feat(ROCm): Allow selecting ROCm version when building llama.cpp backend

.github/workflows/backend.yml

@@ -21,6 +21,8 @@ jobs:
       build-type: ${{ matrix.build-type }}
       cuda-major-version: ${{ matrix.cuda-major-version }}
       cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      rocm-major-version: ${{ matrix.rocm-major-version }}
+      rocm-minor-version: ${{ matrix.rocm-minor-version }}
       platforms: ${{ matrix.platforms }}
       runs-on: ${{ matrix.runs-on }}
       base-image: ${{ matrix.base-image }}
@@ -634,7 +636,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-rerankers'
+            tag-suffix: '-gpu-amd-rocm-6-rerankers'
             runs-on: 'ubuntu-latest'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             skip-drivers: 'false'
@@ -643,13 +645,26 @@ jobs:
             context: "./backend"
             ubuntu-version: '2204'
           - build-type: 'hipblas'
-            cuda-major-version: ""
-            cuda-minor-version: ""
+            rocm-major-version: "6"
+            rocm-minor-version: "4.3"
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-llama-cpp'
+            tag-suffix: '-gpu-amd-rocm-6-llama-cpp'
             runs-on: 'ubuntu-latest'
-            base-image: "rocm/dev-ubuntu-22.04:6.4.3"
+            base-image: "ubuntu:22.04"
+            skip-drivers: 'false'
+            backend: "llama-cpp"
+            dockerfile: "./backend/Dockerfile.llama-cpp"
+            context: "./"
+            ubuntu-version: '2204'
+          - build-type: 'hipblas'
+            rocm-major-version: "7"
+            rocm-minor-version: "1.1"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-amd-rocm-7-llama-cpp'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:22.04"
             skip-drivers: 'false'
             backend: "llama-cpp"
             dockerfile: "./backend/Dockerfile.llama-cpp"
@@ -660,7 +675,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-vllm'
+            tag-suffix: '-gpu-amd-rocm-6-vllm'
             runs-on: 'arc-runner-set'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             skip-drivers: 'false'
@@ -673,7 +688,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-transformers'
+            tag-suffix: '-gpu-amd-rocm-6-transformers'
             runs-on: 'arc-runner-set'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             skip-drivers: 'false'
@@ -686,7 +701,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-diffusers'
+            tag-suffix: '-gpu-amd-rocm-6-diffusers'
             runs-on: 'arc-runner-set'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             skip-drivers: 'false'
@@ -700,7 +715,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-kokoro'
+            tag-suffix: '-gpu-amd-rocm-6-kokoro'
             runs-on: 'arc-runner-set'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             skip-drivers: 'false'
@@ -713,7 +728,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-vibevoice'
+            tag-suffix: '-gpu-amd-rocm-6-vibevoice'
             runs-on: 'arc-runner-set'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             skip-drivers: 'false'
@@ -726,7 +741,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-faster-whisper'
+            tag-suffix: '-gpu-amd-rocm-6-faster-whisper'
             runs-on: 'ubuntu-latest'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             skip-drivers: 'false'
@@ -739,7 +754,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-coqui'
+            tag-suffix: '-gpu-amd-rocm-6-coqui'
             runs-on: 'ubuntu-latest'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             skip-drivers: 'false'
@@ -752,7 +767,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-bark'
+            tag-suffix: '-gpu-amd-rocm-6-bark'
             runs-on: 'arc-runner-set'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             skip-drivers: 'false'
@@ -1161,7 +1176,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-whisper'
+            tag-suffix: '-gpu-amd-rocm-6-whisper'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             runs-on: 'ubuntu-latest'
             skip-drivers: 'false'
@@ -1310,7 +1325,7 @@ jobs:
             platforms: 'linux/amd64'
             skip-drivers: 'true'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-hipblas-exllama2'
+            tag-suffix: '-gpu-amd-rocm-6-exllama2'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             runs-on: 'ubuntu-latest'
             backend: "exllama2"
@@ -1336,7 +1351,7 @@ jobs:
           #   cuda-minor-version: ""
           #   platforms: 'linux/amd64'
           #   tag-latest: 'auto'
-          #   tag-suffix: '-gpu-hipblas-rfdetr'
+          #   tag-suffix: '-gpu-amd-rocm-6-rfdetr'
           #   base-image: "rocm/dev-ubuntu-22.04:6.4.3"
           #   runs-on: 'ubuntu-latest'
           #   skip-drivers: 'false'
@@ -1376,7 +1391,7 @@ jobs:
             cuda-minor-version: ""
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-neutts'
+            tag-suffix: '-gpu-amd-rocm-6-neutts'
             runs-on: 'arc-runner-set'
             base-image: "rocm/dev-ubuntu-22.04:6.4.3"
             skip-drivers: 'false'

.github/workflows/image.yml

@@ -37,14 +37,26 @@ jobs:
       matrix:
         include:
           - build-type: 'hipblas'
+            rocm-major-version: "6"
+            rocm-minor-version: "4.3"
             platforms: 'linux/amd64'
             tag-latest: 'auto'
-            tag-suffix: '-gpu-hipblas'
-            base-image: "rocm/dev-ubuntu-22.04:6.4.3"
-            grpc-base-image: "ubuntu:22.04"
+            tag-suffix: '-gpu-amd-rocm-6'
+            base-image: "ubuntu:22.04"
+            runs-on: 'ubuntu-latest'
+            makeflags: "--jobs=3 --output-sync=target"
+            aio: "-aio-gpu-amd-rocm-6"
+            ubuntu-version: '2204'
+          - build-type: 'hipblas'
+            rocm-major-version: "7"
+            rocm-minor-version: "1.1"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-amd-rocm-7'
+            base-image: "ubuntu:22.04"
             runs-on: 'ubuntu-latest'
             makeflags: "--jobs=3 --output-sync=target"
-            aio: "-aio-gpu-hipblas"
+            aio: "-aio-gpu-amd-rocm-7"
             ubuntu-version: '2204'
 
   core-image-build:

Dockerfile

@@ -19,6 +19,8 @@ FROM requirements AS requirements-drivers
 ARG BUILD_TYPE
 ARG CUDA_MAJOR_VERSION=12
 ARG CUDA_MINOR_VERSION=0
+ARG ROCM_MAJOR_VERSION=6
+ARG ROCM_MINOR_VERSION=4.3 # ROCm version to append to the major version, in the format of their apt repo (https://repo.radeon.com/rocm/apt/). Like `0_alpha` or `3.4`.
 ARG SKIP_DRIVERS=false
 ARG TARGETARCH
 ARG TARGETVARIANT
@@ -111,13 +113,29 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
     ; fi
 
 RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
+        # Setup for specific ROCm version as described here: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/install-methods/package-manager/package-manager-ubuntu.html
+        ROCM_VERSION="${ROCM_MAJOR_VERSION}.${ROCM_MINOR_VERSION}" && \
         apt-get update && \
         apt-get install -y --no-install-recommends \
-            hipblas-dev \
-            rocblas-dev && \
+            gpg wget && \
+        mkdir --parents --mode=0755 /etc/apt/keyrings && \
+        wget -qO - https://repo.radeon.com/rocm/rocm.gpg.key | gpg --yes --dearmor --output /etc/apt/keyrings/rocm.gpg && \
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_VERSION} jammy main" >> /etc/apt/sources.list.d/rocm.list && \
+        if [ "${ROCM_MAJOR_VERSION}" -ge 7 ]; then \
+            echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/graphics/${ROCM_VERSION}/ubuntu jammy main" >> /etc/apt/sources.list.d/rocm.list \
+        ; fi && \
+        echo "Package: *" >> /etc/apt/preferences.d/rocm-pin-600 && \
+        echo "Pin: release o=repo.radeon.com" >> /etc/apt/preferences.d/rocm-pin-600 && \
+        echo "Pin-Priority: 600" >> /etc/apt/preferences.d/rocm-pin-600 && \
+        # End setup steps for specific ROCm version - the packages below will be installed from the configured repositories
+        apt-get update && \
+        apt-get install -y --no-install-recommends \
+            rocm-hip-runtime \
+            rocblas-dev \
+            hipblas-dev && \
         apt-get clean && \
         rm -rf /var/lib/apt/lists/* && \
-        echo "amd" > /run/localai/capability && \
+        echo "amd-rocm-${ROCM_MAJOR_VERSION}" > /run/localai/capability && \
         # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able
         # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency
         ldconfig \

README.md

@@ -43,7 +43,7 @@
 
 > :bulb: Get help - [❓FAQ](https://localai.io/faq/) [💭Discussions](https://github.com/go-skynet/LocalAI/discussions) [:speech_balloon: Discord](https://discord.gg/uJAeKSAGDy) [:book: Documentation website](https://localai.io/)
 >
-> [💻 Quickstart](https://localai.io/basics/getting_started/) [🖼️ Models](https://models.localai.io/) [🚀 Roadmap](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap) [🛫 Examples](https://github.com/mudler/LocalAI-examples) Try on 
+> [💻 Quickstart](https://localai.io/basics/getting_started/) [🖼️ Models](https://models.localai.io/) [🚀 Roadmap](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap) [🛫 Examples](https://github.com/mudler/LocalAI-examples) Try on
 [![Telegram](https://img.shields.io/badge/Telegram-2CA5E0?style=for-the-badge&logo=telegram&logoColor=white)](https://t.me/localaiofficial_bot)
 
 [![tests](https://github.com/go-skynet/LocalAI/actions/workflows/test.yml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/test.yml)[![Build and Release](https://github.com/go-skynet/LocalAI/actions/workflows/release.yaml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/release.yaml)[![build container images](https://github.com/go-skynet/LocalAI/actions/workflows/image.yml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/image.yml)[![Bump dependencies](https://github.com/go-skynet/LocalAI/actions/workflows/bump_deps.yaml/badge.svg)](https://github.com/go-skynet/LocalAI/actions/workflows/bump_deps.yaml)[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/localai)](https://artifacthub.io/packages/search?repo=localai)
@@ -131,10 +131,10 @@ For more installation options, see [Installer Options](https://localai.io/instal
 Or run with docker:
 
 > **💡 Docker Run vs Docker Start**
-> 
+>
 > - `docker run` creates and starts a new container. If a container with the same name already exists, this command will fail.
 > - `docker start` starts an existing container that was previously created with `docker run`.
-> 
+>
 > If you've already run LocalAI before and want to start it again, use: `docker start -i local-ai`
 
 ### CPU only image:
@@ -159,7 +159,7 @@ docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-nv
 ### AMD GPU Images (ROCm):
 
 ```bash
-docker run -ti --name local-ai -p 8080:8080 --device=/dev/kfd --device=/dev/dri --group-add=video localai/localai:latest-gpu-hipblas
+docker run -ti --name local-ai -p 8080:8080 --device=/dev/kfd --device=/dev/dri --group-add=video localai/localai:latest-gpu-amd-rocm-6
 ```
 
 ### Intel GPU Images (oneAPI):
@@ -190,7 +190,7 @@ docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-ai
 docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-aio-gpu-intel
 
 # AMD GPU version
-docker run -ti --name local-ai -p 8080:8080 --device=/dev/kfd --device=/dev/dri --group-add=video localai/localai:latest-aio-gpu-hipblas
+docker run -ti --name local-ai -p 8080:8080 --device=/dev/kfd --device=/dev/dri --group-add=video localai/localai:latest-aio-gpu-amd-rocm-6
 ```
 
 For more information about the AIO images and pre-downloaded models, see [Container Documentation](https://localai.io/basics/container/).
@@ -250,7 +250,7 @@ Roadmap items: [List of issues](https://github.com/mudler/LocalAI/issues?q=is%3A
 - 🗣 [Text to Audio](https://localai.io/features/text-to-audio/)
 - 🔈 [Audio to Text](https://localai.io/features/audio-to-text/) (Audio transcription with `whisper.cpp`)
 - 🎨 [Image generation](https://localai.io/features/image-generation)
-- 🔥 [OpenAI-alike tools API](https://localai.io/features/openai-functions/) 
+- 🔥 [OpenAI-alike tools API](https://localai.io/features/openai-functions/)
 - 🧠 [Embeddings generation for vector databases](https://localai.io/features/embeddings/)
 - ✍️ [Constrained grammars](https://localai.io/features/constrained_grammars/)
 - 🖼️ [Download Models directly from Huggingface ](https://localai.io/models/)
@@ -356,7 +356,7 @@ Other:
 - Github bot which answer on issues, with code and documentation as context https://github.com/JackBekket/GitHelper
 - Github Actions: https://github.com/marketplace/actions/start-localai
 - Examples: https://github.com/mudler/LocalAI/tree/master/examples/
-  
+
 
 ### 🔗 Resources
 

backend/Dockerfile.llama-cpp

@@ -55,9 +55,13 @@ ARG BUILD_TYPE
 ENV BUILD_TYPE=${BUILD_TYPE}
 ARG CUDA_MAJOR_VERSION
 ARG CUDA_MINOR_VERSION
+ARG ROCM_MAJOR_VERSION
+ARG ROCM_MINOR_VERSION
 ARG SKIP_DRIVERS=false
 ENV CUDA_MAJOR_VERSION=${CUDA_MAJOR_VERSION}
 ENV CUDA_MINOR_VERSION=${CUDA_MINOR_VERSION}
+ENV ROCM_MAJOR_VERSION=${ROCM_MAJOR_VERSION}
+ENV ROCM_MINOR_VERSION=${ROCM_MINOR_VERSION}
 ENV DEBIAN_FRONTEND=noninteractive
 ARG TARGETARCH
 ARG TARGETVARIANT
@@ -158,10 +162,27 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
     ; fi
 
 RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
+        # Setup for specific ROCm version as described here: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/install-methods/package-manager/package-manager-ubuntu.html
+        ROCM_VERSION="${ROCM_MAJOR_VERSION}.${ROCM_MINOR_VERSION}" && \
         apt-get update && \
         apt-get install -y --no-install-recommends \
-            hipblas-dev \
-            rocblas-dev && \
+            gpg wget && \
+        mkdir --parents --mode=0755 /etc/apt/keyrings && \
+        wget -qO - https://repo.radeon.com/rocm/rocm.gpg.key | gpg --yes --dearmor --output /etc/apt/keyrings/rocm.gpg && \
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_VERSION} jammy main" >> /etc/apt/sources.list.d/rocm.list && \
+        if [ "${ROCM_MAJOR_VERSION}" -ge 7 ]; then \
+            echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/graphics/${ROCM_VERSION}/ubuntu jammy main" >> /etc/apt/sources.list.d/rocm.list \
+        ; fi && \
+        echo "Package: *" >> /etc/apt/preferences.d/rocm-pin-600 && \
+        echo "Pin: release o=repo.radeon.com" >> /etc/apt/preferences.d/rocm-pin-600 && \
+        echo "Pin-Priority: 600" >> /etc/apt/preferences.d/rocm-pin-600 && \
+        # End setup steps for specific ROCm version
+        apt-get update && \
+        apt-get install -y --no-install-recommends \
+            # Build dependencies
+            rocm-developer-tools \
+            rocm-hip-runtime-dev \
+            rocm-hip-sdk && \
         apt-get clean && \
         rm -rf /var/lib/apt/lists/* && \
         # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able