+ + 📈 **llama.cpp server** for _${{ github.job }}_ on _${{ env.RUNNER_LABEL }}_ for `${{ matrix.model }}`-`${{ matrix.ftype }}`: **${{ env.BENCH_ITERATIONS}} iterations** 🚀 + +
+ +
+
+
diff --git a/llama.cpp/.github/workflows/build-amd.yml b/llama.cpp/.github/workflows/build-amd.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b6fe8de8650a1281626067a0e52365fb5c75653d
--- /dev/null
+++ b/llama.cpp/.github/workflows/build-amd.yml
@@ -0,0 +1,52 @@
+name: CI (AMD)
+
+on:
+ workflow_dispatch: # allows manual triggering
+ push:
+ branches:
+ - master
+ paths: [
+ '.github/workflows/build-amd.yml',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ '**/*.h',
+ '**/*.hpp',
+ '**/*.c',
+ '**/*.cpp',
+ '**/*.cu',
+ '**/*.cuh',
+ '**/*.comp'
+ ]
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ ggml-ci-x64-amd-vulkan:
+ runs-on: [self-hosted, Linux, X64, AMD]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ vulkaninfo --summary
+ GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
+
+ ggml-ci-x64-amd-rocm:
+ runs-on: [self-hosted, Linux, X64, AMD]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ amd-smi static
+ GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
diff --git a/llama.cpp/.github/workflows/build-cmake-pkg.yml b/llama.cpp/.github/workflows/build-cmake-pkg.yml
new file mode 100644
index 0000000000000000000000000000000000000000..fee2ab96bd0e8ba7bae04a9c859ec7ec5ad7e4d2
--- /dev/null
+++ b/llama.cpp/.github/workflows/build-cmake-pkg.yml
@@ -0,0 +1,51 @@
+name: Build relocatable cmake package
+on:
+ workflow_dispatch:
+ workflow_call:
+
+jobs:
+ linux:
+ runs-on: ubuntu-24.04
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Install dependencies
+ run: |
+ sudo apt update
+ sudo apt install -y build-essential tcl
+
+ - name: Build
+ run: |
+ PREFIX="$(pwd)"/inst
+ cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX" \
+ -DLLAMA_CURL=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=OFF \
+ -DLLAMA_BUILD_EXAMPLES=OFF -DCMAKE_BUILD_TYPE=Release
+ cmake --build build --config Release
+ cmake --install build --prefix "$PREFIX" --config Release
+
+ export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
+ tclsh <<'EOF'
+ set build(commit) [string trim [exec git rev-parse --short HEAD]]
+ set build(number) [string trim [exec git rev-list --count HEAD]]
+ set build(version) "0.0.$build(number)"
+
+ set llamaconfig [read [open "$env(LLAMA_CONFIG)" r]]
+ set checks [list "set\\(LLAMA_VERSION \\s+$build(version)\\)" \
+ "set\\(LLAMA_BUILD_COMMIT\\s+$build(commit)\\)" \
+ "set\\(LLAMA_BUILD_NUMBER\\s+$build(number)\\)"]
+
+ puts -nonewline "Checking llama-config.cmake version... "
+ foreach check $checks {
+ if {![regexp -expanded -- $check $llamaconfig]} {
+ puts "\"$check\" failed!"
+ exit 1
+ }
+ }
+ puts "success."
+ EOF
+
+ cd examples/simple-cmake-pkg
+ cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
+ cmake --build build
diff --git a/llama.cpp/.github/workflows/build-linux-cross.yml b/llama.cpp/.github/workflows/build-linux-cross.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2b101876c5abf22bae1eec8680d9eb9c60172995
--- /dev/null
+++ b/llama.cpp/.github/workflows/build-linux-cross.yml
@@ -0,0 +1,299 @@
+name: Build on Linux using cross-compiler
+on:
+ workflow_dispatch:
+ workflow_call:
+
+jobs:
+ ubuntu-24-riscv64-cpu-cross:
+ runs-on: ubuntu-24.04
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup Riscv
+ run: |
+ sudo dpkg --add-architecture riscv64
+
+ # Add arch-specific repositories for non-amd64 architectures
+ cat << EOF | sudo tee /etc/apt/sources.list.d/riscv64-ports.list
+ deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe
+ deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe
+ deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe
+ deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe
+ EOF
+
+ sudo apt-get update || true ;# Prevent failure due to missing URLs.
+
+ sudo apt-get install -y --no-install-recommends \
+ build-essential \
+ gcc-14-riscv64-linux-gnu \
+ g++-14-riscv64-linux-gnu
+
+ - name: Build
+ run: |
+ cmake -B build -DLLAMA_CURL=OFF \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DGGML_OPENMP=OFF \
+ -DLLAMA_BUILD_EXAMPLES=ON \
+ -DLLAMA_BUILD_TOOLS=ON \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DCMAKE_SYSTEM_NAME=Linux \
+ -DCMAKE_SYSTEM_PROCESSOR=riscv64 \
+ -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
+ -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+ -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
+ -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+ -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
+ -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
+
+ cmake --build build --config Release -j $(nproc)
+
+ # ubuntu-24-riscv64-vulkan-cross:
+ # runs-on: ubuntu-24.04
+
+ # steps:
+ # - uses: actions/checkout@v4
+ # - name: Setup Riscv
+ # run: |
+ # sudo dpkg --add-architecture riscv64
+
+ # # Add arch-specific repositories for non-amd64 architectures
+ # cat << EOF | sudo tee /etc/apt/sources.list.d/riscv64-ports.list
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe
+ # EOF
+
+ # sudo apt-get update || true ;# Prevent failure due to missing URLs.
+
+ # sudo apt-get install -y --no-install-recommends \
+ # build-essential \
+ # glslc \
+ # gcc-14-riscv64-linux-gnu \
+ # g++-14-riscv64-linux-gnu \
+ # libvulkan-dev:riscv64
+
+ # - name: Build
+ # run: |
+ # cmake -B build -DLLAMA_CURL=OFF \
+ # -DCMAKE_BUILD_TYPE=Release \
+ # -DGGML_VULKAN=ON \
+ # -DGGML_OPENMP=OFF \
+ # -DLLAMA_BUILD_EXAMPLES=ON \
+ # -DLLAMA_BUILD_TOOLS=ON \
+ # -DLLAMA_BUILD_TESTS=OFF \
+ # -DCMAKE_SYSTEM_NAME=Linux \
+ # -DCMAKE_SYSTEM_PROCESSOR=riscv64 \
+ # -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
+ # -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
+ # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+ # -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
+ # -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+ # -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
+ # -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
+
+ # cmake --build build --config Release -j $(nproc)
+
+ # ubuntu-24-arm64-vulkan-cross:
+ # runs-on: ubuntu-24.04
+
+ # steps:
+ # - uses: actions/checkout@v4
+ # - name: Setup Arm64
+ # run: |
+ # sudo dpkg --add-architecture arm64
+
+ # # Add arch-specific repositories for non-amd64 architectures
+ # cat << EOF | sudo tee /etc/apt/sources.list.d/arm64-ports.list
+ # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe
+ # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe
+ # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe
+ # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe
+ # EOF
+
+ # sudo apt-get update || true ;# Prevent failure due to missing URLs.
+
+ # sudo apt-get install -y --no-install-recommends \
+ # build-essential \
+ # glslc \
+ # crossbuild-essential-arm64 \
+ # libvulkan-dev:arm64
+
+ # - name: Build
+ # run: |
+ # cmake -B build -DLLAMA_CURL=OFF \
+ # -DCMAKE_BUILD_TYPE=Release \
+ # -DGGML_VULKAN=ON \
+ # -DGGML_OPENMP=OFF \
+ # -DLLAMA_BUILD_EXAMPLES=ON \
+ # -DLLAMA_BUILD_TOOLS=ON \
+ # -DLLAMA_BUILD_TESTS=OFF \
+ # -DCMAKE_SYSTEM_NAME=Linux \
+ # -DCMAKE_SYSTEM_PROCESSOR=aarch64 \
+ # -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \
+ # -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ \
+ # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+ # -DCMAKE_FIND_ROOT_PATH=/usr/lib/aarch64-linux-gnu \
+ # -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+ # -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
+ # -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
+
+ # cmake --build build --config Release -j $(nproc)
+
+ debian-13-loongarch64-cpu-cross:
+ runs-on: ubuntu-24.04
+ container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup LoongArch
+ run: |
+ rm -f /etc/apt/sources.list.d/*
+ cat << EOF | tee /etc/apt/sources.list.d/debian-ports.list
+ deb http://snapshot.debian.org/archive/debian/20250515T202920Z/ trixie main
+ EOF
+ ( echo 'quiet "true";'; \
+ echo 'APT::Get::Assume-Yes "true";'; \
+ echo 'APT::Install-Recommends "false";'; \
+ echo 'Acquire::Check-Valid-Until "false";'; \
+ echo 'Acquire::Retries "5";'; \
+ ) > /etc/apt/apt.conf.d/99snapshot-repos
+
+ apt-get update
+ apt-get install -y ca-certificates debian-ports-archive-keyring cmake git zip
+ dpkg --add-architecture loong64
+
+ # Add arch-specific repositories for non-amd64 architectures
+ cat << EOF | tee /etc/apt/sources.list.d/loong64-ports.list
+ deb [arch=loong64] http://snapshot.debian.org/archive/debian-ports/20250515T194251Z/ sid main
+ EOF
+
+ apt-get update || true ;# Prevent failure due to missing URLs.
+
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ gcc-14-loongarch64-linux-gnu \
+ g++-14-loongarch64-linux-gnu
+
+ - name: Build
+ run: |
+ cmake -B build -DLLAMA_CURL=OFF \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DGGML_OPENMP=OFF \
+ -DLLAMA_BUILD_EXAMPLES=ON \
+ -DLLAMA_BUILD_TOOLS=ON \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DCMAKE_SYSTEM_NAME=Linux \
+ -DCMAKE_SYSTEM_PROCESSOR=loongarch64 \
+ -DCMAKE_C_COMPILER=loongarch64-linux-gnu-gcc-14 \
+ -DCMAKE_CXX_COMPILER=loongarch64-linux-gnu-g++-14 \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+ -DCMAKE_FIND_ROOT_PATH=/usr/lib/loongarch64-linux-gnu \
+ -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+ -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
+ -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
+
+ cmake --build build --config Release -j $(nproc)
+
+ debian-13-loongarch64-vulkan-cross:
+ runs-on: ubuntu-24.04
+ container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup LoongArch
+ run: |
+ rm -f /etc/apt/sources.list.d/*
+ cat << EOF | tee /etc/apt/sources.list.d/debian-ports.list
+ deb http://snapshot.debian.org/archive/debian/20250515T202920Z/ trixie main
+ EOF
+ ( echo 'quiet "true";'; \
+ echo 'APT::Get::Assume-Yes "true";'; \
+ echo 'APT::Install-Recommends "false";'; \
+ echo 'Acquire::Check-Valid-Until "false";'; \
+ echo 'Acquire::Retries "5";'; \
+ ) > /etc/apt/apt.conf.d/99snapshot-repos
+
+ apt-get update
+ apt-get install -y ca-certificates debian-ports-archive-keyring cmake git zip
+ dpkg --add-architecture loong64
+
+ # Add arch-specific repositories for non-amd64 architectures
+ cat << EOF | tee /etc/apt/sources.list.d/loong64-ports.list
+ deb [arch=loong64] http://snapshot.debian.org/archive/debian-ports/20250515T194251Z/ sid main
+ EOF
+
+ apt-get update || true ;# Prevent failure due to missing URLs.
+
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ glslc \
+ gcc-14-loongarch64-linux-gnu \
+ g++-14-loongarch64-linux-gnu \
+ libvulkan-dev:loong64
+
+ - name: Build
+ run: |
+ cmake -B build -DLLAMA_CURL=OFF \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DGGML_VULKAN=ON \
+ -DGGML_OPENMP=OFF \
+ -DLLAMA_BUILD_EXAMPLES=ON \
+ -DLLAMA_BUILD_TOOLS=ON \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DCMAKE_SYSTEM_NAME=Linux \
+ -DCMAKE_SYSTEM_PROCESSOR=loongarch64 \
+ -DCMAKE_C_COMPILER=loongarch64-linux-gnu-gcc-14 \
+ -DCMAKE_CXX_COMPILER=loongarch64-linux-gnu-g++-14 \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+ -DCMAKE_FIND_ROOT_PATH=/usr/lib/loongarch64-linux-gnu \
+ -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+ -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
+ -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
+
+ cmake --build build --config Release -j $(nproc)
+
+ ubuntu-24-riscv64-cpu-spacemit-ime-cross:
+ runs-on: ubuntu-24.04
+
+ env:
+ SPACEMIT_IME_TOOLCHAIN_VERSION: "1.1.2"
+ SPACEMIT_IME_TOOLCHAIN_PATH: "spacemit-toolchain-linux-glibc-x86_64"
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Cache Toolchain
+ uses: actions/cache@v4
+ id: cache-spacemit-ime-cross-toolchain
+ with:
+ path: ./${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}
+ key: ${{ runner.os }}-spacemit-ime-toolchain-v${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}
+
+ - name: Setup Toolchain
+ if: steps.cache-spacemit-ime-cross-toolchain.outputs.cache-hit != 'true'
+ run: |
+ wget --quiet --no-check-certificate https://archive.spacemit.com/toolchain/spacemit-toolchain-linux-glibc-x86_64-v${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}.tar.xz -O ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}.tar.xz
+ rm -rf ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}
+ mkdir -p ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}
+ tar xf ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}.tar.xz -C ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }} --strip-components=1
+ rm -rf ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}.tar.xz
+
+ - name: Build
+ run: |
+ export RISCV_ROOT_PATH=${PWD}/${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}
+ cmake -B build -DLLAMA_CURL=OFF \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DGGML_OPENMP=OFF \
+ -DLLAMA_BUILD_EXAMPLES=ON \
+ -DLLAMA_BUILD_TOOLS=ON \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DGGML_CPU_RISCV64_SPACEMIT=ON \
+ -DGGML_RVV=ON \
+ -DGGML_RV_ZFH=ON \
+ -DGGML_RV_ZICBOP=ON \
+ -DRISCV64_SPACEMIT_IME_SPEC=RISCV64_SPACEMIT_IME1 \
+ -DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake
+
+ cmake --build build --config Release -j $(nproc)
diff --git a/llama.cpp/.github/workflows/build-riscv-native.yml b/llama.cpp/.github/workflows/build-riscv-native.yml
new file mode 100644
index 0000000000000000000000000000000000000000..a3a0b0d6638cafd69d0798b81f75956b4dd9d3a7
--- /dev/null
+++ b/llama.cpp/.github/workflows/build-riscv-native.yml
@@ -0,0 +1,120 @@
+name: Build on RISCV Linux Machine by Cloud-V
+on:
+ pull_request:
+ workflow_dispatch:
+ workflow_call:
+
+jobs:
+ debian-13-riscv64-native: # Bianbu 2.2
+ runs-on: [self-hosted, RISCV64]
+
+ steps:
+ - name: Install prerequisites
+ run: |
+ sudo apt-get update || true
+ sudo apt-get install -y libatomic1
+ - uses: actions/checkout@v4
+ - name: Setup Riscv
+ run: |
+ sudo apt-get update || true
+ sudo apt-get install -y --no-install-recommends \
+ build-essential \
+ gcc-14-riscv64-linux-gnu \
+ g++-14-riscv64-linux-gnu \
+ ccache \
+ cmake
+
+ - name: Setup ccache
+ run: |
+ mkdir -p $HOME/.ccache
+ ccache -M 5G -d $HOME/.ccache
+ export CCACHE_LOGFILE=/home/runneruser/ccache_debug/ccache.log
+ export CCACHE_DEBUGDIR="/home/runneruser/ccache_debug"
+ echo "$GITHUB_WORKSPACE"
+ echo "CCACHE_LOGFILE=$CCACHE_LOGFILE" >> $GITHUB_ENV
+ echo "CCACHE_DEBUGDIR=$CCACHE_DEBUGDIR" >> $GITHUB_ENV
+ echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV
+ echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
+
+ - name: Build
+ run: |
+ cmake -B build \
+ -DLLAMA_CURL=OFF \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DGGML_OPENMP=OFF \
+ -DLLAMA_BUILD_EXAMPLES=ON \
+ -DLLAMA_BUILD_TOOLS=ON \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DCMAKE_SYSTEM_NAME=Linux \
+ -DCMAKE_SYSTEM_PROCESSOR=riscv64 \
+ -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
+ -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+ -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
+ -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+ -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
+ -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
+
+ cmake --build build --config Release -j $(nproc)
+
+ # debian-13-riscv64-spacemit-ime-native: # Bianbu 2.2
+ # runs-on: [self-hosted, RISCV64]
+
+ # steps:
+ # - name: Install prerequisites
+ # run: |
+ # sudo apt-get update || true
+ # sudo apt-get install -y libatomic1
+ # - uses: actions/checkout@v4
+ # - name: Setup Riscv
+ # run: |
+ # sudo apt-get update || true
+ # sudo apt-get install -y --no-install-recommends \
+ # build-essential \
+ # gcc-14-riscv64-linux-gnu \
+ # g++-14-riscv64-linux-gnu \
+ # ccache \
+ # cmake
+ # sudo apt-get upgrade binutils -y
+
+ # - name: Setup ccache
+ # run: |
+ # mkdir -p $HOME/.ccache
+ # ccache -M 5G -d $HOME/.ccache
+ # export CCACHE_LOGFILE=/home/runneruser/ccache_debug/ccache.log
+ # export CCACHE_DEBUGDIR="/home/runneruser/ccache_debug"
+ # echo "$GITHUB_WORKSPACE"
+ # echo "CCACHE_LOGFILE=$CCACHE_LOGFILE" >> $GITHUB_ENV
+ # echo "CCACHE_DEBUGDIR=$CCACHE_DEBUGDIR" >> $GITHUB_ENV
+ # echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV
+ # echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
+
+ # - name: Build
+ # run: |
+ # cmake -B build \
+ # -DLLAMA_CURL=OFF \
+ # -DCMAKE_BUILD_TYPE=Release \
+ # -DGGML_OPENMP=OFF \
+ # -DLLAMA_BUILD_EXAMPLES=ON \
+ # -DLLAMA_BUILD_TOOLS=ON \
+ # -DLLAMA_BUILD_TESTS=OFF \
+ # -DCMAKE_SYSTEM_NAME=Linux \
+ # -DCMAKE_SYSTEM_PROCESSOR=riscv64 \
+ # -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
+ # -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
+ # -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+ # -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+ # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+ # -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
+ # -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+ # -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
+ # -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH \
+ # -DGGML_RVV=ON \
+ # -DGGML_RV_ZFH=ON \
+ # -DGGML_RV_ZICBOP=ON \
+ # -DGGML_CPU_RISCV64_SPACEMIT=ON \
+ # -DRISCV64_SPACEMIT_IME_SPEC=RISCV64_SPACEMIT_IME1
+
+ # cmake --build build --config Release -j $(nproc)
diff --git a/llama.cpp/.github/workflows/build.yml b/llama.cpp/.github/workflows/build.yml
new file mode 100644
index 0000000000000000000000000000000000000000..db885907cdaed6007d9ec3f6557837463a1429b7
--- /dev/null
+++ b/llama.cpp/.github/workflows/build.yml
@@ -0,0 +1,1514 @@
+name: CI
+
+on:
+ workflow_dispatch: # allows manual triggering
+ push:
+ branches:
+ - master
+ paths: [
+ '.github/workflows/build.yml',
+ '.github/workflows/build-linux-cross.yml',
+ '.github/workflows/build-cmake-pkg.yml',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ '**/*.h',
+ '**/*.hpp',
+ '**/*.c',
+ '**/*.cpp',
+ '**/*.cu',
+ '**/*.cuh',
+ '**/*.swift',
+ '**/*.m',
+ '**/*.metal',
+ '**/*.comp'
+ ]
+
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: [
+ '.github/workflows/build.yml',
+ '.github/workflows/build-linux-cross.yml',
+ '.github/workflows/build-cmake-pkg.yml',
+ '**/CMakeLists.txt',
+ '**/.cmake',
+ '**/*.h',
+ '**/*.hpp',
+ '**/*.c',
+ '**/*.cpp',
+ '**/*.cu',
+ '**/*.cuh',
+ '**/*.swift',
+ '**/*.m',
+ '**/*.metal',
+ '**/*.comp'
+ ]
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+env:
+ GGML_NLOOP: 3
+ GGML_N_THREADS: 1
+ LLAMA_LOG_COLORS: 1
+ LLAMA_LOG_PREFIX: 1
+ LLAMA_LOG_TIMESTAMPS: 1
+
+jobs:
+ macOS-latest-cmake-arm64:
+ runs-on: macos-latest
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: macOS-latest-cmake-arm64
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ continue-on-error: true
+ run: |
+ brew update
+ brew install curl
+
+ - name: Build
+ id: cmake_build
+ run: |
+ sysctl -a
+ cmake -B build \
+ -DCMAKE_BUILD_RPATH="@loader_path" \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DGGML_METAL_USE_BF16=ON \
+ -DGGML_METAL_EMBED_LIBRARY=OFF \
+ -DGGML_METAL_SHADER_DEBUG=ON \
+ -DGGML_RPC=ON
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
+ leaks -atExit -- ./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1
+
+ - name: Test
+ id: cmake_test
+ run: |
+ cd build
+ ctest -L 'main|curl' --verbose --timeout 900
+
+ macOS-latest-cmake-x64:
+ runs-on: macos-15-intel
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: macOS-latest-cmake-x64
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ continue-on-error: true
+ run: |
+ brew update
+ brew install curl
+
+ - name: Build
+ id: cmake_build
+ run: |
+ sysctl -a
+ # Metal is disabled due to intermittent failures with Github runners not having a GPU:
+ # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
+ cmake -B build \
+ -DCMAKE_BUILD_RPATH="@loader_path" \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DGGML_METAL=OFF \
+ -DGGML_RPC=ON \
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
+
+ - name: Test
+ id: cmake_test
+ run: |
+ cd build
+ ctest -L main --verbose --timeout 900
+
+ macOS-latest-cmake-arm64-webgpu:
+ runs-on: macos-latest
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: macOS-latest-cmake-arm64-webgpu
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ continue-on-error: true
+ run: |
+ brew update
+ brew install curl
+
+ - name: Dawn Dependency
+ id: dawn-depends
+ run: |
+ DAWN_VERSION="v1.0.0"
+ DAWN_OWNER="reeselevine"
+ DAWN_REPO="dawn"
+ DAWN_ASSET_NAME="Dawn-a1a6b45cced25a3b7f4fb491e0ae70796cc7f22b-macos-latest-Release.tar.gz"
+ echo "Fetching release asset from https://github.com/${DAWN_OWNER}/${DAWN_REPO}/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}"
+ curl -L -o artifact.tar.gz \
+ "https://github.com/${DAWN_OWNER}/${DAWN_REPO}/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}"
+ mkdir dawn
+ tar -xvf artifact.tar.gz -C dawn --strip-components=1
+
+ - name: Build
+ id: cmake_build
+ run: |
+ export CMAKE_PREFIX_PATH=dawn
+ cmake -B build -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
+
+ - name: Test
+ id: cmake_test
+ run: |
+ cd build
+ ctest -L main --verbose --timeout 900
+
+ ubuntu-cpu-cmake:
+ strategy:
+ matrix:
+ include:
+ - build: 'x64'
+ os: ubuntu-22.04
+ - build: 'arm64'
+ os: ubuntu-22.04-arm
+ - build: 's390x'
+ os: ubuntu-24.04-s390x
+ - build: 'ppc64le'
+ os: ubuntu-24.04-ppc64le
+
+ runs-on: ${{ matrix.os }}
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-cpu-cmake-${{ matrix.build }}
+ evict-old-files: 1d
+
+ - name: Build Dependencies
+ id: build_depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends \
+ python3 python3-pip python3-dev \
+ libjpeg-dev build-essential libcurl4-openssl-dev \
+ git-lfs
+
+ - name: Python Dependencies
+ id: python_depends
+ run: |
+ python3 -m pip install --upgrade pip
+ pip3 install ./gguf-py
+
+ - name: Swap Endianness
+ id: endianness
+ if: ${{ matrix.build == 's390x' }}
+ run: |
+ for f in models/*.gguf; do
+ echo YES | python3 gguf-py/gguf/scripts/gguf_convert_endian.py $f big
+ done
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DGGML_RPC=ON
+ cmake --build build --config Release -j $(nproc)
+
+ - name: Test
+ id: cmake_test
+ run: |
+ cd build
+ ctest -L 'main|curl' --verbose --timeout 900
+
+ - name: Test llama2c conversion
+ id: llama2c_test
+ if: ${{ matrix.build != 's390x' }}
+ run: |
+ cd build
+ echo "Fetch tokenizer"
+ wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin
+ echo "Fetch llama2c model"
+ wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
+ ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
+ ./bin/llama-cli -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
+
+ - name: Test llama2c (s390x)
+ id: llama2c_test_s390x
+ if: ${{ matrix.build == 's390x' }}
+ run: |
+ cd build
+ echo "Fetch llama2c big-endian model"
+ wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf
+ ./bin/llama-cli -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
+
+ ubuntu-latest-cmake-sanitizer:
+ runs-on: ubuntu-latest
+
+ continue-on-error: true
+
+ strategy:
+ matrix:
+ sanitizer: [ADDRESS, THREAD, UNDEFINED]
+ build_type: [Debug]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-latest-cmake-sanitizer-${{ matrix.sanitizer }}
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install build-essential libcurl4-openssl-dev
+
+ - name: Build
+ id: cmake_build
+ if: ${{ matrix.sanitizer != 'THREAD' }}
+ run: |
+ cmake -B build \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
+ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
+ cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
+
+ - name: Build (no OpenMP)
+ id: cmake_build_no_openmp
+ if: ${{ matrix.sanitizer == 'THREAD' }}
+ run: |
+ cmake -B build \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
+ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+ -DGGML_OPENMP=OFF
+ cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
+
+ - name: Test
+ id: cmake_test
+ run: |
+ cd build
+ ctest -L main --verbose --timeout 900
+
+ ubuntu-latest-llguidance:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install build-essential libcurl4-openssl-dev
+
+ - name: Build
+ id: cmake_build
+ run: |
+ mkdir build
+ cd build
+ cmake .. \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DLLAMA_LLGUIDANCE=ON
+ cmake --build . --config Release -j $(nproc)
+
+ - name: Test
+ id: cmake_test
+ run: |
+ cd build
+ ctest -L main --verbose --timeout 900
+
+ ubuntu-latest-cmake-rpc:
+ runs-on: ubuntu-latest
+
+ continue-on-error: true
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ # - name: ccache
+ # uses: ggml-org/ccache-action@v1.2.16
+ # with:
+ # key: ubuntu-latest-cmake-rpc
+ # evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install build-essential libcurl4-openssl-dev
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DGGML_RPC=ON
+ cmake --build build --config Release -j $(nproc)
+
+ - name: Test
+ id: cmake_test
+ run: |
+ cd build
+ ctest -L main --verbose
+
+ ubuntu-24-cmake-vulkan:
+ runs-on: ubuntu-24.04
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-24-cmake-vulkan
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo add-apt-repository -y ppa:kisak/kisak-mesa
+ sudo apt-get update -y
+ sudo apt-get install -y build-essential mesa-vulkan-drivers libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libcurl4-openssl-dev
+
+ - name: Get latest Vulkan SDK version
+ id: vulkan_sdk_version
+ run: |
+ echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
+
+ - name: Cache Vulkan SDK
+ id: cache_vulkan_sdk
+ uses: actions/cache@v4
+ with:
+ path: ./vulkan_sdk
+ key: vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}
+
+ - name: Install Vulkan SDK
+ if: steps.cache_vulkan_sdk.outputs.cache-hit != 'true'
+ id: vulkan_sdk_install
+ run: |
+ mkdir -p vulkan_sdk
+ cd vulkan_sdk
+ curl --no-progress-meter https://sdk.lunarg.com/sdk/download/latest/linux/vulkan_sdk.tar.xz | tar -Jx --strip-components=1
+
+ - name: Build
+ id: cmake_build
+ run: |
+ source ./vulkan_sdk/setup-env.sh
+ cmake -B build \
+ -DGGML_VULKAN=ON
+ cmake --build build --config Release -j $(nproc)
+
+ - name: Test
+ id: cmake_test
+ run: |
+ cd build
+ export GGML_VK_VISIBLE_DEVICES=0
+ export GGML_VK_DISABLE_F16=1
+ # This is using llvmpipe and runs slower than other backends
+ ctest -L main --verbose --timeout 4200
+
+ ubuntu-22-cmake-webgpu:
+ runs-on: ubuntu-22.04
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-22-cmake-webgpu
+ evict-old-files: 1d
+
+ - name: Vulkan SDK Dependencies
+ id: vulkan-depends
+ run: |
+ wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add -
+ sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+ sudo apt-get update -y
+ sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libcurl4-openssl-dev
+
+ - name: Dawn Dependency
+ id: dawn-depends
+ run: |
+ sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev
+ DAWN_VERSION="v1.0.0"
+ DAWN_OWNER="reeselevine"
+ DAWN_REPO="dawn"
+ DAWN_ASSET_NAME="Dawn-a1a6b45cced25a3b7f4fb491e0ae70796cc7f22b-ubuntu-latest-Release.tar.gz"
+ echo "Fetching release asset from https://github.com/${DAWN_OWNER}/${DAWN_REPO}/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}"
+ curl -L -o artifact.tar.gz \
+ "https://github.com/${DAWN_OWNER}/${DAWN_REPO}/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}"
+ mkdir dawn
+ tar -xvf artifact.tar.gz -C dawn --strip-components=1
+
+ - name: Build
+ id: cmake_build
+ run: |
+ export Dawn_DIR=dawn/lib64/cmake/Dawn
+ cmake -B build -DGGML_WEBGPU=ON
+ cmake --build build --config Release -j $(nproc)
+
+ - name: Test
+ id: cmake_test
+ run: |
+ cd build
+ # This is using llvmpipe and runs slower than other backends
+ ctest -L main --verbose --timeout 3600
+
+ ubuntu-22-cmake-hip:
+ runs-on: ubuntu-22.04
+ container: rocm/dev-ubuntu-22.04:6.1.2
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libcurl4-openssl-dev rocwmma-dev
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-22-cmake-hip
+ evict-old-files: 1d
+
+ - name: Build with native CMake HIP support
+ id: cmake_build
+ run: |
+ cmake -B build -S . \
+ -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
+ -DGGML_HIP_ROCWMMA_FATTN=ON \
+ -DGGML_HIP=ON
+ cmake --build build --config Release -j $(nproc)
+
+ ubuntu-22-cmake-musa:
+ runs-on: ubuntu-22.04
+ container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Dependencies
+ id: depends
+ run: |
+ apt-get update
+ apt-get install -y build-essential git cmake libcurl4-openssl-dev
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-22-cmake-musa
+ evict-old-files: 1d
+
+ - name: Build with native CMake MUSA support
+ id: cmake_build
+ run: |
+ cmake -B build -S . \
+ -DGGML_MUSA=ON
+ cmake --build build --config Release -j $(nproc)
+
+ ubuntu-22-cmake-sycl:
+ runs-on: ubuntu-22.04
+
+ continue-on-error: true
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: add oneAPI to apt
+ shell: bash
+ run: |
+ cd /tmp
+ wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+ sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+ rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+ sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+
+ - name: install oneAPI dpcpp compiler
+ shell: bash
+ run: |
+ sudo apt update
+ sudo apt install intel-oneapi-compiler-dpcpp-cpp libcurl4-openssl-dev
+
+ - name: install oneAPI MKL library
+ shell: bash
+ run: |
+ sudo apt install intel-oneapi-mkl-devel
+
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-22-cmake-sycl
+ evict-old-files: 1d
+
+ - name: Build
+ id: cmake_build
+ run: |
+ source /opt/intel/oneapi/setvars.sh
+ cmake -B build \
+ -DGGML_SYCL=ON \
+ -DCMAKE_C_COMPILER=icx \
+ -DCMAKE_CXX_COMPILER=icpx
+ cmake --build build --config Release -j $(nproc)
+
+ ubuntu-22-cmake-sycl-fp16:
+ runs-on: ubuntu-22.04
+
+ continue-on-error: true
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: add oneAPI to apt
+ shell: bash
+ run: |
+ cd /tmp
+ wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+ sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+ rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+ sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+
+ - name: install oneAPI dpcpp compiler
+ shell: bash
+ run: |
+ sudo apt update
+ sudo apt install intel-oneapi-compiler-dpcpp-cpp libcurl4-openssl-dev
+
+ - name: install oneAPI MKL library
+ shell: bash
+ run: |
+ sudo apt install intel-oneapi-mkl-devel
+
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-22-cmake-sycl-fp16
+ evict-old-files: 1d
+
+ - name: Build
+ id: cmake_build
+ run: |
+ source /opt/intel/oneapi/setvars.sh
+ cmake -B build \
+ -DGGML_SYCL=ON \
+ -DCMAKE_C_COMPILER=icx \
+ -DCMAKE_CXX_COMPILER=icpx \
+ -DGGML_SYCL_F16=ON
+ cmake --build build --config Release -j $(nproc)
+
+ build-linux-cross:
+ uses: ./.github/workflows/build-linux-cross.yml
+
+ build-cmake-pkg:
+ uses: ./.github/workflows/build-cmake-pkg.yml
+
+ macOS-latest-cmake-ios:
+ runs-on: macos-latest
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: macOS-latest-cmake-ios
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ continue-on-error: true
+ run: |
+ brew update
+
+ - name: Build
+ id: cmake_build
+ run: |
+ sysctl -a
+ cmake -B build -G Xcode \
+ -DGGML_METAL_USE_BF16=ON \
+ -DGGML_METAL_EMBED_LIBRARY=ON \
+ -DLLAMA_BUILD_COMMON=OFF \
+ -DLLAMA_BUILD_EXAMPLES=OFF \
+ -DLLAMA_BUILD_TOOLS=OFF \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DLLAMA_BUILD_SERVER=OFF \
+ -DCMAKE_SYSTEM_NAME=iOS \
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+ -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+ macOS-latest-cmake-tvos:
+ runs-on: macos-latest
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: macOS-latest-cmake-tvos
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ continue-on-error: true
+ run: |
+ brew update
+
+ - name: Build
+ id: cmake_build
+ run: |
+ sysctl -a
+ cmake -B build -G Xcode \
+ -DGGML_METAL_USE_BF16=ON \
+ -DGGML_METAL_EMBED_LIBRARY=ON \
+ -DLLAMA_BUILD_COMMON=OFF \
+ -DLLAMA_BUILD_EXAMPLES=OFF \
+ -DLLAMA_BUILD_TOOLS=OFF \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DLLAMA_BUILD_SERVER=OFF \
+ -DCMAKE_SYSTEM_NAME=tvOS \
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+ -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+ macOS-latest-cmake-visionos:
+ runs-on: macos-latest
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Dependencies
+ id: depends
+ continue-on-error: true
+ run: |
+ brew update
+
+ - name: Build
+ id: cmake_build
+ run: |
+ sysctl -a
+ cmake -B build -G Xcode \
+ -DGGML_METAL_USE_BF16=ON \
+ -DGGML_METAL_EMBED_LIBRARY=ON \
+ -DLLAMA_BUILD_COMMON=OFF \
+ -DLLAMA_BUILD_EXAMPLES=OFF \
+ -DLLAMA_BUILD_TOOLS=OFF \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DLLAMA_BUILD_SERVER=OFF \
+ -DCMAKE_SYSTEM_NAME=visionOS \
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
+ -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+ macOS-latest-swift:
+ runs-on: macos-latest
+ needs: ios-xcode-build
+
+ strategy:
+ matrix:
+ destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: macOS-latest-swift
+ evict-old-files: 1d
+
+ - name: Download xcframework artifact
+ uses: actions/download-artifact@v4
+ with:
+ name: llama-xcframework
+ path: build-apple/llama.xcframework/
+
+ - name: Dependencies
+ id: depends
+ continue-on-error: true
+ run: |
+ brew update
+
+ - name: Build llama.cpp with CMake
+ id: cmake_build
+ run: |
+ sysctl -a
+ cmake -B build -G Xcode \
+ -DGGML_METAL_USE_BF16=ON \
+ -DGGML_METAL_EMBED_LIBRARY=ON \
+ -DLLAMA_CURL=OFF \
+ -DLLAMA_BUILD_EXAMPLES=OFF \
+ -DLLAMA_BUILD_TOOLS=OFF \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DLLAMA_BUILD_SERVER=OFF \
+ -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
+
+ windows-msys2:
+ runs-on: windows-2025
+
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - { sys: UCRT64, env: ucrt-x86_64, build: Release }
+ - { sys: CLANG64, env: clang-x86_64, build: Release }
+
+ steps:
+ - name: Clone
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: windows-msys2
+ variant: ccache
+ evict-old-files: 1d
+
+ - name: Setup ${{ matrix.sys }}
+ uses: msys2/setup-msys2@v2
+ with:
+ update: true
+ msystem: ${{matrix.sys}}
+ install: >-
+ base-devel
+ git
+ mingw-w64-${{matrix.env}}-toolchain
+ mingw-w64-${{matrix.env}}-cmake
+ mingw-w64-${{matrix.env}}-openblas
+
+ - name: Build using CMake
+ shell: msys2 {0}
+ run: |
+ cmake -B build
+ cmake --build build --config ${{ matrix.build }} -j $(nproc)
+
+ - name: Clean after building using CMake
+ shell: msys2 {0}
+ run: |
+ rm -rf build
+
+ - name: Build using CMake w/ OpenBLAS
+ shell: msys2 {0}
+ run: |
+ cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
+ cmake --build build --config ${{ matrix.build }} -j $(nproc)
+
+ windows-latest-cmake:
+ runs-on: windows-2025
+
+ env:
+ OPENBLAS_VERSION: 0.3.23
+ SDE_VERSION: 9.33.0-2024-01-07
+ VULKAN_VERSION: 1.4.313.2
+
+ strategy:
+ matrix:
+ include:
+ - build: 'cpu-x64 (static)'
+ arch: 'x64'
+ defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'
+ - build: 'openblas-x64'
+ arch: 'x64'
+ defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
+ - build: 'vulkan-x64'
+ arch: 'x64'
+ defines: '-DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_VULKAN=ON'
+ - build: 'llvm-arm64'
+ arch: 'arm64'
+ defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON'
+ - build: 'llvm-arm64-opencl-adreno'
+ arch: 'arm64'
+ defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON'
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: windows-latest-cmake-${{ matrix.build }}
+ variant: ccache
+ evict-old-files: 1d
+
+ - name: Download OpenBLAS
+ id: get_openblas
+ if: ${{ matrix.build == 'openblas-x64' }}
+ run: |
+ curl.exe -o $env:RUNNER_TEMP/openblas.zip -L "https://github.com/xianyi/OpenBLAS/releases/download/v${env:OPENBLAS_VERSION}/OpenBLAS-${env:OPENBLAS_VERSION}-x64.zip"
+ curl.exe -o $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt -L "https://github.com/xianyi/OpenBLAS/raw/v${env:OPENBLAS_VERSION}/LICENSE"
+ mkdir $env:RUNNER_TEMP/openblas
+ tar.exe -xvf $env:RUNNER_TEMP/openblas.zip -C $env:RUNNER_TEMP/openblas
+ $vcdir = $(vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)
+ $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim()))
+ $lib = $(join-path $msvc 'bin\Hostx64\x64\lib.exe')
+ & $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll
+
+ - name: Install Vulkan SDK
+ id: get_vulkan
+ if: ${{ matrix.build == 'vulkan-x64' }}
+ run: |
+ curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
+ & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
+ Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
+ Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
+
+ - name: Install Ninja
+ id: install_ninja
+ run: |
+ choco install ninja
+
+ - name: Install OpenCL Headers and Libs
+ id: install_opencl
+ if: ${{ matrix.build == 'llvm-arm64-opencl-adreno' }}
+ run: |
+ git clone https://github.com/KhronosGroup/OpenCL-Headers
+ cd OpenCL-Headers
+ cmake -B build `
+ -DBUILD_TESTING=OFF `
+ -DOPENCL_HEADERS_BUILD_TESTING=OFF `
+ -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `
+ -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
+ cmake --build build --target install
+ git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader
+ cd OpenCL-ICD-Loader
+ cmake -B build-arm64-release `
+ -A arm64 `
+ -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" `
+ -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
+ cmake --build build-arm64-release --target install --config release
+
+ - name: libCURL
+ id: get_libcurl
+ uses: ./.github/actions/windows-setup-curl
+ with:
+ architecture: ${{ matrix.arch == 'x64' && 'win64' || 'win64a' }}
+
+ - name: Build
+ id: cmake_build
+ env:
+ CURL_PATH: ${{ steps.get_libcurl.outputs.curl_path }}
+ run: |
+ cmake -S . -B build ${{ matrix.defines }} `
+ -DCURL_LIBRARY="$env:CURL_PATH/lib/libcurl.dll.a" -DCURL_INCLUDE_DIR="$env:CURL_PATH/include"
+ cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
+ cp $env:CURL_PATH/bin/libcurl-*.dll build/bin/Release
+
+ - name: Add libopenblas.dll
+ id: add_libopenblas_dll
+ if: ${{ matrix.build == 'openblas-x64' }}
+ run: |
+ cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll
+ cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt
+
+ - name: Test
+ id: cmake_test
+ if: ${{ matrix.arch == 'x64' }}
+ run: |
+ cd build
+ ctest -L main -C Release --verbose --timeout 900
+
+ # TODO: disabled for now, consider adding tests for all CPU variants instead
+ # - name: Test (Intel SDE)
+ # id: cmake_test_sde
+ # if: ${{ matrix.build == 'avx512-x64' && env.HAS_AVX512F == '0' }} # use Intel SDE for AVX-512 emulation
+ # run: |
+ # curl.exe -o $env:RUNNER_TEMP/sde.tar.xz -L "https://downloadmirror.intel.com/813591/sde-external-${env:SDE_VERSION}-win.tar.xz"
+ # # for some weird reason windows tar doesn't like sde tar.xz
+ # 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar.xz
+ # 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar
+ # $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe)
+ # cd build
+ # $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1
+ # & $sde -future -- ctest -L main -C Release --verbose --timeout 900
+
+ ubuntu-latest-cmake-cuda:
+ runs-on: ubuntu-latest
+ container: nvidia/cuda:12.6.2-devel-ubuntu24.04
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Install dependencies
+ env:
+ DEBIAN_FRONTEND: noninteractive
+ run: |
+ apt update
+ apt install -y cmake build-essential ninja-build libgomp1 git libcurl4-openssl-dev
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-latest-cmake-cuda
+ evict-old-files: 1d
+
+ - name: Build with CMake
+ run: |
+ cmake -S . -B build -G Ninja \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_CUDA_ARCHITECTURES=89-real \
+ -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DGGML_NATIVE=OFF \
+ -DGGML_CUDA=ON
+ cmake --build build
+
+ windows-2022-cmake-cuda:
+ runs-on: windows-2022
+
+ strategy:
+ matrix:
+ cuda: ['12.4']
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Install ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: windows-cuda-${{ matrix.cuda }}
+ variant: ccache
+ evict-old-files: 1d
+
+ - name: Install Cuda Toolkit
+ uses: ./.github/actions/windows-setup-cuda
+ with:
+ cuda_version: ${{ matrix.cuda }}
+
+ - name: Install Ninja
+ id: install_ninja
+ run: |
+ choco install ninja
+
+ - name: libCURL
+ id: get_libcurl
+ uses: ./.github/actions/windows-setup-curl
+
+ - name: Build
+ id: cmake_build
+ shell: cmd
+ env:
+ CURL_PATH: ${{ steps.get_libcurl.outputs.curl_path }}
+ run: |
+ call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+ cmake -S . -B build -G "Ninja Multi-Config" ^
+ -DLLAMA_BUILD_SERVER=ON ^
+ -DGGML_NATIVE=OFF ^
+ -DGGML_BACKEND_DL=ON ^
+ -DGGML_CPU_ALL_VARIANTS=ON ^
+ -DGGML_CUDA=ON ^
+ -DGGML_RPC=ON ^
+ -DCURL_LIBRARY="%CURL_PATH%/lib/libcurl.dll.a" -DCURL_INCLUDE_DIR="%CURL_PATH%/include"
+ set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
+ cmake --build build --config Release -j %NINJA_JOBS% -t ggml
+ cmake --build build --config Release
+
+ windows-latest-cmake-sycl:
+ runs-on: windows-2022
+
+ defaults:
+ run:
+ shell: bash
+
+ env:
+ WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/24751ead-ddc5-4479-b9e6-f9fe2ff8b9f2/intel-deep-learning-essentials-2025.2.1.25_offline.exe
+ WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
+ ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: windows-latest-cmake-sycl
+ variant: ccache
+ evict-old-files: 1d
+
+ - name: Install
+ run: |
+ scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL
+
+ # TODO: add libcurl support ; we will also need to modify win-build-sycl.bat to accept user-specified args
+
+ - name: Build
+ id: cmake_build
+ run: examples/sycl/win-build-sycl.bat
+
+ windows-latest-cmake-hip:
+ runs-on: windows-2022
+
+ env:
+ # The ROCm version must correspond to the version used in the HIP SDK.
+ ROCM_VERSION: "6.4.2"
+ HIPSDK_INSTALLER_VERSION: "25.Q3"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Grab rocWMMA package
+ id: grab_rocwmma
+ run: |
+ curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/${{ env.ROCM_VERSION }}/pool/main/r/rocwmma-dev/rocwmma-dev_1.7.0.60402-120~24.04_amd64.deb"
+ 7z x rocwmma.deb
+ 7z x data.tar
+
+ - name: Cache ROCm Installation
+ id: cache-rocm
+ uses: actions/cache@v4
+ with:
+ path: C:\Program Files\AMD\ROCm
+ key: rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }}
+
+ - name: Install ROCm
+ if: steps.cache-rocm.outputs.cache-hit != 'true'
+ id: depends
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "Downloading AMD HIP SDK Installer"
+ Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-WinSvr2022-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
+ write-host "Installing AMD HIP SDK"
+ $proc = Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -PassThru
+ $completed = $proc.WaitForExit(600000)
+ if (-not $completed) {
+ Write-Error "ROCm installation timed out after 10 minutes. Killing the process"
+ $proc.Kill()
+ exit 1
+ }
+ if ($proc.ExitCode -ne 0) {
+ Write-Error "ROCm installation failed with exit code $($proc.ExitCode)"
+ exit 1
+ }
+ write-host "Completed AMD HIP SDK installation"
+
+ - name: Verify ROCm
+ id: verify
+ run: |
+ # Find and test ROCm installation
+ $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1
+ if (-not $clangPath) {
+ Write-Error "ROCm installation not found"
+ exit 1
+ }
+ & $clangPath.FullName --version
+
+ - name: Install ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ${{ github.job }}
+ evict-old-files: 1d
+
+ - name: libCURL
+ id: get_libcurl
+ uses: ./.github/actions/windows-setup-curl
+
+ - name: Build
+ id: cmake_build
+ env:
+ CURL_PATH: ${{ steps.get_libcurl.outputs.curl_path }}
+ run: |
+ $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
+ $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"
+ cmake -G "Unix Makefiles" -B build -S . `
+ -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" `
+ -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" `
+ -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-${{ env.ROCM_VERSION }}/include/" `
+ -DCMAKE_BUILD_TYPE=Release `
+ -DROCM_DIR="${env:HIP_PATH}" `
+ -DGGML_HIP=ON `
+ -DGGML_HIP_ROCWMMA_FATTN=ON `
+ -DGGML_RPC=ON `
+ -DCURL_LIBRARY="$env:CURL_PATH/lib/libcurl.dll.a" -DCURL_INCLUDE_DIR="$env:CURL_PATH/include"
+ cmake --build build -j ${env:NUMBER_OF_PROCESSORS}
+
+ ios-xcode-build:
+ runs-on: macos-latest
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Setup Xcode
+ uses: maxim-lobanov/setup-xcode@v1
+ with:
+ xcode-version: latest-stable
+
+ - name: Build
+ id: cmake_build
+ run: |
+ sysctl -a
+ cmake -B build -G Xcode \
+ -DGGML_METAL_USE_BF16=ON \
+ -DGGML_METAL_EMBED_LIBRARY=ON \
+ -DLLAMA_CURL=OFF \
+ -DLLAMA_BUILD_EXAMPLES=OFF \
+ -DLLAMA_BUILD_TOOLS=OFF \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DLLAMA_BUILD_SERVER=OFF \
+ -DCMAKE_SYSTEM_NAME=iOS \
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+ -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+ - name: xcodebuild for swift package
+ id: xcodebuild
+ run: |
+ ./build-xcframework.sh
+
+ - name: Upload xcframework artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: llama-xcframework
+ path: build-apple/llama.xcframework/
+ retention-days: 1
+
+ - name: Build Xcode project
+ run: |
+ xcodebuild -downloadPlatform iOS
+ xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
+
+ android-build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Clone
+ uses: actions/checkout@v4
+
+ # Disabled due to size (400MB) and always 0 cache hits
+ # - name: ccache
+ # uses: ggml-org/ccache-action@v1.2.16
+ # with:
+ # key: android-build
+ # evict-old-files: 1d
+
+ - name: Set up JDK
+ uses: actions/setup-java@v3
+ with:
+ java-version: 17
+ distribution: zulu
+
+ - name: Setup Android SDK
+ uses: android-actions/setup-android@v3
+ with:
+ log-accepted-android-sdk-licenses: false
+
+ - name: Build
+ run: |
+ cd examples/llama.android
+ ./gradlew build --no-daemon
+
+ openEuler-latest-cmake-cann:
+ if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'Ascend NPU') }}
+ defaults:
+ run:
+ shell: bash -el {0}
+ strategy:
+ matrix:
+ arch: [x86, aarch64]
+ cann:
+ - '8.1.RC1.alpha001-910b-openeuler22.03-py3.10'
+ device:
+ - 'ascend910b3'
+ build:
+ - 'Release'
+ runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
+ container: ascendai/cann:${{ matrix.cann }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Dependencies
+ run: |
+ yum update -y
+ yum install -y git gcc gcc-c++ make cmake libcurl-devel
+
+ - name: Build
+ run: |
+ export LD_LIBRARY_PATH=${ASCEND_TOOLKIT_HOME}/lib64:${ASCEND_TOOLKIT_HOME}/$(uname -m)-linux/devlib/:${LD_LIBRARY_PATH}
+
+ cmake -S . -B build \
+ -DCMAKE_BUILD_TYPE=${{ matrix.build }} \
+ -DGGML_CANN=on \
+ -DSOC_TYPE=${{ matrix.device }}
+ cmake --build build -j $(nproc)
+
+# TODO: simplify the following workflows using a matrix
+# TODO: run lighter CI on PRs and the full CI only on master (if needed)
+ ggml-ci-x64-cpu-low-perf:
+ runs-on: ubuntu-22.04
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ggml-ci-x64-cpu-low-perf
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install build-essential libcurl4-openssl-dev
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
+
+ ggml-ci-arm64-cpu-low-perf:
+ runs-on: ubuntu-22.04-arm
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ggml-ci-arm64-cpu-low-perf
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install build-essential libcurl4-openssl-dev
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
+
+ ggml-ci-x64-cpu-high-perf:
+ runs-on: ubuntu-22.04
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ggml-ci-x64-cpu-high-perf
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install build-essential libcurl4-openssl-dev
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ LLAMA_ARG_THREADS=$(nproc) bash ./ci/run.sh ./tmp/results ./tmp/mnt
+
+ ggml-ci-arm64-cpu-high-perf:
+ runs-on: ubuntu-22.04-arm
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ggml-ci-arm64-cpu-high-perf
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install build-essential libcurl4-openssl-dev
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_SVE=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
+
+ ggml-ci-arm64-cpu-high-perf-sve:
+ runs-on: ubuntu-22.04-arm
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ggml-ci-arm64-cpu-high-perf-sve
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install build-essential libcurl4-openssl-dev
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
+
+ ggml-ci-x64-nvidia-cuda:
+ runs-on: [self-hosted, Linux, X64, NVIDIA]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ nvidia-smi
+ GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
+
+ ggml-ci-x64-nvidia-vulkan-cm:
+ runs-on: [self-hosted, Linux, X64, NVIDIA]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ vulkaninfo --summary
+ GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
+
+ ggml-ci-x64-nvidia-vulkan-cm2:
+ runs-on: [self-hosted, Linux, X64, NVIDIA, COOPMAT2]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ vulkaninfo --summary
+ GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
+
+ ggml-ci-x64-cpu-amx:
+ runs-on: [self-hosted, Linux, X64, CPU, AMX]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
+
+ ggml-ci-mac-metal:
+ runs-on: [self-hosted, macOS, ARM64]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+ ggml-ci-mac-vulkan:
+ runs-on: [self-hosted, macOS, ARM64]
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Test
+ id: ggml-ci
+ run: |
+ vulkaninfo --summary
+ GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
diff --git a/llama.cpp/.github/workflows/close-issue.yml b/llama.cpp/.github/workflows/close-issue.yml
new file mode 100644
index 0000000000000000000000000000000000000000..cbfc4990dbc802f282f533365f6f1ff8a015d0a7
--- /dev/null
+++ b/llama.cpp/.github/workflows/close-issue.yml
@@ -0,0 +1,28 @@
+name: Close inactive issues
+on:
+ schedule:
+ - cron: "42 0 * * *"
+
+# Fine-grant permission
+# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
+permissions:
+ issues: write
+
+jobs:
+ close-issues:
+ runs-on: ubuntu-latest
+ permissions:
+ issues: write
+ pull-requests: write
+ steps:
+ - uses: actions/stale@v5
+ with:
+ exempt-issue-labels: "refactoring,help wanted,good first issue,research 🔬,bug,roadmap"
+ days-before-issue-stale: 30
+ days-before-issue-close: 14
+ stale-issue-label: "stale"
+ close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+ days-before-pr-stale: -1
+ days-before-pr-close: -1
+ operations-per-run: 10000
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/llama.cpp/.github/workflows/copilot-setup-steps.yml b/llama.cpp/.github/workflows/copilot-setup-steps.yml
new file mode 100644
index 0000000000000000000000000000000000000000..3645e30378b95438b3677ad715a600b56e993068
--- /dev/null
+++ b/llama.cpp/.github/workflows/copilot-setup-steps.yml
@@ -0,0 +1,57 @@
+name: "Copilot Setup Steps"
+
+# Automatically run the setup steps when they are changed to allow for easy validation, and
+# allow manual testing through the repository's "Actions" tab
+on:
+ workflow_dispatch:
+ push:
+ paths:
+ - .github/workflows/copilot-setup-steps.yml
+ pull_request:
+ paths:
+ - .github/workflows/copilot-setup-steps.yml
+
+jobs:
+ # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot.
+ copilot-setup-steps:
+ runs-on: ubuntu-latest
+
+ # Set the permissions to the lowest permissions possible needed for your steps.
+ # Copilot will be given its own token for its operations.
+ permissions:
+ # If you want to clone the repository as part of your setup steps, for example to install dependencies, you'll need the `contents: read` permission. If you don't clone the repository in your setup steps, Copilot will do this for you automatically after the steps complete.
+ contents: read
+
+ # You can define any steps you want, and they will run before the agent starts.
+ # If you do not check out your code, Copilot will do this for you.
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: copilot-setup-steps
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install build-essential libcurl4-openssl-dev
+ # Install git-clang-format script for formatting only changed code
+ wget -O /tmp/git-clang-format https://raw.githubusercontent.com/llvm/llvm-project/release/18.x/clang/tools/clang-format/git-clang-format
+ sudo cp /tmp/git-clang-format /usr/local/bin/git-clang-format
+ sudo chmod +x /usr/local/bin/git-clang-format
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Install Python dependencies
+ run: |
+ python3 -m venv .venv
+ .venv/bin/activate
+ pip install -r requirements/requirements-all.txt -r tools/server/tests/requirements.txt
+ pip install flake8 pyright pre-commit
diff --git a/llama.cpp/.github/workflows/docker.yml b/llama.cpp/.github/workflows/docker.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f73a2bc9f458b67a50adbda2c2b258a920e1a116
--- /dev/null
+++ b/llama.cpp/.github/workflows/docker.yml
@@ -0,0 +1,214 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: Publish Docker image
+
+on:
+ workflow_dispatch: # allows manual triggering
+ schedule:
+ # Rebuild daily rather than on every push because it is expensive
+ - cron: '12 4 * * *'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+# Fine-grant permission
+# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
+permissions:
+ packages: write
+
+jobs:
+ push_to_registry:
+ name: Push Docker image to Docker Hub
+
+ runs-on: ${{ matrix.config.runs_on }}
+ env:
+ COMMIT_SHA: ${{ github.sha }}
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ # Multi-stage build
+ # Note: the arm64 images are failing, which prevents the amd64 images from being built
+ # https://github.com/ggml-org/llama.cpp/issues/11888
+ #- { tag: "cpu", dockerfile: ".devops/cpu.Dockerfile", platforms: "linux/amd64,linux/arm64", full: true, light: true, server: true, free_disk_space: false }
+ - { tag: "cpu", dockerfile: ".devops/cpu.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false, runs_on: "ubuntu-22.04" }
+ - { tag: "cuda", dockerfile: ".devops/cuda.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false, runs_on: "ubuntu-22.04" }
+ - { tag: "musa", dockerfile: ".devops/musa.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: true, runs_on: "ubuntu-22.04" }
+ - { tag: "intel", dockerfile: ".devops/intel.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: true, runs_on: "ubuntu-22.04" }
+ - { tag: "vulkan", dockerfile: ".devops/vulkan.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false, runs_on: "ubuntu-22.04" }
+ - { tag: "s390x", dockerfile: ".devops/s390x.Dockerfile", platforms: "linux/s390x", full: true, light: true, server: true, free_disk_space: false, runs_on: "ubuntu-22.04-s390x" }
+ # Note: the rocm images are failing due to a compiler error and are disabled until this is fixed to allow the workflow to complete
+ #- {tag: "rocm", dockerfile: ".devops/rocm.Dockerfile", platforms: "linux/amd64,linux/arm64", full: true, light: true, server: true, free_disk_space: true }
+ steps:
+ - name: Check out the repo
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0 # preserve git history, so we can determine the build number
+
+ - name: Set up QEMU
+ if: ${{ matrix.config.tag != 's390x' }}
+ uses: docker/setup-qemu-action@v3
+ with:
+ image: tonistiigi/binfmt:qemu-v7.0.0-28
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Log in to Docker Hub
+ uses: docker/login-action@v2
+ with:
+ registry: ghcr.io
+ username: ${{ github.repository_owner }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Determine source tag name
+ id: srctag
+ uses: ./.github/actions/get-tag-name
+ env:
+ BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+
+ - name: Determine image tag name
+ id: tag
+ shell: bash
+ run: |
+ REPO_OWNER="${GITHUB_REPOSITORY_OWNER@L}" # to lower case
+ REPO_NAME="${{ github.event.repository.name }}"
+
+ # list all tags possible
+ if [[ "${{ matrix.config.tag }}" == "cpu" ]]; then
+ TYPE=""
+ else
+ TYPE="-${{ matrix.config.tag }}"
+ fi
+ PREFIX="ghcr.io/${REPO_OWNER}/${REPO_NAME}:"
+ CACHETAGS="${PREFIX}buildcache${TYPE}"
+ FULLTAGS="${PREFIX}full${TYPE},${PREFIX}full${TYPE}-${{ steps.srctag.outputs.name }}"
+ LIGHTTAGS="${PREFIX}light${TYPE},${PREFIX}light${TYPE}-${{ steps.srctag.outputs.name }}"
+ SERVERTAGS="${PREFIX}server${TYPE},${PREFIX}server${TYPE}-${{ steps.srctag.outputs.name }}"
+ echo "cache_output_tags=$CACHETAGS" >> $GITHUB_OUTPUT
+ echo "full_output_tags=$FULLTAGS" >> $GITHUB_OUTPUT
+ echo "light_output_tags=$LIGHTTAGS" >> $GITHUB_OUTPUT
+ echo "server_output_tags=$SERVERTAGS" >> $GITHUB_OUTPUT
+ echo "cache_output_tags=$CACHETAGS" # print out for debugging
+ echo "full_output_tags=$FULLTAGS" # print out for debugging
+ echo "light_output_tags=$LIGHTTAGS" # print out for debugging
+ echo "server_output_tags=$SERVERTAGS" # print out for debugging
+ env:
+ GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'
+
+ - name: Free Disk Space (Ubuntu)
+ if: ${{ matrix.config.free_disk_space == true }}
+ uses: ggml-org/free-disk-space@v1.3.1
+ with:
+ # this might remove tools that are actually needed,
+ # if set to "true" but frees about 6 GB
+ tool-cache: false
+
+ # all of these default to true, but feel free to set to
+ # "false" if necessary for your workflow
+ android: true
+ dotnet: true
+ haskell: true
+ large-packages: true
+ docker-images: true
+ swap-storage: true
+
+ - name: Build and push Full Docker image (tagged + versioned)
+ if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.full == true }}
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ push: true
+ platforms: ${{ matrix.config.platforms }}
+ # tag list is generated from step above
+ tags: ${{ steps.tag.outputs.full_output_tags }}
+ file: ${{ matrix.config.dockerfile }}
+ target: full
+ provenance: false
+ # using github experimental cache
+ #cache-from: type=gha
+ #cache-to: type=gha,mode=max
+ # return to this if the experimental github cache is having issues
+ #cache-to: type=local,dest=/tmp/.buildx-cache
+ #cache-from: type=local,src=/tmp/.buildx-cache
+ # using registry cache (no storage limit)
+ cache-from: type=registry,ref=${{ steps.tag.outputs.cache_output_tags }}
+ cache-to: type=registry,ref=${{ steps.tag.outputs.cache_output_tags }},mode=max
+
+ - name: Build and push Light Docker image (tagged + versioned)
+ if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.light == true }}
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ push: true
+ platforms: ${{ matrix.config.platforms }}
+ # tag list is generated from step above
+ tags: ${{ steps.tag.outputs.light_output_tags }}
+ file: ${{ matrix.config.dockerfile }}
+ target: light
+ provenance: false
+ # using github experimental cache
+ #cache-from: type=gha
+ #cache-to: type=gha,mode=max
+ # return to this if the experimental github cache is having issues
+ #cache-to: type=local,dest=/tmp/.buildx-cache
+ #cache-from: type=local,src=/tmp/.buildx-cache
+ # using registry cache (no storage limit)
+ cache-from: type=registry,ref=${{ steps.tag.outputs.cache_output_tags }}
+ cache-to: type=registry,ref=${{ steps.tag.outputs.cache_output_tags }},mode=max
+
+ - name: Build and push Server Docker image (tagged + versioned)
+ if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.server == true }}
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ push: true
+ platforms: ${{ matrix.config.platforms }}
+ # tag list is generated from step above
+ tags: ${{ steps.tag.outputs.server_output_tags }}
+ file: ${{ matrix.config.dockerfile }}
+ target: server
+ provenance: false
+ # using github experimental cache
+ #cache-from: type=gha
+ #cache-to: type=gha,mode=max
+ # return to this if the experimental github cache is having issues
+ #cache-to: type=local,dest=/tmp/.buildx-cache
+ #cache-from: type=local,src=/tmp/.buildx-cache
+ # using registry cache (no storage limit)
+ cache-from: type=registry,ref=${{ steps.tag.outputs.cache_output_tags }}
+ cache-to: type=registry,ref=${{ steps.tag.outputs.cache_output_tags }},mode=max
+
+ create_tag:
+ name: Create and push git tag
+ runs-on: ubuntu-22.04
+ permissions:
+ contents: write
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Determine source tag name
+ id: srctag
+ uses: ./.github/actions/get-tag-name
+ env:
+ BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+
+ - name: Create and push git tag
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ git tag ${{ steps.srctag.outputs.name }} || exit 0
+ git push origin ${{ steps.srctag.outputs.name }} || exit 0
diff --git a/llama.cpp/.github/workflows/editorconfig.yml b/llama.cpp/.github/workflows/editorconfig.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f02b7c2194bcf57568eb29e1d072ec9b5923f4cd
--- /dev/null
+++ b/llama.cpp/.github/workflows/editorconfig.yml
@@ -0,0 +1,29 @@
+name: EditorConfig Checker
+
+on:
+ workflow_dispatch: # allows manual triggering
+ inputs:
+ create_release:
+ description: 'Create new release'
+ required: true
+ type: boolean
+ push:
+ branches:
+ - master
+ pull_request:
+ branches:
+ - master
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ editorconfig:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: editorconfig-checker/action-editorconfig-checker@v2
+ with:
+ version: v3.0.3
+ - run: editorconfig-checker
diff --git a/llama.cpp/.github/workflows/gguf-publish.yml b/llama.cpp/.github/workflows/gguf-publish.yml
new file mode 100644
index 0000000000000000000000000000000000000000..3ca4d30581074cc1b69d7b7306eef170f0baa0d6
--- /dev/null
+++ b/llama.cpp/.github/workflows/gguf-publish.yml
@@ -0,0 +1,44 @@
+# This workflow will upload a Python Package using Twine when a GGUF release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+# See `gguf-py/README.md` for how to make a release.
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+ workflow_dispatch:
+ push:
+ # Pattern matched against refs/tags
+ tags:
+ - 'gguf-v*' # Push events to every version tag
+
+
+jobs:
+ deploy:
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.9.x'
+ - name: Install dependencies
+ run: |
+ cd gguf-py
+ python -m pip install poetry
+ poetry install
+
+ - name: Build package
+ run: cd gguf-py && poetry build
+ - name: Publish package
+ uses: pypa/gh-action-pypi-publish@release/v1
+ with:
+ password: ${{ secrets.PYPI_API_TOKEN }}
+ packages-dir: gguf-py/dist
diff --git a/llama.cpp/.github/workflows/labeler.yml b/llama.cpp/.github/workflows/labeler.yml
new file mode 100644
index 0000000000000000000000000000000000000000..0b0f300aa402ad80634d1338eb698311e4dc3a9c
--- /dev/null
+++ b/llama.cpp/.github/workflows/labeler.yml
@@ -0,0 +1,17 @@
+name: "Pull Request Labeler"
+on:
+- pull_request_target
+
+jobs:
+ labeler:
+ permissions:
+ contents: read
+ pull-requests: write
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ repository: "ggml-org/llama.cpp"
+ - uses: actions/labeler@v5
+ with:
+ configuration-path: '.github/labeler.yml'
diff --git a/llama.cpp/.github/workflows/pre-tokenizer-hashes.yml b/llama.cpp/.github/workflows/pre-tokenizer-hashes.yml
new file mode 100644
index 0000000000000000000000000000000000000000..dff998e239319a210c17c24809faf00271e43c17
--- /dev/null
+++ b/llama.cpp/.github/workflows/pre-tokenizer-hashes.yml
@@ -0,0 +1,45 @@
+name: Check Pre-Tokenizer Hashes
+
+on:
+ push:
+ paths:
+ - 'convert_hf_to_gguf.py'
+ - 'convert_hf_to_gguf_update.py'
+ pull_request:
+ paths:
+ - 'convert_hf_to_gguf.py'
+ - 'convert_hf_to_gguf_update.py'
+
+jobs:
+ pre-tokenizer-hashes:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Install Python dependencies
+ run: |
+ python3 -m venv .venv
+ .venv/bin/pip install -r requirements/requirements-convert_hf_to_gguf_update.txt
+
+ - name: Update pre-tokenizer hashes
+ run: |
+ cp convert_hf_to_gguf.py /tmp
+ .venv/bin/python convert_hf_to_gguf_update.py --check-missing
+
+ - name: Check if committed pre-tokenizer hashes matches generated version
+ run: |
+ if ! diff -q convert_hf_to_gguf.py /tmp/convert_hf_to_gguf.py; then
+ echo "Model pre-tokenizer hashes (in convert_hf_to_gguf.py) do not match generated hashes (from convert_hf_to_gguf_update.py)."
+ echo "To fix: run ./convert_hf_to_gguf_update.py and commit the updated convert_hf_to_gguf.py along with your changes"
+ echo "Differences found:"
+ diff convert_hf_to_gguf.py /tmp/convert_hf_to_gguf.py || true
+ exit 1
+ fi
+ echo "Model pre-tokenizer hashes are up to date."
diff --git a/llama.cpp/.github/workflows/python-check-requirements.yml b/llama.cpp/.github/workflows/python-check-requirements.yml
new file mode 100644
index 0000000000000000000000000000000000000000..46e80aecd0a0cd9d44847d120e2bc9c0026dfc77
--- /dev/null
+++ b/llama.cpp/.github/workflows/python-check-requirements.yml
@@ -0,0 +1,33 @@
+name: Python check requirements.txt
+
+on:
+ push:
+ paths:
+ - '.github/workflows/python-check-requirements.yml'
+ - 'scripts/check-requirements.sh'
+ - 'convert*.py'
+ - '**/requirements*.txt'
+ pull_request:
+ paths:
+ - '.github/workflows/python-check-requirements.yml'
+ - 'scripts/check-requirements.sh'
+ - 'convert*.py'
+ - '**/requirements*.txt'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ python-check-requirements:
+ runs-on: ubuntu-latest
+ name: check-requirements
+ steps:
+ - name: Check out source repository
+ uses: actions/checkout@v4
+ - name: Set up Python environment
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+ - name: Run check-requirements.sh script
+ run: bash scripts/check-requirements.sh
diff --git a/llama.cpp/.github/workflows/python-lint.yml b/llama.cpp/.github/workflows/python-lint.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ddfdf73b8fce2c12d413127eb24575dc81a8e64d
--- /dev/null
+++ b/llama.cpp/.github/workflows/python-lint.yml
@@ -0,0 +1,30 @@
+name: flake8 Lint
+
+on:
+ push:
+ branches:
+ - master
+ paths: ['.github/workflows/python-lint.yml', '**/*.py']
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: ['.github/workflows/python-lint.yml', '**/*.py']
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ flake8-lint:
+ runs-on: ubuntu-latest
+ name: Lint
+ steps:
+ - name: Check out source repository
+ uses: actions/checkout@v4
+ - name: Set up Python environment
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+ - name: flake8 Lint
+ uses: py-actions/flake8@v2
+ with:
+ plugins: "flake8-no-print"
diff --git a/llama.cpp/.github/workflows/python-type-check.yml b/llama.cpp/.github/workflows/python-type-check.yml
new file mode 100644
index 0000000000000000000000000000000000000000..373bb601020b25597408efdb4bcbf31dc4562a18
--- /dev/null
+++ b/llama.cpp/.github/workflows/python-type-check.yml
@@ -0,0 +1,40 @@
+name: Python Type-Check
+
+on:
+ push:
+ paths:
+ - '.github/workflows/python-type-check.yml'
+ - 'pyrightconfig.json'
+ - '**.py'
+ - '**/requirements*.txt'
+ pull_request:
+ paths:
+ - '.github/workflows/python-type-check.yml'
+ - 'pyrightconfig.json'
+ - '**.py'
+ - '**/requirements*.txt'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ python-type-check:
+ runs-on: ubuntu-latest
+ name: pyright type-check
+ steps:
+ - name: Check out source repository
+ uses: actions/checkout@v4
+ - name: Set up Python environment
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+ - name: Install Python dependencies
+ # TODO: use a venv
+ run: pip install -r requirements/requirements-all.txt
+ - name: Type-check with Pyright
+ uses: jakebailey/pyright-action@v2
+ with:
+ version: 1.1.382
+ level: warning
+ warnings: true
diff --git a/llama.cpp/.github/workflows/release.yml b/llama.cpp/.github/workflows/release.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2ad381159409c8f119e7ad0a92544a80d326225e
--- /dev/null
+++ b/llama.cpp/.github/workflows/release.yml
@@ -0,0 +1,798 @@
+name: Release
+
+on:
+ workflow_dispatch: # allows manual triggering
+ inputs:
+ create_release:
+ description: 'Create new release'
+ required: true
+ type: boolean
+ push:
+ branches:
+ - master
+ paths: ['.github/workflows/release.yml', '**/CMakeLists.txt', '**/.cmake', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.cuh', '**/*.swift', '**/*.m', '**/*.metal', '**/*.comp']
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+ cancel-in-progress: true
+
+env:
+ BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+ CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON"
+
+jobs:
+ macOS-arm64:
+ runs-on: macos-14
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: macOS-latest-cmake-arm64
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ continue-on-error: true
+ run: |
+ brew update
+ brew install curl
+
+ - name: Build
+ id: cmake_build
+ run: |
+ sysctl -a
+ cmake -B build \
+ -DCMAKE_INSTALL_RPATH='@loader_path' \
+ -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DGGML_METAL_USE_BF16=ON \
+ -DGGML_METAL_EMBED_LIBRARY=ON \
+ -DGGML_RPC=ON \
+ ${{ env.CMAKE_ARGS }}
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
+
+ - name: Determine tag name
+ id: tag
+ uses: ./.github/actions/get-tag-name
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ cp LICENSE ./build/bin/
+ zip -r llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.zip ./build/bin/*
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.zip
+ name: llama-bin-macos-arm64.zip
+
+ macOS-x64:
+ runs-on: macos-15-intel
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: macOS-latest-cmake-x64
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ continue-on-error: true
+ run: |
+ brew update
+ brew install curl
+
+ - name: Build
+ id: cmake_build
+ run: |
+ sysctl -a
+ # Metal is disabled due to intermittent failures with Github runners not having a GPU:
+ # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
+ cmake -B build \
+ -DCMAKE_INSTALL_RPATH='@loader_path' \
+ -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ -DGGML_METAL=OFF \
+ -DGGML_RPC=ON \
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
+
+ - name: Determine tag name
+ id: tag
+ uses: ./.github/actions/get-tag-name
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ cp LICENSE ./build/bin/
+ zip -r llama-${{ steps.tag.outputs.name }}-bin-macos-x64.zip ./build/bin/*
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-${{ steps.tag.outputs.name }}-bin-macos-x64.zip
+ name: llama-bin-macos-x64.zip
+
+ ubuntu-22-cpu:
+ strategy:
+ matrix:
+ include:
+ - build: 'x64'
+ os: ubuntu-22.04
+ # GGML_BACKEND_DL and GGML_CPU_ALL_VARIANTS are not currently supported on arm
+ # - build: 'arm64'
+ # os: ubuntu-22.04-arm
+
+ runs-on: ${{ matrix.os }}
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-cpu-cmake-${{ matrix.build }}
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get install build-essential libcurl4-openssl-dev
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DCMAKE_INSTALL_RPATH='$ORIGIN' \
+ -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+ -DGGML_BACKEND_DL=ON \
+ -DGGML_NATIVE=OFF \
+ -DGGML_CPU_ALL_VARIANTS=ON \
+ -DLLAMA_FATAL_WARNINGS=ON \
+ ${{ env.CMAKE_ARGS }}
+ cmake --build build --config Release -j $(nproc)
+
+ - name: Determine tag name
+ id: tag
+ uses: ./.github/actions/get-tag-name
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ cp LICENSE ./build/bin/
+ zip -r llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.zip ./build/bin/*
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-${{ matrix.build }}.zip
+ name: llama-bin-ubuntu-${{ matrix.build }}.zip
+
+ ubuntu-22-vulkan:
+ runs-on: ubuntu-22.04
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: ubuntu-22-cmake-vulkan
+ evict-old-files: 1d
+
+ - name: Dependencies
+ id: depends
+ run: |
+ wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add -
+ sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+ sudo apt-get update -y
+ sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libcurl4-openssl-dev
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -B build \
+ -DCMAKE_INSTALL_RPATH='$ORIGIN' \
+ -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+ -DGGML_BACKEND_DL=ON \
+ -DGGML_NATIVE=OFF \
+ -DGGML_CPU_ALL_VARIANTS=ON \
+ -DGGML_VULKAN=ON \
+ ${{ env.CMAKE_ARGS }}
+ cmake --build build --config Release -j $(nproc)
+
+ - name: Determine tag name
+ id: tag
+ uses: ./.github/actions/get-tag-name
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ cp LICENSE ./build/bin/
+ zip -r llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.zip ./build/bin/*
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.zip
+ name: llama-bin-ubuntu-vulkan-x64.zip
+
+ windows-cpu:
+ runs-on: windows-2025
+
+ strategy:
+ matrix:
+ include:
+ - arch: 'x64'
+ - arch: 'arm64'
+
+ steps:
+ - name: Clone
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: windows-latest-cmake-cpu-${{ matrix.arch }}
+ variant: ccache
+ evict-old-files: 1d
+
+ - name: Install Ninja
+ run: |
+ choco install ninja
+
+ - name: libCURL
+ id: get_libcurl
+ uses: ./.github/actions/windows-setup-curl
+ with:
+ architecture: ${{ matrix.arch == 'x64' && 'win64' || 'win64a' }}
+
+ - name: Build
+ shell: cmd
+ env:
+ CURL_PATH: ${{ steps.get_libcurl.outputs.curl_path }}
+ run: |
+ call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }}
+ cmake -S . -B build -G "Ninja Multi-Config" ^
+ -D CMAKE_TOOLCHAIN_FILE=cmake/${{ matrix.arch }}-windows-llvm.cmake ^
+ -DGGML_NATIVE=OFF ^
+ -DGGML_BACKEND_DL=ON ^
+ -DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} ^
+ -DGGML_OPENMP=ON ^
+ -DCURL_LIBRARY="%CURL_PATH%/lib/libcurl.dll.a" -DCURL_INCLUDE_DIR="%CURL_PATH%/include" ^
+ ${{ env.CMAKE_ARGS }}
+ cmake --build build --config Release
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ env:
+ CURL_PATH: ${{ steps.get_libcurl.outputs.curl_path }}
+ run: |
+ Copy-Item $env:CURL_PATH\bin\libcurl-${{ matrix.arch }}.dll .\build\bin\Release\
+ Copy-Item "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Redist\MSVC\14.44.35112\debug_nonredist\${{ matrix.arch }}\Microsoft.VC143.OpenMP.LLVM\libomp140.${{ matrix.arch == 'x64' && 'x86_64' || 'aarch64' }}.dll" .\build\bin\Release\
+ 7z a llama-bin-win-cpu-${{ matrix.arch }}.zip .\build\bin\Release\*
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-bin-win-cpu-${{ matrix.arch }}.zip
+ name: llama-bin-win-cpu-${{ matrix.arch }}.zip
+
+ windows:
+ runs-on: windows-2025
+
+ env:
+ OPENBLAS_VERSION: 0.3.23
+ VULKAN_VERSION: 1.4.313.2
+
+ strategy:
+ matrix:
+ include:
+ - backend: 'vulkan'
+ arch: 'x64'
+ defines: '-DGGML_VULKAN=ON'
+ target: 'ggml-vulkan'
+ - backend: 'opencl-adreno'
+ arch: 'arm64'
+ defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON'
+ target: 'ggml-opencl'
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: windows-latest-cmake-${{ matrix.backend }}-${{ matrix.arch }}
+ variant: ccache
+ evict-old-files: 1d
+
+ - name: Install Vulkan SDK
+ id: get_vulkan
+ if: ${{ matrix.backend == 'vulkan' }}
+ run: |
+ curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
+ & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
+ Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
+ Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
+
+ - name: Install Ninja
+ id: install_ninja
+ run: |
+ choco install ninja
+
+ - name: Install OpenCL Headers and Libs
+ id: install_opencl
+ if: ${{ matrix.backend == 'opencl-adreno' && matrix.arch == 'arm64' }}
+ run: |
+ git clone https://github.com/KhronosGroup/OpenCL-Headers
+ cd OpenCL-Headers
+ cmake -B build `
+ -DBUILD_TESTING=OFF `
+ -DOPENCL_HEADERS_BUILD_TESTING=OFF `
+ -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `
+ -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
+ cmake --build build --target install
+ git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader
+ cd OpenCL-ICD-Loader
+ cmake -B build-arm64-release `
+ -A arm64 `
+ -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" `
+ -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
+ cmake --build build-arm64-release --target install --config release
+
+ - name: Build
+ id: cmake_build
+ run: |
+ cmake -S . -B build ${{ matrix.defines }} -DGGML_NATIVE=OFF -DGGML_CPU=OFF -DGGML_BACKEND_DL=ON -DLLAMA_CURL=OFF
+ cmake --build build --config Release --target ${{ matrix.target }}
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ 7z a llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip .\build\bin\Release\${{ matrix.target }}.dll
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip
+ name: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip
+
+ windows-cuda:
+ runs-on: windows-2022
+
+ strategy:
+ matrix:
+ cuda: ['12.4']
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Install ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: windows-cuda-${{ matrix.cuda }}
+ variant: ccache
+ evict-old-files: 1d
+
+ - name: Install Cuda Toolkit
+ uses: ./.github/actions/windows-setup-cuda
+ with:
+ cuda_version: ${{ matrix.cuda }}
+
+ - name: Install Ninja
+ id: install_ninja
+ run: |
+ choco install ninja
+
+ - name: Build
+ id: cmake_build
+ shell: cmd
+ run: |
+ call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+ cmake -S . -B build -G "Ninja Multi-Config" ^
+ -DGGML_BACKEND_DL=ON ^
+ -DGGML_NATIVE=OFF ^
+ -DGGML_CPU=OFF ^
+ -DGGML_CUDA=ON ^
+ -DLLAMA_CURL=OFF
+ set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
+ cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ 7z a llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip .\build\bin\Release\ggml-cuda.dll
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
+ name: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
+
+ - name: Copy and pack Cuda runtime
+ run: |
+ echo "Cuda install location: ${{ env.CUDA_PATH }}"
+ $dst='.\build\bin\cudart\'
+ robocopy "${{env.CUDA_PATH}}\bin" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll
+ robocopy "${{env.CUDA_PATH}}\lib" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll
+ 7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip $dst\*
+
+ - name: Upload Cuda runtime
+ uses: actions/upload-artifact@v4
+ with:
+ path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
+ name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
+
+ windows-sycl:
+ runs-on: windows-2022
+
+ defaults:
+ run:
+ shell: bash
+
+ env:
+ WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/24751ead-ddc5-4479-b9e6-f9fe2ff8b9f2/intel-deep-learning-essentials-2025.2.1.25_offline.exe
+ WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
+ ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: windows-latest-cmake-sycl
+ variant: ccache
+ evict-old-files: 1d
+
+ - name: Install
+ run: |
+ scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL
+
+ - name: Build
+ id: cmake_build
+ shell: cmd
+ run: |
+ call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force
+ cmake -G "Ninja" -B build ^
+ -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=icx ^
+ -DCMAKE_BUILD_TYPE=Release ^
+ -DGGML_BACKEND_DL=ON -DBUILD_SHARED_LIBS=ON ^
+ -DGGML_CPU=OFF -DGGML_SYCL=ON ^
+ -DLLAMA_CURL=OFF
+ cmake --build build --target ggml-sycl -j
+
+ - name: Build the release package
+ id: pack_artifacts
+ run: |
+ echo "cp oneAPI running time dll files in ${{ env.ONEAPI_ROOT }} to ./build/bin"
+
+ cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_sycl_blas.5.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_core.2.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.2.dll" ./build/bin
+
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero_v2.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_opencl.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_loader.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_win_proxy_loader.dll" ./build/bin
+
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl8.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/svml_dispmd.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libmmd.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libiomp5md.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl-ls.exe" ./build/bin
+
+ cp "${{ env.ONEAPI_ROOT }}/dnnl/latest/bin/dnnl.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/tbb/latest/bin/tbb12.dll" ./build/bin
+
+ cp "${{ env.ONEAPI_ROOT }}/tcm/latest/bin/tcm.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/tcm/latest/bin/libhwloc-15.dll" ./build/bin
+ cp "${{ env.ONEAPI_ROOT }}/umf/latest/bin/umf.dll" ./build/bin
+
+ echo "cp oneAPI running time dll files to ./build/bin done"
+ 7z a llama-bin-win-sycl-x64.zip ./build/bin/*
+
+ - name: Upload the release package
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-bin-win-sycl-x64.zip
+ name: llama-bin-win-sycl-x64.zip
+
+ windows-hip:
+ runs-on: windows-2022
+
+ env:
+ HIPSDK_INSTALLER_VERSION: "25.Q3"
+
+ strategy:
+ matrix:
+ include:
+ - name: "radeon"
+ gpu_targets: "gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+
+ - name: Grab rocWMMA package
+ id: grab_rocwmma
+ run: |
+ curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/7.0.1/pool/main/r/rocwmma-dev/rocwmma-dev_2.0.0.70001-42~24.04_amd64.deb"
+ 7z x rocwmma.deb
+ 7z x data.tar
+
+ - name: Cache ROCm Installation
+ id: cache-rocm
+ uses: actions/cache@v4
+ with:
+ path: C:\Program Files\AMD\ROCm
+ key: rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }}
+
+ - name: ccache
+ uses: ggml-org/ccache-action@v1.2.16
+ with:
+ key: windows-latest-cmake-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ matrix.name }}-x64
+ evict-old-files: 1d
+
+ - name: Install ROCm
+ if: steps.cache-rocm.outputs.cache-hit != 'true'
+ id: depends
+ run: |
+ $ErrorActionPreference = "Stop"
+ write-host "Downloading AMD HIP SDK Installer"
+ Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-WinSvr2022-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
+ write-host "Installing AMD HIP SDK"
+ $proc = Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -PassThru
+ $completed = $proc.WaitForExit(600000)
+ if (-not $completed) {
+ Write-Error "ROCm installation timed out after 10 minutes. Killing the process"
+ $proc.Kill()
+ exit 1
+ }
+ if ($proc.ExitCode -ne 0) {
+ Write-Error "ROCm installation failed with exit code $($proc.ExitCode)"
+ exit 1
+ }
+ write-host "Completed AMD HIP SDK installation"
+
+ - name: Verify ROCm
+ id: verify
+ run: |
+ # Find and test ROCm installation
+ $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1
+ if (-not $clangPath) {
+ Write-Error "ROCm installation not found"
+ exit 1
+ }
+ & $clangPath.FullName --version
+
+ - name: Build
+ id: cmake_build
+ run: |
+ $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
+ $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"
+ cmake -G "Unix Makefiles" -B build -S . `
+ -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" `
+ -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" `
+ -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-7.0.1/include/ -Wno-ignored-attributes -Wno-nested-anon-types" `
+ -DCMAKE_BUILD_TYPE=Release `
+ -DGGML_BACKEND_DL=ON `
+ -DGGML_NATIVE=OFF `
+ -DGGML_CPU=OFF `
+ -DAMDGPU_TARGETS="${{ matrix.gpu_targets }}" `
+ -DGGML_HIP_ROCWMMA_FATTN=ON `
+ -DGGML_HIP=ON `
+ -DLLAMA_CURL=OFF
+ cmake --build build --target ggml-hip -j ${env:NUMBER_OF_PROCESSORS}
+ md "build\bin\rocblas\library\"
+ md "build\bin\hipblaslt\library"
+ cp "${env:HIP_PATH}\bin\hipblas.dll" "build\bin\"
+ cp "${env:HIP_PATH}\bin\hipblaslt.dll" "build\bin\"
+ cp "${env:HIP_PATH}\bin\rocblas.dll" "build\bin\"
+ cp "${env:HIP_PATH}\bin\rocblas\library\*" "build\bin\rocblas\library\"
+ cp "${env:HIP_PATH}\bin\hipblaslt\library\*" "build\bin\hipblaslt\library\"
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ 7z a llama-bin-win-hip-${{ matrix.name }}-x64.zip .\build\bin\*
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-bin-win-hip-${{ matrix.name }}-x64.zip
+ name: llama-bin-win-hip-${{ matrix.name }}-x64.zip
+
+ ios-xcode-build:
+ runs-on: macos-15
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Setup Xcode
+ run: |
+ sudo xcode-select -s /Applications/Xcode_16.4.app
+
+ - name: Build
+ id: cmake_build
+ run: |
+ sysctl -a
+ cmake -B build -G Xcode \
+ -DGGML_METAL_USE_BF16=ON \
+ -DGGML_METAL_EMBED_LIBRARY=ON \
+ -DLLAMA_CURL=OFF \
+ -DLLAMA_BUILD_EXAMPLES=OFF \
+ -DLLAMA_BUILD_TOOLS=OFF \
+ -DLLAMA_BUILD_TESTS=OFF \
+ -DLLAMA_BUILD_SERVER=OFF \
+ -DCMAKE_SYSTEM_NAME=iOS \
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+ -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+ - name: xcodebuild for swift package
+ id: xcodebuild
+ run: |
+ ./build-xcframework.sh
+
+ - name: Build Xcode project
+ run: xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
+
+ - name: Determine tag name
+ id: tag
+ uses: ./.github/actions/get-tag-name
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ zip --symlinks -r llama-${{ steps.tag.outputs.name }}-xcframework.zip build-apple/llama.xcframework
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-${{ steps.tag.outputs.name }}-xcframework.zip
+ name: llama-${{ steps.tag.outputs.name }}-xcframework
+
+ release:
+ if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+
+ # Fine-grant permission
+ # https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
+ permissions:
+ contents: write # for creating release
+
+ runs-on: ubuntu-latest
+
+ needs:
+ - windows
+ - windows-cpu
+ - windows-cuda
+ - windows-sycl
+ - windows-hip
+ - ubuntu-22-cpu
+ - ubuntu-22-vulkan
+ - macOS-arm64
+ - macOS-x64
+ - ios-xcode-build
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Determine tag name
+ id: tag
+ uses: ./.github/actions/get-tag-name
+
+ - name: Download artifacts
+ id: download-artifact
+ uses: actions/download-artifact@v4
+ with:
+ path: ./artifact
+ merge-multiple: true
+
+ - name: Move artifacts
+ id: move_artifacts
+ run: |
+ mkdir -p release
+
+ echo "Adding CPU backend files to existing zips..."
+ for arch in x64 arm64; do
+ cpu_zip="artifact/llama-bin-win-cpu-${arch}.zip"
+ temp_dir=$(mktemp -d)
+ echo "Extracting CPU backend for $arch..."
+ unzip "$cpu_zip" -d "$temp_dir"
+
+ echo "Adding CPU files to $arch zips..."
+ for target_zip in artifact/llama-bin-win-*-${arch}.zip; do
+ if [[ "$target_zip" == "$cpu_zip" ]]; then
+ continue
+ fi
+ echo "Adding CPU backend to $(basename "$target_zip")"
+ realpath_target_zip=$(realpath "$target_zip")
+ (cd "$temp_dir" && zip -r "$realpath_target_zip" .)
+ done
+
+ rm -rf "$temp_dir"
+ done
+
+ echo "Renaming and moving zips to release..."
+ for zip_file in artifact/llama-bin-win-*.zip; do
+ base_name=$(basename "$zip_file" .zip)
+ zip_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.zip"
+ echo "Moving $zip_file to release/$zip_name"
+ mv "$zip_file" "release/$zip_name"
+ done
+
+ echo "Moving other artifacts..."
+ mv -v artifact/*.zip release
+
+ - name: Create release
+ id: create_release
+ uses: ggml-org/action-create-release@v1
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ tag_name: ${{ steps.tag.outputs.name }}
+
+ - name: Upload release
+ id: upload_release
+ uses: actions/github-script@v3
+ with:
+ github-token: ${{secrets.GITHUB_TOKEN}}
+ script: |
+ const path = require('path');
+ const fs = require('fs');
+ const release_id = '${{ steps.create_release.outputs.id }}';
+ for (let file of await fs.readdirSync('./release')) {
+ if (path.extname(file) === '.zip') {
+ console.log('uploadReleaseAsset', file);
+ await github.repos.uploadReleaseAsset({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ release_id: release_id,
+ name: file,
+ data: await fs.readFileSync(`./release/${file}`)
+ });
+ }
+ }
diff --git a/llama.cpp/.github/workflows/server.yml b/llama.cpp/.github/workflows/server.yml
new file mode 100644
index 0000000000000000000000000000000000000000..1ea1300c2e4c3255c113bf4dfce4f082666b0615
--- /dev/null
+++ b/llama.cpp/.github/workflows/server.yml
@@ -0,0 +1,392 @@
+# Server build and tests
+name: Server
+
+on:
+ workflow_dispatch: # allows manual triggering
+ inputs:
+ sha:
+ description: 'Commit SHA1 to build'
+ required: false
+ type: string
+ slow_tests:
+ description: 'Run slow tests'
+ required: true
+ type: boolean
+ push:
+ branches:
+ - master
+ paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'tools/server/**.*']
+ pull_request:
+ types: [opened, synchronize, reopened]
+ paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'tools/server/**.*']
+
+env:
+ LLAMA_LOG_COLORS: 1
+ LLAMA_LOG_PREFIX: 1
+ LLAMA_LOG_TIMESTAMPS: 1
+ LLAMA_LOG_VERBOSITY: 10
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ server:
+ runs-on: ubuntu-latest
+
+ strategy:
+ matrix:
+ sanitizer: [ADDRESS, UNDEFINED] # THREAD is broken
+ build_type: [RelWithDebInfo]
+ include:
+ - build_type: Release
+ sanitizer: ""
+ fail-fast: false # While -DLLAMA_SANITIZE_THREAD=ON is broken
+
+ steps:
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get -y install \
+ build-essential \
+ xxd \
+ git \
+ cmake \
+ curl \
+ wget \
+ language-pack-en \
+ libcurl4-openssl-dev
+
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
+
+ - name: Python setup
+ id: setup_python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Tests dependencies
+ id: test_dependencies
+ run: |
+ pip install -r tools/server/tests/requirements.txt
+
+ webui-setup:
+ name: WebUI Setup
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: "22"
+ cache: "npm"
+ cache-dependency-path: "tools/server/webui/package-lock.json"
+
+ - name: Cache node_modules
+ uses: actions/cache@v4
+ id: cache-node-modules
+ with:
+ path: tools/server/webui/node_modules
+ key: ${{ runner.os }}-node-modules-${{ hashFiles('tools/server/webui/package-lock.json') }}
+ restore-keys: |
+ ${{ runner.os }}-node-modules-
+
+ - name: Install dependencies
+ if: steps.cache-node-modules.outputs.cache-hit != 'true'
+ run: npm ci
+ working-directory: tools/server/webui
+
+ webui-check:
+ needs: webui-setup
+ name: WebUI Check
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: "22"
+
+ - name: Restore node_modules cache
+ uses: actions/cache@v4
+ with:
+ path: tools/server/webui/node_modules
+ key: ${{ runner.os }}-node-modules-${{ hashFiles('tools/server/webui/package-lock.json') }}
+ restore-keys: |
+ ${{ runner.os }}-node-modules-
+
+ - name: Run type checking
+ run: npm run check
+ working-directory: tools/server/webui
+
+ - name: Run linting
+ run: npm run lint
+ working-directory: tools/server/webui
+
+ webui-build:
+ needs: webui-check
+ name: WebUI Build
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: "22"
+
+ - name: Restore node_modules cache
+ uses: actions/cache@v4
+ with:
+ path: tools/server/webui/node_modules
+ key: ${{ runner.os }}-node-modules-${{ hashFiles('tools/server/webui/package-lock.json') }}
+ restore-keys: |
+ ${{ runner.os }}-node-modules-
+
+ - name: Build application
+ run: npm run build
+ working-directory: tools/server/webui
+
+ webui-tests:
+ needs: webui-build
+ name: Run WebUI tests
+ permissions:
+ contents: read
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: "22"
+
+ - name: Restore node_modules cache
+ uses: actions/cache@v4
+ with:
+ path: tools/server/webui/node_modules
+ key: ${{ runner.os }}-node-modules-${{ hashFiles('tools/server/webui/package-lock.json') }}
+ restore-keys: |
+ ${{ runner.os }}-node-modules-
+
+ - name: Install Playwright browsers
+ run: npx playwright install --with-deps
+ working-directory: tools/server/webui
+
+ - name: Build Storybook
+ run: npm run build-storybook
+ working-directory: tools/server/webui
+
+ - name: Run Client tests
+ run: npm run test:client
+ working-directory: tools/server/webui
+
+ - name: Run Server tests
+ run: npm run test:server
+ working-directory: tools/server/webui
+
+ - name: Run UI tests
+ run: npm run test:ui
+ working-directory: tools/server/webui
+
+ - name: Run E2E tests
+ run: npm run test:e2e
+ working-directory: tools/server/webui
+
+ server-build:
+ needs: [webui-tests]
+ runs-on: ubuntu-latest
+
+ strategy:
+ matrix:
+ sanitizer: [ADDRESS, UNDEFINED] # THREAD is broken
+ build_type: [RelWithDebInfo]
+ include:
+ - build_type: Release
+ sanitizer: ""
+ fail-fast: false # While -DLLAMA_SANITIZE_THREAD=ON is broken
+
+ steps:
+ - name: Dependencies
+ id: depends
+ run: |
+ sudo apt-get update
+ sudo apt-get -y install \
+ build-essential \
+ xxd \
+ git \
+ cmake \
+ curl \
+ wget \
+ language-pack-en \
+ libcurl4-openssl-dev
+
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
+
+ - name: Python setup
+ id: setup_python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Tests dependencies
+ id: test_dependencies
+ run: |
+ pip install -r tools/server/tests/requirements.txt
+
+ - name: Setup Node.js for WebUI
+ uses: actions/setup-node@v4
+ with:
+ node-version: "22"
+ cache: "npm"
+ cache-dependency-path: "tools/server/webui/package-lock.json"
+
+ - name: Install WebUI dependencies
+ run: npm ci
+ working-directory: tools/server/webui
+
+ - name: Build WebUI
+ run: npm run build
+ working-directory: tools/server/webui
+
+ - name: Build (no OpenMP)
+ id: cmake_build_no_openmp
+ if: ${{ matrix.sanitizer == 'THREAD' }}
+ run: |
+ cmake -B build \
+ -DGGML_NATIVE=OFF \
+ -DLLAMA_BUILD_SERVER=ON \
+ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+ -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
+ -DGGML_OPENMP=OFF ;
+ cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target llama-server
+
+ - name: Build (sanitizers)
+ id: cmake_build_sanitizers
+ if: ${{ matrix.sanitizer != '' && matrix.sanitizer != 'THREAD' }}
+ run: |
+ cmake -B build \
+ -DGGML_NATIVE=OFF \
+ -DLLAMA_BUILD_SERVER=ON \
+ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+ -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON ;
+ cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target llama-server
+
+ - name: Build (sanitizers)
+ id: cmake_build
+ if: ${{ matrix.sanitizer == '' }}
+ run: |
+ cmake -B build \
+ -DGGML_NATIVE=OFF \
+ -DLLAMA_BUILD_SERVER=ON \
+ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ;
+ cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target llama-server
+
+ - name: Tests
+ id: server_integration_tests
+ if: ${{ matrix.sanitizer == '' }}
+ env:
+ GITHUB_ACTIONS: "true"
+ run: |
+ cd tools/server/tests
+ ./tests.sh
+
+ - name: Tests (sanitizers)
+ id: server_integration_tests_sanitizers
+ if: ${{ matrix.sanitizer != '' }}
+ run: |
+ cd tools/server/tests
+ LLAMA_SANITIZE=1 ./tests.sh
+
+ - name: Slow tests
+ id: server_integration_tests_slow
+ if: ${{ (github.event.schedule || github.event.inputs.slow_tests == 'true') && matrix.build_type == 'Release' }}
+ run: |
+ cd tools/server/tests
+ SLOW_TESTS=1 ./tests.sh
+
+
+ server-windows:
+ runs-on: windows-2022
+
+ steps:
+ - name: Clone
+ id: checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
+
+ - name: libCURL
+ id: get_libcurl
+ uses: ./.github/actions/windows-setup-curl
+
+ - name: Build
+ id: cmake_build
+ env:
+ CURL_PATH: ${{ steps.get_libcurl.outputs.curl_path }}
+ run: |
+ cmake -B build -DCURL_LIBRARY="$env:CURL_PATH/lib/libcurl.dll.a" -DCURL_INCLUDE_DIR="$env:CURL_PATH/include"
+ cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS} --target llama-server
+
+ - name: Python setup
+ id: setup_python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Tests dependencies
+ id: test_dependencies
+ run: |
+ pip install -r tools/server/tests/requirements.txt
+
+ - name: Copy Libcurl
+ id: prepare_libcurl
+ env:
+ CURL_PATH: ${{ steps.get_libcurl.outputs.curl_path }}
+ run: |
+ cp $env:CURL_PATH/bin/libcurl-x64.dll ./build/bin/Release/libcurl-x64.dll
+
+ - name: Tests
+ id: server_integration_tests
+ if: ${{ !matrix.disabled_on_pr || !github.event.pull_request }}
+ run: |
+ cd tools/server/tests
+ $env:PYTHONIOENCODING = ":replace"
+ pytest -v -x -m "not slow"
+
+ - name: Slow tests
+ id: server_integration_tests_slow
+ if: ${{ (github.event.schedule || github.event.inputs.slow_tests == 'true') && matrix.build_type == 'Release' }}
+ run: |
+ cd tools/server/tests
+ $env:SLOW_TESTS = "1"
+ pytest -v -x
diff --git a/llama.cpp/.github/workflows/update-ops-docs.yml b/llama.cpp/.github/workflows/update-ops-docs.yml
new file mode 100644
index 0000000000000000000000000000000000000000..c0218fa7421735e014151d73cf30b44379fde84c
--- /dev/null
+++ b/llama.cpp/.github/workflows/update-ops-docs.yml
@@ -0,0 +1,40 @@
+name: Update Operations Documentation
+
+on:
+ push:
+ paths:
+ - 'docs/ops/**'
+ - 'scripts/create_ops_docs.py'
+ pull_request:
+ paths:
+ - 'docs/ops/**'
+ - 'scripts/create_ops_docs.py'
+
+jobs:
+ update-ops-docs:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.x'
+
+ - name: Generate operations documentation to temporary file
+ run: |
+ mkdir -p /tmp/ops_check
+ ./scripts/create_ops_docs.py /tmp/ops_check/ops.md
+
+ - name: Check if docs/ops.md matches generated version
+ run: |
+ if ! diff -q docs/ops.md /tmp/ops_check/ops.md; then
+ echo "Operations documentation (docs/ops.md) is not up to date with the backend CSV files."
+ echo "To fix: run ./scripts/create_ops_docs.py and commit the updated docs/ops.md along with your changes"
+ echo "Differences found:"
+ diff docs/ops.md /tmp/ops_check/ops.md || true
+ exit 1
+ fi
+ echo "Operations documentation is up to date."
diff --git a/llama.cpp/.github/workflows/winget.yml b/llama.cpp/.github/workflows/winget.yml
new file mode 100644
index 0000000000000000000000000000000000000000..5c286155951e5ef3bb8d8e5b923ccc3b13456c04
--- /dev/null
+++ b/llama.cpp/.github/workflows/winget.yml
@@ -0,0 +1,42 @@
+name: Update Winget Package
+
+on:
+ workflow_dispatch: # allows manual triggering
+ schedule:
+ - cron: '28 5 * * *' # Update every day at 5:28 UTC
+
+jobs:
+ update:
+ name: Update Winget Package
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Install cargo binstall
+ uses: cargo-bins/cargo-binstall@268643a6b5ea099f5718ee5cd3ff7dc89a5eb49b
+
+ - name: Install komac
+ run: |
+ cargo binstall komac@2.11.2 -y
+
+ - name: Find latest release
+ id: find_latest_release
+ uses: actions/github-script@v6
+ with:
+ script: |
+ const { data: releases } = await github.rest.repos.listReleases({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ });
+ console.log("Latest release:", releases[0].tag_name);
+ return releases[0].tag_name;
+
+ - name: Update manifest
+ env:
+ VERSION: ${{ steps.find_latest_release.outputs.result }}
+ run: |
+ echo "Updating manifest..."
+ komac update --version ${{ env.VERSION }} \
+ --urls "https://github.com/ggml-org/llama.cpp/releases/download/${{ env.VERSION }}/llama-${{ env.VERSION }}-bin-win-vulkan-x64.zip" \
+ --token ${{ secrets.WINGET_GITHUB_TOKEN }} \
+ --submit \
+ ggml.llamacpp
diff --git a/llama.cpp/.gitignore b/llama.cpp/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..c7d000978571aca57eb1a2904d9c875e9cfb9d1d
--- /dev/null
+++ b/llama.cpp/.gitignore
@@ -0,0 +1,154 @@
+# Extensions
+
+*.a
+*.bat
+*.bin
+*.d
+*.dll
+*.dot
+*.etag
+*.exe
+*.gcda
+*.gcno
+*.gcov
+*.gguf
+*.gguf.json
+*.lastModified
+*.log
+*.metallib
+*.o
+*.so
+*.swp
+*.tmp
+
+# IDE / OS
+
+.cache/
+.ccls-cache/
+.direnv/
+.DS_Store
+.envrc
+.idea/
+.swiftpm
+.vs/
+.vscode/
+nppBackup
+
+
+# Coverage
+
+gcovr-report/
+lcov-report/
+
+# Build Artifacts
+
+tags
+.build/
+build*
+release
+debug
+!build-info.cmake
+!build-info.cpp.in
+!build-info.sh
+!build.zig
+!docs/build.md
+/libllama.so
+/llama-*
+/vulkan-shaders-gen
+android-ndk-*
+arm_neon.h
+cmake-build-*
+CMakeSettings.json
+compile_commands.json
+ggml-metal-embed.metal
+llama-batched-swift
+/rpc-server
+out/
+tmp/
+autogen-*.md
+
+# Deprecated
+
+/main
+/server
+
+# CI
+
+!.github/workflows/*.yml
+
+# Models
+
+models/*
+models-mnt
+!models/.editorconfig
+!models/ggml-vocab-*.gguf*
+!models/templates
+
+# Zig
+zig-out/
+zig-cache/
+
+# Logs
+
+ppl-*.txt
+qnt-*.txt
+perf-*.txt
+
+# Examples
+
+examples/jeopardy/results.txt
+tools/server/*.css.hpp
+tools/server/*.html.hpp
+tools/server/*.js.hpp
+tools/server/*.mjs.hpp
+tools/server/*.gz.hpp
+!build_64.sh
+!examples/*.bat
+!examples/*/*.kts
+!examples/*/*/*.kts
+!examples/sycl/*.bat
+!examples/sycl/*.sh
+
+# Server Web UI temporary files
+node_modules
+tools/server/webui/dist
+
+# Python
+
+/.venv
+__pycache__/
+*/poetry.lock
+poetry.toml
+
+# Nix
+/result
+
+# Test binaries
+/tests/test-backend-ops
+/tests/test-double-float
+/tests/test-grad0
+/tests/test-grammar-parser
+/tests/test-llama-grammar
+/tests/test-opt
+/tests/test-quantize-fns
+/tests/test-quantize-perf
+/tests/test-rope
+/tests/test-sampling
+/tests/test-tokenizer-0
+/tests/test-tokenizer-1-bpe
+/tests/test-tokenizer-1-spm
+
+# Scripts
+!/scripts/install-oneapi.bat
+
+# Test models for lora adapters
+/lora-tests
+
+# Local scripts
+/run-vim.sh
+/run-chat.sh
+.ccache/
+
+# IDE
+*.code-workspace
+.windsurf/
diff --git a/llama.cpp/.gitmodules b/llama.cpp/.gitmodules
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/llama.cpp/.pre-commit-config.yaml b/llama.cpp/.pre-commit-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..91d7916285081aa14f5b801935cb35d031f2e601
--- /dev/null
+++ b/llama.cpp/.pre-commit-config.yaml
@@ -0,0 +1,16 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+exclude: prompts/.*.txt
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.6.0
+ hooks:
+ - id: trailing-whitespace
+ - id: end-of-file-fixer
+ - id: check-yaml
+ - id: check-added-large-files
+- repo: https://github.com/PyCQA/flake8
+ rev: 7.0.0
+ hooks:
+ - id: flake8
+ additional_dependencies: [flake8-no-print]
diff --git a/llama.cpp/AUTHORS b/llama.cpp/AUTHORS
new file mode 100644
index 0000000000000000000000000000000000000000..0af9f44ad4a16bd57d739acad7bd1e3f45256677
--- /dev/null
+++ b/llama.cpp/AUTHORS
@@ -0,0 +1,1106 @@
+# date: Sat Mar 8 18:23:52 EET 2025
+# this file is auto-generated by scripts/gen-authors.sh
+
+0cc4m Expand details for performance related PR only
+ + - Concurrent users: ${{ env.N_USERS }}, duration: ${{ github.event.inputs.duration || env.DURATION }} + - HTTP request : avg=${{ env.HTTP_REQ_DURATION_AVG }}ms p(95)=${{ env.HTTP_REQ_DURATION_P_95_ }}ms fails=${{ env.HTTP_REQ_FAILED_PASSES }}, finish reason: stop=${{ env.LLAMACPP_COMPLETIONS_STOP_RATE_PASSES }} truncated=${{ env.LLAMACPP_COMPLETIONS_TRUNCATED_RATE_PASSES }} + - Prompt processing (pp): avg=${{ env.LLAMACPP_PROMPT_PROCESSING_SECOND_AVG }}tk/s p(95)=${{ env.LLAMACPP_PROMPT_PROCESSING_SECOND_P_95_ }}tk/s + - Token generation (tg): avg=${{ env.LLAMACPP_TOKENS_SECOND_AVG }}tk/s p(95)=${{ env.LLAMACPP_TOKENS_SECOND_P_95_ }}tk/s + - ${{ env.BENCH_GRAPH_XLABEL }} + + +
+
+
+
+
+
+
+
+ More
+
+ ```mermaid
+ ${{ env.PROMPT_TOKENS_SECONDS }}
+ ```
+
+
+
+
+
+
+ More
+
+ ```mermaid
+ ${{ env.PREDICTED_TOKENS_SECONDS }}
+ ```
+
+
+
+
+ Details
+ +
+
+
+
+
+
+
+ More
+
+ ```mermaid
+ ${{ env.KV_CACHE_USAGE_RATIO }}
+ ```
+
+
+
+
+
+
+ More
+
+ ```mermaid
+ ${{ env.REQUESTS_PROCESSING }}
+ ```
+
+