Georgi Gerganov
6b9ba4f2e8
* ci : fix data race between runners sharing the model cache Two runners per mac mini share HOME, so concurrent jobs corrupt each other downloads, converted models and venv under the shared MNT cache. - gg_wget: download in a temp dir, publish with mv, so readers only ever see complete files (-N cannot be combined with -O) - convert, quantize and wikitext unzip now produce under a per-runner scratch dir in OUT, published to MNT via hardlink when missing - gguf-split and quantize tests get a per-runner TMP_DIR - python venv moved from MNT to OUT - metal, vulkan and webgpu mac jobs pass a per-runner OUT path Assisted-by: dsh:llama.cpp/agent-qwen4 (RTX 5090 machine) * ci : clean up scratch models after tests Assisted-by: dsh:llama.cpp/agent-qwen4 (RTX 5090 machine) * ci : fix q4_0 publish when a cached model exists cp -n skips an existing file with exit code 1, so the ln fallback aborted the qwen3 job under set -e. Publish through a unique temp file with an atomic mv: readers never see a partial file and a stale cached q4_0 gets replaced. Assisted-by: dsh:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (RTX 5090 machine) * ci : cache-first gg_wget to avoid re-downloads HF sends no Last-Modified, so wget -N re-downloads the full files on every run. Skip the download when a non-empty cache file exists: the HF CI artifacts are static. Downloads still go through a temp dir and an atomic mv, so concurrent runners never read partial files. Assisted-by: dsh:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (RTX 5090 machine)
722 lines
25 KiB
Bash
Executable File
722 lines
25 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# sample usage:
|
|
#
|
|
# mkdir tmp
|
|
#
|
|
# # CPU-only build
|
|
# bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
# # with CUDA support
|
|
# GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
# # with ROCm support
|
|
# GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
# # with SYCL support
|
|
# GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
# # with VULKAN support
|
|
# GG_BUILD_VULKAN=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
# # with WebGPU support
|
|
# GG_BUILD_WEBGPU=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
# # with MUSA support
|
|
# GG_BUILD_MUSA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
# # with KLEIDIAI support
|
|
# GG_BUILD_KLEIDIAI=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
# # with BLAS support
|
|
# GG_BUILD_BLAS=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
# with BLAS support (custom vendor)
|
|
# GG_BUILD_BLAS=1 GG_BUILD_BLAS_VENDOR=Intel10_64lp bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
# with OPENVINO support
|
|
# GG_BUILD_OPENVINO=1 GG_BUILD_LOW_PERF=1 GGML_OPENVINO_DEVICE=CPU bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
|
#
|
|
|
|
if [ -z "$2" ]; then
|
|
echo "usage: $0 <output-dir> <mnt-dir>"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$1"
|
|
mkdir -p "$2"
|
|
|
|
OUT=$(realpath "$1")
|
|
MNT=$(realpath "$2")
|
|
|
|
rm -f $OUT/*.log
|
|
|
|
sd=`dirname $0`
|
|
cd $sd/../
|
|
SRC=`pwd`
|
|
|
|
CMAKE_EXTRA="-DLLAMA_FATAL_WARNINGS=${LLAMA_FATAL_WARNINGS:-ON} -DLLAMA_OPENSSL=OFF -DGGML_SCHED_NO_REALLOC=ON"
|
|
CTEST_EXTRA=""
|
|
|
|
# Default to use make unless specified for compatibility
|
|
CMAKE_GENERATOR="Unix Makefiles"
|
|
|
|
if [ ! -z "${GG_BUILD_NINJA}" ]; then
|
|
CMAKE_GENERATOR="Ninja"
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_METAL} ]; then
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON"
|
|
else
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=OFF"
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_CUDA} ]; then
|
|
# TODO: Drop GGML_CUDA_CCCL_VERSION when CUDA CI uses CTK >= 13.5, which bundles CCCL >= 3.5.
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DGGML_CUDA_CCCL_VERSION=v3.4.3"
|
|
|
|
if command -v nvidia-smi >/dev/null 2>&1; then
|
|
CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')
|
|
if [[ -n "$CUDA_ARCH" && "$CUDA_ARCH" =~ ^[0-9]+$ ]]; then
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}"
|
|
else
|
|
echo "Warning: Using fallback CUDA architectures"
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=61;70;75;80;86;89"
|
|
fi
|
|
else
|
|
echo "Error: nvidia-smi not found, cannot build with CUDA"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_ROCM} ]; then
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_HIP_COMPILER=$(hipconfig -l)/clang -DGGML_HIP=ON"
|
|
if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then
|
|
echo "Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)"
|
|
exit 1
|
|
fi
|
|
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGPU_TARGETS=${GG_BUILD_AMDGPU_TARGETS}"
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_SYCL} ]; then
|
|
if [ -z ${ONEAPI_ROOT} ]; then
|
|
echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:"
|
|
echo "source /opt/intel/oneapi/setvars.sh"
|
|
exit 1
|
|
fi
|
|
# Use only main GPU
|
|
export ONEAPI_DEVICE_SELECTOR="level_zero:0"
|
|
# Enable sysman for correct memory reporting
|
|
export ZES_ENABLE_SYSMAN=1
|
|
# to circumvent precision issues on CPY operations
|
|
export SYCL_PROGRAM_COMPILE_OPTIONS="-cl-fp32-correctly-rounded-divide-sqrt"
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_VULKAN} ]; then
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=1"
|
|
|
|
if [[ "$OSTYPE" == "darwin"* ]]; then
|
|
MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION="/usr/local/lib/cmake/vulkan"
|
|
MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION="${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers/SPIRV-HeadersConfig.cmake"
|
|
if [[ -f "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" || -h "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" ]]; then
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DSPIRV-Headers_DIR=${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers"
|
|
fi
|
|
fi
|
|
|
|
# Build shared libs on Windows
|
|
# to reduce binary size and avoid errors in library loading unit tests
|
|
if uname -s | grep -qi nt; then
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DBUILD_SHARED_LIBS=ON"
|
|
fi
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_WEBGPU} ]; then
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_WEBGPU=1"
|
|
|
|
if [ ! -z "${GG_BUILD_WEBGPU_DAWN_PREFIX}" ]; then
|
|
if [ -z "${CMAKE_PREFIX_PATH}" ]; then
|
|
export CMAKE_PREFIX_PATH="${GG_BUILD_WEBGPU_DAWN_PREFIX}"
|
|
else
|
|
export CMAKE_PREFIX_PATH="${GG_BUILD_WEBGPU_DAWN_PREFIX}:${CMAKE_PREFIX_PATH}"
|
|
fi
|
|
fi
|
|
|
|
# For some systems, Dawn_DIR needs to be set explicitly, e.g., the lib64 path
|
|
if [ ! -z "${GG_BUILD_WEBGPU_DAWN_DIR}" ]; then
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DDawn_DIR=${GG_BUILD_WEBGPU_DAWN_DIR}"
|
|
fi
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_MUSA} ]; then
|
|
# Use ph1 by default (MTT S5000)
|
|
MUSA_ARCH=${MUSA_ARCH:-31}
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_NO_SVE} ]; then
|
|
# arm 9 and newer enables sve by default, adjust these flags depending on the cpu used
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm"
|
|
fi
|
|
|
|
if [ -n "${GG_BUILD_KLEIDIAI}" ]; then
|
|
echo ">>===== Enabling KleidiAI support"
|
|
CMAKE_EXTRA="${CMAKE_EXTRA:+$CMAKE_EXTRA } -DGGML_CPU_KLEIDIAI=ON"
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_BLAS} ]; then
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=${GG_BUILD_BLAS_VENDOR:-OpenBLAS}"
|
|
else
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_BLAS=OFF"
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_OPENVINO} ]; then
|
|
if [ -z ${OpenVINO_DIR} ]; then
|
|
echo "OpenVINO_DIR not found, please install OpenVINO via archives and enable it by:"
|
|
echo "source /opt/intel/openvino/setupvars.sh"
|
|
exit 1
|
|
fi
|
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
|
|
|
|
# TODO: fix failing tests on OpenVINO backend
|
|
CTEST_EXTRA="-E test-llama-archs|test-llama-context"
|
|
fi
|
|
|
|
## helpers
|
|
|
|
# download a file unless already cached in $MNT
|
|
function gg_wget {
|
|
local out=$1
|
|
local url=$2
|
|
|
|
local cwd=`pwd`
|
|
local name=`basename $url`
|
|
local tmp=.gg-wget.$$.tmp
|
|
|
|
mkdir -p $out
|
|
cd $out
|
|
|
|
# cache first: HF artifacts are static, HF sends no Last-Modified for -N
|
|
if [ ! -s $name ]; then
|
|
# publish with mv: concurrent runners never read partial files
|
|
rm -rf $tmp && mkdir $tmp
|
|
(cd $tmp && wget -nv $url)
|
|
if [ -s $tmp/$name ]; then
|
|
mv -f $tmp/$name $name
|
|
fi
|
|
rm -rf $tmp
|
|
fi
|
|
|
|
cd $cwd
|
|
}
|
|
|
|
function gg_run {
|
|
ci=$1
|
|
|
|
set -o pipefail
|
|
set -x
|
|
|
|
gg_run_$ci | tee $OUT/$ci.log
|
|
cur=$?
|
|
|
|
set +x
|
|
set +o pipefail
|
|
|
|
ret=$((ret | cur))
|
|
}
|
|
|
|
## ci
|
|
|
|
# ctest_debug
|
|
|
|
function gg_run_ctest_debug {
|
|
cd ${SRC}
|
|
|
|
rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug
|
|
|
|
set -e
|
|
|
|
# Check required binaries are installed
|
|
gg_check_build_requirements
|
|
|
|
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
|
(time cmake --build . --config Debug -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
|
|
|
|
(time ctest -C Debug --output-on-failure -L main -E "test-opt|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
|
|
|
set +e
|
|
}
|
|
|
|
# ctest_release
|
|
|
|
function gg_run_ctest_release {
|
|
cd ${SRC}
|
|
|
|
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
|
|
|
set -e
|
|
|
|
# Check required binaries are installed
|
|
gg_check_build_requirements
|
|
|
|
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
|
(time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
|
|
|
|
if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
|
(time ctest -C Release --output-on-failure -L main ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
|
else
|
|
(time ctest -C Release --output-on-failure -L main -E test-opt ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
|
fi
|
|
|
|
set +e
|
|
}
|
|
|
|
# test_llama_archs_tensor_split
|
|
|
|
function gg_run_test_llama_archs_tensor_split {
|
|
cd ${SRC}
|
|
|
|
set -e
|
|
|
|
if [ ! -z ${GG_BUILD_CUDA} ]; then
|
|
GGML_CUDA_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
|
|
GGML_CUDA_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
|
|
GGML_CUDA_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
|
|
GGML_CUDA_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
|
|
fi
|
|
|
|
if [ ! -z ${GG_BUILD_METAL} ]; then
|
|
GGML_METAL_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
|
|
GGML_METAL_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
|
|
GGML_METAL_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
|
|
GGML_METAL_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
|
|
fi
|
|
|
|
set +e
|
|
}
|
|
|
|
# test_llama_archs_models
|
|
|
|
function gg_run_test_llama_archs_models {
|
|
cd ${SRC}
|
|
|
|
set -e
|
|
|
|
# TODO: fix and re-enable `test-llama-archs` on OpenVINO
|
|
# TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists
|
|
if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then
|
|
rm -rf build-ci-models && mkdir -p build-ci-models
|
|
|
|
# generate the dummy models used by the model-dependent tests
|
|
./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1
|
|
fi
|
|
|
|
set +e
|
|
}
|
|
|
|
# test_scripts
|
|
|
|
function gg_run_test_scripts {
|
|
cd ${SRC}
|
|
|
|
set -e
|
|
|
|
(cd ./tools/gguf-split && time bash tests.sh "$SRC/build-ci-release/bin" "$OUT/scratch") 2>&1 | tee -a $OUT/${ci}-scripts.log
|
|
(cd ./tools/quantize && time bash tests.sh "$SRC/build-ci-release/bin" "$OUT/scratch") 2>&1 | tee -a $OUT/${ci}-scripts.log
|
|
|
|
set +e
|
|
}
|
|
|
|
function gg_get_model {
|
|
#local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-f16.gguf"
|
|
local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-q4_0.gguf"
|
|
if [[ -s $gguf_0 ]]; then
|
|
echo -n "$gguf_0"
|
|
else
|
|
echo >&2 "No model found. Can't run gg_run_ctest_with_model."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
function gg_run_ctest_with_model_debug {
|
|
cd ${SRC}
|
|
|
|
local model; model=$(gg_get_model)
|
|
cd build-ci-debug
|
|
set -e
|
|
|
|
(LLAMACPP_TEST_MODELFILE="$model" time ctest -C Debug --output-on-failure -L model) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
|
|
|
set +e
|
|
cd ..
|
|
}
|
|
|
|
function gg_run_ctest_with_model_release {
|
|
cd ${SRC}
|
|
|
|
local model; model=$(gg_get_model)
|
|
cd build-ci-release
|
|
set -e
|
|
|
|
(LLAMACPP_TEST_MODELFILE="$model" time ctest -C Release --output-on-failure -L model) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
|
|
|
# test memory leaks
|
|
#if [[ ! -z ${GG_BUILD_METAL} ]]; then
|
|
# # TODO: this hangs for some reason ...
|
|
# (time leaks -quiet -atExit -- ./bin/test-thread-safety -m $model --parallel 2 -t 2 -p "hello") 2>&1 | tee -a $OUT/${ci}-leaks.log
|
|
#fi
|
|
|
|
set +e
|
|
cd ..
|
|
}
|
|
|
|
# qwen3_0_6b
|
|
|
|
function gg_run_qwen3_0_6b {
|
|
cd ${SRC}
|
|
|
|
gg_wget models-mnt/qwen3/0.6B/ https://huggingface.co/Qwen/Qwen3-0.6B-Base/raw/main/config.json
|
|
gg_wget models-mnt/qwen3/0.6B/ https://huggingface.co/Qwen/Qwen3-0.6B-Base/raw/main/tokenizer.json
|
|
gg_wget models-mnt/qwen3/0.6B/ https://huggingface.co/Qwen/Qwen3-0.6B-Base/raw/main/tokenizer_config.json
|
|
#gg_wget models-mnt/qwen3/0.6B/ https://huggingface.co/Qwen/Qwen3-0.6B-Base/raw/main/special_tokens_map.json
|
|
gg_wget models-mnt/qwen3/0.6B/ https://huggingface.co/Qwen/Qwen3-0.6B-Base/resolve/main/model.safetensors
|
|
|
|
|
|
gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip
|
|
|
|
rm -rf $OUT/scratch/qwen3-0.6B $OUT/scratch/wikitext
|
|
mkdir -p $OUT/scratch/qwen3-0.6B
|
|
unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d $OUT/scratch/wikitext/
|
|
|
|
path_models="../models-mnt/qwen3/0.6B"
|
|
path_local="$OUT/scratch/qwen3-0.6B"
|
|
path_wiki="$OUT/scratch/wikitext/wikitext-2-raw"
|
|
|
|
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
|
|
|
set -e
|
|
|
|
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
|
(time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
|
|
|
|
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-f16.gguf --outtype f16
|
|
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-bf16.gguf --outtype bf16
|
|
|
|
model_f16="${path_local}/ggml-model-f16.gguf"
|
|
model_bf16="${path_local}/ggml-model-bf16.gguf"
|
|
model_q4_0="${path_local}/ggml-model-q4_0.gguf"
|
|
|
|
wiki_test="${path_wiki}/wiki.test.raw"
|
|
|
|
function check_ppl {
|
|
qnt="$1"
|
|
ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
|
|
|
|
if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then
|
|
printf ' - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl"
|
|
return 20
|
|
fi
|
|
|
|
printf ' - %s @ %s OK\n' "$qnt" "$ppl"
|
|
return 0
|
|
}
|
|
|
|
for qnt in q8_0 q4_0 q4_1 q5_0 q5_1 q2_k q3_k q4_k q5_k q6_k; do
|
|
model="${path_local}/ggml-model-${qnt}.gguf"
|
|
|
|
./bin/llama-quantize ${model_bf16} ${model} ${qnt} $(nproc)
|
|
|
|
(time ./bin/llama-completion -no-cnv --model ${model} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-${qnt}.log
|
|
(time ./bin/llama-perplexity --model ${model} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-${qnt}.log
|
|
|
|
# note: ppl > 20.0 for q2_k with this model
|
|
if [ "${qnt}" != "q2_k" ]; then
|
|
check_ppl "${qnt}" "$(cat $OUT/${ci}-tg-${qnt}.log | grep "^\[1\]")"
|
|
fi
|
|
|
|
if [ "${qnt}" = "q4_0" ]; then
|
|
# q4_0 is kept for other tests
|
|
q4_0="${path_models}/ggml-model-q4_0.gguf"
|
|
ln ${model} "$q4_0" 2>/dev/null || { cp ${model} $q4_0.tmp.$$ && mv -f $q4_0.tmp.$$ $q4_0; }
|
|
else
|
|
rm -f ${model}
|
|
fi
|
|
done
|
|
|
|
(time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
|
|
|
|
(time ./bin/llama-completion -no-cnv --model ${model_f16} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
|
(time ./bin/llama-completion -no-cnv --model ${model_bf16} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
|
|
|
|
(time ./bin/llama-perplexity --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
|
if [ -z ${GG_BUILD_NO_BF16} ]; then
|
|
(time ./bin/llama-perplexity --model ${model_bf16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
|
|
fi
|
|
|
|
(time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
|
|
|
|
# TODO: these tests are extremely slow on WebGPU - re-enable when fixed
|
|
# https://github.com/ggml-org/llama.cpp/actions/runs/39639985732
|
|
if [ -z ${GG_BUILD_WEBGPU} ]; then
|
|
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
|
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
|
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
|
(time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
|
|
fi
|
|
|
|
check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")"
|
|
if [ -z ${GG_BUILD_NO_BF16} ]; then
|
|
check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")"
|
|
fi
|
|
|
|
# scratch is rebuilt on every run; q4_0 is already published to $MNT
|
|
rm -rf $OUT/scratch/qwen3-0.6B $OUT/scratch/wikitext
|
|
|
|
set +e
|
|
}
|
|
|
|
# bge-small
|
|
|
|
function gg_run_embd_bge_small {
|
|
cd ${SRC}
|
|
|
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/config.json
|
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/tokenizer.json
|
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/tokenizer_config.json
|
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/special_tokens_map.json
|
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/pytorch_model.bin
|
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/sentence_bert_config.json
|
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/vocab.txt
|
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/modules.json
|
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/config.json
|
|
|
|
gg_wget models-mnt/bge-small/1_Pooling https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/1_Pooling/config.json
|
|
|
|
path_models="../models-mnt/bge-small"
|
|
path_local="$OUT/scratch/bge-small"
|
|
rm -rf $path_local && mkdir -p $path_local
|
|
|
|
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
|
|
|
set -e
|
|
|
|
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
|
(time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
|
|
|
|
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-f16.gguf
|
|
|
|
model_f16="${path_local}/ggml-model-f16.gguf"
|
|
model_q8_0="${path_local}/ggml-model-q8_0.gguf"
|
|
|
|
./bin/llama-quantize ${model_f16} ${model_q8_0} q8_0
|
|
|
|
(time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
|
|
|
|
(time ./bin/llama-embedding --model ${model_f16} -p "I believe the meaning of life is" -ngl 99 -c 0 --no-op-offload) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
|
(time ./bin/llama-embedding --model ${model_q8_0} -p "I believe the meaning of life is" -ngl 99 -c 0 --no-op-offload) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
|
|
|
|
rm -rf $path_local
|
|
|
|
set +e
|
|
}
|
|
|
|
# rerank_tiny
|
|
|
|
function gg_run_rerank_tiny {
|
|
cd ${SRC}
|
|
|
|
gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/config.json
|
|
gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/tokenizer.json
|
|
gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/tokenizer_config.json
|
|
gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/special_tokens_map.json
|
|
gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/resolve/main/pytorch_model.bin
|
|
gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/vocab.json
|
|
|
|
path_models="../models-mnt/rerank-tiny"
|
|
path_local="$OUT/scratch/rerank-tiny"
|
|
rm -rf $path_local && mkdir -p $path_local
|
|
|
|
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
|
|
|
set -e
|
|
|
|
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
|
(time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
|
|
|
|
python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-f16.gguf
|
|
|
|
model_f16="${path_local}/ggml-model-f16.gguf"
|
|
|
|
(time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
|
|
|
|
# for this model, the SEP token is "</s>"
|
|
(time ./bin/llama-embedding --model ${model_f16} -p "what is panda?\thi\nwhat is panda?\tit's a bear\nwhat is panda?\tThe giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China." -ngl 99 -c 0 --pooling rank --embd-normalize -1 --no-op-offload --verbose-prompt) 2>&1 | tee -a $OUT/${ci}-rk-f16.log
|
|
|
|
# sample output
|
|
# rerank score 0: 0.029
|
|
# rerank score 1: 0.029
|
|
# rerank score 2: 0.135
|
|
|
|
# check that the score is in the range [$3, $4]
|
|
function check_score {
|
|
qnt="$1"
|
|
score=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
|
|
|
|
if [ $(echo "$score < $3" | bc) -eq 1 ] || [ $(echo "$score > $4" | bc) -eq 1 ]; then
|
|
printf ' - %s @ %s (FAIL: score not in range [%s, %s])\n' "$qnt" "$score" "$3" "$4"
|
|
return 20
|
|
fi
|
|
|
|
printf ' - %s @ %s OK\n' "$qnt" "$score"
|
|
return 0
|
|
}
|
|
|
|
check_score "rerank score 0" "$(cat $OUT/${ci}-rk-f16.log | grep "rerank score 0")" "0.00" "0.05" | tee -a $OUT/${ci}-rk-f16.log
|
|
check_score "rerank score 1" "$(cat $OUT/${ci}-rk-f16.log | grep "rerank score 1")" "0.00" "0.05" | tee -a $OUT/${ci}-rk-f16.log
|
|
check_score "rerank score 2" "$(cat $OUT/${ci}-rk-f16.log | grep "rerank score 2")" "0.10" "0.30" | tee -a $OUT/${ci}-rk-f16.log
|
|
|
|
rm -rf $path_local
|
|
|
|
set +e
|
|
}
|
|
|
|
function gg_check_build_requirements {
|
|
if ! command -v git &> /dev/null; then
|
|
echo 'git not found, please install'
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v git-lfs &> /dev/null; then
|
|
echo 'git-lfs not found, please install'
|
|
exit 1
|
|
fi
|
|
|
|
if ! git config --get filter.lfs.clean &> /dev/null; then
|
|
echo 'git-lfs not initialized, please run `git lfs install`'
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v wget &> /dev/null; then
|
|
echo 'wget not found, please install'
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v python3 &> /dev/null; then
|
|
echo 'python3 not found, please install'
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v pip3 &> /dev/null; then
|
|
echo 'pip3 not found, please install'
|
|
exit 1
|
|
fi
|
|
|
|
if ! python3 -m ensurepip --help &> /dev/null; then
|
|
echo 'ensurepip not found, please install python3-venv package'
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v cmake &> /dev/null; then
|
|
echo 'cmake not found, please install'
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v ccache &> /dev/null; then
|
|
echo 'ccache not found, please consider installing for faster builds'
|
|
fi
|
|
|
|
if ! command -v ctest &> /dev/null; then
|
|
echo 'ctest not found, please install'
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v unzip &> /dev/null; then
|
|
echo 'unzip not found, please install'
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
function gg_run_test_backend_ops {
|
|
cd ${SRC}
|
|
|
|
cd build-ci-release
|
|
|
|
set -e
|
|
|
|
local n_jobs=$(nproc)
|
|
if [ "${n_jobs}" -gt 2 ]; then
|
|
n_jobs=2
|
|
fi
|
|
local args_extra="-j ${n_jobs}"
|
|
|
|
# TODO: MoltenVK bug?
|
|
# https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897
|
|
if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then
|
|
args_extra=""
|
|
fi
|
|
|
|
# TODO: OpenVINO GPU plugin crashes (CL_OUT_OF_RESOURCES) with 2 concurrent workers on GPU.
|
|
if [ ! -z "${GG_BUILD_OPENVINO}" ] && [ "${GGML_OPENVINO_DEVICE:-}" = "GPU" ]; then
|
|
args_extra=""
|
|
fi
|
|
|
|
# TODO: reduce the test-backend-ops timeout to 1800s
|
|
if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
|
|
(time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
|
|
else
|
|
(time timeout 3600 ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
|
|
fi
|
|
|
|
set +e
|
|
}
|
|
|
|
## main
|
|
|
|
export LLAMA_ARG_LOG_PREFIX=1
|
|
export LLAMA_ARG_LOG_TIMESTAMPS=1
|
|
|
|
if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
|
# Create symlink: ./llama.cpp/models-mnt -> $MNT/models
|
|
rm -rf ${SRC}/models-mnt
|
|
mnt_models=${MNT}/models
|
|
mkdir -p ${mnt_models}
|
|
ln -sfn ${mnt_models} ${SRC}/models-mnt
|
|
|
|
# Create a fresh python3 venv and enter it
|
|
if ! python3 -m venv "$OUT/venv"; then
|
|
echo "Error: Failed to create Python virtual environment at $OUT/venv."
|
|
exit 1
|
|
fi
|
|
source "$OUT/venv/bin/activate"
|
|
|
|
pip install -r ${SRC}/requirements.txt --disable-pip-version-check
|
|
pip install --editable gguf-py --disable-pip-version-check
|
|
fi
|
|
|
|
ret=0
|
|
|
|
test $ret -eq 0 && gg_run ctest_debug
|
|
test $ret -eq 0 && gg_run ctest_release
|
|
|
|
test $ret -eq 0 && gg_run test_backend_ops
|
|
|
|
test $ret -eq 0 && gg_run test_llama_archs_models
|
|
test $ret -eq 0 && gg_run test_llama_archs_tensor_split
|
|
|
|
if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
|
test $ret -eq 0 && gg_run embd_bge_small
|
|
test $ret -eq 0 && gg_run rerank_tiny
|
|
|
|
if [ -z ${GG_BUILD_CLOUD} ] || [ ${GG_BUILD_EXTRA_TESTS_0} ]; then
|
|
test $ret -eq 0 && gg_run test_scripts
|
|
fi
|
|
|
|
test $ret -eq 0 && gg_run qwen3_0_6b
|
|
|
|
test $ret -eq 0 && gg_run ctest_with_model_debug
|
|
test $ret -eq 0 && gg_run ctest_with_model_release
|
|
fi
|
|
|
|
exit $ret
|