feat: integrate xrip/llama.cpp-avx1-numa-sm75 fork with NCCL & NUMA optimizations
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
# llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB)
|
||||
# Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism.
|
||||
export LD_LIBRARY_PATH=/opt/llama.cpp/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu
|
||||
export LD_LIBRARY_PATH=/opt/llama.cpp-xrip/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu
|
||||
|
||||
# Low-Latency NCCL & CUDA Driver Pipeline Optimizations
|
||||
export CUDA_DEVICE_MAX_CONNECTIONS=1
|
||||
@@ -11,10 +11,11 @@ export NCCL_P2P_DISABLE=0
|
||||
export NCCL_ALGO=RING
|
||||
export NCCL_PROTO=SIMPLE
|
||||
|
||||
exec /opt/llama.cpp/build-nccl/bin/llama-server \
|
||||
exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
|
||||
-m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
|
||||
--spec-type draft-mtp \
|
||||
--spec-draft-n-max 2 \
|
||||
--numa split \
|
||||
-ngl 99 \
|
||||
-c 262144 \
|
||||
--parallel 1 \
|
||||
|
||||
Reference in New Issue
Block a user