wmantly/turing-multi-gpu-llm-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate xrip/llama.cpp-avx1-numa-sm75 fork with NCCL & NUMA optimizations

Browse Source
This commit is contained in:
wmantly
2026-09-02 16:02:55 +00:00
parent 38b308c6fd
commit ba0aaed7bb
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/start-server.sh
+3 -2
View File
@@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
# llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB) # llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB)
# Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism. # Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism.
export LD_LIBRARY_PATH=/opt/llama.cpp/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu export LD_LIBRARY_PATH=/opt/llama.cpp-xrip/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu
# Low-Latency NCCL & CUDA Driver Pipeline Optimizations # Low-Latency NCCL & CUDA Driver Pipeline Optimizations
export CUDA_DEVICE_MAX_CONNECTIONS=1 export CUDA_DEVICE_MAX_CONNECTIONS=1
@@ -11,10 +11,11 @@ export NCCL_P2P_DISABLE=0
export NCCL_ALGO=RING export NCCL_ALGO=RING
export NCCL_PROTO=SIMPLE export NCCL_PROTO=SIMPLE
exec /opt/llama.cpp/build-nccl/bin/llama-server \ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
-m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \ -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
--spec-type draft-mtp \ --spec-type draft-mtp \
--spec-draft-n-max 2 \ --spec-draft-n-max 2 \
--numa split \
-ngl 99 \ -ngl 99 \
-c 262144 \ -c 262144 \
--parallel 1 \ --parallel 1 \