wmantly/turing-multi-gpu-llm-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: update to Q5_K_P @ 128K ctx, non-blocking proxy sessions, and full multi-GPU benchmark suite

Browse Source
This commit is contained in:
wmantly
2026-09-13 02:14:25 +00:00
parent cfeac35ad6
commit cd989b3e70
5 changed files with 406 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/start-server.sh
+13 -9
View File
@@ -1,5 +1,5 @@
#!/bin/bash
# llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB)
# llama-server launcher for Qwen3.8-27B on Turing GPUs (RTX 2060 12GB + CMP 50HX 10GB)
# Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism.
# Accelerated with HauhauCS FastMTP 32K draft sidecar and official thinking parameters.
export LD_LIBRARY_PATH=/opt/llama.cpp-xrip/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu
@@ -12,16 +12,20 @@ export NCCL_P2P_DISABLE=0
export NCCL_ALGO=RING
export NCCL_PROTO=SIMPLE
# Dynamically target the 12GB RTX 2060 for mmproj and FastMTP draft model
RTX_DEV=$(/opt/llama.cpp-xrip/build-nccl/bin/llama-cli --list-devices | grep -i "RTX 2060" | awk '{print $1}' | tr -d ':')
RTX_DEV=${RTX_DEV:-CUDA0}
exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
-m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
-m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q5_K_P.gguf \
--spec-draft-model /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf \
--spec-draft-device CUDA2 \
--spec-draft-device "$RTX_DEV" \
--spec-draft-ngl all \
--spec-type draft-mtp \
--spec-draft-n-max 3 \
--spec-draft-p-min 0 \
--mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \
--mmproj-device CUDA2 \
--mmproj-device "$RTX_DEV" \
--image-min-tokens 1024 \
--temp 1.0 \
--top-k 20 \
@@ -35,17 +39,17 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
--reasoning-format deepseek \
--numa split \
-ngl 99 \
-c 204800 \
-c 131072 \
--parallel 1 \
--slot-save-path /var/cache/llama-slots \
--cache-ram 16384 \
--split-mode tensor \
--flash-attn on \
--batch-size 1024 \
--ubatch-size 512 \
--batch-size 2048 \
--ubatch-size 1024 \
--jinja \
--threads 12 \
--cache-type-k q4_0 \
--cache-type-v q4_0 \
--cache-type-k q5_0 \
--cache-type-v q5_0 \
--host 0.0.0.0 \
--port 8080