wmantly/turing-multi-gpu-llm-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Configure HauhauCS FastMTP 32K draft head, reasoning parameters, and non-thinking mode

Browse Source
This commit is contained in:
wmantly
2026-09-06 22:46:11 +00:00
parent 2af141b38e
commit cfeac35ad6
2 changed files with 22 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ollama-proxy.py
+4
View File
@@ -143,6 +143,10 @@ MODEL_TAGS = [
"Qwen3.8-27B-Uncensored", "Qwen3.8-27B-Uncensored",
"qwen3.8-27b:latest", "qwen3.8-27b:latest",
"qwen3.8-27b", "qwen3.8-27b",
"qwen3.8-27b-aggressive-q3",
"qwen3.8-27b-aggressive-q3:latest",
"qwen3.8-27b-aggressive",
"qwen3.8-27b-aggressive:latest",
] ]
# Set to False to hide reasoning_content / thinking blocks entirely. # Set to False to hide reasoning_content / thinking blocks entirely.
scripts/start-server.sh
+18 -3
View File
@@ -1,6 +1,7 @@
#!/bin/bash #!/bin/bash
# llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB) # llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB)
# Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism. # Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism.
# Accelerated with HauhauCS FastMTP 32K draft sidecar and official thinking parameters.
export LD_LIBRARY_PATH=/opt/llama.cpp-xrip/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu export LD_LIBRARY_PATH=/opt/llama.cpp-xrip/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu
# Low-Latency NCCL & CUDA Driver Pipeline Optimizations # Low-Latency NCCL & CUDA Driver Pipeline Optimizations
@@ -13,14 +14,28 @@ export NCCL_PROTO=SIMPLE
exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
-m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \ -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
--spec-draft-model /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf \
--spec-draft-device CUDA2 \
--spec-draft-ngl all \
--spec-type draft-mtp \
--spec-draft-n-max 3 \
--spec-draft-p-min 0 \
--mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \ --mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \
--mmproj-device CUDA2 \ --mmproj-device CUDA2 \
--image-min-tokens 1024 \ --image-min-tokens 1024 \
--spec-type draft-mtp \ --temp 1.0 \
--spec-draft-n-max 2 \ --top-k 20 \
--top-p 0.95 \
--min-p 0 \
--presence-penalty 0 \
--repeat-penalty 1.0 \
--reasoning on \
--reasoning-effort xhigh \
--reasoning-preserve \
--reasoning-format deepseek \
--numa split \ --numa split \
-ngl 99 \ -ngl 99 \
-c 262144 \ -c 204800 \
--parallel 1 \ --parallel 1 \
--slot-save-path /var/cache/llama-slots \ --slot-save-path /var/cache/llama-slots \
--cache-ram 16384 \ --cache-ram 16384 \