Configure HauhauCS FastMTP 32K draft head, reasoning parameters, and non-thinking mode
This commit is contained in:
@@ -143,6 +143,10 @@ MODEL_TAGS = [
|
||||
"Qwen3.8-27B-Uncensored",
|
||||
"qwen3.8-27b:latest",
|
||||
"qwen3.8-27b",
|
||||
"qwen3.8-27b-aggressive-q3",
|
||||
"qwen3.8-27b-aggressive-q3:latest",
|
||||
"qwen3.8-27b-aggressive",
|
||||
"qwen3.8-27b-aggressive:latest",
|
||||
]
|
||||
|
||||
# Set to False to hide reasoning_content / thinking blocks entirely.
|
||||
|
||||
+18
-3
@@ -1,6 +1,7 @@
|
||||
#!/bin/bash
|
||||
# llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB)
|
||||
# Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism.
|
||||
# Accelerated with HauhauCS FastMTP 32K draft sidecar and official thinking parameters.
|
||||
export LD_LIBRARY_PATH=/opt/llama.cpp-xrip/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu
|
||||
|
||||
# Low-Latency NCCL & CUDA Driver Pipeline Optimizations
|
||||
@@ -13,14 +14,28 @@ export NCCL_PROTO=SIMPLE
|
||||
|
||||
exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
|
||||
-m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
|
||||
--spec-draft-model /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf \
|
||||
--spec-draft-device CUDA2 \
|
||||
--spec-draft-ngl all \
|
||||
--spec-type draft-mtp \
|
||||
--spec-draft-n-max 3 \
|
||||
--spec-draft-p-min 0 \
|
||||
--mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \
|
||||
--mmproj-device CUDA2 \
|
||||
--image-min-tokens 1024 \
|
||||
--spec-type draft-mtp \
|
||||
--spec-draft-n-max 2 \
|
||||
--temp 1.0 \
|
||||
--top-k 20 \
|
||||
--top-p 0.95 \
|
||||
--min-p 0 \
|
||||
--presence-penalty 0 \
|
||||
--repeat-penalty 1.0 \
|
||||
--reasoning on \
|
||||
--reasoning-effort xhigh \
|
||||
--reasoning-preserve \
|
||||
--reasoning-format deepseek \
|
||||
--numa split \
|
||||
-ngl 99 \
|
||||
-c 262144 \
|
||||
-c 204800 \
|
||||
--parallel 1 \
|
||||
--slot-save-path /var/cache/llama-slots \
|
||||
--cache-ram 16384 \
|
||||
|
||||
Reference in New Issue
Block a user