wmantly/turing-multi-gpu-llm-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Perf: Enable native MTP speculative decoding (draft-n-max 2), achieving 38.73 tok/s (+120% speedup) at 64.8% draft acceptance

Browse Source
This commit is contained in:
wmantly
2026-09-02 01:00:57 +00:00
parent d1353e9b0d
commit 4a03c75710
1 changed files with 2 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/start-server.sh
+2 -5
View File
@@ -13,14 +13,11 @@ export NCCL_PROTO=SIMPLE
exec /opt/llama.cpp/build-nccl/bin/llama-server \ exec /opt/llama.cpp/build-nccl/bin/llama-server \
-m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \ -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
--spec-type ngram-simple \ --spec-type draft-mtp \
--spec-ngram-simple-size-n 24 \ --spec-draft-n-max 2 \
--spec-ngram-simple-size-m 4 \
--mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \
-ngl 99 \ -ngl 99 \
-c 262144 \ -c 262144 \
--parallel 1 \ --parallel 1 \
--image-max-tokens 2048 \
--split-mode tensor \ --split-mode tensor \
--flash-attn on \ --flash-attn on \
--batch-size 1024 \ --batch-size 1024 \