Update start-server.sh with validated ngram context-lookup speculation and NCCL pipeline flags

2026-09-01 15:56:43 +00:00
parent 2b4e826437
commit 261631457c
1 changed files with 3 additions and 0 deletions
@@ -13,6 +13,9 @@ export NCCL_PROTO=SIMPLE

 exec /opt/llama.cpp/build-nccl/bin/llama-server \
  -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
+  --spec-type ngram-simple \
+  --spec-ngram-simple-size-n 24 \
+  --spec-ngram-simple-size-m 4 \
  --mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \
  -ngl 99 \
  -c 262144 \