feat(cache): enable radix prefix sharing, chunk cache reuse, and 32GB disk-backed slot persistence

2026-09-15 17:03:08 +00:00
parent ae7c1e24e3
commit f5f8e5bb79
1 changed files with 5 additions and 1 deletions
@@ -41,8 +41,12 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
  -ngl 99 \
  -c 131072 \
  --parallel 1 \
+  --cache-prompt \
+  --cache-reuse 64 \
+  --cache-idle-slots \
+  --slot-prompt-similarity 0.10 \
+  --cache-ram 32768 \
  --slot-save-path /var/cache/llama-slots \
-  --cache-ram 16384 \
  --split-mode tensor \
  --flash-attn on \
  --batch-size 2048 \