feat: implement session-managed persistent KV cache architecture with slot persistence and management API

2026-09-04 02:53:54 +00:00
parent 5daafe24e8
commit 9ab0583cab
4 changed files with 244 additions and 3 deletions
@@ -19,6 +19,8 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
  -ngl 99 \
  -c 262144 \
  --parallel 1 \
+  --slot-save-path /var/cache/llama-slots \
+  --cache-ram 16384 \
  --split-mode tensor \
  --flash-attn on \
  --batch-size 1024 \