feat: implement session-managed persistent KV cache architecture with slot persistence and management API

This commit is contained in:
wmantly committed 2026-09-04 02:53:54 +00:00
1 parent 5daafe24e8
commit 9ab0583cab
4 files changed
+244 -3

No files matched your search

+1 -1
View File
@@ -19,7 +19,7 @@ TimeoutStopSec=60
# Basic hardening (server only listens on :8080, reads models read-only)
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=
ReadWritePaths=/var/cache/llama-slots /tmp
NoNewPrivileges=true
PrivateTmp=true
# Do not cap memory/CPU; inference needs all of it