feat(cache): enable radix prefix sharing, chunk cache reuse, and 32GB disk-backed slot persistence
This commit is contained in:
@@ -41,8 +41,12 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
|
|||||||
-ngl 99 \
|
-ngl 99 \
|
||||||
-c 131072 \
|
-c 131072 \
|
||||||
--parallel 1 \
|
--parallel 1 \
|
||||||
|
--cache-prompt \
|
||||||
|
--cache-reuse 64 \
|
||||||
|
--cache-idle-slots \
|
||||||
|
--slot-prompt-similarity 0.10 \
|
||||||
|
--cache-ram 32768 \
|
||||||
--slot-save-path /var/cache/llama-slots \
|
--slot-save-path /var/cache/llama-slots \
|
||||||
--cache-ram 16384 \
|
|
||||||
--split-mode tensor \
|
--split-mode tensor \
|
||||||
--flash-attn on \
|
--flash-attn on \
|
||||||
--batch-size 2048 \
|
--batch-size 2048 \
|
||||||
|
|||||||
Reference in New Issue
Block a user