diff --git a/scripts/start-server.sh b/scripts/start-server.sh index 410c4fe..90f4571 100755 --- a/scripts/start-server.sh +++ b/scripts/start-server.sh @@ -41,8 +41,12 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \ -ngl 99 \ -c 131072 \ --parallel 1 \ + --cache-prompt \ + --cache-reuse 64 \ + --cache-idle-slots \ + --slot-prompt-similarity 0.10 \ + --cache-ram 32768 \ --slot-save-path /var/cache/llama-slots \ - --cache-ram 16384 \ --split-mode tensor \ --flash-attn on \ --batch-size 2048 \