feat(context): bump context window to 192K (196,608 tokens) with q4_0 KV cache
This commit is contained in:
1 parent
1e98eea3bd
commit
de36ba8f6e
2 files changed
+4
-4
No files matched your search
@@ -39,7 +39,7 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
|
||||
--reasoning-format deepseek \
|
||||
--numa isolate \
|
||||
-ngl 99 \
|
||||
-c 131072 \
|
||||
-c 196608 \
|
||||
--parallel 1 \
|
||||
--cache-prompt \
|
||||
--cache-idle-slots \
|
||||
@@ -52,7 +52,7 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
|
||||
--ubatch-size 1024 \
|
||||
--jinja \
|
||||
--threads 12 \
|
||||
--cache-type-k q5_0 \
|
||||
--cache-type-v q5_0 \
|
||||
--cache-type-k q4_0 \
|
||||
--cache-type-v q4_0 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8080
|
||||
Reference in new issue
Block a user