From de36ba8f6ee4c886e859cc83816d78dff8a70abc Mon Sep 17 00:00:00 2001 From: wmantly Date: Tue, 22 Sep 2026 01:50:58 +0000 Subject: [PATCH] feat(context): bump context window to 192K (196,608 tokens) with q4_0 KV cache --- scripts/ollama-proxy.py | 2 +- scripts/start-server.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ollama-proxy.py b/scripts/ollama-proxy.py index bfb4206..7948a7c 100644 --- a/scripts/ollama-proxy.py +++ b/scripts/ollama-proxy.py @@ -38,7 +38,7 @@ LLAMA_BASE = "http://127.0.0.1:8080" # llama.cpp server root MODEL_NAME = "Qwen3.8-Uncensored" # primary name BACKEND_MODEL = MODEL_NAME # what we send llama.cpp VERSION = "0.5.4" # fake ollama version -CTX_SIZE = 131072 +CTX_SIZE = 196608 MAX_OUTPUT = 65536 KEEP_ALIVE = 300 diff --git a/scripts/start-server.sh b/scripts/start-server.sh index b882c9b..38be28c 100755 --- a/scripts/start-server.sh +++ b/scripts/start-server.sh @@ -39,7 +39,7 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \ --reasoning-format deepseek \ --numa isolate \ -ngl 99 \ - -c 131072 \ + -c 196608 \ --parallel 1 \ --cache-prompt \ --cache-idle-slots \ @@ -52,7 +52,7 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \ --ubatch-size 1024 \ --jinja \ --threads 12 \ - --cache-type-k q5_0 \ - --cache-type-v q5_0 \ + --cache-type-k q4_0 \ + --cache-type-v q4_0 \ --host 0.0.0.0 \ --port 8080 \ No newline at end of file