wmantly/turing-multi-gpu-llm-server
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(context): bump context window to 192K (196,608 tokens) with q4_0 KV cache

Browse Source
This commit is contained in:
wmantly committed 2026-09-22 01:50:58 +00:00
1 parent 1e98eea3bd
commit de36ba8f6e
2 files changed
+4 -4
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/ollama-proxy.py
+1 -1
View File
@@ -38,7 +38,7 @@ LLAMA_BASE = "http://127.0.0.1:8080" # llama.cpp server root
MODEL_NAME = "Qwen3.8-Uncensored" # primary name MODEL_NAME = "Qwen3.8-Uncensored" # primary name
BACKEND_MODEL = MODEL_NAME # what we send llama.cpp BACKEND_MODEL = MODEL_NAME # what we send llama.cpp
VERSION = "0.5.4" # fake ollama version VERSION = "0.5.4" # fake ollama version
CTX_SIZE = 131072 CTX_SIZE = 196608
MAX_OUTPUT = 65536 MAX_OUTPUT = 65536
KEEP_ALIVE = 300 KEEP_ALIVE = 300
scripts/start-server.sh
+3 -3
View File
@@ -39,7 +39,7 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
--reasoning-format deepseek \ --reasoning-format deepseek \
--numa isolate \ --numa isolate \
-ngl 99 \ -ngl 99 \
-c 131072 \ -c 196608 \
--parallel 1 \ --parallel 1 \
--cache-prompt \ --cache-prompt \
--cache-idle-slots \ --cache-idle-slots \
@@ -52,7 +52,7 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
--ubatch-size 1024 \ --ubatch-size 1024 \
--jinja \ --jinja \
--threads 12 \ --threads 12 \
--cache-type-k q5_0 \ --cache-type-k q4_0 \
--cache-type-v q5_0 \ --cache-type-v q4_0 \
--host 0.0.0.0 \ --host 0.0.0.0 \
--port 8080 --port 8080