wmantly/turing-multi-gpu-llm-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(proxy): dynamically report exact 128K context window (131072) and Q5_K_P quantization across all API endpoints

Browse Source
This commit is contained in:
wmantly
2026-09-18 02:10:00 +00:00
parent ff29e72442
commit 7269a96313
1 changed files with 40 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ollama-proxy.py
+40 -22
View File
@@ -38,10 +38,23 @@ LLAMA_BASE = "http://127.0.0.1:8080" # llama.cpp server root
MODEL_NAME = "Qwen3.8-Uncensored" # primary name
BACKEND_MODEL = MODEL_NAME # what we send llama.cpp
VERSION = "0.5.4" # fake ollama version
CTX_SIZE = 262144
MAX_OUTPUT = 131072
CTX_SIZE = 131072
MAX_OUTPUT = 65536
KEEP_ALIVE = 300
def get_ctx_size():
"""Dynamically get the exact context size configured on the llama-server backend."""
try:
req = urllib.request.Request(f"{LLAMA_BASE}/props", headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=2) as r:
props = json.loads(r.read().decode())
n = props.get("default_generation_settings", {}).get("n_ctx")
if n and isinstance(n, int) and n > 0:
return n
except Exception:
pass
return CTX_SIZE
# Session Management & Slot Persistence
SLOT_SAVE_PATH = "/var/cache/llama-slots"
SESSION_LOCK = threading.Lock()
@@ -328,19 +341,20 @@ def ollama_version():
return {"version": VERSION}
def _model_entry(tag):
ctx = get_ctx_size()
return {
"name": tag,
"model": tag,
"modified_at": now_iso(),
"size": 17923393664,
"size": 19780000000,
"digest": "sha256:" + "0" * 64,
"details": {
"format": "gguf",
"family": "qwen35",
"families": ["qwen35", "qwen2", "clip"],
"parameter_size": "27.0B",
"quantization_level": "Q4_K_P",
"context_length": CTX_SIZE,
"quantization_level": "Q5_K_P",
"context_length": ctx,
},
"capabilities": ["completion", "chat", "tools", "thinking", "vision", "multimodal"],
}
@@ -349,35 +363,37 @@ def ollama_tags():
return {"models": [_model_entry(tag) for tag in MODEL_TAGS]}
def ollama_ps():
ctx = get_ctx_size()
return {
"models": [
{
"name": "Qwen3.8-Uncensored:latest",
"model": "Qwen3.8-Uncensored:latest",
"size": 17923393664,
"size": 19780000000,
"digest": "sha256:" + "0" * 64,
"details": {
"format": "gguf",
"family": "qwen35",
"families": ["qwen35", "qwen2", "clip"],
"parameter_size": "27.0B",
"quantization_level": "Q4_K_P",
"context_length": CTX_SIZE,
"quantization_level": "Q5_K_P",
"context_length": ctx,
},
"expires_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time() + 86400)) + ".000000Z",
"size_vram": 17923393664,
"size_vram": 19780000000,
"processors": 3,
"context_length": CTX_SIZE,
"context_length": ctx,
}
]
}
def ollama_show(payload=None):
req_model = (payload.get("name") or payload.get("model") or MODEL_NAME) if payload else MODEL_NAME
ctx = get_ctx_size()
return {
"license": "Apache-2.0",
"modelfile": f"FROM {req_model}\nPARAMETER temperature 0.7\nPARAMETER top_p 0.95\nPARAMETER num_ctx {CTX_SIZE}",
"parameters": f"temperature 0.7\ntop_p 0.95\nnum_ctx {CTX_SIZE}",
"modelfile": f"FROM {req_model}\nPARAMETER temperature 0.7\nPARAMETER top_p 0.95\nPARAMETER num_ctx {ctx}",
"parameters": f"temperature 0.7\ntop_p 0.95\nnum_ctx {ctx}",
"template": "{{ if .System }}<|im_start|>system\n{{ .System }}<|im_end|>\n{{ end }}{{ if .Prompt }}<|im_start|>user\n{{ .Prompt }}<|im_end|>\n{{ end }}<|im_start|>assistant\n{{ .Response }}<|im_end|>",
"system": "",
"details": {
@@ -385,15 +401,15 @@ def ollama_show(payload=None):
"family": "qwen35",
"families": ["qwen35", "qwen2", "clip"],
"parameter_size": "27.0B",
"quantization_level": "Q4_K_P",
"context_length": CTX_SIZE,
"quantization_level": "Q5_K_P",
"context_length": ctx,
},
"model_info": {
"general.architecture": "qwen35",
"general.name": req_model,
"general.parameter_count": 27320697856,
"general.quantization_version": 2,
"qwen35.context_length": CTX_SIZE,
"qwen35.context_length": ctx,
"qwen35.attention.head_count": 32,
"qwen35.attention.head_count_kv": 4,
"qwen35.max_output_tokens": MAX_OUTPUT,
@@ -411,12 +427,13 @@ def ollama_status():
return {"cloud": {"disabled": True, "source": ""}}
def ollama_recommendations():
ctx = get_ctx_size()
return {
"recommendations": [
{
"model": tag,
"description": "Qwen3.8-27B Uncensored (local, 256K context)",
"context_length": CTX_SIZE,
"description": "Qwen3.8-27B Uncensored (local, 128K context)",
"context_length": ctx,
"max_output_tokens": MAX_OUTPUT,
}
for tag in MODEL_TAGS[:2]
@@ -1303,6 +1320,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
})
self._send(200, {"sessions": sorted(sessions, key=lambda s: s["size_bytes"], reverse=True)})
elif path in ("/v1/models", "/models"):
ctx = get_ctx_size()
self._send(200, {
"object": "list",
"data": [
@@ -1311,17 +1329,17 @@ class Handler(http.server.BaseHTTPRequestHandler):
"object": "model",
"owned_by": "local",
"created": int(time.time()),
"context_length": CTX_SIZE,
"max_context_length": CTX_SIZE,
"context_length": ctx,
"max_context_length": ctx,
"max_tokens": MAX_OUTPUT,
"max_output_tokens": MAX_OUTPUT,
"max_model_len": CTX_SIZE,
"max_model_len": ctx,
"architecture": {
"input_modalities": ["text", "image"],
"output_modalities": ["text"],
},
"meta": {
"n_ctx": CTX_SIZE,
"n_ctx": ctx,
},
"input_modalities": ["text", "image"],
"modalities": ["text", "image"],
@@ -1336,7 +1354,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
with urllib.request.urlopen(f"{LLAMA_BASE}/props", timeout=5) as r:
props = json.loads(r.read().decode())
except Exception:
props = {"n_ctx": CTX_SIZE}
props = {"n_ctx": get_ctx_size()}
props.setdefault("modalities", {})
if isinstance(props["modalities"], dict):
props["modalities"]["vision"] = True