wmantly/turing-multi-gpu-llm-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(proxy): dynamically report exact 128K context window (131072) and Q5_K_P quantization across all API endpoints

Browse Source
This commit is contained in:
wmantly
2026-09-18 02:10:00 +00:00
parent ff29e72442
commit 7269a96313
1 changed files with 40 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ollama-proxy.py
+40 -22
View File
@@ -38,10 +38,23 @@ LLAMA_BASE = "http://127.0.0.1:8080" # llama.cpp server root
MODEL_NAME = "Qwen3.8-Uncensored" # primary name MODEL_NAME = "Qwen3.8-Uncensored" # primary name
BACKEND_MODEL = MODEL_NAME # what we send llama.cpp BACKEND_MODEL = MODEL_NAME # what we send llama.cpp
VERSION = "0.5.4" # fake ollama version VERSION = "0.5.4" # fake ollama version
CTX_SIZE = 262144 CTX_SIZE = 131072
MAX_OUTPUT = 131072 MAX_OUTPUT = 65536
KEEP_ALIVE = 300 KEEP_ALIVE = 300
def get_ctx_size():
"""Dynamically get the exact context size configured on the llama-server backend."""
try:
req = urllib.request.Request(f"{LLAMA_BASE}/props", headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=2) as r:
props = json.loads(r.read().decode())
n = props.get("default_generation_settings", {}).get("n_ctx")
if n and isinstance(n, int) and n > 0:
return n
except Exception:
pass
return CTX_SIZE
# Session Management & Slot Persistence # Session Management & Slot Persistence
SLOT_SAVE_PATH = "/var/cache/llama-slots" SLOT_SAVE_PATH = "/var/cache/llama-slots"
SESSION_LOCK = threading.Lock() SESSION_LOCK = threading.Lock()
@@ -328,19 +341,20 @@ def ollama_version():
return {"version": VERSION} return {"version": VERSION}
def _model_entry(tag): def _model_entry(tag):
ctx = get_ctx_size()
return { return {
"name": tag, "name": tag,
"model": tag, "model": tag,
"modified_at": now_iso(), "modified_at": now_iso(),
"size": 17923393664, "size": 19780000000,
"digest": "sha256:" + "0" * 64, "digest": "sha256:" + "0" * 64,
"details": { "details": {
"format": "gguf", "format": "gguf",
"family": "qwen35", "family": "qwen35",
"families": ["qwen35", "qwen2", "clip"], "families": ["qwen35", "qwen2", "clip"],
"parameter_size": "27.0B", "parameter_size": "27.0B",
"quantization_level": "Q4_K_P", "quantization_level": "Q5_K_P",
"context_length": CTX_SIZE, "context_length": ctx,
}, },
"capabilities": ["completion", "chat", "tools", "thinking", "vision", "multimodal"], "capabilities": ["completion", "chat", "tools", "thinking", "vision", "multimodal"],
} }
@@ -349,35 +363,37 @@ def ollama_tags():
return {"models": [_model_entry(tag) for tag in MODEL_TAGS]} return {"models": [_model_entry(tag) for tag in MODEL_TAGS]}
def ollama_ps(): def ollama_ps():
ctx = get_ctx_size()
return { return {
"models": [ "models": [
{ {
"name": "Qwen3.8-Uncensored:latest", "name": "Qwen3.8-Uncensored:latest",
"model": "Qwen3.8-Uncensored:latest", "model": "Qwen3.8-Uncensored:latest",
"size": 17923393664, "size": 19780000000,
"digest": "sha256:" + "0" * 64, "digest": "sha256:" + "0" * 64,
"details": { "details": {
"format": "gguf", "format": "gguf",
"family": "qwen35", "family": "qwen35",
"families": ["qwen35", "qwen2", "clip"], "families": ["qwen35", "qwen2", "clip"],
"parameter_size": "27.0B", "parameter_size": "27.0B",
"quantization_level": "Q4_K_P", "quantization_level": "Q5_K_P",
"context_length": CTX_SIZE, "context_length": ctx,
}, },
"expires_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time() + 86400)) + ".000000Z", "expires_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time() + 86400)) + ".000000Z",
"size_vram": 17923393664, "size_vram": 19780000000,
"processors": 3, "processors": 3,
"context_length": CTX_SIZE, "context_length": ctx,
} }
] ]
} }
def ollama_show(payload=None): def ollama_show(payload=None):
req_model = (payload.get("name") or payload.get("model") or MODEL_NAME) if payload else MODEL_NAME req_model = (payload.get("name") or payload.get("model") or MODEL_NAME) if payload else MODEL_NAME
ctx = get_ctx_size()
return { return {
"license": "Apache-2.0", "license": "Apache-2.0",
"modelfile": f"FROM {req_model}\nPARAMETER temperature 0.7\nPARAMETER top_p 0.95\nPARAMETER num_ctx {CTX_SIZE}", "modelfile": f"FROM {req_model}\nPARAMETER temperature 0.7\nPARAMETER top_p 0.95\nPARAMETER num_ctx {ctx}",
"parameters": f"temperature 0.7\ntop_p 0.95\nnum_ctx {CTX_SIZE}", "parameters": f"temperature 0.7\ntop_p 0.95\nnum_ctx {ctx}",
"template": "{{ if .System }}<|im_start|>system\n{{ .System }}<|im_end|>\n{{ end }}{{ if .Prompt }}<|im_start|>user\n{{ .Prompt }}<|im_end|>\n{{ end }}<|im_start|>assistant\n{{ .Response }}<|im_end|>", "template": "{{ if .System }}<|im_start|>system\n{{ .System }}<|im_end|>\n{{ end }}{{ if .Prompt }}<|im_start|>user\n{{ .Prompt }}<|im_end|>\n{{ end }}<|im_start|>assistant\n{{ .Response }}<|im_end|>",
"system": "", "system": "",
"details": { "details": {
@@ -385,15 +401,15 @@ def ollama_show(payload=None):
"family": "qwen35", "family": "qwen35",
"families": ["qwen35", "qwen2", "clip"], "families": ["qwen35", "qwen2", "clip"],
"parameter_size": "27.0B", "parameter_size": "27.0B",
"quantization_level": "Q4_K_P", "quantization_level": "Q5_K_P",
"context_length": CTX_SIZE, "context_length": ctx,
}, },
"model_info": { "model_info": {
"general.architecture": "qwen35", "general.architecture": "qwen35",
"general.name": req_model, "general.name": req_model,
"general.parameter_count": 27320697856, "general.parameter_count": 27320697856,
"general.quantization_version": 2, "general.quantization_version": 2,
"qwen35.context_length": CTX_SIZE, "qwen35.context_length": ctx,
"qwen35.attention.head_count": 32, "qwen35.attention.head_count": 32,
"qwen35.attention.head_count_kv": 4, "qwen35.attention.head_count_kv": 4,
"qwen35.max_output_tokens": MAX_OUTPUT, "qwen35.max_output_tokens": MAX_OUTPUT,
@@ -411,12 +427,13 @@ def ollama_status():
return {"cloud": {"disabled": True, "source": ""}} return {"cloud": {"disabled": True, "source": ""}}
def ollama_recommendations(): def ollama_recommendations():
ctx = get_ctx_size()
return { return {
"recommendations": [ "recommendations": [
{ {
"model": tag, "model": tag,
"description": "Qwen3.8-27B Uncensored (local, 256K context)", "description": "Qwen3.8-27B Uncensored (local, 128K context)",
"context_length": CTX_SIZE, "context_length": ctx,
"max_output_tokens": MAX_OUTPUT, "max_output_tokens": MAX_OUTPUT,
} }
for tag in MODEL_TAGS[:2] for tag in MODEL_TAGS[:2]
@@ -1303,6 +1320,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
}) })
self._send(200, {"sessions": sorted(sessions, key=lambda s: s["size_bytes"], reverse=True)}) self._send(200, {"sessions": sorted(sessions, key=lambda s: s["size_bytes"], reverse=True)})
elif path in ("/v1/models", "/models"): elif path in ("/v1/models", "/models"):
ctx = get_ctx_size()
self._send(200, { self._send(200, {
"object": "list", "object": "list",
"data": [ "data": [
@@ -1311,17 +1329,17 @@ class Handler(http.server.BaseHTTPRequestHandler):
"object": "model", "object": "model",
"owned_by": "local", "owned_by": "local",
"created": int(time.time()), "created": int(time.time()),
"context_length": CTX_SIZE, "context_length": ctx,
"max_context_length": CTX_SIZE, "max_context_length": ctx,
"max_tokens": MAX_OUTPUT, "max_tokens": MAX_OUTPUT,
"max_output_tokens": MAX_OUTPUT, "max_output_tokens": MAX_OUTPUT,
"max_model_len": CTX_SIZE, "max_model_len": ctx,
"architecture": { "architecture": {
"input_modalities": ["text", "image"], "input_modalities": ["text", "image"],
"output_modalities": ["text"], "output_modalities": ["text"],
}, },
"meta": { "meta": {
"n_ctx": CTX_SIZE, "n_ctx": ctx,
}, },
"input_modalities": ["text", "image"], "input_modalities": ["text", "image"],
"modalities": ["text", "image"], "modalities": ["text", "image"],
@@ -1336,7 +1354,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
with urllib.request.urlopen(f"{LLAMA_BASE}/props", timeout=5) as r: with urllib.request.urlopen(f"{LLAMA_BASE}/props", timeout=5) as r:
props = json.loads(r.read().decode()) props = json.loads(r.read().decode())
except Exception: except Exception:
props = {"n_ctx": CTX_SIZE} props = {"n_ctx": get_ctx_size()}
props.setdefault("modalities", {}) props.setdefault("modalities", {})
if isinstance(props["modalities"], dict): if isinstance(props["modalities"], dict):
props["modalities"]["vision"] = True props["modalities"]["vision"] = True