diff --git a/scripts/ollama-proxy.py b/scripts/ollama-proxy.py index 64ece16..9a43ac6 100644 --- a/scripts/ollama-proxy.py +++ b/scripts/ollama-proxy.py @@ -38,10 +38,23 @@ LLAMA_BASE = "http://127.0.0.1:8080" # llama.cpp server root MODEL_NAME = "Qwen3.8-Uncensored" # primary name BACKEND_MODEL = MODEL_NAME # what we send llama.cpp VERSION = "0.5.4" # fake ollama version -CTX_SIZE = 262144 -MAX_OUTPUT = 131072 +CTX_SIZE = 131072 +MAX_OUTPUT = 65536 KEEP_ALIVE = 300 +def get_ctx_size(): + """Dynamically get the exact context size configured on the llama-server backend.""" + try: + req = urllib.request.Request(f"{LLAMA_BASE}/props", headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=2) as r: + props = json.loads(r.read().decode()) + n = props.get("default_generation_settings", {}).get("n_ctx") + if n and isinstance(n, int) and n > 0: + return n + except Exception: + pass + return CTX_SIZE + # Session Management & Slot Persistence SLOT_SAVE_PATH = "/var/cache/llama-slots" SESSION_LOCK = threading.Lock() @@ -328,19 +341,20 @@ def ollama_version(): return {"version": VERSION} def _model_entry(tag): + ctx = get_ctx_size() return { "name": tag, "model": tag, "modified_at": now_iso(), - "size": 17923393664, + "size": 19780000000, "digest": "sha256:" + "0" * 64, "details": { "format": "gguf", "family": "qwen35", "families": ["qwen35", "qwen2", "clip"], "parameter_size": "27.0B", - "quantization_level": "Q4_K_P", - "context_length": CTX_SIZE, + "quantization_level": "Q5_K_P", + "context_length": ctx, }, "capabilities": ["completion", "chat", "tools", "thinking", "vision", "multimodal"], } @@ -349,35 +363,37 @@ def ollama_tags(): return {"models": [_model_entry(tag) for tag in MODEL_TAGS]} def ollama_ps(): + ctx = get_ctx_size() return { "models": [ { "name": "Qwen3.8-Uncensored:latest", "model": "Qwen3.8-Uncensored:latest", - "size": 17923393664, + "size": 19780000000, "digest": "sha256:" + "0" * 64, "details": { "format": "gguf", "family": "qwen35", "families": ["qwen35", "qwen2", "clip"], "parameter_size": "27.0B", - "quantization_level": "Q4_K_P", - "context_length": CTX_SIZE, + "quantization_level": "Q5_K_P", + "context_length": ctx, }, "expires_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time() + 86400)) + ".000000Z", - "size_vram": 17923393664, + "size_vram": 19780000000, "processors": 3, - "context_length": CTX_SIZE, + "context_length": ctx, } ] } def ollama_show(payload=None): req_model = (payload.get("name") or payload.get("model") or MODEL_NAME) if payload else MODEL_NAME + ctx = get_ctx_size() return { "license": "Apache-2.0", - "modelfile": f"FROM {req_model}\nPARAMETER temperature 0.7\nPARAMETER top_p 0.95\nPARAMETER num_ctx {CTX_SIZE}", - "parameters": f"temperature 0.7\ntop_p 0.95\nnum_ctx {CTX_SIZE}", + "modelfile": f"FROM {req_model}\nPARAMETER temperature 0.7\nPARAMETER top_p 0.95\nPARAMETER num_ctx {ctx}", + "parameters": f"temperature 0.7\ntop_p 0.95\nnum_ctx {ctx}", "template": "{{ if .System }}<|im_start|>system\n{{ .System }}<|im_end|>\n{{ end }}{{ if .Prompt }}<|im_start|>user\n{{ .Prompt }}<|im_end|>\n{{ end }}<|im_start|>assistant\n{{ .Response }}<|im_end|>", "system": "", "details": { @@ -385,15 +401,15 @@ def ollama_show(payload=None): "family": "qwen35", "families": ["qwen35", "qwen2", "clip"], "parameter_size": "27.0B", - "quantization_level": "Q4_K_P", - "context_length": CTX_SIZE, + "quantization_level": "Q5_K_P", + "context_length": ctx, }, "model_info": { "general.architecture": "qwen35", "general.name": req_model, "general.parameter_count": 27320697856, "general.quantization_version": 2, - "qwen35.context_length": CTX_SIZE, + "qwen35.context_length": ctx, "qwen35.attention.head_count": 32, "qwen35.attention.head_count_kv": 4, "qwen35.max_output_tokens": MAX_OUTPUT, @@ -411,12 +427,13 @@ def ollama_status(): return {"cloud": {"disabled": True, "source": ""}} def ollama_recommendations(): + ctx = get_ctx_size() return { "recommendations": [ { "model": tag, - "description": "Qwen3.8-27B Uncensored (local, 256K context)", - "context_length": CTX_SIZE, + "description": "Qwen3.8-27B Uncensored (local, 128K context)", + "context_length": ctx, "max_output_tokens": MAX_OUTPUT, } for tag in MODEL_TAGS[:2] @@ -1303,6 +1320,7 @@ class Handler(http.server.BaseHTTPRequestHandler): }) self._send(200, {"sessions": sorted(sessions, key=lambda s: s["size_bytes"], reverse=True)}) elif path in ("/v1/models", "/models"): + ctx = get_ctx_size() self._send(200, { "object": "list", "data": [ @@ -1311,17 +1329,17 @@ class Handler(http.server.BaseHTTPRequestHandler): "object": "model", "owned_by": "local", "created": int(time.time()), - "context_length": CTX_SIZE, - "max_context_length": CTX_SIZE, + "context_length": ctx, + "max_context_length": ctx, "max_tokens": MAX_OUTPUT, "max_output_tokens": MAX_OUTPUT, - "max_model_len": CTX_SIZE, + "max_model_len": ctx, "architecture": { "input_modalities": ["text", "image"], "output_modalities": ["text"], }, "meta": { - "n_ctx": CTX_SIZE, + "n_ctx": ctx, }, "input_modalities": ["text", "image"], "modalities": ["text", "image"], @@ -1336,7 +1354,7 @@ class Handler(http.server.BaseHTTPRequestHandler): with urllib.request.urlopen(f"{LLAMA_BASE}/props", timeout=5) as r: props = json.loads(r.read().decode()) except Exception: - props = {"n_ctx": CTX_SIZE} + props = {"n_ctx": get_ctx_size()} props.setdefault("modalities", {}) if isinstance(props["modalities"], dict): props["modalities"]["vision"] = True