feat: update to Q5_K_P @ 128K ctx, non-blocking proxy sessions, and full multi-GPU benchmark suite

scripts/ollama-proxy.py

@@ -92,39 +92,39 @@ def extract_session_id(headers, payload=None):
 
     return "default"
 
+import traceback
+
 def ensure_session(session_id):
-    """Ensure slot 0 contains the KV cache for session_id, saving/restoring as needed."""
-    global CURRENT_SESSION
-    if not session_id:
-        session_id = "default"
-
-    clean_id = re.sub(r'[^a-zA-Z0-9_\-\.]', '_', str(session_id))[:64]
-
-    with SESSION_LOCK:
-        if clean_id == CURRENT_SESSION:
+    """Ensure slot 0 contains the KV cache for explicit named sessions, safely and non-blockingly."""
+    try:
+        global CURRENT_SESSION
+        if not session_id:
             return
 
-        # 1. Save old session if not default
-        if CURRENT_SESSION and CURRENT_SESSION != "default":
-            save_file = f"{CURRENT_SESSION}.bin"
-            print(f"[ollama-proxy] Saving slot 0 for session '{CURRENT_SESSION}' -> {save_file}", flush=True)
-            _slot_action("save", save_file)
+        clean_id = re.sub(r'[^a-zA-Z0-9_\-\.]', '_', str(session_id))[:64]
 
-        # 2. Restore new session or erase
-        target_file = f"{clean_id}.bin"
-        target_path = os.path.join(SLOT_SAVE_PATH, target_file)
+        # Only manage disk snapshots for explicit user session IDs (ignore transient sys- hashes)
+        if clean_id == "default" or clean_id.startswith("sys-"):
+            return
 
-        if os.path.exists(target_path):
-            print(f"[ollama-proxy] Restoring slot 0 for session '{clean_id}' <- {target_file}", flush=True)
-            res = _slot_action("restore", target_file)
-            if not res:
-                print(f"[ollama-proxy] Restore failed for '{clean_id}', falling back to erase", flush=True)
-                _slot_action("erase")
-        else:
-            print(f"[ollama-proxy] Starting fresh slot for session '{clean_id}'", flush=True)
-            _slot_action("erase")
+        with SESSION_LOCK:
+            if clean_id == CURRENT_SESSION:
+                return
 
-        CURRENT_SESSION = clean_id
+            if CURRENT_SESSION and CURRENT_SESSION != "default" and not CURRENT_SESSION.startswith("sys-"):
+                save_file = f"{CURRENT_SESSION}.bin"
+                print(f"[ollama-proxy] Saving slot 0 for session '{CURRENT_SESSION}' -> {save_file}", flush=True)
+                _slot_action("save", save_file)
+
+            target_file = f"{clean_id}.bin"
+            target_path = os.path.join(SLOT_SAVE_PATH, target_file)
+            if os.path.exists(target_path):
+                print(f"[ollama-proxy] Restoring slot 0 for session '{clean_id}' <- {target_file}", flush=True)
+                _slot_action("restore", target_file)
+
+            CURRENT_SESSION = clean_id
+    except Exception as e:
+        print(f"[ollama-proxy] Warning: ensure_session({session_id}) error (non-fatal): {e}", flush=True)
 
 MODEL_TAGS  = [
     "Qwen3.8-Uncensored:latest",
@@ -1444,6 +1444,8 @@ class Handler(http.server.BaseHTTPRequestHandler):
                 detail = ""
             self._send(e.code, {"error": detail or str(e)})
         except Exception as e:
+            print(f"[ollama-proxy] Exception in POST {path}: {e}", flush=True)
+            traceback.print_exc()
             self._send(500, {"error": f"{type(e).__name__}: {e}"})
 
     def do_DELETE(self):

scripts/start-server.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# llama-server launcher for Qwen3.8-27B on 3x Turing GPUs (RTX 2060 12GB + 2x CMP 50HX 10GB)
+# llama-server launcher for Qwen3.8-27B on Turing GPUs (RTX 2060 12GB + CMP 50HX 10GB)
 # Built with NVIDIA NCCL for hardware-accelerated multi-GPU tensor parallelism.
 # Accelerated with HauhauCS FastMTP 32K draft sidecar and official thinking parameters.
 export LD_LIBRARY_PATH=/opt/llama.cpp-xrip/build-nccl/bin:/opt/minicpm-venv/lib/python3.13/site-packages/nvidia/nccl/lib:/lib/x86_64-linux-gnu
@@ -12,16 +12,20 @@ export NCCL_P2P_DISABLE=0
 export NCCL_ALGO=RING
 export NCCL_PROTO=SIMPLE
 
+# Dynamically target the 12GB RTX 2060 for mmproj and FastMTP draft model
+RTX_DEV=$(/opt/llama.cpp-xrip/build-nccl/bin/llama-cli --list-devices | grep -i "RTX 2060" | awk '{print $1}' | tr -d ':')
+RTX_DEV=${RTX_DEV:-CUDA0}
+
 exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
-  -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q4_K_P.gguf \
+  -m /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q5_K_P.gguf \
   --spec-draft-model /opt/models/gguf/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf \
-  --spec-draft-device CUDA2 \
+  --spec-draft-device "$RTX_DEV" \
   --spec-draft-ngl all \
   --spec-type draft-mtp \
   --spec-draft-n-max 3 \
   --spec-draft-p-min 0 \
   --mmproj /opt/models/gguf/mmproj-Qwen3.8-27B-Uncensored-f16.gguf \
-  --mmproj-device CUDA2 \
+  --mmproj-device "$RTX_DEV" \
   --image-min-tokens 1024 \
   --temp 1.0 \
   --top-k 20 \
@@ -35,17 +39,17 @@ exec /opt/llama.cpp-xrip/build-nccl/bin/llama-server \
   --reasoning-format deepseek \
   --numa split \
   -ngl 99 \
-  -c 204800 \
+  -c 131072 \
   --parallel 1 \
   --slot-save-path /var/cache/llama-slots \
   --cache-ram 16384 \
   --split-mode tensor \
   --flash-attn on \
-  --batch-size 1024 \
-  --ubatch-size 512 \
+  --batch-size 2048 \
+  --ubatch-size 1024 \
   --jinja \
   --threads 12 \
-  --cache-type-k q4_0 \
-  --cache-type-v q4_0 \
+  --cache-type-k q5_0 \
+  --cache-type-v q5_0 \
   --host 0.0.0.0 \
   --port 8080