wmantly/turing-multi-gpu-llm-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: update to Q5_K_P @ 128K ctx, non-blocking proxy sessions, and full multi-GPU benchmark suite

Browse Source
This commit is contained in:
wmantly
2026-09-13 02:14:25 +00:00
parent cfeac35ad6
commit cd989b3e70
5 changed files with 406 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ollama-proxy.py
+29 -27
View File
@@ -92,39 +92,39 @@ def extract_session_id(headers, payload=None):
return "default"
import traceback
def ensure_session(session_id):
"""Ensure slot 0 contains the KV cache for session_id, saving/restoring as needed."""
global CURRENT_SESSION
if not session_id:
session_id = "default"
clean_id = re.sub(r'[^a-zA-Z0-9_\-\.]', '_', str(session_id))[:64]
with SESSION_LOCK:
if clean_id == CURRENT_SESSION:
"""Ensure slot 0 contains the KV cache for explicit named sessions, safely and non-blockingly."""
try:
global CURRENT_SESSION
if not session_id:
return
# 1. Save old session if not default
if CURRENT_SESSION and CURRENT_SESSION != "default":
save_file = f"{CURRENT_SESSION}.bin"
print(f"[ollama-proxy] Saving slot 0 for session '{CURRENT_SESSION}' -> {save_file}", flush=True)
_slot_action("save", save_file)
clean_id = re.sub(r'[^a-zA-Z0-9_\-\.]', '_', str(session_id))[:64]
# 2. Restore new session or erase
target_file = f"{clean_id}.bin"
target_path = os.path.join(SLOT_SAVE_PATH, target_file)
# Only manage disk snapshots for explicit user session IDs (ignore transient sys- hashes)
if clean_id == "default" or clean_id.startswith("sys-"):
return
if os.path.exists(target_path):
print(f"[ollama-proxy] Restoring slot 0 for session '{clean_id}' <- {target_file}", flush=True)
res = _slot_action("restore", target_file)
if not res:
print(f"[ollama-proxy] Restore failed for '{clean_id}', falling back to erase", flush=True)
_slot_action("erase")
else:
print(f"[ollama-proxy] Starting fresh slot for session '{clean_id}'", flush=True)
_slot_action("erase")
with SESSION_LOCK:
if clean_id == CURRENT_SESSION:
return
CURRENT_SESSION = clean_id
if CURRENT_SESSION and CURRENT_SESSION != "default" and not CURRENT_SESSION.startswith("sys-"):
save_file = f"{CURRENT_SESSION}.bin"
print(f"[ollama-proxy] Saving slot 0 for session '{CURRENT_SESSION}' -> {save_file}", flush=True)
_slot_action("save", save_file)
target_file = f"{clean_id}.bin"
target_path = os.path.join(SLOT_SAVE_PATH, target_file)
if os.path.exists(target_path):
print(f"[ollama-proxy] Restoring slot 0 for session '{clean_id}' <- {target_file}", flush=True)
_slot_action("restore", target_file)
CURRENT_SESSION = clean_id
except Exception as e:
print(f"[ollama-proxy] Warning: ensure_session({session_id}) error (non-fatal): {e}", flush=True)
MODEL_TAGS = [
"Qwen3.8-Uncensored:latest",
@@ -1444,6 +1444,8 @@ class Handler(http.server.BaseHTTPRequestHandler):
detail = ""
self._send(e.code, {"error": detail or str(e)})
except Exception as e:
print(f"[ollama-proxy] Exception in POST {path}: {e}", flush=True)
traceback.print_exc()
self._send(500, {"error": f"{type(e).__name__}: {e}"})
def do_DELETE(self):