wmantly/turing-multi-gpu-llm-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(proxy): handle client aborts and broken pipe disconnections gracefully without 500 errors

Browse Source
This commit is contained in:
wmantly
2026-09-16 00:34:14 +00:00
parent f5f8e5bb79
commit aebc82e403
1 changed files with 43 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ollama-proxy.py
+16 -6
View File
@@ -1205,6 +1205,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
return {}
def _send(self, code, obj_or_bytes, ctype="application/json"):
try:
if isinstance(obj_or_bytes, bytes):
body = obj_or_bytes
elif isinstance(obj_or_bytes, (dict, list)):
@@ -1216,38 +1217,37 @@ class Handler(http.server.BaseHTTPRequestHandler):
self.send_header("Content-Length", str(len(body)))
self.send_header("Ollama-Proxy", "llama.cpp")
self._set_cors_headers()
try:
self.end_headers()
self.wfile.write(body)
except (BrokenPipeError, ConnectionResetError):
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
def _send_stream(self, gen):
try:
self.send_response(200)
self.send_header("Content-Type", "application/x-ndjson")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "close")
self._set_cors_headers()
self.end_headers()
try:
for line in gen:
self.wfile.write(line.encode() if isinstance(line, str) else line)
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError):
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
def _send_sse(self, gen):
try:
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "close")
self._set_cors_headers()
self.end_headers()
try:
for chunk in gen:
self.wfile.write(chunk.encode() if isinstance(chunk, str) else chunk)
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError):
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
def do_OPTIONS(self):
@@ -1435,12 +1435,15 @@ class Handler(http.server.BaseHTTPRequestHandler):
self.send_header("Connection", "close")
self._set_cors_headers()
self.end_headers()
try:
while True:
line = r.readline()
if not line:
break
self.wfile.write(line)
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
else:
raw = r.read()
self.send_response(200)
@@ -1448,11 +1451,18 @@ class Handler(http.server.BaseHTTPRequestHandler):
self.send_header("Content-Length", str(len(raw)))
self._set_cors_headers()
self.end_headers()
try:
self.wfile.write(raw)
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
except urllib.error.HTTPError as e:
self._send(e.code, {"error": e.read().decode(errors="replace")})
else:
self._send(404, {"error": f"unknown POST path {path}"})
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
except urllib.error.HTTPError as e:
try:
detail = e.read().decode(errors="replace")