wmantly/turing-multi-gpu-llm-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(proxy): handle client aborts and broken pipe disconnections gracefully without 500 errors

Browse Source
This commit is contained in:
wmantly
2026-09-16 00:34:14 +00:00
parent f5f8e5bb79
commit aebc82e403
1 changed files with 43 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ollama-proxy.py
+43 -33
View File
@@ -1205,49 +1205,49 @@ class Handler(http.server.BaseHTTPRequestHandler):
return {} return {}
def _send(self, code, obj_or_bytes, ctype="application/json"): def _send(self, code, obj_or_bytes, ctype="application/json"):
if isinstance(obj_or_bytes, bytes):
body = obj_or_bytes
elif isinstance(obj_or_bytes, (dict, list)):
body = json.dumps(obj_or_bytes).encode()
else:
body = str(obj_or_bytes).encode()
self.send_response(code)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(len(body)))
self.send_header("Ollama-Proxy", "llama.cpp")
self._set_cors_headers()
try: try:
if isinstance(obj_or_bytes, bytes):
body = obj_or_bytes
elif isinstance(obj_or_bytes, (dict, list)):
body = json.dumps(obj_or_bytes).encode()
else:
body = str(obj_or_bytes).encode()
self.send_response(code)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(len(body)))
self.send_header("Ollama-Proxy", "llama.cpp")
self._set_cors_headers()
self.end_headers() self.end_headers()
self.wfile.write(body) self.wfile.write(body)
except (BrokenPipeError, ConnectionResetError): except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass pass
def _send_stream(self, gen): def _send_stream(self, gen):
self.send_response(200)
self.send_header("Content-Type", "application/x-ndjson")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "close")
self._set_cors_headers()
self.end_headers()
try: try:
self.send_response(200)
self.send_header("Content-Type", "application/x-ndjson")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "close")
self._set_cors_headers()
self.end_headers()
for line in gen: for line in gen:
self.wfile.write(line.encode() if isinstance(line, str) else line) self.wfile.write(line.encode() if isinstance(line, str) else line)
self.wfile.flush() self.wfile.flush()
except (BrokenPipeError, ConnectionResetError): except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass pass
def _send_sse(self, gen): def _send_sse(self, gen):
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "close")
self._set_cors_headers()
self.end_headers()
try: try:
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "close")
self._set_cors_headers()
self.end_headers()
for chunk in gen: for chunk in gen:
self.wfile.write(chunk.encode() if isinstance(chunk, str) else chunk) self.wfile.write(chunk.encode() if isinstance(chunk, str) else chunk)
self.wfile.flush() self.wfile.flush()
except (BrokenPipeError, ConnectionResetError): except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass pass
def do_OPTIONS(self): def do_OPTIONS(self):
@@ -1435,12 +1435,15 @@ class Handler(http.server.BaseHTTPRequestHandler):
self.send_header("Connection", "close") self.send_header("Connection", "close")
self._set_cors_headers() self._set_cors_headers()
self.end_headers() self.end_headers()
while True: try:
line = r.readline() while True:
if not line: line = r.readline()
break if not line:
self.wfile.write(line) break
self.wfile.flush() self.wfile.write(line)
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
else: else:
raw = r.read() raw = r.read()
self.send_response(200) self.send_response(200)
@@ -1448,11 +1451,18 @@ class Handler(http.server.BaseHTTPRequestHandler):
self.send_header("Content-Length", str(len(raw))) self.send_header("Content-Length", str(len(raw)))
self._set_cors_headers() self._set_cors_headers()
self.end_headers() self.end_headers()
self.wfile.write(raw) try:
self.wfile.write(raw)
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
self._send(e.code, {"error": e.read().decode(errors="replace")}) self._send(e.code, {"error": e.read().decode(errors="replace")})
else: else:
self._send(404, {"error": f"unknown POST path {path}"}) self._send(404, {"error": f"unknown POST path {path}"})
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
try: try:
detail = e.read().decode(errors="replace") detail = e.read().decode(errors="replace")