m64_ef24b3f62fa7 / proxy.py
139 lines, 6.0 KBraw
1"""qwen38-flash-next-llamacpp-gb10: puts llama-server (127.0.0.1:8080) on $M64_PORT (stdlib only).23llama-server listens only on loopback without a key; start.sh starts this once it reports healthy,4and stops the capsule if either exits. Requests are relayed as they are, with "model" set to5the one llama-server serves; replies and /v1/models name the capsule's model (MODEL) instead, and6streaming responses (SSE) are relayed line by line. GET /health is 200 while llama-server answers7/v1/models, else 503.8"""910import json11import os12import sys13import urllib.error14import urllib.request15from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer1617PORT = int(os.environ.get("M64_PORT", "9000"))18UPSTREAM = "http://127.0.0.1:8080"19MAX_BODY = 64 * 1024 * 102420TIMEOUT_S = 1800 # between two bytes from llama-server (a long prompt is fine)21CHUNK = 16 * 102422MODEL = os.environ.get("M64_MODEL_NAME", "qwen3.8-flash-next-iq3_s-llamacpp") # model_name in capsule.toml2324# Loopback only: the m64 proxy variables must not apply to it.25OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))262728def upstream_model():29 """The id llama-server serves, or None while it does not answer."""30 try:31 with OPENER.open(UPSTREAM + "/v1/models", timeout=5) as resp:32 return json.load(resp)["data"][0]["id"]33 except (OSError, ValueError, KeyError, IndexError):34 return None353637class Handler(BaseHTTPRequestHandler):38 protocol_version = "HTTP/1.1"39 server_version = "qwen38-flash-next-llamacpp-capsule"4041 def log_message(self, fmt, *args): # one short line per request on stderr (not /health)42 if self.path == "/health":43 return44 sys.stderr.write("%s %s\n" % (self.command, fmt % args))4546 def _json(self, status, obj):47 data = json.dumps(obj).encode()48 self.send_response(status)49 self.send_header("Content-Type", "application/json")50 self.send_header("Content-Length", str(len(data)))51 self.end_headers()52 self.wfile.write(data)5354 def _relay(self, req, upstream):55 try:56 resp = OPENER.open(req, timeout=TIMEOUT_S)57 except urllib.error.HTTPError as e:58 resp = e # relay llama-server's own error status and body59 except (urllib.error.URLError, OSError) as e:60 return self._json(502, {"error": f"llama-server unreachable: {type(e).__name__}"})61 with resp:62 self.send_response(resp.status if hasattr(resp, "status") else resp.code)63 self.send_header("Content-Type", resp.headers.get("Content-Type", "application/json"))64 self.send_header("Cache-Control", "no-cache")65 self.send_header("Transfer-Encoding", "chunked")66 self.end_headers()67 # llama-server writes '"model": "<its name>"' in every reply and SSE chunk; complete lines68 # are renamed and sent, so the name is never split across two reads.69 old = b'"model": %s' % json.dumps(upstream).encode()70 new = b'"model": %s' % json.dumps(MODEL).encode()71 buf = b""72 try:73 while True:74 # read1: whatever has arrived (up to CHUNK), without waiting to fill it.75 data = resp.read1(CHUNK) if hasattr(resp, "read1") else resp.read(CHUNK)76 buf += data77 cut = len(buf) if not data else buf.rfind(b"\n") + 178 if cut:79 out = buf[:cut].replace(old, new)80 buf = buf[cut:]81 self.wfile.write(b"%x\r\n%s\r\n" % (len(out), out))82 self.wfile.flush()83 if not data:84 break85 self.wfile.write(b"0\r\n\r\n")86 except (BrokenPipeError, ConnectionResetError):87 self.close_connection = True8889 def do_GET(self):90 if self.path == "/health":91 ok = upstream_model() is not None92 return self._json(200 if ok else 503, {"status": "ok" if ok else "loading"})93 if self.path == "/v1/models":94 try:95 with OPENER.open(UPSTREAM + "/v1/models", timeout=5) as resp:96 models = json.load(resp)97 models["data"][0]["id"] = MODEL98 except (OSError, ValueError, KeyError, IndexError):99 return self._json(503, {"error": "llama-server is not serving"})100 return self._json(200, models)101 return self._json(404, {"error": "not found"})102103 def do_POST(self):104 if self.path != "/v1/chat/completions":105 return self._json(404, {"error": "not found"})106 try:107 length = int(self.headers.get("Content-Length") or 0)108 except ValueError:109 return self._json(400, {"error": "bad Content-Length"})110 if length <= 0 or length > MAX_BODY:111 return self._json(400 if length <= 0 else 413, {"error": "body missing or too large"})112 try:113 body = json.loads(self.rfile.read(length))114 except ValueError:115 return self._json(400, {"error": "body must be JSON"})116 if not isinstance(body, dict):117 return self._json(400, {"error": "body must be a JSON object"})118 model = upstream_model()119 if model is None:120 return self._json(503, {"error": "llama-server is not serving"})121 body["model"] = model122 self._relay(urllib.request.Request(UPSTREAM + "/v1/chat/completions", data=json.dumps(body).encode(),123 method="POST", headers={124 "Content-Type": "application/json",125 "Accept": "text/event-stream" if body.get("stream") is True else "application/json",126 }), model)127128129def main():130 model = upstream_model()131 server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler)132 server.daemon_threads = True133 with open(os.environ["M64_STATUS"], "a") as f:134 f.write(json.dumps({"phase": "ready", "detail": f"llama-server serving {model}"}) + "\n")135 server.serve_forever()136137138if __name__ == "__main__":139 main()