"""qwen38-flash-next-llamacpp-gb10: puts llama-server (127.0.0.1:8080) on $M64_PORT (stdlib only). llama-server listens only on loopback without a key; start.sh starts this once it reports healthy, and stops the capsule if either exits. Requests are relayed as they are, with "model" set to the one llama-server serves; replies and /v1/models name the capsule's model (MODEL) instead, and streaming responses (SSE) are relayed line by line. GET /health is 200 while llama-server answers /v1/models, else 503. """ import json import os import sys import urllib.error import urllib.request from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer PORT = int(os.environ.get("M64_PORT", "9000")) UPSTREAM = "http://127.0.0.1:8080" MAX_BODY = 64 * 1024 * 1024 TIMEOUT_S = 1800 # between two bytes from llama-server (a long prompt is fine) CHUNK = 16 * 1024 MODEL = os.environ.get("M64_MODEL_NAME", "qwen3.8-flash-next-iq3_s-llamacpp") # model_name in capsule.toml # Loopback only: the m64 proxy variables must not apply to it. OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) def upstream_model(): """The id llama-server serves, or None while it does not answer.""" try: with OPENER.open(UPSTREAM + "/v1/models", timeout=5) as resp: return json.load(resp)["data"][0]["id"] except (OSError, ValueError, KeyError, IndexError): return None class Handler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" server_version = "qwen38-flash-next-llamacpp-capsule" def log_message(self, fmt, *args): # one short line per request on stderr (not /health) if self.path == "/health": return sys.stderr.write("%s %s\n" % (self.command, fmt % args)) def _json(self, status, obj): data = json.dumps(obj).encode() self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) def _relay(self, req, upstream): try: resp = OPENER.open(req, timeout=TIMEOUT_S) except urllib.error.HTTPError as e: resp = e # relay llama-server's own error status and body except (urllib.error.URLError, OSError) as e: return self._json(502, {"error": f"llama-server unreachable: {type(e).__name__}"}) with resp: self.send_response(resp.status if hasattr(resp, "status") else resp.code) self.send_header("Content-Type", resp.headers.get("Content-Type", "application/json")) self.send_header("Cache-Control", "no-cache") self.send_header("Transfer-Encoding", "chunked") self.end_headers() # llama-server writes '"model": ""' in every reply and SSE chunk; complete lines # are renamed and sent, so the name is never split across two reads. old = b'"model": %s' % json.dumps(upstream).encode() new = b'"model": %s' % json.dumps(MODEL).encode() buf = b"" try: while True: # read1: whatever has arrived (up to CHUNK), without waiting to fill it. data = resp.read1(CHUNK) if hasattr(resp, "read1") else resp.read(CHUNK) buf += data cut = len(buf) if not data else buf.rfind(b"\n") + 1 if cut: out = buf[:cut].replace(old, new) buf = buf[cut:] self.wfile.write(b"%x\r\n%s\r\n" % (len(out), out)) self.wfile.flush() if not data: break self.wfile.write(b"0\r\n\r\n") except (BrokenPipeError, ConnectionResetError): self.close_connection = True def do_GET(self): if self.path == "/health": ok = upstream_model() is not None return self._json(200 if ok else 503, {"status": "ok" if ok else "loading"}) if self.path == "/v1/models": try: with OPENER.open(UPSTREAM + "/v1/models", timeout=5) as resp: models = json.load(resp) models["data"][0]["id"] = MODEL except (OSError, ValueError, KeyError, IndexError): return self._json(503, {"error": "llama-server is not serving"}) return self._json(200, models) return self._json(404, {"error": "not found"}) def do_POST(self): if self.path != "/v1/chat/completions": return self._json(404, {"error": "not found"}) try: length = int(self.headers.get("Content-Length") or 0) except ValueError: return self._json(400, {"error": "bad Content-Length"}) if length <= 0 or length > MAX_BODY: return self._json(400 if length <= 0 else 413, {"error": "body missing or too large"}) try: body = json.loads(self.rfile.read(length)) except ValueError: return self._json(400, {"error": "body must be JSON"}) if not isinstance(body, dict): return self._json(400, {"error": "body must be a JSON object"}) model = upstream_model() if model is None: return self._json(503, {"error": "llama-server is not serving"}) body["model"] = model self._relay(urllib.request.Request(UPSTREAM + "/v1/chat/completions", data=json.dumps(body).encode(), method="POST", headers={ "Content-Type": "application/json", "Accept": "text/event-stream" if body.get("stream") is True else "application/json", }), model) def main(): model = upstream_model() server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler) server.daemon_threads = True with open(os.environ["M64_STATUS"], "a") as f: f.write(json.dumps({"phase": "ready", "detail": f"llama-server serving {model}"}) + "\n") server.serve_forever() if __name__ == "__main__": main()