motif64

m64_ef24b3f62fa7 / proxy.py

139 lines, 6.0 KBraw
1"""qwen38-flash-next-llamacpp-gb10: puts llama-server (127.0.0.1:8080) on $M64_PORT (stdlib only).23llama-server listens only on loopback without a key; start.sh starts this once it reports healthy,4and stops the capsule if either exits. Requests are relayed as they are, with "model" set to5the one llama-server serves; replies and /v1/models name the capsule's model (MODEL) instead, and6streaming responses (SSE) are relayed line by line. GET /health is 200 while llama-server answers7/v1/models, else 503.8"""910import json11import os12import sys13import urllib.error14import urllib.request15from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer1617PORT = int(os.environ.get("M64_PORT", "9000"))18UPSTREAM = "http://127.0.0.1:8080"19MAX_BODY = 64 * 1024 * 102420TIMEOUT_S = 1800  # between two bytes from llama-server (a long prompt is fine)21CHUNK = 16 * 102422MODEL = os.environ.get("M64_MODEL_NAME", "qwen3.8-flash-next-iq3_s-llamacpp")  # model_name in capsule.toml2324# Loopback only: the m64 proxy variables must not apply to it.25OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))262728def upstream_model():29    """The id llama-server serves, or None while it does not answer."""30    try:31        with OPENER.open(UPSTREAM + "/v1/models", timeout=5) as resp:32            return json.load(resp)["data"][0]["id"]33    except (OSError, ValueError, KeyError, IndexError):34        return None353637class Handler(BaseHTTPRequestHandler):38    protocol_version = "HTTP/1.1"39    server_version = "qwen38-flash-next-llamacpp-capsule"4041    def log_message(self, fmt, *args):  # one short line per request on stderr (not /health)42        if self.path == "/health":43            return44        sys.stderr.write("%s %s\n" % (self.command, fmt % args))4546    def _json(self, status, obj):47        data = json.dumps(obj).encode()48        self.send_response(status)49        self.send_header("Content-Type", "application/json")50        self.send_header("Content-Length", str(len(data)))51        self.end_headers()52        self.wfile.write(data)5354    def _relay(self, req, upstream):55        try:56            resp = OPENER.open(req, timeout=TIMEOUT_S)57        except urllib.error.HTTPError as e:58            resp = e  # relay llama-server's own error status and body59        except (urllib.error.URLError, OSError) as e:60            return self._json(502, {"error": f"llama-server unreachable: {type(e).__name__}"})61        with resp:62            self.send_response(resp.status if hasattr(resp, "status") else resp.code)63            self.send_header("Content-Type", resp.headers.get("Content-Type", "application/json"))64            self.send_header("Cache-Control", "no-cache")65            self.send_header("Transfer-Encoding", "chunked")66            self.end_headers()67            # llama-server writes '"model": "<its name>"' in every reply and SSE chunk; complete lines68            # are renamed and sent, so the name is never split across two reads.69            old = b'"model": %s' % json.dumps(upstream).encode()70            new = b'"model": %s' % json.dumps(MODEL).encode()71            buf = b""72            try:73                while True:74                    # read1: whatever has arrived (up to CHUNK), without waiting to fill it.75                    data = resp.read1(CHUNK) if hasattr(resp, "read1") else resp.read(CHUNK)76                    buf += data77                    cut = len(buf) if not data else buf.rfind(b"\n") + 178                    if cut:79                        out = buf[:cut].replace(old, new)80                        buf = buf[cut:]81                        self.wfile.write(b"%x\r\n%s\r\n" % (len(out), out))82                        self.wfile.flush()83                    if not data:84                        break85                self.wfile.write(b"0\r\n\r\n")86            except (BrokenPipeError, ConnectionResetError):87                self.close_connection = True8889    def do_GET(self):90        if self.path == "/health":91            ok = upstream_model() is not None92            return self._json(200 if ok else 503, {"status": "ok" if ok else "loading"})93        if self.path == "/v1/models":94            try:95                with OPENER.open(UPSTREAM + "/v1/models", timeout=5) as resp:96                    models = json.load(resp)97                models["data"][0]["id"] = MODEL98            except (OSError, ValueError, KeyError, IndexError):99                return self._json(503, {"error": "llama-server is not serving"})100            return self._json(200, models)101        return self._json(404, {"error": "not found"})102103    def do_POST(self):104        if self.path != "/v1/chat/completions":105            return self._json(404, {"error": "not found"})106        try:107            length = int(self.headers.get("Content-Length") or 0)108        except ValueError:109            return self._json(400, {"error": "bad Content-Length"})110        if length <= 0 or length > MAX_BODY:111            return self._json(400 if length <= 0 else 413, {"error": "body missing or too large"})112        try:113            body = json.loads(self.rfile.read(length))114        except ValueError:115            return self._json(400, {"error": "body must be JSON"})116        if not isinstance(body, dict):117            return self._json(400, {"error": "body must be a JSON object"})118        model = upstream_model()119        if model is None:120            return self._json(503, {"error": "llama-server is not serving"})121        body["model"] = model122        self._relay(urllib.request.Request(UPSTREAM + "/v1/chat/completions", data=json.dumps(body).encode(),123                                           method="POST", headers={124            "Content-Type": "application/json",125            "Accept": "text/event-stream" if body.get("stream") is True else "application/json",126        }), model)127128129def main():130    model = upstream_model()131    server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler)132    server.daemon_threads = True133    with open(os.environ["M64_STATUS"], "a") as f:134        f.write(json.dumps({"phase": "ready", "detail": f"llama-server serving {model}"}) + "\n")135    server.serve_forever()136137138if __name__ == "__main__":139    main()