motif64

m64_fbfd8ff7225c / sensenova_u1_server.py

205 lines, 9.6 KBraw
1"""OpenAI-compatible image server for SenseNova-U1.5-8B-MoT.23sensenova/SenseNova-U1.5-8B-MoT (Apache-2.0, OpenSenseNova / SenseTime) is a native4unified multimodal model on the NEO-unify architecture: ONE checkpoint does5text-to-image, image editing and visual question answering. Internally it is a Qwen3-8B6LLM backbone with MoT — Mixture of Transformers — generation branches (the *_mot_gen7weights), and it is a PIXEL-SPACE generator, so there is no separate VAE to load.89Apache-2.0 matters here: most of the strong image models on this rig are10non-commercial (qwen-image-2.1 is Qwen Research, tinydit-256 is CC BY-NC). This one11and Ming-Image are the two that can back something shipped.1213WHY THE VENDOR REPO AND NOT ComfyUI: ComfyUI master does carry a SenseNovaU15 class,14and its detection keys are the official checkpoint's own key names, so a merged15single-file repack does load there. We deliberately run the vendor's own inference16code instead — it is the path the model was tested on, it keeps us on official17Apache-2.0 weights with no community repack in the trust chain, and it gives us18think_mode and the official 8-step LoRA, neither of which the ComfyUI path exposes.19The code is baked into an image (machines/gx10/docker/Dockerfile.sensenova-u1) rather20than pip-installed at boot, because sensenova_u1 must be importable before transformers21sees the checkpoint: it self-registers NEOChatModel at import time, which is what lets22this run without trust_remote_code.2324Generation defaults are hardcoded in the request models below and echoed per call under25"params_used"; query them via GET /v1/config.26"""27import base64, io, os, threading, time28from typing import Optional2930import torch31from fastapi import FastAPI, HTTPException32from fastapi.responses import JSONResponse33from PIL import Image34from pydantic import BaseModel, Field35import uvicorn3637import sensenova_u1  # noqa: F401  — registers NEOChatModel with transformers on import38from sensenova_u1.utils import load_and_merge_lora_weight_from_safetensors3940SERVED_MODEL_ID = os.environ.get("SERVED_MODEL_ID", "sensenova-u1.5-8b-mot")41PORT = int(os.environ.get("PORT", "9204"))42MODEL_DIR = os.environ.get("MODEL_DIR", "/model")43LORA_PATH = os.environ.get("SENSENOVA_LORA", "").strip()44VRAM_MODE = os.environ.get("SENSENOVA_VRAM_MODE", "")45DEVICE_MAP = os.environ.get("SENSENOVA_DEVICE_MAP", "").strip() or None4647# The vendor's trained resolution set. Anything else triggers their own48# "outside the trained resolution set" warning; we expose the buckets by name so49# callers do not have to memorise the pixel pairs.50SUPPORTED_RESOLUTIONS = {51    "1:1": (2048, 2048), "16:9": (2720, 1536), "9:16": (1536, 2720),52    "3:2": (2496, 1664), "2:3": (1664, 2496), "4:3": (2368, 1760),53    "3:4": (1760, 2368), "1:2": (1440, 2880), "2:1": (2880, 1440),54    "1:3": (1152, 3456), "3:1": (3456, 1152),55}5657# With the official 8-step distillation LoRA merged, the step count and CFG both58# change — a few-step model sampled at 50 steps / cfg 4 is wrong, not just slow.59LORA_DEFAULTS = {"num_inference_steps": 8, "guidance_scale": 1.0}60BASE_DEFAULTS = {"num_inference_steps": 50, "guidance_scale": 4.0}6162app = FastAPI()63_lock = threading.Lock()64_state = {}656667def _defaults():68    return LORA_DEFAULTS if _state.get("lora_merged") else BASE_DEFAULTS697071def _load():72    t0 = time.time()73    # Imported lazily so the module can be read without the vendor tree present.74    import sys75    sys.path.insert(0, "/opt/sensenova-u1/examples/t2i")76    from inference import SenseNovaU1T2I  # vendor's engine wrapper7778    kwargs = dict(model_path=MODEL_DIR, device="cuda", dtype=torch.bfloat16)79    if VRAM_MODE:80        kwargs["vram_mode"] = VRAM_MODE81    if DEVICE_MAP:82        kwargs["device_map"] = DEVICE_MAP83    engine = SenseNovaU1T2I(**kwargs)8485    lora_merged = False86    if LORA_PATH:87        if not os.path.exists(LORA_PATH):88            print(f"WARNING: SENSENOVA_LORA={LORA_PATH} does not exist; running full-step.",89                  flush=True)90        else:91            print(f"merging LoRA {LORA_PATH}", flush=True)92            engine.model = load_and_merge_lora_weight_from_safetensors(engine.model, LORA_PATH)93            lora_merged = True9495    _state.update(engine=engine, lora_merged=lora_merged)96    print(f"SenseNova-U1.5 loaded in {time.time()-t0:.1f}s | lora_merged={lora_merged}",97          flush=True)9899100class ImageRequest(BaseModel):101    prompt: str = Field(..., description="Text prompt. Direct natural language works for simple asks; for complex layouts the vendor recommends prompt enhancement upstream and spelling out what must stay fixed.")102    n: int = Field(1, description="Number of images (vendor batch_size).")103    size: str = Field("1:1", description="An aspect key from /v1/config -> supported_resolutions ('1:1' = 2048x2048, '16:9' = 2720x1536, up to '3:1' = 3456x1152), or an explicit WxH. Off-bucket sizes are accepted but the vendor warns quality degrades.")104    num_inference_steps: Optional[int] = Field(None, description="Sampler steps. Defaults to 8 when the official 8-step LoRA is merged, else 50. Setting this explicitly overrides either.")105    guidance_scale: Optional[float] = Field(None, description="CFG scale (vendor cfg_scale). Defaults to 1.0 with the 8-step LoRA merged, else 4.0. The card notes oversaturation/over-detail is usually fixed by LOWERING this.")106    cfg_norm: str = Field("none", description="Vendor cfg_norm strategy.")107    timestep_shift: float = Field(3.0, description="Flow timestep shift. Vendor default 3.0.")108    think_mode: bool = Field(False, description="Run the model's own reasoning pass before generating; the <think> text is returned alongside the image. Slower, and intended for complex or heavily constrained prompts.")109    seed: Optional[int] = Field(None, description="Random seed. null = random per call.")110    response_format: str = Field("b64_json", description="Only b64_json supported.")111112113def _params_doc(model):114    out = {}115    for name, info in model.model_fields.items():116        d = info.default117        out[name] = {"default": None if d is ... else d, "description": info.description}118    return out119120121def _resolve_size(size: str):122    if size in SUPPORTED_RESOLUTIONS:123        return SUPPORTED_RESOLUTIONS[size]124    try:125        w, h = (int(x) for x in size.lower().split("x"))126    except ValueError:127        raise HTTPException(400, f"size must be an aspect key {list(SUPPORTED_RESOLUTIONS)} or WxH; got {size}")128    return w, h129130131@app.get("/health")132def health():133    ok = bool(_state)134    return {"status": "ok" if ok else "degraded", "model": SERVED_MODEL_ID,135            "loaded": ok, "lora_merged": _state.get("lora_merged", False)}136137138@app.get("/v1/models")139def models():140    return {"object": "list", "data": [{"id": SERVED_MODEL_ID, "object": "model"}]}141142143@app.get("/v1/config")144def config():145    return {"model": SERVED_MODEL_ID,146            "params": _params_doc(ImageRequest),147            "supported_resolutions": {k: f"{w}x{h}" for k, (w, h) in SUPPORTED_RESOLUTIONS.items()},148            "lora_merged": _state.get("lora_merged", False),149            "active_defaults": _defaults(),150            "notes": [151                "One checkpoint covers t2i, editing and VQA; this server exposes t2i "152                "and editing.",153                "Pixel-space generator — there is no separate VAE.",154                "Apache-2.0, so unlike qwen-image-2.1 and tinydit-256 this one is not "155                "restricted to non-commercial use.",156                "Known rough edges per the card: oversaturation (lower guidance_scale), "157                "dense/small mixed-script text, exact counts in constrained layouts, "158                "small faces and hands, and drift on broad multi-reference edits.",159            ]}160161162@app.post("/v1/images/generations")163def generate_images(req: ImageRequest):164    if not _state:165        raise HTTPException(503, "model not loaded")166    w, h = _resolve_size(req.size)167    d = _defaults()168    steps = req.num_inference_steps if req.num_inference_steps is not None else d["num_inference_steps"]169    cfg = req.guidance_scale if req.guidance_scale is not None else d["guidance_scale"]170    seed = req.seed if req.seed is not None else int(time.time() * 1000) % (2**31)171172    t0 = time.time()173    # One engine, one CUDA context; FastAPI runs `def` endpoints in a threadpool, so174    # concurrent calls would interleave inside the sampler. Serialize and report the wait.175    with _lock:176        queued = time.time() - t0177        images = _state["engine"].generate(178            req.prompt, image_size=(w, h), cfg_scale=cfg, cfg_norm=req.cfg_norm,179            timestep_shift=req.timestep_shift, num_steps=steps,180            batch_size=req.n, seed=seed, think_mode=req.think_mode)181        think = _state["engine"].last_think_text if req.think_mode else ""182183    data = []184    for im in images:185        buf = io.BytesIO(); im.save(buf, format="PNG")186        data.append({"b64_json": base64.b64encode(buf.getvalue()).decode()})187188    print(f"Generated {len(data)} image(s) at {w}x{h} in {time.time()-t0:.1f}s "189          f"(steps={steps} cfg={cfg})", flush=True)190    body = {"created": int(time.time()), "data": data,191            "params_used": {"n": req.n, "size": f"{w}x{h}", "num_inference_steps": steps,192                            "guidance_scale": cfg, "cfg_norm": req.cfg_norm,193                            "timestep_shift": req.timestep_shift,194                            "think_mode": req.think_mode, "seed": seed,195                            "lora_merged": _state.get("lora_merged", False),196                            "queued_s": round(queued, 3)}}197    if think:198        body["think"] = think199    return JSONResponse(body)200201202if __name__ == "__main__":203    print(f"SenseNova-U1.5 server on :{PORT}", flush=True)204    _load()205    uvicorn.run(app, host="0.0.0.0", port=PORT)