motif64

m64_c2cef7905f1b / qwen_image_21_omni_server.py

515 lines, 23.0 KBraw
1"""Qwen-Image-2.1 server (Qwen/Qwen-Image-2.1) — unified T2I + edit + native RGBA.23Adapted for the qwen-image-2.1-uncensored-omni-gb10 capsule from gx10's4qwen_image_21_unified_server.py (as in the qwen-image-2.1-viggle-turbo-gb10 capsule). Changes:5  * QWEN_IMAGE_21_TRANSFORMER: load the transformer from another diffusers folder (here6    ChatR Labs' Omni adapter merged into the base at strength 1.0, BF16) and the rest of the7    pipeline (text encoder, processor, tokenizer, VAE, scheduler) from MODEL_DIR.8  * The defaults (steps, resolution, true CFG, negative prompt) come from env vars, set to9    that model card's recommended values.10  * When true CFG is on and no negative prompt is given, the substitute is a single space,11    not "": per the model card an empty negative prompt disables true CFG.1213One 7B DiT (32 single-stream layers, mixed-granularity attention, prefix KV14cache reuse) does text-to-image, image editing and transparent-layer work.15Text encoder is Qwen3-VL — it reads condition images as vision context, which16is why reference pixels have to be passed through rather than pre-encoded.17VAE is 4-channel, so transparency is native: a prompt that asks for RGBA comes18back as an RGBA PNG, no matting step.1920Pipeline class: `QwenImage21Pipeline` — diffusers **git main only** (not in the210.40.0 release), transformers >= 5.17 for Qwen3VLForConditionalGeneration.2223Endpoints (both OpenAI-compatible in shape, both go through one code path):24  POST /v1/images/generations   text-to-image; `image`/`images` optional25  POST /v1/images/edits         edit/reference mode; image(s) required, up to 1026  GET  /v1/config               resolved defaults + the native aspect table2728Notes that bite:29  * CFG knob is `true_cfg_scale`, not `guidance_scale`, and the pipeline30    silently disables it unless a negative_prompt is present. We fill in "" so31    the knob does what the caller asked; params_used flags when we did.32  * `use_kv_cache` changes the sample, not just the speed — caching re-tiles33    attention, and a 1-ULP difference amplifies over 32 blocks. Both settings34    are equally valid; pin the flag to reproduce an image.35  * Dimensions are floored to vae_scale_factor*2, so params_used echoes the36    size actually rendered, not the size requested.3738Env vars:39  MODEL_DIR                     container path to weights   (default /model)40  PORT                          listen port                 (default 9049)41  SERVED_MODEL_ID               /v1/models id               (default qwen-image-2.1-7b)42  QWEN_IMAGE_21_LOAD_MODE       device | offload | sequential  (default device)43  QWEN_IMAGE_21_VAE_TILING      1 to tile VAE decode        (default 1)44"""45import base6446import gc47import io48import os49import resource50import threading51import time52from typing import Any, List, Optional, Union5354import torch55import uvicorn56from fastapi import FastAPI, HTTPException57from pydantic import BaseModel, Field58from PIL import Image5960MODEL_DIR = os.environ.get("MODEL_DIR", "/model")61PORT = int(os.environ.get("PORT", 9049))62SERVED_MODEL_ID = os.environ.get("SERVED_MODEL_ID", "qwen-image-2.1-7b")6364# How to get 33 GB of weights onto a 119 GB pool that CPU and GPU share:65#   device      low-mem load, then pipe.to("cuda"). Everything resident, fastest.66#   offload     enable_model_cpu_offload() — one component on device at a time67#               (text_encoder->transformer->vae). Lower peak, slower per image.68#   sequential  enable_sequential_cpu_offload() — submodule granularity. Lowest69#               peak, much slower. Last resort.70LOAD_MODE = os.environ.get("QWEN_IMAGE_21_LOAD_MODE", "device").strip().lower()7172# VAE tiling caps the decode transient — but it is OFF by default, because it is73# not free and it is not rare. This VAE tiles whenever the latent exceeds74# tile_sample_min/scale_factor_spatial = 256/16 = 16, and a 1024² image has a75# 64×64 latent, so tiling engages on *every* normal image: the decode becomes a76# ~5×5 grid of tiles (stride 192 -> 64 px overlap) that are blended back77# together, which shows up as seams and faint streaks. Vanilla diffusers does78# not enable it. With ~80 GB spare on this box the decode fits comfortably79# untiled, so turn it on only for a resolution that genuinely will not fit.80VAE_TILING = os.environ.get("QWEN_IMAGE_21_VAE_TILING", "0") not in ("0", "false", "no")8182# Upstream README: 40 steps, and height/width default to output_resolution when83# unset.84#85# 2048, not the pipeline's own 1024 default: every example on the model card and86# its whole aspect-ratio table are at ~2048, and the difference is not subtle.87# Same prompt and seed at 1024 gives a flat, poster-like image with a floating88# sign and mushy reflections; at 2048 the same seed resolves into a89# photorealistic street with physical 3D letters, storefronts and real90# reflections. 1024 sits below the resolution this checkpoint was tuned for, and91# the result reads as "artifacts".92#93# It costs ~5x the time on GB10 (~337 s vs ~67 s at 40 steps), so pass94# size/output_resolution explicitly for fast previews. Quality is the default.95DEFAULT_STEPS = int(os.environ.get("QWEN_IMAGE_21_STEPS", "40"))96DEFAULT_RESOLUTION = int(os.environ.get("QWEN_IMAGE_21_RESOLUTION", "2048"))97DEFAULT_TRUE_CFG = float(os.environ.get("QWEN_IMAGE_21_TRUE_CFG", "1.0"))98# The negative prompt used when true CFG is on and the request has none. A single space:99# an empty one turns true CFG off in the pipeline.100DEFAULT_NEGATIVE = os.environ.get("QWEN_IMAGE_21_NEGATIVE", " ")101# Optional: a diffusers folder holding a replacement transformer (config.json +102# diffusion_pytorch_model.safetensors) for the same architecture.103TRANSFORMER_DIR = os.environ.get("QWEN_IMAGE_21_TRANSFORMER", "").strip()104105# ── Viggle turbo LoRA (optional) ──────────────────────────────────────────────106# Viggle/Qwen-Image-2.1-viggle-turbo is a few-step DISTILLATION LoRA for this exact107# base transformer, so it costs 1.3 GB instead of another 33 GB checkpoint. v0.3 runs108# 6 steps with NO classifier-free guidance; Viggle measure ~5x faster end to end and109# "hard to tell apart" from the 40-step base on most prompts, with the gaps on small110# dense text and complicated edits.111#112# Loading it CHANGES THE SAMPLER CONTRACT, not just the speed: a distilled few-step113# model sampled at 40 steps is wrong, not merely slow, so the defaults below swap too.114# Everything stays overridable per request.115#116# Licensing is unchanged — the LoRA is Qwen Research, same as the base it adapts.117LORA_PATH = os.environ.get("QWEN_IMAGE_21_LORA", "").strip()118# v0.3's distillation was done against a schedule with shift_terminal null; the repo119# ships that scheduler config for exactly this reason. Applied only when the LoRA is.120LORA_SCHEDULER_DIR = os.environ.get("QWEN_IMAGE_21_LORA_SCHEDULER", "").strip()121TURBO_STEPS = int(os.environ.get("QWEN_IMAGE_21_LORA_STEPS", "6"))122123# From the model card — the resolutions the checkpoint was actually trained on.124NATIVE_ASPECT_RATIOS = {125    "1:1": [2048, 2048],126    "4:3": [2400, 1792],127    "3:4": [1792, 2400],128    "3:2": [2528, 1696],129    "2:3": [1696, 2528],130    "16:9": [2752, 1536],131    "9:16": [1536, 2752],132}133134MAX_REFERENCE_IMAGES = 10135136app = FastAPI()137pipe = None138139# One GPU, one pipeline, and a scheduler that keeps its step counter on itself:140# `FlowMatchEulerDiscreteScheduler.step` reads `self.step_index` and indexes141# `self.sigmas[step_index + 1]`, so two generations in flight share one counter142# and it runs off the end —143#   IndexError: index 41 is out of bounds for dimension 0 with size 41144# for a 40-step call (41 sigmas). FastAPI runs `def` endpoints in a threadpool,145# so concurrent requests land here in parallel by default. Serialize them.146# Queueing rather than rejecting: the second caller waits a generation and gets147# an image, which is what a benchmark driver firing off prompts actually wants.148_gpu_lock = threading.Lock()149150151def _mem(tag: str):152    """GB of host RSS + CUDA allocator state. Printed at every load stage so a153    future OOM says where it happened instead of just dying."""154    rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024 ** 2)  # KiB -> GiB155    if torch.cuda.is_available():156        alloc = torch.cuda.memory_allocated() / (1024 ** 3)157        reserved = torch.cuda.memory_reserved() / (1024 ** 3)158        print(f"  [mem] {tag:<22} peak_rss={rss:6.1f} GiB  cuda_alloc={alloc:6.1f} GiB  "159              f"cuda_reserved={reserved:6.1f} GiB", flush=True)160    else:161        print(f"  [mem] {tag:<22} peak_rss={rss:6.1f} GiB", flush=True)162163164LORA_STATE = {"active": False}165166167def _default_steps():168    return TURBO_STEPS if LORA_STATE["active"] else DEFAULT_STEPS169170171def load_model():172    global pipe173    from diffusers import QwenImage21Pipeline174175    print(f"Loading QwenImage21Pipeline from {MODEL_DIR} "176          f"(~33 GB BF16, mode={LOAD_MODE}, first load several min from NFS)...", flush=True)177    _mem("before load")178    t0 = time.time()179180    # low_cpu_mem_usage stays at its default (True). Do NOT pass False here:181    # that path instantiates every module with real weights in the *default*182    # dtype (fp32) and only then casts to bf16, so ~15B params cost ~60 GB of183    # fp32 on top of the 33 GB bf16 target and the checkpoint itself. On this184    # box CPU and GPU share one 119 GB pool, so it OOMs before it can cast.185    # The default path materializes straight into bf16 from the mmap'd186    # safetensors and peaks near 1x the weights. (The older187    # qwen_image_gen_server.py still passes False — it is fragile for the same188    # reason, just with a bigger margin to hide in.)189    extra = {}190    if TRANSFORMER_DIR:191        from diffusers import QwenImage21Transformer2DModel192        print(f"  transformer from {TRANSFORMER_DIR}", flush=True)193        extra["transformer"] = QwenImage21Transformer2DModel.from_pretrained(194            TRANSFORMER_DIR, torch_dtype=torch.bfloat16)195        _mem("transformer loaded")196    pipe = QwenImage21Pipeline.from_pretrained(197        MODEL_DIR,198        torch_dtype=torch.bfloat16,199        **extra,200    )201    _mem("weights materialized")202203    if LOAD_MODE == "device":204        pipe.to("cuda")205    elif LOAD_MODE == "offload":206        pipe.enable_model_cpu_offload()207    elif LOAD_MODE == "sequential":208        pipe.enable_sequential_cpu_offload()209    else:210        raise SystemExit(211            f"QWEN_IMAGE_21_LOAD_MODE={LOAD_MODE!r} is not one of device|offload|sequential"212        )213    _mem(f"placed ({LOAD_MODE})")214215    if LORA_PATH:216        if not os.path.exists(LORA_PATH):217            print(f"  WARNING: QWEN_IMAGE_21_LORA={LORA_PATH} not found — "218                  f"running the 40-step base model.", flush=True)219        else:220            # Needs `peft` in the container. Kept as load_lora_weights (not merged)221            # so the adapter can be disabled per process without touching the base.222            print(f"  loading Viggle turbo LoRA {os.path.basename(LORA_PATH)}", flush=True)223            pipe.load_lora_weights(LORA_PATH)224            if LORA_SCHEDULER_DIR and os.path.isdir(LORA_SCHEDULER_DIR):225                from diffusers import FlowMatchEulerDiscreteScheduler226                pipe.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(227                    LORA_SCHEDULER_DIR)228                print(f"  scheduler replaced from {LORA_SCHEDULER_DIR} "229                      f"(shift_terminal null, what the distillation targeted)", flush=True)230            LORA_STATE["active"] = True231            print(f"  turbo active: defaults are now {TURBO_STEPS} steps / true_cfg 1.0",232                  flush=True)233        _mem("lora loaded")234235    if VAE_TILING:236        # enable_tiling() is a real method; slicing only exists as a flag on this237        # VAE class (no enable_slicing()), so set it directly when present.238        pipe.vae.enable_tiling()239        if hasattr(pipe.vae, "use_slicing"):240            pipe.vae.use_slicing = True241        print("  VAE tiling enabled (slicing too, where supported).", flush=True)242243    # Hand back the load-time scratch before the first request arrives.244    gc.collect()245    if torch.cuda.is_available():246        torch.cuda.empty_cache()247    _mem("after cleanup")248249    print(f"Qwen-Image-2.1 ready in {time.time() - t0:.1f}s (mode={LOAD_MODE}).", flush=True)250251252class ImageRequest(BaseModel):253    prompt: str = Field(..., description="Text prompt, or the edit instruction when image(s) are supplied.")254    image: Optional[Union[str, List[str]]] = Field(255        None,256        description=(257            "Reference/condition image(s) as base64 PNG or JPEG. A single string or a list "258            f"of up to {MAX_REFERENCE_IMAGES}. Required on /v1/images/edits, optional on "259            "/v1/images/generations. RGBA input is preserved (transparent-layer editing)."260        ),261    )262    images: Optional[List[str]] = Field(263        None, description="Alias for `image` as a list. Use whichever reads better; don't send both."264    )265    negative_prompt: Optional[str] = Field(266        None,267        description="What to steer away from. Required for true_cfg_scale > 1 to take effect; "268                    "the server substitutes \"\" if you raise cfg without one.",269    )270    n: int = Field(1, ge=1, le=4, description="Number of images to generate.")271    size: Optional[str] = Field(272        None,273        description="WxH, e.g. 2048x2048. null = square at `output_resolution`. Floored to a "274                    "multiple of vae_scale_factor*2. See /v1/config for native aspect ratios.",275    )276    output_resolution: int = Field(277        DEFAULT_RESOLUTION, ge=512, le=2752,278        description="Target side length. Derives size when `size` is null, and sets the side "279                    "length condition images are resized to. Default 2048 = the card's native "280                    "operating point; 1024 is ~5x faster but visibly degraded, so use it only "281                    "for previews.",282    )283    num_inference_steps: Optional[int] = Field(284        None, ge=1, le=100,285        description="Diffusion steps. Left unset it resolves to the Viggle turbo LoRA's "286                    "6 when that LoRA is loaded, else the upstream default of 40 — "287                    "sampling a distilled few-step model at 40 steps is wrong, not just "288                    "slow. GET /v1/config reports which is active.",289    )290    true_cfg_scale: float = Field(291        DEFAULT_TRUE_CFG, ge=1.0, le=15.0,292        description="True CFG scale (NOT guidance_scale). 1.0 = off, which is the upstream default. "293                    "Needs a negative_prompt to engage.",294    )295    use_kv_cache: bool = Field(296        True,297        description="Reuse text/condition-image KV after step 1. Faster, and it changes which "298                    "valid sample you get — pin it to reproduce an image.",299    )300    seed: Optional[int] = Field(None, description="Random seed. null = random per call. n>1 uses seed+i.")301    response_format: str = Field("b64_json", description="Only b64_json supported.")302303304def _params_doc():305    out = {}306    for name, info in ImageRequest.model_fields.items():307        default = info.default308        if default is ...:309            default = None310        entry: dict[str, Any] = {"default": default, "description": info.description}311        if name in ("image", "images"):312            # Never echo a payload back as a "default" — these are content, not settings.313            entry["default"] = None314        out[name] = entry315    return out316317318@app.get("/health")319def health():320    return {"status": "ok" if pipe is not None else "loading", "model": SERVED_MODEL_ID}321322323@app.get("/v1/models")324def models_ep():325    return {"object": "list", "data": [{"id": SERVED_MODEL_ID, "object": "model"}]}326327328@app.get("/v1/config")329def config_ep():330    return {331        "model": SERVED_MODEL_ID,332        "repo_id": "Qwen/Qwen-Image-2.1",333        "transformer": TRANSFORMER_DIR or "base",334        "pipeline_class": "QwenImage21Pipeline",335        "endpoints": ["POST /v1/images/generations", "POST /v1/images/edits"],336        "capabilities": [337            "text-to-image",338            "image-edit",339            "transparent-rgba",340            f"multi-reference (up to {MAX_REFERENCE_IMAGES} images)",341        ],342        "params": _params_doc(),343        "memory": {"load_mode": LOAD_MODE, "vae_tiling": VAE_TILING},344        "turbo_lora": {345            "active": LORA_STATE["active"],346            "path": LORA_PATH or None,347            "repo": "Viggle/Qwen-Image-2.1-viggle-turbo",348            "effective_default_steps": _default_steps(),349            "note": ("Viggle few-step distillation LoRA on this same base transformer. "350                     "When active the default drops to 6 steps with no CFG (~5x faster, "351                     "'hard to tell apart' from the 40-step base on most prompts per "352                     "Viggle; the gaps are small dense text and complicated edits). "353                     "Both remain overridable per request. Qwen Research license, same "354                     "as the base — still non-commercial."),355        },356        "native_aspect_ratios": NATIVE_ASPECT_RATIOS,357        "transparency": {358            "how": "Ask for it in the prompt; the 4-channel VAE emits alpha directly.",359            "prompt_template": (360                "This is an RGBA image with transparency. <subject>. "361                "The image has alpha channel and the background is transparent."362            ),363        },364        "license": "Qwen Research License (non-commercial)",365    }366367368def _collect_images(req: ImageRequest) -> Optional[list]:369    """Normalize `image` / `images` into a list of PIL images, or None."""370    if req.image is not None and req.images is not None:371        raise HTTPException(400, "send either `image` or `images`, not both")372    raw = req.images if req.images is not None else req.image373    if raw is None:374        return None375    if isinstance(raw, str):376        raw = [raw]377    if not raw:378        return None379    if len(raw) > MAX_REFERENCE_IMAGES:380        raise HTTPException(400, f"at most {MAX_REFERENCE_IMAGES} reference images ({len(raw)} sent)")381    out = []382    for i, b64 in enumerate(raw):383        try:384            img = Image.open(io.BytesIO(base64.b64decode(b64)))385            img.load()386        except Exception as e:387            raise HTTPException(400, f"image[{i}] is not decodable base64 PNG/JPEG: {e}")388        # No .convert() here: the pipeline promotes to RGBA itself, and forcing RGB389        # first would throw away the alpha this model is meant to edit.390        out.append(img)391    return out392393394def _resolve_size(req: ImageRequest) -> tuple[Optional[int], Optional[int]]:395    """Explicit `size` wins; otherwise let the pipeline derive from output_resolution396    (and, in edit mode, from the reference image's aspect ratio)."""397    if req.size is None:398        return None, None399    try:400        w_str, h_str = req.size.lower().split("x")401        w, h = int(w_str), int(h_str)402    except Exception:403        raise HTTPException(400, "size must be 'WIDTHxHEIGHT', e.g. 2048x2048")404    if not (256 <= w <= 4096 and 256 <= h <= 4096):405        raise HTTPException(400, "each dimension must be in [256, 4096]")406    return w, h407408409def _run(req: ImageRequest, require_image: bool):410    if pipe is None:411        raise HTTPException(503, "Model not loaded")412    if req.response_format != "b64_json":413        raise HTTPException(400, "only response_format=b64_json is supported")414415    condition_images = _collect_images(req)416    if require_image and not condition_images:417        raise HTTPException(400, "/v1/images/edits requires `image` (or `images`)")418419    width, height = _resolve_size(req)420421    # Unset steps resolve here, not at class-definition time, so the turbo LoRA's422    # 6 and the base model's 40 both come from the same request contract.423    steps = req.num_inference_steps if req.num_inference_steps is not None else _default_steps()424425    # The pipeline turns true CFG off unless a negative prompt exists, and only426    # logs a warning. Substitute an empty one so raising the knob has the effect427    # the caller plainly intended, and say so in params_used.428    negative_prompt = req.negative_prompt429    cfg_needs_negative = req.true_cfg_scale > 1.0 and negative_prompt is None430    if cfg_needs_negative:431        negative_prompt = DEFAULT_NEGATIVE432433    if req.seed is not None:434        generator = [torch.Generator("cuda").manual_seed(req.seed + i) for i in range(req.n)]435    else:436        generator = None437438    queued0 = time.time()439    with _gpu_lock:440        queued_s = time.time() - queued0441        if queued_s > 1.0:442            print(f"  waited {queued_s:.1f}s for the GPU (another request was running)", flush=True)443        t0 = time.time()444        with torch.inference_mode():445            images = pipe(446                prompt=req.prompt,447                image=condition_images,448                negative_prompt=negative_prompt,449                true_cfg_scale=req.true_cfg_scale,450                width=width,451                height=height,452                output_resolution=req.output_resolution,453                num_inference_steps=steps,454                num_images_per_prompt=req.n,455                use_kv_cache=req.use_kv_cache,456                generator=generator,457            ).images458    elapsed = time.time() - t0459460    data = []461    for img in images:462        buf = io.BytesIO()463        img.save(buf, format="PNG")  # PNG so a native-alpha result survives the trip464        data.append({"b64_json": base64.b64encode(buf.getvalue()).decode()})465466    # Report what was rendered, not what was asked: the pipeline floors both467    # dimensions and, in edit mode, may derive them from the reference aspect.468    actual = f"{images[0].width}x{images[0].height}" if images else None469    modes = sorted({img.mode for img in images})470471    print(472        f"generated {len(images)} × {actual} in {elapsed:.2f}s "473        f"(steps={steps}, true_cfg={req.true_cfg_scale}, "474        f"refs={len(condition_images) if condition_images else 0}, "475        f"kv_cache={req.use_kv_cache}, seed={req.seed}, mode={'/'.join(modes)})",476        flush=True,477    )478    return {479        "created": int(time.time()),480        "data": data,481        "params_used": {482            "prompt": req.prompt,483            "negative_prompt": negative_prompt,484            "n": req.n,485            "size_requested": req.size,486            "size": actual,487            "image_mode": modes,488            "output_resolution": req.output_resolution,489            "num_inference_steps": steps,490            "turbo_lora": LORA_STATE["active"],491            "true_cfg_scale": req.true_cfg_scale,492            "use_kv_cache": req.use_kv_cache,493            "reference_images": len(condition_images) if condition_images else 0,494            "seed": req.seed,495            "elapsed_s": round(elapsed, 2),496            "queued_s": round(queued_s, 2),497            **({"_note": "true_cfg_scale > 1 with no negative_prompt — substituted the default negative prompt so CFG engages"}498               if cfg_needs_negative else {}),499        },500    }501502503@app.post("/v1/images/generations")504def images_generations(req: ImageRequest):505    return _run(req, require_image=False)506507508@app.post("/v1/images/edits")509def images_edits(req: ImageRequest):510    return _run(req, require_image=True)511512513if __name__ == "__main__":514    load_model()515    uvicorn.run(app, host="0.0.0.0", port=PORT)