m64_c2cef7905f1b / qwen_image_21_omni_server.py
515 lines, 23.0 KBraw
1"""Qwen-Image-2.1 server (Qwen/Qwen-Image-2.1) — unified T2I + edit + native RGBA.23Adapted for the qwen-image-2.1-uncensored-omni-gb10 capsule from gx10's4qwen_image_21_unified_server.py (as in the qwen-image-2.1-viggle-turbo-gb10 capsule). Changes:5 * QWEN_IMAGE_21_TRANSFORMER: load the transformer from another diffusers folder (here6 ChatR Labs' Omni adapter merged into the base at strength 1.0, BF16) and the rest of the7 pipeline (text encoder, processor, tokenizer, VAE, scheduler) from MODEL_DIR.8 * The defaults (steps, resolution, true CFG, negative prompt) come from env vars, set to9 that model card's recommended values.10 * When true CFG is on and no negative prompt is given, the substitute is a single space,11 not "": per the model card an empty negative prompt disables true CFG.1213One 7B DiT (32 single-stream layers, mixed-granularity attention, prefix KV14cache reuse) does text-to-image, image editing and transparent-layer work.15Text encoder is Qwen3-VL — it reads condition images as vision context, which16is why reference pixels have to be passed through rather than pre-encoded.17VAE is 4-channel, so transparency is native: a prompt that asks for RGBA comes18back as an RGBA PNG, no matting step.1920Pipeline class: `QwenImage21Pipeline` — diffusers **git main only** (not in the210.40.0 release), transformers >= 5.17 for Qwen3VLForConditionalGeneration.2223Endpoints (both OpenAI-compatible in shape, both go through one code path):24 POST /v1/images/generations text-to-image; `image`/`images` optional25 POST /v1/images/edits edit/reference mode; image(s) required, up to 1026 GET /v1/config resolved defaults + the native aspect table2728Notes that bite:29 * CFG knob is `true_cfg_scale`, not `guidance_scale`, and the pipeline30 silently disables it unless a negative_prompt is present. We fill in "" so31 the knob does what the caller asked; params_used flags when we did.32 * `use_kv_cache` changes the sample, not just the speed — caching re-tiles33 attention, and a 1-ULP difference amplifies over 32 blocks. Both settings34 are equally valid; pin the flag to reproduce an image.35 * Dimensions are floored to vae_scale_factor*2, so params_used echoes the36 size actually rendered, not the size requested.3738Env vars:39 MODEL_DIR container path to weights (default /model)40 PORT listen port (default 9049)41 SERVED_MODEL_ID /v1/models id (default qwen-image-2.1-7b)42 QWEN_IMAGE_21_LOAD_MODE device | offload | sequential (default device)43 QWEN_IMAGE_21_VAE_TILING 1 to tile VAE decode (default 1)44"""45import base6446import gc47import io48import os49import resource50import threading51import time52from typing import Any, List, Optional, Union5354import torch55import uvicorn56from fastapi import FastAPI, HTTPException57from pydantic import BaseModel, Field58from PIL import Image5960MODEL_DIR = os.environ.get("MODEL_DIR", "/model")61PORT = int(os.environ.get("PORT", 9049))62SERVED_MODEL_ID = os.environ.get("SERVED_MODEL_ID", "qwen-image-2.1-7b")6364# How to get 33 GB of weights onto a 119 GB pool that CPU and GPU share:65# device low-mem load, then pipe.to("cuda"). Everything resident, fastest.66# offload enable_model_cpu_offload() — one component on device at a time67# (text_encoder->transformer->vae). Lower peak, slower per image.68# sequential enable_sequential_cpu_offload() — submodule granularity. Lowest69# peak, much slower. Last resort.70LOAD_MODE = os.environ.get("QWEN_IMAGE_21_LOAD_MODE", "device").strip().lower()7172# VAE tiling caps the decode transient — but it is OFF by default, because it is73# not free and it is not rare. This VAE tiles whenever the latent exceeds74# tile_sample_min/scale_factor_spatial = 256/16 = 16, and a 1024² image has a75# 64×64 latent, so tiling engages on *every* normal image: the decode becomes a76# ~5×5 grid of tiles (stride 192 -> 64 px overlap) that are blended back77# together, which shows up as seams and faint streaks. Vanilla diffusers does78# not enable it. With ~80 GB spare on this box the decode fits comfortably79# untiled, so turn it on only for a resolution that genuinely will not fit.80VAE_TILING = os.environ.get("QWEN_IMAGE_21_VAE_TILING", "0") not in ("0", "false", "no")8182# Upstream README: 40 steps, and height/width default to output_resolution when83# unset.84#85# 2048, not the pipeline's own 1024 default: every example on the model card and86# its whole aspect-ratio table are at ~2048, and the difference is not subtle.87# Same prompt and seed at 1024 gives a flat, poster-like image with a floating88# sign and mushy reflections; at 2048 the same seed resolves into a89# photorealistic street with physical 3D letters, storefronts and real90# reflections. 1024 sits below the resolution this checkpoint was tuned for, and91# the result reads as "artifacts".92#93# It costs ~5x the time on GB10 (~337 s vs ~67 s at 40 steps), so pass94# size/output_resolution explicitly for fast previews. Quality is the default.95DEFAULT_STEPS = int(os.environ.get("QWEN_IMAGE_21_STEPS", "40"))96DEFAULT_RESOLUTION = int(os.environ.get("QWEN_IMAGE_21_RESOLUTION", "2048"))97DEFAULT_TRUE_CFG = float(os.environ.get("QWEN_IMAGE_21_TRUE_CFG", "1.0"))98# The negative prompt used when true CFG is on and the request has none. A single space:99# an empty one turns true CFG off in the pipeline.100DEFAULT_NEGATIVE = os.environ.get("QWEN_IMAGE_21_NEGATIVE", " ")101# Optional: a diffusers folder holding a replacement transformer (config.json +102# diffusion_pytorch_model.safetensors) for the same architecture.103TRANSFORMER_DIR = os.environ.get("QWEN_IMAGE_21_TRANSFORMER", "").strip()104105# ── Viggle turbo LoRA (optional) ──────────────────────────────────────────────106# Viggle/Qwen-Image-2.1-viggle-turbo is a few-step DISTILLATION LoRA for this exact107# base transformer, so it costs 1.3 GB instead of another 33 GB checkpoint. v0.3 runs108# 6 steps with NO classifier-free guidance; Viggle measure ~5x faster end to end and109# "hard to tell apart" from the 40-step base on most prompts, with the gaps on small110# dense text and complicated edits.111#112# Loading it CHANGES THE SAMPLER CONTRACT, not just the speed: a distilled few-step113# model sampled at 40 steps is wrong, not merely slow, so the defaults below swap too.114# Everything stays overridable per request.115#116# Licensing is unchanged — the LoRA is Qwen Research, same as the base it adapts.117LORA_PATH = os.environ.get("QWEN_IMAGE_21_LORA", "").strip()118# v0.3's distillation was done against a schedule with shift_terminal null; the repo119# ships that scheduler config for exactly this reason. Applied only when the LoRA is.120LORA_SCHEDULER_DIR = os.environ.get("QWEN_IMAGE_21_LORA_SCHEDULER", "").strip()121TURBO_STEPS = int(os.environ.get("QWEN_IMAGE_21_LORA_STEPS", "6"))122123# From the model card — the resolutions the checkpoint was actually trained on.124NATIVE_ASPECT_RATIOS = {125 "1:1": [2048, 2048],126 "4:3": [2400, 1792],127 "3:4": [1792, 2400],128 "3:2": [2528, 1696],129 "2:3": [1696, 2528],130 "16:9": [2752, 1536],131 "9:16": [1536, 2752],132}133134MAX_REFERENCE_IMAGES = 10135136app = FastAPI()137pipe = None138139# One GPU, one pipeline, and a scheduler that keeps its step counter on itself:140# `FlowMatchEulerDiscreteScheduler.step` reads `self.step_index` and indexes141# `self.sigmas[step_index + 1]`, so two generations in flight share one counter142# and it runs off the end —143# IndexError: index 41 is out of bounds for dimension 0 with size 41144# for a 40-step call (41 sigmas). FastAPI runs `def` endpoints in a threadpool,145# so concurrent requests land here in parallel by default. Serialize them.146# Queueing rather than rejecting: the second caller waits a generation and gets147# an image, which is what a benchmark driver firing off prompts actually wants.148_gpu_lock = threading.Lock()149150151def _mem(tag: str):152 """GB of host RSS + CUDA allocator state. Printed at every load stage so a153 future OOM says where it happened instead of just dying."""154 rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024 ** 2) # KiB -> GiB155 if torch.cuda.is_available():156 alloc = torch.cuda.memory_allocated() / (1024 ** 3)157 reserved = torch.cuda.memory_reserved() / (1024 ** 3)158 print(f" [mem] {tag:<22} peak_rss={rss:6.1f} GiB cuda_alloc={alloc:6.1f} GiB "159 f"cuda_reserved={reserved:6.1f} GiB", flush=True)160 else:161 print(f" [mem] {tag:<22} peak_rss={rss:6.1f} GiB", flush=True)162163164LORA_STATE = {"active": False}165166167def _default_steps():168 return TURBO_STEPS if LORA_STATE["active"] else DEFAULT_STEPS169170171def load_model():172 global pipe173 from diffusers import QwenImage21Pipeline174175 print(f"Loading QwenImage21Pipeline from {MODEL_DIR} "176 f"(~33 GB BF16, mode={LOAD_MODE}, first load several min from NFS)...", flush=True)177 _mem("before load")178 t0 = time.time()179180 # low_cpu_mem_usage stays at its default (True). Do NOT pass False here:181 # that path instantiates every module with real weights in the *default*182 # dtype (fp32) and only then casts to bf16, so ~15B params cost ~60 GB of183 # fp32 on top of the 33 GB bf16 target and the checkpoint itself. On this184 # box CPU and GPU share one 119 GB pool, so it OOMs before it can cast.185 # The default path materializes straight into bf16 from the mmap'd186 # safetensors and peaks near 1x the weights. (The older187 # qwen_image_gen_server.py still passes False — it is fragile for the same188 # reason, just with a bigger margin to hide in.)189 extra = {}190 if TRANSFORMER_DIR:191 from diffusers import QwenImage21Transformer2DModel192 print(f" transformer from {TRANSFORMER_DIR}", flush=True)193 extra["transformer"] = QwenImage21Transformer2DModel.from_pretrained(194 TRANSFORMER_DIR, torch_dtype=torch.bfloat16)195 _mem("transformer loaded")196 pipe = QwenImage21Pipeline.from_pretrained(197 MODEL_DIR,198 torch_dtype=torch.bfloat16,199 **extra,200 )201 _mem("weights materialized")202203 if LOAD_MODE == "device":204 pipe.to("cuda")205 elif LOAD_MODE == "offload":206 pipe.enable_model_cpu_offload()207 elif LOAD_MODE == "sequential":208 pipe.enable_sequential_cpu_offload()209 else:210 raise SystemExit(211 f"QWEN_IMAGE_21_LOAD_MODE={LOAD_MODE!r} is not one of device|offload|sequential"212 )213 _mem(f"placed ({LOAD_MODE})")214215 if LORA_PATH:216 if not os.path.exists(LORA_PATH):217 print(f" WARNING: QWEN_IMAGE_21_LORA={LORA_PATH} not found — "218 f"running the 40-step base model.", flush=True)219 else:220 # Needs `peft` in the container. Kept as load_lora_weights (not merged)221 # so the adapter can be disabled per process without touching the base.222 print(f" loading Viggle turbo LoRA {os.path.basename(LORA_PATH)}", flush=True)223 pipe.load_lora_weights(LORA_PATH)224 if LORA_SCHEDULER_DIR and os.path.isdir(LORA_SCHEDULER_DIR):225 from diffusers import FlowMatchEulerDiscreteScheduler226 pipe.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(227 LORA_SCHEDULER_DIR)228 print(f" scheduler replaced from {LORA_SCHEDULER_DIR} "229 f"(shift_terminal null, what the distillation targeted)", flush=True)230 LORA_STATE["active"] = True231 print(f" turbo active: defaults are now {TURBO_STEPS} steps / true_cfg 1.0",232 flush=True)233 _mem("lora loaded")234235 if VAE_TILING:236 # enable_tiling() is a real method; slicing only exists as a flag on this237 # VAE class (no enable_slicing()), so set it directly when present.238 pipe.vae.enable_tiling()239 if hasattr(pipe.vae, "use_slicing"):240 pipe.vae.use_slicing = True241 print(" VAE tiling enabled (slicing too, where supported).", flush=True)242243 # Hand back the load-time scratch before the first request arrives.244 gc.collect()245 if torch.cuda.is_available():246 torch.cuda.empty_cache()247 _mem("after cleanup")248249 print(f"Qwen-Image-2.1 ready in {time.time() - t0:.1f}s (mode={LOAD_MODE}).", flush=True)250251252class ImageRequest(BaseModel):253 prompt: str = Field(..., description="Text prompt, or the edit instruction when image(s) are supplied.")254 image: Optional[Union[str, List[str]]] = Field(255 None,256 description=(257 "Reference/condition image(s) as base64 PNG or JPEG. A single string or a list "258 f"of up to {MAX_REFERENCE_IMAGES}. Required on /v1/images/edits, optional on "259 "/v1/images/generations. RGBA input is preserved (transparent-layer editing)."260 ),261 )262 images: Optional[List[str]] = Field(263 None, description="Alias for `image` as a list. Use whichever reads better; don't send both."264 )265 negative_prompt: Optional[str] = Field(266 None,267 description="What to steer away from. Required for true_cfg_scale > 1 to take effect; "268 "the server substitutes \"\" if you raise cfg without one.",269 )270 n: int = Field(1, ge=1, le=4, description="Number of images to generate.")271 size: Optional[str] = Field(272 None,273 description="WxH, e.g. 2048x2048. null = square at `output_resolution`. Floored to a "274 "multiple of vae_scale_factor*2. See /v1/config for native aspect ratios.",275 )276 output_resolution: int = Field(277 DEFAULT_RESOLUTION, ge=512, le=2752,278 description="Target side length. Derives size when `size` is null, and sets the side "279 "length condition images are resized to. Default 2048 = the card's native "280 "operating point; 1024 is ~5x faster but visibly degraded, so use it only "281 "for previews.",282 )283 num_inference_steps: Optional[int] = Field(284 None, ge=1, le=100,285 description="Diffusion steps. Left unset it resolves to the Viggle turbo LoRA's "286 "6 when that LoRA is loaded, else the upstream default of 40 — "287 "sampling a distilled few-step model at 40 steps is wrong, not just "288 "slow. GET /v1/config reports which is active.",289 )290 true_cfg_scale: float = Field(291 DEFAULT_TRUE_CFG, ge=1.0, le=15.0,292 description="True CFG scale (NOT guidance_scale). 1.0 = off, which is the upstream default. "293 "Needs a negative_prompt to engage.",294 )295 use_kv_cache: bool = Field(296 True,297 description="Reuse text/condition-image KV after step 1. Faster, and it changes which "298 "valid sample you get — pin it to reproduce an image.",299 )300 seed: Optional[int] = Field(None, description="Random seed. null = random per call. n>1 uses seed+i.")301 response_format: str = Field("b64_json", description="Only b64_json supported.")302303304def _params_doc():305 out = {}306 for name, info in ImageRequest.model_fields.items():307 default = info.default308 if default is ...:309 default = None310 entry: dict[str, Any] = {"default": default, "description": info.description}311 if name in ("image", "images"):312 # Never echo a payload back as a "default" — these are content, not settings.313 entry["default"] = None314 out[name] = entry315 return out316317318@app.get("/health")319def health():320 return {"status": "ok" if pipe is not None else "loading", "model": SERVED_MODEL_ID}321322323@app.get("/v1/models")324def models_ep():325 return {"object": "list", "data": [{"id": SERVED_MODEL_ID, "object": "model"}]}326327328@app.get("/v1/config")329def config_ep():330 return {331 "model": SERVED_MODEL_ID,332 "repo_id": "Qwen/Qwen-Image-2.1",333 "transformer": TRANSFORMER_DIR or "base",334 "pipeline_class": "QwenImage21Pipeline",335 "endpoints": ["POST /v1/images/generations", "POST /v1/images/edits"],336 "capabilities": [337 "text-to-image",338 "image-edit",339 "transparent-rgba",340 f"multi-reference (up to {MAX_REFERENCE_IMAGES} images)",341 ],342 "params": _params_doc(),343 "memory": {"load_mode": LOAD_MODE, "vae_tiling": VAE_TILING},344 "turbo_lora": {345 "active": LORA_STATE["active"],346 "path": LORA_PATH or None,347 "repo": "Viggle/Qwen-Image-2.1-viggle-turbo",348 "effective_default_steps": _default_steps(),349 "note": ("Viggle few-step distillation LoRA on this same base transformer. "350 "When active the default drops to 6 steps with no CFG (~5x faster, "351 "'hard to tell apart' from the 40-step base on most prompts per "352 "Viggle; the gaps are small dense text and complicated edits). "353 "Both remain overridable per request. Qwen Research license, same "354 "as the base — still non-commercial."),355 },356 "native_aspect_ratios": NATIVE_ASPECT_RATIOS,357 "transparency": {358 "how": "Ask for it in the prompt; the 4-channel VAE emits alpha directly.",359 "prompt_template": (360 "This is an RGBA image with transparency. <subject>. "361 "The image has alpha channel and the background is transparent."362 ),363 },364 "license": "Qwen Research License (non-commercial)",365 }366367368def _collect_images(req: ImageRequest) -> Optional[list]:369 """Normalize `image` / `images` into a list of PIL images, or None."""370 if req.image is not None and req.images is not None:371 raise HTTPException(400, "send either `image` or `images`, not both")372 raw = req.images if req.images is not None else req.image373 if raw is None:374 return None375 if isinstance(raw, str):376 raw = [raw]377 if not raw:378 return None379 if len(raw) > MAX_REFERENCE_IMAGES:380 raise HTTPException(400, f"at most {MAX_REFERENCE_IMAGES} reference images ({len(raw)} sent)")381 out = []382 for i, b64 in enumerate(raw):383 try:384 img = Image.open(io.BytesIO(base64.b64decode(b64)))385 img.load()386 except Exception as e:387 raise HTTPException(400, f"image[{i}] is not decodable base64 PNG/JPEG: {e}")388 # No .convert() here: the pipeline promotes to RGBA itself, and forcing RGB389 # first would throw away the alpha this model is meant to edit.390 out.append(img)391 return out392393394def _resolve_size(req: ImageRequest) -> tuple[Optional[int], Optional[int]]:395 """Explicit `size` wins; otherwise let the pipeline derive from output_resolution396 (and, in edit mode, from the reference image's aspect ratio)."""397 if req.size is None:398 return None, None399 try:400 w_str, h_str = req.size.lower().split("x")401 w, h = int(w_str), int(h_str)402 except Exception:403 raise HTTPException(400, "size must be 'WIDTHxHEIGHT', e.g. 2048x2048")404 if not (256 <= w <= 4096 and 256 <= h <= 4096):405 raise HTTPException(400, "each dimension must be in [256, 4096]")406 return w, h407408409def _run(req: ImageRequest, require_image: bool):410 if pipe is None:411 raise HTTPException(503, "Model not loaded")412 if req.response_format != "b64_json":413 raise HTTPException(400, "only response_format=b64_json is supported")414415 condition_images = _collect_images(req)416 if require_image and not condition_images:417 raise HTTPException(400, "/v1/images/edits requires `image` (or `images`)")418419 width, height = _resolve_size(req)420421 # Unset steps resolve here, not at class-definition time, so the turbo LoRA's422 # 6 and the base model's 40 both come from the same request contract.423 steps = req.num_inference_steps if req.num_inference_steps is not None else _default_steps()424425 # The pipeline turns true CFG off unless a negative prompt exists, and only426 # logs a warning. Substitute an empty one so raising the knob has the effect427 # the caller plainly intended, and say so in params_used.428 negative_prompt = req.negative_prompt429 cfg_needs_negative = req.true_cfg_scale > 1.0 and negative_prompt is None430 if cfg_needs_negative:431 negative_prompt = DEFAULT_NEGATIVE432433 if req.seed is not None:434 generator = [torch.Generator("cuda").manual_seed(req.seed + i) for i in range(req.n)]435 else:436 generator = None437438 queued0 = time.time()439 with _gpu_lock:440 queued_s = time.time() - queued0441 if queued_s > 1.0:442 print(f" waited {queued_s:.1f}s for the GPU (another request was running)", flush=True)443 t0 = time.time()444 with torch.inference_mode():445 images = pipe(446 prompt=req.prompt,447 image=condition_images,448 negative_prompt=negative_prompt,449 true_cfg_scale=req.true_cfg_scale,450 width=width,451 height=height,452 output_resolution=req.output_resolution,453 num_inference_steps=steps,454 num_images_per_prompt=req.n,455 use_kv_cache=req.use_kv_cache,456 generator=generator,457 ).images458 elapsed = time.time() - t0459460 data = []461 for img in images:462 buf = io.BytesIO()463 img.save(buf, format="PNG") # PNG so a native-alpha result survives the trip464 data.append({"b64_json": base64.b64encode(buf.getvalue()).decode()})465466 # Report what was rendered, not what was asked: the pipeline floors both467 # dimensions and, in edit mode, may derive them from the reference aspect.468 actual = f"{images[0].width}x{images[0].height}" if images else None469 modes = sorted({img.mode for img in images})470471 print(472 f"generated {len(images)} × {actual} in {elapsed:.2f}s "473 f"(steps={steps}, true_cfg={req.true_cfg_scale}, "474 f"refs={len(condition_images) if condition_images else 0}, "475 f"kv_cache={req.use_kv_cache}, seed={req.seed}, mode={'/'.join(modes)})",476 flush=True,477 )478 return {479 "created": int(time.time()),480 "data": data,481 "params_used": {482 "prompt": req.prompt,483 "negative_prompt": negative_prompt,484 "n": req.n,485 "size_requested": req.size,486 "size": actual,487 "image_mode": modes,488 "output_resolution": req.output_resolution,489 "num_inference_steps": steps,490 "turbo_lora": LORA_STATE["active"],491 "true_cfg_scale": req.true_cfg_scale,492 "use_kv_cache": req.use_kv_cache,493 "reference_images": len(condition_images) if condition_images else 0,494 "seed": req.seed,495 "elapsed_s": round(elapsed, 2),496 "queued_s": round(queued_s, 2),497 **({"_note": "true_cfg_scale > 1 with no negative_prompt — substituted the default negative prompt so CFG engages"}498 if cfg_needs_negative else {}),499 },500 }501502503@app.post("/v1/images/generations")504def images_generations(req: ImageRequest):505 return _run(req, require_image=False)506507508@app.post("/v1/images/edits")509def images_edits(req: ImageRequest):510 return _run(req, require_image=True)511512513if __name__ == "__main__":514 load_model()515 uvicorn.run(app, host="0.0.0.0", port=PORT)