"""Qwen-Image-2.1 server (Qwen/Qwen-Image-2.1) — unified T2I + edit + native RGBA. Adapted for the qwen-image-2.1-uncensored-omni-gb10 capsule from gx10's qwen_image_21_unified_server.py (as in the qwen-image-2.1-viggle-turbo-gb10 capsule). Changes: * QWEN_IMAGE_21_TRANSFORMER: load the transformer from another diffusers folder (here ChatR Labs' Omni adapter merged into the base at strength 1.0, BF16) and the rest of the pipeline (text encoder, processor, tokenizer, VAE, scheduler) from MODEL_DIR. * The defaults (steps, resolution, true CFG, negative prompt) come from env vars, set to that model card's recommended values. * When true CFG is on and no negative prompt is given, the substitute is a single space, not "": per the model card an empty negative prompt disables true CFG. One 7B DiT (32 single-stream layers, mixed-granularity attention, prefix KV cache reuse) does text-to-image, image editing and transparent-layer work. Text encoder is Qwen3-VL — it reads condition images as vision context, which is why reference pixels have to be passed through rather than pre-encoded. VAE is 4-channel, so transparency is native: a prompt that asks for RGBA comes back as an RGBA PNG, no matting step. Pipeline class: `QwenImage21Pipeline` — diffusers **git main only** (not in the 0.40.0 release), transformers >= 5.17 for Qwen3VLForConditionalGeneration. Endpoints (both OpenAI-compatible in shape, both go through one code path): POST /v1/images/generations text-to-image; `image`/`images` optional POST /v1/images/edits edit/reference mode; image(s) required, up to 10 GET /v1/config resolved defaults + the native aspect table Notes that bite: * CFG knob is `true_cfg_scale`, not `guidance_scale`, and the pipeline silently disables it unless a negative_prompt is present. We fill in "" so the knob does what the caller asked; params_used flags when we did. * `use_kv_cache` changes the sample, not just the speed — caching re-tiles attention, and a 1-ULP difference amplifies over 32 blocks. Both settings are equally valid; pin the flag to reproduce an image. * Dimensions are floored to vae_scale_factor*2, so params_used echoes the size actually rendered, not the size requested. Env vars: MODEL_DIR container path to weights (default /model) PORT listen port (default 9049) SERVED_MODEL_ID /v1/models id (default qwen-image-2.1-7b) QWEN_IMAGE_21_LOAD_MODE device | offload | sequential (default device) QWEN_IMAGE_21_VAE_TILING 1 to tile VAE decode (default 1) """ import base64 import gc import io import os import resource import threading import time from typing import Any, List, Optional, Union import torch import uvicorn from fastapi import FastAPI, HTTPException from pydantic import BaseModel, Field from PIL import Image MODEL_DIR = os.environ.get("MODEL_DIR", "/model") PORT = int(os.environ.get("PORT", 9049)) SERVED_MODEL_ID = os.environ.get("SERVED_MODEL_ID", "qwen-image-2.1-7b") # How to get 33 GB of weights onto a 119 GB pool that CPU and GPU share: # device low-mem load, then pipe.to("cuda"). Everything resident, fastest. # offload enable_model_cpu_offload() — one component on device at a time # (text_encoder->transformer->vae). Lower peak, slower per image. # sequential enable_sequential_cpu_offload() — submodule granularity. Lowest # peak, much slower. Last resort. LOAD_MODE = os.environ.get("QWEN_IMAGE_21_LOAD_MODE", "device").strip().lower() # VAE tiling caps the decode transient — but it is OFF by default, because it is # not free and it is not rare. This VAE tiles whenever the latent exceeds # tile_sample_min/scale_factor_spatial = 256/16 = 16, and a 1024² image has a # 64×64 latent, so tiling engages on *every* normal image: the decode becomes a # ~5×5 grid of tiles (stride 192 -> 64 px overlap) that are blended back # together, which shows up as seams and faint streaks. Vanilla diffusers does # not enable it. With ~80 GB spare on this box the decode fits comfortably # untiled, so turn it on only for a resolution that genuinely will not fit. VAE_TILING = os.environ.get("QWEN_IMAGE_21_VAE_TILING", "0") not in ("0", "false", "no") # Upstream README: 40 steps, and height/width default to output_resolution when # unset. # # 2048, not the pipeline's own 1024 default: every example on the model card and # its whole aspect-ratio table are at ~2048, and the difference is not subtle. # Same prompt and seed at 1024 gives a flat, poster-like image with a floating # sign and mushy reflections; at 2048 the same seed resolves into a # photorealistic street with physical 3D letters, storefronts and real # reflections. 1024 sits below the resolution this checkpoint was tuned for, and # the result reads as "artifacts". # # It costs ~5x the time on GB10 (~337 s vs ~67 s at 40 steps), so pass # size/output_resolution explicitly for fast previews. Quality is the default. DEFAULT_STEPS = int(os.environ.get("QWEN_IMAGE_21_STEPS", "40")) DEFAULT_RESOLUTION = int(os.environ.get("QWEN_IMAGE_21_RESOLUTION", "2048")) DEFAULT_TRUE_CFG = float(os.environ.get("QWEN_IMAGE_21_TRUE_CFG", "1.0")) # The negative prompt used when true CFG is on and the request has none. A single space: # an empty one turns true CFG off in the pipeline. DEFAULT_NEGATIVE = os.environ.get("QWEN_IMAGE_21_NEGATIVE", " ") # Optional: a diffusers folder holding a replacement transformer (config.json + # diffusion_pytorch_model.safetensors) for the same architecture. TRANSFORMER_DIR = os.environ.get("QWEN_IMAGE_21_TRANSFORMER", "").strip() # ── Viggle turbo LoRA (optional) ────────────────────────────────────────────── # Viggle/Qwen-Image-2.1-viggle-turbo is a few-step DISTILLATION LoRA for this exact # base transformer, so it costs 1.3 GB instead of another 33 GB checkpoint. v0.3 runs # 6 steps with NO classifier-free guidance; Viggle measure ~5x faster end to end and # "hard to tell apart" from the 40-step base on most prompts, with the gaps on small # dense text and complicated edits. # # Loading it CHANGES THE SAMPLER CONTRACT, not just the speed: a distilled few-step # model sampled at 40 steps is wrong, not merely slow, so the defaults below swap too. # Everything stays overridable per request. # # Licensing is unchanged — the LoRA is Qwen Research, same as the base it adapts. LORA_PATH = os.environ.get("QWEN_IMAGE_21_LORA", "").strip() # v0.3's distillation was done against a schedule with shift_terminal null; the repo # ships that scheduler config for exactly this reason. Applied only when the LoRA is. LORA_SCHEDULER_DIR = os.environ.get("QWEN_IMAGE_21_LORA_SCHEDULER", "").strip() TURBO_STEPS = int(os.environ.get("QWEN_IMAGE_21_LORA_STEPS", "6")) # From the model card — the resolutions the checkpoint was actually trained on. NATIVE_ASPECT_RATIOS = { "1:1": [2048, 2048], "4:3": [2400, 1792], "3:4": [1792, 2400], "3:2": [2528, 1696], "2:3": [1696, 2528], "16:9": [2752, 1536], "9:16": [1536, 2752], } MAX_REFERENCE_IMAGES = 10 app = FastAPI() pipe = None # One GPU, one pipeline, and a scheduler that keeps its step counter on itself: # `FlowMatchEulerDiscreteScheduler.step` reads `self.step_index` and indexes # `self.sigmas[step_index + 1]`, so two generations in flight share one counter # and it runs off the end — # IndexError: index 41 is out of bounds for dimension 0 with size 41 # for a 40-step call (41 sigmas). FastAPI runs `def` endpoints in a threadpool, # so concurrent requests land here in parallel by default. Serialize them. # Queueing rather than rejecting: the second caller waits a generation and gets # an image, which is what a benchmark driver firing off prompts actually wants. _gpu_lock = threading.Lock() def _mem(tag: str): """GB of host RSS + CUDA allocator state. Printed at every load stage so a future OOM says where it happened instead of just dying.""" rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024 ** 2) # KiB -> GiB if torch.cuda.is_available(): alloc = torch.cuda.memory_allocated() / (1024 ** 3) reserved = torch.cuda.memory_reserved() / (1024 ** 3) print(f" [mem] {tag:<22} peak_rss={rss:6.1f} GiB cuda_alloc={alloc:6.1f} GiB " f"cuda_reserved={reserved:6.1f} GiB", flush=True) else: print(f" [mem] {tag:<22} peak_rss={rss:6.1f} GiB", flush=True) LORA_STATE = {"active": False} def _default_steps(): return TURBO_STEPS if LORA_STATE["active"] else DEFAULT_STEPS def load_model(): global pipe from diffusers import QwenImage21Pipeline print(f"Loading QwenImage21Pipeline from {MODEL_DIR} " f"(~33 GB BF16, mode={LOAD_MODE}, first load several min from NFS)...", flush=True) _mem("before load") t0 = time.time() # low_cpu_mem_usage stays at its default (True). Do NOT pass False here: # that path instantiates every module with real weights in the *default* # dtype (fp32) and only then casts to bf16, so ~15B params cost ~60 GB of # fp32 on top of the 33 GB bf16 target and the checkpoint itself. On this # box CPU and GPU share one 119 GB pool, so it OOMs before it can cast. # The default path materializes straight into bf16 from the mmap'd # safetensors and peaks near 1x the weights. (The older # qwen_image_gen_server.py still passes False — it is fragile for the same # reason, just with a bigger margin to hide in.) extra = {} if TRANSFORMER_DIR: from diffusers import QwenImage21Transformer2DModel print(f" transformer from {TRANSFORMER_DIR}", flush=True) extra["transformer"] = QwenImage21Transformer2DModel.from_pretrained( TRANSFORMER_DIR, torch_dtype=torch.bfloat16) _mem("transformer loaded") pipe = QwenImage21Pipeline.from_pretrained( MODEL_DIR, torch_dtype=torch.bfloat16, **extra, ) _mem("weights materialized") if LOAD_MODE == "device": pipe.to("cuda") elif LOAD_MODE == "offload": pipe.enable_model_cpu_offload() elif LOAD_MODE == "sequential": pipe.enable_sequential_cpu_offload() else: raise SystemExit( f"QWEN_IMAGE_21_LOAD_MODE={LOAD_MODE!r} is not one of device|offload|sequential" ) _mem(f"placed ({LOAD_MODE})") if LORA_PATH: if not os.path.exists(LORA_PATH): print(f" WARNING: QWEN_IMAGE_21_LORA={LORA_PATH} not found — " f"running the 40-step base model.", flush=True) else: # Needs `peft` in the container. Kept as load_lora_weights (not merged) # so the adapter can be disabled per process without touching the base. print(f" loading Viggle turbo LoRA {os.path.basename(LORA_PATH)}", flush=True) pipe.load_lora_weights(LORA_PATH) if LORA_SCHEDULER_DIR and os.path.isdir(LORA_SCHEDULER_DIR): from diffusers import FlowMatchEulerDiscreteScheduler pipe.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( LORA_SCHEDULER_DIR) print(f" scheduler replaced from {LORA_SCHEDULER_DIR} " f"(shift_terminal null, what the distillation targeted)", flush=True) LORA_STATE["active"] = True print(f" turbo active: defaults are now {TURBO_STEPS} steps / true_cfg 1.0", flush=True) _mem("lora loaded") if VAE_TILING: # enable_tiling() is a real method; slicing only exists as a flag on this # VAE class (no enable_slicing()), so set it directly when present. pipe.vae.enable_tiling() if hasattr(pipe.vae, "use_slicing"): pipe.vae.use_slicing = True print(" VAE tiling enabled (slicing too, where supported).", flush=True) # Hand back the load-time scratch before the first request arrives. gc.collect() if torch.cuda.is_available(): torch.cuda.empty_cache() _mem("after cleanup") print(f"Qwen-Image-2.1 ready in {time.time() - t0:.1f}s (mode={LOAD_MODE}).", flush=True) class ImageRequest(BaseModel): prompt: str = Field(..., description="Text prompt, or the edit instruction when image(s) are supplied.") image: Optional[Union[str, List[str]]] = Field( None, description=( "Reference/condition image(s) as base64 PNG or JPEG. A single string or a list " f"of up to {MAX_REFERENCE_IMAGES}. Required on /v1/images/edits, optional on " "/v1/images/generations. RGBA input is preserved (transparent-layer editing)." ), ) images: Optional[List[str]] = Field( None, description="Alias for `image` as a list. Use whichever reads better; don't send both." ) negative_prompt: Optional[str] = Field( None, description="What to steer away from. Required for true_cfg_scale > 1 to take effect; " "the server substitutes \"\" if you raise cfg without one.", ) n: int = Field(1, ge=1, le=4, description="Number of images to generate.") size: Optional[str] = Field( None, description="WxH, e.g. 2048x2048. null = square at `output_resolution`. Floored to a " "multiple of vae_scale_factor*2. See /v1/config for native aspect ratios.", ) output_resolution: int = Field( DEFAULT_RESOLUTION, ge=512, le=2752, description="Target side length. Derives size when `size` is null, and sets the side " "length condition images are resized to. Default 2048 = the card's native " "operating point; 1024 is ~5x faster but visibly degraded, so use it only " "for previews.", ) num_inference_steps: Optional[int] = Field( None, ge=1, le=100, description="Diffusion steps. Left unset it resolves to the Viggle turbo LoRA's " "6 when that LoRA is loaded, else the upstream default of 40 — " "sampling a distilled few-step model at 40 steps is wrong, not just " "slow. GET /v1/config reports which is active.", ) true_cfg_scale: float = Field( DEFAULT_TRUE_CFG, ge=1.0, le=15.0, description="True CFG scale (NOT guidance_scale). 1.0 = off, which is the upstream default. " "Needs a negative_prompt to engage.", ) use_kv_cache: bool = Field( True, description="Reuse text/condition-image KV after step 1. Faster, and it changes which " "valid sample you get — pin it to reproduce an image.", ) seed: Optional[int] = Field(None, description="Random seed. null = random per call. n>1 uses seed+i.") response_format: str = Field("b64_json", description="Only b64_json supported.") def _params_doc(): out = {} for name, info in ImageRequest.model_fields.items(): default = info.default if default is ...: default = None entry: dict[str, Any] = {"default": default, "description": info.description} if name in ("image", "images"): # Never echo a payload back as a "default" — these are content, not settings. entry["default"] = None out[name] = entry return out @app.get("/health") def health(): return {"status": "ok" if pipe is not None else "loading", "model": SERVED_MODEL_ID} @app.get("/v1/models") def models_ep(): return {"object": "list", "data": [{"id": SERVED_MODEL_ID, "object": "model"}]} @app.get("/v1/config") def config_ep(): return { "model": SERVED_MODEL_ID, "repo_id": "Qwen/Qwen-Image-2.1", "transformer": TRANSFORMER_DIR or "base", "pipeline_class": "QwenImage21Pipeline", "endpoints": ["POST /v1/images/generations", "POST /v1/images/edits"], "capabilities": [ "text-to-image", "image-edit", "transparent-rgba", f"multi-reference (up to {MAX_REFERENCE_IMAGES} images)", ], "params": _params_doc(), "memory": {"load_mode": LOAD_MODE, "vae_tiling": VAE_TILING}, "turbo_lora": { "active": LORA_STATE["active"], "path": LORA_PATH or None, "repo": "Viggle/Qwen-Image-2.1-viggle-turbo", "effective_default_steps": _default_steps(), "note": ("Viggle few-step distillation LoRA on this same base transformer. " "When active the default drops to 6 steps with no CFG (~5x faster, " "'hard to tell apart' from the 40-step base on most prompts per " "Viggle; the gaps are small dense text and complicated edits). " "Both remain overridable per request. Qwen Research license, same " "as the base — still non-commercial."), }, "native_aspect_ratios": NATIVE_ASPECT_RATIOS, "transparency": { "how": "Ask for it in the prompt; the 4-channel VAE emits alpha directly.", "prompt_template": ( "This is an RGBA image with transparency. . " "The image has alpha channel and the background is transparent." ), }, "license": "Qwen Research License (non-commercial)", } def _collect_images(req: ImageRequest) -> Optional[list]: """Normalize `image` / `images` into a list of PIL images, or None.""" if req.image is not None and req.images is not None: raise HTTPException(400, "send either `image` or `images`, not both") raw = req.images if req.images is not None else req.image if raw is None: return None if isinstance(raw, str): raw = [raw] if not raw: return None if len(raw) > MAX_REFERENCE_IMAGES: raise HTTPException(400, f"at most {MAX_REFERENCE_IMAGES} reference images ({len(raw)} sent)") out = [] for i, b64 in enumerate(raw): try: img = Image.open(io.BytesIO(base64.b64decode(b64))) img.load() except Exception as e: raise HTTPException(400, f"image[{i}] is not decodable base64 PNG/JPEG: {e}") # No .convert() here: the pipeline promotes to RGBA itself, and forcing RGB # first would throw away the alpha this model is meant to edit. out.append(img) return out def _resolve_size(req: ImageRequest) -> tuple[Optional[int], Optional[int]]: """Explicit `size` wins; otherwise let the pipeline derive from output_resolution (and, in edit mode, from the reference image's aspect ratio).""" if req.size is None: return None, None try: w_str, h_str = req.size.lower().split("x") w, h = int(w_str), int(h_str) except Exception: raise HTTPException(400, "size must be 'WIDTHxHEIGHT', e.g. 2048x2048") if not (256 <= w <= 4096 and 256 <= h <= 4096): raise HTTPException(400, "each dimension must be in [256, 4096]") return w, h def _run(req: ImageRequest, require_image: bool): if pipe is None: raise HTTPException(503, "Model not loaded") if req.response_format != "b64_json": raise HTTPException(400, "only response_format=b64_json is supported") condition_images = _collect_images(req) if require_image and not condition_images: raise HTTPException(400, "/v1/images/edits requires `image` (or `images`)") width, height = _resolve_size(req) # Unset steps resolve here, not at class-definition time, so the turbo LoRA's # 6 and the base model's 40 both come from the same request contract. steps = req.num_inference_steps if req.num_inference_steps is not None else _default_steps() # The pipeline turns true CFG off unless a negative prompt exists, and only # logs a warning. Substitute an empty one so raising the knob has the effect # the caller plainly intended, and say so in params_used. negative_prompt = req.negative_prompt cfg_needs_negative = req.true_cfg_scale > 1.0 and negative_prompt is None if cfg_needs_negative: negative_prompt = DEFAULT_NEGATIVE if req.seed is not None: generator = [torch.Generator("cuda").manual_seed(req.seed + i) for i in range(req.n)] else: generator = None queued0 = time.time() with _gpu_lock: queued_s = time.time() - queued0 if queued_s > 1.0: print(f" waited {queued_s:.1f}s for the GPU (another request was running)", flush=True) t0 = time.time() with torch.inference_mode(): images = pipe( prompt=req.prompt, image=condition_images, negative_prompt=negative_prompt, true_cfg_scale=req.true_cfg_scale, width=width, height=height, output_resolution=req.output_resolution, num_inference_steps=steps, num_images_per_prompt=req.n, use_kv_cache=req.use_kv_cache, generator=generator, ).images elapsed = time.time() - t0 data = [] for img in images: buf = io.BytesIO() img.save(buf, format="PNG") # PNG so a native-alpha result survives the trip data.append({"b64_json": base64.b64encode(buf.getvalue()).decode()}) # Report what was rendered, not what was asked: the pipeline floors both # dimensions and, in edit mode, may derive them from the reference aspect. actual = f"{images[0].width}x{images[0].height}" if images else None modes = sorted({img.mode for img in images}) print( f"generated {len(images)} × {actual} in {elapsed:.2f}s " f"(steps={steps}, true_cfg={req.true_cfg_scale}, " f"refs={len(condition_images) if condition_images else 0}, " f"kv_cache={req.use_kv_cache}, seed={req.seed}, mode={'/'.join(modes)})", flush=True, ) return { "created": int(time.time()), "data": data, "params_used": { "prompt": req.prompt, "negative_prompt": negative_prompt, "n": req.n, "size_requested": req.size, "size": actual, "image_mode": modes, "output_resolution": req.output_resolution, "num_inference_steps": steps, "turbo_lora": LORA_STATE["active"], "true_cfg_scale": req.true_cfg_scale, "use_kv_cache": req.use_kv_cache, "reference_images": len(condition_images) if condition_images else 0, "seed": req.seed, "elapsed_s": round(elapsed, 2), "queued_s": round(queued_s, 2), **({"_note": "true_cfg_scale > 1 with no negative_prompt — substituted the default negative prompt so CFG engages"} if cfg_needs_negative else {}), }, } @app.post("/v1/images/generations") def images_generations(req: ImageRequest): return _run(req, require_image=False) @app.post("/v1/images/edits") def images_edits(req: ImageRequest): return _run(req, require_image=True) if __name__ == "__main__": load_model() uvicorn.run(app, host="0.0.0.0", port=PORT)