#!/usr/bin/env bash # qwen38-flash-next-llamacpp-gb10 entrypoint. Runs inside the container; takes no arguments. # llama-server (llama.cpp v1.0.0) serves the model on 127.0.0.1:8080; proxy.py puts it on $M64_PORT. # The GGUF shards and the picture reader are mounted read-only under /weights; nothing is fetched at runtime. # Shard 1 (the weights) is memory-mapped and offloaded whole to the GPU (one copy in the GB10's unified memory); # shard 2 (the 29 GB PLE n-gram table) is given with --ple and read lazily, one row per token, from the page # cache (--lazy-mode on), as the model card recommends. set -eu QUANT=${QUANT:-IQ3_S} CONTEXT=${CONTEXT:-65536} VISION=${VISION:-on} export M64_MODEL_NAME=${M64_MODEL_NAME:-qwen3.8-flash-next-iq3_s-llamacpp} status() { printf '{"phase":"%s","detail":"%s"}\n' "$1" "$2" >> "$M64_STATUS"; } W=/weights/qwen38fn S1=$(ls "$W/$QUANT"/*-00001-of-00002.gguf) S2=$(ls "$W/$QUANT"/*-00002-of-00002.gguf) args=(-m "$S1" --ple "$S2" --alias "$M64_MODEL_NAME" --host 127.0.0.1 --port 8080 -c "$CONTEXT" -np "${PARALLEL:-1}" -ngl 999 -fa on -ctk q8_0 -ctv q8_0 -b 2048 -ub 2048 -lm mmap --lazy-mode on --temp 1.0 --top-k 20 --top-p 0.95 --min-p 0 # the GGUF's general.sampling values --jinja --no-ui --metrics) if [ "$VISION" = on ]; then args+=(--mmproj "$W/mmproj-Qwen3.8-Flash-Next-BF16.gguf"); fi status loading "loading $QUANT into memory (about 84 GB from disk on a cold start)" /opt/llama/bin/llama-server "${args[@]}" & LLAMA=$! trap 'kill $LLAMA 2>/dev/null' TERM INT until curl -sf http://127.0.0.1:8080/health >/dev/null; do kill -0 $LLAMA 2>/dev/null || { status error "llama-server exited during load"; wait $LLAMA; exit 1; } sleep 2 done /usr/bin/python3 -u /capsule/proxy.py & PROXY=$! trap 'kill $LLAMA $PROXY 2>/dev/null' TERM INT # Whichever stops first ends the capsule. wait -n $LLAMA $PROXY kill $LLAMA $PROXY 2>/dev/null || true exit 1