m64_5deb4065b1cf / start.sh
43 lines, 2.0 KBraw
1#!/usr/bin/env bash2# qwen38-flash-next-llamacpp-gb10 entrypoint. Runs inside the container; takes no arguments.3# llama-server (llama.cpp v1.0.0) serves the model on 127.0.0.1:8080; proxy.py puts it on $M64_PORT.4# The GGUF shards and the picture reader are mounted read-only under /weights; nothing is fetched at runtime.5# Shard 1 (the weights) is memory-mapped and offloaded whole to the GPU (one copy in the GB10's unified memory);6# shard 2 (the 29 GB PLE n-gram table) is given with --ple and read lazily, one row per token, from the page7# cache (--lazy-mode on), as the model card recommends.8set -eu9QUANT=${QUANT:-IQ3_S}10CONTEXT=${CONTEXT:-262144}11VISION=${VISION:-on}12export M64_MODEL_NAME=${M64_MODEL_NAME:-qwen3.8-flash-next-iq3_s-llamacpp}13status() { printf '{"phase":"%s","detail":"%s"}\n' "$1" "$2" >> "$M64_STATUS"; }1415W=/weights/qwen38fn16S1=$(ls "$W/$QUANT"/*-00001-of-00002.gguf)17S2=$(ls "$W/$QUANT"/*-00002-of-00002.gguf)18args=(-m "$S1" --ple "$S2" --alias "$M64_MODEL_NAME"19 --host 127.0.0.1 --port 808020 -c "$CONTEXT" -np "${PARALLEL:-1}" -ngl 999 -fa on21 -ctk q8_0 -ctv q8_0 -b 2048 -ub 204822 -lm mmap --lazy-mode on23 --temp 1.0 --top-k 20 --top-p 0.95 --min-p 0 # the GGUF's general.sampling values24 --jinja --no-ui --metrics)25# At least 1024 tokens per picture: below that Qwen-VL loses fine detail (llama.cpp warns about it at load).26if [ "$VISION" = on ]; then args+=(--mmproj "$W/mmproj-Qwen3.8-Flash-Next-BF16.gguf" --image-min-tokens 1024); fi2728status loading "loading $QUANT into memory (about 84 GB from disk on a cold start)"29/opt/llama/bin/llama-server "${args[@]}" &30LLAMA=$!31trap 'kill $LLAMA 2>/dev/null' TERM INT32until curl -sf http://127.0.0.1:8080/health >/dev/null; do33 kill -0 $LLAMA 2>/dev/null || { status error "llama-server exited during load"; wait $LLAMA; exit 1; }34 sleep 235done3637/usr/bin/python3 -u /capsule/proxy.py &38PROXY=$!39trap 'kill $LLAMA $PROXY 2>/dev/null' TERM INT40# Whichever stops first ends the capsule.41wait -n $LLAMA $PROXY42kill $LLAMA $PROXY 2>/dev/null || true43exit 1