motif64

m64_ef24b3f62fa7 / start.sh

42 lines, 1.9 KBraw
1#!/usr/bin/env bash2# qwen38-flash-next-llamacpp-gb10 entrypoint. Runs inside the container; takes no arguments.3# llama-server (llama.cpp v1.0.0) serves the model on 127.0.0.1:8080; proxy.py puts it on $M64_PORT.4# The GGUF shards and the picture reader are mounted read-only under /weights; nothing is fetched at runtime.5# Shard 1 (the weights) is memory-mapped and offloaded whole to the GPU (one copy in the GB10's unified memory);6# shard 2 (the 29 GB PLE n-gram table) is given with --ple and read lazily, one row per token, from the page7# cache (--lazy-mode on), as the model card recommends.8set -eu9QUANT=${QUANT:-IQ3_S}10CONTEXT=${CONTEXT:-65536}11VISION=${VISION:-on}12export M64_MODEL_NAME=${M64_MODEL_NAME:-qwen3.8-flash-next-iq3_s-llamacpp}13status() { printf '{"phase":"%s","detail":"%s"}\n' "$1" "$2" >> "$M64_STATUS"; }1415W=/weights/qwen38fn16S1=$(ls "$W/$QUANT"/*-00001-of-00002.gguf)17S2=$(ls "$W/$QUANT"/*-00002-of-00002.gguf)18args=(-m "$S1" --ple "$S2" --alias "$M64_MODEL_NAME"19      --host 127.0.0.1 --port 808020      -c "$CONTEXT" -np "${PARALLEL:-1}" -ngl 999 -fa on21      -ctk q8_0 -ctv q8_0 -b 2048 -ub 204822      -lm mmap --lazy-mode on23      --temp 1.0 --top-k 20 --top-p 0.95 --min-p 0  # the GGUF's general.sampling values24      --jinja --no-ui --metrics)25if [ "$VISION" = on ]; then args+=(--mmproj "$W/mmproj-Qwen3.8-Flash-Next-BF16.gguf"); fi2627status loading "loading $QUANT into memory (about 84 GB from disk on a cold start)"28/opt/llama/bin/llama-server "${args[@]}" &29LLAMA=$!30trap 'kill $LLAMA 2>/dev/null' TERM INT31until curl -sf http://127.0.0.1:8080/health >/dev/null; do32  kill -0 $LLAMA 2>/dev/null || { status error "llama-server exited during load"; wait $LLAMA; exit 1; }33  sleep 234done3536/usr/bin/python3 -u /capsule/proxy.py &37PROXY=$!38trap 'kill $LLAMA $PROXY 2>/dev/null' TERM INT39# Whichever stops first ends the capsule.40wait -n $LLAMA $PROXY41kill $LLAMA $PROXY 2>/dev/null || true42exit 1