mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Every mention of another inference engine is rewritten to say what our code does, or the math it computes. Reference values are "the reference value from the published config" or "the recorded reference output", line citations into another engine's source become the rule itself, comparisons with other engines are dropped, and asides about other engines' bugs go. Identifiers follow: - CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels' `hanzo_rotary`; the shuffle/ldg macros are HANZO_*. - GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side. - Tests: `*_matches_vllm` and friends name what they compare against (the reference fixture, the tile rule). - MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op is `rotary-embedding`. - hanzo-bench's board scores ours against a "rival" whose name comes from the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s` columns; dossier.sh and spec_decode.sh take `--rival-bench`. - install.ps1 installs this repository's hanzo-cli, as install.sh does, with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL. - bench_http.py reads only the hanzo: metric families. Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and tools/upstream-map.sh; each ported file's copyright/licence/provenance header; Cargo.toml dependency names; data identifiers (Hugging Face repo IDs, fixture file names and fixture metadata keys the tests read, on-disk paths, executable names the GPU and engine guards match on a host); model and format names. LLM.md says the rule once, under "One name: Hanzo Engine". Co-authored-by: Hanzo Dev <[email protected]>
166 lines
9.6 KiB
Bash
166 lines
9.6 KiB
Bash
#!/usr/bin/env bash
|
|
# dossier.sh — collect count-based prefill/decode samples for the SOTA dossier.
|
|
#
|
|
# Runs OUR `hanzo-engine bench` (--json: raw per-rep [wall_s, tokens], immune to any
|
|
# self-reported-rate bug) and the rival engine's bench binary (-o json rows) on the SAME GGUF,
|
|
# the SAME shapes, on a VERIFIED-QUIET box, and records a pinned manifest so the
|
|
# numbers are reproducible. Timing is wall-clock for BOTH engines (identical
|
|
# method => no cross-engine instrument bias); repetition + Student-t CIs quantify
|
|
# residual noise; the quiet-gate removes contention. Scoring is `hanzo-engine board score`.
|
|
#
|
|
# dossier.sh --backend rocm|vulkan|metal|cuda \
|
|
# --hanzo-engine PATH --rival-bench PATH --rival-name NAME \
|
|
# --model GGUF-or-hub-id --model-tag qwen3-1p7b \
|
|
# [-p 512,500 -n 128 -r 7] [--concurrency 1] [--max-ctx 4096] \
|
|
# [--hanzo-args "--dflash DRAFT"] \
|
|
# [--engine-dir DIR --rival-dir DIR] [--out DIR] [--force]
|
|
#
|
|
# --hanzo-args reach `hanzo-engine bench` verbatim, before the model selection, and are recorded
|
|
# in the manifest; speculation goes there. A GGUF file loads as `text --format gguf`, anything
|
|
# else as `auto -m` (a Hub id), with paged attention sized to --max-ctx as a server sizes it, so
|
|
# what is measured is what is served.
|
|
#
|
|
# The rival bench is called as `<bench> -m GGUF -p P -n N -fa 1 -r REPS -o json` and its rows
|
|
# land in rival_<tag>.json; --rival-name is the subject its rows are filed under, and --rival-dir
|
|
# its source tree (commit and dirty state go in the manifest).
|
|
#
|
|
# One GPU workload per box: the quiet-gate HARD-FAILS if another GPU job (any command
|
|
# named in COMPET below: compilers, profilers, engines) is already running. Desktop CPU load
|
|
# is recorded, not fatal.
|
|
set -euo pipefail
|
|
|
|
BACKEND="" HANZO="" RIVAL="" RIVAL_NAME="" MODEL="" MTAG="" OUT="" FORCE=0 STOCH=0 HARGS=""
|
|
PLIST="512" NGEN=128 REPS=7 CONC=1 MAXCTX=4096
|
|
ENGINE_DIR="$HOME/work/hanzo/engine" RIVAL_DIR=""
|
|
while [[ $# -gt 0 ]]; do case "$1" in
|
|
--backend) BACKEND="$2"; shift 2;;
|
|
--hanzo-engine) HANZO="$2"; shift 2;;
|
|
--rival-bench) RIVAL="$2"; shift 2;;
|
|
--rival-name) RIVAL_NAME="$2"; shift 2;;
|
|
--model) MODEL="$2"; shift 2;;
|
|
--model-tag) MTAG="$2"; shift 2;;
|
|
-p) PLIST="$2"; shift 2;;
|
|
-n) NGEN="$2"; shift 2;;
|
|
-r) REPS="$2"; shift 2;;
|
|
--concurrency) CONC="$2"; shift 2;;
|
|
--max-ctx) MAXCTX="$2"; shift 2;;
|
|
--hanzo-args) HARGS="$2"; shift 2;;
|
|
--engine-dir) ENGINE_DIR="$2"; shift 2;;
|
|
--rival-dir) RIVAL_DIR="$2"; shift 2;;
|
|
--out) OUT="$2"; shift 2;;
|
|
--force) FORCE=1; shift;;
|
|
--stochastic) STOCH=1; shift;;
|
|
*) echo "unknown arg: $1" >&2; exit 2;;
|
|
esac; done
|
|
: "${BACKEND:?--backend required}" "${HANZO:?--hanzo-engine required}" "${MODEL:?--model required}" "${MTAG:?--model-tag required}"
|
|
[[ -z "$RIVAL" || -n "$RIVAL_NAME" ]] || { echo "--rival-bench needs --rival-name" >&2; exit 2; }
|
|
# Sampler is an explicit, recorded choice. Default greedy argmax: the device picks each token, as
|
|
# a bench of the bare forward pass decodes; --stochastic reproduces the full-vocab multinomial
|
|
# only to MEASURE its tax.
|
|
SAMPLER="$([[ "$STOCH" == 1 ]] && echo stochastic-temp1-fullvocab || echo greedy-argmax)"
|
|
[[ -x "$HANZO" ]] || { echo "no hanzo-engine: $HANZO" >&2; exit 2; }
|
|
OUT="${OUT:-$ENGINE_DIR/bench-runs/$(hostname -s)-$BACKEND-$MTAG-$(date +%Y%m%d-%H%M%S)}"
|
|
mkdir -p "$OUT"
|
|
if [[ -f "$MODEL" ]]; then
|
|
MODEL_ARGS=(text --format gguf -m "$(cd "$(dirname "$MODEL")" && pwd)" -f "$(basename "$MODEL")")
|
|
else
|
|
MODEL_ARGS=(auto -m "$MODEL")
|
|
fi
|
|
|
|
# --- backend runtime env (one place) -------------------------------------------
|
|
backend_env() { case "$BACKEND" in
|
|
rocm) echo "LD_LIBRARY_PATH=/opt/rocm/lib HSA_XNACK=1";;
|
|
vulkan) echo "";;
|
|
*) echo "";;
|
|
esac; }
|
|
|
|
# --- GPU busy % (best-effort, per backend) -------------------------------------
|
|
gpu_busy() { case "$BACKEND" in
|
|
rocm|vulkan) rocm-smi --showuse 2>/dev/null | grep -oE "GPU use \(%\): [0-9]+" | grep -oE "[0-9]+$" | head -1;;
|
|
cuda) nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null | head -1;;
|
|
*) echo "na";;
|
|
esac; }
|
|
|
|
# --- quiet gate: one GPU workload per box --------------------------------------
|
|
COMPET='ncu|nsys|rocprof|rocprofv3|metal-capture|xctrace|cargo|rustc|hanzo-engine|llama-bench|llama-server|ollama|vllm|sglang'
|
|
quiet_gate() {
|
|
# The real GPU-contention signals are (a) GPU busy% and (b) a true GPU-compute
|
|
# process. Match the competitor list against the COMMAND NAME, not the full
|
|
# argv, so a guard daemon (earlyoom, whose --prefer string names these tools) or
|
|
# a network-bound eval (python calling an API) that merely mentions them does not
|
|
# false-positive. Desktop/CPU load is recorded as advisory, never fatal -- a
|
|
# shared workstation is never fully idle, and CV>5% flagging in `hanzo-engine board score`
|
|
# surfaces any timing noise it induces.
|
|
local hits busy load
|
|
hits="$(ps -eo pid=,comm= 2>/dev/null | grep -wE "$COMPET" | grep -vw "$$" || true)"
|
|
busy="$(gpu_busy)"; load="$(uptime | grep -oE 'load average.*' || true)"
|
|
{ echo "quiet-gate @ $(date +%Y-%m-%dT%H:%M:%S%z)"; echo "gpu_busy%: $busy"; echo "$load";
|
|
echo "gpu/compile competitors (by command name):"; echo "${hits:- (none)}"; } | tee "$OUT/quiet-gate.txt"
|
|
if [[ -n "$hits" && "$FORCE" != 1 ]]; then
|
|
echo "REFUSING: a GPU/compile workload is running (see above). Re-run when quiet or --force." >&2; exit 3; fi
|
|
# A numeric-expected backend (rocm/vulkan/cuda) whose busy% cannot be read is a gate
|
|
# FAILURE, not a silent degrade to the process gate: an unreadable accelerator is exactly
|
|
# when a stale or contending job hides. Metal has no CLI busy% and is process-gated by design.
|
|
if [[ "$BACKEND" != metal && ! "$busy" =~ ^[0-9]+$ && "$FORCE" != 1 ]]; then
|
|
echo "REFUSING: GPU busy%% unreadable on $BACKEND (got '$busy'); cannot verify idle. --force to override." >&2; exit 3; fi
|
|
if [[ "$busy" =~ ^[0-9]+$ && "$busy" -gt 15 && "$FORCE" != 1 ]]; then
|
|
echo "REFUSING: GPU busy ${busy}% > 15%. Re-run when idle or --force." >&2; exit 3; fi
|
|
}
|
|
|
|
# --- pinned manifest ------------------------------------------------------------
|
|
ver_of() { grep -A1 "^name = \"$1\"" "$ENGINE_DIR/Cargo.lock" 2>/dev/null | grep version | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+'; }
|
|
dirty_of() { [[ -n "$(git -C "$1" status --porcelain 2>/dev/null)" ]] && echo true || echo false; }
|
|
manifest() {
|
|
local ever egit gpu
|
|
ever="$(grep -m1 '^version' "$ENGINE_DIR/Cargo.toml" 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')"
|
|
egit="$(git -C "$ENGINE_DIR" rev-parse HEAD 2>/dev/null || true)"
|
|
gpu="$(rocminfo 2>/dev/null | grep -m1 'Marketing Name' | sed 's/^ *//' || nvidia-smi --query-gpu=name,driver_version --format=csv,noheader 2>/dev/null | head -1 || true)"
|
|
# Values pass as arguments, never interpolated into a script: a model path with a space or a
|
|
# quote cannot break the manifest. The binaries are hashed and each tree's dirty state
|
|
# recorded -- a binary built from an uncommitted tree carries a clean commit SHA and would
|
|
# otherwise misreport its provenance.
|
|
"$HANZO" board manifest "$OUT/manifest.json" \
|
|
--backend "$BACKEND" --sampler "$SAMPLER" --gpu "$gpu" \
|
|
--engine-git-sha "$egit" --engine-git-dirty "$(dirty_of "$ENGINE_DIR")" --engine-version "$ever" \
|
|
--hanzo-ml "$(ver_of hanzo-ml)" --hanzo-rocm-kernels "$(ver_of hanzo-rocm-kernels)" \
|
|
--hanzo-metal-kernels "$(ver_of hanzo-metal-kernels)" \
|
|
--rival "$RIVAL_NAME" \
|
|
--rival-sha "$( [[ -n "$RIVAL_DIR" ]] && git -C "$RIVAL_DIR" rev-parse HEAD 2>/dev/null || true)" \
|
|
--rival-git-dirty "$( [[ -n "$RIVAL_DIR" ]] && dirty_of "$RIVAL_DIR" || echo false)" \
|
|
--hanzo-bench-bin "$HANZO" --rival-bench-bin "$RIVAL" --model-path "$MODEL" \
|
|
--prompt-sizes "$PLIST" --n-gen "$NGEN" --reps "$REPS" --concurrency "$CONC" --max-ctx "$MAXCTX" \
|
|
--hanzo-args "$HARGS" --backend-env "$(backend_env)"
|
|
}
|
|
|
|
run_hanzo() { # $1=phase(pp|tg) $2=n $3=tag
|
|
# The device map is the server's automatic one, every layer on the GPU here.
|
|
local p g; if [[ "$1" == pp ]]; then p="$2"; g=0; else p=0; g="$2"; fi
|
|
echo ">> hanzo-engine bench $BACKEND $3 (p=$p g=$g r=$REPS c=$CONC)" >&2
|
|
env $(backend_env) "$HANZO" bench --prompt-len "$p" --gen-len "$g" --repetitions "$REPS" --concurrency "$CONC" \
|
|
$([[ "$STOCH" == 1 ]] && echo --stochastic) --json "$OUT/hanzo_${3}.json" $HARGS \
|
|
"${MODEL_ARGS[@]}" --max-seq-len "$MAXCTX" --paged-attn on --pa-context-len "$MAXCTX" \
|
|
> "$OUT/hanzo_${3}.log" 2>&1 || { echo "hanzo-engine bench FAILED ($3); tail:" >&2; tail -8 "$OUT/hanzo_${3}.log" >&2; }
|
|
}
|
|
run_rival() { # $1=phase(pp|tg) $2=n $3=tag
|
|
[[ -x "$RIVAL" ]] || { echo '{"skipped":"no rival bench"}' > "$OUT/rival_${3}.json"; return; }
|
|
local p n; if [[ "$1" == pp ]]; then p="$2"; n=0; else p=0; n="$2"; fi
|
|
echo ">> $RIVAL_NAME bench $BACKEND $3 (p=$p n=$n r=$REPS)" >&2
|
|
env $(backend_env) "$RIVAL" -m "$MODEL" -p "$p" -n "$n" -fa 1 -r "$REPS" -o json \
|
|
> "$OUT/rival_${3}.json" 2> "$OUT/rival_${3}.err" || { echo "$RIVAL_NAME bench FAILED ($3)" >&2; tail -5 "$OUT/rival_${3}.err" >&2; }
|
|
}
|
|
|
|
echo "=== dossier: $BACKEND / $MTAG -> $OUT ===" >&2
|
|
quiet_gate
|
|
manifest || echo "manifest step failed (non-fatal); continuing" >&2
|
|
# decode (one shared run), then each prefill size (ragged included via -p list)
|
|
run_hanzo tg "$NGEN" "${MTAG}_${BACKEND}_tg${NGEN}"
|
|
run_rival tg "$NGEN" "${MTAG}_${BACKEND}_tg${NGEN}"
|
|
IFS=',' read -ra PS <<< "$PLIST"
|
|
for P in "${PS[@]}"; do
|
|
run_hanzo pp "$P" "${MTAG}_${BACKEND}_pp${P}"
|
|
run_rival pp "$P" "${MTAG}_${BACKEND}_pp${P}"
|
|
done
|
|
"$HANZO" board score "$OUT" >&2 || echo "scoring failed; the samples are in $OUT" >&2
|
|
echo "=== done: $OUT ===" >&2
|
|
echo "$OUT"
|