Files
engine/scripts/blockwise_fp8_e2e.sh
zeekayandHanzo Dev 42e34c2d41 refactor: name only Hanzo Engine in code, comments and docs
Every mention of another inference engine is rewritten to say what our code
does, or the math it computes. Reference values are "the reference value
from the published config" or "the recorded reference output", line
citations into another engine's source become the rule itself, comparisons
with other engines are dropped, and asides about other engines' bugs go.

Identifiers follow:
- CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's
  MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels'
  `hanzo_rotary`; the shuffle/ldg macros are HANZO_*.
- GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is
  `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side.
- Tests: `*_matches_vllm` and friends name what they compare against (the
  reference fixture, the tile rule).
- MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op
  is `rotary-embedding`.
- hanzo-bench's board scores ours against a "rival" whose name comes from
  the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s`
  columns; dossier.sh and spec_decode.sh take `--rival-bench`.
- install.ps1 installs this repository's hanzo-cli, as install.sh does,
  with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL.
- bench_http.py reads only the hanzo: metric families.

Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and
tools/upstream-map.sh; each ported file's copyright/licence/provenance
header; Cargo.toml dependency names; data identifiers (Hugging Face repo
IDs, fixture file names and fixture metadata keys the tests read, on-disk
paths, executable names the GPU and engine guards match on a host); model
and format names.

LLM.md says the rule once, under "One name: Hanzo Engine".


Co-authored-by: Hanzo Dev <[email protected]>
2026-10-16 20:59:50 -07:00

164 lines
6.6 KiB
Bash

#!/usr/bin/env bash
# Q4 end to end: Qwen3.8-Flash-Next (fp8hybrid snapshot) served by hanzo with
# --fp8-block-gemm-backend auto (the default) and scalar, no speculation.
#
# scripts/blockwise_fp8_e2e.sh --out DIR [--plan]
#
# It needs the box to itself (~75 GiB resident): the user stops the server on :18300 first and
# restarts it afterwards. This script never touches that server, and refuses to run while :18300
# listens.
#
# Per backend: serve, then
# - W0's harness (scripts/bench_http.py): c=1 decode of 512 tokens x5, warm prefill at ~8K and ~32K;
# - nsys over 20 decode steps: block-FP8 kernel µs per step;
# - greedy output identical across 3 runs;
# - the startup log's resolution line.
# Exits 1 unless: auto logs exactly one "fp8-block-gemm-backend auto -> cutlass" line and no
# fallback WARN; cutlass block-FP8 GEMM µs per decode step <= 1.05 x GATE_M1_SUM_US, the M=1
# GEMM time budget in µs (T6's per-shape minima at M=1, summed over the model's GEMM shapes);
# auto decode beats scalar;
# greedy output is identical across 3 runs per backend.
#
# Inputs: HANZO_ENGINE_BIN (default <repo>/target/release/hanzo-engine), PORT (default 30000),
# SNAPSHOT (default the fp8hybrid snapshot below), GATE_M1_SUM_US (required).
set -euo pipefail
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO=$(dirname "$HERE")
BIN=${HANZO_ENGINE_BIN:-$REPO/target/release/hanzo-engine}
PORT=${PORT:-30000}
SNAPSHOT=${SNAPSHOT:-$HOME/.cache/huggingface/hub/models--nvidia--Qwen3.8-Flash-Next-NVFP4/snapshots/fc694b54fb0174e0913e6adf86691ef85a4ead47-fp8hybrid}
URL=http://127.0.0.1:$PORT/v1/chat/completions
MIN_GIB=100
OUT=""
PLAN=0
while [ $# -gt 0 ]; do
case $1 in
--out) OUT=$2; shift 2 ;;
--plan) PLAN=1; shift ;;
*) echo "usage: blockwise_fp8_e2e.sh --out DIR [--plan]" >&2; exit 2 ;;
esac
done
[ -n "$OUT" ] || { echo "usage: blockwise_fp8_e2e.sh --out DIR [--plan]" >&2; exit 2; }
serve_cmd() { # serve_cmd <backend> [extra...]
local backend=$1
shift
printf '%s\n' "$BIN" serve -m "$SNAPSHOT" --port "$PORT" --fp8-block-gemm-backend "$backend" "$@"
}
if [ "$PLAN" = 1 ]; then
for b in auto scalar; do
mapfile -t argv < <(serve_cmd "$b")
printf '%q ' "${argv[@]}"
printf '\n'
done
exit 0
fi
# Preflight: every refusal happens before anything starts.
if ss -ltn 2>/dev/null | awk '{print $4}' | grep -q ':18300$'; then
echo "e2e: :18300 is listening; a full model needs that server stopped" >&2
exit 75
fi
mem=$(awk '/^MemAvailable:/ {printf "%d", $2 / 1048576}' /proc/meminfo)
[ "$mem" -ge "$MIN_GIB" ] || { echo "e2e: MemAvailable $mem GiB < $MIN_GIB GiB" >&2; exit 75; }
[ -x "$BIN" ] || { echo "e2e: no binary at $BIN" >&2; exit 1; }
[ -d "$SNAPSHOT" ] || { echo "e2e: no snapshot at $SNAPSHOT" >&2; exit 1; }
[ -f "$HERE/bench_http.py" ] || { echo "e2e: scripts/bench_http.py (W0's harness) is missing" >&2; exit 1; }
[ -n "${GATE_M1_SUM_US:-}" ] || { echo "e2e: set GATE_M1_SUM_US, the M=1 GEMM budget in us (T6's minima)" >&2; exit 2; }
command -v nsys >/dev/null || { echo "e2e: nsys not on PATH" >&2; exit 1; }
mkdir -p "$OUT"
PID=""
stop() { [ -z "$PID" ] || { kill "$PID" 2>/dev/null || true; wait "$PID" 2>/dev/null || true; PID=""; }; }
trap stop EXIT
wait_ready() {
for _ in $(seq 1 900); do
curl -sf "http://127.0.0.1:$PORT/v1/models" >/dev/null && return 0
kill -0 "$PID" 2>/dev/null || { echo "e2e: server exited during load" >&2; return 1; }
sleep 2
done
echo "e2e: server not ready after 30 min" >&2
return 1
}
greedy() { # greedy <file>: one deterministic completion
curl -sf "$URL" -H 'content-type: application/json' -d '{
"model": "default", "temperature": 0, "max_tokens": 256, "seed": 0,
"messages": [{"role": "user", "content": "List the first twelve prime numbers and explain the sieve of Eratosthenes."}]
}' | python3 -c 'import json,sys; print(json.load(sys.stdin)["choices"][0]["message"]["content"])' >"$1"
}
fail=0
declare -A DECODE
for backend in auto scalar; do
dir=$OUT/$backend
mkdir -p "$dir"
mapfile -t argv < <(serve_cmd "$backend")
"${argv[@]}" >"$dir/serve.log" 2>&1 &
PID=$!
wait_ready
for i in 1 2 3; do greedy "$dir/greedy.$i.txt"; done
if ! cmp -s "$dir/greedy.1.txt" "$dir/greedy.2.txt" || ! cmp -s "$dir/greedy.1.txt" "$dir/greedy.3.txt"; then
echo "FAIL $backend: greedy output differs across 3 runs" >&2
fail=1
fi
for i in 1 2 3 4 5; do
python3 "$HERE/bench_http.py" "$URL" --gen 512 --sizes 4 --label "$backend-decode-$i" --json "$dir/decode.jsonl"
done
# ~8K and ~32K prompt tokens; the second pass is the warm one.
for pass in cold warm; do
python3 "$HERE/bench_http.py" "$URL" --gen 16 --sizes 260,1030 --label "$backend-prefill-$pass" --json "$dir/prefill.jsonl"
done
DECODE[$backend]=$(python3 -c '
import json, sys
rates = [json.loads(l).get("decode_tps") for l in open(sys.argv[1])]
rates = [r for r in rates if r]
print(max(rates) if rates else 0)' "$dir/decode.jsonl")
stop
if [ "$backend" = auto ]; then
lines=$(grep -c 'fp8-block-gemm-backend auto -> cutlass' "$dir/serve.log" || true)
warns=$(grep -c 'cutlass lib not built for sm_121' "$dir/serve.log" || true)
if [ "$lines" != 1 ] || [ "$warns" != 0 ]; then
echo "FAIL auto: $lines 'auto -> cutlass' lines, $warns fallback WARNs" >&2
fail=1
fi
fi
# 20 decode steps under nsys: block-FP8 GEMM µs per step.
mapfile -t argv < <(serve_cmd "$backend" --profiler cuda --profiler-max-iterations 20)
nsys profile --trace=cuda --capture-range=cudaProfilerApi --capture-range-end=stop \
-o "$dir/decode" "${argv[@]}" >"$dir/nsys.log" 2>&1 &
PID=$!
wait_ready
python3 "$HERE/bench_http.py" "$URL" --gen 64 --sizes 4 --label "$backend-nsys" >/dev/null
stop
nsys stats --report cuda_gpu_kern_sum --format csv -o "$dir/kern" "$dir/decode.nsys-rep" >/dev/null
per_step=$(python3 -c '
import csv, sys
total = 0.0
for row in csv.DictReader(open(sys.argv[1])):
name = row["Name"]
if "hanzo_blockwise_fp8" in name or "Sm120Blockwise" in name or "w8a8" in name or "BlockwiseScaling" in name:
total += float(row["Total Time (ns)"])
print(total / 1e3 / 20)' "$dir/kern_cuda_gpu_kern_sum.csv")
echo "$backend: block-FP8 GEMM $per_step us per decode step"
if [ "$backend" = auto ] && python3 -c "import sys; sys.exit(0 if $per_step <= 1.05 * $GATE_M1_SUM_US else 1)"; then :; elif [ "$backend" = auto ]; then
echo "FAIL auto: $per_step us per step > 1.05 x budget $GATE_M1_SUM_US us" >&2
fail=1
fi
done
echo "decode tok/s: auto ${DECODE[auto]}, scalar ${DECODE[scalar]}"
if ! python3 -c "import sys; sys.exit(0 if ${DECODE[auto]} > ${DECODE[scalar]} else 1)"; then
echo "FAIL: auto decode is not faster than scalar" >&2
fail=1
fi
exit $fail