Files
bench-harness 2f1218062f feat(bench): server-driven MoE bench harness (roofline + decode isolation)
Orchestrator (scripts/bench_moe.sh) drives a sweep of hanzo-serve launches:
per-row env/flags, /health poll + load time, resident-memory sampler
(VmRSS curve + VmHWM peak), warmup, load generator, live expert-cache readout
off /v1/system/info, graceful SIGTERM by recorded PID (never pkill/kill -9).

Load generator (scripts/bench_client.py) isolates prefill from decode via the
engine usage superset and cross-checks steady-state decode from client-side ITL
(first K dropped), with per-user prompt salts so prefix-caching can't elide
prefill. Batch phase reports aggregate vs per-stream tok/s.

Roofline probe (hanzo-bench/src/bin/roofline.rs) measures the two ceilings on
the box with no external tools: RAM via a STREAM triad (single/all-thread),
NVMe via an O_DIRECT sequential read. Report assembler (scripts/bench_report.py)
owns the schema + roofline arithmetic, emitting results.json + a rendered table.
Sweep matrix is declarative (scripts/bench_moe.toml).
2026-07-13 00:57:26 -07:00

261 lines
11 KiB
Bash

#!/usr/bin/env bash
# bench_moe — server-driven MoE bench orchestrator (pure executor of bench_moe.toml).
#
# One arg = a config file. For each row of the sweep matrix it launches `hanzo
# serve` with the row's env + flags, proves the server is up, samples resident
# memory, drives the load generator (bench_client.py) for prefill/decode/steady/
# batch, reads the expert-cache + peak RSS off the live process, then terminates
# the server gracefully (SIGTERM by the recorded PID — never pkill / kill -9). A
# roofline probe runs once at the end; bench_report.py folds every row + the probe
# into one results.json plus a rendered table.
#
# Concerns are split, not braided: this file does lifecycle + wiring; bench_client
# drives load; roofline measures bandwidth; bench_report owns the schema + math.
#
# Build/run env (REQUIRED — the sandbox cc shim rejects -m64):
# cd ~/work/hanzo/engine
# PATH="/usr/bin:$PATH:$HOME/.cargo/bin:/opt/rocm/bin" \
# CC=/usr/bin/gcc CXX=/usr/bin/g++ \
# CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=/usr/bin/gcc \
# cargo build --release -p hanzo --bin hanzo --features rocm
# cargo build --release -p hanzo-bench --bin roofline --features rocm
# Runtime adds LD_LIBRARY_PATH=/opt/rocm/lib. Engine main is v1.7.37.
#
# bench_moe.sh [-c config.toml] [-o out-dir] [--hanzo path] [--roofline path]
# [--no-hash] [--dry-run]
#
# --dry-run wires manifest -> synthetic row -> report without launching a server,
# so the harness is self-testing on everything the ROCm serve path doesn't gate.
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG="$HERE/bench_moe.toml"
OUT=""
HANZO="${HANZO:-$HERE/../target/release/hanzo}"
ROOFLINE="${ROOFLINE:-$HERE/../target/release/roofline}"
LD_ROCM="${LD_LIBRARY_PATH:-/opt/rocm/lib}"
DRY_RUN=0
DO_HASH=1
while [[ $# -gt 0 ]]; do
case "$1" in
-c) CONFIG="$2"; shift 2 ;;
-o) OUT="$2"; shift 2 ;;
--hanzo) HANZO="$2"; shift 2 ;;
--roofline) ROOFLINE="$2"; shift 2 ;;
--no-hash) DO_HASH=0; shift ;;
--dry-run) DRY_RUN=1; DO_HASH=0; shift ;;
*) echo "unknown arg: $1" >&2; exit 2 ;;
esac
done
[[ -f "$CONFIG" ]] || { echo "no config: $CONFIG" >&2; exit 2; }
OUT="${OUT:-$(mktemp -d "${TMPDIR:-/tmp}/bench_moe.XXXXXX")}"
mkdir -p "$OUT"
echo "out-dir: $OUT" >&2
# --- config accessors (python tomllib; no jq/toml shell dep) -------------------
cfg() { python3 -c "import tomllib,sys;d=tomllib.load(open(sys.argv[1],'rb'));print(d.get(sys.argv[2],''))" "$CONFIG" "$1"; }
MODEL="$(cfg model)"
PORT="$(cfg port)"; PORT="${PORT:-1234}"
WARMUP="$(cfg warmup)"; WARMUP="${WARMUP:-1}"
PROMPT_TOKENS="$(cfg prompt_tokens)"
MAX_TOKENS="$(cfg max_tokens)"
BATCH="$(cfg batch)"
MODEL_DIR="$(dirname "$MODEL")"
MODEL_FILE="$(basename "$MODEL")"
# --- reproducibility manifest --------------------------------------------------
write_manifest() {
local sha="skipped"
if [[ "$DO_HASH" == 1 && -f "$MODEL" ]]; then
sha="$(sha256sum "$MODEL" | awk '{print $1}')"
fi
local gpu="unknown"
if command -v rocminfo >/dev/null 2>&1; then
gpu="$(rocminfo 2>/dev/null | grep -i 'gfx' | head -1 | awk '{print $NF}' || true)"
gpu="${gpu:-unknown}"
fi
local rocm_ver="unknown"
if command -v hipconfig >/dev/null 2>&1; then
rocm_ver="$(hipconfig --version 2>/dev/null || echo unknown)"
fi
python3 - "$OUT/manifest.json" <<PY
import json, os, platform, subprocess, sys, time
def sh(*a):
try: return subprocess.check_output(a, text=True, stderr=subprocess.DEVNULL).strip()
except Exception: return None
m = {
"git_sha": sh("git","-C","$HERE","rev-parse","HEAD"),
"host": platform.node(),
"uname": " ".join(os.uname()),
"gpu": "$gpu",
"rocm_ver": "$rocm_ver",
"model_path": "$MODEL",
"model_sha256": "$sha",
"build_features": "rocm",
"serve_cmdline": "hanzo serve --port $PORT --isq <q> --max-seqs <n> --prefix-cache-n 0 -m $MODEL_DIR -f $MODEL_FILE --format gguf",
"env": {k: os.environ.get(k) for k in ("STREAM_EXPERTS","STREAM_EXPERTS_RAM_GB","LD_LIBRARY_PATH")},
"timestamp": time.time(),
}
json.dump(m, open(sys.argv[1],"w"), indent=2)
print("manifest ->", sys.argv[1], file=sys.stderr)
PY
}
# --- resident-memory sampler ---------------------------------------------------
rss_sampler() { # $1=pid $2=outfile ; VmRSS every 250ms while the pid lives
local pid="$1" out="$2"
while kill -0 "$pid" 2>/dev/null; do
awk -v t="$(date +%s.%N)" '/VmRSS/{print t, $2}' "/proc/$pid/status" >> "$out" 2>/dev/null || true
sleep 0.25
done
}
peak_rss_gb() { # $1=pid -> VmHWM (kB) as GB, kernel high-water mark
awk '/VmHWM/{printf "%.3f", $2/1048576}' "/proc/$1/status" 2>/dev/null || echo ""
}
# --- one sweep row -------------------------------------------------------------
run_row() { # $1=idx $2=quant $3=stream(0/1) $4=ram_gb $5=max_seqs
local idx="$1" quant="$2" se="$3" ram_gb="$4" max_seqs="$5"
local log="$OUT/row-$idx.serve.log" rss="$OUT/row-$idx.rss.tsv"
local recs="$OUT/row-$idx.records.json" sysinfo="$OUT/row-$idx.sysinfo.json"
local -a envp=(LD_LIBRARY_PATH="$LD_ROCM")
[[ "$se" == 1 ]] && envp+=(STREAM_EXPERTS=1)
[[ "$se" == 1 && -n "$ram_gb" ]] && envp+=(STREAM_EXPERTS_RAM_GB="$ram_gb")
echo "[row $idx] quant=$quant stream=$se ram_gb=${ram_gb:-inf} max_seqs=$max_seqs" >&2
local t_load0 t_load1
t_load0="$(date +%s.%N)"
env "${envp[@]}" "$HANZO" serve --port "$PORT" --isq "$quant" \
--max-seqs "$max_seqs" --prefix-cache-n 0 \
-m "$MODEL_DIR" -f "$MODEL_FILE" --format gguf > "$log" 2>&1 &
local pid=$!
# Ready-poll /health; bail if the process dies.
until curl -sf "http://localhost:$PORT/health" >/dev/null 2>&1; do
kill -0 "$pid" 2>/dev/null || { echo "[row $idx] server exited early; see $log" >&2; return 1; }
sleep 0.5
done
t_load1="$(date +%s.%N)"
echo "[row $idx] ready in $(echo "$t_load1 - $t_load0" | bc)s (pid $pid)" >&2
: > "$rss"
rss_sampler "$pid" "$rss" &
local sampler=$!
# Unscored warmup, then the scored phases.
local i
for ((i=0; i<WARMUP; i++)); do
curl -sf "http://localhost:$PORT/v1/chat/completions" \
-H 'content-type: application/json' \
-d '{"model":"default","messages":[{"role":"user","content":"warmup"}],"max_tokens":8,"temperature":0}' \
>/dev/null 2>&1 || true
done
python3 "$HERE/bench_client.py" --base-url "http://localhost:$PORT" \
--max-seqs "$max_seqs" --prompt-tokens "$PROMPT_TOKENS" \
--max-tokens "$MAX_TOKENS" --batch "$BATCH" --out "$recs" >&2
# Read the expert-cache + peak RSS off the live process before termination.
curl -sf "http://localhost:$PORT/v1/system/info" -o "$sysinfo" 2>/dev/null || echo '{}' > "$sysinfo"
local peak; peak="$(peak_rss_gb "$pid")"
# Graceful terminate by the recorded PID; never pkill / kill -9.
kill -TERM "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
kill "$sampler" 2>/dev/null || true
wait "$sampler" 2>/dev/null || true
[[ -f "$MODEL_DIR/.hanzo_experts_usage" ]] || echo "[row $idx] note: no .hanzo_experts_usage sidecar" >&2
emit_row "$idx" "$quant" "$se" "$ram_gb" "$max_seqs" "$recs" "$sysinfo" "$rss" "$peak"
}
# --- fold one row's artifacts into row-<idx>.json ------------------------------
emit_row() { # idx quant se ram_gb max_seqs recs sysinfo rss peak
python3 - "$@" <<'PY'
import json, sys
idx, quant, se, ram_gb, max_seqs, recs_p, sys_p, rss_p, peak = sys.argv[1:10]
def load(p, default):
try: return json.load(open(p))
except Exception: return default
records = load(recs_p, [])
sysinfo = load(sys_p, {})
ec = sysinfo.get("expert_cache") # None on an engine without the readout surface
resident = ec.get("resident_gb") if ec else None
row = {
"config": {"quant": quant, "stream_experts": se == "1",
"ram_gb": (float(ram_gb) if ram_gb else None), "max_seqs": int(max_seqs)},
"records": records,
"peak_rss_gb": (float(peak) if peak else None),
"resident_gb": resident,
"expert_cache": ec,
}
out = f"{__import__('os').path.dirname(recs_p)}/row-{idx}.json"
json.dump(row, open(out, "w"), indent=2)
print("row ->", out, file=sys.stderr)
PY
}
# --- dry-run: synthesize a row + roofline, exercise the assembler --------------
dry_run() {
echo "[dry-run] no server launch; wiring manifest -> synthetic row -> report" >&2
python3 - "$OUT" <<'PY'
import json, os, sys
out = sys.argv[1]
records = [
{"phase":"prefill","config":{"prompt_tokens":512},"usage_prompt_tok_s":1800.0,"prompt_tokens":512},
{"phase":"decode","config":{"max_tokens":256},"usage_compl_tok_s":95.0,"completion_tokens":256},
{"phase":"steady","config":{"max_tokens":256},"usage_compl_tok_s":95.0,
"itl":{"count":248,"p50_ms":10.4,"p90_ms":11.1,"p99_ms":13.0,"steady_tok_s":96.1}},
{"phase":"batch","config":{"n":1,"max_tokens":256},"wall_s":2.7,"aggregate_tok_s":95.0,"per_stream_tok_s":95.0,"completion_tokens":256},
{"phase":"batch","config":{"n":8,"max_tokens":256},"wall_s":5.0,"aggregate_tok_s":410.0,"per_stream_tok_s":51.0,"completion_tokens":2048},
]
# Illustrative resident-mode numbers (hit_rate 1.0 -> no NVMe miss path): with
# ~1.5 GB active weights/token served from RAM, the ceiling sits near ~120 t/s and
# the measured decode lands under it. Forced-small streamed budgets in a real run
# drive hit_rate down and expose the brutal NVMe-miss ceiling instead.
row = {"config":{"quant":"q4k","stream_experts":True,"ram_gb":4.0,"max_seqs":32},
"records":records,"peak_rss_gb":9.8,"resident_gb":4.1,
"expert_cache":{"hit_rate":1.0,"active_weight_bytes":1.5e9,
"resident_dense_bytes":4.0e8,"kv_read_bytes":5.0e7}}
json.dump(row, open(os.path.join(out,"row-0.json"),"w"), indent=2)
roof = {"ram_bw_gbs":{"single_thread":41.0,"all_thread":248.0,"threads":32,"buffer_gb":2.0,"reps":5},
"nvme_bw_gbs":{"value":3.31,"direct":True,"bytes_read":4294967296,"chunk_mb":4,"file":"model.gguf"}}
json.dump(roof, open(os.path.join(out,"roofline.json"),"w"), indent=2)
print("[dry-run] wrote synthetic row-0.json + roofline.json", file=sys.stderr)
PY
}
# --- main ----------------------------------------------------------------------
write_manifest
if [[ "$DRY_RUN" == 1 ]]; then
dry_run
else
[[ -x "$HANZO" ]] || { echo "no hanzo binary: $HANZO (build it first)" >&2; exit 2; }
idx=0
while IFS=$'\t' read -r quant se ram_gb max_seqs; do
[[ -z "$quant" ]] && continue
run_row "$idx" "$quant" "$se" "$ram_gb" "$max_seqs" || echo "[row $idx] failed" >&2
idx=$((idx + 1))
done < <(python3 -c "import tomllib,sys
d=tomllib.load(open('$CONFIG','rb'))
for c in d.get('configs',[]):
print('\t'.join([c.get('quant','q4k'),'1' if c.get('stream_experts') else '0',str(c.get('ram_gb','')),str(c.get('max_seqs',32))]))")
if [[ -x "$ROOFLINE" ]]; then
echo "roofline probe ..." >&2
LD_LIBRARY_PATH="$LD_ROCM" "$ROOFLINE" --file "$MODEL" > "$OUT/roofline.json" 2>>"$OUT/roofline.log" \
|| echo "roofline probe failed; see $OUT/roofline.log" >&2
else
echo "no roofline binary: $ROOFLINE (skipping probe)" >&2
fi
fi
python3 "$HERE/bench_report.py" --in-dir "$OUT"
echo "done: $OUT" >&2