mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Orchestrator (scripts/bench_moe.sh) drives a sweep of hanzo-serve launches: per-row env/flags, /health poll + load time, resident-memory sampler (VmRSS curve + VmHWM peak), warmup, load generator, live expert-cache readout off /v1/system/info, graceful SIGTERM by recorded PID (never pkill/kill -9). Load generator (scripts/bench_client.py) isolates prefill from decode via the engine usage superset and cross-checks steady-state decode from client-side ITL (first K dropped), with per-user prompt salts so prefix-caching can't elide prefill. Batch phase reports aggregate vs per-stream tok/s. Roofline probe (hanzo-bench/src/bin/roofline.rs) measures the two ceilings on the box with no external tools: RAM via a STREAM triad (single/all-thread), NVMe via an O_DIRECT sequential read. Report assembler (scripts/bench_report.py) owns the schema + roofline arithmetic, emitting results.json + a rendered table. Sweep matrix is declarative (scripts/bench_moe.toml).
214 lines
7.7 KiB
Python
214 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
"""bench_report — assemble one results.json and render the tables (one writer, DRY).
|
|
|
|
Input: a directory of per-row artifacts written by the orchestrator
|
|
manifest.json reproducibility header (git/host/gpu/model/env/serve cmd)
|
|
roofline.json measured RAM BW + NVMe BW (from the roofline probe)
|
|
row-<i>.json { config, records:[client records], peak_rss_gb,
|
|
resident_gb, expert_cache }
|
|
|
|
Output: results.json (machine) + a console/markdown table (human), both derived
|
|
from the same in-memory rows. This is the single home for the roofline arithmetic:
|
|
bytes/token, predicted ceiling, %-of-ceiling — computed here where the row data is,
|
|
not in the bandwidth probe (which only measures) or the load generator (which only
|
|
drives). Ceiling is emitted only when the expert-cache readout is present; on an
|
|
engine without that surface the column degrades to null (no invented numbers).
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
GIB = 1024.0**3
|
|
|
|
|
|
def load(path: str) -> dict | list | None:
|
|
if not os.path.exists(path):
|
|
return None
|
|
with open(path) as f:
|
|
return json.load(f)
|
|
|
|
|
|
def pick(records: list[dict], phase: str) -> list[dict]:
|
|
return [r for r in records if r.get("phase") == phase]
|
|
|
|
|
|
def prefill_tok_s(records: list[dict]) -> float | None:
|
|
vals = [r["usage_prompt_tok_s"] for r in pick(records, "prefill") if r.get("usage_prompt_tok_s")]
|
|
return max(vals) if vals else None
|
|
|
|
|
|
def decode_usage_tok_s(records: list[dict]) -> float | None:
|
|
vals = [r["usage_compl_tok_s"] for r in pick(records, "decode") if r.get("usage_compl_tok_s")]
|
|
return max(vals) if vals else None
|
|
|
|
|
|
def steady(records: list[dict]) -> dict | None:
|
|
best = None
|
|
for r in pick(records, "steady"):
|
|
itl = r.get("itl")
|
|
if itl and itl.get("steady_tok_s"):
|
|
if best is None or itl["steady_tok_s"] > best["steady_tok_s"]:
|
|
best = itl
|
|
return best
|
|
|
|
|
|
def batch_curve(records: list[dict]) -> list[dict]:
|
|
return [
|
|
{"n": r["config"]["n"], "aggregate_tok_s": r.get("aggregate_tok_s"),
|
|
"per_stream_tok_s": r.get("per_stream_tok_s")}
|
|
for r in pick(records, "batch")
|
|
]
|
|
|
|
|
|
def roofline_for_row(row: dict, roof: dict | None, decode_tok_s: float | None) -> dict | None:
|
|
"""Predicted decode ceiling from measured BW + the row's expert-cache readout.
|
|
|
|
hit_bytes serviced from RAM, miss_bytes from NVMe:
|
|
hit_bytes = active_weight_bytes*hit_rate + resident_dense_bytes
|
|
miss_bytes = active_weight_bytes*(1-hit_rate)
|
|
ceiling = 1 / (hit_bytes/RAM_BW + miss_bytes/NVMe_BW + kv_bytes/RAM_BW)
|
|
STREAM_EXPERTS-off collapses to RAM_BW/bytes_per_token (hit_rate treated as 1).
|
|
Requires the expert-cache readout; returns None otherwise (no guessing).
|
|
"""
|
|
ec = row.get("expert_cache")
|
|
if not roof or not ec:
|
|
return None
|
|
ram_bw = (roof.get("ram_bw_gbs") or {}).get("all_thread")
|
|
nvme_bw = (roof.get("nvme_bw_gbs") or {}).get("value")
|
|
if not ram_bw:
|
|
return None
|
|
active = ec.get("active_weight_bytes")
|
|
if not active:
|
|
return None
|
|
hit_rate = ec.get("hit_rate", 1.0)
|
|
resident_dense = ec.get("resident_dense_bytes", 0.0)
|
|
kv_bytes = ec.get("kv_read_bytes", 0.0)
|
|
hit_bytes = active * hit_rate + resident_dense
|
|
miss_bytes = active * (1.0 - hit_rate)
|
|
ram_bw_bps = ram_bw * 1e9
|
|
secs = (hit_bytes + kv_bytes) / ram_bw_bps
|
|
if miss_bytes > 0 and nvme_bw:
|
|
secs += miss_bytes / (nvme_bw * 1e9)
|
|
ceiling = (1.0 / secs) if secs > 0 else None
|
|
pct = (100.0 * decode_tok_s / ceiling) if (ceiling and decode_tok_s) else None
|
|
return {
|
|
"bytes_per_token": hit_bytes + miss_bytes + kv_bytes,
|
|
"predicted_ceiling_tok_s": ceiling,
|
|
"pct_of_ceiling": pct,
|
|
}
|
|
|
|
|
|
def assemble(rows_in: list[dict], roof: dict | None) -> list[dict]:
|
|
rows = []
|
|
for row in rows_in:
|
|
recs = row.get("records", [])
|
|
dec_usage = decode_usage_tok_s(recs)
|
|
st = steady(recs)
|
|
dec_steady = st["steady_tok_s"] if st else None
|
|
rows.append({
|
|
"config": row.get("config", {}),
|
|
"prefill_tok_s": prefill_tok_s(recs),
|
|
"decode_tok_s_usage": dec_usage,
|
|
"decode_tok_s_steady": dec_steady,
|
|
"itl_p50_ms": st["p50_ms"] if st else None,
|
|
"itl_p90_ms": st["p90_ms"] if st else None,
|
|
"peak_rss_gb": row.get("peak_rss_gb"),
|
|
"resident_gb": row.get("resident_gb"),
|
|
"expert_cache": row.get("expert_cache"),
|
|
"batch": batch_curve(recs),
|
|
"roofline": roofline_for_row(row, roof, dec_steady or dec_usage),
|
|
})
|
|
return rows
|
|
|
|
|
|
def fmt(v, prec=1) -> str:
|
|
if v is None:
|
|
return "-"
|
|
if isinstance(v, float):
|
|
return f"{v:.{prec}f}"
|
|
return str(v)
|
|
|
|
|
|
def cfg_label(c: dict) -> str:
|
|
q = c.get("quant", "?")
|
|
se = "on" if c.get("stream_experts") else "off"
|
|
ram = c.get("ram_gb")
|
|
ram = "inf" if ram is None else fmt(ram, 0)
|
|
return f"{q}/se={se}/ram={ram}"
|
|
|
|
|
|
def render_table(rows: list[dict]) -> str:
|
|
cols = ["config", "prefill t/s", "decode t/s", "steady t/s", "p90 ITL ms",
|
|
"peak RSS", "resident", "hit%", "ceiling t/s", "%ceil"]
|
|
lines = ["| " + " | ".join(cols) + " |",
|
|
"|" + "|".join(["---"] * len(cols)) + "|"]
|
|
for r in sorted(rows, key=lambda x: cfg_label(x["config"])):
|
|
ec = r.get("expert_cache") or {}
|
|
roof = r.get("roofline") or {}
|
|
hit = ec.get("hit_rate")
|
|
lines.append("| " + " | ".join([
|
|
cfg_label(r["config"]),
|
|
fmt(r["prefill_tok_s"]),
|
|
fmt(r["decode_tok_s_usage"]),
|
|
fmt(r["decode_tok_s_steady"]),
|
|
fmt(r["itl_p90_ms"], 2),
|
|
fmt(r["peak_rss_gb"]),
|
|
fmt(r["resident_gb"]),
|
|
fmt(hit * 100 if hit is not None else None),
|
|
fmt(roof.get("predicted_ceiling_tok_s")),
|
|
fmt(roof.get("pct_of_ceiling")),
|
|
]) + " |")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def render_roofline(roof: dict | None, rows: list[dict]) -> str:
|
|
if not roof:
|
|
return "roofline: (no probe)"
|
|
ram = roof.get("ram_bw_gbs") or {}
|
|
nvme = roof.get("nvme_bw_gbs") or {}
|
|
out = [
|
|
"roofline probe:",
|
|
f" RAM BW : {fmt(ram.get('single_thread'))} GB/s (1t), "
|
|
f"{fmt(ram.get('all_thread'))} GB/s ({ram.get('threads')}t)",
|
|
f" NVMe BW : {fmt(nvme.get('value'))} GB/s "
|
|
f"(direct={nvme.get('direct')}, {fmt(nvme.get('bytes_read', 0) / GIB)} GiB read)",
|
|
]
|
|
return "\n".join(out)
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
p = argparse.ArgumentParser(description="Assemble bench results.json + table")
|
|
p.add_argument("--in-dir", required=True, help="dir with manifest/roofline/row-*.json")
|
|
p.add_argument("--out", default="", help="results.json path (default <in-dir>/results.json)")
|
|
a = p.parse_args(argv)
|
|
|
|
manifest = load(os.path.join(a.in_dir, "manifest.json")) or {}
|
|
roof = load(os.path.join(a.in_dir, "roofline.json"))
|
|
rows_in = []
|
|
for name in sorted(os.listdir(a.in_dir)):
|
|
if name.startswith("row-") and name.endswith(".json"):
|
|
rows_in.append(load(os.path.join(a.in_dir, name)))
|
|
|
|
rows = assemble([r for r in rows_in if r], roof)
|
|
if roof:
|
|
manifest["ram_bw_gbs"] = (roof.get("ram_bw_gbs") or {}).get("all_thread")
|
|
manifest["nvme_bw_gbs"] = (roof.get("nvme_bw_gbs") or {}).get("value")
|
|
results = {"manifest": manifest, "rows": rows}
|
|
|
|
out = a.out or os.path.join(a.in_dir, "results.json")
|
|
with open(out, "w") as f:
|
|
json.dump(results, f, indent=2)
|
|
|
|
print(render_roofline(roof, rows))
|
|
print()
|
|
print(render_table(rows))
|
|
print()
|
|
print(f"results.json -> {out}", file=sys.stderr)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|