mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
LLM.md section 3b quoted numbers from a script that lived in a home directory and counted chunks, not tokens. Under MTP or DFlash one chunk carries several tokens, and vLLM v0.29 streams thinking as `reasoning` where hanzo sends `reasoning_content`, so chunk counting measured steps and missed vLLM's thinking entirely. bench_http.py counts token chunks by content, reasoning, reasoning_content or tool_calls, takes decode tokens from usage (a stream without usage fails), and reports TTFT, decode rate, per-step gaps and a concurrent aggregate as JSON lines. It also carries the /metrics check and the profile window used by later tests. bytes_per_token.py gives two labeled numbers: the computed weight, KV and state bytes per token from safetensors headers (19.44 GB of 27B weights, matching SGLang's figure), and the measured SM-to-L2 bytes from an nsys gb20y-top export. GB10 has no DRAM counter, so the measured figure is a cross-check, never a substitute. Claude-Session: https://claude.ai/code/session_019ABm6hF8RNyUFpbjQWqfxY
209 lines
9.0 KiB
Python
209 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Bytes one decode step moves, two ways, never substituted for each other.
|
|
|
|
weights COMPUTED weight + KV + recurrent-state bytes per token, from the safetensors headers
|
|
(only the header bytes are read) and config.json. The method behind SGLang's 19.44 GB
|
|
and hanzo's 26.65 GB in LLM.md section 3b.
|
|
nsys MEASURED SM-to-L2 bytes per token ('VidL2 Total from L1TEX [Bytes]', GB10's gb20y-top
|
|
metric set) and GPU seconds per decode step (kernel time inside NVTX `decode` ranges),
|
|
from an `nsys export --type sqlite` file. GB10 exposes no DRAM counter, so this is L2
|
|
traffic, a cross-check on the computed figure and a finder of redundant reads.
|
|
|
|
Standard library only.
|
|
"""
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import struct
|
|
import sys
|
|
|
|
DTYPE_BYTES = {
|
|
"F64": 8, "I64": 8, "U64": 8, "F32": 4, "I32": 4, "U32": 4, "F16": 2, "BF16": 2, "I16": 2,
|
|
"U16": 2, "F8_E4M3": 1, "F8_E5M2": 1, "F8_E8M0": 1, "I8": 1, "U8": 1, "BOOL": 1,
|
|
}
|
|
NAMED = {"bf16": 2, "f16": 2, "fp16": 2, "f32": 4, "fp32": 4, "fp8": 1, "f8": 1, "fp8_e4m3": 1,
|
|
"fp4": 0.5, "nvfp4": 0.5625}
|
|
SCALE_SUFFIXES = (".weight_scale", ".weight_scale_2", ".input_scale")
|
|
SKIPPED = re.compile(r"(^|\.)(visual|mtp)\.|^mtp\.")
|
|
|
|
|
|
def headers(checkpoint):
|
|
"""{tensor name: {dtype, shape, bytes}} over every shard's header."""
|
|
tensors = {}
|
|
for path in sorted(glob.glob(os.path.join(checkpoint, "*.safetensors"))):
|
|
with open(path, "rb") as f:
|
|
(n,) = struct.unpack("<Q", f.read(8))
|
|
header = json.loads(f.read(n))
|
|
for name, info in header.items():
|
|
if name == "__metadata__":
|
|
continue
|
|
start, end = info["data_offsets"]
|
|
tensors[name] = {"dtype": info["dtype"], "shape": info["shape"], "bytes": end - start}
|
|
if not tensors:
|
|
raise SystemExit(f"no safetensors under {checkpoint}")
|
|
return tensors
|
|
|
|
|
|
def numel(shape):
|
|
n = 1
|
|
for d in shape:
|
|
n *= d
|
|
return n
|
|
|
|
|
|
def logical_numel(name, info, tensors):
|
|
"""Elements a weight represents: NVFP4 packs two per U8 byte (it has an F8 block scale)."""
|
|
base = name[: -len(".weight")] if name.endswith(".weight") else None
|
|
if info["dtype"] == "U8" and base and base + ".weight_scale" in tensors:
|
|
return 2 * numel(info["shape"])
|
|
return numel(info["shape"])
|
|
|
|
|
|
def text_config(checkpoint):
|
|
with open(os.path.join(checkpoint, "config.json")) as f:
|
|
config = json.load(f)
|
|
return config.get("text_config", config)
|
|
|
|
|
|
def weights(args):
|
|
tensors = headers(args.checkpoint)
|
|
cfg = text_config(args.checkpoint)
|
|
resident = []
|
|
for spec in args.resident:
|
|
pattern, _, dtype = spec.rpartition("=")
|
|
if not pattern or dtype.lower() not in NAMED:
|
|
raise SystemExit(f"--resident wants REGEX=DTYPE with DTYPE in {sorted(NAMED)}: {spec}")
|
|
resident.append((re.compile(pattern), NAMED[dtype.lower()]))
|
|
|
|
weight_bytes = 0.0
|
|
embed_row = 0.0
|
|
held = {}
|
|
for name, info in tensors.items():
|
|
if SKIPPED.search(name):
|
|
continue
|
|
if "embed_tokens" in name:
|
|
embed_row += info["bytes"] / info["shape"][0]
|
|
continue
|
|
rule = next((r for r in resident if r[0].search(name)), None)
|
|
if rule:
|
|
if name.endswith(SCALE_SUFFIXES):
|
|
continue # the module is held dequantized; its scales are not read
|
|
size = logical_numel(name, info, tensors) * rule[1]
|
|
held[rule[0].pattern] = held.get(rule[0].pattern, 0) + size
|
|
weight_bytes += size
|
|
else:
|
|
weight_bytes += info["bytes"]
|
|
|
|
layer_types = cfg.get("layer_types") or ["full_attention"] * cfg["num_hidden_layers"]
|
|
full = layer_types.count("full_attention")
|
|
linear = layer_types.count("linear_attention")
|
|
kv_dtype = {"bf16": 2, "fp8": 1}[args.kv_dtype]
|
|
head_dim = cfg.get("head_dim") or cfg["hidden_size"] // cfg["num_attention_heads"]
|
|
kv = full * cfg["num_key_value_heads"] * head_dim * 2 * kv_dtype * args.context
|
|
state = 0
|
|
if linear:
|
|
v_heads, k_heads = cfg["linear_num_value_heads"], cfg["linear_num_key_heads"]
|
|
k_dim, v_dim = cfg["linear_key_head_dim"], cfg["linear_value_head_dim"]
|
|
ssm = linear * v_heads * k_dim * v_dim * 4 * 2 # fp32 state, read and written
|
|
conv_dim = 2 * k_heads * k_dim + v_heads * v_dim
|
|
conv = linear * conv_dim * (cfg["linear_conv_kernel_dim"] - 1) * 2 * 2 # bf16, read + write
|
|
state = ssm + conv
|
|
total = weight_bytes + embed_row + kv + state
|
|
out = {
|
|
"kind": "computed",
|
|
"checkpoint": os.path.abspath(args.checkpoint),
|
|
"context": args.context,
|
|
"kv_dtype": args.kv_dtype,
|
|
"weights_bytes": int(weight_bytes),
|
|
"embed_row_bytes": int(embed_row),
|
|
"kv_bytes": int(kv),
|
|
"state_bytes": int(state),
|
|
"resident": {k: int(v) for k, v in held.items()},
|
|
"bytes_per_token": int(total),
|
|
"gb_per_token": round(total / 1e9, 3),
|
|
}
|
|
print(json.dumps(out))
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# nsys export. Every table and column name the tool depends on is in this one function.
|
|
|
|
L2_METRIC = "VidL2 Total from L1TEX [Bytes]"
|
|
|
|
|
|
def read_capture(path, range_name="decode"):
|
|
db = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
tables = {r[0] for r in db.execute("select name from sqlite_master where type='table'")}
|
|
capture = {"l2_bytes": None, "ranges": 0, "kernel_ns": None}
|
|
if {"GPU_METRICS", "TARGET_INFO_GPU_METRICS"} <= tables:
|
|
(capture["l2_bytes"],) = db.execute(
|
|
"select coalesce(sum(g.value), 0) from GPU_METRICS g join TARGET_INFO_GPU_METRICS t"
|
|
" on g.typeId = t.typeId and g.metricId = t.metricId where t.metricName = ?",
|
|
(L2_METRIC,)).fetchone()
|
|
if "NVTX_EVENTS" in tables:
|
|
text = ("coalesce(n.text, (select s.value from StringIds s where s.id = n.textId))"
|
|
if "StringIds" in tables else "n.text")
|
|
ranges = db.execute(
|
|
f"select n.start, n.end, n.globalTid from NVTX_EVENTS n"
|
|
f" where n.end is not null and {text} = ?", (range_name,)).fetchall()
|
|
capture["ranges"] = len(ranges)
|
|
work = [t for t in ("CUPTI_ACTIVITY_KIND_KERNEL", "CUPTI_ACTIVITY_KIND_GRAPH_TRACE")
|
|
if t in tables]
|
|
if ranges and work and "CUPTI_ACTIVITY_KIND_RUNTIME" in tables:
|
|
db.execute("create temp table ranges (start integer, end integer, tid integer)")
|
|
db.executemany("insert into ranges values (?, ?, ?)", ranges)
|
|
total = 0
|
|
for table in work:
|
|
(ns,) = db.execute(
|
|
f"select coalesce(sum(k.end - k.start), 0) from {table} k"
|
|
" join CUPTI_ACTIVITY_KIND_RUNTIME r on r.correlationId = k.correlationId"
|
|
" join ranges g on r.globalTid = g.tid and r.start >= g.start and r.start <= g.end"
|
|
).fetchone()
|
|
total += ns
|
|
capture["kernel_ns"] = total
|
|
return capture
|
|
|
|
|
|
def nsys(args):
|
|
capture = read_capture(args.sqlite, args.range)
|
|
out = {"kind": "measured", "sqlite": os.path.abspath(args.sqlite), "tokens": args.tokens}
|
|
if capture["l2_bytes"] is not None:
|
|
out["l2_bytes"] = capture["l2_bytes"]
|
|
out["l2_bytes_per_token"] = capture["l2_bytes"] / args.tokens
|
|
out["l2_gb_per_token"] = round(capture["l2_bytes"] / args.tokens / 1e9, 3)
|
|
out["decode_ranges"] = capture["ranges"]
|
|
if capture["kernel_ns"] is not None and capture["ranges"]:
|
|
out["gpu_s_per_step"] = capture["kernel_ns"] / capture["ranges"] / 1e9
|
|
if args.computed_bytes:
|
|
out["achieved_gb_s"] = args.computed_bytes / out["gpu_s_per_step"] / 1e9
|
|
print(json.dumps(out))
|
|
return out
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
w = sub.add_parser("weights", help="computed bytes per token")
|
|
w.add_argument("--checkpoint", required=True)
|
|
w.add_argument("--context", type=int, required=True, help="tokens in the KV cache")
|
|
w.add_argument("--kv-dtype", choices=["bf16", "fp8"], required=True)
|
|
w.add_argument("--resident", action="append", default=[], metavar="REGEX=DTYPE",
|
|
help="tensors an engine holds at another dtype (their scales are dropped)")
|
|
n = sub.add_parser("nsys", help="measured bytes per token from an nsys sqlite export")
|
|
n.add_argument("--sqlite", required=True)
|
|
n.add_argument("--tokens", type=int, required=True, help="tokens decoded inside the capture")
|
|
n.add_argument("--range", default="decode", help="NVTX range name of a decode step")
|
|
n.add_argument("--computed-bytes", type=float,
|
|
help="computed bytes per step, to report achieved GB/s over GPU time")
|
|
args = ap.parse_args(argv)
|
|
{"weights": weights, "nsys": nsys}[args.cmd](args)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|