mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Every mention of another inference engine is rewritten to say what our code does, or the math it computes. Reference values are "the reference value from the published config" or "the recorded reference output", line citations into another engine's source become the rule itself, comparisons with other engines are dropped, and asides about other engines' bugs go. Identifiers follow: - CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels' `hanzo_rotary`; the shuffle/ldg macros are HANZO_*. - GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side. - Tests: `*_matches_vllm` and friends name what they compare against (the reference fixture, the tile rule). - MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op is `rotary-embedding`. - hanzo-bench's board scores ours against a "rival" whose name comes from the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s` columns; dossier.sh and spec_decode.sh take `--rival-bench`. - install.ps1 installs this repository's hanzo-cli, as install.sh does, with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL. - bench_http.py reads only the hanzo: metric families. Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and tools/upstream-map.sh; each ported file's copyright/licence/provenance header; Cargo.toml dependency names; data identifiers (Hugging Face repo IDs, fixture file names and fixture metadata keys the tests read, on-disk paths, executable names the GPU and engine guards match on a host); model and format names. LLM.md says the rule once, under "One name: Hanzo Engine". Co-authored-by: Hanzo Dev <[email protected]>
210 lines
9.1 KiB
Python
210 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Bytes one decode step moves, two ways, never substituted for each other.
|
|
|
|
weights COMPUTED weight + KV + recurrent-state bytes per token, from the safetensors headers
|
|
(only the header bytes are read) and config.json. The method behind LLM.md's per-token
|
|
figures: 19.44 GB for the checkpoint as published (NVFP4), 26.65 GB with hanzo's FP8
|
|
layers.
|
|
nsys MEASURED SM-to-L2 bytes per token ('VidL2 Total from L1TEX [Bytes]', GB10's gb20y-top
|
|
metric set) and GPU seconds per decode step (kernel time inside NVTX `decode` ranges),
|
|
from an `nsys export --type sqlite` file. GB10 exposes no DRAM counter, so this is L2
|
|
traffic, a cross-check on the computed figure and a finder of redundant reads.
|
|
|
|
Standard library only.
|
|
"""
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import struct
|
|
import sys
|
|
|
|
DTYPE_BYTES = {
|
|
"F64": 8, "I64": 8, "U64": 8, "F32": 4, "I32": 4, "U32": 4, "F16": 2, "BF16": 2, "I16": 2,
|
|
"U16": 2, "F8_E4M3": 1, "F8_E5M2": 1, "F8_E8M0": 1, "I8": 1, "U8": 1, "BOOL": 1,
|
|
}
|
|
NAMED = {"bf16": 2, "f16": 2, "fp16": 2, "f32": 4, "fp32": 4, "fp8": 1, "f8": 1, "fp8_e4m3": 1,
|
|
"fp4": 0.5, "nvfp4": 0.5625}
|
|
SCALE_SUFFIXES = (".weight_scale", ".weight_scale_2", ".input_scale")
|
|
SKIPPED = re.compile(r"(^|\.)(visual|mtp)\.|^mtp\.")
|
|
|
|
|
|
def headers(checkpoint):
|
|
"""{tensor name: {dtype, shape, bytes}} over every shard's header."""
|
|
tensors = {}
|
|
for path in sorted(glob.glob(os.path.join(checkpoint, "*.safetensors"))):
|
|
with open(path, "rb") as f:
|
|
(n,) = struct.unpack("<Q", f.read(8))
|
|
header = json.loads(f.read(n))
|
|
for name, info in header.items():
|
|
if name == "__metadata__":
|
|
continue
|
|
start, end = info["data_offsets"]
|
|
tensors[name] = {"dtype": info["dtype"], "shape": info["shape"], "bytes": end - start}
|
|
if not tensors:
|
|
raise SystemExit(f"no safetensors under {checkpoint}")
|
|
return tensors
|
|
|
|
|
|
def numel(shape):
|
|
n = 1
|
|
for d in shape:
|
|
n *= d
|
|
return n
|
|
|
|
|
|
def logical_numel(name, info, tensors):
|
|
"""Elements a weight represents: NVFP4 packs two per U8 byte (it has an F8 block scale)."""
|
|
base = name[: -len(".weight")] if name.endswith(".weight") else None
|
|
if info["dtype"] == "U8" and base and base + ".weight_scale" in tensors:
|
|
return 2 * numel(info["shape"])
|
|
return numel(info["shape"])
|
|
|
|
|
|
def text_config(checkpoint):
|
|
with open(os.path.join(checkpoint, "config.json")) as f:
|
|
config = json.load(f)
|
|
return config.get("text_config", config)
|
|
|
|
|
|
def weights(args):
|
|
tensors = headers(args.checkpoint)
|
|
cfg = text_config(args.checkpoint)
|
|
resident = []
|
|
for spec in args.resident:
|
|
pattern, _, dtype = spec.rpartition("=")
|
|
if not pattern or dtype.lower() not in NAMED:
|
|
raise SystemExit(f"--resident wants REGEX=DTYPE with DTYPE in {sorted(NAMED)}: {spec}")
|
|
resident.append((re.compile(pattern), NAMED[dtype.lower()]))
|
|
|
|
weight_bytes = 0.0
|
|
embed_row = 0.0
|
|
held = {}
|
|
for name, info in tensors.items():
|
|
if SKIPPED.search(name):
|
|
continue
|
|
if "embed_tokens" in name:
|
|
embed_row += info["bytes"] / info["shape"][0]
|
|
continue
|
|
rule = next((r for r in resident if r[0].search(name)), None)
|
|
if rule:
|
|
if name.endswith(SCALE_SUFFIXES):
|
|
continue # the module is held dequantized; its scales are not read
|
|
size = logical_numel(name, info, tensors) * rule[1]
|
|
held[rule[0].pattern] = held.get(rule[0].pattern, 0) + size
|
|
weight_bytes += size
|
|
else:
|
|
weight_bytes += info["bytes"]
|
|
|
|
layer_types = cfg.get("layer_types") or ["full_attention"] * cfg["num_hidden_layers"]
|
|
full = layer_types.count("full_attention")
|
|
linear = layer_types.count("linear_attention")
|
|
kv_dtype = {"bf16": 2, "fp8": 1}[args.kv_dtype]
|
|
head_dim = cfg.get("head_dim") or cfg["hidden_size"] // cfg["num_attention_heads"]
|
|
kv = full * cfg["num_key_value_heads"] * head_dim * 2 * kv_dtype * args.context
|
|
state = 0
|
|
if linear:
|
|
v_heads, k_heads = cfg["linear_num_value_heads"], cfg["linear_num_key_heads"]
|
|
k_dim, v_dim = cfg["linear_key_head_dim"], cfg["linear_value_head_dim"]
|
|
ssm = linear * v_heads * k_dim * v_dim * 4 * 2 # fp32 state, read and written
|
|
conv_dim = 2 * k_heads * k_dim + v_heads * v_dim
|
|
conv = linear * conv_dim * (cfg["linear_conv_kernel_dim"] - 1) * 2 * 2 # bf16, read + write
|
|
state = ssm + conv
|
|
total = weight_bytes + embed_row + kv + state
|
|
out = {
|
|
"kind": "computed",
|
|
"checkpoint": os.path.abspath(args.checkpoint),
|
|
"context": args.context,
|
|
"kv_dtype": args.kv_dtype,
|
|
"weights_bytes": int(weight_bytes),
|
|
"embed_row_bytes": int(embed_row),
|
|
"kv_bytes": int(kv),
|
|
"state_bytes": int(state),
|
|
"resident": {k: int(v) for k, v in held.items()},
|
|
"bytes_per_token": int(total),
|
|
"gb_per_token": round(total / 1e9, 3),
|
|
}
|
|
print(json.dumps(out))
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# nsys export. Every table and column name the tool depends on is in this one function.
|
|
|
|
L2_METRIC = "VidL2 Total from L1TEX [Bytes]"
|
|
|
|
|
|
def read_capture(path, range_name="decode"):
|
|
db = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
tables = {r[0] for r in db.execute("select name from sqlite_master where type='table'")}
|
|
capture = {"l2_bytes": None, "ranges": 0, "kernel_ns": None}
|
|
if {"GPU_METRICS", "TARGET_INFO_GPU_METRICS"} <= tables:
|
|
(capture["l2_bytes"],) = db.execute(
|
|
"select coalesce(sum(g.value), 0) from GPU_METRICS g join TARGET_INFO_GPU_METRICS t"
|
|
" on g.typeId = t.typeId and g.metricId = t.metricId where t.metricName = ?",
|
|
(L2_METRIC,)).fetchone()
|
|
if "NVTX_EVENTS" in tables:
|
|
text = ("coalesce(n.text, (select s.value from StringIds s where s.id = n.textId))"
|
|
if "StringIds" in tables else "n.text")
|
|
ranges = db.execute(
|
|
f"select n.start, n.end, n.globalTid from NVTX_EVENTS n"
|
|
f" where n.end is not null and {text} = ?", (range_name,)).fetchall()
|
|
capture["ranges"] = len(ranges)
|
|
work = [t for t in ("CUPTI_ACTIVITY_KIND_KERNEL", "CUPTI_ACTIVITY_KIND_GRAPH_TRACE")
|
|
if t in tables]
|
|
if ranges and work and "CUPTI_ACTIVITY_KIND_RUNTIME" in tables:
|
|
db.execute("create temp table ranges (start integer, end integer, tid integer)")
|
|
db.executemany("insert into ranges values (?, ?, ?)", ranges)
|
|
total = 0
|
|
for table in work:
|
|
(ns,) = db.execute(
|
|
f"select coalesce(sum(k.end - k.start), 0) from {table} k"
|
|
" join CUPTI_ACTIVITY_KIND_RUNTIME r on r.correlationId = k.correlationId"
|
|
" join ranges g on r.globalTid = g.tid and r.start >= g.start and r.start <= g.end"
|
|
).fetchone()
|
|
total += ns
|
|
capture["kernel_ns"] = total
|
|
return capture
|
|
|
|
|
|
def nsys(args):
|
|
capture = read_capture(args.sqlite, args.range)
|
|
out = {"kind": "measured", "sqlite": os.path.abspath(args.sqlite), "tokens": args.tokens}
|
|
if capture["l2_bytes"] is not None:
|
|
out["l2_bytes"] = capture["l2_bytes"]
|
|
out["l2_bytes_per_token"] = capture["l2_bytes"] / args.tokens
|
|
out["l2_gb_per_token"] = round(capture["l2_bytes"] / args.tokens / 1e9, 3)
|
|
out["decode_ranges"] = capture["ranges"]
|
|
if capture["kernel_ns"] is not None and capture["ranges"]:
|
|
out["gpu_s_per_step"] = capture["kernel_ns"] / capture["ranges"] / 1e9
|
|
if args.computed_bytes:
|
|
out["achieved_gb_s"] = args.computed_bytes / out["gpu_s_per_step"] / 1e9
|
|
print(json.dumps(out))
|
|
return out
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
w = sub.add_parser("weights", help="computed bytes per token")
|
|
w.add_argument("--checkpoint", required=True)
|
|
w.add_argument("--context", type=int, required=True, help="tokens in the KV cache")
|
|
w.add_argument("--kv-dtype", choices=["bf16", "fp8"], required=True)
|
|
w.add_argument("--resident", action="append", default=[], metavar="REGEX=DTYPE",
|
|
help="tensors an engine holds at another dtype (their scales are dropped)")
|
|
n = sub.add_parser("nsys", help="measured bytes per token from an nsys sqlite export")
|
|
n.add_argument("--sqlite", required=True)
|
|
n.add_argument("--tokens", type=int, required=True, help="tokens decoded inside the capture")
|
|
n.add_argument("--range", default="decode", help="NVTX range name of a decode step")
|
|
n.add_argument("--computed-bytes", type=float,
|
|
help="computed bytes per step, to report achieved GB/s over GPU time")
|
|
args = ap.parse_args(argv)
|
|
{"weights": weights, "nsys": nsys}[args.cmd](args)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|