Files
engine/scripts/zen6_vectors.py
zeekay eeb464bb67 Rename the qwen4exp architecture to zen6
Every name the engine owns for the Qwen3.8-Flash-Next architecture is
now zen6: models::zen6 and models::quantized_zen6, Zen6Loader,
NormalLoaderType::Zen6 and its `--arch zen6` string,
GGUFArchitecture::Zen6, the zen6_moe fixtures, the zen6_* golden
scripts and LLM.md.

Upstream spellings are read, never written, where a checkpoint names
its architecture: the Hugging Face class
Qwen4ExpForConditionalGeneration in from_causal_lm_name, and the GGUF
general.architecture qwen4exp as a parse alias on
GGUFArchitecture::Zen6.  Content::from_readers moves such a file's
qwen4exp.* keys under zen6.*, so the lookups built from the enum's name
(block_count and head_count in the device mapper, block_count in the
pipeline) find them.

Recorded goldens are untracked: rename a local
tests/fixtures/qwen4exp{,_vectors}.safetensors to zen6{,_vectors}.
2026-09-28 14:09:56 -07:00

601 lines
27 KiB
Python

#!/usr/bin/env python3
"""Served-kernel vectors for zen6: vLLM's own GPU kernels on the golden's inputs.
The CPU golden (zen6_golden.py) transcribes the served kernels with IEEE arithmetic. This
script runs the served kernels themselves, on the golden's inputs and on synthetic edge rows, so the
engine's tests can hold its quantizers to bit equality with the server and its blocks to the
server's own outputs. It records how far the golden sits from the server, stage by stage.
It never touches the running vLLM server: it loads one layer's tensors at a time from the
checkpoint, uses at most 1.2 GiB of device memory, and refuses to JIT-compile FlashInfer (the
cached fused_moe_120 and fp4_quantization_120f modules must load as they are).
systemd-run --user --scope --quiet -p MemoryMax=6G -p MemorySwapMax=0 choom -n 1000 -- \
env MAX_JOBS=1 nice -n19 ionice -c3 ~/vllm-env/bin/python \
scripts/zen6_vectors.py --snapshot <fp8hybrid> \
--golden hanzo-engine/tests/fixtures/zen6.safetensors
"""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
from pathlib import Path
os.environ.setdefault("MAX_JOBS", "1")
import numpy as np # noqa: E402
import torch # noqa: E402
sys.path.insert(0, str(Path(__file__).resolve().parent))
import zen6_golden as G # noqa: E402
DEV = torch.device("cuda")
BF16 = torch.bfloat16
F32 = torch.float32
LIMIT = int(1.2 * (1 << 30))
ROOT = Path(__file__).resolve().parent.parent
def forbid_jit() -> None:
"""FlashInfer must load its cached modules; building one is an error, not a side effect.
A JIT module is loaded after `build()`, where ninja no-ops on an up-to-date cache. This
`build()` leaves the served build.ninja untouched and only asks ninja, dry-run, whether
anything would compile: nothing (or a cached .so of the same version) loads it, anything else
is an error.
(FLASHINFER_DISABLE_JIT refuses before that check, so it cannot be used here.)"""
import subprocess
from flashinfer.jit import core
def check(self, *a, **k):
ninja = Path(self.ninja_path)
if not ninja.exists():
raise RuntimeError(f"FlashInfer module {self.name} is not in the JIT cache")
out = subprocess.run(
["ninja", "-n", "-C", str(Path(self.build_dir).resolve()), "-f", str(ninja.resolve())],
capture_output=True, text=True, check=True,
).stdout
if "no work to do" in out:
return
# A cached .so whose build.ninja another environment wrote (same FlashInfer version, so
# the same sources): load it as it is rather than rebuild.
if Path(self.jit_library_path).exists():
print(f"FlashInfer {self.name}: ninja is stale, loading the cached {self.jit_library_path}")
return
raise RuntimeError(f"FlashInfer would JIT-build {self.name}:\n{out[:2000]}")
for cls in (core.JitSpec, *core.JitSpec.__subclasses__()):
cls.build = check
# --------------------------------------------------------------------------------------------
# inductor kernels, verbatim from the live compile cache
# --------------------------------------------------------------------------------------------
def inductor_kernel(path: Path, name: str):
"""The @triton.jit body of `name` from an inductor output module, compiled standalone."""
import triton
import triton.language as tl
from torch._inductor.runtime import triton_helpers
from torch._inductor.runtime.triton_helpers import libdevice
from torch._inductor.runtime.triton_helpers import math as tl_math
text = path.read_text()
m = re.search(rf"{name} = async_compile\.triton\('{name}', '''(.*?)''', device_str", text, re.S)
if m:
src = m.group(1)
body = src[src.index("@triton.jit") :]
else:
# a standalone kernel module: the file is the kernel
at = text.index(f"@triton.jit\ndef {name}(")
body = text[at:]
scope = {
"triton": triton,
"tl": tl,
"triton_helpers": triton_helpers,
"libdevice": libdevice,
"tl_math": tl_math,
}
# a real file, so triton can read the source back
cache = ROOT / "target" / "zen6-vectors"
cache.mkdir(parents=True, exist_ok=True)
f = cache / f"{name}_{path.stem[:8]}.py"
f.write_text(
"import triton\nimport triton.language as tl\n"
"from torch._inductor.runtime import triton_helpers\n"
"from torch._inductor.runtime.triton_helpers import libdevice, math as tl_math\n\n" + body
)
code = compile(f.read_text(), str(f), "exec")
exec(code, scope)
return scope[name]
INDUCTOR = G.INDUCTOR
FUSED = INDUCTOR / "d7/cd76kp4tkxhvavtr4sv37bs4zyaxtb3ozm22umsosh3ggoncajyl.py"
GATED = INDUCTOR / "7w/c7wgpo357q2hrlrcgvyaumoyoqeqjzziqapm4jzorvvnkqhetdic.py"
EAGER = INDUCTOR / "7l/c7lvd6lba4iozfx55hisw2larwxcmcpvff34nriwmo3gvhsvtg2j.py"
KERN = "triton_per_fused__to_copy_abs_clamp_clone_cutlass_scaled_mm_div_max_permute_squeeze_transpose_view"
KERN_G = "triton_per_fused__to_copy_abs_clamp_clone_cutlass_scaled_mm_div_max_mul_permute_sigmoid_squeeze_transpose_view_0"
KERN_E = "triton_per_fused__to_copy_abs_clamp_div_max_view_0"
def fp8_linear(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""The fused W8A8 input quantizer as served: kernel _2 for 2560-wide rows (hc_gate_mix out),
kernel _0 for 6144-wide rows (the delta-net core into out_proj)."""
t, k = x.shape
kern = inductor_kernel(FUSED, f"{KERN}_{2 if k == 2560 else 0}")
assert k in (2560, 6144), k
s = torch.empty((t, k // 128), dtype=F32, device=DEV).t().contiguous().t() # stride (1, t)
q = torch.empty((t, k), dtype=torch.float8_e4m3fn, device=DEV)
kern[(t * (k // 128),)](x.contiguous(), s, q, t, t * (k // 128), 128, XBLOCK=1, num_warps=2)
return q, s.contiguous()
def fp8_gated(core: torch.Tensor, gate: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""The o_proj quantizer fused with attn * sigmoid(gate) (f32 product, never stored)."""
t, k = core.shape
kern = inductor_kernel(GATED, KERN_G)
s = torch.empty((t, k // 128), dtype=F32, device=DEV).t().contiguous().t()
q = torch.empty((t, k), dtype=torch.float8_e4m3fn, device=DEV)
kern[(t * (k // 128),)](core.contiguous(), gate.contiguous(), s, q, t, t * (k // 128), 128, XBLOCK=1, num_warps=2)
return q, s.contiguous()
def fp8_eager(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""The standalone QuantFP8 kernel (amax / 448 with a runtime divisor), run inside
moe_forward_shared for the shared expert."""
t, k = x.shape
groups = k // 128
quant = inductor_kernel(EAGER, KERN_E)
scale = inductor_kernel(EAGER, "triton_poi_fused__to_copy_clamp_clone_squeeze_transpose_1")
q = torch.empty((t, k), dtype=torch.float8_e4m3fn, device=DEV)
amax = torch.empty((t, groups), dtype=BF16, device=DEV)
quant[(t * groups,)](x.contiguous(), 448.0, amax, q, t * groups, 128, XBLOCK=1, num_warps=2)
# the graph's second kernel: s = max(amax / 448, 1/229376), stored column major
s = torch.empty((groups, t), dtype=F32, device=DEV)
scale[(t, groups, 1)](amax, 448.0, s, k, t, groups, t, YBLOCK=1, XBLOCK=1, num_warps=1)
return q, s.t().contiguous()
# --------------------------------------------------------------------------------------------
# synthetic edge rows
# --------------------------------------------------------------------------------------------
def synthetic() -> torch.Tensor:
g = torch.Generator().manual_seed(1234)
rows = []
k = 2560
rows.append(torch.zeros(k)) # an all-zero row
rows.append(torch.full((k,), -0.0)) # negative zeros
for scale in (1e-3, 5e-4, 1.9e-3): # amax below 1/512
rows.append(torch.randn(k, generator=g) * scale / 4)
for p in range(-6, 7, 3): # amax on powers of two
r = torch.randn(k, generator=g).clamp(-1.9, 1.9) / 2
r[::128] = 2.0**p
rows.append(r * 2.0**p)
for v in (448.0, 1000.0, 3.0e4): # saturating rows
r = torch.randn(k, generator=g)
r[5::128] = v
rows.append(r)
# e4m3 ties: values half-way between codes after scaling by amax/448
r = torch.zeros(k)
r[::128] = 448.0
r[1::128] = 1.0625
r[2::128] = 17.0 / 16 * 2.0**-7
r[3::128] = 2.0**-9 * 1.5
rows.append(r)
# e2m1 ties: block max 6 so the scale is 1, then values on the midpoints
r = torch.zeros(k)
ties = torch.tensor([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0, -0.25, -0.75, -1.25, -1.75, -2.5, -3.5, -5.0, 6.0, -6.0])
r[:] = ties.repeat(k // 16)
rows.append(r)
while len(rows) < 64:
scale = [1e-4, 1e-2, 1.0, 1e2][len(rows) % 4]
rows.append(torch.randn(k, generator=g) * scale)
return torch.stack(rows).to(BF16)
# --------------------------------------------------------------------------------------------
# helpers
# --------------------------------------------------------------------------------------------
def ulps(got: torch.Tensor, want: torch.Tensor) -> torch.Tensor:
"""|got - want| in bf16 ulps at |want|, with a floor of 2^-10 * rms(want) per row."""
got = got.float().cpu()
want = want.float().cpu()
exp = torch.floor(torch.log2(want.abs().clamp(min=1e-30)))
ulp = torch.pow(2.0, exp - 7)
rms = want.pow(2).mean(-1, keepdim=True).sqrt()
return (got - want).abs() / (ulp + rms * 2.0**-10)
def gap(got: torch.Tensor, want: torch.Tensor) -> dict:
u = ulps(got, want).reshape(-1)
g2 = got.float().cpu().reshape(want.shape[0], -1)
w2 = want.float().cpu().reshape(want.shape[0], -1)
cos = torch.nn.functional.cosine_similarity(g2, w2, dim=-1)
cos = torch.where(torch.isnan(cos), torch.ones_like(cos), cos)
return {
"max_ulp": float(u.max()),
"p999_ulp": float(torch.quantile(u.double(), 0.999)),
"min_cos": float(cos.min()),
}
def code_gap(got: torch.Tensor, want: torch.Tensor) -> dict:
"""Quantizer codes: fraction of elements that differ, and the largest step."""
g = got.view(torch.uint8).cpu().to(torch.int32) if got.dtype != torch.uint8 else got.cpu().to(torch.int32)
w = want.view(torch.uint8).cpu().to(torch.int32) if want.dtype != torch.uint8 else want.cpu().to(torch.int32)
diff = (g != w)
return {"differ": float(diff.float().mean()), "count": int(diff.sum())}
# --------------------------------------------------------------------------------------------
# stages
# --------------------------------------------------------------------------------------------
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--snapshot", required=True, type=Path)
ap.add_argument("--golden", required=True, type=Path)
ap.add_argument("--quant-out", type=Path, default=ROOT / "hanzo-quant/tests/fixtures/vllm_vectors.safetensors")
ap.add_argument("--out", type=Path, default=ROOT / "hanzo-engine/tests/fixtures/zen6_vectors.safetensors")
args = ap.parse_args()
G.check_kernels()
free, _ = torch.cuda.mem_get_info()
assert free >= 4 << 30, f"only {free / 2**30:.1f} GiB free on the device"
forbid_jit()
torch.cuda.reset_peak_memory_stats()
from safetensors.torch import load_file, save_file
gold = {k: v for k, v in load_file(str(args.golden)).items()}
def gd(name: str) -> torch.Tensor:
return gold[name].to(DEV)
ck = G.Checkpoint(args.snapshot)
qv: dict[str, torch.Tensor] = {} # quantizer vectors (inputs and outputs)
ev: dict[str, torch.Tensor] = {} # engine vectors (outputs)
report: dict[str, dict] = {}
import vllm._custom_ops as ops
# ---------------- quantizers ----------------
syn = synthetic().to(DEV)
qv["syn.x"] = syn
for tag, x in [("syn", syn), ("l0u", gd("l0.attn.u")), ("l3u", gd("l3.attn.u")), ("l3mu", gd("l3.mlp.u"))]:
if tag != "syn":
qv[f"x.{tag}"] = x
q, s = fp8_linear(x)
qv[f"fp8.linear.{tag}.q"], qv[f"fp8.linear.{tag}.s"] = q, s
if tag != "syn":
gq, gs = G.fp8_quant(x.float().cpu(), "linear")
report[f"fp8.linear.{tag}.golden"] = code_gap(q, gq)
x = gd("l0.gdn.normed")
qv["fp8.linear.l0n.x"] = x
q, s = fp8_linear(x)
qv["fp8.linear.l0n.q"], qv["fp8.linear.l0n.s"] = q, s
report["fp8.linear.l0n.golden"] = code_gap(q, G.fp8_quant(x.float().cpu(), "linear")[0])
core, gate = gd("l3.attn.core"), gd("l3.attn.gate")
q, s = fp8_gated(core, gate)
qv["fp8.gated.core"], qv["fp8.gated.gate"] = core, gate
qv["fp8.gated.q"], qv["fp8.gated.s"] = q, s
report["fp8.gated.golden"] = code_gap(q, gold["l3.attn.oq"])
for tag, x in [("syn", syn), ("l3mu", gd("l3.mlp.u")), ("l3act", gd("l3.shared.act"))]:
q, s = fp8_eager(x)
qv[f"fp8.eager.{tag}.q"], qv[f"fp8.eager.{tag}.s"] = q, s
if tag != "syn":
report[f"fp8.eager.{tag}.golden"] = code_gap(q, G.fp8_quant(x.float().cpu(), "eager")[0])
qv["fp8.eager.l3act.x"] = gd("l3.shared.act")
from vllm.model_executor.layers.quantization.utils.fp8_utils import per_token_group_quant_fp8
q, s = per_token_group_quant_fp8(syn, 128, eps=1e-10, dtype=torch.float8_e4m3fn, use_ue8m0=False)
qv["fp8.group.syn.q"], qv["fp8.group.syn.s"] = q, s
gs1, gs2 = [float(v) for v in gold["l3.mlp.gs"]]
qv["fp4.gs"] = torch.tensor([gs1, gs2], dtype=F32)
for tag, x in [("syn", syn), ("l3mu", gd("l3.mlp.u"))]:
q, s = ops.scaled_fp4_quant(x, torch.tensor([gs1], dtype=F32, device=DEV), is_sf_swizzled_layout=False)
qv[f"fp4.input.{tag}.q"], qv[f"fp4.input.{tag}.s"] = q, s.view(torch.float8_e4m3fn).reshape(x.shape[0], -1)
gq, gs_ = G.nvfp4_quant(gold["l3.mlp.u"].float(), gs1)
packed = (gq[:, 0::2] | (gq[:, 1::2] << 4)).to(torch.uint8)
report["fp4.input.l3mu.golden"] = {
"codes": code_gap(qv["fp4.input.l3mu.q"], packed),
"scales": code_gap(qv["fp4.input.l3mu.s"], gs_),
}
import flashinfer
inter = gd("l3.mlp.swiglu").reshape(-1, 640)
qv["fp4.inter.l3.x"] = inter
syn640 = syn[:, :640].contiguous()
for tag, x in [("syn", syn640), ("l3", inter)]:
q, s = flashinfer.fp4_quantize(
x, torch.tensor([gs2], dtype=F32, device=DEV), sf_vec_size=16, sf_use_ue8m0=False,
is_sf_swizzled_layout=False,
)
qv[f"fp4.inter.{tag}.q"], qv[f"fp4.inter.{tag}.s"] = q, s.view(torch.float8_e4m3fn).reshape(x.shape[0], -1)
gq = gold["l3.mlp.iq"].reshape(-1, 640).to(torch.int32)
packed = (gq[:, 0::2] | (gq[:, 1::2] << 4)).to(torch.uint8)
report["fp4.inter.l3.golden"] = {
"codes": code_gap(qv["fp4.inter.l3.q"], packed),
"scales": code_gap(qv["fp4.inter.l3.s"], gold["l3.mlp.is"].reshape(-1, 40)),
}
print("quantizers done", flush=True)
# ---------------- W8A8 (cutlass_scaled_mm, as CutlassFp8BlockScaledMMKernel calls it) ----------
def scaled_mm(q, s, w, ws):
n, k = w.shape
out = torch.empty((q.shape[0], n), dtype=BF16, device=DEV)
a_s = s.t().contiguous().t() # [M, K/128], column major
torch.ops._C.cutlass_scaled_mm(out, q, w.t(), a_s, ws.t(), None)
return out
p = G.PREFIX + "layers.3.self_attn.o_proj"
w = ck.get(p + ".weight").to(DEV)
ws = ck.get(p + ".weight_scale_inv").to(DEV)
ev["w8a8.o_proj"] = scaled_mm(qv["fp8.gated.q"], qv["fp8.gated.s"], w, ws)
report["w8a8.o_proj.golden"] = gap(ev["w8a8.o_proj"], gold["l3.attn.y"])
del w, ws
p = G.PREFIX + "layers.3.mlp.shared_expert.down_proj"
w = ck.get(p + ".weight").to(DEV)
ws = ck.get(p + ".weight_scale_inv").to(DEV)
ev["w8a8.shared_down"] = scaled_mm(qv["fp8.eager.l3act.q"], qv["fp8.eager.l3act.s"], w, ws)
report["w8a8.shared_down.golden"] = gap(ev["w8a8.shared_down"], gold["l3.shared.down"])
del w, ws
torch.cuda.empty_cache()
print("w8a8 done", flush=True)
# ---------------- MoE: FlashInfer cutlass_fused_moe over layer 3's selected experts ----------
ev["moe.routed"] = moe(ck, gold, report)
torch.cuda.empty_cache()
print("moe done", flush=True)
# ---------------- GDN prefill and decode (layer 0) ----------------
gdn(ck, gold, ev, report)
torch.cuda.empty_cache()
print("gdn done", flush=True)
# ---------------- attention (layer 3) ----------------
attention(ck, gold, ev, report)
print("attention done", flush=True)
peak = torch.cuda.max_memory_allocated()
meta = {
"vllm": __import__("vllm").__version__,
"flashinfer": flashinfer.__version__,
"triton": __import__("triton").__version__,
"torch": torch.__version__,
"gpu": torch.cuda.get_device_name(),
"golden_sha256": G.sha256(args.golden),
"kernels": json.dumps({k: v[2] for k, v in G.KERNELS.items()}),
"gaps": json.dumps(report),
"device_peak": str(peak),
}
for path, tensors in ((args.quant_out, qv), (args.out, ev)):
path.parent.mkdir(parents=True, exist_ok=True)
save_file({k: v.detach().contiguous().cpu() for k, v in tensors.items()}, str(path), metadata=meta)
print(f"wrote {path} ({path.stat().st_size / 1e6:.2f} MB, {len(tensors)} tensors), sha256 {G.sha256(path)}")
print(json.dumps(report, indent=1))
print(f"device peak {peak / 2**30:.3f} GiB")
assert peak <= LIMIT, peak
def moe(ck: "G.Checkpoint", gold: dict, report: dict) -> torch.Tensor:
from flashinfer.fused_moe import cutlass_fused_moe
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import swizzle_blockscale
import vllm._custom_ops as ops
ids = gold["l3.ids"]
wts = gold["l3.weights"]
sel = sorted(set(ids.reshape(-1).tolist()))
remap = {e: n for n, e in enumerate(sel)}
local = torch.tensor([[remap[int(e)] for e in row] for row in ids.tolist()], dtype=torch.int32, device=DEV)
p = G.PREFIX + "layers.3.mlp.experts"
in13 = max(float(ck.get(f"{p}.{e}.{n}_proj.input_scale")) for e in range(512) for n in ("gate", "up"))
in2 = max(float(ck.get(f"{p}.{e}.down_proj.input_scale")) for e in range(512))
w13, s13, w2, s2, a1, a2 = [], [], [], [], [], []
for e in sel:
g = ck.get(f"{p}.{e}.gate_proj.weight")
u = ck.get(f"{p}.{e}.up_proj.weight")
gs = ck.get(f"{p}.{e}.gate_proj.weight_scale")
us = ck.get(f"{p}.{e}.up_proj.weight_scale")
# FlashInfer takes [w3; w1] = [up; gate]
w13.append(torch.cat([u, g]))
s13.append(torch.cat([us, gs]))
w2.append(ck.get(f"{p}.{e}.down_proj.weight"))
s2.append(ck.get(f"{p}.{e}.down_proj.weight_scale"))
a1.append(float(ck.get(f"{p}.{e}.gate_proj.weight_scale_2")))
a2.append(float(ck.get(f"{p}.{e}.down_proj.weight_scale_2")))
w13 = torch.stack(w13).to(DEV)
w2 = torch.stack(w2).to(DEV)
s13 = swizzle_blockscale(torch.stack(s13).to(DEV))
s2 = swizzle_blockscale(torch.stack(s2).to(DEV))
in13_t = torch.tensor(in13, dtype=F32, device=DEV)
in2_t = torch.tensor(in2, dtype=F32, device=DEV)
g1 = torch.tensor(a1, dtype=F32, device=DEV) * in13_t
g2 = torch.tensor(a2, dtype=F32, device=DEV) * in2_t
a1g = 1.0 / in13_t
a2g = 1.0 / in2_t
x = gold["l3.mlp.u"].to(DEV)
xq, xs = ops.scaled_fp4_quant(x, a1g, is_sf_swizzled_layout=True)
out = torch.empty_like(x)
cutlass_fused_moe(
input=xq,
token_selected_experts=local,
token_final_scales=wts.to(DEV),
fc1_expert_weights=w13.view(torch.long),
fc2_expert_weights=w2.view(torch.long),
output_dtype=BF16,
quant_scales=[a1g, s13.view(torch.int32), g1, a2g, s2.view(torch.int32), g2],
input_sf=xs,
output=out,
)
report["moe.routed.golden"] = gap(out, gold["l3.routed"])
del w13, w2, s13, s2
return out
def gdn(ck: "G.Checkpoint", gold: dict, ev: dict, report: dict) -> None:
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn, causal_conv1d_update
from vllm.third_party.flash_linear_attention.ops import chunk_gated_delta_rule
from vllm.third_party.flash_linear_attention.ops.fused_gdn_prefill_post_conv import fused_post_conv_prep
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import layer_norm_fwd
import vllm._custom_ops as ops
p = G.PREFIX + "layers.0.linear_attn"
qkvz = gold["l0.gdn.qkvz"].to(DEV)
ba = gold["l0.gdn.ba"].to(DEV)
t = qkvz.shape[0]
dim = 2 * G.KEY_DIM + G.VAL_DIM
mixed = qkvz[:, :dim].contiguous()
z = qkvz[:, dim:].contiguous().reshape(t, G.NV, G.DV)
b, a = ba[:, : G.NV].contiguous(), ba[:, G.NV :].contiguous()
w = ck.get(p + ".conv1d.weight").to(DEV).reshape(dim, 4)
a_log = ck.get(p + ".A_log").to(DEV).float()
dt = ck.get(p + ".dt_bias").to(DEV)
nw = ck.get(p + ".norm.weight").to(DEV)
def prefill(n: int, conv_state: torch.Tensor, ssm: torch.Tensor):
"""causal_conv1d_fn -> fused_post_conv_prep -> FLA chunk -> gated norm, over tokens [0, n)."""
x = mixed[:n]
qsl = torch.tensor([0, n], dtype=torch.int32, device=DEV)
conv = causal_conv1d_fn(
x.transpose(0, 1),
w,
None,
activation="silu",
conv_states=conv_state,
has_initial_state=torch.zeros(1, dtype=torch.bool, device=DEV),
cache_indices=torch.zeros(1, dtype=torch.int32, device=DEV),
query_start_loc=qsl,
).transpose(0, 1)
q, k, v, g, beta = fused_post_conv_prep(
conv_output=conv.contiguous(), a=a[:n], b=b[:n], A_log=a_log, dt_bias=dt,
num_k_heads=G.NK, head_k_dim=G.DK, head_v_dim=G.DV, apply_l2norm=True, output_g_exp=False,
)
o, state = chunk_gated_delta_rule(
q=q.unsqueeze(0), k=k.unsqueeze(0), v=v.unsqueeze(0), g=g.unsqueeze(0), beta=beta.unsqueeze(0),
initial_state=torch.zeros(1, G.NV, G.DV, G.DK, dtype=F32, device=DEV),
output_final_state=True, cu_seqlens=qsl.long(), use_qk_l2norm_in_kernel=False,
)
core = o.squeeze(0).to(BF16)
ssm[0] = state[0].to(ssm.dtype)
normed = torch.empty_like(core)
layer_norm_fwd(
core.reshape(-1, G.DV), nw.contiguous(), None, 1e-6, z=z[:n].reshape(-1, G.DV),
out=normed.reshape(-1, G.DV), group_size=G.DV, norm_before_gate=True, is_rms_norm=True,
activation="sigmoid",
)
return conv, q, k, v, g, beta, core, normed
conv_state = torch.zeros(1, dim, 3 + 4, dtype=BF16, device=DEV)
ssm = torch.zeros(1, G.NV, G.DV, G.DK, dtype=F32, device=DEV)
conv, q, k, v, g, beta, core, normed = prefill(t, conv_state, ssm)
names = {"conv": conv, "q": q.reshape(t, -1), "k": k.reshape(t, -1), "v": v.reshape(t, -1), "g": g, "beta": beta, "core": core.reshape(t, -1), "normed": normed.reshape(t, -1)}
for n, val in names.items():
ev[f"gdn.prefill.{n}"] = val
report[f"gdn.prefill.{n}.golden"] = gap(val, gold[f"l0.gdn.{n}"])
# served state is [HV, V, K]; the golden keeps every fourth head as [K, V]
ev["gdn.prefill.state"] = ssm[0].contiguous()
gs = gold["l0.gdn.state"]
served = ssm[0][::4].transpose(-1, -2).cpu()
report["gdn.prefill.state.golden_rel"] = float((served - gs).abs().max() / gs.abs().max())
# decode: one step for the last token from the (t-1)-token state, and one 5-token step over
# the last five from the (t-5)-token state, through the fused MTP decode op
for n, tag in ((1, "decode1"), (5, "decode5")):
conv_state = torch.zeros(1, dim, 3 + 4, dtype=BF16, device=DEV)
ssm = torch.zeros(1, G.NV, G.DV, G.DK, dtype=F32, device=DEV)
prefill(t - n, conv_state, ssm)
rows = slice(t - n, t)
qsl = torch.tensor([0, n], dtype=torch.int32, device=DEV)
idx = torch.zeros(1, n, dtype=torch.int32, device=DEV)
acc = torch.ones(1, dtype=torch.int32, device=DEV)
xs = causal_conv1d_update(
mixed[rows].contiguous(), conv_state, w, None, "silu", conv_state_indices=idx[:, 0],
num_accepted_tokens=acc, query_start_loc=qsl, max_query_len=n, validate_data=False,
)
out = torch.zeros(n, G.NV, G.DV, dtype=BF16, device=DEV)
ops.fused_gdn_decode_post_conv_mtp(
mixed_qkv=xs, a=a[rows].contiguous(), b=b[rows].contiguous(), A_log=a_log, dt_bias=dt,
state_indices=idx, cu_seqlens=qsl, num_accepted_tokens=acc, state=ssm,
output_gate=z[rows].contiguous(), norm_weight=nw, out=out, scale=G.DK**-0.5, norm_eps=1e-6,
output_gate_activation="sigmoid",
)
ev[f"gdn.{tag}.normed"] = out.reshape(n, -1)
ev[f"gdn.{tag}.state"] = ssm[0].contiguous()
report[f"gdn.{tag}.normed.golden"] = gap(out.reshape(n, -1), gold["l0.gdn.normed"][rows])
def attention(ck: "G.Checkpoint", gold: dict, ev: dict, report: dict) -> None:
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.models.qwen4_exp.nvidia.ops.qsa import qsa_sparse_paged_attention
cfg = json.loads((Path(ck.snapshot) / "config.json").read_text())["text_config"]
p = G.PREFIX + "layers.3.self_attn"
qkv = gold["l3.attn.qkv"].to(DEV)
t = qkv.shape[0]
from vllm.config import VllmConfig, set_current_vllm_config
# a CustomOp reads the current config when it is built; the defaults are what matters here
with torch.device(DEV), set_current_vllm_config(VllmConfig()):
rope = get_rope(head_size=G.HD, max_position=4096, rope_parameters=cfg["rope_parameters"], dtype=BF16)
ev["attn.cos_sin"] = rope.cos_sin_cache[:t].to(BF16)
pos = torch.arange(t, device=DEV)
positions = pos.unsqueeze(0).expand(3, t).contiguous()
qg = qkv[:, : G.NQ * 2 * G.HD]
k = qkv[:, G.NQ * 2 * G.HD : G.NQ * 2 * G.HD + G.NKV * G.HD]
v = qkv[:, G.NQ * 2 * G.HD + G.NKV * G.HD :]
q, k, gate = fused_qk_rmsnorm_rope_gate(
qg.contiguous(), k.contiguous(), ck.get(p + ".q_norm.weight").to(DEV), ck.get(p + ".k_norm.weight").to(DEV),
rope.cos_sin_cache, positions, 1e-6, G.NQ, G.NKV, G.HD, rope.rotary_dim, norm_beta=1.0,
mrope_section=getattr(rope, "mrope_section", None),
)
for n, val in {"q": q, "k": k, "gate": gate}.items():
ev[f"attn.normrope.{n}"] = val
report[f"attn.normrope.{n}.golden"] = gap(val, gold[f"l3.attn.{n}"])
# dense causal indices over a paged bf16 cache holding the 24 tokens
width = 2051
page = 16
pages = (t + page - 1) // page
kc = torch.zeros(pages, page, G.NKV, G.HD, dtype=BF16, device=DEV)
vc = torch.zeros_like(kc)
kc.reshape(-1, G.NKV, G.HD)[:t] = k.reshape(t, G.NKV, G.HD)
vc.reshape(-1, G.NKV, G.HD)[:t] = v.reshape(t, G.NKV, G.HD)
table = torch.arange(pages, dtype=torch.int32, device=DEV).unsqueeze(0)
idx = torch.full((t, width), -1, dtype=torch.int32, device=DEV)
for r in range(t):
idx[r, : r + 1] = torch.arange(r + 1, dtype=torch.int32, device=DEV)
qh = q.reshape(t, G.NQ, G.HD)
out = qsa_sparse_paged_attention(qh, kc, vc, idx, table, torch.zeros(t, dtype=torch.int32, device=DEV))
ev["attn.core24"] = out.reshape(t, -1)
report["attn.core24.golden"] = gap(out.reshape(t, -1), gold["l3.attn.core"])
one = qsa_sparse_paged_attention(
qh[t - 1 :].contiguous(), kc, vc, idx[t - 1 :].contiguous(), table, torch.zeros(1, dtype=torch.int32, device=DEV)
)
ev["attn.core1"] = one.reshape(1, -1)
report["attn.core1.golden"] = gap(one.reshape(1, -1), gold["l3.attn.core"][t - 1 :])
if __name__ == "__main__":
main()