Files
cklxxandGitHub 2a2a2b1a3d Optional TileLang GPU fast path: 4-5x lower latency per call, identical answers (#25)
* feat: optional TileLang GPU fast path (laya.load(..., fast=True))

Adds laya/tl_kernels.py (fused GEMM+epilogue, GEMM+GEGLU, residual+LayerNorm,
in-place RoPE, padding-aware sliding-window flash attention) and laya/fast.py,
which rebuilds the DecisionModel forward from those kernels with bf16-resident
weights and a CUDA graph per (batch, length) bucket. Agent.accelerate() /
deaccelerate() and the `fast=` flag on load(); the `laya[fast]` extra pulls
tilelang. Nothing is imported or changed unless fast=True is requested; without
CUDA or tilelang the stock forward is used (warning, or raise with strict=True).

Numerics match the stock bf16 path (AG News / emotion accuracy and ECE identical
within noise, 99.5% argmax agreement, same max |dp| vs fp32). 1-question calls go
from 14-18 ms to 3-5 ms on an RTX 4070 Ti SUPER; 30-question batches 1.2-1.9x.

benchmarks/bench_fast.py reproduces the numbers (--eval N adds the dataset
comparison); tests/test_fast.py checks each kernel against torch and the full
forward against the stock model (skips without CUDA/tilelang).

* fix(fast): match HF's sliding window bound, keep the residual stream in fp32; add parity evidence

Two deviations from the stock forward surfaced on a fixed-state parity run
(benchmarks/parity_fast.py, new):

- The local-attention kernel used ModernBertAttention.sliding_window (config
  value + 1, flash-attn's inclusive convention). HF's bidirectional mask keeps
  |i - j| <= config.sliding_window, so every local layer saw one extra key per
  side. On ModernBERT-large this moved noul probabilities by up to 0.6 on some
  inputs (mmBERT-base was barely affected). The kernel now takes
  config.sliding_window.
- The residual stream was stored in bf16 between layers; the stock path keeps
  it in fp32 and ModernBERT-large's residual activations reach ~3e4, where bf16
  loses ~100 units per add. add_ln_kernel now updates an fp32 stream and emits
  the bf16 normalised input; embeddings are gathered from an exact fp16 copy.

Result on 60 fixed states x up to 8 questions (both checkpoints): the fast path
is at least as close to an fp32 forward as the stock bf16 path (max |dp| vs fp32
<= 0.046 fast vs <= 0.058 stock), argmax agreement >= 47/48 per question type,
and on every disagreement the fast path sides with fp32. Per-option
probabilities for all three paths are in benchmarks/results/parity_*.json.
Latency is unchanged (1-question e2e 3.2 ms multilingual / 5.3 ms English).
2026-09-23 19:01:15 +05:30

87 lines
4.7 KiB
Python

"""TileLang fast path: kernels vs torch reference, and full forward vs the stock model.
Skips unless CUDA + tilelang are available. Run: python -m pytest tests/test_fast.py -q
"""
import os, sys
import pytest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
torch = pytest.importorskip("torch")
if not torch.cuda.is_available():
pytest.skip("needs CUDA", allow_module_level=True)
pytest.importorskip("tilelang")
from laya import tl_kernels as K # noqa: E402
dev = "cuda"
def err(a, b): return (a.float() - b.float()).abs().max().item()
def test_gemm_epilogues():
M, Kd = 200, 768 # M not a multiple of the tile on purpose
A = torch.randn(M, Kd, device=dev, dtype=torch.bfloat16)
for N, bias, act in [(2304, False, "none"), (768, True, "gelu"), (3072, True, "relu")]:
W = torch.randn(N, Kd, device=dev, dtype=torch.bfloat16) * 0.02; b = torch.randn(N, device=dev)
C = torch.empty(M, N, device=dev, dtype=torch.bfloat16)
K.gemm_kernel(N, Kd, bias=bias, act=act)(A, W, b, C)
ref = A.float() @ W.float().T + (b if bias else 0)
ref = {"none": ref, "gelu": torch.nn.functional.gelu(ref), "relu": torch.relu(ref)}[act]
assert err(C, ref) < 0.05
def test_geglu():
M, Kd, F = 256, 768, 1152
A = torch.randn(M, Kd, device=dev, dtype=torch.bfloat16); Wi = torch.randn(2 * F, Kd, device=dev, dtype=torch.bfloat16) * 0.02
C = torch.empty(M, F, device=dev, dtype=torch.bfloat16); K.gemm_geglu_kernel(F, Kd)(A, Wi, C)
x = A.float() @ Wi.float().T
assert err(C, torch.nn.functional.gelu(x[:, :F]) * x[:, F:]) < 0.05
def test_add_layernorm():
M, D = 100, 768
X = torch.randn(M, D, device=dev) * 3000 # fp32 residual stream at ModernBERT-large's real magnitude
R = torch.randn(M, D, device=dev, dtype=torch.bfloat16) * 50
w = torch.rand(D, device=dev) + 0.5; b = torch.randn(D, device=dev)
X2 = X.clone(); Y = torch.empty(M, D, device=dev, dtype=torch.bfloat16)
K.add_ln_kernel(D, residual=True, bias=True)(X2, R, w, b, Y)
xr = X + R.float()
assert torch.equal(X2, xr) # the stream is updated exactly, in fp32
assert err(Y, torch.nn.functional.layer_norm(xr, (D,), w, b, 1e-5)) < 0.05
def test_attention_mask_and_window():
H, Dh, B = 12, 64, 3
for L, window, dyn in [(80, 0, True), (200, 65, True), (1024, 65, False), (1024, 0, False)]:
qkv = torch.randn(B, L, 3, H, Dh, device=dev, dtype=torch.bfloat16)
lens = torch.tensor([L, L - 7, max(1, L // 3)], device=dev, dtype=torch.int32)
O = torch.empty(B, L, H * Dh, device=dev, dtype=torch.bfloat16)
K.attn_kernel(None if dyn else B, None if dyn else L, H, Dh, window=window)(qkv, lens, O)
q, k, v = [qkv[:, :, i].transpose(1, 2).float() for i in range(3)]
idx = torch.arange(L, device=dev)
mask = (idx[None, :] < lens[:, None])[:, None, None, :].expand(B, 1, L, L)
if window:
mask = mask & ((idx[:, None] - idx[None, :]).abs() <= window)[None, None]
ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask).transpose(1, 2).reshape(B, L, -1)
valid = idx[None, :] < lens[:, None]
assert torch.isfinite(O).all()
assert (O.float() - ref)[valid].abs().max().item() < 0.02
@pytest.mark.skipif(os.environ.get("LAYA_TEST_MODEL") is None, reason="set LAYA_TEST_MODEL=<repo or path> to run")
def test_full_forward_matches_stock():
import laya
from laya.common import QTYPES, build_sequence, collate_items
agent = laya.load(os.environ["LAYA_TEST_MODEL"], subfolder=os.environ.get("LAYA_TEST_SUBFOLDER"))
q = {"dept": {"type": "choice", "instructions": "Which team?", "criteria": {"billing": "refunds", "tech": "bugs", "sales": "pricing"}},
"urg": {"type": "score", "instructions": "How urgent?", "criteria": ["low", "mid", "high"]},
"churn": {"type": "noul", "instructions": "Threatens to cancel?"}}
st = {"body": "We were billed twice, refund now or we cancel. " * 30}
items = []
for qid in q:
qq = agent._to_internal(q[qid]); seq, m = build_sequence(agent.tok, st, qq, agent.cfg["max_len"], agent.cfg["head_max_len"])
items.append({"ids": seq, "markers": m, "qtype": QTYPES[qq["t"]]})
b = {k: v.cuda() for k, v in collate_items([items], agent.tok.pad_token_id).items() if torch.is_tensor(v)}
def run():
with torch.no_grad(), torch.autocast("cuda", dtype=agent.dtype):
return agent.model(b["input_ids"], b["attention_mask"], b["marker_pos"], b["marker_mask"], b["qtype"])[0]
lo = run(); assert agent.accelerate(strict=True); lf = run()
assert (torch.softmax(lo.float(), -1) - torch.softmax(lf.float(), -1)).abs().max().item() < 0.02