Files
ml/hanzo-nn/tests/cpu_flash_attn.rs
zeekay 35bbf78aea ModernBERT runs quantized inference packed on the CPU and its attention fused on Metal.
quantize turns the projections the CPU tiles into Q8_0 blocks. A quantized backbone then runs
each sequence's real tokens only: embeddings, norms and projections over the packed tokens,
attention per (sequence, head) straight from the QKV projection (attention::cpu_flash::packed:
rotation, window, softmax and both products in one parallel pass), GeGLU in one vectorized pass
(ops::geglu, erf within 1.5e-7 of exact, the composite wherever a gradient flows); pad positions
come back zero. On Metal, inference takes one SDPA kernel for scores, mask, softmax and values.
Training's path is unchanged.

Claude-Session: https://claude.ai/code/session_018Gt5TvrE2rxXuZikWSauv3
2026-09-27 05:00:26 -07:00

340 lines
11 KiB
Rust

use hanzo_ml::{DType, Device, Result, Tensor};
use hanzo_nn::attention::{flash_attn, AttnMask};
/// Reference SDPA: softmax(Q @ K^T * scale) @ V
fn reference_sdpa(q: &Tensor, k: &Tensor, v: &Tensor, scale: f32) -> Result<Tensor> {
let att = (q.clone() * scale as f64)?.matmul(&k.clone().t()?)?;
let att = hanzo_nn::ops::softmax_last_dim(&att.to_dtype(DType::F32)?)?.to_dtype(q.dtype())?;
att.matmul(&v.clone())
}
/// Reference causal SDPA with mask
fn reference_causal_sdpa(q: &Tensor, k: &Tensor, v: &Tensor, scale: f32) -> Result<Tensor> {
let (_, _, s_q, _) = q.dims4()?;
let (_, _, s_kv, _) = k.dims4()?;
let offset = s_kv - s_q;
let mask: Vec<f32> = (0..s_q)
.flat_map(|i| {
(0..s_kv).map(move |j| {
if j as isize <= i as isize + offset as isize {
0.0
} else {
f32::NEG_INFINITY
}
})
})
.collect();
let mask = Tensor::from_vec(mask, (1, 1, s_q, s_kv), q.device())?;
let att = (q.clone() * scale as f64)?.matmul(&k.clone().t()?)?;
let att = att.broadcast_add(&mask)?;
let att = hanzo_nn::ops::softmax_last_dim(&att.to_dtype(DType::F32)?)?.to_dtype(q.dtype())?;
att.matmul(&v.clone())
}
fn assert_close(a: &Tensor, b: &Tensor, tol: f32, label: &str) -> Result<()> {
let a_flat: Vec<f32> = a.to_dtype(DType::F32)?.flatten_all()?.to_vec1()?;
let b_flat: Vec<f32> = b.to_dtype(DType::F32)?.flatten_all()?.to_vec1()?;
let max_diff = a_flat
.iter()
.zip(b_flat.iter())
.map(|(x, y)| (x - y).abs())
.fold(0.0f32, f32::max);
assert!(
max_diff < tol,
"{label}: max diff {max_diff:.6e} > tol {tol:.6e}"
);
Ok(())
}
// Causal decode (q_len=1)
#[test]
fn causal_decode_b1() -> Result<()> {
let (h, d, kv_len) = (4, 16, 32);
let scale = 1.0 / (d as f32).sqrt();
let dev = &Device::Cpu;
let q = Tensor::randn(0f32, 1f32, (1, h, 1, d), dev)?;
let k = Tensor::randn(0f32, 1f32, (1, h, kv_len, d), dev)?;
let v = Tensor::randn(0f32, 1f32, (1, h, kv_len, d), dev)?;
let expected = reference_sdpa(&q, &k, &v, scale)?;
// flash_attn expects (B, S, H, D)
let out = flash_attn::<f32>(
&q.transpose(1, 2)?,
&k.transpose(1, 2)?,
&v.transpose(1, 2)?,
scale,
AttnMask::causal_with_offset(kv_len - 1),
None,
None,
)?;
// Output (B, H, S, D), compare
assert_close(&out, &expected, 1e-5, "causal_decode_b1")
}
// Causal prefill
#[test]
fn causal_prefill_b1() -> Result<()> {
let (h, d, s) = (4, 16, 8);
let scale = 1.0 / (d as f32).sqrt();
let dev = &Device::Cpu;
let q = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
let k = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
let v = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
let expected = reference_causal_sdpa(&q, &k, &v, scale)?;
let out = flash_attn::<f32>(
&q.transpose(1, 2)?,
&k.transpose(1, 2)?,
&v.transpose(1, 2)?,
scale,
AttnMask::causal(),
None,
None,
)?;
assert_close(&out, &expected, 1e-5, "causal_prefill_b1")
}
// Causal prefill with KV offset (simulating KV cache)
#[test]
fn causal_prefill_with_offset() -> Result<()> {
let (h, d, s_q, s_kv) = (4, 16, 4, 12);
let scale = 1.0 / (d as f32).sqrt();
let dev = &Device::Cpu;
let offset = s_kv - s_q; // 8
let q = Tensor::randn(0f32, 1f32, (1, h, s_q, d), dev)?;
let k = Tensor::randn(0f32, 1f32, (1, h, s_kv, d), dev)?;
let v = Tensor::randn(0f32, 1f32, (1, h, s_kv, d), dev)?;
let expected = reference_causal_sdpa(&q, &k, &v, scale)?;
let out = flash_attn::<f32>(
&q.transpose(1, 2)?,
&k.transpose(1, 2)?,
&v.transpose(1, 2)?,
scale,
AttnMask::causal_with_offset(offset),
None,
None,
)?;
assert_close(&out, &expected, 1e-5, "causal_prefill_with_offset")
}
// GQA (fewer KV heads)
#[test]
fn causal_decode_gqa() -> Result<()> {
let (h_q, h_kv, d, kv_len) = (8, 2, 16, 20);
let scale = 1.0 / (d as f32).sqrt();
let dev = &Device::Cpu;
let q = Tensor::randn(0f32, 1f32, (1, h_q, 1, d), dev)?;
let k = Tensor::randn(0f32, 1f32, (1, h_kv, kv_len, d), dev)?;
let v = Tensor::randn(0f32, 1f32, (1, h_kv, kv_len, d), dev)?;
// Reference: expand K/V to match Q heads
let reps = h_q / h_kv;
let k_exp = k
.reshape((1, h_kv, 1, kv_len, d))?
.broadcast_as((1, h_kv, reps, kv_len, d))?
.reshape((1, h_q, kv_len, d))?;
let v_exp = v
.reshape((1, h_kv, 1, kv_len, d))?
.broadcast_as((1, h_kv, reps, kv_len, d))?
.reshape((1, h_q, kv_len, d))?;
let expected = reference_sdpa(&q, &k_exp, &v_exp, scale)?;
let out = flash_attn::<f32>(
&q.transpose(1, 2)?,
&k.transpose(1, 2)?,
&v.transpose(1, 2)?,
scale,
AttnMask::causal_with_offset(kv_len - 1),
None,
None,
)?;
assert_close(&out, &expected, 1e-5, "causal_decode_gqa")
}
// Standard (no mask)
#[test]
fn standard_no_mask_b1() -> Result<()> {
let (h, d, s) = (4, 16, 8);
let scale = 1.0 / (d as f32).sqrt();
let dev = &Device::Cpu;
let q = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
let k = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
let v = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
let expected = reference_sdpa(&q, &k, &v, scale)?;
let out = flash_attn::<f32>(
&q.transpose(1, 2)?,
&k.transpose(1, 2)?,
&v.transpose(1, 2)?,
scale,
AttnMask::None,
None,
None,
)?;
assert_close(&out, &expected, 1e-5, "standard_no_mask_b1")
}
// Standard with explicit mask
#[test]
fn standard_with_mask_b1() -> Result<()> {
let (h, d, s) = (2, 8, 4);
let scale = 1.0 / (d as f32).sqrt();
let dev = &Device::Cpu;
let q = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
let k = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
let v = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
// Causal mask as explicit tensor
let mask: Vec<f32> = (0..s)
.flat_map(|i| (0..s).map(move |j| if j <= i { 0.0 } else { f32::NEG_INFINITY }))
.collect();
let mask_tensor = Tensor::from_vec(mask, (1, s, s), dev)?;
// Reference with mask applied
let att = (q.clone() * scale as f64)?.matmul(&k.clone().t()?)?;
let att = att.broadcast_add(&mask_tensor.unsqueeze(0)?)?;
let att = hanzo_nn::ops::softmax_last_dim(&att)?;
let expected = att.matmul(&v.clone())?;
let out = flash_attn::<f32>(
&q.transpose(1, 2)?,
&k.transpose(1, 2)?,
&v.transpose(1, 2)?,
scale,
AttnMask::Mask(mask_tensor),
None,
None,
)?;
assert_close(&out, &expected, 1e-4, "standard_with_mask_b1")
}
// Interleaved KV decode
#[test]
fn interleaved_kv_decode() -> Result<()> {
use hanzo_nn::attention::cpu_flash::causal::causal_decode_f32_interleaved;
let (h_q, h_kv, d, kv_len) = (8, 2, 16, 20);
let scale = 1.0 / (d as f32).sqrt();
let dev = &Device::Cpu;
let q = Tensor::randn(0f32, 1f32, (1, h_q, 1, d), dev)?;
let k = Tensor::randn(0f32, 1f32, (1, h_kv, kv_len, d), dev)?;
let v = Tensor::randn(0f32, 1f32, (1, h_kv, kv_len, d), dev)?;
// Reference with GQA expansion
let reps = h_q / h_kv;
let k_exp = k
.reshape((1, h_kv, 1, kv_len, d))?
.broadcast_as((1, h_kv, reps, kv_len, d))?
.reshape((1, h_q, kv_len, d))?;
let v_exp = v
.reshape((1, h_kv, 1, kv_len, d))?
.broadcast_as((1, h_kv, reps, kv_len, d))?
.reshape((1, h_q, kv_len, d))?;
let expected = reference_sdpa(&q, &k_exp, &v_exp, scale)?;
// Build interleaved KV: (kv_len, h_kv, 2*d).
let k_seq = k.squeeze(0)?.transpose(0, 1)?.contiguous()?;
let v_seq = v.squeeze(0)?.transpose(0, 1)?.contiguous()?;
let kv = Tensor::cat(&[&k_seq, &v_seq], 2)?.contiguous()?; // (kv_len, h_kv, 2*d)
let kv_data: Vec<f32> = kv.flatten_all()?.to_vec1()?;
let q_flat: Vec<f32> = q
.squeeze(0)?
.squeeze(1)?
.contiguous()?
.flatten_all()?
.to_vec1()?;
let out = causal_decode_f32_interleaved(&q_flat, &kv_data, h_q, h_kv, d, kv_len, scale)?;
let out = out.unsqueeze(0)?;
assert_close(&out, &expected, 1e-5, "interleaved_kv_decode")
}
/// Packed attention equals the composite per sequence: rotary on queries and keys, scaled
/// queries, a softmax over the keys within the window, the product with the values.
#[test]
fn packed_is_the_composite() -> Result<()> {
use hanzo_nn::attention::cpu_flash::packed::packed;
let dev = Device::Cpu;
let (heads, d, p) = (3, 8, 200);
let lens = [5usize, 1, 70, 130];
let t: usize = lens.iter().sum();
let qkv = Tensor::randn(0f32, 1.0, (t, 3 * heads * d), &dev)?;
let freqs: Vec<f32> = (0..p)
.flat_map(|i| (0..d / 2).map(move |j| i as f32 / 1000f32.powf(2. * j as f32 / d as f32)))
.collect();
let freqs = Tensor::from_vec(freqs, (p, d / 2), &dev)?;
let (cos, sin) = (freqs.cos()?, freqs.sin()?);
let scale = 0.35f32;
for window in [None, Some(4), Some(64)] {
let got = packed(&qkv, &lens, heads, window, &cos, &sin, scale)?;
let mut start = 0;
for &l in &lens {
// [l, 3, H, D] -> q, k, v as [1, H, l, D]
let x = qkv.narrow(0, start, l)?.reshape((l, 3, heads, d))?;
let part = |i: usize| {
x.narrow(1, i, 1)?
.squeeze(1)?
.transpose(0, 1)?
.unsqueeze(0)?
.contiguous()
};
let q = hanzo_nn::rotary_emb::rope(&part(0)?, &cos, &sin)?;
let k = hanzo_nn::rotary_emb::rope(&part(1)?, &cos, &sin)?;
let q = (q * scale as f64)?;
let mut s = q.matmul(&k.transpose(2, 3)?)?;
if let Some(w) = window {
let m: Vec<f32> = (0..l)
.flat_map(|i| {
(0..l).map(move |j| {
if i.abs_diff(j) > w {
f32::NEG_INFINITY
} else {
0.
}
})
})
.collect();
s = s.broadcast_add(&Tensor::from_vec(m, (l, l), &dev)?)?;
}
let o = hanzo_nn::ops::softmax_last_dim(&s)?.matmul(&part(2)?)?;
let want = o.squeeze(0)?.transpose(0, 1)?.reshape((l, heads * d))?;
let gap = (got.narrow(0, start, l)? - want)?
.abs()?
.max_all()?
.to_scalar::<f32>()?;
assert!(gap < 1e-5, "window {window:?}, length {l}: {gap}");
start += l;
}
}
Ok(())
}