mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-21 22:51:09 +00:00
quantize turns the projections the CPU tiles into Q8_0 blocks. A quantized backbone then runs each sequence's real tokens only: embeddings, norms and projections over the packed tokens, attention per (sequence, head) straight from the QKV projection (attention::cpu_flash::packed: rotation, window, softmax and both products in one parallel pass), GeGLU in one vectorized pass (ops::geglu, erf within 1.5e-7 of exact, the composite wherever a gradient flows); pad positions come back zero. On Metal, inference takes one SDPA kernel for scores, mask, softmax and values. Training's path is unchanged. Claude-Session: https://claude.ai/code/session_018Gt5TvrE2rxXuZikWSauv3
340 lines
11 KiB
Rust
340 lines
11 KiB
Rust
use hanzo_ml::{DType, Device, Result, Tensor};
|
|
use hanzo_nn::attention::{flash_attn, AttnMask};
|
|
|
|
/// Reference SDPA: softmax(Q @ K^T * scale) @ V
|
|
fn reference_sdpa(q: &Tensor, k: &Tensor, v: &Tensor, scale: f32) -> Result<Tensor> {
|
|
let att = (q.clone() * scale as f64)?.matmul(&k.clone().t()?)?;
|
|
let att = hanzo_nn::ops::softmax_last_dim(&att.to_dtype(DType::F32)?)?.to_dtype(q.dtype())?;
|
|
att.matmul(&v.clone())
|
|
}
|
|
|
|
/// Reference causal SDPA with mask
|
|
fn reference_causal_sdpa(q: &Tensor, k: &Tensor, v: &Tensor, scale: f32) -> Result<Tensor> {
|
|
let (_, _, s_q, _) = q.dims4()?;
|
|
let (_, _, s_kv, _) = k.dims4()?;
|
|
let offset = s_kv - s_q;
|
|
|
|
let mask: Vec<f32> = (0..s_q)
|
|
.flat_map(|i| {
|
|
(0..s_kv).map(move |j| {
|
|
if j as isize <= i as isize + offset as isize {
|
|
0.0
|
|
} else {
|
|
f32::NEG_INFINITY
|
|
}
|
|
})
|
|
})
|
|
.collect();
|
|
let mask = Tensor::from_vec(mask, (1, 1, s_q, s_kv), q.device())?;
|
|
|
|
let att = (q.clone() * scale as f64)?.matmul(&k.clone().t()?)?;
|
|
let att = att.broadcast_add(&mask)?;
|
|
let att = hanzo_nn::ops::softmax_last_dim(&att.to_dtype(DType::F32)?)?.to_dtype(q.dtype())?;
|
|
att.matmul(&v.clone())
|
|
}
|
|
|
|
fn assert_close(a: &Tensor, b: &Tensor, tol: f32, label: &str) -> Result<()> {
|
|
let a_flat: Vec<f32> = a.to_dtype(DType::F32)?.flatten_all()?.to_vec1()?;
|
|
let b_flat: Vec<f32> = b.to_dtype(DType::F32)?.flatten_all()?.to_vec1()?;
|
|
let max_diff = a_flat
|
|
.iter()
|
|
.zip(b_flat.iter())
|
|
.map(|(x, y)| (x - y).abs())
|
|
.fold(0.0f32, f32::max);
|
|
assert!(
|
|
max_diff < tol,
|
|
"{label}: max diff {max_diff:.6e} > tol {tol:.6e}"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
// Causal decode (q_len=1)
|
|
|
|
#[test]
|
|
fn causal_decode_b1() -> Result<()> {
|
|
let (h, d, kv_len) = (4, 16, 32);
|
|
let scale = 1.0 / (d as f32).sqrt();
|
|
let dev = &Device::Cpu;
|
|
|
|
let q = Tensor::randn(0f32, 1f32, (1, h, 1, d), dev)?;
|
|
let k = Tensor::randn(0f32, 1f32, (1, h, kv_len, d), dev)?;
|
|
let v = Tensor::randn(0f32, 1f32, (1, h, kv_len, d), dev)?;
|
|
|
|
let expected = reference_sdpa(&q, &k, &v, scale)?;
|
|
|
|
// flash_attn expects (B, S, H, D)
|
|
let out = flash_attn::<f32>(
|
|
&q.transpose(1, 2)?,
|
|
&k.transpose(1, 2)?,
|
|
&v.transpose(1, 2)?,
|
|
scale,
|
|
AttnMask::causal_with_offset(kv_len - 1),
|
|
None,
|
|
None,
|
|
)?;
|
|
// Output (B, H, S, D), compare
|
|
assert_close(&out, &expected, 1e-5, "causal_decode_b1")
|
|
}
|
|
|
|
// Causal prefill
|
|
|
|
#[test]
|
|
fn causal_prefill_b1() -> Result<()> {
|
|
let (h, d, s) = (4, 16, 8);
|
|
let scale = 1.0 / (d as f32).sqrt();
|
|
let dev = &Device::Cpu;
|
|
|
|
let q = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
|
|
let k = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
|
|
let v = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
|
|
|
|
let expected = reference_causal_sdpa(&q, &k, &v, scale)?;
|
|
|
|
let out = flash_attn::<f32>(
|
|
&q.transpose(1, 2)?,
|
|
&k.transpose(1, 2)?,
|
|
&v.transpose(1, 2)?,
|
|
scale,
|
|
AttnMask::causal(),
|
|
None,
|
|
None,
|
|
)?;
|
|
|
|
assert_close(&out, &expected, 1e-5, "causal_prefill_b1")
|
|
}
|
|
|
|
// Causal prefill with KV offset (simulating KV cache)
|
|
|
|
#[test]
|
|
fn causal_prefill_with_offset() -> Result<()> {
|
|
let (h, d, s_q, s_kv) = (4, 16, 4, 12);
|
|
let scale = 1.0 / (d as f32).sqrt();
|
|
let dev = &Device::Cpu;
|
|
let offset = s_kv - s_q; // 8
|
|
|
|
let q = Tensor::randn(0f32, 1f32, (1, h, s_q, d), dev)?;
|
|
let k = Tensor::randn(0f32, 1f32, (1, h, s_kv, d), dev)?;
|
|
let v = Tensor::randn(0f32, 1f32, (1, h, s_kv, d), dev)?;
|
|
|
|
let expected = reference_causal_sdpa(&q, &k, &v, scale)?;
|
|
|
|
let out = flash_attn::<f32>(
|
|
&q.transpose(1, 2)?,
|
|
&k.transpose(1, 2)?,
|
|
&v.transpose(1, 2)?,
|
|
scale,
|
|
AttnMask::causal_with_offset(offset),
|
|
None,
|
|
None,
|
|
)?;
|
|
|
|
assert_close(&out, &expected, 1e-5, "causal_prefill_with_offset")
|
|
}
|
|
|
|
// GQA (fewer KV heads)
|
|
|
|
#[test]
|
|
fn causal_decode_gqa() -> Result<()> {
|
|
let (h_q, h_kv, d, kv_len) = (8, 2, 16, 20);
|
|
let scale = 1.0 / (d as f32).sqrt();
|
|
let dev = &Device::Cpu;
|
|
|
|
let q = Tensor::randn(0f32, 1f32, (1, h_q, 1, d), dev)?;
|
|
let k = Tensor::randn(0f32, 1f32, (1, h_kv, kv_len, d), dev)?;
|
|
let v = Tensor::randn(0f32, 1f32, (1, h_kv, kv_len, d), dev)?;
|
|
|
|
// Reference: expand K/V to match Q heads
|
|
let reps = h_q / h_kv;
|
|
let k_exp = k
|
|
.reshape((1, h_kv, 1, kv_len, d))?
|
|
.broadcast_as((1, h_kv, reps, kv_len, d))?
|
|
.reshape((1, h_q, kv_len, d))?;
|
|
let v_exp = v
|
|
.reshape((1, h_kv, 1, kv_len, d))?
|
|
.broadcast_as((1, h_kv, reps, kv_len, d))?
|
|
.reshape((1, h_q, kv_len, d))?;
|
|
|
|
let expected = reference_sdpa(&q, &k_exp, &v_exp, scale)?;
|
|
|
|
let out = flash_attn::<f32>(
|
|
&q.transpose(1, 2)?,
|
|
&k.transpose(1, 2)?,
|
|
&v.transpose(1, 2)?,
|
|
scale,
|
|
AttnMask::causal_with_offset(kv_len - 1),
|
|
None,
|
|
None,
|
|
)?;
|
|
|
|
assert_close(&out, &expected, 1e-5, "causal_decode_gqa")
|
|
}
|
|
|
|
// Standard (no mask)
|
|
|
|
#[test]
|
|
fn standard_no_mask_b1() -> Result<()> {
|
|
let (h, d, s) = (4, 16, 8);
|
|
let scale = 1.0 / (d as f32).sqrt();
|
|
let dev = &Device::Cpu;
|
|
|
|
let q = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
|
|
let k = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
|
|
let v = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
|
|
|
|
let expected = reference_sdpa(&q, &k, &v, scale)?;
|
|
|
|
let out = flash_attn::<f32>(
|
|
&q.transpose(1, 2)?,
|
|
&k.transpose(1, 2)?,
|
|
&v.transpose(1, 2)?,
|
|
scale,
|
|
AttnMask::None,
|
|
None,
|
|
None,
|
|
)?;
|
|
|
|
assert_close(&out, &expected, 1e-5, "standard_no_mask_b1")
|
|
}
|
|
|
|
// Standard with explicit mask
|
|
|
|
#[test]
|
|
fn standard_with_mask_b1() -> Result<()> {
|
|
let (h, d, s) = (2, 8, 4);
|
|
let scale = 1.0 / (d as f32).sqrt();
|
|
let dev = &Device::Cpu;
|
|
|
|
let q = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
|
|
let k = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
|
|
let v = Tensor::randn(0f32, 1f32, (1, h, s, d), dev)?;
|
|
|
|
// Causal mask as explicit tensor
|
|
let mask: Vec<f32> = (0..s)
|
|
.flat_map(|i| (0..s).map(move |j| if j <= i { 0.0 } else { f32::NEG_INFINITY }))
|
|
.collect();
|
|
let mask_tensor = Tensor::from_vec(mask, (1, s, s), dev)?;
|
|
|
|
// Reference with mask applied
|
|
let att = (q.clone() * scale as f64)?.matmul(&k.clone().t()?)?;
|
|
let att = att.broadcast_add(&mask_tensor.unsqueeze(0)?)?;
|
|
let att = hanzo_nn::ops::softmax_last_dim(&att)?;
|
|
let expected = att.matmul(&v.clone())?;
|
|
|
|
let out = flash_attn::<f32>(
|
|
&q.transpose(1, 2)?,
|
|
&k.transpose(1, 2)?,
|
|
&v.transpose(1, 2)?,
|
|
scale,
|
|
AttnMask::Mask(mask_tensor),
|
|
None,
|
|
None,
|
|
)?;
|
|
|
|
assert_close(&out, &expected, 1e-4, "standard_with_mask_b1")
|
|
}
|
|
|
|
// Interleaved KV decode
|
|
|
|
#[test]
|
|
fn interleaved_kv_decode() -> Result<()> {
|
|
use hanzo_nn::attention::cpu_flash::causal::causal_decode_f32_interleaved;
|
|
|
|
let (h_q, h_kv, d, kv_len) = (8, 2, 16, 20);
|
|
let scale = 1.0 / (d as f32).sqrt();
|
|
let dev = &Device::Cpu;
|
|
|
|
let q = Tensor::randn(0f32, 1f32, (1, h_q, 1, d), dev)?;
|
|
let k = Tensor::randn(0f32, 1f32, (1, h_kv, kv_len, d), dev)?;
|
|
let v = Tensor::randn(0f32, 1f32, (1, h_kv, kv_len, d), dev)?;
|
|
|
|
// Reference with GQA expansion
|
|
let reps = h_q / h_kv;
|
|
let k_exp = k
|
|
.reshape((1, h_kv, 1, kv_len, d))?
|
|
.broadcast_as((1, h_kv, reps, kv_len, d))?
|
|
.reshape((1, h_q, kv_len, d))?;
|
|
let v_exp = v
|
|
.reshape((1, h_kv, 1, kv_len, d))?
|
|
.broadcast_as((1, h_kv, reps, kv_len, d))?
|
|
.reshape((1, h_q, kv_len, d))?;
|
|
let expected = reference_sdpa(&q, &k_exp, &v_exp, scale)?;
|
|
|
|
// Build interleaved KV: (kv_len, h_kv, 2*d).
|
|
let k_seq = k.squeeze(0)?.transpose(0, 1)?.contiguous()?;
|
|
let v_seq = v.squeeze(0)?.transpose(0, 1)?.contiguous()?;
|
|
let kv = Tensor::cat(&[&k_seq, &v_seq], 2)?.contiguous()?; // (kv_len, h_kv, 2*d)
|
|
|
|
let kv_data: Vec<f32> = kv.flatten_all()?.to_vec1()?;
|
|
let q_flat: Vec<f32> = q
|
|
.squeeze(0)?
|
|
.squeeze(1)?
|
|
.contiguous()?
|
|
.flatten_all()?
|
|
.to_vec1()?;
|
|
|
|
let out = causal_decode_f32_interleaved(&q_flat, &kv_data, h_q, h_kv, d, kv_len, scale)?;
|
|
|
|
let out = out.unsqueeze(0)?;
|
|
assert_close(&out, &expected, 1e-5, "interleaved_kv_decode")
|
|
}
|
|
|
|
/// Packed attention equals the composite per sequence: rotary on queries and keys, scaled
|
|
/// queries, a softmax over the keys within the window, the product with the values.
|
|
#[test]
|
|
fn packed_is_the_composite() -> Result<()> {
|
|
use hanzo_nn::attention::cpu_flash::packed::packed;
|
|
let dev = Device::Cpu;
|
|
let (heads, d, p) = (3, 8, 200);
|
|
let lens = [5usize, 1, 70, 130];
|
|
let t: usize = lens.iter().sum();
|
|
let qkv = Tensor::randn(0f32, 1.0, (t, 3 * heads * d), &dev)?;
|
|
let freqs: Vec<f32> = (0..p)
|
|
.flat_map(|i| (0..d / 2).map(move |j| i as f32 / 1000f32.powf(2. * j as f32 / d as f32)))
|
|
.collect();
|
|
let freqs = Tensor::from_vec(freqs, (p, d / 2), &dev)?;
|
|
let (cos, sin) = (freqs.cos()?, freqs.sin()?);
|
|
let scale = 0.35f32;
|
|
for window in [None, Some(4), Some(64)] {
|
|
let got = packed(&qkv, &lens, heads, window, &cos, &sin, scale)?;
|
|
let mut start = 0;
|
|
for &l in &lens {
|
|
// [l, 3, H, D] -> q, k, v as [1, H, l, D]
|
|
let x = qkv.narrow(0, start, l)?.reshape((l, 3, heads, d))?;
|
|
let part = |i: usize| {
|
|
x.narrow(1, i, 1)?
|
|
.squeeze(1)?
|
|
.transpose(0, 1)?
|
|
.unsqueeze(0)?
|
|
.contiguous()
|
|
};
|
|
let q = hanzo_nn::rotary_emb::rope(&part(0)?, &cos, &sin)?;
|
|
let k = hanzo_nn::rotary_emb::rope(&part(1)?, &cos, &sin)?;
|
|
let q = (q * scale as f64)?;
|
|
let mut s = q.matmul(&k.transpose(2, 3)?)?;
|
|
if let Some(w) = window {
|
|
let m: Vec<f32> = (0..l)
|
|
.flat_map(|i| {
|
|
(0..l).map(move |j| {
|
|
if i.abs_diff(j) > w {
|
|
f32::NEG_INFINITY
|
|
} else {
|
|
0.
|
|
}
|
|
})
|
|
})
|
|
.collect();
|
|
s = s.broadcast_add(&Tensor::from_vec(m, (l, l), &dev)?)?;
|
|
}
|
|
let o = hanzo_nn::ops::softmax_last_dim(&s)?.matmul(&part(2)?)?;
|
|
let want = o.squeeze(0)?.transpose(0, 1)?.reshape((l, heads * d))?;
|
|
let gap = (got.narrow(0, start, l)? - want)?
|
|
.abs()?
|
|
.max_all()?
|
|
.to_scalar::<f32>()?;
|
|
assert!(gap < 1e-5, "window {window:?}, length {l}: {gap}");
|
|
start += l;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|