Files
zeekay 35bbf78aea ModernBERT runs quantized inference packed on the CPU and its attention fused on Metal.
quantize turns the projections the CPU tiles into Q8_0 blocks. A quantized backbone then runs
each sequence's real tokens only: embeddings, norms and projections over the packed tokens,
attention per (sequence, head) straight from the QKV projection (attention::cpu_flash::packed:
rotation, window, softmax and both products in one parallel pass), GeGLU in one vectorized pass
(ops::geglu, erf within 1.5e-7 of exact, the composite wherever a gradient flows); pad positions
come back zero. On Metal, inference takes one SDPA kernel for scores, mask, softmax and values.
Training's path is unchanged.

Claude-Session: https://claude.ai/code/session_018Gt5TvrE2rxXuZikWSauv3
2026-09-27 05:00:26 -07:00

79 lines
2.5 KiB
Rust

//! ModernBERT's CPU inference pass against its training pass, on a tiny random backbone.
use hanzo_ml::quantized::GgmlDType;
use hanzo_ml::{DType, Device, Result, Tensor};
use hanzo_nn::{VarBuilder, VarMap};
use hanzo_transformers::models::modernbert::{Config, ModernBert};
fn config() -> Config {
Config {
vocab_size: 50,
hidden_size: 64,
num_hidden_layers: 3,
num_attention_heads: 2,
intermediate_size: 64,
max_position_embeddings: 64,
layer_norm_eps: 1e-5,
pad_token_id: 0,
global_attn_every_n_layers: 3,
global_rope_theta: 160000.,
local_attention: 8,
local_rope_theta: 10000.,
classifier_config: None,
}
}
/// Rows of 12, 7 and 3 tokens, right-padded, and their mask.
fn rows(dev: &Device) -> Result<(Tensor, Tensor)> {
let lens = [12usize, 7, 3];
let ids: Vec<u32> = lens
.iter()
.enumerate()
.flat_map(|(r, &n)| {
(0..12).map(move |t| {
if t < n {
1 + ((r * 13 + t * 7) % 49) as u32
} else {
0
}
})
})
.collect();
let mask: Vec<u32> = lens
.iter()
.flat_map(|&n| (0..12).map(move |t| u32::from(t < n)))
.collect();
Ok((
Tensor::from_vec(ids, (3, 12), dev)?,
Tensor::from_vec(mask, (3, 12), dev)?,
))
}
/// The packed pass over each row's real tokens gives, at every real position, what the padded
/// pass gives: to rounding with F32 projections, to quantization with Q8_0 blocks; its pad
/// positions are zero.
#[test]
fn the_packed_pass_is_the_padded_pass() -> Result<()> {
let dev = Device::Cpu;
let vm = VarMap::new();
let bert = ModernBert::new(VarBuilder::from_varmap(&vm, DType::F32, &dev), &config())?;
let (ids, mask) = rows(&dev)?;
let keep = mask.to_dtype(DType::F32)?.unsqueeze(2)?;
let want = bert.forward(&ids, &mask)?.broadcast_mul(&keep)?;
let scale = want.abs()?.max_all()?.to_scalar::<f32>()?;
for (dtype, tol) in [(GgmlDType::F32, 1e-5f32), (GgmlDType::Q8_0, 0.05)] {
let mut q = bert.clone();
q.quantize(dtype)?;
let got = q.forward(&ids, &mask)?;
let pads = got
.broadcast_mul(&keep.affine(-1.0, 1.0)?)?
.abs()?
.max_all()?
.to_scalar::<f32>()?;
assert_eq!(pads, 0.0);
let gap = (got - &want)?.abs()?.max_all()?.to_scalar::<f32>()?;
assert!(gap <= tol * scale, "{dtype:?}: {gap} against {scale}");
}
Ok(())
}