Files
ml/hanzo-transformers/tests/modernbert_memory.rs
zeekayandHanzo Dev 565ee00ef1 Green main: rustfmt, clippy 1.98, and two x86 float tolerances
- rustfmt over the tree.
- clippy 1.98: `is_multiple_of` for six modular checks, and `as_chunks` for
  the bf16/f32 row decode.
- cpu_flash::packed: the first chunk against its own pass, and the cache path
  against the block pass, are different kernels. On x86 they agree to about
  1e-7, not bit for bit, so the check is within 1e-6. A key read past the chunk
  would be off by the size of a score.
- geglu: the fused erf is within about 3e-7 of the composite's (measured worst
  2.8e-7 on an AMD EPYC 7571, near g = -0.06), so the bound is
  (½|g||u| + |y|)·4e-7.

hanzo-ml and hanzo-nn: 409 tests pass on x86. The ROCm tests (rocm_moe 6,
rocm_gdn 2, rocm_ops 1) pass on gfx1151.

Co-authored-by: Hanzo Dev <[email protected]>
2026-10-16 18:34:21 -07:00

118 lines
4.0 KiB
Rust

//! A long row's padded pass holds no `[L, L]` tensor of any type, and answers as the packed pass,
//! whose attention is another algorithm (`hanzo_nn::attention::packed`). Its own binary, so the
//! allocator counts this pass alone.
use hanzo_ml::quantized::GgmlDType;
use hanzo_ml::{DType, Device, Result, Tensor};
use hanzo_nn::{VarBuilder, VarMap};
use hanzo_transformers::models::modernbert::{Config, ModernBert};
use std::alloc::{GlobalAlloc, Layout, System};
use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
/// The system allocator, counting the bytes held, their peak and the largest block.
struct Counting;
static HELD: AtomicUsize = AtomicUsize::new(0);
static PEAK: AtomicUsize = AtomicUsize::new(0);
static LARGEST: AtomicUsize = AtomicUsize::new(0);
unsafe impl GlobalAlloc for Counting {
unsafe fn alloc(&self, l: Layout) -> *mut u8 {
let p = unsafe { System.alloc(l) };
if !p.is_null() {
let held = HELD.fetch_add(l.size(), Relaxed) + l.size();
PEAK.fetch_max(held, Relaxed);
LARGEST.fetch_max(l.size(), Relaxed);
}
p
}
unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
unsafe { System.dealloc(p, l) };
HELD.fetch_sub(l.size(), Relaxed);
}
}
#[global_allocator]
static COUNTING: Counting = Counting;
#[test]
fn a_long_row_holds_no_square_tensor() -> Result<()> {
let dev = Device::Cpu;
// two heads, 8,192 trained positions stretched to 32,768: a row past 5,792 tokens tiles
let cfg = Config {
vocab_size: 50,
hidden_size: 64,
num_hidden_layers: 3,
num_attention_heads: 2,
intermediate_size: 64,
max_position_embeddings: 8192,
layer_norm_eps: 1e-5,
pad_token_id: 0,
global_attn_every_n_layers: 3,
global_rope_theta: 160000.,
local_attention: 8,
local_rope_theta: 10000.,
yarn: Some(4.0),
classifier_config: None,
};
// weights apart from their variables, as inference reads them: no pass keeps its graph
let vm = VarMap::new();
ModernBert::new(VarBuilder::from_varmap(&vm, DType::F32, &dev), &cfg)?;
let weights = vm
.data()
.lock()
.expect("varmap lock")
.iter()
.map(|(k, v)| (k.clone(), v.as_tensor().detach()))
.collect();
let bert = ModernBert::new(VarBuilder::from_tensors(weights, DType::F32, &dev), &cfg)?;
let l = 20_480;
let lens = [l, 9_000];
let ids: Vec<u32> = lens
.iter()
.enumerate()
.flat_map(|(r, &n)| {
(0..l).map(move |t| {
if t < n {
1 + ((r * 13 + t * 7) % 49) as u32
} else {
0
}
})
})
.collect();
let mask: Vec<u32> = lens
.iter()
.flat_map(|&n| (0..l).map(move |t| u32::from(t < n)))
.collect();
let ids = Tensor::from_vec(ids, (2, l), &dev)?;
let mask = Tensor::from_vec(mask, (2, l), &dev)?;
let base = HELD.load(Relaxed);
PEAK.store(base, Relaxed);
LARGEST.store(0, Relaxed);
let want = bert.forward(&ids, &mask)?;
let (peak, largest) = (PEAK.load(Relaxed) - base, LARGEST.load(Relaxed));
// whole, a global layer's scores alone are [2, 2, L, L] in F32, 6.7 GB
assert!(
largest < l * l,
"a block of {largest} bytes, {l} x {l} is {}",
l * l
);
assert!(peak < 4 * l * l, "{peak} bytes held at once");
let keep = mask.to_dtype(DType::F32)?.unsqueeze(2)?;
let want = want.broadcast_mul(&keep)?;
let mut packs = bert.clone();
packs.quantize(GgmlDType::F32)?;
assert!(packs.packs());
let got = packs.forward(&ids, &mask)?;
let scale = want.abs()?.max_all()?.to_scalar::<f32>()?;
let gap = (got - &want)?.abs()?.max_all()?.to_scalar::<f32>()?;
// over 16,384 keys the two algorithms' float sums part by 1.6e-5 of the scale, the padded
// pass's scores whole or tiled alike (whole and tiled part by 5e-7)
assert!(gap <= 5e-5 * scale, "packed {gap} against {scale}");
Ok(())
}