mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
175 lines
7.0 KiB
Rust
175 lines
7.0 KiB
Rust
// Numeric unit test for the FUSED ROCm rms_norm kernel (hanzo_nn::ops::rms_norm via the RmsNorm
|
|
// CustomOp2 -> RocmStorage::rms_norm -> reduce.hip `rmsnorm_{f16,f32}`). Compares the fused kernel
|
|
// against rms_norm_slow (the KNOWN-GOOD ~9-op composite) on identical inputs, on the SAME ROCm
|
|
// device, and asserts they agree within f16 tolerance. Covers last_dim in {4096, 12288}, a
|
|
// non-power-of-2 (4097, exercises the strided tail + partial-warp reduction), batch>1, and both
|
|
// F16 and F32 dtypes. A wrong norm = garbage across every model, so this is the critical gate.
|
|
// Lives in hanzo-cli so it links the HIP runtime via hanzo-cli's build.rs.
|
|
#![cfg(feature = "rocm")]
|
|
|
|
use hanzo_ml::{DType, Device, Tensor};
|
|
use std::io::Write;
|
|
|
|
// Deterministic pseudo-random-ish value in roughly [-3, 3], varying per index so each row differs.
|
|
fn val(i: usize) -> f32 {
|
|
let x = ((i.wrapping_mul(2654435761) >> 8) & 0xffff) as f32 / 65535.0; // [0,1)
|
|
(x - 0.5) * 6.0
|
|
}
|
|
|
|
fn to_f32_vec(t: &Tensor) -> Vec<f32> {
|
|
let t = t.to_dtype(DType::F32).expect("to f32");
|
|
let t = t.flatten_all().expect("flatten");
|
|
t.to_vec1::<f32>().expect("to_vec1")
|
|
}
|
|
|
|
fn check_shape(dev: &Device, log: &mut String, rows: usize, cols: usize, dtype: DType, tol: f32) {
|
|
let eps = 1e-5f32;
|
|
let n = rows * cols;
|
|
|
|
// Build x[rows, cols] and alpha[cols] on the rocm device, in the target dtype.
|
|
let x_f32: Vec<f32> = (0..n).map(val).collect();
|
|
// alpha varies around 1.0 (realistic rmsnorm weight), some negative to catch sign bugs.
|
|
let alpha_f32: Vec<f32> = (0..cols).map(|i| 1.0 + 0.25 * val(i)).collect();
|
|
|
|
let x = Tensor::from_vec(x_f32, (rows, cols), dev)
|
|
.expect("x tensor")
|
|
.to_dtype(dtype)
|
|
.expect("x dtype");
|
|
let alpha = Tensor::from_vec(alpha_f32, (cols,), dev)
|
|
.expect("alpha tensor")
|
|
.to_dtype(dtype)
|
|
.expect("alpha dtype");
|
|
|
|
let fused = hanzo_nn::ops::rms_norm(&x, &alpha, eps).expect("fused rms_norm");
|
|
let slow = hanzo_nn::ops::rms_norm_slow(&x, &alpha, eps).expect("rms_norm_slow");
|
|
|
|
// Sanity: fused output keeps the input shape + dtype.
|
|
assert_eq!(fused.dims(), &[rows, cols], "fused shape");
|
|
assert_eq!(fused.dtype(), dtype, "fused dtype");
|
|
|
|
let a = to_f32_vec(&fused);
|
|
let b = to_f32_vec(&slow);
|
|
assert_eq!(a.len(), b.len());
|
|
|
|
let mut max_abs_err = 0f32;
|
|
let mut argmax = 0usize;
|
|
for i in 0..a.len() {
|
|
let e = (a[i] - b[i]).abs();
|
|
if e > max_abs_err {
|
|
max_abs_err = e;
|
|
argmax = i;
|
|
}
|
|
}
|
|
log.push_str(&format!(
|
|
"rows={rows} cols={cols} dtype={dtype:?} max_abs_err={max_abs_err:.6} at idx {argmax} (fused={:.5} slow={:.5})\n",
|
|
a[argmax], b[argmax]
|
|
));
|
|
assert!(
|
|
max_abs_err < tol,
|
|
"rms_norm fused vs slow mismatch: rows={rows} cols={cols} dtype={dtype:?} max_abs_err={max_abs_err} >= tol {tol}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn rmsnorm_numeric() {
|
|
let mut log = String::new();
|
|
let dev = Device::new_rocm(0).expect("rocm device");
|
|
|
|
// F16: the production dtype. f16 round-trip on both sides -> allow ~1e-2.
|
|
check_shape(&dev, &mut log, 1, 4096, DType::F16, 1e-2);
|
|
check_shape(&dev, &mut log, 4, 4096, DType::F16, 1e-2); // batch>1
|
|
check_shape(&dev, &mut log, 2, 12288, DType::F16, 1e-2); // wide FFN-ish hidden
|
|
check_shape(&dev, &mut log, 3, 4097, DType::F16, 1e-2); // non-power-of-2 + batch>1
|
|
check_shape(&dev, &mut log, 1, 31, DType::F16, 1e-2); // < warp size
|
|
check_shape(&dev, &mut log, 5, 1025, DType::F16, 1e-2); // just over the 1024 block switch
|
|
|
|
// F32: tighter tolerance — both paths accumulate in f32, so they should be very close.
|
|
check_shape(&dev, &mut log, 4, 4096, DType::F32, 1e-4);
|
|
check_shape(&dev, &mut log, 2, 4097, DType::F32, 1e-4);
|
|
check_shape(&dev, &mut log, 8, 12288, DType::F32, 1e-4);
|
|
|
|
let _ =
|
|
std::fs::File::create("C:\\rmsnorm-test.txt").and_then(|mut f| f.write_all(log.as_bytes()));
|
|
eprintln!("{log}");
|
|
}
|
|
|
|
// Fused add_rmsnorm (RocmStorage::add_rms_norm -> reduce.hip `add_rmsnorm_f32`): the (sum, normed)
|
|
// outputs must be BIT-IDENTICAL to a separate f32 add then the fused `rms_norm` kernel, because the
|
|
// residual stream is f32 and the sum-of-squares reduction order matches `rmsnorm`. nbad counts any
|
|
// non-zero deviation. Covers narrow/wide rows + the 1024 block-size switch + batch>1.
|
|
fn add_rms_check(dev: &Device, log: &mut String, rows: usize, cols: usize) -> usize {
|
|
let eps = 1e-5f32;
|
|
let n = rows * cols;
|
|
let x_v: Vec<f32> = (0..n).map(val).collect();
|
|
let r_v: Vec<f32> = (0..n).map(|i| val(i.wrapping_mul(7) + 3)).collect();
|
|
let a_v: Vec<f32> = (0..cols).map(|i| 1.0 + 0.25 * val(i)).collect();
|
|
let x = Tensor::from_vec(x_v, (rows, cols), dev).expect("x");
|
|
let r = Tensor::from_vec(r_v, (rows, cols), dev).expect("r");
|
|
let a = Tensor::from_vec(a_v, (cols,), dev).expect("a");
|
|
|
|
let sum_ref = (&x + &r).expect("sum_ref");
|
|
let normed_ref =
|
|
hanzo_nn::ops::rms_norm(&sum_ref.contiguous().expect("c"), &a, eps).expect("ref");
|
|
|
|
let (xs, xl) = x.storage_and_layout();
|
|
let xr = match &*xs {
|
|
hanzo_ml::Storage::Rocm(s) => s,
|
|
_ => panic!("x not rocm"),
|
|
};
|
|
let (rs, rl) = r.storage_and_layout();
|
|
let rr = match &*rs {
|
|
hanzo_ml::Storage::Rocm(s) => s,
|
|
_ => panic!("r not rocm"),
|
|
};
|
|
let (as_, al) = a.storage_and_layout();
|
|
let ar = match &*as_ {
|
|
hanzo_ml::Storage::Rocm(s) => s,
|
|
_ => panic!("a not rocm"),
|
|
};
|
|
let (sum_s, normed_s) = xr
|
|
.add_rms_norm(xl, rr, rl, ar, al, eps)
|
|
.expect("add_rms_norm");
|
|
let shape = x.shape().clone();
|
|
let sum_f = Tensor::from((hanzo_ml::Storage::Rocm(sum_s), shape.clone()));
|
|
let normed_f = Tensor::from((hanzo_ml::Storage::Rocm(normed_s), shape));
|
|
|
|
let (sr, sf) = (to_f32_vec(&sum_ref), to_f32_vec(&sum_f));
|
|
let (nr, nf) = (to_f32_vec(&normed_ref), to_f32_vec(&normed_f));
|
|
let mut nbad = 0usize;
|
|
let mut max_err = 0f32;
|
|
for i in 0..n {
|
|
if sr[i] != sf[i] {
|
|
nbad += 1;
|
|
}
|
|
let e = (nr[i] - nf[i]).abs();
|
|
max_err = max_err.max(e);
|
|
if e != 0.0 {
|
|
nbad += 1;
|
|
}
|
|
}
|
|
log.push_str(&format!(
|
|
"add_rmsnorm rows={rows} cols={cols} nbad={nbad}/{} sum_exact={} normed_max_err={max_err:.3e}\n",
|
|
2 * n,
|
|
sr == sf
|
|
));
|
|
nbad
|
|
}
|
|
|
|
#[test]
|
|
fn add_rmsnorm_numeric() {
|
|
let mut log = String::new();
|
|
let dev = Device::new_rocm(0).expect("rocm device");
|
|
let mut nbad = 0;
|
|
nbad += add_rms_check(&dev, &mut log, 1, 2048); // decode hidden (Qwen3-30B-A3B)
|
|
nbad += add_rms_check(&dev, &mut log, 1, 4096);
|
|
nbad += add_rms_check(&dev, &mut log, 4, 2048); // batch>1
|
|
nbad += add_rms_check(&dev, &mut log, 3, 31); // < warp size
|
|
nbad += add_rms_check(&dev, &mut log, 5, 1025); // just over the 1024 block switch
|
|
nbad += add_rms_check(&dev, &mut log, 2, 12288); // wide
|
|
eprintln!("{log}");
|
|
assert_eq!(
|
|
nbad, 0,
|
|
"add_rmsnorm must be bit-identical to add + rms_norm\n{log}"
|
|
);
|
|
}
|