Files
ml/hanzo-ml/tests/rocm_moe_tests.rs
zeekayandHanzo Dev e655792cb4 fix(quantized): device kernels read a view from its own start
The GPU arms of QMatMul, the indexed MoE and the router take a tensor's
raw storage after .contiguous(). A contiguous view can start inside its
storage: the last row of a hidden state, from which a model reads its
logits, is one. .contiguous() returns such a view as it is, and the
kernels read from the storage's first element, so a model's next token
came from the first prompt position's logits on ROCm. Each raw-storage
site now takes the tensor through packed(), which copies a view that
does not start at element 0.


Co-authored-by: Hanzo Dev <[email protected]>
2026-09-24 12:52:55 -07:00

254 lines
9.3 KiB
Rust

//! The routed-expert path on the device, against the CPU, at a 512-expert model's shapes.
//!
//! A 512-expert, top-10 MoE (hidden 2560, expert width 640) stores gate/up as IQ3_S or IQ4_XS
//! and down as IQ4_NL or Q8_0. Each case runs `indexed_moe_forward` on ROCm for a decode step and
//! a prompt, with f16 and f32 activations, and compares every routed slot with the CPU's dequantized
//! weights. The router's top-k over 512 logits and the Q6_K vocabulary projection are checked too.
#![cfg(feature = "rocm")]
use hanzo_ml::quantized::{ggml_file::qtensor_from_ggml, GgmlDType, QMatMul};
use hanzo_ml::{DType, Device, Module, Result, RocmQuantType, Tensor};
/// Deterministic values: an LCG, so a failure reproduces without a seed file.
struct Lcg(u64);
impl Lcg {
fn next(&mut self) -> u64 {
self.0 = self
.0
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
self.0 >> 33
}
fn byte(&mut self) -> u8 {
self.next() as u8
}
fn unit(&mut self) -> f32 {
(self.next() % 20001) as f32 / 10000.0 - 1.0
}
}
/// Random blocks of `dtype` for `elems` weights, with each block's f16 scale set to a finite
/// value so the comparison is about the decode, not NaN.
fn blocks(dtype: GgmlDType, elems: usize, rng: &mut Lcg) -> Vec<u8> {
let qt = RocmQuantType::from_ggml(dtype).expect("wired");
let (size, per) = (qt.block_bytes(), qt.block_elems());
// Where each type keeps its f16 super-scale: last for Q6_K, first for the rest.
let at = if dtype == GgmlDType::Q6K { size - 2 } else { 0 };
let mut raw = Vec::with_capacity(elems / per * size);
for _ in 0..elems / per {
let start = raw.len();
raw.extend((0..size).map(|_| rng.byte()));
let d = half::f16::from_f32(0.004 + 0.004 * rng.unit().abs()).to_bits();
raw[start + at..start + at + 2].copy_from_slice(&d.to_le_bytes());
}
raw
}
/// Largest |gpu - cpu| over the largest |cpu|.
fn rel(got: &[f32], want: &[f32]) -> f32 {
assert_eq!(got.len(), want.len());
let scale = want.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-12);
got.iter()
.zip(want)
.map(|(g, w)| (g - w).abs())
.fold(0f32, f32::max)
/ scale
}
/// One bank `[experts, n, k]` of `dtype`, `t` tokens, top-`topk` ids spread over every expert.
/// `per_slot`: down's input is one row per routed slot; gate/up share one row per token.
#[allow(clippy::too_many_arguments)]
fn moe_case(
gpu: &Device,
dtype: GgmlDType,
experts: usize,
n: usize,
k: usize,
t: usize,
topk: usize,
per_slot: bool,
act: DType,
) -> Result<f32> {
let mut rng = Lcg(0x5eed ^ (dtype as u64) << 8 ^ (n * k) as u64);
let raw = blocks(dtype, experts * n * k, &mut rng);
let bank = qtensor_from_ggml(dtype, &raw, vec![experts, n, k], &Device::Cpu)?
.dequantize(&Device::Cpu)?
.flatten_all()?
.to_vec1::<f32>()?;
let rows = if per_slot { t * topk } else { t };
let x: Vec<f32> = (0..rows * k).map(|_| rng.unit()).collect();
// Distinct experts per token, reaching past 255.
let ids: Vec<u32> = (0..t * topk)
.map(|i| ((i / topk * 97 + (i % topk) * 53 + 11) % experts) as u32)
.collect();
let mut want = vec![0f32; t * topk * n];
for slot in 0..t * topk {
let e = ids[slot] as usize;
let xr = if per_slot { slot } else { slot / topk };
for r in 0..n {
let w = &bank[(e * n + r) * k..(e * n + r + 1) * k];
want[slot * n + r] = w
.iter()
.zip(&x[xr * k..(xr + 1) * k])
.map(|(a, b)| *a as f64 * *b as f64)
.sum::<f64>() as f32;
}
}
let q = qtensor_from_ggml(dtype, &raw, vec![experts, n, k], gpu)?;
let shape = if per_slot { (t, topk, k) } else { (t, 1, k) };
let xg = Tensor::from_vec(x, shape, gpu)?.to_dtype(act)?;
let idg = Tensor::from_vec(ids, (t, topk), gpu)?;
let got = q
.indexed_moe_forward(&xg, &idg)?
.to_dtype(DType::F32)?
.flatten_all()?
.to_vec1::<f32>()?;
Ok(rel(&got, &want))
}
#[test]
fn routed_experts_match_the_cpu_at_model_shapes() -> Result<()> {
let gpu = Device::new_rocm(0)?;
let mut worst = 0f32;
// (dtype, experts, n, k, per_slot): gate/up then down, each with all 512 experts at a narrow
// width (ids past 255) and with a few experts at the full width.
let cases = [
(GgmlDType::IQ3_S, 512, 32, 2560, false),
(GgmlDType::IQ3_S, 16, 640, 2560, false),
(GgmlDType::IQ4_XS, 16, 640, 2560, false),
(GgmlDType::IQ4_NL, 512, 64, 640, true),
(GgmlDType::IQ4_NL, 16, 2560, 640, true),
(GgmlDType::Q8_0, 16, 2560, 640, true),
];
for (dtype, experts, n, k, per_slot) in cases {
for t in [1usize, 5] {
for act in [DType::F16, DType::F32] {
let e = moe_case(&gpu, dtype, experts, n, k, t, 10, per_slot, act)?;
println!("{dtype:?} E={experts} n={n} k={k} t={t} {act:?}: rel {e:.2e}");
worst = worst.max(e);
}
}
}
// f16 activations and the int8 paths' q8 activations land near 1e-3 of the largest output.
assert!(
worst < 1e-2,
"routed experts off by {worst} of the largest output"
);
Ok(())
}
#[test]
fn top_k_of_512_router_logits_matches_the_cpu() -> Result<()> {
let gpu = Device::new_rocm(0)?;
let mut rng = Lcg(0x7009);
let (t, e) = (7usize, 512usize);
let logits: Vec<f32> = (0..t * e).map(|_| rng.unit() * 4.0).collect();
let sort = |d: &Device| -> Result<Vec<u32>> {
let (_, idx) = Tensor::from_vec(logits.clone(), (t, e), d)?.sort_last_dim(false)?;
idx.narrow(1, 0, 10)?.flatten_all()?.to_vec1::<u32>()
};
assert_eq!(sort(&gpu)?, sort(&Device::Cpu)?);
Ok(())
}
#[test]
fn q6k_vocabulary_projection_matches_the_cpu() -> Result<()> {
let gpu = Device::new_rocm(0)?;
let mut rng = Lcg(0x6b);
// Rows as the vocabulary, k as the hidden width.
let (rows, k) = (248_320usize, 2560usize);
let raw = blocks(GgmlDType::Q6K, rows * k, &mut rng);
let x: Vec<f32> = (0..k).map(|_| rng.unit()).collect();
let reference = qtensor_from_ggml(GgmlDType::Q6K, &raw, vec![rows, k], &Device::Cpu)?
.dequantize(&Device::Cpu)?;
let want = Tensor::from_vec(x.clone(), (1, k), &Device::Cpu)?
.matmul(&reference.t()?)?
.flatten_all()?
.to_vec1::<f32>()?;
let head = QMatMul::from_qtensor(qtensor_from_ggml(
GgmlDType::Q6K,
&raw,
vec![rows, k],
&gpu,
)?)?;
for act in [DType::F16, DType::F32] {
let got = head
.forward(&Tensor::from_vec(x.clone(), (1, k), &gpu)?.to_dtype(act)?)?
.to_dtype(DType::F32)?
.flatten_all()?
.to_vec1::<f32>()?;
let e = rel(&got, &want);
println!("Q6_K head {act:?}: rel {e:.2e}");
assert!(e < 1e-2, "Q6_K head {act:?} off by {e}");
}
Ok(())
}
#[test]
fn an_f32_router_weight_keeps_its_precision() -> Result<()> {
let gpu = Device::new_rocm(0)?;
let mut rng = Lcg(0x60a7);
let (experts, k) = (512usize, 2560usize);
let w: Vec<f32> = (0..experts * k).map(|_| rng.unit() * 0.05).collect();
let bytes: Vec<u8> = w.iter().flat_map(|v| v.to_le_bytes()).collect();
let router = QMatMul::from_qtensor(qtensor_from_ggml(
GgmlDType::F32,
&bytes,
vec![experts, k],
&gpu,
)?)?;
let reference = Tensor::from_vec(w, (experts, k), &Device::Cpu)?;
// A prompt and a decode step, as the router sees them: f32 activations.
for t in [5usize, 1] {
let x: Vec<f32> = (0..t * k).map(|_| rng.unit()).collect();
let want = Tensor::from_vec(x.clone(), (t, k), &Device::Cpu)?
.matmul(&reference.t()?)?
.flatten_all()?
.to_vec1::<f32>()?;
let got = router
.forward(&Tensor::from_vec(x, (t, k), &gpu)?)?
.flatten_all()?
.to_vec1::<f32>()?;
let e = rel(&got, &want);
println!("f32 router t={t}: rel {e:.2e}");
assert!(e < 1e-5, "f32 router t={t} off by {e}");
}
Ok(())
}
#[test]
fn a_row_view_projects_its_own_row() -> Result<()> {
let gpu = Device::new_rocm(0)?;
let mut rng = Lcg(0x1a57);
let (rows, k) = (1024usize, 2560usize);
let raw = blocks(GgmlDType::Q6K, rows * k, &mut rng);
let head = QMatMul::from_qtensor(qtensor_from_ggml(
GgmlDType::Q6K,
&raw,
vec![rows, k],
&gpu,
)?)?;
// A 3-token hidden state; the logits come from its last row, a contiguous view that starts
// two rows into the storage.
let h: Vec<f32> = (0..3 * k).map(|_| rng.unit()).collect();
for act in [DType::F16, DType::F32] {
let batch = Tensor::from_vec(h.clone(), (1, 3, k), &gpu)?.to_dtype(act)?;
let last = batch.narrow(1, 2, 1)?.contiguous()?;
let alone = Tensor::from_vec(h[2 * k..].to_vec(), (1, 1, k), &gpu)?.to_dtype(act)?;
let got = head
.forward(&last)?
.to_dtype(DType::F32)?
.flatten_all()?
.to_vec1::<f32>()?;
let want = head
.forward(&alone)?
.to_dtype(DType::F32)?
.flatten_all()?
.to_vec1::<f32>()?;
assert_eq!(got, want, "{act:?}: the view read another row");
}
Ok(())
}