Files
ml/hanzo-ml/tests/rocm_scatter_tests.rs
zeekayandHanzo Dev d62a609fe5 feat(rocm): scatter and scatter_add
ROCm was the only backend without scatter, so any model whose graph reaches
Tensor::scatter died on the box. Qwen3.5-27B's interleaved MRoPE was the first.

indexing.hip gains two templates and 42 entry points: 7 dtypes (f32, f64, u8,
u32, i64, bf16, f16) x 3 index dtypes (u8, u32, i64) x set/add. One thread owns
a whole (pre, post) column and walks the scattered dim itself, so the repeated
writes to one output slot are ordered and scatter_add needs no atomics. The
16-bit types accumulate in float, as binary.hip already does.

scatter_apply is the one body behind both scatter_set and scatter_add_set; its
root string is both the kernel-name root and the op name in errors, so the two
cannot drift.

Names are checked, not assumed: the 42 symbols the file emits are exactly the
42 kernel_name::<T>() asks for.

Tests: rocm_scatter_tests pins ten cases across all seven dtypes to
hand-computed values on the CPU, then demands bit-identical results from ROCm.


Co-authored-by: Hanzo Dev <[email protected]>
2026-09-15 09:59:35 -07:00

180 lines
5.5 KiB
Rust

//! `scatter` / `scatter_set` / `scatter_add` / `scatter_add_set` on the ROCm backend, checked
//! against the CPU as the oracle.
//!
//! The CPU half runs everywhere and pins the fixture to hand-computed values, so a wrong oracle
//! cannot quietly agree with a wrong kernel. The ROCm half (feature `rocm`) runs the same cases
//! on the GPU for every dtype the ROCm scatter kernels cover and demands bit-identical results:
//! the integer and set paths are exact by construction, and the float cases use small integers
//! that every dtype here, bf16 included, represents exactly.
use hanzo_ml::{DType, Device, Result, Tensor};
// Every dtype the ROCm scatter kernels cover.
const DTYPES: [DType; 7] = [
DType::F32,
DType::F64,
DType::U8,
DType::U32,
DType::I64,
DType::BF16,
DType::F16,
];
fn flat(t: &Tensor) -> Result<Vec<f32>> {
t.to_device(&Device::Cpu)?
.to_dtype(DType::F32)?
.flatten_all()?
.to_vec1::<f32>()
}
/// Every case, as (name, flattened result). `src` and `ids` are [4,3]; the destination is [6,3]
/// for dim 0 and [4,5] for dim 1, so the scattered dim differs in length from the source.
fn cases(device: &Device, dtype: DType) -> Result<Vec<(&'static str, Vec<f32>)>> {
let src = Tensor::arange(0f32, 12f32, &Device::Cpu)?
.reshape((4, 3))?
.to_dtype(dtype)?
.to_device(device)?;
let ids = Tensor::new(
&[[0u32, 1, 2], [3, 4, 0], [3, 3, 1], [2, 0, 4]],
&Device::Cpu,
)?
.to_device(device)?;
let ids_i64 = ids
.to_device(&Device::Cpu)?
.to_dtype(DType::I64)?
.to_device(device)?;
let ids_skip = Tensor::new(
&[
[0u32, u32::MAX, 2],
[3, 4, u32::MAX],
[3, 3, 1],
[u32::MAX, u32::MAX, 4],
],
&Device::Cpu,
)?
.to_device(device)?;
let dst0 = Tensor::ones((6, 3), dtype, &Device::Cpu)?.to_device(device)?;
let dst1 = Tensor::ones((4, 5), dtype, &Device::Cpu)?.to_device(device)?;
let set = Tensor::ones((6, 3), dtype, &Device::Cpu)?.to_device(device)?;
set.scatter_set(&ids, &src, 0)?;
let add_set = Tensor::ones((6, 3), dtype, &Device::Cpu)?.to_device(device)?;
add_set.scatter_add_set(&ids, &src, 0)?;
Ok(vec![
("scatter_dim0", flat(&dst0.scatter(&ids, &src, 0)?)?),
("scatter_add_dim0", flat(&dst0.scatter_add(&ids, &src, 0)?)?),
("scatter_dim1", flat(&dst1.scatter(&ids, &src, 1)?)?),
("scatter_add_dim1", flat(&dst1.scatter_add(&ids, &src, 1)?)?),
("scatter_ids_i64", flat(&dst0.scatter(&ids_i64, &src, 0)?)?),
(
"scatter_add_ids_i64",
flat(&dst0.scatter_add(&ids_i64, &src, 0)?)?,
),
("scatter_skip", flat(&dst0.scatter(&ids_skip, &src, 0)?)?),
(
"scatter_add_skip",
flat(&dst0.scatter_add(&ids_skip, &src, 0)?)?,
),
("scatter_set", flat(&set)?),
("scatter_add_set", flat(&add_set)?),
])
}
fn case<'a>(cases: &'a [(&'static str, Vec<f32>)], name: &str) -> &'a [f32] {
&cases.iter().find(|(n, _)| *n == name).unwrap().1
}
#[rustfmt::skip]
const SCATTER_DIM0: [f32; 18] = [
0.0, 10.0, 5.0,
1.0, 1.0, 8.0,
9.0, 1.0, 2.0,
6.0, 7.0, 1.0,
1.0, 4.0, 11.0,
1.0, 1.0, 1.0,
];
#[rustfmt::skip]
const SCATTER_ADD_DIM0: [f32; 18] = [
1.0, 11.0, 6.0,
1.0, 2.0, 9.0,
10.0, 1.0, 3.0,
10.0, 8.0, 1.0,
1.0, 5.0, 12.0,
1.0, 1.0, 1.0,
];
// u32::MAX in the ids means "leave that slot alone".
#[rustfmt::skip]
const SCATTER_SKIP: [f32; 18] = [
0.0, 1.0, 1.0,
1.0, 1.0, 8.0,
1.0, 1.0, 2.0,
6.0, 7.0, 1.0,
1.0, 4.0, 11.0,
1.0, 1.0, 1.0,
];
#[rustfmt::skip]
const SCATTER_DIM1: [f32; 20] = [
0.0, 1.0, 2.0, 1.0, 1.0,
5.0, 1.0, 1.0, 3.0, 4.0,
1.0, 8.0, 1.0, 7.0, 1.0,
10.0, 1.0, 9.0, 1.0, 11.0,
];
#[rustfmt::skip]
const SCATTER_ADD_DIM1: [f32; 20] = [
1.0, 2.0, 3.0, 1.0, 1.0,
6.0, 1.0, 1.0, 4.0, 5.0,
1.0, 9.0, 1.0, 14.0, 1.0,
11.0, 1.0, 10.0, 1.0, 12.0,
];
#[rustfmt::skip]
const SCATTER_ADD_SKIP: [f32; 18] = [
1.0, 1.0, 1.0,
1.0, 1.0, 9.0,
1.0, 1.0, 3.0,
10.0, 8.0, 1.0,
1.0, 5.0, 12.0,
1.0, 1.0, 1.0,
];
#[test]
fn cpu_oracle_is_pinned() -> Result<()> {
for dtype in DTYPES {
let c = cases(&Device::Cpu, dtype)?;
assert_eq!(case(&c, "scatter_dim0"), SCATTER_DIM0, "{dtype:?}");
assert_eq!(case(&c, "scatter_add_dim0"), SCATTER_ADD_DIM0, "{dtype:?}");
assert_eq!(case(&c, "scatter_dim1"), SCATTER_DIM1, "{dtype:?}");
assert_eq!(case(&c, "scatter_add_dim1"), SCATTER_ADD_DIM1, "{dtype:?}");
assert_eq!(case(&c, "scatter_ids_i64"), SCATTER_DIM0, "{dtype:?}");
assert_eq!(
case(&c, "scatter_add_ids_i64"),
SCATTER_ADD_DIM0,
"{dtype:?}"
);
assert_eq!(case(&c, "scatter_skip"), SCATTER_SKIP, "{dtype:?}");
assert_eq!(case(&c, "scatter_add_skip"), SCATTER_ADD_SKIP, "{dtype:?}");
assert_eq!(case(&c, "scatter_set"), SCATTER_DIM0, "{dtype:?}");
assert_eq!(case(&c, "scatter_add_set"), SCATTER_ADD_DIM0, "{dtype:?}");
}
Ok(())
}
#[cfg(feature = "rocm")]
#[test]
fn rocm_matches_cpu() -> Result<()> {
let rocm = Device::new_rocm(0)?;
for dtype in DTYPES {
let want = cases(&Device::Cpu, dtype)?;
let got = cases(&rocm, dtype)?;
for ((name, want), (_, got)) in want.iter().zip(got.iter()) {
assert_eq!(got, want, "{dtype:?} {name}");
}
}
Ok(())
}