mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-20 23:08:29 +00:00
Three-way per file through the rename: base 39355c6c91, theirs the upstream
merge of main with 35d7ae7c (via 7ef6a694d), ours origin/main.
Lands the barrier thread pool, runtime CPU feature dispatch, aarch64 and x86
repacked quantized kernels (indexed_gemv, gemv_fused_shared_lhs), quantized
embedding on CPU/Metal/CUDA, paged flash-attn kernels, cutile fused MoE, the
in-place op aliasing API, parking_lot storage locks, Metal buffer builders and
debug labels, safetensors 0.8, hf-hub 1.0, tokenizers 0.23.
Ours keeps ROCm, Vulkan, wgpu, the IQ/MXFP4/NVFP4/TQ dtypes, and the i-quant
weight/activation block split in matmul. Our backends ride InplaceOpN and the
quantized embedding; Metal get_rows gains the IQ kernels. ug leaves, as upstream.
Co-authored-by: Hanzo Dev <[email protected]>
267 lines
9.2 KiB
Rust
267 lines
9.2 KiB
Rust
use hanzo_ml::{test_device, DType, Device, IndexOp, Result, Tensor};
|
|
|
|
fn matmul(device: &Device) -> Result<()> {
|
|
let data = vec![1.0f32, 2.0, 3.0, 4.0];
|
|
let a = Tensor::from_slice(&data, (2, 2), device)?;
|
|
let data = vec![1.0f32, 2.0, 3.0, 4.0];
|
|
let b = Tensor::from_slice(&data, (2, 2), device)?;
|
|
|
|
let c = a.matmul(&b)?;
|
|
assert_eq!(c.to_vec2::<f32>()?, &[[7.0f32, 10.0], [15.0, 22.0]]);
|
|
|
|
let data = vec![1.0f32, 2.0];
|
|
let a = Tensor::from_slice(&data, (2, 1), device)?;
|
|
let data = vec![3.0f32, 4.0];
|
|
let b = Tensor::from_slice(&data, (1, 2), device)?;
|
|
let c = a.matmul(&b)?;
|
|
assert_eq!(c.to_vec2::<f32>()?, &[&[3.0, 4.0], &[6.0, 8.0]]);
|
|
|
|
let data: Vec<_> = (0..6).map(|i| i as f32).collect();
|
|
let a = Tensor::from_slice(&data, (2, 3), device)?;
|
|
let data: Vec<_> = (0..6).map(|i| (i + 2) as f32).collect();
|
|
let b = Tensor::from_slice(&data, (3, 2), device)?;
|
|
let c = a.matmul(&b)?;
|
|
assert_eq!(c.to_vec2::<f32>()?, &[&[16., 19.], &[52., 64.]]);
|
|
|
|
let data: Vec<_> = (0..12).map(|i| i as f32).collect();
|
|
let a = Tensor::from_slice(&data, (2, 2, 3), device)?;
|
|
let data: Vec<_> = (0..12).map(|i| (i + 2) as f32).collect();
|
|
let b = Tensor::from_slice(&data, (2, 3, 2), device)?;
|
|
let expected = [[[16., 19.], [52., 64.]], [[214., 235.], [304., 334.]]];
|
|
|
|
let c = a.matmul(&b)?;
|
|
assert_eq!(c.to_vec3::<f32>()?, &expected);
|
|
|
|
// Also perform the matmul on contiguous transposed versions.
|
|
let a_tt = a.t()?.contiguous()?.t()?;
|
|
assert!(!a_tt.is_contiguous());
|
|
assert_eq!(a.dims(), a_tt.dims());
|
|
assert_eq!(a_tt.stride(), &[6, 1, 2]);
|
|
|
|
let b_tt = b.t()?.contiguous()?.t()?;
|
|
assert!(!b_tt.is_contiguous());
|
|
assert_eq!(b.dims(), b_tt.dims());
|
|
assert_eq!(b_tt.stride(), &[6, 1, 3]);
|
|
|
|
assert_eq!(a_tt.matmul(&b)?.to_vec3::<f32>()?, &expected);
|
|
assert_eq!(a.matmul(&b_tt)?.to_vec3::<f32>()?, &expected);
|
|
assert_eq!(a_tt.matmul(&b_tt)?.to_vec3::<f32>()?, &expected);
|
|
Ok(())
|
|
}
|
|
|
|
fn matmul_bf16(device: &Device) -> Result<()> {
|
|
if !device.supports_bf16() {
|
|
return Ok(());
|
|
}
|
|
let data = vec![1.0f32, 2.0, 3.0, 4.0];
|
|
let a = Tensor::from_slice(&data, (2, 2), device)?.to_dtype(DType::BF16)?;
|
|
let data = vec![1.0f32, 2.0, 3.0, 4.0];
|
|
let b = Tensor::from_slice(&data, (2, 2), device)?.to_dtype(DType::BF16)?;
|
|
|
|
let c = a.matmul(&b)?.to_dtype(DType::F32)?;
|
|
assert_eq!(c.to_vec2::<f32>()?, &[[7.0f32, 10.0], [15.0, 22.0]]);
|
|
Ok(())
|
|
}
|
|
|
|
fn broadcast_matmul(device: &Device) -> Result<()> {
|
|
let lhs = Tensor::randn(0f32, 1f32, (3, 1, 4, 5), device)?;
|
|
let rhs = Tensor::randn(0f32, 1f32, (6, 5, 2), device)?;
|
|
let out = lhs.broadcast_matmul(&rhs)?;
|
|
assert_eq!(out.dims(), &[3, 6, 4, 2]);
|
|
for idx1 in 0..3 {
|
|
for idx2 in 0..6 {
|
|
let out = out.i((idx1, idx2))?;
|
|
let lhs = lhs.i((idx1, 0))?;
|
|
let rhs = rhs.i(idx2)?;
|
|
let out2 = lhs.matmul(&rhs);
|
|
let sum_diff2 = (out - out2)?.sqr()?.sum_all()?;
|
|
// With cuda, we see errors of up to ~1e-12.
|
|
assert!(sum_diff2.to_vec0::<f32>()? < 1e-6)
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn zero_matmul(device: &Device) -> Result<()> {
|
|
let lhs = Tensor::zeros((2, 0), DType::F32, device)?;
|
|
let rhs = Tensor::zeros((0, 3), DType::F32, device)?;
|
|
let output = lhs.matmul(&rhs)?;
|
|
assert_eq!(output.dims(), &[2, 3]);
|
|
assert_eq!(output.to_vec2::<f32>()?, &[[0., 0., 0.], [0., 0., 0.]]);
|
|
|
|
let lhs = Tensor::zeros((2, 3, 4), DType::F32, device)?
|
|
.transpose(1, 2)?
|
|
.narrow(1, 0, 0)?;
|
|
let rhs = Tensor::zeros((2, 4, 3), DType::F32, device)?
|
|
.transpose(1, 2)?
|
|
.narrow(2, 0, 0)?;
|
|
assert!(!lhs.is_contiguous());
|
|
assert!(!rhs.is_contiguous());
|
|
assert_eq!(lhs.dims(), &[2, 0, 3]);
|
|
assert_eq!(rhs.dims(), &[2, 3, 0]);
|
|
assert_eq!(lhs.matmul(&rhs)?.dims(), &[2, 0, 0]);
|
|
Ok(())
|
|
}
|
|
|
|
fn assert_matmul_error(
|
|
device: &Device,
|
|
lhs: (&[usize], DType),
|
|
rhs: (&[usize], DType),
|
|
expected: &str,
|
|
) -> Result<()> {
|
|
let lhs = Tensor::zeros(lhs.0, lhs.1, device)?;
|
|
let rhs = Tensor::zeros(rhs.0, rhs.1, device)?;
|
|
let err = lhs.matmul(&rhs).unwrap_err();
|
|
assert!(
|
|
err.to_string().starts_with(expected),
|
|
"unexpected error: {err}"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
fn zero_matmul_validation(device: &Device) -> Result<()> {
|
|
use DType::{F16, F32};
|
|
|
|
let shape_error = "shape mismatch in matmul";
|
|
assert_matmul_error(device, (&[0, 2], F32), (&[3, 4], F16), shape_error)?;
|
|
assert_matmul_error(device, (&[2, 3], F32), (&[4, 0], F32), shape_error)?;
|
|
assert_matmul_error(device, (&[0, 2, 3], F32), (&[1, 3, 4], F32), shape_error)?;
|
|
assert_matmul_error(
|
|
device,
|
|
(&[0, 2], F32),
|
|
(&[2, 3], F16),
|
|
"dtype mismatch in matmul",
|
|
)?;
|
|
Ok(())
|
|
}
|
|
|
|
fn zero_matmul_device_validation(device: &Device) -> Result<()> {
|
|
if device.is_cpu() {
|
|
return Ok(());
|
|
}
|
|
let lhs = Tensor::zeros((0, 2), DType::F32, &Device::Cpu)?;
|
|
let rhs = Tensor::zeros((2, 3), DType::F16, device)?;
|
|
let err = lhs.matmul(&rhs).unwrap_err();
|
|
assert!(
|
|
err.to_string().starts_with("device mismatch in matmul"),
|
|
"unexpected error: {err}"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
// A rank-2 rhs is broadcast over the batch dims only, which `broadcast_matmul` folds into a
|
|
// single 2D matmul instead of copying the rhs. Check the folded path against the per-batch
|
|
// products it stands for, contiguous lhs (folded) and non-contiguous lhs (fallback) alike.
|
|
fn broadcast_matmul_rank2_rhs(device: &Device) -> Result<()> {
|
|
let rhs = Tensor::randn(0f32, 1f32, (5, 2), device)?;
|
|
for lhs in [
|
|
Tensor::randn(0f32, 1f32, (3, 4, 5), device)?,
|
|
Tensor::randn(0f32, 1f32, (3, 6, 4, 5), device)?,
|
|
Tensor::randn(0f32, 1f32, (1, 1, 5), device)?,
|
|
Tensor::randn(0f32, 1f32, (3, 5, 4), device)?.transpose(1, 2)?,
|
|
] {
|
|
let out = lhs.broadcast_matmul(&rhs)?;
|
|
let mut dims = lhs.dims().to_vec();
|
|
let n = dims.len();
|
|
dims[n - 1] = 2;
|
|
assert_eq!(out.dims(), dims.as_slice());
|
|
// Same product, computed the way the doc comment describes it.
|
|
let batch: usize = lhs.dims()[..n - 2].iter().product();
|
|
let (m, k) = (lhs.dims()[n - 2], lhs.dims()[n - 1]);
|
|
let flat = lhs.reshape((batch, m, k))?;
|
|
let out = out.reshape((batch, m, 2))?;
|
|
for b in 0..batch {
|
|
let diff = (out.i(b)? - flat.i(b)?.matmul(&rhs)?)?.sqr()?.sum_all()?;
|
|
assert!(diff.to_vec0::<f32>()? < 1e-6);
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn tensor_dot() -> Result<()> {
|
|
let lhs = Tensor::new(&[1., 2., 3.], &Device::Cpu)?;
|
|
let rhs = Tensor::new(&[4., 5., 6.], &Device::Cpu)?;
|
|
let expected = Tensor::new(32., &Device::Cpu)?;
|
|
let dot_ret = lhs.dot(&rhs)?;
|
|
hanzo_ml::test_utils::assert_tensor_eq(&dot_ret, &expected)?;
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn tensor_mv() -> Result<()> {
|
|
let mat = Tensor::new(&[[1., 2., 3.], [4., 5., 6.]], &Device::Cpu)?;
|
|
let vec = Tensor::new(&[1., 1., 1.], &Device::Cpu)?;
|
|
let expected = Tensor::new(&[6., 15.], &Device::Cpu)?;
|
|
let mv_ret = mat.mv(&vec)?;
|
|
hanzo_ml::test_utils::assert_tensor_eq(&mv_ret, &expected)?;
|
|
Ok(())
|
|
}
|
|
|
|
// https://github.com/hanzoai/ml/issues/1948
|
|
fn squeeze_mm(device: &Device) -> Result<()> {
|
|
let seq_len = 8_usize;
|
|
let a = Tensor::zeros((1, seq_len, 16), DType::F32, device)?;
|
|
let x = a.i((.., seq_len - 1, ..))?;
|
|
let w = Tensor::zeros((32, 16), DType::F32, device)?.t()?;
|
|
let x = x.matmul(&w)?;
|
|
assert_eq!(x.dims(), &[1, 32]);
|
|
Ok(())
|
|
}
|
|
|
|
// https://github.com/hanzoai/ml/issues/1992
|
|
fn mm_layout(device: &Device) -> Result<()> {
|
|
let a = Tensor::arange(0f32, 16f32, device)?.reshape((1, 1, 4, 4))?;
|
|
let b = Tensor::arange(0f32, 8f32, device)?.reshape((1, 1, 4, 2))?;
|
|
let mm1 = a.matmul(&b)?;
|
|
// Forces the layout to be:
|
|
// shape: [1, 1, 4, 2], stride: [8, 2, 2, 1], start_offset: 0
|
|
// This is still a contiguous matrix but matmul checks are only the two last dimensions have
|
|
// non 1 sizes but matmul check may be reluctant to handle it.
|
|
let b = b.transpose(1, 2)?.force_contiguous()?.transpose(1, 2)?;
|
|
let mm2 = a.matmul(&b)?;
|
|
let diff = (mm1 - mm2)?.abs()?.sum_all()?.to_vec0::<f32>()?;
|
|
assert_eq!(diff, 0.);
|
|
Ok(())
|
|
}
|
|
|
|
test_device!(matmul, matmul_cpu, matmul_gpu, matmul_metal);
|
|
test_device!(
|
|
matmul_bf16,
|
|
matmul_bf16_cpu,
|
|
matmul_bf16_gpu,
|
|
matmul_bf16_metal
|
|
);
|
|
test_device!(
|
|
broadcast_matmul,
|
|
broadcast_matmul_cpu,
|
|
broadcast_matmul_gpu,
|
|
broadcast_matmul_metal
|
|
);
|
|
test_device!(
|
|
zero_matmul,
|
|
zero_matmul_cpu,
|
|
zero_matmul_gpu,
|
|
zero_matmul_metal
|
|
);
|
|
test_device!(
|
|
zero_matmul_validation,
|
|
zero_matmul_validation_cpu,
|
|
zero_matmul_validation_gpu,
|
|
zero_matmul_validation_metal
|
|
);
|
|
test_device!(
|
|
zero_matmul_device_validation,
|
|
zero_matmul_device_validation_cpu,
|
|
zero_matmul_device_validation_gpu,
|
|
zero_matmul_device_validation_metal
|
|
);
|
|
test_device!(
|
|
broadcast_matmul_rank2_rhs,
|
|
broadcast_matmul_rank2_rhs_cpu,
|
|
broadcast_matmul_rank2_rhs_gpu,
|
|
broadcast_matmul_rank2_rhs_metal
|
|
);
|
|
test_device!(squeeze_mm, squeeze_mm_cpu, squeeze_mm_gpu, squeeze_mm_metal);
|
|
test_device!(mm_layout, mm_layout_cpu, mm_layout_gpu, mm_layout_metal);
|