Files
zeekayandHanzo Dev 942bf024be Metal trains with fused AdamW, gradient-norm and normalization-backward kernels.
- AdamW::step_scaled updates an F32 parameter on Metal with one kernel that reads the parameter,
  its gradient and both moments once and writes them back once, the gradient scaled first for
  clipping; other devices keep the tensor-op path. Optimizer::step is step_scaled(grads, 1).
- optim::grad_norm is the global L2 norm of a set of gradients, one host read; on Metal each
  gradient adds its sum of squares into one accumulator.
- The LayerNorm and last-dim softmax backward passes run as one threadgroup per row,
  accumulated in F32, with the LayerNorm parameter gradients summed column-wise by a second
  kernel, instead of a dozen tensor ops over the activation.
- InplaceOpN and Tensor::inplace_op are public, for in-place ops with more than two sources.

On an M4 Max, for a bf16 ModernBERT-large training step of 32 rows of 128 tokens, the optimizer
went from 315 to 29 ms. tests/fused_optim.rs checks the fused step and norm against the CPU
tensor path; tests/grad.rs checks the fused backward kernels in F32 and at model size in bf16.

Co-authored-by: Hanzo Dev <[email protected]>
2026-09-24 21:01:28 -07:00

195 lines
6.9 KiB
Rust

//! The fused ops take part in autograd: each one's gradients equal those of its unfused
//! composite, on every device the fused kernel runs on.
#[cfg(feature = "mkl")]
extern crate intel_mkl_src;
#[cfg(feature = "accelerate")]
extern crate accelerate_src;
use hanzo_ml::{test_device, DType, Device, Result, Tensor, Var};
use hanzo_nn::{ops, rotary_emb, Module};
/// Gradients of `Σ f(x…) ⊙ w` with respect to every input, for a fixed random `w`.
fn grads(inputs: &[&Var], y: &Tensor, w: &Tensor) -> Result<Vec<Tensor>> {
let g = (y * w)?.sum_all()?.backward()?;
inputs
.iter()
.map(|v| {
g.get(v.as_tensor())
.cloned()
.ok_or_else(|| hanzo_ml::Error::Msg("no gradient reached an input".into()))
})
.collect()
}
fn max_diff(a: &Tensor, b: &Tensor) -> Result<f32> {
(a.to_dtype(DType::F32)? - b.to_dtype(DType::F32)?)?
.abs()?
.flatten_all()?
.max(0)?
.to_scalar::<f32>()
}
fn assert_close(fused: &[Tensor], slow: &[Tensor], tol: f32, what: &str) -> Result<()> {
assert_eq!(fused.len(), slow.len());
for (i, (a, b)) in fused.iter().zip(slow).enumerate() {
let d = max_diff(a, b)?;
assert!(d < tol, "{what}: input {i} gradient differs by {d}");
}
Ok(())
}
fn var(shape: &[usize], device: &Device) -> Result<Var> {
Var::from_tensor(&Tensor::randn(0f32, 1f32, shape, device)?)
}
fn softmax_last_dim(device: &Device) -> Result<()> {
let x = var(&[3, 4, 37], device)?;
let w = Tensor::randn(0f32, 1f32, (3, 4, 37), device)?;
let fused = grads(&[&x], &ops::softmax_last_dim(x.as_tensor())?, &w)?;
let slow = grads(&[&x], &ops::softmax(x.as_tensor(), 2)?, &w)?;
assert_close(&fused, &slow, 1e-5, "softmax_last_dim")
}
fn layer_norm(device: &Device) -> Result<()> {
let (x, a, b) = (
var(&[2, 5, 64], device)?,
var(&[64], device)?,
var(&[64], device)?,
);
let w = Tensor::randn(0f32, 1f32, (2, 5, 64), device)?;
let ins = [&x, &a, &b];
let fused = grads(
&ins,
&ops::layer_norm(x.as_tensor(), a.as_tensor(), b.as_tensor(), 1e-5)?,
&w,
)?;
let slow = grads(
&ins,
&ops::layer_norm_slow(x.as_tensor(), a.as_tensor(), b.as_tensor(), 1e-5)?,
&w,
)?;
assert_close(&fused, &slow, 1e-4, "layer_norm")
}
fn layer_norm_no_bias(device: &Device) -> Result<()> {
let (x, a) = (var(&[4, 7, 32], device)?, var(&[32], device)?);
let w = Tensor::randn(0f32, 1f32, (4, 7, 32), device)?;
let ln = hanzo_nn::LayerNorm::new_no_bias(a.as_tensor().clone(), 1e-5);
let fused = grads(&[&x, &a], &ln.forward(x.as_tensor())?, &w)?;
let zero = a.as_tensor().zeros_like()?;
let slow = grads(
&[&x, &a],
&ops::layer_norm_slow(x.as_tensor(), a.as_tensor(), &zero, 1e-5)?,
&w,
)?;
assert_close(&fused, &slow, 1e-4, "layer_norm (no bias)")
}
fn rms_norm(device: &Device) -> Result<()> {
let (x, a) = (var(&[3, 6, 48], device)?, var(&[48], device)?);
let w = Tensor::randn(0f32, 1f32, (3, 6, 48), device)?;
let fused = grads(
&[&x, &a],
&ops::rms_norm(x.as_tensor(), a.as_tensor(), 1e-5)?,
&w,
)?;
let slow = grads(
&[&x, &a],
&ops::rms_norm_slow(x.as_tensor(), a.as_tensor(), 1e-5)?,
&w,
)?;
assert_close(&fused, &slow, 1e-4, "rms_norm")
}
fn silu_mul(device: &Device) -> Result<()> {
let (g, u) = (var(&[5, 33], device)?, var(&[5, 33], device)?);
let w = Tensor::randn(0f32, 1f32, (5, 33), device)?;
let fused = grads(&[&g, &u], &ops::silu_mul(g.as_tensor(), u.as_tensor())?, &w)?;
let slow = grads(&[&g, &u], &(g.as_tensor().silu()? * u.as_tensor())?, &w)?;
assert_close(&fused, &slow, 1e-5, "silu_mul")
}
fn rope(device: &Device) -> Result<()> {
let (b, h, t, d) = (2, 3, 11, 16);
let x = var(&[b, h, t, d], device)?;
// tables longer than the sequence, as a model builds them
let cos = Tensor::randn(0f32, 1f32, (t + 5, d / 2), device)?;
let sin = Tensor::randn(0f32, 1f32, (t + 5, d / 2), device)?;
let w = Tensor::randn(0f32, 1f32, (b, h, t, d), device)?;
let fused = grads(&[&x], &rotary_emb::rope(x.as_tensor(), &cos, &sin)?, &w)?;
let slow = grads(
&[&x],
&rotary_emb::rope_slow(x.as_tensor(), &cos, &sin)?,
&w,
)?;
assert_close(&fused, &slow, 1e-5, "rope")?;
let fused = grads(&[&x], &rotary_emb::rope_i(x.as_tensor(), &cos, &sin)?, &w)?;
let slow = grads(
&[&x],
&rotary_emb::rope_i_slow(x.as_tensor(), &cos, &sin)?,
&w,
)?;
assert_close(&fused, &slow, 1e-5, "rope_i")?;
// (b, t, h, d): the same rotation with the head and time axes swapped
let xt = var(&[b, t, h, d], device)?;
let wt = w.transpose(1, 2)?.contiguous()?;
let fused = grads(
&[&xt],
&rotary_emb::rope_thd(xt.as_tensor(), &cos, &sin)?,
&wt,
)?;
let slow = rotary_emb::rope_slow(&xt.as_tensor().transpose(1, 2)?.contiguous()?, &cos, &sin)?
.transpose(1, 2)?;
let slow = grads(&[&xt], &slow, &wt)?;
assert_close(&fused, &slow, 1e-5, "rope_thd")
}
/// Model-sized rows in bf16: the fused kernels accumulate in F32, so their gradients match the
/// F32 composite to bf16 rounding. The row count is not a multiple of the kernel's row chunk.
fn half_precision_rows(device: &Device) -> Result<()> {
let bf = |v: &Var| v.as_tensor().to_dtype(DType::BF16);
let (x, a, b) = (
var(&[3, 70, 1024], device)?,
var(&[1024], device)?,
var(&[1024], device)?,
);
let w = Tensor::randn(0f32, 1f32, (3, 70, 1024), device)?;
let ins = [&x, &a, &b];
let y = ops::layer_norm(&bf(&x)?, &bf(&a)?, &bf(&b)?, 1e-5)?.to_dtype(DType::F32)?;
let fused = grads(&ins, &y, &w)?;
let slow = grads(
&ins,
&ops::layer_norm_slow(x.as_tensor(), a.as_tensor(), b.as_tensor(), 1e-5)?,
&w,
)?;
for (i, (f, s)) in fused.iter().zip(&slow).enumerate() {
let scale = s.abs()?.flatten_all()?.max(0)?.to_scalar::<f32>()?;
let d = max_diff(f, s)?;
assert!(
d < 2e-2 * scale.max(1.0),
"layer_norm bf16 input {i}: {d} (scale {scale})"
);
}
let z = var(&[4, 16, 130], device)?;
let w = Tensor::randn(0f32, 1f32, (4, 16, 130), device)?;
let y = ops::softmax_last_dim(&bf(&z)?)?.to_dtype(DType::F32)?;
let fused = grads(&[&z], &y, &w)?;
let slow = grads(&[&z], &ops::softmax(z.as_tensor(), 2)?, &w)?;
let d = max_diff(&fused[0], &slow[0])?;
assert!(d < 2e-2, "softmax bf16: {d}");
Ok(())
}
test_device!(softmax_last_dim, softmax_cpu, softmax_gpu, softmax_metal);
test_device!(layer_norm, ln_cpu, ln_gpu, ln_metal);
test_device!(layer_norm_no_bias, lnnb_cpu, lnnb_gpu, lnnb_metal);
test_device!(rms_norm, rms_cpu, rms_gpu, rms_metal);
test_device!(silu_mul, silu_mul_cpu, silu_mul_gpu, silu_mul_metal);
test_device!(rope, rope_cpu, rope_gpu, rope_metal);
test_device!(half_precision_rows, half_cpu, half_gpu, half_metal);