Files
ml/hanzo-nn/tests/fused_optim.rs
zeekayandHanzo Dev 942bf024be Metal trains with fused AdamW, gradient-norm and normalization-backward kernels.
- AdamW::step_scaled updates an F32 parameter on Metal with one kernel that reads the parameter,
  its gradient and both moments once and writes them back once, the gradient scaled first for
  clipping; other devices keep the tensor-op path. Optimizer::step is step_scaled(grads, 1).
- optim::grad_norm is the global L2 norm of a set of gradients, one host read; on Metal each
  gradient adds its sum of squares into one accumulator.
- The LayerNorm and last-dim softmax backward passes run as one threadgroup per row,
  accumulated in F32, with the LayerNorm parameter gradients summed column-wise by a second
  kernel, instead of a dozen tensor ops over the activation.
- InplaceOpN and Tensor::inplace_op are public, for in-place ops with more than two sources.

On an M4 Max, for a bf16 ModernBERT-large training step of 32 rows of 128 tokens, the optimizer
went from 315 to 29 ms. tests/fused_optim.rs checks the fused step and norm against the CPU
tensor path; tests/grad.rs checks the fused backward kernels in F32 and at model size in bf16.

Co-authored-by: Hanzo Dev <[email protected]>
2026-09-24 21:01:28 -07:00

69 lines
2.4 KiB
Rust

//! The fused Metal AdamW step and gradient norm equal the tensor-op path on the CPU.
#![cfg(feature = "metal")]
use hanzo_ml::{DType, Device, Result, Tensor, Var};
use hanzo_nn::optim::{grad_norm, AdamW, ParamsAdamW};
use hanzo_nn::Optimizer;
fn params() -> ParamsAdamW {
ParamsAdamW {
lr: 3e-3,
beta1: 0.9,
beta2: 0.98,
eps: 1e-6,
weight_decay: 0.05,
}
}
/// Runs `steps` AdamW steps from `w0` with the gradient of `Σ w ⊙ c` scaled by `scale`.
fn run(dev: &Device, w0: &Tensor, c: &Tensor, steps: usize, scale: f64) -> Result<(Vec<f32>, f64)> {
let w = Var::from_tensor(&w0.to_device(dev)?)?;
let c = c.to_device(dev)?;
let mut opt = AdamW::new(vec![w.clone()], params())?;
let mut norm = 0.0;
for _ in 0..steps {
let loss = (w.as_tensor().sqr()? * &c)?.sum_all()?;
let grads = loss.backward()?;
norm = grad_norm(&grads, &[w.clone()])?;
opt.step_scaled(&grads, scale)?;
}
Ok((w.as_tensor().flatten_all()?.to_vec1::<f32>()?, norm))
}
#[test]
fn adamw_fused_matches_tensor_path() -> Result<()> {
let metal = Device::new_metal(0)?;
// an odd length: the kernel's last threadgroup is partial
let w0 = Tensor::randn(0f32, 1f32, (37, 129), &Device::Cpu)?;
let c = Tensor::randn(0f32, 1f32, (37, 129), &Device::Cpu)?;
for scale in [1.0, 0.25] {
let (cpu, n_cpu) = run(&Device::Cpu, &w0, &c, 5, scale)?;
let (gpu, n_gpu) = run(&metal, &w0, &c, 5, scale)?;
let d = cpu
.iter()
.zip(&gpu)
.map(|(a, b)| (a - b).abs())
.fold(0f32, f32::max);
assert!(d < 1e-5, "scale {scale}: parameters differ by {d}");
assert!(
(n_cpu - n_gpu).abs() / n_cpu < 1e-5,
"norm {n_cpu} vs {n_gpu}"
);
}
Ok(())
}
#[test]
fn grad_norm_mixes_fused_and_other_dtypes() -> Result<()> {
let metal = Device::new_metal(0)?;
let a = Var::from_tensor(&Tensor::randn(0f32, 1f32, 1000, &metal)?)?;
let b = Var::from_tensor(&Tensor::randn(0f32, 1f32, 300, &metal)?.to_dtype(DType::BF16)?)?;
let loss = (a.as_tensor().sum_all()? * 3.0)?
.add(&(b.as_tensor().to_dtype(DType::F32)?.sum_all()? * 2.0)?)?;
let grads = loss.backward()?;
let n = grad_norm(&grads, &[a, b])?;
let want = (1000.0 * 9.0 + 300.0 * 4.0f64).sqrt();
assert!((n - want).abs() / want < 1e-5, "{n} vs {want}");
Ok(())
}