mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-20 23:08:29 +00:00
- AdamW::step_scaled updates an F32 parameter on Metal with one kernel that reads the parameter, its gradient and both moments once and writes them back once, the gradient scaled first for clipping; other devices keep the tensor-op path. Optimizer::step is step_scaled(grads, 1). - optim::grad_norm is the global L2 norm of a set of gradients, one host read; on Metal each gradient adds its sum of squares into one accumulator. - The LayerNorm and last-dim softmax backward passes run as one threadgroup per row, accumulated in F32, with the LayerNorm parameter gradients summed column-wise by a second kernel, instead of a dozen tensor ops over the activation. - InplaceOpN and Tensor::inplace_op are public, for in-place ops with more than two sources. On an M4 Max, for a bf16 ModernBERT-large training step of 32 rows of 128 tokens, the optimizer went from 315 to 29 ms. tests/fused_optim.rs checks the fused step and norm against the CPU tensor path; tests/grad.rs checks the fused backward kernels in F32 and at model size in bf16. Co-authored-by: Hanzo Dev <[email protected]>
69 lines
2.4 KiB
Rust
69 lines
2.4 KiB
Rust
//! The fused Metal AdamW step and gradient norm equal the tensor-op path on the CPU.
|
|
#![cfg(feature = "metal")]
|
|
|
|
use hanzo_ml::{DType, Device, Result, Tensor, Var};
|
|
use hanzo_nn::optim::{grad_norm, AdamW, ParamsAdamW};
|
|
use hanzo_nn::Optimizer;
|
|
|
|
fn params() -> ParamsAdamW {
|
|
ParamsAdamW {
|
|
lr: 3e-3,
|
|
beta1: 0.9,
|
|
beta2: 0.98,
|
|
eps: 1e-6,
|
|
weight_decay: 0.05,
|
|
}
|
|
}
|
|
|
|
/// Runs `steps` AdamW steps from `w0` with the gradient of `Σ w ⊙ c` scaled by `scale`.
|
|
fn run(dev: &Device, w0: &Tensor, c: &Tensor, steps: usize, scale: f64) -> Result<(Vec<f32>, f64)> {
|
|
let w = Var::from_tensor(&w0.to_device(dev)?)?;
|
|
let c = c.to_device(dev)?;
|
|
let mut opt = AdamW::new(vec![w.clone()], params())?;
|
|
let mut norm = 0.0;
|
|
for _ in 0..steps {
|
|
let loss = (w.as_tensor().sqr()? * &c)?.sum_all()?;
|
|
let grads = loss.backward()?;
|
|
norm = grad_norm(&grads, &[w.clone()])?;
|
|
opt.step_scaled(&grads, scale)?;
|
|
}
|
|
Ok((w.as_tensor().flatten_all()?.to_vec1::<f32>()?, norm))
|
|
}
|
|
|
|
#[test]
|
|
fn adamw_fused_matches_tensor_path() -> Result<()> {
|
|
let metal = Device::new_metal(0)?;
|
|
// an odd length: the kernel's last threadgroup is partial
|
|
let w0 = Tensor::randn(0f32, 1f32, (37, 129), &Device::Cpu)?;
|
|
let c = Tensor::randn(0f32, 1f32, (37, 129), &Device::Cpu)?;
|
|
for scale in [1.0, 0.25] {
|
|
let (cpu, n_cpu) = run(&Device::Cpu, &w0, &c, 5, scale)?;
|
|
let (gpu, n_gpu) = run(&metal, &w0, &c, 5, scale)?;
|
|
let d = cpu
|
|
.iter()
|
|
.zip(&gpu)
|
|
.map(|(a, b)| (a - b).abs())
|
|
.fold(0f32, f32::max);
|
|
assert!(d < 1e-5, "scale {scale}: parameters differ by {d}");
|
|
assert!(
|
|
(n_cpu - n_gpu).abs() / n_cpu < 1e-5,
|
|
"norm {n_cpu} vs {n_gpu}"
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn grad_norm_mixes_fused_and_other_dtypes() -> Result<()> {
|
|
let metal = Device::new_metal(0)?;
|
|
let a = Var::from_tensor(&Tensor::randn(0f32, 1f32, 1000, &metal)?)?;
|
|
let b = Var::from_tensor(&Tensor::randn(0f32, 1f32, 300, &metal)?.to_dtype(DType::BF16)?)?;
|
|
let loss = (a.as_tensor().sum_all()? * 3.0)?
|
|
.add(&(b.as_tensor().to_dtype(DType::F32)?.sum_all()? * 2.0)?)?;
|
|
let grads = loss.backward()?;
|
|
let n = grad_norm(&grads, &[a, b])?;
|
|
let want = (1000.0 * 9.0 + 300.0 * 4.0f64).sqrt();
|
|
assert!((n - want).abs() / want < 1e-5, "{n} vs {want}");
|
|
Ok(())
|
|
}
|