mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-21 22:51:09 +00:00
- AdamW::step_scaled updates an F32 parameter on Metal with one kernel that reads the parameter, its gradient and both moments once and writes them back once, the gradient scaled first for clipping; other devices keep the tensor-op path. Optimizer::step is step_scaled(grads, 1). - optim::grad_norm is the global L2 norm of a set of gradients, one host read; on Metal each gradient adds its sum of squares into one accumulator. - The LayerNorm and last-dim softmax backward passes run as one threadgroup per row, accumulated in F32, with the LayerNorm parameter gradients summed column-wise by a second kernel, instead of a dozen tensor ops over the activation. - InplaceOpN and Tensor::inplace_op are public, for in-place ops with more than two sources. On an M4 Max, for a bf16 ModernBERT-large training step of 32 rows of 128 tokens, the optimizer went from 315 to 29 ms. tests/fused_optim.rs checks the fused step and norm against the CPU tensor path; tests/grad.rs checks the fused backward kernels in F32 and at model size in bf16. Co-authored-by: Hanzo Dev <[email protected]>
195 lines
6.9 KiB
Rust
195 lines
6.9 KiB
Rust
//! The fused ops take part in autograd: each one's gradients equal those of its unfused
|
|
//! composite, on every device the fused kernel runs on.
|
|
|
|
#[cfg(feature = "mkl")]
|
|
extern crate intel_mkl_src;
|
|
|
|
#[cfg(feature = "accelerate")]
|
|
extern crate accelerate_src;
|
|
|
|
use hanzo_ml::{test_device, DType, Device, Result, Tensor, Var};
|
|
use hanzo_nn::{ops, rotary_emb, Module};
|
|
|
|
/// Gradients of `Σ f(x…) ⊙ w` with respect to every input, for a fixed random `w`.
|
|
fn grads(inputs: &[&Var], y: &Tensor, w: &Tensor) -> Result<Vec<Tensor>> {
|
|
let g = (y * w)?.sum_all()?.backward()?;
|
|
inputs
|
|
.iter()
|
|
.map(|v| {
|
|
g.get(v.as_tensor())
|
|
.cloned()
|
|
.ok_or_else(|| hanzo_ml::Error::Msg("no gradient reached an input".into()))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn max_diff(a: &Tensor, b: &Tensor) -> Result<f32> {
|
|
(a.to_dtype(DType::F32)? - b.to_dtype(DType::F32)?)?
|
|
.abs()?
|
|
.flatten_all()?
|
|
.max(0)?
|
|
.to_scalar::<f32>()
|
|
}
|
|
|
|
fn assert_close(fused: &[Tensor], slow: &[Tensor], tol: f32, what: &str) -> Result<()> {
|
|
assert_eq!(fused.len(), slow.len());
|
|
for (i, (a, b)) in fused.iter().zip(slow).enumerate() {
|
|
let d = max_diff(a, b)?;
|
|
assert!(d < tol, "{what}: input {i} gradient differs by {d}");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn var(shape: &[usize], device: &Device) -> Result<Var> {
|
|
Var::from_tensor(&Tensor::randn(0f32, 1f32, shape, device)?)
|
|
}
|
|
|
|
fn softmax_last_dim(device: &Device) -> Result<()> {
|
|
let x = var(&[3, 4, 37], device)?;
|
|
let w = Tensor::randn(0f32, 1f32, (3, 4, 37), device)?;
|
|
let fused = grads(&[&x], &ops::softmax_last_dim(x.as_tensor())?, &w)?;
|
|
let slow = grads(&[&x], &ops::softmax(x.as_tensor(), 2)?, &w)?;
|
|
assert_close(&fused, &slow, 1e-5, "softmax_last_dim")
|
|
}
|
|
|
|
fn layer_norm(device: &Device) -> Result<()> {
|
|
let (x, a, b) = (
|
|
var(&[2, 5, 64], device)?,
|
|
var(&[64], device)?,
|
|
var(&[64], device)?,
|
|
);
|
|
let w = Tensor::randn(0f32, 1f32, (2, 5, 64), device)?;
|
|
let ins = [&x, &a, &b];
|
|
let fused = grads(
|
|
&ins,
|
|
&ops::layer_norm(x.as_tensor(), a.as_tensor(), b.as_tensor(), 1e-5)?,
|
|
&w,
|
|
)?;
|
|
let slow = grads(
|
|
&ins,
|
|
&ops::layer_norm_slow(x.as_tensor(), a.as_tensor(), b.as_tensor(), 1e-5)?,
|
|
&w,
|
|
)?;
|
|
assert_close(&fused, &slow, 1e-4, "layer_norm")
|
|
}
|
|
|
|
fn layer_norm_no_bias(device: &Device) -> Result<()> {
|
|
let (x, a) = (var(&[4, 7, 32], device)?, var(&[32], device)?);
|
|
let w = Tensor::randn(0f32, 1f32, (4, 7, 32), device)?;
|
|
let ln = hanzo_nn::LayerNorm::new_no_bias(a.as_tensor().clone(), 1e-5);
|
|
let fused = grads(&[&x, &a], &ln.forward(x.as_tensor())?, &w)?;
|
|
let zero = a.as_tensor().zeros_like()?;
|
|
let slow = grads(
|
|
&[&x, &a],
|
|
&ops::layer_norm_slow(x.as_tensor(), a.as_tensor(), &zero, 1e-5)?,
|
|
&w,
|
|
)?;
|
|
assert_close(&fused, &slow, 1e-4, "layer_norm (no bias)")
|
|
}
|
|
|
|
fn rms_norm(device: &Device) -> Result<()> {
|
|
let (x, a) = (var(&[3, 6, 48], device)?, var(&[48], device)?);
|
|
let w = Tensor::randn(0f32, 1f32, (3, 6, 48), device)?;
|
|
let fused = grads(
|
|
&[&x, &a],
|
|
&ops::rms_norm(x.as_tensor(), a.as_tensor(), 1e-5)?,
|
|
&w,
|
|
)?;
|
|
let slow = grads(
|
|
&[&x, &a],
|
|
&ops::rms_norm_slow(x.as_tensor(), a.as_tensor(), 1e-5)?,
|
|
&w,
|
|
)?;
|
|
assert_close(&fused, &slow, 1e-4, "rms_norm")
|
|
}
|
|
|
|
fn silu_mul(device: &Device) -> Result<()> {
|
|
let (g, u) = (var(&[5, 33], device)?, var(&[5, 33], device)?);
|
|
let w = Tensor::randn(0f32, 1f32, (5, 33), device)?;
|
|
let fused = grads(&[&g, &u], &ops::silu_mul(g.as_tensor(), u.as_tensor())?, &w)?;
|
|
let slow = grads(&[&g, &u], &(g.as_tensor().silu()? * u.as_tensor())?, &w)?;
|
|
assert_close(&fused, &slow, 1e-5, "silu_mul")
|
|
}
|
|
|
|
fn rope(device: &Device) -> Result<()> {
|
|
let (b, h, t, d) = (2, 3, 11, 16);
|
|
let x = var(&[b, h, t, d], device)?;
|
|
// tables longer than the sequence, as a model builds them
|
|
let cos = Tensor::randn(0f32, 1f32, (t + 5, d / 2), device)?;
|
|
let sin = Tensor::randn(0f32, 1f32, (t + 5, d / 2), device)?;
|
|
let w = Tensor::randn(0f32, 1f32, (b, h, t, d), device)?;
|
|
let fused = grads(&[&x], &rotary_emb::rope(x.as_tensor(), &cos, &sin)?, &w)?;
|
|
let slow = grads(
|
|
&[&x],
|
|
&rotary_emb::rope_slow(x.as_tensor(), &cos, &sin)?,
|
|
&w,
|
|
)?;
|
|
assert_close(&fused, &slow, 1e-5, "rope")?;
|
|
|
|
let fused = grads(&[&x], &rotary_emb::rope_i(x.as_tensor(), &cos, &sin)?, &w)?;
|
|
let slow = grads(
|
|
&[&x],
|
|
&rotary_emb::rope_i_slow(x.as_tensor(), &cos, &sin)?,
|
|
&w,
|
|
)?;
|
|
assert_close(&fused, &slow, 1e-5, "rope_i")?;
|
|
|
|
// (b, t, h, d): the same rotation with the head and time axes swapped
|
|
let xt = var(&[b, t, h, d], device)?;
|
|
let wt = w.transpose(1, 2)?.contiguous()?;
|
|
let fused = grads(
|
|
&[&xt],
|
|
&rotary_emb::rope_thd(xt.as_tensor(), &cos, &sin)?,
|
|
&wt,
|
|
)?;
|
|
let slow = rotary_emb::rope_slow(&xt.as_tensor().transpose(1, 2)?.contiguous()?, &cos, &sin)?
|
|
.transpose(1, 2)?;
|
|
let slow = grads(&[&xt], &slow, &wt)?;
|
|
assert_close(&fused, &slow, 1e-5, "rope_thd")
|
|
}
|
|
|
|
/// Model-sized rows in bf16: the fused kernels accumulate in F32, so their gradients match the
|
|
/// F32 composite to bf16 rounding. The row count is not a multiple of the kernel's row chunk.
|
|
fn half_precision_rows(device: &Device) -> Result<()> {
|
|
let bf = |v: &Var| v.as_tensor().to_dtype(DType::BF16);
|
|
let (x, a, b) = (
|
|
var(&[3, 70, 1024], device)?,
|
|
var(&[1024], device)?,
|
|
var(&[1024], device)?,
|
|
);
|
|
let w = Tensor::randn(0f32, 1f32, (3, 70, 1024), device)?;
|
|
let ins = [&x, &a, &b];
|
|
let y = ops::layer_norm(&bf(&x)?, &bf(&a)?, &bf(&b)?, 1e-5)?.to_dtype(DType::F32)?;
|
|
let fused = grads(&ins, &y, &w)?;
|
|
let slow = grads(
|
|
&ins,
|
|
&ops::layer_norm_slow(x.as_tensor(), a.as_tensor(), b.as_tensor(), 1e-5)?,
|
|
&w,
|
|
)?;
|
|
for (i, (f, s)) in fused.iter().zip(&slow).enumerate() {
|
|
let scale = s.abs()?.flatten_all()?.max(0)?.to_scalar::<f32>()?;
|
|
let d = max_diff(f, s)?;
|
|
assert!(
|
|
d < 2e-2 * scale.max(1.0),
|
|
"layer_norm bf16 input {i}: {d} (scale {scale})"
|
|
);
|
|
}
|
|
|
|
let z = var(&[4, 16, 130], device)?;
|
|
let w = Tensor::randn(0f32, 1f32, (4, 16, 130), device)?;
|
|
let y = ops::softmax_last_dim(&bf(&z)?)?.to_dtype(DType::F32)?;
|
|
let fused = grads(&[&z], &y, &w)?;
|
|
let slow = grads(&[&z], &ops::softmax(z.as_tensor(), 2)?, &w)?;
|
|
let d = max_diff(&fused[0], &slow[0])?;
|
|
assert!(d < 2e-2, "softmax bf16: {d}");
|
|
Ok(())
|
|
}
|
|
|
|
test_device!(softmax_last_dim, softmax_cpu, softmax_gpu, softmax_metal);
|
|
test_device!(layer_norm, ln_cpu, ln_gpu, ln_metal);
|
|
test_device!(layer_norm_no_bias, lnnb_cpu, lnnb_gpu, lnnb_metal);
|
|
test_device!(rms_norm, rms_cpu, rms_gpu, rms_metal);
|
|
test_device!(silu_mul, silu_mul_cpu, silu_mul_gpu, silu_mul_metal);
|
|
test_device!(rope, rope_cpu, rope_gpu, rope_metal);
|
|
test_device!(half_precision_rows, half_cpu, half_gpu, half_metal);
|