mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-21 03:12:10 +00:00
Three-way per file through the rename: base 39355c6c91, theirs the upstream
merge of main with 35d7ae7c (via 7ef6a694d), ours origin/main.
Lands the barrier thread pool, runtime CPU feature dispatch, aarch64 and x86
repacked quantized kernels (indexed_gemv, gemv_fused_shared_lhs), quantized
embedding on CPU/Metal/CUDA, paged flash-attn kernels, cutile fused MoE, the
in-place op aliasing API, parking_lot storage locks, Metal buffer builders and
debug labels, safetensors 0.8, hf-hub 1.0, tokenizers 0.23.
Ours keeps ROCm, Vulkan, wgpu, the IQ/MXFP4/NVFP4/TQ dtypes, and the i-quant
weight/activation block split in matmul. Our backends ride InplaceOpN and the
quantized embedding; Metal get_rows gains the IQ kernels. ug leaves, as upstream.
Co-authored-by: Hanzo Dev <[email protected]>
43 lines
1.4 KiB
Rust
43 lines
1.4 KiB
Rust
use hanzo_ml::{DType, Device, Result, Tensor};
|
|
use std::time::Instant;
|
|
|
|
#[test]
|
|
fn bf16_lfm_op_timings() -> Result<()> {
|
|
let dev = Device::Cpu;
|
|
// LFM ShortConv shape: depthwise conv over [1, 2048, 8192], k=3
|
|
let x = Tensor::rand(-1f32, 1f32, (1, 2048, 8192), &dev)?;
|
|
let w = Tensor::rand(-1f32, 1f32, (2048, 1, 3), &dev)?;
|
|
for dt in [DType::F32, DType::BF16] {
|
|
let xd = x.to_dtype(dt)?;
|
|
let wd = w.to_dtype(dt)?;
|
|
let t = Instant::now();
|
|
for _ in 0..3 {
|
|
let _ = xd.conv1d(&wd, 2, 1, 1, 2048)?;
|
|
}
|
|
println!("conv1d {dt:?}: {:?}", t.elapsed() / 3);
|
|
}
|
|
// rms_norm at [8192, 2048]
|
|
let x = Tensor::rand(-1f32, 1f32, (1, 8192, 2048), &dev)?;
|
|
let w = Tensor::rand(0.5f32, 1.5f32, (2048,), &dev)?;
|
|
for dt in [DType::F32, DType::BF16] {
|
|
let xd = x.to_dtype(dt)?;
|
|
let wd = w.to_dtype(dt)?;
|
|
let t = Instant::now();
|
|
for _ in 0..5 {
|
|
let _ = hanzo_nn::ops::rms_norm(&xd, &wd, 1e-6)?;
|
|
}
|
|
println!("rms_norm {dt:?}: {:?}", t.elapsed() / 5);
|
|
}
|
|
// silu + mul at [8192, 6144]
|
|
let a = Tensor::rand(-1f32, 1f32, (8192, 6144), &dev)?;
|
|
for dt in [DType::F32, DType::BF16] {
|
|
let ad = a.to_dtype(dt)?;
|
|
let t = Instant::now();
|
|
for _ in 0..5 {
|
|
let _ = (hanzo_nn::ops::silu(&ad)? * &ad)?;
|
|
}
|
|
println!("silu*mul {dt:?}: {:?}", t.elapsed() / 5);
|
|
}
|
|
Ok(())
|
|
}
|