mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-21 22:51:09 +00:00
106 lines
3.7 KiB
Rust
106 lines
3.7 KiB
Rust
use crate::benchmarks::{BenchDevice, BenchDeviceHandler};
|
|
use criterion::{criterion_group, Criterion, Throughput};
|
|
use hanzo_ml::{DType, Device, Tensor};
|
|
use std::hint::black_box;
|
|
use std::time::Instant;
|
|
|
|
// Representative transformer KV-cache shape: [batch, seq, head_dim].
|
|
const BATCH: usize = 8;
|
|
const SEQ: usize = 512;
|
|
const HIDDEN: usize = 64;
|
|
const PAD: usize = 16;
|
|
|
|
fn run_contiguous(t: &Tensor) {
|
|
t.contiguous().unwrap();
|
|
}
|
|
|
|
fn bench_contiguous(c: &mut Criterion, device: &Device) {
|
|
let bytes = (BATCH * SEQ * HIDDEN * DType::F32.size_in_bytes()) as u64;
|
|
|
|
// Contiguous baseline:
|
|
// Close to theoretical best case / single optimised copy.
|
|
// This is for comparison with the other benches. We use `force_contigious`, because we're
|
|
// not interested in the performance of `Arc::clone()`
|
|
{
|
|
let t = Tensor::zeros((BATCH, SEQ, HIDDEN), DType::F32, device).unwrap();
|
|
let mut group = c.benchmark_group(device.bench_name("contiguous"));
|
|
group.throughput(Throughput::Bytes(bytes));
|
|
group.bench_function("iter", |b| {
|
|
b.iter_custom(|iters| {
|
|
let start = Instant::now();
|
|
for _ in 0..iters {
|
|
black_box(t.force_contiguous().unwrap());
|
|
}
|
|
device.sync().unwrap();
|
|
start.elapsed()
|
|
})
|
|
});
|
|
group.finish();
|
|
}
|
|
|
|
// UniformBlocks
|
|
// Narrow a padded tensor so only the outer stride differs.
|
|
// narrow(1, 0, SEQ) on [BATCH, SEQ + PAD, HIDDEN] gives strides
|
|
// [HIDDEN * (SEQ + PAD), HIDDEN, 1]. Inner block of SEQ * HIDDEN is fully contiguous.
|
|
// This is the layout that copy2d targets.
|
|
{
|
|
let base = Tensor::zeros((BATCH, SEQ + PAD, HIDDEN), DType::F32, device).unwrap();
|
|
let t = base.narrow(1, 0, SEQ).unwrap();
|
|
assert!(
|
|
!t.is_contiguous(),
|
|
"expected non-contiguous tensor for benchmark"
|
|
);
|
|
|
|
let mut group = c.benchmark_group(device.bench_name("contiguous_uniform"));
|
|
group.throughput(Throughput::Bytes(bytes));
|
|
group.bench_function("iter", |b| {
|
|
b.iter_custom(|iters| {
|
|
let start = Instant::now();
|
|
for _ in 0..iters {
|
|
run_contiguous(black_box(&t));
|
|
}
|
|
device.sync().unwrap();
|
|
start.elapsed()
|
|
})
|
|
});
|
|
group.finish();
|
|
}
|
|
|
|
// MultipleBlocks
|
|
// Transpose mixes dimensions so the innermost stride is no longer 1.
|
|
// `contiguous()` must use the general strided kernel.
|
|
{
|
|
let base = Tensor::zeros((BATCH, SEQ, HIDDEN), DType::F32, device).unwrap();
|
|
// transpose(0, 1) - shape: [SEQ, BATCH, HIDDEN], strides: [HIDDEN, SEQ * HIDDEN, 1]
|
|
// aka MultipleBlocks { block_len = HIDDEN } (SEQ * BATCH blocks).
|
|
let t = base.transpose(0, 1).unwrap();
|
|
assert!(
|
|
!t.is_contiguous(),
|
|
"expected non-contiguous tensor for benchmark"
|
|
);
|
|
|
|
let mut group = c.benchmark_group(device.bench_name("contiguous_strided"));
|
|
group.throughput(Throughput::Bytes(bytes));
|
|
group.bench_function("iter", |b| {
|
|
b.iter_custom(|iters| {
|
|
let start = Instant::now();
|
|
for _ in 0..iters {
|
|
run_contiguous(black_box(&t));
|
|
}
|
|
device.sync().unwrap();
|
|
start.elapsed()
|
|
})
|
|
});
|
|
group.finish();
|
|
}
|
|
}
|
|
|
|
fn criterion_benchmark(c: &mut Criterion) {
|
|
let handler = BenchDeviceHandler::new().unwrap();
|
|
for device in &handler.devices {
|
|
bench_contiguous(c, device);
|
|
}
|
|
}
|
|
|
|
criterion_group!(benches, criterion_benchmark);
|