Files
engine/hanzo-engine/src/ops.rs

3967 lines
145 KiB
Rust

use hanzo_ml::{shape::Dim, DType, Result, Tensor, D};
use std::sync::Arc;
#[cfg(feature = "cuda")]
use crate::cuda::ffi;
use crate::layers::Activation;
#[cfg(feature = "cuda")]
use hanzo_ml::Shape;
// ============================================================================
// Optimized parallel topk for CUDA
// Uses a dedicated kernel that's much faster than full sort for small k
// Single kernel call writes both values and indices - no post-processing needed
// ============================================================================
#[cfg(feature = "cuda")]
fn topk_launched(err: i32, dims: &[usize], k: usize) -> Result<()> {
if err == 0 {
Ok(())
} else {
hanzo_ml::bail!("topk kernel failed with CUDA error {err} on {dims:?} (k = {k})")
}
}
#[cfg(feature = "cuda")]
#[allow(clippy::cast_possible_truncation)]
fn cuda_topk(input: &Tensor, k: usize) -> Result<TopKOutput> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::CudaStorageSlice;
use std::ffi::c_void;
// Full last-dim top-k over ALL rows (the kernel runs one block per row); MoE routers call this on
// per-token [tokens, experts] scores and need every token's experts, matching the CPU fallback.
let input = input.contiguous()?;
let dims = input.dims();
let ncols = *dims
.last()
.ok_or_else(|| hanzo_ml::Error::Msg("empty dims".to_string()))?;
if ncols == 0 || k == 0 || k > ncols {
hanzo_ml::bail!("topk of {k} over rows of width {ncols}");
}
let nrows = (input.elem_count() / ncols) as i32;
let ncols_i32 = ncols as i32;
let k_i32 = k as i32;
// Output shapes
let mut out_dims = dims.to_vec();
*out_dims.last_mut().unwrap() = k;
let out_elem_count = nrows as usize * k;
let (storage, layout) = input.storage_and_layout();
let storage = match &*storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_topk requires CUDA tensor"),
};
let dev = storage.device();
let stream = dev.cuda_stream();
let stream_raw = stream.cu_stream() as i64;
let (src_ptr, src_guard) = match &storage.slice {
CudaStorageSlice::BF16(inp) => inp.device_ptr(inp.stream()),
CudaStorageSlice::F16(inp) => inp.device_ptr(inp.stream()),
CudaStorageSlice::F32(inp) => inp.device_ptr(inp.stream()),
_ => hanzo_ml::bail!("cuda_topk only supports BF16/F16/F32"),
};
let byte_offset = layout.start_offset() * input.dtype().size_in_bytes();
let src_ptr = unsafe { (src_ptr as *const u8).add(byte_offset) as *const c_void };
// Allocate both output buffers
let mut indices_dst = unsafe { dev.alloc::<u32>(out_elem_count) }?;
let (indices_ptr, indices_guard) = indices_dst.device_ptr_mut(&stream);
// For wide vocabulary-scale rows (4K..=256K cols, small batch, k <= 128), split across
// multi-block 4096-col chunks so Stage 1 saturates all SMs with register-held values.
let use_multiblock = (4096..=262_144).contains(&ncols) && k <= 128 && nrows > 0 && nrows <= 128;
let num_chunks = ncols.div_ceil(4096);
let mut mb_scratch = if use_multiblock {
let s1_count = (nrows as usize) * num_chunks * k;
let s1_vals = unsafe { dev.alloc::<f32>(s1_count) }?;
let s1_idxs = unsafe { dev.alloc::<u32>(s1_count) }?;
Some((s1_vals, s1_idxs))
} else {
None
};
let (values_tensor, indices_tensor) = match input.dtype() {
DType::BF16 => {
let mut values_dst = unsafe { dev.alloc::<half::bf16>(out_elem_count) }?;
let (values_ptr, values_guard) = values_dst.device_ptr_mut(&stream);
let err = if let Some((ref mut s1_vals, ref mut s1_idxs)) = mb_scratch {
let (s1v_ptr, _s1v_g) = s1_vals.device_ptr_mut(&stream);
let (s1i_ptr, _s1i_g) = s1_idxs.device_ptr_mut(&stream);
unsafe {
ffi::topk_multiblock_bf16(
src_ptr,
s1v_ptr as *mut c_void,
s1i_ptr as *mut c_void,
values_ptr as *mut c_void,
indices_ptr as *mut c_void,
nrows,
ncols_i32,
k_i32,
stream_raw,
)
}
} else {
unsafe {
ffi::topk_bf16(
src_ptr,
values_ptr as *mut c_void,
indices_ptr as *mut c_void,
nrows,
ncols_i32,
k_i32,
stream_raw,
)
}
};
topk_launched(err, dims, k)?;
drop(values_guard);
drop(indices_guard);
let values_storage = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::BF16(values_dst),
device: dev.clone(),
};
let indices_storage = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::U32(indices_dst),
device: dev.clone(),
};
let values_tensor = Tensor::from((
hanzo_ml::Storage::Cuda(values_storage),
Shape::from_dims(&out_dims),
));
let indices_tensor = Tensor::from((
hanzo_ml::Storage::Cuda(indices_storage),
Shape::from_dims(&out_dims),
));
(values_tensor, indices_tensor)
}
DType::F16 => {
let mut values_dst = unsafe { dev.alloc::<half::f16>(out_elem_count) }?;
let (values_ptr, values_guard) = values_dst.device_ptr_mut(&stream);
let err = if let Some((ref mut s1_vals, ref mut s1_idxs)) = mb_scratch {
let (s1v_ptr, _s1v_g) = s1_vals.device_ptr_mut(&stream);
let (s1i_ptr, _s1i_g) = s1_idxs.device_ptr_mut(&stream);
unsafe {
ffi::topk_multiblock_f16(
src_ptr,
s1v_ptr as *mut c_void,
s1i_ptr as *mut c_void,
values_ptr as *mut c_void,
indices_ptr as *mut c_void,
nrows,
ncols_i32,
k_i32,
stream_raw,
)
}
} else {
unsafe {
ffi::topk_f16(
src_ptr,
values_ptr as *mut c_void,
indices_ptr as *mut c_void,
nrows,
ncols_i32,
k_i32,
stream_raw,
)
}
};
topk_launched(err, dims, k)?;
drop(values_guard);
drop(indices_guard);
let values_storage = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F16(values_dst),
device: dev.clone(),
};
let indices_storage = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::U32(indices_dst),
device: dev.clone(),
};
let values_tensor = Tensor::from((
hanzo_ml::Storage::Cuda(values_storage),
Shape::from_dims(&out_dims),
));
let indices_tensor = Tensor::from((
hanzo_ml::Storage::Cuda(indices_storage),
Shape::from_dims(&out_dims),
));
(values_tensor, indices_tensor)
}
DType::F32 => {
let mut values_dst = unsafe { dev.alloc::<f32>(out_elem_count) }?;
let (values_ptr, values_guard) = values_dst.device_ptr_mut(&stream);
let err = if let Some((ref mut s1_vals, ref mut s1_idxs)) = mb_scratch {
let (s1v_ptr, _s1v_g) = s1_vals.device_ptr_mut(&stream);
let (s1i_ptr, _s1i_g) = s1_idxs.device_ptr_mut(&stream);
unsafe {
ffi::topk_multiblock_f32(
src_ptr,
s1v_ptr as *mut c_void,
s1i_ptr as *mut c_void,
values_ptr as *mut c_void,
indices_ptr as *mut c_void,
nrows,
ncols_i32,
k_i32,
stream_raw,
)
}
} else {
unsafe {
ffi::topk_f32(
src_ptr,
values_ptr as *mut c_void,
indices_ptr as *mut c_void,
nrows,
ncols_i32,
k_i32,
stream_raw,
)
}
};
topk_launched(err, dims, k)?;
drop(values_guard);
drop(indices_guard);
let values_storage = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F32(values_dst),
device: dev.clone(),
};
let indices_storage = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::U32(indices_dst),
device: dev.clone(),
};
let values_tensor = Tensor::from((
hanzo_ml::Storage::Cuda(values_storage),
Shape::from_dims(&out_dims),
));
let indices_tensor = Tensor::from((
hanzo_ml::Storage::Cuda(indices_storage),
Shape::from_dims(&out_dims),
));
(values_tensor, indices_tensor)
}
dt => hanzo_ml::bail!("cuda_topk unsupported dtype: {:?}", dt),
};
drop(src_guard);
Ok(TopKOutput {
values: values_tensor,
indices: indices_tensor,
})
}
#[derive(Debug, Clone, Copy)]
#[allow(dead_code)]
pub enum MoeRouterScoreFunction {
Raw,
Softmax,
Sigmoid,
}
#[derive(Debug, Clone, Copy)]
#[allow(dead_code)]
pub enum MoeRouterSelectedWeight {
Score,
Sigmoid,
}
#[derive(Debug, Clone, Copy)]
pub struct MoeRouterTopKConfig {
pub top_k: usize,
pub score_function: MoeRouterScoreFunction,
pub selected_weight: MoeRouterSelectedWeight,
pub renormalize: bool,
pub norm_min: f32,
pub output_scale: f32,
pub logit_clip: Option<(f32, f32)>,
}
/// Deterministic top-k expert selection with a CANONICAL, backend-independent tie-break: order
/// each row by (score DESC, expert-index ASC) using f32 total ordering (NaN-safe), take the first
/// `k`. Unlike hanzo-ml's `.topk()` (whose tie-break is unspecified) and the separate CUDA bf16/f16
/// router kernel, this yields the SAME experts on every machine — the routing analog of the
/// lowest-id argmax tie-break the sampler now uses. `top_k <= num_experts` for every MoE config.
#[allow(clippy::cast_possible_truncation)]
fn deterministic_topk_indices(scores: &Tensor, k: usize) -> Result<Tensor> {
let rows = scores.to_vec2::<f32>()?;
let device = scores.device();
let mut out: Vec<u32> = Vec::with_capacity(rows.len() * k);
for row in &rows {
let mut idx: Vec<u32> = (0..row.len() as u32).collect();
idx.sort_by(|&a, &b| row[b as usize].total_cmp(&row[a as usize]).then(a.cmp(&b)));
out.extend_from_slice(&idx[..k]);
}
Tensor::from_vec(out, (rows.len(), k), device)
}
pub fn moe_router_topk(
logits: &Tensor,
config: MoeRouterTopKConfig,
selection_bias: Option<&Tensor>,
expert_scale: Option<&Tensor>,
) -> Result<TopKOutput> {
// While a poi proof transcript is being emitted (inside `poi_forward::prove`), routing takes the
// single canonical CPU path so a prover and a re-executing verifier on different hardware select
// the IDENTICAL experts (a different expert is a different sub-network). Off during normal
// inference: `proving()` is false, so the fast CUDA/hanzo-ml path is byte-for-byte unchanged.
let proof = crate::poi_forward::proving();
#[cfg(feature = "cuda")]
if !proof
&& logits.device().is_cuda()
&& logits
.dims()
.last()
.is_some_and(|&e| crate::cuda::route::serves(e, config.top_k, &config))
{
return crate::cuda::route::topk(logits, config, selection_bias, expert_scale);
}
let logits = logits.to_dtype(DType::F32)?;
let logits = match config.logit_clip {
Some((min, max)) => logits.clamp(min as f64, max as f64)?,
None => logits,
};
let scores = match config.score_function {
MoeRouterScoreFunction::Raw => logits.clone(),
MoeRouterScoreFunction::Softmax => hanzo_nn::ops::softmax_last_dim(&logits)?,
MoeRouterScoreFunction::Sigmoid => hanzo_nn::ops::sigmoid(&logits)?,
};
let selection_scores = if let Some(selection_bias) = selection_bias {
scores.broadcast_add(&selection_bias.to_dtype(DType::F32)?)?
} else {
scores.clone()
};
let indices = if proof {
// canonical, backend-independent expert selection (Proof-of-AI)
deterministic_topk_indices(&selection_scores, config.top_k)?
} else {
selection_scores
.topk(config.top_k)?
.indices
.to_dtype(DType::U32)?
};
let mut values = match config.selected_weight {
MoeRouterSelectedWeight::Score => scores.gather(&indices, D::Minus1)?,
MoeRouterSelectedWeight::Sigmoid => {
hanzo_nn::ops::sigmoid(&logits.gather(&indices, D::Minus1)?)?
}
};
if config.renormalize {
let denominator = values.sum_keepdim(D::Minus1)?;
let denominator = if config.norm_min > 0.0 {
let min = Tensor::full(config.norm_min, denominator.shape(), denominator.device())?;
denominator.broadcast_maximum(&min)?
} else {
denominator
};
values = values.broadcast_div(&denominator)?;
}
if config.output_scale != 1.0 {
values = (values * config.output_scale as f64)?;
}
if let Some(expert_scale) = expert_scale {
let scales = expert_scale
.to_dtype(DType::F32)?
.index_select(&indices.flatten_all()?, 0)?
.reshape(indices.shape())?;
values = (values * scales)?;
}
Ok(TopKOutput { values, indices })
}
#[cfg(feature = "cuda")]
#[allow(dead_code)]
#[allow(clippy::cast_possible_truncation)]
pub fn cuda_topk_logits_f32(
input: &Tensor,
k: usize,
temperature: f64,
) -> Result<TopKLogitsOutput> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::CudaStorageSlice;
const MAX_K: usize = 128;
const CHUNK_SIZE: usize = 2048;
const MAX_STAGE2_CANDIDATES: usize = 48 * 1024;
if temperature <= 0.0 || !temperature.is_finite() {
hanzo_ml::bail!("cuda_topk_logits_f32 requires a positive finite temperature");
}
let input = input.contiguous()?;
if input.dtype() != DType::F32 {
hanzo_ml::bail!("cuda_topk_logits_f32 requires F32 logits");
}
let ncols = input.elem_count();
if ncols == 0 {
hanzo_ml::bail!("cuda_topk_logits_f32 got empty logits");
}
let k = k.min(ncols);
if k == 0 || k > MAX_K {
hanzo_ml::bail!("cuda_topk_logits_f32 k={} must be in [1, {}]", k, MAX_K);
}
let nblocks = ncols.div_ceil(CHUNK_SIZE);
let stage2_candidates = nblocks * k;
if stage2_candidates > MAX_STAGE2_CANDIDATES {
hanzo_ml::bail!(
"cuda_topk_logits_f32 workspace too large: {} candidates",
stage2_candidates
);
}
let (storage, layout) = input.storage_and_layout();
let storage = match &*storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_topk_logits_f32 requires CUDA tensor"),
};
let dev = storage.device();
let stream = dev.cuda_stream();
let stream_raw = stream.cu_stream() as i64;
let (src_ptr, src_guard) = match &storage.slice {
CudaStorageSlice::F32(inp) => inp.device_ptr(inp.stream()),
_ => hanzo_ml::bail!("cuda_topk_logits_f32 only supports F32"),
};
let src_ptr = unsafe { (src_ptr as *const f32).add(layout.start_offset()) };
let workspace_elems = nblocks * k;
let mut block_values = unsafe { dev.alloc::<f32>(workspace_elems) }?;
let mut block_indices = unsafe { dev.alloc::<u32>(workspace_elems) }?;
let mut block_maxes = unsafe { dev.alloc::<f32>(nblocks) }?;
let mut block_sums = unsafe { dev.alloc::<f32>(nblocks) }?;
let mut values_dst = unsafe { dev.alloc::<f32>(k) }?;
let mut indices_dst = unsafe { dev.alloc::<u32>(k) }?;
let mut softmax_info_dst = unsafe { dev.alloc::<f32>(2) }?;
let (block_values_ptr, block_values_guard) = block_values.device_ptr_mut(&stream);
let (block_indices_ptr, block_indices_guard) = block_indices.device_ptr_mut(&stream);
let (block_maxes_ptr, block_maxes_guard) = block_maxes.device_ptr_mut(&stream);
let (block_sums_ptr, block_sums_guard) = block_sums.device_ptr_mut(&stream);
let (values_ptr, values_guard) = values_dst.device_ptr_mut(&stream);
let (indices_ptr, indices_guard) = indices_dst.device_ptr_mut(&stream);
let (softmax_info_ptr, softmax_info_guard) = softmax_info_dst.device_ptr_mut(&stream);
unsafe {
ffi::topk_large_f32(
src_ptr,
block_values_ptr as *mut f32,
block_indices_ptr as *mut u32,
block_maxes_ptr as *mut f32,
block_sums_ptr as *mut f32,
values_ptr as *mut f32,
indices_ptr as *mut u32,
softmax_info_ptr as *mut f32,
ncols as i32,
k as i32,
CHUNK_SIZE as i32,
nblocks as i32,
(1.0 / temperature) as f32,
stream_raw,
);
}
drop(src_guard);
drop(block_values_guard);
drop(block_indices_guard);
drop(block_maxes_guard);
drop(block_sums_guard);
drop(values_guard);
drop(indices_guard);
drop(softmax_info_guard);
let values_storage = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F32(values_dst),
device: dev.clone(),
};
let indices_storage = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::U32(indices_dst),
device: dev.clone(),
};
let softmax_info_storage = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F32(softmax_info_dst),
device: dev.clone(),
};
let workspace = vec![
Tensor::from((
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F32(block_values),
device: dev.clone(),
}),
Shape::from_dims(&[workspace_elems]),
)),
Tensor::from((
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::U32(block_indices),
device: dev.clone(),
}),
Shape::from_dims(&[workspace_elems]),
)),
Tensor::from((
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F32(block_maxes),
device: dev.clone(),
}),
Shape::from_dims(&[nblocks]),
)),
Tensor::from((
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F32(block_sums),
device: dev.clone(),
}),
Shape::from_dims(&[nblocks]),
)),
];
Ok(TopKLogitsOutput {
values: Tensor::from((
hanzo_ml::Storage::Cuda(values_storage),
Shape::from_dims(&[k]),
)),
indices: Tensor::from((
hanzo_ml::Storage::Cuda(indices_storage),
Shape::from_dims(&[k]),
)),
softmax_info: Tensor::from((
hanzo_ml::Storage::Cuda(softmax_info_storage),
Shape::from_dims(&[2]),
)),
_workspace: workspace,
})
}
#[cfg(feature = "cuda")]
#[allow(clippy::cast_possible_truncation)]
pub fn cuda_topk_logits_f32_packed(
input: &Tensor,
k: usize,
temperature: f64,
) -> Result<TopKLogitsPackedOutput> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::CudaStorageSlice;
const MAX_K: usize = 128;
const CHUNK_SIZE: usize = 2048;
const MAX_STAGE2_CANDIDATES: usize = 48 * 1024;
if temperature <= 0.0 || !temperature.is_finite() {
hanzo_ml::bail!("cuda_topk_logits_f32_packed requires a positive finite temperature");
}
let input = input.contiguous()?;
if input.dtype() != DType::F32 {
hanzo_ml::bail!("cuda_topk_logits_f32_packed requires F32 logits");
}
let ncols = input.elem_count();
if ncols == 0 {
hanzo_ml::bail!("cuda_topk_logits_f32_packed got empty logits");
}
let k = k.min(ncols);
if k == 0 || k > MAX_K {
hanzo_ml::bail!(
"cuda_topk_logits_f32_packed k={} must be in [1, {}]",
k,
MAX_K
);
}
let nblocks = ncols.div_ceil(CHUNK_SIZE);
let stage2_candidates = nblocks * k;
if stage2_candidates > MAX_STAGE2_CANDIDATES {
hanzo_ml::bail!(
"cuda_topk_logits_f32_packed workspace too large: {} candidates",
stage2_candidates
);
}
let (storage, layout) = input.storage_and_layout();
let storage = match &*storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_topk_logits_f32_packed requires CUDA tensor"),
};
let dev = storage.device();
let stream = dev.cuda_stream();
let stream_raw = stream.cu_stream() as i64;
let (src_ptr, src_guard) = match &storage.slice {
CudaStorageSlice::F32(inp) => inp.device_ptr(inp.stream()),
_ => hanzo_ml::bail!("cuda_topk_logits_f32_packed only supports F32"),
};
let src_ptr = unsafe { (src_ptr as *const f32).add(layout.start_offset()) };
let workspace_elems = nblocks * k;
let mut block_values = unsafe { dev.alloc::<f32>(workspace_elems) }?;
let mut block_indices = unsafe { dev.alloc::<u32>(workspace_elems) }?;
let mut block_maxes = unsafe { dev.alloc::<f32>(nblocks) }?;
let mut block_sums = unsafe { dev.alloc::<f32>(nblocks) }?;
let mut packed_dst = unsafe { dev.alloc::<f32>(2 * k + 2) }?;
let (block_values_ptr, block_values_guard) = block_values.device_ptr_mut(&stream);
let (block_indices_ptr, block_indices_guard) = block_indices.device_ptr_mut(&stream);
let (block_maxes_ptr, block_maxes_guard) = block_maxes.device_ptr_mut(&stream);
let (block_sums_ptr, block_sums_guard) = block_sums.device_ptr_mut(&stream);
let (packed_ptr, packed_guard) = packed_dst.device_ptr_mut(&stream);
unsafe {
ffi::topk_large_f32_packed(
src_ptr,
block_values_ptr as *mut f32,
block_indices_ptr as *mut u32,
block_maxes_ptr as *mut f32,
block_sums_ptr as *mut f32,
packed_ptr as *mut f32,
ncols as i32,
k as i32,
CHUNK_SIZE as i32,
nblocks as i32,
(1.0 / temperature) as f32,
stream_raw,
);
}
drop(src_guard);
drop(block_values_guard);
drop(block_indices_guard);
drop(block_maxes_guard);
drop(block_sums_guard);
drop(packed_guard);
let packed_storage = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F32(packed_dst),
device: dev.clone(),
};
let workspace = vec![
Tensor::from((
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F32(block_values),
device: dev.clone(),
}),
Shape::from_dims(&[workspace_elems]),
)),
Tensor::from((
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::U32(block_indices),
device: dev.clone(),
}),
Shape::from_dims(&[workspace_elems]),
)),
Tensor::from((
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F32(block_maxes),
device: dev.clone(),
}),
Shape::from_dims(&[nblocks]),
)),
Tensor::from((
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::F32(block_sums),
device: dev.clone(),
}),
Shape::from_dims(&[nblocks]),
)),
];
Ok(TopKLogitsPackedOutput {
packed: Tensor::from((
hanzo_ml::Storage::Cuda(packed_storage),
Shape::from_dims(&[2 * k + 2]),
)),
k,
_workspace: workspace,
})
}
#[cfg(feature = "cuda")]
pub struct CudaTop1LogitsWorkspace {
ncols: usize,
nblocks: usize,
location: hanzo_ml::DeviceLocation,
block_values: hanzo_ml::cuda_backend::cudarc::driver::CudaSlice<f32>,
block_indices: hanzo_ml::cuda_backend::cudarc::driver::CudaSlice<u32>,
packed: hanzo_ml::cuda_backend::cudarc::driver::CudaSlice<f32>,
}
#[cfg(feature = "cuda")]
#[allow(clippy::cast_possible_truncation)]
pub fn cuda_topk_softmax(input: &Tensor, k: usize) -> Result<TopKOutput> {
// The topk_softmax CUDA kernels (topk_softmax_{bf16,f16,f32}) are unimplemented on this
// branch (no .cu entry point, no FFI decl). This helper is only reachable from the gpt-oss
// MoE router, which the speech path never exercises; bail rather than call phantom symbols.
let _ = (input, k);
hanzo_ml::bail!("cuda_topk_softmax is not implemented on this build")
}
/// Cached single-pass top-1 over F32 logits, returning the packed [logit, index] pair.
/// Reuses a per-device workspace across calls. Used by the CUDA top_k==1 sampler fast path.
#[cfg(feature = "cuda")]
#[allow(clippy::cast_possible_truncation)]
pub fn cuda_top1_logits_f32_cached(
input: &Tensor,
cache: &mut Option<CudaTop1LogitsWorkspace>,
) -> Result<[f32; 2]> {
use hanzo_ml::backend::{BackendDevice, BackendStorage};
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::CudaStorageSlice;
const CHUNK_SIZE: usize = 2048;
let input = input.contiguous()?;
if input.dtype() != DType::F32 {
hanzo_ml::bail!("cuda_top1_logits_f32_cached requires F32 logits");
}
let ncols = input.elem_count();
if ncols == 0 {
hanzo_ml::bail!("cuda_top1_logits_f32_cached got empty logits");
}
let nblocks = ncols.div_ceil(CHUNK_SIZE);
let (storage, layout) = input.storage_and_layout();
let storage = match &*storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_top1_logits_f32_cached requires CUDA tensor"),
};
let dev = storage.device();
let location = dev.location();
let stream = dev.cuda_stream();
let stream_raw = stream.cu_stream() as i64;
let needs_alloc = cache.as_ref().is_none_or(|workspace| {
workspace.ncols != ncols || workspace.nblocks != nblocks || workspace.location != location
});
if needs_alloc {
*cache = Some(CudaTop1LogitsWorkspace {
ncols,
nblocks,
location,
block_values: unsafe { dev.alloc::<f32>(nblocks) }?,
block_indices: unsafe { dev.alloc::<u32>(nblocks) }?,
packed: unsafe { dev.alloc::<f32>(2) }?,
});
}
let (src_ptr, src_guard) = match &storage.slice {
CudaStorageSlice::F32(inp) => inp.device_ptr(inp.stream()),
_ => hanzo_ml::bail!("cuda_top1_logits_f32_cached only supports F32"),
};
let src_ptr = unsafe { (src_ptr as *const f32).add(layout.start_offset()) };
let workspace = cache
.as_mut()
.expect("CUDA top-1 workspace was allocated above");
let (block_values_ptr, block_values_guard) = workspace.block_values.device_ptr_mut(&stream);
let (block_indices_ptr, block_indices_guard) = workspace.block_indices.device_ptr_mut(&stream);
let (packed_ptr, packed_guard) = workspace.packed.device_ptr_mut(&stream);
unsafe {
ffi::top1_large_f32_packed(
src_ptr,
block_values_ptr as *mut f32,
block_indices_ptr as *mut u32,
packed_ptr as *mut f32,
ncols as i32,
CHUNK_SIZE as i32,
nblocks as i32,
stream_raw,
);
}
drop(src_guard);
drop(block_values_guard);
drop(block_indices_guard);
drop(packed_guard);
dev.synchronize()?;
let packed = dev.clone_dtoh(&workspace.packed)?;
Ok([packed[0], packed[1]])
}
#[allow(dead_code)]
#[derive(Debug, Clone)]
struct ArgSort {
asc: bool,
last_dim: usize,
inplace: bool,
}
impl hanzo_ml::CustomOp1 for ArgSort {
fn name(&self) -> &'static str {
"argsort"
}
fn cpu_fwd(
&self,
_: &hanzo_ml::CpuStorage,
_: &hanzo_ml::Layout,
) -> Result<(hanzo_ml::CpuStorage, hanzo_ml::Shape)> {
panic!("not implemented!")
}
#[allow(clippy::cast_possible_truncation)]
#[cfg(feature = "cuda")]
fn cuda_fwd(
&self,
storage: &hanzo_ml::CudaStorage,
layout: &hanzo_ml::Layout,
) -> Result<(hanzo_ml::CudaStorage, hanzo_ml::Shape)> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::DevicePtr;
use hanzo_ml::cuda_backend::CudaStorageSlice;
let dev = storage.device();
let elem_count = layout.shape().elem_count();
let ncols = self.last_dim as i32;
let nrows = elem_count as i32 / ncols;
let dst = unsafe { dev.alloc::<u32>(elem_count) }?;
use std::ffi::c_void;
let (src, _src_guard) = match &storage.slice {
CudaStorageSlice::U8(inp) => inp.device_ptr(inp.stream()),
CudaStorageSlice::U32(inp) => inp.device_ptr(inp.stream()),
CudaStorageSlice::I64(inp) => inp.device_ptr(inp.stream()),
CudaStorageSlice::BF16(inp) => inp.device_ptr(inp.stream()),
CudaStorageSlice::F16(inp) => inp.device_ptr(inp.stream()),
CudaStorageSlice::F32(inp) => inp.device_ptr(inp.stream()),
CudaStorageSlice::F64(inp) => inp.device_ptr(inp.stream()),
_ => hanzo_ml::bail!("Unexpected dtype in asort"),
};
let src_ptr = src as *const c_void;
let (dst_ptr, dst_guard) = dst.device_ptr(dst.stream());
let dst_ptr = dst_ptr as *mut c_void;
let stream = dev.cuda_stream().cu_stream() as i64;
unsafe {
if self.asc {
match storage.dtype() {
hanzo_ml::DType::U8 => {
ffi::asort_asc_u8(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::U32 => {
ffi::asort_asc_u32(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::I64 => {
ffi::asort_asc_i64(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::BF16 => {
ffi::asort_asc_bf16(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::F16 => {
ffi::asort_asc_f16(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::F32 => {
ffi::asort_asc_f32(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::F64 => {
ffi::asort_asc_f64(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
_ => hanzo_ml::bail!("Unexpected dtype in asort"),
}
} else {
match storage.dtype() {
hanzo_ml::DType::U8 => {
ffi::asort_desc_u8(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::U32 => {
ffi::asort_desc_u32(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::I64 => {
ffi::asort_desc_i64(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::BF16 => {
ffi::asort_desc_bf16(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::F16 => {
ffi::asort_desc_f16(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::F32 => {
ffi::asort_desc_f32(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
hanzo_ml::DType::F64 => {
ffi::asort_desc_f64(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
}
_ => hanzo_ml::bail!("Unexpected dtype in asort"),
}
}
}
drop(dst_guard);
let dst_ret = hanzo_ml::cuda_backend::CudaStorage {
slice: CudaStorageSlice::U32(dst),
device: dev.clone(),
};
Ok((dst_ret, layout.shape().clone()))
}
}
#[allow(dead_code)]
pub trait ArgSortOp {
fn arg_sort(&self, asc: bool) -> Result<Tensor>;
fn sort(&self, asc: bool) -> Result<(Tensor, Tensor)>;
}
impl ArgSortOp for Tensor {
/// Returns the indices that sort the tensor along the last dimension.
///
/// If `asc` is `true`, sorting is in ascending order. Otherwise sorting is performed in
/// descending order. The sort is unstable so there is no guarantees on the final order when it
/// comes to ties.
fn arg_sort(&self, asc: bool) -> Result<Tensor> {
if !self.is_contiguous() {
return Err(hanzo_ml::Error::RequiresContiguous { op: "arg_sort" });
}
let last_dim = match self.dims().last() {
Some(last_dim) => *last_dim,
None => hanzo_ml::bail!("empty last-dim in arg-sort"),
};
// No need for a backward pass for arg sort.
self.apply_op1_no_bwd(&ArgSort {
asc,
last_dim,
inplace: false,
})
}
/// Sorts the tensor along the last dimension, returns the sorted tensor together with the
/// sorted indexes.
///
/// If `asc` is `true`, sorting is in ascending order. Otherwise sorting is performed in
/// descending order. The sort is unstable so there is no guarantees on the final order when it
/// comes to ties.
fn sort(&self, asc: bool) -> Result<(Tensor, Tensor)> {
if !self.is_contiguous() {
return Err(hanzo_ml::Error::RequiresContiguous { op: "arg_sort" });
}
let last_dim = match self.dims().last() {
Some(last_dim) => *last_dim,
None => hanzo_ml::bail!("empty last-dim in arg-sort"),
};
let sorted = self.copy()?;
let asort = sorted.apply_op1_no_bwd(&ArgSort {
asc,
last_dim,
inplace: true,
})?;
Ok((sorted, asort))
}
}
#[allow(dead_code)]
pub struct TopKOutput {
pub values: Tensor,
pub indices: Tensor,
}
#[allow(dead_code)]
pub struct TopKLogitsOutput {
pub values: Tensor,
pub indices: Tensor,
/// `[softmax_denominator, global_max]` for the full-vocabulary softmax at
/// the temperature used for top-k selection.
pub softmax_info: Tensor,
_workspace: Vec<Tensor>,
}
#[allow(dead_code)]
pub struct TopKLogitsPackedOutput {
/// Packed as `[values; indices_as_f32; softmax_denominator; global_max]`.
pub packed: Tensor,
pub k: usize,
_workspace: Vec<Tensor>,
}
#[cfg(feature = "cuda")]
pub fn cuda_softcap_f32(input: &Tensor, cap: f32) -> Result<Tensor> {
// The `softcap_f32` CUDA kernel is unimplemented on this branch (no .cu entry point, no FFI
// decl) and this helper has no callers; bail rather than reference a phantom symbol. The
// device-agnostic tanh-softcap path is available via `Tensor` ops where needed.
let _ = (input, cap);
hanzo_ml::bail!("cuda_softcap_f32 is not implemented on this build")
}
#[cfg(feature = "cuda")]
pub fn cuda_apply_sparse_penalties_f32(
input: &Tensor,
token_ids: &Tensor,
counts: &Tensor,
frequency_penalty: f32,
presence_penalty: f32,
repetition_penalty: f32,
) -> Result<Tensor> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
use std::ffi::c_void;
if input.dtype() != DType::F32 {
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires F32 logits");
}
if token_ids.dtype() != DType::U32 {
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires U32 token ids");
}
if counts.dtype() != DType::F32 {
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires F32 counts");
}
if token_ids.elem_count() != counts.elem_count() {
hanzo_ml::bail!(
"cuda_apply_sparse_penalties_f32 token ids/counts length mismatch: {} vs {}",
token_ids.elem_count(),
counts.elem_count()
);
}
if !token_ids.device().same_device(input.device())
|| !counts.device().same_device(input.device())
{
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 tensors must be on the same CUDA device");
}
let input = input.contiguous()?;
let token_ids = token_ids.contiguous()?;
let counts = counts.contiguous()?;
let elem_count = input.elem_count();
let n_tokens = token_ids.elem_count();
if elem_count == 0 {
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 got empty logits");
}
if elem_count > i32::MAX as usize {
hanzo_ml::bail!(
"cuda_apply_sparse_penalties_f32 input is too large: {elem_count} elements"
);
}
if n_tokens > i32::MAX as usize {
hanzo_ml::bail!(
"cuda_apply_sparse_penalties_f32 token list is too large: {n_tokens} elements"
);
}
let elem_count_i32 = i32::try_from(elem_count).map_err(hanzo_ml::Error::wrap)?;
let n_tokens_i32 = i32::try_from(n_tokens).map_err(hanzo_ml::Error::wrap)?;
let (input_storage, input_layout) = input.storage_and_layout();
let input_storage = match &*input_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires CUDA logits"),
};
let CudaStorageSlice::F32(src) = &input_storage.slice else {
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 only supports F32 logits");
};
let (token_storage, token_layout) = token_ids.storage_and_layout();
let token_storage = match &*token_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires CUDA token ids"),
};
let CudaStorageSlice::U32(token_src) = &token_storage.slice else {
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 only supports U32 token ids");
};
let (count_storage, count_layout) = counts.storage_and_layout();
let count_storage = match &*count_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires CUDA counts"),
};
let CudaStorageSlice::F32(count_src) = &count_storage.slice else {
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 only supports F32 counts");
};
let dev = input_storage.device();
let stream = dev.cuda_stream();
let mut out = unsafe { dev.alloc::<f32>(elem_count) }?;
let (src_ptr, src_guard) = src.device_ptr(&stream);
let (token_ptr, token_guard) = token_src.device_ptr(&stream);
let (count_ptr, count_guard) = count_src.device_ptr(&stream);
let (out_ptr, out_guard) = out.device_ptr_mut(&stream);
let src_ptr = unsafe { (src_ptr as *const f32).add(input_layout.start_offset()) };
let token_ptr = unsafe { (token_ptr as *const u32).add(token_layout.start_offset()) };
let count_ptr = unsafe { (count_ptr as *const f32).add(count_layout.start_offset()) };
unsafe {
ffi::apply_sparse_penalties_f32(
src_ptr as *const c_void,
out_ptr as *mut c_void,
token_ptr,
count_ptr,
elem_count_i32,
n_tokens_i32,
frequency_penalty,
presence_penalty,
repetition_penalty,
stream.cu_stream() as i64,
);
}
drop(src_guard);
drop(token_guard);
drop(count_guard);
drop(out_guard);
let out_storage = CudaStorage {
slice: CudaStorageSlice::F32(out),
device: dev.clone(),
};
Ok(Tensor::from((
hanzo_ml::Storage::Cuda(out_storage),
input.shape().clone(),
)))
}
#[cfg(feature = "metal")]
pub fn metal_apply_sparse_penalties(
input: &Tensor,
token_ids: &Tensor,
counts: &Tensor,
frequency_penalty: f32,
presence_penalty: f32,
repetition_penalty: f32,
) -> Result<Tensor> {
use hanzo_ml::{backend::BackendStorage, MetalStorage, Shape, Storage};
if !matches!(input.dtype(), DType::F32 | DType::F16 | DType::BF16) {
hanzo_ml::bail!("metal_apply_sparse_penalties requires F32/F16/BF16 logits");
}
if token_ids.dtype() != DType::U32 || counts.dtype() != DType::F32 {
hanzo_ml::bail!("metal_apply_sparse_penalties token_ids must be u32, counts f32");
}
let dtype = input.dtype();
let n = input.elem_count();
let n_tokens = token_ids.elem_count();
if counts.elem_count() != n_tokens {
hanzo_ml::bail!("token_ids and counts length mismatch");
}
let input = input.contiguous()?;
let token_ids = token_ids.contiguous()?;
let counts = counts.contiguous()?;
let (input_s, input_l) = input.storage_and_layout();
let (tok_s, tok_l) = token_ids.storage_and_layout();
let (cnt_s, cnt_l) = counts.storage_and_layout();
let (Storage::Metal(input_s), Storage::Metal(tok_s), Storage::Metal(cnt_s)) =
(&*input_s, &*tok_s, &*cnt_s)
else {
hanzo_ml::bail!("metal_apply_sparse_penalties requires Metal tensors");
};
let device = input_s.device().clone();
let out_buf = device.new_buffer(n, dtype, "penalties-out")?;
let encoder = device.command_encoder()?;
encoder.set_label("penalties-copy");
{
use hanzo_quant::metal_kernels::Kernels;
hanzo_quant::metal_kernels::call_copy_logits(
device.device(),
&encoder,
&Kernels::new(),
dtype,
input_s.buffer(),
input_l.start_offset() * input.dtype().size_in_bytes(),
&out_buf,
n,
)
.map_err(|e| hanzo_ml::Error::Msg(format!("metal copy: {e}")))?;
}
encoder.set_label("penalties-apply");
hanzo_quant::metal_kernels::call_apply_sparse_penalties(
device.device(),
&encoder,
&hanzo_quant::metal_kernels::Kernels::new(),
dtype,
&out_buf,
tok_s.buffer(),
cnt_s.buffer(),
n,
n_tokens,
frequency_penalty,
presence_penalty,
repetition_penalty,
)
.map_err(|e| hanzo_ml::Error::Msg(format!("metal penalties: {e}")))?;
let _ = (tok_l, cnt_l);
Ok(Tensor::from((
Storage::Metal(MetalStorage::new(out_buf, device.clone(), n, dtype)),
Shape::from(input.dims()),
)))
}
#[cfg(feature = "cuda")]
pub(crate) fn try_cuda_rms_norm_strided_4d(
input: &Tensor,
weight: &Tensor,
eps: f32,
) -> Result<Option<Tensor>> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
use std::ffi::c_void;
if !input.device().is_cuda() || input.rank() != 4 {
return Ok(None);
}
let dtype = input.dtype();
if !matches!(dtype, DType::BF16 | DType::F16 | DType::F32) || weight.dtype() != dtype {
return Ok(None);
}
if !weight.device().same_device(input.device()) {
return Ok(None);
}
let (batch, heads, seq_len, head_dim) = input.dims4()?;
if weight.dims1()? != head_dim {
hanzo_ml::bail!(
"cuda_rms_norm_strided_4d weight size {} does not match head dim {head_dim}",
weight.dims1()?
);
}
if input.elem_count() == 0 {
return Ok(None);
}
for (name, value) in [
("batch", batch),
("heads", heads),
("seq_len", seq_len),
("head_dim", head_dim),
] {
if value > i32::MAX as usize {
hanzo_ml::bail!("cuda_rms_norm_strided_4d {name} is too large: {value}");
}
}
let (input_storage, input_layout) = input.storage_and_layout();
if input_layout.is_contiguous() {
return Ok(None);
}
let input_storage = match &*input_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let weight = weight.contiguous()?;
let (weight_storage, weight_layout) = weight.storage_and_layout();
let weight_storage = match &*weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let dev = input_storage.device();
let stream = dev.cuda_stream();
let stream_ptr = stream.cu_stream() as i64;
let shape = input.shape().clone();
let elem_count = input.elem_count();
let stride = input_layout.stride();
let batch_i32 = i32::try_from(batch).map_err(hanzo_ml::Error::wrap)?;
let heads_i32 = i32::try_from(heads).map_err(hanzo_ml::Error::wrap)?;
let seq_len_i32 = i32::try_from(seq_len).map_err(hanzo_ml::Error::wrap)?;
let head_dim_i32 = i32::try_from(head_dim).map_err(hanzo_ml::Error::wrap)?;
macro_rules! launch {
($variant:ident, $ty:ty, $ffi_fn:ident) => {{
let CudaStorageSlice::$variant(src) = &input_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_strided_4d input dtype mismatch");
};
let CudaStorageSlice::$variant(weight_src) = &weight_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_strided_4d weight dtype mismatch");
};
let mut out = unsafe { dev.alloc::<$ty>(elem_count) }?;
let (src_ptr, src_guard) = src.device_ptr(&stream);
let (weight_ptr, weight_guard) = weight_src.device_ptr(&stream);
let (out_ptr, out_guard) = out.device_ptr_mut(&stream);
let src_ptr = unsafe { (src_ptr as *const $ty).add(input_layout.start_offset()) };
let weight_ptr =
unsafe { (weight_ptr as *const $ty).add(weight_layout.start_offset()) };
unsafe {
ffi::$ffi_fn(
src_ptr as *const c_void,
weight_ptr as *const c_void,
out_ptr as *mut c_void,
stride[0] as i64,
stride[1] as i64,
stride[2] as i64,
stride[3] as i64,
batch_i32,
heads_i32,
seq_len_i32,
head_dim_i32,
eps,
stream_ptr,
);
}
drop(src_guard);
drop(weight_guard);
drop(out_guard);
let out_storage = CudaStorage {
slice: CudaStorageSlice::$variant(out),
device: dev.clone(),
};
Ok(Some(Tensor::from((
hanzo_ml::Storage::Cuda(out_storage),
shape,
))))
}};
}
match dtype {
DType::BF16 => launch!(BF16, half::bf16, rms_norm_strided_4d_bf16),
DType::F16 => launch!(F16, half::f16, rms_norm_strided_4d_f16),
DType::F32 => launch!(F32, f32, rms_norm_strided_4d_f32),
_ => Ok(None),
}
}
#[cfg(feature = "cuda")]
pub fn cuda_rms_norm_residual(
input: &Tensor,
residual: &Tensor,
weight: &Tensor,
scale: Option<&Tensor>,
eps: f32,
) -> Result<Tensor> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
use std::ffi::c_void;
if input.shape() != residual.shape() {
hanzo_ml::bail!(
"cuda_rms_norm_residual input/residual shape mismatch: {:?} vs {:?}",
input.shape(),
residual.shape()
);
}
if input.dtype() != residual.dtype() || input.dtype() != weight.dtype() {
hanzo_ml::bail!(
"cuda_rms_norm_residual dtype mismatch: input {:?}, residual {:?}, weight {:?}",
input.dtype(),
residual.dtype(),
weight.dtype()
);
}
if !matches!(input.dtype(), DType::BF16 | DType::F16 | DType::F32) {
hanzo_ml::bail!(
"cuda_rms_norm_residual only supports BF16/F16/F32, got {:?}",
input.dtype()
);
}
if !residual.device().same_device(input.device())
|| !weight.device().same_device(input.device())
{
hanzo_ml::bail!("cuda_rms_norm_residual tensors must be on the same CUDA device");
}
if let Some(scale) = scale {
if scale.elem_count() != 1 {
hanzo_ml::bail!(
"cuda_rms_norm_residual scale must have one element, got {}",
scale.elem_count()
);
}
if scale.dtype() != input.dtype() {
hanzo_ml::bail!(
"cuda_rms_norm_residual scale dtype mismatch: input {:?}, scale {:?}",
input.dtype(),
scale.dtype()
);
}
if !scale.device().same_device(input.device()) {
hanzo_ml::bail!("cuda_rms_norm_residual scale must be on the same CUDA device");
}
}
let ncols = input.dim(D::Minus1)?;
if weight.dims1()? != ncols {
hanzo_ml::bail!(
"cuda_rms_norm_residual weight size {} does not match last dim {ncols}",
weight.dims1()?
);
}
let elem_count = input.elem_count();
if elem_count == 0 {
hanzo_ml::bail!("cuda_rms_norm_residual got empty input");
}
let nrows = elem_count / ncols;
if nrows > i32::MAX as usize || ncols > i32::MAX as usize {
hanzo_ml::bail!("cuda_rms_norm_residual input is too large: nrows={nrows}, ncols={ncols}");
}
let nrows_i32 = i32::try_from(nrows).map_err(hanzo_ml::Error::wrap)?;
let ncols_i32 = i32::try_from(ncols).map_err(hanzo_ml::Error::wrap)?;
let input = input.contiguous()?;
let residual = residual.contiguous()?;
let weight = weight.contiguous()?;
let scale = scale.map(Tensor::contiguous).transpose()?;
let (input_storage, input_layout) = input.storage_and_layout();
let input_storage = match &*input_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_rms_norm_residual requires CUDA input"),
};
let (residual_storage, residual_layout) = residual.storage_and_layout();
let residual_storage = match &*residual_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_rms_norm_residual requires CUDA residual"),
};
let (weight_storage, weight_layout) = weight.storage_and_layout();
let weight_storage = match &*weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_rms_norm_residual requires CUDA weight"),
};
let scale_storage_and_layout = scale.as_ref().map(|scale| scale.storage_and_layout());
let dev = input_storage.device();
let stream = dev.cuda_stream();
let stream_ptr = stream.cu_stream() as i64;
let shape = input.shape().clone();
macro_rules! launch {
($variant:ident, $ty:ty, $ffi_fn:ident) => {{
let CudaStorageSlice::$variant(src) = &input_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_residual input dtype mismatch");
};
let CudaStorageSlice::$variant(residual_src) = &residual_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_residual residual dtype mismatch");
};
let CudaStorageSlice::$variant(weight_src) = &weight_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_residual weight dtype mismatch");
};
let (scale_ptr, scale_guard) =
if let Some((scale_storage, scale_layout)) = &scale_storage_and_layout {
let scale_storage = match &**scale_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_rms_norm_residual requires CUDA scale"),
};
let CudaStorageSlice::$variant(scale_src) = &scale_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_residual scale dtype mismatch");
};
let (scale_ptr, scale_guard) = scale_src.device_ptr(&stream);
(
unsafe { (scale_ptr as *const $ty).add(scale_layout.start_offset()) }
as *const c_void,
Some(scale_guard),
)
} else {
(std::ptr::null(), None)
};
let mut out = unsafe { dev.alloc::<$ty>(elem_count) }?;
let (src_ptr, src_guard) = src.device_ptr(&stream);
let (residual_ptr, residual_guard) = residual_src.device_ptr(&stream);
let (weight_ptr, weight_guard) = weight_src.device_ptr(&stream);
let (out_ptr, out_guard) = out.device_ptr_mut(&stream);
let src_ptr = unsafe { (src_ptr as *const $ty).add(input_layout.start_offset()) };
let residual_ptr =
unsafe { (residual_ptr as *const $ty).add(residual_layout.start_offset()) };
let weight_ptr =
unsafe { (weight_ptr as *const $ty).add(weight_layout.start_offset()) };
unsafe {
ffi::$ffi_fn(
src_ptr as *const c_void,
residual_ptr as *const c_void,
weight_ptr as *const c_void,
scale_ptr,
out_ptr as *mut c_void,
nrows_i32,
ncols_i32,
eps,
stream_ptr,
);
}
drop(src_guard);
drop(residual_guard);
drop(weight_guard);
drop(scale_guard);
drop(out_guard);
let out_storage = CudaStorage {
slice: CudaStorageSlice::$variant(out),
device: dev.clone(),
};
Ok(Tensor::from((hanzo_ml::Storage::Cuda(out_storage), shape)))
}};
}
match input.dtype() {
DType::BF16 => launch!(BF16, half::bf16, rms_norm_residual_bf16),
DType::F16 => launch!(F16, half::f16, rms_norm_residual_f16),
DType::F32 => launch!(F32, f32, rms_norm_residual_f32),
dtype => hanzo_ml::bail!("cuda_rms_norm_residual unsupported dtype {dtype:?}"),
}
}
/// Fused residual-add + rmsnorm on CUDA: returns Some((sum = input + residual, normed =
/// rmsnorm(sum)*weight)) in one launch, or None when the fused kernel does not cover the case (caller
/// falls back to a separate add + rms_norm). The mean is taken over the stored sum, so the result is
/// bit-identical to the fallback. The pre-norm-transformer pattern; the CUDA twin of rocm_rms_norm_of_sum.
#[cfg(feature = "cuda")]
pub fn cuda_rms_norm_of_sum(
input: &Tensor,
residual: &Tensor,
weight: &Tensor,
eps: f32,
) -> Result<Option<(Tensor, Tensor)>> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
use std::ffi::c_void;
if !matches!(input.dtype(), DType::BF16 | DType::F16 | DType::F32) {
return Ok(None);
}
if input.dtype() != residual.dtype() || input.dtype() != weight.dtype() {
return Ok(None);
}
if !input.device().is_cuda()
|| !residual.device().same_device(input.device())
|| !weight.device().same_device(input.device())
{
return Ok(None);
}
if input.shape() != residual.shape() {
hanzo_ml::bail!(
"cuda_rms_norm_of_sum input/residual shape mismatch: {:?} vs {:?}",
input.shape(),
residual.shape()
);
}
let ncols = input.dim(D::Minus1)?;
if weight.dims1()? != ncols {
hanzo_ml::bail!(
"cuda_rms_norm_of_sum weight size {} does not match last dim {ncols}",
weight.dims1()?
);
}
let elem_count = input.elem_count();
if elem_count == 0 {
hanzo_ml::bail!("cuda_rms_norm_of_sum got empty input");
}
let nrows = elem_count / ncols;
if nrows > i32::MAX as usize || ncols > i32::MAX as usize {
hanzo_ml::bail!("cuda_rms_norm_of_sum input too large: nrows={nrows}, ncols={ncols}");
}
let nrows_i32 = i32::try_from(nrows).map_err(hanzo_ml::Error::wrap)?;
let ncols_i32 = i32::try_from(ncols).map_err(hanzo_ml::Error::wrap)?;
let input = input.contiguous()?;
let residual = residual.contiguous()?;
let weight = weight.contiguous()?;
let (input_storage, input_layout) = input.storage_and_layout();
let input_storage = match &*input_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (residual_storage, residual_layout) = residual.storage_and_layout();
let residual_storage = match &*residual_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (weight_storage, weight_layout) = weight.storage_and_layout();
let weight_storage = match &*weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let dev = input_storage.device();
let stream = dev.cuda_stream();
let stream_ptr = stream.cu_stream() as i64;
let shape = input.shape().clone();
macro_rules! launch {
($variant:ident, $ty:ty, $ffi_fn:ident) => {{
let CudaStorageSlice::$variant(src) = &input_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_of_sum input dtype mismatch");
};
let CudaStorageSlice::$variant(residual_src) = &residual_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_of_sum residual dtype mismatch");
};
let CudaStorageSlice::$variant(weight_src) = &weight_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_of_sum weight dtype mismatch");
};
let mut sum_out = unsafe { dev.alloc::<$ty>(elem_count) }?;
let mut norm_out = unsafe { dev.alloc::<$ty>(elem_count) }?;
let (src_ptr, src_guard) = src.device_ptr(&stream);
let (residual_ptr, residual_guard) = residual_src.device_ptr(&stream);
let (weight_ptr, weight_guard) = weight_src.device_ptr(&stream);
let (sum_out_ptr, sum_out_guard) = sum_out.device_ptr_mut(&stream);
let (norm_out_ptr, norm_out_guard) = norm_out.device_ptr_mut(&stream);
let src_ptr = unsafe { (src_ptr as *const $ty).add(input_layout.start_offset()) };
let residual_ptr =
unsafe { (residual_ptr as *const $ty).add(residual_layout.start_offset()) };
let weight_ptr =
unsafe { (weight_ptr as *const $ty).add(weight_layout.start_offset()) };
unsafe {
ffi::$ffi_fn(
src_ptr as *const c_void,
residual_ptr as *const c_void,
weight_ptr as *const c_void,
sum_out_ptr as *mut c_void,
norm_out_ptr as *mut c_void,
nrows_i32,
ncols_i32,
eps,
stream_ptr,
);
}
drop(src_guard);
drop(residual_guard);
drop(weight_guard);
drop(sum_out_guard);
drop(norm_out_guard);
let sum_storage = CudaStorage {
slice: CudaStorageSlice::$variant(sum_out),
device: dev.clone(),
};
let norm_storage = CudaStorage {
slice: CudaStorageSlice::$variant(norm_out),
device: dev.clone(),
};
Ok(Some((
Tensor::from((hanzo_ml::Storage::Cuda(sum_storage), shape.clone())),
Tensor::from((hanzo_ml::Storage::Cuda(norm_storage), shape)),
)))
}};
}
match input.dtype() {
DType::BF16 => launch!(BF16, half::bf16, rms_norm_of_sum_bf16),
DType::F16 => launch!(F16, half::f16, rms_norm_of_sum_f16),
DType::F32 => launch!(F32, f32, rms_norm_of_sum_f32),
dtype => hanzo_ml::bail!("cuda_rms_norm_of_sum unsupported dtype {dtype:?}"),
}
}
/// Fused residual-add + rmsnorm: returns Some((sum = input + residual, normed = rmsnorm(sum)*weight))
/// in one ROCm launch, or None (caller falls back to a separate add + rms_norm) for any case the fused
/// kernel does not cover. Covers F32 (prefill/router) and F16 (the decode residual stream). Distinct
/// from `cuda_rms_norm_residual` (which is `residual + rmsnorm(x)`): this normalizes the SUM, the
/// pre-norm-transformer pattern.
#[cfg(feature = "rocm")]
pub fn rocm_rms_norm_of_sum(
input: &Tensor,
residual: &Tensor,
weight: &Tensor,
eps: f32,
) -> Result<Option<(Tensor, Tensor)>> {
if !matches!(input.dtype(), DType::F32 | DType::F16)
|| residual.dtype() != input.dtype()
|| weight.dtype() != input.dtype()
{
return Ok(None);
}
if !input.device().is_rocm()
|| !residual.device().same_device(input.device())
|| !weight.device().same_device(input.device())
{
return Ok(None);
}
if input.shape() != residual.shape() {
hanzo_ml::bail!(
"rocm_rms_norm_of_sum input/residual shape mismatch: {:?} vs {:?}",
input.shape(),
residual.shape()
);
}
let input = input.contiguous()?;
let residual = residual.contiguous()?;
let weight = weight.contiguous()?;
let (xs, xl) = input.storage_and_layout();
let xr = match &*xs {
hanzo_ml::Storage::Rocm(r) => r,
_ => return Ok(None),
};
let (rs, rl) = residual.storage_and_layout();
let rr = match &*rs {
hanzo_ml::Storage::Rocm(r) => r,
_ => return Ok(None),
};
let (ws, wl) = weight.storage_and_layout();
let wr = match &*ws {
hanzo_ml::Storage::Rocm(r) => r,
_ => return Ok(None),
};
let (sum, normed) = xr.add_rms_norm(xl, rr, rl, wr, wl, eps)?;
let shape = input.shape().clone();
let sum_t = Tensor::from((hanzo_ml::Storage::Rocm(sum), shape.clone()));
let normed_t = Tensor::from((hanzo_ml::Storage::Rocm(normed), shape));
Ok(Some((sum_t, normed_t)))
}
/// Fused (input + residual) then rmsnorm on Vulkan: returns (sum, normed) where sum = input + residual
/// (the new residual-stream value) and normed = rms_norm(sum) * weight, in one dispatch (the DSL
/// add_rmsnorm_blk kernel) with no barrier between the add and the norm. Bit-identical to a separate add
/// + rms_norm. Returns None (caller falls back) unless every tensor is F32 and Vulkan on one device --
/// the block kernel is f32. Mirrors [`rocm_rms_norm_of_sum`].
#[cfg(feature = "vulkan")]
pub fn vulkan_rms_norm_of_sum(
input: &Tensor,
residual: &Tensor,
weight: &Tensor,
eps: f32,
) -> Result<Option<(Tensor, Tensor)>> {
if input.dtype() != DType::F32 || residual.dtype() != DType::F32 || weight.dtype() != DType::F32
{
return Ok(None);
}
if !input.device().is_vulkan()
|| !residual.device().same_device(input.device())
|| !weight.device().same_device(input.device())
{
return Ok(None);
}
if input.shape() != residual.shape() {
hanzo_ml::bail!(
"vulkan_rms_norm_of_sum input/residual shape mismatch: {:?} vs {:?}",
input.shape(),
residual.shape()
);
}
let input = input.contiguous()?;
let residual = residual.contiguous()?;
let weight = weight.contiguous()?;
let (xs, xl) = input.storage_and_layout();
let xr = match &*xs {
hanzo_ml::Storage::Vulkan(v) => v,
_ => return Ok(None),
};
let (rs, rl) = residual.storage_and_layout();
let rr = match &*rs {
hanzo_ml::Storage::Vulkan(v) => v,
_ => return Ok(None),
};
let (ws, wl) = weight.storage_and_layout();
let wr = match &*ws {
hanzo_ml::Storage::Vulkan(v) => v,
_ => return Ok(None),
};
let (sum, normed) = xr.add_rmsnorm(xl, rr, rl, wr, wl, eps)?;
let shape = input.shape().clone();
let sum_t = Tensor::from((hanzo_ml::Storage::Vulkan(sum), shape.clone()));
let normed_t = Tensor::from((hanzo_ml::Storage::Vulkan(normed), shape));
Ok(Some((sum_t, normed_t)))
}
#[cfg(feature = "metal")]
pub fn metal_rms_norm_residual(
input: &Tensor,
residual: &Tensor,
weight: &Tensor,
scale: Option<&Tensor>,
eps: f32,
) -> Result<Option<Tensor>> {
use hanzo_ml::{backend::BackendStorage, MetalStorage, Shape, Storage};
if input.shape() != residual.shape() {
return Ok(None);
}
let n_cols = input.dim(D::Minus1)?;
if weight.dims1()? != n_cols {
return Ok(None);
}
let n_rows = input.elem_count() / n_cols;
if n_rows == 0 {
return Ok(None);
}
if let Some(scale) = scale {
if scale.elem_count() != 1 {
return Ok(None);
}
}
let input = input.contiguous()?;
let residual = residual.contiguous()?;
let weight = weight.contiguous()?;
let scale_t = scale.map(Tensor::contiguous).transpose()?;
let (input_storage, input_layout) = input.storage_and_layout();
let Storage::Metal(input_storage) = &*input_storage else {
return Ok(None);
};
let (residual_storage, residual_layout) = residual.storage_and_layout();
let Storage::Metal(residual_storage) = &*residual_storage else {
return Ok(None);
};
let (weight_storage, weight_layout) = weight.storage_and_layout();
let Storage::Metal(weight_storage) = &*weight_storage else {
return Ok(None);
};
let scale_storage_and_layout = scale_t.as_ref().map(|s| s.storage_and_layout());
let scale_metal = match scale_storage_and_layout.as_ref() {
Some((s, l)) => {
let Storage::Metal(s) = &**s else {
return Ok(None);
};
Some((s, l))
}
None => None,
};
let device = input_storage.device().clone();
let dtype = input.dtype();
let out_buf = device.new_buffer(input.elem_count(), dtype, "rmsnorm-residual-out")?;
let encoder = device.command_encoder()?;
encoder.set_label("rmsnorm-residual");
let x_offset = input_layout.start_offset() * dtype.size_in_bytes();
let res_offset = residual_layout.start_offset() * dtype.size_in_bytes();
let w_offset = weight_layout.start_offset() * dtype.size_in_bytes();
let scale_arg = scale_metal
.as_ref()
.map(|(s, l)| (s.buffer(), l.start_offset() * dtype.size_in_bytes()));
hanzo_quant::metal_kernels::call_rmsnorm_residual(
device.device(),
&encoder,
&hanzo_quant::metal_kernels::Kernels::new(),
dtype,
(input_storage.buffer(), x_offset),
(residual_storage.buffer(), res_offset),
(weight_storage.buffer(), w_offset),
scale_arg,
&out_buf,
n_cols,
n_rows,
eps,
)
.map_err(hanzo_ml::Error::wrap)?;
let out = Tensor::from((
Storage::Metal(MetalStorage::new(
out_buf,
device.clone(),
input.elem_count(),
dtype,
)),
Shape::from(input.dims()),
));
Ok(Some(out))
}
#[cfg(feature = "metal")]
#[allow(clippy::cast_possible_truncation)]
pub fn metal_topk_logits_packed(
input: &Tensor,
k: usize,
temperature: f64,
) -> Result<TopKLogitsPackedOutput> {
use hanzo_ml::{backend::BackendStorage, MetalStorage, Shape, Storage};
const MAX_K: usize = 128;
const CHUNK_SIZE: usize = 2048;
if temperature <= 0.0 || !temperature.is_finite() {
hanzo_ml::bail!("metal_topk_logits_packed requires a positive finite temperature");
}
let input = input.contiguous()?;
if !matches!(input.dtype(), DType::F32 | DType::F16 | DType::BF16) {
hanzo_ml::bail!("metal_topk_logits_packed requires F32/F16/BF16 logits");
}
let dtype = input.dtype();
let ncols = input.elem_count();
if ncols == 0 {
hanzo_ml::bail!("metal_topk_logits_packed got empty logits");
}
let k = k.min(ncols);
if k == 0 || k > MAX_K {
hanzo_ml::bail!("metal_topk_logits_packed k={k} must be in [1, {MAX_K}]");
}
let nblocks = ncols.div_ceil(CHUNK_SIZE);
let (input_s, input_l) = input.storage_and_layout();
let Storage::Metal(input_s) = &*input_s else {
hanzo_ml::bail!("metal_topk_logits_packed requires Metal tensor");
};
let device = input_s.device().clone();
let block_values_buf = device.new_buffer(nblocks * k, DType::F32, "topk-block-values")?;
let block_indices_buf = device.new_buffer(nblocks * k, DType::U32, "topk-block-indices")?;
let block_maxes_buf = device.new_buffer(nblocks, DType::F32, "topk-block-maxes")?;
let block_sums_buf = device.new_buffer(nblocks, DType::F32, "topk-block-sums")?;
let packed_buf = device.new_buffer(2 * k + 2, DType::F32, "topk-packed")?;
let encoder = device.command_encoder()?;
encoder.set_label("topk-logits-packed");
let inv_temp = (1.0_f64 / temperature) as f32;
let input_offset = input_l.start_offset() * input.dtype().size_in_bytes();
hanzo_quant::metal_kernels::call_topk_logits_packed(
device.device(),
&encoder,
&hanzo_quant::metal_kernels::Kernels::new(),
dtype,
input_s.buffer(),
&block_values_buf,
&block_indices_buf,
&block_maxes_buf,
&block_sums_buf,
&packed_buf,
ncols,
k,
CHUNK_SIZE,
inv_temp,
)
.map_err(|e| hanzo_ml::Error::Msg(format!("metal_topk_logits_packed kernel error: {e}")))?;
let _ = (
input_offset,
&block_values_buf,
&block_indices_buf,
&block_maxes_buf,
&block_sums_buf,
);
let packed = Tensor::from((
Storage::Metal(MetalStorage::new(
packed_buf,
device.clone(),
2 * k + 2,
DType::F32,
)),
Shape::from(vec![2 * k + 2]),
));
Ok(TopKLogitsPackedOutput {
packed,
k,
_workspace: vec![],
})
}
#[cfg(feature = "cuda")]
pub fn cuda_rms_norm_residual_then_rms_norm(
input: &Tensor,
residual: &Tensor,
residual_weight: &Tensor,
scale: Option<&Tensor>,
norm_weight: &Tensor,
residual_eps: f32,
norm_eps: f32,
) -> Result<(Tensor, Tensor)> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
use std::ffi::c_void;
if input.shape() != residual.shape() {
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm input/residual shape mismatch: {:?} vs {:?}",
input.shape(),
residual.shape()
);
}
if input.dtype() != residual.dtype()
|| input.dtype() != residual_weight.dtype()
|| input.dtype() != norm_weight.dtype()
{
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm dtype mismatch: input {:?}, residual {:?}, residual_weight {:?}, norm_weight {:?}",
input.dtype(),
residual.dtype(),
residual_weight.dtype(),
norm_weight.dtype()
);
}
if !matches!(input.dtype(), DType::BF16 | DType::F16 | DType::F32) {
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm only supports BF16/F16/F32, got {:?}",
input.dtype()
);
}
if !residual.device().same_device(input.device())
|| !residual_weight.device().same_device(input.device())
|| !norm_weight.device().same_device(input.device())
{
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm tensors must be on the same CUDA device"
);
}
if let Some(scale) = scale {
if scale.elem_count() != 1 {
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm scale must have one element, got {}",
scale.elem_count()
);
}
if scale.dtype() != input.dtype() {
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm scale dtype mismatch: input {:?}, scale {:?}",
input.dtype(),
scale.dtype()
);
}
if !scale.device().same_device(input.device()) {
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm scale must be on the same CUDA device"
);
}
}
let ncols = input.dim(D::Minus1)?;
if residual_weight.dims1()? != ncols {
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm residual weight size {} does not match last dim {ncols}",
residual_weight.dims1()?
);
}
if norm_weight.dims1()? != ncols {
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm norm weight size {} does not match last dim {ncols}",
norm_weight.dims1()?
);
}
let elem_count = input.elem_count();
if elem_count == 0 {
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm got empty input");
}
let nrows = elem_count / ncols;
if nrows > i32::MAX as usize || ncols > i32::MAX as usize {
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm input is too large: nrows={nrows}, ncols={ncols}"
);
}
let nrows_i32 = i32::try_from(nrows).map_err(hanzo_ml::Error::wrap)?;
let ncols_i32 = i32::try_from(ncols).map_err(hanzo_ml::Error::wrap)?;
let input = input.contiguous()?;
let residual = residual.contiguous()?;
let residual_weight = residual_weight.contiguous()?;
let norm_weight = norm_weight.contiguous()?;
let scale = scale.map(Tensor::contiguous).transpose()?;
let (input_storage, input_layout) = input.storage_and_layout();
let input_storage = match &*input_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm requires CUDA input"),
};
let (residual_storage, residual_layout) = residual.storage_and_layout();
let residual_storage = match &*residual_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm requires CUDA residual"),
};
let (residual_weight_storage, residual_weight_layout) = residual_weight.storage_and_layout();
let residual_weight_storage = match &*residual_weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => {
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm requires CUDA residual weight")
}
};
let (norm_weight_storage, norm_weight_layout) = norm_weight.storage_and_layout();
let norm_weight_storage = match &*norm_weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm requires CUDA norm weight"),
};
let scale_storage_and_layout = scale.as_ref().map(|scale| scale.storage_and_layout());
let dev = input_storage.device();
let stream = dev.cuda_stream();
let stream_ptr = stream.cu_stream() as i64;
let shape = input.shape().clone();
macro_rules! launch {
($variant:ident, $ty:ty, $ffi_fn:ident) => {{
let CudaStorageSlice::$variant(src) = &input_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm input dtype mismatch");
};
let CudaStorageSlice::$variant(residual_src) = &residual_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm residual dtype mismatch");
};
let CudaStorageSlice::$variant(residual_weight_src) = &residual_weight_storage.slice
else {
hanzo_ml::bail!(
"cuda_rms_norm_residual_then_rms_norm residual weight dtype mismatch"
);
};
let CudaStorageSlice::$variant(norm_weight_src) = &norm_weight_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm norm weight dtype mismatch");
};
let (scale_ptr, scale_guard) = if let Some((scale_storage, scale_layout)) =
&scale_storage_and_layout
{
let scale_storage = match &**scale_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => {
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm requires CUDA scale")
}
};
let CudaStorageSlice::$variant(scale_src) = &scale_storage.slice else {
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm scale dtype mismatch");
};
let (scale_ptr, scale_guard) = scale_src.device_ptr(&stream);
(
unsafe { (scale_ptr as *const $ty).add(scale_layout.start_offset()) }
as *const c_void,
Some(scale_guard),
)
} else {
(std::ptr::null(), None)
};
let mut residual_out = unsafe { dev.alloc::<$ty>(elem_count) }?;
let mut norm_out = unsafe { dev.alloc::<$ty>(elem_count) }?;
let (src_ptr, src_guard) = src.device_ptr(&stream);
let (residual_ptr, residual_guard) = residual_src.device_ptr(&stream);
let (residual_weight_ptr, residual_weight_guard) =
residual_weight_src.device_ptr(&stream);
let (norm_weight_ptr, norm_weight_guard) = norm_weight_src.device_ptr(&stream);
let (residual_out_ptr, residual_out_guard) = residual_out.device_ptr_mut(&stream);
let (norm_out_ptr, norm_out_guard) = norm_out.device_ptr_mut(&stream);
let src_ptr = unsafe { (src_ptr as *const $ty).add(input_layout.start_offset()) };
let residual_ptr =
unsafe { (residual_ptr as *const $ty).add(residual_layout.start_offset()) };
let residual_weight_ptr = unsafe {
(residual_weight_ptr as *const $ty).add(residual_weight_layout.start_offset())
};
let norm_weight_ptr =
unsafe { (norm_weight_ptr as *const $ty).add(norm_weight_layout.start_offset()) };
unsafe {
ffi::$ffi_fn(
src_ptr as *const c_void,
residual_ptr as *const c_void,
residual_weight_ptr as *const c_void,
scale_ptr,
norm_weight_ptr as *const c_void,
residual_out_ptr as *mut c_void,
norm_out_ptr as *mut c_void,
nrows_i32,
ncols_i32,
residual_eps,
norm_eps,
stream_ptr,
);
}
drop(src_guard);
drop(residual_guard);
drop(residual_weight_guard);
drop(norm_weight_guard);
drop(scale_guard);
drop(residual_out_guard);
drop(norm_out_guard);
let residual_storage = CudaStorage {
slice: CudaStorageSlice::$variant(residual_out),
device: dev.clone(),
};
let norm_storage = CudaStorage {
slice: CudaStorageSlice::$variant(norm_out),
device: dev.clone(),
};
Ok((
Tensor::from((hanzo_ml::Storage::Cuda(residual_storage), shape.clone())),
Tensor::from((hanzo_ml::Storage::Cuda(norm_storage), shape)),
))
}};
}
match input.dtype() {
DType::BF16 => launch!(BF16, half::bf16, rms_norm_residual_then_rms_norm_bf16),
DType::F16 => launch!(F16, half::f16, rms_norm_residual_then_rms_norm_f16),
DType::F32 => launch!(F32, f32, rms_norm_residual_then_rms_norm_f32),
dtype => {
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm unsupported dtype {dtype:?}")
}
}
}
#[cfg(feature = "cuda")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn try_cuda_qk_rms_norm_rope(
q: &Tensor,
k: Option<&Tensor>,
q_weight: &Tensor,
k_weight: Option<&Tensor>,
q_eps: f32,
k_eps: f32,
cos: &Tensor,
sin: &Tensor,
is_neox: bool,
) -> Result<Option<(Tensor, Option<Tensor>)>> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
use std::ffi::c_void;
if !q.device().is_cuda() {
return Ok(None);
}
let dtype = q.dtype();
if !matches!(dtype, DType::BF16 | DType::F16 | DType::F32)
|| q_weight.dtype() != dtype
|| k_weight.is_some_and(|weight| weight.dtype() != dtype)
|| cos.dtype() != dtype
|| sin.dtype() != dtype
{
return Ok(None);
}
if !q_weight.device().same_device(q.device())
|| !cos.device().same_device(q.device())
|| !sin.device().same_device(q.device())
|| k.is_some_and(|k| !k.device().same_device(q.device()) || k.dtype() != dtype)
|| k_weight.is_some_and(|weight| !weight.device().same_device(q.device()))
{
return Ok(None);
}
let (batch, q_heads, seq_len, head_dim) = q.dims4()?;
// The fused kernel is intended for prompt/multi-token attention prep.
// Decode rows are already cheap in the existing kernels, and this row-wise
// reduction launch is slower for seq_len == 1 on current CUDA targets.
if seq_len == 1 {
return Ok(None);
}
let (k_heads, k_elem_count) = if let Some(k) = k {
let (k_batch, k_heads, k_seq_len, k_head_dim) = k.dims4()?;
if (k_batch, k_seq_len, k_head_dim) != (batch, seq_len, head_dim) {
hanzo_ml::bail!(
"q/k shape mismatch for fused qk norm rope: {:?} vs {:?}",
q.shape(),
k.shape()
);
}
let Some(k_weight) = k_weight else {
hanzo_ml::bail!("missing k norm weight for fused qk norm rope");
};
if k_weight.dims1()? != head_dim {
hanzo_ml::bail!(
"k norm weight size {} does not match head dim {head_dim}",
k_weight.dims1()?
);
}
(k_heads, k.elem_count())
} else {
(0, 0)
};
if q_weight.dims1()? != head_dim {
hanzo_ml::bail!(
"q norm weight size {} does not match head dim {head_dim}",
q_weight.dims1()?
);
}
let (cos_rows, rot_dim) = cos.dims2()?;
if sin.dims2()? != (cos_rows, rot_dim) {
hanzo_ml::bail!(
"cos/sin shape mismatch for fused qk norm rope: {:?} vs {:?}",
cos.shape(),
sin.shape()
);
}
if rot_dim == 0 || rot_dim * 2 > head_dim {
return Ok(None);
}
let cos_batch_stride = if cos_rows == seq_len {
0
} else if cos_rows == batch * seq_len {
seq_len
} else {
hanzo_ml::bail!(
"cos/sin rows {cos_rows} do not match seq_len {seq_len} or batch*seq_len {}",
batch * seq_len
);
};
for (name, value) in [
("batch", batch),
("q_heads", q_heads),
("k_heads", k_heads),
("seq_len", seq_len),
("head_dim", head_dim),
("rot_dim", rot_dim),
("cos_batch_stride", cos_batch_stride),
] {
if value > i32::MAX as usize {
hanzo_ml::bail!("fused qk norm rope {name} is too large: {value}");
}
}
let batch_i32 = i32::try_from(batch).map_err(hanzo_ml::Error::wrap)?;
let q_heads_i32 = i32::try_from(q_heads).map_err(hanzo_ml::Error::wrap)?;
let k_heads_i32 = i32::try_from(k_heads).map_err(hanzo_ml::Error::wrap)?;
let seq_len_i32 = i32::try_from(seq_len).map_err(hanzo_ml::Error::wrap)?;
let head_dim_i32 = i32::try_from(head_dim).map_err(hanzo_ml::Error::wrap)?;
let rot_dim_i32 = i32::try_from(rot_dim).map_err(hanzo_ml::Error::wrap)?;
let cos_batch_stride_i32 = i32::try_from(cos_batch_stride).map_err(hanzo_ml::Error::wrap)?;
let cos = cos.contiguous()?;
let sin = sin.contiguous()?;
let q_weight = q_weight.contiguous()?;
let k_weight = k_weight.map(Tensor::contiguous).transpose()?;
let (q_storage, q_layout) = q.storage_and_layout();
let q_storage = match &*q_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let k_storage_and_layout = k.map(Tensor::storage_and_layout);
let (q_weight_storage, q_weight_layout) = q_weight.storage_and_layout();
let q_weight_storage = match &*q_weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let k_weight_storage_and_layout = k_weight.as_ref().map(Tensor::storage_and_layout);
let (cos_storage, cos_layout) = cos.storage_and_layout();
let cos_storage = match &*cos_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (sin_storage, sin_layout) = sin.storage_and_layout();
let sin_storage = match &*sin_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let dev = q_storage.device();
let stream = dev.cuda_stream();
let stream_ptr = stream.cu_stream() as i64;
let q_shape = Shape::from_dims(&[batch, q_heads, seq_len, head_dim]);
let k_shape = Shape::from_dims(&[batch, k_heads, seq_len, head_dim]);
let q_elem_count = q.elem_count();
let q_stride = q_layout.stride();
let k_stride = k_storage_and_layout
.as_ref()
.map(|(_, layout)| layout.stride())
.unwrap_or(&[0, 0, 0, 0]);
macro_rules! launch {
($variant:ident, $ty:ty, $dtype_id:expr) => {{
let CudaStorageSlice::$variant(q_src) = &q_storage.slice else {
hanzo_ml::bail!("fused qk norm rope q dtype mismatch");
};
let CudaStorageSlice::$variant(q_weight_src) = &q_weight_storage.slice else {
hanzo_ml::bail!("fused qk norm rope q weight dtype mismatch");
};
let CudaStorageSlice::$variant(cos_src) = &cos_storage.slice else {
hanzo_ml::bail!("fused qk norm rope cos dtype mismatch");
};
let CudaStorageSlice::$variant(sin_src) = &sin_storage.slice else {
hanzo_ml::bail!("fused qk norm rope sin dtype mismatch");
};
let mut q_out_buf = unsafe { dev.alloc::<$ty>(q_elem_count) }?;
let mut k_out_buf = if k_elem_count == 0 {
None
} else {
Some(unsafe { dev.alloc::<$ty>(k_elem_count) }?)
};
let (q_ptr, q_guard) = q_src.device_ptr(&stream);
let q_ptr = unsafe { (q_ptr as *const $ty).add(q_layout.start_offset()) };
let (q_weight_ptr, q_weight_guard) = q_weight_src.device_ptr(&stream);
let q_weight_ptr =
unsafe { (q_weight_ptr as *const $ty).add(q_weight_layout.start_offset()) };
let (cos_ptr, cos_guard) = cos_src.device_ptr(&stream);
let cos_ptr = unsafe { (cos_ptr as *const $ty).add(cos_layout.start_offset()) };
let (sin_ptr, sin_guard) = sin_src.device_ptr(&stream);
let sin_ptr = unsafe { (sin_ptr as *const $ty).add(sin_layout.start_offset()) };
let mut k_guard = None;
let k_ptr = if let Some((k_storage, k_layout)) = &k_storage_and_layout {
let k_storage = match &**k_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let CudaStorageSlice::$variant(k_src) = &k_storage.slice else {
hanzo_ml::bail!("fused qk norm rope k dtype mismatch");
};
let (ptr, guard) = k_src.device_ptr(&stream);
k_guard = Some(guard);
unsafe { (ptr as *const $ty).add(k_layout.start_offset()) }
} else {
std::ptr::null()
};
let mut k_weight_guard = None;
let k_weight_ptr =
if let Some((k_weight_storage, k_weight_layout)) = &k_weight_storage_and_layout {
let k_weight_storage = match &**k_weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let CudaStorageSlice::$variant(k_weight_src) = &k_weight_storage.slice else {
hanzo_ml::bail!("fused qk norm rope k weight dtype mismatch");
};
let (ptr, guard) = k_weight_src.device_ptr(&stream);
k_weight_guard = Some(guard);
unsafe { (ptr as *const $ty).add(k_weight_layout.start_offset()) }
} else {
q_weight_ptr
};
let (q_out_ptr, q_out_guard) = q_out_buf.device_ptr_mut(&stream);
let mut k_out_guard = None;
let k_out_ptr = if let Some(k_out_buf) = &mut k_out_buf {
let (ptr, guard) = k_out_buf.device_ptr_mut(&stream);
k_out_guard = Some(guard);
ptr as *mut $ty
} else {
std::ptr::null_mut()
};
unsafe {
ffi::qk_rms_norm_rope(
q_ptr as *const c_void,
k_ptr as *const c_void,
q_weight_ptr as *const c_void,
k_weight_ptr as *const c_void,
cos_ptr as *const c_void,
sin_ptr as *const c_void,
q_out_ptr as *mut c_void,
k_out_ptr as *mut c_void,
q_stride[0] as i64,
q_stride[1] as i64,
q_stride[2] as i64,
q_stride[3] as i64,
k_stride[0] as i64,
k_stride[1] as i64,
k_stride[2] as i64,
k_stride[3] as i64,
batch_i32,
q_heads_i32,
k_heads_i32,
seq_len_i32,
head_dim_i32,
rot_dim_i32,
cos_batch_stride_i32,
q_eps,
k_eps,
i32::from(is_neox),
$dtype_id,
stream_ptr,
);
}
drop(q_guard);
drop(q_weight_guard);
drop(cos_guard);
drop(sin_guard);
drop(k_guard);
drop(k_weight_guard);
drop(q_out_guard);
drop(k_out_guard);
let q_storage = CudaStorage {
slice: CudaStorageSlice::$variant(q_out_buf),
device: dev.clone(),
};
let q_tensor = Tensor::from((hanzo_ml::Storage::Cuda(q_storage), q_shape));
let k_tensor = if let Some(k_out_buf) = k_out_buf {
let k_storage = CudaStorage {
slice: CudaStorageSlice::$variant(k_out_buf),
device: dev.clone(),
};
Some(Tensor::from((hanzo_ml::Storage::Cuda(k_storage), k_shape)))
} else {
None
};
Ok(Some((q_tensor, k_tensor)))
}};
}
match dtype {
DType::BF16 => launch!(BF16, half::bf16, 1),
DType::F16 => launch!(F16, half::f16, 0),
DType::F32 => launch!(F32, f32, 2),
_ => Ok(None),
}
}
#[cfg(feature = "cuda")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn try_cuda_qk_rms_norm_rope_positions(
q: &Tensor,
k: Option<&Tensor>,
q_weight: &Tensor,
k_weight: Option<&Tensor>,
q_eps: f32,
k_eps: f32,
cos: &Tensor,
sin: &Tensor,
positions: &Tensor,
is_neox: bool,
) -> Result<Option<(Tensor, Option<Tensor>)>> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
use std::ffi::c_void;
if !q.device().is_cuda() {
return Ok(None);
}
let dtype = q.dtype();
if !matches!(dtype, DType::BF16 | DType::F16 | DType::F32)
|| q_weight.dtype() != dtype
|| k_weight.is_some_and(|weight| weight.dtype() != dtype)
|| cos.dtype() != dtype
|| sin.dtype() != dtype
|| positions.dtype() != DType::U32
{
return Ok(None);
}
if !q_weight.device().same_device(q.device())
|| !cos.device().same_device(q.device())
|| !sin.device().same_device(q.device())
|| !positions.device().same_device(q.device())
|| k.is_some_and(|k| !k.device().same_device(q.device()) || k.dtype() != dtype)
|| k_weight.is_some_and(|weight| !weight.device().same_device(q.device()))
{
return Ok(None);
}
let (batch, q_heads, seq_len, head_dim) = q.dims4()?;
if positions.dims1()? != batch {
hanzo_ml::bail!(
"positions length {} does not match batch size {batch}",
positions.dims1()?
);
}
let (k_heads, k_elem_count) = if let Some(k) = k {
let (k_batch, k_heads, k_seq_len, k_head_dim) = k.dims4()?;
if (k_batch, k_seq_len, k_head_dim) != (batch, seq_len, head_dim) {
hanzo_ml::bail!(
"q/k shape mismatch for fused qk norm rope positions: {:?} vs {:?}",
q.shape(),
k.shape()
);
}
let Some(k_weight) = k_weight else {
hanzo_ml::bail!("missing k norm weight for fused qk norm rope positions");
};
if k_weight.dims1()? != head_dim {
hanzo_ml::bail!(
"k norm weight size {} does not match head dim {head_dim}",
k_weight.dims1()?
);
}
(k_heads, k.elem_count())
} else {
(0, 0)
};
if q_weight.dims1()? != head_dim {
hanzo_ml::bail!(
"q norm weight size {} does not match head dim {head_dim}",
q_weight.dims1()?
);
}
let (cos_rows, rot_dim) = cos.dims2()?;
if sin.dims2()? != (cos_rows, rot_dim) {
hanzo_ml::bail!(
"cos/sin shape mismatch for fused qk norm rope positions: {:?} vs {:?}",
cos.shape(),
sin.shape()
);
}
if rot_dim == 0 || rot_dim * 2 > head_dim {
return Ok(None);
}
for (name, value) in [
("batch", batch),
("q_heads", q_heads),
("k_heads", k_heads),
("seq_len", seq_len),
("head_dim", head_dim),
("rot_dim", rot_dim),
] {
if value > i32::MAX as usize {
hanzo_ml::bail!("fused qk norm rope positions {name} is too large: {value}");
}
}
let batch_i32 = i32::try_from(batch).map_err(hanzo_ml::Error::wrap)?;
let q_heads_i32 = i32::try_from(q_heads).map_err(hanzo_ml::Error::wrap)?;
let k_heads_i32 = i32::try_from(k_heads).map_err(hanzo_ml::Error::wrap)?;
let seq_len_i32 = i32::try_from(seq_len).map_err(hanzo_ml::Error::wrap)?;
let head_dim_i32 = i32::try_from(head_dim).map_err(hanzo_ml::Error::wrap)?;
let rot_dim_i32 = i32::try_from(rot_dim).map_err(hanzo_ml::Error::wrap)?;
let cos = cos.contiguous()?;
let sin = sin.contiguous()?;
let positions = positions.contiguous()?;
let q_weight = q_weight.contiguous()?;
let k_weight = k_weight.map(Tensor::contiguous).transpose()?;
let (q_storage, q_layout) = q.storage_and_layout();
let q_storage = match &*q_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let k_storage_and_layout = k.map(Tensor::storage_and_layout);
let (q_weight_storage, q_weight_layout) = q_weight.storage_and_layout();
let q_weight_storage = match &*q_weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let k_weight_storage_and_layout = k_weight.as_ref().map(Tensor::storage_and_layout);
let (cos_storage, cos_layout) = cos.storage_and_layout();
let cos_storage = match &*cos_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (sin_storage, sin_layout) = sin.storage_and_layout();
let sin_storage = match &*sin_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (positions_storage, positions_layout) = positions.storage_and_layout();
let positions_storage = match &*positions_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let dev = q_storage.device();
let stream = dev.cuda_stream();
let stream_ptr = stream.cu_stream() as i64;
let q_shape = Shape::from_dims(&[batch, q_heads, seq_len, head_dim]);
let k_shape = Shape::from_dims(&[batch, k_heads, seq_len, head_dim]);
let q_elem_count = q.elem_count();
let q_stride = q_layout.stride();
let k_stride = k_storage_and_layout
.as_ref()
.map(|(_, layout)| layout.stride())
.unwrap_or(&[0, 0, 0, 0]);
macro_rules! launch {
($variant:ident, $ty:ty, $dtype_id:expr) => {{
let CudaStorageSlice::$variant(q_src) = &q_storage.slice else {
hanzo_ml::bail!("fused qk norm rope positions q dtype mismatch");
};
let CudaStorageSlice::$variant(q_weight_src) = &q_weight_storage.slice else {
hanzo_ml::bail!("fused qk norm rope positions q weight dtype mismatch");
};
let CudaStorageSlice::$variant(cos_src) = &cos_storage.slice else {
hanzo_ml::bail!("fused qk norm rope positions cos dtype mismatch");
};
let CudaStorageSlice::$variant(sin_src) = &sin_storage.slice else {
hanzo_ml::bail!("fused qk norm rope positions sin dtype mismatch");
};
let CudaStorageSlice::U32(positions_src) = &positions_storage.slice else {
hanzo_ml::bail!("fused qk norm rope positions dtype mismatch");
};
let mut q_out_buf = unsafe { dev.alloc::<$ty>(q_elem_count) }?;
let mut k_out_buf = if k_elem_count == 0 {
None
} else {
Some(unsafe { dev.alloc::<$ty>(k_elem_count) }?)
};
let (q_ptr, q_guard) = q_src.device_ptr(&stream);
let q_ptr = unsafe { (q_ptr as *const $ty).add(q_layout.start_offset()) };
let (q_weight_ptr, q_weight_guard) = q_weight_src.device_ptr(&stream);
let q_weight_ptr =
unsafe { (q_weight_ptr as *const $ty).add(q_weight_layout.start_offset()) };
let (cos_ptr, cos_guard) = cos_src.device_ptr(&stream);
let cos_ptr = unsafe { (cos_ptr as *const $ty).add(cos_layout.start_offset()) };
let (sin_ptr, sin_guard) = sin_src.device_ptr(&stream);
let sin_ptr = unsafe { (sin_ptr as *const $ty).add(sin_layout.start_offset()) };
let (positions_ptr, positions_guard) = positions_src.device_ptr(&stream);
let positions_ptr =
unsafe { (positions_ptr as *const u32).add(positions_layout.start_offset()) };
let mut k_guard = None;
let k_ptr = if let Some((k_storage, k_layout)) = &k_storage_and_layout {
let k_storage = match &**k_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let CudaStorageSlice::$variant(k_src) = &k_storage.slice else {
hanzo_ml::bail!("fused qk norm rope positions k dtype mismatch");
};
let (ptr, guard) = k_src.device_ptr(&stream);
k_guard = Some(guard);
unsafe { (ptr as *const $ty).add(k_layout.start_offset()) }
} else {
std::ptr::null()
};
let mut k_weight_guard = None;
let k_weight_ptr =
if let Some((k_weight_storage, k_weight_layout)) = &k_weight_storage_and_layout {
let k_weight_storage = match &**k_weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let CudaStorageSlice::$variant(k_weight_src) = &k_weight_storage.slice else {
hanzo_ml::bail!("fused qk norm rope positions k weight dtype mismatch");
};
let (ptr, guard) = k_weight_src.device_ptr(&stream);
k_weight_guard = Some(guard);
unsafe { (ptr as *const $ty).add(k_weight_layout.start_offset()) }
} else {
q_weight_ptr
};
let (q_out_ptr, q_out_guard) = q_out_buf.device_ptr_mut(&stream);
let mut k_out_guard = None;
let k_out_ptr = if let Some(k_out_buf) = &mut k_out_buf {
let (ptr, guard) = k_out_buf.device_ptr_mut(&stream);
k_out_guard = Some(guard);
ptr as *mut $ty
} else {
std::ptr::null_mut()
};
unsafe {
ffi::qk_rms_norm_rope_positions(
q_ptr as *const c_void,
k_ptr as *const c_void,
q_weight_ptr as *const c_void,
k_weight_ptr as *const c_void,
cos_ptr as *const c_void,
sin_ptr as *const c_void,
positions_ptr as *const c_void,
q_out_ptr as *mut c_void,
k_out_ptr as *mut c_void,
q_stride[0] as i64,
q_stride[1] as i64,
q_stride[2] as i64,
q_stride[3] as i64,
k_stride[0] as i64,
k_stride[1] as i64,
k_stride[2] as i64,
k_stride[3] as i64,
batch_i32,
q_heads_i32,
k_heads_i32,
seq_len_i32,
head_dim_i32,
rot_dim_i32,
q_eps,
k_eps,
i32::from(is_neox),
$dtype_id,
stream_ptr,
);
}
drop(q_guard);
drop(q_weight_guard);
drop(cos_guard);
drop(sin_guard);
drop(positions_guard);
drop(k_guard);
drop(k_weight_guard);
drop(q_out_guard);
drop(k_out_guard);
let q_storage = CudaStorage {
slice: CudaStorageSlice::$variant(q_out_buf),
device: dev.clone(),
};
let q_tensor = Tensor::from((hanzo_ml::Storage::Cuda(q_storage), q_shape));
let k_tensor = if let Some(k_out_buf) = k_out_buf {
let k_storage = CudaStorage {
slice: CudaStorageSlice::$variant(k_out_buf),
device: dev.clone(),
};
Some(Tensor::from((hanzo_ml::Storage::Cuda(k_storage), k_shape)))
} else {
None
};
Ok(Some((q_tensor, k_tensor)))
}};
}
match dtype {
DType::BF16 => launch!(BF16, half::bf16, 1),
DType::F16 => launch!(F16, half::f16, 0),
DType::F32 => launch!(F32, f32, 2),
_ => Ok(None),
}
}
#[cfg(feature = "cuda")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn try_cuda_qkv_rms_norm_rope_positions(
q: &Tensor,
k: &Tensor,
v: &Tensor,
q_weight: &Tensor,
k_weight: &Tensor,
v_weight: &Tensor,
q_eps: f32,
k_eps: f32,
v_eps: f32,
cos: &Tensor,
sin: &Tensor,
positions: &Tensor,
is_neox: bool,
) -> Result<Option<(Tensor, Tensor, Tensor)>> {
use hanzo_ml::backend::BackendStorage;
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
use std::ffi::c_void;
if !q.device().is_cuda() {
return Ok(None);
}
let dtype = q.dtype();
if !matches!(dtype, DType::BF16 | DType::F16 | DType::F32)
|| k.dtype() != dtype
|| v.dtype() != dtype
|| q_weight.dtype() != dtype
|| k_weight.dtype() != dtype
|| v_weight.dtype() != dtype
|| cos.dtype() != dtype
|| sin.dtype() != dtype
|| positions.dtype() != DType::U32
{
return Ok(None);
}
if !q_weight.device().same_device(q.device())
|| !k_weight.device().same_device(q.device())
|| !v_weight.device().same_device(q.device())
|| !cos.device().same_device(q.device())
|| !sin.device().same_device(q.device())
|| !positions.device().same_device(q.device())
|| !k.device().same_device(q.device())
|| !v.device().same_device(q.device())
{
return Ok(None);
}
let (batch, q_heads, seq_len, head_dim) = q.dims4()?;
let (k_batch, k_heads, k_seq_len, k_head_dim) = k.dims4()?;
let (v_batch, v_heads, v_seq_len, v_head_dim) = v.dims4()?;
if (k_batch, k_seq_len, k_head_dim) != (batch, seq_len, head_dim)
|| (v_batch, v_heads, v_seq_len, v_head_dim) != (batch, k_heads, seq_len, head_dim)
{
hanzo_ml::bail!(
"q/k/v shape mismatch for fused qkv norm rope positions: {:?}, {:?}, {:?}",
q.shape(),
k.shape(),
v.shape()
);
}
if positions.dims1()? != batch {
hanzo_ml::bail!(
"positions length {} does not match batch size {batch}",
positions.dims1()?
);
}
if q_weight.dims1()? != head_dim
|| k_weight.dims1()? != head_dim
|| v_weight.dims1()? != head_dim
{
hanzo_ml::bail!("qkv norm weight size does not match head dim {head_dim}");
}
let (cos_rows, rot_dim) = cos.dims2()?;
if sin.dims2()? != (cos_rows, rot_dim) {
hanzo_ml::bail!(
"cos/sin shape mismatch for fused qkv norm rope positions: {:?} vs {:?}",
cos.shape(),
sin.shape()
);
}
if rot_dim == 0 || rot_dim * 2 > head_dim {
return Ok(None);
}
for (name, value) in [
("batch", batch),
("q_heads", q_heads),
("k_heads", k_heads),
("seq_len", seq_len),
("head_dim", head_dim),
("rot_dim", rot_dim),
] {
if value > i32::MAX as usize {
hanzo_ml::bail!("fused qkv norm rope positions {name} is too large: {value}");
}
}
let batch_i32 = i32::try_from(batch).map_err(hanzo_ml::Error::wrap)?;
let q_heads_i32 = i32::try_from(q_heads).map_err(hanzo_ml::Error::wrap)?;
let k_heads_i32 = i32::try_from(k_heads).map_err(hanzo_ml::Error::wrap)?;
let seq_len_i32 = i32::try_from(seq_len).map_err(hanzo_ml::Error::wrap)?;
let head_dim_i32 = i32::try_from(head_dim).map_err(hanzo_ml::Error::wrap)?;
let rot_dim_i32 = i32::try_from(rot_dim).map_err(hanzo_ml::Error::wrap)?;
let cos = cos.contiguous()?;
let sin = sin.contiguous()?;
let positions = positions.contiguous()?;
let q_weight = q_weight.contiguous()?;
let k_weight = k_weight.contiguous()?;
let v_weight = v_weight.contiguous()?;
let (q_storage, q_layout) = q.storage_and_layout();
let q_storage = match &*q_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (k_storage, k_layout) = k.storage_and_layout();
let k_storage = match &*k_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (v_storage, v_layout) = v.storage_and_layout();
let v_storage = match &*v_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (q_weight_storage, q_weight_layout) = q_weight.storage_and_layout();
let q_weight_storage = match &*q_weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (k_weight_storage, k_weight_layout) = k_weight.storage_and_layout();
let k_weight_storage = match &*k_weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (v_weight_storage, v_weight_layout) = v_weight.storage_and_layout();
let v_weight_storage = match &*v_weight_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (cos_storage, cos_layout) = cos.storage_and_layout();
let cos_storage = match &*cos_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (sin_storage, sin_layout) = sin.storage_and_layout();
let sin_storage = match &*sin_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let (positions_storage, positions_layout) = positions.storage_and_layout();
let positions_storage = match &*positions_storage {
hanzo_ml::Storage::Cuda(s) => s,
_ => return Ok(None),
};
let dev = q_storage.device();
let stream = dev.cuda_stream();
let stream_ptr = stream.cu_stream() as i64;
let q_shape = Shape::from_dims(&[batch, q_heads, seq_len, head_dim]);
let kv_shape = Shape::from_dims(&[batch, k_heads, seq_len, head_dim]);
let q_elem_count = q.elem_count();
let kv_elem_count = k.elem_count();
let q_stride = q_layout.stride();
let k_stride = k_layout.stride();
let v_stride = v_layout.stride();
macro_rules! launch {
($variant:ident, $ty:ty, $dtype_id:expr) => {{
let CudaStorageSlice::$variant(q_src) = &q_storage.slice else {
hanzo_ml::bail!("fused qkv norm rope positions q dtype mismatch");
};
let CudaStorageSlice::$variant(k_src) = &k_storage.slice else {
hanzo_ml::bail!("fused qkv norm rope positions k dtype mismatch");
};
let CudaStorageSlice::$variant(v_src) = &v_storage.slice else {
hanzo_ml::bail!("fused qkv norm rope positions v dtype mismatch");
};
let CudaStorageSlice::$variant(q_weight_src) = &q_weight_storage.slice else {
hanzo_ml::bail!("fused qkv norm rope positions q weight dtype mismatch");
};
let CudaStorageSlice::$variant(k_weight_src) = &k_weight_storage.slice else {
hanzo_ml::bail!("fused qkv norm rope positions k weight dtype mismatch");
};
let CudaStorageSlice::$variant(v_weight_src) = &v_weight_storage.slice else {
hanzo_ml::bail!("fused qkv norm rope positions v weight dtype mismatch");
};
let CudaStorageSlice::$variant(cos_src) = &cos_storage.slice else {
hanzo_ml::bail!("fused qkv norm rope positions cos dtype mismatch");
};
let CudaStorageSlice::$variant(sin_src) = &sin_storage.slice else {
hanzo_ml::bail!("fused qkv norm rope positions sin dtype mismatch");
};
let CudaStorageSlice::U32(positions_src) = &positions_storage.slice else {
hanzo_ml::bail!("fused qkv norm rope positions dtype mismatch");
};
let mut q_out_buf = unsafe { dev.alloc::<$ty>(q_elem_count) }?;
let mut k_out_buf = unsafe { dev.alloc::<$ty>(kv_elem_count) }?;
let mut v_out_buf = unsafe { dev.alloc::<$ty>(kv_elem_count) }?;
let (q_ptr, q_guard) = q_src.device_ptr(&stream);
let q_ptr = unsafe { (q_ptr as *const $ty).add(q_layout.start_offset()) };
let (k_ptr, k_guard) = k_src.device_ptr(&stream);
let k_ptr = unsafe { (k_ptr as *const $ty).add(k_layout.start_offset()) };
let (v_ptr, v_guard) = v_src.device_ptr(&stream);
let v_ptr = unsafe { (v_ptr as *const $ty).add(v_layout.start_offset()) };
let (q_weight_ptr, q_weight_guard) = q_weight_src.device_ptr(&stream);
let q_weight_ptr =
unsafe { (q_weight_ptr as *const $ty).add(q_weight_layout.start_offset()) };
let (k_weight_ptr, k_weight_guard) = k_weight_src.device_ptr(&stream);
let k_weight_ptr =
unsafe { (k_weight_ptr as *const $ty).add(k_weight_layout.start_offset()) };
let (v_weight_ptr, v_weight_guard) = v_weight_src.device_ptr(&stream);
let v_weight_ptr =
unsafe { (v_weight_ptr as *const $ty).add(v_weight_layout.start_offset()) };
let (cos_ptr, cos_guard) = cos_src.device_ptr(&stream);
let cos_ptr = unsafe { (cos_ptr as *const $ty).add(cos_layout.start_offset()) };
let (sin_ptr, sin_guard) = sin_src.device_ptr(&stream);
let sin_ptr = unsafe { (sin_ptr as *const $ty).add(sin_layout.start_offset()) };
let (positions_ptr, positions_guard) = positions_src.device_ptr(&stream);
let positions_ptr =
unsafe { (positions_ptr as *const u32).add(positions_layout.start_offset()) };
let (q_out_ptr, q_out_guard) = q_out_buf.device_ptr_mut(&stream);
let (k_out_ptr, k_out_guard) = k_out_buf.device_ptr_mut(&stream);
let (v_out_ptr, v_out_guard) = v_out_buf.device_ptr_mut(&stream);
unsafe {
ffi::qkv_rms_norm_rope_positions(
q_ptr as *const c_void,
k_ptr as *const c_void,
v_ptr as *const c_void,
q_weight_ptr as *const c_void,
k_weight_ptr as *const c_void,
v_weight_ptr as *const c_void,
cos_ptr as *const c_void,
sin_ptr as *const c_void,
positions_ptr as *const c_void,
q_out_ptr as *mut c_void,
k_out_ptr as *mut c_void,
v_out_ptr as *mut c_void,
q_stride[0] as i64,
q_stride[1] as i64,
q_stride[2] as i64,
q_stride[3] as i64,
k_stride[0] as i64,
k_stride[1] as i64,
k_stride[2] as i64,
k_stride[3] as i64,
v_stride[0] as i64,
v_stride[1] as i64,
v_stride[2] as i64,
v_stride[3] as i64,
batch_i32,
q_heads_i32,
k_heads_i32,
seq_len_i32,
head_dim_i32,
rot_dim_i32,
q_eps,
k_eps,
v_eps,
i32::from(is_neox),
$dtype_id,
stream_ptr,
);
}
drop(q_guard);
drop(k_guard);
drop(v_guard);
drop(q_weight_guard);
drop(k_weight_guard);
drop(v_weight_guard);
drop(cos_guard);
drop(sin_guard);
drop(positions_guard);
drop(q_out_guard);
drop(k_out_guard);
drop(v_out_guard);
let q_storage = CudaStorage {
slice: CudaStorageSlice::$variant(q_out_buf),
device: dev.clone(),
};
let k_storage = CudaStorage {
slice: CudaStorageSlice::$variant(k_out_buf),
device: dev.clone(),
};
let v_storage = CudaStorage {
slice: CudaStorageSlice::$variant(v_out_buf),
device: dev.clone(),
};
Ok(Some((
Tensor::from((hanzo_ml::Storage::Cuda(q_storage), q_shape)),
Tensor::from((hanzo_ml::Storage::Cuda(k_storage), kv_shape.clone())),
Tensor::from((hanzo_ml::Storage::Cuda(v_storage), kv_shape)),
)))
}};
}
match dtype {
DType::BF16 => launch!(BF16, half::bf16, 1),
DType::F16 => launch!(F16, half::f16, 0),
DType::F32 => launch!(F32, f32, 2),
_ => Ok(None),
}
}
pub trait TopKLastDimOp {
/// Topk in the last dim. `values` retains a gradient but `indices` has none w.r.t self.
/// This expects a contiguous tensor.
/// Note: this implements torch.topk with sorted=True.
fn topk(&self, topk: usize) -> Result<TopKOutput>;
/// Topk in the last dim. `values` retains a gradient but `indices` has none w.r.t self.
/// This expects a contiguous tensor.
/// Note: this implements torch.topk with sorted=False.
fn topk_unsorted(&self, topk: usize) -> Result<TopKOutput>;
}
impl TopKLastDimOp for Tensor {
fn topk(&self, topk: usize) -> Result<TopKOutput> {
// Use optimized parallel topk kernel on CUDA
// Single kernel call, no post-processing overhead
#[cfg(feature = "cuda")]
if self.device().is_cuda() {
return cuda_topk(self, topk);
}
// Fallback: full sort (CPU or non-CUDA)
let (values, sorted_indices) = self.sort_last_dim(false)?;
let topk_indices = sorted_indices.narrow(D::Minus1, 0, topk)?.contiguous()?;
let topk_values = values.narrow(D::Minus1, 0, topk)?.contiguous()?;
Ok(TopKOutput {
values: topk_values,
indices: topk_indices,
})
}
fn topk_unsorted(&self, topk: usize) -> Result<TopKOutput> {
// Sorted descending
let TopKOutput { values, indices } = self.topk(topk)?;
// Reorder the indices ascending
#[cfg(feature = "cuda")]
let reorder_indices = indices.arg_sort(true)?;
#[cfg(not(feature = "cuda"))]
let reorder_indices = indices.arg_sort_last_dim(true)?;
let topk_indices_unsorted = indices
.to_dtype(DType::F32)?
.gather(&reorder_indices, D::Minus1)?
.to_dtype(DType::U32)?;
let topk_values_unsorted = values.gather(&reorder_indices, D::Minus1)?;
Ok(TopKOutput {
values: topk_values_unsorted,
indices: topk_indices_unsorted,
})
}
}
pub trait RepeatInterleaveOp {
fn repeat_interleave<D: Dim>(&self, repeats: usize, dim: D) -> Result<Tensor>;
fn repeat_interleave_flat(&self, repeats: Vec<u32>) -> Result<Tensor>;
}
impl RepeatInterleaveOp for Tensor {
fn repeat_interleave<D: Dim>(&self, repeats: usize, dim: D) -> Result<Tensor> {
let dim = dim.to_index(self.shape(), "repeat_interleave")?;
let dim_elements = self.dim(dim)?;
// For metal
assert!(self.dtype().is_float());
#[allow(clippy::cast_possible_truncation)]
let indices = Tensor::new(
(0..dim_elements)
.flat_map(|i| vec![i as u32; repeats])
.collect::<Vec<_>>(),
self.device(),
)?;
self.index_select(&indices, dim)
}
fn repeat_interleave_flat(&self, repeats: Vec<u32>) -> Result<Tensor> {
let xs = self.flatten_all()?;
if repeats.len() != xs.dim(0)? {
hanzo_ml::bail!(
"repeats ({}) must match flattened self length ({})",
repeats.len(),
xs.dim(0)?
);
}
#[allow(clippy::cast_possible_truncation)]
let indices = Tensor::new(
(0..xs.dim(0)?)
.flat_map(|i| vec![i as u32; repeats[i] as usize])
.collect::<Vec<_>>(),
xs.device(),
)?;
xs.index_select(&indices, 0)
}
}
pub trait SplitOp {
fn split<D: Dim>(&self, splits: &[usize], dim: D) -> Result<Vec<Tensor>>;
}
impl SplitOp for Tensor {
fn split<D: Dim>(&self, splits: &[usize], dim: D) -> Result<Vec<Tensor>> {
let dim = dim.to_index(self.shape(), "split")?;
let mut split_res = Vec::new();
let mut index = 0;
for split in splits {
split_res.push(self.narrow(dim, index, *split)?);
index += *split;
}
Ok(split_res)
}
}
#[allow(dead_code)]
pub trait BincountOp {
fn bincount(&self, minlength: u32) -> Result<Vec<u32>>;
}
#[allow(dead_code)]
fn bincount(values: &[u32], minlength: u32) -> Vec<u32> {
// let max_val = values.iter().max().copied().unwrap_or(0);
// let result_len = (max_val + 1).max(minlength);
// values.iter().fold(
// // Start with a histogram vector of zeros.
// vec![0u32; result_len as usize],
// // For each value, update the histogram.
// |mut histogram, &value| {
// histogram[value as usize] += 1;
// histogram
// },
// )
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
// Early return if there are no values.
if values.is_empty() {
return vec![0u32; minlength as usize];
}
// Compute the maximum value in parallel.
// SAFETY: We just checked that values is nonempty above, so max() will return Some.
// Using expect() for clearer error message if this invariant is somehow violated.
let max_val = *values
.par_iter()
.max()
.expect("values should be non-empty after empty check");
// The histogram length must cover all observed values as well as `minlength`.
let result_len = (max_val + 1).max(minlength) as usize;
// Build per-thread histograms in parallel.
// We use unsafe indexing to eliminate bounds checks in the inner loop.
values
.par_iter()
.fold(
|| vec![0u32; result_len],
|mut local_hist, &v| {
// SAFETY: v is guaranteed to be <= max_val, so it is in bounds.
unsafe {
*local_hist.get_unchecked_mut(v as usize) += 1;
}
local_hist
},
)
// Merge the per-thread histograms in parallel.
.reduce(
|| vec![0u32; result_len],
|mut global_hist, local_hist| {
for i in 0..result_len {
// SAFETY: we know local histogram is at least result_len, as is global_hist
unsafe {
*global_hist.get_unchecked_mut(i) += local_hist.get_unchecked(i);
}
}
global_hist
},
)
}
#[allow(dead_code)]
impl BincountOp for Tensor {
fn bincount(&self, minlength: u32) -> Result<Vec<u32>> {
let values = self.to_vec1::<u32>()?;
Ok(bincount(&values, minlength))
}
}
// an external reference implementation
pub fn apply_triangular(xs: &Tensor, diagonal: isize, upper: bool) -> Result<Tensor> {
let device = xs.device();
let (l, s) = xs.dims2()?;
let mut xs_tri = vec![];
for i in 0..l as isize {
for j in 0..s as isize {
let cond = if upper {
i + diagonal > j
} else {
i + diagonal < j
};
xs_tri.push(if cond { 0u8 } else { 1u8 });
}
}
xs * Tensor::from_vec(xs_tri, (l, s), device)?.to_dtype(xs.dtype())?
}
/// Elementwise multiply and activation. The following activations are supported:
/// - `gelu`
/// - `silu`
/// - `relu`
///
/// This is equivalent to:
/// `act(a) * b`
///
/// With supported dtypes (F16, BF16, F32) and activations (SiLU, GELU, ReLU),
/// this uses a fused kernel for better performance by eliminating intermediate
/// memory allocation. Optimized implementations are available for:
/// - CUDA: Custom CUDA kernel with vec4 optimization
/// - Metal: Native Metal kernel
/// - CPU: Rayon-parallelized implementation
fn glu_activation_type(act: Activation) -> Option<hanzo_quant::GluActivationType> {
match act {
Activation::Silu | Activation::Swish => Some(hanzo_quant::GluActivationType::Silu),
Activation::NewGelu | Activation::GeluPytorchTanh => {
Some(hanzo_quant::GluActivationType::Gelu)
}
Activation::Gelu => Some(hanzo_quant::GluActivationType::GeluErf),
Activation::Relu => Some(hanzo_quant::GluActivationType::Relu),
_ => None,
}
}
fn nn_glu_activation_type(act: hanzo_nn::Activation) -> Option<hanzo_quant::GluActivationType> {
match act {
hanzo_nn::Activation::Silu | hanzo_nn::Activation::Swish => {
Some(hanzo_quant::GluActivationType::Silu)
}
hanzo_nn::Activation::NewGelu | hanzo_nn::Activation::GeluPytorchTanh => {
Some(hanzo_quant::GluActivationType::Gelu)
}
hanzo_nn::Activation::Gelu => Some(hanzo_quant::GluActivationType::GeluErf),
hanzo_nn::Activation::Relu => Some(hanzo_quant::GluActivationType::Relu),
_ => None,
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum GatedActivationOrder {
GateUp,
UpGate,
}
pub fn mul_and_act(a: &Tensor, b: &Tensor, act: Activation) -> Result<Tensor> {
// Vulkan has a fused silu_mul kernel but no general fused_glu; route Silu (the common SwiGLU
// case) to it so the FFN gate does one dispatch instead of silu + mul (a dispatch + barrier each).
#[cfg(feature = "vulkan")]
if a.device().is_vulkan() && a.dtype() == b.dtype() && matches!(act, Activation::Silu) {
return hanzo_nn::ops::silu_mul(a, b);
}
// Check if we can use the fused kernel (works on CUDA, Metal, and CPU)
if matches!(a.dtype(), DType::F16 | DType::BF16 | DType::F32) && a.dtype() == b.dtype() {
if let Some(activation_type) = glu_activation_type(act) {
return hanzo_quant::fused_glu(a, b, activation_type);
}
}
a.apply(&act)? * b
}
/// Clamp SwiGLU gate/up before the gated activation: `gate.clamp(max=limit)`,
/// `up.clamp(-limit, limit)` (DeepSeek-V4 `DeepseekV4MLP`/`Experts`, gpt-oss). `None` returns the
/// inputs unchanged (cheap ref-clone) so the no-limit path stays byte-identical.
pub fn swiglu_clamp(gate: &Tensor, up: &Tensor, limit: Option<f32>) -> Result<(Tensor, Tensor)> {
match limit {
None => Ok((gate.clone(), up.clone())),
Some(limit) => {
let limit = f64::from(limit);
Ok((
gate.clamp(f64::NEG_INFINITY, limit)?,
up.clamp(-limit, limit)?,
))
}
}
}
pub fn mul_and_nn_act(a: &Tensor, b: &Tensor, act: hanzo_nn::Activation) -> Result<Tensor> {
// Check if we can use the fused kernel (works on CUDA, Metal, and CPU)
if matches!(a.dtype(), DType::F16 | DType::BF16 | DType::F32) && a.dtype() == b.dtype() {
if let Some(activation_type) = nn_glu_activation_type(act) {
return hanzo_quant::fused_glu(a, b, activation_type);
}
}
a.apply(&act)? * b
}
pub fn split_mul_and_act(xs: &Tensor, split_size: usize, act: Activation) -> Result<Tensor> {
split_mul_and_act_order(xs, split_size, act, GatedActivationOrder::GateUp)
}
pub fn split_mul_and_act_order(
xs: &Tensor,
split_size: usize,
act: Activation,
order: GatedActivationOrder,
) -> Result<Tensor> {
let last_dim = xs.dim(D::Minus1)?;
let Some(expected_last_dim) = split_size.checked_mul(2) else {
hanzo_ml::bail!("split_mul_and_act split size overflow: {split_size}");
};
if last_dim != expected_last_dim {
hanzo_ml::bail!("split_mul_and_act expected last dim {expected_last_dim}, got {last_dim}");
}
let first = xs.narrow(D::Minus1, 0, split_size)?;
let second = xs.narrow(D::Minus1, split_size, split_size)?;
match order {
GatedActivationOrder::GateUp => mul_and_act(&first, &second, act),
GatedActivationOrder::UpGate => mul_and_act(&second, &first, act),
}
}
#[derive(Clone)]
pub(crate) struct MergedDenseProjection {
proj: Arc<dyn hanzo_quant::QuantMethod>,
output_dims: Vec<usize>,
}
impl MergedDenseProjection {
pub(crate) fn new(projs: &[&dyn hanzo_quant::QuantMethod]) -> Result<Option<Self>> {
let mut weights = Vec::with_capacity(projs.len());
let mut output_dims = Vec::with_capacity(projs.len());
let mut input_dim = None;
let mut dtype_device: Option<(DType, hanzo_ml::Device)> = None;
for proj in projs {
if proj.has_bias() {
return Ok(None);
}
let Some((weight, bias)) = proj.unquant_weight_bias() else {
return Ok(None);
};
if bias.is_some() {
return Ok(None);
}
let (rows, cols) = weight.dims2()?;
if let Some(input_dim) = input_dim {
if input_dim != cols {
return Ok(None);
}
} else {
input_dim = Some(cols);
}
let this_dtype_device = (weight.dtype(), weight.device().clone());
if let Some((dtype, device)) = dtype_device.as_ref() {
if *dtype != this_dtype_device.0
|| device.location() != this_dtype_device.1.location()
{
return Ok(None);
}
} else {
dtype_device = Some(this_dtype_device);
}
output_dims.push(rows);
weights.push(weight);
}
let weight_refs = weights.iter().collect::<Vec<_>>();
let weight = Tensor::cat(&weight_refs, 0)?;
let proj = <hanzo_quant::UnquantLinear as hanzo_quant::QuantMethod>::new(
hanzo_quant::QuantMethodConfig::Unquantized(hanzo_nn::Linear::new(weight, None)),
)?;
Ok(Some(Self {
proj: Arc::new(proj),
output_dims,
}))
}
pub(crate) fn forward(&self, xs: &Tensor) -> Result<Vec<Tensor>> {
let ys = self.proj.forward(xs)?;
let mut parts = Vec::with_capacity(self.output_dims.len());
let mut offset = 0;
for &dim in &self.output_dims {
parts.push(ys.narrow(D::Minus1, offset, dim)?);
offset += dim;
}
Ok(parts)
}
}
/// Feed-forward path for quantized gate/up/down projections.
pub(crate) fn quantized_ffn(
xs: &Tensor,
gate: &dyn hanzo_quant::QuantMethod,
up: &dyn hanzo_quant::QuantMethod,
down: &dyn hanzo_quant::QuantMethod,
act: Activation,
swiglu_limit: Option<f32>,
) -> Result<Tensor> {
// The fused gate-up kernels don't materialize gate/up, so they can't apply the SwiGLU clamp;
// fall through to the split path when a limit is set (`None` keeps the fast path unchanged).
#[cfg(feature = "cuda")]
if swiglu_limit.is_none() {
if let Some(activation_type) = glu_activation_type(act) {
if let Some(inter) =
hanzo_quant::try_fused_quantized_gate_up(xs, gate, up, activation_type)?
{
return down.forward(&inter);
}
}
}
#[cfg(feature = "metal")]
if swiglu_limit.is_none() {
if let Some(activation_type) = glu_activation_type(act) {
if let Some(inter) =
hanzo_quant::try_fused_gate_up_metal(xs, gate, up, activation_type)?
{
return down.forward(&inter);
}
}
}
let lhs = gate.forward(xs)?;
let rhs = up.forward(xs)?;
let (lhs, rhs) = swiglu_clamp(&lhs, &rhs, swiglu_limit)?;
let inter = mul_and_act(&lhs, &rhs, act)?;
down.forward(&inter)
}
pub(crate) fn qkv_projections(
xs: &Tensor,
q_proj: &dyn hanzo_quant::QuantMethod,
k_proj: &dyn hanzo_quant::QuantMethod,
v_proj: &dyn hanzo_quant::QuantMethod,
) -> Result<(Tensor, Tensor, Tensor)> {
#[cfg(feature = "cuda")]
if let Some(qkv) = hanzo_quant::try_fused_quantized_qkv(xs, q_proj, k_proj, v_proj)? {
return Ok(qkv);
}
#[cfg(feature = "metal")]
if let Some(qkv) = hanzo_quant::try_fused_qkv_metal(xs, q_proj, k_proj, v_proj)? {
return Ok(qkv);
}
Ok((
q_proj.forward(xs)?,
k_proj.forward(xs)?,
v_proj.forward(xs)?,
))
}
mod tests {
#[test]
fn test_deterministic_moe_routing_tie_break_lowest_expert() {
// Proof-of-AI: on equal/near-equal router scores, expert selection MUST be canonical
// (lowest expert index) and backend-independent, or a re-executing verifier routes to a
// different sub-network and an honest run false-rejects.
use super::deterministic_topk_indices;
use hanzo_ml::Tensor;
let device = hanzo_ml::Device::Cpu;
// Row 0: experts 1 and 3 tie at 5.0 (the two highest) -> top-2 = [1, 3], lower index
// ordered first (tests tie ORDERING).
// Row 1: all five equal at 7.0 -> top-2 = [0, 1], the two LOWEST indices (tests the
// boundary tie-break: which of the equal experts make the cut).
let scores = Tensor::from_vec(
vec![
1.0f32, 5.0, 2.0, 5.0, 4.0, /* row1 */ 7.0, 7.0, 7.0, 7.0, 7.0,
],
(2, 5),
&device,
)
.unwrap();
let idx = deterministic_topk_indices(&scores, 2).unwrap();
assert_eq!(
idx.to_vec2::<u32>().unwrap(),
vec![vec![1u32, 3], vec![0u32, 1]]
);
// Determinism: identical inputs -> identical selection, every time.
let idx2 = deterministic_topk_indices(&scores, 2).unwrap();
assert_eq!(
idx.to_vec2::<u32>().unwrap(),
idx2.to_vec2::<u32>().unwrap()
);
}
#[test]
fn moe_router_topk_canonical_only_inside_proof_context() {
// Wiring: proof-deterministic routing must engage IFF a poi transcript is being emitted
// (`poi_forward::prove`). Inside prove(), tied scores select the canonical lowest indices so
// prover and verifier agree; outside, proving() is false and the flag is never forced on.
use super::{moe_router_topk, MoeRouterScoreFunction, MoeRouterSelectedWeight};
use hanzo_ml::Tensor;
let device = hanzo_ml::Device::Cpu;
// Row 0: experts 1,3 tie highest -> canonical top-2 = [1,3]. Row 1: all tie -> lowest two [0,1].
let logits = Tensor::from_vec(
vec![1.0f32, 5.0, 2.0, 5.0, 4.0, 7.0, 7.0, 7.0, 7.0, 7.0],
(2, 5),
&device,
)
.unwrap();
let config = super::MoeRouterTopKConfig {
top_k: 2,
score_function: MoeRouterScoreFunction::Raw,
selected_weight: MoeRouterSelectedWeight::Score,
renormalize: false,
norm_min: 0.0,
output_scale: 1.0,
logit_clip: None,
};
assert!(
!crate::poi_forward::proving(),
"no proof context by default"
);
let (out, _transcript) = crate::poi_forward::prove(|| {
assert!(
crate::poi_forward::proving(),
"prove() installs the context"
);
moe_router_topk(&logits, config, None, None).unwrap()
});
assert_eq!(
out.indices.to_vec2::<u32>().unwrap(),
vec![vec![1u32, 3], vec![0u32, 1]],
"proof context forces the canonical, backend-independent tie-break"
);
}
#[test]
fn swiglu_clamp_matches_reference_and_is_noop_when_none() {
use super::swiglu_clamp;
use hanzo_ml::Tensor;
let device = hanzo_ml::Device::Cpu;
let gate = Tensor::from_vec(vec![10.0f32, -10.0, 0.5], (1, 3), &device).unwrap();
let up = Tensor::from_vec(vec![10.0f32, -10.0, 0.5], (1, 3), &device).unwrap();
// None: exact passthrough.
let (g0, u0) = swiglu_clamp(&gate, &up, None).unwrap();
assert_eq!(g0.to_vec2::<f32>().unwrap(), gate.to_vec2::<f32>().unwrap());
assert_eq!(u0.to_vec2::<f32>().unwrap(), up.to_vec2::<f32>().unwrap());
// Some(1.0): DeepSeek-V4 clamp -> gate.clamp(max=1), up.clamp(-1, 1).
let (g1, u1) = swiglu_clamp(&gate, &up, Some(1.0)).unwrap();
assert_eq!(g1.to_vec2::<f32>().unwrap(), vec![vec![1.0f32, -10.0, 0.5]]);
assert_eq!(u1.to_vec2::<f32>().unwrap(), vec![vec![1.0f32, -1.0, 0.5]]);
}
#[test]
fn test_topk() {
use crate::ops::{TopKLastDimOp, TopKOutput};
use hanzo_ml::Tensor;
let device = hanzo_ml::Device::Cpu;
// [[1, 3, 5],
// [2, 4, 6]]
let x = Tensor::arange(1f32, 7f32, &device)
.unwrap()
.reshape((3, 2))
.unwrap()
.t()
.unwrap()
.contiguous()
.unwrap();
let TopKOutput { values, indices } = x.topk(2).unwrap();
assert_eq!(
x.to_vec2::<f32>().unwrap(),
vec![vec![1f32, 3f32, 5f32], vec![2f32, 4f32, 6f32]]
);
assert_eq!(
values.to_vec2::<f32>().unwrap(),
vec![vec![5f32, 3f32], vec![6f32, 4f32]]
);
assert_eq!(
indices.to_vec2::<u32>().unwrap(),
vec![vec![2u32, 1u32], vec![2u32, 1u32]]
);
}
#[test]
fn test_repeat_interleave() -> hanzo_ml::Result<()> {
use crate::ops::RepeatInterleaveOp;
use hanzo_ml::{Device, Tensor};
let input = Tensor::new(
vec![vec![vec![1f32, 2., 3.], vec![4f32, 5., 6.]]],
&Device::Cpu,
)?;
let repeat_interleaved = input.repeat_interleave(2, 2)?;
assert_eq!(
repeat_interleaved.to_vec3::<f32>()?,
vec![vec![
vec![1., 1., 2., 2., 3., 3.],
vec![4., 4., 5., 5., 6., 6.]
]]
);
Ok(())
}
#[test]
fn test_repeat_interleave_flat() -> hanzo_ml::Result<()> {
use crate::ops::RepeatInterleaveOp;
use hanzo_ml::{Device, Tensor};
let input = Tensor::new(vec![1., 2., 3., 4.], &Device::Cpu)?;
let repeat_interleaved = input.repeat_interleave_flat(vec![1u32, 2u32, 3u32, 4u32])?;
assert_eq!(
repeat_interleaved.to_vec1::<f64>()?,
vec![1., 2., 2., 3., 3., 3., 4., 4., 4., 4.]
);
Ok(())
}
#[cfg(feature = "cuda")]
#[test]
fn test_cuda_topk_multiblock_matches_cpu() -> hanzo_ml::Result<()> {
use crate::ops::{TopKLastDimOp, TopKOutput};
use hanzo_ml::{DType, Device, Tensor};
let Ok(cuda) = Device::new_cuda(0) else {
return Ok(());
};
let nrows = 4usize;
let ncols = 32_768usize;
let k = 32usize;
let mut data = vec![0f32; nrows * ncols];
for r in 0..nrows {
for c in 0..ncols {
// Deterministic pattern spanning chunks + exact ties near top.
let v = ((c * 37 + r * 101) % 997) as f32 * 0.25 - 120.0;
data[r * ncols + c] = v;
}
// Inject planted ties across separate 4096-col chunks.
data[r * ncols + 5] = 900.0;
data[r * ncols + 4101] = 900.0;
data[r * ncols + 16390] = 950.0;
data[r * ncols + 28675] = 925.0;
}
let cpu_f32 = Tensor::from_vec(data, (nrows, ncols), &Device::Cpu)?;
let TopKOutput {
values: ref_v,
indices: ref_i,
} = cpu_f32.topk(k)?;
let gpu_f32 = cpu_f32.to_device(&cuda)?;
let TopKOutput {
values: gpu_v,
indices: gpu_i,
} = gpu_f32.topk(k)?;
assert_eq!(
gpu_i.to_vec2::<u32>()?,
ref_i.to_vec2::<u32>()?,
"F32 multiblock topk indices mismatch"
);
assert_eq!(
gpu_v.to_vec2::<f32>()?,
ref_v.to_vec2::<f32>()?,
"F32 multiblock topk values mismatch"
);
// Also test BF16 at DFlash vocab scale (248,320 cols).
let ncols_big = 248_320usize;
let nrows_big = 3usize;
let mut big = vec![0f32; nrows_big * ncols_big];
for r in 0..nrows_big {
for c in 0..ncols_big {
big[r * ncols_big + c] = ((c * 13 + r * 71) % 251) as f32 - 125.0;
}
big[r * ncols_big + 123] = 512.0;
big[r * ncols_big + 123_456] = 512.0;
big[r * ncols_big + 240_000] = 1024.0;
}
let cpu_bf16 =
Tensor::from_vec(big, (nrows_big, ncols_big), &Device::Cpu)?.to_dtype(DType::BF16)?;
let TopKOutput {
values: ref_bv,
indices: ref_bi,
} = cpu_bf16.topk(k)?;
let gpu_bf16 = cpu_bf16.to_device(&cuda)?;
let TopKOutput {
values: gpu_bv,
indices: gpu_bi,
} = gpu_bf16.topk(k)?;
assert_eq!(
gpu_bi.to_vec2::<u32>()?,
ref_bi.to_vec2::<u32>()?,
"BF16 248K multiblock topk indices mismatch"
);
assert_eq!(
gpu_bv.to_dtype(DType::F32)?.to_vec2::<f32>()?,
ref_bv.to_dtype(DType::F32)?.to_vec2::<f32>()?,
"BF16 248K multiblock topk values mismatch"
);
// Partial trailing chunk, non-power-of-two ncols, max multiblock k=128, ties, and -inf.
let ncols_odd = 10_003usize;
let nrows_odd = 2usize;
let k_max = 128usize;
let mut odd = vec![0f32; nrows_odd * ncols_odd];
for r in 0..nrows_odd {
for c in 0..ncols_odd {
odd[r * ncols_odd + c] = if c % 97 == 0 {
f32::NEG_INFINITY
} else {
((c * 19 + r * 53) % 1031) as f32 * 0.5 - 250.0
};
}
odd[r * ncols_odd + 4095] = 777.0;
odd[r * ncols_odd + 4096] = 777.0;
odd[r * ncols_odd + ncols_odd - 1] = 888.0;
}
let cpu_odd = Tensor::from_vec(odd, (nrows_odd, ncols_odd), &Device::Cpu)?;
let TopKOutput {
values: ref_ov,
indices: ref_oi,
} = cpu_odd.topk(k_max)?;
let gpu_odd = cpu_odd.to_device(&cuda)?;
let TopKOutput {
values: gpu_ov,
indices: gpu_oi,
} = gpu_odd.topk(k_max)?;
assert_eq!(
gpu_oi.to_vec2::<u32>()?,
ref_oi.to_vec2::<u32>()?,
"F32 partial-chunk k=128 multiblock topk indices mismatch"
);
assert_eq!(
gpu_ov.to_vec2::<f32>()?,
ref_ov.to_vec2::<f32>()?,
"F32 partial-chunk k=128 multiblock topk values mismatch"
);
let cpu_f16 = cpu_odd.to_dtype(DType::F16)?;
let TopKOutput {
values: ref_hv,
indices: ref_hi,
} = cpu_f16.topk(k_max)?;
let gpu_f16 = cpu_f16.to_device(&cuda)?;
let TopKOutput {
values: gpu_hv,
indices: gpu_hi,
} = gpu_f16.topk(k_max)?;
assert_eq!(
gpu_hi.to_vec2::<u32>()?,
ref_hi.to_vec2::<u32>()?,
"F16 partial-chunk k=128 multiblock topk indices mismatch"
);
assert_eq!(
gpu_hv.to_dtype(DType::F32)?.to_vec2::<f32>()?,
ref_hv.to_dtype(DType::F32)?.to_vec2::<f32>()?,
"F16 partial-chunk k=128 multiblock topk values mismatch"
);
Ok(())
}
}
#[cfg(all(test, feature = "cuda"))]
mod cuda_offset_tests {
use super::*;
use hanzo_ml::{DType, Device, Tensor};
#[test]
fn cuda_sliced_row_topk_respects_start_offset() -> hanzo_ml::Result<()> {
let Ok(cuda) = Device::new_cuda(0) else {
return Ok(());
};
let nrows = 4usize;
let ncols = 4096usize;
let mut data = vec![-10.0f32; nrows * ncols];
for r in 0..nrows {
let peak = 100 + r * 777;
data[r * ncols + peak] = 20.0 + r as f32;
data[r * ncols + peak + 1] = 10.0 + r as f32;
}
let logits = Tensor::from_vec(data, (1, nrows, ncols), &Device::Cpu)?.to_device(&cuda)?;
let mut top1_cache = None;
for r in 0..nrows {
let expected_idx = (100 + r * 777) as u32;
let row_tensor = logits.narrow(1, r, 1)?.squeeze(0)?.squeeze(0)?;
assert_eq!(row_tensor.layout().start_offset(), r * ncols);
let top1 = cuda_top1_logits_f32_cached(&row_tensor, &mut top1_cache)?;
assert_eq!(
top1[1] as u32, expected_idx,
"cuda_top1_logits_f32_cached row {r} read wrong offset"
);
let packed_out = cuda_topk_logits_f32_packed(&row_tensor, 4, 1.0)?;
let packed = packed_out.packed.to_vec1::<f32>()?;
assert_eq!(
packed[packed_out.k] as u32, expected_idx,
"cuda_topk_logits_f32_packed row {r} read wrong offset"
);
let large_out = cuda_topk_logits_f32(&row_tensor, 4, 1.0)?;
let large_idxs = large_out.indices.to_vec1::<u32>()?;
assert_eq!(
large_idxs[0], expected_idx,
"cuda_topk_logits_f32 row {r} read wrong offset"
);
let topk_f32 = cuda_topk(&row_tensor, 4)?;
assert_eq!(
topk_f32.indices.to_vec1::<u32>()?[0],
expected_idx,
"cuda_topk f32 row {r} read wrong offset"
);
let logits_bf16 = logits.to_dtype(DType::BF16)?;
let sliced_bf16 = logits_bf16.narrow(1, r, 1)?.squeeze(0)?.squeeze(0)?;
assert_eq!(sliced_bf16.layout().start_offset(), r * ncols);
let topk_bf16 = cuda_topk(&sliced_bf16, 4)?;
assert_eq!(
topk_bf16.indices.to_vec1::<u32>()?[0],
expected_idx,
"cuda_topk bf16 row {r} read wrong offset"
);
}
Ok(())
}
}