mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
3967 lines
145 KiB
Rust
3967 lines
145 KiB
Rust
use hanzo_ml::{shape::Dim, DType, Result, Tensor, D};
|
|
use std::sync::Arc;
|
|
|
|
#[cfg(feature = "cuda")]
|
|
use crate::cuda::ffi;
|
|
use crate::layers::Activation;
|
|
#[cfg(feature = "cuda")]
|
|
use hanzo_ml::Shape;
|
|
|
|
// ============================================================================
|
|
// Optimized parallel topk for CUDA
|
|
// Uses a dedicated kernel that's much faster than full sort for small k
|
|
// Single kernel call writes both values and indices - no post-processing needed
|
|
// ============================================================================
|
|
|
|
#[cfg(feature = "cuda")]
|
|
fn topk_launched(err: i32, dims: &[usize], k: usize) -> Result<()> {
|
|
if err == 0 {
|
|
Ok(())
|
|
} else {
|
|
hanzo_ml::bail!("topk kernel failed with CUDA error {err} on {dims:?} (k = {k})")
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
fn cuda_topk(input: &Tensor, k: usize) -> Result<TopKOutput> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::CudaStorageSlice;
|
|
use std::ffi::c_void;
|
|
|
|
// Full last-dim top-k over ALL rows (the kernel runs one block per row); MoE routers call this on
|
|
// per-token [tokens, experts] scores and need every token's experts, matching the CPU fallback.
|
|
let input = input.contiguous()?;
|
|
let dims = input.dims();
|
|
let ncols = *dims
|
|
.last()
|
|
.ok_or_else(|| hanzo_ml::Error::Msg("empty dims".to_string()))?;
|
|
if ncols == 0 || k == 0 || k > ncols {
|
|
hanzo_ml::bail!("topk of {k} over rows of width {ncols}");
|
|
}
|
|
let nrows = (input.elem_count() / ncols) as i32;
|
|
let ncols_i32 = ncols as i32;
|
|
let k_i32 = k as i32;
|
|
|
|
// Output shapes
|
|
let mut out_dims = dims.to_vec();
|
|
*out_dims.last_mut().unwrap() = k;
|
|
let out_elem_count = nrows as usize * k;
|
|
|
|
let (storage, layout) = input.storage_and_layout();
|
|
let storage = match &*storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_topk requires CUDA tensor"),
|
|
};
|
|
|
|
let dev = storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let stream_raw = stream.cu_stream() as i64;
|
|
|
|
let (src_ptr, src_guard) = match &storage.slice {
|
|
CudaStorageSlice::BF16(inp) => inp.device_ptr(inp.stream()),
|
|
CudaStorageSlice::F16(inp) => inp.device_ptr(inp.stream()),
|
|
CudaStorageSlice::F32(inp) => inp.device_ptr(inp.stream()),
|
|
_ => hanzo_ml::bail!("cuda_topk only supports BF16/F16/F32"),
|
|
};
|
|
let byte_offset = layout.start_offset() * input.dtype().size_in_bytes();
|
|
let src_ptr = unsafe { (src_ptr as *const u8).add(byte_offset) as *const c_void };
|
|
|
|
// Allocate both output buffers
|
|
let mut indices_dst = unsafe { dev.alloc::<u32>(out_elem_count) }?;
|
|
let (indices_ptr, indices_guard) = indices_dst.device_ptr_mut(&stream);
|
|
|
|
// For wide vocabulary-scale rows (4K..=256K cols, small batch, k <= 128), split across
|
|
// multi-block 4096-col chunks so Stage 1 saturates all SMs with register-held values.
|
|
let use_multiblock = (4096..=262_144).contains(&ncols) && k <= 128 && nrows > 0 && nrows <= 128;
|
|
let num_chunks = ncols.div_ceil(4096);
|
|
let mut mb_scratch = if use_multiblock {
|
|
let s1_count = (nrows as usize) * num_chunks * k;
|
|
let s1_vals = unsafe { dev.alloc::<f32>(s1_count) }?;
|
|
let s1_idxs = unsafe { dev.alloc::<u32>(s1_count) }?;
|
|
Some((s1_vals, s1_idxs))
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let (values_tensor, indices_tensor) = match input.dtype() {
|
|
DType::BF16 => {
|
|
let mut values_dst = unsafe { dev.alloc::<half::bf16>(out_elem_count) }?;
|
|
let (values_ptr, values_guard) = values_dst.device_ptr_mut(&stream);
|
|
|
|
let err = if let Some((ref mut s1_vals, ref mut s1_idxs)) = mb_scratch {
|
|
let (s1v_ptr, _s1v_g) = s1_vals.device_ptr_mut(&stream);
|
|
let (s1i_ptr, _s1i_g) = s1_idxs.device_ptr_mut(&stream);
|
|
unsafe {
|
|
ffi::topk_multiblock_bf16(
|
|
src_ptr,
|
|
s1v_ptr as *mut c_void,
|
|
s1i_ptr as *mut c_void,
|
|
values_ptr as *mut c_void,
|
|
indices_ptr as *mut c_void,
|
|
nrows,
|
|
ncols_i32,
|
|
k_i32,
|
|
stream_raw,
|
|
)
|
|
}
|
|
} else {
|
|
unsafe {
|
|
ffi::topk_bf16(
|
|
src_ptr,
|
|
values_ptr as *mut c_void,
|
|
indices_ptr as *mut c_void,
|
|
nrows,
|
|
ncols_i32,
|
|
k_i32,
|
|
stream_raw,
|
|
)
|
|
}
|
|
};
|
|
topk_launched(err, dims, k)?;
|
|
|
|
drop(values_guard);
|
|
drop(indices_guard);
|
|
|
|
let values_storage = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::BF16(values_dst),
|
|
device: dev.clone(),
|
|
};
|
|
let indices_storage = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::U32(indices_dst),
|
|
device: dev.clone(),
|
|
};
|
|
|
|
let values_tensor = Tensor::from((
|
|
hanzo_ml::Storage::Cuda(values_storage),
|
|
Shape::from_dims(&out_dims),
|
|
));
|
|
let indices_tensor = Tensor::from((
|
|
hanzo_ml::Storage::Cuda(indices_storage),
|
|
Shape::from_dims(&out_dims),
|
|
));
|
|
(values_tensor, indices_tensor)
|
|
}
|
|
DType::F16 => {
|
|
let mut values_dst = unsafe { dev.alloc::<half::f16>(out_elem_count) }?;
|
|
let (values_ptr, values_guard) = values_dst.device_ptr_mut(&stream);
|
|
|
|
let err = if let Some((ref mut s1_vals, ref mut s1_idxs)) = mb_scratch {
|
|
let (s1v_ptr, _s1v_g) = s1_vals.device_ptr_mut(&stream);
|
|
let (s1i_ptr, _s1i_g) = s1_idxs.device_ptr_mut(&stream);
|
|
unsafe {
|
|
ffi::topk_multiblock_f16(
|
|
src_ptr,
|
|
s1v_ptr as *mut c_void,
|
|
s1i_ptr as *mut c_void,
|
|
values_ptr as *mut c_void,
|
|
indices_ptr as *mut c_void,
|
|
nrows,
|
|
ncols_i32,
|
|
k_i32,
|
|
stream_raw,
|
|
)
|
|
}
|
|
} else {
|
|
unsafe {
|
|
ffi::topk_f16(
|
|
src_ptr,
|
|
values_ptr as *mut c_void,
|
|
indices_ptr as *mut c_void,
|
|
nrows,
|
|
ncols_i32,
|
|
k_i32,
|
|
stream_raw,
|
|
)
|
|
}
|
|
};
|
|
topk_launched(err, dims, k)?;
|
|
|
|
drop(values_guard);
|
|
drop(indices_guard);
|
|
|
|
let values_storage = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F16(values_dst),
|
|
device: dev.clone(),
|
|
};
|
|
let indices_storage = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::U32(indices_dst),
|
|
device: dev.clone(),
|
|
};
|
|
|
|
let values_tensor = Tensor::from((
|
|
hanzo_ml::Storage::Cuda(values_storage),
|
|
Shape::from_dims(&out_dims),
|
|
));
|
|
let indices_tensor = Tensor::from((
|
|
hanzo_ml::Storage::Cuda(indices_storage),
|
|
Shape::from_dims(&out_dims),
|
|
));
|
|
(values_tensor, indices_tensor)
|
|
}
|
|
DType::F32 => {
|
|
let mut values_dst = unsafe { dev.alloc::<f32>(out_elem_count) }?;
|
|
let (values_ptr, values_guard) = values_dst.device_ptr_mut(&stream);
|
|
|
|
let err = if let Some((ref mut s1_vals, ref mut s1_idxs)) = mb_scratch {
|
|
let (s1v_ptr, _s1v_g) = s1_vals.device_ptr_mut(&stream);
|
|
let (s1i_ptr, _s1i_g) = s1_idxs.device_ptr_mut(&stream);
|
|
unsafe {
|
|
ffi::topk_multiblock_f32(
|
|
src_ptr,
|
|
s1v_ptr as *mut c_void,
|
|
s1i_ptr as *mut c_void,
|
|
values_ptr as *mut c_void,
|
|
indices_ptr as *mut c_void,
|
|
nrows,
|
|
ncols_i32,
|
|
k_i32,
|
|
stream_raw,
|
|
)
|
|
}
|
|
} else {
|
|
unsafe {
|
|
ffi::topk_f32(
|
|
src_ptr,
|
|
values_ptr as *mut c_void,
|
|
indices_ptr as *mut c_void,
|
|
nrows,
|
|
ncols_i32,
|
|
k_i32,
|
|
stream_raw,
|
|
)
|
|
}
|
|
};
|
|
topk_launched(err, dims, k)?;
|
|
|
|
drop(values_guard);
|
|
drop(indices_guard);
|
|
|
|
let values_storage = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F32(values_dst),
|
|
device: dev.clone(),
|
|
};
|
|
let indices_storage = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::U32(indices_dst),
|
|
device: dev.clone(),
|
|
};
|
|
|
|
let values_tensor = Tensor::from((
|
|
hanzo_ml::Storage::Cuda(values_storage),
|
|
Shape::from_dims(&out_dims),
|
|
));
|
|
let indices_tensor = Tensor::from((
|
|
hanzo_ml::Storage::Cuda(indices_storage),
|
|
Shape::from_dims(&out_dims),
|
|
));
|
|
(values_tensor, indices_tensor)
|
|
}
|
|
dt => hanzo_ml::bail!("cuda_topk unsupported dtype: {:?}", dt),
|
|
};
|
|
|
|
drop(src_guard);
|
|
|
|
Ok(TopKOutput {
|
|
values: values_tensor,
|
|
indices: indices_tensor,
|
|
})
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy)]
|
|
#[allow(dead_code)]
|
|
pub enum MoeRouterScoreFunction {
|
|
Raw,
|
|
Softmax,
|
|
Sigmoid,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy)]
|
|
#[allow(dead_code)]
|
|
pub enum MoeRouterSelectedWeight {
|
|
Score,
|
|
Sigmoid,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy)]
|
|
pub struct MoeRouterTopKConfig {
|
|
pub top_k: usize,
|
|
pub score_function: MoeRouterScoreFunction,
|
|
pub selected_weight: MoeRouterSelectedWeight,
|
|
pub renormalize: bool,
|
|
pub norm_min: f32,
|
|
pub output_scale: f32,
|
|
pub logit_clip: Option<(f32, f32)>,
|
|
}
|
|
|
|
/// Deterministic top-k expert selection with a CANONICAL, backend-independent tie-break: order
|
|
/// each row by (score DESC, expert-index ASC) using f32 total ordering (NaN-safe), take the first
|
|
/// `k`. Unlike hanzo-ml's `.topk()` (whose tie-break is unspecified) and the separate CUDA bf16/f16
|
|
/// router kernel, this yields the SAME experts on every machine — the routing analog of the
|
|
/// lowest-id argmax tie-break the sampler now uses. `top_k <= num_experts` for every MoE config.
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
fn deterministic_topk_indices(scores: &Tensor, k: usize) -> Result<Tensor> {
|
|
let rows = scores.to_vec2::<f32>()?;
|
|
let device = scores.device();
|
|
let mut out: Vec<u32> = Vec::with_capacity(rows.len() * k);
|
|
for row in &rows {
|
|
let mut idx: Vec<u32> = (0..row.len() as u32).collect();
|
|
idx.sort_by(|&a, &b| row[b as usize].total_cmp(&row[a as usize]).then(a.cmp(&b)));
|
|
out.extend_from_slice(&idx[..k]);
|
|
}
|
|
Tensor::from_vec(out, (rows.len(), k), device)
|
|
}
|
|
|
|
pub fn moe_router_topk(
|
|
logits: &Tensor,
|
|
config: MoeRouterTopKConfig,
|
|
selection_bias: Option<&Tensor>,
|
|
expert_scale: Option<&Tensor>,
|
|
) -> Result<TopKOutput> {
|
|
// While a poi proof transcript is being emitted (inside `poi_forward::prove`), routing takes the
|
|
// single canonical CPU path so a prover and a re-executing verifier on different hardware select
|
|
// the IDENTICAL experts (a different expert is a different sub-network). Off during normal
|
|
// inference: `proving()` is false, so the fast CUDA/hanzo-ml path is byte-for-byte unchanged.
|
|
let proof = crate::poi_forward::proving();
|
|
#[cfg(feature = "cuda")]
|
|
if !proof
|
|
&& logits.device().is_cuda()
|
|
&& logits
|
|
.dims()
|
|
.last()
|
|
.is_some_and(|&e| crate::cuda::route::serves(e, config.top_k, &config))
|
|
{
|
|
return crate::cuda::route::topk(logits, config, selection_bias, expert_scale);
|
|
}
|
|
|
|
let logits = logits.to_dtype(DType::F32)?;
|
|
let logits = match config.logit_clip {
|
|
Some((min, max)) => logits.clamp(min as f64, max as f64)?,
|
|
None => logits,
|
|
};
|
|
let scores = match config.score_function {
|
|
MoeRouterScoreFunction::Raw => logits.clone(),
|
|
MoeRouterScoreFunction::Softmax => hanzo_nn::ops::softmax_last_dim(&logits)?,
|
|
MoeRouterScoreFunction::Sigmoid => hanzo_nn::ops::sigmoid(&logits)?,
|
|
};
|
|
let selection_scores = if let Some(selection_bias) = selection_bias {
|
|
scores.broadcast_add(&selection_bias.to_dtype(DType::F32)?)?
|
|
} else {
|
|
scores.clone()
|
|
};
|
|
let indices = if proof {
|
|
// canonical, backend-independent expert selection (Proof-of-AI)
|
|
deterministic_topk_indices(&selection_scores, config.top_k)?
|
|
} else {
|
|
selection_scores
|
|
.topk(config.top_k)?
|
|
.indices
|
|
.to_dtype(DType::U32)?
|
|
};
|
|
let mut values = match config.selected_weight {
|
|
MoeRouterSelectedWeight::Score => scores.gather(&indices, D::Minus1)?,
|
|
MoeRouterSelectedWeight::Sigmoid => {
|
|
hanzo_nn::ops::sigmoid(&logits.gather(&indices, D::Minus1)?)?
|
|
}
|
|
};
|
|
|
|
if config.renormalize {
|
|
let denominator = values.sum_keepdim(D::Minus1)?;
|
|
let denominator = if config.norm_min > 0.0 {
|
|
let min = Tensor::full(config.norm_min, denominator.shape(), denominator.device())?;
|
|
denominator.broadcast_maximum(&min)?
|
|
} else {
|
|
denominator
|
|
};
|
|
values = values.broadcast_div(&denominator)?;
|
|
}
|
|
if config.output_scale != 1.0 {
|
|
values = (values * config.output_scale as f64)?;
|
|
}
|
|
if let Some(expert_scale) = expert_scale {
|
|
let scales = expert_scale
|
|
.to_dtype(DType::F32)?
|
|
.index_select(&indices.flatten_all()?, 0)?
|
|
.reshape(indices.shape())?;
|
|
values = (values * scales)?;
|
|
}
|
|
|
|
Ok(TopKOutput { values, indices })
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
#[allow(dead_code)]
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
pub fn cuda_topk_logits_f32(
|
|
input: &Tensor,
|
|
k: usize,
|
|
temperature: f64,
|
|
) -> Result<TopKLogitsOutput> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::CudaStorageSlice;
|
|
|
|
const MAX_K: usize = 128;
|
|
const CHUNK_SIZE: usize = 2048;
|
|
const MAX_STAGE2_CANDIDATES: usize = 48 * 1024;
|
|
|
|
if temperature <= 0.0 || !temperature.is_finite() {
|
|
hanzo_ml::bail!("cuda_topk_logits_f32 requires a positive finite temperature");
|
|
}
|
|
|
|
let input = input.contiguous()?;
|
|
if input.dtype() != DType::F32 {
|
|
hanzo_ml::bail!("cuda_topk_logits_f32 requires F32 logits");
|
|
}
|
|
|
|
let ncols = input.elem_count();
|
|
if ncols == 0 {
|
|
hanzo_ml::bail!("cuda_topk_logits_f32 got empty logits");
|
|
}
|
|
let k = k.min(ncols);
|
|
if k == 0 || k > MAX_K {
|
|
hanzo_ml::bail!("cuda_topk_logits_f32 k={} must be in [1, {}]", k, MAX_K);
|
|
}
|
|
|
|
let nblocks = ncols.div_ceil(CHUNK_SIZE);
|
|
let stage2_candidates = nblocks * k;
|
|
if stage2_candidates > MAX_STAGE2_CANDIDATES {
|
|
hanzo_ml::bail!(
|
|
"cuda_topk_logits_f32 workspace too large: {} candidates",
|
|
stage2_candidates
|
|
);
|
|
}
|
|
|
|
let (storage, layout) = input.storage_and_layout();
|
|
let storage = match &*storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_topk_logits_f32 requires CUDA tensor"),
|
|
};
|
|
|
|
let dev = storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let stream_raw = stream.cu_stream() as i64;
|
|
|
|
let (src_ptr, src_guard) = match &storage.slice {
|
|
CudaStorageSlice::F32(inp) => inp.device_ptr(inp.stream()),
|
|
_ => hanzo_ml::bail!("cuda_topk_logits_f32 only supports F32"),
|
|
};
|
|
let src_ptr = unsafe { (src_ptr as *const f32).add(layout.start_offset()) };
|
|
|
|
let workspace_elems = nblocks * k;
|
|
let mut block_values = unsafe { dev.alloc::<f32>(workspace_elems) }?;
|
|
let mut block_indices = unsafe { dev.alloc::<u32>(workspace_elems) }?;
|
|
let mut block_maxes = unsafe { dev.alloc::<f32>(nblocks) }?;
|
|
let mut block_sums = unsafe { dev.alloc::<f32>(nblocks) }?;
|
|
let mut values_dst = unsafe { dev.alloc::<f32>(k) }?;
|
|
let mut indices_dst = unsafe { dev.alloc::<u32>(k) }?;
|
|
let mut softmax_info_dst = unsafe { dev.alloc::<f32>(2) }?;
|
|
|
|
let (block_values_ptr, block_values_guard) = block_values.device_ptr_mut(&stream);
|
|
let (block_indices_ptr, block_indices_guard) = block_indices.device_ptr_mut(&stream);
|
|
let (block_maxes_ptr, block_maxes_guard) = block_maxes.device_ptr_mut(&stream);
|
|
let (block_sums_ptr, block_sums_guard) = block_sums.device_ptr_mut(&stream);
|
|
let (values_ptr, values_guard) = values_dst.device_ptr_mut(&stream);
|
|
let (indices_ptr, indices_guard) = indices_dst.device_ptr_mut(&stream);
|
|
let (softmax_info_ptr, softmax_info_guard) = softmax_info_dst.device_ptr_mut(&stream);
|
|
|
|
unsafe {
|
|
ffi::topk_large_f32(
|
|
src_ptr,
|
|
block_values_ptr as *mut f32,
|
|
block_indices_ptr as *mut u32,
|
|
block_maxes_ptr as *mut f32,
|
|
block_sums_ptr as *mut f32,
|
|
values_ptr as *mut f32,
|
|
indices_ptr as *mut u32,
|
|
softmax_info_ptr as *mut f32,
|
|
ncols as i32,
|
|
k as i32,
|
|
CHUNK_SIZE as i32,
|
|
nblocks as i32,
|
|
(1.0 / temperature) as f32,
|
|
stream_raw,
|
|
);
|
|
}
|
|
|
|
drop(src_guard);
|
|
drop(block_values_guard);
|
|
drop(block_indices_guard);
|
|
drop(block_maxes_guard);
|
|
drop(block_sums_guard);
|
|
drop(values_guard);
|
|
drop(indices_guard);
|
|
drop(softmax_info_guard);
|
|
|
|
let values_storage = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F32(values_dst),
|
|
device: dev.clone(),
|
|
};
|
|
let indices_storage = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::U32(indices_dst),
|
|
device: dev.clone(),
|
|
};
|
|
let softmax_info_storage = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F32(softmax_info_dst),
|
|
device: dev.clone(),
|
|
};
|
|
let workspace = vec![
|
|
Tensor::from((
|
|
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F32(block_values),
|
|
device: dev.clone(),
|
|
}),
|
|
Shape::from_dims(&[workspace_elems]),
|
|
)),
|
|
Tensor::from((
|
|
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::U32(block_indices),
|
|
device: dev.clone(),
|
|
}),
|
|
Shape::from_dims(&[workspace_elems]),
|
|
)),
|
|
Tensor::from((
|
|
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F32(block_maxes),
|
|
device: dev.clone(),
|
|
}),
|
|
Shape::from_dims(&[nblocks]),
|
|
)),
|
|
Tensor::from((
|
|
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F32(block_sums),
|
|
device: dev.clone(),
|
|
}),
|
|
Shape::from_dims(&[nblocks]),
|
|
)),
|
|
];
|
|
|
|
Ok(TopKLogitsOutput {
|
|
values: Tensor::from((
|
|
hanzo_ml::Storage::Cuda(values_storage),
|
|
Shape::from_dims(&[k]),
|
|
)),
|
|
indices: Tensor::from((
|
|
hanzo_ml::Storage::Cuda(indices_storage),
|
|
Shape::from_dims(&[k]),
|
|
)),
|
|
softmax_info: Tensor::from((
|
|
hanzo_ml::Storage::Cuda(softmax_info_storage),
|
|
Shape::from_dims(&[2]),
|
|
)),
|
|
_workspace: workspace,
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
pub fn cuda_topk_logits_f32_packed(
|
|
input: &Tensor,
|
|
k: usize,
|
|
temperature: f64,
|
|
) -> Result<TopKLogitsPackedOutput> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::CudaStorageSlice;
|
|
|
|
const MAX_K: usize = 128;
|
|
const CHUNK_SIZE: usize = 2048;
|
|
const MAX_STAGE2_CANDIDATES: usize = 48 * 1024;
|
|
|
|
if temperature <= 0.0 || !temperature.is_finite() {
|
|
hanzo_ml::bail!("cuda_topk_logits_f32_packed requires a positive finite temperature");
|
|
}
|
|
|
|
let input = input.contiguous()?;
|
|
if input.dtype() != DType::F32 {
|
|
hanzo_ml::bail!("cuda_topk_logits_f32_packed requires F32 logits");
|
|
}
|
|
|
|
let ncols = input.elem_count();
|
|
if ncols == 0 {
|
|
hanzo_ml::bail!("cuda_topk_logits_f32_packed got empty logits");
|
|
}
|
|
let k = k.min(ncols);
|
|
if k == 0 || k > MAX_K {
|
|
hanzo_ml::bail!(
|
|
"cuda_topk_logits_f32_packed k={} must be in [1, {}]",
|
|
k,
|
|
MAX_K
|
|
);
|
|
}
|
|
|
|
let nblocks = ncols.div_ceil(CHUNK_SIZE);
|
|
let stage2_candidates = nblocks * k;
|
|
if stage2_candidates > MAX_STAGE2_CANDIDATES {
|
|
hanzo_ml::bail!(
|
|
"cuda_topk_logits_f32_packed workspace too large: {} candidates",
|
|
stage2_candidates
|
|
);
|
|
}
|
|
|
|
let (storage, layout) = input.storage_and_layout();
|
|
let storage = match &*storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_topk_logits_f32_packed requires CUDA tensor"),
|
|
};
|
|
|
|
let dev = storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let stream_raw = stream.cu_stream() as i64;
|
|
|
|
let (src_ptr, src_guard) = match &storage.slice {
|
|
CudaStorageSlice::F32(inp) => inp.device_ptr(inp.stream()),
|
|
_ => hanzo_ml::bail!("cuda_topk_logits_f32_packed only supports F32"),
|
|
};
|
|
let src_ptr = unsafe { (src_ptr as *const f32).add(layout.start_offset()) };
|
|
|
|
let workspace_elems = nblocks * k;
|
|
let mut block_values = unsafe { dev.alloc::<f32>(workspace_elems) }?;
|
|
let mut block_indices = unsafe { dev.alloc::<u32>(workspace_elems) }?;
|
|
let mut block_maxes = unsafe { dev.alloc::<f32>(nblocks) }?;
|
|
let mut block_sums = unsafe { dev.alloc::<f32>(nblocks) }?;
|
|
let mut packed_dst = unsafe { dev.alloc::<f32>(2 * k + 2) }?;
|
|
|
|
let (block_values_ptr, block_values_guard) = block_values.device_ptr_mut(&stream);
|
|
let (block_indices_ptr, block_indices_guard) = block_indices.device_ptr_mut(&stream);
|
|
let (block_maxes_ptr, block_maxes_guard) = block_maxes.device_ptr_mut(&stream);
|
|
let (block_sums_ptr, block_sums_guard) = block_sums.device_ptr_mut(&stream);
|
|
let (packed_ptr, packed_guard) = packed_dst.device_ptr_mut(&stream);
|
|
|
|
unsafe {
|
|
ffi::topk_large_f32_packed(
|
|
src_ptr,
|
|
block_values_ptr as *mut f32,
|
|
block_indices_ptr as *mut u32,
|
|
block_maxes_ptr as *mut f32,
|
|
block_sums_ptr as *mut f32,
|
|
packed_ptr as *mut f32,
|
|
ncols as i32,
|
|
k as i32,
|
|
CHUNK_SIZE as i32,
|
|
nblocks as i32,
|
|
(1.0 / temperature) as f32,
|
|
stream_raw,
|
|
);
|
|
}
|
|
|
|
drop(src_guard);
|
|
drop(block_values_guard);
|
|
drop(block_indices_guard);
|
|
drop(block_maxes_guard);
|
|
drop(block_sums_guard);
|
|
drop(packed_guard);
|
|
|
|
let packed_storage = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F32(packed_dst),
|
|
device: dev.clone(),
|
|
};
|
|
let workspace = vec![
|
|
Tensor::from((
|
|
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F32(block_values),
|
|
device: dev.clone(),
|
|
}),
|
|
Shape::from_dims(&[workspace_elems]),
|
|
)),
|
|
Tensor::from((
|
|
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::U32(block_indices),
|
|
device: dev.clone(),
|
|
}),
|
|
Shape::from_dims(&[workspace_elems]),
|
|
)),
|
|
Tensor::from((
|
|
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F32(block_maxes),
|
|
device: dev.clone(),
|
|
}),
|
|
Shape::from_dims(&[nblocks]),
|
|
)),
|
|
Tensor::from((
|
|
hanzo_ml::Storage::Cuda(hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::F32(block_sums),
|
|
device: dev.clone(),
|
|
}),
|
|
Shape::from_dims(&[nblocks]),
|
|
)),
|
|
];
|
|
|
|
Ok(TopKLogitsPackedOutput {
|
|
packed: Tensor::from((
|
|
hanzo_ml::Storage::Cuda(packed_storage),
|
|
Shape::from_dims(&[2 * k + 2]),
|
|
)),
|
|
k,
|
|
_workspace: workspace,
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
pub struct CudaTop1LogitsWorkspace {
|
|
ncols: usize,
|
|
nblocks: usize,
|
|
location: hanzo_ml::DeviceLocation,
|
|
block_values: hanzo_ml::cuda_backend::cudarc::driver::CudaSlice<f32>,
|
|
block_indices: hanzo_ml::cuda_backend::cudarc::driver::CudaSlice<u32>,
|
|
packed: hanzo_ml::cuda_backend::cudarc::driver::CudaSlice<f32>,
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
pub fn cuda_topk_softmax(input: &Tensor, k: usize) -> Result<TopKOutput> {
|
|
// The topk_softmax CUDA kernels (topk_softmax_{bf16,f16,f32}) are unimplemented on this
|
|
// branch (no .cu entry point, no FFI decl). This helper is only reachable from the gpt-oss
|
|
// MoE router, which the speech path never exercises; bail rather than call phantom symbols.
|
|
let _ = (input, k);
|
|
hanzo_ml::bail!("cuda_topk_softmax is not implemented on this build")
|
|
}
|
|
|
|
/// Cached single-pass top-1 over F32 logits, returning the packed [logit, index] pair.
|
|
/// Reuses a per-device workspace across calls. Used by the CUDA top_k==1 sampler fast path.
|
|
#[cfg(feature = "cuda")]
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
pub fn cuda_top1_logits_f32_cached(
|
|
input: &Tensor,
|
|
cache: &mut Option<CudaTop1LogitsWorkspace>,
|
|
) -> Result<[f32; 2]> {
|
|
use hanzo_ml::backend::{BackendDevice, BackendStorage};
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::CudaStorageSlice;
|
|
|
|
const CHUNK_SIZE: usize = 2048;
|
|
|
|
let input = input.contiguous()?;
|
|
if input.dtype() != DType::F32 {
|
|
hanzo_ml::bail!("cuda_top1_logits_f32_cached requires F32 logits");
|
|
}
|
|
let ncols = input.elem_count();
|
|
if ncols == 0 {
|
|
hanzo_ml::bail!("cuda_top1_logits_f32_cached got empty logits");
|
|
}
|
|
let nblocks = ncols.div_ceil(CHUNK_SIZE);
|
|
|
|
let (storage, layout) = input.storage_and_layout();
|
|
let storage = match &*storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_top1_logits_f32_cached requires CUDA tensor"),
|
|
};
|
|
|
|
let dev = storage.device();
|
|
let location = dev.location();
|
|
let stream = dev.cuda_stream();
|
|
let stream_raw = stream.cu_stream() as i64;
|
|
|
|
let needs_alloc = cache.as_ref().is_none_or(|workspace| {
|
|
workspace.ncols != ncols || workspace.nblocks != nblocks || workspace.location != location
|
|
});
|
|
if needs_alloc {
|
|
*cache = Some(CudaTop1LogitsWorkspace {
|
|
ncols,
|
|
nblocks,
|
|
location,
|
|
block_values: unsafe { dev.alloc::<f32>(nblocks) }?,
|
|
block_indices: unsafe { dev.alloc::<u32>(nblocks) }?,
|
|
packed: unsafe { dev.alloc::<f32>(2) }?,
|
|
});
|
|
}
|
|
|
|
let (src_ptr, src_guard) = match &storage.slice {
|
|
CudaStorageSlice::F32(inp) => inp.device_ptr(inp.stream()),
|
|
_ => hanzo_ml::bail!("cuda_top1_logits_f32_cached only supports F32"),
|
|
};
|
|
let src_ptr = unsafe { (src_ptr as *const f32).add(layout.start_offset()) };
|
|
|
|
let workspace = cache
|
|
.as_mut()
|
|
.expect("CUDA top-1 workspace was allocated above");
|
|
let (block_values_ptr, block_values_guard) = workspace.block_values.device_ptr_mut(&stream);
|
|
let (block_indices_ptr, block_indices_guard) = workspace.block_indices.device_ptr_mut(&stream);
|
|
let (packed_ptr, packed_guard) = workspace.packed.device_ptr_mut(&stream);
|
|
|
|
unsafe {
|
|
ffi::top1_large_f32_packed(
|
|
src_ptr,
|
|
block_values_ptr as *mut f32,
|
|
block_indices_ptr as *mut u32,
|
|
packed_ptr as *mut f32,
|
|
ncols as i32,
|
|
CHUNK_SIZE as i32,
|
|
nblocks as i32,
|
|
stream_raw,
|
|
);
|
|
}
|
|
|
|
drop(src_guard);
|
|
drop(block_values_guard);
|
|
drop(block_indices_guard);
|
|
drop(packed_guard);
|
|
|
|
dev.synchronize()?;
|
|
let packed = dev.clone_dtoh(&workspace.packed)?;
|
|
Ok([packed[0], packed[1]])
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
#[derive(Debug, Clone)]
|
|
struct ArgSort {
|
|
asc: bool,
|
|
last_dim: usize,
|
|
inplace: bool,
|
|
}
|
|
|
|
impl hanzo_ml::CustomOp1 for ArgSort {
|
|
fn name(&self) -> &'static str {
|
|
"argsort"
|
|
}
|
|
|
|
fn cpu_fwd(
|
|
&self,
|
|
_: &hanzo_ml::CpuStorage,
|
|
_: &hanzo_ml::Layout,
|
|
) -> Result<(hanzo_ml::CpuStorage, hanzo_ml::Shape)> {
|
|
panic!("not implemented!")
|
|
}
|
|
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
#[cfg(feature = "cuda")]
|
|
fn cuda_fwd(
|
|
&self,
|
|
storage: &hanzo_ml::CudaStorage,
|
|
layout: &hanzo_ml::Layout,
|
|
) -> Result<(hanzo_ml::CudaStorage, hanzo_ml::Shape)> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::DevicePtr;
|
|
use hanzo_ml::cuda_backend::CudaStorageSlice;
|
|
|
|
let dev = storage.device();
|
|
let elem_count = layout.shape().elem_count();
|
|
let ncols = self.last_dim as i32;
|
|
let nrows = elem_count as i32 / ncols;
|
|
let dst = unsafe { dev.alloc::<u32>(elem_count) }?;
|
|
|
|
use std::ffi::c_void;
|
|
|
|
let (src, _src_guard) = match &storage.slice {
|
|
CudaStorageSlice::U8(inp) => inp.device_ptr(inp.stream()),
|
|
CudaStorageSlice::U32(inp) => inp.device_ptr(inp.stream()),
|
|
CudaStorageSlice::I64(inp) => inp.device_ptr(inp.stream()),
|
|
CudaStorageSlice::BF16(inp) => inp.device_ptr(inp.stream()),
|
|
CudaStorageSlice::F16(inp) => inp.device_ptr(inp.stream()),
|
|
CudaStorageSlice::F32(inp) => inp.device_ptr(inp.stream()),
|
|
CudaStorageSlice::F64(inp) => inp.device_ptr(inp.stream()),
|
|
_ => hanzo_ml::bail!("Unexpected dtype in asort"),
|
|
};
|
|
let src_ptr = src as *const c_void;
|
|
let (dst_ptr, dst_guard) = dst.device_ptr(dst.stream());
|
|
let dst_ptr = dst_ptr as *mut c_void;
|
|
let stream = dev.cuda_stream().cu_stream() as i64;
|
|
unsafe {
|
|
if self.asc {
|
|
match storage.dtype() {
|
|
hanzo_ml::DType::U8 => {
|
|
ffi::asort_asc_u8(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::U32 => {
|
|
ffi::asort_asc_u32(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::I64 => {
|
|
ffi::asort_asc_i64(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::BF16 => {
|
|
ffi::asort_asc_bf16(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::F16 => {
|
|
ffi::asort_asc_f16(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::F32 => {
|
|
ffi::asort_asc_f32(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::F64 => {
|
|
ffi::asort_asc_f64(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
_ => hanzo_ml::bail!("Unexpected dtype in asort"),
|
|
}
|
|
} else {
|
|
match storage.dtype() {
|
|
hanzo_ml::DType::U8 => {
|
|
ffi::asort_desc_u8(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::U32 => {
|
|
ffi::asort_desc_u32(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::I64 => {
|
|
ffi::asort_desc_i64(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::BF16 => {
|
|
ffi::asort_desc_bf16(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::F16 => {
|
|
ffi::asort_desc_f16(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::F32 => {
|
|
ffi::asort_desc_f32(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
hanzo_ml::DType::F64 => {
|
|
ffi::asort_desc_f64(src_ptr, dst_ptr, nrows, ncols, self.inplace, stream)
|
|
}
|
|
_ => hanzo_ml::bail!("Unexpected dtype in asort"),
|
|
}
|
|
}
|
|
}
|
|
drop(dst_guard);
|
|
let dst_ret = hanzo_ml::cuda_backend::CudaStorage {
|
|
slice: CudaStorageSlice::U32(dst),
|
|
device: dev.clone(),
|
|
};
|
|
Ok((dst_ret, layout.shape().clone()))
|
|
}
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
pub trait ArgSortOp {
|
|
fn arg_sort(&self, asc: bool) -> Result<Tensor>;
|
|
fn sort(&self, asc: bool) -> Result<(Tensor, Tensor)>;
|
|
}
|
|
|
|
impl ArgSortOp for Tensor {
|
|
/// Returns the indices that sort the tensor along the last dimension.
|
|
///
|
|
/// If `asc` is `true`, sorting is in ascending order. Otherwise sorting is performed in
|
|
/// descending order. The sort is unstable so there is no guarantees on the final order when it
|
|
/// comes to ties.
|
|
fn arg_sort(&self, asc: bool) -> Result<Tensor> {
|
|
if !self.is_contiguous() {
|
|
return Err(hanzo_ml::Error::RequiresContiguous { op: "arg_sort" });
|
|
}
|
|
let last_dim = match self.dims().last() {
|
|
Some(last_dim) => *last_dim,
|
|
None => hanzo_ml::bail!("empty last-dim in arg-sort"),
|
|
};
|
|
// No need for a backward pass for arg sort.
|
|
self.apply_op1_no_bwd(&ArgSort {
|
|
asc,
|
|
last_dim,
|
|
inplace: false,
|
|
})
|
|
}
|
|
|
|
/// Sorts the tensor along the last dimension, returns the sorted tensor together with the
|
|
/// sorted indexes.
|
|
///
|
|
/// If `asc` is `true`, sorting is in ascending order. Otherwise sorting is performed in
|
|
/// descending order. The sort is unstable so there is no guarantees on the final order when it
|
|
/// comes to ties.
|
|
fn sort(&self, asc: bool) -> Result<(Tensor, Tensor)> {
|
|
if !self.is_contiguous() {
|
|
return Err(hanzo_ml::Error::RequiresContiguous { op: "arg_sort" });
|
|
}
|
|
let last_dim = match self.dims().last() {
|
|
Some(last_dim) => *last_dim,
|
|
None => hanzo_ml::bail!("empty last-dim in arg-sort"),
|
|
};
|
|
let sorted = self.copy()?;
|
|
|
|
let asort = sorted.apply_op1_no_bwd(&ArgSort {
|
|
asc,
|
|
last_dim,
|
|
inplace: true,
|
|
})?;
|
|
|
|
Ok((sorted, asort))
|
|
}
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
pub struct TopKOutput {
|
|
pub values: Tensor,
|
|
pub indices: Tensor,
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
pub struct TopKLogitsOutput {
|
|
pub values: Tensor,
|
|
pub indices: Tensor,
|
|
/// `[softmax_denominator, global_max]` for the full-vocabulary softmax at
|
|
/// the temperature used for top-k selection.
|
|
pub softmax_info: Tensor,
|
|
_workspace: Vec<Tensor>,
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
pub struct TopKLogitsPackedOutput {
|
|
/// Packed as `[values; indices_as_f32; softmax_denominator; global_max]`.
|
|
pub packed: Tensor,
|
|
pub k: usize,
|
|
_workspace: Vec<Tensor>,
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
pub fn cuda_softcap_f32(input: &Tensor, cap: f32) -> Result<Tensor> {
|
|
// The `softcap_f32` CUDA kernel is unimplemented on this branch (no .cu entry point, no FFI
|
|
// decl) and this helper has no callers; bail rather than reference a phantom symbol. The
|
|
// device-agnostic tanh-softcap path is available via `Tensor` ops where needed.
|
|
let _ = (input, cap);
|
|
hanzo_ml::bail!("cuda_softcap_f32 is not implemented on this build")
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
pub fn cuda_apply_sparse_penalties_f32(
|
|
input: &Tensor,
|
|
token_ids: &Tensor,
|
|
counts: &Tensor,
|
|
frequency_penalty: f32,
|
|
presence_penalty: f32,
|
|
repetition_penalty: f32,
|
|
) -> Result<Tensor> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
|
|
use std::ffi::c_void;
|
|
|
|
if input.dtype() != DType::F32 {
|
|
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires F32 logits");
|
|
}
|
|
if token_ids.dtype() != DType::U32 {
|
|
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires U32 token ids");
|
|
}
|
|
if counts.dtype() != DType::F32 {
|
|
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires F32 counts");
|
|
}
|
|
if token_ids.elem_count() != counts.elem_count() {
|
|
hanzo_ml::bail!(
|
|
"cuda_apply_sparse_penalties_f32 token ids/counts length mismatch: {} vs {}",
|
|
token_ids.elem_count(),
|
|
counts.elem_count()
|
|
);
|
|
}
|
|
if !token_ids.device().same_device(input.device())
|
|
|| !counts.device().same_device(input.device())
|
|
{
|
|
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 tensors must be on the same CUDA device");
|
|
}
|
|
|
|
let input = input.contiguous()?;
|
|
let token_ids = token_ids.contiguous()?;
|
|
let counts = counts.contiguous()?;
|
|
|
|
let elem_count = input.elem_count();
|
|
let n_tokens = token_ids.elem_count();
|
|
if elem_count == 0 {
|
|
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 got empty logits");
|
|
}
|
|
if elem_count > i32::MAX as usize {
|
|
hanzo_ml::bail!(
|
|
"cuda_apply_sparse_penalties_f32 input is too large: {elem_count} elements"
|
|
);
|
|
}
|
|
if n_tokens > i32::MAX as usize {
|
|
hanzo_ml::bail!(
|
|
"cuda_apply_sparse_penalties_f32 token list is too large: {n_tokens} elements"
|
|
);
|
|
}
|
|
let elem_count_i32 = i32::try_from(elem_count).map_err(hanzo_ml::Error::wrap)?;
|
|
let n_tokens_i32 = i32::try_from(n_tokens).map_err(hanzo_ml::Error::wrap)?;
|
|
|
|
let (input_storage, input_layout) = input.storage_and_layout();
|
|
let input_storage = match &*input_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires CUDA logits"),
|
|
};
|
|
let CudaStorageSlice::F32(src) = &input_storage.slice else {
|
|
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 only supports F32 logits");
|
|
};
|
|
|
|
let (token_storage, token_layout) = token_ids.storage_and_layout();
|
|
let token_storage = match &*token_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires CUDA token ids"),
|
|
};
|
|
let CudaStorageSlice::U32(token_src) = &token_storage.slice else {
|
|
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 only supports U32 token ids");
|
|
};
|
|
|
|
let (count_storage, count_layout) = counts.storage_and_layout();
|
|
let count_storage = match &*count_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 requires CUDA counts"),
|
|
};
|
|
let CudaStorageSlice::F32(count_src) = &count_storage.slice else {
|
|
hanzo_ml::bail!("cuda_apply_sparse_penalties_f32 only supports F32 counts");
|
|
};
|
|
|
|
let dev = input_storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let mut out = unsafe { dev.alloc::<f32>(elem_count) }?;
|
|
|
|
let (src_ptr, src_guard) = src.device_ptr(&stream);
|
|
let (token_ptr, token_guard) = token_src.device_ptr(&stream);
|
|
let (count_ptr, count_guard) = count_src.device_ptr(&stream);
|
|
let (out_ptr, out_guard) = out.device_ptr_mut(&stream);
|
|
|
|
let src_ptr = unsafe { (src_ptr as *const f32).add(input_layout.start_offset()) };
|
|
let token_ptr = unsafe { (token_ptr as *const u32).add(token_layout.start_offset()) };
|
|
let count_ptr = unsafe { (count_ptr as *const f32).add(count_layout.start_offset()) };
|
|
|
|
unsafe {
|
|
ffi::apply_sparse_penalties_f32(
|
|
src_ptr as *const c_void,
|
|
out_ptr as *mut c_void,
|
|
token_ptr,
|
|
count_ptr,
|
|
elem_count_i32,
|
|
n_tokens_i32,
|
|
frequency_penalty,
|
|
presence_penalty,
|
|
repetition_penalty,
|
|
stream.cu_stream() as i64,
|
|
);
|
|
}
|
|
|
|
drop(src_guard);
|
|
drop(token_guard);
|
|
drop(count_guard);
|
|
drop(out_guard);
|
|
|
|
let out_storage = CudaStorage {
|
|
slice: CudaStorageSlice::F32(out),
|
|
device: dev.clone(),
|
|
};
|
|
Ok(Tensor::from((
|
|
hanzo_ml::Storage::Cuda(out_storage),
|
|
input.shape().clone(),
|
|
)))
|
|
}
|
|
|
|
#[cfg(feature = "metal")]
|
|
pub fn metal_apply_sparse_penalties(
|
|
input: &Tensor,
|
|
token_ids: &Tensor,
|
|
counts: &Tensor,
|
|
frequency_penalty: f32,
|
|
presence_penalty: f32,
|
|
repetition_penalty: f32,
|
|
) -> Result<Tensor> {
|
|
use hanzo_ml::{backend::BackendStorage, MetalStorage, Shape, Storage};
|
|
|
|
if !matches!(input.dtype(), DType::F32 | DType::F16 | DType::BF16) {
|
|
hanzo_ml::bail!("metal_apply_sparse_penalties requires F32/F16/BF16 logits");
|
|
}
|
|
if token_ids.dtype() != DType::U32 || counts.dtype() != DType::F32 {
|
|
hanzo_ml::bail!("metal_apply_sparse_penalties token_ids must be u32, counts f32");
|
|
}
|
|
let dtype = input.dtype();
|
|
let n = input.elem_count();
|
|
let n_tokens = token_ids.elem_count();
|
|
if counts.elem_count() != n_tokens {
|
|
hanzo_ml::bail!("token_ids and counts length mismatch");
|
|
}
|
|
|
|
let input = input.contiguous()?;
|
|
let token_ids = token_ids.contiguous()?;
|
|
let counts = counts.contiguous()?;
|
|
|
|
let (input_s, input_l) = input.storage_and_layout();
|
|
let (tok_s, tok_l) = token_ids.storage_and_layout();
|
|
let (cnt_s, cnt_l) = counts.storage_and_layout();
|
|
let (Storage::Metal(input_s), Storage::Metal(tok_s), Storage::Metal(cnt_s)) =
|
|
(&*input_s, &*tok_s, &*cnt_s)
|
|
else {
|
|
hanzo_ml::bail!("metal_apply_sparse_penalties requires Metal tensors");
|
|
};
|
|
let device = input_s.device().clone();
|
|
|
|
let out_buf = device.new_buffer(n, dtype, "penalties-out")?;
|
|
let encoder = device.command_encoder()?;
|
|
encoder.set_label("penalties-copy");
|
|
{
|
|
use hanzo_quant::metal_kernels::Kernels;
|
|
hanzo_quant::metal_kernels::call_copy_logits(
|
|
device.device(),
|
|
&encoder,
|
|
&Kernels::new(),
|
|
dtype,
|
|
input_s.buffer(),
|
|
input_l.start_offset() * input.dtype().size_in_bytes(),
|
|
&out_buf,
|
|
n,
|
|
)
|
|
.map_err(|e| hanzo_ml::Error::Msg(format!("metal copy: {e}")))?;
|
|
}
|
|
encoder.set_label("penalties-apply");
|
|
hanzo_quant::metal_kernels::call_apply_sparse_penalties(
|
|
device.device(),
|
|
&encoder,
|
|
&hanzo_quant::metal_kernels::Kernels::new(),
|
|
dtype,
|
|
&out_buf,
|
|
tok_s.buffer(),
|
|
cnt_s.buffer(),
|
|
n,
|
|
n_tokens,
|
|
frequency_penalty,
|
|
presence_penalty,
|
|
repetition_penalty,
|
|
)
|
|
.map_err(|e| hanzo_ml::Error::Msg(format!("metal penalties: {e}")))?;
|
|
let _ = (tok_l, cnt_l);
|
|
Ok(Tensor::from((
|
|
Storage::Metal(MetalStorage::new(out_buf, device.clone(), n, dtype)),
|
|
Shape::from(input.dims()),
|
|
)))
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
pub(crate) fn try_cuda_rms_norm_strided_4d(
|
|
input: &Tensor,
|
|
weight: &Tensor,
|
|
eps: f32,
|
|
) -> Result<Option<Tensor>> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
|
|
use std::ffi::c_void;
|
|
|
|
if !input.device().is_cuda() || input.rank() != 4 {
|
|
return Ok(None);
|
|
}
|
|
|
|
let dtype = input.dtype();
|
|
if !matches!(dtype, DType::BF16 | DType::F16 | DType::F32) || weight.dtype() != dtype {
|
|
return Ok(None);
|
|
}
|
|
if !weight.device().same_device(input.device()) {
|
|
return Ok(None);
|
|
}
|
|
|
|
let (batch, heads, seq_len, head_dim) = input.dims4()?;
|
|
if weight.dims1()? != head_dim {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_strided_4d weight size {} does not match head dim {head_dim}",
|
|
weight.dims1()?
|
|
);
|
|
}
|
|
if input.elem_count() == 0 {
|
|
return Ok(None);
|
|
}
|
|
for (name, value) in [
|
|
("batch", batch),
|
|
("heads", heads),
|
|
("seq_len", seq_len),
|
|
("head_dim", head_dim),
|
|
] {
|
|
if value > i32::MAX as usize {
|
|
hanzo_ml::bail!("cuda_rms_norm_strided_4d {name} is too large: {value}");
|
|
}
|
|
}
|
|
|
|
let (input_storage, input_layout) = input.storage_and_layout();
|
|
if input_layout.is_contiguous() {
|
|
return Ok(None);
|
|
}
|
|
let input_storage = match &*input_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let weight = weight.contiguous()?;
|
|
let (weight_storage, weight_layout) = weight.storage_and_layout();
|
|
let weight_storage = match &*weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
|
|
let dev = input_storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let stream_ptr = stream.cu_stream() as i64;
|
|
let shape = input.shape().clone();
|
|
let elem_count = input.elem_count();
|
|
let stride = input_layout.stride();
|
|
let batch_i32 = i32::try_from(batch).map_err(hanzo_ml::Error::wrap)?;
|
|
let heads_i32 = i32::try_from(heads).map_err(hanzo_ml::Error::wrap)?;
|
|
let seq_len_i32 = i32::try_from(seq_len).map_err(hanzo_ml::Error::wrap)?;
|
|
let head_dim_i32 = i32::try_from(head_dim).map_err(hanzo_ml::Error::wrap)?;
|
|
|
|
macro_rules! launch {
|
|
($variant:ident, $ty:ty, $ffi_fn:ident) => {{
|
|
let CudaStorageSlice::$variant(src) = &input_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_strided_4d input dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(weight_src) = &weight_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_strided_4d weight dtype mismatch");
|
|
};
|
|
let mut out = unsafe { dev.alloc::<$ty>(elem_count) }?;
|
|
let (src_ptr, src_guard) = src.device_ptr(&stream);
|
|
let (weight_ptr, weight_guard) = weight_src.device_ptr(&stream);
|
|
let (out_ptr, out_guard) = out.device_ptr_mut(&stream);
|
|
let src_ptr = unsafe { (src_ptr as *const $ty).add(input_layout.start_offset()) };
|
|
let weight_ptr =
|
|
unsafe { (weight_ptr as *const $ty).add(weight_layout.start_offset()) };
|
|
|
|
unsafe {
|
|
ffi::$ffi_fn(
|
|
src_ptr as *const c_void,
|
|
weight_ptr as *const c_void,
|
|
out_ptr as *mut c_void,
|
|
stride[0] as i64,
|
|
stride[1] as i64,
|
|
stride[2] as i64,
|
|
stride[3] as i64,
|
|
batch_i32,
|
|
heads_i32,
|
|
seq_len_i32,
|
|
head_dim_i32,
|
|
eps,
|
|
stream_ptr,
|
|
);
|
|
}
|
|
|
|
drop(src_guard);
|
|
drop(weight_guard);
|
|
drop(out_guard);
|
|
|
|
let out_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(out),
|
|
device: dev.clone(),
|
|
};
|
|
Ok(Some(Tensor::from((
|
|
hanzo_ml::Storage::Cuda(out_storage),
|
|
shape,
|
|
))))
|
|
}};
|
|
}
|
|
|
|
match dtype {
|
|
DType::BF16 => launch!(BF16, half::bf16, rms_norm_strided_4d_bf16),
|
|
DType::F16 => launch!(F16, half::f16, rms_norm_strided_4d_f16),
|
|
DType::F32 => launch!(F32, f32, rms_norm_strided_4d_f32),
|
|
_ => Ok(None),
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
pub fn cuda_rms_norm_residual(
|
|
input: &Tensor,
|
|
residual: &Tensor,
|
|
weight: &Tensor,
|
|
scale: Option<&Tensor>,
|
|
eps: f32,
|
|
) -> Result<Tensor> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
|
|
use std::ffi::c_void;
|
|
|
|
if input.shape() != residual.shape() {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual input/residual shape mismatch: {:?} vs {:?}",
|
|
input.shape(),
|
|
residual.shape()
|
|
);
|
|
}
|
|
if input.dtype() != residual.dtype() || input.dtype() != weight.dtype() {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual dtype mismatch: input {:?}, residual {:?}, weight {:?}",
|
|
input.dtype(),
|
|
residual.dtype(),
|
|
weight.dtype()
|
|
);
|
|
}
|
|
if !matches!(input.dtype(), DType::BF16 | DType::F16 | DType::F32) {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual only supports BF16/F16/F32, got {:?}",
|
|
input.dtype()
|
|
);
|
|
}
|
|
if !residual.device().same_device(input.device())
|
|
|| !weight.device().same_device(input.device())
|
|
{
|
|
hanzo_ml::bail!("cuda_rms_norm_residual tensors must be on the same CUDA device");
|
|
}
|
|
if let Some(scale) = scale {
|
|
if scale.elem_count() != 1 {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual scale must have one element, got {}",
|
|
scale.elem_count()
|
|
);
|
|
}
|
|
if scale.dtype() != input.dtype() {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual scale dtype mismatch: input {:?}, scale {:?}",
|
|
input.dtype(),
|
|
scale.dtype()
|
|
);
|
|
}
|
|
if !scale.device().same_device(input.device()) {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual scale must be on the same CUDA device");
|
|
}
|
|
}
|
|
|
|
let ncols = input.dim(D::Minus1)?;
|
|
if weight.dims1()? != ncols {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual weight size {} does not match last dim {ncols}",
|
|
weight.dims1()?
|
|
);
|
|
}
|
|
let elem_count = input.elem_count();
|
|
if elem_count == 0 {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual got empty input");
|
|
}
|
|
let nrows = elem_count / ncols;
|
|
if nrows > i32::MAX as usize || ncols > i32::MAX as usize {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual input is too large: nrows={nrows}, ncols={ncols}");
|
|
}
|
|
let nrows_i32 = i32::try_from(nrows).map_err(hanzo_ml::Error::wrap)?;
|
|
let ncols_i32 = i32::try_from(ncols).map_err(hanzo_ml::Error::wrap)?;
|
|
|
|
let input = input.contiguous()?;
|
|
let residual = residual.contiguous()?;
|
|
let weight = weight.contiguous()?;
|
|
let scale = scale.map(Tensor::contiguous).transpose()?;
|
|
|
|
let (input_storage, input_layout) = input.storage_and_layout();
|
|
let input_storage = match &*input_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_rms_norm_residual requires CUDA input"),
|
|
};
|
|
let (residual_storage, residual_layout) = residual.storage_and_layout();
|
|
let residual_storage = match &*residual_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_rms_norm_residual requires CUDA residual"),
|
|
};
|
|
let (weight_storage, weight_layout) = weight.storage_and_layout();
|
|
let weight_storage = match &*weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_rms_norm_residual requires CUDA weight"),
|
|
};
|
|
let scale_storage_and_layout = scale.as_ref().map(|scale| scale.storage_and_layout());
|
|
|
|
let dev = input_storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let stream_ptr = stream.cu_stream() as i64;
|
|
let shape = input.shape().clone();
|
|
|
|
macro_rules! launch {
|
|
($variant:ident, $ty:ty, $ffi_fn:ident) => {{
|
|
let CudaStorageSlice::$variant(src) = &input_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual input dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(residual_src) = &residual_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual residual dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(weight_src) = &weight_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual weight dtype mismatch");
|
|
};
|
|
let (scale_ptr, scale_guard) =
|
|
if let Some((scale_storage, scale_layout)) = &scale_storage_and_layout {
|
|
let scale_storage = match &**scale_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_rms_norm_residual requires CUDA scale"),
|
|
};
|
|
let CudaStorageSlice::$variant(scale_src) = &scale_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual scale dtype mismatch");
|
|
};
|
|
let (scale_ptr, scale_guard) = scale_src.device_ptr(&stream);
|
|
(
|
|
unsafe { (scale_ptr as *const $ty).add(scale_layout.start_offset()) }
|
|
as *const c_void,
|
|
Some(scale_guard),
|
|
)
|
|
} else {
|
|
(std::ptr::null(), None)
|
|
};
|
|
|
|
let mut out = unsafe { dev.alloc::<$ty>(elem_count) }?;
|
|
let (src_ptr, src_guard) = src.device_ptr(&stream);
|
|
let (residual_ptr, residual_guard) = residual_src.device_ptr(&stream);
|
|
let (weight_ptr, weight_guard) = weight_src.device_ptr(&stream);
|
|
let (out_ptr, out_guard) = out.device_ptr_mut(&stream);
|
|
|
|
let src_ptr = unsafe { (src_ptr as *const $ty).add(input_layout.start_offset()) };
|
|
let residual_ptr =
|
|
unsafe { (residual_ptr as *const $ty).add(residual_layout.start_offset()) };
|
|
let weight_ptr =
|
|
unsafe { (weight_ptr as *const $ty).add(weight_layout.start_offset()) };
|
|
|
|
unsafe {
|
|
ffi::$ffi_fn(
|
|
src_ptr as *const c_void,
|
|
residual_ptr as *const c_void,
|
|
weight_ptr as *const c_void,
|
|
scale_ptr,
|
|
out_ptr as *mut c_void,
|
|
nrows_i32,
|
|
ncols_i32,
|
|
eps,
|
|
stream_ptr,
|
|
);
|
|
}
|
|
|
|
drop(src_guard);
|
|
drop(residual_guard);
|
|
drop(weight_guard);
|
|
drop(scale_guard);
|
|
drop(out_guard);
|
|
|
|
let out_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(out),
|
|
device: dev.clone(),
|
|
};
|
|
Ok(Tensor::from((hanzo_ml::Storage::Cuda(out_storage), shape)))
|
|
}};
|
|
}
|
|
|
|
match input.dtype() {
|
|
DType::BF16 => launch!(BF16, half::bf16, rms_norm_residual_bf16),
|
|
DType::F16 => launch!(F16, half::f16, rms_norm_residual_f16),
|
|
DType::F32 => launch!(F32, f32, rms_norm_residual_f32),
|
|
dtype => hanzo_ml::bail!("cuda_rms_norm_residual unsupported dtype {dtype:?}"),
|
|
}
|
|
}
|
|
|
|
/// Fused residual-add + rmsnorm on CUDA: returns Some((sum = input + residual, normed =
|
|
/// rmsnorm(sum)*weight)) in one launch, or None when the fused kernel does not cover the case (caller
|
|
/// falls back to a separate add + rms_norm). The mean is taken over the stored sum, so the result is
|
|
/// bit-identical to the fallback. The pre-norm-transformer pattern; the CUDA twin of rocm_rms_norm_of_sum.
|
|
#[cfg(feature = "cuda")]
|
|
pub fn cuda_rms_norm_of_sum(
|
|
input: &Tensor,
|
|
residual: &Tensor,
|
|
weight: &Tensor,
|
|
eps: f32,
|
|
) -> Result<Option<(Tensor, Tensor)>> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
|
|
use std::ffi::c_void;
|
|
|
|
if !matches!(input.dtype(), DType::BF16 | DType::F16 | DType::F32) {
|
|
return Ok(None);
|
|
}
|
|
if input.dtype() != residual.dtype() || input.dtype() != weight.dtype() {
|
|
return Ok(None);
|
|
}
|
|
if !input.device().is_cuda()
|
|
|| !residual.device().same_device(input.device())
|
|
|| !weight.device().same_device(input.device())
|
|
{
|
|
return Ok(None);
|
|
}
|
|
if input.shape() != residual.shape() {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_of_sum input/residual shape mismatch: {:?} vs {:?}",
|
|
input.shape(),
|
|
residual.shape()
|
|
);
|
|
}
|
|
let ncols = input.dim(D::Minus1)?;
|
|
if weight.dims1()? != ncols {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_of_sum weight size {} does not match last dim {ncols}",
|
|
weight.dims1()?
|
|
);
|
|
}
|
|
let elem_count = input.elem_count();
|
|
if elem_count == 0 {
|
|
hanzo_ml::bail!("cuda_rms_norm_of_sum got empty input");
|
|
}
|
|
let nrows = elem_count / ncols;
|
|
if nrows > i32::MAX as usize || ncols > i32::MAX as usize {
|
|
hanzo_ml::bail!("cuda_rms_norm_of_sum input too large: nrows={nrows}, ncols={ncols}");
|
|
}
|
|
let nrows_i32 = i32::try_from(nrows).map_err(hanzo_ml::Error::wrap)?;
|
|
let ncols_i32 = i32::try_from(ncols).map_err(hanzo_ml::Error::wrap)?;
|
|
|
|
let input = input.contiguous()?;
|
|
let residual = residual.contiguous()?;
|
|
let weight = weight.contiguous()?;
|
|
|
|
let (input_storage, input_layout) = input.storage_and_layout();
|
|
let input_storage = match &*input_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (residual_storage, residual_layout) = residual.storage_and_layout();
|
|
let residual_storage = match &*residual_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (weight_storage, weight_layout) = weight.storage_and_layout();
|
|
let weight_storage = match &*weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
|
|
let dev = input_storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let stream_ptr = stream.cu_stream() as i64;
|
|
let shape = input.shape().clone();
|
|
|
|
macro_rules! launch {
|
|
($variant:ident, $ty:ty, $ffi_fn:ident) => {{
|
|
let CudaStorageSlice::$variant(src) = &input_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_of_sum input dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(residual_src) = &residual_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_of_sum residual dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(weight_src) = &weight_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_of_sum weight dtype mismatch");
|
|
};
|
|
|
|
let mut sum_out = unsafe { dev.alloc::<$ty>(elem_count) }?;
|
|
let mut norm_out = unsafe { dev.alloc::<$ty>(elem_count) }?;
|
|
let (src_ptr, src_guard) = src.device_ptr(&stream);
|
|
let (residual_ptr, residual_guard) = residual_src.device_ptr(&stream);
|
|
let (weight_ptr, weight_guard) = weight_src.device_ptr(&stream);
|
|
let (sum_out_ptr, sum_out_guard) = sum_out.device_ptr_mut(&stream);
|
|
let (norm_out_ptr, norm_out_guard) = norm_out.device_ptr_mut(&stream);
|
|
|
|
let src_ptr = unsafe { (src_ptr as *const $ty).add(input_layout.start_offset()) };
|
|
let residual_ptr =
|
|
unsafe { (residual_ptr as *const $ty).add(residual_layout.start_offset()) };
|
|
let weight_ptr =
|
|
unsafe { (weight_ptr as *const $ty).add(weight_layout.start_offset()) };
|
|
|
|
unsafe {
|
|
ffi::$ffi_fn(
|
|
src_ptr as *const c_void,
|
|
residual_ptr as *const c_void,
|
|
weight_ptr as *const c_void,
|
|
sum_out_ptr as *mut c_void,
|
|
norm_out_ptr as *mut c_void,
|
|
nrows_i32,
|
|
ncols_i32,
|
|
eps,
|
|
stream_ptr,
|
|
);
|
|
}
|
|
|
|
drop(src_guard);
|
|
drop(residual_guard);
|
|
drop(weight_guard);
|
|
drop(sum_out_guard);
|
|
drop(norm_out_guard);
|
|
|
|
let sum_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(sum_out),
|
|
device: dev.clone(),
|
|
};
|
|
let norm_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(norm_out),
|
|
device: dev.clone(),
|
|
};
|
|
Ok(Some((
|
|
Tensor::from((hanzo_ml::Storage::Cuda(sum_storage), shape.clone())),
|
|
Tensor::from((hanzo_ml::Storage::Cuda(norm_storage), shape)),
|
|
)))
|
|
}};
|
|
}
|
|
|
|
match input.dtype() {
|
|
DType::BF16 => launch!(BF16, half::bf16, rms_norm_of_sum_bf16),
|
|
DType::F16 => launch!(F16, half::f16, rms_norm_of_sum_f16),
|
|
DType::F32 => launch!(F32, f32, rms_norm_of_sum_f32),
|
|
dtype => hanzo_ml::bail!("cuda_rms_norm_of_sum unsupported dtype {dtype:?}"),
|
|
}
|
|
}
|
|
|
|
/// Fused residual-add + rmsnorm: returns Some((sum = input + residual, normed = rmsnorm(sum)*weight))
|
|
/// in one ROCm launch, or None (caller falls back to a separate add + rms_norm) for any case the fused
|
|
/// kernel does not cover. Covers F32 (prefill/router) and F16 (the decode residual stream). Distinct
|
|
/// from `cuda_rms_norm_residual` (which is `residual + rmsnorm(x)`): this normalizes the SUM, the
|
|
/// pre-norm-transformer pattern.
|
|
#[cfg(feature = "rocm")]
|
|
pub fn rocm_rms_norm_of_sum(
|
|
input: &Tensor,
|
|
residual: &Tensor,
|
|
weight: &Tensor,
|
|
eps: f32,
|
|
) -> Result<Option<(Tensor, Tensor)>> {
|
|
if !matches!(input.dtype(), DType::F32 | DType::F16)
|
|
|| residual.dtype() != input.dtype()
|
|
|| weight.dtype() != input.dtype()
|
|
{
|
|
return Ok(None);
|
|
}
|
|
if !input.device().is_rocm()
|
|
|| !residual.device().same_device(input.device())
|
|
|| !weight.device().same_device(input.device())
|
|
{
|
|
return Ok(None);
|
|
}
|
|
if input.shape() != residual.shape() {
|
|
hanzo_ml::bail!(
|
|
"rocm_rms_norm_of_sum input/residual shape mismatch: {:?} vs {:?}",
|
|
input.shape(),
|
|
residual.shape()
|
|
);
|
|
}
|
|
let input = input.contiguous()?;
|
|
let residual = residual.contiguous()?;
|
|
let weight = weight.contiguous()?;
|
|
let (xs, xl) = input.storage_and_layout();
|
|
let xr = match &*xs {
|
|
hanzo_ml::Storage::Rocm(r) => r,
|
|
_ => return Ok(None),
|
|
};
|
|
let (rs, rl) = residual.storage_and_layout();
|
|
let rr = match &*rs {
|
|
hanzo_ml::Storage::Rocm(r) => r,
|
|
_ => return Ok(None),
|
|
};
|
|
let (ws, wl) = weight.storage_and_layout();
|
|
let wr = match &*ws {
|
|
hanzo_ml::Storage::Rocm(r) => r,
|
|
_ => return Ok(None),
|
|
};
|
|
let (sum, normed) = xr.add_rms_norm(xl, rr, rl, wr, wl, eps)?;
|
|
let shape = input.shape().clone();
|
|
let sum_t = Tensor::from((hanzo_ml::Storage::Rocm(sum), shape.clone()));
|
|
let normed_t = Tensor::from((hanzo_ml::Storage::Rocm(normed), shape));
|
|
Ok(Some((sum_t, normed_t)))
|
|
}
|
|
|
|
/// Fused (input + residual) then rmsnorm on Vulkan: returns (sum, normed) where sum = input + residual
|
|
/// (the new residual-stream value) and normed = rms_norm(sum) * weight, in one dispatch (the DSL
|
|
/// add_rmsnorm_blk kernel) with no barrier between the add and the norm. Bit-identical to a separate add
|
|
/// + rms_norm. Returns None (caller falls back) unless every tensor is F32 and Vulkan on one device --
|
|
/// the block kernel is f32. Mirrors [`rocm_rms_norm_of_sum`].
|
|
#[cfg(feature = "vulkan")]
|
|
pub fn vulkan_rms_norm_of_sum(
|
|
input: &Tensor,
|
|
residual: &Tensor,
|
|
weight: &Tensor,
|
|
eps: f32,
|
|
) -> Result<Option<(Tensor, Tensor)>> {
|
|
if input.dtype() != DType::F32 || residual.dtype() != DType::F32 || weight.dtype() != DType::F32
|
|
{
|
|
return Ok(None);
|
|
}
|
|
if !input.device().is_vulkan()
|
|
|| !residual.device().same_device(input.device())
|
|
|| !weight.device().same_device(input.device())
|
|
{
|
|
return Ok(None);
|
|
}
|
|
if input.shape() != residual.shape() {
|
|
hanzo_ml::bail!(
|
|
"vulkan_rms_norm_of_sum input/residual shape mismatch: {:?} vs {:?}",
|
|
input.shape(),
|
|
residual.shape()
|
|
);
|
|
}
|
|
let input = input.contiguous()?;
|
|
let residual = residual.contiguous()?;
|
|
let weight = weight.contiguous()?;
|
|
let (xs, xl) = input.storage_and_layout();
|
|
let xr = match &*xs {
|
|
hanzo_ml::Storage::Vulkan(v) => v,
|
|
_ => return Ok(None),
|
|
};
|
|
let (rs, rl) = residual.storage_and_layout();
|
|
let rr = match &*rs {
|
|
hanzo_ml::Storage::Vulkan(v) => v,
|
|
_ => return Ok(None),
|
|
};
|
|
let (ws, wl) = weight.storage_and_layout();
|
|
let wr = match &*ws {
|
|
hanzo_ml::Storage::Vulkan(v) => v,
|
|
_ => return Ok(None),
|
|
};
|
|
let (sum, normed) = xr.add_rmsnorm(xl, rr, rl, wr, wl, eps)?;
|
|
let shape = input.shape().clone();
|
|
let sum_t = Tensor::from((hanzo_ml::Storage::Vulkan(sum), shape.clone()));
|
|
let normed_t = Tensor::from((hanzo_ml::Storage::Vulkan(normed), shape));
|
|
Ok(Some((sum_t, normed_t)))
|
|
}
|
|
|
|
#[cfg(feature = "metal")]
|
|
pub fn metal_rms_norm_residual(
|
|
input: &Tensor,
|
|
residual: &Tensor,
|
|
weight: &Tensor,
|
|
scale: Option<&Tensor>,
|
|
eps: f32,
|
|
) -> Result<Option<Tensor>> {
|
|
use hanzo_ml::{backend::BackendStorage, MetalStorage, Shape, Storage};
|
|
|
|
if input.shape() != residual.shape() {
|
|
return Ok(None);
|
|
}
|
|
let n_cols = input.dim(D::Minus1)?;
|
|
if weight.dims1()? != n_cols {
|
|
return Ok(None);
|
|
}
|
|
let n_rows = input.elem_count() / n_cols;
|
|
if n_rows == 0 {
|
|
return Ok(None);
|
|
}
|
|
if let Some(scale) = scale {
|
|
if scale.elem_count() != 1 {
|
|
return Ok(None);
|
|
}
|
|
}
|
|
|
|
let input = input.contiguous()?;
|
|
let residual = residual.contiguous()?;
|
|
let weight = weight.contiguous()?;
|
|
let scale_t = scale.map(Tensor::contiguous).transpose()?;
|
|
|
|
let (input_storage, input_layout) = input.storage_and_layout();
|
|
let Storage::Metal(input_storage) = &*input_storage else {
|
|
return Ok(None);
|
|
};
|
|
let (residual_storage, residual_layout) = residual.storage_and_layout();
|
|
let Storage::Metal(residual_storage) = &*residual_storage else {
|
|
return Ok(None);
|
|
};
|
|
let (weight_storage, weight_layout) = weight.storage_and_layout();
|
|
let Storage::Metal(weight_storage) = &*weight_storage else {
|
|
return Ok(None);
|
|
};
|
|
let scale_storage_and_layout = scale_t.as_ref().map(|s| s.storage_and_layout());
|
|
let scale_metal = match scale_storage_and_layout.as_ref() {
|
|
Some((s, l)) => {
|
|
let Storage::Metal(s) = &**s else {
|
|
return Ok(None);
|
|
};
|
|
Some((s, l))
|
|
}
|
|
None => None,
|
|
};
|
|
|
|
let device = input_storage.device().clone();
|
|
let dtype = input.dtype();
|
|
let out_buf = device.new_buffer(input.elem_count(), dtype, "rmsnorm-residual-out")?;
|
|
|
|
let encoder = device.command_encoder()?;
|
|
encoder.set_label("rmsnorm-residual");
|
|
|
|
let x_offset = input_layout.start_offset() * dtype.size_in_bytes();
|
|
let res_offset = residual_layout.start_offset() * dtype.size_in_bytes();
|
|
let w_offset = weight_layout.start_offset() * dtype.size_in_bytes();
|
|
let scale_arg = scale_metal
|
|
.as_ref()
|
|
.map(|(s, l)| (s.buffer(), l.start_offset() * dtype.size_in_bytes()));
|
|
|
|
hanzo_quant::metal_kernels::call_rmsnorm_residual(
|
|
device.device(),
|
|
&encoder,
|
|
&hanzo_quant::metal_kernels::Kernels::new(),
|
|
dtype,
|
|
(input_storage.buffer(), x_offset),
|
|
(residual_storage.buffer(), res_offset),
|
|
(weight_storage.buffer(), w_offset),
|
|
scale_arg,
|
|
&out_buf,
|
|
n_cols,
|
|
n_rows,
|
|
eps,
|
|
)
|
|
.map_err(hanzo_ml::Error::wrap)?;
|
|
|
|
let out = Tensor::from((
|
|
Storage::Metal(MetalStorage::new(
|
|
out_buf,
|
|
device.clone(),
|
|
input.elem_count(),
|
|
dtype,
|
|
)),
|
|
Shape::from(input.dims()),
|
|
));
|
|
Ok(Some(out))
|
|
}
|
|
|
|
#[cfg(feature = "metal")]
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
pub fn metal_topk_logits_packed(
|
|
input: &Tensor,
|
|
k: usize,
|
|
temperature: f64,
|
|
) -> Result<TopKLogitsPackedOutput> {
|
|
use hanzo_ml::{backend::BackendStorage, MetalStorage, Shape, Storage};
|
|
|
|
const MAX_K: usize = 128;
|
|
const CHUNK_SIZE: usize = 2048;
|
|
|
|
if temperature <= 0.0 || !temperature.is_finite() {
|
|
hanzo_ml::bail!("metal_topk_logits_packed requires a positive finite temperature");
|
|
}
|
|
let input = input.contiguous()?;
|
|
if !matches!(input.dtype(), DType::F32 | DType::F16 | DType::BF16) {
|
|
hanzo_ml::bail!("metal_topk_logits_packed requires F32/F16/BF16 logits");
|
|
}
|
|
let dtype = input.dtype();
|
|
let ncols = input.elem_count();
|
|
if ncols == 0 {
|
|
hanzo_ml::bail!("metal_topk_logits_packed got empty logits");
|
|
}
|
|
let k = k.min(ncols);
|
|
if k == 0 || k > MAX_K {
|
|
hanzo_ml::bail!("metal_topk_logits_packed k={k} must be in [1, {MAX_K}]");
|
|
}
|
|
let nblocks = ncols.div_ceil(CHUNK_SIZE);
|
|
|
|
let (input_s, input_l) = input.storage_and_layout();
|
|
let Storage::Metal(input_s) = &*input_s else {
|
|
hanzo_ml::bail!("metal_topk_logits_packed requires Metal tensor");
|
|
};
|
|
let device = input_s.device().clone();
|
|
|
|
let block_values_buf = device.new_buffer(nblocks * k, DType::F32, "topk-block-values")?;
|
|
let block_indices_buf = device.new_buffer(nblocks * k, DType::U32, "topk-block-indices")?;
|
|
let block_maxes_buf = device.new_buffer(nblocks, DType::F32, "topk-block-maxes")?;
|
|
let block_sums_buf = device.new_buffer(nblocks, DType::F32, "topk-block-sums")?;
|
|
let packed_buf = device.new_buffer(2 * k + 2, DType::F32, "topk-packed")?;
|
|
|
|
let encoder = device.command_encoder()?;
|
|
encoder.set_label("topk-logits-packed");
|
|
|
|
let inv_temp = (1.0_f64 / temperature) as f32;
|
|
let input_offset = input_l.start_offset() * input.dtype().size_in_bytes();
|
|
|
|
hanzo_quant::metal_kernels::call_topk_logits_packed(
|
|
device.device(),
|
|
&encoder,
|
|
&hanzo_quant::metal_kernels::Kernels::new(),
|
|
dtype,
|
|
input_s.buffer(),
|
|
&block_values_buf,
|
|
&block_indices_buf,
|
|
&block_maxes_buf,
|
|
&block_sums_buf,
|
|
&packed_buf,
|
|
ncols,
|
|
k,
|
|
CHUNK_SIZE,
|
|
inv_temp,
|
|
)
|
|
.map_err(|e| hanzo_ml::Error::Msg(format!("metal_topk_logits_packed kernel error: {e}")))?;
|
|
let _ = (
|
|
input_offset,
|
|
&block_values_buf,
|
|
&block_indices_buf,
|
|
&block_maxes_buf,
|
|
&block_sums_buf,
|
|
);
|
|
|
|
let packed = Tensor::from((
|
|
Storage::Metal(MetalStorage::new(
|
|
packed_buf,
|
|
device.clone(),
|
|
2 * k + 2,
|
|
DType::F32,
|
|
)),
|
|
Shape::from(vec![2 * k + 2]),
|
|
));
|
|
Ok(TopKLogitsPackedOutput {
|
|
packed,
|
|
k,
|
|
_workspace: vec![],
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
pub fn cuda_rms_norm_residual_then_rms_norm(
|
|
input: &Tensor,
|
|
residual: &Tensor,
|
|
residual_weight: &Tensor,
|
|
scale: Option<&Tensor>,
|
|
norm_weight: &Tensor,
|
|
residual_eps: f32,
|
|
norm_eps: f32,
|
|
) -> Result<(Tensor, Tensor)> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
|
|
use std::ffi::c_void;
|
|
|
|
if input.shape() != residual.shape() {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm input/residual shape mismatch: {:?} vs {:?}",
|
|
input.shape(),
|
|
residual.shape()
|
|
);
|
|
}
|
|
if input.dtype() != residual.dtype()
|
|
|| input.dtype() != residual_weight.dtype()
|
|
|| input.dtype() != norm_weight.dtype()
|
|
{
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm dtype mismatch: input {:?}, residual {:?}, residual_weight {:?}, norm_weight {:?}",
|
|
input.dtype(),
|
|
residual.dtype(),
|
|
residual_weight.dtype(),
|
|
norm_weight.dtype()
|
|
);
|
|
}
|
|
if !matches!(input.dtype(), DType::BF16 | DType::F16 | DType::F32) {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm only supports BF16/F16/F32, got {:?}",
|
|
input.dtype()
|
|
);
|
|
}
|
|
if !residual.device().same_device(input.device())
|
|
|| !residual_weight.device().same_device(input.device())
|
|
|| !norm_weight.device().same_device(input.device())
|
|
{
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm tensors must be on the same CUDA device"
|
|
);
|
|
}
|
|
if let Some(scale) = scale {
|
|
if scale.elem_count() != 1 {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm scale must have one element, got {}",
|
|
scale.elem_count()
|
|
);
|
|
}
|
|
if scale.dtype() != input.dtype() {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm scale dtype mismatch: input {:?}, scale {:?}",
|
|
input.dtype(),
|
|
scale.dtype()
|
|
);
|
|
}
|
|
if !scale.device().same_device(input.device()) {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm scale must be on the same CUDA device"
|
|
);
|
|
}
|
|
}
|
|
|
|
let ncols = input.dim(D::Minus1)?;
|
|
if residual_weight.dims1()? != ncols {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm residual weight size {} does not match last dim {ncols}",
|
|
residual_weight.dims1()?
|
|
);
|
|
}
|
|
if norm_weight.dims1()? != ncols {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm norm weight size {} does not match last dim {ncols}",
|
|
norm_weight.dims1()?
|
|
);
|
|
}
|
|
let elem_count = input.elem_count();
|
|
if elem_count == 0 {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm got empty input");
|
|
}
|
|
let nrows = elem_count / ncols;
|
|
if nrows > i32::MAX as usize || ncols > i32::MAX as usize {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm input is too large: nrows={nrows}, ncols={ncols}"
|
|
);
|
|
}
|
|
let nrows_i32 = i32::try_from(nrows).map_err(hanzo_ml::Error::wrap)?;
|
|
let ncols_i32 = i32::try_from(ncols).map_err(hanzo_ml::Error::wrap)?;
|
|
|
|
let input = input.contiguous()?;
|
|
let residual = residual.contiguous()?;
|
|
let residual_weight = residual_weight.contiguous()?;
|
|
let norm_weight = norm_weight.contiguous()?;
|
|
let scale = scale.map(Tensor::contiguous).transpose()?;
|
|
|
|
let (input_storage, input_layout) = input.storage_and_layout();
|
|
let input_storage = match &*input_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm requires CUDA input"),
|
|
};
|
|
let (residual_storage, residual_layout) = residual.storage_and_layout();
|
|
let residual_storage = match &*residual_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm requires CUDA residual"),
|
|
};
|
|
let (residual_weight_storage, residual_weight_layout) = residual_weight.storage_and_layout();
|
|
let residual_weight_storage = match &*residual_weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm requires CUDA residual weight")
|
|
}
|
|
};
|
|
let (norm_weight_storage, norm_weight_layout) = norm_weight.storage_and_layout();
|
|
let norm_weight_storage = match &*norm_weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm requires CUDA norm weight"),
|
|
};
|
|
let scale_storage_and_layout = scale.as_ref().map(|scale| scale.storage_and_layout());
|
|
|
|
let dev = input_storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let stream_ptr = stream.cu_stream() as i64;
|
|
let shape = input.shape().clone();
|
|
|
|
macro_rules! launch {
|
|
($variant:ident, $ty:ty, $ffi_fn:ident) => {{
|
|
let CudaStorageSlice::$variant(src) = &input_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm input dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(residual_src) = &residual_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm residual dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(residual_weight_src) = &residual_weight_storage.slice
|
|
else {
|
|
hanzo_ml::bail!(
|
|
"cuda_rms_norm_residual_then_rms_norm residual weight dtype mismatch"
|
|
);
|
|
};
|
|
let CudaStorageSlice::$variant(norm_weight_src) = &norm_weight_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm norm weight dtype mismatch");
|
|
};
|
|
let (scale_ptr, scale_guard) = if let Some((scale_storage, scale_layout)) =
|
|
&scale_storage_and_layout
|
|
{
|
|
let scale_storage = match &**scale_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm requires CUDA scale")
|
|
}
|
|
};
|
|
let CudaStorageSlice::$variant(scale_src) = &scale_storage.slice else {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm scale dtype mismatch");
|
|
};
|
|
let (scale_ptr, scale_guard) = scale_src.device_ptr(&stream);
|
|
(
|
|
unsafe { (scale_ptr as *const $ty).add(scale_layout.start_offset()) }
|
|
as *const c_void,
|
|
Some(scale_guard),
|
|
)
|
|
} else {
|
|
(std::ptr::null(), None)
|
|
};
|
|
|
|
let mut residual_out = unsafe { dev.alloc::<$ty>(elem_count) }?;
|
|
let mut norm_out = unsafe { dev.alloc::<$ty>(elem_count) }?;
|
|
let (src_ptr, src_guard) = src.device_ptr(&stream);
|
|
let (residual_ptr, residual_guard) = residual_src.device_ptr(&stream);
|
|
let (residual_weight_ptr, residual_weight_guard) =
|
|
residual_weight_src.device_ptr(&stream);
|
|
let (norm_weight_ptr, norm_weight_guard) = norm_weight_src.device_ptr(&stream);
|
|
let (residual_out_ptr, residual_out_guard) = residual_out.device_ptr_mut(&stream);
|
|
let (norm_out_ptr, norm_out_guard) = norm_out.device_ptr_mut(&stream);
|
|
|
|
let src_ptr = unsafe { (src_ptr as *const $ty).add(input_layout.start_offset()) };
|
|
let residual_ptr =
|
|
unsafe { (residual_ptr as *const $ty).add(residual_layout.start_offset()) };
|
|
let residual_weight_ptr = unsafe {
|
|
(residual_weight_ptr as *const $ty).add(residual_weight_layout.start_offset())
|
|
};
|
|
let norm_weight_ptr =
|
|
unsafe { (norm_weight_ptr as *const $ty).add(norm_weight_layout.start_offset()) };
|
|
|
|
unsafe {
|
|
ffi::$ffi_fn(
|
|
src_ptr as *const c_void,
|
|
residual_ptr as *const c_void,
|
|
residual_weight_ptr as *const c_void,
|
|
scale_ptr,
|
|
norm_weight_ptr as *const c_void,
|
|
residual_out_ptr as *mut c_void,
|
|
norm_out_ptr as *mut c_void,
|
|
nrows_i32,
|
|
ncols_i32,
|
|
residual_eps,
|
|
norm_eps,
|
|
stream_ptr,
|
|
);
|
|
}
|
|
|
|
drop(src_guard);
|
|
drop(residual_guard);
|
|
drop(residual_weight_guard);
|
|
drop(norm_weight_guard);
|
|
drop(scale_guard);
|
|
drop(residual_out_guard);
|
|
drop(norm_out_guard);
|
|
|
|
let residual_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(residual_out),
|
|
device: dev.clone(),
|
|
};
|
|
let norm_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(norm_out),
|
|
device: dev.clone(),
|
|
};
|
|
Ok((
|
|
Tensor::from((hanzo_ml::Storage::Cuda(residual_storage), shape.clone())),
|
|
Tensor::from((hanzo_ml::Storage::Cuda(norm_storage), shape)),
|
|
))
|
|
}};
|
|
}
|
|
|
|
match input.dtype() {
|
|
DType::BF16 => launch!(BF16, half::bf16, rms_norm_residual_then_rms_norm_bf16),
|
|
DType::F16 => launch!(F16, half::f16, rms_norm_residual_then_rms_norm_f16),
|
|
DType::F32 => launch!(F32, f32, rms_norm_residual_then_rms_norm_f32),
|
|
dtype => {
|
|
hanzo_ml::bail!("cuda_rms_norm_residual_then_rms_norm unsupported dtype {dtype:?}")
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn try_cuda_qk_rms_norm_rope(
|
|
q: &Tensor,
|
|
k: Option<&Tensor>,
|
|
q_weight: &Tensor,
|
|
k_weight: Option<&Tensor>,
|
|
q_eps: f32,
|
|
k_eps: f32,
|
|
cos: &Tensor,
|
|
sin: &Tensor,
|
|
is_neox: bool,
|
|
) -> Result<Option<(Tensor, Option<Tensor>)>> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
|
|
use std::ffi::c_void;
|
|
|
|
if !q.device().is_cuda() {
|
|
return Ok(None);
|
|
}
|
|
|
|
let dtype = q.dtype();
|
|
if !matches!(dtype, DType::BF16 | DType::F16 | DType::F32)
|
|
|| q_weight.dtype() != dtype
|
|
|| k_weight.is_some_and(|weight| weight.dtype() != dtype)
|
|
|| cos.dtype() != dtype
|
|
|| sin.dtype() != dtype
|
|
{
|
|
return Ok(None);
|
|
}
|
|
|
|
if !q_weight.device().same_device(q.device())
|
|
|| !cos.device().same_device(q.device())
|
|
|| !sin.device().same_device(q.device())
|
|
|| k.is_some_and(|k| !k.device().same_device(q.device()) || k.dtype() != dtype)
|
|
|| k_weight.is_some_and(|weight| !weight.device().same_device(q.device()))
|
|
{
|
|
return Ok(None);
|
|
}
|
|
|
|
let (batch, q_heads, seq_len, head_dim) = q.dims4()?;
|
|
// The fused kernel is intended for prompt/multi-token attention prep.
|
|
// Decode rows are already cheap in the existing kernels, and this row-wise
|
|
// reduction launch is slower for seq_len == 1 on current CUDA targets.
|
|
if seq_len == 1 {
|
|
return Ok(None);
|
|
}
|
|
|
|
let (k_heads, k_elem_count) = if let Some(k) = k {
|
|
let (k_batch, k_heads, k_seq_len, k_head_dim) = k.dims4()?;
|
|
if (k_batch, k_seq_len, k_head_dim) != (batch, seq_len, head_dim) {
|
|
hanzo_ml::bail!(
|
|
"q/k shape mismatch for fused qk norm rope: {:?} vs {:?}",
|
|
q.shape(),
|
|
k.shape()
|
|
);
|
|
}
|
|
let Some(k_weight) = k_weight else {
|
|
hanzo_ml::bail!("missing k norm weight for fused qk norm rope");
|
|
};
|
|
if k_weight.dims1()? != head_dim {
|
|
hanzo_ml::bail!(
|
|
"k norm weight size {} does not match head dim {head_dim}",
|
|
k_weight.dims1()?
|
|
);
|
|
}
|
|
(k_heads, k.elem_count())
|
|
} else {
|
|
(0, 0)
|
|
};
|
|
|
|
if q_weight.dims1()? != head_dim {
|
|
hanzo_ml::bail!(
|
|
"q norm weight size {} does not match head dim {head_dim}",
|
|
q_weight.dims1()?
|
|
);
|
|
}
|
|
|
|
let (cos_rows, rot_dim) = cos.dims2()?;
|
|
if sin.dims2()? != (cos_rows, rot_dim) {
|
|
hanzo_ml::bail!(
|
|
"cos/sin shape mismatch for fused qk norm rope: {:?} vs {:?}",
|
|
cos.shape(),
|
|
sin.shape()
|
|
);
|
|
}
|
|
if rot_dim == 0 || rot_dim * 2 > head_dim {
|
|
return Ok(None);
|
|
}
|
|
|
|
let cos_batch_stride = if cos_rows == seq_len {
|
|
0
|
|
} else if cos_rows == batch * seq_len {
|
|
seq_len
|
|
} else {
|
|
hanzo_ml::bail!(
|
|
"cos/sin rows {cos_rows} do not match seq_len {seq_len} or batch*seq_len {}",
|
|
batch * seq_len
|
|
);
|
|
};
|
|
|
|
for (name, value) in [
|
|
("batch", batch),
|
|
("q_heads", q_heads),
|
|
("k_heads", k_heads),
|
|
("seq_len", seq_len),
|
|
("head_dim", head_dim),
|
|
("rot_dim", rot_dim),
|
|
("cos_batch_stride", cos_batch_stride),
|
|
] {
|
|
if value > i32::MAX as usize {
|
|
hanzo_ml::bail!("fused qk norm rope {name} is too large: {value}");
|
|
}
|
|
}
|
|
let batch_i32 = i32::try_from(batch).map_err(hanzo_ml::Error::wrap)?;
|
|
let q_heads_i32 = i32::try_from(q_heads).map_err(hanzo_ml::Error::wrap)?;
|
|
let k_heads_i32 = i32::try_from(k_heads).map_err(hanzo_ml::Error::wrap)?;
|
|
let seq_len_i32 = i32::try_from(seq_len).map_err(hanzo_ml::Error::wrap)?;
|
|
let head_dim_i32 = i32::try_from(head_dim).map_err(hanzo_ml::Error::wrap)?;
|
|
let rot_dim_i32 = i32::try_from(rot_dim).map_err(hanzo_ml::Error::wrap)?;
|
|
let cos_batch_stride_i32 = i32::try_from(cos_batch_stride).map_err(hanzo_ml::Error::wrap)?;
|
|
|
|
let cos = cos.contiguous()?;
|
|
let sin = sin.contiguous()?;
|
|
let q_weight = q_weight.contiguous()?;
|
|
let k_weight = k_weight.map(Tensor::contiguous).transpose()?;
|
|
|
|
let (q_storage, q_layout) = q.storage_and_layout();
|
|
let q_storage = match &*q_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let k_storage_and_layout = k.map(Tensor::storage_and_layout);
|
|
let (q_weight_storage, q_weight_layout) = q_weight.storage_and_layout();
|
|
let q_weight_storage = match &*q_weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let k_weight_storage_and_layout = k_weight.as_ref().map(Tensor::storage_and_layout);
|
|
let (cos_storage, cos_layout) = cos.storage_and_layout();
|
|
let cos_storage = match &*cos_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (sin_storage, sin_layout) = sin.storage_and_layout();
|
|
let sin_storage = match &*sin_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
|
|
let dev = q_storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let stream_ptr = stream.cu_stream() as i64;
|
|
let q_shape = Shape::from_dims(&[batch, q_heads, seq_len, head_dim]);
|
|
let k_shape = Shape::from_dims(&[batch, k_heads, seq_len, head_dim]);
|
|
let q_elem_count = q.elem_count();
|
|
|
|
let q_stride = q_layout.stride();
|
|
let k_stride = k_storage_and_layout
|
|
.as_ref()
|
|
.map(|(_, layout)| layout.stride())
|
|
.unwrap_or(&[0, 0, 0, 0]);
|
|
|
|
macro_rules! launch {
|
|
($variant:ident, $ty:ty, $dtype_id:expr) => {{
|
|
let CudaStorageSlice::$variant(q_src) = &q_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope q dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(q_weight_src) = &q_weight_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope q weight dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(cos_src) = &cos_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope cos dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(sin_src) = &sin_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope sin dtype mismatch");
|
|
};
|
|
|
|
let mut q_out_buf = unsafe { dev.alloc::<$ty>(q_elem_count) }?;
|
|
let mut k_out_buf = if k_elem_count == 0 {
|
|
None
|
|
} else {
|
|
Some(unsafe { dev.alloc::<$ty>(k_elem_count) }?)
|
|
};
|
|
|
|
let (q_ptr, q_guard) = q_src.device_ptr(&stream);
|
|
let q_ptr = unsafe { (q_ptr as *const $ty).add(q_layout.start_offset()) };
|
|
let (q_weight_ptr, q_weight_guard) = q_weight_src.device_ptr(&stream);
|
|
let q_weight_ptr =
|
|
unsafe { (q_weight_ptr as *const $ty).add(q_weight_layout.start_offset()) };
|
|
let (cos_ptr, cos_guard) = cos_src.device_ptr(&stream);
|
|
let cos_ptr = unsafe { (cos_ptr as *const $ty).add(cos_layout.start_offset()) };
|
|
let (sin_ptr, sin_guard) = sin_src.device_ptr(&stream);
|
|
let sin_ptr = unsafe { (sin_ptr as *const $ty).add(sin_layout.start_offset()) };
|
|
|
|
let mut k_guard = None;
|
|
let k_ptr = if let Some((k_storage, k_layout)) = &k_storage_and_layout {
|
|
let k_storage = match &**k_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let CudaStorageSlice::$variant(k_src) = &k_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope k dtype mismatch");
|
|
};
|
|
let (ptr, guard) = k_src.device_ptr(&stream);
|
|
k_guard = Some(guard);
|
|
unsafe { (ptr as *const $ty).add(k_layout.start_offset()) }
|
|
} else {
|
|
std::ptr::null()
|
|
};
|
|
|
|
let mut k_weight_guard = None;
|
|
let k_weight_ptr =
|
|
if let Some((k_weight_storage, k_weight_layout)) = &k_weight_storage_and_layout {
|
|
let k_weight_storage = match &**k_weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let CudaStorageSlice::$variant(k_weight_src) = &k_weight_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope k weight dtype mismatch");
|
|
};
|
|
let (ptr, guard) = k_weight_src.device_ptr(&stream);
|
|
k_weight_guard = Some(guard);
|
|
unsafe { (ptr as *const $ty).add(k_weight_layout.start_offset()) }
|
|
} else {
|
|
q_weight_ptr
|
|
};
|
|
|
|
let (q_out_ptr, q_out_guard) = q_out_buf.device_ptr_mut(&stream);
|
|
let mut k_out_guard = None;
|
|
let k_out_ptr = if let Some(k_out_buf) = &mut k_out_buf {
|
|
let (ptr, guard) = k_out_buf.device_ptr_mut(&stream);
|
|
k_out_guard = Some(guard);
|
|
ptr as *mut $ty
|
|
} else {
|
|
std::ptr::null_mut()
|
|
};
|
|
|
|
unsafe {
|
|
ffi::qk_rms_norm_rope(
|
|
q_ptr as *const c_void,
|
|
k_ptr as *const c_void,
|
|
q_weight_ptr as *const c_void,
|
|
k_weight_ptr as *const c_void,
|
|
cos_ptr as *const c_void,
|
|
sin_ptr as *const c_void,
|
|
q_out_ptr as *mut c_void,
|
|
k_out_ptr as *mut c_void,
|
|
q_stride[0] as i64,
|
|
q_stride[1] as i64,
|
|
q_stride[2] as i64,
|
|
q_stride[3] as i64,
|
|
k_stride[0] as i64,
|
|
k_stride[1] as i64,
|
|
k_stride[2] as i64,
|
|
k_stride[3] as i64,
|
|
batch_i32,
|
|
q_heads_i32,
|
|
k_heads_i32,
|
|
seq_len_i32,
|
|
head_dim_i32,
|
|
rot_dim_i32,
|
|
cos_batch_stride_i32,
|
|
q_eps,
|
|
k_eps,
|
|
i32::from(is_neox),
|
|
$dtype_id,
|
|
stream_ptr,
|
|
);
|
|
}
|
|
|
|
drop(q_guard);
|
|
drop(q_weight_guard);
|
|
drop(cos_guard);
|
|
drop(sin_guard);
|
|
drop(k_guard);
|
|
drop(k_weight_guard);
|
|
drop(q_out_guard);
|
|
drop(k_out_guard);
|
|
|
|
let q_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(q_out_buf),
|
|
device: dev.clone(),
|
|
};
|
|
let q_tensor = Tensor::from((hanzo_ml::Storage::Cuda(q_storage), q_shape));
|
|
|
|
let k_tensor = if let Some(k_out_buf) = k_out_buf {
|
|
let k_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(k_out_buf),
|
|
device: dev.clone(),
|
|
};
|
|
Some(Tensor::from((hanzo_ml::Storage::Cuda(k_storage), k_shape)))
|
|
} else {
|
|
None
|
|
};
|
|
|
|
Ok(Some((q_tensor, k_tensor)))
|
|
}};
|
|
}
|
|
|
|
match dtype {
|
|
DType::BF16 => launch!(BF16, half::bf16, 1),
|
|
DType::F16 => launch!(F16, half::f16, 0),
|
|
DType::F32 => launch!(F32, f32, 2),
|
|
_ => Ok(None),
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn try_cuda_qk_rms_norm_rope_positions(
|
|
q: &Tensor,
|
|
k: Option<&Tensor>,
|
|
q_weight: &Tensor,
|
|
k_weight: Option<&Tensor>,
|
|
q_eps: f32,
|
|
k_eps: f32,
|
|
cos: &Tensor,
|
|
sin: &Tensor,
|
|
positions: &Tensor,
|
|
is_neox: bool,
|
|
) -> Result<Option<(Tensor, Option<Tensor>)>> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
|
|
use std::ffi::c_void;
|
|
|
|
if !q.device().is_cuda() {
|
|
return Ok(None);
|
|
}
|
|
|
|
let dtype = q.dtype();
|
|
if !matches!(dtype, DType::BF16 | DType::F16 | DType::F32)
|
|
|| q_weight.dtype() != dtype
|
|
|| k_weight.is_some_and(|weight| weight.dtype() != dtype)
|
|
|| cos.dtype() != dtype
|
|
|| sin.dtype() != dtype
|
|
|| positions.dtype() != DType::U32
|
|
{
|
|
return Ok(None);
|
|
}
|
|
|
|
if !q_weight.device().same_device(q.device())
|
|
|| !cos.device().same_device(q.device())
|
|
|| !sin.device().same_device(q.device())
|
|
|| !positions.device().same_device(q.device())
|
|
|| k.is_some_and(|k| !k.device().same_device(q.device()) || k.dtype() != dtype)
|
|
|| k_weight.is_some_and(|weight| !weight.device().same_device(q.device()))
|
|
{
|
|
return Ok(None);
|
|
}
|
|
|
|
let (batch, q_heads, seq_len, head_dim) = q.dims4()?;
|
|
if positions.dims1()? != batch {
|
|
hanzo_ml::bail!(
|
|
"positions length {} does not match batch size {batch}",
|
|
positions.dims1()?
|
|
);
|
|
}
|
|
|
|
let (k_heads, k_elem_count) = if let Some(k) = k {
|
|
let (k_batch, k_heads, k_seq_len, k_head_dim) = k.dims4()?;
|
|
if (k_batch, k_seq_len, k_head_dim) != (batch, seq_len, head_dim) {
|
|
hanzo_ml::bail!(
|
|
"q/k shape mismatch for fused qk norm rope positions: {:?} vs {:?}",
|
|
q.shape(),
|
|
k.shape()
|
|
);
|
|
}
|
|
let Some(k_weight) = k_weight else {
|
|
hanzo_ml::bail!("missing k norm weight for fused qk norm rope positions");
|
|
};
|
|
if k_weight.dims1()? != head_dim {
|
|
hanzo_ml::bail!(
|
|
"k norm weight size {} does not match head dim {head_dim}",
|
|
k_weight.dims1()?
|
|
);
|
|
}
|
|
(k_heads, k.elem_count())
|
|
} else {
|
|
(0, 0)
|
|
};
|
|
|
|
if q_weight.dims1()? != head_dim {
|
|
hanzo_ml::bail!(
|
|
"q norm weight size {} does not match head dim {head_dim}",
|
|
q_weight.dims1()?
|
|
);
|
|
}
|
|
|
|
let (cos_rows, rot_dim) = cos.dims2()?;
|
|
if sin.dims2()? != (cos_rows, rot_dim) {
|
|
hanzo_ml::bail!(
|
|
"cos/sin shape mismatch for fused qk norm rope positions: {:?} vs {:?}",
|
|
cos.shape(),
|
|
sin.shape()
|
|
);
|
|
}
|
|
if rot_dim == 0 || rot_dim * 2 > head_dim {
|
|
return Ok(None);
|
|
}
|
|
|
|
for (name, value) in [
|
|
("batch", batch),
|
|
("q_heads", q_heads),
|
|
("k_heads", k_heads),
|
|
("seq_len", seq_len),
|
|
("head_dim", head_dim),
|
|
("rot_dim", rot_dim),
|
|
] {
|
|
if value > i32::MAX as usize {
|
|
hanzo_ml::bail!("fused qk norm rope positions {name} is too large: {value}");
|
|
}
|
|
}
|
|
let batch_i32 = i32::try_from(batch).map_err(hanzo_ml::Error::wrap)?;
|
|
let q_heads_i32 = i32::try_from(q_heads).map_err(hanzo_ml::Error::wrap)?;
|
|
let k_heads_i32 = i32::try_from(k_heads).map_err(hanzo_ml::Error::wrap)?;
|
|
let seq_len_i32 = i32::try_from(seq_len).map_err(hanzo_ml::Error::wrap)?;
|
|
let head_dim_i32 = i32::try_from(head_dim).map_err(hanzo_ml::Error::wrap)?;
|
|
let rot_dim_i32 = i32::try_from(rot_dim).map_err(hanzo_ml::Error::wrap)?;
|
|
|
|
let cos = cos.contiguous()?;
|
|
let sin = sin.contiguous()?;
|
|
let positions = positions.contiguous()?;
|
|
let q_weight = q_weight.contiguous()?;
|
|
let k_weight = k_weight.map(Tensor::contiguous).transpose()?;
|
|
|
|
let (q_storage, q_layout) = q.storage_and_layout();
|
|
let q_storage = match &*q_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let k_storage_and_layout = k.map(Tensor::storage_and_layout);
|
|
let (q_weight_storage, q_weight_layout) = q_weight.storage_and_layout();
|
|
let q_weight_storage = match &*q_weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let k_weight_storage_and_layout = k_weight.as_ref().map(Tensor::storage_and_layout);
|
|
let (cos_storage, cos_layout) = cos.storage_and_layout();
|
|
let cos_storage = match &*cos_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (sin_storage, sin_layout) = sin.storage_and_layout();
|
|
let sin_storage = match &*sin_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (positions_storage, positions_layout) = positions.storage_and_layout();
|
|
let positions_storage = match &*positions_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
|
|
let dev = q_storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let stream_ptr = stream.cu_stream() as i64;
|
|
let q_shape = Shape::from_dims(&[batch, q_heads, seq_len, head_dim]);
|
|
let k_shape = Shape::from_dims(&[batch, k_heads, seq_len, head_dim]);
|
|
let q_elem_count = q.elem_count();
|
|
|
|
let q_stride = q_layout.stride();
|
|
let k_stride = k_storage_and_layout
|
|
.as_ref()
|
|
.map(|(_, layout)| layout.stride())
|
|
.unwrap_or(&[0, 0, 0, 0]);
|
|
|
|
macro_rules! launch {
|
|
($variant:ident, $ty:ty, $dtype_id:expr) => {{
|
|
let CudaStorageSlice::$variant(q_src) = &q_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope positions q dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(q_weight_src) = &q_weight_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope positions q weight dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(cos_src) = &cos_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope positions cos dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(sin_src) = &sin_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope positions sin dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::U32(positions_src) = &positions_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope positions dtype mismatch");
|
|
};
|
|
|
|
let mut q_out_buf = unsafe { dev.alloc::<$ty>(q_elem_count) }?;
|
|
let mut k_out_buf = if k_elem_count == 0 {
|
|
None
|
|
} else {
|
|
Some(unsafe { dev.alloc::<$ty>(k_elem_count) }?)
|
|
};
|
|
|
|
let (q_ptr, q_guard) = q_src.device_ptr(&stream);
|
|
let q_ptr = unsafe { (q_ptr as *const $ty).add(q_layout.start_offset()) };
|
|
let (q_weight_ptr, q_weight_guard) = q_weight_src.device_ptr(&stream);
|
|
let q_weight_ptr =
|
|
unsafe { (q_weight_ptr as *const $ty).add(q_weight_layout.start_offset()) };
|
|
let (cos_ptr, cos_guard) = cos_src.device_ptr(&stream);
|
|
let cos_ptr = unsafe { (cos_ptr as *const $ty).add(cos_layout.start_offset()) };
|
|
let (sin_ptr, sin_guard) = sin_src.device_ptr(&stream);
|
|
let sin_ptr = unsafe { (sin_ptr as *const $ty).add(sin_layout.start_offset()) };
|
|
let (positions_ptr, positions_guard) = positions_src.device_ptr(&stream);
|
|
let positions_ptr =
|
|
unsafe { (positions_ptr as *const u32).add(positions_layout.start_offset()) };
|
|
|
|
let mut k_guard = None;
|
|
let k_ptr = if let Some((k_storage, k_layout)) = &k_storage_and_layout {
|
|
let k_storage = match &**k_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let CudaStorageSlice::$variant(k_src) = &k_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope positions k dtype mismatch");
|
|
};
|
|
let (ptr, guard) = k_src.device_ptr(&stream);
|
|
k_guard = Some(guard);
|
|
unsafe { (ptr as *const $ty).add(k_layout.start_offset()) }
|
|
} else {
|
|
std::ptr::null()
|
|
};
|
|
|
|
let mut k_weight_guard = None;
|
|
let k_weight_ptr =
|
|
if let Some((k_weight_storage, k_weight_layout)) = &k_weight_storage_and_layout {
|
|
let k_weight_storage = match &**k_weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let CudaStorageSlice::$variant(k_weight_src) = &k_weight_storage.slice else {
|
|
hanzo_ml::bail!("fused qk norm rope positions k weight dtype mismatch");
|
|
};
|
|
let (ptr, guard) = k_weight_src.device_ptr(&stream);
|
|
k_weight_guard = Some(guard);
|
|
unsafe { (ptr as *const $ty).add(k_weight_layout.start_offset()) }
|
|
} else {
|
|
q_weight_ptr
|
|
};
|
|
|
|
let (q_out_ptr, q_out_guard) = q_out_buf.device_ptr_mut(&stream);
|
|
let mut k_out_guard = None;
|
|
let k_out_ptr = if let Some(k_out_buf) = &mut k_out_buf {
|
|
let (ptr, guard) = k_out_buf.device_ptr_mut(&stream);
|
|
k_out_guard = Some(guard);
|
|
ptr as *mut $ty
|
|
} else {
|
|
std::ptr::null_mut()
|
|
};
|
|
|
|
unsafe {
|
|
ffi::qk_rms_norm_rope_positions(
|
|
q_ptr as *const c_void,
|
|
k_ptr as *const c_void,
|
|
q_weight_ptr as *const c_void,
|
|
k_weight_ptr as *const c_void,
|
|
cos_ptr as *const c_void,
|
|
sin_ptr as *const c_void,
|
|
positions_ptr as *const c_void,
|
|
q_out_ptr as *mut c_void,
|
|
k_out_ptr as *mut c_void,
|
|
q_stride[0] as i64,
|
|
q_stride[1] as i64,
|
|
q_stride[2] as i64,
|
|
q_stride[3] as i64,
|
|
k_stride[0] as i64,
|
|
k_stride[1] as i64,
|
|
k_stride[2] as i64,
|
|
k_stride[3] as i64,
|
|
batch_i32,
|
|
q_heads_i32,
|
|
k_heads_i32,
|
|
seq_len_i32,
|
|
head_dim_i32,
|
|
rot_dim_i32,
|
|
q_eps,
|
|
k_eps,
|
|
i32::from(is_neox),
|
|
$dtype_id,
|
|
stream_ptr,
|
|
);
|
|
}
|
|
|
|
drop(q_guard);
|
|
drop(q_weight_guard);
|
|
drop(cos_guard);
|
|
drop(sin_guard);
|
|
drop(positions_guard);
|
|
drop(k_guard);
|
|
drop(k_weight_guard);
|
|
drop(q_out_guard);
|
|
drop(k_out_guard);
|
|
|
|
let q_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(q_out_buf),
|
|
device: dev.clone(),
|
|
};
|
|
let q_tensor = Tensor::from((hanzo_ml::Storage::Cuda(q_storage), q_shape));
|
|
|
|
let k_tensor = if let Some(k_out_buf) = k_out_buf {
|
|
let k_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(k_out_buf),
|
|
device: dev.clone(),
|
|
};
|
|
Some(Tensor::from((hanzo_ml::Storage::Cuda(k_storage), k_shape)))
|
|
} else {
|
|
None
|
|
};
|
|
|
|
Ok(Some((q_tensor, k_tensor)))
|
|
}};
|
|
}
|
|
|
|
match dtype {
|
|
DType::BF16 => launch!(BF16, half::bf16, 1),
|
|
DType::F16 => launch!(F16, half::f16, 0),
|
|
DType::F32 => launch!(F32, f32, 2),
|
|
_ => Ok(None),
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn try_cuda_qkv_rms_norm_rope_positions(
|
|
q: &Tensor,
|
|
k: &Tensor,
|
|
v: &Tensor,
|
|
q_weight: &Tensor,
|
|
k_weight: &Tensor,
|
|
v_weight: &Tensor,
|
|
q_eps: f32,
|
|
k_eps: f32,
|
|
v_eps: f32,
|
|
cos: &Tensor,
|
|
sin: &Tensor,
|
|
positions: &Tensor,
|
|
is_neox: bool,
|
|
) -> Result<Option<(Tensor, Tensor, Tensor)>> {
|
|
use hanzo_ml::backend::BackendStorage;
|
|
use hanzo_ml::cuda_backend::cudarc::driver::{DevicePtr, DevicePtrMut};
|
|
use hanzo_ml::cuda_backend::{CudaStorage, CudaStorageSlice};
|
|
use std::ffi::c_void;
|
|
|
|
if !q.device().is_cuda() {
|
|
return Ok(None);
|
|
}
|
|
|
|
let dtype = q.dtype();
|
|
if !matches!(dtype, DType::BF16 | DType::F16 | DType::F32)
|
|
|| k.dtype() != dtype
|
|
|| v.dtype() != dtype
|
|
|| q_weight.dtype() != dtype
|
|
|| k_weight.dtype() != dtype
|
|
|| v_weight.dtype() != dtype
|
|
|| cos.dtype() != dtype
|
|
|| sin.dtype() != dtype
|
|
|| positions.dtype() != DType::U32
|
|
{
|
|
return Ok(None);
|
|
}
|
|
|
|
if !q_weight.device().same_device(q.device())
|
|
|| !k_weight.device().same_device(q.device())
|
|
|| !v_weight.device().same_device(q.device())
|
|
|| !cos.device().same_device(q.device())
|
|
|| !sin.device().same_device(q.device())
|
|
|| !positions.device().same_device(q.device())
|
|
|| !k.device().same_device(q.device())
|
|
|| !v.device().same_device(q.device())
|
|
{
|
|
return Ok(None);
|
|
}
|
|
|
|
let (batch, q_heads, seq_len, head_dim) = q.dims4()?;
|
|
let (k_batch, k_heads, k_seq_len, k_head_dim) = k.dims4()?;
|
|
let (v_batch, v_heads, v_seq_len, v_head_dim) = v.dims4()?;
|
|
if (k_batch, k_seq_len, k_head_dim) != (batch, seq_len, head_dim)
|
|
|| (v_batch, v_heads, v_seq_len, v_head_dim) != (batch, k_heads, seq_len, head_dim)
|
|
{
|
|
hanzo_ml::bail!(
|
|
"q/k/v shape mismatch for fused qkv norm rope positions: {:?}, {:?}, {:?}",
|
|
q.shape(),
|
|
k.shape(),
|
|
v.shape()
|
|
);
|
|
}
|
|
if positions.dims1()? != batch {
|
|
hanzo_ml::bail!(
|
|
"positions length {} does not match batch size {batch}",
|
|
positions.dims1()?
|
|
);
|
|
}
|
|
if q_weight.dims1()? != head_dim
|
|
|| k_weight.dims1()? != head_dim
|
|
|| v_weight.dims1()? != head_dim
|
|
{
|
|
hanzo_ml::bail!("qkv norm weight size does not match head dim {head_dim}");
|
|
}
|
|
|
|
let (cos_rows, rot_dim) = cos.dims2()?;
|
|
if sin.dims2()? != (cos_rows, rot_dim) {
|
|
hanzo_ml::bail!(
|
|
"cos/sin shape mismatch for fused qkv norm rope positions: {:?} vs {:?}",
|
|
cos.shape(),
|
|
sin.shape()
|
|
);
|
|
}
|
|
if rot_dim == 0 || rot_dim * 2 > head_dim {
|
|
return Ok(None);
|
|
}
|
|
|
|
for (name, value) in [
|
|
("batch", batch),
|
|
("q_heads", q_heads),
|
|
("k_heads", k_heads),
|
|
("seq_len", seq_len),
|
|
("head_dim", head_dim),
|
|
("rot_dim", rot_dim),
|
|
] {
|
|
if value > i32::MAX as usize {
|
|
hanzo_ml::bail!("fused qkv norm rope positions {name} is too large: {value}");
|
|
}
|
|
}
|
|
let batch_i32 = i32::try_from(batch).map_err(hanzo_ml::Error::wrap)?;
|
|
let q_heads_i32 = i32::try_from(q_heads).map_err(hanzo_ml::Error::wrap)?;
|
|
let k_heads_i32 = i32::try_from(k_heads).map_err(hanzo_ml::Error::wrap)?;
|
|
let seq_len_i32 = i32::try_from(seq_len).map_err(hanzo_ml::Error::wrap)?;
|
|
let head_dim_i32 = i32::try_from(head_dim).map_err(hanzo_ml::Error::wrap)?;
|
|
let rot_dim_i32 = i32::try_from(rot_dim).map_err(hanzo_ml::Error::wrap)?;
|
|
|
|
let cos = cos.contiguous()?;
|
|
let sin = sin.contiguous()?;
|
|
let positions = positions.contiguous()?;
|
|
let q_weight = q_weight.contiguous()?;
|
|
let k_weight = k_weight.contiguous()?;
|
|
let v_weight = v_weight.contiguous()?;
|
|
|
|
let (q_storage, q_layout) = q.storage_and_layout();
|
|
let q_storage = match &*q_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (k_storage, k_layout) = k.storage_and_layout();
|
|
let k_storage = match &*k_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (v_storage, v_layout) = v.storage_and_layout();
|
|
let v_storage = match &*v_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (q_weight_storage, q_weight_layout) = q_weight.storage_and_layout();
|
|
let q_weight_storage = match &*q_weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (k_weight_storage, k_weight_layout) = k_weight.storage_and_layout();
|
|
let k_weight_storage = match &*k_weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (v_weight_storage, v_weight_layout) = v_weight.storage_and_layout();
|
|
let v_weight_storage = match &*v_weight_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (cos_storage, cos_layout) = cos.storage_and_layout();
|
|
let cos_storage = match &*cos_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (sin_storage, sin_layout) = sin.storage_and_layout();
|
|
let sin_storage = match &*sin_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
let (positions_storage, positions_layout) = positions.storage_and_layout();
|
|
let positions_storage = match &*positions_storage {
|
|
hanzo_ml::Storage::Cuda(s) => s,
|
|
_ => return Ok(None),
|
|
};
|
|
|
|
let dev = q_storage.device();
|
|
let stream = dev.cuda_stream();
|
|
let stream_ptr = stream.cu_stream() as i64;
|
|
let q_shape = Shape::from_dims(&[batch, q_heads, seq_len, head_dim]);
|
|
let kv_shape = Shape::from_dims(&[batch, k_heads, seq_len, head_dim]);
|
|
let q_elem_count = q.elem_count();
|
|
let kv_elem_count = k.elem_count();
|
|
|
|
let q_stride = q_layout.stride();
|
|
let k_stride = k_layout.stride();
|
|
let v_stride = v_layout.stride();
|
|
|
|
macro_rules! launch {
|
|
($variant:ident, $ty:ty, $dtype_id:expr) => {{
|
|
let CudaStorageSlice::$variant(q_src) = &q_storage.slice else {
|
|
hanzo_ml::bail!("fused qkv norm rope positions q dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(k_src) = &k_storage.slice else {
|
|
hanzo_ml::bail!("fused qkv norm rope positions k dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(v_src) = &v_storage.slice else {
|
|
hanzo_ml::bail!("fused qkv norm rope positions v dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(q_weight_src) = &q_weight_storage.slice else {
|
|
hanzo_ml::bail!("fused qkv norm rope positions q weight dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(k_weight_src) = &k_weight_storage.slice else {
|
|
hanzo_ml::bail!("fused qkv norm rope positions k weight dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(v_weight_src) = &v_weight_storage.slice else {
|
|
hanzo_ml::bail!("fused qkv norm rope positions v weight dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(cos_src) = &cos_storage.slice else {
|
|
hanzo_ml::bail!("fused qkv norm rope positions cos dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::$variant(sin_src) = &sin_storage.slice else {
|
|
hanzo_ml::bail!("fused qkv norm rope positions sin dtype mismatch");
|
|
};
|
|
let CudaStorageSlice::U32(positions_src) = &positions_storage.slice else {
|
|
hanzo_ml::bail!("fused qkv norm rope positions dtype mismatch");
|
|
};
|
|
|
|
let mut q_out_buf = unsafe { dev.alloc::<$ty>(q_elem_count) }?;
|
|
let mut k_out_buf = unsafe { dev.alloc::<$ty>(kv_elem_count) }?;
|
|
let mut v_out_buf = unsafe { dev.alloc::<$ty>(kv_elem_count) }?;
|
|
|
|
let (q_ptr, q_guard) = q_src.device_ptr(&stream);
|
|
let q_ptr = unsafe { (q_ptr as *const $ty).add(q_layout.start_offset()) };
|
|
let (k_ptr, k_guard) = k_src.device_ptr(&stream);
|
|
let k_ptr = unsafe { (k_ptr as *const $ty).add(k_layout.start_offset()) };
|
|
let (v_ptr, v_guard) = v_src.device_ptr(&stream);
|
|
let v_ptr = unsafe { (v_ptr as *const $ty).add(v_layout.start_offset()) };
|
|
let (q_weight_ptr, q_weight_guard) = q_weight_src.device_ptr(&stream);
|
|
let q_weight_ptr =
|
|
unsafe { (q_weight_ptr as *const $ty).add(q_weight_layout.start_offset()) };
|
|
let (k_weight_ptr, k_weight_guard) = k_weight_src.device_ptr(&stream);
|
|
let k_weight_ptr =
|
|
unsafe { (k_weight_ptr as *const $ty).add(k_weight_layout.start_offset()) };
|
|
let (v_weight_ptr, v_weight_guard) = v_weight_src.device_ptr(&stream);
|
|
let v_weight_ptr =
|
|
unsafe { (v_weight_ptr as *const $ty).add(v_weight_layout.start_offset()) };
|
|
let (cos_ptr, cos_guard) = cos_src.device_ptr(&stream);
|
|
let cos_ptr = unsafe { (cos_ptr as *const $ty).add(cos_layout.start_offset()) };
|
|
let (sin_ptr, sin_guard) = sin_src.device_ptr(&stream);
|
|
let sin_ptr = unsafe { (sin_ptr as *const $ty).add(sin_layout.start_offset()) };
|
|
let (positions_ptr, positions_guard) = positions_src.device_ptr(&stream);
|
|
let positions_ptr =
|
|
unsafe { (positions_ptr as *const u32).add(positions_layout.start_offset()) };
|
|
|
|
let (q_out_ptr, q_out_guard) = q_out_buf.device_ptr_mut(&stream);
|
|
let (k_out_ptr, k_out_guard) = k_out_buf.device_ptr_mut(&stream);
|
|
let (v_out_ptr, v_out_guard) = v_out_buf.device_ptr_mut(&stream);
|
|
|
|
unsafe {
|
|
ffi::qkv_rms_norm_rope_positions(
|
|
q_ptr as *const c_void,
|
|
k_ptr as *const c_void,
|
|
v_ptr as *const c_void,
|
|
q_weight_ptr as *const c_void,
|
|
k_weight_ptr as *const c_void,
|
|
v_weight_ptr as *const c_void,
|
|
cos_ptr as *const c_void,
|
|
sin_ptr as *const c_void,
|
|
positions_ptr as *const c_void,
|
|
q_out_ptr as *mut c_void,
|
|
k_out_ptr as *mut c_void,
|
|
v_out_ptr as *mut c_void,
|
|
q_stride[0] as i64,
|
|
q_stride[1] as i64,
|
|
q_stride[2] as i64,
|
|
q_stride[3] as i64,
|
|
k_stride[0] as i64,
|
|
k_stride[1] as i64,
|
|
k_stride[2] as i64,
|
|
k_stride[3] as i64,
|
|
v_stride[0] as i64,
|
|
v_stride[1] as i64,
|
|
v_stride[2] as i64,
|
|
v_stride[3] as i64,
|
|
batch_i32,
|
|
q_heads_i32,
|
|
k_heads_i32,
|
|
seq_len_i32,
|
|
head_dim_i32,
|
|
rot_dim_i32,
|
|
q_eps,
|
|
k_eps,
|
|
v_eps,
|
|
i32::from(is_neox),
|
|
$dtype_id,
|
|
stream_ptr,
|
|
);
|
|
}
|
|
|
|
drop(q_guard);
|
|
drop(k_guard);
|
|
drop(v_guard);
|
|
drop(q_weight_guard);
|
|
drop(k_weight_guard);
|
|
drop(v_weight_guard);
|
|
drop(cos_guard);
|
|
drop(sin_guard);
|
|
drop(positions_guard);
|
|
drop(q_out_guard);
|
|
drop(k_out_guard);
|
|
drop(v_out_guard);
|
|
|
|
let q_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(q_out_buf),
|
|
device: dev.clone(),
|
|
};
|
|
let k_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(k_out_buf),
|
|
device: dev.clone(),
|
|
};
|
|
let v_storage = CudaStorage {
|
|
slice: CudaStorageSlice::$variant(v_out_buf),
|
|
device: dev.clone(),
|
|
};
|
|
Ok(Some((
|
|
Tensor::from((hanzo_ml::Storage::Cuda(q_storage), q_shape)),
|
|
Tensor::from((hanzo_ml::Storage::Cuda(k_storage), kv_shape.clone())),
|
|
Tensor::from((hanzo_ml::Storage::Cuda(v_storage), kv_shape)),
|
|
)))
|
|
}};
|
|
}
|
|
|
|
match dtype {
|
|
DType::BF16 => launch!(BF16, half::bf16, 1),
|
|
DType::F16 => launch!(F16, half::f16, 0),
|
|
DType::F32 => launch!(F32, f32, 2),
|
|
_ => Ok(None),
|
|
}
|
|
}
|
|
|
|
pub trait TopKLastDimOp {
|
|
/// Topk in the last dim. `values` retains a gradient but `indices` has none w.r.t self.
|
|
/// This expects a contiguous tensor.
|
|
/// Note: this implements torch.topk with sorted=True.
|
|
fn topk(&self, topk: usize) -> Result<TopKOutput>;
|
|
|
|
/// Topk in the last dim. `values` retains a gradient but `indices` has none w.r.t self.
|
|
/// This expects a contiguous tensor.
|
|
/// Note: this implements torch.topk with sorted=False.
|
|
fn topk_unsorted(&self, topk: usize) -> Result<TopKOutput>;
|
|
}
|
|
|
|
impl TopKLastDimOp for Tensor {
|
|
fn topk(&self, topk: usize) -> Result<TopKOutput> {
|
|
// Use optimized parallel topk kernel on CUDA
|
|
// Single kernel call, no post-processing overhead
|
|
#[cfg(feature = "cuda")]
|
|
if self.device().is_cuda() {
|
|
return cuda_topk(self, topk);
|
|
}
|
|
|
|
// Fallback: full sort (CPU or non-CUDA)
|
|
let (values, sorted_indices) = self.sort_last_dim(false)?;
|
|
|
|
let topk_indices = sorted_indices.narrow(D::Minus1, 0, topk)?.contiguous()?;
|
|
let topk_values = values.narrow(D::Minus1, 0, topk)?.contiguous()?;
|
|
Ok(TopKOutput {
|
|
values: topk_values,
|
|
indices: topk_indices,
|
|
})
|
|
}
|
|
|
|
fn topk_unsorted(&self, topk: usize) -> Result<TopKOutput> {
|
|
// Sorted descending
|
|
let TopKOutput { values, indices } = self.topk(topk)?;
|
|
// Reorder the indices ascending
|
|
#[cfg(feature = "cuda")]
|
|
let reorder_indices = indices.arg_sort(true)?;
|
|
#[cfg(not(feature = "cuda"))]
|
|
let reorder_indices = indices.arg_sort_last_dim(true)?;
|
|
let topk_indices_unsorted = indices
|
|
.to_dtype(DType::F32)?
|
|
.gather(&reorder_indices, D::Minus1)?
|
|
.to_dtype(DType::U32)?;
|
|
let topk_values_unsorted = values.gather(&reorder_indices, D::Minus1)?;
|
|
Ok(TopKOutput {
|
|
values: topk_values_unsorted,
|
|
indices: topk_indices_unsorted,
|
|
})
|
|
}
|
|
}
|
|
|
|
pub trait RepeatInterleaveOp {
|
|
fn repeat_interleave<D: Dim>(&self, repeats: usize, dim: D) -> Result<Tensor>;
|
|
fn repeat_interleave_flat(&self, repeats: Vec<u32>) -> Result<Tensor>;
|
|
}
|
|
|
|
impl RepeatInterleaveOp for Tensor {
|
|
fn repeat_interleave<D: Dim>(&self, repeats: usize, dim: D) -> Result<Tensor> {
|
|
let dim = dim.to_index(self.shape(), "repeat_interleave")?;
|
|
let dim_elements = self.dim(dim)?;
|
|
// For metal
|
|
assert!(self.dtype().is_float());
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
let indices = Tensor::new(
|
|
(0..dim_elements)
|
|
.flat_map(|i| vec![i as u32; repeats])
|
|
.collect::<Vec<_>>(),
|
|
self.device(),
|
|
)?;
|
|
self.index_select(&indices, dim)
|
|
}
|
|
|
|
fn repeat_interleave_flat(&self, repeats: Vec<u32>) -> Result<Tensor> {
|
|
let xs = self.flatten_all()?;
|
|
if repeats.len() != xs.dim(0)? {
|
|
hanzo_ml::bail!(
|
|
"repeats ({}) must match flattened self length ({})",
|
|
repeats.len(),
|
|
xs.dim(0)?
|
|
);
|
|
}
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
let indices = Tensor::new(
|
|
(0..xs.dim(0)?)
|
|
.flat_map(|i| vec![i as u32; repeats[i] as usize])
|
|
.collect::<Vec<_>>(),
|
|
xs.device(),
|
|
)?;
|
|
xs.index_select(&indices, 0)
|
|
}
|
|
}
|
|
|
|
pub trait SplitOp {
|
|
fn split<D: Dim>(&self, splits: &[usize], dim: D) -> Result<Vec<Tensor>>;
|
|
}
|
|
|
|
impl SplitOp for Tensor {
|
|
fn split<D: Dim>(&self, splits: &[usize], dim: D) -> Result<Vec<Tensor>> {
|
|
let dim = dim.to_index(self.shape(), "split")?;
|
|
let mut split_res = Vec::new();
|
|
let mut index = 0;
|
|
for split in splits {
|
|
split_res.push(self.narrow(dim, index, *split)?);
|
|
index += *split;
|
|
}
|
|
Ok(split_res)
|
|
}
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
pub trait BincountOp {
|
|
fn bincount(&self, minlength: u32) -> Result<Vec<u32>>;
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
fn bincount(values: &[u32], minlength: u32) -> Vec<u32> {
|
|
// let max_val = values.iter().max().copied().unwrap_or(0);
|
|
// let result_len = (max_val + 1).max(minlength);
|
|
// values.iter().fold(
|
|
// // Start with a histogram vector of zeros.
|
|
// vec![0u32; result_len as usize],
|
|
// // For each value, update the histogram.
|
|
// |mut histogram, &value| {
|
|
// histogram[value as usize] += 1;
|
|
// histogram
|
|
// },
|
|
// )
|
|
|
|
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
|
|
|
|
// Early return if there are no values.
|
|
if values.is_empty() {
|
|
return vec![0u32; minlength as usize];
|
|
}
|
|
|
|
// Compute the maximum value in parallel.
|
|
// SAFETY: We just checked that values is nonempty above, so max() will return Some.
|
|
// Using expect() for clearer error message if this invariant is somehow violated.
|
|
let max_val = *values
|
|
.par_iter()
|
|
.max()
|
|
.expect("values should be non-empty after empty check");
|
|
|
|
// The histogram length must cover all observed values as well as `minlength`.
|
|
let result_len = (max_val + 1).max(minlength) as usize;
|
|
|
|
// Build per-thread histograms in parallel.
|
|
// We use unsafe indexing to eliminate bounds checks in the inner loop.
|
|
values
|
|
.par_iter()
|
|
.fold(
|
|
|| vec![0u32; result_len],
|
|
|mut local_hist, &v| {
|
|
// SAFETY: v is guaranteed to be <= max_val, so it is in bounds.
|
|
unsafe {
|
|
*local_hist.get_unchecked_mut(v as usize) += 1;
|
|
}
|
|
local_hist
|
|
},
|
|
)
|
|
// Merge the per-thread histograms in parallel.
|
|
.reduce(
|
|
|| vec![0u32; result_len],
|
|
|mut global_hist, local_hist| {
|
|
for i in 0..result_len {
|
|
// SAFETY: we know local histogram is at least result_len, as is global_hist
|
|
unsafe {
|
|
*global_hist.get_unchecked_mut(i) += local_hist.get_unchecked(i);
|
|
}
|
|
}
|
|
global_hist
|
|
},
|
|
)
|
|
}
|
|
|
|
#[allow(dead_code)]
|
|
impl BincountOp for Tensor {
|
|
fn bincount(&self, minlength: u32) -> Result<Vec<u32>> {
|
|
let values = self.to_vec1::<u32>()?;
|
|
|
|
Ok(bincount(&values, minlength))
|
|
}
|
|
}
|
|
|
|
// an external reference implementation
|
|
pub fn apply_triangular(xs: &Tensor, diagonal: isize, upper: bool) -> Result<Tensor> {
|
|
let device = xs.device();
|
|
let (l, s) = xs.dims2()?;
|
|
let mut xs_tri = vec![];
|
|
for i in 0..l as isize {
|
|
for j in 0..s as isize {
|
|
let cond = if upper {
|
|
i + diagonal > j
|
|
} else {
|
|
i + diagonal < j
|
|
};
|
|
xs_tri.push(if cond { 0u8 } else { 1u8 });
|
|
}
|
|
}
|
|
xs * Tensor::from_vec(xs_tri, (l, s), device)?.to_dtype(xs.dtype())?
|
|
}
|
|
|
|
/// Elementwise multiply and activation. The following activations are supported:
|
|
/// - `gelu`
|
|
/// - `silu`
|
|
/// - `relu`
|
|
///
|
|
/// This is equivalent to:
|
|
/// `act(a) * b`
|
|
///
|
|
/// With supported dtypes (F16, BF16, F32) and activations (SiLU, GELU, ReLU),
|
|
/// this uses a fused kernel for better performance by eliminating intermediate
|
|
/// memory allocation. Optimized implementations are available for:
|
|
/// - CUDA: Custom CUDA kernel with vec4 optimization
|
|
/// - Metal: Native Metal kernel
|
|
/// - CPU: Rayon-parallelized implementation
|
|
fn glu_activation_type(act: Activation) -> Option<hanzo_quant::GluActivationType> {
|
|
match act {
|
|
Activation::Silu | Activation::Swish => Some(hanzo_quant::GluActivationType::Silu),
|
|
Activation::NewGelu | Activation::GeluPytorchTanh => {
|
|
Some(hanzo_quant::GluActivationType::Gelu)
|
|
}
|
|
Activation::Gelu => Some(hanzo_quant::GluActivationType::GeluErf),
|
|
Activation::Relu => Some(hanzo_quant::GluActivationType::Relu),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn nn_glu_activation_type(act: hanzo_nn::Activation) -> Option<hanzo_quant::GluActivationType> {
|
|
match act {
|
|
hanzo_nn::Activation::Silu | hanzo_nn::Activation::Swish => {
|
|
Some(hanzo_quant::GluActivationType::Silu)
|
|
}
|
|
hanzo_nn::Activation::NewGelu | hanzo_nn::Activation::GeluPytorchTanh => {
|
|
Some(hanzo_quant::GluActivationType::Gelu)
|
|
}
|
|
hanzo_nn::Activation::Gelu => Some(hanzo_quant::GluActivationType::GeluErf),
|
|
hanzo_nn::Activation::Relu => Some(hanzo_quant::GluActivationType::Relu),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub enum GatedActivationOrder {
|
|
GateUp,
|
|
UpGate,
|
|
}
|
|
|
|
pub fn mul_and_act(a: &Tensor, b: &Tensor, act: Activation) -> Result<Tensor> {
|
|
// Vulkan has a fused silu_mul kernel but no general fused_glu; route Silu (the common SwiGLU
|
|
// case) to it so the FFN gate does one dispatch instead of silu + mul (a dispatch + barrier each).
|
|
#[cfg(feature = "vulkan")]
|
|
if a.device().is_vulkan() && a.dtype() == b.dtype() && matches!(act, Activation::Silu) {
|
|
return hanzo_nn::ops::silu_mul(a, b);
|
|
}
|
|
// Check if we can use the fused kernel (works on CUDA, Metal, and CPU)
|
|
if matches!(a.dtype(), DType::F16 | DType::BF16 | DType::F32) && a.dtype() == b.dtype() {
|
|
if let Some(activation_type) = glu_activation_type(act) {
|
|
return hanzo_quant::fused_glu(a, b, activation_type);
|
|
}
|
|
}
|
|
|
|
a.apply(&act)? * b
|
|
}
|
|
|
|
/// Clamp SwiGLU gate/up before the gated activation: `gate.clamp(max=limit)`,
|
|
/// `up.clamp(-limit, limit)` (DeepSeek-V4 `DeepseekV4MLP`/`Experts`, gpt-oss). `None` returns the
|
|
/// inputs unchanged (cheap ref-clone) so the no-limit path stays byte-identical.
|
|
pub fn swiglu_clamp(gate: &Tensor, up: &Tensor, limit: Option<f32>) -> Result<(Tensor, Tensor)> {
|
|
match limit {
|
|
None => Ok((gate.clone(), up.clone())),
|
|
Some(limit) => {
|
|
let limit = f64::from(limit);
|
|
Ok((
|
|
gate.clamp(f64::NEG_INFINITY, limit)?,
|
|
up.clamp(-limit, limit)?,
|
|
))
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn mul_and_nn_act(a: &Tensor, b: &Tensor, act: hanzo_nn::Activation) -> Result<Tensor> {
|
|
// Check if we can use the fused kernel (works on CUDA, Metal, and CPU)
|
|
if matches!(a.dtype(), DType::F16 | DType::BF16 | DType::F32) && a.dtype() == b.dtype() {
|
|
if let Some(activation_type) = nn_glu_activation_type(act) {
|
|
return hanzo_quant::fused_glu(a, b, activation_type);
|
|
}
|
|
}
|
|
|
|
a.apply(&act)? * b
|
|
}
|
|
|
|
pub fn split_mul_and_act(xs: &Tensor, split_size: usize, act: Activation) -> Result<Tensor> {
|
|
split_mul_and_act_order(xs, split_size, act, GatedActivationOrder::GateUp)
|
|
}
|
|
|
|
pub fn split_mul_and_act_order(
|
|
xs: &Tensor,
|
|
split_size: usize,
|
|
act: Activation,
|
|
order: GatedActivationOrder,
|
|
) -> Result<Tensor> {
|
|
let last_dim = xs.dim(D::Minus1)?;
|
|
let Some(expected_last_dim) = split_size.checked_mul(2) else {
|
|
hanzo_ml::bail!("split_mul_and_act split size overflow: {split_size}");
|
|
};
|
|
if last_dim != expected_last_dim {
|
|
hanzo_ml::bail!("split_mul_and_act expected last dim {expected_last_dim}, got {last_dim}");
|
|
}
|
|
|
|
let first = xs.narrow(D::Minus1, 0, split_size)?;
|
|
let second = xs.narrow(D::Minus1, split_size, split_size)?;
|
|
match order {
|
|
GatedActivationOrder::GateUp => mul_and_act(&first, &second, act),
|
|
GatedActivationOrder::UpGate => mul_and_act(&second, &first, act),
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub(crate) struct MergedDenseProjection {
|
|
proj: Arc<dyn hanzo_quant::QuantMethod>,
|
|
output_dims: Vec<usize>,
|
|
}
|
|
|
|
impl MergedDenseProjection {
|
|
pub(crate) fn new(projs: &[&dyn hanzo_quant::QuantMethod]) -> Result<Option<Self>> {
|
|
let mut weights = Vec::with_capacity(projs.len());
|
|
let mut output_dims = Vec::with_capacity(projs.len());
|
|
let mut input_dim = None;
|
|
let mut dtype_device: Option<(DType, hanzo_ml::Device)> = None;
|
|
|
|
for proj in projs {
|
|
if proj.has_bias() {
|
|
return Ok(None);
|
|
}
|
|
let Some((weight, bias)) = proj.unquant_weight_bias() else {
|
|
return Ok(None);
|
|
};
|
|
if bias.is_some() {
|
|
return Ok(None);
|
|
}
|
|
let (rows, cols) = weight.dims2()?;
|
|
if let Some(input_dim) = input_dim {
|
|
if input_dim != cols {
|
|
return Ok(None);
|
|
}
|
|
} else {
|
|
input_dim = Some(cols);
|
|
}
|
|
let this_dtype_device = (weight.dtype(), weight.device().clone());
|
|
if let Some((dtype, device)) = dtype_device.as_ref() {
|
|
if *dtype != this_dtype_device.0
|
|
|| device.location() != this_dtype_device.1.location()
|
|
{
|
|
return Ok(None);
|
|
}
|
|
} else {
|
|
dtype_device = Some(this_dtype_device);
|
|
}
|
|
output_dims.push(rows);
|
|
weights.push(weight);
|
|
}
|
|
|
|
let weight_refs = weights.iter().collect::<Vec<_>>();
|
|
let weight = Tensor::cat(&weight_refs, 0)?;
|
|
let proj = <hanzo_quant::UnquantLinear as hanzo_quant::QuantMethod>::new(
|
|
hanzo_quant::QuantMethodConfig::Unquantized(hanzo_nn::Linear::new(weight, None)),
|
|
)?;
|
|
|
|
Ok(Some(Self {
|
|
proj: Arc::new(proj),
|
|
output_dims,
|
|
}))
|
|
}
|
|
|
|
pub(crate) fn forward(&self, xs: &Tensor) -> Result<Vec<Tensor>> {
|
|
let ys = self.proj.forward(xs)?;
|
|
let mut parts = Vec::with_capacity(self.output_dims.len());
|
|
let mut offset = 0;
|
|
for &dim in &self.output_dims {
|
|
parts.push(ys.narrow(D::Minus1, offset, dim)?);
|
|
offset += dim;
|
|
}
|
|
Ok(parts)
|
|
}
|
|
}
|
|
|
|
/// Feed-forward path for quantized gate/up/down projections.
|
|
pub(crate) fn quantized_ffn(
|
|
xs: &Tensor,
|
|
gate: &dyn hanzo_quant::QuantMethod,
|
|
up: &dyn hanzo_quant::QuantMethod,
|
|
down: &dyn hanzo_quant::QuantMethod,
|
|
act: Activation,
|
|
swiglu_limit: Option<f32>,
|
|
) -> Result<Tensor> {
|
|
// The fused gate-up kernels don't materialize gate/up, so they can't apply the SwiGLU clamp;
|
|
// fall through to the split path when a limit is set (`None` keeps the fast path unchanged).
|
|
#[cfg(feature = "cuda")]
|
|
if swiglu_limit.is_none() {
|
|
if let Some(activation_type) = glu_activation_type(act) {
|
|
if let Some(inter) =
|
|
hanzo_quant::try_fused_quantized_gate_up(xs, gate, up, activation_type)?
|
|
{
|
|
return down.forward(&inter);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "metal")]
|
|
if swiglu_limit.is_none() {
|
|
if let Some(activation_type) = glu_activation_type(act) {
|
|
if let Some(inter) =
|
|
hanzo_quant::try_fused_gate_up_metal(xs, gate, up, activation_type)?
|
|
{
|
|
return down.forward(&inter);
|
|
}
|
|
}
|
|
}
|
|
|
|
let lhs = gate.forward(xs)?;
|
|
let rhs = up.forward(xs)?;
|
|
let (lhs, rhs) = swiglu_clamp(&lhs, &rhs, swiglu_limit)?;
|
|
let inter = mul_and_act(&lhs, &rhs, act)?;
|
|
down.forward(&inter)
|
|
}
|
|
|
|
pub(crate) fn qkv_projections(
|
|
xs: &Tensor,
|
|
q_proj: &dyn hanzo_quant::QuantMethod,
|
|
k_proj: &dyn hanzo_quant::QuantMethod,
|
|
v_proj: &dyn hanzo_quant::QuantMethod,
|
|
) -> Result<(Tensor, Tensor, Tensor)> {
|
|
#[cfg(feature = "cuda")]
|
|
if let Some(qkv) = hanzo_quant::try_fused_quantized_qkv(xs, q_proj, k_proj, v_proj)? {
|
|
return Ok(qkv);
|
|
}
|
|
|
|
#[cfg(feature = "metal")]
|
|
if let Some(qkv) = hanzo_quant::try_fused_qkv_metal(xs, q_proj, k_proj, v_proj)? {
|
|
return Ok(qkv);
|
|
}
|
|
|
|
Ok((
|
|
q_proj.forward(xs)?,
|
|
k_proj.forward(xs)?,
|
|
v_proj.forward(xs)?,
|
|
))
|
|
}
|
|
|
|
mod tests {
|
|
#[test]
|
|
fn test_deterministic_moe_routing_tie_break_lowest_expert() {
|
|
// Proof-of-AI: on equal/near-equal router scores, expert selection MUST be canonical
|
|
// (lowest expert index) and backend-independent, or a re-executing verifier routes to a
|
|
// different sub-network and an honest run false-rejects.
|
|
use super::deterministic_topk_indices;
|
|
use hanzo_ml::Tensor;
|
|
let device = hanzo_ml::Device::Cpu;
|
|
// Row 0: experts 1 and 3 tie at 5.0 (the two highest) -> top-2 = [1, 3], lower index
|
|
// ordered first (tests tie ORDERING).
|
|
// Row 1: all five equal at 7.0 -> top-2 = [0, 1], the two LOWEST indices (tests the
|
|
// boundary tie-break: which of the equal experts make the cut).
|
|
let scores = Tensor::from_vec(
|
|
vec![
|
|
1.0f32, 5.0, 2.0, 5.0, 4.0, /* row1 */ 7.0, 7.0, 7.0, 7.0, 7.0,
|
|
],
|
|
(2, 5),
|
|
&device,
|
|
)
|
|
.unwrap();
|
|
let idx = deterministic_topk_indices(&scores, 2).unwrap();
|
|
assert_eq!(
|
|
idx.to_vec2::<u32>().unwrap(),
|
|
vec![vec![1u32, 3], vec![0u32, 1]]
|
|
);
|
|
// Determinism: identical inputs -> identical selection, every time.
|
|
let idx2 = deterministic_topk_indices(&scores, 2).unwrap();
|
|
assert_eq!(
|
|
idx.to_vec2::<u32>().unwrap(),
|
|
idx2.to_vec2::<u32>().unwrap()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn moe_router_topk_canonical_only_inside_proof_context() {
|
|
// Wiring: proof-deterministic routing must engage IFF a poi transcript is being emitted
|
|
// (`poi_forward::prove`). Inside prove(), tied scores select the canonical lowest indices so
|
|
// prover and verifier agree; outside, proving() is false and the flag is never forced on.
|
|
use super::{moe_router_topk, MoeRouterScoreFunction, MoeRouterSelectedWeight};
|
|
use hanzo_ml::Tensor;
|
|
let device = hanzo_ml::Device::Cpu;
|
|
// Row 0: experts 1,3 tie highest -> canonical top-2 = [1,3]. Row 1: all tie -> lowest two [0,1].
|
|
let logits = Tensor::from_vec(
|
|
vec![1.0f32, 5.0, 2.0, 5.0, 4.0, 7.0, 7.0, 7.0, 7.0, 7.0],
|
|
(2, 5),
|
|
&device,
|
|
)
|
|
.unwrap();
|
|
let config = super::MoeRouterTopKConfig {
|
|
top_k: 2,
|
|
score_function: MoeRouterScoreFunction::Raw,
|
|
selected_weight: MoeRouterSelectedWeight::Score,
|
|
renormalize: false,
|
|
norm_min: 0.0,
|
|
output_scale: 1.0,
|
|
logit_clip: None,
|
|
};
|
|
|
|
assert!(
|
|
!crate::poi_forward::proving(),
|
|
"no proof context by default"
|
|
);
|
|
let (out, _transcript) = crate::poi_forward::prove(|| {
|
|
assert!(
|
|
crate::poi_forward::proving(),
|
|
"prove() installs the context"
|
|
);
|
|
moe_router_topk(&logits, config, None, None).unwrap()
|
|
});
|
|
assert_eq!(
|
|
out.indices.to_vec2::<u32>().unwrap(),
|
|
vec![vec![1u32, 3], vec![0u32, 1]],
|
|
"proof context forces the canonical, backend-independent tie-break"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn swiglu_clamp_matches_reference_and_is_noop_when_none() {
|
|
use super::swiglu_clamp;
|
|
use hanzo_ml::Tensor;
|
|
let device = hanzo_ml::Device::Cpu;
|
|
let gate = Tensor::from_vec(vec![10.0f32, -10.0, 0.5], (1, 3), &device).unwrap();
|
|
let up = Tensor::from_vec(vec![10.0f32, -10.0, 0.5], (1, 3), &device).unwrap();
|
|
|
|
// None: exact passthrough.
|
|
let (g0, u0) = swiglu_clamp(&gate, &up, None).unwrap();
|
|
assert_eq!(g0.to_vec2::<f32>().unwrap(), gate.to_vec2::<f32>().unwrap());
|
|
assert_eq!(u0.to_vec2::<f32>().unwrap(), up.to_vec2::<f32>().unwrap());
|
|
|
|
// Some(1.0): DeepSeek-V4 clamp -> gate.clamp(max=1), up.clamp(-1, 1).
|
|
let (g1, u1) = swiglu_clamp(&gate, &up, Some(1.0)).unwrap();
|
|
assert_eq!(g1.to_vec2::<f32>().unwrap(), vec![vec![1.0f32, -10.0, 0.5]]);
|
|
assert_eq!(u1.to_vec2::<f32>().unwrap(), vec![vec![1.0f32, -1.0, 0.5]]);
|
|
}
|
|
|
|
#[test]
|
|
fn test_topk() {
|
|
use crate::ops::{TopKLastDimOp, TopKOutput};
|
|
use hanzo_ml::Tensor;
|
|
let device = hanzo_ml::Device::Cpu;
|
|
// [[1, 3, 5],
|
|
// [2, 4, 6]]
|
|
let x = Tensor::arange(1f32, 7f32, &device)
|
|
.unwrap()
|
|
.reshape((3, 2))
|
|
.unwrap()
|
|
.t()
|
|
.unwrap()
|
|
.contiguous()
|
|
.unwrap();
|
|
let TopKOutput { values, indices } = x.topk(2).unwrap();
|
|
assert_eq!(
|
|
x.to_vec2::<f32>().unwrap(),
|
|
vec![vec![1f32, 3f32, 5f32], vec![2f32, 4f32, 6f32]]
|
|
);
|
|
assert_eq!(
|
|
values.to_vec2::<f32>().unwrap(),
|
|
vec![vec![5f32, 3f32], vec![6f32, 4f32]]
|
|
);
|
|
assert_eq!(
|
|
indices.to_vec2::<u32>().unwrap(),
|
|
vec![vec![2u32, 1u32], vec![2u32, 1u32]]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_repeat_interleave() -> hanzo_ml::Result<()> {
|
|
use crate::ops::RepeatInterleaveOp;
|
|
use hanzo_ml::{Device, Tensor};
|
|
|
|
let input = Tensor::new(
|
|
vec![vec![vec![1f32, 2., 3.], vec![4f32, 5., 6.]]],
|
|
&Device::Cpu,
|
|
)?;
|
|
|
|
let repeat_interleaved = input.repeat_interleave(2, 2)?;
|
|
assert_eq!(
|
|
repeat_interleaved.to_vec3::<f32>()?,
|
|
vec![vec![
|
|
vec![1., 1., 2., 2., 3., 3.],
|
|
vec![4., 4., 5., 5., 6., 6.]
|
|
]]
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn test_repeat_interleave_flat() -> hanzo_ml::Result<()> {
|
|
use crate::ops::RepeatInterleaveOp;
|
|
use hanzo_ml::{Device, Tensor};
|
|
|
|
let input = Tensor::new(vec![1., 2., 3., 4.], &Device::Cpu)?;
|
|
|
|
let repeat_interleaved = input.repeat_interleave_flat(vec![1u32, 2u32, 3u32, 4u32])?;
|
|
assert_eq!(
|
|
repeat_interleaved.to_vec1::<f64>()?,
|
|
vec![1., 2., 2., 3., 3., 3., 4., 4., 4., 4.]
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
#[test]
|
|
fn test_cuda_topk_multiblock_matches_cpu() -> hanzo_ml::Result<()> {
|
|
use crate::ops::{TopKLastDimOp, TopKOutput};
|
|
use hanzo_ml::{DType, Device, Tensor};
|
|
|
|
let Ok(cuda) = Device::new_cuda(0) else {
|
|
return Ok(());
|
|
};
|
|
let nrows = 4usize;
|
|
let ncols = 32_768usize;
|
|
let k = 32usize;
|
|
let mut data = vec![0f32; nrows * ncols];
|
|
for r in 0..nrows {
|
|
for c in 0..ncols {
|
|
// Deterministic pattern spanning chunks + exact ties near top.
|
|
let v = ((c * 37 + r * 101) % 997) as f32 * 0.25 - 120.0;
|
|
data[r * ncols + c] = v;
|
|
}
|
|
// Inject planted ties across separate 4096-col chunks.
|
|
data[r * ncols + 5] = 900.0;
|
|
data[r * ncols + 4101] = 900.0;
|
|
data[r * ncols + 16390] = 950.0;
|
|
data[r * ncols + 28675] = 925.0;
|
|
}
|
|
|
|
let cpu_f32 = Tensor::from_vec(data, (nrows, ncols), &Device::Cpu)?;
|
|
let TopKOutput {
|
|
values: ref_v,
|
|
indices: ref_i,
|
|
} = cpu_f32.topk(k)?;
|
|
let gpu_f32 = cpu_f32.to_device(&cuda)?;
|
|
let TopKOutput {
|
|
values: gpu_v,
|
|
indices: gpu_i,
|
|
} = gpu_f32.topk(k)?;
|
|
assert_eq!(
|
|
gpu_i.to_vec2::<u32>()?,
|
|
ref_i.to_vec2::<u32>()?,
|
|
"F32 multiblock topk indices mismatch"
|
|
);
|
|
assert_eq!(
|
|
gpu_v.to_vec2::<f32>()?,
|
|
ref_v.to_vec2::<f32>()?,
|
|
"F32 multiblock topk values mismatch"
|
|
);
|
|
|
|
// Also test BF16 at DFlash vocab scale (248,320 cols).
|
|
let ncols_big = 248_320usize;
|
|
let nrows_big = 3usize;
|
|
let mut big = vec![0f32; nrows_big * ncols_big];
|
|
for r in 0..nrows_big {
|
|
for c in 0..ncols_big {
|
|
big[r * ncols_big + c] = ((c * 13 + r * 71) % 251) as f32 - 125.0;
|
|
}
|
|
big[r * ncols_big + 123] = 512.0;
|
|
big[r * ncols_big + 123_456] = 512.0;
|
|
big[r * ncols_big + 240_000] = 1024.0;
|
|
}
|
|
let cpu_bf16 =
|
|
Tensor::from_vec(big, (nrows_big, ncols_big), &Device::Cpu)?.to_dtype(DType::BF16)?;
|
|
let TopKOutput {
|
|
values: ref_bv,
|
|
indices: ref_bi,
|
|
} = cpu_bf16.topk(k)?;
|
|
let gpu_bf16 = cpu_bf16.to_device(&cuda)?;
|
|
let TopKOutput {
|
|
values: gpu_bv,
|
|
indices: gpu_bi,
|
|
} = gpu_bf16.topk(k)?;
|
|
assert_eq!(
|
|
gpu_bi.to_vec2::<u32>()?,
|
|
ref_bi.to_vec2::<u32>()?,
|
|
"BF16 248K multiblock topk indices mismatch"
|
|
);
|
|
assert_eq!(
|
|
gpu_bv.to_dtype(DType::F32)?.to_vec2::<f32>()?,
|
|
ref_bv.to_dtype(DType::F32)?.to_vec2::<f32>()?,
|
|
"BF16 248K multiblock topk values mismatch"
|
|
);
|
|
|
|
// Partial trailing chunk, non-power-of-two ncols, max multiblock k=128, ties, and -inf.
|
|
let ncols_odd = 10_003usize;
|
|
let nrows_odd = 2usize;
|
|
let k_max = 128usize;
|
|
let mut odd = vec![0f32; nrows_odd * ncols_odd];
|
|
for r in 0..nrows_odd {
|
|
for c in 0..ncols_odd {
|
|
odd[r * ncols_odd + c] = if c % 97 == 0 {
|
|
f32::NEG_INFINITY
|
|
} else {
|
|
((c * 19 + r * 53) % 1031) as f32 * 0.5 - 250.0
|
|
};
|
|
}
|
|
odd[r * ncols_odd + 4095] = 777.0;
|
|
odd[r * ncols_odd + 4096] = 777.0;
|
|
odd[r * ncols_odd + ncols_odd - 1] = 888.0;
|
|
}
|
|
let cpu_odd = Tensor::from_vec(odd, (nrows_odd, ncols_odd), &Device::Cpu)?;
|
|
let TopKOutput {
|
|
values: ref_ov,
|
|
indices: ref_oi,
|
|
} = cpu_odd.topk(k_max)?;
|
|
let gpu_odd = cpu_odd.to_device(&cuda)?;
|
|
let TopKOutput {
|
|
values: gpu_ov,
|
|
indices: gpu_oi,
|
|
} = gpu_odd.topk(k_max)?;
|
|
assert_eq!(
|
|
gpu_oi.to_vec2::<u32>()?,
|
|
ref_oi.to_vec2::<u32>()?,
|
|
"F32 partial-chunk k=128 multiblock topk indices mismatch"
|
|
);
|
|
assert_eq!(
|
|
gpu_ov.to_vec2::<f32>()?,
|
|
ref_ov.to_vec2::<f32>()?,
|
|
"F32 partial-chunk k=128 multiblock topk values mismatch"
|
|
);
|
|
|
|
let cpu_f16 = cpu_odd.to_dtype(DType::F16)?;
|
|
let TopKOutput {
|
|
values: ref_hv,
|
|
indices: ref_hi,
|
|
} = cpu_f16.topk(k_max)?;
|
|
let gpu_f16 = cpu_f16.to_device(&cuda)?;
|
|
let TopKOutput {
|
|
values: gpu_hv,
|
|
indices: gpu_hi,
|
|
} = gpu_f16.topk(k_max)?;
|
|
assert_eq!(
|
|
gpu_hi.to_vec2::<u32>()?,
|
|
ref_hi.to_vec2::<u32>()?,
|
|
"F16 partial-chunk k=128 multiblock topk indices mismatch"
|
|
);
|
|
assert_eq!(
|
|
gpu_hv.to_dtype(DType::F32)?.to_vec2::<f32>()?,
|
|
ref_hv.to_dtype(DType::F32)?.to_vec2::<f32>()?,
|
|
"F16 partial-chunk k=128 multiblock topk values mismatch"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[cfg(all(test, feature = "cuda"))]
|
|
mod cuda_offset_tests {
|
|
use super::*;
|
|
use hanzo_ml::{DType, Device, Tensor};
|
|
|
|
#[test]
|
|
fn cuda_sliced_row_topk_respects_start_offset() -> hanzo_ml::Result<()> {
|
|
let Ok(cuda) = Device::new_cuda(0) else {
|
|
return Ok(());
|
|
};
|
|
let nrows = 4usize;
|
|
let ncols = 4096usize;
|
|
let mut data = vec![-10.0f32; nrows * ncols];
|
|
for r in 0..nrows {
|
|
let peak = 100 + r * 777;
|
|
data[r * ncols + peak] = 20.0 + r as f32;
|
|
data[r * ncols + peak + 1] = 10.0 + r as f32;
|
|
}
|
|
let logits = Tensor::from_vec(data, (1, nrows, ncols), &Device::Cpu)?.to_device(&cuda)?;
|
|
|
|
let mut top1_cache = None;
|
|
for r in 0..nrows {
|
|
let expected_idx = (100 + r * 777) as u32;
|
|
let row_tensor = logits.narrow(1, r, 1)?.squeeze(0)?.squeeze(0)?;
|
|
assert_eq!(row_tensor.layout().start_offset(), r * ncols);
|
|
|
|
let top1 = cuda_top1_logits_f32_cached(&row_tensor, &mut top1_cache)?;
|
|
assert_eq!(
|
|
top1[1] as u32, expected_idx,
|
|
"cuda_top1_logits_f32_cached row {r} read wrong offset"
|
|
);
|
|
|
|
let packed_out = cuda_topk_logits_f32_packed(&row_tensor, 4, 1.0)?;
|
|
let packed = packed_out.packed.to_vec1::<f32>()?;
|
|
assert_eq!(
|
|
packed[packed_out.k] as u32, expected_idx,
|
|
"cuda_topk_logits_f32_packed row {r} read wrong offset"
|
|
);
|
|
|
|
let large_out = cuda_topk_logits_f32(&row_tensor, 4, 1.0)?;
|
|
let large_idxs = large_out.indices.to_vec1::<u32>()?;
|
|
assert_eq!(
|
|
large_idxs[0], expected_idx,
|
|
"cuda_topk_logits_f32 row {r} read wrong offset"
|
|
);
|
|
|
|
let topk_f32 = cuda_topk(&row_tensor, 4)?;
|
|
assert_eq!(
|
|
topk_f32.indices.to_vec1::<u32>()?[0],
|
|
expected_idx,
|
|
"cuda_topk f32 row {r} read wrong offset"
|
|
);
|
|
|
|
let logits_bf16 = logits.to_dtype(DType::BF16)?;
|
|
let sliced_bf16 = logits_bf16.narrow(1, r, 1)?.squeeze(0)?.squeeze(0)?;
|
|
assert_eq!(sliced_bf16.layout().start_offset(), r * ncols);
|
|
let topk_bf16 = cuda_topk(&sliced_bf16, 4)?;
|
|
assert_eq!(
|
|
topk_bf16.indices.to_vec1::<u32>()?[0],
|
|
expected_idx,
|
|
"cuda_topk bf16 row {r} read wrong offset"
|
|
);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
}
|