Files
zeekay 37ff59034b merge: upstream/master (-X ours, brand preserved)
# Conflicts:
#	hanzo-engine/src/device_map.rs
#	hanzo-engine/src/device_map/mappers.rs
#	hanzo-engine/src/device_map/mask.rs
#	hanzo-engine/src/device_map/mod.rs
#	hanzo-engine/src/device_map/peer.rs
#	hanzo-engine/src/moe/experts.rs
#	hanzo-engine/src/moe/experts/backends.rs
#	hanzo-engine/src/moe/experts/checkpoint.rs
#	hanzo-engine/src/moe/experts/config.rs
#	hanzo-engine/src/moe/experts/forward.rs
#	hanzo-engine/src/moe/experts/mod.rs
#	hanzo-engine/src/paged_attention/attention_backend.rs
#	hanzo-engine/src/perf_flags.rs
#	hanzo-engine/src/pipeline/cuda_graph.rs
#	hanzo-paged-attn/src/cuda/backend/flashinfer.rs
#	hanzo-paged-attn/src/cuda/flashinfer/attention/default_prefill_params.cuh
#	hanzo-paged-attn/src/cuda/flashinfer/attention/mask.cuh
#	hanzo-paged-attn/src/cuda/flashinfer/attention/prefill.cuh
#	hanzo-paged-attn/src/cuda/flashinfer/fp16.h
#	hanzo-paged-attn/src/cuda/flashinfer/frag_layout_swizzle.cuh
#	hanzo-paged-attn/src/cuda/flashinfer/mma.cuh
#	hanzo-paged-attn/src/cuda/flashinfer/permuted_smem.cuh
#	hanzo-paged-attn/src/cuda/flashinfer_decode.cu
#	hanzo-quant/src/metal_kernels/moe.metal
#	hanzo-quant/src/metal_kernels/rotary.metal
#	hanzo-quant/src/metal_kernels/softcap.metal
#	hanzo-server-core/src/anthropic.rs
#	hanzo-server-core/src/route_registry.rs
#	mistralrs-core/Cargo.toml
#	mistralrs-core/src/lib.rs
2026-06-02 10:21:05 -07:00

1008 lines
36 KiB
Plaintext

// Get inspiration from
// https://github.com/pytorch/pytorch/blob/65aa16f968af2cd18ff8c25cc657e7abda594bfc/aten/src/ATen/native/cuda/Nonzero.cu
#include <assert.h>
#include <cub/cub.cuh>
#include <stdint.h>
#include <stdio.h>
#if __CUDACC_VER_MAJOR__ >= 13
#include <thrust/iterator/counting_iterator.h>
#include <thrust/iterator/transform_iterator.h>
#endif
#if __CUDACC_VER_MAJOR__ >= 13
#define USE_THRUST_INPUT_ITERS 1
#else
#define USE_THRUST_INPUT_ITERS 0
#endif
#if USE_THRUST_INPUT_ITERS
#define DECLARE_NONZERO_ITER(NAME, TYPE, PTR) \
auto NAME = thrust::make_transform_iterator(PTR, NonZeroOp<TYPE>());
#define DECLARE_COUNTING_ITER(NAME, START) \
auto NAME = thrust::make_counting_iterator<uint32_t>(START);
#else
#define DECLARE_NONZERO_ITER(NAME, TYPE, PTR) \
cub::TransformInputIterator<bool, NonZeroOp<TYPE>, const TYPE *> NAME( \
PTR, NonZeroOp<TYPE>());
#define DECLARE_COUNTING_ITER(NAME, START) \
cub::CountingInputIterator<uint32_t> NAME(START);
#endif
#define CUDA_CHECK(call) \
do { \
cudaError_t err = call; \
if (err != cudaSuccess) { \
fprintf(stderr, "CUDA error at %s:%d: %s\n", __FILE__, __LINE__, \
cudaGetErrorString(err)); \
exit(err); \
} \
} while (0)
#define MAX(a, b) ((a) > (b) ? (a) : (b))
#define MIN(a, b) ((a) < (b) ? (a) : (b))
int next_power_of_2(const uint32_t num_nonzero) {
int result = 1;
while (result < num_nonzero) {
result <<= 1;
}
return result;
}
template <typename T> struct NonZeroOp {
__host__ __device__ __forceinline__ bool operator()(const T &a) const {
return (a != T(0));
}
};
// count the number of non-zero elements in an array, to better allocate memory
template <typename T>
void count_nonzero(const T *d_in, const uint32_t N, uint32_t *h_out,
cudaStream_t stream) {
DECLARE_NONZERO_ITER(itr, T, d_in);
size_t temp_storage_bytes = 0;
uint32_t *d_num_nonzero;
CUDA_CHECK(
cudaMallocAsync((void **)&d_num_nonzero, sizeof(uint32_t), stream));
CUDA_CHECK(cub::DeviceReduce::Sum(nullptr, temp_storage_bytes, itr,
d_num_nonzero, N, stream));
void **d_temp_storage;
CUDA_CHECK(cudaMallocAsync(&d_temp_storage, temp_storage_bytes, stream));
CUDA_CHECK(cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, itr,
d_num_nonzero, N, stream));
CUDA_CHECK(cudaMemcpyAsync(h_out, d_num_nonzero, sizeof(uint32_t),
cudaMemcpyDeviceToHost, stream));
CUDA_CHECK(cudaFreeAsync(d_num_nonzero, stream));
CUDA_CHECK(cudaFreeAsync(d_temp_storage, stream));
}
#define COUNT_NONZERO_OP(TYPENAME, RUST_NAME) \
extern "C" uint32_t count_nonzero_##RUST_NAME( \
const TYPENAME *d_in, uint32_t N, cudaStream_t stream) { \
uint32_t result; \
count_nonzero(d_in, N, &result, stream); \
return result; \
}
#define COUNT_NONZERO_OP_DUMMY(RUST_NAME) \
extern "C" uint32_t count_nonzero_##RUST_NAME( \
const uint16_t *d_in, uint32_t N, cudaStream_t stream) { \
return 0; \
}
#if __CUDA_ARCH__ >= 800
COUNT_NONZERO_OP(__nv_bfloat16, bf16)
#else
COUNT_NONZERO_OP_DUMMY(bf16)
#endif
#if __CUDA_ARCH__ >= 530
COUNT_NONZERO_OP(__half, f16)
#else
COUNT_NONZERO_OP_DUMMY(f16)
#endif
COUNT_NONZERO_OP(float, f32)
COUNT_NONZERO_OP(double, f64)
COUNT_NONZERO_OP(uint8_t, u8)
COUNT_NONZERO_OP(uint32_t, u32)
COUNT_NONZERO_OP(int64_t, i64)
COUNT_NONZERO_OP(int32_t, i32)
COUNT_NONZERO_OP(int16_t, i16)
__global__ void transform_indices(const uint32_t *temp_indices,
const uint32_t num_nonzero,
const uint32_t *dims, const uint32_t num_dims,
uint32_t *d_out) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < num_nonzero) {
int temp_index = temp_indices[idx];
for (int i = num_dims - 1; i >= 0; i--) {
d_out[idx * num_dims + i] = temp_index % dims[i];
if (dims[i] == 0) {
// Handle the error appropriately
} else {
temp_index /= dims[i];
}
}
}
}
// get the indices of non-zero elements in an array
template <typename T>
void nonzero(const T *d_in, const uint32_t N, const uint32_t num_nonzero,
const uint32_t *dims, const uint32_t num_dims, uint32_t *d_out,
cudaStream_t stream) {
DECLARE_NONZERO_ITER(itr, T, d_in);
DECLARE_COUNTING_ITER(counting_itr, 0);
uint32_t *out_temp;
uint32_t *num_selected_out;
CUDA_CHECK(cudaMallocAsync((void **)&out_temp, num_nonzero * sizeof(uint32_t),
stream));
CUDA_CHECK(
cudaMallocAsync((void **)&num_selected_out, sizeof(uint32_t), stream));
size_t temp_storage_bytes = 0;
CUDA_CHECK(cub::DeviceSelect::Flagged(nullptr, temp_storage_bytes,
counting_itr, itr, out_temp,
num_selected_out, N, stream));
void **d_temp_storage;
CUDA_CHECK(cudaMallocAsync(&d_temp_storage, temp_storage_bytes, stream));
CUDA_CHECK(cub::DeviceSelect::Flagged(d_temp_storage, temp_storage_bytes,
counting_itr, itr, out_temp,
num_selected_out, (int)N, stream));
int nthreads = next_power_of_2(num_nonzero);
if (nthreads > 1024) {
nthreads = 1024;
}
const int nblocks = (num_nonzero + nthreads - 1) / nthreads;
transform_indices<<<nblocks, nthreads, 0, stream>>>(out_temp, num_nonzero,
dims, num_dims, d_out);
CUDA_CHECK(cudaGetLastError());
CUDA_CHECK(cudaFreeAsync(out_temp, stream));
CUDA_CHECK(cudaFreeAsync(d_temp_storage, stream));
CUDA_CHECK(cudaFreeAsync(num_selected_out, stream));
}
#define NONZERO_OP(TYPENAME, RUST_NAME) \
extern "C" void nonzero_##RUST_NAME(const TYPENAME *d_in, uint32_t N, \
uint32_t num_nonzero, \
const uint32_t *dims, uint32_t num_dims, \
uint32_t *d_out, cudaStream_t stream) { \
nonzero(d_in, N, num_nonzero, dims, num_dims, d_out, stream); \
}
#define NONZERO_OP_DUMMY(RUST_NAME) \
extern "C" void nonzero_##RUST_NAME(const uint16_t *d_in, uint32_t N, \
uint32_t num_nonzero, \
const uint32_t *dims, uint32_t num_dims, \
uint32_t *d_out, cudaStream_t stream) { \
assert(false); \
}
#if __CUDA_ARCH__ >= 800
NONZERO_OP(__nv_bfloat16, bf16)
#else
NONZERO_OP_DUMMY(bf16)
#endif
#if __CUDA_ARCH__ >= 530
NONZERO_OP(__half, f16)
#else
NONZERO_OP_DUMMY(f16)
#endif
NONZERO_OP(float, f32)
NONZERO_OP(double, f64)
NONZERO_OP(uint8_t, u8)
NONZERO_OP(uint32_t, u32)
NONZERO_OP(int64_t, i64)
NONZERO_OP(int32_t, i32)
NONZERO_OP(int16_t, i16)
template <typename T>
__global__ void bitwise_and__kernel(const T *d_in1, const T *d_in2, T *d_out,
const uint32_t N) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < N) {
d_out[idx] = d_in1[idx] & d_in2[idx];
}
}
template <typename T>
__global__ void bitwise_or__kernel(const T *d_in1, const T *d_in2, T *d_out,
const uint32_t N) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < N) {
d_out[idx] = d_in1[idx] | d_in2[idx];
}
}
template <typename T>
__global__ void bitwise_xor__kernel(const T *d_in1, const T *d_in2, T *d_out,
const uint32_t N) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < N) {
d_out[idx] = d_in1[idx] ^ d_in2[idx];
}
}
template <typename T>
void bitwise_and(const T *d_in1, const T *d_in2, T *d_out, int N) {
int nthreads = next_power_of_2(N);
if (nthreads > 1024) {
nthreads = 1024;
}
const int nblocks = (N + nthreads - 1) / nthreads;
bitwise_and__kernel<<<nblocks, nthreads>>>(d_in1, d_in2, d_out, N);
CUDA_CHECK(cudaGetLastError());
}
template <typename T>
void bitwise_or(const T *d_in1, const T *d_in2, T *d_out, int N) {
int nthreads = next_power_of_2(N);
if (nthreads > 1024) {
nthreads = 1024;
}
const int nblocks = (N + nthreads - 1) / nthreads;
bitwise_or__kernel<<<nblocks, nthreads>>>(d_in1, d_in2, d_out, N);
CUDA_CHECK(cudaGetLastError());
}
template <typename T>
void bitwise_xor(const T *d_in1, const T *d_in2, T *d_out, int N) {
int nthreads = next_power_of_2(N);
if (nthreads > 1024) {
nthreads = 1024;
}
const int nblocks = (N + nthreads - 1) / nthreads;
bitwise_xor__kernel<<<nblocks, nthreads>>>(d_in1, d_in2, d_out, N);
CUDA_CHECK(cudaGetLastError());
}
#define BITWISE_OP(TYPENAME, RUST_NAME) \
extern "C" void bitwise_and_##RUST_NAME(const TYPENAME *d_in1, \
const TYPENAME *d_in2, \
TYPENAME *d_out, uint32_t N) { \
bitwise_and(d_in1, d_in2, d_out, N); \
} \
extern "C" void bitwise_or_##RUST_NAME(const TYPENAME *d_in1, \
const TYPENAME *d_in2, \
TYPENAME *d_out, uint32_t N) { \
bitwise_or(d_in1, d_in2, d_out, N); \
} \
extern "C" void bitwise_xor_##RUST_NAME(const TYPENAME *d_in1, \
const TYPENAME *d_in2, \
TYPENAME *d_out, uint32_t N) { \
bitwise_xor(d_in1, d_in2, d_out, N); \
}
BITWISE_OP(uint8_t, u8)
BITWISE_OP(uint32_t, u32)
BITWISE_OP(int64_t, i64)
BITWISE_OP(int32_t, i32)
template <typename T>
__global__ void leftshift_kernel(const T *d_in1, T *d_out, const uint32_t N,
const int32_t k) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < N) {
d_out[idx] = d_in1[idx] << k;
}
}
template <typename T>
void leftshift(const T *d_in1, T *d_out, int N, const int32_t k) {
int nthreads = next_power_of_2(N);
if (nthreads > 1024) {
nthreads = 1024;
}
const int nblocks = (N + nthreads - 1) / nthreads;
leftshift_kernel<<<nblocks, nthreads>>>(d_in1, d_out, N, k);
CUDA_CHECK(cudaGetLastError());
}
#define LEFTSHIFT_OP(TYPENAME, RUST_NAME) \
extern "C" void leftshift_##RUST_NAME( \
const TYPENAME *d_in1, TYPENAME *d_out, uint32_t N, int32_t k) { \
leftshift(d_in1, d_out, N, k); \
}
LEFTSHIFT_OP(uint8_t, u8)
LEFTSHIFT_OP(int32_t, i32)
LEFTSHIFT_OP(uint32_t, u32)
LEFTSHIFT_OP(int64_t, i64)
// ============================================================================
// Fused GPT-OSS SwiGLU kernel
// Fuses 7 operations into 1: clamp, multiply, sigmoid, etc.
//
// Formula:
// gate_clamped = min(gate, limit)
// up_clamped = clamp(up, -limit, limit)
// glu = gate_clamped * sigmoid(gate_clamped * alpha)
// output = (up_clamped + 1) * glu
// ============================================================================
#include <cuda_bf16.h>
#include <cuda_fp16.h>
__device__ __forceinline__ float fast_sigmoid(float x) {
return 1.0f / (1.0f + expf(-x));
}
template <typename T>
__global__ void gptoss_swiglu_kernel(const T *__restrict__ gate,
const T *__restrict__ up,
T *__restrict__ output, const uint32_t N,
const float alpha, const float limit) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= N)
return;
// Load inputs
float g = (float)gate[idx];
float u = (float)up[idx];
// Clamp gate (max only) and up (both min and max)
float gate_clamped = fminf(g, limit);
float up_clamped = fmaxf(fminf(u, limit), -limit);
// glu = gate_clamped * sigmoid(gate_clamped * alpha)
float glu = gate_clamped * fast_sigmoid(gate_clamped * alpha);
// output = (up_clamped + 1) * glu
float result = (up_clamped + 1.0f) * glu;
output[idx] = (T)result;
}
// Vectorized version for better memory bandwidth (4 elements at a time)
template <typename T, typename T4>
__global__ void gptoss_swiglu_kernel_vec4(const T4 *__restrict__ gate,
const T4 *__restrict__ up,
T4 *__restrict__ output,
const uint32_t N4, const float alpha,
const float limit) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= N4)
return;
T4 g4 = gate[idx];
T4 u4 = up[idx];
float g0 = (float)((T *)&g4)[0];
float g1 = (float)((T *)&g4)[1];
float g2 = (float)((T *)&g4)[2];
float g3 = (float)((T *)&g4)[3];
float u0 = (float)((T *)&u4)[0];
float u1 = (float)((T *)&u4)[1];
float u2 = (float)((T *)&u4)[2];
float u3 = (float)((T *)&u4)[3];
// Process 4 elements
#pragma unroll
for (int i = 0; i < 4; i++) {
float g = (i == 0) ? g0 : (i == 1) ? g1 : (i == 2) ? g2 : g3;
float u = (i == 0) ? u0 : (i == 1) ? u1 : (i == 2) ? u2 : u3;
float gate_clamped = fminf(g, limit);
float up_clamped = fmaxf(fminf(u, limit), -limit);
float glu = gate_clamped * fast_sigmoid(gate_clamped * alpha);
float result = (up_clamped + 1.0f) * glu;
if (i == 0)
((T *)&g4)[0] = (T)result;
else if (i == 1)
((T *)&g4)[1] = (T)result;
else if (i == 2)
((T *)&g4)[2] = (T)result;
else
((T *)&g4)[3] = (T)result;
}
output[idx] = g4;
}
extern "C" void gptoss_swiglu_f16(const __half *gate, const __half *up,
__half *output, uint32_t N, float alpha,
float limit, cudaStream_t stream) {
// Use vectorized kernel when N is divisible by 4
if (N % 4 == 0) {
const int N4 = N / 4;
const int nthreads = 256;
const int nblocks = (N4 + nthreads - 1) / nthreads;
gptoss_swiglu_kernel_vec4<__half, uint64_t>
<<<nblocks, nthreads, 0, stream>>>(
(const uint64_t *)gate, (const uint64_t *)up, (uint64_t *)output,
N4, alpha, limit);
} else {
const int nthreads = 256;
const int nblocks = (N + nthreads - 1) / nthreads;
gptoss_swiglu_kernel<<<nblocks, nthreads, 0, stream>>>(gate, up, output, N,
alpha, limit);
}
CUDA_CHECK(cudaGetLastError());
}
extern "C" void gptoss_swiglu_bf16(const __nv_bfloat16 *gate,
const __nv_bfloat16 *up,
__nv_bfloat16 *output, uint32_t N,
float alpha, float limit,
cudaStream_t stream) {
// Use vectorized kernel when N is divisible by 4
if (N % 4 == 0) {
const int N4 = N / 4;
const int nthreads = 256;
const int nblocks = (N4 + nthreads - 1) / nthreads;
gptoss_swiglu_kernel_vec4<__nv_bfloat16, uint64_t>
<<<nblocks, nthreads, 0, stream>>>(
(const uint64_t *)gate, (const uint64_t *)up, (uint64_t *)output,
N4, alpha, limit);
} else {
const int nthreads = 256;
const int nblocks = (N + nthreads - 1) / nthreads;
gptoss_swiglu_kernel<<<nblocks, nthreads, 0, stream>>>(gate, up, output, N,
alpha, limit);
}
CUDA_CHECK(cudaGetLastError());
}
extern "C" void gptoss_swiglu_f32(const float *gate, const float *up,
float *output, uint32_t N, float alpha,
float limit, cudaStream_t stream) {
// Use vectorized kernel when N is divisible by 4
if (N % 4 == 0) {
const int N4 = N / 4;
const int nthreads = 256;
const int nblocks = (N4 + nthreads - 1) / nthreads;
gptoss_swiglu_kernel_vec4<float, float4><<<nblocks, nthreads, 0, stream>>>(
(const float4 *)gate, (const float4 *)up, (float4 *)output, N4, alpha,
limit);
} else {
const int nthreads = 256;
const int nblocks = (N + nthreads - 1) / nthreads;
gptoss_swiglu_kernel<<<nblocks, nthreads, 0, stream>>>(gate, up, output, N,
alpha, limit);
}
CUDA_CHECK(cudaGetLastError());
}
// ============================================================================
// Fused GPT-OSS SwiGLU kernel for INTERLEAVED gate/up data
//
// This kernel handles interleaved gate/up format: [..., intermediate_size, 2]
// where gate = data[..., :, 0] and up = data[..., :, 1]
//
// Avoids 2 tensor copies from narrow().squeeze().contiguous()
// ============================================================================
template <typename T>
__global__ void gptoss_swiglu_interleaved_kernel(
const T *__restrict__ gate_up, // [N, intermediate_size, 2] interleaved
T *__restrict__ output, // [N, intermediate_size]
const uint32_t N, // num_tokens * topk
const uint32_t intermediate_size, const float alpha, const float limit) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
const uint32_t total_elements = N * intermediate_size;
if (idx >= total_elements)
return;
// Decode position
const int n = idx / intermediate_size;
const int i = idx % intermediate_size;
// Read interleaved values: gate at offset 0, up at offset 1
const int base_idx = (n * intermediate_size + i) * 2;
float g = (float)gate_up[base_idx]; // gate
float u = (float)gate_up[base_idx + 1]; // up
// Clamp gate (max only) and up (both min and max)
float gate_clamped = fminf(g, limit);
float up_clamped = fmaxf(fminf(u, limit), -limit);
// glu = gate_clamped * sigmoid(gate_clamped * alpha)
float glu = gate_clamped * fast_sigmoid(gate_clamped * alpha);
// output = (up_clamped + 1) * glu
float result = (up_clamped + 1.0f) * glu;
output[idx] = (T)result;
}
extern "C" void gptoss_swiglu_interleaved_f16(const __half *gate_up,
__half *output, uint32_t N,
uint32_t intermediate_size,
float alpha, float limit,
cudaStream_t stream) {
const uint32_t total = N * intermediate_size;
const int nthreads = 256;
const int nblocks = (total + nthreads - 1) / nthreads;
gptoss_swiglu_interleaved_kernel<<<nblocks, nthreads, 0, stream>>>(
gate_up, output, N, intermediate_size, alpha, limit);
CUDA_CHECK(cudaGetLastError());
}
extern "C" void gptoss_swiglu_interleaved_bf16(
const __nv_bfloat16 *gate_up, __nv_bfloat16 *output, uint32_t N,
uint32_t intermediate_size, float alpha, float limit, cudaStream_t stream) {
const uint32_t total = N * intermediate_size;
const int nthreads = 256;
const int nblocks = (total + nthreads - 1) / nthreads;
gptoss_swiglu_interleaved_kernel<<<nblocks, nthreads, 0, stream>>>(
gate_up, output, N, intermediate_size, alpha, limit);
CUDA_CHECK(cudaGetLastError());
}
extern "C" void gptoss_swiglu_interleaved_f32(const float *gate_up,
float *output, uint32_t N,
uint32_t intermediate_size,
float alpha, float limit,
cudaStream_t stream) {
const uint32_t total = N * intermediate_size;
const int nthreads = 256;
const int nblocks = (total + nthreads - 1) / nthreads;
gptoss_swiglu_interleaved_kernel<<<nblocks, nthreads, 0, stream>>>(
gate_up, output, N, intermediate_size, alpha, limit);
CUDA_CHECK(cudaGetLastError());
}
// ============================================================================
// Fused Softmax with Sinks kernel for GPT-OSS attention
//
// This kernel computes softmax over attention logits while including a per-head
// "sink" value in the normalization, then drops the sink from the output.
//
// This avoids:
// 1. Tensor concatenation to add sinks
// 2. Broadcast operations for max subtraction
// 3. narrow().contiguous() copy to drop sinks
//
// Input:
// logits: [batch, heads, q_len, k_len] - attention scores after [email protected] * scale
// sinks: [heads] - per-head sink values
// mask: [batch, 1, q_len, k_len] or nullptr - attention mask (0 = attend,
// -inf = mask)
//
// Output:
// scores: [batch, heads, q_len, k_len] - softmax probabilities (sink dropped)
// ============================================================================
// Warp-level reduction for max
__device__ __forceinline__ float warp_reduce_max(float val) {
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
val = fmaxf(val, __shfl_down_sync(0xffffffff, val, offset));
}
return val;
}
// Warp-level reduction for sum
__device__ __forceinline__ float warp_reduce_sum(float val) {
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
val += __shfl_down_sync(0xffffffff, val, offset);
}
return val;
}
// Fused softmax with sinks - one block per (batch, head, query) position
// Each block processes one row of the attention matrix
template <typename T>
__global__ void softmax_with_sinks_kernel(
const T *__restrict__ logits, // [batch, heads, q_len, k_len]
const T *__restrict__ sinks, // [heads]
const T *__restrict__ mask, // [batch, 1, q_len, k_len] or nullptr
T *__restrict__ output, // [batch, heads, q_len, k_len]
const int batch_size, const int num_heads, const int q_len, const int k_len,
const float scale // softmax scale (usually 1.0, already applied)
) {
// Each block handles one (batch, head, query) = one row of length k_len
const int row_idx = blockIdx.x;
const int total_rows = batch_size * num_heads * q_len;
if (row_idx >= total_rows)
return;
// Decode indices
const int b = row_idx / (num_heads * q_len);
const int h = (row_idx / q_len) % num_heads;
const int q = row_idx % q_len;
// Pointers to this row
const int logits_offset = ((b * num_heads + h) * q_len + q) * k_len;
const T *row_logits = logits + logits_offset;
T *row_output = output + logits_offset;
// Get sink value for this head
const float sink_val = (float)sinks[h];
// Mask offset (mask has shape [batch, 1, q_len, k_len])
const int mask_offset = (b * q_len + q) * k_len;
const T *row_mask = mask ? (mask + mask_offset) : nullptr;
// Shared memory for reductions
__shared__ float s_max;
__shared__ float s_sum;
const int tid = threadIdx.x;
const int block_size = blockDim.x;
// Step 1: Find max (including sink)
float local_max = -INFINITY;
for (int k = tid; k < k_len; k += block_size) {
float val = (float)row_logits[k];
if (row_mask) {
val += (float)row_mask[k]; // mask is 0 or -inf
}
local_max = fmaxf(local_max, val);
}
// Include sink in max computation
if (tid == 0) {
local_max = fmaxf(local_max, sink_val);
}
// Warp reduction for max
local_max = warp_reduce_max(local_max);
// Block reduction for max (if more than 1 warp)
if (block_size > 32) {
__shared__ float warp_maxes[32];
const int warp_id = tid / 32;
const int lane_id = tid % 32;
if (lane_id == 0) {
warp_maxes[warp_id] = local_max;
}
__syncthreads();
if (tid < 32) {
local_max = (tid < (block_size + 31) / 32) ? warp_maxes[tid] : -INFINITY;
local_max = warp_reduce_max(local_max);
}
}
if (tid == 0) {
s_max = local_max;
}
__syncthreads();
const float row_max = s_max;
// Step 2: Compute exp(x - max) and sum (including sink)
float local_sum = 0.0f;
for (int k = tid; k < k_len; k += block_size) {
float val = (float)row_logits[k];
if (row_mask) {
val += (float)row_mask[k];
}
local_sum += expf(val - row_max);
}
// Include sink in sum computation
if (tid == 0) {
local_sum += expf(sink_val - row_max);
}
// Warp reduction for sum
local_sum = warp_reduce_sum(local_sum);
// Block reduction for sum
if (block_size > 32) {
__shared__ float warp_sums[32];
const int warp_id = tid / 32;
const int lane_id = tid % 32;
if (lane_id == 0) {
warp_sums[warp_id] = local_sum;
}
__syncthreads();
if (tid < 32) {
local_sum = (tid < (block_size + 31) / 32) ? warp_sums[tid] : 0.0f;
local_sum = warp_reduce_sum(local_sum);
}
}
if (tid == 0) {
s_sum = local_sum;
}
__syncthreads();
const float row_sum = s_sum;
// Step 3: Write normalized outputs (sink is NOT written - it's dropped)
const float inv_sum = 1.0f / row_sum;
for (int k = tid; k < k_len; k += block_size) {
float val = (float)row_logits[k];
if (row_mask) {
val += (float)row_mask[k];
}
row_output[k] = (T)(expf(val - row_max) * inv_sum);
}
}
// Launch wrapper for f16
extern "C" void softmax_with_sinks_f16(const __half *logits,
const __half *sinks, const __half *mask,
__half *output, int batch_size,
int num_heads, int q_len, int k_len,
float scale, cudaStream_t stream) {
const int total_rows = batch_size * num_heads * q_len;
// Choose block size based on k_len
int block_size = 256;
if (k_len <= 64)
block_size = 64;
else if (k_len <= 128)
block_size = 128;
else if (k_len <= 256)
block_size = 256;
else if (k_len <= 512)
block_size = 512;
else
block_size = 1024;
softmax_with_sinks_kernel<<<total_rows, block_size, 0, stream>>>(
logits, sinks, mask, output, batch_size, num_heads, q_len, k_len, scale);
CUDA_CHECK(cudaGetLastError());
}
// Launch wrapper for bf16
extern "C" void softmax_with_sinks_bf16(const __nv_bfloat16 *logits,
const __nv_bfloat16 *sinks,
const __nv_bfloat16 *mask,
__nv_bfloat16 *output, int batch_size,
int num_heads, int q_len, int k_len,
float scale, cudaStream_t stream) {
const int total_rows = batch_size * num_heads * q_len;
int block_size = 256;
if (k_len <= 64)
block_size = 64;
else if (k_len <= 128)
block_size = 128;
else if (k_len <= 256)
block_size = 256;
else if (k_len <= 512)
block_size = 512;
else
block_size = 1024;
softmax_with_sinks_kernel<<<total_rows, block_size, 0, stream>>>(
logits, sinks, mask, output, batch_size, num_heads, q_len, k_len, scale);
CUDA_CHECK(cudaGetLastError());
}
// Launch wrapper for f32
extern "C" void softmax_with_sinks_f32(const float *logits, const float *sinks,
const float *mask, float *output,
int batch_size, int num_heads, int q_len,
int k_len, float scale,
cudaStream_t stream) {
const int total_rows = batch_size * num_heads * q_len;
int block_size = 256;
if (k_len <= 64)
block_size = 64;
else if (k_len <= 128)
block_size = 128;
else if (k_len <= 256)
block_size = 256;
else if (k_len <= 512)
block_size = 512;
else
block_size = 1024;
softmax_with_sinks_kernel<<<total_rows, block_size, 0, stream>>>(
logits, sinks, mask, output, batch_size, num_heads, q_len, k_len, scale);
CUDA_CHECK(cudaGetLastError());
}
// ============================================================================
// Fused GLU (Gated Linear Unit) kernel: output = activation(a) * b
// Supports SiLU, GELU (approximate), and ReLU activations
//
// This fuses the activation function and element-wise multiplication into
// a single kernel pass, eliminating intermediate tensor allocation.
// ============================================================================
// Activation type enum - must match Rust GluActivationType
enum GluActivation {
GLU_SILU = 0,
GLU_GELU = 1,
GLU_RELU = 2,
GLU_GELU_ERF = 3
};
// SiLU activation: x * sigmoid(x)
__device__ __forceinline__ float glu_silu(float x) {
return x / (1.0f + expf(-x));
}
// GELU approximation: 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3)))
__device__ __forceinline__ float glu_gelu(float x) {
const float kSqrt2OverPi = 0.7978845608f;
const float kCoeff = 0.044715f;
float x3 = x * x * x;
float inner = kSqrt2OverPi * (x + kCoeff * x3);
return 0.5f * x * (1.0f + tanhf(inner));
}
// ReLU activation: max(0, x)
__device__ __forceinline__ float glu_relu(float x) { return fmaxf(x, 0.0f); }
// GELU (exact ERF version): x * normcdf(x), matching hanzo-ml's CUDA impl
__device__ __forceinline__ float glu_gelu_erf(float x) {
return x * normcdff(x);
}
__device__ __forceinline__ float apply_glu_activation(float x, int act) {
switch (act) {
case GLU_SILU:
return glu_silu(x);
case GLU_GELU:
return glu_gelu(x);
case GLU_RELU:
return glu_relu(x);
case GLU_GELU_ERF:
return glu_gelu_erf(x);
default:
return glu_silu(x);
}
}
// Scalar kernel for general case
template <typename T>
__global__ void fused_glu_kernel(const T *__restrict__ a, // input to activation
const T *__restrict__ b, // multiplier
T *__restrict__ output, const uint32_t N,
const int activation) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= N)
return;
float a_val = (float)a[idx];
// Cast activation back to T before multiplying, matching hanzo-ml's
// two-step behavior: unary op in float32 -> cast to T -> binary mul in T
T activated = (T)apply_glu_activation(a_val, activation);
output[idx] = activated * b[idx];
}
// Vectorized version for 4 elements at a time
template <typename T, typename T4>
__global__ void fused_glu_kernel_vec4(const T4 *__restrict__ a,
const T4 *__restrict__ b,
T4 *__restrict__ output,
const uint32_t N4, const int activation) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= N4)
return;
T4 a4 = a[idx];
T4 b4 = b[idx];
T4 out4;
float a0 = (float)((T *)&a4)[0];
float a1 = (float)((T *)&a4)[1];
float a2 = (float)((T *)&a4)[2];
float a3 = (float)((T *)&a4)[3];
// Cast activation back to T before multiplying, matching hanzo-ml's
// two-step behavior: unary op in float32 -> cast to T -> binary mul in T
T act0 = (T)apply_glu_activation(a0, activation);
T act1 = (T)apply_glu_activation(a1, activation);
T act2 = (T)apply_glu_activation(a2, activation);
T act3 = (T)apply_glu_activation(a3, activation);
((T *)&out4)[0] = act0 * ((T *)&b4)[0];
((T *)&out4)[1] = act1 * ((T *)&b4)[1];
((T *)&out4)[2] = act2 * ((T *)&b4)[2];
((T *)&out4)[3] = act3 * ((T *)&b4)[3];
output[idx] = out4;
}
extern "C" void fused_glu_f16(const __half *a, const __half *b, __half *output,
uint32_t N, int activation, cudaStream_t stream) {
if (N % 4 == 0) {
const int N4 = N / 4;
const int nthreads = 256;
const int nblocks = (N4 + nthreads - 1) / nthreads;
fused_glu_kernel_vec4<__half, uint64_t><<<nblocks, nthreads, 0, stream>>>(
(const uint64_t *)a, (const uint64_t *)b, (uint64_t *)output, N4,
activation);
} else {
const int nthreads = 256;
const int nblocks = (N + nthreads - 1) / nthreads;
fused_glu_kernel<<<nblocks, nthreads, 0, stream>>>(a, b, output, N,
activation);
}
CUDA_CHECK(cudaGetLastError());
}
extern "C" void fused_glu_bf16(const __nv_bfloat16 *a, const __nv_bfloat16 *b,
__nv_bfloat16 *output, uint32_t N,
int activation, cudaStream_t stream) {
if (N % 4 == 0) {
const int N4 = N / 4;
const int nthreads = 256;
const int nblocks = (N4 + nthreads - 1) / nthreads;
fused_glu_kernel_vec4<__nv_bfloat16, uint64_t>
<<<nblocks, nthreads, 0, stream>>>((const uint64_t *)a,
(const uint64_t *)b,
(uint64_t *)output, N4, activation);
} else {
const int nthreads = 256;
const int nblocks = (N + nthreads - 1) / nthreads;
fused_glu_kernel<<<nblocks, nthreads, 0, stream>>>(a, b, output, N,
activation);
}
CUDA_CHECK(cudaGetLastError());
}
extern "C" void fused_glu_f32(const float *a, const float *b, float *output,
uint32_t N, int activation, cudaStream_t stream) {
if (N % 4 == 0) {
const int N4 = N / 4;
const int nthreads = 256;
const int nblocks = (N4 + nthreads - 1) / nthreads;
fused_glu_kernel_vec4<float, float4><<<nblocks, nthreads, 0, stream>>>(
(const float4 *)a, (const float4 *)b, (float4 *)output, N4, activation);
} else {
const int nthreads = 256;
const int nblocks = (N + nthreads - 1) / nthreads;
fused_glu_kernel<<<nblocks, nthreads, 0, stream>>>(a, b, output, N,
activation);
}
CUDA_CHECK(cudaGetLastError());
}
__global__ void softcap_f32_kernel(const float *__restrict__ input,
float *__restrict__ output,
const uint32_t N, const float cap) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= N) {
return;
}
output[idx] = tanhf(input[idx] / cap) * cap;
}
__global__ void softcap_f16_to_f32_kernel(const __half *__restrict__ input,
float *__restrict__ output,
const uint32_t N, const float cap) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= N) {
return;
}
const float value = __half2float(input[idx]);
output[idx] = tanhf(value / cap) * cap;
}
__global__ void softcap_bf16_to_f32_kernel(
const __nv_bfloat16 *__restrict__ input, float *__restrict__ output,
const uint32_t N, const float cap) {
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= N) {
return;
}
const float value = __bfloat162float(input[idx]);
output[idx] = tanhf(value / cap) * cap;
}
extern "C" void softcap_f32(const float *input, float *output, uint32_t N,
float cap, cudaStream_t stream) {
const int nthreads = 256;
const int nblocks = (N + nthreads - 1) / nthreads;
softcap_f32_kernel<<<nblocks, nthreads, 0, stream>>>(input, output, N, cap);
CUDA_CHECK(cudaGetLastError());
}
extern "C" void softcap_f16_to_f32(const __half *input, float *output,
uint32_t N, float cap,
cudaStream_t stream) {
const int nthreads = 256;
const int nblocks = (N + nthreads - 1) / nthreads;
softcap_f16_to_f32_kernel<<<nblocks, nthreads, 0, stream>>>(
input, output, N, cap);
CUDA_CHECK(cudaGetLastError());
}
extern "C" void softcap_bf16_to_f32(const __nv_bfloat16 *input, float *output,
uint32_t N, float cap,
cudaStream_t stream) {
const int nthreads = 256;
const int nblocks = (N + nthreads - 1) / nthreads;
softcap_bf16_to_f32_kernel<<<nblocks, nthreads, 0, stream>>>(
input, output, N, cap);
CUDA_CHECK(cudaGetLastError());
}