mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-21 02:36:58 +00:00
Every mention of another inference engine is rewritten to say what our code does, or the math it computes. Reference values are "the reference value from the published config" or "the recorded reference output", line citations into another engine's source become the rule itself, comparisons with other engines are dropped, and asides about other engines' bugs go. Identifiers follow: - CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels' `hanzo_rotary`; the shuffle/ldg macros are HANZO_*. - GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side. - Tests: `*_matches_vllm` and friends name what they compare against (the reference fixture, the tile rule). - MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op is `rotary-embedding`. - hanzo-bench's board scores ours against a "rival" whose name comes from the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s` columns; dossier.sh and spec_decode.sh take `--rival-bench`. - install.ps1 installs this repository's hanzo-cli, as install.sh does, with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL. - bench_http.py reads only the hanzo: metric families. Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and tools/upstream-map.sh; each ported file's copyright/licence/provenance header; Cargo.toml dependency names; data identifiers (Hugging Face repo IDs, fixture file names and fixture metadata keys the tests read, on-disk paths, executable names the GPU and engine guards match on a host); model and format names. LLM.md says the rule once, under "One name: Hanzo Engine". Co-authored-by: Hanzo Dev <[email protected]>
283 lines
11 KiB
Plaintext
283 lines
11 KiB
Plaintext
/*
|
|
* Custom GEMV/GEMM CUDA Kernel for small batch sizes
|
|
*
|
|
* Optimized for LLM decode-phase inference (batch_size=1-8).
|
|
* Computes: Y = X @ A^T + bias where:
|
|
* - A: [M, K] weight matrix (row-major)
|
|
* - X: [B, K] input matrix (B = batch size, 1-8)
|
|
* - bias: [M] optional bias vector
|
|
* - Y: [B, M] output matrix
|
|
*
|
|
* Design:
|
|
* - Simple loop without heavy unrolling
|
|
* - Vectorized loads (float2, half2, nv_bfloat162)
|
|
* - __ldg() for read-only cache path
|
|
* - Warp-level reduction using XOR shuffle
|
|
* - Supports batch sizes 1-8 efficiently
|
|
*/
|
|
|
|
#include <cstdint>
|
|
#include <cuda.h>
|
|
#include <cuda_bf16.h>
|
|
#include <cuda_fp16.h>
|
|
#include <cuda_runtime.h>
|
|
|
|
#define WARP_SIZE 32
|
|
#define MAX_BATCH_SIZE 8
|
|
|
|
// Warp-level reduction sum using XOR shuffle (butterfly pattern)
|
|
__device__ __forceinline__ float warp_reduce_sum(float val) {
|
|
#pragma unroll
|
|
for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) {
|
|
val += __shfl_xor_sync(0xffffffff, val, offset);
|
|
}
|
|
return val;
|
|
}
|
|
|
|
// Type conversion helpers
|
|
__device__ __forceinline__ float to_float(float x) { return x; }
|
|
__device__ __forceinline__ float to_float(__half x) { return __half2float(x); }
|
|
#if __CUDA_ARCH__ >= 800
|
|
__device__ __forceinline__ float to_float(__nv_bfloat16 x) {
|
|
return __bfloat162float(x);
|
|
}
|
|
#endif
|
|
|
|
// ============================================================================
|
|
// Batched GEMV Kernel - supports batch sizes 1-8
|
|
// ============================================================================
|
|
|
|
template <typename T, typename Vec2, int BLOCK_SIZE, int BATCH_SIZE>
|
|
__global__ void
|
|
gemv_kernel_batched(const T *__restrict__ A, // [M, K] weights (row-major)
|
|
const T *__restrict__ X, // [B, K] input matrix
|
|
const T *__restrict__ bias, // [M] optional bias
|
|
T *__restrict__ Y, // [B, M] output matrix
|
|
int M, int K, bool has_bias) {
|
|
const int row = blockIdx.x;
|
|
if (row >= M)
|
|
return;
|
|
|
|
const int tid = threadIdx.x;
|
|
const int K2 = K / 2;
|
|
|
|
// Static shared memory for block-level reduction
|
|
constexpr int NUM_WARPS = BLOCK_SIZE / WARP_SIZE;
|
|
__shared__ float warp_sums[NUM_WARPS][BATCH_SIZE];
|
|
|
|
// Direct pointer to weight row
|
|
const Vec2 *A_vec = reinterpret_cast<const Vec2 *>(A + row * K);
|
|
|
|
// Accumulators for each batch element
|
|
float acc[BATCH_SIZE];
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
acc[b] = 0.0f;
|
|
}
|
|
|
|
// Main loop over K dimension
|
|
for (int col2 = tid; col2 < K2; col2 += BLOCK_SIZE) {
|
|
Vec2 a_val = __ldg(A_vec + col2);
|
|
|
|
// Process each batch element
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
const Vec2 *x_vec = reinterpret_cast<const Vec2 *>(X + b * K);
|
|
Vec2 x_val = __ldg(x_vec + col2);
|
|
|
|
if constexpr (std::is_same_v<T, float>) {
|
|
acc[b] = fmaf(a_val.x, x_val.x, acc[b]);
|
|
acc[b] = fmaf(a_val.y, x_val.y, acc[b]);
|
|
} else if constexpr (std::is_same_v<T, __half>) {
|
|
float2 a_f = __half22float2(a_val);
|
|
float2 x_f = __half22float2(x_val);
|
|
acc[b] = fmaf(a_f.x, x_f.x, acc[b]);
|
|
acc[b] = fmaf(a_f.y, x_f.y, acc[b]);
|
|
}
|
|
#if __CUDA_ARCH__ >= 800
|
|
else if constexpr (std::is_same_v<T, __nv_bfloat16>) {
|
|
acc[b] =
|
|
fmaf(__bfloat162float(a_val.x), __bfloat162float(x_val.x), acc[b]);
|
|
acc[b] =
|
|
fmaf(__bfloat162float(a_val.y), __bfloat162float(x_val.y), acc[b]);
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
|
|
// Handle remainder if K is odd
|
|
if (K % 2 != 0 && tid == 0) {
|
|
int last_idx = K - 1;
|
|
float a_last = to_float(__ldg(A + row * K + last_idx));
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
float x_last = to_float(__ldg(X + b * K + last_idx));
|
|
acc[b] = fmaf(a_last, x_last, acc[b]);
|
|
}
|
|
}
|
|
|
|
// Warp-level reduction for each batch element
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
acc[b] = warp_reduce_sum(acc[b]);
|
|
}
|
|
|
|
// Block-level reduction via shared memory
|
|
if constexpr (BLOCK_SIZE > WARP_SIZE) {
|
|
int warp_id = tid / WARP_SIZE;
|
|
int lane_id = tid % WARP_SIZE;
|
|
|
|
if (lane_id == 0) {
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
warp_sums[warp_id][b] = acc[b];
|
|
}
|
|
}
|
|
__syncthreads();
|
|
|
|
if (warp_id == 0) {
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
acc[b] = (lane_id < NUM_WARPS) ? warp_sums[lane_id][b] : 0.0f;
|
|
acc[b] = warp_reduce_sum(acc[b]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Thread 0 writes the final results
|
|
if (tid == 0) {
|
|
float bias_val = has_bias ? to_float(__ldg(bias + row)) : 0.0f;
|
|
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
float result = acc[b] + bias_val;
|
|
|
|
if constexpr (std::is_same_v<T, float>) {
|
|
Y[b * M + row] = result;
|
|
} else if constexpr (std::is_same_v<T, __half>) {
|
|
Y[b * M + row] = __float2half(result);
|
|
}
|
|
#if __CUDA_ARCH__ >= 800
|
|
else if constexpr (std::is_same_v<T, __nv_bfloat16>) {
|
|
Y[b * M + row] = __float2bfloat16(result);
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// Block Size Selection
|
|
// ============================================================================
|
|
|
|
__host__ int get_optimal_block_size(int K) {
|
|
const int K2 = K / 2;
|
|
if (K2 <= 32)
|
|
return 32;
|
|
if (K2 <= 64)
|
|
return 64;
|
|
if (K2 <= 128)
|
|
return 128;
|
|
return 256;
|
|
}
|
|
|
|
// ============================================================================
|
|
// Launch macro to reduce code duplication
|
|
// ============================================================================
|
|
|
|
#define LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, BATCH_SIZE) \
|
|
gemv_kernel_batched<T, Vec2, BLOCK_SIZE, BATCH_SIZE> \
|
|
<<<grid, dim3(BLOCK_SIZE), 0, stream>>>(A, X, bias, Y, M, K, has_bias)
|
|
|
|
#define DISPATCH_BATCH_SIZE(T, Vec2, BLOCK_SIZE) \
|
|
switch (batch_size) { \
|
|
case 1: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 1); \
|
|
break; \
|
|
case 2: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 2); \
|
|
break; \
|
|
case 3: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 3); \
|
|
break; \
|
|
case 4: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 4); \
|
|
break; \
|
|
case 5: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 5); \
|
|
break; \
|
|
case 6: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 6); \
|
|
break; \
|
|
case 7: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 7); \
|
|
break; \
|
|
case 8: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 8); \
|
|
break; \
|
|
default: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 1); \
|
|
break; \
|
|
}
|
|
|
|
#define DISPATCH_BLOCK_SIZE(T, Vec2) \
|
|
switch (block_size) { \
|
|
case 32: \
|
|
DISPATCH_BATCH_SIZE(T, Vec2, 32); \
|
|
break; \
|
|
case 64: \
|
|
DISPATCH_BATCH_SIZE(T, Vec2, 64); \
|
|
break; \
|
|
case 128: \
|
|
DISPATCH_BATCH_SIZE(T, Vec2, 128); \
|
|
break; \
|
|
case 256: \
|
|
DISPATCH_BATCH_SIZE(T, Vec2, 256); \
|
|
break; \
|
|
default: \
|
|
DISPATCH_BATCH_SIZE(T, Vec2, 256); \
|
|
break; \
|
|
}
|
|
|
|
// ============================================================================
|
|
// Launch Functions - BF16
|
|
// ============================================================================
|
|
|
|
extern "C" void launch_gemv_bf16(const __nv_bfloat16 *A, const __nv_bfloat16 *X,
|
|
const __nv_bfloat16 *bias, __nv_bfloat16 *Y,
|
|
int M, int K, int batch_size, bool has_bias,
|
|
cudaStream_t stream) {
|
|
int block_size = get_optimal_block_size(K);
|
|
dim3 grid(M);
|
|
|
|
DISPATCH_BLOCK_SIZE(__nv_bfloat16, __nv_bfloat162);
|
|
}
|
|
|
|
// ============================================================================
|
|
// Launch Functions - F16
|
|
// ============================================================================
|
|
|
|
extern "C" void launch_gemv_f16(const __half *A, const __half *X,
|
|
const __half *bias, __half *Y, int M, int K,
|
|
int batch_size, bool has_bias,
|
|
cudaStream_t stream) {
|
|
int block_size = get_optimal_block_size(K);
|
|
dim3 grid(M);
|
|
|
|
DISPATCH_BLOCK_SIZE(__half, half2);
|
|
}
|
|
|
|
// ============================================================================
|
|
// Launch Functions - F32
|
|
// ============================================================================
|
|
|
|
extern "C" void launch_gemv_f32(const float *A, const float *X,
|
|
const float *bias, float *Y, int M, int K,
|
|
int batch_size, bool has_bias,
|
|
cudaStream_t stream) {
|
|
int block_size = get_optimal_block_size(K);
|
|
dim3 grid(M);
|
|
|
|
DISPATCH_BLOCK_SIZE(float, float2);
|
|
}
|