mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-21 02:09:47 +00:00
283 lines
11 KiB
Plaintext
283 lines
11 KiB
Plaintext
/*
|
|
* Custom GEMV/GEMM CUDA Kernel for small batch sizes
|
|
*
|
|
* Optimized for LLM decode-phase inference (batch_size=1-8).
|
|
* Computes: Y = X @ A^T + bias where:
|
|
* - A: [M, K] weight matrix (row-major)
|
|
* - X: [B, K] input matrix (B = batch size, 1-8)
|
|
* - bias: [M] optional bias vector
|
|
* - Y: [B, M] output matrix
|
|
*
|
|
* Design follows llama.cpp mmvf.cu approach:
|
|
* - Simple loop without heavy unrolling
|
|
* - Vectorized loads (float2, half2, nv_bfloat162)
|
|
* - __ldg() for read-only cache path
|
|
* - Warp-level reduction using XOR shuffle
|
|
* - Supports batch sizes 1-8 efficiently
|
|
*/
|
|
|
|
#include <cstdint>
|
|
#include <cuda.h>
|
|
#include <cuda_bf16.h>
|
|
#include <cuda_fp16.h>
|
|
#include <cuda_runtime.h>
|
|
|
|
#define WARP_SIZE 32
|
|
#define MAX_BATCH_SIZE 8
|
|
|
|
// Warp-level reduction sum using XOR shuffle (butterfly pattern)
|
|
__device__ __forceinline__ float warp_reduce_sum(float val) {
|
|
#pragma unroll
|
|
for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) {
|
|
val += __shfl_xor_sync(0xffffffff, val, offset);
|
|
}
|
|
return val;
|
|
}
|
|
|
|
// Type conversion helpers
|
|
__device__ __forceinline__ float to_float(float x) { return x; }
|
|
__device__ __forceinline__ float to_float(__half x) { return __half2float(x); }
|
|
#if __CUDA_ARCH__ >= 800
|
|
__device__ __forceinline__ float to_float(__nv_bfloat16 x) {
|
|
return __bfloat162float(x);
|
|
}
|
|
#endif
|
|
|
|
// ============================================================================
|
|
// Batched GEMV Kernel - supports batch sizes 1-8
|
|
// ============================================================================
|
|
|
|
template <typename T, typename Vec2, int BLOCK_SIZE, int BATCH_SIZE>
|
|
__global__ void
|
|
gemv_kernel_batched(const T *__restrict__ A, // [M, K] weights (row-major)
|
|
const T *__restrict__ X, // [B, K] input matrix
|
|
const T *__restrict__ bias, // [M] optional bias
|
|
T *__restrict__ Y, // [B, M] output matrix
|
|
int M, int K, bool has_bias) {
|
|
const int row = blockIdx.x;
|
|
if (row >= M)
|
|
return;
|
|
|
|
const int tid = threadIdx.x;
|
|
const int K2 = K / 2;
|
|
|
|
// Static shared memory for block-level reduction
|
|
constexpr int NUM_WARPS = BLOCK_SIZE / WARP_SIZE;
|
|
__shared__ float warp_sums[NUM_WARPS][BATCH_SIZE];
|
|
|
|
// Direct pointer to weight row
|
|
const Vec2 *A_vec = reinterpret_cast<const Vec2 *>(A + row * K);
|
|
|
|
// Accumulators for each batch element
|
|
float acc[BATCH_SIZE];
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
acc[b] = 0.0f;
|
|
}
|
|
|
|
// Main loop over K dimension
|
|
for (int col2 = tid; col2 < K2; col2 += BLOCK_SIZE) {
|
|
Vec2 a_val = __ldg(A_vec + col2);
|
|
|
|
// Process each batch element
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
const Vec2 *x_vec = reinterpret_cast<const Vec2 *>(X + b * K);
|
|
Vec2 x_val = __ldg(x_vec + col2);
|
|
|
|
if constexpr (std::is_same_v<T, float>) {
|
|
acc[b] = fmaf(a_val.x, x_val.x, acc[b]);
|
|
acc[b] = fmaf(a_val.y, x_val.y, acc[b]);
|
|
} else if constexpr (std::is_same_v<T, __half>) {
|
|
float2 a_f = __half22float2(a_val);
|
|
float2 x_f = __half22float2(x_val);
|
|
acc[b] = fmaf(a_f.x, x_f.x, acc[b]);
|
|
acc[b] = fmaf(a_f.y, x_f.y, acc[b]);
|
|
}
|
|
#if __CUDA_ARCH__ >= 800
|
|
else if constexpr (std::is_same_v<T, __nv_bfloat16>) {
|
|
acc[b] =
|
|
fmaf(__bfloat162float(a_val.x), __bfloat162float(x_val.x), acc[b]);
|
|
acc[b] =
|
|
fmaf(__bfloat162float(a_val.y), __bfloat162float(x_val.y), acc[b]);
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
|
|
// Handle remainder if K is odd
|
|
if (K % 2 != 0 && tid == 0) {
|
|
int last_idx = K - 1;
|
|
float a_last = to_float(__ldg(A + row * K + last_idx));
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
float x_last = to_float(__ldg(X + b * K + last_idx));
|
|
acc[b] = fmaf(a_last, x_last, acc[b]);
|
|
}
|
|
}
|
|
|
|
// Warp-level reduction for each batch element
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
acc[b] = warp_reduce_sum(acc[b]);
|
|
}
|
|
|
|
// Block-level reduction via shared memory
|
|
if constexpr (BLOCK_SIZE > WARP_SIZE) {
|
|
int warp_id = tid / WARP_SIZE;
|
|
int lane_id = tid % WARP_SIZE;
|
|
|
|
if (lane_id == 0) {
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
warp_sums[warp_id][b] = acc[b];
|
|
}
|
|
}
|
|
__syncthreads();
|
|
|
|
if (warp_id == 0) {
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
acc[b] = (lane_id < NUM_WARPS) ? warp_sums[lane_id][b] : 0.0f;
|
|
acc[b] = warp_reduce_sum(acc[b]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Thread 0 writes the final results
|
|
if (tid == 0) {
|
|
float bias_val = has_bias ? to_float(__ldg(bias + row)) : 0.0f;
|
|
|
|
#pragma unroll
|
|
for (int b = 0; b < BATCH_SIZE; b++) {
|
|
float result = acc[b] + bias_val;
|
|
|
|
if constexpr (std::is_same_v<T, float>) {
|
|
Y[b * M + row] = result;
|
|
} else if constexpr (std::is_same_v<T, __half>) {
|
|
Y[b * M + row] = __float2half(result);
|
|
}
|
|
#if __CUDA_ARCH__ >= 800
|
|
else if constexpr (std::is_same_v<T, __nv_bfloat16>) {
|
|
Y[b * M + row] = __float2bfloat16(result);
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// Block Size Selection
|
|
// ============================================================================
|
|
|
|
__host__ int get_optimal_block_size(int K) {
|
|
const int K2 = K / 2;
|
|
if (K2 <= 32)
|
|
return 32;
|
|
if (K2 <= 64)
|
|
return 64;
|
|
if (K2 <= 128)
|
|
return 128;
|
|
return 256;
|
|
}
|
|
|
|
// ============================================================================
|
|
// Launch macro to reduce code duplication
|
|
// ============================================================================
|
|
|
|
#define LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, BATCH_SIZE) \
|
|
gemv_kernel_batched<T, Vec2, BLOCK_SIZE, BATCH_SIZE> \
|
|
<<<grid, dim3(BLOCK_SIZE), 0, stream>>>(A, X, bias, Y, M, K, has_bias)
|
|
|
|
#define DISPATCH_BATCH_SIZE(T, Vec2, BLOCK_SIZE) \
|
|
switch (batch_size) { \
|
|
case 1: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 1); \
|
|
break; \
|
|
case 2: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 2); \
|
|
break; \
|
|
case 3: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 3); \
|
|
break; \
|
|
case 4: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 4); \
|
|
break; \
|
|
case 5: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 5); \
|
|
break; \
|
|
case 6: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 6); \
|
|
break; \
|
|
case 7: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 7); \
|
|
break; \
|
|
case 8: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 8); \
|
|
break; \
|
|
default: \
|
|
LAUNCH_GEMV_BATCHED(T, Vec2, BLOCK_SIZE, 1); \
|
|
break; \
|
|
}
|
|
|
|
#define DISPATCH_BLOCK_SIZE(T, Vec2) \
|
|
switch (block_size) { \
|
|
case 32: \
|
|
DISPATCH_BATCH_SIZE(T, Vec2, 32); \
|
|
break; \
|
|
case 64: \
|
|
DISPATCH_BATCH_SIZE(T, Vec2, 64); \
|
|
break; \
|
|
case 128: \
|
|
DISPATCH_BATCH_SIZE(T, Vec2, 128); \
|
|
break; \
|
|
case 256: \
|
|
DISPATCH_BATCH_SIZE(T, Vec2, 256); \
|
|
break; \
|
|
default: \
|
|
DISPATCH_BATCH_SIZE(T, Vec2, 256); \
|
|
break; \
|
|
}
|
|
|
|
// ============================================================================
|
|
// Launch Functions - BF16
|
|
// ============================================================================
|
|
|
|
extern "C" void launch_gemv_bf16(const __nv_bfloat16 *A, const __nv_bfloat16 *X,
|
|
const __nv_bfloat16 *bias, __nv_bfloat16 *Y,
|
|
int M, int K, int batch_size, bool has_bias,
|
|
cudaStream_t stream) {
|
|
int block_size = get_optimal_block_size(K);
|
|
dim3 grid(M);
|
|
|
|
DISPATCH_BLOCK_SIZE(__nv_bfloat16, __nv_bfloat162);
|
|
}
|
|
|
|
// ============================================================================
|
|
// Launch Functions - F16
|
|
// ============================================================================
|
|
|
|
extern "C" void launch_gemv_f16(const __half *A, const __half *X,
|
|
const __half *bias, __half *Y, int M, int K,
|
|
int batch_size, bool has_bias,
|
|
cudaStream_t stream) {
|
|
int block_size = get_optimal_block_size(K);
|
|
dim3 grid(M);
|
|
|
|
DISPATCH_BLOCK_SIZE(__half, half2);
|
|
}
|
|
|
|
// ============================================================================
|
|
// Launch Functions - F32
|
|
// ============================================================================
|
|
|
|
extern "C" void launch_gemv_f32(const float *A, const float *X,
|
|
const float *bias, float *Y, int M, int K,
|
|
int batch_size, bool has_bias,
|
|
cudaStream_t stream) {
|
|
int block_size = get_optimal_block_size(K);
|
|
dim3 grid(M);
|
|
|
|
DISPATCH_BLOCK_SIZE(float, float2);
|
|
}
|