mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-21 02:36:58 +00:00
Upstream is 201 commits past our merge base and `git merge` cannot reach them: the crates were renamed at the fork and the contents have diverged past any similarity threshold, so it re-adds their whole tree at the old mistralrs-* paths, 1,622 duplicate files, merging nothing. Routing each commit through tools/upstream-map.sh and applying it strictly is what works, and six land: 28614903c6 build(quant): harden the Metal kernel build against silent failures 00496e6818 fix(metal): register PR #2166 kernels in the runtime-compile path e5b9b93d37 feat(metal): fix sdpa offset handling, remove the f32 upcast bb5ee55de9 fix(afq): fix afq launch/index math for large MoE expert stacks b6521a41c3 fix(tune): detect the CUDA backend on Windows 56d686b060 fix(webui): send chat images as data URLs Refused by rule: anything touching Cargo.toml, LICENSE, NOTICE, README or the install scripts. Their manifests pin candle from huggingface where we pin our own hanzo-ml fork, and a 3-way apply did try to swap them. The rest of what is worth having is new subsystems, not patches: NVFP4 through cuTile and CUTLASS, paged flash attention, cuTile GDN prefill. Those get ported by hand. Co-authored-by: Hanzo Dev <[email protected]>
310 lines
9.8 KiB
Plaintext
310 lines
9.8 KiB
Plaintext
/*
|
|
* AFQ (Affine Fast Quantization) CUDA Utilities
|
|
*
|
|
* Common definitions and helper functions for AFQ CUDA kernels.
|
|
*/
|
|
|
|
#ifndef AFQ_UTILS_CUH
|
|
#define AFQ_UTILS_CUH
|
|
|
|
#include <cuda.h>
|
|
#include <cuda_bf16.h>
|
|
#include <cuda_fp16.h>
|
|
#include <cuda_runtime.h>
|
|
#include <cstdint>
|
|
|
|
// Block sizes for various operations
|
|
#define AFQ_BLOCK_SIZE 256
|
|
#define AFQ_WARP_SIZE 32
|
|
|
|
// Tile sizes for matrix operations
|
|
#define AFQ_TILE_M 32
|
|
#define AFQ_TILE_N 32
|
|
#define AFQ_TILE_K 32
|
|
|
|
// Utility: ceiling division
|
|
__host__ __device__ __forceinline__ unsigned int cdiv(unsigned int a,
|
|
unsigned int b) {
|
|
return (a + b - 1) / b;
|
|
}
|
|
|
|
__host__ __device__ __forceinline__ uint64_t cdiv_u64(uint64_t a, uint64_t b) {
|
|
return (a + b - 1) / b;
|
|
}
|
|
|
|
// ============================================================================
|
|
// Bit extraction helpers
|
|
// ============================================================================
|
|
|
|
// Extract N-bit value from packed u32 at given position
|
|
template <int bits>
|
|
__device__ __forceinline__ uint32_t extract_bits(uint32_t packed, int pos) {
|
|
constexpr uint32_t mask = (1u << bits) - 1;
|
|
return (packed >> (pos * bits)) & mask;
|
|
}
|
|
|
|
// Specialized extraction for 3-bit (spans bytes)
|
|
__device__ __forceinline__ uint32_t extract_3bit(const uint8_t *data,
|
|
int idx) {
|
|
// 8 values packed into 3 bytes (24 bits)
|
|
int byte_offset = (idx * 3) / 8;
|
|
int bit_offset = (idx * 3) % 8;
|
|
|
|
uint32_t val;
|
|
if (bit_offset <= 5) {
|
|
val = (data[byte_offset] >> bit_offset) & 0x7;
|
|
} else {
|
|
val = ((data[byte_offset] >> bit_offset) |
|
|
(data[byte_offset + 1] << (8 - bit_offset))) &
|
|
0x7;
|
|
}
|
|
return val;
|
|
}
|
|
|
|
// Specialized extraction for 6-bit (spans bytes)
|
|
__device__ __forceinline__ uint32_t extract_6bit(const uint8_t *data,
|
|
int idx) {
|
|
// 4 values packed into 3 bytes (24 bits)
|
|
int byte_offset = (idx * 6) / 8;
|
|
int bit_offset = (idx * 6) % 8;
|
|
|
|
uint32_t val;
|
|
if (bit_offset <= 2) {
|
|
val = (data[byte_offset] >> bit_offset) & 0x3f;
|
|
} else {
|
|
val = ((data[byte_offset] >> bit_offset) |
|
|
(data[byte_offset + 1] << (8 - bit_offset))) &
|
|
0x3f;
|
|
}
|
|
return val;
|
|
}
|
|
|
|
// ============================================================================
|
|
// Dequantization helpers (q * scale + bias)
|
|
// ============================================================================
|
|
|
|
template <typename T>
|
|
__device__ __forceinline__ T dequant_value(uint32_t q, T scale, T bias);
|
|
|
|
template <>
|
|
__device__ __forceinline__ float dequant_value<float>(uint32_t q, float scale,
|
|
float bias) {
|
|
return fmaf((float)q, scale, bias);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ __half dequant_value<__half>(uint32_t q, __half scale,
|
|
__half bias) {
|
|
return __hfma(__uint2half_rn(q), scale, bias);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ __nv_bfloat16
|
|
dequant_value<__nv_bfloat16>(uint32_t q, __nv_bfloat16 scale,
|
|
__nv_bfloat16 bias) {
|
|
#if __CUDA_ARCH__ >= 800
|
|
return __hfma(__uint2bfloat16_rn(q), scale, bias);
|
|
#else
|
|
float out =
|
|
fmaf((float)q, __bfloat162float(scale), __bfloat162float(bias));
|
|
return __float2bfloat16(out);
|
|
#endif
|
|
}
|
|
|
|
// ============================================================================
|
|
// Quantization helpers (round((w - bias) / scale))
|
|
// ============================================================================
|
|
|
|
template <typename T, int bits>
|
|
__device__ __forceinline__ uint32_t quant_value(T w, T scale, T bias);
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t quant_value<float, 2>(float w, float scale,
|
|
float bias) {
|
|
float q = roundf((w - bias) / scale);
|
|
return min(max((int)q, 0), 3);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t quant_value<float, 3>(float w, float scale,
|
|
float bias) {
|
|
float q = roundf((w - bias) / scale);
|
|
return min(max((int)q, 0), 7);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t quant_value<float, 4>(float w, float scale,
|
|
float bias) {
|
|
float q = roundf((w - bias) / scale);
|
|
return min(max((int)q, 0), 15);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t quant_value<float, 6>(float w, float scale,
|
|
float bias) {
|
|
float q = roundf((w - bias) / scale);
|
|
return min(max((int)q, 0), 63);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t quant_value<float, 8>(float w, float scale,
|
|
float bias) {
|
|
float q = roundf((w - bias) / scale);
|
|
return min(max((int)q, 0), 255);
|
|
}
|
|
|
|
// Half precision versions
|
|
template <>
|
|
__device__ __forceinline__ uint32_t quant_value<__half, 2>(__half w,
|
|
__half scale,
|
|
__half bias) {
|
|
float wf = __half2float(w);
|
|
float sf = __half2float(scale);
|
|
float bf = __half2float(bias);
|
|
float q = roundf((wf - bf) / sf);
|
|
return min(max((int)q, 0), 3);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t quant_value<__half, 3>(__half w,
|
|
__half scale,
|
|
__half bias) {
|
|
float wf = __half2float(w);
|
|
float sf = __half2float(scale);
|
|
float bf = __half2float(bias);
|
|
float q = roundf((wf - bf) / sf);
|
|
return min(max((int)q, 0), 7);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t quant_value<__half, 4>(__half w,
|
|
__half scale,
|
|
__half bias) {
|
|
float wf = __half2float(w);
|
|
float sf = __half2float(scale);
|
|
float bf = __half2float(bias);
|
|
float q = roundf((wf - bf) / sf);
|
|
return min(max((int)q, 0), 15);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t quant_value<__half, 6>(__half w,
|
|
__half scale,
|
|
__half bias) {
|
|
float wf = __half2float(w);
|
|
float sf = __half2float(scale);
|
|
float bf = __half2float(bias);
|
|
float q = roundf((wf - bf) / sf);
|
|
return min(max((int)q, 0), 63);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t quant_value<__half, 8>(__half w,
|
|
__half scale,
|
|
__half bias) {
|
|
float wf = __half2float(w);
|
|
float sf = __half2float(scale);
|
|
float bf = __half2float(bias);
|
|
float q = roundf((wf - bf) / sf);
|
|
return min(max((int)q, 0), 255);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t
|
|
quant_value<__nv_bfloat16, 2>(__nv_bfloat16 w, __nv_bfloat16 scale,
|
|
__nv_bfloat16 bias) {
|
|
float wf = __bfloat162float(w);
|
|
float sf = __bfloat162float(scale);
|
|
float bf = __bfloat162float(bias);
|
|
float q = roundf((wf - bf) / sf);
|
|
return min(max((int)q, 0), 3);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t
|
|
quant_value<__nv_bfloat16, 3>(__nv_bfloat16 w, __nv_bfloat16 scale,
|
|
__nv_bfloat16 bias) {
|
|
float wf = __bfloat162float(w);
|
|
float sf = __bfloat162float(scale);
|
|
float bf = __bfloat162float(bias);
|
|
float q = roundf((wf - bf) / sf);
|
|
return min(max((int)q, 0), 7);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t
|
|
quant_value<__nv_bfloat16, 4>(__nv_bfloat16 w, __nv_bfloat16 scale,
|
|
__nv_bfloat16 bias) {
|
|
float wf = __bfloat162float(w);
|
|
float sf = __bfloat162float(scale);
|
|
float bf = __bfloat162float(bias);
|
|
float q = roundf((wf - bf) / sf);
|
|
return min(max((int)q, 0), 15);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t
|
|
quant_value<__nv_bfloat16, 6>(__nv_bfloat16 w, __nv_bfloat16 scale,
|
|
__nv_bfloat16 bias) {
|
|
float wf = __bfloat162float(w);
|
|
float sf = __bfloat162float(scale);
|
|
float bf = __bfloat162float(bias);
|
|
float q = roundf((wf - bf) / sf);
|
|
return min(max((int)q, 0), 63);
|
|
}
|
|
|
|
template <>
|
|
__device__ __forceinline__ uint32_t
|
|
quant_value<__nv_bfloat16, 8>(__nv_bfloat16 w, __nv_bfloat16 scale,
|
|
__nv_bfloat16 bias) {
|
|
float wf = __bfloat162float(w);
|
|
float sf = __bfloat162float(scale);
|
|
float bf = __bfloat162float(bias);
|
|
float q = roundf((wf - bf) / sf);
|
|
return min(max((int)q, 0), 255);
|
|
}
|
|
|
|
// ============================================================================
|
|
// Warp-level primitives
|
|
// ============================================================================
|
|
|
|
__device__ __forceinline__ float warp_reduce_sum(float val) {
|
|
#pragma unroll
|
|
for (int offset = AFQ_WARP_SIZE / 2; offset > 0; offset /= 2) {
|
|
val += __shfl_down_sync(0xffffffff, val, offset);
|
|
}
|
|
return val;
|
|
}
|
|
|
|
__device__ __forceinline__ float warp_reduce_max(float val) {
|
|
#pragma unroll
|
|
for (int offset = AFQ_WARP_SIZE / 2; offset > 0; offset /= 2) {
|
|
val = fmaxf(val, __shfl_down_sync(0xffffffff, val, offset));
|
|
}
|
|
return val;
|
|
}
|
|
|
|
__device__ __forceinline__ float warp_reduce_min(float val) {
|
|
#pragma unroll
|
|
for (int offset = AFQ_WARP_SIZE / 2; offset > 0; offset /= 2) {
|
|
val = fminf(val, __shfl_down_sync(0xffffffff, val, offset));
|
|
}
|
|
return val;
|
|
}
|
|
|
|
// Broadcast value from lane 0 to all lanes
|
|
__device__ __forceinline__ float warp_broadcast(float val) {
|
|
return __shfl_sync(0xffffffff, val, 0);
|
|
}
|
|
|
|
// ============================================================================
|
|
// Group index calculation
|
|
// ============================================================================
|
|
|
|
// Given a column index, compute the group index
|
|
__device__ __forceinline__ int get_group_idx(int col, int group_size) {
|
|
return col / group_size;
|
|
}
|
|
|
|
#endif // AFQ_UTILS_CUH
|