mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-21 02:36:58 +00:00
103 lines
3.8 KiB
Plaintext
103 lines
3.8 KiB
Plaintext
/**
|
|
* Per-token, per-128-group E4M3 activation quantization, as the served kernels compute it.
|
|
*
|
|
* Modes (one scale s per row and group, q = e4m3 RN satfinite of clamp(x / s, +-448)):
|
|
* 0 linear: s = max(amax * RN(1/448), 1/229376), x / s by div.full.f32. The inductor kernel that
|
|
* quantizes every W8A8 input in the compiled graph (it folds the /448 into a multiply).
|
|
* 1 eager: s = max(amax div.full 448, 1/229376). The standalone QuantFP8 kernel that runs inside
|
|
* custom ops (the shared expert inside moe_forward_shared), divisor passed at runtime.
|
|
* 2 gather: vLLM's per_token_group_fp8_quant (eps 1e-10), used by the fused-MoE gathers:
|
|
* s = max(amax, 1e-10) / 448 (IEEE), q = x / s (IEEE).
|
|
*
|
|
* Built without --use_fast_math and with --fmad=false: div.full.f32 is written as inline PTX where
|
|
* the served kernel uses it, and every other operation is the IEEE one.
|
|
*/
|
|
|
|
#include "e2m1.cuh"
|
|
|
|
#include <cuda_bf16.h>
|
|
#include <cuda_fp16.h>
|
|
#include <cuda_fp8.h>
|
|
#include <cuda_runtime.h>
|
|
#include <stdint.h>
|
|
|
|
namespace {
|
|
|
|
constexpr int GROUP = 128;
|
|
constexpr float RCP448 = 0.002232142857142857f;
|
|
constexpr float FLOOR = 4.359654017857143e-06f;
|
|
|
|
template <typename T> __device__ __forceinline__ float to_f32(T v);
|
|
template <> __device__ __forceinline__ float to_f32<float>(float v) { return v; }
|
|
template <> __device__ __forceinline__ float to_f32<__half>(__half v) { return __half2float(v); }
|
|
template <> __device__ __forceinline__ float to_f32<__nv_bfloat16>(__nv_bfloat16 v) {
|
|
return __bfloat162float(v);
|
|
}
|
|
|
|
// One warp per (row, group): 32 lanes x 4 values.
|
|
template <typename T>
|
|
__global__ void fp8_kernel(const T *__restrict__ x, uint8_t *__restrict__ q,
|
|
float *__restrict__ s, int m, int k, int mode) {
|
|
const int groups = k / GROUP;
|
|
const long warp = ((long)blockIdx.x * blockDim.x + threadIdx.x) / 32;
|
|
const int lane = threadIdx.x % 32;
|
|
if (warp >= (long)m * groups)
|
|
return;
|
|
const long row = warp / groups;
|
|
const int g = warp % groups;
|
|
const T *src = x + row * k + g * GROUP;
|
|
float v[4];
|
|
float amax = 0.0f;
|
|
#pragma unroll
|
|
for (int j = 0; j < 4; j++) {
|
|
v[j] = to_f32<T>(src[lane * 4 + j]);
|
|
amax = fmaxf(amax, fabsf(v[j]));
|
|
}
|
|
#pragma unroll
|
|
for (int off = 16; off > 0; off >>= 1)
|
|
amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, off));
|
|
|
|
float scale;
|
|
if (mode == 0)
|
|
scale = fmaxf(amax * RCP448, FLOOR);
|
|
else if (mode == 1)
|
|
scale = fmaxf(e2m1::div_full(amax, 448.0f), FLOOR);
|
|
else
|
|
scale = fmaxf(amax, 1e-10f) / 448.0f;
|
|
|
|
uint8_t *dst = q + row * k + g * GROUP + lane * 4;
|
|
#pragma unroll
|
|
for (int j = 0; j < 4; j++) {
|
|
float r = mode == 2 ? v[j] / scale : e2m1::div_full(v[j], scale);
|
|
r = fminf(fmaxf(r, -448.0f), 448.0f);
|
|
dst[j] = __nv_cvt_float_to_fp8(r, __NV_SATFINITE, __NV_E4M3);
|
|
}
|
|
if (lane == 0)
|
|
s[row * groups + g] = scale;
|
|
}
|
|
|
|
template <typename T>
|
|
void launch(const T *x, uint8_t *q, float *s, int m, int k, int mode, cudaStream_t stream) {
|
|
const long warps = (long)m * (k / GROUP);
|
|
const int threads = 256;
|
|
const long blocks = (warps * 32 + threads - 1) / threads;
|
|
fp8_kernel<T><<<(unsigned)blocks, threads, 0, stream>>>(x, q, s, m, k, mode);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
extern "C" void hanzo_quantize_fp8_f32(const float *x, uint8_t *q, float *s, int m, int k,
|
|
int mode, cudaStream_t stream) {
|
|
launch<float>(x, q, s, m, k, mode, stream);
|
|
}
|
|
|
|
extern "C" void hanzo_quantize_fp8_f16(const __half *x, uint8_t *q, float *s, int m, int k,
|
|
int mode, cudaStream_t stream) {
|
|
launch<__half>(x, q, s, m, k, mode, stream);
|
|
}
|
|
|
|
extern "C" void hanzo_quantize_fp8_bf16(const __nv_bfloat16 *x, uint8_t *q, float *s, int m,
|
|
int k, int mode, cudaStream_t stream) {
|
|
launch<__nv_bfloat16>(x, q, s, m, k, mode, stream);
|
|
}
|