Files
engine/hanzo-quant/kernels/quantize/nvfp4.cu
zeekayandHanzo Dev 42e34c2d41 refactor: name only Hanzo Engine in code, comments and docs
Every mention of another inference engine is rewritten to say what our code
does, or the math it computes. Reference values are "the reference value
from the published config" or "the recorded reference output", line
citations into another engine's source become the rule itself, comparisons
with other engines are dropped, and asides about other engines' bugs go.

Identifiers follow:
- CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's
  MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels'
  `hanzo_rotary`; the shuffle/ldg macros are HANZO_*.
- GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is
  `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side.
- Tests: `*_matches_vllm` and friends name what they compare against (the
  reference fixture, the tile rule).
- MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op
  is `rotary-embedding`.
- hanzo-bench's board scores ours against a "rival" whose name comes from
  the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s`
  columns; dossier.sh and spec_decode.sh take `--rival-bench`.
- install.ps1 installs this repository's hanzo-cli, as install.sh does,
  with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL.
- bench_http.py reads only the hanzo: metric families.

Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and
tools/upstream-map.sh; each ported file's copyright/licence/provenance
header; Cargo.toml dependency names; data identifiers (Hugging Face repo
IDs, fixture file names and fixture metadata keys the tests read, on-disk
paths, executable names the GPU and engine guards match on a host); model
and format names.

LLM.md says the rule once, under "One name: Hanzo Engine".


Co-authored-by: Hanzo Dev <[email protected]>
2026-10-16 20:59:50 -07:00

86 lines
3.0 KiB
Plaintext

/**
* NVFP4 activation quantization with the served fast-math formula (e2m1.cuh), linear scale
* layout: codes [M, K/2] (low nibble first), E4M3 scales [M, K/16].
*
* The same formula is scaled_fp4_quant's (the fc1 input of the NVFP4 MoE and the dense NVFP4
* layers) and FlashInfer's cvt_warp_fp16_to_fp4's (the fc2 input, after the bf16 SwiGLU).
*/
#include "e2m1.cuh"
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include <stdint.h>
namespace {
constexpr int BLOCK = 16;
template <typename T> __device__ __forceinline__ float to_f32(T v);
template <> __device__ __forceinline__ float to_f32<__half>(__half v) { return __half2float(v); }
template <> __device__ __forceinline__ float to_f32<__nv_bfloat16>(__nv_bfloat16 v) {
return __bfloat162float(v);
}
// One thread per block of 16.
template <typename T>
__global__ void nvfp4_kernel(const T *__restrict__ x, uint8_t *__restrict__ codes,
uint8_t *__restrict__ scales, long blocks, float gs) {
for (long b = (long)blockIdx.x * blockDim.x + threadIdx.x; b < blocks;
b += (long)gridDim.x * blockDim.x) {
const T *src = x + b * BLOCK;
float v[BLOCK];
float vmax = 0.0f;
#pragma unroll
for (int j = 0; j < BLOCK; j++) {
v[j] = to_f32<T>(src[j]);
vmax = fmaxf(vmax, fabsf(v[j]));
}
const e2m1::Block blk = e2m1::block(vmax, gs);
scales[b] = blk.scale;
uint8_t *dst = codes + b * (BLOCK / 2);
#pragma unroll
for (int j = 0; j < BLOCK / 2; j++) {
const uint8_t lo = e2m1::encode(v[2 * j] * blk.out);
const uint8_t hi = e2m1::encode(v[2 * j + 1] * blk.out);
dst[j] = lo | (hi << 4);
}
}
}
template <typename T>
void launch(const T *x, uint8_t *codes, uint8_t *scales, int m, int k, float gs,
cudaStream_t stream) {
const long blocks = (long)m * (k / BLOCK);
const int threads = 256;
long grid = (blocks + threads - 1) / threads;
if (grid > 65535)
grid = 65535;
nvfp4_kernel<T><<<(unsigned)grid, threads, 0, stream>>>(x, codes, scales, blocks, gs);
}
// Round-trip probes for the encoder tests: one code per input.
__global__ void e2m1_encode_kernel(const float *v, uint8_t *codes, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < n)
codes[i] = e2m1::encode(v[i]);
}
} // namespace
extern "C" void hanzo_quantize_nvfp4_f16(const __half *x, uint8_t *codes, uint8_t *scales, int m,
int k, float gs, cudaStream_t stream) {
launch<__half>(x, codes, scales, m, k, gs, stream);
}
extern "C" void hanzo_quantize_nvfp4_bf16(const __nv_bfloat16 *x, uint8_t *codes,
uint8_t *scales, int m, int k, float gs,
cudaStream_t stream) {
launch<__nv_bfloat16>(x, codes, scales, m, k, gs, stream);
}
extern "C" void hanzo_e2m1_encode(const float *v, uint8_t *codes, int n, cudaStream_t stream) {
e2m1_encode_kernel<<<(n + 255) / 256, 256, 0, stream>>>(v, codes, n);
}