mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Every mention of another inference engine is rewritten to say what our code does, or the math it computes. Reference values are "the reference value from the published config" or "the recorded reference output", line citations into another engine's source become the rule itself, comparisons with other engines are dropped, and asides about other engines' bugs go. Identifiers follow: - CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels' `hanzo_rotary`; the shuffle/ldg macros are HANZO_*. - GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side. - Tests: `*_matches_vllm` and friends name what they compare against (the reference fixture, the tile rule). - MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op is `rotary-embedding`. - hanzo-bench's board scores ours against a "rival" whose name comes from the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s` columns; dossier.sh and spec_decode.sh take `--rival-bench`. - install.ps1 installs this repository's hanzo-cli, as install.sh does, with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL. - bench_http.py reads only the hanzo: metric families. Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and tools/upstream-map.sh; each ported file's copyright/licence/provenance header; Cargo.toml dependency names; data identifiers (Hugging Face repo IDs, fixture file names and fixture metadata keys the tests read, on-disk paths, executable names the GPU and engine guards match on a host); model and format names. LLM.md says the rule once, under "One name: Hanzo Engine". Co-authored-by: Hanzo Dev <[email protected]>
99 lines
2.9 KiB
Plaintext
99 lines
2.9 KiB
Plaintext
/**
|
|
* NVFP4 building blocks shared by every activation quantizer: the served approximate
|
|
* instructions, the E2M1 encoder and decoder, and the block-scale formula.
|
|
*
|
|
* The served quantizers (scaled_fp4_quant, and FlashInfer's cvt_warp_fp16_to_fp4 from its
|
|
* quantization_utils.cuh, built with fast math on) compute, per block of 16 with global scale gs:
|
|
*
|
|
* SF = gs * (vecMax * rcp.approx.ftz(6))
|
|
* sf8 = e4m3 RN satfinite of SF
|
|
* out = vecMax != 0 ? rcp.approx.ftz(f32(sf8) * rcp.approx.ftz(gs)) : 0
|
|
* code = e2m1 RN satfinite of x * out
|
|
*
|
|
* The approximate reciprocal is inline PTX so no compiler flag can change it; files that include
|
|
* this header are built without --use_fast_math and with --fmad=false.
|
|
*
|
|
* Plain sm_121 has no cvt.rn.satfinite.e2m1x2.f32, so the encoder is software: round to nearest,
|
|
* ties to the even code, saturate at 6, and keep the sign of a zero result, as the instruction
|
|
* does.
|
|
*/
|
|
#pragma once
|
|
|
|
#include <cmath>
|
|
#include <cstdint>
|
|
#include <cuda_fp8.h>
|
|
|
|
namespace e2m1 {
|
|
|
|
__device__ __forceinline__ float rcp_approx_ftz(float a) {
|
|
float b;
|
|
asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(b) : "f"(a));
|
|
return b;
|
|
}
|
|
|
|
__device__ __forceinline__ float div_full(float a, float b) {
|
|
float c;
|
|
asm("div.full.f32 %0, %1, %2;" : "=f"(c) : "f"(a), "f"(b));
|
|
return c;
|
|
}
|
|
|
|
/// E2M1 magnitudes by code: 0, 0.5, 1, 1.5, 2, 3, 4, 6.
|
|
__host__ __device__ __forceinline__ float decode(uint8_t code) {
|
|
const float mag[8] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f};
|
|
const float m = mag[code & 7];
|
|
return (code & 8) ? -m : m;
|
|
}
|
|
|
|
/// Round to nearest E2M1, ties to the even code, saturating; NaN saturates. Returns the 4-bit
|
|
/// code with the input's sign bit, so -0 and negatives rounding to zero encode as 0b1000.
|
|
__host__ __device__ __forceinline__ uint8_t encode(float v) {
|
|
#ifdef __CUDA_ARCH__
|
|
const uint32_t bits = __float_as_uint(v);
|
|
#else
|
|
uint32_t bits;
|
|
__builtin_memcpy(&bits, &v, 4);
|
|
#endif
|
|
const uint8_t sign = (bits >> 31) ? 8 : 0;
|
|
const float a = fabsf(v);
|
|
uint8_t code;
|
|
if (a <= 0.25f)
|
|
code = 0;
|
|
else if (a < 0.75f)
|
|
code = 1;
|
|
else if (a <= 1.25f)
|
|
code = 2;
|
|
else if (a < 1.75f)
|
|
code = 3;
|
|
else if (a <= 2.5f)
|
|
code = 4;
|
|
else if (a < 3.5f)
|
|
code = 5;
|
|
else if (a <= 5.0f)
|
|
code = 6;
|
|
else
|
|
code = 7; // also +inf and NaN
|
|
return code | sign;
|
|
}
|
|
|
|
__device__ __forceinline__ float e4m3_to_float(uint8_t v) {
|
|
__nv_fp8_e4m3 f;
|
|
f.__x = v;
|
|
return static_cast<float>(f);
|
|
}
|
|
|
|
/// The block's E4M3 scale byte and the multiplier its values are encoded with.
|
|
struct Block {
|
|
uint8_t scale;
|
|
float out;
|
|
};
|
|
|
|
__device__ __forceinline__ Block block(float vec_max, float gs) {
|
|
const float sf = gs * (vec_max * rcp_approx_ftz(6.0f));
|
|
const uint8_t sf8 = __nv_cvt_float_to_fp8(sf, __NV_SATFINITE, __NV_E4M3);
|
|
const float out =
|
|
vec_max != 0.0f ? rcp_approx_ftz(e4m3_to_float(sf8) * rcp_approx_ftz(gs)) : 0.0f;
|
|
return {sf8, out};
|
|
}
|
|
|
|
} // namespace e2m1
|