mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
The WMMA kernel dequantizes weights to bf16 and runs the MMA there, which
caps it at the bf16 tensor-core rate. A GB10 measures 123.5 TFLOP/s on a
register-resident bf16 WMMA chain and this kernel reached 18.5 of it; the
block-scaled FP4 MMA reaches 357 on the same shapes, so the ceiling is the
dtype, not the tiling.
Both operands go in as E2M1: packed weights feed the MMA untouched, and
activations are quantized against the `input_scale` the checkpoint
calibrated for exactly this. A layer takes the path only when the device
carries the MMA, the shape meets its alignment and that scale is present;
any other layer keeps the dequantizing kernel. Decode stays on the matvec
path.
The GEMM is upstream's block-scaled FP4 kernel (0a29442f9b). Upstream
reaches it only through cuTile, which needs CUDA 13.2, and pins its build
to 13.3. The kernel compiles and runs correctly on 13.0, so activations
are quantized by a native CUDA kernel here and the build accepts the
version measured to work.
In-engine on a GB10, Qwen3.8-27B-NVFP4, prefill T/s:
prompt tokens before after
178 297 665
1,291 358 954
12,449 310 657
Decode is unchanged at 7.01 / 6.95 / 6.60.
Co-authored-by: Hanzo Dev <[email protected]>
399 lines
16 KiB
Rust
399 lines
16 KiB
Rust
#[cfg(feature = "cuda")]
|
|
#[allow(unused)]
|
|
fn cuda_version_from_build_system() -> (usize, usize) {
|
|
let output = std::process::Command::new("nvcc")
|
|
.arg("--version")
|
|
.output()
|
|
.expect("Failed to execute `nvcc`");
|
|
|
|
if !output.status.success() {
|
|
panic!(
|
|
"`nvcc --version` failed.\nstdout:\n{}\n\nstderr:\n{}",
|
|
String::from_utf8_lossy(&output.stdout),
|
|
String::from_utf8_lossy(&output.stderr),
|
|
);
|
|
}
|
|
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
let version_line = stdout.lines().nth(3).unwrap();
|
|
let release_section = version_line.split(", ").nth(1).unwrap();
|
|
let version_number = release_section.split(' ').nth(1).unwrap();
|
|
|
|
match version_number {
|
|
"13.2" => (13, 2),
|
|
"13.1" => (13, 1),
|
|
"13.0" => (13, 0),
|
|
"12.9" => (12, 9),
|
|
"12.8" => (12, 8),
|
|
"12.6" => (12, 6),
|
|
"12.5" => (12, 5),
|
|
"12.4" => (12, 4),
|
|
"12.3" => (12, 3),
|
|
"12.2" => (12, 2),
|
|
"12.1" => (12, 1),
|
|
"12.0" => (12, 0),
|
|
"11.8" => (11, 8),
|
|
"11.7" => (11, 7),
|
|
"11.6" => (11, 6),
|
|
"11.5" => (11, 5),
|
|
"11.4" => (11, 4),
|
|
v => panic!("Unsupported cuda toolkit version: `{v}`. Please raise a github issue."),
|
|
}
|
|
}
|
|
|
|
/// CUTLASS commit carrying the sm_120 block-scaled collective this kernel builds on.
|
|
#[cfg(feature = "cuda")]
|
|
const NVFP4_CUTLASS_COMMIT: &str = "b46b16d003484063bca4ed365e44095c4c6ed633";
|
|
|
|
/// Block-scaled NVFP4 needs Blackwell tensor cores and the CUDA 13 toolkit that
|
|
/// can target them. Measured on a GB10 with CUDA 13.0.
|
|
#[cfg(feature = "cuda")]
|
|
fn nvfp4_cutlass_supported(cuda_major: usize, compute_cap: usize, target: &str) -> bool {
|
|
cuda_major >= 13 && compute_cap == 121 && target.contains("linux")
|
|
}
|
|
|
|
fn main() -> Result<(), String> {
|
|
// Declare expected cfg values for check-cfg lint
|
|
println!("cargo::rustc-check-cfg=cfg(has_marlin_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_blockwise_fp8_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_scalar_fp8_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_vector_fp8_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_mxfp4_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_mxfp4_wmma_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_wmma_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_cutlass_kernels)");
|
|
|
|
#[cfg(feature = "cuda")]
|
|
{
|
|
use std::{path::PathBuf, vec};
|
|
const CUDA_NVCC_FLAGS: Option<&'static str> = option_env!("CUDA_NVCC_FLAGS");
|
|
|
|
println!("cargo:rerun-if-changed=build.rs");
|
|
println!("cargo:rerun-if-env-changed=CUDA_NVCC_FLAGS");
|
|
ensure_real_nvcc();
|
|
let build_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
|
|
|
let mut builder = cudaforge::KernelBuilder::new()
|
|
.source_glob("kernels/*/*.cu")
|
|
.out_dir(build_dir.clone())
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_HALF_OPERATORS__")
|
|
.arg("-U__CUDA_NO_HALF_CONVERSIONS__")
|
|
.arg("-U__CUDA_NO_HALF2_OPERATORS__")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
.arg("--use_fast_math")
|
|
.arg("--verbose")
|
|
.arg("--compiler-options")
|
|
.arg("-fPIC");
|
|
|
|
let compute_cap = builder.get_compute_cap().unwrap_or(80);
|
|
// ======== Handle optional kernel compilation via rustc-cfg flags
|
|
let cc_over_80 = compute_cap >= 80;
|
|
|
|
if cc_over_80 {
|
|
println!("cargo:rustc-cfg=has_marlin_kernels");
|
|
println!("cargo:rustc-cfg=has_blockwise_fp8_kernels");
|
|
println!("cargo:rustc-cfg=has_scalar_fp8_kernels");
|
|
println!("cargo:rustc-cfg=has_vector_fp8_kernels");
|
|
// WMMA tensor core MXFP4 kernel (FP16/BF16 WMMA requires SM >= 80)
|
|
println!("cargo:rustc-cfg=has_mxfp4_wmma_kernels");
|
|
println!("cargo:rustc-cfg=has_nvfp4_wmma_kernels");
|
|
}
|
|
// MXFP4 and NVFP4 are always enabled with CUDA (LUT-based dequantization in registers)
|
|
println!("cargo:rustc-cfg=has_mxfp4_kernels");
|
|
println!("cargo:rustc-cfg=has_nvfp4_kernels");
|
|
|
|
let mut excluded_files = if cc_over_80 {
|
|
vec!["dummy_*.cu", "*_dummy.cu"]
|
|
} else {
|
|
vec!["marlin_*.cu", "*_fp8.cu", "*_fp8_gemm.cu", "*_wmma.cu"]
|
|
};
|
|
// The block-scaled path needs CUTLASS headers and sm_121a, so it has its own builder.
|
|
excluded_files.push("nvfp4_cutlass.cu");
|
|
excluded_files.push("nvfp4_quantize.cu");
|
|
builder = builder.exclude(&excluded_files);
|
|
|
|
// https://github.com/hanzoai/engine/issues/286
|
|
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
|
|
builder = builder.arg("--compiler-options");
|
|
builder = builder.arg(cuda_nvcc_flags_env);
|
|
}
|
|
|
|
let target = std::env::var("TARGET").unwrap();
|
|
let (cuda_major, cuda_minor) = cuda_version_from_build_system();
|
|
let build_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
|
// https://github.com/hanzoai/engine/issues/588
|
|
let out_file = if target.contains("msvc") {
|
|
// Windows case
|
|
build_dir.join("hanzoquant.lib")
|
|
} else {
|
|
build_dir.join("libhanzoquant.a")
|
|
};
|
|
builder
|
|
.build_lib(out_file)
|
|
.expect("Build mistral quant lib failed!");
|
|
println!("cargo:rustc-link-search={}", build_dir.display());
|
|
println!("cargo:rustc-link-lib=hanzoquant");
|
|
println!("cargo:rustc-link-lib=dylib=cudart");
|
|
|
|
if target.contains("msvc") {
|
|
// nothing to link to
|
|
} else if target.contains("apple")
|
|
|| target.contains("freebsd")
|
|
|| target.contains("openbsd")
|
|
{
|
|
println!("cargo:rustc-link-lib=dylib=c++");
|
|
} else if target.contains("android") {
|
|
println!("cargo:rustc-link-lib=dylib=c++_shared");
|
|
} else {
|
|
println!("cargo:rustc-link-lib=dylib=stdc++");
|
|
}
|
|
|
|
println!("cargo:rustc-cfg=feature=\"cuda-{cuda_major}0{cuda_minor}0\"");
|
|
|
|
if nvfp4_cutlass_supported(cuda_major, compute_cap, &target) {
|
|
let mut nvfp4_builder = cudaforge::KernelBuilder::new()
|
|
.source_files([
|
|
"kernels/nvfp4_cutlass/nvfp4_cutlass.cu",
|
|
"kernels/nvfp4_cutlass/nvfp4_quantize.cu",
|
|
])
|
|
.watch(["kernels/nvfp4_cutlass"])
|
|
.out_dir(build_dir.clone())
|
|
// Block-scaled MMA is a family-specific feature: plain sm_121 will not do.
|
|
.compute_cap_arch("121a")
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
// The epilogue folds the two global scales with explicit round-to-nearest
|
|
// multiplies, which a contraction would silently change.
|
|
.arg("--fmad=false")
|
|
.arg("--compiler-options")
|
|
.arg("-fPIC")
|
|
.with_cutlass(Some(NVFP4_CUTLASS_COMMIT));
|
|
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
|
|
nvfp4_builder = nvfp4_builder
|
|
.arg("--compiler-options")
|
|
.arg(cuda_nvcc_flags_env);
|
|
}
|
|
nvfp4_builder
|
|
.build_lib(build_dir.join("libhanzonvfp4.a"))
|
|
.expect("Build hanzo NVFP4 block-scaled lib failed!");
|
|
println!("cargo:rustc-link-lib=hanzonvfp4");
|
|
println!("cargo:rustc-cfg=has_nvfp4_cutlass_kernels");
|
|
}
|
|
|
|
// cuTile needs CUDA >= 13.1: its JIT toolchain (`tileiras`) ships with 13.1+, not 13.0, so a
|
|
// 13.0 build compiles but fails to JIT at runtime.
|
|
let cuda_ge_131 = cuda_major > 13 || (cuda_major == 13 && cuda_minor >= 1);
|
|
if std::env::var("CARGO_FEATURE_CUTILE").is_ok() {
|
|
if !cuda_ge_131 {
|
|
panic!(
|
|
"the `cutile` feature requires CUDA >= 13.1 to build (found {cuda_major}.{cuda_minor}); \
|
|
build without `--features cutile`"
|
|
);
|
|
}
|
|
} else if cuda_ge_131 {
|
|
println!(
|
|
"cargo:warning=CUDA {cuda_major}.{cuda_minor} detected: enable the `cutile` feature for \
|
|
optimized kernels."
|
|
);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(feature = "metal")]
|
|
{
|
|
use std::path::PathBuf;
|
|
use std::process::Command;
|
|
use std::{env, str};
|
|
|
|
const METAL_SOURCES: [&str; 21] = [
|
|
"bitwise",
|
|
"blockwise_fp8",
|
|
"bnb_dequantize",
|
|
"f8q8",
|
|
"flash_attn",
|
|
"fused_glu",
|
|
"hqq_dequantize",
|
|
"hqq_bitpack",
|
|
"moe",
|
|
"mxfp4",
|
|
"quantized",
|
|
"rotary",
|
|
"rmsnorm_residual",
|
|
"scalar_fp8",
|
|
"scan",
|
|
"sdpa_with_sinks",
|
|
"softcap",
|
|
"softmax_with_sinks",
|
|
"sort",
|
|
"copy",
|
|
"topk_logits",
|
|
];
|
|
const HEADER_SOURCES: [&str; 5] = ["utils", "bf16", "scan_impl", "sort_impl", "copy_impl"];
|
|
// Include-only headers (not compiled directly, just tracked for changes)
|
|
const INCLUDE_ONLY: [&str; 2] = ["float8", "float4"];
|
|
for src in METAL_SOURCES {
|
|
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
|
|
}
|
|
for src in HEADER_SOURCES {
|
|
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
|
|
}
|
|
for src in INCLUDE_ONLY {
|
|
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
|
|
}
|
|
println!("cargo::rerun-if-changed=build.rs");
|
|
|
|
// Check if precompilation should be skipped
|
|
// https://github.com/hanzoai/engine/pull/1311#issuecomment-3001309885
|
|
println!("cargo:rerun-if-env-changed=METAL_PRECOMPILE");
|
|
let skip_precompile = env::var("METAL_PRECOMPILE")
|
|
.map(|v| v == "0" || v.to_lowercase() == "false")
|
|
.unwrap_or(false);
|
|
|
|
if skip_precompile {
|
|
println!("cargo:warning=Skipping Metal kernel precompilation (METAL_PRECOMPILE=0)");
|
|
// Write a dummy metallib file to satisfy the include_bytes! macro
|
|
let out_dir = PathBuf::from(std::env::var("OUT_DIR").map_err(|_| "OUT_DIR not set")?);
|
|
std::fs::write(out_dir.join("hanzo_quant.metallib"), []).unwrap();
|
|
std::fs::write(out_dir.join("hanzo_quant_ios.metallib"), []).unwrap();
|
|
std::fs::write(out_dir.join("hanzo_quant_tvos.metallib"), []).unwrap();
|
|
return Ok(());
|
|
}
|
|
|
|
enum Platform {
|
|
MacOS,
|
|
Ios,
|
|
TvOS,
|
|
}
|
|
|
|
impl Platform {
|
|
fn sdk(&self) -> &str {
|
|
match self {
|
|
Platform::MacOS => "macosx",
|
|
Platform::Ios => "iphoneos",
|
|
Platform::TvOS => "appletvos",
|
|
}
|
|
}
|
|
|
|
fn metal_std(&self) -> &str {
|
|
// Use Metal 3.1 unified standard for all platforms.
|
|
// This fixes Xcode 26+ where the default Metal standard may be too low.
|
|
// https://github.com/hanzoai/engine/issues/1844
|
|
//
|
|
// Note: tvOS devices with A15+ (Apple TV 4K 3rd gen) support Metal 3.1.
|
|
match self {
|
|
Platform::MacOS | Platform::Ios | Platform::TvOS => "metal3.1",
|
|
}
|
|
}
|
|
}
|
|
|
|
fn compile(platform: Platform) -> Result<(), String> {
|
|
let current_dir = env::current_dir().expect("Failed to get current directory");
|
|
let out_dir = PathBuf::from(std::env::var("OUT_DIR").map_err(|_| "OUT_DIR not set")?);
|
|
let working_directory = out_dir.to_string_lossy().to_string();
|
|
let sources = current_dir.join("src").join("metal_kernels");
|
|
|
|
// Compile metal to air
|
|
let mut compile_air_cmd = Command::new("xcrun");
|
|
compile_air_cmd
|
|
.arg("--sdk")
|
|
.arg(platform.sdk())
|
|
.arg("metal")
|
|
.arg(format!("-std={}", platform.metal_std()))
|
|
.arg(format!("-working-directory={working_directory}"))
|
|
.arg("-Wall")
|
|
.arg("-Wextra")
|
|
.arg("-O3")
|
|
.arg("-c")
|
|
.arg("-w");
|
|
for metal_file in METAL_SOURCES {
|
|
compile_air_cmd.arg(sources.join(format!("{metal_file}.metal")));
|
|
}
|
|
for metal_file in HEADER_SOURCES {
|
|
compile_air_cmd.arg(sources.join(format!("{metal_file}.metal")));
|
|
}
|
|
// Run the Metal compiler once and fail the build on a nonzero exit.
|
|
let status = compile_air_cmd
|
|
.status()
|
|
.expect("Failed to invoke metal compiler (metal -> air)");
|
|
if !status.success() {
|
|
panic!("Compiling metal -> air failed. Exit with status: {status}")
|
|
}
|
|
|
|
// Compile air to metallib
|
|
let lib_name = match platform {
|
|
Platform::MacOS => "hanzo_quant.metallib",
|
|
Platform::Ios => "hanzo_quant_ios.metallib",
|
|
Platform::TvOS => "hanzo_quant_tvos.metallib",
|
|
};
|
|
let metallib = out_dir.join(lib_name);
|
|
let mut compile_metallib_cmd = Command::new("xcrun");
|
|
// Keep the link step on the same SDK/std as the AIR compile step.
|
|
compile_metallib_cmd
|
|
.arg("--sdk")
|
|
.arg(platform.sdk())
|
|
.arg("metal")
|
|
.arg(format!("-std={}", platform.metal_std()))
|
|
.arg("-o")
|
|
.arg(&metallib);
|
|
|
|
for metal_file in METAL_SOURCES {
|
|
compile_metallib_cmd.arg(out_dir.join(format!("{metal_file}.air")));
|
|
}
|
|
for metal_file in HEADER_SOURCES {
|
|
compile_metallib_cmd.arg(out_dir.join(format!("{metal_file}.air")));
|
|
}
|
|
|
|
let status = compile_metallib_cmd
|
|
.status()
|
|
.expect("Failed to invoke metal linker (air -> metallib)");
|
|
if !status.success() {
|
|
panic!("Compiling air -> metallib failed. Exit with status: {status}")
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
compile(Platform::MacOS)?;
|
|
compile(Platform::Ios)?;
|
|
compile(Platform::TvOS)?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(not(any(feature = "metal", feature = "cuda")))]
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
fn ensure_real_nvcc() {
|
|
use std::path::PathBuf;
|
|
if std::env::var_os("NVCC").is_some() {
|
|
return;
|
|
}
|
|
let from_root = ["CUDA_HOME", "CUDA_PATH", "CUDA_ROOT"]
|
|
.iter()
|
|
.filter_map(|k| std::env::var_os(k))
|
|
.map(|r| PathBuf::from(r).join("bin").join("nvcc"))
|
|
.find(|p| p.exists());
|
|
let from_path = std::env::var_os("PATH").and_then(|paths| {
|
|
std::env::split_paths(&paths)
|
|
.map(|d| d.join("nvcc"))
|
|
.find(|p| p.exists())
|
|
});
|
|
if let Some(nvcc) = from_root.or(from_path) {
|
|
if let Ok(real) = std::fs::canonicalize(&nvcc) {
|
|
println!("cargo:warning=hanzo: pinning NVCC to {}", real.display());
|
|
std::env::set_var("NVCC", real);
|
|
}
|
|
}
|
|
}
|