mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-21 02:36:58 +00:00
481 lines
20 KiB
Rust
481 lines
20 KiB
Rust
#[cfg(feature = "cuda")]
|
|
#[allow(unused)]
|
|
fn cuda_version_from_build_system() -> (usize, usize) {
|
|
let output = std::process::Command::new("nvcc")
|
|
.arg("--version")
|
|
.output()
|
|
.expect("Failed to execute `nvcc`");
|
|
|
|
if !output.status.success() {
|
|
panic!(
|
|
"`nvcc --version` failed.\nstdout:\n{}\n\nstderr:\n{}",
|
|
String::from_utf8_lossy(&output.stdout),
|
|
String::from_utf8_lossy(&output.stderr),
|
|
);
|
|
}
|
|
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
let version_line = stdout.lines().nth(3).unwrap();
|
|
let release_section = version_line.split(", ").nth(1).unwrap();
|
|
let version_number = release_section.split(' ').nth(1).unwrap();
|
|
|
|
match version_number {
|
|
"13.2" => (13, 2),
|
|
"13.1" => (13, 1),
|
|
"13.0" => (13, 0),
|
|
"12.9" => (12, 9),
|
|
"12.8" => (12, 8),
|
|
"12.6" => (12, 6),
|
|
"12.5" => (12, 5),
|
|
"12.4" => (12, 4),
|
|
"12.3" => (12, 3),
|
|
"12.2" => (12, 2),
|
|
"12.1" => (12, 1),
|
|
"12.0" => (12, 0),
|
|
"11.8" => (11, 8),
|
|
"11.7" => (11, 7),
|
|
"11.6" => (11, 6),
|
|
"11.5" => (11, 5),
|
|
"11.4" => (11, 4),
|
|
v => panic!("Unsupported cuda toolkit version: `{v}`. Please raise a github issue."),
|
|
}
|
|
}
|
|
|
|
/// CUTLASS commit carrying the sm_120 block-scaled and blockwise collectives
|
|
/// the NVFP4 and block-FP8 libs build on.
|
|
#[cfg(feature = "cuda")]
|
|
const CUTLASS_COMMIT: &str = "b46b16d003484063bca4ed365e44095c4c6ed633";
|
|
|
|
/// The CUTLASS sm_121a libs need Blackwell tensor cores and the CUDA 13 toolkit
|
|
/// that can target them. Measured on a GB10 with CUDA 13.0.
|
|
#[cfg(feature = "cuda")]
|
|
fn cutlass_sm121_supported(cuda_major: usize, compute_cap: usize, target: &str) -> bool {
|
|
cuda_major >= 13 && compute_cap == 121 && target.contains("linux")
|
|
}
|
|
|
|
fn main() -> Result<(), String> {
|
|
// Declare expected cfg values for check-cfg lint
|
|
println!("cargo::rustc-check-cfg=cfg(has_marlin_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_blockwise_fp8_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_scalar_fp8_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_vector_fp8_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_mxfp4_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_mxfp4_wmma_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_wmma_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_cutlass_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_blockwise_fp8_cutlass_kernels)");
|
|
|
|
#[cfg(feature = "cuda")]
|
|
{
|
|
use std::{path::PathBuf, vec};
|
|
const CUDA_NVCC_FLAGS: Option<&'static str> = option_env!("CUDA_NVCC_FLAGS");
|
|
|
|
println!("cargo:rerun-if-changed=build.rs");
|
|
println!("cargo:rerun-if-env-changed=CUDA_NVCC_FLAGS");
|
|
ensure_real_nvcc();
|
|
let build_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
|
|
|
let mut builder = cudaforge::KernelBuilder::new()
|
|
.source_glob("kernels/*/*.cu")
|
|
.out_dir(build_dir.clone())
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_HALF_OPERATORS__")
|
|
.arg("-U__CUDA_NO_HALF_CONVERSIONS__")
|
|
.arg("-U__CUDA_NO_HALF2_OPERATORS__")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
.arg("--use_fast_math")
|
|
.arg("--verbose")
|
|
.arg("--compiler-options")
|
|
.arg("-fPIC");
|
|
|
|
let compute_cap = builder.get_compute_cap().unwrap_or(80);
|
|
// ======== Handle optional kernel compilation via rustc-cfg flags
|
|
let cc_over_80 = compute_cap >= 80;
|
|
|
|
if cc_over_80 {
|
|
println!("cargo:rustc-cfg=has_marlin_kernels");
|
|
println!("cargo:rustc-cfg=has_blockwise_fp8_kernels");
|
|
println!("cargo:rustc-cfg=has_scalar_fp8_kernels");
|
|
println!("cargo:rustc-cfg=has_vector_fp8_kernels");
|
|
// WMMA tensor core MXFP4 kernel (FP16/BF16 WMMA requires SM >= 80)
|
|
println!("cargo:rustc-cfg=has_mxfp4_wmma_kernels");
|
|
println!("cargo:rustc-cfg=has_nvfp4_wmma_kernels");
|
|
}
|
|
// MXFP4 and NVFP4 are always enabled with CUDA (LUT-based dequantization in registers)
|
|
println!("cargo:rustc-cfg=has_mxfp4_kernels");
|
|
println!("cargo:rustc-cfg=has_nvfp4_kernels");
|
|
|
|
let mut excluded_files = if cc_over_80 {
|
|
vec!["dummy_*.cu", "*_dummy.cu"]
|
|
} else {
|
|
vec!["marlin_*.cu", "*_fp8.cu", "*_fp8_gemm.cu", "*_wmma.cu"]
|
|
};
|
|
// The block-scaled path needs CUTLASS headers and sm_121a, so it has its own builder.
|
|
excluded_files.push("nvfp4_cutlass.cu");
|
|
excluded_files.push("nvfp4_quantize.cu");
|
|
// The activation quantizers must not take --use_fast_math: they have their own builder.
|
|
excluded_files.push("quantize/*");
|
|
excluded_files.push("blockwise_fp8_cutlass/*");
|
|
// Routed experts build with CUTLASS for sm_121a, without fast math: their own builder.
|
|
excluded_files.push("moe/*");
|
|
builder = builder.exclude(&excluded_files);
|
|
|
|
// https://github.com/hanzoai/engine/issues/286
|
|
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
|
|
builder = builder.arg("--compiler-options");
|
|
builder = builder.arg(cuda_nvcc_flags_env);
|
|
}
|
|
|
|
let target = std::env::var("TARGET").unwrap();
|
|
let (cuda_major, cuda_minor) = cuda_version_from_build_system();
|
|
let build_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
|
// https://github.com/hanzoai/engine/issues/588
|
|
let out_file = if target.contains("msvc") {
|
|
// Windows case
|
|
build_dir.join("hanzoquant.lib")
|
|
} else {
|
|
build_dir.join("libhanzoquant.a")
|
|
};
|
|
builder
|
|
.build_lib(out_file)
|
|
.expect("Build mistral quant lib failed!");
|
|
println!("cargo:rustc-link-search={}", build_dir.display());
|
|
println!("cargo:rustc-link-lib=hanzoquant");
|
|
|
|
// Activation quantizers: bit-exact with the served kernels, so no --use_fast_math (every
|
|
// division and reciprocal would become an approximation the server does not use) and no
|
|
// contraction. The approximate instructions the server does use are inline PTX.
|
|
let mut quantize_builder = cudaforge::KernelBuilder::new()
|
|
.source_glob("kernels/quantize/*.cu")
|
|
.watch(["kernels/quantize"])
|
|
.out_dir(build_dir.clone())
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_HALF_OPERATORS__")
|
|
.arg("-U__CUDA_NO_HALF_CONVERSIONS__")
|
|
.arg("-U__CUDA_NO_HALF2_OPERATORS__")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
.arg("--fmad=false")
|
|
.arg("--compiler-options")
|
|
.arg("-fPIC");
|
|
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
|
|
quantize_builder = quantize_builder
|
|
.arg("--compiler-options")
|
|
.arg(cuda_nvcc_flags_env);
|
|
}
|
|
let quantize_lib = if target.contains("msvc") {
|
|
build_dir.join("hanzoquantize.lib")
|
|
} else {
|
|
build_dir.join("libhanzoquantize.a")
|
|
};
|
|
quantize_builder
|
|
.build_lib(quantize_lib)
|
|
.expect("Build hanzo activation quantizers failed!");
|
|
println!("cargo:rustc-link-lib=hanzoquantize");
|
|
println!("cargo:rustc-link-lib=dylib=cudart");
|
|
|
|
if target.contains("msvc") {
|
|
// nothing to link to
|
|
} else if target.contains("apple")
|
|
|| target.contains("freebsd")
|
|
|| target.contains("openbsd")
|
|
{
|
|
println!("cargo:rustc-link-lib=dylib=c++");
|
|
} else if target.contains("android") {
|
|
println!("cargo:rustc-link-lib=dylib=c++_shared");
|
|
} else {
|
|
println!("cargo:rustc-link-lib=dylib=stdc++");
|
|
}
|
|
|
|
println!("cargo:rustc-cfg=feature=\"cuda-{cuda_major}0{cuda_minor}0\"");
|
|
|
|
let cutlass_sm121 = cutlass_sm121_supported(cuda_major, compute_cap, &target);
|
|
if cutlass_sm121 {
|
|
let mut nvfp4_builder = cudaforge::KernelBuilder::new()
|
|
.source_files([
|
|
"kernels/nvfp4_cutlass/nvfp4_cutlass.cu",
|
|
"kernels/nvfp4_cutlass/nvfp4_quantize.cu",
|
|
])
|
|
.source_glob("kernels/moe/*.cu")
|
|
.watch(["kernels/nvfp4_cutlass", "kernels/quantize"])
|
|
.watch(["kernels/moe"])
|
|
.out_dir(build_dir.clone())
|
|
// Block-scaled MMA is a family-specific feature: plain sm_121 will not do.
|
|
.compute_cap_arch("121a")
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
// The epilogue folds the two global scales with explicit round-to-nearest
|
|
// multiplies, which a contraction would silently change.
|
|
.arg("--fmad=false")
|
|
.arg("--compiler-options")
|
|
.arg("-fPIC")
|
|
.with_cutlass(Some(CUTLASS_COMMIT));
|
|
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
|
|
nvfp4_builder = nvfp4_builder
|
|
.arg("--compiler-options")
|
|
.arg(cuda_nvcc_flags_env);
|
|
}
|
|
nvfp4_builder
|
|
.build_lib(build_dir.join("libhanzonvfp4.a"))
|
|
.expect("Build hanzo NVFP4 block-scaled lib failed!");
|
|
println!("cargo:rustc-link-lib=hanzonvfp4");
|
|
println!("cargo:rustc-cfg=has_nvfp4_cutlass_kernels");
|
|
}
|
|
|
|
if cutlass_sm121 {
|
|
// vLLM v0.29.0's three sm_12x blockwise FP8 tiles, one per translation unit.
|
|
// Default contraction (--fmad=true, no fast-math) as vLLM compiles them: the
|
|
// mainloop's per-128-K promotion is a single FFMA, and splitting it rounds twice.
|
|
let mut blockwise_builder = cudaforge::KernelBuilder::new()
|
|
.source_files([
|
|
"kernels/blockwise_fp8_cutlass/blockwise_fp8_cutlass.cu",
|
|
"kernels/blockwise_fp8_cutlass/swap.cu",
|
|
"kernels/blockwise_fp8_cutlass/pingpong.cu",
|
|
"kernels/blockwise_fp8_cutlass/cooperative.cu",
|
|
])
|
|
.watch(["kernels/blockwise_fp8_cutlass"])
|
|
.out_dir(build_dir.clone())
|
|
.compute_cap_arch("121a")
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
.arg("--compiler-options")
|
|
.arg("-fPIC")
|
|
.with_cutlass(Some(CUTLASS_COMMIT));
|
|
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
|
|
blockwise_builder = blockwise_builder
|
|
.arg("--compiler-options")
|
|
.arg(cuda_nvcc_flags_env);
|
|
}
|
|
// cudaforge compiles on rayon's global pool, which the main builder already
|
|
// sized; a local one-thread pool keeps these multi-GB CUTLASS units serial.
|
|
rayon::ThreadPoolBuilder::new()
|
|
.num_threads(1)
|
|
.build()
|
|
.map_err(|e| e.to_string())?
|
|
.install(|| blockwise_builder.build_lib(build_dir.join("libhanzoblockwisefp8.a")))
|
|
.expect("Build hanzo block-FP8 CUTLASS lib failed!");
|
|
println!("cargo:rustc-link-lib=hanzoblockwisefp8");
|
|
println!("cargo:rustc-cfg=has_blockwise_fp8_cutlass_kernels");
|
|
}
|
|
|
|
// cuTile needs CUDA >= 13.1: its JIT toolchain (`tileiras`) ships with 13.1+, not 13.0, so a
|
|
// 13.0 build compiles but fails to JIT at runtime.
|
|
let cuda_ge_131 = cuda_major > 13 || (cuda_major == 13 && cuda_minor >= 1);
|
|
if std::env::var("CARGO_FEATURE_CUTILE").is_ok() {
|
|
if !cuda_ge_131 {
|
|
panic!(
|
|
"the `cutile` feature requires CUDA >= 13.1 to build (found {cuda_major}.{cuda_minor}); \
|
|
build without `--features cutile`"
|
|
);
|
|
}
|
|
} else if cuda_ge_131 {
|
|
println!(
|
|
"cargo:warning=CUDA {cuda_major}.{cuda_minor} detected: enable the `cutile` feature for \
|
|
optimized kernels."
|
|
);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(feature = "metal")]
|
|
{
|
|
use std::path::PathBuf;
|
|
use std::process::Command;
|
|
use std::{env, str};
|
|
|
|
const METAL_SOURCES: [&str; 21] = [
|
|
"bitwise",
|
|
"blockwise_fp8",
|
|
"bnb_dequantize",
|
|
"f8q8",
|
|
"flash_attn",
|
|
"fused_glu",
|
|
"hqq_dequantize",
|
|
"hqq_bitpack",
|
|
"moe",
|
|
"mxfp4",
|
|
"quantized",
|
|
"rotary",
|
|
"rmsnorm_residual",
|
|
"scalar_fp8",
|
|
"scan",
|
|
"sdpa_with_sinks",
|
|
"softcap",
|
|
"softmax_with_sinks",
|
|
"sort",
|
|
"copy",
|
|
"topk_logits",
|
|
];
|
|
const HEADER_SOURCES: [&str; 5] = ["utils", "bf16", "scan_impl", "sort_impl", "copy_impl"];
|
|
// Include-only headers (not compiled directly, just tracked for changes)
|
|
const INCLUDE_ONLY: [&str; 2] = ["float8", "float4"];
|
|
for src in METAL_SOURCES {
|
|
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
|
|
}
|
|
for src in HEADER_SOURCES {
|
|
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
|
|
}
|
|
for src in INCLUDE_ONLY {
|
|
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
|
|
}
|
|
println!("cargo::rerun-if-changed=build.rs");
|
|
|
|
// Check if precompilation should be skipped
|
|
// https://github.com/hanzoai/engine/pull/1311#issuecomment-3001309885
|
|
println!("cargo:rerun-if-env-changed=METAL_PRECOMPILE");
|
|
let skip_precompile = env::var("METAL_PRECOMPILE")
|
|
.map(|v| v == "0" || v.to_lowercase() == "false")
|
|
.unwrap_or(false);
|
|
|
|
if skip_precompile {
|
|
println!("cargo:warning=Skipping Metal kernel precompilation (METAL_PRECOMPILE=0)");
|
|
// Write a dummy metallib file to satisfy the include_bytes! macro
|
|
let out_dir = PathBuf::from(std::env::var("OUT_DIR").map_err(|_| "OUT_DIR not set")?);
|
|
std::fs::write(out_dir.join("hanzo_quant.metallib"), []).unwrap();
|
|
std::fs::write(out_dir.join("hanzo_quant_ios.metallib"), []).unwrap();
|
|
std::fs::write(out_dir.join("hanzo_quant_tvos.metallib"), []).unwrap();
|
|
return Ok(());
|
|
}
|
|
|
|
enum Platform {
|
|
MacOS,
|
|
Ios,
|
|
TvOS,
|
|
}
|
|
|
|
impl Platform {
|
|
fn sdk(&self) -> &str {
|
|
match self {
|
|
Platform::MacOS => "macosx",
|
|
Platform::Ios => "iphoneos",
|
|
Platform::TvOS => "appletvos",
|
|
}
|
|
}
|
|
|
|
fn metal_std(&self) -> &str {
|
|
// Use Metal 3.1 unified standard for all platforms.
|
|
// This fixes Xcode 26+ where the default Metal standard may be too low.
|
|
// https://github.com/hanzoai/engine/issues/1844
|
|
//
|
|
// Note: tvOS devices with A15+ (Apple TV 4K 3rd gen) support Metal 3.1.
|
|
match self {
|
|
Platform::MacOS | Platform::Ios | Platform::TvOS => "metal3.1",
|
|
}
|
|
}
|
|
}
|
|
|
|
fn compile(platform: Platform) -> Result<(), String> {
|
|
let current_dir = env::current_dir().expect("Failed to get current directory");
|
|
let out_dir = PathBuf::from(std::env::var("OUT_DIR").map_err(|_| "OUT_DIR not set")?);
|
|
let working_directory = out_dir.to_string_lossy().to_string();
|
|
let sources = current_dir.join("src").join("metal_kernels");
|
|
|
|
// Compile metal to air
|
|
let mut compile_air_cmd = Command::new("xcrun");
|
|
compile_air_cmd
|
|
.arg("--sdk")
|
|
.arg(platform.sdk())
|
|
.arg("metal")
|
|
.arg(format!("-std={}", platform.metal_std()))
|
|
.arg(format!("-working-directory={working_directory}"))
|
|
.arg("-Wall")
|
|
.arg("-Wextra")
|
|
.arg("-O3")
|
|
.arg("-c")
|
|
.arg("-w");
|
|
for metal_file in METAL_SOURCES {
|
|
compile_air_cmd.arg(sources.join(format!("{metal_file}.metal")));
|
|
}
|
|
for metal_file in HEADER_SOURCES {
|
|
compile_air_cmd.arg(sources.join(format!("{metal_file}.metal")));
|
|
}
|
|
// Run the Metal compiler once and fail the build on a nonzero exit.
|
|
let status = compile_air_cmd
|
|
.status()
|
|
.expect("Failed to invoke metal compiler (metal -> air)");
|
|
if !status.success() {
|
|
panic!("Compiling metal -> air failed. Exit with status: {status}")
|
|
}
|
|
|
|
// Compile air to metallib
|
|
let lib_name = match platform {
|
|
Platform::MacOS => "hanzo_quant.metallib",
|
|
Platform::Ios => "hanzo_quant_ios.metallib",
|
|
Platform::TvOS => "hanzo_quant_tvos.metallib",
|
|
};
|
|
let metallib = out_dir.join(lib_name);
|
|
let mut compile_metallib_cmd = Command::new("xcrun");
|
|
// Keep the link step on the same SDK/std as the AIR compile step.
|
|
compile_metallib_cmd
|
|
.arg("--sdk")
|
|
.arg(platform.sdk())
|
|
.arg("metal")
|
|
.arg(format!("-std={}", platform.metal_std()))
|
|
.arg("-o")
|
|
.arg(&metallib);
|
|
|
|
for metal_file in METAL_SOURCES {
|
|
compile_metallib_cmd.arg(out_dir.join(format!("{metal_file}.air")));
|
|
}
|
|
for metal_file in HEADER_SOURCES {
|
|
compile_metallib_cmd.arg(out_dir.join(format!("{metal_file}.air")));
|
|
}
|
|
|
|
let status = compile_metallib_cmd
|
|
.status()
|
|
.expect("Failed to invoke metal linker (air -> metallib)");
|
|
if !status.success() {
|
|
panic!("Compiling air -> metallib failed. Exit with status: {status}")
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
compile(Platform::MacOS)?;
|
|
compile(Platform::Ios)?;
|
|
compile(Platform::TvOS)?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(not(any(feature = "metal", feature = "cuda")))]
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
fn ensure_real_nvcc() {
|
|
use std::path::PathBuf;
|
|
if std::env::var_os("NVCC").is_some() {
|
|
return;
|
|
}
|
|
let from_root = ["CUDA_HOME", "CUDA_PATH", "CUDA_ROOT"]
|
|
.iter()
|
|
.filter_map(|k| std::env::var_os(k))
|
|
.map(|r| PathBuf::from(r).join("bin").join("nvcc"))
|
|
.find(|p| p.exists());
|
|
let from_path = std::env::var_os("PATH").and_then(|paths| {
|
|
std::env::split_paths(&paths)
|
|
.map(|d| d.join("nvcc"))
|
|
.find(|p| p.exists())
|
|
});
|
|
if let Some(nvcc) = from_root.or(from_path) {
|
|
if let Ok(real) = std::fs::canonicalize(&nvcc) {
|
|
println!("cargo:warning=hanzo: pinning NVCC to {}", real.display());
|
|
std::env::set_var("NVCC", real);
|
|
}
|
|
}
|
|
}
|