mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
The backend in attention/backends/lsa.rs, its wiring (LsaConfig::install, LoaderBuilder::with_lsa, a TOML [lsa] section, GET/POST/DELETE /v1/lsa) and the CUDA 13.3/13.4 arms in hanzo-quant's version table, as they sat uncommitted in the shared checkout (lsa.rs sha256 ded839afdd01eb4c...), merged onto main and rustfmt'd. No commit on any ref held it before.
483 lines
20 KiB
Rust
483 lines
20 KiB
Rust
#[cfg(feature = "cuda")]
|
|
#[allow(unused)]
|
|
fn cuda_version_from_build_system() -> (usize, usize) {
|
|
let output = std::process::Command::new("nvcc")
|
|
.arg("--version")
|
|
.output()
|
|
.expect("Failed to execute `nvcc`");
|
|
|
|
if !output.status.success() {
|
|
panic!(
|
|
"`nvcc --version` failed.\nstdout:\n{}\n\nstderr:\n{}",
|
|
String::from_utf8_lossy(&output.stdout),
|
|
String::from_utf8_lossy(&output.stderr),
|
|
);
|
|
}
|
|
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
let version_line = stdout.lines().nth(3).unwrap();
|
|
let release_section = version_line.split(", ").nth(1).unwrap();
|
|
let version_number = release_section.split(' ').nth(1).unwrap();
|
|
|
|
match version_number {
|
|
"13.4" => (13, 4),
|
|
"13.3" => (13, 3),
|
|
"13.2" => (13, 2),
|
|
"13.1" => (13, 1),
|
|
"13.0" => (13, 0),
|
|
"12.9" => (12, 9),
|
|
"12.8" => (12, 8),
|
|
"12.6" => (12, 6),
|
|
"12.5" => (12, 5),
|
|
"12.4" => (12, 4),
|
|
"12.3" => (12, 3),
|
|
"12.2" => (12, 2),
|
|
"12.1" => (12, 1),
|
|
"12.0" => (12, 0),
|
|
"11.8" => (11, 8),
|
|
"11.7" => (11, 7),
|
|
"11.6" => (11, 6),
|
|
"11.5" => (11, 5),
|
|
"11.4" => (11, 4),
|
|
v => panic!("Unsupported cuda toolkit version: `{v}`. Please raise a github issue."),
|
|
}
|
|
}
|
|
|
|
/// CUTLASS commit carrying the sm_120 block-scaled and blockwise collectives
|
|
/// the NVFP4 and block-FP8 libs build on.
|
|
#[cfg(feature = "cuda")]
|
|
const CUTLASS_COMMIT: &str = "b46b16d003484063bca4ed365e44095c4c6ed633";
|
|
|
|
/// The CUTLASS sm_121a libs need Blackwell tensor cores and the CUDA 13 toolkit
|
|
/// that can target them. Measured on a GB10 with CUDA 13.0.
|
|
#[cfg(feature = "cuda")]
|
|
fn cutlass_sm121_supported(cuda_major: usize, compute_cap: usize, target: &str) -> bool {
|
|
cuda_major >= 13 && compute_cap == 121 && target.contains("linux")
|
|
}
|
|
|
|
fn main() -> Result<(), String> {
|
|
// Declare expected cfg values for check-cfg lint
|
|
println!("cargo::rustc-check-cfg=cfg(has_marlin_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_blockwise_fp8_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_scalar_fp8_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_vector_fp8_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_mxfp4_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_mxfp4_wmma_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_wmma_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_cutlass_kernels)");
|
|
println!("cargo::rustc-check-cfg=cfg(has_blockwise_fp8_cutlass_kernels)");
|
|
|
|
#[cfg(feature = "cuda")]
|
|
{
|
|
use std::{path::PathBuf, vec};
|
|
const CUDA_NVCC_FLAGS: Option<&'static str> = option_env!("CUDA_NVCC_FLAGS");
|
|
|
|
println!("cargo:rerun-if-changed=build.rs");
|
|
println!("cargo:rerun-if-env-changed=CUDA_NVCC_FLAGS");
|
|
ensure_real_nvcc();
|
|
let build_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
|
|
|
let mut builder = cudaforge::KernelBuilder::new()
|
|
.source_glob("kernels/*/*.cu")
|
|
.out_dir(build_dir.clone())
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_HALF_OPERATORS__")
|
|
.arg("-U__CUDA_NO_HALF_CONVERSIONS__")
|
|
.arg("-U__CUDA_NO_HALF2_OPERATORS__")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
.arg("--use_fast_math")
|
|
.arg("--verbose")
|
|
.arg("--compiler-options")
|
|
.arg("-fPIC");
|
|
|
|
let compute_cap = builder.get_compute_cap().unwrap_or(80);
|
|
// ======== Handle optional kernel compilation via rustc-cfg flags
|
|
let cc_over_80 = compute_cap >= 80;
|
|
|
|
if cc_over_80 {
|
|
println!("cargo:rustc-cfg=has_marlin_kernels");
|
|
println!("cargo:rustc-cfg=has_blockwise_fp8_kernels");
|
|
println!("cargo:rustc-cfg=has_scalar_fp8_kernels");
|
|
println!("cargo:rustc-cfg=has_vector_fp8_kernels");
|
|
// WMMA tensor core MXFP4 kernel (FP16/BF16 WMMA requires SM >= 80)
|
|
println!("cargo:rustc-cfg=has_mxfp4_wmma_kernels");
|
|
println!("cargo:rustc-cfg=has_nvfp4_wmma_kernels");
|
|
}
|
|
// MXFP4 and NVFP4 are always enabled with CUDA (LUT-based dequantization in registers)
|
|
println!("cargo:rustc-cfg=has_mxfp4_kernels");
|
|
println!("cargo:rustc-cfg=has_nvfp4_kernels");
|
|
|
|
let mut excluded_files = if cc_over_80 {
|
|
vec!["dummy_*.cu", "*_dummy.cu"]
|
|
} else {
|
|
vec!["marlin_*.cu", "*_fp8.cu", "*_fp8_gemm.cu", "*_wmma.cu"]
|
|
};
|
|
// The block-scaled path needs CUTLASS headers and sm_121a, so it has its own builder.
|
|
excluded_files.push("nvfp4_cutlass.cu");
|
|
excluded_files.push("nvfp4_quantize.cu");
|
|
// The activation quantizers must not take --use_fast_math: they have their own builder.
|
|
excluded_files.push("quantize/*");
|
|
excluded_files.push("blockwise_fp8_cutlass/*");
|
|
// Routed experts build with CUTLASS for sm_121a, without fast math: their own builder.
|
|
excluded_files.push("moe/*");
|
|
builder = builder.exclude(&excluded_files);
|
|
|
|
// https://github.com/hanzoai/engine/issues/286
|
|
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
|
|
builder = builder.arg("--compiler-options");
|
|
builder = builder.arg(cuda_nvcc_flags_env);
|
|
}
|
|
|
|
let target = std::env::var("TARGET").unwrap();
|
|
let (cuda_major, cuda_minor) = cuda_version_from_build_system();
|
|
let build_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
|
// https://github.com/hanzoai/engine/issues/588
|
|
let out_file = if target.contains("msvc") {
|
|
// Windows case
|
|
build_dir.join("hanzoquant.lib")
|
|
} else {
|
|
build_dir.join("libhanzoquant.a")
|
|
};
|
|
builder
|
|
.build_lib(out_file)
|
|
.expect("Build mistral quant lib failed!");
|
|
println!("cargo:rustc-link-search={}", build_dir.display());
|
|
println!("cargo:rustc-link-lib=hanzoquant");
|
|
|
|
// Activation quantizers: bit-exact with the served kernels, so no --use_fast_math (every
|
|
// division and reciprocal would become an approximation the server does not use) and no
|
|
// contraction. The approximate instructions the server does use are inline PTX.
|
|
let mut quantize_builder = cudaforge::KernelBuilder::new()
|
|
.source_glob("kernels/quantize/*.cu")
|
|
.watch(["kernels/quantize"])
|
|
.out_dir(build_dir.clone())
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_HALF_OPERATORS__")
|
|
.arg("-U__CUDA_NO_HALF_CONVERSIONS__")
|
|
.arg("-U__CUDA_NO_HALF2_OPERATORS__")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
.arg("--fmad=false")
|
|
.arg("--compiler-options")
|
|
.arg("-fPIC");
|
|
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
|
|
quantize_builder = quantize_builder
|
|
.arg("--compiler-options")
|
|
.arg(cuda_nvcc_flags_env);
|
|
}
|
|
let quantize_lib = if target.contains("msvc") {
|
|
build_dir.join("hanzoquantize.lib")
|
|
} else {
|
|
build_dir.join("libhanzoquantize.a")
|
|
};
|
|
quantize_builder
|
|
.build_lib(quantize_lib)
|
|
.expect("Build hanzo activation quantizers failed!");
|
|
println!("cargo:rustc-link-lib=hanzoquantize");
|
|
println!("cargo:rustc-link-lib=dylib=cudart");
|
|
|
|
if target.contains("msvc") {
|
|
// nothing to link to
|
|
} else if target.contains("apple")
|
|
|| target.contains("freebsd")
|
|
|| target.contains("openbsd")
|
|
{
|
|
println!("cargo:rustc-link-lib=dylib=c++");
|
|
} else if target.contains("android") {
|
|
println!("cargo:rustc-link-lib=dylib=c++_shared");
|
|
} else {
|
|
println!("cargo:rustc-link-lib=dylib=stdc++");
|
|
}
|
|
|
|
println!("cargo:rustc-cfg=feature=\"cuda-{cuda_major}0{cuda_minor}0\"");
|
|
|
|
let cutlass_sm121 = cutlass_sm121_supported(cuda_major, compute_cap, &target);
|
|
if cutlass_sm121 {
|
|
let mut nvfp4_builder = cudaforge::KernelBuilder::new()
|
|
.source_files([
|
|
"kernels/nvfp4_cutlass/nvfp4_cutlass.cu",
|
|
"kernels/nvfp4_cutlass/nvfp4_quantize.cu",
|
|
])
|
|
.source_glob("kernels/moe/*.cu")
|
|
.watch(["kernels/nvfp4_cutlass", "kernels/quantize"])
|
|
.watch(["kernels/moe"])
|
|
.out_dir(build_dir.clone())
|
|
// Block-scaled MMA is a family-specific feature: plain sm_121 will not do.
|
|
.compute_cap_arch("121a")
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
// The epilogue folds the two global scales with explicit round-to-nearest
|
|
// multiplies, which a contraction would silently change.
|
|
.arg("--fmad=false")
|
|
.arg("--compiler-options")
|
|
.arg("-fPIC")
|
|
.with_cutlass(Some(CUTLASS_COMMIT));
|
|
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
|
|
nvfp4_builder = nvfp4_builder
|
|
.arg("--compiler-options")
|
|
.arg(cuda_nvcc_flags_env);
|
|
}
|
|
nvfp4_builder
|
|
.build_lib(build_dir.join("libhanzonvfp4.a"))
|
|
.expect("Build hanzo NVFP4 block-scaled lib failed!");
|
|
println!("cargo:rustc-link-lib=hanzonvfp4");
|
|
println!("cargo:rustc-cfg=has_nvfp4_cutlass_kernels");
|
|
}
|
|
|
|
if cutlass_sm121 {
|
|
// The three sm_12x blockwise FP8 tiles (swap-AB, pingpong, cooperative), one per
|
|
// translation unit. Default contraction (--fmad=true, no fast-math): the mainloop's
|
|
// per-128-K promotion is a single FFMA, and splitting it rounds twice.
|
|
let mut blockwise_builder = cudaforge::KernelBuilder::new()
|
|
.source_files([
|
|
"kernels/blockwise_fp8_cutlass/blockwise_fp8_cutlass.cu",
|
|
"kernels/blockwise_fp8_cutlass/swap.cu",
|
|
"kernels/blockwise_fp8_cutlass/pingpong.cu",
|
|
"kernels/blockwise_fp8_cutlass/cooperative.cu",
|
|
])
|
|
.watch(["kernels/blockwise_fp8_cutlass"])
|
|
.out_dir(build_dir.clone())
|
|
.compute_cap_arch("121a")
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
.arg("--compiler-options")
|
|
.arg("-fPIC")
|
|
.with_cutlass(Some(CUTLASS_COMMIT));
|
|
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
|
|
blockwise_builder = blockwise_builder
|
|
.arg("--compiler-options")
|
|
.arg(cuda_nvcc_flags_env);
|
|
}
|
|
// cudaforge compiles on rayon's global pool, which the main builder already
|
|
// sized; a local one-thread pool keeps these multi-GB CUTLASS units serial.
|
|
rayon::ThreadPoolBuilder::new()
|
|
.num_threads(1)
|
|
.build()
|
|
.map_err(|e| e.to_string())?
|
|
.install(|| blockwise_builder.build_lib(build_dir.join("libhanzoblockwisefp8.a")))
|
|
.expect("Build hanzo block-FP8 CUTLASS lib failed!");
|
|
println!("cargo:rustc-link-lib=hanzoblockwisefp8");
|
|
println!("cargo:rustc-cfg=has_blockwise_fp8_cutlass_kernels");
|
|
}
|
|
|
|
// cuTile needs CUDA >= 13.1: its JIT toolchain (`tileiras`) ships with 13.1+, not 13.0, so a
|
|
// 13.0 build compiles but fails to JIT at runtime.
|
|
let cuda_ge_131 = cuda_major > 13 || (cuda_major == 13 && cuda_minor >= 1);
|
|
if std::env::var("CARGO_FEATURE_CUTILE").is_ok() {
|
|
if !cuda_ge_131 {
|
|
panic!(
|
|
"the `cutile` feature requires CUDA >= 13.1 to build (found {cuda_major}.{cuda_minor}); \
|
|
build without `--features cutile`"
|
|
);
|
|
}
|
|
} else if cuda_ge_131 {
|
|
println!(
|
|
"cargo:warning=CUDA {cuda_major}.{cuda_minor} detected: enable the `cutile` feature for \
|
|
optimized kernels."
|
|
);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(feature = "metal")]
|
|
{
|
|
use std::path::PathBuf;
|
|
use std::process::Command;
|
|
use std::{env, str};
|
|
|
|
const METAL_SOURCES: [&str; 21] = [
|
|
"bitwise",
|
|
"blockwise_fp8",
|
|
"bnb_dequantize",
|
|
"f8q8",
|
|
"flash_attn",
|
|
"fused_glu",
|
|
"hqq_dequantize",
|
|
"hqq_bitpack",
|
|
"moe",
|
|
"mxfp4",
|
|
"quantized",
|
|
"rotary",
|
|
"rmsnorm_residual",
|
|
"scalar_fp8",
|
|
"scan",
|
|
"sdpa_with_sinks",
|
|
"softcap",
|
|
"softmax_with_sinks",
|
|
"sort",
|
|
"copy",
|
|
"topk_logits",
|
|
];
|
|
const HEADER_SOURCES: [&str; 5] = ["utils", "bf16", "scan_impl", "sort_impl", "copy_impl"];
|
|
// Include-only headers (not compiled directly, just tracked for changes)
|
|
const INCLUDE_ONLY: [&str; 2] = ["float8", "float4"];
|
|
for src in METAL_SOURCES {
|
|
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
|
|
}
|
|
for src in HEADER_SOURCES {
|
|
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
|
|
}
|
|
for src in INCLUDE_ONLY {
|
|
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
|
|
}
|
|
println!("cargo::rerun-if-changed=build.rs");
|
|
|
|
// Check if precompilation should be skipped
|
|
// https://github.com/hanzoai/engine/pull/1311#issuecomment-3001309885
|
|
println!("cargo:rerun-if-env-changed=METAL_PRECOMPILE");
|
|
let skip_precompile = env::var("METAL_PRECOMPILE")
|
|
.map(|v| v == "0" || v.to_lowercase() == "false")
|
|
.unwrap_or(false);
|
|
|
|
if skip_precompile {
|
|
println!("cargo:warning=Skipping Metal kernel precompilation (METAL_PRECOMPILE=0)");
|
|
// Write a dummy metallib file to satisfy the include_bytes! macro
|
|
let out_dir = PathBuf::from(std::env::var("OUT_DIR").map_err(|_| "OUT_DIR not set")?);
|
|
std::fs::write(out_dir.join("hanzo_quant.metallib"), []).unwrap();
|
|
std::fs::write(out_dir.join("hanzo_quant_ios.metallib"), []).unwrap();
|
|
std::fs::write(out_dir.join("hanzo_quant_tvos.metallib"), []).unwrap();
|
|
return Ok(());
|
|
}
|
|
|
|
enum Platform {
|
|
MacOS,
|
|
Ios,
|
|
TvOS,
|
|
}
|
|
|
|
impl Platform {
|
|
fn sdk(&self) -> &str {
|
|
match self {
|
|
Platform::MacOS => "macosx",
|
|
Platform::Ios => "iphoneos",
|
|
Platform::TvOS => "appletvos",
|
|
}
|
|
}
|
|
|
|
fn metal_std(&self) -> &str {
|
|
// Use Metal 3.1 unified standard for all platforms.
|
|
// This fixes Xcode 26+ where the default Metal standard may be too low.
|
|
// https://github.com/hanzoai/engine/issues/1844
|
|
//
|
|
// Note: tvOS devices with A15+ (Apple TV 4K 3rd gen) support Metal 3.1.
|
|
match self {
|
|
Platform::MacOS | Platform::Ios | Platform::TvOS => "metal3.1",
|
|
}
|
|
}
|
|
}
|
|
|
|
fn compile(platform: Platform) -> Result<(), String> {
|
|
let current_dir = env::current_dir().expect("Failed to get current directory");
|
|
let out_dir = PathBuf::from(std::env::var("OUT_DIR").map_err(|_| "OUT_DIR not set")?);
|
|
let working_directory = out_dir.to_string_lossy().to_string();
|
|
let sources = current_dir.join("src").join("metal_kernels");
|
|
|
|
// Compile metal to air
|
|
let mut compile_air_cmd = Command::new("xcrun");
|
|
compile_air_cmd
|
|
.arg("--sdk")
|
|
.arg(platform.sdk())
|
|
.arg("metal")
|
|
.arg(format!("-std={}", platform.metal_std()))
|
|
.arg(format!("-working-directory={working_directory}"))
|
|
.arg("-Wall")
|
|
.arg("-Wextra")
|
|
.arg("-O3")
|
|
.arg("-c")
|
|
.arg("-w");
|
|
for metal_file in METAL_SOURCES {
|
|
compile_air_cmd.arg(sources.join(format!("{metal_file}.metal")));
|
|
}
|
|
for metal_file in HEADER_SOURCES {
|
|
compile_air_cmd.arg(sources.join(format!("{metal_file}.metal")));
|
|
}
|
|
// Run the Metal compiler once and fail the build on a nonzero exit.
|
|
let status = compile_air_cmd
|
|
.status()
|
|
.expect("Failed to invoke metal compiler (metal -> air)");
|
|
if !status.success() {
|
|
panic!("Compiling metal -> air failed. Exit with status: {status}")
|
|
}
|
|
|
|
// Compile air to metallib
|
|
let lib_name = match platform {
|
|
Platform::MacOS => "hanzo_quant.metallib",
|
|
Platform::Ios => "hanzo_quant_ios.metallib",
|
|
Platform::TvOS => "hanzo_quant_tvos.metallib",
|
|
};
|
|
let metallib = out_dir.join(lib_name);
|
|
let mut compile_metallib_cmd = Command::new("xcrun");
|
|
// Keep the link step on the same SDK/std as the AIR compile step.
|
|
compile_metallib_cmd
|
|
.arg("--sdk")
|
|
.arg(platform.sdk())
|
|
.arg("metal")
|
|
.arg(format!("-std={}", platform.metal_std()))
|
|
.arg("-o")
|
|
.arg(&metallib);
|
|
|
|
for metal_file in METAL_SOURCES {
|
|
compile_metallib_cmd.arg(out_dir.join(format!("{metal_file}.air")));
|
|
}
|
|
for metal_file in HEADER_SOURCES {
|
|
compile_metallib_cmd.arg(out_dir.join(format!("{metal_file}.air")));
|
|
}
|
|
|
|
let status = compile_metallib_cmd
|
|
.status()
|
|
.expect("Failed to invoke metal linker (air -> metallib)");
|
|
if !status.success() {
|
|
panic!("Compiling air -> metallib failed. Exit with status: {status}")
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
compile(Platform::MacOS)?;
|
|
compile(Platform::Ios)?;
|
|
compile(Platform::TvOS)?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(not(any(feature = "metal", feature = "cuda")))]
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
fn ensure_real_nvcc() {
|
|
use std::path::PathBuf;
|
|
if std::env::var_os("NVCC").is_some() {
|
|
return;
|
|
}
|
|
let from_root = ["CUDA_HOME", "CUDA_PATH", "CUDA_ROOT"]
|
|
.iter()
|
|
.filter_map(|k| std::env::var_os(k))
|
|
.map(|r| PathBuf::from(r).join("bin").join("nvcc"))
|
|
.find(|p| p.exists());
|
|
let from_path = std::env::var_os("PATH").and_then(|paths| {
|
|
std::env::split_paths(&paths)
|
|
.map(|d| d.join("nvcc"))
|
|
.find(|p| p.exists())
|
|
});
|
|
if let Some(nvcc) = from_root.or(from_path) {
|
|
if let Ok(real) = std::fs::canonicalize(&nvcc) {
|
|
println!("cargo:warning=hanzo: pinning NVCC to {}", real.display());
|
|
std::env::set_var("NVCC", real);
|
|
}
|
|
}
|
|
}
|