Files
zeekay 10edbb538b attention: latent sparse attention (LSA) backend
The backend in attention/backends/lsa.rs, its wiring (LsaConfig::install,
LoaderBuilder::with_lsa, a TOML [lsa] section, GET/POST/DELETE /v1/lsa)
and the CUDA 13.3/13.4 arms in hanzo-quant's version table, as they sat
uncommitted in the shared checkout (lsa.rs sha256 ded839afdd01eb4c...),
merged onto main and rustfmt'd. No commit on any ref held it before.
2026-11-12 15:57:50 -08:00

483 lines
20 KiB
Rust

#[cfg(feature = "cuda")]
#[allow(unused)]
fn cuda_version_from_build_system() -> (usize, usize) {
let output = std::process::Command::new("nvcc")
.arg("--version")
.output()
.expect("Failed to execute `nvcc`");
if !output.status.success() {
panic!(
"`nvcc --version` failed.\nstdout:\n{}\n\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
}
let stdout = String::from_utf8_lossy(&output.stdout);
let version_line = stdout.lines().nth(3).unwrap();
let release_section = version_line.split(", ").nth(1).unwrap();
let version_number = release_section.split(' ').nth(1).unwrap();
match version_number {
"13.4" => (13, 4),
"13.3" => (13, 3),
"13.2" => (13, 2),
"13.1" => (13, 1),
"13.0" => (13, 0),
"12.9" => (12, 9),
"12.8" => (12, 8),
"12.6" => (12, 6),
"12.5" => (12, 5),
"12.4" => (12, 4),
"12.3" => (12, 3),
"12.2" => (12, 2),
"12.1" => (12, 1),
"12.0" => (12, 0),
"11.8" => (11, 8),
"11.7" => (11, 7),
"11.6" => (11, 6),
"11.5" => (11, 5),
"11.4" => (11, 4),
v => panic!("Unsupported cuda toolkit version: `{v}`. Please raise a github issue."),
}
}
/// CUTLASS commit carrying the sm_120 block-scaled and blockwise collectives
/// the NVFP4 and block-FP8 libs build on.
#[cfg(feature = "cuda")]
const CUTLASS_COMMIT: &str = "b46b16d003484063bca4ed365e44095c4c6ed633";
/// The CUTLASS sm_121a libs need Blackwell tensor cores and the CUDA 13 toolkit
/// that can target them. Measured on a GB10 with CUDA 13.0.
#[cfg(feature = "cuda")]
fn cutlass_sm121_supported(cuda_major: usize, compute_cap: usize, target: &str) -> bool {
cuda_major >= 13 && compute_cap == 121 && target.contains("linux")
}
fn main() -> Result<(), String> {
// Declare expected cfg values for check-cfg lint
println!("cargo::rustc-check-cfg=cfg(has_marlin_kernels)");
println!("cargo::rustc-check-cfg=cfg(has_blockwise_fp8_kernels)");
println!("cargo::rustc-check-cfg=cfg(has_scalar_fp8_kernels)");
println!("cargo::rustc-check-cfg=cfg(has_vector_fp8_kernels)");
println!("cargo::rustc-check-cfg=cfg(has_mxfp4_kernels)");
println!("cargo::rustc-check-cfg=cfg(has_mxfp4_wmma_kernels)");
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_kernels)");
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_wmma_kernels)");
println!("cargo::rustc-check-cfg=cfg(has_nvfp4_cutlass_kernels)");
println!("cargo::rustc-check-cfg=cfg(has_blockwise_fp8_cutlass_kernels)");
#[cfg(feature = "cuda")]
{
use std::{path::PathBuf, vec};
const CUDA_NVCC_FLAGS: Option<&'static str> = option_env!("CUDA_NVCC_FLAGS");
println!("cargo:rerun-if-changed=build.rs");
println!("cargo:rerun-if-env-changed=CUDA_NVCC_FLAGS");
ensure_real_nvcc();
let build_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
let mut builder = cudaforge::KernelBuilder::new()
.source_glob("kernels/*/*.cu")
.out_dir(build_dir.clone())
.arg("-std=c++17")
.arg("-O3")
.arg("-U__CUDA_NO_HALF_OPERATORS__")
.arg("-U__CUDA_NO_HALF_CONVERSIONS__")
.arg("-U__CUDA_NO_HALF2_OPERATORS__")
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
.arg("--expt-relaxed-constexpr")
.arg("--expt-extended-lambda")
.arg("--use_fast_math")
.arg("--verbose")
.arg("--compiler-options")
.arg("-fPIC");
let compute_cap = builder.get_compute_cap().unwrap_or(80);
// ======== Handle optional kernel compilation via rustc-cfg flags
let cc_over_80 = compute_cap >= 80;
if cc_over_80 {
println!("cargo:rustc-cfg=has_marlin_kernels");
println!("cargo:rustc-cfg=has_blockwise_fp8_kernels");
println!("cargo:rustc-cfg=has_scalar_fp8_kernels");
println!("cargo:rustc-cfg=has_vector_fp8_kernels");
// WMMA tensor core MXFP4 kernel (FP16/BF16 WMMA requires SM >= 80)
println!("cargo:rustc-cfg=has_mxfp4_wmma_kernels");
println!("cargo:rustc-cfg=has_nvfp4_wmma_kernels");
}
// MXFP4 and NVFP4 are always enabled with CUDA (LUT-based dequantization in registers)
println!("cargo:rustc-cfg=has_mxfp4_kernels");
println!("cargo:rustc-cfg=has_nvfp4_kernels");
let mut excluded_files = if cc_over_80 {
vec!["dummy_*.cu", "*_dummy.cu"]
} else {
vec!["marlin_*.cu", "*_fp8.cu", "*_fp8_gemm.cu", "*_wmma.cu"]
};
// The block-scaled path needs CUTLASS headers and sm_121a, so it has its own builder.
excluded_files.push("nvfp4_cutlass.cu");
excluded_files.push("nvfp4_quantize.cu");
// The activation quantizers must not take --use_fast_math: they have their own builder.
excluded_files.push("quantize/*");
excluded_files.push("blockwise_fp8_cutlass/*");
// Routed experts build with CUTLASS for sm_121a, without fast math: their own builder.
excluded_files.push("moe/*");
builder = builder.exclude(&excluded_files);
// https://github.com/hanzoai/engine/issues/286
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
builder = builder.arg("--compiler-options");
builder = builder.arg(cuda_nvcc_flags_env);
}
let target = std::env::var("TARGET").unwrap();
let (cuda_major, cuda_minor) = cuda_version_from_build_system();
let build_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
// https://github.com/hanzoai/engine/issues/588
let out_file = if target.contains("msvc") {
// Windows case
build_dir.join("hanzoquant.lib")
} else {
build_dir.join("libhanzoquant.a")
};
builder
.build_lib(out_file)
.expect("Build mistral quant lib failed!");
println!("cargo:rustc-link-search={}", build_dir.display());
println!("cargo:rustc-link-lib=hanzoquant");
// Activation quantizers: bit-exact with the served kernels, so no --use_fast_math (every
// division and reciprocal would become an approximation the server does not use) and no
// contraction. The approximate instructions the server does use are inline PTX.
let mut quantize_builder = cudaforge::KernelBuilder::new()
.source_glob("kernels/quantize/*.cu")
.watch(["kernels/quantize"])
.out_dir(build_dir.clone())
.arg("-std=c++17")
.arg("-O3")
.arg("-U__CUDA_NO_HALF_OPERATORS__")
.arg("-U__CUDA_NO_HALF_CONVERSIONS__")
.arg("-U__CUDA_NO_HALF2_OPERATORS__")
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
.arg("--expt-relaxed-constexpr")
.arg("--expt-extended-lambda")
.arg("--fmad=false")
.arg("--compiler-options")
.arg("-fPIC");
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
quantize_builder = quantize_builder
.arg("--compiler-options")
.arg(cuda_nvcc_flags_env);
}
let quantize_lib = if target.contains("msvc") {
build_dir.join("hanzoquantize.lib")
} else {
build_dir.join("libhanzoquantize.a")
};
quantize_builder
.build_lib(quantize_lib)
.expect("Build hanzo activation quantizers failed!");
println!("cargo:rustc-link-lib=hanzoquantize");
println!("cargo:rustc-link-lib=dylib=cudart");
if target.contains("msvc") {
// nothing to link to
} else if target.contains("apple")
|| target.contains("freebsd")
|| target.contains("openbsd")
{
println!("cargo:rustc-link-lib=dylib=c++");
} else if target.contains("android") {
println!("cargo:rustc-link-lib=dylib=c++_shared");
} else {
println!("cargo:rustc-link-lib=dylib=stdc++");
}
println!("cargo:rustc-cfg=feature=\"cuda-{cuda_major}0{cuda_minor}0\"");
let cutlass_sm121 = cutlass_sm121_supported(cuda_major, compute_cap, &target);
if cutlass_sm121 {
let mut nvfp4_builder = cudaforge::KernelBuilder::new()
.source_files([
"kernels/nvfp4_cutlass/nvfp4_cutlass.cu",
"kernels/nvfp4_cutlass/nvfp4_quantize.cu",
])
.source_glob("kernels/moe/*.cu")
.watch(["kernels/nvfp4_cutlass", "kernels/quantize"])
.watch(["kernels/moe"])
.out_dir(build_dir.clone())
// Block-scaled MMA is a family-specific feature: plain sm_121 will not do.
.compute_cap_arch("121a")
.arg("-std=c++17")
.arg("-O3")
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
.arg("--expt-relaxed-constexpr")
.arg("--expt-extended-lambda")
// The epilogue folds the two global scales with explicit round-to-nearest
// multiplies, which a contraction would silently change.
.arg("--fmad=false")
.arg("--compiler-options")
.arg("-fPIC")
.with_cutlass(Some(CUTLASS_COMMIT));
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
nvfp4_builder = nvfp4_builder
.arg("--compiler-options")
.arg(cuda_nvcc_flags_env);
}
nvfp4_builder
.build_lib(build_dir.join("libhanzonvfp4.a"))
.expect("Build hanzo NVFP4 block-scaled lib failed!");
println!("cargo:rustc-link-lib=hanzonvfp4");
println!("cargo:rustc-cfg=has_nvfp4_cutlass_kernels");
}
if cutlass_sm121 {
// The three sm_12x blockwise FP8 tiles (swap-AB, pingpong, cooperative), one per
// translation unit. Default contraction (--fmad=true, no fast-math): the mainloop's
// per-128-K promotion is a single FFMA, and splitting it rounds twice.
let mut blockwise_builder = cudaforge::KernelBuilder::new()
.source_files([
"kernels/blockwise_fp8_cutlass/blockwise_fp8_cutlass.cu",
"kernels/blockwise_fp8_cutlass/swap.cu",
"kernels/blockwise_fp8_cutlass/pingpong.cu",
"kernels/blockwise_fp8_cutlass/cooperative.cu",
])
.watch(["kernels/blockwise_fp8_cutlass"])
.out_dir(build_dir.clone())
.compute_cap_arch("121a")
.arg("-std=c++17")
.arg("-O3")
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
.arg("--expt-relaxed-constexpr")
.arg("--expt-extended-lambda")
.arg("--compiler-options")
.arg("-fPIC")
.with_cutlass(Some(CUTLASS_COMMIT));
if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
blockwise_builder = blockwise_builder
.arg("--compiler-options")
.arg(cuda_nvcc_flags_env);
}
// cudaforge compiles on rayon's global pool, which the main builder already
// sized; a local one-thread pool keeps these multi-GB CUTLASS units serial.
rayon::ThreadPoolBuilder::new()
.num_threads(1)
.build()
.map_err(|e| e.to_string())?
.install(|| blockwise_builder.build_lib(build_dir.join("libhanzoblockwisefp8.a")))
.expect("Build hanzo block-FP8 CUTLASS lib failed!");
println!("cargo:rustc-link-lib=hanzoblockwisefp8");
println!("cargo:rustc-cfg=has_blockwise_fp8_cutlass_kernels");
}
// cuTile needs CUDA >= 13.1: its JIT toolchain (`tileiras`) ships with 13.1+, not 13.0, so a
// 13.0 build compiles but fails to JIT at runtime.
let cuda_ge_131 = cuda_major > 13 || (cuda_major == 13 && cuda_minor >= 1);
if std::env::var("CARGO_FEATURE_CUTILE").is_ok() {
if !cuda_ge_131 {
panic!(
"the `cutile` feature requires CUDA >= 13.1 to build (found {cuda_major}.{cuda_minor}); \
build without `--features cutile`"
);
}
} else if cuda_ge_131 {
println!(
"cargo:warning=CUDA {cuda_major}.{cuda_minor} detected: enable the `cutile` feature for \
optimized kernels."
);
}
Ok(())
}
#[cfg(feature = "metal")]
{
use std::path::PathBuf;
use std::process::Command;
use std::{env, str};
const METAL_SOURCES: [&str; 21] = [
"bitwise",
"blockwise_fp8",
"bnb_dequantize",
"f8q8",
"flash_attn",
"fused_glu",
"hqq_dequantize",
"hqq_bitpack",
"moe",
"mxfp4",
"quantized",
"rotary",
"rmsnorm_residual",
"scalar_fp8",
"scan",
"sdpa_with_sinks",
"softcap",
"softmax_with_sinks",
"sort",
"copy",
"topk_logits",
];
const HEADER_SOURCES: [&str; 5] = ["utils", "bf16", "scan_impl", "sort_impl", "copy_impl"];
// Include-only headers (not compiled directly, just tracked for changes)
const INCLUDE_ONLY: [&str; 2] = ["float8", "float4"];
for src in METAL_SOURCES {
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
}
for src in HEADER_SOURCES {
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
}
for src in INCLUDE_ONLY {
println!("cargo::rerun-if-changed=src/metal_kernels/{src}.metal");
}
println!("cargo::rerun-if-changed=build.rs");
// Check if precompilation should be skipped
// https://github.com/hanzoai/engine/pull/1311#issuecomment-3001309885
println!("cargo:rerun-if-env-changed=METAL_PRECOMPILE");
let skip_precompile = env::var("METAL_PRECOMPILE")
.map(|v| v == "0" || v.to_lowercase() == "false")
.unwrap_or(false);
if skip_precompile {
println!("cargo:warning=Skipping Metal kernel precompilation (METAL_PRECOMPILE=0)");
// Write a dummy metallib file to satisfy the include_bytes! macro
let out_dir = PathBuf::from(std::env::var("OUT_DIR").map_err(|_| "OUT_DIR not set")?);
std::fs::write(out_dir.join("hanzo_quant.metallib"), []).unwrap();
std::fs::write(out_dir.join("hanzo_quant_ios.metallib"), []).unwrap();
std::fs::write(out_dir.join("hanzo_quant_tvos.metallib"), []).unwrap();
return Ok(());
}
enum Platform {
MacOS,
Ios,
TvOS,
}
impl Platform {
fn sdk(&self) -> &str {
match self {
Platform::MacOS => "macosx",
Platform::Ios => "iphoneos",
Platform::TvOS => "appletvos",
}
}
fn metal_std(&self) -> &str {
// Use Metal 3.1 unified standard for all platforms.
// This fixes Xcode 26+ where the default Metal standard may be too low.
// https://github.com/hanzoai/engine/issues/1844
//
// Note: tvOS devices with A15+ (Apple TV 4K 3rd gen) support Metal 3.1.
match self {
Platform::MacOS | Platform::Ios | Platform::TvOS => "metal3.1",
}
}
}
fn compile(platform: Platform) -> Result<(), String> {
let current_dir = env::current_dir().expect("Failed to get current directory");
let out_dir = PathBuf::from(std::env::var("OUT_DIR").map_err(|_| "OUT_DIR not set")?);
let working_directory = out_dir.to_string_lossy().to_string();
let sources = current_dir.join("src").join("metal_kernels");
// Compile metal to air
let mut compile_air_cmd = Command::new("xcrun");
compile_air_cmd
.arg("--sdk")
.arg(platform.sdk())
.arg("metal")
.arg(format!("-std={}", platform.metal_std()))
.arg(format!("-working-directory={working_directory}"))
.arg("-Wall")
.arg("-Wextra")
.arg("-O3")
.arg("-c")
.arg("-w");
for metal_file in METAL_SOURCES {
compile_air_cmd.arg(sources.join(format!("{metal_file}.metal")));
}
for metal_file in HEADER_SOURCES {
compile_air_cmd.arg(sources.join(format!("{metal_file}.metal")));
}
// Run the Metal compiler once and fail the build on a nonzero exit.
let status = compile_air_cmd
.status()
.expect("Failed to invoke metal compiler (metal -> air)");
if !status.success() {
panic!("Compiling metal -> air failed. Exit with status: {status}")
}
// Compile air to metallib
let lib_name = match platform {
Platform::MacOS => "hanzo_quant.metallib",
Platform::Ios => "hanzo_quant_ios.metallib",
Platform::TvOS => "hanzo_quant_tvos.metallib",
};
let metallib = out_dir.join(lib_name);
let mut compile_metallib_cmd = Command::new("xcrun");
// Keep the link step on the same SDK/std as the AIR compile step.
compile_metallib_cmd
.arg("--sdk")
.arg(platform.sdk())
.arg("metal")
.arg(format!("-std={}", platform.metal_std()))
.arg("-o")
.arg(&metallib);
for metal_file in METAL_SOURCES {
compile_metallib_cmd.arg(out_dir.join(format!("{metal_file}.air")));
}
for metal_file in HEADER_SOURCES {
compile_metallib_cmd.arg(out_dir.join(format!("{metal_file}.air")));
}
let status = compile_metallib_cmd
.status()
.expect("Failed to invoke metal linker (air -> metallib)");
if !status.success() {
panic!("Compiling air -> metallib failed. Exit with status: {status}")
}
Ok(())
}
compile(Platform::MacOS)?;
compile(Platform::Ios)?;
compile(Platform::TvOS)?;
Ok(())
}
#[cfg(not(any(feature = "metal", feature = "cuda")))]
Ok(())
}
#[cfg(feature = "cuda")]
fn ensure_real_nvcc() {
use std::path::PathBuf;
if std::env::var_os("NVCC").is_some() {
return;
}
let from_root = ["CUDA_HOME", "CUDA_PATH", "CUDA_ROOT"]
.iter()
.filter_map(|k| std::env::var_os(k))
.map(|r| PathBuf::from(r).join("bin").join("nvcc"))
.find(|p| p.exists());
let from_path = std::env::var_os("PATH").and_then(|paths| {
std::env::split_paths(&paths)
.map(|d| d.join("nvcc"))
.find(|p| p.exists())
});
if let Some(nvcc) = from_root.or(from_path) {
if let Ok(real) = std::fs::canonicalize(&nvcc) {
println!("cargo:warning=hanzo: pinning NVCC to {}", real.display());
std::env::set_var("NVCC", real);
}
}
}