Files
zeekayandHanzo Dev 6a363f7ee3 merge: upstream main ddf1b879 and the aarch64 repack branch 35d7ae7c
Three-way per file through the rename: base 39355c6c91, theirs the upstream
merge of main with 35d7ae7c (via 7ef6a694d), ours origin/main.

Lands the barrier thread pool, runtime CPU feature dispatch, aarch64 and x86
repacked quantized kernels (indexed_gemv, gemv_fused_shared_lhs), quantized
embedding on CPU/Metal/CUDA, paged flash-attn kernels, cutile fused MoE, the
in-place op aliasing API, parking_lot storage locks, Metal buffer builders and
debug labels, safetensors 0.8, hf-hub 1.0, tokenizers 0.23.

Ours keeps ROCm, Vulkan, wgpu, the IQ/MXFP4/NVFP4/TQ dtypes, and the i-quant
weight/activation block split in matmul. Our backends ride InplaceOpN and the
quantized embedding; Metal get_rows gains the IQ kernels. ug leaves, as upstream.

Co-authored-by: Hanzo Dev <[email protected]>
2026-09-15 15:52:28 -07:00

176 lines
6.7 KiB
Rust

// Build script to run nvcc and generate the C glue code for launching the flash-attention kernel.
// The cuda build time is very long so one can set the FLASH_ATTN_BUILD_DIR environment
// variable in order to cache the compiled artifacts and avoid recompiling too often.
use anyhow::{Context, Result};
use hanzo_flash_attn_build::{cutlass_include_arg, fetch_cutlass};
use std::{fs, path::PathBuf};
const CUTLASS_COMMIT: &str = "7d49e6c7e2f8896c47f586706e67e1fb215529dc";
const KERNEL_FILES: [&str; 53] = [
"kernels/flash_api.cu",
"kernels/flash_fwd_splitkv_hdim512_fp16_sm80.cu",
"kernels/flash_fwd_splitkv_hdim512_bf16_sm80.cu",
"kernels/flash_fwd_splitkv_hdim512_fp16_causal_sm80.cu",
"kernels/flash_fwd_splitkv_hdim512_bf16_causal_sm80.cu",
"kernels/flash_fwd_splitkv_hdim64_fp16_sm80.cu",
"kernels/flash_fwd_splitkv_hdim128_fp16_sm80.cu",
"kernels/flash_fwd_splitkv_hdim256_fp16_sm80.cu",
"kernels/flash_fwd_splitkv_hdim64_bf16_sm80.cu",
"kernels/flash_fwd_splitkv_hdim128_bf16_sm80.cu",
"kernels/flash_fwd_splitkv_hdim256_bf16_sm80.cu",
"kernels/flash_fwd_splitkv_hdim64_fp16_causal_sm80.cu",
"kernels/flash_fwd_splitkv_hdim128_fp16_causal_sm80.cu",
"kernels/flash_fwd_splitkv_hdim256_fp16_causal_sm80.cu",
"kernels/flash_fwd_splitkv_hdim64_bf16_causal_sm80.cu",
"kernels/flash_fwd_splitkv_hdim128_bf16_causal_sm80.cu",
"kernels/flash_fwd_splitkv_hdim256_bf16_causal_sm80.cu",
"kernels/flash_fwd_hdim128_fp16_sm80.cu",
"kernels/flash_fwd_hdim160_fp16_sm80.cu",
"kernels/flash_fwd_hdim192_fp16_sm80.cu",
"kernels/flash_fwd_hdim224_fp16_sm80.cu",
"kernels/flash_fwd_hdim256_fp16_sm80.cu",
"kernels/flash_fwd_hdim512_fp16_sm80.cu",
"kernels/flash_fwd_hdim32_fp16_sm80.cu",
"kernels/flash_fwd_hdim64_fp16_sm80.cu",
"kernels/flash_fwd_hdim96_fp16_sm80.cu",
"kernels/flash_fwd_hdim128_bf16_sm80.cu",
"kernels/flash_fwd_hdim160_bf16_sm80.cu",
"kernels/flash_fwd_hdim192_bf16_sm80.cu",
"kernels/flash_fwd_hdim224_bf16_sm80.cu",
"kernels/flash_fwd_hdim256_bf16_sm80.cu",
"kernels/flash_fwd_hdim512_bf16_sm80.cu",
"kernels/flash_fwd_hdim32_bf16_sm80.cu",
"kernels/flash_fwd_hdim64_bf16_sm80.cu",
"kernels/flash_fwd_hdim96_bf16_sm80.cu",
"kernels/flash_fwd_hdim128_fp16_causal_sm80.cu",
"kernels/flash_fwd_hdim160_fp16_causal_sm80.cu",
"kernels/flash_fwd_hdim192_fp16_causal_sm80.cu",
"kernels/flash_fwd_hdim224_fp16_causal_sm80.cu",
"kernels/flash_fwd_hdim256_fp16_causal_sm80.cu",
"kernels/flash_fwd_hdim512_fp16_causal_sm80.cu",
"kernels/flash_fwd_hdim32_fp16_causal_sm80.cu",
"kernels/flash_fwd_hdim64_fp16_causal_sm80.cu",
"kernels/flash_fwd_hdim96_fp16_causal_sm80.cu",
"kernels/flash_fwd_hdim128_bf16_causal_sm80.cu",
"kernels/flash_fwd_hdim160_bf16_causal_sm80.cu",
"kernels/flash_fwd_hdim192_bf16_causal_sm80.cu",
"kernels/flash_fwd_hdim224_bf16_causal_sm80.cu",
"kernels/flash_fwd_hdim256_bf16_causal_sm80.cu",
"kernels/flash_fwd_hdim512_bf16_causal_sm80.cu",
"kernels/flash_fwd_hdim32_bf16_causal_sm80.cu",
"kernels/flash_fwd_hdim64_bf16_causal_sm80.cu",
"kernels/flash_fwd_hdim96_bf16_causal_sm80.cu",
];
const HEADER_FILES: [&str; 18] = [
"kernels/alibi.h",
"kernels/block_info.h",
"kernels/dropout.h",
"kernels/error.h",
"kernels/flash.h",
"kernels/flash_fwd_kernel.h",
"kernels/flash_fwd_launch_template.h",
"kernels/hardware_info.h",
"kernels/kernel_helpers.h",
"kernels/kernel_traits.h",
"kernels/kernel_traits_sm90.h",
"kernels/kernels.h",
"kernels/mask.h",
"kernels/philox.cuh",
"kernels/rotary.h",
"kernels/softmax.h",
"kernels/static_switch.h",
"kernels/utils.h",
];
fn update_hash(hash: &mut u64, bytes: &[u8]) {
const FNV_PRIME: u64 = 1099511628211;
for &byte in bytes {
*hash ^= u64::from(byte);
*hash = hash.wrapping_mul(FNV_PRIME);
}
}
fn header_hash() -> Result<u64> {
let mut hash = 14695981039346656037;
for file in HEADER_FILES {
update_hash(&mut hash, file.as_bytes());
update_hash(&mut hash, &fs::read(file)?);
}
Ok(hash)
}
fn main() -> Result<()> {
println!("cargo::rerun-if-changed=build.rs");
for kernel_file in KERNEL_FILES.iter() {
println!("cargo::rerun-if-changed={kernel_file}");
}
for header_file in HEADER_FILES.iter() {
println!("cargo::rerun-if-changed={header_file}");
}
let out_dir = PathBuf::from(std::env::var("OUT_DIR").context("OUT_DIR not set")?);
let build_dir = match std::env::var("FLASH_ATTN_BUILD_DIR") {
Err(_) =>
{
#[allow(clippy::redundant_clone)]
out_dir.clone()
}
Ok(build_dir) => {
let path = PathBuf::from(build_dir);
path.canonicalize().expect(&format!(
"Directory doesn't exists: {} (the current directory is {})",
&path.display(),
std::env::current_dir()?.display()
))
}
};
// Fetch cutlass headers on-demand
let cutlass_dir = fetch_cutlass(&out_dir, CUTLASS_COMMIT)?;
let cutlass_include: &'static str =
Box::leak(cutlass_include_arg(&cutlass_dir).into_boxed_str());
// Kernel objects cached in FLASH_ATTN_BUILD_DIR must rebuild when a shared header changes.
let header_hash_arg: &'static str =
Box::leak(format!("-DFLASH_ATTN_HEADER_HASH=0x{:016x}", header_hash()?).into_boxed_str());
let kernels = KERNEL_FILES.iter().collect();
let mut builder = bindgen_cuda::Builder::default()
.kernel_paths(kernels)
.out_dir(build_dir.clone())
.arg("-std=c++17")
.arg("-O3")
.arg("-U__CUDA_NO_HALF_OPERATORS__")
.arg("-U__CUDA_NO_HALF_CONVERSIONS__")
.arg("-U__CUDA_NO_HALF2_OPERATORS__")
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
.arg(&cutlass_include)
.arg("--expt-relaxed-constexpr")
.arg("--expt-extended-lambda")
.arg("--use_fast_math")
.arg("--verbose")
.arg(header_hash_arg);
let mut is_target_msvc = false;
if let Ok(target) = std::env::var("TARGET") {
if target.contains("msvc") {
is_target_msvc = true;
builder = builder.arg("-D_USE_MATH_DEFINES");
}
}
if !is_target_msvc {
builder = builder.arg("-Xcompiler").arg("-fPIC");
}
let out_file = build_dir.join("libflashattention.a");
builder.build_lib(out_file);
println!("cargo::rustc-link-search={}", build_dir.display());
println!("cargo::rustc-link-lib=flashattention");
println!("cargo::rustc-link-lib=dylib=cudart");
if !is_target_msvc {
println!("cargo::rustc-link-lib=dylib=stdc++");
}
Ok(())
}