mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-21 22:51:09 +00:00
Three-way per file through the rename: base 39355c6c91, theirs the upstream
merge of main with 35d7ae7c (via 7ef6a694d), ours origin/main.
Lands the barrier thread pool, runtime CPU feature dispatch, aarch64 and x86
repacked quantized kernels (indexed_gemv, gemv_fused_shared_lhs), quantized
embedding on CPU/Metal/CUDA, paged flash-attn kernels, cutile fused MoE, the
in-place op aliasing API, parking_lot storage locks, Metal buffer builders and
debug labels, safetensors 0.8, hf-hub 1.0, tokenizers 0.23.
Ours keeps ROCm, Vulkan, wgpu, the IQ/MXFP4/NVFP4/TQ dtypes, and the i-quant
weight/activation block split in matmul. Our backends ride InplaceOpN and the
quantized embedding; Metal get_rows gains the IQ kernels. ug leaves, as upstream.
Co-authored-by: Hanzo Dev <[email protected]>
176 lines
6.7 KiB
Rust
176 lines
6.7 KiB
Rust
// Build script to run nvcc and generate the C glue code for launching the flash-attention kernel.
|
|
// The cuda build time is very long so one can set the FLASH_ATTN_BUILD_DIR environment
|
|
// variable in order to cache the compiled artifacts and avoid recompiling too often.
|
|
use anyhow::{Context, Result};
|
|
use hanzo_flash_attn_build::{cutlass_include_arg, fetch_cutlass};
|
|
use std::{fs, path::PathBuf};
|
|
const CUTLASS_COMMIT: &str = "7d49e6c7e2f8896c47f586706e67e1fb215529dc";
|
|
|
|
const KERNEL_FILES: [&str; 53] = [
|
|
"kernels/flash_api.cu",
|
|
"kernels/flash_fwd_splitkv_hdim512_fp16_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim512_bf16_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim512_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim512_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim64_fp16_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim128_fp16_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim256_fp16_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim64_bf16_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim128_bf16_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim256_bf16_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim64_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim128_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim256_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim64_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim128_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_splitkv_hdim256_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim128_fp16_sm80.cu",
|
|
"kernels/flash_fwd_hdim160_fp16_sm80.cu",
|
|
"kernels/flash_fwd_hdim192_fp16_sm80.cu",
|
|
"kernels/flash_fwd_hdim224_fp16_sm80.cu",
|
|
"kernels/flash_fwd_hdim256_fp16_sm80.cu",
|
|
"kernels/flash_fwd_hdim512_fp16_sm80.cu",
|
|
"kernels/flash_fwd_hdim32_fp16_sm80.cu",
|
|
"kernels/flash_fwd_hdim64_fp16_sm80.cu",
|
|
"kernels/flash_fwd_hdim96_fp16_sm80.cu",
|
|
"kernels/flash_fwd_hdim128_bf16_sm80.cu",
|
|
"kernels/flash_fwd_hdim160_bf16_sm80.cu",
|
|
"kernels/flash_fwd_hdim192_bf16_sm80.cu",
|
|
"kernels/flash_fwd_hdim224_bf16_sm80.cu",
|
|
"kernels/flash_fwd_hdim256_bf16_sm80.cu",
|
|
"kernels/flash_fwd_hdim512_bf16_sm80.cu",
|
|
"kernels/flash_fwd_hdim32_bf16_sm80.cu",
|
|
"kernels/flash_fwd_hdim64_bf16_sm80.cu",
|
|
"kernels/flash_fwd_hdim96_bf16_sm80.cu",
|
|
"kernels/flash_fwd_hdim128_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim160_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim192_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim224_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim256_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim512_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim32_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim64_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim96_fp16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim128_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim160_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim192_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim224_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim256_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim512_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim32_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim64_bf16_causal_sm80.cu",
|
|
"kernels/flash_fwd_hdim96_bf16_causal_sm80.cu",
|
|
];
|
|
|
|
const HEADER_FILES: [&str; 18] = [
|
|
"kernels/alibi.h",
|
|
"kernels/block_info.h",
|
|
"kernels/dropout.h",
|
|
"kernels/error.h",
|
|
"kernels/flash.h",
|
|
"kernels/flash_fwd_kernel.h",
|
|
"kernels/flash_fwd_launch_template.h",
|
|
"kernels/hardware_info.h",
|
|
"kernels/kernel_helpers.h",
|
|
"kernels/kernel_traits.h",
|
|
"kernels/kernel_traits_sm90.h",
|
|
"kernels/kernels.h",
|
|
"kernels/mask.h",
|
|
"kernels/philox.cuh",
|
|
"kernels/rotary.h",
|
|
"kernels/softmax.h",
|
|
"kernels/static_switch.h",
|
|
"kernels/utils.h",
|
|
];
|
|
|
|
fn update_hash(hash: &mut u64, bytes: &[u8]) {
|
|
const FNV_PRIME: u64 = 1099511628211;
|
|
for &byte in bytes {
|
|
*hash ^= u64::from(byte);
|
|
*hash = hash.wrapping_mul(FNV_PRIME);
|
|
}
|
|
}
|
|
|
|
fn header_hash() -> Result<u64> {
|
|
let mut hash = 14695981039346656037;
|
|
for file in HEADER_FILES {
|
|
update_hash(&mut hash, file.as_bytes());
|
|
update_hash(&mut hash, &fs::read(file)?);
|
|
}
|
|
Ok(hash)
|
|
}
|
|
|
|
fn main() -> Result<()> {
|
|
println!("cargo::rerun-if-changed=build.rs");
|
|
for kernel_file in KERNEL_FILES.iter() {
|
|
println!("cargo::rerun-if-changed={kernel_file}");
|
|
}
|
|
for header_file in HEADER_FILES.iter() {
|
|
println!("cargo::rerun-if-changed={header_file}");
|
|
}
|
|
let out_dir = PathBuf::from(std::env::var("OUT_DIR").context("OUT_DIR not set")?);
|
|
let build_dir = match std::env::var("FLASH_ATTN_BUILD_DIR") {
|
|
Err(_) =>
|
|
{
|
|
#[allow(clippy::redundant_clone)]
|
|
out_dir.clone()
|
|
}
|
|
Ok(build_dir) => {
|
|
let path = PathBuf::from(build_dir);
|
|
path.canonicalize().expect(&format!(
|
|
"Directory doesn't exists: {} (the current directory is {})",
|
|
&path.display(),
|
|
std::env::current_dir()?.display()
|
|
))
|
|
}
|
|
};
|
|
|
|
// Fetch cutlass headers on-demand
|
|
let cutlass_dir = fetch_cutlass(&out_dir, CUTLASS_COMMIT)?;
|
|
let cutlass_include: &'static str =
|
|
Box::leak(cutlass_include_arg(&cutlass_dir).into_boxed_str());
|
|
// Kernel objects cached in FLASH_ATTN_BUILD_DIR must rebuild when a shared header changes.
|
|
let header_hash_arg: &'static str =
|
|
Box::leak(format!("-DFLASH_ATTN_HEADER_HASH=0x{:016x}", header_hash()?).into_boxed_str());
|
|
|
|
let kernels = KERNEL_FILES.iter().collect();
|
|
let mut builder = bindgen_cuda::Builder::default()
|
|
.kernel_paths(kernels)
|
|
.out_dir(build_dir.clone())
|
|
.arg("-std=c++17")
|
|
.arg("-O3")
|
|
.arg("-U__CUDA_NO_HALF_OPERATORS__")
|
|
.arg("-U__CUDA_NO_HALF_CONVERSIONS__")
|
|
.arg("-U__CUDA_NO_HALF2_OPERATORS__")
|
|
.arg("-U__CUDA_NO_BFLOAT16_CONVERSIONS__")
|
|
.arg(&cutlass_include)
|
|
.arg("--expt-relaxed-constexpr")
|
|
.arg("--expt-extended-lambda")
|
|
.arg("--use_fast_math")
|
|
.arg("--verbose")
|
|
.arg(header_hash_arg);
|
|
|
|
let mut is_target_msvc = false;
|
|
if let Ok(target) = std::env::var("TARGET") {
|
|
if target.contains("msvc") {
|
|
is_target_msvc = true;
|
|
builder = builder.arg("-D_USE_MATH_DEFINES");
|
|
}
|
|
}
|
|
|
|
if !is_target_msvc {
|
|
builder = builder.arg("-Xcompiler").arg("-fPIC");
|
|
}
|
|
|
|
let out_file = build_dir.join("libflashattention.a");
|
|
builder.build_lib(out_file);
|
|
|
|
println!("cargo::rustc-link-search={}", build_dir.display());
|
|
println!("cargo::rustc-link-lib=flashattention");
|
|
println!("cargo::rustc-link-lib=dylib=cudart");
|
|
if !is_target_msvc {
|
|
println!("cargo::rustc-link-lib=dylib=stdc++");
|
|
}
|
|
Ok(())
|
|
}
|