mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-20 23:08:29 +00:00
Quantized Qwen3 weights now matmul ON the Vulkan GPU reading the GGML block format straight from a device buffer, instead of dequantizing the whole weight to f32 first (4x the decode bandwidth). Three fused on-GPU quant matvec compute shaders: - mul_mat_vec_q4_0.comp — 18 B/block (f16 d + 16×u8), nibble dequant (q-8)*d - mul_mat_vec_q8_0.comp — 34 B/block (f16 d + 32×i8), reads GGUF bytes as-is - mul_mat_vec_q4k.comp — 144 B super-block, 8×32 with 6-bit packed scales/mins Each reads the byte-packed GGML block out of a u32 SSBO (byte-addressed; blocks are not 4-aligned) and is byte-exact with the k_quants.rs CPU dequant. Wiring: QMatMul gains the GGML dtype on its VulkanQuant variant and, in from_arc, uploads the raw quantized bytes (upload_qweight, no re-pack/no re-quant) for Q4_0/Q8_0/Q4_K on a Vulkan device; the decode (1-row) forward dispatches the matching matvec_*_gpu. Prefill still dequantizes to a temp f32 weight. The prior Q8_0 path re-quantized via a CPU dequant + custom 9-u32 pack; the native Q8_0 kernel removes that lossy round-trip. Validation on the GB10 (NVIDIA Vulkan driver), vs an f64 dequant reference over identical quantized weights, Qwen3 shapes (hidden 2048/4096, nout up to 4096): raw kernels — Q4_0 max_abs 1.4e-5, Q8_0 2.3e-5, Q4_K 7.4e-5 (RMS-rel ≤ 1.1e-6) QMatMul.forward— Q4_0 8e-6, Q8_0 7e-6, Q4_K 4.2e-5 i.e. pure fp32 accumulation noise. New test: tests/vulkan_quant_tests.rs (skips w/o a GPU). build.rs / regen-spv.sh: fall back to glslangValidator (-V) when glslc is absent, so the shaders compile on glslang-only hosts (this box has no shaderc). Also repairs a pre-existing broken `--features vulkan` lib build: a half-landed host_visible / PooledBuf refactor left alloc_f32/alloc_f16/alloc_u32/free_scratch/reclaim on the old 2-tuple raw_buffer return; threaded the host-visible flag through so the crate compiles. Touches only the Vulkan backend.
127 lines
5.3 KiB
Rust
127 lines
5.3 KiB
Rust
// Build script for hanzo-ml.
|
|
//
|
|
// Vulkan backend: provides every GLSL compute kernel under
|
|
// `src/vulkan/shaders/*.comp` to the backend as `$OUT_DIR/<name>.spv`, which
|
|
// it loads via `include_bytes!(concat!(env!("OUT_DIR"), "/<name>.spv"))`.
|
|
//
|
|
// Two ways to obtain the SPIR-V, in priority order:
|
|
// 1. If `glslc` (shaderc) is available, compile each `*.comp` fresh
|
|
// (`--target-env=vulkan1.2`). The `.comp` sources are the source of truth.
|
|
// 2. Otherwise fall back to the pre-compiled `.spv` checked in under
|
|
// `src/vulkan/spv/<name>.spv`. This lets the crate build on hosts WITHOUT
|
|
// a Vulkan SDK / glslc (e.g. native Windows) — only a glslc-equipped host
|
|
// (any Linux dev box) needs to regenerate the committed `.spv` when a
|
|
// shader changes (see `src/vulkan/regen-spv.sh`).
|
|
//
|
|
// The whole step is guarded so it never breaks a non-Vulkan build:
|
|
// - It only runs when the `vulkan` cargo feature is enabled
|
|
// (`CARGO_FEATURE_VULKAN` is set by cargo), AND
|
|
// - it only runs when the shaders directory actually exists.
|
|
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::Command;
|
|
|
|
fn main() {
|
|
// Always re-run if the build script itself changes.
|
|
println!("cargo:rerun-if-changed=build.rs");
|
|
|
|
let vulkan_feature = std::env::var_os("CARGO_FEATURE_VULKAN").is_some();
|
|
let shaders_dir = Path::new("src/vulkan/shaders");
|
|
let prebuilt_dir = Path::new("src/vulkan/spv");
|
|
|
|
// No-op for non-vulkan builds, or if the shaders haven't been added yet.
|
|
if !vulkan_feature || !shaders_dir.is_dir() {
|
|
return;
|
|
}
|
|
|
|
// Re-run if a shader is added/removed from the directory.
|
|
println!("cargo:rerun-if-changed={}", shaders_dir.display());
|
|
println!("cargo:rerun-if-changed={}", prebuilt_dir.display());
|
|
|
|
let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set by cargo");
|
|
|
|
let mut entries: Vec<_> = std::fs::read_dir(shaders_dir)
|
|
.unwrap_or_else(|e| panic!("failed to read {}: {e}", shaders_dir.display()))
|
|
.filter_map(|e| e.ok().map(|e| e.path()))
|
|
.filter(|p| p.extension().map(|e| e == "comp").unwrap_or(false))
|
|
.collect();
|
|
// Deterministic compile order.
|
|
entries.sort();
|
|
|
|
if entries.is_empty() {
|
|
return;
|
|
}
|
|
|
|
// Resolve glslc once (allow override via GLSLC for unusual toolchains).
|
|
let glslc = std::env::var("GLSLC").unwrap_or_else(|_| "glslc".to_string());
|
|
let have_glslc = Command::new(&glslc)
|
|
.arg("--version")
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false);
|
|
|
|
// Second-choice compiler: glslangValidator (the `glslang-tools` package, present on hosts
|
|
// that ship glslang but not shaderc/glslc). `-V` emits Vulkan SPIR-V; the same `.comp`
|
|
// sources compile under it (incl. GL_EXT_shader_explicit_arithmetic_types_float16). Only
|
|
// probed when glslc is missing so the glslc fast path is unchanged.
|
|
let glslang = std::env::var("GLSLANG").unwrap_or_else(|_| "glslangValidator".to_string());
|
|
let have_glslang = !have_glslc
|
|
&& Command::new(&glslang)
|
|
.arg("--version")
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false);
|
|
|
|
if !have_glslc && !have_glslang {
|
|
println!(
|
|
"cargo:warning=neither glslc nor glslangValidator found; using pre-compiled SPIR-V from {}",
|
|
prebuilt_dir.display()
|
|
);
|
|
} else if have_glslang {
|
|
println!("cargo:warning=glslc not found; compiling shaders with glslangValidator");
|
|
}
|
|
|
|
for src in &entries {
|
|
// <name>.comp -> <name>.spv
|
|
let stem = src
|
|
.file_stem()
|
|
.and_then(|s| s.to_str())
|
|
.unwrap_or_else(|| panic!("invalid shader file name: {}", src.display()));
|
|
let dst = format!("{out_dir}/{stem}.spv");
|
|
let src_str = src.to_str().expect("non-UTF8 shader path");
|
|
println!("cargo:rerun-if-changed={src_str}");
|
|
|
|
if have_glslc {
|
|
// Source of truth: compile the .comp fresh.
|
|
let status = Command::new(&glslc)
|
|
.args(["--target-env=vulkan1.2", src_str, "-o", &dst])
|
|
.status()
|
|
.unwrap_or_else(|e| panic!("failed to run `{glslc}`: {e}"));
|
|
assert!(
|
|
status.success(),
|
|
"glslc failed to compile {src_str} (exit status: {status})"
|
|
);
|
|
} else if have_glslang {
|
|
// Compile fresh with glslangValidator (`-V` => Vulkan SPIR-V binary).
|
|
let status = Command::new(&glslang)
|
|
.args(["--target-env", "vulkan1.2", "-V", src_str, "-o", &dst])
|
|
.status()
|
|
.unwrap_or_else(|e| panic!("failed to run `{glslang}`: {e}"));
|
|
assert!(
|
|
status.success(),
|
|
"glslangValidator failed to compile {src_str} (exit status: {status})"
|
|
);
|
|
} else {
|
|
// Fallback: copy the committed pre-compiled SPIR-V.
|
|
let prebuilt: PathBuf = prebuilt_dir.join(format!("{stem}.spv"));
|
|
println!("cargo:rerun-if-changed={}", prebuilt.display());
|
|
std::fs::copy(&prebuilt, &dst).unwrap_or_else(|e| {
|
|
panic!(
|
|
"no glslc and no pre-compiled SPIR-V at {} (run src/vulkan/regen-spv.sh on a host with glslc): {e}",
|
|
prebuilt.display()
|
|
)
|
|
});
|
|
}
|
|
}
|
|
}
|