Files
engine/hanzo-quant/src/lib.rs

1987 lines
64 KiB
Rust
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use std::{
borrow::Cow,
fmt::Debug,
num::NonZeroUsize,
sync::{atomic::AtomicUsize, Arc, Condvar, Mutex, MutexGuard},
};
use blockwise_fp8::blockwise_fp8_linear_b;
#[cfg(feature = "metal")]
use hanzo_ml::D;
use hanzo_ml::{
quantized::{GgmlDType, QMatMul, QTensor},
DType, Device, Result, Tensor,
};
use pertensor_fp8::pertensor_fp8_linear_b;
#[cfg(feature = "metal")]
pub mod metal_kernels;
mod afq;
mod bitsandbytes;
pub mod blockwise_fp8;
pub mod cublaslt;
#[cfg(all(feature = "cuda", feature = "cutile"))]
pub mod cutile;
pub mod distributed;
mod dummy;
#[cfg(all(feature = "cuda", has_nvfp4_cutlass_kernels))]
pub mod experts;
pub mod f8q8;
mod fp8;
pub mod gemv;
mod gguf;
pub mod gguf_export;
mod gptq;
mod hqq;
mod imatrix;
mod lora;
#[cfg(all(feature = "cuda", feature = "cutile"))]
pub mod moe;
mod mxfp4;
pub mod nvfp4;
mod pending_layer;
mod pertensor_fp8;
pub mod quantize;
pub mod rotary;
pub mod safetensors;
mod scalar_fp8;
mod unquantized;
mod utils;
mod vector_fp8;
pub use gguf_export::{export_gguf, GgufOutputType};
use gptq::gptq_linear;
use lora::merge_lora_weights;
use regex::Regex;
pub use safetensors::{Shard, ShardedSafeTensors, ShardedVarBuilder};
#[cfg(feature = "metal")]
pub use afq::ops::{
afq_gather_qmm_rhs_sorted, afq_gather_qmm_rhs_sorted_gate_up, metal_arg_sort_u32_1d,
metal_moe_weighted_reduce_flat,
};
pub use afq::{AfqBits, AfqGroupSize, AfqInner, AfqLayer};
pub use bitsandbytes::{BnbLinear, BnbQuantParams, BnbQuantType};
pub use blockwise_fp8::{
blockwise_fp8_act, blockwise_fp8_moe, fp8_blockwise_dequantize, fp8_blockwise_quantize,
BlockwiseFP8Linear,
};
pub use distributed::{
layers::{
compute_kv_shard, compute_n_kv_groups, ColumnParallelLayer, FusedExperts, PackedExperts,
ReplicatedLayer, RowParallelLayer,
},
pipeline::{broadcast_head_tokens, PpHeader, RingPipeline, PP_OP_FORWARD, PP_OP_TERMINATE},
socket::{Client, Server},
BarrierLike, Comm, Id, RingConfig, SumAllReduce,
};
pub use dummy::{DummyLayer, DummyLayerInfo};
pub use f8q8::F8Q8Linear;
pub use fp8::FP8Linear;
#[cfg(feature = "cuda")]
pub use gemv::gemv;
pub use gemv::{should_use_gemv, GEMV_CONTROLLER};
#[cfg(feature = "cuda")]
pub use gguf::cuda::{
grouped_moe_gemm_prequantized, indexed_moe_fused_decode, moe_dispatch_build,
moe_weighted_reduce_flat, moe_weighted_reduce_flat_bf16, quantize_input_q8_1,
ACT_GELU_PYTORCH_TANH, ACT_SILU,
};
#[cfg(feature = "cuda")]
pub use gguf::fast_mmq::{
grouped as grouped_moe_mmq, grouped_from_glu_pair as grouped_moe_mmq_from_glu_pair,
grouped_pair as grouped_moe_mmq_pair, supports as supports_mmq,
};
pub use gguf::GgufMatMul;
pub use gptq::GptqLayer;
pub use hqq::{HqqAxis, HqqBits, HqqConfig, HqqLayer};
pub use imatrix::{CollectedImatrixData, ImatrixLayerStats};
pub use lora::{
clear_applied_loras, get_applied_loras, linear_no_bias_static_lora, push_applied_lora,
LoraAdapter, LoraConfig, StaticLoraConfig, MULTI_LORA_DELIMITER,
};
pub use mxfp4::MXFP4Layer;
pub use nvfp4::NVFP4Layer;
pub use pending_layer::PendingIsqLayer;
pub use pertensor_fp8::PerTensorFP8Linear;
pub use unquantized::UnquantLinear;
pub use utils::flash_attn_sinks_metal;
pub use utils::flash_attn_sinks_varlen_metal;
#[cfg(feature = "cuda")]
pub use utils::gptoss_swiglu_fused;
#[cfg(feature = "cuda")]
pub use utils::gptoss_swiglu_interleaved;
pub use utils::isq::apply_immediate_isq;
pub use utils::softcap;
pub use utils::softmax_with_sinks;
pub use utils::{fused_glu, GluActivationType};
pub use utils::{log, BitWiseOp, CumSumOp, LeftshiftOp, NonZeroOp, SortOp, UQFF_QUANT_TYPE_OFFSET};
pub use vector_fp8::{fp8_vector_dequantize, fp8_vector_quantize};
use hanzo_nn::{Conv1d, Conv2d, Linear, Module};
use serde::{Deserialize, Deserializer, Serialize};
/// Limits outstanding async ISQ jobs to prevent unbounded memory growth.
///
/// Without backpressure, MoE models (e.g. Gemma4 with 128 experts × 30 layers)
/// queue BF16 tensor data in the rayon pool faster than the pool can quantize,
/// causing OOM on memory-constrained systems like macOS Metal with unified memory.
pub struct IsqBackpressure {
count: Mutex<usize>,
cvar: Condvar,
max: usize,
}
impl IsqBackpressure {
pub fn new(max: usize) -> Self {
Self {
count: Mutex::new(0),
cvar: Condvar::new(),
max,
}
}
/// Block until a slot is available, then increment the outstanding count.
pub fn acquire(&self) {
let mut count = self.count.lock().expect("ISQ backpressure lock poisoned");
while *count >= self.max {
count = self
.cvar
.wait(count)
.expect("ISQ backpressure lock poisoned");
}
*count += 1;
}
/// Decrement the outstanding count and wake a blocked loader thread.
pub fn release(&self) {
let mut count = self.count.lock().expect("ISQ backpressure lock poisoned");
*count = count.saturating_sub(1);
self.cvar.notify_one();
}
}
impl Debug for IsqBackpressure {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let count = self.count.lock().map(|c| *c).unwrap_or(0);
f.debug_struct("IsqBackpressure")
.field("outstanding", &count)
.field("max", &self.max)
.finish()
}
}
#[derive(Clone, Debug)]
pub struct ImmediateIsqParams {
pub guard: QuantizeOntoGuard,
pub ty: Option<IsqType>,
pub predicates: Vec<Regex>,
pub overrides: Vec<ImmediateIsqOverride>,
/// Thread pool for parallel immediate ISQ on discrete GPUs.
/// When `Some`, `apply_immediate_isq` will spawn quantization tasks
/// on this pool and return `PendingIsqLayer` wrappers.
pub pool: Option<Arc<rayon::ThreadPool>>,
/// Backpressure to limit outstanding async ISQ jobs.
pub backpressure: Arc<IsqBackpressure>,
}
#[derive(Clone, Debug)]
pub struct ImmediateIsqOverride {
pub predicate: Regex,
pub ty: Option<IsqType>,
pub device: Option<Device>,
}
#[derive(Clone, Debug)]
pub struct ImmediateIsqMatch {
pub ty: IsqType,
pub device: Option<Device>,
}
thread_local! {
static ENGINE_IMMEDIATE_ISQ: std::cell::RefCell<Option<ImmediateIsqParams>> = const { std::cell::RefCell::new(None) } ;
}
pub fn set_immediate_isq(isq: Option<IsqType>, predicates: Vec<Regex>) {
let (pool, _) = create_isq_thread_pool(isq);
set_immediate_isq_with_pool(isq, predicates, Vec::new(), pool);
}
pub fn set_immediate_isq_with_pool(
isq: Option<IsqType>,
predicates: Vec<Regex>,
overrides: Vec<ImmediateIsqOverride>,
pool: rayon::ThreadPool,
) {
// Allow pool threads + 1 outstanding jobs: enough for pipeline overlap
// (load next tensor while pool quantizes current) without unbounded growth.
let max_outstanding = pool.current_num_threads() + 1;
ENGINE_IMMEDIATE_ISQ.with(|cell| {
*cell.borrow_mut() = Some(ImmediateIsqParams {
guard: QuantizeOntoGuard::new(),
ty: isq,
predicates,
overrides,
backpressure: Arc::new(IsqBackpressure::new(max_outstanding)),
pool: Some(Arc::new(pool)),
});
});
}
/// Create a rayon thread pool for parallel immediate ISQ.
/// Returns `(pool, num_threads)` so callers can log the thread count.
///
/// Thread count is based on the quantization type:
/// - GGML types (Q2K-Q8K) and F8E4M3: `rayon::current_num_threads()` (CPU quantization)
/// - HQQ/AFQ: 1 thread (GPU quantization, serialized by `QuantizeOntoGuard`)
pub fn create_isq_thread_pool(ty: Option<IsqType>) -> (rayon::ThreadPool, usize) {
let num_threads = if std::env::var("ISQ_SINGLETHREAD").is_ok() {
1
} else if let Some(ty) = ty {
ty.get_max_isq_cpu_threads()
.map(usize::from)
.unwrap_or_else(rayon::current_num_threads)
} else {
rayon::current_num_threads()
};
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(num_threads)
.build()
.expect("Failed to create ISQ thread pool");
(pool, num_threads)
}
pub fn get_immediate_isq() -> Option<ImmediateIsqParams> {
ENGINE_IMMEDIATE_ISQ.with(|cell| cell.borrow().clone())
}
pub fn clear_immediate_isq() {
ENGINE_IMMEDIATE_ISQ.with(|cell| {
*cell.borrow_mut() = None;
});
}
pub fn should_apply_immediate_isq(vb: &ShardedVarBuilder) -> bool {
immediate_isq_match(vb).is_some()
}
pub fn immediate_isq_match(vb: &ShardedVarBuilder) -> Option<ImmediateIsqMatch> {
let immediate_isq = get_immediate_isq()?;
// Add a .weight to match the ISQ regexes!
let prefix = format!("{}.weight", vb.prefix());
resolve_immediate_isq(&immediate_isq, &prefix)
}
fn resolve_immediate_isq(params: &ImmediateIsqParams, prefix: &str) -> Option<ImmediateIsqMatch> {
if let Some(override_hit) = params
.overrides
.iter()
.find(|override_pred| override_pred.predicate.is_match(prefix))
{
if let Some(ty) = override_hit.ty.or(params.ty) {
return Some(ImmediateIsqMatch {
ty,
device: override_hit.device.clone(),
});
}
return None;
}
if let Some(ty) = params.ty {
if params
.predicates
.iter()
.any(|predicate| predicate.is_match(prefix))
{
return Some(ImmediateIsqMatch { ty, device: None });
}
}
None
}
#[derive(Debug, Clone, Serialize)]
#[serde(tag = "quant_method", rename_all = "lowercase")]
pub enum QuantizedConfig {
GptqAwq {
bits: usize,
group_size: usize,
checkpoint_format: Option<String>,
is_awq: bool,
},
Fp8 {
weight_block_size: Option<Vec<usize>>,
},
Bitsandbytes {
bnb_4bit_quant_type: Option<String>,
},
Afq {
bits: usize,
group_size: usize,
/// MLX's per-module entries, by module path: its own bits and group size, or left
/// unquantized.
#[serde(skip_serializing_if = "std::collections::HashMap::is_empty")]
modules: std::collections::HashMap<String, AfqModule>,
},
MXFP4 {},
ModelOpt {
quant_algo: Option<String>,
quantized_layers: std::collections::HashMap<String, ModelOptLayerConfig>,
},
}
/// One module's entry in an MLX `quantization` block.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum AfqModule {
/// `false`: the module stays unquantized. (`true` means the block's defaults.)
Quantized(bool),
/// The module's own affine recipe.
Recipe { bits: usize, group_size: usize },
}
/// The entry for the module at `prefix` in a per-module recipe keyed by module path: the exact
/// path, or one path ending the other at a `.` boundary (checkpoints nest the same modules under
/// different roots).
pub(crate) fn recipe_entry<'a, V>(
recipe: &'a std::collections::HashMap<String, V>,
prefix: &str,
) -> Option<&'a V> {
recipe.get(prefix).or_else(|| {
recipe.iter().find_map(|(k, v)| {
(k.ends_with(&format!(".{prefix}")) || prefix.ends_with(&format!(".{k}"))).then_some(v)
})
})
}
impl QuantizedConfig {
/// The AFQ (bits, group size) for the module at `prefix`: its own entry, else the block's
/// defaults; `None` when the recipe leaves it unquantized or the config is not AFQ.
pub fn afq_at(&self, prefix: &str) -> Option<(usize, usize)> {
let QuantizedConfig::Afq {
bits,
group_size,
modules,
} = self
else {
return None;
};
match recipe_entry(modules, prefix) {
Some(AfqModule::Quantized(false)) => None,
Some(AfqModule::Recipe { bits, group_size }) => Some((*bits, *group_size)),
Some(AfqModule::Quantized(true)) | None => Some((*bits, *group_size)),
}
}
}
/// MLX's per-module entries among a quantization block's keys: booleans and objects carrying
/// `bits`. Only the affine mode exists in AFQ.
fn afq_modules<E: serde::de::Error>(
rest: std::collections::HashMap<String, serde_json::Value>,
) -> std::result::Result<std::collections::HashMap<String, AfqModule>, E> {
let mut modules = std::collections::HashMap::new();
for (key, value) in rest {
let mode = match &value {
serde_json::Value::Bool(_) => None,
serde_json::Value::Object(o) if o.contains_key("bits") => o.get("mode"),
serde_json::Value::String(_) if key == "mode" => Some(&value),
_ => continue,
};
if let Some(mode) = mode.and_then(|m| m.as_str()).filter(|m| *m != "affine") {
return Err(E::custom(format!(
"AFQ supports the affine mode only; {key} asks for {mode}"
)));
}
if key != "mode" {
let module = serde_json::from_value(value).map_err(E::custom)?;
modules.insert(key, module);
}
}
Ok(modules)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelOptLayerConfig {
#[serde(default)]
pub quant_algo: Option<String>,
#[serde(default)]
pub group_size: Option<usize>,
}
// Common fields for all variants
#[derive(Deserialize)]
struct RawConfig {
quant_method: Option<String>,
quant_algo: Option<String>,
quantized_layers: Option<std::collections::HashMap<String, ModelOptLayerConfig>>,
bits: Option<usize>,
group_size: Option<usize>,
checkpoint_format: Option<String>,
weight_block_size: Option<Vec<usize>>,
bnb_4bit_quant_type: Option<String>,
#[serde(flatten)]
rest: std::collections::HashMap<String, serde_json::Value>,
}
// Custom deserializer implementation
impl<'de> Deserialize<'de> for QuantizedConfig {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let raw = RawConfig::deserialize(deserializer)?;
match &raw.quant_method {
Some(m) if m == "gptq" || m == "awq" => {
let bits = raw
.bits
.ok_or_else(|| serde::de::Error::missing_field("bits"))?;
let group_size = raw
.group_size
.ok_or_else(|| serde::de::Error::missing_field("group_size"))?;
Ok(QuantizedConfig::GptqAwq {
bits,
group_size,
checkpoint_format: raw.checkpoint_format,
is_awq: m == "awq",
})
}
Some(m) if m == "fp8" => {
// weight_block_size is optional - None means per-tensor quantization
Ok(QuantizedConfig::Fp8 {
weight_block_size: raw.weight_block_size,
})
}
Some(m) if m == "bitsandbytes" => Ok(QuantizedConfig::Bitsandbytes {
bnb_4bit_quant_type: raw.bnb_4bit_quant_type,
}),
Some(m) if m == "afq" => {
let bits = raw
.bits
.ok_or_else(|| serde::de::Error::missing_field("bits"))?;
let group_size = raw
.group_size
.ok_or_else(|| serde::de::Error::missing_field("group_size"))?;
Ok(QuantizedConfig::Afq {
bits,
group_size,
modules: afq_modules(raw.rest)?,
})
}
Some(m) if m == "mxfp4" => {
Ok(QuantizedConfig::MXFP4 { })
}
Some(m) if m == "modelopt" => {
Ok(QuantizedConfig::ModelOpt {
quant_algo: raw.quant_algo,
quantized_layers: raw.quantized_layers.unwrap_or_default(),
})
}
None => {
let bits = raw
.bits
.ok_or_else(|| serde::de::Error::missing_field("bits"))?;
let group_size = raw
.group_size
.ok_or_else(|| serde::de::Error::missing_field("group_size"))?;
Ok(QuantizedConfig::Afq {
bits,
group_size,
modules: afq_modules(raw.rest)?,
})
}
Some(unknown_method) => {
Err(serde::de::Error::custom(format!(
"Unknown quantization method: {unknown_method}. Expected one of: gptq, fp8, bitsandbytes, afq, mxfp4, modelopt, or not specified"
)))
},
}
}
}
impl QuantizedConfig {
pub fn name(&self) -> &'static str {
match self {
Self::GptqAwq { .. } => "gptq",
Self::Fp8 { .. } => "fp8",
Self::Bitsandbytes { .. } => "bitsandbytes",
Self::Afq { .. } => "afq",
Self::MXFP4 { .. } => "mxfp4",
Self::ModelOpt { .. } => "modelopt",
}
}
pub fn get_bits_name(&self, vb: &ShardedVarBuilder) -> String {
match self {
Self::GptqAwq { bits, .. } => format!("{bits} bits"),
Self::Fp8 { .. } => "8 bits".to_string(),
Self::Bitsandbytes {
bnb_4bit_quant_type: Some(_),
} => "4 bits".to_string(),
Self::Bitsandbytes {
bnb_4bit_quant_type: None,
} => "8 bits".to_string(),
Self::Afq { bits, .. } => format!("{bits} bits"),
Self::MXFP4 {} => format!("{} bits", mxfp4::N_BITS),
Self::ModelOpt {
quantized_layers, ..
} => {
let prefix = vb.prefix();
if let Some(cfg) = quantized_layers.iter().find_map(|(k, v)| {
if k == &prefix || k.ends_with(&format!(".{prefix}")) || prefix.ends_with(k) {
Some(v)
} else {
None
}
}) {
match cfg.quant_algo.as_deref() {
Some("NVFP4") => "4 bits (NVFP4)".to_string(),
Some("FP8") => "8 bits (FP8)".to_string(),
_ => "mixed precision".to_string(),
}
} else {
"16 bits".to_string()
}
}
}
}
pub fn pack_factor(&self, dtype: DType) -> usize {
match self {
Self::GptqAwq { bits, .. } | Self::Afq { bits, .. } => match bits {
2 => IsqType::Q2K.pack_factor(dtype),
3 => IsqType::Q3K.pack_factor(dtype),
4 => IsqType::Q4K.pack_factor(dtype),
5 => IsqType::Q5K.pack_factor(dtype),
6 => IsqType::Q6K.pack_factor(dtype),
8 => IsqType::Q8_0.pack_factor(dtype),
40 => 4, // mxfp4: 2 FP4 values per byte = factor of 4
other => panic!("Unexpected bits in `pack_factor` {other}"),
},
Self::Fp8 { .. } => IsqType::Q8_0.pack_factor(dtype),
Self::Bitsandbytes {
bnb_4bit_quant_type: Some(_),
}
| Self::Bitsandbytes {
bnb_4bit_quant_type: None,
} => IsqType::Q4K.pack_factor(dtype),
Self::MXFP4 {} => IsqType::Q4_0.pack_factor(dtype),
Self::ModelOpt { .. } => 4,
}
}
}
#[derive(Debug, Clone)]
pub enum QuantMethodConfig {
GptqAwq {
bits: i32,
/// The weights are in the shuffled layout (rows sequential, packed values reordered).
is_shuffled: bool,
q_weight: Tensor,
qzeros: Option<Tensor>,
scales: Tensor,
g_idx: Option<Tensor>,
bias: Option<Tensor>,
workspace: Option<Tensor>,
is_marlin: bool,
is_awq: bool,
},
Gguf {
q_weight: Arc<QTensor>,
b: Option<Tensor>,
},
Unquantized(Linear),
Hqq {
tensor: Tensor,
bits: HqqBits,
group_size: NonZeroUsize,
axis: HqqAxis,
optimization_steps: Option<usize>,
round_zeros: Option<bool>,
channel_wise: Option<bool>,
bias: Option<Tensor>,
},
Dummy,
FP8 {
lin: Linear,
dtype: DType,
},
Bnb {
weight: Tensor,
bias: Option<Tensor>,
params: BnbQuantParams,
quant_ty: BnbQuantType,
},
BlockwiseFP8 {
weight: Tensor,
weight_scale_inv: Tensor,
bias: Option<Tensor>,
dequant_dtype: DType,
weight_block_size: Vec<usize>,
},
PerTensorFP8 {
weight: Tensor,
weight_scale: Tensor,
/// The checkpoint's calibrated activation scale, present only when it quantized
/// activations too.
input_scale: Option<Tensor>,
bias: Option<Tensor>,
dequant_dtype: DType,
},
Afq {
weight: Tensor,
bias: Option<Tensor>,
bits: AfqBits,
group_size: AfqGroupSize,
},
MXFP4 {
blocks: Tensor,
scales: Tensor,
bias: Option<Tensor>,
},
NVFP4 {
weight: Tensor,
weight_scale: Tensor,
weight_scale_2: Option<Tensor>,
/// The checkpoint's calibrated activation scale, present only when it was
/// quantized for FP4 activations.
input_scale: Option<Tensor>,
bias: Option<Tensor>,
dequant_dtype: DType,
},
}
/// Device/configurable intelligent matrix multiplication
/// - Handles limitation of `accelerate` which requires f32
pub struct MatMul;
impl MatMul {
/// Compute matrix-matrix product.
pub fn matmul(&self, a: &Tensor, b: &Tensor) -> Result<Tensor> {
#[cfg(feature = "accelerate")]
{
let original_dtype = a.dtype();
a.to_dtype(DType::F32)?
.matmul(&b.to_dtype(DType::F32)?)?
.to_dtype(original_dtype)
}
#[cfg(not(feature = "accelerate"))]
{
// The CPU multiplies half-precision inputs in F32 and F32 inputs as they are. Rounding
// them to F16 instead kept three significant digits of every attention score and sent
// any score past 65504 to infinity.
if a.device().is_cpu() && matches!(a.dtype(), DType::F16 | DType::BF16) {
let original_dtype = a.dtype();
a.to_dtype(DType::F32)?
.matmul(&b.to_dtype(DType::F32)?)?
.to_dtype(original_dtype)
} else {
a.matmul(b)
}
}
}
/// Compute matrix-matrix product.
/// The result will be divided by the `scale` parameter in an affine division.
pub fn matmul_affine_div(&self, a: &Tensor, b: &Tensor, scale: f64) -> Result<Tensor> {
// TODO(hanzoai): Optimize this by using the gemm parameter?
self.matmul(a, b)? / scale
}
/// Compute matrix-matrix product.
/// The result will be divided by the `scale` parameter in an affine multiplication.
pub fn matmul_affine_mul(&self, a: &Tensor, b: &Tensor, scale: f64) -> Result<Tensor> {
// TODO(hanzoai): Optimize this by using the gemm parameter?
self.matmul(a, b)? * scale
}
/// Compute quantized matrix-matrix product.
pub fn qmatmul(&self, x: &Tensor, matmul: &QMatMul) -> Result<Tensor> {
matmul.forward(x)
}
}
/// Device/configurable intelligent convolution
/// - Handles limitation of cpu which requires f32
pub struct Convolution;
impl Convolution {
pub fn forward_1d(&self, layer: &Conv1d, x: &Tensor) -> Result<Tensor> {
if x.device().is_cpu() {
let original_dtype = x.dtype();
Conv1d::new(
layer.weight().to_dtype(DType::F32)?,
layer.bias().map(|b| b.to_dtype(DType::F32)).transpose()?,
*layer.config(),
)
.forward(&x.to_dtype(DType::F32)?)?
.to_dtype(original_dtype)
} else {
layer.forward(x)
}
}
pub fn forward_2d(&self, layer: &Conv2d, x: &Tensor) -> Result<Tensor> {
if x.device().is_cpu() {
let original_dtype = x.dtype();
Conv2d::new(
layer.weight().to_dtype(DType::F32)?,
layer.bias().map(|b| b.to_dtype(DType::F32)).transpose()?,
*layer.config(),
)
.forward(&x.to_dtype(DType::F32)?)?
.to_dtype(original_dtype)
} else {
layer.forward(x)
}
}
}
/// In-situ quantization type specifying the format to apply to model weights.
#[derive(Clone, Copy, Debug, PartialEq, Hash, Eq, Serialize, Deserialize)]
pub enum IsqType {
Q4_0,
Q4_1,
Q5_0,
Q5_1,
Q8_0,
Q8_1,
Q2K,
Q3K,
Q4K,
Q5K,
Q6K,
Q8K,
HQQ8,
HQQ4,
// HQQ3,
// HQQ2,
// HQQ1,
F8E4M3,
AFQ8,
AFQ6,
AFQ4,
AFQ3,
AFQ2,
F8Q8,
MXFP4,
}
/// Target bit width for automatic ISQ quantization.
///
/// On Metal, these select AFQ variants; on CUDA/CPU, they select Q*K variants.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum IsqBits {
/// 2-bit quantization (AFQ2 on Metal, Q2K otherwise).
Two,
/// 3-bit quantization (AFQ3 on Metal, Q3K otherwise).
Three,
/// 4-bit quantization (AFQ4 on Metal, Q4K otherwise).
Four,
/// 5-bit quantization (Q5K on all platforms).
Five,
/// 6-bit quantization (AFQ6 on Metal, Q6K otherwise).
Six,
/// 8-bit quantization (AFQ8 on Metal, Q8_0 otherwise).
Eight,
}
impl IsqBits {
/// Resolve to the platform-appropriate `IsqType` for the given device.
pub fn resolve(self, device: &Device) -> IsqType {
match (self, device.is_metal()) {
(Self::Two, true) => IsqType::AFQ2,
(Self::Two, false) => IsqType::Q2K,
(Self::Three, true) => IsqType::AFQ3,
(Self::Three, false) => IsqType::Q3K,
(Self::Four, true) => IsqType::AFQ4,
(Self::Four, false) => IsqType::Q4K,
(Self::Five, _) => IsqType::Q5K,
(Self::Six, true) => IsqType::AFQ6,
(Self::Six, false) => IsqType::Q6K,
(Self::Eight, true) => IsqType::AFQ8,
(Self::Eight, false) => IsqType::Q8_0,
}
}
/// Return all platform variants, with the current platform's preferred variant first.
/// On Metal, AFQ variants come first; on other platforms, GGUF/Q variants come first.
pub fn expand(self) -> Vec<IsqType> {
#[cfg(feature = "metal")]
match self {
Self::Two => vec![IsqType::AFQ2, IsqType::Q2K],
Self::Three => vec![IsqType::AFQ3, IsqType::Q3K],
Self::Four => vec![IsqType::AFQ4, IsqType::Q4K],
Self::Five => vec![IsqType::Q5K],
Self::Six => vec![IsqType::AFQ6, IsqType::Q6K],
Self::Eight => vec![IsqType::AFQ8, IsqType::Q8_0],
}
#[cfg(not(feature = "metal"))]
match self {
Self::Two => vec![IsqType::Q2K, IsqType::AFQ2],
Self::Three => vec![IsqType::Q3K, IsqType::AFQ3],
Self::Four => vec![IsqType::Q4K, IsqType::AFQ4],
Self::Five => vec![IsqType::Q5K],
Self::Six => vec![IsqType::Q6K, IsqType::AFQ6],
Self::Eight => vec![IsqType::Q8_0, IsqType::AFQ8],
}
}
}
impl TryFrom<&str> for IsqBits {
type Error = ();
fn try_from(s: &str) -> std::result::Result<Self, ()> {
match s {
"2" => Ok(Self::Two),
"3" => Ok(Self::Three),
"4" => Ok(Self::Four),
"5" => Ok(Self::Five),
"6" => Ok(Self::Six),
"8" => Ok(Self::Eight),
_ => Err(()),
}
}
}
impl std::fmt::Display for IsqType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Q4_0 => write!(f, "q4_0"),
Self::Q4_1 => write!(f, "q4_1"),
Self::Q5_0 => write!(f, "q5_0"),
Self::Q5_1 => write!(f, "q5_1"),
Self::Q8_0 => write!(f, "q8_0"),
Self::Q8_1 => write!(f, "q8_1"),
Self::Q2K => write!(f, "q2k"),
Self::Q3K => write!(f, "q3k"),
Self::Q4K => write!(f, "q4k"),
Self::Q5K => write!(f, "q5k"),
Self::Q6K => write!(f, "q6k"),
Self::Q8K => write!(f, "q8k"),
Self::HQQ8 => write!(f, "hqq8"),
Self::HQQ4 => write!(f, "hqq4"),
Self::F8E4M3 => write!(f, "fp8"),
Self::AFQ8 => write!(f, "afq8"),
Self::AFQ6 => write!(f, "afq6"),
Self::AFQ4 => write!(f, "afq4"),
Self::AFQ3 => write!(f, "afq3"),
Self::AFQ2 => write!(f, "afq2"),
Self::F8Q8 => write!(f, "f8q8"),
Self::MXFP4 => write!(f, "mxfp4"),
}
}
}
impl IsqType {
/// Factor by which the weight size is reduced over the given dtype.
/// original size / pack factor = quantized size
pub fn pack_factor(&self, dtype: DType) -> usize {
match self {
Self::Q4_0 | Self::AFQ4 => (dtype.size_in_bytes() * GgmlDType::Q4_0.block_size())
.div_ceil(GgmlDType::Q4_0.type_size()),
Self::Q4_1 => (dtype.size_in_bytes() * GgmlDType::Q4_1.block_size())
.div_ceil(GgmlDType::Q4_1.type_size()),
Self::Q5_0 => (dtype.size_in_bytes() * GgmlDType::Q5_0.block_size())
.div_ceil(GgmlDType::Q5_0.type_size()),
Self::Q5_1 => (dtype.size_in_bytes() * GgmlDType::Q5_1.block_size())
.div_ceil(GgmlDType::Q5_1.type_size()),
Self::Q8_0 | Self::AFQ8 => (dtype.size_in_bytes() * GgmlDType::Q8_0.block_size())
.div_ceil(GgmlDType::Q8_0.type_size()),
Self::Q8_1 => (dtype.size_in_bytes() * GgmlDType::Q8_1.block_size())
.div_ceil(GgmlDType::Q8_1.type_size()),
Self::Q2K | Self::AFQ2 => (dtype.size_in_bytes() * GgmlDType::Q2K.block_size())
.div_ceil(GgmlDType::Q2K.type_size()),
Self::Q3K | Self::AFQ3 => (dtype.size_in_bytes() * GgmlDType::Q3K.block_size())
.div_ceil(GgmlDType::Q3K.type_size()),
Self::Q4K => (dtype.size_in_bytes() * GgmlDType::Q4K.block_size())
.div_ceil(GgmlDType::Q4K.type_size()),
Self::Q5K => (dtype.size_in_bytes() * GgmlDType::Q5K.block_size())
.div_ceil(GgmlDType::Q5K.type_size()),
Self::Q6K | Self::AFQ6 => (dtype.size_in_bytes() * GgmlDType::Q6K.block_size())
.div_ceil(GgmlDType::Q6K.type_size()),
Self::Q8K => (dtype.size_in_bytes() * GgmlDType::Q8K.block_size())
.div_ceil(GgmlDType::Q8K.type_size()),
// F8Q8: 33 bytes per 32 values -> similar to Q8_0
Self::F8Q8 => (dtype.size_in_bytes() * 32).div_ceil(33),
// Estimates
Self::HQQ4 => 4,
Self::HQQ8 => 2,
Self::F8E4M3 => 2,
// MXFP4: 4 bits per value + 1 byte scale per 32 values
// For BF16 (2 bytes): (2*32)/(16+1) ≈ 3.76 → 3
Self::MXFP4 => 3,
}
}
pub fn get_max_isq_cpu_threads(&self) -> Option<NonZeroUsize> {
match self {
/*IsqType::HQQ1 | IsqType::HQQ2 | IsqType::HQQ3 | */
IsqType::HQQ4
| IsqType::HQQ8
| IsqType::AFQ2
| IsqType::AFQ3
| IsqType::AFQ4
| IsqType::AFQ6
| IsqType::AFQ8
| IsqType::MXFP4 => {
// Use 1 because our HQQ quantizes on the GPU
Some(1.try_into().unwrap())
}
IsqType::F8E4M3 | IsqType::F8Q8 => None,
IsqType::Q2K
| IsqType::Q3K
| IsqType::Q4K
| IsqType::Q4_0
| IsqType::Q4_1
| IsqType::Q5K
| IsqType::Q5_0
| IsqType::Q5_1
| IsqType::Q6K
| IsqType::Q8K
| IsqType::Q8_0
| IsqType::Q8_1 => None,
}
}
}
impl TryFrom<IsqType> for GgmlDType {
type Error = hanzo_ml::Error;
fn try_from(value: IsqType) -> Result<Self> {
let tp = match value {
IsqType::Q2K => Self::Q2K,
IsqType::Q3K => Self::Q3K,
IsqType::Q4K => Self::Q4K,
IsqType::Q4_0 => Self::Q4_0,
IsqType::Q4_1 => Self::Q4_1,
IsqType::Q5K => Self::Q5K,
IsqType::Q5_0 => Self::Q5_0,
IsqType::Q5_1 => Self::Q5_1,
IsqType::Q6K => Self::Q6K,
IsqType::Q8K => Self::Q8K,
IsqType::Q8_0 => Self::Q8_0,
IsqType::Q8_1 => Self::Q8_1,
_ => hanzo_ml::bail!("Expected valid GGML ISQ type."),
};
#[cfg(feature = "cuda")]
{
if !matches!(
tp,
GgmlDType::Q4_0
| GgmlDType::Q4_1
| GgmlDType::Q5_0
| GgmlDType::Q5_1
| GgmlDType::Q8_0
| GgmlDType::Q2K
| GgmlDType::Q3K
| GgmlDType::Q4K
| GgmlDType::Q5K
| GgmlDType::Q6K
) {
hanzo_ml::bail!("GGML ISQ type on CUDA must be one of `Q4_0`, `Q4_1`, `Q5_0`, `Q5_1`, `Q8_0`, `Q2K`, `Q3K`, `Q4K`, `Q5K`, `Q6K`, `HQQ8`, `HQQ4`")
}
}
Ok(tp)
}
}
impl TryFrom<GgmlDType> for IsqType {
type Error = hanzo_ml::Error;
fn try_from(value: GgmlDType) -> Result<Self> {
match value {
GgmlDType::Q2K => Ok(Self::Q2K),
GgmlDType::Q3K => Ok(Self::Q3K),
GgmlDType::Q4K => Ok(Self::Q4K),
GgmlDType::Q5K => Ok(Self::Q5K),
GgmlDType::Q6K => Ok(Self::Q6K),
GgmlDType::Q4_0 => Ok(Self::Q4_0),
GgmlDType::Q4_1 => Ok(Self::Q4_1),
GgmlDType::Q5_0 => Ok(Self::Q5_0),
GgmlDType::Q5_1 => Ok(Self::Q5_1),
GgmlDType::Q8_0 => Ok(Self::Q8_0),
GgmlDType::Q8_1 => Ok(Self::Q8_1),
GgmlDType::Q8K => Ok(Self::Q8K),
GgmlDType::MXFP4 => Ok(Self::MXFP4),
GgmlDType::IQ4_NL
| GgmlDType::IQ4_XS
| GgmlDType::BF16
| GgmlDType::F32
| GgmlDType::F16
// IQ / ternary / 1-bit / FP4 codec types are decode-only, not ISQ targets.
| GgmlDType::ROCMFP4
| GgmlDType::ROCMFP4_FAST
| GgmlDType::IQ2_XXS
| GgmlDType::IQ2_XS
| GgmlDType::IQ3_XXS
| GgmlDType::IQ1_S
| GgmlDType::IQ3_S
| GgmlDType::IQ2_S
| GgmlDType::IQ1_M
| GgmlDType::TQ1_0
| GgmlDType::TQ2_0
| GgmlDType::NVFP4
| GgmlDType::Q1_0
| GgmlDType::PQ2_0
| GgmlDType::PTQ1_0
| GgmlDType::I32 => {
hanzo_ml::bail!("Expected valid GGML ISQ type.")
}
}
}
}
#[derive(Debug, Clone, Copy)]
pub enum QuantizedSerdeType {
Gguf = 0,
Unquant = 1,
Hqq = 2,
Fp8 = 3,
Afq = 4,
F8Q8 = 5,
Mxfp4 = 6,
}
impl TryFrom<usize> for QuantizedSerdeType {
type Error = hanzo_ml::Error;
fn try_from(value: usize) -> std::result::Result<Self, Self::Error> {
match value {
0 => Ok(Self::Gguf),
1 => Ok(Self::Unquant),
2 => Ok(Self::Hqq),
3 => Ok(Self::Fp8),
4 => Ok(Self::Afq),
5 => Ok(Self::F8Q8),
6 => Ok(Self::Mxfp4),
other => hanzo_ml::bail!("QuantizedSerdeType {other} is invalid."),
}
}
}
pub trait QuantizedSerde {
fn name(&self) -> &'static str;
fn isq_serde_supported(&self) -> bool {
false
}
fn serialize(&self) -> Result<Cow<'_, [u8]>> {
hanzo_ml::bail!("`QuantizedSerde::serialize` is not supported.")
}
fn deserialize(
_data: Cow<[u8]>,
_device: &Device,
_comm: &Arc<crate::Comm>,
_guard: QuantizeOntoGuard,
) -> Result<Arc<dyn QuantMethod>>
where
Self: Sized,
{
hanzo_ml::bail!("`QuantizedSerde::deserialize` is not supported.")
}
fn deserialize_ext_bias(
_data: Cow<[u8]>,
_device: &Device,
_guard: QuantizeOntoGuard,
) -> Result<(Arc<dyn QuantMethod>, Option<Tensor>)>
where
Self: Sized,
{
hanzo_ml::bail!("`QuantizedSerde::deserialize_ext_bias` is not supported.")
}
/// NOT meant for external calling
fn serialize_with_bias(&self, _bias: Option<Tensor>) -> Result<Cow<'_, [u8]>> {
hanzo_ml::bail!("`QuantizedSerde::serialize_with_bias` is not supported.")
}
}
/// Used to gate access to quantizing onto the host device
#[derive(Clone, Debug)]
#[allow(unused)]
pub struct QuantizeOntoGuard {
pub inner: Arc<Mutex<()>>,
}
/// Real (for Metal) and Fake (for CUDA)
pub enum QuantizeOntoDropGuard<'a> {
Real(MutexGuard<'a, ()>),
Fake,
}
impl Default for QuantizeOntoGuard {
fn default() -> Self {
Self::new()
}
}
impl QuantizeOntoGuard {
pub fn new() -> Self {
QuantizeOntoGuard {
inner: Arc::new(Mutex::new(())),
}
}
/// Acquire the quantize drop guard to protect the critical section.
///
/// On metal, this waits for outstanding work to finish to avoid "A command encoder is already encoding to this command buffer"
pub fn acquire(&self, device: &Device) -> QuantizeOntoDropGuard<'_> {
#[cfg(feature = "cuda")]
{
let _ = device;
QuantizeOntoDropGuard::Fake
}
#[cfg(not(feature = "cuda"))]
{
#[cfg(feature = "metal")]
if let Device::Metal(dev) = device {
// This is necessary to avoid the errors of "A command encoder is already encoding to this command buffer"
dev.wait_until_completed()
.expect("Failed to flush command buffer.");
}
#[cfg(not(feature = "metal"))]
let _ = device;
QuantizeOntoDropGuard::Real(self.inner.lock().expect("QuantizeOntoGuard was poisoned!"))
}
}
}
pub enum DistributedKind {
ColumnParallel,
RowParallel,
Replicated,
}
/// Quantized method for a quantized matmul.
pub trait QuantMethod: Send + Sync + Debug + QuantizedSerde {
fn new(method: QuantMethodConfig) -> Result<Self>
where
Self: Sized;
fn dequantize_w(&self) -> Result<Tensor>;
/// Compute matmul of `self` and `a`. `self` should contain the weights.
/// Automatically casts to the required quantization activation type and back.
fn forward(&self, a: &Tensor) -> Result<Tensor> {
if let Some(t) = self.quantized_act_type() {
let original_ty = a.dtype();
self.forward_raw(&a.to_dtype(t)?)?.to_dtype(original_ty)
} else {
self.forward_raw(a)
}
}
/// Raw matmul without dtype casting. Implementors override this.
/// Callers should use `forward` instead.
fn forward_raw(&self, a: &Tensor) -> Result<Tensor>;
/// Compute gather matmul of `self` and `a`. `self` should contain the weights.
/// Automatically casts to the required quantization activation type and back.
///
/// If `a` is (n_tokens, n_experts, cols), `self` weights are (n_experts, rows, cols),
/// then the indices are (n_tokens, n_experts).
fn gather_forward(&self, a: &Tensor, indices: &Tensor) -> Result<Tensor> {
if let Some(t) = self.quantized_act_type() {
let original_ty = a.dtype();
self.gather_forward_raw(&a.to_dtype(t)?, indices)?
.to_dtype(original_ty)
} else {
self.gather_forward_raw(a, indices)
}
}
/// Raw gather matmul without dtype casting. Implementors override this.
/// Callers should use `gather_forward` instead.
fn gather_forward_raw(&self, _a: &Tensor, _indices: &Tensor) -> Result<Tensor> {
hanzo_ml::bail!(
"{} does not support `gather_forward`. Please raise an issue.",
self.name()
)
}
/// Get the underlying QTensor if this is a GGUF quantized layer.
/// Used for direct kernel access in the grouped MoE prefill path.
#[cfg(feature = "cuda")]
fn get_qtensor(&self) -> Option<&hanzo_ml::quantized::QTensor> {
None
}
/// If this is an AFQ layer, return its (w_q, scales, biases, bits, group_size).
/// Used by Metal fused QKV / gate-up paths.
fn afq_inner(&self) -> Option<crate::afq::AfqInner<'_>> {
None
}
/// If a quantized method, return the activation dtype.
fn quantized_act_type(&self) -> Option<DType>;
/// Weight dtype and device
fn dtype_and_device(&self) -> (DType, Device);
/// Add a delta weight from LoRA to the weights. This should be prescaled with alpha.
fn add_delta_w(&self, delta: &Tensor) -> Result<Arc<dyn QuantMethod>>;
/// If the quant is backed by a qmatmul.
fn apply_isq(
self: Arc<Self>,
dtype: Option<IsqType>,
device: Device,
n_quantized: &AtomicUsize,
imatrix_weight: Option<Vec<f32>>,
guard: QuantizeOntoGuard,
) -> Result<Arc<dyn QuantMethod>>;
fn unquant_weight_bias(&self) -> Option<(Tensor, Option<Tensor>)> {
None
}
fn has_bias(&self) -> bool {
false
}
/// Begin tracking stats into an ImatrixLayerStats
fn begin_track_stats(&mut self) -> Result<()> {
hanzo_ml::bail!("`{}` does not support tracking stats.", self.name())
}
/// End tracking stats into an ImatrixLayerStats. Returns the computed imatrix.
fn end_track_stats(&self) -> Result<Tensor> {
hanzo_ml::bail!("`{}` does not support tracking stats.", self.name())
}
fn is_distributed(&self) -> Option<DistributedKind> {
None
}
fn dummy_info(&self) -> Option<&DummyLayerInfo> {
None
}
}
impl Module for dyn QuantMethod {
fn forward(&self, xs: &Tensor) -> Result<Tensor> {
QuantMethod::forward(self, xs)
}
}
#[cfg(feature = "cuda")]
pub fn try_fused_quantized_gate_up(
xs: &Tensor,
gate: &dyn QuantMethod,
up: &dyn QuantMethod,
activation: GluActivationType,
) -> Result<Option<Tensor>> {
if gate.has_bias() || up.has_bias() {
return Ok(None);
}
if !matches!(xs.dtype(), DType::BF16 | DType::F16 | DType::F32) {
return Ok(None);
}
let Some(gate_q) = gate.get_qtensor() else {
return Ok(None);
};
let Some(up_q) = up.get_qtensor() else {
return Ok(None);
};
let dtype = gate_q.dtype();
if dtype != up_q.dtype() || !gguf::fast_mmvq::supports(dtype) {
return Ok(None);
}
if gate_q.shape() != up_q.shape() {
return Ok(None);
}
let Some((&k, batch_dims)) = xs.dims().split_last() else {
return Ok(None);
};
let flat_batch = batch_dims.iter().product::<usize>();
if flat_batch == 0 || flat_batch > gguf::fast_mmvq::MMVQ_MAX_BATCH {
return Ok(None);
}
let (_, ncols) = gate_q.shape().dims2()?;
if k != ncols {
return Ok(None);
}
Ok(Some(gguf::fast_mmvq::fused_glu(
gate_q, up_q, xs, activation,
)?))
}
#[cfg(feature = "cuda")]
pub fn try_fused_quantized_qkv(
xs: &Tensor,
q: &dyn QuantMethod,
k: &dyn QuantMethod,
v: &dyn QuantMethod,
) -> Result<Option<(Tensor, Tensor, Tensor)>> {
if q.has_bias() || k.has_bias() || v.has_bias() {
return Ok(None);
}
if !matches!(xs.dtype(), DType::BF16 | DType::F16 | DType::F32) {
return Ok(None);
}
let Some(q_q) = q.get_qtensor() else {
return Ok(None);
};
let Some(k_q) = k.get_qtensor() else {
return Ok(None);
};
let Some(v_q) = v.get_qtensor() else {
return Ok(None);
};
let dtype = q_q.dtype();
if dtype != k_q.dtype() || dtype != v_q.dtype() || !gguf::fast_mmvq::supports(dtype) {
return Ok(None);
}
let Some((&input_cols, batch_dims)) = xs.dims().split_last() else {
return Ok(None);
};
let flat_batch = batch_dims.iter().product::<usize>();
if flat_batch == 0 || flat_batch > gguf::fast_mmvq::MMVQ_MAX_BATCH {
return Ok(None);
}
let (_, q_cols) = q_q.shape().dims2()?;
let (_, k_cols) = k_q.shape().dims2()?;
let (_, v_cols) = v_q.shape().dims2()?;
if input_cols != q_cols || input_cols != k_cols || input_cols != v_cols {
return Ok(None);
}
Ok(Some(gguf::fast_mmvq::fused_qkv(q_q, k_q, v_q, xs)?))
}
/// Metal fused gate+up: single Metal kernel that does both matmuls with shared
/// x reads and applies the GLU activation in-register before writing one output.
#[cfg(feature = "metal")]
pub fn try_fused_gate_up_metal(
xs: &Tensor,
gate: &dyn QuantMethod,
up: &dyn QuantMethod,
activation: GluActivationType,
) -> Result<Option<Tensor>> {
use hanzo_ml::{backend::BackendStorage, MetalStorage, Shape, Storage};
if gate.has_bias() || up.has_bias() {
return Ok(None);
}
if !matches!(xs.dtype(), DType::BF16 | DType::F16 | DType::F32) {
return Ok(None);
}
if !xs.device().is_metal() {
return Ok(None);
}
let Some(gi) = gate.afq_inner() else {
return Ok(None);
};
let Some(ui) = up.afq_inner() else {
return Ok(None);
};
if gi.bits != ui.bits || gi.group_size != ui.group_size {
return Ok(None);
}
if gi.scales.dtype() != ui.scales.dtype() {
return Ok(None);
}
if gi.w_q.rank() != 2 || ui.w_q.rank() != 2 {
return Ok(None);
}
let k = xs.dim(D::Minus1)?;
let n_gate = gi.w_q.dim(0)?;
let n_up = ui.w_q.dim(0)?;
if n_gate != n_up {
return Ok(None);
}
let n = n_gate;
// qmm_t kernel uses BM=32 tiles; for small M (decode) it wastes most of the
// tile. Let the caller fall back to separate qmv-based forwards.
let probe_m = xs.elem_count() / k;
if probe_m < 16 {
return Ok(None);
}
if k * gi.bits as usize / 8 / 4 != gi.w_q.dim(1)? {
// unexpected pack factor; let the generic path handle it
return Ok(None);
}
let act_code: u32 = match activation {
GluActivationType::Silu => 0,
GluActivationType::Gelu => 1,
GluActivationType::GeluErf => 2,
GluActivationType::Relu => 3,
};
let xs = xs.contiguous()?;
let m = xs.elem_count() / k;
if m == 0 {
return Ok(None);
}
let (xs_storage, xs_layout) = xs.storage_and_layout();
let Storage::Metal(xs_storage) = &*xs_storage else {
return Ok(None);
};
let (g_w_s, _) = gi.w_q.storage_and_layout();
let Storage::Metal(g_w_s) = &*g_w_s else {
return Ok(None);
};
let (g_s_s, _) = gi.scales.storage_and_layout();
let Storage::Metal(g_s_s) = &*g_s_s else {
return Ok(None);
};
let (g_b_s, _) = gi.biases.storage_and_layout();
let Storage::Metal(g_b_s) = &*g_b_s else {
return Ok(None);
};
let (u_w_s, _) = ui.w_q.storage_and_layout();
let Storage::Metal(u_w_s) = &*u_w_s else {
return Ok(None);
};
let (u_s_s, _) = ui.scales.storage_and_layout();
let Storage::Metal(u_s_s) = &*u_s_s else {
return Ok(None);
};
let (u_b_s, _) = ui.biases.storage_and_layout();
let Storage::Metal(u_b_s) = &*u_b_s else {
return Ok(None);
};
let device = xs_storage.device().clone();
let dtype = xs.dtype();
let mut out_shape = xs.dims().to_vec();
*out_shape.last_mut().unwrap() = n;
let out = device.new_buffer(out_shape.iter().product(), dtype, "afq-gate-up-out")?;
let encoder = device.command_encoder()?;
encoder.set_label("afq-gate-up");
metal_kernels::call_afq_qmm_gate_up(
device.device(),
&encoder,
&metal_kernels::Kernels::new(),
dtype,
(
xs_storage.buffer(),
xs_layout.start_offset() * dtype.size_in_bytes(),
),
g_w_s.buffer(),
g_s_s.buffer(),
g_b_s.buffer(),
u_w_s.buffer(),
u_s_s.buffer(),
u_b_s.buffer(),
&out,
m,
n,
k,
gi.bits as usize,
gi.group_size as usize,
act_code,
)
.map_err(hanzo_ml::Error::wrap)?;
let out_t = Tensor::from((
Storage::Metal(MetalStorage::new(
out,
device.clone(),
out_shape.iter().product(),
dtype,
)),
Shape::from(out_shape),
));
Ok(Some(out_t))
}
/// Metal fused QKV: single Metal kernel that handles all three projections,
/// routing per-tile to the right weight matrix.
#[cfg(feature = "metal")]
pub fn try_fused_qkv_metal(
xs: &Tensor,
q: &dyn QuantMethod,
k: &dyn QuantMethod,
v: &dyn QuantMethod,
) -> Result<Option<(Tensor, Tensor, Tensor)>> {
use hanzo_ml::{backend::BackendStorage, MetalStorage, Shape, Storage};
if q.has_bias() || k.has_bias() || v.has_bias() {
return Ok(None);
}
if !matches!(xs.dtype(), DType::BF16 | DType::F16 | DType::F32) {
return Ok(None);
}
if !xs.device().is_metal() {
return Ok(None);
}
let Some(qi) = q.afq_inner() else {
return Ok(None);
};
let Some(ki) = k.afq_inner() else {
return Ok(None);
};
let Some(vi) = v.afq_inner() else {
return Ok(None);
};
if qi.bits != ki.bits || qi.bits != vi.bits {
return Ok(None);
}
if qi.group_size != ki.group_size || qi.group_size != vi.group_size {
return Ok(None);
}
if qi.scales.dtype() != ki.scales.dtype() || qi.scales.dtype() != vi.scales.dtype() {
return Ok(None);
}
if qi.w_q.rank() != 2 || ki.w_q.rank() != 2 || vi.w_q.rank() != 2 {
return Ok(None);
}
let n_q = qi.w_q.dim(0)?;
let n_k = ki.w_q.dim(0)?;
let n_v = vi.w_q.dim(0)?;
// The kernel routes by tile-aligned column boundaries; require N_q and
// N_k to be multiples of the tile width (32). For Gemma-style models
// those are already 32-multiples; fall back when they're not.
if n_q % 32 != 0 || n_k % 32 != 0 || n_v % 32 != 0 {
return Ok(None);
}
let k_dim = xs.dim(D::Minus1)?;
// qmm_t kernel uses BM=32; for small M (decode) the tile is mostly empty.
// Fall back to separate qmv calls.
let probe_m = xs.elem_count() / k_dim;
if probe_m < 16 {
return Ok(None);
}
let xs = xs.contiguous()?;
let m = xs.elem_count() / k_dim;
if m == 0 {
return Ok(None);
}
let (xs_s, xs_l) = xs.storage_and_layout();
let Storage::Metal(xs_s) = &*xs_s else {
return Ok(None);
};
let qws = qi.w_q.storage_and_layout().0;
let qss = qi.scales.storage_and_layout().0;
let qbs = qi.biases.storage_and_layout().0;
let kws = ki.w_q.storage_and_layout().0;
let kss = ki.scales.storage_and_layout().0;
let kbs = ki.biases.storage_and_layout().0;
let vws = vi.w_q.storage_and_layout().0;
let vss = vi.scales.storage_and_layout().0;
let vbs = vi.biases.storage_and_layout().0;
let (Storage::Metal(qw_m), Storage::Metal(qs_m), Storage::Metal(qb_m)) = (&*qws, &*qss, &*qbs)
else {
return Ok(None);
};
let (Storage::Metal(kw_m), Storage::Metal(ks_m), Storage::Metal(kb_m)) = (&*kws, &*kss, &*kbs)
else {
return Ok(None);
};
let (Storage::Metal(vw_m), Storage::Metal(vs_m), Storage::Metal(vb_m)) = (&*vws, &*vss, &*vbs)
else {
return Ok(None);
};
let device = xs_s.device().clone();
let dtype = xs.dtype();
let mut q_shape = xs.dims().to_vec();
let mut k_shape = q_shape.clone();
let mut v_shape = q_shape.clone();
*q_shape.last_mut().unwrap() = n_q;
*k_shape.last_mut().unwrap() = n_k;
*v_shape.last_mut().unwrap() = n_v;
let q_out = device.new_buffer(q_shape.iter().product(), dtype, "afq-qkv-q")?;
let k_out = device.new_buffer(k_shape.iter().product(), dtype, "afq-qkv-k")?;
let v_out = device.new_buffer(v_shape.iter().product(), dtype, "afq-qkv-v")?;
let encoder = device.command_encoder()?;
encoder.set_label("afq-qkv");
metal_kernels::call_afq_qmm_qkv(
device.device(),
&encoder,
&metal_kernels::Kernels::new(),
dtype,
(xs_s.buffer(), xs_l.start_offset() * dtype.size_in_bytes()),
qw_m.buffer(),
qs_m.buffer(),
qb_m.buffer(),
kw_m.buffer(),
ks_m.buffer(),
kb_m.buffer(),
vw_m.buffer(),
vs_m.buffer(),
vb_m.buffer(),
&q_out,
&k_out,
&v_out,
m,
n_q,
n_k,
n_v,
k_dim,
qi.bits as usize,
qi.group_size as usize,
)
.map_err(hanzo_ml::Error::wrap)?;
let q_t = Tensor::from((
Storage::Metal(MetalStorage::new(
q_out,
device.clone(),
q_shape.iter().product(),
dtype,
)),
Shape::from(q_shape),
));
let k_t = Tensor::from((
Storage::Metal(MetalStorage::new(
k_out,
device.clone(),
k_shape.iter().product(),
dtype,
)),
Shape::from(k_shape),
));
let v_t = Tensor::from((
Storage::Metal(MetalStorage::new(
v_out,
device.clone(),
v_shape.iter().product(),
dtype,
)),
Shape::from(v_shape),
));
Ok(Some((q_t, k_t, v_t)))
}
fn tensor_prefix(vb: &ShardedVarBuilder) -> String {
let prefix = vb.prefix();
if prefix.is_empty() {
"<root>".to_string()
} else {
prefix
}
}
fn missing_required_tensors(vb: &ShardedVarBuilder, required: &[&str]) -> Vec<String> {
required
.iter()
.copied()
.filter(|name| !vb.contains_tensor(name))
.map(|name| safetensors::full_tensor_name(vb, name))
.collect()
}
pub(crate) fn has_missing_required_tensors(vb: &ShardedVarBuilder, required: &[&str]) -> bool {
required.iter().any(|name| !vb.contains_tensor(name))
}
pub(crate) fn make_dummy_or_error(
context: &str,
vb: &ShardedVarBuilder,
required: &[&str],
) -> Result<Arc<dyn QuantMethod>> {
let missing = missing_required_tensors(vb, required);
if missing.is_empty() {
hanzo_ml::bail!(
"Internal error: requested DummyLayer for {context} without missing tensors"
);
}
let has_uqff_placeholder = required
.iter()
.any(|name| safetensors::is_uqff_dummy_tensor(vb, name));
if !has_uqff_placeholder {
hanzo_ml::bail!(
"Missing required tensor(s) for {context} at prefix `{}`: {}. Dummy layers are only allowed for tensors intentionally omitted while loading UQFF artifacts.",
tensor_prefix(vb),
missing.join(", ")
);
}
Ok(Arc::new(DummyLayer::placeholder(DummyLayerInfo {
context: context.to_string(),
prefix: tensor_prefix(vb),
missing_tensors: missing,
})))
}
pub fn linear_no_bias(
in_dim: usize,
out_dim: usize,
config: &Option<QuantizedConfig>,
vb: ShardedVarBuilder,
) -> Result<Arc<dyn QuantMethod>> {
let base_vb = vb.clone();
let vb = if should_apply_immediate_isq(&vb) {
vb.set_device(Device::Cpu)
} else {
vb
};
let layer = if let Some(quant_conf) = &config {
match quant_conf {
QuantizedConfig::GptqAwq { .. } => gptq_linear(in_dim, out_dim, quant_conf, vb)?,
QuantizedConfig::Fp8 { weight_block_size } => {
if weight_block_size.is_some() {
blockwise_fp8_linear_b(
in_dim,
out_dim,
quant_conf,
false,
Default::default(),
vb,
)?
} else {
pertensor_fp8_linear_b(
in_dim,
out_dim,
quant_conf,
false,
Default::default(),
vb,
)?
}
}
QuantizedConfig::Bitsandbytes { .. } => {
Arc::new(BnbLinear::linear_b(in_dim, out_dim, false, vb)?) as Arc<_>
}
QuantizedConfig::Afq { .. } => {
AfqLayer::afq_linear_b(in_dim, out_dim, quant_conf, false, vb)?
}
QuantizedConfig::MXFP4 {} => {
MXFP4Layer::linear_b(in_dim, out_dim, quant_conf, false, vb)?
}
QuantizedConfig::ModelOpt {
quantized_layers, ..
} => distributed::layers::load_modelopt_linear(
in_dim,
out_dim,
false,
quant_conf,
quantized_layers,
&vb,
)?,
}
} else {
if !vb.contains_tensor("weight") {
make_dummy_or_error("linear_no_bias", &vb, &["weight"])?
} else {
let weight = vb.get_with_hints((out_dim, in_dim), "weight", Default::default())?;
let weight = merge_lora_weights(&vb, weight, in_dim, out_dim, Default::default())?;
let layer = <UnquantLinear as QuantMethod>::new(QuantMethodConfig::Unquantized(
Linear::new(weight, None),
))?;
Arc::new(layer) as Arc<dyn QuantMethod>
}
};
apply_immediate_isq(layer, base_vb)
}
pub fn linear(
in_dim: usize,
out_dim: usize,
config: &Option<QuantizedConfig>,
vb: ShardedVarBuilder,
) -> Result<Arc<dyn QuantMethod>> {
let base_vb = vb.clone();
let vb = if should_apply_immediate_isq(&vb) {
vb.set_device(Device::Cpu)
} else {
vb
};
let layer = if let Some(quant_conf) = &config {
match quant_conf {
QuantizedConfig::GptqAwq { .. } => gptq_linear(in_dim, out_dim, quant_conf, vb)?,
QuantizedConfig::Fp8 { weight_block_size } => {
if weight_block_size.is_some() {
blockwise_fp8_linear_b(
in_dim,
out_dim,
quant_conf,
true,
Default::default(),
vb,
)?
} else {
pertensor_fp8_linear_b(
in_dim,
out_dim,
quant_conf,
true,
Default::default(),
vb,
)?
}
}
QuantizedConfig::Bitsandbytes { .. } => {
Arc::new(BnbLinear::linear_b(in_dim, out_dim, true, vb)?) as Arc<_>
}
QuantizedConfig::Afq { .. } => {
AfqLayer::afq_linear_b(in_dim, out_dim, quant_conf, true, vb)?
}
QuantizedConfig::MXFP4 {} => {
MXFP4Layer::linear_b(in_dim, out_dim, quant_conf, true, vb)?
}
QuantizedConfig::ModelOpt {
quantized_layers, ..
} => distributed::layers::load_modelopt_linear(
in_dim,
out_dim,
true,
quant_conf,
quantized_layers,
&vb,
)?,
}
} else {
if has_missing_required_tensors(&vb, &["weight", "bias"]) {
make_dummy_or_error("linear", &vb, &["weight", "bias"])?
} else {
let weight = vb.get_with_hints((out_dim, in_dim), "weight", Default::default())?;
let weight = merge_lora_weights(&vb, weight, in_dim, out_dim, Default::default())?;
let bias = vb.get_with_hints((out_dim,), "bias", Default::default())?;
let layer = <UnquantLinear as QuantMethod>::new(QuantMethodConfig::Unquantized(
Linear::new(weight, Some(bias)),
))?;
Arc::new(layer) as Arc<dyn QuantMethod>
}
};
apply_immediate_isq(layer, base_vb)
}
pub fn linear_b(
in_dim: usize,
out_dim: usize,
bias: bool,
config: &Option<QuantizedConfig>,
vb: ShardedVarBuilder,
) -> Result<Arc<dyn QuantMethod>> {
if bias {
linear(in_dim, out_dim, config, vb)
} else {
linear_no_bias(in_dim, out_dim, config, vb)
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use super::*;
fn empty_vb(make_dummy_regexes: Option<Vec<&str>>) -> ShardedVarBuilder {
let backend: HashMap<String, Tensor> = HashMap::new();
let make_dummy_regexes = make_dummy_regexes.map(|regexes| {
Arc::new(
regexes
.into_iter()
.map(Regex::new)
.collect::<std::result::Result<Vec<_>, _>>()
.unwrap(),
)
});
ShardedSafeTensors::wrap_with_dummy_regexes(
Box::new(backend),
DType::F32,
Device::Cpu,
make_dummy_regexes,
)
}
#[test]
fn missing_linear_weight_outside_uqff_errors() {
let err = linear_no_bias(2, 3, &None, empty_vb(None).pp("foo")).unwrap_err();
let msg = err.to_string();
assert!(msg.contains("Missing required tensor(s)"));
assert!(msg.contains("foo.weight"));
assert!(msg.contains("UQFF"));
}
#[test]
fn missing_uqff_placeholder_creates_contextual_dummy() -> Result<()> {
let layer = linear_no_bias(
2,
3,
&None,
empty_vb(Some(vec![r"^foo\.weight$"])).pp("foo"),
)?;
let info = layer.dummy_info().unwrap();
assert_eq!(layer.name(), "dummy");
assert_eq!(info.context, "linear_no_bias");
assert_eq!(info.prefix, "foo");
assert_eq!(info.missing_tensors, vec!["foo.weight"]);
let input = Tensor::zeros((1, 2), DType::F32, &Device::Cpu)?;
let err = layer.forward_raw(&input).unwrap_err();
let msg = err.to_string();
assert!(msg.contains("forward pass"));
assert!(msg.contains("foo.weight"));
assert!(msg.contains("temporary UQFF placeholders"));
Ok(())
}
}
#[cfg(test)]
mod afq_recipe_tests {
use super::*;
// Entries copied from the MTPLX pack of Qwen3.8-Flash-Next (config.json `quantization`):
// 4-bit g64 by default, routers and the indexer at 8 bits, hyper-connections unquantized.
const RECIPE: &str = r#"{
"bits": 4, "group_size": 64, "mode": "affine",
"language_model.lm_head": {"bits": 4, "group_size": 64, "mode": "affine"},
"language_model.model.layers.0.mlp.gate": {"bits": 8, "group_size": 64, "mode": "affine"},
"language_model.model.layers.11.self_attn.indexer.index_qk_proj": {"bits": 8, "group_size": 64, "mode": "affine"},
"language_model.model.layers.0.attn_hyper_connection.input_mix_weight_down": false
}"#;
#[test]
fn mlx_recipe_resolves_per_module() {
let config: QuantizedConfig = serde_json::from_str(RECIPE).unwrap();
let at = |prefix: &str| config.afq_at(prefix);
assert_eq!(at("language_model.lm_head"), Some((4, 64)));
assert_eq!(at("language_model.model.layers.0.mlp.gate"), Some((8, 64)));
assert_eq!(
at("language_model.model.layers.11.self_attn.indexer.index_qk_proj"),
Some((8, 64))
);
assert_eq!(
at("language_model.model.layers.0.attn_hyper_connection.input_mix_weight_down"),
None
);
// A module the recipe does not list takes the defaults.
assert_eq!(
at("language_model.model.layers.3.mlp.switch_mlp.gate_proj"),
Some((4, 64))
);
// The same path nested under another root matches on a `.` boundary; a bare suffix does not.
assert_eq!(
at("model.language_model.model.layers.0.mlp.gate"),
Some((8, 64))
);
assert_eq!(at("layers.10.mlp.gate"), Some((4, 64)));
}
#[test]
fn mlx_recipe_refuses_other_modes() {
let recipe = r#"{"bits": 4, "group_size": 32, "a.b": {"bits": 4, "group_size": 32, "mode": "mxfp4"}}"#;
assert!(serde_json::from_str::<QuantizedConfig>(recipe).is_err());
let recipe = r#"{"bits": 4, "group_size": 32, "mode": "nvfp4"}"#;
assert!(serde_json::from_str::<QuantizedConfig>(recipe).is_err());
}
}