mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-21 02:36:58 +00:00
1987 lines
64 KiB
Rust
1987 lines
64 KiB
Rust
use std::{
|
||
borrow::Cow,
|
||
fmt::Debug,
|
||
num::NonZeroUsize,
|
||
sync::{atomic::AtomicUsize, Arc, Condvar, Mutex, MutexGuard},
|
||
};
|
||
|
||
use blockwise_fp8::blockwise_fp8_linear_b;
|
||
#[cfg(feature = "metal")]
|
||
use hanzo_ml::D;
|
||
use hanzo_ml::{
|
||
quantized::{GgmlDType, QMatMul, QTensor},
|
||
DType, Device, Result, Tensor,
|
||
};
|
||
use pertensor_fp8::pertensor_fp8_linear_b;
|
||
|
||
#[cfg(feature = "metal")]
|
||
pub mod metal_kernels;
|
||
|
||
mod afq;
|
||
mod bitsandbytes;
|
||
pub mod blockwise_fp8;
|
||
pub mod cublaslt;
|
||
#[cfg(all(feature = "cuda", feature = "cutile"))]
|
||
pub mod cutile;
|
||
pub mod distributed;
|
||
mod dummy;
|
||
#[cfg(all(feature = "cuda", has_nvfp4_cutlass_kernels))]
|
||
pub mod experts;
|
||
pub mod f8q8;
|
||
mod fp8;
|
||
pub mod gemv;
|
||
mod gguf;
|
||
pub mod gguf_export;
|
||
mod gptq;
|
||
mod hqq;
|
||
mod imatrix;
|
||
mod lora;
|
||
#[cfg(all(feature = "cuda", feature = "cutile"))]
|
||
pub mod moe;
|
||
mod mxfp4;
|
||
pub mod nvfp4;
|
||
mod pending_layer;
|
||
mod pertensor_fp8;
|
||
pub mod quantize;
|
||
pub mod rotary;
|
||
pub mod safetensors;
|
||
mod scalar_fp8;
|
||
mod unquantized;
|
||
mod utils;
|
||
mod vector_fp8;
|
||
|
||
pub use gguf_export::{export_gguf, GgufOutputType};
|
||
use gptq::gptq_linear;
|
||
use lora::merge_lora_weights;
|
||
use regex::Regex;
|
||
pub use safetensors::{Shard, ShardedSafeTensors, ShardedVarBuilder};
|
||
|
||
#[cfg(feature = "metal")]
|
||
pub use afq::ops::{
|
||
afq_gather_qmm_rhs_sorted, afq_gather_qmm_rhs_sorted_gate_up, metal_arg_sort_u32_1d,
|
||
metal_moe_weighted_reduce_flat,
|
||
};
|
||
pub use afq::{AfqBits, AfqGroupSize, AfqInner, AfqLayer};
|
||
pub use bitsandbytes::{BnbLinear, BnbQuantParams, BnbQuantType};
|
||
pub use blockwise_fp8::{
|
||
blockwise_fp8_act, blockwise_fp8_moe, fp8_blockwise_dequantize, fp8_blockwise_quantize,
|
||
BlockwiseFP8Linear,
|
||
};
|
||
pub use distributed::{
|
||
layers::{
|
||
compute_kv_shard, compute_n_kv_groups, ColumnParallelLayer, FusedExperts, PackedExperts,
|
||
ReplicatedLayer, RowParallelLayer,
|
||
},
|
||
pipeline::{broadcast_head_tokens, PpHeader, RingPipeline, PP_OP_FORWARD, PP_OP_TERMINATE},
|
||
socket::{Client, Server},
|
||
BarrierLike, Comm, Id, RingConfig, SumAllReduce,
|
||
};
|
||
pub use dummy::{DummyLayer, DummyLayerInfo};
|
||
pub use f8q8::F8Q8Linear;
|
||
pub use fp8::FP8Linear;
|
||
#[cfg(feature = "cuda")]
|
||
pub use gemv::gemv;
|
||
pub use gemv::{should_use_gemv, GEMV_CONTROLLER};
|
||
#[cfg(feature = "cuda")]
|
||
pub use gguf::cuda::{
|
||
grouped_moe_gemm_prequantized, indexed_moe_fused_decode, moe_dispatch_build,
|
||
moe_weighted_reduce_flat, moe_weighted_reduce_flat_bf16, quantize_input_q8_1,
|
||
ACT_GELU_PYTORCH_TANH, ACT_SILU,
|
||
};
|
||
#[cfg(feature = "cuda")]
|
||
pub use gguf::fast_mmq::{
|
||
grouped as grouped_moe_mmq, grouped_from_glu_pair as grouped_moe_mmq_from_glu_pair,
|
||
grouped_pair as grouped_moe_mmq_pair, supports as supports_mmq,
|
||
};
|
||
pub use gguf::GgufMatMul;
|
||
pub use gptq::GptqLayer;
|
||
pub use hqq::{HqqAxis, HqqBits, HqqConfig, HqqLayer};
|
||
pub use imatrix::{CollectedImatrixData, ImatrixLayerStats};
|
||
pub use lora::{
|
||
clear_applied_loras, get_applied_loras, linear_no_bias_static_lora, push_applied_lora,
|
||
LoraAdapter, LoraConfig, StaticLoraConfig, MULTI_LORA_DELIMITER,
|
||
};
|
||
pub use mxfp4::MXFP4Layer;
|
||
pub use nvfp4::NVFP4Layer;
|
||
pub use pending_layer::PendingIsqLayer;
|
||
pub use pertensor_fp8::PerTensorFP8Linear;
|
||
pub use unquantized::UnquantLinear;
|
||
pub use utils::flash_attn_sinks_metal;
|
||
pub use utils::flash_attn_sinks_varlen_metal;
|
||
#[cfg(feature = "cuda")]
|
||
pub use utils::gptoss_swiglu_fused;
|
||
#[cfg(feature = "cuda")]
|
||
pub use utils::gptoss_swiglu_interleaved;
|
||
pub use utils::isq::apply_immediate_isq;
|
||
pub use utils::softcap;
|
||
pub use utils::softmax_with_sinks;
|
||
pub use utils::{fused_glu, GluActivationType};
|
||
pub use utils::{log, BitWiseOp, CumSumOp, LeftshiftOp, NonZeroOp, SortOp, UQFF_QUANT_TYPE_OFFSET};
|
||
pub use vector_fp8::{fp8_vector_dequantize, fp8_vector_quantize};
|
||
|
||
use hanzo_nn::{Conv1d, Conv2d, Linear, Module};
|
||
use serde::{Deserialize, Deserializer, Serialize};
|
||
|
||
/// Limits outstanding async ISQ jobs to prevent unbounded memory growth.
|
||
///
|
||
/// Without backpressure, MoE models (e.g. Gemma4 with 128 experts × 30 layers)
|
||
/// queue BF16 tensor data in the rayon pool faster than the pool can quantize,
|
||
/// causing OOM on memory-constrained systems like macOS Metal with unified memory.
|
||
pub struct IsqBackpressure {
|
||
count: Mutex<usize>,
|
||
cvar: Condvar,
|
||
max: usize,
|
||
}
|
||
|
||
impl IsqBackpressure {
|
||
pub fn new(max: usize) -> Self {
|
||
Self {
|
||
count: Mutex::new(0),
|
||
cvar: Condvar::new(),
|
||
max,
|
||
}
|
||
}
|
||
|
||
/// Block until a slot is available, then increment the outstanding count.
|
||
pub fn acquire(&self) {
|
||
let mut count = self.count.lock().expect("ISQ backpressure lock poisoned");
|
||
while *count >= self.max {
|
||
count = self
|
||
.cvar
|
||
.wait(count)
|
||
.expect("ISQ backpressure lock poisoned");
|
||
}
|
||
*count += 1;
|
||
}
|
||
|
||
/// Decrement the outstanding count and wake a blocked loader thread.
|
||
pub fn release(&self) {
|
||
let mut count = self.count.lock().expect("ISQ backpressure lock poisoned");
|
||
*count = count.saturating_sub(1);
|
||
self.cvar.notify_one();
|
||
}
|
||
}
|
||
|
||
impl Debug for IsqBackpressure {
|
||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
let count = self.count.lock().map(|c| *c).unwrap_or(0);
|
||
f.debug_struct("IsqBackpressure")
|
||
.field("outstanding", &count)
|
||
.field("max", &self.max)
|
||
.finish()
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub struct ImmediateIsqParams {
|
||
pub guard: QuantizeOntoGuard,
|
||
pub ty: Option<IsqType>,
|
||
pub predicates: Vec<Regex>,
|
||
pub overrides: Vec<ImmediateIsqOverride>,
|
||
/// Thread pool for parallel immediate ISQ on discrete GPUs.
|
||
/// When `Some`, `apply_immediate_isq` will spawn quantization tasks
|
||
/// on this pool and return `PendingIsqLayer` wrappers.
|
||
pub pool: Option<Arc<rayon::ThreadPool>>,
|
||
/// Backpressure to limit outstanding async ISQ jobs.
|
||
pub backpressure: Arc<IsqBackpressure>,
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub struct ImmediateIsqOverride {
|
||
pub predicate: Regex,
|
||
pub ty: Option<IsqType>,
|
||
pub device: Option<Device>,
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub struct ImmediateIsqMatch {
|
||
pub ty: IsqType,
|
||
pub device: Option<Device>,
|
||
}
|
||
|
||
thread_local! {
|
||
static ENGINE_IMMEDIATE_ISQ: std::cell::RefCell<Option<ImmediateIsqParams>> = const { std::cell::RefCell::new(None) } ;
|
||
}
|
||
|
||
pub fn set_immediate_isq(isq: Option<IsqType>, predicates: Vec<Regex>) {
|
||
let (pool, _) = create_isq_thread_pool(isq);
|
||
set_immediate_isq_with_pool(isq, predicates, Vec::new(), pool);
|
||
}
|
||
|
||
pub fn set_immediate_isq_with_pool(
|
||
isq: Option<IsqType>,
|
||
predicates: Vec<Regex>,
|
||
overrides: Vec<ImmediateIsqOverride>,
|
||
pool: rayon::ThreadPool,
|
||
) {
|
||
// Allow pool threads + 1 outstanding jobs: enough for pipeline overlap
|
||
// (load next tensor while pool quantizes current) without unbounded growth.
|
||
let max_outstanding = pool.current_num_threads() + 1;
|
||
ENGINE_IMMEDIATE_ISQ.with(|cell| {
|
||
*cell.borrow_mut() = Some(ImmediateIsqParams {
|
||
guard: QuantizeOntoGuard::new(),
|
||
ty: isq,
|
||
predicates,
|
||
overrides,
|
||
backpressure: Arc::new(IsqBackpressure::new(max_outstanding)),
|
||
pool: Some(Arc::new(pool)),
|
||
});
|
||
});
|
||
}
|
||
|
||
/// Create a rayon thread pool for parallel immediate ISQ.
|
||
/// Returns `(pool, num_threads)` so callers can log the thread count.
|
||
///
|
||
/// Thread count is based on the quantization type:
|
||
/// - GGML types (Q2K-Q8K) and F8E4M3: `rayon::current_num_threads()` (CPU quantization)
|
||
/// - HQQ/AFQ: 1 thread (GPU quantization, serialized by `QuantizeOntoGuard`)
|
||
pub fn create_isq_thread_pool(ty: Option<IsqType>) -> (rayon::ThreadPool, usize) {
|
||
let num_threads = if std::env::var("ISQ_SINGLETHREAD").is_ok() {
|
||
1
|
||
} else if let Some(ty) = ty {
|
||
ty.get_max_isq_cpu_threads()
|
||
.map(usize::from)
|
||
.unwrap_or_else(rayon::current_num_threads)
|
||
} else {
|
||
rayon::current_num_threads()
|
||
};
|
||
|
||
let pool = rayon::ThreadPoolBuilder::new()
|
||
.num_threads(num_threads)
|
||
.build()
|
||
.expect("Failed to create ISQ thread pool");
|
||
(pool, num_threads)
|
||
}
|
||
|
||
pub fn get_immediate_isq() -> Option<ImmediateIsqParams> {
|
||
ENGINE_IMMEDIATE_ISQ.with(|cell| cell.borrow().clone())
|
||
}
|
||
|
||
pub fn clear_immediate_isq() {
|
||
ENGINE_IMMEDIATE_ISQ.with(|cell| {
|
||
*cell.borrow_mut() = None;
|
||
});
|
||
}
|
||
|
||
pub fn should_apply_immediate_isq(vb: &ShardedVarBuilder) -> bool {
|
||
immediate_isq_match(vb).is_some()
|
||
}
|
||
|
||
pub fn immediate_isq_match(vb: &ShardedVarBuilder) -> Option<ImmediateIsqMatch> {
|
||
let immediate_isq = get_immediate_isq()?;
|
||
// Add a .weight to match the ISQ regexes!
|
||
let prefix = format!("{}.weight", vb.prefix());
|
||
resolve_immediate_isq(&immediate_isq, &prefix)
|
||
}
|
||
|
||
fn resolve_immediate_isq(params: &ImmediateIsqParams, prefix: &str) -> Option<ImmediateIsqMatch> {
|
||
if let Some(override_hit) = params
|
||
.overrides
|
||
.iter()
|
||
.find(|override_pred| override_pred.predicate.is_match(prefix))
|
||
{
|
||
if let Some(ty) = override_hit.ty.or(params.ty) {
|
||
return Some(ImmediateIsqMatch {
|
||
ty,
|
||
device: override_hit.device.clone(),
|
||
});
|
||
}
|
||
return None;
|
||
}
|
||
|
||
if let Some(ty) = params.ty {
|
||
if params
|
||
.predicates
|
||
.iter()
|
||
.any(|predicate| predicate.is_match(prefix))
|
||
{
|
||
return Some(ImmediateIsqMatch { ty, device: None });
|
||
}
|
||
}
|
||
|
||
None
|
||
}
|
||
|
||
#[derive(Debug, Clone, Serialize)]
|
||
#[serde(tag = "quant_method", rename_all = "lowercase")]
|
||
pub enum QuantizedConfig {
|
||
GptqAwq {
|
||
bits: usize,
|
||
group_size: usize,
|
||
checkpoint_format: Option<String>,
|
||
is_awq: bool,
|
||
},
|
||
Fp8 {
|
||
weight_block_size: Option<Vec<usize>>,
|
||
},
|
||
Bitsandbytes {
|
||
bnb_4bit_quant_type: Option<String>,
|
||
},
|
||
Afq {
|
||
bits: usize,
|
||
group_size: usize,
|
||
/// MLX's per-module entries, by module path: its own bits and group size, or left
|
||
/// unquantized.
|
||
#[serde(skip_serializing_if = "std::collections::HashMap::is_empty")]
|
||
modules: std::collections::HashMap<String, AfqModule>,
|
||
},
|
||
MXFP4 {},
|
||
ModelOpt {
|
||
quant_algo: Option<String>,
|
||
quantized_layers: std::collections::HashMap<String, ModelOptLayerConfig>,
|
||
},
|
||
}
|
||
|
||
/// One module's entry in an MLX `quantization` block.
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(untagged)]
|
||
pub enum AfqModule {
|
||
/// `false`: the module stays unquantized. (`true` means the block's defaults.)
|
||
Quantized(bool),
|
||
/// The module's own affine recipe.
|
||
Recipe { bits: usize, group_size: usize },
|
||
}
|
||
|
||
/// The entry for the module at `prefix` in a per-module recipe keyed by module path: the exact
|
||
/// path, or one path ending the other at a `.` boundary (checkpoints nest the same modules under
|
||
/// different roots).
|
||
pub(crate) fn recipe_entry<'a, V>(
|
||
recipe: &'a std::collections::HashMap<String, V>,
|
||
prefix: &str,
|
||
) -> Option<&'a V> {
|
||
recipe.get(prefix).or_else(|| {
|
||
recipe.iter().find_map(|(k, v)| {
|
||
(k.ends_with(&format!(".{prefix}")) || prefix.ends_with(&format!(".{k}"))).then_some(v)
|
||
})
|
||
})
|
||
}
|
||
|
||
impl QuantizedConfig {
|
||
/// The AFQ (bits, group size) for the module at `prefix`: its own entry, else the block's
|
||
/// defaults; `None` when the recipe leaves it unquantized or the config is not AFQ.
|
||
pub fn afq_at(&self, prefix: &str) -> Option<(usize, usize)> {
|
||
let QuantizedConfig::Afq {
|
||
bits,
|
||
group_size,
|
||
modules,
|
||
} = self
|
||
else {
|
||
return None;
|
||
};
|
||
match recipe_entry(modules, prefix) {
|
||
Some(AfqModule::Quantized(false)) => None,
|
||
Some(AfqModule::Recipe { bits, group_size }) => Some((*bits, *group_size)),
|
||
Some(AfqModule::Quantized(true)) | None => Some((*bits, *group_size)),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// MLX's per-module entries among a quantization block's keys: booleans and objects carrying
|
||
/// `bits`. Only the affine mode exists in AFQ.
|
||
fn afq_modules<E: serde::de::Error>(
|
||
rest: std::collections::HashMap<String, serde_json::Value>,
|
||
) -> std::result::Result<std::collections::HashMap<String, AfqModule>, E> {
|
||
let mut modules = std::collections::HashMap::new();
|
||
for (key, value) in rest {
|
||
let mode = match &value {
|
||
serde_json::Value::Bool(_) => None,
|
||
serde_json::Value::Object(o) if o.contains_key("bits") => o.get("mode"),
|
||
serde_json::Value::String(_) if key == "mode" => Some(&value),
|
||
_ => continue,
|
||
};
|
||
if let Some(mode) = mode.and_then(|m| m.as_str()).filter(|m| *m != "affine") {
|
||
return Err(E::custom(format!(
|
||
"AFQ supports the affine mode only; {key} asks for {mode}"
|
||
)));
|
||
}
|
||
if key != "mode" {
|
||
let module = serde_json::from_value(value).map_err(E::custom)?;
|
||
modules.insert(key, module);
|
||
}
|
||
}
|
||
Ok(modules)
|
||
}
|
||
|
||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||
pub struct ModelOptLayerConfig {
|
||
#[serde(default)]
|
||
pub quant_algo: Option<String>,
|
||
#[serde(default)]
|
||
pub group_size: Option<usize>,
|
||
}
|
||
|
||
// Common fields for all variants
|
||
#[derive(Deserialize)]
|
||
struct RawConfig {
|
||
quant_method: Option<String>,
|
||
quant_algo: Option<String>,
|
||
quantized_layers: Option<std::collections::HashMap<String, ModelOptLayerConfig>>,
|
||
bits: Option<usize>,
|
||
group_size: Option<usize>,
|
||
checkpoint_format: Option<String>,
|
||
weight_block_size: Option<Vec<usize>>,
|
||
bnb_4bit_quant_type: Option<String>,
|
||
#[serde(flatten)]
|
||
rest: std::collections::HashMap<String, serde_json::Value>,
|
||
}
|
||
|
||
// Custom deserializer implementation
|
||
impl<'de> Deserialize<'de> for QuantizedConfig {
|
||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||
where
|
||
D: Deserializer<'de>,
|
||
{
|
||
let raw = RawConfig::deserialize(deserializer)?;
|
||
|
||
match &raw.quant_method {
|
||
Some(m) if m == "gptq" || m == "awq" => {
|
||
let bits = raw
|
||
.bits
|
||
.ok_or_else(|| serde::de::Error::missing_field("bits"))?;
|
||
let group_size = raw
|
||
.group_size
|
||
.ok_or_else(|| serde::de::Error::missing_field("group_size"))?;
|
||
Ok(QuantizedConfig::GptqAwq {
|
||
bits,
|
||
group_size,
|
||
checkpoint_format: raw.checkpoint_format,
|
||
is_awq: m == "awq",
|
||
})
|
||
}
|
||
Some(m) if m == "fp8" => {
|
||
// weight_block_size is optional - None means per-tensor quantization
|
||
Ok(QuantizedConfig::Fp8 {
|
||
weight_block_size: raw.weight_block_size,
|
||
})
|
||
}
|
||
Some(m) if m == "bitsandbytes" => Ok(QuantizedConfig::Bitsandbytes {
|
||
bnb_4bit_quant_type: raw.bnb_4bit_quant_type,
|
||
}),
|
||
Some(m) if m == "afq" => {
|
||
let bits = raw
|
||
.bits
|
||
.ok_or_else(|| serde::de::Error::missing_field("bits"))?;
|
||
let group_size = raw
|
||
.group_size
|
||
.ok_or_else(|| serde::de::Error::missing_field("group_size"))?;
|
||
Ok(QuantizedConfig::Afq {
|
||
bits,
|
||
group_size,
|
||
modules: afq_modules(raw.rest)?,
|
||
})
|
||
}
|
||
Some(m) if m == "mxfp4" => {
|
||
Ok(QuantizedConfig::MXFP4 { })
|
||
}
|
||
Some(m) if m == "modelopt" => {
|
||
Ok(QuantizedConfig::ModelOpt {
|
||
quant_algo: raw.quant_algo,
|
||
quantized_layers: raw.quantized_layers.unwrap_or_default(),
|
||
})
|
||
}
|
||
None => {
|
||
let bits = raw
|
||
.bits
|
||
.ok_or_else(|| serde::de::Error::missing_field("bits"))?;
|
||
let group_size = raw
|
||
.group_size
|
||
.ok_or_else(|| serde::de::Error::missing_field("group_size"))?;
|
||
Ok(QuantizedConfig::Afq {
|
||
bits,
|
||
group_size,
|
||
modules: afq_modules(raw.rest)?,
|
||
})
|
||
}
|
||
Some(unknown_method) => {
|
||
Err(serde::de::Error::custom(format!(
|
||
"Unknown quantization method: {unknown_method}. Expected one of: gptq, fp8, bitsandbytes, afq, mxfp4, modelopt, or not specified"
|
||
)))
|
||
},
|
||
}
|
||
}
|
||
}
|
||
|
||
impl QuantizedConfig {
|
||
pub fn name(&self) -> &'static str {
|
||
match self {
|
||
Self::GptqAwq { .. } => "gptq",
|
||
Self::Fp8 { .. } => "fp8",
|
||
Self::Bitsandbytes { .. } => "bitsandbytes",
|
||
Self::Afq { .. } => "afq",
|
||
Self::MXFP4 { .. } => "mxfp4",
|
||
Self::ModelOpt { .. } => "modelopt",
|
||
}
|
||
}
|
||
|
||
pub fn get_bits_name(&self, vb: &ShardedVarBuilder) -> String {
|
||
match self {
|
||
Self::GptqAwq { bits, .. } => format!("{bits} bits"),
|
||
Self::Fp8 { .. } => "8 bits".to_string(),
|
||
Self::Bitsandbytes {
|
||
bnb_4bit_quant_type: Some(_),
|
||
} => "4 bits".to_string(),
|
||
Self::Bitsandbytes {
|
||
bnb_4bit_quant_type: None,
|
||
} => "8 bits".to_string(),
|
||
Self::Afq { bits, .. } => format!("{bits} bits"),
|
||
Self::MXFP4 {} => format!("{} bits", mxfp4::N_BITS),
|
||
Self::ModelOpt {
|
||
quantized_layers, ..
|
||
} => {
|
||
let prefix = vb.prefix();
|
||
if let Some(cfg) = quantized_layers.iter().find_map(|(k, v)| {
|
||
if k == &prefix || k.ends_with(&format!(".{prefix}")) || prefix.ends_with(k) {
|
||
Some(v)
|
||
} else {
|
||
None
|
||
}
|
||
}) {
|
||
match cfg.quant_algo.as_deref() {
|
||
Some("NVFP4") => "4 bits (NVFP4)".to_string(),
|
||
Some("FP8") => "8 bits (FP8)".to_string(),
|
||
_ => "mixed precision".to_string(),
|
||
}
|
||
} else {
|
||
"16 bits".to_string()
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
pub fn pack_factor(&self, dtype: DType) -> usize {
|
||
match self {
|
||
Self::GptqAwq { bits, .. } | Self::Afq { bits, .. } => match bits {
|
||
2 => IsqType::Q2K.pack_factor(dtype),
|
||
3 => IsqType::Q3K.pack_factor(dtype),
|
||
4 => IsqType::Q4K.pack_factor(dtype),
|
||
5 => IsqType::Q5K.pack_factor(dtype),
|
||
6 => IsqType::Q6K.pack_factor(dtype),
|
||
8 => IsqType::Q8_0.pack_factor(dtype),
|
||
40 => 4, // mxfp4: 2 FP4 values per byte = factor of 4
|
||
other => panic!("Unexpected bits in `pack_factor` {other}"),
|
||
},
|
||
Self::Fp8 { .. } => IsqType::Q8_0.pack_factor(dtype),
|
||
Self::Bitsandbytes {
|
||
bnb_4bit_quant_type: Some(_),
|
||
}
|
||
| Self::Bitsandbytes {
|
||
bnb_4bit_quant_type: None,
|
||
} => IsqType::Q4K.pack_factor(dtype),
|
||
Self::MXFP4 {} => IsqType::Q4_0.pack_factor(dtype),
|
||
Self::ModelOpt { .. } => 4,
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Clone)]
|
||
pub enum QuantMethodConfig {
|
||
GptqAwq {
|
||
bits: i32,
|
||
/// The weights are in the shuffled layout (rows sequential, packed values reordered).
|
||
is_shuffled: bool,
|
||
q_weight: Tensor,
|
||
qzeros: Option<Tensor>,
|
||
scales: Tensor,
|
||
g_idx: Option<Tensor>,
|
||
bias: Option<Tensor>,
|
||
workspace: Option<Tensor>,
|
||
is_marlin: bool,
|
||
is_awq: bool,
|
||
},
|
||
Gguf {
|
||
q_weight: Arc<QTensor>,
|
||
b: Option<Tensor>,
|
||
},
|
||
Unquantized(Linear),
|
||
Hqq {
|
||
tensor: Tensor,
|
||
bits: HqqBits,
|
||
group_size: NonZeroUsize,
|
||
axis: HqqAxis,
|
||
optimization_steps: Option<usize>,
|
||
round_zeros: Option<bool>,
|
||
channel_wise: Option<bool>,
|
||
bias: Option<Tensor>,
|
||
},
|
||
Dummy,
|
||
FP8 {
|
||
lin: Linear,
|
||
dtype: DType,
|
||
},
|
||
Bnb {
|
||
weight: Tensor,
|
||
bias: Option<Tensor>,
|
||
params: BnbQuantParams,
|
||
quant_ty: BnbQuantType,
|
||
},
|
||
BlockwiseFP8 {
|
||
weight: Tensor,
|
||
weight_scale_inv: Tensor,
|
||
bias: Option<Tensor>,
|
||
dequant_dtype: DType,
|
||
weight_block_size: Vec<usize>,
|
||
},
|
||
PerTensorFP8 {
|
||
weight: Tensor,
|
||
weight_scale: Tensor,
|
||
/// The checkpoint's calibrated activation scale, present only when it quantized
|
||
/// activations too.
|
||
input_scale: Option<Tensor>,
|
||
bias: Option<Tensor>,
|
||
dequant_dtype: DType,
|
||
},
|
||
Afq {
|
||
weight: Tensor,
|
||
bias: Option<Tensor>,
|
||
bits: AfqBits,
|
||
group_size: AfqGroupSize,
|
||
},
|
||
MXFP4 {
|
||
blocks: Tensor,
|
||
scales: Tensor,
|
||
bias: Option<Tensor>,
|
||
},
|
||
NVFP4 {
|
||
weight: Tensor,
|
||
weight_scale: Tensor,
|
||
weight_scale_2: Option<Tensor>,
|
||
/// The checkpoint's calibrated activation scale, present only when it was
|
||
/// quantized for FP4 activations.
|
||
input_scale: Option<Tensor>,
|
||
bias: Option<Tensor>,
|
||
dequant_dtype: DType,
|
||
},
|
||
}
|
||
|
||
/// Device/configurable intelligent matrix multiplication
|
||
/// - Handles limitation of `accelerate` which requires f32
|
||
pub struct MatMul;
|
||
|
||
impl MatMul {
|
||
/// Compute matrix-matrix product.
|
||
pub fn matmul(&self, a: &Tensor, b: &Tensor) -> Result<Tensor> {
|
||
#[cfg(feature = "accelerate")]
|
||
{
|
||
let original_dtype = a.dtype();
|
||
a.to_dtype(DType::F32)?
|
||
.matmul(&b.to_dtype(DType::F32)?)?
|
||
.to_dtype(original_dtype)
|
||
}
|
||
#[cfg(not(feature = "accelerate"))]
|
||
{
|
||
// The CPU multiplies half-precision inputs in F32 and F32 inputs as they are. Rounding
|
||
// them to F16 instead kept three significant digits of every attention score and sent
|
||
// any score past 65504 to infinity.
|
||
if a.device().is_cpu() && matches!(a.dtype(), DType::F16 | DType::BF16) {
|
||
let original_dtype = a.dtype();
|
||
a.to_dtype(DType::F32)?
|
||
.matmul(&b.to_dtype(DType::F32)?)?
|
||
.to_dtype(original_dtype)
|
||
} else {
|
||
a.matmul(b)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Compute matrix-matrix product.
|
||
/// The result will be divided by the `scale` parameter in an affine division.
|
||
pub fn matmul_affine_div(&self, a: &Tensor, b: &Tensor, scale: f64) -> Result<Tensor> {
|
||
// TODO(hanzoai): Optimize this by using the gemm parameter?
|
||
self.matmul(a, b)? / scale
|
||
}
|
||
|
||
/// Compute matrix-matrix product.
|
||
/// The result will be divided by the `scale` parameter in an affine multiplication.
|
||
pub fn matmul_affine_mul(&self, a: &Tensor, b: &Tensor, scale: f64) -> Result<Tensor> {
|
||
// TODO(hanzoai): Optimize this by using the gemm parameter?
|
||
self.matmul(a, b)? * scale
|
||
}
|
||
|
||
/// Compute quantized matrix-matrix product.
|
||
pub fn qmatmul(&self, x: &Tensor, matmul: &QMatMul) -> Result<Tensor> {
|
||
matmul.forward(x)
|
||
}
|
||
}
|
||
|
||
/// Device/configurable intelligent convolution
|
||
/// - Handles limitation of cpu which requires f32
|
||
pub struct Convolution;
|
||
|
||
impl Convolution {
|
||
pub fn forward_1d(&self, layer: &Conv1d, x: &Tensor) -> Result<Tensor> {
|
||
if x.device().is_cpu() {
|
||
let original_dtype = x.dtype();
|
||
Conv1d::new(
|
||
layer.weight().to_dtype(DType::F32)?,
|
||
layer.bias().map(|b| b.to_dtype(DType::F32)).transpose()?,
|
||
*layer.config(),
|
||
)
|
||
.forward(&x.to_dtype(DType::F32)?)?
|
||
.to_dtype(original_dtype)
|
||
} else {
|
||
layer.forward(x)
|
||
}
|
||
}
|
||
|
||
pub fn forward_2d(&self, layer: &Conv2d, x: &Tensor) -> Result<Tensor> {
|
||
if x.device().is_cpu() {
|
||
let original_dtype = x.dtype();
|
||
Conv2d::new(
|
||
layer.weight().to_dtype(DType::F32)?,
|
||
layer.bias().map(|b| b.to_dtype(DType::F32)).transpose()?,
|
||
*layer.config(),
|
||
)
|
||
.forward(&x.to_dtype(DType::F32)?)?
|
||
.to_dtype(original_dtype)
|
||
} else {
|
||
layer.forward(x)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// In-situ quantization type specifying the format to apply to model weights.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Hash, Eq, Serialize, Deserialize)]
|
||
pub enum IsqType {
|
||
Q4_0,
|
||
Q4_1,
|
||
Q5_0,
|
||
Q5_1,
|
||
Q8_0,
|
||
Q8_1,
|
||
Q2K,
|
||
Q3K,
|
||
Q4K,
|
||
Q5K,
|
||
Q6K,
|
||
Q8K,
|
||
HQQ8,
|
||
HQQ4,
|
||
// HQQ3,
|
||
// HQQ2,
|
||
// HQQ1,
|
||
F8E4M3,
|
||
AFQ8,
|
||
AFQ6,
|
||
AFQ4,
|
||
AFQ3,
|
||
AFQ2,
|
||
F8Q8,
|
||
MXFP4,
|
||
}
|
||
|
||
/// Target bit width for automatic ISQ quantization.
|
||
///
|
||
/// On Metal, these select AFQ variants; on CUDA/CPU, they select Q*K variants.
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||
pub enum IsqBits {
|
||
/// 2-bit quantization (AFQ2 on Metal, Q2K otherwise).
|
||
Two,
|
||
/// 3-bit quantization (AFQ3 on Metal, Q3K otherwise).
|
||
Three,
|
||
/// 4-bit quantization (AFQ4 on Metal, Q4K otherwise).
|
||
Four,
|
||
/// 5-bit quantization (Q5K on all platforms).
|
||
Five,
|
||
/// 6-bit quantization (AFQ6 on Metal, Q6K otherwise).
|
||
Six,
|
||
/// 8-bit quantization (AFQ8 on Metal, Q8_0 otherwise).
|
||
Eight,
|
||
}
|
||
|
||
impl IsqBits {
|
||
/// Resolve to the platform-appropriate `IsqType` for the given device.
|
||
pub fn resolve(self, device: &Device) -> IsqType {
|
||
match (self, device.is_metal()) {
|
||
(Self::Two, true) => IsqType::AFQ2,
|
||
(Self::Two, false) => IsqType::Q2K,
|
||
(Self::Three, true) => IsqType::AFQ3,
|
||
(Self::Three, false) => IsqType::Q3K,
|
||
(Self::Four, true) => IsqType::AFQ4,
|
||
(Self::Four, false) => IsqType::Q4K,
|
||
(Self::Five, _) => IsqType::Q5K,
|
||
(Self::Six, true) => IsqType::AFQ6,
|
||
(Self::Six, false) => IsqType::Q6K,
|
||
(Self::Eight, true) => IsqType::AFQ8,
|
||
(Self::Eight, false) => IsqType::Q8_0,
|
||
}
|
||
}
|
||
|
||
/// Return all platform variants, with the current platform's preferred variant first.
|
||
/// On Metal, AFQ variants come first; on other platforms, GGUF/Q variants come first.
|
||
pub fn expand(self) -> Vec<IsqType> {
|
||
#[cfg(feature = "metal")]
|
||
match self {
|
||
Self::Two => vec![IsqType::AFQ2, IsqType::Q2K],
|
||
Self::Three => vec![IsqType::AFQ3, IsqType::Q3K],
|
||
Self::Four => vec![IsqType::AFQ4, IsqType::Q4K],
|
||
Self::Five => vec![IsqType::Q5K],
|
||
Self::Six => vec![IsqType::AFQ6, IsqType::Q6K],
|
||
Self::Eight => vec![IsqType::AFQ8, IsqType::Q8_0],
|
||
}
|
||
#[cfg(not(feature = "metal"))]
|
||
match self {
|
||
Self::Two => vec![IsqType::Q2K, IsqType::AFQ2],
|
||
Self::Three => vec![IsqType::Q3K, IsqType::AFQ3],
|
||
Self::Four => vec![IsqType::Q4K, IsqType::AFQ4],
|
||
Self::Five => vec![IsqType::Q5K],
|
||
Self::Six => vec![IsqType::Q6K, IsqType::AFQ6],
|
||
Self::Eight => vec![IsqType::Q8_0, IsqType::AFQ8],
|
||
}
|
||
}
|
||
}
|
||
|
||
impl TryFrom<&str> for IsqBits {
|
||
type Error = ();
|
||
fn try_from(s: &str) -> std::result::Result<Self, ()> {
|
||
match s {
|
||
"2" => Ok(Self::Two),
|
||
"3" => Ok(Self::Three),
|
||
"4" => Ok(Self::Four),
|
||
"5" => Ok(Self::Five),
|
||
"6" => Ok(Self::Six),
|
||
"8" => Ok(Self::Eight),
|
||
_ => Err(()),
|
||
}
|
||
}
|
||
}
|
||
|
||
impl std::fmt::Display for IsqType {
|
||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
match self {
|
||
Self::Q4_0 => write!(f, "q4_0"),
|
||
Self::Q4_1 => write!(f, "q4_1"),
|
||
Self::Q5_0 => write!(f, "q5_0"),
|
||
Self::Q5_1 => write!(f, "q5_1"),
|
||
Self::Q8_0 => write!(f, "q8_0"),
|
||
Self::Q8_1 => write!(f, "q8_1"),
|
||
Self::Q2K => write!(f, "q2k"),
|
||
Self::Q3K => write!(f, "q3k"),
|
||
Self::Q4K => write!(f, "q4k"),
|
||
Self::Q5K => write!(f, "q5k"),
|
||
Self::Q6K => write!(f, "q6k"),
|
||
Self::Q8K => write!(f, "q8k"),
|
||
Self::HQQ8 => write!(f, "hqq8"),
|
||
Self::HQQ4 => write!(f, "hqq4"),
|
||
Self::F8E4M3 => write!(f, "fp8"),
|
||
Self::AFQ8 => write!(f, "afq8"),
|
||
Self::AFQ6 => write!(f, "afq6"),
|
||
Self::AFQ4 => write!(f, "afq4"),
|
||
Self::AFQ3 => write!(f, "afq3"),
|
||
Self::AFQ2 => write!(f, "afq2"),
|
||
Self::F8Q8 => write!(f, "f8q8"),
|
||
Self::MXFP4 => write!(f, "mxfp4"),
|
||
}
|
||
}
|
||
}
|
||
|
||
impl IsqType {
|
||
/// Factor by which the weight size is reduced over the given dtype.
|
||
/// original size / pack factor = quantized size
|
||
pub fn pack_factor(&self, dtype: DType) -> usize {
|
||
match self {
|
||
Self::Q4_0 | Self::AFQ4 => (dtype.size_in_bytes() * GgmlDType::Q4_0.block_size())
|
||
.div_ceil(GgmlDType::Q4_0.type_size()),
|
||
Self::Q4_1 => (dtype.size_in_bytes() * GgmlDType::Q4_1.block_size())
|
||
.div_ceil(GgmlDType::Q4_1.type_size()),
|
||
Self::Q5_0 => (dtype.size_in_bytes() * GgmlDType::Q5_0.block_size())
|
||
.div_ceil(GgmlDType::Q5_0.type_size()),
|
||
Self::Q5_1 => (dtype.size_in_bytes() * GgmlDType::Q5_1.block_size())
|
||
.div_ceil(GgmlDType::Q5_1.type_size()),
|
||
Self::Q8_0 | Self::AFQ8 => (dtype.size_in_bytes() * GgmlDType::Q8_0.block_size())
|
||
.div_ceil(GgmlDType::Q8_0.type_size()),
|
||
Self::Q8_1 => (dtype.size_in_bytes() * GgmlDType::Q8_1.block_size())
|
||
.div_ceil(GgmlDType::Q8_1.type_size()),
|
||
Self::Q2K | Self::AFQ2 => (dtype.size_in_bytes() * GgmlDType::Q2K.block_size())
|
||
.div_ceil(GgmlDType::Q2K.type_size()),
|
||
Self::Q3K | Self::AFQ3 => (dtype.size_in_bytes() * GgmlDType::Q3K.block_size())
|
||
.div_ceil(GgmlDType::Q3K.type_size()),
|
||
Self::Q4K => (dtype.size_in_bytes() * GgmlDType::Q4K.block_size())
|
||
.div_ceil(GgmlDType::Q4K.type_size()),
|
||
Self::Q5K => (dtype.size_in_bytes() * GgmlDType::Q5K.block_size())
|
||
.div_ceil(GgmlDType::Q5K.type_size()),
|
||
Self::Q6K | Self::AFQ6 => (dtype.size_in_bytes() * GgmlDType::Q6K.block_size())
|
||
.div_ceil(GgmlDType::Q6K.type_size()),
|
||
Self::Q8K => (dtype.size_in_bytes() * GgmlDType::Q8K.block_size())
|
||
.div_ceil(GgmlDType::Q8K.type_size()),
|
||
// F8Q8: 33 bytes per 32 values -> similar to Q8_0
|
||
Self::F8Q8 => (dtype.size_in_bytes() * 32).div_ceil(33),
|
||
// Estimates
|
||
Self::HQQ4 => 4,
|
||
Self::HQQ8 => 2,
|
||
Self::F8E4M3 => 2,
|
||
// MXFP4: 4 bits per value + 1 byte scale per 32 values
|
||
// For BF16 (2 bytes): (2*32)/(16+1) ≈ 3.76 → 3
|
||
Self::MXFP4 => 3,
|
||
}
|
||
}
|
||
|
||
pub fn get_max_isq_cpu_threads(&self) -> Option<NonZeroUsize> {
|
||
match self {
|
||
/*IsqType::HQQ1 | IsqType::HQQ2 | IsqType::HQQ3 | */
|
||
IsqType::HQQ4
|
||
| IsqType::HQQ8
|
||
| IsqType::AFQ2
|
||
| IsqType::AFQ3
|
||
| IsqType::AFQ4
|
||
| IsqType::AFQ6
|
||
| IsqType::AFQ8
|
||
| IsqType::MXFP4 => {
|
||
// Use 1 because our HQQ quantizes on the GPU
|
||
Some(1.try_into().unwrap())
|
||
}
|
||
IsqType::F8E4M3 | IsqType::F8Q8 => None,
|
||
IsqType::Q2K
|
||
| IsqType::Q3K
|
||
| IsqType::Q4K
|
||
| IsqType::Q4_0
|
||
| IsqType::Q4_1
|
||
| IsqType::Q5K
|
||
| IsqType::Q5_0
|
||
| IsqType::Q5_1
|
||
| IsqType::Q6K
|
||
| IsqType::Q8K
|
||
| IsqType::Q8_0
|
||
| IsqType::Q8_1 => None,
|
||
}
|
||
}
|
||
}
|
||
|
||
impl TryFrom<IsqType> for GgmlDType {
|
||
type Error = hanzo_ml::Error;
|
||
|
||
fn try_from(value: IsqType) -> Result<Self> {
|
||
let tp = match value {
|
||
IsqType::Q2K => Self::Q2K,
|
||
IsqType::Q3K => Self::Q3K,
|
||
IsqType::Q4K => Self::Q4K,
|
||
IsqType::Q4_0 => Self::Q4_0,
|
||
IsqType::Q4_1 => Self::Q4_1,
|
||
IsqType::Q5K => Self::Q5K,
|
||
IsqType::Q5_0 => Self::Q5_0,
|
||
IsqType::Q5_1 => Self::Q5_1,
|
||
IsqType::Q6K => Self::Q6K,
|
||
IsqType::Q8K => Self::Q8K,
|
||
IsqType::Q8_0 => Self::Q8_0,
|
||
IsqType::Q8_1 => Self::Q8_1,
|
||
_ => hanzo_ml::bail!("Expected valid GGML ISQ type."),
|
||
};
|
||
#[cfg(feature = "cuda")]
|
||
{
|
||
if !matches!(
|
||
tp,
|
||
GgmlDType::Q4_0
|
||
| GgmlDType::Q4_1
|
||
| GgmlDType::Q5_0
|
||
| GgmlDType::Q5_1
|
||
| GgmlDType::Q8_0
|
||
| GgmlDType::Q2K
|
||
| GgmlDType::Q3K
|
||
| GgmlDType::Q4K
|
||
| GgmlDType::Q5K
|
||
| GgmlDType::Q6K
|
||
) {
|
||
hanzo_ml::bail!("GGML ISQ type on CUDA must be one of `Q4_0`, `Q4_1`, `Q5_0`, `Q5_1`, `Q8_0`, `Q2K`, `Q3K`, `Q4K`, `Q5K`, `Q6K`, `HQQ8`, `HQQ4`")
|
||
}
|
||
}
|
||
Ok(tp)
|
||
}
|
||
}
|
||
|
||
impl TryFrom<GgmlDType> for IsqType {
|
||
type Error = hanzo_ml::Error;
|
||
|
||
fn try_from(value: GgmlDType) -> Result<Self> {
|
||
match value {
|
||
GgmlDType::Q2K => Ok(Self::Q2K),
|
||
GgmlDType::Q3K => Ok(Self::Q3K),
|
||
GgmlDType::Q4K => Ok(Self::Q4K),
|
||
GgmlDType::Q5K => Ok(Self::Q5K),
|
||
GgmlDType::Q6K => Ok(Self::Q6K),
|
||
GgmlDType::Q4_0 => Ok(Self::Q4_0),
|
||
GgmlDType::Q4_1 => Ok(Self::Q4_1),
|
||
GgmlDType::Q5_0 => Ok(Self::Q5_0),
|
||
GgmlDType::Q5_1 => Ok(Self::Q5_1),
|
||
GgmlDType::Q8_0 => Ok(Self::Q8_0),
|
||
GgmlDType::Q8_1 => Ok(Self::Q8_1),
|
||
GgmlDType::Q8K => Ok(Self::Q8K),
|
||
GgmlDType::MXFP4 => Ok(Self::MXFP4),
|
||
GgmlDType::IQ4_NL
|
||
| GgmlDType::IQ4_XS
|
||
| GgmlDType::BF16
|
||
| GgmlDType::F32
|
||
| GgmlDType::F16
|
||
// IQ / ternary / 1-bit / FP4 codec types are decode-only, not ISQ targets.
|
||
| GgmlDType::ROCMFP4
|
||
| GgmlDType::ROCMFP4_FAST
|
||
| GgmlDType::IQ2_XXS
|
||
| GgmlDType::IQ2_XS
|
||
| GgmlDType::IQ3_XXS
|
||
| GgmlDType::IQ1_S
|
||
| GgmlDType::IQ3_S
|
||
| GgmlDType::IQ2_S
|
||
| GgmlDType::IQ1_M
|
||
| GgmlDType::TQ1_0
|
||
| GgmlDType::TQ2_0
|
||
| GgmlDType::NVFP4
|
||
| GgmlDType::Q1_0
|
||
| GgmlDType::PQ2_0
|
||
| GgmlDType::PTQ1_0
|
||
| GgmlDType::I32 => {
|
||
hanzo_ml::bail!("Expected valid GGML ISQ type.")
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Clone, Copy)]
|
||
pub enum QuantizedSerdeType {
|
||
Gguf = 0,
|
||
Unquant = 1,
|
||
Hqq = 2,
|
||
Fp8 = 3,
|
||
Afq = 4,
|
||
F8Q8 = 5,
|
||
Mxfp4 = 6,
|
||
}
|
||
|
||
impl TryFrom<usize> for QuantizedSerdeType {
|
||
type Error = hanzo_ml::Error;
|
||
fn try_from(value: usize) -> std::result::Result<Self, Self::Error> {
|
||
match value {
|
||
0 => Ok(Self::Gguf),
|
||
1 => Ok(Self::Unquant),
|
||
2 => Ok(Self::Hqq),
|
||
3 => Ok(Self::Fp8),
|
||
4 => Ok(Self::Afq),
|
||
5 => Ok(Self::F8Q8),
|
||
6 => Ok(Self::Mxfp4),
|
||
other => hanzo_ml::bail!("QuantizedSerdeType {other} is invalid."),
|
||
}
|
||
}
|
||
}
|
||
|
||
pub trait QuantizedSerde {
|
||
fn name(&self) -> &'static str;
|
||
fn isq_serde_supported(&self) -> bool {
|
||
false
|
||
}
|
||
fn serialize(&self) -> Result<Cow<'_, [u8]>> {
|
||
hanzo_ml::bail!("`QuantizedSerde::serialize` is not supported.")
|
||
}
|
||
fn deserialize(
|
||
_data: Cow<[u8]>,
|
||
_device: &Device,
|
||
_comm: &Arc<crate::Comm>,
|
||
_guard: QuantizeOntoGuard,
|
||
) -> Result<Arc<dyn QuantMethod>>
|
||
where
|
||
Self: Sized,
|
||
{
|
||
hanzo_ml::bail!("`QuantizedSerde::deserialize` is not supported.")
|
||
}
|
||
fn deserialize_ext_bias(
|
||
_data: Cow<[u8]>,
|
||
_device: &Device,
|
||
_guard: QuantizeOntoGuard,
|
||
) -> Result<(Arc<dyn QuantMethod>, Option<Tensor>)>
|
||
where
|
||
Self: Sized,
|
||
{
|
||
hanzo_ml::bail!("`QuantizedSerde::deserialize_ext_bias` is not supported.")
|
||
}
|
||
/// NOT meant for external calling
|
||
fn serialize_with_bias(&self, _bias: Option<Tensor>) -> Result<Cow<'_, [u8]>> {
|
||
hanzo_ml::bail!("`QuantizedSerde::serialize_with_bias` is not supported.")
|
||
}
|
||
}
|
||
|
||
/// Used to gate access to quantizing onto the host device
|
||
#[derive(Clone, Debug)]
|
||
#[allow(unused)]
|
||
pub struct QuantizeOntoGuard {
|
||
pub inner: Arc<Mutex<()>>,
|
||
}
|
||
|
||
/// Real (for Metal) and Fake (for CUDA)
|
||
pub enum QuantizeOntoDropGuard<'a> {
|
||
Real(MutexGuard<'a, ()>),
|
||
Fake,
|
||
}
|
||
|
||
impl Default for QuantizeOntoGuard {
|
||
fn default() -> Self {
|
||
Self::new()
|
||
}
|
||
}
|
||
|
||
impl QuantizeOntoGuard {
|
||
pub fn new() -> Self {
|
||
QuantizeOntoGuard {
|
||
inner: Arc::new(Mutex::new(())),
|
||
}
|
||
}
|
||
|
||
/// Acquire the quantize drop guard to protect the critical section.
|
||
///
|
||
/// On metal, this waits for outstanding work to finish to avoid "A command encoder is already encoding to this command buffer"
|
||
pub fn acquire(&self, device: &Device) -> QuantizeOntoDropGuard<'_> {
|
||
#[cfg(feature = "cuda")]
|
||
{
|
||
let _ = device;
|
||
QuantizeOntoDropGuard::Fake
|
||
}
|
||
|
||
#[cfg(not(feature = "cuda"))]
|
||
{
|
||
#[cfg(feature = "metal")]
|
||
if let Device::Metal(dev) = device {
|
||
// This is necessary to avoid the errors of "A command encoder is already encoding to this command buffer"
|
||
dev.wait_until_completed()
|
||
.expect("Failed to flush command buffer.");
|
||
}
|
||
#[cfg(not(feature = "metal"))]
|
||
let _ = device;
|
||
|
||
QuantizeOntoDropGuard::Real(self.inner.lock().expect("QuantizeOntoGuard was poisoned!"))
|
||
}
|
||
}
|
||
}
|
||
|
||
pub enum DistributedKind {
|
||
ColumnParallel,
|
||
RowParallel,
|
||
Replicated,
|
||
}
|
||
|
||
/// Quantized method for a quantized matmul.
|
||
pub trait QuantMethod: Send + Sync + Debug + QuantizedSerde {
|
||
fn new(method: QuantMethodConfig) -> Result<Self>
|
||
where
|
||
Self: Sized;
|
||
|
||
fn dequantize_w(&self) -> Result<Tensor>;
|
||
|
||
/// Compute matmul of `self` and `a`. `self` should contain the weights.
|
||
/// Automatically casts to the required quantization activation type and back.
|
||
fn forward(&self, a: &Tensor) -> Result<Tensor> {
|
||
if let Some(t) = self.quantized_act_type() {
|
||
let original_ty = a.dtype();
|
||
self.forward_raw(&a.to_dtype(t)?)?.to_dtype(original_ty)
|
||
} else {
|
||
self.forward_raw(a)
|
||
}
|
||
}
|
||
|
||
/// Raw matmul without dtype casting. Implementors override this.
|
||
/// Callers should use `forward` instead.
|
||
fn forward_raw(&self, a: &Tensor) -> Result<Tensor>;
|
||
|
||
/// Compute gather matmul of `self` and `a`. `self` should contain the weights.
|
||
/// Automatically casts to the required quantization activation type and back.
|
||
///
|
||
/// If `a` is (n_tokens, n_experts, cols), `self` weights are (n_experts, rows, cols),
|
||
/// then the indices are (n_tokens, n_experts).
|
||
fn gather_forward(&self, a: &Tensor, indices: &Tensor) -> Result<Tensor> {
|
||
if let Some(t) = self.quantized_act_type() {
|
||
let original_ty = a.dtype();
|
||
self.gather_forward_raw(&a.to_dtype(t)?, indices)?
|
||
.to_dtype(original_ty)
|
||
} else {
|
||
self.gather_forward_raw(a, indices)
|
||
}
|
||
}
|
||
|
||
/// Raw gather matmul without dtype casting. Implementors override this.
|
||
/// Callers should use `gather_forward` instead.
|
||
fn gather_forward_raw(&self, _a: &Tensor, _indices: &Tensor) -> Result<Tensor> {
|
||
hanzo_ml::bail!(
|
||
"{} does not support `gather_forward`. Please raise an issue.",
|
||
self.name()
|
||
)
|
||
}
|
||
|
||
/// Get the underlying QTensor if this is a GGUF quantized layer.
|
||
/// Used for direct kernel access in the grouped MoE prefill path.
|
||
#[cfg(feature = "cuda")]
|
||
fn get_qtensor(&self) -> Option<&hanzo_ml::quantized::QTensor> {
|
||
None
|
||
}
|
||
|
||
/// If this is an AFQ layer, return its (w_q, scales, biases, bits, group_size).
|
||
/// Used by Metal fused QKV / gate-up paths.
|
||
fn afq_inner(&self) -> Option<crate::afq::AfqInner<'_>> {
|
||
None
|
||
}
|
||
|
||
/// If a quantized method, return the activation dtype.
|
||
fn quantized_act_type(&self) -> Option<DType>;
|
||
|
||
/// Weight dtype and device
|
||
fn dtype_and_device(&self) -> (DType, Device);
|
||
|
||
/// Add a delta weight from LoRA to the weights. This should be prescaled with alpha.
|
||
fn add_delta_w(&self, delta: &Tensor) -> Result<Arc<dyn QuantMethod>>;
|
||
|
||
/// If the quant is backed by a qmatmul.
|
||
fn apply_isq(
|
||
self: Arc<Self>,
|
||
dtype: Option<IsqType>,
|
||
device: Device,
|
||
n_quantized: &AtomicUsize,
|
||
imatrix_weight: Option<Vec<f32>>,
|
||
guard: QuantizeOntoGuard,
|
||
) -> Result<Arc<dyn QuantMethod>>;
|
||
|
||
fn unquant_weight_bias(&self) -> Option<(Tensor, Option<Tensor>)> {
|
||
None
|
||
}
|
||
|
||
fn has_bias(&self) -> bool {
|
||
false
|
||
}
|
||
|
||
/// Begin tracking stats into an ImatrixLayerStats
|
||
fn begin_track_stats(&mut self) -> Result<()> {
|
||
hanzo_ml::bail!("`{}` does not support tracking stats.", self.name())
|
||
}
|
||
|
||
/// End tracking stats into an ImatrixLayerStats. Returns the computed imatrix.
|
||
fn end_track_stats(&self) -> Result<Tensor> {
|
||
hanzo_ml::bail!("`{}` does not support tracking stats.", self.name())
|
||
}
|
||
|
||
fn is_distributed(&self) -> Option<DistributedKind> {
|
||
None
|
||
}
|
||
|
||
fn dummy_info(&self) -> Option<&DummyLayerInfo> {
|
||
None
|
||
}
|
||
}
|
||
|
||
impl Module for dyn QuantMethod {
|
||
fn forward(&self, xs: &Tensor) -> Result<Tensor> {
|
||
QuantMethod::forward(self, xs)
|
||
}
|
||
}
|
||
|
||
#[cfg(feature = "cuda")]
|
||
pub fn try_fused_quantized_gate_up(
|
||
xs: &Tensor,
|
||
gate: &dyn QuantMethod,
|
||
up: &dyn QuantMethod,
|
||
activation: GluActivationType,
|
||
) -> Result<Option<Tensor>> {
|
||
if gate.has_bias() || up.has_bias() {
|
||
return Ok(None);
|
||
}
|
||
if !matches!(xs.dtype(), DType::BF16 | DType::F16 | DType::F32) {
|
||
return Ok(None);
|
||
}
|
||
|
||
let Some(gate_q) = gate.get_qtensor() else {
|
||
return Ok(None);
|
||
};
|
||
let Some(up_q) = up.get_qtensor() else {
|
||
return Ok(None);
|
||
};
|
||
let dtype = gate_q.dtype();
|
||
if dtype != up_q.dtype() || !gguf::fast_mmvq::supports(dtype) {
|
||
return Ok(None);
|
||
}
|
||
if gate_q.shape() != up_q.shape() {
|
||
return Ok(None);
|
||
}
|
||
|
||
let Some((&k, batch_dims)) = xs.dims().split_last() else {
|
||
return Ok(None);
|
||
};
|
||
let flat_batch = batch_dims.iter().product::<usize>();
|
||
if flat_batch == 0 || flat_batch > gguf::fast_mmvq::MMVQ_MAX_BATCH {
|
||
return Ok(None);
|
||
}
|
||
let (_, ncols) = gate_q.shape().dims2()?;
|
||
if k != ncols {
|
||
return Ok(None);
|
||
}
|
||
|
||
Ok(Some(gguf::fast_mmvq::fused_glu(
|
||
gate_q, up_q, xs, activation,
|
||
)?))
|
||
}
|
||
|
||
#[cfg(feature = "cuda")]
|
||
pub fn try_fused_quantized_qkv(
|
||
xs: &Tensor,
|
||
q: &dyn QuantMethod,
|
||
k: &dyn QuantMethod,
|
||
v: &dyn QuantMethod,
|
||
) -> Result<Option<(Tensor, Tensor, Tensor)>> {
|
||
if q.has_bias() || k.has_bias() || v.has_bias() {
|
||
return Ok(None);
|
||
}
|
||
if !matches!(xs.dtype(), DType::BF16 | DType::F16 | DType::F32) {
|
||
return Ok(None);
|
||
}
|
||
|
||
let Some(q_q) = q.get_qtensor() else {
|
||
return Ok(None);
|
||
};
|
||
let Some(k_q) = k.get_qtensor() else {
|
||
return Ok(None);
|
||
};
|
||
let Some(v_q) = v.get_qtensor() else {
|
||
return Ok(None);
|
||
};
|
||
let dtype = q_q.dtype();
|
||
if dtype != k_q.dtype() || dtype != v_q.dtype() || !gguf::fast_mmvq::supports(dtype) {
|
||
return Ok(None);
|
||
}
|
||
|
||
let Some((&input_cols, batch_dims)) = xs.dims().split_last() else {
|
||
return Ok(None);
|
||
};
|
||
let flat_batch = batch_dims.iter().product::<usize>();
|
||
if flat_batch == 0 || flat_batch > gguf::fast_mmvq::MMVQ_MAX_BATCH {
|
||
return Ok(None);
|
||
}
|
||
let (_, q_cols) = q_q.shape().dims2()?;
|
||
let (_, k_cols) = k_q.shape().dims2()?;
|
||
let (_, v_cols) = v_q.shape().dims2()?;
|
||
if input_cols != q_cols || input_cols != k_cols || input_cols != v_cols {
|
||
return Ok(None);
|
||
}
|
||
|
||
Ok(Some(gguf::fast_mmvq::fused_qkv(q_q, k_q, v_q, xs)?))
|
||
}
|
||
|
||
/// Metal fused gate+up: single Metal kernel that does both matmuls with shared
|
||
/// x reads and applies the GLU activation in-register before writing one output.
|
||
#[cfg(feature = "metal")]
|
||
pub fn try_fused_gate_up_metal(
|
||
xs: &Tensor,
|
||
gate: &dyn QuantMethod,
|
||
up: &dyn QuantMethod,
|
||
activation: GluActivationType,
|
||
) -> Result<Option<Tensor>> {
|
||
use hanzo_ml::{backend::BackendStorage, MetalStorage, Shape, Storage};
|
||
|
||
if gate.has_bias() || up.has_bias() {
|
||
return Ok(None);
|
||
}
|
||
if !matches!(xs.dtype(), DType::BF16 | DType::F16 | DType::F32) {
|
||
return Ok(None);
|
||
}
|
||
if !xs.device().is_metal() {
|
||
return Ok(None);
|
||
}
|
||
|
||
let Some(gi) = gate.afq_inner() else {
|
||
return Ok(None);
|
||
};
|
||
let Some(ui) = up.afq_inner() else {
|
||
return Ok(None);
|
||
};
|
||
if gi.bits != ui.bits || gi.group_size != ui.group_size {
|
||
return Ok(None);
|
||
}
|
||
if gi.scales.dtype() != ui.scales.dtype() {
|
||
return Ok(None);
|
||
}
|
||
if gi.w_q.rank() != 2 || ui.w_q.rank() != 2 {
|
||
return Ok(None);
|
||
}
|
||
let k = xs.dim(D::Minus1)?;
|
||
let n_gate = gi.w_q.dim(0)?;
|
||
let n_up = ui.w_q.dim(0)?;
|
||
if n_gate != n_up {
|
||
return Ok(None);
|
||
}
|
||
let n = n_gate;
|
||
// qmm_t kernel uses BM=32 tiles; for small M (decode) it wastes most of the
|
||
// tile. Let the caller fall back to separate qmv-based forwards.
|
||
let probe_m = xs.elem_count() / k;
|
||
if probe_m < 16 {
|
||
return Ok(None);
|
||
}
|
||
if k * gi.bits as usize / 8 / 4 != gi.w_q.dim(1)? {
|
||
// unexpected pack factor; let the generic path handle it
|
||
return Ok(None);
|
||
}
|
||
|
||
let act_code: u32 = match activation {
|
||
GluActivationType::Silu => 0,
|
||
GluActivationType::Gelu => 1,
|
||
GluActivationType::GeluErf => 2,
|
||
GluActivationType::Relu => 3,
|
||
};
|
||
|
||
let xs = xs.contiguous()?;
|
||
let m = xs.elem_count() / k;
|
||
if m == 0 {
|
||
return Ok(None);
|
||
}
|
||
|
||
let (xs_storage, xs_layout) = xs.storage_and_layout();
|
||
let Storage::Metal(xs_storage) = &*xs_storage else {
|
||
return Ok(None);
|
||
};
|
||
let (g_w_s, _) = gi.w_q.storage_and_layout();
|
||
let Storage::Metal(g_w_s) = &*g_w_s else {
|
||
return Ok(None);
|
||
};
|
||
let (g_s_s, _) = gi.scales.storage_and_layout();
|
||
let Storage::Metal(g_s_s) = &*g_s_s else {
|
||
return Ok(None);
|
||
};
|
||
let (g_b_s, _) = gi.biases.storage_and_layout();
|
||
let Storage::Metal(g_b_s) = &*g_b_s else {
|
||
return Ok(None);
|
||
};
|
||
let (u_w_s, _) = ui.w_q.storage_and_layout();
|
||
let Storage::Metal(u_w_s) = &*u_w_s else {
|
||
return Ok(None);
|
||
};
|
||
let (u_s_s, _) = ui.scales.storage_and_layout();
|
||
let Storage::Metal(u_s_s) = &*u_s_s else {
|
||
return Ok(None);
|
||
};
|
||
let (u_b_s, _) = ui.biases.storage_and_layout();
|
||
let Storage::Metal(u_b_s) = &*u_b_s else {
|
||
return Ok(None);
|
||
};
|
||
|
||
let device = xs_storage.device().clone();
|
||
let dtype = xs.dtype();
|
||
let mut out_shape = xs.dims().to_vec();
|
||
*out_shape.last_mut().unwrap() = n;
|
||
let out = device.new_buffer(out_shape.iter().product(), dtype, "afq-gate-up-out")?;
|
||
|
||
let encoder = device.command_encoder()?;
|
||
encoder.set_label("afq-gate-up");
|
||
|
||
metal_kernels::call_afq_qmm_gate_up(
|
||
device.device(),
|
||
&encoder,
|
||
&metal_kernels::Kernels::new(),
|
||
dtype,
|
||
(
|
||
xs_storage.buffer(),
|
||
xs_layout.start_offset() * dtype.size_in_bytes(),
|
||
),
|
||
g_w_s.buffer(),
|
||
g_s_s.buffer(),
|
||
g_b_s.buffer(),
|
||
u_w_s.buffer(),
|
||
u_s_s.buffer(),
|
||
u_b_s.buffer(),
|
||
&out,
|
||
m,
|
||
n,
|
||
k,
|
||
gi.bits as usize,
|
||
gi.group_size as usize,
|
||
act_code,
|
||
)
|
||
.map_err(hanzo_ml::Error::wrap)?;
|
||
|
||
let out_t = Tensor::from((
|
||
Storage::Metal(MetalStorage::new(
|
||
out,
|
||
device.clone(),
|
||
out_shape.iter().product(),
|
||
dtype,
|
||
)),
|
||
Shape::from(out_shape),
|
||
));
|
||
Ok(Some(out_t))
|
||
}
|
||
|
||
/// Metal fused QKV: single Metal kernel that handles all three projections,
|
||
/// routing per-tile to the right weight matrix.
|
||
#[cfg(feature = "metal")]
|
||
pub fn try_fused_qkv_metal(
|
||
xs: &Tensor,
|
||
q: &dyn QuantMethod,
|
||
k: &dyn QuantMethod,
|
||
v: &dyn QuantMethod,
|
||
) -> Result<Option<(Tensor, Tensor, Tensor)>> {
|
||
use hanzo_ml::{backend::BackendStorage, MetalStorage, Shape, Storage};
|
||
|
||
if q.has_bias() || k.has_bias() || v.has_bias() {
|
||
return Ok(None);
|
||
}
|
||
if !matches!(xs.dtype(), DType::BF16 | DType::F16 | DType::F32) {
|
||
return Ok(None);
|
||
}
|
||
if !xs.device().is_metal() {
|
||
return Ok(None);
|
||
}
|
||
|
||
let Some(qi) = q.afq_inner() else {
|
||
return Ok(None);
|
||
};
|
||
let Some(ki) = k.afq_inner() else {
|
||
return Ok(None);
|
||
};
|
||
let Some(vi) = v.afq_inner() else {
|
||
return Ok(None);
|
||
};
|
||
if qi.bits != ki.bits || qi.bits != vi.bits {
|
||
return Ok(None);
|
||
}
|
||
if qi.group_size != ki.group_size || qi.group_size != vi.group_size {
|
||
return Ok(None);
|
||
}
|
||
if qi.scales.dtype() != ki.scales.dtype() || qi.scales.dtype() != vi.scales.dtype() {
|
||
return Ok(None);
|
||
}
|
||
if qi.w_q.rank() != 2 || ki.w_q.rank() != 2 || vi.w_q.rank() != 2 {
|
||
return Ok(None);
|
||
}
|
||
let n_q = qi.w_q.dim(0)?;
|
||
let n_k = ki.w_q.dim(0)?;
|
||
let n_v = vi.w_q.dim(0)?;
|
||
// The kernel routes by tile-aligned column boundaries; require N_q and
|
||
// N_k to be multiples of the tile width (32). For Gemma-style models
|
||
// those are already 32-multiples; fall back when they're not.
|
||
if n_q % 32 != 0 || n_k % 32 != 0 || n_v % 32 != 0 {
|
||
return Ok(None);
|
||
}
|
||
let k_dim = xs.dim(D::Minus1)?;
|
||
// qmm_t kernel uses BM=32; for small M (decode) the tile is mostly empty.
|
||
// Fall back to separate qmv calls.
|
||
let probe_m = xs.elem_count() / k_dim;
|
||
if probe_m < 16 {
|
||
return Ok(None);
|
||
}
|
||
|
||
let xs = xs.contiguous()?;
|
||
let m = xs.elem_count() / k_dim;
|
||
if m == 0 {
|
||
return Ok(None);
|
||
}
|
||
|
||
let (xs_s, xs_l) = xs.storage_and_layout();
|
||
let Storage::Metal(xs_s) = &*xs_s else {
|
||
return Ok(None);
|
||
};
|
||
let qws = qi.w_q.storage_and_layout().0;
|
||
let qss = qi.scales.storage_and_layout().0;
|
||
let qbs = qi.biases.storage_and_layout().0;
|
||
let kws = ki.w_q.storage_and_layout().0;
|
||
let kss = ki.scales.storage_and_layout().0;
|
||
let kbs = ki.biases.storage_and_layout().0;
|
||
let vws = vi.w_q.storage_and_layout().0;
|
||
let vss = vi.scales.storage_and_layout().0;
|
||
let vbs = vi.biases.storage_and_layout().0;
|
||
let (Storage::Metal(qw_m), Storage::Metal(qs_m), Storage::Metal(qb_m)) = (&*qws, &*qss, &*qbs)
|
||
else {
|
||
return Ok(None);
|
||
};
|
||
let (Storage::Metal(kw_m), Storage::Metal(ks_m), Storage::Metal(kb_m)) = (&*kws, &*kss, &*kbs)
|
||
else {
|
||
return Ok(None);
|
||
};
|
||
let (Storage::Metal(vw_m), Storage::Metal(vs_m), Storage::Metal(vb_m)) = (&*vws, &*vss, &*vbs)
|
||
else {
|
||
return Ok(None);
|
||
};
|
||
|
||
let device = xs_s.device().clone();
|
||
let dtype = xs.dtype();
|
||
let mut q_shape = xs.dims().to_vec();
|
||
let mut k_shape = q_shape.clone();
|
||
let mut v_shape = q_shape.clone();
|
||
*q_shape.last_mut().unwrap() = n_q;
|
||
*k_shape.last_mut().unwrap() = n_k;
|
||
*v_shape.last_mut().unwrap() = n_v;
|
||
let q_out = device.new_buffer(q_shape.iter().product(), dtype, "afq-qkv-q")?;
|
||
let k_out = device.new_buffer(k_shape.iter().product(), dtype, "afq-qkv-k")?;
|
||
let v_out = device.new_buffer(v_shape.iter().product(), dtype, "afq-qkv-v")?;
|
||
|
||
let encoder = device.command_encoder()?;
|
||
encoder.set_label("afq-qkv");
|
||
|
||
metal_kernels::call_afq_qmm_qkv(
|
||
device.device(),
|
||
&encoder,
|
||
&metal_kernels::Kernels::new(),
|
||
dtype,
|
||
(xs_s.buffer(), xs_l.start_offset() * dtype.size_in_bytes()),
|
||
qw_m.buffer(),
|
||
qs_m.buffer(),
|
||
qb_m.buffer(),
|
||
kw_m.buffer(),
|
||
ks_m.buffer(),
|
||
kb_m.buffer(),
|
||
vw_m.buffer(),
|
||
vs_m.buffer(),
|
||
vb_m.buffer(),
|
||
&q_out,
|
||
&k_out,
|
||
&v_out,
|
||
m,
|
||
n_q,
|
||
n_k,
|
||
n_v,
|
||
k_dim,
|
||
qi.bits as usize,
|
||
qi.group_size as usize,
|
||
)
|
||
.map_err(hanzo_ml::Error::wrap)?;
|
||
|
||
let q_t = Tensor::from((
|
||
Storage::Metal(MetalStorage::new(
|
||
q_out,
|
||
device.clone(),
|
||
q_shape.iter().product(),
|
||
dtype,
|
||
)),
|
||
Shape::from(q_shape),
|
||
));
|
||
let k_t = Tensor::from((
|
||
Storage::Metal(MetalStorage::new(
|
||
k_out,
|
||
device.clone(),
|
||
k_shape.iter().product(),
|
||
dtype,
|
||
)),
|
||
Shape::from(k_shape),
|
||
));
|
||
let v_t = Tensor::from((
|
||
Storage::Metal(MetalStorage::new(
|
||
v_out,
|
||
device.clone(),
|
||
v_shape.iter().product(),
|
||
dtype,
|
||
)),
|
||
Shape::from(v_shape),
|
||
));
|
||
Ok(Some((q_t, k_t, v_t)))
|
||
}
|
||
|
||
fn tensor_prefix(vb: &ShardedVarBuilder) -> String {
|
||
let prefix = vb.prefix();
|
||
if prefix.is_empty() {
|
||
"<root>".to_string()
|
||
} else {
|
||
prefix
|
||
}
|
||
}
|
||
|
||
fn missing_required_tensors(vb: &ShardedVarBuilder, required: &[&str]) -> Vec<String> {
|
||
required
|
||
.iter()
|
||
.copied()
|
||
.filter(|name| !vb.contains_tensor(name))
|
||
.map(|name| safetensors::full_tensor_name(vb, name))
|
||
.collect()
|
||
}
|
||
|
||
pub(crate) fn has_missing_required_tensors(vb: &ShardedVarBuilder, required: &[&str]) -> bool {
|
||
required.iter().any(|name| !vb.contains_tensor(name))
|
||
}
|
||
|
||
pub(crate) fn make_dummy_or_error(
|
||
context: &str,
|
||
vb: &ShardedVarBuilder,
|
||
required: &[&str],
|
||
) -> Result<Arc<dyn QuantMethod>> {
|
||
let missing = missing_required_tensors(vb, required);
|
||
if missing.is_empty() {
|
||
hanzo_ml::bail!(
|
||
"Internal error: requested DummyLayer for {context} without missing tensors"
|
||
);
|
||
}
|
||
|
||
let has_uqff_placeholder = required
|
||
.iter()
|
||
.any(|name| safetensors::is_uqff_dummy_tensor(vb, name));
|
||
if !has_uqff_placeholder {
|
||
hanzo_ml::bail!(
|
||
"Missing required tensor(s) for {context} at prefix `{}`: {}. Dummy layers are only allowed for tensors intentionally omitted while loading UQFF artifacts.",
|
||
tensor_prefix(vb),
|
||
missing.join(", ")
|
||
);
|
||
}
|
||
|
||
Ok(Arc::new(DummyLayer::placeholder(DummyLayerInfo {
|
||
context: context.to_string(),
|
||
prefix: tensor_prefix(vb),
|
||
missing_tensors: missing,
|
||
})))
|
||
}
|
||
|
||
pub fn linear_no_bias(
|
||
in_dim: usize,
|
||
out_dim: usize,
|
||
config: &Option<QuantizedConfig>,
|
||
vb: ShardedVarBuilder,
|
||
) -> Result<Arc<dyn QuantMethod>> {
|
||
let base_vb = vb.clone();
|
||
let vb = if should_apply_immediate_isq(&vb) {
|
||
vb.set_device(Device::Cpu)
|
||
} else {
|
||
vb
|
||
};
|
||
|
||
let layer = if let Some(quant_conf) = &config {
|
||
match quant_conf {
|
||
QuantizedConfig::GptqAwq { .. } => gptq_linear(in_dim, out_dim, quant_conf, vb)?,
|
||
QuantizedConfig::Fp8 { weight_block_size } => {
|
||
if weight_block_size.is_some() {
|
||
blockwise_fp8_linear_b(
|
||
in_dim,
|
||
out_dim,
|
||
quant_conf,
|
||
false,
|
||
Default::default(),
|
||
vb,
|
||
)?
|
||
} else {
|
||
pertensor_fp8_linear_b(
|
||
in_dim,
|
||
out_dim,
|
||
quant_conf,
|
||
false,
|
||
Default::default(),
|
||
vb,
|
||
)?
|
||
}
|
||
}
|
||
QuantizedConfig::Bitsandbytes { .. } => {
|
||
Arc::new(BnbLinear::linear_b(in_dim, out_dim, false, vb)?) as Arc<_>
|
||
}
|
||
QuantizedConfig::Afq { .. } => {
|
||
AfqLayer::afq_linear_b(in_dim, out_dim, quant_conf, false, vb)?
|
||
}
|
||
QuantizedConfig::MXFP4 {} => {
|
||
MXFP4Layer::linear_b(in_dim, out_dim, quant_conf, false, vb)?
|
||
}
|
||
QuantizedConfig::ModelOpt {
|
||
quantized_layers, ..
|
||
} => distributed::layers::load_modelopt_linear(
|
||
in_dim,
|
||
out_dim,
|
||
false,
|
||
quant_conf,
|
||
quantized_layers,
|
||
&vb,
|
||
)?,
|
||
}
|
||
} else {
|
||
if !vb.contains_tensor("weight") {
|
||
make_dummy_or_error("linear_no_bias", &vb, &["weight"])?
|
||
} else {
|
||
let weight = vb.get_with_hints((out_dim, in_dim), "weight", Default::default())?;
|
||
let weight = merge_lora_weights(&vb, weight, in_dim, out_dim, Default::default())?;
|
||
|
||
let layer = <UnquantLinear as QuantMethod>::new(QuantMethodConfig::Unquantized(
|
||
Linear::new(weight, None),
|
||
))?;
|
||
Arc::new(layer) as Arc<dyn QuantMethod>
|
||
}
|
||
};
|
||
apply_immediate_isq(layer, base_vb)
|
||
}
|
||
|
||
pub fn linear(
|
||
in_dim: usize,
|
||
out_dim: usize,
|
||
config: &Option<QuantizedConfig>,
|
||
vb: ShardedVarBuilder,
|
||
) -> Result<Arc<dyn QuantMethod>> {
|
||
let base_vb = vb.clone();
|
||
let vb = if should_apply_immediate_isq(&vb) {
|
||
vb.set_device(Device::Cpu)
|
||
} else {
|
||
vb
|
||
};
|
||
|
||
let layer = if let Some(quant_conf) = &config {
|
||
match quant_conf {
|
||
QuantizedConfig::GptqAwq { .. } => gptq_linear(in_dim, out_dim, quant_conf, vb)?,
|
||
QuantizedConfig::Fp8 { weight_block_size } => {
|
||
if weight_block_size.is_some() {
|
||
blockwise_fp8_linear_b(
|
||
in_dim,
|
||
out_dim,
|
||
quant_conf,
|
||
true,
|
||
Default::default(),
|
||
vb,
|
||
)?
|
||
} else {
|
||
pertensor_fp8_linear_b(
|
||
in_dim,
|
||
out_dim,
|
||
quant_conf,
|
||
true,
|
||
Default::default(),
|
||
vb,
|
||
)?
|
||
}
|
||
}
|
||
QuantizedConfig::Bitsandbytes { .. } => {
|
||
Arc::new(BnbLinear::linear_b(in_dim, out_dim, true, vb)?) as Arc<_>
|
||
}
|
||
QuantizedConfig::Afq { .. } => {
|
||
AfqLayer::afq_linear_b(in_dim, out_dim, quant_conf, true, vb)?
|
||
}
|
||
QuantizedConfig::MXFP4 {} => {
|
||
MXFP4Layer::linear_b(in_dim, out_dim, quant_conf, true, vb)?
|
||
}
|
||
QuantizedConfig::ModelOpt {
|
||
quantized_layers, ..
|
||
} => distributed::layers::load_modelopt_linear(
|
||
in_dim,
|
||
out_dim,
|
||
true,
|
||
quant_conf,
|
||
quantized_layers,
|
||
&vb,
|
||
)?,
|
||
}
|
||
} else {
|
||
if has_missing_required_tensors(&vb, &["weight", "bias"]) {
|
||
make_dummy_or_error("linear", &vb, &["weight", "bias"])?
|
||
} else {
|
||
let weight = vb.get_with_hints((out_dim, in_dim), "weight", Default::default())?;
|
||
let weight = merge_lora_weights(&vb, weight, in_dim, out_dim, Default::default())?;
|
||
let bias = vb.get_with_hints((out_dim,), "bias", Default::default())?;
|
||
|
||
let layer = <UnquantLinear as QuantMethod>::new(QuantMethodConfig::Unquantized(
|
||
Linear::new(weight, Some(bias)),
|
||
))?;
|
||
Arc::new(layer) as Arc<dyn QuantMethod>
|
||
}
|
||
};
|
||
apply_immediate_isq(layer, base_vb)
|
||
}
|
||
|
||
pub fn linear_b(
|
||
in_dim: usize,
|
||
out_dim: usize,
|
||
bias: bool,
|
||
config: &Option<QuantizedConfig>,
|
||
vb: ShardedVarBuilder,
|
||
) -> Result<Arc<dyn QuantMethod>> {
|
||
if bias {
|
||
linear(in_dim, out_dim, config, vb)
|
||
} else {
|
||
linear_no_bias(in_dim, out_dim, config, vb)
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use std::collections::HashMap;
|
||
|
||
use super::*;
|
||
|
||
fn empty_vb(make_dummy_regexes: Option<Vec<&str>>) -> ShardedVarBuilder {
|
||
let backend: HashMap<String, Tensor> = HashMap::new();
|
||
let make_dummy_regexes = make_dummy_regexes.map(|regexes| {
|
||
Arc::new(
|
||
regexes
|
||
.into_iter()
|
||
.map(Regex::new)
|
||
.collect::<std::result::Result<Vec<_>, _>>()
|
||
.unwrap(),
|
||
)
|
||
});
|
||
ShardedSafeTensors::wrap_with_dummy_regexes(
|
||
Box::new(backend),
|
||
DType::F32,
|
||
Device::Cpu,
|
||
make_dummy_regexes,
|
||
)
|
||
}
|
||
|
||
#[test]
|
||
fn missing_linear_weight_outside_uqff_errors() {
|
||
let err = linear_no_bias(2, 3, &None, empty_vb(None).pp("foo")).unwrap_err();
|
||
let msg = err.to_string();
|
||
|
||
assert!(msg.contains("Missing required tensor(s)"));
|
||
assert!(msg.contains("foo.weight"));
|
||
assert!(msg.contains("UQFF"));
|
||
}
|
||
|
||
#[test]
|
||
fn missing_uqff_placeholder_creates_contextual_dummy() -> Result<()> {
|
||
let layer = linear_no_bias(
|
||
2,
|
||
3,
|
||
&None,
|
||
empty_vb(Some(vec![r"^foo\.weight$"])).pp("foo"),
|
||
)?;
|
||
|
||
let info = layer.dummy_info().unwrap();
|
||
assert_eq!(layer.name(), "dummy");
|
||
assert_eq!(info.context, "linear_no_bias");
|
||
assert_eq!(info.prefix, "foo");
|
||
assert_eq!(info.missing_tensors, vec!["foo.weight"]);
|
||
|
||
let input = Tensor::zeros((1, 2), DType::F32, &Device::Cpu)?;
|
||
let err = layer.forward_raw(&input).unwrap_err();
|
||
let msg = err.to_string();
|
||
assert!(msg.contains("forward pass"));
|
||
assert!(msg.contains("foo.weight"));
|
||
assert!(msg.contains("temporary UQFF placeholders"));
|
||
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod afq_recipe_tests {
|
||
use super::*;
|
||
|
||
// Entries copied from the MTPLX pack of Qwen3.8-Flash-Next (config.json `quantization`):
|
||
// 4-bit g64 by default, routers and the indexer at 8 bits, hyper-connections unquantized.
|
||
const RECIPE: &str = r#"{
|
||
"bits": 4, "group_size": 64, "mode": "affine",
|
||
"language_model.lm_head": {"bits": 4, "group_size": 64, "mode": "affine"},
|
||
"language_model.model.layers.0.mlp.gate": {"bits": 8, "group_size": 64, "mode": "affine"},
|
||
"language_model.model.layers.11.self_attn.indexer.index_qk_proj": {"bits": 8, "group_size": 64, "mode": "affine"},
|
||
"language_model.model.layers.0.attn_hyper_connection.input_mix_weight_down": false
|
||
}"#;
|
||
|
||
#[test]
|
||
fn mlx_recipe_resolves_per_module() {
|
||
let config: QuantizedConfig = serde_json::from_str(RECIPE).unwrap();
|
||
let at = |prefix: &str| config.afq_at(prefix);
|
||
assert_eq!(at("language_model.lm_head"), Some((4, 64)));
|
||
assert_eq!(at("language_model.model.layers.0.mlp.gate"), Some((8, 64)));
|
||
assert_eq!(
|
||
at("language_model.model.layers.11.self_attn.indexer.index_qk_proj"),
|
||
Some((8, 64))
|
||
);
|
||
assert_eq!(
|
||
at("language_model.model.layers.0.attn_hyper_connection.input_mix_weight_down"),
|
||
None
|
||
);
|
||
// A module the recipe does not list takes the defaults.
|
||
assert_eq!(
|
||
at("language_model.model.layers.3.mlp.switch_mlp.gate_proj"),
|
||
Some((4, 64))
|
||
);
|
||
// The same path nested under another root matches on a `.` boundary; a bare suffix does not.
|
||
assert_eq!(
|
||
at("model.language_model.model.layers.0.mlp.gate"),
|
||
Some((8, 64))
|
||
);
|
||
assert_eq!(at("layers.10.mlp.gate"), Some((4, 64)));
|
||
}
|
||
|
||
#[test]
|
||
fn mlx_recipe_refuses_other_modes() {
|
||
let recipe = r#"{"bits": 4, "group_size": 32, "a.b": {"bits": 4, "group_size": 32, "mode": "mxfp4"}}"#;
|
||
assert!(serde_json::from_str::<QuantizedConfig>(recipe).is_err());
|
||
let recipe = r#"{"bits": 4, "group_size": 32, "mode": "nvfp4"}"#;
|
||
assert!(serde_json::from_str::<QuantizedConfig>(recipe).is_err());
|
||
}
|
||
}
|