mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 23:45:50 +00:00
The indexer q/k norms ran through QRmsNorm at the activation dtype, and
rms_norm casts the weight to that dtype. With bf16 activations the folded
Gemma weight 1 + w (f32 in the GGUF, mostly not bf16-exact) was rounded
before the multiply; on ROCm the normed value was rounded too. vLLM keeps x,
rrms and 1 + w in f32 and rounds once before RoPE
(nvidia/ops/qsa_pre_indexer.py:68-71). project and compress now norm in f32
and round once. norms_round_once holds every bf16 query and compressed key
within half an ulp of the f64 norm; it fails on either path without the fix.
The fixture's norm weights are now folded Gemma values.
The per-entry KV budget from cf0c1c7292 stays. Gemma4's floored per-layer
mean charged 21490 of the 21504 elements a token that the cache allocates,
over-running an MbAmount budget, and tune charged a hybrid for its linear
layers. paged_budget_is_the_per_layer_sum pins Gemma4's sum and block count
against CacheEngine's allocation. calculate_max_context now takes the
ModelConfigLike, and hybrid_context_counts_only_attention_layers pins the
hybrid figure.
Co-authored-by: Hanzo Dev <[email protected]>
722 lines
24 KiB
Rust
722 lines
24 KiB
Rust
use std::path::{Path, PathBuf};
|
|
|
|
use anyhow::{Context, Result};
|
|
use hanzo_ml::{DType, Device};
|
|
use hf_hub::{
|
|
api::sync::{ApiBuilder, ApiRepo},
|
|
Cache, Repo, RepoType,
|
|
};
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
use crate::device_map::{DeviceLayerMapMetadata, DeviceMapMetadata};
|
|
use crate::model_loader::{get_auto_device_map_params, get_model_dtype};
|
|
use crate::paged_attention::ModelConfigLike;
|
|
use crate::pipeline::{
|
|
AutoDeviceMapParams, AutoEmbeddingLoader, AutoMultimodalLoader, AutoNormalLoader,
|
|
DeviceMappedModelLoader, EmbeddingLoaderType, MultimodalLoaderType, NormalLoaderType,
|
|
TokenSource,
|
|
};
|
|
use crate::utils::tokens::get_token;
|
|
use crate::{paged_attn_supported, IsqType, ModelSelected, TryIntoDType, GLOBAL_HF_CACHE};
|
|
|
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
|
#[serde(rename_all = "kebab-case")]
|
|
pub enum TuneProfile {
|
|
Quality,
|
|
Balanced,
|
|
Fast,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct AutoTuneRequest {
|
|
pub model: ModelSelected,
|
|
pub token_source: TokenSource,
|
|
pub hf_revision: Option<String>,
|
|
pub force_cpu: bool,
|
|
pub profile: TuneProfile,
|
|
pub requested_isq: Option<IsqType>,
|
|
}
|
|
|
|
/// Quality tier for a quantization level
|
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
|
#[serde(rename_all = "kebab-case")]
|
|
pub enum QualityTier {
|
|
/// Full precision, baseline quality
|
|
Baseline,
|
|
/// Near-lossless (8-bit)
|
|
NearLossless,
|
|
/// Good quality (6-bit, 5-bit)
|
|
Good,
|
|
/// Acceptable quality (4-bit)
|
|
Acceptable,
|
|
/// Degraded quality (3-bit, 2-bit)
|
|
Degraded,
|
|
}
|
|
|
|
/// Fit status for a quantization candidate
|
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
|
#[serde(rename_all = "kebab-case")]
|
|
pub enum FitStatus {
|
|
/// Model fits entirely on GPU(s)
|
|
Fits,
|
|
/// Model requires CPU offloading (hybrid)
|
|
Hybrid,
|
|
/// Model doesn't fit even with CPU offload
|
|
TooLarge,
|
|
}
|
|
|
|
/// A tuning candidate with all calculated metrics
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TuneCandidate {
|
|
/// Quantization type (None = no quantization)
|
|
pub isq: Option<IsqType>,
|
|
/// Display name for the quantization
|
|
pub isq_name: String,
|
|
/// Estimated model size in bytes
|
|
pub estimated_size_bytes: u64,
|
|
/// VRAM usage as percentage (0.0 - 1.0)
|
|
pub vram_usage_percent: f32,
|
|
/// Maximum context length that fits (model-specific calculation)
|
|
pub max_context_tokens: usize,
|
|
/// Whether max_context_tokens is the model's native maximum (not VRAM limited)
|
|
pub context_is_model_max: bool,
|
|
/// Quality tier
|
|
pub quality: QualityTier,
|
|
/// Whether this candidate fits
|
|
pub fit_status: FitStatus,
|
|
/// Device layer mapping (if hybrid)
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub device_layers_cli: Option<String>,
|
|
/// Whether this is the recommended option
|
|
pub recommended: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AutoTuneResult {
|
|
pub model_id: String,
|
|
pub profile: TuneProfile,
|
|
pub backend: String,
|
|
/// All evaluated candidates with their metrics
|
|
pub candidates: Vec<TuneCandidate>,
|
|
/// The recommended ISQ type
|
|
pub recommended_isq: Option<IsqType>,
|
|
/// Device layers for the recommended option
|
|
pub device_layers: Option<Vec<DeviceLayerMapMetadata>>,
|
|
pub device_layers_cli: Option<String>,
|
|
pub paged_attn_mode: Option<String>,
|
|
/// Copy-paste command for the recommended option
|
|
pub recommended_command: String,
|
|
/// Total VRAM available across all GPUs (bytes)
|
|
pub total_vram_bytes: u64,
|
|
pub warnings: Vec<String>,
|
|
pub notes: Vec<String>,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
enum TuneBackend {
|
|
Cpu,
|
|
Cuda,
|
|
Metal,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug)]
|
|
enum TuneKind {
|
|
Normal,
|
|
Multimodal,
|
|
Embedding,
|
|
}
|
|
|
|
fn backend_from_devices(devices: &[Device]) -> TuneBackend {
|
|
if devices.iter().any(|d| matches!(d, Device::Cuda(_))) {
|
|
TuneBackend::Cuda
|
|
} else if devices.iter().any(|d| matches!(d, Device::Metal(_))) {
|
|
TuneBackend::Metal
|
|
} else {
|
|
TuneBackend::Cpu
|
|
}
|
|
}
|
|
|
|
fn backend_name(backend: TuneBackend) -> String {
|
|
match backend {
|
|
TuneBackend::Cpu => "cpu".to_string(),
|
|
TuneBackend::Cuda => "cuda".to_string(),
|
|
TuneBackend::Metal => "metal".to_string(),
|
|
}
|
|
}
|
|
|
|
fn select_devices(force_cpu: bool) -> Result<Vec<Device>> {
|
|
if force_cpu {
|
|
return Ok(vec![Device::Cpu]);
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
{
|
|
if let Ok(dev) = Device::new_cuda(0) {
|
|
return Ok(crate::device_map::get_all_similar_devices(&dev)?);
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "metal")]
|
|
{
|
|
if let Ok(dev) = Device::new_metal(0) {
|
|
return Ok(crate::device_map::get_all_similar_devices(&dev)?);
|
|
}
|
|
}
|
|
|
|
Ok(vec![Device::Cpu])
|
|
}
|
|
|
|
fn hf_cache_path_from_model(model: &ModelSelected) -> Option<PathBuf> {
|
|
match model {
|
|
ModelSelected::Plain { hf_cache_path, .. }
|
|
| ModelSelected::Lora { hf_cache_path, .. }
|
|
| ModelSelected::XLora { hf_cache_path, .. }
|
|
| ModelSelected::MultimodalPlain { hf_cache_path, .. }
|
|
| ModelSelected::Embedding { hf_cache_path, .. }
|
|
| ModelSelected::Run { hf_cache_path, .. } => hf_cache_path.clone(),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn model_id_from_selected(model: &ModelSelected) -> String {
|
|
match model {
|
|
ModelSelected::Plain { model_id, .. }
|
|
| ModelSelected::Lora {
|
|
model_id: Some(model_id),
|
|
..
|
|
}
|
|
| ModelSelected::XLora {
|
|
model_id: Some(model_id),
|
|
..
|
|
}
|
|
| ModelSelected::MultimodalPlain { model_id, .. }
|
|
| ModelSelected::Embedding { model_id, .. }
|
|
| ModelSelected::Run { model_id, .. } => model_id.clone(),
|
|
ModelSelected::GGUF {
|
|
quantized_model_id, ..
|
|
}
|
|
| ModelSelected::GGML {
|
|
quantized_model_id, ..
|
|
}
|
|
| ModelSelected::LoraGGUF {
|
|
quantized_model_id, ..
|
|
}
|
|
| ModelSelected::XLoraGGUF {
|
|
quantized_model_id, ..
|
|
}
|
|
| ModelSelected::LoraGGML {
|
|
quantized_model_id, ..
|
|
}
|
|
| ModelSelected::XLoraGGML {
|
|
quantized_model_id, ..
|
|
} => quantized_model_id.clone(),
|
|
ModelSelected::DiffusionPlain { model_id, .. } => model_id.clone(),
|
|
ModelSelected::Speech { model_id, .. } => model_id.clone(),
|
|
ModelSelected::Toml { file } => file.clone(),
|
|
ModelSelected::MultiModel { .. } => "multi-model".to_string(),
|
|
_ => "unknown".to_string(),
|
|
}
|
|
}
|
|
|
|
fn load_config_artifacts(
|
|
model_id: &str,
|
|
token_source: &TokenSource,
|
|
hf_revision: Option<String>,
|
|
hf_cache_path: Option<PathBuf>,
|
|
) -> Result<(String, bool)> {
|
|
if Path::new(model_id).exists() {
|
|
let config_path = Path::new(model_id).join("config.json");
|
|
let config = std::fs::read_to_string(&config_path)
|
|
.with_context(|| format!("Failed to read config.json at {}", config_path.display()))?;
|
|
let sentence_transformers = Path::new(model_id)
|
|
.join("config_sentence_transformers.json")
|
|
.exists();
|
|
return Ok((config, sentence_transformers));
|
|
}
|
|
|
|
let cache = hf_cache_path
|
|
.map(Cache::new)
|
|
.unwrap_or_else(Cache::from_env);
|
|
GLOBAL_HF_CACHE.get_or_init(|| cache.clone());
|
|
|
|
let mut api = ApiBuilder::from_cache(cache)
|
|
.with_progress(false)
|
|
.with_token(get_token(token_source)?);
|
|
if let Some(cache_dir) = crate::hf_hub_cache_dir() {
|
|
api = api.with_cache_dir(cache_dir);
|
|
}
|
|
let api = api.build()?;
|
|
let revision = hf_revision.unwrap_or_else(|| "main".to_string());
|
|
let api = api.repo(Repo::with_revision(
|
|
model_id.to_string(),
|
|
RepoType::Model,
|
|
revision,
|
|
));
|
|
|
|
let config_path = api_get_file(&api, model_id, "config.json")?;
|
|
let config = std::fs::read_to_string(&config_path).with_context(|| {
|
|
format!(
|
|
"Failed to read config.json from cache at {}",
|
|
config_path.display()
|
|
)
|
|
})?;
|
|
|
|
let sentence_transformers =
|
|
api_get_file(&api, model_id, "config_sentence_transformers.json").is_ok();
|
|
|
|
Ok((config, sentence_transformers))
|
|
}
|
|
|
|
fn api_get_file(api: &ApiRepo, model_id: &str, file: &str) -> Result<PathBuf> {
|
|
let model_id = Path::new(model_id);
|
|
if model_id.exists() {
|
|
let path = model_id.join(file);
|
|
if path.exists() {
|
|
Ok(path)
|
|
} else {
|
|
anyhow::bail!("File {file} not found at {}", model_id.display())
|
|
}
|
|
} else {
|
|
Ok(api.get(file)?)
|
|
}
|
|
}
|
|
|
|
fn infer_kind(config: &str, sentence_transformers: bool) -> Result<TuneKind> {
|
|
if sentence_transformers {
|
|
return Ok(TuneKind::Embedding);
|
|
}
|
|
#[derive(Deserialize)]
|
|
struct AutoConfig {
|
|
architectures: Vec<String>,
|
|
}
|
|
let cfg: AutoConfig = serde_json::from_str(config)?;
|
|
if cfg.architectures.len() != 1 {
|
|
anyhow::bail!("Expected exactly one architecture in config");
|
|
}
|
|
let name = &cfg.architectures[0];
|
|
if MultimodalLoaderType::from_causal_lm_name(name).is_ok() {
|
|
return Ok(TuneKind::Multimodal);
|
|
}
|
|
if EmbeddingLoaderType::from_causal_lm_name(name).is_ok() {
|
|
return Ok(TuneKind::Embedding);
|
|
}
|
|
let _ = NormalLoaderType::from_causal_lm_name(name)?;
|
|
Ok(TuneKind::Normal)
|
|
}
|
|
|
|
/// Get all candidates to evaluate (in order from highest to lowest quality)
|
|
fn all_candidates(backend: TuneBackend) -> Vec<Option<IsqType>> {
|
|
match backend {
|
|
TuneBackend::Metal => vec![
|
|
None, // No quantization
|
|
Some(IsqType::AFQ8),
|
|
Some(IsqType::AFQ6),
|
|
Some(IsqType::AFQ4),
|
|
Some(IsqType::AFQ3),
|
|
Some(IsqType::AFQ2),
|
|
],
|
|
_ => vec![
|
|
None, // No quantization
|
|
Some(IsqType::Q8_0),
|
|
Some(IsqType::Q6K),
|
|
Some(IsqType::Q5K),
|
|
Some(IsqType::Q4K),
|
|
Some(IsqType::Q3K),
|
|
Some(IsqType::Q2K),
|
|
],
|
|
}
|
|
}
|
|
|
|
fn default_candidates(profile: TuneProfile, backend: TuneBackend) -> Vec<IsqType> {
|
|
match backend {
|
|
TuneBackend::Metal => match profile {
|
|
TuneProfile::Quality => {
|
|
vec![IsqType::AFQ8, IsqType::AFQ6, IsqType::AFQ4, IsqType::AFQ3]
|
|
}
|
|
TuneProfile::Balanced => vec![IsqType::AFQ6, IsqType::AFQ4, IsqType::AFQ3],
|
|
TuneProfile::Fast => vec![IsqType::AFQ4, IsqType::AFQ3, IsqType::AFQ2],
|
|
},
|
|
_ => match profile {
|
|
TuneProfile::Quality => vec![
|
|
IsqType::Q8_0,
|
|
IsqType::Q6K,
|
|
IsqType::Q5K,
|
|
IsqType::Q4K,
|
|
IsqType::Q3K,
|
|
IsqType::Q2K,
|
|
],
|
|
TuneProfile::Balanced => vec![IsqType::Q6K, IsqType::Q5K, IsqType::Q4K, IsqType::Q3K],
|
|
TuneProfile::Fast => vec![IsqType::Q4K, IsqType::Q3K, IsqType::Q2K],
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Map ISQ type to quality tier
|
|
fn quality_tier(isq: Option<IsqType>) -> QualityTier {
|
|
match isq {
|
|
None => QualityTier::Baseline,
|
|
Some(t) => match t {
|
|
IsqType::Q8_0 | IsqType::Q8_1 | IsqType::Q8K | IsqType::AFQ8 | IsqType::HQQ8 => {
|
|
QualityTier::NearLossless
|
|
}
|
|
IsqType::Q6K | IsqType::AFQ6 => QualityTier::Good,
|
|
IsqType::Q5_0 | IsqType::Q5_1 | IsqType::Q5K => QualityTier::Good,
|
|
IsqType::Q4_0 | IsqType::Q4_1 | IsqType::Q4K | IsqType::AFQ4 | IsqType::HQQ4 => {
|
|
QualityTier::Acceptable
|
|
}
|
|
IsqType::Q3K | IsqType::AFQ3 => QualityTier::Degraded,
|
|
IsqType::Q2K | IsqType::AFQ2 => QualityTier::Degraded,
|
|
_ => QualityTier::Acceptable,
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Get display name for ISQ type
|
|
fn isq_display_name(isq: Option<IsqType>) -> String {
|
|
match isq {
|
|
None => "None (FP16)".to_string(),
|
|
Some(t) => format!("{t:?}"),
|
|
}
|
|
}
|
|
|
|
/// Get total VRAM across all GPU devices
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
fn total_vram(devices: &[Device]) -> u64 {
|
|
use crate::MemoryUsage;
|
|
devices
|
|
.iter()
|
|
.filter(|d| !matches!(d, Device::Cpu))
|
|
.filter_map(|d| MemoryUsage.query(d).ok().map(|m| m.total()))
|
|
.sum::<usize>() as u64
|
|
}
|
|
|
|
/// Get available VRAM across all GPU devices
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
fn available_vram(devices: &[Device]) -> u64 {
|
|
use crate::MemoryUsage;
|
|
devices
|
|
.iter()
|
|
.filter(|d| !matches!(d, Device::Cpu))
|
|
.filter_map(|d| MemoryUsage.query(d).ok().map(|m| m.available()))
|
|
.sum::<usize>() as u64
|
|
}
|
|
|
|
/// Calculate the maximum context length that fits in remaining VRAM
|
|
/// Uses the model's actual KV cache configuration from ModelConfigLike
|
|
/// Returns (max_context, is_at_model_max) where is_at_model_max is true if
|
|
/// the context is limited by the model's native max, not VRAM
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
fn calculate_max_context(
|
|
model_cfg: &dyn ModelConfigLike,
|
|
model_size_bytes: u64,
|
|
available_vram_bytes: u64,
|
|
dtype: DType,
|
|
) -> (usize, bool) {
|
|
let native_max_seq_len = model_cfg.max_seq_len();
|
|
|
|
if model_size_bytes >= available_vram_bytes {
|
|
return (0, false);
|
|
}
|
|
|
|
let remaining_bytes = available_vram_bytes - model_size_bytes;
|
|
|
|
// KV cache elements per token across every cached layer, each at its own layer's
|
|
// num_kv_heads, k_head_dim, v_head_dim (from ModelConfigLike trait)
|
|
let kv_elems_per_token = model_cfg.kv_cache_elements_per_token();
|
|
|
|
// Total KV cache bytes per token = elements * dtype_size
|
|
let dtype_size = dtype.size_in_bytes();
|
|
let kv_bytes_per_token = kv_elems_per_token * dtype_size;
|
|
|
|
if kv_bytes_per_token == 0 {
|
|
return (native_max_seq_len, true);
|
|
}
|
|
|
|
let calculated_max = remaining_bytes as usize / kv_bytes_per_token;
|
|
|
|
// Return the minimum of calculated max and model's native max
|
|
let is_at_model_max = calculated_max >= native_max_seq_len;
|
|
(calculated_max.min(native_max_seq_len), is_at_model_max)
|
|
}
|
|
|
|
fn map_for_candidate(
|
|
loader: &dyn DeviceMappedModelLoader,
|
|
config: &str,
|
|
dtype: DType,
|
|
params: &AutoDeviceMapParams,
|
|
devices: &[Device],
|
|
isq: Option<IsqType>,
|
|
) -> Result<(DeviceMapMetadata, usize)> {
|
|
let pack_factor = isq.map(|i| i.pack_factor(dtype)).unwrap_or(1);
|
|
let layer_sizes = loader.layer_sizes_in_bytes(config, dtype, pack_factor, None)?;
|
|
let non_mapped = loader.non_mapped_size_in_bytes(config, dtype, pack_factor, None)?;
|
|
let total = layer_sizes.iter().sum::<usize>() + non_mapped;
|
|
let map = crate::pipeline::get_device_layers_for_loader(
|
|
loader,
|
|
config,
|
|
loader.num_layers(config)?,
|
|
layer_sizes,
|
|
non_mapped,
|
|
total,
|
|
devices,
|
|
dtype,
|
|
params,
|
|
None,
|
|
)?;
|
|
Ok((map, total))
|
|
}
|
|
|
|
#[allow(
|
|
clippy::cast_precision_loss,
|
|
clippy::cast_possible_truncation,
|
|
clippy::cast_sign_loss
|
|
)]
|
|
pub fn auto_tune(req: AutoTuneRequest) -> Result<AutoTuneResult> {
|
|
let model_id = model_id_from_selected(&req.model);
|
|
match &req.model {
|
|
ModelSelected::GGUF { .. }
|
|
| ModelSelected::GGML { .. }
|
|
| ModelSelected::LoraGGUF { .. }
|
|
| ModelSelected::XLoraGGUF { .. }
|
|
| ModelSelected::LoraGGML { .. }
|
|
| ModelSelected::XLoraGGML { .. } => {
|
|
anyhow::bail!("Auto-tuning is not supported for pre-quantized GGUF/GGML models.");
|
|
}
|
|
ModelSelected::DiffusionPlain { .. } | ModelSelected::Speech { .. } => {
|
|
anyhow::bail!("Auto-tuning is not supported for diffusion or speech models.");
|
|
}
|
|
_ => {}
|
|
}
|
|
|
|
let hf_cache_path = hf_cache_path_from_model(&req.model);
|
|
let (config, sentence_transformers) = load_config_artifacts(
|
|
&model_id,
|
|
&req.token_source,
|
|
req.hf_revision.clone(),
|
|
hf_cache_path,
|
|
)?;
|
|
|
|
let kind = match &req.model {
|
|
ModelSelected::MultimodalPlain { .. } => TuneKind::Multimodal,
|
|
ModelSelected::Embedding { .. } => TuneKind::Embedding,
|
|
_ => infer_kind(&config, sentence_transformers)?,
|
|
};
|
|
|
|
let mut params = get_auto_device_map_params(&req.model)?;
|
|
if matches!(kind, TuneKind::Multimodal) {
|
|
params = params.maybe_promote_to_multimodal();
|
|
}
|
|
|
|
let devices = select_devices(req.force_cpu)?;
|
|
let backend = backend_from_devices(&devices);
|
|
|
|
let dtype = {
|
|
let model_dtype = get_model_dtype(&req.model)?;
|
|
let refs = devices.iter().collect::<Vec<_>>();
|
|
model_dtype.try_into_dtype(&refs)?
|
|
};
|
|
|
|
let loader_normal = AutoNormalLoader;
|
|
let loader_multimodal = AutoMultimodalLoader;
|
|
let loader_embedding = AutoEmbeddingLoader;
|
|
let loader: &dyn DeviceMappedModelLoader = match kind {
|
|
TuneKind::Normal => &loader_normal,
|
|
TuneKind::Multimodal => &loader_multimodal,
|
|
TuneKind::Embedding => &loader_embedding,
|
|
};
|
|
|
|
// Get preferred candidates based on profile (for determining recommendation)
|
|
let preferred_candidates: Vec<Option<IsqType>> =
|
|
req.requested_isq.map(|t| vec![Some(t)]).unwrap_or_else(|| {
|
|
default_candidates(req.profile, backend)
|
|
.into_iter()
|
|
.map(Some)
|
|
.collect()
|
|
});
|
|
|
|
let mut warnings = Vec::new();
|
|
let mut notes = Vec::new();
|
|
|
|
if matches!(kind, TuneKind::Embedding) {
|
|
notes.push("Detected embedding model configuration.".to_string());
|
|
}
|
|
if matches!(kind, TuneKind::Multimodal) {
|
|
notes.push("Detected multimodal model configuration.".to_string());
|
|
}
|
|
|
|
// Get total VRAM for calculations
|
|
let total_vram_bytes = total_vram(&devices);
|
|
let avail_vram_bytes = available_vram(&devices);
|
|
|
|
// Evaluate ALL candidates
|
|
let all_isq_candidates = all_candidates(backend);
|
|
let mut tune_candidates: Vec<TuneCandidate> = Vec::new();
|
|
let mut recommended_idx: Option<usize> = None;
|
|
|
|
for isq in all_isq_candidates {
|
|
let result = map_for_candidate(loader, &config, dtype, ¶ms, &devices, isq);
|
|
|
|
let (fit_status, estimated_size, device_layers_cli) = match &result {
|
|
Ok((map, total_size)) => {
|
|
let layers = map.device_layers();
|
|
let is_hybrid = layers
|
|
.map(|l| l.iter().any(|d| d.ordinal == usize::MAX))
|
|
.unwrap_or(false);
|
|
let status = if is_hybrid {
|
|
FitStatus::Hybrid
|
|
} else {
|
|
FitStatus::Fits
|
|
};
|
|
(status, *total_size as u64, map.to_cli_spec())
|
|
}
|
|
Err(_) => {
|
|
// Calculate estimated size even for non-fitting candidates
|
|
let pack_factor = isq.map(|i| i.pack_factor(dtype)).unwrap_or(1);
|
|
let layer_sizes = loader
|
|
.layer_sizes_in_bytes(&config, dtype, pack_factor, None)
|
|
.unwrap_or_default();
|
|
let non_mapped = loader
|
|
.non_mapped_size_in_bytes(&config, dtype, pack_factor, None)
|
|
.unwrap_or(0);
|
|
let est_size = (layer_sizes.iter().sum::<usize>() + non_mapped) as u64;
|
|
(FitStatus::TooLarge, est_size, None)
|
|
}
|
|
};
|
|
|
|
let vram_usage = if total_vram_bytes > 0 {
|
|
(estimated_size as f32) / (total_vram_bytes as f32)
|
|
} else {
|
|
1.0
|
|
};
|
|
|
|
let (context_room, context_is_model_max) = loader
|
|
.model_config(&config)
|
|
.map(|cfg| calculate_max_context(&*cfg, estimated_size, avail_vram_bytes, dtype))
|
|
.unwrap_or((0, false));
|
|
|
|
let candidate = TuneCandidate {
|
|
isq,
|
|
isq_name: isq_display_name(isq),
|
|
estimated_size_bytes: estimated_size,
|
|
vram_usage_percent: vram_usage,
|
|
max_context_tokens: context_room,
|
|
context_is_model_max,
|
|
quality: quality_tier(isq),
|
|
fit_status,
|
|
device_layers_cli,
|
|
recommended: false, // Set later
|
|
};
|
|
|
|
tune_candidates.push(candidate);
|
|
}
|
|
|
|
// Mark the recommended candidate (first from preferred list that fits)
|
|
for pref in &preferred_candidates {
|
|
if let Some(idx) = tune_candidates.iter().position(|c| {
|
|
c.isq == *pref && matches!(c.fit_status, FitStatus::Fits | FitStatus::Hybrid)
|
|
}) {
|
|
tune_candidates[idx].recommended = true;
|
|
recommended_idx = Some(idx);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// If no preferred candidate fits, recommend the first that fits
|
|
if recommended_idx.is_none() {
|
|
if let Some(idx) = tune_candidates
|
|
.iter()
|
|
.position(|c| matches!(c.fit_status, FitStatus::Fits | FitStatus::Hybrid))
|
|
{
|
|
tune_candidates[idx].recommended = true;
|
|
recommended_idx = Some(idx);
|
|
}
|
|
}
|
|
|
|
// Build the result
|
|
let (recommended_isq, device_layers, device_layers_cli, recommended_command) =
|
|
if let Some(idx) = recommended_idx {
|
|
let rec = &tune_candidates[idx];
|
|
let isq_flag = rec
|
|
.isq
|
|
.map(|i| format!(" --isq {:?}", i).to_lowercase())
|
|
.unwrap_or_default();
|
|
let cmd = format!("hanzo serve -m {model_id}{isq_flag}");
|
|
(rec.isq, None, rec.device_layers_cli.clone(), cmd)
|
|
} else {
|
|
(None, None, None, format!("hanzo serve -m {model_id}"))
|
|
};
|
|
|
|
let paged_attn_mode = if backend != TuneBackend::Cpu && paged_attn_supported() {
|
|
Some("auto".to_string())
|
|
} else {
|
|
Some("off".to_string())
|
|
};
|
|
|
|
// Add warnings for candidates that don't fit
|
|
for c in &tune_candidates {
|
|
if matches!(c.fit_status, FitStatus::TooLarge) && c.isq.is_some() {
|
|
warnings.push(format!(
|
|
"{} ({:.1} GB) exceeds available VRAM",
|
|
c.isq_name,
|
|
c.estimated_size_bytes as f64 / 1e9
|
|
));
|
|
}
|
|
}
|
|
|
|
if recommended_idx.is_none() {
|
|
anyhow::bail!(
|
|
"No suitable quantization level fits on the available devices. Try a smaller model or enable CPU offload."
|
|
);
|
|
}
|
|
|
|
Ok(AutoTuneResult {
|
|
model_id,
|
|
profile: req.profile,
|
|
backend: backend_name(backend),
|
|
candidates: tune_candidates,
|
|
recommended_isq,
|
|
device_layers,
|
|
device_layers_cli,
|
|
paged_attn_mode,
|
|
recommended_command,
|
|
total_vram_bytes,
|
|
warnings,
|
|
notes,
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::paged_attention::{KvCacheLayout, KvLayers, ModelConfigMetadata};
|
|
|
|
/// A hybrid's context is priced by the attention layers it caches, not by every decoder layer:
|
|
/// the Qwen3.5-27B shape, 16 of 64 layers at 4 KV heads of 128, costs 16 x 2 x 4 x 128 x 2 B
|
|
/// = 32 KiB a token, a quarter of what the same model with attention everywhere costs.
|
|
#[test]
|
|
fn hybrid_context_counts_only_attention_layers() {
|
|
let dense = ModelConfigMetadata {
|
|
max_seq_len: 262144,
|
|
num_layers: 64,
|
|
hidden_size: 4096,
|
|
num_kv_heads: 4,
|
|
num_attn_heads: 32,
|
|
sliding_window: None,
|
|
k_head_dim: 128,
|
|
v_head_dim: 128,
|
|
kv_cache_layout: KvCacheLayout::StandardNoFlashInfer,
|
|
};
|
|
let hybrid = KvLayers::new(
|
|
dense.clone(),
|
|
(0..64).filter(|i| (i + 1) % 4 == 0).collect(),
|
|
);
|
|
let (model, gib) = (3 << 30, 1u64 << 30);
|
|
let fits = |config: &dyn ModelConfigLike, free| {
|
|
calculate_max_context(config, model, model + free, DType::BF16)
|
|
};
|
|
assert_eq!(fits(&hybrid, 4 * gib), (131072, false));
|
|
assert_eq!(fits(&dense, 4 * gib), (32768, false));
|
|
assert_eq!(fits(&hybrid, 16 * gib), (262144, true));
|
|
}
|
|
}
|