Files
079edc6182 fix(engine): isolate the prefix cache per tenant and cache only written KV (#119)
Prefix blocks were hashed from token content alone, so any two requests with the
same prefix shared KV across orgs, and the sequence-level cache and its disk spill
were keyed by tokens too. Every block hash, sequence-cache key and recurrent-prefix
key now carries the request's cache salt, which the server derives from the
gateway's X-Org-Id plus an optional cache_salt body field. Salted prefixes are
dropped instead of spilled to disk.

The scheduler also cached blocks no forward had written: a prompt preempted in the
pass that admitted it, the last sampled token's slot of every finished or preempted
sequence, and sequences that ended in error. Now only written positions become
reusable.

kv_isolation drives the real scheduler against a simulated KV store and checks every
read for unwritten, foreign-context or other-tenant KV; hanzo.yml gates on it.

Co-authored-by: Claude Opus 5.5 <[email protected]>
2026-11-19 16:39:53 -05:00

165 lines
5.1 KiB
Rust

//! Compute perplexity of a text file using a loaded model.
//!
//! Run with: `cargo run --release --example perplexity -p hanzo`
use std::{fs::read_to_string, path::PathBuf, time::Instant};
use anyhow::{Context, Result};
use clap::Parser;
use either::Either;
use hanzo::{
cross_entropy_loss, parse_isq_value, Constraint, DType, Device, Hanzo, ModelBuilder,
NormalRequest, Request, ResponseOk, SamplingParams, Tensor,
};
use tokio::sync::mpsc::channel;
/// Calculate perplexity of a model. By default, this uses the Llama 3.1 8B model.
#[derive(Parser)]
struct Args {
/// The model to run.
#[arg(short, long, default_value = "google/gemma-4-E4B-it")]
model_id: String,
/// Filename to text to run the model on. This is recommended to be the Wikitext 2 dataset:
/// https://huggingface.co/datasets/hanzoai/wikitext2
#[arg(short, long)]
file: String,
/// ISQ quantization to run with.
#[arg(short, long)]
isq: Option<String>,
/// Generate and utilize an imatrix to enhance GGUF quantizations.
#[arg(short, long)]
calibration_file: Option<PathBuf>,
}
async fn process_chunk(runner: &Hanzo, chunk: Vec<u32>) -> anyhow::Result<(Tensor, Vec<u32>)> {
let (tx, mut rx) = channel(1);
let request = Request::Normal(Box::new(NormalRequest {
messages: hanzo::RequestMessage::CompletionTokens(chunk),
sampling_params: SamplingParams {
max_len: Some(0),
..SamplingParams::deterministic()
},
response: tx,
return_logprobs: false,
is_streaming: false,
id: 0,
constraint: Constraint::None,
suffix: None,
tools: None,
tool_choice: None,
logits_processors: None,
return_raw_logits: true,
web_search_options: None,
enable_code_execution: false,
code_execution_permission: None,
code_execution_approval_notifier: None,
agent_permission: None,
agent_approval_handler: None,
agent_approval_notifier: None,
max_tool_rounds: None,
tool_dispatch_url: None,
model_id: None,
truncate_sequence: false,
session_id: None,
files: None,
cache_salt: None,
}));
runner.get_sender(None)?.send(request).await?;
let ResponseOk::Raw {
logits_chunks,
tokens,
} = rx
.recv()
.await
.context("Channel was erroneously closed!")?
.as_result()?
else {
anyhow::bail!("Got unexpected response type.")
};
Ok((logits_chunks[0].clone(), tokens))
}
#[tokio::main]
async fn main() -> Result<()> {
let args = Args::parse();
let quant = if let Some(isq) = &args.isq {
Some(parse_isq_value(isq, None).map_err(anyhow::Error::msg)?)
} else {
None
};
let prompt_chunksize = 1024;
let mut model_builder = ModelBuilder::new(&args.model_id).with_logging();
if let Some(quant) = quant {
model_builder = model_builder.with_isq(quant);
}
if let Some(calibration_file) = &args.calibration_file {
model_builder = model_builder.with_calibration_file(calibration_file.clone());
}
let model = model_builder.build().await?;
let text = read_to_string(&args.file)?;
let tokens = model
.tokenize(Either::Right(text), None, false, false, None)
.await?;
let bos_token = model
.tokenize(Either::Right(" ".to_string()), None, true, false, None)
.await?[0];
let inner = model.inner();
println!("Using bos token id `{bos_token}`.");
let n_chunks = tokens.len().div_ceil(prompt_chunksize);
let mut ppl_measurements = Vec::new();
for (i, chunk) in tokens.chunks(prompt_chunksize).enumerate() {
let start = Instant::now();
let (logits, tokens) = {
let chunk = [vec![bos_token], chunk.to_vec()].concat();
process_chunk(inner, chunk).await?
};
// Upcast to float if we need to compute the loss to avoid potential precision issues
let logits = logits.to_device(&Device::Cpu)?.to_dtype(DType::F32)?;
// Shift so that tokens < n predict n
let shift_logits = logits.narrow(0, 0, logits.dim(0)? - 1)?.contiguous()?;
let shift_labels = Tensor::from_slice(&tokens[1..], (tokens.len() - 1,), &Device::Cpu)?;
let loss_fct = cross_entropy_loss(&shift_logits, &shift_labels)?;
let perplexity = loss_fct.exp()?.to_scalar::<f32>()?;
let end = Instant::now();
ppl_measurements.push(perplexity);
println!(
"Chunk {i}/{n_chunks} ({} tokens): Perplexity for `{}`, ISQ `{:?}`, {}s: {perplexity}",
tokens.len(),
args.file,
quant,
end.duration_since(start).as_secs_f32(),
);
}
let mean = ppl_measurements.iter().sum::<f32>() / ppl_measurements.len() as f32;
let variance = ppl_measurements
.iter()
.map(|e| (mean - e).powf(2.))
.sum::<f32>()
/ ppl_measurements.len() as f32;
let std_dev = variance.sqrt();
println!();
println!(
"Final perplexity for `{}`, ISQ `{:?}`: {}±{} ppl",
args.file, quant, mean, std_dev
);
Ok(())
}