mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
656 lines
23 KiB
Rust
656 lines
23 KiB
Rust
// Copyright 2026 Hanzo AI, Inc.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
//! CPU release benchmark harness for Hanzo Engine hot paths:
|
|
//! 1. Clean-room `hanzo-tokenizer` + `PromptTokenCache` vs `tokenizers` (single-shot & 20-turn)
|
|
//! 2. `O(1)` incremental `SuffixAutomaton` + `SA_SPEC` bypass vs naive n-gram scan
|
|
//! 3. Cached vs uncached `minijinja` 12-turn + 2-tool chat template compilation/rendering
|
|
//! 4. `hanzo-vision` SIMD RGB resize + planar CHW normalization (`1080p` & `4K`)
|
|
//! 5. `hanzo-kv-fabric` epoch-fenced KV directory + `HKV1` TCP KV block transport
|
|
|
|
use std::collections::HashMap;
|
|
use std::path::PathBuf;
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use anyhow::Result;
|
|
use hanzo_kv_fabric::{
|
|
compute_block_digest, kv_block_hash, EpochFencedKvDirectory, InMemoryKvBackend, KvBlockMeta,
|
|
KvBlockTransport,
|
|
};
|
|
use hanzo_ml::Device;
|
|
use hanzo_tokenizer::{PromptTokenCache, Tokenizer as FastTokenizer};
|
|
use hanzo_vision::{
|
|
simd_resize_rgb, ApplyTransforms, ImageTransform, Normalize, ToNormalizedTensor, ToTensor,
|
|
Transforms,
|
|
};
|
|
use image::{imageops::FilterType, DynamicImage, RgbImage};
|
|
use minijinja::{context, Environment};
|
|
use serde::Serialize;
|
|
use tokenizers::Tokenizer as HfTokenizer;
|
|
|
|
fn percentile_us(samples: &mut [Duration], pct: f64) -> f64 {
|
|
samples.sort_unstable();
|
|
let idx = (((samples.len() - 1) as f64) * pct).round() as usize;
|
|
samples[idx].as_secs_f64() * 1_000_000.0
|
|
}
|
|
|
|
fn percentile_ms(samples: &mut [Duration], pct: f64) -> f64 {
|
|
percentile_us(samples, pct) / 1_000.0
|
|
}
|
|
|
|
fn naive_lookup(
|
|
toks: &[u32],
|
|
ngram_min: usize,
|
|
ngram_max: usize,
|
|
gamma: usize,
|
|
) -> (usize, Vec<u32>) {
|
|
let len = toks.len();
|
|
let longest = ngram_max.min(len.saturating_sub(1));
|
|
for n in (ngram_min..=longest).rev() {
|
|
let needle = &toks[len - n..];
|
|
for start in (0..len - n).rev() {
|
|
if &toks[start..start + n] == needle {
|
|
let from = start + n;
|
|
let to = (from + gamma).min(len);
|
|
return (n, toks[from..to].to_vec());
|
|
}
|
|
}
|
|
}
|
|
(0, Vec::new())
|
|
}
|
|
|
|
fn synthetic_bytelevel_bpe_json() -> Vec<u8> {
|
|
let mut bs: Vec<u32> = (u32::from(b'!')..=u32::from(b'~'))
|
|
.chain(0xa1u32..=0xac)
|
|
.chain(0xaeu32..=0xff)
|
|
.collect();
|
|
let mut cs = bs.clone();
|
|
let mut n = 0u32;
|
|
for b in 0..256u32 {
|
|
if !bs.contains(&b) {
|
|
bs.push(b);
|
|
cs.push(256 + n);
|
|
n += 1;
|
|
}
|
|
}
|
|
let mut vocab = serde_json::Map::new();
|
|
vocab.insert("<|im_start|>".to_string(), serde_json::json!(0));
|
|
vocab.insert("<|im_end|>".to_string(), serde_json::json!(1));
|
|
let mut next_id = 2u32;
|
|
for code in cs {
|
|
let ch = char::from_u32(code).unwrap();
|
|
vocab.insert(ch.to_string(), serde_json::json!(next_id));
|
|
next_id += 1;
|
|
}
|
|
for tok in [
|
|
"fn", "re", "et", "ret", "ur", "rn", "retu", "return", "in", "nt", "int",
|
|
] {
|
|
vocab.insert(tok.to_string(), serde_json::json!(next_id));
|
|
next_id += 1;
|
|
}
|
|
|
|
let doc = serde_json::json!({
|
|
"version": "1.0",
|
|
"added_tokens": [
|
|
{"id": 0, "content": "<|im_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
|
|
{"id": 1, "content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}
|
|
],
|
|
"normalizer": { "type": "NFC" },
|
|
"pre_tokenizer": {
|
|
"type": "Sequence",
|
|
"pretokenizers": [
|
|
{
|
|
"type": "Split",
|
|
"pattern": { "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" },
|
|
"behavior": "Isolated",
|
|
"invert": false
|
|
},
|
|
{"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}
|
|
]
|
|
},
|
|
"model": {
|
|
"type": "BPE",
|
|
"vocab": vocab,
|
|
"merges": ["f n", "r e", "e t", "re t", "u r", "r n", "ret u", "retu rn", "i n", "n t", "in t"]
|
|
}
|
|
});
|
|
serde_json::to_vec(&doc).unwrap()
|
|
}
|
|
|
|
fn discover_hf_tokenizers() -> Vec<(String, PathBuf)> {
|
|
let mut bases = Vec::new();
|
|
if let Ok(hf_home) = std::env::var("HF_HOME") {
|
|
bases.push(PathBuf::from(hf_home).join("hub"));
|
|
}
|
|
if let Ok(home) = std::env::var("HOME") {
|
|
bases.push(
|
|
PathBuf::from(home)
|
|
.join(".cache")
|
|
.join("huggingface")
|
|
.join("hub"),
|
|
);
|
|
}
|
|
|
|
let targets = [
|
|
("Qwen3-8B", "models--Qwen--Qwen3-8B"),
|
|
("Kimi-K2.5", "models--hoangquan456--Kimi-K2.5"),
|
|
("gpt-oss-120b", "models--openai--gpt-oss-120b"),
|
|
("GLM-4.7", "models--zai-org--GLM-4.7"),
|
|
(
|
|
"Nemotron-3-Nano-30B",
|
|
"models--nvidia--NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
|
|
),
|
|
("DeepSeek-V3.2", "models--deepseek-ai--DeepSeek-V3.2"),
|
|
];
|
|
|
|
let mut out = Vec::new();
|
|
for (label, dir) in targets {
|
|
for base in &bases {
|
|
let snaps = base.join(dir).join("snapshots");
|
|
if let Ok(entries) = std::fs::read_dir(&snaps) {
|
|
for entry in entries.flatten() {
|
|
let p = entry.path().join("tokenizer.json");
|
|
if p.is_file() {
|
|
out.push((label.to_string(), p));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if out.last().is_some_and(|(n, _)| n == label) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct TokenizerBenchRow {
|
|
model: String,
|
|
hf_single_us: f64,
|
|
hanzo_single_us: f64,
|
|
speedup_vs_hf: f64,
|
|
hanzo_20t_cold_us: f64,
|
|
hanzo_20t_cached_us: f64,
|
|
cache_hits: u64,
|
|
cache_bypassed: u64,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct SamBenchRow {
|
|
ctx_len: usize,
|
|
hit_naive_us: f64,
|
|
hit_sam_us: f64,
|
|
miss_naive_us: f64,
|
|
miss_sam_us: f64,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct VisionBenchRow {
|
|
shape: String,
|
|
baseline_ms: f64,
|
|
simd_ms: f64,
|
|
speedup: f64,
|
|
max_abs_diff: f32,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct HotpathReport {
|
|
tokenizer: Vec<TokenizerBenchRow>,
|
|
sam: Vec<SamBenchRow>,
|
|
sa_spec_bypassed_rounds: usize,
|
|
sa_spec_total_rounds: usize,
|
|
chat_template_uncached_us: f64,
|
|
chat_template_cached_us: f64,
|
|
vision: Vec<VisionBenchRow>,
|
|
kv_dir_512_p50_us: f64,
|
|
kv_tcp_256k_push_us: f64,
|
|
kv_tcp_256k_pull_us: f64,
|
|
}
|
|
|
|
fn bench_tokenizer_pair(
|
|
label: &str,
|
|
hf_tok: &HfTokenizer,
|
|
fast_tok: &FastTokenizer,
|
|
) -> Result<TokenizerBenchRow> {
|
|
let unit = "def compute_norm(xs: list[float]) -> float:\n # café naïve 東京 🚀\n return sum(x * x for x in xs)\n";
|
|
let large_prompt = unit.repeat(320); // ~33 KiB
|
|
|
|
let hf_ids = hf_tok.encode_fast(large_prompt.as_str(), true).unwrap();
|
|
let hz_ids = fast_tok.encode_with_special_tokens(&large_prompt, true)?;
|
|
anyhow::ensure!(
|
|
hf_ids.get_ids() == hz_ids.as_slice(),
|
|
"{label}: token parity mismatch on 33 KiB prompt"
|
|
);
|
|
|
|
let iters = 15;
|
|
let mut hf_times = Vec::with_capacity(iters);
|
|
let mut hz_times = Vec::with_capacity(iters);
|
|
for _ in 0..iters {
|
|
let t0 = Instant::now();
|
|
let e = hf_tok.encode_fast(large_prompt.as_str(), true).unwrap();
|
|
hf_times.push(t0.elapsed());
|
|
std::hint::black_box(e);
|
|
|
|
let t1 = Instant::now();
|
|
let ids = fast_tok.encode_with_special_tokens(&large_prompt, true)?;
|
|
hz_times.push(t1.elapsed());
|
|
std::hint::black_box(ids);
|
|
}
|
|
let hf_p50 = percentile_us(&mut hf_times, 0.50);
|
|
let hz_p50 = percentile_us(&mut hz_times, 0.50);
|
|
|
|
// 20-turn incremental multi-turn benchmark
|
|
let mut turns = Vec::with_capacity(20);
|
|
let mut conv = String::from("<|im_start|>system\nYou are an assistant.<|im_end|>\n");
|
|
for t in 0..20 {
|
|
conv.push_str(&format!(
|
|
"<|im_start|>user\nRefactor fn step_{t}(x: i32) -> i32 {{ x + {t} }}<|im_end|>\n\
|
|
<|im_start|>assistant\n```rust\nfn step_{t}(x: i32) -> i32 {{ x + {t} }}\n```<|im_end|>\n"
|
|
));
|
|
turns.push(conv.clone());
|
|
}
|
|
// Verify 20-turn PromptTokenCache parity against HF outside timed loops.
|
|
let mut parity_cache = PromptTokenCache::new(16);
|
|
for p in &turns {
|
|
let got = fast_tok.encode_with_cache(p, true, &mut parity_cache)?;
|
|
let exp = hf_tok.encode_fast(p.as_str(), true).unwrap();
|
|
anyhow::ensure!(
|
|
got.as_slice() == exp.get_ids(),
|
|
"{label}: 20-turn PromptTokenCache parity mismatch"
|
|
);
|
|
}
|
|
|
|
// Warmup cold path before measuring median 20-turn cold vs cached latency.
|
|
for p in &turns {
|
|
std::hint::black_box(fast_tok.encode_with_special_tokens(p, true)?);
|
|
}
|
|
|
|
let mut cold_samples = Vec::with_capacity(5);
|
|
for _ in 0..5 {
|
|
let t_cold = Instant::now();
|
|
for p in &turns {
|
|
std::hint::black_box(fast_tok.encode_with_special_tokens(p, true)?);
|
|
}
|
|
cold_samples.push(t_cold.elapsed().as_secs_f64() * 1_000_000.0);
|
|
}
|
|
cold_samples.sort_by(f64::total_cmp);
|
|
let cold_20t_us = cold_samples[cold_samples.len() / 2];
|
|
|
|
let mut cached_samples = Vec::with_capacity(5);
|
|
let mut stats = parity_cache.stats();
|
|
for _ in 0..5 {
|
|
let mut cache = PromptTokenCache::new(16);
|
|
let t_cached = Instant::now();
|
|
for p in &turns {
|
|
std::hint::black_box(fast_tok.encode_with_cache(p, true, &mut cache)?);
|
|
}
|
|
cached_samples.push(t_cached.elapsed().as_secs_f64() * 1_000_000.0);
|
|
stats = cache.stats();
|
|
}
|
|
cached_samples.sort_by(f64::total_cmp);
|
|
let cached_20t_us = cached_samples[cached_samples.len() / 2];
|
|
|
|
println!(
|
|
" tokenizer [{label:<20}]: hf={hf_p50:>8.1}us hanzo={hz_p50:>7.1}us ({:.2}x) | 20T cold={cold_20t_us:>7.1}us cached={cached_20t_us:>7.1}us (hits={} bypassed={})",
|
|
hf_p50 / hz_p50.max(1e-6),
|
|
stats.exact_hits + stats.prefix_hits,
|
|
stats.bypassed
|
|
);
|
|
|
|
Ok(TokenizerBenchRow {
|
|
model: label.to_string(),
|
|
hf_single_us: hf_p50,
|
|
hanzo_single_us: hz_p50,
|
|
speedup_vs_hf: hf_p50 / hz_p50.max(1e-6),
|
|
hanzo_20t_cold_us: cold_20t_us,
|
|
hanzo_20t_cached_us: cached_20t_us,
|
|
cache_hits: stats.exact_hits + stats.prefix_hits,
|
|
cache_bypassed: stats.bypassed,
|
|
})
|
|
}
|
|
|
|
fn bench_speculative_sam() -> Result<(Vec<SamBenchRow>, usize, usize)> {
|
|
let mut rows = Vec::new();
|
|
for &ctx_len in &[1024usize, 4096, 16384] {
|
|
let mut hit_seq: Vec<u32> = (0..ctx_len as u32).map(|i| (i % 97) + 100).collect();
|
|
let needle: Vec<u32> = (0..8u32).map(|i| 10 + i).collect();
|
|
let mid = ctx_len / 2;
|
|
hit_seq[mid..mid + 8].copy_from_slice(&needle);
|
|
hit_seq.extend_from_slice(&needle);
|
|
|
|
let mut miss_seq: Vec<u32> = (0..ctx_len as u32).map(|i| i + 100).collect();
|
|
miss_seq.extend_from_slice(&[999_991, 999_992, 999_993, 999_994]);
|
|
|
|
let mut sam_hit = hanzo_engine::speculative::SuffixAutomaton::new();
|
|
let mut sam_miss = hanzo_engine::speculative::SuffixAutomaton::new();
|
|
let _ = sam_hit.lookup(&hit_seq, 2, 8, 6);
|
|
let _ = sam_miss.lookup(&miss_seq, 2, 8, 6);
|
|
|
|
let iters = 30;
|
|
let mut h_naive = Vec::with_capacity(iters);
|
|
let mut h_sam = Vec::with_capacity(iters);
|
|
let mut m_naive = Vec::with_capacity(iters);
|
|
let mut m_sam = Vec::with_capacity(iters);
|
|
|
|
for k in 0..iters {
|
|
let tok = 5000 + k as u32;
|
|
hit_seq.push(tok);
|
|
let t0 = Instant::now();
|
|
let n_res = naive_lookup(&hit_seq, 2, 8, 6);
|
|
h_naive.push(t0.elapsed());
|
|
|
|
let t1 = Instant::now();
|
|
let s_res = sam_hit.lookup(&hit_seq, 2, 8, 6);
|
|
h_sam.push(t1.elapsed());
|
|
anyhow::ensure!(n_res == s_res, "SAM hit parity mismatch at len={ctx_len}");
|
|
|
|
miss_seq.push(1_000_000 + k as u32);
|
|
let t2 = Instant::now();
|
|
let mn = naive_lookup(&miss_seq, 2, 8, 6);
|
|
m_naive.push(t2.elapsed());
|
|
|
|
let t3 = Instant::now();
|
|
let ms = sam_miss.lookup(&miss_seq, 2, 8, 6);
|
|
m_sam.push(t3.elapsed());
|
|
anyhow::ensure!(mn == ms, "SAM miss parity mismatch at len={ctx_len}");
|
|
}
|
|
|
|
let row = SamBenchRow {
|
|
ctx_len,
|
|
hit_naive_us: percentile_us(&mut h_naive, 0.50),
|
|
hit_sam_us: percentile_us(&mut h_sam, 0.50),
|
|
miss_naive_us: percentile_us(&mut m_naive, 0.50),
|
|
miss_sam_us: percentile_us(&mut m_sam, 0.50),
|
|
};
|
|
println!(
|
|
" SAM [ctx={:>5}]: hit naive={:.2}us sam={:.2}us | miss naive={:.2}us sam={:.2}us ({:.1}x miss speedup)",
|
|
row.ctx_len,
|
|
row.hit_naive_us,
|
|
row.hit_sam_us,
|
|
row.miss_naive_us,
|
|
row.miss_sam_us,
|
|
row.miss_naive_us / row.miss_sam_us.max(1e-6)
|
|
);
|
|
rows.push(row);
|
|
}
|
|
|
|
// 64-step grounded edit decode SA_SPEC bypass check
|
|
let base: Vec<u32> = (0..256u32).cycle().take(8192).collect();
|
|
let mut seq = base.clone();
|
|
seq.extend_from_slice(&base[400..406]);
|
|
let mut sam = hanzo_engine::speculative::SuffixAutomaton::new();
|
|
let mut bypassed = 0usize;
|
|
let total_steps = 64usize;
|
|
for step in 0..total_steps {
|
|
let (matched_len, prop) = sam.lookup(&seq, 2, 8, 6);
|
|
if matched_len >= 4 && !prop.is_empty() {
|
|
bypassed += 1;
|
|
}
|
|
seq.push(base[406 + step]);
|
|
}
|
|
println!(
|
|
" SA_SPEC grounded edit decode: {bypassed}/{total_steps} neural draft passes bypassed"
|
|
);
|
|
Ok((rows, bypassed, total_steps))
|
|
}
|
|
|
|
fn bench_chat_template() -> Result<(f64, f64)> {
|
|
let tmpl_src = r#"{%- if tools -%}
|
|
<|im_start|>system
|
|
Tools:
|
|
{%- for tool in tools %}
|
|
- {{ tool.name }}: {{ tool.description }}
|
|
{%- endfor %}<|im_end|>
|
|
{%- endif -%}
|
|
{%- for m in messages %}
|
|
<|im_start|>{{ m.role }}
|
|
{{ m.content | trim }}<|im_end|>
|
|
{%- endfor %}
|
|
{%- if add_generation_prompt %}
|
|
<|im_start|>assistant
|
|
{% endif %}"#;
|
|
|
|
let mut messages = Vec::new();
|
|
for i in 0..12 {
|
|
messages.push(HashMap::from([
|
|
(
|
|
"role",
|
|
if i % 2 == 0 { "user" } else { "assistant" }.to_string(),
|
|
),
|
|
(
|
|
"content",
|
|
format!("Turn {i}: update module src/engine_{i}.rs and verify tests."),
|
|
),
|
|
]));
|
|
}
|
|
let tools = [
|
|
HashMap::from([
|
|
("name", "read_file"),
|
|
("description", "Read workspace file lines"),
|
|
]),
|
|
HashMap::from([
|
|
("name", "edit_file"),
|
|
("description", "Apply surgical edit to file"),
|
|
]),
|
|
];
|
|
|
|
let mut cached_env = Environment::new();
|
|
cached_env.set_unknown_method_callback(minijinja_contrib::pycompat::unknown_method_callback);
|
|
cached_env.add_template("chat", tmpl_src)?;
|
|
|
|
let mut uncached_times = Vec::with_capacity(50);
|
|
let mut cached_times = Vec::with_capacity(50);
|
|
for _ in 0..50 {
|
|
let t0 = Instant::now();
|
|
let mut env = Environment::new();
|
|
env.set_unknown_method_callback(minijinja_contrib::pycompat::unknown_method_callback);
|
|
env.add_template("chat", tmpl_src)?;
|
|
let out0 = env.get_template("chat")?.render(
|
|
context! { messages => &messages, tools => &tools, add_generation_prompt => true },
|
|
)?;
|
|
uncached_times.push(t0.elapsed());
|
|
|
|
let t1 = Instant::now();
|
|
let out1 = cached_env.get_template("chat")?.render(
|
|
context! { messages => &messages, tools => &tools, add_generation_prompt => true },
|
|
)?;
|
|
cached_times.push(t1.elapsed());
|
|
anyhow::ensure!(out0 == out1, "chat template output parity mismatch");
|
|
}
|
|
let uncached_us = percentile_us(&mut uncached_times, 0.50);
|
|
let cached_us = percentile_us(&mut cached_times, 0.50);
|
|
println!(
|
|
" chat_template (12T+2tools): uncached={uncached_us:.1}us | cached={cached_us:.1}us ({:.2}x)",
|
|
uncached_us / cached_us.max(1e-6)
|
|
);
|
|
Ok((uncached_us, cached_us))
|
|
}
|
|
|
|
fn bench_vision() -> Result<Vec<VisionBenchRow>> {
|
|
let mut rows = Vec::new();
|
|
let dev = Device::Cpu;
|
|
let baseline_tf = Transforms {
|
|
input: &ToTensor,
|
|
inner_transforms: &[&Normalize {
|
|
mean: vec![0.5, 0.5, 0.5],
|
|
std: vec![0.5, 0.5, 0.5],
|
|
}],
|
|
};
|
|
let fast_tf = ToNormalizedTensor {
|
|
mean: [0.5, 0.5, 0.5],
|
|
std: [0.5, 0.5, 0.5],
|
|
};
|
|
|
|
for (label, w, h) in [("1080p", 1920u32, 1080u32), ("4K", 3840u32, 2160u32)] {
|
|
let mut raw = vec![0u8; (w * h * 3) as usize];
|
|
for (i, b) in raw.iter_mut().enumerate() {
|
|
*b = ((i * 31) % 251) as u8;
|
|
}
|
|
let img = DynamicImage::ImageRgb8(RgbImage::from_raw(w, h, raw).unwrap());
|
|
|
|
let base_t = img
|
|
.resize_exact(896, 896, FilterType::CatmullRom)
|
|
.apply(baseline_tf, &dev)?
|
|
.flatten_all()?
|
|
.to_vec1::<f32>()?;
|
|
let fast_img = simd_resize_rgb(&img, 896, 896)?;
|
|
let ref_norm_on_fast = fast_img
|
|
.clone()
|
|
.apply(baseline_tf, &dev)?
|
|
.flatten_all()?
|
|
.to_vec1::<f32>()?;
|
|
let fast_t = fast_tf
|
|
.map(&fast_img, &dev)?
|
|
.flatten_all()?
|
|
.to_vec1::<f32>()?;
|
|
let max_diff = ref_norm_on_fast
|
|
.iter()
|
|
.zip(fast_t.iter())
|
|
.fold(0.0f32, |acc, (a, b)| acc.max((a - b).abs()));
|
|
anyhow::ensure!(
|
|
max_diff < 1e-5,
|
|
"vision normalization drift too high on {label}: {max_diff}"
|
|
);
|
|
let resize_diff = base_t
|
|
.iter()
|
|
.zip(fast_t.iter())
|
|
.fold(0.0f32, |acc, (a, b)| acc.max((a - b).abs()));
|
|
anyhow::ensure!(
|
|
resize_diff <= (1.0 / 127.5) + 1e-4,
|
|
"vision resize+norm drift exceeds 1 u8 LSB on {label}: {resize_diff}"
|
|
);
|
|
|
|
let iters = 8;
|
|
let mut base_times = Vec::with_capacity(iters);
|
|
let mut fast_times = Vec::with_capacity(iters);
|
|
for _ in 0..iters {
|
|
let t0 = Instant::now();
|
|
let b_out = img
|
|
.resize_exact(896, 896, FilterType::CatmullRom)
|
|
.apply(baseline_tf, &dev)?;
|
|
base_times.push(t0.elapsed());
|
|
std::hint::black_box(b_out);
|
|
|
|
let t1 = Instant::now();
|
|
let r = simd_resize_rgb(&img, 896, 896)?;
|
|
let f_out = fast_tf.map(&r, &dev)?;
|
|
fast_times.push(t1.elapsed());
|
|
std::hint::black_box(f_out);
|
|
}
|
|
let b_ms = percentile_ms(&mut base_times, 0.50);
|
|
let f_ms = percentile_ms(&mut fast_times, 0.50);
|
|
let spd = b_ms / f_ms.max(1e-6);
|
|
println!(
|
|
" vision [{label} -> 896x896 CHW]: baseline={b_ms:.2}ms | hanzo_simd={f_ms:.2}ms ({spd:.2}x, max_diff={max_diff:.2e})"
|
|
);
|
|
rows.push(VisionBenchRow {
|
|
shape: format!("{w}x{h}->896x896"),
|
|
baseline_ms: b_ms,
|
|
simd_ms: f_ms,
|
|
speedup: spd,
|
|
max_abs_diff: max_diff,
|
|
});
|
|
}
|
|
Ok(rows)
|
|
}
|
|
|
|
async fn bench_kv_fabric() -> Result<(f64, f64, f64)> {
|
|
let dir = EpochFencedKvDirectory::new();
|
|
let backend = Arc::new(InMemoryKvBackend::new(1));
|
|
let transport = KvBlockTransport::bind("127.0.0.1:0".parse()?, Arc::clone(&backend)).await?;
|
|
let peer = transport.local_addr();
|
|
|
|
let digest = compute_block_digest(b"bench-meta");
|
|
let metas: Vec<KvBlockMeta> = (0u32..512)
|
|
.map(|i| KvBlockMeta {
|
|
block_hash: kv_block_hash(&[i, i + 1], 0),
|
|
num_bytes: 262_144,
|
|
sha256: digest,
|
|
})
|
|
.collect();
|
|
let mut dir_times = Vec::with_capacity(30);
|
|
for epoch in 1u64..=30 {
|
|
let t0 = Instant::now();
|
|
dir.publish_blocks("worker-0", epoch, peer, &metas)?;
|
|
let _ = dir.lookup_fresh_owners(metas[256].block_hash);
|
|
let _ = dir.fence_epoch("worker-0", epoch + 1);
|
|
dir_times.push(t0.elapsed());
|
|
}
|
|
let dir_p50 = percentile_us(&mut dir_times, 0.50);
|
|
|
|
backend.set_epoch(100);
|
|
let payload = vec![0x5Au8; 256 * 1024];
|
|
let hash = kv_block_hash(&[7, 256], 0);
|
|
let mut push_times = Vec::with_capacity(20);
|
|
let mut pull_times = Vec::with_capacity(20);
|
|
for i in 0..20 {
|
|
let ep = 100 + i as u64;
|
|
let t0 = Instant::now();
|
|
KvBlockTransport::<InMemoryKvBackend>::push_block(peer, hash, ep, &payload).await?;
|
|
push_times.push(t0.elapsed());
|
|
let t1 = Instant::now();
|
|
let (_, got) = KvBlockTransport::<InMemoryKvBackend>::pull_block(peer, hash, ep).await?;
|
|
pull_times.push(t1.elapsed());
|
|
anyhow::ensure!(got.len() == payload.len());
|
|
}
|
|
let push_p50 = percentile_us(&mut push_times, 0.50);
|
|
let pull_p50 = percentile_us(&mut pull_times, 0.50);
|
|
println!(
|
|
" kv_fabric: dir512={dir_p50:.1}us | HKV1 256KiB push={push_p50:.1}us pull={pull_p50:.1}us"
|
|
);
|
|
Ok((dir_p50, push_p50, pull_p50))
|
|
}
|
|
|
|
#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn main() -> Result<()> {
|
|
let args: Vec<String> = std::env::args().collect();
|
|
let json_out = args
|
|
.windows(2)
|
|
.find(|w| w[0] == "--json")
|
|
.map(|w| PathBuf::from(&w[1]));
|
|
|
|
println!("=== Hanzo Hot-Path CPU Release Benchmark ===");
|
|
|
|
let mut tok_rows = Vec::new();
|
|
let synth_bytes = synthetic_bytelevel_bpe_json();
|
|
let synth_hf = HfTokenizer::from_bytes(&synth_bytes).unwrap();
|
|
let synth_fast = FastTokenizer::from_bytes(&synth_bytes)?;
|
|
tok_rows.push(bench_tokenizer_pair(
|
|
"synthetic-cl100k",
|
|
&synth_hf,
|
|
&synth_fast,
|
|
)?);
|
|
|
|
for (label, path) in discover_hf_tokenizers() {
|
|
let hf_tok = HfTokenizer::from_file(&path).unwrap();
|
|
let fast_tok = FastTokenizer::from_file(&path)?;
|
|
tok_rows.push(bench_tokenizer_pair(&label, &hf_tok, &fast_tok)?);
|
|
}
|
|
|
|
let (sam_rows, sa_bypassed, sa_total) = bench_speculative_sam()?;
|
|
let (chat_uncached, chat_cached) = bench_chat_template()?;
|
|
let vision_rows = bench_vision()?;
|
|
let (dir_p50, push_p50, pull_p50) = bench_kv_fabric().await?;
|
|
|
|
if let Some(path) = json_out {
|
|
let report = HotpathReport {
|
|
tokenizer: tok_rows,
|
|
sam: sam_rows,
|
|
sa_spec_bypassed_rounds: sa_bypassed,
|
|
sa_spec_total_rounds: sa_total,
|
|
chat_template_uncached_us: chat_uncached,
|
|
chat_template_cached_us: chat_cached,
|
|
vision: vision_rows,
|
|
kv_dir_512_p50_us: dir_p50,
|
|
kv_tcp_256k_push_us: push_p50,
|
|
kv_tcp_256k_pull_us: pull_p50,
|
|
};
|
|
std::fs::write(&path, serde_json::to_string_pretty(&report)?)?;
|
|
println!("wrote report to {}", path.display());
|
|
}
|
|
|
|
Ok(())
|
|
}
|