Files
engine/hanzo-tokenizer/tests/parity.rs

909 lines
32 KiB
Rust

// Copyright 2026 Hanzo AI, Inc.
// SPDX-License-Identifier: MIT
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::thread;
use hanzo_tokenizer::{PromptCacheStats, PromptTokenCache, Tokenizer, TokenizerError};
use serde_json::json;
fn cached_hf_tokenizer(repo_dir: &str) -> Option<PathBuf> {
let base = PathBuf::from("/home/z/.cache/huggingface/hub")
.join(repo_dir)
.join("snapshots");
let entries = std::fs::read_dir(base).ok()?;
for entry in entries.flatten() {
let cand = entry.path().join("tokenizer.json");
if cand.is_file() {
return Some(cand);
}
}
None
}
fn synthetic_byte_level_config(
normalizer: serde_json::Value,
pre_tokenizer: serde_json::Value,
post_processor: serde_json::Value,
added_tokens: serde_json::Value,
ignore_merges: bool,
array_merges: bool,
) -> Vec<u8> {
let mut vocab = BTreeMap::new();
let mut id = 0u32;
let base_chars: Vec<String> = {
let mut bs: Vec<u8> = (b'!'..=b'~')
.chain(0xA1..=0xAC)
.chain(0xAE..=0xFF)
.collect();
let mut out = Vec::new();
for b in 0u16..=255 {
let b = b as u8;
if let Some(pos) = bs.iter().position(|&x| x == b) {
let _ = pos;
out.push((b as char).to_string());
} else {
let extra_idx = (0u8..b)
.filter(|&x| {
!((b'!'..=b'~').contains(&x)
|| (0xA1..=0xAC).contains(&x)
|| (0xAE..=0xFF).contains(&x))
})
.count();
let cp = 256u32 + extra_idx as u32;
out.push(char::from_u32(cp).unwrap().to_string());
}
}
let _ = &mut bs;
out
};
for s in base_chars {
vocab.insert(s, id);
id += 1;
}
let extra_tokens = [
"he", "ll", "lo", "hel", "llo", "hello", "wo", "rl", "wor", "ld", "world", "Ġw", "Ġwo",
"Ġwor", "Ġworld", "foo", "fo", "bar", "ba", "<unk>", "<bos>", "<eos>", "<pad>", "<spec>",
"<norm>",
];
for t in extra_tokens {
vocab.entry(t.to_string()).or_insert_with(|| {
let cur = id;
id += 1;
cur
});
}
let raw_merges = [
("h", "e"),
("l", "l"),
("ll", "o"),
("he", "llo"),
("w", "o"),
("r", "l"),
("wo", "r"),
("wor", "ld"),
("Ġ", "w"),
("Ġw", "o"),
("Ġwo", "r"),
("f", "o"),
("fo", "o"),
("b", "a"),
("ba", "r"),
];
let merges: serde_json::Value = if array_merges {
serde_json::Value::Array(raw_merges.iter().map(|(a, b)| json!([a, b])).collect())
} else {
serde_json::Value::Array(
raw_merges
.iter()
.map(|(a, b)| json!(format!("{a} {b}")))
.collect(),
)
};
serde_json::to_vec(&json!({
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": added_tokens,
"normalizer": normalizer,
"pre_tokenizer": pre_tokenizer,
"post_processor": post_processor,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<unk>",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": ignore_merges,
"vocab": vocab,
"merges": merges
}
}))
.unwrap()
}
fn synthetic_char_bpe_config(
normalizer: serde_json::Value,
pre_tokenizer: serde_json::Value,
byte_fallback: bool,
fuse_unk: bool,
continuing_subword_prefix: Option<&str>,
end_of_word_suffix: Option<&str>,
unk_token: Option<&str>,
) -> Vec<u8> {
let mut vocab = BTreeMap::new();
let mut id = 0u32;
let mut insert = |s: &str| {
if !vocab.contains_key(s) {
vocab.insert(s.to_string(), id);
id += 1;
}
};
insert("<unk>");
if byte_fallback {
for b in 0u16..=255 {
insert(&format!("<{:#04X}>", b as u8));
}
}
for ch in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 !.,:-_".chars() {
let s = ch.to_string();
insert(&s);
if let Some(p) = continuing_subword_prefix {
insert(&format!("{p}{s}"));
}
if let Some(suf) = end_of_word_suffix {
insert(&format!("{s}{suf}"));
}
if let (Some(p), Some(suf)) = (continuing_subword_prefix, end_of_word_suffix) {
insert(&format!("{p}{s}{suf}"));
}
}
for extra in [
"ab", "bc", "abc", "12", "23", "123", "##bc", "bc</w>", "abc</w>", "##bc</w>", "123</w>",
"a</w>", "b</w>", "c</w>", "1</w>", "2</w>", "3</w>", "##c</w>",
] {
insert(extra);
}
let p = continuing_subword_prefix.unwrap_or("");
let s = end_of_word_suffix.unwrap_or("");
let merges = vec![
format!("a {p}b"),
format!("ab {p}c{s}"),
format!("{p}b {p}c{s}"),
format!("1 {p}2"),
format!("12 {p}3{s}"),
];
serde_json::to_vec(&json!({
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": normalizer,
"pre_tokenizer": pre_tokenizer,
"post_processor": null,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": unk_token,
"continuing_subword_prefix": continuing_subword_prefix,
"end_of_word_suffix": end_of_word_suffix,
"fuse_unk": fuse_unk,
"byte_fallback": byte_fallback,
"ignore_merges": false,
"vocab": vocab,
"merges": merges
}
}))
.unwrap()
}
fn assert_parity_bytes(json_bytes: &[u8], samples: &[&str]) {
let hf = tokenizers::Tokenizer::from_bytes(json_bytes).expect("hf load");
let hz = Tokenizer::from_bytes(json_bytes).expect("hz load");
for &sample in samples {
for &add_special in &[false, true] {
let expected = hf
.encode(sample, add_special)
.expect("hf encode")
.get_ids()
.to_vec();
let actual = hz
.encode_with_special_tokens(sample, add_special)
.expect("hz encode");
assert_eq!(
actual, expected,
"mismatch for add_special={add_special}, sample={sample:?}"
);
}
let default_expected = hf.encode(sample, true).unwrap().get_ids().to_vec();
let default_actual = hz.encode(sample).unwrap();
assert_eq!(default_actual, default_expected);
}
}
#[test]
fn synthetic_normalizers_parity() {
let normalizers = [
json!(null),
json!({"type": "NFC"}),
json!({"type": "NFKC"}),
json!({"type": "NFD"}),
json!({"type": "NFKD"}),
json!({"type": "Lowercase"}),
json!({"type": "Prepend", "prepend": ">>"}),
json!({"type": "Strip", "strip_left": true, "strip_right": true}),
json!({"type": "Replace", "pattern": {"String": "foo"}, "content": "bar"}),
json!({"type": "Replace", "pattern": {"Regex": "\\s+"}, "content": " "}),
json!({"type": "Sequence", "normalizers": [
{"type": "NFC"},
{"type": "Lowercase"},
{"type": "Strip", "strip_left": true, "strip_right": true},
{"type": "Replace", "pattern": {"String": " "}, "content": " "}
]}),
];
let pre = json!({
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": false,
"use_regex": false
});
let samples = [
"",
"hello world",
" Foo WORLD ",
"cafe\u{0301} CAFÉ",
"fi fl ① ② ²",
"foo bar hello",
" ",
];
for (idx, norm) in normalizers.into_iter().enumerate() {
let cfg = synthetic_byte_level_config(
norm,
pre.clone(),
json!(null),
json!([]),
idx % 2 == 0,
idx % 2 == 1,
);
assert_parity_bytes(&cfg, &samples);
}
}
#[test]
fn synthetic_pre_tokenizers_and_ignore_merges_parity() {
let pre_tokenizers = [
json!(null),
json!({"type": "Digits", "individual_digits": true}),
json!({"type": "Digits", "individual_digits": false}),
json!({"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": false, "use_regex": true}),
json!({"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": true}),
json!({"type": "Split", "pattern": {"String": " "}, "behavior": "Isolated", "invert": false}),
json!({"type": "Split", "pattern": {"String": " "}, "behavior": "Removed", "invert": false}),
json!({"type": "Split", "pattern": {"String": " "}, "behavior": "MergedWithPrevious", "invert": false}),
json!({"type": "Split", "pattern": {"String": " "}, "behavior": "MergedWithNext", "invert": false}),
json!({"type": "Split", "pattern": {"Regex": "[a-z]+"}, "behavior": "Isolated", "invert": true}),
json!({"type": "Split", "pattern": {"Regex": "\\w+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false}),
json!({"type": "Sequence", "pretokenizers": [
{"type": "Split", "pattern": {"Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false},
{"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false}
]}),
];
let samples = [
"",
"hello world",
" hello world! \n\nI'm 12345 ",
"foobar hello world",
"a b c d\te\nf",
"123abc456def",
" ",
];
for (i, pre) in pre_tokenizers.into_iter().enumerate() {
let is_byte_level = serde_json::to_string(&pre).unwrap().contains("ByteLevel");
let cfg = if is_byte_level {
synthetic_byte_level_config(json!(null), pre, json!(null), json!([]), i % 2 == 0, true)
} else {
synthetic_char_bpe_config(json!(null), pre, false, false, None, None, Some("<unk>"))
};
assert_parity_bytes(&cfg, &samples);
}
// Verify ignore_merges forces direct whole-word lookup without merging "Ġwor" + "ld" -> wait
// Note in synthetic vocab "Ġworld" IS in vocab, while merge table stops at "Ġwor" without merging "Ġwor ld"!
let pre_seq = json!({
"type": "Sequence",
"pretokenizers": [
{"type": "Split", "pattern": {"String": " "}, "behavior": "MergedWithNext", "invert": false},
{"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false}
]
});
let cfg_ignore = synthetic_byte_level_config(
json!(null),
pre_seq.clone(),
json!(null),
json!([]),
true,
true,
);
let cfg_no_ignore =
synthetic_byte_level_config(json!(null), pre_seq, json!(null), json!([]), false, true);
let hz_ign = Tokenizer::from_bytes(&cfg_ignore).unwrap();
let hz_no = Tokenizer::from_bytes(&cfg_no_ignore).unwrap();
let t_ign = hz_ign.encode("hello world").unwrap();
let t_no = hz_no.encode("hello world").unwrap();
assert_ne!(t_ign, t_no);
assert_parity_bytes(&cfg_ignore, &["hello world", "foo bar"]);
assert_parity_bytes(&cfg_no_ignore, &["hello world", "foo bar"]);
}
#[test]
fn synthetic_char_bpe_byte_fallback_fuse_unk_prefix_suffix_parity() {
let cases = [
(true, false, None, None, Some("<unk>")),
(false, true, None, None, Some("<unk>")),
(false, false, Some("##"), None, Some("<unk>")),
(false, false, None, Some("</w>"), Some("<unk>")),
(false, false, Some("##"), Some("</w>"), Some("<unk>")),
];
let pre = json!({"type": "Split", "pattern": {"String": " "}, "behavior": "Removed", "invert": false});
let samples = [
"abc 123",
"abc ??? 123",
"hello 世界 café 🚀",
"abc def ghi",
];
for (bf, fu, cp, eow, unk) in cases {
let cfg = synthetic_char_bpe_config(json!(null), pre.clone(), bf, fu, cp, eow, unk);
assert_parity_bytes(&cfg, &samples);
}
}
#[test]
fn synthetic_added_tokens_and_template_post_processor_parity() {
let added = json!([
{"id": 276, "content": "<bos>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
{"id": 277, "content": "<eos>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
{"id": 279, "content": "<spec>", "single_word": true, "lstrip": true, "rstrip": true, "normalized": false, "special": false},
{"id": 281, "content": "<NORM>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false},
{"id": 278, "content": "<pad>", "single_word": true, "lstrip": false, "rstrip": false, "normalized": false, "special": true}
]);
let post = json!({
"type": "Sequence",
"processors": [
{"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false},
{
"type": "TemplateProcessing",
"single": [
{"SpecialToken": {"id": "<bos>", "type_id": 0}},
{"Sequence": {"id": "A", "type_id": 0}},
{"SpecialToken": {"id": "<eos>", "type_id": 0}}
],
"pair": [
{"SpecialToken": {"id": "<bos>", "type_id": 0}},
{"Sequence": {"id": "A", "type_id": 0}},
{"SpecialToken": {"id": "<eos>", "type_id": 0}},
{"Sequence": {"id": "B", "type_id": 1}},
{"SpecialToken": {"id": "<eos>", "type_id": 1}}
],
"special_tokens": {
"<bos>": {"id": "<bos>", "ids": [276], "tokens": ["<bos>"]},
"<eos>": {"id": "<eos>", "ids": [277], "tokens": ["<eos>"]}
}
}
]
});
let pre = json!({
"type": "Sequence",
"pretokenizers": [
{"type": "Split", "pattern": {"String": " "}, "behavior": "Isolated", "invert": false},
{"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false}
]
});
let cfg =
synthetic_byte_level_config(json!({"type": "Lowercase"}), pre, post, added, false, true);
let samples = [
"",
"hello world",
"<bos>hello<eos>",
"hello <spec> world",
"foo<spec>bar",
"foo <spec>bar",
"<norm> <NORM> hello <pad> foo<pad>bar",
"hello <pad> <spec> world",
];
assert_parity_bytes(&cfg, &samples);
}
#[test]
fn unsupported_features_return_explicit_errors() {
let base_pre = json!({"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false});
// 1. Unknown normalizer
let bad_norm = synthetic_byte_level_config(
json!({"type": "BertNormalizer"}),
base_pre.clone(),
json!(null),
json!([]),
false,
true,
);
assert!(matches!(
Tokenizer::from_bytes(&bad_norm),
Err(TokenizerError::Unsupported(_))
));
// 2. Arbitrary lookaround in Split
let bad_split = synthetic_byte_level_config(
json!(null),
json!({"type": "Split", "pattern": {"Regex": "(?<=a)b"}, "behavior": "Isolated", "invert": false}),
json!(null),
json!([]),
false,
true,
);
assert!(matches!(
Tokenizer::from_bytes(&bad_split),
Err(TokenizerError::Unsupported(_))
));
// 3. Unsupported TemplateProcessing with Sequence B in single
let bad_pp = synthetic_byte_level_config(
json!(null),
base_pre.clone(),
json!({
"type": "TemplateProcessing",
"single": [{"Sequence": {"id": "A", "type_id": 0}}, {"Sequence": {"id": "B", "type_id": 1}}],
"special_tokens": {}
}),
json!([]),
false,
true,
);
assert!(matches!(
Tokenizer::from_bytes(&bad_pp),
Err(TokenizerError::Unsupported(_))
));
// 4. Non-null truncation / padding / non-1.0 version / WordPiece model
let mut doc: serde_json::Value = serde_json::from_slice(&synthetic_byte_level_config(
json!(null),
base_pre,
json!(null),
json!([]),
false,
true,
))
.unwrap();
let mut doc_trunc = doc.clone();
doc_trunc["truncation"] = json!({"max_length": 128});
assert!(matches!(
Tokenizer::from_bytes(&serde_json::to_vec(&doc_trunc).unwrap()),
Err(TokenizerError::Unsupported(_))
));
let mut doc_pad = doc.clone();
doc_pad["padding"] = json!({"strategy": "BatchLongest"});
assert!(matches!(
Tokenizer::from_bytes(&serde_json::to_vec(&doc_pad).unwrap()),
Err(TokenizerError::Unsupported(_))
));
let mut doc_ver = doc.clone();
doc_ver["version"] = json!("2.0");
assert!(matches!(
Tokenizer::from_bytes(&serde_json::to_vec(&doc_ver).unwrap()),
Err(TokenizerError::Unsupported(_))
));
doc["model"]["type"] = json!("WordPiece");
assert!(matches!(
Tokenizer::from_bytes(&serde_json::to_vec(&doc).unwrap()),
Err(TokenizerError::Unsupported(_))
));
}
#[test]
fn cached_hf_models_full_parity() {
let models = [
"models--Qwen--Qwen3-8B",
"models--hoangquan456--Kimi-K2.5",
"models--openai--gpt-oss-120b",
"models--zai-org--GLM-4.7",
"models--nvidia--NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
"models--deepseek-ai--DeepSeek-V3.2",
];
let corpus = [
"",
" ",
" ",
"\n\n\n \t \r\n",
"Hello, world!",
"I'm don't we're they've I'll he'd IT'S WE'RE",
"foo1234567890bar 1 12 123 1234 999999999",
"http://example.com/a/b/c?x=1&y=2#frag\n/usr/local/bin/cargo\n// comment /// doc",
"fn main() -> Result<(), Box<dyn std::error::Error>> {\n let xs = vec![1, 2, 3];\n println!(\"{xs:?}\");\n Ok(())\n}\n",
"{\"role\":\"assistant\",\"content\":\"hello\\nworld\",\"ok\":true}\n",
"Café naïve résumé — e\u{0301} a\u{0300} o\u{0302} fi fl ①",
"你好世界!こんにちは世界。안녕하세요! 123日本語456中文。",
"مرحبا بالعالم שלום עולם नमस्ते दुनिया ไทย ภาษาไทย",
"Emoji ZWJ: 👨‍👩‍👧‍👦 👍🏽 🏳️‍🌈 © ™ ∞ ≠ ≈ ∑ √",
"Mixed spaces: foo \n bar baz\t\t\r\nqux ",
"<|endoftext|>user\nHello!<think>step 1</think>Done.",
];
let large_prompt = {
let unit = "The quick brown fox jumps over 123 lazy dogs! Café 世界 🚀 // path/to/file\n";
let mut s = String::with_capacity(unit.len() * 400);
for _ in 0..400 {
s.push_str(unit);
}
s
};
let mut tested_count = 0usize;
for repo in models {
let Some(path) = cached_hf_tokenizer(repo) else {
continue;
};
tested_count += 1;
let bytes = std::fs::read(&path).expect("read cached tokenizer.json");
let hf = tokenizers::Tokenizer::from_bytes(&bytes).expect("hf load");
let hz = Tokenizer::from_bytes(&bytes).expect("hz load");
for &sample in &corpus {
for &add_special in &[false, true] {
let expected = hf.encode(sample, add_special).unwrap().get_ids().to_vec();
let actual = hz.encode_with_special_tokens(sample, add_special).unwrap();
assert_eq!(
actual, expected,
"parity mismatch in {repo} (add_special={add_special}) on {sample:?}"
);
}
}
let expected_large = hf
.encode(large_prompt.as_str(), true)
.unwrap()
.get_ids()
.to_vec();
let actual_large = hz.encode(&large_prompt).unwrap();
assert_eq!(
actual_large, expected_large,
"large parallel parity mismatch in {repo}"
);
}
assert!(
tested_count > 0,
"expected at least one cached HF model under /home/z/.cache/huggingface/hub"
);
}
#[test]
fn unicode_scalar_parity_against_hf_onig() {
let models = [
"models--Qwen--Qwen3-8B",
"models--hoangquan456--Kimi-K2.5",
"models--openai--gpt-oss-120b",
"models--zai-org--GLM-4.7",
"models--nvidia--NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
"models--deepseek-ai--DeepSeek-V3.2",
];
// Sample all Unicode scalars in batches separated by lowercase letters and spaces.
// Using `a{ch}a ` probes both preceding (`U*L+` / `[^\r\n\p{L}\p{N}]?`) and trailing
// (`U+L*` / `[punct][A-Za-z]+`) transitions across all 6 regex families.
let all_chars: Vec<char> = (0u32..=0x10_FFFF)
.filter_map(char::from_u32)
.filter(|&c| c != '\0')
.collect();
for repo in models {
let Some(path) = cached_hf_tokenizer(repo) else {
continue;
};
let bytes = std::fs::read(&path).expect("read tokenizer.json");
let hf = tokenizers::Tokenizer::from_bytes(&bytes).expect("hf load");
let hz = Tokenizer::from_bytes(&bytes).expect("hz load");
let check_slice = |chars: &[char]| -> bool {
let mut s = String::with_capacity(chars.len() * 8);
for &ch in chars {
s.push('a');
s.push(ch);
s.push('a');
s.push(' ');
}
let expected = hf.encode(s.as_str(), false).unwrap().get_ids().to_vec();
let actual = hz.encode_with_special_tokens(&s, false).unwrap();
expected == actual
};
for chunk in all_chars.chunks(256) {
if !check_slice(chunk) {
let mut lo = 0usize;
let mut hi = chunk.len();
while hi - lo > 1 {
let mid = (lo + hi) / 2;
if check_slice(&chunk[lo..mid]) {
lo = mid;
} else {
hi = mid;
}
}
let bad = chunk[lo];
let probes = [
format!("{bad}"),
format!("a{bad}a "),
format!("a{bad}b"),
format!(" {bad} "),
format!("\n{bad}x"),
];
for p in probes {
let exp = hf.encode(p.as_str(), false).unwrap().get_ids().to_vec();
let act = hz.encode_with_special_tokens(&p, false).unwrap();
assert_eq!(
act, exp,
"Unicode parity mismatch in {repo} for U+{:04X} ({bad:?}) on {p:?}",
bad as u32
);
}
panic!(
"Unicode parity batch mismatch in {repo} around U+{:04X} ({bad:?})",
bad as u32
);
}
}
}
}
#[test]
fn file_loading_and_invalid_config_errors() {
let missing = Tokenizer::from_file("/nonexistent/path/to/tokenizer.json");
assert!(matches!(missing, Err(TokenizerError::Io(_))));
let bad_json = Tokenizer::from_bytes(b"{not valid json");
assert!(matches!(bad_json, Err(TokenizerError::Json(_))));
let bad_merge = json!({
"version": "1.0",
"added_tokens": [],
"normalizer": null,
"pre_tokenizer": null,
"post_processor": null,
"model": {
"type": "BPE",
"continuing_subword_prefix": "##",
"vocab": {"a": 0, "b": 1, "ab": 2},
"merges": ["a b"]
}
});
assert!(matches!(
Tokenizer::from_bytes(&serde_json::to_vec(&bad_merge).unwrap()),
Err(TokenizerError::InvalidConfig(_))
));
if let Some(path) = cached_hf_tokenizer("models--Qwen--Qwen3-8B") {
let tok = Tokenizer::from_file(path).unwrap();
assert!(!tok.encode("hello").unwrap().is_empty());
}
}
#[test]
fn prompt_cache_hits_boundary_safety_and_auto_bypass() {
// 1. Test TemplateProcessing (BOS + EOS) + ByteLevel checkpoint parity
let pre = json!({
"type": "Sequence",
"pretokenizers": [
{"type": "Split", "pattern": {"Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false},
{"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false}
]
});
let added = json!([
{"id": 276, "content": "<bos>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true},
{"id": 277, "content": "<eos>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}
]);
let post = json!({
"type": "TemplateProcessing",
"single": [
{"SpecialToken": {"id": "<bos>", "type_id": 0}},
{"Sequence": {"id": "A", "type_id": 0}},
{"SpecialToken": {"id": "<eos>", "type_id": 0}}
],
"special_tokens": {
"<bos>": {"id": "<bos>", "ids": [276], "tokens": ["<bos>"]},
"<eos>": {"id": "<eos>", "ids": [277], "tokens": ["<eos>"]}
}
});
let cfg = synthetic_byte_level_config(json!({"type": "NFC"}), pre, post, added, false, true);
let hz = Tokenizer::from_bytes(&cfg).unwrap();
let mut cache = PromptTokenCache::new(4);
// Multi-turn incremental growth with non-ASCII NFD->NFC before `\n`!
let t1 = "Cafe\u{0301} line 1\n";
let t2 = "Cafe\u{0301} line 1\nhello world\n";
let t3 = "Cafe\u{0301} line 1\nhello world\n<bos>tail\n";
let t4_slash = "Cafe\u{0301} line 1\nhello world\n/slash/path";
let t5_quote = "Cafe\u{0301} line 1\nhello world\n's contraction";
for &prompt in &[t1, t2, t3, t4_slash, t5_quote] {
let cached_out = hz.encode_with_cache(prompt, true, &mut cache).unwrap();
let uncached_out = hz.encode_with_special_tokens(prompt, true).unwrap();
assert_eq!(cached_out, uncached_out, "prompt={prompt:?}");
}
let stats = cache.stats();
assert_eq!(stats.misses, 1);
assert_eq!(stats.prefix_hits, 4);
// Exact hit
assert_eq!(
hz.encode_with_cache(t2, true, &mut cache).unwrap(),
hz.encode_with_special_tokens(t2, true).unwrap()
);
assert_eq!(cache.stats().exact_hits, 1);
// Zero-capacity cache always bypasses
let mut zero_cache = PromptTokenCache::new(0);
assert_eq!(
hz.encode_with_cache(t1, true, &mut zero_cache).unwrap(),
hz.encode_with_special_tokens(t1, true).unwrap()
);
assert_eq!(
zero_cache.stats(),
PromptCacheStats {
exact_hits: 0,
prefix_hits: 0,
misses: 0,
bypassed: 1,
}
);
// 2. Test cached HF models with PromptTokenCache multi-turn + auto-bypass
for repo in [
"models--Qwen--Qwen3-8B",
"models--openai--gpt-oss-120b",
"models--deepseek-ai--DeepSeek-V3.2",
] {
let Some(path) = cached_hf_tokenizer(repo) else {
continue;
};
let model_tok = Tokenizer::from_file(path).unwrap();
let mut mcache = PromptTokenCache::new(8);
let mut acc = String::from("<|im_start|>system\nYou are helpful!<|im_end|>\n");
let _ = model_tok
.encode_with_cache(&acc, true, &mut mcache)
.unwrap();
for turn in 0..5 {
acc.push_str(&format!("<|im_start|>user\nTurn {turn} café!\n"));
let got = model_tok
.encode_with_cache(&acc, true, &mut mcache)
.unwrap();
let exp = model_tok.encode_with_special_tokens(&acc, true).unwrap();
assert_eq!(got, exp, "incremental parity failed on {repo} turn {turn}");
}
assert!(mcache.stats().prefix_hits >= 4);
// Online self-bypass check on 15 completely unrelated single-line misses:
// 2 consecutive misses -> 8 bypassed -> 1 recovery probe (turns out miss) -> bypassed...
let mut uc = PromptTokenCache::new(4);
for i in 0..14 {
let p = format!("unrelated prompt number {i} without prefix overlap");
let got = model_tok.encode_with_cache(&p, true, &mut uc).unwrap();
let exp = model_tok.encode(&p).unwrap();
assert_eq!(got, exp);
}
let ustats = uc.stats();
assert!(ustats.bypassed >= 8, "expected auto-bypass, got {ustats:?}");
assert!(
ustats.misses <= 4,
"expected bounded misses, got {ustats:?}"
);
// Recovery after cooldown when incremental prefix resumes:
// First flush remaining 5 cooldown calls, insert `p_base` on the recovery probe,
// flush the subsequent 8 cooldown calls, then verify the next probe hits `p_base`.
for _ in 0..5 {
let _ = model_tok
.encode_with_cache("drain cooldown", true, &mut uc)
.unwrap();
}
let p_base = "system header line\n";
let _ = model_tok.encode_with_cache(p_base, true, &mut uc).unwrap();
for _ in 0..8 {
let _ = model_tok
.encode_with_cache("drain cooldown", true, &mut uc)
.unwrap();
}
let got_rec = model_tok
.encode_with_cache("system header line\nnext line", true, &mut uc)
.unwrap();
assert_eq!(
got_rec,
model_tok.encode("system header line\nnext line").unwrap()
);
assert!(uc.stats().prefix_hits > 0);
// Once recovered, consecutive hits stay active with zero bypass increment
let bypassed_before = uc.stats().bypassed;
let got_exact = model_tok
.encode_with_cache("system header line\nnext line", true, &mut uc)
.unwrap();
assert_eq!(got_exact, got_rec);
assert_eq!(uc.stats().exact_hits, 1);
assert_eq!(uc.stats().bypassed, bypassed_before);
}
}
#[test]
fn concurrent_16_thread_encoding_parity() {
let pre = json!({
"type": "Sequence",
"pretokenizers": [
{"type": "Split", "pattern": {"Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false},
{"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": false, "use_regex": false}
]
});
let tok = if let Some(path) = cached_hf_tokenizer("models--Qwen--Qwen3-8B") {
Tokenizer::from_file(path).unwrap()
} else {
let cfg = synthetic_byte_level_config(
json!({"type": "NFC"}),
pre,
json!(null),
json!([]),
false,
true,
);
Tokenizer::from_bytes(&cfg).unwrap()
};
let samples: Vec<String> = (0..16)
.map(|i| {
format!(
"Thread {i}: hello world! Café 你好 世界 🚀 // path/item_{i} 123456\n{}\n",
"repeated_segment ".repeat(20)
)
})
.collect();
let expected: Vec<Vec<u32>> = samples.iter().map(|s| tok.encode(s).unwrap()).collect();
let shared = Arc::new(tok);
let handles: Vec<_> = (0..16)
.map(|tid| {
let t = Arc::clone(&shared);
let s = samples[tid].clone();
let exp = expected[tid].clone();
thread::spawn(move || {
for _ in 0..25 {
let got = t.encode(&s).unwrap();
assert_eq!(got, exp);
}
})
})
.collect();
for h in handles {
h.join().expect("worker thread panicked");
}
}