Files
zeekayandHanzo Dev 6a363f7ee3 merge: upstream main ddf1b879 and the aarch64 repack branch 35d7ae7c
Three-way per file through the rename: base 39355c6c91, theirs the upstream
merge of main with 35d7ae7c (via 7ef6a694d), ours origin/main.

Lands the barrier thread pool, runtime CPU feature dispatch, aarch64 and x86
repacked quantized kernels (indexed_gemv, gemv_fused_shared_lhs), quantized
embedding on CPU/Metal/CUDA, paged flash-attn kernels, cutile fused MoE, the
in-place op aliasing API, parking_lot storage locks, Metal buffer builders and
debug labels, safetensors 0.8, hf-hub 1.0, tokenizers 0.23.

Ours keeps ROCm, Vulkan, wgpu, the IQ/MXFP4/NVFP4/TQ dtypes, and the i-quant
weight/activation block split in matmul. Our backends ride InplaceOpN and the
quantized embedding; Metal get_rows gains the IQ kernels. ug leaves, as upstream.

Co-authored-by: Hanzo Dev <[email protected]>
2026-09-15 15:52:28 -07:00

96 lines
3.1 KiB
Rust

use std::{
fs::File,
io::BufReader,
path::{Path, PathBuf},
};
use anyhow::{Context, Result};
use clap::Parser;
use hanzo_ml::quantized::gguf_file;
use hanzo_ml::quantized::tokenizer::TokenizerFromGguf;
use hanzo_ml_examples::hub::Api;
use tokenizers::Tokenizer;
#[derive(Parser, Debug)]
struct Args {
/// Path to the GGUF file that stores the tokenizer metadata.
#[arg(long)]
model: String,
/// Optional revision (branch/tag/commit) when pulling from the Hugging Face Hub.
#[arg(long)]
revision: Option<String>,
/// Text prompt to tokenize with the GGUF tokenizer.
#[arg(long, default_value = "Hello Hanzo!")]
prompt: String,
}
fn main() -> Result<()> {
let args = Args::parse();
let gguf_path = resolve_model_path(&args.model, args.revision.clone())
.with_context(|| format!("failed to locate GGUF file {}", args.model))?;
let file = File::open(&gguf_path)
.with_context(|| format!("failed to open GGUF file {}", gguf_path.display()))?;
let mut reader = BufReader::new(file);
let content = gguf_file::Content::read(&mut reader).context("failed to load GGUF metadata")?;
// Build the tokenizer directly from the GGUF metadata (tokens, merges, and post-processing).
let tokenizer =
Tokenizer::from_gguf(&content).context("failed to initialize tokenizer from GGUF")?;
let encoding = tokenizer
.encode(args.prompt.clone(), true)
.map_err(anyhow::Error::msg)
.context("failed to tokenize prompt")?;
println!("Prompt: {}", args.prompt);
println!("Source: {}", gguf_path.display());
println!("Token ids: {:?}", encoding.get_ids());
println!("Tokens: {:?}", encoding.get_tokens());
println!(
"Special tokens mask: {:?}",
encoding.get_special_tokens_mask()
);
let decoded = tokenizer
.decode(encoding.get_ids(), true)
.map_err(anyhow::Error::msg)
.context("failed to decode tokens")?;
println!("Decoded (special tokens stripped): {decoded}");
Ok(())
}
fn resolve_model_path(model: &str, revision: Option<String>) -> Result<PathBuf> {
// Local path: use as-is if it exists.
let candidate = Path::new(model);
if candidate.exists() {
return Ok(candidate.to_path_buf());
}
// Hugging Face Hub: accept strings like `author/repo/weights.gguf` or
// `author/repo/subdir/weights.gguf`. An optional `revision` can be provided.
let trimmed = model
.trim_start_matches("hf://")
.trim_start_matches("https://huggingface.co/")
.trim_start_matches("huggingface.co/");
let parts: Vec<_> = trimmed.split('/').filter(|s| !s.is_empty()).collect();
if parts.len() < 3 {
anyhow::bail!(
"model must be a local file or an HF path like `author/repo/file.gguf`, got `{model}`"
);
}
let repo_id = format!("{}/{}", parts[0], parts[1]);
let filename = parts[2..].join("/");
let api = Api::new()?;
let path = api
.model(repo_id)
.with_revision(revision.unwrap_or_else(|| "main".to_string()))
.get(&filename)?;
Ok(path)
}