mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-20 23:08:29 +00:00
Three-way per file through the rename: base 39355c6c91, theirs the upstream
merge of main with 35d7ae7c (via 7ef6a694d), ours origin/main.
Lands the barrier thread pool, runtime CPU feature dispatch, aarch64 and x86
repacked quantized kernels (indexed_gemv, gemv_fused_shared_lhs), quantized
embedding on CPU/Metal/CUDA, paged flash-attn kernels, cutile fused MoE, the
in-place op aliasing API, parking_lot storage locks, Metal buffer builders and
debug labels, safetensors 0.8, hf-hub 1.0, tokenizers 0.23.
Ours keeps ROCm, Vulkan, wgpu, the IQ/MXFP4/NVFP4/TQ dtypes, and the i-quant
weight/activation block split in matmul. Our backends ride InplaceOpN and the
quantized embedding; Metal get_rows gains the IQ kernels. ug leaves, as upstream.
Co-authored-by: Hanzo Dev <[email protected]>
177 lines
5.2 KiB
Rust
177 lines
5.2 KiB
Rust
use std::path::PathBuf;
|
|
|
|
use anyhow::{Error as E, Result};
|
|
use clap::{Parser, ValueEnum};
|
|
use hanzo_ml::{Device, Tensor};
|
|
use hanzo_ml_examples::hub::Api;
|
|
use hanzo_nn::VarBuilder;
|
|
use hanzo_transformers::models::modernbert;
|
|
use tokenizers::{PaddingParams, Tokenizer};
|
|
|
|
#[derive(Debug, Clone, ValueEnum)]
|
|
enum Model {
|
|
ModernBertBase,
|
|
ModernBertLarge,
|
|
}
|
|
|
|
#[derive(Parser, Debug)]
|
|
#[command(author, version, about, long_about = None)]
|
|
struct Args {
|
|
/// Run on CPU rather than on GPU.
|
|
#[arg(long)]
|
|
cpu: bool,
|
|
|
|
/// Enable tracing (generates a trace-timestamp.json file).
|
|
#[arg(long)]
|
|
tracing: bool,
|
|
|
|
#[arg(long)]
|
|
model_id: Option<String>,
|
|
|
|
#[arg(long, default_value = "main")]
|
|
revision: String,
|
|
|
|
#[arg(long, default_value = "modern-bert-base")]
|
|
model: Model,
|
|
|
|
// Path to the tokenizer file.
|
|
#[arg(long)]
|
|
tokenizer_file: Option<String>,
|
|
|
|
// Path to the weight files.
|
|
#[arg(long)]
|
|
weight_files: Option<String>,
|
|
|
|
// Path to the config file.
|
|
#[arg(long)]
|
|
config_file: Option<String>,
|
|
|
|
/// When set, compute embeddings for this prompt.
|
|
#[arg(long)]
|
|
prompt: Option<String>,
|
|
}
|
|
|
|
fn main() -> Result<()> {
|
|
let args = Args::parse();
|
|
let api = Api::new()?;
|
|
let model_id = match &args.model_id {
|
|
Some(model_id) => model_id.to_string(),
|
|
None => match args.model {
|
|
Model::ModernBertBase => "answerdotai/ModernBERT-base".to_string(),
|
|
Model::ModernBertLarge => "answerdotai/ModernBERT-large".to_string(),
|
|
},
|
|
};
|
|
let repo = api.model(model_id).with_revision(args.revision);
|
|
|
|
let tokenizer_filename = match args.tokenizer_file {
|
|
Some(file) => std::path::PathBuf::from(file),
|
|
None => repo.get("tokenizer.json")?,
|
|
};
|
|
|
|
let config_filename = match args.config_file {
|
|
Some(file) => std::path::PathBuf::from(file),
|
|
None => repo.get("config.json")?,
|
|
};
|
|
|
|
let weights_filename = match args.weight_files {
|
|
Some(files) => PathBuf::from(files),
|
|
None => match repo.get("model.safetensors") {
|
|
Ok(safetensors) => safetensors,
|
|
Err(_) => match repo.get("pytorch_model.bin") {
|
|
Ok(pytorch_model) => pytorch_model,
|
|
Err(e) => {
|
|
anyhow::bail!("Model weights not found. The weights should either be a `model.safetensors` or `pytorch_model.bin` file. Error: {e}")
|
|
}
|
|
},
|
|
},
|
|
};
|
|
|
|
let config = std::fs::read_to_string(config_filename)?;
|
|
let config: modernbert::Config = serde_json::from_str(&config)?;
|
|
let mut tokenizer = Tokenizer::from_file(tokenizer_filename).map_err(E::msg)?;
|
|
|
|
let device = hanzo_ml_examples::device(args.cpu)?;
|
|
|
|
let vb = if weights_filename.ends_with("model.safetensors") {
|
|
unsafe {
|
|
VarBuilder::from_mmaped_safetensors(&[weights_filename], hanzo_ml::DType::F32, &device)
|
|
.unwrap()
|
|
}
|
|
} else {
|
|
println!("Loading weights from pytorch_model.bin");
|
|
VarBuilder::from_pth(&weights_filename, hanzo_ml::DType::F32, &device).unwrap()
|
|
};
|
|
tokenizer
|
|
.with_padding(Some(PaddingParams {
|
|
strategy: tokenizers::PaddingStrategy::BatchLongest,
|
|
pad_id: config.pad_token_id,
|
|
..Default::default()
|
|
}))
|
|
.with_truncation(None)
|
|
.map_err(E::msg)?;
|
|
|
|
let prompt = match &args.prompt {
|
|
Some(p) => vec![p.as_str()],
|
|
None => vec![
|
|
"Hello I'm a [MASK] model.",
|
|
"I'm a [MASK] boy.",
|
|
"I'm [MASK] in berlin.",
|
|
"The capital of France is [MASK].",
|
|
],
|
|
};
|
|
let model = modernbert::ModernBertForMaskedLM::load(vb, &config)?;
|
|
|
|
let input_ids = tokenize_batch(&tokenizer, prompt.clone(), &device)?;
|
|
let attention_mask = get_attention_mask(&tokenizer, prompt.clone(), &device)?;
|
|
|
|
let output = model
|
|
.forward(&input_ids, &attention_mask)?
|
|
.to_dtype(hanzo_ml::DType::F32)?;
|
|
|
|
let max_outs = output.argmax(2)?;
|
|
|
|
let max_out = max_outs.to_vec2::<u32>()?;
|
|
let max_out_refs: Vec<&[u32]> = max_out.iter().map(|v| v.as_slice()).collect();
|
|
let decoded = tokenizer.decode_batch(&max_out_refs, true).unwrap();
|
|
for (i, sentence) in decoded.iter().enumerate() {
|
|
println!("Sentence: {} : {}", i + 1, sentence);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
pub fn tokenize_batch(
|
|
tokenizer: &Tokenizer,
|
|
input: Vec<&str>,
|
|
device: &Device,
|
|
) -> anyhow::Result<Tensor> {
|
|
let tokens = tokenizer.encode_batch(input, true).map_err(E::msg)?;
|
|
|
|
let token_ids = tokens
|
|
.iter()
|
|
.map(|tokens| {
|
|
let tokens = tokens.get_ids().to_vec();
|
|
Tensor::new(tokens.as_slice(), device)
|
|
})
|
|
.collect::<hanzo_ml::Result<Vec<_>>>()?;
|
|
|
|
Ok(Tensor::stack(&token_ids, 0)?)
|
|
}
|
|
|
|
pub fn get_attention_mask(
|
|
tokenizer: &Tokenizer,
|
|
input: Vec<&str>,
|
|
device: &Device,
|
|
) -> anyhow::Result<Tensor> {
|
|
let tokens = tokenizer.encode_batch(input, true).map_err(E::msg)?;
|
|
|
|
let attention_mask = tokens
|
|
.iter()
|
|
.map(|tokens| {
|
|
let tokens = tokens.get_attention_mask().to_vec();
|
|
Tensor::new(tokens.as_slice(), device)
|
|
})
|
|
.collect::<hanzo_ml::Result<Vec<_>>>()?;
|
|
Ok(Tensor::stack(&attention_mask, 0)?)
|
|
}
|