Files
zeekayandHanzo Dev 6a363f7ee3 merge: upstream main ddf1b879 and the aarch64 repack branch 35d7ae7c
Three-way per file through the rename: base 39355c6c91, theirs the upstream
merge of main with 35d7ae7c (via 7ef6a694d), ours origin/main.

Lands the barrier thread pool, runtime CPU feature dispatch, aarch64 and x86
repacked quantized kernels (indexed_gemv, gemv_fused_shared_lhs), quantized
embedding on CPU/Metal/CUDA, paged flash-attn kernels, cutile fused MoE, the
in-place op aliasing API, parking_lot storage locks, Metal buffer builders and
debug labels, safetensors 0.8, hf-hub 1.0, tokenizers 0.23.

Ours keeps ROCm, Vulkan, wgpu, the IQ/MXFP4/NVFP4/TQ dtypes, and the i-quant
weight/activation block split in matmul. Our backends ride InplaceOpN and the
quantized embedding; Metal get_rows gains the IQ kernels. ug leaves, as upstream.

Co-authored-by: Hanzo Dev <[email protected]>
2026-09-15 15:52:28 -07:00

792 lines
26 KiB
Rust

#[cfg(feature = "mkl")]
extern crate intel_mkl_src;
#[cfg(feature = "accelerate")]
extern crate accelerate_src;
use anyhow::Result;
use clap::{Parser, ValueEnum};
use hanzo_transformers::models::quantized_rwkv_v5::Model as Q5;
use hanzo_transformers::models::quantized_rwkv_v6::Model as Q6;
use hanzo_transformers::models::rwkv_v5::{Config, Model as M5, State, Tokenizer};
use hanzo_transformers::models::rwkv_v6::Model as M6;
use hanzo_transformers::models::rwkv_v7::{
Config as ConfigV7, Model as M7, ModelVersion, State as StateV7,
};
use hanzo_ml::{DType, Device, Tensor};
use hanzo_ml_examples::hub::Api;
use hanzo_nn::VarBuilder;
use hanzo_transformers::generation::LogitsProcessor;
const EOS_TOKEN_ID: u32 = 261;
enum Model {
M5(M5),
Q5(Q5),
M6(M6),
Q6(Q6),
}
impl Model {
fn forward(&self, xs: &Tensor, state: &mut State) -> hanzo_ml::Result<Tensor> {
match self {
Self::M5(m) => m.forward(xs, state),
Self::Q5(m) => m.forward(xs, state),
Self::M6(m) => m.forward(xs, state),
Self::Q6(m) => m.forward(xs, state),
}
}
}
struct TextGeneration {
model: Model,
config: Config,
device: Device,
tokenizer: Tokenizer,
logits_processor: LogitsProcessor,
repeat_penalty: f32,
repeat_last_n: usize,
}
impl TextGeneration {
#[allow(clippy::too_many_arguments)]
fn new(
model: Model,
config: Config,
tokenizer: Tokenizer,
seed: u64,
temp: Option<f64>,
top_p: Option<f64>,
repeat_penalty: f32,
repeat_last_n: usize,
device: &Device,
) -> Self {
let logits_processor = LogitsProcessor::new(seed, temp, top_p);
Self {
model,
config,
tokenizer,
logits_processor,
repeat_penalty,
repeat_last_n,
device: device.clone(),
}
}
fn run(&mut self, prompt: &str, sample_len: usize) -> Result<()> {
use std::io::Write;
let mut tokens = self.tokenizer.encode(prompt)?;
let mut generated_tokens = 0usize;
let mut state = State::new(1, &self.config, &self.device)?;
let mut next_logits = None;
for &t in tokens.iter() {
let input = Tensor::new(&[[t]], &self.device)?;
let logits = self.model.forward(&input, &mut state)?;
next_logits = Some(logits);
print!("{}", self.tokenizer.decode(&[t])?)
}
std::io::stdout().flush()?;
let start_gen = std::time::Instant::now();
for _ in 0..sample_len {
let logits = match next_logits.as_ref() {
Some(logits) => logits,
None => anyhow::bail!("cannot work on an empty prompt"),
};
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
let logits = if self.repeat_penalty == 1. {
logits
} else {
let start_at = tokens.len().saturating_sub(self.repeat_last_n);
hanzo_transformers::utils::apply_repeat_penalty(
&logits,
self.repeat_penalty,
&tokens[start_at..],
)?
};
let next_token = self.logits_processor.sample(&logits)?;
tokens.push(next_token);
generated_tokens += 1;
if next_token == EOS_TOKEN_ID || next_token == 0 {
break;
}
print!("{}", self.tokenizer.decode(&[next_token])?);
std::io::stdout().flush()?;
let input = Tensor::new(&[[next_token]], &self.device)?;
next_logits = Some(self.model.forward(&input, &mut state)?)
}
let dt = start_gen.elapsed();
println!(
"\n{generated_tokens} tokens generated ({:.2} token/s)",
generated_tokens as f64 / dt.as_secs_f64(),
);
Ok(())
}
}
/// Text generation pipeline for RWKV v7 models.
/// Separate from v5/v6 because v7 has different Config, State, and forward signature.
struct TextGenerationV7 {
model: M7,
config: ConfigV7,
device: Device,
dtype: DType,
tokenizer: Tokenizer,
logits_processor: LogitsProcessor,
alpha_presence: f32,
alpha_frequency: f32,
alpha_decay: f32,
stop: Option<String>,
}
impl TextGenerationV7 {
#[allow(clippy::too_many_arguments)]
fn new(
model: M7,
config: ConfigV7,
tokenizer: Tokenizer,
seed: u64,
temp: Option<f64>,
top_p: Option<f64>,
alpha_presence: f32,
alpha_frequency: f32,
alpha_decay: f32,
device: &Device,
dtype: DType,
stop: Option<String>,
) -> Self {
let logits_processor = LogitsProcessor::new(seed, temp, top_p);
Self {
model,
config,
tokenizer,
logits_processor,
alpha_presence,
alpha_frequency,
alpha_decay,
device: device.clone(),
dtype,
stop,
}
}
fn run(&mut self, prompt: &str, sample_len: usize) -> Result<()> {
use std::io::Write;
// Strip trailing whitespace — RWKV tokenizer produces non-English output otherwise
let prompt = prompt.trim_end();
let mut tokens = self.tokenizer.encode(prompt)?;
let mut generated_tokens = 0usize;
let mut state = StateV7::new_with_dtype(&self.config, &self.device, self.dtype)?;
// RWKV penalty state: per-token occurrence counts with exponential decay
let vocab_size = self.config.vocab_size;
let penalties_enabled = self.alpha_presence != 0.0 || self.alpha_frequency != 0.0;
let mut occurrence: Vec<f32> = vec![0.0; vocab_size];
// Process prompt using batched forward_seq for efficiency
let start_prompt = std::time::Instant::now();
let next_logits = self.model.forward_seq(&tokens, &mut state)?;
let prompt_time = start_prompt.elapsed();
// Update penalty counts for prompt tokens
if penalties_enabled {
for &t in tokens.iter() {
for count in occurrence.iter_mut() {
*count *= self.alpha_decay;
}
if (t as usize) < vocab_size {
occurrence[t as usize] += 1.0;
}
}
}
// Print the prompt
print!("{}", self.tokenizer.decode(&tokens)?);
std::io::stdout().flush()?;
let mut next_logits = Some(next_logits);
println!(
"\n[prompt: {} tokens in {:.2}s, {:.1} tok/s]",
tokens.len(),
prompt_time.as_secs_f64(),
tokens.len() as f64 / prompt_time.as_secs_f64()
);
// Track generated text for stop sequence detection
let mut generated_text = String::new();
let mut printed_len = 0; // How many chars we've already printed
let start_gen = std::time::Instant::now();
for _ in 0..sample_len {
let logits = match next_logits.as_ref() {
Some(logits) => logits,
None => anyhow::bail!("cannot work on an empty prompt"),
};
let logits = logits.to_dtype(DType::F32)?;
// Apply RWKV presence + frequency penalty
let logits = if penalties_enabled {
let mut logits_vec = logits.to_vec1::<f32>()?;
for (i, logit) in logits_vec.iter_mut().enumerate() {
if occurrence[i] > 0.0 {
*logit -= self.alpha_presence + self.alpha_frequency * occurrence[i];
}
}
Tensor::from_vec(logits_vec, vocab_size, logits.device())?
} else {
logits
};
let next_token = self.logits_processor.sample(&logits)?;
tokens.push(next_token);
generated_tokens += 1;
if penalties_enabled {
for count in occurrence.iter_mut() {
*count *= self.alpha_decay;
}
if (next_token as usize) < vocab_size {
occurrence[next_token as usize] += 1.0;
}
}
if next_token == EOS_TOKEN_ID || next_token == 0 {
break;
}
let token_text = self.tokenizer.decode(&[next_token])?;
generated_text.push_str(&token_text);
// Check for stop sequence
if let Some(stop) = &self.stop {
if let Some(pos) = generated_text.find(stop.as_str()) {
// Print only up to the stop sequence
if pos > printed_len {
print!("{}", &generated_text[printed_len..pos]);
std::io::stdout().flush()?;
}
break;
}
// Only print text that can't be the start of stop sequence
// Keep the last (stop.chars().count() - 1) chars buffered
// Use char boundaries to avoid splitting multi-byte UTF-8 characters
let stop_char_count = stop.chars().count();
let total_chars = generated_text.chars().count();
let safe_char_count = total_chars.saturating_sub(stop_char_count - 1);
// Convert char count back to byte offset at a valid boundary
let safe_len = generated_text
.char_indices()
.nth(safe_char_count)
.map(|(i, _)| i)
.unwrap_or(generated_text.len());
if safe_len > printed_len {
print!("{}", &generated_text[printed_len..safe_len]);
std::io::stdout().flush()?;
printed_len = safe_len;
}
} else {
print!("{}", token_text);
std::io::stdout().flush()?;
}
let input = Tensor::new(&[[next_token]], &self.device)?;
next_logits = Some(self.model.forward(&input, &mut state, &[next_token])?)
}
let dt = start_gen.elapsed();
println!(
"\n{generated_tokens} tokens generated ({:.2} token/s)",
generated_tokens as f64 / dt.as_secs_f64(),
);
Ok(())
}
}
#[derive(ValueEnum, Clone, Copy, PartialEq, Eq, Debug)]
enum Which {
// RWKV v5 models
Eagle7b,
World1b5,
World3b,
// RWKV v6 models
World6_1b6,
// RWKV v7 models: rwkv7-g1d (original v7 architecture, generation 1 dataset d)
#[value(name = "rwkv7-g1d-0.1b")]
Rwkv7G1d0_1b,
#[value(name = "rwkv7-g1d-0.4b")]
Rwkv7G1d0_4b,
#[value(name = "rwkv7-g1d-1.5b")]
Rwkv7G1d1_5b,
#[value(name = "rwkv7-g1d-2.9b")]
Rwkv7G1d2_9b,
#[value(name = "rwkv7-g1d-7.2b")]
Rwkv7G1d7_2b,
#[value(name = "rwkv7-g1d-13.3b")]
Rwkv7G1d13_3b,
// RWKV v7a models: rwkv7a-g1d (v7a variant, generation 1 dataset d)
#[value(name = "rwkv7a-g1d-0.1b")]
Rwkv7aG1d0_1b,
// RWKV v7b models: rwkv7b-g1b (v7b variant, generation 1 dataset b)
#[value(name = "rwkv7b-g1b-0.1b")]
Rwkv7bG1b0_1b,
}
impl std::fmt::Display for Which {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{self:?}")
}
}
impl Which {
fn is_v7(&self) -> bool {
matches!(
self,
Self::Rwkv7G1d0_1b
| Self::Rwkv7G1d0_4b
| Self::Rwkv7G1d1_5b
| Self::Rwkv7G1d2_9b
| Self::Rwkv7G1d7_2b
| Self::Rwkv7G1d13_3b
| Self::Rwkv7aG1d0_1b
| Self::Rwkv7bG1b0_1b
)
}
fn model_id(&self) -> &'static str {
match self {
Self::Eagle7b => "RWKV/v5-Eagle-7B-HF",
Self::World1b5 => "RWKV/rwkv-5-world-1b5",
Self::World3b => "RWKV/rwkv-5-world-3b",
Self::World6_1b6 => "paperfun/rwkv",
Self::Rwkv7G1d0_1b
| Self::Rwkv7G1d0_4b
| Self::Rwkv7G1d1_5b
| Self::Rwkv7G1d2_9b
| Self::Rwkv7G1d7_2b
| Self::Rwkv7G1d13_3b
| Self::Rwkv7aG1d0_1b
| Self::Rwkv7bG1b0_1b => "DanielClough/rwkv7-g1-safetensors",
}
}
fn revision(&self) -> &'static str {
match self {
Self::Eagle7b => "refs/pr/1",
Self::World1b5 | Self::World3b => "refs/pr/2",
Self::World6_1b6 => "main",
Self::Rwkv7G1d0_1b
| Self::Rwkv7G1d0_4b
| Self::Rwkv7G1d1_5b
| Self::Rwkv7G1d2_9b
| Self::Rwkv7G1d7_2b
| Self::Rwkv7G1d13_3b
| Self::Rwkv7aG1d0_1b
| Self::Rwkv7bG1b0_1b => "main",
}
}
fn v7_version(&self) -> Option<ModelVersion> {
match self {
Self::Rwkv7G1d0_1b
| Self::Rwkv7G1d0_4b
| Self::Rwkv7G1d1_5b
| Self::Rwkv7G1d2_9b
| Self::Rwkv7G1d7_2b
| Self::Rwkv7G1d13_3b => Some(ModelVersion::V7),
Self::Rwkv7aG1d0_1b => Some(ModelVersion::V7a),
Self::Rwkv7bG1b0_1b => Some(ModelVersion::V7b),
_ => None,
}
}
fn v7_config(&self) -> Option<ConfigV7> {
let version = self.v7_version()?;
let (hidden_size, num_hidden_layers) = match self {
Self::Rwkv7G1d0_1b | Self::Rwkv7aG1d0_1b | Self::Rwkv7bG1b0_1b => (768, 12),
Self::Rwkv7G1d0_4b => (1024, 24),
Self::Rwkv7G1d1_5b => (2048, 24),
Self::Rwkv7G1d2_9b => (2560, 32),
Self::Rwkv7G1d7_2b => (4096, 32),
Self::Rwkv7G1d13_3b => (4096, 61),
_ => return None,
};
Some(ConfigV7 {
version,
vocab_size: 65536,
hidden_size,
num_hidden_layers,
head_size: 64,
intermediate_size: None, // defaults to hidden_size * 4
rescale_every: 0,
})
}
}
#[derive(ValueEnum, Clone, Copy, PartialEq, Eq, Debug)]
enum Preset {
/// Chat: temp 1.0, top_p 0.5, presence 2.0, frequency 0.1, decay 0.99
Chat,
/// Creative (fiction etc.): temp 0.6, top_p 0.7, presence 2.0, frequency 0.2, decay 0.99
Creative,
}
#[derive(ValueEnum, Clone, Copy, PartialEq, Eq, Debug)]
enum PromptTemplate {
/// Pass prompt as-is with no formatting.
Raw,
/// Chat format: User: {prompt}\n\nA:
Chat,
/// Think format: User: {prompt}\n\nA: <think>
Think,
/// Fake think (recommended): User: {prompt}\n\nA: <think></think
FakeThink,
/// Fill-in-middle for G1c+ models (text, code, everything): ✿prefix✿✿suffix✿{suffix}✿middle✿{prompt}
Fim,
}
/// Format the user prompt according to the selected template.
fn apply_template(
template: PromptTemplate,
prompt: &str,
system: Option<&str>,
suffix: Option<&str>,
) -> String {
match template {
PromptTemplate::Raw => prompt.to_string(),
PromptTemplate::Chat => {
// Replace \n\n in user prompt with \n (double newline is chat round separator)
let prompt = prompt.replace("\n\n", "\n");
let mut out = String::new();
if let Some(sys) = system {
out.push_str(&format!("System: {sys}\n\n"));
}
out.push_str(&format!("User: {prompt}\n\nA:"));
out
}
PromptTemplate::Think => {
let prompt = prompt.replace("\n\n", "\n");
let mut out = String::new();
if let Some(sys) = system {
out.push_str(&format!("System: {sys}\n\n"));
}
out.push_str(&format!("User: {prompt}\n\nA: <think>"));
out
}
PromptTemplate::FakeThink => {
let prompt = prompt.replace("\n\n", "\n");
let mut out = String::new();
if let Some(sys) = system {
out.push_str(&format!("System: {sys}\n\n"));
}
out.push_str(&format!("User: {prompt}\n\nA: <think></think"));
out
}
PromptTemplate::Fim => {
let suffix = suffix.unwrap_or("");
// FIM prompt for G1c and newer models (works for text, code, and everything)
// Recommended format: ✿prefix✿✿suffix✿<suffix>✿middle✿<prompt>
// The model continues from <prompt> and generates until it reaches <suffix>
format!("✿prefix✿✿suffix✿{suffix}✿middle✿{prompt}")
}
}
}
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Args {
/// Run on CPU rather than on GPU.
#[arg(long)]
cpu: bool,
/// Enable tracing (generates a trace-timestamp.json file).
#[arg(long)]
tracing: bool,
#[arg(long)]
prompt: String,
/// Prompt template to apply (v7 only).
#[arg(long, default_value = "raw")]
template: PromptTemplate,
/// System prompt for chat/think templates.
#[arg(long)]
system: Option<String>,
/// Suffix text for FIM (fill-in-middle) template.
#[arg(long)]
suffix: Option<String>,
/// Sampling preset (v7 only). Overrides temperature, top_p, and penalty defaults.
#[arg(long)]
preset: Option<Preset>,
/// The temperature used to generate samples.
#[arg(long, default_value_t = 1.0)]
temperature: f64,
/// Nucleus sampling probability cutoff.
#[arg(long, default_value = "0.5")]
top_p: Option<f64>,
/// The seed to use when generating random samples.
#[arg(long, default_value_t = 299792458)]
seed: u64,
/// The length of the sample to generate (in tokens).
#[arg(long, short = 'n', default_value_t = 5000)]
sample_len: usize,
/// Stop generation when this text is produced (e.g., --stop "User:").
#[arg(long)]
stop: Option<String>,
#[arg(long, default_value = "world1b5")]
which: Which,
#[arg(long)]
model_id: Option<String>,
#[arg(long)]
revision: Option<String>,
#[arg(long)]
tokenizer: Option<String>,
#[arg(long)]
weight_files: Option<String>,
#[arg(long)]
config_file: Option<String>,
#[arg(long)]
quantized: bool,
/// Data type for inference: f32, f16, or bf16. Half precision (f16/bf16) is faster.
#[arg(long, default_value = "f32")]
dtype: String,
/// Penalty to be applied for repeating tokens, 1. means no penalty (v5/v6 only).
#[arg(long, default_value_t = 1.1)]
repeat_penalty: f32,
/// The context size to consider for the repeat penalty (v5/v6 only).
#[arg(long, default_value_t = 64)]
repeat_last_n: usize,
/// RWKV presence penalty (v7 only). Flat additive penalty for any token that has appeared.
#[arg(long, default_value_t = 2.0)]
alpha_presence: f32,
/// RWKV frequency penalty (v7 only). Additive penalty proportional to token count.
#[arg(long, default_value_t = 0.1)]
alpha_frequency: f32,
/// RWKV penalty count decay (v7 only). Exponential decay applied to token counts each step.
#[arg(long, default_value_t = 0.99)]
alpha_decay: f32,
}
fn main() -> Result<()> {
use tracing_chrome::ChromeLayerBuilder;
use tracing_subscriber::prelude::*;
let mut args = Args::parse();
let _guard = if args.tracing {
let (chrome_layer, guard) = ChromeLayerBuilder::new().build();
tracing_subscriber::registry().with(chrome_layer).init();
Some(guard)
} else {
None
};
// Apply preset overrides (v7 only)
if let Some(preset) = args.preset {
match preset {
Preset::Chat => {
args.temperature = 1.0;
args.top_p = Some(0.5);
args.alpha_presence = 2.0;
args.alpha_frequency = 0.1;
args.alpha_decay = 0.99;
}
Preset::Creative => {
args.temperature = 0.6;
args.top_p = Some(0.7);
args.alpha_presence = 2.0;
args.alpha_frequency = 0.2;
args.alpha_decay = 0.99;
}
}
}
let api = Api::new()?;
let repo = api
.model(
args.model_id
.unwrap_or_else(|| args.which.model_id().to_string()),
)
.with_revision(
args.revision
.unwrap_or_else(|| args.which.revision().to_string()),
);
let tokenizer = match args.tokenizer {
Some(file) => std::path::PathBuf::from(file),
None => api
.model("lmz/hanzo-ml-rwkv")
.get("rwkv_vocab_v20230424.json")?,
};
let config_filename = match (&args.config_file, args.which.is_v7()) {
(Some(file), _) => Some(std::path::PathBuf::from(file)),
(None, true) => None, // v7 models use built-in config, no config.json needed
(None, false) => Some(repo.get("config.json")?),
};
let filenames = match args.weight_files {
Some(files) => files
.split(',')
.map(std::path::PathBuf::from)
.collect::<Vec<_>>(),
None => {
if args.quantized {
if args.which.is_v7() {
anyhow::bail!("quantized RWKV v7 models are not yet supported");
}
vec![match args.which {
Which::World1b5 => api.model("lmz/hanzo-ml-rwkv").get("world1b5-q4k.gguf")?,
Which::World3b => api.model("lmz/hanzo-ml-rwkv").get("world3b-q4k.gguf")?,
Which::Eagle7b => api.model("lmz/hanzo-ml-rwkv").get("eagle7b-q4k.gguf")?,
Which::World6_1b6 => repo.get("rwkv-6-world-1b6-q4k.gguf")?,
_ => unreachable!(),
}]
} else {
vec![match args.which {
Which::World1b5 | Which::World3b | Which::Eagle7b => {
repo.get("model.safetensors")?
}
Which::World6_1b6 => repo.get("rwkv-6-world-1b6.safetensors")?,
Which::Rwkv7G1d0_1b => {
repo.get("rwkv7-g1d-0.1b-20260129-ctx8192.safetensors")?
}
Which::Rwkv7G1d0_4b => {
repo.get("rwkv7-g1d-0.4b-20260210-ctx8192.safetensors")?
}
Which::Rwkv7G1d1_5b => {
repo.get("rwkv7-g1d-1.5b-20260212-ctx8192.safetensors")?
}
Which::Rwkv7G1d2_9b => {
repo.get("rwkv7-g1d-2.9b-20260131-ctx8192.safetensors")?
}
Which::Rwkv7G1d7_2b => {
repo.get("rwkv7-g1d-7.2b-20260131-ctx8192.safetensors")?
}
Which::Rwkv7G1d13_3b => {
repo.get("rwkv7-g1d-13.3b-20260131-ctx8192.safetensors")?
}
Which::Rwkv7aG1d0_1b => {
repo.get("rwkv7a-g1d-0.1b-20260212-ctx8192.safetensors")?
}
Which::Rwkv7bG1b0_1b => {
repo.get("rwkv7b-g1b-0.1b-20250822-ctx4096.safetensors")?
}
}]
}
}
};
let tokenizer = Tokenizer::new(tokenizer)?;
let device = hanzo_ml_examples::device(args.cpu)?;
if args.which.is_v7() {
// RWKV v7 path — different Config, State, and forward signature
let config: ConfigV7 = if let Some(config_file) = &config_filename {
serde_json::from_slice(&std::fs::read(config_file)?)?
} else {
args.which
.v7_config()
.expect("v7 variant must have built-in config")
};
// Parse dtype from string
let dtype = match args.dtype.to_lowercase().as_str() {
"f16" => DType::F16,
"bf16" => DType::BF16,
"f32" => DType::F32,
other => anyhow::bail!("Unknown dtype '{}'. Use f32, f16, or bf16.", other),
};
let vb = unsafe { VarBuilder::from_mmaped_safetensors(&filenames, dtype, &device)? };
let model = M7::new(&config, vb)?;
// For FIM template, auto-set stop sequence to ✿ (delimiter signals end of middle)
let stop = match (&args.stop, args.template) {
(Some(s), _) => Some(s.clone()), // User-specified stop takes precedence
(None, PromptTemplate::Fim) => Some("✿".to_string()), // FIM auto-stops on delimiter
(None, _) => None,
};
let mut pipeline = TextGenerationV7::new(
model,
config,
tokenizer,
args.seed,
Some(args.temperature),
args.top_p,
args.alpha_presence,
args.alpha_frequency,
args.alpha_decay,
&device,
dtype,
stop,
);
let prompt = apply_template(
args.template,
&args.prompt,
args.system.as_deref(),
args.suffix.as_deref(),
);
pipeline.run(&prompt, args.sample_len)?;
} else {
// v5/v6 path (existing behavior)
let config: Config = serde_json::from_slice(&std::fs::read(config_filename.unwrap())?)?;
let model = if args.quantized {
let filename = &filenames[0];
let vb = hanzo_transformers::quantized_var_builder::VarBuilder::from_gguf(
filename, &device,
)?;
match args.which {
Which::World1b5 | Which::World3b | Which::Eagle7b => {
Model::Q5(Q5::new(&config, vb)?)
}
Which::World6_1b6 => Model::Q6(Q6::new(&config, vb)?),
_ => unreachable!(),
}
} else {
let vb =
unsafe { VarBuilder::from_mmaped_safetensors(&filenames, DType::F32, &device)? };
match args.which {
Which::World1b5 | Which::World3b | Which::Eagle7b => {
Model::M5(M5::new(&config, vb)?)
}
Which::World6_1b6 => Model::M6(M6::new(&config, vb)?),
_ => unreachable!(),
}
};
let mut pipeline = TextGeneration::new(
model,
config,
tokenizer,
args.seed,
Some(args.temperature),
args.top_p,
args.repeat_penalty,
args.repeat_last_n,
&device,
);
pipeline.run(&args.prompt, args.sample_len)?;
}
Ok(())
}