mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-22 01:49:59 +00:00
Three-way per file through the rename: base 39355c6c91, theirs the upstream
merge of main with 35d7ae7c (via 7ef6a694d), ours origin/main.
Lands the barrier thread pool, runtime CPU feature dispatch, aarch64 and x86
repacked quantized kernels (indexed_gemv, gemv_fused_shared_lhs), quantized
embedding on CPU/Metal/CUDA, paged flash-attn kernels, cutile fused MoE, the
in-place op aliasing API, parking_lot storage locks, Metal buffer builders and
debug labels, safetensors 0.8, hf-hub 1.0, tokenizers 0.23.
Ours keeps ROCm, Vulkan, wgpu, the IQ/MXFP4/NVFP4/TQ dtypes, and the i-quant
weight/activation block split in matmul. Our backends ride InplaceOpN and the
quantized embedding; Metal get_rows gains the IQ kernels. ug leaves, as upstream.
Co-authored-by: Hanzo Dev <[email protected]>
611 lines
19 KiB
Rust
611 lines
19 KiB
Rust
#[cfg(feature = "mkl")]
|
|
extern crate intel_mkl_src;
|
|
|
|
#[cfg(feature = "accelerate")]
|
|
extern crate accelerate_src;
|
|
|
|
use anyhow::{Error as E, Result};
|
|
use clap::{Parser, ValueEnum};
|
|
use std::io::Write;
|
|
|
|
use hanzo_ml::{DType, Device, Tensor};
|
|
use hanzo_ml_examples::chat_template::{ChatTemplate, ChatTemplateOptions, Message};
|
|
use hanzo_ml_examples::token_output_stream::TokenOutputStream;
|
|
|
|
use hanzo_ml_examples::hub::Api;
|
|
use hanzo_nn::VarBuilder;
|
|
use hanzo_transformers::generation::{LogitsProcessor, Sampling};
|
|
use tokenizers::Tokenizer;
|
|
|
|
// Import both model implementations
|
|
use hanzo_transformers::models::smol::quantized_smollm3::QuantizedModelForCausalLM;
|
|
use hanzo_transformers::models::smol::smollm3::{Config, ModelForCausalLM};
|
|
|
|
const DEFAULT_PROMPT: &str = "Write a Rust function to calculate the factorial of a given number.";
|
|
|
|
// ==================== Model Type Enum ====================
|
|
|
|
enum SmolLM3Model {
|
|
Quantized(QuantizedModelForCausalLM),
|
|
Full(ModelForCausalLM, Config), // Store config alongside model
|
|
}
|
|
|
|
impl SmolLM3Model {
|
|
fn forward(&mut self, input: &Tensor, pos: usize) -> Result<Tensor> {
|
|
match self {
|
|
Self::Quantized(model) => Ok(model.forward(input, pos)?),
|
|
Self::Full(model, _) => Ok(model.forward(input, pos)?),
|
|
}
|
|
}
|
|
|
|
fn config(&self) -> ModelConfig {
|
|
match self {
|
|
Self::Quantized(model) => {
|
|
let cfg = model.config();
|
|
ModelConfig {
|
|
vocab_size: cfg.vocab_size,
|
|
hidden_size: cfg.hidden_size,
|
|
num_hidden_layers: cfg.num_hidden_layers,
|
|
num_attention_heads: cfg.num_attention_heads,
|
|
num_key_value_heads: cfg.num_key_value_heads,
|
|
rope_theta: cfg.rope_theta as f32, // Convert f64 to f32
|
|
eos_token_id: Some(128012), // Default SmolLM3 EOS
|
|
no_rope_layers: None,
|
|
no_rope_layer_interval: None,
|
|
}
|
|
}
|
|
Self::Full(_, cfg) => {
|
|
ModelConfig {
|
|
vocab_size: cfg.vocab_size,
|
|
hidden_size: cfg.hidden_size,
|
|
num_hidden_layers: cfg.num_hidden_layers,
|
|
num_attention_heads: cfg.num_attention_heads,
|
|
num_key_value_heads: cfg.num_key_value_heads,
|
|
rope_theta: cfg.rope_theta as f32, // Convert f64 to f32
|
|
eos_token_id: cfg.eos_token_id,
|
|
no_rope_layers: cfg
|
|
.no_rope_layers
|
|
.as_ref()
|
|
.map(|v| v.iter().map(|&x| x as u32).collect()), // Convert Vec<usize> to Vec<u32>
|
|
no_rope_layer_interval: cfg.no_rope_layer_interval,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Unified config representation
|
|
struct ModelConfig {
|
|
vocab_size: usize,
|
|
hidden_size: usize,
|
|
num_hidden_layers: usize,
|
|
num_attention_heads: usize,
|
|
num_key_value_heads: usize,
|
|
rope_theta: f32,
|
|
eos_token_id: Option<u32>,
|
|
no_rope_layers: Option<Vec<u32>>,
|
|
no_rope_layer_interval: Option<usize>,
|
|
}
|
|
|
|
impl ModelConfig {
|
|
fn head_dim(&self) -> usize {
|
|
self.hidden_size / self.num_attention_heads
|
|
}
|
|
}
|
|
|
|
// ==================== CLI Arguments ====================
|
|
|
|
#[derive(Clone, Debug, Copy, PartialEq, Eq, ValueEnum)]
|
|
enum ModelType {
|
|
/// Use quantized GGUF model (smaller, faster)
|
|
#[value(name = "quantized")]
|
|
Quantized,
|
|
/// Use full precision safetensors model (larger, more accurate)
|
|
#[value(name = "full")]
|
|
Full,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Copy, PartialEq, Eq, ValueEnum)]
|
|
enum Quantization {
|
|
#[value(name = "q4_k_m")]
|
|
Q4KM,
|
|
#[value(name = "q8_0")]
|
|
Q8_0,
|
|
#[value(name = "f16")]
|
|
F16,
|
|
}
|
|
|
|
impl Quantization {
|
|
fn filename_unsloth(&self) -> &'static str {
|
|
match self {
|
|
Self::Q4KM => "SmolLM3-3B-Q4_K_M.gguf",
|
|
Self::Q8_0 => "SmolLM3-3B-Q8_0.gguf",
|
|
Self::F16 => "SmolLM3-3B-F16.gguf",
|
|
}
|
|
}
|
|
|
|
fn size_gb(&self) -> f32 {
|
|
match self {
|
|
Self::Q4KM => 1.92,
|
|
Self::Q8_0 => 3.28,
|
|
Self::F16 => 6.16,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Copy, PartialEq, Eq, ValueEnum)]
|
|
enum WhichModel {
|
|
#[value(name = "3b")]
|
|
W3b,
|
|
#[value(name = "3b-base")]
|
|
W3bBase,
|
|
}
|
|
|
|
#[derive(Parser, Debug)]
|
|
#[command(author, version, about, long_about = None)]
|
|
struct Args {
|
|
/// Model type: 'quantized' for GGUF or 'full' for safetensors
|
|
#[arg(long, default_value = "quantized")]
|
|
model_type: ModelType,
|
|
|
|
/// Which model variant to use
|
|
#[arg(long, default_value = "3b")]
|
|
model: WhichModel,
|
|
|
|
/// Quantization level (only for quantized models)
|
|
/// Q8_0: 3.3GB, best quality | Q4_K_M: 1.9GB, good balance | F16: 6.2GB, full precision
|
|
#[arg(long, default_value = "q8_0")]
|
|
quantization: Quantization,
|
|
|
|
/// Data type (only for full models: f32, f16, bf16, or auto)
|
|
#[arg(long, default_value = "auto")]
|
|
dtype: String,
|
|
|
|
/// Path to model file (optional, will auto-download if not provided)
|
|
#[arg(long)]
|
|
model_path: Option<String>,
|
|
|
|
/// Path to tokenizer file (optional, will auto-download if not provided)
|
|
#[arg(long)]
|
|
tokenizer: Option<String>,
|
|
|
|
/// The initial prompt
|
|
#[arg(long)]
|
|
prompt: Option<String>,
|
|
|
|
/// The length of the sample to generate (in tokens)
|
|
#[arg(short = 'n', long, default_value_t = 1000)]
|
|
sample_len: usize,
|
|
|
|
/// The temperature used to generate samples, use 0 for greedy sampling
|
|
#[arg(long, default_value_t = 0.8)]
|
|
temperature: f64,
|
|
|
|
/// Nucleus sampling probability cutoff
|
|
#[arg(long)]
|
|
top_p: Option<f64>,
|
|
|
|
/// Only sample among the top K samples
|
|
#[arg(long)]
|
|
top_k: Option<usize>,
|
|
|
|
/// The seed to use when generating random samples
|
|
#[arg(long, default_value_t = 299792458)]
|
|
seed: u64,
|
|
|
|
/// Penalty to be applied for repeating tokens, 1. means no penalty
|
|
#[arg(long, default_value_t = 1.1)]
|
|
repeat_penalty: f32,
|
|
|
|
/// The context size to consider for the repeat penalty
|
|
#[arg(long, default_value_t = 64)]
|
|
repeat_last_n: usize,
|
|
|
|
/// Skip chat template formatting (use raw prompt, like base model)
|
|
#[arg(long)]
|
|
no_chat_template: bool,
|
|
|
|
/// Enable thinking/reasoning mode (allows model to show its reasoning process)
|
|
#[arg(long)]
|
|
thinking: bool,
|
|
|
|
/// Process prompt elements separately (slower, for debugging)
|
|
#[arg(long)]
|
|
split_prompt: bool,
|
|
|
|
/// Enable tracing (generates a trace-timestamp.json file)
|
|
#[arg(long)]
|
|
tracing: bool,
|
|
|
|
/// Use CPU flash attention (fused kernel, no mask materialization)
|
|
#[arg(long)]
|
|
use_flash_attn: bool,
|
|
}
|
|
|
|
impl Args {
|
|
fn get_tokenizer(&self) -> Result<Tokenizer> {
|
|
let tokenizer_path = match &self.tokenizer {
|
|
Some(path) => std::path::PathBuf::from(path),
|
|
None => {
|
|
let api = Api::new()?;
|
|
let api = api.model("HuggingFaceTB/SmolLM3-3B");
|
|
api.get("tokenizer.json")?
|
|
}
|
|
};
|
|
Tokenizer::from_file(tokenizer_path).map_err(E::msg)
|
|
}
|
|
|
|
fn should_use_chat_template(&self) -> bool {
|
|
matches!(self.model, WhichModel::W3b) && !self.no_chat_template
|
|
}
|
|
}
|
|
|
|
// ==================== Model Loading ====================
|
|
|
|
fn load_quantized_model(args: &Args, device: &Device) -> Result<SmolLM3Model> {
|
|
let model_path = match &args.model_path {
|
|
Some(path) => std::path::PathBuf::from(path),
|
|
None => {
|
|
let filename = args.quantization.filename_unsloth();
|
|
let repo_id = "unsloth/SmolLM3-3B-GGUF";
|
|
let api = Api::new()?;
|
|
println!(
|
|
"Downloading {} from {} (~{:.2}GB)...",
|
|
filename,
|
|
repo_id,
|
|
args.quantization.size_gb()
|
|
);
|
|
api.model(repo_id).with_revision("main").get(filename)?
|
|
}
|
|
};
|
|
|
|
println!("Loading quantized model from {:?}...", model_path);
|
|
let model = QuantizedModelForCausalLM::from_gguf(&model_path, device, args.use_flash_attn)?;
|
|
Ok(SmolLM3Model::Quantized(model))
|
|
}
|
|
|
|
fn load_full_model(args: &Args, device: &Device) -> Result<SmolLM3Model> {
|
|
let api = Api::new()?;
|
|
let model_id = match args.model {
|
|
WhichModel::W3b => "HuggingFaceTB/SmolLM3-3B",
|
|
WhichModel::W3bBase => "HuggingFaceTB/SmolLM3-3B-Base",
|
|
};
|
|
|
|
println!("Loading full model from: {}", model_id);
|
|
let repo = api.model(model_id).with_revision("main");
|
|
|
|
let filenames = match &args.model_path {
|
|
Some(path) => vec![std::path::PathBuf::from(path)],
|
|
None => hanzo_ml_examples::hub_load_safetensors(&repo, "model.safetensors.index.json")?,
|
|
};
|
|
|
|
let config_file = repo.get("config.json")?;
|
|
let config: Config = serde_json::from_slice(&std::fs::read(config_file)?)?;
|
|
|
|
let dtype = match args.dtype.as_str() {
|
|
"f16" => DType::F16,
|
|
"bf16" => DType::BF16,
|
|
"f32" => DType::F32,
|
|
"auto" => {
|
|
if device.is_cuda() || device.is_metal() {
|
|
DType::BF16
|
|
} else {
|
|
DType::F32
|
|
}
|
|
}
|
|
other => anyhow::bail!("Unsupported dtype: {}, use f16, bf16, f32, or auto", other),
|
|
};
|
|
|
|
println!("Using dtype: {:?}", dtype);
|
|
|
|
let vb = unsafe { VarBuilder::from_mmaped_safetensors(&filenames, dtype, device)? };
|
|
let model = ModelForCausalLM::new(&config, vb)?;
|
|
|
|
Ok(SmolLM3Model::Full(model, config))
|
|
}
|
|
|
|
// ==================== Text Generation ====================
|
|
|
|
fn format_prompt(prompt: &str, use_chat_template: bool, enable_thinking: bool) -> String {
|
|
if !use_chat_template {
|
|
return prompt.to_string();
|
|
}
|
|
|
|
let template = ChatTemplate::chatml_with_thinking();
|
|
|
|
// Build system message with SmolLM3's metadata format
|
|
let now = chrono::Local::now();
|
|
let today_date = now.format("%d %B %Y").to_string();
|
|
let reasoning_mode = if enable_thinking {
|
|
"/think"
|
|
} else {
|
|
"/no_think"
|
|
};
|
|
|
|
let system_content = format!(
|
|
"## Metadata\n\n\
|
|
Knowledge Cutoff Date: June 2025\n\
|
|
Today Date: {}\n\
|
|
Reasoning Mode: {}\n\n\
|
|
## Custom Instructions\n\n\
|
|
You are a helpful AI assistant named SmolLM, trained by Hugging Face.",
|
|
today_date, reasoning_mode
|
|
);
|
|
|
|
let messages = vec![Message::system(system_content), Message::user(prompt)];
|
|
|
|
let options = if enable_thinking {
|
|
ChatTemplateOptions::for_generation().with_thinking()
|
|
} else {
|
|
ChatTemplateOptions::for_generation()
|
|
};
|
|
|
|
template.apply(&messages, &options).unwrap()
|
|
}
|
|
|
|
fn get_eos_token(tokenizer: &Tokenizer, config: &ModelConfig) -> u32 {
|
|
if let Some(eos_id) = config.eos_token_id {
|
|
return eos_id;
|
|
}
|
|
|
|
let vocab = tokenizer.get_vocab(true);
|
|
if let Some(&eos_id) = vocab.get("<|im_end|>") {
|
|
return eos_id;
|
|
}
|
|
if let Some(&eos_id) = vocab.get("<|endoftext|>") {
|
|
return eos_id;
|
|
}
|
|
|
|
128012 // Default SmolLM3 EOS token
|
|
}
|
|
|
|
fn run_generation(
|
|
model: &mut SmolLM3Model,
|
|
tokenizer: Tokenizer,
|
|
args: &Args,
|
|
device: &Device,
|
|
) -> Result<()> {
|
|
let mut tos = TokenOutputStream::new(tokenizer);
|
|
|
|
// Prepare prompt
|
|
let prompt_str = args
|
|
.prompt
|
|
.clone()
|
|
.unwrap_or_else(|| DEFAULT_PROMPT.to_string());
|
|
let use_chat_template = args.should_use_chat_template();
|
|
let formatted_prompt = format_prompt(&prompt_str, use_chat_template, args.thinking);
|
|
|
|
println!("\n=== Generation Settings ===");
|
|
println!("Model type: {:?}", args.model_type);
|
|
println!(
|
|
"Chat template: {}",
|
|
if use_chat_template {
|
|
"enabled"
|
|
} else {
|
|
"disabled"
|
|
}
|
|
);
|
|
println!(
|
|
"Thinking mode: {}",
|
|
if args.thinking {
|
|
"enabled (/think)"
|
|
} else {
|
|
"disabled (/no_think)"
|
|
}
|
|
);
|
|
println!("Raw prompt: {}", prompt_str);
|
|
|
|
// Encode prompt
|
|
let tokens = tos
|
|
.tokenizer()
|
|
.encode(formatted_prompt.as_str(), false)
|
|
.map_err(E::msg)?;
|
|
let tokens = tokens.get_ids();
|
|
println!("Encoded {} tokens", tokens.len());
|
|
|
|
// Setup logits processor
|
|
let sampling = if args.temperature <= 0.0 {
|
|
Sampling::ArgMax
|
|
} else {
|
|
match (args.top_k, args.top_p) {
|
|
(None, None) => Sampling::All {
|
|
temperature: args.temperature,
|
|
},
|
|
(Some(k), None) => Sampling::TopK {
|
|
k,
|
|
temperature: args.temperature,
|
|
},
|
|
(None, Some(p)) => Sampling::TopP {
|
|
p,
|
|
temperature: args.temperature,
|
|
},
|
|
(Some(k), Some(p)) => Sampling::TopKThenTopP {
|
|
k,
|
|
p,
|
|
temperature: args.temperature,
|
|
},
|
|
}
|
|
};
|
|
let mut logits_processor = LogitsProcessor::from_sampling(args.seed, sampling);
|
|
|
|
// Process prompt
|
|
let start_prompt = std::time::Instant::now();
|
|
let mut next_token = if !args.split_prompt {
|
|
let input = Tensor::new(tokens, device)?.unsqueeze(0)?;
|
|
let logits = model.forward(&input, 0)?;
|
|
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
|
|
logits_processor.sample(&logits)?
|
|
} else {
|
|
let mut next_token = 0;
|
|
for (pos, &token) in tokens.iter().enumerate() {
|
|
let input = Tensor::new(&[token], device)?.unsqueeze(0)?;
|
|
let logits = model.forward(&input, pos)?;
|
|
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
|
|
next_token = logits_processor.sample(&logits)?;
|
|
}
|
|
next_token
|
|
};
|
|
let prompt_dt = start_prompt.elapsed();
|
|
|
|
// Get EOS token
|
|
let config = model.config();
|
|
let eos_token = get_eos_token(tos.tokenizer(), &config);
|
|
|
|
// Generate tokens
|
|
let mut all_tokens = vec![next_token];
|
|
print!("\n=== Output ===\n");
|
|
if let Some(t) = tos.next_token(next_token)? {
|
|
print!("{t}");
|
|
std::io::stdout().flush()?;
|
|
}
|
|
|
|
let start_generation = std::time::Instant::now();
|
|
let to_sample = args.sample_len.saturating_sub(1);
|
|
let mut sampled = 0;
|
|
|
|
for index in 0..to_sample {
|
|
let input = Tensor::new(&[next_token], device)?.unsqueeze(0)?;
|
|
let logits = model.forward(&input, tokens.len() + index)?;
|
|
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
|
|
|
|
let logits = if args.repeat_penalty == 1.0 {
|
|
logits
|
|
} else {
|
|
let start_at = all_tokens.len().saturating_sub(args.repeat_last_n);
|
|
hanzo_transformers::utils::apply_repeat_penalty(
|
|
&logits,
|
|
args.repeat_penalty,
|
|
&all_tokens[start_at..],
|
|
)?
|
|
};
|
|
|
|
next_token = logits_processor.sample(&logits)?;
|
|
all_tokens.push(next_token);
|
|
|
|
if let Some(t) = tos.next_token(next_token)? {
|
|
print!("{t}");
|
|
std::io::stdout().flush()?;
|
|
}
|
|
|
|
sampled += 1;
|
|
if next_token == eos_token {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if let Some(rest) = tos.decode_rest().map_err(E::msg)? {
|
|
print!("{rest}");
|
|
}
|
|
|
|
let generation_dt = start_generation.elapsed();
|
|
|
|
// Print statistics
|
|
println!(
|
|
"\n\n=== Statistics ===\n\
|
|
{:4} prompt tokens processed: {:.2} token/s\n\
|
|
{:4} tokens generated: {:.2} token/s",
|
|
tokens.len(),
|
|
tokens.len() as f64 / prompt_dt.as_secs_f64(),
|
|
sampled,
|
|
sampled as f64 / generation_dt.as_secs_f64(),
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
// ==================== Main ====================
|
|
|
|
fn print_model_info(config: &ModelConfig) {
|
|
println!("\n=== Model Configuration ===");
|
|
println!("Vocab size: {}", config.vocab_size);
|
|
println!("Hidden size: {}", config.hidden_size);
|
|
println!("Num layers: {}", config.num_hidden_layers);
|
|
println!("Num attention heads: {}", config.num_attention_heads);
|
|
println!("Num KV heads: {}", config.num_key_value_heads);
|
|
println!("Head dim: {}", config.head_dim());
|
|
println!("RoPE theta: {:.0}", config.rope_theta);
|
|
|
|
// Print RoPE/NoPE layer info for full models
|
|
if let Some(ref no_rope_layers) = config.no_rope_layers {
|
|
let num_rope_layers = no_rope_layers.iter().filter(|&&x| x == 1).count();
|
|
let num_nope_layers = no_rope_layers.iter().filter(|&&x| x == 0).count();
|
|
println!("\nLayer Configuration:");
|
|
println!(
|
|
" RoPE layers: {} ({}%)",
|
|
num_rope_layers,
|
|
num_rope_layers * 100 / config.num_hidden_layers
|
|
);
|
|
println!(
|
|
" NoPE layers: {} ({}%)",
|
|
num_nope_layers,
|
|
num_nope_layers * 100 / config.num_hidden_layers
|
|
);
|
|
} else if let Some(interval) = config.no_rope_layer_interval {
|
|
let num_nope_layers = config.num_hidden_layers / interval;
|
|
let num_rope_layers = config.num_hidden_layers - num_nope_layers;
|
|
println!("\nLayer Configuration:");
|
|
println!(
|
|
" RoPE layers: {} ({}%)",
|
|
num_rope_layers,
|
|
num_rope_layers * 100 / config.num_hidden_layers
|
|
);
|
|
println!(
|
|
" NoPE layers: {} ({}%) - every {}th layer",
|
|
num_nope_layers,
|
|
num_nope_layers * 100 / config.num_hidden_layers,
|
|
interval
|
|
);
|
|
}
|
|
}
|
|
|
|
fn main() -> Result<()> {
|
|
use tracing_chrome::ChromeLayerBuilder;
|
|
use tracing_subscriber::prelude::*;
|
|
|
|
let args = Args::parse();
|
|
|
|
let _guard = if args.tracing {
|
|
let (chrome_layer, guard) = ChromeLayerBuilder::new().build();
|
|
tracing_subscriber::registry().with(chrome_layer).init();
|
|
Some(guard)
|
|
} else {
|
|
None
|
|
};
|
|
|
|
println!("=== SmolLM3 Unified Inference ===");
|
|
println!(
|
|
"avx: {}, neon: {}, simd128: {}, f16c: {}",
|
|
hanzo_ml::utils::with_avx(),
|
|
hanzo_ml::utils::with_neon(),
|
|
hanzo_ml::utils::with_simd128(),
|
|
hanzo_ml::utils::with_f16c()
|
|
);
|
|
println!(
|
|
"temp: {:.2}, repeat-penalty: {:.2}, repeat-last-n: {}",
|
|
args.temperature, args.repeat_penalty, args.repeat_last_n
|
|
);
|
|
|
|
let start = std::time::Instant::now();
|
|
let device = hanzo_ml_examples::device(false)?;
|
|
|
|
// Load model
|
|
let mut model = match args.model_type {
|
|
ModelType::Quantized => load_quantized_model(&args, &device)?,
|
|
ModelType::Full => load_full_model(&args, &device)?,
|
|
};
|
|
|
|
println!("Model loaded in {:.2}s", start.elapsed().as_secs_f32());
|
|
|
|
// Print model info
|
|
let config = model.config();
|
|
print_model_info(&config);
|
|
|
|
// Load tokenizer
|
|
let tokenizer = args.get_tokenizer()?;
|
|
|
|
// Run generation
|
|
run_generation(&mut model, tokenizer, &args, &device)?;
|
|
|
|
Ok(())
|
|
}
|