mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-20 23:08:29 +00:00
Three-way per file through the rename: base 39355c6c91, theirs the upstream
merge of main with 35d7ae7c (via 7ef6a694d), ours origin/main.
Lands the barrier thread pool, runtime CPU feature dispatch, aarch64 and x86
repacked quantized kernels (indexed_gemv, gemv_fused_shared_lhs), quantized
embedding on CPU/Metal/CUDA, paged flash-attn kernels, cutile fused MoE, the
in-place op aliasing API, parking_lot storage locks, Metal buffer builders and
debug labels, safetensors 0.8, hf-hub 1.0, tokenizers 0.23.
Ours keeps ROCm, Vulkan, wgpu, the IQ/MXFP4/NVFP4/TQ dtypes, and the i-quant
weight/activation block split in matmul. Our backends ride InplaceOpN and the
quantized embedding; Metal get_rows gains the IQ kernels. ug leaves, as upstream.
Co-authored-by: Hanzo Dev <[email protected]>
76 lines
2.2 KiB
Rust
76 lines
2.2 KiB
Rust
use anyhow::{Context, Result};
|
|
use clap::Parser;
|
|
use hanzo_ml_examples::hub::Api;
|
|
use model::VoxtralModel;
|
|
|
|
mod download;
|
|
mod model;
|
|
|
|
#[derive(Parser, Debug)]
|
|
#[command(author, version, about, long_about = None)]
|
|
struct Args {
|
|
/// Run on CPU rather than on GPU.
|
|
#[arg(long, default_value_t = false)]
|
|
cpu: bool,
|
|
|
|
/// The input to be processed, in wav format, will default to `jfk.wav`. Alternatively
|
|
/// this can be set to sample:jfk, sample:gb1, ... to fetch a sample from the following
|
|
/// repo: https://huggingface.co/datasets/Narsil/hanzo_demo/
|
|
#[arg(long)]
|
|
input: Option<String>,
|
|
|
|
#[arg(long, default_value = "mistralai/Voxtral-Mini-3B-2507")]
|
|
model_id: Option<String>,
|
|
}
|
|
|
|
#[cfg(feature = "cuda")]
|
|
fn use_cpu() -> bool {
|
|
true
|
|
}
|
|
|
|
#[cfg(not(feature = "cuda"))]
|
|
fn use_cpu() -> bool {
|
|
false
|
|
}
|
|
|
|
fn main() -> Result<()> {
|
|
let args = Args::parse();
|
|
|
|
let use_cpu = args.cpu || !use_cpu();
|
|
|
|
let model_id = args.model_id.unwrap();
|
|
|
|
// Create model - equivalent to loading the model and processor in Python
|
|
let mut model =
|
|
VoxtralModel::new(&model_id, use_cpu).context("Failed to load Voxtral model")?;
|
|
|
|
println!("Model loaded successfully on device: {:?}", model.device());
|
|
|
|
let api = Api::new()?;
|
|
let dataset = api.dataset("Narsil/hanzo-ml-examples");
|
|
|
|
let audio_file = if let Some(input) = args.input {
|
|
if let Some(sample) = input.strip_prefix("sample:") {
|
|
dataset.get(format!("samples_{sample}.wav"))?
|
|
} else {
|
|
std::path::PathBuf::from(input)
|
|
}
|
|
} else {
|
|
println!("No audio file submitted: Downloading https://huggingface.co/datasets/Narsil/hanzo_demo/blob/main/samples_jfk.wav");
|
|
dataset.get("samples_jfk.wav")?
|
|
};
|
|
|
|
let (audio_data, sample_rate) =
|
|
hanzo_ml_examples::audio::pcm_decode(audio_file).context("Failed to decode audio file")?;
|
|
|
|
// Transcribe audio with token output
|
|
let result = model
|
|
.transcribe_audio(&audio_data, sample_rate)
|
|
.context("Failed to transcribe audio with tokens")?;
|
|
|
|
println!("\n===================================================\n");
|
|
println!("{}", result.text);
|
|
|
|
Ok(())
|
|
}
|