mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
Eric BuehlerandGitHub
0e027790f2
* Initial * Markdown * Copy code * Add model loading sidebar * Support vision models * Tweak isq * Links go to another page * Clear when switch model * Fix html tags * Add image support! * More then one images * Fix * Improved textarea * Tab for switching between vision and text * No paged attn for now * Prettier format * Multiple models at once * Better switching, clearing ability * Mobile support * Inline markdown parser * Update examples * Typos * Support specifying isq * Fix mobile * Fixes * Fix button on mobile * Image height is capped * Thumbnail * Fix rotating kv cache edge case * Add drag and drop for images * Small things * Sidebar is frozen now * Better listner * Add readme * Tweak readme
48 lines
1.4 KiB
Rust
48 lines
1.4 KiB
Rust
use anyhow::Result;
|
|
use mistralrs::{
|
|
AutoDeviceMapParams, DeviceMapSetting, IsqType, TextMessageRole, VisionMessages,
|
|
VisionModelBuilder,
|
|
};
|
|
|
|
// const MODEL_ID: &str = "meta-llama/Llama-3.2-11B-Vision-Instruct";
|
|
const MODEL_ID: &str = "lamm-mit/Cephalo-Llama-3.2-11B-Vision-Instruct-128k";
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
let auto_map_params = AutoDeviceMapParams::Text {
|
|
max_seq_len: 4096,
|
|
max_batch_size: 2,
|
|
};
|
|
let model = VisionModelBuilder::new(MODEL_ID)
|
|
.with_isq(IsqType::Q4K)
|
|
.with_logging()
|
|
.with_device_mapping(DeviceMapSetting::Auto(auto_map_params))
|
|
.build()
|
|
.await?;
|
|
|
|
let bytes = match reqwest::blocking::get(
|
|
"https://cdn.britannica.com/45/5645-050-B9EC0205/head-treasure-flower-disk-flowers-inflorescence-ray.jpg",
|
|
) {
|
|
Ok(http_resp) => http_resp.bytes()?.to_vec(),
|
|
Err(e) => anyhow::bail!(e),
|
|
};
|
|
let image = image::load_from_memory(&bytes)?;
|
|
|
|
let messages = VisionMessages::new().add_image_message(
|
|
TextMessageRole::User,
|
|
"What is depicted here? Please describe the scene in detail.",
|
|
vec![image],
|
|
&model,
|
|
)?;
|
|
|
|
let response = model.send_chat_request(messages).await?;
|
|
|
|
println!("{}", response.choices[0].message.content.as_ref().unwrap());
|
|
dbg!(
|
|
response.usage.avg_prompt_tok_per_sec,
|
|
response.usage.avg_compl_tok_per_sec
|
|
);
|
|
|
|
Ok(())
|
|
}
|