mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
Eric BuehlerandGitHub
ebf1c27adc
* Nice, easy rust API * Nicer example * Docs * Update docs * Support lora rust api model * Reorganize * Add the examples * Update tool calling example * Fix crate name * Add messages_ref * Add llama 8b specific example * Add gguf, lora/xlora models * Export them * Doctests build * Add support for vision models * Add some docs * Even *more* docs * Format
32 lines
962 B
Rust
32 lines
962 B
Rust
use std::fs::File;
|
|
|
|
use anyhow::Result;
|
|
use mistralrs::{TextMessageRole, TextMessages, TextModelBuilder, XLoraModelBuilder};
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
let model =
|
|
XLoraModelBuilder::from_text_model_builder(
|
|
TextModelBuilder::new("HuggingFaceH4/zephyr-7b-beta").with_logging(),
|
|
"lamm-mit/x-lora",
|
|
serde_json::from_reader(File::open("my-ordering-file.json").unwrap_or_else(|_| {
|
|
panic!("Could not load ordering file at my-ordering-file.json")
|
|
}))?,
|
|
)
|
|
.build()
|
|
.await?;
|
|
|
|
let messages =
|
|
TextMessages::new().add_message(TextMessageRole::User, "Hello! What is graphene.");
|
|
|
|
let response = model.send_chat_request(messages).await?;
|
|
|
|
println!("{}", response.choices[0].message.content.as_ref().unwrap());
|
|
dbg!(
|
|
response.usage.avg_prompt_tok_per_sec,
|
|
response.usage.avg_compl_tok_per_sec
|
|
);
|
|
|
|
Ok(())
|
|
}
|