mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
Eric BuehlerandGitHub
d8c48b2326
* Nice, easy rust API * Nicer example * Docs * Update docs * Support lora rust api model * Reorganize * Add the examples * Update tool calling example * Fix crate name * Add messages_ref * Add llama 8b specific example * Add gguf, lora/xlora models * Export them * Doctests build * Add support for vision models * Add some docs * Even *more* docs * Format
28 lines
831 B
Rust
28 lines
831 B
Rust
use anyhow::Result;
|
|
use mistralrs::{
|
|
IsqType, PagedAttentionMetaBuilder, RequestBuilder, TextMessageRole, TextModelBuilder,
|
|
};
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
|
|
.with_isq(IsqType::Q4K)
|
|
.with_logging()
|
|
.with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
|
|
.build()
|
|
.await?;
|
|
|
|
// Bullet list regex
|
|
let request = RequestBuilder::new()
|
|
.set_constraint(mistralrs::Constraint::Regex(
|
|
"(- [^\n]*\n)+(- [^\n]*)(\n\n)?".to_string(),
|
|
))
|
|
.add_message(TextMessageRole::User, "Please write a few jokes.");
|
|
|
|
let response = model.send_chat_request(request).await?;
|
|
|
|
println!("{}", response.choices[0].message.content.as_ref().unwrap());
|
|
|
|
Ok(())
|
|
}
|