mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
c3be7b6be4
* Add model * Update loader * Fix llama config usage * Docs * Fix config no_rope_layers * Fix tie_word_embeddings default * Add chat template * Embed the chat templates * Fix embedding template * enable_thinking default true * Update examples * XML tools for smollm3 * Add smollm3 docs * Fix openai examples * Clippy --------- Co-authored-by: Eric Buehler <[email protected]>
36 lines
1.0 KiB
Rust
36 lines
1.0 KiB
Rust
use anyhow::Result;
|
|
use mistralrs::{
|
|
IsqType, PagedAttentionMetaBuilder, TextMessageRole, TextMessages, TextModelBuilder,
|
|
};
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
let model = TextModelBuilder::new("HuggingFaceTB/SmolLM3-3B")
|
|
.with_isq(IsqType::Q8_0)
|
|
.with_logging()
|
|
.with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
|
|
.build()
|
|
.await?;
|
|
|
|
let messages = TextMessages::new()
|
|
// .enable_thinking(false)
|
|
.add_message(
|
|
TextMessageRole::System,
|
|
"You are an AI agent with a specialty in programming.",
|
|
)
|
|
.add_message(
|
|
TextMessageRole::User,
|
|
"Hello! How are you? Please write generic binary search function in Rust.",
|
|
);
|
|
|
|
let response = model.send_chat_request(messages).await?;
|
|
|
|
println!("{}", response.choices[0].message.content.as_ref().unwrap());
|
|
dbg!(
|
|
response.usage.avg_prompt_tok_per_sec,
|
|
response.usage.avg_compl_tok_per_sec
|
|
);
|
|
|
|
Ok(())
|
|
}
|