mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
c3be7b6be4
* Add model * Update loader * Fix llama config usage * Docs * Fix config no_rope_layers * Fix tie_word_embeddings default * Add chat template * Embed the chat templates * Fix embedding template * enable_thinking default true * Update examples * XML tools for smollm3 * Add smollm3 docs * Fix openai examples * Clippy --------- Co-authored-by: Eric Buehler <[email protected]>
50 lines
2.0 KiB
Rust
50 lines
2.0 KiB
Rust
use anyhow::Result;
|
|
use mistralrs::{IsqType, TextMessageRole, TextMessages, TextModelBuilder};
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
let model = TextModelBuilder::new("Qwen/Qwen3-30B-A3B")
|
|
.with_isq(IsqType::Q4K)
|
|
.with_logging()
|
|
.build()
|
|
.await?;
|
|
|
|
let mut messages = TextMessages::new();
|
|
|
|
// ------------------------------------------------------------------
|
|
// First question, thinking mode is enabled by default
|
|
// ------------------------------------------------------------------
|
|
messages = messages.add_message(TextMessageRole::User, "Hello! How many rs in strawberry?");
|
|
let response = model.send_chat_request(messages.clone()).await?;
|
|
|
|
println!("{}", response.choices[0].message.content.as_ref().unwrap());
|
|
|
|
messages = messages.add_message(
|
|
TextMessageRole::Assistant,
|
|
response.choices[0].message.content.as_ref().unwrap(),
|
|
);
|
|
|
|
// ------------------------------------------------------------------
|
|
// Second question, disable thinking mode with RequestBuilder or /no_think
|
|
// ------------------------------------------------------------------
|
|
messages = messages.add_message(TextMessageRole::User, "How many rs in blueberry? /no_think");
|
|
let response = model.send_chat_request(messages.clone()).await?;
|
|
|
|
println!("{}", response.choices[0].message.content.as_ref().unwrap());
|
|
|
|
messages = messages.add_message(
|
|
TextMessageRole::Assistant,
|
|
response.choices[0].message.content.as_ref().unwrap(),
|
|
);
|
|
|
|
// ------------------------------------------------------------------
|
|
// Third question, reenable thinking mode with RequestBuilder or /think
|
|
// ------------------------------------------------------------------
|
|
messages = messages.add_message(TextMessageRole::User, "Are you sure? /think");
|
|
let response = model.send_chat_request(messages).await?;
|
|
|
|
println!("{}", response.choices[0].message.content.as_ref().unwrap());
|
|
|
|
Ok(())
|
|
}
|