mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
2c9da3438a
* Add the tool * Actually search * Clippy * Sort of works * Remove some debuggers * tweak * Add some rules * Works great * Tweak 'system' prompt * Update mistralrs-core/src/search/mod.rs Co-authored-by: Copilot <[email protected]> * Typo * Add it to all the apis * Add bert model for similarity reranking * Typos * Early detection of tools * Alias max_tokens -> max_completion_tokens too * Customizable bert model * Flip the enabler around * Add docs * Update readme * Typo --------- Co-authored-by: Copilot <[email protected]>
34 lines
1004 B
Rust
34 lines
1004 B
Rust
use anyhow::Result;
|
|
use mistralrs::{
|
|
BertEmbeddingModel, IsqType, RequestBuilder, TextMessageRole, TextMessages, TextModelBuilder,
|
|
WebSearchOptions,
|
|
};
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
let model = TextModelBuilder::new("NousResearch/Hermes-3-Llama-3.1-8B")
|
|
.with_isq(IsqType::Q4K)
|
|
.with_logging()
|
|
.with_search(BertEmbeddingModel::default())
|
|
.build()
|
|
.await?;
|
|
|
|
let messages = TextMessages::new().add_message(
|
|
TextMessageRole::User,
|
|
"What is the weather forecast for Boston?",
|
|
);
|
|
let messages =
|
|
RequestBuilder::from(messages).with_web_search_options(WebSearchOptions::default());
|
|
|
|
let response = model.send_chat_request(messages).await?;
|
|
|
|
println!("What is the weather forecast for Boston?\n\n");
|
|
println!("{}", response.choices[0].message.content.as_ref().unwrap());
|
|
dbg!(
|
|
response.usage.avg_prompt_tok_per_sec,
|
|
response.usage.avg_compl_tok_per_sec
|
|
);
|
|
|
|
Ok(())
|
|
}
|