mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
Michał MoskalandGitHub
4b92d4b49b
* import llguidance modules * llg constraint types * integrate llguidance * fix handling of stop tokens * update toktrie * remove submodules * fix version conflicts * tok_trie -> tok_env rename * update to latest llguidance * sync lock * bump llg (lazy_static fix) * update to latest llguidance, fix conflicts * import toktrie via llguidance * n=1 * test with llama1b * remove aici folder (no longer used) * use more specific type for llg grammars * update python APIs to support json schema and llg * update example to use lark not yacc * rename example * remove testing scripts * re-export llguidance for easier LlguidanceGrammar construction * fix formatting * fix clippy * add python samples * add server samples * add rust samples
39 lines
1.3 KiB
Rust
39 lines
1.3 KiB
Rust
use anyhow::Result;
|
|
use mistralrs::{
|
|
IsqType, PagedAttentionMetaBuilder, RequestBuilder, TextMessageRole, TextModelBuilder,
|
|
};
|
|
use serde_json::json;
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
let model = TextModelBuilder::new("microsoft/Phi-3.5-mini-instruct")
|
|
.with_isq(IsqType::Q4K)
|
|
.with_logging()
|
|
.with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
|
|
.build()
|
|
.await?;
|
|
|
|
let request = RequestBuilder::new()
|
|
.set_constraint(mistralrs::Constraint::JsonSchema(json!(
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"street": {"type": "string"},
|
|
"city": {"type": "string"},
|
|
"state": {"type": "string", "pattern": "^[A-Z]{2}$"},
|
|
"zip": {"type": "integer", "minimum": 10000, "maximum": 99999},
|
|
},
|
|
"required": ["street", "city", "state", "zip"],
|
|
"additionalProperties": false,
|
|
}
|
|
)))
|
|
.set_sampler_max_len(100)
|
|
.add_message(TextMessageRole::User, "A sample address please.");
|
|
|
|
let response = model.send_chat_request(request).await?;
|
|
|
|
println!("{}", response.choices[0].message.content.as_ref().unwrap());
|
|
|
|
Ok(())
|
|
}
|