Files
engine/hanzo-engine/src/pipeline/sampling.rs

1019 lines
41 KiB
Rust

use std::sync::Arc;
use hanzo_ml::{DType, Result, Tensor};
use rand_isaac::Isaac64Rng;
use crate::{
prefix_cacher::PrefixCacheManagerV2,
sampler::Logprobs,
sequence::{Sequence, SequenceRecognizer, SequenceState, StopReason},
tools::{parse_text_tools, ToolCallResponse, ToolCallType},
};
use hanzo_llm_mcp::CalledFunction;
use super::Pipeline;
macro_rules! fixup_sentencepiece {
($txt:expr) => {
$txt.to_string().replace("▁", " ")
};
(Option $txt:expr) => {
match &$txt {
Some(txt) => Some(fixup_sentencepiece!(txt)),
None => None,
}
};
}
fn parse_text_and_tool_calls(
raw_text: &str,
matcher: Option<Arc<crate::tools::ToolCallingMatcher>>,
) -> Result<(Option<String>, Vec<ToolCallResponse>)> {
parse_text_tools(raw_text, matcher).map_err(hanzo_ml::Error::msg)
}
/// One streamed delta's content and tool calls. Calls are read from the answer only: with a
/// reasoning parser, from its content delta, never the raw text, which holds the think block too.
fn parse_streaming_text_and_tool_calls(
content_delta: Option<String>,
raw_delta: &str,
has_reasoning_parser: bool,
matcher: Option<Arc<crate::tools::ToolCallingMatcher>>,
) -> Result<(Option<String>, Vec<ToolCallResponse>)> {
let raw_text = match content_delta {
Some(content_delta) => content_delta,
None if has_reasoning_parser => return Ok((None, Vec::new())),
None => raw_delta.to_string(),
};
parse_text_and_tool_calls(raw_text.as_str(), matcher)
}
fn status_for_stop_reason(
reason: crate::sequence::StopReason,
) -> crate::engine::ledger::TerminalStatus {
if matches!(reason, crate::sequence::StopReason::Canceled) {
crate::engine::ledger::TerminalStatus::Canceled
} else {
crate::engine::ledger::TerminalStatus::Completed
}
}
/// Reads usage + execution trace under one `SequenceGroup` lock, releases the lock before
/// entering the ledger, and records the terminal request event once (`serve_api:` + `serve_trace:`).
#[cfg(test)]
fn record_group_finish(
drafter: Option<crate::speculative::SpeculativeAttachInfo>,
seq: &Sequence,
clear_group_totals: bool,
) -> crate::Usage {
record_group_finish_with_status(
drafter,
seq,
clear_group_totals,
crate::engine::ledger::TerminalStatus::Completed,
)
}
fn record_group_finish_with_status(
drafter: Option<crate::speculative::SpeculativeAttachInfo>,
seq: &Sequence,
clear_group_totals: bool,
status: crate::engine::ledger::TerminalStatus,
) -> crate::Usage {
let (should_log, usage, trace) = {
let mut group = seq.get_mut_group();
let should_log = !std::mem::replace(&mut group.trace_logged, true);
let usage = group.get_usage();
if clear_group_totals {
group.total_prompt_toks = 0;
group.total_toks = 0;
}
(should_log, usage, group.execution_trace.clone())
};
if should_log {
let structured = matches!(seq.recognizer, SequenceRecognizer::Llguidance(_))
&& !seq.is_tool_grammar_active();
crate::engine::ledger::record_with_trace(
&usage,
drafter.as_ref().map(|d| d.name()),
structured,
&trace,
*seq.id(),
status,
);
}
usage
}
pub(crate) async fn finish_or_add_toks_to_seq(
this: &dyn Pipeline,
prefix_cacher: &mut PrefixCacheManagerV2,
seq: &mut Sequence,
logprobs: Logprobs,
eos_tok: Option<&[u32]>,
use_prefix_cacher: bool,
) -> Result<()> {
// Record prefill wall-time once, at the first sampled token and before any
// detokenization or response construction below. `step_start_instant` is set
// just before the prompt forward; sampling has already forced that forward's
// device sync, so this elapsed span is the true prompt-forward -> first-token
// latency -- identical for single-token (max_len=1 / prompt benchmarks, which
// finish inside the prompt step) and multi-token requests, on every backend.
// Recording it here (not after step() returns) keeps post-forward overhead
// out of the number so avg_prompt_tok_per_sec reflects real prefill throughput.
if seq.total_prompt_time.is_none() {
if let Some(start) = seq.step_start_instant {
seq.total_prompt_time = Some(start.elapsed().as_millis());
}
}
let mut is_done = seq.is_done(logprobs.token, eos_tok, this.get_metadata().max_seq_len);
let metadata = this.get_metadata();
let tok_env = metadata.tok_env().ok_or(hanzo_ml::Error::Msg(
"`finish_or_add_toks_to_seq` requires the pipeline to have a token trie".to_string(),
))?;
// Include special tokens when tool calling is active (so tool parsers can see
// delimiters like <tool_call>, [TOOL_CALLS], <|python_tag|>) or when think tag
// mode is enabled (so <think>/<\/think> delimiters are visible in the output).
let include_special = seq.tools.is_some() || seq.needs_special_tokens();
let completion_bytes = tok_env
.tok_trie()
.decode_ext(&[logprobs.token], include_special);
if let Some(stop) = seq.add_token(logprobs.clone(), completion_bytes, &is_done) {
is_done = Some(stop);
}
// If we can have a tool and we got a tool, stop the sequence early.
// Doesn't conflict with the logic below because it does the same thing anyway.
if let Some(ref t) = seq.tools {
if let Ok(Some(ref d)) = seq.peek_delta() {
let (_tool_use_still_possible, tool_use_is_done) =
t.prefix_could_be_tool(d.as_str())?;
if tool_use_is_done
&& parse_text_tools(d, seq.tools.clone()).is_ok_and(|(_, calls)| !calls.is_empty())
{
seq.set_state(SequenceState::Done(StopReason::Eos));
seq.release_held();
is_done = Some(StopReason::Eos);
}
}
};
// Mid-stream grammar activation for tool calls.
// When a tool call prefix is detected and no grammar is already active,
// build a format-specific grammar and activate it so subsequent tokens
// are constrained to valid tool call syntax.
// For Harmony mode, tool calls are signaled by token-level markers (not
// text prefixes), so we also check HarmonyContext for new tool calls.
//
// Skip when the sequence is already done — peek_delta() still contains
// the tool call prefix from earlier generation, which would spuriously
// re-activate the grammar on a completed sequence.
if matches!(seq.recognizer, SequenceRecognizer::None) && is_done.is_none() {
// Try text-based detection first, then Harmony token-based detection.
let grm = if let Some(ref t) = seq.tools {
seq.peek_delta()
.ok()
.flatten()
.and_then(|text| t.build_tool_call_grammar(text.as_str()))
} else {
None
};
let grm = grm.or_else(|| {
if seq.needs_harmony_tool_grammar() {
let recipient = seq.harmony_current_tool_recipient();
seq.tools
.as_ref()?
.build_harmony_tool_grammar(recipient.as_deref())
} else {
None
}
});
if let Some(grm) = grm {
if let Some(ref factory) = metadata.llg_factory {
match crate::pipeline::llg::constraint_from_llg_grammar(factory, grm) {
Ok(matcher) => {
tracing::debug!("Activated tool call grammar");
seq.recognizer = SequenceRecognizer::Llguidance(Box::new(matcher));
seq.set_tool_grammar_active(true);
}
Err(e) => {
tracing::warn!(
"Failed to build tool call grammar: {e}. \
Continuing without constraint."
);
}
}
}
}
}
// Handle streaming requests
if seq.get_mut_group().is_streaming {
let mut tool_use_still_possible = false;
let mut tool_use_is_done = false;
if let Some(ref t) = seq.tools {
if let Ok(Some(ref d)) = seq.peek_delta() {
(tool_use_still_possible, tool_use_is_done) = t.prefix_could_be_tool(d.as_str())?;
}
};
// Send chunks when:
// 1. Tool call is not possible (!tool_use_still_possible) - normal streaming
// 2. Tool call is complete (tool_use_is_done) - send the tool call
// 3. Sequence is done (is_done.is_some()) - send buffered output as text since it wasn't a valid tool call
if !tool_use_still_possible || tool_use_is_done || is_done.is_some() {
if is_done.is_some() && seq.reasoning_mode().is_some() {
seq.finalize_reasoning();
}
let delta_result = seq.get_delta();
if let Some(delta) = crate::handle_seq_error_stateaware_ok!(delta_result, seq) {
if seq.get_mut_group().is_chat {
let (content_delta, reasoning_delta, tool_calls) = if seq.is_harmony_mode() {
let content_delta = seq.get_response_content_delta();
let reasoning_delta = seq.get_reasoning_content_delta();
// In Harmony mode, only finalize tool calls when the sequence is done
// (EOS token or stop string), not when we first detect a tool call.
// This ensures tool call arguments are fully generated.
let tool_calls = if is_done.is_some() && seq.has_harmony_tool_calls() {
// Sequence is done and has tool calls - finalize and send them
is_done = Some(StopReason::ToolCalls);
let harmony_tool_calls = seq.get_harmony_tool_calls();
harmony_tool_calls
.into_iter()
.enumerate()
.map(|(i, tc)| ToolCallResponse {
index: i,
id: tc.id,
tp: ToolCallType::Function,
function: CalledFunction {
name: tc.name,
arguments: tc.arguments,
},
})
.collect()
} else {
vec![]
};
(content_delta, reasoning_delta, tool_calls)
} else {
// A call's own text is not content.
let (content_delta, tool_calls) = parse_streaming_text_and_tool_calls(
seq.get_response_content_delta(),
delta.as_str(),
seq.reasoning_mode().is_some(),
seq.tools.clone(),
)?;
if !tool_calls.is_empty() {
is_done = Some(StopReason::ToolCalls);
}
(content_delta, seq.get_reasoning_content_delta(), tool_calls)
};
seq.add_streaming_chunk_choice_to_group(crate::ChunkChoice {
delta: crate::Delta {
content: fixup_sentencepiece!(Option content_delta),
role: "assistant".to_string(),
tool_calls: Some(tool_calls).filter(|v| !v.is_empty()),
reasoning_content: reasoning_delta,
},
index: seq.get_response_index(),
finish_reason: is_done.map(|x| x.to_string()),
logprobs: if seq.return_logprobs() {
Some(crate::ResponseLogprob {
token: delta,
bytes: logprobs.bytes.clone().map(|b| b.into_bytes()),
logprob: logprobs.logprob,
top_logprobs: logprobs.top_logprobs.unwrap().clone(),
})
} else {
None
},
});
} else {
seq.add_streaming_completion_chunk_choice_to_group(
crate::CompletionChunkChoice {
text: fixup_sentencepiece!(delta),
index: seq.get_response_index(),
finish_reason: is_done.map(|x| x.to_string()),
logprobs: if seq.return_logprobs() {
Some(crate::CompletionLogprobs::new(
&[crate::ResponseLogprob {
token: delta,
bytes: logprobs.bytes.clone().map(|b| b.into_bytes()),
logprob: logprobs.logprob,
top_logprobs: logprobs.top_logprobs.unwrap().clone(),
}],
0,
))
} else {
None
},
},
);
}
}
// Send usage on final chunk, recording terminal status after delivery check.
let final_snapshot = if let Some(reason) = is_done {
seq.update_time_info();
let (should_log, usage, trace) = {
let mut group = seq.get_mut_group();
let should_log = !std::mem::replace(&mut group.trace_logged, true);
let usage = group.get_usage();
group.total_prompt_toks = 0;
group.total_toks = 0;
(should_log, usage, group.execution_trace.clone())
};
Some((reason, should_log, usage, trace))
} else {
None
};
let usage_opt = final_snapshot.as_ref().map(|(_, _, u, _)| u.clone());
let send_failed = seq
.get_mut_group()
.maybe_send_streaming_response(seq, this.name().clone(), usage_opt)
.await
.is_err();
if let Some((reason, should_log, usage, trace)) = final_snapshot {
if should_log {
let status = if send_failed {
crate::engine::ledger::TerminalStatus::Canceled
} else {
status_for_stop_reason(reason)
};
let structured = matches!(seq.recognizer, SequenceRecognizer::Llguidance(_))
&& !seq.is_tool_grammar_active();
crate::engine::ledger::record_with_trace(
&usage,
this.drafter().as_ref().map(|d| d.name()),
structured,
&trace,
*seq.id(),
status,
);
}
}
if send_failed {
if is_done.is_none() {
seq.update_time_info();
record_group_finish_with_status(
this.drafter(),
seq,
true,
crate::engine::ledger::TerminalStatus::Canceled,
);
}
// If we can't send the response, cancel the sequence
seq.set_state(crate::sequence::SequenceState::Done(
crate::sequence::StopReason::Canceled,
));
this.reset_non_granular_state();
}
}
// Handle Done state regardless of tool detection - must be outside the tool_use check
// to ensure sequence completes even when tool detection thinks output might be a tool call
if let Some(reason) = is_done {
if use_prefix_cacher {
let recurrent_snapshots = if this.cache().is_hybrid() {
seq.recurrent_state_idx()
.and_then(|idx| this.cache().hybrid().snapshot_recurrent_state(idx).ok())
} else {
None
};
prefix_cacher.add_sequence(seq, recurrent_snapshots);
prefix_cacher.evict_caches()?;
}
seq.set_state(crate::sequence::SequenceState::Done(reason));
this.reset_non_granular_state();
}
} else if let Some(mut reason) = is_done {
/*
***********************
Finish the sequence now
***********************
*/
{
seq.set_state(crate::sequence::SequenceState::Done(reason));
let (tokenizer, pipeline_name) = {
let pipeline_name = this.name();
let tokenizer = this.tokenizer();
(tokenizer, pipeline_name)
};
let logprobs = if seq.return_logprobs() {
let mut logprobs = Vec::new();
for logprob in seq.logprobs() {
let resp_logprob = crate::ResponseLogprob {
token: crate::handle_seq_error_stateaware_ok!(
tokenizer
.as_ref()
.ok_or(hanzo_ml::Error::Msg(
"`finish_or_add_toks_to_seq` requires the pipeline to have a tokenizer"
.to_string(),
))?.decode(&[logprob.token], false),
seq
),
bytes: logprob.bytes.clone().map(|b| b.into_bytes()),
logprob: logprob.logprob,
top_logprobs: logprob.top_logprobs.clone().unwrap(),
};
logprobs.push(resp_logprob);
}
Some(logprobs)
} else {
None
};
// Signal EOS to all reasoning parsers
seq.finalize_reasoning();
// A stop string never enters the completion, so its text is the whole completion.
let text = match reason {
crate::sequence::StopReason::Length(_)
| crate::sequence::StopReason::ModelLength(_)
| crate::sequence::StopReason::Eos
| crate::sequence::StopReason::StopTok(_)
| crate::sequence::StopReason::Canceled
| crate::sequence::StopReason::ToolCalls
| crate::sequence::StopReason::StopString { .. } => {
String::from_utf8_lossy(seq.completion_bytes()).to_string()
}
crate::sequence::StopReason::GeneratedImage
| crate::sequence::StopReason::GeneratedSpeech
| crate::sequence::StopReason::GeneratedFrames
| crate::sequence::StopReason::GeneratedTranscription => {
hanzo_ml::bail!("Stop reason was a non-text generation.")
}
};
// A completion is the exact continuation of its prompt; a chat message drops the
// whitespace the first token carries.
let is_chat = seq.get_mut_group().is_chat;
let text = if is_chat {
text.trim_start().to_string()
} else {
text
};
let mut declined = false;
if is_chat {
let (text_new, tool_calls, reasoning_content) =
if let Some(mode) = seq.reasoning_mode() {
let final_content = seq.get_response_content();
let reasoning = seq.get_reasoning_content();
let (final_content, tool_calls) =
if mode == crate::reasoning_parsers::ReasoningMode::Harmony {
let harmony_tool_calls = seq.get_harmony_tool_calls();
let tool_calls = harmony_tool_calls
.into_iter()
.enumerate()
.map(|(i, tc)| ToolCallResponse {
index: i,
id: tc.id,
tp: ToolCallType::Function,
function: CalledFunction {
name: tc.name,
arguments: tc.arguments,
},
})
.collect();
(final_content, tool_calls)
} else if let Some(ref content) = final_content {
// A call's own text is not content.
parse_text_and_tool_calls(content.as_str(), seq.tools.clone())?
} else {
(None, vec![])
};
(final_content, tool_calls, reasoning)
} else {
let (text_new, tool_calls) =
parse_text_and_tool_calls(text.as_str(), seq.tools.clone())?;
(text_new, tool_calls, None)
};
if !tool_calls.is_empty() {
reason = StopReason::ToolCalls;
}
// A request that had to call a tool and did not is refused,
// not answered with text.
declined = tool_calls.is_empty()
&& seq.tools.as_ref().is_some_and(|tools| tools.must_call());
let choice = crate::Choice {
finish_reason: fixup_sentencepiece!(reason),
index: seq.get_response_index(),
message: crate::ResponseMessage {
content: text_new,
role: "assistant".to_string(),
tool_calls: Some(tool_calls).filter(|v| !v.is_empty()),
reasoning_content,
},
logprobs: logprobs.map(|l| crate::Logprobs { content: Some(l) }),
};
seq.add_choice_to_group(choice);
} else {
let choice = crate::CompletionChoice {
finish_reason: fixup_sentencepiece!(reason),
index: seq.get_response_index(),
text,
logprobs: logprobs.map(|l| crate::CompletionLogprobs::new(&l, 0)),
};
seq.add_completion_choice_to_group(choice);
}
if use_prefix_cacher {
let recurrent_snapshots = if this.cache().is_hybrid() {
seq.recurrent_state_idx()
.and_then(|idx| this.cache().hybrid().snapshot_recurrent_state(idx).ok())
} else {
None
};
prefix_cacher.add_sequence(seq, recurrent_snapshots);
prefix_cacher.evict_caches()?;
}
// Ensure timing info is synced to group before sending response
seq.update_time_info();
let non_stream_status = if declined {
crate::engine::ledger::TerminalStatus::Error
} else {
status_for_stop_reason(reason)
};
record_group_finish_with_status(this.drafter(), seq, false, non_stream_status);
let mut group = seq.get_mut_group();
if group.is_chat {
let response = crate::ChatCompletionResponse {
id: seq.id().to_string(),
choices: group.get_choices().to_vec(),
created: seq.creation_time(),
model: pipeline_name,
system_fingerprint: crate::SYSTEM_FINGERPRINT.to_string(),
object: "chat.completion".to_string(),
usage: group.get_usage(),
agentic_tool_calls: None,
files: None,
session_id: None,
};
if declined {
group
.send_chat_model_error(
crate::tools::FORCED_CALL_DECLINED.to_string(),
response,
seq.responder(),
)
.await
.map_err(hanzo_ml::Error::msg)?;
} else {
group
.maybe_send_chat_done_response(response, seq.responder())
.await
.map_err(hanzo_ml::Error::msg)?;
}
} else {
let completion_response = crate::CompletionResponse {
id: seq.id().to_string(),
choices: group.get_completion_choices(),
created: seq.creation_time(),
model: pipeline_name,
system_fingerprint: crate::SYSTEM_FINGERPRINT.to_string(),
object: "text_completion".to_string(),
usage: group.get_usage(),
};
group
.maybe_send_completion_done_response(completion_response, seq.responder())
.await
.map_err(hanzo_ml::Error::msg)?;
}
}
this.reset_non_granular_state();
}
Ok(())
}
pub async fn sample_and_add_toks(
this: &dyn Pipeline,
seqs: &mut [&mut Sequence],
logits_seq: Vec<Tensor>,
prefix_cacher: &mut PrefixCacheManagerV2,
disable_eos_stop: bool,
rng: Arc<std::sync::Mutex<Isaac64Rng>>,
) -> Result<()> {
let seqs_len = seqs.len();
debug_assert_eq!(logits_seq.len(), seqs_len);
let use_async_pool = seqs_len > 1;
let sampling_futures: Vec<_> = std::iter::zip(logits_seq, seqs.iter_mut())
.map(|(logits_per_seq, seq)| {
let return_logprobs = seq.return_logprobs();
sample_sequence(
logits_per_seq,
seq,
return_logprobs,
rng.clone(),
use_async_pool,
false,
use_async_pool,
)
})
.collect();
let mut sampled_vec = futures::future::join_all(sampling_futures).await;
// Ring tensor-parallel: replace every rank's independently-sampled token with the head's, so a
// near-tie argmax can't silently desync the lockstep KV caches. No-op unless ring TP is active.
if hanzo_quant::distributed::use_ring() && !crate::pipeline_parallel::use_pipeline_parallel() {
let mut token_ids: Vec<u32> = sampled_vec
.iter()
.map(|s| s.as_ref().map(|lp| lp.token).unwrap_or_default())
.collect();
hanzo_quant::distributed::broadcast_head_tokens(&mut token_ids)?;
for (sampled, id) in sampled_vec.iter_mut().zip(token_ids) {
if let Ok(lp) = sampled {
lp.token = id;
}
}
}
for (sampled, seq) in std::iter::zip(sampled_vec, seqs.iter_mut()) {
let next_token = crate::handle_seq_error_stateaware_ok!(sampled, seq);
let metadata = this.get_metadata();
let eos_tok = if disable_eos_stop {
None
} else {
Some(&metadata.eos_tok[..])
};
finish_or_add_toks_to_seq(this, prefix_cacher, seq, next_token, eos_tok, true).await?;
}
Ok(())
}
/// Async sample optionally adding to trie. `rng` is the engine's shared RNG; a seeded sequence
/// draws from its own instead.
#[allow(clippy::too_many_arguments)]
pub async fn sample_sequence(
logits: Tensor,
seq: &mut Sequence,
return_logprobs: bool,
rng: Arc<std::sync::Mutex<Isaac64Rng>>,
use_async_pool: bool,
sample_speculative: bool,
multiple_sequences: bool,
) -> Result<Logprobs> {
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
// The engine is closing the think block: its token, not the model's draw.
if let Some(token) = seq.forced_token() {
return seq.sampler().forced(&logits, token, return_logprobs);
}
let rng = seq.rng(&rng);
let sampler = seq.sampler();
let ctx_clone = seq.get_toks().to_vec();
let rng_clone = rng.clone();
let logits_clone = logits.clone();
let first_lobprobs_response = if use_async_pool {
tokio_rayon::spawn(move || {
sampler.sample(
logits_clone,
&ctx_clone,
return_logprobs,
rng_clone,
sample_speculative,
multiple_sequences,
)
})
.await?
} else {
sampler.sample(
logits_clone,
&ctx_clone,
return_logprobs,
rng_clone,
sample_speculative,
multiple_sequences,
)?
};
let bias_if_not_allowed = match &mut seq.recognizer {
SequenceRecognizer::Llguidance(ref mut llg) => {
if !llg.is_stopped()
&& llg
.validate_tokens(&[first_lobprobs_response.token])
.unwrap_or(0)
== 1
{
None
} else {
let mask = llg.compute_mask_or_eos().map_err(hanzo_ml::Error::msg)?;
if mask.is_allowed(first_lobprobs_response.token) {
// shouldn't really happen, except for EOS
None
} else {
let mut acc = vec![-f32::INFINITY; logits.shape().dims1().unwrap()];
mask.iter_set_entries(|idx| {
if idx < acc.len() {
acc[idx] = 0.0;
}
});
Some(acc)
}
}
}
SequenceRecognizer::None => None,
};
let second_logprobs_response = match bias_if_not_allowed {
Some(acc) => {
let new_logits = (&logits + Tensor::from_slice(&acc, acc.len(), logits.device())?)?;
let ctx_clone = seq.get_toks().to_vec();
let rng_clone = rng.clone();
let sampler = seq.sampler();
if use_async_pool {
tokio_rayon::spawn(move || {
sampler.sample(
new_logits,
&ctx_clone,
return_logprobs,
rng_clone,
sample_speculative,
multiple_sequences,
)
})
.await?
} else {
sampler.sample(
new_logits,
&ctx_clone,
return_logprobs,
rng_clone,
sample_speculative,
multiple_sequences,
)?
}
}
None => first_lobprobs_response,
};
match seq.recognizer {
SequenceRecognizer::Llguidance(ref mut llg) => {
if !llg.is_stopped() {
llg.consume_token(second_logprobs_response.token)
.map_err(hanzo_ml::Error::msg)?;
}
}
SequenceRecognizer::None => {}
}
// Deactivate mid-stream tool grammar when the grammar has completed
// (i.e. the full tool call body and closing delimiter have been
// generated). This allows re-activation for subsequent tool calls
// in multi-tool-call turns.
if let SequenceRecognizer::Llguidance(ref llg) = seq.recognizer {
if llg.is_stopped() && seq.is_tool_grammar_active() {
seq.recognizer = SequenceRecognizer::None;
seq.set_tool_grammar_active(false);
tracing::debug!("Deactivated tool call grammar (body complete)");
}
}
Ok(second_logprobs_response)
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use hanzo_llm_mcp::{Function, Tool, ToolType};
use super::*;
use crate::tools::{ToolCallingMatcher, ToolChoice};
fn weather_tool() -> Tool {
Tool {
tp: ToolType::Function,
function: Function {
description: Some("Get the current weather for a city.".to_string()),
name: "get_weather".to_string(),
parameters: None,
strict: None,
},
}
}
fn shared_rng(seed: u64) -> Arc<std::sync::Mutex<Isaac64Rng>> {
use rand::SeedableRng;
Arc::new(std::sync::Mutex::new(Isaac64Rng::seed_from_u64(seed)))
}
async fn draw(seq: &mut Sequence, rng: &Arc<std::sync::Mutex<Isaac64Rng>>) -> u32 {
let logits =
Tensor::new(&[[[0.5f32, 1.0, 0.2, 0.9, 0.7]]], &hanzo_ml::Device::Cpu).unwrap();
sample_sequence(logits, seq, false, rng.clone(), false, false, false)
.await
.unwrap()
.token
}
#[tokio::test]
async fn a_seeded_request_replays_whatever_the_shared_rng_did() {
let run = |seed: Option<u64>, shared: u64| async move {
let mut seq = crate::sequence::test_sequence(vec![1, 2], Some(1.0));
if let Some(seed) = seed {
seq.set_seed(seed);
}
let rng = shared_rng(shared);
let mut tokens = Vec::new();
for _ in 0..24 {
tokens.push(draw(&mut seq, &rng).await);
}
tokens
};
assert_eq!(run(Some(3), 0).await, run(Some(3), 99).await);
assert_ne!(run(Some(3), 0).await, run(Some(4), 0).await);
assert_ne!(run(None, 0).await, run(None, 99).await);
}
#[tokio::test]
async fn a_think_close_in_progress_is_written_instead_of_sampled() {
let mut seq = crate::sequence::test_sequence(vec![1, 2], None);
seq.enable_reasoning(
crate::reasoning_parsers::ReasoningMode::TagBased,
Box::new(crate::reasoning_parsers::TagReasoningContext::new_in_think_block()),
);
seq.set_thinking_budget(0, vec![4, 2]);
// Greedy would pick 1; the close's first token comes out instead.
assert_eq!(draw(&mut seq, &shared_rng(0)).await, 4);
}
#[test]
fn gemma4_tool_call_suppresses_raw_content_without_suffix() {
let tool = weather_tool();
let matcher = Arc::new(ToolCallingMatcher::new(ToolChoice::Auto, Some(&[tool])).unwrap());
let raw = r#"<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}"#;
let (content, tool_calls) = parse_text_and_tool_calls(raw, Some(matcher)).unwrap();
assert_eq!(content, None);
assert_eq!(tool_calls.len(), 1);
assert_eq!(tool_calls[0].function.name, "get_weather");
assert_eq!(tool_calls[0].function.arguments, r#"{"city":"Paris"}"#);
}
#[test]
fn reasoning_stream_does_not_fallback_to_raw_delta() {
let tool = weather_tool();
let matcher = Arc::new(ToolCallingMatcher::new(ToolChoice::Auto, Some(&[tool])).unwrap());
let raw = r#"<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}"#;
let (content, tool_calls) =
parse_streaming_text_and_tool_calls(None, raw, true, Some(matcher)).unwrap();
assert_eq!(content, None);
assert!(tool_calls.is_empty());
}
#[test]
fn a_streamed_call_is_not_content_and_calls_in_the_think_block_are_not_calls() {
let tool = weather_tool();
let matcher = Arc::new(
ToolCallingMatcher::new(ToolChoice::Auto, Some(&[tool]))
.unwrap()
.with_xml_calls(true),
);
let call = "<tool_call>\n<function=get_weather>\n<parameter=city>\nOslo\n</parameter>\n</function>\n</tool_call>";
let content = format!("Let me check.\n{call}");
let raw = format!("I could use <tool_call> here.</think>{content}");
let (text, calls) =
parse_streaming_text_and_tool_calls(Some(content), &raw, true, Some(matcher.clone()))
.unwrap();
assert_eq!(text.as_deref(), Some("Let me check."));
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].function.arguments, r#"{"city":"Oslo"}"#);
// A call written in the reasoning, with no answer yet, is not a call.
let raw = format!("thinking {call}");
let (text, calls) =
parse_streaming_text_and_tool_calls(None, &raw, true, Some(matcher)).unwrap();
assert_eq!(text, None);
assert!(calls.is_empty());
}
#[test]
fn non_reasoning_stream_uses_raw_delta() {
let tool = weather_tool();
let matcher = Arc::new(ToolCallingMatcher::new(ToolChoice::Auto, Some(&[tool])).unwrap());
let raw = r#"<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}"#;
let (content, tool_calls) =
parse_streaming_text_and_tool_calls(None, raw, false, Some(matcher)).unwrap();
assert_eq!(content, None);
assert_eq!(tool_calls.len(), 1);
assert_eq!(tool_calls[0].function.name, "get_weather");
}
#[test]
fn record_group_finish_does_not_reenter_sequence_group_mutex() {
use tokio::sync::mpsc::channel;
use tokio::sync::Mutex;
use crate::sampler::Sampler;
use crate::sequence::{ForwardStepTelemetry, SeqStepType, SequenceGroup};
use crate::speculative::{SpeculativeAttachInfo, SpeculativeAttachKind};
let make_seq = |is_streaming: bool| {
let (tx, _rx) = channel(1);
let sampler =
Sampler::new(None, 0, None, None, None, None, None, 32, 1.0, 0.0, vec![]).unwrap();
let group = Arc::new(Mutex::new(SequenceGroup::new(1, is_streaming, false, None)));
let seq = Sequence::new_waiting(
vec![1, 2, 3, 4],
"prompt".to_string(),
0,
0,
1,
tx,
sampler,
vec![],
vec![],
None,
false,
false,
group,
0,
0,
SequenceRecognizer::None,
None,
None,
None,
None,
None,
None,
None,
None,
SeqStepType::PromptAndDecode,
None,
None,
None,
false,
vec![],
);
seq.record_drafts(7, 2);
seq.record_draft_block_size(8);
seq.begin_speculative_round();
for _ in 0..3 {
seq.record_speculative_committed_token();
}
seq.record_speculative_stand_down("slower_than_plain_probe");
seq.record_plain_decode_step();
seq.record_forward_step_telemetry(
ForwardStepTelemetry {
cuda_decode_graph_capture: true,
cuda_decode_graph_replay: false,
cuda_prefill_graph_capture: false,
cuda_prefill_graph_replay: false,
},
false,
);
seq
};
let drafter = Some(SpeculativeAttachInfo {
kind: SpeculativeAttachKind::Dflash { block_size: 8 },
});
let before = crate::engine::ledger::totals();
let non_stream = make_seq(false);
let usage_non_stream = record_group_finish(drafter.clone(), &non_stream, false);
let trace_str = crate::engine::ledger::trace_line(
&usage_non_stream,
Some("dflash"),
&non_stream.execution_trace(),
);
assert!(trace_str.contains("mode=dflash->dense"));
assert!(trace_str.contains("spec_rounds=1 spec_committed=3 plain_steps=1"));
assert!(trace_str.contains("cuda_decode_graph=capture:1,replay:0"));
assert!(trace_str.contains("stand_down=slower_than_plain_probe"));
let stream = make_seq(true);
let _usage_stream = record_group_finish(drafter, &stream, true);
let after = crate::engine::ledger::totals();
assert!(after.requests >= before.requests + 2);
// Mutex remains unlocked after returning.
assert_eq!(stream.get_mut_group().total_toks, 0);
}
}