mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-21 02:09:47 +00:00
1019 lines
41 KiB
Rust
1019 lines
41 KiB
Rust
use std::sync::Arc;
|
|
|
|
use hanzo_ml::{DType, Result, Tensor};
|
|
use rand_isaac::Isaac64Rng;
|
|
|
|
use crate::{
|
|
prefix_cacher::PrefixCacheManagerV2,
|
|
sampler::Logprobs,
|
|
sequence::{Sequence, SequenceRecognizer, SequenceState, StopReason},
|
|
tools::{parse_text_tools, ToolCallResponse, ToolCallType},
|
|
};
|
|
use hanzo_llm_mcp::CalledFunction;
|
|
|
|
use super::Pipeline;
|
|
|
|
macro_rules! fixup_sentencepiece {
|
|
($txt:expr) => {
|
|
$txt.to_string().replace("▁", " ")
|
|
};
|
|
(Option $txt:expr) => {
|
|
match &$txt {
|
|
Some(txt) => Some(fixup_sentencepiece!(txt)),
|
|
None => None,
|
|
}
|
|
};
|
|
}
|
|
|
|
fn parse_text_and_tool_calls(
|
|
raw_text: &str,
|
|
matcher: Option<Arc<crate::tools::ToolCallingMatcher>>,
|
|
) -> Result<(Option<String>, Vec<ToolCallResponse>)> {
|
|
parse_text_tools(raw_text, matcher).map_err(hanzo_ml::Error::msg)
|
|
}
|
|
|
|
/// One streamed delta's content and tool calls. Calls are read from the answer only: with a
|
|
/// reasoning parser, from its content delta, never the raw text, which holds the think block too.
|
|
fn parse_streaming_text_and_tool_calls(
|
|
content_delta: Option<String>,
|
|
raw_delta: &str,
|
|
has_reasoning_parser: bool,
|
|
matcher: Option<Arc<crate::tools::ToolCallingMatcher>>,
|
|
) -> Result<(Option<String>, Vec<ToolCallResponse>)> {
|
|
let raw_text = match content_delta {
|
|
Some(content_delta) => content_delta,
|
|
None if has_reasoning_parser => return Ok((None, Vec::new())),
|
|
None => raw_delta.to_string(),
|
|
};
|
|
parse_text_and_tool_calls(raw_text.as_str(), matcher)
|
|
}
|
|
|
|
fn status_for_stop_reason(
|
|
reason: crate::sequence::StopReason,
|
|
) -> crate::engine::ledger::TerminalStatus {
|
|
if matches!(reason, crate::sequence::StopReason::Canceled) {
|
|
crate::engine::ledger::TerminalStatus::Canceled
|
|
} else {
|
|
crate::engine::ledger::TerminalStatus::Completed
|
|
}
|
|
}
|
|
|
|
/// Reads usage + execution trace under one `SequenceGroup` lock, releases the lock before
|
|
/// entering the ledger, and records the terminal request event once (`serve_api:` + `serve_trace:`).
|
|
#[cfg(test)]
|
|
fn record_group_finish(
|
|
drafter: Option<crate::speculative::SpeculativeAttachInfo>,
|
|
seq: &Sequence,
|
|
clear_group_totals: bool,
|
|
) -> crate::Usage {
|
|
record_group_finish_with_status(
|
|
drafter,
|
|
seq,
|
|
clear_group_totals,
|
|
crate::engine::ledger::TerminalStatus::Completed,
|
|
)
|
|
}
|
|
|
|
fn record_group_finish_with_status(
|
|
drafter: Option<crate::speculative::SpeculativeAttachInfo>,
|
|
seq: &Sequence,
|
|
clear_group_totals: bool,
|
|
status: crate::engine::ledger::TerminalStatus,
|
|
) -> crate::Usage {
|
|
let (should_log, usage, trace) = {
|
|
let mut group = seq.get_mut_group();
|
|
let should_log = !std::mem::replace(&mut group.trace_logged, true);
|
|
let usage = group.get_usage();
|
|
if clear_group_totals {
|
|
group.total_prompt_toks = 0;
|
|
group.total_toks = 0;
|
|
}
|
|
(should_log, usage, group.execution_trace.clone())
|
|
};
|
|
if should_log {
|
|
let structured = matches!(seq.recognizer, SequenceRecognizer::Llguidance(_))
|
|
&& !seq.is_tool_grammar_active();
|
|
crate::engine::ledger::record_with_trace(
|
|
&usage,
|
|
drafter.as_ref().map(|d| d.name()),
|
|
structured,
|
|
&trace,
|
|
*seq.id(),
|
|
status,
|
|
);
|
|
}
|
|
usage
|
|
}
|
|
|
|
pub(crate) async fn finish_or_add_toks_to_seq(
|
|
this: &dyn Pipeline,
|
|
prefix_cacher: &mut PrefixCacheManagerV2,
|
|
seq: &mut Sequence,
|
|
logprobs: Logprobs,
|
|
eos_tok: Option<&[u32]>,
|
|
use_prefix_cacher: bool,
|
|
) -> Result<()> {
|
|
// Record prefill wall-time once, at the first sampled token and before any
|
|
// detokenization or response construction below. `step_start_instant` is set
|
|
// just before the prompt forward; sampling has already forced that forward's
|
|
// device sync, so this elapsed span is the true prompt-forward -> first-token
|
|
// latency -- identical for single-token (max_len=1 / prompt benchmarks, which
|
|
// finish inside the prompt step) and multi-token requests, on every backend.
|
|
// Recording it here (not after step() returns) keeps post-forward overhead
|
|
// out of the number so avg_prompt_tok_per_sec reflects real prefill throughput.
|
|
if seq.total_prompt_time.is_none() {
|
|
if let Some(start) = seq.step_start_instant {
|
|
seq.total_prompt_time = Some(start.elapsed().as_millis());
|
|
}
|
|
}
|
|
|
|
let mut is_done = seq.is_done(logprobs.token, eos_tok, this.get_metadata().max_seq_len);
|
|
let metadata = this.get_metadata();
|
|
let tok_env = metadata.tok_env().ok_or(hanzo_ml::Error::Msg(
|
|
"`finish_or_add_toks_to_seq` requires the pipeline to have a token trie".to_string(),
|
|
))?;
|
|
// Include special tokens when tool calling is active (so tool parsers can see
|
|
// delimiters like <tool_call>, [TOOL_CALLS], <|python_tag|>) or when think tag
|
|
// mode is enabled (so <think>/<\/think> delimiters are visible in the output).
|
|
let include_special = seq.tools.is_some() || seq.needs_special_tokens();
|
|
let completion_bytes = tok_env
|
|
.tok_trie()
|
|
.decode_ext(&[logprobs.token], include_special);
|
|
if let Some(stop) = seq.add_token(logprobs.clone(), completion_bytes, &is_done) {
|
|
is_done = Some(stop);
|
|
}
|
|
|
|
// If we can have a tool and we got a tool, stop the sequence early.
|
|
// Doesn't conflict with the logic below because it does the same thing anyway.
|
|
if let Some(ref t) = seq.tools {
|
|
if let Ok(Some(ref d)) = seq.peek_delta() {
|
|
let (_tool_use_still_possible, tool_use_is_done) =
|
|
t.prefix_could_be_tool(d.as_str())?;
|
|
|
|
if tool_use_is_done
|
|
&& parse_text_tools(d, seq.tools.clone()).is_ok_and(|(_, calls)| !calls.is_empty())
|
|
{
|
|
seq.set_state(SequenceState::Done(StopReason::Eos));
|
|
seq.release_held();
|
|
is_done = Some(StopReason::Eos);
|
|
}
|
|
}
|
|
};
|
|
|
|
// Mid-stream grammar activation for tool calls.
|
|
// When a tool call prefix is detected and no grammar is already active,
|
|
// build a format-specific grammar and activate it so subsequent tokens
|
|
// are constrained to valid tool call syntax.
|
|
// For Harmony mode, tool calls are signaled by token-level markers (not
|
|
// text prefixes), so we also check HarmonyContext for new tool calls.
|
|
//
|
|
// Skip when the sequence is already done — peek_delta() still contains
|
|
// the tool call prefix from earlier generation, which would spuriously
|
|
// re-activate the grammar on a completed sequence.
|
|
if matches!(seq.recognizer, SequenceRecognizer::None) && is_done.is_none() {
|
|
// Try text-based detection first, then Harmony token-based detection.
|
|
let grm = if let Some(ref t) = seq.tools {
|
|
seq.peek_delta()
|
|
.ok()
|
|
.flatten()
|
|
.and_then(|text| t.build_tool_call_grammar(text.as_str()))
|
|
} else {
|
|
None
|
|
};
|
|
let grm = grm.or_else(|| {
|
|
if seq.needs_harmony_tool_grammar() {
|
|
let recipient = seq.harmony_current_tool_recipient();
|
|
seq.tools
|
|
.as_ref()?
|
|
.build_harmony_tool_grammar(recipient.as_deref())
|
|
} else {
|
|
None
|
|
}
|
|
});
|
|
|
|
if let Some(grm) = grm {
|
|
if let Some(ref factory) = metadata.llg_factory {
|
|
match crate::pipeline::llg::constraint_from_llg_grammar(factory, grm) {
|
|
Ok(matcher) => {
|
|
tracing::debug!("Activated tool call grammar");
|
|
seq.recognizer = SequenceRecognizer::Llguidance(Box::new(matcher));
|
|
seq.set_tool_grammar_active(true);
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!(
|
|
"Failed to build tool call grammar: {e}. \
|
|
Continuing without constraint."
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Handle streaming requests
|
|
if seq.get_mut_group().is_streaming {
|
|
let mut tool_use_still_possible = false;
|
|
let mut tool_use_is_done = false;
|
|
if let Some(ref t) = seq.tools {
|
|
if let Ok(Some(ref d)) = seq.peek_delta() {
|
|
(tool_use_still_possible, tool_use_is_done) = t.prefix_could_be_tool(d.as_str())?;
|
|
}
|
|
};
|
|
|
|
// Send chunks when:
|
|
// 1. Tool call is not possible (!tool_use_still_possible) - normal streaming
|
|
// 2. Tool call is complete (tool_use_is_done) - send the tool call
|
|
// 3. Sequence is done (is_done.is_some()) - send buffered output as text since it wasn't a valid tool call
|
|
if !tool_use_still_possible || tool_use_is_done || is_done.is_some() {
|
|
if is_done.is_some() && seq.reasoning_mode().is_some() {
|
|
seq.finalize_reasoning();
|
|
}
|
|
let delta_result = seq.get_delta();
|
|
if let Some(delta) = crate::handle_seq_error_stateaware_ok!(delta_result, seq) {
|
|
if seq.get_mut_group().is_chat {
|
|
let (content_delta, reasoning_delta, tool_calls) = if seq.is_harmony_mode() {
|
|
let content_delta = seq.get_response_content_delta();
|
|
let reasoning_delta = seq.get_reasoning_content_delta();
|
|
// In Harmony mode, only finalize tool calls when the sequence is done
|
|
// (EOS token or stop string), not when we first detect a tool call.
|
|
// This ensures tool call arguments are fully generated.
|
|
let tool_calls = if is_done.is_some() && seq.has_harmony_tool_calls() {
|
|
// Sequence is done and has tool calls - finalize and send them
|
|
is_done = Some(StopReason::ToolCalls);
|
|
let harmony_tool_calls = seq.get_harmony_tool_calls();
|
|
harmony_tool_calls
|
|
.into_iter()
|
|
.enumerate()
|
|
.map(|(i, tc)| ToolCallResponse {
|
|
index: i,
|
|
id: tc.id,
|
|
tp: ToolCallType::Function,
|
|
function: CalledFunction {
|
|
name: tc.name,
|
|
arguments: tc.arguments,
|
|
},
|
|
})
|
|
.collect()
|
|
} else {
|
|
vec![]
|
|
};
|
|
(content_delta, reasoning_delta, tool_calls)
|
|
} else {
|
|
// A call's own text is not content.
|
|
let (content_delta, tool_calls) = parse_streaming_text_and_tool_calls(
|
|
seq.get_response_content_delta(),
|
|
delta.as_str(),
|
|
seq.reasoning_mode().is_some(),
|
|
seq.tools.clone(),
|
|
)?;
|
|
if !tool_calls.is_empty() {
|
|
is_done = Some(StopReason::ToolCalls);
|
|
}
|
|
(content_delta, seq.get_reasoning_content_delta(), tool_calls)
|
|
};
|
|
|
|
seq.add_streaming_chunk_choice_to_group(crate::ChunkChoice {
|
|
delta: crate::Delta {
|
|
content: fixup_sentencepiece!(Option content_delta),
|
|
role: "assistant".to_string(),
|
|
tool_calls: Some(tool_calls).filter(|v| !v.is_empty()),
|
|
reasoning_content: reasoning_delta,
|
|
},
|
|
index: seq.get_response_index(),
|
|
finish_reason: is_done.map(|x| x.to_string()),
|
|
logprobs: if seq.return_logprobs() {
|
|
Some(crate::ResponseLogprob {
|
|
token: delta,
|
|
bytes: logprobs.bytes.clone().map(|b| b.into_bytes()),
|
|
logprob: logprobs.logprob,
|
|
top_logprobs: logprobs.top_logprobs.unwrap().clone(),
|
|
})
|
|
} else {
|
|
None
|
|
},
|
|
});
|
|
} else {
|
|
seq.add_streaming_completion_chunk_choice_to_group(
|
|
crate::CompletionChunkChoice {
|
|
text: fixup_sentencepiece!(delta),
|
|
index: seq.get_response_index(),
|
|
finish_reason: is_done.map(|x| x.to_string()),
|
|
logprobs: if seq.return_logprobs() {
|
|
Some(crate::CompletionLogprobs::new(
|
|
&[crate::ResponseLogprob {
|
|
token: delta,
|
|
bytes: logprobs.bytes.clone().map(|b| b.into_bytes()),
|
|
logprob: logprobs.logprob,
|
|
top_logprobs: logprobs.top_logprobs.unwrap().clone(),
|
|
}],
|
|
0,
|
|
))
|
|
} else {
|
|
None
|
|
},
|
|
},
|
|
);
|
|
}
|
|
}
|
|
|
|
// Send usage on final chunk, recording terminal status after delivery check.
|
|
let final_snapshot = if let Some(reason) = is_done {
|
|
seq.update_time_info();
|
|
let (should_log, usage, trace) = {
|
|
let mut group = seq.get_mut_group();
|
|
let should_log = !std::mem::replace(&mut group.trace_logged, true);
|
|
let usage = group.get_usage();
|
|
group.total_prompt_toks = 0;
|
|
group.total_toks = 0;
|
|
(should_log, usage, group.execution_trace.clone())
|
|
};
|
|
Some((reason, should_log, usage, trace))
|
|
} else {
|
|
None
|
|
};
|
|
let usage_opt = final_snapshot.as_ref().map(|(_, _, u, _)| u.clone());
|
|
|
|
let send_failed = seq
|
|
.get_mut_group()
|
|
.maybe_send_streaming_response(seq, this.name().clone(), usage_opt)
|
|
.await
|
|
.is_err();
|
|
|
|
if let Some((reason, should_log, usage, trace)) = final_snapshot {
|
|
if should_log {
|
|
let status = if send_failed {
|
|
crate::engine::ledger::TerminalStatus::Canceled
|
|
} else {
|
|
status_for_stop_reason(reason)
|
|
};
|
|
let structured = matches!(seq.recognizer, SequenceRecognizer::Llguidance(_))
|
|
&& !seq.is_tool_grammar_active();
|
|
crate::engine::ledger::record_with_trace(
|
|
&usage,
|
|
this.drafter().as_ref().map(|d| d.name()),
|
|
structured,
|
|
&trace,
|
|
*seq.id(),
|
|
status,
|
|
);
|
|
}
|
|
}
|
|
|
|
if send_failed {
|
|
if is_done.is_none() {
|
|
seq.update_time_info();
|
|
record_group_finish_with_status(
|
|
this.drafter(),
|
|
seq,
|
|
true,
|
|
crate::engine::ledger::TerminalStatus::Canceled,
|
|
);
|
|
}
|
|
// If we can't send the response, cancel the sequence
|
|
seq.set_state(crate::sequence::SequenceState::Done(
|
|
crate::sequence::StopReason::Canceled,
|
|
));
|
|
this.reset_non_granular_state();
|
|
}
|
|
}
|
|
|
|
// Handle Done state regardless of tool detection - must be outside the tool_use check
|
|
// to ensure sequence completes even when tool detection thinks output might be a tool call
|
|
if let Some(reason) = is_done {
|
|
if use_prefix_cacher {
|
|
let recurrent_snapshots = if this.cache().is_hybrid() {
|
|
seq.recurrent_state_idx()
|
|
.and_then(|idx| this.cache().hybrid().snapshot_recurrent_state(idx).ok())
|
|
} else {
|
|
None
|
|
};
|
|
prefix_cacher.add_sequence(seq, recurrent_snapshots);
|
|
prefix_cacher.evict_caches()?;
|
|
}
|
|
seq.set_state(crate::sequence::SequenceState::Done(reason));
|
|
this.reset_non_granular_state();
|
|
}
|
|
} else if let Some(mut reason) = is_done {
|
|
/*
|
|
***********************
|
|
Finish the sequence now
|
|
***********************
|
|
*/
|
|
{
|
|
seq.set_state(crate::sequence::SequenceState::Done(reason));
|
|
let (tokenizer, pipeline_name) = {
|
|
let pipeline_name = this.name();
|
|
let tokenizer = this.tokenizer();
|
|
(tokenizer, pipeline_name)
|
|
};
|
|
|
|
let logprobs = if seq.return_logprobs() {
|
|
let mut logprobs = Vec::new();
|
|
for logprob in seq.logprobs() {
|
|
let resp_logprob = crate::ResponseLogprob {
|
|
token: crate::handle_seq_error_stateaware_ok!(
|
|
tokenizer
|
|
.as_ref()
|
|
.ok_or(hanzo_ml::Error::Msg(
|
|
"`finish_or_add_toks_to_seq` requires the pipeline to have a tokenizer"
|
|
.to_string(),
|
|
))?.decode(&[logprob.token], false),
|
|
seq
|
|
),
|
|
bytes: logprob.bytes.clone().map(|b| b.into_bytes()),
|
|
logprob: logprob.logprob,
|
|
top_logprobs: logprob.top_logprobs.clone().unwrap(),
|
|
};
|
|
logprobs.push(resp_logprob);
|
|
}
|
|
Some(logprobs)
|
|
} else {
|
|
None
|
|
};
|
|
|
|
// Signal EOS to all reasoning parsers
|
|
seq.finalize_reasoning();
|
|
|
|
// A stop string never enters the completion, so its text is the whole completion.
|
|
let text = match reason {
|
|
crate::sequence::StopReason::Length(_)
|
|
| crate::sequence::StopReason::ModelLength(_)
|
|
| crate::sequence::StopReason::Eos
|
|
| crate::sequence::StopReason::StopTok(_)
|
|
| crate::sequence::StopReason::Canceled
|
|
| crate::sequence::StopReason::ToolCalls
|
|
| crate::sequence::StopReason::StopString { .. } => {
|
|
String::from_utf8_lossy(seq.completion_bytes()).to_string()
|
|
}
|
|
crate::sequence::StopReason::GeneratedImage
|
|
| crate::sequence::StopReason::GeneratedSpeech
|
|
| crate::sequence::StopReason::GeneratedFrames
|
|
| crate::sequence::StopReason::GeneratedTranscription => {
|
|
hanzo_ml::bail!("Stop reason was a non-text generation.")
|
|
}
|
|
};
|
|
|
|
// A completion is the exact continuation of its prompt; a chat message drops the
|
|
// whitespace the first token carries.
|
|
let is_chat = seq.get_mut_group().is_chat;
|
|
let text = if is_chat {
|
|
text.trim_start().to_string()
|
|
} else {
|
|
text
|
|
};
|
|
let mut declined = false;
|
|
if is_chat {
|
|
let (text_new, tool_calls, reasoning_content) =
|
|
if let Some(mode) = seq.reasoning_mode() {
|
|
let final_content = seq.get_response_content();
|
|
let reasoning = seq.get_reasoning_content();
|
|
|
|
let (final_content, tool_calls) =
|
|
if mode == crate::reasoning_parsers::ReasoningMode::Harmony {
|
|
let harmony_tool_calls = seq.get_harmony_tool_calls();
|
|
let tool_calls = harmony_tool_calls
|
|
.into_iter()
|
|
.enumerate()
|
|
.map(|(i, tc)| ToolCallResponse {
|
|
index: i,
|
|
id: tc.id,
|
|
tp: ToolCallType::Function,
|
|
function: CalledFunction {
|
|
name: tc.name,
|
|
arguments: tc.arguments,
|
|
},
|
|
})
|
|
.collect();
|
|
(final_content, tool_calls)
|
|
} else if let Some(ref content) = final_content {
|
|
// A call's own text is not content.
|
|
parse_text_and_tool_calls(content.as_str(), seq.tools.clone())?
|
|
} else {
|
|
(None, vec![])
|
|
};
|
|
|
|
(final_content, tool_calls, reasoning)
|
|
} else {
|
|
let (text_new, tool_calls) =
|
|
parse_text_and_tool_calls(text.as_str(), seq.tools.clone())?;
|
|
(text_new, tool_calls, None)
|
|
};
|
|
|
|
if !tool_calls.is_empty() {
|
|
reason = StopReason::ToolCalls;
|
|
}
|
|
// A request that had to call a tool and did not is refused,
|
|
// not answered with text.
|
|
declined = tool_calls.is_empty()
|
|
&& seq.tools.as_ref().is_some_and(|tools| tools.must_call());
|
|
|
|
let choice = crate::Choice {
|
|
finish_reason: fixup_sentencepiece!(reason),
|
|
index: seq.get_response_index(),
|
|
message: crate::ResponseMessage {
|
|
content: text_new,
|
|
role: "assistant".to_string(),
|
|
tool_calls: Some(tool_calls).filter(|v| !v.is_empty()),
|
|
reasoning_content,
|
|
},
|
|
logprobs: logprobs.map(|l| crate::Logprobs { content: Some(l) }),
|
|
};
|
|
seq.add_choice_to_group(choice);
|
|
} else {
|
|
let choice = crate::CompletionChoice {
|
|
finish_reason: fixup_sentencepiece!(reason),
|
|
index: seq.get_response_index(),
|
|
text,
|
|
logprobs: logprobs.map(|l| crate::CompletionLogprobs::new(&l, 0)),
|
|
};
|
|
seq.add_completion_choice_to_group(choice);
|
|
}
|
|
|
|
if use_prefix_cacher {
|
|
let recurrent_snapshots = if this.cache().is_hybrid() {
|
|
seq.recurrent_state_idx()
|
|
.and_then(|idx| this.cache().hybrid().snapshot_recurrent_state(idx).ok())
|
|
} else {
|
|
None
|
|
};
|
|
prefix_cacher.add_sequence(seq, recurrent_snapshots);
|
|
prefix_cacher.evict_caches()?;
|
|
}
|
|
|
|
// Ensure timing info is synced to group before sending response
|
|
seq.update_time_info();
|
|
let non_stream_status = if declined {
|
|
crate::engine::ledger::TerminalStatus::Error
|
|
} else {
|
|
status_for_stop_reason(reason)
|
|
};
|
|
record_group_finish_with_status(this.drafter(), seq, false, non_stream_status);
|
|
|
|
let mut group = seq.get_mut_group();
|
|
if group.is_chat {
|
|
let response = crate::ChatCompletionResponse {
|
|
id: seq.id().to_string(),
|
|
choices: group.get_choices().to_vec(),
|
|
created: seq.creation_time(),
|
|
model: pipeline_name,
|
|
system_fingerprint: crate::SYSTEM_FINGERPRINT.to_string(),
|
|
object: "chat.completion".to_string(),
|
|
usage: group.get_usage(),
|
|
agentic_tool_calls: None,
|
|
files: None,
|
|
session_id: None,
|
|
};
|
|
if declined {
|
|
group
|
|
.send_chat_model_error(
|
|
crate::tools::FORCED_CALL_DECLINED.to_string(),
|
|
response,
|
|
seq.responder(),
|
|
)
|
|
.await
|
|
.map_err(hanzo_ml::Error::msg)?;
|
|
} else {
|
|
group
|
|
.maybe_send_chat_done_response(response, seq.responder())
|
|
.await
|
|
.map_err(hanzo_ml::Error::msg)?;
|
|
}
|
|
} else {
|
|
let completion_response = crate::CompletionResponse {
|
|
id: seq.id().to_string(),
|
|
choices: group.get_completion_choices(),
|
|
created: seq.creation_time(),
|
|
model: pipeline_name,
|
|
system_fingerprint: crate::SYSTEM_FINGERPRINT.to_string(),
|
|
object: "text_completion".to_string(),
|
|
usage: group.get_usage(),
|
|
};
|
|
group
|
|
.maybe_send_completion_done_response(completion_response, seq.responder())
|
|
.await
|
|
.map_err(hanzo_ml::Error::msg)?;
|
|
}
|
|
}
|
|
this.reset_non_granular_state();
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn sample_and_add_toks(
|
|
this: &dyn Pipeline,
|
|
seqs: &mut [&mut Sequence],
|
|
logits_seq: Vec<Tensor>,
|
|
prefix_cacher: &mut PrefixCacheManagerV2,
|
|
disable_eos_stop: bool,
|
|
rng: Arc<std::sync::Mutex<Isaac64Rng>>,
|
|
) -> Result<()> {
|
|
let seqs_len = seqs.len();
|
|
debug_assert_eq!(logits_seq.len(), seqs_len);
|
|
|
|
let use_async_pool = seqs_len > 1;
|
|
|
|
let sampling_futures: Vec<_> = std::iter::zip(logits_seq, seqs.iter_mut())
|
|
.map(|(logits_per_seq, seq)| {
|
|
let return_logprobs = seq.return_logprobs();
|
|
sample_sequence(
|
|
logits_per_seq,
|
|
seq,
|
|
return_logprobs,
|
|
rng.clone(),
|
|
use_async_pool,
|
|
false,
|
|
use_async_pool,
|
|
)
|
|
})
|
|
.collect();
|
|
let mut sampled_vec = futures::future::join_all(sampling_futures).await;
|
|
|
|
// Ring tensor-parallel: replace every rank's independently-sampled token with the head's, so a
|
|
// near-tie argmax can't silently desync the lockstep KV caches. No-op unless ring TP is active.
|
|
if hanzo_quant::distributed::use_ring() && !crate::pipeline_parallel::use_pipeline_parallel() {
|
|
let mut token_ids: Vec<u32> = sampled_vec
|
|
.iter()
|
|
.map(|s| s.as_ref().map(|lp| lp.token).unwrap_or_default())
|
|
.collect();
|
|
hanzo_quant::distributed::broadcast_head_tokens(&mut token_ids)?;
|
|
for (sampled, id) in sampled_vec.iter_mut().zip(token_ids) {
|
|
if let Ok(lp) = sampled {
|
|
lp.token = id;
|
|
}
|
|
}
|
|
}
|
|
|
|
for (sampled, seq) in std::iter::zip(sampled_vec, seqs.iter_mut()) {
|
|
let next_token = crate::handle_seq_error_stateaware_ok!(sampled, seq);
|
|
|
|
let metadata = this.get_metadata();
|
|
let eos_tok = if disable_eos_stop {
|
|
None
|
|
} else {
|
|
Some(&metadata.eos_tok[..])
|
|
};
|
|
|
|
finish_or_add_toks_to_seq(this, prefix_cacher, seq, next_token, eos_tok, true).await?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Async sample optionally adding to trie. `rng` is the engine's shared RNG; a seeded sequence
|
|
/// draws from its own instead.
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub async fn sample_sequence(
|
|
logits: Tensor,
|
|
seq: &mut Sequence,
|
|
return_logprobs: bool,
|
|
rng: Arc<std::sync::Mutex<Isaac64Rng>>,
|
|
use_async_pool: bool,
|
|
sample_speculative: bool,
|
|
multiple_sequences: bool,
|
|
) -> Result<Logprobs> {
|
|
let logits = logits.squeeze(0)?.squeeze(0)?.to_dtype(DType::F32)?;
|
|
|
|
// The engine is closing the think block: its token, not the model's draw.
|
|
if let Some(token) = seq.forced_token() {
|
|
return seq.sampler().forced(&logits, token, return_logprobs);
|
|
}
|
|
|
|
let rng = seq.rng(&rng);
|
|
let sampler = seq.sampler();
|
|
let ctx_clone = seq.get_toks().to_vec();
|
|
let rng_clone = rng.clone();
|
|
let logits_clone = logits.clone();
|
|
let first_lobprobs_response = if use_async_pool {
|
|
tokio_rayon::spawn(move || {
|
|
sampler.sample(
|
|
logits_clone,
|
|
&ctx_clone,
|
|
return_logprobs,
|
|
rng_clone,
|
|
sample_speculative,
|
|
multiple_sequences,
|
|
)
|
|
})
|
|
.await?
|
|
} else {
|
|
sampler.sample(
|
|
logits_clone,
|
|
&ctx_clone,
|
|
return_logprobs,
|
|
rng_clone,
|
|
sample_speculative,
|
|
multiple_sequences,
|
|
)?
|
|
};
|
|
|
|
let bias_if_not_allowed = match &mut seq.recognizer {
|
|
SequenceRecognizer::Llguidance(ref mut llg) => {
|
|
if !llg.is_stopped()
|
|
&& llg
|
|
.validate_tokens(&[first_lobprobs_response.token])
|
|
.unwrap_or(0)
|
|
== 1
|
|
{
|
|
None
|
|
} else {
|
|
let mask = llg.compute_mask_or_eos().map_err(hanzo_ml::Error::msg)?;
|
|
if mask.is_allowed(first_lobprobs_response.token) {
|
|
// shouldn't really happen, except for EOS
|
|
None
|
|
} else {
|
|
let mut acc = vec![-f32::INFINITY; logits.shape().dims1().unwrap()];
|
|
mask.iter_set_entries(|idx| {
|
|
if idx < acc.len() {
|
|
acc[idx] = 0.0;
|
|
}
|
|
});
|
|
|
|
Some(acc)
|
|
}
|
|
}
|
|
}
|
|
SequenceRecognizer::None => None,
|
|
};
|
|
let second_logprobs_response = match bias_if_not_allowed {
|
|
Some(acc) => {
|
|
let new_logits = (&logits + Tensor::from_slice(&acc, acc.len(), logits.device())?)?;
|
|
|
|
let ctx_clone = seq.get_toks().to_vec();
|
|
let rng_clone = rng.clone();
|
|
let sampler = seq.sampler();
|
|
if use_async_pool {
|
|
tokio_rayon::spawn(move || {
|
|
sampler.sample(
|
|
new_logits,
|
|
&ctx_clone,
|
|
return_logprobs,
|
|
rng_clone,
|
|
sample_speculative,
|
|
multiple_sequences,
|
|
)
|
|
})
|
|
.await?
|
|
} else {
|
|
sampler.sample(
|
|
new_logits,
|
|
&ctx_clone,
|
|
return_logprobs,
|
|
rng_clone,
|
|
sample_speculative,
|
|
multiple_sequences,
|
|
)?
|
|
}
|
|
}
|
|
None => first_lobprobs_response,
|
|
};
|
|
|
|
match seq.recognizer {
|
|
SequenceRecognizer::Llguidance(ref mut llg) => {
|
|
if !llg.is_stopped() {
|
|
llg.consume_token(second_logprobs_response.token)
|
|
.map_err(hanzo_ml::Error::msg)?;
|
|
}
|
|
}
|
|
SequenceRecognizer::None => {}
|
|
}
|
|
|
|
// Deactivate mid-stream tool grammar when the grammar has completed
|
|
// (i.e. the full tool call body and closing delimiter have been
|
|
// generated). This allows re-activation for subsequent tool calls
|
|
// in multi-tool-call turns.
|
|
if let SequenceRecognizer::Llguidance(ref llg) = seq.recognizer {
|
|
if llg.is_stopped() && seq.is_tool_grammar_active() {
|
|
seq.recognizer = SequenceRecognizer::None;
|
|
seq.set_tool_grammar_active(false);
|
|
tracing::debug!("Deactivated tool call grammar (body complete)");
|
|
}
|
|
}
|
|
|
|
Ok(second_logprobs_response)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::sync::Arc;
|
|
|
|
use hanzo_llm_mcp::{Function, Tool, ToolType};
|
|
|
|
use super::*;
|
|
use crate::tools::{ToolCallingMatcher, ToolChoice};
|
|
|
|
fn weather_tool() -> Tool {
|
|
Tool {
|
|
tp: ToolType::Function,
|
|
function: Function {
|
|
description: Some("Get the current weather for a city.".to_string()),
|
|
name: "get_weather".to_string(),
|
|
parameters: None,
|
|
strict: None,
|
|
},
|
|
}
|
|
}
|
|
|
|
fn shared_rng(seed: u64) -> Arc<std::sync::Mutex<Isaac64Rng>> {
|
|
use rand::SeedableRng;
|
|
Arc::new(std::sync::Mutex::new(Isaac64Rng::seed_from_u64(seed)))
|
|
}
|
|
|
|
async fn draw(seq: &mut Sequence, rng: &Arc<std::sync::Mutex<Isaac64Rng>>) -> u32 {
|
|
let logits =
|
|
Tensor::new(&[[[0.5f32, 1.0, 0.2, 0.9, 0.7]]], &hanzo_ml::Device::Cpu).unwrap();
|
|
sample_sequence(logits, seq, false, rng.clone(), false, false, false)
|
|
.await
|
|
.unwrap()
|
|
.token
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_seeded_request_replays_whatever_the_shared_rng_did() {
|
|
let run = |seed: Option<u64>, shared: u64| async move {
|
|
let mut seq = crate::sequence::test_sequence(vec![1, 2], Some(1.0));
|
|
if let Some(seed) = seed {
|
|
seq.set_seed(seed);
|
|
}
|
|
let rng = shared_rng(shared);
|
|
let mut tokens = Vec::new();
|
|
for _ in 0..24 {
|
|
tokens.push(draw(&mut seq, &rng).await);
|
|
}
|
|
tokens
|
|
};
|
|
assert_eq!(run(Some(3), 0).await, run(Some(3), 99).await);
|
|
assert_ne!(run(Some(3), 0).await, run(Some(4), 0).await);
|
|
assert_ne!(run(None, 0).await, run(None, 99).await);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_think_close_in_progress_is_written_instead_of_sampled() {
|
|
let mut seq = crate::sequence::test_sequence(vec![1, 2], None);
|
|
seq.enable_reasoning(
|
|
crate::reasoning_parsers::ReasoningMode::TagBased,
|
|
Box::new(crate::reasoning_parsers::TagReasoningContext::new_in_think_block()),
|
|
);
|
|
seq.set_thinking_budget(0, vec![4, 2]);
|
|
// Greedy would pick 1; the close's first token comes out instead.
|
|
assert_eq!(draw(&mut seq, &shared_rng(0)).await, 4);
|
|
}
|
|
|
|
#[test]
|
|
fn gemma4_tool_call_suppresses_raw_content_without_suffix() {
|
|
let tool = weather_tool();
|
|
let matcher = Arc::new(ToolCallingMatcher::new(ToolChoice::Auto, Some(&[tool])).unwrap());
|
|
let raw = r#"<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}"#;
|
|
|
|
let (content, tool_calls) = parse_text_and_tool_calls(raw, Some(matcher)).unwrap();
|
|
|
|
assert_eq!(content, None);
|
|
assert_eq!(tool_calls.len(), 1);
|
|
assert_eq!(tool_calls[0].function.name, "get_weather");
|
|
assert_eq!(tool_calls[0].function.arguments, r#"{"city":"Paris"}"#);
|
|
}
|
|
|
|
#[test]
|
|
fn reasoning_stream_does_not_fallback_to_raw_delta() {
|
|
let tool = weather_tool();
|
|
let matcher = Arc::new(ToolCallingMatcher::new(ToolChoice::Auto, Some(&[tool])).unwrap());
|
|
let raw = r#"<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}"#;
|
|
|
|
let (content, tool_calls) =
|
|
parse_streaming_text_and_tool_calls(None, raw, true, Some(matcher)).unwrap();
|
|
|
|
assert_eq!(content, None);
|
|
assert!(tool_calls.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn a_streamed_call_is_not_content_and_calls_in_the_think_block_are_not_calls() {
|
|
let tool = weather_tool();
|
|
let matcher = Arc::new(
|
|
ToolCallingMatcher::new(ToolChoice::Auto, Some(&[tool]))
|
|
.unwrap()
|
|
.with_xml_calls(true),
|
|
);
|
|
let call = "<tool_call>\n<function=get_weather>\n<parameter=city>\nOslo\n</parameter>\n</function>\n</tool_call>";
|
|
let content = format!("Let me check.\n{call}");
|
|
let raw = format!("I could use <tool_call> here.</think>{content}");
|
|
let (text, calls) =
|
|
parse_streaming_text_and_tool_calls(Some(content), &raw, true, Some(matcher.clone()))
|
|
.unwrap();
|
|
assert_eq!(text.as_deref(), Some("Let me check."));
|
|
assert_eq!(calls.len(), 1);
|
|
assert_eq!(calls[0].function.arguments, r#"{"city":"Oslo"}"#);
|
|
// A call written in the reasoning, with no answer yet, is not a call.
|
|
let raw = format!("thinking {call}");
|
|
let (text, calls) =
|
|
parse_streaming_text_and_tool_calls(None, &raw, true, Some(matcher)).unwrap();
|
|
assert_eq!(text, None);
|
|
assert!(calls.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn non_reasoning_stream_uses_raw_delta() {
|
|
let tool = weather_tool();
|
|
let matcher = Arc::new(ToolCallingMatcher::new(ToolChoice::Auto, Some(&[tool])).unwrap());
|
|
let raw = r#"<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}"#;
|
|
|
|
let (content, tool_calls) =
|
|
parse_streaming_text_and_tool_calls(None, raw, false, Some(matcher)).unwrap();
|
|
|
|
assert_eq!(content, None);
|
|
assert_eq!(tool_calls.len(), 1);
|
|
assert_eq!(tool_calls[0].function.name, "get_weather");
|
|
}
|
|
|
|
#[test]
|
|
fn record_group_finish_does_not_reenter_sequence_group_mutex() {
|
|
use tokio::sync::mpsc::channel;
|
|
use tokio::sync::Mutex;
|
|
|
|
use crate::sampler::Sampler;
|
|
use crate::sequence::{ForwardStepTelemetry, SeqStepType, SequenceGroup};
|
|
use crate::speculative::{SpeculativeAttachInfo, SpeculativeAttachKind};
|
|
|
|
let make_seq = |is_streaming: bool| {
|
|
let (tx, _rx) = channel(1);
|
|
let sampler =
|
|
Sampler::new(None, 0, None, None, None, None, None, 32, 1.0, 0.0, vec![]).unwrap();
|
|
let group = Arc::new(Mutex::new(SequenceGroup::new(1, is_streaming, false, None)));
|
|
let seq = Sequence::new_waiting(
|
|
vec![1, 2, 3, 4],
|
|
"prompt".to_string(),
|
|
0,
|
|
0,
|
|
1,
|
|
tx,
|
|
sampler,
|
|
vec![],
|
|
vec![],
|
|
None,
|
|
false,
|
|
false,
|
|
group,
|
|
0,
|
|
0,
|
|
SequenceRecognizer::None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
SeqStepType::PromptAndDecode,
|
|
None,
|
|
None,
|
|
None,
|
|
false,
|
|
vec![],
|
|
);
|
|
seq.record_drafts(7, 2);
|
|
seq.record_draft_block_size(8);
|
|
seq.begin_speculative_round();
|
|
for _ in 0..3 {
|
|
seq.record_speculative_committed_token();
|
|
}
|
|
seq.record_speculative_stand_down("slower_than_plain_probe");
|
|
seq.record_plain_decode_step();
|
|
seq.record_forward_step_telemetry(
|
|
ForwardStepTelemetry {
|
|
cuda_decode_graph_capture: true,
|
|
cuda_decode_graph_replay: false,
|
|
cuda_prefill_graph_capture: false,
|
|
cuda_prefill_graph_replay: false,
|
|
},
|
|
false,
|
|
);
|
|
seq
|
|
};
|
|
|
|
let drafter = Some(SpeculativeAttachInfo {
|
|
kind: SpeculativeAttachKind::Dflash { block_size: 8 },
|
|
});
|
|
let before = crate::engine::ledger::totals();
|
|
|
|
let non_stream = make_seq(false);
|
|
let usage_non_stream = record_group_finish(drafter.clone(), &non_stream, false);
|
|
let trace_str = crate::engine::ledger::trace_line(
|
|
&usage_non_stream,
|
|
Some("dflash"),
|
|
&non_stream.execution_trace(),
|
|
);
|
|
assert!(trace_str.contains("mode=dflash->dense"));
|
|
assert!(trace_str.contains("spec_rounds=1 spec_committed=3 plain_steps=1"));
|
|
assert!(trace_str.contains("cuda_decode_graph=capture:1,replay:0"));
|
|
assert!(trace_str.contains("stand_down=slower_than_plain_probe"));
|
|
|
|
let stream = make_seq(true);
|
|
let _usage_stream = record_group_finish(drafter, &stream, true);
|
|
let after = crate::engine::ledger::totals();
|
|
assert!(after.requests >= before.requests + 2);
|
|
// Mutex remains unlocked after returning.
|
|
assert_eq!(stream.get_mut_group().total_toks, 0);
|
|
}
|
|
}
|