hanzoai/dev
2
0
Fork 0
mirror of https://github.com/hanzoai/dev.git synced 2026-10-25 03:32:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(codex-rs): commit outstanding changes across core, tui, and browser

Browse Source
This commit is contained in:
James Peter
2025-08-29 00:56:02 +10:00
parent cfbfa6ca42
commit ca102f4564
15 changed files with 3365 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/browser/src/config.rs
+3 -1
View File
@@ -130,7 +130,9 @@ fn default_viewport() -> ViewportConfig {
}
fn default_wait() -> WaitStrategy {
WaitStrategy::Event("networkidle".to_string())
// "load" is more reliable than a fixed 1s "networkidle" sleep in our
// navigation implementation and better matches SPA hydration delays.
WaitStrategy::Event("load".to_string())
}
fn default_segments_max() -> usize {
codex-rs/core/src/client.rs
+25 -9
View File
@@ -451,6 +451,9 @@ struct SseEvent {
response: Option<Value>,
item: Option<Value>,
delta: Option<String>,
// Present on delta events from the Responses API; used to correlate
// streaming chunks with the final OutputItemDone.
item_id: Option<String>,
}
#[derive(Debug, Deserialize)]
@@ -650,33 +653,46 @@ async fn process_sse<S>(
}
"response.output_text.delta" => {
if let Some(delta) = event.delta {
let event = ResponseEvent::OutputTextDelta {
// Prefer the explicit item_id from the SSE event; fall back to last seen.
if let Some(ref id) = event.item_id {
current_item_id = Some(id.clone());
}
tracing::debug!("sse.delta output_text id={:?} len={}", current_item_id, delta.len());
let ev = ResponseEvent::OutputTextDelta {
delta,
item_id: current_item_id.clone(),
item_id: event.item_id.or_else(|| current_item_id.clone()),
};
if tx_event.send(Ok(event)).await.is_err() {
if tx_event.send(Ok(ev)).await.is_err() {
return;
}
}
}
"response.reasoning_summary_text.delta" => {
if let Some(delta) = event.delta {
let event = ResponseEvent::ReasoningSummaryDelta {
if let Some(ref id) = event.item_id {
current_item_id = Some(id.clone());
}
tracing::debug!("sse.delta reasoning_summary id={:?} len={}", current_item_id, delta.len());
let ev = ResponseEvent::ReasoningSummaryDelta {
delta,
item_id: current_item_id.clone(),
item_id: event.item_id.or_else(|| current_item_id.clone()),
};
if tx_event.send(Ok(event)).await.is_err() {
if tx_event.send(Ok(ev)).await.is_err() {
return;
}
}
}
"response.reasoning_text.delta" => {
if let Some(delta) = event.delta {
let event = ResponseEvent::ReasoningContentDelta {
if let Some(ref id) = event.item_id {
current_item_id = Some(id.clone());
}
tracing::debug!("sse.delta reasoning_content id={:?} len={}", current_item_id, delta.len());
let ev = ResponseEvent::ReasoningContentDelta {
delta,
item_id: current_item_id.clone(),
item_id: event.item_id.or_else(|| current_item_id.clone()),
};
if tx_event.send(Ok(event)).await.is_err() {
if tx_event.send(Ok(ev)).await.is_err() {
return;
}
}
codex-rs/core/src/codex.rs
+495 -68
View File
@@ -25,6 +25,7 @@ use futures::prelude::*;
use mcp_types::CallToolResult;
use serde::Serialize;
use serde_json;
use serde_json::json;
use tokio::sync::oneshot;
use tokio::task::AbortHandle;
use tracing::debug;
@@ -2007,6 +2008,9 @@ async fn run_agent(sess: Arc<Session>, sub_id: String, input: Vec<InputItem>) {
last_task_message = get_last_assistant_message_from_turn(
&items_to_record_in_conversation_history,
);
if let Some(m) = last_task_message.as_ref() {
tracing::info!("core.turn completed: last_assistant_message.len={}", m.len());
}
sess.maybe_notify(UserNotification::AgentTurnComplete {
turn_id: sub_id.clone(),
input_messages: turn_input_messages,
@@ -2036,6 +2040,12 @@ async fn run_agent(sess: Arc<Session>, sub_id: String, input: Vec<InputItem>) {
last_agent_message: last_task_message,
}),
};
match &event.msg {
EventMsg::TaskComplete(TaskCompleteEvent { last_agent_message: Some(m) }) => {
tracing::info!("core.emit TaskComplete last_agent_message.len={}", m.len());
}
_ => {}
}
sess.tx_event.send(event).await.ok();
}
@@ -2744,7 +2754,7 @@ async fn handle_browser_cleanup(
call_id: String,
) -> ResponseInputItem {
let call_id_clone = call_id.clone();
let sess_clone = sess;
let _sess_clone = sess;
execute_custom_tool(
sess,
&sub_id,
@@ -2752,7 +2762,7 @@ async fn handle_browser_cleanup(
"browser_cleanup".to_string(),
Some(serde_json::json!({})),
|| async move {
if let Some(browser_manager) = get_browser_manager_for_session(sess_clone).await {
if let Some(browser_manager) = get_browser_manager_for_session(_sess_clone).await {
match browser_manager.cleanup().await {
Ok(_) => ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
@@ -2796,6 +2806,8 @@ async fn handle_web_fetch(
url: String,
#[serde(default)]
timeout_ms: Option<u64>,
#[serde(default)]
mode: Option<String>, // "auto" (default), "browser", or "http"
}
let parsed: Result<WebFetchParams, _> = serde_json::from_str(&arguments_clone);
@@ -2812,86 +2824,434 @@ async fn handle_web_fetch(
}
};
let timeout = Duration::from_millis(params.timeout_ms.unwrap_or(15000));
let user_agent = crate::user_agent::get_codex_user_agent(Some("web_fetch"));
let client = match reqwest::Client::builder()
.user_agent(user_agent)
.timeout(timeout)
.build()
{
Ok(c) => c,
Err(e) => {
return ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload {
content: format!("Failed to build HTTP client: {e}"),
success: Some(false),
},
};
// Helper: build a client with a specific UA and common headers.
async fn do_request(
url: &str,
ua: &str,
timeout: Duration,
extra_headers: Option<&[(reqwest::header::HeaderName, &'static str)]>,
) -> Result<reqwest::Response, reqwest::Error> {
let client = reqwest::Client::builder()
.timeout(timeout)
.user_agent(ua)
.build()?;
let mut req = client.get(url)
// Add a few browser-like headers to reduce blocks
.header(reqwest::header::ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
.header(reqwest::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9");
if let Some(pairs) = extra_headers {
for (k, v) in pairs.iter() {
req = req.header(k, *v);
}
}
};
req.send().await
}
let resp = match client.get(&params.url).send().await {
// Helper: remove obvious noisy blocks before markdown conversion.
fn strip_noisy_tags(mut html: String) -> String {
// Basic, case-insensitive removal of <script>, <style>, and <noscript> blocks.
// Avoids dumping challenge JS into markdown on blocked pages.
let patterns = [("script", "</script>"), ("style", "</style>"), ("noscript", "</noscript>")];
for (open, close) in patterns {
// Iterate until none remain (with a safety cap on iterations)
let mut guard = 0;
loop {
if guard > 64 { break; }
let start = html.to_lowercase().find(&format!("<{open}"));
let Some(s) = start else { break; };
let end_search_from = s + 1;
let lower = html.to_lowercase();
let end = lower[end_search_from..].find(&close).map(|i| end_search_from + i + close.len());
if let Some(e) = end {
html.replace_range(s..e, "");
} else {
// No close tag found; remove from open to end
html.truncate(s);
break;
}
guard += 1;
}
}
html
}
// Helper: convert HTML to markdown and truncate if too large.
fn convert_html_to_markdown_trimmed(html: String, max_chars: usize) -> crate::error::Result<(String, bool)> {
let options = htmd::options::Options {
heading_style: htmd::options::HeadingStyle::Atx,
code_block_style: htmd::options::CodeBlockStyle::Fenced,
link_style: htmd::options::LinkStyle::Inlined,
..Default::default()
};
let converter = htmd::HtmlToMarkdown::builder().options(options).build();
let sanitized = strip_noisy_tags(html);
let markdown = converter.convert(&sanitized)?;
let mut truncated = false;
let rendered = if markdown.len() > max_chars {
truncated = true;
let mut s = markdown;
s.truncate(max_chars);
s.push_str("\n\n… (truncated)\n");
s
} else {
markdown
};
Ok((rendered, truncated))
}
// Helper: detect WAF/challenge pages to avoid dumping challenge content.
fn detect_block_vendor(_status: reqwest::StatusCode, body: &str) -> Option<&'static str> {
// Identify common bot-challenge pages regardless of HTTP status.
// Cloudflare often returns 200 with a challenge that requires JS/cookies.
let lower = body.to_lowercase();
if lower.contains("cloudflare")
|| lower.contains("cf-ray")
|| lower.contains("_cf_chl_opt")
|| lower.contains("challenge-platform")
|| lower.contains("checking if the site connection is secure")
|| lower.contains("waiting for")
|| lower.contains("just a moment")
{
return Some("cloudflare");
}
None
}
fn headers_indicate_block(headers: &reqwest::header::HeaderMap) -> bool {
let h = headers;
let has_cf_ray = h.get("cf-ray").is_some();
let has_cf_mitigated = h.get("cf-mitigated").is_some();
let has_cf_bm = h.get("set-cookie").and_then(|v| v.to_str().ok()).map(|s| s.contains("__cf_bm=")).unwrap_or(false);
let has_chlray = h.get("server-timing").and_then(|v| v.to_str().ok()).map(|s| s.to_lowercase().contains("chlray")).unwrap_or(false);
has_cf_ray || has_cf_mitigated || has_cf_bm || has_chlray
}
fn looks_like_challenge_markdown(md: &str) -> bool {
let l = md.to_lowercase();
l.contains("just a moment") || l.contains("enable javascript and cookies") || l.contains("waiting for ")
}
let timeout = Duration::from_millis(params.timeout_ms.unwrap_or(15000));
let codex_ua = crate::user_agent::get_codex_user_agent(Some("web_fetch"));
// If explicit browser mode requested, try the internal browser first.
if matches!(params.mode.as_deref(), Some("browser")) {
{
let browser_manager = codex_browser::global::get_or_create_browser_manager().await;
if let Ok(res) = browser_manager.goto(&params.url).await {
// Allow a few short settles for JS/cookie challenges to auto-resolve
let mut html: Option<String> = None;
for _ in 0..4 {
if let Ok(val) = browser_manager.execute_javascript("(function(){ return { html: document.documentElement.outerHTML, title: document.title||'' }; })()").await {
html = val.get("value").and_then(|v| v.get("html")).and_then(|v| v.as_str()).map(|s| s.to_string());
let t_low = val.get("value").and_then(|v| v.get("title")).and_then(|v| v.as_str()).unwrap_or("").to_lowercase();
if !(t_low.contains("just a moment") || t_low.contains("checking if") || t_low.contains("waiting for")) {
break;
}
}
tokio::time::sleep(Duration::from_millis(1200)).await;
}
if let Some(html) = html {
let (markdown, truncated) = match convert_html_to_markdown_trimmed(html, 120_000) {
Ok(t) => t,
Err(e) => {
return ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload { content: format!("Markdown conversion failed: {e}"), success: Some(false) },
};
}
};
let body = serde_json::json!({
"url": params.url,
"status": 200,
"final_url": res.url,
"content_type": "text/html",
"used_browser_ua": true,
"via_browser": true,
"truncated": truncated,
"markdown": markdown,
});
return ResponseInputItem::FunctionCallOutput { call_id: call_id_clone, output: FunctionCallOutputPayload { content: body.to_string(), success: Some(true) } };
}
}
}
}
// Attempt 1: Codex UA + polite headers
let resp = match do_request(&params.url, &codex_ua, timeout, None).await {
Ok(r) => r,
Err(e) => {
return ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload {
content: format!("Request failed: {e}"),
success: Some(false),
},
output: FunctionCallOutputPayload { content: format!("Request failed: {e}"), success: Some(false) },
};
}
};
let status = resp.status();
let text = match resp.text().await {
// Capture metadata before consuming the response body.
let mut status = resp.status();
let mut final_url = resp.url().to_string();
let mut headers = resp.headers().clone();
// Read body
let mut body_text = match resp.text().await {
Ok(t) => t,
Err(e) => {
return ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload {
content: format!("Failed to read response body: {e}"),
success: Some(false),
},
output: FunctionCallOutputPayload { content: format!("Failed to read response body: {e}"), success: Some(false) },
};
}
};
let mut used_browser_ua = false;
let browser_ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36";
if !matches!(params.mode.as_deref(), Some("http")) && (detect_block_vendor(status, &body_text).is_some() || headers_indicate_block(&headers)) {
// Simple retry with a browser UA and extra headers
let extra = [
(reqwest::header::HeaderName::from_static("upgrade-insecure-requests"), "1"),
];
if let Ok(r2) = do_request(&params.url, browser_ua, timeout, Some(&extra)).await {
let status2 = r2.status();
let final_url2 = r2.url().to_string();
let headers2 = r2.headers().clone();
if let Ok(t2) = r2.text().await {
used_browser_ua = true;
status = status2;
final_url = final_url2;
headers = headers2;
body_text = t2;
}
}
}
// Response metadata
let content_type = headers
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
// Provide structured diagnostics if blocked by WAF (even if HTTP 200)
if !matches!(params.mode.as_deref(), Some("http")) && (detect_block_vendor(status, &body_text).is_some() || headers_indicate_block(&headers)) {
let vendor = "cloudflare";
let retry_after = headers
.get(reqwest::header::RETRY_AFTER)
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
let cf_ray = headers
.get("cf-ray")
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
let mut diag = serde_json::json!({
"final_url": final_url,
"content_type": content_type,
"used_browser_ua": used_browser_ua,
"blocked_by_waf": true,
"vendor": vendor,
});
if let Some(ra) = retry_after { diag["retry_after"] = serde_json::json!(ra); }
if let Some(ray) = cf_ray { diag["cf_ray"] = serde_json::json!(ray); }
// Attempt a last-resort browser-based fetch if the live browser is available.
{
let browser_manager = codex_browser::global::get_or_create_browser_manager().await;
browser_manager.set_enabled_sync(true);
// Try navigate and extract outerHTML via JS (after SPA settles per manager config)
if browser_manager.goto(&params.url).await.is_ok() {
let js = "(function(){ return { html: document.documentElement.outerHTML, title: document.title||'' }; })()";
if let Ok(val) = browser_manager.execute_javascript(js).await {
let html = val.get("value").and_then(|v| v.get("html")).and_then(|v| v.as_str()).unwrap_or("").to_string();
let title = val.get("value").and_then(|v| v.get("title")).and_then(|v| v.as_str()).unwrap_or("");
if !html.is_empty() && title.to_lowercase() != "just a moment..." {
let (markdown, truncated) = match convert_html_to_markdown_trimmed(html, 120_000) {
Ok(t) => t,
Err(e) => {
return ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload { content: format!("Markdown conversion failed: {e}"), success: Some(false) },
};
}
};
diag["via_browser"] = serde_json::json!(true);
let body = serde_json::json!({
"url": params.url,
"status": 200,
"final_url": final_url,
"content_type": content_type,
"used_browser_ua": used_browser_ua,
"truncated": truncated,
"markdown": markdown,
});
return ResponseInputItem::FunctionCallOutput { call_id: call_id_clone, output: FunctionCallOutputPayload { content: body.to_string(), success: Some(true) } };
}
}
// If JS extraction failed, try a CDP outerHTML of root
let root = browser_manager.execute_cdp("DOM.getDocument", json!({"depth": 1})).await.ok();
if let Some(root) = root {
if let Some(node_id) = root.get("root").and_then(|r| r.get("nodeId")).and_then(|n| n.as_u64()) {
if let Ok(outer) = browser_manager.execute_cdp("DOM.getOuterHTML", json!({"nodeId": node_id})).await {
if let Some(html) = outer.get("outerHTML").and_then(|v| v.as_str()) {
let (markdown, truncated) = match convert_html_to_markdown_trimmed(html.to_string(), 120_000) {
Ok(t) => t,
Err(e) => {
return ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload { content: format!("Markdown conversion failed: {e}"), success: Some(false) },
};
}
};
diag["via_browser"] = serde_json::json!(true);
let body = serde_json::json!({
"url": params.url,
"status": 200,
"final_url": final_url,
"content_type": content_type,
"used_browser_ua": used_browser_ua,
"truncated": truncated,
"markdown": markdown,
});
return ResponseInputItem::FunctionCallOutput { call_id: call_id_clone, output: FunctionCallOutputPayload { content: body.to_string(), success: Some(true) } };
}
}
}
}
}
}
let (md_preview, _trunc) = match convert_html_to_markdown_trimmed(body_text, 2000) {
Ok(t) => t,
Err(_) => ("".to_string(), false),
};
let body = serde_json::json!({
"url": params.url,
"status": status.as_u16(),
"error": "Blocked by site challenge",
"diagnostics": diag,
"markdown": md_preview,
});
return ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload { content: body.to_string(), success: Some(false) },
};
}
// If not success, provide structured, minimal diagnostics without dumping content.
if !status.is_success() {
let waf_vendor = detect_block_vendor(status, &body_text);
let retry_after = headers
.get(reqwest::header::RETRY_AFTER)
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
let cf_ray = headers
.get("cf-ray")
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string());
let mut diag = serde_json::json!({
"final_url": final_url,
"content_type": content_type,
"used_browser_ua": used_browser_ua,
});
if let Some(vendor) = waf_vendor { diag["blocked_by_waf"] = serde_json::json!(true); diag["vendor"] = serde_json::json!(vendor); }
if let Some(ra) = retry_after { diag["retry_after"] = serde_json::json!(ra); }
if let Some(ray) = cf_ray { diag["cf_ray"] = serde_json::json!(ray); }
// Provide a tiny, safe preview of visible text only (converted and truncated).
let (md_preview, _trunc) = match convert_html_to_markdown_trimmed(body_text, 2000) {
Ok(t) => t,
Err(_) => ("".to_string(), false),
};
let body = serde_json::json!({
"url": params.url,
"status": status.as_u16(),
"error": format!("HTTP {} {}", status.as_u16(), status.canonical_reason().unwrap_or("")),
"diagnostics": diag,
// Keep a short, human-friendly preview; avoid dumping raw HTML or long JS.
"markdown": md_preview,
});
return ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload { content: body.to_string(), success: Some(false) },
};
}
// Success: convert to markdown (sanitized and size-limited)
let (markdown, truncated) = match convert_html_to_markdown_trimmed(body_text, 120_000) {
Ok(t) => t,
Err(e) => {
return ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload { content: format!("Markdown conversion failed: {e}"), success: Some(false) },
};
}
};
// Configure htmd with conservative, GFM‑like defaults
let options = htmd::options::Options {
heading_style: htmd::options::HeadingStyle::Atx,
code_block_style: htmd::options::CodeBlockStyle::Fenced,
link_style: htmd::options::LinkStyle::Inlined,
..Default::default()
};
let converter = htmd::HtmlToMarkdown::builder().options(options).build();
let markdown = match converter.convert(&text) {
Ok(m) => m,
Err(e) => {
return ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload {
content: format!("Markdown conversion failed: {e}"),
success: Some(false),
},
};
// If the rendered markdown still looks like a challenge page, attempt browser fallback (unless http-only).
if !matches!(params.mode.as_deref(), Some("http")) && looks_like_challenge_markdown(&markdown) {
{
let browser_manager = codex_browser::global::get_or_create_browser_manager().await;
browser_manager.set_enabled_sync(true);
if browser_manager.goto(&params.url).await.is_ok() {
let js = "(function(){ return { html: document.documentElement.outerHTML, title: document.title||'' }; })()";
let mut html: Option<String> = None;
for _ in 0..3 {
if let Ok(val) = browser_manager.execute_javascript(js).await {
html = val.get("value").and_then(|v| v.get("html")).and_then(|v| v.as_str()).map(|s| s.to_string());
let t_low = val.get("value").and_then(|v| v.get("title")).and_then(|v| v.as_str()).unwrap_or("").to_lowercase();
if !(t_low.contains("just a moment") || t_low.contains("checking if") || t_low.contains("waiting for")) {
break;
}
}
tokio::time::sleep(Duration::from_millis(1200)).await;
}
if let Some(html) = html {
let (md2, truncated2) = match convert_html_to_markdown_trimmed(html, 120_000) {
Ok(t) => t,
Err(e) => {
return ResponseInputItem::FunctionCallOutput { call_id: call_id_clone, output: FunctionCallOutputPayload { content: format!("Markdown conversion failed: {e}"), success: Some(false) } };
}
};
let body = serde_json::json!({
"url": params.url,
"status": 200,
"final_url": final_url,
"content_type": content_type,
"used_browser_ua": true,
"via_browser": true,
"truncated": truncated2,
"markdown": md2,
});
return ResponseInputItem::FunctionCallOutput { call_id: call_id_clone, output: FunctionCallOutputPayload { content: body.to_string(), success: Some(true) } };
}
}
}
};
// If fallback not possible, return structured error rather than a useless challenge page
let body = serde_json::json!({
"url": params.url,
"status": 200,
"error": "Blocked by site challenge",
"diagnostics": { "final_url": final_url, "content_type": content_type, "used_browser_ua": used_browser_ua, "blocked_by_waf": true, "vendor": "cloudflare", "detected_via": "markdown" },
"markdown": markdown.chars().take(2000).collect::<String>(),
});
return ResponseInputItem::FunctionCallOutput { call_id: call_id_clone, output: FunctionCallOutputPayload { content: body.to_string(), success: Some(false) } };
}
let body = serde_json::json!({
"url": params.url,
"status": status.as_u16(),
"final_url": final_url,
"content_type": content_type,
"used_browser_ua": used_browser_ua,
"truncated": truncated,
"markdown": markdown,
});
ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload {
content: body.to_string(),
success: Some(status.is_success()),
},
}
ResponseInputItem::FunctionCallOutput { call_id: call_id_clone, output: FunctionCallOutputPayload { content: body.to_string(), success: Some(true) } }
},
).await
}
@@ -3911,7 +4271,7 @@ async fn handle_container_exec_with_params(
let ExecToolCallOutput { exit_code, .. } = &output;
let is_success = *exit_code == 0;
let content = format_exec_output(&output);
let content = format_exec_output_with_limit(sess, &sub_id, &call_id, &output);
ResponseInputItem::FunctionCallOutput {
call_id: call_id.clone(),
output: FunctionCallOutputPayload {
@@ -4042,7 +4402,7 @@ async fn handle_sandbox_error(
let ExecToolCallOutput { exit_code, .. } = &retry_output;
let is_success = *exit_code == 0;
let content = format_exec_output(&retry_output);
let content = format_exec_output_with_limit(sess, &sub_id, &call_id, &retry_output);
ResponseInputItem::FunctionCallOutput {
call_id: call_id.clone(),
@@ -4074,6 +4434,51 @@ async fn handle_sandbox_error(
}
}
// Limit extremely large tool outputs before sending to the model to avoid
// context overflows. Keep this conservative because multiple tool outputs
// can appear in a single turn. The limit is in bytes (on the UTF‑8 string).
const MAX_TOOL_OUTPUT_BYTES_FOR_MODEL: usize = 32 * 1024; // 32 KiB
fn truncate_middle_bytes(s: &str, max_bytes: usize) -> (String, bool) {
if s.len() <= max_bytes {
return (s.to_string(), false);
}
if max_bytes == 0 {
return ("…truncated…".to_string(), true);
}
// Try to keep some head/tail, favoring newline boundaries when possible.
let keep = max_bytes.saturating_sub("…truncated…\n".len());
let left_budget = keep / 2;
let right_budget = keep - left_budget;
// Safe prefix end on a char boundary, prefer last newline within budget.
let prefix_end = {
let mut end = left_budget.min(s.len());
if let Some(head) = s.get(..end) {
if let Some(i) = head.rfind('\n') { end = i + 1; }
}
while end > 0 && !s.is_char_boundary(end) { end -= 1; }
end
};
// Safe suffix start on a char boundary, prefer first newline within budget.
let suffix_start = {
let mut start = s.len().saturating_sub(right_budget);
if let Some(tail) = s.get(start..) {
if let Some(i) = tail.find('\n') { start += i + 1; }
}
while start < s.len() && !s.is_char_boundary(start) { start += 1; }
start
};
let mut out = String::with_capacity(max_bytes);
out.push_str(&s[..prefix_end]);
out.push_str("…truncated…\n");
out.push_str(&s[suffix_start..]);
(out, true)
}
fn format_exec_output_str(exec_output: &ExecToolCallOutput) -> String {
let ExecToolCallOutput {
exit_code,
@@ -4095,8 +4500,14 @@ fn format_exec_output_str(exec_output: &ExecToolCallOutput) -> String {
formatted_output
}
/// Exec output is a pre-serialized JSON payload
fn format_exec_output(exec_output: &ExecToolCallOutput) -> String {
/// Exec output serialized for the model. If the payload is too large,
/// write the full output to a file and include a truncated preview here.
fn format_exec_output_with_limit(
sess: &Session,
sub_id: &str,
call_id: &str,
exec_output: &ExecToolCallOutput,
) -> String {
let ExecToolCallOutput {
exit_code,
duration,
@@ -4110,18 +4521,34 @@ fn format_exec_output(exec_output: &ExecToolCallOutput) -> String {
}
#[derive(Serialize)]
struct ExecOutput<'a> {
output: &'a str,
metadata: ExecMetadata,
}
struct ExecOutput<'a> { output: &'a str, metadata: ExecMetadata }
// round to 1 decimal place
let duration_seconds = ((duration.as_secs_f32()) * 10.0).round() / 10.0;
let formatted_output = format_exec_output_str(exec_output);
let full = format_exec_output_str(exec_output);
let (maybe_truncated, was_truncated) =
truncate_middle_bytes(&full, MAX_TOOL_OUTPUT_BYTES_FOR_MODEL);
// If truncated, persist the full output under .code/agents/<agent>/exec-<call_id>.txt
// so users can inspect it and the model can refer to a short, stable path.
let final_output = if was_truncated {
let cwd = sess.get_cwd().to_path_buf();
let file_note = match ensure_agent_dir(&cwd, sub_id)
.and_then(|dir| write_agent_file(&dir, &format!("exec-{call_id}.txt"), &full))
{
Ok(path) => format!("\n\n[Full output saved to: {}]", path.display()),
Err(e) => format!("\n\n[Full output was too large and truncation applied; failed to save file: {e}]")
};
let mut s = maybe_truncated;
s.push_str(&file_note);
s
} else {
maybe_truncated
};
let payload = ExecOutput {
output: &formatted_output,
output: &final_output,
metadata: ExecMetadata {
exit_code: *exit_code,
duration_seconds,
@@ -4444,13 +4871,13 @@ async fn handle_browser_open(
tracing::info!("Creating new browser manager");
let new_manager =
codex_browser::global::get_or_create_browser_manager().await;
// Enable the browser
new_manager.set_enabled_sync(true);
Some(new_manager)
}
};
if let Some(browser_manager) = browser_manager {
// Ensure the browser manager is marked enabled so status reflects reality
browser_manager.set_enabled_sync(true);
// Clear any lingering node highlight from previous commands
let _ = browser_manager
.execute_cdp("Overlay.hideHighlight", serde_json::json!({}))
codex-rs/core/src/openai_tools.rs
+8
View File
@@ -1486,6 +1486,14 @@ fn create_web_fetch_tool() -> OpenAiTool {
},
);
// Optional mode: auto (default), browser (use internal browser/CDP), http (raw HTTP only)
properties.insert(
"mode".to_string(),
JsonSchema::String {
description: Some("Optional: 'auto' (default) falls back to the internal browser on challenges; 'browser' forces CDP-based fetch; 'http' disables browser fallback.".to_string()),
},
);
OpenAiTool::Function(ResponsesApiTool {
name: "web_fetch".to_string(),
description: "Fetches a webpage over HTTP(S) and converts the HTML to Markdown using htmd.".to_string(),
codex-rs/tui/src/app.rs
+14 -5
View File
@@ -272,15 +272,24 @@ impl App<'_> {
while let Ok(event) = self.app_event_rx.recv() {
match event {
AppEvent::InsertHistory(lines) => match &mut self.app_state {
AppState::Chat { widget } => widget.insert_history_lines(lines),
AppState::Chat { widget } => {
tracing::debug!("app: InsertHistory lines={}", lines.len());
widget.insert_history_lines(lines)
},
AppState::Onboarding { .. } => {}
},
AppEvent::InsertHistoryWithKind { kind, lines } => match &mut self.app_state {
AppState::Chat { widget } => widget.insert_history_lines_with_kind(kind, lines),
AppEvent::InsertHistoryWithKind { id, kind, lines } => match &mut self.app_state {
AppState::Chat { widget } => {
tracing::debug!("app: InsertHistoryWithKind kind={:?} id={:?} lines={}", kind, id, lines.len());
widget.insert_history_lines_with_kind(kind, id, lines)
},
AppState::Onboarding { .. } => {}
},
AppEvent::InsertFinalAnswer { lines, source } => match &mut self.app_state {
AppState::Chat { widget } => widget.insert_final_answer(lines, source),
AppEvent::InsertFinalAnswer { id, lines, source } => match &mut self.app_state {
AppState::Chat { widget } => {
tracing::debug!("app: InsertFinalAnswer id={:?} lines={} source_len={}", id, lines.len(), source.len());
widget.insert_final_answer_with_id(id, lines, source)
},
AppState::Onboarding { .. } => {}
},
AppEvent::RequestRedraw => {
codex-rs/tui/src/app_event.rs
+2 -2
View File
@@ -85,9 +85,9 @@ pub(crate) enum AppEvent {
DiffResult(String),
InsertHistory(Vec<Line<'static>>),
InsertHistoryWithKind { kind: StreamKind, lines: Vec<Line<'static>> },
InsertHistoryWithKind { id: Option<String>, kind: StreamKind, lines: Vec<Line<'static>> },
/// Finalized assistant answer with raw markdown for re-rendering under theme changes.
InsertFinalAnswer { lines: Vec<Line<'static>>, source: String },
InsertFinalAnswer { id: Option<String>, lines: Vec<Line<'static>>, source: String },
#[allow(dead_code)]
StartCommitAnimation,
codex-rs/tui/src/chatwidget.rs
+326 -46
View File
@@ -1,4 +1,4 @@
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use once_cell::sync::Lazy;
@@ -186,6 +186,12 @@ pub(crate) struct ChatWidget<'a> {
// Interrupt manager for handling cancellations
interrupts: interrupts::InterruptManager,
// Track ids that have produced a finalized Answer/Reasoning in the current turn.
// Any further deltas for these ids should be ignored to avoid post-final updates.
closed_answer_ids: HashSet<String>,
closed_reasoning_ids: HashSet<String>,
// Accumulated patch change sets for this session (latest last)
session_patch_sets: Vec<HashMap<PathBuf, codex_core::protocol::FileChange>>,
// Baseline original contents captured when a file first appears in a change set
@@ -630,7 +636,7 @@ impl ChatWidget<'_> {
}
/// Handle streaming delta for both answer and reasoning
fn handle_streaming_delta(&mut self, kind: StreamKind, delta: String) {
fn handle_streaming_delta(&mut self, kind: StreamKind, id: String, delta: String) {
tracing::debug!("handle_streaming_delta kind={:?}, delta={:?}", kind, delta);
// Remember which stream is currently active so we can group inserts
self.current_stream_kind = Some(kind);
@@ -638,7 +644,7 @@ impl ChatWidget<'_> {
// Always begin the correct stream for this delta's kind
// This will switch streams if needed or no-op if already active
self.stream.begin(kind, &sink);
self.stream.begin_with_id(kind, Some(id), &sink);
// Append delta to the stream
self.stream.push_and_maybe_commit(&delta, &sink);
@@ -1393,6 +1399,8 @@ impl ChatWidget<'_> {
stream: crate::streaming::controller::StreamController::new(config.clone()),
current_stream_kind: None,
interrupts: interrupts::InterruptManager::new(),
closed_answer_ids: HashSet::new(),
closed_reasoning_ids: HashSet::new(),
session_patch_sets: Vec::new(),
baseline_file_contents: HashMap::new(),
diff_overlay: None,
@@ -1499,6 +1507,8 @@ impl ChatWidget<'_> {
stream: crate::streaming::controller::StreamController::new(config.clone()),
current_stream_kind: None,
interrupts: interrupts::InterruptManager::new(),
closed_answer_ids: HashSet::new(),
closed_reasoning_ids: HashSet::new(),
session_patch_sets: Vec::new(),
baseline_file_contents: HashMap::new(),
diff_overlay: None,
@@ -2487,7 +2497,7 @@ impl ChatWidget<'_> {
}
pub(crate) fn handle_codex_event(&mut self, event: Event) {
tracing::info!(
tracing::debug!(
"handle_codex_event({})",
serde_json::to_string_pretty(&event).unwrap_or_default()
);
@@ -2534,13 +2544,22 @@ impl ChatWidget<'_> {
self.mark_needs_redraw();
}
EventMsg::AgentMessage(AgentMessageEvent { message }) => {
tracing::debug!("AgentMessage event with message: {:?}...", message.chars().take(100).collect::<String>());
tracing::debug!(
"AgentMessage final id={} bytes={} preview={:?}",
id,
message.len(),
message.chars().take(80).collect::<String>()
);
// Use StreamController for final answer
let sink = AppEventHistorySink(self.app_event_tx.clone());
self.current_stream_kind = Some(StreamKind::Answer);
// Seed the controller with this id so finalize targets the right streaming cell
self.stream.begin_with_id(StreamKind::Answer, Some(id.clone()), &sink);
let _finished = self.stream.apply_final_answer(&message, &sink);
// Stream finishing is handled by StreamController
self.last_assistant_message = Some(message);
// Mark this id closed for further answer deltas in this turn
self.closed_answer_ids.insert(id.clone());
self.mark_needs_redraw();
}
EventMsg::ReplayHistory(ev) => {
@@ -2640,8 +2659,13 @@ impl ChatWidget<'_> {
}
EventMsg::AgentMessageDelta(AgentMessageDeltaEvent { delta }) => {
tracing::debug!("AgentMessageDelta: {:?}", delta);
// Ignore late deltas for ids that have already finalized in this turn
if self.closed_answer_ids.contains(&id) {
tracing::debug!("Ignoring Answer delta for closed id={}", id);
return;
}
// Stream answer delta through StreamController
self.handle_streaming_delta(StreamKind::Answer, delta);
self.handle_streaming_delta(StreamKind::Answer, id.clone(), delta);
// Show responding state while assistant streams
self.bottom_pane.update_status_text("responding".to_string());
}
@@ -2650,17 +2674,25 @@ impl ChatWidget<'_> {
// Use StreamController for final reasoning
let sink = AppEventHistorySink(self.app_event_tx.clone());
self.current_stream_kind = Some(StreamKind::Reasoning);
self.stream.begin_with_id(StreamKind::Reasoning, Some(id.clone()), &sink);
// The StreamController now properly handles duplicate detection and prevents
// re-injecting content when we're already finishing a stream
let _finished = self.stream.apply_final_reasoning(&text, &sink);
// Stream finishing is handled by StreamController
// Mark this id closed for further reasoning deltas in this turn
self.closed_reasoning_ids.insert(id.clone());
self.mark_needs_redraw();
}
EventMsg::AgentReasoningDelta(AgentReasoningDeltaEvent { delta }) => {
tracing::debug!("AgentReasoningDelta: {:?}", delta);
// Ignore late deltas for ids that have already finalized in this turn
if self.closed_reasoning_ids.contains(&id) {
tracing::debug!("Ignoring Reasoning delta for closed id={}", id);
return;
}
// Stream reasoning delta through StreamController
self.handle_streaming_delta(StreamKind::Reasoning, delta);
self.handle_streaming_delta(StreamKind::Reasoning, id.clone(), delta);
// Show thinking state while reasoning streams
self.bottom_pane.update_status_text("thinking".to_string());
}
@@ -2673,6 +2705,9 @@ impl ChatWidget<'_> {
// Reset stream headers for new turn
self.stream.reset_headers_for_new_turn();
self.current_stream_kind = None;
// New turn: clear closed id guards
self.closed_answer_ids.clear();
self.closed_reasoning_ids.clear();
self.bottom_pane.clear_ctrl_c_quit_hint();
self.bottom_pane.set_task_running(true);
self.bottom_pane.update_status_text("waiting for model".to_string());
@@ -2749,13 +2784,14 @@ impl ChatWidget<'_> {
delta,
}) => {
// Treat raw reasoning content the same as summarized reasoning
self.handle_streaming_delta(StreamKind::Reasoning, delta);
self.handle_streaming_delta(StreamKind::Reasoning, id.clone(), delta);
}
EventMsg::AgentReasoningRawContent(AgentReasoningRawContentEvent { text }) => {
tracing::debug!("AgentReasoningRawContent event with text: {:?}...", text.chars().take(100).collect::<String>());
// Use StreamController for final raw reasoning
let sink = AppEventHistorySink(self.app_event_tx.clone());
self.current_stream_kind = Some(StreamKind::Reasoning);
self.stream.begin_with_id(StreamKind::Reasoning, Some(id.clone()), &sink);
let _finished = self.stream.apply_final_reasoning(&text, &sink);
// Stream finishing is handled by StreamController
self.mark_needs_redraw();
@@ -3756,12 +3792,13 @@ impl ChatWidget<'_> {
pub(crate) fn insert_history_lines(&mut self, lines: Vec<ratatui::text::Line<'static>>) {
let kind = self.current_stream_kind.unwrap_or(StreamKind::Answer);
self.insert_history_lines_with_kind(kind, lines);
self.insert_history_lines_with_kind(kind, None, lines);
}
pub(crate) fn insert_history_lines_with_kind(
&mut self,
kind: StreamKind,
id: Option<String>,
mut lines: Vec<ratatui::text::Line<'static>>,
) {
// No debug logging: we rely on preserving span modifiers end-to-end.
@@ -3814,6 +3851,11 @@ impl ChatWidget<'_> {
self.history_cells.push(Box::new(cell));
}
StreamKind::Answer => {
tracing::debug!(
"history.insert Answer id={:?} incoming_lines={}",
id,
lines.len()
);
// Any incoming Answer means reasoning is no longer bottom-most
self.clear_reasoning_in_progress();
// Keep a single StreamingContentCell and append to it
@@ -3822,29 +3864,78 @@ impl ChatWidget<'_> {
.as_any_mut()
.downcast_mut::<history_cell::StreamingContentCell>()
{
// Guard against stray header sneaking into a later chunk
if lines.first().map(|l| {
l.spans
.iter()
.map(|s| s.content.as_ref())
.collect::<String>()
.trim()
.eq_ignore_ascii_case("codex")
}).unwrap_or(false) {
if lines.len() == 1 {
return;
// If id is specified, only append when ids match
if let Some(ref want) = id {
if stream_cell.id.as_ref() != Some(want) {
// fall through to create/find matching cell below
} else {
lines.remove(0);
tracing::debug!("history.append -> last StreamingContentCell (id match) lines+={}", lines.len());
// Guard against stray header sneaking into a later chunk
if lines.first().map(|l| {
l.spans
.iter()
.map(|s| s.content.as_ref())
.collect::<String>()
.trim()
.eq_ignore_ascii_case("codex")
}).unwrap_or(false) {
if lines.len() == 1 {
return;
} else {
lines.remove(0);
}
}
stream_cell.lines.extend(lines);
self.invalidate_height_cache();
self.autoscroll_if_near_bottom();
self.request_redraw();
return;
}
} else {
// No id — legacy: append to last
tracing::debug!("history.append -> last StreamingContentCell (no id provided) lines+={}", lines.len());
if lines.first().map(|l| {
l.spans
.iter()
.map(|s| s.content.as_ref())
.collect::<String>()
.trim()
.eq_ignore_ascii_case("codex")
}).unwrap_or(false) {
if lines.len() == 1 { return; } else { lines.remove(0); }
}
stream_cell.lines.extend(lines);
self.invalidate_height_cache();
self.autoscroll_if_near_bottom();
self.request_redraw();
return;
}
}
}
// If id is specified, try to locate an existing streaming cell with that id
if let Some(ref want) = id {
if let Some(idx) = self.history_cells.iter().rposition(|c| c
.as_any()
.downcast_ref::<history_cell::StreamingContentCell>()
.map(|sc| sc.id.as_ref() == Some(want)).unwrap_or(false))
{
if let Some(stream_cell) = self.history_cells[idx]
.as_any_mut()
.downcast_mut::<history_cell::StreamingContentCell>()
{
tracing::debug!("history.append -> StreamingContentCell by id at idx={} lines+={}", idx, lines.len());
if lines.first().map(|l| {
l.spans.iter().map(|s| s.content.as_ref()).collect::<String>().trim().eq_ignore_ascii_case("codex")
}).unwrap_or(false) {
if lines.len() == 1 { return; } else { lines.remove(0); }
}
stream_cell.lines.extend(lines);
self.invalidate_height_cache();
self.autoscroll_if_near_bottom();
self.request_redraw();
return;
}
// No debug logging in steady-state.
stream_cell.lines.extend(lines);
// Content height changed; clear memoized heights
self.invalidate_height_cache();
self.autoscroll_if_near_bottom();
self.request_redraw();
return;
}
}
@@ -3864,7 +3955,9 @@ impl ChatWidget<'_> {
with_header.extend(lines);
lines = with_header;
}
self.history_cells.push(Box::new(history_cell::new_streaming_content(lines)));
let new_idx = self.history_cells.len();
self.history_cells.push(Box::new(history_cell::new_streaming_content_with_id(id.clone(), lines)));
tracing::debug!("history.new StreamingContentCell at idx={} id={:?}", new_idx, id);
}
}
@@ -3875,11 +3968,70 @@ impl ChatWidget<'_> {
/// Replace the in-progress streaming assistant cell with a final markdown cell that
/// stores raw markdown for future re-rendering.
pub(crate) fn insert_final_answer(
pub(crate) fn insert_final_answer_with_id(
&mut self,
id: Option<String>,
lines: Vec<ratatui::text::Line<'static>>,
source: String,
) {
tracing::debug!("insert_final_answer_with_id id={:?} source_len={} lines={}", id, source.len(), lines.len());
// Debug: list last few history cell kinds so we can see what's present
let tail_kinds: String = self
.history_cells
.iter()
.rev()
.take(5)
.map(|c| {
if c.as_any().downcast_ref::<history_cell::StreamingContentCell>().is_some() {
"Streaming".to_string()
} else if c.as_any().downcast_ref::<history_cell::AssistantMarkdownCell>().is_some() {
"AssistantFinal".to_string()
} else if c.as_any().downcast_ref::<history_cell::CollapsibleReasoningCell>().is_some() {
"Reasoning".to_string()
} else {
format!("{:?}", c.kind())
}
})
.collect::<Vec<_>>()
.join(", ");
tracing::debug!("history.tail kinds(last5) = [{}]", tail_kinds);
// When we have an id but could not find a streaming cell by id, dump ids
if id.is_some() {
let ids: Vec<String> = self.history_cells.iter().enumerate().filter_map(|(i,c)| {
c.as_any().downcast_ref::<history_cell::StreamingContentCell>().and_then(|sc| {
sc.id.as_ref().map(|s| format!("{}:{}", i, s))
})
}).collect();
tracing::debug!("history.streaming ids={}", ids.join(" | "));
}
// If we already finalized this id in the current turn with identical content,
// drop this event to avoid duplicates (belt-and-suspenders against upstream repeats).
if let Some(ref want) = id {
if self.closed_answer_ids.contains(want) {
if let Some(existing_idx) = self
.history_cells
.iter()
.rposition(|c| c
.as_any()
.downcast_ref::<history_cell::AssistantMarkdownCell>()
.map(|amc| amc.id.as_ref() == Some(want))
.unwrap_or(false))
{
if let Some(amc) = self.history_cells[existing_idx]
.as_any()
.downcast_ref::<history_cell::AssistantMarkdownCell>()
{
let prev = Self::normalize_text(&amc.raw);
let newn = Self::normalize_text(&source);
if prev == newn {
tracing::debug!("InsertFinalAnswer: dropping duplicate final for id={}", want);
return;
}
}
}
}
}
// Ensure a hidden 'codex' header is present
let has_header = lines.first().map(|l| {
l.spans.iter().map(|s| s.content.as_ref()).collect::<String>().trim().eq_ignore_ascii_case("codex")
@@ -3888,21 +4040,140 @@ impl ChatWidget<'_> {
// No need to mutate `lines` further since we rebuild from `source` below.
}
// If the last cell is a StreamingContentCell, replace it; else append
if let Some(last) = self.history_cells.last_mut() {
if last.as_any().downcast_ref::<history_cell::StreamingContentCell>().is_some() {
let cell = history_cell::AssistantMarkdownCell::new(source, &self.config);
*last = Box::new(cell);
// Replace the matching StreamingContentCell if one exists for this id; else fallback to most recent.
// NOTE (dup‑guard): This relies on `StreamingContentCell::as_any()` returning `self`.
// If that impl is removed, downcast_ref will fail and we won't find the streaming cell,
// causing the final to append a new Assistant cell (duplicate).
let streaming_idx = if let Some(ref want) = id {
// Prefer an exact id match
let by_id = self
.history_cells
.iter()
.rposition(|c| {
if let Some(sc) = c.as_any().downcast_ref::<history_cell::StreamingContentCell>() {
sc.id.as_ref() == Some(want)
} else {
false
}
});
// Fallback: use the latest streaming cell regardless of id if no exact match
by_id.or_else(|| {
self.history_cells
.iter()
.rposition(|c| c.as_any().downcast_ref::<history_cell::StreamingContentCell>().is_some())
})
} else {
self.history_cells
.iter()
.rposition(|c| c.as_any().downcast_ref::<history_cell::StreamingContentCell>().is_some())
};
if let Some(idx) = streaming_idx {
tracing::debug!("final-answer: replacing StreamingContentCell at idx={} by id match", idx);
// Replace the matching streaming cell in-place, preserving the id
let cell = history_cell::AssistantMarkdownCell::new_with_id(source, id.clone(), &self.config);
self.history_cells[idx] = Box::new(cell);
self.invalidate_height_cache();
self.autoscroll_if_near_bottom();
self.request_redraw();
return;
}
// No streaming cell found. First, try to replace a finalized assistant cell
// that was created for the same stream id (e.g., we already finalized due to
// a lifecycle event and this InsertFinalAnswer arrived slightly later).
if let Some(ref want) = id {
if let Some(idx) = self
.history_cells
.iter()
.rposition(|c| {
if let Some(amc) = c.as_any().downcast_ref::<history_cell::AssistantMarkdownCell>() {
amc.id.as_ref() == Some(want)
} else {
false
}
})
{
tracing::debug!("final-answer: replacing existing AssistantMarkdownCell at idx={} by id match", idx);
let cell = history_cell::AssistantMarkdownCell::new_with_id(source, id.clone(), &self.config);
self.history_cells[idx] = Box::new(cell);
self.invalidate_height_cache();
self.autoscroll_if_near_bottom();
self.request_redraw();
return;
}
}
let cell = history_cell::AssistantMarkdownCell::new(source, &self.config);
// Otherwise, if a finalized assistant cell exists at the tail,
// replace it in place to avoid duplicate assistant messages when a second
// InsertFinalAnswer (e.g., from an AgentMessage event) arrives after we already
// finalized due to a side event.
if let Some(idx) = self
.history_cells
.iter()
.rposition(|c| c.as_any().downcast_ref::<history_cell::AssistantMarkdownCell>().is_some())
{
// Replace the tail finalized assistant cell if the new content is identical OR
// a superset revision of the previous content (common provider behavior where
// a later final slightly extends the earlier one). Otherwise append a new
// assistant message so distinct messages remain separate.
let (should_replace, _prev_len, _new_len) = self.history_cells[idx]
.as_any()
.downcast_ref::<history_cell::AssistantMarkdownCell>()
.map(|amc| {
let prev = Self::normalize_text(&amc.raw);
let newn = Self::normalize_text(&source);
let identical = prev == newn;
let is_superset = !identical && newn.contains(&prev);
// Heuristic: treat as revision when previous is reasonably long to
// avoid collapsing very short replies unintentionally.
let long_enough = prev.len() >= 80;
(identical || (is_superset && long_enough), prev.len(), newn.len())
})
.unwrap_or((false, 0, 0));
if should_replace {
tracing::debug!("final-answer: replacing tail AssistantMarkdownCell via heuristic identical/superset");
let cell = history_cell::AssistantMarkdownCell::new_with_id(source, id.clone(), &self.config);
self.history_cells[idx] = Box::new(cell);
self.invalidate_height_cache();
self.autoscroll_if_near_bottom();
self.request_redraw();
return;
}
}
// Fallback: no prior assistant cell found; append as new.
tracing::debug!("final-answer: append new AssistantMarkdownCell (no prior cell) id={:?}", id);
let cell = history_cell::AssistantMarkdownCell::new_with_id(source, id, &self.config);
self.add_to_history(cell);
}
/// Normalize text for duplicate detection (trim trailing whitespace and normalize newlines)
fn normalize_text(s: &str) -> String {
// 1) Normalize newlines
let s = s.replace("\r\n", "\n");
// 2) Trim trailing whitespace per line; collapse repeated blank lines
let mut out: Vec<String> = Vec::new();
let mut saw_blank = false;
for line in s.lines() {
// Replace common Unicode bullets with ASCII to stabilize equality checks
let line = line
.replace('\u{2022}', "-") // •
.replace('\u{25E6}', "-") // ◦
.replace('\u{2219}', "-"); // ∙
let trimmed = line.trim_end();
if trimmed.chars().all(|c| c.is_whitespace()) {
if !saw_blank { out.push(String::new()); }
saw_blank = true;
} else {
out.push(trimmed.to_string());
saw_blank = false;
}
}
// 3) Remove trailing blank lines
while out.last().is_some_and(|l| l.is_empty()) { out.pop(); }
out.join("\n")
}
pub(crate) fn toggle_reasoning_visibility(&mut self) {
// Track whether any reasoning cells are found and their new state
let mut has_reasoning_cells = false;
@@ -6434,18 +6705,27 @@ impl WidgetRef for &ChatWidget<'_> {
}
};
// Draw the symbol aligned with the first visible content row.
// Assistant cells render with 1-row top padding; if not scrolled past it,
// place the dot on the second row to align with the text.
// Draw the symbol anchored to the top of the message (not the viewport).
// "Top of the message" accounts for any intentional top padding per cell type.
// As you scroll past that anchor, the icon scrolls away with the message.
if gutter_area.width >= 2 {
let pad_adjust: u16 = match item.kind() {
crate::history_cell::HistoryCellType::Assistant => if skip_top == 0 { 1 } else { 0 },
// Anchor offset counted from the very start of the item's painted area
// to the first line of its content that the icon should align with.
let anchor_offset: u16 = match item.kind() {
// Assistant messages render with one row of top padding so that
// the content visually aligns; anchor to that second row.
crate::history_cell::HistoryCellType::Assistant => 1,
_ => 0,
};
let symbol_y = gutter_area.y.saturating_add(pad_adjust);
if symbol_y < gutter_area.y.saturating_add(gutter_area.height) {
let symbol_style = Style::default().fg(color).bg(gutter_bg);
buf.set_string(gutter_area.x, symbol_y, symbol, symbol_style);
// If we've scrolled past the anchor line, don't render the icon.
if skip_top <= anchor_offset {
let rel = anchor_offset - skip_top; // rows from current viewport top
let symbol_y = gutter_area.y.saturating_add(rel);
if symbol_y < gutter_area.y.saturating_add(gutter_area.height) {
let symbol_style = Style::default().fg(color).bg(gutter_bg);
buf.set_string(gutter_area.x, symbol_y, symbol, symbol_style);
}
}
}
}
codex-rs/tui/src/chatwidget/tests.rs
+160
View File
@@ -1104,6 +1104,110 @@ fn multiple_agent_messages_in_single_turn_emit_multiple_headers() {
assert!(first_idx < second_idx, "messages out of order: {combined}");
}
#[test]
fn two_final_answers_append_not_overwrite_when_no_deltas() {
// Directly exercise ChatWidget::insert_final_answer to validate we do not
// overwrite a prior finalized assistant message when a new final arrives
// without any streaming deltas (regression guard for overwrite bug).
let (mut chat, rx, _op_rx) = make_chatwidget_manual();
// First finalized assistant message (no streaming cell exists yet)
chat.insert_final_answer_with_id(None, Vec::new(), "First message".to_string());
// Second finalized assistant message (also without streaming)
chat.insert_final_answer_with_id(None, Vec::new(), "Second message".to_string());
// Drain any history insert side-effects (not strictly required here)
let _ = drain_insert_history(&rx);
// Verify via exported ResponseItems so we don't reach into private fields
let items = chat.export_response_items();
let assistants: Vec<String> = items
.into_iter()
.filter_map(|it| match it {
codex_protocol::models::ResponseItem::Message { role, content, .. } if role == "assistant" => {
let text = content.into_iter().filter_map(|c| match c {
codex_protocol::models::ContentItem::OutputText { text } => Some(text),
codex_protocol::models::ContentItem::InputText { text } => Some(text),
_ => None,
}).collect::<Vec<_>>().join("\n");
Some(text)
}
_ => None,
})
.collect();
assert_eq!(assistants.len(), 2, "expected two assistant messages, got {}", assistants.len());
assert!(assistants.iter().any(|s| s.contains("First message")), "missing first message");
assert!(assistants.iter().any(|s| s.contains("Second message")), "missing second message");
}
#[test]
fn second_final_that_is_superset_replaces_first() {
let (mut chat, rx, _op_rx) = make_chatwidget_manual();
let base = "• Behavior\n ◦ One\n\nNotes\n ◦ Alpha\n".to_string();
let extended = format!("{} ◦ Beta\n", base);
chat.insert_final_answer_with_id(None, Vec::new(), base);
chat.insert_final_answer_with_id(None, Vec::new(), extended.clone());
// Drain history events
let _ = drain_insert_history(&rx);
// Expect exactly one assistant message containing the extended text
let items = chat.export_response_items();
let assistants: Vec<String> = items
.into_iter()
.filter_map(|it| match it {
codex_protocol::models::ResponseItem::Message { role, content, .. } if role == "assistant" => {
let text = content.into_iter().filter_map(|c| match c {
codex_protocol::models::ContentItem::OutputText { text } => Some(text),
codex_protocol::models::ContentItem::InputText { text } => Some(text),
_ => None,
}).collect::<Vec<_>>().join("\n");
Some(text)
}
_ => None,
})
.collect();
assert_eq!(assistants.len(), 1, "expected single assistant after superset replace");
assert!(assistants[0].contains("Beta"), "expected extended content present");
}
#[test]
fn identical_content_with_unicode_bullets_dedupes() {
let (mut chat, rx, _op_rx) = make_chatwidget_manual();
let a = "• Behavior\n ◦ One\n\nNotes\n ◦ Alpha".to_string();
let b = "- Behavior\n - One\n\nNotes\n - Alpha".to_string();
// Two finals that only differ in bullet glyphs should dedupe into one cell
chat.insert_final_answer_with_id(None, Vec::new(), a);
chat.insert_final_answer_with_id(None, Vec::new(), b);
// Drain history events
let _ = drain_insert_history(&rx);
let items = chat.export_response_items();
let assistants: Vec<String> = items
.into_iter()
.filter_map(|it| match it {
codex_protocol::models::ResponseItem::Message { role, content, .. } if role == "assistant" => {
let text = content.into_iter().filter_map(|c| match c {
codex_protocol::models::ContentItem::OutputText { text } => Some(text),
codex_protocol::models::ContentItem::InputText { text } => Some(text),
_ => None,
}).collect::<Vec<_>>().join("\n");
Some(text)
}
_ => None,
})
.collect();
assert_eq!(assistants.len(), 1, "expected deduped single assistant cell");
}
#[test]
fn final_reasoning_then_message_without_deltas_are_rendered() {
let (mut chat, rx, _op_rx) = make_chatwidget_manual();
@@ -1192,6 +1296,62 @@ fn deltas_then_same_final_message_are_rendered_snapshot() {
assert_snapshot!(combined);
}
#[test]
fn late_final_does_not_duplicate_when_stream_finalized_early() {
use codex_core::protocol::*;
let (mut chat, rx, _op_rx) = make_chatwidget_manual();
// Stream some answer deltas for id "s1"
chat.handle_codex_event(Event {
id: "s1".into(),
msg: EventMsg::AgentMessageDelta(AgentMessageDeltaEvent {
delta: "What I Can Do\n".into(),
}),
});
chat.handle_codex_event(Event {
id: "s1".into(),
msg: EventMsg::AgentMessageDelta(AgentMessageDeltaEvent {
delta: "- Explore/Modify Code\n".into(),
}),
});
// Simulate a side event that forces finalization (e.g., tool start)
chat.finalize_active_stream();
// Now a late final AgentMessage arrives with the full text
let final_text = "What I Can Do\n- Explore/Modify Code\n".to_string();
chat.handle_codex_event(Event {
id: "s1".into(),
msg: EventMsg::AgentMessage(AgentMessageEvent {
message: final_text.clone(),
}),
});
// Drain any history insert side-effects (not strictly required here)
let _ = drain_insert_history(&rx);
// Export and assert that only a single assistant message exists
let items = chat.export_response_items();
let assistants: Vec<String> = items
.into_iter()
.filter_map(|it| match it {
codex_protocol::models::ResponseItem::Message { role, content, .. } if role == "assistant" => {
let text = content.into_iter().filter_map(|c| match c {
codex_protocol::models::ContentItem::OutputText { text } => Some(text),
codex_protocol::models::ContentItem::InputText { text } => Some(text),
_ => None,
}).collect::<Vec<_>>().join("\n");
Some(text)
}
_ => None,
})
.collect();
assert_eq!(assistants.len(), 1, "late final should replace, not duplicate");
assert!(assistants[0].contains("Explore/Modify Code"));
}
#[test]
fn streaming_answer_then_finalize_does_not_truncate() {
let (mut chat, rx, _op_rx) = make_chatwidget_manual();
codex-rs/tui/src/history_cell.rs
+1461 -365
View File
File diff suppressed because it is too large Load Diff
codex-rs/tui/src/markdown.rs
+61 -23
View File
@@ -64,7 +64,7 @@ fn append_markdown_with_opener_and_cwd_and_bold(
};
lines.extend(rendered);
}
Segment::Code { _lang, content } => {
Segment::Code { _lang, content, fenced } => {
// Use syntect-based syntax highlighting when available, preserving exact text.
let lang = _lang.as_deref();
// Apply a solid background and pad trailing spaces so the block forms
@@ -77,25 +77,46 @@ fn append_markdown_with_opener_and_cwd_and_bold(
.map(|l| l.spans.iter().map(|s| UnicodeWidthStr::width(s.content.as_ref())).sum::<usize>())
.max()
.unwrap_or(0);
// Add 1 space of horizontal padding on both sides for fenced blocks
let left_right_pad = if fenced { 1 } else { 0 };
let target_w = max_w + (left_right_pad * 2);
// Optional: add a blank row above for fenced code blocks
if fenced {
let top = Line::from(Span::styled(" ".repeat(target_w), Style::default().bg(code_bg)));
lines.push(top);
}
for l in highlighted.iter_mut() {
l.style = l.style.bg(code_bg);
// Apply background to all existing spans instead of the line,
// so the painted region matches our explicit padding width.
for sp in l.spans.iter_mut() {
sp.style = sp.style.bg(code_bg);
}
// Prepend left padding when requested
if left_right_pad > 0 {
l.spans.insert(0, Span::styled(" ", Style::default().bg(code_bg)));
}
let w: usize = l
.spans
.iter()
.map(|s| UnicodeWidthStr::width(s.content.as_ref()))
.sum();
if max_w > w {
let pad = " ".repeat(max_w - w);
if target_w > w {
let pad = " ".repeat(target_w - w);
l.spans.push(Span::styled(pad, Style::default().bg(code_bg)));
} else if w == 0 {
// Defensive: if this code block segment happens to have
// max_w == 0 (e.g., a split artifact), still paint the
// row with at least one space so the code block
// background shows consistently.
// Defensive: paint at least one cell so background shows
l.spans.push(Span::styled(" ", Style::default().bg(code_bg)));
}
}
lines.extend(highlighted);
// Optional: add a blank row below for fenced code blocks
if fenced {
let bottom = Line::from(Span::styled(" ".repeat(target_w), Style::default().bg(code_bg)));
lines.push(bottom);
}
}
}
}
@@ -160,6 +181,8 @@ enum Segment {
Code {
_lang: Option<String>,
content: String,
// true when originated from ```/~~~ fenced block; false for indented
fenced: bool,
},
}
@@ -282,6 +305,7 @@ fn split_text_and_fences(src: &str) -> Vec<Segment> {
segments.push(Segment::Code {
_lang: code_lang.take(),
content: code_content.clone(),
fenced: true,
});
code_content.clear();
code_mode = CodeMode::None;
@@ -307,6 +331,7 @@ fn split_text_and_fences(src: &str) -> Vec<Segment> {
segments.push(Segment::Code {
_lang: None,
content: code_content.clone(),
fenced: false,
});
code_content.clear();
code_mode = CodeMode::None;
@@ -324,6 +349,7 @@ fn split_text_and_fences(src: &str) -> Vec<Segment> {
segments.push(Segment::Code {
_lang: code_lang.take(),
content: code_content.clone(),
fenced: matches!(code_mode, CodeMode::Fenced),
});
} else if !curr_text.is_empty() {
segments.push(Segment::Text(curr_text.clone()));
@@ -409,14 +435,16 @@ mod tests {
.collect::<String>()
})
.collect();
assert_eq!(
rendered,
vec![
" indented".to_string(),
"\t\twith tabs".to_string(),
" four spaces".to_string()
]
);
// Expect: top padding row, the 3 code lines (each with a leading space and optional trailing spaces), bottom padding row.
assert_eq!(rendered.len(), 5, "expected fenced padding rows added: {:?}", rendered);
// Helpers: identify padded blank line and strip left pad+trailing spaces
let is_padded_blank = |s: &str| !s.is_empty() && s.trim().is_empty();
let strip_line = |s: &str| s.strip_prefix(' ').unwrap_or(s).trim_end().to_string();
assert!(is_padded_blank(&rendered[0]));
assert_eq!(strip_line(&rendered[1]), " indented");
assert_eq!(strip_line(&rendered[2]), "\t\twith tabs");
assert_eq!(strip_line(&rendered[3]), " four spaces");
assert!(is_padded_blank(&rendered[4]));
}
#[test]
@@ -434,10 +462,12 @@ mod tests {
.collect::<String>()
})
.collect();
// Expect first and last lines rewritten, middle line unchanged.
assert!(rendered[0].contains("vscode://file"));
assert_eq!(rendered[1], "Inside 【F:/x.rs†L2】");
assert!(matches!(rendered.last(), Some(s) if s.contains("vscode://file")));
// Expect first and last lines rewritten, and the interior fenced code line
// unchanged (but wrapped with left/right padding rows).
let strip_line = |s: &str| s.strip_prefix(' ').unwrap_or(s).trim_end().to_string();
assert!(rendered.first().is_some_and(|s| s.contains("vscode://file")));
assert!(rendered.iter().any(|s| strip_line(s) == "Inside 【F:/x.rs†L2】"));
assert!(rendered.last().is_some_and(|s| s.contains("vscode://file")));
}
#[test]
@@ -532,15 +562,23 @@ mod tests {
.map(|l| l.spans.iter().map(|s| s.content.clone()).collect::<String>())
.collect();
// There should be at least 7 lines: comment, fn greet header, format!, closing },
// the internal blank (spaces), fn main header, println!, closing }.
assert!(rendered.len() >= 7, "unexpected line count: {:?}", rendered);
// There should be at least 9 lines including top/bottom padding.
assert!(rendered.len() >= 9, "unexpected line count: {:?}", rendered);
// Find the internal blank: it should be a line consisting only of spaces
// (inserted by padding logic inside code block), not an actually empty string
// produced by the non-code renderer.
assert!(rendered.iter().any(|s| !s.is_empty() && s.trim().is_empty()),
"expected a space-padded blank line inside the code block, got: {:?}", rendered);
// Validate uniform rectangular width including padding rows
use unicode_width::UnicodeWidthStr;
let widths: Vec<usize> = out
.iter()
.map(|l| l.spans.iter().map(|s| UnicodeWidthStr::width(s.content.as_ref())).sum())
.collect();
let maxw = *widths.iter().max().unwrap_or(&0);
assert!(widths.iter().all(|w| *w == maxw), "all lines must be padded to same width: {:?}", widths);
}
#[test]
codex-rs/tui/src/markdown_renderer.rs
+547 -180
View File
File diff suppressed because it is too large Load Diff
codex-rs/tui/src/markdown_stream.rs
+11 -6
View File
@@ -95,12 +95,17 @@ impl MarkdownStreamCollector {
}
let mut complete_line_count = rendered.len();
if complete_line_count > 0
&& crate::render::line_utils::is_blank_line_spaces_only(
&rendered[complete_line_count - 1],
)
{
complete_line_count -= 1;
if complete_line_count > 0 {
let last = &rendered[complete_line_count - 1];
// Do not drop a trailing blank when it is part of a code block; that
// would cause the blank to be emitted later next to a previously
// committed plain blank separator, producing a visible double-gap
// (one painted, one unpainted).
let is_blank = crate::render::line_utils::is_blank_line_spaces_only(last);
let is_code_bg = crate::render::line_utils::is_code_block_painted(last);
if is_blank && !is_code_bg {
complete_line_count -= 1;
}
}
if !self.buffer.ends_with('\n') {
complete_line_count = complete_line_count.saturating_sub(1);
codex-rs/tui/src/render/line_utils.rs
+12
View File
@@ -46,3 +46,15 @@ pub fn is_blank_line_trim(line: &Line<'_>) -> bool {
}
line.spans.iter().all(|s| s.content.trim().is_empty())
}
/// Whether this line is painted with the code-block background color.
/// Used to distinguish a truly blank paragraph separator (no background)
/// from a blank line that is part of a code block (should not be dropped
/// during streaming commit logic).
pub fn is_code_block_painted(line: &Line<'_>) -> bool {
let code_bg = crate::colors::code_block_bg();
if line.style.bg == Some(code_bg) {
return true;
}
line.spans.iter().any(|s| s.style.bg == Some(code_bg))
}
codex-rs/tui/src/streaming/controller.rs
+94 -30
View File
@@ -10,8 +10,8 @@ use super::StreamState;
/// Sink for history insertions and animation control.
pub(crate) trait HistorySink {
fn insert_history(&self, lines: Vec<Line<'static>>);
fn insert_history_with_kind(&self, kind: StreamKind, lines: Vec<Line<'static>>);
fn insert_final_answer(&self, lines: Vec<Line<'static>>, full_markdown_source: String);
fn insert_history_with_kind(&self, id: Option<String>, kind: StreamKind, lines: Vec<Line<'static>>);
fn insert_final_answer(&self, id: Option<String>, lines: Vec<Line<'static>>, full_markdown_source: String);
fn start_commit_animation(&self);
fn stop_commit_animation(&self);
}
@@ -21,27 +21,19 @@ pub(crate) struct AppEventHistorySink(pub(crate) crate::app_event_sender::AppEve
impl HistorySink for AppEventHistorySink {
fn insert_history(&self, lines: Vec<Line<'static>>) {
tracing::debug!("insert_history called with {} lines:", lines.len());
for (i, line) in lines.iter().enumerate() {
let text: String = line.spans.iter().map(|s| s.content.to_string()).collect();
if i < 3 || i >= lines.len() - 1 {
tracing::debug!(" Line {}: {:?}", i, text);
} else if i == 3 {
tracing::debug!(" ... ({} more lines) ...", lines.len() - 4);
}
}
tracing::debug!("sink.insert_history lines={}", lines.len());
self.0
.send(crate::app_event::AppEvent::InsertHistory(lines))
}
fn insert_history_with_kind(&self, kind: StreamKind, lines: Vec<Line<'static>>) {
tracing::debug!("insert_history_with_kind({:?}) {} lines", kind, lines.len());
fn insert_history_with_kind(&self, id: Option<String>, kind: StreamKind, lines: Vec<Line<'static>>) {
tracing::debug!("sink.insert_history_with_kind kind={:?} id={:?} lines={}", kind, id, lines.len());
self.0
.send(crate::app_event::AppEvent::InsertHistoryWithKind { kind, lines })
.send(crate::app_event::AppEvent::InsertHistoryWithKind { id, kind, lines })
}
fn insert_final_answer(&self, lines: Vec<Line<'static>>, full_markdown_source: String) {
tracing::debug!("insert_final_answer {} lines ({} bytes source)", lines.len(), full_markdown_source.len());
fn insert_final_answer(&self, id: Option<String>, lines: Vec<Line<'static>>, full_markdown_source: String) {
tracing::debug!("sink.insert_final_answer id={:?} lines={} source_len={}", id, lines.len(), full_markdown_source.len());
self.0
.send(crate::app_event::AppEvent::InsertFinalAnswer { lines, source: full_markdown_source })
.send(crate::app_event::AppEvent::InsertFinalAnswer { id, lines, source: full_markdown_source })
}
fn start_commit_animation(&self) {
self.0
@@ -61,6 +53,7 @@ pub(crate) struct StreamController {
header: HeaderEmitter,
states: [StreamState; 2],
current_stream: Option<StreamKind>,
current_stream_id: Option<String>,
finishing_after_drain: bool,
thinking_placeholder_shown: bool,
}
@@ -72,6 +65,7 @@ impl StreamController {
header: HeaderEmitter::new(),
states: [StreamState::new_for_kind(StreamKind::Answer), StreamState::new_for_kind(StreamKind::Reasoning)],
current_stream: None,
current_stream_id: None,
finishing_after_drain: false,
thinking_placeholder_shown: false,
}
@@ -107,7 +101,11 @@ impl StreamController {
}
fn emit_header_if_needed(&mut self, kind: StreamKind, out_lines: &mut Lines) -> bool {
self.header.maybe_emit(kind, out_lines)
let emitted = self.header.maybe_emit(kind, out_lines);
if emitted {
tracing::debug!("stream: emitted header for {:?}", kind);
}
emitted
}
#[inline]
@@ -118,8 +116,14 @@ impl StreamController {
/// Begin a stream, flushing previously completed lines from any other
/// active stream to maintain ordering.
pub(crate) fn begin(&mut self, kind: StreamKind, sink: &impl HistorySink) {
tracing::debug!("begin called for {:?}, current_stream={:?}", kind, self.current_stream);
pub(crate) fn begin_with_id(&mut self, kind: StreamKind, id: Option<String>, sink: &impl HistorySink) {
tracing::debug!("stream.begin kind={:?} prev={:?} new_id={:?}", kind, self.current_stream, id);
// NOTE (dup‑guard): Historically we cleared `current_stream[_id]` even when
// `kind` did not change, which caused the active Answer stream to lose its id.
// Downstream, the UI could not find the streaming cell by id on finalization
// and appended a new Assistant cell (visible duplicate). Keep state when the
// kind is unchanged, and if the id changes mid‑stream, flush under the old id
// and adopt the new one so the final can match and replace in‑place.
if let Some(current) = self.current_stream {
if current != kind {
tracing::debug!("Switching from {:?} to {:?}, flushing previous", current, kind);
@@ -138,21 +142,53 @@ impl StreamController {
result
};
if !step.history.is_empty() {
tracing::debug!("Flushing {} lines from {:?} stream", step.history.len(), current);
tracing::debug!("stream.flush prev={:?} lines={}", current, step.history.len());
let mut lines: Lines = Vec::new();
self.emit_header_if_needed(current, &mut lines);
lines.extend(step.history);
// Don't add extra blank line - markdown renderer handles spacing
sink.insert_history_with_kind(current, lines);
sink.insert_history_with_kind(self.current_stream_id.clone(), current, lines);
}
// Only clear current stream tracking when actually switching kinds.
self.current_stream = None;
self.current_stream_id = None;
}
// If the kind is unchanged, we may still need to handle id transitions.
// If the incoming id differs from our current id, flush any buffered
// content under the old id and then adopt the new id so downstream
// finalize uses a matching identifier.
if current == kind {
if let Some(ref new_id) = id {
if self.current_stream_id.as_ref() != Some(new_id) {
let cfg = self.config.clone();
let step = {
let prev_state = self.state_mut(current);
let newly_completed = prev_state.collector.commit_complete_lines(&cfg);
if !newly_completed.is_empty() { prev_state.enqueue(newly_completed); }
let result = prev_state.drain_all();
tracing::debug!("Flushing {:?} due to id change {:?} -> {:?}", current, self.current_stream_id, id);
result
};
if !step.history.is_empty() {
let mut lines: Lines = Vec::new();
self.emit_header_if_needed(current, &mut lines);
lines.extend(step.history);
sink.insert_history_with_kind(self.current_stream_id.clone(), current, lines);
}
// Now adopt the new id; do not reset kind.
self.current_stream_id = Some(new_id.clone());
}
} else if self.current_stream_id.is_none() {
// If we previously had no id and a None arrives again, keep as None.
}
}
self.current_stream = None;
}
if self.current_stream != Some(kind) {
let prev = self.current_stream;
self.current_stream = Some(kind);
// Always adopt the provided id when starting a new stream
self.current_stream_id = id;
// Starting a new stream cancels any pending finish-from-previous-stream animation.
self.finishing_after_drain = false;
if prev.is_some() {
@@ -169,13 +205,18 @@ impl StreamController {
}
}
/// Backwards-compatible entry point without an id.
pub(crate) fn begin(&mut self, kind: StreamKind, sink: &impl HistorySink) {
self.begin_with_id(kind, None, sink);
}
/// Push a delta; if it contains a newline, commit completed lines and start animation.
pub(crate) fn push_and_maybe_commit(&mut self, delta: &str, sink: &impl HistorySink) {
let Some(kind) = self.current_stream else {
tracing::debug!("push_and_maybe_commit called but no current_stream");
return;
};
tracing::debug!("push_and_maybe_commit for {:?}, delta={:?}", kind, delta);
tracing::debug!("push_and_maybe_commit for {:?}, delta.len={} contains_nl={}", kind, delta.len(), delta.contains('\n'));
let cfg = self.config.clone();
// Check header flag before borrowing state (used only to avoid double headers)
@@ -216,6 +257,8 @@ impl StreamController {
// No per-span overrides needed for Answer: line FG is already bright.
styled.push(line);
}
let count = styled.len();
tracing::debug!("stream.commit {:?} newly_completed_lines={}", kind, count);
state.enqueue(styled);
sink.start_commit_animation();
}
@@ -302,7 +345,7 @@ impl StreamController {
// Build output regardless of whether out_lines is empty so we can still
// replace the streaming cell with a re-renderable final cell.
let mut lines_with_header: Lines = Vec::new();
let _emitted_header = self.emit_header_if_needed(kind, &mut lines_with_header);
let emitted_header = self.emit_header_if_needed(kind, &mut lines_with_header);
// Reduce leading blanks to at most one
let mut skip_count = 0;
while skip_count < out_lines.len()
@@ -332,9 +375,22 @@ impl StreamController {
if matches!(kind, StreamKind::Answer) {
// Use the source captured before draining so we don't lose content
// when the collector was cleared by finalize_and_drain.
sink.insert_final_answer(lines_with_header, full_source_before_drain);
tracing::debug!(
"stream.finalize ANSWER id={:?} header={} out_lines={} source_len={}",
self.current_stream_id,
emitted_header,
lines_with_header.len(),
full_source_before_drain.len()
);
sink.insert_final_answer(self.current_stream_id.clone(), lines_with_header, full_source_before_drain);
} else if !lines_with_header.is_empty() {
sink.insert_history_with_kind(kind, lines_with_header);
tracing::debug!(
"stream.finalize REASONING id={:?} header={} out_lines={}",
self.current_stream_id,
emitted_header,
lines_with_header.len()
);
sink.insert_history_with_kind(self.current_stream_id.clone(), kind, lines_with_header);
}
// Cleanup
@@ -344,7 +400,10 @@ impl StreamController {
// Also clear the per-stream emitted flag so the header can render again.
self.header.reset_for_stream(kind);
self.current_stream = None;
self.current_stream_id = None;
self.finishing_after_drain = false;
// Ensure any commit animation thread is stopped when we finalize immediately.
sink.stop_commit_animation();
true
} else {
if !remaining.is_empty() {
@@ -400,7 +459,7 @@ impl StreamController {
})
.collect();
out.extend(history);
sink.insert_history_with_kind(kind, out);
sink.insert_history_with_kind(self.current_stream_id.clone(), kind, out);
}
let is_idle = self.state(kind).is_idle();
@@ -414,6 +473,7 @@ impl StreamController {
// Also clear the per-stream emitted flag so the header can render again.
self.header.reset_for_stream(kind);
self.current_stream = None;
self.current_stream_id = None;
self.finishing_after_drain = false;
return true;
}
@@ -463,8 +523,12 @@ impl StreamController {
// Fall through to inject the message
}
// This is a new section (no deltas received)
self.begin(kind, sink);
// If no stream is active for this kind, begin a new section now.
// Otherwise, preserve the existing stream context (including id)
// so finalization can correctly target the matching cell.
if self.current_stream != Some(kind) {
self.begin(kind, sink);
}
{
let state = self.state_mut(kind);
codex-rs/tui/src/syntax_highlight.rs
+146 -7
View File
@@ -72,13 +72,16 @@ fn default_theme<'a>() -> &'a Theme {
.unwrap_or_else(|| ts.themes.values().next().expect("at least one syntect theme"))
}
fn syntax_for_lang<'a>(ps: &'a SyntaxSet, lang: &str) -> &'a SyntaxReference {
// Try token, then extension, then name; fall back to plain text.
fn try_syntax_for_lang<'a>(ps: &'a SyntaxSet, lang: &str) -> Option<&'a SyntaxReference> {
// Try token, then extension, then name; return None if not found.
let lang = normalize_lang(lang);
ps.find_syntax_by_token(lang)
.or_else(|| ps.find_syntax_by_extension(lang))
.or_else(|| ps.find_syntax_by_name(lang))
.unwrap_or_else(|| ps.find_syntax_plain_text())
}
fn syntax_for_lang<'a>(ps: &'a SyntaxSet, lang: &str) -> &'a SyntaxReference {
try_syntax_for_lang(ps, lang).unwrap_or_else(|| ps.find_syntax_plain_text())
}
fn span_from_syn((SynStyle { foreground, font_style, .. }, text): (SynStyle, &str)) -> Span<'static> {
@@ -104,10 +107,26 @@ fn span_from_syn((SynStyle { foreground, font_style, .. }, text): (SynStyle, &st
pub(crate) fn highlight_code_block(content: &str, lang: Option<&str>) -> Vec<Line<'static>> {
let ps = syntax_set();
let theme = default_theme();
// Resolve language if provided
let syntax = lang
.map(|l| syntax_for_lang(ps, l))
.unwrap_or_else(|| ps.find_syntax_plain_text());
// Resolve language
let syntax = {
// Prefer the provided language when it maps to a known syntax.
if let Some(l) = lang.and_then(|l| if l.trim().is_empty() { None } else { Some(l) }) {
if let Some(s) = try_syntax_for_lang(ps, l) {
s
} else {
// Unknown language label: try a best-effort auto-detect before
// falling back to plain text.
autodetect_lang(content)
.map(|dl| syntax_for_lang(ps, dl))
.unwrap_or_else(|| ps.find_syntax_plain_text())
}
} else {
// No label provided: attempt auto-detection, otherwise plain text.
autodetect_lang(content)
.map(|dl| syntax_for_lang(ps, dl))
.unwrap_or_else(|| ps.find_syntax_plain_text())
}
};
let mut highlighter = HighlightLines::new(syntax, theme);
let mut out: Vec<Line<'static>> = Vec::new();
@@ -245,3 +264,123 @@ fn normalize_lang(lang: &str) -> &str {
}
// Theme cycling removed; we always use Solarized matching UI brightness.
// --- Lightweight language auto-detection ---
// Heuristics only; must be fast and avoid extra deps.
fn autodetect_lang(content: &str) -> Option<&'static str> {
let s = content.trim_start();
if s.is_empty() {
return None;
}
// 1) Shebang on first line
if let Some(first) = s.lines().next() {
if let Some(rest) = first.strip_prefix("#!") {
let l = rest.to_ascii_lowercase();
if l.contains("bash") || l.contains("sh") || l.contains("zsh") {
return Some("bash");
}
if l.contains("python") {
return Some("python");
}
if l.contains("node") || l.contains("deno") {
return Some("javascript");
}
if l.contains("ruby") {
return Some("ruby");
}
}
}
// Narrow to a small sample for sniffing
let sample: String = s.lines().take(24).collect::<Vec<_>>().join("\n");
let lower = sample.to_ascii_lowercase();
let trimmed_all = s.trim();
// 2) Diff/patch
if trimmed_all.starts_with("diff --git ")
|| sample.lines().take(6).any(|l| l.starts_with("--- "))
&& sample.lines().take(6).any(|l| l.starts_with("+++ "))
{
return Some("diff");
}
// 3) JSON (parse to be certain when it looks like JSON)
if (trimmed_all.starts_with('{') && trimmed_all.ends_with('}'))
|| (trimmed_all.starts_with('[') && trimmed_all.ends_with(']'))
{
if serde_json::from_str::<serde_json::Value>(trimmed_all).is_ok() {
return Some("json");
}
}
// 4) HTML/XML
if trimmed_all.starts_with('<') && trimmed_all.contains('>') {
let l = lower.trim_start();
if l.starts_with("<!doctype html") || l.starts_with("<html") || l.starts_with("<head") || l.starts_with("<body") {
return Some("html");
}
if l.starts_with("<?xml") || l.starts_with("<svg") {
return Some("xml");
}
}
// 5) Rust (common keywords + braces)
let rust_hits = ["fn ", "let ", "mod ", "impl ", "use ", "pub "]
.iter()
.filter(|k| lower.contains(**k))
.count();
if rust_hits >= 2 && sample.contains('{') {
return Some("rust");
}
// 6) Python (def/class/import and minimal punctuation typical of Python)
if (lower.contains("def ") || lower.contains("class ") || lower.contains("import "))
&& !sample.contains(";")
{
return Some("python");
}
// 7) Shell-ish commands (very loose; prefer only when signals are clear)
if sample.lines().take(8).any(|l| l.trim_start().starts_with("$ "))
|| sample.contains(" && ")
|| sample.contains(" | ")
|| sample.lines().take(8).any(|l| l.trim_start().starts_with("echo "))
{
return Some("bash");
}
// 8) SQL
if ["select ", "insert ", "update ", "delete ", "create ", "alter ", "drop "]
.iter()
.any(|kw| lower.contains(*kw))
&& sample.contains(';')
{
return Some("sql");
}
// 9) INI / TOML / YAML (rough heuristics)
if let Some(first_line) = s.lines().find(|l| !l.trim().is_empty()) {
let fl = first_line.trim();
if fl.starts_with('[') && fl.ends_with(']') {
if s.contains("[[") || s.contains("]]") || s.contains("=\"") {
return Some("toml");
}
if s.lines().any(|l| l.contains('=')) && !s.contains(": ") {
return Some("ini");
}
}
}
let yaml_signals = sample
.lines()
.map(|l| l.trim())
.filter(|l| !l.is_empty())
.take(20)
.filter(|l| l.starts_with("- ") || (l.contains(':') && !l.contains("://") && !l.contains("::") && !l.contains(":=")))
.count();
if yaml_signals >= 3 {
return Some("yaml");
}
None
}