mirror of
https://github.com/hanzoai/dev.git
synced 2026-10-17 15:34:54 +00:00
- Applied cargo fmt to fix code formatting - Fixed trailing whitespace issues - Updated colors to use theme system - Fixed merge-related syntax errors - Recreated build-fast.sh for monorepo structure
4388 lines
167 KiB
Rust
4388 lines
167 KiB
Rust
// Poisoned mutex should fail the program
|
||
#![allow(clippy::unwrap_used)]
|
||
|
||
use std::borrow::Cow;
|
||
use std::collections::HashMap;
|
||
use std::collections::HashSet;
|
||
use std::path::Path;
|
||
use std::path::PathBuf;
|
||
use std::sync::Arc;
|
||
use std::sync::Mutex;
|
||
use std::sync::atomic::AtomicU64;
|
||
use std::time::Duration;
|
||
|
||
use async_channel::Receiver;
|
||
use async_channel::Sender;
|
||
use base64::Engine;
|
||
use codex_apply_patch::ApplyPatchAction;
|
||
use codex_apply_patch::MaybeApplyPatchVerified;
|
||
use codex_apply_patch::maybe_parse_apply_patch_verified;
|
||
use codex_login::CodexAuth;
|
||
use futures::prelude::*;
|
||
use mcp_types::CallToolResult;
|
||
use serde::Serialize;
|
||
use serde_json;
|
||
use tokio::sync::oneshot;
|
||
use tokio::task::AbortHandle;
|
||
use tracing::debug;
|
||
use tracing::error;
|
||
use tracing::info;
|
||
use tracing::trace;
|
||
use tracing::warn;
|
||
use uuid::Uuid;
|
||
|
||
/// Gather ephemeral, per-turn context that should not be persisted to history.
|
||
/// Combines environment info and (when enabled) a live browser snapshot and status.
|
||
struct EphemeralJar {
|
||
items: Vec<ResponseItem>,
|
||
}
|
||
|
||
impl EphemeralJar {
|
||
fn new() -> Self {
|
||
Self { items: Vec::new() }
|
||
}
|
||
|
||
fn into_items(self) -> Vec<ResponseItem> {
|
||
self.items
|
||
}
|
||
}
|
||
|
||
fn get_git_branch(cwd: &std::path::Path) -> Option<String> {
|
||
let head_path = cwd.join(".git/HEAD");
|
||
if let Ok(contents) = std::fs::read_to_string(&head_path) {
|
||
if let Some(rest) = contents.trim().strip_prefix("ref: ") {
|
||
if let Some(branch) = rest.trim().rsplit('/').next() {
|
||
return Some(branch.to_string());
|
||
}
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
async fn build_turn_status_items(sess: &Session) -> Vec<ResponseItem> {
|
||
let mut jar = EphemeralJar::new();
|
||
|
||
// Collect environment context
|
||
let cwd = sess.cwd.to_string_lossy().to_string();
|
||
let branch = get_git_branch(&sess.cwd).unwrap_or_else(|| "unknown".to_string());
|
||
let reasoning_effort = sess.client.get_reasoning_effort();
|
||
|
||
// Build current system status
|
||
let mut current_status = format!(
|
||
r#"== System Status ==
|
||
[automatic message added by system]
|
||
|
||
cwd: {}
|
||
branch: {}
|
||
reasoning: {:?}"#,
|
||
cwd, branch, reasoning_effort
|
||
);
|
||
|
||
// Prepare browser context + optional screenshot
|
||
let mut screenshot_content: Option<ContentItem> = None;
|
||
let mut include_screenshot = false;
|
||
|
||
if let Some(browser_manager) = codex_browser::global::get_browser_manager().await {
|
||
if browser_manager.is_enabled().await {
|
||
// Get current URL and browser info
|
||
let url = browser_manager
|
||
.get_current_url()
|
||
.await
|
||
.unwrap_or_else(|| "unknown".to_string());
|
||
|
||
// Try to get a tab title if available
|
||
let title = match browser_manager.get_or_create_page().await {
|
||
Ok(page) => page.get_title().await,
|
||
Err(_) => None,
|
||
};
|
||
|
||
// Get browser type description
|
||
let browser_type = browser_manager.get_browser_type().await;
|
||
|
||
// Get viewport dimensions
|
||
let (viewport_width, viewport_height) = browser_manager.get_viewport_size().await;
|
||
let viewport_info = format!(" | Viewport: {}x{}", viewport_width, viewport_height);
|
||
|
||
// Get cursor position
|
||
let cursor_info = match browser_manager.get_cursor_position().await {
|
||
Ok((x, y)) => format!(
|
||
" | Mouse position: ({:.0}, {:.0}) [shown as a blue cursor in the screenshot]",
|
||
x, y
|
||
),
|
||
Err(_) => String::new(),
|
||
};
|
||
|
||
// Try to capture screenshot and compare with last one
|
||
let screenshot_status = match capture_browser_screenshot(sess).await {
|
||
Ok((screenshot_path, _url)) => {
|
||
// Check if screenshot has changed using image hashing
|
||
let mut last_screenshot_info = sess.last_screenshot_info.lock().unwrap();
|
||
|
||
// Compute hash for current screenshot
|
||
let current_hash =
|
||
crate::image_comparison::compute_image_hash(&screenshot_path).ok();
|
||
|
||
let should_include_screenshot = if let (
|
||
Some((_last_path, last_phash, last_dhash)),
|
||
Some((cur_phash, cur_dhash)),
|
||
) =
|
||
(last_screenshot_info.as_ref(), current_hash.as_ref())
|
||
{
|
||
// Compare hashes to see if screenshots are similar
|
||
let similar = crate::image_comparison::are_hashes_similar(
|
||
last_phash, last_dhash, cur_phash, cur_dhash,
|
||
);
|
||
|
||
if !similar {
|
||
// Screenshot has changed, include it
|
||
*last_screenshot_info = Some((
|
||
screenshot_path.clone(),
|
||
cur_phash.clone(),
|
||
cur_dhash.clone(),
|
||
));
|
||
true
|
||
} else {
|
||
// Screenshot unchanged
|
||
false
|
||
}
|
||
} else {
|
||
// No previous screenshot or hash computation failed, include it
|
||
if let Some((phash, dhash)) = current_hash {
|
||
*last_screenshot_info = Some((screenshot_path.clone(), phash, dhash));
|
||
}
|
||
true
|
||
};
|
||
|
||
if should_include_screenshot {
|
||
if let Ok(bytes) = std::fs::read(&screenshot_path) {
|
||
let mime = mime_guess::from_path(&screenshot_path)
|
||
.first()
|
||
.map(|m| m.to_string())
|
||
.unwrap_or_else(|| "image/png".to_string());
|
||
let encoded = base64::engine::general_purpose::STANDARD.encode(bytes);
|
||
screenshot_content = Some(ContentItem::InputImage {
|
||
image_url: format!("data:{mime};base64,{encoded}"),
|
||
detail: Some("high".to_string()),
|
||
});
|
||
include_screenshot = true;
|
||
""
|
||
} else {
|
||
" [Screenshot file read failed]"
|
||
}
|
||
} else {
|
||
" [Screenshot unchanged]"
|
||
}
|
||
}
|
||
Err(err_msg) => {
|
||
// Include error message so LLM knows screenshot failed
|
||
format!(" [Screenshot unavailable: {}]", err_msg).leak()
|
||
}
|
||
};
|
||
|
||
let status_line = if let Some(t) = title {
|
||
format!(
|
||
"Browser url: {} — {} ({}){}{}{}. You can interact with it using browser_* tools.",
|
||
url, t, browser_type, viewport_info, cursor_info, screenshot_status
|
||
)
|
||
} else {
|
||
format!(
|
||
"Browser url: {} ({}){}{}{}. You can interact with it using browser_* tools.",
|
||
url, browser_type, viewport_info, cursor_info, screenshot_status
|
||
)
|
||
};
|
||
current_status.push_str("\n");
|
||
current_status.push_str(&status_line);
|
||
}
|
||
}
|
||
|
||
// Check if system status has changed
|
||
let mut last_status = sess.last_system_status.lock().unwrap();
|
||
let status_changed = last_status.as_ref() != Some(¤t_status);
|
||
|
||
if status_changed {
|
||
// Update last status
|
||
*last_status = Some(current_status.clone());
|
||
}
|
||
|
||
// Only include items if something has changed or is new
|
||
let mut content: Vec<ContentItem> = Vec::new();
|
||
|
||
if status_changed {
|
||
content.push(ContentItem::InputText {
|
||
text: current_status,
|
||
});
|
||
}
|
||
|
||
if include_screenshot {
|
||
if let Some(image) = screenshot_content {
|
||
content.push(image);
|
||
}
|
||
}
|
||
|
||
if !content.is_empty() {
|
||
jar.items.push(ResponseItem::Message {
|
||
id: None,
|
||
role: "user".to_string(),
|
||
content,
|
||
});
|
||
}
|
||
|
||
jar.into_items()
|
||
}
|
||
use crate::agent_tool::AGENT_MANAGER;
|
||
use crate::agent_tool::AgentStatus;
|
||
use crate::agent_tool::CancelAgentParams;
|
||
use crate::agent_tool::CheckAgentStatusParams;
|
||
use crate::agent_tool::GetAgentResultParams;
|
||
use crate::agent_tool::ListAgentsParams;
|
||
use crate::agent_tool::RunAgentParams;
|
||
use crate::agent_tool::WaitForAgentParams;
|
||
use crate::apply_patch::ApplyPatchExec;
|
||
use crate::apply_patch::CODEX_APPLY_PATCH_ARG1;
|
||
use crate::apply_patch::InternalApplyPatchInvocation;
|
||
use crate::apply_patch::convert_apply_patch_to_protocol;
|
||
use crate::apply_patch::get_writable_roots;
|
||
use crate::apply_patch::{self};
|
||
use crate::client::ModelClient;
|
||
use crate::client_common::EnvironmentContext;
|
||
use crate::client_common::Prompt;
|
||
use crate::client_common::ResponseEvent;
|
||
use crate::config::Config;
|
||
use crate::config_types::ShellEnvironmentPolicy;
|
||
use crate::conversation_history::ConversationHistory;
|
||
use crate::error::CodexErr;
|
||
use crate::error::Result as CodexResult;
|
||
use crate::error::SandboxErr;
|
||
use crate::error::get_error_message_ui;
|
||
use crate::exec::ExecParams;
|
||
use crate::exec::ExecToolCallOutput;
|
||
use crate::exec::SandboxType;
|
||
use crate::exec::StdoutStream;
|
||
use crate::exec::StreamOutput;
|
||
use crate::exec::process_exec_tool_call;
|
||
use crate::exec_env::create_env;
|
||
use crate::mcp_connection_manager::McpConnectionManager;
|
||
use crate::mcp_tool_call::handle_mcp_tool_call;
|
||
use crate::models::ContentItem;
|
||
use crate::models::FunctionCallOutputPayload;
|
||
use crate::models::LocalShellAction;
|
||
use crate::models::ReasoningItemContent;
|
||
use crate::models::ReasoningItemReasoningSummary;
|
||
use crate::models::ResponseInputItem;
|
||
use crate::models::ResponseItem;
|
||
use crate::models::ShellToolCallParams;
|
||
use crate::openai_tools::ToolsConfig;
|
||
use crate::openai_tools::get_openai_tools;
|
||
use crate::parse_command::parse_command;
|
||
use crate::plan_tool::handle_update_plan;
|
||
use crate::project_doc::get_user_instructions;
|
||
use crate::protocol::AgentMessageDeltaEvent;
|
||
use crate::protocol::AgentMessageEvent;
|
||
use crate::protocol::AgentReasoningDeltaEvent;
|
||
use crate::protocol::AgentReasoningEvent;
|
||
use crate::protocol::AgentReasoningRawContentDeltaEvent;
|
||
use crate::protocol::AgentReasoningRawContentEvent;
|
||
use crate::protocol::AgentReasoningSectionBreakEvent;
|
||
use crate::protocol::AgentStatusUpdateEvent;
|
||
use crate::protocol::ApplyPatchApprovalRequestEvent;
|
||
use crate::protocol::AskForApproval;
|
||
use crate::protocol::BackgroundEventEvent;
|
||
use crate::protocol::BrowserScreenshotUpdateEvent;
|
||
use crate::protocol::ErrorEvent;
|
||
use crate::protocol::Event;
|
||
use crate::protocol::EventMsg;
|
||
use crate::protocol::ExecApprovalRequestEvent;
|
||
use crate::protocol::ExecCommandBeginEvent;
|
||
use crate::protocol::ExecCommandEndEvent;
|
||
use crate::protocol::FileChange;
|
||
use crate::protocol::InputItem;
|
||
use crate::protocol::Op;
|
||
use crate::protocol::PatchApplyBeginEvent;
|
||
use crate::protocol::PatchApplyEndEvent;
|
||
use crate::protocol::ReviewDecision;
|
||
use crate::protocol::SandboxPolicy;
|
||
use crate::protocol::SessionConfiguredEvent;
|
||
use crate::protocol::Submission;
|
||
use crate::protocol::TaskCompleteEvent;
|
||
use crate::protocol::TurnDiffEvent;
|
||
use crate::rollout::RolloutRecorder;
|
||
use crate::safety::SafetyCheck;
|
||
use crate::safety::assess_command_safety;
|
||
use crate::safety::assess_safety_for_untrusted_command;
|
||
use crate::shell;
|
||
use crate::turn_diff_tracker::TurnDiffTracker;
|
||
use crate::user_notification::UserNotification;
|
||
use crate::util::backoff;
|
||
use serde_json::Value;
|
||
|
||
/// The high-level interface to the Codex system.
|
||
/// It operates as a queue pair where you send submissions and receive events.
|
||
pub struct Codex {
|
||
next_id: AtomicU64,
|
||
tx_sub: Sender<Submission>,
|
||
rx_event: Receiver<Event>,
|
||
}
|
||
|
||
/// Wrapper returned by [`Codex::spawn`] containing the spawned [`Codex`],
|
||
/// the submission id for the initial `ConfigureSession` request and the
|
||
/// unique session id.
|
||
pub struct CodexSpawnOk {
|
||
pub codex: Codex,
|
||
pub init_id: String,
|
||
pub session_id: Uuid,
|
||
}
|
||
|
||
impl Codex {
|
||
/// Spawn a new [`Codex`] and initialize the session.
|
||
pub async fn spawn(config: Config, auth: Option<CodexAuth>) -> CodexResult<CodexSpawnOk> {
|
||
// experimental resume path (undocumented)
|
||
let resume_path = config.experimental_resume.clone();
|
||
info!("resume_path: {resume_path:?}");
|
||
let (tx_sub, rx_sub) = async_channel::bounded(64);
|
||
let (tx_event, rx_event) = async_channel::unbounded();
|
||
|
||
let user_instructions = get_user_instructions(&config).await;
|
||
|
||
let configure_session = Op::ConfigureSession {
|
||
provider: config.model_provider.clone(),
|
||
model: config.model.clone(),
|
||
model_reasoning_effort: config.model_reasoning_effort,
|
||
model_reasoning_summary: config.model_reasoning_summary,
|
||
user_instructions,
|
||
base_instructions: config.base_instructions.clone(),
|
||
approval_policy: config.approval_policy,
|
||
sandbox_policy: config.sandbox_policy.clone(),
|
||
disable_response_storage: config.disable_response_storage,
|
||
notify: config.notify.clone(),
|
||
cwd: config.cwd.clone(),
|
||
resume_path: resume_path.clone(),
|
||
};
|
||
|
||
let config = Arc::new(config);
|
||
|
||
// Generate a unique ID for the lifetime of this Codex session.
|
||
let session_id = Uuid::new_v4();
|
||
|
||
// This task will run until Op::Shutdown is received.
|
||
tokio::spawn(submission_loop(session_id, config, auth, rx_sub, tx_event));
|
||
let codex = Codex {
|
||
next_id: AtomicU64::new(0),
|
||
tx_sub,
|
||
rx_event,
|
||
};
|
||
let init_id = codex.submit(configure_session).await?;
|
||
|
||
Ok(CodexSpawnOk {
|
||
codex,
|
||
init_id,
|
||
session_id,
|
||
})
|
||
}
|
||
|
||
/// Submit the `op` wrapped in a `Submission` with a unique ID.
|
||
pub async fn submit(&self, op: Op) -> CodexResult<String> {
|
||
let id = self
|
||
.next_id
|
||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
|
||
.to_string();
|
||
let sub = Submission { id: id.clone(), op };
|
||
self.submit_with_id(sub).await?;
|
||
Ok(id)
|
||
}
|
||
|
||
/// Use sparingly: prefer `submit()` so Codex is responsible for generating
|
||
/// unique IDs for each submission.
|
||
pub async fn submit_with_id(&self, sub: Submission) -> CodexResult<()> {
|
||
self.tx_sub
|
||
.send(sub)
|
||
.await
|
||
.map_err(|_| CodexErr::InternalAgentDied)?;
|
||
Ok(())
|
||
}
|
||
|
||
pub async fn next_event(&self) -> CodexResult<Event> {
|
||
let event = self
|
||
.rx_event
|
||
.recv()
|
||
.await
|
||
.map_err(|_| CodexErr::InternalAgentDied)?;
|
||
Ok(event)
|
||
}
|
||
}
|
||
|
||
/// Mutable state of the agent
|
||
#[derive(Default)]
|
||
struct State {
|
||
approved_commands: HashSet<Vec<String>>,
|
||
current_agent: Option<AgentAgent>,
|
||
pending_approvals: HashMap<String, oneshot::Sender<ReviewDecision>>,
|
||
pending_input: Vec<ResponseInputItem>,
|
||
history: ConversationHistory,
|
||
}
|
||
|
||
/// Context for an initialized model agent
|
||
///
|
||
/// A session has at most 1 running agent at a time, and can be interrupted by user input.
|
||
pub(crate) struct Session {
|
||
client: ModelClient,
|
||
tx_event: Sender<Event>,
|
||
|
||
/// The session's current working directory. All relative paths provided by
|
||
/// the model as well as sandbox policies are resolved against this path
|
||
/// instead of `std::env::current_dir()`.
|
||
cwd: PathBuf,
|
||
base_instructions: Option<String>,
|
||
user_instructions: Option<String>,
|
||
approval_policy: AskForApproval,
|
||
sandbox_policy: SandboxPolicy,
|
||
shell_environment_policy: ShellEnvironmentPolicy,
|
||
writable_roots: Vec<PathBuf>,
|
||
disable_response_storage: bool,
|
||
tools_config: ToolsConfig,
|
||
|
||
/// Manager for external MCP servers/tools.
|
||
mcp_connection_manager: McpConnectionManager,
|
||
|
||
/// Configuration for available agent models
|
||
agents: Vec<crate::config_types::AgentConfig>,
|
||
|
||
/// External notifier command (will be passed as args to exec()). When
|
||
/// `None` this feature is disabled.
|
||
notify: Option<Vec<String>>,
|
||
|
||
/// Optional rollout recorder for persisting the conversation transcript so
|
||
/// sessions can be replayed or inspected later.
|
||
rollout: Mutex<Option<RolloutRecorder>>,
|
||
state: Mutex<State>,
|
||
codex_linux_sandbox_exe: Option<PathBuf>,
|
||
user_shell: shell::Shell,
|
||
show_raw_agent_reasoning: bool,
|
||
/// Pending browser screenshots to include in the next model request
|
||
#[allow(dead_code)]
|
||
pending_browser_screenshots: Mutex<Vec<PathBuf>>,
|
||
/// Track the last system status to detect changes
|
||
last_system_status: Mutex<Option<String>>,
|
||
/// Track the last screenshot path and hash to detect changes
|
||
last_screenshot_info: Mutex<Option<(PathBuf, Vec<u8>, Vec<u8>)>>, // (path, phash, dhash)
|
||
}
|
||
|
||
impl Session {
|
||
pub(crate) fn get_writable_roots(&self) -> &[PathBuf] {
|
||
&self.writable_roots
|
||
}
|
||
|
||
pub(crate) fn get_approval_policy(&self) -> AskForApproval {
|
||
self.approval_policy
|
||
}
|
||
|
||
pub(crate) fn get_cwd(&self) -> &Path {
|
||
&self.cwd
|
||
}
|
||
|
||
fn resolve_path(&self, path: Option<String>) -> PathBuf {
|
||
path.as_ref()
|
||
.map(PathBuf::from)
|
||
.map_or_else(|| self.cwd.clone(), |p| self.cwd.join(p))
|
||
}
|
||
}
|
||
|
||
impl Session {
|
||
pub fn set_agent(&self, agent: AgentAgent) {
|
||
let mut state = self.state.lock().unwrap();
|
||
if let Some(current_agent) = state.current_agent.take() {
|
||
current_agent.abort();
|
||
}
|
||
state.current_agent = Some(agent);
|
||
}
|
||
|
||
pub fn remove_agent(&self, sub_id: &str) {
|
||
let mut state = self.state.lock().unwrap();
|
||
if let Some(agent) = &state.current_agent {
|
||
if agent.sub_id == sub_id {
|
||
state.current_agent.take();
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Sends the given event to the client and swallows the send event, if
|
||
/// any, logging it as an error.
|
||
pub(crate) async fn send_event(&self, event: Event) {
|
||
if let Err(e) = self.tx_event.send(event).await {
|
||
error!("failed to send tool call event: {e}");
|
||
}
|
||
}
|
||
|
||
pub async fn request_command_approval(
|
||
&self,
|
||
sub_id: String,
|
||
call_id: String,
|
||
command: Vec<String>,
|
||
cwd: PathBuf,
|
||
reason: Option<String>,
|
||
) -> oneshot::Receiver<ReviewDecision> {
|
||
let (tx_approve, rx_approve) = oneshot::channel();
|
||
let event = Event {
|
||
id: sub_id.clone(),
|
||
msg: EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent {
|
||
call_id,
|
||
command,
|
||
cwd,
|
||
reason,
|
||
}),
|
||
};
|
||
let _ = self.tx_event.send(event).await;
|
||
{
|
||
let mut state = self.state.lock().unwrap();
|
||
state.pending_approvals.insert(sub_id, tx_approve);
|
||
}
|
||
rx_approve
|
||
}
|
||
|
||
pub async fn request_patch_approval(
|
||
&self,
|
||
sub_id: String,
|
||
call_id: String,
|
||
action: &ApplyPatchAction,
|
||
reason: Option<String>,
|
||
grant_root: Option<PathBuf>,
|
||
) -> oneshot::Receiver<ReviewDecision> {
|
||
let (tx_approve, rx_approve) = oneshot::channel();
|
||
let event = Event {
|
||
id: sub_id.clone(),
|
||
msg: EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent {
|
||
call_id,
|
||
changes: convert_apply_patch_to_protocol(action),
|
||
reason,
|
||
grant_root,
|
||
}),
|
||
};
|
||
let _ = self.tx_event.send(event).await;
|
||
{
|
||
let mut state = self.state.lock().unwrap();
|
||
state.pending_approvals.insert(sub_id, tx_approve);
|
||
}
|
||
rx_approve
|
||
}
|
||
|
||
pub fn notify_approval(&self, sub_id: &str, decision: ReviewDecision) {
|
||
let mut state = self.state.lock().unwrap();
|
||
if let Some(tx_approve) = state.pending_approvals.remove(sub_id) {
|
||
tx_approve.send(decision).ok();
|
||
}
|
||
}
|
||
|
||
pub fn add_approved_command(&self, cmd: Vec<String>) {
|
||
let mut state = self.state.lock().unwrap();
|
||
state.approved_commands.insert(cmd);
|
||
}
|
||
|
||
/// Records items to both the rollout and the chat completions/ZDR
|
||
/// transcript, if enabled.
|
||
async fn record_conversation_items(&self, items: &[ResponseItem]) {
|
||
debug!("Recording items for conversation: {items:?}");
|
||
self.record_state_snapshot(items).await;
|
||
|
||
self.state.lock().unwrap().history.record_items(items);
|
||
}
|
||
|
||
/// Clean up old screenshots and system status messages from conversation history
|
||
/// This is called when a new user message arrives to keep history manageable
|
||
async fn cleanup_old_status_items(&self) {
|
||
let mut state = self.state.lock().unwrap();
|
||
|
||
// Get current history items
|
||
let current_items = state.history.contents();
|
||
|
||
// Track various message types and their positions
|
||
let mut real_user_messages = Vec::new(); // Non-status user messages
|
||
let mut status_messages = Vec::new(); // Messages with screenshots or status
|
||
|
||
for (idx, item) in current_items.iter().enumerate() {
|
||
match item {
|
||
ResponseItem::Message { role, content, .. } if role == "user" => {
|
||
// Check message content
|
||
let has_status = content.iter().any(|c| {
|
||
if let ContentItem::InputText { text } = c {
|
||
text.contains("== System Status ==")
|
||
|| text.contains("Current working directory:")
|
||
|| text.contains("Git branch:")
|
||
} else {
|
||
false
|
||
}
|
||
});
|
||
|
||
let has_screenshot = content
|
||
.iter()
|
||
.any(|c| matches!(c, ContentItem::InputImage { .. }));
|
||
|
||
let has_real_text = content.iter().any(|c| {
|
||
if let ContentItem::InputText { text } = c {
|
||
// Real user text doesn't contain system status markers
|
||
!text.contains("== System Status ==")
|
||
&& !text.contains("Current working directory:")
|
||
&& !text.contains("Git branch:")
|
||
&& !text.trim().is_empty()
|
||
} else {
|
||
false
|
||
}
|
||
});
|
||
|
||
if has_real_text && !has_status && !has_screenshot {
|
||
// This is a real user message
|
||
real_user_messages.push(idx);
|
||
} else if has_status || has_screenshot {
|
||
// This is a status/screenshot message
|
||
status_messages.push(idx);
|
||
}
|
||
}
|
||
_ => {}
|
||
}
|
||
}
|
||
|
||
// Find screenshots to keep: last 2 that directly follow real user commands
|
||
let mut screenshots_to_keep = std::collections::HashSet::new();
|
||
|
||
// Work backwards through real user messages
|
||
for &user_idx in real_user_messages.iter().rev().take(2) {
|
||
// Find the first status message after this user message
|
||
for &status_idx in status_messages.iter() {
|
||
if status_idx > user_idx {
|
||
// Check if this status message contains a screenshot
|
||
if let Some(ResponseItem::Message { content, .. }) =
|
||
current_items.get(status_idx)
|
||
{
|
||
let has_screenshot = content
|
||
.iter()
|
||
.any(|c| matches!(c, ContentItem::InputImage { .. }));
|
||
if has_screenshot {
|
||
screenshots_to_keep.insert(status_idx);
|
||
break; // Only keep one screenshot per user message
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Build the filtered history
|
||
let mut items_to_keep = Vec::new();
|
||
let mut removed_screenshots = 0;
|
||
let mut removed_status = 0;
|
||
|
||
for (idx, item) in current_items.iter().enumerate() {
|
||
let should_keep = if status_messages.contains(&idx) {
|
||
// This is a status/screenshot message
|
||
if screenshots_to_keep.contains(&idx) {
|
||
true // Keep this screenshot
|
||
} else {
|
||
// Count what we're removing
|
||
if let ResponseItem::Message { content, .. } = item {
|
||
let has_screenshot = content
|
||
.iter()
|
||
.any(|c| matches!(c, ContentItem::InputImage { .. }));
|
||
if has_screenshot {
|
||
removed_screenshots += 1;
|
||
} else {
|
||
removed_status += 1;
|
||
}
|
||
}
|
||
false // Remove this status/screenshot
|
||
}
|
||
} else {
|
||
true // Keep all non-status messages (real user messages, assistant messages, etc.)
|
||
};
|
||
|
||
if should_keep {
|
||
items_to_keep.push(item.clone());
|
||
}
|
||
}
|
||
|
||
// Replace the history with cleaned items
|
||
state.history = ConversationHistory::new();
|
||
state.history.record_items(&items_to_keep);
|
||
|
||
if removed_screenshots > 0 || removed_status > 0 {
|
||
info!(
|
||
"Cleaned up history: removed {} old screenshots and {} status messages, kept {} recent screenshots",
|
||
removed_screenshots,
|
||
removed_status,
|
||
screenshots_to_keep.len()
|
||
);
|
||
}
|
||
}
|
||
|
||
async fn record_state_snapshot(&self, items: &[ResponseItem]) {
|
||
let snapshot = { crate::rollout::SessionStateSnapshot {} };
|
||
|
||
let recorder = {
|
||
let guard = self.rollout.lock().unwrap();
|
||
guard.as_ref().cloned()
|
||
};
|
||
|
||
if let Some(rec) = recorder {
|
||
if let Err(e) = rec.record_state(snapshot).await {
|
||
error!("failed to record rollout state: {e:#}");
|
||
}
|
||
if let Err(e) = rec.record_items(items).await {
|
||
error!("failed to record rollout items: {e:#}");
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn on_exec_command_begin(
|
||
&self,
|
||
turn_diff_tracker: &mut TurnDiffTracker,
|
||
exec_command_context: ExecCommandContext,
|
||
) {
|
||
let ExecCommandContext {
|
||
sub_id,
|
||
call_id,
|
||
command_for_display,
|
||
cwd,
|
||
apply_patch,
|
||
} = exec_command_context;
|
||
let msg = match apply_patch {
|
||
Some(ApplyPatchCommandContext {
|
||
user_explicitly_approved_this_action,
|
||
changes,
|
||
}) => {
|
||
turn_diff_tracker.on_patch_begin(&changes);
|
||
|
||
EventMsg::PatchApplyBegin(PatchApplyBeginEvent {
|
||
call_id,
|
||
auto_approved: !user_explicitly_approved_this_action,
|
||
changes,
|
||
})
|
||
}
|
||
None => EventMsg::ExecCommandBegin(ExecCommandBeginEvent {
|
||
call_id,
|
||
command: command_for_display.clone(),
|
||
cwd,
|
||
parsed_cmd: parse_command(&command_for_display),
|
||
}),
|
||
};
|
||
let event = Event {
|
||
id: sub_id.to_string(),
|
||
msg,
|
||
};
|
||
let _ = self.tx_event.send(event).await;
|
||
}
|
||
|
||
#[allow(clippy::too_many_arguments)]
|
||
async fn on_exec_command_end(
|
||
&self,
|
||
turn_diff_tracker: &mut TurnDiffTracker,
|
||
sub_id: &str,
|
||
call_id: &str,
|
||
output: &ExecToolCallOutput,
|
||
is_apply_patch: bool,
|
||
) {
|
||
let ExecToolCallOutput {
|
||
stdout,
|
||
stderr,
|
||
duration,
|
||
exit_code,
|
||
} = output;
|
||
// Because stdout and stderr could each be up to 100 KiB, we send
|
||
// truncated versions.
|
||
const MAX_STREAM_OUTPUT: usize = 5 * 1024; // 5KiB
|
||
let stdout = stdout.text.chars().take(MAX_STREAM_OUTPUT).collect();
|
||
let stderr = stderr.text.chars().take(MAX_STREAM_OUTPUT).collect();
|
||
|
||
let msg = if is_apply_patch {
|
||
EventMsg::PatchApplyEnd(PatchApplyEndEvent {
|
||
call_id: call_id.to_string(),
|
||
stdout,
|
||
stderr,
|
||
success: *exit_code == 0,
|
||
})
|
||
} else {
|
||
EventMsg::ExecCommandEnd(ExecCommandEndEvent {
|
||
call_id: call_id.to_string(),
|
||
stdout,
|
||
stderr,
|
||
duration: *duration,
|
||
exit_code: *exit_code,
|
||
})
|
||
};
|
||
|
||
let event = Event {
|
||
id: sub_id.to_string(),
|
||
msg,
|
||
};
|
||
let _ = self.tx_event.send(event).await;
|
||
|
||
// If this is an apply_patch, after we emit the end patch, emit a second event
|
||
// with the full turn diff if there is one.
|
||
if is_apply_patch {
|
||
let unified_diff = turn_diff_tracker.get_unified_diff();
|
||
if let Ok(Some(unified_diff)) = unified_diff {
|
||
let msg = EventMsg::TurnDiff(TurnDiffEvent { unified_diff });
|
||
let event = Event {
|
||
id: sub_id.into(),
|
||
msg,
|
||
};
|
||
let _ = self.tx_event.send(event).await;
|
||
}
|
||
}
|
||
}
|
||
/// Runs the exec tool call and emits events for the begin and end of the
|
||
/// command even on error.
|
||
///
|
||
/// Returns the output of the exec tool call.
|
||
async fn run_exec_with_events<'a>(
|
||
&self,
|
||
turn_diff_tracker: &mut TurnDiffTracker,
|
||
begin_ctx: ExecCommandContext,
|
||
exec_args: ExecInvokeArgs<'a>,
|
||
) -> crate::error::Result<ExecToolCallOutput> {
|
||
let is_apply_patch = begin_ctx.apply_patch.is_some();
|
||
let sub_id = begin_ctx.sub_id.clone();
|
||
let call_id = begin_ctx.call_id.clone();
|
||
|
||
self.on_exec_command_begin(turn_diff_tracker, begin_ctx.clone())
|
||
.await;
|
||
|
||
let result = process_exec_tool_call(
|
||
exec_args.params,
|
||
exec_args.sandbox_type,
|
||
exec_args.sandbox_policy,
|
||
exec_args.codex_linux_sandbox_exe,
|
||
exec_args.stdout_stream,
|
||
)
|
||
.await;
|
||
|
||
let output_stderr;
|
||
let borrowed: &ExecToolCallOutput = match &result {
|
||
Ok(output) => output,
|
||
Err(e) => {
|
||
output_stderr = ExecToolCallOutput {
|
||
exit_code: -1,
|
||
stdout: StreamOutput::new(String::new()),
|
||
stderr: StreamOutput::new(get_error_message_ui(e)),
|
||
duration: Duration::default(),
|
||
};
|
||
&output_stderr
|
||
}
|
||
};
|
||
self.on_exec_command_end(
|
||
turn_diff_tracker,
|
||
&sub_id,
|
||
&call_id,
|
||
borrowed,
|
||
is_apply_patch,
|
||
)
|
||
.await;
|
||
|
||
result
|
||
}
|
||
|
||
/// Helper that emits a BackgroundEvent with the given message. This keeps
|
||
/// the call‑sites terse so adding more diagnostics does not clutter the
|
||
/// core agent logic.
|
||
async fn notify_background_event(&self, sub_id: &str, message: impl Into<String>) {
|
||
let event = Event {
|
||
id: sub_id.to_string(),
|
||
msg: EventMsg::BackgroundEvent(BackgroundEventEvent {
|
||
message: message.into(),
|
||
}),
|
||
};
|
||
let _ = self.tx_event.send(event).await;
|
||
}
|
||
|
||
/// Build the full turn input by concatenating the current conversation
|
||
/// history with additional items for this turn.
|
||
/// Browser screenshots are filtered out from history to keep them ephemeral.
|
||
pub fn turn_input_with_history(&self, extra: Vec<ResponseItem>) -> Vec<ResponseItem> {
|
||
let history = self.state.lock().unwrap().history.contents();
|
||
|
||
// Debug: Count function call outputs in history
|
||
let fc_output_count = history
|
||
.iter()
|
||
.filter(|item| matches!(item, ResponseItem::FunctionCallOutput { .. }))
|
||
.count();
|
||
if fc_output_count > 0 {
|
||
debug!(
|
||
"History contains {} FunctionCallOutput items",
|
||
fc_output_count
|
||
);
|
||
}
|
||
|
||
// Count images in extra for debugging (we can't distinguish ephemeral at this level anymore)
|
||
let images_in_extra = extra
|
||
.iter()
|
||
.filter(|item| {
|
||
if let ResponseItem::Message { content, .. } = item {
|
||
content
|
||
.iter()
|
||
.any(|c| matches!(c, crate::models::ContentItem::InputImage { .. }))
|
||
} else {
|
||
false
|
||
}
|
||
})
|
||
.count();
|
||
|
||
if images_in_extra > 0 {
|
||
tracing::info!(
|
||
"Found {} images in current turn's extra items",
|
||
images_in_extra
|
||
);
|
||
}
|
||
|
||
// Filter out browser screenshots from historical messages
|
||
// We identify them by the [EPHEMERAL:...] marker that precedes them
|
||
let filtered_history: Vec<ResponseItem> = history
|
||
.into_iter()
|
||
.map(|item| {
|
||
if let ResponseItem::Message { id, role, content } = item {
|
||
if role == "user" {
|
||
// Filter out ephemeral content from user messages
|
||
let mut filtered_content: Vec<crate::models::ContentItem> = Vec::new();
|
||
let mut skip_next_image = false;
|
||
|
||
for content_item in content {
|
||
match &content_item {
|
||
crate::models::ContentItem::InputText { text }
|
||
if text.starts_with("[EPHEMERAL:") =>
|
||
{
|
||
// This is an ephemeral marker, skip it and the next image
|
||
skip_next_image = true;
|
||
tracing::info!("Filtering out ephemeral marker: {}", text);
|
||
}
|
||
crate::models::ContentItem::InputImage { .. }
|
||
if skip_next_image =>
|
||
{
|
||
// Skip this image as it follows an ephemeral marker
|
||
skip_next_image = false;
|
||
tracing::info!("Filtering out ephemeral image from history");
|
||
}
|
||
_ => {
|
||
// Keep everything else
|
||
filtered_content.push(content_item);
|
||
}
|
||
}
|
||
}
|
||
|
||
ResponseItem::Message {
|
||
id,
|
||
role,
|
||
content: filtered_content,
|
||
}
|
||
} else {
|
||
// Keep assistant messages unchanged
|
||
ResponseItem::Message { id, role, content }
|
||
}
|
||
} else {
|
||
item
|
||
}
|
||
})
|
||
.collect();
|
||
|
||
// Concatenate filtered history with current turn's extras (which includes current ephemeral images)
|
||
let result = [filtered_history, extra].concat();
|
||
|
||
// Count total images in result for debugging
|
||
let total_images = result
|
||
.iter()
|
||
.filter(|item| {
|
||
if let ResponseItem::Message { content, .. } = item {
|
||
content
|
||
.iter()
|
||
.any(|c| matches!(c, crate::models::ContentItem::InputImage { .. }))
|
||
} else {
|
||
false
|
||
}
|
||
})
|
||
.count();
|
||
|
||
if total_images > 0 {
|
||
tracing::info!("Total images being sent to model: {}", total_images);
|
||
}
|
||
|
||
result
|
||
}
|
||
|
||
/// Returns the input if there was no agent running to inject into
|
||
pub fn inject_input(&self, input: Vec<InputItem>) -> Result<(), Vec<InputItem>> {
|
||
let mut state = self.state.lock().unwrap();
|
||
if state.current_agent.is_some() {
|
||
state.pending_input.push(input.into());
|
||
Ok(())
|
||
} else {
|
||
Err(input)
|
||
}
|
||
}
|
||
|
||
pub fn get_pending_input(&self) -> Vec<ResponseInputItem> {
|
||
let mut state = self.state.lock().unwrap();
|
||
if state.pending_input.is_empty() {
|
||
Vec::with_capacity(0)
|
||
} else {
|
||
let mut ret = Vec::new();
|
||
std::mem::swap(&mut ret, &mut state.pending_input);
|
||
ret
|
||
}
|
||
}
|
||
|
||
pub fn add_pending_input(&self, input: ResponseInputItem) {
|
||
let mut state = self.state.lock().unwrap();
|
||
state.pending_input.push(input);
|
||
}
|
||
|
||
pub async fn call_tool(
|
||
&self,
|
||
server: &str,
|
||
tool: &str,
|
||
arguments: Option<serde_json::Value>,
|
||
timeout: Option<Duration>,
|
||
) -> anyhow::Result<CallToolResult> {
|
||
self.mcp_connection_manager
|
||
.call_tool(server, tool, arguments, timeout)
|
||
.await
|
||
}
|
||
|
||
fn abort(&self) {
|
||
info!("Aborting existing session");
|
||
let mut state = self.state.lock().unwrap();
|
||
state.pending_approvals.clear();
|
||
state.pending_input.clear();
|
||
if let Some(agent) = state.current_agent.take() {
|
||
agent.abort();
|
||
}
|
||
}
|
||
|
||
/// Spawn the configured notifier (if any) with the given JSON payload as
|
||
/// the last argument. Failures are logged but otherwise ignored so that
|
||
/// notification issues do not interfere with the main workflow.
|
||
fn maybe_notify(&self, notification: UserNotification) {
|
||
let Some(notify_command) = &self.notify else {
|
||
return;
|
||
};
|
||
|
||
if notify_command.is_empty() {
|
||
return;
|
||
}
|
||
|
||
let Ok(json) = serde_json::to_string(¬ification) else {
|
||
error!("failed to serialise notification payload");
|
||
return;
|
||
};
|
||
|
||
let mut command = std::process::Command::new(¬ify_command[0]);
|
||
if notify_command.len() > 1 {
|
||
command.args(¬ify_command[1..]);
|
||
}
|
||
command.arg(json);
|
||
|
||
// Fire-and-forget – we do not wait for completion.
|
||
if let Err(e) = command.spawn() {
|
||
warn!("failed to spawn notifier '{}': {e}", notify_command[0]);
|
||
}
|
||
}
|
||
}
|
||
|
||
impl Drop for Session {
|
||
fn drop(&mut self) {
|
||
self.abort();
|
||
}
|
||
}
|
||
|
||
impl State {
|
||
pub fn partial_clone(&self) -> Self {
|
||
Self {
|
||
approved_commands: self.approved_commands.clone(),
|
||
history: self.history.clone(),
|
||
..Default::default()
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub(crate) struct ExecCommandContext {
|
||
pub(crate) sub_id: String,
|
||
pub(crate) call_id: String,
|
||
pub(crate) command_for_display: Vec<String>,
|
||
pub(crate) cwd: PathBuf,
|
||
pub(crate) apply_patch: Option<ApplyPatchCommandContext>,
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub(crate) struct ApplyPatchCommandContext {
|
||
pub(crate) user_explicitly_approved_this_action: bool,
|
||
pub(crate) changes: HashMap<PathBuf, FileChange>,
|
||
}
|
||
|
||
/// A series of Turns in response to user input.
|
||
pub(crate) struct AgentAgent {
|
||
sess: Arc<Session>,
|
||
sub_id: String,
|
||
handle: AbortHandle,
|
||
}
|
||
|
||
impl AgentAgent {
|
||
fn spawn(sess: Arc<Session>, sub_id: String, input: Vec<InputItem>) -> Self {
|
||
let handle =
|
||
tokio::spawn(run_agent(Arc::clone(&sess), sub_id.clone(), input)).abort_handle();
|
||
Self {
|
||
sess,
|
||
sub_id,
|
||
handle,
|
||
}
|
||
}
|
||
|
||
fn compact(
|
||
sess: Arc<Session>,
|
||
sub_id: String,
|
||
input: Vec<InputItem>,
|
||
compact_instructions: String,
|
||
) -> Self {
|
||
let handle = tokio::spawn(run_compact_agent(
|
||
Arc::clone(&sess),
|
||
sub_id.clone(),
|
||
input,
|
||
compact_instructions,
|
||
))
|
||
.abort_handle();
|
||
Self {
|
||
sess,
|
||
sub_id,
|
||
handle,
|
||
}
|
||
}
|
||
|
||
fn abort(self) {
|
||
if !self.handle.is_finished() {
|
||
self.handle.abort();
|
||
let event = Event {
|
||
id: self.sub_id,
|
||
msg: EventMsg::Error(ErrorEvent {
|
||
message: " Turn interrupted".to_string(),
|
||
}),
|
||
};
|
||
let tx_event = self.sess.tx_event.clone();
|
||
tokio::spawn(async move {
|
||
tx_event.send(event).await.ok();
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn submission_loop(
|
||
mut session_id: Uuid,
|
||
config: Arc<Config>,
|
||
auth: Option<CodexAuth>,
|
||
rx_sub: Receiver<Submission>,
|
||
tx_event: Sender<Event>,
|
||
) {
|
||
// Initialize agent manager with event sender
|
||
{
|
||
let mut manager = AGENT_MANAGER.write().await;
|
||
let (agent_tx, mut agent_rx) = tokio::sync::mpsc::unbounded_channel();
|
||
manager.set_event_sender(agent_tx);
|
||
drop(manager);
|
||
|
||
// Forward agent events to the main event channel
|
||
let tx_event_clone = tx_event.clone();
|
||
tokio::spawn(async move {
|
||
while let Some(event) = agent_rx.recv().await {
|
||
let _ = tx_event_clone.send(event).await;
|
||
}
|
||
});
|
||
}
|
||
|
||
let mut sess: Option<Arc<Session>> = None;
|
||
// shorthand - send an event when there is no active session
|
||
let send_no_session_event = |sub_id: String| async {
|
||
let event = Event {
|
||
id: sub_id,
|
||
msg: EventMsg::Error(ErrorEvent {
|
||
message: "No session initialized, expected 'ConfigureSession' as first Op"
|
||
.to_string(),
|
||
}),
|
||
};
|
||
tx_event.send(event).await.ok();
|
||
};
|
||
|
||
// To break out of this loop, send Op::Shutdown.
|
||
while let Ok(sub) = rx_sub.recv().await {
|
||
debug!(?sub, "Submission");
|
||
match sub.op {
|
||
Op::Interrupt => {
|
||
let sess = match sess.as_ref() {
|
||
Some(sess) => sess,
|
||
None => {
|
||
send_no_session_event(sub.id).await;
|
||
continue;
|
||
}
|
||
};
|
||
sess.abort();
|
||
}
|
||
Op::ConfigureSession {
|
||
provider,
|
||
model,
|
||
model_reasoning_effort,
|
||
model_reasoning_summary,
|
||
user_instructions,
|
||
base_instructions,
|
||
approval_policy,
|
||
sandbox_policy,
|
||
disable_response_storage,
|
||
notify,
|
||
cwd,
|
||
resume_path,
|
||
} => {
|
||
debug!(
|
||
"Configuring session: model={model}; provider={provider:?}; resume={resume_path:?}"
|
||
);
|
||
if !cwd.is_absolute() {
|
||
let message = format!("cwd is not absolute: {cwd:?}");
|
||
error!(message);
|
||
let event = Event {
|
||
id: sub.id,
|
||
msg: EventMsg::Error(ErrorEvent { message }),
|
||
};
|
||
if let Err(e) = tx_event.send(event).await {
|
||
error!("failed to send error message: {e:?}");
|
||
}
|
||
return;
|
||
}
|
||
// Optionally resume an existing rollout.
|
||
let mut restored_items: Option<Vec<ResponseItem>> = None;
|
||
let rollout_recorder: Option<RolloutRecorder> =
|
||
if let Some(path) = resume_path.as_ref() {
|
||
match RolloutRecorder::resume(path, cwd.clone()).await {
|
||
Ok((rec, saved)) => {
|
||
session_id = saved.session_id;
|
||
if !saved.items.is_empty() {
|
||
restored_items = Some(saved.items);
|
||
}
|
||
Some(rec)
|
||
}
|
||
Err(e) => {
|
||
warn!("failed to resume rollout from {path:?}: {e}");
|
||
None
|
||
}
|
||
}
|
||
} else {
|
||
None
|
||
};
|
||
|
||
let rollout_recorder = match rollout_recorder {
|
||
Some(rec) => Some(rec),
|
||
None => {
|
||
match RolloutRecorder::new(&config, session_id, user_instructions.clone())
|
||
.await
|
||
{
|
||
Ok(r) => Some(r),
|
||
Err(e) => {
|
||
warn!("failed to initialise rollout recorder: {e}");
|
||
None
|
||
}
|
||
}
|
||
}
|
||
};
|
||
|
||
// Create debug logger based on config
|
||
let debug_logger = match crate::debug_logger::DebugLogger::new(config.debug) {
|
||
Ok(logger) => std::sync::Arc::new(std::sync::Mutex::new(logger)),
|
||
Err(e) => {
|
||
warn!("Failed to create debug logger: {}", e);
|
||
// Create a disabled logger as fallback
|
||
std::sync::Arc::new(std::sync::Mutex::new(
|
||
crate::debug_logger::DebugLogger::new(false).unwrap(),
|
||
))
|
||
}
|
||
};
|
||
|
||
let client = ModelClient::new(
|
||
config.clone(),
|
||
auth.clone(),
|
||
provider.clone(),
|
||
model_reasoning_effort,
|
||
model_reasoning_summary,
|
||
session_id,
|
||
debug_logger,
|
||
);
|
||
|
||
// abort any current running session and clone its state
|
||
let state = match sess.take() {
|
||
Some(sess) => {
|
||
sess.abort();
|
||
sess.state.lock().unwrap().partial_clone()
|
||
}
|
||
None => State {
|
||
history: ConversationHistory::new(),
|
||
..Default::default()
|
||
},
|
||
};
|
||
|
||
let writable_roots = get_writable_roots(&cwd);
|
||
|
||
// Error messages to dispatch after SessionConfigured is sent.
|
||
let mut mcp_connection_errors = Vec::<Event>::new();
|
||
let (mcp_connection_manager, failed_clients) =
|
||
match McpConnectionManager::new(config.mcp_servers.clone()).await {
|
||
Ok((mgr, failures)) => (mgr, failures),
|
||
Err(e) => {
|
||
let message = format!("Failed to create MCP connection manager: {e:#}");
|
||
error!("{message}");
|
||
mcp_connection_errors.push(Event {
|
||
id: sub.id.clone(),
|
||
msg: EventMsg::Error(ErrorEvent { message }),
|
||
});
|
||
(McpConnectionManager::default(), Default::default())
|
||
}
|
||
};
|
||
|
||
// Surface individual client start-up failures to the user.
|
||
if !failed_clients.is_empty() {
|
||
for (server_name, err) in failed_clients {
|
||
let message =
|
||
format!("MCP client for `{server_name}` failed to start: {err:#}");
|
||
error!("{message}");
|
||
mcp_connection_errors.push(Event {
|
||
id: sub.id.clone(),
|
||
msg: EventMsg::Error(ErrorEvent { message }),
|
||
});
|
||
}
|
||
}
|
||
let default_shell = shell::default_user_shell().await;
|
||
sess = Some(Arc::new(Session {
|
||
client,
|
||
tools_config: ToolsConfig::new(
|
||
&config.model_family,
|
||
approval_policy,
|
||
sandbox_policy.clone(),
|
||
config.include_plan_tool,
|
||
),
|
||
tx_event: tx_event.clone(),
|
||
user_instructions,
|
||
base_instructions,
|
||
approval_policy,
|
||
sandbox_policy,
|
||
shell_environment_policy: config.shell_environment_policy.clone(),
|
||
cwd,
|
||
writable_roots,
|
||
mcp_connection_manager,
|
||
agents: config.agents.clone(),
|
||
notify,
|
||
state: Mutex::new(state),
|
||
rollout: Mutex::new(rollout_recorder),
|
||
codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(),
|
||
disable_response_storage,
|
||
user_shell: default_shell,
|
||
show_raw_agent_reasoning: config.show_raw_agent_reasoning,
|
||
pending_browser_screenshots: Mutex::new(Vec::new()),
|
||
last_system_status: Mutex::new(None),
|
||
last_screenshot_info: Mutex::new(None),
|
||
}));
|
||
|
||
// Patch restored state into the newly created session.
|
||
if let Some(sess_arc) = &sess {
|
||
if restored_items.is_some() {
|
||
let mut st = sess_arc.state.lock().unwrap();
|
||
st.history.record_items(restored_items.unwrap().iter());
|
||
}
|
||
}
|
||
|
||
// Gather history metadata for SessionConfiguredEvent.
|
||
let (history_log_id, history_entry_count) =
|
||
crate::message_history::history_metadata(&config).await;
|
||
|
||
// ack
|
||
let events = std::iter::once(Event {
|
||
id: sub.id.clone(),
|
||
msg: EventMsg::SessionConfigured(SessionConfiguredEvent {
|
||
session_id,
|
||
model,
|
||
history_log_id,
|
||
history_entry_count,
|
||
}),
|
||
})
|
||
.chain(mcp_connection_errors.into_iter());
|
||
for event in events {
|
||
if let Err(e) = tx_event.send(event).await {
|
||
error!("failed to send event: {e:?}");
|
||
}
|
||
}
|
||
}
|
||
Op::UserInput { items } => {
|
||
let sess = match sess.as_ref() {
|
||
Some(sess) => sess,
|
||
None => {
|
||
send_no_session_event(sub.id).await;
|
||
continue;
|
||
}
|
||
};
|
||
|
||
// Clean up old status items when new user input arrives
|
||
// This prevents token buildup from old screenshots/status messages
|
||
sess.cleanup_old_status_items().await;
|
||
|
||
// attempt to inject input into current agent
|
||
if let Err(items) = sess.inject_input(items) {
|
||
// no current agent, spawn a new one
|
||
let agent = AgentAgent::spawn(Arc::clone(sess), sub.id, items);
|
||
sess.set_agent(agent);
|
||
}
|
||
}
|
||
Op::ExecApproval { id, decision } => {
|
||
let sess = match sess.as_ref() {
|
||
Some(sess) => sess,
|
||
None => {
|
||
send_no_session_event(sub.id).await;
|
||
continue;
|
||
}
|
||
};
|
||
match decision {
|
||
ReviewDecision::Abort => {
|
||
sess.abort();
|
||
}
|
||
other => sess.notify_approval(&id, other),
|
||
}
|
||
}
|
||
Op::PatchApproval { id, decision } => {
|
||
let sess = match sess.as_ref() {
|
||
Some(sess) => sess,
|
||
None => {
|
||
send_no_session_event(sub.id).await;
|
||
continue;
|
||
}
|
||
};
|
||
match decision {
|
||
ReviewDecision::Abort => {
|
||
sess.abort();
|
||
}
|
||
other => sess.notify_approval(&id, other),
|
||
}
|
||
}
|
||
Op::AddToHistory { text } => {
|
||
// TODO: What should we do if we got AddToHistory before ConfigureSession?
|
||
// currently, if ConfigureSession has resume path, this history will be ignored
|
||
let id = session_id;
|
||
let config = config.clone();
|
||
tokio::spawn(async move {
|
||
if let Err(e) = crate::message_history::append_entry(&text, &id, &config).await
|
||
{
|
||
warn!("failed to append to message history: {e}");
|
||
}
|
||
});
|
||
}
|
||
|
||
Op::GetHistoryEntryRequest { offset, log_id } => {
|
||
let config = config.clone();
|
||
let tx_event = tx_event.clone();
|
||
let sub_id = sub.id.clone();
|
||
|
||
tokio::spawn(async move {
|
||
// Run lookup in blocking thread because it does file IO + locking.
|
||
let entry_opt = tokio::task::spawn_blocking(move || {
|
||
crate::message_history::lookup(log_id, offset, &config)
|
||
})
|
||
.await
|
||
.unwrap_or(None);
|
||
|
||
let event = Event {
|
||
id: sub_id,
|
||
msg: EventMsg::GetHistoryEntryResponse(
|
||
crate::protocol::GetHistoryEntryResponseEvent {
|
||
offset,
|
||
log_id,
|
||
entry: entry_opt,
|
||
},
|
||
),
|
||
};
|
||
|
||
if let Err(e) = tx_event.send(event).await {
|
||
warn!("failed to send GetHistoryEntryResponse event: {e}");
|
||
}
|
||
});
|
||
}
|
||
Op::Compact => {
|
||
let sess = match sess.as_ref() {
|
||
Some(sess) => sess,
|
||
None => {
|
||
send_no_session_event(sub.id).await;
|
||
continue;
|
||
}
|
||
};
|
||
|
||
// Create a summarization request as user input
|
||
const SUMMARIZATION_PROMPT: &str = include_str!("prompt_for_compact_command.md");
|
||
|
||
// Attempt to inject input into current agent
|
||
if let Err(items) = sess.inject_input(vec![InputItem::Text {
|
||
text: "Start Summarization".to_string(),
|
||
}]) {
|
||
let agent = AgentAgent::compact(
|
||
sess.clone(),
|
||
sub.id,
|
||
items,
|
||
SUMMARIZATION_PROMPT.to_string(),
|
||
);
|
||
sess.set_agent(agent);
|
||
}
|
||
}
|
||
Op::Shutdown => {
|
||
info!("Shutting down Codex instance");
|
||
|
||
// Gracefully flush and shutdown rollout recorder on session end so tests
|
||
// that inspect the rollout file do not race with the background writer.
|
||
if let Some(sess_arc) = sess {
|
||
let recorder_opt = sess_arc.rollout.lock().unwrap().take();
|
||
if let Some(rec) = recorder_opt {
|
||
if let Err(e) = rec.shutdown().await {
|
||
warn!("failed to shutdown rollout recorder: {e}");
|
||
let event = Event {
|
||
id: sub.id.clone(),
|
||
msg: EventMsg::Error(ErrorEvent {
|
||
message: "Failed to shutdown rollout recorder".to_string(),
|
||
}),
|
||
};
|
||
if let Err(e) = tx_event.send(event).await {
|
||
warn!("failed to send error message: {e:?}");
|
||
}
|
||
}
|
||
}
|
||
}
|
||
let event = Event {
|
||
id: sub.id.clone(),
|
||
msg: EventMsg::ShutdownComplete,
|
||
};
|
||
if let Err(e) = tx_event.send(event).await {
|
||
warn!("failed to send Shutdown event: {e}");
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
debug!("Agent loop exited");
|
||
}
|
||
|
||
/// Takes a user message as input and runs a loop where, at each turn, the model
|
||
/// replies with either:
|
||
///
|
||
/// - requested function calls
|
||
/// - an assistant message
|
||
///
|
||
/// While it is possible for the model to return multiple of these items in a
|
||
/// single turn, in practice, we generally one item per turn:
|
||
///
|
||
/// - If the model requests a function call, we execute it and send the output
|
||
/// back to the model in the next turn.
|
||
/// - If the model sends only an assistant message, we record it in the
|
||
/// conversation history and consider the agent complete.
|
||
async fn run_agent(sess: Arc<Session>, sub_id: String, input: Vec<InputItem>) {
|
||
if input.is_empty() {
|
||
return;
|
||
}
|
||
let event = Event {
|
||
id: sub_id.clone(),
|
||
msg: EventMsg::TaskStarted,
|
||
};
|
||
if sess.tx_event.send(event).await.is_err() {
|
||
return;
|
||
}
|
||
|
||
// Debug logging for ephemeral images
|
||
let ephemeral_count = input
|
||
.iter()
|
||
.filter(|item| matches!(item, InputItem::EphemeralImage { .. }))
|
||
.count();
|
||
|
||
if ephemeral_count > 0 {
|
||
tracing::info!(
|
||
"Processing {} ephemeral images in user input",
|
||
ephemeral_count
|
||
);
|
||
}
|
||
|
||
// Convert input to ResponseInputItem
|
||
let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input);
|
||
let initial_response_item: ResponseItem = initial_input_for_turn.clone().into();
|
||
|
||
// Record to history but we'll handle ephemeral images separately
|
||
sess.record_conversation_items(&[initial_response_item.clone()])
|
||
.await;
|
||
|
||
let mut last_task_message: Option<String> = None;
|
||
// Although from the perspective of codex.rs, TurnDiffTracker has the lifecycle of a Agent which contains
|
||
// many turns, from the perspective of the user, it is a single turn.
|
||
let mut turn_diff_tracker = TurnDiffTracker::new();
|
||
|
||
// Track if this is the first iteration - if so, include the initial input
|
||
let mut first_iteration = true;
|
||
|
||
loop {
|
||
// Note that pending_input would be something like a message the user
|
||
// submitted through the UI while the model was running. Though the UI
|
||
// may support this, the model might not.
|
||
let pending_input = sess
|
||
.get_pending_input()
|
||
.into_iter()
|
||
.map(ResponseItem::from)
|
||
.collect::<Vec<ResponseItem>>();
|
||
|
||
// Do not duplicate the initial input in `pending_input`.
|
||
// It is already recorded to history above; ephemeral items are appended separately.
|
||
if first_iteration {
|
||
first_iteration = false;
|
||
} else {
|
||
// Only record pending input to history on subsequent iterations
|
||
sess.record_conversation_items(&pending_input).await;
|
||
}
|
||
|
||
// Construct the input that we will send to the model. When using the
|
||
// Chat completions API (or ZDR clients), the model needs the full
|
||
// conversation history on each turn. The rollout file, however, should
|
||
// only record the new items that originated in this turn so that it
|
||
// represents an append-only log without duplicates.
|
||
let turn_input: Vec<ResponseItem> = sess.turn_input_with_history(pending_input);
|
||
|
||
let turn_input_messages: Vec<String> = turn_input
|
||
.iter()
|
||
.filter_map(|item| match item {
|
||
ResponseItem::Message { content, .. } => Some(content),
|
||
_ => None,
|
||
})
|
||
.flat_map(|content| {
|
||
content.iter().filter_map(|item| match item {
|
||
ContentItem::OutputText { text } => Some(text.clone()),
|
||
_ => None,
|
||
})
|
||
})
|
||
.collect();
|
||
match run_turn(&sess, &mut turn_diff_tracker, sub_id.clone(), turn_input).await {
|
||
Ok(turn_output) => {
|
||
let mut items_to_record_in_conversation_history = Vec::<ResponseItem>::new();
|
||
let mut responses = Vec::<ResponseInputItem>::new();
|
||
for processed_response_item in turn_output {
|
||
let ProcessedResponseItem { item, response } = processed_response_item;
|
||
match (&item, &response) {
|
||
(ResponseItem::Message { role, .. }, None) if role == "assistant" => {
|
||
// If the model returned a message, we need to record it.
|
||
items_to_record_in_conversation_history.push(item);
|
||
}
|
||
(
|
||
ResponseItem::LocalShellCall { .. },
|
||
Some(ResponseInputItem::FunctionCallOutput { call_id, output }),
|
||
) => {
|
||
items_to_record_in_conversation_history.push(item);
|
||
items_to_record_in_conversation_history.push(
|
||
ResponseItem::FunctionCallOutput {
|
||
call_id: call_id.clone(),
|
||
output: output.clone(),
|
||
},
|
||
);
|
||
}
|
||
(
|
||
ResponseItem::FunctionCall { .. },
|
||
Some(ResponseInputItem::FunctionCallOutput { call_id, output }),
|
||
) => {
|
||
debug!(
|
||
"Recording function call and output for call_id: {}",
|
||
call_id
|
||
);
|
||
items_to_record_in_conversation_history.push(item);
|
||
items_to_record_in_conversation_history.push(
|
||
ResponseItem::FunctionCallOutput {
|
||
call_id: call_id.clone(),
|
||
output: output.clone(),
|
||
},
|
||
);
|
||
}
|
||
(
|
||
ResponseItem::FunctionCall { .. },
|
||
Some(ResponseInputItem::McpToolCallOutput { call_id, result }),
|
||
) => {
|
||
items_to_record_in_conversation_history.push(item);
|
||
let (content, success): (String, Option<bool>) = match result {
|
||
Ok(CallToolResult {
|
||
content,
|
||
is_error,
|
||
structured_content: _,
|
||
}) => match serde_json::to_string(content) {
|
||
Ok(content) => (content, *is_error),
|
||
Err(e) => {
|
||
warn!("Failed to serialize MCP tool call output: {e}");
|
||
(e.to_string(), Some(true))
|
||
}
|
||
},
|
||
Err(e) => (e.clone(), Some(true)),
|
||
};
|
||
items_to_record_in_conversation_history.push(
|
||
ResponseItem::FunctionCallOutput {
|
||
call_id: call_id.clone(),
|
||
output: FunctionCallOutputPayload { content, success },
|
||
},
|
||
);
|
||
}
|
||
(
|
||
ResponseItem::Reasoning {
|
||
id,
|
||
summary,
|
||
content,
|
||
encrypted_content,
|
||
},
|
||
None,
|
||
) => {
|
||
items_to_record_in_conversation_history.push(ResponseItem::Reasoning {
|
||
id: id.clone(),
|
||
summary: summary.clone(),
|
||
content: content.clone(),
|
||
encrypted_content: encrypted_content.clone(),
|
||
});
|
||
}
|
||
_ => {
|
||
warn!("Unexpected response item: {item:?} with response: {response:?}");
|
||
}
|
||
};
|
||
if let Some(response) = response {
|
||
responses.push(response);
|
||
}
|
||
}
|
||
|
||
// Only attempt to take the lock if there is something to record.
|
||
if !items_to_record_in_conversation_history.is_empty() {
|
||
// Record items in their original chronological order to maintain
|
||
// proper sequence of events. This ensures function calls and their
|
||
// outputs appear in the correct order in conversation history.
|
||
sess.record_conversation_items(&items_to_record_in_conversation_history)
|
||
.await;
|
||
}
|
||
|
||
// If there are responses, add them to pending input for the next iteration
|
||
if !responses.is_empty() {
|
||
for response in &responses {
|
||
sess.add_pending_input(response.clone());
|
||
}
|
||
}
|
||
|
||
if responses.is_empty() {
|
||
debug!("Turn completed");
|
||
last_task_message = get_last_assistant_message_from_turn(
|
||
&items_to_record_in_conversation_history,
|
||
);
|
||
sess.maybe_notify(UserNotification::AgentTurnComplete {
|
||
turn_id: sub_id.clone(),
|
||
input_messages: turn_input_messages,
|
||
last_assistant_message: last_task_message.clone(),
|
||
});
|
||
break;
|
||
}
|
||
}
|
||
Err(e) => {
|
||
info!("Turn error: {e:#}");
|
||
let event = Event {
|
||
id: sub_id.clone(),
|
||
msg: EventMsg::Error(ErrorEvent {
|
||
message: e.to_string(),
|
||
}),
|
||
};
|
||
sess.tx_event.send(event).await.ok();
|
||
// let the user continue the conversation
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
sess.remove_agent(&sub_id);
|
||
let event = Event {
|
||
id: sub_id,
|
||
msg: EventMsg::TaskComplete(TaskCompleteEvent {
|
||
last_agent_message: last_task_message,
|
||
}),
|
||
};
|
||
sess.tx_event.send(event).await.ok();
|
||
}
|
||
|
||
async fn run_turn(
|
||
sess: &Session,
|
||
turn_diff_tracker: &mut TurnDiffTracker,
|
||
sub_id: String,
|
||
input: Vec<ResponseItem>,
|
||
) -> CodexResult<Vec<ProcessedResponseItem>> {
|
||
let tools = get_openai_tools(
|
||
&sess.tools_config,
|
||
Some(sess.mcp_connection_manager.list_all_tools()),
|
||
);
|
||
|
||
let mut retries = 0;
|
||
loop {
|
||
// Build status items (screenshots, system status) fresh for each attempt
|
||
let status_items = build_turn_status_items(sess).await;
|
||
|
||
let prompt = Prompt {
|
||
input: input.clone(),
|
||
user_instructions: sess.user_instructions.clone(),
|
||
store: !sess.disable_response_storage,
|
||
tools: tools.clone(),
|
||
base_instructions_override: sess.base_instructions.clone(),
|
||
environment_context: Some(EnvironmentContext {
|
||
cwd: sess.cwd.clone(),
|
||
approval_policy: sess.approval_policy,
|
||
sandbox_policy: sess.sandbox_policy.clone(),
|
||
}),
|
||
status_items, // Include status items with this request
|
||
};
|
||
|
||
match try_run_turn(sess, turn_diff_tracker, &sub_id, &prompt).await {
|
||
Ok(output) => {
|
||
// Record status items to conversation history after successful turn
|
||
// This ensures they persist for future requests in the right chronological order
|
||
if !prompt.status_items.is_empty() {
|
||
sess.record_conversation_items(&prompt.status_items).await;
|
||
}
|
||
return Ok(output);
|
||
}
|
||
Err(CodexErr::Interrupted) => return Err(CodexErr::Interrupted),
|
||
Err(CodexErr::EnvVar(var)) => return Err(CodexErr::EnvVar(var)),
|
||
Err(e @ (CodexErr::UsageLimitReached(_) | CodexErr::UsageNotIncluded)) => {
|
||
return Err(e);
|
||
}
|
||
Err(e) => {
|
||
// Use the configured provider-specific stream retry budget.
|
||
let max_retries = sess.client.get_provider().stream_max_retries();
|
||
if retries < max_retries {
|
||
retries += 1;
|
||
let delay = match e {
|
||
CodexErr::Stream(_, Some(delay)) => delay,
|
||
_ => backoff(retries),
|
||
};
|
||
warn!(
|
||
"stream disconnected - retrying turn ({retries}/{max_retries} in {delay:?})...",
|
||
);
|
||
|
||
// Surface retry information to any UI/front‑end so the
|
||
// user understands what is happening instead of staring
|
||
// at a seemingly frozen screen.
|
||
sess.notify_background_event(
|
||
&sub_id,
|
||
format!(
|
||
"stream error: {e}; retrying {retries}/{max_retries} in {delay:?}…"
|
||
),
|
||
)
|
||
.await;
|
||
|
||
tokio::time::sleep(delay).await;
|
||
} else {
|
||
return Err(e);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// When the model is prompted, it returns a stream of events. Some of these
|
||
/// events map to a `ResponseItem`. A `ResponseItem` may need to be
|
||
/// "handled" such that it produces a `ResponseInputItem` that needs to be
|
||
/// sent back to the model on the next turn.
|
||
#[derive(Debug)]
|
||
struct ProcessedResponseItem {
|
||
item: ResponseItem,
|
||
response: Option<ResponseInputItem>,
|
||
}
|
||
|
||
async fn try_run_turn(
|
||
sess: &Session,
|
||
turn_diff_tracker: &mut TurnDiffTracker,
|
||
sub_id: &str,
|
||
prompt: &Prompt,
|
||
) -> CodexResult<Vec<ProcessedResponseItem>> {
|
||
// call_ids that are part of this response.
|
||
let completed_call_ids = prompt
|
||
.input
|
||
.iter()
|
||
.filter_map(|ri| match ri {
|
||
ResponseItem::FunctionCallOutput { call_id, .. } => Some(call_id),
|
||
ResponseItem::LocalShellCall {
|
||
call_id: Some(call_id),
|
||
..
|
||
} => Some(call_id),
|
||
_ => None,
|
||
})
|
||
.collect::<Vec<_>>();
|
||
|
||
// call_ids that were pending but are not part of this response.
|
||
// This usually happens because the user interrupted the model before we responded to one of its tool calls
|
||
// and then the user sent a follow-up message.
|
||
let missing_calls = {
|
||
prompt
|
||
.input
|
||
.iter()
|
||
.filter_map(|ri| match ri {
|
||
ResponseItem::FunctionCall { call_id, .. } => Some(call_id),
|
||
ResponseItem::LocalShellCall {
|
||
call_id: Some(call_id),
|
||
..
|
||
} => Some(call_id),
|
||
_ => None,
|
||
})
|
||
.filter_map(|call_id| {
|
||
if completed_call_ids.contains(&call_id) {
|
||
None
|
||
} else {
|
||
Some(call_id.clone())
|
||
}
|
||
})
|
||
.map(|call_id| ResponseItem::FunctionCallOutput {
|
||
call_id: call_id.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: "aborted".to_string(),
|
||
success: Some(false),
|
||
},
|
||
})
|
||
.collect::<Vec<_>>()
|
||
};
|
||
let prompt: Cow<Prompt> = if missing_calls.is_empty() {
|
||
Cow::Borrowed(prompt)
|
||
} else {
|
||
// Add the synthetic aborted missing calls to the beginning of the input to ensure all call ids have responses.
|
||
let input = [missing_calls, prompt.input.clone()].concat();
|
||
Cow::Owned(Prompt {
|
||
input,
|
||
..prompt.clone()
|
||
})
|
||
};
|
||
|
||
let mut stream = sess.client.clone().stream(&prompt).await?;
|
||
|
||
let mut output = Vec::new();
|
||
loop {
|
||
// Poll the next item from the model stream. We must inspect *both* Ok and Err
|
||
// cases so that transient stream failures (e.g., dropped SSE connection before
|
||
// `response.completed`) bubble up and trigger the caller's retry logic.
|
||
let event = stream.next().await;
|
||
let Some(event) = event else {
|
||
// Channel closed without yielding a final Completed event or explicit error.
|
||
// Treat as a disconnected stream so the caller can retry.
|
||
return Err(CodexErr::Stream(
|
||
"stream closed before response.completed".into(),
|
||
None,
|
||
));
|
||
};
|
||
|
||
let event = match event {
|
||
Ok(ev) => ev,
|
||
Err(e) => {
|
||
// Propagate the underlying stream error to the caller (run_turn), which
|
||
// will apply the configured `stream_max_retries` policy.
|
||
return Err(e);
|
||
}
|
||
};
|
||
|
||
match event {
|
||
ResponseEvent::Created => {}
|
||
ResponseEvent::OutputItemDone(item) => {
|
||
let response =
|
||
handle_response_item(sess, turn_diff_tracker, sub_id, item.clone()).await?;
|
||
|
||
output.push(ProcessedResponseItem { item, response });
|
||
}
|
||
ResponseEvent::Completed {
|
||
response_id: _,
|
||
token_usage,
|
||
} => {
|
||
if let Some(token_usage) = token_usage {
|
||
sess.tx_event
|
||
.send(Event {
|
||
id: sub_id.to_string(),
|
||
msg: EventMsg::TokenCount(token_usage),
|
||
})
|
||
.await
|
||
.ok();
|
||
}
|
||
|
||
let unified_diff = turn_diff_tracker.get_unified_diff();
|
||
if let Ok(Some(unified_diff)) = unified_diff {
|
||
let msg = EventMsg::TurnDiff(TurnDiffEvent { unified_diff });
|
||
let event = Event {
|
||
id: sub_id.to_string(),
|
||
msg,
|
||
};
|
||
let _ = sess.tx_event.send(event).await;
|
||
}
|
||
|
||
return Ok(output);
|
||
}
|
||
ResponseEvent::OutputTextDelta { delta, item_id } => {
|
||
// Don't append to history during streaming - only send UI events.
|
||
// The complete message will be added to history when OutputItemDone arrives.
|
||
// This ensures items are recorded in the correct chronological order.
|
||
|
||
// Use the item_id if present, otherwise fall back to sub_id
|
||
let event_id = item_id.unwrap_or_else(|| sub_id.to_string());
|
||
let event = Event {
|
||
id: event_id,
|
||
msg: EventMsg::AgentMessageDelta(AgentMessageDeltaEvent { delta }),
|
||
};
|
||
sess.tx_event.send(event).await.ok();
|
||
}
|
||
ResponseEvent::ReasoningSummaryDelta { delta, item_id } => {
|
||
// Use the item_id if present, otherwise fall back to sub_id
|
||
let event_id = item_id.unwrap_or_else(|| sub_id.to_string());
|
||
let event = Event {
|
||
id: event_id,
|
||
msg: EventMsg::AgentReasoningDelta(AgentReasoningDeltaEvent { delta }),
|
||
};
|
||
sess.tx_event.send(event).await.ok();
|
||
}
|
||
ResponseEvent::ReasoningSummaryPartAdded => {
|
||
let event = Event {
|
||
id: sub_id.to_string(),
|
||
msg: EventMsg::AgentReasoningSectionBreak(AgentReasoningSectionBreakEvent {}),
|
||
};
|
||
sess.tx_event.send(event).await.ok();
|
||
}
|
||
ResponseEvent::ReasoningContentDelta { delta, item_id } => {
|
||
if sess.show_raw_agent_reasoning {
|
||
// Use the item_id if present, otherwise fall back to sub_id
|
||
let event_id = item_id.unwrap_or_else(|| sub_id.to_string());
|
||
let event = Event {
|
||
id: event_id,
|
||
msg: EventMsg::AgentReasoningRawContentDelta(
|
||
AgentReasoningRawContentDeltaEvent { delta },
|
||
),
|
||
};
|
||
sess.tx_event.send(event).await.ok();
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn run_compact_agent(
|
||
sess: Arc<Session>,
|
||
sub_id: String,
|
||
input: Vec<InputItem>,
|
||
compact_instructions: String,
|
||
) {
|
||
let start_event = Event {
|
||
id: sub_id.clone(),
|
||
msg: EventMsg::TaskStarted,
|
||
};
|
||
if sess.tx_event.send(start_event).await.is_err() {
|
||
return;
|
||
}
|
||
|
||
let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input);
|
||
let turn_input: Vec<ResponseItem> =
|
||
sess.turn_input_with_history(vec![initial_input_for_turn.clone().into()]);
|
||
|
||
let max_retries = sess.client.get_provider().stream_max_retries();
|
||
let mut retries = 0;
|
||
|
||
loop {
|
||
// Build status items (screenshots, system status) fresh for each attempt
|
||
let status_items = build_turn_status_items(&sess).await;
|
||
|
||
let prompt = Prompt {
|
||
input: turn_input.clone(),
|
||
user_instructions: None,
|
||
store: !sess.disable_response_storage,
|
||
environment_context: None,
|
||
tools: Vec::new(),
|
||
base_instructions_override: Some(compact_instructions.clone()),
|
||
status_items, // Include status items with this request
|
||
};
|
||
|
||
let attempt_result = drain_to_completed(&sess, &sub_id, &prompt).await;
|
||
|
||
match attempt_result {
|
||
Ok(()) => {
|
||
// Record status items to conversation history after successful turn
|
||
if !prompt.status_items.is_empty() {
|
||
sess.record_conversation_items(&prompt.status_items).await;
|
||
}
|
||
break;
|
||
}
|
||
Err(CodexErr::Interrupted) => return,
|
||
Err(e) => {
|
||
if retries < max_retries {
|
||
retries += 1;
|
||
let delay = backoff(retries);
|
||
sess.notify_background_event(
|
||
&sub_id,
|
||
format!(
|
||
"stream error: {e}; retrying {retries}/{max_retries} in {delay:?}…"
|
||
),
|
||
)
|
||
.await;
|
||
tokio::time::sleep(delay).await;
|
||
continue;
|
||
} else {
|
||
let event = Event {
|
||
id: sub_id.clone(),
|
||
msg: EventMsg::Error(ErrorEvent {
|
||
message: e.to_string(),
|
||
}),
|
||
};
|
||
sess.send_event(event).await;
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
sess.remove_agent(&sub_id);
|
||
let event = Event {
|
||
id: sub_id.clone(),
|
||
msg: EventMsg::AgentMessage(AgentMessageEvent {
|
||
message: "Compact agent completed".to_string(),
|
||
}),
|
||
};
|
||
sess.send_event(event).await;
|
||
let event = Event {
|
||
id: sub_id.clone(),
|
||
msg: EventMsg::TaskComplete(TaskCompleteEvent {
|
||
last_agent_message: None,
|
||
}),
|
||
};
|
||
sess.send_event(event).await;
|
||
|
||
let mut state = sess.state.lock().unwrap();
|
||
state.history.keep_last_messages(1);
|
||
}
|
||
|
||
async fn handle_response_item(
|
||
sess: &Session,
|
||
turn_diff_tracker: &mut TurnDiffTracker,
|
||
sub_id: &str,
|
||
item: ResponseItem,
|
||
) -> CodexResult<Option<ResponseInputItem>> {
|
||
debug!(?item, "Output item");
|
||
let output = match item {
|
||
ResponseItem::Message { content, id, .. } => {
|
||
// Use the item_id if present, otherwise fall back to sub_id
|
||
let event_id = id.unwrap_or_else(|| sub_id.to_string());
|
||
for item in content {
|
||
if let ContentItem::OutputText { text } = item {
|
||
let event = Event {
|
||
id: event_id.clone(),
|
||
msg: EventMsg::AgentMessage(AgentMessageEvent { message: text }),
|
||
};
|
||
sess.tx_event.send(event).await.ok();
|
||
}
|
||
}
|
||
None
|
||
}
|
||
ResponseItem::Reasoning {
|
||
id,
|
||
summary,
|
||
content,
|
||
encrypted_content: _,
|
||
} => {
|
||
// Use the item_id if present and not empty, otherwise fall back to sub_id
|
||
let event_id = if !id.is_empty() {
|
||
id.clone()
|
||
} else {
|
||
sub_id.to_string()
|
||
};
|
||
for item in summary {
|
||
let text = match item {
|
||
ReasoningItemReasoningSummary::SummaryText { text } => text,
|
||
};
|
||
let event = Event {
|
||
id: event_id.clone(),
|
||
msg: EventMsg::AgentReasoning(AgentReasoningEvent { text }),
|
||
};
|
||
sess.tx_event.send(event).await.ok();
|
||
}
|
||
if sess.show_raw_agent_reasoning && content.is_some() {
|
||
let content = content.unwrap();
|
||
for item in content {
|
||
let text = match item {
|
||
ReasoningItemContent::ReasoningText { text } => text,
|
||
ReasoningItemContent::Text { text } => text,
|
||
};
|
||
let event = Event {
|
||
id: event_id.clone(),
|
||
msg: EventMsg::AgentReasoningRawContent(AgentReasoningRawContentEvent {
|
||
text,
|
||
}),
|
||
};
|
||
sess.tx_event.send(event).await.ok();
|
||
}
|
||
}
|
||
None
|
||
}
|
||
ResponseItem::FunctionCall {
|
||
name,
|
||
arguments,
|
||
call_id,
|
||
..
|
||
} => {
|
||
info!("FunctionCall: {arguments}");
|
||
Some(
|
||
handle_function_call(
|
||
sess,
|
||
turn_diff_tracker,
|
||
sub_id.to_string(),
|
||
name,
|
||
arguments,
|
||
call_id,
|
||
)
|
||
.await,
|
||
)
|
||
}
|
||
ResponseItem::LocalShellCall {
|
||
id,
|
||
call_id,
|
||
status: _,
|
||
action,
|
||
} => {
|
||
let LocalShellAction::Exec(action) = action;
|
||
tracing::info!("LocalShellCall: {action:?}");
|
||
let params = ShellToolCallParams {
|
||
command: action.command,
|
||
workdir: action.working_directory,
|
||
timeout_ms: action.timeout_ms,
|
||
with_escalated_permissions: None,
|
||
justification: None,
|
||
};
|
||
let effective_call_id = match (call_id, id) {
|
||
(Some(call_id), _) => call_id,
|
||
(None, Some(id)) => id,
|
||
(None, None) => {
|
||
error!("LocalShellCall without call_id or id");
|
||
return Ok(Some(ResponseInputItem::FunctionCallOutput {
|
||
call_id: "".to_string(),
|
||
output: FunctionCallOutputPayload {
|
||
content: "LocalShellCall without call_id or id".to_string(),
|
||
success: None,
|
||
},
|
||
}));
|
||
}
|
||
};
|
||
|
||
let exec_params = to_exec_params(params, sess);
|
||
Some(
|
||
handle_container_exec_with_params(
|
||
exec_params,
|
||
sess,
|
||
turn_diff_tracker,
|
||
sub_id.to_string(),
|
||
effective_call_id,
|
||
)
|
||
.await,
|
||
)
|
||
}
|
||
ResponseItem::FunctionCallOutput { .. } => {
|
||
debug!("unexpected FunctionCallOutput from stream");
|
||
None
|
||
}
|
||
ResponseItem::Other => None,
|
||
};
|
||
Ok(output)
|
||
}
|
||
|
||
async fn handle_function_call(
|
||
sess: &Session,
|
||
turn_diff_tracker: &mut TurnDiffTracker,
|
||
sub_id: String,
|
||
name: String,
|
||
arguments: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
match name.as_str() {
|
||
"container.exec" | "shell" => {
|
||
let params = match parse_container_exec_arguments(arguments, sess, &call_id) {
|
||
Ok(params) => params,
|
||
Err(output) => {
|
||
return *output;
|
||
}
|
||
};
|
||
handle_container_exec_with_params(params, sess, turn_diff_tracker, sub_id, call_id)
|
||
.await
|
||
}
|
||
"update_plan" => handle_update_plan(sess, arguments, sub_id, call_id).await,
|
||
"run_agent" => handle_run_agent(sess, arguments, sub_id, call_id).await,
|
||
"check_agent_status" => handle_check_agent_status(sess, arguments, sub_id, call_id).await,
|
||
"get_agent_result" => handle_get_agent_result(sess, arguments, sub_id, call_id).await,
|
||
"cancel_agent" => handle_cancel_agent(sess, arguments, sub_id, call_id).await,
|
||
"wait_for_agent" => handle_wait_for_agent(sess, arguments, sub_id, call_id).await,
|
||
"list_agents" => handle_list_agents(sess, arguments, sub_id, call_id).await,
|
||
"browser_open" => handle_browser_open(sess, arguments, sub_id, call_id).await,
|
||
"browser_close" => handle_browser_close(sess, sub_id, call_id).await,
|
||
"browser_status" => handle_browser_status(sess, sub_id, call_id).await,
|
||
"browser_click" => handle_browser_click(sess, arguments, sub_id, call_id).await,
|
||
"browser_move" => handle_browser_move(sess, arguments, sub_id, call_id).await,
|
||
"browser_type" => handle_browser_type(sess, arguments, sub_id, call_id).await,
|
||
"browser_key" => handle_browser_key(sess, arguments, sub_id, call_id).await,
|
||
"browser_javascript" => handle_browser_javascript(sess, arguments, sub_id, call_id).await,
|
||
"browser_scroll" => handle_browser_scroll(sess, arguments, sub_id, call_id).await,
|
||
"browser_history" => handle_browser_history(sess, arguments, sub_id, call_id).await,
|
||
_ => {
|
||
match sess.mcp_connection_manager.parse_tool_name(&name) {
|
||
Some((server, tool_name)) => {
|
||
// TODO(mbolin): Determine appropriate timeout for tool call.
|
||
let timeout = None;
|
||
handle_mcp_tool_call(
|
||
sess, &sub_id, call_id, server, tool_name, arguments, timeout,
|
||
)
|
||
.await
|
||
}
|
||
None => {
|
||
// Unknown function: reply with structured failure so the model can adapt.
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("unsupported call: {name}"),
|
||
success: None,
|
||
},
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
fn to_exec_params(params: ShellToolCallParams, sess: &Session) -> ExecParams {
|
||
ExecParams {
|
||
command: params.command,
|
||
cwd: sess.resolve_path(params.workdir.clone()),
|
||
timeout_ms: params.timeout_ms,
|
||
env: create_env(&sess.shell_environment_policy),
|
||
with_escalated_permissions: params.with_escalated_permissions,
|
||
justification: params.justification,
|
||
}
|
||
}
|
||
|
||
fn parse_container_exec_arguments(
|
||
arguments: String,
|
||
sess: &Session,
|
||
call_id: &str,
|
||
) -> Result<ExecParams, Box<ResponseInputItem>> {
|
||
// parse command
|
||
match serde_json::from_str::<ShellToolCallParams>(&arguments) {
|
||
Ok(shell_tool_call_params) => Ok(to_exec_params(shell_tool_call_params, sess)),
|
||
Err(e) => {
|
||
// allow model to re-sample
|
||
let output = ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id.to_string(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("failed to parse function arguments: {e}"),
|
||
success: None,
|
||
},
|
||
};
|
||
Err(Box::new(output))
|
||
}
|
||
}
|
||
}
|
||
|
||
pub struct ExecInvokeArgs<'a> {
|
||
pub params: ExecParams,
|
||
pub sandbox_type: SandboxType,
|
||
pub sandbox_policy: &'a SandboxPolicy,
|
||
pub codex_linux_sandbox_exe: &'a Option<PathBuf>,
|
||
pub stdout_stream: Option<StdoutStream>,
|
||
}
|
||
|
||
fn maybe_run_with_user_profile(params: ExecParams, sess: &Session) -> ExecParams {
|
||
if sess.shell_environment_policy.use_profile {
|
||
let command = sess
|
||
.user_shell
|
||
.format_default_shell_invocation(params.command.clone());
|
||
if let Some(command) = command {
|
||
return ExecParams { command, ..params };
|
||
}
|
||
}
|
||
params
|
||
}
|
||
|
||
async fn handle_run_agent(
|
||
sess: &Session,
|
||
arguments: String,
|
||
_sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
match serde_json::from_str::<RunAgentParams>(&arguments) {
|
||
Ok(params) => {
|
||
let mut manager = AGENT_MANAGER.write().await;
|
||
|
||
// Handle model parameter (can be string or array)
|
||
let models = match params.model {
|
||
Some(serde_json::Value::String(model)) => vec![model],
|
||
Some(serde_json::Value::Array(models)) => models
|
||
.into_iter()
|
||
.filter_map(|m| m.as_str().map(String::from))
|
||
.collect(),
|
||
_ => vec!["codex".to_string()], // Default model
|
||
};
|
||
|
||
let batch_id = if models.len() > 1 {
|
||
Some(Uuid::new_v4().to_string())
|
||
} else {
|
||
None
|
||
};
|
||
|
||
let mut agent_ids = Vec::new();
|
||
for model in models {
|
||
// Check if this model is configured and enabled
|
||
let agent_config = sess.agents.iter().find(|a| {
|
||
a.name.to_lowercase() == model.to_lowercase()
|
||
|| a.command.to_lowercase() == model.to_lowercase()
|
||
});
|
||
|
||
if let Some(config) = agent_config {
|
||
if !config.enabled {
|
||
continue; // Skip disabled agents
|
||
}
|
||
|
||
// Override read_only if agent is configured as read-only
|
||
let read_only = config.read_only || params.read_only.unwrap_or(false);
|
||
|
||
let agent_id = manager
|
||
.create_agent_with_config(
|
||
model,
|
||
params.task.clone(),
|
||
params.context.clone(),
|
||
params.output.clone(),
|
||
params.files.clone().unwrap_or_default(),
|
||
read_only,
|
||
batch_id.clone(),
|
||
config.clone(),
|
||
)
|
||
.await;
|
||
agent_ids.push(agent_id);
|
||
} else {
|
||
// Use default configuration for unknown agents
|
||
let agent_id = manager
|
||
.create_agent(
|
||
model,
|
||
params.task.clone(),
|
||
params.context.clone(),
|
||
params.output.clone(),
|
||
params.files.clone().unwrap_or_default(),
|
||
params.read_only.unwrap_or(false),
|
||
batch_id.clone(),
|
||
)
|
||
.await;
|
||
agent_ids.push(agent_id);
|
||
}
|
||
}
|
||
|
||
// Send agent status update event
|
||
drop(manager); // Release the write lock first
|
||
if agent_ids.len() > 0 {
|
||
send_agent_status_update(sess).await;
|
||
}
|
||
|
||
let response = if let Some(batch_id) = batch_id {
|
||
serde_json::json!({
|
||
"batch_id": batch_id,
|
||
"agent_ids": agent_ids,
|
||
"status": "started",
|
||
"message": format!("Started {} agents", agent_ids.len())
|
||
})
|
||
} else {
|
||
serde_json::json!({
|
||
"agent_id": agent_ids[0],
|
||
"status": "started",
|
||
"message": "Agent started successfully"
|
||
})
|
||
};
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: response.to_string(),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Invalid run_agent arguments: {}", e),
|
||
success: None,
|
||
},
|
||
},
|
||
}
|
||
}
|
||
|
||
async fn handle_check_agent_status(
|
||
_sess: &Session,
|
||
arguments: String,
|
||
_sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
match serde_json::from_str::<CheckAgentStatusParams>(&arguments) {
|
||
Ok(params) => {
|
||
let manager = AGENT_MANAGER.read().await;
|
||
|
||
if let Some(agent) = manager.get_agent(¶ms.agent_id) {
|
||
let response = serde_json::json!({
|
||
"agent_id": agent.id,
|
||
"status": agent.status,
|
||
"model": agent.model,
|
||
"created_at": agent.created_at,
|
||
"started_at": agent.started_at,
|
||
"completed_at": agent.completed_at,
|
||
"progress": agent.progress,
|
||
"error": agent.error,
|
||
"worktree_path": agent.worktree_path,
|
||
"branch_name": agent.branch_name,
|
||
});
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: response.to_string(),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Agent not found: {}", params.agent_id),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Invalid check_agent_status arguments: {}", e),
|
||
success: None,
|
||
},
|
||
},
|
||
}
|
||
}
|
||
|
||
async fn handle_get_agent_result(
|
||
_sess: &Session,
|
||
arguments: String,
|
||
_sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
match serde_json::from_str::<GetAgentResultParams>(&arguments) {
|
||
Ok(params) => {
|
||
let manager = AGENT_MANAGER.read().await;
|
||
|
||
if let Some(agent) = manager.get_agent(¶ms.agent_id) {
|
||
if agent.status == AgentStatus::Completed {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: agent
|
||
.result
|
||
.unwrap_or_else(|| "No result available".to_string()),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
} else if agent.status == AgentStatus::Failed {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"Agent failed: {}",
|
||
agent.error.unwrap_or_else(|| "Unknown error".to_string())
|
||
),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"Agent is still {}: cannot get result yet",
|
||
serde_json::to_string(&agent.status)
|
||
.unwrap_or_else(|_| "running".to_string())
|
||
),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Agent not found: {}", params.agent_id),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Invalid get_agent_result arguments: {}", e),
|
||
success: None,
|
||
},
|
||
},
|
||
}
|
||
}
|
||
|
||
async fn handle_cancel_agent(
|
||
_sess: &Session,
|
||
arguments: String,
|
||
_sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
match serde_json::from_str::<CancelAgentParams>(&arguments) {
|
||
Ok(params) => {
|
||
let mut manager = AGENT_MANAGER.write().await;
|
||
|
||
if let Some(agent_id) = params.agent_id {
|
||
if manager.cancel_agent(&agent_id).await {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Agent {} cancelled", agent_id),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to cancel agent {}", agent_id),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
} else if let Some(batch_id) = params.batch_id {
|
||
let count = manager.cancel_batch(&batch_id).await;
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Cancelled {} agents in batch {}", count, batch_id),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: "Either agent_id or batch_id must be provided".to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Invalid cancel_agent arguments: {}", e),
|
||
success: None,
|
||
},
|
||
},
|
||
}
|
||
}
|
||
|
||
async fn handle_wait_for_agent(
|
||
_sess: &Session,
|
||
arguments: String,
|
||
_sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
match serde_json::from_str::<WaitForAgentParams>(&arguments) {
|
||
Ok(params) => {
|
||
let timeout =
|
||
std::time::Duration::from_secs(params.timeout_seconds.unwrap_or(300).min(600));
|
||
let start = std::time::Instant::now();
|
||
|
||
loop {
|
||
if start.elapsed() > timeout {
|
||
return ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: "Timeout waiting for agent completion".to_string(),
|
||
success: Some(false),
|
||
},
|
||
};
|
||
}
|
||
|
||
let manager = AGENT_MANAGER.read().await;
|
||
|
||
if let Some(agent_id) = ¶ms.agent_id {
|
||
if let Some(agent) = manager.get_agent(agent_id) {
|
||
if matches!(
|
||
agent.status,
|
||
AgentStatus::Completed | AgentStatus::Failed | AgentStatus::Cancelled
|
||
) {
|
||
let response = serde_json::json!({
|
||
"agent_id": agent.id,
|
||
"status": agent.status,
|
||
"wait_time_seconds": start.elapsed().as_secs(),
|
||
});
|
||
return ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: response.to_string(),
|
||
success: Some(true),
|
||
},
|
||
};
|
||
}
|
||
}
|
||
} else if let Some(batch_id) = ¶ms.batch_id {
|
||
let agents = manager.list_agents(None, Some(batch_id.clone()), false);
|
||
let completed_agents: Vec<_> = agents
|
||
.into_iter()
|
||
.filter(|t| {
|
||
matches!(
|
||
t.status,
|
||
AgentStatus::Completed
|
||
| AgentStatus::Failed
|
||
| AgentStatus::Cancelled
|
||
)
|
||
})
|
||
.collect();
|
||
|
||
if !completed_agents.is_empty() {
|
||
let response = if params.return_all.unwrap_or(false) {
|
||
serde_json::json!({
|
||
"batch_id": batch_id,
|
||
"completed_agents": completed_agents.iter().map(|t| t.id.clone()).collect::<Vec<_>>(),
|
||
"wait_time_seconds": start.elapsed().as_secs(),
|
||
})
|
||
} else {
|
||
serde_json::json!({
|
||
"agent_id": completed_agents[0].id,
|
||
"status": completed_agents[0].status,
|
||
"wait_time_seconds": start.elapsed().as_secs(),
|
||
})
|
||
};
|
||
return ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: response.to_string(),
|
||
success: Some(true),
|
||
},
|
||
};
|
||
}
|
||
}
|
||
|
||
drop(manager);
|
||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Invalid wait_for_agent arguments: {}", e),
|
||
success: None,
|
||
},
|
||
},
|
||
}
|
||
}
|
||
|
||
async fn handle_list_agents(
|
||
sess: &Session,
|
||
arguments: String,
|
||
_sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
match serde_json::from_str::<ListAgentsParams>(&arguments) {
|
||
Ok(params) => {
|
||
let manager = AGENT_MANAGER.read().await;
|
||
|
||
let status_filter =
|
||
params
|
||
.status_filter
|
||
.and_then(|s| match s.to_lowercase().as_str() {
|
||
"pending" => Some(AgentStatus::Pending),
|
||
"running" => Some(AgentStatus::Running),
|
||
"completed" => Some(AgentStatus::Completed),
|
||
"failed" => Some(AgentStatus::Failed),
|
||
"cancelled" => Some(AgentStatus::Cancelled),
|
||
_ => None,
|
||
});
|
||
|
||
let agents = manager.list_agents(
|
||
status_filter,
|
||
params.batch_id,
|
||
params.recent_only.unwrap_or(false),
|
||
);
|
||
|
||
// Count running agents for status update
|
||
let running_count = agents
|
||
.iter()
|
||
.filter(|a| a.status == AgentStatus::Running)
|
||
.count();
|
||
if running_count > 0 {
|
||
let status_msg = format!(
|
||
"🤖 {} agent{} currently running",
|
||
running_count,
|
||
if running_count != 1 { "s" } else { "" }
|
||
);
|
||
let event = Event {
|
||
id: "agent-status".to_string(),
|
||
msg: EventMsg::BackgroundEvent(BackgroundEventEvent {
|
||
message: status_msg,
|
||
}),
|
||
};
|
||
let _ = sess.tx_event.send(event).await;
|
||
}
|
||
|
||
// Add status counts to summary
|
||
let pending_count = agents
|
||
.iter()
|
||
.filter(|a| a.status == AgentStatus::Pending)
|
||
.count();
|
||
let running_count = agents
|
||
.iter()
|
||
.filter(|a| a.status == AgentStatus::Running)
|
||
.count();
|
||
let completed_count = agents
|
||
.iter()
|
||
.filter(|a| a.status == AgentStatus::Completed)
|
||
.count();
|
||
let failed_count = agents
|
||
.iter()
|
||
.filter(|a| a.status == AgentStatus::Failed)
|
||
.count();
|
||
let cancelled_count = agents
|
||
.iter()
|
||
.filter(|a| a.status == AgentStatus::Cancelled)
|
||
.count();
|
||
|
||
let summary = serde_json::json!({
|
||
"total_agents": agents.len(),
|
||
"status_counts": {
|
||
"pending": pending_count,
|
||
"running": running_count,
|
||
"completed": completed_count,
|
||
"failed": failed_count,
|
||
"cancelled": cancelled_count,
|
||
},
|
||
"agents": agents.iter().map(|t| {
|
||
serde_json::json!({
|
||
"id": t.id,
|
||
"model": t.model,
|
||
"status": t.status,
|
||
"created_at": t.created_at,
|
||
"batch_id": t.batch_id,
|
||
"worktree_path": t.worktree_path,
|
||
"branch_name": t.branch_name,
|
||
})
|
||
}).collect::<Vec<_>>(),
|
||
});
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: summary.to_string(),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Invalid list_agents arguments: {}", e),
|
||
success: None,
|
||
},
|
||
},
|
||
}
|
||
}
|
||
|
||
async fn handle_container_exec_with_params(
|
||
params: ExecParams,
|
||
sess: &Session,
|
||
turn_diff_tracker: &mut TurnDiffTracker,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
// check if this was a patch, and apply it if so
|
||
let apply_patch_exec = match maybe_parse_apply_patch_verified(¶ms.command, ¶ms.cwd) {
|
||
MaybeApplyPatchVerified::Body(changes) => {
|
||
match apply_patch::apply_patch(sess, &sub_id, &call_id, changes).await {
|
||
InternalApplyPatchInvocation::Output(item) => return item,
|
||
InternalApplyPatchInvocation::DelegateToExec(apply_patch_exec) => {
|
||
Some(apply_patch_exec)
|
||
}
|
||
}
|
||
}
|
||
MaybeApplyPatchVerified::CorrectnessError(parse_error) => {
|
||
// It looks like an invocation of `apply_patch`, but we
|
||
// could not resolve it into a patch that would apply
|
||
// cleanly. Return to model for resample.
|
||
return ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("error: {parse_error:#}"),
|
||
success: None,
|
||
},
|
||
};
|
||
}
|
||
MaybeApplyPatchVerified::ShellParseError(error) => {
|
||
trace!("Failed to parse shell command, {error:?}");
|
||
None
|
||
}
|
||
MaybeApplyPatchVerified::NotApplyPatch => None,
|
||
};
|
||
|
||
let (params, safety, command_for_display) = match &apply_patch_exec {
|
||
Some(ApplyPatchExec {
|
||
action: ApplyPatchAction { patch, cwd, .. },
|
||
user_explicitly_approved_this_action,
|
||
}) => {
|
||
let path_to_codex = std::env::current_exe()
|
||
.ok()
|
||
.map(|p| p.to_string_lossy().to_string());
|
||
let Some(path_to_codex) = path_to_codex else {
|
||
return ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: "failed to determine path to codex executable".to_string(),
|
||
success: None,
|
||
},
|
||
};
|
||
};
|
||
|
||
let params = ExecParams {
|
||
command: vec![
|
||
path_to_codex,
|
||
CODEX_APPLY_PATCH_ARG1.to_string(),
|
||
patch.clone(),
|
||
],
|
||
cwd: cwd.clone(),
|
||
timeout_ms: params.timeout_ms,
|
||
env: HashMap::new(),
|
||
with_escalated_permissions: params.with_escalated_permissions,
|
||
justification: params.justification.clone(),
|
||
};
|
||
let safety = if *user_explicitly_approved_this_action {
|
||
SafetyCheck::AutoApprove {
|
||
sandbox_type: SandboxType::None,
|
||
}
|
||
} else {
|
||
assess_safety_for_untrusted_command(
|
||
sess.approval_policy,
|
||
&sess.sandbox_policy,
|
||
params.with_escalated_permissions.unwrap_or(false),
|
||
)
|
||
};
|
||
(
|
||
params,
|
||
safety,
|
||
vec!["apply_patch".to_string(), patch.clone()],
|
||
)
|
||
}
|
||
None => {
|
||
let safety = {
|
||
let state = sess.state.lock().unwrap();
|
||
assess_command_safety(
|
||
¶ms.command,
|
||
sess.approval_policy,
|
||
&sess.sandbox_policy,
|
||
&state.approved_commands,
|
||
params.with_escalated_permissions.unwrap_or(false),
|
||
)
|
||
};
|
||
let command_for_display = params.command.clone();
|
||
(params, safety, command_for_display)
|
||
}
|
||
};
|
||
|
||
let sandbox_type = match safety {
|
||
SafetyCheck::AutoApprove { sandbox_type } => sandbox_type,
|
||
SafetyCheck::AskUser => {
|
||
let rx_approve = sess
|
||
.request_command_approval(
|
||
sub_id.clone(),
|
||
call_id.clone(),
|
||
params.command.clone(),
|
||
params.cwd.clone(),
|
||
params.justification.clone(),
|
||
)
|
||
.await;
|
||
match rx_approve.await.unwrap_or_default() {
|
||
ReviewDecision::Approved => (),
|
||
ReviewDecision::ApprovedForSession => {
|
||
sess.add_approved_command(params.command.clone());
|
||
}
|
||
ReviewDecision::Denied | ReviewDecision::Abort => {
|
||
return ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: "exec command rejected by user".to_string(),
|
||
success: None,
|
||
},
|
||
};
|
||
}
|
||
}
|
||
// No sandboxing is applied because the user has given
|
||
// explicit approval. Often, we end up in this case because
|
||
// the command cannot be run in a sandbox, such as
|
||
// installing a new dependency that requires network access.
|
||
SandboxType::None
|
||
}
|
||
SafetyCheck::Reject { reason } => {
|
||
return ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("exec command rejected: {reason}"),
|
||
success: None,
|
||
},
|
||
};
|
||
}
|
||
};
|
||
|
||
let exec_command_context = ExecCommandContext {
|
||
sub_id: sub_id.clone(),
|
||
call_id: call_id.clone(),
|
||
command_for_display: command_for_display.clone(),
|
||
cwd: params.cwd.clone(),
|
||
apply_patch: apply_patch_exec.map(
|
||
|ApplyPatchExec {
|
||
action,
|
||
user_explicitly_approved_this_action,
|
||
}| ApplyPatchCommandContext {
|
||
user_explicitly_approved_this_action,
|
||
changes: convert_apply_patch_to_protocol(&action),
|
||
},
|
||
),
|
||
};
|
||
|
||
let params = maybe_run_with_user_profile(params, sess);
|
||
let output_result = sess
|
||
.run_exec_with_events(
|
||
turn_diff_tracker,
|
||
exec_command_context.clone(),
|
||
ExecInvokeArgs {
|
||
params: params.clone(),
|
||
sandbox_type,
|
||
sandbox_policy: &sess.sandbox_policy,
|
||
codex_linux_sandbox_exe: &sess.codex_linux_sandbox_exe,
|
||
stdout_stream: Some(StdoutStream {
|
||
sub_id: sub_id.clone(),
|
||
call_id: call_id.clone(),
|
||
tx_event: sess.tx_event.clone(),
|
||
}),
|
||
},
|
||
)
|
||
.await;
|
||
|
||
match output_result {
|
||
Ok(output) => {
|
||
let ExecToolCallOutput { exit_code, .. } = &output;
|
||
|
||
let is_success = *exit_code == 0;
|
||
let content = format_exec_output(output);
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content,
|
||
success: Some(is_success),
|
||
},
|
||
}
|
||
}
|
||
Err(CodexErr::Sandbox(error)) => {
|
||
handle_sandbox_error(
|
||
turn_diff_tracker,
|
||
params,
|
||
exec_command_context,
|
||
error,
|
||
sandbox_type,
|
||
sess,
|
||
)
|
||
.await
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("execution error: {e}"),
|
||
success: None,
|
||
},
|
||
},
|
||
}
|
||
}
|
||
|
||
async fn handle_sandbox_error(
|
||
turn_diff_tracker: &mut TurnDiffTracker,
|
||
params: ExecParams,
|
||
exec_command_context: ExecCommandContext,
|
||
error: SandboxErr,
|
||
sandbox_type: SandboxType,
|
||
sess: &Session,
|
||
) -> ResponseInputItem {
|
||
let call_id = exec_command_context.call_id.clone();
|
||
let sub_id = exec_command_context.sub_id.clone();
|
||
let cwd = exec_command_context.cwd.clone();
|
||
|
||
// Early out if either the user never wants to be asked for approval, or
|
||
// we're letting the model manage escalation requests. Otherwise, continue
|
||
match sess.approval_policy {
|
||
AskForApproval::Never | AskForApproval::OnRequest => {
|
||
return ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"failed in sandbox {sandbox_type:?} with execution error: {error}"
|
||
),
|
||
success: Some(false),
|
||
},
|
||
};
|
||
}
|
||
AskForApproval::UnlessTrusted | AskForApproval::OnFailure => (),
|
||
}
|
||
|
||
// similarly, if the command timed out, we can simply return this failure to the model
|
||
if matches!(error, SandboxErr::Timeout) {
|
||
return ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"command timed out after {} milliseconds",
|
||
params.timeout_duration().as_millis()
|
||
),
|
||
success: Some(false),
|
||
},
|
||
};
|
||
}
|
||
|
||
// Note that when `error` is `SandboxErr::Denied`, it could be a false
|
||
// positive. That is, it may have exited with a non-zero exit code, not
|
||
// because the sandbox denied it, but because that is its expected behavior,
|
||
// i.e., a grep command that did not match anything. Ideally we would
|
||
// include additional metadata on the command to indicate whether non-zero
|
||
// exit codes merit a retry.
|
||
|
||
// For now, we categorically ask the user to retry without sandbox and
|
||
// emit the raw error as a background event.
|
||
sess.notify_background_event(&sub_id, format!("Execution failed: {error}"))
|
||
.await;
|
||
|
||
let rx_approve = sess
|
||
.request_command_approval(
|
||
sub_id.clone(),
|
||
call_id.clone(),
|
||
params.command.clone(),
|
||
cwd.clone(),
|
||
Some("command failed; retry without sandbox?".to_string()),
|
||
)
|
||
.await;
|
||
|
||
match rx_approve.await.unwrap_or_default() {
|
||
ReviewDecision::Approved | ReviewDecision::ApprovedForSession => {
|
||
// Persist this command as pre‑approved for the
|
||
// remainder of the session so future
|
||
// executions skip the sandbox directly.
|
||
// TODO(ragona): Isn't this a bug? It always saves the command in an | fork?
|
||
sess.add_approved_command(params.command.clone());
|
||
// Inform UI we are retrying without sandbox.
|
||
sess.notify_background_event(&sub_id, "retrying command without sandbox")
|
||
.await;
|
||
|
||
// This is an escalated retry; the policy will not be
|
||
// examined and the sandbox has been set to `None`.
|
||
let retry_output_result = sess
|
||
.run_exec_with_events(
|
||
turn_diff_tracker,
|
||
exec_command_context.clone(),
|
||
ExecInvokeArgs {
|
||
params,
|
||
sandbox_type: SandboxType::None,
|
||
sandbox_policy: &sess.sandbox_policy,
|
||
codex_linux_sandbox_exe: &sess.codex_linux_sandbox_exe,
|
||
stdout_stream: Some(StdoutStream {
|
||
sub_id: sub_id.clone(),
|
||
call_id: call_id.clone(),
|
||
tx_event: sess.tx_event.clone(),
|
||
}),
|
||
},
|
||
)
|
||
.await;
|
||
|
||
match retry_output_result {
|
||
Ok(retry_output) => {
|
||
let ExecToolCallOutput { exit_code, .. } = &retry_output;
|
||
|
||
let is_success = *exit_code == 0;
|
||
let content = format_exec_output(retry_output);
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content,
|
||
success: Some(is_success),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("retry failed: {e}"),
|
||
success: None,
|
||
},
|
||
},
|
||
}
|
||
}
|
||
ReviewDecision::Denied | ReviewDecision::Abort => {
|
||
// Fall through to original failure handling.
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id,
|
||
output: FunctionCallOutputPayload {
|
||
content: "exec command rejected by user".to_string(),
|
||
success: None,
|
||
},
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Exec output is a pre-serialized JSON payload
|
||
fn format_exec_output(exec_output: ExecToolCallOutput) -> String {
|
||
let ExecToolCallOutput {
|
||
exit_code,
|
||
stdout,
|
||
stderr,
|
||
duration,
|
||
} = exec_output;
|
||
|
||
#[derive(Serialize)]
|
||
struct ExecMetadata {
|
||
exit_code: i32,
|
||
duration_seconds: f32,
|
||
}
|
||
|
||
#[derive(Serialize)]
|
||
struct ExecOutput<'a> {
|
||
output: &'a str,
|
||
metadata: ExecMetadata,
|
||
}
|
||
|
||
// round to 1 decimal place
|
||
let duration_seconds = ((duration.as_secs_f32()) * 10.0).round() / 10.0;
|
||
|
||
let is_success = exit_code == 0;
|
||
let output = if is_success { stdout } else { stderr };
|
||
|
||
let mut formatted_output = output.text;
|
||
if let Some(truncated_after_lines) = output.truncated_after_lines {
|
||
formatted_output.push_str(&format!(
|
||
"\n\n[Output truncated after {truncated_after_lines} lines: too many lines or bytes.]",
|
||
));
|
||
}
|
||
|
||
let payload = ExecOutput {
|
||
output: &formatted_output,
|
||
metadata: ExecMetadata {
|
||
exit_code,
|
||
duration_seconds,
|
||
},
|
||
};
|
||
|
||
#[expect(clippy::expect_used)]
|
||
serde_json::to_string(&payload).expect("serialize ExecOutput")
|
||
}
|
||
|
||
fn get_last_assistant_message_from_turn(responses: &[ResponseItem]) -> Option<String> {
|
||
responses.iter().rev().find_map(|item| {
|
||
if let ResponseItem::Message { role, content, .. } = item {
|
||
if role == "assistant" {
|
||
content.iter().rev().find_map(|ci| {
|
||
if let ContentItem::OutputText { text } = ci {
|
||
Some(text.clone())
|
||
} else {
|
||
None
|
||
}
|
||
})
|
||
} else {
|
||
None
|
||
}
|
||
} else {
|
||
None
|
||
}
|
||
})
|
||
}
|
||
|
||
async fn drain_to_completed(sess: &Session, sub_id: &str, prompt: &Prompt) -> CodexResult<()> {
|
||
let mut stream = sess.client.clone().stream(prompt).await?;
|
||
loop {
|
||
let maybe_event = stream.next().await;
|
||
let Some(event) = maybe_event else {
|
||
return Err(CodexErr::Stream(
|
||
"stream closed before response.completed".into(),
|
||
None,
|
||
));
|
||
};
|
||
match event {
|
||
Ok(ResponseEvent::OutputItemDone(item)) => {
|
||
// Record only to in-memory conversation history; avoid state snapshot.
|
||
let mut state = sess.state.lock().unwrap();
|
||
state.history.record_items(std::slice::from_ref(&item));
|
||
}
|
||
Ok(ResponseEvent::Completed {
|
||
response_id: _,
|
||
token_usage,
|
||
}) => {
|
||
let token_usage = match token_usage {
|
||
Some(usage) => usage,
|
||
None => {
|
||
return Err(CodexErr::Stream(
|
||
"token_usage was None in ResponseEvent::Completed".into(),
|
||
None,
|
||
));
|
||
}
|
||
};
|
||
sess.tx_event
|
||
.send(Event {
|
||
id: sub_id.to_string(),
|
||
msg: EventMsg::TokenCount(token_usage),
|
||
})
|
||
.await
|
||
.ok();
|
||
return Ok(());
|
||
}
|
||
Ok(_) => continue,
|
||
Err(e) => return Err(e),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Capture a screenshot from the browser and store it for the next model request
|
||
async fn capture_browser_screenshot(_sess: &Session) -> Result<(PathBuf, String), String> {
|
||
let browser_manager = codex_browser::global::get_browser_manager()
|
||
.await
|
||
.ok_or_else(|| "No browser manager available".to_string())?;
|
||
|
||
if !browser_manager.is_enabled().await {
|
||
return Err("Browser manager is not enabled".to_string());
|
||
}
|
||
|
||
// Get current URL first
|
||
let url = browser_manager
|
||
.get_current_url()
|
||
.await
|
||
.unwrap_or_else(|| "Browser".to_string());
|
||
tracing::debug!("Attempting to capture screenshot at URL: {}", url);
|
||
|
||
match browser_manager.capture_screenshot().await {
|
||
Ok(screenshots) => {
|
||
if let Some(first_screenshot) = screenshots.first() {
|
||
tracing::info!(
|
||
"Captured browser screenshot: {} at URL: {}",
|
||
first_screenshot.display(),
|
||
url
|
||
);
|
||
Ok((first_screenshot.clone(), url))
|
||
} else {
|
||
let msg = format!("Screenshot capture returned empty results at URL: {}", url);
|
||
tracing::warn!("{}", msg);
|
||
Err(msg)
|
||
}
|
||
}
|
||
Err(e) => {
|
||
let msg = format!("Failed to capture screenshot at {}: {}", url, e);
|
||
tracing::warn!("{}", msg);
|
||
Err(msg)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Send agent status update event to the TUI
|
||
async fn send_agent_status_update(sess: &Session) {
|
||
let manager = AGENT_MANAGER.read().await;
|
||
|
||
// Collect all active agents (not completed/failed/cancelled)
|
||
let agents: Vec<crate::protocol::AgentInfo> = manager
|
||
.get_all_agents()
|
||
.filter(|agent| {
|
||
!matches!(
|
||
agent.status,
|
||
AgentStatus::Completed | AgentStatus::Failed | AgentStatus::Cancelled
|
||
)
|
||
})
|
||
.map(|agent| crate::protocol::AgentInfo {
|
||
id: agent.id.clone(),
|
||
name: agent.model.clone(), // Use model name as the display name
|
||
status: match agent.status {
|
||
AgentStatus::Pending => "pending".to_string(),
|
||
AgentStatus::Running => "running".to_string(),
|
||
AgentStatus::Completed => "completed".to_string(),
|
||
AgentStatus::Failed => "failed".to_string(),
|
||
AgentStatus::Cancelled => "cancelled".to_string(),
|
||
},
|
||
model: Some(agent.model.clone()),
|
||
})
|
||
.collect();
|
||
|
||
let event = Event {
|
||
id: "agent_status".to_string(),
|
||
msg: EventMsg::AgentStatusUpdate(AgentStatusUpdateEvent {
|
||
agents,
|
||
context: None,
|
||
task: None,
|
||
}),
|
||
};
|
||
|
||
// Send event asynchronously
|
||
let tx_event = sess.tx_event.clone();
|
||
tokio::spawn(async move {
|
||
if let Err(e) = tx_event.send(event).await {
|
||
tracing::error!("Failed to send agent status update event: {}", e);
|
||
}
|
||
});
|
||
}
|
||
|
||
/// Add a screenshot to pending screenshots for the next model request
|
||
fn add_pending_screenshot(sess: &Session, screenshot_path: PathBuf, url: String) {
|
||
// Do not queue screenshots for next turn anymore; we inject fresh per-turn.
|
||
tracing::info!("Captured screenshot; updating UI and using per-turn injection");
|
||
|
||
// Also send an immediate event to update the TUI display
|
||
let event = Event {
|
||
id: "browser_screenshot".to_string(),
|
||
msg: EventMsg::BrowserScreenshotUpdate(BrowserScreenshotUpdateEvent {
|
||
screenshot_path,
|
||
url,
|
||
}),
|
||
};
|
||
|
||
// Send event asynchronously to avoid blocking
|
||
let tx_event = sess.tx_event.clone();
|
||
tokio::spawn(async move {
|
||
if let Err(e) = tx_event.send(event).await {
|
||
tracing::error!("Failed to send browser screenshot update event: {}", e);
|
||
}
|
||
});
|
||
}
|
||
|
||
/// Consume pending screenshots and return them as ResponseInputItems
|
||
#[allow(dead_code)]
|
||
fn consume_pending_screenshots(sess: &Session) -> Vec<ResponseInputItem> {
|
||
let mut pending = sess.pending_browser_screenshots.lock().unwrap();
|
||
let screenshots = pending.drain(..).collect::<Vec<_>>();
|
||
|
||
screenshots
|
||
.into_iter()
|
||
.map(|path| {
|
||
let metadata = format!(
|
||
"[EPHEMERAL:browser_screenshot] Browser screenshot at {}",
|
||
chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC")
|
||
);
|
||
|
||
// Read the screenshot file and create an ephemeral image
|
||
match std::fs::read(&path) {
|
||
Ok(bytes) => {
|
||
let mime = mime_guess::from_path(&path)
|
||
.first()
|
||
.map(|m| m.to_string())
|
||
.unwrap_or_else(|| "image/png".to_string());
|
||
let encoded = base64::engine::general_purpose::STANDARD.encode(bytes);
|
||
|
||
ResponseInputItem::Message {
|
||
role: "user".to_string(),
|
||
content: vec![
|
||
ContentItem::InputText { text: metadata },
|
||
ContentItem::InputImage {
|
||
image_url: format!("data:{mime};base64,{encoded}"),
|
||
detail: Some("high".to_string()),
|
||
},
|
||
],
|
||
}
|
||
}
|
||
Err(e) => {
|
||
tracing::error!("Failed to read screenshot {}: {}", path.display(), e);
|
||
ResponseInputItem::Message {
|
||
role: "user".to_string(),
|
||
content: vec![ContentItem::InputText {
|
||
text: format!("Failed to load browser screenshot: {}", e),
|
||
}],
|
||
}
|
||
}
|
||
}
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
/// Helper function to wrap custom tool calls with events
|
||
async fn execute_custom_tool<F, Fut>(
|
||
sess: &Session,
|
||
sub_id: &str,
|
||
call_id: String,
|
||
tool_name: String,
|
||
parameters: Option<serde_json::Value>,
|
||
tool_fn: F,
|
||
) -> ResponseInputItem
|
||
where
|
||
F: FnOnce() -> Fut,
|
||
Fut: std::future::Future<Output = ResponseInputItem>,
|
||
{
|
||
use crate::protocol::{CustomToolCallBeginEvent, CustomToolCallEndEvent};
|
||
use std::time::Instant;
|
||
|
||
// Send begin event
|
||
let begin_event = Event {
|
||
id: sub_id.to_string(),
|
||
msg: EventMsg::CustomToolCallBegin(CustomToolCallBeginEvent {
|
||
call_id: call_id.clone(),
|
||
tool_name: tool_name.clone(),
|
||
parameters: parameters.clone(),
|
||
}),
|
||
};
|
||
sess.send_event(begin_event).await;
|
||
|
||
// Execute the tool
|
||
let start = Instant::now();
|
||
let result = tool_fn().await;
|
||
let duration = start.elapsed();
|
||
|
||
// Extract success/failure from result
|
||
let (success, message) = match &result {
|
||
ResponseInputItem::FunctionCallOutput { output, .. } => {
|
||
let content = &output.content;
|
||
let is_error = content.contains("error") || content.contains("failed");
|
||
(!is_error, content.clone())
|
||
}
|
||
_ => (true, String::from("Tool completed")),
|
||
};
|
||
|
||
// Send end event
|
||
let end_event = Event {
|
||
id: sub_id.to_string(),
|
||
msg: EventMsg::CustomToolCallEnd(CustomToolCallEndEvent {
|
||
call_id: call_id.clone(),
|
||
tool_name,
|
||
parameters,
|
||
duration,
|
||
result: if success { Ok(message) } else { Err(message) },
|
||
}),
|
||
};
|
||
sess.send_event(end_event).await;
|
||
|
||
result
|
||
}
|
||
|
||
async fn handle_browser_open(
|
||
sess: &Session,
|
||
arguments: String,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
// Parse arguments as JSON for the event
|
||
let params = serde_json::from_str(&arguments).ok();
|
||
|
||
let sess_clone = sess;
|
||
let arguments_clone = arguments.clone();
|
||
let call_id_clone = call_id.clone();
|
||
|
||
execute_custom_tool(
|
||
sess,
|
||
&sub_id,
|
||
call_id,
|
||
"browser_open".to_string(),
|
||
params,
|
||
|| async move {
|
||
// Parse the URL from arguments
|
||
let args: Result<Value, _> = serde_json::from_str(&arguments_clone);
|
||
|
||
match args {
|
||
Ok(json) => {
|
||
let url = json
|
||
.get("url")
|
||
.and_then(|v| v.as_str())
|
||
.unwrap_or("about:blank");
|
||
|
||
// Use the global browser manager (create if needed)
|
||
let browser_manager = {
|
||
let existing_global = codex_browser::global::get_browser_manager().await;
|
||
if let Some(existing) = existing_global {
|
||
tracing::info!("Using existing global browser manager");
|
||
Some(existing)
|
||
} else {
|
||
tracing::info!("Creating new browser manager");
|
||
let new_manager =
|
||
codex_browser::global::get_or_create_browser_manager().await;
|
||
// Enable the browser
|
||
new_manager.set_enabled_sync(true);
|
||
Some(new_manager)
|
||
}
|
||
};
|
||
|
||
if let Some(browser_manager) = browser_manager {
|
||
// Navigate to the URL
|
||
match browser_manager.goto(url).await {
|
||
Ok(_) => {
|
||
tracing::info!(
|
||
"Browser navigation to {} completed successfully",
|
||
url
|
||
);
|
||
// Capture screenshot after navigation
|
||
match capture_browser_screenshot(sess_clone).await {
|
||
Ok((screenshot_path, updated_url)) => {
|
||
tracing::info!(
|
||
"Screenshot captured after navigation: {} at URL: {}",
|
||
screenshot_path.display(),
|
||
updated_url
|
||
);
|
||
add_pending_screenshot(
|
||
sess_clone,
|
||
screenshot_path,
|
||
updated_url,
|
||
);
|
||
}
|
||
Err(e) => {
|
||
tracing::warn!(
|
||
"Screenshot capture failed after navigation: {}",
|
||
e
|
||
);
|
||
}
|
||
}
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"Browser opened to: {} [new page shown in screenshot]",
|
||
url
|
||
),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"Failed to navigate browser to {}: {}",
|
||
url, e
|
||
),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: "Failed to initialize browser manager.".to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to parse browser_open arguments: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
|
||
/// Get the browser manager for the session (always uses global)
|
||
async fn get_browser_manager_for_session(
|
||
_sess: &Session,
|
||
) -> Option<Arc<codex_browser::BrowserManager>> {
|
||
// Always use the global browser manager
|
||
codex_browser::global::get_browser_manager().await
|
||
}
|
||
|
||
async fn handle_browser_close(
|
||
sess: &Session,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
let sess_clone = sess;
|
||
let call_id_clone = call_id.clone();
|
||
|
||
execute_custom_tool(
|
||
sess,
|
||
&sub_id,
|
||
call_id,
|
||
"browser_close".to_string(),
|
||
None,
|
||
|| async move {
|
||
let browser_manager = get_browser_manager_for_session(sess_clone).await;
|
||
if let Some(browser_manager) = browser_manager {
|
||
match browser_manager.stop().await {
|
||
Ok(_) => {
|
||
// Clear the browser manager from global
|
||
codex_browser::global::clear_browser_manager().await;
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: "Browser closed. Screenshot capture disabled.".to_string(),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to close browser: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content: "Browser is not currently open.".to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
|
||
async fn handle_browser_status(
|
||
sess: &Session,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
let sess_clone = sess;
|
||
let call_id_clone = call_id.clone();
|
||
|
||
execute_custom_tool(
|
||
sess,
|
||
&sub_id,
|
||
call_id,
|
||
"browser_status".to_string(),
|
||
None,
|
||
|| async move {
|
||
let browser_manager = get_browser_manager_for_session(sess_clone).await;
|
||
if let Some(browser_manager) = browser_manager {
|
||
let status = browser_manager.get_status().await;
|
||
let status_msg = if status.enabled {
|
||
if let Some(url) = status.current_url {
|
||
format!("Browser status: Enabled, currently at {}", url)
|
||
} else {
|
||
"Browser status: Enabled, no page loaded".to_string()
|
||
}
|
||
} else {
|
||
"Browser status: Disabled".to_string()
|
||
};
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: status_msg,
|
||
success: Some(true),
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content:
|
||
"Browser is not initialized. Use browser_open to start the browser."
|
||
.to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
|
||
async fn handle_browser_click(
|
||
sess: &Session,
|
||
arguments: String,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
let params = serde_json::from_str(&arguments).ok();
|
||
let sess_clone = sess;
|
||
let call_id_clone = call_id.clone();
|
||
|
||
execute_custom_tool(
|
||
sess,
|
||
&sub_id,
|
||
call_id,
|
||
"browser_click".to_string(),
|
||
params,
|
||
|| async move {
|
||
let browser_manager = get_browser_manager_for_session(sess_clone).await;
|
||
|
||
if let Some(browser_manager) = browser_manager {
|
||
// No arguments needed - click at current position
|
||
match browser_manager.click_at_current().await {
|
||
Ok((x, y)) => {
|
||
// Capture screenshot after clicking
|
||
if let Ok((screenshot_path, url)) = capture_browser_screenshot(sess_clone).await
|
||
{
|
||
add_pending_screenshot(sess_clone, screenshot_path, url);
|
||
}
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Clicked at current mouse position ({}, {}) [updated status shown in screenshot]", x, y),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to click: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content: "Browser is not initialized. Use browser_open to start the browser."
|
||
.to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
|
||
async fn handle_browser_move(
|
||
sess: &Session,
|
||
arguments: String,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
let params = serde_json::from_str(&arguments).ok();
|
||
let sess_clone = sess;
|
||
let arguments_clone = arguments.clone();
|
||
let call_id_clone = call_id.clone();
|
||
|
||
execute_custom_tool(
|
||
sess,
|
||
&sub_id,
|
||
call_id,
|
||
"browser_move".to_string(),
|
||
params,
|
||
|| async move {
|
||
let browser_manager = get_browser_manager_for_session(sess_clone).await;
|
||
|
||
if let Some(browser_manager) = browser_manager {
|
||
let args: Result<Value, _> = serde_json::from_str(&arguments_clone);
|
||
match args {
|
||
Ok(json) => {
|
||
// Check if we have relative movement (dx, dy) or absolute (x, y)
|
||
let has_dx = json.get("dx").is_some();
|
||
let has_dy = json.get("dy").is_some();
|
||
let has_x = json.get("x").is_some();
|
||
let has_y = json.get("y").is_some();
|
||
|
||
let result = if has_dx || has_dy {
|
||
// Relative movement
|
||
let dx = json.get("dx").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||
let dy = json.get("dy").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||
browser_manager.move_mouse_relative(dx, dy).await
|
||
} else if has_x || has_y {
|
||
// Absolute movement
|
||
let x = json.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||
let y = json.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||
browser_manager.move_mouse(x, y).await.map(|_| (x, y))
|
||
} else {
|
||
// No parameters provided, just return current position
|
||
browser_manager.get_cursor_position().await
|
||
};
|
||
|
||
match result {
|
||
Ok((x, y)) => {
|
||
// Capture screenshot after moving mouse
|
||
if let Ok((screenshot_path, url)) = capture_browser_screenshot(sess_clone).await
|
||
{
|
||
add_pending_screenshot(sess_clone, screenshot_path, url);
|
||
}
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Moved mouse position to ({}, {}) [updated mouse shown in screenshot]", x, y),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to move mouse: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to parse browser_move arguments: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content: "Browser is not initialized. Use browser_open to start the browser."
|
||
.to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
|
||
async fn handle_browser_type(
|
||
sess: &Session,
|
||
arguments: String,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
let params = serde_json::from_str(&arguments).ok();
|
||
let sess_clone = sess;
|
||
let arguments_clone = arguments.clone();
|
||
let call_id_clone = call_id.clone();
|
||
|
||
execute_custom_tool(
|
||
sess,
|
||
&sub_id,
|
||
call_id,
|
||
"browser_type".to_string(),
|
||
params,
|
||
|| async move {
|
||
let browser_manager = get_browser_manager_for_session(sess_clone).await;
|
||
if let Some(browser_manager) = browser_manager {
|
||
let args: Result<Value, _> = serde_json::from_str(&arguments_clone);
|
||
match args {
|
||
Ok(json) => {
|
||
let text = json.get("text").and_then(|v| v.as_str()).unwrap_or("");
|
||
|
||
match browser_manager.type_text(text).await {
|
||
Ok(_) => {
|
||
// Capture screenshot after typing
|
||
if let Ok((screenshot_path, url)) =
|
||
capture_browser_screenshot(sess_clone).await
|
||
{
|
||
add_pending_screenshot(sess_clone, screenshot_path, url);
|
||
}
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"Typed: {} [updated page shown in screenshot]",
|
||
text
|
||
),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to type text: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to parse browser_type arguments: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content:
|
||
"Browser is not initialized. Use browser_open to start the browser."
|
||
.to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
|
||
async fn handle_browser_key(
|
||
sess: &Session,
|
||
arguments: String,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
let params = serde_json::from_str(&arguments).ok();
|
||
let sess_clone = sess;
|
||
let arguments_clone = arguments.clone();
|
||
let call_id_clone = call_id.clone();
|
||
|
||
execute_custom_tool(
|
||
sess,
|
||
&sub_id,
|
||
call_id,
|
||
"browser_key".to_string(),
|
||
params,
|
||
|| async move {
|
||
let browser_manager = get_browser_manager_for_session(sess_clone).await;
|
||
if let Some(browser_manager) = browser_manager {
|
||
let args: Result<Value, _> = serde_json::from_str(&arguments_clone);
|
||
match args {
|
||
Ok(json) => {
|
||
let key = json.get("key").and_then(|v| v.as_str()).unwrap_or("");
|
||
|
||
match browser_manager.press_key(key).await {
|
||
Ok(_) => {
|
||
// Capture screenshot after pressing key
|
||
if let Ok((screenshot_path, url)) =
|
||
capture_browser_screenshot(sess_clone).await
|
||
{
|
||
add_pending_screenshot(sess_clone, screenshot_path, url);
|
||
}
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"Pressed key: {} [updated page shown in screenshot]",
|
||
key
|
||
),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to press key: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to parse browser_key arguments: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content:
|
||
"Browser is not initialized. Use browser_open to start the browser."
|
||
.to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
|
||
async fn handle_browser_javascript(
|
||
sess: &Session,
|
||
arguments: String,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
let params = serde_json::from_str(&arguments).ok();
|
||
let sess_clone = sess;
|
||
let arguments_clone = arguments.clone();
|
||
let call_id_clone = call_id.clone();
|
||
|
||
execute_custom_tool(
|
||
sess,
|
||
&sub_id,
|
||
call_id,
|
||
"browser_javascript".to_string(),
|
||
params,
|
||
|| async move {
|
||
let browser_manager = get_browser_manager_for_session(sess_clone).await;
|
||
if let Some(browser_manager) = browser_manager {
|
||
let args: Result<Value, _> = serde_json::from_str(&arguments_clone);
|
||
match args {
|
||
Ok(json) => {
|
||
let code = json.get("code").and_then(|v| v.as_str()).unwrap_or("");
|
||
|
||
match browser_manager.execute_javascript(code).await {
|
||
Ok(result) => {
|
||
// Log the JavaScript execution result
|
||
tracing::info!("JavaScript execution returned: {:?}", result);
|
||
|
||
// Format the result for the LLM
|
||
let formatted_result = if let Some(obj) = result.as_object() {
|
||
// Check if it's our wrapped result format
|
||
if let (Some(success), Some(value)) =
|
||
(obj.get("success"), obj.get("value"))
|
||
{
|
||
let logs = obj.get("logs").and_then(|v| v.as_array());
|
||
let mut output = String::new();
|
||
|
||
if let Some(logs) = logs {
|
||
if !logs.is_empty() {
|
||
output.push_str("Console logs:\n");
|
||
for log in logs {
|
||
if let Some(log_str) = log.as_str() {
|
||
output
|
||
.push_str(&format!(" {}\n", log_str));
|
||
}
|
||
}
|
||
output.push_str("\n");
|
||
}
|
||
}
|
||
|
||
if success.as_bool().unwrap_or(false) {
|
||
output.push_str("Result: ");
|
||
output.push_str(
|
||
&serde_json::to_string_pretty(value)
|
||
.unwrap_or_else(|_| "null".to_string()),
|
||
);
|
||
} else if let Some(error) = obj.get("error") {
|
||
output.push_str("Error: ");
|
||
output.push_str(&error.to_string());
|
||
}
|
||
|
||
output
|
||
} else {
|
||
// Fallback to raw JSON if not in expected format
|
||
serde_json::to_string_pretty(&result)
|
||
.unwrap_or_else(|_| "null".to_string())
|
||
}
|
||
} else {
|
||
// Not an object, return as-is
|
||
serde_json::to_string_pretty(&result)
|
||
.unwrap_or_else(|_| "null".to_string())
|
||
};
|
||
|
||
tracing::info!("Returning to LLM: {}", formatted_result);
|
||
|
||
// Capture screenshot after executing JavaScript
|
||
if let Ok((screenshot_path, url)) =
|
||
capture_browser_screenshot(sess_clone).await
|
||
{
|
||
add_pending_screenshot(sess_clone, screenshot_path, url);
|
||
}
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"{} [updated page shown in screenshot]",
|
||
formatted_result
|
||
),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to execute JavaScript: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to parse browser_javascript arguments: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content:
|
||
"Browser is not initialized. Use browser_open to start the browser."
|
||
.to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
|
||
async fn handle_browser_scroll(
|
||
sess: &Session,
|
||
arguments: String,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
let params = serde_json::from_str(&arguments).ok();
|
||
let sess_clone = sess;
|
||
let arguments_clone = arguments.clone();
|
||
let call_id_clone = call_id.clone();
|
||
|
||
execute_custom_tool(
|
||
sess,
|
||
&sub_id,
|
||
call_id,
|
||
"browser_scroll".to_string(),
|
||
params,
|
||
|| async move {
|
||
let browser_manager = get_browser_manager_for_session(sess_clone).await;
|
||
if let Some(browser_manager) = browser_manager {
|
||
let args: Result<Value, _> = serde_json::from_str(&arguments_clone);
|
||
match args {
|
||
Ok(json) => {
|
||
let dx = json.get("dx").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||
let dy = json.get("dy").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||
|
||
match browser_manager.scroll_by(dx, dy).await {
|
||
Ok(_) => {
|
||
if let Ok((screenshot_path, url)) = capture_browser_screenshot(sess_clone).await
|
||
{
|
||
add_pending_screenshot(sess_clone, screenshot_path, url);
|
||
}
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Scrolled by ({}, {}) [scrolled page shown in screenshot]", dx, dy),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to scroll: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to parse browser_scroll arguments: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content: "Browser is not initialized. Use browser_open to start the browser.".to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
|
||
async fn handle_browser_history(
|
||
sess: &Session,
|
||
arguments: String,
|
||
sub_id: String,
|
||
call_id: String,
|
||
) -> ResponseInputItem {
|
||
let params = serde_json::from_str(&arguments).ok();
|
||
let sess_clone = sess;
|
||
let arguments_clone = arguments.clone();
|
||
let call_id_clone = call_id.clone();
|
||
|
||
execute_custom_tool(
|
||
sess,
|
||
&sub_id,
|
||
call_id,
|
||
"browser_history".to_string(),
|
||
params,
|
||
|| async move {
|
||
let browser_manager = get_browser_manager_for_session(sess_clone).await;
|
||
if let Some(browser_manager) = browser_manager {
|
||
let args: Result<Value, _> = serde_json::from_str(&arguments_clone);
|
||
match args {
|
||
Ok(json) => {
|
||
let direction =
|
||
json.get("direction").and_then(|v| v.as_str()).unwrap_or("");
|
||
|
||
if direction != "back" && direction != "forward" {
|
||
return ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"Unsupported direction: {} (expected 'back' or 'forward')",
|
||
direction
|
||
),
|
||
success: Some(false),
|
||
},
|
||
};
|
||
}
|
||
|
||
let action_res = if direction == "back" {
|
||
browser_manager.history_back().await
|
||
} else {
|
||
browser_manager.history_forward().await
|
||
};
|
||
|
||
match action_res {
|
||
Ok(_) => {
|
||
if let Ok((screenshot_path, url)) =
|
||
capture_browser_screenshot(sess_clone).await
|
||
{
|
||
add_pending_screenshot(sess_clone, screenshot_path, url);
|
||
}
|
||
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!(
|
||
"History {} triggered [new page shown in screenshot]",
|
||
direction
|
||
),
|
||
success: Some(true),
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone.clone(),
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to navigate history: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
}
|
||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content: format!("Failed to parse browser_history arguments: {}", e),
|
||
success: Some(false),
|
||
},
|
||
},
|
||
}
|
||
} else {
|
||
ResponseInputItem::FunctionCallOutput {
|
||
call_id: call_id_clone,
|
||
output: FunctionCallOutputPayload {
|
||
content:
|
||
"Browser is not initialized. Use browser_open to start the browser."
|
||
.to_string(),
|
||
success: Some(false),
|
||
},
|
||
}
|
||
}
|
||
},
|
||
)
|
||
.await
|
||
}
|