hanzoai/dev
2
0
Fork 0
mirror of https://github.com/hanzoai/dev.git synced 2026-10-25 03:32:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add browser_console tool and conditional browser tool inclusion

Browse Source 
- Add new browser_console tool to capture and return console output from the browser
- Implement get_console_logs() method in BrowserManager with JavaScript hooks to capture console.log, warn, error, info, and debug messages
- Add handle_browser_console() handler in codex.rs to format console logs with timestamps and levels
- Modify get_openai_tools() to conditionally include browser tools based on browser state
- When browser is disabled: only browser_open and browser_status tools are available
- When browser is enabled: all browser interaction tools are included
- Browser state is determined by checking if global browser manager exists

This allows the LLM to debug web pages by capturing console output and ensures browser interaction tools are only available when a browser session is active.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>
This commit is contained in:
James Peter
2025-08-15 08:12:37 +10:00
co-authored by Claude
parent f37c31dd47
commit d6da1a47c6
3 changed files with 240 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/browser/src/manager.rs
+92
View File
@@ -1120,6 +1120,98 @@ impl BrowserManager {
page.go_forward().await
}
/// Capture console logs from the browser
pub async fn get_console_logs(&self, lines: Option<usize>) -> Result<serde_json::Value> {
let page = self.get_or_create_page().await?;
// JavaScript to capture console logs
let script = r#"
(() => {
// Get or create console log buffer
if (!window.__codex_console_logs) {
window.__codex_console_logs = [];
// Override console methods to capture logs
const originalLog = console.log;
const originalWarn = console.warn;
const originalError = console.error;
const originalInfo = console.info;
const originalDebug = console.debug;
const captureLog = (level, args) => {
const timestamp = new Date().toISOString();
const message = args.map(arg => {
try {
if (typeof arg === 'object') {
return JSON.stringify(arg, null, 2);
} else {
return String(arg);
}
} catch (e) {
return String(arg);
}
}).join(' ');
window.__codex_console_logs.push({
timestamp,
level,
message
});
// Keep only last 1000 logs to prevent memory issues
if (window.__codex_console_logs.length > 1000) {
window.__codex_console_logs.shift();
}
};
console.log = function(...args) {
captureLog('log', args);
originalLog.apply(console, args);
};
console.warn = function(...args) {
captureLog('warn', args);
originalWarn.apply(console, args);
};
console.error = function(...args) {
captureLog('error', args);
originalError.apply(console, args);
};
console.info = function(...args) {
captureLog('info', args);
originalInfo.apply(console, args);
};
console.debug = function(...args) {
captureLog('debug', args);
originalDebug.apply(console, args);
};
}
// Return the requested number of logs
const logs = window.__codex_console_logs || [];
const requestedLines = arguments[0];
if (requestedLines && requestedLines > 0) {
return logs.slice(-requestedLines);
} else {
return logs;
}
})()
"#;
// Pass the lines parameter to the script
let script_with_params = if let Some(lines) = lines {
format!("(({}) => {{{}}})({})", lines, script, lines)
} else {
format!("(() => {{{}}})()", script)
};
page.execute_javascript(&script_with_params).await
}
/// Get the current cursor position
pub async fn get_cursor_position(&self) -> Result<(f64, f64)> {
let page = self.get_or_create_page().await?;
codex-rs/core/src/codex.rs
+94
View File
@@ -348,6 +348,7 @@ impl Codex {
model: config.model.clone(),
model_reasoning_effort: config.model_reasoning_effort,
model_reasoning_summary: config.model_reasoning_summary,
model_text_verbosity: config.model_text_verbosity,
user_instructions,
base_instructions: config.base_instructions.clone(),
approval_policy: config.approval_policy,
@@ -1221,6 +1222,7 @@ async fn submission_loop(
model,
model_reasoning_effort,
model_reasoning_summary,
model_text_verbosity,
user_instructions,
base_instructions,
approval_policy,
@@ -1299,6 +1301,7 @@ async fn submission_loop(
provider.clone(),
model_reasoning_effort,
model_reasoning_summary,
model_text_verbosity,
session_id,
debug_logger,
);
@@ -1800,9 +1803,13 @@ async fn run_turn(
sub_id: String,
input: Vec<ResponseItem>,
) -> CodexResult<Vec<ProcessedResponseItem>> {
// Check if browser is enabled
let browser_enabled = codex_browser::global::get_browser_manager().await.is_some();
let tools = get_openai_tools(
&sess.tools_config,
Some(sess.mcp_connection_manager.list_all_tools()),
browser_enabled,
);
let mut retries = 0;
@@ -2310,6 +2317,7 @@ async fn handle_function_call(
"browser_javascript" => handle_browser_javascript(sess, arguments, sub_id, call_id).await,
"browser_scroll" => handle_browser_scroll(sess, arguments, sub_id, call_id).await,
"browser_history" => handle_browser_history(sess, arguments, sub_id, call_id).await,
"browser_console" => handle_browser_console(sess, arguments, sub_id, call_id).await,
_ => {
match sess.mcp_connection_manager.parse_tool_name(&name) {
Some((server, tool_name)) => {
@@ -4289,6 +4297,92 @@ async fn handle_browser_scroll(
.await
}
async fn handle_browser_console(
sess: &Session,
arguments: String,
sub_id: String,
call_id: String,
) -> ResponseInputItem {
let params = serde_json::from_str(&arguments).ok();
let sess_clone = sess;
let arguments_clone = arguments.clone();
let call_id_clone = call_id.clone();
execute_custom_tool(
sess,
&sub_id,
call_id,
"browser_console".to_string(),
params,
|| async move {
let browser_manager = get_browser_manager_for_session(sess_clone).await;
if let Some(browser_manager) = browser_manager {
let args: Result<Value, _> = serde_json::from_str(&arguments_clone);
let lines = match args {
Ok(json) => json.get("lines").and_then(|v| v.as_u64()).map(|n| n as usize),
Err(_) => None,
};
match browser_manager.get_console_logs(lines).await {
Ok(logs) => {
// Format the logs for display
let formatted = if let Some(logs_array) = logs.as_array() {
if logs_array.is_empty() {
"No console logs captured.".to_string()
} else {
let mut output = String::new();
output.push_str("Console logs:\n");
for log in logs_array {
if let Some(log_obj) = log.as_object() {
let timestamp = log_obj.get("timestamp")
.and_then(|v| v.as_str())
.unwrap_or("");
let level = log_obj.get("level")
.and_then(|v| v.as_str())
.unwrap_or("log");
let message = log_obj.get("message")
.and_then(|v| v.as_str())
.unwrap_or("");
output.push_str(&format!("[{}] [{}] {}\n", timestamp, level.to_uppercase(), message));
}
}
output
}
} else {
"No console logs captured.".to_string()
};
ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload {
content: formatted,
success: Some(true),
},
}
}
Err(e) => ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload {
content: format!("Failed to get console logs: {}", e),
success: Some(false),
},
},
}
} else {
ResponseInputItem::FunctionCallOutput {
call_id: call_id_clone,
output: FunctionCallOutputPayload {
content: "Browser is not enabled. Use browser_open to enable it first.".to_string(),
success: Some(false),
},
}
}
},
)
.await
}
async fn handle_browser_history(
sess: &Session,
arguments: String,
codex-rs/core/src/openai_tools.rs
+54 -20
View File
@@ -442,6 +442,7 @@ fn sanitize_json_schema(value: &mut JsonValue) {
pub(crate) fn get_openai_tools(
config: &ToolsConfig,
mcp_tools: Option<HashMap<String, mcp_types::Tool>>,
browser_enabled: bool,
) -> Vec<OpenAiTool> {
let mut tools: Vec<OpenAiTool> = Vec::new();
@@ -461,17 +462,24 @@ pub(crate) fn get_openai_tools(
tools.push(PLAN_TOOL.clone());
}
// Add browser tools
tools.push(create_browser_open_tool());
tools.push(create_browser_close_tool());
tools.push(create_browser_status_tool());
tools.push(create_browser_click_tool());
tools.push(create_browser_move_tool());
tools.push(create_browser_type_tool());
tools.push(create_browser_key_tool());
tools.push(create_browser_javascript_tool());
tools.push(create_browser_scroll_tool());
tools.push(create_browser_history_tool());
// Add browser tools only when browser is enabled
if browser_enabled {
tools.push(create_browser_open_tool());
tools.push(create_browser_close_tool());
tools.push(create_browser_status_tool());
tools.push(create_browser_click_tool());
tools.push(create_browser_move_tool());
tools.push(create_browser_type_tool());
tools.push(create_browser_key_tool());
tools.push(create_browser_javascript_tool());
tools.push(create_browser_scroll_tool());
tools.push(create_browser_history_tool());
tools.push(create_browser_console_tool());
} else {
// Only include browser_open and browser_status when browser is disabled
tools.push(create_browser_open_tool());
tools.push(create_browser_status_tool());
}
// Add agent management tools for calling external LLMs asynchronously
tools.push(create_run_agent_tool());
@@ -536,9 +544,9 @@ mod tests {
SandboxPolicy::ReadOnly,
true,
);
let tools = get_openai_tools(&config, Some(HashMap::new()));
let tools = get_openai_tools(&config, Some(HashMap::new()), false);
assert_eq_tool_names(&tools, &["local_shell", "update_plan"]);
assert_eq_tool_names(&tools, &["local_shell", "update_plan", "browser_open", "browser_status", "run_agent", "check_agent_status", "get_agent_result", "cancel_agent", "wait_for_agent", "list_agents"]);
}
#[test]
@@ -550,9 +558,9 @@ mod tests {
SandboxPolicy::ReadOnly,
true,
);
let tools = get_openai_tools(&config, Some(HashMap::new()));
let tools = get_openai_tools(&config, Some(HashMap::new()), false);
assert_eq_tool_names(&tools, &["shell", "update_plan"]);
assert_eq_tool_names(&tools, &["shell", "update_plan", "browser_open", "browser_status", "run_agent", "check_agent_status", "get_agent_result", "cancel_agent", "wait_for_agent", "list_agents"]);
}
#[test]
@@ -600,9 +608,10 @@ mod tests {
description: Some("Do something cool".to_string()),
},
)])),
false,
);
assert_eq_tool_names(&tools, &["shell", "test_server/do_something_cool"]);
assert_eq_tool_names(&tools, &["shell", "browser_open", "browser_status", "run_agent", "check_agent_status", "get_agent_result", "cancel_agent", "wait_for_agent", "list_agents", "test_server/do_something_cool"]);
assert_eq!(
tools[1],
@@ -679,9 +688,10 @@ mod tests {
description: Some("Search docs".to_string()),
},
)])),
false,
);
assert_eq_tool_names(&tools, &["shell", "dash/search"]);
assert_eq_tool_names(&tools, &["shell", "browser_open", "browser_status", "run_agent", "check_agent_status", "get_agent_result", "cancel_agent", "wait_for_agent", "list_agents", "dash/search"]);
assert_eq!(
tools[1],
@@ -732,9 +742,10 @@ mod tests {
description: Some("Pagination".to_string()),
},
)])),
false,
);
assert_eq_tool_names(&tools, &["shell", "dash/paginate"]);
assert_eq_tool_names(&tools, &["shell", "browser_open", "browser_status", "run_agent", "check_agent_status", "get_agent_result", "cancel_agent", "wait_for_agent", "list_agents", "dash/paginate"]);
assert_eq!(
tools[1],
OpenAiTool::Function(ResponsesApiTool {
@@ -782,9 +793,10 @@ mod tests {
description: Some("Tags".to_string()),
},
)])),
false,
);
assert_eq_tool_names(&tools, &["shell", "dash/tags"]);
assert_eq_tool_names(&tools, &["shell", "browser_open", "browser_status", "run_agent", "check_agent_status", "get_agent_result", "cancel_agent", "wait_for_agent", "list_agents", "dash/tags"]);
assert_eq!(
tools[1],
OpenAiTool::Function(ResponsesApiTool {
@@ -835,9 +847,10 @@ mod tests {
description: Some("AnyOf Value".to_string()),
},
)])),
false,
);
assert_eq_tool_names(&tools, &["shell", "dash/value"]);
assert_eq_tool_names(&tools, &["shell", "browser_open", "browser_status", "run_agent", "check_agent_status", "get_agent_result", "cancel_agent", "wait_for_agent", "list_agents", "dash/value"]);
assert_eq!(
tools[1],
OpenAiTool::Function(ResponsesApiTool {
@@ -1084,3 +1097,24 @@ fn create_browser_history_tool() -> OpenAiTool {
},
})
}
fn create_browser_console_tool() -> OpenAiTool {
let mut properties = BTreeMap::new();
properties.insert(
"lines".to_string(),
JsonSchema::Number {
description: Some("Optional: Number of recent console lines to return (default: all available)".to_string()),
},
);
OpenAiTool::Function(ResponsesApiTool {
name: "browser_console".to_string(),
description: "Captures and returns the console output from the browser, including logs, warnings, and errors.".to_string(),
strict: false,
parameters: JsonSchema::Object {
properties,
required: Some(vec![]),
additional_properties: Some(false),
},
})
}