mirror of
https://github.com/hanzoai/dev.git
synced 2026-10-25 03:32:32 +00:00
feat: add browser control tools with automatic screenshot feedback
- Add browser_click, browser_type, browser_key, browser_javascript tools - Implement automatic screenshot capture after browser actions - Fix JavaScript execution bug where code was not being executed - Add comprehensive logging for JavaScript results - Format JavaScript results with console logs and return values - Include ephemeral screenshots in tool responses for immediate LLM feedback - Add global browser manager singleton for state sharing - Fix "runtime within runtime" panic in TUI The LLM agent can now: - Control browser with click, type, and key press actions - Execute JavaScript and see console logs + return values - Receive visual feedback immediately after each action - Perform multi-step browser automation workflows 🤖 Generated with Claude Code Co-Authored-By: Claude <[email protected]>
This commit is contained in:
James Peter
Generated
+246
-4
@@ -283,6 +283,20 @@ dependencies = [
|
||||
"syn 2.0.104",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-tungstenite"
|
||||
version = "0.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c5359381fd414fbdb272c48f2111c16cb0bb3447bfacd59311ff3736da9f6664"
|
||||
dependencies = [
|
||||
"futures-io",
|
||||
"futures-util",
|
||||
"log",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tungstenite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atomic-waker"
|
||||
version = "1.1.2"
|
||||
@@ -503,6 +517,72 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
|
||||
|
||||
[[package]]
|
||||
name = "chromiumoxide"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8380ce7721cc895fe8a184c49d615fe755b0c9a3d7986355cee847439fff907f"
|
||||
dependencies = [
|
||||
"async-tungstenite",
|
||||
"base64 0.22.1",
|
||||
"cfg-if",
|
||||
"chromiumoxide_cdp",
|
||||
"chromiumoxide_types",
|
||||
"dunce",
|
||||
"fnv",
|
||||
"futures",
|
||||
"futures-timer",
|
||||
"pin-project-lite",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
"which",
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "chromiumoxide_cdp"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cadbfb52fa0aeca43626f6c42ca04184b108b786f8e45198dc41a42aedcf2e50"
|
||||
dependencies = [
|
||||
"chromiumoxide_pdl",
|
||||
"chromiumoxide_types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "chromiumoxide_pdl"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c197aeb42872c5d4c923e7d8ad46d99a58fd0fec37f6491554ff677a6791d3c9"
|
||||
dependencies = [
|
||||
"chromiumoxide_types",
|
||||
"either",
|
||||
"heck 0.4.1",
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "chromiumoxide_types"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "923486888790528d55ac37ec2f7483ed19eb8ccbb44701878e5856d1ceadf5d8"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.41"
|
||||
@@ -556,7 +636,7 @@ version = "4.5.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.104",
|
||||
@@ -617,6 +697,30 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "codex-browser"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"chromiumoxide",
|
||||
"chrono",
|
||||
"futures",
|
||||
"image",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.12",
|
||||
"tokio",
|
||||
"tokio-test",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "codex-chatgpt"
|
||||
version = "0.0.0"
|
||||
@@ -675,6 +779,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"chrono",
|
||||
"codex-apply-patch",
|
||||
"codex-browser",
|
||||
"codex-login",
|
||||
"codex-mcp-client",
|
||||
"core_test_support",
|
||||
@@ -870,6 +975,7 @@ dependencies = [
|
||||
"clap",
|
||||
"codex-ansi-escape",
|
||||
"codex-arg0",
|
||||
"codex-browser",
|
||||
"codex-common",
|
||||
"codex-core",
|
||||
"codex-file-search",
|
||||
@@ -1155,6 +1261,12 @@ dependencies = [
|
||||
"syn 2.0.104",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "data-encoding"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
|
||||
|
||||
[[package]]
|
||||
name = "deadpool"
|
||||
version = "0.10.0"
|
||||
@@ -1354,6 +1466,12 @@ version = "0.15.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
|
||||
|
||||
[[package]]
|
||||
name = "dunce"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
|
||||
|
||||
[[package]]
|
||||
name = "dupe"
|
||||
version = "0.9.1"
|
||||
@@ -1907,6 +2025,12 @@ dependencies = [
|
||||
"foldhash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
@@ -4394,7 +4518,7 @@ version = "0.26.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustversion",
|
||||
@@ -4407,7 +4531,7 @@ version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.104",
|
||||
@@ -4520,7 +4644,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349"
|
||||
dependencies = [
|
||||
"cfg-expr",
|
||||
"heck",
|
||||
"heck 0.5.0",
|
||||
"pkg-config",
|
||||
"toml 0.8.23",
|
||||
"version-compare",
|
||||
@@ -5069,6 +5193,24 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"data-encoding",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.8.5",
|
||||
"sha1",
|
||||
"thiserror 1.0.69",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.18.0"
|
||||
@@ -5146,6 +5288,12 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf-8"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
@@ -5375,6 +5523,18 @@ version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3"
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "6.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f"
|
||||
dependencies = [
|
||||
"either",
|
||||
"home",
|
||||
"rustix 0.38.44",
|
||||
"winsafe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "whoami"
|
||||
version = "1.6.0"
|
||||
@@ -5557,6 +5717,15 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
|
||||
dependencies = [
|
||||
"windows-targets 0.48.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.52.0"
|
||||
@@ -5584,6 +5753,21 @@ dependencies = [
|
||||
"windows-targets 0.53.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm 0.48.5",
|
||||
"windows_aarch64_msvc 0.48.5",
|
||||
"windows_i686_gnu 0.48.5",
|
||||
"windows_i686_msvc 0.48.5",
|
||||
"windows_x86_64_gnu 0.48.5",
|
||||
"windows_x86_64_gnullvm 0.48.5",
|
||||
"windows_x86_64_msvc 0.48.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.52.6"
|
||||
@@ -5616,6 +5800,12 @@ dependencies = [
|
||||
"windows_x86_64_msvc 0.53.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.6"
|
||||
@@ -5628,6 +5818,12 @@ version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.52.6"
|
||||
@@ -5640,6 +5836,12 @@ version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.52.6"
|
||||
@@ -5664,6 +5866,12 @@ version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.52.6"
|
||||
@@ -5676,6 +5884,12 @@ version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.52.6"
|
||||
@@ -5688,6 +5902,12 @@ version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.52.6"
|
||||
@@ -5700,6 +5920,12 @@ version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.52.6"
|
||||
@@ -5721,6 +5947,22 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winsafe"
|
||||
version = "0.0.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904"
|
||||
|
||||
[[package]]
|
||||
name = "wiremock"
|
||||
version = "0.6.4"
|
||||
|
||||
@@ -3,6 +3,7 @@ members = [
|
||||
"ansi-escape",
|
||||
"apply-patch",
|
||||
"arg0",
|
||||
"browser",
|
||||
"cli",
|
||||
"common",
|
||||
"core",
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
[package]
|
||||
edition = "2024"
|
||||
name = "codex-browser"
|
||||
version = { workspace = true }
|
||||
|
||||
[lib]
|
||||
name = "codex_browser"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
async-trait = "0.1"
|
||||
base64 = "0.22"
|
||||
bytes = "1"
|
||||
chromiumoxide = { version = "0.7", features = ["tokio-runtime"], default-features = false }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
futures = "0.3"
|
||||
image = "0.25"
|
||||
once_cell = "1.20"
|
||||
regex = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
tempfile = "3"
|
||||
thiserror = "2"
|
||||
tokio = { version = "1", features = ["fs", "io-util", "macros", "rt-multi-thread", "sync", "time"] }
|
||||
tracing = "0.1"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-test = "0.4"
|
||||
@@ -0,0 +1,138 @@
|
||||
use crate::{config::ImageFormat, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::fs;
|
||||
use uuid::Uuid;
|
||||
use chrono::{DateTime, Utc, Duration};
|
||||
use std::collections::HashMap;
|
||||
use tokio::sync::RwLock;
|
||||
use std::sync::Arc;
|
||||
use serde::{Serialize, Deserialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ImageRef {
|
||||
pub path: String,
|
||||
pub mime: String,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub ttl_ms: u64,
|
||||
pub created_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
pub struct AssetManager {
|
||||
base_dir: PathBuf,
|
||||
#[allow(dead_code)]
|
||||
session_id: String,
|
||||
assets: Arc<RwLock<HashMap<String, ImageRef>>>,
|
||||
}
|
||||
|
||||
impl AssetManager {
|
||||
pub async fn new() -> Result<Self> {
|
||||
let session_id = Uuid::new_v4().to_string();
|
||||
let base_dir = PathBuf::from("/tmp/codex/browser").join(&session_id);
|
||||
|
||||
fs::create_dir_all(&base_dir).await?;
|
||||
|
||||
Ok(Self {
|
||||
base_dir,
|
||||
session_id,
|
||||
assets: Arc::new(RwLock::new(HashMap::new())),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn store_screenshot(
|
||||
&self,
|
||||
data: &[u8],
|
||||
format: ImageFormat,
|
||||
width: u32,
|
||||
height: u32,
|
||||
ttl_ms: u64,
|
||||
) -> Result<ImageRef> {
|
||||
let filename = format!("{}.{}", Uuid::new_v4(), match format {
|
||||
ImageFormat::Png => "png",
|
||||
ImageFormat::Webp => "webp",
|
||||
});
|
||||
|
||||
let path = self.base_dir.join(&filename);
|
||||
fs::write(&path, data).await?;
|
||||
|
||||
let mime = match format {
|
||||
ImageFormat::Png => "image/png",
|
||||
ImageFormat::Webp => "image/webp",
|
||||
}.to_string();
|
||||
|
||||
let image_ref = ImageRef {
|
||||
path: path.to_string_lossy().to_string(),
|
||||
mime,
|
||||
width,
|
||||
height,
|
||||
ttl_ms,
|
||||
created_at: Utc::now(),
|
||||
};
|
||||
|
||||
let mut assets = self.assets.write().await;
|
||||
assets.insert(filename, image_ref.clone());
|
||||
|
||||
Ok(image_ref)
|
||||
}
|
||||
|
||||
pub async fn store_screenshots(
|
||||
&self,
|
||||
screenshots: Vec<crate::page::Screenshot>,
|
||||
ttl_ms: u64,
|
||||
) -> Result<Vec<ImageRef>> {
|
||||
let mut refs = Vec::new();
|
||||
|
||||
for screenshot in screenshots {
|
||||
let image_ref = self.store_screenshot(
|
||||
&screenshot.data,
|
||||
screenshot.format,
|
||||
screenshot.width,
|
||||
screenshot.height,
|
||||
ttl_ms,
|
||||
).await?;
|
||||
refs.push(image_ref);
|
||||
}
|
||||
|
||||
Ok(refs)
|
||||
}
|
||||
|
||||
pub async fn cleanup_expired(&self) -> Result<()> {
|
||||
let now = Utc::now();
|
||||
let mut assets = self.assets.write().await;
|
||||
let mut to_remove = Vec::new();
|
||||
|
||||
for (key, asset) in assets.iter() {
|
||||
let age = now - asset.created_at;
|
||||
if age > Duration::milliseconds(asset.ttl_ms as i64) {
|
||||
to_remove.push(key.clone());
|
||||
let _ = fs::remove_file(&asset.path).await;
|
||||
}
|
||||
}
|
||||
|
||||
for key in to_remove {
|
||||
assets.remove(&key);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn cleanup_all(&self) -> Result<()> {
|
||||
if self.base_dir.exists() {
|
||||
fs::remove_dir_all(&self.base_dir).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_session_dir(&self) -> &Path {
|
||||
&self.base_dir
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AssetManager {
|
||||
fn drop(&mut self) {
|
||||
let base_dir = self.base_dir.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = fs::remove_dir_all(&base_dir).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BrowserConfig {
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
|
||||
#[serde(default = "default_viewport")]
|
||||
pub viewport: ViewportConfig,
|
||||
|
||||
#[serde(default = "default_wait")]
|
||||
pub wait: WaitStrategy,
|
||||
|
||||
#[serde(default)]
|
||||
pub fullpage: bool,
|
||||
|
||||
#[serde(default = "default_segments_max")]
|
||||
pub segments_max: usize,
|
||||
|
||||
#[serde(default = "default_idle_timeout_ms")]
|
||||
pub idle_timeout_ms: u64,
|
||||
|
||||
#[serde(default = "default_format")]
|
||||
pub format: ImageFormat,
|
||||
}
|
||||
|
||||
impl Default for BrowserConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
viewport: default_viewport(),
|
||||
wait: default_wait(),
|
||||
fullpage: false,
|
||||
segments_max: default_segments_max(),
|
||||
idle_timeout_ms: default_idle_timeout_ms(),
|
||||
format: default_format(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ViewportConfig {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
|
||||
#[serde(default = "default_device_scale_factor")]
|
||||
pub device_scale_factor: f64,
|
||||
|
||||
#[serde(default)]
|
||||
pub mobile: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum WaitStrategy {
|
||||
Event(String),
|
||||
Delay { delay_ms: u64 },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ImageFormat {
|
||||
Png,
|
||||
Webp,
|
||||
}
|
||||
|
||||
fn default_viewport() -> ViewportConfig {
|
||||
ViewportConfig {
|
||||
width: 1024,
|
||||
height: 768,
|
||||
device_scale_factor: 1.0,
|
||||
mobile: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn default_wait() -> WaitStrategy {
|
||||
WaitStrategy::Event("networkidle".to_string())
|
||||
}
|
||||
|
||||
fn default_segments_max() -> usize {
|
||||
8
|
||||
}
|
||||
|
||||
fn default_idle_timeout_ms() -> u64 {
|
||||
60000
|
||||
}
|
||||
|
||||
fn default_device_scale_factor() -> f64 {
|
||||
1.0
|
||||
}
|
||||
|
||||
fn default_format() -> ImageFormat {
|
||||
ImageFormat::Png
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
use crate::manager::BrowserManager;
|
||||
use crate::config::BrowserConfig;
|
||||
use std::sync::Arc;
|
||||
use once_cell::sync::Lazy;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// Global browser manager instance shared between TUI and Session
|
||||
static GLOBAL_BROWSER_MANAGER: Lazy<Arc<RwLock<Option<Arc<BrowserManager>>>>> =
|
||||
Lazy::new(|| Arc::new(RwLock::new(None)));
|
||||
|
||||
/// Get or create the global browser manager
|
||||
pub async fn get_or_create_browser_manager() -> Arc<BrowserManager> {
|
||||
let mut guard = GLOBAL_BROWSER_MANAGER.write().await;
|
||||
if let Some(manager) = guard.as_ref() {
|
||||
manager.clone()
|
||||
} else {
|
||||
let mut config = BrowserConfig::default();
|
||||
config.enabled = true;
|
||||
let manager = Arc::new(BrowserManager::new(config));
|
||||
manager.set_enabled_sync(true);
|
||||
*guard = Some(manager.clone());
|
||||
manager
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the global browser manager if it exists
|
||||
pub async fn get_browser_manager() -> Option<Arc<BrowserManager>> {
|
||||
GLOBAL_BROWSER_MANAGER.read().await.clone()
|
||||
}
|
||||
|
||||
/// Clear the global browser manager
|
||||
pub async fn clear_browser_manager() {
|
||||
*GLOBAL_BROWSER_MANAGER.write().await = None;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
use crate::{
|
||||
assets::{AssetManager, ImageRef},
|
||||
manager::BrowserManager,
|
||||
page::ScreenshotMode,
|
||||
Result,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, info};
|
||||
|
||||
pub struct BrowserInjector {
|
||||
manager: Arc<BrowserManager>,
|
||||
asset_manager: Arc<AssetManager>,
|
||||
}
|
||||
|
||||
impl BrowserInjector {
|
||||
pub fn new(manager: Arc<BrowserManager>, asset_manager: Arc<AssetManager>) -> Self {
|
||||
Self {
|
||||
manager,
|
||||
asset_manager,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn inject_pre_llm_call(&self) -> Result<Option<InjectionResult>> {
|
||||
if !self.manager.is_enabled().await {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
debug!("Browser enabled, capturing pre-LLM screenshot");
|
||||
|
||||
let page = match self.manager.get_or_create_page().await {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
info!("No active page, using about:blank");
|
||||
let page = self.manager.get_or_create_page().await?;
|
||||
page.goto("about:blank", None).await?;
|
||||
page
|
||||
}
|
||||
};
|
||||
|
||||
let config = self.manager.get_config().await;
|
||||
let mode = if config.fullpage {
|
||||
ScreenshotMode::FullPage {
|
||||
segments_max: Some(config.segments_max),
|
||||
}
|
||||
} else {
|
||||
ScreenshotMode::Viewport
|
||||
};
|
||||
|
||||
let screenshots = page.screenshot(mode).await?;
|
||||
let ttl_ms = 300000;
|
||||
let images = self.asset_manager.store_screenshots(screenshots, ttl_ms).await?;
|
||||
|
||||
let current_url = page.get_url().await.unwrap_or_else(|_| "about:blank".to_string());
|
||||
|
||||
let system_hint = format!(
|
||||
"A fresh screenshot of the active page ({}) is attached; use browser.* tools to navigate or capture more.",
|
||||
current_url
|
||||
);
|
||||
|
||||
let segments_captured = images.len();
|
||||
|
||||
Ok(Some(InjectionResult {
|
||||
images,
|
||||
system_hint,
|
||||
metadata: InjectionMetadata {
|
||||
url: current_url,
|
||||
fullpage: config.fullpage,
|
||||
segments_captured,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn cleanup_expired(&self) -> Result<()> {
|
||||
self.asset_manager.cleanup_expired().await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct InjectionResult {
|
||||
pub images: Vec<ImageRef>,
|
||||
pub system_hint: String,
|
||||
pub metadata: InjectionMetadata,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct InjectionMetadata {
|
||||
pub url: String,
|
||||
pub fullpage: bool,
|
||||
pub segments_captured: usize,
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod browser_injector;
|
||||
|
||||
pub use browser_injector::{BrowserInjector, InjectionResult};
|
||||
@@ -0,0 +1,45 @@
|
||||
pub mod assets;
|
||||
pub mod config;
|
||||
pub mod global;
|
||||
pub mod hooks;
|
||||
pub mod manager;
|
||||
pub mod page;
|
||||
pub mod tools;
|
||||
|
||||
pub use config::{BrowserConfig, ViewportConfig, WaitStrategy};
|
||||
pub use manager::BrowserManager;
|
||||
pub use page::{Page, ScreenshotMode, ScreenshotRegion};
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum BrowserError {
|
||||
#[error("Browser not initialized")]
|
||||
NotInitialized,
|
||||
|
||||
#[error("Page not loaded")]
|
||||
PageNotLoaded,
|
||||
|
||||
#[error("CDP error: {0}")]
|
||||
CdpError(String),
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
|
||||
#[error("Screenshot failed: {0}")]
|
||||
ScreenshotError(String),
|
||||
|
||||
#[error("Invalid configuration: {0}")]
|
||||
ConfigError(String),
|
||||
|
||||
#[error("Asset storage error: {0}")]
|
||||
AssetError(String),
|
||||
}
|
||||
|
||||
impl From<chromiumoxide::error::CdpError> for BrowserError {
|
||||
fn from(e: chromiumoxide::error::CdpError) -> Self {
|
||||
BrowserError::CdpError(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, BrowserError>;
|
||||
@@ -0,0 +1,412 @@
|
||||
use crate::{config::BrowserConfig, page::Page, BrowserError, Result};
|
||||
use chromiumoxide::{Browser, BrowserConfig as CdpConfig};
|
||||
use futures::StreamExt;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tokio::time::{sleep, Duration, Instant};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
pub struct BrowserManager {
|
||||
pub config: Arc<RwLock<BrowserConfig>>,
|
||||
browser: Arc<Mutex<Option<Browser>>>,
|
||||
page: Arc<Mutex<Option<Arc<Page>>>>,
|
||||
last_activity: Arc<Mutex<Instant>>,
|
||||
idle_monitor_handle: Arc<Mutex<Option<tokio::task::JoinHandle<()>>>>,
|
||||
assets: Arc<Mutex<Option<Arc<crate::assets::AssetManager>>>>,
|
||||
}
|
||||
|
||||
impl BrowserManager {
|
||||
pub fn new(config: BrowserConfig) -> Self {
|
||||
Self {
|
||||
config: Arc::new(RwLock::new(config)),
|
||||
browser: Arc::new(Mutex::new(None)),
|
||||
page: Arc::new(Mutex::new(None)),
|
||||
last_activity: Arc::new(Mutex::new(Instant::now())),
|
||||
idle_monitor_handle: Arc::new(Mutex::new(None)),
|
||||
assets: Arc::new(Mutex::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn start(&self) -> Result<()> {
|
||||
let mut browser_guard = self.browser.lock().await;
|
||||
if browser_guard.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!("Starting browser");
|
||||
|
||||
let (browser, mut handler) = Browser::launch(
|
||||
CdpConfig::builder()
|
||||
.window_size(1024, 768)
|
||||
.build()
|
||||
.map_err(|e| BrowserError::CdpError(e.to_string()))?
|
||||
)
|
||||
.await?;
|
||||
|
||||
let _handle = tokio::spawn(async move {
|
||||
while let Some(event) = handler.next().await {
|
||||
debug!("Browser event: {:?}", event);
|
||||
}
|
||||
});
|
||||
|
||||
*browser_guard = Some(browser);
|
||||
|
||||
self.start_idle_monitor().await;
|
||||
self.update_activity().await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn stop(&self) -> Result<()> {
|
||||
self.stop_idle_monitor().await;
|
||||
|
||||
let mut page_guard = self.page.lock().await;
|
||||
*page_guard = None;
|
||||
|
||||
let mut browser_guard = self.browser.lock().await;
|
||||
if let Some(mut browser) = browser_guard.take() {
|
||||
info!("Stopping browser");
|
||||
browser.close().await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_or_create_page(&self) -> Result<Arc<Page>> {
|
||||
self.ensure_browser().await?;
|
||||
self.update_activity().await;
|
||||
|
||||
let mut page_guard = self.page.lock().await;
|
||||
if let Some(page) = page_guard.as_ref() {
|
||||
return Ok(Arc::clone(page));
|
||||
}
|
||||
|
||||
let browser_guard = self.browser.lock().await;
|
||||
let browser = browser_guard
|
||||
.as_ref()
|
||||
.ok_or(BrowserError::NotInitialized)?;
|
||||
|
||||
let cdp_page = browser.new_page("about:blank").await?;
|
||||
let config = self.config.read().await;
|
||||
|
||||
// Set viewport dimensions using CDP protocol
|
||||
use chromiumoxide::cdp::browser_protocol::emulation::SetDeviceMetricsOverrideParams;
|
||||
let params = SetDeviceMetricsOverrideParams::builder()
|
||||
.width(config.viewport.width as i64)
|
||||
.height(config.viewport.height as i64)
|
||||
.device_scale_factor(config.viewport.device_scale_factor)
|
||||
.mobile(config.viewport.mobile)
|
||||
.build()
|
||||
.map_err(|e| BrowserError::CdpError(e))?;
|
||||
|
||||
cdp_page.execute(params).await?;
|
||||
|
||||
let page = Arc::new(Page::new(cdp_page, config.clone()));
|
||||
*page_guard = Some(Arc::clone(&page));
|
||||
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
pub async fn close_page(&self) -> Result<()> {
|
||||
let mut page_guard = self.page.lock().await;
|
||||
if let Some(page) = page_guard.take() {
|
||||
page.close().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn is_enabled(&self) -> bool {
|
||||
self.config.read().await.enabled
|
||||
}
|
||||
|
||||
pub async fn set_enabled(&self, enabled: bool) -> Result<()> {
|
||||
let mut config = self.config.write().await;
|
||||
config.enabled = enabled;
|
||||
|
||||
if enabled {
|
||||
self.start().await?;
|
||||
} else {
|
||||
self.stop().await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn update_config(&self, updates: impl FnOnce(&mut BrowserConfig)) -> Result<()> {
|
||||
let mut config = self.config.write().await;
|
||||
updates(&mut config);
|
||||
|
||||
if let Some(page) = self.page.lock().await.as_ref() {
|
||||
page.update_viewport(config.viewport.clone()).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_config(&self) -> BrowserConfig {
|
||||
self.config.read().await.clone()
|
||||
}
|
||||
|
||||
pub async fn get_status(&self) -> BrowserStatus {
|
||||
let config = self.config.read().await;
|
||||
let browser_active = self.browser.lock().await.is_some();
|
||||
let page_guard = self.page.lock().await;
|
||||
let current_url = if let Some(page) = page_guard.as_ref() {
|
||||
page.get_url().await.ok()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
BrowserStatus {
|
||||
enabled: config.enabled,
|
||||
browser_active,
|
||||
current_url,
|
||||
viewport: config.viewport.clone(),
|
||||
fullpage: config.fullpage,
|
||||
}
|
||||
}
|
||||
|
||||
async fn ensure_browser(&self) -> Result<()> {
|
||||
let browser_guard = self.browser.lock().await;
|
||||
if browser_guard.is_none() {
|
||||
drop(browser_guard);
|
||||
self.start().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn update_activity(&self) {
|
||||
let mut last_activity = self.last_activity.lock().await;
|
||||
*last_activity = Instant::now();
|
||||
}
|
||||
|
||||
pub fn set_enabled_sync(&self, enabled: bool) {
|
||||
// Try to set immediately if possible, otherwise spawn a task
|
||||
if let Ok(mut cfg) = self.config.try_write() {
|
||||
cfg.enabled = enabled;
|
||||
} else {
|
||||
let config = self.config.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut cfg = config.write().await;
|
||||
cfg.enabled = enabled;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_fullpage_sync(&self, fullpage: bool) {
|
||||
if let Ok(mut cfg) = self.config.try_write() {
|
||||
cfg.fullpage = fullpage;
|
||||
} else {
|
||||
let config = self.config.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut cfg = config.write().await;
|
||||
cfg.fullpage = fullpage;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_viewport_sync(&self, width: u32, height: u32) {
|
||||
if let Ok(mut cfg) = self.config.try_write() {
|
||||
cfg.viewport.width = width;
|
||||
cfg.viewport.height = height;
|
||||
} else {
|
||||
let config = self.config.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut cfg = config.write().await;
|
||||
cfg.viewport.width = width;
|
||||
cfg.viewport.height = height;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_segments_max_sync(&self, segments_max: usize) {
|
||||
if let Ok(mut cfg) = self.config.try_write() {
|
||||
cfg.segments_max = segments_max;
|
||||
} else {
|
||||
let config = self.config.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut cfg = config.write().await;
|
||||
cfg.segments_max = segments_max;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_status_sync(&self) -> String {
|
||||
// Use try operations to avoid blocking - return cached/default values if locks are held
|
||||
let cfg = self.config.try_read().map(|c| {
|
||||
let enabled = c.enabled;
|
||||
let viewport_width = c.viewport.width;
|
||||
let viewport_height = c.viewport.height;
|
||||
let fullpage = c.fullpage;
|
||||
(enabled, viewport_width, viewport_height, fullpage)
|
||||
}).unwrap_or((false, 1024, 768, false));
|
||||
|
||||
let browser_active = self.browser.try_lock().map(|b| b.is_some()).unwrap_or(false);
|
||||
|
||||
let mode = if cfg.0 { "enabled" } else { "disabled" };
|
||||
let fullpage = if cfg.3 { "on" } else { "off" };
|
||||
|
||||
let mut status = format!(
|
||||
"Browser status:\n• Mode: {}\n• Viewport: {}×{}\n• Full-page: {}",
|
||||
mode, cfg.1, cfg.2, fullpage
|
||||
);
|
||||
|
||||
if browser_active {
|
||||
status.push_str("\n• Browser: active");
|
||||
}
|
||||
|
||||
status
|
||||
}
|
||||
|
||||
async fn start_idle_monitor(&self) {
|
||||
let config = self.config.read().await;
|
||||
let idle_timeout = Duration::from_millis(config.idle_timeout_ms);
|
||||
drop(config);
|
||||
|
||||
let browser = Arc::clone(&self.browser);
|
||||
let last_activity = Arc::clone(&self.last_activity);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
loop {
|
||||
sleep(Duration::from_secs(10)).await;
|
||||
|
||||
let last = *last_activity.lock().await;
|
||||
if last.elapsed() > idle_timeout {
|
||||
warn!("Browser idle timeout reached, closing");
|
||||
let mut browser_guard = browser.lock().await;
|
||||
if let Some(mut browser) = browser_guard.take() {
|
||||
let _ = browser.close().await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut handle_guard = self.idle_monitor_handle.lock().await;
|
||||
*handle_guard = Some(handle);
|
||||
}
|
||||
|
||||
async fn stop_idle_monitor(&self) {
|
||||
let mut handle_guard = self.idle_monitor_handle.lock().await;
|
||||
if let Some(handle) = handle_guard.take() {
|
||||
handle.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn goto(&self, url: &str) -> Result<crate::page::GotoResult> {
|
||||
// Get or create page
|
||||
let page = self.get_or_create_page().await?;
|
||||
|
||||
let config = self.config.read().await;
|
||||
let result = page.goto(url, Some(config.wait.clone())).await?;
|
||||
|
||||
self.update_activity().await;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn capture_screenshot_with_url(&self) -> Result<(Vec<std::path::PathBuf>, Option<String>)> {
|
||||
let (paths, url) = self.capture_screenshot_internal().await?;
|
||||
Ok((paths, Some(url)))
|
||||
}
|
||||
|
||||
pub async fn capture_screenshot(&self) -> Result<Vec<std::path::PathBuf>> {
|
||||
let (paths, _) = self.capture_screenshot_internal().await?;
|
||||
Ok(paths)
|
||||
}
|
||||
|
||||
async fn capture_screenshot_internal(&self) -> Result<(Vec<std::path::PathBuf>, String)> {
|
||||
// Ensure we have a browser and page
|
||||
let page = self.get_or_create_page().await?;
|
||||
|
||||
// Initialize assets manager if needed
|
||||
let mut assets_guard = self.assets.lock().await;
|
||||
if assets_guard.is_none() {
|
||||
*assets_guard = Some(Arc::new(crate::assets::AssetManager::new().await?));
|
||||
}
|
||||
let assets = assets_guard.as_ref().unwrap().clone();
|
||||
drop(assets_guard);
|
||||
|
||||
// Get current config
|
||||
let config = self.config.read().await;
|
||||
|
||||
// Determine screenshot mode
|
||||
let mode = if config.fullpage {
|
||||
crate::page::ScreenshotMode::FullPage {
|
||||
segments_max: Some(config.segments_max)
|
||||
}
|
||||
} else {
|
||||
crate::page::ScreenshotMode::Viewport
|
||||
};
|
||||
|
||||
// Get current URL
|
||||
let current_url = page.get_url().await.unwrap_or_else(|_| "about:blank".to_string());
|
||||
|
||||
// Capture screenshots
|
||||
let screenshots = page.screenshot(mode).await?;
|
||||
|
||||
// Store screenshots and get paths
|
||||
let mut paths = Vec::new();
|
||||
for screenshot in screenshots {
|
||||
let image_ref = assets.store_screenshot(
|
||||
&screenshot.data,
|
||||
screenshot.format,
|
||||
screenshot.width,
|
||||
screenshot.height,
|
||||
300000, // 5 minute TTL
|
||||
).await?;
|
||||
paths.push(std::path::PathBuf::from(image_ref.path));
|
||||
}
|
||||
|
||||
self.update_activity().await;
|
||||
Ok((paths, current_url))
|
||||
}
|
||||
|
||||
pub async fn close(&self) -> Result<()> {
|
||||
self.stop_idle_monitor().await;
|
||||
|
||||
let mut page_guard = self.page.lock().await;
|
||||
if let Some(page) = page_guard.take() {
|
||||
page.close().await?;
|
||||
}
|
||||
|
||||
let mut browser_guard = self.browser.lock().await;
|
||||
if let Some(mut browser) = browser_guard.take() {
|
||||
browser.close().await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Click at the specified coordinates
|
||||
pub async fn click(&self, x: f64, y: f64) -> Result<()> {
|
||||
let page = self.get_or_create_page().await?;
|
||||
page.click(x, y).await
|
||||
}
|
||||
|
||||
/// Type text into the currently focused element
|
||||
pub async fn type_text(&self, text: &str) -> Result<()> {
|
||||
let page = self.get_or_create_page().await?;
|
||||
page.type_text(text).await
|
||||
}
|
||||
|
||||
/// Press a key (e.g., "Enter", "Tab", "Escape", "ArrowDown")
|
||||
pub async fn press_key(&self, key: &str) -> Result<()> {
|
||||
let page = self.get_or_create_page().await?;
|
||||
page.press_key(key).await
|
||||
}
|
||||
|
||||
/// Execute JavaScript code with enhanced return value handling
|
||||
pub async fn execute_javascript(&self, code: &str) -> Result<serde_json::Value> {
|
||||
let page = self.get_or_create_page().await?;
|
||||
page.execute_javascript(code).await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct BrowserStatus {
|
||||
pub enabled: bool,
|
||||
pub browser_active: bool,
|
||||
pub current_url: Option<String>,
|
||||
pub viewport: crate::config::ViewportConfig,
|
||||
pub fullpage: bool,
|
||||
}
|
||||
@@ -0,0 +1,512 @@
|
||||
use crate::{
|
||||
config::{BrowserConfig, ImageFormat, ViewportConfig, WaitStrategy},
|
||||
BrowserError, Result,
|
||||
};
|
||||
use chromiumoxide::page::Page as CdpPage;
|
||||
use chromiumoxide::cdp::browser_protocol::page::{CaptureScreenshotFormat, CaptureScreenshotParams};
|
||||
use chromiumoxide::cdp::browser_protocol::emulation::SetDeviceMetricsOverrideParams;
|
||||
use chromiumoxide::cdp::browser_protocol::input::{
|
||||
DispatchMouseEventParams, DispatchMouseEventType,
|
||||
DispatchKeyEventParams, DispatchKeyEventType,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, info};
|
||||
|
||||
pub struct Page {
|
||||
cdp_page: Arc<CdpPage>,
|
||||
config: BrowserConfig,
|
||||
current_url: Arc<RwLock<Option<String>>>,
|
||||
}
|
||||
|
||||
impl Page {
|
||||
pub fn new(cdp_page: CdpPage, config: BrowserConfig) -> Self {
|
||||
Self {
|
||||
cdp_page: Arc::new(cdp_page),
|
||||
config,
|
||||
current_url: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn goto(&self, url: &str, wait: Option<WaitStrategy>) -> Result<GotoResult> {
|
||||
info!("Navigating to {}", url);
|
||||
|
||||
let wait_strategy = wait.unwrap_or_else(|| self.config.wait.clone());
|
||||
|
||||
self.cdp_page.goto(url).await?;
|
||||
|
||||
match wait_strategy {
|
||||
WaitStrategy::Event(event) => {
|
||||
match event.as_str() {
|
||||
"domcontentloaded" => {
|
||||
self.cdp_page.wait_for_navigation().await?;
|
||||
}
|
||||
"networkidle" | "networkidle0" => {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
"networkidle2" => {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
"load" => {
|
||||
self.cdp_page.wait_for_navigation().await?;
|
||||
}
|
||||
_ => {
|
||||
return Err(BrowserError::ConfigError(format!(
|
||||
"Unknown wait event: {}",
|
||||
event
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
WaitStrategy::Delay { delay_ms } => {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await;
|
||||
}
|
||||
}
|
||||
|
||||
let title = self.cdp_page.get_title().await.ok().flatten();
|
||||
let final_url = self.cdp_page.url().await?.unwrap_or_else(|| url.to_string());
|
||||
|
||||
let mut current_url = self.current_url.write().await;
|
||||
*current_url = Some(final_url.clone());
|
||||
|
||||
Ok(GotoResult {
|
||||
url: final_url,
|
||||
title,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn screenshot(&self, mode: ScreenshotMode) -> Result<Vec<Screenshot>> {
|
||||
match mode {
|
||||
ScreenshotMode::Viewport => self.screenshot_viewport().await,
|
||||
ScreenshotMode::FullPage { segments_max } => {
|
||||
self.screenshot_fullpage(segments_max.unwrap_or(self.config.segments_max))
|
||||
.await
|
||||
}
|
||||
ScreenshotMode::Region(region) => self.screenshot_region(region).await,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn screenshot_viewport(&self) -> Result<Vec<Screenshot>> {
|
||||
debug!("Taking viewport screenshot");
|
||||
|
||||
let format = match self.config.format {
|
||||
ImageFormat::Png => CaptureScreenshotFormat::Png,
|
||||
ImageFormat::Webp => CaptureScreenshotFormat::Webp,
|
||||
};
|
||||
|
||||
let params = CaptureScreenshotParams::builder()
|
||||
.format(format)
|
||||
.build();
|
||||
|
||||
let data = self.cdp_page.screenshot(params).await?;
|
||||
|
||||
Ok(vec![Screenshot {
|
||||
data,
|
||||
width: self.config.viewport.width,
|
||||
height: self.config.viewport.height,
|
||||
format: self.config.format,
|
||||
}])
|
||||
}
|
||||
|
||||
pub async fn screenshot_fullpage(&self, segments_max: usize) -> Result<Vec<Screenshot>> {
|
||||
debug!("Taking full page screenshot with max {} segments", segments_max);
|
||||
|
||||
let viewport = &self.config.viewport;
|
||||
let overlap = 24;
|
||||
let scroll_height = viewport.height - overlap;
|
||||
|
||||
let mut screenshots = Vec::new();
|
||||
let mut current_y = 0;
|
||||
|
||||
for segment_idx in 0..segments_max {
|
||||
self.cdp_page
|
||||
.evaluate(format!("window.scrollTo(0, {})", current_y))
|
||||
.await?;
|
||||
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
||||
|
||||
let format = match self.config.format {
|
||||
ImageFormat::Png => CaptureScreenshotFormat::Png,
|
||||
ImageFormat::Webp => CaptureScreenshotFormat::Webp,
|
||||
};
|
||||
|
||||
let params = CaptureScreenshotParams::builder()
|
||||
.format(format)
|
||||
.build();
|
||||
|
||||
let data = self.cdp_page.screenshot(params).await?;
|
||||
|
||||
screenshots.push(Screenshot {
|
||||
data,
|
||||
width: viewport.width,
|
||||
height: viewport.height,
|
||||
format: self.config.format,
|
||||
});
|
||||
|
||||
let eval_result = self
|
||||
.cdp_page
|
||||
.evaluate("document.body.scrollHeight")
|
||||
.await?;
|
||||
|
||||
let page_height: i64 = eval_result
|
||||
.value()
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(0);
|
||||
|
||||
current_y += scroll_height as i64;
|
||||
|
||||
if current_y >= page_height {
|
||||
debug!("Reached end of page after {} segments", segment_idx + 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if screenshots.len() == segments_max {
|
||||
info!("[full page truncated at {} segments]", segments_max);
|
||||
}
|
||||
|
||||
Ok(screenshots)
|
||||
}
|
||||
|
||||
pub async fn screenshot_region(&self, region: ScreenshotRegion) -> Result<Vec<Screenshot>> {
|
||||
debug!(
|
||||
"Taking region screenshot: {}x{} at ({}, {})",
|
||||
region.width, region.height, region.x, region.y
|
||||
);
|
||||
|
||||
let format = match self.config.format {
|
||||
ImageFormat::Png => CaptureScreenshotFormat::Png,
|
||||
ImageFormat::Webp => CaptureScreenshotFormat::Webp,
|
||||
};
|
||||
|
||||
let params = CaptureScreenshotParams::builder()
|
||||
.format(format)
|
||||
.clip(chromiumoxide::cdp::browser_protocol::page::Viewport {
|
||||
x: region.x as f64,
|
||||
y: region.y as f64,
|
||||
width: region.width as f64,
|
||||
height: region.height as f64,
|
||||
scale: 1.0,
|
||||
})
|
||||
.build();
|
||||
|
||||
let data = self.cdp_page.screenshot(params).await?;
|
||||
|
||||
let final_width = if region.width > 1024 {
|
||||
1024
|
||||
} else {
|
||||
region.width
|
||||
};
|
||||
|
||||
Ok(vec![Screenshot {
|
||||
data,
|
||||
width: final_width,
|
||||
height: region.height,
|
||||
format: self.config.format,
|
||||
}])
|
||||
}
|
||||
|
||||
pub async fn set_viewport(&self, viewport: SetViewportParams) -> Result<ViewportResult> {
|
||||
let params = SetDeviceMetricsOverrideParams::builder()
|
||||
.width(viewport.width as i64)
|
||||
.height(viewport.height as i64)
|
||||
.device_scale_factor(viewport.device_scale_factor.unwrap_or(1.0))
|
||||
.mobile(viewport.mobile.unwrap_or(false))
|
||||
.build()
|
||||
.map_err(|e| BrowserError::CdpError(e))?;
|
||||
|
||||
self.cdp_page.execute(params).await?;
|
||||
|
||||
Ok(ViewportResult {
|
||||
width: viewport.width,
|
||||
height: viewport.height,
|
||||
dpr: viewport.device_scale_factor.unwrap_or(1.0),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn inject_js(&self, script: &str) -> Result<serde_json::Value> {
|
||||
let result = self.cdp_page.evaluate(script).await?;
|
||||
Ok(result.value().cloned().unwrap_or(serde_json::Value::Null))
|
||||
}
|
||||
|
||||
pub async fn close(&self) -> Result<()> {
|
||||
// Note: chromiumoxide's close() takes ownership, so we can't call it on Arc<Page>
|
||||
// The page will be closed when the Arc is dropped
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_url(&self) -> Result<String> {
|
||||
let url_guard = self.current_url.read().await;
|
||||
url_guard
|
||||
.clone()
|
||||
.ok_or(BrowserError::PageNotLoaded)
|
||||
}
|
||||
|
||||
pub async fn update_viewport(&self, viewport: ViewportConfig) -> Result<()> {
|
||||
let params = SetDeviceMetricsOverrideParams::builder()
|
||||
.width(viewport.width as i64)
|
||||
.height(viewport.height as i64)
|
||||
.device_scale_factor(viewport.device_scale_factor)
|
||||
.mobile(viewport.mobile)
|
||||
.build()
|
||||
.map_err(|e| BrowserError::CdpError(e))?;
|
||||
|
||||
self.cdp_page.execute(params).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Click at the specified coordinates
|
||||
pub async fn click(&self, x: f64, y: f64) -> Result<()> {
|
||||
debug!("Clicking at ({}, {})", x, y);
|
||||
|
||||
// Move to position
|
||||
let move_params = DispatchMouseEventParams::builder()
|
||||
.r#type(DispatchMouseEventType::MouseMoved)
|
||||
.x(x)
|
||||
.y(y)
|
||||
.build()
|
||||
.map_err(|e| BrowserError::CdpError(e))?;
|
||||
self.cdp_page.execute(move_params).await?;
|
||||
|
||||
// Mouse down
|
||||
let down_params = DispatchMouseEventParams::builder()
|
||||
.r#type(DispatchMouseEventType::MousePressed)
|
||||
.x(x)
|
||||
.y(y)
|
||||
.button(chromiumoxide::cdp::browser_protocol::input::MouseButton::Left)
|
||||
.click_count(1)
|
||||
.build()
|
||||
.map_err(|e| BrowserError::CdpError(e))?;
|
||||
self.cdp_page.execute(down_params).await?;
|
||||
|
||||
// Mouse up
|
||||
let up_params = DispatchMouseEventParams::builder()
|
||||
.r#type(DispatchMouseEventType::MouseReleased)
|
||||
.x(x)
|
||||
.y(y)
|
||||
.button(chromiumoxide::cdp::browser_protocol::input::MouseButton::Left)
|
||||
.click_count(1)
|
||||
.build()
|
||||
.map_err(|e| BrowserError::CdpError(e))?;
|
||||
self.cdp_page.execute(up_params).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Type text into the currently focused element
|
||||
pub async fn type_text(&self, text: &str) -> Result<()> {
|
||||
debug!("Typing text: {}", text);
|
||||
|
||||
for ch in text.chars() {
|
||||
let params = DispatchKeyEventParams::builder()
|
||||
.r#type(DispatchKeyEventType::Char)
|
||||
.text(ch.to_string())
|
||||
.build()
|
||||
.map_err(|e| BrowserError::CdpError(e))?;
|
||||
self.cdp_page.execute(params).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Press a key (e.g., "Enter", "Tab", "Escape", "ArrowDown")
|
||||
pub async fn press_key(&self, key: &str) -> Result<()> {
|
||||
debug!("Pressing key: {}", key);
|
||||
|
||||
// Key down
|
||||
let down_params = DispatchKeyEventParams::builder()
|
||||
.r#type(DispatchKeyEventType::KeyDown)
|
||||
.key(key.to_string())
|
||||
.build()
|
||||
.map_err(|e| BrowserError::CdpError(e))?;
|
||||
self.cdp_page.execute(down_params).await?;
|
||||
|
||||
// Key up
|
||||
let up_params = DispatchKeyEventParams::builder()
|
||||
.r#type(DispatchKeyEventType::KeyUp)
|
||||
.key(key.to_string())
|
||||
.build()
|
||||
.map_err(|e| BrowserError::CdpError(e))?;
|
||||
self.cdp_page.execute(up_params).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Execute JavaScript code with enhanced return value handling
|
||||
pub async fn execute_javascript(&self, code: &str) -> Result<serde_json::Value> {
|
||||
debug!("Executing JavaScript: {}...", &code.chars().take(100).collect::<String>());
|
||||
|
||||
// Helper to check if a line looks like an expression that should be returned
|
||||
let is_expression_line = |line: &str| -> bool {
|
||||
let line = line.trim();
|
||||
if line.is_empty() { return false; }
|
||||
|
||||
// Skip lines that are already returns or start with control keywords
|
||||
let control_keywords = [
|
||||
"return", "const", "let", "var", "class", "function", "async",
|
||||
"if", "for", "while", "switch", "try", "catch", "finally",
|
||||
"throw", "import", "export", "yield", "await", "break",
|
||||
"continue", "debugger"
|
||||
];
|
||||
|
||||
for keyword in &control_keywords {
|
||||
if line.starts_with(keyword) &&
|
||||
(line.len() == keyword.len() ||
|
||||
line.chars().nth(keyword.len()).map_or(false, |c| !c.is_alphanumeric())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Skip lines that end with block closures
|
||||
if line.starts_with('}') { return false; }
|
||||
|
||||
true
|
||||
};
|
||||
|
||||
// Analyze the code
|
||||
let lines: Vec<&str> = code.split('\n').collect();
|
||||
let mut last_expr_line = None;
|
||||
let mut has_explicit_return = false;
|
||||
|
||||
// Scan for significant lines from the end
|
||||
for (i, line) in lines.iter().enumerate().rev() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with("//") { continue; }
|
||||
|
||||
if line.starts_with("return") {
|
||||
has_explicit_return = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if last_expr_line.is_none() && is_expression_line(line) {
|
||||
last_expr_line = Some(i);
|
||||
if !line.ends_with(';') && !line.ends_with('}') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build the function body
|
||||
let body = if has_explicit_return {
|
||||
code.to_string()
|
||||
} else if let Some(expr_line_idx) = last_expr_line {
|
||||
let prefix = lines[..expr_line_idx].join("\n");
|
||||
let expr_line = lines[expr_line_idx].trim();
|
||||
let suffix = lines[expr_line_idx + 1..].join("\n");
|
||||
|
||||
// Handle object literals and array literals - wrap in parentheses
|
||||
let wrapped_expr = if expr_line.starts_with('{') || expr_line.starts_with('[') {
|
||||
format!("({})", expr_line)
|
||||
} else {
|
||||
expr_line.to_string()
|
||||
};
|
||||
|
||||
format!("{}\nreturn {};\n{}", prefix, wrapped_expr, suffix)
|
||||
} else {
|
||||
// Try to capture the last variable defined
|
||||
let var_regex = regex::Regex::new(r"(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=").unwrap();
|
||||
if let Some(captures) = var_regex.captures(code) {
|
||||
if let Some(var_name) = captures.get(1) {
|
||||
format!("{}; return typeof {} !== 'undefined' ? {} : undefined;",
|
||||
code, var_name.as_str(), var_name.as_str())
|
||||
} else {
|
||||
code.to_string()
|
||||
}
|
||||
} else {
|
||||
code.to_string()
|
||||
}
|
||||
};
|
||||
|
||||
// Wrap in async IIFE with error handling and console.log capture
|
||||
let wrapped = format!(r#"
|
||||
(async () => {{
|
||||
const __logs = [];
|
||||
const __origLog = console.log;
|
||||
|
||||
function safe(arg) {{
|
||||
if (arg === null || typeof arg !== 'object') return String(arg);
|
||||
try {{ return JSON.stringify(arg); }} catch {{ return '[Circular]'; }}
|
||||
}}
|
||||
|
||||
console.log = function(...args) {{
|
||||
__logs.push(args.map(safe).join(' '));
|
||||
__origLog.apply(console, args);
|
||||
}};
|
||||
|
||||
try {{
|
||||
const AsyncFunction = Object.getPrototypeOf(async function(){{}}).constructor;
|
||||
const __userCode = {};
|
||||
const fn = new AsyncFunction(__userCode);
|
||||
const value = await fn();
|
||||
return {{
|
||||
success: true,
|
||||
value: (value === undefined && __logs.length > 0) ? __logs[__logs.length-1] : value,
|
||||
logs: __logs
|
||||
}};
|
||||
}} catch (err) {{
|
||||
return {{
|
||||
success: false,
|
||||
error: err.toString() + (err.stack ? '\n' + err.stack : ''),
|
||||
logs: __logs
|
||||
}};
|
||||
}} finally {{
|
||||
console.log = __origLog;
|
||||
}}
|
||||
}})()
|
||||
"#, serde_json::to_string(&body).unwrap());
|
||||
|
||||
tracing::debug!("Executing JavaScript code: {}", code);
|
||||
tracing::debug!("Wrapped code: {}", wrapped);
|
||||
|
||||
let result = self.cdp_page.evaluate(wrapped).await?;
|
||||
let result_value = result.value().cloned().unwrap_or(serde_json::Value::Null);
|
||||
|
||||
tracing::info!("JavaScript execution result: {}", result_value);
|
||||
|
||||
Ok(result_value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum ScreenshotMode {
|
||||
Viewport,
|
||||
FullPage { segments_max: Option<usize> },
|
||||
Region(ScreenshotRegion),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScreenshotRegion {
|
||||
pub x: u32,
|
||||
pub y: u32,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Screenshot {
|
||||
pub data: Vec<u8>,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub format: ImageFormat,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
pub struct GotoResult {
|
||||
pub url: String,
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
pub struct SetViewportParams {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub device_scale_factor: Option<f64>,
|
||||
pub mobile: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
pub struct ViewportResult {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub dpr: f64,
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
use crate::{
|
||||
assets::{AssetManager, ImageRef},
|
||||
manager::BrowserManager,
|
||||
page::{ScreenshotMode, ScreenshotRegion, SetViewportParams},
|
||||
config::WaitStrategy,
|
||||
Result,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct BrowserTools {
|
||||
manager: Arc<BrowserManager>,
|
||||
asset_manager: Arc<AssetManager>,
|
||||
}
|
||||
|
||||
impl BrowserTools {
|
||||
pub fn new(manager: Arc<BrowserManager>, asset_manager: Arc<AssetManager>) -> Self {
|
||||
Self {
|
||||
manager,
|
||||
asset_manager,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle_tool_call(&self, call: BrowserToolCall) -> Result<BrowserToolResult> {
|
||||
match call {
|
||||
BrowserToolCall::Goto { url, wait } => {
|
||||
let page = self.manager.get_or_create_page().await?;
|
||||
let result = page.goto(&url, wait).await?;
|
||||
Ok(BrowserToolResult::Goto(result))
|
||||
}
|
||||
|
||||
BrowserToolCall::Screenshot {
|
||||
mode,
|
||||
segments_max,
|
||||
region,
|
||||
inject_js,
|
||||
format: _,
|
||||
} => {
|
||||
let page = self.manager.get_or_create_page().await?;
|
||||
|
||||
if let Some(script) = inject_js {
|
||||
page.inject_js(&script).await?;
|
||||
}
|
||||
|
||||
let screenshot_mode = match mode.as_deref() {
|
||||
Some("full_page") => ScreenshotMode::FullPage { segments_max },
|
||||
Some("viewport") | None => ScreenshotMode::Viewport,
|
||||
Some(_) => {
|
||||
if let Some(r) = region {
|
||||
ScreenshotMode::Region(ScreenshotRegion {
|
||||
x: r.x,
|
||||
y: r.y,
|
||||
width: r.width,
|
||||
height: r.height,
|
||||
})
|
||||
} else {
|
||||
ScreenshotMode::Viewport
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let screenshots = page.screenshot(screenshot_mode).await?;
|
||||
let ttl_ms = 300000;
|
||||
let images = self.asset_manager.store_screenshots(screenshots, ttl_ms).await?;
|
||||
|
||||
Ok(BrowserToolResult::Screenshot(ScreenshotResult { images }))
|
||||
}
|
||||
|
||||
BrowserToolCall::SetViewport {
|
||||
width,
|
||||
height,
|
||||
device_scale_factor,
|
||||
mobile,
|
||||
} => {
|
||||
let page = self.manager.get_or_create_page().await?;
|
||||
let result = page
|
||||
.set_viewport(SetViewportParams {
|
||||
width,
|
||||
height,
|
||||
device_scale_factor,
|
||||
mobile,
|
||||
})
|
||||
.await?;
|
||||
Ok(BrowserToolResult::SetViewport(result))
|
||||
}
|
||||
|
||||
BrowserToolCall::Close { what } => {
|
||||
match what.as_deref() {
|
||||
Some("browser") => {
|
||||
self.manager.stop().await?;
|
||||
Ok(BrowserToolResult::Close(CloseResult {
|
||||
closed: "browser".to_string(),
|
||||
}))
|
||||
}
|
||||
Some("page") | None => {
|
||||
self.manager.close_page().await?;
|
||||
Ok(BrowserToolResult::Close(CloseResult {
|
||||
closed: "page".to_string(),
|
||||
}))
|
||||
}
|
||||
Some(other) => Err(crate::BrowserError::ConfigError(format!(
|
||||
"Unknown close target: {}",
|
||||
other
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "tool", rename_all = "snake_case")]
|
||||
pub enum BrowserToolCall {
|
||||
#[serde(rename = "browser.goto")]
|
||||
Goto {
|
||||
url: String,
|
||||
wait: Option<WaitStrategy>,
|
||||
},
|
||||
|
||||
#[serde(rename = "browser.screenshot")]
|
||||
Screenshot {
|
||||
mode: Option<String>,
|
||||
segments_max: Option<usize>,
|
||||
region: Option<RegionParams>,
|
||||
inject_js: Option<String>,
|
||||
format: Option<String>,
|
||||
},
|
||||
|
||||
#[serde(rename = "browser.setViewport")]
|
||||
SetViewport {
|
||||
width: u32,
|
||||
height: u32,
|
||||
device_scale_factor: Option<f64>,
|
||||
mobile: Option<bool>,
|
||||
},
|
||||
|
||||
#[serde(rename = "browser.close")]
|
||||
Close {
|
||||
what: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct RegionParams {
|
||||
pub x: u32,
|
||||
pub y: u32,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum BrowserToolResult {
|
||||
Goto(crate::page::GotoResult),
|
||||
Screenshot(ScreenshotResult),
|
||||
SetViewport(crate::page::ViewportResult),
|
||||
Close(CloseResult),
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ScreenshotResult {
|
||||
pub images: Vec<ImageRef>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct CloseResult {
|
||||
pub closed: String,
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod browser_tools;
|
||||
pub mod schema;
|
||||
|
||||
pub use browser_tools::{BrowserTools, BrowserToolCall, BrowserToolResult};
|
||||
pub use schema::{get_browser_tools_schema, BrowserToolSchema};
|
||||
@@ -0,0 +1,126 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BrowserToolSchema {
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
pub parameters: serde_json::Value,
|
||||
}
|
||||
|
||||
pub fn get_browser_tools_schema() -> Vec<BrowserToolSchema> {
|
||||
vec![
|
||||
BrowserToolSchema {
|
||||
name: "browser.goto".to_string(),
|
||||
description: "Navigate to a URL and wait for the page to load".to_string(),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "The URL to navigate to"
|
||||
},
|
||||
"wait": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"enum": ["domcontentloaded", "networkidle", "networkidle0", "networkidle2", "load"],
|
||||
"description": "Wait for a specific event"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"delay_ms": {
|
||||
"type": "number",
|
||||
"description": "Wait for a specific delay in milliseconds"
|
||||
}
|
||||
},
|
||||
"required": ["delay_ms"]
|
||||
}
|
||||
],
|
||||
"description": "Wait strategy for page load"
|
||||
}
|
||||
},
|
||||
"required": ["url"]
|
||||
}),
|
||||
},
|
||||
BrowserToolSchema {
|
||||
name: "browser.screenshot".to_string(),
|
||||
description: "Take a screenshot of the current page".to_string(),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"enum": ["viewport", "full_page"],
|
||||
"description": "Screenshot mode (default: viewport)"
|
||||
},
|
||||
"segments_max": {
|
||||
"type": "number",
|
||||
"description": "Maximum number of segments for full_page mode (default: 8)"
|
||||
},
|
||||
"region": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"x": { "type": "number" },
|
||||
"y": { "type": "number" },
|
||||
"width": { "type": "number" },
|
||||
"height": { "type": "number" }
|
||||
},
|
||||
"required": ["x", "y", "width", "height"],
|
||||
"description": "Optional region to capture"
|
||||
},
|
||||
"inject_js": {
|
||||
"type": "string",
|
||||
"description": "JavaScript to inject before screenshot"
|
||||
},
|
||||
"format": {
|
||||
"type": "string",
|
||||
"enum": ["png", "webp"],
|
||||
"description": "Image format (default: png)"
|
||||
}
|
||||
}
|
||||
}),
|
||||
},
|
||||
BrowserToolSchema {
|
||||
name: "browser.setViewport".to_string(),
|
||||
description: "Set the browser viewport size".to_string(),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"width": {
|
||||
"type": "number",
|
||||
"description": "Viewport width in pixels"
|
||||
},
|
||||
"height": {
|
||||
"type": "number",
|
||||
"description": "Viewport height in pixels"
|
||||
},
|
||||
"device_scale_factor": {
|
||||
"type": "number",
|
||||
"description": "Device scale factor (default: 1.0)"
|
||||
},
|
||||
"mobile": {
|
||||
"type": "boolean",
|
||||
"description": "Enable mobile mode (default: false)"
|
||||
}
|
||||
},
|
||||
"required": ["width", "height"]
|
||||
}),
|
||||
},
|
||||
BrowserToolSchema {
|
||||
name: "browser.close".to_string(),
|
||||
description: "Close the page or browser".to_string(),
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"what": {
|
||||
"type": "string",
|
||||
"enum": ["page", "browser"],
|
||||
"description": "What to close (default: page)"
|
||||
}
|
||||
}
|
||||
}),
|
||||
},
|
||||
]
|
||||
}
|
||||
@@ -17,6 +17,7 @@ base64 = "0.22"
|
||||
bytes = "1.10.1"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
codex-apply-patch = { path = "../apply-patch" }
|
||||
codex-browser = { path = "../browser" }
|
||||
codex-login = { path = "../login" }
|
||||
codex-mcp-client = { path = "../mcp-client" }
|
||||
dirs = "6"
|
||||
|
||||
+637
-4
@@ -12,6 +12,7 @@ use std::time::Duration;
|
||||
|
||||
use async_channel::Receiver;
|
||||
use async_channel::Sender;
|
||||
use base64::Engine;
|
||||
use codex_apply_patch::ApplyPatchAction;
|
||||
use codex_apply_patch::MaybeApplyPatchVerified;
|
||||
use codex_apply_patch::maybe_parse_apply_patch_verified;
|
||||
@@ -66,6 +67,8 @@ use crate::models::ShellToolCallParams;
|
||||
use crate::openai_tools::ToolsConfig;
|
||||
use crate::openai_tools::get_openai_tools;
|
||||
use crate::plan_tool::handle_update_plan;
|
||||
use serde_json::Value;
|
||||
use codex_browser::manager::BrowserManager;
|
||||
use crate::project_doc::get_user_instructions;
|
||||
use crate::protocol::AgentMessageDeltaEvent;
|
||||
use crate::protocol::AgentMessageEvent;
|
||||
@@ -236,6 +239,10 @@ pub(crate) struct Session {
|
||||
codex_linux_sandbox_exe: Option<PathBuf>,
|
||||
user_shell: shell::Shell,
|
||||
show_raw_agent_reasoning: bool,
|
||||
/// Browser manager for controlling headless browser
|
||||
browser_manager: Mutex<Option<Arc<BrowserManager>>>,
|
||||
/// Pending browser screenshots to include in the next model request
|
||||
pending_browser_screenshots: Mutex<Vec<PathBuf>>,
|
||||
}
|
||||
|
||||
impl Session {
|
||||
@@ -536,8 +543,87 @@ impl Session {
|
||||
|
||||
/// Build the full turn input by concatenating the current conversation
|
||||
/// history with additional items for this turn.
|
||||
/// Browser screenshots are filtered out from history to keep them ephemeral.
|
||||
pub fn turn_input_with_history(&self, extra: Vec<ResponseItem>) -> Vec<ResponseItem> {
|
||||
[self.state.lock().unwrap().history.contents(), extra].concat()
|
||||
let history = self.state.lock().unwrap().history.contents();
|
||||
|
||||
// Count images in extra for debugging (we can't distinguish ephemeral at this level anymore)
|
||||
let images_in_extra = extra.iter().filter(|item| {
|
||||
if let ResponseItem::Message { content, .. } = item {
|
||||
content.iter().any(|c| {
|
||||
matches!(c, crate::models::ContentItem::InputImage { .. })
|
||||
})
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}).count();
|
||||
|
||||
if images_in_extra > 0 {
|
||||
tracing::info!("Found {} images in current turn's extra items", images_in_extra);
|
||||
}
|
||||
|
||||
// Filter out browser screenshots from historical messages
|
||||
// We identify them by the [EPHEMERAL:...] marker that precedes them
|
||||
let filtered_history: Vec<ResponseItem> = history
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
if let ResponseItem::Message { id, role, content } = item {
|
||||
if role == "user" {
|
||||
// Filter out ephemeral content from user messages
|
||||
let mut filtered_content: Vec<crate::models::ContentItem> = Vec::new();
|
||||
let mut skip_next_image = false;
|
||||
|
||||
for content_item in content {
|
||||
match &content_item {
|
||||
crate::models::ContentItem::InputText { text } if text.starts_with("[EPHEMERAL:") => {
|
||||
// This is an ephemeral marker, skip it and the next image
|
||||
skip_next_image = true;
|
||||
tracing::info!("Filtering out ephemeral marker: {}", text);
|
||||
}
|
||||
crate::models::ContentItem::InputImage { .. } if skip_next_image => {
|
||||
// Skip this image as it follows an ephemeral marker
|
||||
skip_next_image = false;
|
||||
tracing::info!("Filtering out ephemeral image from history");
|
||||
}
|
||||
_ => {
|
||||
// Keep everything else
|
||||
filtered_content.push(content_item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ResponseItem::Message {
|
||||
id,
|
||||
role,
|
||||
content: filtered_content,
|
||||
}
|
||||
} else {
|
||||
// Keep assistant messages unchanged
|
||||
ResponseItem::Message { id, role, content }
|
||||
}
|
||||
} else {
|
||||
item
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Concatenate filtered history with current turn's extras (which includes current ephemeral images)
|
||||
let result = [filtered_history, extra].concat();
|
||||
|
||||
// Count total images in result for debugging
|
||||
let total_images = result.iter().filter(|item| {
|
||||
if let ResponseItem::Message { content, .. } = item {
|
||||
content.iter().any(|c| matches!(c, crate::models::ContentItem::InputImage { .. }))
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}).count();
|
||||
|
||||
if total_images > 0 {
|
||||
tracing::info!("Total images being sent to model: {}", total_images);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Returns the input if there was no task running to inject into
|
||||
@@ -562,6 +648,11 @@ impl Session {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_pending_input(&self, input: ResponseInputItem) {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
state.pending_input.push(input);
|
||||
}
|
||||
|
||||
pub async fn call_tool(
|
||||
&self,
|
||||
server: &str,
|
||||
@@ -889,6 +980,8 @@ async fn submission_loop(
|
||||
disable_response_storage,
|
||||
user_shell: default_shell,
|
||||
show_raw_agent_reasoning: config.show_raw_agent_reasoning,
|
||||
browser_manager: Mutex::new(None), // Initialize on-demand when browser tools are used
|
||||
pending_browser_screenshots: Mutex::new(Vec::new()),
|
||||
}));
|
||||
|
||||
// Patch restored state into the newly created session.
|
||||
@@ -1094,25 +1187,50 @@ async fn run_task(sess: Arc<Session>, sub_id: String, input: Vec<InputItem>) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Debug logging for ephemeral images
|
||||
let ephemeral_count = input.iter().filter(|item| {
|
||||
matches!(item, InputItem::EphemeralImage { .. })
|
||||
}).count();
|
||||
|
||||
if ephemeral_count > 0 {
|
||||
tracing::info!("Processing {} ephemeral images in user input", ephemeral_count);
|
||||
}
|
||||
|
||||
// Convert input to ResponseInputItem
|
||||
let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input);
|
||||
sess.record_conversation_items(&[initial_input_for_turn.clone().into()])
|
||||
let initial_response_item: ResponseItem = initial_input_for_turn.clone().into();
|
||||
|
||||
// Record to history but we'll handle ephemeral images separately
|
||||
sess.record_conversation_items(&[initial_response_item.clone()])
|
||||
.await;
|
||||
|
||||
let mut last_agent_message: Option<String> = None;
|
||||
// Although from the perspective of codex.rs, TurnDiffTracker has the lifecycle of a Task which contains
|
||||
// many turns, from the perspective of the user, it is a single turn.
|
||||
let mut turn_diff_tracker = TurnDiffTracker::new();
|
||||
|
||||
// Track if this is the first iteration - if so, include the initial input
|
||||
let mut first_iteration = true;
|
||||
|
||||
loop {
|
||||
// Note that pending_input would be something like a message the user
|
||||
// submitted through the UI while the model was running. Though the UI
|
||||
// may support this, the model might not.
|
||||
let pending_input = sess
|
||||
let mut pending_input = sess
|
||||
.get_pending_input()
|
||||
.into_iter()
|
||||
.map(ResponseItem::from)
|
||||
.collect::<Vec<ResponseItem>>();
|
||||
sess.record_conversation_items(&pending_input).await;
|
||||
|
||||
// On the first iteration, add the initial input to pending_input
|
||||
// This ensures ephemeral images are included in the current turn
|
||||
if first_iteration {
|
||||
pending_input.insert(0, initial_response_item.clone());
|
||||
first_iteration = false;
|
||||
} else {
|
||||
// Only record pending input to history on subsequent iterations
|
||||
sess.record_conversation_items(&pending_input).await;
|
||||
}
|
||||
|
||||
// Construct the input that we will send to the model. When using the
|
||||
// Chat completions API (or ZDR clients), the model needs the full
|
||||
@@ -1226,6 +1344,19 @@ async fn run_task(sess: Arc<Session>, sub_id: String, input: Vec<InputItem>) {
|
||||
.await;
|
||||
}
|
||||
|
||||
// Add any pending browser screenshots to the responses
|
||||
let screenshot_responses = consume_pending_screenshots(&sess);
|
||||
for screenshot_response in screenshot_responses {
|
||||
responses.push(screenshot_response);
|
||||
}
|
||||
|
||||
// If there are responses, add them to pending input for the next iteration
|
||||
if !responses.is_empty() {
|
||||
for response in &responses {
|
||||
sess.add_pending_input(response.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if responses.is_empty() {
|
||||
debug!("Turn completed");
|
||||
last_agent_message = get_last_assistant_message_from_turn(
|
||||
@@ -1715,6 +1846,13 @@ async fn handle_function_call(
|
||||
.await
|
||||
}
|
||||
"update_plan" => handle_update_plan(sess, arguments, sub_id, call_id).await,
|
||||
"browser_open" => handle_browser_open(sess, arguments, sub_id, call_id).await,
|
||||
"browser_close" => handle_browser_close(sess, sub_id, call_id).await,
|
||||
"browser_status" => handle_browser_status(sess, sub_id, call_id).await,
|
||||
"browser_click" => handle_browser_click(sess, arguments, sub_id, call_id).await,
|
||||
"browser_type" => handle_browser_type(sess, arguments, sub_id, call_id).await,
|
||||
"browser_key" => handle_browser_key(sess, arguments, sub_id, call_id).await,
|
||||
"browser_javascript" => handle_browser_javascript(sess, arguments, sub_id, call_id).await,
|
||||
_ => {
|
||||
match sess.mcp_connection_manager.parse_tool_name(&name) {
|
||||
Some((server, tool_name)) => {
|
||||
@@ -2250,3 +2388,498 @@ async fn drain_to_completed(sess: &Session, sub_id: &str, prompt: &Prompt) -> Co
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture a screenshot from the browser and store it for the next model request
|
||||
async fn capture_browser_screenshot(sess: &Session) -> Option<PathBuf> {
|
||||
let browser_manager = sess.browser_manager.lock().unwrap().clone();
|
||||
if let Some(browser_manager) = browser_manager {
|
||||
if browser_manager.is_enabled().await {
|
||||
match browser_manager.capture_screenshot().await {
|
||||
Ok(screenshots) => {
|
||||
if let Some(first_screenshot) = screenshots.first() {
|
||||
tracing::info!("Captured browser screenshot: {}", first_screenshot.display());
|
||||
return Some(first_screenshot.clone());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to capture browser screenshot: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Add a screenshot to pending screenshots for the next model request
|
||||
fn add_pending_screenshot(sess: &Session, screenshot_path: PathBuf) {
|
||||
let mut pending = sess.pending_browser_screenshots.lock().unwrap();
|
||||
pending.push(screenshot_path);
|
||||
tracing::info!("Added pending screenshot for next model request");
|
||||
}
|
||||
|
||||
/// Consume pending screenshots and return them as ResponseInputItems
|
||||
fn consume_pending_screenshots(sess: &Session) -> Vec<ResponseInputItem> {
|
||||
let mut pending = sess.pending_browser_screenshots.lock().unwrap();
|
||||
let screenshots = pending.drain(..).collect::<Vec<_>>();
|
||||
|
||||
screenshots.into_iter().map(|path| {
|
||||
let metadata = format!("[EPHEMERAL:browser_screenshot] Browser screenshot at {}",
|
||||
chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC"));
|
||||
|
||||
// Read the screenshot file and create an ephemeral image
|
||||
match std::fs::read(&path) {
|
||||
Ok(bytes) => {
|
||||
let mime = mime_guess::from_path(&path)
|
||||
.first()
|
||||
.map(|m| m.to_string())
|
||||
.unwrap_or_else(|| "image/png".to_string());
|
||||
let encoded = base64::engine::general_purpose::STANDARD.encode(bytes);
|
||||
|
||||
ResponseInputItem::Message {
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputText { text: metadata },
|
||||
ContentItem::InputImage {
|
||||
image_url: format!("data:{mime};base64,{encoded}"),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to read screenshot {}: {}", path.display(), e);
|
||||
ResponseInputItem::Message {
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputText {
|
||||
text: format!("Failed to load browser screenshot: {}", e),
|
||||
}],
|
||||
}
|
||||
}
|
||||
}
|
||||
}).collect()
|
||||
}
|
||||
|
||||
async fn handle_browser_open(
|
||||
sess: &Session,
|
||||
arguments: String,
|
||||
_sub_id: String,
|
||||
call_id: String,
|
||||
) -> ResponseInputItem {
|
||||
// Parse the URL from arguments
|
||||
let args: Result<Value, _> = serde_json::from_str(&arguments);
|
||||
|
||||
match args {
|
||||
Ok(json) => {
|
||||
let url = json.get("url")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("about:blank");
|
||||
|
||||
// Use the global browser manager
|
||||
let needs_init = sess.browser_manager.lock().unwrap().is_none();
|
||||
let browser_manager = if needs_init {
|
||||
let manager = codex_browser::global::get_or_create_browser_manager().await;
|
||||
// Enable the browser
|
||||
manager.set_enabled_sync(true);
|
||||
*sess.browser_manager.lock().unwrap() = Some(manager.clone());
|
||||
Some(manager)
|
||||
} else {
|
||||
sess.browser_manager.lock().unwrap().clone()
|
||||
};
|
||||
|
||||
if let Some(browser_manager) = browser_manager {
|
||||
// Navigate to the URL
|
||||
match browser_manager.goto(url).await {
|
||||
Ok(_) => {
|
||||
// Capture screenshot after navigation
|
||||
if let Some(screenshot_path) = capture_browser_screenshot(sess).await {
|
||||
add_pending_screenshot(sess, screenshot_path);
|
||||
}
|
||||
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Browser opened to: {}. Screenshots will be automatically attached to messages.", url),
|
||||
success: Some(true),
|
||||
},
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to navigate browser to {}: {}", url, e),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "Failed to initialize browser manager.".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to parse browser_open arguments: {}", e),
|
||||
success: Some(false),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_browser_close(
|
||||
sess: &Session,
|
||||
_sub_id: String,
|
||||
call_id: String,
|
||||
) -> ResponseInputItem {
|
||||
let browser_manager = sess.browser_manager.lock().unwrap().clone();
|
||||
if let Some(browser_manager) = browser_manager {
|
||||
match browser_manager.stop().await {
|
||||
Ok(_) => {
|
||||
// Clear the browser manager from the session and global
|
||||
*sess.browser_manager.lock().unwrap() = None;
|
||||
codex_browser::global::clear_browser_manager().await;
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "Browser closed. Screenshot capture disabled.".to_string(),
|
||||
success: Some(true),
|
||||
},
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to close browser: {}", e),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "Browser is not currently open.".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_browser_status(
|
||||
sess: &Session,
|
||||
_sub_id: String,
|
||||
call_id: String,
|
||||
) -> ResponseInputItem {
|
||||
let browser_manager = sess.browser_manager.lock().unwrap().clone();
|
||||
if let Some(browser_manager) = browser_manager {
|
||||
let status = browser_manager.get_status().await;
|
||||
let status_msg = if status.enabled {
|
||||
if let Some(url) = status.current_url {
|
||||
format!("Browser status: Enabled, currently at {}", url)
|
||||
} else {
|
||||
"Browser status: Enabled, no page loaded".to_string()
|
||||
}
|
||||
} else {
|
||||
"Browser status: Disabled".to_string()
|
||||
};
|
||||
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: status_msg,
|
||||
success: Some(true),
|
||||
},
|
||||
}
|
||||
} else {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "Browser is not initialized. Use browser_open to start the browser.".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_browser_click(
|
||||
sess: &Session,
|
||||
arguments: String,
|
||||
_sub_id: String,
|
||||
call_id: String,
|
||||
) -> ResponseInputItem {
|
||||
let browser_manager = sess.browser_manager.lock().unwrap().clone();
|
||||
if let Some(browser_manager) = browser_manager {
|
||||
let args: Result<Value, _> = serde_json::from_str(&arguments);
|
||||
match args {
|
||||
Ok(json) => {
|
||||
let x = json.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let y = json.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
|
||||
match browser_manager.click(x, y).await {
|
||||
Ok(_) => {
|
||||
// Capture screenshot after clicking
|
||||
if let Some(screenshot_path) = capture_browser_screenshot(sess).await {
|
||||
add_pending_screenshot(sess, screenshot_path);
|
||||
}
|
||||
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Clicked at ({}, {})", x, y),
|
||||
success: Some(true),
|
||||
},
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to click: {}", e),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to parse browser_click arguments: {}", e),
|
||||
success: Some(false),
|
||||
},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "Browser is not initialized. Use browser_open to start the browser.".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_browser_type(
|
||||
sess: &Session,
|
||||
arguments: String,
|
||||
_sub_id: String,
|
||||
call_id: String,
|
||||
) -> ResponseInputItem {
|
||||
let browser_manager = sess.browser_manager.lock().unwrap().clone();
|
||||
if let Some(browser_manager) = browser_manager {
|
||||
let args: Result<Value, _> = serde_json::from_str(&arguments);
|
||||
match args {
|
||||
Ok(json) => {
|
||||
let text = json.get("text").and_then(|v| v.as_str()).unwrap_or("");
|
||||
|
||||
match browser_manager.type_text(text).await {
|
||||
Ok(_) => {
|
||||
// Capture screenshot after typing
|
||||
if let Some(screenshot_path) = capture_browser_screenshot(sess).await {
|
||||
add_pending_screenshot(sess, screenshot_path);
|
||||
}
|
||||
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Typed: {}", text),
|
||||
success: Some(true),
|
||||
},
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to type text: {}", e),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to parse browser_type arguments: {}", e),
|
||||
success: Some(false),
|
||||
},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "Browser is not initialized. Use browser_open to start the browser.".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_browser_key(
|
||||
sess: &Session,
|
||||
arguments: String,
|
||||
_sub_id: String,
|
||||
call_id: String,
|
||||
) -> ResponseInputItem {
|
||||
let browser_manager = sess.browser_manager.lock().unwrap().clone();
|
||||
if let Some(browser_manager) = browser_manager {
|
||||
let args: Result<Value, _> = serde_json::from_str(&arguments);
|
||||
match args {
|
||||
Ok(json) => {
|
||||
let key = json.get("key").and_then(|v| v.as_str()).unwrap_or("");
|
||||
|
||||
match browser_manager.press_key(key).await {
|
||||
Ok(_) => {
|
||||
// Capture screenshot after pressing key
|
||||
if let Some(screenshot_path) = capture_browser_screenshot(sess).await {
|
||||
add_pending_screenshot(sess, screenshot_path);
|
||||
}
|
||||
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Pressed key: {}", key),
|
||||
success: Some(true),
|
||||
},
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to press key: {}", e),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to parse browser_key arguments: {}", e),
|
||||
success: Some(false),
|
||||
},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "Browser is not initialized. Use browser_open to start the browser.".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_browser_javascript(
|
||||
sess: &Session,
|
||||
arguments: String,
|
||||
_sub_id: String,
|
||||
call_id: String,
|
||||
) -> ResponseInputItem {
|
||||
let browser_manager = sess.browser_manager.lock().unwrap().clone();
|
||||
if let Some(browser_manager) = browser_manager {
|
||||
let args: Result<Value, _> = serde_json::from_str(&arguments);
|
||||
match args {
|
||||
Ok(json) => {
|
||||
let code = json.get("code").and_then(|v| v.as_str()).unwrap_or("");
|
||||
|
||||
match browser_manager.execute_javascript(code).await {
|
||||
Ok(result) => {
|
||||
// Log the JavaScript execution result
|
||||
tracing::info!("JavaScript execution returned: {:?}", result);
|
||||
|
||||
// Format the result for the LLM
|
||||
let formatted_result = if let Some(obj) = result.as_object() {
|
||||
// Check if it's our wrapped result format
|
||||
if let (Some(success), Some(value)) = (obj.get("success"), obj.get("value")) {
|
||||
let logs = obj.get("logs").and_then(|v| v.as_array());
|
||||
let mut output = String::new();
|
||||
|
||||
if let Some(logs) = logs {
|
||||
if !logs.is_empty() {
|
||||
output.push_str("Console logs:\n");
|
||||
for log in logs {
|
||||
if let Some(log_str) = log.as_str() {
|
||||
output.push_str(&format!(" {}\n", log_str));
|
||||
}
|
||||
}
|
||||
output.push_str("\n");
|
||||
}
|
||||
}
|
||||
|
||||
if success.as_bool().unwrap_or(false) {
|
||||
output.push_str("Result: ");
|
||||
output.push_str(&serde_json::to_string_pretty(value).unwrap_or_else(|_| "null".to_string()));
|
||||
} else if let Some(error) = obj.get("error") {
|
||||
output.push_str("Error: ");
|
||||
output.push_str(&error.to_string());
|
||||
}
|
||||
|
||||
output
|
||||
} else {
|
||||
// Fallback to raw JSON if not in expected format
|
||||
serde_json::to_string_pretty(&result).unwrap_or_else(|_| "null".to_string())
|
||||
}
|
||||
} else {
|
||||
// Not an object, return as-is
|
||||
serde_json::to_string_pretty(&result).unwrap_or_else(|_| "null".to_string())
|
||||
};
|
||||
|
||||
tracing::info!("Returning to LLM: {}", formatted_result);
|
||||
|
||||
// Capture screenshot after executing JavaScript
|
||||
if let Some(screenshot_path) = capture_browser_screenshot(sess).await {
|
||||
add_pending_screenshot(sess, screenshot_path);
|
||||
}
|
||||
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: formatted_result,
|
||||
success: Some(true),
|
||||
},
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to execute JavaScript: {}", e),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: format!("Failed to parse browser_javascript arguments: {}", e),
|
||||
success: Some(false),
|
||||
},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id,
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "Browser is not initialized. Use browser_open to start the browser.".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use crate::config_profile::ConfigProfile;
|
||||
use crate::config_types::BrowserConfig;
|
||||
use crate::config_types::History;
|
||||
use crate::config_types::McpServerConfig;
|
||||
use crate::config_types::ReasoningEffort;
|
||||
@@ -377,6 +378,9 @@ pub struct ConfigToml {
|
||||
/// Collection of settings that are specific to the TUI.
|
||||
pub tui: Option<Tui>,
|
||||
|
||||
/// Browser configuration for integrated screenshot capabilities.
|
||||
pub browser: Option<BrowserConfig>,
|
||||
|
||||
/// When set to `true`, `AgentReasoning` events will be hidden from the
|
||||
/// UI/output. Defaults to `false`.
|
||||
pub hide_agent_reasoning: Option<bool>,
|
||||
|
||||
@@ -78,6 +78,57 @@ pub enum HistoryPersistence {
|
||||
#[derive(Deserialize, Debug, Clone, PartialEq, Default)]
|
||||
pub struct Tui {}
|
||||
|
||||
/// Browser configuration for integrated screenshot capabilities.
|
||||
#[derive(Deserialize, Debug, Clone, PartialEq, Default)]
|
||||
pub struct BrowserConfig {
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub viewport: Option<BrowserViewportConfig>,
|
||||
|
||||
#[serde(default)]
|
||||
pub wait: Option<BrowserWaitStrategy>,
|
||||
|
||||
#[serde(default)]
|
||||
pub fullpage: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub segments_max: Option<usize>,
|
||||
|
||||
#[serde(default)]
|
||||
pub idle_timeout_ms: Option<u64>,
|
||||
|
||||
#[serde(default)]
|
||||
pub format: Option<BrowserImageFormat>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone, PartialEq)]
|
||||
pub struct BrowserViewportConfig {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
|
||||
#[serde(default)]
|
||||
pub device_scale_factor: Option<f64>,
|
||||
|
||||
#[serde(default)]
|
||||
pub mobile: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone, PartialEq)]
|
||||
#[serde(untagged)]
|
||||
pub enum BrowserWaitStrategy {
|
||||
Event(String),
|
||||
Delay { delay_ms: u64 },
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone, Copy, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum BrowserImageFormat {
|
||||
Png,
|
||||
Webp,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone, Copy, PartialEq, Default, Serialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum SandboxMode {
|
||||
|
||||
+53
-14
@@ -146,23 +146,27 @@ pub enum ReasoningItemContent {
|
||||
|
||||
impl From<Vec<InputItem>> for ResponseInputItem {
|
||||
fn from(items: Vec<InputItem>) -> Self {
|
||||
Self::Message {
|
||||
role: "user".to_string(),
|
||||
content: items
|
||||
.into_iter()
|
||||
.filter_map(|c| match c {
|
||||
InputItem::Text { text } => Some(ContentItem::InputText { text }),
|
||||
InputItem::Image { image_url } => Some(ContentItem::InputImage { image_url }),
|
||||
InputItem::LocalImage { path } => match std::fs::read(&path) {
|
||||
let mut content_items = Vec::new();
|
||||
|
||||
for item in items {
|
||||
match item {
|
||||
InputItem::Text { text } => {
|
||||
content_items.push(ContentItem::InputText { text });
|
||||
}
|
||||
InputItem::Image { image_url } => {
|
||||
content_items.push(ContentItem::InputImage { image_url });
|
||||
}
|
||||
InputItem::LocalImage { path } => {
|
||||
match std::fs::read(&path) {
|
||||
Ok(bytes) => {
|
||||
let mime = mime_guess::from_path(&path)
|
||||
.first()
|
||||
.map(|m| m.essence_str().to_owned())
|
||||
.unwrap_or_else(|| "application/octet-stream".to_string());
|
||||
let encoded = base64::engine::general_purpose::STANDARD.encode(bytes);
|
||||
Some(ContentItem::InputImage {
|
||||
content_items.push(ContentItem::InputImage {
|
||||
image_url: format!("data:{mime};base64,{encoded}"),
|
||||
})
|
||||
});
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
@@ -170,11 +174,46 @@ impl From<Vec<InputItem>> for ResponseInputItem {
|
||||
path.display(),
|
||||
err
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
})
|
||||
.collect::<Vec<ContentItem>>(),
|
||||
}
|
||||
}
|
||||
InputItem::EphemeralImage { path, metadata } => {
|
||||
tracing::info!("Processing ephemeral image: {} with metadata: {:?}", path.display(), metadata);
|
||||
|
||||
// Add metadata text BEFORE the image so the LLM sees context first
|
||||
if let Some(meta) = metadata {
|
||||
content_items.push(ContentItem::InputText {
|
||||
text: format!("[EPHEMERAL:{}]", meta)
|
||||
});
|
||||
}
|
||||
|
||||
match std::fs::read(&path) {
|
||||
Ok(bytes) => {
|
||||
let mime = mime_guess::from_path(&path)
|
||||
.first()
|
||||
.map(|m| m.essence_str().to_owned())
|
||||
.unwrap_or_else(|| "application/octet-stream".to_string());
|
||||
let encoded = base64::engine::general_purpose::STANDARD.encode(bytes);
|
||||
tracing::info!("Created ephemeral image data URL with mime: {}", mime);
|
||||
content_items.push(ContentItem::InputImage {
|
||||
image_url: format!("data:{mime};base64,{encoded}"),
|
||||
});
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::error!(
|
||||
"Failed to read ephemeral image {} – {}",
|
||||
path.display(),
|
||||
err
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Self::Message {
|
||||
role: "user".to_string(),
|
||||
content: content_items,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -331,6 +331,15 @@ pub(crate) fn get_openai_tools(
|
||||
if config.plan_tool {
|
||||
tools.push(PLAN_TOOL.clone());
|
||||
}
|
||||
|
||||
// Add browser tools
|
||||
tools.push(create_browser_open_tool());
|
||||
tools.push(create_browser_close_tool());
|
||||
tools.push(create_browser_status_tool());
|
||||
tools.push(create_browser_click_tool());
|
||||
tools.push(create_browser_type_tool());
|
||||
tools.push(create_browser_key_tool());
|
||||
tools.push(create_browser_javascript_tool());
|
||||
|
||||
if let Some(mcp_tools) = mcp_tools {
|
||||
for (name, tool) in mcp_tools {
|
||||
@@ -498,3 +507,144 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn create_browser_open_tool() -> OpenAiTool {
|
||||
let mut properties = BTreeMap::new();
|
||||
properties.insert(
|
||||
"url".to_string(),
|
||||
JsonSchema::String {
|
||||
description: Some("The URL to navigate to (e.g., https://example.com)".to_string()),
|
||||
},
|
||||
);
|
||||
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "browser_open".to_string(),
|
||||
description: "Opens a browser window and navigates to the specified URL. Screenshots will be automatically attached to subsequent messages.".to_string(),
|
||||
strict: false,
|
||||
parameters: JsonSchema::Object {
|
||||
properties,
|
||||
required: Some(vec!["url".to_string()]),
|
||||
additional_properties: Some(false),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn create_browser_close_tool() -> OpenAiTool {
|
||||
let properties = BTreeMap::new();
|
||||
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "browser_close".to_string(),
|
||||
description: "Closes the browser window and disables screenshot capture.".to_string(),
|
||||
strict: false,
|
||||
parameters: JsonSchema::Object {
|
||||
properties,
|
||||
required: Some(vec![]),
|
||||
additional_properties: Some(false),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn create_browser_status_tool() -> OpenAiTool {
|
||||
let properties = BTreeMap::new();
|
||||
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "browser_status".to_string(),
|
||||
description: "Gets the current browser status including whether it's enabled, current URL, and viewport settings.".to_string(),
|
||||
strict: false,
|
||||
parameters: JsonSchema::Object {
|
||||
properties,
|
||||
required: Some(vec![]),
|
||||
additional_properties: Some(false),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn create_browser_click_tool() -> OpenAiTool {
|
||||
let mut properties = BTreeMap::new();
|
||||
properties.insert(
|
||||
"x".to_string(),
|
||||
JsonSchema::Number {
|
||||
description: Some("The X coordinate to click at".to_string()),
|
||||
},
|
||||
);
|
||||
properties.insert(
|
||||
"y".to_string(),
|
||||
JsonSchema::Number {
|
||||
description: Some("The Y coordinate to click at".to_string()),
|
||||
},
|
||||
);
|
||||
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "browser_click".to_string(),
|
||||
description: "Clicks at the specified coordinates in the browser window.".to_string(),
|
||||
strict: false,
|
||||
parameters: JsonSchema::Object {
|
||||
properties,
|
||||
required: Some(vec!["x".to_string(), "y".to_string()]),
|
||||
additional_properties: Some(false),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn create_browser_type_tool() -> OpenAiTool {
|
||||
let mut properties = BTreeMap::new();
|
||||
properties.insert(
|
||||
"text".to_string(),
|
||||
JsonSchema::String {
|
||||
description: Some("The text to type into the currently focused element".to_string()),
|
||||
},
|
||||
);
|
||||
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "browser_type".to_string(),
|
||||
description: "Types text into the currently focused element in the browser.".to_string(),
|
||||
strict: false,
|
||||
parameters: JsonSchema::Object {
|
||||
properties,
|
||||
required: Some(vec!["text".to_string()]),
|
||||
additional_properties: Some(false),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn create_browser_key_tool() -> OpenAiTool {
|
||||
let mut properties = BTreeMap::new();
|
||||
properties.insert(
|
||||
"key".to_string(),
|
||||
JsonSchema::String {
|
||||
description: Some("The key to press (e.g., 'Enter', 'Tab', 'Escape', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', 'Backspace', 'Delete')".to_string()),
|
||||
},
|
||||
);
|
||||
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "browser_key".to_string(),
|
||||
description: "Presses a keyboard key in the browser (e.g., Enter, Tab, Escape, arrow keys).".to_string(),
|
||||
strict: false,
|
||||
parameters: JsonSchema::Object {
|
||||
properties,
|
||||
required: Some(vec!["key".to_string()]),
|
||||
additional_properties: Some(false),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn create_browser_javascript_tool() -> OpenAiTool {
|
||||
let mut properties = BTreeMap::new();
|
||||
properties.insert(
|
||||
"code".to_string(),
|
||||
JsonSchema::String {
|
||||
description: Some("The JavaScript code to execute in the browser context".to_string()),
|
||||
},
|
||||
);
|
||||
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "browser_javascript".to_string(),
|
||||
description: "Executes JavaScript code in the browser and returns the result. The code is wrapped to automatically capture return values and console.log output.".to_string(),
|
||||
strict: false,
|
||||
parameters: JsonSchema::Object {
|
||||
properties,
|
||||
required: Some(vec!["code".to_string()]),
|
||||
additional_properties: Some(false),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -338,6 +338,14 @@ pub enum InputItem {
|
||||
LocalImage {
|
||||
path: std::path::PathBuf,
|
||||
},
|
||||
|
||||
/// Ephemeral image (like browser screenshots) that should not be persisted in history.
|
||||
/// This will be converted to an `Image` variant but marked as ephemeral.
|
||||
EphemeralImage {
|
||||
path: std::path::PathBuf,
|
||||
/// Optional metadata to help identify the image (e.g., "screenshot:1234567890:https://example.com")
|
||||
metadata: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Event Queue Entry - events from agent
|
||||
|
||||
@@ -25,6 +25,7 @@ chrono = { version = "0.4", features = ["serde"] }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
codex-ansi-escape = { path = "../ansi-escape" }
|
||||
codex-arg0 = { path = "../arg0" }
|
||||
codex-browser = { path = "../browser" }
|
||||
codex-common = { path = "../common", features = [
|
||||
"cli",
|
||||
"elapsed",
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::slash_command::SlashCommand;
|
||||
use crate::tui;
|
||||
use codex_core::config::Config;
|
||||
use codex_core::protocol::Event;
|
||||
#[cfg(debug_assertions)]
|
||||
use codex_core::protocol::EventMsg;
|
||||
use codex_core::protocol::Op;
|
||||
use color_eyre::eyre::Result;
|
||||
@@ -381,6 +382,11 @@ impl App<'_> {
|
||||
widget.add_prompts_output();
|
||||
}
|
||||
}
|
||||
SlashCommand::Browser => {
|
||||
if let AppState::Chat { widget } = &mut self.app_state {
|
||||
widget.handle_browser_command(command_text);
|
||||
}
|
||||
}
|
||||
#[cfg(debug_assertions)]
|
||||
SlashCommand::TestApproval => {
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -34,6 +34,7 @@ use ratatui::buffer::Buffer;
|
||||
use ratatui::layout::Constraint;
|
||||
use ratatui::layout::Layout;
|
||||
use ratatui::layout::Rect;
|
||||
use ratatui::text::Line;
|
||||
use ratatui::widgets::Widget;
|
||||
use ratatui::widgets::WidgetRef;
|
||||
use tokio::sync::mpsc::UnboundedSender;
|
||||
@@ -50,7 +51,9 @@ use crate::history_cell::CommandOutput;
|
||||
use crate::history_cell::HistoryCell;
|
||||
use crate::history_cell::PatchEventType;
|
||||
use crate::live_wrap::RowBuilder;
|
||||
use crate::text_block::TextBlock;
|
||||
use crate::user_approval_widget::ApprovalRequest;
|
||||
use codex_browser::{BrowserManager, config::BrowserConfig};
|
||||
use codex_file_search::FileMatch;
|
||||
use ratatui::style::Stylize;
|
||||
|
||||
@@ -83,6 +86,8 @@ pub(crate) struct ChatWidget<'a> {
|
||||
// Store pending image paths keyed by their placeholder text
|
||||
pending_images: HashMap<String, PathBuf>,
|
||||
welcome_shown: bool,
|
||||
// Browser manager for screenshot functionality
|
||||
browser_manager: Arc<BrowserManager>,
|
||||
}
|
||||
|
||||
struct UserMessage {
|
||||
@@ -274,6 +279,11 @@ impl ChatWidget<'_> {
|
||||
}
|
||||
});
|
||||
|
||||
// Browser manager will be initialized lazily when needed
|
||||
// For now, create a placeholder that will be replaced on first use
|
||||
let browser_config = BrowserConfig::default();
|
||||
let browser_manager = Arc::new(BrowserManager::new(browser_config));
|
||||
|
||||
Self {
|
||||
app_event_tx: app_event_tx.clone(),
|
||||
codex_op_tx,
|
||||
@@ -300,6 +310,7 @@ impl ChatWidget<'_> {
|
||||
live_max_rows: 3,
|
||||
pending_images: HashMap::new(),
|
||||
welcome_shown: false,
|
||||
browser_manager,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -416,11 +427,110 @@ impl ChatWidget<'_> {
|
||||
fn submit_user_message(&mut self, user_message: UserMessage) {
|
||||
let UserMessage { text, image_paths } = user_message;
|
||||
let mut items: Vec<InputItem> = Vec::new();
|
||||
let mut browser_screenshot_count = 0;
|
||||
|
||||
// Check if browser mode is enabled and capture screenshot
|
||||
// Always check both local and global browser managers
|
||||
let local_browser_manager = self.browser_manager.clone();
|
||||
|
||||
// Check if either local or global browser is enabled
|
||||
let local_enabled = local_browser_manager.config.try_read()
|
||||
.map(|c| c.enabled)
|
||||
.unwrap_or(false);
|
||||
|
||||
// Check global browser manager async
|
||||
let (tx_global, rx_global) = std::sync::mpsc::channel();
|
||||
tokio::spawn(async move {
|
||||
let global_manager = codex_browser::global::get_browser_manager().await;
|
||||
let enabled = if let Some(ref mgr) = global_manager {
|
||||
mgr.is_enabled().await
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let _ = tx_global.send((global_manager, enabled));
|
||||
});
|
||||
|
||||
// Get global browser state
|
||||
let (global_manager, global_enabled) = rx_global.recv_timeout(std::time::Duration::from_millis(100))
|
||||
.unwrap_or((None, false));
|
||||
|
||||
// Use global manager if it exists and is enabled, otherwise use local
|
||||
let (browser_manager, browser_enabled) = if global_enabled && global_manager.is_some() {
|
||||
(global_manager.unwrap(), true)
|
||||
} else if local_enabled {
|
||||
(local_browser_manager, true)
|
||||
} else if let Some(global) = global_manager {
|
||||
// Global exists but not enabled, still prefer it over local
|
||||
(global, false)
|
||||
} else {
|
||||
(local_browser_manager, false)
|
||||
};
|
||||
|
||||
// Get current browser URL for display
|
||||
let mut browser_url = None;
|
||||
|
||||
if browser_enabled {
|
||||
tracing::info!("Browser is enabled, attempting to capture screenshot...");
|
||||
|
||||
// We need to capture screenshots synchronously before sending the message
|
||||
// Use a channel to get the result from the async task
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let browser_manager_capture = browser_manager.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
tracing::info!("Starting async screenshot capture...");
|
||||
let result = browser_manager_capture.capture_screenshot_with_url().await;
|
||||
match &result {
|
||||
Ok((paths, _)) => tracing::info!("Screenshot capture succeeded with {} images", paths.len()),
|
||||
Err(e) => tracing::error!("Screenshot capture failed: {}", e),
|
||||
}
|
||||
let _ = tx.send(result);
|
||||
});
|
||||
|
||||
// Wait for screenshot capture (with timeout)
|
||||
match rx.recv_timeout(std::time::Duration::from_secs(5)) {
|
||||
Ok(Ok((screenshot_paths, url))) => {
|
||||
tracing::info!("Captured {} browser screenshot(s)", screenshot_paths.len());
|
||||
browser_screenshot_count = screenshot_paths.len();
|
||||
let screenshot_url = url.clone();
|
||||
browser_url = url;
|
||||
// Add browser screenshots directly to items (ephemeral - not stored in image_paths for history)
|
||||
for path in screenshot_paths {
|
||||
tracing::info!("Browser screenshot attached: {}", path.display());
|
||||
// Check if the file actually exists
|
||||
if path.exists() {
|
||||
tracing::info!("Screenshot file exists at: {}", path.display());
|
||||
// Add as EphemeralImage with metadata for identification
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
let metadata = format!("screenshot:{}:{}", timestamp, screenshot_url.as_deref().unwrap_or("unknown"));
|
||||
items.push(InputItem::EphemeralImage {
|
||||
path,
|
||||
metadata: Some(metadata),
|
||||
});
|
||||
} else {
|
||||
tracing::error!("Screenshot file does not exist at: {}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
tracing::error!("Failed to capture browser screenshot: {}", e);
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::error!("Browser screenshot capture timed out");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tracing::info!("Browser is not enabled, skipping screenshot capture");
|
||||
}
|
||||
|
||||
if !text.is_empty() {
|
||||
items.push(InputItem::Text { text: text.clone() });
|
||||
}
|
||||
|
||||
// Add user-provided images (these are persistent in history)
|
||||
for path in image_paths {
|
||||
items.push(InputItem::LocalImage { path });
|
||||
}
|
||||
@@ -429,6 +539,15 @@ impl ChatWidget<'_> {
|
||||
return;
|
||||
}
|
||||
|
||||
// Debug logging for what we're sending
|
||||
let ephemeral_count = items.iter().filter(|item| {
|
||||
matches!(item, InputItem::EphemeralImage { .. })
|
||||
}).count();
|
||||
let total_items = items.len();
|
||||
if ephemeral_count > 0 {
|
||||
tracing::info!("Sending {} items to model (including {} ephemeral images)", total_items, ephemeral_count);
|
||||
}
|
||||
|
||||
self.codex_op_tx
|
||||
.send(Op::UserInput { items })
|
||||
.unwrap_or_else(|e| {
|
||||
@@ -444,9 +563,25 @@ impl ChatWidget<'_> {
|
||||
});
|
||||
}
|
||||
|
||||
// Only show text portion in conversation history for now.
|
||||
// Show text in conversation history, with a note about browser screenshots
|
||||
if !text.is_empty() {
|
||||
self.add_to_history(HistoryCell::new_user_prompt(text.clone()));
|
||||
let display_text = if browser_screenshot_count > 0 {
|
||||
if let Some(url) = browser_url {
|
||||
format!("{}\n\n[Browser: {}]", text, url)
|
||||
} else {
|
||||
format!("{}\n\n[Browser screenshot attached]", text)
|
||||
}
|
||||
} else {
|
||||
text.clone()
|
||||
};
|
||||
self.add_to_history(HistoryCell::new_user_prompt(display_text));
|
||||
} else if browser_screenshot_count > 0 {
|
||||
let prompt_text = if let Some(url) = browser_url {
|
||||
format!("[Browser: {}]", url)
|
||||
} else {
|
||||
"[Browser screenshot attached]".to_string()
|
||||
};
|
||||
self.add_to_history(HistoryCell::new_user_prompt(prompt_text));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -831,6 +966,136 @@ impl ChatWidget<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn handle_browser_command(&mut self, command_text: String) {
|
||||
// Parse the browser subcommand
|
||||
let parts: Vec<&str> = command_text.trim().split_whitespace().collect();
|
||||
|
||||
let browser_manager = self.browser_manager.clone();
|
||||
let response = if parts.len() > 1 {
|
||||
let first_arg = parts[1];
|
||||
|
||||
// Check if the first argument looks like a URL (has a dot or protocol)
|
||||
let is_url = first_arg.contains("://") || first_arg.contains(".");
|
||||
|
||||
if is_url {
|
||||
// It's a URL - enable browser mode and navigate to it
|
||||
let url = parts[1..].join(" ");
|
||||
|
||||
// Ensure URL has protocol
|
||||
let full_url = if !url.contains("://") {
|
||||
format!("https://{}", url)
|
||||
} else {
|
||||
url.clone()
|
||||
};
|
||||
|
||||
// Enable browser mode
|
||||
browser_manager.set_enabled_sync(true);
|
||||
|
||||
// Navigate to URL and wait for it to load
|
||||
let browser_manager_open = browser_manager.clone();
|
||||
let url_for_goto = full_url.clone();
|
||||
tokio::spawn(async move {
|
||||
match browser_manager_open.goto(&url_for_goto).await {
|
||||
Ok(result) => {
|
||||
tracing::info!("Browser opened to: {} (title: {:?})", result.url, result.title);
|
||||
// Add a small delay to ensure page is fully rendered
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
|
||||
tracing::info!("Page should be loaded now");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to open browser: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
format!("Browser mode enabled. Opening: {}\nScreenshots will be attached to model requests.", full_url)
|
||||
} else {
|
||||
// It's a subcommand
|
||||
match first_arg {
|
||||
"off" => {
|
||||
// Disable browser mode
|
||||
browser_manager.set_enabled_sync(false);
|
||||
// Close any open browser
|
||||
let browser_manager_close = browser_manager.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = browser_manager_close.close().await {
|
||||
tracing::error!("Failed to close browser: {}", e);
|
||||
}
|
||||
});
|
||||
"Browser mode disabled.".to_string()
|
||||
}
|
||||
"status" => {
|
||||
// Get status from BrowserManager
|
||||
browser_manager.get_status_sync()
|
||||
}
|
||||
"fullpage" => {
|
||||
if parts.len() > 2 {
|
||||
match parts[2] {
|
||||
"on" => {
|
||||
// Enable full-page mode
|
||||
browser_manager.set_fullpage_sync(true);
|
||||
"Full-page screenshot mode enabled (max 8 segments).".to_string()
|
||||
}
|
||||
"off" => {
|
||||
// Disable full-page mode
|
||||
browser_manager.set_fullpage_sync(false);
|
||||
"Full-page screenshot mode disabled.".to_string()
|
||||
}
|
||||
_ => "Usage: /browser fullpage [on|off]".to_string()
|
||||
}
|
||||
} else {
|
||||
"Usage: /browser fullpage [on|off]".to_string()
|
||||
}
|
||||
}
|
||||
"config" => {
|
||||
if parts.len() > 3 {
|
||||
let key = parts[2];
|
||||
let value = parts[3..].join(" ");
|
||||
// Update browser config
|
||||
match key {
|
||||
"viewport" => {
|
||||
// Parse viewport dimensions like "1920x1080"
|
||||
if let Some((width_str, height_str)) = value.split_once('x') {
|
||||
if let (Ok(width), Ok(height)) = (width_str.parse::<u32>(), height_str.parse::<u32>()) {
|
||||
browser_manager.set_viewport_sync(width, height);
|
||||
format!("Browser viewport updated: {}x{}", width, height)
|
||||
} else {
|
||||
"Invalid viewport format. Use: /browser config viewport 1920x1080".to_string()
|
||||
}
|
||||
} else {
|
||||
"Invalid viewport format. Use: /browser config viewport 1920x1080".to_string()
|
||||
}
|
||||
}
|
||||
"segments_max" => {
|
||||
if let Ok(max) = value.parse::<usize>() {
|
||||
browser_manager.set_segments_max_sync(max);
|
||||
format!("Browser segments_max updated: {}", max)
|
||||
} else {
|
||||
"Invalid segments_max value. Use a number.".to_string()
|
||||
}
|
||||
}
|
||||
_ => format!("Unknown config key: {}. Available: viewport, segments_max", key)
|
||||
}
|
||||
} else {
|
||||
"Usage: /browser config <key> <value>\nAvailable keys: viewport, segments_max".to_string()
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
format!("Unknown browser command: '{}'\nUsage: /browser <url> | off | status | fullpage | config", first_arg)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
"Browser commands:\n• /browser <url> - Open URL and enable browser mode\n• /browser off - Disable browser mode\n• /browser status - Show current status\n• /browser fullpage [on|off] - Toggle full-page mode\n• /browser config <key> <value> - Update configuration".to_string()
|
||||
};
|
||||
|
||||
// Add the response to the UI as a background event
|
||||
let lines = response.lines().map(|line| Line::from(line.to_string())).collect();
|
||||
self.add_to_history(HistoryCell::BackgroundEvent {
|
||||
view: TextBlock::new(lines),
|
||||
});
|
||||
}
|
||||
|
||||
/// Programmatically submit a user text message as if typed in the
|
||||
/// composer. The text will be added to conversation history and sent to
|
||||
/// the agent.
|
||||
|
||||
@@ -17,6 +17,7 @@ pub enum SlashCommand {
|
||||
Compact,
|
||||
Diff,
|
||||
Status,
|
||||
Browser,
|
||||
Reasoning,
|
||||
Prompts,
|
||||
Logout,
|
||||
@@ -35,6 +36,7 @@ impl SlashCommand {
|
||||
SlashCommand::Quit => "exit Codex",
|
||||
SlashCommand::Diff => "show git diff (including untracked files)",
|
||||
SlashCommand::Status => "show current session configuration and token usage",
|
||||
SlashCommand::Browser => "manage browser mode (on/off/status/config)",
|
||||
SlashCommand::Reasoning => "change reasoning effort (low/medium/high/none)",
|
||||
SlashCommand::Prompts => "show example prompts",
|
||||
SlashCommand::Logout => "log out of Codex",
|
||||
|
||||
Reference in New Issue
Block a user