feat(core/bridge): make screenshot/javascript actions

This commit is contained in:
James Peter
2025-12-03 20:51:48 +10:00
parent f54b84f711
commit 5380f2cb41
4 changed files with 24 additions and 69 deletions
+2 -2
View File
@@ -25,7 +25,7 @@ Only run the lint when explicitly requested be by the user OR only the code you'
- Local Sentry-style telemetry plus two-way control: error/console streaming, pageviews/screenshots, and control commands. Install in apps via npm: `@just-every/code-bridge`.
- Host writes `.code/code-bridge.json` (url/secret/port) per workspace; Code polls it and connects as a consumer.
- Bridge clients send console/errors/screenshot/pageview/control events.
- Adjust subscriptions with the internal tool `code_bridge` (actions: subscribe | command). `subscribe` sets the workspace default level (errors|warn|info|trace) and enables all capabilities; `command` supports `screenshot` and `javascript` (requires `code`). Examples: `{"action":"subscribe","level":"trace"}`, `{"action":"command","command":"screenshot"}`, `{"action":"command","command":"javascript","code":"window.location.href"}`.
- Adjust subscriptions with the internal tool `code_bridge` (actions: subscribe | screenshot | javascript). `subscribe` sets the workspace default level (errors|warn|info|trace) and enables all capabilities; `screenshot` requests a capture; `javascript` runs provided JS on the bridge client (requires `code`). Examples: `{"action":"subscribe","level":"trace"}`, `{"action":"screenshot"}`, `{"action":"javascript","code":"window.location.href"}`.
- Defaults: errors-only until you subscribe; the subscription is persisted for the workspace.
# Tools
@@ -45,7 +45,7 @@ When you run shell tools with Code they will run in the foreground for up to 10
Use the browser tools to open a live page, interact with it, and harvest results. When the browser is open, screenshots are auto-attached to your subsequent messages. The browser will either be an internal headless browser, or a CPD connection to the user's active Chrome browser. Your screenshots will be 1024×768 which exactly matches the viewport.
## Code Bridge
A local Sentry-like bridge for development environments: add `@just-every/code-bridge` to your JavaScript app to stream errors/console, pageviews/screenshots, and expose a control channel for two-way, real-time debugging. The `code_bridge` tool supports: `{"action":"subscribe","level":"trace|info|warn|errors"}` (persists workspace defaults and always requests full capabilities), `{"action":"command","command":"screenshot"}` to ask connected bridges for a screenshot, and `{"action":"command","command":"javascript","code":"<JS to run>"}` to execute JS on the bridge and return the result.
A local Sentry-like bridge for development environments: add `@just-every/code-bridge` to your JavaScript app to stream errors/console, pageviews/screenshots, and expose a control channel for two-way, real-time debugging. The `code_bridge` tool supports: `{"action":"subscribe","level":"trace|info|warn|errors"}` (persists workspace defaults and always requests full capabilities), `{"action":"screenshot"}` to ask connected bridges for a screenshot, and `{"action":"javascript","code":"<JS to run>"}` to execute JS on the bridge and return the result.
## Web tools
Use `web.run` when you need multi-step browsing—search, opens, clicks, screenshots, or specialized lookups. Use `browser {"action":"fetch","url":"https://example.com"}` when you already know the URL and just need its Markdown content in a single fetch.
+1 -1
View File
@@ -729,7 +729,7 @@ async fn connect_and_listen(meta: BridgeMeta, session: Arc<Session>, cwd: &Path)
if !BRIDGE_HINT_EMITTED.swap(true, Ordering::SeqCst) && workspace_has_code_bridge(cwd) {
session
.record_bridge_event(
"Code Bridge is a local, real-time debug stream (errors/console like Sentry, plus pageviews/screenshots and a control channel). Use the `code_bridge` tool: `action=subscribe` with level (errors|warn|info|trace) to persist full-capability logging, or `action=command` with `command=screenshot` or `command=javascript` (plus `code`)."
"Code Bridge is a local, real-time debug stream (errors/console like Sentry, plus pageviews/screenshots and a control channel). Use the `code_bridge` tool: `action=subscribe` with level (errors|warn|info|trace) to persist full-capability logging, `action=screenshot` to request a capture, or `action=javascript` with `code` to run JS on the bridge client."
.to_string(),
)
.await;
+16 -45
View File
@@ -6731,8 +6731,6 @@ struct BridgeControlArgs {
#[serde(default)]
level: Option<String>,
#[serde(default)]
command: Option<String>,
#[serde(default)]
code: Option<String>,
}
@@ -6823,64 +6821,37 @@ async fn handle_code_bridge_with_cwd(
output: FunctionCallOutputPayload { content: "ok".to_string(), success: Some(true) },
}
}
"command" => {
let cmd = match args.command.as_ref().map(|c| c.trim().to_lowercase()) {
Some(c) if c == "screenshot" || c == "javascript" => c,
Some(_) => {
return ResponseInputItem::FunctionCallOutput {
call_id: ctx.call_id.clone(),
output: FunctionCallOutputPayload {
content: "unsupported command (use screenshot|javascript)".to_string(),
success: Some(false),
},
}
}
"screenshot" => {
send_bridge_control("screenshot", serde_json::json!({}));
ResponseInputItem::FunctionCallOutput {
call_id: ctx.call_id.clone(),
output: FunctionCallOutputPayload { content: "requested screenshot".to_string(), success: Some(true) },
}
}
"javascript" => {
let code = match args.code.as_ref().map(|c| c.trim()).filter(|c| !c.is_empty()) {
Some(c) => c,
None => {
return ResponseInputItem::FunctionCallOutput {
call_id: ctx.call_id.clone(),
output: FunctionCallOutputPayload {
content: "missing command".to_string(),
content: "missing code for javascript action".to_string(),
success: Some(false),
},
}
}
};
match cmd.as_str() {
"screenshot" => {
send_bridge_control("screenshot", serde_json::json!({}));
ResponseInputItem::FunctionCallOutput {
call_id: ctx.call_id.clone(),
output: FunctionCallOutputPayload { content: "requested screenshot".to_string(), success: Some(true) },
}
}
"javascript" => {
let code = match args.code.as_ref().map(|c| c.trim()).filter(|c| !c.is_empty()) {
Some(c) => c,
None => {
return ResponseInputItem::FunctionCallOutput {
call_id: ctx.call_id.clone(),
output: FunctionCallOutputPayload {
content: "missing code for javascript command".to_string(),
success: Some(false),
},
}
}
};
send_bridge_control("javascript", serde_json::json!({ "code": code }));
ResponseInputItem::FunctionCallOutput {
call_id: ctx.call_id.clone(),
output: FunctionCallOutputPayload { content: "sent javascript".to_string(), success: Some(true) },
}
}
_ => unreachable!(),
send_bridge_control("javascript", serde_json::json!({ "code": code }));
ResponseInputItem::FunctionCallOutput {
call_id: ctx.call_id.clone(),
output: FunctionCallOutputPayload { content: "sent javascript".to_string(), success: Some(true) },
}
}
// Keep legacy actions for backward compatibility with older prompts/tools
"show" | "set" | "clear" => ResponseInputItem::FunctionCallOutput {
call_id: ctx.call_id.clone(),
output: FunctionCallOutputPayload {
content: "deprecated action; use subscribe or command (screenshot|javascript)".to_string(),
content: "deprecated action; use subscribe, screenshot, or javascript".to_string(),
success: Some(false),
},
},
+5 -21
View File
@@ -806,12 +806,13 @@ pub fn create_bridge_tool() -> OpenAiTool {
"action".to_string(),
JsonSchema::String {
description: Some(
"Required: subscribe (set level + persist), command (screenshot | javascript)."
"Required: subscribe (set level + persist), screenshot (request a screenshot), javascript (run JS on the bridge client)."
.to_string(),
),
allowed_values: Some(vec![
"subscribe".to_string(),
"command".to_string(),
"screenshot".to_string(),
"javascript".to_string(),
]),
},
);
@@ -832,27 +833,10 @@ pub fn create_bridge_tool() -> OpenAiTool {
},
);
properties.insert(
"command".to_string(),
JsonSchema::String {
description: Some("For action=command: screenshot | javascript.".to_string()),
allowed_values: Some(vec!["screenshot".to_string(), "javascript".to_string()]),
},
);
properties.insert(
"args".to_string(),
JsonSchema::Object {
properties: BTreeMap::new(),
required: None,
additional_properties: None,
},
);
properties.insert(
"code".to_string(),
JsonSchema::String {
description: Some("For action=command & command=javascript: JS to execute on the bridge client.".to_string()),
description: Some("For action=javascript: JS to execute on the bridge client.".to_string()),
allowed_values: None,
},
);
@@ -860,7 +844,7 @@ pub fn create_bridge_tool() -> OpenAiTool {
OpenAiTool::Function(ResponsesApiTool {
name: "code_bridge".to_string(),
description:
"Code Bridge = local Sentry-style event stream + two-way control (errors/console/pageviews/screenshots/control). Actions: subscribe (set level, persists, requests full capabilities), command=screenshot (ask bridges for a screenshot), command=javascript (send JS to execute and return result). Examples: {\"action\":\"subscribe\",\"level\":\"trace\"}, {\"action\":\"command\",\"command\":\"screenshot\"}, {\"action\":\"command\",\"command\":\"javascript\",\"code\":\"window.location.href\"}.".to_string(),
"Code Bridge = local Sentry-style event stream + two-way control (errors/console/pageviews/screenshots/control). Actions: subscribe (set level, persists, requests full capabilities), screenshot (ask bridges for a screenshot), javascript (send JS to execute and return result). Examples: {\"action\":\"subscribe\",\"level\":\"trace\"}, {\"action\":\"screenshot\"}, {\"action\":\"javascript\",\"code\":\"window.location.href\"}.".to_string(),
strict: false,
parameters: JsonSchema::Object {
properties,