mirror of
https://github.com/hanzoai/dev.git
synced 2026-10-25 03:32:32 +00:00
feat(core/bridge): make screenshot/javascript actions
This commit is contained in:
@@ -25,7 +25,7 @@ Only run the lint when explicitly requested be by the user OR only the code you'
|
||||
- Local Sentry-style telemetry plus two-way control: error/console streaming, pageviews/screenshots, and control commands. Install in apps via npm: `@just-every/code-bridge`.
|
||||
- Host writes `.code/code-bridge.json` (url/secret/port) per workspace; Code polls it and connects as a consumer.
|
||||
- Bridge clients send console/errors/screenshot/pageview/control events.
|
||||
- Adjust subscriptions with the internal tool `code_bridge` (actions: subscribe | command). `subscribe` sets the workspace default level (errors|warn|info|trace) and enables all capabilities; `command` supports `screenshot` and `javascript` (requires `code`). Examples: `{"action":"subscribe","level":"trace"}`, `{"action":"command","command":"screenshot"}`, `{"action":"command","command":"javascript","code":"window.location.href"}`.
|
||||
- Adjust subscriptions with the internal tool `code_bridge` (actions: subscribe | screenshot | javascript). `subscribe` sets the workspace default level (errors|warn|info|trace) and enables all capabilities; `screenshot` requests a capture; `javascript` runs provided JS on the bridge client (requires `code`). Examples: `{"action":"subscribe","level":"trace"}`, `{"action":"screenshot"}`, `{"action":"javascript","code":"window.location.href"}`.
|
||||
- Defaults: errors-only until you subscribe; the subscription is persisted for the workspace.
|
||||
|
||||
# Tools
|
||||
@@ -45,7 +45,7 @@ When you run shell tools with Code they will run in the foreground for up to 10
|
||||
Use the browser tools to open a live page, interact with it, and harvest results. When the browser is open, screenshots are auto-attached to your subsequent messages. The browser will either be an internal headless browser, or a CPD connection to the user's active Chrome browser. Your screenshots will be 1024×768 which exactly matches the viewport.
|
||||
|
||||
## Code Bridge
|
||||
A local Sentry-like bridge for development environments: add `@just-every/code-bridge` to your JavaScript app to stream errors/console, pageviews/screenshots, and expose a control channel for two-way, real-time debugging. The `code_bridge` tool supports: `{"action":"subscribe","level":"trace|info|warn|errors"}` (persists workspace defaults and always requests full capabilities), `{"action":"command","command":"screenshot"}` to ask connected bridges for a screenshot, and `{"action":"command","command":"javascript","code":"<JS to run>"}` to execute JS on the bridge and return the result.
|
||||
A local Sentry-like bridge for development environments: add `@just-every/code-bridge` to your JavaScript app to stream errors/console, pageviews/screenshots, and expose a control channel for two-way, real-time debugging. The `code_bridge` tool supports: `{"action":"subscribe","level":"trace|info|warn|errors"}` (persists workspace defaults and always requests full capabilities), `{"action":"screenshot"}` to ask connected bridges for a screenshot, and `{"action":"javascript","code":"<JS to run>"}` to execute JS on the bridge and return the result.
|
||||
|
||||
## Web tools
|
||||
Use `web.run` when you need multi-step browsing—search, opens, clicks, screenshots, or specialized lookups. Use `browser {"action":"fetch","url":"https://example.com"}` when you already know the URL and just need its Markdown content in a single fetch.
|
||||
|
||||
@@ -729,7 +729,7 @@ async fn connect_and_listen(meta: BridgeMeta, session: Arc<Session>, cwd: &Path)
|
||||
if !BRIDGE_HINT_EMITTED.swap(true, Ordering::SeqCst) && workspace_has_code_bridge(cwd) {
|
||||
session
|
||||
.record_bridge_event(
|
||||
"Code Bridge is a local, real-time debug stream (errors/console like Sentry, plus pageviews/screenshots and a control channel). Use the `code_bridge` tool: `action=subscribe` with level (errors|warn|info|trace) to persist full-capability logging, or `action=command` with `command=screenshot` or `command=javascript` (plus `code`)."
|
||||
"Code Bridge is a local, real-time debug stream (errors/console like Sentry, plus pageviews/screenshots and a control channel). Use the `code_bridge` tool: `action=subscribe` with level (errors|warn|info|trace) to persist full-capability logging, `action=screenshot` to request a capture, or `action=javascript` with `code` to run JS on the bridge client."
|
||||
.to_string(),
|
||||
)
|
||||
.await;
|
||||
|
||||
+16
-45
@@ -6731,8 +6731,6 @@ struct BridgeControlArgs {
|
||||
#[serde(default)]
|
||||
level: Option<String>,
|
||||
#[serde(default)]
|
||||
command: Option<String>,
|
||||
#[serde(default)]
|
||||
code: Option<String>,
|
||||
}
|
||||
|
||||
@@ -6823,64 +6821,37 @@ async fn handle_code_bridge_with_cwd(
|
||||
output: FunctionCallOutputPayload { content: "ok".to_string(), success: Some(true) },
|
||||
}
|
||||
}
|
||||
"command" => {
|
||||
let cmd = match args.command.as_ref().map(|c| c.trim().to_lowercase()) {
|
||||
Some(c) if c == "screenshot" || c == "javascript" => c,
|
||||
Some(_) => {
|
||||
return ResponseInputItem::FunctionCallOutput {
|
||||
call_id: ctx.call_id.clone(),
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "unsupported command (use screenshot|javascript)".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
"screenshot" => {
|
||||
send_bridge_control("screenshot", serde_json::json!({}));
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id: ctx.call_id.clone(),
|
||||
output: FunctionCallOutputPayload { content: "requested screenshot".to_string(), success: Some(true) },
|
||||
}
|
||||
}
|
||||
"javascript" => {
|
||||
let code = match args.code.as_ref().map(|c| c.trim()).filter(|c| !c.is_empty()) {
|
||||
Some(c) => c,
|
||||
None => {
|
||||
return ResponseInputItem::FunctionCallOutput {
|
||||
call_id: ctx.call_id.clone(),
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "missing command".to_string(),
|
||||
content: "missing code for javascript action".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
match cmd.as_str() {
|
||||
"screenshot" => {
|
||||
send_bridge_control("screenshot", serde_json::json!({}));
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id: ctx.call_id.clone(),
|
||||
output: FunctionCallOutputPayload { content: "requested screenshot".to_string(), success: Some(true) },
|
||||
}
|
||||
}
|
||||
"javascript" => {
|
||||
let code = match args.code.as_ref().map(|c| c.trim()).filter(|c| !c.is_empty()) {
|
||||
Some(c) => c,
|
||||
None => {
|
||||
return ResponseInputItem::FunctionCallOutput {
|
||||
call_id: ctx.call_id.clone(),
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "missing code for javascript command".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
};
|
||||
send_bridge_control("javascript", serde_json::json!({ "code": code }));
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id: ctx.call_id.clone(),
|
||||
output: FunctionCallOutputPayload { content: "sent javascript".to_string(), success: Some(true) },
|
||||
}
|
||||
}
|
||||
_ => unreachable!(),
|
||||
send_bridge_control("javascript", serde_json::json!({ "code": code }));
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id: ctx.call_id.clone(),
|
||||
output: FunctionCallOutputPayload { content: "sent javascript".to_string(), success: Some(true) },
|
||||
}
|
||||
}
|
||||
// Keep legacy actions for backward compatibility with older prompts/tools
|
||||
"show" | "set" | "clear" => ResponseInputItem::FunctionCallOutput {
|
||||
call_id: ctx.call_id.clone(),
|
||||
output: FunctionCallOutputPayload {
|
||||
content: "deprecated action; use subscribe or command (screenshot|javascript)".to_string(),
|
||||
content: "deprecated action; use subscribe, screenshot, or javascript".to_string(),
|
||||
success: Some(false),
|
||||
},
|
||||
},
|
||||
|
||||
@@ -806,12 +806,13 @@ pub fn create_bridge_tool() -> OpenAiTool {
|
||||
"action".to_string(),
|
||||
JsonSchema::String {
|
||||
description: Some(
|
||||
"Required: subscribe (set level + persist), command (screenshot | javascript)."
|
||||
"Required: subscribe (set level + persist), screenshot (request a screenshot), javascript (run JS on the bridge client)."
|
||||
.to_string(),
|
||||
),
|
||||
allowed_values: Some(vec![
|
||||
"subscribe".to_string(),
|
||||
"command".to_string(),
|
||||
"screenshot".to_string(),
|
||||
"javascript".to_string(),
|
||||
]),
|
||||
},
|
||||
);
|
||||
@@ -832,27 +833,10 @@ pub fn create_bridge_tool() -> OpenAiTool {
|
||||
},
|
||||
);
|
||||
|
||||
properties.insert(
|
||||
"command".to_string(),
|
||||
JsonSchema::String {
|
||||
description: Some("For action=command: screenshot | javascript.".to_string()),
|
||||
allowed_values: Some(vec!["screenshot".to_string(), "javascript".to_string()]),
|
||||
},
|
||||
);
|
||||
|
||||
properties.insert(
|
||||
"args".to_string(),
|
||||
JsonSchema::Object {
|
||||
properties: BTreeMap::new(),
|
||||
required: None,
|
||||
additional_properties: None,
|
||||
},
|
||||
);
|
||||
|
||||
properties.insert(
|
||||
"code".to_string(),
|
||||
JsonSchema::String {
|
||||
description: Some("For action=command & command=javascript: JS to execute on the bridge client.".to_string()),
|
||||
description: Some("For action=javascript: JS to execute on the bridge client.".to_string()),
|
||||
allowed_values: None,
|
||||
},
|
||||
);
|
||||
@@ -860,7 +844,7 @@ pub fn create_bridge_tool() -> OpenAiTool {
|
||||
OpenAiTool::Function(ResponsesApiTool {
|
||||
name: "code_bridge".to_string(),
|
||||
description:
|
||||
"Code Bridge = local Sentry-style event stream + two-way control (errors/console/pageviews/screenshots/control). Actions: subscribe (set level, persists, requests full capabilities), command=screenshot (ask bridges for a screenshot), command=javascript (send JS to execute and return result). Examples: {\"action\":\"subscribe\",\"level\":\"trace\"}, {\"action\":\"command\",\"command\":\"screenshot\"}, {\"action\":\"command\",\"command\":\"javascript\",\"code\":\"window.location.href\"}.".to_string(),
|
||||
"Code Bridge = local Sentry-style event stream + two-way control (errors/console/pageviews/screenshots/control). Actions: subscribe (set level, persists, requests full capabilities), screenshot (ask bridges for a screenshot), javascript (send JS to execute and return result). Examples: {\"action\":\"subscribe\",\"level\":\"trace\"}, {\"action\":\"screenshot\"}, {\"action\":\"javascript\",\"code\":\"window.location.href\"}.".to_string(),
|
||||
strict: false,
|
||||
parameters: JsonSchema::Object {
|
||||
properties,
|
||||
|
||||
Reference in New Issue
Block a user