mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
945 lines
51 KiB
HTML
945 lines
51 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0015</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0015.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0015">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0015.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0015</h1><p class="lede">HIP-15: Computer Control Standard. Status Final. Hanzo architectural specification.</p><h1>HIP-0015: Computer Control Standard</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal defines the Computer Control Standard for Hanzo Operative, a unified interface that enables AI agents to perceive, interpret, and interact with computer environments. The standard covers screen capture and recording, mouse and keyboard input, touch emulation, browser automation via Playwright, AI-driven visual interpretation of screen content, and device emulation across platforms. All computer control in the Hanzo ecosystem MUST conform to this interface.</p>
|
|
<p><strong>Repository</strong>: <a href="https://github.com/hanzoai/operative">github.com/hanzoai/operative</a> <strong>MCP Package</strong>: <code>hanzo-mcp</code> (computer use tools) <strong>Platforms</strong>: macOS (Quartz), Linux (X11/Wayland), Windows (Win32)</p>
|
|
<h2>Design Philosophy</h2>
|
|
<h3>Why Computer Use Matters for AI</h3>
|
|
<p>LLMs are powerful reasoning engines constrained to text input and text output. They can write code but not run it in an IDE. They can describe a UI fix but not verify it visually. They can plan a workflow but not execute it across applications.</p>
|
|
<p>Computer use removes this constraint. When an AI agent can see the screen, move the mouse, type on the keyboard, and navigate between applications, it gains the ability to operate any software that a human can operate. This transforms AI from a conversational assistant into a productive worker that can:</p>
|
|
<ul><li>Navigate complex enterprise applications (SAP, Salesforce, internal tools)</li><li>Perform multi-step workflows across multiple applications</li><li>Verify visual output (design reviews, chart validation, UI testing)</li><li>Operate legacy software that has no API</li><li>Debug applications by observing their runtime behavior</li></ul>
|
|
<h3>Why Playwright for Browser Automation</h3>
|
|
<p>For web-specific tasks, screen-based computer use is unnecessarily imprecise. Playwright (by Microsoft) provides a purpose-built browser automation layer with:</p>
|
|
<ul><li><strong>Cross-browser support</strong>: Chromium, Firefox, WebKit</li><li><strong>Reliable selectors</strong>: CSS, XPath, text, role, test-id, and other locator strategies</li><li><strong>Built-in waiting</strong>: Auto-waits for elements to be actionable before interacting</li><li><strong>Network interception</strong>: Mock APIs, block requests, intercept responses</li><li><strong>Mobile emulation</strong>: Simulate any device viewport, user agent, and touch input</li><li><strong>Trace recording</strong>: Debug failing automations with full execution traces</li></ul>
|
|
<p>Operative uses both approaches: Playwright for web applications (faster, more reliable, semantically rich) and screen recording for native applications (universal, no API required). The agent chooses the appropriate backend based on the target application.</p>
|
|
<h2>Specification</h2>
|
|
<h3>Core Interfaces</h3>
|
|
<h4>Screen Capture</h4>
|
|
<pre><code class="lang-typescript">interface ScreenCapture {
|
|
// Single screenshot
|
|
screenshot(options?: {
|
|
region?: Region; // Capture specific area
|
|
optimize?: boolean; // Compress for LLM (default: true)
|
|
maxSize?: number; // Max dimension in pixels (default: 768)
|
|
quality?: number; // JPEG quality 1-100 (default: 60)
|
|
}): Promise<CaptureResult>;
|
|
|
|
// Screen recording session
|
|
session(options?: {
|
|
duration?: number; // Seconds (default: 30, max: 120)
|
|
fps?: number; // Frame rate (default: 30)
|
|
quality?: "low" | "medium" | "high";
|
|
region?: Region; // Record specific area
|
|
targetFrames?: number; // Target keyframes to extract (default: 30)
|
|
activityThreshold?: number; // Activity sensitivity (default: 0.02)
|
|
}): Promise<SessionResult>;
|
|
|
|
// Background recording (start/stop)
|
|
record(options?: RecordOptions): Promise<RecordingHandle>;
|
|
stop(handle: RecordingHandle): Promise<SessionResult>;
|
|
|
|
// Get display information
|
|
getScreens(): Promise<ScreenInfo[]>;
|
|
screenSize(): Promise<Dimensions>;
|
|
currentScreen(): Promise<ScreenInfo>;
|
|
}
|
|
|
|
interface CaptureResult {
|
|
image: string; // Base64-encoded image data
|
|
width: number;
|
|
height: number;
|
|
timestamp: number;
|
|
format: "jpeg" | "png";
|
|
}
|
|
|
|
interface SessionResult {
|
|
frames: CaptureResult[]; // Extracted keyframes
|
|
activities: Activity[]; // Detected activity events
|
|
duration: number; // Actual recording duration
|
|
totalSize: number; // Total payload size in bytes
|
|
}
|
|
|
|
interface Activity {
|
|
timestamp: number;
|
|
type: "movement" | "click" | "typing" | "window_change" | "scroll";
|
|
position?: Point;
|
|
details?: string;
|
|
}</code></pre>
|
|
<h4>Input Control</h4>
|
|
<pre><code class="lang-typescript">interface InputControl {
|
|
// Mouse operations (< 5ms native latency)
|
|
click(x: number, y: number, options?: ClickOptions): Promise<void>;
|
|
doubleClick(x: number, y: number): Promise<void>;
|
|
rightClick(x: number, y: number): Promise<void>;
|
|
middleClick(x: number, y: number): Promise<void>;
|
|
move(x: number, y: number): Promise<void>;
|
|
moveRelative(dx: number, dy: number): Promise<void>;
|
|
drag(startX: number, startY: number, endX: number, endY: number,
|
|
options?: DragOptions): Promise<void>;
|
|
scroll(amount: number, x?: number, y?: number): Promise<void>;
|
|
|
|
// Keyboard operations (< 2ms native latency)
|
|
type(text: string, options?: TypeOptions): Promise<void>;
|
|
press(key: string): Promise<void>;
|
|
keyDown(key: string): Promise<void>;
|
|
keyUp(key: string): Promise<void>;
|
|
hotkey(keys: string[]): Promise<void>;
|
|
|
|
// Touch operations
|
|
tap(x: number, y: number): Promise<void>;
|
|
swipe(direction: Direction, distance?: number): Promise<void>;
|
|
pinch(scale: number): Promise<void>;
|
|
|
|
// Batch execution
|
|
batch(actions: Action[]): Promise<BatchResult>;
|
|
}
|
|
|
|
interface ClickOptions {
|
|
button?: "left" | "right" | "middle";
|
|
clickCount?: number;
|
|
delay?: number; // Delay between mousedown and mouseup
|
|
}
|
|
|
|
interface TypeOptions {
|
|
interval?: number; // Delay between keystrokes (default: 0.02s)
|
|
clear?: boolean; // Clear field before typing
|
|
}
|
|
|
|
interface DragOptions {
|
|
duration?: number; // Drag duration in seconds
|
|
steps?: number; // Intermediate points for smooth drag
|
|
}</code></pre>
|
|
<h4>Window Management</h4>
|
|
<pre><code class="lang-typescript">interface WindowManager {
|
|
getActiveWindow(): Promise<WindowInfo>;
|
|
listWindows(): Promise<WindowInfo[]>;
|
|
focusWindow(title: string, options?: {
|
|
useRegex?: boolean;
|
|
}): Promise<void>;
|
|
}
|
|
|
|
interface WindowInfo {
|
|
title: string;
|
|
bounds: Region;
|
|
pid: number;
|
|
isActive: boolean;
|
|
application: string;
|
|
}</code></pre>
|
|
<h4>Region Operations</h4>
|
|
<pre><code class="lang-typescript">interface RegionManager {
|
|
// Define a named region for repeated operations
|
|
defineRegion(name: string, x: number, y: number,
|
|
width: number, height: number): Promise<void>;
|
|
|
|
// Capture screenshot of named region
|
|
regionScreenshot(name: string): Promise<CaptureResult>;
|
|
|
|
// Find image within named region
|
|
regionLocate(name: string, imagePath: string): Promise<Point | null>;
|
|
}</code></pre>
|
|
<h4>Image Location</h4>
|
|
<pre><code class="lang-typescript">interface ImageLocator {
|
|
// Find image on screen, return center point
|
|
locate(imagePath: string, options?: {
|
|
confidence?: number; // Match confidence (default: 0.9)
|
|
}): Promise<Point | null>;
|
|
|
|
// Find all instances of image
|
|
locateAll(imagePath: string): Promise<Point[]>;
|
|
|
|
// Wait for image to appear
|
|
waitForImage(imagePath: string, timeout?: number): Promise<Point>;
|
|
|
|
// Wait for image to disappear
|
|
waitWhileImage(imagePath: string, timeout?: number): Promise<void>;
|
|
|
|
// Get pixel color at point
|
|
pixel(x: number, y: number): Promise<Color>;
|
|
|
|
// Check if pixel matches color
|
|
pixelMatches(x: number, y: number, color: Color,
|
|
tolerance?: number): Promise<boolean>;
|
|
}</code></pre>
|
|
<h4>Browser Automation (Playwright Integration)</h4>
|
|
<pre><code class="lang-typescript">interface BrowserControl {
|
|
// Navigation
|
|
navigate(url: string): Promise<NavigateResult>;
|
|
reload(): Promise<void>;
|
|
goBack(): Promise<void>;
|
|
goForward(): Promise<void>;
|
|
url(): Promise<string>;
|
|
title(): Promise<string>;
|
|
content(): Promise<string>;
|
|
|
|
// Element interaction
|
|
click(selector: string, options?: BrowserClickOptions): Promise<void>;
|
|
fill(selector: string, text: string): Promise<void>;
|
|
type(selector: string, text: string): Promise<void>;
|
|
press(selector: string, key: string): Promise<void>;
|
|
selectOption(selector: string, value: string): Promise<void>;
|
|
check(selector: string): Promise<void>;
|
|
uncheck(selector: string): Promise<void>;
|
|
upload(selector: string, files: string[]): Promise<void>;
|
|
hover(selector: string): Promise<void>;
|
|
|
|
// Element locators
|
|
locator(selector: string): Locator;
|
|
getByRole(role: string, options?: { name?: string }): Locator;
|
|
getByText(text: string, options?: { exact?: boolean }): Locator;
|
|
getByLabel(text: string): Locator;
|
|
getByPlaceholder(text: string): Locator;
|
|
getByTestId(testId: string): Locator;
|
|
|
|
// Element state
|
|
getText(selector: string): Promise<string>;
|
|
getAttribute(selector: string, name: string): Promise<string | null>;
|
|
getValue(selector: string): Promise<string>;
|
|
isVisible(selector: string): Promise<boolean>;
|
|
isEnabled(selector: string): Promise<boolean>;
|
|
|
|
// Assertions
|
|
expectVisible(selector: string): Promise<void>;
|
|
expectText(selector: string, expected: string): Promise<void>;
|
|
expectValue(selector: string, expected: string): Promise<void>;
|
|
expectUrl(pattern: string): Promise<void>;
|
|
expectTitle(expected: string): Promise<void>;
|
|
|
|
// Waiting
|
|
wait(timeout: number): Promise<void>;
|
|
waitForLoad(state?: "load" | "domcontentloaded" | "networkidle"): Promise<void>;
|
|
waitForUrl(pattern: string): Promise<void>;
|
|
waitForSelector(selector: string, options?: WaitOptions): Promise<void>;
|
|
|
|
// Screenshots and capture
|
|
screenshot(options?: { fullPage?: boolean }): Promise<CaptureResult>;
|
|
pdf(): Promise<Buffer>;
|
|
snapshot(): Promise<string>; // Accessibility tree snapshot
|
|
|
|
// JavaScript execution
|
|
evaluate(code: string): Promise<unknown>;
|
|
|
|
// Device emulation
|
|
emulate(device: DevicePreset): Promise<void>;
|
|
viewport(width: number, height: number): Promise<void>;
|
|
|
|
// Network interception
|
|
route(pattern: string, options: RouteOptions): Promise<void>;
|
|
unroute(pattern: string): Promise<void>;
|
|
|
|
// Multi-tab / multi-context
|
|
newPage(): Promise<PageHandle>;
|
|
newContext(options?: ContextOptions): Promise<ContextHandle>;
|
|
newTab(): Promise<TabHandle>;
|
|
tabs(): Promise<TabInfo[]>;
|
|
|
|
// Storage and cookies
|
|
cookies(): Promise<Cookie[]>;
|
|
clearCookies(): Promise<void>;
|
|
storage(type: "local" | "session"): Promise<Record<string, string>>;
|
|
|
|
// Debugging
|
|
highlight(selector: string): Promise<void>;
|
|
traceStart(): Promise<void>;
|
|
traceStop(path: string): Promise<void>;
|
|
console(): Promise<ConsoleMessage[]>;
|
|
errors(): Promise<Error[]>;
|
|
}
|
|
|
|
type DevicePreset =
|
|
| "mobile" | "tablet" | "laptop"
|
|
| "iphone_14" | "pixel_7" | "ipad_pro";
|
|
|
|
interface ContextOptions {
|
|
device?: DevicePreset;
|
|
locale?: string;
|
|
geolocation?: { latitude: number; longitude: number };
|
|
permissions?: string[];
|
|
storageState?: string; // Path to saved auth state
|
|
}</code></pre>
|
|
<h3>AI Interpretation Pipeline</h3>
|
|
<p>The AI interpretation pipeline is the core innovation that distinguishes Operative from traditional automation frameworks. Rather than requiring explicit selectors or coordinates, the agent observes the screen and reasons about what to do next.</p>
|
|
<pre><code>┌──────────────────────────────────────────────────────────────────┐
|
|
│ AI Interpretation Pipeline │
|
|
├──────────────────────────────────────────────────────────────────┤
|
|
│ │
|
|
│ Screen Recording ──> Activity Detection ──> Keyframe Extraction │
|
|
│ │ │
|
|
│ v │
|
|
│ Frame Compression │
|
|
│ (~768px, 60% JPEG) │
|
|
│ │ │
|
|
│ v │
|
|
│ LLM Visual Analysis │
|
|
│ (multimodal reasoning) │
|
|
│ │ │
|
|
│ v │
|
|
│ Action Planning │
|
|
│ (click, type, scroll) │
|
|
│ │ │
|
|
│ v │
|
|
│ Action Execution │
|
|
│ (native or Playwright) │
|
|
│ │ │
|
|
│ v │
|
|
│ Verification Capture │
|
|
│ (screenshot after action) │
|
|
│ │ │
|
|
│ v │
|
|
│ Loop / Complete │
|
|
│ │
|
|
└──────────────────────────────────────────────────────────────────┘</code></pre>
|
|
<h4>Activity Detection</h4>
|
|
<p>Activity detection identifies moments of significance in a screen recording. The system monitors for:</p>
|
|
<p>| Activity Type | Detection Method | Threshold | |---------------|-----------------|-----------| | Mouse movement | Frame-to-frame cursor position delta | > 10px displacement | | Click events | OS-level input event hooks | Any click | | Typing | Keyboard event hooks + screen text change | Any keystroke | | Window change | Active window title change | Title differs | | Scroll | Viewport content shift without cursor movement | > 5% content shift | | Animation | Pixel-level frame difference | > 2% of frame area |</p>
|
|
<h4>Keyframe Selection</h4>
|
|
<p>From a 30-second recording at 30fps (900 raw frames), the system selects approximately 30 keyframes using the following algorithm:</p>
|
|
<ol><li>Score each frame by the sum of activity intensities at that timestamp</li><li>Apply non-maximum suppression with a 500ms window to avoid duplicates</li><li>Always include the first frame, last frame, and frames immediately after click events</li><li>Fill remaining slots with the highest-scoring frames distributed evenly across the timeline</li></ol>
|
|
<h4>Frame Compression</h4>
|
|
<p>Each keyframe is compressed for efficient LLM consumption:</p>
|
|
<ul><li>Maximum dimension: 768 pixels (configurable via <code>HANZO_SCREEN_MAX_SIZE</code>)</li><li>JPEG quality: 60% (configurable via <code>HANZO_SCREEN_QUALITY</code>)</li><li>Target total payload: ~500KB for a 30-frame session</li><li>Average per-frame size: ~15-20KB</li></ul>
|
|
<h3>Control Flow: Agent Observe-Act Loop</h3>
|
|
<p>The standard defines a control flow for agents interacting with computers:</p>
|
|
<pre><code class="lang-python">class ComputerUseAgent:
|
|
"""
|
|
Agent that controls a computer through the Operative interface.
|
|
Integrates with HIP-9 Agent SDK and HIP-10 MCP tools.
|
|
"""
|
|
|
|
def __init__(self, operative: OperativeClient, model: LLMClient):
|
|
self.operative = operative
|
|
self.model = model
|
|
self.history = []
|
|
|
|
async def execute_task(self, task: str) -> TaskResult:
|
|
"""
|
|
Execute a task on the computer by observing and acting in a loop.
|
|
"""
|
|
# Initial observation
|
|
screen = await self.operative.screenshot()
|
|
self.history.append({"role": "observation", "image": screen})
|
|
|
|
while not self.is_complete():
|
|
# Ask the LLM what to do given the screen and task
|
|
action = await self.model.reason(
|
|
task=task,
|
|
current_screen=screen,
|
|
history=self.history,
|
|
available_actions=self.operative.capabilities()
|
|
)
|
|
|
|
if action.type == "done":
|
|
return TaskResult(success=True, output=action.summary)
|
|
|
|
# Execute the action
|
|
await self.operative.execute(action)
|
|
self.history.append({"role": "action", "action": action})
|
|
|
|
# Observe the result
|
|
await asyncio.sleep(action.wait_after or 0.5)
|
|
screen = await self.operative.screenshot()
|
|
self.history.append({"role": "observation", "image": screen})
|
|
|
|
return TaskResult(success=False, output="Max iterations reached")</code></pre>
|
|
<h3>Batch Operations</h3>
|
|
<p>Batch operations execute multiple actions atomically with guaranteed ordering:</p>
|
|
<pre><code class="lang-typescript">interface BatchOperation {
|
|
actions: Action[];
|
|
options?: {
|
|
stopOnError?: boolean; // Halt on first failure (default: true)
|
|
delayBetween?: number; // Milliseconds between actions (default: 50)
|
|
};
|
|
}
|
|
|
|
// Example: login form automation
|
|
const loginBatch: BatchOperation = {
|
|
actions: [
|
|
{ action: "click", x: 400, y: 300 },
|
|
{ action: "type", text: "[email protected]" },
|
|
{ action: "press", key: "Tab" },
|
|
{ action: "type", text: "password123" },
|
|
{ action: "press", key: "Enter" }
|
|
],
|
|
options: { delayBetween: 100 }
|
|
};</code></pre>
|
|
<h3>Platform Backends</h3>
|
|
<p>| Platform | Backend | Mouse Latency | Keyboard Latency | Screen Capture | |----------|---------|---------------|-------------------|----------------| | macOS | Quartz (native) | < 5ms | < 2ms | < 50ms | | Linux | X11 | < 5ms | < 2ms | < 50ms | | Linux | Wayland | < 10ms | < 5ms | < 100ms | | Windows | Win32 | < 5ms | < 2ms | < 50ms | | Docker | Virtual Display (Xvfb) | < 10ms | < 5ms | < 100ms |</p>
|
|
<h3>MCP Tool Interface</h3>
|
|
<p>Computer control is exposed as MCP tools (HIP-10) so any MCP-compatible agent can invoke computer use without custom integration.</p>
|
|
<pre><code class="lang-yaml">MCP Tool: computer
|
|
Description: Control local computer with native API acceleration
|
|
Actions:
|
|
# Mouse
|
|
- click(x, y)
|
|
- double_click(x, y)
|
|
- right_click(x, y)
|
|
- move(x, y)
|
|
- drag(start_x, start_y, end_x, end_y)
|
|
- scroll(amount, x, y)
|
|
|
|
# Keyboard
|
|
- type(text, interval)
|
|
- press(key)
|
|
- hotkey(keys[])
|
|
- key_down(key) / key_up(key)
|
|
|
|
# Screen
|
|
- screenshot()
|
|
- screenshot_region(region)
|
|
- get_screens()
|
|
- screen_size()
|
|
|
|
# Image location
|
|
- locate(image_path)
|
|
- locate_all(image_path)
|
|
- wait_for_image(image_path, timeout)
|
|
|
|
# Pixel
|
|
- pixel(x, y)
|
|
- pixel_matches(x, y, color, tolerance)
|
|
|
|
# Windows
|
|
- get_active_window()
|
|
- list_windows()
|
|
- focus_window(title)
|
|
|
|
# Regions
|
|
- define_region(name, x, y, w, h)
|
|
- region_screenshot(name)
|
|
- region_locate(name, image)
|
|
|
|
# Batch
|
|
- batch(actions[])
|
|
|
|
# Timing
|
|
- sleep(seconds)
|
|
- set_pause(seconds)
|
|
|
|
MCP Tool: screen
|
|
Description: Screen recording and AI interpretation
|
|
Actions:
|
|
- session(duration) # Record + analyze + compress + return
|
|
- capture() # Single optimized screenshot
|
|
- record(duration, fps) # Start background recording
|
|
- stop() # Stop and process recording
|
|
- analyze(path) # Process existing video file
|
|
- status() # Check recording state
|
|
- info() # System capabilities
|
|
|
|
MCP Tool: browser
|
|
Description: Full Playwright browser automation
|
|
Actions:
|
|
# Navigation: navigate, reload, go_back, go_forward, url, title
|
|
# Input: click, fill, type, press, select_option, check, upload
|
|
# Locators: locator, get_by_role, get_by_text, get_by_label
|
|
# State: is_visible, is_enabled, is_checked, get_text, get_attribute
|
|
# Assertions: expect_visible, expect_text, expect_value, expect_url
|
|
# Wait: wait, wait_for_load, wait_for_url, wait_for_selector
|
|
# Capture: screenshot, pdf, snapshot, evaluate
|
|
# Device: viewport, emulate, geolocation, permissions
|
|
# Network: route, unroute
|
|
# Multi-tab: new_page, new_context, tabs, close_tab
|
|
# Debug: trace_start, trace_stop, highlight, console, errors</code></pre>
|
|
<h3>Configuration</h3>
|
|
<p>Configuration via environment variables:</p>
|
|
<pre><code class="lang-bash"># Screen recording defaults
|
|
HANZO_SCREEN_DURATION=30 # Default session duration (seconds)
|
|
HANZO_SCREEN_TARGET_FRAMES=30 # Target keyframes per session
|
|
HANZO_SCREEN_MAX_SIZE=768 # Max frame dimension (pixels)
|
|
HANZO_SCREEN_QUALITY=60 # JPEG compression quality (1-100)
|
|
HANZO_SCREEN_ACTIVITY_THRESHOLD=0.02 # Activity detection sensitivity
|
|
|
|
# Browser automation
|
|
HANZO_BROWSER_HEADLESS=true # Run browser in headless mode
|
|
HANZO_BROWSER_TIMEOUT=30000 # Default action timeout (ms)
|
|
HANZO_BROWSER_SLOW_MO=0 # Slow down actions for debugging (ms)
|
|
|
|
# Platform
|
|
HANZO_OPERATIVE_BACKEND=auto # auto, quartz, x11, wayland, win32
|
|
HANZO_OPERATIVE_DISPLAY=:0 # X11 display (Linux)</code></pre>
|
|
<h2>Implementation</h2>
|
|
<h3>Architecture</h3>
|
|
<pre><code>┌─────────────────────────────────────────────────────────────────┐
|
|
│ Agent (HIP-9) │
|
|
│ (reasoning + planning) │
|
|
├─────────────────────────────────────────────────────────────────┤
|
|
│ MCP Protocol (HIP-10) │
|
|
│ (tool invocation + context management) │
|
|
├──────────────────┬──────────────────┬───────────────────────────┤
|
|
│ computer tool │ screen tool │ browser tool │
|
|
│ (input control) │ (recording) │ (Playwright) │
|
|
├──────────────────┴──────────────────┴───────────────────────────┤
|
|
│ Operative Core Library │
|
|
│ (platform abstraction + dispatch) │
|
|
├──────────────┬──────────────┬──────────────┬────────────────────┤
|
|
│ Quartz │ X11 │ Wayland │ Win32 │
|
|
│ (macOS) │ (Linux) │ (Linux) │ (Windows) │
|
|
├──────────────┴──────────────┴──────────────┴────────────────────┤
|
|
│ Operating System │
|
|
│ (display server + input subsystem) │
|
|
└─────────────────────────────────────────────────────────────────┘</code></pre>
|
|
<h3>Repository Structure</h3>
|
|
<pre><code>operative/
|
|
├── src/
|
|
│ ├── core/ # Platform abstraction layer
|
|
│ │ ├── backend.py # Backend interface
|
|
│ │ ├── quartz.py # macOS Quartz backend
|
|
│ │ ├── x11.py # Linux X11 backend
|
|
│ │ ├── wayland.py # Linux Wayland backend
|
|
│ │ └── win32.py # Windows Win32 backend
|
|
│ ├── screen/ # Screen capture and recording
|
|
│ │ ├── capture.py # Screenshot operations
|
|
│ │ ├── recorder.py # Screen recording
|
|
│ │ ├── activity.py # Activity detection
|
|
│ │ ├── keyframe.py # Keyframe extraction
|
|
│ │ └── compress.py # Frame compression
|
|
│ ├── input/ # Input control
|
|
│ │ ├── mouse.py # Mouse operations
|
|
│ │ ├── keyboard.py # Keyboard operations
|
|
│ │ ├── touch.py # Touch emulation
|
|
│ │ └── batch.py # Batch action execution
|
|
│ ├── browser/ # Playwright integration
|
|
│ │ ├── engine.py # Browser engine management
|
|
│ │ ├── page.py # Page interaction
|
|
│ │ ├── locators.py # Element location strategies
|
|
│ │ ├── network.py # Network interception
|
|
│ │ └── devices.py # Device emulation presets
|
|
│ ├── window/ # Window management
|
|
│ │ ├── manager.py # Window operations
|
|
│ │ └── regions.py # Named region management
|
|
│ ├── image/ # Image location
|
|
│ │ ├── locator.py # Template matching
|
|
│ │ └── pixel.py # Pixel operations
|
|
│ ├── mcp/ # MCP tool definitions
|
|
│ │ ├── computer_tool.py # computer() MCP tool
|
|
│ │ ├── screen_tool.py # screen() MCP tool
|
|
│ │ └── browser_tool.py # browser() MCP tool
|
|
│ └── security/ # Security and audit
|
|
│ ├── permissions.py # Permission system
|
|
│ ├── audit.py # Action audit trail
|
|
│ ├── sandbox.py # Execution sandbox
|
|
│ └── pii.py # PII detection and blurring
|
|
├── tests/
|
|
│ ├── test_capture.py
|
|
│ ├── test_input.py
|
|
│ ├── test_browser.py
|
|
│ ├── test_activity.py
|
|
│ ├── test_batch.py
|
|
│ └── test_security.py
|
|
├── pyproject.toml
|
|
├── Makefile
|
|
└── README.md</code></pre>
|
|
<h3>Platform Support Matrix</h3>
|
|
<p>| Feature | macOS (Quartz) | Linux (X11) | Linux (Wayland) | Windows (Win32) | Docker (Xvfb) | |---------|---------------|-------------|-----------------|-----------------|----------------| | Screenshot | Yes | Yes | Yes | Yes | Yes | | Screen recording | Yes | Yes | Yes | Yes | Yes | | Mouse control | Yes | Yes | Yes | Yes | Yes | | Keyboard control | Yes | Yes | Yes | Yes | Yes | | Touch emulation | Yes | No | Yes | Yes | No | | Window management | Yes | Yes | Partial | Yes | Yes | | Image location | Yes | Yes | Yes | Yes | Yes | | Browser (Playwright) | Yes | Yes | Yes | Yes | Yes | | HiDPI support | Yes | Yes | Yes | Yes | N/A |</p>
|
|
<h3>Agent SDK Integration</h3>
|
|
<p>Agents built with the Agent SDK (HIP-9) can invoke computer use as a first-class capability:</p>
|
|
<pre><code class="lang-python">from hanzoai.agent import Agent
|
|
from hanzoai.operative import ComputerTool, ScreenTool, BrowserTool
|
|
|
|
agent = Agent(
|
|
name="desktop-worker",
|
|
model="claude-sonnet-4-20250514",
|
|
tools=[
|
|
ComputerTool(), # Native mouse/keyboard/screen
|
|
ScreenTool(), # Screen recording + AI interpretation
|
|
BrowserTool(), # Playwright browser automation
|
|
],
|
|
instructions="""You can control this computer. Use the browser tool
|
|
for web tasks and the computer tool for native applications. Take
|
|
screenshots to verify your actions succeeded before moving on."""
|
|
)
|
|
|
|
result = await agent.run("Open the settings app and enable dark mode")</code></pre>
|
|
<h3>Deployment Modes</h3>
|
|
<h4>Local Development</h4>
|
|
<p>The agent runs directly on the developer machine. Screen capture and input control target the local display.</p>
|
|
<pre><code class="lang-bash">pip install hanzo-operative
|
|
# or
|
|
pip install hanzo-mcp # includes operative tools</code></pre>
|
|
<h4>Docker Container</h4>
|
|
<p>For CI/CD and headless environments, Operative runs in a Docker container with a virtual display:</p>
|
|
<pre><code class="lang-dockerfile">FROM hanzoai/operative:latest
|
|
|
|
# Virtual display is pre-configured
|
|
# Playwright browsers are pre-installed
|
|
# MCP tools are registered automatically
|
|
|
|
ENV DISPLAY=:99
|
|
ENV HANZO_OPERATIVE_BACKEND=x11</code></pre>
|
|
<pre><code class="lang-yaml"># compose.yml
|
|
services:
|
|
operative:
|
|
image: hanzoai/operative:latest
|
|
environment:
|
|
- DISPLAY=:99
|
|
ports:
|
|
- "6080:6080" # noVNC for visual debugging
|
|
volumes:
|
|
- /tmp/.X11-unix:/tmp/.X11-unix</code></pre>
|
|
<h4>Cloud Deployment</h4>
|
|
<p>For production agents running on Hanzo infrastructure, Operative instances are provisioned per-agent with isolated virtual displays:</p>
|
|
<pre><code>Agent Request → Hanzo Cloud → Provision VM with Operative →
|
|
→ Agent connects via MCP → Executes task → Returns result →
|
|
→ VM destroyed</code></pre>
|
|
<h2>Security</h2>
|
|
<h3>Permission System</h3>
|
|
<p>All computer use actions require explicit permission grants. Permissions are scoped to specific capabilities and can be restricted to specific applications or screen regions.</p>
|
|
<pre><code class="lang-yaml">permissions:
|
|
# Screen capture permissions
|
|
screen.capture:
|
|
allowed: true
|
|
restricted_regions: # Never capture these areas
|
|
- name: "password_field"
|
|
bounds: { x: 100, y: 200, w: 300, h: 50 }
|
|
|
|
# Input control permissions
|
|
input.mouse:
|
|
allowed: true
|
|
restricted_windows: # Cannot interact with these windows
|
|
- "1Password"
|
|
- "Keychain Access"
|
|
|
|
input.keyboard:
|
|
allowed: true
|
|
block_hotkeys: # Cannot execute these key combinations
|
|
- ["Command", "Q"] # Prevent quitting applications
|
|
- ["Command", "Shift", "Delete"] # Prevent emptying trash
|
|
|
|
# Browser permissions
|
|
browser.navigate:
|
|
allowed: true
|
|
blocked_domains:
|
|
- "*.bank.com"
|
|
- "mail.google.com"
|
|
|
|
browser.network:
|
|
allowed: false # Cannot intercept network traffic</code></pre>
|
|
<h3>Sandboxed Execution</h3>
|
|
<p>Computer use actions run in a restricted context:</p>
|
|
<ol><li><strong>Process isolation</strong>: Operative runs as a separate process with minimal OS privileges</li><li><strong>Application allowlisting</strong>: Only designated applications can be targeted</li><li><strong>Input rate limiting</strong>: Maximum actions per second to prevent runaway automation</li><li><strong>Timeout enforcement</strong>: All operations have hard timeouts to prevent hangs</li><li><strong>Resource limits</strong>: CPU, memory, and disk usage are capped</li></ol>
|
|
<h3>Audit Trail</h3>
|
|
<p>Every action is logged with full context for forensic analysis:</p>
|
|
<pre><code class="lang-json">{
|
|
"id": "act_01HQ3X7...",
|
|
"timestamp": "2025-01-15T10:30:45.123Z",
|
|
"agent_id": "agent_desktop-worker",
|
|
"session_id": "sess_01HQ3X...",
|
|
"action": {
|
|
"type": "click",
|
|
"x": 450,
|
|
"y": 300,
|
|
"button": "left"
|
|
},
|
|
"context": {
|
|
"active_window": "Google Chrome - Settings",
|
|
"screen_resolution": "2560x1440",
|
|
"before_screenshot": "scr_01HQ3X7_before.jpg",
|
|
"after_screenshot": "scr_01HQ3X7_after.jpg"
|
|
},
|
|
"permission": {
|
|
"granted": true,
|
|
"policy": "default_allow_chrome"
|
|
},
|
|
"duration_ms": 3
|
|
}</code></pre>
|
|
<h3>PII Detection and Protection</h3>
|
|
<p>When screen captures contain sensitive information, the system detects and protects it:</p>
|
|
<ol><li><strong>Detection</strong>: On-device ML model identifies common PII patterns (credit card numbers, SSNs, email addresses, passwords) in captured frames</li><li><strong>Blurring</strong>: Detected PII regions are blurred before frames are sent to the LLM</li><li><strong>Redaction in logs</strong>: Audit trail screenshots have PII regions redacted</li><li><strong>Opt-out</strong>: Specific screen regions can be marked as "never capture" in the permission configuration</li></ol>
|
|
<pre><code class="lang-python">class PIIDetector:
|
|
"""
|
|
Detects and protects personally identifiable information
|
|
in screen captures before they are sent to the LLM.
|
|
"""
|
|
|
|
PATTERNS = [
|
|
"credit_card", # 16-digit card numbers
|
|
"ssn", # Social security numbers
|
|
"email", # Email addresses in input fields
|
|
"password_field", # Active password input fields
|
|
"api_key", # API keys and tokens
|
|
]
|
|
|
|
async def protect(self, frame: CaptureResult) -> CaptureResult:
|
|
"""Detect and blur PII regions in a captured frame."""
|
|
detections = await self.detect(frame)
|
|
if detections:
|
|
frame = self.blur_regions(frame, detections)
|
|
return frame</code></pre>
|
|
<h3>Network Security</h3>
|
|
<p>When Operative is deployed remotely (Docker, cloud), the connection between agent and Operative instance is secured:</p>
|
|
<ul><li><strong>mTLS</strong>: Mutual TLS authentication between agent and Operative</li><li><strong>Encrypted transport</strong>: All screen data and control commands encrypted in transit</li><li><strong>Session tokens</strong>: Short-lived tokens for each computer use session</li><li><strong>IP allowlisting</strong>: Operative instances only accept connections from authorized agents</li></ul>
|
|
<h2>Compatibility</h2>
|
|
<h3>Relationship to Other HIPs</h3>
|
|
<p>| HIP | Relationship | |-----|-------------| | HIP-9 (Agent SDK) | Agents invoke computer use through the Agent SDK's tool interface | | HIP-10 (MCP) | Computer use exposed as MCP tools (<code>computer</code>, <code>screen</code>, <code>browser</code>) | | HIP-4 (LLM Gateway) | Visual reasoning requests routed through the LLM Gateway | | HIP-1211 (Chat Interface) | Chat UI can display computer use sessions inline | | HIP-14 (App Deployment) | Operative containers deployed via the application platform |</p>
|
|
<h2>Test Plan</h2>
|
|
<h3>Unit Tests</h3>
|
|
<pre><code class="lang-bash"># Run full test suite
|
|
pytest tests/ -v
|
|
|
|
# Platform-specific tests
|
|
pytest tests/test_capture.py -v -k "quartz" # macOS
|
|
pytest tests/test_capture.py -v -k "x11" # Linux</code></pre>
|
|
<h3>Integration Tests</h3>
|
|
<pre><code class="lang-python">async def test_observe_act_loop():
|
|
"""Test the full observe-act cycle."""
|
|
operative = OperativeClient()
|
|
|
|
# Take screenshot
|
|
screen = await operative.screenshot()
|
|
assert screen.width > 0
|
|
assert screen.height > 0
|
|
assert len(screen.image) > 0
|
|
|
|
# Click at a position
|
|
await operative.click(100, 100)
|
|
|
|
# Verify screen changed
|
|
screen_after = await operative.screenshot()
|
|
assert screen_after.timestamp > screen.timestamp
|
|
|
|
|
|
async def test_screen_recording_session():
|
|
"""Test screen recording with activity detection."""
|
|
operative = OperativeClient()
|
|
|
|
result = await operative.session(duration=5)
|
|
|
|
assert len(result.frames) > 0
|
|
assert len(result.frames) <= 30
|
|
assert result.duration >= 4.5
|
|
assert result.totalSize < 1_000_000 # Under 1MB
|
|
|
|
|
|
async def test_browser_navigation():
|
|
"""Test Playwright browser automation."""
|
|
browser = BrowserClient()
|
|
|
|
await browser.navigate("https://example.com")
|
|
|
|
title = await browser.title()
|
|
assert "Example" in title
|
|
|
|
text = await browser.getText("h1")
|
|
assert "Example Domain" in text</code></pre>
|
|
<h3>Performance Benchmarks</h3>
|
|
<p>| Operation | Target Latency | Platform | |-----------|---------------|----------| | Screenshot | < 50ms | All | | Mouse click | < 5ms | Native | | Keyboard type (per char) | < 2ms | Native | | Screen recording start | < 100ms | All | | Session processing (30s) | < 5s | All | | Browser navigate | < 3s | All | | Image locate | < 200ms | All |</p>
|
|
<h2>References</h2>
|
|
<ol><li><a href="./hip-0009-agent-sdk-multi-agent-orchestration-framework.md">HIP-9: Agent SDK</a></li><li><a href="./hip-0010-model-context-protocol-mcp-integration-standards.md">HIP-10: MCP Integration Standards</a></li><li><a href="./hip-0004-llm-gateway-unified-ai-provider-interface.md">HIP-4: LLM Gateway</a></li><li><a href="https://github.com/hanzoai/operative">Operative Repository</a></li><li><a href="https://playwright.dev/">Playwright Documentation</a></li><li><a href="https://modelcontextprotocol.io/">Model Context Protocol Specification</a></li><li><a href="https://developer.apple.com/documentation/coregraphics/quartz_event_services">Apple Quartz Event Services</a></li><li><a href="https://www.x.org/releases/current/doc/">X11 Protocol Reference</a></li></ol>
|
|
<h2>Copyright</h2>
|
|
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0015.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |