Files

1089 lines
43 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import * as Schema from "effect/Schema";
export const DeviceExpect = Schema.Struct({
briefs: Schema.optional(Schema.Number),
recording: Schema.optional(Schema.String),
artifact: Schema.optional(Schema.String),
noRawUi: Schema.optional(Schema.Boolean),
capture: Schema.optional(
Schema.Struct({
hostId: Schema.String,
deviceId: Schema.String,
app: Schema.optional(Schema.String),
}),
),
noShutdown: Schema.optional(
Schema.Array(Schema.Struct({ hostId: Schema.String, deviceId: Schema.String })),
),
report: Schema.optional(Schema.Array(Schema.String)),
reportOrCapture: Schema.optional(Schema.Boolean),
maxFailedAttemptsPerTurn: Schema.optional(Schema.Number),
maxCaptureAttempts: Schema.optional(Schema.Number),
});
type DeviceRun = {
readonly launchers?: ReadonlyArray<string>;
readonly text?: string;
readonly finalText?: string;
readonly stopReason?: string;
readonly transcript?: ReadonlyArray<unknown>;
readonly calls: ReadonlyArray<{
readonly step: number;
readonly tool: string;
readonly input: unknown;
readonly turn?: number;
readonly failed?: boolean;
readonly output?: string;
}>;
};
const record = (value: unknown): value is Readonly<Record<string, unknown>> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const inputString = (input: unknown, key: string) =>
record(input) && typeof input[key] === "string" ? input[key] : "";
const resultTarget = (call: DeviceRun["calls"][number]) => {
try {
const device: unknown = JSON.parse(call.output ?? "").device;
if (record(device) && typeof device.hostId === "string" && typeof device.id === "string")
return { hostId: device.hostId, deviceId: device.id };
} catch {}
};
const inputTarget = (call: DeviceRun["calls"][number]) => ({
hostId: inputString(call.input, "hostId") || "local",
deviceId: inputString(call.input, "deviceId"),
});
const openTargets = (previous: DeviceRun["calls"]) => {
const sessions = new Map<string, ReturnType<typeof inputTarget>>();
for (const prior of previous) {
if (prior.failed !== false) continue;
if (prior.tool === "device_open") {
const target = resultTarget(prior) ?? inputTarget(prior);
const key = `${target.hostId}/${target.deviceId}`;
sessions.delete(key);
sessions.set(key, target);
} else if (prior.tool === "device_close") {
const host = inputString(prior.input, "hostId"),
id = inputString(prior.input, "deviceId");
for (const [key, target] of sessions)
if ((!host || target.hostId === host) && (!id || target.deviceId === id))
sessions.delete(key);
}
}
return [...sessions.values()];
};
const flagValue = (words: ReadonlyArray<string>, flag: string) => {
for (const [i, word] of words.entries()) {
if (i === 0) continue;
if (word === flag) return words[i + 1];
if (word.startsWith(`${flag}=`)) return word.slice(flag.length + 1);
}
};
/** Each successful device_open's launcher, with the device and session it targets. */
const launcherOpens = (previous: DeviceRun["calls"]) =>
previous.flatMap((call) => {
if (call.tool !== "device_open" || call.failed !== false) return [];
try {
const { device, agentDevice }: Record<string, unknown> = JSON.parse(call.output ?? "");
if (
!record(device) ||
!record(agentDevice) ||
typeof device.hostId !== "string" ||
typeof device.id !== "string" ||
typeof agentDevice.command !== "string"
)
return [];
const targetArgs = Array.isArray(agentDevice.targetArgs)
? agentDevice.targetArgs.filter((arg): arg is string => typeof arg === "string")
: [];
const session =
typeof agentDevice.session === "string"
? agentDevice.session
: flagValue(["", ...targetArgs], "--session");
return [
{ command: agentDevice.command, hostId: device.hostId, deviceId: device.id, session },
];
} catch {
return [];
}
});
/**
* The device a launcher command drives: the one its --udid, --serial or --session names among
* the devices its launcher was returned for, else its launcher's only device. Without launcher
* results, only the sole open device.
*/
const launcherTarget = (words: ReadonlyArray<string>, previous: DeviceRun["calls"]) => {
const id = flagValue(words, "--udid") ?? flagValue(words, "--serial");
const session = flagValue(words, "--session");
const launched = launcherOpens(previous).filter((open) => open.command === words[0]);
const candidates = (
launched.length > 0
? launched
: openTargets(previous).map((t) => ({ ...t, session: undefined }))
).filter(
(t) =>
(id === undefined || t.deviceId === id) &&
(session === undefined || t.session === undefined || t.session === session),
);
return new Set(candidates.map((t) => `${t.hostId}/${t.deviceId}`)).size === 1
? candidates.at(-1)
: undefined;
};
const screenshotTarget = (call: DeviceRun["calls"][number], previous: DeviceRun["calls"]) => {
const result = resultTarget(call);
if (result) return result;
if (call.failed && /open in this thread/.test(call.output ?? "")) return undefined;
const host = inputString(call.input, "hostId"),
id = inputString(call.input, "deviceId");
if (host && id) return inputTarget(call);
const matches = openTargets(previous).filter(
(t) => (!host || t.hostId === host) && (!id || t.deviceId === id),
);
return id ? matches.at(-1) : matches.length === 1 ? matches[0] : undefined;
};
const managed =
/device_(?:open|screenshot|action)\b|device (?:panel|manager|tools)|returned\s+(?:(?:bound|pinned|agent-device|cli)\s+)*(?:launcher|driver|command)/;
const ui =
/\b(?:ui|screens?|screenshots?|screen captures?|record(?:ing|video)?s?|settings|taps?|typ(?:e|ing)|launch(?:es|ing)?|relaunch(?:es|ing)?|deep links?|swipes?|simulator (?:control|actions)|device actions|(?:drive|control|inspect)\s+(?:the\s+)?(?:simulator|device|app))\b/;
const rawCommand =
/\bsimctl\s+(?:launch|terminate|openurl|ui|privacy|location|push)\b|\bsimctl\s+io\s+(?:[^\s;]+\s+)?(?:screenshot|recordvideo)\b|\bidb\b[^.;]*\b(?:ui|screenshot|launch|video|record)\b|\badb\b[^.;]*\b(?:input|screencap|screenrecord)\b|\blab-(?:shot|ui|launch|capture(?:-set)?)\b|\bmac_agent_device\b/;
export function briefRoute(text: string) {
const positive = text
.toLowerCase()
.replaceAll("’", "'")
.replace(/[`*]/g, "")
// "SSH for the non-UI operations" restricts SSH rather than routing UI through it.
.replace(/\bnon-(?:ui|device|simulator|screen)\b/g, "auxiliary")
// A trailing prohibition negates its subject, not an earlier independent predicate.
.replace(
/(^|[.;\n]\s*)((?:-\s*)?(?:use|capture|take|record|run|call|drive)\b[^.;,\n]*?)\s+and\s+(?=[^.;,\n]*?\b(?:is|are)\s+(?:strictly\s+)?(?:forbidden|prohibited|disallowed|banned|not allowed|not permitted)\b)/g,
"$1$2. ",
)
.replace(
/\b(?!(?:because|since|as|if|unless|when|so|but)\b)(?:(?!\b(?:because|since|as|if|unless|when|so|but)\b)[^.;,\n])*?\b(?:is|are)\s+(?:strictly\s+)?(?:forbidden|prohibited|disallowed|banned|not allowed|not permitted)\b/g,
"",
)
.replace(/\b(?:do not|don't)\s+forget\s+to\b/g, "")
.replace(/\b(?:do not|don't)\s+avoid\b/g, "use")
.replace(/\b(?:should|must)\s+not\s+be\s+avoided\b/g, "is allowed")
.replace(/\b(?:there is )?no need to avoid\b/g, "use")
.replace(/\bnot just\b/g, "")
.replace(/\bbut (?:also )?for\b/g, "and for")
.replace(/\bavoid limiting (ssh|simctl|adb|idb) to [^.;]+;\s*use it\b/g, "use $1")
.replace(/(^|[.!;\n]\s*)(for [^.;,\n]+),\s*(?=(?:use|run)\b)/g, "$1$2 ")
.split(
/[\n.!;]+(?=\s|$)|\b(?:but|however|while)\b|,\s*(?=(?:screenshots?|recordings?)\s+(?:via|through|go))|(?:,\s*|\band\s+|\bthen\s+)(?=(?:use|run|call)\b)/,
)
.map((clause) =>
(
clause.split(
/\b(?:not|never|no|don't|cannot|can't|avoid|forbidden|prohibited|disallowed|rather than|instead of|without)\b/,
)[0] ?? ""
)
.replace(
/\b(?:ui|screen|settings|simulator|device|app)\s+(logs?|builds?|test runners?)\b/g,
"$1",
)
.replace(/\b(?:record|capture|stream)\s+(?:the\s+)?logs\b/g, "logs")
.replace(/\b(?:logs? from|reading)\s+screenshot\s+(?:tests?|logs?)\b/g, "logs")
.replace(
/\b(?:copy(?:ing)?|fetch(?:ing)?|transfer(?:ring)?|download(?:ing)?|sync(?:ing)?)\s+(?:(?:the|already|saved)\s+)*(?:screenshots?|recordings?|artifacts?)\b/g,
"artifact transfer",
),
);
const hasManaged = positive.some((clause) => managed.test(clause));
const hasRaw = positive.some((clause) => {
if (rawCommand.test(clause)) return true;
const routes = clause.matchAll(/\b(?:ssh|simctl|idb|adb|lab scripts?)\b/g);
for (const route of routes) {
const before = clause.slice(0, route.index);
const after = clause.slice(route.index + route[0].length).split(managed)[0] ?? "";
// Copying already-saved artifacts is auxiliary; keep later capture verbs visible.
const copies =
/^\s+(?:to\s+)?(?:copy|fetch|transfer|download|sync)\b/.test(after) ||
/\b(?:copy|fetch|transfer|download|sync)\s+(?:(?:the|saved)\s+)*(?:screenshots?|recordings?|artifacts?)\s+(?:over|via|through|using)\s*$/.test(
before,
);
if (copies && !/\b(?:take|capture|inspect|record|drive|control)\b/.test(after)) continue;
const purpose =
/^\s+(?:(?:is|are|can|may|must|should|be|used|only|allowed|permitted|fine)\s+)*(?:for|to|into)\b/.test(
after,
);
const hostPurpose =
/^\s+(?:-\S+\s+\S+\s+)*\S+\s+(?:for|and|then)\b|^\s+(?:handles|and|then)\b/.test(after);
if ((purpose || hostPurpose || new RegExp(`^\\s+${ui.source}`).test(after)) && ui.test(after))
return true;
const via = new RegExp(
`${ui.source}(?:\\s+(?:may|can|must|should|be|is|are|taken|captured|saved|recorded|only|directly))*\\s+(?:via|by|over|through|using|with|use|run)\\s+(?:raw\\s+)?$`,
);
if (
via.test(before) ||
/\b(?:screenshots?|screen(?: capture)?|recordings?)(?:\s+on\s+\S+)?\s+(?:over|via|using|through|with)\s*$/.test(
before,
) ||
/\bfor\s+[^.;]*\b(?:screenshots?|recordings?|screen capture)\s*,?\s*$/.test(before)
)
return true;
}
return false;
});
return hasManaged && hasRaw ? "mixed" : hasManaged ? "managed" : hasRaw ? "raw" : "neutral";
}
const executable = (word = "") => word.split("/").at(-1) ?? "";
const uiCommand =
/^(?:simctl\s+(?:launch|terminate|openurl|ui|privacy|location|push)\b|simctl\s+io\s+\S+\s+(?:screenshot|recordvideo)\b|idb\b.*\b(?:ui|screenshot|launch|video|record)\b|adb\b.*\b(?:shell\s+(?:input|am\s+start|settings)|(?:shell|exec-out)\s+(?:screencap|screenrecord))\b|screencapture\b|lab-(?:shot|ui|launch|capture(?:-set)?)\b)/i;
const deviceArgs = (words: ReadonlyArray<string>) => {
const args: string[] = [];
for (let i = 1; i < words.length; i++) {
const word = words[i]!;
if (word === "--") return [...args, ...words.slice(i + 1)];
if (!word.startsWith("-")) args.push(word);
else if (
/^--(?:platform|udid|serial|device|target|level|config|remote-config|session|state-dir|scope|timeout|timeout-ms|wait|metro-port|out|quality)$/.test(
word,
)
)
i++;
}
return args;
};
const auxiliaryAgent =
/^(?:(?:devices|apps|doctor|capabilities|install|uninstall|reinstall|install-from-source|logs|help|version)(?: |$)|session list(?: |$)|record contact-sheet(?: |$))/;
const deviceOperation = (words: ReadonlyArray<string>) => {
const args = deviceArgs(words);
return !!args[0] && !auxiliaryAgent.test(args.join(" "));
};
const interpreter = /^(?:python\d*(?:\.\d+)?|node|ruby|perl|osascript)$/;
const shells = new Set(["bash", "sh", "zsh"]);
/** Human and JSON agent-device failure envelopes. */
const AGENT_DEVICE_ERROR = /\bError \([A-Z_]+\):|"success"\s*:\s*false/;
export type ShellInvocation = {
readonly words: ReadonlyArray<string>;
readonly remote: boolean;
readonly raw: boolean;
readonly managed: boolean;
/** Raw-route checks inspect possible actions; managed credit requires established execution. */
readonly executed: boolean;
/** An aggregate success cannot establish success of an action masked by ||. */
readonly unmasked: boolean;
/** A later `;` or `&` command owns the call's exit status. */
readonly sequenced: boolean;
};
// A deliberately bounded shell grammar, not a shell executor. Keep quotes until expansion so
// literal command examples, comments and quoted heredocs never become invocations.
const tokens = (source: string) =>
source
.replace(/'[^']*'|\\[\s\S]/g, (part) => (part === "\\\n" ? "" : part))
.match(
/`(?:\\.|[^`])*`|[<>]\((?:[^()]|\([^()]*\))*\)|\$\((?:[^()'"\\]|\\.|'[^']*'|"(?:\\.|[^"\\])*"|\([^()]*\))*\)|\d+(?:>>?|<)(?:&\d+)?|(?:"(?:\\.|[^"\\])*"|'[^']*'|\\.|\$\{\w+\}|[^\s;&|(){}<>"'\\])+|&&|\|\||[;\n&|(){}]|\d*(?:>>?|<)(?:&\d+)?/g,
) ?? [];
const unquote = (word: string, variables: Map<string, string>) =>
word.replace(
/'([^']*)'|"((?:\\.|[^"\\])*)"|\\(.)|([^'"\\]+)/g,
(
_part,
single: string | undefined,
double: string | undefined,
escaped: string | undefined,
plain: string | undefined,
) => {
if (single !== undefined) return single;
if (escaped !== undefined) return escaped;
return (double ?? plain ?? "").replace(
/\\(["\\$])|\$(?:\{(\w+)\}|(\w+))/g,
(ref, escaped: string | undefined, braced: string | undefined, bare: string | undefined) =>
escaped ?? variables.get(braced ?? bare ?? "") ?? ref,
);
},
);
function unwrap(words: string[]) {
while (words.length) {
if (/^\w+=/.test(words[0]!)) {
words.shift();
continue;
}
const name = executable(words[0]);
if (!/^(?:env|sudo|command|nohup|exec|time|caffeinate|timeout|nice)$/.test(name)) break;
words.shift();
const takesValue: Record<string, RegExp> = {
env: /^(?:-u|--unset|-C|--chdir)$/,
sudo: /^(?:-[ughpCT]|--user|--group|--host|--prompt)$/,
caffeinate: /^-[tw]$/,
timeout: /^(?:-[sk]|--signal|--kill-after)$/,
nice: /^(?:-n|--adjustment)$/,
};
while (words[0]?.startsWith("-")) {
const option = words.shift()!;
if (option === "--") break;
if (takesValue[name]?.test(option)) words.shift();
}
if (name === "timeout") words.shift();
}
return words;
}
/** Literal process-launch APIs only. Definitions and strings are not executed programs. */
function scriptInvocations(
script: string,
remote: boolean,
variables: Map<string, string>,
language: string,
): ShellInvocation[] {
const actions: ShellInvocation[] = [];
const python = language.startsWith("python");
// A function body is source, not an invocation. Calls to computed/script functions are
// outside this grammar; do not grant managed credit for them.
let definitionIndent: number | undefined;
if (python)
script = script
.split("\n")
.map((line) => {
const indent = line.match(/^\s*/)?.[0].length ?? 0;
if (line.trim() && definitionIndent !== undefined && indent <= definitionIndent)
definitionIndent = undefined;
if (/^\s*(?:async\s+)?def\s/.test(line)) definitionIndent = indent;
return definitionIndent === undefined ? line : "";
})
.join("\n");
const launch =
/^(subprocess\.(?:run|Popen|call|check_call|check_output)|os\.(?:system|popen)|(?:child_process\.)?(?:execFileSync|execFile|execSync|exec|spawnSync|spawn))\s*\(/;
const literal = "(?:'[^']*'|\"(?:\\\\.|[^\"\\\\])*\")";
const quoteArg = (arg: string) => `'${unquote(arg, variables).replaceAll("'", "'\\''")}'`;
const array = (source: string) => {
const match = new RegExp(`^\\[\\s*(${literal}(?:\\s*,\\s*${literal})*\\s*,?)?\\s*\\]`).exec(
source,
);
return match
? (match[1]?.match(new RegExp(literal, "g")) ?? []).map(quoteArg).join(" ")
: undefined;
};
for (let i = 0; i < script.length; i++) {
const char = script[i];
if (char === "#" || script.slice(i, i + 2) === "//") {
const end = script.indexOf("\n", i);
i = end < 0 ? script.length : end;
continue;
}
if (char === "'" || char === '"' || char === "`") {
const quote = python && script.slice(i, i + 3) === char.repeat(3) ? char.repeat(3) : char;
i += quote.length;
while (i < script.length && script.slice(i, i + quote.length) !== quote) {
if (script[i] === "\\") i++;
i++;
}
i += quote.length - 1;
continue;
}
const match = launch.exec(script.slice(i));
if (!match) continue;
const rest = script.slice(i + match[0].length).trimStart();
let program: string | undefined;
if (rest.startsWith("[")) program = array(rest);
else {
const arg = rest.match(new RegExp(`^${literal}`))?.[0];
const tail = arg === undefined ? "" : rest.slice(arg.length);
if (arg && /(?:spawn|execFile)/.test(match[1]!)) {
const argv = array(tail.replace(/^\s*,\s*/, ""));
if (argv !== undefined) program = `${quoteArg(arg)} ${argv}`;
} else if (
arg &&
(!python || match[1]!.startsWith("os.") || /^\s*,[^)]*\bshell\s*=\s*True\b/.test(tail))
) {
program = unquote(arg, variables);
}
}
if (program !== undefined) {
const line = script.slice(script.lastIndexOf("\n", i) + 1, i);
const boundary = Math.max(
script.lastIndexOf(";", i),
python ? script.lastIndexOf("\n", i) : -1,
);
const statement = script.slice(boundary + 1, i);
const certain =
!/=>|\bfunction\b|\blambda\b/.test(statement) &&
(python ? !/^\s/.test(line) : !script.slice(0, i).includes("{"));
actions.push(
...shellInvocations(program, new Map(variables), remote).map((action) => ({
...action,
executed: action.executed && certain,
// A script's exit status needn't reflect its subprocesses (`subprocess.run` doesn't check).
sequenced: true,
})),
);
}
i += match[0].length - 1;
}
return actions;
}
/** Shared by raw-UI, recording and failure grading. Literal aliases may span shell calls. */
export function shellInvocations(
command: string,
variables = new Map<string, string>(),
remote = false,
succeeded = false,
/** Shared with function calls: the shell's functions, and whether it exited. */
shell: {
functions: Map<string, { text: string; certain: boolean }>;
exited: boolean;
depth: number;
} = {
functions: new Map(),
exited: false,
depth: 0,
},
): ShellInvocation[] {
if (shell.depth > 8) return [];
const bodies = new Map<string, { text: string; expand: boolean }>();
command = command.replace(
/^([^\n]*?)<<(-?)\s*(['"]?)(\w+)\3([^\n]*)\n([\s\S]*?)(?:\n[ \t]*\4[ \t]*(?=\n|(?![\s\S]))|(?![\s\S]))/gm,
(
_match,
line: string,
strip: string,
quote: string,
_end: string,
rest: string,
body: string,
) => {
const key = `__HEREDOC_${bodies.size}`;
bodies.set(key, { text: strip ? body.replace(/^\t+/gm, "") : body, expand: !quote });
return `${line} ${key} ${rest}`;
},
);
const actions: ShellInvocation[] = [];
let clause: string[] = [];
let pipe: string | undefined;
let gate = "";
let status: boolean | undefined;
// `before` says whether an earlier if/elif branch of this statement ran: true, false or unknown.
const conditions: Array<{
parent: boolean;
certain: boolean;
take: boolean | undefined;
before: boolean | undefined;
}> = [];
let active = true;
let certain = true;
let currentCertain = true;
let currentUnmasked = true;
let currentSequenced = false;
// After a certain `exit` or `return`, nothing later in its scope runs.
let exited = false;
const append = (nested: ShellInvocation[]) =>
actions.push(
...nested.map((action) => ({
...action,
executed: action.executed && currentCertain,
unmasked: action.unmasked && currentUnmasked,
sequenced: action.sequenced || currentSequenced,
})),
);
const truth = (words: string[]): boolean | undefined => {
if (words[0] === "!") {
const value = truth(words.slice(1));
return value === undefined ? undefined : !value;
}
return ["true", ":"].includes(words[0] ?? "") ? true : words[0] === "false" ? false : undefined;
};
const commandTokens = tokens(command.replace(/(^|\n)[ \t]*#[^\n]*/g, "$1"));
let tokenIndex = 0;
const functions = shell.functions;
const listTokens = () => {
let start = tokenIndex - 1,
end = tokenIndex;
while (start >= 0 && !/^[;\n&]$/.test(commandTokens[start]!)) start--;
while (end < commandTokens.length && !/^[;\n&]$/.test(commandTokens[end]!)) end++;
return commandTokens.slice(start + 1, end);
};
const substitutions = (source: string, heredoc = false) => {
const expanded = heredoc
? source
: source.replace(
/"((?:\\.|[^"\\])*)"|'[^']*'|\\./g,
(part, double: string | undefined) => double ?? (part.startsWith("'") ? "" : part),
);
for (const match of expanded.matchAll(/(?<!\\)\$\(((?:[^()]|\([^()]*\))*)\)/g))
append(shellInvocations(match[1]!, new Map(variables), remote));
for (const match of expanded.matchAll(/(?<!\\)`([^`]*)`|[<>]\(((?:[^()]|\([^()]*\))*)\)/g))
append(shellInvocations(match[1] ?? match[2]!, new Map(variables), remote));
};
const flush = () => {
if (!clause.length) return;
const rawWords = clause;
const redirects = redirectOperands;
redirectOperands = [];
clause = [];
let words = rawWords.map((word) => unquote(word, variables));
if (words[0] === "if") {
const parent = active;
conditions.push({
parent,
certain,
take: undefined,
before: false,
});
words.shift();
}
if (words[0] === "then") {
const condition = conditions.at(-1);
// The branch follows the condition list's final status (`if true && false`), not its
// first command's.
if (condition) condition.take = condition.before === true ? false : status;
active =
condition?.parent !== false && condition?.take !== false && condition?.before !== true;
certain =
condition?.certain !== false && condition?.take === true && condition?.before === false;
words.shift();
}
if (words[0] === "else") {
const condition = conditions.at(-1);
active = !!condition?.parent && condition.take !== true && condition.before !== true;
certain = !!condition?.certain && condition.take === false && condition.before === false;
words.shift();
}
if (words[0] === "fi") {
const condition = conditions.pop();
active = condition?.parent ?? true;
certain = condition?.certain ?? true;
words.shift();
}
if (words[0] === "elif") {
// An elif's condition runs only when no earlier branch did.
const condition = conditions.at(-1);
if (condition) {
condition.before =
condition.before === true || condition.take === true
? true
: condition.before === false && condition.take === false
? false
: undefined;
condition.take = condition.before === true ? false : truth(words.slice(1));
active = condition.parent && condition.before !== true;
certain = condition.certain && condition.before === false;
}
words.shift();
}
const loopList = /^(?:for|select)$/.test(words[0] ?? "");
if (/^(?:while|until)$/.test(words[0] ?? "") || loopList) {
const value = loopList
? words[2] === "in" && words.length > 3
? true
: undefined
: truth(words.slice(1));
conditions.push({
parent: active,
certain,
take: words[0] === "until" && value !== undefined ? !value : value,
before: false,
});
if (!loopList) words.shift();
}
if (words[0] === "do") {
const condition = conditions.at(-1);
active = condition?.parent !== false && condition?.take !== false;
certain = condition?.certain !== false && condition?.take === true;
words.shift();
}
if (words[0] === "done") {
const condition = conditions.pop();
active = condition?.parent ?? true;
certain = condition?.certain ?? true;
words.shift();
}
const reachable =
gate === "&&" ? status : gate === "||" ? (status === undefined ? undefined : !status) : true;
if (!active || exited || !words.length || reachable === false) return;
const list = listTokens();
currentUnmasked = !list.includes("||") && words[0] !== "!";
// The call's exit status is the last command's; an action followed by `;` or `&` doesn't own it.
// A command piped into another doesn't own the pipeline's status either.
currentSequenced =
commandTokens[tokenIndex] === "|" ||
commandTokens.slice(tokenIndex).some((t) => /^[;\n&]$/.test(t));
const finalAndSuccess =
succeeded &&
gate === "&&" &&
currentUnmasked &&
!commandTokens.some((t) => /^(?:if|for|while|until|then|else|fi|do|done)$/.test(t)) &&
!commandTokens.slice(tokenIndex).some((t) => /^[;\n&]$/.test(t));
currentCertain = certain && (reachable === true || finalAndSuccess);
// A list's status is known only when this command certainly ran, or when its own result
// decides the list either way (`x && false`, `x || true`).
const value = truth(words);
status =
reachable === true
? value
: gate === "&&" && value === false
? false
: gate === "||" && value === true
? true
: undefined;
for (const word of [...rawWords, ...redirects]) if (!bodies.has(word)) substitutions(word);
if (loopList) return;
const body = words.map((word) => bodies.get(word)).find((value) => value !== undefined);
if (body?.expand) substitutions(body.text, true);
words = words.filter((word) => !bodies.has(word));
const literalAssignments = words.every((word) => /^\w+=/.test(word));
if (literalAssignments || words[0] === "export") {
for (const word of words) {
const at = word.indexOf("=");
if (at > 0) {
if (currentCertain) variables.set(word.slice(0, at), word.slice(at + 1));
else variables.delete(word.slice(0, at));
}
}
return;
}
if (executable(words[0]) === "command" && /^-[vV]$/.test(words[1] ?? "")) return;
while (words[0] === "!") words.shift();
words = unwrap(words);
if (/^(?:exit|return)$/.test(executable(words[0]))) {
if (currentCertain) {
exited = true;
// `exit` in a function ends the shell; `return` ends only the function.
if (executable(words[0]) === "exit") shell.exited = true;
} else certain = false;
return;
}
const name = executable(words[0]);
let input = body?.text ?? pipe;
pipe = undefined;
if (name === "cat" || name === "tee") {
pipe = input;
return;
}
if (name === "printf" || name === "echo") {
pipe =
name === "echo"
? words.slice(1).join(" ")
: words[1]?.includes("%s")
? words.slice(2).join("\n")
: words[1]?.replaceAll("\\n", "\n");
return;
}
if (functions.has(name)) {
const definition = functions.get(name)!;
const call = { functions, exited: false, depth: shell.depth + 1 };
append(
shellInvocations(definition.text, new Map(variables), remote, succeeded, call).map(
(action) => ({ ...action, executed: action.executed && definition.certain }),
),
);
if (call.exited && definition.certain && currentCertain) exited = shell.exited = true;
return;
}
if (name === "ssh") {
let at = 1;
while (words[at]?.startsWith("-")) {
const option = words[at++]!;
if (option === "--") break;
if (/^-[bBcDEeFIiJLlmOoPpQRSWw]$/.test(option)) at++;
}
at++; // destination
if (words[at] === "--") at++;
const script = words.slice(at).join(" ");
// A remote shell/interpreter inherits redirected/piped stdin, never the local aliases.
const destination = unwrap(tokens(script).map((word) => unquote(word, new Map())));
const reader = executable(destination[0]);
// Shells take a program with -c (`bash -se` reads stdin); interpreters with -c or -e.
const programFlag = shells.has(reader) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/;
const hasProgram = destination.some((word) => programFlag.test(word));
if (
input !== undefined &&
(!reader || ((shells.has(reader) || interpreter.test(reader)) && !hasProgram))
) {
if (interpreter.test(reader)) append(scriptInvocations(input, true, new Map(), reader));
else append(shellInvocations(input, new Map(), true));
}
append(shellInvocations(script, new Map(), true));
return;
}
if (shells.has(name) || interpreter.test(name)) {
const at = words.findIndex((word) =>
(shells.has(name) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/).test(word),
);
if (at >= 0) input = words[at + 1];
if (input !== undefined)
append(
shells.has(name)
? shellInvocations(input, new Map(variables), remote, succeeded)
: scriptInvocations(input, remote, variables, name),
);
return;
}
const endOptions = words.indexOf("--");
if (
words
.slice(0, endOptions < 0 ? words.length : endOptions)
.some((word) => word === "--help" || word === "-h") ||
words[1] === "help"
)
return;
const normalized = name === "xcrun" ? words.slice(1) : [...words];
if (name === "xcrun")
while (normalized[0]?.startsWith("-")) {
const option = normalized.shift();
if (/^(?:--sdk|-sdk|--toolchain|-toolchain)$/.test(option ?? "")) normalized.shift();
}
const cli = executable(normalized[0]);
const agent = ["agent-device", "mac_agent_device"].includes(cli) && deviceOperation(normalized);
actions.push({
words,
remote,
executed: currentCertain,
unmasked: currentUnmasked,
sequenced: currentSequenced,
raw:
uiCommand.test([cli, ...normalized.slice(1)].join(" ")) ||
(agent && (remote || cli === "mac_agent_device")),
managed: agent && !remote && cli === "agent-device",
});
};
const scopes: Array<{
variables: Map<string, string>;
active: boolean;
certain: boolean;
exited: boolean;
subshell: boolean;
}> = [];
let comment = false;
let redirect = false;
let redirectOperands: string[] = [];
for (; tokenIndex < commandTokens.length; tokenIndex++) {
const token = commandTokens[tokenIndex]!;
// Comments come first, so a commented-out function definition is never registered.
if (token === "\n") comment = false;
if (comment) continue;
if (token.startsWith("#")) {
comment = true;
continue;
}
// Shell function definitions don't execute their bodies. Keep literal calls supported.
const nameAt = tokenIndex + (token === "function" ? 1 : 0);
const bodyAt =
commandTokens[nameAt + 1] === "(" && commandTokens[nameAt + 2] === ")"
? nameAt + 3
: token === "function"
? nameAt + 1
: -1;
if (commandTokens[bodyAt] === "{") {
flush();
let end = bodyAt + 1,
depth = 1;
for (; end < commandTokens.length && depth; end++) {
if (commandTokens[end] === "{") depth++;
if (commandTokens[end] === "}") depth--;
}
if (active)
functions.set(commandTokens[nameAt]!, {
text: commandTokens.slice(bodyAt + 1, end - 1).join(" "),
certain,
});
tokenIndex = end - 1;
continue;
}
if (/^\d*(?:>>?|<)(?:&\d+)?$/.test(token)) {
redirect = !token.includes("&");
continue;
}
if (redirect) {
redirect = false;
redirectOperands.push(token);
continue;
}
if (/^(?:&&|\|\||[;\n&|(){}])$/.test(token)) {
flush();
if (token === "(" || token === "{") {
scopes.push({ variables, active, certain, exited, subshell: token === "(" });
if (token === "(") variables = new Map(variables);
const reachable =
gate === "&&"
? status
: gate === "||"
? status === undefined
? undefined
: !status
: true;
active = active && reachable !== false;
certain = certain && reachable === true;
}
if (token === ")" || token === "}") {
const scope = scopes.pop();
// A subshell's exit ends only the subshell; a group's ends the shell.
if (scope) exited = scope.exited || (!scope.subshell && exited);
if (scope) ({ variables, active, certain } = scope);
}
if (token !== "|") pipe = undefined;
gate = token;
} else clause.push(token);
}
flush();
return actions;
}
export function gradeDevice(expect: typeof DeviceExpect.Type | undefined, run: DeviceRun) {
const checks: Array<{ check: string; pass: boolean }> = [];
if (!expect) return checks;
const add = (check: string, pass: boolean) => checks.push({ check, pass });
if (expect.briefs !== undefined) {
const briefs = run.calls.filter((call) =>
["delegate_task", "hanzo_thread_launch"].includes(call.tool),
);
add("device briefs count", briefs.length >= expect.briefs);
for (const [i, call] of briefs.entries()) {
const route = briefRoute(
inputString(call.input, call.tool === "delegate_task" ? "task" : "message"),
);
add(`device brief ${i + 1} ${call.tool} ${route}`, route === "managed");
}
}
if (expect.artifact !== undefined)
add(
`device final artifact ${expect.artifact}`,
run.stopReason === "end_turn" && (run.finalText ?? "").includes(expect.artifact),
);
const variables = new Map<string, string>();
const shell = run.calls
.filter((call) => ["Bash", "Monitor"].includes(call.tool))
.map(
(call) =>
[
call,
shellInvocations(
inputString(call.input, "command"),
variables,
false,
call.failed === false,
).map((action) => ({
...action,
args: deviceArgs(action.words),
bound: !action.remote && (run.launchers ?? []).includes(action.words[0] ?? ""),
})),
] as const,
);
const allActions = shell.flatMap(([, actions]) => actions);
const usableImage = (call: DeviceRun["calls"][number]) => {
if (call.failed !== false) return false;
try {
const image: unknown = JSON.parse(call.output ?? "").screenshot;
return (
record(image) &&
typeof image.width === "number" &&
image.width > 0 &&
typeof image.height === "number" &&
image.height > 0
);
} catch {
return false;
}
};
const successfulScreenshots = ([call, actions]: (typeof shell)[number]) => {
const captures = actions.filter((a) => a.managed && a.bound && a.args[0] === "screenshot");
if (
call.failed !== false ||
AGENT_DEVICE_ERROR.test(call.output ?? "") ||
captures.length !== 1 ||
/"(?:width|height)"\s*:\s*0\b/.test(call.output ?? "")
)
return [];
const paths = (call.output ?? "").split("\n").flatMap((line) => {
// The simulated shell words the success line several ways; each must still name the path.
const saved = line.match(
/^[^\w\n]*(?:Screenshot saved|Saved screenshot|Captured screenshot|Wrote screenshot)(?: to)?:?\s+(.+)$/i,
)?.[1];
if (saved) return [saved];
try {
const image: unknown = JSON.parse(line);
if (
record(image) &&
typeof image.path === "string" &&
typeof image.width === "number" &&
image.width > 0 &&
typeof image.height === "number" &&
image.height > 0
)
return [image.path];
} catch {}
return [];
});
return captures.filter(
(a) =>
a.unmasked &&
((paths.length === 0 && a.executed && !a.sequenced) ||
paths.some(
(path) =>
path === a.args[1] ||
(a.args[1]?.startsWith("$/") && path.endsWith(a.args[1].slice(1))),
)),
);
};
const recovered =
expect.reportOrCapture &&
(run.calls.some((call) => call.tool === "device_screenshot" && usableImage(call)) ||
shell.some((entry) => successfulScreenshots(entry).length > 0));
if (expect.noShutdown)
add(
"callsMustNot shutdown occupied device",
!run.calls.some(
(call, index) =>
call.tool === "device_close" &&
record(call.input) &&
call.input.shutdown === true &&
openTargets(run.calls.slice(0, index)).some(
(target) =>
expect.noShutdown!.some(
(protectedTarget) =>
target.hostId === protectedTarget.hostId &&
target.deviceId === protectedTarget.deviceId,
) &&
(!inputString(call.input, "hostId") ||
inputString(call.input, "hostId") === target.hostId) &&
(!inputString(call.input, "deviceId") ||
inputString(call.input, "deviceId") === target.deviceId),
),
),
);
for (const pattern of expect.report ?? [])
add(
`device final report /${pattern}/`,
run.stopReason === "end_turn" &&
(!!recovered || new RegExp(pattern).test(run.finalText ?? "")),
);
if (expect.noRawUi) add("shellMustNot device UI", !allActions.some((action) => action.raw));
if (expect.maxCaptureAttempts !== undefined)
add(
`maxCalls device captures <= ${expect.maxCaptureAttempts}`,
run.calls.filter((call) => call.tool === "device_screenshot").length +
allActions.filter((action) => action.managed && action.args[0] === "screenshot").length <=
expect.maxCaptureAttempts,
);
if (expect.capture) {
const target = expect.capture;
const matches = (value: ReturnType<typeof inputTarget> | undefined) =>
value?.hostId === target.hostId && value?.deviceId === target.deviceId;
const appSessions = new Set<string>();
let opened = false,
captured = false;
for (const [index, call] of run.calls.entries()) {
if (call.tool === "device_open" && call.failed === false && matches(inputTarget(call)))
opened = true;
const appReady = !target.app || (call.failed === false && appSessions.size > 0);
if (
opened &&
appReady &&
call.tool === "device_screenshot" &&
matches(screenshotTarget(call, run.calls.slice(0, index))) &&
(!target.app || usableImage(call))
)
captured = true;
const entry = shell.find(([source]) => source === call);
const actions = (entry?.[1] ?? []).filter((action) => action.managed && action.bound);
const screenshots = entry ? successfulScreenshots(entry) : [];
for (const action of actions) {
const { args, words } = action;
const onTarget = matches(launcherTarget(words, run.calls.slice(0, index)));
// Only an open or close that ran and succeeded changes the app session.
const ran =
call.failed === false &&
action.executed &&
action.unmasked &&
!AGENT_DEVICE_ERROR.test(call.output ?? "");
if (target.app && ["open", "close"].includes(args[0] ?? "") && ran) {
appSessions.delete(words[0]!);
if (
args[0] === "open" &&
args[1] === target.app &&
onTarget &&
(!action.sequenced ||
(actions.filter((a) => a.args[0] === "open").length === 1 &&
/^Opened:/m.test(call.output ?? "")))
)
appSessions.add(words[0]!);
}
if (
opened &&
onTarget &&
(target.app
? appSessions.has(words[0]!) && screenshots.includes(action)
: action.executed &&
(["screenshot", "snapshot"].includes(args[0] ?? "") ||
(args[0] === "record" && args[1] === "start")))
)
captured = true;
}
}
add("device successful open and capture attempt", captured);
}
if (expect.recording !== undefined) {
const steps = [...shell].flatMap(([call, actions]) => {
const operations = actions.filter(
({ args }) => args[0] === "record" && /^(?:start|stop)$/.test(args[1] ?? ""),
);
return operations
.filter(
(action) =>
action.executed &&
action.unmasked &&
action.bound &&
((!action.sequenced &&
call.failed === false &&
!AGENT_DEVICE_ERROR.test(call.output ?? "")) ||
(operations.filter((o) => o.args[1] === action.args[1]).length === 1 &&
new RegExp(
`(?:^|\\n)[^\\w\\n]*Recording ${action.args[1] === "start" ? "started" : "stopped"}\\b|"kind"\\s*:\\s*"recording${action.args[1] === "start" ? "Started" : "Stopped"}"`,
"i",
).test(call.output ?? ""))),
)
.map(({ args }) => args[1]!);
});
const start = steps.indexOf("start");
checks.push({
check: `device recording ${expect.recording}`,
pass:
start >= 0 &&
steps.indexOf("stop", start + 1) >= 0 &&
run.stopReason === "end_turn" &&
(run.finalText ?? "").includes(expect.recording),
});
}
if (expect.maxFailedAttemptsPerTurn !== undefined) {
const failedByTurn = new Map<number, number>();
for (const call of run.calls) {
const actions = shell.find(([source]) => source === call)?.[1] ?? [];
// A failed raw attempt, such as agent-device over ssh, is still a failed device attempt.
const device = actions.filter((action) => action.managed || action.raw).length;
// agent-device prints one error per failed command, even when a later command succeeds.
const errors = (call.output ?? "").match(new RegExp(AGENT_DEVICE_ERROR, "g"))?.length ?? 0;
let discoveryFailed = false;
if (call.tool === "device_list") {
try {
const statuses: unknown = JSON.parse(call.output ?? "").hostStatuses;
discoveryFailed =
record(statuses) &&
Object.values(statuses).some((s) => record(s) && s.status === "failed");
} catch {}
}
const attempts =
call.tool === "device_list"
? Number(!!call.failed || discoveryFailed)
: /^device_(?:open|screenshot|close)$/.test(call.tool)
? Number(!!call.failed)
: call.failed && device
? Math.max(1, Math.min(errors, device))
: Math.min(errors, device);
const turn = call.turn ?? 0;
failedByTurn.set(turn, (failedByTurn.get(turn) ?? 0) + attempts);
}
const max = expect.maxFailedAttemptsPerTurn;
add(
`maxCalls device failures per user turn <= ${max}`,
[...failedByTurn.values()].every((count) => count <= max),
);
}
return checks;
}