mirror of
https://github.com/hanzoai/dev-desktop.git
synced 2026-11-22 00:14:30 +00:00
338 lines
14 KiB
JavaScript
338 lines
14 KiB
JavaScript
#!/usr/bin/env node
|
|
// An eval script, not server code: it shells out to git and reads files directly.
|
|
// @effect-diagnostics nodeBuiltinImport:off
|
|
import * as NodeChildProcess from "node:child_process";
|
|
import * as NodeCrypto from "node:crypto";
|
|
import * as NodeFS from "node:fs";
|
|
import * as NodeOS from "node:os";
|
|
import * as NodePath from "node:path";
|
|
import * as NodeUtil from "node:util";
|
|
import * as Effect from "effect/Effect";
|
|
import * as Schedule from "effect/Schedule";
|
|
import * as Schema from "effect/Schema";
|
|
import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http";
|
|
import { proxyConfig } from "../shared/proxy.ts";
|
|
import { cases, type Case } from "./cases.ts";
|
|
import { grade, testRange, type Comment } from "./grade.ts";
|
|
|
|
const { values } = NodeUtil.parseArgs({
|
|
options: {
|
|
out: { type: "string" },
|
|
model: { type: "string", multiple: true },
|
|
prompt: { type: "string" },
|
|
case: { type: "string", multiple: true },
|
|
samples: { type: "string", default: "2" },
|
|
concurrency: { type: "string", default: "6" },
|
|
reasoning: { type: "string", default: "low" },
|
|
},
|
|
});
|
|
if (!values.out) throw new Error("Use --out <directory> [--model <id>] [--prompt <file>].");
|
|
const outDirectory = values.out;
|
|
const models = values.model ?? ["gpt-6.1-terra", "gpt-6-luna"];
|
|
const repo = NodePath.resolve(import.meta.dirname, "../..");
|
|
const checkFile =
|
|
values.prompt ?? NodePath.join(repo, ".macroscope/check-run-agents/test-signal.md");
|
|
// Macroscope gives the model the body; the front matter only configures the run.
|
|
const instructions = NodeFS.readFileSync(checkFile, "utf8")
|
|
.replace(/^---\n[\s\S]*?\n---\n/, "")
|
|
.trim();
|
|
const words = instructions.split(/\s+/).length;
|
|
const promptHash = NodeCrypto.createHash("sha256").update(instructions).digest("hex").slice(0, 12);
|
|
const selected = values.case ? cases.filter((entry) => values.case!.includes(entry.id)) : cases;
|
|
const proxy = proxyConfig();
|
|
|
|
const git = (args: ReadonlyArray<string>) => {
|
|
const result = NodeChildProcess.spawnSync("git", args, {
|
|
cwd: repo,
|
|
encoding: "utf8",
|
|
maxBuffer: 1 << 26,
|
|
});
|
|
return result.status === 0 ? result.stdout : undefined;
|
|
};
|
|
|
|
/** The case's PR head as files the model can read, plus the diff it reviews. */
|
|
const world = (entry: Case) => {
|
|
const files = new Map<string, string>();
|
|
if (entry.type === "real") {
|
|
const diff = git(["diff", `${entry.rev}~1`, entry.rev, "--", ...entry.paths]);
|
|
// An empty diff would grade a guard case as a clean pass.
|
|
if (!diff) throw new Error(`${entry.id}: no diff at ${entry.rev}.`);
|
|
return { diff, files };
|
|
}
|
|
const before = git(["show", `${entry.rev}:${entry.path}`]);
|
|
if (!before?.includes(entry.after)) throw new Error(`${entry.id}: anchor not found.`);
|
|
const after = before.replace(entry.after, entry.after + entry.insert);
|
|
files.set(entry.path, after);
|
|
const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "hanzo-test-signal-"));
|
|
NodeFS.writeFileSync(NodePath.join(directory, "before"), before);
|
|
NodeFS.writeFileSync(NodePath.join(directory, "after"), after);
|
|
const raw = NodeChildProcess.spawnSync("git", ["diff", "--no-index", "before", "after"], {
|
|
cwd: directory,
|
|
encoding: "utf8",
|
|
}).stdout;
|
|
NodeFS.rmSync(directory, { recursive: true });
|
|
const diff = raw
|
|
.replace("diff --git a/before b/after", `diff --git a/${entry.path} b/${entry.path}`)
|
|
.replace("--- a/before", `--- a/${entry.path}`)
|
|
.replace("+++ b/after", `+++ b/${entry.path}`);
|
|
return { diff, files };
|
|
};
|
|
|
|
// The check reads the rules at the PR head, where they always exist. Old case commits predate
|
|
// them, so serve the rules from this checkout.
|
|
const ruleFiles = new Set(["docs/internals/testing.md", "AGENTS.md"]);
|
|
const readAt = (entry: Case, files: Map<string, string>, path: string) =>
|
|
ruleFiles.has(path)
|
|
? NodeFS.readFileSync(NodePath.join(repo, path), "utf8")
|
|
: (files.get(path) ?? git(["show", `${entry.rev}:${path}`]));
|
|
|
|
/** `git grep` at `rev`, with overlay and rule files searched in their served form instead. */
|
|
const grepAt = (entry: Case, files: Map<string, string>, pattern: string, scope: string) => {
|
|
// Status 2 is a bad pattern. Show grep's message, as a real tool would, so the model can fix it.
|
|
const check = NodeChildProcess.spawnSync("grep", ["-E", "--", pattern], {
|
|
input: "",
|
|
encoding: "utf8",
|
|
});
|
|
if (check.status !== 0 && check.status !== 1) return check.stderr.trim() || "grep failed.";
|
|
const served = new Map(files);
|
|
for (const path of ruleFiles) served.set(path, readAt(entry, files, path) ?? "");
|
|
// Served files also exist in this checkout, so git applies its own pathspec rules to the scope.
|
|
const matched = new Set((git(["ls-files", "--", scope || "."]) ?? "").split("\n"));
|
|
const inScope = (path: string) => matched.has(path);
|
|
const fromRev = (git(["grep", "-n", "-E", "-e", pattern, entry.rev, "--", scope || "."]) ?? "")
|
|
.split("\n")
|
|
.filter((line) => line !== "")
|
|
.map((line) => line.replace(`${entry.rev}:`, ""))
|
|
.filter((line) => !served.has(line.slice(0, line.indexOf(":"))));
|
|
const fromServed = [...served]
|
|
.filter(([path]) => inScope(path))
|
|
.flatMap(([path, text]) =>
|
|
NodeChildProcess.spawnSync("grep", ["-n", "-E", "--", pattern], {
|
|
input: text,
|
|
encoding: "utf8",
|
|
})
|
|
.stdout.split("\n")
|
|
.filter((line) => line !== "")
|
|
.map((line) => `${path}:${line}`),
|
|
);
|
|
return [...fromServed, ...fromRev].slice(0, 80).join("\n") || "No matches.";
|
|
};
|
|
|
|
const tools = [
|
|
{
|
|
type: "function",
|
|
name: "read_file",
|
|
description: "Read a file at the pull request head, with line numbers.",
|
|
parameters: {
|
|
type: "object",
|
|
properties: {
|
|
path: { type: "string" },
|
|
start_line: { type: "integer" },
|
|
end_line: { type: "integer" },
|
|
},
|
|
required: ["path"],
|
|
},
|
|
},
|
|
{
|
|
type: "function",
|
|
name: "grep",
|
|
description:
|
|
"Search the repository at the pull request head with an extended regular expression.",
|
|
parameters: {
|
|
type: "object",
|
|
properties: { pattern: { type: "string" }, path: { type: "string" } },
|
|
required: ["pattern"],
|
|
},
|
|
},
|
|
{
|
|
type: "function",
|
|
name: "post_comment",
|
|
description: "Post an inline review comment on a line of the pull request head.",
|
|
parameters: {
|
|
type: "object",
|
|
properties: { path: { type: "string" }, line: { type: "integer" }, body: { type: "string" } },
|
|
required: ["path", "line", "body"],
|
|
},
|
|
},
|
|
];
|
|
|
|
const system = (body: string) =>
|
|
[
|
|
"You are a Macroscope Check Run Agent reviewing a GitHub pull request.",
|
|
"Follow the check instructions below. Read the repository at the pull request head with read_file and grep.",
|
|
"Leave inline review comments with post_comment. Then give your final response.",
|
|
"",
|
|
"<check_instructions>",
|
|
body,
|
|
"</check_instructions>",
|
|
].join("\n");
|
|
|
|
const decodeResponse = Schema.decodeUnknownSync(
|
|
Schema.fromJsonString(
|
|
Schema.Struct({
|
|
output: Schema.Array(Schema.Record(Schema.String, Schema.Unknown)),
|
|
usage: Schema.optional(Schema.Struct({ total_tokens: Schema.optional(Schema.Number) })),
|
|
}),
|
|
),
|
|
);
|
|
const decodeArguments = Schema.decodeUnknownSync(
|
|
Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)),
|
|
);
|
|
const encodeRow = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
|
|
|
|
const post = Effect.fn("evaluateTestSignal.post")(function* (body: unknown) {
|
|
const client = (yield* HttpClient.HttpClient).pipe(
|
|
HttpClient.retryTransient({
|
|
retryOn: "errors-and-responses",
|
|
schedule: Schedule.exponential("2 seconds"),
|
|
times: 4,
|
|
}),
|
|
);
|
|
const response = yield* HttpClientRequest.post(`${proxy.url}/responses`).pipe(
|
|
HttpClientRequest.setHeaders(proxy.headers),
|
|
HttpClientRequest.bodyJson(body),
|
|
Effect.flatMap(client.execute),
|
|
);
|
|
const text = yield* response.text;
|
|
if (response.status !== 200)
|
|
return yield* Effect.die(new Error(`Model API ${response.status}: ${text}`));
|
|
return decodeResponse(text);
|
|
});
|
|
|
|
const runSample = Effect.fn("evaluateTestSignal.sample")(function* (entry: Case, model: string) {
|
|
const { diff, files } = world(entry);
|
|
const comments: Array<Comment> = [];
|
|
const input: Array<unknown> = [
|
|
{
|
|
role: "user",
|
|
content: `Pull request diff for the files this check includes:\n\n\`\`\`diff\n${diff}\`\`\``,
|
|
},
|
|
];
|
|
let final = "";
|
|
let tokens = 0;
|
|
for (let step = 0; step < 16; step++) {
|
|
const response = yield* post({
|
|
model,
|
|
instructions: system(instructions),
|
|
input,
|
|
tools,
|
|
reasoning: { effort: values.reasoning },
|
|
store: false,
|
|
include: ["reasoning.encrypted_content"],
|
|
});
|
|
tokens += response.usage?.total_tokens ?? 0;
|
|
input.push(...response.output);
|
|
const calls = response.output.filter((item) => item.type === "function_call");
|
|
final = response.output
|
|
.filter((item) => item.type === "message")
|
|
.flatMap((item) => (item.content as Array<{ text?: string }>).map((part) => part.text ?? ""))
|
|
.join("");
|
|
if (calls.length === 0) break;
|
|
for (const call of calls) {
|
|
const args = decodeArguments(String(call.arguments));
|
|
let output: string;
|
|
if (call.name === "post_comment") {
|
|
comments.push({
|
|
path: String(args.path),
|
|
line: Number(args.line),
|
|
body: String(args.body),
|
|
});
|
|
output = "Posted.";
|
|
} else if (call.name === "read_file") {
|
|
const text = readAt(entry, files, String(args.path));
|
|
const lines = text?.split("\n") ?? [];
|
|
const start = Math.max(1, Number(args.start_line ?? 1));
|
|
const end = Math.min(lines.length, Number(args.end_line ?? start + 399));
|
|
output =
|
|
text === undefined
|
|
? "File not found."
|
|
: lines
|
|
.slice(start - 1, end)
|
|
.map((line, index) => `${start + index}: ${line}`)
|
|
.join("\n");
|
|
} else {
|
|
output = grepAt(entry, files, String(args.pattern), String(args.path ?? ""));
|
|
}
|
|
input.push({ type: "function_call_output", call_id: call.call_id, output });
|
|
}
|
|
}
|
|
const targets = entry.targets.flatMap((target) => {
|
|
const range = testRange(readAt(entry, files, target.path) ?? "", target.title);
|
|
if (!range) throw new Error(`${entry.id}: target "${target.title}" not found.`);
|
|
return [{ path: target.path, ...range }];
|
|
});
|
|
return { ...grade({ kind: entry.kind, targets, comments, final }), comments, final, tokens };
|
|
});
|
|
|
|
const sampleSafely = (entry: Case, model: string, sample: number) =>
|
|
runSample(entry, model).pipe(
|
|
Effect.map((result) => ({ case: entry.id, kind: entry.kind, model, sample, ...result })),
|
|
Effect.catchCause((cause) =>
|
|
Effect.succeed({ case: entry.id, kind: entry.kind, model, sample, error: String(cause) }),
|
|
),
|
|
);
|
|
|
|
type Row = Effect.Success<ReturnType<typeof sampleSafely>>;
|
|
const isGraded = (row: Row) => "pass" in row;
|
|
const passed = (row: Row) => "pass" in row && row.pass;
|
|
|
|
const summary = await Effect.runPromise(
|
|
Effect.gen(function* () {
|
|
NodeFS.mkdirSync(outDirectory, { recursive: true });
|
|
const samples = Number(values.samples);
|
|
const pairs = models.flatMap((model) => selected.map((entry) => ({ model, entry })));
|
|
const rows: Array<Row> = (yield* Effect.forEach(
|
|
pairs.flatMap(({ model, entry }) =>
|
|
Array.from({ length: samples }, (_, n) => ({ model, entry, n })),
|
|
),
|
|
({ model, entry, n }) => sampleSafely(entry, model, n),
|
|
{ concurrency: Number(values.concurrency) },
|
|
)).slice();
|
|
// A third sample settles a split.
|
|
const splits = pairs.filter(({ model, entry }) => {
|
|
const own = rows.filter((row) => row.model === model && row.case === entry.id);
|
|
return own.length === 2 && own.some(passed) && !own.every(passed);
|
|
});
|
|
rows.push(
|
|
...(yield* Effect.forEach(splits, ({ model, entry }) => sampleSafely(entry, model, samples), {
|
|
concurrency: Number(values.concurrency),
|
|
})),
|
|
);
|
|
NodeFS.writeFileSync(
|
|
NodePath.join(outDirectory, "results.ndjson"),
|
|
rows.map((row) => encodeRow(row)).join("\n") + "\n",
|
|
);
|
|
|
|
const lines = [`prompt ${promptHash}, ${words} words, ${checkFile}`, ""];
|
|
lines.push("| model | bad caught | guards clean | errors | tokens |", "|---|---|---|---|---|");
|
|
for (const model of models) {
|
|
const verdicts = selected.map((entry) => {
|
|
const own = rows.filter((row) => row.model === model && row.case === entry.id);
|
|
// A run error counts as a failed sample, so one pass and one error is not a majority.
|
|
const graded = own.filter(isGraded);
|
|
const passes = own.filter(passed).length;
|
|
return {
|
|
entry,
|
|
pass: passes > own.length / 2,
|
|
errors: own.length - graded.length,
|
|
};
|
|
});
|
|
const count = (kind: Case["kind"]) => {
|
|
const own = verdicts.filter((verdict) => verdict.entry.kind === kind);
|
|
return `${own.filter((verdict) => verdict.pass).length}/${own.length}`;
|
|
};
|
|
const tokens = rows
|
|
.filter((row) => row.model === model && "tokens" in row)
|
|
.reduce((sum, row) => sum + ("tokens" in row ? row.tokens : 0), 0);
|
|
const errors = verdicts.reduce((sum, verdict) => sum + verdict.errors, 0);
|
|
lines.push(`| ${model} | ${count("bad")} | ${count("guard")} | ${errors} | ${tokens} |`);
|
|
const failed = verdicts.filter((verdict) => !verdict.pass).map((verdict) => verdict.entry.id);
|
|
if (failed.length > 0) lines.push(`| failed: ${failed.join(", ")} | | | | |`);
|
|
}
|
|
const report = lines.join("\n") + "\n";
|
|
NodeFS.writeFileSync(NodePath.join(outDirectory, "summary.md"), report);
|
|
return report;
|
|
}).pipe(Effect.provide(FetchHttpClient.layer)),
|
|
);
|
|
process.stdout.write(summary);
|