mirror of
https://github.com/hanzoai/dev-desktop.git
synced 2026-11-20 01:07:23 +00:00
421 lines
15 KiB
TypeScript
421 lines
15 KiB
TypeScript
// @effect-diagnostics nodeBuiltinImport:off globalTimers:off - the eval owns a server subprocess and a scratch workspace.
|
|
/**
|
|
* An isolated Hanzo server for one device run. Its only provider is `evalbot`, an Fx instance
|
|
* backed by `fake-fx.ts`, so every reply is scripted. State lives under the run's `--out`.
|
|
*/
|
|
import * as NodeChildProcess from "node:child_process";
|
|
import * as NodeFS from "node:fs";
|
|
import * as NodeNet from "node:net";
|
|
import * as NodePath from "node:path";
|
|
import * as NodeWS from "@effect/platform-node/NodeWS";
|
|
import {
|
|
CommandId,
|
|
EnvironmentId,
|
|
ExecutionEnvironmentDescriptor,
|
|
MessageId,
|
|
ORCHESTRATION_PROTOCOL_HEADER,
|
|
ORCHESTRATION_PROTOCOL_VERSION,
|
|
OrchestrationV2ShellSnapshot,
|
|
ProjectId,
|
|
ProviderInstanceId,
|
|
ThreadId,
|
|
WsRpcGroup,
|
|
type OrchestrationV2ThreadProjection,
|
|
} from "@hanzodev/contracts";
|
|
import * as SecureRandom from "effect/crypto/SecureRandom";
|
|
import * as Effect from "effect/Effect";
|
|
import * as Layer from "effect/Layer";
|
|
import * as Schema from "effect/Schema";
|
|
import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
|
|
import { RpcClient, RpcSerialization } from "effect/rpc";
|
|
import { Socket } from "effect/socket";
|
|
|
|
const REPO_ROOT = NodePath.resolve(import.meta.dirname, "../..");
|
|
export const EVAL_INSTANCE = ProviderInstanceId.make("evalbot");
|
|
export const EVAL_MODEL = "eval/scripted";
|
|
|
|
export const uuid = Effect.flatMap(SecureRandom.SecureRandom, (random) => random.uuidV4());
|
|
|
|
export class EnvironmentError extends Schema.TaggedError<EnvironmentError>()("EnvironmentError", {
|
|
message: Schema.String,
|
|
}) {}
|
|
|
|
export const freePort = () =>
|
|
Effect.callback<number>((resume) => {
|
|
const server = NodeNet.createServer();
|
|
server.listen(0, () => {
|
|
const address = server.address();
|
|
server.close(() =>
|
|
resume(Effect.succeed(typeof address === "object" && address ? address.port : 0)),
|
|
);
|
|
});
|
|
});
|
|
|
|
const waitForPort = Effect.fn("environment.waitForPort")(function* (port: number) {
|
|
for (let attempt = 0; attempt < 120; attempt++) {
|
|
const open = yield* Effect.callback<boolean>((resume) => {
|
|
const socket = NodeNet.connect(port, "127.0.0.1");
|
|
socket.once("connect", () => (socket.destroy(), resume(Effect.succeed(true))));
|
|
socket.once("error", () => resume(Effect.succeed(false)));
|
|
});
|
|
if (open) return;
|
|
yield* Effect.sleep("500 millis");
|
|
}
|
|
return yield* new EnvironmentError({ message: `The server never listened on ${port}.` });
|
|
});
|
|
|
|
/** Runs a server CLI command and returns both output streams. */
|
|
const runCli = (args: ReadonlyArray<string>) =>
|
|
Effect.try({
|
|
try: () => {
|
|
const result = NodeChildProcess.spawnSync("node", ["apps/server/src/bin.ts", ...args], {
|
|
cwd: REPO_ROOT,
|
|
encoding: "utf8",
|
|
});
|
|
if (result.status !== 0) throw new Error(result.stderr);
|
|
return { stdout: result.stdout, stderr: result.stderr };
|
|
},
|
|
catch: (cause) =>
|
|
new EnvironmentError({ message: `bin.ts ${args.join(" ")} failed: ${cause}` }),
|
|
});
|
|
|
|
function writeScratch(baseDir: string) {
|
|
const workspace = NodePath.join(baseDir, "workspace/acme-app");
|
|
const control = NodePath.join(baseDir, "control");
|
|
NodeFS.mkdirSync(NodePath.join(baseDir, "userdata"), { recursive: true });
|
|
NodeFS.mkdirSync(control, { recursive: true });
|
|
NodeFS.mkdirSync(workspace, { recursive: true });
|
|
NodeFS.writeFileSync(NodePath.join(workspace, "README.md"), "# acme app\n");
|
|
// The / menu's pickable row; without it, fx finds only skills in folders above `--out`.
|
|
const skill = NodePath.join(workspace, ".fx/skills/eval-notes");
|
|
NodeFS.mkdirSync(skill, { recursive: true });
|
|
NodeFS.writeFileSync(
|
|
NodePath.join(skill, "SKILL.md"),
|
|
"---\nname: eval-notes\ndescription: Takes notes for the eval.\n---\nTake notes.\n",
|
|
);
|
|
const git = (...args: string[]) =>
|
|
NodeChildProcess.execFileSync("git", ["-C", workspace, ...args], { stdio: "ignore" });
|
|
git("init", "-q");
|
|
git("add", "-A");
|
|
git("-c", "user.name=eval", "-c", "[email protected]", "commit", "-qm", "init");
|
|
const launcher = NodePath.join(baseDir, "fx");
|
|
NodeFS.writeFileSync(
|
|
launcher,
|
|
`#!/bin/sh\nexec node ${JSON.stringify(NodePath.join(import.meta.dirname, "fake-fx.ts"))} "$@"\n`,
|
|
{ mode: 0o755 },
|
|
);
|
|
NodeFS.writeFileSync(
|
|
NodePath.join(baseDir, "userdata/settings.json"),
|
|
JSON.stringify({
|
|
providerInstances: {
|
|
[EVAL_INSTANCE]: {
|
|
driver: "fx",
|
|
displayName: "Eval Bot",
|
|
environment: [{ name: "HANZO_EVAL_FX_CONTROL_DIR", value: control }],
|
|
config: { enabled: true, binaryPath: launcher },
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
return { workspace, control };
|
|
}
|
|
|
|
/** Starts the server in the caller's scope; the scope's end stops it. */
|
|
export const startEnvironment = Effect.fn("environment.start")(function* (input: {
|
|
readonly baseDir: string;
|
|
/** Origin the device uses to reach this machine, without a port. */
|
|
readonly deviceHost: string;
|
|
}) {
|
|
NodeFS.rmSync(input.baseDir, { recursive: true, force: true });
|
|
const { workspace, control } = writeScratch(input.baseDir);
|
|
const home = NodePath.join(input.baseDir, "home");
|
|
NodeFS.mkdirSync(home, { recursive: true });
|
|
const port = yield* freePort();
|
|
const log = NodeFS.openSync(NodePath.join(input.baseDir, "server.log"), "w");
|
|
yield* Effect.acquireRelease(
|
|
Effect.sync(() =>
|
|
NodeChildProcess.spawn(
|
|
"node",
|
|
[
|
|
"apps/server/src/bin.ts",
|
|
"serve",
|
|
"--host",
|
|
"0.0.0.0",
|
|
"--port",
|
|
String(port),
|
|
"--base-dir",
|
|
input.baseDir,
|
|
"--no-browser",
|
|
"--log-level",
|
|
"warn",
|
|
workspace,
|
|
],
|
|
// A scratch HOME keeps the user's own skills and config out of the app under test.
|
|
{ cwd: REPO_ROOT, stdio: ["ignore", log, log], env: { ...process.env, HOME: home } },
|
|
),
|
|
),
|
|
(child) => Effect.sync(() => child.kill("SIGTERM")),
|
|
);
|
|
yield* waitForPort(port);
|
|
const origin = `http://127.0.0.1:${port}`;
|
|
const issued = yield* runCli([
|
|
"auth",
|
|
"session",
|
|
"issue",
|
|
"--base-dir",
|
|
input.baseDir,
|
|
"--token-only",
|
|
]);
|
|
// The token is the last line of stdout; stderr is only a fallback.
|
|
const lastLine = (text: string) =>
|
|
text
|
|
.split("\n")
|
|
.filter((line) => line.trim())
|
|
.at(-1)
|
|
?.trim();
|
|
const token = lastLine(issued.stdout) ?? lastLine(issued.stderr) ?? "";
|
|
const http = (yield* HttpClient.HttpClient).pipe(
|
|
HttpClient.mapRequest((request) =>
|
|
request.pipe(
|
|
HttpClientRequest.bearerToken(token),
|
|
HttpClientRequest.setHeader(
|
|
ORCHESTRATION_PROTOCOL_HEADER,
|
|
String(ORCHESTRATION_PROTOCOL_VERSION),
|
|
),
|
|
),
|
|
),
|
|
HttpClient.filterStatusOk,
|
|
);
|
|
const descriptor = yield* http
|
|
.get(`${origin}/.well-known/hanzo/environment`)
|
|
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(ExecutionEnvironmentDescriptor)));
|
|
const shell = () =>
|
|
http
|
|
.get(`${origin}/v1/event/shell`)
|
|
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(OrchestrationV2ShellSnapshot)));
|
|
const createProject = Effect.fn("environment.createProject")(function* (
|
|
title: string,
|
|
workspaceRoot: string,
|
|
) {
|
|
const projectId = ProjectId.make(yield* uuid);
|
|
yield* http.execute(
|
|
HttpClientRequest.post(`${origin}/v1/projects/mutate`).pipe(
|
|
HttpClientRequest.bodyJsonUnsafe({
|
|
type: "project.create",
|
|
commandId: yield* uuid,
|
|
projectId,
|
|
title,
|
|
workspaceRoot,
|
|
}),
|
|
),
|
|
);
|
|
return projectId;
|
|
});
|
|
const projectId = yield* createProject("acme-app", workspace);
|
|
const ticket = yield* http
|
|
.execute(HttpClientRequest.post(`${origin}/v1/auth/websocket-ticket`))
|
|
.pipe(
|
|
Effect.flatMap(HttpClientResponse.schemaBodyJson(Schema.Struct({ ticket: Schema.String }))),
|
|
);
|
|
// Built into the caller's scope: `Effect.provide` would close the socket once `make` returns.
|
|
const protocol = yield* Layer.build(
|
|
RpcClient.layerProtocolSocket().pipe(
|
|
Layer.provide(
|
|
Socket.layerWebSocket(
|
|
`${origin.replace(/^http/, "ws")}/ws?orchestrationProtocol=${ORCHESTRATION_PROTOCOL_VERSION}&wsTicket=${encodeURIComponent(ticket.ticket)}`,
|
|
),
|
|
),
|
|
Layer.provide(RpcSerialization.layerJson),
|
|
Layer.provide(NodeWS.layerWebSocketConstructor),
|
|
),
|
|
);
|
|
const client = yield* RpcClient.make(WsRpcGroup).pipe(Effect.provideContext(protocol));
|
|
const environmentId = EnvironmentId.make(descriptor.environmentId);
|
|
|
|
const command = Effect.map(uuid, CommandId.make);
|
|
const projection = (threadId: ThreadId) =>
|
|
client["orchestration.getThreadProjection"]({ threadId });
|
|
|
|
const create = Effect.fn("environment.create")(function* (
|
|
title: string,
|
|
prompt?: string,
|
|
project: ProjectId = projectId,
|
|
) {
|
|
const threadId = ThreadId.make(yield* uuid);
|
|
yield* client["orchestration.launchThread"]({
|
|
commandId: yield* command,
|
|
threadId,
|
|
projectId: project,
|
|
title,
|
|
modelSelection: { instanceId: EVAL_INSTANCE, model: EVAL_MODEL },
|
|
runtimeMode: "full-access",
|
|
interactionMode: "default",
|
|
workspaceStrategy: { type: "root" },
|
|
...(prompt === undefined ? {} : { initialMessage: { text: prompt, attachments: [] } }),
|
|
});
|
|
return threadId;
|
|
});
|
|
|
|
/** Starts a thread whose first reply follows `prompt`'s `[[eval:…]]` script. */
|
|
const launch = (title: string, prompt: string) => create(title, prompt);
|
|
|
|
const until = Effect.fn("environment.until")(function* <A>(
|
|
description: string,
|
|
threadId: ThreadId,
|
|
pick: (value: OrchestrationV2ThreadProjection) => A | undefined,
|
|
) {
|
|
for (let attempt = 0; attempt < 150; attempt++) {
|
|
const found = pick(yield* projection(threadId));
|
|
if (found !== undefined) return found;
|
|
yield* Effect.sleep("200 millis");
|
|
}
|
|
return yield* new EnvironmentError({ message: `Thread ${threadId} never had ${description}.` });
|
|
});
|
|
|
|
const send = Effect.fn("environment.send")(function* (threadId: ThreadId, text: string) {
|
|
yield* client["orchestration.dispatchCommand"]({
|
|
type: "message.dispatch",
|
|
commandId: yield* command,
|
|
threadId,
|
|
messageId: MessageId.make(yield* uuid),
|
|
text,
|
|
attachments: [],
|
|
dispatchMode: { type: "queue_after_active" },
|
|
createdBy: "user",
|
|
creationSource: "server",
|
|
});
|
|
});
|
|
|
|
const isSettled = (value: OrchestrationV2ThreadProjection) =>
|
|
value.runs.length > 0 &&
|
|
value.runs.every((run) =>
|
|
["completed", "failed", "interrupted", "cancelled"].includes(run.status),
|
|
);
|
|
|
|
/** Waits until every run of the thread has ended. */
|
|
const settled = Effect.fn("environment.settled")(function* (threadId: ThreadId) {
|
|
for (let attempt = 0; attempt < 600; attempt++) {
|
|
const value = yield* projection(threadId);
|
|
if (isSettled(value)) return value;
|
|
yield* Effect.sleep("200 millis");
|
|
}
|
|
return yield* new EnvironmentError({ message: `Thread ${threadId} never settled.` });
|
|
});
|
|
|
|
/** A settled thread of `turns` request and reply pairs, for scroll and jank cases. */
|
|
const longThread = Effect.fn("environment.longThread")(function* (title: string, turns: number) {
|
|
const threadId = yield* launch(title, "Summarize the module. [[eval:stream:400:0]]");
|
|
yield* settled(threadId);
|
|
for (let turn = 1; turn < turns; turn++) {
|
|
const text = `Next part ${turn}. [[eval:stream:${turn % 3 === 0 ? 900 : 300}:0]]`;
|
|
yield* send(threadId, text);
|
|
// The message and its run land in one command, so once it shows, `settled` sees the run.
|
|
yield* until("its next message", threadId, (value) =>
|
|
value.messages.some((message) => message.text === text) ? true : undefined,
|
|
);
|
|
yield* settled(threadId);
|
|
}
|
|
return threadId;
|
|
});
|
|
|
|
return {
|
|
/** An empty thread in its own project; `lock` then disables `evalbot` there, so sends are rejected. */
|
|
launchLockable: Effect.fn("environment.launchLockable")(function* (title: string) {
|
|
const root = NodePath.join(input.baseDir, `workspace/locked-${(yield* uuid).slice(0, 6)}`);
|
|
NodeFS.mkdirSync(root, { recursive: true });
|
|
const project = yield* createProject("locked", root);
|
|
const threadId = yield* create(title, undefined, project);
|
|
const lock = client["server.updateSettings"]({
|
|
patch: {
|
|
projectSettingsOverrides: {
|
|
[project]: { providerInstances: { [EVAL_INSTANCE]: { enabled: false } } },
|
|
},
|
|
},
|
|
});
|
|
return { threadId, lock };
|
|
}),
|
|
/** Archives every thread but `keep`, so earlier samples' rows never crowd Home. */
|
|
archiveAllExcept: Effect.fn("environment.archiveAllExcept")(function* (keep: ThreadId) {
|
|
for (const thread of (yield* shell()).threads) {
|
|
if (thread.id === keep) continue;
|
|
yield* client["orchestration.dispatchCommand"]({
|
|
type: "thread.archive",
|
|
commandId: yield* command,
|
|
threadId: thread.id,
|
|
});
|
|
}
|
|
}),
|
|
/** Stops every unfinished run, so one sample's held turns never leak into the next. */
|
|
interruptAll: Effect.fn("environment.interruptAll")(function* () {
|
|
const threads = (yield* shell()).threads;
|
|
for (const thread of threads)
|
|
for (const run of (yield* projection(thread.id)).runs) {
|
|
if (["completed", "failed", "interrupted", "cancelled"].includes(run.status)) continue;
|
|
yield* client["orchestration.dispatchCommand"]({
|
|
type: "run.interrupt",
|
|
commandId: yield* command,
|
|
threadId: thread.id,
|
|
runId: run.id,
|
|
});
|
|
}
|
|
for (const thread of threads)
|
|
if ((yield* projection(thread.id)).runs.length) yield* settled(thread.id);
|
|
}),
|
|
waitForPendingInput: (threadId: ThreadId) =>
|
|
until("a pending question", threadId, (value) =>
|
|
value.runtimeRequests.find((request) => request.status === "pending"),
|
|
),
|
|
waitForSubagents: (threadId: ThreadId) =>
|
|
until("running subagents", threadId, (value) =>
|
|
value.subagents.length >= 2 ? value.subagents : undefined,
|
|
),
|
|
origin,
|
|
environmentId,
|
|
archive: (threadId: ThreadId) =>
|
|
command.pipe(
|
|
Effect.flatMap((commandId) =>
|
|
client["orchestration.dispatchCommand"]({ type: "thread.archive", commandId, threadId }),
|
|
),
|
|
Effect.asVoid,
|
|
),
|
|
unpin: (threadId: ThreadId) =>
|
|
command.pipe(
|
|
Effect.flatMap((commandId) =>
|
|
client["orchestration.dispatchCommand"]({ type: "thread.unpin", commandId, threadId }),
|
|
),
|
|
Effect.asVoid,
|
|
),
|
|
projection,
|
|
launch,
|
|
send,
|
|
settled,
|
|
longThread,
|
|
shell,
|
|
release: (key: string) =>
|
|
Effect.sync(() => NodeFS.writeFileSync(NodePath.join(control, `release-${key}`), "")),
|
|
pairingUrl: () =>
|
|
runCli([
|
|
"auth",
|
|
"pairing",
|
|
"create",
|
|
"--base-dir",
|
|
input.baseDir,
|
|
"--base-url",
|
|
`http://${input.deviceHost}:${port}`,
|
|
"--ttl",
|
|
"15m",
|
|
"--label",
|
|
"mobile-behavior-eval",
|
|
]).pipe(
|
|
Effect.flatMap((out) => {
|
|
const url = /Pair URL:\s*(\S+)/.exec(`${out.stdout}\n${out.stderr}`)?.[1];
|
|
return url
|
|
? Effect.succeed(url)
|
|
: Effect.fail(new EnvironmentError({ message: "No pairing URL in CLI output." }));
|
|
}),
|
|
),
|
|
};
|
|
}, Effect.provide(FetchHttpClient.layer));
|
|
|
|
export type EvalEnvironment = Effect.Success<ReturnType<typeof startEnvironment>>;
|