mirror of
https://github.com/hanzoai/dev-desktop.git
synced 2026-11-20 01:07:23 +00:00
411 lines
16 KiB
TypeScript
411 lines
16 KiB
TypeScript
// See docs/internals/mobile-development.md#measuring-renders.
|
|
import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient";
|
|
import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
|
|
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
|
|
import * as NodeServices from "@effect/platform-node/NodeServices";
|
|
import * as NodeWS from "@effect/platform-node/NodeWS";
|
|
import {
|
|
CommandId,
|
|
EnvironmentId,
|
|
ExecutionEnvironmentDescriptor,
|
|
ORCHESTRATION_PROTOCOL_HEADER,
|
|
ORCHESTRATION_PROTOCOL_VERSION,
|
|
OrchestrationV2ShellSnapshot,
|
|
ProjectId,
|
|
ProviderInstanceId,
|
|
ThreadId,
|
|
WsRpcGroup,
|
|
type OrchestrationV2ThreadProjection,
|
|
} from "@hanzodev/contracts";
|
|
import * as Console from "effect/Console";
|
|
import * as SecureRandom from "effect/crypto/SecureRandom";
|
|
import * as Effect from "effect/Effect";
|
|
import * as FileSystem from "effect/FileSystem";
|
|
import * as Layer from "effect/Layer";
|
|
import * as Option from "effect/Option";
|
|
import * as Schema from "effect/Schema";
|
|
import * as Stream from "effect/Stream";
|
|
import { Command, Flag } from "effect/cli";
|
|
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
|
|
import { RpcClient, RpcSerialization } from "effect/rpc";
|
|
import { Socket } from "effect/socket";
|
|
|
|
const RenderReport = Schema.Struct({
|
|
commits: Schema.Number,
|
|
totalRenders: Schema.Number,
|
|
components: Schema.Array(Schema.Struct({ key: Schema.String, count: Schema.Number })),
|
|
causes: Schema.Array(Schema.Struct({ key: Schema.String, count: Schema.Number })),
|
|
});
|
|
type RenderReport = typeof RenderReport.Type;
|
|
const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(RenderReport));
|
|
const InspectorTargets = Schema.Array(
|
|
Schema.Struct({ appId: Schema.String, webSocketDebuggerUrl: Schema.String }),
|
|
);
|
|
const CdpReply = Schema.Struct({
|
|
id: Schema.Number,
|
|
result: Schema.optional(
|
|
Schema.Struct({
|
|
result: Schema.optional(Schema.Struct({ value: Schema.optional(Schema.Unknown) })),
|
|
exceptionDetails: Schema.optional(Schema.Unknown),
|
|
}),
|
|
),
|
|
});
|
|
const decodeCdpReply = Schema.decodeUnknownOption(Schema.fromJsonString(CdpReply));
|
|
const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
|
|
|
|
class MeasureError extends Schema.TaggedError<MeasureError>()("MeasureError", {
|
|
message: Schema.String,
|
|
}) {}
|
|
|
|
const STREAM_PROMPT =
|
|
"Think it through step by step, then write a 3000-word essay on the history of typesetting in plain paragraphs with a few headings. Do not use any tools.";
|
|
|
|
interface Options {
|
|
readonly server: string;
|
|
readonly token: string;
|
|
readonly metro: string;
|
|
readonly scheme: string;
|
|
readonly runs: number;
|
|
readonly windowSeconds: number;
|
|
readonly provider: string;
|
|
readonly model: string;
|
|
readonly projectRoot: string;
|
|
}
|
|
|
|
const uuid = Effect.flatMap(SecureRandom.SecureRandom, (random) => random.uuidV4());
|
|
|
|
const median = (values: ReadonlyArray<number>) => {
|
|
const sorted = [...values].sort((a, b) => a - b);
|
|
const middle = Math.floor(sorted.length / 2);
|
|
return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2;
|
|
};
|
|
|
|
const connectInspector = Effect.fn("measure.connectInspector")(function* (metro: string) {
|
|
const http = yield* HttpClient.HttpClient;
|
|
const targets = yield* http
|
|
.get(`${metro}/json/list`)
|
|
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(InspectorTargets)));
|
|
const target = targets.find((entry) => entry.appId.startsWith("ai.hanzo.dev"));
|
|
if (!target)
|
|
return yield* new MeasureError({
|
|
message: `No Hanzo Dev app is attached to Metro at ${metro}.`,
|
|
});
|
|
// Expo closes inspector sockets unless Origin matches Metro's advertised URL; pass that URL as --metro.
|
|
const socket = yield* Socket.makeWebSocket(target.webSocketDebuggerUrl, {
|
|
headers: { origin: new URL(metro).origin },
|
|
openTimeout: "10 seconds",
|
|
});
|
|
const writer = yield* socket.writer;
|
|
const reader = yield* socket.reader;
|
|
let nextId = 0;
|
|
return Effect.fn("measure.evaluate")(function* (expression: string) {
|
|
const id = ++nextId;
|
|
yield* writer.write(
|
|
encodeJson({ id, method: "Runtime.evaluate", params: { expression, returnByValue: true } }),
|
|
);
|
|
while (true) {
|
|
for (const frame of yield* reader.pull) {
|
|
const text = typeof frame === "string" ? frame : new TextDecoder().decode(frame);
|
|
const reply = decodeCdpReply(text);
|
|
if (Option.isNone(reply) || reply.value.id !== id) continue;
|
|
if (reply.value.result?.exceptionDetails)
|
|
return yield* new MeasureError({
|
|
message: `Evaluation failed: ${encodeJson(reply.value.result.exceptionDetails)}`,
|
|
});
|
|
return reply.value.result?.result?.value;
|
|
}
|
|
}
|
|
});
|
|
});
|
|
|
|
const connectServer = Effect.fn("measure.connectServer")(function* (options: Options) {
|
|
const http = (yield* HttpClient.HttpClient).pipe(
|
|
HttpClient.mapRequest((request) =>
|
|
request.pipe(
|
|
HttpClientRequest.bearerToken(options.token),
|
|
HttpClientRequest.setHeader(
|
|
ORCHESTRATION_PROTOCOL_HEADER,
|
|
String(ORCHESTRATION_PROTOCOL_VERSION),
|
|
),
|
|
),
|
|
),
|
|
HttpClient.filterStatusOk,
|
|
);
|
|
const descriptor = yield* http
|
|
.get(`${options.server}/.well-known/hanzo/environment`)
|
|
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(ExecutionEnvironmentDescriptor)));
|
|
const shell = yield* http
|
|
.get(`${options.server}/v1/event/shell`)
|
|
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(OrchestrationV2ShellSnapshot)));
|
|
let projectId = shell.projects.find(
|
|
(project) => project.workspaceRoot === options.projectRoot,
|
|
)?.id;
|
|
// A project record outlives its folder, and turns fail without it.
|
|
yield* (yield* FileSystem.FileSystem).makeDirectory(options.projectRoot, { recursive: true });
|
|
if (!projectId) {
|
|
projectId = ProjectId.make(yield* uuid);
|
|
yield* http.execute(
|
|
HttpClientRequest.post(`${options.server}/v1/projects/mutate`).pipe(
|
|
HttpClientRequest.bodyJsonUnsafe({
|
|
type: "project.create",
|
|
commandId: yield* uuid,
|
|
projectId,
|
|
title: "Render measurement",
|
|
workspaceRoot: options.projectRoot,
|
|
}),
|
|
),
|
|
);
|
|
}
|
|
const ticket = yield* http
|
|
.execute(HttpClientRequest.post(`${options.server}/v1/auth/websocket-ticket`))
|
|
.pipe(
|
|
Effect.flatMap(HttpClientResponse.schemaBodyJson(Schema.Struct({ ticket: Schema.String }))),
|
|
);
|
|
// Built into the caller's scope: `Effect.provide` would close the socket once `make` returns.
|
|
const protocol = yield* Layer.build(
|
|
RpcClient.layerProtocolSocket().pipe(
|
|
Layer.provide(
|
|
Socket.layerWebSocket(
|
|
`${options.server.replace(/^http/, "ws")}/ws?orchestrationProtocol=${ORCHESTRATION_PROTOCOL_VERSION}&wsTicket=${encodeURIComponent(ticket.ticket)}`,
|
|
),
|
|
),
|
|
Layer.provide(RpcSerialization.layerJson),
|
|
),
|
|
);
|
|
const client = yield* RpcClient.make(WsRpcGroup).pipe(Effect.provideContext(protocol));
|
|
return { client, environmentId: EnvironmentId.make(descriptor.environmentId), projectId };
|
|
});
|
|
|
|
// Some providers stream only reasoning, so any streaming text item counts.
|
|
const streamingItem = (projection: OrchestrationV2ThreadProjection) =>
|
|
projection.turnItems.find(
|
|
(item) => (item.type === "assistant_message" || item.type === "reasoning") && item.streaming,
|
|
);
|
|
|
|
const measureStreamReply = Effect.fn("measure.streamReply")(function* (
|
|
options: Options,
|
|
run: number,
|
|
route: Option.Option<string>,
|
|
) {
|
|
if (Option.isSome(route)) yield* reloadAndOpen(options, route.value);
|
|
const evaluate = yield* connectInspector(options.metro);
|
|
const { client, environmentId, projectId } = yield* connectServer(options);
|
|
const threadId = ThreadId.make(yield* uuid);
|
|
yield* client["orchestration.launchThread"]({
|
|
commandId: CommandId.make(yield* uuid),
|
|
threadId,
|
|
projectId,
|
|
title: `Render measurement ${run}`,
|
|
modelSelection: {
|
|
instanceId: ProviderInstanceId.make(options.provider),
|
|
model: options.model,
|
|
},
|
|
// The prompt needs no tools; approval-required keeps a stray call from acting.
|
|
runtimeMode: "approval-required",
|
|
interactionMode: "default",
|
|
workspaceStrategy: { type: "root" },
|
|
initialMessage: { text: STREAM_PROMPT, attachments: [] },
|
|
});
|
|
// A failed or interrupted window must not leave a provider turn running.
|
|
yield* Effect.addFinalizer(() =>
|
|
Effect.gen(function* () {
|
|
const after = yield* client["orchestration.getThreadProjection"]({ threadId });
|
|
const activeRun = after.runs.find(
|
|
(entry) => !["completed", "failed", "interrupted", "cancelled"].includes(entry.status),
|
|
);
|
|
if (!activeRun) return;
|
|
yield* client["orchestration.dispatchCommand"]({
|
|
type: "run.interrupt",
|
|
commandId: CommandId.make(yield* uuid),
|
|
threadId,
|
|
runId: activeRun.id,
|
|
});
|
|
}).pipe(Effect.ignore),
|
|
);
|
|
if (Option.isNone(route))
|
|
yield* evaluate(
|
|
`__hanzoRenderCounterOpen(${encodeJson(`${options.scheme}://threads/${environmentId}/${threadId}`)})`,
|
|
);
|
|
const projectionOf = (item: { kind: string; projection?: OrchestrationV2ThreadProjection }) =>
|
|
item.kind === "snapshot" && item.projection
|
|
? Effect.succeed(item.projection)
|
|
: client["orchestration.getThreadProjection"]({ threadId });
|
|
const streaming = yield* client["orchestration.subscribeThread"]({ threadId }).pipe(
|
|
Stream.mapEffect(projectionOf),
|
|
Stream.filter((projection) => streamingItem(projection) !== undefined),
|
|
Stream.runHead,
|
|
Effect.timeout("90 seconds"),
|
|
);
|
|
if (Option.isNone(streaming))
|
|
return yield* new MeasureError({ message: `Run ${run}: the reply never started streaming.` });
|
|
yield* evaluate("__hanzoRenderCounterStart()");
|
|
yield* Effect.sleep(`${options.windowSeconds} seconds`);
|
|
const report = yield* decodeReport(yield* evaluate("JSON.stringify(__hanzoRenderCounterStop())"));
|
|
// An in-view streaming thread must commit; out-of-view streaming may commit nothing.
|
|
if (report.commits === 0 && Option.isNone(route))
|
|
return yield* new MeasureError({
|
|
message: `Run ${run}: no React commits in the window. Check the device for a red box.`,
|
|
});
|
|
const after = yield* client["orchestration.getThreadProjection"]({ threadId });
|
|
return { report, stillStreaming: streamingItem(after) !== undefined };
|
|
}, Effect.scoped);
|
|
|
|
const appLink = (options: Options, route: string) =>
|
|
`__hanzoRenderCounterOpen(${encodeJson(`${options.scheme}://${route.replace(/^\//, "")}`)})`;
|
|
|
|
const reloadAndOpen = Effect.fn("measure.reloadAndOpen")(function* (
|
|
options: Options,
|
|
route: string,
|
|
) {
|
|
// A link to an already-open screen only refocuses it, so each run starts from a reload.
|
|
yield* Effect.scoped(
|
|
Effect.flatMap(connectInspector(options.metro), (evaluate) =>
|
|
evaluate("__hanzoRenderCounterReload()"),
|
|
),
|
|
);
|
|
yield* Effect.sleep("15 seconds");
|
|
if (route === "/" || route === "") return;
|
|
yield* Effect.scoped(
|
|
Effect.flatMap(connectInspector(options.metro), (evaluate) =>
|
|
evaluate(appLink(options, route)),
|
|
),
|
|
);
|
|
yield* Effect.sleep("5 seconds");
|
|
});
|
|
|
|
const measureRouteOpen = Effect.fn("measure.routeOpen")(function* (
|
|
options: Options,
|
|
route: string,
|
|
run: number,
|
|
) {
|
|
yield* Effect.scoped(
|
|
Effect.flatMap(connectInspector(options.metro), (evaluate) =>
|
|
evaluate("__hanzoRenderCounterReload()"),
|
|
),
|
|
);
|
|
yield* Effect.sleep("15 seconds");
|
|
const evaluate = yield* connectInspector(options.metro);
|
|
yield* evaluate("__hanzoRenderCounterStart()");
|
|
yield* evaluate(appLink(options, route));
|
|
yield* Effect.sleep(`${options.windowSeconds} seconds`);
|
|
const report = yield* decodeReport(yield* evaluate("JSON.stringify(__hanzoRenderCounterStop())"));
|
|
if (report.commits === 0)
|
|
return yield* new MeasureError({
|
|
message: `Run ${run}: no React commits in the window. Check that ${route} is a valid app path.`,
|
|
});
|
|
return { report, stillStreaming: undefined };
|
|
}, Effect.scoped);
|
|
|
|
const medianTable = (
|
|
title: string,
|
|
reports: ReadonlyArray<RenderReport>,
|
|
pick: (report: RenderReport) => RenderReport["causes"],
|
|
limit: number,
|
|
) => {
|
|
const keys = new Set(reports.flatMap((report) => pick(report).map((entry) => entry.key)));
|
|
const rows = [...keys]
|
|
.map((key) => ({
|
|
key,
|
|
count: median(reports.map((report) => pick(report).find((e) => e.key === key)?.count ?? 0)),
|
|
}))
|
|
.filter((row) => row.count > 0)
|
|
.sort((a, b) => b.count - a.count)
|
|
.slice(0, limit);
|
|
return [
|
|
`| ${title} | Renders |`,
|
|
"|---|---:|",
|
|
...rows.map((r) => `| \`${r.key}\` | ${r.count} |`),
|
|
];
|
|
};
|
|
|
|
const formatTable = (reports: ReadonlyArray<RenderReport>) => {
|
|
const totals = (pick: (report: RenderReport) => number) => median(reports.map(pick));
|
|
return [
|
|
`Median of ${reports.length} runs.`,
|
|
"",
|
|
"| | |",
|
|
"|---|---:|",
|
|
`| renders | ${totals((r) => r.totalRenders)} |`,
|
|
`| commits | ${totals((r) => r.commits)} |`,
|
|
// Provider pace sets how many commits land in the window; per-commit cost compares across runs.
|
|
`| renders per commit | ${totals((r) => Math.round(r.totalRenders / Math.max(r.commits, 1)))} |`,
|
|
"",
|
|
...medianTable("Component", reports, (r) => r.components, 20),
|
|
"",
|
|
...medianTable("Cause", reports, (r) => r.causes, 15),
|
|
].join("\n");
|
|
};
|
|
|
|
const command = Command.make(
|
|
"measure-renders",
|
|
{
|
|
server: Flag.String("server").pipe(
|
|
Flag.withDescription("Hanzo server origin the app is paired with."),
|
|
Flag.withDefault(""),
|
|
),
|
|
token: Flag.String("token").pipe(
|
|
Flag.withDescription("Bearer token from `hanzo auth session issue --token-only`."),
|
|
Flag.withDefault(""),
|
|
),
|
|
route: Flag.String("route").pipe(
|
|
Flag.withDescription(
|
|
"App path to open and measure. With --server, the path stays open while a new thread streams.",
|
|
),
|
|
Flag.optional,
|
|
),
|
|
metro: Flag.String("metro").pipe(Flag.withDefault("http://127.0.0.1:8081")),
|
|
scheme: Flag.String("scheme").pipe(Flag.withDefault("hanzo-dev")),
|
|
runs: Flag.Int("runs").pipe(Flag.withDefault(3)),
|
|
window: Flag.Int("window").pipe(
|
|
Flag.withDescription("Seconds to record; defaults to 12 for a reply and 6 for --route."),
|
|
Flag.optional,
|
|
),
|
|
provider: Flag.String("provider").pipe(Flag.withDefault("claudeAgent")),
|
|
model: Flag.String("model").pipe(Flag.withDefault("claude-haiku-5-5")),
|
|
projectRoot: Flag.String("project-root").pipe(
|
|
Flag.withDescription("Scratch directory for the measurement project; created if missing."),
|
|
Flag.withDefault(""),
|
|
),
|
|
},
|
|
Effect.fn("measure.main")(function* (flags) {
|
|
const route = flags.route;
|
|
const options: Options = {
|
|
...flags,
|
|
windowSeconds: Option.getOrElse(flags.window, () =>
|
|
Option.isSome(route) && !flags.server ? 6 : 12,
|
|
),
|
|
};
|
|
const streams = Option.isNone(route) || flags.server !== "";
|
|
if (streams && (!flags.server || !flags.token || !flags.projectRoot))
|
|
return yield* new MeasureError({
|
|
message: "Streaming a reply needs --server, --token and --project-root.",
|
|
});
|
|
const reports: RenderReport[] = [];
|
|
for (let run = 1; run <= flags.runs; run++) {
|
|
const { report, stillStreaming } =
|
|
Option.isSome(route) && !streams
|
|
? yield* measureRouteOpen(options, route.value, run)
|
|
: yield* measureStreamReply(options, run, route);
|
|
yield* Console.error(
|
|
`run ${run}: ${report.totalRenders} renders in ${report.commits} commits${stillStreaming === false ? " (warning: streaming ended inside the window)" : ""}`,
|
|
);
|
|
reports.push(report);
|
|
// Let the interrupted turn settle so the next run starts from a quiet app.
|
|
yield* Effect.sleep("3 seconds");
|
|
}
|
|
yield* Console.log(formatTable(reports));
|
|
}),
|
|
);
|
|
|
|
Command.run(command, { version: "0.0.0" }).pipe(
|
|
Effect.provide(
|
|
Layer.mergeAll(
|
|
NodeServices.layer,
|
|
NodeCrypto.layer,
|
|
NodeHttpClient.layerUndici,
|
|
NodeWS.layerWebSocketConstructor,
|
|
),
|
|
),
|
|
NodeRuntime.runMain,
|
|
);
|