mirror of
https://github.com/hanzoai/dev-desktop.git
synced 2026-11-20 01:07:23 +00:00
108 lines
4.0 KiB
TypeScript
108 lines
4.0 KiB
TypeScript
import * as NodeUtil from "node:util";
|
|
import * as NodeServices from "@effect/platform-node/NodeServices";
|
|
import {
|
|
CodexSettings,
|
|
DEFAULT_SPEECH_POST_PROCESSING_PROMPT,
|
|
ProviderInstanceId,
|
|
} from "@hanzodev/contracts";
|
|
import * as Effect from "effect/Effect";
|
|
import * as FileSystem from "effect/FileSystem";
|
|
import * as Layer from "effect/Layer";
|
|
import * as Path from "effect/Path";
|
|
import * as Random from "effect/Random";
|
|
import * as Schema from "effect/Schema";
|
|
import * as ServerConfig from "../../apps/server/src/config.ts";
|
|
import { makeCodexTextGeneration } from "../../apps/server/src/textGeneration/CodexTextGeneration.ts";
|
|
import { buildTranscriptionPostProcessingPrompt } from "../../apps/server/src/textGeneration/TranscriptionPostProcessing.ts";
|
|
import { cases } from "./cases.ts";
|
|
|
|
const { values } = NodeUtil.parseArgs({
|
|
options: {
|
|
out: { type: "string" },
|
|
model: { type: "string" },
|
|
baseline: { type: "string" },
|
|
case: { type: "string" },
|
|
samples: { type: "string", default: "3" },
|
|
},
|
|
});
|
|
const { out, model } = values;
|
|
const samples = Number(values.samples);
|
|
const fixtures = cases.filter((fixture) => !values.case || fixture.id === values.case);
|
|
if (!out || !model || !fixtures.length || !Number.isSafeInteger(samples) || samples < 1) {
|
|
throw new Error(
|
|
"Use --out <directory> --model <configured model> [--case <id>] [--samples <count>].",
|
|
);
|
|
}
|
|
const Results = Schema.fromJsonString(
|
|
Schema.Array(
|
|
Schema.Struct({
|
|
id: Schema.String,
|
|
sample: Schema.Number,
|
|
text: Schema.String,
|
|
}),
|
|
),
|
|
);
|
|
const decodeSettings = Schema.decodeEffect(CodexSettings);
|
|
const decodeResults = Schema.decodeEffect(Results);
|
|
const encodeRows = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
|
|
await Effect.runPromise(
|
|
Effect.gen(function* () {
|
|
const fs = yield* FileSystem.FileSystem;
|
|
const path = yield* Path.Path;
|
|
const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "hanzo-voice-eval-" });
|
|
const service = yield* makeCodexTextGeneration(yield* decodeSettings({}));
|
|
const baseline = values.baseline
|
|
? yield* fs.readFileString(values.baseline).pipe(Effect.flatMap(decodeResults))
|
|
: [];
|
|
const results = [];
|
|
const review = [];
|
|
const key = [];
|
|
for (const fixture of fixtures) {
|
|
const start = fixture.before.length;
|
|
const prompt = buildTranscriptionPostProcessingPrompt(
|
|
DEFAULT_SPEECH_POST_PROCESSING_PROMPT,
|
|
fixture.transcript,
|
|
{
|
|
text: fixture.before + fixture.selection + fixture.after,
|
|
selection: { start, end: start + fixture.selection.length },
|
|
},
|
|
fixture.cue,
|
|
);
|
|
for (let sample = 1; sample <= samples; sample++) {
|
|
const previous = baseline.find((row) => row.id === fixture.id && row.sample === sample);
|
|
if (values.baseline && !previous)
|
|
throw new Error(`Missing baseline for ${fixture.id} sample ${sample}.`);
|
|
const { transcription } = yield* service.generateTranscriptionPostProcessing({
|
|
cwd,
|
|
prompt,
|
|
modelSelection: { instanceId: ProviderInstanceId.make("codex"), model },
|
|
});
|
|
results.push({ id: fixture.id, sample, text: transcription });
|
|
const first = !previous || (yield* Random.nextBoolean);
|
|
review.push({
|
|
...fixture,
|
|
sample,
|
|
A: first ? transcription : previous?.text,
|
|
B: first ? previous?.text : transcription,
|
|
});
|
|
key.push({ id: fixture.id, sample, candidate: first ? "A" : "B" });
|
|
}
|
|
}
|
|
yield* fs.makeDirectory(out, { recursive: true });
|
|
for (const [name, rows] of [
|
|
["results", results],
|
|
["review", review],
|
|
["answer-key", key],
|
|
] as const) {
|
|
yield* fs.writeFileString(path.join(out, name + ".json"), yield* encodeRows(rows));
|
|
}
|
|
}).pipe(
|
|
Effect.provide(
|
|
ServerConfig.layerTest(process.cwd(), { prefix: "hanzo-voice-eval-state-" }).pipe(
|
|
Layer.provideMerge(NodeServices.layer),
|
|
),
|
|
),
|
|
Effect.scoped,
|
|
),
|
|
);
|