mirror of
https://github.com/hanzoai/dev-desktop.git
synced 2026-11-20 01:07:23 +00:00
158 lines
6.2 KiB
TypeScript
158 lines
6.2 KiB
TypeScript
// @effect-diagnostics nodeBuiltinImport:off - compares the generated bundle directly with its checked-in bytes.
|
|
import * as NodeFS from "node:fs";
|
|
|
|
import { catalog } from "@handy-computer/models";
|
|
import { recommendSpeechModels } from "@hanzodev/shared/speech";
|
|
import policy from "../../../packages/shared/src/speechRecommendationPolicy.json" with { type: "json" };
|
|
import { expect, it } from "vite-plus/test";
|
|
|
|
import manifest from "../package.json" with { type: "json" };
|
|
import { generateSpeechCatalog, renderSpeechCatalog } from "./speech-catalog.ts";
|
|
|
|
const generate = (models = catalog.models) =>
|
|
generateSpeechCatalog(
|
|
{ ...catalog.data, models },
|
|
manifest.devDependencies["@handy-computer/models"],
|
|
manifest.dependencies["transcribe-cpp"],
|
|
);
|
|
|
|
it("reproduces the bundled registry byte for byte without upstream requests", async () => {
|
|
const { fastCpuMaxWaitSeconds, ...methodology } = catalog.methodology;
|
|
expect(policy).toEqual({ ...methodology, fastMaxWaitSeconds: fastCpuMaxWaitSeconds });
|
|
expect(NodeFS.readFileSync(new URL("../src/speech/catalog.json", import.meta.url), "utf8")).toBe(
|
|
await renderSpeechCatalog(
|
|
catalog.data,
|
|
manifest.devDependencies["@handy-computer/models"],
|
|
manifest.dependencies["transcribe-cpp"],
|
|
),
|
|
);
|
|
for (const model of generate().models) {
|
|
const source = catalog.models.find((entry) => entry.repository === model.id)!;
|
|
expect(model).toMatchObject({
|
|
revision: source.revision,
|
|
name: source.name,
|
|
...source.files[source.quant],
|
|
supportsStreaming: source.capabilities.streaming,
|
|
supportsLanguageDetection: source.capabilities.languageDetection,
|
|
supportsTranslation: source.capabilities.translate,
|
|
});
|
|
}
|
|
const parakeet = catalog.models[0]!;
|
|
const withoutFrenchOrSpeed = generateSpeechCatalog(
|
|
{
|
|
...catalog.data,
|
|
models: [{ ...parakeet, languages: ["en", "fr"] }],
|
|
results: { [parakeet.id]: { accuracy: { en: { error: 3.99 } }, performance: {} } },
|
|
},
|
|
"0.2.3",
|
|
"0.4.1",
|
|
);
|
|
expect(withoutFrenchOrSpeed.models[0]?.benchmark).toEqual({
|
|
accuracy: { en: { grade: "A-", errorPercent: 3.99 } },
|
|
secondsPer30s: { cpu: null, accelerated: null },
|
|
});
|
|
expect(generate().models[0]?.benchmark.secondsPer30s.accelerated).toBeCloseTo(1.19, 2);
|
|
for (const performance of [{ cpu: 2 }, { accelerated: 10 }, { cpu: 2, accelerated: 10 }]) {
|
|
const result = generateSpeechCatalog(
|
|
{
|
|
...catalog.data,
|
|
models: [parakeet],
|
|
results: { [parakeet.id]: { accuracy: { en: { error: 3.99 } }, performance } },
|
|
},
|
|
"0.2.3",
|
|
"0.4.1",
|
|
);
|
|
expect(result.models[0]?.benchmark.secondsPer30s).toEqual({
|
|
cpu: performance.cpu ? 30 / performance.cpu : null,
|
|
accelerated: performance.accelerated ? 30 / performance.accelerated : null,
|
|
});
|
|
}
|
|
});
|
|
|
|
it("preserves Handy's accelerated balanced pick after filtering Hanzo's offered candidates", () => {
|
|
const offered = generate().models;
|
|
const sources = catalog.models.filter((model) =>
|
|
offered.some(({ id }) => id === model.repository),
|
|
);
|
|
for (const languages of [["en"], ["de"], ["ja"], ["en", "de"], ["en", "fr"], ["hi", "mr"]]) {
|
|
const expected = catalog
|
|
.recommendModels(sources, languages)
|
|
.find((pick) => pick.status !== "unbenchmarked" && pick.roles.includes("best"));
|
|
expect(
|
|
recommendSpeechModels(offered, languages, "accelerated").keys().next().value,
|
|
languages.join(","),
|
|
).toBe(expected?.model.repository);
|
|
}
|
|
const excluded = { ...catalog.models[0]!, license: "cc-by-nc-4.0" };
|
|
const allowed = { ...catalog.models[1]!, languages: ["en"] };
|
|
const filtered = generate([excluded, allowed]).models;
|
|
expect([...recommendSpeechModels(filtered, ["en"], "accelerated").keys()]).toEqual([
|
|
allowed.repository,
|
|
]);
|
|
});
|
|
|
|
it("filters restricted licenses, unsupported roles, engines, architectures and files", () => {
|
|
const model = catalog.models[0]!;
|
|
for (const override of [
|
|
{ role: "langid" as const },
|
|
{ role: "vad" as const },
|
|
{ role: "diarize" as const },
|
|
{ license: "cc-by-nc-4.0" },
|
|
{ license: "CC-BY-NC-SA-4.0" },
|
|
{ license: "cc-by-nc-nd-3.0" },
|
|
{ license: "cc-by-nc" },
|
|
{ license: "CC BY-NC 4.0" },
|
|
{ license: "custom_NC" },
|
|
{ minEngine: "0.4.2" },
|
|
{ minEngine: "0.5.0" },
|
|
{ family: "future_arch" },
|
|
...[
|
|
"nested/model.gguf",
|
|
"nested\\model.gguf",
|
|
"../model.gguf",
|
|
"model.bin",
|
|
"model.safetensors",
|
|
].map((filename) => ({ files: { [model.quant]: { ...model.files[model.quant]!, filename } } })),
|
|
]) {
|
|
expect(generate([{ ...model, ...override }]).models).toEqual([]);
|
|
}
|
|
for (const license of ["other", "apache-2.0", "cc-by-4.0", "cc-by-sa-4.0", "unlicense"]) {
|
|
expect(generate([{ ...model, license, minEngine: "0.4.1" }]).models).toHaveLength(1);
|
|
}
|
|
const moonshine = catalog.models.find((entry) => entry.id === "moonshine-tiny-ja")!;
|
|
for (const [id, allowed] of [
|
|
["moonshine-tiny-ja", false],
|
|
["moonshine-base-vi", false],
|
|
["moonshine-base-es", false],
|
|
["moonshine-base", true],
|
|
["moonshine-streaming-tiny-ja", true],
|
|
] as const) {
|
|
const entry = { ...moonshine, id, repository: `handy-computer/${id}-gguf`, license: "mit" };
|
|
expect(generate([entry]).models, id).toHaveLength(allowed ? 1 : 0);
|
|
}
|
|
});
|
|
|
|
it("rejects missing defaults, unpinned or corrupt metadata, collisions, and unreviewed engine/schema changes", () => {
|
|
const model = catalog.models[0]!;
|
|
const file = model.files[model.quant]!;
|
|
for (const override of [
|
|
{ files: {} },
|
|
{ revision: "main" },
|
|
{ repository: "../model" },
|
|
{ languages: [] },
|
|
...[{ sha256: "bad" }, { size: 0 }, { size: 1.5 }].map((entry) => ({
|
|
files: { [model.quant]: { ...file, ...entry } },
|
|
})),
|
|
]) {
|
|
expect(() => generate([{ ...model, ...override }])).toThrow();
|
|
}
|
|
expect(() => generate([model, model])).toThrow(/Duplicate/);
|
|
expect(() => generate([model, { ...model, repository: "handy-computer/another" }])).toThrow(
|
|
/Duplicate/,
|
|
);
|
|
expect(() => generateSpeechCatalog(catalog.data, "0.2.3", "0.4.2")).toThrow(/loader support/);
|
|
expect(() =>
|
|
generateSpeechCatalog({ ...catalog.data, schema: 3 as 2 }, "0.2.3", "0.4.1"),
|
|
).toThrow(/schema/);
|
|
});
|