mirror of
https://github.com/hanzoai/kai.git
synced 2026-10-17 11:18:06 +00:00
zooqueenandHanzo Dev
b00044ece2
Every runner printed whatever came back and exited 0, and fell back to a second raw HTTP call when the SDK import failed. Now each recipe carries expect.json, the answers a careful reader gives on its own input: a list of acceptable options for a choice, of level indices for a score, true or false for a noul. run.py and run.ts send request.json through the SDK, print every answer, and exit 1 when a checked one is wrong. A question with no obvious answer on its input is printed and not checked. Recipes 09 and 11 had no input with an obvious answer, so each state gains the one fact that decides it. The recipe READMEs describe what each recipe asks instead of capabilities it never exercised, and copy the request and checked answers from the two JSON files. Co-authored-by: Hanzo Dev <[email protected]>
47 lines
2.1 KiB
TypeScript
47 lines
2.1 KiB
TypeScript
// Send request.json to Kai with the hanzoai SDK and check each answer against expect.json.
|
|
import { readFileSync } from 'node:fs';
|
|
import { AiApi, Configuration } from 'hanzoai';
|
|
|
|
const read = (name: string) => JSON.parse(readFileSync(new URL(name, import.meta.url), 'utf8'));
|
|
const body = read('request.json');
|
|
body.model = process.env.KAI_MODEL || body.model;
|
|
const expect: Record<string, boolean | Array<string | number>> = read('expect.json');
|
|
|
|
const ai = new AiApi(new Configuration({ accessToken: process.env.HANZO_API_KEY }));
|
|
// The SDK's HTTP error carries the request headers, the API key among them, so keep only status and body.
|
|
const { data } = await ai.postDecisions({ aiDecisionsRequest: body }).catch((e) => {
|
|
throw new Error(e.response ? `HTTP ${e.response.status} ${JSON.stringify(e.response.data)}` : e.message);
|
|
});
|
|
console.log(`${data.id} ${data.model}: ${data.usage.input_tokens} input tokens, ${Math.round(data.latency_ms)} ms`);
|
|
|
|
let wrong = 0;
|
|
for (const name of Object.keys(expect).filter((n) => !(n in data.answers))) {
|
|
wrong++;
|
|
console.log(` FAIL ${name}: expected an answer, got none`);
|
|
}
|
|
for (const [name, a] of Object.entries(data.answers)) {
|
|
let got: boolean | string | number;
|
|
let said: string;
|
|
if (a.type === 'noul') {
|
|
got = a.noul! > 0.5;
|
|
said = `P(true) ${a.noul!.toFixed(3)}`;
|
|
} else if (a.type === 'score') {
|
|
const [top, p] = Object.entries(a.probabilities!).sort((x, y) => y[1] - x[1])[0];
|
|
got = Number(top);
|
|
said = `level ${top} '${a.legend![top]}' at ${p.toFixed(3)} (score ${a.score!.toFixed(2)})`;
|
|
} else {
|
|
got = a.choice!;
|
|
said = `${a.choice} at ${a.answer_confidence!.toFixed(3)}`;
|
|
}
|
|
const want = expect[name];
|
|
if (want === undefined) {
|
|
console.log(` ---- ${name}: ${said} (not checked: no obvious answer on this input)`);
|
|
} else if (Array.isArray(want) ? want.includes(got as string | number) : got === want) {
|
|
console.log(` pass ${name}: ${said}`);
|
|
} else {
|
|
wrong++;
|
|
console.log(` FAIL ${name}: ${said}, expected ${JSON.stringify(want)}`);
|
|
}
|
|
}
|
|
process.exitCode = wrong ? 1 : 0;
|