Files
laya/laya-ts/tests/answer-confidence.test.ts
Melvin 3eb5039b8f feat(laya-ts): report answer_confidence (max(p)) on every answer type
Parity with the Python side (#126): every decoded answer now carries
answer_confidence alongside the existing confidence, so callers can gate
across question types on a single calibrated number instead of the
entropy-based confidence, whose scale moves with the option count k.
Identical to confidence for noul (max(p) over two options); documented
divergence encoded in tests.
2026-09-24 20:31:00 +08:00

57 lines
2.3 KiB
TypeScript

import { describe, expect, it } from "vitest";
import { answerConfidence, confidenceFromProbs } from "../src/common.js";
import { Agent } from "../src/agent.js";
const fakeProvider = (n: number) => ({
// n logits with a known argmax at 0; softmax gives a fixed, checkable distribution
async runEncoder(_b: any) { return { lastHidden: [[1, 0], [0, 1]] }; },
async runHead(_h: any) {
const row = Array.from({ length: n }, (_, i) => (i === 0 ? 2 : 0));
return { logits: [row], act: [[3, 0]] };
},
});
function probs(provider: any, a: Agent) { return a as any; }
describe("answerConfidence", () => {
it("is max(p) on the reported answer", () => {
expect(answerConfidence([0.25, 0.6, 0.15])).toBeCloseTo(0.6);
expect(answerConfidence([1.0])).toBe(1.0);
});
it("returns 1.0 for an empty distribution (Python parity: k < 1)", () => {
expect(answerConfidence([])).toBe(1.0);
});
it("stays at the same value across option counts, unlike the entropy confidence", () => {
// The #361 table: same top probability 0.9, different label-set sizes.
const fill = (k: number) => {
const rest = Array.from({ length: k - 1 }, () => 0.1 / (k - 1));
return [0.9, ...rest];
};
for (const k of [2, 5, 20, 77]) {
expect(answerConfidence(fill(k))).toBeCloseTo(0.9, 5);
}
// entropy confidence rises with k for the same max(p)
expect(confidenceFromProbs(fill(2))).toBeLessThan(confidenceFromProbs(fill(20)));
});
it("is attached to every answer type by the agent decode", async () => {
const mk = (n: number) => new Agent({ provider: fakeProvider(n) } as any);
const choice: any = await mk(3).systemOne("s", {
c: { type: "choice", instructions: "q?", criteria: { a: "x", b: "y", c: "z" } },
});
const top = Math.max(...Object.values(choice.answers.c.probabilities).map(Number));
expect(choice.answers.c.answer_confidence).toBeCloseTo(top, 4);
const score: any = await mk(3).systemOne("s", {
s: { type: "score", instructions: "q?", criteria: ["lo", "mid", "hi"] },
});
expect(score.answers.s.answer_confidence).toBeCloseTo(top, 4);
const noul: any = await mk(2).systemOne("s", { n: { type: "noul", instructions: "q?" } });
// over two options max(p) equals the existing noul confidence
expect(noul.answers.n.answer_confidence).toBeCloseTo(noul.answers.n.confidence, 4);
});
});