mirror of
https://github.com/hanzo-apps/hanzo.ai.git
synced 2026-10-28 07:32:52 +00:00
zeekay
7a66fd03aa
Two of Kai's newest results are not research-API runs, so the page could not carry them: the tool shortlist (one run on Hanzo's internal tool catalog, dev partition, 2026-10-11) and kai-1.2's yes/no reads beside Jev's on the same questions and rows (2026-10-08 and 2026-10-06). Each is now a data file copied unchanged from its run's report, read by name in lib/bench-data under the pages' wins rule: a shortlist figure shows only where it beats BM25, a yes/no test only where Kai leads Jev on every half of it. So the Workbench pair (Jev holds the counterfactual), the predicate matrix and the tied sets stay off; tally's tie is exported for the pooled accuracy rather than typed twice. The shortlist note says what it is: an internal dev set, one run, retrieval only, and that a zero-shot reranker made the ranking worse. e2e/unit/reads.spec.ts holds both files to the figures their runs recorded and the drawn rows to the files; the benchmarks gate checks the export renders them.
112 lines
3.6 KiB
JSON
112 lines
3.6 KiB
JSON
{
|
|
"generated": "2026-10-11",
|
|
"about": "Yes/no questions answered by the production Kai (kai-1.2: kai-1 with its yes/no reader) and by Jev on the same template sets and the same validation rows. Kai through kai serve on CPU over the release folder, 2026-10-08; Jev typesafe/jev-1.13 through api.hanzo.ai, 2026-10-06. Metrics copied unchanged from each read's served.json (hanzo-inc/kai results/bernoulli/reader12/Z2, results/bernoulli/jev).",
|
|
"kai": {
|
|
"model": "kai-1.2",
|
|
"pin": "e45dd919623b83f01595b6dffbcd63315adeab70bfb7dd5d6a942380b320d575",
|
|
"device": "cpu",
|
|
"measured": "2026-10-08",
|
|
"metrics": {
|
|
"critical.yes": 0.8734610296764017,
|
|
"critical.no": 0.13493492623640885,
|
|
"fields.enterprise": 1.0,
|
|
"fields.seats": 1.0,
|
|
"fields.verified": 1.0,
|
|
"fields.language": 1.0,
|
|
"negation.right": 1.0,
|
|
"paraphrase": 0.12443293641855188,
|
|
"absent.within": 0.8166666666666667,
|
|
"matrix.in_band": 40.0,
|
|
"matrix.ordered": 7.0,
|
|
"unknown.mean": 0.4924252931873772,
|
|
"unknown.band": 0.9333333333333333,
|
|
"retained.pooled": 0.8832807570977917,
|
|
"retained.answer": 0.8533333333333334,
|
|
"retained.answer.rows": 150.0,
|
|
"retained.credential_compromise": 0.8235294117647058,
|
|
"retained.credential_compromise.rows": 17.0,
|
|
"retained.is_phishing": 0.9866666666666667,
|
|
"retained.is_phishing.rows": 150.0,
|
|
"retained.is_spam": 1.0,
|
|
"retained.is_spam.rows": 150.0,
|
|
"retained.jailbreak": 0.9866666666666667,
|
|
"retained.jailbreak.rows": 150.0,
|
|
"retained.needs_human": 0.8823529411764706,
|
|
"retained.needs_human.rows": 17.0,
|
|
"retained.relevant": 0.5666666666666667,
|
|
"retained.relevant.rows": 150.0,
|
|
"retained.toxic": 0.9333333333333333,
|
|
"retained.toxic.rows": 150.0,
|
|
"retained.true_positive": 0.7058823529411765,
|
|
"retained.true_positive.rows": 17.0,
|
|
"golden.critical": 0.8734610296764017,
|
|
"golden.counter": 0.13493492623640885,
|
|
"golden.primitive": 1.0,
|
|
"golden.team": 1.0,
|
|
"golden.base": 1.0,
|
|
"isolation.requests": 261.0,
|
|
"isolation.moved": 0.0,
|
|
"latency.p50_ms": 50.68598,
|
|
"latency.p95_ms": 57.196452
|
|
}
|
|
},
|
|
"jev": {
|
|
"model": "typesafe/jev-1.13",
|
|
"measured": "2026-10-06",
|
|
"metrics": {
|
|
"critical.yes": 0.78,
|
|
"critical.no": 0.09,
|
|
"fields.enterprise": 1.0,
|
|
"fields.seats": 1.0,
|
|
"fields.verified": 1.0,
|
|
"fields.language": 1.0,
|
|
"negation.right": 1.0,
|
|
"paraphrase": 0.151,
|
|
"absent.within": 0.38333333333333336,
|
|
"matrix.in_band": 40,
|
|
"matrix.ordered": 8,
|
|
"unknown.mean": 0.36533333333333334,
|
|
"unknown.band": 0.4666666666666667,
|
|
"retained.pooled": 0.8748685594111462,
|
|
"retained.is_spam": 0.98,
|
|
"retained.is_spam.rows": 150,
|
|
"retained.is_phishing": 0.8933333333333333,
|
|
"retained.is_phishing.rows": 150,
|
|
"retained.toxic": 0.9133333333333333,
|
|
"retained.toxic.rows": 150,
|
|
"retained.jailbreak": 0.9866666666666667,
|
|
"retained.jailbreak.rows": 150,
|
|
"retained.relevant": 0.6133333333333333,
|
|
"retained.relevant.rows": 150,
|
|
"retained.answer": 0.9466666666666667,
|
|
"retained.answer.rows": 150,
|
|
"retained.credential_compromise": 0.6470588235294118,
|
|
"retained.credential_compromise.rows": 17,
|
|
"retained.needs_human": 0.5294117647058824,
|
|
"retained.needs_human.rows": 17,
|
|
"retained.true_positive": 0.7058823529411765,
|
|
"retained.true_positive.rows": 17
|
|
}
|
|
},
|
|
"rows": "d953e624e14c9d8e26329102d94ddb18a93a36391b8846d856b626622d2f58a4",
|
|
"n": {
|
|
"critical": 2,
|
|
"paraphrase": 20,
|
|
"absent": 60,
|
|
"negation": 20,
|
|
"fields": {
|
|
"enterprise": 60,
|
|
"seats": 60,
|
|
"verified": 20,
|
|
"language": 25
|
|
},
|
|
"matrix": 48,
|
|
"states": 8,
|
|
"retained": 951
|
|
},
|
|
"band": [
|
|
0.35,
|
|
0.65
|
|
]
|
|
}
|