mirror of
https://github.com/hanzo-apps/hanzo.ai.git
synced 2026-10-29 07:20:43 +00:00
zeekay
7a66fd03aa
Two of Kai's newest results are not research-API runs, so the page could not carry them: the tool shortlist (one run on Hanzo's internal tool catalog, dev partition, 2026-10-11) and kai-1.2's yes/no reads beside Jev's on the same questions and rows (2026-10-08 and 2026-10-06). Each is now a data file copied unchanged from its run's report, read by name in lib/bench-data under the pages' wins rule: a shortlist figure shows only where it beats BM25, a yes/no test only where Kai leads Jev on every half of it. So the Workbench pair (Jev holds the counterfactual), the predicate matrix and the tied sets stay off; tally's tie is exported for the pooled accuracy rather than typed twice. The shortlist note says what it is: an internal dev set, one run, retrieval only, and that a zero-shot reranker made the ranking worse. e2e/unit/reads.spec.ts holds both files to the figures their runs recorded and the drawn rows to the files; the benchmarks gate checks the export renders them.
204 lines
4.6 KiB
JSON
204 lines
4.6 KiB
JSON
{
|
|
"generated": "2026-10-11",
|
|
"measured": "2026-10-11",
|
|
"about": "One run of a tool shortlist over Hanzo's internal tool catalog, dev partition: for each request, the k catalog tools an embedding model and a BM25 term rank first, scored against BM25 alone. Retrieval only, nothing trained. Copied unchanged from the run's report (hanzo-inc/kai train/scripts/dense_shortlist.py --report, commit d26ded1); the reranker figures are the same run's 1,217-request sample.",
|
|
"set": "dev",
|
|
"public": false,
|
|
"embedder": "zen-embedding-0.6B",
|
|
"model": "Qwen/Qwen3-Embedding-0.6B",
|
|
"bm25_weight": 0.1,
|
|
"k": 32,
|
|
"ndcg_k": 10,
|
|
"catalog": 2928,
|
|
"queries": 6608,
|
|
"groups": {
|
|
"all": {
|
|
"n": 6608,
|
|
"bm25": {
|
|
"any": 0.7356234866828087,
|
|
"all": 0.6915859564164649,
|
|
"top1": 0.4138922518159806,
|
|
"ndcg10": 0.5157563072160748
|
|
},
|
|
"shortlist": {
|
|
"any": 0.9355326876513317,
|
|
"all": 0.9185835351089588,
|
|
"top1": 0.5549334140435835,
|
|
"ndcg10": 0.7230150553969997
|
|
}
|
|
},
|
|
"apibank/tool": {
|
|
"n": 281,
|
|
"bm25": {
|
|
"any": 0.8398576512455516,
|
|
"all": 0.8398576512455516,
|
|
"top1": 0.5836298932384342,
|
|
"ndcg10": 0.6777931586984294
|
|
},
|
|
"shortlist": {
|
|
"any": 0.9145907473309609,
|
|
"all": 0.9145907473309609,
|
|
"top1": 0.7224199288256228,
|
|
"ndcg10": 0.799590432991736
|
|
}
|
|
},
|
|
"glaive/tool": {
|
|
"n": 773,
|
|
"bm25": {
|
|
"any": 0.49417852522639066,
|
|
"all": 0.49417852522639066,
|
|
"top1": 0.09184993531694696,
|
|
"ndcg10": 0.19787884423517488
|
|
},
|
|
"shortlist": {
|
|
"any": 0.6701164294954722,
|
|
"all": 0.6701164294954722,
|
|
"top1": 0.13195342820181113,
|
|
"ndcg10": 0.25207521925505244
|
|
}
|
|
},
|
|
"hanzo/domain": {
|
|
"n": 670,
|
|
"bm25": {
|
|
"any": 0.15522388059701492,
|
|
"all": 0.15522388059701492,
|
|
"top1": 0.0,
|
|
"ndcg10": 0.03476401622163185
|
|
},
|
|
"shortlist": {
|
|
"any": 0.9985074626865672,
|
|
"all": 0.9985074626865672,
|
|
"top1": 0.5104477611940299,
|
|
"ndcg10": 0.8145124940024002
|
|
}
|
|
},
|
|
"hanzo/op": {
|
|
"n": 670,
|
|
"bm25": {
|
|
"any": 0.9253731343283582,
|
|
"all": 0.9253731343283582,
|
|
"top1": 0.5880597014925373,
|
|
"ndcg10": 0.7342856947141974
|
|
},
|
|
"shortlist": {
|
|
"any": 0.9955223880597015,
|
|
"all": 0.9955223880597015,
|
|
"top1": 0.382089552238806,
|
|
"ndcg10": 0.7261068271886492
|
|
}
|
|
},
|
|
"hermes/tool": {
|
|
"n": 45,
|
|
"bm25": {
|
|
"any": 1.0,
|
|
"all": 1.0,
|
|
"top1": 0.8666666666666667,
|
|
"ndcg10": 0.9363522056679319
|
|
},
|
|
"shortlist": {
|
|
"any": 1.0,
|
|
"all": 1.0,
|
|
"top1": 0.9333333333333333,
|
|
"ndcg10": 0.9724857668253981
|
|
}
|
|
},
|
|
"pivot/tool": {
|
|
"n": 315,
|
|
"bm25": {
|
|
"any": 0.5142857142857142,
|
|
"all": 0.4888888888888889,
|
|
"top1": 0.14285714285714285,
|
|
"ndcg10": 0.2623293143213184
|
|
},
|
|
"shortlist": {
|
|
"any": 0.7841269841269841,
|
|
"all": 0.7682539682539683,
|
|
"top1": 0.2634920634920635,
|
|
"ndcg10": 0.46487733093457756
|
|
}
|
|
},
|
|
"toolace/tool": {
|
|
"n": 493,
|
|
"bm25": {
|
|
"any": 0.920892494929006,
|
|
"all": 0.9168356997971603,
|
|
"top1": 0.6713995943204868,
|
|
"ndcg10": 0.7641459763767033
|
|
},
|
|
"shortlist": {
|
|
"any": 0.9939148073022313,
|
|
"all": 0.9939148073022313,
|
|
"top1": 0.8904665314401623,
|
|
"ndcg10": 0.9397446265715074
|
|
}
|
|
},
|
|
"toolret/tool": {
|
|
"n": 534,
|
|
"bm25": {
|
|
"any": 0.9213483146067416,
|
|
"all": 0.846441947565543,
|
|
"top1": 0.6441947565543071,
|
|
"ndcg10": 0.7267848743239644
|
|
},
|
|
"shortlist": {
|
|
"any": 0.9943820224719101,
|
|
"all": 0.9569288389513109,
|
|
"top1": 0.8164794007490637,
|
|
"ndcg10": 0.8894977981467306
|
|
}
|
|
},
|
|
"toucan/tool": {
|
|
"n": 1511,
|
|
"bm25": {
|
|
"any": 0.7392455327597618,
|
|
"all": 0.5810721376571807,
|
|
"top1": 0.29119788219722037,
|
|
"ndcg10": 0.3972368924335187
|
|
},
|
|
"shortlist": {
|
|
"any": 0.9596293845135672,
|
|
"all": 0.9027134348113832,
|
|
"top1": 0.45400397088021177,
|
|
"ndcg10": 0.660198138064596
|
|
}
|
|
},
|
|
"when2call/tool": {
|
|
"n": 575,
|
|
"bm25": {
|
|
"any": 0.9565217391304348,
|
|
"all": 0.9565217391304348,
|
|
"top1": 0.6869565217391305,
|
|
"ndcg10": 0.8127209624470496
|
|
},
|
|
"shortlist": {
|
|
"any": 0.9860869565217392,
|
|
"all": 0.9860869565217392,
|
|
"top1": 0.7756521739130435,
|
|
"ndcg10": 0.8847412746563103
|
|
}
|
|
},
|
|
"xlam/tool": {
|
|
"n": 741,
|
|
"bm25": {
|
|
"any": 0.9433198380566802,
|
|
"all": 0.9406207827260459,
|
|
"top1": 0.6909581646423751,
|
|
"ndcg10": 0.7993225063235152
|
|
},
|
|
"shortlist": {
|
|
"any": 1.0,
|
|
"all": 0.9986504723346828,
|
|
"top1": 0.8529014844804319,
|
|
"ndcg10": 0.9327391650551753
|
|
}
|
|
}
|
|
},
|
|
"reranker": {
|
|
"n": 1217,
|
|
"metric": "ndcg10",
|
|
"candidates": 32,
|
|
"reranked": 0.6914111482515909,
|
|
"embedder": 0.7182230402366315
|
|
}
|
|
}
|