hanzoai/hanzo.ai
2
0
Fork 0
mirror of https://github.com/hanzo-apps/hanzo.ai.git synced 2026-10-28 07:32:52 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
hanzo.ai/lib/data/benchmarks.json
T
zeekayandHanzo Dev db2525dfd3 A page per benchmark, and the walk arm that proves it reads the graph 
/benchmarks was one URL covering ten benchmarks, which is one page ranking for
none of them. Somebody searching MemoryAgentBench wants the page about
MemoryAgentBench. Each now has its own route, canonical URL, Dataset and
BreadcrumbList, and a sitemap entry whose lastModified is the `generated` stamp
its run file carries — the date the runs were rolled up, not the date somebody
edited the prose beside them. Where a bench has no stamped run it is offered no
date, because no date is the true answer and a build timestamp would be a
plausible-looking lie that changes on every deploy.

The registry has carried `sandbox` since the directory was built and the route
never existed, so the index and the footer of every other bench page linked to a
404. It exists, and it leads with the row it would rather not: a V8 context
starts in 0.15 ms and cannot run pytest, pip, cargo or a shell.

The boards vary one thing each — the reader with retrieval held, retrieval with
the reader held, the embedder, the unit of memory — and the heading no longer
counts them, because it counted four and rendered five.

The Go harness is no longer transcribed. `node scripts/locomo.mjs` runs it and
parses what it prints into lib/data/benchmarks-semantic.json, with the tag, the
commit, the dataset digest, and the exact command beside every table. That
caught a published instruction that does not run: the 2x2 was captioned
`-factor`, which is not a flag. It also carries the v0.3.0 walk arm — following
the graph's relations scores 0.032 on multi-hop against 0.020 and 0.023, and a
reader that states every piece of evidence it was handed moves the oracle
ceiling 0.068 -> 0.119, so the ceiling moved further than the retrieval did.

The control is the part worth having. `-arms walk -blind` empties every graph
and changes nothing else; the capture compares the two cell by cell and records
which arms survived. Five of seven come back byte-identical and two collapse to
constant abstention. The site used to assert graph inertness in prose. It now
reports the outcome of a comparison, and the sentence cannot outlive the result.

The Metal prefill row was quoted from a warm, unalternated pass at 0.91x. A
cooled, order-alternated pass on the same accelerator reads 0.8645 across three
rounds, so 13.6% behind is the figure, not 9%. It is a different quantization
and sits in its own table rather than replacing a cell.

Co-authored-by: Hanzo Dev <[email protected]>
2026-09-10 23:37:24 -07:00

13876 lines
326 KiB
JSON
Raw Permalink Blame History

{
"generated": "2026-09-11T03:22:43.895Z",
"sections": [
{
"bench": "conv",
"run": "conv-proxy-naive-rag-all-minilm-k10",
"row": null,
"split": null,
"k": 10,
"reader": null,
"facts": null,
"commit": null,
"record": {
"commit": null,
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": null,
"facts_digest": null,
"reader": null,
"embedding": "all-minilm",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:09:17.510Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "original",
"n": 1977,
"metrics": {
"recall": 0.42857701260337777,
"all": 0.3899848254931715,
"any": 0.47749114820435,
"tokens": 261.7020738492666
}
},
{
"name": "original·cat2",
"n": 320,
"metrics": {
"recall": 0.4940104166666666,
"all": 0.4625,
"any": 0.525,
"tokens": 253.2875
}
},
{
"name": "original·no-adversarial",
"n": 1531,
"metrics": {
"recall": 0.45479866356425724,
"all": 0.4069235793598955,
"any": 0.5160026126714565,
"tokens": 265.0979751796212
}
},
{
"name": "original·cat3",
"n": 89,
"metrics": {
"recall": 0.26255783212161266,
"all": 0.16853932584269662,
"any": 0.39325842696629215,
"tokens": 250.4494382022472
}
},
{
"name": "original·cat1",
"n": 281,
"metrics": {
"recall": 0.26694818573447526,
"all": 0.09608540925266904,
"any": 0.498220640569395,
"tokens": 270.4661921708185
}
},
{
"name": "original·cat4",
"n": 841,
"metrics": {
"recall": 0.5229885057471265,
"all": 0.5148632580261593,
"any": 0.5315101070154578,
"tokens": 269.3483947681332
}
},
{
"name": "original·cat5",
"n": 446,
"metrics": {
"recall": 0.33856502242152464,
"all": 0.33183856502242154,
"any": 0.3452914798206278,
"tokens": 250.04484304932734
}
}
],
"table": null
},
{
"bench": "conv",
"run": "conv-proxy-naive-rag-st-minilm-k10",
"row": null,
"split": null,
"k": 10,
"reader": null,
"facts": null,
"commit": null,
"record": {
"commit": null,
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": null,
"facts_digest": null,
"reader": null,
"embedding": "st-minilm",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:07:58.770Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "original",
"n": 1977,
"metrics": {
"recall": 0.428956375274091,
"all": 0.3899848254931715,
"any": 0.47850278199291857,
"tokens": 261.79868487607484
}
},
{
"name": "original·cat2",
"n": 320,
"metrics": {
"recall": 0.4940104166666666,
"all": 0.4625,
"any": 0.525,
"tokens": 253.409375
}
},
{
"name": "original·no-adversarial",
"n": 1531,
"metrics": {
"recall": 0.45528853946236303,
"all": 0.4069235793598955,
"any": 0.5173089483997387,
"tokens": 265.18419333768776
}
},
{
"name": "original·cat3",
"n": 89,
"metrics": {
"recall": 0.26255783212161266,
"all": 0.16853932584269662,
"any": 0.39325842696629215,
"tokens": 250.80898876404495
}
},
{
"name": "original·cat1",
"n": 281,
"metrics": {
"recall": 0.26961722488038276,
"all": 0.09608540925266904,
"any": 0.505338078291815,
"tokens": 270.4199288256228
}
},
{
"name": "original·cat4",
"n": 841,
"metrics": {
"recall": 0.5229885057471265,
"all": 0.5148632580261593,
"any": 0.5315101070154578,
"tokens": 269.436385255648
}
},
{
"name": "original·cat5",
"n": 446,
"metrics": {
"recall": 0.33856502242152464,
"all": 0.33183856502242154,
"any": 0.3452914798206278,
"tokens": 250.17713004484304
}
}
],
"table": null
},
{
"bench": "conv",
"run": "conv-proxy-naive-rag-zenlm_zen-embedding-0.6b-k10",
"row": null,
"split": null,
"k": 10,
"reader": null,
"facts": null,
"commit": null,
"record": {
"commit": null,
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": null,
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:28:50.937Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "original",
"n": 1977,
"metrics": {
"recall": 0.5965582525541794,
"all": 0.548811330298432,
"any": 0.656044511886697,
"tokens": 315.3419322205362
}
},
{
"name": "original·cat2",
"n": 320,
"metrics": {
"recall": 0.7395833333333333,
"all": 0.709375,
"any": 0.76875,
"tokens": 331.9625
}
},
{
"name": "original·no-adversarial",
"n": 1531,
"metrics": {
"recall": 0.6351375998037967,
"all": 0.5760940561724364,
"any": 0.7093403004572175,
"tokens": 319.14369693011105
}
},
{
"name": "original·cat3",
"n": 89,
"metrics": {
"recall": 0.4044943820224719,
"all": 0.33707865168539325,
"any": 0.4943820224719101,
"tokens": 286.1573033707865
}
},
{
"name": "original·cat1",
"n": 281,
"metrics": {
"recall": 0.38100474009352087,
"all": 0.1494661921708185,
"any": 0.6868327402135231,
"tokens": 311.4555160142349
}
},
{
"name": "original·cat4",
"n": 841,
"metrics": {
"recall": 0.7047166072136345,
"all": 0.6932223543400713,
"any": 0.7170035671819263,
"tokens": 320.3258026159334
}
},
{
"name": "original·cat5",
"n": 446,
"metrics": {
"recall": 0.4641255605381166,
"all": 0.4551569506726457,
"any": 0.4730941704035874,
"tokens": 302.2914798206278
}
}
],
"table": null
},
{
"bench": "conv",
"run": "conv-proxy-units-st-minilm-k10",
"row": null,
"split": null,
"k": null,
"reader": null,
"facts": null,
"commit": null,
"record": {
"commit": null,
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": null,
"facts_digest": null,
"reader": null,
"embedding": null,
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": null,
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "turn·all",
"n": 1982,
"metrics": {
"recall": 0.4276940519978481,
"tokens": 279.2228809283552,
"units_per_conv": 568
}
},
{
"name": "turn·no-adversarial",
"n": 1536,
"metrics": {
"recall": 0.4535739655336816,
"tokens": 282.6170247395833,
"units_per_conv": 568
}
},
{
"name": "w2·all",
"n": 1982,
"metrics": {
"recall": 0.5630920252505619,
"tokens": 570.5774470232088,
"units_per_conv": 568
}
},
{
"name": "w2·no-adversarial",
"n": 1536,
"metrics": {
"recall": 0.5433257773740975,
"tokens": 566.5851236979166,
"units_per_conv": 568
}
},
{
"name": "w3·all",
"n": 1982,
"metrics": {
"recall": 0.6248028692274583,
"tokens": 828.5285065590313,
"units_per_conv": 568
}
},
{
"name": "w3·no-adversarial",
"n": 1536,
"metrics": {
"recall": 0.6066792231828271,
"tokens": 823.1243489583334,
"units_per_conv": 568
}
},
{
"name": "w5·all",
"n": 1982,
"metrics": {
"recall": 0.602606916652366,
"tokens": 1258.3917759838546,
"units_per_conv": 568
}
},
{
"name": "w5·no-adversarial",
"n": 1536,
"metrics": {
"recall": 0.5816190812532482,
"tokens": 1240.3352864583333,
"units_per_conv": 568
}
},
{
"name": "session·all",
"n": 1982,
"metrics": {
"recall": 0.6208879701195605,
"tokens": 6854.727674066599,
"units_per_conv": 30
}
},
{
"name": "session·no-adversarial",
"n": 1536,
"metrics": {
"recall": 0.617578096860006,
"tokens": 6847.644694010417,
"units_per_conv": 30
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-cer-k20-enso-flash",
"row": null,
"split": null,
"k": 20,
"reader": "enso-flash",
"facts": null,
"commit": "885a21a63c3254415e40555f7905c8c04aa0be65",
"record": {
"commit": "885a21a63c3254415e40555f7905c8c04aa0be65",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "enso-flash",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 282,
"answered": 282,
"started": "2026-09-09T09:11:57.018Z",
"ended": "2026-09-10T01:18:55.312Z",
"completion": "complete",
"wall_seconds": 55895,
"tokens_per_question": 968,
"reader_tokens": 99823
},
"categories": [
{
"name": "all",
"n": 282,
"metrics": {
"of": 1536,
"f1": {
"mean": 0.42489476674409804,
"lo": 0.38122477595934917,
"hi": 0.4646630704046311
},
"em": {
"mean": 0.11702127659574468,
"lo": 0.08156028368794327,
"hi": 0.1524822695035461
},
"exact": 0.8723404255319149,
"supported": 0.1099290780141844,
"answered": 0.450354609929078,
"tokens": 968.2588652482269
}
},
{
"name": "all:multi-hop",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.42489476674409804,
"lo": 0.38122477595934917,
"hi": 0.4646630704046311
},
"em": {
"mean": 0.11702127659574468,
"lo": 0.08156028368794327,
"hi": 0.1524822695035461
},
"exact": 0.8723404255319149,
"supported": 0.1099290780141844,
"answered": 0.450354609929078,
"tokens": 968.2588652482269
}
},
{
"name": "dev",
"n": 74,
"metrics": {
"of": 383,
"f1": {
"mean": 0.49631866340677494,
"lo": 0.4127368877368879,
"hi": 0.5770937859173152
},
"em": {
"mean": 0.14864864864864866,
"lo": 0.06756756756756757,
"hi": 0.24324324324324326
},
"exact": 0.8648648648648649,
"supported": 0.10810810810810811,
"answered": 0.581081081081081,
"tokens": 963.9324324324324
}
},
{
"name": "dev:multi-hop",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.49631866340677494,
"lo": 0.4127368877368879,
"hi": 0.5770937859173152
},
"em": {
"mean": 0.14864864864864866,
"lo": 0.06756756756756757,
"hi": 0.24324324324324326
},
"exact": 0.8648648648648649,
"supported": 0.10810810810810811,
"answered": 0.581081081081081,
"tokens": 963.9324324324324
}
},
{
"name": "test",
"n": 208,
"metrics": {
"of": 1153,
"f1": {
"mean": 0.3994843419698761,
"lo": 0.3547845292710734,
"hi": 0.4501778901165273
},
"em": {
"mean": 0.10576923076923077,
"lo": 0.0673076923076923,
"hi": 0.14903846153846154
},
"exact": 0.875,
"supported": 0.11057692307692307,
"answered": 0.40384615384615385,
"tokens": 969.7980769230769
}
},
{
"name": "test:multi-hop",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.3994843419698761,
"lo": 0.3547845292710734,
"hi": 0.4501778901165273
},
"em": {
"mean": 0.10576923076923077,
"lo": 0.0673076923076923,
"hi": 0.14903846153846154
},
"exact": 0.875,
"supported": 0.11057692307692307,
"answered": 0.40384615384615385,
"tokens": 969.7980769230769
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-cer-k20-gemma4-31b",
"row": null,
"split": null,
"k": 20,
"reader": "gemma4:31b",
"facts": null,
"commit": "ddc076f5dd0506ce50428416c336e45c4a7ada73",
"record": {
"commit": "ddc076f5dd0506ce50428416c336e45c4a7ada73",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "gemma4:31b",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 282,
"answered": 282,
"started": "2026-09-09T09:11:57.021Z",
"ended": "2026-09-09T12:34:27.896Z",
"completion": "complete",
"wall_seconds": 760,
"tokens_per_question": 968,
"reader_tokens": 402677
},
"categories": [
{
"name": "all",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.40657429797513206,
"lo": 0.36655341299441235,
"hi": 0.4485834501120564
},
"em": {
"mean": 0.1099290780141844,
"lo": 0.07446808510638298,
"hi": 0.1453900709219858
},
"exact": 0.8723404255319149,
"answered": 0.44680851063829785,
"tokens": 968.2695035460993,
"p50_ms": 19474,
"p95_ms": 22867
}
},
{
"name": "all:multi-hop",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.40657429797513206,
"lo": 0.36655341299441235,
"hi": 0.4485834501120564
},
"em": {
"mean": 0.1099290780141844,
"lo": 0.07446808510638298,
"hi": 0.1453900709219858
},
"exact": 0.8723404255319149,
"answered": 0.44680851063829785,
"tokens": 968.2695035460993,
"p50_ms": 19474,
"p95_ms": 22867
}
},
{
"name": "dev",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.4453517171938225,
"lo": 0.3661274966538125,
"hi": 0.5312691267954425
},
"em": {
"mean": 0.10810810810810811,
"lo": 0.04054054054054054,
"hi": 0.17567567567567569
},
"exact": 0.8648648648648649,
"answered": 0.5135135135135135,
"tokens": 963.972972972973,
"p50_ms": 19888,
"p95_ms": 31502
}
},
{
"name": "dev:multi-hop",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.4453517171938225,
"lo": 0.3661274966538125,
"hi": 0.5312691267954425
},
"em": {
"mean": 0.10810810810810811,
"lo": 0.04054054054054054,
"hi": 0.17567567567567569
},
"exact": 0.8648648648648649,
"answered": 0.5135135135135135,
"tokens": 963.972972972973,
"p50_ms": 19888,
"p95_ms": 31502
}
},
{
"name": "test",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.3927784853684824,
"lo": 0.34558310285389193,
"hi": 0.4422367278270195
},
"em": {
"mean": 0.11057692307692307,
"lo": 0.07211538461538461,
"hi": 0.15384615384615385
},
"exact": 0.875,
"answered": 0.4230769230769231,
"tokens": 969.7980769230769,
"p50_ms": 19350,
"p95_ms": 22284
}
},
{
"name": "test:multi-hop",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.3927784853684824,
"lo": 0.34558310285389193,
"hi": 0.4422367278270195
},
"em": {
"mean": 0.11057692307692307,
"lo": 0.07211538461538461,
"hi": 0.15384615384615385
},
"exact": 0.875,
"answered": 0.4230769230769231,
"tokens": 969.7980769230769,
"p50_ms": 19350,
"p95_ms": 22284
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-cer-k20-gpt-oss-120b",
"row": null,
"split": null,
"k": 20,
"reader": "gpt-oss-120b",
"facts": null,
"commit": "885a21a63c3254415e40555f7905c8c04aa0be65",
"record": {
"commit": "885a21a63c3254415e40555f7905c8c04aa0be65",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "gpt-oss-120b",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 282,
"answered": 282,
"started": "2026-09-09T09:11:57.070Z",
"ended": "2026-09-10T01:16:16.096Z",
"completion": "complete",
"wall_seconds": 55736,
"tokens_per_question": 968,
"reader_tokens": 367354
},
"categories": [
{
"name": "all",
"n": 282,
"metrics": {
"of": 1536,
"f1": {
"mean": 0.42644300151111564,
"lo": 0.38803062177423187,
"hi": 0.4690826695335125
},
"em": {
"mean": 0.11347517730496454,
"lo": 0.07801418439716312,
"hi": 0.1524822695035461
},
"exact": 0.8723404255319149,
"supported": 0.1099290780141844,
"answered": 0.44680851063829785,
"tokens": 968.2695035460993
}
},
{
"name": "all:multi-hop",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.42644300151111564,
"lo": 0.38803062177423187,
"hi": 0.4690826695335125
},
"em": {
"mean": 0.11347517730496454,
"lo": 0.07801418439716312,
"hi": 0.1524822695035461
},
"exact": 0.8723404255319149,
"supported": 0.1099290780141844,
"answered": 0.44680851063829785,
"tokens": 968.2695035460993
}
},
{
"name": "dev",
"n": 74,
"metrics": {
"of": 383,
"f1": {
"mean": 0.4894098152157783,
"lo": 0.40524786561265225,
"hi": 0.5697896918485152
},
"em": {
"mean": 0.13513513513513514,
"lo": 0.06756756756756757,
"hi": 0.21621621621621623
},
"exact": 0.8648648648648649,
"supported": 0.10810810810810811,
"answered": 0.5540540540540541,
"tokens": 963.972972972973
}
},
{
"name": "dev:multi-hop",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.4894098152157783,
"lo": 0.40524786561265225,
"hi": 0.5697896918485152
},
"em": {
"mean": 0.13513513513513514,
"lo": 0.06756756756756757,
"hi": 0.21621621621621623
},
"exact": 0.8648648648648649,
"supported": 0.10810810810810811,
"answered": 0.5540540540540541,
"tokens": 963.972972972973
}
},
{
"name": "test",
"n": 208,
"metrics": {
"of": 1153,
"f1": {
"mean": 0.4040413466354181,
"lo": 0.3593656991605714,
"hi": 0.4515793988039147
},
"em": {
"mean": 0.10576923076923077,
"lo": 0.0673076923076923,
"hi": 0.14903846153846154
},
"exact": 0.875,
"supported": 0.11057692307692307,
"answered": 0.40865384615384615,
"tokens": 969.7980769230769
}
},
{
"name": "test:multi-hop",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.4040413466354181,
"lo": 0.3593656991605714,
"hi": 0.4515793988039147
},
"em": {
"mean": 0.10576923076923077,
"lo": 0.0673076923076923,
"hi": 0.14903846153846154
},
"exact": 0.875,
"supported": 0.11057692307692307,
"answered": 0.40865384615384615,
"tokens": 969.7980769230769
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-cer-k20-qwen3.6-35b-a3b",
"row": null,
"split": null,
"k": 20,
"reader": "qwen3.6:35b-a3b",
"facts": null,
"commit": "12468828cfba9497ebbb927af840534bb7e3fce2",
"record": {
"commit": "12468828cfba9497ebbb927af840534bb7e3fce2",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "qwen3.6:35b-a3b",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 1536,
"answered": 1525,
"started": "2026-09-09T10:17:13.162Z",
"ended": "2026-09-09T14:03:37.744Z",
"completion": "partial",
"wall_seconds": 1548,
"tokens_per_question": 975,
"reader_tokens": 2315795
},
"categories": [
{
"name": "all",
"n": 1525,
"metrics": {
"of": 1525,
"f1": {
"mean": 0.5031942773941983,
"lo": 0.48158906451353417,
"hi": 0.5235561531857813
},
"em": {
"mean": 0.26688524590163937,
"lo": 0.24459016393442623,
"hi": 0.28918032786885245
},
"exact": 0.8413114754098361,
"answered": 0.5291803278688525,
"tokens": 969.5567213114754,
"p50_ms": 3411,
"p95_ms": 5457
}
},
{
"name": "all:multi-hop",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.4016390206569068,
"lo": 0.35854944983621084,
"hi": 0.44374061839026
},
"em": {
"mean": 0.11347517730496454,
"lo": 0.07801418439716312,
"hi": 0.1524822695035461
},
"exact": 0.8723404255319149,
"answered": 0.4078014184397163,
"tokens": 968.2695035460993,
"p50_ms": 4992,
"p95_ms": 5595
}
},
{
"name": "all:temporal",
"n": 321,
"metrics": {
"of": 321,
"f1": {
"mean": 0.48609211833510946,
"lo": 0.4433158914467327,
"hi": 0.5278921839669506
},
"em": {
"mean": 0.16510903426791276,
"lo": 0.1277258566978193,
"hi": 0.2087227414330218
},
"exact": 0.8785046728971962,
"answered": 0.5264797507788161,
"tokens": 995.4641744548287,
"p50_ms": 5139,
"p95_ms": 5537
}
},
{
"name": "all:open-domain",
"n": 92,
"metrics": {
"of": 92,
"f1": {
"mean": 0.14938364336523866,
"lo": 0.0907644995423732,
"hi": 0.22019723995390883
},
"em": {
"mean": 0.09782608695652174,
"lo": 0.043478260869565216,
"hi": 0.16304347826086957
},
"exact": 0.6521739130434783,
"answered": 0.13043478260869565,
"tokens": 905.5978260869565,
"p50_ms": 4755,
"p95_ms": 5249
}
},
{
"name": "all:single-hop",
"n": 830,
"metrics": {
"of": 830,
"f1": {
"mean": 0.5835303662960664,
"lo": 0.5566727190876097,
"hi": 0.6118201898573855
},
"em": {
"mean": 0.37710843373493974,
"lo": 0.3457831325301205,
"hi": 0.40843373493975904
},
"exact": 0.8373493975903614,
"answered": 0.6156626506024097,
"tokens": 967.0638554216868,
"p50_ms": 2866,
"p95_ms": 5149
}
},
{
"name": "dev",
"n": 383,
"metrics": {
"of": 383,
"f1": {
"mean": 0.504113837479905,
"lo": 0.46581309684221794,
"hi": 0.5441940698177973
},
"em": {
"mean": 0.2506527415143603,
"lo": 0.20887728459530025,
"hi": 0.29765013054830286
},
"exact": 0.8511749347258486,
"answered": 0.5430809399477807,
"tokens": 995.9869451697128,
"p50_ms": 5060,
"p95_ms": 5494
}
},
{
"name": "dev:multi-hop",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.4327986211599659,
"lo": 0.34809720780309006,
"hi": 0.5121190462366931
},
"em": {
"mean": 0.10810810810810811,
"lo": 0.04054054054054054,
"hi": 0.1891891891891892
},
"exact": 0.8648648648648649,
"answered": 0.4594594594594595,
"tokens": 963.972972972973,
"p50_ms": 5239,
"p95_ms": 5824
}
},
{
"name": "dev:temporal",
"n": 90,
"metrics": {
"of": 90,
"f1": {
"mean": 0.5267460317460316,
"lo": 0.44782186948853603,
"hi": 0.6047971781305113
},
"em": {
"mean": 0.17777777777777778,
"lo": 0.1111111111111111,
"hi": 0.26666666666666666
},
"exact": 0.9111111111111111,
"answered": 0.6444444444444445,
"tokens": 1039.1333333333334,
"p50_ms": 5165,
"p95_ms": 5468
}
},
{
"name": "dev:open-domain",
"n": 19,
"metrics": {
"of": 19,
"f1": {
"mean": 0.07318295739348372,
"lo": 0.010526315789473686,
"hi": 0.16090225563909777
},
"em": {
"mean": 0,
"lo": 0,
"hi": 0
},
"exact": 0.7368421052631579,
"answered": 0.05263157894736842,
"tokens": 940.1052631578947,
"p50_ms": 4607,
"p95_ms": 5071
}
},
{
"name": "dev:single-hop",
"n": 200,
"metrics": {
"of": 200,
"f1": {
"mean": 0.5612544137067349,
"lo": 0.5060953275382087,
"hi": 0.618356486668572
},
"em": {
"mean": 0.36,
"lo": 0.3,
"hi": 0.425
},
"exact": 0.83,
"answered": 0.575,
"tokens": 993.725,
"p50_ms": 4899,
"p95_ms": 5343
}
},
{
"name": "test",
"n": 1142,
"metrics": {
"of": 1142,
"f1": {
"mean": 0.5028858785213229,
"lo": 0.47856269622743625,
"hi": 0.5263999050050839
},
"em": {
"mean": 0.2723292469352014,
"lo": 0.2469352014010508,
"hi": 0.29859894921190894
},
"exact": 0.8380035026269702,
"answered": 0.5245183887915937,
"tokens": 960.6926444833625,
"p50_ms": 3099,
"p95_ms": 5410
}
},
{
"name": "test:multi-hop",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.39055339355485674,
"lo": 0.3430059864075119,
"hi": 0.43725924650995174
},
"em": {
"mean": 0.11538461538461539,
"lo": 0.07211538461538461,
"hi": 0.15865384615384615
},
"exact": 0.875,
"answered": 0.3894230769230769,
"tokens": 969.7980769230769,
"p50_ms": 3384,
"p95_ms": 5545
}
},
{
"name": "test:temporal",
"n": 231,
"metrics": {
"of": 231,
"f1": {
"mean": 0.47025293129189233,
"lo": 0.4200586570716443,
"hi": 0.5210190963437719
},
"em": {
"mean": 0.16017316017316016,
"lo": 0.11688311688311688,
"hi": 0.21212121212121213
},
"exact": 0.8658008658008658,
"answered": 0.4805194805194805,
"tokens": 978.4502164502164,
"p50_ms": 5101,
"p95_ms": 5551
}
},
{
"name": "test:open-domain",
"n": 73,
"metrics": {
"of": 73,
"f1": {
"mean": 0.16921669861816122,
"lo": 0.09800184981490381,
"hi": 0.2476808395387684
},
"em": {
"mean": 0.1232876712328767,
"lo": 0.0547945205479452,
"hi": 0.2054794520547945
},
"exact": 0.6301369863013698,
"answered": 0.1506849315068493,
"tokens": 896.6164383561644,
"p50_ms": 4797,
"p95_ms": 5254
}
},
{
"name": "test:single-hop",
"n": 630,
"metrics": {
"of": 630,
"f1": {
"mean": 0.5906020972768059,
"lo": 0.5571437841750114,
"hi": 0.6228101921328763
},
"em": {
"mean": 0.3825396825396825,
"lo": 0.346031746031746,
"hi": 0.42063492063492064
},
"exact": 0.8396825396825397,
"answered": 0.6285714285714286,
"tokens": 958.6,
"p50_ms": 2804,
"p95_ms": 3374
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-context-k20-enso-flash",
"row": null,
"split": null,
"k": 20,
"reader": "enso-flash",
"facts": null,
"commit": "9eab30f73493a28244d1cd00b3fa53b83fdfba3c",
"record": {
"commit": "9eab30f73493a28244d1cd00b3fa53b83fdfba3c",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "enso-flash",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 282,
"answered": 282,
"started": "2026-09-10T01:19:00.084Z",
"ended": "2026-09-10T21:24:01.507Z",
"completion": "complete",
"wall_seconds": 72301,
"tokens_per_question": 971,
"reader_tokens": 366988
},
"categories": [
{
"name": "all",
"n": 282,
"metrics": {
"of": 1536,
"f1": {
"mean": 0.45132705699486847,
"lo": 0.4116788544913449,
"hi": 0.49065341163812487
},
"em": {
"mean": 0.13829787234042554,
"lo": 0.09929078014184398,
"hi": 0.18085106382978725
},
"exact": 0.8794326241134752,
"supported": 0.1347517730496454,
"answered": 0.475177304964539,
"tokens": 971.4893617021277,
"p50_ms": 5048,
"p95_ms": 25064
}
},
{
"name": "dev",
"n": 74,
"metrics": {
"of": 383,
"f1": {
"mean": 0.5089346344145105,
"lo": 0.4275865640726322,
"hi": 0.5863709363709363
},
"em": {
"mean": 0.14864864864864866,
"lo": 0.06756756756756757,
"hi": 0.22972972972972974
},
"exact": 0.8783783783783784,
"supported": 0.13513513513513514,
"answered": 0.5540540540540541,
"tokens": 983.7162162162163,
"p50_ms": 4516,
"p95_ms": 13074
}
},
{
"name": "test",
"n": 208,
"metrics": {
"of": 1153,
"f1": {
"mean": 0.4308320534898031,
"lo": 0.3785297202138698,
"hi": 0.47480667072337407
},
"em": {
"mean": 0.1346153846153846,
"lo": 0.09134615384615384,
"hi": 0.18269230769230768
},
"exact": 0.8798076923076923,
"supported": 0.1346153846153846,
"answered": 0.44711538461538464,
"tokens": 967.1394230769231,
"p50_ms": 5627,
"p95_ms": 31554
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-context-k20-gemma4-31b",
"row": null,
"split": null,
"k": 20,
"reader": "gemma4:31b",
"facts": null,
"commit": "ddc076f5dd0506ce50428416c336e45c4a7ada73",
"record": {
"commit": "ddc076f5dd0506ce50428416c336e45c4a7ada73",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "gemma4:31b",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 282,
"answered": 282,
"started": "2026-09-09T10:01:09.280Z",
"ended": "2026-09-09T10:44:25.089Z",
"completion": "complete",
"wall_seconds": 1620,
"tokens_per_question": 971,
"reader_tokens": 402503
},
"categories": [
{
"name": "all",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.40489303105085517,
"lo": 0.36396036510511787,
"hi": 0.44294751758679163
},
"em": {
"mean": 0.09929078014184398,
"lo": 0.0673758865248227,
"hi": 0.1347517730496454
},
"exact": 0.8794326241134752,
"answered": 0.4397163120567376,
"tokens": 971.4893617021277,
"p50_ms": 18107,
"p95_ms": 21677
}
},
{
"name": "all:multi-hop",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.40489303105085517,
"lo": 0.36396036510511787,
"hi": 0.44294751758679163
},
"em": {
"mean": 0.09929078014184398,
"lo": 0.0673758865248227,
"hi": 0.1347517730496454
},
"exact": 0.8794326241134752,
"answered": 0.4397163120567376,
"tokens": 971.4893617021277,
"p50_ms": 18107,
"p95_ms": 21677
}
},
{
"name": "dev",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.4468046805448822,
"lo": 0.3682755810751332,
"hi": 0.5315233086845628
},
"em": {
"mean": 0.0945945945945946,
"lo": 0.04054054054054054,
"hi": 0.17567567567567569
},
"exact": 0.8783783783783784,
"answered": 0.5,
"tokens": 983.7162162162163,
"p50_ms": 19126,
"p95_ms": 23443
}
},
{
"name": "dev:multi-hop",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.4468046805448822,
"lo": 0.3682755810751332,
"hi": 0.5315233086845628
},
"em": {
"mean": 0.0945945945945946,
"lo": 0.04054054054054054,
"hi": 0.17567567567567569
},
"exact": 0.8783783783783784,
"answered": 0.5,
"tokens": 983.7162162162163,
"p50_ms": 19126,
"p95_ms": 23443
}
},
{
"name": "test",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.38998215575009537,
"lo": 0.3472438491830753,
"hi": 0.4362389540941655
},
"em": {
"mean": 0.10096153846153846,
"lo": 0.0625,
"hi": 0.14423076923076922
},
"exact": 0.8798076923076923,
"answered": 0.4182692307692308,
"tokens": 967.1394230769231,
"p50_ms": 17809,
"p95_ms": 20364
}
},
{
"name": "test:multi-hop",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.38998215575009537,
"lo": 0.3472438491830753,
"hi": 0.4362389540941655
},
"em": {
"mean": 0.10096153846153846,
"lo": 0.0625,
"hi": 0.14423076923076922
},
"exact": 0.8798076923076923,
"answered": 0.4182692307692308,
"tokens": 967.1394230769231,
"p50_ms": 17809,
"p95_ms": 20364
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-context-k20-gpt-oss-120b",
"row": null,
"split": null,
"k": 20,
"reader": "gpt-oss-120b",
"facts": null,
"commit": "9eab30f73493a28244d1cd00b3fa53b83fdfba3c",
"record": {
"commit": "9eab30f73493a28244d1cd00b3fa53b83fdfba3c",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "gpt-oss-120b",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 282,
"answered": 282,
"started": "2026-09-10T01:16:19.890Z",
"ended": "2026-09-10T21:22:38.922Z",
"completion": "complete",
"wall_seconds": 72379,
"tokens_per_question": 971,
"reader_tokens": 367151
},
"categories": [
{
"name": "all",
"n": 282,
"metrics": {
"of": 1536,
"f1": {
"mean": 0.4534756986788095,
"lo": 0.4158818967929915,
"hi": 0.49451375567443207
},
"em": {
"mean": 0.1347517730496454,
"lo": 0.09929078014184398,
"hi": 0.17375886524822695
},
"exact": 0.8794326241134752,
"supported": 0.1347517730496454,
"answered": 0.4787234042553192,
"tokens": 971.4893617021277,
"p50_ms": 2726,
"p95_ms": 31152
}
},
{
"name": "dev",
"n": 74,
"metrics": {
"of": 383,
"f1": {
"mean": 0.5153133044702501,
"lo": 0.42909021506582484,
"hi": 0.5944699847138869
},
"em": {
"mean": 0.14864864864864866,
"lo": 0.06756756756756757,
"hi": 0.22972972972972974
},
"exact": 0.8783783783783784,
"supported": 0.13513513513513514,
"answered": 0.5540540540540541,
"tokens": 983.7162162162163,
"p50_ms": 2358,
"p95_ms": 35269
}
},
{
"name": "test",
"n": 208,
"metrics": {
"of": 1153,
"f1": {
"mean": 0.4314757812337771,
"lo": 0.38433063035006787,
"hi": 0.4784483402345417
},
"em": {
"mean": 0.12980769230769232,
"lo": 0.08653846153846154,
"hi": 0.17307692307692307
},
"exact": 0.8798076923076923,
"supported": 0.1346153846153846,
"answered": 0.4519230769230769,
"tokens": 967.1394230769231,
"p50_ms": 2843,
"p95_ms": 15562
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-context-k20-qwen3.6-35b-a3b",
"row": null,
"split": null,
"k": 20,
"reader": "qwen3.6:35b-a3b",
"facts": null,
"commit": "12468828cfba9497ebbb927af840534bb7e3fce2",
"record": {
"commit": "12468828cfba9497ebbb927af840534bb7e3fce2",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "qwen3.6:35b-a3b",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 1536,
"answered": 1525,
"started": "2026-09-09T12:33:00.876Z",
"ended": "2026-09-09T13:59:29.450Z",
"completion": "partial",
"wall_seconds": 2449,
"tokens_per_question": 968,
"reader_tokens": 2554957
},
"categories": [
{
"name": "all",
"n": 1525,
"metrics": {
"of": 1525,
"f1": {
"mean": 0.5327296100737436,
"lo": 0.5116468673290611,
"hi": 0.5541630203476279
},
"em": {
"mean": 0.2898360655737705,
"lo": 0.26885245901639343,
"hi": 0.31344262295081965
},
"exact": 0.8701639344262295,
"answered": 0.5593442622950819,
"tokens": 969.0380327868852,
"p50_ms": 5007,
"p95_ms": 5845
}
},
{
"name": "all:multi-hop",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.43004774582728983,
"lo": 0.38662569047088013,
"hi": 0.47125305728073796
},
"em": {
"mean": 0.13120567375886524,
"lo": 0.09574468085106383,
"hi": 0.17375886524822695
},
"exact": 0.8794326241134752,
"answered": 0.43617021276595747,
"tokens": 971.4893617021277,
"p50_ms": 5498,
"p95_ms": 6513
}
},
{
"name": "all:temporal",
"n": 321,
"metrics": {
"of": 321,
"f1": {
"mean": 0.49451839264923403,
"lo": 0.4514980692550787,
"hi": 0.5363413844722259
},
"em": {
"mean": 0.1806853582554517,
"lo": 0.14018691588785046,
"hi": 0.22741433021806853
},
"exact": 0.8847352024922118,
"answered": 0.5389408099688473,
"tokens": 993.7445482866044,
"p50_ms": 4237,
"p95_ms": 5907
}
},
{
"name": "all:open-domain",
"n": 92,
"metrics": {
"of": 92,
"f1": {
"mean": 0.1981289838083316,
"lo": 0.1296371601534645,
"hi": 0.2740259642433555
},
"em": {
"mean": 0.13043478260869565,
"lo": 0.06521739130434782,
"hi": 0.20652173913043478
},
"exact": 0.6521739130434783,
"answered": 0.18478260869565216,
"tokens": 916.8804347826087,
"p50_ms": 3673,
"p95_ms": 4117
}
},
{
"name": "all:single-hop",
"n": 830,
"metrics": {
"of": 830,
"f1": {
"mean": 0.6194830367330058,
"lo": 0.5911709329285919,
"hi": 0.6442718660923942
},
"em": {
"mean": 0.4036144578313253,
"lo": 0.3710843373493976,
"hi": 0.43493975903614457
},
"exact": 0.8855421686746988,
"answered": 0.6506024096385542,
"tokens": 964.4313253012048,
"p50_ms": 4966,
"p95_ms": 5520
}
},
{
"name": "dev",
"n": 383,
"metrics": {
"of": 383,
"f1": {
"mean": 0.51866848343928,
"lo": 0.47958475137924317,
"hi": 0.5578697110813269
},
"em": {
"mean": 0.26370757180156656,
"lo": 0.22193211488250653,
"hi": 0.30809399477806787
},
"exact": 0.8668407310704961,
"answered": 0.566579634464752,
"tokens": 1015.0809399477807,
"p50_ms": 4240,
"p95_ms": 5848
}
},
{
"name": "dev:multi-hop",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.4505046593679344,
"lo": 0.3656066613141335,
"hi": 0.5364501055677525
},
"em": {
"mean": 0.13513513513513514,
"lo": 0.06756756756756757,
"hi": 0.21621621621621623
},
"exact": 0.8783783783783784,
"answered": 0.5,
"tokens": 983.7162162162163,
"p50_ms": 4014,
"p95_ms": 5772
}
},
{
"name": "dev:temporal",
"n": 90,
"metrics": {
"of": 90,
"f1": {
"mean": 0.5198500881834213,
"lo": 0.44246031746031733,
"hi": 0.5896119929453261
},
"em": {
"mean": 0.18888888888888888,
"lo": 0.1111111111111111,
"hi": 0.26666666666666666
},
"exact": 0.9222222222222223,
"answered": 0.6333333333333333,
"tokens": 1058.3444444444444,
"p50_ms": 5539,
"p95_ms": 6020
}
},
{
"name": "dev:open-domain",
"n": 19,
"metrics": {
"of": 19,
"f1": {
"mean": 0.07393483709273184,
"lo": 0.013157894736842105,
"hi": 0.1604010025062657
},
"em": {
"mean": 0,
"lo": 0,
"hi": 0
},
"exact": 0.7368421052631579,
"answered": 0.05263157894736842,
"tokens": 987.2105263157895,
"p50_ms": 3785,
"p95_ms": 4117
}
},
{
"name": "dev:single-hop",
"n": 200,
"metrics": {
"of": 200,
"f1": {
"mean": 0.5856070726137352,
"lo": 0.529489549836353,
"hi": 0.6404945010067329
},
"em": {
"mean": 0.37,
"lo": 0.305,
"hi": 0.44
},
"exact": 0.85,
"answered": 0.61,
"tokens": 1009.865,
"p50_ms": 4037,
"p95_ms": 5419
}
},
{
"name": "test",
"n": 1142,
"metrics": {
"of": 1142,
"f1": {
"mean": 0.5374453819660394,
"lo": 0.5154234373257967,
"hi": 0.5610125382812944
},
"em": {
"mean": 0.29859894921190894,
"lo": 0.2732049036777583,
"hi": 0.3257443082311734
},
"exact": 0.8712784588441331,
"answered": 0.5569176882661997,
"tokens": 953.5963222416813,
"p50_ms": 5063,
"p95_ms": 5830
}
},
{
"name": "test:multi-hop",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.42276980543302195,
"lo": 0.37856647019399686,
"hi": 0.4702564228935132
},
"em": {
"mean": 0.12980769230769232,
"lo": 0.08653846153846154,
"hi": 0.1778846153846154
},
"exact": 0.8798076923076923,
"answered": 0.41346153846153844,
"tokens": 967.1394230769231,
"p50_ms": 5604,
"p95_ms": 6735
}
},
{
"name": "test:temporal",
"n": 231,
"metrics": {
"of": 231,
"f1": {
"mean": 0.4846489008826672,
"lo": 0.43327773717384155,
"hi": 0.53288419132575
},
"em": {
"mean": 0.1774891774891775,
"lo": 0.12987012987012986,
"hi": 0.22943722943722944
},
"exact": 0.8701298701298701,
"answered": 0.5021645021645021,
"tokens": 968.5757575757576,
"p50_ms": 3903,
"p95_ms": 5598
}
},
{
"name": "test:open-domain",
"n": 73,
"metrics": {
"of": 73,
"f1": {
"mean": 0.2304534877480083,
"lo": 0.15118645916591117,
"hi": 0.31680608112114955
},
"em": {
"mean": 0.1643835616438356,
"lo": 0.0821917808219178,
"hi": 0.2602739726027397
},
"exact": 0.6301369863013698,
"answered": 0.2191780821917808,
"tokens": 898.5753424657535,
"p50_ms": 3585,
"p95_ms": 4213
}
},
{
"name": "test:single-hop",
"n": 630,
"metrics": {
"of": 630,
"f1": {
"mean": 0.6302373110565829,
"lo": 0.598657935655634,
"hi": 0.6607275639401357
},
"em": {
"mean": 0.4142857142857143,
"lo": 0.3761904761904762,
"hi": 0.4523809523809524
},
"exact": 0.8968253968253969,
"answered": 0.6634920634920635,
"tokens": 950.0079365079365,
"p50_ms": 5065,
"p95_ms": 5526
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-oracle-k20-gemma4-31b",
"row": null,
"split": null,
"k": 20,
"reader": "gemma4:31b",
"facts": null,
"commit": "ddc076f5dd0506ce50428416c336e45c4a7ada73",
"record": {
"commit": "ddc076f5dd0506ce50428416c336e45c4a7ada73",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "gemma4:31b",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 282,
"answered": 282,
"started": "2026-09-09T10:44:25.916Z",
"ended": "2026-09-09T10:56:08.299Z",
"completion": "complete",
"wall_seconds": 702,
"tokens_per_question": 169,
"reader_tokens": 114470
},
"categories": [
{
"name": "all",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.5301416917271639,
"lo": 0.4900480818101769,
"hi": 0.570121753819505
},
"em": {
"mean": 0.1595744680851064,
"lo": 0.12056737588652482,
"hi": 0.20212765957446807
},
"exact": 0.9964539007092199,
"answered": 0.5851063829787234,
"tokens": 168.5177304964539,
"p50_ms": 4759,
"p95_ms": 7501
}
},
{
"name": "all:multi-hop",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.5301416917271639,
"lo": 0.4900480818101769,
"hi": 0.570121753819505
},
"em": {
"mean": 0.1595744680851064,
"lo": 0.12056737588652482,
"hi": 0.20212765957446807
},
"exact": 0.9964539007092199,
"answered": 0.5851063829787234,
"tokens": 168.5177304964539,
"p50_ms": 4759,
"p95_ms": 7501
}
},
{
"name": "dev",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.5919549061189925,
"lo": 0.5193204261315717,
"hi": 0.6701349190513274
},
"em": {
"mean": 0.13513513513513514,
"lo": 0.06756756756756757,
"hi": 0.21621621621621623
},
"exact": 0.9864864864864865,
"answered": 0.6621621621621622,
"tokens": 135.0810810810811,
"p50_ms": 3839,
"p95_ms": 5649
}
},
{
"name": "dev:multi-hop",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.5919549061189925,
"lo": 0.5193204261315717,
"hi": 0.6701349190513274
},
"em": {
"mean": 0.13513513513513514,
"lo": 0.06756756756756757,
"hi": 0.21621621621621623
},
"exact": 0.9864864864864865,
"answered": 0.6621621621621622,
"tokens": 135.0810810810811,
"p50_ms": 3839,
"p95_ms": 5649
}
},
{
"name": "test",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.5081504519916098,
"lo": 0.46000693776643786,
"hi": 0.5568810797341097
},
"em": {
"mean": 0.16826923076923078,
"lo": 0.1201923076923077,
"hi": 0.21634615384615385
},
"exact": 1,
"answered": 0.5576923076923077,
"tokens": 180.41346153846155,
"p50_ms": 5017,
"p95_ms": 7969
}
},
{
"name": "test:multi-hop",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.5081504519916098,
"lo": 0.46000693776643786,
"hi": 0.5568810797341097
},
"em": {
"mean": 0.16826923076923078,
"lo": 0.1201923076923077,
"hi": 0.21634615384615385
},
"exact": 1,
"answered": 0.5576923076923077,
"tokens": 180.41346153846155,
"p50_ms": 5017,
"p95_ms": 7969
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-oracle-k20-qwen3.6-35b-a3b",
"row": null,
"split": null,
"k": 20,
"reader": "qwen3.6:35b-a3b",
"facts": null,
"commit": "12468828cfba9497ebbb927af840534bb7e3fce2",
"record": {
"commit": "12468828cfba9497ebbb927af840534bb7e3fce2",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "qwen3.6:35b-a3b",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 1536,
"answered": 1525,
"started": "2026-09-09T12:45:37.890Z",
"ended": "2026-09-09T13:59:30.537Z",
"completion": "partial",
"wall_seconds": 1989,
"tokens_per_question": 82,
"reader_tokens": 460599
},
"categories": [
{
"name": "all",
"n": 1525,
"metrics": {
"of": 1525,
"f1": {
"mean": 0.6184550729877653,
"lo": 0.5974999677344102,
"hi": 0.6378452097018928
},
"em": {
"mean": 0.3462295081967213,
"lo": 0.32459016393442625,
"hi": 0.36918032786885246
},
"exact": 0.9967213114754099,
"answered": 0.659672131147541,
"tokens": 82.1488524590164,
"p50_ms": 2929,
"p95_ms": 4015
}
},
{
"name": "all:multi-hop",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.556304737641278,
"lo": 0.5122132496547763,
"hi": 0.5990094280623881
},
"em": {
"mean": 0.20212765957446807,
"lo": 0.15602836879432624,
"hi": 0.24822695035460993
},
"exact": 0.9964539007092199,
"answered": 0.6063829787234043,
"tokens": 168.5177304964539,
"p50_ms": 3800,
"p95_ms": 4387
}
},
{
"name": "all:temporal",
"n": 321,
"metrics": {
"of": 321,
"f1": {
"mean": 0.5438301573815596,
"lo": 0.5029226398385278,
"hi": 0.587450158010906
},
"em": {
"mean": 0.22741433021806853,
"lo": 0.1838006230529595,
"hi": 0.2772585669781931
},
"exact": 0.9968847352024922,
"answered": 0.5887850467289719,
"tokens": 63.74766355140187,
"p50_ms": 3283,
"p95_ms": 3809
}
},
{
"name": "all:open-domain",
"n": 92,
"metrics": {
"of": 92,
"f1": {
"mean": 0.2644291893812354,
"lo": 0.1867861394076228,
"hi": 0.34891484261688865
},
"em": {
"mean": 0.1956521739130435,
"lo": 0.11956521739130435,
"hi": 0.2826086956521739
},
"exact": 0.967391304347826,
"answered": 0.25,
"tokens": 117.19565217391305,
"p50_ms": 3335,
"p95_ms": 3809
}
},
{
"name": "all:single-hop",
"n": 830,
"metrics": {
"of": 830,
"f1": {
"mean": 0.7076735956011446,
"lo": 0.6815724588154725,
"hi": 0.7305743080790545
},
"em": {
"mean": 0.4578313253012048,
"lo": 0.42168674698795183,
"hi": 0.4903614457831325
},
"exact": 1,
"answered": 0.7506024096385542,
"tokens": 56.036144578313255,
"p50_ms": 2751,
"p95_ms": 3036
}
},
{
"name": "dev",
"n": 383,
"metrics": {
"of": 383,
"f1": {
"mean": 0.6123530750809747,
"lo": 0.5739952530041554,
"hi": 0.6510492411694527
},
"em": {
"mean": 0.3263707571801567,
"lo": 0.2793733681462141,
"hi": 0.3733681462140992
},
"exact": 0.9973890339425587,
"answered": 0.6710182767624021,
"tokens": 75.6710182767624,
"p50_ms": 2945,
"p95_ms": 3809
}
},
{
"name": "dev:multi-hop",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.6059866488813856,
"lo": 0.5304390133337502,
"hi": 0.692674600569337
},
"em": {
"mean": 0.1891891891891892,
"lo": 0.10810810810810811,
"hi": 0.28378378378378377
},
"exact": 0.9864864864864865,
"answered": 0.7297297297297297,
"tokens": 135.0810810810811,
"p50_ms": 3660,
"p95_ms": 4025
}
},
{
"name": "dev:temporal",
"n": 90,
"metrics": {
"of": 90,
"f1": {
"mean": 0.5699559082892414,
"lo": 0.4914373897707229,
"hi": 0.6448941798941797
},
"em": {
"mean": 0.28888888888888886,
"lo": 0.2,
"hi": 0.37777777777777777
},
"exact": 1,
"answered": 0.6555555555555556,
"tokens": 62.644444444444446,
"p50_ms": 3257,
"p95_ms": 3809
}
},
{
"name": "dev:open-domain",
"n": 19,
"metrics": {
"of": 19,
"f1": {
"mean": 0.09089390142021721,
"lo": 0.021052631578947368,
"hi": 0.17443609022556394
},
"em": {
"mean": 0,
"lo": 0,
"hi": 0
},
"exact": 1,
"answered": 0.05263157894736842,
"tokens": 113.78947368421052,
"p50_ms": 3489,
"p95_ms": 3809
}
},
{
"name": "dev:single-hop",
"n": 200,
"metrics": {
"of": 200,
"f1": {
"mean": 0.6833259993288737,
"lo": 0.6319834423238181,
"hi": 0.733010626287422
},
"em": {
"mean": 0.425,
"lo": 0.36,
"hi": 0.49
},
"exact": 1,
"answered": 0.715,
"tokens": 55.93,
"p50_ms": 2739,
"p95_ms": 3111
}
},
{
"name": "test",
"n": 1142,
"metrics": {
"of": 1142,
"f1": {
"mean": 0.6205015398864543,
"lo": 0.5993450330898592,
"hi": 0.6440107175972996
},
"em": {
"mean": 0.35288966725043786,
"lo": 0.3248686514886165,
"hi": 0.38178633975481613
},
"exact": 0.9964973730297724,
"answered": 0.6558669001751314,
"tokens": 84.32136602451838,
"p50_ms": 2925,
"p95_ms": 4084
}
},
{
"name": "test:multi-hop",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.5386294422962394,
"lo": 0.492068842196796,
"hi": 0.5886182319408546
},
"em": {
"mean": 0.20673076923076922,
"lo": 0.15384615384615385,
"hi": 0.2644230769230769
},
"exact": 1,
"answered": 0.5625,
"tokens": 180.41346153846155,
"p50_ms": 3875,
"p95_ms": 4462
}
},
{
"name": "test:temporal",
"n": 231,
"metrics": {
"of": 231,
"f1": {
"mean": 0.5336512933915534,
"lo": 0.4833092833092835,
"hi": 0.5819687287219759
},
"em": {
"mean": 0.20346320346320346,
"lo": 0.15151515151515152,
"hi": 0.2554112554112554
},
"exact": 0.9956709956709957,
"answered": 0.5627705627705628,
"tokens": 64.17748917748918,
"p50_ms": 3303,
"p95_ms": 3857
}
},
{
"name": "test:open-domain",
"n": 73,
"metrics": {
"of": 73,
"f1": {
"mean": 0.30959590816561,
"lo": 0.22156145403526545,
"hi": 0.41588848746382995
},
"em": {
"mean": 0.2465753424657534,
"lo": 0.1643835616438356,
"hi": 0.3561643835616438
},
"exact": 0.958904109589041,
"answered": 0.3013698630136986,
"tokens": 118.08219178082192,
"p50_ms": 3246,
"p95_ms": 3817
}
},
{
"name": "test:single-hop",
"n": 630,
"metrics": {
"of": 630,
"f1": {
"mean": 0.7154029912431358,
"lo": 0.6876197753820203,
"hi": 0.7446131495412637
},
"em": {
"mean": 0.46825396825396826,
"lo": 0.4301587301587302,
"hi": 0.5095238095238095
},
"exact": 1,
"answered": 0.7619047619047619,
"tokens": 56.06984126984127,
"p50_ms": 2753,
"p95_ms": 3007
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-single-k20-gemma4-31b",
"row": null,
"split": null,
"k": 20,
"reader": "gemma4:31b",
"facts": null,
"commit": "ddc076f5dd0506ce50428416c336e45c4a7ada73",
"record": {
"commit": "ddc076f5dd0506ce50428416c336e45c4a7ada73",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "gemma4:31b",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 282,
"answered": 282,
"started": "2026-09-09T10:56:09.853Z",
"ended": "2026-09-09T11:33:57.673Z",
"completion": "complete",
"wall_seconds": 2268,
"tokens_per_question": 835,
"reader_tokens": 367970
},
"categories": [
{
"name": "all",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.375885104034787,
"lo": 0.3355962049368216,
"hi": 0.4151224565726375
},
"em": {
"mean": 0.09929078014184398,
"lo": 0.0673758865248227,
"hi": 0.1347517730496454
},
"exact": 0.7978723404255319,
"answered": 0.425531914893617,
"tokens": 834.7482269503546,
"p50_ms": 16067,
"p95_ms": 18487
}
},
{
"name": "all:multi-hop",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.375885104034787,
"lo": 0.3355962049368216,
"hi": 0.4151224565726375
},
"em": {
"mean": 0.09929078014184398,
"lo": 0.0673758865248227,
"hi": 0.1347517730496454
},
"exact": 0.7978723404255319,
"answered": 0.425531914893617,
"tokens": 834.7482269503546,
"p50_ms": 16067,
"p95_ms": 18487
}
},
{
"name": "dev",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.4114971654938449,
"lo": 0.3320832180651915,
"hi": 0.4901185901185901
},
"em": {
"mean": 0.06756756756756757,
"lo": 0.013513513513513514,
"hi": 0.12162162162162163
},
"exact": 0.7837837837837838,
"answered": 0.4864864864864865,
"tokens": 830.5945945945946,
"p50_ms": 15797,
"p95_ms": 19048
}
},
{
"name": "dev:multi-hop",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.4114971654938449,
"lo": 0.3320832180651915,
"hi": 0.4901185901185901
},
"em": {
"mean": 0.06756756756756757,
"lo": 0.013513513513513514,
"hi": 0.12162162162162163
},
"exact": 0.7837837837837838,
"answered": 0.4864864864864865,
"tokens": 830.5945945945946,
"p50_ms": 15797,
"p95_ms": 19048
}
},
{
"name": "test",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.36321542832339115,
"lo": 0.31528468639006363,
"hi": 0.40943817960368684
},
"em": {
"mean": 0.11057692307692307,
"lo": 0.07211538461538461,
"hi": 0.15384615384615385
},
"exact": 0.8028846153846154,
"answered": 0.40384615384615385,
"tokens": 836.2259615384615,
"p50_ms": 16105,
"p95_ms": 18456
}
},
{
"name": "test:multi-hop",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.36321542832339115,
"lo": 0.31528468639006363,
"hi": 0.40943817960368684
},
"em": {
"mean": 0.11057692307692307,
"lo": 0.07211538461538461,
"hi": 0.15384615384615385
},
"exact": 0.8028846153846154,
"answered": 0.40384615384615385,
"tokens": 836.2259615384615,
"p50_ms": 16105,
"p95_ms": 18456
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-all-single-k20-qwen3.6-35b-a3b",
"row": null,
"split": null,
"k": 20,
"reader": "qwen3.6:35b-a3b",
"facts": null,
"commit": "bea5d0ad184462b76db19ad2d601f39c329b7c09",
"record": {
"commit": "bea5d0ad184462b76db19ad2d601f39c329b7c09",
"prompt": "prompts/reader-locomo.txt",
"prompt_digest": "afa0a591ae62c574",
"dataset_digest": "79fa87e90f040813",
"store_digest": "688cea6b889e484c",
"facts_digest": "cc680d252a3538b9",
"reader": "qwen3.6:35b-a3b",
"embedding": null,
"temperature": 0,
"max_tokens": 64,
"questions": 1536,
"answered": 1525,
"started": "2026-09-09T12:54:38.083Z",
"ended": "2026-09-09T13:59:32.379Z",
"completion": "partial",
"wall_seconds": 2390,
"tokens_per_question": 840,
"reader_tokens": 1989142
},
"categories": [
{
"name": "all",
"n": 1525,
"metrics": {
"of": 1525,
"f1": {
"mean": 0.4881578802950179,
"lo": 0.46623196367843434,
"hi": 0.5096951323870532
},
"em": {
"mean": 0.26229508196721313,
"lo": 0.23934426229508196,
"hi": 0.2845901639344262
},
"exact": 0.7927868852459017,
"answered": 0.5134426229508197,
"tokens": 841.088524590164,
"p50_ms": 4564,
"p95_ms": 5311
}
},
{
"name": "all:multi-hop",
"n": 282,
"metrics": {
"of": 282,
"f1": {
"mean": 0.36708319324537253,
"lo": 0.32934244507282956,
"hi": 0.41035123904910403
},
"em": {
"mean": 0.09219858156028368,
"lo": 0.06028368794326241,
"hi": 0.1276595744680851
},
"exact": 0.7978723404255319,
"answered": 0.3900709219858156,
"tokens": 834.7482269503546,
"p50_ms": 4936,
"p95_ms": 5402
}
},
{
"name": "all:temporal",
"n": 321,
"metrics": {
"of": 321,
"f1": {
"mean": 0.4748220972520042,
"lo": 0.432913019828908,
"hi": 0.5174956058133628
},
"em": {
"mean": 0.17445482866043613,
"lo": 0.13707165109034267,
"hi": 0.21806853582554517
},
"exact": 0.8255451713395638,
"answered": 0.5046728971962616,
"tokens": 856.3333333333334,
"p50_ms": 4785,
"p95_ms": 5359
}
},
{
"name": "all:open-domain",
"n": 92,
"metrics": {
"of": 92,
"f1": {
"mean": 0.17725317047567687,
"lo": 0.11491596638655462,
"hi": 0.25445891884510813
},
"em": {
"mean": 0.11956521739130435,
"lo": 0.05434782608695652,
"hi": 0.1956521739130435
},
"exact": 0.532608695652174,
"answered": 0.17391304347826086,
"tokens": 780.7717391304348,
"p50_ms": 3123,
"p95_ms": 3627
}
},
{
"name": "all:single-hop",
"n": 830,
"metrics": {
"of": 830,
"f1": {
"mean": 0.5689134000639201,
"lo": 0.5409475566165869,
"hi": 0.5953400767933723
},
"em": {
"mean": 0.36987951807228914,
"lo": 0.3385542168674699,
"hi": 0.40120481927710844
},
"exact": 0.8072289156626506,
"answered": 0.5963855421686747,
"tokens": 844.032530120482,
"p50_ms": 3742,
"p95_ms": 5232
}
},
{
"name": "dev",
"n": 383,
"metrics": {
"of": 383,
"f1": {
"mean": 0.4882440215211369,
"lo": 0.4489697660699844,
"hi": 0.5267893540177094
},
"em": {
"mean": 0.24804177545691905,
"lo": 0.206266318537859,
"hi": 0.28720626631853785
},
"exact": 0.8067885117493473,
"answered": 0.5221932114882507,
"tokens": 864.1253263707572,
"p50_ms": 3629,
"p95_ms": 5264
}
},
{
"name": "dev:multi-hop",
"n": 74,
"metrics": {
"of": 74,
"f1": {
"mean": 0.4093514962952304,
"lo": 0.3335098991620731,
"hi": 0.49587308369917077
},
"em": {
"mean": 0.08108108108108109,
"lo": 0.02702702702702703,
"hi": 0.14864864864864866
},
"exact": 0.7837837837837838,
"answered": 0.4594594594594595,
"tokens": 830.5945945945946,
"p50_ms": 4953,
"p95_ms": 5475
}
},
{
"name": "dev:temporal",
"n": 90,
"metrics": {
"of": 90,
"f1": {
"mean": 0.5272222222222221,
"lo": 0.45180776014109336,
"hi": 0.6028483245149912
},
"em": {
"mean": 0.2111111111111111,
"lo": 0.13333333333333333,
"hi": 0.3
},
"exact": 0.9,
"answered": 0.6111111111111112,
"tokens": 892.7,
"p50_ms": 4904,
"p95_ms": 5313
}
},
{
"name": "dev:open-domain",
"n": 19,
"metrics": {
"of": 19,
"f1": {
"mean": 0.04511278195488722,
"lo": 0,
"hi": 0.12030075187969927
},
"em": {
"mean": 0,
"lo": 0,
"hi": 0
},
"exact": 0.631578947368421,
"answered": 0.05263157894736842,
"tokens": 781.6315789473684,
"p50_ms": 3090,
"p95_ms": 3495
}
},
{
"name": "dev:single-hop",
"n": 200,
"metrics": {
"of": 200,
"f1": {
"mean": 0.5419915332980269,
"lo": 0.48233670399401085,
"hi": 0.5974412875547035
},
"em": {
"mean": 0.35,
"lo": 0.285,
"hi": 0.415
},
"exact": 0.79,
"answered": 0.55,
"tokens": 871.51,
"p50_ms": 3412,
"p95_ms": 3819
}
},
{
"name": "test",
"n": 1142,
"metrics": {
"of": 1142,
"f1": {
"mean": 0.48812899054930775,
"lo": 0.46491974479757947,
"hi": 0.5143194824535225
},
"em": {
"mean": 0.2670753064798599,
"lo": 0.24168126094570927,
"hi": 0.29509632224168125
},
"exact": 0.7880910683012259,
"answered": 0.510507880910683,
"tokens": 833.3625218914185,
"p50_ms": 4646,
"p95_ms": 5315
}
},
{
"name": "test:multi-hop",
"n": 208,
"metrics": {
"of": 208,
"f1": {
"mean": 0.3520454315834038,
"lo": 0.31027902774413185,
"hi": 0.40342120017895494
},
"em": {
"mean": 0.09615384615384616,
"lo": 0.057692307692307696,
"hi": 0.1346153846153846
},
"exact": 0.8028846153846154,
"answered": 0.36538461538461536,
"tokens": 836.2259615384615,
"p50_ms": 4931,
"p95_ms": 5393
}
},
{
"name": "test:temporal",
"n": 231,
"metrics": {
"of": 231,
"f1": {
"mean": 0.45440646414672403,
"lo": 0.40529225464290414,
"hi": 0.5048428002973461
},
"em": {
"mean": 0.16017316017316016,
"lo": 0.11688311688311688,
"hi": 0.2077922077922078
},
"exact": 0.7965367965367965,
"answered": 0.46320346320346323,
"tokens": 842.1645021645022,
"p50_ms": 4601,
"p95_ms": 5374
}
},
{
"name": "test:open-domain",
"n": 73,
"metrics": {
"of": 73,
"f1": {
"mean": 0.21164587433725224,
"lo": 0.13354761114632185,
"hi": 0.2959229669141031
},
"em": {
"mean": 0.1506849315068493,
"lo": 0.0684931506849315,
"hi": 0.2328767123287671
},
"exact": 0.5068493150684932,
"answered": 0.2054794520547945,
"tokens": 780.5479452054794,
"p50_ms": 3143,
"p95_ms": 3662
}
},
{
"name": "test:single-hop",
"n": 630,
"metrics": {
"of": 630,
"f1": {
"mean": 0.5774600244340442,
"lo": 0.5449694737675975,
"hi": 0.6118638554360045
},
"em": {
"mean": 0.3761904761904762,
"lo": 0.3380952380952381,
"hi": 0.4142857142857143
},
"exact": 0.8126984126984127,
"answered": 0.6111111111111112,
"tokens": 835.3095238095239,
"p50_ms": 4547,
"p95_ms": 5282
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-adjacency-locomo",
"row": "+adjacency",
"split": "all",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:41.917Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.14893617021276595,
"lo": 0.10283687943262411,
"hi": 0.1950354609929078
},
"any@5": {
"mean": 0.6702127659574468,
"lo": 0.6170212765957447,
"hi": 0.7269503546099291
},
"recall@5": {
"mean": 0.3736380832265492,
"lo": 0.3333683647653413,
"hi": 0.41702238752910653
},
"all@10": {
"mean": 0.25177304964539005,
"lo": 0.19858156028368795,
"hi": 0.3049645390070922
},
"any@10": {
"mean": 0.8014184397163121,
"lo": 0.7553191489361702,
"hi": 0.8475177304964538
},
"recall@10": {
"mean": 0.5081476728033282,
"lo": 0.4630435561286626,
"hi": 0.5504454371475647
},
"all@20": {
"mean": 0.3900709219858156,
"lo": 0.3333333333333333,
"hi": 0.45390070921985815
},
"any@20": {
"mean": 0.8865248226950354,
"lo": 0.8475177304964538,
"hi": 0.9219858156028369
},
"recall@20": {
"mean": 0.6361280646275044,
"lo": 0.5938793362085633,
"hi": 0.6801066644151746
},
"mrr": {
"mean": 0.5002651733194846,
"lo": 0.4539708026410152,
"hi": 0.5482347292187718
},
"ndcg@5": {
"mean": 0.3557429598347188,
"lo": 0.3165164434059352,
"hi": 0.39540612184723173
},
"ndcg@10": {
"mean": 0.41081855082313207,
"lo": 0.3740640145622385,
"hi": 0.4470992852123337
},
"ndcg@20": {
"mean": 0.4561045382021939,
"lo": 0.42175002802365136,
"hi": 0.4913200568961985
},
"exact": {
"mean": 0.8865248226950354,
"lo": 0.8475177304964538,
"hi": 0.9219858156028369
},
"supported": {
"mean": 0.9078014184397163,
"lo": 0.8723404255319149,
"hi": 0.9397163120567376
},
"examined": 74.63829787234043,
"tokens": 788.7482269503546,
"p50": 0.8235,
"p95": 0.946208
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.7383177570093458,
"lo": 0.6884735202492211,
"hi": 0.7881619937694704
},
"any@5": {
"mean": 0.8037383177570093,
"lo": 0.7570093457943925,
"hi": 0.8442367601246106
},
"recall@5": {
"mean": 0.7718068535825544,
"lo": 0.724558670820353,
"hi": 0.8161993769470405
},
"all@10": {
"mean": 0.7975077881619937,
"lo": 0.7507788161993769,
"hi": 0.8411214953271028
},
"any@10": {
"mean": 0.8535825545171339,
"lo": 0.8099688473520249,
"hi": 0.8909657320872274
},
"recall@10": {
"mean": 0.8265835929387331,
"lo": 0.7819314641744548,
"hi": 0.8639667705088268
},
"all@20": {
"mean": 0.8348909657320872,
"lo": 0.7912772585669782,
"hi": 0.8753894080996885
},
"any@20": {
"mean": 0.8940809968847352,
"lo": 0.8566978193146417,
"hi": 0.9252336448598131
},
"recall@20": {
"mean": 0.8650051921079958,
"lo": 0.8276220145379023,
"hi": 0.8992731048805814
},
"mrr": {
"mean": 0.6858439081432484,
"lo": 0.6402053813268767,
"hi": 0.7305105689314046
},
"ndcg@5": {
"mean": 0.6883831546972707,
"lo": 0.6443898487237105,
"hi": 0.7334991544003355
},
"ndcg@10": {
"mean": 0.70768732676272,
"lo": 0.6641165087717769,
"hi": 0.7496356736456137
},
"ndcg@20": {
"mean": 0.7183945295655887,
"lo": 0.6774583628155739,
"hi": 0.7578542894951453
},
"exact": {
"mean": 0.8940809968847352,
"lo": 0.8566978193146417,
"hi": 0.9252336448598131
},
"supported": {
"mean": 0.9003115264797508,
"lo": 0.8660436137071651,
"hi": 0.9314641744548287
},
"examined": 73.33644859813084,
"tokens": 804.2398753894081,
"p50": 0.833666,
"p95": 0.930375
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.33695652173913043,
"lo": 0.2391304347826087,
"hi": 0.42391304347826086
},
"any@5": {
"mean": 0.5434782608695652,
"lo": 0.44565217391304346,
"hi": 0.6413043478260869
},
"recall@5": {
"mean": 0.4299872122762149,
"lo": 0.344575873827792,
"hi": 0.5182225063938618
},
"all@10": {
"mean": 0.3804347826086957,
"lo": 0.2826086956521739,
"hi": 0.4782608695652174
},
"any@10": {
"mean": 0.6195652173913043,
"lo": 0.5217391304347826,
"hi": 0.717391304347826
},
"recall@10": {
"mean": 0.479539641943734,
"lo": 0.38858695652173914,
"hi": 0.5652173913043479
},
"all@20": {
"mean": 0.44565217391304346,
"lo": 0.34782608695652173,
"hi": 0.5434782608695652
},
"any@20": {
"mean": 0.6847826086956522,
"lo": 0.5869565217391305,
"hi": 0.7717391304347826
},
"recall@20": {
"mean": 0.5368761140819964,
"lo": 0.44405855227466484,
"hi": 0.621376811594203
},
"mrr": {
"mean": 0.38323064314912153,
"lo": 0.3074844388703084,
"hi": 0.46476913786696405
},
"ndcg@5": {
"mean": 0.3488133003703941,
"lo": 0.27454512871061104,
"hi": 0.4255917007439612
},
"ndcg@10": {
"mean": 0.3673963896894526,
"lo": 0.2953490942098949,
"hi": 0.4409483863739728
},
"ndcg@20": {
"mean": 0.38612528021220527,
"lo": 0.31391787549012845,
"hi": 0.4606605123610225
},
"exact": {
"mean": 0.6847826086956522,
"lo": 0.5869565217391305,
"hi": 0.7717391304347826
},
"supported": {
"mean": 0.6956521739130435,
"lo": 0.5978260869565217,
"hi": 0.782608695652174
},
"examined": 77.83695652173913,
"tokens": 733.4565217391304,
"p50": 0.848916,
"p95": 0.969375
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.7241379310344828,
"lo": 0.6920332936979786,
"hi": 0.7526753864447087
},
"any@5": {
"mean": 0.7431629013079667,
"lo": 0.7122473246135553,
"hi": 0.7717003567181926
},
"recall@5": {
"mean": 0.7338485929449069,
"lo": 0.7017439556084027,
"hi": 0.7623860483551327
},
"all@10": {
"mean": 0.8026159334126041,
"lo": 0.7752675386444708,
"hi": 0.8299643281807372
},
"any@10": {
"mean": 0.8240190249702735,
"lo": 0.797859690844233,
"hi": 0.8501783590963139
},
"recall@10": {
"mean": 0.8133174791914387,
"lo": 0.7875544986127626,
"hi": 0.839476813317479
},
"all@20": {
"mean": 0.8418549346016647,
"lo": 0.8156956004756243,
"hi": 0.8668252080856124
},
"any@20": {
"mean": 0.8632580261593341,
"lo": 0.8382877526753865,
"hi": 0.8858501783590963
},
"recall@20": {
"mean": 0.8529528339278636,
"lo": 0.827982560443916,
"hi": 0.8769322235434007
},
"mrr": {
"mean": 0.59431907827495,
"lo": 0.567923170046867,
"hi": 0.6217656584198329
},
"ndcg@5": {
"mean": 0.6154626450982685,
"lo": 0.5878391828234506,
"hi": 0.6421302011719054
},
"ndcg@10": {
"mean": 0.6418285102633962,
"lo": 0.616575507017145,
"hi": 0.6667961361953328
},
"ndcg@20": {
"mean": 0.6521453070378274,
"lo": 0.628098485584856,
"hi": 0.6764483354406272
},
"exact": {
"mean": 0.8632580261593341,
"lo": 0.8382877526753865,
"hi": 0.8858501783590963
},
"supported": {
"mean": 0.8787158145065398,
"lo": 0.8549346016646849,
"hi": 0.9001189060642093
},
"examined": 73.8359096313912,
"tokens": 776.8145065398336,
"p50": 0.829625,
"p95": 0.943708
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.5983072916666666,
"lo": 0.57421875,
"hi": 0.623046875
},
"any@5": {
"mean": 0.73046875,
"lo": 0.7102864583333334,
"hi": 0.7526041666666666
},
"recall@5": {
"mean": 0.6574488474387793,
"lo": 0.6366685831529583,
"hi": 0.6791428049697085
},
"all@10": {
"mean": 0.6751302083333334,
"lo": 0.6510416666666666,
"hi": 0.6985677083333334
},
"any@10": {
"mean": 0.8138020833333334,
"lo": 0.794921875,
"hi": 0.8333333333333334
},
"recall@10": {
"mean": 0.7400707188298793,
"lo": 0.7198756151766265,
"hi": 0.7603860965987274
},
"all@20": {
"mean": 0.7337239583333334,
"lo": 0.7115885416666666,
"hi": 0.755859375
},
"any@20": {
"mean": 0.86328125,
"lo": 0.845703125,
"hi": 0.8795572916666666
},
"recall@20": {
"mean": 0.796732237448242,
"lo": 0.7775335653121723,
"hi": 0.8143594351803224
},
"mrr": {
"mean": 0.5835353108001504,
"lo": 0.5649756854023954,
"hi": 0.6049196897104847
},
"ndcg@5": {
"mean": 0.5670477965448777,
"lo": 0.5482820606434095,
"hi": 0.5876925919778916
},
"ndcg@10": {
"mean": 0.596742648571549,
"lo": 0.5782365022668652,
"hi": 0.6162300045376051
},
"ndcg@20": {
"mean": 0.6140650083085327,
"lo": 0.5969190277399955,
"hi": 0.6327275531316976
},
"exact": {
"mean": 0.86328125,
"lo": 0.845703125,
"hi": 0.8795572916666666
},
"supported": {
"mean": 0.8776041666666666,
"lo": 0.861328125,
"hi": 0.8932291666666666
},
"examined": 74.11848958333333,
"tokens": 782.1399739583334,
"p50": 0.8315,
"p95": 0.945667
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-adjacency-locomo-minilm",
"row": "+adjacency",
"split": "all",
"k": 10,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "all-minilm",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:53.889Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.13829787234042554,
"lo": 0.09574468085106383,
"hi": 0.18085106382978725
},
"any@5": {
"mean": 0.6063829787234043,
"lo": 0.549645390070922,
"hi": 0.6631205673758865
},
"recall@5": {
"mean": 0.33177722540824434,
"lo": 0.29175875294296333,
"hi": 0.37451985460104137
},
"all@10": {
"mean": 0.21631205673758866,
"lo": 0.1702127659574468,
"hi": 0.26595744680851063
},
"any@10": {
"mean": 0.7446808510638298,
"lo": 0.6914893617021277,
"hi": 0.7943262411347518
},
"recall@10": {
"mean": 0.44965125445248627,
"lo": 0.40495914593059046,
"hi": 0.49445898757208934
},
"all@20": {
"mean": 0.21631205673758866,
"lo": 0.1702127659574468,
"hi": 0.26595744680851063
},
"any@20": {
"mean": 0.7446808510638298,
"lo": 0.6914893617021277,
"hi": 0.7943262411347518
},
"recall@20": {
"mean": 0.44965125445248627,
"lo": 0.40495914593059046,
"hi": 0.49445898757208934
},
"mrr": {
"mean": 0.4451212991106605,
"lo": 0.39677473826409976,
"hi": 0.49334824946527067
},
"ndcg@5": {
"mean": 0.31384132239633733,
"lo": 0.27589643844443096,
"hi": 0.3547876996703277
},
"ndcg@10": {
"mean": 0.36112199071254336,
"lo": 0.3238667682511073,
"hi": 0.4009750676774452
},
"ndcg@20": {
"mean": 0.3605063850926038,
"lo": 0.3230689820898773,
"hi": 0.39983694654785057
},
"exact": {
"mean": 0.7446808510638298,
"lo": 0.6914893617021277,
"hi": 0.7943262411347518
},
"supported": {
"mean": 0.7588652482269503,
"lo": 0.7092198581560284,
"hi": 0.8049645390070922
},
"examined": 77.11702127659575,
"tokens": 368.51418439716315,
"p50": 0.410208,
"p95": 0.493667
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.660436137071651,
"lo": 0.6074766355140186,
"hi": 0.7165109034267912
},
"any@5": {
"mean": 0.7258566978193146,
"lo": 0.6728971962616822,
"hi": 0.7757009345794392
},
"recall@5": {
"mean": 0.6934060228452751,
"lo": 0.6448598130841122,
"hi": 0.7471443406022845
},
"all@10": {
"mean": 0.7258566978193146,
"lo": 0.6728971962616822,
"hi": 0.7757009345794392
},
"any@10": {
"mean": 0.794392523364486,
"lo": 0.7445482866043613,
"hi": 0.838006230529595
},
"recall@10": {
"mean": 0.7609034267912772,
"lo": 0.7100207684319834,
"hi": 0.8060747663551402
},
"all@20": {
"mean": 0.7258566978193146,
"lo": 0.6728971962616822,
"hi": 0.7757009345794392
},
"any@20": {
"mean": 0.794392523364486,
"lo": 0.7445482866043613,
"hi": 0.838006230529595
},
"recall@20": {
"mean": 0.7609034267912772,
"lo": 0.7100207684319834,
"hi": 0.8060747663551402
},
"mrr": {
"mean": 0.591236463432725,
"lo": 0.5477785195074913,
"hi": 0.6364201651584829
},
"ndcg@5": {
"mean": 0.5985029627401264,
"lo": 0.5573659625066172,
"hi": 0.6443253296070415
},
"ndcg@10": {
"mean": 0.6212745413137054,
"lo": 0.5796426833175173,
"hi": 0.6641861762320057
},
"ndcg@20": {
"mean": 0.6212745413137054,
"lo": 0.5796426833175173,
"hi": 0.6641861762320057
},
"exact": {
"mean": 0.794392523364486,
"lo": 0.7445482866043613,
"hi": 0.838006230529595
},
"supported": {
"mean": 0.8037383177570093,
"lo": 0.7538940809968847,
"hi": 0.8473520249221184
},
"examined": 77.20560747663552,
"tokens": 359.6448598130841,
"p50": 0.4175,
"p95": 0.475084
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.22826086956521738,
"lo": 0.14130434782608695,
"hi": 0.31521739130434784
},
"any@5": {
"mean": 0.43478260869565216,
"lo": 0.33695652173913043,
"hi": 0.532608695652174
},
"recall@5": {
"mean": 0.31947996589940325,
"lo": 0.23769181585677748,
"hi": 0.40217391304347827
},
"all@10": {
"mean": 0.2717391304347826,
"lo": 0.18478260869565216,
"hi": 0.3695652173913043
},
"any@10": {
"mean": 0.5,
"lo": 0.40217391304347827,
"hi": 0.5978260869565217
},
"recall@10": {
"mean": 0.37356138107416886,
"lo": 0.28985507246376807,
"hi": 0.45561594202898553
},
"all@20": {
"mean": 0.2717391304347826,
"lo": 0.18478260869565216,
"hi": 0.3695652173913043
},
"any@20": {
"mean": 0.5,
"lo": 0.40217391304347827,
"hi": 0.5978260869565217
},
"recall@20": {
"mean": 0.37356138107416886,
"lo": 0.28985507246376807,
"hi": 0.45561594202898553
},
"mrr": {
"mean": 0.3088768115942028,
"lo": 0.2322463768115942,
"hi": 0.39003623188405806
},
"ndcg@5": {
"mean": 0.2721647815944582,
"lo": 0.20200848191110457,
"hi": 0.34717076434121763
},
"ndcg@10": {
"mean": 0.29324046648344543,
"lo": 0.22329021677141261,
"hi": 0.36244271810549084
},
"ndcg@20": {
"mean": 0.2927730987528042,
"lo": 0.22243328640479898,
"hi": 0.36241885604831564
},
"exact": {
"mean": 0.5,
"lo": 0.40217391304347827,
"hi": 0.5978260869565217
},
"supported": {
"mean": 0.5217391304347826,
"lo": 0.41304347826086957,
"hi": 0.6195652173913043
},
"examined": 78.3804347826087,
"tokens": 341.5326086956522,
"p50": 0.428166,
"p95": 0.48425
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.6706302021403091,
"lo": 0.6397146254458977,
"hi": 0.703923900118906
},
"any@5": {
"mean": 0.6944114149821641,
"lo": 0.6623067776456599,
"hi": 0.7265160523186682
},
"recall@5": {
"mean": 0.6827189853349188,
"lo": 0.6521997621878716,
"hi": 0.7146254458977408
},
"all@10": {
"mean": 0.7455410225921522,
"lo": 0.7146254458977408,
"hi": 0.7740784780023782
},
"any@10": {
"mean": 0.7645659928656362,
"lo": 0.7360285374554102,
"hi": 0.7931034482758621
},
"recall@10": {
"mean": 0.7552516845025764,
"lo": 0.7265160523186681,
"hi": 0.7847800237812128
},
"all@20": {
"mean": 0.7455410225921522,
"lo": 0.7146254458977408,
"hi": 0.7740784780023782
},
"any@20": {
"mean": 0.7645659928656362,
"lo": 0.7360285374554102,
"hi": 0.7931034482758621
},
"recall@20": {
"mean": 0.7552516845025764,
"lo": 0.7265160523186681,
"hi": 0.7847800237812128
},
"mrr": {
"mean": 0.5317847234018453,
"lo": 0.5036171979691595,
"hi": 0.5609011381008999
},
"ndcg@5": {
"mean": 0.5588070587220914,
"lo": 0.5308803972718089,
"hi": 0.5875119382918671
},
"ndcg@10": {
"mean": 0.5825290101461325,
"lo": 0.5554143276659207,
"hi": 0.6099078364942478
},
"ndcg@20": {
"mean": 0.5825290101461325,
"lo": 0.5554143276659207,
"hi": 0.6099078364942478
},
"exact": {
"mean": 0.7645659928656362,
"lo": 0.7360285374554102,
"hi": 0.7931034482758621
},
"supported": {
"mean": 0.7752675386444708,
"lo": 0.7455410225921522,
"hi": 0.8038049940546967
},
"examined": 76.57788347205708,
"tokens": 361.871581450654,
"p50": 0.422,
"p95": 0.486875
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.5442708333333334,
"lo": 0.5188802083333334,
"hi": 0.5703125
},
"any@5": {
"mean": 0.6692708333333334,
"lo": 0.6477864583333334,
"hi": 0.6946614583333334
},
"recall@5": {
"mean": 0.5987651916848111,
"lo": 0.5767857847447694,
"hi": 0.6223155537114333
},
"all@10": {
"mean": 0.6158854166666666,
"lo": 0.5924479166666666,
"hi": 0.640625
},
"any@10": {
"mean": 0.7513020833333334,
"lo": 0.7311197916666666,
"hi": 0.7734375
},
"recall@10": {
"mean": 0.6774648225788354,
"lo": 0.65609202752697,
"hi": 0.699643262097004
},
"all@20": {
"mean": 0.6158854166666666,
"lo": 0.5924479166666666,
"hi": 0.640625
},
"any@20": {
"mean": 0.7513020833333334,
"lo": 0.7311197916666666,
"hi": 0.7734375
},
"recall@20": {
"mean": 0.6774648225788354,
"lo": 0.65609202752697,
"hi": 0.699643262097004
},
"mrr": {
"mean": 0.5149470899470906,
"lo": 0.4952887318121699,
"hi": 0.5364433490410067
},
"ndcg@5": {
"mean": 0.5049600262026789,
"lo": 0.4845486776249875,
"hi": 0.5265352808267991
},
"ndcg@10": {
"mean": 0.5326500973906306,
"lo": 0.5131742466833328,
"hi": 0.5525642093539673
},
"ndcg@20": {
"mean": 0.5325090826666451,
"lo": 0.5130332319593471,
"hi": 0.5523436204944483
},
"exact": {
"mean": 0.7513020833333334,
"lo": 0.7311197916666666,
"hi": 0.7734375
},
"supported": {
"mean": 0.7630208333333334,
"lo": 0.7434895833333334,
"hi": 0.7838541666666666
},
"examined": 76.916015625,
"tokens": 361.4075520833333,
"p50": 0.419166,
"p95": 0.486875
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-facts-locomo",
"row": "+facts",
"split": "all",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:40.125Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.15602836879432624,
"lo": 0.1099290780141844,
"hi": 0.20212765957446807
},
"any@5": {
"mean": 0.6595744680851063,
"lo": 0.6063829787234043,
"hi": 0.7163120567375887
},
"recall@5": {
"mean": 0.3784253172691024,
"lo": 0.3371049049660473,
"hi": 0.41994704416316997
},
"all@10": {
"mean": 0.25177304964539005,
"lo": 0.19858156028368795,
"hi": 0.30141843971631205
},
"any@10": {
"mean": 0.8120567375886525,
"lo": 0.7659574468085106,
"hi": 0.8581560283687943
},
"recall@10": {
"mean": 0.5127955958023148,
"lo": 0.4697313009043134,
"hi": 0.5547926540507728
},
"all@20": {
"mean": 0.36879432624113473,
"lo": 0.31560283687943264,
"hi": 0.4326241134751773
},
"any@20": {
"mean": 0.8829787234042553,
"lo": 0.8439716312056738,
"hi": 0.9184397163120568
},
"recall@20": {
"mean": 0.6268237755359815,
"lo": 0.586611172515428,
"hi": 0.668644781144781
},
"mrr": {
"mean": 0.49882458112276634,
"lo": 0.4530576692810735,
"hi": 0.5486985708747286
},
"ndcg@5": {
"mean": 0.35804983448867833,
"lo": 0.319253889604083,
"hi": 0.3969424608487085
},
"ndcg@10": {
"mean": 0.4139198294630372,
"lo": 0.3769989198621411,
"hi": 0.45065282364439163
},
"ndcg@20": {
"mean": 0.45418003087496917,
"lo": 0.4186971800949777,
"hi": 0.4900268331024795
},
"exact": {
"mean": 0.8829787234042553,
"lo": 0.8439716312056738,
"hi": 0.9184397163120568
},
"supported": {
"mean": 0.900709219858156,
"lo": 0.8617021276595744,
"hi": 0.9326241134751773
},
"examined": 70.97163120567376,
"tokens": 792.1914893617021,
"p50": 0.814959,
"p95": 1.027958
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.7165109034267912,
"lo": 0.6666666666666666,
"hi": 0.7663551401869159
},
"any@5": {
"mean": 0.7850467289719626,
"lo": 0.735202492211838,
"hi": 0.8286604361370716
},
"recall@5": {
"mean": 0.7525960539979231,
"lo": 0.7045690550363448,
"hi": 0.798546209761163
},
"all@10": {
"mean": 0.778816199376947,
"lo": 0.7320872274143302,
"hi": 0.822429906542056
},
"any@10": {
"mean": 0.8442367601246106,
"lo": 0.8006230529595015,
"hi": 0.8785046728971962
},
"recall@10": {
"mean": 0.8141225337487019,
"lo": 0.7715472481827622,
"hi": 0.8504672897196262
},
"all@20": {
"mean": 0.8255451713395638,
"lo": 0.7850467289719626,
"hi": 0.8660436137071651
},
"any@20": {
"mean": 0.8847352024922118,
"lo": 0.8473520249221184,
"hi": 0.9158878504672897
},
"recall@20": {
"mean": 0.856178608515057,
"lo": 0.8187954309449634,
"hi": 0.8894080996884736
},
"mrr": {
"mean": 0.6658630236763827,
"lo": 0.6194195052702471,
"hi": 0.7114160631683998
},
"ndcg@5": {
"mean": 0.6670960773442423,
"lo": 0.6222683683436729,
"hi": 0.7132155644781653
},
"ndcg@10": {
"mean": 0.6883375050188185,
"lo": 0.6460833467504105,
"hi": 0.7316830387638942
},
"ndcg@20": {
"mean": 0.6999622362783545,
"lo": 0.6591947052001706,
"hi": 0.7411160873037704
},
"exact": {
"mean": 0.8847352024922118,
"lo": 0.8473520249221184,
"hi": 0.9158878504672897
},
"supported": {
"mean": 0.8909657320872274,
"lo": 0.8566978193146417,
"hi": 0.9221183800623053
},
"examined": 69.68535825545172,
"tokens": 808.4672897196261,
"p50": 0.824542,
"p95": 1.000833
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.32608695652173914,
"lo": 0.22826086956521738,
"hi": 0.41304347826086957
},
"any@5": {
"mean": 0.532608695652174,
"lo": 0.43478260869565216,
"hi": 0.6304347826086957
},
"recall@5": {
"mean": 0.41911764705882354,
"lo": 0.33423913043478265,
"hi": 0.5018115942028986
},
"all@10": {
"mean": 0.3695652173913043,
"lo": 0.2717391304347826,
"hi": 0.45652173913043476
},
"any@10": {
"mean": 0.6086956521739131,
"lo": 0.5,
"hi": 0.7065217391304348
},
"recall@10": {
"mean": 0.4686700767263427,
"lo": 0.37990196078431376,
"hi": 0.5498188405797102
},
"all@20": {
"mean": 0.42391304347826086,
"lo": 0.31521739130434784,
"hi": 0.5217391304347826
},
"any@20": {
"mean": 0.6847826086956522,
"lo": 0.5869565217391305,
"hi": 0.7717391304347826
},
"recall@20": {
"mean": 0.5266459350538634,
"lo": 0.4377131287297527,
"hi": 0.610618654576455
},
"mrr": {
"mean": 0.3836170321224669,
"lo": 0.30260073561703993,
"hi": 0.4660230017566974
},
"ndcg@5": {
"mean": 0.34575283184102795,
"lo": 0.2657344189365719,
"hi": 0.42301366785867917
},
"ndcg@10": {
"mean": 0.364972985332826,
"lo": 0.28737739315169347,
"hi": 0.44085502240542396
},
"ndcg@20": {
"mean": 0.3838326518781968,
"lo": 0.3080254619768419,
"hi": 0.4601459211454409
},
"exact": {
"mean": 0.6847826086956522,
"lo": 0.5869565217391305,
"hi": 0.7717391304347826
},
"supported": {
"mean": 0.6956521739130435,
"lo": 0.5978260869565217,
"hi": 0.782608695652174
},
"examined": 73.91304347826087,
"tokens": 741.9565217391304,
"p50": 0.843834,
"p95": 1.015916
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.6813317479191439,
"lo": 0.6480380499405469,
"hi": 0.7110582639714625
},
"any@5": {
"mean": 0.7098692033293698,
"lo": 0.6789536266349584,
"hi": 0.7395957193816884
},
"recall@5": {
"mean": 0.695798652397939,
"lo": 0.6646848989298454,
"hi": 0.7249306381292113
},
"all@10": {
"mean": 0.7657550535077289,
"lo": 0.7384066587395958,
"hi": 0.7931034482758621
},
"any@10": {
"mean": 0.7883472057074911,
"lo": 0.7598097502972652,
"hi": 0.8156956004756243
},
"recall@10": {
"mean": 0.7772493063812922,
"lo": 0.7491082045184304,
"hi": 0.8045977011494252
},
"all@20": {
"mean": 0.8228299643281808,
"lo": 0.7954815695600476,
"hi": 0.8489892984542212
},
"any@20": {
"mean": 0.845422116527943,
"lo": 0.8204518430439952,
"hi": 0.8703923900118906
},
"recall@20": {
"mean": 0.8345223939754259,
"lo": 0.8091557669441143,
"hi": 0.8592944906856916
},
"mrr": {
"mean": 0.5686064273281685,
"lo": 0.5403861131074789,
"hi": 0.596240029934483
},
"ndcg@5": {
"mean": 0.584179056109971,
"lo": 0.5553567343757194,
"hi": 0.6105417405672364
},
"ndcg@10": {
"mean": 0.6115383125455156,
"lo": 0.5850127381025877,
"hi": 0.637743935789305
},
"ndcg@20": {
"mean": 0.626607591951164,
"lo": 0.6012893996486375,
"hi": 0.651790285714053
},
"exact": {
"mean": 0.845422116527943,
"lo": 0.8204518430439952,
"hi": 0.8703923900118906
},
"supported": {
"mean": 0.8596908442330559,
"lo": 0.8347205707491082,
"hi": 0.882282996432818
},
"examined": 70.46016646848989,
"tokens": 782.30677764566,
"p50": 0.836167,
"p95": 1.065834
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.5709635416666666,
"lo": 0.546875,
"hi": 0.5970052083333334
},
"any@5": {
"mean": 0.7057291666666666,
"lo": 0.6842447916666666,
"hi": 0.7278645833333334
},
"recall@5": {
"mean": 0.6328286217443347,
"lo": 0.6122348156039562,
"hi": 0.6549106737117694
},
"all@10": {
"mean": 0.650390625,
"lo": 0.626953125,
"hi": 0.673828125
},
"any@10": {
"mean": 0.7936197916666666,
"lo": 0.7727864583333334,
"hi": 0.814453125
},
"recall@10": {
"mean": 0.7179205762207527,
"lo": 0.6961097293719621,
"hi": 0.7392319935828365
},
"all@20": {
"mean": 0.7161458333333334,
"lo": 0.693359375,
"hi": 0.7376302083333334
},
"any@20": {
"mean": 0.8509114583333334,
"lo": 0.8326822916666666,
"hi": 0.8678385416666666
},
"recall@20": {
"mean": 0.7824755191359168,
"lo": 0.7627444459475711,
"hi": 0.8000453312035707
},
"mrr": {
"mean": 0.56503993152018,
"lo": 0.54550531862589,
"hi": 0.5862521572798466
},
"ndcg@5": {
"mean": 0.5457107687963331,
"lo": 0.5261140633560843,
"hi": 0.5664966299433811
},
"ndcg@10": {
"mean": 0.5765383896621187,
"lo": 0.557925319868244,
"hi": 0.5966687006539495
},
"ndcg@20": {
"mean": 0.5957397365597754,
"lo": 0.5786760609669855,
"hi": 0.6147927782003465
},
"exact": {
"mean": 0.8509114583333334,
"lo": 0.8326822916666666,
"hi": 0.8678385416666666
},
"supported": {
"mean": 0.8639322916666666,
"lo": 0.8470052083333334,
"hi": 0.8802083333333334
},
"examined": 70.59895833333333,
"tokens": 787.171875,
"p50": 0.8305,
"p95": 1.037916
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-facts-locomo-minilm",
"row": "+facts",
"split": "all",
"k": 10,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "all-minilm",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:52.827Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.14184397163120568,
"lo": 0.10283687943262411,
"hi": 0.18439716312056736
},
"any@5": {
"mean": 0.599290780141844,
"lo": 0.5425531914893617,
"hi": 0.6560283687943262
},
"recall@5": {
"mean": 0.3297086674886227,
"lo": 0.2863221884498479,
"hi": 0.37225904461346776
},
"all@10": {
"mean": 0.20921985815602837,
"lo": 0.16312056737588654,
"hi": 0.25886524822695034
},
"any@10": {
"mean": 0.7340425531914894,
"lo": 0.6843971631205674,
"hi": 0.7836879432624113
},
"recall@10": {
"mean": 0.43901295658014583,
"lo": 0.3927979806059314,
"hi": 0.483596180166729
},
"all@20": {
"mean": 0.20921985815602837,
"lo": 0.16312056737588654,
"hi": 0.25886524822695034
},
"any@20": {
"mean": 0.7340425531914894,
"lo": 0.6843971631205674,
"hi": 0.7836879432624113
},
"recall@20": {
"mean": 0.43901295658014583,
"lo": 0.3927979806059314,
"hi": 0.483596180166729
},
"mrr": {
"mean": 0.44080969267139475,
"lo": 0.39303726218619833,
"hi": 0.4884329618372171
},
"ndcg@5": {
"mean": 0.3114959898872857,
"lo": 0.27172404015536705,
"hi": 0.35185994604593424
},
"ndcg@10": {
"mean": 0.3551896753145083,
"lo": 0.3169873376736437,
"hi": 0.39532550246021014
},
"ndcg@20": {
"mean": 0.3545740696945687,
"lo": 0.3161895515124137,
"hi": 0.3946756572849495
},
"exact": {
"mean": 0.7340425531914894,
"lo": 0.6843971631205674,
"hi": 0.7836879432624113
},
"supported": {
"mean": 0.74822695035461,
"lo": 0.6985815602836879,
"hi": 0.7943262411347518
},
"examined": 72.98936170212765,
"tokens": 369.26241134751774,
"p50": 0.40875,
"p95": 0.491917
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.6542056074766355,
"lo": 0.6012461059190031,
"hi": 0.7071651090342679
},
"any@5": {
"mean": 0.7165109034267912,
"lo": 0.6666666666666666,
"hi": 0.7694704049844237
},
"recall@5": {
"mean": 0.6866562824506749,
"lo": 0.6383696780893041,
"hi": 0.7372793354101765
},
"all@10": {
"mean": 0.719626168224299,
"lo": 0.6697819314641744,
"hi": 0.7725856697819314
},
"any@10": {
"mean": 0.7881619937694704,
"lo": 0.7383177570093458,
"hi": 0.8317757009345794
},
"recall@10": {
"mean": 0.7546728971962616,
"lo": 0.7040498442367601,
"hi": 0.8006230529595015
},
"all@20": {
"mean": 0.719626168224299,
"lo": 0.6697819314641744,
"hi": 0.7725856697819314
},
"any@20": {
"mean": 0.7881619937694704,
"lo": 0.7383177570093458,
"hi": 0.8317757009345794
},
"recall@20": {
"mean": 0.7546728971962616,
"lo": 0.7040498442367601,
"hi": 0.8006230529595015
},
"mrr": {
"mean": 0.5791524501804874,
"lo": 0.5362050635415121,
"hi": 0.6248875043267568
},
"ndcg@5": {
"mean": 0.5887256368089216,
"lo": 0.5473882661655016,
"hi": 0.6341792265008279
},
"ndcg@10": {
"mean": 0.610746795714004,
"lo": 0.5692841609848157,
"hi": 0.6539432471616377
},
"ndcg@20": {
"mean": 0.610746795714004,
"lo": 0.5692841609848157,
"hi": 0.6539432471616377
},
"exact": {
"mean": 0.7881619937694704,
"lo": 0.7383177570093458,
"hi": 0.8317757009345794
},
"supported": {
"mean": 0.7975077881619937,
"lo": 0.7507788161993769,
"hi": 0.8411214953271028
},
"examined": 72.97507788161994,
"tokens": 363.4018691588785,
"p50": 0.410833,
"p95": 0.487667
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.21739130434782608,
"lo": 0.13043478260869565,
"hi": 0.30434782608695654
},
"any@5": {
"mean": 0.43478260869565216,
"lo": 0.33695652173913043,
"hi": 0.532608695652174
},
"recall@5": {
"mean": 0.31223358908780907,
"lo": 0.23161764705882354,
"hi": 0.3885869565217392
},
"all@10": {
"mean": 0.25,
"lo": 0.16304347826086957,
"hi": 0.33695652173913043
},
"any@10": {
"mean": 0.4782608695652174,
"lo": 0.3804347826086957,
"hi": 0.5760869565217391
},
"recall@10": {
"mean": 0.35182225063938627,
"lo": 0.2666773231031543,
"hi": 0.42700341005967607
},
"all@20": {
"mean": 0.25,
"lo": 0.16304347826086957,
"hi": 0.33695652173913043
},
"any@20": {
"mean": 0.4782608695652174,
"lo": 0.3804347826086957,
"hi": 0.5760869565217391
},
"recall@20": {
"mean": 0.35182225063938627,
"lo": 0.2666773231031543,
"hi": 0.42700341005967607
},
"mrr": {
"mean": 0.2940217391304347,
"lo": 0.21739130434782608,
"hi": 0.3717391304347826
},
"ndcg@5": {
"mean": 0.26053421706515895,
"lo": 0.192475624430639,
"hi": 0.33240978673019417
},
"ndcg@10": {
"mean": 0.2760796957925401,
"lo": 0.20593561748064262,
"hi": 0.34642428669643033
},
"ndcg@20": {
"mean": 0.2755825529215823,
"lo": 0.205874468908378,
"hi": 0.34642428669643033
},
"exact": {
"mean": 0.4782608695652174,
"lo": 0.3804347826086957,
"hi": 0.5760869565217391
},
"supported": {
"mean": 0.5217391304347826,
"lo": 0.42391304347826086,
"hi": 0.6195652173913043
},
"examined": 74.25,
"tokens": 345.5978260869565,
"p50": 0.418375,
"p95": 0.497375
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.6504161712247325,
"lo": 0.6183115338882283,
"hi": 0.6813317479191439
},
"any@5": {
"mean": 0.6753864447086801,
"lo": 0.6444708680142688,
"hi": 0.7074910820451843
},
"recall@5": {
"mean": 0.6630994847403885,
"lo": 0.6311930241775663,
"hi": 0.6936187078874356
},
"all@10": {
"mean": 0.7312722948870393,
"lo": 0.7003567181926278,
"hi": 0.760998810939358
},
"any@10": {
"mean": 0.7562425683709869,
"lo": 0.727705112960761,
"hi": 0.7859690844233056
},
"recall@10": {
"mean": 0.7439556084026953,
"lo": 0.7148236226714229,
"hi": 0.7728894173602854
},
"all@20": {
"mean": 0.7312722948870393,
"lo": 0.7003567181926278,
"hi": 0.760998810939358
},
"any@20": {
"mean": 0.7562425683709869,
"lo": 0.727705112960761,
"hi": 0.7859690844233056
},
"recall@20": {
"mean": 0.7439556084026953,
"lo": 0.7148236226714229,
"hi": 0.7728894173602854
},
"mrr": {
"mean": 0.5163741200007548,
"lo": 0.488997886114414,
"hi": 0.5455240360115502
},
"ndcg@5": {
"mean": 0.5403970052139379,
"lo": 0.5125466314168684,
"hi": 0.5691151080280196
},
"ndcg@10": {
"mean": 0.5669616279169497,
"lo": 0.5399011006595326,
"hi": 0.5945311190540494
},
"ndcg@20": {
"mean": 0.5669616279169497,
"lo": 0.5399011006595326,
"hi": 0.5945311190540494
},
"exact": {
"mean": 0.7562425683709869,
"lo": 0.727705112960761,
"hi": 0.7859690844233056
},
"supported": {
"mean": 0.7681331747919143,
"lo": 0.7384066587395958,
"hi": 0.7954815695600476
},
"examined": 72.66587395957194,
"tokens": 364.67776456599285,
"p50": 0.420125,
"p95": 0.50575
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.5319010416666666,
"lo": 0.5071614583333334,
"hi": 0.5572916666666666
},
"any@5": {
"mean": 0.6555989583333334,
"lo": 0.6334635416666666,
"hi": 0.6783854166666666
},
"recall@5": {
"mean": 0.5857986118237,
"lo": 0.5640059061819173,
"hi": 0.6089268062438296
},
"all@10": {
"mean": 0.6041666666666666,
"lo": 0.5813802083333334,
"hi": 0.6302083333333334
},
"any@10": {
"mean": 0.7421875,
"lo": 0.7213541666666666,
"hi": 0.763671875
},
"recall@10": {
"mean": 0.6667226350788354,
"lo": 0.6453447706833931,
"hi": 0.688848371223988
},
"all@20": {
"mean": 0.6041666666666666,
"lo": 0.5813802083333334,
"hi": 0.6302083333333334
},
"any@20": {
"mean": 0.7421875,
"lo": 0.7213541666666666,
"hi": 0.763671875
},
"recall@20": {
"mean": 0.6667226350788354,
"lo": 0.6453447706833931,
"hi": 0.688848371223988
},
"mrr": {
"mean": 0.5023026723710321,
"lo": 0.48160342261904815,
"hi": 0.5237327938988102
},
"ndcg@5": {
"mean": 0.49170952338463053,
"lo": 0.4715256905375293,
"hi": 0.5122302824612829
},
"ndcg@10": {
"mean": 0.5198094211939798,
"lo": 0.5007460244576395,
"hi": 0.5394914416094047
},
"ndcg@20": {
"mean": 0.5196666230631524,
"lo": 0.5005507850720555,
"hi": 0.539378420265119
},
"exact": {
"mean": 0.7421875,
"lo": 0.7213541666666666,
"hi": 0.763671875
},
"supported": {
"mean": 0.755859375,
"lo": 0.7356770833333334,
"hi": 0.77734375
},
"examined": 72.884765625,
"tokens": 364.1100260416667,
"p50": 0.41625,
"p95": 0.501792
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-failed-global-rrf-locomo",
"row": "failed: global RRF",
"split": "all",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:49.797Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.10638297872340426,
"lo": 0.07092198581560284,
"hi": 0.1453900709219858
},
"any@5": {
"mean": 0.6666666666666666,
"lo": 0.6170212765957447,
"hi": 0.7198581560283688
},
"recall@5": {
"mean": 0.33617261642177665,
"lo": 0.2966045432730762,
"hi": 0.3736519193966002
},
"all@10": {
"mean": 0.24468085106382978,
"lo": 0.19148936170212766,
"hi": 0.2978723404255319
},
"any@10": {
"mean": 0.7872340425531915,
"lo": 0.7375886524822695,
"hi": 0.8368794326241135
},
"recall@10": {
"mean": 0.4962747711207957,
"lo": 0.4501541504201078,
"hi": 0.5415253357442609
},
"all@20": {
"mean": 0.33687943262411346,
"lo": 0.2801418439716312,
"hi": 0.3971631205673759
},
"any@20": {
"mean": 0.8900709219858156,
"lo": 0.851063829787234,
"hi": 0.9290780141843972
},
"recall@20": {
"mean": 0.6124988015385552,
"lo": 0.5704361241595283,
"hi": 0.6549414652578149
},
"mrr": {
"mean": 0.47964464758113057,
"lo": 0.43262626361030615,
"hi": 0.5256016850275299
},
"ndcg@5": {
"mean": 0.3231759933977223,
"lo": 0.2859251049725175,
"hi": 0.3625421794602101
},
"ndcg@10": {
"mean": 0.3898386454905944,
"lo": 0.3527343705621711,
"hi": 0.4292144858084069
},
"ndcg@20": {
"mean": 0.42936621834983685,
"lo": 0.39411386024180317,
"hi": 0.4645137922182303
},
"exact": {
"mean": 0.8900709219858156,
"lo": 0.851063829787234,
"hi": 0.9290780141843972
},
"supported": {
"mean": 0.9078014184397163,
"lo": 0.8723404255319149,
"hi": 0.9432624113475178
},
"examined": 137.45744680851064,
"tokens": 766.3368794326241,
"p50": 1.817625,
"p95": 2.102875
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.7133956386292835,
"lo": 0.6635514018691588,
"hi": 0.7632398753894081
},
"any@5": {
"mean": 0.7570093457943925,
"lo": 0.7040498442367601,
"hi": 0.8037383177570093
},
"recall@5": {
"mean": 0.7365005192107995,
"lo": 0.6866562824506749,
"hi": 0.7832294911734163
},
"all@10": {
"mean": 0.794392523364486,
"lo": 0.7476635514018691,
"hi": 0.838006230529595
},
"any@10": {
"mean": 0.8535825545171339,
"lo": 0.8099688473520249,
"hi": 0.8940809968847352
},
"recall@10": {
"mean": 0.8226895119418485,
"lo": 0.7793354101765317,
"hi": 0.8624091381100728
},
"all@20": {
"mean": 0.838006230529595,
"lo": 0.794392523364486,
"hi": 0.8785046728971962
},
"any@20": {
"mean": 0.8940809968847352,
"lo": 0.8629283489096573,
"hi": 0.9252336448598131
},
"recall@20": {
"mean": 0.8665628245067496,
"lo": 0.8302180685358256,
"hi": 0.902907580477674
},
"mrr": {
"mean": 0.564189313256542,
"lo": 0.5195261690162011,
"hi": 0.6088478981450827
},
"ndcg@5": {
"mean": 0.5879804428301514,
"lo": 0.542845131931765,
"hi": 0.6323605410238466
},
"ndcg@10": {
"mean": 0.6172037062053691,
"lo": 0.5755890347317931,
"hi": 0.6585487371765604
},
"ndcg@20": {
"mean": 0.6287761352480469,
"lo": 0.5891905318601749,
"hi": 0.6684055636156414
},
"exact": {
"mean": 0.8940809968847352,
"lo": 0.8629283489096573,
"hi": 0.9252336448598131
},
"supported": {
"mean": 0.9065420560747663,
"lo": 0.8753894080996885,
"hi": 0.9345794392523364
},
"examined": 137.46105919003116,
"tokens": 775.3239875389409,
"p50": 1.806708,
"p95": 2.113459
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.2826086956521739,
"lo": 0.1956521739130435,
"hi": 0.3695652173913043
},
"any@5": {
"mean": 0.4891304347826087,
"lo": 0.391304347826087,
"hi": 0.5869565217391305
},
"recall@5": {
"mean": 0.37563938618925835,
"lo": 0.29140025575447565,
"hi": 0.46078431372549017
},
"all@10": {
"mean": 0.358695652173913,
"lo": 0.2608695652173913,
"hi": 0.44565217391304346
},
"any@10": {
"mean": 0.5652173913043478,
"lo": 0.4673913043478261,
"hi": 0.6630434782608695
},
"recall@10": {
"mean": 0.44602514919011077,
"lo": 0.360347399829497,
"hi": 0.5298913043478262
},
"all@20": {
"mean": 0.40217391304347827,
"lo": 0.30434782608695654,
"hi": 0.4891304347826087
},
"any@20": {
"mean": 0.6413043478260869,
"lo": 0.5434782608695652,
"hi": 0.7282608695652174
},
"recall@20": {
"mean": 0.49312369216461294,
"lo": 0.4031620553359684,
"hi": 0.5763824304425329
},
"mrr": {
"mean": 0.3238724182538121,
"lo": 0.24972179089026914,
"hi": 0.39925516193099314
},
"ndcg@5": {
"mean": 0.2868802369405238,
"lo": 0.22151621486035744,
"hi": 0.35200230111110564
},
"ndcg@10": {
"mean": 0.31237711191443046,
"lo": 0.24772720888559766,
"hi": 0.37775440465510535
},
"ndcg@20": {
"mean": 0.32676139670333565,
"lo": 0.26229624092851334,
"hi": 0.3934783393461619
},
"exact": {
"mean": 0.6413043478260869,
"lo": 0.5434782608695652,
"hi": 0.7282608695652174
},
"supported": {
"mean": 0.6630434782608695,
"lo": 0.5652173913043478,
"hi": 0.75
},
"examined": 142.5108695652174,
"tokens": 712.2608695652174,
"p50": 1.880458,
"p95": 2.137167
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.6872770511296076,
"lo": 0.6551724137931034,
"hi": 0.718192627824019
},
"any@5": {
"mean": 0.7063020214030915,
"lo": 0.6753864447086801,
"hi": 0.737217598097503
},
"recall@5": {
"mean": 0.6965913594926675,
"lo": 0.6647839873166865,
"hi": 0.7279032897344431
},
"all@10": {
"mean": 0.7871581450653984,
"lo": 0.7598097502972652,
"hi": 0.8156956004756243
},
"any@10": {
"mean": 0.8061831153388823,
"lo": 0.7788347205707491,
"hi": 0.8335315101070154
},
"recall@10": {
"mean": 0.7966706302021404,
"lo": 0.7685295283392787,
"hi": 0.825009908838684
},
"all@20": {
"mean": 0.8680142687277052,
"lo": 0.8442330558858502,
"hi": 0.8894173602853745
},
"any@20": {
"mean": 0.8906064209274673,
"lo": 0.8692033293697978,
"hi": 0.9108204518430439
},
"recall@20": {
"mean": 0.8797066983749504,
"lo": 0.8571145461751883,
"hi": 0.899722552516845
},
"mrr": {
"mean": 0.5279396578772041,
"lo": 0.502193791472274,
"hi": 0.5559127311118887
},
"ndcg@5": {
"mean": 0.552514432164153,
"lo": 0.5249202110718919,
"hi": 0.5817656773206705
},
"ndcg@10": {
"mean": 0.5854099804353688,
"lo": 0.5604760561415895,
"hi": 0.6120953641306569
},
"ndcg@20": {
"mean": 0.6069290151296998,
"lo": 0.5837597521214232,
"hi": 0.6312770377169584
},
"exact": {
"mean": 0.8906064209274673,
"lo": 0.8692033293697978,
"hi": 0.9108204518430439
},
"supported": {
"mean": 0.9024970273483948,
"lo": 0.882282996432818,
"hi": 0.9215219976218787
},
"examined": 136.94411414982164,
"tokens": 760.7514863258026,
"p50": 1.657916,
"p95": 2.08775
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.5618489583333334,
"lo": 0.5377604166666666,
"hi": 0.5865885416666666
},
"any@5": {
"mean": 0.6966145833333334,
"lo": 0.6751302083333334,
"hi": 0.7194010416666666
},
"recall@5": {
"mean": 0.6195374357814797,
"lo": 0.5981211897430095,
"hi": 0.6425188730795347
},
"all@10": {
"mean": 0.6634114583333334,
"lo": 0.6399739583333334,
"hi": 0.6868489583333334
},
"any@10": {
"mean": 0.7981770833333334,
"lo": 0.7779947916666666,
"hi": 0.8177083333333334
},
"recall@10": {
"mean": 0.7259551643977137,
"lo": 0.7048062449577787,
"hi": 0.7466934257795693
},
"all@20": {
"mean": 0.736328125,
"lo": 0.7141927083333334,
"hi": 0.759765625
},
"any@20": {
"mean": 0.8763020833333334,
"lo": 0.8587239583333334,
"hi": 0.8919270833333334
},
"recall@20": {
"mean": 0.8047474229902454,
"lo": 0.786382218503264,
"hi": 0.8229655541983211
},
"mrr": {
"mean": 0.5144258300307999,
"lo": 0.4947179120135754,
"hi": 0.5336567409487608
},
"ndcg@5": {
"mean": 0.5019107887599054,
"lo": 0.48248427824319884,
"hi": 0.5211851926404428
},
"ndcg@10": {
"mean": 0.5397951663818629,
"lo": 0.5208817926945056,
"hi": 0.5577321978518185
},
"ndcg@20": {
"mean": 0.5621144291732155,
"lo": 0.5446772027653415,
"hi": 0.5785255424147758
},
"exact": {
"mean": 0.8763020833333334,
"lo": 0.8587239583333334,
"hi": 0.8919270833333334
},
"supported": {
"mean": 0.8899739583333334,
"lo": 0.873046875,
"hi": 0.9049479166666666
},
"examined": 137.47981770833334,
"tokens": 761.91796875,
"p50": 1.727875,
"p95": 2.103042
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-failed-global-rrf-locomo-minilm",
"row": "failed: global RRF",
"split": "all",
"k": 10,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "all-minilm",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:57.003Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.1099290780141844,
"lo": 0.07446808510638298,
"hi": 0.14893617021276595
},
"any@5": {
"mean": 0.6276595744680851,
"lo": 0.5709219858156028,
"hi": 0.6843971631205674
},
"recall@5": {
"mean": 0.3254368526624966,
"lo": 0.28610765711829544,
"hi": 0.3665432049390614
},
"all@10": {
"mean": 0.20212765957446807,
"lo": 0.15602836879432624,
"hi": 0.25177304964539005
},
"any@10": {
"mean": 0.74822695035461,
"lo": 0.6950354609929078,
"hi": 0.7978723404255319
},
"recall@10": {
"mean": 0.44185149904634785,
"lo": 0.3951282070962921,
"hi": 0.4850380706763684
},
"all@20": {
"mean": 0.20212765957446807,
"lo": 0.15602836879432624,
"hi": 0.25177304964539005
},
"any@20": {
"mean": 0.74822695035461,
"lo": 0.6950354609929078,
"hi": 0.7978723404255319
},
"recall@20": {
"mean": 0.44185149904634785,
"lo": 0.3951282070962921,
"hi": 0.4850380706763684
},
"mrr": {
"mean": 0.41332601598559027,
"lo": 0.36832151300236404,
"hi": 0.45690926488798816
},
"ndcg@5": {
"mean": 0.2953361567522654,
"lo": 0.2613199040317171,
"hi": 0.33141295998834724
},
"ndcg@10": {
"mean": 0.3422178830279593,
"lo": 0.3069053133473708,
"hi": 0.3787475552592692
},
"ndcg@20": {
"mean": 0.34181232861718075,
"lo": 0.30625745334236176,
"hi": 0.37850787845148004
},
"exact": {
"mean": 0.74822695035461,
"lo": 0.6950354609929078,
"hi": 0.7978723404255319
},
"supported": {
"mean": 0.7659574468085106,
"lo": 0.7127659574468085,
"hi": 0.8156028368794326
},
"examined": 141.46099290780143,
"tokens": 386.7695035460993,
"p50": 0.417625,
"p95": 0.489791
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.6479750778816199,
"lo": 0.5950155763239875,
"hi": 0.7009345794392523
},
"any@5": {
"mean": 0.7102803738317757,
"lo": 0.660436137071651,
"hi": 0.7632398753894081
},
"recall@5": {
"mean": 0.6799065420560747,
"lo": 0.628245067497404,
"hi": 0.7305295950155763
},
"all@10": {
"mean": 0.7414330218068536,
"lo": 0.6884735202492211,
"hi": 0.7912772585669782
},
"any@10": {
"mean": 0.7819314641744548,
"lo": 0.7320872274143302,
"hi": 0.8286604361370716
},
"recall@10": {
"mean": 0.7619418483904464,
"lo": 0.7118380062305295,
"hi": 0.8094496365524402
},
"all@20": {
"mean": 0.7414330218068536,
"lo": 0.6884735202492211,
"hi": 0.7912772585669782
},
"any@20": {
"mean": 0.7819314641744548,
"lo": 0.7320872274143302,
"hi": 0.8286604361370716
},
"recall@20": {
"mean": 0.7619418483904464,
"lo": 0.7118380062305295,
"hi": 0.8094496365524402
},
"mrr": {
"mean": 0.4994004351480985,
"lo": 0.4539830885625277,
"hi": 0.5424652623250755
},
"ndcg@5": {
"mean": 0.5284135280733528,
"lo": 0.4839216728229938,
"hi": 0.5711733618113314
},
"ndcg@10": {
"mean": 0.5570986232754994,
"lo": 0.5146818464202254,
"hi": 0.5982571130911756
},
"ndcg@20": {
"mean": 0.5570986232754994,
"lo": 0.5146818464202254,
"hi": 0.5982571130911756
},
"exact": {
"mean": 0.7819314641744548,
"lo": 0.7320872274143302,
"hi": 0.8286604361370716
},
"supported": {
"mean": 0.794392523364486,
"lo": 0.7445482866043613,
"hi": 0.838006230529595
},
"examined": 144.71651090342678,
"tokens": 383.3302180685358,
"p50": 0.426417,
"p95": 0.483792
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.1956521739130435,
"lo": 0.11956521739130435,
"hi": 0.2826086956521739
},
"any@5": {
"mean": 0.41304347826086957,
"lo": 0.31521739130434784,
"hi": 0.5108695652173914
},
"recall@5": {
"mean": 0.30072463768115937,
"lo": 0.223731884057971,
"hi": 0.3840579710144928
},
"all@10": {
"mean": 0.2826086956521739,
"lo": 0.18478260869565216,
"hi": 0.3695652173913043
},
"any@10": {
"mean": 0.5217391304347826,
"lo": 0.42391304347826086,
"hi": 0.6195652173913043
},
"recall@10": {
"mean": 0.3967391304347826,
"lo": 0.3134057971014493,
"hi": 0.4782608695652174
},
"all@20": {
"mean": 0.2826086956521739,
"lo": 0.18478260869565216,
"hi": 0.3695652173913043
},
"any@20": {
"mean": 0.5217391304347826,
"lo": 0.42391304347826086,
"hi": 0.6195652173913043
},
"recall@20": {
"mean": 0.3967391304347826,
"lo": 0.3134057971014493,
"hi": 0.4782608695652174
},
"mrr": {
"mean": 0.2681720151828847,
"lo": 0.19512163561076604,
"hi": 0.33951863354037265
},
"ndcg@5": {
"mean": 0.22952717005759646,
"lo": 0.17016675190901823,
"hi": 0.29250324964498636
},
"ndcg@10": {
"mean": 0.2643194161362118,
"lo": 0.20553561868688033,
"hi": 0.3252113944332185
},
"ndcg@20": {
"mean": 0.2643194161362118,
"lo": 0.20553561868688033,
"hi": 0.3252113944332185
},
"exact": {
"mean": 0.5217391304347826,
"lo": 0.42391304347826086,
"hi": 0.6195652173913043
},
"supported": {
"mean": 0.5434782608695652,
"lo": 0.44565217391304346,
"hi": 0.6413043478260869
},
"examined": 144.8586956521739,
"tokens": 364.85869565217394,
"p50": 0.440708,
"p95": 0.5045
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.633769322235434,
"lo": 0.6016646848989299,
"hi": 0.6682520808561236
},
"any@5": {
"mean": 0.6587395957193817,
"lo": 0.6266349583828775,
"hi": 0.6932223543400713
},
"recall@5": {
"mean": 0.64645263575109,
"lo": 0.6141498216409037,
"hi": 0.6809353943717796
},
"all@10": {
"mean": 0.7336504161712247,
"lo": 0.703923900118906,
"hi": 0.7645659928656362
},
"any@10": {
"mean": 0.7574316290130797,
"lo": 0.727705112960761,
"hi": 0.7871581450653984
},
"recall@10": {
"mean": 0.7455410225921522,
"lo": 0.7168053904082441,
"hi": 0.7748711850971066
},
"all@20": {
"mean": 0.7336504161712247,
"lo": 0.703923900118906,
"hi": 0.7645659928656362
},
"any@20": {
"mean": 0.7574316290130797,
"lo": 0.727705112960761,
"hi": 0.7871581450653984
},
"recall@20": {
"mean": 0.7455410225921522,
"lo": 0.7168053904082441,
"hi": 0.7748711850971066
},
"mrr": {
"mean": 0.46843515844704864,
"lo": 0.441765283204046,
"hi": 0.497761546156314
},
"ndcg@5": {
"mean": 0.4984177516553171,
"lo": 0.47074223363845413,
"hi": 0.5279911005906487
},
"ndcg@10": {
"mean": 0.5311514344483234,
"lo": 0.5055717032429449,
"hi": 0.5588872915442351
},
"ndcg@20": {
"mean": 0.5311514344483234,
"lo": 0.5055717032429449,
"hi": 0.5588872915442351
},
"exact": {
"mean": 0.7574316290130797,
"lo": 0.727705112960761,
"hi": 0.7871581450653984
},
"supported": {
"mean": 0.7705112960760999,
"lo": 0.7419738406658739,
"hi": 0.8014268727705113
},
"examined": 141.89417360285375,
"tokens": 381.0523186682521,
"p50": 0.429,
"p95": 0.48925
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.5143229166666666,
"lo": 0.4889322916666667,
"hi": 0.5403645833333334
},
"any@5": {
"mean": 0.6490885416666666,
"lo": 0.626953125,
"hi": 0.6731770833333334
},
"recall@5": {
"mean": 0.5737998214740607,
"lo": 0.5518683108398366,
"hi": 0.5975481633376373
},
"all@10": {
"mean": 0.6106770833333334,
"lo": 0.587890625,
"hi": 0.63671875
},
"any@10": {
"mean": 0.7467447916666666,
"lo": 0.7252604166666666,
"hi": 0.7682291666666666
},
"recall@10": {
"mean": 0.6723212604585962,
"lo": 0.6515283706227379,
"hi": 0.6942383170975895
},
"all@20": {
"mean": 0.6106770833333334,
"lo": 0.587890625,
"hi": 0.63671875
},
"any@20": {
"mean": 0.7467447916666666,
"lo": 0.7252604166666666,
"hi": 0.7682291666666666
},
"recall@20": {
"mean": 0.6723212604585962,
"lo": 0.6515283706227379,
"hi": 0.6942383170975895
},
"mrr": {
"mean": 0.4527937954695767,
"lo": 0.4335844494047625,
"hi": 0.47270275297619074
},
"ndcg@5": {
"mean": 0.4512964632181668,
"lo": 0.4329189759661802,
"hi": 0.4713155941483333
},
"ndcg@10": {
"mean": 0.4859048461854749,
"lo": 0.4678396716670474,
"hi": 0.5044367788092247
},
"ndcg@20": {
"mean": 0.48583038893037106,
"lo": 0.4678246859358408,
"hi": 0.5043854742565462
},
"exact": {
"mean": 0.7467447916666666,
"lo": 0.7252604166666666,
"hi": 0.7682291666666666
},
"supported": {
"mean": 0.7610677083333334,
"lo": 0.7389322916666666,
"hi": 0.7825520833333334
},
"examined": 142.58203125,
"tokens": 381.6080729166667,
"p50": 0.426792,
"p95": 0.491375
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-iterative-hops-locomo",
"row": "+iterative hops",
"split": "all",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:46.669Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.17375886524822695,
"lo": 0.13120567375886524,
"hi": 0.2198581560283688
},
"any@5": {
"mean": 0.6879432624113475,
"lo": 0.6347517730496454,
"hi": 0.7446808510638298
},
"recall@5": {
"mean": 0.39114483770138864,
"lo": 0.3485971268166116,
"hi": 0.4377199042092658
},
"all@10": {
"mean": 0.2695035460992908,
"lo": 0.2198581560283688,
"hi": 0.3262411347517731
},
"any@10": {
"mean": 0.7978723404255319,
"lo": 0.74822695035461,
"hi": 0.8439716312056738
},
"recall@10": {
"mean": 0.5210654712754379,
"lo": 0.4779094132817535,
"hi": 0.5667677616025881
},
"all@20": {
"mean": 0.3900709219858156,
"lo": 0.3333333333333333,
"hi": 0.45390070921985815
},
"any@20": {
"mean": 0.8794326241134752,
"lo": 0.8404255319148937,
"hi": 0.9184397163120568
},
"recall@20": {
"mean": 0.6315383478037453,
"lo": 0.5892845629547756,
"hi": 0.6734203133559239
},
"mrr": {
"mean": 0.5262498676202482,
"lo": 0.4784057907378173,
"hi": 0.5771365056079498
},
"ndcg@5": {
"mean": 0.3780334912627326,
"lo": 0.3396787873496082,
"hi": 0.42036243342882973
},
"ndcg@10": {
"mean": 0.43130557404097647,
"lo": 0.39247205592152073,
"hi": 0.4718915908508174
},
"ndcg@20": {
"mean": 0.4704082639994522,
"lo": 0.43153702653024706,
"hi": 0.5096304855903746
},
"exact": {
"mean": 0.8794326241134752,
"lo": 0.8404255319148937,
"hi": 0.9184397163120568
},
"supported": {
"mean": 0.8971631205673759,
"lo": 0.8617021276595744,
"hi": 0.9326241134751773
},
"examined": 76.79078014184397,
"tokens": 792.4113475177305,
"p50": 1.635958,
"p95": 1.974875
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.7538940809968847,
"lo": 0.7009345794392523,
"hi": 0.8006230529595015
},
"any@5": {
"mean": 0.822429906542056,
"lo": 0.7757009345794392,
"hi": 0.8629283489096573
},
"recall@5": {
"mean": 0.788681204569055,
"lo": 0.7429906542056075,
"hi": 0.8296988577362409
},
"all@10": {
"mean": 0.8099688473520249,
"lo": 0.7663551401869159,
"hi": 0.8504672897196262
},
"any@10": {
"mean": 0.8660436137071651,
"lo": 0.8255451713395638,
"hi": 0.9003115264797508
},
"recall@10": {
"mean": 0.8374870197300105,
"lo": 0.7969885773624091,
"hi": 0.8733125649013501
},
"all@20": {
"mean": 0.8442367601246106,
"lo": 0.8006230529595015,
"hi": 0.881619937694704
},
"any@20": {
"mean": 0.8847352024922118,
"lo": 0.8473520249221184,
"hi": 0.9190031152647975
},
"recall@20": {
"mean": 0.8639667705088268,
"lo": 0.8239875389408098,
"hi": 0.8977154724818277
},
"mrr": {
"mean": 0.6984800446679442,
"lo": 0.6543287736475144,
"hi": 0.7407966957893175
},
"ndcg@5": {
"mean": 0.7036915102489739,
"lo": 0.6604307708878624,
"hi": 0.7439738420540648
},
"ndcg@10": {
"mean": 0.7204511441293081,
"lo": 0.6794019325622825,
"hi": 0.758939686145592
},
"ndcg@20": {
"mean": 0.7279922917893658,
"lo": 0.6865400516705366,
"hi": 0.7653775401991423
},
"exact": {
"mean": 0.8847352024922118,
"lo": 0.8473520249221184,
"hi": 0.9190031152647975
},
"supported": {
"mean": 0.8940809968847352,
"lo": 0.8566978193146417,
"hi": 0.9252336448598131
},
"examined": 75.34267912772586,
"tokens": 814.7445482866044,
"p50": 1.63725,
"p95": 2.008584
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.2826086956521739,
"lo": 0.1956521739130435,
"hi": 0.3695652173913043
},
"any@5": {
"mean": 0.5108695652173914,
"lo": 0.40217391304347827,
"hi": 0.6086956521739131
},
"recall@5": {
"mean": 0.38650895140664965,
"lo": 0.3031756180733163,
"hi": 0.47202685421994894
},
"all@10": {
"mean": 0.358695652173913,
"lo": 0.2717391304347826,
"hi": 0.45652173913043476
},
"any@10": {
"mean": 0.5652173913043478,
"lo": 0.4673913043478261,
"hi": 0.6630434782608695
},
"recall@10": {
"mean": 0.44629156010230175,
"lo": 0.35789641943734013,
"hi": 0.5335144927536232
},
"all@20": {
"mean": 0.45652173913043476,
"lo": 0.34782608695652173,
"hi": 0.5543478260869565
},
"any@20": {
"mean": 0.6521739130434783,
"lo": 0.5434782608695652,
"hi": 0.75
},
"recall@20": {
"mean": 0.542039641943734,
"lo": 0.44565217391304346,
"hi": 0.6308077578857629
},
"mrr": {
"mean": 0.39068502895229246,
"lo": 0.30511739610205074,
"hi": 0.4766659562375676
},
"ndcg@5": {
"mean": 0.3388180485253245,
"lo": 0.2648579010253863,
"hi": 0.41880848606796683
},
"ndcg@10": {
"mean": 0.3619988533579641,
"lo": 0.287445251429063,
"hi": 0.44226165614820206
},
"ndcg@20": {
"mean": 0.3904771158959641,
"lo": 0.3146355789824112,
"hi": 0.4683849789970929
},
"exact": {
"mean": 0.6521739130434783,
"lo": 0.5434782608695652,
"hi": 0.75
},
"supported": {
"mean": 0.6739130434782609,
"lo": 0.5652173913043478,
"hi": 0.7717391304347826
},
"examined": 80.58695652173913,
"tokens": 737.554347826087,
"p50": 1.640625,
"p95": 2.014917
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.7265160523186682,
"lo": 0.6944114149821641,
"hi": 0.7562425683709869
},
"any@5": {
"mean": 0.7479191438763377,
"lo": 0.7158145065398336,
"hi": 0.7776456599286563
},
"recall@5": {
"mean": 0.737217598097503,
"lo": 0.7061038446294093,
"hi": 0.7655568767340468
},
"all@10": {
"mean": 0.8038049940546967,
"lo": 0.7764565992865636,
"hi": 0.8287752675386445
},
"any@10": {
"mean": 0.8204518430439952,
"lo": 0.7954815695600476,
"hi": 0.845422116527943
},
"recall@10": {
"mean": 0.8125247720967101,
"lo": 0.786563614744352,
"hi": 0.8372968688069757
},
"all@20": {
"mean": 0.8680142687277052,
"lo": 0.8442330558858502,
"hi": 0.8906064209274673
},
"any@20": {
"mean": 0.8846611177170036,
"lo": 0.8632580261593341,
"hi": 0.906064209274673
},
"recall@20": {
"mean": 0.8769322235434007,
"lo": 0.8551327784383671,
"hi": 0.8975426080063417
},
"mrr": {
"mean": 0.6044913231128677,
"lo": 0.5767457768396489,
"hi": 0.632388538320542
},
"ndcg@5": {
"mean": 0.622812298921069,
"lo": 0.5950570424812236,
"hi": 0.6517861865621283
},
"ndcg@10": {
"mean": 0.647300319438318,
"lo": 0.6214799076770401,
"hi": 0.6731405666534839
},
"ndcg@20": {
"mean": 0.6641907214769633,
"lo": 0.6397265428747496,
"hi": 0.6880178906794555
},
"exact": {
"mean": 0.8846611177170036,
"lo": 0.8632580261593341,
"hi": 0.906064209274673
},
"supported": {
"mean": 0.89179548156956,
"lo": 0.8703923900118906,
"hi": 0.9120095124851367
},
"examined": 75.7705112960761,
"tokens": 782.5945303210464,
"p50": 1.651291,
"p95": 2.377875
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.6041666666666666,
"lo": 0.5813802083333334,
"hi": 0.6282552083333334
},
"any@5": {
"mean": 0.73828125,
"lo": 0.71875,
"hi": 0.759765625
},
"recall@5": {
"mean": 0.6634299052264779,
"lo": 0.642090108498626,
"hi": 0.6852074737328646
},
"all@10": {
"mean": 0.6803385416666666,
"lo": 0.65625,
"hi": 0.7044270833333334
},
"any@10": {
"mean": 0.810546875,
"lo": 0.791015625,
"hi": 0.8307291666666666
},
"recall@10": {
"mean": 0.7422955423800465,
"lo": 0.7216522049783823,
"hi": 0.7625596701665779
},
"all@20": {
"mean": 0.7506510416666666,
"lo": 0.7291666666666666,
"hi": 0.7721354166666666
},
"any@20": {
"mean": 0.8697916666666666,
"lo": 0.8528645833333334,
"hi": 0.88671875
},
"recall@20": {
"mean": 0.8091111943182375,
"lo": 0.7924974782493401,
"hi": 0.8271150287828947
},
"mrr": {
"mean": 0.5969627489640992,
"lo": 0.5766193048596492,
"hi": 0.6185202966629849
},
"ndcg@5": {
"mean": 0.5777648588430715,
"lo": 0.5575979369263989,
"hi": 0.5982473886165646
},
"ndcg@10": {
"mean": 0.6058440444672003,
"lo": 0.5868194541799111,
"hi": 0.6262510739850687
},
"ndcg@20": {
"mean": 0.6255527002192611,
"lo": 0.6075115431694088,
"hi": 0.6451486637297992
},
"exact": {
"mean": 0.8697916666666666,
"lo": 0.8528645833333334,
"hi": 0.88671875
},
"supported": {
"mean": 0.8802083333333334,
"lo": 0.8645833333333334,
"hi": 0.896484375
},
"examined": 76.15690104166667,
"tokens": 788.41796875,
"p50": 1.646208,
"p95": 2.1975
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-iterative-hops-locomo-minilm",
"row": "+iterative hops",
"split": "all",
"k": 10,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "all-minilm",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:55.935Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.16666666666666666,
"lo": 0.12411347517730496,
"hi": 0.2127659574468085
},
"any@5": {
"mean": 0.6560283687943262,
"lo": 0.6028368794326241,
"hi": 0.7092198581560284
},
"recall@5": {
"mean": 0.368805947277392,
"lo": 0.32494689065237775,
"hi": 0.41094195299122505
},
"all@10": {
"mean": 0.24113475177304963,
"lo": 0.19148936170212766,
"hi": 0.29432624113475175
},
"any@10": {
"mean": 0.7801418439716312,
"lo": 0.7304964539007093,
"hi": 0.8226950354609929
},
"recall@10": {
"mean": 0.48571848032934273,
"lo": 0.44014128109872813,
"hi": 0.5307370551351517
},
"all@20": {
"mean": 0.24113475177304963,
"lo": 0.19148936170212766,
"hi": 0.29432624113475175
},
"any@20": {
"mean": 0.7801418439716312,
"lo": 0.7304964539007093,
"hi": 0.8226950354609929
},
"recall@20": {
"mean": 0.48571848032934273,
"lo": 0.44014128109872813,
"hi": 0.5307370551351517
},
"mrr": {
"mean": 0.4885933806146569,
"lo": 0.4409405606214116,
"hi": 0.5351781492738938
},
"ndcg@5": {
"mean": 0.35311866078871745,
"lo": 0.31310874886862405,
"hi": 0.3923532877841448
},
"ndcg@10": {
"mean": 0.39928944300688035,
"lo": 0.35932285662615965,
"hi": 0.4402021855343431
},
"ndcg@20": {
"mean": 0.39856072302884915,
"lo": 0.35874276464289706,
"hi": 0.43940233579555704
},
"exact": {
"mean": 0.7801418439716312,
"lo": 0.7304964539007093,
"hi": 0.8226950354609929
},
"supported": {
"mean": 0.7943262411347518,
"lo": 0.7446808510638298,
"hi": 0.8368794326241135
},
"examined": 78.84751773049645,
"tokens": 385.63475177304963,
"p50": 0.752208,
"p95": 0.860125
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.67601246105919,
"lo": 0.6261682242990654,
"hi": 0.7289719626168224
},
"any@5": {
"mean": 0.7414330218068536,
"lo": 0.6884735202492211,
"hi": 0.7912772585669782
},
"recall@5": {
"mean": 0.708982346832814,
"lo": 0.6591381100726895,
"hi": 0.7580477673935619
},
"all@10": {
"mean": 0.735202492211838,
"lo": 0.6853582554517134,
"hi": 0.7850467289719626
},
"any@10": {
"mean": 0.794392523364486,
"lo": 0.7476635514018691,
"hi": 0.838006230529595
},
"recall@10": {
"mean": 0.7650571131879542,
"lo": 0.7188473520249222,
"hi": 0.809190031152648
},
"all@20": {
"mean": 0.735202492211838,
"lo": 0.6853582554517134,
"hi": 0.7850467289719626
},
"any@20": {
"mean": 0.794392523364486,
"lo": 0.7476635514018691,
"hi": 0.838006230529595
},
"recall@20": {
"mean": 0.7650571131879542,
"lo": 0.7188473520249222,
"hi": 0.809190031152648
},
"mrr": {
"mean": 0.6179028828561539,
"lo": 0.5730443059882314,
"hi": 0.6633931661969045
},
"ndcg@5": {
"mean": 0.6242436927016979,
"lo": 0.5805031523949948,
"hi": 0.6707236698057869
},
"ndcg@10": {
"mean": 0.6438440826718334,
"lo": 0.600545380808979,
"hi": 0.689634288219117
},
"ndcg@20": {
"mean": 0.6438440826718334,
"lo": 0.600545380808979,
"hi": 0.689634288219117
},
"exact": {
"mean": 0.794392523364486,
"lo": 0.7476635514018691,
"hi": 0.838006230529595
},
"supported": {
"mean": 0.8037383177570093,
"lo": 0.7538940809968847,
"hi": 0.8442367601246106
},
"examined": 78.7632398753894,
"tokens": 377.8317757009346,
"p50": 0.757958,
"p95": 0.84325
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.21739130434782608,
"lo": 0.13043478260869565,
"hi": 0.30434782608695654
},
"any@5": {
"mean": 0.42391304347826086,
"lo": 0.32608695652173914,
"hi": 0.5217391304347826
},
"recall@5": {
"mean": 0.30951619778346123,
"lo": 0.23044543904518328,
"hi": 0.3894927536231885
},
"all@10": {
"mean": 0.2717391304347826,
"lo": 0.18478260869565216,
"hi": 0.358695652173913
},
"any@10": {
"mean": 0.4782608695652174,
"lo": 0.3804347826086957,
"hi": 0.5760869565217391
},
"recall@10": {
"mean": 0.35997442455242973,
"lo": 0.27445652173913043,
"hi": 0.44059036658141515
},
"all@20": {
"mean": 0.2717391304347826,
"lo": 0.18478260869565216,
"hi": 0.358695652173913
},
"any@20": {
"mean": 0.4782608695652174,
"lo": 0.3804347826086957,
"hi": 0.5760869565217391
},
"recall@20": {
"mean": 0.35997442455242973,
"lo": 0.27445652173913043,
"hi": 0.44059036658141515
},
"mrr": {
"mean": 0.31485507246376804,
"lo": 0.23541666666666664,
"hi": 0.39402173913043476
},
"ndcg@5": {
"mean": 0.2759371576775228,
"lo": 0.20399805153817102,
"hi": 0.34639832641444834
},
"ndcg@10": {
"mean": 0.2958054059298273,
"lo": 0.22306651501118943,
"hi": 0.3666713965428709
},
"ndcg@20": {
"mean": 0.29529845557770623,
"lo": 0.22288747862230876,
"hi": 0.36606555405652236
},
"exact": {
"mean": 0.4782608695652174,
"lo": 0.3804347826086957,
"hi": 0.5760869565217391
},
"supported": {
"mean": 0.5,
"lo": 0.40217391304347827,
"hi": 0.5978260869565217
},
"examined": 79.81521739130434,
"tokens": 349.4021739130435,
"p50": 0.78025,
"p95": 0.863333
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.6777645659928656,
"lo": 0.6456599286563615,
"hi": 0.7086801426872771
},
"any@5": {
"mean": 0.703923900118906,
"lo": 0.6730083234244947,
"hi": 0.7348394768133175
},
"recall@5": {
"mean": 0.6910424098295681,
"lo": 0.6593341260404281,
"hi": 0.7215616329766151
},
"all@10": {
"mean": 0.7752675386444708,
"lo": 0.7455410225921522,
"hi": 0.8026159334126041
},
"any@10": {
"mean": 0.7966706302021404,
"lo": 0.7681331747919143,
"hi": 0.8228299643281808
},
"recall@10": {
"mean": 0.7861672611969878,
"lo": 0.7572334522393974,
"hi": 0.8125247720967101
},
"all@20": {
"mean": 0.7752675386444708,
"lo": 0.7455410225921522,
"hi": 0.8026159334126041
},
"any@20": {
"mean": 0.7966706302021404,
"lo": 0.7681331747919143,
"hi": 0.8228299643281808
},
"recall@20": {
"mean": 0.7861672611969878,
"lo": 0.7572334522393974,
"hi": 0.8125247720967101
},
"mrr": {
"mean": 0.5547590736651377,
"lo": 0.5263778004265518,
"hi": 0.5825745993998072
},
"ndcg@5": {
"mean": 0.574870827099968,
"lo": 0.5466450393944562,
"hi": 0.6030176821549715
},
"ndcg@10": {
"mean": 0.6066477748100206,
"lo": 0.5792788340072618,
"hi": 0.6326413057539124
},
"ndcg@20": {
"mean": 0.6066477748100206,
"lo": 0.5792788340072618,
"hi": 0.6326413057539124
},
"exact": {
"mean": 0.7966706302021404,
"lo": 0.7681331747919143,
"hi": 0.8228299643281808
},
"supported": {
"mean": 0.8049940546967895,
"lo": 0.7776456599286563,
"hi": 0.83115338882283
},
"examined": 77.99048751486326,
"tokens": 376.7502972651605,
"p50": 0.767291,
"p95": 0.86075
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.5559895833333334,
"lo": 0.5319010416666666,
"hi": 0.5807291666666666
},
"any@5": {
"mean": 0.6861979166666666,
"lo": 0.6647135416666666,
"hi": 0.7083333333333334
},
"recall@5": {
"mean": 0.6127791453960303,
"lo": 0.5911780566077443,
"hi": 0.6360486571938051
},
"all@10": {
"mean": 0.638671875,
"lo": 0.615234375,
"hi": 0.6640625
},
"any@10": {
"mean": 0.7740885416666666,
"lo": 0.7545572916666666,
"hi": 0.7955729166666666
},
"recall@10": {
"mean": 0.7010678766352201,
"lo": 0.6806027112050962,
"hi": 0.7233280762879981
},
"all@20": {
"mean": 0.638671875,
"lo": 0.615234375,
"hi": 0.6640625
},
"any@20": {
"mean": 0.7740885416666666,
"lo": 0.7545572916666666,
"hi": 0.7955729166666666
},
"recall@20": {
"mean": 0.7010678766352201,
"lo": 0.6806027112050962,
"hi": 0.7233280762879981
},
"mrr": {
"mean": 0.5414382853835985,
"lo": 0.5216949301421959,
"hi": 0.5620047433035716
},
"ndcg@5": {
"mean": 0.5265717915345479,
"lo": 0.5074071662685651,
"hi": 0.5481944279418088
},
"ndcg@10": {
"mean": 0.557733365511958,
"lo": 0.5391070757578454,
"hi": 0.5775451209279486
},
"ndcg@20": {
"mean": 0.5575692128646922,
"lo": 0.538934162636885,
"hi": 0.5773809682806831
},
"exact": {
"mean": 0.7740885416666666,
"lo": 0.7545572916666666,
"hi": 0.7955729166666666
},
"supported": {
"mean": 0.7845052083333334,
"lo": 0.7649739583333334,
"hi": 0.8053385416666666
},
"examined": 78.41861979166667,
"tokens": 376.9694010416667,
"p50": 0.763917,
"p95": 0.859125
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-lexical-locomo",
"row": "+lexical",
"split": "all",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:38.310Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.09219858156028368,
"lo": 0.06028368794326241,
"hi": 0.1276595744680851
},
"any@5": {
"mean": 0.5141843971631206,
"lo": 0.45390070921985815,
"hi": 0.574468085106383
},
"recall@5": {
"mean": 0.26724782620639287,
"lo": 0.22984900193746793,
"hi": 0.3040882347713254
},
"all@10": {
"mean": 0.14893617021276595,
"lo": 0.1099290780141844,
"hi": 0.19148936170212766
},
"any@10": {
"mean": 0.6524822695035462,
"lo": 0.599290780141844,
"hi": 0.7056737588652482
},
"recall@10": {
"mean": 0.3662516690932146,
"lo": 0.32500357518554607,
"hi": 0.40819772540097304
},
"all@20": {
"mean": 0.22695035460992907,
"lo": 0.17375886524822695,
"hi": 0.2801418439716312
},
"any@20": {
"mean": 0.7801418439716312,
"lo": 0.7304964539007093,
"hi": 0.8297872340425532
},
"recall@20": {
"mean": 0.48746697498097286,
"lo": 0.44506079027355633,
"hi": 0.5322452851988126
},
"mrr": {
"mean": 0.35577061478229055,
"lo": 0.3111462237343276,
"hi": 0.4011096759948122
},
"ndcg@5": {
"mean": 0.2424533837253265,
"lo": 0.20765701891655247,
"hi": 0.27842455561280394
},
"ndcg@10": {
"mean": 0.28326828743130944,
"lo": 0.2495611420993408,
"hi": 0.3194520480864223
},
"ndcg@20": {
"mean": 0.324802068823288,
"lo": 0.29207593389360353,
"hi": 0.36153770299607446
},
"exact": {
"mean": 0.7801418439716312,
"lo": 0.7304964539007093,
"hi": 0.8297872340425532
},
"supported": {
"mean": 0.8049645390070922,
"lo": 0.7588652482269503,
"hi": 0.8546099290780141
},
"examined": 52.755319148936174,
"tokens": 666.2446808510638,
"p50": 0.589917,
"p95": 0.668542
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.6417445482866043,
"lo": 0.5887850467289719,
"hi": 0.6978193146417445
},
"any@5": {
"mean": 0.7071651090342679,
"lo": 0.6573208722741433,
"hi": 0.7538940809968847
},
"recall@5": {
"mean": 0.6741952232606437,
"lo": 0.6264278296988578,
"hi": 0.7248182762201454
},
"all@10": {
"mean": 0.7133956386292835,
"lo": 0.6666666666666666,
"hi": 0.7632398753894081
},
"any@10": {
"mean": 0.7757009345794392,
"lo": 0.7289719626168224,
"hi": 0.8193146417445483
},
"recall@10": {
"mean": 0.7448078920041536,
"lo": 0.6998961578400831,
"hi": 0.7904984423676013
},
"all@20": {
"mean": 0.7881619937694704,
"lo": 0.7414330218068536,
"hi": 0.8317757009345794
},
"any@20": {
"mean": 0.8411214953271028,
"lo": 0.8006230529595015,
"hi": 0.8785046728971962
},
"recall@20": {
"mean": 0.8151609553478713,
"lo": 0.7725856697819314,
"hi": 0.8561786085150572
},
"mrr": {
"mean": 0.5792451743972246,
"lo": 0.5350022067485394,
"hi": 0.6257222755747106
},
"ndcg@5": {
"mean": 0.5804134401221316,
"lo": 0.5358175470381371,
"hi": 0.6270376313761218
},
"ndcg@10": {
"mean": 0.6047313578774792,
"lo": 0.5626818136983928,
"hi": 0.6485184721032711
},
"ndcg@20": {
"mean": 0.6240441156669049,
"lo": 0.5843115325395902,
"hi": 0.6669673710795354
},
"exact": {
"mean": 0.8411214953271028,
"lo": 0.8006230529595015,
"hi": 0.8785046728971962
},
"supported": {
"mean": 0.8535825545171339,
"lo": 0.8130841121495327,
"hi": 0.8909657320872274
},
"examined": 52.087227414330215,
"tokens": 667.6915887850467,
"p50": 0.596958,
"p95": 0.688417
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.20652173913043478,
"lo": 0.13043478260869565,
"hi": 0.29347826086956524
},
"any@5": {
"mean": 0.40217391304347827,
"lo": 0.30434782608695654,
"hi": 0.5
},
"recall@5": {
"mean": 0.29502344416027276,
"lo": 0.21648550724637683,
"hi": 0.37228260869565216
},
"all@10": {
"mean": 0.31521739130434784,
"lo": 0.21739130434782608,
"hi": 0.41304347826086957
},
"any@10": {
"mean": 0.4891304347826087,
"lo": 0.391304347826087,
"hi": 0.5869565217391305
},
"recall@10": {
"mean": 0.38895993179880645,
"lo": 0.3025362318840579,
"hi": 0.47431798806479114
},
"all@20": {
"mean": 0.3695652173913043,
"lo": 0.2717391304347826,
"hi": 0.45652173913043476
},
"any@20": {
"mean": 0.5760869565217391,
"lo": 0.4673913043478261,
"hi": 0.6630434782608695
},
"recall@20": {
"mean": 0.4551654653956444,
"lo": 0.36384464853134924,
"hi": 0.5439335813376733
},
"mrr": {
"mean": 0.2855780486358289,
"lo": 0.21175668581990087,
"hi": 0.3562263902509898
},
"ndcg@5": {
"mean": 0.24621552279591924,
"lo": 0.17792011369427793,
"hi": 0.3154571761778389
},
"ndcg@10": {
"mean": 0.28341897277553085,
"lo": 0.21720576484830612,
"hi": 0.357648531585174
},
"ndcg@20": {
"mean": 0.30320517963864035,
"lo": 0.23514747405799527,
"hi": 0.37248897691466
},
"exact": {
"mean": 0.5760869565217391,
"lo": 0.4673913043478261,
"hi": 0.6630434782608695
},
"supported": {
"mean": 0.6195652173913043,
"lo": 0.5217391304347826,
"hi": 0.717391304347826
},
"examined": 53.69565217391305,
"tokens": 604.0217391304348,
"p50": 0.611291,
"p95": 0.7075
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.652794292508918,
"lo": 0.619500594530321,
"hi": 0.6825208085612366
},
"any@5": {
"mean": 0.6801426872770512,
"lo": 0.6492271105826397,
"hi": 0.7098692033293698
},
"recall@5": {
"mean": 0.6666666666666667,
"lo": 0.6349583828775267,
"hi": 0.6963931827189854
},
"all@10": {
"mean": 0.7300832342449465,
"lo": 0.6979785969084423,
"hi": 0.7598097502972652
},
"any@10": {
"mean": 0.7574316290130797,
"lo": 0.7288941736028538,
"hi": 0.7847800237812128
},
"recall@10": {
"mean": 0.7439556084026953,
"lo": 0.7142290923503766,
"hi": 0.7715021799445105
},
"all@20": {
"mean": 0.8252080856123662,
"lo": 0.8002378121284186,
"hi": 0.8501783590963139
},
"any@20": {
"mean": 0.8513674197384067,
"lo": 0.8287752675386445,
"hi": 0.8739595719381689
},
"recall@20": {
"mean": 0.8382877526753865,
"lo": 0.8139120095124851,
"hi": 0.8616726119698772
},
"mrr": {
"mean": 0.5244113950014242,
"lo": 0.4973038262262069,
"hi": 0.5541539717003924
},
"ndcg@5": {
"mean": 0.542742585985731,
"lo": 0.5148223165956123,
"hi": 0.5723955383585498
},
"ndcg@10": {
"mean": 0.5679299691865375,
"lo": 0.5422253193299387,
"hi": 0.5963369929989933
},
"ndcg@20": {
"mean": 0.5917466262174708,
"lo": 0.5671936830682968,
"hi": 0.6169577866306147
},
"exact": {
"mean": 0.8513674197384067,
"lo": 0.8287752675386445,
"hi": 0.8739595719381689
},
"supported": {
"mean": 0.8585017835909632,
"lo": 0.8347205707491082,
"hi": 0.8799048751486326
},
"examined": 52.37931034482759,
"tokens": 669.3448275862069,
"p50": 0.614375,
"p95": 0.685166
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.5208333333333334,
"lo": 0.4954427083333333,
"hi": 0.5455729166666666
},
"any@5": {
"mean": 0.638671875,
"lo": 0.6158854166666666,
"hi": 0.662109375
},
"recall@5": {
"mean": 0.5726493341056519,
"lo": 0.5496957854322287,
"hi": 0.5965495731120734
},
"all@10": {
"mean": 0.5950520833333334,
"lo": 0.5703125,
"hi": 0.6184895833333334
},
"any@10": {
"mean": 0.7259114583333334,
"lo": 0.703125,
"hi": 0.748046875
},
"recall@10": {
"mean": 0.6535268778709487,
"lo": 0.631911139561736,
"hi": 0.6749505599937343
},
"all@20": {
"mean": 0.6803385416666666,
"lo": 0.658203125,
"hi": 0.703125
},
"any@20": {
"mean": 0.8196614583333334,
"lo": 0.80078125,
"hi": 0.837890625
},
"recall@20": {
"mean": 0.7460986825701176,
"lo": 0.7266972181491215,
"hi": 0.7654867123617123
},
"mrr": {
"mean": 0.4906042825656325,
"lo": 0.469387384512615,
"hi": 0.5120915762524136
},
"ndcg@5": {
"mean": 0.4777232496100061,
"lo": 0.4562425087370611,
"hi": 0.49947162893276903
},
"ndcg@10": {
"mean": 0.5063177555439617,
"lo": 0.48660432254786196,
"hi": 0.5272948405159219
},
"ndcg@20": {
"mean": 0.5322045141359967,
"lo": 0.5132224539477597,
"hi": 0.5518094350608949
},
"exact": {
"mean": 0.8196614583333334,
"lo": 0.80078125,
"hi": 0.837890625
},
"supported": {
"mean": 0.8333333333333334,
"lo": 0.814453125,
"hi": 0.8509114583333334
},
"examined": 52.466145833333336,
"tokens": 664.517578125,
"p50": 0.606875,
"p95": 0.686542
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-lexical-locomo-minilm",
"row": "+lexical",
"split": "all",
"k": 10,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "all-minilm",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:51.764Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.08865248226950355,
"lo": 0.05673758865248227,
"hi": 0.12411347517730496
},
"any@5": {
"mean": 0.475177304964539,
"lo": 0.41843971631205673,
"hi": 0.5354609929078015
},
"recall@5": {
"mean": 0.23930681528665865,
"lo": 0.20286693522975735,
"hi": 0.2814651069970218
},
"all@10": {
"mean": 0.13829787234042554,
"lo": 0.09929078014184398,
"hi": 0.18085106382978725
},
"any@10": {
"mean": 0.5886524822695035,
"lo": 0.5283687943262412,
"hi": 0.6453900709219859
},
"recall@10": {
"mean": 0.3271469628832451,
"lo": 0.28533471681567996,
"hi": 0.3710577149406938
},
"all@20": {
"mean": 0.13829787234042554,
"lo": 0.09929078014184398,
"hi": 0.18085106382978725
},
"any@20": {
"mean": 0.5886524822695035,
"lo": 0.5283687943262412,
"hi": 0.6453900709219859
},
"recall@20": {
"mean": 0.3271469628832451,
"lo": 0.28533471681567996,
"hi": 0.3710577149406938
},
"mrr": {
"mean": 0.3174012158054713,
"lo": 0.2702648317010021,
"hi": 0.3662543622650007
},
"ndcg@5": {
"mean": 0.217287852806138,
"lo": 0.1824993876735486,
"hi": 0.2557597236706764
},
"ndcg@10": {
"mean": 0.2521201798600354,
"lo": 0.21687475734446424,
"hi": 0.28970379352740416
},
"ndcg@20": {
"mean": 0.25163695022907123,
"lo": 0.21655070700374734,
"hi": 0.28913333710785677
},
"exact": {
"mean": 0.5886524822695035,
"lo": 0.5283687943262412,
"hi": 0.6453900709219859
},
"supported": {
"mean": 0.6205673758865248,
"lo": 0.5638297872340425,
"hi": 0.6773049645390071
},
"examined": 53.797872340425535,
"tokens": 305.5496453900709,
"p50": 0.300959,
"p95": 0.360625
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.5545171339563862,
"lo": 0.5015576323987538,
"hi": 0.6074766355140186
},
"any@5": {
"mean": 0.6105919003115264,
"lo": 0.5607476635514018,
"hi": 0.6635514018691588
},
"recall@5": {
"mean": 0.5838525441329179,
"lo": 0.5347871235721704,
"hi": 0.6357736240913809
},
"all@10": {
"mean": 0.6199376947040498,
"lo": 0.5669781931464174,
"hi": 0.6728971962616822
},
"any@10": {
"mean": 0.6947040498442367,
"lo": 0.6448598130841121,
"hi": 0.7445482866043613
},
"recall@10": {
"mean": 0.6565420560747663,
"lo": 0.6056593977154725,
"hi": 0.7076843198338526
},
"all@20": {
"mean": 0.6199376947040498,
"lo": 0.5669781931464174,
"hi": 0.6728971962616822
},
"any@20": {
"mean": 0.6947040498442367,
"lo": 0.6448598130841121,
"hi": 0.7445482866043613
},
"recall@20": {
"mean": 0.6565420560747663,
"lo": 0.6056593977154725,
"hi": 0.7076843198338526
},
"mrr": {
"mean": 0.472646244375216,
"lo": 0.4292488750432676,
"hi": 0.5182663304158631
},
"ndcg@5": {
"mean": 0.48025863516203665,
"lo": 0.4378761481334199,
"hi": 0.5258420994572574
},
"ndcg@10": {
"mean": 0.5057474624073696,
"lo": 0.4650944661813467,
"hi": 0.5497305656770777
},
"ndcg@20": {
"mean": 0.5057474624073696,
"lo": 0.4650944661813467,
"hi": 0.5497305656770777
},
"exact": {
"mean": 0.6947040498442367,
"lo": 0.6448598130841121,
"hi": 0.7445482866043613
},
"supported": {
"mean": 0.7009345794392523,
"lo": 0.6510903426791277,
"hi": 0.7507788161993769
},
"examined": 53.601246105919,
"tokens": 292.20249221183803,
"p50": 0.305875,
"p95": 0.361875
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.15217391304347827,
"lo": 0.07608695652173914,
"hi": 0.22826086956521738
},
"any@5": {
"mean": 0.33695652173913043,
"lo": 0.2391304347826087,
"hi": 0.42391304347826086
},
"recall@5": {
"mean": 0.2334292412617221,
"lo": 0.1585144927536232,
"hi": 0.3067988064791134
},
"all@10": {
"mean": 0.2391304347826087,
"lo": 0.15217391304347827,
"hi": 0.32608695652173914
},
"any@10": {
"mean": 0.4782608695652174,
"lo": 0.3804347826086957,
"hi": 0.5760869565217391
},
"recall@10": {
"mean": 0.3366901108269395,
"lo": 0.25309036658141515,
"hi": 0.4166666666666667
},
"all@20": {
"mean": 0.2391304347826087,
"lo": 0.15217391304347827,
"hi": 0.32608695652173914
},
"any@20": {
"mean": 0.4782608695652174,
"lo": 0.3804347826086957,
"hi": 0.5760869565217391
},
"recall@20": {
"mean": 0.3366901108269395,
"lo": 0.25309036658141515,
"hi": 0.4166666666666667
},
"mrr": {
"mean": 0.23897515527950305,
"lo": 0.1648550724637681,
"hi": 0.30749654934437537
},
"ndcg@5": {
"mean": 0.1865642063157956,
"lo": 0.12632289769600144,
"hi": 0.24274011632066503
},
"ndcg@10": {
"mean": 0.22491496987275422,
"lo": 0.16299302405602398,
"hi": 0.2843842344371714
},
"ndcg@20": {
"mean": 0.22423554057364717,
"lo": 0.16230226955439112,
"hi": 0.2841834541470522
},
"exact": {
"mean": 0.4782608695652174,
"lo": 0.3804347826086957,
"hi": 0.5760869565217391
},
"supported": {
"mean": 0.5108695652173914,
"lo": 0.41304347826086957,
"hi": 0.6086956521739131
},
"examined": 53.68478260869565,
"tokens": 285.5652173913044,
"p50": 0.31675,
"p95": 0.385209
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.5743162901307967,
"lo": 0.5422116527942925,
"hi": 0.6076099881093936
},
"any@5": {
"mean": 0.5969084423305588,
"lo": 0.5659928656361475,
"hi": 0.6302021403091558
},
"recall@5": {
"mean": 0.586008719778042,
"lo": 0.5544986127625842,
"hi": 0.6189060642092746
},
"all@10": {
"mean": 0.6718192627824019,
"lo": 0.6409036860879905,
"hi": 0.7051129607609988
},
"any@10": {
"mean": 0.6967895362663495,
"lo": 0.6658739595719382,
"hi": 0.7288941736028538
},
"recall@10": {
"mean": 0.6848989298454221,
"lo": 0.6547760602457393,
"hi": 0.7179944510503368
},
"all@20": {
"mean": 0.6718192627824019,
"lo": 0.6409036860879905,
"hi": 0.7051129607609988
},
"any@20": {
"mean": 0.6967895362663495,
"lo": 0.6658739595719382,
"hi": 0.7288941736028538
},
"recall@20": {
"mean": 0.6848989298454221,
"lo": 0.6547760602457393,
"hi": 0.7179944510503368
},
"mrr": {
"mean": 0.4426995451371193,
"lo": 0.41607402374346486,
"hi": 0.47057027725874284
},
"ndcg@5": {
"mean": 0.4638079037857048,
"lo": 0.4373407832302931,
"hi": 0.49270256509903027
},
"ndcg@10": {
"mean": 0.49684950666268685,
"lo": 0.47147547618447183,
"hi": 0.5243614829520726
},
"ndcg@20": {
"mean": 0.49684950666268685,
"lo": 0.47147547618447183,
"hi": 0.5243614829520726
},
"exact": {
"mean": 0.6967895362663495,
"lo": 0.6658739595719382,
"hi": 0.7288941736028538
},
"supported": {
"mean": 0.7027348394768134,
"lo": 0.6730083234244947,
"hi": 0.7360285374554102
},
"examined": 53.4922711058264,
"tokens": 304.8513674197384,
"p50": 0.31525,
"p95": 0.376625
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.4557291666666667,
"lo": 0.4322916666666667,
"hi": 0.4811197916666667
},
"any@5": {
"mean": 0.5618489583333334,
"lo": 0.5397135416666666,
"hi": 0.5885416666666666
},
"recall@5": {
"mean": 0.5007877682987736,
"lo": 0.47854452455573543,
"hi": 0.526024226414549
},
"all@10": {
"mean": 0.537109375,
"lo": 0.51171875,
"hi": 0.5638020833333334
},
"any@10": {
"mean": 0.6634114583333334,
"lo": 0.640625,
"hi": 0.6875
},
"recall@10": {
"mean": 0.5924355037299177,
"lo": 0.5706794703901297,
"hi": 0.6175470428859676
},
"all@20": {
"mean": 0.537109375,
"lo": 0.51171875,
"hi": 0.5638020833333334
},
"any@20": {
"mean": 0.6634114583333334,
"lo": 0.640625,
"hi": 0.6875
},
"recall@20": {
"mean": 0.5924355037299177,
"lo": 0.5706794703901297,
"hi": 0.6175470428859676
},
"mrr": {
"mean": 0.4137517051091268,
"lo": 0.39354332010582027,
"hi": 0.43572849495701066
},
"ndcg@5": {
"mean": 0.4053805666947761,
"lo": 0.38600023083286955,
"hi": 0.4271868694822411
},
"ndcg@10": {
"mean": 0.43749051984694537,
"lo": 0.418266438159254,
"hi": 0.458996312798903
},
"ndcg@20": {
"mean": 0.4373611069228245,
"lo": 0.4181662106912021,
"hi": 0.458931624235823
},
"exact": {
"mean": 0.6634114583333334,
"lo": 0.640625,
"hi": 0.6875
},
"supported": {
"mean": 0.67578125,
"lo": 0.654296875,
"hi": 0.6998697916666666
},
"examined": 53.582682291666664,
"tokens": 301.1809895833333,
"p50": 0.310917,
"p95": 0.370541
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-semantic-only-locomo",
"row": "semantic only",
"split": "all",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:36.864Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.10283687943262411,
"lo": 0.07092198581560284,
"hi": 0.13829787234042554
},
"any@5": {
"mean": 0.5425531914893617,
"lo": 0.4858156028368794,
"hi": 0.6028368794326241
},
"recall@5": {
"mean": 0.2790904162404723,
"lo": 0.24113360044211127,
"hi": 0.3201503315025153
},
"all@10": {
"mean": 0.14893617021276595,
"lo": 0.1099290780141844,
"hi": 0.19148936170212766
},
"any@10": {
"mean": 0.6843971631205674,
"lo": 0.6312056737588653,
"hi": 0.7411347517730497
},
"recall@10": {
"mean": 0.379062642906428,
"lo": 0.3373358713784247,
"hi": 0.4225936779408225
},
"all@20": {
"mean": 0.22695035460992907,
"lo": 0.18085106382978725,
"hi": 0.2765957446808511
},
"any@20": {
"mean": 0.7978723404255319,
"lo": 0.75177304964539,
"hi": 0.8439716312056738
},
"recall@20": {
"mean": 0.49525277302432963,
"lo": 0.4522372245356566,
"hi": 0.5382053820883609
},
"mrr": {
"mean": 0.38848695231509267,
"lo": 0.3438219372536943,
"hi": 0.4355049481066376
},
"ndcg@5": {
"mean": 0.26001886725658546,
"lo": 0.22468361701064868,
"hi": 0.2975818396134799
},
"ndcg@10": {
"mean": 0.3017461285836642,
"lo": 0.2658754026729805,
"hi": 0.33945454497647226
},
"ndcg@20": {
"mean": 0.3430927557635451,
"lo": 0.3083184091126181,
"hi": 0.37934373420923795
},
"exact": {
"mean": 0.7978723404255319,
"lo": 0.75177304964539,
"hi": 0.8439716312056738
},
"supported": {
"mean": 0.8262411347517731,
"lo": 0.7801418439716312,
"hi": 0.8687943262411347
},
"examined": 40,
"tokens": 655.7375886524823,
"p50": 0.545916,
"p95": 0.634709
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.6355140186915887,
"lo": 0.5856697819314641,
"hi": 0.6915887850467289
},
"any@5": {
"mean": 0.6978193146417445,
"lo": 0.6479750778816199,
"hi": 0.7507788161993769
},
"recall@5": {
"mean": 0.666147455867082,
"lo": 0.6173416407061267,
"hi": 0.7175493250259607
},
"all@10": {
"mean": 0.7071651090342679,
"lo": 0.660436137071651,
"hi": 0.7570093457943925
},
"any@10": {
"mean": 0.7663551401869159,
"lo": 0.719626168224299,
"hi": 0.8130841121495327
},
"recall@10": {
"mean": 0.7372793354101765,
"lo": 0.6905503634475597,
"hi": 0.7840083073727933
},
"all@20": {
"mean": 0.7725856697819314,
"lo": 0.7258566978193146,
"hi": 0.8193146417445483
},
"any@20": {
"mean": 0.8255451713395638,
"lo": 0.7850467289719626,
"hi": 0.8660436137071651
},
"recall@20": {
"mean": 0.8006230529595015,
"lo": 0.7580477673935617,
"hi": 0.8426791277258567
},
"mrr": {
"mean": 0.5542694810373698,
"lo": 0.5092054820751908,
"hi": 0.5996680122540919
},
"ndcg@5": {
"mean": 0.5594218332348924,
"lo": 0.5137702637184411,
"hi": 0.6064731533758588
},
"ndcg@10": {
"mean": 0.5841456825874469,
"lo": 0.5396180705016614,
"hi": 0.6291961993381358
},
"ndcg@20": {
"mean": 0.6009873424351589,
"lo": 0.5605640942893734,
"hi": 0.6431103971439202
},
"exact": {
"mean": 0.8255451713395638,
"lo": 0.7850467289719626,
"hi": 0.8660436137071651
},
"supported": {
"mean": 0.8411214953271028,
"lo": 0.8006230529595015,
"hi": 0.881619937694704
},
"examined": 40,
"tokens": 677.1651090342679,
"p50": 0.548625,
"p95": 0.633875
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.2826086956521739,
"lo": 0.1956521739130435,
"hi": 0.3695652173913043
},
"any@5": {
"mean": 0.40217391304347827,
"lo": 0.30434782608695654,
"hi": 0.5
},
"recall@5": {
"mean": 0.3342391304347826,
"lo": 0.2454710144927536,
"hi": 0.42028985507246375
},
"all@10": {
"mean": 0.32608695652173914,
"lo": 0.22826086956521738,
"hi": 0.42391304347826086
},
"any@10": {
"mean": 0.4782608695652174,
"lo": 0.3804347826086957,
"hi": 0.5760869565217391
},
"recall@10": {
"mean": 0.391304347826087,
"lo": 0.30163043478260876,
"hi": 0.4764492753623188
},
"all@20": {
"mean": 0.33695652173913043,
"lo": 0.2391304347826087,
"hi": 0.42391304347826086
},
"any@20": {
"mean": 0.532608695652174,
"lo": 0.42391304347826086,
"hi": 0.6304347826086957
},
"recall@20": {
"mean": 0.42572463768115937,
"lo": 0.3333333333333333,
"hi": 0.5135869565217392
},
"mrr": {
"mean": 0.26781479390175045,
"lo": 0.1989012798795408,
"hi": 0.3377846790890268
},
"ndcg@5": {
"mean": 0.2531332567053892,
"lo": 0.18505824610048294,
"hi": 0.319565649501838
},
"ndcg@10": {
"mean": 0.2747667202914771,
"lo": 0.20531101372899216,
"hi": 0.33980288980864276
},
"ndcg@20": {
"mean": 0.286333606334597,
"lo": 0.21710358535469887,
"hi": 0.3522677731545342
},
"exact": {
"mean": 0.532608695652174,
"lo": 0.42391304347826086,
"hi": 0.6304347826086957
},
"supported": {
"mean": 0.5652173913043478,
"lo": 0.45652173913043476,
"hi": 0.6630434782608695
},
"examined": 40,
"tokens": 601.0326086956521,
"p50": 0.555958,
"p95": 0.642
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.5909631391200951,
"lo": 0.558858501783591,
"hi": 0.6230677764565993
},
"any@5": {
"mean": 0.6218787158145065,
"lo": 0.5909631391200951,
"hi": 0.652794292508918
},
"recall@5": {
"mean": 0.6058263971462544,
"lo": 0.5753071739992073,
"hi": 0.6379310344827588
},
"all@10": {
"mean": 0.6932223543400713,
"lo": 0.6599286563614745,
"hi": 0.72294887039239
},
"any@10": {
"mean": 0.7170035671819263,
"lo": 0.6860879904875149,
"hi": 0.746730083234245
},
"recall@10": {
"mean": 0.7045184304399524,
"lo": 0.6730083234244945,
"hi": 0.7344431232659532
},
"all@20": {
"mean": 0.7824019024970273,
"lo": 0.7538644470868014,
"hi": 0.8097502972651606
},
"any@20": {
"mean": 0.807372175980975,
"lo": 0.7812128418549346,
"hi": 0.8335315101070154
},
"recall@20": {
"mean": 0.7952833927863654,
"lo": 0.7685295283392786,
"hi": 0.8212445501387239
},
"mrr": {
"mean": 0.45584178162074446,
"lo": 0.42915765020360874,
"hi": 0.4845099646358093
},
"ndcg@5": {
"mean": 0.4741038978110057,
"lo": 0.44739695519035344,
"hi": 0.5031513241476366
},
"ndcg@10": {
"mean": 0.5063576315581536,
"lo": 0.4808234474179655,
"hi": 0.5350764574163313
},
"ndcg@20": {
"mean": 0.5296042773703612,
"lo": 0.5059459678819334,
"hi": 0.5557429987091624
},
"exact": {
"mean": 0.807372175980975,
"lo": 0.7812128418549346,
"hi": 0.8335315101070154
},
"supported": {
"mean": 0.8192627824019025,
"lo": 0.7931034482758621,
"hi": 0.8442330558858502
},
"examined": 40,
"tokens": 661.8656361474435,
"p50": 0.545667,
"p95": 0.577667
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.4921875,
"lo": 0.4674479166666667,
"hi": 0.5169270833333334
},
"any@5": {
"mean": 0.6100260416666666,
"lo": 0.5865885416666666,
"hi": 0.634765625
},
"recall@5": {
"mean": 0.5421789262455383,
"lo": 0.520131253352368,
"hi": 0.5658849457035109
},
"all@10": {
"mean": 0.57421875,
"lo": 0.5501302083333334,
"hi": 0.6002604166666666
},
"any@10": {
"mean": 0.70703125,
"lo": 0.6861979166666666,
"hi": 0.7311197916666666
},
"recall@10": {
"mean": 0.6328530807072131,
"lo": 0.6113224882756131,
"hi": 0.656630753315581
},
"all@20": {
"mean": 0.6516927083333334,
"lo": 0.6295572916666666,
"hi": 0.6764322916666666
},
"any@20": {
"mean": 0.79296875,
"lo": 0.7740885416666666,
"hi": 0.8138020833333334
},
"recall@20": {
"mean": 0.7191805221307689,
"lo": 0.6993386243386245,
"hi": 0.7402145228522694
},
"mrr": {
"mean": 0.4527836740545968,
"lo": 0.433171438480175,
"hi": 0.474956606205095
},
"ndcg@5": {
"mean": 0.43939411895228403,
"lo": 0.41909674712345396,
"hi": 0.4603782642117336
},
"ndcg@10": {
"mean": 0.47117739503801154,
"lo": 0.45219071240813347,
"hi": 0.4912215449103277
},
"ndcg@20": {
"mean": 0.495708973371264,
"lo": 0.47751681430798104,
"hi": 0.5149560029827353
},
"exact": {
"mean": 0.79296875,
"lo": 0.7740885416666666,
"hi": 0.8138020833333334
},
"supported": {
"mean": 0.8098958333333334,
"lo": 0.7916666666666666,
"hi": 0.8287760416666666
},
"examined": 40,
"tokens": 660.2942708333334,
"p50": 0.546541,
"p95": 0.613792
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-all-semantic-only-locomo-minilm",
"row": "semantic only",
"split": "all",
"k": 10,
"reader": null,
"facts": "locomo",
"commit": "885a21a63c32",
"record": {
"commit": "885a21a63c32",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "all-minilm",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T10:03:50.863Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 282,
"metrics": {
"all@5": {
"mean": 0.06028368794326241,
"lo": 0.03546099290780142,
"hi": 0.08865248226950355
},
"any@5": {
"mean": 0.38652482269503546,
"lo": 0.32978723404255317,
"hi": 0.4432624113475177
},
"recall@5": {
"mean": 0.18768267110932288,
"lo": 0.15561784188603892,
"hi": 0.22322157074396726
},
"all@10": {
"mean": 0.09574468085106383,
"lo": 0.06028368794326241,
"hi": 0.1347517730496454
},
"any@10": {
"mean": 0.49645390070921985,
"lo": 0.4326241134751773,
"hi": 0.5531914893617021
},
"recall@10": {
"mean": 0.26591713002687245,
"lo": 0.22579209548694434,
"hi": 0.3038654017377422
},
"all@20": {
"mean": 0.09574468085106383,
"lo": 0.06028368794326241,
"hi": 0.1347517730496454
},
"any@20": {
"mean": 0.49645390070921985,
"lo": 0.4326241134751773,
"hi": 0.5531914893617021
},
"recall@20": {
"mean": 0.26591713002687245,
"lo": 0.22579209548694434,
"hi": 0.3038654017377422
},
"mrr": {
"mean": 0.2366711696498933,
"lo": 0.19631881121242842,
"hi": 0.2782027468197683
},
"ndcg@5": {
"mean": 0.1631464044217895,
"lo": 0.13356257931416207,
"hi": 0.19495456290886304
},
"ndcg@10": {
"mean": 0.19634128794145747,
"lo": 0.1650449687978228,
"hi": 0.22802947469330737
},
"ndcg@20": {
"mean": 0.19589534667040084,
"lo": 0.1648817491625309,
"hi": 0.22732911446142612
},
"exact": {
"mean": 0.49645390070921985,
"lo": 0.4326241134751773,
"hi": 0.5531914893617021
},
"supported": {
"mean": 0.5177304964539007,
"lo": 0.45390070921985815,
"hi": 0.5780141843971631
},
"examined": 40,
"tokens": 287.9255319148936,
"p50": 0.252583,
"p95": 0.277
}
},
{
"name": "temporal",
"n": 321,
"metrics": {
"all@5": {
"mean": 0.38317757009345793,
"lo": 0.3333333333333333,
"hi": 0.43613707165109034
},
"any@5": {
"mean": 0.4454828660436137,
"lo": 0.3925233644859813,
"hi": 0.5015576323987538
},
"recall@5": {
"mean": 0.4145898234683282,
"lo": 0.3634475597092419,
"hi": 0.4672897196261682
},
"all@10": {
"mean": 0.46105919003115264,
"lo": 0.411214953271028,
"hi": 0.5109034267912772
},
"any@10": {
"mean": 0.5233644859813084,
"lo": 0.4735202492211838,
"hi": 0.5763239875389408
},
"recall@10": {
"mean": 0.4924714434060228,
"lo": 0.4439252336448599,
"hi": 0.5415368639667706
},
"all@20": {
"mean": 0.46105919003115264,
"lo": 0.411214953271028,
"hi": 0.5109034267912772
},
"any@20": {
"mean": 0.5233644859813084,
"lo": 0.4735202492211838,
"hi": 0.5763239875389408
},
"recall@20": {
"mean": 0.4924714434060228,
"lo": 0.4439252336448599,
"hi": 0.5415368639667706
},
"mrr": {
"mean": 0.3270051426593484,
"lo": 0.2844199673638926,
"hi": 0.3723013400583495
},
"ndcg@5": {
"mean": 0.3279908596763101,
"lo": 0.2845474020403503,
"hi": 0.37230258982144837
},
"ndcg@10": {
"mean": 0.35464881688853356,
"lo": 0.3133309326908929,
"hi": 0.3972869580407622
},
"ndcg@20": {
"mean": 0.35464881688853356,
"lo": 0.3133309326908929,
"hi": 0.3972869580407622
},
"exact": {
"mean": 0.5233644859813084,
"lo": 0.4735202492211838,
"hi": 0.5763239875389408
},
"supported": {
"mean": 0.5295950155763239,
"lo": 0.4797507788161994,
"hi": 0.5794392523364486
},
"examined": 40,
"tokens": 270.8722741433022,
"p50": 0.252958,
"p95": 0.279125
}
},
{
"name": "open-domain",
"n": 92,
"metrics": {
"all@5": {
"mean": 0.13043478260869565,
"lo": 0.06521739130434782,
"hi": 0.20652173913043478
},
"any@5": {
"mean": 0.2717391304347826,
"lo": 0.18478260869565216,
"hi": 0.358695652173913
},
"recall@5": {
"mean": 0.1863277919863598,
"lo": 0.11956521739130435,
"hi": 0.25969735720375103
},
"all@10": {
"mean": 0.16304347826086957,
"lo": 0.08695652173913043,
"hi": 0.2391304347826087
},
"any@10": {
"mean": 0.3804347826086957,
"lo": 0.2826086956521739,
"hi": 0.4782608695652174
},
"recall@10": {
"mean": 0.25399616368286443,
"lo": 0.17881500426257463,
"hi": 0.32971014492753625
},
"all@20": {
"mean": 0.16304347826086957,
"lo": 0.08695652173913043,
"hi": 0.2391304347826087
},
"any@20": {
"mean": 0.3804347826086957,
"lo": 0.2826086956521739,
"hi": 0.4782608695652174
},
"recall@20": {
"mean": 0.25399616368286443,
"lo": 0.17881500426257463,
"hi": 0.32971014492753625
},
"mrr": {
"mean": 0.16343599033816422,
"lo": 0.10926932367149755,
"hi": 0.22035024154589372
},
"ndcg@5": {
"mean": 0.13654916827296718,
"lo": 0.08732658337546208,
"hi": 0.18873575886836158
},
"ndcg@10": {
"mean": 0.16304481346022165,
"lo": 0.11320686000784023,
"hi": 0.21499496526783507
},
"ndcg@20": {
"mean": 0.16256763824841727,
"lo": 0.11320686000784023,
"hi": 0.2145196423342657
},
"exact": {
"mean": 0.3804347826086957,
"lo": 0.2826086956521739,
"hi": 0.4782608695652174
},
"supported": {
"mean": 0.41304347826086957,
"lo": 0.31521739130434784,
"hi": 0.5108695652173914
},
"examined": 40,
"tokens": 269.0108695652174,
"p50": 0.256125,
"p95": 0.292625
}
},
{
"name": "single-hop",
"n": 841,
"metrics": {
"all@5": {
"mean": 0.4126040428061831,
"lo": 0.3793103448275862,
"hi": 0.4470868014268728
},
"any@5": {
"mean": 0.43876337693222356,
"lo": 0.4054696789536266,
"hi": 0.47205707491082044
},
"recall@5": {
"mean": 0.4256837098692034,
"lo": 0.3926872770511296,
"hi": 0.4572929052715022
},
"all@10": {
"mean": 0.5136741973840666,
"lo": 0.47919143876337694,
"hi": 0.5481569560047562
},
"any@10": {
"mean": 0.5315101070154578,
"lo": 0.4982164090368609,
"hi": 0.5659928656361475
},
"recall@10": {
"mean": 0.5225921521997623,
"lo": 0.4892984542211653,
"hi": 0.5574712643678161
},
"all@20": {
"mean": 0.5136741973840666,
"lo": 0.47919143876337694,
"hi": 0.5481569560047562
},
"any@20": {
"mean": 0.5315101070154578,
"lo": 0.4982164090368609,
"hi": 0.5659928656361475
},
"recall@20": {
"mean": 0.5225921521997623,
"lo": 0.4892984542211653,
"hi": 0.5574712643678161
},
"mrr": {
"mean": 0.29981739425853554,
"lo": 0.2740591321744707,
"hi": 0.3277334239284297
},
"ndcg@5": {
"mean": 0.3171267990889507,
"lo": 0.2901344946902622,
"hi": 0.34489950164327043
},
"ndcg@10": {
"mean": 0.34860905819310756,
"lo": 0.32259710124919555,
"hi": 0.37621447539307695
},
"ndcg@20": {
"mean": 0.34860905819310756,
"lo": 0.32259710124919555,
"hi": 0.37621447539307695
},
"exact": {
"mean": 0.5315101070154578,
"lo": 0.4982164090368609,
"hi": 0.5659928656361475
},
"supported": {
"mean": 0.5481569560047562,
"lo": 0.5148632580261593,
"hi": 0.5814506539833532
},
"examined": 40,
"tokens": 286.8680142687277,
"p50": 0.252583,
"p95": 0.267583
}
},
{
"name": "all",
"n": 1536,
"metrics": {
"all@5": {
"mean": 0.3248697916666667,
"lo": 0.3020833333333333,
"hi": 0.3502604166666667
},
"any@5": {
"mean": 0.4205729166666667,
"lo": 0.3971354166666667,
"hi": 0.4466145833333333
},
"recall@5": {
"mean": 0.3653333355787158,
"lo": 0.34399171905853626,
"hi": 0.3902917286706349
},
"all@10": {
"mean": 0.4049479166666667,
"lo": 0.380859375,
"hi": 0.4303385416666667
},
"any@10": {
"mean": 0.5143229166666666,
"lo": 0.4889322916666667,
"hi": 0.5403645833333334
},
"recall@10": {
"mean": 0.4530856842836816,
"lo": 0.43122914084767666,
"hi": 0.47801113075064566
},
"all@20": {
"mean": 0.4049479166666667,
"lo": 0.380859375,
"hi": 0.4303385416666667
},
"any@20": {
"mean": 0.5143229166666666,
"lo": 0.4889322916666667,
"hi": 0.5403645833333334
},
"recall@20": {
"mean": 0.4530856842836816,
"lo": 0.43122914084767666,
"hi": 0.47801113075064566
},
"mrr": {
"mean": 0.28573727885251304,
"lo": 0.26809637483465604,
"hi": 0.3050207713293649
},
"ndcg@5": {
"mean": 0.2803115322382567,
"lo": 0.26200074160777115,
"hi": 0.29947908335862217
},
"ndcg@10": {
"mean": 0.3108013373694369,
"lo": 0.29381584073662215,
"hi": 0.3295557222367796
},
"ndcg@20": {
"mean": 0.3106908845322468,
"lo": 0.29371357384768365,
"hi": 0.32940418708676733
},
"exact": {
"mean": 0.5143229166666666,
"lo": 0.4889322916666667,
"hi": 0.5403645833333334
},
"supported": {
"mean": 0.5305989583333334,
"lo": 0.5045572916666666,
"hi": 0.556640625
},
"examined": 40,
"tokens": 282.6497395833333,
"p50": 0.25275,
"p95": 0.272541
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-adjacency-locomo",
"row": "+adjacency",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "33d0f8749bd1",
"record": {
"commit": "33d0f8749bd1",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:31:03.410Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.12980769230769232,
"lo": 0.08173076923076923,
"hi": 0.1778846153846154
},
"any@5": {
"mean": 0.6586538461538461,
"lo": 0.5961538461538461,
"hi": 0.7163461538461539
},
"recall@5": {
"mean": 0.35167919616932775,
"lo": 0.30563186813186827,
"hi": 0.39840105070368226
},
"all@10": {
"mean": 0.23076923076923078,
"lo": 0.17307692307692307,
"hi": 0.28846153846153844
},
"any@10": {
"mean": 0.8028846153846154,
"lo": 0.7451923076923077,
"hi": 0.8557692307692307
},
"recall@10": {
"mean": 0.48861046665322977,
"lo": 0.4407730062335326,
"hi": 0.5376833875189136
},
"all@20": {
"mean": 0.375,
"lo": 0.3076923076923077,
"hi": 0.44711538461538464
},
"any@20": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"recall@20": {
"mean": 0.6190934978763927,
"lo": 0.5723412095944987,
"hi": 0.6687131490585435
},
"mrr": {
"mean": 0.48788638578302035,
"lo": 0.43393214691291626,
"hi": 0.538372584947104
},
"ndcg@20": {
"mean": 0.4432587162151551,
"lo": 0.4022063231280031,
"hi": 0.48446323683833525
},
"exact": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"supported": {
"mean": 0.8990384615384616,
"lo": 0.8557692307692307,
"hi": 0.9375
},
"examined": 75.13461538461539,
"tokens": 785.0673076923077,
"p50": 1.0255,
"p95": 1.239083
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.683982683982684,
"lo": 0.6190476190476191,
"hi": 0.7445887445887446
},
"any@5": {
"mean": 0.7662337662337663,
"lo": 0.70995670995671,
"hi": 0.8181818181818182
},
"recall@5": {
"mean": 0.7261904761904762,
"lo": 0.66991341991342,
"hi": 0.7781385281385281
},
"all@10": {
"mean": 0.7619047619047619,
"lo": 0.7012987012987013,
"hi": 0.8181818181818182
},
"any@10": {
"mean": 0.8354978354978355,
"lo": 0.7878787878787878,
"hi": 0.8787878787878788
},
"recall@10": {
"mean": 0.8001443001443,
"lo": 0.7467532467532467,
"hi": 0.8463203463203464
},
"all@20": {
"mean": 0.8008658008658008,
"lo": 0.7445887445887446,
"hi": 0.8528138528138528
},
"any@20": {
"mean": 0.8787878787878788,
"lo": 0.8354978354978355,
"hi": 0.9177489177489178
},
"recall@20": {
"mean": 0.8405483405483406,
"lo": 0.7950937950937951,
"hi": 0.8838383838383838
},
"mrr": {
"mean": 0.6347992753645924,
"lo": 0.5794278292654917,
"hi": 0.6858974076315557
},
"ndcg@20": {
"mean": 0.6722673726443379,
"lo": 0.6212415550282798,
"hi": 0.7197764705975214
},
"exact": {
"mean": 0.8787878787878788,
"lo": 0.8354978354978355,
"hi": 0.9177489177489178
},
"supported": {
"mean": 0.8874458874458875,
"lo": 0.8441558441558441,
"hi": 0.9264069264069265
},
"examined": 73.66233766233766,
"tokens": 777.008658008658,
"p50": 1.034875,
"p95": 1.657125
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.3287671232876712,
"lo": 0.2328767123287671,
"hi": 0.4383561643835616
},
"any@5": {
"mean": 0.5205479452054794,
"lo": 0.410958904109589,
"hi": 0.6301369863013698
},
"recall@5": {
"mean": 0.4117647058823529,
"lo": 0.31406124093473003,
"hi": 0.5167875369325812
},
"all@10": {
"mean": 0.3835616438356164,
"lo": 0.273972602739726,
"hi": 0.4931506849315068
},
"any@10": {
"mean": 0.6027397260273972,
"lo": 0.4931506849315068,
"hi": 0.726027397260274
},
"recall@10": {
"mean": 0.46964813322589316,
"lo": 0.3689900617781358,
"hi": 0.5776255707762558
},
"all@20": {
"mean": 0.4520547945205479,
"lo": 0.3424657534246575,
"hi": 0.5753424657534246
},
"any@20": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"recall@20": {
"mean": 0.5350584816741143,
"lo": 0.43171440431714403,
"hi": 0.6415525114155252
},
"mrr": {
"mean": 0.3873135046080251,
"lo": 0.2951820629902821,
"hi": 0.4817606670346397
},
"ndcg@20": {
"mean": 0.38620687414325416,
"lo": 0.30204649790048815,
"hi": 0.47683526263650616
},
"exact": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"supported": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"examined": 77.89041095890411,
"tokens": 715.3424657534247,
"p50": 1.0375,
"p95": 1.473292
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.7254290171606864,
"lo": 0.6911076443057722,
"hi": 0.7597503900156006
},
"any@5": {
"mean": 0.7472698907956318,
"lo": 0.7145085803432137,
"hi": 0.781591263650546
},
"recall@5": {
"mean": 0.7366094643785752,
"lo": 0.7048881955278211,
"hi": 0.7688507540301612
},
"all@10": {
"mean": 0.8034321372854915,
"lo": 0.7737909516380655,
"hi": 0.8330733229329174
},
"any@10": {
"mean": 0.828393135725429,
"lo": 0.8003120124804992,
"hi": 0.859594383775351
},
"recall@10": {
"mean": 0.8159126365054602,
"lo": 0.7867914716588662,
"hi": 0.8447737909516381
},
"all@20": {
"mean": 0.8455538221528861,
"lo": 0.8174726989079563,
"hi": 0.875195007800312
},
"any@20": {
"mean": 0.8705148205928237,
"lo": 0.84399375975039,
"hi": 0.8970358814352574
},
"recall@20": {
"mean": 0.858554342173687,
"lo": 0.8320332813312531,
"hi": 0.8871554862194487
},
"mrr": {
"mean": 0.6005214329034746,
"lo": 0.5674326056311919,
"hi": 0.6314057740227662
},
"ndcg@20": {
"mean": 0.6574000985786645,
"lo": 0.6278209589648567,
"hi": 0.6853441656033034
},
"exact": {
"mean": 0.8705148205928237,
"lo": 0.84399375975039,
"hi": 0.8970358814352574
},
"supported": {
"mean": 0.8845553822152886,
"lo": 0.859594383775351,
"hi": 0.9095163806552262
},
"examined": 74.31825273010921,
"tokens": 762.7067082683308,
"p50": 1.030042,
"p95": 1.112375
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.5845620121422377,
"lo": 0.5542064180398959,
"hi": 0.611448395490026
},
"any@5": {
"mean": 0.7207285342584562,
"lo": 0.6938421509106678,
"hi": 0.7467476149176062
},
"recall@5": {
"mean": 0.6445141049430171,
"lo": 0.6186099763068846,
"hi": 0.6702628753149131
},
"all@10": {
"mean": 0.6652211621856028,
"lo": 0.6366001734605378,
"hi": 0.6938421509106678
},
"any@10": {
"mean": 0.810928013876843,
"lo": 0.7866435385949696,
"hi": 0.8334778837814397
},
"recall@10": {
"mean": 0.7317854502365094,
"lo": 0.7059838166500282,
"hi": 0.7543627640548719
},
"all@20": {
"mean": 0.7267996530789246,
"lo": 0.6999132697311362,
"hi": 0.7545533391153513
},
"any@20": {
"mean": 0.8629661751951431,
"lo": 0.8421509106678231,
"hi": 0.8820468343451865
},
"recall@20": {
"mean": 0.7912668835390286,
"lo": 0.7689623461397325,
"hi": 0.8119539943580204
},
"mrr": {
"mean": 0.5735707937377295,
"lo": 0.548177782571671,
"hi": 0.5966556147111853
},
"ndcg@20": {
"mean": 0.604577745927992,
"lo": 0.5818404410285393,
"hi": 0.6260280189916164
},
"exact": {
"mean": 0.8629661751951431,
"lo": 0.8421509106678231,
"hi": 0.8820468343451865
},
"supported": {
"mean": 0.8751084128360798,
"lo": 0.8542931483087598,
"hi": 0.8941890719861232
},
"examined": 74.56027753686037,
"tokens": 766.6071118820469,
"p50": 1.031166,
"p95": 1.211292
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-entities-locomo",
"row": "+entities",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "33d0f8749bd1",
"record": {
"commit": "33d0f8749bd1",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:31:00.258Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.125,
"lo": 0.07692307692307693,
"hi": 0.17307692307692307
},
"any@5": {
"mean": 0.6394230769230769,
"lo": 0.5769230769230769,
"hi": 0.7019230769230769
},
"recall@5": {
"mean": 0.3457497089898405,
"lo": 0.2980309347085664,
"hi": 0.3937844593272225
},
"all@10": {
"mean": 0.23076923076923078,
"lo": 0.1778846153846154,
"hi": 0.28846153846153844
},
"any@10": {
"mean": 0.8125,
"lo": 0.7548076923076923,
"hi": 0.8653846153846154
},
"recall@10": {
"mean": 0.4937100545653177,
"lo": 0.44578477078477086,
"hi": 0.5414014804639806
},
"all@20": {
"mean": 0.35096153846153844,
"lo": 0.28846153846153844,
"hi": 0.4182692307692308
},
"any@20": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"recall@20": {
"mean": 0.6112867213196161,
"lo": 0.5637543103825996,
"hi": 0.6591054778554777
},
"mrr": {
"mean": 0.4846737997579344,
"lo": 0.43055864408027894,
"hi": 0.5358538363346056
},
"ndcg@20": {
"mean": 0.4381448923889576,
"lo": 0.3975402047272317,
"hi": 0.47914224255201265
},
"exact": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"supported": {
"mean": 0.8990384615384616,
"lo": 0.8605769230769231,
"hi": 0.9375
},
"examined": 71.38461538461539,
"tokens": 789.0432692307693,
"p50": 1.090208,
"p95": 1.358416
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.6536796536796536,
"lo": 0.5887445887445888,
"hi": 0.7186147186147186
},
"any@5": {
"mean": 0.7445887445887446,
"lo": 0.683982683982684,
"hi": 0.7965367965367965
},
"recall@5": {
"mean": 0.7016594516594515,
"lo": 0.6410533910533911,
"hi": 0.7539682539682541
},
"all@10": {
"mean": 0.7359307359307359,
"lo": 0.6796536796536796,
"hi": 0.7922077922077922
},
"any@10": {
"mean": 0.8225108225108225,
"lo": 0.7705627705627706,
"hi": 0.8701298701298701
},
"recall@10": {
"mean": 0.7828282828282827,
"lo": 0.7294372294372294,
"hi": 0.8333333333333333
},
"all@20": {
"mean": 0.7878787878787878,
"lo": 0.7316017316017316,
"hi": 0.8398268398268398
},
"any@20": {
"mean": 0.8658008658008658,
"lo": 0.8181818181818182,
"hi": 0.9047619047619048
},
"recall@20": {
"mean": 0.8282828282828283,
"lo": 0.7792207792207794,
"hi": 0.8715728715728717
},
"mrr": {
"mean": 0.6165548634513492,
"lo": 0.5604183730425135,
"hi": 0.6696670265902506
},
"ndcg@20": {
"mean": 0.6536386963254479,
"lo": 0.6030115224296215,
"hi": 0.7013252265488457
},
"exact": {
"mean": 0.8658008658008658,
"lo": 0.8181818181818182,
"hi": 0.9047619047619048
},
"supported": {
"mean": 0.8744588744588745,
"lo": 0.8311688311688312,
"hi": 0.9134199134199135
},
"examined": 69.92640692640693,
"tokens": 781.0346320346321,
"p50": 1.095958,
"p95": 1.294625
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.3150684931506849,
"lo": 0.2054794520547945,
"hi": 0.4246575342465753
},
"any@5": {
"mean": 0.5068493150684932,
"lo": 0.3972602739726027,
"hi": 0.6301369863013698
},
"recall@5": {
"mean": 0.3980660757453666,
"lo": 0.29646790222938496,
"hi": 0.5022831050228309
},
"all@10": {
"mean": 0.3698630136986301,
"lo": 0.2602739726027397,
"hi": 0.4931506849315068
},
"any@10": {
"mean": 0.589041095890411,
"lo": 0.4794520547945205,
"hi": 0.7123287671232876
},
"recall@10": {
"mean": 0.45594950308890686,
"lo": 0.35777598710717157,
"hi": 0.5627853881278538
},
"all@20": {
"mean": 0.4383561643835616,
"lo": 0.3150684931506849,
"hi": 0.547945205479452
},
"any@20": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"recall@20": {
"mean": 0.529014968378385,
"lo": 0.4228505848167411,
"hi": 0.6338302444265379
},
"mrr": {
"mean": 0.3889420130858487,
"lo": 0.29925304908181627,
"hi": 0.4902547984739764
},
"ndcg@20": {
"mean": 0.38624272004940585,
"lo": 0.3034266550839264,
"hi": 0.47834278834474303
},
"exact": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"supported": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"examined": 73.9041095890411,
"tokens": 720.4794520547945,
"p50": 1.11025,
"p95": 1.361542
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.6786271450858035,
"lo": 0.6427457098283932,
"hi": 0.7129485179407177
},
"any@5": {
"mean": 0.7129485179407177,
"lo": 0.6770670826833073,
"hi": 0.7472698907956318
},
"recall@5": {
"mean": 0.6960478419136765,
"lo": 0.6612064482579303,
"hi": 0.7298491939677588
},
"all@10": {
"mean": 0.7706708268330733,
"lo": 0.7379095163806553,
"hi": 0.8034321372854915
},
"any@10": {
"mean": 0.797191887675507,
"lo": 0.7675507020280812,
"hi": 0.828393135725429
},
"recall@10": {
"mean": 0.7841913676547062,
"lo": 0.7532501300052002,
"hi": 0.8161726469058762
},
"all@20": {
"mean": 0.8299531981279251,
"lo": 0.8018720748829953,
"hi": 0.859594383775351
},
"any@20": {
"mean": 0.8564742589703588,
"lo": 0.8299531981279251,
"hi": 0.8814352574102964
},
"recall@20": {
"mean": 0.843733749349974,
"lo": 0.8177327093083725,
"hi": 0.8705148205928237
},
"mrr": {
"mean": 0.5761941396262525,
"lo": 0.544112559826264,
"hi": 0.606800425944637
},
"ndcg@20": {
"mean": 0.6335368610039814,
"lo": 0.6044092466982808,
"hi": 0.6614255524886651
},
"exact": {
"mean": 0.8564742589703588,
"lo": 0.8299531981279251,
"hi": 0.8814352574102964
},
"supported": {
"mean": 0.8689547581903276,
"lo": 0.84399375975039,
"hi": 0.8954758190327613
},
"examined": 70.8377535101404,
"tokens": 767.3151326053043,
"p50": 1.112333,
"p95": 1.4905
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.5507372072853426,
"lo": 0.5212489158716392,
"hi": 0.577623590633131
},
"any@5": {
"mean": 0.6929748482220295,
"lo": 0.6652211621856028,
"hi": 0.7189939288811795
},
"recall@5": {
"mean": 0.6151125437981775,
"lo": 0.5878877625876302,
"hi": 0.6413635487573043
},
"all@10": {
"mean": 0.6409366869037294,
"lo": 0.6123156981786644,
"hi": 0.6686903729401561
},
"any@10": {
"mean": 0.7918473547267997,
"lo": 0.7675628794449263,
"hi": 0.8152645273200347
},
"recall@10": {
"mean": 0.7107337424762152,
"lo": 0.6852655804946103,
"hi": 0.735598646518202
},
"all@20": {
"mean": 0.7103209019947961,
"lo": 0.6834345186470078,
"hi": 0.7389418907198613
},
"any@20": {
"mean": 0.852558542931483,
"lo": 0.8308759757155247,
"hi": 0.8716392020815265
},
"recall@20": {
"mean": 0.7787791824742142,
"lo": 0.756281812049375,
"hi": 0.801120970775647
},
"mrr": {
"mean": 0.5559146004012201,
"lo": 0.5305414483221604,
"hi": 0.5783457259625566
},
"ndcg@20": {
"mean": 0.5866587362838163,
"lo": 0.5646208726277896,
"hi": 0.6077338311154385
},
"exact": {
"mean": 0.852558542931483,
"lo": 0.8308759757155247,
"hi": 0.8716392020815265
},
"supported": {
"mean": 0.8638334778837814,
"lo": 0.8447528187337381,
"hi": 0.8829141370338248
},
"examined": 70.9479618386817,
"tokens": 771.0182133564614,
"p50": 1.104708,
"p95": 1.373125
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-facts-locomo",
"row": "+facts",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "33d0f8749bd1",
"record": {
"commit": "33d0f8749bd1",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:30:58.545Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.125,
"lo": 0.07692307692307693,
"hi": 0.17307692307692307
},
"any@5": {
"mean": 0.6394230769230769,
"lo": 0.5769230769230769,
"hi": 0.7019230769230769
},
"recall@5": {
"mean": 0.3457497089898405,
"lo": 0.2980309347085664,
"hi": 0.3937844593272225
},
"all@10": {
"mean": 0.23076923076923078,
"lo": 0.1778846153846154,
"hi": 0.28846153846153844
},
"any@10": {
"mean": 0.8125,
"lo": 0.7548076923076923,
"hi": 0.8653846153846154
},
"recall@10": {
"mean": 0.4937100545653177,
"lo": 0.44578477078477086,
"hi": 0.5414014804639806
},
"all@20": {
"mean": 0.35096153846153844,
"lo": 0.28846153846153844,
"hi": 0.4182692307692308
},
"any@20": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"recall@20": {
"mean": 0.6112867213196161,
"lo": 0.5637543103825996,
"hi": 0.6591054778554777
},
"mrr": {
"mean": 0.4846737997579344,
"lo": 0.43055864408027894,
"hi": 0.5358538363346056
},
"ndcg@20": {
"mean": 0.4381448923889576,
"lo": 0.3975402047272317,
"hi": 0.47914224255201265
},
"exact": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"supported": {
"mean": 0.8990384615384616,
"lo": 0.8605769230769231,
"hi": 0.9375
},
"examined": 71.38461538461539,
"tokens": 789.0432692307693,
"p50": 1.111041,
"p95": 1.371666
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.6536796536796536,
"lo": 0.5887445887445888,
"hi": 0.7186147186147186
},
"any@5": {
"mean": 0.7445887445887446,
"lo": 0.683982683982684,
"hi": 0.7965367965367965
},
"recall@5": {
"mean": 0.7016594516594515,
"lo": 0.6410533910533911,
"hi": 0.7539682539682541
},
"all@10": {
"mean": 0.7359307359307359,
"lo": 0.6796536796536796,
"hi": 0.7922077922077922
},
"any@10": {
"mean": 0.8225108225108225,
"lo": 0.7705627705627706,
"hi": 0.8701298701298701
},
"recall@10": {
"mean": 0.7828282828282827,
"lo": 0.7294372294372294,
"hi": 0.8333333333333333
},
"all@20": {
"mean": 0.7878787878787878,
"lo": 0.7316017316017316,
"hi": 0.8398268398268398
},
"any@20": {
"mean": 0.8658008658008658,
"lo": 0.8181818181818182,
"hi": 0.9047619047619048
},
"recall@20": {
"mean": 0.8282828282828283,
"lo": 0.7792207792207794,
"hi": 0.8715728715728717
},
"mrr": {
"mean": 0.6165548634513492,
"lo": 0.5604183730425135,
"hi": 0.6696670265902506
},
"ndcg@20": {
"mean": 0.6536386963254479,
"lo": 0.6030115224296215,
"hi": 0.7013252265488457
},
"exact": {
"mean": 0.8658008658008658,
"lo": 0.8181818181818182,
"hi": 0.9047619047619048
},
"supported": {
"mean": 0.8744588744588745,
"lo": 0.8311688311688312,
"hi": 0.9134199134199135
},
"examined": 69.92640692640693,
"tokens": 781.0346320346321,
"p50": 1.140791,
"p95": 1.389041
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.3150684931506849,
"lo": 0.2054794520547945,
"hi": 0.4246575342465753
},
"any@5": {
"mean": 0.5068493150684932,
"lo": 0.3972602739726027,
"hi": 0.6301369863013698
},
"recall@5": {
"mean": 0.3980660757453666,
"lo": 0.29646790222938496,
"hi": 0.5022831050228309
},
"all@10": {
"mean": 0.3698630136986301,
"lo": 0.2602739726027397,
"hi": 0.4931506849315068
},
"any@10": {
"mean": 0.589041095890411,
"lo": 0.4794520547945205,
"hi": 0.7123287671232876
},
"recall@10": {
"mean": 0.45594950308890686,
"lo": 0.35777598710717157,
"hi": 0.5627853881278538
},
"all@20": {
"mean": 0.4383561643835616,
"lo": 0.3150684931506849,
"hi": 0.547945205479452
},
"any@20": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"recall@20": {
"mean": 0.529014968378385,
"lo": 0.4228505848167411,
"hi": 0.6338302444265379
},
"mrr": {
"mean": 0.3889420130858487,
"lo": 0.29925304908181627,
"hi": 0.4902547984739764
},
"ndcg@20": {
"mean": 0.38624272004940585,
"lo": 0.3034266550839264,
"hi": 0.47834278834474303
},
"exact": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"supported": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"examined": 73.9041095890411,
"tokens": 720.4794520547945,
"p50": 1.152916,
"p95": 1.379208
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.6786271450858035,
"lo": 0.6427457098283932,
"hi": 0.7129485179407177
},
"any@5": {
"mean": 0.7129485179407177,
"lo": 0.6770670826833073,
"hi": 0.7472698907956318
},
"recall@5": {
"mean": 0.6960478419136765,
"lo": 0.6612064482579303,
"hi": 0.7298491939677588
},
"all@10": {
"mean": 0.7706708268330733,
"lo": 0.7379095163806553,
"hi": 0.8034321372854915
},
"any@10": {
"mean": 0.797191887675507,
"lo": 0.7675507020280812,
"hi": 0.828393135725429
},
"recall@10": {
"mean": 0.7841913676547062,
"lo": 0.7532501300052002,
"hi": 0.8161726469058762
},
"all@20": {
"mean": 0.8299531981279251,
"lo": 0.8018720748829953,
"hi": 0.859594383775351
},
"any@20": {
"mean": 0.8564742589703588,
"lo": 0.8299531981279251,
"hi": 0.8814352574102964
},
"recall@20": {
"mean": 0.843733749349974,
"lo": 0.8177327093083725,
"hi": 0.8705148205928237
},
"mrr": {
"mean": 0.5761941396262525,
"lo": 0.544112559826264,
"hi": 0.606800425944637
},
"ndcg@20": {
"mean": 0.6335368610039814,
"lo": 0.6044092466982808,
"hi": 0.6614255524886651
},
"exact": {
"mean": 0.8564742589703588,
"lo": 0.8299531981279251,
"hi": 0.8814352574102964
},
"supported": {
"mean": 0.8689547581903276,
"lo": 0.84399375975039,
"hi": 0.8954758190327613
},
"examined": 70.8377535101404,
"tokens": 767.3151326053043,
"p50": 1.16275,
"p95": 1.415792
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.5507372072853426,
"lo": 0.5212489158716392,
"hi": 0.577623590633131
},
"any@5": {
"mean": 0.6929748482220295,
"lo": 0.6652211621856028,
"hi": 0.7189939288811795
},
"recall@5": {
"mean": 0.6151125437981775,
"lo": 0.5878877625876302,
"hi": 0.6413635487573043
},
"all@10": {
"mean": 0.6409366869037294,
"lo": 0.6123156981786644,
"hi": 0.6686903729401561
},
"any@10": {
"mean": 0.7918473547267997,
"lo": 0.7675628794449263,
"hi": 0.8152645273200347
},
"recall@10": {
"mean": 0.7107337424762152,
"lo": 0.6852655804946103,
"hi": 0.735598646518202
},
"all@20": {
"mean": 0.7103209019947961,
"lo": 0.6834345186470078,
"hi": 0.7389418907198613
},
"any@20": {
"mean": 0.852558542931483,
"lo": 0.8308759757155247,
"hi": 0.8716392020815265
},
"recall@20": {
"mean": 0.7787791824742142,
"lo": 0.756281812049375,
"hi": 0.801120970775647
},
"mrr": {
"mean": 0.5559146004012201,
"lo": 0.5305414483221604,
"hi": 0.5783457259625566
},
"ndcg@20": {
"mean": 0.5866587362838163,
"lo": 0.5646208726277896,
"hi": 0.6077338311154385
},
"exact": {
"mean": 0.852558542931483,
"lo": 0.8308759757155247,
"hi": 0.8716392020815265
},
"supported": {
"mean": 0.8638334778837814,
"lo": 0.8447528187337381,
"hi": 0.8829141370338248
},
"examined": 70.9479618386817,
"tokens": 771.0182133564614,
"p50": 1.14775,
"p95": 1.405125
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-failed-chain-search-locomo",
"row": "failed: chain search",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "47f46c9ef14f",
"record": {
"commit": "47f46c9ef14f",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:41:15.253Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.12980769230769232,
"lo": 0.08173076923076923,
"hi": 0.1778846153846154
},
"any@5": {
"mean": 0.6490384615384616,
"lo": 0.5865384615384616,
"hi": 0.7115384615384616
},
"recall@5": {
"mean": 0.3566917274976485,
"lo": 0.3080266772700985,
"hi": 0.40285513098013087
},
"all@10": {
"mean": 0.22596153846153846,
"lo": 0.16826923076923078,
"hi": 0.28365384615384615
},
"any@10": {
"mean": 0.7692307692307693,
"lo": 0.7115384615384616,
"hi": 0.8269230769230769
},
"recall@10": {
"mean": 0.4863479722032353,
"lo": 0.4405319826372459,
"hi": 0.5363772831700463
},
"all@20": {
"mean": 0.375,
"lo": 0.3076923076923077,
"hi": 0.4423076923076923
},
"any@20": {
"mean": 0.8798076923076923,
"lo": 0.8365384615384616,
"hi": 0.9230769230769231
},
"recall@20": {
"mean": 0.6266715101419048,
"lo": 0.5799915453698349,
"hi": 0.6748567404817403
},
"mrr": {
"mean": 0.45409886959511875,
"lo": 0.4013457350637708,
"hi": 0.50986032460286
},
"ndcg@20": {
"mean": 0.4359625785149342,
"lo": 0.3930799569415861,
"hi": 0.47853027982250385
},
"exact": {
"mean": 0.8798076923076923,
"lo": 0.8365384615384616,
"hi": 0.9230769230769231
},
"supported": {
"mean": 0.8990384615384616,
"lo": 0.8557692307692307,
"hi": 0.9375
},
"examined": 79.05288461538461,
"tokens": 773.7788461538462,
"p50": 1.627125,
"p95": 1.991417
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.6320346320346321,
"lo": 0.5627705627705628,
"hi": 0.6926406926406926
},
"any@5": {
"mean": 0.7316017316017316,
"lo": 0.670995670995671,
"hi": 0.7878787878787878
},
"recall@5": {
"mean": 0.683982683982684,
"lo": 0.6219336219336219,
"hi": 0.7388167388167388
},
"all@10": {
"mean": 0.7186147186147186,
"lo": 0.658008658008658,
"hi": 0.7748917748917749
},
"any@10": {
"mean": 0.7965367965367965,
"lo": 0.7402597402597403,
"hi": 0.8484848484848485
},
"recall@10": {
"mean": 0.7604617604617604,
"lo": 0.707070707070707,
"hi": 0.8109668109668111
},
"all@20": {
"mean": 0.7878787878787878,
"lo": 0.7316017316017316,
"hi": 0.8398268398268398
},
"any@20": {
"mean": 0.8528138528138528,
"lo": 0.8008658008658008,
"hi": 0.9004329004329005
},
"recall@20": {
"mean": 0.8225108225108225,
"lo": 0.7705627705627704,
"hi": 0.8686868686868687
},
"mrr": {
"mean": 0.6036878362986079,
"lo": 0.5492682790429162,
"hi": 0.6592153937933162
},
"ndcg@20": {
"mean": 0.6403171578773605,
"lo": 0.5895066069149429,
"hi": 0.6904196431858053
},
"exact": {
"mean": 0.8528138528138528,
"lo": 0.8008658008658008,
"hi": 0.9004329004329005
},
"supported": {
"mean": 0.8658008658008658,
"lo": 0.8181818181818182,
"hi": 0.9090909090909091
},
"examined": 77.80086580086581,
"tokens": 775.991341991342,
"p50": 1.663958,
"p95": 2.711958
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.2602739726027397,
"lo": 0.1643835616438356,
"hi": 0.3698630136986301
},
"any@5": {
"mean": 0.4246575342465753,
"lo": 0.3150684931506849,
"hi": 0.547945205479452
},
"recall@5": {
"mean": 0.3264840182648402,
"lo": 0.23173515981735163,
"hi": 0.430365296803653
},
"all@10": {
"mean": 0.3698630136986301,
"lo": 0.2602739726027397,
"hi": 0.4794520547945205
},
"any@10": {
"mean": 0.5616438356164384,
"lo": 0.4520547945205479,
"hi": 0.684931506849315
},
"recall@10": {
"mean": 0.4486301369863014,
"lo": 0.3436073059360731,
"hi": 0.5536529680365295
},
"all@20": {
"mean": 0.4520547945205479,
"lo": 0.3424657534246575,
"hi": 0.5616438356164384
},
"any@20": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"recall@20": {
"mean": 0.5440566014699777,
"lo": 0.4413168754425805,
"hi": 0.6527665860864894
},
"mrr": {
"mean": 0.31270251035561275,
"lo": 0.22914604197843516,
"hi": 0.4064213049910068
},
"ndcg@20": {
"mean": 0.3431396362187805,
"lo": 0.2632092225532381,
"hi": 0.43111309421113453
},
"exact": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"supported": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"examined": 82.87671232876713,
"tokens": 700.2054794520548,
"p50": 1.651625,
"p95": 2.332792
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.6957878315132605,
"lo": 0.6630265210608425,
"hi": 0.7316692667706708
},
"any@5": {
"mean": 0.7269890795631825,
"lo": 0.6942277691107644,
"hi": 0.7613104524180967
},
"recall@5": {
"mean": 0.7119084763390536,
"lo": 0.6788871554862194,
"hi": 0.7459698387935517
},
"all@10": {
"mean": 0.7800312012480499,
"lo": 0.750390015600624,
"hi": 0.8112324492979719
},
"any@10": {
"mean": 0.8049921996879875,
"lo": 0.7753510140405616,
"hi": 0.8361934477379095
},
"recall@10": {
"mean": 0.7927717108684348,
"lo": 0.7628705148205929,
"hi": 0.8231929277171088
},
"all@20": {
"mean": 0.8377535101404057,
"lo": 0.8096723868954758,
"hi": 0.8642745709828393
},
"any@20": {
"mean": 0.8611544461778471,
"lo": 0.8330733229329174,
"hi": 0.8861154446177847
},
"recall@20": {
"mean": 0.8502340093603744,
"lo": 0.8224128965158608,
"hi": 0.875455018200728
},
"mrr": {
"mean": 0.5407060710075053,
"lo": 0.5101199456017053,
"hi": 0.570326391386509
},
"ndcg@20": {
"mean": 0.6084782636544991,
"lo": 0.5802871114819035,
"hi": 0.6351862144663595
},
"exact": {
"mean": 0.8611544461778471,
"lo": 0.8330733229329174,
"hi": 0.8861154446177847
},
"supported": {
"mean": 0.8705148205928237,
"lo": 0.84399375975039,
"hi": 0.8939157566302652
},
"examined": 78.4399375975039,
"tokens": 763.3400936037441,
"p50": 1.633917,
"p95": 1.861458
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.5533391153512576,
"lo": 0.5247181266261925,
"hi": 0.5836947094535994
},
"any@5": {
"mean": 0.6947094535993061,
"lo": 0.6686903729401561,
"hi": 0.7207285342584562
},
"recall@5": {
"mean": 0.6178304822083066,
"lo": 0.5910547364363498,
"hi": 0.6451765830777103
},
"all@10": {
"mean": 0.6418039895923677,
"lo": 0.6149176062445794,
"hi": 0.6712922810060711
},
"any@10": {
"mean": 0.7814397224631396,
"lo": 0.7571552471812663,
"hi": 0.805724197745013
},
"recall@10": {
"mean": 0.709231319645799,
"lo": 0.684319082417864,
"hi": 0.7344023183138532
},
"all@20": {
"mean": 0.7198612315698178,
"lo": 0.6929748482220295,
"hi": 0.7476149176062445
},
"any@20": {
"mean": 0.8516912402428447,
"lo": 0.8317432784041631,
"hi": 0.8716392020815265
},
"recall@20": {
"mean": 0.7849642723476362,
"lo": 0.7631516871249536,
"hi": 0.8066369048079167
},
"mrr": {
"mean": 0.523264813384678,
"lo": 0.49945492932008095,
"hi": 0.5463819313506426
},
"ndcg@20": {
"mean": 0.5669360279681543,
"lo": 0.5459046520104331,
"hi": 0.5889749460119293
},
"exact": {
"mean": 0.8516912402428447,
"lo": 0.8317432784041631,
"hi": 0.8716392020815265
},
"supported": {
"mean": 0.8629661751951431,
"lo": 0.8438855160450998,
"hi": 0.8811795316565482
},
"examined": 78.70338248048569,
"tokens": 763.7606244579358,
"p50": 1.640375,
"p95": 1.961375
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-failed-global-rrf-locomo",
"row": "failed: global RRF",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "47f46c9ef14f",
"record": {
"commit": "47f46c9ef14f",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:41:19.209Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.11057692307692307,
"lo": 0.0673076923076923,
"hi": 0.15384615384615385
},
"any@5": {
"mean": 0.6682692307692307,
"lo": 0.6057692307692307,
"hi": 0.7259615384615384
},
"recall@5": {
"mean": 0.33109300239234457,
"lo": 0.2888030372405374,
"hi": 0.3762095543345542
},
"all@10": {
"mean": 0.22596153846153846,
"lo": 0.16826923076923078,
"hi": 0.27884615384615385
},
"any@10": {
"mean": 0.7884615384615384,
"lo": 0.7355769230769231,
"hi": 0.8365384615384616
},
"recall@10": {
"mean": 0.47852316725671984,
"lo": 0.43209915084915085,
"hi": 0.5262531693452747
},
"all@20": {
"mean": 0.32211538461538464,
"lo": 0.2548076923076923,
"hi": 0.3894230769230769
},
"any@20": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"recall@20": {
"mean": 0.5950704905474642,
"lo": 0.5473974399138875,
"hi": 0.6434327824952825
},
"mrr": {
"mean": 0.4853218982814572,
"lo": 0.4330538538259129,
"hi": 0.5372254481629481
},
"ndcg@20": {
"mean": 0.4248527669565008,
"lo": 0.38551746297787204,
"hi": 0.46375699892190875
},
"exact": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"supported": {
"mean": 0.8990384615384616,
"lo": 0.8557692307692307,
"hi": 0.9375
},
"examined": 138.09134615384616,
"tokens": 765.3990384615385,
"p50": 1.208458,
"p95": 1.862959
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.658008658008658,
"lo": 0.5974025974025974,
"hi": 0.7186147186147186
},
"any@5": {
"mean": 0.70995670995671,
"lo": 0.6493506493506493,
"hi": 0.7619047619047619
},
"recall@5": {
"mean": 0.6857864357864358,
"lo": 0.6262626262626262,
"hi": 0.7402597402597402
},
"all@10": {
"mean": 0.7489177489177489,
"lo": 0.6926406926406926,
"hi": 0.8008658008658008
},
"any@10": {
"mean": 0.8268398268398268,
"lo": 0.7748917748917749,
"hi": 0.8744588744588745
},
"recall@10": {
"mean": 0.7860750360750361,
"lo": 0.7323232323232323,
"hi": 0.8326118326118327
},
"all@20": {
"mean": 0.8008658008658008,
"lo": 0.7445887445887446,
"hi": 0.8484848484848485
},
"any@20": {
"mean": 0.8744588744588745,
"lo": 0.8268398268398268,
"hi": 0.9134199134199135
},
"recall@20": {
"mean": 0.8383838383838385,
"lo": 0.79004329004329,
"hi": 0.8816738816738816
},
"mrr": {
"mean": 0.5220363223826091,
"lo": 0.46962401581385543,
"hi": 0.5718391029506388
},
"ndcg@20": {
"mean": 0.5881376426602214,
"lo": 0.5416243297005839,
"hi": 0.6315579595158466
},
"exact": {
"mean": 0.8744588744588745,
"lo": 0.8268398268398268,
"hi": 0.9134199134199135
},
"supported": {
"mean": 0.8874458874458875,
"lo": 0.8441558441558441,
"hi": 0.922077922077922
},
"examined": 138.08225108225108,
"tokens": 755.5714285714286,
"p50": 1.207209,
"p95": 2.667833
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.2876712328767123,
"lo": 0.1917808219178082,
"hi": 0.3972602739726027
},
"any@5": {
"mean": 0.4931506849315068,
"lo": 0.3835616438356164,
"hi": 0.6027397260273972
},
"recall@5": {
"mean": 0.3775181305398872,
"lo": 0.2798146655922643,
"hi": 0.48025785656728437
},
"all@10": {
"mean": 0.3561643835616438,
"lo": 0.2465753424657534,
"hi": 0.4657534246575342
},
"any@10": {
"mean": 0.547945205479452,
"lo": 0.4383561643835616,
"hi": 0.6712328767123288
},
"recall@10": {
"mean": 0.4342600053720118,
"lo": 0.33313188289014234,
"hi": 0.5376712328767124
},
"all@20": {
"mean": 0.3835616438356164,
"lo": 0.273972602739726,
"hi": 0.5068493150684932
},
"any@20": {
"mean": 0.6301369863013698,
"lo": 0.5205479452054794,
"hi": 0.7397260273972602
},
"recall@20": {
"mean": 0.4730691280248089,
"lo": 0.3715649647156496,
"hi": 0.5799086757990867
},
"mrr": {
"mean": 0.31292988393109267,
"lo": 0.23065361198922846,
"hi": 0.3946219987819504
},
"ndcg@20": {
"mean": 0.31536509414222413,
"lo": 0.2412395475185427,
"hi": 0.3945705779469032
},
"exact": {
"mean": 0.6301369863013698,
"lo": 0.5205479452054794,
"hi": 0.7397260273972602
},
"supported": {
"mean": 0.6438356164383562,
"lo": 0.5342465753424658,
"hi": 0.7534246575342466
},
"examined": 143.13698630136986,
"tokens": 703.068493150685,
"p50": 1.2215,
"p95": 2.2955
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.6895475819032761,
"lo": 0.6536661466458659,
"hi": 0.7254290171606864
},
"any@5": {
"mean": 0.7113884555382215,
"lo": 0.6755070202808112,
"hi": 0.7472698907956318
},
"recall@5": {
"mean": 0.7002080083203329,
"lo": 0.6651066042641707,
"hi": 0.734659386375455
},
"all@10": {
"mean": 0.7893915756630265,
"lo": 0.7566302652106084,
"hi": 0.8205928237129485
},
"any@10": {
"mean": 0.8112324492979719,
"lo": 0.7800312012480499,
"hi": 0.8424336973478939
},
"recall@10": {
"mean": 0.8003120124804992,
"lo": 0.7688507540301612,
"hi": 0.8307332293291732
},
"all@20": {
"mean": 0.8673946957878315,
"lo": 0.8393135725429017,
"hi": 0.8923556942277691
},
"any@20": {
"mean": 0.8954758190327613,
"lo": 0.8705148205928237,
"hi": 0.9204368174726989
},
"recall@20": {
"mean": 0.8819552782111285,
"lo": 0.8543941757670308,
"hi": 0.905616224648986
},
"mrr": {
"mean": 0.5368133335227127,
"lo": 0.50616102807435,
"hi": 0.568451067522818
},
"ndcg@20": {
"mean": 0.6134382835824326,
"lo": 0.5858124666204244,
"hi": 0.6413922130765786
},
"exact": {
"mean": 0.8954758190327613,
"lo": 0.8705148205928237,
"hi": 0.9204368174726989
},
"supported": {
"mean": 0.9048361934477379,
"lo": 0.8814352574102964,
"hi": 0.9266770670826833
},
"examined": 138.3666146645866,
"tokens": 750.8892355694228,
"p50": 1.199792,
"p95": 1.474292
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.5533391153512576,
"lo": 0.5221162185602776,
"hi": 0.5819601040763226
},
"any@5": {
"mean": 0.6895056374674762,
"lo": 0.6626192541196878,
"hi": 0.7155247181266262
},
"recall@5": {
"mean": 0.6103002324605549,
"lo": 0.5816407963346626,
"hi": 0.6362304549778873
},
"all@10": {
"mean": 0.6522116218560278,
"lo": 0.6235906331309627,
"hi": 0.6816999132697311
},
"any@10": {
"mean": 0.7935819601040763,
"lo": 0.769297484822203,
"hi": 0.8169991326973114
},
"recall@10": {
"mean": 0.7162334193537625,
"lo": 0.6898124117109992,
"hi": 0.7413246182817556
},
"all@20": {
"mean": 0.7250650477016479,
"lo": 0.6981786643538594,
"hi": 0.753686036426713
},
"any@20": {
"mean": 0.8725065047701648,
"lo": 0.852558542931483,
"hi": 0.8915871639202082
},
"recall@20": {
"mean": 0.795584309089058,
"lo": 0.7725343749947086,
"hi": 0.8179874268600946
},
"mrr": {
"mean": 0.5103890491135773,
"lo": 0.4861962672982314,
"hi": 0.5329583454906008
},
"ndcg@20": {
"mean": 0.555476810607272,
"lo": 0.5323202764538849,
"hi": 0.5751166908703932
},
"exact": {
"mean": 0.8725065047701648,
"lo": 0.852558542931483,
"hi": 0.8915871639202082
},
"supported": {
"mean": 0.8837814397224631,
"lo": 0.8638334778837814,
"hi": 0.9028620988725065
},
"examined": 138.56201214223765,
"tokens": 751.417172593235,
"p50": 1.20425,
"p95": 1.720541
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-failed-prf-locomo",
"row": "failed: PRF",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "47f46c9ef14f",
"record": {
"commit": "47f46c9ef14f",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:38:08.419Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.10576923076923077,
"lo": 0.0673076923076923,
"hi": 0.15384615384615385
},
"any@5": {
"mean": 0.5240384615384616,
"lo": 0.4567307692307692,
"hi": 0.5961538461538461
},
"recall@5": {
"mean": 0.26918931762681775,
"lo": 0.22596587440337443,
"hi": 0.3142652486402487
},
"all@10": {
"mean": 0.1346153846153846,
"lo": 0.08653846153846154,
"hi": 0.18269230769230768
},
"any@10": {
"mean": 0.6201923076923077,
"lo": 0.5576923076923077,
"hi": 0.6826923076923077
},
"recall@10": {
"mean": 0.33322179465271573,
"lo": 0.28586743117993124,
"hi": 0.37932794544636655
},
"all@20": {
"mean": 0.1971153846153846,
"lo": 0.14423076923076922,
"hi": 0.25
},
"any@20": {
"mean": 0.7548076923076923,
"lo": 0.6971153846153846,
"hi": 0.8076923076923077
},
"recall@20": {
"mean": 0.44976166012350216,
"lo": 0.40219131197749625,
"hi": 0.4968465197083619
},
"mrr": {
"mean": 0.378700701140568,
"lo": 0.3230679827964899,
"hi": 0.43331210723037655
},
"ndcg@20": {
"mean": 0.3246869448854136,
"lo": 0.284925776155068,
"hi": 0.36574283023028953
},
"exact": {
"mean": 0.7548076923076923,
"lo": 0.6971153846153846,
"hi": 0.8076923076923077
},
"supported": {
"mean": 0.7740384615384616,
"lo": 0.7211538461538461,
"hi": 0.8269230769230769
},
"examined": 623.5240384615385,
"tokens": 671.4759615384615,
"p50": 1.0915,
"p95": 1.521792
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.5670995670995671,
"lo": 0.5021645021645021,
"hi": 0.6320346320346321
},
"any@5": {
"mean": 0.6406926406926406,
"lo": 0.5800865800865801,
"hi": 0.7012987012987013
},
"recall@5": {
"mean": 0.602092352092352,
"lo": 0.5386002886002885,
"hi": 0.6637806637806638
},
"all@10": {
"mean": 0.6320346320346321,
"lo": 0.5670995670995671,
"hi": 0.6926406926406926
},
"any@10": {
"mean": 0.7142857142857143,
"lo": 0.6536796536796536,
"hi": 0.7705627705627706
},
"recall@10": {
"mean": 0.6746031746031746,
"lo": 0.613997113997114,
"hi": 0.7308802308802309
},
"all@20": {
"mean": 0.6926406926406926,
"lo": 0.6277056277056277,
"hi": 0.7489177489177489
},
"any@20": {
"mean": 0.7705627705627706,
"lo": 0.7142857142857143,
"hi": 0.8225108225108225
},
"recall@20": {
"mean": 0.7323232323232322,
"lo": 0.673881673881674,
"hi": 0.7849927849927849
},
"mrr": {
"mean": 0.5217816710895998,
"lo": 0.46689454661942703,
"hi": 0.5786338661338659
},
"ndcg@20": {
"mean": 0.5588126719266567,
"lo": 0.5076231493977125,
"hi": 0.609275150746681
},
"exact": {
"mean": 0.7705627705627706,
"lo": 0.7142857142857143,
"hi": 0.8225108225108225
},
"supported": {
"mean": 0.7878787878787878,
"lo": 0.7316017316017316,
"hi": 0.8398268398268398
},
"examined": 632.2121212121212,
"tokens": 678.4329004329004,
"p50": 1.103625,
"p95": 1.867459
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.2602739726027397,
"lo": 0.1643835616438356,
"hi": 0.3561643835616438
},
"any@5": {
"mean": 0.3835616438356164,
"lo": 0.273972602739726,
"hi": 0.4931506849315068
},
"recall@5": {
"mean": 0.3116438356164384,
"lo": 0.2191780821917808,
"hi": 0.4132420091324201
},
"all@10": {
"mean": 0.3287671232876712,
"lo": 0.2191780821917808,
"hi": 0.4383561643835616
},
"any@10": {
"mean": 0.4520547945205479,
"lo": 0.3424657534246575,
"hi": 0.5753424657534246
},
"recall@10": {
"mean": 0.37785388127853875,
"lo": 0.28082191780821913,
"hi": 0.4863013698630138
},
"all@20": {
"mean": 0.3561643835616438,
"lo": 0.2465753424657534,
"hi": 0.4657534246575342
},
"any@20": {
"mean": 0.5342465753424658,
"lo": 0.410958904109589,
"hi": 0.6438356164383562
},
"recall@20": {
"mean": 0.42626913779210307,
"lo": 0.3278941713671769,
"hi": 0.530150416330916
},
"mrr": {
"mean": 0.26469789022961343,
"lo": 0.18564304597109207,
"hi": 0.3548407262448358
},
"ndcg@20": {
"mean": 0.28357041599673233,
"lo": 0.20691991981153923,
"hi": 0.3695673711119848
},
"exact": {
"mean": 0.5342465753424658,
"lo": 0.410958904109589,
"hi": 0.6438356164383562
},
"supported": {
"mean": 0.547945205479452,
"lo": 0.4246575342465753,
"hi": 0.6712328767123288
},
"examined": 635.4520547945206,
"tokens": 610.7945205479452,
"p50": 1.099959,
"p95": 1.563708
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.5507020280811232,
"lo": 0.514820592823713,
"hi": 0.5897035881435257
},
"any@5": {
"mean": 0.5881435257410297,
"lo": 0.5507020280811232,
"hi": 0.625585023400936
},
"recall@5": {
"mean": 0.5686427457098284,
"lo": 0.531461258450338,
"hi": 0.6058242329693188
},
"all@10": {
"mean": 0.6209048361934477,
"lo": 0.5834633385335414,
"hi": 0.6583463338533542
},
"any@10": {
"mean": 0.6505460218408736,
"lo": 0.6146645865834633,
"hi": 0.6879875195007801
},
"recall@10": {
"mean": 0.6349453978159125,
"lo": 0.5988039521580864,
"hi": 0.6703068122724909
},
"all@20": {
"mean": 0.7238689547581904,
"lo": 0.6895475819032761,
"hi": 0.7581903276131046
},
"any@20": {
"mean": 0.750390015600624,
"lo": 0.717628705148206,
"hi": 0.7831513260530422
},
"recall@20": {
"mean": 0.736349453978159,
"lo": 0.7025481019240769,
"hi": 0.7691107644305772
},
"mrr": {
"mean": 0.4295069540508288,
"lo": 0.39596377242971204,
"hi": 0.45999161476785655
},
"ndcg@20": {
"mean": 0.4943878751363225,
"lo": 0.4643900802485395,
"hi": 0.523245254042786
},
"exact": {
"mean": 0.750390015600624,
"lo": 0.717628705148206,
"hi": 0.7831513260530422
},
"supported": {
"mean": 0.765990639625585,
"lo": 0.733229329173167,
"hi": 0.7987519500780031
},
"examined": 637.3588143525741,
"tokens": 663.5553822152887,
"p50": 1.097417,
"p95": 1.297125
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.45533391153512576,
"lo": 0.424978317432784,
"hi": 0.4839549002601908
},
"any@5": {
"mean": 0.5741543798785776,
"lo": 0.5455333911535126,
"hi": 0.6027753686036427
},
"recall@5": {
"mean": 0.505051787857512,
"lo": 0.4770526163631107,
"hi": 0.5317733148609126
},
"all@10": {
"mean": 0.5169124024284475,
"lo": 0.48742411101474414,
"hi": 0.5455333911535126
},
"any@10": {
"mean": 0.645273200346921,
"lo": 0.6175195143104943,
"hi": 0.6730268863833477
},
"recall@10": {
"mean": 0.5721828273672431,
"lo": 0.5443170365135945,
"hi": 0.5990633565700896
},
"all@20": {
"mean": 0.5993061578490894,
"lo": 0.5715524718126627,
"hi": 0.6287944492627927
},
"any@20": {
"mean": 0.7415437987857763,
"lo": 0.7155247181266262,
"hi": 0.7658282740676496
},
"recall@20": {
"mean": 0.6642105282143788,
"lo": 0.6389785849285096,
"hi": 0.6893719171148628
},
"mrr": {
"mean": 0.4283939422309436,
"lo": 0.40242142535221626,
"hi": 0.4533665115432221
},
"ndcg@20": {
"mean": 0.4633338075293741,
"lo": 0.4407187860104831,
"hi": 0.48578222759827966
},
"exact": {
"mean": 0.7415437987857763,
"lo": 0.7155247181266262,
"hi": 0.7658282740676496
},
"supported": {
"mean": 0.7580225498699046,
"lo": 0.7328707718993929,
"hi": 0.7814397224631396
},
"examined": 633.7111882046835,
"tokens": 664.6244579358196,
"p50": 1.098459,
"p95": 1.42575
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-failed-surface-entities-locomo",
"row": "failed: surface entities",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "47f46c9ef14f",
"record": {
"commit": "47f46c9ef14f",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:41:17.264Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.125,
"lo": 0.08173076923076923,
"hi": 0.17307692307692307
},
"any@5": {
"mean": 0.6442307692307693,
"lo": 0.5817307692307693,
"hi": 0.7067307692307693
},
"recall@5": {
"mean": 0.3479131705283021,
"lo": 0.3012230277032909,
"hi": 0.395274864024864
},
"all@10": {
"mean": 0.22596153846153846,
"lo": 0.17307692307692307,
"hi": 0.28365384615384615
},
"any@10": {
"mean": 0.8076923076923077,
"lo": 0.7548076923076923,
"hi": 0.8557692307692307
},
"recall@10": {
"mean": 0.4856972340524973,
"lo": 0.4381052974802975,
"hi": 0.5346135954688588
},
"all@20": {
"mean": 0.3557692307692308,
"lo": 0.2932692307692308,
"hi": 0.42788461538461536
},
"any@20": {
"mean": 0.8798076923076923,
"lo": 0.8365384615384616,
"hi": 0.9230769230769231
},
"recall@20": {
"mean": 0.6133166358495306,
"lo": 0.5657980086434032,
"hi": 0.6626895765053659
},
"mrr": {
"mean": 0.4777713272304619,
"lo": 0.42413581343869783,
"hi": 0.5268045856387202
},
"ndcg@20": {
"mean": 0.43626545808598927,
"lo": 0.3957373004039599,
"hi": 0.4765680141199041
},
"exact": {
"mean": 0.8798076923076923,
"lo": 0.8365384615384616,
"hi": 0.9230769230769231
},
"supported": {
"mean": 0.8942307692307693,
"lo": 0.8509615384615384,
"hi": 0.9326923076923077
},
"examined": 82.76923076923077,
"tokens": 790.7548076923077,
"p50": 0.96725,
"p95": 1.232958
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.6623376623376623,
"lo": 0.5974025974025974,
"hi": 0.7229437229437229
},
"any@5": {
"mean": 0.7532467532467533,
"lo": 0.6926406926406926,
"hi": 0.8008658008658008
},
"recall@5": {
"mean": 0.7095959595959596,
"lo": 0.6507936507936508,
"hi": 0.7611832611832612
},
"all@10": {
"mean": 0.7359307359307359,
"lo": 0.6796536796536796,
"hi": 0.7922077922077922
},
"any@10": {
"mean": 0.8311688311688312,
"lo": 0.7835497835497836,
"hi": 0.8787878787878788
},
"recall@10": {
"mean": 0.7864357864357862,
"lo": 0.7373737373737371,
"hi": 0.8347763347763348
},
"all@20": {
"mean": 0.7878787878787878,
"lo": 0.7316017316017316,
"hi": 0.8398268398268398
},
"any@20": {
"mean": 0.8658008658008658,
"lo": 0.8181818181818182,
"hi": 0.9047619047619048
},
"recall@20": {
"mean": 0.8282828282828283,
"lo": 0.7792207792207794,
"hi": 0.8715728715728717
},
"mrr": {
"mean": 0.6073430713448205,
"lo": 0.5536268879017461,
"hi": 0.6604468095778258
},
"ndcg@20": {
"mean": 0.6465322857031848,
"lo": 0.596984511735602,
"hi": 0.6952982844174961
},
"exact": {
"mean": 0.8658008658008658,
"lo": 0.8181818181818182,
"hi": 0.9047619047619048
},
"supported": {
"mean": 0.8744588744588745,
"lo": 0.8311688311688312,
"hi": 0.9134199134199135
},
"examined": 80.87012987012987,
"tokens": 782.9264069264069,
"p50": 0.997709,
"p95": 1.481834
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.3150684931506849,
"lo": 0.2054794520547945,
"hi": 0.4246575342465753
},
"any@5": {
"mean": 0.5068493150684932,
"lo": 0.3972602739726027,
"hi": 0.6301369863013698
},
"recall@5": {
"mean": 0.3980660757453666,
"lo": 0.29646790222938496,
"hi": 0.5022831050228309
},
"all@10": {
"mean": 0.3698630136986301,
"lo": 0.2602739726027397,
"hi": 0.4931506849315068
},
"any@10": {
"mean": 0.5753424657534246,
"lo": 0.4657534246575342,
"hi": 0.684931506849315
},
"recall@10": {
"mean": 0.4513832930432447,
"lo": 0.35367982809562176,
"hi": 0.5593607305936072
},
"all@20": {
"mean": 0.4383561643835616,
"lo": 0.3150684931506849,
"hi": 0.547945205479452
},
"any@20": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"recall@20": {
"mean": 0.529014968378385,
"lo": 0.4228505848167411,
"hi": 0.6338302444265379
},
"mrr": {
"mean": 0.39346884090034767,
"lo": 0.3012210202278695,
"hi": 0.4913859732352884
},
"ndcg@20": {
"mean": 0.3867692324145459,
"lo": 0.3040943960374479,
"hi": 0.47779070338360696
},
"exact": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"supported": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"examined": 85.08219178082192,
"tokens": 724.0410958904109,
"p50": 0.977917,
"p95": 1.486
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.6770670826833073,
"lo": 0.641185647425897,
"hi": 0.7113884555382215
},
"any@5": {
"mean": 0.7098283931357254,
"lo": 0.6755070202808112,
"hi": 0.7441497659906396
},
"recall@5": {
"mean": 0.6939677587103484,
"lo": 0.6588663546541862,
"hi": 0.7262090483619346
},
"all@10": {
"mean": 0.765990639625585,
"lo": 0.734789391575663,
"hi": 0.797191887675507
},
"any@10": {
"mean": 0.7909516380655226,
"lo": 0.7613104524180967,
"hi": 0.8221528861154446
},
"recall@10": {
"mean": 0.7787311492459699,
"lo": 0.7485699427977118,
"hi": 0.80941237649506
},
"all@20": {
"mean": 0.8299531981279251,
"lo": 0.8018720748829953,
"hi": 0.859594383775351
},
"any@20": {
"mean": 0.8549141965678627,
"lo": 0.8299531981279251,
"hi": 0.8814352574102964
},
"recall@20": {
"mean": 0.842953718148726,
"lo": 0.8172126885075404,
"hi": 0.8692147685907438
},
"mrr": {
"mean": 0.5702985794501804,
"lo": 0.5371639402117971,
"hi": 0.601544058069936
},
"ndcg@20": {
"mean": 0.6286365944670231,
"lo": 0.5993702228869107,
"hi": 0.6556428837760447
},
"exact": {
"mean": 0.8549141965678627,
"lo": 0.8299531981279251,
"hi": 0.8814352574102964
},
"supported": {
"mean": 0.8673946957878315,
"lo": 0.8424336973478939,
"hi": 0.8923556942277691
},
"examined": 81.62558502340093,
"tokens": 770.0889235569423,
"p50": 0.975292,
"p95": 1.316375
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.5516045099739809,
"lo": 0.5221162185602776,
"hi": 0.5802254986990459
},
"any@5": {
"mean": 0.6938421509106678,
"lo": 0.6660884648742411,
"hi": 0.7198612315698178
},
"recall@5": {
"mean": 0.6159364813523839,
"lo": 0.5888301023664686,
"hi": 0.6417459191093189
},
"all@10": {
"mean": 0.6374674761491761,
"lo": 0.6079791847354726,
"hi": 0.6660884648742411
},
"any@10": {
"mean": 0.7883781439722463,
"lo": 0.7649609713790113,
"hi": 0.810928013876843
},
"recall@10": {
"mean": 0.7066863299292361,
"lo": 0.6818972108859974,
"hi": 0.7319578101736771
},
"all@20": {
"mean": 0.7111882046834345,
"lo": 0.6851691240242844,
"hi": 0.7389418907198613
},
"any@20": {
"mean": 0.8508239375542064,
"lo": 0.8300086730268864,
"hi": 0.8707718993928881
},
"recall@20": {
"mean": 0.7787117255984312,
"lo": 0.7559673217841841,
"hi": 0.8015946099934833
},
"mrr": {
"mean": 0.5498328710822903,
"lo": 0.525394707616647,
"hi": 0.5727028397539973
},
"ndcg@20": {
"mean": 0.582205016738027,
"lo": 0.5607012134786218,
"hi": 0.6036769455215404
},
"exact": {
"mean": 0.8508239375542064,
"lo": 0.8300086730268864,
"hi": 0.8707718993928881
},
"supported": {
"mean": 0.8620988725065047,
"lo": 0.8430182133564614,
"hi": 0.8811795316565482
},
"examined": 81.89939288811796,
"tokens": 773.4735472679965,
"p50": 0.976833,
"p95": 1.35675
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-iterative-hops-locomo",
"row": "+iterative hops",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "47f46c9ef14f",
"record": {
"commit": "47f46c9ef14f",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:38:06.789Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.15865384615384615,
"lo": 0.11057692307692307,
"hi": 0.21153846153846154
},
"any@5": {
"mean": 0.6923076923076923,
"lo": 0.625,
"hi": 0.7548076923076923
},
"recall@5": {
"mean": 0.37902008444771584,
"lo": 0.33082400968914133,
"hi": 0.4277220495148129
},
"all@10": {
"mean": 0.2644230769230769,
"lo": 0.20192307692307693,
"hi": 0.3269230769230769
},
"any@10": {
"mean": 0.7884615384615384,
"lo": 0.7259615384615384,
"hi": 0.8413461538461539
},
"recall@10": {
"mean": 0.5070855588125324,
"lo": 0.4546904484404485,
"hi": 0.561892210494184
},
"all@20": {
"mean": 0.375,
"lo": 0.3125,
"hi": 0.4423076923076923
},
"any@20": {
"mean": 0.8798076923076923,
"lo": 0.8365384615384616,
"hi": 0.9230769230769231
},
"recall@20": {
"mean": 0.6186401317980266,
"lo": 0.5702433987795831,
"hi": 0.6668583152958154
},
"mrr": {
"mean": 0.5174447874156882,
"lo": 0.4651763111214023,
"hi": 0.5730865306112015
},
"ndcg@20": {
"mean": 0.46077161012352147,
"lo": 0.41844990921300673,
"hi": 0.5040718899183362
},
"exact": {
"mean": 0.8798076923076923,
"lo": 0.8365384615384616,
"hi": 0.9230769230769231
},
"supported": {
"mean": 0.8894230769230769,
"lo": 0.8509615384615384,
"hi": 0.9326923076923077
},
"examined": 77.26923076923077,
"tokens": 786.8269230769231,
"p50": 1.670792,
"p95": 2.226458
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.70995670995671,
"lo": 0.645021645021645,
"hi": 0.7705627705627706
},
"any@5": {
"mean": 0.7965367965367965,
"lo": 0.7445887445887446,
"hi": 0.8441558441558441
},
"recall@5": {
"mean": 0.7539682539682541,
"lo": 0.6984126984126985,
"hi": 0.8059163059163059
},
"all@10": {
"mean": 0.7748917748917749,
"lo": 0.7186147186147186,
"hi": 0.8311688311688312
},
"any@10": {
"mean": 0.8484848484848485,
"lo": 0.8008658008658008,
"hi": 0.8917748917748918
},
"recall@10": {
"mean": 0.810966810966811,
"lo": 0.7611832611832611,
"hi": 0.8585858585858585
},
"all@20": {
"mean": 0.8138528138528138,
"lo": 0.7575757575757576,
"hi": 0.8614718614718615
},
"any@20": {
"mean": 0.8701298701298701,
"lo": 0.8268398268398268,
"hi": 0.9134199134199135
},
"recall@20": {
"mean": 0.8412698412698413,
"lo": 0.7936507936507937,
"hi": 0.8845598845598848
},
"mrr": {
"mean": 0.6486701129558271,
"lo": 0.5966828145399575,
"hi": 0.6991854706140421
},
"ndcg@20": {
"mean": 0.6831518522468351,
"lo": 0.6353603067095851,
"hi": 0.7301670811130756
},
"exact": {
"mean": 0.8701298701298701,
"lo": 0.8268398268398268,
"hi": 0.9134199134199135
},
"supported": {
"mean": 0.8787878787878788,
"lo": 0.8354978354978355,
"hi": 0.9177489177489178
},
"examined": 75.64502164502164,
"tokens": 787.8268398268398,
"p50": 1.694375,
"p95": 2.656541
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.2876712328767123,
"lo": 0.1917808219178082,
"hi": 0.3972602739726027
},
"any@5": {
"mean": 0.4794520547945205,
"lo": 0.3698630136986301,
"hi": 0.6027397260273972
},
"recall@5": {
"mean": 0.370668815471394,
"lo": 0.27900886381950035,
"hi": 0.4768331990330379
},
"all@10": {
"mean": 0.3561643835616438,
"lo": 0.2465753424657534,
"hi": 0.4657534246575342
},
"any@10": {
"mean": 0.547945205479452,
"lo": 0.4383561643835616,
"hi": 0.6575342465753424
},
"recall@10": {
"mean": 0.4323126510878324,
"lo": 0.3299086757990868,
"hi": 0.5353881278538812
},
"all@20": {
"mean": 0.4520547945205479,
"lo": 0.3424657534246575,
"hi": 0.5616438356164384
},
"any@20": {
"mean": 0.6301369863013698,
"lo": 0.5205479452054794,
"hi": 0.7534246575342466
},
"recall@20": {
"mean": 0.5278673113080848,
"lo": 0.4182782702121945,
"hi": 0.640881009938222
},
"mrr": {
"mean": 0.3744581767950019,
"lo": 0.27834904821206186,
"hi": 0.4731159877815801
},
"ndcg@20": {
"mean": 0.37607553858319165,
"lo": 0.2910236925008464,
"hi": 0.4668658814391134
},
"exact": {
"mean": 0.6301369863013698,
"lo": 0.5205479452054794,
"hi": 0.7534246575342466
},
"supported": {
"mean": 0.6438356164383562,
"lo": 0.5342465753424658,
"hi": 0.7534246575342466
},
"examined": 80.38356164383562,
"tokens": 718.013698630137,
"p50": 1.67925,
"p95": 2.508125
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.7410296411856474,
"lo": 0.7082683307332294,
"hi": 0.7737909516380655
},
"any@5": {
"mean": 0.765990639625585,
"lo": 0.734789391575663,
"hi": 0.797191887675507
},
"recall@5": {
"mean": 0.7535101404056163,
"lo": 0.7220488819552783,
"hi": 0.7847113884555381
},
"all@10": {
"mean": 0.8174726989079563,
"lo": 0.7893915756630265,
"hi": 0.8455538221528861
},
"any@10": {
"mean": 0.8361934477379095,
"lo": 0.8096723868954758,
"hi": 0.8642745709828393
},
"recall@10": {
"mean": 0.827353094123765,
"lo": 0.7995319812792512,
"hi": 0.8541341653666148
},
"all@20": {
"mean": 0.8767550702028081,
"lo": 0.8517940717628705,
"hi": 0.9001560062402496
},
"any@20": {
"mean": 0.8954758190327613,
"lo": 0.8736349453978159,
"hi": 0.9173166926677067
},
"recall@20": {
"mean": 0.8868954758190327,
"lo": 0.8642745709828393,
"hi": 0.9087363494539783
},
"mrr": {
"mean": 0.6102558984551486,
"lo": 0.5803301250300179,
"hi": 0.6407525121920794
},
"ndcg@20": {
"mean": 0.6703539111547279,
"lo": 0.6435301667340992,
"hi": 0.6972491395281866
},
"exact": {
"mean": 0.8954758190327613,
"lo": 0.8736349453978159,
"hi": 0.9173166926677067
},
"supported": {
"mean": 0.9001560062402496,
"lo": 0.8783151326053042,
"hi": 0.921996879875195
},
"examined": 76.16380655226209,
"tokens": 766.3853354134166,
"p50": 1.669,
"p95": 1.901417
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.601040763226366,
"lo": 0.5715524718126627,
"hi": 0.6305290546400694
},
"any@5": {
"mean": 0.7406764960971379,
"lo": 0.7146574154379879,
"hi": 0.7649609713790113
},
"recall@5": {
"mean": 0.6618054360461437,
"lo": 0.6347645213859745,
"hi": 0.6875525432151169
},
"all@10": {
"mean": 0.6799653078924545,
"lo": 0.6530789245446661,
"hi": 0.7094535993061578
},
"any@10": {
"mean": 0.8117953165654813,
"lo": 0.7883781439722463,
"hi": 0.834345186470078
},
"recall@10": {
"mean": 0.7412829890972117,
"lo": 0.7179273530747948,
"hi": 0.7646945312109417
},
"all@20": {
"mean": 0.7467476149176062,
"lo": 0.7198612315698178,
"hi": 0.7718993928881179
},
"any@20": {
"mean": 0.8707718993928881,
"lo": 0.849956634865568,
"hi": 0.8898525585429314
},
"recall@20": {
"mean": 0.8066303508003583,
"lo": 0.7846149733860216,
"hi": 0.8278642906775466
},
"mrr": {
"mean": 0.5862799563669082,
"lo": 0.5619925647051685,
"hi": 0.6083587749656751
},
"ndcg@20": {
"mean": 0.6164778353351822,
"lo": 0.5951503268427839,
"hi": 0.6369324696789576
},
"exact": {
"mean": 0.8707718993928881,
"lo": 0.849956634865568,
"hi": 0.8898525585429314
},
"supported": {
"mean": 0.8777103209019947,
"lo": 0.8568950563746748,
"hi": 0.8967909800520382
},
"examined": 76.52645273200346,
"tokens": 771.3061578490893,
"p50": 1.674209,
"p95": 2.000625
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-lexical-locomo",
"row": "+lexical",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "33d0f8749bd1",
"record": {
"commit": "33d0f8749bd1",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:30:56.727Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.08653846153846154,
"lo": 0.04807692307692308,
"hi": 0.125
},
"any@5": {
"mean": 0.5048076923076923,
"lo": 0.4375,
"hi": 0.5673076923076923
},
"recall@5": {
"mean": 0.25920137976059043,
"lo": 0.21361034798534798,
"hi": 0.29813917807338863
},
"all@10": {
"mean": 0.14423076923076922,
"lo": 0.09615384615384616,
"hi": 0.19230769230769232
},
"any@10": {
"mean": 0.6634615384615384,
"lo": 0.6009615384615384,
"hi": 0.7211538461538461
},
"recall@10": {
"mean": 0.35753030777701833,
"lo": 0.31113400488400494,
"hi": 0.40082122738372744
},
"all@20": {
"mean": 0.21153846153846154,
"lo": 0.15865384615384615,
"hi": 0.2644230769230769
},
"any@20": {
"mean": 0.7836538461538461,
"lo": 0.7307692307692307,
"hi": 0.8365384615384616
},
"recall@20": {
"mean": 0.475636315438947,
"lo": 0.4286607215883531,
"hi": 0.520567946308078
},
"mrr": {
"mean": 0.36730230708651923,
"lo": 0.316604551816433,
"hi": 0.4172857025048394
},
"ndcg@20": {
"mean": 0.3254027080429509,
"lo": 0.28484644750248517,
"hi": 0.3627710369447173
},
"exact": {
"mean": 0.7836538461538461,
"lo": 0.7307692307692307,
"hi": 0.8365384615384616
},
"supported": {
"mean": 0.8028846153846154,
"lo": 0.75,
"hi": 0.8509615384615384
},
"examined": 53.28846153846154,
"tokens": 664.9615384615385,
"p50": 0.605584,
"p95": 0.675042
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.5714285714285714,
"lo": 0.5064935064935064,
"hi": 0.6320346320346321
},
"any@5": {
"mean": 0.6536796536796536,
"lo": 0.5887445887445888,
"hi": 0.70995670995671
},
"recall@5": {
"mean": 0.6121933621933622,
"lo": 0.5461760461760462,
"hi": 0.6691919191919192
},
"all@10": {
"mean": 0.6493506493506493,
"lo": 0.5844155844155844,
"hi": 0.70995670995671
},
"any@10": {
"mean": 0.7316017316017316,
"lo": 0.670995670995671,
"hi": 0.7878787878787878
},
"recall@10": {
"mean": 0.6908369408369409,
"lo": 0.6313131313131314,
"hi": 0.7489177489177491
},
"all@20": {
"mean": 0.7359307359307359,
"lo": 0.6796536796536796,
"hi": 0.7922077922077922
},
"any@20": {
"mean": 0.8095238095238095,
"lo": 0.7532467532467533,
"hi": 0.8528138528138528
},
"recall@20": {
"mean": 0.7734487734487733,
"lo": 0.7186147186147185,
"hi": 0.8210678210678211
},
"mrr": {
"mean": 0.5319195466113564,
"lo": 0.47584868851605217,
"hi": 0.5870079896053919
},
"ndcg@20": {
"mean": 0.575878058777001,
"lo": 0.5247323346555869,
"hi": 0.6227898909956452
},
"exact": {
"mean": 0.8095238095238095,
"lo": 0.7532467532467533,
"hi": 0.8528138528138528
},
"supported": {
"mean": 0.8225108225108225,
"lo": 0.7705627705627706,
"hi": 0.8701298701298701
},
"examined": 52.36363636363637,
"tokens": 652.056277056277,
"p50": 0.610875,
"p95": 0.683875
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.2328767123287671,
"lo": 0.136986301369863,
"hi": 0.3424657534246575
},
"any@5": {
"mean": 0.4246575342465753,
"lo": 0.3150684931506849,
"hi": 0.5342465753424658
},
"recall@5": {
"mean": 0.31701584743486433,
"lo": 0.22683319903303786,
"hi": 0.41633091592801497
},
"all@10": {
"mean": 0.3287671232876712,
"lo": 0.2191780821917808,
"hi": 0.4383561643835616
},
"any@10": {
"mean": 0.4931506849315068,
"lo": 0.3835616438356164,
"hi": 0.6164383561643836
},
"recall@10": {
"mean": 0.3943056674724684,
"lo": 0.2892828364222401,
"hi": 0.499798549556809
},
"all@20": {
"mean": 0.3835616438356164,
"lo": 0.2602739726027397,
"hi": 0.4931506849315068
},
"any@20": {
"mean": 0.5753424657534246,
"lo": 0.4657534246575342,
"hi": 0.684931506849315
},
"recall@20": {
"mean": 0.45947793812419113,
"lo": 0.35482138060703733,
"hi": 0.5621138866505506
},
"mrr": {
"mean": 0.294400614606094,
"lo": 0.2147986716479868,
"hi": 0.3844645080946451
},
"ndcg@20": {
"mean": 0.3123971228699728,
"lo": 0.23405635818963522,
"hi": 0.3991294505955169
},
"exact": {
"mean": 0.5753424657534246,
"lo": 0.4657534246575342,
"hi": 0.684931506849315
},
"supported": {
"mean": 0.6164383561643836,
"lo": 0.5068493150684932,
"hi": 0.726027397260274
},
"examined": 53.95890410958904,
"tokens": 605.9589041095891,
"p50": 0.624458,
"p95": 0.705917
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.6614664586583463,
"lo": 0.62402496099844,
"hi": 0.6989079563182528
},
"any@5": {
"mean": 0.6957878315132605,
"lo": 0.6583463338533542,
"hi": 0.733229329173167
},
"recall@5": {
"mean": 0.6788871554862195,
"lo": 0.6422256890275612,
"hi": 0.7155486219448778
},
"all@10": {
"mean": 0.734789391575663,
"lo": 0.7020280811232449,
"hi": 0.7691107644305772
},
"any@10": {
"mean": 0.7675507020280812,
"lo": 0.734789391575663,
"hi": 0.8003120124804992
},
"recall@10": {
"mean": 0.7514300572022882,
"lo": 0.7199687987519501,
"hi": 0.7839313572542902
},
"all@20": {
"mean": 0.828393135725429,
"lo": 0.8003120124804992,
"hi": 0.858034321372855
},
"any@20": {
"mean": 0.859594383775351,
"lo": 0.8315132605304212,
"hi": 0.8876755070202809
},
"recall@20": {
"mean": 0.84399375975039,
"lo": 0.8166926677067082,
"hi": 0.8720748829953198
},
"mrr": {
"mean": 0.5380483988943944,
"lo": 0.5052907246562063,
"hi": 0.5691097857617007
},
"ndcg@20": {
"mean": 0.602869664118745,
"lo": 0.5730801808058196,
"hi": 0.631216745799341
},
"exact": {
"mean": 0.859594383775351,
"lo": 0.8315132605304212,
"hi": 0.8876755070202809
},
"supported": {
"mean": 0.8689547581903276,
"lo": 0.8424336973478939,
"hi": 0.8970358814352574
},
"examined": 52.85647425897036,
"tokens": 659.9063962558503,
"p50": 0.621041,
"p95": 0.684458
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.5125758889852559,
"lo": 0.4813529921942758,
"hi": 0.5420641803989592
},
"any@5": {
"mean": 0.6357328707718994,
"lo": 0.6071118820468343,
"hi": 0.6643538594969645
},
"recall@5": {
"mean": 0.5669031892335483,
"lo": 0.5371406754700838,
"hi": 0.5945781988941711
},
"all@10": {
"mean": 0.585429314830876,
"lo": 0.5550737207285342,
"hi": 0.614050303555941
},
"any@10": {
"mean": 0.7241977450130095,
"lo": 0.6973113616652211,
"hi": 0.7502168256721596
},
"recall@10": {
"mean": 0.6456206571926367,
"lo": 0.6187121147898524,
"hi": 0.6712817682462089
},
"all@20": {
"mean": 0.6704249783174328,
"lo": 0.6426712922810061,
"hi": 0.6999132697311362
},
"any@20": {
"mean": 0.8178664353859497,
"lo": 0.795316565481353,
"hi": 0.8412836079791848
},
"recall@20": {
"mean": 0.739064102134461,
"lo": 0.7153105838015844,
"hi": 0.7629561225525717
},
"mrr": {
"mean": 0.4905919893311104,
"lo": 0.4654642481635726,
"hi": 0.5150992129484433
},
"ndcg@20": {
"mean": 0.5290165130269249,
"lo": 0.5051311709831977,
"hi": 0.5505917490241086
},
"exact": {
"mean": 0.8178664353859497,
"lo": 0.795316565481353,
"hi": 0.8412836079791848
},
"supported": {
"mean": 0.8317432784041631,
"lo": 0.8100607111882047,
"hi": 0.8542931483087598
},
"examined": 52.90546400693842,
"tokens": 655.8300086730269,
"p50": 0.616,
"p95": 0.685417
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-semantic-only-locomo",
"row": "semantic only",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "33d0f8749bd1",
"record": {
"commit": "33d0f8749bd1",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:30:55.680Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.09615384615384616,
"lo": 0.057692307692307696,
"hi": 0.13942307692307693
},
"any@5": {
"mean": 0.5432692307692307,
"lo": 0.4807692307692308,
"hi": 0.6105769230769231
},
"recall@5": {
"mean": 0.26732450663371726,
"lo": 0.22537739124910178,
"hi": 0.3094032612124718
},
"all@10": {
"mean": 0.12980769230769232,
"lo": 0.08173076923076923,
"hi": 0.17307692307692307
},
"any@10": {
"mean": 0.6971153846153846,
"lo": 0.6394230769230769,
"hi": 0.7548076923076923
},
"recall@10": {
"mean": 0.36680608317121466,
"lo": 0.32094604043617214,
"hi": 0.41264097744360906
},
"all@20": {
"mean": 0.1875,
"lo": 0.1346153846153846,
"hi": 0.2403846153846154
},
"any@20": {
"mean": 0.8028846153846154,
"lo": 0.75,
"hi": 0.8557692307692307
},
"recall@20": {
"mean": 0.47537475317080563,
"lo": 0.43224067599067595,
"hi": 0.5252327533577532
},
"mrr": {
"mean": 0.3976824408441311,
"lo": 0.3437570779423008,
"hi": 0.45152536608128724
},
"ndcg@20": {
"mean": 0.3397717156608031,
"lo": 0.30280721880296124,
"hi": 0.37964207488855084
},
"exact": {
"mean": 0.8028846153846154,
"lo": 0.75,
"hi": 0.8557692307692307
},
"supported": {
"mean": 0.8269230769230769,
"lo": 0.7788461538461539,
"hi": 0.875
},
"examined": 40,
"tokens": 655.7596153846154,
"p50": 0.545666,
"p95": 0.762542
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.5800865800865801,
"lo": 0.5151515151515151,
"hi": 0.6406926406926406
},
"any@5": {
"mean": 0.658008658008658,
"lo": 0.5974025974025974,
"hi": 0.7142857142857143
},
"recall@5": {
"mean": 0.6183261183261183,
"lo": 0.5562770562770563,
"hi": 0.6760461760461761
},
"all@10": {
"mean": 0.6666666666666666,
"lo": 0.6060606060606061,
"hi": 0.7229437229437229
},
"any@10": {
"mean": 0.7402597402597403,
"lo": 0.683982683982684,
"hi": 0.7965367965367965
},
"recall@10": {
"mean": 0.7041847041847041,
"lo": 0.6464646464646464,
"hi": 0.7582972582972584
},
"all@20": {
"mean": 0.7229437229437229,
"lo": 0.6623376623376623,
"hi": 0.7748917748917749
},
"any@20": {
"mean": 0.7965367965367965,
"lo": 0.7445887445887446,
"hi": 0.8441558441558441
},
"recall@20": {
"mean": 0.7619047619047618,
"lo": 0.7056277056277056,
"hi": 0.8095238095238094
},
"mrr": {
"mean": 0.5306430737866947,
"lo": 0.47920992476951235,
"hi": 0.5838765028509109
},
"ndcg@20": {
"mean": 0.5718485997519132,
"lo": 0.5222930854878948,
"hi": 0.6203708214134132
},
"exact": {
"mean": 0.7965367965367965,
"lo": 0.7445887445887446,
"hi": 0.8441558441558441
},
"supported": {
"mean": 0.8138528138528138,
"lo": 0.7619047619047619,
"hi": 0.8614718614718615
},
"examined": 40,
"tokens": 661.1601731601731,
"p50": 0.54875,
"p95": 0.90875
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.2876712328767123,
"lo": 0.1917808219178082,
"hi": 0.3972602739726027
},
"any@5": {
"mean": 0.410958904109589,
"lo": 0.3013698630136986,
"hi": 0.5342465753424658
},
"recall@5": {
"mean": 0.339041095890411,
"lo": 0.24429223744292242,
"hi": 0.4452054794520548
},
"all@10": {
"mean": 0.3424657534246575,
"lo": 0.2328767123287671,
"hi": 0.4520547945205479
},
"any@10": {
"mean": 0.4657534246575342,
"lo": 0.3561643835616438,
"hi": 0.589041095890411
},
"recall@10": {
"mean": 0.3926940639269406,
"lo": 0.29109589041095885,
"hi": 0.4965753424657534
},
"all@20": {
"mean": 0.3561643835616438,
"lo": 0.2465753424657534,
"hi": 0.4657534246575342
},
"any@20": {
"mean": 0.5068493150684932,
"lo": 0.3972602739726027,
"hi": 0.6164383561643836
},
"recall@20": {
"mean": 0.4223744292237443,
"lo": 0.319634703196347,
"hi": 0.526255707762557
},
"mrr": {
"mean": 0.27187481468303387,
"lo": 0.19182974559686894,
"hi": 0.3655088062622309
},
"ndcg@20": {
"mean": 0.28694090646781806,
"lo": 0.2111605046501864,
"hi": 0.37032842959964085
},
"exact": {
"mean": 0.5068493150684932,
"lo": 0.3972602739726027,
"hi": 0.6164383561643836
},
"supported": {
"mean": 0.5342465753424658,
"lo": 0.4246575342465753,
"hi": 0.6575342465753424
},
"examined": 40,
"tokens": 599.5890410958904,
"p50": 0.55025,
"p95": 0.92025
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.5897035881435257,
"lo": 0.5522620904836193,
"hi": 0.6271450858034321
},
"any@5": {
"mean": 0.6271450858034321,
"lo": 0.5897035881435257,
"hi": 0.6645865834633385
},
"recall@5": {
"mean": 0.6076443057722309,
"lo": 0.5702028081123245,
"hi": 0.6443057722308891
},
"all@10": {
"mean": 0.6973478939157566,
"lo": 0.6614664586583463,
"hi": 0.734789391575663
},
"any@10": {
"mean": 0.7254290171606864,
"lo": 0.6895475819032761,
"hi": 0.7597503900156006
},
"recall@10": {
"mean": 0.7106084243369734,
"lo": 0.6747269890795631,
"hi": 0.7459698387935517
},
"all@20": {
"mean": 0.7831513260530422,
"lo": 0.750390015600624,
"hi": 0.8159126365054602
},
"any@20": {
"mean": 0.8127925117004681,
"lo": 0.7831513260530422,
"hi": 0.8424336973478939
},
"recall@20": {
"mean": 0.798491939677587,
"lo": 0.7670306812272492,
"hi": 0.828653146125845
},
"mrr": {
"mean": 0.4547518955359953,
"lo": 0.4217949321419973,
"hi": 0.4848926725286338
},
"ndcg@20": {
"mean": 0.5287422583244219,
"lo": 0.49944231537926476,
"hi": 0.5556159678730394
},
"exact": {
"mean": 0.8127925117004681,
"lo": 0.7831513260530422,
"hi": 0.8424336973478939
},
"supported": {
"mean": 0.8252730109204368,
"lo": 0.7956318252730109,
"hi": 0.8549141965678627
},
"examined": 40,
"tokens": 651.182527301092,
"p50": 0.5485,
"p95": 0.58275
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.47961838681699914,
"lo": 0.4483954900260191,
"hi": 0.5099739809193409
},
"any@5": {
"mean": 0.6045099739809193,
"lo": 0.575021682567216,
"hi": 0.6331309627059843
},
"recall@5": {
"mean": 0.5313849355708122,
"lo": 0.501374948448943,
"hi": 0.5586053164133229
},
"all@10": {
"mean": 0.5663486556808326,
"lo": 0.5359930615784909,
"hi": 0.5949696444058976
},
"any@10": {
"mean": 0.7068516912402428,
"lo": 0.6808326105810928,
"hi": 0.7320034692107545
},
"recall@10": {
"mean": 0.6271717247467009,
"lo": 0.5998230780241094,
"hi": 0.6526765200077382
},
"all@20": {
"mean": 0.6366001734605378,
"lo": 0.6079791847354726,
"hi": 0.6652211621856028
},
"any@20": {
"mean": 0.7883781439722463,
"lo": 0.7623590633130962,
"hi": 0.8100607111882047
},
"recall@20": {
"mean": 0.7090586429542014,
"lo": 0.6836520326546341,
"hi": 0.7329121658913507
},
"mrr": {
"mean": 0.4480826749789595,
"lo": 0.4233628662868976,
"hi": 0.47175110962578837
},
"ndcg@20": {
"mean": 0.4879791996168637,
"lo": 0.4671145165653039,
"hi": 0.5092383408442205
},
"exact": {
"mean": 0.7883781439722463,
"lo": 0.7623590633130962,
"hi": 0.8100607111882047
},
"supported": {
"mean": 0.8048568950563747,
"lo": 0.7805724197745013,
"hi": 0.8274067649609714
},
"examined": 40,
"tokens": 650.7406764960972,
"p50": 0.548333,
"p95": 0.597875
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-timeline-locomo",
"row": "+timeline",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "33d0f8749bd1",
"record": {
"commit": "33d0f8749bd1",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:31:01.864Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.1346153846153846,
"lo": 0.08653846153846154,
"hi": 0.18269230769230768
},
"any@5": {
"mean": 0.6346153846153846,
"lo": 0.5721153846153846,
"hi": 0.6971153846153846
},
"recall@5": {
"mean": 0.34815355514368673,
"lo": 0.30046984265734267,
"hi": 0.3954288766788766
},
"all@10": {
"mean": 0.23557692307692307,
"lo": 0.18269230769230768,
"hi": 0.2932692307692308
},
"any@10": {
"mean": 0.8125,
"lo": 0.7548076923076923,
"hi": 0.8653846153846154
},
"recall@10": {
"mean": 0.49611390071916384,
"lo": 0.4491053537106167,
"hi": 0.5438030171582804
},
"all@20": {
"mean": 0.34615384615384615,
"lo": 0.28365384615384615,
"hi": 0.4182692307692308
},
"any@20": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"recall@20": {
"mean": 0.60888287516577,
"lo": 0.5607238247863247,
"hi": 0.6567054473304469
},
"mrr": {
"mean": 0.4941957014673361,
"lo": 0.43971566627816644,
"hi": 0.5476737672050171
},
"ndcg@20": {
"mean": 0.4435250499492416,
"lo": 0.40231396875664227,
"hi": 0.4848359964643688
},
"exact": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"supported": {
"mean": 0.8990384615384616,
"lo": 0.8605769230769231,
"hi": 0.9375
},
"examined": 71.39423076923077,
"tokens": 787.9567307692307,
"p50": 1.03275,
"p95": 1.365125
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.6666666666666666,
"lo": 0.6017316017316018,
"hi": 0.7272727272727273
},
"any@5": {
"mean": 0.7575757575757576,
"lo": 0.7012987012987013,
"hi": 0.8095238095238095
},
"recall@5": {
"mean": 0.7146464646464645,
"lo": 0.6587301587301588,
"hi": 0.7655122655122656
},
"all@10": {
"mean": 0.7445887445887446,
"lo": 0.6883116883116883,
"hi": 0.8008658008658008
},
"any@10": {
"mean": 0.8311688311688312,
"lo": 0.7792207792207793,
"hi": 0.8787878787878788
},
"recall@10": {
"mean": 0.7907647907647906,
"lo": 0.7373737373737375,
"hi": 0.8405483405483404
},
"all@20": {
"mean": 0.7965367965367965,
"lo": 0.7402597402597403,
"hi": 0.8484848484848485
},
"any@20": {
"mean": 0.8744588744588745,
"lo": 0.8311688311688312,
"hi": 0.9134199134199135
},
"recall@20": {
"mean": 0.8362193362193363,
"lo": 0.7886002886002885,
"hi": 0.8802308802308801
},
"mrr": {
"mean": 0.6378219561470132,
"lo": 0.5789315818154627,
"hi": 0.6912945573086903
},
"ndcg@20": {
"mean": 0.6714383789237611,
"lo": 0.620733519820475,
"hi": 0.7190888722432123
},
"exact": {
"mean": 0.8744588744588745,
"lo": 0.8311688311688312,
"hi": 0.9134199134199135
},
"supported": {
"mean": 0.8831168831168831,
"lo": 0.8398268398268398,
"hi": 0.922077922077922
},
"examined": 70.23809523809524,
"tokens": 777.6926406926407,
"p50": 1.042708,
"p95": 1.762084
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.3287671232876712,
"lo": 0.2328767123287671,
"hi": 0.4383561643835616
},
"any@5": {
"mean": 0.5205479452054794,
"lo": 0.410958904109589,
"hi": 0.6301369863013698
},
"recall@5": {
"mean": 0.4117647058823529,
"lo": 0.31406124093473003,
"hi": 0.5167875369325812
},
"all@10": {
"mean": 0.3835616438356164,
"lo": 0.273972602739726,
"hi": 0.4931506849315068
},
"any@10": {
"mean": 0.6027397260273972,
"lo": 0.4931506849315068,
"hi": 0.726027397260274
},
"recall@10": {
"mean": 0.46964813322589316,
"lo": 0.3689900617781358,
"hi": 0.5776255707762558
},
"all@20": {
"mean": 0.4520547945205479,
"lo": 0.3424657534246575,
"hi": 0.5753424657534246
},
"any@20": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"recall@20": {
"mean": 0.5358642834468781,
"lo": 0.43312455741948086,
"hi": 0.6415525114155252
},
"mrr": {
"mean": 0.39454599814188857,
"lo": 0.30572505880725065,
"hi": 0.49585878353001633
},
"ndcg@20": {
"mean": 0.3933342621546875,
"lo": 0.30896459893545547,
"hi": 0.48342238137466076
},
"exact": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"supported": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"examined": 74.08219178082192,
"tokens": 717.3835616438356,
"p50": 1.0435,
"p95": 1.599084
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.6973478939157566,
"lo": 0.6614664586583463,
"hi": 0.7316692667706708
},
"any@5": {
"mean": 0.733229329173167,
"lo": 0.6989079563182528,
"hi": 0.765990639625585
},
"recall@5": {
"mean": 0.7152886115444618,
"lo": 0.6809672386895476,
"hi": 0.7480499219968799
},
"all@10": {
"mean": 0.7878315132605305,
"lo": 0.7566302652106084,
"hi": 0.8190327613104524
},
"any@10": {
"mean": 0.8143525741029641,
"lo": 0.7847113884555382,
"hi": 0.84399375975039
},
"recall@10": {
"mean": 0.8010920436817472,
"lo": 0.7717108684347374,
"hi": 0.8302132085283412
},
"all@20": {
"mean": 0.8330733229329174,
"lo": 0.8049921996879875,
"hi": 0.8627145085803433
},
"any@20": {
"mean": 0.859594383775351,
"lo": 0.8330733229329174,
"hi": 0.8845553822152886
},
"recall@20": {
"mean": 0.8468538741549663,
"lo": 0.8205928237129485,
"hi": 0.8725949037961519
},
"mrr": {
"mean": 0.5916945656355463,
"lo": 0.5602100278307032,
"hi": 0.6224894800477528
},
"ndcg@20": {
"mean": 0.6466226025086881,
"lo": 0.617359084017186,
"hi": 0.6738880882521819
},
"exact": {
"mean": 0.859594383775351,
"lo": 0.8330733229329174,
"hi": 0.8845553822152886
},
"supported": {
"mean": 0.8736349453978159,
"lo": 0.8486739469578783,
"hi": 0.8985959438377535
},
"examined": 71.05304212168487,
"tokens": 765.7613104524181,
"p50": 1.048208,
"p95": 1.186541
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.5663486556808326,
"lo": 0.5359930615784909,
"hi": 0.593235039028621
},
"any@5": {
"mean": 0.7068516912402428,
"lo": 0.6808326105810928,
"hi": 0.7320034692107545
},
"recall@5": {
"mean": 0.6297121390569228,
"lo": 0.6030960212439277,
"hi": 0.6547472655733106
},
"all@10": {
"mean": 0.6539462272333044,
"lo": 0.6253252385082394,
"hi": 0.6825672159583694
},
"any@10": {
"mean": 0.8039895923677364,
"lo": 0.779705117085863,
"hi": 0.826539462272333
},
"recall@10": {
"mean": 0.7230205305652583,
"lo": 0.6973637543009007,
"hi": 0.746974495072806
},
"all@20": {
"mean": 0.7137901127493496,
"lo": 0.6869037294015612,
"hi": 0.7424111014744146
},
"any@20": {
"mean": 0.8560277536860365,
"lo": 0.8352124891587164,
"hi": 0.8759757155247181
},
"recall@20": {
"mean": 0.7821038427806611,
"lo": 0.759638233138444,
"hi": 0.8038792415176274
},
"mrr": {
"mean": 0.5708652664457147,
"lo": 0.5456264741183747,
"hi": 0.5938913999542285
},
"ndcg@20": {
"mean": 0.5989193107252315,
"lo": 0.5767339135661658,
"hi": 0.6196109275092635
},
"exact": {
"mean": 0.8560277536860365,
"lo": 0.8352124891587164,
"hi": 0.8759757155247181
},
"supported": {
"mean": 0.8681699913269731,
"lo": 0.8490893321769297,
"hi": 0.8863833477883781
},
"examined": 71.14310494362533,
"tokens": 769.0928013876843,
"p50": 1.044584,
"p95": 1.318708
}
}
],
"table": null
},
{
"bench": "locomo",
"run": "locomo-retrieval-test-typed-graph-locomo",
"row": "+typed graph",
"split": "test",
"k": 20,
"reader": null,
"facts": "locomo",
"commit": "33d0f8749bd1",
"record": {
"commit": "33d0f8749bd1",
"prompt": null,
"prompt_digest": null,
"dataset_digest": null,
"store_digest": "688cea6b889e484c",
"facts_digest": null,
"reader": null,
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": null,
"max_tokens": null,
"questions": null,
"answered": null,
"started": "2026-09-09T09:31:04.904Z",
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "multi-hop",
"n": 208,
"metrics": {
"all@5": {
"mean": 0.12980769230769232,
"lo": 0.08173076923076923,
"hi": 0.1778846153846154
},
"any@5": {
"mean": 0.6586538461538461,
"lo": 0.5961538461538461,
"hi": 0.7163461538461539
},
"recall@5": {
"mean": 0.35167919616932775,
"lo": 0.30563186813186827,
"hi": 0.39840105070368226
},
"all@10": {
"mean": 0.23076923076923078,
"lo": 0.17307692307692307,
"hi": 0.28846153846153844
},
"any@10": {
"mean": 0.8028846153846154,
"lo": 0.7451923076923077,
"hi": 0.8557692307692307
},
"recall@10": {
"mean": 0.48861046665322977,
"lo": 0.4407730062335326,
"hi": 0.5376833875189136
},
"all@20": {
"mean": 0.375,
"lo": 0.3076923076923077,
"hi": 0.44711538461538464
},
"any@20": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"recall@20": {
"mean": 0.6190934978763927,
"lo": 0.5723412095944987,
"hi": 0.6687131490585435
},
"mrr": {
"mean": 0.48788638578302035,
"lo": 0.43393214691291626,
"hi": 0.538372584947104
},
"ndcg@20": {
"mean": 0.4432587162151551,
"lo": 0.4022063231280031,
"hi": 0.48446323683833525
},
"exact": {
"mean": 0.8846153846153846,
"lo": 0.8413461538461539,
"hi": 0.9278846153846154
},
"supported": {
"mean": 0.8990384615384616,
"lo": 0.8557692307692307,
"hi": 0.9375
},
"examined": 75.13461538461539,
"tokens": 785.0673076923077,
"p50": 0.970125,
"p95": 1.207333
}
},
{
"name": "temporal",
"n": 231,
"metrics": {
"all@5": {
"mean": 0.683982683982684,
"lo": 0.6190476190476191,
"hi": 0.7445887445887446
},
"any@5": {
"mean": 0.7662337662337663,
"lo": 0.70995670995671,
"hi": 0.8181818181818182
},
"recall@5": {
"mean": 0.7261904761904762,
"lo": 0.66991341991342,
"hi": 0.7781385281385281
},
"all@10": {
"mean": 0.7619047619047619,
"lo": 0.7012987012987013,
"hi": 0.8181818181818182
},
"any@10": {
"mean": 0.8354978354978355,
"lo": 0.7878787878787878,
"hi": 0.8787878787878788
},
"recall@10": {
"mean": 0.8001443001443,
"lo": 0.7467532467532467,
"hi": 0.8463203463203464
},
"all@20": {
"mean": 0.8008658008658008,
"lo": 0.7445887445887446,
"hi": 0.8528138528138528
},
"any@20": {
"mean": 0.8787878787878788,
"lo": 0.8354978354978355,
"hi": 0.9177489177489178
},
"recall@20": {
"mean": 0.8405483405483406,
"lo": 0.7950937950937951,
"hi": 0.8838383838383838
},
"mrr": {
"mean": 0.6347992753645924,
"lo": 0.5794278292654917,
"hi": 0.6858974076315557
},
"ndcg@20": {
"mean": 0.6722673726443379,
"lo": 0.6212415550282798,
"hi": 0.7197764705975214
},
"exact": {
"mean": 0.8787878787878788,
"lo": 0.8354978354978355,
"hi": 0.9177489177489178
},
"supported": {
"mean": 0.8874458874458875,
"lo": 0.8441558441558441,
"hi": 0.9264069264069265
},
"examined": 73.66233766233766,
"tokens": 777.008658008658,
"p50": 0.986375,
"p95": 1.559833
}
},
{
"name": "open-domain",
"n": 73,
"metrics": {
"all@5": {
"mean": 0.3287671232876712,
"lo": 0.2328767123287671,
"hi": 0.4383561643835616
},
"any@5": {
"mean": 0.5205479452054794,
"lo": 0.410958904109589,
"hi": 0.6301369863013698
},
"recall@5": {
"mean": 0.4117647058823529,
"lo": 0.31406124093473003,
"hi": 0.5167875369325812
},
"all@10": {
"mean": 0.3835616438356164,
"lo": 0.273972602739726,
"hi": 0.4931506849315068
},
"any@10": {
"mean": 0.6027397260273972,
"lo": 0.4931506849315068,
"hi": 0.726027397260274
},
"recall@10": {
"mean": 0.46964813322589316,
"lo": 0.3689900617781358,
"hi": 0.5776255707762558
},
"all@20": {
"mean": 0.4520547945205479,
"lo": 0.3424657534246575,
"hi": 0.5753424657534246
},
"any@20": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"recall@20": {
"mean": 0.5350584816741143,
"lo": 0.43171440431714403,
"hi": 0.6415525114155252
},
"mrr": {
"mean": 0.3873135046080251,
"lo": 0.2951820629902821,
"hi": 0.4817606670346397
},
"ndcg@20": {
"mean": 0.38620687414325416,
"lo": 0.30204649790048815,
"hi": 0.47683526263650616
},
"exact": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"supported": {
"mean": 0.684931506849315,
"lo": 0.5753424657534246,
"hi": 0.7945205479452054
},
"examined": 77.89041095890411,
"tokens": 715.3424657534247,
"p50": 0.986667,
"p95": 1.405584
}
},
{
"name": "single-hop",
"n": 641,
"metrics": {
"all@5": {
"mean": 0.7254290171606864,
"lo": 0.6911076443057722,
"hi": 0.7597503900156006
},
"any@5": {
"mean": 0.7472698907956318,
"lo": 0.7145085803432137,
"hi": 0.781591263650546
},
"recall@5": {
"mean": 0.7366094643785752,
"lo": 0.7048881955278211,
"hi": 0.7688507540301612
},
"all@10": {
"mean": 0.8034321372854915,
"lo": 0.7737909516380655,
"hi": 0.8330733229329174
},
"any@10": {
"mean": 0.828393135725429,
"lo": 0.8003120124804992,
"hi": 0.859594383775351
},
"recall@10": {
"mean": 0.8159126365054602,
"lo": 0.7867914716588662,
"hi": 0.8447737909516381
},
"all@20": {
"mean": 0.8455538221528861,
"lo": 0.8174726989079563,
"hi": 0.875195007800312
},
"any@20": {
"mean": 0.8705148205928237,
"lo": 0.84399375975039,
"hi": 0.8970358814352574
},
"recall@20": {
"mean": 0.858554342173687,
"lo": 0.8320332813312531,
"hi": 0.8871554862194487
},
"mrr": {
"mean": 0.6005214329034746,
"lo": 0.5674326056311919,
"hi": 0.6314057740227662
},
"ndcg@20": {
"mean": 0.6574000985786645,
"lo": 0.6278209589648567,
"hi": 0.6853441656033034
},
"exact": {
"mean": 0.8705148205928237,
"lo": 0.84399375975039,
"hi": 0.8970358814352574
},
"supported": {
"mean": 0.8845553822152886,
"lo": 0.859594383775351,
"hi": 0.9095163806552262
},
"examined": 74.31825273010921,
"tokens": 762.7067082683308,
"p50": 0.976375,
"p95": 1.073958
}
},
{
"name": "all",
"n": 1153,
"metrics": {
"all@5": {
"mean": 0.5845620121422377,
"lo": 0.5542064180398959,
"hi": 0.611448395490026
},
"any@5": {
"mean": 0.7207285342584562,
"lo": 0.6938421509106678,
"hi": 0.7467476149176062
},
"recall@5": {
"mean": 0.6445141049430171,
"lo": 0.6186099763068846,
"hi": 0.6702628753149131
},
"all@10": {
"mean": 0.6652211621856028,
"lo": 0.6366001734605378,
"hi": 0.6938421509106678
},
"any@10": {
"mean": 0.810928013876843,
"lo": 0.7866435385949696,
"hi": 0.8334778837814397
},
"recall@10": {
"mean": 0.7317854502365094,
"lo": 0.7059838166500282,
"hi": 0.7543627640548719
},
"all@20": {
"mean": 0.7267996530789246,
"lo": 0.6999132697311362,
"hi": 0.7545533391153513
},
"any@20": {
"mean": 0.8629661751951431,
"lo": 0.8421509106678231,
"hi": 0.8820468343451865
},
"recall@20": {
"mean": 0.7912668835390286,
"lo": 0.7689623461397325,
"hi": 0.8119539943580204
},
"mrr": {
"mean": 0.5735707937377295,
"lo": 0.548177782571671,
"hi": 0.5966556147111853
},
"ndcg@20": {
"mean": 0.604577745927992,
"lo": 0.5818404410285393,
"hi": 0.6260280189916164
},
"exact": {
"mean": 0.8629661751951431,
"lo": 0.8421509106678231,
"hi": 0.8820468343451865
},
"supported": {
"mean": 0.8751084128360798,
"lo": 0.8542931483087598,
"hi": 0.8941890719861232
},
"examined": 74.56027753686037,
"tokens": 766.6071118820469,
"p50": 0.977042,
"p95": 1.156292
}
}
],
"table": null
},
{
"bench": "mab",
"run": "mab-dev-beam-none",
"row": "beam",
"split": "dev",
"k": 10,
"reader": "none",
"facts": null,
"commit": "b2bb961938b",
"record": {
"commit": "b2bb961938b",
"prompt": "prompts/reader-mab.txt",
"prompt_digest": "7d9cee398c4536d60756c86bd5d073679202e5a63ad594ecd1a56704873d2eee",
"dataset_digest": "24d5c3f09ce0ce15625cb9f8a98f44f0d864ca6c94d7b4ad04eb697ca3a5ff45",
"store_digest": null,
"facts_digest": null,
"reader": "none",
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": 0,
"max_tokens": 64,
"questions": null,
"answered": null,
"started": null,
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "mh_6k",
"n": 100,
"metrics": {
"substring_em": 0.88
}
},
{
"name": "sh_6k",
"n": 100,
"metrics": {
"substring_em": 0.98
}
}
],
"table": null
},
{
"bench": "mab",
"run": "mab-dev-beamr-none",
"row": "beamr",
"split": "dev",
"k": 10,
"reader": "none",
"facts": null,
"commit": "b2bb961938",
"record": {
"commit": "b2bb961938",
"prompt": "prompts/reader-mab.txt",
"prompt_digest": "7d9cee398c4536d60756c86bd5d073679202e5a63ad594ecd1a56704873d2eee",
"dataset_digest": "24d5c3f09ce0ce15625cb9f8a98f44f0d864ca6c94d7b4ad04eb697ca3a5ff45",
"store_digest": null,
"facts_digest": null,
"reader": "none",
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": 0,
"max_tokens": 64,
"questions": null,
"answered": null,
"started": null,
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "mh_6k",
"n": 100,
"metrics": {
"substring_em": 0.89
}
},
{
"name": "sh_6k",
"n": 100,
"metrics": {
"substring_em": 0.98
}
}
],
"table": null
},
{
"bench": "mab",
"run": "mab-dev-beamx-none",
"row": "beamx",
"split": "dev",
"k": 10,
"reader": "none",
"facts": null,
"commit": "2bce2e8f3e1",
"record": {
"commit": "2bce2e8f3e1",
"prompt": "prompts/reader-mab.txt",
"prompt_digest": "7d9cee398c4536d60756c86bd5d073679202e5a63ad594ecd1a56704873d2eee",
"dataset_digest": "24d5c3f09ce0ce15625cb9f8a98f44f0d864ca6c94d7b4ad04eb697ca3a5ff45",
"store_digest": null,
"facts_digest": null,
"reader": "none",
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": 0,
"max_tokens": 64,
"questions": null,
"answered": null,
"started": null,
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "mh_6k",
"n": 100,
"metrics": {
"substring_em": 0.93
}
},
{
"name": "sh_6k",
"n": 100,
"metrics": {
"substring_em": 1
}
}
],
"table": null
},
{
"bench": "mab",
"run": "mab-dev-noreader-none",
"row": "noreader",
"split": "dev",
"k": 10,
"reader": "none",
"facts": null,
"commit": "12468828cf",
"record": {
"commit": "12468828cf",
"prompt": "prompts/reader-mab.txt",
"prompt_digest": "7d9cee398c4536d60756c86bd5d073679202e5a63ad594ecd1a56704873d2eee",
"dataset_digest": "24d5c3f09ce0ce15625cb9f8a98f44f0d864ca6c94d7b4ad04eb697ca3a5ff45",
"store_digest": null,
"facts_digest": null,
"reader": "none",
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": 0,
"max_tokens": 64,
"questions": null,
"answered": null,
"started": null,
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "mh_6k",
"n": 100,
"metrics": {
"substring_em": 0.76
}
},
{
"name": "sh_6k",
"n": 100,
"metrics": {
"substring_em": 0.96
}
}
],
"table": null
},
{
"bench": "mab",
"run": "mab-test-beam-none",
"row": "beam",
"split": "test",
"k": 10,
"reader": "none",
"facts": null,
"commit": "9eab30f734",
"record": {
"commit": "9eab30f734",
"prompt": "prompts/reader-mab.txt",
"prompt_digest": "7d9cee398c4536d60756c86bd5d073679202e5a63ad594ecd1a56704873d2eee",
"dataset_digest": "24d5c3f09ce0ce15625cb9f8a98f44f0d864ca6c94d7b4ad04eb697ca3a5ff45",
"store_digest": null,
"facts_digest": null,
"reader": "none",
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": 0,
"max_tokens": 64,
"questions": null,
"answered": null,
"started": null,
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "mh_32k",
"n": 100,
"metrics": {
"substring_em": 0.87
}
},
{
"name": "mh_64k",
"n": 100,
"metrics": {
"substring_em": 0.84
}
},
{
"name": "mh_262k",
"n": 100,
"metrics": {
"substring_em": 0.64
}
},
{
"name": "sh_32k",
"n": 100,
"metrics": {
"substring_em": 0.97
}
},
{
"name": "sh_64k",
"n": 100,
"metrics": {
"substring_em": 0.97
}
},
{
"name": "sh_262k",
"n": 100,
"metrics": {
"substring_em": 0.94
}
}
],
"table": null
},
{
"bench": "mab",
"run": "mab-test-beamx-none",
"row": "beamx",
"split": "test",
"k": 10,
"reader": "none",
"facts": null,
"commit": "2bce2e8f3e1",
"record": {
"commit": "2bce2e8f3e1",
"prompt": "prompts/reader-mab.txt",
"prompt_digest": "7d9cee398c4536d60756c86bd5d073679202e5a63ad594ecd1a56704873d2eee",
"dataset_digest": "24d5c3f09ce0ce15625cb9f8a98f44f0d864ca6c94d7b4ad04eb697ca3a5ff45",
"store_digest": null,
"facts_digest": null,
"reader": "none",
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": 0,
"max_tokens": 64,
"questions": null,
"answered": null,
"started": null,
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "mh_32k",
"n": 100,
"metrics": {
"substring_em": 0.89
}
},
{
"name": "mh_64k",
"n": 100,
"metrics": {
"substring_em": 0.89
}
},
{
"name": "mh_262k",
"n": 100,
"metrics": {
"substring_em": 0.67
}
},
{
"name": "sh_32k",
"n": 100,
"metrics": {
"substring_em": 0.98
}
},
{
"name": "sh_64k",
"n": 100,
"metrics": {
"substring_em": 0.98
}
},
{
"name": "sh_262k",
"n": 100,
"metrics": {
"substring_em": 0.97
}
}
],
"table": null
},
{
"bench": "mab",
"run": "mab-test-noreader-none",
"row": "noreader",
"split": "test",
"k": 10,
"reader": "none",
"facts": null,
"commit": "12468828cf",
"record": {
"commit": "12468828cf",
"prompt": "prompts/reader-mab.txt",
"prompt_digest": "7d9cee398c4536d60756c86bd5d073679202e5a63ad594ecd1a56704873d2eee",
"dataset_digest": "24d5c3f09ce0ce15625cb9f8a98f44f0d864ca6c94d7b4ad04eb697ca3a5ff45",
"store_digest": null,
"facts_digest": null,
"reader": "none",
"embedding": "zenlm/zen-embedding-0.6b",
"temperature": 0,
"max_tokens": 64,
"questions": null,
"answered": null,
"started": null,
"ended": null,
"completion": "unknown",
"wall_seconds": null,
"tokens_per_question": null,
"reader_tokens": null
},
"categories": [
{
"name": "mh_32k",
"n": 100,
"metrics": {
"substring_em": 0.59
}
},
{
"name": "mh_64k",
"n": 100,
"metrics": {
"substring_em": 0.59
}
},
{
"name": "mh_262k",
"n": 100,
"metrics": {
"substring_em": 0.33
}
},
{
"name": "sh_32k",
"n": 100,
"metrics": {
"substring_em": 0.97
}
},
{
"name": "sh_64k",
"n": 100,
"metrics": {
"substring_em": 0.95
}
},
{
"name": "sh_262k",
"n": 100,
"metrics": {
"substring_em": 0.94
}
}
],
"table": null
}
]
}
Reference in New Issue View Git Blame Copy Permalink