mirror of
https://github.com/hanzo-apps/hanzo.ai.git
synced 2026-10-28 07:32:52 +00:00
198 lines
3.1 KiB
JSON
198 lines
3.1 KiB
JSON
{
|
|
"generated": "2026-09-19T20:26:44.033Z",
|
|
"embed": "all-minilm",
|
|
"k": [
|
|
5,
|
|
10
|
|
],
|
|
"tally": {
|
|
"ALL": {
|
|
"n": 500,
|
|
"mrr": {
|
|
"tm": 451.34175316279504,
|
|
"sm": 433.87047775238176,
|
|
"ctx": 459.31608946608947
|
|
},
|
|
"all": {
|
|
"tm@5": 429,
|
|
"tm@10": 467,
|
|
"sm@5": 413,
|
|
"sm@10": 461,
|
|
"ctx@5": 453,
|
|
"ctx@10": 485
|
|
},
|
|
"any": {
|
|
"tm@5": 487,
|
|
"tm@10": 494,
|
|
"sm@5": 479,
|
|
"sm@10": 487,
|
|
"ctx@5": 491,
|
|
"ctx@10": 499
|
|
}
|
|
},
|
|
"single-session-user": {
|
|
"n": 70,
|
|
"mrr": {
|
|
"tm": 59.63155878201252,
|
|
"sm": 55.083901420217224,
|
|
"ctx": 61.01666666666667
|
|
},
|
|
"all": {
|
|
"tm@5": 67,
|
|
"tm@10": 68,
|
|
"sm@5": 64,
|
|
"sm@10": 66,
|
|
"ctx@5": 69,
|
|
"ctx@10": 70
|
|
},
|
|
"any": {
|
|
"tm@5": 67,
|
|
"tm@10": 68,
|
|
"sm@5": 64,
|
|
"sm@10": 66,
|
|
"ctx@5": 69,
|
|
"ctx@10": 70
|
|
}
|
|
},
|
|
"multi-session": {
|
|
"n": 133,
|
|
"mrr": {
|
|
"tm": 121.33333333333333,
|
|
"sm": 117.53015873015873,
|
|
"ctx": 121.97619047619047
|
|
},
|
|
"all": {
|
|
"tm@5": 101,
|
|
"tm@10": 119,
|
|
"sm@5": 98,
|
|
"sm@10": 115,
|
|
"ctx@5": 114,
|
|
"ctx@10": 126
|
|
},
|
|
"any": {
|
|
"tm@5": 131,
|
|
"tm@10": 133,
|
|
"sm@5": 129,
|
|
"sm@10": 132,
|
|
"ctx@5": 132,
|
|
"ctx@10": 133
|
|
}
|
|
},
|
|
"single-session-preference": {
|
|
"n": 30,
|
|
"mrr": {
|
|
"tm": 24.096969696969698,
|
|
"sm": 23.83888888888889,
|
|
"ctx": 24.962121212121215
|
|
},
|
|
"all": {
|
|
"tm@5": 29,
|
|
"tm@10": 29,
|
|
"sm@5": 28,
|
|
"sm@10": 29,
|
|
"ctx@5": 29,
|
|
"ctx@10": 29
|
|
},
|
|
"any": {
|
|
"tm@5": 29,
|
|
"tm@10": 29,
|
|
"sm@5": 28,
|
|
"sm@10": 29,
|
|
"ctx@5": 29,
|
|
"ctx@10": 29
|
|
}
|
|
},
|
|
"temporal-reasoning": {
|
|
"n": 133,
|
|
"mrr": {
|
|
"tm": 115.0798913504796,
|
|
"sm": 109.56752871311697,
|
|
"ctx": 118.86111111111111
|
|
},
|
|
"all": {
|
|
"tm@5": 101,
|
|
"tm@10": 118,
|
|
"sm@5": 96,
|
|
"sm@10": 118,
|
|
"ctx@5": 108,
|
|
"ctx@10": 126
|
|
},
|
|
"any": {
|
|
"tm@5": 126,
|
|
"tm@10": 130,
|
|
"sm@5": 125,
|
|
"sm@10": 126,
|
|
"ctx@5": 127,
|
|
"ctx@10": 133
|
|
}
|
|
},
|
|
"knowledge-update": {
|
|
"n": 78,
|
|
"mrr": {
|
|
"tm": 75.2,
|
|
"sm": 71.85,
|
|
"ctx": 76.5
|
|
},
|
|
"all": {
|
|
"tm@5": 75,
|
|
"tm@10": 77,
|
|
"sm@5": 71,
|
|
"sm@10": 77,
|
|
"ctx@5": 77,
|
|
"ctx@10": 78
|
|
},
|
|
"any": {
|
|
"tm@5": 78,
|
|
"tm@10": 78,
|
|
"sm@5": 77,
|
|
"sm@10": 78,
|
|
"ctx@5": 78,
|
|
"ctx@10": 78
|
|
}
|
|
},
|
|
"single-session-assistant": {
|
|
"n": 56,
|
|
"mrr": {
|
|
"tm": 56,
|
|
"sm": 56,
|
|
"ctx": 56
|
|
},
|
|
"all": {
|
|
"tm@5": 56,
|
|
"tm@10": 56,
|
|
"sm@5": 56,
|
|
"sm@10": 56,
|
|
"ctx@5": 56,
|
|
"ctx@10": 56
|
|
},
|
|
"any": {
|
|
"tm@5": 56,
|
|
"tm@10": 56,
|
|
"sm@5": 56,
|
|
"sm@10": 56,
|
|
"ctx@5": 56,
|
|
"ctx@10": 56
|
|
}
|
|
}
|
|
},
|
|
"latency_ms": {
|
|
"p50": 0.202292,
|
|
"p95": 0.292708
|
|
},
|
|
"context": {
|
|
"run": "longmemeval-all-context-minilm",
|
|
"row": "+iterative hops",
|
|
"frozen": {
|
|
"commit": "33d0f8749bd1",
|
|
"engine": "b467901f3950",
|
|
"objective": "LoCoMo dev; no tuning on LongMemEval"
|
|
},
|
|
"commit": "34d0e737dbf5",
|
|
"engine": "b467901f3950",
|
|
"latency_ms": {
|
|
"p50": 1.958167,
|
|
"p95": 5.870834
|
|
},
|
|
"wall_seconds": 97
|
|
}
|
|
} |