mirror of
https://github.com/hanzo-apps/hanzo.ai.git
synced 2026-10-28 07:32:52 +00:00
zeekayandHanzo Dev
db2525dfd3
/benchmarks was one URL covering ten benchmarks, which is one page ranking for none of them. Somebody searching MemoryAgentBench wants the page about MemoryAgentBench. Each now has its own route, canonical URL, Dataset and BreadcrumbList, and a sitemap entry whose lastModified is the `generated` stamp its run file carries — the date the runs were rolled up, not the date somebody edited the prose beside them. Where a bench has no stamped run it is offered no date, because no date is the true answer and a build timestamp would be a plausible-looking lie that changes on every deploy. The registry has carried `sandbox` since the directory was built and the route never existed, so the index and the footer of every other bench page linked to a 404. It exists, and it leads with the row it would rather not: a V8 context starts in 0.15 ms and cannot run pytest, pip, cargo or a shell. The boards vary one thing each — the reader with retrieval held, retrieval with the reader held, the embedder, the unit of memory — and the heading no longer counts them, because it counted four and rendered five. The Go harness is no longer transcribed. `node scripts/locomo.mjs` runs it and parses what it prints into lib/data/benchmarks-semantic.json, with the tag, the commit, the dataset digest, and the exact command beside every table. That caught a published instruction that does not run: the 2x2 was captioned `-factor`, which is not a flag. It also carries the v0.3.0 walk arm — following the graph's relations scores 0.032 on multi-hop against 0.020 and 0.023, and a reader that states every piece of evidence it was handed moves the oracle ceiling 0.068 -> 0.119, so the ceiling moved further than the retrieval did. The control is the part worth having. `-arms walk -blind` empties every graph and changes nothing else; the capture compares the two cell by cell and records which arms survived. Five of seven come back byte-identical and two collapse to constant abstention. The site used to assert graph inertness in prose. It now reports the outcome of a comparison, and the sentence cannot outlive the result. The Metal prefill row was quoted from a warm, unalternated pass at 0.91x. A cooled, order-alternated pass on the same accelerator reads 0.8645 across three rounds, so 13.6% behind is the figure, not 9%. It is a different quantization and sits in its own table rather than replacing a cell. Co-authored-by: Hanzo Dev <[email protected]>
1139 lines
23 KiB
JSON
1139 lines
23 KiB
JSON
{
|
|
"harness": "hanzoai/semantic · bench/locomo",
|
|
"tag": "v0.3.0",
|
|
"commit": "b34efcf6bf8769096ce57fff945f556685d8f153",
|
|
"dataset": "snap-research/locomo",
|
|
"dataset_sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4",
|
|
"settings": {
|
|
"k": 5,
|
|
"floor": 0.3,
|
|
"span": 4,
|
|
"band": 0.9,
|
|
"parts": 2
|
|
},
|
|
"generated": "2026-09-11T05:54:16Z",
|
|
"main": {
|
|
"command": "go run ./bench/locomo",
|
|
"arms": [
|
|
"graph",
|
|
"graph+edge",
|
|
"oracle",
|
|
"vector"
|
|
],
|
|
"rows": [
|
|
{
|
|
"category": "single-hop",
|
|
"n": 841,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.121,
|
|
"recall": 0.507,
|
|
"quiet": 0.24
|
|
},
|
|
"graph+edge": {
|
|
"f1": 0.056,
|
|
"recall": 0.507,
|
|
"quiet": 0.21
|
|
},
|
|
"oracle": {
|
|
"f1": 0.199,
|
|
"recall": 1,
|
|
"quiet": 0.42
|
|
},
|
|
"vector": {
|
|
"f1": 0.118,
|
|
"recall": 0.546,
|
|
"quiet": 0.2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "multi-hop",
|
|
"n": 282,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.02,
|
|
"recall": 0.228,
|
|
"quiet": 0.18
|
|
},
|
|
"graph+edge": {
|
|
"f1": 0.025,
|
|
"recall": 0.228,
|
|
"quiet": 0.17
|
|
},
|
|
"oracle": {
|
|
"f1": 0.068,
|
|
"recall": 0.973,
|
|
"quiet": 0.44
|
|
},
|
|
"vector": {
|
|
"f1": 0.023,
|
|
"recall": 0.194,
|
|
"quiet": 0.17
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "temporal",
|
|
"n": 321,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.338,
|
|
"recall": 0.532,
|
|
"quiet": 0.23
|
|
},
|
|
"graph+edge": {
|
|
"f1": 0.336,
|
|
"recall": 0.532,
|
|
"quiet": 0.22
|
|
},
|
|
"oracle": {
|
|
"f1": 0.379,
|
|
"recall": 0.997,
|
|
"quiet": 0.33
|
|
},
|
|
"vector": {
|
|
"f1": 0.357,
|
|
"recall": 0.581,
|
|
"quiet": 0.16
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "open-domain",
|
|
"n": 96,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.018,
|
|
"recall": 0.238,
|
|
"quiet": 0.56
|
|
},
|
|
"graph+edge": {
|
|
"f1": 0.018,
|
|
"recall": 0.238,
|
|
"quiet": 0.55
|
|
},
|
|
"oracle": {
|
|
"f1": 0.032,
|
|
"recall": 0.951,
|
|
"quiet": 0.79
|
|
},
|
|
"vector": {
|
|
"f1": 0.016,
|
|
"recall": 0.258,
|
|
"quiet": 0.45
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "adversarial",
|
|
"n": 446,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.502,
|
|
"recall": 0.135,
|
|
"quiet": 0.5
|
|
},
|
|
"graph+edge": {
|
|
"f1": 0.484,
|
|
"recall": 0.135,
|
|
"quiet": 0.48
|
|
},
|
|
"oracle": {
|
|
"f1": 0.426,
|
|
"recall": 1,
|
|
"quiet": 0.43
|
|
},
|
|
"vector": {
|
|
"f1": 0.2,
|
|
"recall": 0.511,
|
|
"quiet": 0.2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "answerable",
|
|
"n": 1540,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.142,
|
|
"recall": 0.444,
|
|
"quiet": 0.25
|
|
},
|
|
"graph+edge": {
|
|
"f1": 0.107,
|
|
"recall": 0.444,
|
|
"quiet": 0.23
|
|
},
|
|
"oracle": {
|
|
"f1": 0.202,
|
|
"recall": 0.991,
|
|
"quiet": 0.43
|
|
},
|
|
"vector": {
|
|
"f1": 0.144,
|
|
"recall": 0.471,
|
|
"quiet": 0.2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "overall",
|
|
"n": 1986,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.223,
|
|
"recall": 0.375,
|
|
"quiet": 0.31
|
|
},
|
|
"graph+edge": {
|
|
"f1": 0.191,
|
|
"recall": 0.375,
|
|
"quiet": 0.29
|
|
},
|
|
"oracle": {
|
|
"f1": 0.252,
|
|
"recall": 0.993,
|
|
"quiet": 0.43
|
|
},
|
|
"vector": {
|
|
"f1": 0.156,
|
|
"recall": 0.48,
|
|
"quiet": 0.2
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"factor": {
|
|
"command": "go run ./bench/locomo -arms factor",
|
|
"arms": [
|
|
"graph",
|
|
"scoped",
|
|
"span",
|
|
"vector"
|
|
],
|
|
"rows": [
|
|
{
|
|
"category": "single-hop",
|
|
"n": 841,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.121,
|
|
"recall": 0.507,
|
|
"quiet": 0.24
|
|
},
|
|
"scoped": {
|
|
"f1": 0.129,
|
|
"recall": 0.533,
|
|
"quiet": 0.21
|
|
},
|
|
"span": {
|
|
"f1": 0.129,
|
|
"recall": 0.541,
|
|
"quiet": 0.16
|
|
},
|
|
"vector": {
|
|
"f1": 0.118,
|
|
"recall": 0.546,
|
|
"quiet": 0.2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "multi-hop",
|
|
"n": 282,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.02,
|
|
"recall": 0.228,
|
|
"quiet": 0.18
|
|
},
|
|
"scoped": {
|
|
"f1": 0.021,
|
|
"recall": 0.219,
|
|
"quiet": 0.18
|
|
},
|
|
"span": {
|
|
"f1": 0.023,
|
|
"recall": 0.209,
|
|
"quiet": 0.15
|
|
},
|
|
"vector": {
|
|
"f1": 0.023,
|
|
"recall": 0.194,
|
|
"quiet": 0.17
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "temporal",
|
|
"n": 321,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.338,
|
|
"recall": 0.532,
|
|
"quiet": 0.23
|
|
},
|
|
"scoped": {
|
|
"f1": 0.328,
|
|
"recall": 0.559,
|
|
"quiet": 0.23
|
|
},
|
|
"span": {
|
|
"f1": 0.362,
|
|
"recall": 0.553,
|
|
"quiet": 0.16
|
|
},
|
|
"vector": {
|
|
"f1": 0.357,
|
|
"recall": 0.581,
|
|
"quiet": 0.16
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "open-domain",
|
|
"n": 96,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.018,
|
|
"recall": 0.238,
|
|
"quiet": 0.56
|
|
},
|
|
"scoped": {
|
|
"f1": 0.016,
|
|
"recall": 0.239,
|
|
"quiet": 0.47
|
|
},
|
|
"span": {
|
|
"f1": 0.016,
|
|
"recall": 0.225,
|
|
"quiet": 0.43
|
|
},
|
|
"vector": {
|
|
"f1": 0.016,
|
|
"recall": 0.258,
|
|
"quiet": 0.45
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "adversarial",
|
|
"n": 446,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.502,
|
|
"recall": 0.135,
|
|
"quiet": 0.5
|
|
},
|
|
"scoped": {
|
|
"f1": 0.352,
|
|
"recall": 0.207,
|
|
"quiet": 0.35
|
|
},
|
|
"span": {
|
|
"f1": 0.17,
|
|
"recall": 0.571,
|
|
"quiet": 0.17
|
|
},
|
|
"vector": {
|
|
"f1": 0.2,
|
|
"recall": 0.511,
|
|
"quiet": 0.2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "answerable",
|
|
"n": 1540,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.142,
|
|
"recall": 0.444,
|
|
"quiet": 0.25
|
|
},
|
|
"scoped": {
|
|
"f1": 0.144,
|
|
"recall": 0.463,
|
|
"quiet": 0.23
|
|
},
|
|
"span": {
|
|
"f1": 0.151,
|
|
"recall": 0.463,
|
|
"quiet": 0.17
|
|
},
|
|
"vector": {
|
|
"f1": 0.144,
|
|
"recall": 0.471,
|
|
"quiet": 0.2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "overall",
|
|
"n": 1986,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.223,
|
|
"recall": 0.375,
|
|
"quiet": 0.31
|
|
},
|
|
"scoped": {
|
|
"f1": 0.19,
|
|
"recall": 0.405,
|
|
"quiet": 0.25
|
|
},
|
|
"span": {
|
|
"f1": 0.156,
|
|
"recall": 0.487,
|
|
"quiet": 0.17
|
|
},
|
|
"vector": {
|
|
"f1": 0.156,
|
|
"recall": 0.48,
|
|
"quiet": 0.2
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"silent": {
|
|
"command": "go run ./bench/locomo -stats out/answers.json",
|
|
"f1": {
|
|
"single-hop": 0.001,
|
|
"multi-hop": 0.007,
|
|
"temporal": 0.002,
|
|
"open-domain": 0.035,
|
|
"adversarial": 1,
|
|
"answerable": 0.004,
|
|
"overall": 0.228
|
|
}
|
|
},
|
|
"auc": {
|
|
"graph": {
|
|
"sure": 0.698,
|
|
"decided": 0.627,
|
|
"quiet_answerable": 0.249,
|
|
"quiet_adversarial": 0.502
|
|
},
|
|
"graph+edge": {
|
|
"sure": 0.698,
|
|
"decided": 0.628,
|
|
"quiet_answerable": 0.228,
|
|
"quiet_adversarial": 0.484
|
|
},
|
|
"oracle": {
|
|
"sure": 0.503,
|
|
"decided": 0.499,
|
|
"quiet_answerable": 0.429,
|
|
"quiet_adversarial": 0.426
|
|
},
|
|
"vector": {
|
|
"sure": 0.514,
|
|
"decided": 0.5,
|
|
"quiet_answerable": 0.2,
|
|
"quiet_adversarial": 0.2
|
|
}
|
|
},
|
|
"paired": [
|
|
{
|
|
"a": "graph",
|
|
"b": "graph+edge",
|
|
"adversarial": {
|
|
"mean": 0.018,
|
|
"lo": 0.004,
|
|
"hi": 0.036
|
|
},
|
|
"answerable": {
|
|
"mean": 0.035,
|
|
"lo": 0.026,
|
|
"hi": 0.044
|
|
}
|
|
},
|
|
{
|
|
"a": "graph",
|
|
"b": "oracle",
|
|
"adversarial": {
|
|
"mean": 0.076,
|
|
"lo": 0.042,
|
|
"hi": 0.107
|
|
},
|
|
"answerable": {
|
|
"mean": -0.06,
|
|
"lo": -0.068,
|
|
"hi": -0.053
|
|
}
|
|
},
|
|
{
|
|
"a": "graph",
|
|
"b": "vector",
|
|
"adversarial": {
|
|
"mean": 0.303,
|
|
"lo": 0.247,
|
|
"hi": 0.347
|
|
},
|
|
"answerable": {
|
|
"mean": -0.002,
|
|
"lo": -0.01,
|
|
"hi": 0.006
|
|
}
|
|
},
|
|
{
|
|
"a": "graph+edge",
|
|
"b": "oracle",
|
|
"adversarial": {
|
|
"mean": 0.058,
|
|
"lo": 0.025,
|
|
"hi": 0.087
|
|
},
|
|
"answerable": {
|
|
"mean": -0.095,
|
|
"lo": -0.109,
|
|
"hi": -0.082
|
|
}
|
|
},
|
|
{
|
|
"a": "graph+edge",
|
|
"b": "vector",
|
|
"adversarial": {
|
|
"mean": 0.285,
|
|
"lo": 0.223,
|
|
"hi": 0.332
|
|
},
|
|
"answerable": {
|
|
"mean": -0.037,
|
|
"lo": -0.05,
|
|
"hi": -0.025
|
|
}
|
|
},
|
|
{
|
|
"a": "oracle",
|
|
"b": "vector",
|
|
"adversarial": {
|
|
"mean": 0.226,
|
|
"lo": 0.16,
|
|
"hi": 0.275
|
|
},
|
|
"answerable": {
|
|
"mean": 0.058,
|
|
"lo": 0.048,
|
|
"hi": 0.068
|
|
}
|
|
}
|
|
],
|
|
"walk": {
|
|
"command": "go run ./bench/locomo -arms walk",
|
|
"arms": [
|
|
"graph",
|
|
"oracle",
|
|
"oracle+set",
|
|
"path",
|
|
"path+set",
|
|
"vector",
|
|
"vector+set"
|
|
],
|
|
"rows": [
|
|
{
|
|
"category": "single-hop",
|
|
"n": 841,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.121,
|
|
"recall": 0.507,
|
|
"quiet": 0.24
|
|
},
|
|
"oracle": {
|
|
"f1": 0.199,
|
|
"recall": 1,
|
|
"quiet": 0.42
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.203,
|
|
"recall": 1,
|
|
"quiet": 0.42
|
|
},
|
|
"path": {
|
|
"f1": 0.111,
|
|
"recall": 0.467,
|
|
"quiet": 0.27
|
|
},
|
|
"path+set": {
|
|
"f1": 0.049,
|
|
"recall": 0.467,
|
|
"quiet": 0.24
|
|
},
|
|
"vector": {
|
|
"f1": 0.118,
|
|
"recall": 0.546,
|
|
"quiet": 0.2
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.089,
|
|
"recall": 0.546,
|
|
"quiet": 0.12
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "multi-hop",
|
|
"n": 282,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.02,
|
|
"recall": 0.228,
|
|
"quiet": 0.18
|
|
},
|
|
"oracle": {
|
|
"f1": 0.068,
|
|
"recall": 0.973,
|
|
"quiet": 0.44
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.119,
|
|
"recall": 0.973,
|
|
"quiet": 0.44
|
|
},
|
|
"path": {
|
|
"f1": 0.032,
|
|
"recall": 0.222,
|
|
"quiet": 0.26
|
|
},
|
|
"path+set": {
|
|
"f1": 0.071,
|
|
"recall": 0.222,
|
|
"quiet": 0.25
|
|
},
|
|
"vector": {
|
|
"f1": 0.023,
|
|
"recall": 0.194,
|
|
"quiet": 0.17
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.066,
|
|
"recall": 0.194,
|
|
"quiet": 0.13
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "temporal",
|
|
"n": 321,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.338,
|
|
"recall": 0.532,
|
|
"quiet": 0.23
|
|
},
|
|
"oracle": {
|
|
"f1": 0.379,
|
|
"recall": 0.997,
|
|
"quiet": 0.33
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.379,
|
|
"recall": 0.997,
|
|
"quiet": 0.33
|
|
},
|
|
"path": {
|
|
"f1": 0.353,
|
|
"recall": 0.512,
|
|
"quiet": 0.21
|
|
},
|
|
"path+set": {
|
|
"f1": 0.354,
|
|
"recall": 0.512,
|
|
"quiet": 0.21
|
|
},
|
|
"vector": {
|
|
"f1": 0.357,
|
|
"recall": 0.581,
|
|
"quiet": 0.16
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.356,
|
|
"recall": 0.581,
|
|
"quiet": 0.15
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "open-domain",
|
|
"n": 96,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.018,
|
|
"recall": 0.238,
|
|
"quiet": 0.56
|
|
},
|
|
"oracle": {
|
|
"f1": 0.032,
|
|
"recall": 0.951,
|
|
"quiet": 0.79
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.036,
|
|
"recall": 0.951,
|
|
"quiet": 0.78
|
|
},
|
|
"path": {
|
|
"f1": 0.016,
|
|
"recall": 0.213,
|
|
"quiet": 0.61
|
|
},
|
|
"path+set": {
|
|
"f1": 0.024,
|
|
"recall": 0.213,
|
|
"quiet": 0.6
|
|
},
|
|
"vector": {
|
|
"f1": 0.016,
|
|
"recall": 0.258,
|
|
"quiet": 0.45
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.017,
|
|
"recall": 0.258,
|
|
"quiet": 0.36
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "adversarial",
|
|
"n": 446,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.502,
|
|
"recall": 0.135,
|
|
"quiet": 0.5
|
|
},
|
|
"oracle": {
|
|
"f1": 0.426,
|
|
"recall": 1,
|
|
"quiet": 0.43
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.426,
|
|
"recall": 1,
|
|
"quiet": 0.43
|
|
},
|
|
"path": {
|
|
"f1": 0.576,
|
|
"recall": 0.13,
|
|
"quiet": 0.58
|
|
},
|
|
"path+set": {
|
|
"f1": 0.563,
|
|
"recall": 0.13,
|
|
"quiet": 0.56
|
|
},
|
|
"vector": {
|
|
"f1": 0.2,
|
|
"recall": 0.511,
|
|
"quiet": 0.2
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.128,
|
|
"recall": 0.511,
|
|
"quiet": 0.13
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "answerable",
|
|
"n": 1540,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.142,
|
|
"recall": 0.444,
|
|
"quiet": 0.25
|
|
},
|
|
"oracle": {
|
|
"f1": 0.202,
|
|
"recall": 0.991,
|
|
"quiet": 0.43
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.214,
|
|
"recall": 0.991,
|
|
"quiet": 0.43
|
|
},
|
|
"path": {
|
|
"f1": 0.141,
|
|
"recall": 0.415,
|
|
"quiet": 0.28
|
|
},
|
|
"path+set": {
|
|
"f1": 0.115,
|
|
"recall": 0.415,
|
|
"quiet": 0.26
|
|
},
|
|
"vector": {
|
|
"f1": 0.144,
|
|
"recall": 0.471,
|
|
"quiet": 0.2
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.136,
|
|
"recall": 0.471,
|
|
"quiet": 0.15
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "overall",
|
|
"n": 1986,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.223,
|
|
"recall": 0.375,
|
|
"quiet": 0.31
|
|
},
|
|
"oracle": {
|
|
"f1": 0.252,
|
|
"recall": 0.993,
|
|
"quiet": 0.43
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.262,
|
|
"recall": 0.993,
|
|
"quiet": 0.43
|
|
},
|
|
"path": {
|
|
"f1": 0.239,
|
|
"recall": 0.351,
|
|
"quiet": 0.34
|
|
},
|
|
"path+set": {
|
|
"f1": 0.216,
|
|
"recall": 0.351,
|
|
"quiet": 0.33
|
|
},
|
|
"vector": {
|
|
"f1": 0.156,
|
|
"recall": 0.48,
|
|
"quiet": 0.2
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.134,
|
|
"recall": 0.48,
|
|
"quiet": 0.14
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"blind": {
|
|
"command": "go run ./bench/locomo -arms walk -blind",
|
|
"arms": [
|
|
"graph",
|
|
"oracle",
|
|
"oracle+set",
|
|
"path",
|
|
"path+set",
|
|
"vector",
|
|
"vector+set"
|
|
],
|
|
"rows": [
|
|
{
|
|
"category": "single-hop",
|
|
"n": 841,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.121,
|
|
"recall": 0.507,
|
|
"quiet": 0.24
|
|
},
|
|
"oracle": {
|
|
"f1": 0.199,
|
|
"recall": 1,
|
|
"quiet": 0.42
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.203,
|
|
"recall": 1,
|
|
"quiet": 0.42
|
|
},
|
|
"path": {
|
|
"f1": 0.001,
|
|
"recall": 0,
|
|
"quiet": 1
|
|
},
|
|
"path+set": {
|
|
"f1": 0.001,
|
|
"recall": 0,
|
|
"quiet": 1
|
|
},
|
|
"vector": {
|
|
"f1": 0.118,
|
|
"recall": 0.546,
|
|
"quiet": 0.2
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.089,
|
|
"recall": 0.546,
|
|
"quiet": 0.12
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "multi-hop",
|
|
"n": 282,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.02,
|
|
"recall": 0.228,
|
|
"quiet": 0.18
|
|
},
|
|
"oracle": {
|
|
"f1": 0.068,
|
|
"recall": 0.973,
|
|
"quiet": 0.44
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.119,
|
|
"recall": 0.973,
|
|
"quiet": 0.44
|
|
},
|
|
"path": {
|
|
"f1": 0.007,
|
|
"recall": 0,
|
|
"quiet": 1
|
|
},
|
|
"path+set": {
|
|
"f1": 0.007,
|
|
"recall": 0,
|
|
"quiet": 1
|
|
},
|
|
"vector": {
|
|
"f1": 0.023,
|
|
"recall": 0.194,
|
|
"quiet": 0.17
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.066,
|
|
"recall": 0.194,
|
|
"quiet": 0.13
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "temporal",
|
|
"n": 321,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.338,
|
|
"recall": 0.532,
|
|
"quiet": 0.23
|
|
},
|
|
"oracle": {
|
|
"f1": 0.379,
|
|
"recall": 0.997,
|
|
"quiet": 0.33
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.379,
|
|
"recall": 0.997,
|
|
"quiet": 0.33
|
|
},
|
|
"path": {
|
|
"f1": 0.002,
|
|
"recall": 0,
|
|
"quiet": 1
|
|
},
|
|
"path+set": {
|
|
"f1": 0.002,
|
|
"recall": 0,
|
|
"quiet": 1
|
|
},
|
|
"vector": {
|
|
"f1": 0.357,
|
|
"recall": 0.581,
|
|
"quiet": 0.16
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.356,
|
|
"recall": 0.581,
|
|
"quiet": 0.15
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "open-domain",
|
|
"n": 96,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.018,
|
|
"recall": 0.238,
|
|
"quiet": 0.56
|
|
},
|
|
"oracle": {
|
|
"f1": 0.032,
|
|
"recall": 0.951,
|
|
"quiet": 0.79
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.036,
|
|
"recall": 0.951,
|
|
"quiet": 0.78
|
|
},
|
|
"path": {
|
|
"f1": 0.035,
|
|
"recall": 0.042,
|
|
"quiet": 1
|
|
},
|
|
"path+set": {
|
|
"f1": 0.035,
|
|
"recall": 0.042,
|
|
"quiet": 1
|
|
},
|
|
"vector": {
|
|
"f1": 0.016,
|
|
"recall": 0.258,
|
|
"quiet": 0.45
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.017,
|
|
"recall": 0.258,
|
|
"quiet": 0.36
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "adversarial",
|
|
"n": 446,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.502,
|
|
"recall": 0.135,
|
|
"quiet": 0.5
|
|
},
|
|
"oracle": {
|
|
"f1": 0.426,
|
|
"recall": 1,
|
|
"quiet": 0.43
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.426,
|
|
"recall": 1,
|
|
"quiet": 0.43
|
|
},
|
|
"path": {
|
|
"f1": 1,
|
|
"recall": 0,
|
|
"quiet": 1
|
|
},
|
|
"path+set": {
|
|
"f1": 1,
|
|
"recall": 0,
|
|
"quiet": 1
|
|
},
|
|
"vector": {
|
|
"f1": 0.2,
|
|
"recall": 0.511,
|
|
"quiet": 0.2
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.128,
|
|
"recall": 0.511,
|
|
"quiet": 0.13
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "answerable",
|
|
"n": 1540,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.142,
|
|
"recall": 0.444,
|
|
"quiet": 0.25
|
|
},
|
|
"oracle": {
|
|
"f1": 0.202,
|
|
"recall": 0.991,
|
|
"quiet": 0.43
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.214,
|
|
"recall": 0.991,
|
|
"quiet": 0.43
|
|
},
|
|
"path": {
|
|
"f1": 0.004,
|
|
"recall": 0.003,
|
|
"quiet": 1
|
|
},
|
|
"path+set": {
|
|
"f1": 0.004,
|
|
"recall": 0.003,
|
|
"quiet": 1
|
|
},
|
|
"vector": {
|
|
"f1": 0.144,
|
|
"recall": 0.471,
|
|
"quiet": 0.2
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.136,
|
|
"recall": 0.471,
|
|
"quiet": 0.15
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"category": "overall",
|
|
"n": 1986,
|
|
"arms": {
|
|
"graph": {
|
|
"f1": 0.223,
|
|
"recall": 0.375,
|
|
"quiet": 0.31
|
|
},
|
|
"oracle": {
|
|
"f1": 0.252,
|
|
"recall": 0.993,
|
|
"quiet": 0.43
|
|
},
|
|
"oracle+set": {
|
|
"f1": 0.262,
|
|
"recall": 0.993,
|
|
"quiet": 0.43
|
|
},
|
|
"path": {
|
|
"f1": 0.228,
|
|
"recall": 0.002,
|
|
"quiet": 1
|
|
},
|
|
"path+set": {
|
|
"f1": 0.228,
|
|
"recall": 0.002,
|
|
"quiet": 1
|
|
},
|
|
"vector": {
|
|
"f1": 0.156,
|
|
"recall": 0.48,
|
|
"quiet": 0.2
|
|
},
|
|
"vector+set": {
|
|
"f1": 0.134,
|
|
"recall": 0.48,
|
|
"quiet": 0.14
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"identical": [
|
|
"graph",
|
|
"oracle",
|
|
"oracle+set",
|
|
"vector",
|
|
"vector+set"
|
|
],
|
|
"changed": [
|
|
"path",
|
|
"path+set"
|
|
]
|
|
}
|
|
}
|