mirror of
https://github.com/hanzoai/semantic.git
synced 2026-10-30 15:04:19 +00:00
zeekayandHanzo Dev
c99f2a73c1
The recall benchmark run twice from these packages over the same 5,882 turns, with the same reader and the dataset's own scorer: turns in a store.Mem retrieved by cosine, and a kg graph of who did what retrieved by looking up the person a question names. An oracle arm holding the annotated evidence bounds what the reader can do at all. The graph scores 0.223 against the flat index's 0.156 overall — level on the 1,540 answerable questions, 0.502 against 0.200 on the 446 adversarial ones — and leads at every matched rate of declining to answer. It gets there by being selectively blind: it reaches 58-79% of the evidence for questions that have an answer and 27% for questions that do not, because a turn about somebody else is not a candidate however close its wording. Where it loses is reported too. The metric is ported from task_eval/evaluation.py, NLTK's Porter stemmer included, and held against the Python by three golden tests and once end to end: score.py runs the official code over the same predictions and reproduces every figure to three decimals. LoCoMo is CC BY-NC and this module is MIT, so neither the dataset nor their scorer is kept here. Both are fetched on first use, the dataset through ingest.Web. Co-authored-by: Hanzo Dev <[email protected]>
74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
"""Score a predictions file with the official LoCoMo scorer.
|
|
|
|
The Go harness prints its own numbers; this runs task_eval/evaluation.py over
|
|
the same answers so the two can be diffed. They should agree to the last
|
|
decimal, and the Go tests hold the port against this Python word for word.
|
|
|
|
.venv/bin/python score.py out/answers.json
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent / "ref"))
|
|
from shim import eval_question_answering # noqa: E402
|
|
|
|
NAMES = {4: "single hop", 1: "multi hop", 2: "temporal", 3: "open domain", 5: "adversarial"}
|
|
ORDER = [4, 1, 2, 3, 5]
|
|
|
|
|
|
def main(path):
|
|
samples = json.load(open(path))
|
|
arms = sorted(
|
|
k[: -len("_prediction")]
|
|
for k in samples[0]["qa"][0]
|
|
if k.endswith("_prediction")
|
|
)
|
|
|
|
score = {a: defaultdict(float) for a in arms}
|
|
reach = {a: defaultdict(float) for a in arms}
|
|
count = defaultdict(int)
|
|
for sample in samples:
|
|
for qa in sample["qa"]:
|
|
count[qa["category"]] += 1
|
|
for arm in arms:
|
|
key = arm + "_prediction_context"
|
|
for qa in sample["qa"]:
|
|
# The official recall reads context[0] to tell a session id
|
|
# from a dialogue id, and a memory that returned nothing has
|
|
# no context[0]. A placeholder that matches no evidence scores
|
|
# the same zero without touching the vendored file.
|
|
if not qa[key]:
|
|
qa[key] = ["-"]
|
|
marks, _, recall = eval_question_answering(sample["qa"], arm + "_prediction")
|
|
for qa, mark, got in zip(sample["qa"], marks, recall):
|
|
score[arm][qa["category"]] += mark
|
|
reach[arm][qa["category"]] += got
|
|
|
|
print(f"\n{'':14}{'n':>6}", end="")
|
|
for arm in arms:
|
|
print(f" {arm:>20}", end="")
|
|
print(f"\n{'category':14}{'':6}", end="")
|
|
for _ in arms:
|
|
print(f" {'F1':>6} {'recall':>6} ", end="")
|
|
print()
|
|
|
|
def row(name, kinds):
|
|
n = sum(count[k] for k in kinds)
|
|
print(f"{name:14}{n:6}", end="")
|
|
for arm in arms:
|
|
f1 = sum(score[arm][k] for k in kinds) / n
|
|
rc = sum(reach[arm][k] for k in kinds) / n
|
|
print(f" {f1:6.3f} {rc:6.3f} ", end="")
|
|
print()
|
|
|
|
for kind in ORDER:
|
|
row(NAMES[kind], [kind])
|
|
row("answerable", [1, 2, 3, 4])
|
|
row("overall", ORDER)
|
|
|
|
|
|
main(sys.argv[1] if len(sys.argv) > 1 else "out/answers.json")
|