mirror of
https://github.com/hanzoai/semantic.git
synced 2026-10-30 15:04:19 +00:00
zeekayandHanzo Dev
c99f2a73c1
The recall benchmark run twice from these packages over the same 5,882 turns, with the same reader and the dataset's own scorer: turns in a store.Mem retrieved by cosine, and a kg graph of who did what retrieved by looking up the person a question names. An oracle arm holding the annotated evidence bounds what the reader can do at all. The graph scores 0.223 against the flat index's 0.156 overall — level on the 1,540 answerable questions, 0.502 against 0.200 on the 446 adversarial ones — and leads at every matched rate of declining to answer. It gets there by being selectively blind: it reaches 58-79% of the evidence for questions that have an answer and 27% for questions that do not, because a turn about somebody else is not a candidate however close its wording. Where it loses is reported too. The metric is ported from task_eval/evaluation.py, NLTK's Porter stemmer included, and held against the Python by three golden tests and once end to end: score.py runs the official code over the same predictions and reproduces every figure to three decimals. LoCoMo is CC BY-NC and this module is MIT, so neither the dataset nor their scorer is kept here. Both are fetched on first use, the dataset through ingest.Web. Co-authored-by: Hanzo Dev <[email protected]>
111 lines
2.9 KiB
Go
111 lines
2.9 KiB
Go
package main
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// The dataset, read back. LoCoMo keeps its sessions under numbered keys rather
|
|
// than in an array and dates them in prose, so the loader has real work to do
|
|
// and getting any of it wrong moves every number in the report.
|
|
|
|
func TestLoad(t *testing.T) {
|
|
samples, err := Load("data/locomo10.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(samples) != 10 {
|
|
t.Fatalf("%d conversations, not 10", len(samples))
|
|
}
|
|
|
|
turns, asks := 0, 0
|
|
kind := map[int]int{}
|
|
for _, s := range samples {
|
|
turns += len(s.Turns)
|
|
asks += len(s.Asks)
|
|
for _, a := range s.Asks {
|
|
kind[a.Kind]++
|
|
}
|
|
if s.Who[0] == "" || s.Who[1] == "" || s.Who[0] == s.Who[1] {
|
|
t.Errorf("%s: speakers %q and %q", s.ID, s.Who[0], s.Who[1])
|
|
}
|
|
for i, turn := range s.Turns {
|
|
if turn.ID == "" || turn.Who == "" || turn.Text == "" {
|
|
t.Fatalf("%s: turn %d is missing something: %+v", s.ID, i, turn)
|
|
}
|
|
if turn.Date.IsZero() {
|
|
t.Fatalf("%s: turn %s has no date", s.ID, turn.ID)
|
|
}
|
|
if i > 0 && s.Turns[i-1].Session > turn.Session {
|
|
t.Fatalf("%s: session %d follows %d", s.ID, turn.Session, s.Turns[i-1].Session)
|
|
}
|
|
}
|
|
for _, a := range s.Asks {
|
|
if a.Text == "" || a.Answer == "" {
|
|
t.Errorf("%s: question %q answered %q", s.ID, a.Text, a.Answer)
|
|
}
|
|
if a.Kind < 1 || a.Kind > 5 {
|
|
t.Errorf("%s: category %d", s.ID, a.Kind)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The counts the paper reports, and what this file actually holds.
|
|
if turns != 5882 {
|
|
t.Errorf("%d turns, not 5882", turns)
|
|
}
|
|
if asks != 1986 {
|
|
t.Errorf("%d questions, not 1986", asks)
|
|
}
|
|
for got, want := range map[int]int{1: 282, 2: 321, 3: 96, 4: 841, 5: 446} {
|
|
if kind[got] != want {
|
|
t.Errorf("category %d has %d questions, not %d", got, kind[got], want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDates(t *testing.T) {
|
|
samples, err := Load("data/locomo10.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first := samples[0].Turns[0]
|
|
want := time.Date(2023, time.May, 8, 13, 56, 0, 0, time.UTC)
|
|
if !first.Date.Equal(want) {
|
|
t.Errorf("first session is dated %v, not %v", first.Date, want)
|
|
}
|
|
if got := Day(first.Date); got != "8 May, 2023" {
|
|
t.Errorf("written as %q", got)
|
|
}
|
|
if got := first.Line(); got != `(8 May, 2023) Caroline said, "Hey Mel! Good to see you! How have you been?"` {
|
|
t.Errorf("indexed as %q", got)
|
|
}
|
|
}
|
|
|
|
// Evidence points at turns that exist, or the recall measure is meaningless.
|
|
func TestEvidenceResolves(t *testing.T) {
|
|
samples, err := Load("data/locomo10.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cited, lost := 0, 0
|
|
for _, s := range samples {
|
|
known := map[string]bool{}
|
|
for _, turn := range s.Turns {
|
|
known[turn.ID] = true
|
|
}
|
|
for _, a := range s.Asks {
|
|
for _, e := range a.Evidence {
|
|
cited++
|
|
if !known[trim(e)] {
|
|
lost++
|
|
}
|
|
}
|
|
}
|
|
}
|
|
t.Logf("%d evidence turns cited, %d of them not in the conversation", cited, lost)
|
|
if share := float64(lost) / float64(cited); share > 0.02 {
|
|
t.Errorf("%.1f%% of evidence does not resolve to a turn", 100*share)
|
|
}
|
|
}
|