mirror of
https://github.com/hanzoai/semantic.git
synced 2026-10-30 15:04:19 +00:00
zeekayandHanzo Dev
c99f2a73c1
The recall benchmark run twice from these packages over the same 5,882 turns, with the same reader and the dataset's own scorer: turns in a store.Mem retrieved by cosine, and a kg graph of who did what retrieved by looking up the person a question names. An oracle arm holding the annotated evidence bounds what the reader can do at all. The graph scores 0.223 against the flat index's 0.156 overall — level on the 1,540 answerable questions, 0.502 against 0.200 on the 446 adversarial ones — and leads at every matched rate of declining to answer. It gets there by being selectively blind: it reaches 58-79% of the evidence for questions that have an answer and 27% for questions that do not, because a turn about somebody else is not a candidate however close its wording. Where it loses is reported too. The metric is ported from task_eval/evaluation.py, NLTK's Porter stemmer included, and held against the Python by three golden tests and once end to end: score.py runs the official code over the same predictions and reproduces every figure to three decimals. LoCoMo is CC BY-NC and this module is MIT, so neither the dataset nor their scorer is kept here. Both are fetched on first use, the dataset through ingest.Web. Co-authored-by: Hanzo Dev <[email protected]>
71 lines
2.0 KiB
Go
71 lines
2.0 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"sort"
|
|
)
|
|
|
|
// A third memory that cheats, and is in the run for exactly that reason. It
|
|
// returns the turns the dataset itself names as the evidence for a question:
|
|
// perfect retrieval, nothing else changed. What it scores is the ceiling the
|
|
// reader puts on the whole benchmark, so the gap between an arm and the oracle
|
|
// is what better remembering could still be worth, and the gap between the
|
|
// oracle and 1.0 is what the reader costs everybody equally.
|
|
//
|
|
// Without it a low number is unreadable — it could mean the memory failed to
|
|
// find the turn or that the reader failed to quote it, and those call for
|
|
// different work.
|
|
|
|
// Oracle answers from the dataset's own evidence.
|
|
type Oracle struct {
|
|
turns []Turn
|
|
at map[string]int
|
|
told map[string][]string // question to the turns that answer it
|
|
words *Words
|
|
}
|
|
|
|
// NewOracle indexes the answers the dataset gives away.
|
|
func NewOracle(s Sample, words *Words) *Oracle {
|
|
o := &Oracle{
|
|
turns: s.Turns,
|
|
at: make(map[string]int, len(s.Turns)),
|
|
told: make(map[string][]string, len(s.Asks)),
|
|
words: words,
|
|
}
|
|
for i, t := range s.Turns {
|
|
o.at[t.ID] = i
|
|
}
|
|
for _, ask := range s.Asks {
|
|
o.told[ask.Text] = ask.Evidence
|
|
}
|
|
return o
|
|
}
|
|
|
|
// Name is what this arm is called in the results.
|
|
func (o *Oracle) Name() string { return "oracle" }
|
|
|
|
// Recall returns the evidence turns, ranked among themselves by how much of
|
|
// the question they cover so that the reader still has to choose.
|
|
func (o *Oracle) Recall(ctx context.Context, question string, k int) ([]Cite, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
focus := o.words.Focus(question, "")
|
|
var out []Cite
|
|
for _, id := range o.told[question] {
|
|
i, known := o.at[trim(id)]
|
|
if !known {
|
|
continue
|
|
}
|
|
out = append(out, Cite{
|
|
Turn: o.turns[i],
|
|
Score: o.words.Cover(focus, Terms(o.turns[i].Text)),
|
|
})
|
|
}
|
|
sort.SliceStable(out, func(i, j int) bool { return out[i].Score > out[j].Score })
|
|
if len(out) > k {
|
|
out = out[:k]
|
|
}
|
|
return out, nil
|
|
}
|