mirror of
https://github.com/hanzoai/semantic.git
synced 2026-10-30 15:04:19 +00:00
zeekayandHanzo Dev
0c787e9f4d
Two changes that only mean something together. The graph was never read: emptying it left every number identical, so the subject filter was doing all the work and the structure none. The path arm stands on the nodes a question names, gathers the assertions out of them into relations, and cites the turns behind the one that fits — multi-hop 0.020 to 0.032, adversarial 0.502 to 0.576. Run it with the graph emptied and it goes to nothing at all: recall 0.000, quiet on every question, overall 0.228, which is the score for answering nothing. That is the control the old arms could not fail. The reader was the other half. Sixty percent of multi-hop answers are lists and the reader stopped at two parts, so it discarded the shape of the answer before retrieval had a say. Gather states what the memory returned and the oracle moves 0.068 to 0.119, which is what lets a retrieval gain show underneath it at all. Eighty-eight percent of the distance to one is still what an extractive reader cannot write. What was measured and taken back out: coreference touches three percent of sentences and moves multi-hop by nothing; a second hop reaches more evidence and costs adversarial questions dearly, because reaching the trap turn is how those are lost; scoring whole relations has an arity bias that settles on the loose large one over the exact small one. Folding predicates by stem stays, at 3,472 relations to 2,866. Losses: single hop falls 0.121 to 0.111 walking, and to 0.049 when the reader states everything, because category one is marked part by part and every other answerable category is one bag of tokens where an extra part costs precision. Co-authored-by: Hanzo Dev <[email protected]>
95 lines
2.6 KiB
Go
95 lines
2.6 KiB
Go
package main
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// What an extractive reader could get at best.
|
|
//
|
|
// The report's ceiling is the oracle column: the deterministic reader, handed
|
|
// the turns the dataset itself names as the evidence. That number is low, and
|
|
// two different things could make it low. The reader may be quoting the wrong
|
|
// words out of evidence that contains the right ones, in which case a better
|
|
// reader lifts it. Or the reference answer may not be in the evidence at all —
|
|
// LoCoMo's answers are written by an annotator, not copied out of the turns —
|
|
// in which case no reader that quotes can ever reach it and only one that
|
|
// writes its own words can.
|
|
//
|
|
// This separates the two. For every answerable question it takes the stems of
|
|
// the reference answer and asks how many of them appear anywhere in the turns
|
|
// the dataset named. That share is the recall an ideal extractor would have,
|
|
// and it is an upper bound on the recall half of the F1 the metric marks.
|
|
func TestAnswersAreInTheEvidence(t *testing.T) {
|
|
samples, err := Load("data/locomo10.json")
|
|
if err != nil {
|
|
t.Skipf("dataset: %v", err)
|
|
}
|
|
|
|
type count struct {
|
|
n int
|
|
have, whole float64
|
|
}
|
|
by := map[int]*count{}
|
|
for _, s := range samples {
|
|
at := make(map[string]Turn, len(s.Turns))
|
|
for _, turn := range s.Turns {
|
|
at[turn.ID] = turn
|
|
}
|
|
for _, ask := range s.Asks {
|
|
if ask.Kind == 5 {
|
|
continue
|
|
}
|
|
var said []string
|
|
for _, id := range ask.Evidence {
|
|
if turn, known := at[trim(id)]; known {
|
|
said = append(said, turn.Text)
|
|
}
|
|
}
|
|
held := map[string]int{}
|
|
for _, w := range stems(strings.Join(said, " ")) {
|
|
held[w]++
|
|
}
|
|
want := stems(ask.Answer)
|
|
if len(want) == 0 {
|
|
continue
|
|
}
|
|
same := 0
|
|
for _, w := range want {
|
|
if held[w] > 0 {
|
|
held[w]--
|
|
same++
|
|
}
|
|
}
|
|
if by[ask.Kind] == nil {
|
|
by[ask.Kind] = &count{}
|
|
}
|
|
c := by[ask.Kind]
|
|
c.n++
|
|
c.have += float64(same) / float64(len(want))
|
|
if same == len(want) {
|
|
c.whole++
|
|
}
|
|
}
|
|
}
|
|
|
|
t.Log("share of the reference answer's words that appear in the evidence the dataset names")
|
|
all := &count{}
|
|
for _, k := range Kinds {
|
|
c := by[k.Kind]
|
|
if c == nil {
|
|
continue
|
|
}
|
|
t.Logf("%-12s n=%4d words in evidence %.3f answers wholly in evidence %.3f",
|
|
k.Name, c.n, c.have/float64(c.n), c.whole/float64(c.n))
|
|
all.n += c.n
|
|
all.have += c.have
|
|
all.whole += c.whole
|
|
}
|
|
t.Logf("%-12s n=%4d words in evidence %.3f answers wholly in evidence %.3f",
|
|
"answerable", all.n, all.have/float64(all.n), all.whole/float64(all.n))
|
|
if all.n == 0 {
|
|
t.Fatal("no answerable questions were measured")
|
|
}
|
|
}
|