mirror of
https://github.com/hanzoai/papers.git
synced 2026-10-29 07:34:31 +00:00
zeekayandHanzo Dev
2b594ab2c9
A rerun of the measurement at hanzoai/semantic v0.2.4 found the result sound and some of the prose around it not. Three statements were false. The subject resolver matches any name the conversation uses and not only the two speakers, so 11 questions fall back to searching everything, not 22. Deleting the derived facts moves six more cells than the sentence admitted, and moves them both ways: single-hop F1 up a thousandth, reach down two. And constant abstention at 0.228 is above every arm that has to retrieve, but not above the oracle's 0.252, which the paper's own Table 1 prints two pages later. Two claims outran their evidence. "At every matched silence rate" generalized from four wide-gap points past an 8% crossing where the gap is 0.002, and below that rate the flat index cannot be matched at all; the sentence now says which rates were measured and what happens at the quiet end. And the adversarial questions carry no reference answer — the field the dataset gives them holds the distractor, "yoga" for the worked example — so the paper no longer describes a reference saying the conversation does not say. "Scoped" named both a specific arm and a general property. Arms are now set in typewriter and mean one thing each, subject scope is the factor, and the abstract says which effect +0.152 is. Differences are taken from the unrounded scores and rounded once, which §2.2 states and which moves the claim-span effect to -0.029 and the interaction to +0.179. Section 13 claimed seven commands produced every number, and at least six sets came from nowhere in the repository. The harness now computes them: -stats reads a predictions file back for the constant-abstention score, the AUCs and the bootstrap intervals, and the run prints sessions, build totals and the subject counts. The commands missing from the table are in it, and the four figures that no command produces say where they come from. The measurement, the tables and the conclusion are unchanged. Five interval endpoints move by a thousandth, because the bootstrap is now enumerated over all 92,378 resamples instead of sampling 10,000 of them. Co-authored-by: Hanzo Dev <[email protected]>
173 lines
5.7 KiB
BibTeX
173 lines
5.7 KiB
BibTeX
@inproceedings{locomo,
|
|
author = {Adyasha Maharana and Dong-Ho Lee and Sergey Tulyakov and
|
|
Mohit Bansal and Francesco Barbieri and Yuwei Fang},
|
|
title = {{Evaluating Very Long-Term Conversational Memory of LLM Agents}},
|
|
booktitle = {Proceedings of the 62nd Annual Meeting of the Association for
|
|
Computational Linguistics (ACL)},
|
|
year = {2024},
|
|
note = {Dataset and evaluation code:
|
|
\url{https://github.com/snap-research/locomo}. Dataset licensed
|
|
CC BY-NC 4.0}
|
|
}
|
|
|
|
@misc{semantic,
|
|
author = {{Hanzo Industries}},
|
|
title = {{semantic: knowledge graphs, context graphs and decision
|
|
intelligence for agents, in Go}},
|
|
howpublished = {\url{https://github.com/hanzoai/semantic}},
|
|
year = {2026},
|
|
note = {MIT. Benchmark harness under \texttt{bench/locomo/}, tag v0.2.4}
|
|
}
|
|
|
|
@techreport{multihop,
|
|
author = {{Hanzo AI Research}},
|
|
title = {{Hops Do Not Fix Multi-Hop Retrieval: A Negative Result on
|
|
Long-Horizon Conversational Memory}},
|
|
institution = {Hanzo Industries Inc},
|
|
year = {2026},
|
|
note = {Measurement pass 2026-09-07, LoCoMo with
|
|
\texttt{zenlm/zen-embedding-0.6b}}
|
|
}
|
|
|
|
@inproceedings{squad2,
|
|
author = {Pranav Rajpurkar and Robin Jia and Percy Liang},
|
|
title = {{Know What You Don't Know: Unanswerable Questions for SQuAD}},
|
|
booktitle = {Proceedings of the 56th Annual Meeting of the Association for
|
|
Computational Linguistics (ACL)},
|
|
pages = {784--789},
|
|
year = {2018}
|
|
}
|
|
|
|
@inproceedings{dpr,
|
|
author = {Vladimir Karpukhin and Barlas O{\u{g}}uz and Sewon Min and
|
|
Patrick Lewis and Ledell Wu and Sergey Edunov and Danqi Chen and
|
|
Wen-tau Yih},
|
|
title = {{Dense Passage Retrieval for Open-Domain Question Answering}},
|
|
booktitle = {Proceedings of the 2020 Conference on Empirical Methods in
|
|
Natural Language Processing (EMNLP)},
|
|
pages = {6769--6781},
|
|
year = {2020}
|
|
}
|
|
|
|
@article{bm25,
|
|
author = {Stephen Robertson and Hugo Zaragoza},
|
|
title = {{The Probabilistic Relevance Framework: BM25 and Beyond}},
|
|
journal = {Foundations and Trends in Information Retrieval},
|
|
volume = {3},
|
|
number = {4},
|
|
pages = {333--389},
|
|
year = {2009}
|
|
}
|
|
|
|
@inproceedings{rag,
|
|
author = {Patrick Lewis and Ethan Perez and Aleksandra Piktus and
|
|
Fabio Petroni and Vladimir Karpukhin and Naman Goyal and
|
|
Heinrich K{\"u}ttler and Mike Lewis and Wen-tau Yih and
|
|
Tim Rockt{\"a}schel and Sebastian Riedel and Douwe Kiela},
|
|
title = {{Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks}},
|
|
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
|
|
year = {2020}
|
|
}
|
|
|
|
@misc{graphrag,
|
|
author = {Darren Edge and Ha Trinh and Newman Cheng and Joshua Bradley
|
|
and Alex Chao and Apurva Mody and Steven Truitt and
|
|
Jonathan Larson},
|
|
title = {{From Local to Global: A Graph RAG Approach to
|
|
Query-Focused Summarization}},
|
|
howpublished = {arXiv:2404.16130},
|
|
year = {2024}
|
|
}
|
|
|
|
@inproceedings{hipporag,
|
|
author = {Bernal Jim{\'e}nez Guti{\'e}rrez and Yiheng Shu and Yu Gu and
|
|
Michihiro Yasunaga and Yu Su},
|
|
title = {{HippoRAG: Neurobiologically Inspired Long-Term Memory for
|
|
Large Language Models}},
|
|
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
|
|
year = {2024}
|
|
}
|
|
|
|
@misc{memgpt,
|
|
author = {Charles Packer and Sarah Wooders and Kevin Lin and
|
|
Vivian Fang and Shishir G. Patil and Ion Stoica and
|
|
Joseph E. Gonzalez},
|
|
title = {{MemGPT: Towards LLMs as Operating Systems}},
|
|
howpublished = {arXiv:2310.08560},
|
|
year = {2023}
|
|
}
|
|
|
|
@article{negativecontrol,
|
|
author = {Marc Lipsitch and Eric Tchetgen Tchetgen and Ted Cohen},
|
|
title = {{Negative Controls: A Tool for Detecting Confounding and Bias in
|
|
Observational Studies}},
|
|
journal = {Epidemiology},
|
|
volume = {21},
|
|
number = {3},
|
|
pages = {383--388},
|
|
year = {2010}
|
|
}
|
|
|
|
@article{neuralhype,
|
|
author = {Jimmy Lin},
|
|
title = {{The Neural Hype and Comparisons Against Weak Baselines}},
|
|
journal = {SIGIR Forum},
|
|
volume = {52},
|
|
number = {2},
|
|
pages = {40--51},
|
|
year = {2019}
|
|
}
|
|
|
|
@inproceedings{realitycheck,
|
|
author = {Kevin Musgrave and Serge Belongie and Ser-Nam Lim},
|
|
title = {{A Metric Learning Reality Check}},
|
|
booktitle = {European Conference on Computer Vision (ECCV)},
|
|
year = {2020}
|
|
}
|
|
|
|
@article{hanleymcneil,
|
|
author = {James A. Hanley and Barbara J. McNeil},
|
|
title = {{The Meaning and Use of the Area under a Receiver Operating
|
|
Characteristic (ROC) Curve}},
|
|
journal = {Radiology},
|
|
volume = {143},
|
|
number = {1},
|
|
pages = {29--36},
|
|
year = {1982}
|
|
}
|
|
|
|
@article{mannwhitney,
|
|
author = {Henry B. Mann and Donald R. Whitney},
|
|
title = {{On a Test of Whether One of Two Random Variables is
|
|
Stochastically Larger than the Other}},
|
|
journal = {Annals of Mathematical Statistics},
|
|
volume = {18},
|
|
number = {1},
|
|
pages = {50--60},
|
|
year = {1947}
|
|
}
|
|
|
|
@book{bootstrap,
|
|
author = {Bradley Efron and Robert J. Tibshirani},
|
|
title = {{An Introduction to the Bootstrap}},
|
|
publisher = {Chapman \& Hall},
|
|
year = {1993}
|
|
}
|
|
|
|
@article{porter,
|
|
author = {Martin F. Porter},
|
|
title = {{An Algorithm for Suffix Stripping}},
|
|
journal = {Program},
|
|
volume = {14},
|
|
number = {3},
|
|
pages = {130--137},
|
|
year = {1980}
|
|
}
|
|
|
|
@book{nltk,
|
|
author = {Steven Bird and Ewan Klein and Edward Loper},
|
|
title = {{Natural Language Processing with Python}},
|
|
publisher = {O'Reilly Media},
|
|
year = {2009}
|
|
}
|