hanzoai/papers
2
0
Fork 0
mirror of https://github.com/hanzoai/papers.git synced 2026-10-22 17:20:17 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
Files
main
papers/hanzo-temporal-assertion-graph/results-data.tex
T
zeekayandHanzo Dev 07694f8e88 A paper on the temporal assertion graph as it stands on main 
Describes /v1/graph at hanzo-inc/cloud 1fffdf408: one append-only table of
assertions, a server-derived knowable instant, reads at a valid and a
transaction instant, versions that correct each other only when learned
later, declared cardinality and schema as assertions, diff, path, erasure
with a receipt, Louvain communities and a citation-checked answer.

CronQuestions from hanzoai/benchmarks: 99.88% Hits@1 for the control,
semantica 0.7.0 and the store, question for question; 0.063 ms against
5,357 ms per call. The first run's 70.03% is reported with its two causes
and the two defects the harness found in the fix. LongMemEval-S is
included as a measurement of the retrieval engine, not of the store.

Co-authored-by: Hanzo Dev <[email protected]>
2026-09-23 00:00:10 -07:00

199 lines
8.8 KiB
TeX
Raw Permalink Blame History

% Read from the run artifacts named below -- do not hand-edit values.
%
% CronQuestions lane: hanzoai/benchmarks, cronquestions/, METHOD.md beside it.
% Two passes, each a set of runs/*/{metrics,meta}.json:
%
% current cloud hanzo-inc/cloud 1fffdf408f2e, apps/graph clean; lane
% commit 2e6d3942d32b; results committed in fa30104, table in 57a55cf.
% runs/hanzo-wire-test, runs/hanzo-replay-test (2026-09-23 UTC)
% first cloud c47fd3692e9a, apps/graph clean; lane commit f8a63ef311ae;
% results committed in 2e6d394 (git show 2e6d394:cronquestions/runs/...)
% (2026-09-19 UTC)
%
% runs/kg-test the control: the plan over dictionaries
% runs/semantica-test semantica 0.7.0 TemporalGraphQuery, semantica-agi/semantica 3910c33bc88c
%
% The first pass's miss counts by relation are the replay predictions at
% 2e6d394 joined to data/plan-test.jsonl on the question index.
%
% Data: CronQuestions data_v2.zip, sha256 0a2f7714...fd4734e;
% facts.tsv 445b1048...7576706; plan-test.jsonl 45d055ec...1817da.
% Host: one Apple M1 Max, 10 cores, arm64, macOS 26.6.2; every store in-process.
%
% LongMemEval lane: hanzoai/benchmarks, brain/longmemeval/, run directory
% brain/runs/longmemeval-all-context-minilm/{metrics,meta}.json (2026-09-19 UTC).
% Engine b467901f3950, configuration frozen on LoCoMo dev at 33d0f8749bd1;
% nothing tuned on LongMemEval. all-MiniLM-L6-v2 through Ollama; the cosine and
% engine columns read the same cached vectors.
%
% Hits@1 and recall are percentages; intervals are 95% bootstrap over questions.
% -- when each run was taken, UTC, from meta.json
\setres{cron/when}{2026-09-23}
\setres{cron/first/when}{2026-09-19}
\setres{cron/sem/when}{2026-09-20}
\setres{lme/when}{2026-09-19}
% -- CronQuestions: the knowledge graph and the question set
\setres{cron/kgfacts}{328,635}
\setres{cron/inverted}{652}
\setres{cron/facts}{327,983}
\setres{cron/entities}{125,726}
\setres{cron/n}{26,168}
\setres{cron/of}{30,000}
\setres{cron/n/se}{7,812}
\setres{cron/n/st}{5,046}
\setres{cron/n/ba}{2,151}
\setres{cron/n/fl}{11,159}
\setres{cron/n/tj}{3,832}
\setres{cron/n/tjtext}{1,407}
\setres{cron/n/tjjoin}{2,425}
% statements repeated in facts.tsv: (subject, relation, object, start year)
% keys on more than one line, and those whose lines differ in end year;
% computed here from facts.tsv rather than read from a run
\setres{cron/dupkeys}{2,919}
\setres{cron/duprows}{5,974}
\setres{cron/dupends}{1,772}
\setres{cron/validn}{26,122}
\setres{cron/validall}{99.93}
% -- CronQuestions, current pass: Hits@1 by type
\setres{cron/kg/all}{99.88} \setres{cron/kg/allci}{[99.84, 99.92]}
\setres{cron/kg/se}{100.00} \setres{cron/kg/st}{100.00}
\setres{cron/kg/ba}{98.56} \setres{cron/kg/fl}{100.00}
\setres{cron/kg/baci}{[98.05, 99.02]}
\setres{cron/sem/all}{99.88} \setres{cron/sem/allci}{[99.84, 99.92]}
\setres{cron/sem/se}{100.00} \setres{cron/sem/st}{100.00}
\setres{cron/sem/ba}{98.56} \setres{cron/sem/fl}{100.00}
\setres{cron/wire/all}{99.88} \setres{cron/wire/allci}{[99.84, 99.92]}
\setres{cron/wire/se}{100.00} \setres{cron/wire/st}{100.00}
\setres{cron/wire/ba}{98.56} \setres{cron/wire/fl}{100.00}
\setres{cron/replay/all}{99.88} \setres{cron/replay/allci}{[99.84, 99.92]}
\setres{cron/replay/se}{100.00} \setres{cron/replay/st}{100.00}
\setres{cron/replay/ba}{98.56} \setres{cron/replay/fl}{100.00}
% predictions differing from the control's, question by question
\setres{cron/sem/differ}{0}
\setres{cron/wire/differ}{0}
\setres{cron/replay/differ}{0}
% -- CronQuestions, current pass: load and query cost
\setres{cron/sem/load}{0.08} \setres{cron/sem/units}{327,983 relationships}
\setres{cron/sem/mb}{---} \setres{cron/sem/p50}{5,357}
\setres{cron/sem/p99}{7,531} \setres{cron/sem/timed}{150}
\setres{cron/sem/distinct}{339}
\setres{cron/sem/at/p50}{6,211} \setres{cron/sem/history/p50}{5,240}
\setres{cron/sem/touch/p50}{5,276}
\setres{cron/wire/load}{116} \setres{cron/wire/units}{655,856 assertions}
\setres{cron/wire/assertions}{655,856}
\setres{cron/wire/mb}{284} \setres{cron/wire/p50}{0.063}
\setres{cron/wire/p99}{2.39} \setres{cron/wire/timed}{27,096}
\setres{cron/wire/at/p50}{0.088} \setres{cron/wire/history/p50}{0.047}
\setres{cron/wire/touch/p50}{0.044}
\setres{cron/wire/refused}{110}
\setres{cron/replay/load}{145} \setres{cron/replay/units}{655,966 assertions}
\setres{cron/replay/assertions}{655,966}
\setres{cron/replay/mb}{284} \setres{cron/replay/p50}{0.058}
\setres{cron/replay/p99}{2.07} \setres{cron/replay/timed}{27,096}
\setres{cron/replay/refused}{0}
% the ratio of the two medians, 5,357 / 0.063, rounded
\setres{cron/ratio}{85,000}
% one-minute load average: at the start, and the peak through each timed phase
\setres{cron/wire/loadavg}{18.65, then 25.04 loading and 26.52 querying}
\setres{cron/replay/loadavg}{25.14, then 41.90 loading and 33.28 querying}
\setres{cron/sem/loadavg}{11.00, then 28.65 querying}
% -- CronQuestions, first pass (cloud c47fd3692e9a): Hits@1
\setres{cron/first/wire/all}{70.03} \setres{cron/first/wire/allci}{[69.47, 70.55]}
\setres{cron/first/wire/se}{0.00} \setres{cron/first/wire/st}{100.00}
\setres{cron/first/wire/ba}{98.56} \setres{cron/first/wire/fl}{100.00}
\setres{cron/first/replay/all}{95.58} \setres{cron/first/replay/allci}{[95.30, 95.83]}
\setres{cron/first/replay/se}{85.59} \setres{cron/first/replay/seci}{[84.82, 86.30]}
\setres{cron/first/replay/st}{100.00}
\setres{cron/first/replay/ba}{98.56} \setres{cron/first/replay/fl}{100.00}
% -- CronQuestions, first pass: load
\setres{cron/first/wire/load}{174} \setres{cron/first/wire/assertions}{1,311,666}
\setres{cron/first/wire/mb}{478}
\setres{cron/first/replay/load}{266} \setres{cron/first/replay/assertions}{1,311,918}
\setres{cron/first/replay/mb}{479}
\setres{cron/first/refused/zero}{14}
\setres{cron/first/refused/zerofacts}{7}
\setres{cron/first/refused/skew}{252}
% -- CronQuestions, first pass: the replayed clock's misses, by relation
\setres{cron/first/replay/misses}{1,126}
\setres{cron/first/replay/miss/position}{552}
\setres{cron/first/replay/miss/team}{540}
\setres{cron/first/replay/miss/employer}{29}
\setres{cron/first/replay/miss/award}{4}
\setres{cron/first/replay/miss/spouse}{1}
\setres{cron/first/replay/bamiss}{31}
% -- LongMemEval-S: session recall over answer_session_ids, 500 questions
\setres{lme/all/n}{500}
\setres{lme/all/cosfive}{85.8} \setres{lme/all/engfive}{90.6}
\setres{lme/all/costen}{93.4} \setres{lme/all/engten}{97.0}
\setres{lme/all/cosmrr}{0.903} \setres{lme/all/engmrr}{0.919}
\setres{lme/all/cosfiveci}{[82.6, 88.8]}
\setres{lme/all/engfiveci}{[87.8, 93.0]}
\setres{lme/all/costenci}{[91.2, 95.6]}
\setres{lme/all/engtenci}{[95.4, 98.4]}
\setres{lme/all/cosanyten}{98.8} \setres{lme/all/enganyten}{99.8}
\setres{lme/ssu/n}{70}
\setres{lme/ssu/cosfive}{95.7} \setres{lme/ssu/engfive}{98.6}
\setres{lme/ssu/costen}{97.1} \setres{lme/ssu/engten}{100.0}
\setres{lme/ssu/cosmrr}{0.852} \setres{lme/ssu/engmrr}{0.872}
\setres{lme/ssa/n}{56}
\setres{lme/ssa/cosfive}{100.0} \setres{lme/ssa/engfive}{100.0}
\setres{lme/ssa/costen}{100.0} \setres{lme/ssa/engten}{100.0}
\setres{lme/ssa/cosmrr}{1.000} \setres{lme/ssa/engmrr}{1.000}
\setres{lme/ssp/n}{30}
\setres{lme/ssp/cosfive}{96.7} \setres{lme/ssp/engfive}{96.7}
\setres{lme/ssp/costen}{96.7} \setres{lme/ssp/engten}{96.7}
\setres{lme/ssp/cosmrr}{0.803} \setres{lme/ssp/engmrr}{0.832}
\setres{lme/ms/n}{133}
\setres{lme/ms/cosfive}{75.9} \setres{lme/ms/engfive}{85.7}
\setres{lme/ms/costen}{89.5} \setres{lme/ms/engten}{94.7}
\setres{lme/ms/cosmrr}{0.912} \setres{lme/ms/engmrr}{0.917}
\setres{lme/tr/n}{133}
\setres{lme/tr/cosfive}{75.9} \setres{lme/tr/engfive}{81.2}
\setres{lme/tr/costen}{88.7} \setres{lme/tr/engten}{94.7}
\setres{lme/tr/cosmrr}{0.865} \setres{lme/tr/engmrr}{0.894}
\setres{lme/ku/n}{78}
\setres{lme/ku/cosfive}{96.2} \setres{lme/ku/engfive}{98.7}
\setres{lme/ku/costen}{98.7} \setres{lme/ku/engten}{100.0}
\setres{lme/ku/cosmrr}{0.964} \setres{lme/ku/engmrr}{0.981}
\setres{lme/abs/n}{30}
\setres{lme/abs/cosfive}{70.0} \setres{lme/abs/engfive}{83.3}
\setres{lme/abs/costen}{83.3} \setres{lme/abs/engten}{100.0}
\setres{lme/abs/cosmrr}{0.774} \setres{lme/abs/engmrr}{0.796}
% -- LongMemEval-S: what moved, and what it cost
\setres{lme/changedfive}{30} \setres{lme/gainfive}{27} \setres{lme/lostfive}{3}
\setres{lme/changedten}{20} \setres{lme/gainten}{19} \setres{lme/lostten}{1}
\setres{lme/firstup}{37} \setres{lme/firstdown}{11} \setres{lme/firstdowntr}{6}
\setres{lme/shortten}{15}
% exact two-sided sign test on the questions that changed, computed from the
% counts above rather than read from the run
\setres{lme/signfive}{$8\times10^{-6}$}
\setres{lme/signten}{$4\times10^{-5}$}
\setres{lme/p50}{1.96} \setres{lme/pninetyfive}{5.87}
\setres{lme/wall}{97}
\setres{lme/cos/p50}{0.20}
Reference in New Issue View Git Blame Copy Permalink