hanzo.ai/app/(marketing)/benchmarks/memoryagentbench/page.tsx

/**
 * /benchmarks/memoryagentbench — FactConsolidation, with no model reading anything.
 *
 * Figures read from lib/data/benchmarks.json. The Pith comparison is a vendor
 * figure and is written here as a constant, labelled as theirs; the lanes
 * quoted are the ones typed search scores above it, derived rather than asserted.
 */
import { phrase } from '@hanzo/i18n'
import { T } from '@hanzo/i18n/react'
import { pageMeta } from '@/lib/page-meta'
import { BENCHMARKS, bench } from '@/lib/benchmarks'
import { BenchJsonLd } from '@/components/benchmarks/JsonLd'
import { F, Grid, H2, H3, Harness, Meta, More, Note, P, Shell, Spec, list, type Col } from '@/components/benchmarks/kit'
import { cat, num, pool, sec } from '@/lib/bench-data'

const B = bench('memoryagentbench')!

export const metadata = pageMeta({
  title: 'MemoryAgentBench benchmark',
  description:
    'MemoryAgentBench benchmark: Hanzo Context doubles multi-hop substring EM from 33 to 67 on 262k haystacks without LLM memory reads.',
  path: '/benchmarks/memoryagentbench',
})

const MEASURES = ['substring exact match', 'plan recall@N', 'execution success@N']

const LANES = ['6k', '32k', '64k', '262k'] as const
const DEV = { base: 'mab-dev-noreader-none', ours: 'mab-dev-beamx-none' }
const TEST = { base: 'mab-test-noreader-none', ours: 'mab-test-beamx-none' }

/** 6k is the dev split; 32k, 64k and 262k are test. One row, four haystacks. */
const lane = (run: { base: string; ours: string }, which: 'base' | 'ours', kind: 'mh' | 'sh', size: string) =>
  num(cat(which === 'base' ? run.base : run.ours, `${kind}_${size}`)?.substring_em)

type Row = { row: string; kind: 'mh' | 'sh'; cells: (number | null)[]; pooled: number | null }
const rows: Row[] = (['sh', 'mh'] as const).flatMap((kind) =>
  ([[phrase('plain resolver'), 'base'], [phrase('typed search'), 'ours']] as const).map(([label, which]) => ({
    row: label,
    kind,
    cells: LANES.map((size) => lane(size === '6k' ? DEV : TEST, which, kind, size)),
    pooled: pool(which === 'base' ? DEV.base : DEV.ours, which === 'base' ? TEST.base : TEST.ours, kind),
  })),
)

const cols: Col<Row>[] = [
  { head: <T>configuration</T>, get: (r) => <T>{r.row}</T>, left: true },
  ...LANES.map((size, i) => ({ head: size, get: (r: Row) => F(r.cells[i]) })),
  { head: <T>pooled</T>, get: (r) => F(r.pooled) },
]

/* Pith's per-lane multi-hop figures, from its vendor page. Theirs, not ours,
   and not re-run here — so they are a constant, and only the reading of them
   is derived. */
const PITH = [0.95, 0.83, 0.84, 0.68]
const ours = rows.find((r) => r.kind === 'mh' && r.row === 'typed search')!
/** The lanes typed search scores above Pith's figure, by index. */
const above = LANES.flatMap((_, i) => ((ours.cells[i] ?? 0) > PITH[i] ? [i] : []))

const N = {
  mh262Base: cat(TEST.base, 'mh_262k')?.substring_em ?? null,
  mh262Ours: cat(TEST.ours, 'mh_262k')?.substring_em ?? null,
  mhPoolBase: pool(DEV.base, TEST.base, 'mh'),
  mhPoolOurs: pool(DEV.ours, TEST.ours, 'mh'),
  shPoolBase: pool(DEV.base, TEST.base, 'sh'),
  shPoolOurs: pool(DEV.ours, TEST.ours, 'sh'),
}

export default function Page() {
  const n = sec(TEST.ours)?.categories[0]?.n ?? null
  return (
    <>
      <BenchJsonLd b={B} measures={MEASURES} />
      <Shell b={B}>
        <Spec b={B} />

        <H2><T>What is measured</T></H2>
        <P>
          {n ? (
            <T>
              {'FactConsolidation. A haystack is a numbered list of templated facts in which a later line silently replaces an earlier one about the same subject and relation, and nothing marks the update. Questions ask for the current value, single-hop and multi-hop, over haystacks of 6k to 262k tokens. One hundred questions per haystack. The metric is substring exact match, the benchmark’s own; an unanswered question scores zero. Dev is the 6k haystacks, test is 32k, 64k and 262k.'}
            </T>
          ) : (
            <T>
              {'FactConsolidation. A haystack is a numbered list of templated facts in which a later line silently replaces an earlier one about the same subject and relation, and nothing marks the update. Questions ask for the current value, single-hop and multi-hop, over haystacks of 6k to 262k tokens. The metric is substring exact match, the benchmark’s own; an unanswered question scores zero. Dev is the 6k haystacks, test is 32k, 64k and 262k.'}
            </T>
          )}
        </P>
        <P>
          <T>{'No language model reads the memory or writes the answer in any row here. The question is compiled into a plan sketch and a deterministic typed search over the parsed fact store resolves it — 45 templates, no line unparsed at any size, the supersession chain keyed by subject and relation. That is why these rows are not comparable to a pipeline that reads the haystack with a frontier model, and it is also the point: the retrieval is doing the work.'}</T>
        </P>

        <H2><T>Results</T></H2>
        <Meta><T>substring exact match · 6k from dev, 32k / 64k / 262k from test · no reader · pooled = the mean of the four haystacks</T></Meta>
        <H3><T>single-hop</T></H3>
        <Grid cols={cols} rows={rows.filter((r) => r.kind === 'sh')} keyOf={(r) => `sh-${r.row}`} />
        <H3><T>multi-hop</T></H3>
        <Grid cols={cols} rows={rows.filter((r) => r.kind === 'mh')} keyOf={(r) => `mh-${r.row}`} />
        <Note>
          <T
            values={{
              shOurs: F(N.shPoolOurs),
              mhOurs: F(N.mhPoolOurs),
              shBase: F(N.shPoolBase),
              mhBase: F(N.mhPoolBase),
              from: F(N.mh262Base),
              to: F(N.mh262Ours),
            }}
          >
            {'Typed search pools to {shOurs} single-hop and {mhOurs} multi-hop against the plain resolver’s {shBase} and {mhBase}. The largest single move is 262k multi-hop, from {from} to {to}.'}
          </T>
        </Note>

        <H2><T>Numbers other systems publish</T></H2>
        <P>
          <T>{'Quoted with their protocols, because the protocols differ, and none of them re-run here. CAR (arXiv 2606.01435) reports 78.0 single-hop / 30.2 multi-hop pooled over 6k–262k with gpt-4o-mini and 94.8 / 51.5 with gpt-4o, and 82 / 27 and 93 / 41 respectively at 262k. Its own baselines at 262k multi-hop: HippoRAG-v2 5, gpt-4o whole-context 5, BM25 3, Cognee and MemGPT 3, Mem0 2, Zep 3. That is the nearest thing to a like-for-like comparison anywhere on this site — same data, same split, same substring-EM metric — and it is still not one, because CAR reads the haystack with a frontier model and the rows above use no model at all.'}</T>
        </P>
        {above.length ? (
          <P>
            <T
              values={{
                theirs: above.map((i) => (PITH[i] * 100).toFixed(0)).join(' / '),
                lanes: list(above.map((i) => LANES[i])),
                ours: above.map((i) => F(ours.cells[i])).join(' / '),
              }}
            >
              {'Pith publishes per-lane multi-hop figures on a vendor page: {theirs} at {lanes}, where typed search scores {ours}.'}
            </T>
          </P>
        ) : null}

        <More b={B} all={BENCHMARKS} />
        <Meta>
          <T values={{ harness: <Harness of={B.harness} />, command: <code>{B.harness.command}</code> }}>
            {'raw runs, plans, traces and scoring: {harness} · every table above regenerated by {command}'}
          </T>
        </Meta>
      </Shell>
    </>
  )
}