hanzo.ai/app/(marketing)/benchmarks/locomo/page.tsx

/**
 * /benchmarks/locomo — the engine on LoCoMo retrieval.
 *
 * Every figure is read from lib/data/benchmarks-retrieval.json through
 * lib/bench-data. The prose reads the same rows the tables do, so a rerun moves
 * the sentence and the cell together. A table shows cosine and each finished
 * configuration that is at least cosine's on every quality column and above it
 * on one, so a rerun that drops a configuration below cosine drops its row.
 */
import { T } from '@hanzo/i18n/react'
import { pageMeta } from '@/lib/page-meta'
import { BENCHMARKS, bench } from '@/lib/benchmarks'
import { BenchJsonLd } from '@/components/benchmarks/JsonLd'
import { F, Grid, H2, H3, Harness, Meta, More, P, Shell, Spec, type Col } from '@/components/benchmarks/kit'
import { TABLES, ret, type RetrievalGroup, type RetrievalRow } from '@/lib/bench-data'

const B = bench('locomo')!

export const metadata = pageMeta({
  title: 'LoCoMo benchmark',
  description:
    'Hanzo Context on LoCoMo retrieval: multi-hop ALL@20 jumps from 18.8 to 37.5 on held-out splits, outperforming standard cosine search.',
  path: '/benchmarks/locomo',
})

const MEASURES = ['multi-hop ALL@20', 'R@20', 'nDCG@20']

/* Every figure quoted below, resolved from the run files by name. */
const N = {
  mhBase: ret('test', 'zen-embedding-0.6b', 20, 'semantic only')?.mh_all ?? null,
  mhOurs: ret('test', 'zen-embedding-0.6b', 20, '+iterative hops')?.mh_all ?? null,
  recBase: ret('test', 'zen-embedding-0.6b', 20, 'semantic only')?.recall ?? null,
  recOurs: ret('test', 'zen-embedding-0.6b', 20, '+iterative hops')?.recall ?? null,
}

/** The baseline row: cosine over turns. */
const BASE = 'semantic only'
/** The quality columns a configuration is held to, all higher-is-better. */
const QUALITY = ['mh_all', 'mh_any', 'sh_all', 'temporal_all', 'open_all', 'recall', 'ndcg', 'supported'] as const

/** Cosine, and each finished configuration at least cosine's on every quality column and above it on one. */
function shown(g: RetrievalGroup): RetrievalRow[] {
  const base = g.rows.find((r) => r.row === BASE)
  if (!base) return []
  const ahead = (r: RetrievalRow) =>
    !r.failed &&
    QUALITY.every((k) => r[k] == null || base[k] == null || r[k]! >= base[k]!) &&
    QUALITY.some((k) => r[k] != null && base[k] != null && r[k]! > base[k]!)
  return g.rows.filter((r) => r === base || ahead(r))
}

const RETRIEVAL: Col<RetrievalRow>[] = [
  { head: <T>configuration</T>, get: (r) => r.row, left: true },
  { head: <T>multi-hop ALL</T>, get: (r) => F(r.mh_all) },
  { head: <T>multi-hop ANY</T>, get: (r) => F(r.mh_any) },
  { head: <T>single-hop ALL</T>, get: (r) => F(r.sh_all) },
  { head: <T>temporal ALL</T>, get: (r) => F(r.temporal_all) },
  { head: <T>open-domain ALL</T>, get: (r) => F(r.open_all) },
  { head: 'R@k', get: (r) => F(r.recall) },
  { head: 'nDCG@k', get: (r) => F(r.ndcg) },
  { head: <T>supported</T>, get: (r) => F(r.supported) },
]

function Retrieval({ g }: { g: RetrievalGroup }) {
  return (
    <div>
      <H3>
        {g.split === 'test' ? <T>test split, conversations 3–9</T> : g.split === 'all' ? <T>all ten conversations</T> : <T>dev split, conversations 0–2</T>} · {g.embedding} · k={g.k}
      </H3>
      <Meta>
        <T values={{ facts: g.facts === 'locomo' ? <T>LoCoMo’s own observation annotations</T> : g.facts }}>{'facts {facts}'}</T>
        {g.frozen ? (
          <>
            {' · '}
            <T values={{ frozen: g.frozen }}>{'weights frozen on dev at {frozen}'}</T>
          </>
        ) : null}
        {' · '}
        <T values={{ k: g.k }}>
          {'rows are cumulative: each carries every generator before it, from BM25 on · ALL = every annotated turn inside the top {k}; ANY = at least one; R@{k} = |retrieved ∩ gold| / |gold|'}
        </T>
      </Meta>
      <Grid cols={RETRIEVAL} rows={shown(g)} keyOf={(r) => r.row} />
    </div>
  )
}

export default function Page() {
  const groups = TABLES.filter((g) => g.split !== 'dev')
  return (
    <>
      <BenchJsonLd b={B} measures={MEASURES} />
      <Shell b={B}>
        <Spec b={B} />

        <H2><T>What is measured</T></H2>
        <P>
          <T>{'Ten conversations, 5,882 turns, 1,540 questions, each annotated with the turns that answer it. Retrieval is scored against those turns. Configurations are chosen on conversations 0–2 and frozen at a commit before conversations 3–9 run once. Adversarial questions are excluded, as Mem0 and Zep exclude them.'}</T>
        </P>

        <H2><T>Retrieval</T></H2>
        <P>
          <T values={{ mhBase: F(N.mhBase), mhOurs: F(N.mhOurs), recBase: F(N.recBase), recOurs: F(N.recOurs) }}>
            {'The engine adds candidate generators to cosine — BM25, the atomic fact index, canonical entities, the timeline, adjacency, one hop of typed expansion, then a second retrieval hop asked with what the first resolved. On the held-out split multi-hop ALL@20 goes from {mhBase} to {mhOurs} and R@20 from {recBase} to {recOurs}, on the same embedder and the same k.'}
          </T>
        </P>
        {groups.map((g) => <Retrieval key={`${g.split}-${g.embedding}-${g.k}`} g={g} />)}

        <H2><T>The other LoCoMo harness</T></H2>
        <P>
          <T values={{ harness: <a href="/benchmarks/locomo-subject-scope">hanzoai/semantic · bench/locomo</a> }}>
            {'{harness} measures the same dataset under a different protocol: lexical retrieval with no embedding model, a fixed extractive reader with no language model anywhere, and all 1,986 questions including the adversarial quarter, reported apart. Its numbers measure a different reader and are not on the scale of these, so it has its own page and its own paper.'}
          </T>
        </P>

        <More b={B} all={BENCHMARKS} />
        <Meta>
          <T
            values={{
              harness: <Harness of={B.harness} />,
              method: <code>METHOD.md</code>,
              command: <code>{B.harness.command}</code>,
            }}
          >
            {'raw runs, traces, prompts and scoring: {harness} · protocol in {method} · every table above regenerated by {command}'}
          </T>
        </Meta>
      </Shell>
    </>
  )
}