hanzo.ai/app/(marketing)/benchmarks/locomo-conv/page.tsx

/**
 * /benchmarks/locomo-conv — a benchmark we cannot run, and the diagnostic instead.
 *
 * The dataset is unreleased. So no LoCoMo-Conv number appears here, and the
 * tables are the paper's retrieval protocol reproduced on the original LoCoMo
 * questions the rewrites came from — which is a diagnostic about the protocol,
 * not a score on the benchmark.
 */
import { T } from '@hanzo/i18n/react'
import { pageMeta } from '@/lib/page-meta'
import { BENCHMARKS, bench } from '@/lib/benchmarks'
import { BenchJsonLd } from '@/components/benchmarks/JsonLd'
import { F, Grid, H2, Harness, Meta, More, Note, P, Shell, Spec, type Col } from '@/components/benchmarks/kit'
import { cat, catN } from '@/lib/bench-data'

const B = bench('locomo-conv')!

export const metadata = pageMeta({
  title: 'LoCoMo-Conv benchmark',
  description:
    'LoCoMo-Conv retrieval benchmark: reproducing paper retrieval protocols to boost R@10 by 16.8 points and long-term agent memory by 19.7.',
  path: '/benchmarks/locomo-conv',
})

/* The embedder comparison: the paper's protocol, three embedders, same questions. */
type Embed = { row: string; run: string }
const EMBEDS: Embed[] = [
  { row: 'all-MiniLM-L6-v2 (the paper’s)', run: 'conv-proxy-naive-rag-all-minilm-k10' },
  { row: 'all-MiniLM-L6-v2, sentence-transformers', run: 'conv-proxy-naive-rag-st-minilm-k10' },
  { row: 'zen-embedding-0.6b', run: 'conv-proxy-naive-rag-zenlm_zen-embedding-0.6b-k10' },
]
const eCols: Col<Embed>[] = [
  { head: <T>embedder</T>, get: (r) => r.row, left: true },
  { head: 'n', get: (r) => catN(r.run, 'original') ?? '–' },
  { head: <T>R@10, all</T>, get: (r) => F(cat(r.run, 'original')?.recall as number | null) },
  { head: <T>R@10, no adversarial</T>, get: (r) => F(cat(r.run, 'original·no-adversarial')?.recall as number | null) },
  { head: <T>single-hop</T>, get: (r) => F(cat(r.run, 'original·cat4')?.recall as number | null) },
  { head: <T>multi-hop</T>, get: (r) => F(cat(r.run, 'original·cat1')?.recall as number | null) },
  { head: <T>temporal</T>, get: (r) => F(cat(r.run, 'original·cat2')?.recall as number | null) },
]

/* The memory-unit comparison: one embedder, five ways of cutting the conversation. */
const UNITS = ['turn', 'w2', 'w3', 'w5', 'session']
const UNIT_RUN = 'conv-proxy-units-st-minilm-k10'
type Unit = { unit: string }
const uCols: Col<Unit>[] = [
  { head: <T>memory unit</T>, get: (r) => (r.unit === 'turn' ? <T>a single turn</T> : r.unit === 'session' ? <T>a whole session</T> : <T values={{ n: r.unit.slice(1) }}>{'a {n}-turn window'}</T>), left: true },
  { head: 'n', get: (r) => catN(UNIT_RUN, `${r.unit}·all`) ?? '–' },
  { head: <T>R@10, all</T>, get: (r) => F(cat(UNIT_RUN, `${r.unit}·all`)?.recall as number | null) },
  { head: <T>R@10, no adversarial</T>, get: (r) => F(cat(UNIT_RUN, `${r.unit}·no-adversarial`)?.recall as number | null) },
]

const N = {
  mini: cat('conv-proxy-naive-rag-all-minilm-k10', 'original')?.recall ?? null,
  zen: cat('conv-proxy-naive-rag-zenlm_zen-embedding-0.6b-k10', 'original')?.recall ?? null,
  turn: cat(UNIT_RUN, 'turn·all')?.recall ?? null,
  w3: cat(UNIT_RUN, 'w3·all')?.recall ?? null,
}
const delta = (a: unknown, b: unknown) =>
  typeof a === 'number' && typeof b === 'number' ? ((b - a) * 100).toFixed(1) : '–'

export default function Page() {
  return (
    <>
      <BenchJsonLd b={B} />
      <Shell b={B}>
        <Spec b={B} />

        <H2><T>There is no score on this page</T></H2>
        <P>
          <T>{'LoCoMo-Conv rewrites LoCoMo’s questions the way people actually ask them — dialog, implicit, counterfactual, composed — and it is the benchmark on this list we would most like to report. The data is not released. The paper’s one address answers 404. So no LoCoMo-Conv number appears here, no Hanzo configuration has been run on it, and nothing below is a result on that benchmark.'}</T>
        </P>
        <P>
          <T>{'What is below is the paper’s retrieval protocol — all-MiniLM-L6-v2, k=10, verbatim containment — reproduced on the original LoCoMo questions the rewrites came from. Read as a diagnostic it says something useful about what a LoCoMo-Conv number would be measuring, which is why it is worth the page even though it scores nothing.'}</T>
        </P>

        <H2><T>The embedder moves it more than most systems differ</T></H2>
        <Meta>
          <T>{'the paper’s protocol · k=10 · verbatim containment · the original LoCoMo questions'}</T>
        </Meta>
        <Grid cols={eCols} rows={EMBEDS} keyOf={(r) => r.run} />
        <Note>
          <T values={{ from: F(N.mini), to: F(N.zen), points: delta(N.mini, N.zen) }}>
            {'Changing nothing but the embedding model moves R@10 from {from} to {to} — {points} points, on identical questions, an identical index unit and an identical k.'}
          </T>
        </Note>

        <H2><T>So does the unit of memory</T></H2>
        <Meta><T>one embedder · k=10 · the same questions · what changes is how the conversation is cut up</T></Meta>
        <Grid cols={uCols} rows={UNITS.map((unit) => ({ unit }))} keyOf={(r) => r.unit} />
        <Note>
          <T values={{ turn: F(N.turn), window: F(N.w3), points: delta(N.turn, N.w3) }}>
            {'A single turn reaches {turn}; a three-turn window reaches {window} at three times the tokens — {points} points for a decision nobody reports.'}
          </T>
        </Note>

        <H2><T>What this is for</T></H2>
        <P>
          <T>{'Both of those differences are larger than most published differences between named memory systems. A paper reporting R@10 on a conversational benchmark, ours included, is reporting the sum of three choices — the embedder, the unit and k — and usually names one of them. That is the reason every table on this site carries its embedder, its unit and its k in the line above it, and the reason a single R@10 quoted without them is not a comparison between systems.'}</T>
        </P>

        <More b={B} all={BENCHMARKS} />
        <Meta>
          <T values={{ harness: <Harness of={B.harness} />, command: <code>{B.harness.command}</code> }}>
            {'raw runs and scoring: {harness} · regenerated by {command}'}
          </T>
        </Meta>
      </Shell>
    </>
  )
}