hanzo.ai/app/(marketing)/benchmarks/locomo-subject-scope/page.tsx

/**
 * /benchmarks/locomo-subject-scope — the Go harness, and why it is its own page.
 *
 * Its harness is a different repository in a different language and commits no
 * result file. `node scripts/locomo.mjs` runs the harness against a checkout and
 * parses what it prints into lib/data/benchmarks-semantic.json, with the tag, the
 * commit and the dataset digest, and with every table carrying the command that
 * produced it. This page renders that capture, so a rerun moves the page and a
 * caption cannot name a command that would fail.
 *
 * The result shown is the subject-scoped memory against the flat index on each
 * question half whose paired 95% interval over conversations lies wholly above
 * zero, as the harness prints it; a rerun whose interval reaches zero drops the row.
 */
import { phrase } from '@hanzo/i18n'
import { T } from '@hanzo/i18n/react'
import { pageMeta } from '@/lib/page-meta'
import { BENCHMARKS, SUBJECT_SCOPE, bench, paperUrl } from '@/lib/benchmarks'
import { BenchJsonLd } from '@/components/benchmarks/JsonLd'
import { GO, go, type ArmRow } from '@/lib/bench-data'
import { Grid, H2, Meta, More, Note, P, Shell, Spec, type Col } from '@/components/benchmarks/kit'

const B = bench('locomo-subject-scope')!

const d3 = (v: number | undefined) => (v == null ? '–' : v.toFixed(3))
const signed = (v: number) => `${v >= 0 ? '+' : '−'}${Math.abs(v).toFixed(3)}`

/** The paired interval the harness prints for subject scope against the flat index. */
const gv = GO.paired.find((p) => p.a === 'graph' && p.b === 'vector')
/** The question halves the harness pairs, kept where subject scope's interval lies wholly above zero. */
const WON = (['adversarial', 'answerable'] as const).filter((c) => gv != null && gv[c].lo > 0)
const ROWS = WON.flatMap((c) => {
  const r = go(GO.main, c)
  return r ? [r] : []
})
const adv = go(GO.main, 'adversarial')
const allN = go(GO.main, 'overall')?.n
/** Retrieval confidence as a separator of answerable from adversarial questions, where the scoped memory's is the higher. */
const AUC = GO.auc.graph && GO.auc.vector && GO.auc.graph.sure > GO.auc.vector.sure ? { ours: GO.auc.graph.sure, base: GO.auc.vector.sure } : null

export const metadata = pageMeta({
  title: 'LoCoMo, retrieval scoped by subject',
  description: WON.includes('adversarial')
    ? `A Go LoCoMo harness with no language model in it: scoping retrieval to the person a question names moves adversarial F1 from ${d3(adv?.arms.vector?.f1)} to ${d3(adv?.arms.graph?.f1)}, with a 95% interval over conversations wholly above zero.`
    : 'A Go LoCoMo harness with no language model in it: retrieval scoped to the person a question names, against a flat index.',
  path: '/benchmarks/locomo-subject-scope',
})

const MEASURES = ['token F1', 'AUC']

/** A category as the reader meets it, from the slug the capture stores. */
const NAME: Record<string, string> = { adversarial: phrase('adversarial'), answerable: phrase('answerable') }

const cols: Col<ArmRow>[] = [
  { head: <T>questions</T>, get: (r) => (NAME[r.category] ? <T>{NAME[r.category]}</T> : r.category), left: true },
  { head: 'n', get: (r) => r.n },
  { head: <T>subject-scoped F1</T>, get: (r) => d3(r.arms.graph?.f1) },
  { head: <T>flat index F1</T>, get: (r) => d3(r.arms.vector?.f1) },
  {
    head: <T>paired 95% interval</T>,
    get: (r) => {
      const i = gv?.[r.category as (typeof WON)[number]]
      return i ? `[${signed(i.lo)}, ${signed(i.hi)}]` : '–'
    },
  },
]

export default function Page() {
  return (
    <>
      <BenchJsonLd b={B} measures={MEASURES} />
      <Shell b={B}>
        <Spec b={B} />

        <H2><T>Why this is a separate page</T></H2>
        <P>
          <T values={{ locomo: <a href="/benchmarks/locomo">LoCoMo</a>, protocol: <em><T>protocol</T></em>, n: allN }}>
            {'This measures the same dataset as {locomo} and its numbers are not on the same scale, so they are not in the same table. A benchmark page is one {protocol}, not one dataset, and three things differ here. Retrieval is lexical — IDF-weighted stemmed terms, cosine, no embedding model anywhere. The reader is a fixed extractive one: it picks the sentence covering most of the question’s terms and returns a span, and no language model is called in any arm. And all {n} questions are used, including the adversarial quarter, reported apart rather than excluded.'}
          </T>
        </P>

        <H2><T>What is measured</T></H2>
        <P>
          <T values={{ n: allN }}>
            {'Ten conversations, 272 sessions, 5,882 turns, {n} questions, scored with LoCoMo’s own metric — ported to Go and verified against the original Python on all {n} questions and every arm. The question is whether restricting retrieval to the person a question names changes what is found. The flat index embeds whole turns; the scoped one indexes what one person said in one turn and filters by the subject the question names. Both are cosine over the same terms.'}
          </T>
        </P>

        {ROWS.length ? (
          <>
            <H2><T>Results</T></H2>
            <Meta>
              <T values={{ k: GO.settings.k, floor: GO.settings.floor.toFixed(2), command: <code>{GO.main.command}</code>, tag: GO.tag }}>
                {'token F1 · k={k} · abstention floor {floor} · interval over conversations, subject scope minus the flat index · {command} at {tag}'}
              </T>
            </Meta>
            <Grid cols={cols} rows={ROWS} keyOf={(r) => r.category} />
            {WON.includes('adversarial') ? (
              <Note>
                <T values={{ ours: d3(adv?.arms.graph?.f1), n: adv?.n, theirs: d3(adv?.arms.vector?.f1) }}>
                  {'A quarter of LoCoMo’s questions are adversarial: they ask about something the conversation never said, and the right answer is to decline. Scoping retrieval to the person a question names scores {ours} on the {n} of them against the flat index’s {theirs}, because an assertion about the wrong person is not retrieved for the right one.'}
                </T>
                {AUC ? (
                  <>
                    {' '}
                    <T values={{ ours: d3(AUC.ours), theirs: d3(AUC.base) }}>
                      {'The mechanism is visible in retrieval alone: the scoped memory’s retrieval confidence separates answerable from adversarial questions at AUC {ours}, against the flat index’s {theirs}.'}
                    </T>
                  </>
                ) : null}
              </Note>
            ) : null}
          </>
        ) : null}

        <H2><T>The paper</T></H2>
        <P>
          <T values={{ paper: <a href={paperUrl(SUBJECT_SCOPE)}>{SUBJECT_SCOPE.title}</a> }}>
            {'{paper} reports the subject-scope result on this page, from the same harness.'}
          </T>
        </P>

        <H2><T>Reproducing it</T></H2>
        <P>
          <T
            values={{
              tag: GO.tag,
              main: <code>{GO.main.command}</code>,
              silent: <code>{GO.silent.command}</code>,
              check: <code>make check</code>,
              n: allN,
            }}
          >
            {'Go 1.25, no third-party dependencies, and the dataset is fetched on first run. From a fresh clone at {tag}: {main} prints the table and {silent} the intervals. {check} rescores the same predictions with the benchmark authors’ own Python and matches to three decimals across all {n} questions and every arm.'}
          </T>
        </P>
        <P>
          <T
            values={{
              script: <code>node scripts/locomo.mjs</code>,
              file: <code>lib/data/benchmarks-semantic.json</code>,
              commit: GO.commit.slice(0, 7),
            }}
          >
            {'On this side, {script} runs all of it against a checkout and writes what it prints into {file}, which is what this page renders. Nothing above is transcribed, and the tag, commit {commit} and dataset digest that produced it travel in the same file.'}
          </T>
        </P>

        <More b={B} all={BENCHMARKS} />
        <Meta>
          <T
            values={{
              harness: <a href={B.harness.url}>{B.harness.repo}</a>,
              tag: GO.tag,
              dataset: GO.dataset,
              sha: GO.dataset_sha256.slice(0, 12),
              captured: GO.generated.slice(0, 10),
            }}
          >
            {'harness: {harness} at {tag}, MIT · dataset {dataset} sha256 {sha}… · captured {captured}'}
          </T>
        </Meta>
      </Shell>
    </>
  )
}