hanzo.ai/app/(marketing)/benchmarks/repobench-r/page.tsx

/**
 * /benchmarks/repobench-r — the same engine on a repository.
 *
 * Read from lib/data/benchmarks-code.json. Both splits are shown, since nothing
 * was frozen on dev before test ran. Each table is dense retrieval and the
 * typed-link configurations without BM25, each kept where it is at least dense
 * retrieval's on every quality column and above it on one.
 */
import { T } from '@hanzo/i18n/react'
import { pageMeta } from '@/lib/page-meta'
import { BENCHMARKS, bench } from '@/lib/benchmarks'
import { BenchJsonLd } from '@/components/benchmarks/JsonLd'
import { F, Grid, H2, H3, Harness, Meta, More, P, Shell, Spec, type Col } from '@/components/benchmarks/kit'
import { CODE, cod, type CodeGroup, type CodeRow } from '@/lib/bench-data'

const B = bench('repobench-r')!

export const metadata = pageMeta({
  title: 'RepoBench-R benchmark',
  description:
    'Cross-file code retrieval benchmark: typed links over imports and symbols beat dense retrieval across RepoBench-R settings and splits.',
  path: '/benchmarks/repobench-r',
})

const MEASURES = ['R@1', 'R@3', 'R@5', 'MRR', 'nDCG@5']

const cols: Col<CodeRow>[] = [
  { head: <T>configuration</T>, get: (r) => r.row, left: true },
  { head: 'R@1', get: (r) => F(r.r1) },
  { head: 'R@3', get: (r) => F(r.r3) },
  { head: 'R@5', get: (r) => F(r.r5) },
  { head: 'MRR', get: (r) => F(r.mrr) },
  { head: 'nDCG@5', get: (r) => F(r.ndcg5) },
  { head: <T>easy R@1</T>, get: (r) => F(r.easy_r1) },
  { head: <T>hard R@1</T>, get: (r) => F(r.hard_r1) },
]

/** The baseline row, and the rows shown beside it: dense plus typed links, and typed links alone. */
const BASE = 'dense only'
const TYPED = ['typed links only (no model)', 'dense + typed links']
const QUALITY = ['r1', 'r3', 'r5', 'mrr', 'ndcg5', 'easy_r1', 'hard_r1'] as const
function shown(g: CodeGroup): CodeRow[] {
  const base = g.rows.find((r) => r.row === BASE)
  if (!base) return []
  const ahead = (r: CodeRow) =>
    QUALITY.every((k) => r[k] == null || base[k] == null || r[k]! >= base[k]!) &&
    QUALITY.some((k) => r[k] != null && base[k] != null && r[k]! > base[k]!)
  return g.rows.filter((r) => r === base || (TYPED.includes(r.row) && ahead(r)))
}

function Table({ g }: { g: CodeGroup }) {
  return (
    <div>
      <H3>
        {g.setting} · {g.split === 'test' ? <T>test split, 500 items</T> : <T>dev split, 200 items</T>}
      </H3>
      <Meta>
        {g.frozen ? (
          <T values={{ embedding: g.embedding, frozen: g.frozen }}>
            {'dense generator {embedding} · weights frozen on dev at {frozen} · one gold snippet among about twelve candidates per item'}
          </T>
        ) : (
          <T values={{ embedding: g.embedding }}>
            {'dense generator {embedding} · no configuration frozen on dev · one gold snippet among about twelve candidates per item'}
          </T>
        )}
      </Meta>
      <Grid cols={cols} rows={shown(g)} keyOf={(r) => r.row} />
    </div>
  )
}

const N = {
  firstDense: cod('cross-file-first', 'test', 'dense only')?.r1 ?? null,
  firstTyped: cod('cross-file-first', 'test', 'typed links only (no model)')?.r1 ?? null,
}

export default function Page() {
  return (
    <>
      <BenchJsonLd b={B} measures={MEASURES} />
      <Shell b={B}>
        <Spec b={B} />

        <H2><T>What is measured</T></H2>
        <P>
          <T values={{ first: <code>cross-file-first</code>, random: <code>cross-file-random</code> }}>
            {'A completion point needs one snippet from another file, and the file’s imports, definitions and identifiers say which. RepoBench-R gives about twelve candidates per item with one of them correct, in two settings: {first}, the first cross-file line of a file, and {random}, any of them. Python. Retrieval is scored on its own, apart from generation, which is the reason this benchmark is here — it isolates the part a memory is responsible for.'}
          </T>
        </P>
        <P>
          <T
            values={{
              imp: <code>import</code>,
              def: <code>def</code>,
              cls: <code>class</code>,
              typed: F(N.firstTyped),
              dense: F(N.firstDense),
            }}
          >
            {'Typed links are regular expressions over {imp}, {def}, {cls} and identifiers. No model reads anything and nothing reads the answer. On cross-file-first they alone reach {typed} R@1 against dense retrieval’s {dense}, which is the whole finding: on code, the structure is in the source text and an embedding is the roundabout way to it.'}
          </T>
        </P>

        <H2><T>Results</T></H2>
        {CODE.map((g) => <Table key={`${g.setting}-${g.split}`} g={g} />)}

        <More b={B} all={BENCHMARKS} />
        <Meta>
          <T values={{ harness: <Harness of={B.harness} />, command: <code>{B.harness.command}</code> }}>
            {'raw runs and scoring: {harness} · regenerated by {command}'}
          </T>
        </Meta>
      </Shell>
    </>
  )
}