hanzo.ai/lib/bench-data.ts

/**
 * The published runs, read by name.
 *
 * `node results.mjs` in hanzoai/benchmarks/brain writes four of these files
 * from the metrics file each run directory carries; nothing here is typed in,
 * and a rerun that changes a number changes every page that quotes it. On a
 * fresh anonymous clone, with no dataset fetched and nothing installed, that
 * command reproduces all four — they differ from the committed copies only in
 * the `generated` stamp. Three more come from elsewhere, and each says so
 * where it is read below: the Go harness capture (GO), CronQuestions' own
 * harness (CRON), and the runtime board, the one transcription (RUNTIME).
 *
 * Lookups are BY NAME rather than by position. A table and the sentence beside
 * it read the same row through the same call, so the two cannot drift, and a
 * row that is renamed or removed in a rerun resolves to `null` and renders as
 * `–` rather than silently quoting its neighbour.
 */
import retrieval from '@/lib/data/benchmarks-retrieval.json' with { type: 'json' }
import code from '@/lib/data/benchmarks-code.json' with { type: 'json' }
import runs from '@/lib/data/benchmarks.json' with { type: 'json' }
import longmem from '@/lib/data/benchmarks-longmemeval.json' with { type: 'json' }
import semantic from '@/lib/data/benchmarks-semantic.json' with { type: 'json' }
import cronq from '@/lib/data/benchmarks-cronquestions.json' with { type: 'json' }
import runtime from '@/lib/data/benchmarks-runtime.json' with { type: 'json' }
import decision from '@/lib/data/benchmarks-kai.json' with { type: 'json' }
import shortlist from '@/lib/data/benchmarks-shortlist.json' with { type: 'json' }
import grounding from '@/lib/data/benchmarks-grounding.json' with { type: 'json' }
import coding from '@/lib/data/benchmarks-coding.json' with { type: 'json' }
import { derive as repairs } from '@/lib/coding'
import tools from '@/lib/data/benchmarks-tools.json' with { type: 'json' }
import { derive as handlers } from '@/lib/tools'
import { derive, type Run } from '@/lib/kai'
import { TROLLEY } from '@/lib/trolley'
import { TIE, count, ece, rate } from '@/lib/tally'
import { board } from '@/lib/board'
import { ledger } from '@/lib/results'
import type { Interval, Metric } from '@/components/benchmarks/kit'

export type RetrievalRow = {
  row: string; failed: boolean
  mh_all: number | null; mh_any: number | null; sh_all: number | null
  temporal_all: number | null; open_all: number | null
  recall: number | null; ndcg: number | null; supported: number | null
  examined: number | null; tokens: number | null; p50: number | null
}
export type RetrievalGroup = { split: string; facts: string; embedding: string; k: number; frozen: string | null; rows: RetrievalRow[] }

export type CodeRow = {
  row: string
  r1: number | null; r3: number | null; r5: number | null; mrr: number | null; ndcg5: number | null
  easy_r1: number | null; hard_r1: number | null; examined: number | null; p50: number | null; n: number | null
}
export type CodeGroup = { setting: string; split: string; embedding: string; frozen: string | null; rows: CodeRow[] }

export type Category = { name: string; n: number | null; metrics: Record<string, Metric> }

/**
 * What one execution was, and what it cost.
 *
 * `results.mjs` writes this from the run's own meta.json: the commit it was
 * frozen at, the digests that pin its inputs, the counts, and the wall time and
 * tokens it spent. Every field can be null, because a field the harness did not
 * write is absent rather than zero — a run that says nothing about its counts
 * reads `unknown`, which is not the same claim as `complete`.
 */
export type RunRecord = {
  commit: string | null; prompt: string | null
  prompt_digest: string | null; dataset_digest: string | null
  store_digest: string | null; facts_digest: string | null
  reader: string | null; embedding: string | null
  temperature: number | null; max_tokens: number | null
  questions: number | null; answered: number | null
  started: string | null; ended: string | null
  completion: Completion
  wall_seconds: number | null; tokens_per_question: number | null; reader_tokens: number | null
}

/**
 * Whether a run finished, derived where the record is written and read here.
 *
 * `complete` needs the counts to agree AND the record to say when the run
 * ended, because a harness that checkpoints produces the record a finished run
 * produces minus the denominator. `unknown` is a real answer and the commonest
 * one: the retrieval, code and MemoryAgentBench runs are single deterministic
 * passes that record no counts, so nothing about them says they stopped early
 * and nothing says they did not.
 */
export type Completion = 'complete' | 'partial' | 'inconsistent' | 'unknown'

export type Section = {
  bench: string; run: string; row: string | null; split: string | null; k: number | null
  reader: string | null; facts: string | null; commit: string | null
  record: RunRecord
  categories: Category[]; table: string | null
}

export type Tally = { n: number; mrr: Record<string, number>; all: Record<string, number>; any: Record<string, number> }

export const TABLES = (retrieval as { tables: RetrievalGroup[] }).tables
export const SECTIONS = (runs as { sections: Section[] }).sections
export const CODE = (code as { tables: CodeGroup[] }).tables
export const LME = longmem as {
  embed: string; k: number[]; tally: Record<string, Tally>; latency_ms: { p50: number; p95: number }
  context: { run: string; row: string; frozen: { commit: string; engine: string; objective: string }; latency_ms: { p50: number; p95: number }; wall_seconds: number }
}

/** When each file was last rolled up. `app/sitemap.ts` offers these as `lastModified`. */
export const GENERATED = {
  retrieval: (retrieval as { generated: string }).generated,
  benchmarks: (runs as { generated: string }).generated,
  code: (code as { generated: string }).generated,
  semantic: (semantic as { generated: string }).generated,
  cronquestions: (cronq as { generated: string }).generated,
} as const

export const group = (split: string, embedding: string, k: number) =>
  TABLES.find((t) => t.split === split && t.embedding === embedding && t.k === k) ?? null
export const ret = (split: string, embedding: string, k: number, row: string) =>
  group(split, embedding, k)?.rows.find((r) => r.row === row) ?? null
export const sec = (run: string) => SECTIONS.find((s) => s.run === run) ?? null
export const cat = (run: string, name: string) => sec(run)?.categories.find((c) => c.name === name)?.metrics ?? null
export const catN = (run: string, name: string) => sec(run)?.categories.find((c) => c.name === name)?.n ?? null
export const cod = (setting: string, split: string, row: string) =>
  CODE.find((t) => t.setting === setting && t.split === split)?.rows.find((r) => r.row === row) ?? null
export const num = (m: Metric | undefined) => (typeof m === 'number' ? m : null)

/** LongMemEval is counts, not rates: a tally over that question type's own n. */
export const lme = (type: string, key: 'tm@5' | 'tm@10' | 'ctx@5' | 'ctx@10', kind: 'all' | 'any') => {
  const t = LME.tally[type]
  return t ? t[kind][key] / t.n : null
}

/**
 * MemoryAgentBench pools four haystacks: 6k is the dev split, the rest is test.
 * All four or nothing — a mean over three lanes is a different quantity with
 * the same name.
 */
export const pool = (devRun: string, testRun: string, kind: 'mh' | 'sh') => {
  const lanes: [string, string][] = [[devRun, `${kind}_6k`], [testRun, `${kind}_32k`], [testRun, `${kind}_64k`], [testRun, `${kind}_262k`]]
  const vs = lanes.map(([r, c]) => num(cat(r, c)?.substring_em)).filter((v): v is number => v != null)
  return vs.length === 4 ? vs.reduce((a, b) => a + b, 0) / 4 : null
}

/** One run's execution record, by run name. */
export const rec = (run: string) => sec(run)?.record ?? null

/** How far a run got, as the record has it. A run with no record says nothing. */
export const done = (run: string): Completion => rec(run)?.completion ?? 'unknown'

/**
 * Whether a row may stand in a comparison.
 *
 * A comparison ranks systems, so every row in one has to be a system that
 * finished the work — `partial` and `inconsistent` are excluded from ranking and
 * appear only in a table with their counts showing. `unknown` is admitted: a
 * deterministic single-pass run records no counts, and refusing every row that
 * does not is refusing most of what has actually been measured.
 */
export const rankable = (run: string) => { const c = done(run); return c === 'complete' || c === 'unknown' }

/* ---- the Go harness, which prints tables rather than writing run files ---- */

/** One arm's three figures for one category: what it scored, found, and declined. */
export type Arm = { f1: number; recall: number; quiet: number }
export type ArmRow = { category: string; n: number; arms: Record<string, Arm> }
export type ArmTable = { command: string; arms: string[]; rows: ArmRow[] }

/**
 * Everything hanzoai/semantic's LoCoMo harness prints, captured.
 *
 * `node scripts/locomo.mjs` runs the five commands against a checkout and
 * parses their output into lib/data/benchmarks-semantic.json. It is the one
 * lane on this site whose harness lives in another repository and commits no
 * result file, so the capture stands in for one — with the tag, the commit and
 * the dataset digest that produced it, and with each table carrying the exact
 * command that printed it, so a caption cannot name a command that does not run.
 *
 * `blind` is the same table over emptied graphs. Its `identical` and `changed`
 * lists are computed cell by cell at capture time rather than asserted, so the
 * page reports the outcome of the control instead of its conclusion.
 */
export const GO = semantic as {
  harness: string
  tag: string
  commit: string
  dataset: string
  dataset_sha256: string
  settings: { k: number; floor: number; span: number; band: number; parts: number }
  generated: string
  main: ArmTable
  factor: ArmTable
  silent: { command: string; f1: Record<string, number> }
  auc: Record<string, { sure: number; decided: number; quiet_answerable: number; quiet_adversarial: number }>
  paired: { a: string; b: string; adversarial: Interval; answerable: Interval }[]
  walk: ArmTable
  blind: ArmTable & { identical: string[]; changed: string[] }
}

/** One category of one of those tables, by name. Absent reads null, never a neighbour. */
export const go = (t: ArmTable, category: string) => t.rows.find((r) => r.category === category) ?? null

/**
 * CronQuestions: every scored row of the test split, written by `python3
 * cron.py site test` in hanzoai/benchmarks/cronquestions from the runs it
 * scored. A row is a store; its hits carry bootstrap intervals per question
 * type, and its query_ms the latency of every call it answered.
 */
export type CronRow = {
  row: string; store: string; subject: string
  hits: Record<string, Interval>
  query_ms: Record<string, { n: number; p50: number; p99: number; mean: number }>
  load: { seconds?: number; facts?: number; assertions?: number; bytes?: number; refused?: Record<string, number> }
  load_1m: { at_start?: number; peak_loading?: number; peak_queries?: number }
}
export const CRON = cronq as { generated: string; split: string; covered: string[]; rows: CronRow[] }
export const cron = (row: string) => CRON.rows.find((r) => r.row === row) ?? null

/* ---- the runtime pass, which a README prints rather than a run file ---- */

/**
 * What an agent costs to keep, to start, to wake and to give a shell to.
 *
 * One pass of hanzoai/benchmarks, and the one lane here that is transcribed: the
 * harness prints its board into README.md and commits no raw output, so
 * lib/data/benchmarks-runtime.json is typed in from that board once and every
 * page renders it from there. Each pass names the command that prints its rows,
 * short enough to run while reading the page, and the machine it ran on — the
 * goroutine timings read about half as fast on a 10-core M1 Max as on the M4 Max
 * they were published from, while the memory figure holds.
 *
 * Three columns rather than two, because two was how a wrong number survived:
 *
 *   ours       what this pass measured.
 *   theirs     a figure attributed to somebody else and not re-run.
 *   checked    what their own library did when run here.
 *
 * The sandbox lane once set a V8 context against a cited isolate figure and
 * called the gap seventeenfold. A context shares its isolate's heap, so the two
 * are not one primitive; running the library the citation names put an isolate
 * at 0.35 ms and 1.00 MiB, and that measurement is the `checked` column.
 */
export type Part = 'fleet' | 'goroutine' | 'sandbox'
export type Pass = { repo: string; url: string; board: string; when: string; machine: string; command: string }
export type Shot = { part: Part; what: string; ours: string | null; theirs: string | null; checked: string | null }
/** A language runtime an agent can host in-process, from the same pass. */
export type Host = { runtime: string; language: string; instantiate: string; call: string }

/**
 * The rows the pages publish: a figure of ours that no figure published beside
 * it beats. hanzo-vm's cold boot and its resume from a checkpoint are beaten by
 * the times E2B and Morph publish, and a row with no figure of ours is the other
 * side of a comparison, not a result, so none of those is shown.
 */
const BEATEN = new Set(['hanzo-vm, cold boot', 'hanzo-vm, from a checkpoint'])
const PASS = runtime as { passes: Record<Part, Pass>; shots: Shot[]; hosts: Host[] }
export const RUNTIME = { ...PASS, shots: PASS.shots.filter((s) => s.ours != null && !BEATEN.has(s.what)) }
export const HOSTS = RUNTIME.hosts

/** One part of the pass, in the order an agent's life runs: kept, started, woken, given a shell. */
export const shots = (part: Part) => RUNTIME.shots.filter((s) => s.part === part)

/** One row of the pass by part and name. Absent reads null, never a neighbour. */
export const shot = (part: Part, what: string) => RUNTIME.shots.find((s) => s.part === part && s.what === what) ?? null

/* ---- what the index features ---- */

/**
 * The results /benchmarks leads with, each resolved here by name.
 *
 * The index quotes a few rows from pages that carry the whole table. Every
 * lookup it needs is here rather than in the component, so the index holds no
 * number of its own — e2e/unit/figures.spec.ts fails on a digit in its source —
 * and a rerun moves the index with the pages it summarises. Each `base` and
 * `ours` pair is one metric over the same data, with the same embedder and the
 * same k.
 */
const MAB = {
  dev: { base: 'mab-dev-noreader-none', ours: 'mab-dev-beamx-none' },
  test: { base: 'mab-test-noreader-none', ours: 'mab-test-beamx-none' },
}
/** The largest MemoryAgentBench haystack, as its lanes are named. */
const HAYSTACK = '262k'
const LOCOMO = { split: 'test', embedding: 'zen-embedding-0.6b', k: 20 } as const
const loco = (row: string) => ret(LOCOMO.split, LOCOMO.embedding, LOCOMO.k, row)
/** The LoCoMo answer runs from the one reader that finished cosine, the engine and the gold turns alike. */
const READER = (policy: string) => `locomo-all-${policy}-k${LOCOMO.k}-gemma4-31b`
/** LongMemEval's cut-off: every evidence session in the top K. */
const K = 5
const semantica = cron('semantica')
const graph = cron('hanzo-wire')

/* ---- Kai's reads the research API does not file ---- */

/**
 * Two of Kai's results are reads whose harness is private and which no research
 * run records: the tool shortlist (benchmarks-shortlist.json, one run's report)
 * and Kai's yes/no answers beside Jev's (benchmarks-grounding.json, the two
 * served reads' metrics). Each file is its read's own numbers, copied unchanged.
 * A figure reaches the page only while ours leads on it, the rule lib/kai keeps
 * for the harness: a shortlist metric where the shortlist beats BM25, a yes/no
 * test where Kai leads Jev on every half of it.
 */
type Ranked = { any: number; all: number; top1: number; ndcg10: number }
const SL = shortlist as {
  measured: string; embedder: string; model: string; bm25_weight: number; k: number; ndcg_k: number
  catalog: number; queries: number
  groups: Record<string, { n: number; bm25: Ranked; shortlist: Ranked }>
}
/** The figures a shortlist is read by, each with the cut it is taken at. */
const RANKED: { key: keyof Ranked; label: string }[] = [
  { key: 'any', label: `Recall@${SL.k}` },
  { key: 'ndcg10', label: `nDCG@${SL.ndcg_k}` },
  { key: 'top1', label: 'Top-1' },
]
/** One group of requests by name, BM25 beside the shortlist on each figure the shortlist leads. */
const ranked = (name: string) => {
  const g = SL.groups[name]
  if (!g) return null
  return {
    n: count(g.n),
    rows: RANKED.filter((m) => g.shortlist[m.key] > g.bm25[m.key]).map((m) => ({ label: m.label, base: g.bm25[m.key], ours: g.shortlist[m.key] })),
  }
}

type Side = { model: string; measured: string; metrics: Record<string, number> }
const YN = grounding as {
  kai: Side; jev: Side; band: number[]
  n: { paraphrase: number; absent: number; negation: number; fields: Record<string, number>; matrix: number; states: number; retained: number }
}
/** How many suites the pooled accuracy pools: one `retained.<suite>.rows` count each. */
const SUITES = Object.keys(YN.kai.metrics).filter((k) => /^retained\.[^.]+\.rows$/.test(k)).length
const of = (total: number) => (v: number) => `${count(v)} of ${count(total)}`
/**
 * The yes/no tests both reads carry. A test is drawn only when Kai leads on every
 * half of it: a pair of states, or a count and its order, is one test, and a
 * half Jev holds keeps the whole test off the page. `tie` is the lead a half
 * needs past equal; accuracy takes lib/kai's, two questions in four hundred.
 */
type Half = { key: string; label: string; better: 'high' | 'low'; shown: (v: number) => string; tie?: number }
const TESTS: Half[][] = [
  [
    { key: 'critical.yes', label: 'Workbench starter, a state that warrants yes: p(yes)', better: 'high', shown: ece },
    { key: 'critical.no', label: 'Its counterfactual, a state that warrants no: p(yes)', better: 'low', shown: ece },
  ],
  [{ key: 'paraphrase', label: `A question and its paraphrase, mean |Δp| over ${YN.n.paraphrase} pairs`, better: 'low', shown: ece }],
  [{ key: 'negation.right', label: `A state and its negation answered apart: share of ${YN.n.negation} pairs`, better: 'high', shown: rate }],
  [
    { key: 'fields.enterprise', label: `Typed field, plan: share of ${YN.n.fields.enterprise} pairs right`, better: 'high', shown: rate },
    { key: 'fields.seats', label: `Typed field, seats: share of ${YN.n.fields.seats} pairs right`, better: 'high', shown: rate },
    { key: 'fields.verified', label: `Typed field, verified email: share of ${YN.n.fields.verified} pairs right`, better: 'high', shown: rate },
    { key: 'fields.language', label: `Typed field, message language: share of ${YN.n.fields.language} pairs right`, better: 'high', shown: rate },
  ],
  [{ key: 'absent.within', label: `A state silent on the question, answered between ${YN.band[0]} and ${YN.band[1]}: share of ${YN.n.absent}`, better: 'high', shown: rate }],
  [
    { key: 'matrix.in_band', label: 'Predicate matrix, cells in band', better: 'high', shown: of(YN.n.matrix) },
    { key: 'matrix.ordered', label: 'Predicate matrix, states in order', better: 'high', shown: of(YN.n.states) },
  ],
  [{ key: 'retained.pooled', label: `Accuracy over ${count(YN.n.retained)} validation rows of ${SUITES} yes/no suites, pooled`, better: 'high', shown: rate, tie: TIE }],
]
const leads = (h: Half) => {
  const k = YN.kai.metrics[h.key]
  const j = YN.jev.metrics[h.key]
  if (typeof k !== 'number' || typeof j !== 'number') return false
  const lead = h.better === 'high' ? k - j : j - k
  return lead > 0 && lead >= (h.tie ?? 0)
}

export const INDEX = {
  mab: {
    metric: 'substring EM',
    record: rec(MAB.test.ours),
    single: { base: pool(MAB.dev.base, MAB.test.base, 'sh'), ours: pool(MAB.dev.ours, MAB.test.ours, 'sh') },
    multi: { base: pool(MAB.dev.base, MAB.test.base, 'mh'), ours: pool(MAB.dev.ours, MAB.test.ours, 'mh') },
    haystack: HAYSTACK,
    largest: {
      base: cat(MAB.test.base, `mh_${HAYSTACK}`)?.substring_em ?? null,
      ours: cat(MAB.test.ours, `mh_${HAYSTACK}`)?.substring_em ?? null,
    },
  },
  locomo: {
    ...LOCOMO,
    all: { base: loco('semantic only')?.mh_all ?? null, ours: loco('+iterative hops')?.mh_all ?? null },
    recall: { base: loco('semantic only')?.recall ?? null, ours: loco('+iterative hops')?.recall ?? null },
    answer: {
      n: catN(READER('context'), 'test:multi-hop'),
      base: cat(READER('single'), 'test:multi-hop')?.f1 ?? null,
      ours: cat(READER('context'), 'test:multi-hop')?.f1 ?? null,
      ceiling: cat(READER('oracle'), 'test:multi-hop')?.f1 ?? null,
    },
  },
  lme: {
    k: K,
    n: LME.tally.ALL?.n ?? null,
    embed: LME.embed,
    base: lme('ALL', `tm@${K}`, 'all'),
    ours: lme('ALL', `ctx@${K}`, 'all'),
    ms: { base: LME.latency_ms.p50, ours: LME.context.latency_ms.p50 },
  },
  code: {
    metric: 'R@1',
    split: 'test',
    first: {
      setting: 'cross-file-first',
      base: cod('cross-file-first', 'test', 'dense only')?.r1 ?? null,
      ours: cod('cross-file-first', 'test', 'typed links only (no model)')?.r1 ?? null,
    },
    // Where the full combination scores below dense plus typed links alone.
    random: {
      setting: 'cross-file-random',
      full: cod('cross-file-random', 'test', 'full: dense + BM25 + typed links')?.r1 ?? null,
      pair: cod('cross-file-random', 'test', 'dense + typed links')?.r1 ?? null,
    },
  },
  cron: {
    metric: 'Hits@1',
    hits: { base: semantica?.hits.overall ?? null, ours: graph?.hits.overall ?? null },
    ms: { base: semantica?.query_ms.all?.p50 ?? null, ours: graph?.query_ms.all?.p50 ?? null },
  },
  runtime: {
    passes: RUNTIME.passes,
    dormant: shot('fleet', 'state per dormant agent'),
    heap: shot('goroutine', 'heap per live agent'),
    spawn: shot('goroutine', 'spawn one'),
    wake: shot('goroutine', 'wake one million'),
  },
  /** The tool shortlist Kai's tool-relevance head ranks, against BM25 alone on the same requests. */
  shortlist: {
    measured: SL.measured,
    embedder: SL.embedder,
    model: SL.model,
    k: SL.k,
    ndcg: SL.ndcg_k,
    catalog: count(SL.catalog),
    queries: count(SL.queries),
    all: ranked('all'),
    /** Requests whose gold is a Hanzo service: a domain, scored as the best of its operations. */
    hanzo: ranked('hanzo/domain'),
  },
  /** Kai's yes/no answers beside Jev's on the same questions and rows: every half of every test Kai leads. */
  yesno: {
    kai: { model: YN.kai.model, measured: YN.kai.measured },
    jev: { model: YN.jev.model, measured: YN.jev.measured },
    rows: TESTS.filter((t) => t.every(leads)).flat().map((h) => ({ label: h.label, kai: h.shown(YN.kai.metrics[h.key]), jev: h.shown(YN.jev.metrics[h.key]) })),
  },
  /** The trolley problem: the public benchmark's checks Kai leads Jev on, and the sealed trolley-structured scenes it was qualified on (lib/trolley). */
  trolley: TROLLEY,
  /** Hanzo Dev against Claude Code on held-out repairs: the research API's runs of coding.repair (lib/coding). */
  coding: repairs(coding.data as Run[]),
  /** Hanzo MCP's tool handlers beside the ones they replaced: the research API's runs of the mcp.* benchmarks (lib/tools). */
  tools: handlers(tools.data as Run[]),
}

/* ---- Kai: the research API's runs, as lib/kai derives them ---- */

/**
 * Kai against Jev, from the build-time snapshot of `GET
 * /v1/research/runs?org=hanzo` (`node scripts/kai.mjs`). The pages fetch the
 * same address at runtime through `useKai` and render this until it returns.
 */
export const KAI = derive((decision as unknown as { data: Run[] }).data)

/* ---- the index's own build-time figures: headline, sections, library (lib/board) ---- */

export const BOARD = board(KAI, INDEX)
/** Every result and every benchmark page, one ledger (lib/results). */
export const LEDGER = ledger({ index: INDEX, runtime: RUNTIME, generated: { ...GENERATED, longmemeval: (longmem as { generated: string }).generated } })
export type { Entry, Result, Standing, State, Product } from '@/lib/results'
export { PRODUCTS } from '@/lib/results'
export type { Board as Page, Rate, Story } from '@/lib/board'
export type { Board, Claim, Cell, Kai } from '@/lib/kai'