hanzo.ai/lib/results.ts

/**
 * Every result the benchmark pages carry, as one ledger.
 *
 * A result is one measured figure beside the baseline it was measured against:
 * a Kai revision on one suite against Jev, an engine cell against llama.cpp, a
 * memory run against cosine. Each says who measured it, when, and whether it is the live result,
 * a frozen transcription, or superseded by a later run of the same benchmark.
 * An entry is the benchmark itself, the page a result opens: what it measures,
 * the conditions, the hardware, the source and the command that reproduces it.
 *
 * Nothing here is typed. Each figure is read from the file that holds it: the
 * research API's runs (lib/data/benchmarks-kai.json), the engine's filed
 * experiments, the committed memory and code runs, and the runtime passes. The
 * index page and the detail pages render this; the library filters it in the
 * browser. Enso's results are not here: its boards are the research API's
 * `enso.*` runs, read when a page opens (lib/enso), so no build holds them.
 *
 * Build-time only. It takes the index figures as an argument because
 * lib/bench-data, which computes them, is the module that calls it.
 */
import runs from './data/benchmarks-kai.json' with { type: 'json' }
import { MASSIVE, RUNS as API, SUITES, type Run } from './kai'
import { BENCHMARKS as REGISTRY, LANE, type Bench, type Lane } from './benchmarks'
import { CELLS, COMMAND as ENGINE_COMMAND, RUNS as ENGINE_RUNS, RUNS_URL as ENGINE_URL, name as modelName, wins } from './engine'
import cloud from './data/bench.json' with { type: 'json' }
import { DESKS, MEASURED } from '../components/zen/desks'
import { count, day, rate } from './tally'

import { PRODUCTS, type Product } from './products'
export { PRODUCTS, type Product }
/** How the result stands against its baseline. `unscored` where there is no baseline to stand against, or the harness says the cell is unsettled. */
export type Standing = 'win' | 'level' | 'loss' | 'unscored'
/** `live`: the newest run the API serves. `frozen`: a committed run file, snapshot or transcription. `superseded`: an earlier run a later one of the same benchmark replaced. */
export type State = 'live' | 'frozen' | 'superseded'
export type By = 'Hanzo' | 'Vendor' | 'Board'

export type Result = {
  id: string
  /** The entry's slug: the page this result opens. */
  entry: string
  name: string
  product: Product
  category: string
  metric: string
  /** What was measured, and what it was measured against. */
  subject: string
  against: string | null
  value: number | null
  base: number | null
  better: 'high' | 'low' | null
  shown: string
  baseShown: string | null
  by: By
  /** ISO day (or month, where the source records only the month), or null where it records none. */
  date: string | null
  /** The date as a reader reads it, or why there is none. */
  when: string
  state: State
  standing: Standing
}

export type Entry = {
  slug: string
  name: string
  product: Product
  category: string
  what: string
  href: string
  /** A hand-written page already exists at href; otherwise the detail route draws one. */
  own: boolean
  conditions: string[]
  hardware: string | null
  source: { label: string; href: string } | null
  command: string | null
}

/** The ledger, and the products it holds in the story's order. */
export type Ledger = { results: Result[]; entries: Entry[]; products: Product[] }

/** The index figures lib/bench-data computes, as this file needs them. */
type Metric = number | { mean: number; lo?: number; hi?: number } | null
type Pair = { base: Metric; ours: Metric }
export type Inputs = {
  index: {
    mab: { metric: string; single: Pair; multi: Pair; haystack: string; largest: Pair }
    locomo: { k: number; split: string; embedding: string; all: Pair; recall: Pair; answer: { n: number | null; base: Metric; ours: Metric; ceiling: Metric } }
    lme: { k: number; n: number | null; embed: string; base: Metric; ours: Metric; ms: { base: number | null; ours: number | null } }
    code: { metric: string; split: string; first: { setting: string; base: Metric; ours: Metric }; random: { setting: string; full: Metric; pair: Metric } }
    cron: { metric: string; hits: { base: Metric; ours: Metric }; ms: { base: number | null; ours: number | null } }
  }
  runtime: {
    passes: Record<string, { url: string; when: string; machine: string; command: string }>
    shots: { part: string; what: string; ours: string | null; theirs: string | null }[]
  }
  /** Each run file's roll-up stamp, by the registry's `data` key, or by slug where the registry names no file. */
  generated: Record<string, string>
}

/* ---- shared readers ---- */

/** A run as the API files it: lib/kai's fields, and the counts and digests it does not read. */
type Filed = Run & { questions?: number | null; answered?: number | null; dataset_digest?: string }
const RAW = (runs as unknown as { data: Filed[] }).data
const val = (m: Metric | undefined) => (m == null ? null : typeof m === 'number' ? m : m.mean)
const measure = (r: Run | null | undefined, category: string, metric: string) =>
  r?.measures?.find((m) => m.category === category && m.metric === metric)?.value ?? null
const isoDay = (iso: string | null | undefined) => (iso ? iso.slice(0, 10) : null)
const slug = (s: string) => s.toLowerCase().replace(/['’]/g, '').replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
const stage = (version: string) => {
  const [s, w] = version.split('@')
  return w ? `${s}@${w.slice(0, 8)}` : version
}
/** A Kai revision as a row names it: `kai-1 · 0834a74f`, `Kai a5 · df1c16a2`. */
const revision = (version: string) => {
  const [s, w] = version.split('@')
  return `${/^kai/i.test(s) ? s : `Kai ${s}`}${w ? ` · ${w.slice(0, 8)}` : ''}`
}
/** A field of a run's notes, `key value` among `; `-separated parts. */
const note = (r: Run | null | undefined, key: string) =>
  (r?.notes ?? '').split('; ').find((p) => p.startsWith(`${key} `))?.slice(key.length + 1).trim() ?? null
const newest = (rs: Filed[]) => [...rs].sort((a, b) => (a.ended < b.ended ? 1 : a.ended > b.ended ? -1 : a.version < b.version ? 1 : -1))[0] ?? null
/** Two questions in four hundred: lib/kai's tie, and lib/tally's. */
const TIE = 0.005

/** Ahead, level or behind, in the metric's own direction; within `tie` either way is level. */
function stand(value: number | null, base: number | null, better: 'high' | 'low', tie = 0): Standing {
  if (value == null || base == null) return 'unscored'
  const g = better === 'high' ? value - base : base - value
  return g > tie ? 'win' : g < -tie ? 'loss' : 'level'
}

const DEVICES: Record<string, string> = { cpu: 'CPU', cuda: 'CUDA', rocm: 'ROCm', vulkan: 'Vulkan', metal: 'Metal' }
function hardware(r: Run | null | undefined): string | null {
  const d = note(r, 'device')
  const host = note(r, 'host') ?? (r?.by || null)
  const on = d ? [DEVICES[d] ?? d, note(r, 'dtype')?.toUpperCase()].filter(Boolean).join(' ') : null
  return on || host ? [on, host ? `on ${host}` : null].filter(Boolean).join(' ') : null
}
const runLink = (r: Run) => `${API}&id=${encodeURIComponent(r.id)}`

/* ---- Kai: every revision of every suite, against Jev's run of it ---- */

function kai(results: Result[], entries: Entry[]) {
  const ofBench = (bench: string, split?: string) => RAW.filter((r) => r.benchmark === bench && (!split || r.split === split))
  const jev = (bench: string) => newest(ofBench(bench).filter((r) => r.system === 'jev' && r.baseline))
  const kais = (bench: string, split?: string) => ofBench(bench, split).filter((r) => r.system === 'kai').sort((a, b) => (a.ended < b.ended ? 1 : -1))

  /** One entry per suite, every Kai revision a result, accuracy and calibration error each. */
  const suite = (bench: string, label: string, category: string, metrics: [string, string, 'high' | 'low'][], split?: string) => {
    const mine = kais(bench, split)
    if (!mine.length) return
    const top = mine[0]
    const base = jev(bench)
    const e = `kai-${slug(label)}`
    entries.push({
      slug: e,
      name: label,
      product: 'Kai',
      category,
      what: base
        ? `Kai and Jev on the same ${top.questions ? count(top.questions) : ''} questions of ${label}, one run per Kai revision.`.replace('same  ', 'same ')
        : `Kai on ${label}, one run per revision. No baseline was run on this split.`,
      href: `/benchmarks/${e}`,
      own: false,
      conditions: [
        `split: ${top.split}`,
        ...(top.questions ? [`${count(top.questions)} questions, ${count(top.answered ?? top.questions)} answered`] : []),
        ...(top.dataset_digest ? [`dataset ${top.dataset_digest.slice(0, 12)}`] : []),
        `${mine.length} Kai ${mine.length === 1 ? 'revision' : 'revisions'} measured`,
        ...(base ? [`Jev: ${base.version}, measured ${day(base.ended)}`] : []),
      ],
      hardware: hardware(top),
      source: { label: `research run ${top.id}`, href: runLink(top) },
      command: top.split === 'harness' ? `bench score --harness . --kai results/kai/preds.json.gz --out results/scores.json   # in hanzoai/benchmarks/decision @ ${top.commit?.slice(0, 7) ?? 'HEAD'}` : null,
    })
    for (const r of mine) {
      for (const [metric, shownAs, better] of metrics) {
        const v = measure(r, 'all', metric)
        if (v == null) continue
        const b = base ? measure(base, 'all', metric) : null
        const pct = better === 'high'
        results.push({
          id: `${e}.${metric}.${slug(stage(r.version))}`,
          entry: e,
          name: label,
          product: 'Kai',
          category,
          metric: shownAs,
          subject: revision(r.version),
          against: base ? 'Jev' : null,
          value: v,
          base: b,
          better,
          shown: pct ? rate(v) : v.toFixed(3),
          baseShown: b == null ? null : pct ? rate(b) : b.toFixed(3),
          by: 'Hanzo',
          date: isoDay(r.ended),
          when: day(r.ended),
          state: r === mine[0] ? 'live' : 'superseded',
          standing: stand(v, b, better, pct ? TIE : 0),
        })
      }
    }
  }

  const ACC: [string, string, 'high' | 'low'][] = [
    ['accuracy', 'Accuracy', 'high'],
    ['ece', 'Calibration error (ECE)', 'low'],
  ]
  for (const [bench, label] of SUITES) suite(bench, label, 'Decision harness', ACC, 'harness')
  suite(MASSIVE, 'MASSIVE', 'Decision harness', [
    ['macro_accuracy', 'Accuracy', 'high'],
    ['macro_ece', 'Calibration error (ECE)', 'low'],
  ], 'harness')
  for (const bench of [...new Set(RAW.filter((r) => r.system === 'kai' && r.split.startsWith('val:')).map((r) => r.benchmark))]) {
    const [, task] = bench.split('.')
    const label = SUITES.find(([b]) => b.endsWith(`.${task}`) || b === task)?.[1] ?? task.replace(/_/g, ' ')
    suite(bench, `${label}, validation split`, 'Validation split', ACC)
  }

  /* choosing among many options: one result per choice size */
  const card = kais('cap.cardinality')[0]
  const cardJev = jev('cap.cardinality')
  if (card) {
    const ks = [...new Set((card.measures ?? []).flatMap((m) => { const k = /^k(\d+)\.accall$/.exec(m.metric); return k && m.category === 'all' ? [Number(k[1])] : [] }))].sort((a, b) => a - b)
    /** The sizes Kai leads Jev at, the ones the page shows. */
    const won = ks.filter((k) => stand(measure(card, 'all', `k${k}.accall`), measure(cardJev, 'all', `k${k}.rejected`) === 1 ? null : measure(cardJev, 'all', `k${k}.accall`), 'high', TIE) === 'win')
    entries.push({
      slug: 'kai-choice-size',
      name: 'Choosing among many options',
      product: 'Kai',
      category: 'Capability',
      what: 'Accuracy at each number of listed options, every option scored, the same cases put to both models.',
      href: '/benchmarks/kai-choice-size',
      own: false,
      conditions: [`choice sizes: ${won.map(count).join(', ')}`, 'every option scored', ...(cardJev ? [`Jev: ${cardJev.version}, measured ${day(cardJev.ended)}`] : [])],
      hardware: hardware(card),
      source: { label: `research run ${card.id}`, href: runLink(card) },
      command: 'bench cap --harness . --who kai,jev --kai <checkpoint>   # in hanzoai/benchmarks/decision',
    })
    for (const k of ks) {
      const v = measure(card, 'all', `k${k}.accall`)
      const refused = measure(cardJev, 'all', `k${k}.rejected`) === 1
      const b = refused ? null : measure(cardJev, 'all', `k${k}.accall`)
      results.push({
        id: `kai-choice-size.k${k}`,
        entry: 'kai-choice-size',
        name: 'Choosing among many options',
        product: 'Kai',
        category: 'Capability',
        metric: 'Accuracy',
        subject: `${revision(card.version)} · ${count(k)} options`,
        against: cardJev ? 'Jev' : null,
        value: v,
        base: b,
        better: 'high',
        shown: rate(v),
        baseShown: refused ? 'refused' : b == null ? null : rate(b),
        by: 'Hanzo',
        date: isoDay(card.ended),
        when: day(card.ended),
        state: 'live',
        standing: refused ? 'unscored' : stand(v, b, 'high', TIE),
      })
    }
  }

  /* joint decisions: one capability suite with a plain accuracy both models report */
  const joint = kais('cap.joint')[0]
  const jointJev = jev('cap.joint')
  if (joint && measure(joint, 'all', 'acc') != null) {
    entries.push({
      slug: 'kai-joint',
      name: 'Joint decisions',
      product: 'Kai',
      category: 'Capability',
      what: 'Several typed fields decided together on one case, scored on all of them at once.',
      href: '/benchmarks/kai-joint',
      own: false,
      conditions: ['the same cases to both models', ...(jointJev ? [`Jev: ${jointJev.version}, measured ${day(jointJev.ended)}`] : [])],
      hardware: hardware(joint),
      source: { label: `research run ${joint.id}`, href: runLink(joint) },
      command: 'bench cap --harness . --who kai,jev --kai <checkpoint>   # in hanzoai/benchmarks/decision',
    })
    const v = measure(joint, 'all', 'acc')
    const b = measure(jointJev, 'all', 'acc')
    results.push({
      id: 'kai-joint.acc',
      entry: 'kai-joint',
      name: 'Joint decisions',
      product: 'Kai',
      category: 'Capability',
      metric: 'Accuracy',
      subject: revision(joint.version),
      against: jointJev ? 'Jev' : null,
      value: v,
      base: b,
      better: 'high',
      shown: rate(v),
      baseShown: b == null ? null : rate(b),
      by: 'Hanzo',
      date: isoDay(joint.ended),
      when: day(joint.ended),
      state: 'live',
      standing: stand(v, b, 'high', TIE),
    })
  }

  /* the release gate: every revision put to it, and whether it passed */
  const gates = kais('gate.jev')
  if (gates.length) {
    entries.push({
      slug: 'kai-release-gate',
      name: 'Release gate against Jev',
      product: 'Kai',
      category: 'Release gate',
      what: 'Whether a Kai revision regresses past tolerance against Jev on any gated suite. Passing it is the condition for calling a revision state of the art.',
      href: '/benchmarks/kai-release-gate',
      own: false,
      conditions: [note(gates[0], 'tolerance') ? `tolerance ${note(gates[0], 'tolerance')}` : (gates[0].notes ?? '').split(';')[0]].filter(Boolean),
      hardware: null,
      source: { label: `research run ${gates[0].id}`, href: runLink(gates[0]) },
      command: null,
    })
    for (const r of gates) {
      const pass = measure(r, 'all', 'pass')
      const regressions = measure(r, 'all', 'regressions')
      const suites = measure(r, 'all', 'suites')
      results.push({
        id: `kai-release-gate.${slug(stage(r.version))}`,
        entry: 'kai-release-gate',
        name: 'Release gate against Jev',
        product: 'Kai',
        category: 'Release gate',
        metric: 'Gate result',
        subject: revision(r.version),
        against: 'Jev',
        value: pass,
        base: null,
        better: 'high',
        shown: pass === 1 ? 'passes' : `fails${regressions != null ? ` · ${count(regressions)} regressed` : ''}${suites != null ? ` of ${count(suites)} suites` : ''}`,
        baseShown: null,
        by: 'Hanzo',
        date: isoDay(r.ended),
        when: day(r.ended),
        state: r === gates[0] ? 'live' : 'superseded',
        standing: pass === 1 ? 'win' : pass === 0 ? 'loss' : 'unscored',
      })
    }
  }

  /* latency: one decision, per serving mode and device */
  const lat = kais('latency')[0]
  if (lat) {
    entries.push({
      slug: 'kai-latency',
      name: 'Latency of one decision',
      product: 'Kai',
      category: 'Latency',
      what: 'Time to one decision, by serving mode and device. No baseline was timed on the same hardware, so these stand alone.',
      href: '/benchmarks/kai-latency',
      own: false,
      conditions: [(lat.notes ?? '').split(';')[0]].filter(Boolean),
      hardware: hardware(lat),
      source: { label: `research run ${lat.id}`, href: runLink(lat) },
      command: 'bench speed --states results/states.json.gz --results capability/results --who kai --kai <checkpoint>',
    })
    for (const m of lat.measures ?? []) {
      const [mode, on] = m.category.split('/')
      results.push({
        id: `kai-latency.${slug(m.category)}.${m.metric}`,
        entry: 'kai-latency',
        name: 'Latency of one decision',
        product: 'Kai',
        category: 'Latency',
        metric: m.metric.replace(/ms$/, ' latency'),
        subject: `${revision(lat.version)} · ${mode}${on ? `, ${on.replace(/-/g, ' ').toUpperCase()}` : ''}`,
        against: null,
        value: m.value,
        base: null,
        better: 'low',
        shown: `${m.value < 10 ? m.value.toFixed(2) : m.value.toFixed(1)} ms`,
        baseShown: null,
        by: 'Hanzo',
        date: isoDay(lat.ended),
        when: day(lat.ended),
        state: 'live',
        standing: 'unscored',
      })
    }
  }
}

/* ---- Zen 6: the latest open model's throughput on machines Hanzo owns ---- */

/** The leading figure of a reading as a number, for ordering and drawing: `~2,400` → 2400, `1,150–1,400` → 1150, the conservative end. */
const lead = (shown: string) => {
  const m = /[\d,.]+/.exec(shown)
  return m ? Number(m[0].replace(/,/g, '')) : null
}

function zen6(results: Result[], entries: Entry[]) {
  entries.push({
    slug: 'zen-6',
    name: 'Zen 6 throughput',
    product: 'Zen 6',
    category: 'Throughput on Hanzo hardware',
    what: 'Zen 6, the latest open-weight generation, on three machines Hanzo owns: prompt prefill and single-request decode, in tokens per second, one machine each.',
    href: '/benchmarks/zen-6',
    own: false,
    conditions: [
      ...DESKS.map((d) => `${d.machine}: ${d.memory}, ${d.build}`),
      'decode is one request unless a row says otherwise',
      'a range is what the runs spread over; the low end is the figure ordered and drawn',
    ],
    hardware: DESKS.map((d) => d.machine).join(' · '),
    source: { label: `Zen 6, measured by Hanzo on ${day(MEASURED)}`, href: '/zen' },
    command: null,
  })
  for (const d of DESKS) {
    for (const [metric, shown] of [['Prefill throughput', d.prefill], ['Decode throughput', d.decode]] as const) {
      results.push({
        id: `zen-6.${slug(d.machine)}.${metric.split(' ')[0].toLowerCase()}`,
        entry: 'zen-6',
        name: 'Zen 6 throughput',
        product: 'Zen 6',
        category: 'Throughput on Hanzo hardware',
        metric,
        subject: `Zen 6 · ${d.machine}`,
        against: null,
        value: lead(shown),
        base: null,
        better: 'high',
        shown: `${shown} tok/s`,
        baseShown: null,
        by: 'Hanzo',
        date: MEASURED,
        when: day(MEASURED),
        state: 'frozen',
        standing: 'unscored',
      })
    }
    if (d.more)
      results.push({
        id: `zen-6.${slug(d.machine)}.more`,
        entry: 'zen-6',
        name: 'Zen 6 throughput',
        product: 'Zen 6',
        category: 'Throughput on Hanzo hardware',
        metric: 'Also measured',
        subject: `Zen 6 · ${d.machine}`,
        against: null,
        value: null,
        base: null,
        better: null,
        shown: d.more,
        baseShown: null,
        by: 'Hanzo',
        date: MEASURED,
        when: day(MEASURED),
        state: 'frozen',
        standing: 'unscored',
      })
  }
}

/* ---- the agent platform's own bench: residency, a cold sandbox, a call ---- */

const BENCH = 'https://github.com/hanzoai/benchmarks'
function platform(results: Result[], entries: Entry[]) {
  const rows = (cloud as { source: string; machine: string; rows: { id: string; claim: string; ours: string; published: string; multiple: string; detail: string; command: string }[] }).rows
  const machine = (cloud as { machine: string }).machine
  const month = /(\w+) (\d{4})/.exec(machine)
  const iso = month ? `${month[2]}-${String(new Date(`${month[1]} 1, 2000`).getMonth() + 1).padStart(2, '0')}` : null
  entries.push({
    slug: 'platform-bench',
    name: 'Platform bench',
    product: 'Agent platform',
    category: 'Cost and start-up',
    what: 'What an agent costs at rest, what a sandbox that runs a shell costs to start cold, and what one call costs at the margin, each beside the figure its alternative publishes. Every script re-runs to produce its row.',
    href: '/benchmarks/platform-bench',
    own: false,
    conditions: [...rows.map((r) => `${r.claim}: ${r.detail}`), 'the alternative figures are as their publishers state them, not re-run here'],
    hardware: machine,
    source: { label: 'hanzoai/benchmarks · fleet, sandbox, pricing', href: BENCH },
    command: rows.map((r) => r.command).join(' && '),
  })
  for (const r of rows)
    results.push({
      id: `platform-bench.${r.id}`,
      entry: 'platform-bench',
      name: 'Platform bench',
      product: 'Agent platform',
      category: 'Cost and start-up',
      metric: r.claim,
      subject: 'Hanzo',
      against: 'as its alternative publishes',
      value: null,
      base: null,
      better: null,
      shown: r.ours,
      baseShown: `${r.published} · ${r.multiple}`,
      by: 'Hanzo',
      date: iso,
      when: machine.replace(/^.*?,\s*/, ''),
      state: 'frozen',
      standing: 'unscored',
    })
}

/* ---- the engine against llama.cpp: every filed cell ---- */

function engine(results: Result[]) {
  const when = new Map(ENGINE_RUNS.map((r) => [r.run, r]))
  for (const c of CELLS) {
    const run = when.get(c.run)
    const phase = c.phase === 'prefill' ? 'Prefill' : 'Decode'
    results.push({
      id: `inference.${c.run}.${c.phase}.${c.n}`,
      entry: 'inference',
      name: 'Engine prefill on CUDA vs llama.cpp',
      product: 'Zen 6',
      category: `Engine ${phase.toLowerCase()} vs llama.cpp`,
      metric: `${phase} throughput vs llama.cpp`,
      subject: `${modelName(c.model)} · ${c.backend} · ${count(c.n)} tokens`,
      against: 'llama.cpp',
      value: c.ratio,
      base: 1,
      better: 'high',
      shown: `${c.ratio.toFixed(2)}× (${c.lo.toFixed(2)}–${c.hi.toFixed(2)})`,
      baseShown: '1.00×',
      by: 'Hanzo',
      date: isoDay(run?.iso),
      when: run?.iso ? day(run.iso) : 'undated',
      state: 'frozen',
      standing: wins.includes(c) ? 'win' : c.noisy ? 'unscored' : c.verdict === 'PARITY' ? 'level' : 'loss',
    })
  }
}

/* ---- memory, graph and code: the committed runs, each beside its baseline ---- */

function memory(results: Result[], { index, generated }: Inputs, registry: Map<string, Bench>) {
  const stamp = (s: string) => { const g = registry.get(s)?.data; const iso = (g ? generated[g] : null) ?? generated[s] ?? null; return { date: isoDay(iso), when: iso ? day(iso) : 'undated' } }
  const add = (s: string, key: string, metric: string, subject: string, against: string, ours: Metric, base: Metric, better: 'high' | 'low', fmt: (v: number) => string, standing?: Standing) => {
    const v = val(ours)
    const b = val(base)
    const b0 = registry.get(s)
    results.push({
      id: `${s}.${key}`,
      entry: s,
      name: b0?.name ?? s,
      product: b0 ? PRODUCT[b0.lane](b0) : 'Agent harness',
      category: b0 ? LANE[b0.lane].title : 'Memory',
      metric,
      subject,
      against,
      value: v,
      base: b,
      better,
      shown: v == null ? '–' : fmt(v),
      baseShown: b == null ? null : fmt(b),
      by: 'Hanzo',
      ...stamp(s),
      state: 'frozen',
      standing: standing ?? stand(v, b, better),
    })
  }
  const pct = (v: number) => `${(v * 100).toFixed(1)}%`
  const msf = (v: number) => (v >= 100 ? `${Math.round(v).toLocaleString('en-US')} ms` : `${v.toFixed(v < 1 ? 3 : 2)} ms`)
  const { mab, locomo, lme, code, cron } = index
  add('memoryagentbench', 'single', mab.metric, 'Hanzo typed search · single-hop, pooled', 'plain resolver', mab.single.ours, mab.single.base, 'high', pct)
  add('memoryagentbench', 'multi', mab.metric, 'Hanzo typed search · multi-hop, pooled', 'plain resolver', mab.multi.ours, mab.multi.base, 'high', pct)
  add('memoryagentbench', 'largest', mab.metric, `Hanzo typed search · ${mab.haystack} multi-hop`, 'plain resolver', mab.largest.ours, mab.largest.base, 'high', pct)
  add('locomo', 'all', `multi-hop ALL@${locomo.k}`, 'Hanzo Context', 'cosine over turns', locomo.all.ours, locomo.all.base, 'high', pct)
  add('locomo', 'recall', `R@${locomo.k}`, 'Hanzo Context', 'cosine over turns', locomo.recall.ours, locomo.recall.base, 'high', pct)
  const a = locomo.answer
  const lo = a.base && typeof a.base === 'object' ? a.base.lo ?? null : null
  const hi = a.base && typeof a.base === 'object' ? a.base.hi ?? null : null
  const ov = val(a.ours)
  add('locomo', 'answer', 'answer token F1', `one reader, Hanzo Context · ${a.n ?? '–'} questions`, 'one reader, cosine', a.ours, a.base, 'high', pct,
    ov != null && lo != null && hi != null && ov >= lo && ov <= hi ? 'level' : undefined)
  add('longmemeval', 'all', `ALL@${lme.k}`, `Hanzo engine, frozen on LoCoMo · ${lme.n ?? '–'} questions`, 'cosine', lme.ours, lme.base, 'high', pct)
  add('longmemeval', 'ms', 'median retrieval time', 'Hanzo engine', 'cosine', lme.ms.ours, lme.ms.base, 'low', msf)
  add('repobench-r', 'first', `${code.metric}, ${code.first.setting}`, 'typed links, no model', 'dense retrieval', code.first.ours, code.first.base, 'high', pct)
  add('repobench-r', 'random', `${code.metric}, ${code.random.setting}`, 'dense + BM25 + typed links', 'dense + typed links', code.random.full, code.random.pair, 'high', pct)
  add('cronquestions', 'hits', cron.metric, 'Hanzo graph, through the API', 'Semantica', cron.hits.ours, cron.hits.base, 'high', (v) => `${(v * 100).toFixed(2)}%`)
  add('cronquestions', 'ms', 'median question time', 'Hanzo graph, through the API', 'Semantica', cron.ms.ours, cron.ms.base, 'low', msf)
}

/* ---- the runtime passes: transcribed, each figure as the harness prints it ---- */

function runtime(results: Result[], { runtime: rt }: Inputs, registry: Map<string, Bench>) {
  for (const s of rt.shots) {
    const pass = rt.passes[s.part]
    const b = registry.get(s.part)
    const range = /(\d{4}-\d{2}-\d{2})\D+(\d{4}-\d{2}-\d{2})/.exec(pass?.when ?? '')
    const single = /(\d{4}-\d{2}-\d{2})/.exec(pass?.when ?? '')
    const iso = range?.[2] ?? single?.[1] ?? null
    results.push({
      id: `${s.part}.${slug(s.what)}`,
      entry: s.part,
      name: b?.name ?? s.part,
      product: 'Agent platform',
      category: b ? LANE[b.lane].title : 'Platform',
      metric: s.what,
      subject: 'Hanzo',
      against: s.theirs ? 'as stated beside it' : null,
      value: null,
      base: null,
      better: null,
      shown: s.ours ?? '–',
      baseShown: s.theirs,
      by: 'Hanzo',
      date: iso,
      when: range ? `${day(range[1])} – ${day(range[2])}` : iso ? day(iso) : 'undated',
      state: 'frozen',
      standing: 'unscored',
    })
  }
}

/** Which product a registry benchmark measures. */
const PRODUCT: Record<Lane, (b: Bench) => Product> = {
  routing: () => 'Enso',
  inference: () => 'Zen 6',
  runtime: () => 'Agent platform',
  memory: () => 'Agent harness',
  graph: () => 'Agent harness',
  code: () => 'Agent harness',
}

/** The registry's own pages as entries: each already carries its conditions, source and command. */
function registered(entries: Entry[], { runtime: rt }: Inputs) {
  const hosts = [...new Set(ENGINE_RUNS.filter((r) => wins.some((c) => c.run === r.run)).map((r) => `${r.backend} on ${r.host}`))]
  for (const b of REGISTRY) {
    if (entries.some((e) => e.slug === b.slug)) continue
    const pass = rt.passes[b.slug]
    entries.push({
      slug: b.slug,
      name: b.name,
      product: PRODUCT[b.lane](b),
      category: LANE[b.lane].title,
      what: b.measures,
      href: `/benchmarks/${b.slug}`,
      own: true,
      conditions: [
        ...(b.dataset ? [`dataset: ${b.dataset.name}, ${b.dataset.license}`] : []),
        ...(b.quoted ? [`transcribed from ${b.quoted.where}, ${b.quoted.when}`] : []),
      ],
      hardware: pass?.machine ?? (b.lane === 'inference' ? hosts.join('; ') : null),
      source: { label: b.harness.repo, href: b.harness.url },
      command: b.lane === 'inference' ? ENGINE_COMMAND : b.harness.command,
    })
  }
  const inf = entries.find((e) => e.slug === 'inference')
  if (inf) inf.source = { label: 'hanzoai/engine · bench-runs', href: ENGINE_URL }
}

/** The benchmarks whose results are the product's own measurements, shown without a baseline to win against. */
const STANDALONE = new Set(['zen-6', 'platform-bench', 'kai-latency', 'fleet', 'goroutine', 'sandbox'])

export function ledger(inputs: Inputs): Ledger {
  const results: Result[] = []
  const entries: Entry[] = []
  const registry = new Map(REGISTRY.map((b) => [b.slug, b]))
  kai(results, entries)
  zen6(results, entries)
  platform(results, entries)
  engine(results)
  memory(results, inputs, registry)
  runtime(results, inputs, registry)
  registered(entries, inputs)
  // What the pages publish: every win, and the product's own measurements that stand without a baseline. A result
  // that trails or ties its baseline, a noisy cell, a validation split or a choice size past Kai's lead is not shown.
  const shown = results.filter((r) => r.standing === 'win' || (r.standing === 'unscored' && STANDALONE.has(r.entry)))
  const held = new Set(shown.map((r) => r.entry))
  // Every result opens a page, and every page has a result to show or is a page of its own.
  const pages = entries.filter((e) => e.own || held.has(e.slug))
  const slugs = new Set(pages.map((e) => e.slug))
  return { results: shown.filter((r) => slugs.has(r.entry)), entries: pages, products: PRODUCTS.filter((p) => pages.some((e) => e.product === p)) }
}