hanzo.ai/app/(marketing)/benchmarks/inference/page.tsx

/**
 * /benchmarks/inference — hanzo-engine's prefill on CUDA against llama.cpp.
 *
 * The cells are the harness's own floats from the run directories on branch
 * `bench/dossier` of hanzoai/engine (lib/engine), each with its per-repetition
 * samples, its quiet-gate log and the pinned model digest. The page publishes
 * the cells the harness calls a WIN — the whole 95% interval above parity — with
 * no variance flag on either engine, which are prefill cells on CUDA. Nothing
 * here is typed: a rerun that moves a cell moves the page.
 */
import { T } from '@hanzo/i18n/react'
import { pageMeta } from '@/lib/page-meta'
import { BENCHMARKS, REFUTATION, bench, paperUrl } from '@/lib/benchmarks'
import { BenchJsonLd } from '@/components/benchmarks/JsonLd'
import { Grid, H2, Meta, More, P, Shell, Spec, list, type Col } from '@/components/benchmarks/kit'
import { BRANCH, RUNS, RUNS_URL, name, quant, wins, type Cell } from '@/lib/engine'

const B = bench('inference')!

export const metadata = pageMeta({
  title: B.title,
  description:
    'hanzo-engine against llama.cpp on CUDA, same weights, same box: prefill throughput where the whole 95% interval of the ratio lies above parity.',
  path: '/benchmarks/inference',
})

const MEASURES = ['prefill tokens/s', 'ratio against llama.cpp']

const r3 = (v: number) => v.toFixed(3)
const t1 = (v: number) => v.toFixed(1)

const ROWS = [...wins].sort((a, b) => a.model.localeCompare(b.model) || a.n - b.n)
/** The runs the published cells come from. */
const SHOWN = RUNS.filter((r) => ROWS.some((c) => c.run === r.run))
const REPS = [...new Set(SHOWN.map((r) => r.reps))].sort((a, b) => a - b)
const ENGINES = [...new Set(SHOWN.map((r) => r.engine))]
const models = list([...new Set(ROWS.map((c) => c.model))].map((m) => `${name(m)} at ${quant(m)}`))

const cols: Col<Cell>[] = [
  { head: <T>backend</T>, get: (c) => c.backend, left: true },
  { head: <T>model</T>, get: (c) => name(c.model), left: true },
  { head: <T>quant</T>, get: (c) => quant(c.model), left: true },
  { head: <T>prompt tokens</T>, get: (c) => c.n },
  { head: 'llama.cpp t/s', get: (c) => t1(c.llama) },
  { head: 'hanzo t/s', get: (c) => t1(c.hanzo) },
  { head: <T>ratio</T>, get: (c) => r3(c.ratio) },
  { head: <T>95% interval</T>, get: (c) => `${r3(c.lo)} – ${r3(c.hi)}` },
  { head: <T>verdict</T>, get: (c) => c.verdict, left: true },
]

export default function Page() {
  return (
    <>
      <BenchJsonLd b={B} measures={MEASURES} />
      <Shell b={B}>
        <Spec b={B} />

        <H2><T>What is measured</T></H2>
        <P>
          <T values={{ models, reps: REPS.join(' or ') }}>
            {"hanzo-engine and llama.cpp, built against the same weights, run on the same box, alternating. Prefill is tokens per second at a given prompt length, and the ratio is ours over theirs, so above 1.000 is a win. The models are {models}; {reps} repetitions per cell, as its run records, with the first discarded as warmup, a Student-t 95% interval on each engine, and the ratio's interval formed by adding the two relative intervals in quadrature."}
          </T>
        </P>
        <P>
          <T values={{ win: <strong><T>WIN</T></strong> }}>
            {"The verdict is the harness's, not a reading of ours: {win} where the whole interval lies above 1.000 and neither engine's coefficient of variation exceeded 5%, which is how the harness says the box was quiet. Every cell below is one."}
          </T>
        </P>

        <H2><T>Prefill on CUDA, cell by cell</T></H2>
        <Meta>
          <T values={{ cells: ROWS.length, runs: SHOWN.length, branch: <code>{BRANCH}</code>, engines: ENGINES.join(', ') }}>
            {'{cells} cells from {runs} runs on branch {branch} · engine {engines} · each run pins its engine release, its llama.cpp build and its model digest, and is linked below'}
          </T>
        </Meta>
        <Grid cols={cols} rows={ROWS} keyOf={(c) => `${c.run}-${c.n}`} />

        <H2><T>The work that produced it</T></H2>
        <P>
          <T values={{ paper: <a href={paperUrl(REFUTATION)}>{REFUTATION.title}</a> }}>
            {'{paper} is the campaign that produced these kernels, written as a log of numbered hypotheses, each settled by a cheap decisive experiment. A long prefill is compute-bound, which is where that work landed.'}
          </T>
        </P>

        <More b={B} all={BENCHMARKS} />
        <Meta>
          <T values={{ runs: <a href={RUNS_URL}>bench-runs</a> }}>{'the committed runs, in {runs}:'}</T>{' '}
          {SHOWN.map((r, i) => (
            <span key={r.run}>{i ? ' · ' : ''}{r.backend} {name(r.model)} {quant(r.model)}</span>
          ))}
        </Meta>
      </Shell>
    </>
  )
}