hanzo.ai/lib/engine.ts

/**
 * The engine against llama.cpp, as the evidence has it.
 *
 * `lib/data/engine.json` is what `hanzo-bench publish` files in Hanzo Research for every run
 * on branch `bench/dossier` of hanzoai/engine: one `kernel-perf` experiment per engine per
 * shape, computed from the raw per-repetition samples committed beside each run. Nothing here
 * is retyped from a table; a figure is the harness's own float.
 *
 * The harness's statistics contract, which the verdict column depends on: the first of three
 * or more repetitions discarded as warmup, Student-t two-sided 95% intervals, the ratio's
 * interval formed by adding the two relative intervals in quadrature, and WIN only where the
 * whole interval lies above 1.0. `noisy` is the harness's flag: either engine's coefficient of
 * variation exceeded 5%, which is how it says the box was not quiet.
 */
import filed from './data/engine.json' with { type: 'json' }

export type Verdict = 'WIN' | 'PARITY' | 'LOSS'

export interface Cell {
  run: string
  backend: string
  model: string
  phase: string
  /** Prompt tokens for prefill; generated tokens for decode. */
  n: number
  hanzo: number
  hanzo_ci: number
  hanzo_cv: number
  llama: number
  llama_ci: number
  llama_cv: number
  /** Ours over theirs. Above 1.00 is a win. */
  ratio: number
  ci: number
  lo: number
  hi: number
  verdict: Verdict
  /** The harness flagged one of the two engines' variance. Read the cell as unsettled. */
  noisy: boolean
}

export interface Run {
  run: string
  host: string
  backend: string
  iso: string
  model: string
  model_sha256: string
  engine: string
  engine_sha: string
  llama_sha: string
  reps: number
}

/** One experiment as filed. Ours carries `versus`, the rival's beside it does not. */
interface Experiment {
  subject: string
  task: string
  value: number
  meta: {
    run: string
    host: string
    backend: string
    model: string
    model_sha256: string
    reps: number
    ci95: number | null
    cv_pct: number | null
    noisy: boolean
    versus?: { ratio: number; ci: number; lo: number; hi: number; verdict: Verdict }
  }
  git_sha: string
  lib_versions: Record<string, string>
  ts: number
}

/**
 * A model is named by its GGUF file: family, size, quantization. The site names it by its Zen
 * generation and keeps the quantization beside it, since two lanes of one model at different
 * quantizations are different measurements. The rename happens here, as the evidence is read,
 * so no id, key, label or link the pages build carries the file's family: run directories
 * (`spark-cuda-…-4b-…`) are renamed with the model they measured. A 3.8 build is Zen 6:
 * zenlm/zen6 publishes it.
 */
const FAMILY: [RegExp, string][] = [
  [/Qwen3\.8-/g, 'Zen6-'],
  [/Qwen3-/g, 'Zen-'],
  [/qwen3p8-/g, 'zen6-'],
  [/qwen3-/g, 'zen-'],
]
const zen = (s: string) => FAMILY.reduce((t, [stem, family]) => t.replace(stem, family), s)

const FILED = (filed as { experiments: Experiment[] }).experiments.map((e) => ({
  ...e,
  subject: zen(e.subject),
  meta: { ...e.meta, run: zen(e.meta.run), model: zen(e.meta.model) },
}))
const key = (e: Experiment) => `${e.meta.run}:${e.task}`
const ours = FILED.filter((e) => e.subject.startsWith('hanzo-engine/'))
const theirs = new Map(FILED.filter((e) => e.subject.startsWith('llama.cpp/')).map((e) => [key(e), e]))

const QUANT = /-((?:I?Q\d\w*|ROCmFP4(?:-FAST)?|MXFP4|NVFP4|F16|BF16))$/

export const quant = (model: string) => QUANT.exec(model)?.[1] ?? ''
/** A model's Zen name, its quantization left off. */
export const name = (model: string) => model.replace(QUANT, '')

export const BRANCH = 'bench/dossier'

/** Every shape both engines completed. */
export const CELLS: Cell[] = ours.flatMap((e) => {
  const rival = theirs.get(key(e))
  const v = e.meta.versus
  if (!rival || !v) return []
  const [phase, n] = e.task.split('-')
  return [{
    run: e.meta.run, backend: e.meta.backend, model: e.meta.model, phase, n: Number(n),
    hanzo: e.value, hanzo_ci: e.meta.ci95 ?? NaN, hanzo_cv: e.meta.cv_pct ?? NaN,
    llama: rival.value, llama_ci: rival.meta.ci95 ?? NaN, llama_cv: rival.meta.cv_pct ?? NaN,
    ratio: v.ratio, ci: v.ci, lo: v.lo, hi: v.hi, verdict: v.verdict, noisy: e.meta.noisy,
  }]
})

export const RUNS: Run[] = [...new Map(ours.map((e) => [e.meta.run, e])).values()].map((e) => ({
  run: e.meta.run, host: e.meta.host, backend: e.meta.backend,
  iso: new Date(e.ts * 1000).toISOString(), model: e.meta.model, model_sha256: e.meta.model_sha256,
  engine: e.lib_versions['hanzo-engine'], engine_sha: e.git_sha,
  llama_sha: theirs.get(key(e))?.git_sha ?? '', reps: e.meta.reps,
}))

/** Cells the harness did not flag. A verdict on a flagged cell is not a verdict. */
export const settled = CELLS.filter((c) => !c.noisy)

/** Prefill cells whose whole interval is above parity, the harness agrees, and neither engine was flagged: the cells the pages publish. */
export const wins = settled.filter((c) => c.phase === 'prefill' && c.verdict === 'WIN')

/** The run directories on the bench branch, every run's samples, logs and model digest. */
export const RUNS_URL = `https://github.com/hanzoai/engine/tree/${BRANCH}/bench-runs`

/** The one line that rebuilds a CUDA run directory, in the paper's appendix's form. */
export const COMMAND =
  'scripts/dossier.sh --backend cuda --model <same.gguf> -p 512,500,2048,4096 -n 128 -r 7'