hanzo.ai/lib/kai.ts

/**
 * Kai against Jev, derived from the research API's runs. Only Kai's and Jev's
 * runs are read; the answer's runs of any other system are not.
 *
 * `GET /v1/research/runs?org=hanzo` is the one data path: anonymous, and the
 * public runs only. `bench post` in hanzo-inc/kai writes the runs;
 * `node scripts/kai.mjs` saves the answer as lib/data/benchmarks-kai.json,
 * which the pages render first; `useKai` (hooks/useKai) fetches the same
 * address in the browser, so a newly posted revision shows without a rebuild.
 *
 * One revision per page: the newest Kai version with harness runs. Every Kai
 * number is a run of exactly that version, so a board whose runs are not of it
 * is not drawn, and each board names the device its runs were measured on.
 * Latency is drawn only from a CPU run, the device production answers on. A
 * claim is a condition on the data: it shows only while the numbers under it
 * say what its label says, and the release-gate headline only when the gate
 * against Jev passes. No sentence here is a superlative its runs do not hold.
 *
 * The pages publish wins only (owner, 2026-10-16), and this is where that is
 * decided, on the server and again in the browser: a row reaches a page only
 * while Kai leads Jev on it — a task of the Decision Suite, a suite of the
 * frozen harness, a language of the multilingual suite, a calibration pair
 * where Kai's error is the lower, a choice size where Kai is ahead. A row Kai
 * does not lead is not drawn, counted or named, and a count is of the rows
 * drawn, never "N of M"; the one total said is every one of them, the
 * Decision Suite's "16 of 16" while Kai wins all its tasks. The runs
 * themselves stay whole at the API; only what a page draws is chosen here.
 * Nothing else derive reads leaves this file.
 *
 * The Decision Suite leads both pages: tasks Hanzo fixed, each a figure Kai
 * and Jev were both measured on, each row saying where its figures come from
 * and how to re-run it (`DECISION`). Each row names a benchmark and reads
 * Kai's run of it at the revision a page names, the run its board reads,
 * beside Jev's newest recorded run; its price row is the price list, which the
 * page reads when it opens (`table`), and the board draws the tasks Kai wins.
 */
import { phrase } from '@hanzo/i18n'

export const RUNS = 'https://api.hanzo.ai/v1/research/runs?org=hanzo'

/** What /benchmarks says, in words, while Kai's consequence reading is retrained. */
export const UNPUBLISHED =
  'Consequence reading is being retrained and validated end to end; no safety result is published until a sealed, independently authored evaluation reads it. Nio policy profiles (experimental) decide only from consequences a caller states, and escalate to a person when they cannot be counted.'

/** One reported number of a run, as the API returns it. */
export type Measure = { category: string; metric: string; value: number; n?: number | null; of?: number | null }
/** One execution: a system at a version on one benchmark, as the API returns it. */
export type Run = {
  id: string
  benchmark: string
  split: string
  system: string
  version: string
  baseline: boolean
  commit: string
  ended: string
  by?: string
  notes: string
  measures: Measure[] | null
}

export const WHO = ['kai', 'jev'] as const
export type Who = (typeof WHO)[number]
const NAMES: Record<Who, string> = { kai: 'Kai', jev: 'Jev' }
const HEADS = WHO.map((w) => NAMES[w])

type Figure = number | string | null
/** A printed cell, and whether it is its row's best. */
export type Cell = { who: Who; text: string; best: boolean }
/**
 * What a row says under its label, where a board says more than its figures:
 * the public data it was measured on, a second read in words, and where the
 * figures come from and how to re-run them. A link without `href` is a file
 * named but not public.
 */
export type About = { data: Link[]; also: string | null; links: Link[] }
export type Link = { text: string; href?: string }
export type Line = { label: string; cells: Cell[]; about?: About }
export type Block = { title: string | null; lines: Line[] }
/** Where a board's numbers are: the run or harness commit they were recorded at, and the checkpoint. `lead` is the word before the link, `Source` unless it says. */
export type Cite = { text: string; href: string; lead?: string }
/** One table: the label column's head, column groups of `heads`, blocks of rows; `span` counts every column. `lede` is the line under its title. */
export type Board = { id: string; title: string; lede?: string; first: string; groups: string[]; heads: string[]; span: number; blocks: Block[]; note: string; cite: Cite }
/** A headline figure, the board that holds the row it comes from, and the sentence it lends the headline. */
export type Claim = { figure: string; unit?: string; label: string; note: string; board: string; lede?: string }
/**
 * One suite of an index, Kai beside Jev on it: accuracy and calibration as rates,
 * and where the figures come from — a research run, or (`board`) the sealed
 * board's count the index carries in place of a run.
 */
export type IndexRow = {
  label: string; kai: number | null; jev: number | null; kaiEce: number | null; jevEce: number | null; from: 'run' | 'board'
  /** When each side's run ended, as the run records it; null for a count from the sealed board, which records no date. */
  kaiEnded: string | null; jevEnded: string | null
}
/** The multilingual suite language by language: how many languages Kai is ahead of Jev on. */
export type Languages = { ahead: number }
/** The harness suites Kai leads, the set the pages publish (see derive), when its runs ended, and the language tally. */
export type Indexes = { wins: IndexRow[]; ended: string | null; languages: Languages | null }

/**
 * One task of the Decision Suite: Kai's figure and Jev's, which way is better,
 * how they print, and what the row says under its label. A figure is null
 * until a run holds it; the price row's until the price list is read.
 */
export type Task = {
  key: string
  label: string
  better: 'high' | 'low'
  /** A rate to one place, a rate to two, or dollars. */
  as: 'rate' | 'fine' | 'dollars'
  kai: number | null
  jev: number | null
  /** Its research runs are not yet public: the row prints `pending` and holds the count back. */
  pending: boolean
  /** The day the task joined the suite after it was fixed, or null for a task it was fixed with. */
  added: string | null
  about: About
  /** The cost row's p50s in milliseconds, once its latency runs are public: cached and cold, GPU then CPU, and Jev's hosted call. */
  timing?: { cached: [number, number]; cold: [number, number]; hosted: number }
}
/** The Decision Suite as derived: its tasks in order, the day they were fixed, and how many suites of the frozen harness below it Kai leads. */
export type Suite = { id: string; name: string; fixed: string; rerun: string; tasks: Task[]; led: number }

export type Kai = {
  run: string
  sota: boolean
  headline: { title: string; lede: string }
  who: { jev: string }
  claims: Claim[]
  /** Each only where this version was measured and Kai leads a row of it: the harness, cardinality, latency, the languages and stability. */
  boards: { harness: Board | null; capability: Board | null; latency: Board | null; languages: Board | null; stability: Board | null }
  /** The suites Kai leads on the frozen harness, every row a research run of `run`. */
  index: Indexes
  /** The Decision Suite, its price row unread (`table` reads it in). */
  suite: Suite
}

/** The first latency row the runs measured: its mode, device, dtype and p50, or null when there is none. */
export function p50(k: Kai): { mode: string; device: string; dtype: string; ms: string } | null {
  const line = k.boards.latency?.blocks.flatMap((b) => b.lines).find((l) => Number.isFinite(Number.parseFloat(l.cells[2]?.text ?? '')))
  if (!line) return null
  const [device, dtype, ms] = line.cells.map((c) => c.text)
  return { mode: line.label, device, dtype, ms }
}

const DASH = '–'
const real = (v: Figure | undefined) => (typeof v === 'number' && Number.isFinite(v) ? v : null)
/** A rate: the paper's three places, as a percentage to one. */
const rate = (v: Figure | undefined) => { const x = real(v); return x == null ? DASH : `${(x * 100).toFixed(1)}%` }
/** A rate to two places, where its source states it so. */
const fine = (v: Figure | undefined) => { const x = real(v); return x == null ? DASH : `${(x * 100).toFixed(2)}%` }
/** Within half a point either way is level: two questions in four hundred, lib/tally's tie. */
const TIE = 0.005
/** A time as bench tex's `ms` prints it: none from 100, one from 10, else two, thousands separated. */
const tex = (v: Figure | undefined) => {
  const x = real(v)
  if (x == null) return DASH
  const p = x >= 100 ? 0 : x >= 10 ? 1 : 2
  return x.toLocaleString('en-US', { minimumFractionDigits: p, maximumFractionDigits: p })
}
const count = (v: number) => v.toLocaleString('en-US')
/** "A, B and C". */
const and = (xs: string[]) => (xs.length < 2 ? (xs[0] ?? '') : `${xs.slice(0, -1).join(', ')} and ${xs[xs.length - 1]}`)

type Better = 'high' | 'low' | null

/**
 * A group of cells, one per backend, and which of them is best. `rank` is what
 * is compared when the printed text is not a number. No mark on a tie of all.
 */
function cells(vs: Figure[], fmt: (v: Figure) => string, better: Better, rank: (number | null)[] = vs.map(real)): Cell[] {
  const got = rank.filter((v): v is number => v != null)
  const top = !better || got.length < 2 ? null : better === 'high' ? Math.max(...got) : Math.min(...got)
  const tie = top != null && got.every((v) => v === top)
  return vs.map((v, i) => ({ who: WHO[i], text: fmt(v), best: top != null && !tie && rank[i] === top }))
}

/** The harness suites the pages read, in bench tex's order. A dataset's name stands as written; a label that says what the task is, is a phrase a page may translate where it draws it. */
export const SUITES: [string, string][] = [
  ['jev.ag_news', 'AG News'],
  ['jev.emotion', 'DAIR Emotion'],
  ['jev.banking77_full', 'Banking77'],
  ['app.support_triage', phrase('Support triage')],
  ['app.email_spam', phrase('Email spam')],
  ['app.phishing', phrase('Phishing')],
  ['app.moderation_toxicity', phrase('Toxicity')],
  ['app.rag_relevance', phrase('RAG relevance')],
  ['app.model_routing_domain', phrase('Model routing')],
  ['typed_decisions', phrase('Typed decisions')],
]
export const MASSIVE = 'massive'
/**
 * The attack question as this file names it, each name once: the public
 * prompts the Decision Suite reads false alarms on, the folder that re-runs
 * that read, and the benchmarks it reads. A page prints these only in that
 * row, which Kai leads, and e2e/gates/wins.spec.ts cuts exactly these, and
 * the runs the row links, before it looks for the word.
 */
export const ATTACK = {
  data: 'TrustAIRLab/in-the-wild-jailbreak-prompts',
  rerun: 'decision/jailbreak-suite',
  wild: 'jailbreak.wild',
  sealed: 'jailbreak.guard3-sealed',
} as const

/* ---- the Decision Suite: what it holds, which runs each row reads, where it is re-run ---- */

/** The public repository every row is re-run from. */
const REPO = 'https://github.com/hanzoai/benchmarks'
/** A folder of that repository. */
const rerun = (dir: string): Link => ({ text: 're-run', href: `${REPO}/tree/main/${dir}` })
/** A public dataset, by its Hugging Face id. */
const hf = (id: string): Link => ({ text: id, href: `https://huggingface.co/datasets/${id}` })
/** The suite: its name, its anchor, the day its first tasks were fixed, and the page that re-runs each row in one command. */
export const DECISION = { name: 'Decision Suite', id: 'decision-suite', fixed: '2026-10-15', rerun: `${REPO}/blob/main/decision/SUITE.md` } as const

/**
 * A task as the suite fixes it: the benchmark whose runs it reads, Kai's at the
 * page's revision and Jev's newest, and the measure (`category`, `all` unless it
 * says, and `metric`), or none for the cost, which the page reads from the
 * model list. `cases` says what was put to both models; `counted` names what a
 * run's counts count (`Flagged: Kai 24 of 600`); `also` is a second read the
 * row states in words while the revision has one, `say` naming its set from its
 * size; `timing` the latency runs the cost row states.
 */
type Spec = {
  key: string
  label: string
  better: 'high' | 'low'
  as: Task['as']
  runs: string | 'price'
  category?: string
  metric: string
  data: Link[]
  cases?: string
  counted?: string
  also?: { runs: string; category?: string; metric: string; say: (of: string) => string }
  timing?: Timing
  /** The day the task joined the suite after it was fixed (hanzoai/benchmarks decision/SUITE.md). */
  added?: string
  rerun: Link | null
}
/** A p50 a latency run holds: the run, the measure's category, and what it ran on. */
type Point = { id: string; category: string; on: string }
/** Cost and deployment's timings, each a p50 of its run: a cached decision in process and a cold one, on a GPU and a CPU, and Jev's hosted call. */
type Timing = { cached: [Point, Point]; cold: [Point, Point]; hosted: Point }
/** `bench decide` over kai-1's weights in process (dgx CUDA, evo's CPU at eight threads), and Jev's one question a call over the network (`bench speed`). */
const TIMING: Timing = {
  cached: [
    { id: 'decision.latency-window2-dgx.cuda.f32.q8.tall-kai', category: 'cached/plan-device/cuda-bf16/n1', on: 'GPU' },
    { id: 'decision.latency-window2-evo.cpu.f32.q8.t8-kai', category: 'cached/plan-host/cpu-q8_0/n1', on: 'CPU' },
  ],
  cold: [
    { id: 'decision.latency-window2-dgx.cuda.f32.q8.tall-kai', category: 'cold/tensor/cuda-f32/n1', on: 'GPU' },
    { id: 'decision.latency-window2-evo.cpu.f32.q8.t8-kai', category: 'cold/tensor/cpu-f32-q8_0/n1', on: 'CPU' },
  ],
  hosted: { id: 'decision.latency-network-n1-jev-1.13', category: 'served/network/n1', on: 'Jev' },
}

/** The harness suites the Decision Suite holds, each with the public dataset its questions are drawn from (the frozen harness's builders). */
const DRAWN: Record<string, Link[]> = {
  'jev.ag_news': [hf('fancyzhx/ag_news')],
  'jev.emotion': [hf('dair-ai/emotion')],
  'jev.banking77_full': [hf('mteb/banking77')],
  'app.support_triage': [hf('Tobi-Bueck/customer-support-tickets')],
  'app.email_spam': [hf('SetFit/enron_spam')],
  'app.phishing': [hf('zefang-liu/phishing-email-dataset')],
  'app.moderation_toxicity': [hf('lmsys/toxic-chat')],
  'app.rag_relevance': [hf('microsoft/ms_marco')],
  'app.model_routing_domain': [hf('openai/gsm8k'), hf('google-research-datasets/mbpp'), hf('fancyzhx/ag_news')],
  typed_decisions: [hf('LocalLLaMA/typed-decisions')],
  [MASSIVE]: [hf('mteb/amazon_massive_intent')],
}
/** The choice sizes the suite holds, on Banking77's test cases: the gold intent among its nearest, then among every intent. */
const SIZES: [number, string][] = [
  [16, 'Its test cases, the gold intent and its fifteen nearest as the options.'],
  [77, 'Its test cases, every intent an option.'],
]
/** Where a deployment of one's own is asked for. */
const ENTERPRISE: Link = { text: 'Enterprise', href: '/enterprise' }
/**
 * The suite's tasks in the order they were fixed: eleven harness suites, two
 * choice sizes, a repeated request, false alarms on real benign prompts, cost
 * and deployment. `massive` is the
 * multilingual suite's label as the harness names it.
 */
function specs(massive: string): Spec[] {
  const frozen = rerun('decision')
  return [
    ...SUITES.map(([bench, label]): Spec => ({ key: bench, label, better: 'high', as: 'rate', runs: bench, metric: 'accuracy', data: DRAWN[bench], rerun: frozen })),
    { key: MASSIVE, label: `${massive}, macro`, better: 'high', as: 'rate', runs: MASSIVE, metric: 'macro_accuracy', data: DRAWN[MASSIVE], rerun: frozen },
    ...SIZES.map(([k, cases]): Spec => ({
      key: `cap.cardinality.k${k}`,
      label: `Choice among ${count(k)} options`,
      better: 'high',
      as: 'rate',
      runs: 'cap.cardinality',
      metric: `k${k}.accall`,
      data: [hf('mteb/banking77')],
      cases,
      rerun: rerun('decision/capability'),
    })),
    {
      key: 'decision.stability',
      label: phrase('Same request three times, identical answers'),
      better: 'high',
      as: 'rate',
      runs: 'decision.stability',
      metric: 'identical',
      data: [{ text: 'support, cancellation and severity requests the suite generates' }],
      counted: 'Identical',
      rerun: rerun('decision/stability'),
    },
    {
      key: 'decision.alarms',
      label: phrase('False alarms on real benign prompts (in the wild), lower is better'),
      better: 'low',
      as: 'rate',
      // The public suite's question H over in-the-wild's regular prompts; then guard-3's sealed set of real human prompts.
      runs: ATTACK.wild,
      category: 'benign',
      metric: 'false_alarm_rate',
      also: {
        runs: ATTACK.sealed,
        category: 'real_benign',
        metric: 'false_alarm_rate',
        say: (of) => `On a second, sealed set of ${of} real human prompts`,
      },
      data: [hf(ATTACK.data)],
      cases: 'Its regular prompts, each asked whether it is an attack.',
      counted: 'Flagged',
      rerun: rerun(ATTACK.rerun),
    },
    {
      key: 'cost',
      label: 'Cost & deployment, per million input tokens',
      better: 'low',
      as: 'dollars',
      runs: 'price',
      metric: 'input_per_million',
      data: [{ text: 'the model list', href: 'https://api.hanzo.ai/v1/models' }, ENTERPRISE],
      timing: TIMING,
      rerun: null,
    },
  ]
}

/** Kai ahead of Jev on a task: past the harness's tie on a rate, at all on a price; null until both figures are in. */
export const ahead = (t: Task): boolean | null => {
  if (t.kai == null || t.jev == null) return null
  const g = t.better === 'high' ? t.kai - t.jev : t.jev - t.kai
  return t.as === 'dollars' ? g > 0 : g >= TIE
}

/** The model list's input rates, dollars per million tokens, for Kai and Jev: what the page reads when it opens. */
export type Prices = { kai: number | null; jev: number | null }
/**
 * The suite drawn: its board, which holds the tasks Kai wins and no other, how
 * many those are, whether every task's figures are in, and the claim a page
 * leads with, which it makes only then. `tasks` is every task as derived, for
 * the code that reads a row by its key; nothing draws a task Kai does not win.
 */
export type Table = { board: Board; tasks: Task[]; won: number; whole: boolean; claim: Claim | null }

/** A p50 as a sentence says it, in milliseconds: a tenth under one, whole above. */
const ms = (v: number) => (v < 1 ? v.toFixed(1) : `${Math.round(v)}`)
/** The largest of 1, 2, 5, 10, 20, 50, … not above x: how many times faster, said as "over". */
const over = (x: number) => {
  const p = 10 ** Math.floor(Math.log10(x))
  return [5, 2, 1].map((m) => m * p).find((v) => v <= x) ?? p
}
/** How much faster a cached decision in process is than a hosted call, at the slower of its two devices. */
export const faster = (t: NonNullable<Task['timing']>) => over(t.hosted / Math.max(...t.cached))
/**
 * What cost and deployment says of each side: the list price, and, once the
 * latency runs are public, a cached and a cold decision in process on one's own
 * hardware against a hosted call. Every figure is a run's p50 or the model
 * list's rate.
 */
function deployment(t: Task): string {
  const time = t.timing
  const [c0, c1] = time?.cached ?? []
  const cold = [...(time?.cold ?? [])].sort((a, b) => a - b)
  const kai =
    `${t.kai != null ? `$${t.kai} / M input tokens on api.hanzo.ai` : 'On api.hanzo.ai'}; on your own hardware (Enterprise)` +
    (time
      ? `: a cached decision in process in ${ms(c0!)} ms (GPU) to ${ms(c1!)} ms (CPU) — over ${faster(time)}× faster than a hosted call's ${Math.round(time.hosted)} ms; ` +
        `a cold decision in ${Math.floor(cold[0])}–${Math.floor(cold[1])} ms (${Math.round(time.hosted / cold[1])}–${Math.round(time.hosted / cold[0])}×).`
      : ', with no network and no per-call fee.')
  const jev = `${t.jev != null ? `$${t.jev} / M; ` : ''}hosted API only${time ? `, p50 ${Math.round(time.hosted)} ms per call` : ''}.`
  return `Kai: ${kai} Jev: ${jev}`
}

/** A count of tasks as a sentence says it. */
const tasksOf = (n: number) => `${count(n)} ${n === 1 ? 'task' : 'tasks'}`
/** The tasks won, out of every task only when that is all of them: "16 of 16", else "15", never "15 of 16". */
const share = (won: number, all: number) => (won === all ? `${count(won)} of ${count(all)}` : count(won))

/**
 * The Decision Suite as a board, its cost row read in from `prices` (null until
 * the model list answers). The board draws the tasks Kai wins and no other: a
 * task Jev leads, or one whose runs are not yet public, is not drawn, counted
 * or named. The count is the tasks drawn, said out of the suite's tasks only
 * when Kai wins every one ("16 of 16"), else bare ("15 tasks"); it is stated
 * only once every task's figures are in, so it never moves once drawn.
 */
export function table(s: Suite, prices: Prices | null): Table {
  const tasks = s.tasks.map((t) => {
    if (t.key !== 'cost') return t
    const priced = { ...t, kai: prices?.kai ?? null, jev: prices?.jev ?? null }
    return { ...priced, about: { ...t.about, also: deployment(priced) } }
  })
  /** The tasks Kai wins: the rows the board draws. */
  const wins = tasks.filter((t) => ahead(t))
  const won = wins.length
  const every = won === tasks.length
  const whole = tasks.every((t) => ahead(t) != null)
  /** The days the drawn tasks joined after the suite was fixed, and how many joined on each. */
  const later = [...new Set(wins.flatMap((t) => (t.added ? [t.added] : [])))].map((d) => `${count(wins.filter((t) => t.added === d).length)} joined on ${d}`)
  const fixed = later.length ? `The tasks were fixed on ${s.fixed}; ${and(later)}.` : `Each task was fixed on ${s.fixed}.`
  /** A price prints as the model list states it: dollars a million tokens, every digit it gives. */
  const PRINT: Record<Task['as'], (v: Figure) => string> = { rate, fine, dollars: (v) => (real(v) == null ? DASH : `$${v}\u00a0/\u00a0M`) }
  return {
    tasks,
    won,
    whole,
    claim:
      whole && won
        ? {
            figure: share(won, tasks.length),
            label: `${s.name} tasks won · Kai vs Jev`,
            note: `${every ? 'Every task of the suite' : 'Each task Kai wins'}, its row linking its runs and how to re-run it.`,
            board: s.id,
            lede: `Kai wins ${every ? `${share(won, tasks.length)} tasks` : tasksOf(won)} of the ${s.name} against Jev.`,
          }
        : null,
    board: {
      id: s.id,
      title: whole ? `${s.name}: Kai wins ${every ? `${share(won, tasks.length)} tasks` : tasksOf(won)} against Jev` : `${s.name}: the tasks Kai wins against Jev`,
      lede: `${fixed}${s.led ? " The frozen harness's suites Kai leads are below." : ''}`,
      first: 'Task',
      groups: [''],
      heads: HEADS,
      span: WHO.length + 1,
      blocks: [{ title: null, lines: wins.map((t) => ({ label: t.label, cells: cells([t.kai, t.jev], PRINT[t.as], t.better), about: t.about })) }],
      note: 'Accuracy unless a row names its measure; the same questions to both models. Each row names its data and links its runs.',
      cite: { lead: 'Re-run any row', text: s.rerun.replace(/^https:\/\/github\.com\//, '').replace('/blob/main/', ' '), href: s.rerun },
    },
  }
}

/** A rate to the whole percent, as a headline says it. */
const percent = (v: number) => `${Math.round(v * 100)}%`

/**
 * What /kai leads with, in the owner's order, each from the suite's own rows
 * and only while they hold it: the count, a cached decision in process against
 * a hosted call, identical answers to a repeated request, false alarms on real
 * traffic, and the price. A row Kai does not win, or one not yet public or
 * read, gives no line; no figure here is one the board does not print.
 */
export function headlines(t: Table): Claim[] {
  const row = (key: string) => t.tasks.find((x) => x.key === key && ahead(x))
  const out: Claim[] = []
  if (t.claim) out.push({ ...t.claim, label: `${DECISION.name} tasks won vs Jev` })
  // The timings are what cost and deployment's row says beside its price, so they lead only while that row is drawn.
  const time = row('cost')?.timing
  if (time) {
    out.push({
      figure: `over ${faster(time)}×`,
      label: `faster per cached decision, in process (${ms(time.cached[0])}–${ms(time.cached[1])} ms vs a hosted call's ${Math.round(time.hosted)} ms)`,
      note: 'Cost & deployment',
      board: DECISION.id,
    })
  }
  const same = row('decision.stability')
  if (same) out.push({ figure: `${percent(same.kai!)} vs ${percent(same.jev!)}`, label: same.kai === 1 ? 'identical answers every time' : 'identical answers to a repeated request', note: same.label, board: DECISION.id })
  const alarms = row('decision.alarms')
  if (alarms && alarms.kai! > 0) {
    out.push({
      figure: `${Math.round((alarms.jev! / alarms.kai!) * 10) / 10}×`,
      label: `fewer false alarms on real traffic (${percent(alarms.kai!)} vs ${percent(alarms.jev!)})`,
      note: alarms.label,
      board: DECISION.id,
    })
  }
  const cost = row('cost')
  if (cost) {
    const times = cost.jev! / cost.kai!
    out.push({
      figure: Math.abs(times - 2) < 0.01 ? 'half' : `${Math.round(times * 10) / 10}× less`,
      label: `the price ($${cost.kai} vs $${cost.jev} / M input tokens)`,
      note: cost.label,
      board: DECISION.id,
    })
  }
  return out
}

/** Kai's serving modes, as the latency rows name them. */
const MODES: Record<string, string> = { cold: 'Cold encode', incremental: 'Incremental encode', cached: 'Cached decision' }
/** The devices a run's notes may name (`device cuda; dtype bf16`). */
const DEVICES: Record<string, string> = { cpu: 'CPU', cuda: 'CUDA', rocm: 'ROCm', vulkan: 'Vulkan', metal: 'Metal' }
/** The device production answers on. */
const SERVED = 'cpu'
const TIMES = ['p50ms', 'p95ms', 'p99ms']
const HARNESS = 'https://github.com/hanzoai/benchmarks/tree'
/** Kai's head scores this many options; a larger choice is shortlisted to it by retrieval first. */
const SHORTLIST = 160

/** The latest-ended run of the list. */
const newest = (rs: Run[]): Run | null =>
  rs.reduce<Run | null>((a, r) => (!a || r.ended > a.ended || (r.ended === a.ended && r.version > a.version) ? r : a), null)
const value = (r: Run | null, category: string, metric: string) =>
  r?.measures?.find((m) => m.category === category && m.metric === metric)?.value ?? null
const stage = (version: string) => { const [s, w] = version.split('@'); return w ? `${s}@${w.slice(0, 8)}` : version }
/** A revision as prose names it: `kai-1`, or `Kai a7` for a stage. */
export const named = (version: string) => { const s = version.split('@')[0]; return /^kai/i.test(s) ? s : `Kai ${s}` }
/** A field of a run's notes, `key value` among `; `-separated parts. */
const field = (r: Run | null, key: string) => (r?.notes ?? '').split('; ').find((p) => p.startsWith(`${key} `))?.slice(key.length + 1).trim() ?? null
/** The device and dtype a run's notes record, as printed: `CUDA BF16`; null when they record none. */
const device = (r: Run | null) => {
  const d = field(r, 'device')
  return d ? [DEVICES[d] ?? d, field(r, 'dtype')?.toUpperCase()].filter(Boolean).join(' ') : null
}
/** Where a run was measured, and whether that is the device production answers on. */
const measured = (r: Run | null) => {
  const on = device(r)
  const host = field(r, 'host') ?? r?.by
  if (!on) return 'The run records no device.'
  return field(r, 'device') === SERVED
    ? `Kai measured on ${on}${host ? ` (${host})` : ''}, the device production answers on.`
    : `Kai measured on ${on}${host ? ` (${host})` : ''}; production answers on CPU.`
}

/** The boards and claims of the newest Kai revision with harness runs, and the Decision Suite's rows, or null when there is no such revision. */
export function derive(runs: Run[]): Kai | null {
  const top = newest(runs.filter((r) => r.system === 'kai' && r.split === 'harness'))
  if (!top) return null
  const V = top.version

  /** Kai's run of `bench` at exactly this version; a baseline's newest run of it. */
  const run = (bench: string, who: Who) =>
    newest(runs.filter((r) => r.benchmark === bench && r.system === who && (who === 'kai' ? r.version === V : r.baseline)))
  /**
   * A board's provenance, off the run it read. The commit says which code
   * produced the number; the date says when, which is the half a reader quoting
   * this elsewhere needs and a sha does not give. A run with no `ended` says
   * nothing rather than guessing. The harness board links the frozen harness at
   * its commit; every other board links the run's own record.
   */
  const cite = (r: Run | null, what: string, harness = false): Cite => ({
    text:
      `${RUNS.replace(/^https:\/\//, '')} · ${what} · Kai ${stage(V)} · ${harness ? 'harness' : 'commit'} ${r?.commit ? r.commit.slice(0, 7) : DASH}` +
      (r?.ended ? ` · measured ${r.ended.slice(0, 10)}` : ''),
    href: harness ? `${HARNESS}/${r?.commit ?? ''}/decision` : `${RUNS}&id=${encodeURIComponent(r?.id ?? '')}`,
  })

  /* the frozen harness */

  const acc = (bench: string, category: string, metric: string) => WHO.map((w) => value(run(bench, w), category, metric))
  const massive = WHO.map((w) => run(MASSIVE, w))
  const langs = (massive[0]?.measures ?? []).map((m) => m.category).filter((c, i, a) => c !== 'all' && a.indexOf(c) === i)
  /** A suite of the frozen harness, as its builders drew it, with both models' runs. */
  const printed = ([bench, label]: [string, string]) => ({ suite: bench, label, acc: acc(bench, 'all', 'accuracy'), ece: acc(bench, 'all', 'ece'), ended: WHO.map((w) => run(bench, w)?.ended ?? null) })
  const multi = { suite: MASSIVE, label: `MASSIVE, ${langs.length || DASH} languages`, acc: acc(MASSIVE, 'all', 'macro_accuracy'), ece: acc(MASSIVE, 'all', 'macro_ece'), ended: massive.map((r) => r?.ended ?? null) }
  /** The suites the pages may publish as wins, each with both models' runs. */
  const SUITE = [...SUITES.map(printed), multi]
  /**
   * What the pages publish: the frozen harness's suites Kai leads, every one a
   * research run of this revision and of Jev on the same questions. A suite joins
   * or leaves with its runs: a rerun that loses one drops it from the count.
   */
  const leads = (vs: (number | null)[], better: 'high' | 'low') =>
    vs.every((v) => v != null) && vs.slice(1).every((v) => (better === 'high' ? vs[0] > v : vs[0] < v))
  /** Kai's accuracy lead on a suite: past two questions of 400 is a win, lib/tally's tie. */
  const gap = (r: { acc: (number | null)[] }) => {
    const [k, j] = r.acc.map(real)
    return k == null || j == null ? null : k - j
  }
  const WON = SUITE.filter((r) => { const g = gap(r); return g != null && g >= TIE })
  /** The mean accuracy over exactly the rows given, MASSIVE's macro among them, only when every one is present. */
  const mean = (rows: { acc: (number | null)[] }[], i: number) => {
    const vs = rows.map((r) => r.acc[i])
    return vs.every((v) => v != null) ? (vs as number[]).reduce((a, b) => a + b, 0) / vs.length : null
  }
  const MEAN = WHO.map((_, i) => mean(WON, i))
  /** The record in a sentence: the suites Kai leads, counted and listed. */
  const RECORD = `Ahead of Jev on ${WON.length} suites: ${WON.map((r) => r.label).join(', ')}.`
  const scored = run(SUITES[0][0], 'kai')

  /** The frozen harness's suites Kai leads and their mean, linking the harness at its commit; none while Kai leads none. */
  const harness: Board | null = WON.length
    ? {
        id: 'kai-harness',
        title: `The frozen harness: Kai leads Jev on ${WON.length} suites`,
        first: 'Suite',
        groups: ['Accuracy'],
        heads: HEADS,
        span: WHO.length + 1,
        blocks: [
          {
            title: null,
            lines: [...WON, { label: `Mean, these ${WON.length} suites`, acc: MEAN }].map((r) => ({ label: r.label, cells: cells(r.acc, rate, 'high') })),
          },
        ],
        note: `The suites of the frozen harness Kai leads, each a research run of ${named(V)} and of Jev on the same questions.`,
        cite: cite(scored, 'harness', true),
      }
    : null

  /* choosing among many options: the cardinality suite at this version */

  const card = WHO.map((w) => run('cap.cardinality', w))
  /** The choice sizes Kai's run measured, read off its metrics. */
  const KS = [
    ...new Set(
      (card[0]?.measures ?? []).flatMap((m) => {
        const k = /^k(\d+)\.accall$/.exec(m.metric)
        return k && m.category === 'all' ? [Number(k[1])] : []
      }),
    ),
  ].sort((a, b) => a - b)
  const options = (k: number) => `${count(k)} options`
  const accall = (k: number) => card.map((r) => value(r, 'all', `k${k}.accall`))
  /** The choice sizes where every option is scored, smallest first, while Kai is ahead of Jev at each: the rows published. */
  const AHEAD = KS.filter((k) => k <= SHORTLIST).reduce<{ ks: number[]; run: boolean }>(
    (a, k) => {
      const [x, y] = accall(k).map(real)
      const ahead = a.run && x != null && y != null && x > y
      return { ks: ahead ? [...a.ks, k] : a.ks, run: ahead }
    },
    { ks: [], run: true },
  ).ks

  const capability: Board | null =
    card[0] && AHEAD.length
      ? {
          id: 'kai-capability',
          title: `Ahead of Jev at ${and(AHEAD.map(count))} options`,
          first: 'Measure',
          groups: [''],
          heads: HEADS,
          span: WHO.length + 1,
          blocks: [{ title: `Every option scored`, lines: AHEAD.map((k) => ({ label: `Accuracy, ${options(k)}`, cells: cells(accall(k), rate, 'high') })) }],
          note: `The same cases to both. ${measured(card[0])}`,
          cite: cite(card[0], 'capability'),
        }
      : null

  /* the languages of the multilingual suite Kai leads, by the harness's tie */

  const LANG = langs
    .map((l) => ({ label: l, acc: WHO.map((_, i) => value(massive[i], l, 'accuracy')) }))
    .filter((r) => { const g = gap(r); return g != null && g >= TIE })
  const TALLY: Languages = { ahead: LANG.length }
  const languages: Board | null = LANG.length
    ? {
        id: 'kai-languages',
        title: `MASSIVE: ahead of Jev on ${TALLY.ahead} ${TALLY.ahead === 1 ? 'language' : 'languages'}`,
        first: 'Language',
        groups: ['Accuracy'],
        heads: HEADS,
        span: WHO.length + 1,
        blocks: [{ title: null, lines: LANG.map((r) => ({ label: r.label, cells: cells(r.acc, rate, 'high') })) }],
        note: `The languages of the multilingual suite Kai leads, each as MASSIVE names it, the same questions to both.`,
        cite: cite(massive[0], 'harness', true),
      }
    : null

  /* stability: the same request three times, a condition of thresholding a decision at all */

  const stab = WHO.map((w) => run('decision.stability', w))
  const IDENT = stab.map((r) => value(r, 'all', 'identical'))
  const stability: Board | null =
    stab[0] && leads(IDENT, 'high')
      ? {
          id: 'kai-stability',
          title: 'The same request, answered three times',
          first: 'Measure',
          groups: [''],
          heads: HEADS,
          span: WHO.length + 1,
          blocks: [{ title: null, lines: [{ label: 'Requests whose three answers are identical', cells: cells(IDENT, rate, 'high') }] }],
          note: `Every request of the decision-hijack suite sent three times to each model; identical when all three agree on the decision and its probabilities. Kai on CPU, the device production answers on; Jev through its API.`,
          cite: cite(stab[0], 'stability'),
        }
      : null

  /* the Decision Suite: each row from its benchmark's runs at this revision, pending until both are public */

  const byId = new Map(runs.map((r) => [r.id, r]))
  /** Kai's run of a benchmark at this revision and Jev's, or null while either is not public. */
  const pair = (bench: string) => {
    const rs = WHO.map((w) => run(bench, w))
    return rs.every((r): r is Run => r != null) ? rs : null
  }
  const figure = (r: Run, category: string | undefined, metric: string) => r.measures?.find((m) => m.category === (category ?? 'all') && m.metric === metric) ?? null
  const link = (r: Run, text: string): Link => ({ text, href: `${RUNS}&id=${encodeURIComponent(r.id)}` })
  /** The cost row's timings: every p50 its runs hold, or none while any run is not public. */
  const timed = (t: Timing) => {
    const points = [...t.cached, ...t.cold, t.hosted]
    const got = points.map((p) => {
      const r = byId.get(p.id)
      return r ? { p, r, v: real(figure(r, p.category, 'p50ms')?.value) } : null
    })
    if (!got.every((g) => g != null && g.v != null)) return null
    const v = got.map((g) => g!.v!)
    const links = got.map((g) => link(g!.r, `${g!.p.on} timing`)).filter((l, i, all) => all.findIndex((x) => x.href === l.href && x.text === l.text) === i)
    return { timing: { cached: [v[0], v[1]] as [number, number], cold: [v[2], v[3]] as [number, number], hosted: v[4] }, links }
  }
  const tasks = specs(multi.label).map((s): Task => {
    const base = { key: s.key, label: s.label, better: s.better, as: s.as, added: s.added ?? null }
    const links = s.rerun ? [s.rerun] : []
    if (s.runs === 'price') {
      const t = s.timing ? timed(s.timing) : null
      return { ...base, kai: null, jev: null, pending: false, timing: t?.timing, about: { data: s.data, also: null, links: [...(t?.links ?? []), ...links] } }
    }
    const one = pair(s.runs)
    if (!one) return { ...base, kai: null, jev: null, pending: true, about: { data: s.data, also: [s.cases, 'Its research run is not yet public.'].filter(Boolean).join(' '), links } }
    const [mk, mj] = one.map((r) => figure(r, s.category, s.metric))
    const tally =
      s.counted && mk?.n != null && mk.of != null && mj?.n != null && mj.of != null
        ? `${s.counted}: Kai ${count(mk.n)} of ${count(mk.of)}, Jev ${count(mj.n)} of ${count(mj.of)}.`
        : null
    const two = s.also ? pair(s.also.runs) : null
    const [sk, sj] = two && s.also ? two.map((r) => figure(r, s.also!.category, s.also!.metric)) : []
    const second = s.also && sk?.of != null && real(sk.value) != null && real(sj?.value) != null ? `${s.also.say(count(sk.of))}: Kai ${fine(sk.value)}, Jev ${fine(sj!.value)}.` : null
    const runsOf = [...one, ...(two ?? [])].map((r, i) =>
      link(r, `${r.system === 'kai' ? named(r.version) : NAMES.jev} run${i < 2 ? '' : ', second read'}`),
    )
    return { ...base, kai: real(mk?.value), jev: real(mj?.value), pending: false, about: { data: s.data, also: [s.cases, tally, second].filter(Boolean).join(' ') || null, links: [...runsOf, ...links] } }
  })
  const decision: Suite = { ...DECISION, tasks, led: WON.length }

  /* latency: one decision, on the device production answers on */

  const lat = run('latency', 'kai')
  const served = [...new Set((lat?.measures ?? []).map((m) => m.category))].filter((c) => c.split('/')[1]?.split('-')[0] === SERVED)
  const latency: Board | null = served.length
    ? {
        id: 'kai-latency',
        title: 'Latency of one decision',
        first: 'Serving mode',
        groups: [''],
        heads: ['Device', 'Dtype', 'p50 ms', 'p95 ms', 'p99 ms'],
        span: 6,
        blocks: [
          {
            title: null,
            lines: served.map((c) => {
              const [mode, tag] = c.split('/')
              const dtype = tag.split('-').slice(1).join('-')
              return {
                label: MODES[mode] ?? mode,
                cells: [DEVICES[SERVED], dtype.toUpperCase(), ...TIMES.map((m) => tex(value(lat, c, m)))].map((text) => ({ who: 'kai' as Who, text, best: false })),
              }
            }),
          },
        ],
        note: `Kai ${stage(V)}, one decision a call, on CPU, the device production answers on${lat?.by ? `, timed on ${lat.by}` : ''}. Nothing here is projected.`,
        cite: cite(lat, 'latency'),
      }
    : null

  /* the claims */

  /** Pass against Jev: the one condition the release headline turns on. */
  const gate = run('gate.jev', 'kai')
  const SOTA = value(gate, 'all', 'pass') === 1
  const GATED = SOTA ? value(gate, 'all', 'suites') : null

  const claims: (false | Claim)[] = [
    WON.length > 0 && {
      figure: `${WON.length}`,
      board: 'kai-harness',
      label: 'Suites won vs Jev',
      note: `Every row a research run of ${named(V)} and of Jev.`,
      lede: RECORD,
    },
    leads(MEAN, 'high') && {
      figure: rate(MEAN[0]),
      board: 'kai-harness',
      label: `Mean accuracy across these ${WON.length} suites`,
      note: `Jev ${rate(MEAN[1])}, the same suites`,
      lede: `${rate(MEAN[0])} mean accuracy across these ${WON.length} suites, against Jev's ${rate(MEAN[1])}.`,
    },
    stability != null && leads(IDENT, 'high') && {
      figure: rate(IDENT[0]),
      board: 'kai-stability',
      label: 'Identical answers to a repeated request',
      note: `Jev ${rate(IDENT[1])}, the same requests sent three times`,
      lede: `${rate(IDENT[0])} of repeated requests answer identically, against Jev's ${rate(IDENT[1])}.`,
    },
    leads(MEAN, 'high') && MEAN[0] != null && MEAN[1] != null && {
      figure: `+${((MEAN[0] - MEAN[1]) * 100).toFixed(1)}`,
      unit: 'pp',
      board: 'kai-harness',
      label: 'Mean accuracy lead vs Jev',
      note: `Percentage points across ${WON.length} suites`,
    },
  ]
  const HELD = claims.filter((c): c is Claim => Boolean(c))
  const jev = run(SUITES[0][0], 'jev')?.version ?? DASH
  /** A published row. Its calibration pair is carried only where Kai's error is the lower, so no row holds a figure Jev leads on. */
  const indexRow = (r: { label: string; acc: (number | null)[]; ece: (number | null)[]; ended: (string | null)[] }): IndexRow => {
    const calibrated = leads(r.ece, 'low')
    return {
      label: r.label,
      kai: real(r.acc[0]),
      jev: real(r.acc[1]),
      kaiEce: calibrated ? real(r.ece[0]) : null,
      jevEce: calibrated ? real(r.ece[1]) : null,
      from: 'run',
      kaiEnded: r.ended[0],
      jevEnded: r.ended[1],
    }
  }

  return {
    run: V,
    sota: SOTA,
    headline: {
      title: SOTA
        ? 'Kai passes the release gate against Jev.'
        : WON.length
          ? `Kai leads Jev on ${WON.length} suites, every one a recorded run.`
          : 'Kai beside Jev, on the same questions.',
      lede: [
        ...(SOTA ? [`No suite of the ${GATED} gated regresses past tolerance against Jev.`] : []),
        ...HELD.flatMap((c) => (c.lede ? [c.lede] : [])),
      ].join(' '),
    },
    who: { jev: `${jev} through OpenRouter` },
    claims: HELD,
    boards: { harness, capability, latency, languages, stability },
    index: { wins: WON.map(indexRow), ended: scored?.ended ?? top.ended ?? null, languages: LANG.length ? TALLY : null },
    suite: decision,
  }
}