hanzo.ai/lib/tally.ts

/**
 * The counting /benchmarks does, with no data of its own: how a rate is printed,
 * who leads a suite, an index's record, and which points of one benchmark are
 * Pareto-efficient. Shared by lib/board (the build) and the board components
 * (the browser), so the hero and an interactive view can never count differently.
 */
import { named, type Claim, type IndexRow, type Kai } from './kai'
import { rate as billed, type Pricing } from './api/read'

/** A rate as a percentage to one place: 0.9175 reads 91.8%. */
export const rate = (v: number | null | undefined) => (v == null ? '–' : `${(v * 100).toFixed(1)}%`)
/** A gap between two rates in percentage points, signed: +9.0 pp, −2.3 pp. */
export const points = (v: number | null | undefined) =>
  v == null ? '–' : `${v >= 0 ? '+' : '−'}${Math.abs(v * 100).toFixed(1)} pp`
/** A calibration error to three places, as the harness prints it. */
export const ece = (v: number | null | undefined) => (v == null ? '–' : v.toFixed(3))
/** A count as a reader reads it: 1,000. */
export const count = (n: number) => n.toLocaleString('en-US')
/** Dollars at the precision the rate carries: $0.021, $4, $138.60. */
export const dollars = (v: number) =>
  v >= 1 ? (Number.isInteger(v) ? `$${v}` : `$${v.toFixed(2)}`) : `$${v.toFixed(3).replace(/0+$/, '')}`
/** A run's end as a reader of `locale` dates it: 2026-10-03T… reads Oct 3, 2026 in en-US, 3 أكتوبر 2026 in ar. */
export const date = (iso: string | null, locale: string) =>
  iso ? new Date(iso).toLocaleDateString(locale, { month: 'short', day: 'numeric', year: 'numeric', timeZone: 'UTC' }) : '–'
/** A run's end as an English reader dates it: 2026-10-03T… reads Oct 3, 2026. */
export const day = (iso: string | null) => date(iso, 'en-US')

/** Within half a point either way is level: two questions in four hundred, lib/kai's tie. */
export const TIE = 0.005
export type Verdict = 'won' | 'level' | 'behind' | 'unscored'
/** Who leads a suite on accuracy (higher is better) or calibration error (lower is better). */
export const verdict = (r: IndexRow, on: 'acc' | 'ece' = 'acc'): Verdict => {
  const [k, j] = on === 'acc' ? [r.kai, r.jev] : [r.kaiEce, r.jevEce]
  if (k == null || j == null) return 'unscored'
  const g = on === 'acc' ? k - j : j - k
  return g >= (on === 'acc' ? TIE : 0) && g !== 0 ? 'won' : g <= -(on === 'acc' ? TIE : 0) && g !== 0 ? 'behind' : 'level'
}
/** Kai's lead on a suite, positive where Kai is better, in the metric's own units. */
export const lead = (r: IndexRow, on: 'acc' | 'ece' = 'acc') => {
  const [k, j] = on === 'acc' ? [r.kai, r.jev] : [r.kaiEce, r.jevEce]
  return k == null || j == null ? null : on === 'acc' ? k - j : j - k
}

/** The mean over every row, only when every row is scored. */
const mean = (vs: (number | null)[]) => (vs.every((v): v is number => v != null) ? vs.reduce((a, b) => a + b, 0) / vs.length : null)
export type Tally = { won: number; level: number; behind: IndexRow[]; total: number; kai: number | null; jev: number | null }
/** An index's record on accuracy: suites won, level and behind, and both means. */
export const record = (rows: IndexRow[]): Tally => ({
  won: rows.filter((r) => verdict(r) === 'won').length,
  level: rows.filter((r) => verdict(r) === 'level').length,
  behind: rows.filter((r) => verdict(r) === 'behind'),
  total: rows.length,
  kai: mean(rows.map((r) => r.kai)),
  jev: mean(rows.map((r) => r.jev)),
})

/**
 * The non-dominated points of ONE benchmark: sorted by price, each that scores
 * above every cheaper point. Never computed across benchmarks.
 */
export function frontier<P extends { price: number; score: number }>(ps: P[]): P[] {
  const out: P[] = []
  let best = -Infinity
  for (const p of [...ps].sort((a, b) => a.price - b.price || b.score - a.score)) {
    if (p.score > best) {
      out.push(p)
      best = p.score
    }
  }
  return out
}

/**
 * A board printed as a list by score publishes a Hanzo row only while it
 * outscores every other row on the list: a Hanzo row under a rival's is that
 * rival's win, and a page does not print it. Empty when no Hanzo row is left,
 * so the board is not drawn at all. Rows come sorted high to low.
 */
export function topped<R extends { value: number }>(rows: R[], ours: (r: R) => boolean): R[] {
  const rival = Math.max(...rows.filter((r) => !ours(r)).map((r) => r.value), -Infinity)
  const kept = rows.filter((r) => !ours(r) || r.value > rival)
  return kept.some(ours) ? kept : []
}

/** A headline figure: the number set large, what it measures, where it came from, and where it is drawn. */
export type Tile = { figure: string; label: string; note: string; href: string }

/** Jev as the price list names it: the model the Kai harness compares against. */
export const JEV = 'typesafe/jev-1.13'

/**
 * Kai's price beside Jev's, as the price list states both (GET /v1/pricing,
 * read when a page opens, lib/api): how much less an input token costs, and
 * that a decision writes no output to pay for. Each tile shows only while the
 * list says so, and no build holds either figure.
 */
export function kaiPrice(doc: Pricing): Tile[] {
  const kai = billed(doc, 'kai')
  const jev = billed(doc, JEV)
  const out: Tile[] = []
  if (kai?.input != null && jev?.input != null && kai.input < jev.input) {
    out.push({
      figure: `${Math.floor((1 - kai.input / jev.input) * 100)}% less`,
      label: 'Input-token price · Kai vs Jev',
      note: `${dollars(kai.input)} against ${dollars(jev.input)} per million input tokens, as each is billed. A price, not a cost per task.`,
      href: '/pricing',
    })
  }
  if (kai?.output === 0) {
    out.push({
      figure: '$0',
      label: 'Per output token',
      note: 'A decision returns a choice, a score or a yes/no probability, and writes no text, so the price list bills no output.',
      href: '/pricing',
    })
  }
  return out
}

/**
 * Kai's headline tiles, from the suites the pages publish: suites won and the
 * mean lead. Each shows only while the rows hold it, so a rerun that loses the
 * lead drops the tile rather than printing a stale win. `revision` is the Kai
 * revision the rows were measured on, as lib/kai's `named` prints it.
 */
export function kaiTiles(t: Tally, revision: string): Tile[] {
  const out: Tile[] = []
  if (!t.total || !t.won) return out
  out.push({
    figure: `${t.won}`,
    label: `Suites won · Kai vs Jev`,
    note: `Every row a research run of ${revision} and of Jev.`,
    href: '#leaderboard',
  })
  if (t.kai != null && t.jev != null && t.kai > t.jev) {
    out.push({
      figure: points(t.kai - t.jev),
      label: 'Mean accuracy lead · Kai vs Jev',
      note: `${rate(t.kai)} against ${rate(t.jev)} across these ${t.total} suites.`,
      href: '#leaderboard',
    })
  }
  return out
}

/**
 * Kai's one headline figure, the same wherever a page leads with it (the
 * landing's Kai tile, /benchmarks' hero): the Decision Suite's count while every
 * row of it holds both figures (lib/kai `table`'s claim), else the harness
 * suites Kai leads, else none. `href` is the section of /benchmarks that draws
 * its evidence.
 */
export function kaiLead(claim: Claim | null, kai: Kai | null): Tile | null {
  if (claim) return { figure: claim.figure, label: claim.label, note: claim.note, href: `#${claim.board}` }
  const [won] = kai ? kaiTiles(record(kai.index.wins), named(kai.run)) : []
  return won ? { ...won, href: '#kai' } : null
}

/** The Kai section's title, counted from the suites the pages publish. */
export function kaiHeadline(t: Tally): string {
  return t.won ? `Kai leads Jev on ${t.won} suites, every one a recorded run.` : 'Kai beside Jev, on the same questions.'
}

/** What the Kai section says under its number: the suites, by name. */
export function kaiLede(t: Tally, rows: IndexRow[], revision: string): string {
  if (!t.total) return ''
  return `${rows.map((r) => r.label).join(', ')}. Every row a research run of ${revision} and of Jev on the same questions; mean accuracy across these ${t.total} suites ${rate(t.kai)} against ${rate(t.jev)}.`
}