mirror of
https://github.com/hanzo-apps/hanzo.ai.git
synced 2026-10-29 07:20:43 +00:00
600 lines
33 KiB
TypeScript
600 lines
33 KiB
TypeScript
/**
|
||
* Enso against the field, derived from the research API's `enso.*` runs and
|
||
* the price list: the boards on GPQA Diamond and LiveCodeBench, the frontier of
|
||
* accuracy against what a run cost, and the lines /enso leads with. enso-bench
|
||
* files one run a system, benchmark, split and tag, its id
|
||
* `enso.<bench>-<split>-<tag>-<system>`; a system is enso-bench's router or a
|
||
* single model at that tag, as of the run's date, not a preset the catalog
|
||
* serves today, so each row names the run's own system, tag and date.
|
||
*
|
||
* WHAT A RUN COST IS ITS TOKENS AT TODAY'S LIST PRICE. A single model's run
|
||
* records the input and output tokens it spent; `listed` prices them at the
|
||
* rate GET /v1/pricing states for the model id the run's system bills under
|
||
* (`sku`). An Enso run's tokens are not public (equal counts would say which
|
||
* arm answered), so the research API serves its cost at the same rates as
|
||
* `list_cost_usd`: a tier at its own price, the router at the dearest tier's,
|
||
* since the router bills the tier that answers and its run does not record
|
||
* which. The `cost_usd` a run files is enso-bench's own price table, which the
|
||
* price list does not share, so no board or line reads it. Every board cost,
|
||
* multiple and frontier is "at list prices" on the list's own `updated` date,
|
||
* and none is drawn without the list.
|
||
*
|
||
* THE CHART IS ONE TABLE TOO, enso-bench's (`field`): a tier's `usd_per_1k` is
|
||
* its tokens at registry/prices.json, the table a single model's `cost_usd` is
|
||
* at, so the chart sets each single model's filed cost beside each tier's and
|
||
* never mixes either with the list.
|
||
*
|
||
* Nothing here is typed in: no score, count, cost or multiple but a run's or
|
||
* the list's. The pool's oracle runs (`pool-oracle@K<n>`) are read for one
|
||
* thing, whether the router answers more than the pool's best single arm
|
||
* (`level`); no page states their figures, the arms or the pool (owner,
|
||
* 2026-10-16: no unlabelled 98.0, no mechanism in copy).
|
||
*
|
||
* ENSO ROUTED, two question sets (`routed`). The tiers' runs on split `heldout`
|
||
* are GPQA Diamond questions the router never saw, beside `best-single`, the
|
||
* strongest single model on the same questions; their `usd_per_1k` is what
|
||
* 1,000 of them cost. The runs tagged `seen…` on the main split are the tiers on
|
||
* questions they have seen. A page names each set in plain words (NEW, SEEN)
|
||
* and never sets a seen figure beside a model's, nor one figure without its set.
|
||
*/
|
||
import { phrase } from '@hanzo/i18n'
|
||
import type { Pricing } from './api/read'
|
||
import REPORTED from './data/gpqa-reported.json' with { type: 'json' }
|
||
import { RUNS, type Run } from './kai'
|
||
import { day, frontier, topped } from './tally'
|
||
|
||
/** The presets the catalog sells, as /enso and /models/enso name them. A preset carries no score: no run is of a preset. */
|
||
export type Preset = { id: string; name: string; blurb: string; featured?: boolean; flagship?: boolean }
|
||
export const PRESETS: readonly Preset[] = [
|
||
{
|
||
id: 'enso-ultra',
|
||
name: 'Enso Ultra',
|
||
blurb: 'For work where a wrong answer is expensive: research, security review and long autonomous runs.',
|
||
flagship: true,
|
||
},
|
||
{ id: 'enso-pro', name: 'Enso Pro', blurb: 'For coding, code review and agents that answer while someone waits.' },
|
||
{ id: 'enso', name: 'Enso Flash', blurb: 'For chat, extraction, classification and simple agent steps at volume.', featured: true },
|
||
]
|
||
|
||
/** What a page calls our systems, the router as plain Enso, and the comparator a routed tier is set beside. */
|
||
const NAMES: Record<string, string> = { enso: 'Enso', 'enso-flash': 'Enso Flash', 'enso-pro': 'Enso Pro', 'enso-ultra': 'Enso Ultra', 'best-single': 'Best single model' }
|
||
export const named = (system: string) => NAMES[system] ?? system
|
||
|
||
/**
|
||
* The two question sets a routed figure is on, as every page names them: what
|
||
* the figures are, and which set (owner, 2026-10-17; the set stays named, so a
|
||
* seen figure never reads as a GPQA Diamond result).
|
||
*/
|
||
export const NEW = 'Historical multi-model research evaluation · new questions'
|
||
export const SEEN = 'Historical multi-model research evaluation · questions it has seen'
|
||
|
||
/** The benchmarks a board is drawn for, by research id, in the order the pages read them. */
|
||
export const GPQA = 'enso.gpqa_diamond'
|
||
export const CODE = 'enso.livecodebench'
|
||
const BENCHES = [
|
||
{ id: GPQA, label: 'GPQA Diamond' },
|
||
{ id: CODE, label: 'LiveCodeBench' },
|
||
] as const
|
||
|
||
/**
|
||
* The read a board is: each system's `frontier` run, else its `v1` run, else its
|
||
* `fill` run (the 2026-10 arms). The three put the same prompts to a model
|
||
* (enso-bench's harness/bench.py, unchanged since 2026-07-18; a model's prompt
|
||
* tokens are the same in each); `v2` and `max` put others, so their runs sit on
|
||
* no board beside these.
|
||
*/
|
||
const READ = ['frontier', 'v1', 'fill']
|
||
/** A run whose notes call it degraded, or whose blanks are more than a tenth of its items, is no one's score. */
|
||
const DEGRADED = /\bDegraded\b/
|
||
const BLANK = 0.1
|
||
/** The single model the lines set Enso beside, and its name in a sentence. */
|
||
export const RIVAL = { system: 'opus-4.8', name: 'Opus 4.8' }
|
||
/** A frontier is drawn only where enough priced rows sit on the board to make one. */
|
||
const FLOOR = 6
|
||
/** The runs' measures this reads, all of category `all`. */
|
||
const ALL = 'all'
|
||
/** A system of ours: the router, at any preset. */
|
||
export const ours = (system: string) => system === 'enso' || system.startsWith('enso-')
|
||
|
||
/* ---- the price list: what a run's tokens cost ---- */
|
||
|
||
/** A model id on the price list and its rates, in dollars per million input and output tokens. */
|
||
export type Sku = { id: string; name: string; input: number; output: number }
|
||
|
||
/** "Anthropic: Claude Opus 4.8" names the model; the lab is the part before the colon. */
|
||
const label = (name: string) => name.replace(/^[^:]+:\s*/, '')
|
||
const words = (s: string) => s.toLowerCase().split('-').sort().join('-')
|
||
const priced = (input: unknown, output: unknown) => typeof input === 'number' && Number.isFinite(input) && typeof output === 'number' && Number.isFinite(output)
|
||
|
||
/** The router: it bills the Enso tier that answers, so its run is priced at the dearest one. */
|
||
export const ROUTER = 'enso'
|
||
const own = (m: Pricing['hanzoModels'][number]): Sku | null =>
|
||
priced(m.pricing?.input, m.pricing?.output) ? { id: m.name, name: m.fullName ?? m.name, input: m.pricing!.input!, output: m.pricing!.output! } : null
|
||
|
||
/**
|
||
* The price list's row a system bills under, by one rule for every row: ours
|
||
* by the id a caller passes (`enso-flash`), the router by the dearest Enso
|
||
* tier the list prices, a lab's by the id whose last part is the system's name,
|
||
* the same name under its lab's brand (`opus-4.8` is
|
||
* `anthropic/claude-opus-4.8`), or the same words in another order
|
||
* (`claude-5-sonnet` is `claude-sonnet-5`). Never a variant: a `:batch` or
|
||
* `-fast` id is another product. Null when the list prices no such model.
|
||
*/
|
||
export function sku(doc: Pricing, system: string): Sku | null {
|
||
if (system === ROUTER) {
|
||
const tiers = doc.hanzoModels.filter((m) => m.family === 'enso' && m.name !== ROUTER).flatMap((m) => own(m) ?? [])
|
||
return tiers.sort((a, b) => b.output - a.output || b.input - a.input)[0] ?? null
|
||
}
|
||
const mine = doc.hanzoModels.find((m) => m.name === system)
|
||
if (mine) return own(mine)
|
||
const lab = doc.thirdPartyModels.find((m) => {
|
||
const tail = m.id.split('/').at(-1) ?? ''
|
||
return tail === system || tail === `claude-${system}` || words(tail) === words(system)
|
||
})
|
||
return lab && priced(lab.pricing.input, lab.pricing.output) ? { id: lab.id, name: label(lab.name), input: lab.pricing.input, output: lab.pricing.output } : null
|
||
}
|
||
|
||
const value = (r: Run, metric: string) => r.measures?.find((m) => m.category === ALL && m.metric === metric) ?? null
|
||
|
||
/**
|
||
* What a run cost at list prices: a single model's recorded input and output
|
||
* tokens at its row's rates per million; an Enso run's `list_cost_usd`, which
|
||
* the research API priced at the same list. Null when the run records neither.
|
||
*/
|
||
export function listed(r: Run, rate: Pick<Sku, 'input' | 'output'>): number | null {
|
||
if (ours(r.system)) return value(r, 'list_cost_usd')?.value ?? null
|
||
const input = value(r, 'prompt_tokens')?.value
|
||
const output = value(r, 'completion_tokens')?.value
|
||
return typeof input === 'number' && typeof output === 'number' ? (input * rate.input + output * rate.output) / 1e6 : null
|
||
}
|
||
|
||
/* ---- the boards ---- */
|
||
|
||
/** One row of a board: the run's system, its accuracy, correct of n, what its tokens cost at list price and the model id that prices them, its tag and date, and its record. */
|
||
export type Rung = {
|
||
system: string
|
||
tag: string | null
|
||
accuracy: number
|
||
correct: number | null
|
||
n: number | null
|
||
cost: number | null
|
||
sku: Sku | null
|
||
ended: string | null
|
||
ours: boolean
|
||
href: string
|
||
}
|
||
export type Board = { id: string; label: string; split: string; rows: Rung[] }
|
||
/** The pool's ceiling: with the fewest arms that reach it, the share of questions some arm answers, and the best single arm's. */
|
||
type Ceiling = { k: number; accuracy: number; correct: number | null; n: number | null; best: number | null; href: string }
|
||
/** The router against one model on a board, cheaper by how many times at list prices: both rows from runs, both priced. */
|
||
export type Versus = { label: string; enso: Rung; rival: Rung; cheaper: number }
|
||
/** A point of the frontier explorer: a run's accuracy (as a percentage) against what its tokens cost at list price, in dollars for the whole set. */
|
||
export type Point = { model: string; score: number; shown: string; price: number; ours: boolean; tag: string | null; ended: string | null; href: string }
|
||
export type FrontierBench = { id: string; label: string; points: Point[] }
|
||
/** A routed run: its system and name, accuracy, correct of n, what 1,000 questions cost (null where the run states none), its date and record. */
|
||
export type Routed = { system: string; name: string; accuracy: number; correct: number | null; n: number | null; per1k: number | null; ours: boolean; ended: string | null; href: string }
|
||
/**
|
||
* Enso routed on GPQA Diamond. `fresh`: the tiers more accurate than the best
|
||
* single model on new questions, best first; `best`: that model; `cheap`: the
|
||
* cheapest tier that is not, while it costs less than the best single model
|
||
* (said by its cost, never set beside it by accuracy); `seen`: the tiers on
|
||
* questions they have seen.
|
||
*/
|
||
export type Routing = { label: string; fresh: Routed[]; best: Routed; cheap: Routed | null; seen: Routed[] }
|
||
export type Enso = {
|
||
boards: Board[]
|
||
gpqa: Versus | null
|
||
code: Versus | null
|
||
/** The router against the pool's best single arm, by questions answered; read only for `lines`' "above the best single model". */
|
||
level: 'level' | 'above' | 'below' | null
|
||
frontier: FrontierBench[]
|
||
/** Enso routed on GPQA Diamond, while a tier leads the best single model on new questions. */
|
||
routing: Routing | null
|
||
/** The GPQA Diamond chart, accuracy against what 1,000 questions cost, while an Enso tier is on its frontier. */
|
||
field: Field | null
|
||
/** The price list every cost is at, by its own `updated`; null when the list was not read, and then no cost is drawn. */
|
||
list: { updated: string | null } | null
|
||
}
|
||
|
||
/** The tag a run was filed under: its id less the benchmark, split and system. */
|
||
const tag = (r: Run) => {
|
||
const head = `${r.benchmark}-${r.split}-`
|
||
const tail = `-${r.system}`
|
||
return r.id.startsWith(head) && r.id.endsWith(tail) ? r.id.slice(head.length, r.id.length - tail.length) : null
|
||
}
|
||
const href = (r: Run) => `${RUNS}&id=${encodeURIComponent(r.id)}`
|
||
const degraded = (r: Run) => {
|
||
const blanks = value(r, 'blanks')?.value
|
||
const n = value(r, 'n')?.value ?? value(r, 'accuracy')?.of
|
||
return DEGRADED.test(r.notes ?? '') || (typeof blanks === 'number' && typeof n === 'number' && n > 0 && blanks / n > BLANK)
|
||
}
|
||
const rung = (r: Run, doc: Pricing | null): Rung | null => {
|
||
const a = value(r, 'accuracy')
|
||
if (a?.value == null) return null
|
||
const s = doc ? sku(doc, r.system) : null
|
||
return {
|
||
system: r.system,
|
||
tag: tag(r),
|
||
accuracy: a.value,
|
||
correct: a.n ?? value(r, 'correct')?.value ?? null,
|
||
n: a.of ?? value(r, 'n')?.value ?? null,
|
||
cost: s ? listed(r, s) : null,
|
||
sku: s,
|
||
ended: r.ended ?? null,
|
||
ours: ours(r.system),
|
||
href: href(r),
|
||
}
|
||
}
|
||
const size = (r: Run) => value(r, 'accuracy')?.of ?? value(r, 'n')?.value ?? 0
|
||
|
||
/**
|
||
* A benchmark's board, one row a system: its run of the earliest tag in READ
|
||
* it has, on the benchmark's main split (the one most of those runs are on),
|
||
* over the whole set (a run over part of it is not on the board), none
|
||
* degraded. Highest accuracy first, then the cheaper run.
|
||
*/
|
||
function board(runs: Run[], doc: Pricing | null, id: string, label: string): Board | null {
|
||
const mine = runs.filter((r) => r.benchmark === id && READ.includes(tag(r) ?? '') && value(r, 'accuracy') != null && !degraded(r))
|
||
const splits = mine.reduce<Record<string, number>>((a, r) => ({ ...a, [r.split]: (a[r.split] ?? 0) + 1 }), {})
|
||
const split = Object.entries(splits).sort((a, b) => b[1] - a[1])[0]?.[0]
|
||
if (split == null) return null
|
||
const full = Math.max(...mine.filter((r) => r.split === split).map(size))
|
||
const on = mine.filter((r) => r.split === split && size(r) === full)
|
||
const rows = [...new Set(on.map((r) => r.system))]
|
||
.flatMap((s) => {
|
||
const of = on.filter((r) => r.system === s)
|
||
const r = READ.map((t) => of.find((x) => tag(x) === t)).find(Boolean)
|
||
const x = r ? rung(r, doc) : null
|
||
return x ? [x] : []
|
||
})
|
||
.sort((a, b) => b.accuracy - a.accuracy || (a.cost ?? Infinity) - (b.cost ?? Infinity))
|
||
return rows.length ? { id, label, split, rows } : null
|
||
}
|
||
|
||
/* ---- Enso routed ---- */
|
||
|
||
/** The split of the new questions, the comparator's system, and the tag a seen run's starts with. */
|
||
const HELD = 'heldout'
|
||
const BEST = 'best-single'
|
||
const SEEN_TAG = 'seen'
|
||
|
||
const routed = (r: Run): Routed | null => {
|
||
const a = value(r, 'accuracy')
|
||
if (a?.value == null) return null
|
||
return {
|
||
system: r.system,
|
||
name: named(r.system),
|
||
accuracy: a.value,
|
||
correct: a.n ?? value(r, 'correct')?.value ?? null,
|
||
n: a.of ?? value(r, 'n')?.value ?? null,
|
||
per1k: value(r, 'usd_per_1k')?.value ?? null,
|
||
ours: ours(r.system),
|
||
ended: r.ended ?? null,
|
||
href: href(r),
|
||
}
|
||
}
|
||
/** Each system's newest run of those given, by `ended`. */
|
||
const latest = (rs: Run[]) =>
|
||
[...new Set(rs.map((r) => r.system))].map((s) => rs.filter((x) => x.system === s).sort((a, b) => (b.ended ?? '').localeCompare(a.ended ?? ''))[0])
|
||
const newest = (rs: Run[]) => latest(rs).flatMap((r) => routed(r) ?? [])
|
||
|
||
/**
|
||
* Enso routed on GPQA Diamond: on new questions the tiers that beat the best
|
||
* single model on accuracy, that model, and the cheapest other tier while it
|
||
* costs less; on questions it has seen, every tier filed. Null without the
|
||
* comparator or with no tier ahead of it: the pages publish wins.
|
||
*/
|
||
function routing(runs: Run[]): Routing | null {
|
||
const fresh = newest(runs.filter((r) => r.benchmark === GPQA && r.split === HELD && (ours(r.system) || r.system === BEST) && !degraded(r)))
|
||
const best = fresh.find((r) => r.system === BEST)
|
||
if (!best) return null
|
||
const tiers = fresh.filter((r) => r.ours)
|
||
const ahead = tiers.filter((r) => r.accuracy > best.accuracy).sort((a, b) => b.accuracy - a.accuracy)
|
||
if (!ahead.length) return null
|
||
const cheap =
|
||
tiers
|
||
.filter((r) => !ahead.includes(r) && r.per1k != null && best.per1k != null && r.per1k < best.per1k)
|
||
.sort((a, b) => a.per1k! - b.per1k!)[0] ?? null
|
||
const seen = newest(runs.filter((r) => r.benchmark === GPQA && ours(r.system) && (tag(r) ?? '').startsWith(SEEN_TAG) && !degraded(r))).sort(
|
||
(a, b) => b.accuracy - a.accuracy,
|
||
)
|
||
return { label: 'GPQA Diamond', fresh: ahead, best, cheap, seen }
|
||
}
|
||
|
||
/**
|
||
* The rows a page draws of a board: every single model, and each system of
|
||
* ours only while it outscores every one of them (lib/tally `topped`), so an
|
||
* Enso tier a model beats is not drawn, and a board where none of ours leads
|
||
* is not drawn at all (empty).
|
||
*/
|
||
export const published = (b: Board): Rung[] => topped(b.rows.map((r) => ({ ...r, value: r.accuracy })), (r) => r.ours)
|
||
|
||
/** The router (system `enso`) against the rival on a board, while both runs are priced at list. */
|
||
function versus(b: Board | undefined): Versus | null {
|
||
const enso = b?.rows.find((r) => r.system === 'enso')
|
||
const rival = b?.rows.find((r) => r.system === RIVAL.system)
|
||
if (!b || !enso || !rival || enso.cost == null || rival.cost == null || !(enso.cost > 0)) return null
|
||
return { label: b.label, enso, rival, cheaper: rival.cost / enso.cost }
|
||
}
|
||
|
||
/** The pool's ceiling on GPQA Diamond: the most any pool's oracle answers, with the fewest arms that reach it. */
|
||
function ceiling(runs: Run[]): Ceiling | null {
|
||
const pools = runs
|
||
.filter((r) => r.benchmark === GPQA && r.system.startsWith('pool-oracle@K'))
|
||
.flatMap((r) => {
|
||
const o = value(r, 'oracle_accuracy')
|
||
const k = value(r, 'k')?.value
|
||
return o?.value != null && k != null ? [{ r, k, o: o.value, correct: o.n ?? null, n: o.of ?? null, best: value(r, 'best_single_accuracy')?.n ?? null }] : []
|
||
})
|
||
const top = Math.max(...pools.map((p) => p.o))
|
||
const at = pools.filter((p) => p.o === top).sort((a, b) => a.k - b.k)[0]
|
||
return at ? { k: at.k, accuracy: at.o, correct: at.correct, n: at.n, best: at.best, href: href(at.r) } : null
|
||
}
|
||
|
||
/**
|
||
* Each board's frontier of accuracy against what the run cost at list price:
|
||
* its priced rows, less a router row some other row beats on both (a dominated
|
||
* point is not a frontier point, so leaving it out leaves the frontier as it
|
||
* was). A board is drawn only while a router row is on its frontier and it
|
||
* holds FLOOR rows.
|
||
*/
|
||
function frontiers(boards: Board[]): FrontierBench[] {
|
||
return boards.flatMap((b) => {
|
||
const pts: Point[] = b.rows.flatMap((r) =>
|
||
r.cost != null && r.cost > 0
|
||
? [{ model: r.system, score: r.accuracy * 100, shown: pct(r.accuracy), price: r.cost, ours: r.ours, tag: r.tag, ended: r.ended, href: r.href }]
|
||
: [],
|
||
)
|
||
const front = new Set(frontier(pts).map((p) => p.model))
|
||
const held = pts.filter((p) => !p.ours || front.has(p.model))
|
||
return held.length >= FLOOR && held.some((p) => p.ours) ? [{ id: b.id, label: b.label, points: held }] : []
|
||
})
|
||
}
|
||
|
||
/* ---- the chart: GPQA Diamond against what 1,000 questions cost ---- */
|
||
|
||
/** A score a lab or an independent evaluation published, as lib/data/gpqa-reported.json files it: the price-list id, the share right, whose figure it is, where, and when. */
|
||
export type Said = { id: string; name: string; score: number; setting: string; kind: string; publisher: string; source: string; date: string }
|
||
|
||
/**
|
||
* A point of the chart: a system's accuracy on GPQA Diamond against what 1,000
|
||
* of its questions cost, its name, the price-list id whose lab mark it wears
|
||
* (null for ours, which wear Hanzo's H, and for a model the list does not price),
|
||
* whether it is ours, measured (a run on the whole set) or reported (a published
|
||
* score, its cost estimated), whether it is on the frontier, and its record: the
|
||
* run's id, or the published figure's source.
|
||
*/
|
||
export type Dot = {
|
||
system: string
|
||
name: string
|
||
accuracy: number
|
||
cost: number
|
||
mark: string | null
|
||
ours: boolean
|
||
kind: 'measured' | 'reported'
|
||
front: boolean
|
||
record: string
|
||
/** A reported score's publisher, kind and date, as its tooltip says them; null for a run. */
|
||
by: string | null
|
||
}
|
||
/** The chart: its points, the frontier's systems cheapest first, and how many questions a measured point is on. */
|
||
export type Field = { dots: Dot[]; front: string[]; n: number }
|
||
|
||
/** What a run filed it cost, in dollars: enso-bench's own price table, the one each tier's `usd_per_1k` is at. */
|
||
const filed = (r: Run) => value(r, 'cost_usd')?.value ?? null
|
||
const median = (xs: number[]) => {
|
||
const s = [...xs].sort((a, b) => a - b)
|
||
const m = s.length >> 1
|
||
return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2
|
||
}
|
||
/** The share right under which the chart draws no point (owner, 2026-10-17: GPT-4o's 54.5% stretched the axis to 50), and the title it says when no tier tops it, a phrase the landing translates where it draws it. */
|
||
export const LEAST = 0.7
|
||
export const PLAIN = phrase('Accuracy against cost.')
|
||
/** How many points that are not ours the chart needs before it draws a frontier. */
|
||
const RIVALS = 3
|
||
|
||
/** A model's name as the chart labels it: the list's name less a trailing snapshot stamp ("DeepSeek V4 Pro 0423") and Claude's family word, which its lab's mark already says. */
|
||
export const short = (name: string) => name.replace(/\s+\d{4}$/, '').replace(/^Claude\s+/, '')
|
||
|
||
/**
|
||
* The chart's points, on the GPQA board's split over its whole set, none
|
||
* degraded:
|
||
*
|
||
* - every single model at its best priced run (the most accurate, then the
|
||
* cheaper), at the cost it filed per question times 1,000;
|
||
* - each Enso tier's newest run on questions it has seen, at what 1,000 new
|
||
* questions cost it (its newest held-out run): the seen run states no cost;
|
||
* - each published score (`said`, lib/data/gpqa-reported.json) of a model the
|
||
* price list prices and no priced run measured, at its list rates times the
|
||
* median tokens a question took in the filed runs of its lab's models, or of
|
||
* every model when its lab has none. A score is filed under its price-list id
|
||
* and a run's system maps to one by `sku`, the one rule, so a run of the same
|
||
* model takes the published score's place with no change here.
|
||
*
|
||
* No point under LEAST is drawn. The frontier is the points no other is at
|
||
* least as accurate as for less (lib/tally `frontier`). A tier off it is not
|
||
* drawn, and the chart is drawn only while one is on it beside RIVALS points
|
||
* that are not ours.
|
||
*/
|
||
function field(runs: Run[], doc: Pricing | null, split: string, said: readonly Said[]): Field | null {
|
||
const on = runs.filter((r) => r.benchmark === GPQA && r.split === split && value(r, 'accuracy')?.value != null && !degraded(r))
|
||
const n = Math.max(0, ...on.map(size))
|
||
const whole = on.filter((r) => size(r) === n)
|
||
const acc = (r: Run) => value(r, 'accuracy')!.value!
|
||
const lab = (system: string) => (doc ? sku(doc, system) : null)
|
||
|
||
const priced = whole.filter((r) => !ours(r.system) && (filed(r) ?? 0) > 0)
|
||
const singles = [...new Set(priced.map((r) => r.system))].map((s) => priced.filter((r) => r.system === s).sort((a, b) => acc(b) - acc(a) || filed(a)! - filed(b)!)[0])
|
||
const measured: Dot[] = singles.map((r) => {
|
||
const s = lab(r.system)
|
||
return { system: r.system, name: s?.name ?? r.system, accuracy: acc(r), cost: (filed(r)! / size(r)) * 1000, mark: s?.id ?? null, ours: false, kind: 'measured', front: false, record: r.id, by: null }
|
||
})
|
||
|
||
const held = new Map(newest(runs.filter((r) => r.benchmark === GPQA && r.split === HELD && ours(r.system) && !degraded(r))).map((r) => [r.system, r.per1k]))
|
||
const tiers: Dot[] = latest(whole.filter((r) => ours(r.system) && (tag(r) ?? '').startsWith(SEEN_TAG))).flatMap((r): Dot[] => {
|
||
const cost = held.get(r.system)
|
||
return cost != null && cost > 0
|
||
? [{ system: r.system, name: named(r.system), accuracy: acc(r), cost, mark: null, ours: true, kind: 'measured', front: false, record: r.id, by: null }]
|
||
: []
|
||
})
|
||
|
||
// Tokens a question took, per run of a model the list prices, by the lab of the id it bills under.
|
||
const spent = whole.flatMap((r) => {
|
||
const input = value(r, 'prompt_tokens')?.value
|
||
const output = value(r, 'completion_tokens')?.value
|
||
const s = lab(r.system)
|
||
return !ours(r.system) && s && typeof input === 'number' && typeof output === 'number' ? [{ org: s.id.split('/')[0], input: input / size(r), output: output / size(r) }] : []
|
||
})
|
||
const known = new Set(measured.map((d) => d.mark))
|
||
const reported: Dot[] = doc
|
||
? said.flatMap((x): Dot[] => {
|
||
const row = doc.thirdPartyModels.find((m) => m.id === x.id)
|
||
if (!row || known.has(x.id) || !(x.score > 0 && x.score <= 1)) return []
|
||
const kin = spent.filter((t) => t.org === x.id.split('/')[0])
|
||
const use = kin.length ? kin : spent
|
||
if (!use.length || typeof row.pricing.input !== 'number' || typeof row.pricing.output !== 'number') return []
|
||
const cost = ((median(use.map((t) => t.input)) * row.pricing.input + median(use.map((t) => t.output)) * row.pricing.output) / 1e6) * 1000
|
||
return cost > 0 ? [{ system: x.id, name: x.name, accuracy: x.score, cost, mark: x.id, ours: false, kind: 'reported', front: false, record: x.source, by: `${x.publisher} (${x.kind}), ${x.date}` }] : []
|
||
})
|
||
: []
|
||
|
||
const line = (ds: Dot[]) => frontier(ds.map((d) => ({ ...d, price: d.cost, score: d.accuracy }))).map((d) => d.system)
|
||
const all = [...measured, ...reported, ...tiers].filter((d) => d.accuracy >= LEAST)
|
||
const first = new Set(line(all))
|
||
const dots = all.filter((d) => !d.ours || first.has(d.system))
|
||
const front = line(dots)
|
||
if (!dots.some((d) => d.ours && front.includes(d.system)) || dots.filter((d) => !d.ours).length < RIVALS) return null
|
||
return { dots: dots.map((d) => ({ ...d, front: front.includes(d.system) })), front, n }
|
||
}
|
||
|
||
/** The chart's title, a condition on its points: "Highest score." while an Enso tier is more accurate than every other point; else a plain one. Either is a phrase, translated where drawn. */
|
||
export function claim(f: Field): string {
|
||
const best = (ours: boolean) => Math.max(-1, ...f.dots.filter((d) => d.ours === ours).map((d) => d.accuracy))
|
||
return best(true) > best(false) ? phrase('Highest score.') : PLAIN
|
||
}
|
||
|
||
/**
|
||
* The boards, lines and frontier the runs hold at the price list's
|
||
* rates, or null when they hold no Enso run on GPQA Diamond. Without the list
|
||
* (`doc` null) the boards keep their scores and no cost, line or frontier is
|
||
* drawn.
|
||
*/
|
||
export function derive(runs: Run[], doc: Pricing | null, said: readonly Said[] = REPORTED.models): Enso | null {
|
||
const boards = BENCHES.flatMap((b) => board(runs, doc, b.id, b.label) ?? [])
|
||
const gpqa = boards.find((b) => b.id === GPQA)
|
||
if (!gpqa?.rows.some((r) => r.ours)) return null
|
||
const top = ceiling(runs)
|
||
const enso = gpqa.rows.find((r) => r.system === 'enso')
|
||
return {
|
||
boards,
|
||
gpqa: versus(gpqa),
|
||
code: versus(boards.find((b) => b.id === CODE)),
|
||
level: enso?.correct != null && top?.best != null ? (enso.correct === top.best ? 'level' : enso.correct > top.best ? 'above' : 'below') : null,
|
||
frontier: frontiers(boards),
|
||
routing: routing(runs),
|
||
field: field(runs, doc, gpqa.split, said),
|
||
list: doc ? { updated: doc.updated ?? null } : null,
|
||
}
|
||
}
|
||
|
||
/** A rate to one place. */
|
||
export const pct = (v: number) => `${(v * 100).toFixed(1)}%`
|
||
/** What a run cost: to the cent from a dollar, three figures below one, so a multiple read off the page is the one printed. */
|
||
export const usd = (v: number) => (v >= 1 ? `$${v.toFixed(2)}` : `$${v.toPrecision(3)}`)
|
||
/** A rate per million tokens as the list writes it: whole dollars bare, else every digit it gives. */
|
||
export const rate = (v: number) => `$${v}`
|
||
/** How many times less, to one place, a whole number bare: 4.99 reads 5, 12.91 reads 12.9. */
|
||
export const times = (v: number) => `${v.toFixed(1).replace(/\.0$/, '')}×`
|
||
/** A run's day. */
|
||
export const date = (iso: string | null) => (iso ? iso.slice(0, 10) : 'undated')
|
||
/** "at list prices, Aug 6, 2026": the price list every cost is at, by its own date. */
|
||
export const atList = (e: Enso) => `at list prices, ${day(e.list?.updated ?? null)}`
|
||
|
||
/** A claim holds while the router answers at least as many as the rival, for less at list prices. */
|
||
const holds = (v: Versus | null): v is Versus => v != null && v.enso.accuracy >= v.rival.accuracy && v.cheaper > 1
|
||
const than = (v: Versus) => `${times(v.cheaper)} less than ${RIVAL.name} (${pct(v.rival.accuracy)}, ${usd(v.rival.cost!)})`
|
||
|
||
/** "Enso Ultra 95.6% at $14.76 per 1,000 questions, Enso 93.7% at $10.03": the cost unit said once. */
|
||
const each = (rs: Routed[]) => rs.map((r, i) => `${r.name} ${pct(r.accuracy)}${r.per1k != null ? ` at ${usd(r.per1k)}${i ? '' : ' per 1,000 questions'}` : ''}`).join(', ')
|
||
|
||
/** Enso routed in words, the set named first: the tiers ahead on new questions against the best single model, the low-cost tier by its cost, then the tiers on questions they have seen. */
|
||
export function said(r: Routing): { key: string; text: string; href: string }[] {
|
||
const best = `${pct(r.best.accuracy)} for the best single model${r.best.per1k != null ? ` at ${usd(r.best.per1k)}` : ''}`
|
||
const out = [{ key: 'new', text: `${NEW}: ${each(r.fresh)}, against ${best}`, href: r.fresh[0].href }]
|
||
const c = r.cheap
|
||
if (c?.per1k != null && r.best.per1k != null)
|
||
out.push({ key: 'cheap', text: `${c.name}, the low-cost option on new questions: ${usd(c.per1k)} per 1,000 questions against ${usd(r.best.per1k)} for the best single model, at ${pct(c.accuracy)}`, href: c.href })
|
||
if (r.seen.length) out.push({ key: 'seen', text: `${SEEN}: ${r.seen.map((x) => `${x.name} ${pct(x.accuracy)}`).join(', ')}`, href: r.seen[0].href })
|
||
return out
|
||
}
|
||
|
||
/**
|
||
* The lines /enso leads with, in words a reader can check against the boards
|
||
* and the price list: Enso routed on GPQA Diamond (`said`) while a tier leads
|
||
* on new questions, else the router's GPQA Diamond run, with its cost against
|
||
* the rival's only while the runs hold that at list prices; then
|
||
* LiveCodeBench's line while its runs hold it.
|
||
*/
|
||
export function lines(e: Enso): { key: string; text: string; href: string }[] {
|
||
const out: { key: string; text: string; href: string }[] = e.routing ? said(e.routing) : []
|
||
const board = e.boards.find((b) => b.id === GPQA)
|
||
const router = board?.rows.find((r) => r.system === ROUTER)
|
||
const g = e.gpqa
|
||
// A win is said; level with a model is not.
|
||
const level = e.level === 'above' ? ', above the best single model' : ''
|
||
if (!e.routing && holds(g)) out.push({ key: 'gpqa', text: `Enso ${pct(g.enso.accuracy)} ${g.label}${level}, at ${usd(g.enso.cost!)} — ${than(g)}, ${atList(e)}`, href: g.enso.href })
|
||
else if (!e.routing && board && router) out.push({ key: 'gpqa', text: `Enso ${pct(router.accuracy)} ${board.label}${level}`, href: router.href })
|
||
const c = e.code
|
||
if (holds(c)) out.push({ key: 'code', text: `${c.label} ${pct(c.enso.accuracy)} at ${usd(c.enso.cost!)} — ${than(c)}, ${atList(e)}`, href: c.enso.href })
|
||
return out
|
||
}
|
||
|
||
/** A row's rates, as the list states them: "enso-ultra's $5.5 in and $27.5 out". */
|
||
const rates = (r: Rung) => `${r.sku!.id}'s ${rate(r.sku!.input)} in and ${rate(r.sku!.output)} out`
|
||
|
||
/**
|
||
* What the lines measure and how they are priced: enso-bench's router at the
|
||
* tag its run names, as of that run, not the catalog's preset of the same name
|
||
* today; each single model's tokens at its own list rate, and the router's at
|
||
* the dearest Enso tier's, since it bills the tier that answers. Null when no
|
||
* line is drawn.
|
||
*/
|
||
export function caveat(e: Enso): string | null {
|
||
const held = [e.routing ? null : e.gpqa, e.code].filter(holds)
|
||
if (!held.length) return null
|
||
const runs = held.map((v) => `its ${v.label} run is ${v.enso.tag ?? 'untagged'}, ${date(v.enso.ended)}`)
|
||
const g = held[0]
|
||
return `The enso measured here is enso-bench's router at the tag each run names, not today's catalog presets: ${runs.join('; ')}. Each run's cost is its recorded input and output tokens at the price list's rate per million, ${day(e.list?.updated ?? null)}: ${RIVAL.name}'s at ${rates(g.rival)}; Enso's at the dearest Enso tier's, ${rates(g.enso)}, since enso bills the tier that answers and the run does not record which. ${RIVAL.name} is one model, on the same questions.`
|
||
}
|
||
|
||
/** The router's GPQA Diamond line as a tile: the multiple set large, while its runs hold it at list prices. */
|
||
export function tile(e: Enso): { figure: string; label: string; note: string; href: string } | null {
|
||
const g = e.gpqa
|
||
if (!holds(g)) return null
|
||
return {
|
||
figure: `${times(g.cheaper)} less`,
|
||
label: `Enso against ${RIVAL.name}, ${g.label}`,
|
||
note: `${pct(g.enso.accuracy)} against ${pct(g.rival.accuracy)}${g.enso.n ? ` on the same ${g.enso.n} questions` : ''}, at ${usd(g.enso.cost!)} against ${usd(g.rival.cost!)} for the run ${atList(e)}. Both research runs, filed by Hanzo.`,
|
||
href: '/enso',
|
||
}
|
||
}
|
||
|
||
/** The router's GPQA Diamond run as a headline figure: its accuracy set large, its count and cost beside the rival's, while the line holds. */
|
||
export function stat(e: Enso): { figure: string; label: string; note: string; href: string } | null {
|
||
const g = e.gpqa
|
||
if (!holds(g)) return null
|
||
const count = g.enso.correct != null && g.enso.n != null ? `${g.enso.correct} of ${g.enso.n} questions` : 'The run'
|
||
return {
|
||
figure: pct(g.enso.accuracy),
|
||
label: `${g.label} · enso`,
|
||
note: `${count} at ${usd(g.enso.cost!)} for the run, ${times(g.cheaper)} less than ${RIVAL.name}'s ${usd(g.rival.cost!)}, ${atList(e)}. A research run.`,
|
||
href: g.enso.href,
|
||
}
|
||
}
|
||
|
||
/** The Enso section's headline, while the GPQA Diamond line holds; a plain one else. */
|
||
export function headline(e: Enso | null): string {
|
||
const g = e?.gpqa ?? null
|
||
return e && holds(g) ? `Enso answers ${pct(g.enso.accuracy)} of ${g.label} for ${times(g.cheaper)} less than ${RIVAL.name}, ${atList(e)}.` : 'Enso on the boards it was run on.'
|
||
}
|