mirror of
https://github.com/hanzo-apps/hanzo.ai.git
synced 2026-10-29 07:20:43 +00:00
zandzeekay
6f8c6e2bdb
/ and /enso draw one chart again, the scatter d0199249 removed, read from the
runs when the page opens (lib/enso `field`): every single model at its best
priced run on the 198 questions, at the cost it filed; Enso Ultra, Enso and
Enso Flash on questions they have seen (194, 191 and 189 of 198), each at its
tier's cost on new questions; and the published scores of GPT-6 Sol, GPT-6
Luna, Kimi K3, GLM 5.3, Gemini 4 Argon and Grok 4.7, which no run has
measured, at their list rates times the median tokens a question took, a
dashed disc linking its source. A run of the same model replaces its
published score with no change to the code. The frontier is a line through
the points no other beats on both; a tier off it is not drawn.
The landing's title is a condition on the points: "Highest score. Cheapest
above 90%." while a tier tops every point and another is the cheapest above
90%, else "Accuracy against cost." The captions under the chart and over the
table are gone; each figure names its set where it stands. /enso says how
to route your own way: pinned, by policy, or trained on your own traffic.
141 lines
7.8 KiB
TypeScript
141 lines
7.8 KiB
TypeScript
import type { Run } from '../../lib/kai'
|
|
import { PRICING } from './api'
|
|
|
|
/**
|
|
* Invented `enso.*` runs, in the shape `GET /v1/research/runs?org=hanzo`
|
|
* answers, and the price list they are priced at: scores, token counts and
|
|
* rates that exist nowhere in the source, so a page printing a figure derived
|
|
* from them can only have read it here. A single model's run carries its
|
|
* token counts; an Enso run carries `list_cost_usd` in their place, as the
|
|
* research API serves it to a stranger. Each case lib/enso reads is in it: a
|
|
* system's frontier run beside an earlier one; a system read at v1 where the
|
|
* frontier read has none, and its v2 run left off; a system run only at fill; a run over part of the set;
|
|
* a run whose notes call it degraded and one whose blanks make it so; a run
|
|
* that records no tokens; the pool's oracles at three sizes, two reaching the
|
|
* same ceiling. Enso routed: the tiers and the best single model on new
|
|
* questions (split `heldout`, each with `usd_per_1k`, Ultra's earlier run
|
|
* beside its newest), and two tiers on questions they have seen (tag `seen…`),
|
|
* which no board row and no comparison may read. Each single model's run files
|
|
* a `cost_usd` of its own, enso-bench's price table: no board or line may print
|
|
* it (a board's cost is a run's tokens at the list's rate), and the chart reads
|
|
* nothing else for a single model (lib/enso `field`).
|
|
*/
|
|
|
|
const BASE: Run = { id: '', benchmark: '', split: 'test', system: '', version: '', baseline: false, commit: '', ended: '', notes: '', measures: [] }
|
|
/** A single model's tokens and the cost it filed, or an Enso run's cost at list prices as the API serves it. */
|
|
type Spent = { input: number; output: number; filed: number; blanks?: number } | { listed: number }
|
|
export const run = (benchmark: string, split: string, tag: string, system: string, correct: number, n: number, spent: Spent | null, ended: string, notes = ''): Run => ({
|
|
...BASE,
|
|
id: `${benchmark}-${split}-${tag}-${system}`,
|
|
benchmark,
|
|
split,
|
|
system,
|
|
version: system,
|
|
ended,
|
|
notes: `Fixture: run '${tag}' of ${system}.${notes ? ` ${notes}` : ''}`,
|
|
measures: [
|
|
{ category: 'all', metric: 'accuracy', value: correct / n, n: correct, of: n },
|
|
{ category: 'all', metric: 'n', value: n },
|
|
...(spent && 'listed' in spent ? [{ category: 'all', metric: 'list_cost_usd', value: spent.listed }] : []),
|
|
...(spent && 'input' in spent
|
|
? [
|
|
{ category: 'all', metric: 'prompt_tokens', value: spent.input },
|
|
{ category: 'all', metric: 'completion_tokens', value: spent.output },
|
|
{ category: 'all', metric: 'blanks', value: spent.blanks ?? 0 },
|
|
{ category: 'all', metric: 'cost_usd', value: spent.filed },
|
|
]
|
|
: []),
|
|
],
|
|
})
|
|
const pool = (k: number, oracle: number, best: number): Run => ({
|
|
...BASE,
|
|
id: `${G}-test-pool-oracle-pool-oracle@K${k}`,
|
|
benchmark: G,
|
|
system: `pool-oracle@K${k}`,
|
|
version: '',
|
|
ended: '2026-07-23T00:00:00Z',
|
|
measures: [
|
|
{ category: 'all', metric: 'oracle_accuracy', value: oracle / 198, n: oracle, of: 198 },
|
|
{ category: 'all', metric: 'best_single_accuracy', value: best / 198, n: best, of: 198 },
|
|
{ category: 'all', metric: 'k', value: k },
|
|
],
|
|
})
|
|
|
|
/** A routed run: correct of n on a split, and what 1,000 of its questions cost when it says. */
|
|
const routed = (split: string, tag: string, system: string, correct: number, n: number, per1k: number | null, ended: string): Run => ({
|
|
...BASE,
|
|
id: `${G}-${split}-${tag}-${system}`,
|
|
benchmark: G,
|
|
split,
|
|
system,
|
|
ended,
|
|
notes: `Fixture: routed run '${tag}' of ${system}.`,
|
|
measures: [
|
|
{ category: 'all', metric: 'accuracy', value: correct / n, n: correct, of: n },
|
|
{ category: 'all', metric: 'n', value: n },
|
|
...(per1k == null ? [] : [{ category: 'all', metric: 'usd_per_1k', value: per1k }]),
|
|
],
|
|
})
|
|
|
|
const G = 'enso.gpqa_diamond'
|
|
const L = 'enso.livecodebench'
|
|
const D = '2026-07-19T00:00:00Z'
|
|
export const FIELD: Run[] = [
|
|
run(G, 'test', 'frontier', 'enso', 165, 198, { listed: 5.077 }, D),
|
|
run(G, 'test', 'v1', 'enso', 157, 198, { listed: 3.2 }, '2026-07-16T00:00:00Z'),
|
|
run(G, 'test', 'frontier', 'enso-ultra', 165, 198, { listed: 35.1 }, D),
|
|
run(G, 'test', 'frontier', 'enso-flash', 121, 198, { listed: 1.26 }, D),
|
|
routed('heldout', '1.0', 'enso-ultra', 120, 150, 30, '2026-09-01T00:00:00Z'),
|
|
routed('heldout', '2.0', 'enso-ultra', 140, 150, 21.5, '2026-10-01T00:00:00Z'),
|
|
routed('heldout', '2.0', 'enso', 136, 150, 13.25, '2026-10-01T00:00:00Z'),
|
|
routed('heldout', '2.0', 'enso-flash', 125, 150, 2.75, '2026-10-01T00:00:00Z'),
|
|
routed('heldout', '2.0', 'best-single', 133, 150, 15.4, '2026-10-01T00:00:00Z'),
|
|
routed('test', 'seen-2.0', 'enso-ultra', 190, 198, null, '2026-10-01T00:00:00Z'),
|
|
routed('test', 'seen-2.0', 'enso', 187, 198, null, '2026-10-01T00:00:00Z'),
|
|
routed('test', 'seen-2.0', 'enso-flash', 180, 198, null, '2026-10-01T00:00:00Z'),
|
|
run(G, 'test', 'frontier', 'opus-4.8', 139, 198, { input: 80_000, output: 290_000, filed: 9.9 }, D),
|
|
run(G, 'test', 'frontier', 'cheap-3', 100, 198, { input: 55_000, output: 100_000, filed: 0.198 }, D),
|
|
run(G, 'test', 'frontier', 'mid-4', 150, 198, { input: 55_000, output: 400_000, filed: 2.97 }, D),
|
|
run(G, 'test', 'frontier', 'wide-5', 158, 198, { input: 55_000, output: 380_000, filed: 1.485 }, D),
|
|
run(G, 'test', 'v2', 'solo-9', 131, 198, { input: 55_000, output: 90_000, filed: 0.594 }, '2026-07-20T00:00:00Z'),
|
|
run(G, 'test', 'v1', 'solo-9', 129, 198, null, '2026-07-16T00:00:00Z'),
|
|
run(G, 'test', 'part', 'half-model', 37, 40, null, '2026-07-20T00:00:00Z'),
|
|
run(G, 'test', 'frontier', 'blank-6', 160, 198, { input: 55_000, output: 50_000, filed: 0.99 }, D, 'Degraded: more than 10% blank, so neither accuracy is the model\'s score.'),
|
|
run(G, 'test', 'frontier', 'gap-8', 162, 198, { input: 55_000, output: 50_000, filed: 0.99, blanks: 25 }, D),
|
|
pool(2, 180, 165),
|
|
pool(4, 189, 165),
|
|
pool(6, 189, 165),
|
|
run(L, 'recent', 'frontier', 'enso', 158, 175, { listed: 9.31 }, D),
|
|
run(L, 'recent', 'v1', 'opus-4.8', 111, 175, { input: 120_000, output: 500_000, filed: 7.17 }, '2026-07-16T00:00:00Z'),
|
|
run(L, 'recent', 'v2', 'opus-4.8', 113, 175, { input: 120_000, output: 90_000, filed: 7.18 }, '2026-07-20T00:00:00Z'),
|
|
run(L, 'recent', 'frontier', 'cheap-3', 60, 175, { input: 90_000, output: 90_000, filed: 7.19 }, D),
|
|
run(L, 'recent', 'frontier', 'mid-4', 140, 175, { input: 90_000, output: 300_000, filed: 7.21 }, D),
|
|
run(L, 'recent', 'frontier', 'wide-5', 150, 175, { input: 90_000, output: 330_000, filed: 7.22 }, D),
|
|
run(L, 'recent', 'frontier', 'solo-9', 155, 175, { input: 90_000, output: 280_000, filed: 7.23 }, D),
|
|
run(L, 'recent', 'frontier', 'enso-flash', 130, 175, { listed: 5.4 }, D),
|
|
run(L, 'recent', 'fill', 'late-7', 120, 175, null, '2026-10-16T00:00:00Z'),
|
|
]
|
|
|
|
/** A lab's model on the price list, at invented rates per million tokens. */
|
|
export const lab = (id: string, name: string, input: number, output: number) => ({ id, name, provider: 'Fixture lab', features: ['Fixture lab'], contextWindow: 200_000, isFree: false, pricing: { input, output, cacheRead: null, cacheWrite: null } })
|
|
|
|
/**
|
|
* The price list the field is priced at: e2e/gates/api's, with a row for each
|
|
* single model the field measured but blank-6, so every board row but solo-9's
|
|
* v1 run (no tokens) carries a cost. Ours keep api's rates (enso $7 / $31,
|
|
* enso-flash $3 / $9, enso-ultra $11 / $47), which the router's line names:
|
|
* the dearest tier is enso-ultra.
|
|
*/
|
|
export const PRICED = {
|
|
...PRICING,
|
|
thirdPartyModels: [
|
|
...PRICING.thirdPartyModels,
|
|
lab('anthropic/claude-opus-4.8', 'Anthropic: Claude Opus 4.8', 13, 61),
|
|
lab('fixture/cheap-3', 'Fixture: Cheap 3', 0.4, 1.3),
|
|
lab('fixture/mid-4', 'Fixture: Mid 4', 2.2, 8.8),
|
|
lab('fixture/wide-5', 'Fixture: Wide 5', 1.1, 4.4),
|
|
lab('fixture/solo-9', 'Fixture: Solo 9', 1.7, 6.9),
|
|
lab('fixture/gap-8', 'Fixture: Gap 8', 0.9, 2.9),
|
|
],
|
|
}
|