hanzo.ai/e2e/unit/field.spec.ts

import { test, expect } from '@playwright/test'
import { toPricing, type Pricing } from '../../lib/api/read'
import { GPQA, PLAIN, claim, derive, short, type Said } from '../../lib/enso'
import { FIELD, PRICED, lab, run } from '../gates/field'

/**
 * THE GPQA CHART IS EVERY SINGLE MODEL AT ITS BEST PRICED RUN, ENSO'S TIERS ON
 * QUESTIONS THEY HAVE SEEN, AND A PUBLISHED SCORE ONLY UNTIL A RUN MEASURES IT.
 *
 * Held to lib/enso `field` over the invented runs of e2e/gates/field, with every
 * figure worked by hand: a single model's point is the cost it filed per
 * question times 1,000 (opus-4.8: $9.90 for 198 questions is $50 per 1,000), a
 * tier's is its newest held-out run's `usd_per_1k`, and a published score's is
 * its list rates times the median tokens a question took in the runs of its
 * lab's models. A run of the same model, by the one rule that maps a run's
 * system to its price-list id (lib/enso `sku`), takes the published score's
 * place with no change to the code.
 */

const LIST = toPricing(PRICED)!
const D = '2026-07-19T00:00:00Z'
/** A published score for a fixture lab's model, 97% by its own account. */
const SAID: Said[] = [
  { id: 'fixture/said-1', name: 'Said 1', score: 0.97, setting: 'fixture', kind: 'lab', publisher: 'Fixture lab', source: 'https://example.com/said-1', date: '2026-10-01' },
  { id: 'fixture/unlisted', name: 'Unlisted', score: 0.99, setting: 'fixture', kind: 'lab', publisher: 'Fixture lab', source: 'https://example.com/unlisted', date: '2026-10-01' },
]
/** The list with the published model on it, at $1 in and $4 out. */
const SAYS = toPricing({ ...PRICED, thirdPartyModels: [...PRICED.thirdPartyModels, lab('fixture/said-1', 'Fixture: Said 1', 1, 4)] })!
const points = (runs = FIELD, doc: Pricing | null = LIST, said: Said[] = []) => derive(runs, doc, said)!.field
const rows = (f: ReturnType<typeof points>) => f!.dots.map((d) => [d.system, d.name, d.accuracy, Math.round(d.cost * 1e6) / 1e6, d.kind, d.front])

test('the chart: each single model at its best priced run, each tier on questions it has seen at its held-out cost, and the frontier through them', () => {
  const f = points()!
  expect(f.n).toBe(198)
  expect(rows(f)).toEqual([
    ['opus-4.8', 'Claude Opus 4.8', 139 / 198, 50, 'measured', false],
    ['mid-4', 'Mid 4', 150 / 198, 15, 'measured', false],
    ['wide-5', 'Wide 5', 158 / 198, 7.5, 'measured', false],
    ['enso-ultra', 'Enso Ultra', 190 / 198, 21.5, 'measured', true],
    ['enso', 'Enso', 187 / 198, 13.25, 'measured', true],
    ['enso-flash', 'Enso Flash', 180 / 198, 2.75, 'measured', true],
  ])
  expect(f.front).toEqual(['enso-flash', 'enso', 'enso-ultra'])
  expect(f.dots.find((d) => d.system === 'opus-4.8')!.record).toBe(`${GPQA}-test-frontier-opus-4.8`)
  expect(f.dots.find((d) => d.system === 'enso-flash')!.record).toBe(`${GPQA}-test-seen-2.0-enso-flash`)
  // Ultra is the most accurate point.
  expect(claim(f)).toBe('Highest score.')
  // Nothing under 70%: cheap-3's 50.5% and solo-9's 66.2% (its v2 run, the one it filed a cost for) are not drawn.
  // No degraded run, no run over part of the set, no router run off the seen set, no pool.
  for (const s of ['cheap-3', 'solo-9', 'blank-6', 'gap-8', 'half-model', 'best-single']) expect(f.dots.map((d) => d.system)).not.toContain(s)
  // A label's name: the list's less its snapshot stamp and Claude's family word.
  expect(['DeepSeek: DeepSeek V4 Pro 0423', 'Claude Opus 4.8', 'GPT-5.6 Sol'].map((n) => short(n.replace(/^[^:]+:\s*/, '')))).toEqual(['DeepSeek V4 Pro', 'Opus 4.8', 'GPT-5.6 Sol'])
  // Without the list the chart stands, its costs its own; a model goes by its run's name and wears no mark.
  const bare = points(FIELD, null)!
  expect(bare.dots.map((d) => [d.system, d.accuracy, d.cost])).toEqual(f.dots.map((d) => [d.system, d.accuracy, d.cost]))
  expect(bare.dots.find((d) => d.system === 'opus-4.8')).toMatchObject({ name: 'opus-4.8', mark: null })
})

test('a published score is drawn at its list rates times the median tokens a question took for its lab, and a run of the same model replaces it', () => {
  const f = points(FIELD, SAYS, SAID)!
  // Fixture runs with tokens: 55,000 in each, 100,000 / 400,000 / 380,000 / 90,000 out over 198 questions: (55,000 x 1 + 240,000 x 4) / 198 / 1,000.
  const said = f.dots.find((d) => d.kind === 'reported')!
  expect([said.system, said.name, said.accuracy, said.mark, said.record]).toEqual(['fixture/said-1', 'Said 1', 0.97, 'fixture/said-1', 'https://example.com/said-1'])
  expect(said.cost).toBeCloseTo(1_015_000 / 198_000, 9)
  // A model the list does not price is not drawn.
  expect(f.dots.map((d) => d.system)).not.toContain('fixture/unlisted')
  // It beats Enso and Enso Ultra on both, so they are off the chart; Flash is cheaper and stays on the frontier; the title is plain.
  expect(f.front).toEqual(['enso-flash', 'fixture/said-1'])
  expect(f.dots.map((d) => d.system)).not.toContain('enso')
  expect(f.dots.map((d) => d.system)).not.toContain('enso-ultra')
  expect(claim(f)).toBe(PLAIN)

  // A run of the model on the whole set, priced: its measured point takes the published one's place.
  const ran = points([...FIELD, run(GPQA, 'test', 'frontier', 'said-1', 150, 198, { input: 55_000, output: 100_000, filed: 1.98 }, D)], SAYS, SAID)!
  expect(ran.dots.filter((d) => d.kind === 'reported')).toEqual([])
  expect(ran.dots.find((d) => d.system === 'said-1')).toMatchObject({ name: 'Said 1', accuracy: 150 / 198, cost: 10, kind: 'measured', mark: 'fixture/said-1', record: `${GPQA}-test-frontier-said-1` })
  expect(ran.front).toEqual(['enso-flash', 'enso', 'enso-ultra'])
})

test('a tier another point beats on both is not drawn, the chart is not drawn while no tier is on the frontier, and the title says only what the points hold', () => {
  const at = (id: string, correct: number, filed: number) => (r: (typeof FIELD)[number]) =>
    r.id === id ? { ...r, measures: r.measures!.map((m) => (m.metric === 'accuracy' ? { ...m, value: correct / 198, n: correct } : m.metric === 'cost_usd' ? { ...m, value: filed } : m)) } : r
  const dear = FIELD.map((r) => (r.id === `${GPQA}-heldout-2.0-enso` ? { ...r, measures: r.measures!.map((m) => (m.metric === 'usd_per_1k' ? { ...m, value: 25 } : m)) } : r))
  // Enso at $25 is beaten on both by Enso Ultra at $21.50.
  expect(points(dear)!.front).toEqual(['enso-flash', 'enso-ultra'])
  expect(points(dear)!.dots.map((d) => d.system)).not.toContain('enso')
  // opus-4.8 at 196 of 198 for $50 per 1,000 is the highest score and beats no tier on both: the chart stands, its title plain.
  const higher = points(FIELD.map(at(`${GPQA}-test-frontier-opus-4.8`, 196, 9.9)))!
  expect(higher.front).toEqual(['enso-flash', 'enso', 'enso-ultra', 'opus-4.8'])
  expect(claim(higher)).toBe(PLAIN)
  // Fewer than three points that are not ours: no chart.
  expect(points(FIELD.filter((r) => r.system !== 'mid-4'))).toBeNull()
  // wide-5 at 197 of 198 for 50 cents per 1,000 beats every tier: no chart.
  expect(points(FIELD.map(at(`${GPQA}-test-frontier-wide-5`, 197, 0.099)))).toBeNull()
  // No seen run, no tier: no chart.
  expect(points(FIELD.filter((r) => !r.id.includes('-seen-')))).toBeNull()
})