hanzo.ai/e2e/gates/wins.spec.ts

import { test, expect } from '@playwright/test'
import { existsSync, readdirSync, readFileSync } from 'node:fs'
import { extname, join, relative } from 'node:path'
import { requireExport, serveExport } from './export'
import { api, listing } from './revision'
import { MODELS, serve } from './api'
import snapshot from '../../lib/data/benchmarks-kai.json' with { type: 'json' }
import { INDEX, KAI, LEDGER, RUNTIME } from '../../lib/bench-data'
import { JEV, frontier, verdict } from '../../lib/tally'
import { derive as enso, published } from '../../lib/enso'
import { FIELD, PRICED } from './field'
import { toPricing } from '../../lib/api/read'
import { wins } from '../../lib/engine'
import { ahead as solvedMore, gains, type Side } from '../../lib/coding'
import { ATTACK, ahead, derive, headlines, table, type Board, type Run } from '../../lib/kai'
import { against, lose } from './wins'

/**
 * THE PAGES PUBLISH WINS, AND NOTHING ELSE.
 *
 * The owner, 2026-10-16: for marketing we highlight what we win at, so people
 * can study and understand it, and we do not reveal benchmarks we are still
 * working on. So every row a public page draws is one Hanzo leads on its own
 * metric: a Decision Suite task, a frozen-harness suite and a multilingual
 * language where Kai leads Jev, a calibration pair where Kai's error is the
 * lower, an Enso row that outscores every single model on its board, an Enso
 * point on its board's frontier, a coding path whose whole interval is above
 * one, a tool split whose current handler holds against the one it replaced,
 * an engine cell whose whole interval is above parity, a runtime row no
 * published figure beats. A row Hanzo does not lead is not drawn, not counted
 * and not named, and a count is of the rows drawn with no total beside it:
 * "Kai wins 15 tasks", never "15 of 16", "all 12 suites" or "ahead on 31 of
 * 51". The one total a page may state is every one of them: the Decision
 * Suite's "16 of 16" while Kai wins all its tasks (lib/kai `table`). Every
 * figure left is a recorded run, unedited: rows are removed, never a number
 * changed.
 *
 * Two halves. The data the pages are handed (lib/bench-data, lib/kai over the
 * snapshot, lib/enso over the invented field), checked row by row. Then the
 * bytes that ship and the pages as a reader sees them once the runs are read:
 * no word a loss is told in, no count against a total, no row Jev leads, and
 * nothing in the flight data or the scripts the page does not draw.
 *
 * The one place a page names the attack question is where the Decision
 * Suite's false-alarm row's figures come from, a row Kai leads; those names
 * are cut by their own text (`NAMED`) before the word is looked for.
 */

/** Words a loss is told in. Bare "behind" is one on a benchmark page; on a product page it also means "the family behind Hanzo". */
const LOSS = /\b(trails|trailing|loses|losses|behind|collapses|falls away|worse than|where it loses|boundaries|level on|jailbreak)\b/i
/**
 * The same on a product page, where words have their ordinary senses too: a
 * product is "behind one endpoint", and the landing's chat example says
 * "gravity collapses the core". A loss there is a figure's subject doing it.
 */
const PRODUCT_LOSS = /\b(trails|trailing|loses|falls away|worse than|where it loses|boundaries|level on|jailbreak)\b|\b(accuracy|score|it|Kai|Enso|Zen) collapses\b|\b(is|are|points|Kai|Enso|Zen) behind\b|\bbehind (on|at|by|Jev)\b/i
/** Kai's suite rows as the flight data carries them. */
const ROW = /"label":"([^"]+)","kai":(null|[\d.e-]+),"jev":(null|[\d.e-]+),"kaiEce":(null|[\d.e-]+),"jevEce":(null|[\d.e-]+)/g
/** Within two questions of four hundred is level, lib/tally's tie. */
const TIE = 0.005

/** A route's files in the export: the page, its flight data and markdown, and the flight segments in its folder. */
function route(out: string, path: string): string[] {
  const base = path === '/' ? join(out, 'index') : join(out, path)
  const own = ['.html', '.txt', '.md'].map((x) => base + x).filter(existsSync)
  const dir = path === '/' ? out : base
  const segments = existsSync(dir) ? readdirSync(dir).filter((f) => f.startsWith('__next') && extname(f) === '.txt').map((f) => join(dir, f)) : []
  return [...own, ...segments]
}
/** A page's reading text: no script, no style, no tag. */
const reading = (html: string) =>
  html
    .replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/g, ' ')
    .replace(/&amp;/g, '&')
    .replace(/&#x27;|&#39;/g, "'")
    .replace(/\s+/g, ' ')
/** Flight data as JSON text, whether it is a .txt file or strings pushed from the page's scripts. */
const flight = (text: string) => text.replace(/\\"/g, '"')
const around = (text: string, at: number) => `…${text.slice(Math.max(0, at - 60), at + 60)}…`

/** Rates no source holds, so the Decision Suite's cost row holds and the suite is whole. */
const RATES = { kai: 0.017, jev: 0.061 }
const LIST = {
  ...MODELS,
  data: [
    ...MODELS.data.map((m) => (m.id === 'kai' ? { ...m, pricing: { input_per_million: RATES.kai, output_per_million: 0 } } : m)),
    { id: JEV, owned_by: 'typesafe', outputs: ['decision'], pricing: { input_per_million: RATES.jev, output_per_million: 0 } },
  ],
}
/** The Decision Suite as a page draws it once both answers are in. */
const SUITE = table(KAI.suite, RATES)
/** The tasks Kai does not win: none is drawn, counted or named. */
const LOST = SUITE.tasks.filter((t) => !ahead(t))
/** The suite with Jev answering AG News perfectly, so a task Jev leads is always there to cut. */
const LOSING = table(derive(lose(snapshot.data as unknown as Run[], 'jev.ag_news'))!.suite, RATES)
/** Every Kai board a page may draw, the Decision Suite's among them. */
const BOARDS = [...Object.values(KAI.boards), SUITE.board].filter((b): b is Board => b != null)
/** The languages the multilingual board draws. */
const LANGUAGE = new Set(KAI.boards.languages?.blocks.flatMap((b) => b.lines.map((l) => l.label)) ?? [])
/**
 * What a page prints that names the attack question: lib/kai's own names for
 * it, and every name the rows drawn carry (their data, the files and folders
 * they are read and re-run from). Longest first, so no name is cut out of a
 * longer one.
 */
const NAMED = [
  ...new Set([
    ...Object.values(ATTACK),
    ...SUITE.board.blocks.flatMap((b) => b.lines).flatMap((l) => [...(l.about?.data ?? []), ...(l.about?.links ?? [])].flatMap((x) => [x.text, x.href ?? ''])),
  ]),
]
  .filter((x) => /jailbreak/i.test(x))
  .sort((a, b) => b.length - a.length)
/** Text less every string of NAMED. */
const told = (text: string) => NAMED.reduce((t, x) => t.split(x).join(' '), text)

const BENCHMARK_PAGES = () => {
  const out = requireExport()
  const pages = ['/benchmarks', ...readdirSync(join(out, 'benchmarks')).filter((f) => f.endsWith('.html')).map((f) => `/benchmarks/${f.replace(/\.html$/, '')}`)]
  return { out, pages }
}
const PRODUCT_PAGES = ['/', '/kai', '/kai/guide', '/enso', '/zen']

test('the checks can fail', () => {
  expect(LOSS.test('decode trails everywhere')).toBe(true)
  expect(LOSS.test('Where it loses')).toBe(true)
  expect(LOSS.test('5.1 points behind at $4')).toBe(true)
  expect(LOSS.test('losses included')).toBe(true)
  expect(PRODUCT_LOSS.test('the open-weight family behind Hanzo')).toBe(false)
  expect(PRODUCT_LOSS.test('gravity collapses the core into a singularity')).toBe(false)
  expect(PRODUCT_LOSS.test('Where Kai is behind')).toBe(true)
  expect(PRODUCT_LOSS.test('accuracy collapses past 77 options')).toBe(true)
  for (const said of [
    'Kai wins 16 of 18 tasks against Jev',
    '16 of 18 Decision Suite tasks won · Kai vs Jev',
    'Every language of MASSIVE: ahead on 31 of 51, level on 6',
    'The frozen harness, all 12 suites',
    'Kai leads 11 of 12 suites',
    'Result Win 129 Unscored 22 151 of 160 results',
  ])
    expect(against(said), said).not.toBeNull()
  for (const said of [
    'Kai wins 15 tasks of the Decision Suite against Jev.',
    'Kai wins 16 of 16 tasks of the Decision Suite against Jev.',
    '16 of 16 Decision Suite tasks won · Kai vs Jev',
    '184 of 198 right, against 91.9% for gpt-5.6-sol, the next model on the same questions.',
    'Flagged: Kai 83 of 600, Jev 126 of 600.',
    'MASSIVE: ahead of Jev on 31 languages',
    'The frozen harness: Kai leads Jev on 11 suites',
    'Result Win 129 Unscored 22 151 of 151 results, 24 benchmarks',
  ])
    expect(against(said), said).toBeNull()
  expect(NAMED.length, 'a row drawn names the attack question where its figures come from').toBeGreaterThan(0)
  expect(LOSS.test(told(`${ATTACK.data} Kai leads.`))).toBe(false)
  expect(LOSS.test(told('Jailbreak'))).toBe(true)
  const row = [...flight('{\\"label\\":\\"Jailbreak\\",\\"kai\\":0.9175,\\"jev\\":0.94,\\"kaiEce\\":0.0741,\\"jevEce\\":0.0469}').matchAll(ROW)]
  expect(row.map((m) => m[1])).toEqual(['Jailbreak'])
})

test.describe('the data the pages are handed', () => {
  test('every Kai row a board draws is one Kai leads, marked Kai’s, and no title, note or claim counts against a total', () => {
    expect(BOARDS.length).toBeGreaterThan(1)
    for (const b of [...BOARDS, LOSING.board]) {
      for (const said of [b.title, b.lede ?? '', b.note]) expect(against(said), b.id).toBeNull()
      for (const line of b.blocks.flatMap((x) => x.lines)) {
        const best = line.cells.filter((c) => c.best).map((c) => c.who)
        // Latency is Kai's own measure, compared with nothing; every other row is Kai beside Jev, and Kai's.
        expect(best, `${b.id}: ${line.label}`).toEqual(b.id === 'kai-latency' ? [] : ['kai'])
      }
    }
    for (const c of [...KAI.claims, ...headlines(SUITE), ...headlines(LOSING)]) {
      expect(against(`${c.figure} ${c.label} ${c.note} ${c.lede ?? ''}`), c.label).toBeNull()
      expect(`${c.figure} ${c.label} ${c.note} ${c.lede ?? ''}`, c.label).not.toMatch(/1,000,000|Option Memory|refuses/)
    }
    expect(against(`${KAI.headline.title} ${KAI.headline.lede}`)).toBeNull()
  })

  test('the Decision Suite draws the tasks Kai wins, counts them with no total but every one, and names no other', () => {
    for (const t of [SUITE, LOSING]) {
      const drawn = t.board.blocks.flatMap((b) => b.lines.map((l) => l.label))
      expect(drawn).toEqual(t.tasks.filter((x) => ahead(x)).map((x) => x.label))
      for (const x of t.tasks.filter((y) => !ahead(y))) expect(JSON.stringify([t.board, t.claim]), x.label).not.toContain(x.label)
      expect(t.whole).toBe(true)
      expect(t.won).toBe(drawn.length)
      // "16 of 16" while Kai wins every task; the bare count once one is lost.
      expect(t.claim?.figure).toBe(t.won === t.tasks.length ? `${t.won} of ${t.won}` : String(t.won))
    }
    expect(LOSING.tasks.filter((x) => !ahead(x)).map((x) => x.key), 'Jev leads the task the gate gave him').toContain('jev.ag_news')
    expect(LOSING.claim?.figure).toBe(String(LOSING.won))
  })

  test('every Kai suite and language is one Kai leads, every calibration pair one where its error is lower, and the harness board is those suites', () => {
    expect(KAI.index.wins.length).toBeGreaterThan(0)
    expect(Object.keys(KAI.index).sort()).toEqual(['ended', 'languages', 'wins'])
    for (const r of KAI.index.wins) {
      expect(verdict(r, 'acc'), r.label).toBe('won')
      expect(r.kai! - r.jev!, r.label).toBeGreaterThanOrEqual(TIE)
      if (r.kaiEce != null || r.jevEce != null) expect(verdict(r, 'ece'), `${r.label} calibration`).toBe('won')
    }
    const t = KAI.index.languages
    if (t) {
      expect(Object.keys(t), 'the tally is the languages Kai leads, and nothing about the rest').toEqual(['ahead'])
      expect(LANGUAGE.size, 'the languages drawn are the ones counted').toBe(t.ahead)
    }
    const harness = KAI.boards.harness!.blocks.flatMap((b) => b.lines.map((l) => l.label))
    expect(harness).toEqual([...KAI.index.wins.map((r) => r.label), `Mean, these ${KAI.index.wins.length} suites`])
    expect(harness).not.toContain('Jailbreak')
    expect(KAI.headline.title).toBe(`Kai leads Jev on ${KAI.index.wins.length} suites, every one a recorded run.`)
    expect(KAI.headline.lede).toContain(`across these ${KAI.index.wins.length} suites`)
  })

  test('every Enso row a board draws tops every single model on it, and every Enso point drawn on a frontier is on it', () => {
    // Enso's boards and frontier are the research runs at the price list's rates, read when the page opens (lib/enso); this holds the derivation to them over invented runs.
    const e = enso(FIELD, toPricing(PRICED))!
    for (const b of e.boards) {
      const rows = published(b)
      const rival = Math.max(...rows.filter((r) => !r.ours).map((r) => r.accuracy))
      for (const r of rows.filter((x) => x.ours)) expect(r.accuracy, `${b.id}: ${r.system}`).toBeGreaterThan(rival)
    }
    expect(e.boards.some((b) => b.rows.some((r) => r.ours && !published(b).some((x) => x.system === r.system))), 'the field holds an Enso row a model beats, so the cut is exercised').toBe(true)
    expect(e.frontier.length).toBeGreaterThan(0)
    for (const b of e.frontier) {
      const on = new Set(frontier(b.points).map((p) => p.model))
      for (const p of b.points.filter((x) => x.ours)) expect(on.has(p.model), `${b.id}: ${p.model}`).toBe(true)
    }
  })

  test('a coding path is drawn only as a resolved gain, and the solved counts only while Hanzo Dev solved more', () => {
    const c = INDEX.coding
    if (c) for (const p of gains(c.paths)) expect([p.label, p.lo! > 1]).toEqual([p.label, true])
    const side = (solved: number): Side => ({ id: '', system: '', version: '', solved, of: 44, median: 1, calls: 1 })
    expect(solvedMore({ base: side(44), ours: side(42) })).toBe(false)
    expect(solvedMore({ base: side(42), ours: side(42) })).toBe(false)
    expect(solvedMore({ base: side(41), ours: side(42) })).toBe(true)
  })

  test('a tool split is the current handler beside the one it replaced, and no experiment', () => {
    for (const t of INDEX.tools) for (const s of t.splits) expect(s.rows.map((r) => r.role), `${t.id} ${s.split}`).toEqual(s.rows.length === 2 ? ['now', 'before'] : ['now'])
  })

  test('every shelf row is a win or a standalone measurement', () => {
    for (const r of LEDGER.results) expect(['win', 'unscored'], r.id).toContain(r.standing)
  })

  test('every engine cell published is a settled win, and every runtime row is ours', () => {
    expect(wins.length).toBeGreaterThan(0)
    for (const c of wins) expect([c.verdict, c.noisy, c.lo > 1, c.phase], c.run).toEqual(['WIN', false, true, 'prefill'])
    for (const s of RUNTIME.shots) expect(s.ours, `${s.part}: ${s.what}`).not.toBeNull()
    expect(RUNTIME.shots.map((s) => s.what)).not.toContain('hanzo-vm, cold boot')
  })
})

test.describe('the bytes that ship', () => {
  test('no benchmark page tells a loss or counts against a total', () => {
    const { out, pages } = BENCHMARK_PAGES()
    const found: string[] = []
    for (const path of pages) {
      for (const file of [join(out, `${path}.html`), join(out, `${path}.md`)].filter(existsSync)) {
        const text = told(file.endsWith('.html') ? reading(readFileSync(file, 'utf8')) : readFileSync(file, 'utf8').replace(/&#x27;|&#39;/g, "'"))
        for (const m of [LOSS.exec(text), against(text)]) if (m) found.push(`${relative(out, file)}: ${around(text, m.index!)}`)
      }
    }
    expect(pages.length).toBeGreaterThan(10)
    expect(found).toEqual([])
  })

  test('no product page tells a benchmark loss or counts against a total', () => {
    const out = requireExport()
    const found: string[] = []
    for (const path of PRODUCT_PAGES) {
      const file = join(out, path === '/' ? 'index.html' : `${path}.html`)
      const text = told(reading(readFileSync(file, 'utf8')))
      for (const m of [PRODUCT_LOSS.exec(text), against(text)]) if (m) found.push(`${path}: ${around(text, m.index!)}`)
    }
    expect(found).toEqual([])
  })

  test('the flight data carries only the rows the pages draw', () => {
    const { out, pages } = BENCHMARK_PAGES()
    const labels = new Set(KAI.index.wins.map((r) => r.label))
    const found: string[] = []
    let rows = 0
    for (const path of [...pages, ...PRODUCT_PAGES]) {
      for (const file of route(out, path)) {
        const text = flight(readFileSync(file, 'utf8'))
        const where = relative(out, file)
        for (const m of text.matchAll(ROW)) {
          rows++
          const [, label, k, j, ke, je] = m
          const n = (v: string) => (v === 'null' ? null : Number(v))
          if (!labels.has(label)) found.push(`${where}: suite ${label} is not one Kai leads`)
          if (n(k) == null || n(j) == null || n(k)! - n(j)! < TIE) found.push(`${where}: ${label} accuracy ${k} against ${j}`)
          if ((n(ke) != null || n(je) != null) && !(n(ke)! < n(je)!)) found.push(`${where}: ${label} calibration ${ke} against ${je}`)
        }
        for (const m of text.matchAll(/"standing":"(loss|level)"/g)) found.push(`${where}: a result ${m[1]} ${around(text, m.index!)}`)
        for (const m of text.matchAll(/"who":"jev","text":"[^"]*","best":true/g)) found.push(`${where}: a row Jev leads ${around(text, m.index!)}`)
        const jail = /jailbreak/i.exec(told(text))
        if (jail) found.push(`${where}: ${around(text, jail.index)}`)
      }
    }
    // Kai's rows are read when a page opens (hooks/useKai), so none ships in the export.
    expect(rows, 'Kai rows in the flight data').toBe(0)
    expect(found).toEqual([])
  })

  test('once the runs are read, no page draws a row Kai does not lead, names a task it does not win, or counts against a total', async ({ page }) => {
    const server = await serveExport()
    try {
      await serve(page, { '/v1/models': LIST })
      await page.route(api, (r) => r.fulfill({ json: listing(r, snapshot.data) }))
      for (const [path, ready, rule] of [
        ['/benchmarks', '#leaderboard [role="listitem"]', LOSS],
        ['/kai', `[id="${KAI.boards.harness!.id}"] table`, PRODUCT_LOSS],
        // The landing draws no Decision Suite since e3948ea5f; its Kai figures are the race's replay (Faster).
        ['/', '[data-slot="replay"]', PRODUCT_LOSS],
      ] as const) {
        await page.goto(server.url + path)
        await page.locator('html[data-hydrated]').waitFor({ state: 'attached' })
        await expect(page.locator(ready).first()).toBeVisible({ timeout: 15_000 })
        // The suite is whole, so its count is drawn, and it is the tasks Kai wins.
        if (path !== '/') await expect(page.locator(`[id="${SUITE.board.id}"] tbody tr`)).toHaveCount(SUITE.won)
        const text = told((await page.locator('main').last().textContent() ?? '').replace(/\s+/g, ' '))
        for (const m of [rule.exec(text), against(text)]) expect(m ? `${path}: ${around(text, m.index!)}` : null).toBeNull()
        for (const t of LOST) expect(text, `${path}: ${t.label}`).not.toContain(t.label)
        await expect(page.locator('[data-who="jev"][data-best]'), `${path}: a cell Jev leads`).toHaveCount(0)
        if (KAI.boards.languages && path !== '/') await expect(page.locator(`[id="${KAI.boards.languages.id}"] tbody tr`)).toHaveCount(LANGUAGE.size)
      }
    } finally {
      await server.close()
    }
  })

  test('the scripts the Kai pages load bundle no run', () => {
    const out = requireExport()
    const found: string[] = []
    for (const page of ['benchmarks.html', 'kai.html']) {
      const html = readFileSync(join(out, page), 'utf8')
      const chunks = [...new Set([...html.matchAll(/src="(\/_next\/static\/[^"]+\.js)"/g)].map((m) => decodeURIComponent(m[1])))]
      expect(chunks.length, page).toBeGreaterThan(0)
      for (const src of chunks) {
        const js = told(readFileSync(join(out, src), 'utf8'))
        const m = /jailbreak|"measures":\[/i.exec(js)
        if (m) found.push(`${page} → ${src}: ${around(js, m.index)}`)
      }
    }
    expect(found).toEqual([])
  })
})