hanzo.ai/e2e/gates/suite.spec.ts

import { test, expect, type Page } from '@playwright/test'
import { read, serveExport } from './export'
import { DECISION, ahead, derive, headlines, table, type Run, type Table } from '../../lib/kai'
import { JEV, kaiLead } from '../../lib/tally'
import { MODELS, block, serve } from './api'
import { api } from './revision'
import snapshot from '../../lib/data/benchmarks-kai.json' with { type: 'json' }
import { toRuns } from '../../lib/api/research'
import { FIELD } from './field'
import { against, lose } from './wins'

/**
 * THE DECISION SUITE LEADS /kai AND /benchmarks, READ WHEN THE PAGE OPENS.
 *
 * The pages publish wins only (owner, 2026-10-16): the board draws the tasks
 * Kai wins against Jev and no other, and counts them out of the suite only
 * while that is every one ("Kai wins 16 of 16 tasks"), else bare ("Kai wins 15
 * tasks"); a task Jev leads, or one whose runs are not yet public, is not
 * drawn, counted or named. The gate gives Jev one task (`lose`) to see the cut. Every row names its public data and links its runs
 * and the folder that re-runs it. Every figure is read when the page opens: the
 * research runs (`GET /v1/research/runs?org=hanzo`, hooks/useKai) and the cost
 * row from the model list (`GET /v1/models`). So the gate serves both, the runs
 * as the snapshot holds them and the model list with rates invented here, and
 * holds the page to what lib/kai derives from exactly those answers: the count
 * waits for every task's figures, the board says it is reading while it reads,
 * and says the API is unreachable when it is.
 *
 * Below it, the frozen harness's suites Kai leads: the note says so, and this
 * holds the page to it.
 */

const RUNS = snapshot.data as unknown as Run[]
/** Rates no source holds, so a page printing one can only have read it here. */
const RATES = { kai: 0.017, jev: 0.061 }
const LIST = {
  ...MODELS,
  data: [
    ...MODELS.data.map((m) => (m.id === 'kai' ? { ...m, pricing: { input_per_million: RATES.kai, output_per_million: 0 } } : m)),
    { id: JEV, owned_by: 'typesafe', outputs: ['decision'], pricing: { input_per_million: RATES.jev, output_per_million: 0 } },
  ],
}
const KAI = derive(RUNS)!
/** The suite as the page draws it once both answers are in, and with the model list unread. */
const SERVED = table(KAI.suite, RATES)
const UNREAD = table(KAI.suite, null)
/** The words under the board's title: the day the tasks it draws were fixed, and the harness below. */
const NOTE = "Each task was fixed on 2026-10-15. The frozen harness's suites Kai leads are below."
/** The suite with Jev answering AG News perfectly: a task Kai does not win, so the cut is always exercised. */
const LOSING = table(derive(lose(RUNS, 'jev.ag_news'))!.suite, RATES)

const PAGES = [
  { path: '/kai', heading: '#benchmarks h2' },
  { path: '/benchmarks', heading: '#kai-title' },
] as const

let server: Awaited<ReturnType<typeof serveExport>>
test.beforeAll(async () => {
  server = await serveExport()
})
test.afterAll(async () => {
  await server.close()
})

const runs = (page: Page) => page.route(api, (r) => r.fulfill({ json: { data: RUNS, total: RUNS.length } }))

/** Every row as the table draws it: the label with what it says under it, both figures, and every public link. */
async function drawn(page: Page, t: Table) {
  const rows = page.locator(`[id="${t.board.id}"] table tbody tr`)
  const lines = t.board.blocks.flatMap((b) => b.lines)
  await expect(rows).toHaveCount(lines.length)
  for (const [i, l] of lines.entries()) {
    const cells = rows.nth(i).locator('td')
    await expect(cells.nth(0), l.label).toContainText(l.label)
    await expect(cells.nth(1), `${l.label}, Kai`).toHaveText(l.cells[0].text)
    await expect(cells.nth(2), `${l.label}, Jev`).toHaveText(l.cells[1].text)
    for (const link of [...l.about!.data, ...l.about!.links].filter((x) => x.href)) await expect(cells.nth(0).locator(`a[href="${link.href}"]`), `${l.label}: ${link.text}`).toHaveCount(1)
    if (l.about?.also) await expect(cells.nth(0)).toContainText(l.about.also)
  }
}

test('the board draws the tasks Kai wins and no other, and counts them out of the suite only while that is every one', () => {
  expect(SERVED.board.lede).toBe(NOTE)
  for (const t of [SERVED, LOSING]) {
    const lines = t.board.blocks.flatMap((b) => b.lines)
    const won = t.tasks.filter((x) => ahead(x))
    // Every row drawn is a task Kai wins, marked Kai's; no task Jev leads, or whose runs are not public, is drawn.
    expect(lines.map((l) => l.label)).toEqual(won.map((x) => x.label))
    for (const l of lines) expect([l.label, l.cells.filter((c) => c.best).map((c) => c.who)]).toEqual([l.label, ['kai']])
    expect(t.won).toBe(won.length)
    expect(t.whole).toBe(t.tasks.every((x) => ahead(x) != null))
    for (const said of [t.board.title, t.board.lede!, t.board.note, ...(t.claim ? [t.claim.figure, t.claim.label, t.claim.note, t.claim.lede!] : [])]) expect(against(said), said).toBeNull()
  }
  expect(LOSING.tasks.filter((x) => !ahead(x)).map((x) => x.key), 'Jev leads the task the gate gave him').toEqual(['jev.ag_news'])
  expect(LOSING.claim?.figure).toBe(String(LOSING.won))
  expect(LOSING.board.title).toBe(`${DECISION.name}: Kai wins ${LOSING.won} tasks against Jev`)
  if (SERVED.whole) {
    const every = SERVED.won === SERVED.tasks.length
    expect(SERVED.claim?.figure).toBe(every ? `${SERVED.won} of ${SERVED.won}` : String(SERVED.won))
    expect(SERVED.board.title).toBe(`${DECISION.name}: Kai wins ${SERVED.claim!.figure} tasks against Jev`)
  } else expect(SERVED.claim).toBeNull()
  // Unread, the cost row has no figures: it is not drawn, and the board states no count.
  expect(UNREAD.claim).toBeNull()
  expect(UNREAD.board.blocks.flatMap((b) => b.lines).map((l) => l.label)).not.toContain(SERVED.tasks.find((t) => t.key === 'cost')!.label)
})

test('a page keeps Kai and Jev on a decision benchmark, and every system on an Enso one', () => {
  const run = (benchmark: string, system: string) => ({ ...RUNS[0], id: `${benchmark}-${system}`, benchmark, system })
  const read = toRuns({ data: [run('jev.ag_news', 'kai'), run('jev.ag_news', 'jev'), run('jev.ag_news', 'other'), run('enso.gpqa_diamond', 'other')] })
  expect(read?.map((r) => r.id)).toEqual(['jev.ag_news-kai', 'jev.ag_news-jev', 'enso.gpqa_diamond-other'])
  expect(toRuns({ rows: [] })).toBeNull()
})

for (const { path, heading } of PAGES) {
  test(`${path} leads with the Decision Suite, read when the page opens, the frozen harness below it`, async ({ page }) => {
    await runs(page)
    const asked = await serve(page, { '/v1/models': LIST })
    await page.setViewportSize({ width: 1440, height: 900 })
    await page.goto(server.url + path)
    const suite = page.locator(`[id="${SERVED.board.id}"]`)
    await expect(suite).toContainText(SERVED.board.title, { ignoreCase: true, timeout: 15_000 })
    expect(asked, 'the cost row is the model list’s').toContain('/v1/models')
    await expect(suite).toContainText(NOTE)
    if (SERVED.claim) await expect(page.locator(heading)).toHaveText(SERVED.claim.lede!)
    await drawn(page, SERVED)
    await expect(suite.locator(`a[href="${KAI.suite.rerun}"]`)).toHaveCount(1)

    // No task Kai does not win is named on the page.
    for (const t of SERVED.tasks.filter((x) => !ahead(x))) await expect(page.locator('main'), t.label).not.toContainText(t.label)

    // The harness the note names: below the suite, the suites Kai leads and their mean, and no suite Jev leads.
    const board = KAI.boards.harness!
    const harness = page.locator(`[id="${board.id}"]`)
    const below = await suite.evaluate((s, id) => !!(s.compareDocumentPosition(document.getElementById(id)!) & Node.DOCUMENT_POSITION_FOLLOWING), board.id)
    expect(below, 'the harness is below the suite').toBe(true)
    const lines = board.blocks.flatMap((b) => b.lines)
    await expect(harness.locator('table tbody tr')).toHaveCount(lines.length)
    expect(lines.length).toBe(KAI.index.wins.length + 1)
    for (const l of lines) expect([l.label, l.cells.filter((c) => c.best).map((c) => c.who)]).toEqual([l.label, ['kai']])
    await expect(harness.locator('table tbody tr', { hasText: 'Jailbreak' })).toHaveCount(0)
    expect(against((await harness.textContent()) ?? '')).toBeNull()
  })

  test(`${path} draws no task Kai does not win, and counts the rest bare`, async ({ page }) => {
    const runs = lose(RUNS, 'jev.ag_news')
    await page.route(api, (r) => r.fulfill({ json: { data: runs, total: runs.length } }))
    await serve(page, { '/v1/models': LIST })
    await page.goto(server.url + path)
    const suite = page.locator(`[id="${LOSING.board.id}"]`)
    await expect(suite).toContainText(LOSING.board.title, { ignoreCase: true, timeout: 15_000 })
    await drawn(page, LOSING)
    if (LOSING.claim) await expect(page.locator(heading)).toHaveText(LOSING.claim.lede!)
    await expect(suite).not.toContainText('AG News')
    expect(against((await page.locator('main').last().textContent()) ?? '')).toBeNull()
  })

  test(`${path} says it is reading, then says the API is unreachable, and draws no figure in either`, async ({ page }) => {
    let release: () => void = () => {}
    const held = new Promise<void>((done) => (release = done))
    await page.route(api, async (r) => {
      await held
      await r.abort('connectionrefused')
    })
    await block(page)
    await page.goto(server.url + path)
    await expect(page.locator('main [data-api="loading"]').first()).toBeVisible({ timeout: 15_000 })
    release()
    await expect(page.locator('main [data-api="unreachable"]').first()).toBeVisible({ timeout: 15_000 })
    await expect(page.locator('main [data-api="loading"]')).toHaveCount(0)
    await expect(page.locator(`[id="${SERVED.board.id}"] table`)).toHaveCount(0)
    await expect(page.locator(`[id="${KAI.boards.harness!.id}"] table`)).toHaveCount(0)
    await expect(page.locator(heading)).not.toContainText(`tasks of the ${DECISION.name}`)
  })
}

test('/kai leads with the suite’s own rows as headlines, each linking the board', async ({ page }) => {
  await runs(page)
  await serve(page, { '/v1/models': LIST })
  await page.goto(server.url + '/kai')
  const lines = headlines(SERVED)
  const lead = page.locator('#lead [data-lead]')
  await expect(lead).toHaveCount(lines.length, { timeout: 15_000 })
  for (const [i, c] of lines.entries()) {
    await expect(lead.nth(i)).toContainText(c.figure)
    await expect(lead.nth(i)).toContainText(c.label)
    await expect(lead.nth(i)).toHaveAttribute('href', `#${SERVED.board.id}`)
  }
})

/** /benchmarks' hero tile, by the anchor it links. */
const hero = (page: Page, href: string) => page.locator(`#lead a[href="${href}"]`)

test('/benchmarks leads with the Kai figure, the Decision Suite’s while it holds', async ({ page }) => {
  await runs(page)
  await serve(page, { '/v1/models': LIST })
  const lead = kaiLead(SERVED.claim, KAI)
  if (SERVED.whole) expect(lead?.href, 'a whole suite leads').toBe(`#${SERVED.board.id}`)
  await page.goto(server.url + '/benchmarks')
  if (!lead) {
    // The runs hold no Kai figure: the page states none.
    await expect(page.locator('#lead [data-slot="lead-tiles"] > [aria-hidden]')).toHaveCount(0, { timeout: 15_000 })
    await expect(page.locator('#lead a[href="#decision-suite"], #lead a[href="#kai"]')).toHaveCount(0)
    return
  }
  await expect(hero(page, lead.href)).toContainText(lead.figure, { timeout: 15_000 })
  await expect(hero(page, lead.href)).toContainText(lead.label)
  // The tile opens the board it counts.
  await expect(page.locator(`[id="${lead.href.slice(1)}"]`)).toHaveCount(1)
})

test('while the model list is on its way /benchmarks states no Kai count; once it fails it falls back to the harness record', async ({ page }) => {
  // The Kai runs and Enso's, so a section drawn from the runs alone says they were read.
  await page.route(api, (r) => r.fulfill({ json: { data: [...RUNS, ...FIELD], total: RUNS.length + FIELD.length } }))
  let release: () => void = () => {}
  const held = new Promise<void>((done) => (release = done))
  // Every other read fails at once; the model list, registered after, is held, then fails.
  await block(page)
  await page.route(
    (u) => u.pathname === '/v1/models',
    async (r) => {
      await held
      await r.abort('connectionrefused')
    },
  )
  const fallback = kaiLead(null, KAI)
  if (fallback) expect(fallback.href).toBe('#kai')
  // /benchmarks has drawn Enso's board from the runs; its lead holds its four places, empty, and states no figure.
  await page.goto(server.url + '/benchmarks')
  await expect(page.locator('#enso [data-board] tbody tr').first()).toBeVisible({ timeout: 15_000 })
  await expect(page.locator('#lead [data-slot="lead-tiles"] > [aria-hidden]')).toHaveCount(4)
  await expect(page.locator('#lead [data-slot="lead-tiles"] a')).toHaveCount(0)
  await expect(hero(page, `#${SERVED.board.id}`)).toHaveCount(0)
  await expect(hero(page, '#kai')).toHaveCount(0)
  if (KAI.index.wins.length) await expect(page.locator('#kai-title')).not.toHaveText(KAI.headline.title)
  release()
  await page.goto(server.url + '/benchmarks')
  if (fallback) {
    await expect(hero(page, fallback.href)).toContainText(fallback.figure, { timeout: 15_000 })
    await expect(hero(page, fallback.href)).toContainText(fallback.label)
  } else await expect(page.locator('#lead [data-slot="lead-tiles"] > [aria-hidden]')).toHaveCount(0, { timeout: 15_000 })
  await expect(hero(page, `#${SERVED.board.id}`)).toHaveCount(0)
})

test('a link to the suite from another page lands on its title, clear of the header and the chapter nav, at 390, 1440 and 1920', async ({ page }) => {
  // Enso's runs too: its sections above the suite grow as they are read, which is what moves the anchor.
  await page.route(api, (r) => r.fulfill({ json: { data: [...RUNS, ...FIELD], total: RUNS.length + FIELD.length } }))
  await serve(page, { '/v1/models': LIST })
  for (const [w, h] of [[390, 844], [1440, 900], [1920, 1080]] as const) {
    await page.setViewportSize({ width: w, height: h })
    // A fresh document each time: the same address again would only move to its anchor.
    await page.goto('about:blank')
    await page.goto(`${server.url}/benchmarks#${SERVED.board.id}`)
    const suite = page.locator(`[id="${SERVED.board.id}"]`)
    await expect(suite).toContainText(SERVED.board.title, { ignoreCase: true, timeout: 15_000 })
    await expect(page.locator('#lead [data-slot="lead-tiles"] > [aria-hidden]')).toHaveCount(0)
    await expect(page.locator('#enso [data-board] tbody tr').first()).toBeAttached()
    // The title is on the first screen and nothing is drawn over it.
    await expect
      .poll(
        () =>
          suite.evaluate((s, h) => {
            const title = s.querySelector('h3, h2')!
            const r = title.getBoundingClientRect()
            const at = document.elementFromPoint(r.left + 8, r.top + r.height / 2)
            return r.top >= 0 && r.bottom <= h && !!at && title.contains(at)
          }, h),
        { message: `at ${w}, the suite's title on screen and uncovered`, timeout: 10_000 },
      )
      .toBe(true)
  }
})

test('a reader who scrolls away before the page has settled is left where they are, before the page runs and after', async ({ page }) => {
  await page.route(api, (r) => r.fulfill({ json: { data: [...RUNS, ...FIELD], total: RUNS.length + FIELD.length } }))
  await page.setViewportSize({ width: 1440, height: 900 })
  const suite = page.locator(`[id="${SERVED.board.id}"]`)
  const away = async () => {
    // The answers are in and the suite drawn; the reader is still past it.
    await expect(suite).toContainText(SERVED.board.title, { ignoreCase: true, timeout: 15_000 })
    await page.waitForTimeout(500)
    expect((await suite.boundingBox())!.y, 'the reader is not brought back').toBeLessThan(-1000)
  }

  // After the page runs: the model list is held, the reader scrolls, then it answers.
  let release: () => void = () => {}
  const held = new Promise<void>((done) => (release = done))
  await serve(page, { '/v1/models': LIST })
  await page.route(
    (u) => u.pathname === '/v1/models',
    async (r) => {
      await held
      await r.fulfill({ json: LIST })
    },
  )
  await page.goto(`${server.url}/benchmarks#${SERVED.board.id}`)
  await page.locator('html[data-hydrated]').waitFor({ state: 'attached' })
  await expect(page.locator('#enso [data-board] tbody tr').first()).toBeAttached({ timeout: 15_000 })
  // To the foot of the page by key, which no chart on the way takes for itself.
  await page.keyboard.press('End')
  await page.waitForTimeout(1500)
  release()
  await away()

  // Before the page runs: its script is held, the reader scrolls the drawn page, then it runs.
  await page.goto('about:blank')
  let run: () => void = () => {}
  const script = new Promise<void>((done) => (run = done))
  let holds = 0
  // The page's own scripts: Turbopack names chunks by their content, so they are
  // the ones /benchmarks loads and a page with none of its own (/about) does not.
  const scripts = (html: string) => [...html.matchAll(/<script\b[^>]*\bsrc="([^"]+\.js)"/g)].map((m) => m[1])
  const shared = new Set(scripts(read('about.html')))
  const own = scripts(read('benchmarks.html')).filter((src) => !shared.has(src))
  expect(own.length, 'the page has scripts of its own').toBeGreaterThan(0)
  await page.route((u) => own.includes(u.pathname), async (r) => {
    holds += 1
    await script
    await r.continue()
  })
  await page.goto(`${server.url}/benchmarks#${SERVED.board.id}`, { waitUntil: 'domcontentloaded' })
  await expect(suite).toBeAttached()
  await page.waitForTimeout(800)
  await page.keyboard.press('End')
  await page.waitForTimeout(1500)
  expect(holds, 'the page script was held while the reader scrolled').toBeGreaterThan(0)
  run()
  await away()
})

test('an address whose anchor is not well formed draws the page', async ({ page }) => {
  await runs(page)
  await serve(page, { '/v1/models': LIST })
  const errors: string[] = []
  page.on('pageerror', (e) => errors.push(e.message))
  for (const hash of ['%', `${SERVED.board.id}%`]) {
    await page.goto('about:blank')
    await page.goto(`${server.url}/benchmarks#${hash}`)
    await expect(page.locator(`[id="${SERVED.board.id}"]`)).toContainText(SERVED.board.title, { ignoreCase: true, timeout: 15_000 })
  }
  expect(errors).toEqual([])
})

test('/benchmarks\' lead holds its places from first paint, so its figures arriving move nothing, at 390 and 1440', async ({ page }) => {
  await page.route(api, (r) => r.fulfill({ json: { data: [...RUNS, ...FIELD], total: RUNS.length + FIELD.length } }))
  await serve(page, { '/v1/models': LIST })
  await page.addInitScript(() => {
    const w = window as unknown as { shift: number }
    w.shift = 0
    new PerformanceObserver((list) => {
      for (const e of list.getEntries() as (PerformanceEntry & { value: number; hadRecentInput: boolean })[]) if (!e.hadRecentInput) w.shift += e.value
    }).observe({ type: 'layout-shift', buffered: true })
  })
  for (const [w, h] of [[390, 844], [1440, 900]] as const) {
    await page.setViewportSize({ width: w, height: h })
    await page.goto('about:blank')
    await page.goto(`${server.url}/benchmarks`)
    await expect(page.locator('#lead [data-slot="lead-tiles"] > [aria-hidden]')).toHaveCount(0, { timeout: 15_000 })
    await expect(page.locator('#lead [data-slot="lead-tiles"] a')).not.toHaveCount(0)
    await page.waitForTimeout(500)
    const shift = await page.evaluate(() => (window as unknown as { shift: number }).shift)
    expect(shift, `layout shift at ${w}`).toBeLessThan(0.05)
  }
})

test('at 390 the suite fits and nothing scrolls sideways', async ({ page }) => {
  await runs(page)
  await serve(page, { '/v1/models': LIST })
  await page.setViewportSize({ width: 390, height: 844 })
  for (const { path } of PAGES) {
    await page.goto(server.url + path)
    const suite = page.locator(`[id="${SERVED.board.id}"]`)
    await expect(suite).toContainText(SERVED.board.title, { ignoreCase: true, timeout: 15_000 })
    const box = await suite.boundingBox()
    expect(box && box.x >= 0 && box.x + box.width <= 390, `${path}: the suite inside the viewport`).toBe(true)
    const wide = await page.evaluate(() => {
      document.documentElement.style.overflowX = 'visible'
      document.body.style.overflowX = 'visible'
      return document.documentElement.scrollWidth - document.documentElement.clientWidth
    })
    expect(wide, `${path} scrolls sideways`).toBeLessThanOrEqual(0)
  }
})