hanzo.ai/e2e/gates/leaderboard.spec.ts

// /benchmarks' leaderboard and frontier, driven the way a reader drives them, over the export.
//
// Each control re-sorts, re-filters or re-draws figures handed in from the data,
// so the claims worth a test are that it does what its label says, that the view
// survives a copied link, that an export carries exactly the rows on screen, and
// that the page holds at every width with nothing in the console.

import { test, expect, type Page } from '@playwright/test'
import { readFileSync } from 'node:fs'
import { serveExport } from './export'
import { serve } from './api'
import { api } from './revision'
import { KAI } from '../../lib/bench-data'
import { frontier, record, verdict } from '../../lib/tally'
import { derive as enso } from '../../lib/enso'
import { FIELD, PRICED } from './field'
import { toPricing } from '../../lib/api/read'
import { BENCH } from '../../lib/coding'
import { TOOLS, runs } from '../../lib/tools'
import snapshot from '../../lib/data/benchmarks-kai.json' with { type: 'json' }
import coding from '../../lib/data/benchmarks-coding.json' with { type: 'json' }
import tools from '../../lib/data/benchmarks-tools.json' with { type: 'json' }

/** The coding board's runs, on whichever origin the page asks them of (lib/coding `runs`). */
const CODING = (u: URL): boolean => u.pathname === '/v1/research/runs' && u.searchParams.get('benchmark') === BENCH

let site: Awaited<ReturnType<typeof serveExport>>
test.beforeAll(async () => {
  site = await serveExport()
})
test.afterAll(async () => {
  await site.close()
})

/** Enso's frontier as the page derives it from the invented runs served beside the snapshot, at the price list served with them (./field). */
const FRONTIER = enso(FIELD, toPricing(PRICED))!.frontier

/** The research API answers every board's address with its snapshot, and Enso's with the invented runs, so every request succeeds. */
const open = async (page: Page, query = '') => {
  // The prices beside the runs are the price list's, read when the page opens (e2e/gates/api), with a row for each of the field's models (./field).
  await serve(page, { '/v1/pricing': PRICED })
  const data = [...snapshot.data, ...FIELD]
  await page.route(api, (r) => r.fulfill({ json: { data, total: data.length } }))
  await page.route(CODING, (r) => r.fulfill({ json: { data: coding.data, total: coding.data.length } }))
  for (const { id } of TOOLS) {
    const data = tools.data.filter((run) => run.benchmark === id)
    await page.route((u) => u.href === runs(id), (r) => r.fulfill({ json: { data, total: data.length } }))
  }
  // The event stream's batch is answered as the backend answers it: a slow run that flushes one before its end
  // otherwise reads a 404 from the export server into the console.
  await page.route((u) => u.pathname === '/v1/event', (r) => r.fulfill({ json: {} }))
  // The board asks the research API from an effect, so the request is the sign the page has hydrated.
  const hydrated = page.waitForRequest((r) => api(new URL(r.url())))
  await page.goto(`${site.url}/benchmarks${query}`)
  await hydrated
  return page.locator('main').last()
}
const board = (page: Page) => page.locator('#leaderboard')
const press = (page: Page, scope: string, group: string, name: string) =>
  page.locator(scope).getByRole('group', { name: group }).getByRole('button', { name, exact: true })
/** Press a control until it reports itself pressed: a click that lands before hydration is a click on static HTML. */
const choose = async (page: Page, scope: string, group: string, name: string) => {
  const b = press(page, scope, group, name)
  await expect(async () => {
    await b.click()
    await expect(b).toHaveAttribute('aria-pressed', 'true', { timeout: 500 })
  }).toPass()
}
const suites = (page: Page) =>
  board(page).locator('[role="listitem"] > :first-child > :first-child').evaluateAll((els) => els.map((e) => (e.textContent ?? '').replace(/ · board count$/, '').trim()))
const query = (page: Page) => new URL(page.url()).searchParams

test.describe('/benchmarks, the leaderboard', () => {
  test('sorts by the largest lead first, and by name on request', async ({ page }) => {
    await open(page)
    const rows = KAI.index.wins
    const byLead = [...rows].sort((a, b) => (b.kai! - b.jev!) - (a.kai! - a.jev!)).map((r) => r.label)
    await expect.poll(() => suites(page)).toEqual(byLead)
    await choose(page, '#leaderboard', 'Sort', 'Name')
    await expect.poll(() => suites(page)).toEqual(rows.map((r) => r.label).sort((a, b) => a.localeCompare(b)))
    expect(query(page).get('sort')).toBe('name')
    await expect(press(page, '#leaderboard', 'Sort', 'Name')).toHaveAttribute('aria-pressed', 'true')
  })

  test('the board lists the suites Kai leads, and only those', async ({ page }) => {
    await open(page)
    await expect(board(page).getByRole('group', { name: 'Index' })).toHaveCount(0)
    await choose(page, '#leaderboard', 'View', 'Table')
    await expect(board(page).locator('table tbody tr')).toHaveCount(KAI.index.wins.length)
    const v2 = record(KAI.index.wins)
    await expect(board(page).locator('p', { hasText: 'suites won' })).toContainText(`${v2.won} suites won`)
    await expect(board(page).locator('table tbody')).not.toContainText('Loss')
  })

  test('calibration error is drawn where both models report it', async ({ page }) => {
    await open(page)
    await choose(page, '#leaderboard', 'Metric', 'Calibration error')
    const scored = KAI.index.wins.filter((r) => verdict(r, 'ece') === 'won')
    await expect.poll(() => suites(page).then((s) => s.length)).toBe(scored.length)
    await expect(board(page)).toContainText(scored[0].kaiEce!.toFixed(3))
  })

  test('a copied link opens the view it was copied from', async ({ page }) => {
    await open(page, '?view=table&sort=name&metric=acc')
    await expect(press(page, '#leaderboard', 'View', 'Table')).toHaveAttribute('aria-pressed', 'true')
    await expect(press(page, '#leaderboard', 'Sort', 'Name')).toHaveAttribute('aria-pressed', 'true')
    const first = await board(page).locator('tbody tr td').first().textContent()
    expect(first).toBe(KAI.index.wins.map((r) => r.label).sort((a, b) => a.localeCompare(b))[0])
  })

  test('exports the rows on screen, as CSV and as JSON', async ({ page }) => {
    await open(page)
    const [csv] = await Promise.all([page.waitForEvent('download'), board(page).getByRole('button', { name: 'CSV' }).click()])
    expect(csv.suggestedFilename()).toBe('kai-vs-jev-acc.csv')
    const lines = readFileSync((await csv.path())!, 'utf8').trim().split('\n')
    expect(lines[0]).toBe('suite,kai_accuracy,jev_accuracy,kai_ece,jev_ece,lead,verdict,source,kai_measured,jev_measured')
    expect(lines).toHaveLength(KAI.index.wins.length + 1)
    const [json] = await Promise.all([page.waitForEvent('download'), board(page).getByRole('button', { name: 'JSON' }).click()])
    const rows = JSON.parse(readFileSync((await json.path())!, 'utf8')) as { suite: string; kai_accuracy: number }[]
    expect(rows.map((r) => r.suite).sort()).toEqual(KAI.index.wins.map((r) => r.label).sort())
    for (const r of rows) expect(r.kai_accuracy).toBe(KAI.index.wins.find((x) => x.label === r.suite)!.kai)
  })

  test('copy link puts the current view on the clipboard', async ({ page, context }) => {
    await context.grantPermissions(['clipboard-read', 'clipboard-write'], { origin: site.url })
    await open(page)
    await choose(page, '#leaderboard', 'Sort', 'Jev')
    await board(page).getByRole('button', { name: 'Copy link' }).click()
    await expect(board(page).getByRole('button', { name: 'Link copied' })).toBeVisible()
    expect(new URL(await page.evaluate(() => navigator.clipboard.readText())).searchParams.get('sort')).toBe('jev')
  })
})

test.describe('/benchmarks, the frontier', () => {
  const rows = (page: Page) => page.locator('#frontier details tbody tr')

  test('draws one benchmark at a time, and its frontier is the runs’', async ({ page }) => {
    await open(page)
    const first = FRONTIER[0]
    await page.locator('#frontier details summary').click()
    await expect(rows(page)).toHaveCount(first.points.length)
    const efficient = await rows(page).evaluateAll((trs) => trs.filter((tr) => tr.lastElementChild?.textContent === 'yes').map((tr) => tr.firstElementChild?.textContent))
    expect(efficient.sort()).toEqual(frontier(first.points).map((p) => p.model).sort())

    const next = FRONTIER[1]
    await page.locator('#frontier select').selectOption(next.id)
    await expect(rows(page)).toHaveCount(next.points.length)
    expect(query(page).get('bench')).toBe(next.id)
  })

  test('a point opens its record from the keyboard', async ({ page }) => {
    await open(page)
    const ours = FRONTIER[0].points.find((p) => p.ours)!
    await page.locator(`#frontier circle[aria-label^="${ours.model},"]`).focus()
    await expect(page.locator('#frontier')).toContainText(`${ours.shown} · `)
    await expect(page.locator('#frontier')).toContainText('A research run')
  })
})

test.describe('/benchmarks, every width', () => {
  for (const width of [375, 430, 768, 1024, 1440, 1728]) {
    test(`holds at ${width}px with nothing in the console`, async ({ page }) => {
      const errors: string[] = []
      page.on('console', (m) => m.type() === 'error' && errors.push(m.text()))
      page.on('pageerror', (e) => errors.push(String(e)))
      await page.setViewportSize({ width, height: 900 })
      await open(page)
      await expect(page.locator('#leaderboard [role="listitem"]')).toHaveCount(KAI.index.wins.length)
      for (const id of ['enso', 'frontier', 'kai', 'leaderboard', 'zen', 'harness', 'platform', 'methods', 'methodology', 'all']) await page.locator(`#${id}`).scrollIntoViewIfNeeded()
      expect(await page.evaluate(() => document.documentElement.scrollWidth - window.innerWidth), 'sideways scroll').toBeLessThanOrEqual(0)
      expect(errors).toEqual([])
    })
  }

  test('honours reduced motion: nothing on the board animates', async ({ page }) => {
    await page.emulateMedia({ reducedMotion: 'reduce' })
    await open(page)
    const moving = await page.locator('#leaderboard, #frontier').evaluateAll((els) =>
      els.flatMap((el) => [...el.querySelectorAll('*')]).filter((e) => {
        const s = getComputedStyle(e)
        return s.animationName !== 'none' && s.animationDuration !== '0s'
      }).length,
    )
    expect(moving).toBe(0)
  })
})