hanzo.ai/e2e/gates/shelf.spec.ts

// /benchmarks' shelf, story and detail pages, over the export.
//
// The shelf is one ledger (lib/results) searched, faceted, compared and opened.
// The claims worth a test: a facet narrows to exactly the rows the ledger says,
// a search marks what it matched, a view survives its link, two rows of one
// benchmark and metric compare and two of different ones say why not, the
// palette jumps from the keyboard, the chapter nav stays put and says where you
// are, every benchmark opens a page with its history, source and command, and
// the landing, without the model list, states no retired count.

import { test, expect, type Page } from '@playwright/test'
import { serveExport } from './export'
import { api } from './revision'
import { block } from './api'
import { LEDGER } from '../../lib/bench-data'
import snapshot from '../../lib/data/benchmarks-kai.json' with { type: 'json' }

let site: Awaited<ReturnType<typeof serveExport>>
test.beforeAll(async () => {
  site = await serveExport()
})
test.afterAll(async () => {
  await site.close()
})

const open = async (page: Page, path = '/benchmarks') => {
  await page.route(api, (r) => r.fulfill({ json: { data: snapshot.data, total: snapshot.data.length } }))
  // The index asks the research API from an effect, so the request is the sign it has hydrated. A detail page asks nothing.
  const index = /^\/benchmarks(\?|#|$)/.test(path)
  const hydrated = index ? page.waitForRequest((r) => api(new URL(r.url()))) : null
  await page.goto(`${site.url}${path}`)
  await hydrated
}
const shelf = (page: Page) => page.locator('#all')
const rows = (page: Page) => shelf(page).locator('[role="list"][aria-label="Results"] > [role="listitem"]')
const tally = (page: Page) => shelf(page).locator('p[aria-live="polite"]')
const query = (page: Page) => new URL(page.url()).searchParams
/** Press until it takes: a click that lands before hydration is a click on static HTML. */
const until = async (act: () => Promise<void>, check: () => Promise<void>) =>
  expect(async () => {
    await act()
    await check()
  }).toPass()

test.describe('/benchmarks, the shelf', () => {
  test('starts with every result, and every row carries a date or says why it has none', async ({ page }) => {
    await open(page)
    await expect(tally(page)).toContainText(`${LEDGER.results.length} of ${LEDGER.results.length} results`)
    for (const r of LEDGER.results) expect(r.when, r.id).toMatch(/\d{4}|undated/)
    expect(LEDGER.results.filter((r) => !r.date).every((r) => r.when.includes('undated'))).toBe(true)
  })

  test('the shelf publishes wins and standalone measurements, nothing that trails', async ({ page }) => {
    for (const r of LEDGER.results) expect(['win', 'unscored'], r.id).toContain(r.standing)
    await open(page)
    await expect(shelf(page).getByRole('group', { name: 'Result' }).getByRole('button', { name: /^Loss/ })).toHaveCount(0)
  })

  test('a facet narrows to exactly the rows the ledger holds, and the URL keeps it', async ({ page }) => {
    await open(page)
    const wins = LEDGER.results.filter((r) => r.standing === 'win')
    const chip = shelf(page).getByRole('group', { name: 'Result' }).getByRole('button', { name: /^Win/ })
    await until(() => chip.click(), () => expect(tally(page)).toContainText(`${wins.length} of`, { timeout: 500 }))
    expect(query(page).getAll('result')).toEqual(['win'])
    await expect(chip).toHaveAttribute('aria-pressed', 'true')
    const ids = await rows(page).evaluateAll((els) => els.map((e) => e.getAttribute('data-id')))
    for (const id of ids) expect(wins.map((r) => r.id)).toContain(id)
  })

  test('two facets combine, and a value with a comma in it survives the URL', async ({ page }) => {
    const category = LEDGER.results.find((r) => r.category.includes(','))!.category
    await open(page, `/benchmarks?category=${encodeURIComponent(category)}&state=frozen#all`)
    const want = LEDGER.results.filter((r) => r.category === category && r.state === 'frozen')
    await expect(tally(page)).toContainText(`${want.length} of ${LEDGER.results.length}`)
  })

  test('search narrows instantly and marks what it matched', async ({ page }) => {
    await open(page)
    const box = shelf(page).getByRole('searchbox', { name: 'Search every benchmark and run' })
    const want = LEDGER.results.filter((r) => `${r.name} ${r.subject}`.toLowerCase().includes('banking'))
    expect(want.length).toBeGreaterThan(0)
    await until(() => box.fill('banking'), () => expect(rows(page).first().locator('mark').first()).toBeVisible({ timeout: 500 }))
    await expect(tally(page)).toContainText(`${want.length} of`)
    expect(query(page).get('q')).toBe('banking')
  })

  test('a saved view is a link, and opens as it was saved', async ({ page }) => {
    await open(page)
    const view = shelf(page).getByRole('link', { name: 'Superseded runs' })
    await view.click()
    await page.waitForURL(/state=superseded/)
    // The view opens as a page of its own; its script has run before the wait below starts, so the wait times the shelf.
    await page.locator('html[data-hydrated]').waitFor({ state: 'attached' })
    const want = LEDGER.results.filter((r) => r.state === 'superseded')
    await expect(tally(page)).toContainText(`${want.length} of`)
    await expect(shelf(page).getByRole('group', { name: 'Status' }).getByRole('button', { name: /^Superseded/ })).toHaveAttribute('aria-pressed', 'true')
  })

  test('two revisions of one suite compare; rows of different benchmarks say why not', async ({ page }) => {
    const same = LEDGER.results.filter((r) => r.entry === 'kai-banking77' && r.metric === 'Accuracy').slice(0, 2)
    const other = LEDGER.results.find((r) => r.entry === 'kai-ag-news' && r.metric === 'Accuracy')!
    await open(page, `/benchmarks?q=banking#all`)
    for (const r of same) {
      const box = shelf(page).locator(`[data-id="${r.id}"]`).getByRole('checkbox')
      await until(() => box.click(), () => expect(box).toHaveAttribute('aria-checked', 'true', { timeout: 500 }))
    }
    const tray = page.getByRole('region', { name: 'Compare' })
    await expect(tray).toContainText('2 picked')
    expect(query(page).getAll('pick').sort()).toEqual(same.map((r) => r.id).sort())
    await tray.getByRole('button', { name: 'Compare' }).click()
    for (const r of same) await expect(tray).toContainText(r.shown)

    await page.goto(`${site.url}/benchmarks?pick=${same[0].id}&pick=${other.id}#all`)
    const again = page.getByRole('region', { name: 'Compare' })
    await until(() => again.getByRole('button', { name: 'Compare' }).click(), () => expect(again).toContainText('one benchmark and one metric', { timeout: 500 }))
  })

  test('exports what is on the shelf', async ({ page }) => {
    await open(page, '/benchmarks?product=Zen%206#all')
    const zen = LEDGER.results.filter((r) => r.product === 'Zen 6')
    await expect(tally(page)).toContainText(`${zen.length} of`)
    const [json] = await Promise.all([page.waitForEvent('download'), shelf(page).getByRole('button', { name: 'JSON' }).click()])
    const got = JSON.parse(await (await import('node:fs')).promises.readFile((await json.path())!, 'utf8')) as { product: string; date: string | null }[]
    expect(got).toHaveLength(zen.length)
    expect(got.every((r) => r.product === 'Zen 6')).toBe(true)
  })
})

test.describe('/benchmarks, the story', () => {
  test('reads lead, then Enso, Kai, Zen 6, the agent harness and the agent platform, then methods and the shelf', async ({ page }) => {
    await open(page)
    const order = await page.locator('main section[id]').evaluateAll((els) => els.map((e) => e.id))
    const chapters = ['lead', 'enso', 'kai', 'zen', 'harness', 'platform', 'methods', 'all']
    expect(order.filter((id) => chapters.includes(id))).toEqual(chapters)
  })

  test('the chapter nav stays under the header and marks the chapter in view', async ({ page }) => {
    await page.setViewportSize({ width: 1440, height: 900 })
    await open(page)
    const nav = page.getByRole('navigation', { name: 'Benchmark chapters' })
    // Kai's chapter is drawn once its runs are read; scroll after, or the chapter grows under the reader.
    await expect(page.locator('#leaderboard [role="listitem"]').first()).toBeVisible({ timeout: 15_000 })
    await page.evaluate(() => document.getElementById('zen')!.scrollIntoView({ block: 'start' }))
    await expect(nav.getByRole('link', { name: 'Zen' })).toHaveAttribute('aria-current', 'location')
    const box = await nav.boundingBox()
    expect(box!.y).toBeGreaterThanOrEqual(0)
    expect(box!.y).toBeLessThan(120)
  })

  test('the palette opens on / and jumps to any benchmark, leaving ⌘K to the site search', async ({ page }) => {
    await open(page)
    const entry = LEDGER.entries.find((e) => e.slug === 'kai-choice-size')!
    await until(
      () => page.keyboard.press('/'),
      () => expect(page.getByRole('dialog', { name: 'Jump to a benchmark' })).toBeVisible({ timeout: 500 }),
    )
    await page.keyboard.type(entry.name)
    await page.keyboard.press('Enter')
    await page.waitForURL(new RegExp(`${entry.href}$`))
    await expect(page.locator('h1')).toHaveText(entry.name)
  })
})

test.describe('/benchmarks/<slug>, a benchmark of its own', () => {
  test('every entry without a hand-written page has one, linked from the index', async ({ page }) => {
    await open(page)
    const hrefs = new Set(await page.locator('#all a[href^="/benchmarks/"]').evaluateAll((as) => as.map((a) => a.getAttribute('href'))))
    for (const e of LEDGER.entries) expect(hrefs, e.slug).toContain(e.href)
  })

  test('a Kai suite carries its history, its conditions, its source and its command', async ({ page }) => {
    const e = LEDGER.entries.find((x) => x.slug === 'kai-banking77')!
    const results = LEDGER.results.filter((r) => r.entry === e.slug)
    await open(page, e.href)
    await expect(page.locator('h1')).toHaveText(e.name)
    await expect(page.locator('main table').first().locator('tbody tr')).toHaveCount(results.length)
    await expect(page.locator('main table').first()).toContainText('Superseded')
    await expect(page.locator('main svg[role="img"]').first()).toBeVisible()
    await expect(page.locator(`main a[href="${e.source!.href}"]`)).toHaveCount(1)
    await expect(page.locator('main code')).toContainText('bench score')
    for (const c of e.conditions) await expect(page.locator('main').last()).toContainText(c)
  })
})

test.describe('the published index leads everywhere', () => {
  test('with no model list, the landing states no retired count', async ({ page }) => {
    // The runs, and no other answer: the Decision Suite's cost row is unread, so its count is not stated (e2e/gates/suite.spec.ts reads the other case).
    await page.route(api, (r) => r.fulfill({ json: { data: snapshot.data, total: snapshot.data.length } }))
    await block(page)
    await page.goto(`${site.url}/`)
    await expect(page.locator('main [data-api="loading"]')).toHaveCount(0, { timeout: 15_000 })
    for (const claim of ['11 of 12', '12 of 12', 'of 16', 'Decision Index v2', 'HarmBench']) await expect(page.locator('body')).not.toContainText(claim)
  })
})