hanzo.ai/e2e/gates/benchmarks.spec.ts

// The benchmark pages say what was measured, and link where it was measured.
//
// A number on these pages is read from lib/data/*.json, so the claim worth a
// test is that the export renders the row the data holds — the engine beside
// the floor — that a sentence counting the data counts what is there, and that
// "reproduce this" points at a repository that exists.

import { test, expect, type Page, type Route } from '@playwright/test'
import { serveExport } from './export'
import { serve } from './api'
import { BENCHMARKS } from '../../lib/benchmarks'
import { BOARD, INDEX, KAI, RUNTIME, type Board, type Part } from '../../lib/bench-data'
import { kaiHeadline, kaiTiles, record } from '../../lib/tally'
import { derive, named, type Run } from '../../lib/kai'
import snapshot from '../../lib/data/benchmarks-kai.json' with { type: 'json' }
import { api, newer } from './revision'
import { block } from './api'
import { RUNS as ENGINE_RUNS, wins } from '../../lib/engine'
import { TOOLS, derive as handlers, runs as address, type Board as Tool } from '../../lib/tools'
import tools from '../../lib/data/benchmarks-tools.json' with { type: 'json' }

type Metric = number | { mean: number } | null
const pct = (m: Metric) => (m == null ? '–' : ((typeof m === 'number' ? m : m.mean) * 100).toFixed(1))

let site: Awaited<ReturnType<typeof serveExport>>
test.beforeAll(async () => {
  site = await serveExport()
})
test.afterAll(async () => {
  await site.close()
})

/**
 * The page's own <main>: the kit renders one inside the layout's, and the innermost is the page. Its script has run
 * (`html[data-hydrated]`, app/providers) before a test's waits start, so they time the page and not a slow runner's boot.
 */
const open = async (page: Page, path: string) => {
  await page.goto(site.url + path)
  await page.locator('html[data-hydrated]').waitFor({ state: 'attached' })
  return page.locator('main').last()
}
const words = async (page: Page, path: string) => ((await (await open(page, path)).innerText()).replace(/\s+/g, ' '))

// A page nobody links to is a page nobody reads. The index's cards and lists
// are written by hand, so a benchmark can ship, be in the registry, have its
// own page, and still be reachable only by typing the URL — which is how
// /benchmarks/locomo-conv sat unlinked. This holds the registry and the index
// to each other.
test('every benchmark in the registry is linked from the index', async ({ page }) => {
  await open(page, '/benchmarks')
  const hrefs = await page.locator('a[href^="/benchmarks/"]').evaluateAll((as) =>
    as.map((a) => a.getAttribute('href') ?? ''),
  )
  const linked = new Set(hrefs.map((h) => h.split('#')[0].replace(/\/$/, '')))
  const missing = BENCHMARKS.map((b) => b.slug)
    .filter((slug, i, all) => all.indexOf(slug) === i)
    .filter((slug) => !linked.has(`/benchmarks/${slug}`))
  expect(missing, `benchmarks with no link on /benchmarks: ${missing.join(', ')}`).toEqual([])
})

test.describe('/benchmarks/longmemeval', () => {
  test('shows the engine beside the cosine floor', async ({ page }) => {
    await open(page, '/benchmarks/longmemeval')
    await expect(page.locator('h1')).toHaveCount(1)

    const table = page.locator('table').first()
    await expect(table.locator('th', { hasText: 'ALL@5 engine' })).toBeVisible()
    await expect(table.locator('th', { hasText: 'ALL@5 cosine' })).toBeVisible()

    const all = table.locator('tr', { hasText: 'all questions' })
    await expect(all).toContainText(pct(INDEX.lme.ours))
    await expect(all).toContainText(pct(INDEX.lme.base))

    await expect(page.getByText('There is no Hanzo configuration in this table')).toHaveCount(0)
  })

  test('links the harness where it lives', async ({ page }) => {
    await open(page, '/benchmarks/longmemeval')
    const harness = page.locator('a[href*="github.com/hanzoai/"]')
    await expect(harness.first()).toHaveAttribute('href', /github\.com\/hanzoai\/benchmarks/)
    await expect(page.locator('a[href*="hanzoai/cloud/tree"]')).toHaveCount(0)
  })
})

test.describe('/benchmarks/cronquestions', () => {
  test('shows the graph API beside Semantica and the control', async ({ page }) => {
    await open(page, '/benchmarks/cronquestions')
    await expect(page.locator('h1')).toHaveCount(1)
    const table = page.locator('table').first()
    for (const row of ['control: the plan over dictionaries', 'Semantica 0.7.0', 'Hanzo graph, through the API']) {
      await expect(table.locator('tr', { hasText: row })).toContainText('99.9')
    }
  })
})

test.describe('/benchmarks', () => {
  // The index quotes rows from the run files through INDEX; the render has to
  // carry the value the file holds, in the section that quotes it, not a figure
  // typed beside it or the same digits from a neighbouring panel.
  test('renders the figures the run files hold', async ({ page }) => {
    await open(page, '/benchmarks')
    const { mab, locomo, lme, code, runtime } = INDEX
    const at: [string, Metric[]][] = [
      ['#memory', [mab.single, mab.multi, mab.largest, locomo.all, locomo.recall, lme].flatMap((m) => [m.base, m.ours] as Metric[])],
      ['#code', [code.first.base, code.first.ours]],
    ]
    for (const [section, ms] of at) for (const m of ms) await expect(page.locator(section)).toContainText(pct(m))
    for (const s of [runtime.dormant, runtime.heap, runtime.spawn, runtime.wake]) await expect(page.locator('#runtime')).toContainText(s!.ours!)
  })

  // Kai's own reads, the tool shortlist and the yes/no table, render the figures INDEX reads from their files.
  test('renders the shortlist and the yes/no rows the files hold', async ({ page }) => {
    await open(page, '/benchmarks')
    const { shortlist, yesno } = INDEX
    for (const g of [shortlist.all, shortlist.hanzo]) for (const r of g!.rows) for (const v of [r.base, r.ours]) await expect(page.locator('#shortlist')).toContainText(pct(v))
    await expect(page.locator('#shortlist')).toContainText(`${shortlist.catalog} tools`)
    await expect(page.locator('#shortlist')).toContainText('not a public benchmark')
    const rows = await page.locator('#kai-yesno tbody tr').evaluateAll((trs) => trs.map((tr) => [...tr.querySelectorAll('td')].map((td) => (td.textContent ?? '').trim())))
    expect(rows).toEqual(yesno.rows.map((r) => [r.label, r.kai, r.jev]))
  })

  // The trolley problem: the public checks Kai leads on and the sealed families, the figures INDEX reads from its file.
  test('renders the trolley rows and the sealed families the file holds', async ({ page }) => {
    await open(page, '/benchmarks')
    const { trolley } = INDEX
    const cells = (id: string) => page.locator(`${id} tbody tr`).evaluateAll((trs) => trs.map((tr) => [...tr.querySelectorAll('td')].map((td) => (td.textContent ?? '').trim())))
    expect(await cells('#kai-trolley')).toEqual(trolley.rows.map((r) => [r.label, r.kai, r.jev, r.luna]))
    expect(await cells('#kai-trolley-sealed')).toEqual(trolley.sealed.families.map((f) => [f.name, f.scenes, f.coverage, f.accuracy, f.wrong]))
    await expect(page.locator('#trolley')).toContainText(`${trolley.sealed.perfect.scenes} in all, was decided right`)
    await expect(page.locator('#trolley')).toContainText(trolley.source.setups)
  })

  // A timing belongs to the pass that took it, and the passes ran on different machines.
  test('names the machine each runtime figure was taken on', async ({ page }) => {
    await open(page, '/benchmarks')
    await expect(page.locator('#runtime p', { hasText: 'goroutine timings' })).toContainText(RUNTIME.passes.goroutine.machine)
  })

  // Each of these was on the index with no run behind it, or against one.
  test('carries none of the withdrawn claims', async ({ page }) => {
    const main = await open(page, '/benchmarks')
    const text = (await main.innerText()).replace(/\s+/g, ' ')
    // A count of twelve would need research runs of the served Kai on HarmBench and the harness; none is posted.
    for (const claim of ['1,800', 'Ranked #1', '12 Frontier', '10×', '1.024', 'Execution Sandboxes', '0.706', 'Zero Model Invocations', "World's Fastest", '12 of 12', '12 / 12', '11 of 12', 'behind on', 'Jailbreak', 'Decision Index v2', 'HarmBench']) {
      expect(text, claim).not.toContain(claim)
    }
  })

  // Enso's figures are its research runs, read when the page opens (e2e/gates/ladder.spec.ts serves them): none ships.
  test('ships no Enso score', async () => {
    const html = await (await fetch(site.url + '/benchmarks')).text()
    for (const typed of ['98.0%', '96.0%', 'Enso Ultra scores', 'GPQA-Diamond · Enso']) expect(html, typed).not.toContain(typed)
  })

  // A price is not a speed: the comparison is labelled as the input rate.
  test('labels the price claim as an input-token price', async ({ page }) => {
    // The price is the price list's, read when the page opens (e2e/gates/api).
    await serve(page)
    const main = await open(page, '/benchmarks')
    const tiles = main.locator('a[href="/pricing"]', { hasText: 'Input-token price' })
    await expect(tiles).not.toHaveCount(0)
    for (const t of await tiles.all()) {
      await expect(t).toContainText('not a cost per task')
      expect((await t.innerText()).toLowerCase()).not.toMatch(/\b(faster|fastest)\b/)
    }
  })
})

// Kai's section renders the capture: the headline its data turns on, every
// figure it claims, every row of every table with each cell as the data prints
// it and its best marked, and a link to the commit each table is from.
const rendered = async (page: Page, b: Board) => {
  const table = page.locator(`[id="${b.id}"] table`)
  // Read when the page opens: the table is drawn once the runs are.
  await expect(table.locator('tbody tr').first()).toBeAttached({ timeout: 15_000 })
  const rows = await table.locator('tbody tr').evaluateAll((trs) =>
    trs.map((tr) => [...tr.querySelectorAll('td')].map((td) => (td.textContent ?? '').trim())),
  )
  const want = b.blocks.flatMap((block) => [
    ...(block.title ? [[block.title]] : []),
    ...block.lines.map((l) => [l.label, ...l.cells.map((c) => c.text)]),
  ])
  expect(rows, b.id).toEqual(want)
  await expect(page.locator(`[id="${b.id}"] a[href="${b.cite.href}"]`)).toHaveCount(1)
}

test.describe('/benchmarks, Kai', () => {
  test.beforeEach(async ({ page }) => {
    // The runs the page reads when it opens, as the snapshot holds them, so KAI is what it draws.
    await page.route(api, (r) => r.fulfill({ json: { data: snapshot.data, total: snapshot.data.length } }))
    // No model list: the Decision Suite states no count, so the harness's record leads (e2e/gates/suite.spec.ts reads the other case).
    await block(page)
  })

  // The hero counts the published index, every row a research run of the Kai revision it names and of Jev.
  test('leads with the record the runs hold', async ({ page }) => {
    const main = await open(page, '/benchmarks')
    const v2 = record(KAI.index.wins)
    expect(v2.total).toBeGreaterThan(0)
    await expect(main.locator('#kai-title')).toHaveText(kaiHeadline(v2))
    const [won] = kaiTiles(v2, named(KAI.run))
    const hero = main.locator('#lead')
    await expect(hero.locator('a[href="#kai"]', { hasText: won.label })).toContainText(`research run of ${named(KAI.run)} and of Jev`)
    await expect(hero.locator('a[href="#kai"]', { hasText: won.label })).toContainText(won.figure)
    for (const t of BOARD.also) await expect(hero.locator(`a[href="${t.href}"]`, { hasText: t.label })).toContainText(t.figure)
    for (const r of KAI.index.wins) await expect(main.locator('#kai')).toContainText(r.label)
    // The release claim is the gate's to make.
    if (!KAI.sota) await expect(main.locator('#kai')).not.toContainText('State of the art')
  })

  test('renders every table as the capture prints it, each citing its commit', async ({ page }) => {
    await open(page, '/benchmarks')
    const boards = Object.values(KAI.boards).filter((b) => b != null)
    for (const b of boards) await rendered(page, b)
    // The harness links its frozen commit; every other board links the run it read.
    for (const b of boards) {
      expect(b.cite.href).toMatch(/^https:\/\/(github\.com\/hanzoai\/benchmarks\/(blob|tree)\/[0-9a-f]{40}\/|api\.hanzo\.ai\/v1\/research\/runs\?org=hanzo&id=.+)/)
    }
  })

  test('an API that does not answer is said to, and no earlier figure stands in', async ({ page }) => {
    await page.unroute(api)
    await page.route(api, (r) => r.abort('connectionrefused'))
    const main = await open(page, '/benchmarks')
    await expect(main.locator('#kai [data-api="unreachable"]').first()).toBeVisible({ timeout: 15_000 })
    await expect(main.locator('#leaderboard [role="listitem"]')).toHaveCount(0)
    await expect(main.locator(`[id="${KAI.boards.harness.id}"]`)).toHaveCount(0)
  })

  test('a revision the API holds is the one drawn, hero and section alike', async ({ page }) => {
    const later = newer(snapshot.data as unknown as Run[])
    const want = derive(later)
    await page.unroute(api)
    await page.route(api, (r) => r.fulfill({ json: { data: later, total: later.length } }))
    const main = await open(page, '/benchmarks')
    const v2 = record(want.index.wins)
    await expect(main.locator('#kai-title')).toHaveText(kaiHeadline(v2))
    const [won] = kaiTiles(v2, named(want.run))
    await expect(main.locator('#lead a[href="#kai"]', { hasText: won.label })).toContainText(won.figure)
    // The headline reads the same before and after; the revised cell is what says the answer arrived.
    await expect(page.locator('[id="kai-harness"] table')).toContainText(want.boards.harness.blocks[0].lines[0].cells[0].text)
    for (const b of Object.values(want.boards).filter((x) => x != null)) await rendered(page, b)
  })
})

// The tools board renders each table as the runs derive it: every head, every
// row with what decides it and each cell as its run prints it, the split's
// definition, and a link to the tool's runs. Each tool's address is answered
// here, so the page renders what the test hands it.
test.describe('/benchmarks, agent tools', () => {
  const asked = (u: URL) => TOOLS.some((t) => u.href === address(t.id))
  const answer = (runs: Run[]) => (r: Route) => {
    const data = runs.filter((x) => x.benchmark === new URL(r.request().url()).searchParams.get('benchmark'))
    return r.fulfill({ json: { data, total: data.length } })
  }
  const drawn = async (page: Page, boards: Tool[]) => {
    await expect(page.locator('#tools h3')).toHaveText(boards.map((t) => t.title))
    for (const t of boards) {
      for (const s of t.splits) {
        const block = page.locator(`[id="tools-${t.slug}-${s.split}"]`)
        await expect(block).toContainText(s.definition)
        await expect(block.locator('thead th')).toHaveText(['Handler', 'What decides', ...s.heads])
        const rows = await block.locator('tbody tr').evaluateAll((trs) => trs.map((tr) => [...tr.querySelectorAll('td')].map((td) => (td.textContent ?? '').trim())))
        expect(rows, `${t.id} ${s.split}`).toEqual(s.rows.map((r) => [r.label, r.decides, ...r.cells]))
        await expect(block.locator(`a[href="${t.href}"]`)).toHaveCount(1)
      }
    }
  }

  test('draws every tool as its runs hold it, each table linking them', async ({ page }) => {
    // The tools whose current handler holds against the one it replaced, in TOOLS order (lib/tools); the rest are not drawn.
    expect(INDEX.tools.map((t) => t.id)).toEqual(TOOLS.filter((t) => INDEX.tools.some((x) => x.id === t.id)).map((t) => t.id))
    expect(INDEX.tools.length).toBeGreaterThan(0)
    await page.route(asked, answer(tools.data as unknown as Run[]))
    await open(page, '/benchmarks')
    await drawn(page, INDEX.tools)
  })

  test('a run the API holds replaces the snapshot', async ({ page }) => {
    const runs = tools.data as unknown as Run[]
    const [first] = INDEX.tools[0].splits[0].rows
    const run = runs.find((r) => r.id === first.id)!
    const later = [...runs, { ...run, id: `${run.id}-later`, ended: '2999-01-01T00:00:00Z', measures: (run.measures ?? []).map((m) => ({ ...m, value: m.value / 2 })) }]
    const want = handlers(later)
    expect(want[0].splits[0].rows[0].cells).not.toEqual(first.cells)
    await page.route(asked, answer(later))
    await open(page, '/benchmarks')
    await expect(page.locator(`[id="tools-${want[0].slug}-${want[0].splits[0].split}"] tbody tr`).first()).toContainText(want[0].splits[0].rows[0].cells[0])
    await drawn(page, want)
  })
})

for (const part of ['fleet', 'goroutine', 'sandbox'] as Part[]) {
  test(`/benchmarks/${part} names the machine its pass ran on`, async ({ page }) => {
    await expect(await open(page, `/benchmarks/${part}`)).toContainText(RUNTIME.passes[part].machine)
  })
}

test.describe('/benchmarks/inference', () => {
  // The page publishes the harness's wins and nothing else: every cell a settled WIN, its interval wholly above parity.
  test('renders exactly the winning cells, and counts the runs they come from', async ({ page }) => {
    expect(wins.length, 'winning cells').toBeGreaterThan(0)
    for (const c of wins) {
      expect(c.verdict, c.run).toBe('WIN')
      expect(c.noisy, c.run).toBe(false)
      expect(c.lo, c.run).toBeGreaterThan(1)
    }
    const main = await open(page, '/benchmarks/inference')
    await expect(page.locator('h1')).toHaveText('Engine prefill on CUDA vs llama.cpp')
    const rows = await main.locator('table').first().locator('tbody tr').evaluateAll((trs) => trs.map((tr) => [...tr.querySelectorAll('td')].map((td) => (td.textContent ?? '').trim())))
    expect(rows).toHaveLength(wins.length)
    for (const r of rows) expect(r.at(-1)).toBe('WIN')
    const reps = [...new Set(ENGINE_RUNS.filter((r) => wins.some((c) => c.run === r.run)).map((r) => r.reps))].sort((a, b) => a - b)
    const text = await words(page, '/benchmarks/inference')
    for (const [, n] of text.matchAll(/\b(\w+(?: or \w+)*) repetitions\b/g)) expect(n, 'repetitions').toBe(reps.join(' or '))
  })
})

test.describe('/benchmarks/gpqa', () => {
  // Every row is a research run, read when the page opens (e2e/gates/ladder.spec.ts serves them); the page says so and types no score.
  test('says every row is a run, and types no score', async ({ page }) => {
    const main = await open(page, '/benchmarks/gpqa')
    await expect(main.locator('p', { hasText: 'A row is a run, as of its date.' })).toHaveCount(1)
    const html = await (await fetch(site.url + '/benchmarks/gpqa')).text()
    // The figures it once transcribed, which no run holds.
    for (const typed of ['194, 190 and 184', '98.0', '96.0']) expect(html, typed).not.toContain(typed)
  })
})