hanzo.ai/e2e/gates/kai.spec.ts

import { test, expect, type Page } from '@playwright/test'
import { serveExport } from './export'
import { KAI } from '../../lib/bench-data'
import { derive, type Board, type Run } from '../../lib/kai'
import { STAGE, api, newer } from './revision'
import { block } from './api'
import snapshot from '../../lib/data/benchmarks-kai.json' with { type: 'json' }

/**
 * /kai, on the bytes that ship.
 *
 * The hero is one decision forming: a state, a question with three options, and
 * the distribution that is Kai's answer, which deforms when the reader changes
 * the one fact that matters. Further down, the program graph: evidence in, typed
 * decisions out. So the gate asks the figures the questions the copy answers —
 * does the answer move with the evidence, does every decision settle, does
 * switching a source off re-run only the decisions that read it, does a
 * decision the evidence cannot carry defer and say what it needs — and then
 * what every page owes: nothing thrown, nothing hidden from a reader who
 * declines motion, and nothing wider than a phone.
 *
 * The numbers asserted are the SCRIPTED ones in components/kai/Instrument.tsx,
 * Surface.tsx and scenario.ts, which is why they can be exact: each figure is an
 * example and says so, and a change to a script is a change to this gate in the
 * same commit.
 *
 * Each family page is about its own family; /models is the one place the
 * three are compared. That is checked here because /kai is the page that
 * introduced the comparison.
 */

let server: { url: string; close: () => Promise<void> }

test.beforeAll(async () => {
  server = await serveExport()
})

test.afterAll(async () => {
  await server.close()
})

/** The drawing the reader is shown — the wide one, or the tall one on a phone. */
const drawing = (page: Page) => page.locator('svg[data-shape]:visible')
const node = (page: Page, id: string) => drawing(page).locator(`[data-node="${id}"]`)
const settled = (page: Page) => expect(page.locator('[data-status]')).toContainText('settled', { timeout: 20_000 })
const instrument = (page: Page) => page.locator('[data-figure="instrument"]')
const option = (page: Page, o: string) => instrument(page).locator(`[data-option="${o}"]`)
/**
 * The page is live: the hero's decision has begun to converge, which only a
 * hydrated page does. A key pressed before then moves the native control and
 * nothing React reads.
 */
const live = (page: Page) => expect(instrument(page)).toHaveAttribute('data-phase', /^(live|still)$/, { timeout: 15_000 })

/** How far the page scrolls sideways once the clip that would hide it is lifted. */
const sideways = (page: Page) =>
  page.evaluate(async () => {
    for (let y = 0; y < document.body.scrollHeight; y += 700) {
      window.scrollTo(0, y)
      await new Promise((r) => setTimeout(r, 30))
    }
    const d = document.documentElement
    d.style.overflowX = 'visible'
    document.body.style.overflowX = 'visible'
    return d.scrollWidth - d.clientWidth
  })

/** A board's rows as rendered, against the rows derived from the data. */
const rows = async (page: Page, b: Board) => {
  const got = await page.locator(`[id="${b.id}"] table tbody tr`).evaluateAll((trs) =>
    trs.map((tr) => [...tr.querySelectorAll('td')].map((td) => (td.textContent ?? '').trim())),
  )
  const want = b.blocks.flatMap((block) => [
    ...(block.title ? [[block.title]] : []),
    ...block.lines.map((l) => [l.label, ...l.cells.map((c) => c.text)]),
  ])
  expect(got, b.id).toEqual(want)
}

test.describe('/kai', () => {
  test('renders the hero: the promise, one decision, and a graph that settles', async ({ page }) => {
    await page.setViewportSize({ width: 1440, height: 900 })
    await page.goto(server.url + '/kai')

    await expect(page.locator('h1')).toHaveText('See. Hear. Understand. When the answer has a type, Decide.')
    // The decision converges to the answer the state carries, and says so in words.
    await expect(option(page, 'review')).toHaveText('.76', { timeout: 5_000 })
    await expect(instrument(page).locator('[data-answer]')).toHaveText('choice · review · p .76')
    await expect(instrument(page)).toContainText('usage.output_tokens · 0')
    await expect(page.getByText('Kai · Multimodal Decision Intelligence')).toBeVisible()
    // The hero's ways in: the API on this page, the early-access track, and the docs.
    const hero = page.locator('main section').first()
    await expect(hero.getByRole('link', { name: 'Get decisions' })).toHaveAttribute('href', '#api')
    await expect(hero.getByRole('link', { name: /Join Kai-1\.7 Early Access/ })).toHaveAttribute('href', '/research-access?track=kai')

    await expect(drawing(page)).toHaveAttribute('data-shape', 'wide')
    await expect(drawing(page).locator('[data-node]')).toHaveCount(12)
    await settled(page)
    await expect(drawing(page).locator('[data-node][data-phase="resolved"]')).toHaveCount(12)
    await expect(node(page, 'fault')).toHaveAttribute('data-p', '.91')
    await expect(node(page, 'readiness')).toHaveAttribute('data-p', '.96')

    // Every figure with a number in it says it is an example: the graph, each
    // typed answer, and the call.
    await expect(page.getByText('example · scripted')).toBeVisible()
    await expect(page.locator('#types [data-tag]')).toHaveCount(3)
    await expect(page.locator('#api [data-tag]')).toHaveCount(2)
    await expect(instrument(page).locator('[data-tag]')).toHaveCount(1)
    await expect(page.locator('[data-figure="surface"] [data-tag]')).toHaveCount(1)
  })

  test('the evidence moves the answer: one fact flips the distribution, and back', async ({ page }) => {
    await page.setViewportSize({ width: 1440, height: 900 })
    await page.goto(server.url + '/kai')
    await live(page)
    const pending = instrument(page).getByRole('button', { name: 'pending' })
    const passed = instrument(page).getByRole('button', { name: 'passed' })
    await expect(pending).toHaveAttribute('aria-pressed', 'true')
    await expect(option(page, 'review')).toHaveText('.76', { timeout: 5_000 })

    await passed.click()
    await expect(passed).toHaveAttribute('aria-pressed', 'true')
    await expect(pending).toHaveAttribute('aria-pressed', 'false')
    // The bars travel; they arrive where the new evidence puts them, still summing to one.
    await expect(option(page, 'continue')).toHaveText('.91')
    await expect(option(page, 'review')).toHaveText('.07')
    await expect(option(page, 'stop')).toHaveText('.02')
    await expect(instrument(page).locator('[data-answer]')).toHaveText('choice · continue · p .91')

    await pending.click()
    await expect(option(page, 'review')).toHaveText('.76')
    await expect(instrument(page).locator('[data-answer]')).toHaveText('choice · review · p .76')
  })

  test('the surface says what would flip the decision', async ({ page }) => {
    await page.setViewportSize({ width: 1440, height: 900 })
    await page.goto(server.url + '/kai')
    await live(page)
    const surface = page.locator('[data-figure="surface"]')
    const flip = surface.locator('[data-flip]')
    // 55 evidence against 50 risk sits under the line: 10 points short of continuing.
    await expect(flip).toHaveText('+10 points of evidence turns review into continue.')
    // From the keyboard, the way a reader without a pointer moves it.
    const evidence = surface.getByRole('slider', { name: 'evidence completeness' })
    await evidence.focus()
    for (let i = 0; i < 10; i++) await page.keyboard.press('ArrowRight')
    await expect(evidence).toHaveValue('65')
    await expect(flip).toHaveText('−1 point of evidence turns continue into review.')
    await page.keyboard.press('End')
    await expect(flip).toHaveText('−36 points of evidence turns continue into review.')
    await surface.getByRole('slider', { name: 'risk' }).focus()
    await page.keyboard.press('End')
    await expect(flip).toHaveText('No amount of evidence clears this risk. It stays with review.')
  })

  test('switching a source off re-runs what reads it, and a decision it carried defers', async ({ page }) => {
    await page.setViewportSize({ width: 1440, height: 900 })
    await page.goto(server.url + '/kai')
    await settled(page)

    const can = page.locator('button[data-source="can"]')
    await expect(can).toHaveAttribute('aria-pressed', 'true')
    await can.click()
    await expect(can).toHaveAttribute('aria-pressed', 'false')
    await settled(page)
    // The decisions CAN carried lose confidence; the ones it never read keep theirs.
    await expect(node(page, 'fault')).toHaveAttribute('data-p', '.71')
    await expect(node(page, 'severity')).toHaveAttribute('data-p', '.71')
    await expect(node(page, 'visible_damage')).toHaveAttribute('data-p', '.89')
    await expect(node(page, 'mission_range')).toHaveAttribute('data-p', '.92')

    // Without the camera, the damage call cannot be made, and it asks for the camera.
    await page.locator('button[data-source="camera"]').click()
    await settled(page)
    await expect(node(page, 'visible_damage')).toHaveAttribute('data-defer', '')
    await expect(node(page, 'visible_damage')).toContainText('defer → camera')
    await expect(page.locator('[data-status]')).toContainText('1 deferred')

    // And back: CAN restored, the camera still off.
    await can.click()
    await settled(page)
    await expect(node(page, 'fault')).toHaveAttribute('data-p', '.86')
  })

  test('throws nothing and logs no error, through load and every control', async ({ page }) => {
    const errors: string[] = []
    page.on('pageerror', (e) => errors.push(`pageerror: ${e.message}`))
    page.on('console', (m) => {
      if (m.type() !== 'error') return
      // A request the export makes to another origin — the identity provider,
      // the pricing feed — is not this page's code, and a local server is not
      // the origin those hosts answer. Nor is `/v1/*`: the chrome's event stream
      // posts there on every page, and a static server serves no API. What is
      // left is the page's own.
      const at = m.location().url
      if (at && !at.startsWith(server.url)) return
      if (at && new URL(at).pathname.startsWith('/v1/')) return
      const text = m.text()
      const away = /(?:Access to \w+ at '|GET |POST )(https?:\/\/[^\s']+)/.exec(text)?.[1]
      if (away && !away.startsWith(server.url)) return
      errors.push(text)
    })
    await page.setViewportSize({ width: 1440, height: 900 })
    await page.goto(server.url + '/kai')
    await settled(page)
    await instrument(page).getByRole('button', { name: 'passed' }).click()
    await instrument(page).getByRole('button', { name: 'pending' }).click()
    await page.locator('[data-figure="surface"]').getByRole('slider', { name: 'risk' }).focus()
    await page.keyboard.press('Home')
    for (const s of ['camera', 'can', 'audio', 'sysml', 'sim']) await page.locator(`button[data-source="${s}"]`).click()
    await settled(page)
    await page.getByRole('button', { name: 'replay' }).click()
    await settled(page)
    await page.mouse.wheel(0, 20_000)
    await page.waitForTimeout(500)
    expect(errors).toEqual([])
  })

  test('the results lead with the headline the runs turn on, and each table cites its commit', async ({ page }) => {
    // The runs the page reads when it opens, as the snapshot holds them, so KAI is what it draws.
    await page.route(api, (r) => r.fulfill({ json: { data: snapshot.data, total: snapshot.data.length } }))
    // No model list: the Decision Suite states no count, so the harness's record leads (e2e/gates/suite.spec.ts reads the other case).
    await block(page)
    await page.setViewportSize({ width: 1440, height: 900 })
    await page.goto(server.url + '/kai')
    // The page's script has run before the wait starts, so it times the page and not a slow runner's boot.
    await page.locator('html[data-hydrated]').waitFor({ state: 'attached' })
    const results = page.locator('#benchmarks')
    await expect(results.locator('h2')).toHaveText(KAI.headline.title)
    await expect(results).toContainText(KAI.headline.lede)
    if (!KAI.sota) await expect(page.locator('main')).not.toContainText('State of the art')
    for (const c of KAI.claims) await expect(page.locator(`a[href="#${c.board}"]`, { hasText: c.label })).toContainText(c.figure)
    for (const b of [KAI.boards.harness, KAI.boards.capability, KAI.boards.latency].filter((x) => x != null)) {
      const table = page.locator(`[id="${b.id}"] table`)
      await expect(table.locator('tbody tr')).toHaveCount(b.blocks.reduce((n, x) => n + x.lines.length + (x.title ? 1 : 0), 0))
      const best = b.blocks.flatMap((x) => x.lines.flatMap((l) => l.cells)).filter((c) => c.best).length
      await expect(table.locator('td[data-best]')).toHaveCount(best)
      await expect(page.locator(`[id="${b.id}"] a[href="${b.cite.href}"]`)).toHaveCount(1)
    }
    // The hero's call to action lands on them.
    await expect(page.locator('a[href="#benchmarks"]').first()).toBeVisible()
  })

  test('the results are read when the page opens: the export carries none, and an API that does not answer is said to', async ({ page }) => {
    await page.route(api, (r) => r.abort('connectionrefused'))
    await block(page)
    await page.goto(server.url + '/kai')
    await expect(page.locator('main [data-api="unreachable"]').first()).toBeVisible({ timeout: 15_000 })
    await expect(page.locator(`[id="${KAI.boards.harness.id}"]`)).toHaveCount(0)
    // Nothing the runs say is in the bytes that ship.
    const html = await (await fetch(server.url + '/kai')).text()
    for (const l of KAI.boards.harness.blocks[0].lines) for (const c of l.cells) expect(html, `${l.label}: ${c.text}`).not.toContain(`>${c.text}<`)
  })

  test('a revision the API holds is the one drawn, with no rebuild', async ({ page }) => {
    const later = newer(snapshot.data as unknown as Run[])
    const want = derive(later)
    let asked = 0
    await page.route(api, (r) => {
      asked++
      return r.fulfill({ json: { data: later, total: later.length } })
    })
    await block(page)
    await page.setViewportSize({ width: 1440, height: 900 })
    await page.goto(server.url + '/kai')
    // The later revision's own mark first: the rows are read once, after the swap.
    // The fetch waits for hydration, which on a page this long and with every
    // worker busy runs past the default five seconds.
    await expect(page.locator(`[id="${want.boards.harness.id}"]`)).toContainText(`Kai ${STAGE}@`, { timeout: 15_000 })
    await expect(page.locator('#benchmarks h2')).toHaveText(want.headline.title)
    for (const b of [want.boards.harness, want.boards.capability, want.boards.latency].filter((x) => x != null)) await rows(page, b)
    expect(asked).toBe(1)
  })

  test('at 390 the graph stacks and nothing scrolls sideways', async ({ page }) => {
    await page.setViewportSize({ width: 390, height: 844 })
    await page.goto(server.url + '/kai')
    await expect(drawing(page)).toHaveAttribute('data-shape', 'tall')
    await settled(page)
    // Measured the way e2e/gates/mobile.spec.ts measures: the clip that stops a
    // sideways scroll is lifted first, so a page relying on it is caught.
    expect(await sideways(page)).toBeLessThanOrEqual(0)
  })

  for (const width of [375, 430, 768, 1024, 1728]) {
    test(`at ${width} nothing scrolls sideways and the hero decision fits`, async ({ page }) => {
      await page.setViewportSize({ width, height: 900 })
      await page.goto(server.url + '/kai')
      const box = await instrument(page).boundingBox()
      expect(box && box.x >= 0 && box.x + box.width <= width, 'the instrument inside the viewport').toBe(true)
      expect(await sideways(page)).toBeLessThanOrEqual(0)
    })
  }

  test('declining motion shows the settled program at once', async ({ browser }) => {
    const context = await browser.newContext({ reducedMotion: 'reduce', viewport: { width: 1440, height: 900 } })
    const page = await context.newPage()
    await page.goto(server.url + '/kai')
    await page.locator('html[data-hydrated]').waitFor({ state: 'attached' })
    await expect(page.locator('[data-status]')).toContainText('settled', { timeout: 5_000 })
    await expect(drawing(page).locator('[data-node][data-phase="resolved"]')).toHaveCount(12)
    await expect(node(page, 'fault').locator('[data-v]').first()).toBeVisible()
    // The hero's answer is there on the first paint, and the evidence moves it at once.
    await expect(option(page, 'review')).toHaveText('.76', { timeout: 1_000 })
    await expect(instrument(page)).toHaveAttribute('data-phase', 'still', { timeout: 15_000 })
    await instrument(page).getByRole('button', { name: 'passed' }).click()
    await expect(option(page, 'continue')).toHaveText('.91', { timeout: 250 })
    // Every line of the heading is where it belongs.
    for (const t of ['See. Hear. Understand.', 'When the answer has a type,', 'Decide.']) await expect(page.locator('h1').getByText(t)).toBeVisible()
    await context.close()
  })
})

test('/models compares the three families; each family page keeps to its own', async ({ page }) => {
  await page.goto(server.url + '/models')
  const family = page.locator('section[aria-labelledby="family-title"]')
  await expect(family).toHaveCount(1)
  await expect(family.locator('h2')).toHaveText('Enso routes. Kai decides. Zen reasons.')
  for (const href of ['/enso', '/kai', '/zen']) await expect(family.locator(`a[href="${href}"]`)).toHaveCount(1)
  const versus = page.locator('section[aria-labelledby="versus-title"]')
  await expect(versus).toHaveCount(1)
  for (const href of ['/zen', '/enso']) await expect(versus.locator(`a[href="${href}"]`)).toHaveCount(1)

  for (const [route, name] of [
    ['/enso', 'Enso'],
    ['/zen', 'Zen'],
    ['/kai', 'Kai'],
  ] as const) {
    await page.goto(server.url + route)
    await expect(page.locator('section[aria-labelledby="family-title"], section[aria-labelledby="versus-title"]'), route).toHaveCount(0)
    // One line back to the comparison, and no other family's page linked from the copy.
    await expect(page.locator('main a[href="/models"]'), route).toHaveCount(1)
    for (const other of ['/zen', '/enso', '/kai'].filter((h) => h !== route)) {
      await expect(page.locator(`main a[href="${other}"]`), `${route} links ${other}`).toHaveCount(0)
    }
    await expect(page.locator('h1'), route).toHaveCount(1)
    await expect(page.locator('main'), route).toContainText(`${name} is one of three Hanzo model families`)
  }
})