hanzo.ai/e2e/unit/reads.spec.ts

import { test, expect } from '@playwright/test'
import { INDEX } from '../../lib/bench-data'
import shortlist from '../../lib/data/benchmarks-shortlist.json' with { type: 'json' }
import grounding from '../../lib/data/benchmarks-grounding.json' with { type: 'json' }
import handlers from '../../lib/data/benchmarks-tools.json' with { type: 'json' }
import repairs from '../../lib/data/benchmarks-coding.json' with { type: 'json' }
import { TOOLS, cell, decides, derive } from '../../lib/tools'
import { gains } from '../../lib/coding'
import type { Run } from '../../lib/kai'

/**
 * KAI'S OWN READS SAY WHAT THEIR RUNS SAY.
 *
 * The tool shortlist and the yes/no table on /benchmarks come from two files no
 * research run writes: a run's report and two served reads, copied unchanged.
 * This holds the files to the figures those runs recorded (the shortlist at
 * hanzo-inc/kai d26ded1, the yes/no reads in kai-1.2's promotion on 2026-10-08),
 * then holds what INDEX hands the page to the files, and to the rule every
 * benchmark surface keeps: a figure shows only while ours leads on it.
 */

const three = (v: number) => Number(v.toFixed(3))
type Ranked = { any: number; ndcg10: number; top1: number }
const groups = shortlist.groups as Record<string, { n: number; bm25: Ranked; shortlist: Ranked }>

test('the shortlist file is the run it names', () => {
  expect([shortlist.catalog, shortlist.queries, shortlist.k, shortlist.bm25_weight]).toEqual([2928, 6608, 32, 0.1])
  const { bm25, shortlist: ours } = groups.all
  expect([bm25.any, bm25.ndcg10, bm25.top1].map(three)).toEqual([0.736, 0.516, 0.414])
  expect([ours.any, ours.ndcg10, ours.top1].map(three)).toEqual([0.936, 0.723, 0.555])
  expect([groups['hanzo/domain'].bm25.any, groups['hanzo/domain'].shortlist.any].map(three)).toEqual([0.155, 0.999])
  expect([shortlist.reranker.n, three(shortlist.reranker.embedder), three(shortlist.reranker.reranked)]).toEqual([1217, 0.718, 0.691])
})

test('the page draws the shortlist as the file holds it, and only where it leads BM25', () => {
  const s = INDEX.shortlist
  for (const [name, drawn] of [['all', s.all], ['hanzo/domain', s.hanzo]] as const) {
    const g = groups[name]
    expect(drawn, name).not.toBeNull()
    expect(drawn!.rows.length, name).toBeGreaterThan(0)
    for (const r of drawn!.rows) {
      const key = r.label.startsWith('Recall@') ? 'any' : r.label.startsWith('nDCG@') ? 'ndcg10' : 'top1'
      expect([r.base, r.ours], `${name} ${r.label}`).toEqual([g.bm25[key], g.shortlist[key]])
      expect(r.ours, `${name} ${r.label}`).toBeGreaterThan(r.base)
    }
  }
  expect(s.all!.rows.map((r) => r.label)).toEqual([`Recall@${shortlist.k}`, `nDCG@${shortlist.ndcg_k}`, 'Top-1'])
  expect([s.catalog, s.queries]).toEqual(['2,928', '6,608'])
  // The reranker the run tried and did not keep leads nothing, so the page is handed none of it.
  expect(Object.keys(s)).not.toContain('reranker')
})

test('the yes/no file is the two reads it names', () => {
  const k = grounding.kai.metrics as Record<string, number>
  const j = grounding.jev.metrics as Record<string, number>
  expect([k['critical.yes'], k['critical.no'], k.paraphrase, k['absent.within'], k['retained.pooled']].map(three)).toEqual([0.873, 0.135, 0.124, 0.817, 0.883])
  expect([j['critical.yes'], j['critical.no'], j.paraphrase, j['absent.within'], j['retained.pooled']].map(three)).toEqual([0.78, 0.09, 0.151, 0.383, 0.875])
  expect([k['matrix.in_band'], j['matrix.in_band']]).toEqual([40, 40])
})

test('the yes/no table draws only tests Kai leads on every half', () => {
  const rows = INDEX.yesno.rows
  // The Workbench pair is one test and Jev holds its counterfactual; the matrix and the field and negation sets are level or Jev's.
  expect(rows.map((r) => r.label.split(',')[0].split(':')[0])).toEqual([
    'A question and its paraphrase',
    'A state silent on the question',
    'Accuracy over 951 validation rows of 9 yes/no suites',
  ])
  expect(rows.map((r) => [r.kai, r.jev])).toEqual([
    ['0.124', '0.151'],
    ['81.7%', '38.3%'],
    ['88.3%', '87.5%'],
  ])
  expect(rows.some((r) => /workbench|counterfactual|matrix/i.test(r.label))).toBe(false)
})

test('the trolley table draws only checks Kai leads Jev on, and names the sealed families decided right every time', () => {
  const t = INDEX.trolley
  // Dominated choices, the stranger-not-server pairs and the switch onto one person: Kai's counts under Jev's past the tie.
  expect(t.rows.map((r) => r.key)).toEqual(['dominated', 'server_pairs', 'switch_one'])
  expect(t.rows.map((r) => [r.kai, r.jev])).toEqual([
    ['17 of 672 (2.5%)', '69 of 672 (10.3%)'],
    ['48 of 480 (10.0%)', '180 of 480 (37.5%)'],
    ['0 of 24 (0.0%)', '1 of 24 (4.2%)'],
  ])
  // Contradictions, server20, creator, monotone and self are Jev's or level, so none is drawn.
  expect(t.rows.some((r) => /contradiction|twenty|developer|fewer people|at least one/i.test(r.label))).toBe(false)
  expect(t.sealed.perfect).toEqual({ families: ['switch', 'server', 'robot', 'developer', 'empty'], scenes: '8,300' })
  expect(t.sealed.families.filter((f) => f.wrong !== '0').map((f) => f.name)).toEqual(['spelled', 'bridge'])
})

test('the coding board is the research runs it names, and draws only the paths that are gains', () => {
  const c = INDEX.coding
  expect(c, 'no coding.repair pass in lib/data/benchmarks-coding.json').not.toBeNull()
  expect(c!.base.system).toBe('claude-code')
  expect(c!.ours.system).toBe('hanzo-dev')
  expect(c!.base.of).toBe(c!.ours.of)
  expect(c!.speed.gain).toBe(true)
  expect(c!.speed.lo!).toBeGreaterThan(1)
  expect(c!.speed.n!).toBeLessThanOrEqual(Math.min(c!.base.solved, c!.ours.solved))
  // Every path the run states is read, a slowdown as one; a page draws only the resolved gains among them (lib/coding `gains`).
  const ours = (repairs.data as unknown as Run[]).filter((r) => r.system === 'hanzo-dev').sort((a, b) => b.ended.localeCompare(a.ended))[0]
  const stated = new Set((ours.measures ?? []).map((m) => m.category).filter((x) => x.startsWith('vs-claude-code/')))
  expect(c!.paths).toHaveLength(stated.size)
  expect(c!.paths[0].label).toBe('Settled by search, no model call')
  for (const p of c!.paths) expect(p.slower, p.label).toBe(p.hi != null && p.hi < 1)
  expect(gains(c!.paths).length).toBeGreaterThan(0)
  for (const p of gains(c!.paths)) expect([p.label, p.lo! > 1]).toEqual([p.label, true])
  expect(gains(c!.paths).some((p) => p.slower)).toBe(false)
})


test('the tools board is the research runs it names, each handler beside the one it replaced, while it holds against it', () => {
  const runs = handlers.data as unknown as Run[]
  const tools = INDEX.tools
  expect(tools.map((t) => t.id), 'the tools drawn, in TOOLS order').toEqual(TOOLS.filter((t) => tools.some((x) => x.id === t.id)).map((t) => t.id))
  expect(tools.length).toBeGreaterThan(0)
  // A split not drawn is one whose current handler trails the one it replaced on a measure both runs hold, or leads it on none.
  const value = (r: Run, c: (typeof TOOLS)[number]['columns'][number]) => r.measures?.find((m) => m.category === c.category && m.metric === c.metric)?.value
  for (const tool of TOOLS) {
    for (const split of new Set(runs.filter((r) => r.benchmark === tool.id).map((r) => r.split))) {
      if (tools.find((t) => t.id === tool.id)?.splits.some((s) => s.split === split)) continue
      const own = runs.filter((r) => r.benchmark === tool.id && r.split === split).sort((a, b) => b.ended.localeCompare(a.ended))
      const now = own.find((r) => r.system.endsWith('/measured') && !r.baseline)
      const before = own.find((r) => r.baseline)
      if (!now || !before) continue
      const said = tool.columns.flatMap((c) => {
        const [a, b] = [value(now, c), value(before, c)]
        return a == null || b == null ? [] : [Math.sign(c.better === 'high' ? a - b : b - a)]
      })
      expect(said.some((v) => v < 0) || !said.some((v) => v > 0), `${tool.id} ${split} is drawn while it holds`).toBe(true)
    }
  }
  for (const t of tools) {
    const tool = TOOLS.find((x) => x.id === t.id)!
    // Held out first: the splits the tool names, in its order.
    expect(t.splits.map((s) => s.split), t.id).toEqual(tool.splits.filter((s) => t.splits.some((x) => x.split === s)))
    for (const s of t.splits) {
      const where = `${t.id} ${s.split}`
      expect(s.rows[0].role, where).toBe('now')
      expect(s.rows.filter((r) => r.role === 'now'), where).toHaveLength(1)
      for (const r of s.rows) {
        const run = runs.find((x) => x.id === r.id)!
        expect([run.benchmark, run.split], r.id).toEqual([t.id, s.split])
        expect(run.baseline, r.id).toBe(r.role === 'before')
        expect(r.decides, r.id).toBe(decides(run.version))
        const cells = s.heads.map((h) => cell(run, tool.columns.find((x) => x.head === h)!))
        expect(r.cells, r.id).toEqual(cells)
      }
    }
  }
})

test('what decides is read from each version a step at a time', () => {
  expect(decides('rules, then a Kai question')).toBe('Rules, then Kai')
  expect(decides('a tool the task names, then BM25')).toBe('Rules, then BM25')
  expect(decides('one Kai choice over the names')).toBe('Kai')
  expect(decides('kai-1.2 alone, at 0.5')).toBe('Kai')
  expect(decides('failure signatures, root cause first')).toBe('Rules')
  expect(decides('')).toBe('–')
})

test('the tools board hides with no run, and a latency its notes disown', () => {
  expect(derive([])).toEqual([])
  const runs = (handlers.data as unknown as Run[]).map((r) => ({ ...r, notes: `${r.notes}; not a serving latency` }))
  const heads = derive(runs).flatMap((t) => t.splits.flatMap((s) => s.heads))
  expect(heads.length).toBeGreaterThan(0)
  expect(heads).not.toContain('Median time')
})