hanzo.ai/e2e/unit/snapshot.spec.ts

import { test, expect } from '@playwright/test'
import { execFile } from 'node:child_process'
import { readFileSync, writeFileSync } from 'node:fs'
import { createServer } from 'node:http'
import type { AddressInfo } from 'node:net'
import { promisify } from 'node:util'

/**
 * SCRIPTS/KAI.MJS KEEPS THE COMMITTED SNAPSHOT WHEN THE GATE DOES NOT PASS.
 *
 * `prebuild` runs it on every build. An answer whose revision fails its release
 * gate against Jev is not a snapshot the pages and their gates can read, so the
 * script leaves lib/data/benchmarks-kai.json byte for byte and exits 0, and the
 * build goes on with the committed file. The answer served here holds one run
 * more than the snapshot, which is the case the script used to write.
 */

const FILE = new URL('../../lib/data/benchmarks-kai.json', import.meta.url)
const SCRIPT = new URL('../../scripts/kai.mjs', import.meta.url).pathname
const run = promisify(execFile)

type Run = { id: string; system: string; benchmark: string; measures?: { category: string; metric: string; value: number }[] }

test('a failing gate leaves the committed snapshot byte-identical', async () => {
  const before = readFileSync(FILE)
  const held = JSON.parse(before.toString('utf8')) as { data: Run[] }
  const extra = { ...held.data.find((r) => r.system === 'kai')!, id: 'snapshot-spec-extra' }
  const failing = (r: Run): Run =>
    r.benchmark === 'gate.jev'
      ? { ...r, measures: (r.measures ?? []).map((m) => (m.category === 'all' && m.metric === 'pass' ? { ...m, value: 0 } : m)) }
      : r
  const answer = JSON.stringify({ ...held, data: [...held.data.map(failing), extra] })

  const server = createServer((req, res) => {
    res.writeHead(req.url?.startsWith('/v1/research/runs') ? 200 : 404, { 'content-type': 'application/json' })
    res.end(answer)
  })
  await new Promise<void>((done) => server.listen(0, '127.0.0.1', done))
  const { port } = server.address() as AddressInfo
  try {
    const { stdout } = await run(process.execPath, [SCRIPT, `http://127.0.0.1:${port}`])
    expect(stdout).toContain('kept the frozen snapshot')
    expect(stdout).toContain('gate does not pass')
    expect(readFileSync(FILE).equals(before)).toBe(true)
  } finally {
    server.close()
    if (!readFileSync(FILE).equals(before)) writeFileSync(FILE, before)
  }
})