hanzo.ai/scripts/kai.mjs

#!/usr/bin/env node
/**
 * Kai's results, as the research API holds them.
 *
 *   node scripts/kai.mjs [api base URL]
 *
 * Saves the runs of `GET /v1/research/runs?org=hanzo` whose system the pages
 * compare (lib/kai.ts WHO) as lib/data/benchmarks-kai.json: the snapshot the
 * pages render before their own fetch of the same address returns, and merge
 * with it after. `bench post` in hanzo-inc/kai writes those runs, one per
 * system, revision and benchmark. The base defaults to api.hanzo.ai. The read
 * is anonymous, as the browser's is, so the snapshot holds only public runs.
 *
 * Part of `prebuild`, so every build ships the latest validated runs: canonical,
 * public, and not flagged inconsistent by the API. The committed snapshot is the
 * frozen fallback. It is kept, and the build goes on, when the API cannot be
 * reached, answers with an error, answers with fewer runs of Kai or Jev than
 * the snapshot already holds — a read that loses runs is a broken read, not news
 * — or answers with a revision whose release gate against Jev does not pass
 * (`derive`'s `sota`): such a snapshot is not one the pages and their gates can
 * read, so the build goes on with the committed file, byte for byte
 * (e2e/unit/snapshot.spec.ts).
 */
import { readFileSync, writeFileSync } from 'node:fs'

const FILE = new URL('../lib/data/benchmarks-kai.json', import.meta.url)
const keep = (why) => {
  console.log(`kai: kept the frozen snapshot (${why})`)
  process.exit(0)
}

try {
  const { RUNS, WHO, derive } = await import('../lib/kai.ts')
  const base = process.argv[2]?.replace(/\/$/, '')
  const url = base ? RUNS.replace(new URL(RUNS).origin, base) : RUNS
  const res = await fetch(url, { signal: AbortSignal.timeout(20_000) })
  if (!res.ok) keep(`${url}: ${res.status}`)
  const body = await res.json()
  const kai = derive(body.data ?? [])
  if (!kai) keep(`${url}: no Kai run on the harness`)
  if (!kai.sota) keep(`${url}: ${kai.run}: gate does not pass`)

  // The answer as the API gave it, one run a line, so a diff names the runs a post changed.
  // Only what the pages compare: the snapshot ships to the browser, which merges it with the live
  // answer, so a run of another system, or one whose benchmark or metric names one, stays out.
  const { data: all, ...rest } = body
  // A gate (`gate.<system>`) and a tally (`beats.<system>`) name the system they compare with, so they name it
  // whether or not the answer still lists that system's own runs.
  const others = new Set(all.map((r) => r.system).filter((s) => !WHO.includes(s)))
  const against = (part, i, parts) => i > 0 && ['gate', 'beats'].includes(parts[i - 1]) && !WHO.includes(part)
  const names = (key) => key.split('.').some((part, i, parts) => others.has(part) || against(part, i, parts))
  const valid = (r) => r.canonical !== false && (r.visibility ?? 'public') === 'public' && r.completion !== 'inconsistent'
  const data = all.filter((r) => WHO.includes(r.system) && valid(r) && !names(r.benchmark) && !(r.measures ?? []).some((m) => names(m.metric)))

  const held = JSON.parse(readFileSync(FILE, 'utf8')).data ?? []
  if (data.length < held.length) keep(`${url}: ${data.length} runs against the snapshot's ${held.length}`)

  const tail = Object.entries(rest).map(([k, v]) => `${JSON.stringify(k)}: ${JSON.stringify(v)}`)
  writeFileSync(FILE, `{\n"data": [\n${data.map((r) => JSON.stringify(r)).join(',\n')}\n],\n${tail.join(',\n')}\n}\n`)
  console.log(`kai: ${kai.run}: ${data.length} runs (${data.length - held.length >= 0 ? '+' : ''}${data.length - held.length}), ${kai.claims.length} claims; gate passes`)
} catch (e) {
  keep(e instanceof Error ? e.message : String(e))
}