hanzo.ai/scripts/locomo.mjs

#!/usr/bin/env node
/**
 * The Go LoCoMo harness, captured.
 *
 *   node scripts/locomo.mjs [path to a hanzoai/semantic checkout]
 *
 * Every other bench page on this site renders a run file that the harness
 * itself writes. This one cannot: its harness is a different repository in a
 * different language, it commits no result file, and it prints its tables to a
 * terminal. So the tables were transcribed by hand into the page, which is the
 * weakest guarantee anything here offers — a number that nobody can check
 * against the thing that produced it.
 *
 * This runs the five commands and parses what they print. The site then
 * renders the capture, so the page and the harness agree by construction, and a
 * rerun that moves a number moves the page.
 *
 * It is NOT part of `prebuild`. It needs a Go toolchain, the checkout, and the
 * dataset fetched — none of which a site build should require — and it is run
 * by hand when the harness moves, the way the committed capture it writes is
 * read by hand out of the repository it names.
 *
 * The control is the reason the shape has two halves. `-blind` empties every
 * graph after building it and changes nothing else; the claim is that the arms
 * that never read the graph come back identical and the two that do collapse.
 * That claim is COMPUTED here, cell by cell, rather than written down: the
 * capture carries which arms survived the emptying and which moved, so the page
 * states the result of the comparison instead of asserting its conclusion.
 */
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { readFileSync, writeFileSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { homedir } from 'node:os'

const REPO = resolve(process.argv[2] ?? join(homedir(), 'work/hanzo/semantic'))
const OUT = new URL('../lib/data/benchmarks-semantic.json', import.meta.url)
const DATA = 'bench/locomo/data/locomo10.json'
const PREDS = '/tmp/locomo-capture.json'

/** The harness, from the checkout, with the workspace off so it builds alone. */
const run = (...args) =>
  execFileSync('go', ['run', './bench/locomo', ...args], {
    cwd: REPO,
    encoding: 'utf8',
    maxBuffer: 64 << 20,
    env: { ...process.env, GOWORK: 'off' },
  })

const git = (...args) => execFileSync('git', args, { cwd: REPO, encoding: 'utf8' }).trim()

/** As the JSON spells categories, which is not how a terminal column spells them. */
const slug = (s) => s.trim().replace(/\s+/g, '-')
const pct = (s) => Number((Number(s.replace('%', '')) / 100).toFixed(4))

/**
 * One printed table: an arm header, then a row per category.
 *
 * The arm names are the only thing in the block that is not a number, and they
 * are read rather than supplied — an arm added to the harness appears in the
 * capture without this script being told about it.
 */
function table(text) {
  const lines = text.split('\n')
  const head = lines.findIndex((l) => /^category\s+n\s+F1/.test(l))
  if (head < 1) throw new Error('no table in output')
  const arms = lines[head - 1].trim().split(/\s{2,}/)
  const rows = []
  for (const line of lines.slice(head + 1)) {
    const m = /^(\S.*?)\s{2,}(\d+)\s+(.*)$/.exec(line)
    if (!m) break
    const cells = m[3].trim().split(/\s+/)
    if (cells.length !== arms.length * 3) throw new Error(`row "${m[1]}" has ${cells.length} cells for ${arms.length} arms`)
    rows.push({
      category: slug(m[1]),
      n: Number(m[2]),
      arms: Object.fromEntries(
        arms.map((a, i) => [a, { f1: Number(cells[i * 3]), recall: Number(cells[i * 3 + 1]), quiet: pct(cells[i * 3 + 2]) }]),
      ),
    })
  }
  return { arms, rows }
}

/** The settings line the harness prints, so the capture cannot claim a k it did not use. */
function settings(text) {
  const m = /^k=(\d+) floor=([\d.]+) span=(\d+) band=([\d.]+) parts=(\d+)/m.exec(text)
  if (!m) throw new Error('no settings line')
  const [k, floor, span, band, parts] = m.slice(1).map(Number)
  return { k, floor, span, band, parts }
}

/** `-stats` prints three blocks: constant abstention, the AUCs, the paired intervals. */
function stats(text) {
  const f1 = {}
  for (const [, c, , v] of text.matchAll(/^(\S.*?)\s{2,}(\d+)\s+([\d.]+)\s*$/gm)) f1[slug(c)] = Number(v)

  const auc = {}
  for (const [, arm, sure, decided, qa, qv] of text.matchAll(
    /^(\S+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)%\s+([\d.]+)%\s*$/gm,
  )) {
    auc[arm] = { sure: Number(sure), decided: Number(decided), quiet_answerable: pct(qa), quiet_adversarial: pct(qv) }
  }

  const paired = []
  for (const [, a, b, ...n] of text.matchAll(
    /^(\S+)\s+-\s+(\S+)\s+([-+][\d.]+) \[([-+][\d.]+), ([-+][\d.]+)\]\s+([-+][\d.]+) \[([-+][\d.]+), ([-+][\d.]+)\]\s*$/gm,
  )) {
    const [am, alo, ahi, nm, nlo, nhi] = n.map(Number)
    paired.push({ a, b, adversarial: { mean: am, lo: alo, hi: ahi }, answerable: { mean: nm, lo: nlo, hi: nhi } })
  }
  return { f1, auc, paired }
}

/**
 * Which arms the emptied graph left alone, decided cell by cell.
 *
 * An arm counts as unchanged only if all three of its figures match in every
 * category. That is the strong reading of "byte-identical" and the one the
 * claim needs: an arm that shifted in the third decimal of one category read
 * something, and would be reported as changed.
 */
function control(open, blind) {
  const same = []
  const moved = []
  for (const arm of open.arms) {
    const differs = open.rows.some((r, i) => {
      const a = r.arms[arm]
      const b = blind.rows[i]?.arms[arm]
      return !b || a.f1 !== b.f1 || a.recall !== b.recall || a.quiet !== b.quiet
    })
    ;(differs ? moved : same).push(arm)
  }
  return { identical: same, changed: moved }
}

const main = run('-quiet', '-out', PREDS)
const factor = run('-arms', 'factor', '-quiet')
const statsOut = run('-stats', PREDS)
const walk = run('-arms', 'walk', '-quiet')
const blind = run('-arms', 'walk', '-blind', '-quiet')

const open = table(walk)
const dark = table(blind)
const { f1, auc, paired } = stats(statsOut)

const capture = {
  harness: 'hanzoai/semantic · bench/locomo',
  tag: git('describe', '--tags', '--always'),
  commit: git('rev-parse', 'HEAD'),
  dataset: 'snap-research/locomo',
  dataset_sha256: createHash('sha256').update(readFileSync(join(REPO, DATA))).digest('hex'),
  settings: settings(main),
  generated: new Date().toISOString().replace(/\.\d+Z$/, 'Z'),
  main: { command: 'go run ./bench/locomo', ...table(main) },
  factor: { command: 'go run ./bench/locomo -arms factor', ...table(factor) },
  silent: { command: 'go run ./bench/locomo -stats out/answers.json', f1 },
  auc,
  paired,
  walk: { command: 'go run ./bench/locomo -arms walk', ...open },
  blind: { command: 'go run ./bench/locomo -arms walk -blind', ...dark, ...control(open, dark) },
}

writeFileSync(OUT, `${JSON.stringify(capture, null, 2)}\n`)
console.log(
  `${capture.tag} ${capture.commit.slice(0, 7)} — ${open.arms.length} arms over ${open.rows.at(-1)?.n} questions; ` +
    `emptying the graph left ${capture.blind.identical.join(', ')} identical and moved ${capture.blind.changed.join(', ')}`,
)