hanzo.ai/e2e/gates/wins.ts

import type { Run } from '../../lib/kai'

/**
 * What the wins-only gates share (wins.spec.ts, suite.spec.ts).
 *
 * A count told against a total: a tally of tasks, suites or languages out of
 * how many there are, "all N", or a win counted "N of M". A run's own count
 * ("184 of 198 right", "Flagged: Kai 83 of 600") is a measure, not a tally.
 */
export const TOTAL =
  /\b\d[\d,]* (?:of|out of) \d[\d,]*(?= (?:tasks|suites|languages)\b)|\ball \d[\d,]* (?:tasks|suites|languages)\b|\b(?:wins?|won|leads?|ahead(?: of Jev)? on)\b[^.;]{0,24}?\b\d[\d,]* of \d[\d,]*\b|\b\d[\d,]* of \d[\d,]*\b(?=[^.;]{0,40}\b(?:won|wins)\b)/gi

/** The first count in `text` told against a total, or null. "N of N" is every one of them, which tells no loss, so it passes; "all N" never does. */
export function against(text: string): RegExpExecArray | null {
  for (const m of text.matchAll(TOTAL)) {
    const pair = /(\d[\d,]*) (?:of|out of) (\d[\d,]*)/.exec(m[0])
    if (!pair || pair[1] !== pair[2]) return m as RegExpExecArray
  }
  return null
}

/**
 * The runs with Jev's run of one benchmark answering every question right, so
 * Kai no longer leads it: a suite every task of which Kai wins still exercises
 * the cut, a task Jev leads.
 */
export function lose(runs: Run[], benchmark: string): Run[] {
  return runs.map((r) =>
    r.benchmark === benchmark && r.system === 'jev' ? { ...r, measures: (r.measures ?? []).map((m) => (m.metric === 'accuracy' ? { ...m, value: 1 } : m)) } : r,
  )
}