hanzo.ai/lib/coding.ts

/**
 * Hanzo Dev against Claude Code on single-fault repairs, derived from the research
 * API's runs of benchmark `coding.repair`.
 *
 * `bench/code/post.py` in hanzo-inc/kai files one run a system: Claude Code as the
 * baseline and Hanzo Dev with its paired speed in category `vs-claude-code` (all
 * tasks both solved, then by how Dev settled each). `node scripts/coding.mjs` saves
 * the public answer as lib/data/benchmarks-coding.json, which the page renders
 * first; `useCoding` fetches the same address in the browser, so a newly posted
 * pass shows without a rebuild.
 *
 * A speed is conditional on the tasks both solved, so it is always said over
 * them ("over the 42 tasks both solved"). The pages publish wins only: a path is
 * drawn only while it is a resolved gain (`gains`), and the solved counts only
 * while Hanzo Dev solved more of the same tasks (`ahead`). The board is drawn
 * only while the speed over all tasks both solved is a resolved gain, its
 * interval above one.
 */
import type { Run } from '@/lib/kai'
import { phrase } from '@hanzo/i18n'
import { api } from '@hanzo/app/lib/api'

export const BENCH = 'coding.repair'
/** The board's runs, on the gateway of the host the page is open on. */
export const runs = (): string => `${api()}/v1/research/runs?org=hanzo&benchmark=${BENCH}`

/** One side of the comparison: its run, what it is, how many tasks it solved, its median time and model calls a task. */
export type Side = { id: string; system: string; version: string; solved: number; of: number; median: number | null; calls: number | null }
/**
 * A paired speed with its 95% interval over the tasks both solved; resolved when
 * the interval excludes one, a gain above it, a slowdown below. `label` is
 * authored English (`phrase`): a page draws it with `t()`.
 */
export type Speed = { label: string; value: number; lo: number | null; hi: number | null; n: number | null; resolved: boolean; gain: boolean; slower: boolean }
export type Coding = { split: string; ended: string; commit: string; base: Side; ours: Side; speed: Speed; paths: Speed[] }

/** How Dev settled a task, in the order the page lists them, and what each means (bench/code post.py's paths). */
const PATHS: [string, string][] = [
  ['search', phrase('Settled by search, no model call')],
  ['model', phrase('A model wrote the edit')],
  ['generated', phrase('The model’s first edit')],
  ['recovered', phrase('After recover changed strategy')],
  ['escalated', phrase('The escalation model wrote the edit')],
]

type Measure = NonNullable<Run['measures']>[number] & { lo?: number | null; hi?: number | null }
const find = (r: Run, category: string, metric: string) =>
  (r.measures ?? []).find((m) => m.category === category && m.metric === metric) as Measure | undefined

const side = (r: Run): Side | null => {
  const s = find(r, 'all', 'solved')
  if (!s || s.of == null) return null
  return { id: r.id, system: r.system, version: r.version, solved: s.value, of: s.of, median: find(r, 'all', 'median_s')?.value ?? null, calls: find(r, 'all', 'model_calls')?.value ?? null }
}

const speed = (label: string, m: Measure | undefined): Speed | null =>
  m
    ? { label, value: m.value, lo: m.lo ?? null, hi: m.hi ?? null, n: m.n ?? null, resolved: m.lo != null && m.hi != null && (m.lo > 1 || m.hi < 1), gain: m.lo != null && m.lo > 1, slower: m.hi != null && m.hi < 1 }
    : null

type Public = Run & { study?: string; visibility?: string; canonical?: boolean; completion?: string }

/** The newest public pass of Hanzo Dev and Claude Code on one split, or null when there is none or Dev's speed is not a resolved gain. */
export function derive(runs: Run[]): Coding | null {
  const valid = (runs as Public[]).filter(
    (r) => r.benchmark === BENCH && r.canonical !== false && (r.visibility ?? 'public') === 'public' && r.completion === 'complete',
  )
  const ours = valid.filter((r) => r.system === 'hanzo-dev' && !r.baseline).sort((a, b) => b.ended.localeCompare(a.ended))[0]
  if (!ours) return null
  const base = valid.find((r) => r.system === 'claude-code' && r.baseline && r.split === ours.split && r.study === ours.study)
  const all = speed(phrase('Every task both solved'), find(ours, 'vs-claude-code', 'speedup'))
  const b = base && side(base)
  const o = side(ours)
  if (!b || !o || !all || !all.gain) return null
  const paths = PATHS.map(([key, label]) => speed(label, find(ours, `vs-claude-code/${key}`, 'speedup'))).filter((s): s is Speed => s != null)
  return { split: ours.split, ended: ours.ended.slice(0, 10), commit: ours.commit, base: b, ours: o, speed: all, paths }
}

/** Whether Hanzo Dev solved more of the same tasks than Claude Code: the solved counts are drawn only then. */
export const ahead = (c: Pick<Coding, 'base' | 'ours'>) => c.ours.of === c.base.of && c.ours.solved > c.base.solved
/** The paths a page draws: each whose whole interval is above one, a resolved gain. */
export const gains = (ps: Speed[]) => ps.filter((p) => p.gain)

/** A speed as the page prints it: `3.83×`, with its interval when it has one. */
export const times = (s: Speed) => `${s.value.toFixed(2)}×`
/** The interval's level, as every speed here states it. */
export const LEVEL = '95%'
/** A commit as the page cites it. */
export const short = (sha: string) => sha.slice(0, 7)
export const range = (s: Speed) => (s.lo == null || s.hi == null ? '–' : `${s.lo.toFixed(2)}–${s.hi.toFixed(2)}×`)