hanzo.ai/lib/tools.ts

/**
 * Hanzo MCP's tool handlers, each beside the handler it replaced, derived from
 * the research API's runs of the `mcp.*` benchmarks.
 *
 * Each tool has one address, `runs(id)`: `GET /v1/research/runs?org=hanzo&benchmark=<id>`.
 * `node scripts/tools.mjs` saves the answers as lib/data/benchmarks-tools.json,
 * which the page renders first; `useTools` fetches the same addresses in the
 * browser, so a newly posted run shows without a rebuild; the page links them.
 *
 * A run's system is `hanzo-mcp/<arm>`. `measured` is the handler now and the run
 * marked `baseline` is the one it replaced; the split cites the current
 * handler's commit. Any other arm of the same study (a part read alone, or
 * another handler that joins them) is an experiment, not a handler that ships,
 * and is not drawn. The pages publish wins only: a split is drawn while the
 * current handler is at least level with the one it replaced on every measure
 * both runs hold and ahead on one, each by the way its column is better
 * (`better`). What decides a row
 * is read from its run's `version`, one step at a time in the order it names them
 * (`X, then Y`): a step that names Kai is Kai, one that names BM25 is BM25, any
 * other is rules. A latency shows only where the run's notes do not say it is not
 * a serving latency. A split is drawn while it has a run of the current handler,
 * a tool while it has a split, and the board while it has a tool.
 */
import type { Measure, Run } from '@/lib/kai'

/** A column: the measure it reads, its head, and which way is better. */
type Column = { category: string; metric: string; head: string; better: 'high' | 'low' }
/** A tool: its benchmark, its title, its splits held out first, and its columns. */
type Tool = { id: string; title: string; splits: string[]; columns: Column[] }

const MEDIAN: Column = { category: 'all', metric: 'p50_ms', head: 'Median time', better: 'low' }

/** The tools, in the page's order. */
export const TOOLS: Tool[] = [
  {
    id: 'mcp.injection',
    title: 'Instructions planted in tool output',
    splits: ['test', 'dev'],
    columns: [
      { category: 'attack', metric: 'recall', head: 'Attacks caught', better: 'high' },
      { category: 'benign', metric: 'false_alarm', head: 'False alarms', better: 'low' },
      { category: 'benign/plain', metric: 'false_alarm', head: 'False alarms, plain', better: 'low' },
      { category: 'benign/imperative', metric: 'false_alarm', head: 'False alarms, imperative', better: 'low' },
      { category: 'benign/readme', metric: 'false_alarm', head: 'False alarms, README', better: 'low' },
      MEDIAN,
    ],
  },
  {
    id: 'mcp.command',
    title: 'Shell commands an agent should not run unasked',
    splits: ['hermes'],
    columns: [
      { category: 'deny_or_ask', metric: 'false_allow', head: 'Allowed, should deny or ask', better: 'low' },
      { category: 'ordinary', metric: 'friction', head: 'Ordinary, stopped', better: 'low' },
      { category: 'must_deny', metric: 'denied', head: 'Must deny, denied', better: 'high' },
      MEDIAN,
    ],
  },
  {
    id: 'mcp.select',
    title: 'Which registered tool a task needs',
    splits: ['t1dev', 'hand'],
    columns: [
      { category: 'all', metric: 'top1', head: 'Right tool first', better: 'high' },
      { category: 'all', metric: 'mrr', head: 'MRR', better: 'high' },
      { category: 'all', metric: 'abstained', head: 'Abstained', better: 'low' },
      MEDIAN,
    ],
  },
  {
    id: 'mcp.triage',
    title: 'What kind of failure a CI log shows',
    splits: ['ci60'],
    columns: [
      { category: 'all', metric: 'accuracy', head: 'Right class', better: 'high' },
      { category: 'nameable', metric: 'accuracy', head: 'Right class, nameable', better: 'high' },
      { category: 'all', metric: 'unknown', head: 'Unknown', better: 'low' },
      MEDIAN,
    ],
  },
]

/** A tool's address at the research API: its public runs. */
export const runs = (id: string) => `https://api.hanzo.ai/v1/research/runs?org=hanzo&benchmark=${id}`

/** `now` is the current handler, `before` the handler it replaced. */
export type Role = 'now' | 'before'
export type Row = { id: string; role: Role; label: string; decides: string; cells: string[] }
export type Split = { split: string; definition: string; ended: string; commit: string; heads: string[]; rows: Row[] }
export type Board = { id: string; slug: string; title: string; href: string; splits: Split[] }

type Public = Run & { study?: string; visibility?: string; canonical?: boolean; completion?: string }

/** A commit as the page cites it. */
export const short = (sha: string) => sha.slice(0, 7)

/** A duration at the precision its size needs, as the bench kit's `elapsed` prints one. */
const ms = (v: number) => (v >= 100 ? `${Math.round(v).toLocaleString('en-US')} ms` : `${v.toFixed(v < 1 ? 3 : 1)} ms`)

/** A measure as its cell prints it: a rate with its count, a duration, a count, or a score. */
export function shown(m: Measure | undefined): string {
  if (!m) return '–'
  if (m.of != null) return `${(m.value * 100).toFixed(1)}% (${m.n ?? '–'} / ${m.of})`
  if (m.metric.endsWith('_ms')) return ms(m.value)
  return Number.isInteger(m.value) ? String(m.value) : m.value.toFixed(3)
}

/** The steps a version names, in order: each `Kai`, `BM25` or `rules`. */
const steps = (version: string) =>
  version.trim()
    ? version
        .split(/,\s*then\s+/i)
        .map((s) => (/\bkai\b/i.test(s) ? 'Kai' : /\bbm25\b/i.test(s) ? 'BM25' : 'rules'))
        .filter((s, i, all) => s !== all[i - 1])
    : []

/** What decides, as the page says it: `Rules, then Kai`. */
export function decides(version: string): string {
  const said = steps(version).join(', then ')
  return said ? said[0].toUpperCase() + said.slice(1) : '–'
}

const arm = (r: Run) => r.system.slice(r.system.lastIndexOf('/') + 1)
const newest = (rs: Public[]): Public | undefined => [...rs].sort((a, b) => b.ended.localeCompare(a.ended))[0]
const serving = (r: Run) => !/not a serving latency/i.test(r.notes ?? '')
const find = (r: Run, c: Column) =>
  c.metric.endsWith('_ms') && !serving(r) ? undefined : (r.measures ?? []).find((m) => m.category === c.category && m.metric === c.metric)
/** A run's cell in a column: its measure as the page prints it, a latency only where the run names it a serving one. */
export const cell = (r: Run, c: Column) => shown(find(r, c))
const label = (role: Role, r: Run) => (role === 'now' ? 'Current' : `Replaced, ${short(r.commit)}`)
/** The current handler ahead of the one it replaced on a measure both runs hold (1), level (0), behind (-1), or not compared (null). */
const versus = (now: Run, before: Run, c: Column) => {
  const [a, b] = [find(now, c)?.value, find(before, c)?.value]
  return a == null || b == null ? null : Math.sign(c.better === 'high' ? a - b : b - a)
}
/** The current handler at least level with the one it replaced on every measure both runs hold, and ahead on one. */
const holds = (now: Run, before: Run, columns: Column[]) => {
  const said = columns.map((c) => versus(now, before, c)).filter((v): v is number => v != null)
  return said.every((v) => v >= 0) && said.some((v) => v > 0)
}

/** One split's table: the current handler, then the handler it replaced; null while the current one trails it on any measure. */
function table(tool: Tool, split: string, rs: Public[]): Split | null {
  const now = newest(rs.filter((r) => arm(r) === 'measured' && !r.baseline))
  if (!now) return null
  const before = newest(rs.filter((r) => r.baseline && r.study === now.study))
  if (before && !holds(now, before, tool.columns)) return null
  const lines: [Role, Public][] = [['now', now], ...(before ? [['before', before] as [Role, Public]] : [])]
  const columns = tool.columns.filter((c) => lines.some(([, r]) => find(r, c)))
  return {
    split,
    definition: (now.notes ?? '').startsWith(`${split}:`) ? now.notes.slice(`${split}:`.length).trim() : (now.notes ?? ''),
    ended: now.ended.slice(0, 10),
    commit: short(now.commit),
    heads: columns.map((c) => c.head),
    rows: lines.map(([role, r]) => ({
      id: r.id,
      role,
      label: label(role, r),
      decides: decides(r.version),
      cells: columns.map((c) => cell(r, c)),
    })),
  }
}

/** Every tool with a split to draw, in the page's order; empty when no run is present. */
export function derive(all: Run[]): Board[] {
  const valid = (all as Public[]).filter((r) => r.canonical !== false && (r.visibility ?? 'public') === 'public' && r.completion === 'complete')
  return TOOLS.flatMap((tool) => {
    const own = valid.filter((r) => r.benchmark === tool.id)
    const named = [...new Set(own.map((r) => r.split))]
    const splits = [...tool.splits.filter((s) => named.includes(s)), ...named.filter((s) => !tool.splits.includes(s))]
      .map((s) => table(tool, s, own.filter((r) => r.split === s)))
      .filter((s): s is Split => s != null)
    return splits.length ? [{ id: tool.id, slug: tool.id.replace(/\W+/g, '-'), title: tool.title, href: runs(tool.id), splits }] : []
  })
}