hanzo.ai/lib/benchmarks.ts

import { PAPERS, papers as library } from './data/papers'

/**
 * The benchmarks, as identities rather than as tables.
 *
 * One page per benchmark, and this is the list those pages, the index, the
 * sitemap and every JSON-LD block are built from. The reason it exists is that
 * the same few facts about a benchmark have to agree in four places at once —
 * the card on /benchmarks, the <h1> and metadata of its own page, the
 * `lastModified` a crawler is offered, and the `Dataset` a search engine
 * reads — and four copies of a licence string is three chances to be wrong
 * about somebody else's terms.
 *
 * What is NOT here: numbers. Every figure on every bench page is read from
 * lib/data/benchmarks*.json, written by `node results.mjs` in
 * hanzoai/benchmarks/brain from the raw runs. This file carries what a run
 * cannot state about itself — who made the dataset, under what licence, which
 * command regenerates the table, and whether anything has been run at all.
 *
 * `status` is the honest three-way answer to "what do we know here", and the
 * index sorts by it:
 *
 *   measured   a Hanzo configuration ran against a matched baseline.
 *   baseline   the benchmark's floor is measured and no Hanzo row exists yet.
 *              A gap, published as one, because a reader deciding whether to
 *              trust the measured rows should see which lanes we have not
 *              entered.
 *   soon       no run. No number, no table, no estimate.
 *
 * A benchmark is one PROTOCOL, not one dataset. LoCoMo appears twice — once
 * for the engine's runs under brain/, once for the lexical Go harness in
 * hanzoai/semantic — because their readers, their scope and their question
 * sets differ, so their scores are not on one scale. Two pages that each say
 * "this does not compare with that, here is why" make the wrong reading harder
 * than one page with two tables under a single heading would.
 */

export type Status = 'measured' | 'baseline' | 'soon'

/** The dataset a benchmark is built on: somebody else's work, on their terms. */
export interface Dataset {
  /** As its authors capitalise it. */
  name: string
  /** Where the data actually comes from — the repo or dataset card we fetch. */
  url: string
  /** SPDX-ish, as the dataset states it. Quoted, never guessed. */
  license: string
  /** The paper that introduced it. */
  paper: { title: string; authors: string; url: string; venue: string }
}

/** A paper of ours that reports this benchmark's numbers. */
export interface Paper {
  /** The directory in hanzoai/papers, which is also its slug on papers.hanzo.ai. */
  slug: string
  title: string
  /**
   * Which of the two addresses serves it.
   *
   * `library` is papers.hanzo.ai, where a paper is published. `source` is its
   * PDF in hanzoai/papers, which is public — the address for a paper the
   * library does not serve yet. A paper the library publishes moves this field
   * in the same commit.
   */
  at: 'library' | 'source'
}

/**
 * The lane a benchmark belongs to, and the order the lanes are read in.
 *
 * Stated once, here. The index groups by it and is written in it, so neither
 * can drift from the other.
 */
export type Lane = 'runtime' | 'inference' | 'routing' | 'memory' | 'graph' | 'code'

/* The index leads with the models, then the platform they run on, then memory,
 * then the rest of the suite. Routing is the model board. Runtime and
 * inference are the platform: agents, sandboxes, the cloud. Memory, the graph
 * and code follow, read from run files. */
export const LANES: Lane[] = ['routing', 'runtime', 'inference', 'memory', 'graph', 'code']

/** What a lane is called where a reader sees it, and the one line that says why it is a lane. */
export const LANE: Record<Lane, { title: string; what: string }> = {
  runtime: {
    title: 'Platform, agents, and sandboxes',
    what: 'What an agent costs at rest, what a live one costs to wake, and what a sandbox costs to start.',
  },
  inference: {
    title: 'Cloud inference',
    what: 'The engine against llama.cpp, on the same box and the same weights.',
  },
  routing: {
    title: 'Enso, Zen, and Kai',
    what: 'Enso routes the call. The published board is the graduate-question set the router is scored on.',
  },
  memory: {
    title: 'Memory and retrieval',
    what: 'What a memory finds in a long conversation, and what a reader does with it.',
  },
  graph: {
    title: 'Knowledge graph',
    what: 'Point-in-time questions over facts that begin and end, against another temporal store on the same data.',
  },
  code: {
    title: 'Code',
    what: 'Cross-file retrieval in a repository, scored apart from generation.',
  },
}

export interface Bench {
  /** URL segment under /benchmarks. The name people type into a search box. */
  slug: string
  /** As the dataset's authors capitalise it. */
  name: string
  /** The page's own <h1>. */
  title: string
  /** One sentence: the page's lede and its meta description, which are one thing. */
  lede: string
  /** The index card's line — what is measured, not how it went. */
  measures: string
  /** Which part of the stack it measures. Sets where it sits on the index. */
  lane: Lane
  status: Status
  dataset?: Dataset
  /** The harness, and the literal command that regenerates the published table. */
  harness: { repo: string; url: string; command: string }
  /**
   * Where a page's numbers come from, when they are NOT read from a run file.
   *
   * Most of these pages render lib/data/benchmarks*.json, so a rerun moves the
   * page and no one transcribes anything. Three do not: their harness is a
   * different repository that commits no result file, and publishes its table
   * in a README pinned to a tag or a dated pass. Those numbers are transcribed,
   * which is a weaker guarantee than the rest of this site offers — so it is
   * said on the page rather than left for a reader to discover, and the command
   * that prints them is beside it.
   */
  quoted?: { where: string; when: string }
  papers: Paper[]
  /**
   * Which published run file carries this bench's numbers.
   *
   * `app/sitemap.ts` reads the `generated` stamp out of it, so a bench route's
   * `lastModified` is the moment its runs were last rolled up rather than the
   * moment somebody edited the prose beside them. `null` where the file carries
   * no stamp or there is no file: the sitemap then offers no date, which is the
   * true answer and better than a plausible one.
   */
  data: 'benchmarks' | 'retrieval' | 'code' | 'semantic' | 'cronquestions' | null
}

const LIBRARY = 'https://papers.hanzo.ai'
const SOURCE = 'https://github.com/hanzoai/papers/blob/main/site/public/pdfs'
const BRAIN = 'https://github.com/hanzoai/benchmarks/tree/main/brain'

/** Where a paper can be read. One spelling, so every page agrees and none of them 404. */
export const paperUrl = (p: Paper) => (p.at === 'library' ? `${LIBRARY}/${p.slug}` : `${SOURCE}/${p.slug}.pdf`)

/*
 * The papers these benchmarks report, named rather than spelled out at each
 * link. A title written twice is a title that disagrees with itself once.
 */
export const DORMANT: Paper = { slug: 'hanzo-dormant-agents', title: 'What a Dormant Agent Costs', at: 'source' }
export const MULTI_HOP: Paper = { slug: 'hanzo-multi-hop-retrieval', title: 'Hops Do Not Fix Multi-Hop Retrieval', at: 'source' }
export const SUBJECT_SCOPE: Paper = { slug: 'locomo-subject-scope', title: 'Retrieval Scoped by Subject', at: 'source' }
export const CONTEXT_MEMORY: Paper = { slug: 'hanzo-context-memory', title: 'Linkage, Not Similarity', at: 'source' }
export const REFUTATION: Paper = { slug: 'refutation-driven-perf', title: 'Refutation-Driven Performance Engineering', at: 'library' }

/** A bench page's address. Also its canonical URL and its sitemap entry. */
export const benchUrl = (slug: string) => `https://hanzo.ai/benchmarks/${slug}`

export const BENCHMARKS: Bench[] = [
  {
    slug: 'locomo',
    name: 'LoCoMo',
    title: 'LoCoMo',
    lede:
      'Ten long conversations, 1,540 questions with annotated evidence turns. Hanzo Context measured against cosine over turns and against the annotated gold turns, on the same split, embedder, reader and k.',
    measures: 'Long-conversation retrieval and answering, with the evidence turns annotated.',
    lane: 'memory',
    status: 'measured',
    dataset: {
      name: 'LoCoMo',
      url: 'https://github.com/snap-research/locomo',
      license: 'CC BY-NC 4.0',
      paper: {
        title: 'Evaluating Very Long-Term Conversational Memory of LLM Agents',
        authors: 'Maharana, Lee, Tulyakov, Bansal, Barbieri, Fang',
        url: 'https://arxiv.org/abs/2402.17753',
        venue: 'ACL 2024',
      },
    },
    harness: { repo: 'hanzoai/benchmarks · brain', url: BRAIN, command: 'node results.mjs' },
    papers: [MULTI_HOP, CONTEXT_MEMORY],
    data: 'retrieval',
  },
  {
    slug: 'memoryagentbench',
    name: 'MemoryAgentBench',
    title: 'MemoryAgentBench',
    lede:
      'FactConsolidation: haystacks of 6k to 262k tokens where a later line silently overwrites an earlier one. A typed search over parsed facts resolves the question with no language model reading anything.',
    measures: 'Conflict resolution — which version of a fact is current, over four haystack sizes.',
    lane: 'memory',
    status: 'measured',
    dataset: {
      name: 'MemoryAgentBench',
      url: 'https://huggingface.co/datasets/ai-hyz/MemoryAgentBench',
      license: 'MIT',
      paper: {
        title: 'Evaluating Memory in LLM Agents via Incremental Multi-Turn Interactions',
        authors: 'Hu, Wang, McAuley',
        url: 'https://arxiv.org/abs/2507.05257',
        venue: 'UC San Diego, 2025',
      },
    },
    harness: { repo: 'hanzoai/benchmarks · brain', url: `${BRAIN}/mab`, command: 'node results.mjs' },
    papers: [CONTEXT_MEMORY],
    data: 'benchmarks',
  },
  {
    slug: 'longmemeval',
    name: 'LongMemEval',
    title: 'LongMemEval',
    lede:
      'Five hundred questions, each with its own haystack of chat sessions around 115k tokens. The engine, frozen on LoCoMo and run here untuned, against the cosine floor on the same vectors across 4,846 sessions.',
    measures: 'Session recall over long chat histories, engine against cosine on the same embedder.',
    lane: 'memory',
    status: 'measured',
    dataset: {
      name: 'LongMemEval',
      url: 'https://huggingface.co/datasets/xiaowu0162/longmemeval',
      license: 'MIT',
      paper: {
        title: 'LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory',
        authors: 'Wu et al.',
        url: 'https://arxiv.org/abs/2410.10813',
        venue: '2024',
      },
    },
    harness: {
      repo: 'hanzoai/benchmarks · brain',
      url: `${BRAIN}/longmemeval`,
      command: 'node longmemeval/engine.mjs --embed=minilm && node results.mjs',
    },
    papers: [],
    data: null,
  },
  {
    slug: 'cronquestions',
    name: 'CronQuestions',
    title: 'CronQuestions',
    lede:
      'Wikidata facts with start and end years, and 26,168 questions about who held what, and when. The graph API and Semantica’s temporal query engine, on the same facts and the same machine, with no model reading anything.',
    measures: 'Point-in-time and history questions over a temporal knowledge graph, and the cost of each question.',
    lane: 'graph',
    status: 'measured',
    dataset: {
      name: 'CronQuestions',
      url: 'https://github.com/apoorvumang/CronKGQA',
      license: 'see the dataset repository',
      paper: {
        title: 'Question Answering Over Temporal Knowledge Graphs',
        authors: 'Saxena, Chakrabarti, Talukdar',
        url: 'https://aclanthology.org/2021.acl-long.520/',
        venue: 'ACL 2021',
      },
    },
    harness: {
      repo: 'hanzoai/benchmarks · cronquestions',
      url: 'https://github.com/hanzoai/benchmarks/tree/main/cronquestions',
      command: 'bash cronquestions/run.sh && python3 cron.py site test',
    },
    papers: [],
    data: 'cronquestions',
  },
  {
    slug: 'repobench-r',
    name: 'RepoBench-R',
    title: 'RepoBench-R',
    lede:
      'The retrieval half of RepoBench: a completion point needs one snippet from another file, and the imports, definitions and identifiers say which. Regular expressions over the source, no model reading anything.',
    measures: 'Cross-file retrieval for code completion, scored apart from generation.',
    lane: 'code',
    status: 'measured',
    dataset: {
      name: 'RepoBench-R',
      url: 'https://huggingface.co/datasets/tianyang/repobench-r',
      license: 'CC BY-NC-ND 4.0',
      paper: {
        title: 'RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems',
        authors: 'Liu, Xu, McAuley',
        url: 'https://arxiv.org/abs/2306.03091',
        venue: 'ICLR 2024',
      },
    },
    harness: { repo: 'hanzoai/benchmarks · brain', url: BRAIN, command: 'node results.mjs' },
    papers: [],
    data: 'code',
  },
  {
    slug: 'locomo-subject-scope',
    name: 'LoCoMo · subject scope',
    title: 'Retrieval scoped by subject',
    lede:
      'A second LoCoMo harness, in Go, with no language model anywhere in it: lexical retrieval and a fixed extractive reader, with retrieval scoped to the person a question names, against a flat index. Its numbers are on their own scale, not the engine’s.',
    measures: 'What scoping retrieval to the person a question names changes, against a flat index.',
    lane: 'memory',
    status: 'measured',
    dataset: {
      name: 'LoCoMo',
      url: 'https://github.com/snap-research/locomo',
      license: 'CC BY-NC 4.0',
      paper: {
        title: 'Evaluating Very Long-Term Conversational Memory of LLM Agents',
        authors: 'Maharana, Lee, Tulyakov, Bansal, Barbieri, Fang',
        url: 'https://arxiv.org/abs/2402.17753',
        venue: 'ACL 2024',
      },
    },
    harness: {
      repo: 'hanzoai/semantic · bench/locomo',
      url: 'https://github.com/hanzoai/semantic/tree/v0.3.0/bench/locomo',
      command: 'go run ./bench/locomo',
    },
    papers: [SUBJECT_SCOPE, CONTEXT_MEMORY],
    data: 'semantic',
  },
  {
    slug: 'locomo-conv',
    name: 'LoCoMo-Conv',
    title: 'LoCoMo-Conv',
    lede:
      'The same memories asked the way people ask them — dialog, implicit, counterfactual, composed. The dataset is not released, so what is here is the paper’s retrieval protocol run on the original questions the rewrites came from, and nothing more.',
    measures: 'Conversational rewrites of LoCoMo questions. Dataset unreleased; proxy only.',
    lane: 'memory',
    status: 'baseline',
    dataset: {
      name: 'LoCoMo-Conv',
      url: 'https://arxiv.org/abs/2609.03467',
      license: 'unreleased',
      paper: {
        title: 'LoCoMo-Conv',
        authors: 'Chang & Chen, NTU MiuLab',
        url: 'https://arxiv.org/abs/2609.03467',
        venue: 'September 2026',
      },
    },
    harness: { repo: 'hanzoai/benchmarks · brain', url: `${BRAIN}/conv`, command: 'node results.mjs' },
    papers: [CONTEXT_MEMORY],
    data: 'benchmarks',
  },

  // --- the runtime: what an agent costs at rest, in flight, and on wake ---
  // One measurement pass, on one machine, on one day. The numbers live in
  // hanzoai/cloud bench/README.md rather than in a results JSON, so they are
  // transcribed and say so.
  {
    slug: 'fleet',
    name: 'Fleet residency',
    title: 'What a dormant agent costs',
    lede:
      'A million dormant agents written to disk and read back: 477 bytes of state each, 455 MB for the million, and a resume in 0.034 ms. Measured on a laptop, against a competitor figure that was modelled and never billed.',
    measures: 'Bytes, write rate and resume latency for a dormant agent, at a million of them.',
    lane: 'runtime',
    status: 'measured',
    harness: {
      repo: 'hanzoai/benchmarks · fleet',
      url: 'https://github.com/hanzoai/benchmarks/tree/main/fleet',
      command: 'node fleet/fleet.mjs /tmp/fleet',
    },
    quoted: { where: 'bench/README.md', when: 'the pass of 2026-09-07' },
    papers: [DORMANT],
    data: null,
  },
  {
    slug: 'goroutine',
    name: 'Live agent footprint',
    title: 'An agent as a goroutine',
    lede:
      'A dormant agent is a row; a running one still has to hold its place. 601 bytes of heap per live agent, 125 ns to spawn one, 107 ms to wake a million — and the three language runtimes that can sit inside one.',
    measures: 'Heap and spawn cost of a running agent, and of the runtimes it can host.',
    lane: 'runtime',
    status: 'measured',
    harness: {
      repo: 'hanzoai/benchmarks · goroutine',
      url: 'https://github.com/hanzoai/benchmarks/tree/main/goroutine',
      command: 'cd goroutine && go build -o /tmp/g . && /tmp/g',
    },
    quoted: { where: 'bench/README.md', when: 'the pass of 2026-09-07' },
    papers: [DORMANT],
    data: null,
  },
  {
    slug: 'sandbox',
    name: 'Sandbox cold start',
    title: 'Cold start, by primitive',
    lede:
      'A V8 context ready to execute JavaScript in 0.13 ms, the median of one pass, reproducible by one command.',
    measures: 'Time to a running sandbox, for a V8 context.',
    lane: 'runtime',
    status: 'measured',
    harness: {
      repo: 'hanzoai/benchmarks · sandbox',
      url: 'https://github.com/hanzoai/benchmarks/tree/main/sandbox',
      command: 'node sandbox/sandbox.mjs',
    },
    quoted: { where: 'bench/README.md', when: 'the pass of 2026-09-07' },
    papers: [],
    data: null,
  },
  {
    slug: 'inference',
    name: 'Engine prefill on CUDA vs llama.cpp',
    title: 'Engine prefill on CUDA vs llama.cpp',
    lede:
      'hanzo-engine against llama.cpp on CUDA, built against the same weights and run on the same box: prefill throughput at the prompt lengths where the whole 95% interval of the ratio lies above parity.',
    measures: 'Prefill throughput against llama.cpp on CUDA, by prompt length, with intervals.',
    lane: 'inference',
    status: 'measured',
    harness: {
      repo: 'hanzoai/engine · bench/dossier',
      url: 'https://github.com/hanzoai/engine/tree/bench/dossier/bench-runs',
      command: 'scripts/dossier.sh --backend cuda --model <same.gguf> -p 512,500,2048,4096 -n 128 -r 7',
    },
    quoted: { where: 'the committed run files on branch bench/dossier', when: 'the pass of 2026-07-21' },
    papers: [REFUTATION],
    data: null,
  },

  // --- the router: one hard question set, the router and the field ---
  // The board reads the research API's enso.gpqa_diamond runs when the page
  // opens (components/benchmarks/Ladder); nothing here states a score. No paper
  // is cited: the enso papers carry earlier runs that the runs supersede.
  {
    slug: 'gpqa',
    name: 'GPQA-Diamond',
    title: 'The router on GPQA-Diamond',
    lede:
      'One set of 198 graduate-level questions, answered by Enso\'s router and by single models from other labs, every row a research run.',
    measures: 'Accuracy and what each run\'s tokens cost at list prices, for the router and each single model.',
    lane: 'routing',
    status: 'measured',
    dataset: {
      name: 'GPQA',
      url: 'https://huggingface.co/datasets/Idavidrein/gpqa',
      license: 'CC BY 4.0',
      paper: {
        title: 'GPQA: A Graduate-Level Google-Proof Q&A Benchmark',
        authors: 'Rein, Hou, Stickland, Petty, Pang, Dirani, Michael, Bowman',
        url: 'https://arxiv.org/abs/2311.12022',
        venue: 'COLM 2024',
      },
    },
    harness: { repo: 'api.hanzo.ai/v1/research', url: 'https://api.hanzo.ai/v1/research/runs?org=hanzo&benchmark=enso.gpqa_diamond', command: 'curl -s "https://api.hanzo.ai/v1/research/runs?org=hanzo&benchmark=enso.gpqa_diamond"' },
    quoted: { where: 'the research API', when: 'read when the page opens' },
    papers: [],
    data: null,
  },
]

/** Every paper the library publishes. A card cannot name one the library does not serve. */
export const STUDIES = PAPERS.map((paper) => ({
  title: paper.title,
  note: paper.subtitle,
  href: `${library.source}/${paper.slug}/`,
}))

export const bench = (slug: string) => BENCHMARKS.find((b) => b.slug === slug)

/** The benchmarks of one lane, in the order they were declared. */
export const inLane = (lane: Lane) => BENCHMARKS.filter((b) => b.lane === lane)

/** What a reader is told a status means, in the one place it is spelled. */
export const STATUS: Record<Status, string> = {
  measured: 'measured',
  baseline: 'baseline only',
  soon: 'soon',
}