hanzo.ai/app/(marketing)/benchmarks/cronquestions/page.tsx

/**
 * /benchmarks/cronquestions — the graph API against Semantica's temporal store,
 * same facts, same questions, same machine.
 *
 * Every figure is read from lib/data/benchmarks-cronquestions.json, which the
 * harness writes from the runs it scored. Both stores give the same answers, so
 * the accuracy table is the context and the price of a question is the result.
 */
import { phrase } from '@hanzo/i18n'
import { T } from '@hanzo/i18n/react'
import { pageMeta } from '@/lib/page-meta'
import { BENCHMARKS, bench } from '@/lib/benchmarks'
import { BenchJsonLd } from '@/components/benchmarks/JsonLd'
import { Cell, F, Grid, H2, Harness, Meta, More, P, Shell, Spec, elapsed, type Col } from '@/components/benchmarks/kit'
import { CRON, cron, type CronRow } from '@/lib/bench-data'

const B = bench('cronquestions')!

export const metadata = pageMeta({
  title: 'CronQuestions benchmark',
  description:
    'CronQuestions benchmark: Hanzo graph API answers 99.9% across 26,168 questions at 0.06 ms per question vs 5.4s for Semantica.',
  path: '/benchmarks/cronquestions',
})

const MEASURES = ['Hits@1', 'query latency p50', 'query latency p99']

const NAME: Record<string, string> = {
  kg: phrase('control: the plan over dictionaries'),
  semantica: 'Semantica 0.7.0',
  'hanzo-wire': phrase('Hanzo graph, through the API'),
  'hanzo-replay': phrase('Hanzo graph, replayed write clock'),
}
const TYPES: [string, string][] = [
  ['overall', phrase('all')],
  ['simple_entity', phrase('who held it in year T')],
  ['simple_time', phrase('when')],
  ['before_after', phrase('before / after')],
  ['first_last', phrase('first / last')],
]

const accuracy: Col<CronRow>[] = [
  { head: <T>store</T>, get: (r) => (NAME[r.row] ? <T>{NAME[r.row]}</T> : r.row), left: true },
  ...TYPES.map(([key, head]) => ({ head: <T>{head}</T>, get: (r: CronRow) => <Cell m={r.hits[key]} /> })),
]

const cost: Col<CronRow>[] = [
  { head: <T>store</T>, get: (r) => (NAME[r.row] ? <T>{NAME[r.row]}</T> : r.row), left: true },
  { head: <T>query p50</T>, get: (r) => elapsed(r.query_ms.all?.p50) },
  { head: <T>query p99</T>, get: (r) => elapsed(r.query_ms.all?.p99) },
  { head: <T>load average</T>, get: (r) => (r.load_1m.at_start == null ? '–' : r.load_1m.at_start.toFixed(1)) },
]

const ours = cron('hanzo-wire')
const theirs = cron('semantica')
const control = cron('kg')
const N = {
  ours: ours?.hits.overall,
  theirs: theirs?.hits.overall,
  control: control?.hits.overall,
  oursP50: ours?.query_ms.all?.p50,
  theirsP50: theirs?.query_ms.all?.p50,
  ratio: ours?.query_ms.all?.p50 && theirs?.query_ms.all?.p50 ? theirs.query_ms.all.p50 / ours.query_ms.all.p50 : null,
  facts: ours?.load.facts,
  refused: Object.values(ours?.load.refused ?? {}).reduce((a, b) => a + b, 0),
}

export default function Page() {
  return (
    <>
      <BenchJsonLd b={B} measures={MEASURES} />
      <Shell b={B}>
        <Spec b={B} />

        <H2><T>What is measured</T></H2>
        <P>
          <T values={{ facts: N.facts?.toLocaleString('en-US') }}>
            {'CronQuestions is {facts} Wikidata facts — who held a position, played for a team, received an award — each with a start and an end year, and questions about them: who held a position in a given year, when someone held it, who held it before or after someone else, who held it first or last. The questions come annotated with their entities and years, so each is answered by a fixed plan of store calls and no model reads anything. What differs between rows is only the store the plan calls.'}
          </T>
        </P>
        <P>
          <T values={{ graph: <code>POST /v1/graph</code>, resolve: <code>POST /v1/graph/resolve</code>, known: <code>as_known</code> }}>
            {'Both stores load the same facts and run in-process on one machine; nothing crosses a network. The Hanzo rows go through the graph’s own admission and its own resolution — what {graph} and {resolve} do — asking every question at {known} now, as any caller does. Semantica is driven through its documented temporal query calls.'}
          </T>
        </P>

        <H2><T>Results</T></H2>
        <Meta>
          <T values={{ n: CRON.covered.length, ours: ours?.subject, theirs: theirs?.subject }}>
            {'test split, {n} question types, Hits@1 with bootstrap 95% intervals · {ours} · {theirs}'}
          </T>
        </Meta>
        <Grid cols={accuracy} rows={CRON.rows} keyOf={(r) => r.row} />
        <Meta><T>cost of a question</T></Meta>
        <Grid cols={cost} rows={CRON.rows.filter((r) => r.row !== 'kg')} keyOf={(r) => r.row} />

        <H2><T>What it says</T></H2>
        <P>
          <T values={{ ours: F(N.ours), theirs: F(N.theirs), control: F(N.control) }}>
            {'The graph API answers {ours}% of the questions, and Semantica {theirs}% — each the control’s answer, question for question. {control}% is the plan’s own ceiling: the before/after questions it misses, it misses with the facts in a dictionary too.'}
          </T>
        </P>
        <P>
          <T
            values={{
              ours: elapsed(N.oursP50),
              theirs: elapsed(N.theirsP50),
              ratio: N.ratio == null ? '–' : Math.round(N.ratio / 1000).toLocaleString('en-US'),
            }}
          >
            {'The difference is the price of a question. A point-in-time question about one entity costs the graph one indexed read of that entity’s relation, {ours} at the median. Semantica’s temporal calls take no entity, so every question about one person reconstructs the whole graph at that year: {theirs}. That is about {ratio} thousand times the cost for the same answer.'}
          </T>
        </P>

        <H2><T>What this table is not</T></H2>
        <P>
          <T values={{ refused: N.refused }}>
            {'It is not question answering from text. The questions arrive annotated, so this measures the store — whether it gives the right facts for a point in time, and at what cost — not a model’s reading of a question. {refused} facts that start after today were refused at admission, which rejects a start more than five minutes ahead of the server clock; no covered question asks about them. The 3,832 time-join questions are not covered. The Hanzo timings were taken on a loaded machine, and each run records the load it had.'}
          </T>
        </P>

        <More b={B} all={BENCHMARKS} />
        <Meta>
          <T values={{ harness: <Harness of={B.harness} />, command: <code>{B.harness.command}</code> }}>
            {'raw runs and scoring: {harness} · regenerated by {command}'}
          </T>
        </Meta>
      </Shell>
    </>
  )
}