merge: console enso leaderboard + reported-vs-measured source toggle
This commit is contained in:
hanzo-dev
@@ -10,9 +10,14 @@
|
||||
* Every row shows the score AND ITS SOURCE, because they are not the same kind of
|
||||
* number: `hanzo-measured` is our own harness through api.hanzo.ai (what the router
|
||||
* actually gets), while the provider- and third-party-reported figures are context
|
||||
* measured on someone else's harness at someone else's effort setting. Collapsing
|
||||
* those into one "score" column without provenance would be quietly dishonest, so the
|
||||
* source rides along and `hanzo-measured` is badged.
|
||||
* measured on someone else's harness at someone else's effort setting. The Source
|
||||
* toggle filters the two apart — the honest transparency this surface exists for —
|
||||
* and every row is badged by its class (Enso · Hanzo-measured · reported).
|
||||
*
|
||||
* The Enso family (enso-flash / enso / enso-ultra) is our own product; it ranks in
|
||||
* the table on merit like every other model — never floated to the top — and the
|
||||
* tier strip above places the three tiers side by side, monotonic and priced, from
|
||||
* the same corpus numbers.
|
||||
*
|
||||
* Models with no published score on the selected benchmark are OMITTED, never shown
|
||||
* as zero — an absent measurement is not a bad measurement.
|
||||
@@ -25,9 +30,13 @@ import {
|
||||
BENCHMARK_IDS,
|
||||
benchmarkLabel,
|
||||
coverage,
|
||||
isEnsoModel,
|
||||
leaderboard,
|
||||
priorFor,
|
||||
sourceClass,
|
||||
vendors,
|
||||
type Ranked,
|
||||
type SourceClass,
|
||||
} from '~/lib/api/benchmarks'
|
||||
import { DataTable, type Column } from '~/components/ui/DataTable'
|
||||
import { ProviderLogo } from '~/components/ui/ProviderLogo'
|
||||
@@ -37,28 +46,34 @@ const TNUM = 'hz-tnum'
|
||||
|
||||
const ALL = '__all__'
|
||||
|
||||
/** The Source filter: all rows, only our harness, or only vendor/third-party. */
|
||||
type SrcFilter = 'all' | SourceClass
|
||||
const SRC_OPTIONS: { key: SrcFilter; label: string }[] = [
|
||||
{ key: 'all', label: 'All' },
|
||||
{ key: 'measured', label: 'Hanzo-measured' },
|
||||
{ key: 'reported', label: 'Vendor-reported' },
|
||||
]
|
||||
|
||||
/** The default benchmark: the one the corpus covers most broadly. */
|
||||
const defaultBenchmark = (): string =>
|
||||
BENCHMARK_IDS.slice().sort((a, b) => coverage(b) - coverage(a))[0] ?? BENCHMARK_IDS[0] ?? ''
|
||||
|
||||
/** A quiet chip. `tone="measured"` marks our own harness — the number we trust most. */
|
||||
function Chip({ label, tone = 'muted' }: { label: string; tone?: 'muted' | 'measured' }) {
|
||||
/**
|
||||
* A quiet class chip. `enso` marks our own orchestrated family (Hanzo red),
|
||||
* `measured` marks our own harness (green) — the numbers we trust most — and
|
||||
* `muted` carries a reported source string verbatim.
|
||||
*/
|
||||
function Chip({ label, tone = 'muted' }: { label: string; tone?: 'muted' | 'measured' | 'enso' }) {
|
||||
const bg = tone === 'enso' ? '$red3' : tone === 'measured' ? '$green3' : '$color3'
|
||||
const color = tone === 'enso' ? '$red11' : tone === 'measured' ? '$green11' : '$color11'
|
||||
return (
|
||||
<Text
|
||||
fontSize="$1"
|
||||
px="$2"
|
||||
py="$1"
|
||||
rounded="$2"
|
||||
bg={tone === 'measured' ? '$green3' : '$color3'}
|
||||
color={tone === 'measured' ? '$green11' : '$color11'}
|
||||
numberOfLines={1}
|
||||
>
|
||||
<Text fontSize="$1" px="$2" py="$1" rounded="$2" bg={bg} color={color} numberOfLines={1}>
|
||||
{label}
|
||||
</Text>
|
||||
)
|
||||
}
|
||||
|
||||
/** A horizontally scrolling row of selector pills (benchmarks, vendors). */
|
||||
/** A horizontally scrolling row of selector pills (benchmarks, vendors, source). */
|
||||
function PillRow({
|
||||
options,
|
||||
value,
|
||||
@@ -88,9 +103,83 @@ function PillRow({
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The three Enso tiers, side by side and monotonic in quality (Ultra > Pro > Flash),
|
||||
* priced. Every number is read from the SAME corpus the table ranks on — no separate
|
||||
* marketing figures — so the strip renders only when all three tiers carry a
|
||||
* GPQA-Diamond score (they do once the enso family is synced in). Pro is marked as the
|
||||
* balanced default. An honest surface: no projected numbers, no fabricated #1.
|
||||
*/
|
||||
const ENSO_TIERS: { id: string; name: string; note: string; isDefault?: boolean }[] = [
|
||||
{ id: 'enso-flash', name: 'Enso Flash', note: 'Cheapest — high-volume, escalate only if needed' },
|
||||
{ id: 'enso', name: 'Enso Pro', note: 'The balanced default — routed to the best-fit model per request', isDefault: true },
|
||||
{ id: 'enso-ultra', name: 'Enso Ultra', note: 'Maximum verified quality — adaptive fan-out on the hardest problems' },
|
||||
]
|
||||
|
||||
function EnsoTiers({ benchmark }: { benchmark: string }) {
|
||||
// Prefer the selected benchmark; fall back to GPQA-Diamond (the tiers' shared
|
||||
// headline) so the strip is populated even when the table is on another benchmark.
|
||||
const tiers = ENSO_TIERS.map((t) => {
|
||||
const prior = priorFor(t.id)
|
||||
const score = prior?.scores[benchmark] ?? prior?.scores.gpqa_diamond ?? null
|
||||
const bench = prior?.scores[benchmark] ? benchmark : 'gpqa_diamond'
|
||||
return { ...t, score, benchLabel: benchmarkLabel(bench), price: prior?.price ?? null }
|
||||
})
|
||||
// Honest: only render if the corpus actually carries the family (post-sync).
|
||||
if (tiers.some((t) => t.score == null)) return null
|
||||
|
||||
return (
|
||||
<YStack gap="$2">
|
||||
<Text fontSize="$1" color="$color10">
|
||||
The Enso tiers — one price/quality contract each, from the corpus below
|
||||
</Text>
|
||||
<XStack gap="$2.5" flexWrap="wrap">
|
||||
{tiers.map((t) => (
|
||||
<YStack
|
||||
key={t.id}
|
||||
flex={1}
|
||||
minW={190}
|
||||
gap="$1.5"
|
||||
p="$3"
|
||||
rounded="$4"
|
||||
borderWidth={1}
|
||||
borderColor={t.isDefault ? '$red7' : '$borderColor'}
|
||||
bg="$color1"
|
||||
>
|
||||
<XStack items="center" gap="$2" justify="space-between">
|
||||
<Text fontSize="$3" fontWeight="700" color="$color12">
|
||||
{t.name}
|
||||
</Text>
|
||||
{t.isDefault ? <Chip label="default" tone="enso" /> : null}
|
||||
</XStack>
|
||||
<XStack items="baseline" gap="$2">
|
||||
<Text className={TNUM} fontSize="$7" fontWeight="800" color="$color12">
|
||||
{t.score!.value.toFixed(1)}
|
||||
</Text>
|
||||
<Text fontSize="$1" color="$color10">
|
||||
{t.benchLabel}
|
||||
</Text>
|
||||
</XStack>
|
||||
<XStack items="center" gap="$2" flexWrap="wrap">
|
||||
<Text className={TNUM} fontSize="$2" color="$color11">
|
||||
{t.price == null ? '—' : `$${t.price.toFixed(2)}/M blended`}
|
||||
</Text>
|
||||
<Chip label="Hanzo-measured" tone="measured" />
|
||||
</XStack>
|
||||
<Text fontSize="$1" color="$color10" numberOfLines={2}>
|
||||
{t.note}
|
||||
</Text>
|
||||
</YStack>
|
||||
))}
|
||||
</XStack>
|
||||
</YStack>
|
||||
)
|
||||
}
|
||||
|
||||
export function LeaderboardView() {
|
||||
const [benchmark, setBenchmark] = useState<string>(defaultBenchmark)
|
||||
const [vendor, setVendor] = useState<string>(ALL)
|
||||
const [src, setSrc] = useState<SrcFilter>('all')
|
||||
|
||||
// Only offer benchmarks the corpus actually covers — a benchmark with zero scored
|
||||
// models is not a view worth selecting.
|
||||
@@ -106,8 +195,13 @@ export function LeaderboardView() {
|
||||
|
||||
const ranked = useMemo(() => leaderboard(benchmark), [benchmark])
|
||||
const rows = useMemo(
|
||||
() => (vendor === ALL ? ranked : ranked.filter((r) => r.vendor === vendor)),
|
||||
[ranked, vendor],
|
||||
() =>
|
||||
ranked.filter(
|
||||
(r) =>
|
||||
(vendor === ALL || r.vendor === vendor) &&
|
||||
(src === 'all' || sourceClass(r.score.source) === src),
|
||||
),
|
||||
[ranked, vendor, src],
|
||||
)
|
||||
|
||||
const vendorOptions = useMemo(
|
||||
@@ -115,7 +209,7 @@ export function LeaderboardView() {
|
||||
[],
|
||||
)
|
||||
|
||||
const measured = rows.filter((r) => r.score.source === 'hanzo-measured').length
|
||||
const measured = rows.filter((r) => sourceClass(r.score.source) === 'measured').length
|
||||
|
||||
const columns: Column<Ranked>[] = [
|
||||
{
|
||||
@@ -139,6 +233,7 @@ export function LeaderboardView() {
|
||||
<Text fontSize="$3" color="$color12" numberOfLines={1}>
|
||||
{r.model}
|
||||
</Text>
|
||||
{isEnsoModel(r.model) ? <Chip label="Enso" tone="enso" /> : null}
|
||||
</XStack>
|
||||
),
|
||||
},
|
||||
@@ -169,7 +264,9 @@ export function LeaderboardView() {
|
||||
header: 'Source',
|
||||
width: 190,
|
||||
render: (r) =>
|
||||
r.score.source === 'hanzo-measured' ? (
|
||||
isEnsoModel(r.model) ? (
|
||||
<Chip label="Enso · measured" tone="enso" />
|
||||
) : sourceClass(r.score.source) === 'measured' ? (
|
||||
<Chip label="Hanzo-measured" tone="measured" />
|
||||
) : (
|
||||
<Text fontSize="$1" color="$color10" numberOfLines={1}>
|
||||
@@ -195,9 +292,11 @@ export function LeaderboardView() {
|
||||
<YStack gap="$3.5">
|
||||
<PageHeader
|
||||
title="Leaderboard"
|
||||
subtitle="Published benchmark scores from the enso-bench prior corpus. Every score carries its source; models without a published score on the selected benchmark are not listed."
|
||||
subtitle="Published benchmark scores from the enso-bench prior corpus. Vendors report on their own harness; we measure everyone on one — toggle Source to see which is which. Models without a published score on the selected benchmark are not listed."
|
||||
/>
|
||||
|
||||
<EnsoTiers benchmark={benchmark} />
|
||||
|
||||
<YStack gap="$2">
|
||||
<Text fontSize="$1" color="$color10">
|
||||
Benchmark
|
||||
@@ -205,6 +304,13 @@ export function LeaderboardView() {
|
||||
<PillRow options={benchOptions} value={benchmark} onSelect={setBenchmark} />
|
||||
</YStack>
|
||||
|
||||
<YStack gap="$2">
|
||||
<Text fontSize="$1" color="$color10">
|
||||
Source
|
||||
</Text>
|
||||
<PillRow options={SRC_OPTIONS} value={src} onSelect={(k) => setSrc(k as SrcFilter)} />
|
||||
</YStack>
|
||||
|
||||
<YStack gap="$2">
|
||||
<Text fontSize="$1" color="$color10">
|
||||
Vendor
|
||||
@@ -225,9 +331,13 @@ export function LeaderboardView() {
|
||||
rows={rows}
|
||||
rowKey={(r) => r.model}
|
||||
empty={
|
||||
vendor === ALL
|
||||
? `No published scores for ${benchmarkLabel(benchmark)} in the corpus.`
|
||||
: `No ${vendor} model has a published ${benchmarkLabel(benchmark)} score in the corpus.`
|
||||
src === 'measured'
|
||||
? `No Hanzo-measured ${benchmarkLabel(benchmark)} score${vendor === ALL ? '' : ` for ${vendor}`} in the corpus.`
|
||||
: src === 'reported'
|
||||
? `No vendor-reported ${benchmarkLabel(benchmark)} score${vendor === ALL ? '' : ` for ${vendor}`} in the corpus.`
|
||||
: vendor === ALL
|
||||
? `No published scores for ${benchmarkLabel(benchmark)} in the corpus.`
|
||||
: `No ${vendor} model has a published ${benchmarkLabel(benchmark)} score in the corpus.`
|
||||
}
|
||||
/>
|
||||
|
||||
|
||||
@@ -51,6 +51,62 @@
|
||||
]
|
||||
},
|
||||
"models": [
|
||||
{
|
||||
"model": "enso-ultra",
|
||||
"vendor": "Hanzo",
|
||||
"scores": {
|
||||
"gpqa_diamond": {
|
||||
"value": 92.9,
|
||||
"source": "hanzo-measured"
|
||||
},
|
||||
"charxiv_reasoning": {
|
||||
"value": 85.5,
|
||||
"source": "hanzo-measured"
|
||||
},
|
||||
"humanitys_last_exam": {
|
||||
"value": 34.6,
|
||||
"source": "hanzo-measured"
|
||||
}
|
||||
},
|
||||
"price": 75,
|
||||
"intelligence": 92.9
|
||||
},
|
||||
{
|
||||
"model": "enso",
|
||||
"vendor": "Hanzo",
|
||||
"scores": {
|
||||
"gpqa_diamond": {
|
||||
"value": 87.9,
|
||||
"source": "hanzo-measured"
|
||||
},
|
||||
"livecodebench": {
|
||||
"value": 92,
|
||||
"source": "hanzo-measured"
|
||||
},
|
||||
"charxiv_reasoning": {
|
||||
"value": 78,
|
||||
"source": "hanzo-measured"
|
||||
},
|
||||
"humanitys_last_exam": {
|
||||
"value": 29.4,
|
||||
"source": "hanzo-measured"
|
||||
}
|
||||
},
|
||||
"price": 75,
|
||||
"intelligence": 87.9
|
||||
},
|
||||
{
|
||||
"model": "enso-flash",
|
||||
"vendor": "Hanzo",
|
||||
"scores": {
|
||||
"gpqa_diamond": {
|
||||
"value": 75.8,
|
||||
"source": "hanzo-measured"
|
||||
}
|
||||
},
|
||||
"price": 6,
|
||||
"intelligence": 75.8
|
||||
},
|
||||
{
|
||||
"model": "gpt-5.6-sol",
|
||||
"vendor": "OpenAI",
|
||||
@@ -185,7 +241,7 @@
|
||||
},
|
||||
"livecodebench": {
|
||||
"value": 92.9,
|
||||
"source": "fugu-report"
|
||||
"source": "provider-reported"
|
||||
}
|
||||
},
|
||||
"price": 42,
|
||||
@@ -693,15 +749,15 @@
|
||||
"scores": {
|
||||
"gpqa_diamond": {
|
||||
"value": 94.3,
|
||||
"source": "fugu-report"
|
||||
"source": "provider-reported"
|
||||
},
|
||||
"swe_bench_pro": {
|
||||
"value": 54.2,
|
||||
"source": "fugu-report"
|
||||
"source": "provider-reported"
|
||||
},
|
||||
"livecodebench": {
|
||||
"value": 88.5,
|
||||
"source": "fugu-report"
|
||||
"source": "provider-reported"
|
||||
},
|
||||
"livecodebench_pro": {
|
||||
"value": 2887,
|
||||
@@ -885,15 +941,15 @@
|
||||
"scores": {
|
||||
"gpqa_diamond": {
|
||||
"value": 93.6,
|
||||
"source": "fugu-report"
|
||||
"source": "provider-reported"
|
||||
},
|
||||
"swe_bench_pro": {
|
||||
"value": 58.6,
|
||||
"source": "fugu-report"
|
||||
"source": "provider-reported"
|
||||
},
|
||||
"livecodebench": {
|
||||
"value": 85.3,
|
||||
"source": "fugu-report"
|
||||
"source": "provider-reported"
|
||||
},
|
||||
"terminal_bench": {
|
||||
"value": 84.3,
|
||||
@@ -937,11 +993,11 @@
|
||||
},
|
||||
"swe_bench_pro": {
|
||||
"value": 69.2,
|
||||
"source": "fugu-report"
|
||||
"source": "provider-reported"
|
||||
},
|
||||
"livecodebench": {
|
||||
"value": 87.8,
|
||||
"source": "fugu-report"
|
||||
"source": "provider-reported"
|
||||
},
|
||||
"terminal_bench": {
|
||||
"value": 84.6,
|
||||
@@ -2488,6 +2544,50 @@
|
||||
[
|
||||
"openai/gpt-5.6-sol",
|
||||
"gpt-5.6-sol@openrouter"
|
||||
],
|
||||
[
|
||||
"google/gemini-3.1-pro-preview",
|
||||
"gemini-3.1-pro@openrouter"
|
||||
],
|
||||
[
|
||||
"x-ai/grok-4.5",
|
||||
"grok-4.5@openrouter"
|
||||
],
|
||||
[
|
||||
"moonshotai/kimi-k3",
|
||||
"kimi-k3@openrouter"
|
||||
],
|
||||
[
|
||||
"anthropic/claude-fable-5",
|
||||
"fable-5@openrouter"
|
||||
],
|
||||
[
|
||||
"openai/gpt-5.5",
|
||||
"gpt-5.5@openrouter"
|
||||
],
|
||||
[
|
||||
"deepseek/deepseek-v4-flash",
|
||||
"deepseek-v4-flash@openrouter"
|
||||
],
|
||||
[
|
||||
"openai/gpt-5.6-luna",
|
||||
"gpt-5.6-luna@openrouter"
|
||||
],
|
||||
[
|
||||
"xiaomi/mimo-v2.5-pro",
|
||||
"mimo-v2.5-pro@openrouter"
|
||||
],
|
||||
[
|
||||
"minimax/minimax-m3",
|
||||
"minimax-m3@openrouter"
|
||||
],
|
||||
[
|
||||
"tencent/hy3",
|
||||
"hy3@openrouter"
|
||||
],
|
||||
[
|
||||
"sakana/fugu-ultra",
|
||||
"fugu-ultra@openrouter"
|
||||
]
|
||||
]
|
||||
}
|
||||
|
||||
@@ -10,10 +10,12 @@ import {
|
||||
BENCHMARK_IDS,
|
||||
benchmarkLabel,
|
||||
coverage,
|
||||
isEnsoModel,
|
||||
leaderboard,
|
||||
normalizeModelKey,
|
||||
priorFor,
|
||||
scoreFor,
|
||||
sourceClass,
|
||||
vendors,
|
||||
} from './benchmarks'
|
||||
|
||||
@@ -108,6 +110,66 @@ describe('priorFor / scoreFor', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('sourceClass — the measured-vs-reported binary', () => {
|
||||
it('classes only the literal `hanzo-measured` as our own harness', () => {
|
||||
expect(sourceClass('hanzo-measured')).toBe('measured')
|
||||
})
|
||||
|
||||
it('classes every vendor/third-party source as reported, however reputable', () => {
|
||||
for (const s of [
|
||||
'provider-reported',
|
||||
'Vals AI',
|
||||
'Vals AI — GPQA Diamond leaderboard',
|
||||
'Artificial Analysis',
|
||||
'do-catalog',
|
||||
'OpenAI — Introducing GPT-5',
|
||||
]) {
|
||||
expect(sourceClass(s), s).toBe('reported')
|
||||
}
|
||||
})
|
||||
|
||||
it('every corpus score classes into exactly one of the two', () => {
|
||||
for (const m of allPriors()) {
|
||||
for (const s of Object.values(m.scores)) {
|
||||
expect(['measured', 'reported']).toContain(sourceClass(s.source))
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('the Enso family — synced in, honest numbers, differentiated', () => {
|
||||
it('recognizes the three tiers and nothing else', () => {
|
||||
expect(isEnsoModel('enso')).toBe(true)
|
||||
expect(isEnsoModel('enso-flash')).toBe(true)
|
||||
expect(isEnsoModel('enso-ultra')).toBe(true)
|
||||
expect(isEnsoModel('ensoteric')).toBe(false) // not a tier — must not false-match
|
||||
expect(isEnsoModel('zen5-flash')).toBe(false)
|
||||
expect(isEnsoModel('gpt-5.6-sol')).toBe(false)
|
||||
})
|
||||
|
||||
it('carries the measured GPQA-Diamond numbers, monotonic Ultra > Pro > Flash', () => {
|
||||
const ultra = priorFor('enso-ultra')?.scores.gpqa_diamond
|
||||
const pro = priorFor('enso')?.scores.gpqa_diamond
|
||||
const flash = priorFor('enso-flash')?.scores.gpqa_diamond
|
||||
expect(ultra?.value).toBe(92.9)
|
||||
expect(pro?.value).toBe(87.9)
|
||||
expect(flash?.value).toBe(75.8)
|
||||
// Every tier is our own harness, and the family is strictly monotonic in quality.
|
||||
for (const s of [ultra, pro, flash]) expect(s?.source).toBe('hanzo-measured')
|
||||
expect(ultra!.value).toBeGreaterThan(pro!.value)
|
||||
expect(pro!.value).toBeGreaterThan(flash!.value)
|
||||
})
|
||||
|
||||
it('ranks on merit in the GPQA board — present, but never floated to #1', () => {
|
||||
const board = leaderboard('gpqa_diamond')
|
||||
const ultra = board.find((r) => r.model === 'enso-ultra')
|
||||
expect(ultra).toBeDefined()
|
||||
// Honest placement: strong but not the top of the corpus (reported frontier
|
||||
// models score higher). We never fabricate a #1.
|
||||
expect(ultra!.rank).toBeGreaterThan(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('leaderboard', () => {
|
||||
it('ranks by score, best first, with 1-based ranks', () => {
|
||||
const rows = leaderboard('gpqa_diamond')
|
||||
|
||||
@@ -95,6 +95,29 @@ export function normalizeModelKey(rawKey: string): string {
|
||||
return s
|
||||
}
|
||||
|
||||
// ── Source provenance (the honest measured-vs-reported binary) ────────────────
|
||||
|
||||
/**
|
||||
* The two kinds of number in the corpus. `measured` is our own harness through
|
||||
* api.hanzo.ai — one common effort setting, complete runs, the figure the router
|
||||
* actually gets. `reported` is a vendor's or a third party's leaderboard, measured
|
||||
* on someone else's harness at someone else's effort. The corpus carries a fine
|
||||
* `source` string on every score; this folds it to the one distinction a reader
|
||||
* filters on. Only the literal `hanzo-measured` is ours — everything else, however
|
||||
* reputable (Vals AI, Artificial Analysis, a model card), is reported.
|
||||
*/
|
||||
export type SourceClass = 'measured' | 'reported'
|
||||
export const sourceClass = (source: string): SourceClass =>
|
||||
source === 'hanzo-measured' ? 'measured' : 'reported'
|
||||
|
||||
/**
|
||||
* The Enso family — our own orchestrated tiers (enso-flash / enso / enso-ultra),
|
||||
* measured on our harness. Badged apart from other measured models because they
|
||||
* are the product this surface exists to place honestly among its peers.
|
||||
*/
|
||||
export const isEnsoModel = (modelIdOrName: string): boolean =>
|
||||
/^enso(-|$)/.test(normalizeModelKey(modelIdOrName))
|
||||
|
||||
/** Corpus alias pairs: a gateway model id → its canonical corpus name. */
|
||||
const ALIASES: Record<string, string> = (() => {
|
||||
const m: Record<string, string> = {}
|
||||
|
||||
Reference in New Issue
Block a user