mirror of
https://github.com/hanzo-apps/hanzo.ai.git
synced 2026-10-28 07:32:52 +00:00
zooqueenandHanzo Dev
a10c549d0b
Every run on the GPQA Diamond and LiveCodeBench boards is priced by one rule: its recorded input and output tokens at the rate GET /v1/pricing states for the model id its system bills under, the router's included (enso at enso's rate). The cost a run files is enso-bench's own table and is no longer read. The landing tile, /benchmarks, /enso and /home2 draw the multiple from lib/enso alone, labelled with the list's own date, and draw none without the list. The boards drop the router's arms and the run notes; the copy beside them says the rows are single models from other labs; the client brand filter that listed Zen's bases is gone (the research API withholds those runs). A gate reads every text file in out/ and fails on a model, vendor or router id said to be inside Enso, Zen or Kai, and absorbs the Laya gate. The setup prompt ends with `hanzo dev`, and /home2 runs Claude Code on Hanzo with `hanzo dev --claude --cloud`; `hanzo up` boots a local Kubernetes and `hanzo code claude` is refused by the CLI hanzo.sh installs. Co-authored-by: Hanzo Dev <[email protected]>
148 lines
7.1 KiB
TypeScript
148 lines
7.1 KiB
TypeScript
import type { Comparison } from './compare'
|
|
|
|
/**
|
|
* Every /compare page that exists, in one list.
|
|
*
|
|
* Populated from two places only: our own benchmark runs, and the rival's own
|
|
* published pages read on a dated pass. A row with neither does not get written.
|
|
*
|
|
* All vendor values below were read on 2026-10-04 from the primary vendor page
|
|
* linked in the row. None of them were verified by us — they are relayed. Where a
|
|
* vendor's own pages disagree, the disagreement is on the row rather than
|
|
* smoothed over.
|
|
*/
|
|
export const COMPARISONS: Comparison[] = [
|
|
{
|
|
slug: 'model-routing',
|
|
category: 'Model routing APIs',
|
|
question: 'Which API should you route your model calls through?',
|
|
framing:
|
|
'All four put many models behind one key. They differ less than any of them advertise — and where they do differ, it is not always in our favour.',
|
|
subject: { name: 'Enso', href: '/enso' },
|
|
rivals: [
|
|
{ id: 'openrouter', name: 'OpenRouter', href: 'https://openrouter.ai' },
|
|
{ id: 'portkey', name: 'Portkey', href: 'https://portkey.ai' },
|
|
{ id: 'litellm', name: 'LiteLLM', href: 'https://www.litellm.ai' },
|
|
],
|
|
rows: [
|
|
{
|
|
attribute: 'Models reachable',
|
|
source: 'reported',
|
|
ours: '600+',
|
|
theirs: {
|
|
openrouter: '500+ (80+ providers)',
|
|
portkey: '1,600+',
|
|
litellm: '1,800+ (140+ providers)',
|
|
},
|
|
theirRefs: {
|
|
openrouter: { href: 'https://openrouter.ai', date: '2026-10-04' },
|
|
portkey: { href: 'https://portkey.ai', date: '2026-10-04' },
|
|
litellm: { href: 'https://www.litellm.ai', date: '2026-10-04' },
|
|
},
|
|
note: 'We are behind here. Portkey also states 200+ in its Universal API docs and 3,000+ in its Messages API docs — three counts on three pages of one product.',
|
|
},
|
|
{
|
|
attribute: 'OpenAI-compatible endpoint',
|
|
source: 'reported',
|
|
ours: 'Yes — /v1/chat/completions',
|
|
theirs: { openrouter: 'Yes', portkey: 'Yes', litellm: 'Yes' },
|
|
theirRefs: {
|
|
openrouter: { href: 'https://openrouter.ai/docs/faq', date: '2026-10-04' },
|
|
portkey: { href: 'https://docs.portkey.ai/docs/product/ai-gateway/universal-api', date: '2026-10-04' },
|
|
litellm: { href: 'https://docs.litellm.ai/docs/proxy/client_setup/claude_code', date: '2026-10-04' },
|
|
},
|
|
note: 'Table stakes. Nobody wins this row.',
|
|
},
|
|
{
|
|
attribute: 'Native Anthropic /v1/messages',
|
|
source: 'reported',
|
|
ours: 'Yes — same key',
|
|
theirs: {
|
|
openrouter: 'Yes — Anthropic Messages format, tools and extended thinking',
|
|
portkey: 'Yes — /v1/messages routes to any of their models',
|
|
litellm: 'Yes — serves the format at /v1/messages',
|
|
},
|
|
theirRefs: {
|
|
openrouter: {
|
|
href: 'https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-a-message',
|
|
date: '2026-10-04',
|
|
},
|
|
portkey: {
|
|
href: 'https://docs.portkey.ai/docs/product/ai-gateway/messages-api',
|
|
date: '2026-10-04',
|
|
},
|
|
litellm: {
|
|
href: 'https://docs.litellm.ai/docs/pass_through/anthropic_completion',
|
|
date: '2026-10-04',
|
|
},
|
|
},
|
|
note: 'Worth stating plainly because it is not a differentiator. All four have it. A page that claimed otherwise would be checked in a minute.',
|
|
},
|
|
{
|
|
attribute: 'Cost known before the call runs',
|
|
source: 'reported',
|
|
ours: 'Yes — model calls are quoted from above before they happen, and an agent that would breach its cap is refused rather than discovered on an invoice',
|
|
theirs: {
|
|
openrouter: 'No — blocks once a limit is reached (403), Enterprise plan only. No pre-flight quote published.',
|
|
portkey: 'No — budget limit expires the key once reached, Enterprise plan. No pre-flight quote published.',
|
|
litellm: 'No — caps budgets and stops at the cap. No pre-flight quote published.',
|
|
},
|
|
theirRefs: {
|
|
openrouter: {
|
|
href: 'https://openrouter.ai/docs/guides/features/workspaces/workspace-budgets',
|
|
date: '2026-10-04',
|
|
},
|
|
portkey: {
|
|
href: 'https://docs.portkey.ai/docs/product/ai-gateway/virtual-keys/budget-limits',
|
|
date: '2026-10-04',
|
|
},
|
|
litellm: { href: 'https://www.litellm.ai', date: '2026-10-04' },
|
|
},
|
|
note: 'Capping and quoting are different things. A cap stops you after you have spent; a quote tells you the price before you ask. None of the three publish the second.',
|
|
},
|
|
{
|
|
attribute: 'GPQA-Diamond, run by us',
|
|
source: 'measured',
|
|
ours: 'The router beside single models from other labs on the same 198 questions, every row a research run with its accuracy and what its tokens cost at list prices',
|
|
oursRef: { href: 'https://hanzo.ai/benchmarks/gpqa', date: '2026-10-15' },
|
|
theirs: {
|
|
openrouter: 'not measured by us',
|
|
portkey: 'not measured by us',
|
|
litellm: 'not measured by us',
|
|
},
|
|
note: 'One set of 198 graduate-level questions through the router and the field, read from the research API. We have not run the other three routers on this set, so their cells say that rather than borrowing a marketing number to sit beside a measurement.',
|
|
},
|
|
{
|
|
attribute: 'Routing quality published by the vendor',
|
|
source: 'reported',
|
|
ours: 'Yes — GPQA-Diamond, 198 graduate-level questions, every row a research run',
|
|
theirs: {
|
|
openrouter: 'No — their Benchmarks API scores the models, not the router',
|
|
portkey: 'Not published',
|
|
litellm: 'Yes — Auto Routing Cost Ladders: 58.2% savings, 46.8% win rate vs flagship',
|
|
},
|
|
theirRefs: {
|
|
openrouter: { href: 'https://openrouter.ai/docs/llms.txt', date: '2026-10-04' },
|
|
litellm: { href: 'https://docs.litellm.ai/docs/proxy/auto_routing_benchmark', date: '2026-10-04' },
|
|
},
|
|
note: 'Only one of the three publishes anything about how well the routing itself performs. Ours is a fixed question set with a per-tier price attached; theirs is a savings rate. Not the same measurement, and not directly comparable.',
|
|
},
|
|
{
|
|
attribute: 'Runs on your own infrastructure',
|
|
source: 'reported',
|
|
ours: 'Yes — the whole cloud is one binary and one API, on your own Kubernetes or your laptop',
|
|
theirs: {
|
|
openrouter: 'No — a hosted service',
|
|
portkey: 'Self-hosting offered at enterprise tier',
|
|
litellm: 'Yes — open source, self-hosted free forever',
|
|
},
|
|
theirRefs: {
|
|
portkey: { href: 'https://portkey.ai/pricing', date: '2026-10-04' },
|
|
litellm: { href: 'https://www.litellm.ai/pricing', date: '2026-10-04' },
|
|
},
|
|
note: 'LiteLLM is genuinely free to self-host and we are not going to pretend otherwise. Our claim is platform-level — the whole cloud, not one gateway — which is a wider thing to run and a wider thing to trust.',
|
|
},
|
|
],
|
|
},
|
|
]
|