Files
ai/controllers/zen_client.go
zeekay 4c6933c2a0 A refused Enso tier falls to enso-free first
Live 2026-10-27: enso-ultra refused (its Anthropic key out of credit) fell
to the pool's nemotron-3-ultra:free, relayed as it answered, and forced
calls came back as text. enso-free is the engine's free lane, which judges
a free answer and reads a forced call written out as text as the call; a
refused Enso tier now asks it before the pool's own routes.
2026-10-27 01:54:20 -07:00

3231 lines
130 KiB
Go

// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
// zen_client.go — ai's thin client of the Hanzo model-family services.
//
// A model family (Zen, Enso, …) is a self-contained serving layer: it owns which
// upstream serves a SKU, the identity, the reasoning-vocabulary fold, the 1M context
// ladder, vision, and the enso-ultra fan-out. ai authenticates, meters, and forwards;
// it discovers each family from GET <family>/v1/models and pipes requests verbatim.
// A family is a VALUE — a name, a base URL, a service key, an ownership prefix — so
// zen and enso are two instances of the SAME machinery (modelFamily), not two copies.
// ai holds only configuration and the public brand, never an upstream mapping. See
// hip-00NN for the catalog, pricing, and discovery contract.
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"slices"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/hanzoai/ai/conf"
iam "github.com/hanzoai/ai/internal/iam"
"github.com/hanzoai/ai/log"
"github.com/hanzoai/ai/object"
"github.com/hanzoai/ai/plugin"
"github.com/hanzoai/ai/upstream"
"github.com/hanzoai/decimal"
)
// ── the families (values, not places) ────────────────────────────────────────
//
// Each family's address is configuration (a base URL + service key, set in the
// deployment), so this repository carries no routing detail and reads as open
// source. A family is added by adding an instance here — nothing else in ai knows a
// family name.
// flagshipWindow is the served context window (1M tokens) ai guarantees for the
// flagship Zen/Enso SKUs, independent of what a given discovery snapshot advertises.
const flagshipWindow = 1_000_000
// modelFamily is one serving family as a value: its public name/brand + cold-start
// ownership prefix, the config keys that hold its address, the provider record ai
// forwards through, and a discovered-catalog snapshot refreshed from the family on a
// TTL. All of zen's original machinery is a method on this value; zen and enso are
// two instances.
type modelFamily struct {
name string // "zen" | "enso" | "openrouter" — the brand label
// prices is the family's own price list: each model it names lists and bills at
// its price there, whatever the family's service quotes, and is sent only while
// no arm of its route costs more than that (rates.go). Enso's is ensoPrices.
prices map[string]tag
// ladder is the family's route as its admin catalog was last read, which the
// guard holds each model's price to (rates.go).
ladder ladder
typ string // the object.Provider Type this family serves ("Zen" | "Enso" | "OpenRouter")
prefix string // public brand prefix for cold-start ownership (before first discovery)
owner string // public /v1/models owned_by: "zenlm" (Zen LM) | "hanzo" (Hanzo)
urlKey string // config key for the base URL ("ZEN_URL" | "ENSO_URL")
keyKey string // config key for the service key ("ZEN_API_KEY" | "ENSO_API_KEY")
keyNames []string // the vendor accounts a request tries in order (family_keys.go); nil = the provider's one key
keyList string // one more name, holding further accounts' keys, one a line, tried after keyNames
// credits reads what one vendor account has left, in USD, from the body of GET
// base + creditsPath (family_keys.go). nil for a family whose vendor states none.
credits func([]byte) (float64, error)
creditsPath string
providerFn func() *object.Provider // the virtual provider ai forwards through
// windows is the served context window ai guarantees for a DISCOVERED SKU,
// keyed by SKU id — a flagship reports its real 1M window even when a given
// discovery snapshot advertises less. It cannot add a SKU: a model appears in
// /v1/models iff the family serves it. A map (not a list ai could iterate into
// the listing) is what makes an unservable entry unrepresentable — the previous
// list synthesized a listing for any SKU the family "would" serve, which is how
// enso-pro was advertised, unpriced, for a family that answers 404 for it.
windows map[string]int
// decode turns a family's raw GET /v1/models body into discovered SKUs. nil means
// the Hanzo family wire shape (zenWireModel), which already carries every field ai
// needs. A family that publishes a DIFFERENT catalog dialect — OpenRouter states
// price per TOKEN and never states a tier or a funding class — supplies its own
// translation here, so the dialect is the ONLY thing that varies and discovery,
// TTL, listing, routing, and gating stay one implementation.
decode func([]byte) ([]zenModel, error)
// spare names the routes this family still serves once its account with the
// vendor is spent — the SKUs the vendor charges nothing for. It reads the same
// /v1/models body decode reads, in the family's own dialect, because what "free"
// looks like on the wire is exactly as dialect-specific as what a price is.
//
// It is a SUBSET of the catalog, not a second one: the routes the vendor charges
// nothing for AND answers in text with. Price is what qualifies a route as a
// spare, so a spare bills nothing whether a caller named it or a refusal for
// money landed on it — a downgrade can never quietly become something we charge
// for, and a route we charge for can never be handed out as a remedy.
//
// nil means the family has none, and a refusal for money from it is final.
spare func([]byte) []string
// freeName is the one id this family publishes for the free pool: a route a
// caller can name, and the route a refusal from this family falls to. It is
// branded per family because that is where a caller looks for it, and it is
// served from the SAME pool for all of them, because a route that costs nothing
// is not made different by whose name is on it.
freeName string
// terms states, in the family's own dialect, what the vendor may keep of an
// exchange. It is called with the outgoing body and whether the route being
// asked for is one the vendor charges nothing for, because that is the trade a
// free route IS: the vendor serves it for nothing and keeps what it carried.
// A priced route pays instead, and keeps nothing. m is the SKU as listed, whose
// retail bounds what the vendor may route a priced request to.
//
// nil for a family whose dialect states no such thing, and the body then goes
// out untouched — vendor-specific fields are never sent to a vendor that did
// not define them.
terms func(body []byte, m zenModel, free bool, session string) []byte
// makers are the APIs of the vendors that MAKE some of this family's SKUs, each
// asked for its own (its prefix) before the family's own address (native.go): a
// model a reseller lists is its maker's to serve first. Empty for a family with
// none.
makers []*native
// aliases maps an id a caller may send onto the discovered SKU that serves it,
// lowercase on both sides, for a vendor whose catalog names none (OpenRouter's,
// openrouter_alias.go). A Hanzo family's catalog states its own (named). lookup
// reads through both, so everything that asks the family about a model — whether
// it serves it, its price, its floors, its terms — answers for the alias exactly
// as for the SKU, and sku is what goes upstream.
aliases map[string]string
// lists are the paths of the family's model lists, read as one catalog
// (GET base + each): "/v1/models" when it names none. OpenRouter lists its
// video models apart (/v1/videos/models).
lists []string
// load is where the family's catalog body comes from when it is not the family's
// own GET /v1/models: a vendor whose list is synced into the store (listing.go)
// reads it from there. nil asks the family's address.
load func(context.Context, *object.Provider) ([]byte, error)
// started is whether this process has read the family's catalog once.
started atomic.Bool
// discovered catalog — a read-mostly snapshot; discovery failure keeps the last
// good snapshot so a transient blip never empties ai's model list.
mu sync.RWMutex
byID map[string]zenModel
ids []string
spares []string
fetchedAt time.Time
loaded bool
// named is the aliases the family's own catalog states (catalogHead), lowercase.
named map[string]string
// paid is the catalog's `paid` line: whether it opens the paid lane (open).
paid bool
// loading serializes the first discovery (warm), and tried is when it last ran
// without loading, so a family that cannot be reached is asked again after
// warmRetry rather than on every read.
loading sync.Mutex
tried time.Time
}
var (
zenFam = &modelFamily{
name: "zen", typ: "Zen", prefix: "zen", owner: "zenlm", urlKey: "ZEN_URL", keyKey: "ZEN_API_KEY",
freeName: "zen-free",
providerFn: object.ZenProvider,
windows: map[string]int{
"zen5": flagshipWindow,
"zen5-coder": flagshipWindow,
"zen5-pro": flagshipWindow,
},
}
ensoFam = &modelFamily{
name: "enso", typ: "Enso", prefix: "enso", owner: "hanzo", urlKey: "ENSO_URL", keyKey: "ENSO_API_KEY",
prices: ensoPrices,
freeName: "enso-free",
providerFn: object.EnsoProvider,
// The flagship window. Every other SKU, enso-pro among them, is listed and
// served from discovery.
windows: map[string]int{
"enso-ultra": flagshipWindow,
},
}
// engineFam is OUR OWN compute: the models this deployment serves itself, at no
// marginal cost to anybody's account. It publishes no catalog of its own here —
// it exists so the free pool has a route that cannot be taken away by a vendor's
// balance — so it is deliberately absent from modelFamilies below and is reached
// only through freeRoutes.
engineFam = &modelFamily{
name: "engine", typ: "Engine", prefix: "engine/", owner: "hanzo",
urlKey: "ENGINE_URL", keyKey: "ENGINE_API_KEY",
}
// modelFamilies is the discovery/route/merge iteration order — the ONE list every
// generic family helper walks. openrouter is last: the first-party families claim
// their own SKUs first, and the resale catalog answers for what is left.
modelFamilies = []*modelFamily{zenFam, ensoFam, openrouterFam}
)
// ours reports whether f is a Hanzo service ai fronts (Zen, Enso), which may be
// told who a call is for. A vendor's API is not one.
func (f *modelFamily) ours() bool { return f == zenFam || f == ensoFam }
// window returns the context window ai guarantees when listing and routing a SKU,
// or 0 when the family guarantees none. It applies only to a configured family, so
// a bare or unconfigured family reports no guaranteed window.
func (f *modelFamily) window(model string) int {
if !f.enabled() {
return 0
}
return f.windows[strings.ToLower(strings.TrimSpace(model))]
}
// provider resolves the family to the record ai forwards through — its OWN
// constructor where it has one, so the address that gates listing is the address
// that serves. That constructor reads the admin row first and falls back to
// deployment config, which is what makes admin.hanzo.ai the one control:
// disabling a family there makes provider nil, so enabled() goes false and its
// models leave /v1/models on the next resolution.
//
// Reading conf directly here (which is what this did) meant the console toggle
// moved a row that listing never consulted: a family could be disabled in the
// admin UI and still be listed and served.
//
// A family reached only by configuration — our own compute — has the same record
// built for it here, so every caller holds one shape and the credential stays in
// it: nothing in this file hands a key to anybody, it hands the record to
// authorize.
func (f *modelFamily) provider() *object.Provider {
if f.providerFn != nil {
return f.providerFn()
}
base := strings.TrimRight(strings.TrimSpace(conf.GetConfigString(f.urlKey)), "/")
if base == "" {
return nil
}
return &object.Provider{
Owner: "admin",
Name: f.name,
Category: "Model",
Type: f.typ,
State: "Active",
ProviderUrl: base,
ClientSecret: strings.TrimSpace(conf.GetConfigString(f.keyKey)),
}
}
func (f *modelFamily) baseURL() string {
if p := f.provider(); p != nil {
return strings.TrimRight(strings.TrimSpace(p.ProviderUrl), "/")
}
return ""
}
// enabled reports whether the family service is configured AND admin-enabled. When
// it is not, ai serves no model of that family (serves is false) and it is simply
// absent from /v1/models.
func (f *modelFamily) enabled() bool { return f.baseURL() != "" }
// familyForProviderType maps a virtual provider Type onto its family, resolved from the
// SAME modelFamilies list that discovery, listing and routing all read.
//
// It used to be a hand-written switch, which is a second list that must be kept in step
// with the first — and it was not. OpenRouter was added to modelFamilies, so its 338
// prepaid models were discovered, listed and routable, but the switch never named it. Every
// dispatch site guards on `if fam := familyForProviderType(...); fam != nil`, so a nil sent
// those requests down the generic relay — around pipeToFamily, which is where the funding
// gate lives. Listed, served, and ungated, on real cash.
//
// Deriving the lookup from the one list makes that shape unrepresentable: a family that
// exists is a family this resolves, and adding one cannot silently skip the gate.
func familyForProviderType(t string) *modelFamily {
for _, f := range modelFamilies {
if f.typ == t {
return f
}
}
return nil
}
// familyNamed is the family whose provider row is named name, or nil.
func familyNamed(name string) *modelFamily {
for _, f := range modelFamilies {
if f.name == name {
return f
}
}
return nil
}
// freeID is the platform's OWN name for the pool: one id every surface can send and
// always get a free answer from, wearing no brand at all. enso-free and zen-free are
// the same pool behind a brand; this is the same pool in front of all of them, so a
// client that does not care which family answered has one id to remember.
const freeID = "free"
// freeDoor is the family and id the platform's free pool is served under. Wherever
// Enso's own catalog publishes its free id, the pool answers as Enso: Enso's service
// serves it and gives it Enso's name and maker, so no vendor's model introduces itself
// to a caller who asked for the free tier. Where Enso does not carry it, the family
// that holds the free routes chooses among them.
func freeDoor() (*modelFamily, string) {
if ensoFam.enabled() {
if _, own := ensoFam.lookup(ensoFam.freeName); own {
return ensoFam, ensoFam.freeName
}
}
return freeFamily(), freeID
}
// frontDoor reports that this id is a name this family publishes for the pool — its
// own branded one, and, for the family that HOLDS the pool, the platform's unbranded
// one. Such an id is not a model any vendor has: it is a name for a choice.
//
// A family whose own catalog publishes its branded free id serves it like any other
// SKU, with that catalog's rungs, fallbacks and vision, so the id has one free path.
func (f *modelFamily) frontDoor(model string) bool {
m := strings.ToLower(strings.TrimSpace(model))
if f.freeName != "" && m == strings.ToLower(f.freeName) {
_, own := f.lookup(m)
return !own
}
return f == freeFamily() && m == freeID
}
// freeNames are every id this family publishes for the pool, with the owner each is
// branded to. The platform's own id is owned by us because it is our product, not the
// vendor's, whichever vendor happens to answer it.
func (f *modelFamily) freeNames() []struct{ id, owner string } {
var out []struct{ id, owner string }
if f.freeName != "" && f.frontDoor(f.freeName) {
out = append(out, struct{ id, owner string }{f.freeName, f.owner})
}
if f == freeFamily() {
out = append(out, struct{ id, owner string }{freeID, "hanzo"})
}
return out
}
// freeFamily holds the platform's free routes: the family whose vendor publishes
// routes it charges nothing for. Every family falls back to it, itself included,
// because a route that costs nothing is not made different by whose refusal sent a
// request to it.
func freeFamily() *modelFamily { return openrouterFam }
// spare is a route that can answer when the one asked for cannot: the family that
// carries it and the id it is served under. A pair, because the vendor that ran out
// of money is rarely the one still holding a free route.
type spare struct {
fam *modelFamily
id string
}
// freeRoutes are the routes that cost nothing and answer a chat turn, best first.
// ONE pool for the whole platform: one list to curate, and every family falls back
// to the same places.
//
// OUR OWN COMPUTE IS LAST, and last is the point of it. A route we serve ourselves
// cannot run out of a vendor's money and cannot be rate limited by one, so it is the
// answer that remains when every borrowed one is gone — which is exactly a last
// resort and not a first choice. Measured: the engine serves one small model, and
// asked to reply with a single word it answers with several sentences about editing
// a sentence. Putting it first would spend that on every free answer to insure
// against an outage that has not happened; putting it last insures against it for
// nothing. Its place moves the day it serves a model that answers as well as the
// borrowed ones, and only the order below changes.
func freeRoutes() []spare {
var out []spare
for _, id := range freeFamily().spareRoutes() {
out = append(out, spare{freeFamily(), id})
}
if engineFam.enabled() {
out = append(out, spare{engineFam, engineModel})
}
return out
}
// fallback names where a request its vendor refused may be offered next, and an
// empty list means the refusal stands as itself.
//
// It is the WHOLE of the decision and none of the walking. Three facts settle it
// and not one of them needs a socket, a clock or a request: who the refusal is
// about, whether the vendor can serve at all, and the terms the route was bought
// under. Kept apart from the walk so the policy can be read — and asserted —
// without standing a vendor up to refuse.
func fallback(fam *modelFamily, sku string, err error, body []byte) []spare {
// Our own spend gate, relayed by a service that fronts for us, wears the same
// 402 and means the opposite thing: the CUSTOMER owes money. That has to reach
// them, and answering it with a free model would tell somebody their payment
// problem had fixed itself.
if billingNotice(body) {
return nil
}
// A vendor that cannot serve this request moves it: its account with us is
// spent, it failed, it could not be reached, or every account it holds is rate
// limited. A malformed body (400), a model it has not got (404) and refused
// content (422) stay where they are: every vendor would answer them the same.
if err == nil || !unserved(err) {
return nil
}
// A third-party model is served as itself or not at all: its vendor's refusal
// stands. Only a Hanzo SKU, or the pool's own name, is ours to route elsewhere.
if FamilyOf(sku) == "" && !fam.frontDoor(sku) {
return nil
}
return standIn(fam, sku)
}
// standIn names the routes that answer for sku when its own vendor does not.
//
// A priced route is answered by the model closest to the one asked for: the Enso
// SKU nearest it, whose service walks every account it holds, and then our own
// compute. The answer names the model that wrote it (pipeToFamily), so the caller
// always knows which one did. Any other route falls to the free pool.
func standIn(fam *modelFamily, sku string) []spare {
if word, _ := fam.collection(sku); word == collectionDeny {
// A route bought under `deny` never reaches a borrowed free route, which
// keeps what it carries: its stand-ins are ours.
out := standIns(fam, sku)
if engineFam.enabled() {
out = append(out, spare{engineFam, engineModel})
}
return out
}
// An Enso tier falls to Enso's own free id first, where Enso serves it: its service
// judges a free answer and walks on (a call that breaks its schema, an empty answer,
// a forced call not made), and reads a forced call a model wrote out as text as the
// call, which the pool's own routes, relayed as they answer, do not.
if fam == ensoFam {
if f, id := freeDoor(); f == ensoFam && !strings.EqualFold(id, sku) {
return append([]spare{{f, id}}, freeRoutes()...)
}
}
return freeRoutes()
}
// FreeOnly reports whether the paid lane is off for everyone: it is on only where the
// zen catalog says `paid`, as zen's own discovery states it (GET ZEN_URL/v1/models,
// zenFam.open). One line decides for ai's routes and for the family's alike. Spending
// is an opt-in, so a catalog that is unread, unreachable from the start, or silent on
// it keeps the lane off. A request asks FreeOnlyFor, which also asks whether its
// caller was put on the paid lane (lane.go).
var FreeOnly = func() bool { return !zenFam.open() }
// unserved reports that a vendor could not serve a request at all: its account is
// spent, it answered with its own failure, it was never reached, or it answered
// 429 after every account it holds was asked.
func unserved(err error) bool {
st := upstreamHTTPStatus(err)
return st == 0 || st == http.StatusTooManyRequests || down(err, strings.ToLower(err.Error()))
}
// frontierOut is the output price, in USD per million tokens, from which a
// model is answered by enso-pro rather than enso-flash when it stands in.
const frontierOut = 10.0
// standIns are the Enso SKUs that answer a priced route its vendor cannot serve,
// closest first: enso-pro and then enso-flash for a frontier model, enso-flash
// for any other.
func standIns(fam *modelFamily, sku string) []spare {
if fam == ensoFam || !ensoFam.enabled() {
return nil
}
ids := []string{"enso-flash"}
if p, ok := fam.modelPrice(sku); ok && p.OutputPerMillion >= frontierOut {
ids = []string{"enso-pro", "enso-flash"}
}
out := make([]spare, 0, len(ids))
for _, id := range ids {
out = append(out, spare{ensoFam, id})
}
return out
}
// freeLane reports that sku is a route of the free lane: one its family charges
// nothing for, a name for the pool, or a route out of the pool.
func freeLane(fam *modelFamily, sku string) bool {
return fam.frontDoor(sku) || fam.free(sku) || inPool(sku)
}
// inPool reports that a route is one the pool serves, whichever family carries it.
// It is what prices a pool answer at nothing: every route in the pool is there
// BECAUSE it costs nothing, so billing one would charge for the thing that makes it
// a fallback. Read from the pool itself rather than from any one family's list,
// since the pool now spans families — our own compute among them.
func inPool(id string) bool {
// The SNAPSHOT, never a refresh. This is the billing path: discovery is a
// network round trip with its own timeout, and settling a debit must not wait
// on a vendor to answer — a slow vendor would otherwise make every debit slow,
// and an unreachable one would stall the ledger behind a request that already
// succeeded.
if freeFamily().isSpare(id) {
return true
}
return engineFam.enabled() && strings.EqualFold(strings.TrimSpace(id), engineModel)
}
// servingFamily returns the family that will actually carry this sku — the free
// family for a route out of the pool, whichever family claims it otherwise.
func servingFamily(sku string) *modelFamily {
if freeFamily().isSpare(sku) {
return freeFamily()
}
f, _ := familyServing(sku)
return f
}
// familyServing returns the family that owns a model, if any.
func familyServing(model string) (*modelFamily, bool) {
for _, f := range modelFamilies {
if f.serves(model) {
return f, true
}
}
return nil, false
}
// familyLookup returns the discovered model for an id/alias from whichever family
// holds it.
func familyLookup(model string) (zenModel, bool) {
for _, f := range modelFamilies {
if m, ok := f.lookup(model); ok {
return m, true
}
}
return zenModel{}, false
}
// familyLookupFresh is familyLookup that first refreshes each family's catalog on
// its TTL (a no-op when warm) before reading it. lookup() alone reads the cache
// with no refresh, so on a replica whose family cache is cold or stale a real SKU
// reads as unknown — which would make a gated SKU (enso) read as non-gated on a cold
// replica and skip the access gate. Access/gating decisions must see the same catalog
// the /v1/models merge does; they go through here (FamilyModelGated).
func familyLookupFresh(model string) (zenModel, bool) {
for _, f := range modelFamilies {
f.fresh()
if m, ok := f.lookup(model); ok {
return m, true
}
}
return zenModel{}, false
}
// familyPassthroughRoute routes any family SKU to its family service.
func familyPassthroughRoute(model string) *modelRoute {
for _, f := range modelFamilies {
if r := f.passthroughRoute(model); r != nil {
return r
}
}
return nil
}
// familyModelPrice returns the discovered headline price for a family model.
func familyModelPrice(model string) (modelPrice, bool) {
for _, f := range modelFamilies {
f.warm()
if p, ok := f.modelPrice(model); ok {
return p, true
}
}
return modelPrice{}, false
}
// mergeFamilyModels overlays every configured family's discovered lineup onto the
// base /v1/models list (each SKU wins over a same-named base entry; new SKUs append).
func mergeFamilyModels(base []modelInfo) []modelInfo {
for _, f := range modelFamilies {
base = f.mergeModels(base)
}
return base
}
// ── discovered catalog (values, not places) ──────────────────────────────────
// zenTier is one context tier's retail price ($/MTok) as EXACT decimals — the same
// values the family serves, parsed losslessly (never through float).
type zenTier struct {
MaxCtx int
In decimal.Decimal
Out decimal.Decimal
// CacheRead is what one input token served from the upstream's prompt cache
// bills at. Zero means the family states none, and such a token bills at In.
CacheRead decimal.Decimal
// CacheWrite is what one input token written to the upstream's prompt cache
// bills at, which a vendor may price above In. Zero means the family states none,
// and such a token bills at In.
CacheWrite decimal.Decimal
}
// cacheRate is what one cached input token bills at: the stated cache price, else
// the full input rate. An unpriced cache is never a free cache.
func (t zenTier) cacheRate() decimal.Decimal {
if t.CacheRead.Sign() > 0 {
return t.CacheRead
}
return t.In
}
// writeRate is what one input token written to the cache bills at: the stated write
// price, else the full input rate.
func (t zenTier) writeRate() decimal.Decimal {
if t.CacheWrite.Sign() > 0 {
return t.CacheWrite
}
return t.In
}
// tokens is one answer's usage as it is billed: prompt tokens read fresh, prompt
// tokens served from the upstream's cache, prompt tokens written to it, and
// completion tokens — of which reasoning is the part the model spent thinking. The
// three prompt parts are disjoint and sum to the prompt.
type tokens struct {
fresh, cached, written, completion, reasoning int
// reported says the upstream stated what the answer's output cost: a usage
// object carried its completion. Without it the counts are partial or estimated.
reported bool
}
// prompt is every prompt token: fresh, read from the cache or written to it.
func (t tokens) prompt() int { return t.fresh + t.cached + t.written }
// zenModel is a discovered SKU: what ai needs to list, route, and bill it — the SKU's
// public contract (id, context, price ladder, vision). Not its upstream, which the
// family does not disclose (hip-00NN).
type zenModel struct {
ID string
// Created is when the model was released (Unix seconds), where the family's
// catalog records it; 0 = not recorded, and the listing's own time is used.
Created int64
OwnedBy string
// Name and Description are what the family's catalog calls the SKU and says it
// is; empty where it says nothing.
Name string
Description string
MaxCtx int
// MaxOut is the most one answer may hold, where the catalog states it; 0 is
// unstated.
MaxOut int
// Expires is the day the vendor stops serving the SKU (2006-01-02), where its
// catalog states one.
Expires string
Vision bool
// Tools and Reasoning are whether the SKU takes tool calls and a reasoning
// request, as the family's catalog states; false where it states nothing.
Tools bool
Reasoning bool
// Inputs are the kinds of input this SKU takes, as the family advertises them
// ("text", "image", "audio", "file", "video"). Empty reads as unknown.
Inputs []string
// Outputs are the kinds of answer this SKU produces, as the family advertises
// them ("text", "audio", "image", "embeddings", "rerank"). Empty when the family
// advertises none, which reads as unknown and never as text.
Outputs []string
Access string // "" = generally available; "waitlist" = access-gated (limited preview) — ai enforces the grant
MinTier string // "" | "free" | "trial" | "paid" — min subscription tier the family advertises for this SKU; ai enforces it (Seams A/B). "" ⇒ free (all tiers). Orthogonal to Access.
// Funding is how the SKU's usage is PAID FOR upstream: "prepaid" means every path it
// can take spends a real-cash balance; "" means credits. It is a different KIND of
// floor from MinTier and is enforced differently — see familyFundingAllowed.
Funding string
// Plan is whether the family opens paid plan rungs for this SKU to a host that
// reads the cost its committed answer states (its listing's "plan").
Plan bool
Base zenTier // headline RETAIL price (the in-window tier)
Tiers []zenTier // full ladder, ascending by MaxCtx — the billing contract
// CostIn/CostOut are the upstream COGS ($/MTok) behind Base, when the family
// discloses what the SKU costs us. Zero means undisclosed, which modelPrice
// already reads as cost == price (zero margin) — so a family that states only a
// price behaves exactly as before. A resale family (OpenRouter) states both, and
// the retail it publishes is this cost times a margin, never below it.
CostIn decimal.Decimal
CostOut decimal.Decimal
// Margin is the multiple a SKU we resell (OpenRouter's) bills over its vendor's
// price, 1 + the vendor's fee: its rates are the vendor's times Margin, and a call
// never bills less than the cost the vendor states for it times Margin (bill).
// Zero for a family that prices its own SKUs.
Margin decimal.Decimal
// Variable is a SKU the vendor prices per call by whatever served it (OpenRouter's
// routers, which it lists at a price of -1). No rate describes such a call, so it
// bills at the cost its answer states times Margin, and Base and Tiers hold the
// ceiling a hold reserves: the dearest SKU it could be routed to.
Variable bool
// Card is the vendor's whole price list as it bills to a caller, for a SKU we
// resell: what /v1/models publishes as the SKU's pricing, exactly.
Card *rateCard
// Video is what a video SKU takes and what a clip of it costs (video_spec.go);
// nil for every other SKU.
Video *videoSpec
// Voice is what a speech SKU takes and costs (voice_vendor.go); nil for every
// other SKU.
Voice *voiceSpec
}
// rateCard is a resold SKU's price list as billed: every rate its vendor states, in
// the vendor's names and units, times the markup, and its conditional rates the same.
type rateCard struct {
Rates map[string]string
Overrides []map[string]any
}
// variable reports that this SKU bills each call at the cost its answer states.
func (m zenModel) variable() bool { return m.Variable }
// bill is what one call to this SKU charges, in nano-dollars: its tokens at the tier
// their prompt reaches. A SKU we resell never bills less than the cost its answer
// states times its margin: a cache write, a web search or a dearer provider behind
// the vendor costs what it costs, and billing below cost is the one error resale
// cannot absorb. A variable SKU bills that stated cost alone, or its tokens at the
// ceiling its hold reserved when the answer states none. A stated cost of zero or
// less is no statement, never a credit.
func (m zenModel) bill(cost *int64, t tokens) int64 {
retail := m.retailNano(t)
if m.Margin.Sign() <= 0 || cost == nil || *cost <= 0 {
return retail
}
stated := nanoUp(decimal.New(*cost, 9).Mul(m.Margin))
if m.Variable {
return stated
}
return max(retail, stated)
}
// retailNano is retailUSD in nano-dollars: the exact charge the hold settles, the
// ledger debits and the usage row and its span read. Rounded up, so a priced answer
// never bills nothing; never floored to a cent, so a call worth a thousandth of one
// bills a thousandth of one.
func (m zenModel) retailNano(t tokens) int64 {
return nanoUp(m.retailUSD(t))
}
// unitNano is what units of a media SKU (images, calls) bill at its per-unit price,
// in nano-dollars, rounded up.
func (m zenModel) unitNano(units int) int64 {
return nanoUp(m.Base.In.Mul(decimal.New(int64(units), 0)))
}
// nanoUp is usd in nano-dollars, rounded up; nothing for a negative amount.
func nanoUp(usd decimal.Decimal) int64 {
n, _ := usdNanos(usd.String())
return n
}
// releasedOr is the model's release time, or now when the catalog records none.
func (m zenModel) releasedOr(now int64) int64 {
if m.Created > 0 {
return m.Created
}
return now
}
// gated reports whether the family SKU is access-controlled (a limited-preview SKU
// that is LISTED but callable only with a granted ModelAccess row). ai learns this
// from discovery (the family's advertised access), never a hardcode.
func (m zenModel) gated() bool { return m.Access == "waitlist" }
// priced reports whether the family charges for this SKU on either side. It is the
// one reading of "costs money" the listing takes, so premium and free are answers to
// the same question rather than two opinions of it.
func (m zenModel) priced() bool {
return !m.Base.In.IsZero() || !m.Base.Out.IsZero() || m.Video != nil || (m.Voice != nil && m.Variable)
}
// minTier is the SKU's advertised minimum subscription tier on the enso ladder,
// normalized: "" ⇒ "free" (every tier may use it). One of "free" | "trial" | "paid".
// ai learns this from discovery (the family's advertised min_tier), never a hardcode;
// the tier gate (familyTierAllowed, Seams A/B) enforces it.
func (m zenModel) minTier() string {
if t := strings.ToLower(strings.TrimSpace(m.MinTier)); t != "" {
return t
}
return "free"
}
// tierFor picks the tier that serves promptTokens: the smallest whose context covers
// the prompt, else the largest. This is the family's Tier rule (hip-00NN) — cost keys
// off the SERVED tier, so a 1M-overflow bills at the pricier tier and never underbills.
func (m zenModel) tierFor(promptTokens int) zenTier {
for _, t := range m.Tiers {
if promptTokens <= t.MaxCtx {
return t
}
}
if len(m.Tiers) > 0 {
return m.Tiers[len(m.Tiers)-1]
}
return m.Base
}
var zenMillion = decimal.New(1_000_000, 0)
// retailUSD is what the tokens cost at the tier their prompt reaches, in exact dollars
// to 18 places — identical to the family's own arithmetic.
func (m zenModel) retailUSD(u tokens) decimal.Decimal {
t := m.tierFor(u.prompt())
in := t.In.Mul(decimal.New(int64(u.fresh), 0))
out := t.Out.Mul(decimal.New(int64(u.completion), 0))
read := t.cacheRate().Mul(decimal.New(int64(u.cached), 0))
write := t.writeRate().Mul(decimal.New(int64(u.written), 0))
return in.Add(read).Add(write).Add(out).Quo(zenMillion, 18)
}
// centsUp is nano-dollars as the whole cents a hold reserves (the ledger reserves in
// cents): rounded up, so the hold covers what settles.
func centsUp(nano int64) int64 {
if nano <= 0 {
return 0
}
return (nano + 10_000_000 - 1) / 10_000_000
}
// price projects the headline tier into ai's legacy per-model price struct (float,
// used only for the balance-reservation estimate and the /v1/models display). The
// exact debit path uses retailNano, never this projection.
//
// A discovered SKU always HAS a price, and zero is one of them. "No price" is a
// different fact — nobody stated one — and only a model discovery never saw can be
// in that state. Reading zero as absent is what let a route the vendor charges
// nothing for be listed without a price and gated as though it cost money.
func (m zenModel) price() (modelPrice, bool) {
in, _ := strconv.ParseFloat(m.Base.In.String(), 64)
out, _ := strconv.ParseFloat(m.Base.Out.String(), 64)
cache, _ := strconv.ParseFloat(m.Base.cacheRate().String(), 64)
costIn, _ := strconv.ParseFloat(m.CostIn.String(), 64)
costOut, _ := strconv.ParseFloat(m.CostOut.String(), 64)
return modelPrice{
InputPerMillion: in, OutputPerMillion: out, CacheReadPerMillion: cache,
CostInPerMillion: costIn, CostOutPerMillion: costOut,
Variable: m.variable(), Card: m.Card,
}, true
}
// zenWireModel is the /v1/models item shape a family serves. Prices are JSON strings
// decoded straight into decimal (decimal.UnmarshalJSON), preserving the exact value.
// There is deliberately no upstream field to read.
type zenWireModel struct {
ID string `json:"id"`
OwnedBy string `json:"owned_by"`
Description string `json:"description"`
Mode string `json:"mode"` // the kind of SKU: chat | embedding | rerank | image | audio | video
Access string `json:"access"` // "" | "waitlist" — access gating advertised by the family
MinTier string `json:"min_tier"` // "" | "free" | "trial" | "paid" — min subscription tier advertised for this SKU (Seams A/B)
Funding string `json:"funding"` // "prepaid" = every path this SKU can take spends a real-cash balance
Plan bool `json:"plan"` // the family opens plan rungs for this SKU to a host that reads its cost trailer
ContextWindow int `json:"context_window"`
Pricing struct {
Input decimal.Decimal `json:"input"`
Output decimal.Decimal `json:"output"`
CacheRead decimal.Decimal `json:"cache_read"`
CacheWrite decimal.Decimal `json:"cache_write"`
} `json:"pricing"`
PricingTiers []struct {
MaxContext int `json:"max_context"`
Input decimal.Decimal `json:"input"`
Output decimal.Decimal `json:"output"`
CacheRead decimal.Decimal `json:"cache_read"`
CacheWrite decimal.Decimal `json:"cache_write"`
} `json:"pricing_tiers"`
Capabilities struct {
Vision bool `json:"vision"`
} `json:"capabilities"`
}
func (w zenWireModel) model() zenModel {
zm := zenModel{
ID: w.ID, OwnedBy: w.OwnedBy, Description: strings.TrimSpace(w.Description), MaxCtx: w.ContextWindow, Vision: w.Capabilities.Vision,
Outputs: modeOutputs[strings.ToLower(strings.TrimSpace(w.Mode))],
Access: w.Access, MinTier: w.MinTier, Funding: w.Funding, Plan: w.Plan,
Base: zenTier{MaxCtx: w.ContextWindow, In: w.Pricing.Input, Out: w.Pricing.Output, CacheRead: w.Pricing.CacheRead, CacheWrite: w.Pricing.CacheWrite},
}
for _, t := range w.PricingTiers {
zm.Tiers = append(zm.Tiers, zenTier{MaxCtx: t.MaxContext, In: t.Input, Out: t.Output, CacheRead: t.CacheRead, CacheWrite: t.CacheWrite})
}
if len(zm.Tiers) == 0 {
zm.Tiers = []zenTier{zm.Base}
}
return zm
}
// modeOutputs is what a family SKU of each mode answers with, in OpenRouter's words:
// a chat SKU writes text, an embedding SKU returns embeddings, and so on. A mode not
// named here, or none, states no outputs.
var modeOutputs = map[string][]string{
"chat": {"text"},
"embedding": {"embeddings"},
"rerank": {"rerank"},
"image": {"image"},
"audio": {"audio"},
"video": {"video"},
}
// decodeCatalog translates a family's raw /v1/models body into discovered SKUs using
// the family's own dialect, defaulting to the Hanzo family wire shape.
func (f *modelFamily) decodeCatalog(body []byte) ([]zenModel, error) {
if f.decode != nil {
return f.decode(body)
}
var out struct {
Data []zenWireModel `json:"data"`
}
if err := json.Unmarshal(body, &out); err != nil {
return nil, err
}
models := make([]zenModel, 0, len(out.Data))
for _, w := range out.Data {
models = append(models, w.model())
}
return models, nil
}
// catalogHead is what a Hanzo family's /v1/models states beside its SKUs: the other
// ids it answers to, each naming a SKU it lists ({"claude-zen": "zen6"}), and its
// catalog's `paid` line. A vendor dialect (decode) states neither.
type catalogHead struct {
Aliases map[string]string `json:"aliases"`
Paid bool `json:"paid"`
}
// head reads the catalog-level fields of a family's /v1/models body: its aliases,
// lowercase, less any that spells a SKU it lists (the SKU answers for itself), and
// its paid line. Nothing for a vendor dialect.
func (f *modelFamily) head(body []byte, byID map[string]zenModel) (map[string]string, bool) {
var h catalogHead
if f.decode != nil || json.Unmarshal(body, &h) != nil {
return nil, false
}
named := make(map[string]string, len(h.Aliases))
for alias, sku := range h.Aliases {
alias, sku = strings.ToLower(strings.TrimSpace(alias)), strings.ToLower(strings.TrimSpace(sku))
if _, listed := byID[alias]; alias == "" || sku == "" || listed {
continue
}
named[alias] = sku
}
return named, h.Paid
}
const zenCatalogTTL = 5 * time.Minute
// engineModel is the id the engine answers to for whatever it currently has loaded.
// It is an alias on the engine's side, so the pool names the ROLE rather than a
// checkpoint that changes when the deployment loads a different one.
const engineModel = "default"
// spareTries bounds how many of a vendor's free routes one refused request may be
// offered to. Three, the same number of attempts the failover loop allows across
// vendors (maxProviders) and for the same reason: worst-case latency has to be a
// property of a constant here rather than of a catalogue an operator does not own.
const spareTries = 3
var zenDiscoveryClient = &http.Client{Timeout: listWait, Transport: reaching(&http.Transport{Proxy: http.ProxyFromEnvironment})}
// reaching is t with the plugin scheme registered on it: a family or decision
// service whose address is plugin://<name> is answered in this process by the
// plugin registered under that name (package plugin), every other address by t.
// The address decides; the code that relays is the same either way.
func reaching(t *http.Transport) *http.Transport {
t.RegisterProtocol(plugin.Scheme, plugin.Transport{Head: t.ResponseHeaderTimeout})
return t
}
// refresh fetches GET <family>/v1/models and swaps in a fresh snapshot. Best-effort:
// on any error the previous snapshot stands.
func (f *modelFamily) refresh() error {
p := f.provider()
if p == nil {
return nil
}
read := f.list
if f.load != nil {
read = f.load
}
ctx, cancel := context.WithTimeout(context.Background(), listWait)
defer cancel()
body, err := read(ctx, p)
if err != nil {
return err
}
models, err := f.decodeCatalog(body)
if err != nil {
return err
}
models = f.fix(models)
byID := make(map[string]zenModel, len(models))
ids := make([]string, 0, len(models))
for _, m := range models {
if strings.TrimSpace(m.ID) == "" {
continue
}
byID[strings.ToLower(m.ID)] = m
ids = append(ids, m.ID)
}
var spares []string
if f.spare != nil {
spares = f.spare(body)
}
named, paid := f.head(body, byID)
f.mu.Lock()
f.byID = byID
f.ids = ids
f.spares = spares
f.named = named
f.paid = paid
f.fetchedAt = time.Now()
f.loaded = true
f.mu.Unlock()
return nil
}
// listWait bounds one read of a family's catalog.
const listWait = 30 * time.Second
// list reads the family's GET /v1/models from its address.
func (f *modelFamily) list(ctx context.Context, p *object.Provider) ([]byte, error) {
paths := f.lists
if len(paths) == 0 {
paths = []string{"/v1/models"}
}
if len(paths) == 1 {
return f.listAt(ctx, p, paths[0])
}
// Several lists are one catalog: every entry of each, read whole or not at all,
// so a list that could not be read never marks its models gone. A list after the
// first that the vendor does not have (404) lists nothing.
var all []json.RawMessage
for i, path := range paths {
body, err := f.listAt(ctx, p, path)
if errors.Is(err, errNoList) && i > 0 {
continue
}
if err != nil {
return nil, err
}
var one struct {
Data []json.RawMessage `json:"data"`
}
if err := json.Unmarshal(body, &one); err != nil {
return nil, fmt.Errorf("%s %s: %w", f.name, path, err)
}
all = append(all, one.Data...)
}
return json.Marshal(map[string]any{"data": all})
}
// listAt reads one of the family's lists, GET base + path.
func (f *modelFamily) listAt(ctx context.Context, p *object.Provider, path string) ([]byte, error) {
base := strings.TrimRight(strings.TrimSpace(p.ProviderUrl), "/")
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+path, nil)
if err != nil {
return nil, err
}
upstream.Authorize(req, p)
resp, err := zenDiscoveryClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, fmt.Errorf("%s %s: %w", f.name, path, errNoList)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%s %s: status %d", f.name, path, resp.StatusCode)
}
return io.ReadAll(io.LimitReader(resp.Body, 64<<20))
}
// errNoList is a vendor that has no list at a path.
var errNoList = errors.New("no such list")
// open reports whether the family's catalog, as last discovered, opens the paid lane.
// It reads the snapshot and never waits on the family: discovery keeps it, loaded at
// start (WarmFamilies) and again on the TTL wherever the family is read, and a failed
// refresh keeps the last catalog's line. A catalog never read keeps the lane closed.
func (f *modelFamily) open() bool {
f.mu.RLock()
defer f.mu.RUnlock()
return f.paid
}
// spends reports whether serving sku can spend the platform's own cash: the family's
// catalog funds some path of it from a prepaid balance (zenModel.Funding), or does not
// describe it at all. A SKU the catalog funds from grants and our own compute on every
// path spends none, whatever it is priced at to the caller.
func (f *modelFamily) spends(sku string) bool {
m, ok := f.lookup(sku)
return !ok || strings.EqualFold(strings.TrimSpace(m.Funding), "prepaid")
}
// spareRoutes are the routes this family serves once its account is spent, best
// first. Empty for a family that declared none, and for one whose discovery has
// never succeeded — an unknown vendor gets no fallback invented for it.
func (f *modelFamily) spareRoutes() []string {
f.fresh()
f.mu.RLock()
defer f.mu.RUnlock()
return f.spares
}
// free reports that this family serves the named route for nothing. Read from the
// discovered price, which is the same number the ledger bills, so what a call costs
// and the terms it is served under are one fact.
func (f *modelFamily) free(id string) bool {
m, ok := f.lookup(id)
return ok && !m.priced()
}
// isSpare reports whether a SKU is one of the free routes rather than a catalog
// entry. It is what prices a spare at nothing: the vendor charges us nothing for
// it — that is the whole reason it can answer at all when the account is empty —
// so billing a customer retail for one would be charging for a downgrade.
func (f *modelFamily) isSpare(model string) bool {
model = strings.ToLower(strings.TrimSpace(model))
f.mu.RLock()
defer f.mu.RUnlock()
for _, s := range f.spares {
if strings.ToLower(s) == model {
return true
}
}
return false
}
// warmRetry is how long a family whose first discovery failed waits before a read
// asks it again.
const warmRetry = 10 * time.Second
// warm loads the family's catalog when it has never been read. A price asked for
// before the first discovery is then the discovered one: on a pod that has just
// started, enso-free reads as the free SKU it is, not as an unknown and so priced id.
// Concurrent readers wait for the one load; after a failed one they read the empty
// snapshot until warmRetry has passed.
func (f *modelFamily) warm() {
f.mu.RLock()
loaded := f.loaded
f.mu.RUnlock()
if loaded || f.baseURL() == "" {
return
}
f.loading.Lock()
defer f.loading.Unlock()
f.mu.RLock()
loaded = f.loaded
f.mu.RUnlock()
if loaded || time.Since(f.tried) < warmRetry {
return
}
f.tried = time.Now()
if err := f.refresh(); err != nil {
log.Warning("%s catalog discovery failed: %v", f.name, err)
}
}
// WarmFamilies reads every family's catalog once, in the background, so a pod that
// has just started prices family SKUs from discovery on its first request, and
// starts reading the balance of every account a family spends.
func WarmFamilies() {
for _, f := range modelFamilies {
go f.warm()
}
watching.Do(func() {
for _, f := range modelFamilies {
if f.credits != nil {
go f.watchCredits()
}
if f.load != nil {
go f.keepListed()
}
for _, n := range f.makers {
go n.watch()
}
}
})
}
var watching sync.Once
// fresh refreshes the snapshot if stale (or never loaded) and reports the address
// the family resolved to — empty when it is not configured. Errors are logged, not
// surfaced: a stale snapshot still serves.
//
// The address is returned rather than left for the caller to ask for again, because
// resolving it reads the family's admin row and one read answers both questions.
func (f *modelFamily) fresh() string {
url := f.baseURL()
if url == "" {
return ""
}
f.mu.RLock()
stale := !f.loaded || time.Since(f.fetchedAt) > zenCatalogTTL
f.mu.RUnlock()
if stale {
if err := f.refresh(); err != nil {
log.Warning("%s catalog refresh failed: %v", f.name, err)
}
}
return url
}
// lookup returns the discovered model for an id or one of the family's aliases, if
// any. An alias whose SKU the vendor no longer lists resolves to nothing.
func (f *modelFamily) lookup(model string) (zenModel, bool) {
f.mu.RLock()
defer f.mu.RUnlock()
m, ok := f.byID[strings.ToLower(strings.TrimSpace(f.alias(model)))]
return m, ok
}
// sku is the id the family's vendor serves a model under: the SKU an alias names,
// and the id itself for anything else.
func (f *modelFamily) sku(model string) string {
f.mu.RLock()
defer f.mu.RUnlock()
return f.alias(model)
}
// alias is sku with the snapshot's lock held: the catalog's own aliases, then the
// vendor table ai keeps for a catalog that names none.
func (f *modelFamily) alias(model string) string {
key := strings.ToLower(strings.TrimSpace(model))
if id, ok := f.named[key]; ok {
return id
}
if id, ok := f.aliases[key]; ok {
return id
}
return model
}
// snapshot returns the family as last discovered. It does not refresh: fresh is
// what refreshes, and a caller that wants current data calls it first.
//
// The two were one call, and that put a 15-second discovery inside whatever lock
// the caller held. The catalogue's build holds one, and every family is already
// refreshed before it is taken, so the refresh in here only ever ran a SECOND
// time — for a family that is failing, which is the one case where it costs the
// full timeout and blocks every other reader for it.
func (f *modelFamily) snapshot() []zenModel {
f.mu.RLock()
defer f.mu.RUnlock()
out := make([]zenModel, 0, len(f.ids))
for _, id := range f.ids {
if m, ok := f.byID[strings.ToLower(id)]; ok {
out = append(out, m)
}
}
return out
}
// serves reports whether the family owns this model. True for any SKU the discovered
// catalog resolves, and for the public brand prefix so a request routes correctly even
// before the first discovery completes (cold start). The prefix is the public brand,
// not a confidential mapping. False when the family is not configured.
func (f *modelFamily) serves(model string) bool {
// The family's own free id is served by choosing from the platform pool, so it
// belongs to this family whether or not its vendor is configured or has ever
// heard of it — but only while the pool actually holds something to choose.
// It is asked before the vendor is, because the pool is what answers it and the
// vendor is not: a brand whose service is unreachable still has a free door.
if f.frontDoor(model) {
return len(freeRoutes()) > 0
}
// fresh resolves the family's address once and answers "" when it has none, which
// is exactly !enabled(): one admin-row read where asking both made two.
if f.fresh() == "" {
return false
}
if _, ok := f.lookup(model); ok {
return true
}
return strings.HasPrefix(strings.ToLower(strings.TrimSpace(model)), f.prefix)
}
// modelPrice returns the discovered headline price for a family model, for the
// balance-reservation estimate and the model listing. The exact debit uses
// zenModel.costCents (money/decimal), never this float projection.
func (f *modelFamily) modelPrice(model string) (modelPrice, bool) {
// The family's free id is served from the pool, and the pool is the routes that
// cost nothing. It is a stated price of zero, not an absent one — which is what
// keeps the balance gate from asking a caller to fund a call that bills nothing.
if f.frontDoor(model) {
return modelPrice{}, true
}
m, ok := f.lookup(model)
if !ok {
return modelPrice{}, false
}
return m.price()
}
// passthroughRoute routes any family model to its family provider, forwarding the SKU
// name verbatim. ai holds no SKU→upstream mapping: the family resolves the SKU, injects
// identity, folds reasoning, and picks the upstream (hip-00NN).
func (f *modelFamily) passthroughRoute(model string) *modelRoute {
if !f.serves(model) {
return nil
}
ctx := 0
if m, ok := f.lookup(model); ok {
ctx = m.MaxCtx
}
if w := f.window(model); w > 0 {
ctx = w // guarantee the flagship served window
}
p, priced := f.modelPrice(model)
return &modelRoute{
providerName: f.name,
// The family maps the SKU to its real upstream. An alias travels as the SKU
// it names, because that is the id the vendor serves.
upstreamModel: f.sku(model),
// Premium is a claim about money, so the price makes it — the free id and the
// free routes are not premium, and saying they were would read to a client as
// a model their plan cannot afford.
premium: !priced || p.InputPerMillion > 0 || p.OutputPerMillion > 0,
ownedBy: f.owner, // public branding: enso→"hanzo", zen→"zenlm"
contextWindow: ctx,
}
}
// mergeModels overlays this family's DISCOVERED lineup onto the base /v1/models list:
// a discovered SKU wins over any same-named base entry, new SKUs are appended, and
// each carries its served context window (from discovery, raised to the guaranteed
// window for the flagship SKUs).
//
// Discovery is the only source of a listing. ai never adds a SKU the family does not
// serve: a listed model that 404s upstream is worse than an absent one, and it is how
// enso-pro reached /v1/models with no price at all.
func (f *modelFamily) mergeModels(base []modelInfo) []modelInfo {
zs := f.snapshot()
now := time.Now().Unix()
// ONE id, ONE entry, on every path through this function. An id already in
// base is REPLACED rather than appended beside itself: /v1/models is keyed by
// id, so a second row for one id is not extra information — it is two answers
// to the same question, and a client building a map keeps whichever it read
// last.
//
// The free ids used to be appended blind on the empty-lineup path below, which
// was invisible while the pool was empty (freeInfo returns nothing then, so
// nothing was appended) and became three duplicate rows the moment a route
// existed — enabling our own compute is exactly what makes one exist.
idx := make(map[string]int, len(base))
for i, m := range base {
idx[strings.ToLower(m.ID)] = i
}
upsert := func(info modelInfo) {
k := strings.ToLower(info.ID)
if i, ok := idx[k]; ok {
base[i] = info
} else {
idx[k] = len(base)
base = append(base, info)
}
}
if len(zs) == 0 {
// A family whose own lineup is empty still publishes its free id: that id is
// served from the platform pool, not from anything this family discovered.
for _, info := range f.freeInfo(now) {
upsert(info)
}
sort.Slice(base, func(i, j int) bool { return base[i].ID < base[j].ID })
return base
}
for _, z := range zs {
owner := z.OwnedBy
if owner == "" {
owner = f.owner // public branding default: enso→"hanzo", zen→"zenlm"
}
window := z.MaxCtx
if w := f.window(z.ID); w > 0 {
window = w // flagship SKUs report their guaranteed served window
}
info := modelInfo{
// Premium is a statement about money, so the price is what makes it:
// a SKU the family charges for is premium, and one it charges nothing
// for is not. A free route reported as premium reads to a client as a
// SKU their plan cannot afford, which is the opposite of true.
ID: z.ID, Object: "model", Created: z.releasedOr(now), OwnedBy: owner, Premium: z.priced(),
Name: z.Name, Description: z.Description, ContextWindow: window, MaxOutputTokens: z.MaxOut, Expires: z.Expires,
Inputs: z.Inputs, Outputs: z.Outputs,
SupportsVision: z.Vision, SupportsTools: z.Tools, SupportsReasoning: z.Reasoning,
Video: z.Video, Voice: z.Voice,
}
// A gated SKU is LISTED but access-controlled; advertise the default standing
// ("waitlist"). ListModels upgrades this to the caller's real status when authed.
if z.gated() {
info.Access = &modelAccessInfo{State: "waitlist"}
}
upsert(info)
}
for _, info := range f.freeInfo(now) {
upsert(info)
}
sort.Slice(base, func(i, j int) bool { return base[i].ID < base[j].ID })
return base
}
// freeInfo is the listing entry for this family's free router, present only while
// the pool holds a route to serve it with — a listed model that answers nothing is
// worse than an absent one. Its context is the narrowest in the pool, because that
// is the prompt every route in the pool can hold, and the widest would promise a
// window the answer might not have.
func (f *modelFamily) freeInfo(now int64) []modelInfo {
names := f.freeNames()
routes := freeRoutes()
if len(names) == 0 || len(routes) == 0 {
return nil
}
window := 0
for _, r := range routes {
m, ok := r.fam.lookup(r.id)
if !ok {
continue
}
if window == 0 || (m.MaxCtx > 0 && m.MaxCtx < window) {
window = m.MaxCtx
}
}
out := make([]modelInfo, 0, len(names))
for _, n := range names {
out = append(out, modelInfo{
ID: n.id, Object: "model", Created: now, OwnedBy: n.owner,
Premium: false, ContextWindow: window, Outputs: []string{"text"},
})
}
return out
}
// ── the pipe (IO edge) ────────────────────────────────────────────────────────
// zenPipeHeader bounds ONE attempt: how long a family may take to say it has begun.
//
// It is not a round number chosen for tidiness. On a buffered request the header does
// not arrive until the whole answer does, so this ceiling is the generation ceiling,
// and real whole-site builds through this path complete at ~112s — a shorter bound
// would clip work that is doing exactly what it should. The body is deliberately not
// bounded at all: a streamed 1M-token answer is long by design, and the caller
// staying on the line is the only honest measure of whether it is still wanted.
const zenPipeHeader = 120 * time.Second
// commitWait is how long ai waits for a committed answer to begin before a paid rung
// accepted it: as long as it waits for any family's headers.
var commitWait = zenPipeHeader
// zenPipeClient forwards inference to a family. No client-level timeout: that would
// count the body, which is the part that is long on purpose.
var zenPipeClient = &http.Client{
Transport: reaching(&http.Transport{
Proxy: http.ProxyFromEnvironment,
ResponseHeaderTimeout: zenPipeHeader,
}),
}
var (
zenNewline = []byte("\n")
zenDataPrefix = []byte("data:")
)
// withModel returns body with its top-level "model" field set to model, preserving
// every other field. It reconciles the forwarded body with the model ai resolved: an
// `auto`/`zen-router` request rewrites request.Model to a concrete SKU while the raw
// body still says "auto", and the family serves by the body's model field. RawMessage
// preserves all sibling fields losslessly; on any parse/marshal error the body is
// returned unchanged so the rewrite never drops a request (the family then surfaces a
// precise error on the original body).
func withModel(body []byte, model string) []byte {
var m map[string]json.RawMessage
if err := json.Unmarshal(body, &m); err != nil || m == nil {
return body
}
mb, err := json.Marshal(model)
if err != nil {
return body
}
m["model"] = mb
out, err := json.Marshal(m)
if err != nil {
return body
}
return out
}
// familyFields are the top-level request fields a family is sent, per API path.
// Every other field the caller wrote stays behind: a vendor that resells other
// vendors reads fields that choose other models (models), other endpoints
// (provider, transforms, route), extra completions (n) and paid tools (plugins,
// web_search_options), and each would be charged to our account at a price the SKU
// never quoted. openrouterTerms writes its own provider field after this.
//
// The cache fields go on: prompt_cache_key names the conversation, so every turn is
// sent to the machine that holds its prefix, and cache_control asks for the prompt
// to be cached (the quote prices it as written).
var familyFields = map[string]map[string]bool{
"chat/completions": fieldSet("model", "messages", "stream", "stream_options",
"max_tokens", "temperature", "top_p", "top_k", "min_p", "top_a",
"frequency_penalty", "presence_penalty", "repetition_penalty", "seed", "stop",
"logit_bias", "logprobs", "top_logprobs", "response_format", "tools", "tool_choice",
"parallel_tool_calls", "user", "reasoning_effort", "reasoning",
"prompt_cache_key", "cache_control"),
"messages": fieldSet("model", "messages", "system", "max_tokens", "stream",
"temperature", "top_p", "top_k", "stop_sequences", "tools", "tool_choice",
"thinking", "metadata", "cache_control"),
"embeddings": fieldSet("model", "input", "encoding_format", "dimensions", "user"),
}
func fieldSet(names ...string) map[string]bool {
m := make(map[string]bool, len(names))
for _, n := range names {
m[n] = true
}
return m
}
// searchesWeb reports a model id that runs a web search on every call: a vendor's
// ":online" variant, billed per search on top of the tokens.
func searchesWeb(model string) bool {
return strings.HasSuffix(strings.ToLower(strings.TrimSpace(model)), ":online")
}
// familyBody is the body a family is sent for apiPath: the familyFields of the
// caller's request, with max_tokens set to maxTokens (the ceiling the hold was
// reserved for) when it is positive. max_completion_tokens is not sent, so the one
// ceiling is the reserved one. A path with no field list, or a body that is not a
// JSON object, is sent as it came.
func familyBody(body []byte, apiPath string, maxTokens int) []byte {
allowed, ok := familyFields[apiPath]
if !ok {
return body
}
var in map[string]json.RawMessage
if json.Unmarshal(body, &in) != nil || in == nil {
return body
}
out := make(map[string]json.RawMessage, len(in))
for k, v := range in {
if allowed[k] {
out[k] = v
}
}
if maxTokens > 0 && allowed["max_tokens"] {
out["max_tokens"], _ = json.Marshal(maxTokens)
}
b, err := json.Marshal(out)
if err != nil {
return body
}
return b
}
// pipeToFamily forwards the caller's request to the family service verbatim and relays
// the response back unchanged — the family owns identity, reasoning, upstream, and
// dialect; ai only authenticates, meters, and forwards. apiPath is "messages"
// (Anthropic) or "chat/completions" (OpenAI). Billing settles from the response usage
// at the discovered exact retail price; the settle is idempotent with any deferred
// fail-safe settle at the call site.
// It returns nil when the request is finished with — served, or refused for a
// reason no other vendor would fix, in which case the client already has the
// answer. It returns the refusal when the FAMILY could not serve it and NOTHING
// has been written to the client, so the caller is free to offer the same
// request to the route's declared alternates. That distinction is the whole
// reason a vendor running out of money stopped being an outage: the family is
// one provider among several, and a 402 from it is a fact about that vendor's
// balance, not about the request.
//
// The two answers differ in what they owe the reservation, and it follows from
// what they mean:
//
// nil — the request is over, so the hold is RELEASED. Nothing downstream
// is going to run and settle it.
// a refusal — the request is still moving, so the hold is UNTOUCHED. settle is
// one-shot, and releasing it here would leave whatever provider
// eventually serves this request unbilled.
//
// So: returning nil settles, always, via done. The success path has already
// settled the real cost by then and settle is idempotent, which is what lets one
// rule cover every ending rather than each ending remembering for itself — which
// is how a 400 from an embeddings family came to hold a customer's cents with
// nothing left running that would ever give them back.
func (c *ApiController) pipeToFamily(fam *modelFamily, apiPath, dialect, model string, rawBody []byte, stream bool, maxTokens int, orgId string, authUser *iam.User, isPremium bool, hold *budgetHold, start time.Time) []attempt {
// done ends the request here: the client has its answer, or has gone.
// A covered request's paid upstream: a committed family answer whose cost was never
// stated — refused, cut, or its caller gone — is charged, once, what its mark owes:
// its bound once a paid rung accepted it, else nothing.
// ctx is the request's context, read once: dispatch and pool also run from a
// stream's writer (awaitCommitted's fill), after fiber has recycled c.
grant, pm, ctx := grantOf(c.Ctx), markOf(c.Ctx), c.Context()
place := seatOf(ctx)
unsettled := func() {
if grant != nil && pm.settled.CompareAndSwap(false, true) {
grant.Settle(pm.owed.Load())
}
}
done := func() []attempt {
hold.settle(0)
// The request is over: a plan's hold ends here, at nothing more than a usage
// record already settled.
unsettled()
if grant != nil {
grant.Settle(0)
}
return nil
}
// sku is the route being asked for. requested is what the CALLER named, and is
// set only when a different route ends up answering — so it is both the tag on
// the served generation and the test for whether this was a fallback at all.
sku, requested := model, ""
// note records what one refusal says about the account that answered it: the
// account rests, and somebody is told. Separate from refused because a refusal
// the request RECOVERS from is still a refusal — an empty account nobody logs
// is an empty account nobody tops up.
note := func(err error) attempt {
cooled.rest(orgId, fam.name, err)
at := attempt{
provider: fam.name,
upstream: sku,
origin: fam.name,
status: upstreamHTTPStatus(err),
fault: faultProvider,
err: err,
}
announce(model, at)
return at
}
// refused reports a provider-side failure with nothing written: the request
// is still movable, so hand the reason back rather than spending it on an
// error page.
refused := func(err error) []attempt {
// The caller hung up. There is nobody left to serve, so offering the
// request to a second vendor would spend money answering an empty room —
// and would blame a healthy vendor for the client's disconnect.
if c.Context().Err() != nil {
return done()
}
unsettled()
return []attempt{note(err)}
}
prov := fam.providerFn()
if prov == nil {
// Unconfigured or switched off. From the request's point of view that is
// a provider that cannot serve, so try the alternates before saying no.
return refused(&apiError{status: http.StatusServiceUnavailable, msg: fam.name + " service is not configured"})
}
// Access gate (the ONE choke point): a gated SKU (enso limited preview) is
// callable only with a granted ModelAccess row. Discovery tells us which models
// are gated (zenModel.gated), so this is policy learned from data, not a hardcode.
// The deferred hold.settle(0) at the call site releases the reservation on this
// early return, so a denied request is never billed.
if msg := c.familyRefusal(fam, model, orgId, authUser); msg != "" {
// OUR gate, not the vendor's: this caller may not have this SKU. No other
// provider changes that, so answer it here.
c.zenError(dialect, msg, http.StatusForbidden)
return done()
}
// The price guard (rates.go): a model sold at a fixed price is sent only while
// no arm of its route costs us more than it bills. Ours too, and no other
// provider is asked in its place: a route priced above its model is ours to fix.
if err := fam.guard(ctx, model); err != nil {
c.zenError(dialect, err.Error(), http.StatusServiceUnavailable)
return done()
}
// Forward the model ai RESOLVED, not the caller's raw model id. An
// `auto`/`zen-router` request rewrote request.Model to a concrete family SKU, but
// the raw body still carries "model":"auto" — and the family serves by the body's
// model field, so it 404s on the unknown id "auto" while a direct request for the
// same SKU (body model == SKU) serves. Reconcile the forwarded body's model with
// the resolved SKU; byte-identical for a direct request (model already matches).
rawBody = withModel(rawBody, model)
// What a family is sent is the request that was priced and reserved for: the
// fields that shape the answer, and the token ceiling the hold covers. A field
// that picks other models, endpoints or paid tools on the vendor's side stays
// here (familyBody), and so does a web-searching model id.
if searchesWeb(model) {
c.zenError(dialect, fmt.Sprintf("%s runs a web search on every call and is not served", model), http.StatusBadRequest)
return done()
}
rawBody = familyBody(rawBody, apiPath, maxTokens)
// A chat answer the caller wants whole is still asked of the family as a stream,
// and assembled here: the stream's first frame decides the status exactly as it
// does for a streaming caller, and while the rest arrives the connection is kept
// open with whitespace, which a JSON reader skips, so an answer that takes longer
// than the edge waits for a first byte still arrives (assembleZenStream).
assemble := !stream && apiPath == "chat/completions" && dialect == "openai"
if assemble {
rawBody = streamed(rawBody)
}
reqID := uuid.NewString()
// The family relays whoever it bought the inference from, so the answer leaves
// in our envelope wearing our id, the SKU asked for, and the seller (see
// envelope.go). The id is the one this call is already metered under, which is
// also what the reward join keys on, so the customer, the ledger and the routing
// event all name this completion the same way.
//
// EVERY relayed dialect gets a mark. Only chat had one, so the Anthropic
// dialect — which is what Claude Code and every agent on that SDK speaks — and
// the embeddings shape went on publishing the sub-provider's name, `gen-` ids
// and a finish reason we do not define, long after the chat path stopped. The
// id shape differs because the dialects do: a message is msg_, a completion is
// chatcmpl-, and an embeddings list has no id at all.
mk := &mark{model: model, seller: seller(prov, authUser), routing: routingOf(c.Ctx)}
switch {
case dialect == "anthropic":
mk.speaks, mk.id = messageShape, "msg_"+reqID
case apiPath == "embeddings":
mk.speaks = listShape
default:
mk.speaks, mk.id = chatShape, "chatcmpl-"+reqID
}
url := prov.ProviderUrl + "/v1/" + apiPath
// One build, one place to fail: the method, the URL and the headers do not vary
// by SKU, so a request that cannot be built cannot be built for any of them.
req, err := http.NewRequestWithContext(c.Context(), http.MethodPost, url, nil)
if err != nil {
// Our own request is malformed. Another vendor would not fix that.
c.zenError(dialect, "build "+fam.name+" request: "+err.Error(), http.StatusInternalServerError)
return done()
}
req.Header.Set("Content-Type", "application/json")
if a := c.Header("Accept"); a != "" {
req.Header.Set("Accept", a)
}
upstream.Authorize(req, prov)
// Tenant attribution: the family needs a billable tenant, and ai — which settles
// the ledger — tells the family it fronts this call so it meters without
// double-charging. A Hanzo family is also told who pays (dispatch), by which it
// keeps each payer's share of its free lane; a vendor's API never is.
var who string
if org := tenant(ctx, orgId, authUser); org != "" {
req.Header.Set("X-Org-Id", org)
who = payerOf(ctx, org, authUser)
}
req.Header.Set("X-Hanzo-Fronted-By", "ai")
// What the caller's plan holds for paid upstream travels to a Hanzo family only:
// the family may answer from a paid rung whose cost fits, and tells us its rate.
if spend := spendFor(c.Ctx, fam, model); spend != "" {
req.Header.Set(spendHeader, spend)
req.Header.Set("TE", planTE)
}
// fill answers a covered request from the free pool once its committed answer
// refused inside an answer the caller already holds open (awaitCommitted): the
// same routes a refusal moves to, with no clock on the walk, since the caller is
// kept waiting with keep-alives rather than with silence. Set below, after pool.
var fill func(status int, body []byte) (*http.Response, string)
fills := func() bool {
return len(fallback(fam, sku, &apiError{status: http.StatusBadGateway, msg: "unavailable"}, nil)) > 0
}
// The API version and betas a caller in the Anthropic dialect stated, read while
// the request is still ours: a maker that speaks that dialect is sent them.
var version string
var betas []string
if dialect == "anthropic" {
version, betas = c.anthropicHeaders()
}
// made is the answer a maker gave, when one did: what says a maker served this
// request is the response dispatch was handed, never a header on it.
var made *http.Response
var madeBy *native
// dispatch offers this request to ONE route of ONE family: the family decides the
// address and the credential, the sku decides the model, and every other part of
// the request is the caller's own. That is what lets a route on a DIFFERENT
// vendor answer a request this one cannot — the vendor that ran out of money is
// rarely the one still holding a free route.
dispatch := func(f *modelFamily, s string) (*http.Response, error) {
p := prov
if f != fam {
// provider already encodes the one address rule — the admin row where a
// family has one, configuration otherwise — so a family reached only by
// configuration (our own compute) resolves the same way as one with a
// provider row behind it.
p = f.provider()
if p == nil {
return nil, &apiError{status: http.StatusServiceUnavailable, msg: f.name + " service is not configured"}
}
}
r, rErr := http.NewRequestWithContext(ctx, http.MethodPost, p.ProviderUrl+"/v1/"+apiPath, nil)
if rErr != nil {
return nil, rErr
}
r.Header = req.Header.Clone()
if f != fam {
// The credential is the family's own: another family's never travels.
r.Header.Del("Authorization")
}
if f.ours() && who != "" {
r.Header.Set(payerHeader, who)
}
if spend := spendOf(grant, place, pm, f, s); spend != "" {
r.Header.Set(spendHeader, spend)
r.Header.Set("TE", planTE)
} else {
r.Header.Del(spendHeader)
r.Header.Del("TE")
}
upstream.Authorize(r, p)
// The vendor is sent its own id for the SKU. An alias is ours, so it goes
// upstream as the SKU it names while the answer still wears the alias.
b := withModel(rawBody, f.sku(s))
// A model its maker serves is asked of the maker first, on every key of its
// pool, and of this family only once none of them answered (native.go). A
// maker's API bills every call, so only a request seated on the paid lane
// reaches it, and only for a route this family charges for: a free route and
// the pool's never do.
//
// And only the route the caller named: a route that stands in for another, or
// one out of the pool, is never a maker's, whatever it is priced at.
var missed []miss
if n := f.maker(s); n != nil && f == fam && s == model && !FreeOnlyFor(ctx) {
if price, listed := f.lookup(s); listed && price.priced() {
own, refused := n.send(ctx, nativeCall{path: apiPath, sku: f.sku(s), body: b, stream: stream || assemble,
version: version, betas: betas, price: price, start: start})
if own != nil {
made, madeBy = own, n
return own, nil
}
missed = refused
}
}
if f.terms != nil {
m, _ := f.lookup(s)
b = f.terms(b, m, f.free(s), object.GenAIAttributionFromContext(ctx).Session)
}
r.Body = io.NopCloser(bytes.NewReader(b))
r.ContentLength = int64(len(b))
// GetBody is what http.NewRequest sets for a bytes.Reader and what the
// transport needs to replay a body across a redirect or an HTTP/2 refusal.
// Setting the body by hand without it silently drops that.
r.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(b)), nil }
// A stream's 200 arrives before its answer; send judges its opening frames
// on each key (family_open.go).
resp, err := f.send(r, p, f.free(s), stream || assemble)
// Each key of the maker's that refused first is on the books beside whatever
// this family answers, as a family's own failed arms are.
if resp != nil && len(missed) > 0 {
resp.Header.Add(failoverHeader, chain(missed))
}
// A Hanzo family that committed its answer to a request that may buy a paid
// rung is the request's one paid attempt, whatever it then says: the request
// buys no second paid answer. A whole answer is taken as accepted by a paid rung
// at the commit; a stream says so with its paid marker. A dispatch that carried
// the spend and got no answer at all — its headers came after the wait, or the
// connection was cut — buys none either. It owes nothing: a family commits
// before it asks any rung, so nothing was bought.
// The commit is judged here, before any byte reaches the caller, so a refusal
// inside it is as movable as any family's: what the attempt cost is settled
// now, and the request goes on to the free pool like any other.
if resp != nil && hanzoFamily(f) && committedPlan(resp) {
pm.bound.Store(min(boundOf(resp.Header, grant), spendCap(grant, place)))
pm.tried.Store(true)
if resp, err = awaitCommitted(resp, waiting{pm: pm, spend: spendCap(grant, place), dialect: dialect, settle: unsettled, fill: fill, fills: fills}); resp == nil || resp.StatusCode != http.StatusOK {
unsettled()
}
}
if resp == nil && r.Header.Get(spendHeader) != "" {
pm.asked.Store(true)
}
// A family that answered a stream request whole is a whole answer: judged by
// its status like any buffered one, and relayed as it came.
if assemble && err == nil && resp != nil && resp.StatusCode == http.StatusOK && !eventStream(resp) {
return resp, nil
}
return resp, err
}
send := func(s string) (*http.Response, error) { return dispatch(fam, s) }
// pool walks the platform's free routes in order and reports the one that
// answered. It is the whole of what a free router does, and the whole of what a
// refusal falls back to, so both are one walk rather than two that can drift.
//
// A SHORT walk, not the whole list. More than one try because a free tier is rate
// limited by design — the first route is about as likely to answer 429 as to
// answer — and bounded because every try is a round trip the caller waits
// through, so worst-case latency is a property of spareTries rather than of how
// many free routes a vendor happens to advertise.
pool := func(routes []spare, skip string, bounded bool) (*http.Response, spare) {
// The pool holds routes that hold a CONVERSATION. An embeddings request has
// no free chat route to fall to, and offering it three anyway spends three
// round trips of the caller's time to be told three times that a chat model
// does not embed.
if apiPath == "embeddings" {
return nil, spare{}
}
tried := 0
// unreached holds the families a dispatch could not reach: their other
// routes sit behind the same address, so they are not asked again.
unreached := map[*modelFamily]bool{}
for _, alt := range routes {
if strings.EqualFold(alt.id, skip) || ctx.Err() != nil || unreached[alt.fam] {
continue
}
// A stand-in answers only a caller its own gate admits, and only while
// its route fits under its price.
if c.familyRefusal(alt.fam, alt.id, orgId, authUser) != "" || alt.fam.guard(ctx, alt.id) != nil {
continue
}
// A SECOND bound, on the caller's clock rather than on the count.
// zenPipeHeader bounds one attempt; spareTries bounds how many we
// borrow; neither bounds their PRODUCT, and the product is what a
// person waits through — five attempts of two minutes is ten minutes
// of a request that was expected to take twenty seconds.
//
// The walk is worth its cost because a free tier refuses FAST: a 429 is
// the answer arriving, not the answer being slow, so skipping past one
// is cheap. Once a request has already spent a whole attempt's worth of
// time, nothing here is cheap any more and the next try is just a
// longer wait before the same refusal.
//
// It bounds our own route too. spareTries deliberately does not — one
// more local call is the difference between an answer and a refusal —
// but that rule is about a SHORT walk, and this only fires where the
// walk has already stopped being short.
if bounded && time.Since(start) > zenPipeHeader {
break
}
// The budget bounds how many BORROWED routes we ask, because each is a
// round trip spent waiting on somebody else. Our own is not borrowed and
// is the last thing asked, so it is always asked: one more local call is
// the difference between an answer and a refusal, and it is the only
// route here that cannot be withdrawn by a vendor.
// Our own families — our compute, and the Enso SKUs that stand in for a
// priced route — are not borrowed either.
borrowed := alt.fam != engineFam && alt.fam != ensoFam
if borrowed && tried == spareTries {
continue
}
if borrowed {
tried++
}
r, e := dispatch(alt.fam, alt.id)
if e != nil {
unreached[alt.fam] = true
continue
}
if r.StatusCode == http.StatusOK {
return r, alt
}
r.Body.Close()
}
return nil, spare{}
}
fill = func(status int, body []byte) (*http.Response, string) {
routes := fallback(fam, sku, &apiError{status: status, msg: upstreamErrorMessage(body)}, body)
if len(routes) == 0 {
return nil, ""
}
r, alt := pool(routes, sku, false)
return r, alt.id
}
// spared offers the SAME request to a free route once this vendor cannot serve
// it, and reports the route that answered.
//
// The route need not be this vendor's. It used to have to be, which is why the
// two families whose vendor is reached through a service of their own — enso and
// zen — had no fallback at all: they declare no free route, so a refusal from
// either ended the request even while free routes elsewhere were answering.
//
// It fires on the vendor being unable to serve at all — its account with us spent,
// or its own failure — and on nothing else. `down` reads the STATUS first, because
// a 402 and a 5xx are facts, and consults text only where there is no status to
// read, and then only a narrow list of phrases measured from real upstreams. A
// malformed body (400), a model this vendor has not got (404), refused content
// (422), a rate limit (429) and a timeout all miss it, so none of them can quietly
// hand the caller a smaller model in place of an error. Our OWN balance gate
// cannot reach here either — it refuses before a vendor is dialled — so a 402 at
// this point is a statement about our account with the vendor, never about the
// customer's.
//
// A SHORT walk, not the whole list. More than one try because a free tier is
// rate limited by design — the first route is about as likely to answer 429 as
// to answer — and bounded because every try is a round trip the caller waits
// through, so worst-case latency stays a property of spareTries rather than of
// how many free routes a vendor happens to advertise.
spared := func(err error, body []byte) (*http.Response, spare, bool) {
routes := fallback(fam, sku, err, body)
if len(routes) == 0 || c.Context().Err() != nil {
return nil, spare{}, false
}
r, alt := pool(routes, sku, true)
return r, alt, true
}
// lane is whether the caller is on the free lane: the route they named costs
// nothing. Read before any route stands in, since a stand-in rewrites sku.
lane := freeLane(fam, sku)
// pooled reports whether a free-lane request the pool could not serve is the
// POOL's to answer: some account in it is benched by the vendor. Any other
// failure — a vendor down, nothing discovered, a slow walk — is the request's
// own and keeps its own path to the route's alternates.
pooled := func() (Pool, bool) {
p := FreePool()
return p, p.State != PoolAvailable
}
// poolRefused answers a free-lane request the pool could not serve: no account
// the platform shares among free users had room for it. It is the Free plan's
// own refusal — busy for a minute, or spent until the vendor resets — with the
// page that lifts it, never a vendor's words about an account the caller has
// never heard of. A caller whose plan is paid is told to pick a paid model
// instead: they have nothing to upgrade to.
poolRefused := func(p Pool) []attempt {
plan := familyTier(familyAccessSubject(orgId, authUser))
paid := plan != "" && !strings.EqualFold(plan, "free")
n := object.PoolRefused(c.Host(), c.billingOrg(authUser), p.State, p.Resets, paid)
log.Warn("free pool refused %s: state=%s ready=%d/%d", model, p.State, p.Ready, p.Keys)
if wait := n.RetryAfter(time.Now()); wait > 0 {
c.SetHeader("Retry-After", strconv.FormatInt(wait, 10))
}
// Spent until a reset hours away: a client's automatic retry would only
// be refused again, so it is told not to.
if n.Code == object.CodePoolExhausted {
c.SetHeader("x-should-retry", "false")
}
if dialect == "anthropic" {
c.respondAnthropicError("rate_limit_error", n.Message, n.Status)
return done()
}
c.SetHeader("Content-Type", "application/json")
_ = c.Bytes(n.Status, n.ErrorJSON())
return done()
}
// A free router is served by CHOOSING. The caller named the family's free id,
// which is not a model any vendor holds — it is this family's name for
// "whichever free route answers" — so the pool decides, and the answer leaves
// wearing the id the caller named.
//
// requested is set here for the same reason it is set on the fallback below: a
// different route answered, and the pair (Requested, Model) is what lets the
// ledger be read against the answer the customer holds. Without it the envelope
// would say the free id, the row would say the route, and nothing would join them.
// stood is whether a route stood in for the one named after it could not serve,
// and by is the family that answered.
var resp *http.Response
stood, by := false, fam
if fam.frontDoor(sku) {
r, alt := pool(freeRoutes(), "", true)
if r == nil {
if c.Context().Err() != nil {
return done()
}
if p, ok := pooled(); ok {
return poolRefused(p)
}
return refused(&apiError{status: http.StatusServiceUnavailable,
msg: fmt.Sprintf("model %q: no free route answered", model)})
}
sku, requested, resp, by = alt.id, model, r, alt.fam
} else if !lane && FreeOnlyFor(ctx) && FamilyOf(sku) == "" {
// The paid lane is off and the model named is a third-party one: it is not
// sent, and nothing answers in its place.
err := laneOff(ctx, model)
c.zenError(dialect, err.Error(), statusOf(err))
return done()
} else if !lane && FreeOnlyFor(ctx) && (apiPath != "embeddings" || fam.spends(sku)) {
// The paid lane is off: a priced Hanzo SKU is never sent to its paid route, and
// the Hanzo routes that stand in for it answer in its place. An embedding has no
// stand-in, and one the family funds from grants and our own compute on every
// path (spends) costs the platform no cash, which is all a closed lane bounds:
// it is sent as asked.
r, alt := pool(standIn(fam, sku), sku, true)
if r == nil {
if c.Context().Err() != nil {
return done()
}
return refused(&apiError{status: http.StatusServiceUnavailable,
msg: fmt.Sprintf("model %q: the paid lane is off and no free route answered", model)})
}
resp, requested, sku, stood, by = r, model, alt.id, true, alt.fam
log.Info("family=%s free only: %s served by %s", fam.name, model, alt.id)
} else if resp, err = send(sku); err != nil {
// Never reached the family at all. Nothing is written and nothing is
// billed — deliberately no recordFamilyUsage here, because its
// hold.settle would release the reservation one-shot and leave the cost
// of whichever provider ends up serving this request uncharged.
r, alt, tried := spared(err, nil)
if r == nil {
if tried && lane && c.Context().Err() == nil {
if p, ok := pooled(); ok {
note(err)
return poolRefused(p)
}
}
return refused(err)
}
note(err)
resp, requested, sku, stood, by = r, model, alt.id, true, alt.fam
log.Warn("family=%s could not be reached for %s (%v) — served by %s", fam.name, model, err, alt.id)
}
// Closed by whoever consumes it. That is this function for an answer read whole,
// and the stream callback for one that is relayed — the callback outlives this
// call, so it takes the body with it and this defer stops seeing one.
defer func() {
if resp != nil {
resp.Body.Close()
}
}()
// READ THE REQUEST WHILE IT IS STILL OURS. A streamed answer settles from
// inside fasthttp's writer, which runs after this handler returned; fiber has
// recycled the request by then, so reaching through `c` there is a
// use-after-free — and it does not fail the one request, it SIGSEGVs the whole
// plugin process, so every other call in flight dies with it. Measured in
// production: `DefaultCtx.App()` nil-dereferenced under
// recordFamilyUsage -> billingOrg -> Ctx.Header, twelve times an hour, each one
// answering `mount /v1: http: read response: EOF` to whoever was mid-answer.
//
// So the three request-derived facts a usage row needs are read HERE, on the
// request's own goroutine, and travel by value. recordFamilyUsage takes no
// receiver for the same reason: with no `c` in scope it cannot reach a request
// at all, which is a compile-time property rather than a rule to remember.
w := whence{ledger: c.billingOrg(authUser), ip: c.Fiber().IP(), ctx: c.Context(), asked: fam, plan: grantOf(c.Ctx)}
if fam.prices != nil {
// The row of a family that sets its own prices says how hard it was asked to
// think, beside the arm that answered (rates.go).
w.depth = depthOf(rawBody)
}
// ONE status decision, ahead of the split, because the answer to "can this
// request still be moved" is the same for both shapes. The status arrives
// BEFORE any byte of a stream is written — the WriteHeader below is still
// ahead of us — so a streaming request is exactly as movable here as a
// buffered one. This is the only window in which that is true, which is why
// the decision lives at the status and not somewhere down in the relay. A
// stream's status here is what its opening frames said (dispatch → opening).
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
status, said := resp.StatusCode, upstreamErrorMessage(b)
if status == http.StatusUnauthorized && by.ours() {
// Our own family refusing our credential is OUR outage: the caller's own
// credential was checked at our edge. It moves like a family that is down,
// to the free pool, and never reaches the caller as their auth failure.
barred(by, model, said, time.Now())
status, said = http.StatusServiceUnavailable, fmt.Sprintf("model %q is temporarily unavailable", model)
b = nil
}
err := &apiError{status: status, msg: said}
// Each arm the family asked before it gave up is on the books beside
// whatever answers next, as it would be beside an answer.
failed(w, by, sku, servingOf(resp.Header).failover, authUser, isPremium, stream, reqID, start)
r, alt, tried := spared(err, b)
if r == nil {
if tried && lane && c.Context().Err() == nil {
if p, ok := pooled(); ok {
note(err)
return poolRefused(p)
}
}
if faultOf(err) == faultProvider {
return refused(err)
}
c.zenError(dialect, said, status)
return done()
}
// The vendor could not serve — its account is empty, it is down, or a free
// route is at its rate limit — and a free route in the pool could. The
// refusal is recorded anyway — it is why the caller is about to get a
// different model, and the account it names does not top itself up. The answer leaves wearing the SKU the caller asked for; the route that
// wrote it is the ledger's Model against that Requested, which is where the
// substitution is a fact a query can filter on.
//
// After the floor above, both routes are served under the same terms — a
// route under `deny` never arrives here — so this substitution costs the
// caller nothing it was told about.
note(err)
resp.Body.Close()
resp, requested, sku, stood, by = r, model, alt.id, true, alt.fam
log.Warn("family=%s refused %s (%d) — served by %s",
fam.name, model, err.status, alt.id)
}
// A model somebody named from the resale catalog that another model answered
// is answered in that model's name: the envelope and X-Hanzo-Served say which
// one wrote it. A Hanzo family's own SKU keeps its name, since what serves it is
// the family's to say.
//
// X-Hanzo-Served names the Hanzo SKU that answered and nothing behind it: the
// family's own word when it is one of the family's SKUs (a priced SKU answered
// by its free fallback says so), the stand-in's public name, else the model asked.
name := model
if stood {
name = publicName(fam, by, sku)
if fam == freeFamily() {
mk.model = name
}
} else if said := resp.Header.Get(servedHeader); said != "" && fam != freeFamily() {
if _, ok := fam.lookup(said); ok {
name = said
}
}
c.SetHeader(servedHeader, name)
// The client is told the terms its answer was served under, because a free route
// is free in exchange for what the vendor keeps and somebody has to be able to
// say so — in a consent notice, in a log, in a policy page. Set before any byte
// of a stream goes out, which is the last moment a header can be set at all.
// The rest of what the family said about this answer is for our records only:
// the vendor and the failed arms never reach the client.
sv := servingOf(resp.Header)
if made != nil && resp == made {
sv.maker = madeBy
}
// answer is the response whose trailer states what a committed answer cost, read
// once its body has ended.
answer := resp
if word, stated := servingFamily(sku).collection(sku); stated {
c.SetHeader(headerCollection, word)
}
// WHAT AN ANSWER COST IS KNOWN ONLY WHEN IT IS COMPLETE, so both shapes settle
// here and the streamed one settles from inside its own callback.
//
// A stamped answer carries OUR id, so that is the id the client will thread back
// to /v1/ai/feedback and the id the reward has to join on. An embeddings list has no
// id of its own, so nothing is stamped and the join falls back to the internal
// reqID — which is honest: the client was handed no id to correlate on.
//
// The learning ledger records this served family call (source="family") and, when
// the engine endpoint is configured, the shadow A/B pick — off the hot path, no
// prompt text ever stored (the ledger holds none). The usage row reads only w,
// taken above while the request was ours.
// bill is the hold the answer settles: this request's, until a stream carries
// it into the writer (carry) so done's settle(0) cannot take it first.
bill := hold
settle := func(t tokens, served, respID string, first time.Time) {
if mk.id != "" {
respID = mk.id
}
if t.prompt() == 0 {
t.fresh = coarseTokenEstimate(rawBody)
}
if sv.arm != "" {
served = sv.arm
}
if !first.IsZero() {
sv.first = first.Sub(start)
}
// A committed answer states its records and its cost once its body ended: its
// usage record settles it (planCost).
// One whose cost was never stated settles what its mark owes, here, so the
// usage record is the one settle of every answer that ended. A paid attempt
// already settled — refused, and this answer the pool's — charges nothing more.
//
// The body is closed first. An answer its caller hung up on was not read to
// its end, and a maker's states what it cost as far as it got when it is
// hung up on in turn (native.go): the vendor bills what it wrote either way.
if answer.Body != nil {
_ = answer.Body.Close()
}
sv.merge(answer.Trailer)
if owed := pm.owed.Load(); sv.cost == nil && pm.tried.Load() {
sv.cost = &owed
}
if sv.cost != nil && !pm.settled.CompareAndSwap(false, true) {
sv.cost = nil
}
nano := recordFamilyUsage(w, by, sku, requested, prov, mk, authUser, isPremium, stream, reqID, t, sv, start, bill, "success", "")
c.recordFamilyRouting(model, served, respID, reqID, rawBody, orgId, authUser, t.prompt(), t.completion, nanoToCents(nano), start)
}
if stream {
c.SetHeader("Content-Type", "text/event-stream")
c.SetHeader("Cache-Control", "no-cache")
c.SetHeader("Connection", "keep-alive")
c.Status(http.StatusOK)
// THE CALLBACK RUNS AFTER THIS FUNCTION RETURNS. fasthttp produces a streamed
// body by draining a writer while it serialises the response, so the relay has
// not started when SendStreamWriter comes back — measured, not assumed.
//
// Two things follow. Everything the stream READS must outlive this call, so the
// upstream body travels into the callback and is closed there; left to the defer
// above it would be closed first and the client would be sent nothing.
//
// And everything the stream LEARNS has to be used in here. Settled outside,
// prompt and completion are still zero, so every streamed answer would be
// charged a token estimate for its prompt and nothing at all for its completion.
// The context is still the request's: fasthttp finishes writing the response
// before fiber releases it.
//
// So the stream owns the hold. It settles with what the stream cost, and at zero
// when it ended before a cost could be read; releasing it here would win the one
// settle first and leave the ledger blind to the answer. The caller hands its own
// release over for the same reason.
upstream := resp.Body
resp = nil
bill = hand(hold)
_ = c.SendStreamWriter(func(w *bufio.Writer) {
defer bill.settle(0)
defer upstream.Close()
defer hold.settle(0)
defer unsettled()
settle(relayZenStream(w, upstream, mk, pm.paid))
})
return nil
}
if assemble && eventStream(resp) {
c.SetHeader("Content-Type", "application/json")
c.Status(http.StatusOK)
upstream := resp.Body
resp = nil
bill = hand(hold)
// The answer is whole, so it takes the request's whole shape (a Responses
// object for /v1/responses), never its stream's.
shape := c.answer.shape
_ = c.sendWhole(func(w *bufio.Writer) {
defer bill.settle(0)
defer upstream.Close()
defer hold.settle(0)
defer unsettled()
settle(assembleZenStream(w, upstream, mk, heartbeat, pm.paid, shape))
})
return nil
}
b, rErr := io.ReadAll(resp.Body)
if rErr != nil {
// The family's answer was truncated on the way to us. Nothing has reached the
// client, so this is still movable.
return refused(rErr)
}
// A buffered answer's first token arrives with the rest of it.
read := time.Now()
var t tokens
sniffZenUsage(b, &t)
ct := resp.Header.Get("Content-Type")
if ct == "" {
ct = "application/json"
}
c.SetHeader("Content-Type", ct)
c.answerBody(mk.stamp(b))
settle(t, sniffZenModel(b), sniffZenId(b), read)
return done()
}
// servedHeader names the arm that answered, set by the family and relayed as is.
const servedHeader = "X-Hanzo-Served"
// The vendor that ran the arm and the arms that failed before it, set by a family
// for the gateway fronting it. Read into the usage record, never relayed.
//
// A failed arm is `<upstream> (<provider>): <reason>`, one per header field or
// several joined by "; " in one; each becomes a "failover" row beside the answer
// (failed).
const (
providerHeader = "X-Hanzo-Provider"
failoverHeader = "X-Hanzo-Failover"
// freeHeader is set when the rung that answered bills nothing.
freeHeader = "X-Hanzo-Free"
// cogsHeader is what the request cost Hanzo upstream, in USD: every arm the
// family ran for it, the one that answered and each that failed. A header on a
// whole answer, a trailer on a stream (its cost is known at its end). It is the
// usage row's COGS; the family's answer body never carries ours.
cogsHeader = "X-Hanzo-Cogs"
// keyHeader names the vendor account the arm that answered ran on, by its name
// (OPENROUTER_API_KEY_3), never its key: the free view's which-account
// (listing.go). A header, or a trailer on a stream.
keyHeader = "X-Hanzo-Key"
)
// serving is what a family said about an answer beyond its body: the arm that
// wrote it, the vendor that ran that arm, the arms that failed first, the time to
// its first token, and whether that arm bills nothing.
type serving struct {
arm, vendor, failover, account string
first time.Duration
free bool
// cost is what a committed answer's paid upstream cost, nano-dollars, as its
// family stated it at the end (costHeader); nil when none was stated.
cost *int64
// cogs is what the request cost upstream, nano-dollars (cogsHeader); nil when
// the family stated none.
cogs *int64
// maker is the vendor's own API that gave this answer (native.go), nil for
// every answer a family's address gave. The pipe sets it from the response it
// was handed, so no header an address sends can claim it.
maker *native
}
// merge reads what a family stated in its trailer, at an answer's end, over what its
// headers said.
func (sv *serving) merge(tr http.Header) {
if len(tr) == 0 {
return
}
for dst, k := range map[*string]string{&sv.arm: armHeader, &sv.vendor: providerHeader, &sv.account: keyHeader} {
if v := tr.Get(k); v != "" {
*dst = v
}
}
if v := strings.Join(tr.Values(failoverHeader), "; "); v != "" {
sv.failover = v
}
if sv.arm == "" {
sv.arm = tr.Get(servedHeader)
}
if tr.Get(freeHeader) == "true" {
sv.free = true
}
if n, ok := usdNanos(tr.Get(costHeader)); ok {
sv.cost = &n
}
if n, ok := usdNanos(tr.Get(cogsHeader)); ok {
sv.cogs = &n
}
}
// servingOf reads a family response's headers into a serving.
func servingOf(h http.Header) serving {
arm := h.Get(armHeader)
if arm == "" {
arm = h.Get(servedHeader)
}
sv := serving{arm: arm, vendor: h.Get(providerHeader), account: h.Get(keyHeader), failover: strings.Join(h.Values(failoverHeader), "; "), free: h.Get(freeHeader) == "true"}
if n, ok := usdNanos(h.Get(cogsHeader)); ok {
sv.cogs = &n
}
return sv
}
// miss is one arm a family asked and could not use: the upstream model, the vendor
// that ran it, and why.
type miss struct{ upstream, provider, reason string }
// arms reads a failover chain (failoverHeader) into its arms. An entry opens with
// `<upstream> (<provider>): `; text that does not is the previous entry's reason
// carrying a "; " of its own.
func arms(chain string) []miss {
var out []miss
for _, part := range strings.Split(chain, "; ") {
if strings.TrimSpace(part) == "" {
continue
}
head, reason, ok := strings.Cut(part, "): ")
upstream, provider, paren := strings.Cut(head, " (")
switch {
case ok && paren && upstream != "" && !strings.Contains(upstream, " ") && !strings.Contains(provider, " "):
out = append(out, miss{upstream: upstream, provider: provider, reason: reason})
case len(out) > 0:
out[len(out)-1].reason += "; " + part
default:
out = append(out, miss{reason: part})
}
}
return out
}
// failed files each arm in chain as a "failover" row beside the request reqID: the
// SKU asked of fam, the arm and its vendor, and why, billed nothing. What the arm cost
// upstream is in the request's COGS (cogsHeader), on the row of whatever answered.
func failed(w whence, fam *modelFamily, sku, chain string, u *iam.User, premium, stream bool, reqID string, start time.Time) {
for _, a := range arms(chain) {
if f := listedFamily(a.provider); f != nil && f.free(a.upstream) {
f.served(a.upstream, "", false)
}
}
for _, rec := range failoverRows(w, fam, sku, chain, u, premium, stream, reqID) {
recordTrace(w.ctx, rec, start)
}
}
// failoverRows are the rows failed files, one an arm, each bound to its caller.
// Nothing for a call with no caller to file under.
func failoverRows(w whence, fam *modelFamily, sku, chain string, u *iam.User, premium, stream bool, reqID string) []*usageRecord {
if u == nil {
return nil
}
var out []*usageRecord
for _, a := range arms(chain) {
msg := a.reason
if a.upstream != "" {
msg = a.upstream + ": " + a.reason
}
rec := &usageRecord{
Owner: w.ledger, Organization: u.Owner,
Model: sku, Provider: fam.name, Served: a.upstream, Vendor: a.provider,
Premium: premium, Stream: stream, Status: "failover", ErrorMsg: msg,
ClientIP: w.ip, RequestID: reqID, Account: "hanzo",
}
rec.bind(w.ctx, u)
out = append(out, rec)
}
return out
}
// armHeader is the upstream model a family says answered, for our records only.
const armHeader = "X-Hanzo-Arm"
// publicName is the Hanzo name an answer from route sku of family by goes by: an
// Enso or Zen SKU is its own name; any other route — a borrowed free model, our own
// compute — is the free tier of the family that was asked, and Enso's for the rest.
func publicName(asked, by *modelFamily, sku string) string {
if by == ensoFam || by == zenFam {
return sku
}
if asked != nil && asked.freeName != "" {
return asked.freeName
}
return ensoFam.freeName
}
// eventStream reports whether resp is a server-sent event stream.
func eventStream(resp *http.Response) bool {
return resp != nil && strings.HasPrefix(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream")
}
// heartbeat is how often a whole answer being assembled writes a space to keep its
// connection open.
var heartbeat = 15 * time.Second
// streamed is a chat body asking for a stream that reports its usage.
func streamed(body []byte) []byte {
var in map[string]json.RawMessage
if json.Unmarshal(body, &in) != nil || in == nil {
return body
}
in["stream"] = json.RawMessage("true")
in["stream_options"] = json.RawMessage(`{"include_usage":true}`)
out, err := json.Marshal(in)
if err != nil {
return body
}
return out
}
// assembleZenStream reads a family's chat stream to its end and writes it as one
// chat completion, stamped by mk and put in the request's dialect by shape (nil: as
// it is). Until the answer is whole it writes a space every beat, which a JSON reader
// skips as leading whitespace. An error frame after the first becomes the answer's
// error object, which every dialect reads alike.
//
// A beat that cannot be written means the client is gone: body is closed then, which
// hangs up on the family so its model stops, and what was read so far is returned.
// An answer whose usage never arrived — its client left, or the family's stream
// broke off or ran out its time before stating it — is billed for what it said: its
// text, reasoning and tool arguments, about four characters a token.
func assembleZenStream(w *bufio.Writer, body io.ReadCloser, mk *mark, beat time.Duration, paid func(), shape func([]byte) ([]byte, error)) (t tokens, served, respID string, first time.Time) {
type call struct {
ID string `json:"id,omitempty"`
Type string `json:"type"`
Function struct {
Name string `json:"name"`
Arguments string `json:"arguments"`
} `json:"function"`
}
var (
content, reasoning strings.Builder
calls = map[int]*call{}
order []int
finish string
created int64
usage json.RawMessage
failure json.RawMessage
)
done := make(chan struct{})
// Runs once the reader below is done: every return waits on done first.
defer func() {
said := content.Len() + reasoning.Len()
for _, c := range calls {
said += len(c.Function.Arguments)
}
if !t.reported && said > 0 {
t.completion = said/4 + 1
}
}()
go func() {
defer close(done)
sc := bufio.NewScanner(body)
sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
for sc.Scan() {
line := sc.Bytes()
if paid != nil && string(bytes.TrimSpace(line)) == paidMarker {
paid()
}
if !bytes.HasPrefix(line, zenDataPrefix) {
continue
}
payload := bytes.TrimSpace(line[len(zenDataPrefix):])
if len(payload) == 0 || payload[0] != '{' {
continue
}
if first.IsZero() {
first = time.Now()
}
sniffZenUsage(payload, &t)
if served == "" {
served = sniffZenModel(payload)
}
if respID == "" {
respID = sniffZenId(payload)
}
var f struct {
Created int64 `json:"created"`
Usage json.RawMessage `json:"usage"`
Error json.RawMessage `json:"error"`
Choices []struct {
Delta struct {
Content string `json:"content"`
Reasoning string `json:"reasoning"`
ReasoningContent string `json:"reasoning_content"`
ToolCalls []struct {
Index int `json:"index"`
ID string `json:"id"`
Type string `json:"type"`
Function struct {
Name string `json:"name"`
Arguments string `json:"arguments"`
} `json:"function"`
} `json:"tool_calls"`
} `json:"delta"`
Finish string `json:"finish_reason"`
} `json:"choices"`
}
if json.Unmarshal(payload, &f) != nil {
continue
}
if len(f.Error) > 0 && string(f.Error) != "null" {
failure = f.Error
}
if f.Created != 0 && created == 0 {
created = f.Created
}
if len(f.Usage) > 0 && string(f.Usage) != "null" {
usage = f.Usage
}
for _, ch := range f.Choices {
content.WriteString(ch.Delta.Content)
reasoning.WriteString(ch.Delta.ReasoningContent)
reasoning.WriteString(ch.Delta.Reasoning)
for _, tc := range ch.Delta.ToolCalls {
c := calls[tc.Index]
if c == nil {
c = &call{Type: "function"}
calls[tc.Index] = c
order = append(order, tc.Index)
}
if tc.ID != "" {
c.ID = tc.ID
}
if tc.Type != "" {
c.Type = tc.Type
}
c.Function.Name += tc.Function.Name
c.Function.Arguments += tc.Function.Arguments
}
if ch.Finish != "" {
finish = ch.Finish
}
}
}
}()
tick := time.NewTicker(beat)
defer tick.Stop()
wait:
for {
select {
case <-done:
break wait
case <-tick.C:
_, _ = w.WriteString(" ")
if w.Flush() != nil {
_ = body.Close()
<-done
return
}
}
}
var out []byte
if failure != nil && content.Len() == 0 && len(calls) == 0 {
out, _ = json.Marshal(map[string]json.RawMessage{"error": failure})
_, _ = w.Write(out)
_ = w.Flush()
return
}
msg := map[string]any{"role": "assistant", "content": content.String()}
if reasoning.Len() > 0 {
msg["reasoning_content"] = reasoning.String()
}
if len(order) > 0 {
list := make([]*call, 0, len(order))
for _, i := range order {
list = append(list, calls[i])
}
msg["tool_calls"] = list
if content.Len() == 0 {
msg["content"] = nil
}
}
if finish == "" {
finish = "stop"
}
whole := map[string]any{
"id": respID, "object": "chat.completion", "created": created, "model": served,
"choices": []map[string]any{{"index": 0, "message": msg, "finish_reason": finish}},
}
if usage != nil {
whole["usage"] = usage
}
out, _ = json.Marshal(whole)
if mk != nil {
out = mk.stamp(out)
}
if shape != nil {
shaped, err := shape(out)
if err != nil {
shaped, _ = json.Marshal(map[string]any{"error": map[string]string{"message": err.Error(), "type": "server_error"}})
}
out = shaped
}
_, _ = w.Write(out)
_ = w.Flush()
return
}
// barredSaid is when each family's refusal of our key was last logged, in Unix seconds.
var barredSaid sync.Map // family name → *atomic.Int64
// barred files one refusal of our credential by our own family fam: every one counts
// in cloud_supply_refused{provider=fam, reason=credential}, which an alert reads, and
// a storm of them is one error line a minute per family, naming what to check.
// Whether this one was logged.
func barred(fam *modelFamily, model, said string, now time.Time) bool {
supplyRefused.WithLabelValues(fam.name, reasonCredential).Inc()
v, _ := barredSaid.LoadOrStore(fam.name, new(atomic.Int64))
last := v.(*atomic.Int64)
prev := last.Load()
if prev != 0 && now.Unix()-prev < 60 || !last.CompareAndSwap(prev, now.Unix()) {
return false
}
log.Error("family=%s refused ai's credential for %s (401: %s): is %s at KMS hanzo/prod:/ai, has cloud rolled since, and does no admin provider row named %q carry a secret of its own? Every refusal counts in cloud_supply_refused{reason=credential}; this line repeats at most once a minute.",
fam.name, model, said, fam.keyKey, fam.name)
return true
}
// relayZenStream copies a family's SSE response to the client and captures the final
// usage for billing. The family already emits correct dialect SSE, so ai does not
// translate — it reads usage as it passes and, for the one dialect whose envelope is
// ours (a non-nil mark), stamps each event before it goes out. Every chunk of one
// completion is stamped from the same mark, so the id a client correlates on holds
// for the whole stream. first is when the first data chunk was written.
//
// A write that fails ends the relay: the client is gone, and the caller's close of
// body is what hangs up on the family, so its model stops generating rather than
// running to max_tokens for nobody. The request's context cannot say this — a
// fasthttp request's context does not end when its client leaves — so the failed
// write is the only signal there is. The usage a family states arrives last, so an
// answer whose usage never arrived — its client left, or the family's stream broke
// off or ran out its time before stating it — is billed for the text relayed.
func relayZenStream(w *bufio.Writer, body io.Reader, mk *mark, paid func()) (t tokens, served, respID string, first time.Time) {
said := 0 // characters of answer relayed
defer func() {
if !t.reported && said > 0 {
t.completion = said/4 + 1
}
}()
sc := bufio.NewScanner(body)
sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
for sc.Scan() {
line := sc.Bytes()
if paid != nil && string(bytes.TrimSpace(line)) == paidMarker {
paid()
}
// An SSE comment is a vendor keeping its connection open in its own words
// (": OPENROUTER PROCESSING"). The keep-alive goes on; the words do not.
if bytes.HasPrefix(line, []byte(":")) {
line = []byte(":")
}
if bytes.HasPrefix(line, zenDataPrefix) {
payload := bytes.TrimSpace(line[len(zenDataPrefix):])
if len(payload) > 0 && payload[0] == '{' {
sniffZenUsage(payload, &t)
said += answerChars(payload)
if served == "" {
served = sniffZenModel(payload)
}
if respID == "" {
respID = sniffZenId(payload)
}
if mk != nil {
line = append([]byte("data: "), mk.stamp(payload)...)
}
}
}
_, _ = w.Write(line)
_, _ = w.Write(zenNewline)
if w.Flush() != nil {
return
}
if first.IsZero() && bytes.HasPrefix(line, zenDataPrefix) {
first = time.Now()
}
}
return
}
// answerChars is how many characters of answer one stream frame carries, in either
// dialect a family speaks: an OpenAI chunk's content, reasoning and tool-call
// arguments, or an Anthropic delta's text, thinking and partial JSON.
func answerChars(payload []byte) int {
var f struct {
Choices []struct {
Delta struct {
Content string `json:"content"`
Reasoning string `json:"reasoning"`
ReasoningContent string `json:"reasoning_content"`
ToolCalls []struct {
Function struct {
Arguments string `json:"arguments"`
} `json:"function"`
} `json:"tool_calls"`
} `json:"delta"`
} `json:"choices"`
Delta *struct {
Text string `json:"text"`
Thinking string `json:"thinking"`
PartialJSON string `json:"partial_json"`
} `json:"delta"`
}
if json.Unmarshal(payload, &f) != nil {
return 0
}
n := 0
for _, c := range f.Choices {
n += len(c.Delta.Content) + len(c.Delta.Reasoning) + len(c.Delta.ReasoningContent)
for _, tc := range c.Delta.ToolCalls {
n += len(tc.Function.Arguments)
}
}
if d := f.Delta; d != nil {
n += len(d.Text) + len(d.Thinking) + len(d.PartialJSON)
}
return n
}
// sniffZenModel reads the top-level "model" (or Anthropic's message.model) from a
// family response payload — the concrete arm/rung that served, when the family
// discloses it. "" when absent; the caller then falls back to the requested SKU.
func sniffZenModel(payload []byte) string {
var p struct {
Model string `json:"model"`
Message *struct {
Model string `json:"model"`
} `json:"message"`
}
if json.Unmarshal(payload, &p) != nil {
return ""
}
if p.Model != "" {
return p.Model
}
if p.Message != nil {
return p.Message.Model
}
return ""
}
// sniffZenId reads the response id the CLIENT sees — OpenAI's top-level "id"
// (chatcmpl-…) or Anthropic's message id (msg_…, at "id" for messages or nested under
// "message"). This is the exact id the client threads back to /v1/ai/feedback, so it is
// the routing-event join key. "" when absent.
func sniffZenId(payload []byte) string {
var p struct {
ID string `json:"id"`
Message *struct {
ID string `json:"id"`
} `json:"message"`
}
if json.Unmarshal(payload, &p) != nil {
return ""
}
if p.ID != "" {
return p.ID
}
if p.Message != nil {
return p.Message.ID
}
return ""
}
// sniffZenUsage reads token usage from a family response — an SSE data payload or a
// full body — in the Anthropic (input_tokens/output_tokens, possibly nested under
// message) and OpenAI (prompt_tokens/completion_tokens) shapes. Present fields update
// the running counts; absent ones leave them, so Anthropic's message_start (input) and
// message_delta (output) accumulate across events.
//
// The dialects count a cached prompt differently, and all are read into the one
// split: OpenAI's prompt_tokens INCLUDES prompt_tokens_details.cached_tokens (and
// OpenRouter's, its cache_write_tokens beside them), while Anthropic's
// cache_read_input_tokens and cache_creation_input_tokens are IN ADDITION to
// input_tokens.
func sniffZenUsage(payload []byte, t *tokens) {
type usage struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheRead int `json:"cache_read_input_tokens"`
CacheWrite int `json:"cache_creation_input_tokens"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
PromptDetails *struct {
Cached int `json:"cached_tokens"`
Written int `json:"cache_write_tokens"`
} `json:"prompt_tokens_details"`
CompletionDetails *struct {
Reasoning int `json:"reasoning_tokens"`
} `json:"completion_tokens_details"`
}
var p struct {
Usage *usage `json:"usage"`
Message *struct {
Usage *usage `json:"usage"`
} `json:"message"`
}
if json.Unmarshal(payload, &p) != nil {
return
}
// whole says the usage is the answer's own count. The one a Messages stream
// opens with (message_start) is not: it states the prompt and an output of one
// token before a word is written, so it never says what the answer's output
// cost. Taking it as that billed a stream its caller hung up on for one token.
read := func(u *usage, whole bool) {
if u == nil {
return
}
if u.InputTokens > 0 {
t.fresh = u.InputTokens
}
if u.CacheWrite > 0 {
t.written = u.CacheWrite
}
if u.CacheRead > 0 {
t.cached = u.CacheRead
}
if u.PromptTokens > 0 {
cached, written := 0, 0
if u.PromptDetails != nil {
cached = min(max(u.PromptDetails.Cached, 0), u.PromptTokens)
written = min(max(u.PromptDetails.Written, 0), u.PromptTokens-cached)
}
t.fresh, t.cached, t.written = u.PromptTokens-cached-written, cached, written
}
if u.OutputTokens > 0 {
t.completion = u.OutputTokens
t.reported = t.reported || whole
}
if u.CompletionTokens > 0 {
t.completion, t.reported = u.CompletionTokens, true
}
if u.CompletionDetails != nil && u.CompletionDetails.Reasoning > 0 {
t.reasoning = u.CompletionDetails.Reasoning
}
}
read(p.Usage, true)
if p.Message != nil {
read(p.Message.Usage, false)
}
}
// coarseTokenEstimate is a last-resort prompt-token estimate (~4 chars/token) so a
// successful call whose usage ai could not read is still billed, never zero.
func coarseTokenEstimate(body []byte) int {
return len(body)/4 + 1
}
// recordFamilyUsage settles the budget hold at the exact discovered price and records
// the usage + trace. Billing lives in one place for both stream and buffered paths and
// for every family. origin is who actually served the call (originOf) — the row's
// answer to a question the response no longer answers.
// model is the SKU that ANSWERED and requested the one the caller named, set only
// when a spare route answered in its place. Both travel because the ledger is
// answering two questions — what did we serve, and was it what was asked for — and
// a row that can only answer the first cannot be read as a downgrade at all.
// whence is what the REQUEST said, read on the request's own goroutine and
// carried by value. A usage row needs exactly three things from the request, and
// all three are only safe to read before the handler returns — see the capture in
// pipeToFamily for what reading them later costs.
type whence struct {
ledger string // billingOrg: X-Org-Id, the bearer, the session cookie
ip string // the peer, fiber's own read
ctx context.Context // what the usage row and its span hang off
asked *modelFamily // the family of the model the caller named
plan *object.LimitGrant // the plan that covers the request, nil for none
depth string // how hard the request asks its model to think (depthOf)
}
// recordFamilyUsage takes NO RECEIVER, deliberately. It is called from inside a
// stream writer that outlives the request, so a `c` in scope here is a request
// this function must not touch — and the way to keep a rule like that is to leave
// nothing to touch.
//
// It returns the charge in nano-dollars: exact, never floored to a cent, so an
// embedding worth a thousandth of a cent bills a thousandth of a cent at the hold,
// the debit, the row and the span alike.
func recordFamilyUsage(w whence, fam *modelFamily, model, requested string, prov *object.Provider, mk *mark, authUser *iam.User, isPremium, stream bool, reqID string, t tokens, sv serving, start time.Time, hold *budgetHold, status, errMsg string) int64 {
// The vendor charges nothing for a spare route — that is the whole reason it can
// answer while the account is empty — so it is billed at nothing. Charging
// retail for a downgrade the customer did not choose would be taking money for
// the outage. Stated once, on the record, so the hold, the debit, the warehouse
// row and the span all read the same zero (see usageCostNano). A priced SKU the
// family says it answered from a free rung costs the caller what that rung costs.
free := fam.isSpare(model) || inPool(model) || sv.free
// A free model a family ran on a listed vendor's account counts toward the free
// view (listing.go); what ai sent on its own accounts counted as it was sent.
if lf := listedFamily(sv.vendor); lf != nil && sv.arm != "" && lf.free(sv.arm) {
lf.served(sv.arm, sv.account, status == "success")
}
// What the call cost us to buy: the family's own statement (cogsHeader) where it
// made one, else the cost the answer's usage stated (a resale vendor's), with the
// fee of the vendor whose credit bought it on top (fee.go). It is the row's COGS,
// so usageMargin reads the margin on a relayed call as a fact rather than a guess.
cogs := sv.cogs
if cogs == nil {
cogs = mk.cogs()
}
cost := costed(cogs, buyers(fam, sv)...)
var nano int64
var exact *int64 // the charge the ledger, row and span read
if status == "success" && !free {
if zm, ok := fam.lookup(model); ok {
switch {
case sv.maker != nil && cost != nil:
// A maker's answer is priced at cost: what it charged, the platform's
// fee on top and nothing else (native.go, fee.go).
nano = plus(*cost, false)
default:
// The model's price for its tokens. A resold SKU never bills below
// the cost its answer states; a cost in the headers of any other
// answer is its family's record, not a price. An Enso model bills its
// fixed price whatever its arm cost (rates.go).
nano = zm.bill(mk.cogs(), t)
}
} else {
nano = tokenCostNano(model, t.prompt(), t.completion, t.cached, t.written)
}
// A model that stood in for the one named costs no more than the named one
// would have for the same tokens: the caller pays for an answer, never for
// our outage.
if requested != "" && w.asked != nil {
if zm, ok := w.asked.lookup(requested); ok && zm.priced() {
nano = min(nano, zm.retailNano(t))
}
}
exact = &nano
}
// A call that cost us more than it billed is a loss, and a loss is never
// silent: the row says how much (loss_nano), and the ledger counts it (rates.go).
var loss int64
if status == "success" && cost != nil && *cost > nano {
loss = *cost - nano
log.Warn("loss: %s answered by %s (%s) cost %d nano and billed %d", model, sv.arm, sv.vendor, *cost, nano)
}
var trailed string
if fam.prices != nil {
trailed = trail(fam, model, sv, t)
if status == "success" {
fam.answered(model, sv)
}
}
hold.settleNano(nano)
if authUser == nil {
return nano
}
origin, via := reached(prov, mk, sv)
rec := &usageRecord{
Owner: w.ledger, Organization: authUser.Owner,
Model: model, Requested: requested, Free: free, Provider: fam.name, Origin: origin, Via: via,
PromptTokens: t.prompt(), CacheReadTokens: t.cached, CacheWriteTokens: t.written, CompletionTokens: t.completion, TotalTokens: t.prompt() + t.completion,
ReasoningTokens: t.reasoning, Served: sv.arm, Vendor: sv.vendor, Failover: sv.failover, First: sv.first,
Reasoning: w.depth, Arms: trailed, LossNano: loss,
Cost: float64(nano) / 1e9, Currency: "USD",
Premium: isPremium, Stream: stream, Status: status, ErrorMsg: errMsg,
ClientIP: w.ip, RequestID: reqID, Account: "hanzo", Routing: string(mk.routing),
CostNanoExact: cost,
BilledNanoExact: exact,
plan: w.plan,
planNanos: planCost(w.plan, sv.cost),
}
rec.bind(w.ctx, authUser)
recordUsage(rec)
recordTrace(w.ctx, rec, start)
failed(w, fam, model, sv.failover, authUser, isPremium, stream, reqID, start)
return nano
}
// recordFamilyRouting writes the privacy-preserving learning ledger row for a served
// family call and, when the engine endpoint is configured, the shadow A/B pick. It is
// fire-and-forget off the request hot path (exactly like AddRoutingEvent's auto-route
// callers): a collection failure never touches the served response. It NEVER stores
// prompt text — only token/cost/latency metrics, the served arm, and (for the shadow)
// the engine's own opaque feature vector + counterfactual model.
//
// The request-derived routing inputs (last user text, media flag) are extracted
// synchronously from rawBody before the goroutine spawns, so nothing reads the router
// context off-thread. The shadow /v1/route call is hard-capped by router.DefaultTimeout.
func (c *ApiController) recordFamilyRouting(model, served, respID, reqID string, rawBody []byte, orgId string, authUser *iam.User, prompt, completion int, cents int64, start time.Time) {
owner := orgId
user := ""
if authUser != nil {
if owner == "" {
owner = authUser.Owner
}
user = authUser.Owner + "/" + authUser.Name
}
// The join key is the response id the client sees; fall back to the internal reqID
// when the family disclosed none (RecordFamilyRouting normalizes it identically to
// /v1/ai/feedback so both sides key on the same value).
responseId := respID
if strings.TrimSpace(responseId) == "" {
responseId = reqID
}
text, hasMedia := familyRoutingText(rawBody)
endpoint := ""
if cfg := GetModelConfig(); cfg != nil {
endpoint = cfg.RouterClient(nil).Endpoint
}
in := object.FamilyRoutingInput{
Owner: owner, User: user,
RequestedModel: model, RoutedModel: served, ResponseId: responseId,
PromptTokens: prompt, CompletionTokens: completion,
CostCents: cents, LatencyMs: time.Since(start).Milliseconds(),
RouterEndpoint: endpoint, ShadowText: text,
ShadowApproxTokens: coarseTokenEstimate(rawBody), ShadowHasMedia: hasMedia,
}
// Fire-and-forget off the request hot path — the ONE shared writer (object) is
// called identically by cloud's embedded-zen meter.
go object.RecordFamilyRouting(in)
}
// familyRoutingText extracts the last user turn's text and whether the request carries
// media from a raw chat/completions or messages body — enough for the shadow router to
// classify, working for both dialects (both use messages[].role/content, content a
// string or an array of typed parts). It is a pure function of rawBody; it returns no
// prompt text to any store — only the shadow /v1/route call (which the engine already
// receives for auto routing) reads it, and only the engine's derived features persist.
func familyRoutingText(rawBody []byte) (text string, hasMedia bool) {
var body struct {
Messages []struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
} `json:"messages"`
}
if json.Unmarshal(rawBody, &body) != nil {
return "", false
}
for _, m := range slices.Backward(body.Messages) {
if m.Role != "user" {
continue
}
var s string
if json.Unmarshal(m.Content, &s) == nil {
if s != "" {
return s, false
}
continue
}
var parts []struct {
Type string `json:"type"`
Text string `json:"text"`
}
if json.Unmarshal(m.Content, &parts) != nil {
continue
}
var b strings.Builder
for _, p := range parts {
if strings.Contains(p.Type, "image") || strings.Contains(p.Type, "video") {
hasMedia = true
}
if p.Text != "" {
if b.Len() > 0 {
b.WriteByte('\n')
}
b.WriteString(p.Text)
}
}
if b.Len() > 0 || hasMedia {
return b.String(), hasMedia
}
}
return "", hasMedia
}
// zenError writes a dialect-correct error. Anthropic callers (Claude Code) get the
// Anthropic error envelope; OpenAI callers get the OpenAI one.
func (c *ApiController) zenError(dialect, msg string, status int) {
if dialect == "anthropic" {
c.respondAnthropicError(anthropicErrorTypeForStatus(status), msg, status)
return
}
body, _ := json.Marshal(map[string]any{
"error": map[string]any{"message": msg, "type": "api_error", "code": status},
})
c.SetHeader("Content-Type", "application/json")
_ = c.Bytes(status, body)
}