mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
Red's review of never-stop:
- A masked refusal ("the model could not serve this request") is one a
caller's own request can provoke at will, and its fall read no
allowance: three such calls with the allowance spent all answered 200.
The floor it falls to now takes one call of the free allowance first;
spent, the allowance's own 429 answers and the floor is never asked.
- A masked 403 or 422 is a moderation or content refusal and stands.
- relax's "auto" step appends its sentence after the caller's own
instructions, never as a system message ahead of them.
- Every fallen answer says X-Hanzo-Data-Collection: allow.
- A stream's stop written with a space ("finish_reason": "stop") is a
stop, not a cut.
Testing: go test ./controllers/ -run 'TestAMaskedRefusalFalls|TestRelaxLoosens|TestAFallenAnswerIsOnTheFreeLane|TestAStreamThatBreaksOff|TestARefusalThatIsTheCallers' (the masked body is the engine's own bytes)
Risk: low.
How to revert: git revert this commit.
3478 lines
139 KiB
Go
3478 lines
139 KiB
Go
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
// zen_client.go — ai's thin client of the Hanzo model-family services.
|
|
//
|
|
// A model family (Zen, Enso, …) is a self-contained serving layer: it owns which
|
|
// upstream serves a SKU, the identity, the reasoning-vocabulary fold, the 1M context
|
|
// ladder, vision, and the enso-ultra fan-out. ai authenticates, meters, and forwards;
|
|
// it discovers each family from GET <family>/v1/models and pipes requests verbatim.
|
|
// A family is a VALUE — a name, a base URL, a service key, an ownership prefix — so
|
|
// zen and enso are two instances of the SAME machinery (modelFamily), not two copies.
|
|
// ai holds only configuration and the public brand, never an upstream mapping. See
|
|
// hip-00NN for the catalog, pricing, and discovery contract.
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"cmp"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"slices"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/hanzoai/ai/conf"
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/hanzoai/ai/object"
|
|
"github.com/hanzoai/ai/plugin"
|
|
"github.com/hanzoai/ai/upstream"
|
|
"github.com/hanzoai/decimal"
|
|
)
|
|
|
|
// ── the families (values, not places) ────────────────────────────────────────
|
|
//
|
|
// Each family's address is configuration (a base URL + service key, set in the
|
|
// deployment), so this repository carries no routing detail and reads as open
|
|
// source. A family is added by adding an instance here — nothing else in ai knows a
|
|
// family name.
|
|
|
|
// flagshipWindow is the served context window (1M tokens) ai guarantees for the
|
|
// flagship Zen/Enso SKUs, independent of what a given discovery snapshot advertises.
|
|
const flagshipWindow = 1_000_000
|
|
|
|
// modelFamily is one serving family as a value: its public name/brand + cold-start
|
|
// ownership prefix, the config keys that hold its address, the provider record ai
|
|
// forwards through, and a discovered-catalog snapshot refreshed from the family on a
|
|
// TTL. All of zen's original machinery is a method on this value; zen and enso are
|
|
// two instances.
|
|
type modelFamily struct {
|
|
name string // "zen" | "enso" | "openrouter" — the brand label
|
|
// prices is the family's own price list: each model it names lists and bills at
|
|
// its price there, whatever the family's service quotes, and is sent only while
|
|
// no arm of its route costs more than that (rates.go). Enso's is ensoPrices.
|
|
prices map[string]tag
|
|
// ladder is the family's route as its admin catalog was last read, which the
|
|
// guard holds each model's price to (rates.go).
|
|
ladder ladder
|
|
typ string // the object.Provider Type this family serves ("Zen" | "Enso" | "OpenRouter")
|
|
prefix string // public brand prefix for cold-start ownership (before first discovery)
|
|
owner string // public /v1/models owned_by: "zenlm" (Zen LM) | "hanzo" (Hanzo)
|
|
urlKey string // config key for the base URL ("ZEN_URL" | "ENSO_URL")
|
|
keyKey string // config key for the service key ("ZEN_API_KEY" | "ENSO_API_KEY")
|
|
keyNames []string // the vendor accounts a request tries in order (family_keys.go); nil = the provider's one key
|
|
keyList string // one more name, holding further accounts' keys, one a line, tried after keyNames
|
|
// credits reads what one vendor account has left, in USD, from the body of GET
|
|
// base + creditsPath (family_keys.go). nil for a family whose vendor states none.
|
|
credits func([]byte) (float64, error)
|
|
creditsPath string
|
|
providerFn func() *object.Provider // the virtual provider ai forwards through
|
|
// windows is the served context window ai guarantees for a DISCOVERED SKU,
|
|
// keyed by SKU id — a flagship reports its real 1M window even when a given
|
|
// discovery snapshot advertises less. It cannot add a SKU: a model appears in
|
|
// /v1/models iff the family serves it. A map (not a list ai could iterate into
|
|
// the listing) is what makes an unservable entry unrepresentable — the previous
|
|
// list synthesized a listing for any SKU the family "would" serve, which is how
|
|
// enso-pro was advertised, unpriced, for a family that answers 404 for it.
|
|
windows map[string]int
|
|
|
|
// decode turns a family's raw GET /v1/models body into discovered SKUs. nil means
|
|
// the Hanzo family wire shape (zenWireModel), which already carries every field ai
|
|
// needs. A family that publishes a DIFFERENT catalog dialect — OpenRouter states
|
|
// price per TOKEN and never states a tier or a funding class — supplies its own
|
|
// translation here, so the dialect is the ONLY thing that varies and discovery,
|
|
// TTL, listing, routing, and gating stay one implementation.
|
|
decode func([]byte) ([]zenModel, error)
|
|
|
|
// spare names the routes this family still serves once its account with the
|
|
// vendor is spent — the SKUs the vendor charges nothing for. It reads the same
|
|
// /v1/models body decode reads, in the family's own dialect, because what "free"
|
|
// looks like on the wire is exactly as dialect-specific as what a price is.
|
|
//
|
|
// It is a SUBSET of the catalog, not a second one: the routes the vendor charges
|
|
// nothing for AND answers in text with. Price is what qualifies a route as a
|
|
// spare, so a spare bills nothing whether a caller named it or a refusal for
|
|
// money landed on it — a downgrade can never quietly become something we charge
|
|
// for, and a route we charge for can never be handed out as a remedy.
|
|
//
|
|
// nil means the family has none, and a refusal for money from it is final.
|
|
spare func([]byte) []string
|
|
|
|
// freeName is the one id this family publishes for the free pool: a route a
|
|
// caller can name, and the route a refusal from this family falls to. It is
|
|
// branded per family because that is where a caller looks for it, and it is
|
|
// served from the SAME pool for all of them, because a route that costs nothing
|
|
// is not made different by whose name is on it.
|
|
freeName string
|
|
|
|
// terms states, in the family's own dialect, what the vendor may keep of an
|
|
// exchange. It is called with the outgoing body and whether the route being
|
|
// asked for is one the vendor charges nothing for, because that is the trade a
|
|
// free route IS: the vendor serves it for nothing and keeps what it carried.
|
|
// A priced route pays instead, and keeps nothing. m is the SKU as listed, whose
|
|
// retail bounds what the vendor may route a priced request to.
|
|
//
|
|
// nil for a family whose dialect states no such thing, and the body then goes
|
|
// out untouched — vendor-specific fields are never sent to a vendor that did
|
|
// not define them.
|
|
terms func(body []byte, m zenModel, free bool, session string) []byte
|
|
|
|
// makers are the APIs of the vendors that MAKE some of this family's SKUs, each
|
|
// asked for its own (its prefix) before the family's own address (native.go): a
|
|
// model a reseller lists is its maker's to serve first. Empty for a family with
|
|
// none.
|
|
makers []*native
|
|
|
|
// aliases maps an id a caller may send onto the discovered SKU that serves it,
|
|
// lowercase on both sides, for a vendor whose catalog names none (OpenRouter's,
|
|
// openrouter_alias.go). A Hanzo family's catalog states its own (named). lookup
|
|
// reads through both, so everything that asks the family about a model — whether
|
|
// it serves it, its price, its floors, its terms — answers for the alias exactly
|
|
// as for the SKU, and sku is what goes upstream.
|
|
aliases map[string]string
|
|
|
|
// lists are the paths of the family's model lists, read as one catalog
|
|
// (GET base + each): "/v1/models" when it names none. OpenRouter lists its
|
|
// video models apart (/v1/videos/models).
|
|
lists []string
|
|
|
|
// load is where the family's catalog body comes from when it is not the family's
|
|
// own GET /v1/models: a vendor whose list is synced into the store (listing.go)
|
|
// reads it from there. nil asks the family's address.
|
|
load func(context.Context, *object.Provider) ([]byte, error)
|
|
// started is whether this process has read the family's catalog once.
|
|
started atomic.Bool
|
|
|
|
// discovered catalog — a read-mostly snapshot; discovery failure keeps the last
|
|
// good snapshot so a transient blip never empties ai's model list.
|
|
mu sync.RWMutex
|
|
byID map[string]zenModel
|
|
ids []string
|
|
spares []string
|
|
fetchedAt time.Time
|
|
loaded bool
|
|
// named is the aliases the family's own catalog states (catalogHead), lowercase.
|
|
named map[string]string
|
|
// paid is the catalog's `paid` line: whether it opens the paid lane (open).
|
|
paid bool
|
|
|
|
// loading serializes the first discovery (warm), and tried is when it last ran
|
|
// without loading, so a family that cannot be reached is asked again after
|
|
// warmRetry rather than on every read.
|
|
loading sync.Mutex
|
|
tried time.Time
|
|
}
|
|
|
|
var (
|
|
zenFam = &modelFamily{
|
|
name: "zen", typ: "Zen", prefix: "zen", owner: "zenlm", urlKey: "ZEN_URL", keyKey: "ZEN_API_KEY",
|
|
freeName: "zen-free",
|
|
providerFn: object.ZenProvider,
|
|
windows: map[string]int{
|
|
"zen5": flagshipWindow,
|
|
"zen5-coder": flagshipWindow,
|
|
"zen5-pro": flagshipWindow,
|
|
},
|
|
}
|
|
ensoFam = &modelFamily{
|
|
name: "enso", typ: "Enso", prefix: "enso", owner: "hanzo", urlKey: "ENSO_URL", keyKey: "ENSO_API_KEY",
|
|
prices: ensoPrices,
|
|
freeName: "enso-free",
|
|
providerFn: object.EnsoProvider,
|
|
// The flagship window. Every other SKU, enso-pro among them, is listed and
|
|
// served from discovery.
|
|
windows: map[string]int{
|
|
"enso-ultra": flagshipWindow,
|
|
},
|
|
}
|
|
// engineFam is OUR OWN compute: the models this deployment serves itself, at no
|
|
// marginal cost to anybody's account. It publishes no catalog of its own here —
|
|
// it exists so the free pool has a route that cannot be taken away by a vendor's
|
|
// balance — so it is deliberately absent from modelFamilies below and is reached
|
|
// only through freeRoutes.
|
|
engineFam = &modelFamily{
|
|
name: "engine", typ: "Engine", prefix: "engine/", owner: "hanzo",
|
|
urlKey: "ENGINE_URL", keyKey: "ENGINE_API_KEY",
|
|
}
|
|
|
|
// modelFamilies is the discovery/route/merge iteration order — the ONE list every
|
|
// generic family helper walks. openrouter is last: the first-party families claim
|
|
// their own SKUs first, and the resale catalog answers for what is left.
|
|
modelFamilies = []*modelFamily{zenFam, ensoFam, openrouterFam}
|
|
)
|
|
|
|
// ours reports whether f is a Hanzo service ai fronts (Zen, Enso), which may be
|
|
// told who a call is for. A vendor's API is not one.
|
|
func (f *modelFamily) ours() bool { return f == zenFam || f == ensoFam }
|
|
|
|
// window returns the context window ai guarantees when listing and routing a SKU,
|
|
// or 0 when the family guarantees none. It applies only to a configured family, so
|
|
// a bare or unconfigured family reports no guaranteed window.
|
|
func (f *modelFamily) window(model string) int {
|
|
if !f.enabled() {
|
|
return 0
|
|
}
|
|
return f.windows[strings.ToLower(strings.TrimSpace(model))]
|
|
}
|
|
|
|
// provider resolves the family to the record ai forwards through — its OWN
|
|
// constructor where it has one, so the address that gates listing is the address
|
|
// that serves. That constructor reads the admin row first and falls back to
|
|
// deployment config, which is what makes admin.hanzo.ai the one control:
|
|
// disabling a family there makes provider nil, so enabled() goes false and its
|
|
// models leave /v1/models on the next resolution.
|
|
//
|
|
// Reading conf directly here (which is what this did) meant the console toggle
|
|
// moved a row that listing never consulted: a family could be disabled in the
|
|
// admin UI and still be listed and served.
|
|
//
|
|
// A family reached only by configuration — our own compute — has the same record
|
|
// built for it here, so every caller holds one shape and the credential stays in
|
|
// it: nothing in this file hands a key to anybody, it hands the record to
|
|
// authorize.
|
|
func (f *modelFamily) provider() *object.Provider {
|
|
if f.providerFn != nil {
|
|
return f.providerFn()
|
|
}
|
|
base := strings.TrimRight(strings.TrimSpace(conf.GetConfigString(f.urlKey)), "/")
|
|
if base == "" {
|
|
return nil
|
|
}
|
|
return &object.Provider{
|
|
Owner: "admin",
|
|
Name: f.name,
|
|
Category: "Model",
|
|
Type: f.typ,
|
|
State: "Active",
|
|
ProviderUrl: base,
|
|
ClientSecret: strings.TrimSpace(conf.GetConfigString(f.keyKey)),
|
|
}
|
|
}
|
|
|
|
func (f *modelFamily) baseURL() string {
|
|
if p := f.provider(); p != nil {
|
|
return strings.TrimRight(strings.TrimSpace(p.ProviderUrl), "/")
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// enabled reports whether the family service is configured AND admin-enabled. When
|
|
// it is not, ai serves no model of that family (serves is false) and it is simply
|
|
// absent from /v1/models.
|
|
func (f *modelFamily) enabled() bool { return f.baseURL() != "" }
|
|
|
|
// familyForProviderType maps a virtual provider Type onto its family, resolved from the
|
|
// SAME modelFamilies list that discovery, listing and routing all read.
|
|
//
|
|
// It used to be a hand-written switch, which is a second list that must be kept in step
|
|
// with the first — and it was not. OpenRouter was added to modelFamilies, so its 338
|
|
// prepaid models were discovered, listed and routable, but the switch never named it. Every
|
|
// dispatch site guards on `if fam := familyForProviderType(...); fam != nil`, so a nil sent
|
|
// those requests down the generic relay — around pipeToFamily, which is where the funding
|
|
// gate lives. Listed, served, and ungated, on real cash.
|
|
//
|
|
// Deriving the lookup from the one list makes that shape unrepresentable: a family that
|
|
// exists is a family this resolves, and adding one cannot silently skip the gate.
|
|
func familyForProviderType(t string) *modelFamily {
|
|
for _, f := range modelFamilies {
|
|
if f.typ == t {
|
|
return f
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// familyNamed is the family whose provider row is named name, or nil.
|
|
func familyNamed(name string) *modelFamily {
|
|
for _, f := range modelFamilies {
|
|
if f.name == name {
|
|
return f
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// freeID is the platform's OWN name for the pool: one id every surface can send and
|
|
// always get a free answer from, wearing no brand at all. enso-free and zen-free are
|
|
// the same pool behind a brand; this is the same pool in front of all of them, so a
|
|
// client that does not care which family answered has one id to remember.
|
|
const freeID = "free"
|
|
|
|
// freeDoor is the family and id the platform's free pool is served under. Wherever
|
|
// Enso's own catalog publishes its free id, the pool answers as Enso: Enso's service
|
|
// serves it and gives it Enso's name and maker, so no vendor's model introduces itself
|
|
// to a caller who asked for the free tier. Where Enso does not carry it, the family
|
|
// that holds the free routes chooses among them.
|
|
func freeDoor() (*modelFamily, string) {
|
|
if ensoFam.enabled() {
|
|
if _, own := ensoFam.lookup(ensoFam.freeName); own {
|
|
return ensoFam, ensoFam.freeName
|
|
}
|
|
}
|
|
return freeFamily(), freeID
|
|
}
|
|
|
|
// frontDoor reports that this id is a name this family publishes for the pool — its
|
|
// own branded one, and, for the family that HOLDS the pool, the platform's unbranded
|
|
// one. Such an id is not a model any vendor has: it is a name for a choice.
|
|
//
|
|
// A family whose own catalog publishes its branded free id serves it like any other
|
|
// SKU, with that catalog's rungs, fallbacks and vision, so the id has one free path.
|
|
func (f *modelFamily) frontDoor(model string) bool {
|
|
m := strings.ToLower(strings.TrimSpace(model))
|
|
if f.freeName != "" && m == strings.ToLower(f.freeName) {
|
|
_, own := f.lookup(m)
|
|
return !own
|
|
}
|
|
return f == freeFamily() && m == freeID
|
|
}
|
|
|
|
// freeNames are every id this family publishes for the pool, with the owner each is
|
|
// branded to. The platform's own id is owned by us because it is our product, not the
|
|
// vendor's, whichever vendor happens to answer it.
|
|
func (f *modelFamily) freeNames() []struct{ id, owner string } {
|
|
var out []struct{ id, owner string }
|
|
if f.freeName != "" && f.frontDoor(f.freeName) {
|
|
out = append(out, struct{ id, owner string }{f.freeName, f.owner})
|
|
}
|
|
if f == freeFamily() {
|
|
out = append(out, struct{ id, owner string }{freeID, "hanzo"})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// freeFamily holds the platform's free routes: the family whose vendor publishes
|
|
// routes it charges nothing for. Every family falls back to it, itself included,
|
|
// because a route that costs nothing is not made different by whose refusal sent a
|
|
// request to it.
|
|
func freeFamily() *modelFamily { return openrouterFam }
|
|
|
|
// spare is a route that can answer when the one asked for cannot: the family that
|
|
// carries it and the id it is served under. A pair, because the vendor that ran out
|
|
// of money is rarely the one still holding a free route.
|
|
type spare struct {
|
|
fam *modelFamily
|
|
id string
|
|
}
|
|
|
|
// freeRoutes are the routes that cost nothing and answer a chat turn, best first.
|
|
// ONE pool for the whole platform: one list to curate, and every family falls back
|
|
// to the same places.
|
|
//
|
|
// OUR OWN COMPUTE IS LAST, and last is the point of it. A route we serve ourselves
|
|
// cannot run out of a vendor's money and cannot be rate limited by one, so it is the
|
|
// answer that remains when every borrowed one is gone — which is exactly a last
|
|
// resort and not a first choice. Measured: the engine serves one small model, and
|
|
// asked to reply with a single word it answers with several sentences about editing
|
|
// a sentence. Putting it first would spend that on every free answer to insure
|
|
// against an outage that has not happened; putting it last insures against it for
|
|
// nothing. Its place moves the day it serves a model that answers as well as the
|
|
// borrowed ones, and only the order below changes.
|
|
func freeRoutes() []spare {
|
|
var out []spare
|
|
for _, id := range freeFamily().spareRoutes() {
|
|
out = append(out, spare{freeFamily(), id})
|
|
}
|
|
if engineFam.enabled() {
|
|
out = append(out, spare{engineFam, engineModel})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// fallback names where a request its vendor refused may be offered next, and an
|
|
// empty list means the refusal stands as itself.
|
|
//
|
|
// It is the WHOLE of the decision and none of the walking. Three facts settle it
|
|
// and not one of them needs a socket, a clock or a request: who the refusal is
|
|
// about, whether the vendor can serve at all, and the terms the route was bought
|
|
// under. Kept apart from the walk so the policy can be read — and asserted —
|
|
// without standing a vendor up to refuse.
|
|
func fallback(fam *modelFamily, sku string, err error, body []byte) []spare {
|
|
// Our own spend gate, relayed by a service that fronts for us, wears the same
|
|
// 402 and means the opposite thing: the CUSTOMER owes money. That has to reach
|
|
// them, and answering it with a free model would tell somebody their payment
|
|
// problem had fixed itself.
|
|
if billingNotice(body) {
|
|
return nil
|
|
}
|
|
// A vendor that cannot serve this request moves it: its account with us is
|
|
// spent, it failed, it could not be reached, or every account it holds is rate
|
|
// limited, or our own family hid why its vendor refused (masked). A malformed
|
|
// body (400), a model it has not got (404) and refused content (422) stay where
|
|
// they are: every vendor would answer them the same.
|
|
if err == nil || !unserved(err) && !masked(err) {
|
|
return nil
|
|
}
|
|
// An Enso or Zen id falls to its floor, its family's free model (floor.go).
|
|
if to := Floor(sku); to != "" {
|
|
return floorRoutes(to)
|
|
}
|
|
// A third-party model is served as itself or not at all: its vendor's refusal
|
|
// stands. Only a Hanzo SKU, or the pool's own name, is ours to route elsewhere.
|
|
if FamilyOf(sku) == "" && !fam.frontDoor(sku) {
|
|
return nil
|
|
}
|
|
return standIn(fam, sku)
|
|
}
|
|
|
|
// floorRoutes are the routes that serve the floor to (floor.go), in the order a
|
|
// request naming it is served: the floor itself where its family's catalog lists
|
|
// it, whose service walks its own free chain, then the free pool it is a name for.
|
|
func floorRoutes(to string) []spare {
|
|
var out []spare
|
|
if f := familyNamed(FamilyOf(to)); f != nil && f.enabled() {
|
|
if _, own := f.lookup(to); own {
|
|
out = append(out, spare{f, to})
|
|
}
|
|
}
|
|
return append(out, freeRoutes()...)
|
|
}
|
|
|
|
// standIn names the routes that answer for sku when its own vendor does not.
|
|
//
|
|
// A priced route is answered by the model closest to the one asked for: the Enso
|
|
// SKU nearest it, whose service walks every account it holds, and then our own
|
|
// compute. The answer names the model that wrote it (pipeToFamily), so the caller
|
|
// always knows which one did. Any other route falls to the free pool.
|
|
func standIn(fam *modelFamily, sku string) []spare {
|
|
if word, _ := fam.collection(sku); word == collectionDeny {
|
|
// A route bought under `deny` never reaches a borrowed free route, which
|
|
// keeps what it carries: its stand-ins are ours.
|
|
out := standIns(fam, sku)
|
|
if engineFam.enabled() {
|
|
out = append(out, spare{engineFam, engineModel})
|
|
}
|
|
return out
|
|
}
|
|
return freeRoutes()
|
|
}
|
|
|
|
// FreeOnly reports whether the paid lane is off for everyone: it is on only where the
|
|
// zen catalog says `paid`, as zen's own discovery states it (GET ZEN_URL/v1/models,
|
|
// zenFam.open). One line decides for ai's routes and for the family's alike. Spending
|
|
// is an opt-in, so a catalog that is unread, unreachable from the start, or silent on
|
|
// it keeps the lane off. A request asks FreeOnlyFor, which also asks whether its
|
|
// caller was put on the paid lane (lane.go).
|
|
var FreeOnly = func() bool { return !zenFam.open() }
|
|
|
|
// unserved reports that a vendor could not serve a request at all: its account is
|
|
// spent, it answered with its own failure, it was never reached, or it answered
|
|
// 429 after every account it holds was asked.
|
|
func unserved(err error) bool {
|
|
st := upstreamHTTPStatus(err)
|
|
return st == 0 || st == http.StatusTooManyRequests || down(err, strings.ToLower(err.Error()))
|
|
}
|
|
|
|
// frontierOut is the output price, in USD per million tokens, from which a
|
|
// model is answered by enso-pro rather than enso-flash when it stands in.
|
|
const frontierOut = 10.0
|
|
|
|
// standIns are the Enso SKUs that answer a priced route its vendor cannot serve,
|
|
// closest first: enso-pro and then enso-flash for a frontier model, enso-flash
|
|
// for any other.
|
|
func standIns(fam *modelFamily, sku string) []spare {
|
|
if fam == ensoFam || !ensoFam.enabled() {
|
|
return nil
|
|
}
|
|
ids := []string{"enso-flash"}
|
|
if p, ok := fam.modelPrice(sku); ok && p.OutputPerMillion >= frontierOut {
|
|
ids = []string{"enso-pro", "enso-flash"}
|
|
}
|
|
out := make([]spare, 0, len(ids))
|
|
for _, id := range ids {
|
|
out = append(out, spare{ensoFam, id})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// freeLane reports that sku is a route of the free lane: one its family charges
|
|
// nothing for, a name for the pool, or a route out of the pool.
|
|
func freeLane(fam *modelFamily, sku string) bool {
|
|
return fam.frontDoor(sku) || fam.free(sku) || inPool(sku)
|
|
}
|
|
|
|
// inPool reports that a route is one the pool serves, whichever family carries it.
|
|
// It is what prices a pool answer at nothing: every route in the pool is there
|
|
// BECAUSE it costs nothing, so billing one would charge for the thing that makes it
|
|
// a fallback. Read from the pool itself rather than from any one family's list,
|
|
// since the pool now spans families — our own compute among them.
|
|
func inPool(id string) bool {
|
|
// The SNAPSHOT, never a refresh. This is the billing path: discovery is a
|
|
// network round trip with its own timeout, and settling a debit must not wait
|
|
// on a vendor to answer — a slow vendor would otherwise make every debit slow,
|
|
// and an unreachable one would stall the ledger behind a request that already
|
|
// succeeded.
|
|
if freeFamily().isSpare(id) {
|
|
return true
|
|
}
|
|
return engineFam.enabled() && strings.EqualFold(strings.TrimSpace(id), engineModel)
|
|
}
|
|
|
|
// servingFamily returns the family that will actually carry this sku — the free
|
|
// family for a route out of the pool, whichever family claims it otherwise.
|
|
func servingFamily(sku string) *modelFamily {
|
|
if freeFamily().isSpare(sku) {
|
|
return freeFamily()
|
|
}
|
|
f, _ := familyServing(sku)
|
|
return f
|
|
}
|
|
|
|
// familyServing returns the family that owns a model, if any.
|
|
func familyServing(model string) (*modelFamily, bool) {
|
|
for _, f := range modelFamilies {
|
|
if f.serves(model) {
|
|
return f, true
|
|
}
|
|
}
|
|
return nil, false
|
|
}
|
|
|
|
// familyLookup returns the discovered model for an id/alias from whichever family
|
|
// holds it.
|
|
func familyLookup(model string) (zenModel, bool) {
|
|
for _, f := range modelFamilies {
|
|
if m, ok := f.lookup(model); ok {
|
|
return m, true
|
|
}
|
|
}
|
|
return zenModel{}, false
|
|
}
|
|
|
|
// familyLookupFresh is familyLookup that first refreshes each family's catalog on
|
|
// its TTL (a no-op when warm) before reading it. lookup() alone reads the cache
|
|
// with no refresh, so on a replica whose family cache is cold or stale a real SKU
|
|
// reads as unknown — which would make a gated SKU (enso) read as non-gated on a cold
|
|
// replica and skip the access gate. Access/gating decisions must see the same catalog
|
|
// the /v1/models merge does; they go through here (FamilyModelGated).
|
|
func familyLookupFresh(model string) (zenModel, bool) {
|
|
for _, f := range modelFamilies {
|
|
f.fresh()
|
|
if m, ok := f.lookup(model); ok {
|
|
return m, true
|
|
}
|
|
}
|
|
return zenModel{}, false
|
|
}
|
|
|
|
// familyPassthroughRoute routes any family SKU to its family service.
|
|
func familyPassthroughRoute(model string) *modelRoute {
|
|
for _, f := range modelFamilies {
|
|
if r := f.passthroughRoute(model); r != nil {
|
|
return r
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// familyModelPrice returns the discovered headline price for a family model.
|
|
func familyModelPrice(model string) (modelPrice, bool) {
|
|
for _, f := range modelFamilies {
|
|
f.warm()
|
|
if p, ok := f.modelPrice(model); ok {
|
|
return p, true
|
|
}
|
|
}
|
|
return modelPrice{}, false
|
|
}
|
|
|
|
// mergeFamilyModels overlays every configured family's discovered lineup onto the
|
|
// base /v1/models list (each SKU wins over a same-named base entry; new SKUs append).
|
|
func mergeFamilyModels(base []modelInfo) []modelInfo {
|
|
for _, f := range modelFamilies {
|
|
base = f.mergeModels(base)
|
|
}
|
|
return base
|
|
}
|
|
|
|
// ── discovered catalog (values, not places) ──────────────────────────────────
|
|
|
|
// zenTier is one context tier's retail price ($/MTok) as EXACT decimals — the same
|
|
// values the family serves, parsed losslessly (never through float).
|
|
type zenTier struct {
|
|
MaxCtx int
|
|
In decimal.Decimal
|
|
Out decimal.Decimal
|
|
// CacheRead is what one input token served from the upstream's prompt cache
|
|
// bills at. Zero means the family states none, and such a token bills at In.
|
|
CacheRead decimal.Decimal
|
|
// CacheWrite is what one input token written to the upstream's prompt cache
|
|
// bills at, which a vendor may price above In. Zero means the family states none,
|
|
// and such a token bills at In.
|
|
CacheWrite decimal.Decimal
|
|
}
|
|
|
|
// cacheRate is what one cached input token bills at: the stated cache price, else
|
|
// the full input rate. An unpriced cache is never a free cache.
|
|
func (t zenTier) cacheRate() decimal.Decimal {
|
|
if t.CacheRead.Sign() > 0 {
|
|
return t.CacheRead
|
|
}
|
|
return t.In
|
|
}
|
|
|
|
// writeRate is what one input token written to the cache bills at: the stated write
|
|
// price, else the full input rate.
|
|
func (t zenTier) writeRate() decimal.Decimal {
|
|
if t.CacheWrite.Sign() > 0 {
|
|
return t.CacheWrite
|
|
}
|
|
return t.In
|
|
}
|
|
|
|
// tokens is one answer's usage as it is billed: prompt tokens read fresh, prompt
|
|
// tokens served from the upstream's cache, prompt tokens written to it, and
|
|
// completion tokens — of which reasoning is the part the model spent thinking. The
|
|
// three prompt parts are disjoint and sum to the prompt.
|
|
type tokens struct {
|
|
fresh, cached, written, completion, reasoning int
|
|
// reported says the upstream stated what the answer's output cost: a usage
|
|
// object carried its completion. Without it the counts are partial or estimated.
|
|
reported bool
|
|
}
|
|
|
|
// prompt is every prompt token: fresh, read from the cache or written to it.
|
|
func (t tokens) prompt() int { return t.fresh + t.cached + t.written }
|
|
|
|
// zenModel is a discovered SKU: what ai needs to list, route, and bill it — the SKU's
|
|
// public contract (id, context, price ladder, vision). Not its upstream, which the
|
|
// family does not disclose (hip-00NN).
|
|
type zenModel struct {
|
|
ID string
|
|
// Created is when the model was released (Unix seconds), where the family's
|
|
// catalog records it; 0 = not recorded, and the listing's own time is used.
|
|
Created int64
|
|
OwnedBy string
|
|
// Name and Description are what the family's catalog calls the SKU and says it
|
|
// is; empty where it says nothing.
|
|
Name string
|
|
Description string
|
|
MaxCtx int
|
|
// MaxOut is the most one answer may hold, where the catalog states it; 0 is
|
|
// unstated.
|
|
MaxOut int
|
|
// Expires is the day the vendor stops serving the SKU (2006-01-02), where its
|
|
// catalog states one.
|
|
Expires string
|
|
Vision bool
|
|
// Tools and Reasoning are whether the SKU takes tool calls and a reasoning
|
|
// request, as the family's catalog states; false where it states nothing.
|
|
Tools bool
|
|
Reasoning bool
|
|
// Inputs are the kinds of input this SKU takes, as the family advertises them
|
|
// ("text", "image", "audio", "file", "video"). Empty reads as unknown.
|
|
Inputs []string
|
|
// Outputs are the kinds of answer this SKU produces, as the family advertises
|
|
// them ("text", "audio", "image", "embeddings", "rerank"). Empty when the family
|
|
// advertises none, which reads as unknown and never as text.
|
|
Outputs []string
|
|
Access string // "" = generally available; "waitlist" = access-gated (limited preview) — ai enforces the grant
|
|
MinTier string // "" | "free" | "trial" | "paid" — min subscription tier the family advertises for this SKU; ai enforces it (Seams A/B). "" ⇒ free (all tiers). Orthogonal to Access.
|
|
// Funding is how the SKU's usage is PAID FOR upstream: "prepaid" means every path it
|
|
// can take spends a real-cash balance; "" means credits. It is a different KIND of
|
|
// floor from MinTier and is enforced differently — see familyFundingAllowed.
|
|
Funding string
|
|
// Plan is whether the family opens paid plan rungs for this SKU to a host that
|
|
// reads the cost its committed answer states (its listing's "plan").
|
|
Plan bool
|
|
Base zenTier // headline RETAIL price (the in-window tier)
|
|
Tiers []zenTier // full ladder, ascending by MaxCtx — the billing contract
|
|
|
|
// CostIn/CostOut are the upstream COGS ($/MTok) behind Base, when the family
|
|
// discloses what the SKU costs us. Zero means undisclosed, which modelPrice
|
|
// already reads as cost == price (zero margin) — so a family that states only a
|
|
// price behaves exactly as before. A resale family (OpenRouter) states both, and
|
|
// the retail it publishes is this cost times a margin, never below it.
|
|
CostIn decimal.Decimal
|
|
CostOut decimal.Decimal
|
|
|
|
// Margin is the multiple a SKU we resell (OpenRouter's) bills over its vendor's
|
|
// price, 1 + the vendor's fee: its rates are the vendor's times Margin, and a call
|
|
// never bills less than the cost the vendor states for it times Margin (bill).
|
|
// Zero for a family that prices its own SKUs.
|
|
Margin decimal.Decimal
|
|
// Variable is a SKU the vendor prices per call by whatever served it (OpenRouter's
|
|
// routers, which it lists at a price of -1). No rate describes such a call, so it
|
|
// bills at the cost its answer states times Margin, and Base and Tiers hold the
|
|
// ceiling a hold reserves: the dearest SKU it could be routed to.
|
|
Variable bool
|
|
// Card is the vendor's whole price list as it bills to a caller, for a SKU we
|
|
// resell: what /v1/models publishes as the SKU's pricing, exactly.
|
|
Card *rateCard
|
|
// Video is what a video SKU takes and what a clip of it costs (video_spec.go);
|
|
// nil for every other SKU.
|
|
Video *videoSpec
|
|
// Voice is what a speech SKU takes and costs (voice_vendor.go); nil for every
|
|
// other SKU.
|
|
Voice *voiceSpec
|
|
}
|
|
|
|
// rateCard is a resold SKU's price list as billed: every rate its vendor states, in
|
|
// the vendor's names and units, times the markup, and its conditional rates the same.
|
|
type rateCard struct {
|
|
Rates map[string]string
|
|
Overrides []map[string]any
|
|
}
|
|
|
|
// variable reports that this SKU bills each call at the cost its answer states.
|
|
func (m zenModel) variable() bool { return m.Variable }
|
|
|
|
// bill is what one call to this SKU charges, in nano-dollars: its tokens at the tier
|
|
// their prompt reaches. A SKU we resell never bills less than the cost its answer
|
|
// states times its margin: a cache write, a web search or a dearer provider behind
|
|
// the vendor costs what it costs, and billing below cost is the one error resale
|
|
// cannot absorb. A variable SKU bills that stated cost alone, or its tokens at the
|
|
// ceiling its hold reserved when the answer states none. A stated cost of zero or
|
|
// less is no statement, never a credit.
|
|
func (m zenModel) bill(cost *int64, t tokens) int64 {
|
|
retail := m.retailNano(t)
|
|
if m.Margin.Sign() <= 0 || cost == nil || *cost <= 0 {
|
|
return retail
|
|
}
|
|
stated := nanoUp(decimal.New(*cost, 9).Mul(m.Margin))
|
|
if m.Variable {
|
|
return stated
|
|
}
|
|
return max(retail, stated)
|
|
}
|
|
|
|
// retailNano is retailUSD in nano-dollars: the exact charge the hold settles, the
|
|
// ledger debits and the usage row and its span read. Rounded up, so a priced answer
|
|
// never bills nothing; never floored to a cent, so a call worth a thousandth of one
|
|
// bills a thousandth of one.
|
|
func (m zenModel) retailNano(t tokens) int64 {
|
|
return nanoUp(m.retailUSD(t))
|
|
}
|
|
|
|
// unitNano is what units of a media SKU (images, calls) bill at its per-unit price,
|
|
// in nano-dollars, rounded up.
|
|
func (m zenModel) unitNano(units int) int64 {
|
|
return nanoUp(m.Base.In.Mul(decimal.New(int64(units), 0)))
|
|
}
|
|
|
|
// nanoUp is usd in nano-dollars, rounded up; nothing for a negative amount.
|
|
func nanoUp(usd decimal.Decimal) int64 {
|
|
n, _ := usdNanos(usd.String())
|
|
return n
|
|
}
|
|
|
|
// releasedOr is the model's release time, or now when the catalog records none.
|
|
func (m zenModel) releasedOr(now int64) int64 {
|
|
if m.Created > 0 {
|
|
return m.Created
|
|
}
|
|
return now
|
|
}
|
|
|
|
// gated reports whether the family SKU is access-controlled (a limited-preview SKU
|
|
// that is LISTED but callable only with a granted ModelAccess row). ai learns this
|
|
// from discovery (the family's advertised access), never a hardcode.
|
|
func (m zenModel) gated() bool { return m.Access == "waitlist" }
|
|
|
|
// priced reports whether the family charges for this SKU on either side. It is the
|
|
// one reading of "costs money" the listing takes, so premium and free are answers to
|
|
// the same question rather than two opinions of it.
|
|
func (m zenModel) priced() bool {
|
|
return !m.Base.In.IsZero() || !m.Base.Out.IsZero() || m.Video != nil || (m.Voice != nil && m.Variable)
|
|
}
|
|
|
|
// minTier is the SKU's advertised minimum subscription tier on the enso ladder,
|
|
// normalized: "" ⇒ "free" (every tier may use it). One of "free" | "trial" | "paid".
|
|
// ai learns this from discovery (the family's advertised min_tier), never a hardcode;
|
|
// the tier gate (familyTierAllowed, Seams A/B) enforces it.
|
|
func (m zenModel) minTier() string {
|
|
if t := strings.ToLower(strings.TrimSpace(m.MinTier)); t != "" {
|
|
return t
|
|
}
|
|
return "free"
|
|
}
|
|
|
|
// tierFor picks the tier that serves promptTokens: the smallest whose context covers
|
|
// the prompt, else the largest. This is the family's Tier rule (hip-00NN) — cost keys
|
|
// off the SERVED tier, so a 1M-overflow bills at the pricier tier and never underbills.
|
|
func (m zenModel) tierFor(promptTokens int) zenTier {
|
|
for _, t := range m.Tiers {
|
|
if promptTokens <= t.MaxCtx {
|
|
return t
|
|
}
|
|
}
|
|
if len(m.Tiers) > 0 {
|
|
return m.Tiers[len(m.Tiers)-1]
|
|
}
|
|
return m.Base
|
|
}
|
|
|
|
var zenMillion = decimal.New(1_000_000, 0)
|
|
|
|
// retailUSD is what the tokens cost at the tier their prompt reaches, in exact dollars
|
|
// to 18 places — identical to the family's own arithmetic.
|
|
func (m zenModel) retailUSD(u tokens) decimal.Decimal {
|
|
t := m.tierFor(u.prompt())
|
|
in := t.In.Mul(decimal.New(int64(u.fresh), 0))
|
|
out := t.Out.Mul(decimal.New(int64(u.completion), 0))
|
|
read := t.cacheRate().Mul(decimal.New(int64(u.cached), 0))
|
|
write := t.writeRate().Mul(decimal.New(int64(u.written), 0))
|
|
return in.Add(read).Add(write).Add(out).Quo(zenMillion, 18)
|
|
}
|
|
|
|
// centsUp is nano-dollars as the whole cents a hold reserves (the ledger reserves in
|
|
// cents): rounded up, so the hold covers what settles.
|
|
func centsUp(nano int64) int64 {
|
|
if nano <= 0 {
|
|
return 0
|
|
}
|
|
return (nano + 10_000_000 - 1) / 10_000_000
|
|
}
|
|
|
|
// price projects the headline tier into ai's legacy per-model price struct (float,
|
|
// used only for the balance-reservation estimate and the /v1/models display). The
|
|
// exact debit path uses retailNano, never this projection.
|
|
//
|
|
// A discovered SKU always HAS a price, and zero is one of them. "No price" is a
|
|
// different fact — nobody stated one — and only a model discovery never saw can be
|
|
// in that state. Reading zero as absent is what let a route the vendor charges
|
|
// nothing for be listed without a price and gated as though it cost money.
|
|
func (m zenModel) price() (modelPrice, bool) {
|
|
in, _ := strconv.ParseFloat(m.Base.In.String(), 64)
|
|
out, _ := strconv.ParseFloat(m.Base.Out.String(), 64)
|
|
cache, _ := strconv.ParseFloat(m.Base.cacheRate().String(), 64)
|
|
costIn, _ := strconv.ParseFloat(m.CostIn.String(), 64)
|
|
costOut, _ := strconv.ParseFloat(m.CostOut.String(), 64)
|
|
return modelPrice{
|
|
InputPerMillion: in, OutputPerMillion: out, CacheReadPerMillion: cache,
|
|
CostInPerMillion: costIn, CostOutPerMillion: costOut,
|
|
Variable: m.variable(), Card: m.Card,
|
|
}, true
|
|
}
|
|
|
|
// zenWireModel is the /v1/models item shape a family serves. Prices are JSON strings
|
|
// decoded straight into decimal (decimal.UnmarshalJSON), preserving the exact value.
|
|
// There is deliberately no upstream field to read.
|
|
type zenWireModel struct {
|
|
ID string `json:"id"`
|
|
OwnedBy string `json:"owned_by"`
|
|
Description string `json:"description"`
|
|
Mode string `json:"mode"` // the kind of SKU: chat | embedding | rerank | image | audio | video
|
|
Access string `json:"access"` // "" | "waitlist" — access gating advertised by the family
|
|
MinTier string `json:"min_tier"` // "" | "free" | "trial" | "paid" — min subscription tier advertised for this SKU (Seams A/B)
|
|
Funding string `json:"funding"` // "prepaid" = every path this SKU can take spends a real-cash balance
|
|
Plan bool `json:"plan"` // the family opens plan rungs for this SKU to a host that reads its cost trailer
|
|
ContextWindow int `json:"context_window"`
|
|
Pricing struct {
|
|
Input decimal.Decimal `json:"input"`
|
|
Output decimal.Decimal `json:"output"`
|
|
CacheRead decimal.Decimal `json:"cache_read"`
|
|
CacheWrite decimal.Decimal `json:"cache_write"`
|
|
} `json:"pricing"`
|
|
PricingTiers []struct {
|
|
MaxContext int `json:"max_context"`
|
|
Input decimal.Decimal `json:"input"`
|
|
Output decimal.Decimal `json:"output"`
|
|
CacheRead decimal.Decimal `json:"cache_read"`
|
|
CacheWrite decimal.Decimal `json:"cache_write"`
|
|
} `json:"pricing_tiers"`
|
|
Capabilities struct {
|
|
Vision bool `json:"vision"`
|
|
} `json:"capabilities"`
|
|
}
|
|
|
|
func (w zenWireModel) model() zenModel {
|
|
zm := zenModel{
|
|
ID: w.ID, OwnedBy: w.OwnedBy, Description: strings.TrimSpace(w.Description), MaxCtx: w.ContextWindow, Vision: w.Capabilities.Vision,
|
|
Outputs: modeOutputs[strings.ToLower(strings.TrimSpace(w.Mode))],
|
|
Access: w.Access, MinTier: w.MinTier, Funding: w.Funding, Plan: w.Plan,
|
|
Base: zenTier{MaxCtx: w.ContextWindow, In: w.Pricing.Input, Out: w.Pricing.Output, CacheRead: w.Pricing.CacheRead, CacheWrite: w.Pricing.CacheWrite},
|
|
}
|
|
for _, t := range w.PricingTiers {
|
|
zm.Tiers = append(zm.Tiers, zenTier{MaxCtx: t.MaxContext, In: t.Input, Out: t.Output, CacheRead: t.CacheRead, CacheWrite: t.CacheWrite})
|
|
}
|
|
if len(zm.Tiers) == 0 {
|
|
zm.Tiers = []zenTier{zm.Base}
|
|
}
|
|
return zm
|
|
}
|
|
|
|
// modeOutputs is what a family SKU of each mode answers with, in OpenRouter's words:
|
|
// a chat SKU writes text, an embedding SKU returns embeddings, and so on. A mode not
|
|
// named here, or none, states no outputs.
|
|
var modeOutputs = map[string][]string{
|
|
"chat": {"text"},
|
|
"embedding": {"embeddings"},
|
|
"rerank": {"rerank"},
|
|
"image": {"image"},
|
|
"audio": {"audio"},
|
|
"video": {"video"},
|
|
}
|
|
|
|
// decodeCatalog translates a family's raw /v1/models body into discovered SKUs using
|
|
// the family's own dialect, defaulting to the Hanzo family wire shape.
|
|
func (f *modelFamily) decodeCatalog(body []byte) ([]zenModel, error) {
|
|
if f.decode != nil {
|
|
return f.decode(body)
|
|
}
|
|
var out struct {
|
|
Data []zenWireModel `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(body, &out); err != nil {
|
|
return nil, err
|
|
}
|
|
models := make([]zenModel, 0, len(out.Data))
|
|
for _, w := range out.Data {
|
|
models = append(models, w.model())
|
|
}
|
|
return models, nil
|
|
}
|
|
|
|
// catalogHead is what a Hanzo family's /v1/models states beside its SKUs: the other
|
|
// ids it answers to, each naming a SKU it lists ({"claude-zen": "zen6"}), and its
|
|
// catalog's `paid` line. A vendor dialect (decode) states neither.
|
|
type catalogHead struct {
|
|
Aliases map[string]string `json:"aliases"`
|
|
Paid bool `json:"paid"`
|
|
}
|
|
|
|
// head reads the catalog-level fields of a family's /v1/models body: its aliases,
|
|
// lowercase, less any that spells a SKU it lists (the SKU answers for itself), and
|
|
// its paid line. Nothing for a vendor dialect.
|
|
func (f *modelFamily) head(body []byte, byID map[string]zenModel) (map[string]string, bool) {
|
|
var h catalogHead
|
|
if f.decode != nil || json.Unmarshal(body, &h) != nil {
|
|
return nil, false
|
|
}
|
|
named := make(map[string]string, len(h.Aliases))
|
|
for alias, sku := range h.Aliases {
|
|
alias, sku = strings.ToLower(strings.TrimSpace(alias)), strings.ToLower(strings.TrimSpace(sku))
|
|
if _, listed := byID[alias]; alias == "" || sku == "" || listed {
|
|
continue
|
|
}
|
|
named[alias] = sku
|
|
}
|
|
return named, h.Paid
|
|
}
|
|
|
|
const zenCatalogTTL = 5 * time.Minute
|
|
|
|
// engineModel is the id the engine answers to for whatever it currently has loaded.
|
|
// It is an alias on the engine's side, so the pool names the ROLE rather than a
|
|
// checkpoint that changes when the deployment loads a different one.
|
|
const engineModel = "default"
|
|
|
|
// spareTries bounds how many of a vendor's free routes one refused request may be
|
|
// offered to. Three, the same number of attempts the failover loop allows across
|
|
// vendors (maxProviders) and for the same reason: worst-case latency has to be a
|
|
// property of a constant here rather than of a catalogue an operator does not own.
|
|
const spareTries = 3
|
|
|
|
var zenDiscoveryClient = &http.Client{Timeout: listWait, Transport: reaching(&http.Transport{Proxy: http.ProxyFromEnvironment})}
|
|
|
|
// reaching is t with the plugin scheme registered on it: a family or decision
|
|
// service whose address is plugin://<name> is answered in this process by the
|
|
// plugin registered under that name (package plugin), every other address by t.
|
|
// The address decides; the code that relays is the same either way.
|
|
func reaching(t *http.Transport) *http.Transport {
|
|
t.RegisterProtocol(plugin.Scheme, plugin.Transport{Head: t.ResponseHeaderTimeout})
|
|
return t
|
|
}
|
|
|
|
// refresh fetches GET <family>/v1/models and swaps in a fresh snapshot. Best-effort:
|
|
// on any error the previous snapshot stands.
|
|
func (f *modelFamily) refresh() error {
|
|
p := f.provider()
|
|
if p == nil {
|
|
return nil
|
|
}
|
|
read := f.list
|
|
if f.load != nil {
|
|
read = f.load
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), listWait)
|
|
defer cancel()
|
|
body, err := read(ctx, p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
models, err := f.decodeCatalog(body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
models = f.fix(models)
|
|
byID := make(map[string]zenModel, len(models))
|
|
ids := make([]string, 0, len(models))
|
|
for _, m := range models {
|
|
if strings.TrimSpace(m.ID) == "" {
|
|
continue
|
|
}
|
|
byID[strings.ToLower(m.ID)] = m
|
|
ids = append(ids, m.ID)
|
|
}
|
|
var spares []string
|
|
if f.spare != nil {
|
|
spares = f.spare(body)
|
|
}
|
|
named, paid := f.head(body, byID)
|
|
f.mu.Lock()
|
|
f.byID = byID
|
|
f.ids = ids
|
|
f.spares = spares
|
|
f.named = named
|
|
f.paid = paid
|
|
f.fetchedAt = time.Now()
|
|
f.loaded = true
|
|
f.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
// listWait bounds one read of a family's catalog.
|
|
const listWait = 30 * time.Second
|
|
|
|
// list reads the family's GET /v1/models from its address.
|
|
func (f *modelFamily) list(ctx context.Context, p *object.Provider) ([]byte, error) {
|
|
paths := f.lists
|
|
if len(paths) == 0 {
|
|
paths = []string{"/v1/models"}
|
|
}
|
|
if len(paths) == 1 {
|
|
return f.listAt(ctx, p, paths[0])
|
|
}
|
|
// Several lists are one catalog: every entry of each, read whole or not at all,
|
|
// so a list that could not be read never marks its models gone. A list after the
|
|
// first that the vendor does not have (404) lists nothing.
|
|
var all []json.RawMessage
|
|
for i, path := range paths {
|
|
body, err := f.listAt(ctx, p, path)
|
|
if errors.Is(err, errNoList) && i > 0 {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var one struct {
|
|
Data []json.RawMessage `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(body, &one); err != nil {
|
|
return nil, fmt.Errorf("%s %s: %w", f.name, path, err)
|
|
}
|
|
all = append(all, one.Data...)
|
|
}
|
|
return json.Marshal(map[string]any{"data": all})
|
|
}
|
|
|
|
// listAt reads one of the family's lists, GET base + path.
|
|
func (f *modelFamily) listAt(ctx context.Context, p *object.Provider, path string) ([]byte, error) {
|
|
base := strings.TrimRight(strings.TrimSpace(p.ProviderUrl), "/")
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+path, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
upstream.Authorize(req, p)
|
|
resp, err := zenDiscoveryClient.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode == http.StatusNotFound {
|
|
return nil, fmt.Errorf("%s %s: %w", f.name, path, errNoList)
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("%s %s: status %d", f.name, path, resp.StatusCode)
|
|
}
|
|
return io.ReadAll(io.LimitReader(resp.Body, 64<<20))
|
|
}
|
|
|
|
// errNoList is a vendor that has no list at a path.
|
|
var errNoList = errors.New("no such list")
|
|
|
|
// open reports whether the family's catalog, as last discovered, opens the paid lane.
|
|
// It reads the snapshot and never waits on the family: discovery keeps it, loaded at
|
|
// start (WarmFamilies) and again on the TTL wherever the family is read, and a failed
|
|
// refresh keeps the last catalog's line. A catalog never read keeps the lane closed.
|
|
func (f *modelFamily) open() bool {
|
|
f.mu.RLock()
|
|
defer f.mu.RUnlock()
|
|
return f.paid
|
|
}
|
|
|
|
// spends reports whether serving sku can spend the platform's own cash: the family's
|
|
// catalog funds some path of it from a prepaid balance (zenModel.Funding), or does not
|
|
// describe it at all. A SKU the catalog funds from grants and our own compute on every
|
|
// path spends none, whatever it is priced at to the caller.
|
|
func (f *modelFamily) spends(sku string) bool {
|
|
m, ok := f.lookup(sku)
|
|
return !ok || strings.EqualFold(strings.TrimSpace(m.Funding), "prepaid")
|
|
}
|
|
|
|
// spareRoutes are the routes this family serves once its account is spent, best
|
|
// first. Empty for a family that declared none, and for one whose discovery has
|
|
// never succeeded — an unknown vendor gets no fallback invented for it.
|
|
func (f *modelFamily) spareRoutes() []string {
|
|
f.fresh()
|
|
f.mu.RLock()
|
|
defer f.mu.RUnlock()
|
|
return f.spares
|
|
}
|
|
|
|
// free reports that this family serves the named route for nothing. Read from the
|
|
// discovered price, which is the same number the ledger bills, so what a call costs
|
|
// and the terms it is served under are one fact.
|
|
func (f *modelFamily) free(id string) bool {
|
|
m, ok := f.lookup(id)
|
|
return ok && !m.priced()
|
|
}
|
|
|
|
// isSpare reports whether a SKU is one of the free routes rather than a catalog
|
|
// entry. It is what prices a spare at nothing: the vendor charges us nothing for
|
|
// it — that is the whole reason it can answer at all when the account is empty —
|
|
// so billing a customer retail for one would be charging for a downgrade.
|
|
func (f *modelFamily) isSpare(model string) bool {
|
|
model = strings.ToLower(strings.TrimSpace(model))
|
|
f.mu.RLock()
|
|
defer f.mu.RUnlock()
|
|
for _, s := range f.spares {
|
|
if strings.ToLower(s) == model {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// warmRetry is how long a family whose first discovery failed waits before a read
|
|
// asks it again.
|
|
const warmRetry = 10 * time.Second
|
|
|
|
// warm loads the family's catalog when it has never been read. A price asked for
|
|
// before the first discovery is then the discovered one: on a pod that has just
|
|
// started, enso-free reads as the free SKU it is, not as an unknown and so priced id.
|
|
// Concurrent readers wait for the one load; after a failed one they read the empty
|
|
// snapshot until warmRetry has passed.
|
|
func (f *modelFamily) warm() {
|
|
f.mu.RLock()
|
|
loaded := f.loaded
|
|
f.mu.RUnlock()
|
|
if loaded || f.baseURL() == "" {
|
|
return
|
|
}
|
|
f.loading.Lock()
|
|
defer f.loading.Unlock()
|
|
f.mu.RLock()
|
|
loaded = f.loaded
|
|
f.mu.RUnlock()
|
|
if loaded || time.Since(f.tried) < warmRetry {
|
|
return
|
|
}
|
|
f.tried = time.Now()
|
|
if err := f.refresh(); err != nil {
|
|
log.Warning("%s catalog discovery failed: %v", f.name, err)
|
|
}
|
|
}
|
|
|
|
// WarmFamilies reads every family's catalog once, in the background, so a pod that
|
|
// has just started prices family SKUs from discovery on its first request, and
|
|
// starts reading the balance of every account a family spends.
|
|
func WarmFamilies() {
|
|
for _, f := range modelFamilies {
|
|
go f.warm()
|
|
}
|
|
watching.Do(func() {
|
|
for _, f := range modelFamilies {
|
|
if f.credits != nil {
|
|
go f.watchCredits()
|
|
}
|
|
if f.load != nil {
|
|
go f.keepListed()
|
|
}
|
|
for _, n := range f.makers {
|
|
go n.watch()
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
var watching sync.Once
|
|
|
|
// fresh refreshes the snapshot if stale (or never loaded) and reports the address
|
|
// the family resolved to — empty when it is not configured. Errors are logged, not
|
|
// surfaced: a stale snapshot still serves.
|
|
//
|
|
// The address is returned rather than left for the caller to ask for again, because
|
|
// resolving it reads the family's admin row and one read answers both questions.
|
|
func (f *modelFamily) fresh() string {
|
|
url := f.baseURL()
|
|
if url == "" {
|
|
return ""
|
|
}
|
|
f.mu.RLock()
|
|
stale := !f.loaded || time.Since(f.fetchedAt) > zenCatalogTTL
|
|
f.mu.RUnlock()
|
|
if stale {
|
|
if err := f.refresh(); err != nil {
|
|
log.Warning("%s catalog refresh failed: %v", f.name, err)
|
|
}
|
|
}
|
|
return url
|
|
}
|
|
|
|
// lookup returns the discovered model for an id or one of the family's aliases, if
|
|
// any. An alias whose SKU the vendor no longer lists resolves to nothing.
|
|
func (f *modelFamily) lookup(model string) (zenModel, bool) {
|
|
f.mu.RLock()
|
|
defer f.mu.RUnlock()
|
|
m, ok := f.byID[strings.ToLower(strings.TrimSpace(f.alias(model)))]
|
|
return m, ok
|
|
}
|
|
|
|
// sku is the id the family's vendor serves a model under: the SKU an alias names,
|
|
// and the id itself for anything else.
|
|
func (f *modelFamily) sku(model string) string {
|
|
f.mu.RLock()
|
|
defer f.mu.RUnlock()
|
|
return f.alias(model)
|
|
}
|
|
|
|
// alias is sku with the snapshot's lock held: the catalog's own aliases, then the
|
|
// vendor table ai keeps for a catalog that names none.
|
|
func (f *modelFamily) alias(model string) string {
|
|
key := strings.ToLower(strings.TrimSpace(model))
|
|
if id, ok := f.named[key]; ok {
|
|
return id
|
|
}
|
|
if id, ok := f.aliases[key]; ok {
|
|
return id
|
|
}
|
|
return model
|
|
}
|
|
|
|
// snapshot returns the family as last discovered. It does not refresh: fresh is
|
|
// what refreshes, and a caller that wants current data calls it first.
|
|
//
|
|
// The two were one call, and that put a 15-second discovery inside whatever lock
|
|
// the caller held. The catalogue's build holds one, and every family is already
|
|
// refreshed before it is taken, so the refresh in here only ever ran a SECOND
|
|
// time — for a family that is failing, which is the one case where it costs the
|
|
// full timeout and blocks every other reader for it.
|
|
func (f *modelFamily) snapshot() []zenModel {
|
|
f.mu.RLock()
|
|
defer f.mu.RUnlock()
|
|
out := make([]zenModel, 0, len(f.ids))
|
|
for _, id := range f.ids {
|
|
if m, ok := f.byID[strings.ToLower(id)]; ok {
|
|
out = append(out, m)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// serves reports whether the family owns this model. True for any SKU the discovered
|
|
// catalog resolves, and for the public brand prefix so a request routes correctly even
|
|
// before the first discovery completes (cold start). The prefix is the public brand,
|
|
// not a confidential mapping. False when the family is not configured.
|
|
func (f *modelFamily) serves(model string) bool {
|
|
// The family's own free id is served by choosing from the platform pool, so it
|
|
// belongs to this family whether or not its vendor is configured or has ever
|
|
// heard of it — but only while the pool actually holds something to choose.
|
|
// It is asked before the vendor is, because the pool is what answers it and the
|
|
// vendor is not: a brand whose service is unreachable still has a free door.
|
|
if f.frontDoor(model) {
|
|
return len(freeRoutes()) > 0
|
|
}
|
|
// fresh resolves the family's address once and answers "" when it has none, which
|
|
// is exactly !enabled(): one admin-row read where asking both made two.
|
|
if f.fresh() == "" {
|
|
return false
|
|
}
|
|
if _, ok := f.lookup(model); ok {
|
|
return true
|
|
}
|
|
return strings.HasPrefix(strings.ToLower(strings.TrimSpace(model)), f.prefix)
|
|
}
|
|
|
|
// modelPrice returns the discovered headline price for a family model, for the
|
|
// balance-reservation estimate and the model listing. The exact debit uses
|
|
// zenModel.costCents (money/decimal), never this float projection.
|
|
func (f *modelFamily) modelPrice(model string) (modelPrice, bool) {
|
|
// The family's free id is served from the pool, and the pool is the routes that
|
|
// cost nothing. It is a stated price of zero, not an absent one — which is what
|
|
// keeps the balance gate from asking a caller to fund a call that bills nothing.
|
|
if f.frontDoor(model) {
|
|
return modelPrice{}, true
|
|
}
|
|
m, ok := f.lookup(model)
|
|
if !ok {
|
|
return modelPrice{}, false
|
|
}
|
|
return m.price()
|
|
}
|
|
|
|
// passthroughRoute routes any family model to its family provider, forwarding the SKU
|
|
// name verbatim. ai holds no SKU→upstream mapping: the family resolves the SKU, injects
|
|
// identity, folds reasoning, and picks the upstream (hip-00NN).
|
|
func (f *modelFamily) passthroughRoute(model string) *modelRoute {
|
|
if !f.serves(model) {
|
|
return nil
|
|
}
|
|
ctx := 0
|
|
if m, ok := f.lookup(model); ok {
|
|
ctx = m.MaxCtx
|
|
}
|
|
if w := f.window(model); w > 0 {
|
|
ctx = w // guarantee the flagship served window
|
|
}
|
|
p, priced := f.modelPrice(model)
|
|
return &modelRoute{
|
|
providerName: f.name,
|
|
// The family maps the SKU to its real upstream. An alias travels as the SKU
|
|
// it names, because that is the id the vendor serves.
|
|
upstreamModel: f.sku(model),
|
|
// Premium is a claim about money, so the price makes it — the free id and the
|
|
// free routes are not premium, and saying they were would read to a client as
|
|
// a model their plan cannot afford.
|
|
premium: !priced || p.InputPerMillion > 0 || p.OutputPerMillion > 0,
|
|
ownedBy: f.owner, // public branding: enso→"hanzo", zen→"zenlm"
|
|
contextWindow: ctx,
|
|
}
|
|
}
|
|
|
|
// mergeModels overlays this family's DISCOVERED lineup onto the base /v1/models list:
|
|
// a discovered SKU wins over any same-named base entry, new SKUs are appended, and
|
|
// each carries its served context window (from discovery, raised to the guaranteed
|
|
// window for the flagship SKUs).
|
|
//
|
|
// Discovery is the only source of a listing. ai never adds a SKU the family does not
|
|
// serve: a listed model that 404s upstream is worse than an absent one, and it is how
|
|
// enso-pro reached /v1/models with no price at all.
|
|
func (f *modelFamily) mergeModels(base []modelInfo) []modelInfo {
|
|
zs := f.snapshot()
|
|
now := time.Now().Unix()
|
|
// ONE id, ONE entry, on every path through this function. An id already in
|
|
// base is REPLACED rather than appended beside itself: /v1/models is keyed by
|
|
// id, so a second row for one id is not extra information — it is two answers
|
|
// to the same question, and a client building a map keeps whichever it read
|
|
// last.
|
|
//
|
|
// The free ids used to be appended blind on the empty-lineup path below, which
|
|
// was invisible while the pool was empty (freeInfo returns nothing then, so
|
|
// nothing was appended) and became three duplicate rows the moment a route
|
|
// existed — enabling our own compute is exactly what makes one exist.
|
|
idx := make(map[string]int, len(base))
|
|
for i, m := range base {
|
|
idx[strings.ToLower(m.ID)] = i
|
|
}
|
|
upsert := func(info modelInfo) {
|
|
k := strings.ToLower(info.ID)
|
|
if i, ok := idx[k]; ok {
|
|
base[i] = info
|
|
} else {
|
|
idx[k] = len(base)
|
|
base = append(base, info)
|
|
}
|
|
}
|
|
if len(zs) == 0 {
|
|
// A family whose own lineup is empty still publishes its free id: that id is
|
|
// served from the platform pool, not from anything this family discovered.
|
|
for _, info := range f.freeInfo(now) {
|
|
upsert(info)
|
|
}
|
|
sort.Slice(base, func(i, j int) bool { return base[i].ID < base[j].ID })
|
|
return base
|
|
}
|
|
for _, z := range zs {
|
|
owner := z.OwnedBy
|
|
if owner == "" {
|
|
owner = f.owner // public branding default: enso→"hanzo", zen→"zenlm"
|
|
}
|
|
window := z.MaxCtx
|
|
if w := f.window(z.ID); w > 0 {
|
|
window = w // flagship SKUs report their guaranteed served window
|
|
}
|
|
info := modelInfo{
|
|
// Premium is a statement about money, so the price is what makes it:
|
|
// a SKU the family charges for is premium, and one it charges nothing
|
|
// for is not. A free route reported as premium reads to a client as a
|
|
// SKU their plan cannot afford, which is the opposite of true.
|
|
ID: z.ID, Object: "model", Created: z.releasedOr(now), OwnedBy: owner, Premium: z.priced(),
|
|
Name: z.Name, Description: z.Description, ContextWindow: window, MaxOutputTokens: z.MaxOut, Expires: z.Expires,
|
|
Inputs: z.Inputs, Outputs: z.Outputs,
|
|
SupportsVision: z.Vision, SupportsTools: z.Tools, SupportsReasoning: z.Reasoning,
|
|
Video: z.Video, Voice: z.Voice,
|
|
}
|
|
// A gated SKU is LISTED but access-controlled; advertise the default standing
|
|
// ("waitlist"). ListModels upgrades this to the caller's real status when authed.
|
|
if z.gated() {
|
|
info.Access = &modelAccessInfo{State: "waitlist"}
|
|
}
|
|
upsert(info)
|
|
}
|
|
for _, info := range f.freeInfo(now) {
|
|
upsert(info)
|
|
}
|
|
sort.Slice(base, func(i, j int) bool { return base[i].ID < base[j].ID })
|
|
return base
|
|
}
|
|
|
|
// freeInfo is the listing entry for this family's free router, present only while
|
|
// the pool holds a route to serve it with — a listed model that answers nothing is
|
|
// worse than an absent one. Its context is the narrowest in the pool, because that
|
|
// is the prompt every route in the pool can hold, and the widest would promise a
|
|
// window the answer might not have.
|
|
func (f *modelFamily) freeInfo(now int64) []modelInfo {
|
|
names := f.freeNames()
|
|
routes := freeRoutes()
|
|
if len(names) == 0 || len(routes) == 0 {
|
|
return nil
|
|
}
|
|
window := 0
|
|
for _, r := range routes {
|
|
m, ok := r.fam.lookup(r.id)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if window == 0 || (m.MaxCtx > 0 && m.MaxCtx < window) {
|
|
window = m.MaxCtx
|
|
}
|
|
}
|
|
out := make([]modelInfo, 0, len(names))
|
|
for _, n := range names {
|
|
out = append(out, modelInfo{
|
|
ID: n.id, Object: "model", Created: now, OwnedBy: n.owner,
|
|
Premium: false, ContextWindow: window, Outputs: []string{"text"},
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ── the pipe (IO edge) ────────────────────────────────────────────────────────
|
|
|
|
// zenPipeHeader bounds ONE attempt: how long a family may take to say it has begun.
|
|
//
|
|
// It is not a round number chosen for tidiness. On a buffered request the header does
|
|
// not arrive until the whole answer does, so this ceiling is the generation ceiling,
|
|
// and real whole-site builds through this path complete at ~112s — a shorter bound
|
|
// would clip work that is doing exactly what it should. The body is deliberately not
|
|
// bounded at all: a streamed 1M-token answer is long by design, and the caller
|
|
// staying on the line is the only honest measure of whether it is still wanted.
|
|
const zenPipeHeader = 120 * time.Second
|
|
|
|
// commitWait is how long ai waits for a committed answer to begin before a paid rung
|
|
// accepted it: as long as it waits for any family's headers.
|
|
var commitWait = zenPipeHeader
|
|
|
|
// zenPipeClient forwards inference to a family. No client-level timeout: that would
|
|
// count the body, which is the part that is long on purpose.
|
|
var zenPipeClient = &http.Client{
|
|
Transport: reaching(&http.Transport{
|
|
Proxy: http.ProxyFromEnvironment,
|
|
ResponseHeaderTimeout: zenPipeHeader,
|
|
}),
|
|
}
|
|
|
|
var (
|
|
zenNewline = []byte("\n")
|
|
zenDataPrefix = []byte("data:")
|
|
)
|
|
|
|
// withModel returns body with its top-level "model" field set to model, preserving
|
|
// every other field. It reconciles the forwarded body with the model ai resolved: an
|
|
// `auto`/`zen-router` request rewrites request.Model to a concrete SKU while the raw
|
|
// body still says "auto", and the family serves by the body's model field. RawMessage
|
|
// preserves all sibling fields losslessly; on any parse/marshal error the body is
|
|
// returned unchanged so the rewrite never drops a request (the family then surfaces a
|
|
// precise error on the original body).
|
|
func withModel(body []byte, model string) []byte {
|
|
var m map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &m); err != nil || m == nil {
|
|
return body
|
|
}
|
|
mb, err := json.Marshal(model)
|
|
if err != nil {
|
|
return body
|
|
}
|
|
m["model"] = mb
|
|
out, err := json.Marshal(m)
|
|
if err != nil {
|
|
return body
|
|
}
|
|
return out
|
|
}
|
|
|
|
// familyFields are the top-level request fields a family is sent, per API path.
|
|
// Every other field the caller wrote stays behind: a vendor that resells other
|
|
// vendors reads fields that choose other models (models), other endpoints
|
|
// (provider, transforms, route), extra completions (n) and paid tools (plugins,
|
|
// web_search_options), and each would be charged to our account at a price the SKU
|
|
// never quoted. openrouterTerms writes its own provider field after this.
|
|
//
|
|
// The cache fields go on: prompt_cache_key names the conversation, so every turn is
|
|
// sent to the machine that holds its prefix, and cache_control asks for the prompt
|
|
// to be cached (the quote prices it as written).
|
|
var familyFields = map[string]map[string]bool{
|
|
"chat/completions": fieldSet("model", "messages", "stream", "stream_options",
|
|
"max_tokens", "temperature", "top_p", "top_k", "min_p", "top_a",
|
|
"frequency_penalty", "presence_penalty", "repetition_penalty", "seed", "stop",
|
|
"logit_bias", "logprobs", "top_logprobs", "response_format", "tools", "tool_choice",
|
|
"parallel_tool_calls", "user", "reasoning_effort", "reasoning",
|
|
"prompt_cache_key", "cache_control"),
|
|
"messages": fieldSet("model", "messages", "system", "max_tokens", "stream",
|
|
"temperature", "top_p", "top_k", "stop_sequences", "tools", "tool_choice",
|
|
"thinking", "metadata", "cache_control"),
|
|
"embeddings": fieldSet("model", "input", "encoding_format", "dimensions", "user"),
|
|
}
|
|
|
|
func fieldSet(names ...string) map[string]bool {
|
|
m := make(map[string]bool, len(names))
|
|
for _, n := range names {
|
|
m[n] = true
|
|
}
|
|
return m
|
|
}
|
|
|
|
// searchesWeb reports a model id that runs a web search on every call: a vendor's
|
|
// ":online" variant, billed per search on top of the tokens.
|
|
func searchesWeb(model string) bool {
|
|
return strings.HasSuffix(strings.ToLower(strings.TrimSpace(model)), ":online")
|
|
}
|
|
|
|
// familyBody is the body a family is sent for apiPath: the familyFields of the
|
|
// caller's request, with max_tokens set to maxTokens (the ceiling the hold was
|
|
// reserved for) when it is positive. max_completion_tokens is not sent, so the one
|
|
// ceiling is the reserved one. A path with no field list, or a body that is not a
|
|
// JSON object, is sent as it came.
|
|
func familyBody(body []byte, apiPath string, maxTokens int) []byte {
|
|
allowed, ok := familyFields[apiPath]
|
|
if !ok {
|
|
return body
|
|
}
|
|
var in map[string]json.RawMessage
|
|
if json.Unmarshal(body, &in) != nil || in == nil {
|
|
return body
|
|
}
|
|
out := make(map[string]json.RawMessage, len(in))
|
|
for k, v := range in {
|
|
if allowed[k] {
|
|
out[k] = v
|
|
}
|
|
}
|
|
if maxTokens > 0 && allowed["max_tokens"] {
|
|
out["max_tokens"], _ = json.Marshal(maxTokens)
|
|
}
|
|
b, err := json.Marshal(out)
|
|
if err != nil {
|
|
return body
|
|
}
|
|
return b
|
|
}
|
|
|
|
// pipeToFamily forwards the caller's request to the family service verbatim and relays
|
|
// the response back unchanged — the family owns identity, reasoning, upstream, and
|
|
// dialect; ai only authenticates, meters, and forwards. apiPath is "messages"
|
|
// (Anthropic) or "chat/completions" (OpenAI). Billing settles from the response usage
|
|
// at the discovered exact retail price; the settle is idempotent with any deferred
|
|
// fail-safe settle at the call site.
|
|
// It returns nil when the request is finished with — served, or refused for a
|
|
// reason no other vendor would fix, in which case the client already has the
|
|
// answer. It returns the refusal when the FAMILY could not serve it and NOTHING
|
|
// has been written to the client, so the caller is free to offer the same
|
|
// request to the route's declared alternates. That distinction is the whole
|
|
// reason a vendor running out of money stopped being an outage: the family is
|
|
// one provider among several, and a 402 from it is a fact about that vendor's
|
|
// balance, not about the request.
|
|
//
|
|
// The two answers differ in what they owe the reservation, and it follows from
|
|
// what they mean:
|
|
//
|
|
// nil — the request is over, so the hold is RELEASED. Nothing downstream
|
|
// is going to run and settle it.
|
|
// a refusal — the request is still moving, so the hold is UNTOUCHED. settle is
|
|
// one-shot, and releasing it here would leave whatever provider
|
|
// eventually serves this request unbilled.
|
|
//
|
|
// So: returning nil settles, always, via done. The success path has already
|
|
// settled the real cost by then and settle is idempotent, which is what lets one
|
|
// rule cover every ending rather than each ending remembering for itself — which
|
|
// is how a 400 from an embeddings family came to hold a customer's cents with
|
|
// nothing left running that would ever give them back.
|
|
func (c *ApiController) pipeToFamily(fam *modelFamily, apiPath, dialect, model string, rawBody []byte, stream bool, maxTokens int, orgId string, authUser *iam.User, isPremium bool, hold *budgetHold, start time.Time) []attempt {
|
|
// done ends the request here: the client has its answer, or has gone.
|
|
// A covered request's paid upstream: a committed family answer whose cost was never
|
|
// stated — refused, cut, or its caller gone — is charged, once, what its mark owes:
|
|
// its bound once a paid rung accepted it, else nothing.
|
|
// ctx is the request's context, read once: dispatch and pool also run from a
|
|
// stream's writer (awaitCommitted's fill), after fiber has recycled c.
|
|
grant, pm, ctx := grantOf(c.Ctx), markOf(c.Ctx), c.Context()
|
|
place := seatOf(ctx)
|
|
unsettled := func() {
|
|
if grant != nil && pm.settled.CompareAndSwap(false, true) {
|
|
grant.Settle(pm.owed.Load())
|
|
}
|
|
}
|
|
done := func() []attempt {
|
|
hold.settle(0)
|
|
// The request is over: a plan's hold ends here, at nothing more than a usage
|
|
// record already settled.
|
|
unsettled()
|
|
if grant != nil {
|
|
grant.Settle(0)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// sku is the route being asked for. requested is what the CALLER named, and is
|
|
// set only when a different route ends up answering — so it is both the tag on
|
|
// the served generation and the test for whether this was a fallback at all.
|
|
sku, requested := model, ""
|
|
|
|
// to is the floor of the model asked (floor.go): its family's free model, which
|
|
// serves this request when the model cannot, "" for an embedding or a model with
|
|
// none. fell is why it did, "" while the model asked answers; falling is set while
|
|
// its routes are asked, so they carry no plan spend: a fallen request bills nothing.
|
|
to := ""
|
|
if apiPath != "embeddings" {
|
|
to = Floor(model)
|
|
}
|
|
fell, falling := "", false
|
|
|
|
// note records what one refusal says about the account that answered it: the
|
|
// account rests, and somebody is told. Separate from refused because a refusal
|
|
// the request RECOVERS from is still a refusal — an empty account nobody logs
|
|
// is an empty account nobody tops up.
|
|
note := func(err error) attempt {
|
|
cooled.rest(orgId, fam.name, err)
|
|
at := attempt{
|
|
provider: fam.name,
|
|
upstream: sku,
|
|
origin: fam.name,
|
|
status: upstreamHTTPStatus(err),
|
|
fault: faultProvider,
|
|
err: err,
|
|
}
|
|
announce(model, at)
|
|
return at
|
|
}
|
|
|
|
// refused reports a provider-side failure with nothing written: the request
|
|
// is still movable, so hand the reason back rather than spending it on an
|
|
// error page.
|
|
refused := func(err error) []attempt {
|
|
// The caller hung up. There is nobody left to serve, so offering the
|
|
// request to a second vendor would spend money answering an empty room —
|
|
// and would blame a healthy vendor for the client's disconnect.
|
|
if c.Context().Err() != nil {
|
|
return done()
|
|
}
|
|
unsettled()
|
|
return []attempt{note(err)}
|
|
}
|
|
|
|
prov := fam.providerFn()
|
|
if prov == nil {
|
|
// Unconfigured or switched off. From the request's point of view that is
|
|
// a provider that cannot serve, so try the alternates before saying no.
|
|
return refused(&apiError{status: http.StatusServiceUnavailable, msg: fam.name + " service is not configured"})
|
|
}
|
|
// Access gate (the ONE choke point): a gated SKU (enso limited preview) is
|
|
// callable only with a granted ModelAccess row. Discovery tells us which models
|
|
// are gated (zenModel.gated), so this is policy learned from data, not a hardcode.
|
|
// The deferred hold.settle(0) at the call site releases the reservation on this
|
|
// early return, so a denied request is never billed.
|
|
if msg := c.familyRefusal(fam, model, orgId, authUser); msg != "" {
|
|
// OUR gate, not the vendor's: this caller may not have this SKU. No other
|
|
// provider changes that, so answer it here.
|
|
c.zenError(dialect, msg, http.StatusForbidden)
|
|
return done()
|
|
}
|
|
// The price guard (rates.go): a model sold at a fixed price is sent only while
|
|
// no arm of its route costs us more than it bills. Ours too, and no other
|
|
// provider is asked in its place: a route priced above its model is ours to fix,
|
|
// and while it is, a model with a floor is answered there (unpriced, below).
|
|
unpriced := fam.guard(ctx, model)
|
|
if unpriced != nil && to == "" {
|
|
c.zenError(dialect, unpriced.Error(), http.StatusServiceUnavailable)
|
|
return done()
|
|
}
|
|
// Forward the model ai RESOLVED, not the caller's raw model id. An
|
|
// `auto`/`zen-router` request rewrote request.Model to a concrete family SKU, but
|
|
// the raw body still carries "model":"auto" — and the family serves by the body's
|
|
// model field, so it 404s on the unknown id "auto" while a direct request for the
|
|
// same SKU (body model == SKU) serves. Reconcile the forwarded body's model with
|
|
// the resolved SKU; byte-identical for a direct request (model already matches).
|
|
rawBody = withModel(rawBody, model)
|
|
// What a family is sent is the request that was priced and reserved for: the
|
|
// fields that shape the answer, and the token ceiling the hold covers. A field
|
|
// that picks other models, endpoints or paid tools on the vendor's side stays
|
|
// here (familyBody), and so does a web-searching model id.
|
|
if searchesWeb(model) {
|
|
c.zenError(dialect, fmt.Sprintf("%s runs a web search on every call and is not served", model), http.StatusBadRequest)
|
|
return done()
|
|
}
|
|
rawBody = familyBody(rawBody, apiPath, maxTokens)
|
|
// A chat answer the caller wants whole is still asked of the family as a stream,
|
|
// and assembled here: the stream's first frame decides the status exactly as it
|
|
// does for a streaming caller, and while the rest arrives the connection is kept
|
|
// open with whitespace, which a JSON reader skips, so an answer that takes longer
|
|
// than the edge waits for a first byte still arrives (assembleZenStream).
|
|
assemble := !stream && apiPath == "chat/completions" && dialect == "openai"
|
|
if assemble {
|
|
rawBody = streamed(rawBody)
|
|
}
|
|
reqID := uuid.NewString()
|
|
// The family relays whoever it bought the inference from, so the answer leaves
|
|
// in our envelope wearing our id, the SKU asked for, and the seller (see
|
|
// envelope.go). The id is the one this call is already metered under, which is
|
|
// also what the reward join keys on, so the customer, the ledger and the routing
|
|
// event all name this completion the same way.
|
|
//
|
|
// EVERY relayed dialect gets a mark. Only chat had one, so the Anthropic
|
|
// dialect — which is what Claude Code and every agent on that SDK speaks — and
|
|
// the embeddings shape went on publishing the sub-provider's name, `gen-` ids
|
|
// and a finish reason we do not define, long after the chat path stopped. The
|
|
// id shape differs because the dialects do: a message is msg_, a completion is
|
|
// chatcmpl-, and an embeddings list has no id at all.
|
|
mk := &mark{model: model, seller: seller(prov, authUser), routing: routingOf(c.Ctx)}
|
|
switch {
|
|
case dialect == "anthropic":
|
|
mk.speaks, mk.id = messageShape, "msg_"+reqID
|
|
case apiPath == "embeddings":
|
|
mk.speaks = listShape
|
|
default:
|
|
mk.speaks, mk.id = chatShape, "chatcmpl-"+reqID
|
|
}
|
|
url := prov.ProviderUrl + "/v1/" + apiPath
|
|
// One build, one place to fail: the method, the URL and the headers do not vary
|
|
// by SKU, so a request that cannot be built cannot be built for any of them.
|
|
req, err := http.NewRequestWithContext(c.Context(), http.MethodPost, url, nil)
|
|
if err != nil {
|
|
// Our own request is malformed. Another vendor would not fix that.
|
|
c.zenError(dialect, "build "+fam.name+" request: "+err.Error(), http.StatusInternalServerError)
|
|
return done()
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
if a := c.Header("Accept"); a != "" {
|
|
req.Header.Set("Accept", a)
|
|
}
|
|
upstream.Authorize(req, prov)
|
|
// Tenant attribution: the family needs a billable tenant, and ai — which settles
|
|
// the ledger — tells the family it fronts this call so it meters without
|
|
// double-charging. A Hanzo family is also told who pays (dispatch), by which it
|
|
// keeps each payer's share of its free lane; a vendor's API never is.
|
|
var who string
|
|
if org := tenant(ctx, orgId, authUser); org != "" {
|
|
req.Header.Set("X-Org-Id", org)
|
|
who = payerOf(ctx, org, authUser)
|
|
}
|
|
req.Header.Set("X-Hanzo-Fronted-By", "ai")
|
|
// What the caller's plan holds for paid upstream travels to a Hanzo family only:
|
|
// the family may answer from a paid rung whose cost fits, and tells us its rate.
|
|
if spend := spendFor(c.Ctx, fam, model); spend != "" {
|
|
req.Header.Set(spendHeader, spend)
|
|
req.Header.Set("TE", planTE)
|
|
}
|
|
|
|
// fill answers a covered request from the free pool once its committed answer
|
|
// refused inside an answer the caller already holds open (awaitCommitted): the
|
|
// same routes a refusal moves to, with no clock on the walk, since the caller is
|
|
// kept waiting with keep-alives rather than with silence. Set below, after pool.
|
|
var fill func(status int, body []byte) (*http.Response, string)
|
|
fills := func() bool {
|
|
return len(fallback(fam, sku, &apiError{status: http.StatusBadGateway, msg: "unavailable"}, nil)) > 0
|
|
}
|
|
|
|
// The API version and betas a caller in the Anthropic dialect stated, read while
|
|
// the request is still ours: a maker that speaks that dialect is sent them.
|
|
var version string
|
|
var betas []string
|
|
if dialect == "anthropic" {
|
|
version, betas = c.anthropicHeaders()
|
|
}
|
|
|
|
// made is the answer a maker gave, when one did: what says a maker served this
|
|
// request is the response dispatch was handed, never a header on it.
|
|
var made *http.Response
|
|
var madeBy *native
|
|
|
|
// dispatch offers this request to ONE route of ONE family: the family decides the
|
|
// address and the credential, the sku decides the model, and every other part of
|
|
// the request is the caller's own. That is what lets a route on a DIFFERENT
|
|
// vendor answer a request this one cannot — the vendor that ran out of money is
|
|
// rarely the one still holding a free route.
|
|
dispatch := func(f *modelFamily, s string) (*http.Response, error) {
|
|
p := prov
|
|
if f != fam {
|
|
// provider already encodes the one address rule — the admin row where a
|
|
// family has one, configuration otherwise — so a family reached only by
|
|
// configuration (our own compute) resolves the same way as one with a
|
|
// provider row behind it.
|
|
p = f.provider()
|
|
if p == nil {
|
|
return nil, &apiError{status: http.StatusServiceUnavailable, msg: f.name + " service is not configured"}
|
|
}
|
|
}
|
|
r, rErr := http.NewRequestWithContext(ctx, http.MethodPost, p.ProviderUrl+"/v1/"+apiPath, nil)
|
|
if rErr != nil {
|
|
return nil, rErr
|
|
}
|
|
r.Header = req.Header.Clone()
|
|
if f != fam {
|
|
// The credential is the family's own: another family's never travels.
|
|
r.Header.Del("Authorization")
|
|
}
|
|
if f.ours() && who != "" {
|
|
r.Header.Set(payerHeader, who)
|
|
}
|
|
if spend := spendOf(grant, place, pm, f, s); spend != "" && !falling {
|
|
r.Header.Set(spendHeader, spend)
|
|
r.Header.Set("TE", planTE)
|
|
} else {
|
|
r.Header.Del(spendHeader)
|
|
r.Header.Del("TE")
|
|
}
|
|
upstream.Authorize(r, p)
|
|
// The vendor is sent its own id for the SKU. An alias is ours, so it goes
|
|
// upstream as the SKU it names while the answer still wears the alias. A
|
|
// route of the free pool has room to think (room).
|
|
b := withModel(rawBody, f.sku(s))
|
|
if inPool(s) {
|
|
b = room(b)
|
|
}
|
|
// A model its maker serves is asked of the maker first, on every key of its
|
|
// pool, and of this family only once none of them answered (native.go). A
|
|
// maker's API bills every call, so only a request seated on the paid lane
|
|
// reaches it, and only for a route this family charges for: a free route and
|
|
// the pool's never do.
|
|
//
|
|
// And only the route the caller named: a route that stands in for another, or
|
|
// one out of the pool, is never a maker's, whatever it is priced at.
|
|
var missed []miss
|
|
if n := f.maker(s); n != nil && f == fam && s == model && !FreeOnlyFor(ctx) {
|
|
if price, listed := f.lookup(s); listed && price.priced() {
|
|
own, refused := n.send(ctx, nativeCall{path: apiPath, sku: f.sku(s), body: b, stream: stream || assemble,
|
|
version: version, betas: betas, price: price, start: start})
|
|
if own != nil {
|
|
made, madeBy = own, n
|
|
return own, nil
|
|
}
|
|
missed = refused
|
|
}
|
|
}
|
|
if f.terms != nil {
|
|
m, _ := f.lookup(s)
|
|
b = f.terms(b, m, f.free(s), object.GenAIAttributionFromContext(ctx).Session)
|
|
}
|
|
r.Body = io.NopCloser(bytes.NewReader(b))
|
|
r.ContentLength = int64(len(b))
|
|
// GetBody is what http.NewRequest sets for a bytes.Reader and what the
|
|
// transport needs to replay a body across a redirect or an HTTP/2 refusal.
|
|
// Setting the body by hand without it silently drops that.
|
|
r.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(b)), nil }
|
|
// A stream's 200 arrives before its answer; send judges its opening frames
|
|
// on each key (family_open.go).
|
|
resp, err := f.send(r, p, f.free(s), stream || assemble)
|
|
// Each key of the maker's that refused first is on the books beside whatever
|
|
// this family answers, as a family's own failed arms are.
|
|
if resp != nil && len(missed) > 0 {
|
|
resp.Header.Add(failoverHeader, chain(missed))
|
|
}
|
|
// A Hanzo family that committed its answer to a request that may buy a paid
|
|
// rung is the request's one paid attempt, whatever it then says: the request
|
|
// buys no second paid answer. A whole answer is taken as accepted by a paid rung
|
|
// at the commit; a stream says so with its paid marker. A dispatch that carried
|
|
// the spend and got no answer at all — its headers came after the wait, or the
|
|
// connection was cut — buys none either. It owes nothing: a family commits
|
|
// before it asks any rung, so nothing was bought.
|
|
// The commit is judged here, before any byte reaches the caller, so a refusal
|
|
// inside it is as movable as any family's: what the attempt cost is settled
|
|
// now, and the request goes on to the free pool like any other.
|
|
if resp != nil && hanzoFamily(f) && committedPlan(resp) {
|
|
pm.bound.Store(min(boundOf(resp.Header, grant), spendCap(grant, place)))
|
|
pm.tried.Store(true)
|
|
if resp, err = awaitCommitted(resp, waiting{pm: pm, spend: spendCap(grant, place), dialect: dialect, settle: unsettled, fill: fill, fills: fills}); resp == nil || resp.StatusCode != http.StatusOK {
|
|
unsettled()
|
|
}
|
|
}
|
|
if resp == nil && r.Header.Get(spendHeader) != "" {
|
|
pm.asked.Store(true)
|
|
}
|
|
// A family that answered a stream request whole is a whole answer: judged by
|
|
// its status like any buffered one, and relayed as it came.
|
|
if assemble && err == nil && resp != nil && resp.StatusCode == http.StatusOK && !eventStream(resp) {
|
|
return resp, nil
|
|
}
|
|
return resp, err
|
|
}
|
|
|
|
send := func(s string) (*http.Response, error) { return dispatch(fam, s) }
|
|
|
|
// pool walks the platform's free routes in order and reports the one that
|
|
// answered. It is the whole of what a free router does, and the whole of what a
|
|
// refusal falls back to, so both are one walk rather than two that can drift.
|
|
//
|
|
// A SHORT walk, not the whole list. More than one try because a free tier is rate
|
|
// limited by design — the first route is about as likely to answer 429 as to
|
|
// answer — and bounded because every try is a round trip the caller waits
|
|
// through, so worst-case latency is a property of spareTries rather than of how
|
|
// many free routes a vendor happens to advertise.
|
|
// balked is how the last route the pool asked refused, nil when it asked none.
|
|
var balked *balk
|
|
pool := func(routes []spare, skip string, bounded bool) (*http.Response, spare) {
|
|
// The pool holds routes that hold a CONVERSATION. An embeddings request has
|
|
// no free chat route to fall to, and offering it three anyway spends three
|
|
// round trips of the caller's time to be told three times that a chat model
|
|
// does not embed.
|
|
if apiPath == "embeddings" {
|
|
return nil, spare{}
|
|
}
|
|
tried, asked := 0, 0
|
|
balked = nil
|
|
// unreached holds the families a dispatch could not reach: their other
|
|
// routes sit behind the same address, so they are not asked again.
|
|
unreached := map[*modelFamily]bool{}
|
|
for _, alt := range routes {
|
|
if strings.EqualFold(alt.id, skip) || ctx.Err() != nil || unreached[alt.fam] {
|
|
continue
|
|
}
|
|
// A stand-in answers only a caller its own gate admits, and only while
|
|
// its route fits under its price.
|
|
if c.familyRefusal(alt.fam, alt.id, orgId, authUser) != "" || alt.fam.guard(ctx, alt.id) != nil {
|
|
continue
|
|
}
|
|
// A SECOND bound, on the caller's clock rather than on the count.
|
|
// zenPipeHeader bounds one attempt; spareTries bounds how many we
|
|
// borrow; neither bounds their PRODUCT, and the product is what a
|
|
// person waits through — five attempts of two minutes is ten minutes
|
|
// of a request that was expected to take twenty seconds.
|
|
//
|
|
// The walk is worth its cost because a free tier refuses FAST: a 429 is
|
|
// the answer arriving, not the answer being slow, so skipping past one
|
|
// is cheap. Once a request has already spent a whole attempt's worth of
|
|
// time, nothing here is cheap any more and the next try is just a
|
|
// longer wait before the same refusal.
|
|
//
|
|
// It bounds our own route too. spareTries deliberately does not — one
|
|
// more local call is the difference between an answer and a refusal —
|
|
// but that rule is about a SHORT walk, and this only fires where the
|
|
// walk has already stopped being short.
|
|
//
|
|
// A floor is always asked once, however long the model asked took to fail:
|
|
// a request that timed out is exactly the one its floor is for.
|
|
if bounded && (asked > 0 || !falling) && time.Since(start) > zenPipeHeader {
|
|
break
|
|
}
|
|
// The budget bounds how many BORROWED routes we ask, because each is a
|
|
// round trip spent waiting on somebody else. Our own is not borrowed and
|
|
// is the last thing asked, so it is always asked: one more local call is
|
|
// the difference between an answer and a refusal, and it is the only
|
|
// route here that cannot be withdrawn by a vendor.
|
|
// Our own families — our compute, and the Enso SKUs that stand in for a
|
|
// priced route — are not borrowed either.
|
|
borrowed := alt.fam != engineFam && alt.fam != ensoFam
|
|
if borrowed && tried == spareTries {
|
|
continue
|
|
}
|
|
if borrowed {
|
|
tried++
|
|
}
|
|
asked++
|
|
r, e := dispatch(alt.fam, alt.id)
|
|
if e != nil {
|
|
unreached[alt.fam] = true
|
|
balked = &balk{by: alt.fam, err: e}
|
|
continue
|
|
}
|
|
if r.StatusCode == http.StatusOK {
|
|
return r, alt
|
|
}
|
|
b, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
|
|
r.Body.Close()
|
|
balked = &balk{by: alt.fam, err: &apiError{status: r.StatusCode, msg: upstreamErrorMessage(b)}}
|
|
}
|
|
return nil, spare{}
|
|
}
|
|
|
|
fill = func(status int, body []byte) (*http.Response, string) {
|
|
routes := fallback(fam, sku, &apiError{status: status, msg: upstreamErrorMessage(body)}, body)
|
|
if len(routes) == 0 {
|
|
return nil, ""
|
|
}
|
|
r, alt := pool(routes, sku, false)
|
|
return r, alt.id
|
|
}
|
|
|
|
// spared offers the SAME request to a free route once this vendor cannot serve
|
|
// it, and reports the route that answered.
|
|
//
|
|
// The route need not be this vendor's. It used to have to be, which is why the
|
|
// two families whose vendor is reached through a service of their own — enso and
|
|
// zen — had no fallback at all: they declare no free route, so a refusal from
|
|
// either ended the request even while free routes elsewhere were answering.
|
|
//
|
|
// It fires on the vendor being unable to serve at all — its account with us spent,
|
|
// or its own failure — and on nothing else. `down` reads the STATUS first, because
|
|
// a 402 and a 5xx are facts, and consults text only where there is no status to
|
|
// read, and then only a narrow list of phrases measured from real upstreams. A
|
|
// malformed body (400), a model this vendor has not got (404), refused content
|
|
// (422), a rate limit (429) and a timeout all miss it, so none of them can quietly
|
|
// hand the caller a smaller model in place of an error. Our OWN balance gate
|
|
// cannot reach here either — it refuses before a vendor is dialled — so a 402 at
|
|
// this point is a statement about our account with the vendor, never about the
|
|
// customer's.
|
|
//
|
|
// A SHORT walk, not the whole list. More than one try because a free tier is
|
|
// rate limited by design — the first route is about as likely to answer 429 as
|
|
// to answer — and bounded because every try is a round trip the caller waits
|
|
// through, so worst-case latency stays a property of spareTries rather than of
|
|
// how many free routes a vendor happens to advertise.
|
|
// tools is what the floor was asked in place of a tool choice it may not honor
|
|
// (relax), "" when the choice went as sent. stopped is set once the free
|
|
// allowance answered the request instead of its floor.
|
|
tools, stopped := "", false
|
|
|
|
// walk asks the floor's routes for this request, fallen for why: one hop, from
|
|
// the model asked to its floor, and no further. A choice naming one tool goes as
|
|
// "required" over that tool; a floor that refuses the request's shape is asked
|
|
// once more with the choice "auto" and a sentence asking for the call.
|
|
walk := func(routes []spare, why string) (*http.Response, spare) {
|
|
fell, falling = why, true
|
|
if b, did, ok := relax(rawBody, dialect, 1); ok {
|
|
rawBody, tools = b, did
|
|
}
|
|
r, alt := pool(routes, sku, true)
|
|
if r == nil && balked != nil && shaped(balked.err) && ctx.Err() == nil {
|
|
if b, did, ok := relax(rawBody, dialect, 2); ok {
|
|
rawBody, tools = b, did
|
|
r, alt = pool(routes, sku, true)
|
|
}
|
|
}
|
|
return r, alt
|
|
}
|
|
|
|
spared := func(err error, body []byte) (*http.Response, spare, bool) {
|
|
routes := fallback(fam, sku, err, body)
|
|
if len(routes) == 0 || c.Context().Err() != nil {
|
|
return nil, spare{}, false
|
|
}
|
|
if to != "" && strings.EqualFold(sku, model) {
|
|
why := cmp.Or(cause(err), onUpstream)
|
|
// A masked refusal is one a caller's own request can provoke, again and
|
|
// again, so the floor it falls to is held to the free allowance first, as
|
|
// a free model is at the gate: it still falls, and is bounded.
|
|
if why == onMasked && !c.allowance(authUser, dialect) {
|
|
stopped = true
|
|
return nil, spare{}, false
|
|
}
|
|
r, alt := walk(routes, why)
|
|
return r, alt, true
|
|
}
|
|
r, alt := pool(routes, sku, true)
|
|
return r, alt, true
|
|
}
|
|
|
|
// lane is whether the caller is on the free lane: the route they named costs
|
|
// nothing. Read before any route stands in, since a stand-in rewrites sku.
|
|
lane := freeLane(fam, sku)
|
|
|
|
// pooled reports whether a free-lane request the pool could not serve is the
|
|
// POOL's to answer: some account in it is benched by the vendor. Any other
|
|
// failure — a vendor down, nothing discovered, a slow walk — is the request's
|
|
// own and keeps its own path to the route's alternates.
|
|
pooled := func() (Pool, bool) {
|
|
p := FreePool()
|
|
return p, p.State != PoolAvailable
|
|
}
|
|
|
|
// poolRefused answers a free-lane request the pool could not serve: no account
|
|
// the platform shares among free users had room for it. It is the Free plan's
|
|
// own refusal — busy for a minute, or spent until the vendor resets — with the
|
|
// page that lifts it, never a vendor's words about an account the caller has
|
|
// never heard of. A caller whose plan is paid is told to pick a paid model
|
|
// instead: they have nothing to upgrade to.
|
|
poolRefused := func(p Pool) []attempt {
|
|
plan := familyTier(familyAccessSubject(orgId, authUser))
|
|
paid := plan != "" && !strings.EqualFold(plan, "free")
|
|
n := object.PoolRefused(c.Host(), c.billingOrg(authUser), p.State, p.Resets, paid)
|
|
log.Warn("free pool refused %s: state=%s ready=%d/%d", model, p.State, p.Ready, p.Keys)
|
|
if wait := n.RetryAfter(time.Now()); wait > 0 {
|
|
c.SetHeader("Retry-After", strconv.FormatInt(wait, 10))
|
|
}
|
|
// Spent until a reset hours away: a client's automatic retry would only
|
|
// be refused again, so it is told not to.
|
|
if n.Code == object.CodePoolExhausted {
|
|
c.SetHeader("x-should-retry", "false")
|
|
}
|
|
if dialect == "anthropic" {
|
|
c.respondAnthropicError("rate_limit_error", n.Message, n.Status)
|
|
return done()
|
|
}
|
|
c.SetHeader("Content-Type", "application/json")
|
|
_ = c.Bytes(n.Status, n.ErrorJSON())
|
|
return done()
|
|
}
|
|
|
|
// floored serves the request on its floor for why, before the model asked was
|
|
// sent: its route is unpriced, or the paid lane is closed to the caller.
|
|
floored := func(why string) (*http.Response, spare) {
|
|
return walk(floorRoutes(to), why)
|
|
}
|
|
|
|
// floorless answers a request whose floor could not serve it either: the floor's
|
|
// own refusal, with its status, said as the floor's (Fell), and never another
|
|
// route after it. A free pool every key of which the vendor benched is the Free
|
|
// plan's own refusal (poolRefused). With no route of the floor asked at all, the
|
|
// model's own refusal stands (false).
|
|
floorless := func() ([]attempt, bool) {
|
|
if fell == "" || balked == nil || c.Context().Err() != nil {
|
|
return nil, false
|
|
}
|
|
Fell(c.Ctx, model, to, fell)
|
|
if tools != "" {
|
|
c.SetHeader(toolsHeader, tools)
|
|
}
|
|
if p, ok := pooled(); ok {
|
|
return poolRefused(p), true
|
|
}
|
|
status, said := balked.said(to)
|
|
c.zenError(dialect, said, status)
|
|
return done(), true
|
|
}
|
|
|
|
// A free router is served by CHOOSING. The caller named the family's free id,
|
|
// which is not a model any vendor holds — it is this family's name for
|
|
// "whichever free route answers" — so the pool decides, and the answer leaves
|
|
// wearing the id the caller named.
|
|
//
|
|
// requested is set here for the same reason it is set on the fallback below: a
|
|
// different route answered, and the pair (Requested, Model) is what lets the
|
|
// ledger be read against the answer the customer holds. Without it the envelope
|
|
// would say the free id, the row would say the route, and nothing would join them.
|
|
// stood is whether a route stood in for the one named after it could not serve,
|
|
// and by is the family that answered.
|
|
var resp *http.Response
|
|
stood, by := false, fam
|
|
if fam.frontDoor(sku) {
|
|
r, alt := pool(freeRoutes(), "", true)
|
|
if r == nil {
|
|
if c.Context().Err() != nil {
|
|
return done()
|
|
}
|
|
if p, ok := pooled(); ok {
|
|
return poolRefused(p)
|
|
}
|
|
return refused(&apiError{status: http.StatusServiceUnavailable,
|
|
msg: fmt.Sprintf("model %q: no free route answered", model)})
|
|
}
|
|
sku, requested, resp, by = alt.id, model, r, alt.fam
|
|
} else if !lane && FreeOnlyFor(ctx) && FamilyOf(sku) == "" {
|
|
// The paid lane is off and the model named is a third-party one: it is not
|
|
// sent, and nothing answers in its place.
|
|
err := laneOff(ctx, model)
|
|
c.zenError(dialect, err.Error(), statusOf(err))
|
|
return done()
|
|
} else if unpriced != nil || to != "" && !lane && FreeOnlyFor(ctx) {
|
|
// The model asked is not sent: its route costs more than it bills, or the paid
|
|
// lane is closed to the caller. Its floor answers in its place (floor.go).
|
|
why := onClosed
|
|
if unpriced != nil {
|
|
why = onUnpriced
|
|
}
|
|
r, alt := floored(why)
|
|
if r == nil {
|
|
if out, ok := floorless(); ok {
|
|
return out
|
|
}
|
|
if c.Context().Err() != nil {
|
|
return done()
|
|
}
|
|
if unpriced != nil {
|
|
c.zenError(dialect, unpriced.Error(), http.StatusServiceUnavailable)
|
|
return done()
|
|
}
|
|
return refused(&apiError{status: http.StatusServiceUnavailable,
|
|
msg: fmt.Sprintf("model %q: the paid lane is off and no free route answered", model)})
|
|
}
|
|
resp, requested, sku, stood, by = r, model, alt.id, true, alt.fam
|
|
log.Info("family=%s %s: %s served by %s", fam.name, why, model, alt.id)
|
|
} else if !lane && FreeOnlyFor(ctx) && (apiPath != "embeddings" || fam.spends(sku)) {
|
|
// The paid lane is off: a priced Hanzo SKU is never sent to its paid route, and
|
|
// the Hanzo routes that stand in for it answer in its place. An embedding has no
|
|
// stand-in, and one the family funds from grants and our own compute on every
|
|
// path (spends) costs the platform no cash, which is all a closed lane bounds:
|
|
// it is sent as asked.
|
|
r, alt := pool(standIn(fam, sku), sku, true)
|
|
if r == nil {
|
|
if c.Context().Err() != nil {
|
|
return done()
|
|
}
|
|
return refused(&apiError{status: http.StatusServiceUnavailable,
|
|
msg: fmt.Sprintf("model %q: the paid lane is off and no free route answered", model)})
|
|
}
|
|
resp, requested, sku, stood, by = r, model, alt.id, true, alt.fam
|
|
log.Info("family=%s free only: %s served by %s", fam.name, model, alt.id)
|
|
} else if resp, err = send(sku); err != nil {
|
|
// Never reached the family at all. Nothing is written and nothing is
|
|
// billed — deliberately no recordFamilyUsage here, because its
|
|
// hold.settle would release the reservation one-shot and leave the cost
|
|
// of whichever provider ends up serving this request uncharged.
|
|
r, alt, tried := spared(err, nil)
|
|
if stopped {
|
|
note(err)
|
|
return done()
|
|
}
|
|
if r == nil {
|
|
if out, ok := floorless(); ok {
|
|
note(err)
|
|
return out
|
|
}
|
|
if tried && lane && c.Context().Err() == nil {
|
|
if p, ok := pooled(); ok {
|
|
note(err)
|
|
return poolRefused(p)
|
|
}
|
|
}
|
|
return refused(err)
|
|
}
|
|
note(err)
|
|
resp, requested, sku, stood, by = r, model, alt.id, true, alt.fam
|
|
log.Warn("family=%s could not be reached for %s (%v) — served by %s", fam.name, model, err, alt.id)
|
|
}
|
|
// Closed by whoever consumes it. That is this function for an answer read whole,
|
|
// and the stream callback for one that is relayed — the callback outlives this
|
|
// call, so it takes the body with it and this defer stops seeing one.
|
|
defer func() {
|
|
if resp != nil {
|
|
resp.Body.Close()
|
|
}
|
|
}()
|
|
|
|
// READ THE REQUEST WHILE IT IS STILL OURS. A streamed answer settles from
|
|
// inside fasthttp's writer, which runs after this handler returned; fiber has
|
|
// recycled the request by then, so reaching through `c` there is a
|
|
// use-after-free — and it does not fail the one request, it SIGSEGVs the whole
|
|
// plugin process, so every other call in flight dies with it. Measured in
|
|
// production: `DefaultCtx.App()` nil-dereferenced under
|
|
// recordFamilyUsage -> billingOrg -> Ctx.Header, twelve times an hour, each one
|
|
// answering `mount /v1: http: read response: EOF` to whoever was mid-answer.
|
|
//
|
|
// So the three request-derived facts a usage row needs are read HERE, on the
|
|
// request's own goroutine, and travel by value. recordFamilyUsage takes no
|
|
// receiver for the same reason: with no `c` in scope it cannot reach a request
|
|
// at all, which is a compile-time property rather than a rule to remember.
|
|
w := whence{ledger: c.billingOrg(authUser), ip: c.Fiber().IP(), ctx: c.Context(), asked: fam, plan: grantOf(c.Ctx)}
|
|
if fam.prices != nil {
|
|
// The row of a family that sets its own prices says how hard it was asked to
|
|
// think, beside the arm that answered (rates.go).
|
|
w.depth = depthOf(rawBody)
|
|
}
|
|
|
|
// ONE status decision, ahead of the split, because the answer to "can this
|
|
// request still be moved" is the same for both shapes. The status arrives
|
|
// BEFORE any byte of a stream is written — the WriteHeader below is still
|
|
// ahead of us — so a streaming request is exactly as movable here as a
|
|
// buffered one. This is the only window in which that is true, which is why
|
|
// the decision lives at the status and not somewhere down in the relay. A
|
|
// stream's status here is what its opening frames said (dispatch → opening).
|
|
if resp.StatusCode != http.StatusOK {
|
|
b, _ := io.ReadAll(resp.Body)
|
|
status, said := resp.StatusCode, upstreamErrorMessage(b)
|
|
if status == http.StatusUnauthorized && by.ours() {
|
|
// Our own family refusing our credential is OUR outage: the caller's own
|
|
// credential was checked at our edge. It moves like a family that is down,
|
|
// to the free pool, and never reaches the caller as their auth failure.
|
|
barred(by, model, said, time.Now())
|
|
status, said = http.StatusServiceUnavailable, fmt.Sprintf("model %q is temporarily unavailable", model)
|
|
b = nil
|
|
}
|
|
err := &apiError{status: status, msg: said}
|
|
// Each arm the family asked before it gave up is on the books beside
|
|
// whatever answers next, as it would be beside an answer.
|
|
failed(w, by, sku, servingOf(resp.Header).failover, authUser, isPremium, stream, reqID, start)
|
|
r, alt, tried := spared(err, b)
|
|
if stopped {
|
|
note(err)
|
|
return done()
|
|
}
|
|
if r == nil {
|
|
if out, ok := floorless(); ok {
|
|
note(err)
|
|
return out
|
|
}
|
|
if tried && lane && c.Context().Err() == nil {
|
|
if p, ok := pooled(); ok {
|
|
note(err)
|
|
return poolRefused(p)
|
|
}
|
|
}
|
|
if faultOf(err) == faultProvider {
|
|
return refused(err)
|
|
}
|
|
c.zenError(dialect, said, status)
|
|
return done()
|
|
}
|
|
// The vendor could not serve — its account is empty, it is down, or a free
|
|
// route is at its rate limit — and a free route in the pool could. The
|
|
// refusal is recorded anyway — it is why the caller is about to get a
|
|
// different model, and the account it names does not top itself up. The answer leaves wearing the SKU the caller asked for; the route that
|
|
// wrote it is the ledger's Model against that Requested, which is where the
|
|
// substitution is a fact a query can filter on.
|
|
//
|
|
// After the floor above, both routes are served under the same terms — a
|
|
// route under `deny` never arrives here — so this substitution costs the
|
|
// caller nothing it was told about.
|
|
note(err)
|
|
resp.Body.Close()
|
|
resp, requested, sku, stood, by = r, model, alt.id, true, alt.fam
|
|
log.Warn("family=%s refused %s (%d) — served by %s",
|
|
fam.name, model, err.status, alt.id)
|
|
}
|
|
|
|
// A model somebody named from the resale catalog that another model answered
|
|
// is answered in that model's name: the envelope and X-Hanzo-Served say which
|
|
// one wrote it. A Hanzo family's own SKU keeps its name, since what serves it is
|
|
// the family's to say.
|
|
//
|
|
// X-Hanzo-Served names the Hanzo SKU that answered and nothing behind it: the
|
|
// family's own word when it is one of the family's SKUs (a priced SKU answered
|
|
// by its free fallback says so), the stand-in's public name, else the model asked.
|
|
//
|
|
// An answer from the model's floor is the floor's, in its body as in its headers,
|
|
// whether ai moved the request there or the family's own catalog did when every
|
|
// rung of the model refused (floor.go). It bills nothing (whence.fell).
|
|
name := model
|
|
if stood {
|
|
name = publicName(fam, by, sku)
|
|
if fam == freeFamily() || fell != "" {
|
|
mk.model = name
|
|
}
|
|
} else if said := resp.Header.Get(servedHeader); said != "" && fam != freeFamily() {
|
|
if _, ok := fam.lookup(said); ok {
|
|
name = said
|
|
}
|
|
if to != "" && strings.EqualFold(said, to) {
|
|
name, mk.model, fell = to, to, onRoute
|
|
}
|
|
}
|
|
c.SetHeader(servedHeader, name)
|
|
if fell != "" {
|
|
Fell(c.Ctx, model, name, fell)
|
|
if tools != "" {
|
|
c.SetHeader(toolsHeader, tools)
|
|
}
|
|
w.fell = true
|
|
}
|
|
|
|
// The client is told the terms its answer was served under, because a free route
|
|
// is free in exchange for what the vendor keeps and somebody has to be able to
|
|
// say so — in a consent notice, in a log, in a policy page. Set before any byte
|
|
// of a stream goes out, which is the last moment a header can be set at all.
|
|
// The rest of what the family said about this answer is for our records only:
|
|
// the vendor and the failed arms never reach the client.
|
|
sv := servingOf(resp.Header)
|
|
if made != nil && resp == made {
|
|
sv.maker = madeBy
|
|
}
|
|
// answer is the response whose trailer states what a committed answer cost, read
|
|
// once its body has ended.
|
|
answer := resp
|
|
if word, stated := servingFamily(sku).collection(sku); stated {
|
|
c.SetHeader(headerCollection, word)
|
|
}
|
|
|
|
// WHAT AN ANSWER COST IS KNOWN ONLY WHEN IT IS COMPLETE, so both shapes settle
|
|
// here and the streamed one settles from inside its own callback.
|
|
//
|
|
// A stamped answer carries OUR id, so that is the id the client will thread back
|
|
// to /v1/ai/feedback and the id the reward has to join on. An embeddings list has no
|
|
// id of its own, so nothing is stamped and the join falls back to the internal
|
|
// reqID — which is honest: the client was handed no id to correlate on.
|
|
//
|
|
// The learning ledger records this served family call (source="family") and, when
|
|
// the engine endpoint is configured, the shadow A/B pick — off the hot path, no
|
|
// prompt text ever stored (the ledger holds none). The usage row reads only w,
|
|
// taken above while the request was ours.
|
|
|
|
// bill is the hold the answer settles: this request's, until a stream carries
|
|
// it into the writer (carry) so done's settle(0) cannot take it first.
|
|
bill := hold
|
|
settle := func(t tokens, served, respID string, first time.Time) {
|
|
if mk.id != "" {
|
|
respID = mk.id
|
|
}
|
|
if t.prompt() == 0 {
|
|
t.fresh = coarseTokenEstimate(rawBody)
|
|
}
|
|
if sv.arm != "" {
|
|
served = sv.arm
|
|
}
|
|
if !first.IsZero() {
|
|
sv.first = first.Sub(start)
|
|
}
|
|
// A committed answer states its records and its cost once its body ended: its
|
|
// usage record settles it (planCost).
|
|
// One whose cost was never stated settles what its mark owes, here, so the
|
|
// usage record is the one settle of every answer that ended. A paid attempt
|
|
// already settled — refused, and this answer the pool's — charges nothing more.
|
|
//
|
|
// The body is closed first. An answer its caller hung up on was not read to
|
|
// its end, and a maker's states what it cost as far as it got when it is
|
|
// hung up on in turn (native.go): the vendor bills what it wrote either way.
|
|
if answer.Body != nil {
|
|
_ = answer.Body.Close()
|
|
}
|
|
sv.merge(answer.Trailer)
|
|
if owed := pm.owed.Load(); sv.cost == nil && pm.tried.Load() {
|
|
sv.cost = &owed
|
|
}
|
|
if sv.cost != nil && !pm.settled.CompareAndSwap(false, true) {
|
|
sv.cost = nil
|
|
}
|
|
nano := recordFamilyUsage(w, by, sku, requested, prov, mk, authUser, isPremium, stream, reqID, t, sv, start, bill, "success", "")
|
|
c.recordFamilyRouting(model, served, respID, reqID, rawBody, orgId, authUser, t.prompt(), t.completion, nanoToCents(nano), start)
|
|
}
|
|
|
|
if stream {
|
|
c.SetHeader("Content-Type", "text/event-stream")
|
|
c.SetHeader("Cache-Control", "no-cache")
|
|
c.SetHeader("Connection", "keep-alive")
|
|
c.Status(http.StatusOK)
|
|
// THE CALLBACK RUNS AFTER THIS FUNCTION RETURNS. fasthttp produces a streamed
|
|
// body by draining a writer while it serialises the response, so the relay has
|
|
// not started when SendStreamWriter comes back — measured, not assumed.
|
|
//
|
|
// Two things follow. Everything the stream READS must outlive this call, so the
|
|
// upstream body travels into the callback and is closed there; left to the defer
|
|
// above it would be closed first and the client would be sent nothing.
|
|
//
|
|
// And everything the stream LEARNS has to be used in here. Settled outside,
|
|
// prompt and completion are still zero, so every streamed answer would be
|
|
// charged a token estimate for its prompt and nothing at all for its completion.
|
|
// The context is still the request's: fasthttp finishes writing the response
|
|
// before fiber releases it.
|
|
//
|
|
// So the stream owns the hold. It settles with what the stream cost, and at zero
|
|
// when it ended before a cost could be read; releasing it here would win the one
|
|
// settle first and leave the ledger blind to the answer. The caller hands its own
|
|
// release over for the same reason.
|
|
upstream := resp.Body
|
|
resp = nil
|
|
bill = hand(hold)
|
|
_ = c.SendStreamWriter(func(w *bufio.Writer) {
|
|
defer bill.settle(0)
|
|
defer upstream.Close()
|
|
defer hold.settle(0)
|
|
defer unsettled()
|
|
settle(relayZenStream(w, upstream, mk, pm.paid))
|
|
})
|
|
return nil
|
|
}
|
|
|
|
if assemble && eventStream(resp) {
|
|
c.SetHeader("Content-Type", "application/json")
|
|
c.Status(http.StatusOK)
|
|
upstream := resp.Body
|
|
resp = nil
|
|
bill = hand(hold)
|
|
// The answer is whole, so it takes the request's whole shape (a Responses
|
|
// object for /v1/responses), never its stream's.
|
|
shape := c.answer.shape
|
|
_ = c.sendWhole(func(w *bufio.Writer) {
|
|
defer bill.settle(0)
|
|
defer upstream.Close()
|
|
defer hold.settle(0)
|
|
defer unsettled()
|
|
settle(assembleZenStream(w, upstream, mk, heartbeat, pm.paid, shape))
|
|
})
|
|
return nil
|
|
}
|
|
|
|
b, rErr := io.ReadAll(resp.Body)
|
|
if rErr != nil {
|
|
// The family's answer was truncated on the way to us. Nothing has reached the
|
|
// client, so this is still movable.
|
|
return refused(rErr)
|
|
}
|
|
// A buffered answer's first token arrives with the rest of it.
|
|
read := time.Now()
|
|
var t tokens
|
|
sniffZenUsage(b, &t)
|
|
ct := resp.Header.Get("Content-Type")
|
|
if ct == "" {
|
|
ct = "application/json"
|
|
}
|
|
c.SetHeader("Content-Type", ct)
|
|
c.answerBody(mk.stamp(b))
|
|
settle(t, sniffZenModel(b), sniffZenId(b), read)
|
|
return done()
|
|
}
|
|
|
|
// servedHeader names the arm that answered, set by the family and relayed as is.
|
|
const servedHeader = "X-Hanzo-Served"
|
|
|
|
// The vendor that ran the arm and the arms that failed before it, set by a family
|
|
// for the gateway fronting it. Read into the usage record, never relayed.
|
|
//
|
|
// A failed arm is `<upstream> (<provider>): <reason>`, one per header field or
|
|
// several joined by "; " in one; each becomes a "failover" row beside the answer
|
|
// (failed).
|
|
const (
|
|
providerHeader = "X-Hanzo-Provider"
|
|
failoverHeader = "X-Hanzo-Failover"
|
|
// freeHeader is set when the rung that answered bills nothing.
|
|
freeHeader = "X-Hanzo-Free"
|
|
// cogsHeader is what the request cost Hanzo upstream, in USD: every arm the
|
|
// family ran for it, the one that answered and each that failed. A header on a
|
|
// whole answer, a trailer on a stream (its cost is known at its end). It is the
|
|
// usage row's COGS; the family's answer body never carries ours.
|
|
cogsHeader = "X-Hanzo-Cogs"
|
|
// keyHeader names the vendor account the arm that answered ran on, by its name
|
|
// (OPENROUTER_API_KEY_3), never its key: the free view's which-account
|
|
// (listing.go). A header, or a trailer on a stream.
|
|
keyHeader = "X-Hanzo-Key"
|
|
)
|
|
|
|
// serving is what a family said about an answer beyond its body: the arm that
|
|
// wrote it, the vendor that ran that arm, the arms that failed first, the time to
|
|
// its first token, and whether that arm bills nothing.
|
|
type serving struct {
|
|
arm, vendor, failover, account string
|
|
first time.Duration
|
|
free bool
|
|
// cost is what a committed answer's paid upstream cost, nano-dollars, as its
|
|
// family stated it at the end (costHeader); nil when none was stated.
|
|
cost *int64
|
|
// cogs is what the request cost upstream, nano-dollars (cogsHeader); nil when
|
|
// the family stated none.
|
|
cogs *int64
|
|
// maker is the vendor's own API that gave this answer (native.go), nil for
|
|
// every answer a family's address gave. The pipe sets it from the response it
|
|
// was handed, so no header an address sends can claim it.
|
|
maker *native
|
|
}
|
|
|
|
// merge reads what a family stated in its trailer, at an answer's end, over what its
|
|
// headers said.
|
|
func (sv *serving) merge(tr http.Header) {
|
|
if len(tr) == 0 {
|
|
return
|
|
}
|
|
for dst, k := range map[*string]string{&sv.arm: armHeader, &sv.vendor: providerHeader, &sv.account: keyHeader} {
|
|
if v := tr.Get(k); v != "" {
|
|
*dst = v
|
|
}
|
|
}
|
|
if v := strings.Join(tr.Values(failoverHeader), "; "); v != "" {
|
|
sv.failover = v
|
|
}
|
|
if sv.arm == "" {
|
|
sv.arm = tr.Get(servedHeader)
|
|
}
|
|
if tr.Get(freeHeader) == "true" {
|
|
sv.free = true
|
|
}
|
|
if n, ok := usdNanos(tr.Get(costHeader)); ok {
|
|
sv.cost = &n
|
|
}
|
|
if n, ok := usdNanos(tr.Get(cogsHeader)); ok {
|
|
sv.cogs = &n
|
|
}
|
|
}
|
|
|
|
// servingOf reads a family response's headers into a serving.
|
|
func servingOf(h http.Header) serving {
|
|
arm := h.Get(armHeader)
|
|
if arm == "" {
|
|
arm = h.Get(servedHeader)
|
|
}
|
|
sv := serving{arm: arm, vendor: h.Get(providerHeader), account: h.Get(keyHeader), failover: strings.Join(h.Values(failoverHeader), "; "), free: h.Get(freeHeader) == "true"}
|
|
if n, ok := usdNanos(h.Get(cogsHeader)); ok {
|
|
sv.cogs = &n
|
|
}
|
|
return sv
|
|
}
|
|
|
|
// miss is one arm a family asked and could not use: the upstream model, the vendor
|
|
// that ran it, and why.
|
|
type miss struct{ upstream, provider, reason string }
|
|
|
|
// arms reads a failover chain (failoverHeader) into its arms. An entry opens with
|
|
// `<upstream> (<provider>): `; text that does not is the previous entry's reason
|
|
// carrying a "; " of its own.
|
|
func arms(chain string) []miss {
|
|
var out []miss
|
|
for _, part := range strings.Split(chain, "; ") {
|
|
if strings.TrimSpace(part) == "" {
|
|
continue
|
|
}
|
|
head, reason, ok := strings.Cut(part, "): ")
|
|
upstream, provider, paren := strings.Cut(head, " (")
|
|
switch {
|
|
case ok && paren && upstream != "" && !strings.Contains(upstream, " ") && !strings.Contains(provider, " "):
|
|
out = append(out, miss{upstream: upstream, provider: provider, reason: reason})
|
|
case len(out) > 0:
|
|
out[len(out)-1].reason += "; " + part
|
|
default:
|
|
out = append(out, miss{reason: part})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// failed files each arm in chain as a "failover" row beside the request reqID: the
|
|
// SKU asked of fam, the arm and its vendor, and why, billed nothing. What the arm cost
|
|
// upstream is in the request's COGS (cogsHeader), on the row of whatever answered.
|
|
func failed(w whence, fam *modelFamily, sku, chain string, u *iam.User, premium, stream bool, reqID string, start time.Time) {
|
|
for _, a := range arms(chain) {
|
|
if f := listedFamily(a.provider); f != nil && f.free(a.upstream) {
|
|
f.served(a.upstream, "", false)
|
|
}
|
|
}
|
|
for _, rec := range failoverRows(w, fam, sku, chain, u, premium, stream, reqID) {
|
|
recordTrace(w.ctx, rec, start)
|
|
}
|
|
}
|
|
|
|
// failoverRows are the rows failed files, one an arm, each bound to its caller.
|
|
// Nothing for a call with no caller to file under.
|
|
func failoverRows(w whence, fam *modelFamily, sku, chain string, u *iam.User, premium, stream bool, reqID string) []*usageRecord {
|
|
if u == nil {
|
|
return nil
|
|
}
|
|
var out []*usageRecord
|
|
for _, a := range arms(chain) {
|
|
msg := a.reason
|
|
if a.upstream != "" {
|
|
msg = a.upstream + ": " + a.reason
|
|
}
|
|
rec := &usageRecord{
|
|
Owner: w.ledger, Organization: u.Owner,
|
|
Model: sku, Provider: fam.name, Served: a.upstream, Vendor: a.provider,
|
|
Premium: premium, Stream: stream, Status: "failover", ErrorMsg: msg,
|
|
ClientIP: w.ip, RequestID: reqID, Account: "hanzo",
|
|
}
|
|
rec.bind(w.ctx, u)
|
|
out = append(out, rec)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// balk is how one route of a floor refused: the family that carried it and its
|
|
// error.
|
|
type balk struct {
|
|
by *modelFamily
|
|
err error
|
|
}
|
|
|
|
// said is the floor to's refusal as the caller reads it: the status the route
|
|
// answered, and its sentence when the route is one of our own services, which
|
|
// speak in our voice. A borrowed route's sentence is its vendor's, so the floor is
|
|
// named instead, by what its status means.
|
|
func (b *balk) said(to string) (int, string) {
|
|
status := upstreamHTTPStatus(b.err)
|
|
if status == 0 {
|
|
status = http.StatusServiceUnavailable
|
|
}
|
|
switch {
|
|
case b.by.ours():
|
|
return status, b.err.Error()
|
|
case status == http.StatusTooManyRequests:
|
|
return status, to + " is busy; try again shortly"
|
|
}
|
|
return status, to + " is temporarily unavailable"
|
|
}
|
|
|
|
// armHeader is the upstream model a family says answered, for our records only.
|
|
const armHeader = "X-Hanzo-Arm"
|
|
|
|
// publicName is the Hanzo name an answer from route sku of family by goes by: an
|
|
// Enso or Zen SKU is its own name; any other route — a borrowed free model, our own
|
|
// compute — is the free tier of the family that was asked, and Enso's for the rest.
|
|
func publicName(asked, by *modelFamily, sku string) string {
|
|
if by == ensoFam || by == zenFam {
|
|
return sku
|
|
}
|
|
if asked != nil && asked.freeName != "" {
|
|
return asked.freeName
|
|
}
|
|
return ensoFam.freeName
|
|
}
|
|
|
|
// eventStream reports whether resp is a server-sent event stream.
|
|
func eventStream(resp *http.Response) bool {
|
|
return resp != nil && strings.HasPrefix(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream")
|
|
}
|
|
|
|
// heartbeat is how often a whole answer being assembled writes a space to keep its
|
|
// connection open.
|
|
var heartbeat = 15 * time.Second
|
|
|
|
// streamed is a chat body asking for a stream that reports its usage.
|
|
func streamed(body []byte) []byte {
|
|
var in map[string]json.RawMessage
|
|
if json.Unmarshal(body, &in) != nil || in == nil {
|
|
return body
|
|
}
|
|
in["stream"] = json.RawMessage("true")
|
|
in["stream_options"] = json.RawMessage(`{"include_usage":true}`)
|
|
out, err := json.Marshal(in)
|
|
if err != nil {
|
|
return body
|
|
}
|
|
return out
|
|
}
|
|
|
|
// assembleZenStream reads a family's chat stream to its end and writes it as one
|
|
// chat completion, stamped by mk and put in the request's dialect by shape (nil: as
|
|
// it is). Until the answer is whole it writes a space every beat, which a JSON reader
|
|
// skips as leading whitespace. An error frame after the first becomes the answer's
|
|
// error object, which every dialect reads alike.
|
|
//
|
|
// A beat that cannot be written means the client is gone: body is closed then, which
|
|
// hangs up on the family so its model stops, and what was read so far is returned.
|
|
// An answer whose usage never arrived — its client left, or the family's stream
|
|
// broke off or ran out its time before stating it — is billed for what it said: its
|
|
// text, reasoning and tool arguments, about four characters a token.
|
|
func assembleZenStream(w *bufio.Writer, body io.ReadCloser, mk *mark, beat time.Duration, paid func(), shape func([]byte) ([]byte, error)) (t tokens, served, respID string, first time.Time) {
|
|
type call struct {
|
|
ID string `json:"id,omitempty"`
|
|
Type string `json:"type"`
|
|
Function struct {
|
|
Name string `json:"name"`
|
|
Arguments string `json:"arguments"`
|
|
} `json:"function"`
|
|
}
|
|
var (
|
|
content, reasoning strings.Builder
|
|
calls = map[int]*call{}
|
|
order []int
|
|
finish string
|
|
created int64
|
|
usage json.RawMessage
|
|
failure json.RawMessage
|
|
)
|
|
done := make(chan struct{})
|
|
// Runs once the reader below is done: every return waits on done first.
|
|
defer func() {
|
|
said := content.Len() + reasoning.Len()
|
|
for _, c := range calls {
|
|
said += len(c.Function.Arguments)
|
|
}
|
|
if !t.reported && said > 0 {
|
|
t.completion = said/4 + 1
|
|
}
|
|
}()
|
|
go func() {
|
|
defer close(done)
|
|
sc := bufio.NewScanner(body)
|
|
sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
|
for sc.Scan() {
|
|
line := sc.Bytes()
|
|
if paid != nil && string(bytes.TrimSpace(line)) == paidMarker {
|
|
paid()
|
|
}
|
|
if !bytes.HasPrefix(line, zenDataPrefix) {
|
|
continue
|
|
}
|
|
payload := bytes.TrimSpace(line[len(zenDataPrefix):])
|
|
if len(payload) == 0 || payload[0] != '{' {
|
|
continue
|
|
}
|
|
if first.IsZero() {
|
|
first = time.Now()
|
|
}
|
|
sniffZenUsage(payload, &t)
|
|
if served == "" {
|
|
served = sniffZenModel(payload)
|
|
}
|
|
if respID == "" {
|
|
respID = sniffZenId(payload)
|
|
}
|
|
var f struct {
|
|
Created int64 `json:"created"`
|
|
Usage json.RawMessage `json:"usage"`
|
|
Error json.RawMessage `json:"error"`
|
|
Choices []struct {
|
|
Delta struct {
|
|
Content string `json:"content"`
|
|
Reasoning string `json:"reasoning"`
|
|
ReasoningContent string `json:"reasoning_content"`
|
|
ToolCalls []struct {
|
|
Index int `json:"index"`
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
Function struct {
|
|
Name string `json:"name"`
|
|
Arguments string `json:"arguments"`
|
|
} `json:"function"`
|
|
} `json:"tool_calls"`
|
|
} `json:"delta"`
|
|
Finish string `json:"finish_reason"`
|
|
} `json:"choices"`
|
|
}
|
|
if json.Unmarshal(payload, &f) != nil {
|
|
continue
|
|
}
|
|
if len(f.Error) > 0 && string(f.Error) != "null" {
|
|
failure = f.Error
|
|
}
|
|
if f.Created != 0 && created == 0 {
|
|
created = f.Created
|
|
}
|
|
if len(f.Usage) > 0 && string(f.Usage) != "null" {
|
|
usage = f.Usage
|
|
}
|
|
for _, ch := range f.Choices {
|
|
content.WriteString(ch.Delta.Content)
|
|
reasoning.WriteString(ch.Delta.ReasoningContent)
|
|
reasoning.WriteString(ch.Delta.Reasoning)
|
|
for _, tc := range ch.Delta.ToolCalls {
|
|
c := calls[tc.Index]
|
|
if c == nil {
|
|
c = &call{Type: "function"}
|
|
calls[tc.Index] = c
|
|
order = append(order, tc.Index)
|
|
}
|
|
if tc.ID != "" {
|
|
c.ID = tc.ID
|
|
}
|
|
if tc.Type != "" {
|
|
c.Type = tc.Type
|
|
}
|
|
c.Function.Name += tc.Function.Name
|
|
c.Function.Arguments += tc.Function.Arguments
|
|
}
|
|
if ch.Finish != "" {
|
|
finish = ch.Finish
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
tick := time.NewTicker(beat)
|
|
defer tick.Stop()
|
|
wait:
|
|
for {
|
|
select {
|
|
case <-done:
|
|
break wait
|
|
case <-tick.C:
|
|
_, _ = w.WriteString(" ")
|
|
if w.Flush() != nil {
|
|
_ = body.Close()
|
|
<-done
|
|
return
|
|
}
|
|
}
|
|
}
|
|
var out []byte
|
|
if failure != nil && content.Len() == 0 && len(calls) == 0 {
|
|
out, _ = json.Marshal(map[string]json.RawMessage{"error": failure})
|
|
_, _ = w.Write(out)
|
|
_ = w.Flush()
|
|
return
|
|
}
|
|
msg := map[string]any{"role": "assistant", "content": content.String()}
|
|
if reasoning.Len() > 0 {
|
|
msg["reasoning_content"] = reasoning.String()
|
|
}
|
|
if len(order) > 0 {
|
|
list := make([]*call, 0, len(order))
|
|
for _, i := range order {
|
|
list = append(list, calls[i])
|
|
}
|
|
msg["tool_calls"] = list
|
|
if content.Len() == 0 {
|
|
msg["content"] = nil
|
|
}
|
|
}
|
|
// An answer that says nothing is never a finished one: a model that spent its
|
|
// whole budget thinking (reasoning, no content, no call) stopped for length, and
|
|
// says so, so an agent reads a cut answer rather than an empty success.
|
|
if finish == "" || finish == "stop" && content.Len() == 0 && len(order) == 0 {
|
|
finish = "stop"
|
|
if content.Len() == 0 && len(order) == 0 {
|
|
finish = "length"
|
|
}
|
|
}
|
|
whole := map[string]any{
|
|
"id": respID, "object": "chat.completion", "created": created, "model": served,
|
|
"choices": []map[string]any{{"index": 0, "message": msg, "finish_reason": finish}},
|
|
}
|
|
if usage != nil {
|
|
whole["usage"] = usage
|
|
}
|
|
out, _ = json.Marshal(whole)
|
|
if mk != nil {
|
|
out = mk.stamp(out)
|
|
}
|
|
if shape != nil {
|
|
shaped, err := shape(out)
|
|
if err != nil {
|
|
shaped, _ = json.Marshal(map[string]any{"error": map[string]string{"message": err.Error(), "type": "server_error"}})
|
|
}
|
|
out = shaped
|
|
}
|
|
_, _ = w.Write(out)
|
|
_ = w.Flush()
|
|
return
|
|
}
|
|
|
|
// barredSaid is when each family's refusal of our key was last logged, in Unix seconds.
|
|
var barredSaid sync.Map // family name → *atomic.Int64
|
|
|
|
// barred files one refusal of our credential by our own family fam: every one counts
|
|
// in cloud_supply_refused{provider=fam, reason=credential}, which an alert reads, and
|
|
// a storm of them is one error line a minute per family, naming what to check.
|
|
// Whether this one was logged.
|
|
func barred(fam *modelFamily, model, said string, now time.Time) bool {
|
|
supplyRefused.WithLabelValues(fam.name, reasonCredential).Inc()
|
|
v, _ := barredSaid.LoadOrStore(fam.name, new(atomic.Int64))
|
|
last := v.(*atomic.Int64)
|
|
prev := last.Load()
|
|
if prev != 0 && now.Unix()-prev < 60 || !last.CompareAndSwap(prev, now.Unix()) {
|
|
return false
|
|
}
|
|
log.Error("family=%s refused ai's credential for %s (401: %s): is %s at KMS hanzo/prod:/ai, has cloud rolled since, and does no admin provider row named %q carry a secret of its own? Every refusal counts in cloud_supply_refused{reason=credential}; this line repeats at most once a minute.",
|
|
fam.name, model, said, fam.keyKey, fam.name)
|
|
return true
|
|
}
|
|
|
|
// relayZenStream copies a family's SSE response to the client and captures the final
|
|
// usage for billing. The family already emits correct dialect SSE, so ai does not
|
|
// translate — it reads usage as it passes and, for the one dialect whose envelope is
|
|
// ours (a non-nil mark), stamps each event before it goes out. Every chunk of one
|
|
// completion is stamped from the same mark, so the id a client correlates on holds
|
|
// for the whole stream. first is when the first data chunk was written.
|
|
//
|
|
// A write that fails ends the relay: the client is gone, and the caller's close of
|
|
// body is what hangs up on the family, so its model stops generating rather than
|
|
// running to max_tokens for nobody. The request's context cannot say this — a
|
|
// fasthttp request's context does not end when its client leaves — so the failed
|
|
// write is the only signal there is. The usage a family states arrives last, so an
|
|
// answer whose usage never arrived — its client left, or the family's stream broke
|
|
// off or ran out its time before stating it — is billed for the text relayed.
|
|
//
|
|
// A stream that ends without its last frame ([DONE], message_stop, or an error the
|
|
// family sent) broke off: bytes are on the wire, so it is never asked again of
|
|
// another model. It ends with an error event in its dialect (cutStream), so a
|
|
// client reads a broken answer as broken rather than as complete.
|
|
func relayZenStream(w *bufio.Writer, body io.Reader, mk *mark, paid func()) (t tokens, served, respID string, first time.Time) {
|
|
said := 0 // characters of answer relayed
|
|
ended := false // the stream's last frame was relayed
|
|
defer func() {
|
|
if !t.reported && said > 0 {
|
|
t.completion = said/4 + 1
|
|
}
|
|
}()
|
|
sc := bufio.NewScanner(body)
|
|
sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
|
for sc.Scan() {
|
|
line := sc.Bytes()
|
|
if paid != nil && string(bytes.TrimSpace(line)) == paidMarker {
|
|
paid()
|
|
}
|
|
// An SSE comment is a vendor keeping its connection open in its own words
|
|
// (": OPENROUTER PROCESSING"). The keep-alive goes on; the words do not.
|
|
if bytes.HasPrefix(line, []byte(":")) {
|
|
line = []byte(":")
|
|
}
|
|
ended = ended || closing(line)
|
|
if bytes.HasPrefix(line, zenDataPrefix) {
|
|
payload := bytes.TrimSpace(line[len(zenDataPrefix):])
|
|
if len(payload) > 0 && payload[0] == '{' {
|
|
sniffZenUsage(payload, &t)
|
|
said += answerChars(payload)
|
|
if served == "" {
|
|
served = sniffZenModel(payload)
|
|
}
|
|
if respID == "" {
|
|
respID = sniffZenId(payload)
|
|
}
|
|
if mk != nil {
|
|
line = append([]byte("data: "), mk.stamp(payload)...)
|
|
}
|
|
}
|
|
}
|
|
_, _ = w.Write(line)
|
|
_, _ = w.Write(zenNewline)
|
|
if w.Flush() != nil {
|
|
return
|
|
}
|
|
if first.IsZero() && bytes.HasPrefix(line, zenDataPrefix) {
|
|
first = time.Now()
|
|
}
|
|
}
|
|
if !ended && mk != nil {
|
|
cutStream(w, mk.speaks)
|
|
}
|
|
return
|
|
}
|
|
|
|
// cutWords is what a client reads at the end of an answer whose stream broke off.
|
|
const cutWords = "the answer was cut off before it finished; send the request again"
|
|
|
|
// closing reports a stream line that ends its answer: [DONE], Anthropic's
|
|
// message_stop, the model saying why it stopped (finish_reason, stop_reason), or an
|
|
// error frame in either dialect. What follows a stop — usage, [DONE] — adds nothing
|
|
// to the answer, so a stream cut after it is whole.
|
|
func closing(line []byte) bool {
|
|
t := bytes.TrimSpace(line)
|
|
switch {
|
|
case bytes.Equal(t, []byte("data: [DONE]")), bytes.Equal(t, []byte("data:[DONE]")),
|
|
bytes.Equal(t, []byte("event: message_stop")), bytes.Equal(t, []byte("event: error")):
|
|
return true
|
|
}
|
|
payload, ok := bytes.CutPrefix(t, zenDataPrefix)
|
|
if !ok || !bytes.Contains(payload, []byte(`"error"`)) && !bytes.Contains(payload, []byte(`"message_stop"`)) &&
|
|
!bytes.Contains(payload, []byte(`"finish_reason"`)) && !bytes.Contains(payload, []byte(`"stop_reason"`)) {
|
|
return false
|
|
}
|
|
var f struct {
|
|
Type string `json:"type"`
|
|
Error json.RawMessage `json:"error"`
|
|
Choices []struct {
|
|
Finish string `json:"finish_reason"`
|
|
} `json:"choices"`
|
|
Delta struct {
|
|
Stop string `json:"stop_reason"`
|
|
} `json:"delta"`
|
|
}
|
|
if json.Unmarshal(bytes.TrimSpace(payload), &f) != nil {
|
|
return false
|
|
}
|
|
for _, c := range f.Choices {
|
|
if c.Finish != "" {
|
|
return true
|
|
}
|
|
}
|
|
return f.Type == "message_stop" || f.Type == "error" || f.Delta.Stop != "" || len(f.Error) > 0 && string(f.Error) != "null"
|
|
}
|
|
|
|
// cutStream ends a stream that broke off with an error event in its dialect.
|
|
func cutStream(w *bufio.Writer, speaks shape) {
|
|
switch speaks {
|
|
case messageShape:
|
|
streamAnthropicError(w, "api_error", cutWords)
|
|
case chatShape:
|
|
b, _ := json.Marshal(map[string]any{"error": map[string]string{"message": cutWords, "type": "upstream_error", "code": onCut}})
|
|
_, _ = w.Write(append(append([]byte("data: "), b...), '\n', '\n'))
|
|
_ = w.Flush()
|
|
}
|
|
}
|
|
|
|
// answerChars is how many characters of answer one stream frame carries, in either
|
|
// dialect a family speaks: an OpenAI chunk's content, reasoning and tool-call
|
|
// arguments, or an Anthropic delta's text, thinking and partial JSON.
|
|
func answerChars(payload []byte) int {
|
|
var f struct {
|
|
Choices []struct {
|
|
Delta struct {
|
|
Content string `json:"content"`
|
|
Reasoning string `json:"reasoning"`
|
|
ReasoningContent string `json:"reasoning_content"`
|
|
ToolCalls []struct {
|
|
Function struct {
|
|
Arguments string `json:"arguments"`
|
|
} `json:"function"`
|
|
} `json:"tool_calls"`
|
|
} `json:"delta"`
|
|
} `json:"choices"`
|
|
Delta *struct {
|
|
Text string `json:"text"`
|
|
Thinking string `json:"thinking"`
|
|
PartialJSON string `json:"partial_json"`
|
|
} `json:"delta"`
|
|
}
|
|
if json.Unmarshal(payload, &f) != nil {
|
|
return 0
|
|
}
|
|
n := 0
|
|
for _, c := range f.Choices {
|
|
n += len(c.Delta.Content) + len(c.Delta.Reasoning) + len(c.Delta.ReasoningContent)
|
|
for _, tc := range c.Delta.ToolCalls {
|
|
n += len(tc.Function.Arguments)
|
|
}
|
|
}
|
|
if d := f.Delta; d != nil {
|
|
n += len(d.Text) + len(d.Thinking) + len(d.PartialJSON)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// sniffZenModel reads the top-level "model" (or Anthropic's message.model) from a
|
|
// family response payload — the concrete arm/rung that served, when the family
|
|
// discloses it. "" when absent; the caller then falls back to the requested SKU.
|
|
func sniffZenModel(payload []byte) string {
|
|
var p struct {
|
|
Model string `json:"model"`
|
|
Message *struct {
|
|
Model string `json:"model"`
|
|
} `json:"message"`
|
|
}
|
|
if json.Unmarshal(payload, &p) != nil {
|
|
return ""
|
|
}
|
|
if p.Model != "" {
|
|
return p.Model
|
|
}
|
|
if p.Message != nil {
|
|
return p.Message.Model
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// sniffZenId reads the response id the CLIENT sees — OpenAI's top-level "id"
|
|
// (chatcmpl-…) or Anthropic's message id (msg_…, at "id" for messages or nested under
|
|
// "message"). This is the exact id the client threads back to /v1/ai/feedback, so it is
|
|
// the routing-event join key. "" when absent.
|
|
func sniffZenId(payload []byte) string {
|
|
var p struct {
|
|
ID string `json:"id"`
|
|
Message *struct {
|
|
ID string `json:"id"`
|
|
} `json:"message"`
|
|
}
|
|
if json.Unmarshal(payload, &p) != nil {
|
|
return ""
|
|
}
|
|
if p.ID != "" {
|
|
return p.ID
|
|
}
|
|
if p.Message != nil {
|
|
return p.Message.ID
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// sniffZenUsage reads token usage from a family response — an SSE data payload or a
|
|
// full body — in the Anthropic (input_tokens/output_tokens, possibly nested under
|
|
// message) and OpenAI (prompt_tokens/completion_tokens) shapes. Present fields update
|
|
// the running counts; absent ones leave them, so Anthropic's message_start (input) and
|
|
// message_delta (output) accumulate across events.
|
|
//
|
|
// The dialects count a cached prompt differently, and all are read into the one
|
|
// split: OpenAI's prompt_tokens INCLUDES prompt_tokens_details.cached_tokens (and
|
|
// OpenRouter's, its cache_write_tokens beside them), while Anthropic's
|
|
// cache_read_input_tokens and cache_creation_input_tokens are IN ADDITION to
|
|
// input_tokens.
|
|
func sniffZenUsage(payload []byte, t *tokens) {
|
|
type usage struct {
|
|
InputTokens int `json:"input_tokens"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
CacheRead int `json:"cache_read_input_tokens"`
|
|
CacheWrite int `json:"cache_creation_input_tokens"`
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
PromptDetails *struct {
|
|
Cached int `json:"cached_tokens"`
|
|
Written int `json:"cache_write_tokens"`
|
|
} `json:"prompt_tokens_details"`
|
|
CompletionDetails *struct {
|
|
Reasoning int `json:"reasoning_tokens"`
|
|
} `json:"completion_tokens_details"`
|
|
}
|
|
var p struct {
|
|
Usage *usage `json:"usage"`
|
|
Message *struct {
|
|
Usage *usage `json:"usage"`
|
|
} `json:"message"`
|
|
}
|
|
if json.Unmarshal(payload, &p) != nil {
|
|
return
|
|
}
|
|
// whole says the usage is the answer's own count. The one a Messages stream
|
|
// opens with (message_start) is not: it states the prompt and an output of one
|
|
// token before a word is written, so it never says what the answer's output
|
|
// cost. Taking it as that billed a stream its caller hung up on for one token.
|
|
read := func(u *usage, whole bool) {
|
|
if u == nil {
|
|
return
|
|
}
|
|
if u.InputTokens > 0 {
|
|
t.fresh = u.InputTokens
|
|
}
|
|
if u.CacheWrite > 0 {
|
|
t.written = u.CacheWrite
|
|
}
|
|
if u.CacheRead > 0 {
|
|
t.cached = u.CacheRead
|
|
}
|
|
if u.PromptTokens > 0 {
|
|
cached, written := 0, 0
|
|
if u.PromptDetails != nil {
|
|
cached = min(max(u.PromptDetails.Cached, 0), u.PromptTokens)
|
|
written = min(max(u.PromptDetails.Written, 0), u.PromptTokens-cached)
|
|
}
|
|
t.fresh, t.cached, t.written = u.PromptTokens-cached-written, cached, written
|
|
}
|
|
if u.OutputTokens > 0 {
|
|
t.completion = u.OutputTokens
|
|
t.reported = t.reported || whole
|
|
}
|
|
if u.CompletionTokens > 0 {
|
|
t.completion, t.reported = u.CompletionTokens, true
|
|
}
|
|
if u.CompletionDetails != nil && u.CompletionDetails.Reasoning > 0 {
|
|
t.reasoning = u.CompletionDetails.Reasoning
|
|
}
|
|
}
|
|
read(p.Usage, true)
|
|
if p.Message != nil {
|
|
read(p.Message.Usage, false)
|
|
}
|
|
}
|
|
|
|
// coarseTokenEstimate is a last-resort prompt-token estimate (~4 chars/token) so a
|
|
// successful call whose usage ai could not read is still billed, never zero.
|
|
func coarseTokenEstimate(body []byte) int {
|
|
return len(body)/4 + 1
|
|
}
|
|
|
|
// recordFamilyUsage settles the budget hold at the exact discovered price and records
|
|
// the usage + trace. Billing lives in one place for both stream and buffered paths and
|
|
// for every family. origin is who actually served the call (originOf) — the row's
|
|
// answer to a question the response no longer answers.
|
|
// model is the SKU that ANSWERED and requested the one the caller named, set only
|
|
// when a spare route answered in its place. Both travel because the ledger is
|
|
// answering two questions — what did we serve, and was it what was asked for — and
|
|
// a row that can only answer the first cannot be read as a downgrade at all.
|
|
// whence is what the REQUEST said, read on the request's own goroutine and
|
|
// carried by value. A usage row needs exactly three things from the request, and
|
|
// all three are only safe to read before the handler returns — see the capture in
|
|
// pipeToFamily for what reading them later costs.
|
|
type whence struct {
|
|
ledger string // billingOrg: X-Org-Id, the bearer, the session cookie
|
|
ip string // the peer, fiber's own read
|
|
ctx context.Context // what the usage row and its span hang off
|
|
asked *modelFamily // the family of the model the caller named
|
|
plan *object.LimitGrant // the plan that covers the request, nil for none
|
|
depth string // how hard the request asks its model to think (depthOf)
|
|
fell bool // the model's floor answered (floor.go): it bills nothing
|
|
}
|
|
|
|
// recordFamilyUsage takes NO RECEIVER, deliberately. It is called from inside a
|
|
// stream writer that outlives the request, so a `c` in scope here is a request
|
|
// this function must not touch — and the way to keep a rule like that is to leave
|
|
// nothing to touch.
|
|
//
|
|
// It returns the charge in nano-dollars: exact, never floored to a cent, so an
|
|
// embedding worth a thousandth of a cent bills a thousandth of a cent at the hold,
|
|
// the debit, the row and the span alike.
|
|
func recordFamilyUsage(w whence, fam *modelFamily, model, requested string, prov *object.Provider, mk *mark, authUser *iam.User, isPremium, stream bool, reqID string, t tokens, sv serving, start time.Time, hold *budgetHold, status, errMsg string) int64 {
|
|
// The vendor charges nothing for a spare route — that is the whole reason it can
|
|
// answer while the account is empty — so it is billed at nothing. Charging
|
|
// retail for a downgrade the customer did not choose would be taking money for
|
|
// the outage. Stated once, on the record, so the hold, the debit, the warehouse
|
|
// row and the span all read the same zero (see usageCostNano). A priced SKU the
|
|
// family says it answered from a free rung costs the caller what that rung costs.
|
|
free := w.fell || fam.isSpare(model) || inPool(model) || sv.free
|
|
// A free model a family ran on a listed vendor's account counts toward the free
|
|
// view (listing.go); what ai sent on its own accounts counted as it was sent.
|
|
if lf := listedFamily(sv.vendor); lf != nil && sv.arm != "" && lf.free(sv.arm) {
|
|
lf.served(sv.arm, sv.account, status == "success")
|
|
}
|
|
// What the call cost us to buy: the family's own statement (cogsHeader) where it
|
|
// made one, else the cost the answer's usage stated (a resale vendor's), with the
|
|
// fee of the vendor whose credit bought it on top (fee.go). It is the row's COGS,
|
|
// so usageMargin reads the margin on a relayed call as a fact rather than a guess.
|
|
cogs := sv.cogs
|
|
if cogs == nil {
|
|
cogs = mk.cogs()
|
|
}
|
|
cost := costed(cogs, buyers(fam, sv)...)
|
|
var nano int64
|
|
var exact *int64 // the charge the ledger, row and span read
|
|
if status == "success" && !free {
|
|
if zm, ok := fam.lookup(model); ok {
|
|
switch {
|
|
case sv.maker != nil && cost != nil:
|
|
// A maker's answer is priced at cost: what it charged, the platform's
|
|
// fee on top and nothing else (native.go, fee.go).
|
|
nano = plus(*cost, false)
|
|
default:
|
|
// The model's price for its tokens. A resold SKU never bills below
|
|
// the cost its answer states; a cost in the headers of any other
|
|
// answer is its family's record, not a price. An Enso model bills its
|
|
// fixed price whatever its arm cost (rates.go).
|
|
nano = zm.bill(mk.cogs(), t)
|
|
}
|
|
} else {
|
|
nano = tokenCostNano(model, t.prompt(), t.completion, t.cached, t.written)
|
|
}
|
|
// A model that stood in for the one named costs no more than the named one
|
|
// would have for the same tokens: the caller pays for an answer, never for
|
|
// our outage.
|
|
if requested != "" && w.asked != nil {
|
|
if zm, ok := w.asked.lookup(requested); ok && zm.priced() {
|
|
nano = min(nano, zm.retailNano(t))
|
|
}
|
|
}
|
|
exact = &nano
|
|
}
|
|
// A call that cost us more than it billed is a loss, and a loss is never
|
|
// silent: the row says how much (loss_nano), and the ledger counts it (rates.go).
|
|
var loss int64
|
|
if status == "success" && cost != nil && *cost > nano {
|
|
loss = *cost - nano
|
|
log.Warn("loss: %s answered by %s (%s) cost %d nano and billed %d", model, sv.arm, sv.vendor, *cost, nano)
|
|
}
|
|
var trailed string
|
|
if fam.prices != nil {
|
|
trailed = trail(fam, model, sv, t)
|
|
if status == "success" {
|
|
fam.answered(model, sv)
|
|
}
|
|
}
|
|
hold.settleNano(nano)
|
|
if authUser == nil {
|
|
return nano
|
|
}
|
|
origin, via := reached(prov, mk, sv)
|
|
rec := &usageRecord{
|
|
Owner: w.ledger, Organization: authUser.Owner,
|
|
Model: model, Requested: requested, Free: free, Provider: fam.name, Origin: origin, Via: via,
|
|
PromptTokens: t.prompt(), CacheReadTokens: t.cached, CacheWriteTokens: t.written, CompletionTokens: t.completion, TotalTokens: t.prompt() + t.completion,
|
|
ReasoningTokens: t.reasoning, Served: sv.arm, Vendor: sv.vendor, Failover: sv.failover, First: sv.first,
|
|
Reasoning: w.depth, Arms: trailed, LossNano: loss,
|
|
Cost: float64(nano) / 1e9, Currency: "USD",
|
|
Premium: isPremium, Stream: stream, Status: status, ErrorMsg: errMsg,
|
|
ClientIP: w.ip, RequestID: reqID, Account: "hanzo", Routing: string(mk.routing),
|
|
CostNanoExact: cost,
|
|
BilledNanoExact: exact,
|
|
plan: w.plan,
|
|
planNanos: planCost(w.plan, sv.cost),
|
|
}
|
|
rec.bind(w.ctx, authUser)
|
|
recordUsage(rec)
|
|
recordTrace(w.ctx, rec, start)
|
|
failed(w, fam, model, sv.failover, authUser, isPremium, stream, reqID, start)
|
|
return nano
|
|
}
|
|
|
|
// recordFamilyRouting writes the privacy-preserving learning ledger row for a served
|
|
// family call and, when the engine endpoint is configured, the shadow A/B pick. It is
|
|
// fire-and-forget off the request hot path (exactly like AddRoutingEvent's auto-route
|
|
// callers): a collection failure never touches the served response. It NEVER stores
|
|
// prompt text — only token/cost/latency metrics, the served arm, and (for the shadow)
|
|
// the engine's own opaque feature vector + counterfactual model.
|
|
//
|
|
// The request-derived routing inputs (last user text, media flag) are extracted
|
|
// synchronously from rawBody before the goroutine spawns, so nothing reads the router
|
|
// context off-thread. The shadow /v1/route call is hard-capped by router.DefaultTimeout.
|
|
func (c *ApiController) recordFamilyRouting(model, served, respID, reqID string, rawBody []byte, orgId string, authUser *iam.User, prompt, completion int, cents int64, start time.Time) {
|
|
owner := orgId
|
|
user := ""
|
|
if authUser != nil {
|
|
if owner == "" {
|
|
owner = authUser.Owner
|
|
}
|
|
user = authUser.Owner + "/" + authUser.Name
|
|
}
|
|
// The join key is the response id the client sees; fall back to the internal reqID
|
|
// when the family disclosed none (RecordFamilyRouting normalizes it identically to
|
|
// /v1/ai/feedback so both sides key on the same value).
|
|
responseId := respID
|
|
if strings.TrimSpace(responseId) == "" {
|
|
responseId = reqID
|
|
}
|
|
text, hasMedia := familyRoutingText(rawBody)
|
|
endpoint := ""
|
|
if cfg := GetModelConfig(); cfg != nil {
|
|
endpoint = cfg.RouterClient(nil).Endpoint
|
|
}
|
|
in := object.FamilyRoutingInput{
|
|
Owner: owner, User: user,
|
|
RequestedModel: model, RoutedModel: served, ResponseId: responseId,
|
|
PromptTokens: prompt, CompletionTokens: completion,
|
|
CostCents: cents, LatencyMs: time.Since(start).Milliseconds(),
|
|
RouterEndpoint: endpoint, ShadowText: text,
|
|
ShadowApproxTokens: coarseTokenEstimate(rawBody), ShadowHasMedia: hasMedia,
|
|
}
|
|
// Fire-and-forget off the request hot path — the ONE shared writer (object) is
|
|
// called identically by cloud's embedded-zen meter.
|
|
go object.RecordFamilyRouting(in)
|
|
}
|
|
|
|
// familyRoutingText extracts the last user turn's text and whether the request carries
|
|
// media from a raw chat/completions or messages body — enough for the shadow router to
|
|
// classify, working for both dialects (both use messages[].role/content, content a
|
|
// string or an array of typed parts). It is a pure function of rawBody; it returns no
|
|
// prompt text to any store — only the shadow /v1/route call (which the engine already
|
|
// receives for auto routing) reads it, and only the engine's derived features persist.
|
|
func familyRoutingText(rawBody []byte) (text string, hasMedia bool) {
|
|
var body struct {
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
if json.Unmarshal(rawBody, &body) != nil {
|
|
return "", false
|
|
}
|
|
for _, m := range slices.Backward(body.Messages) {
|
|
|
|
if m.Role != "user" {
|
|
continue
|
|
}
|
|
var s string
|
|
if json.Unmarshal(m.Content, &s) == nil {
|
|
if s != "" {
|
|
return s, false
|
|
}
|
|
continue
|
|
}
|
|
var parts []struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text"`
|
|
}
|
|
if json.Unmarshal(m.Content, &parts) != nil {
|
|
continue
|
|
}
|
|
var b strings.Builder
|
|
for _, p := range parts {
|
|
if strings.Contains(p.Type, "image") || strings.Contains(p.Type, "video") {
|
|
hasMedia = true
|
|
}
|
|
if p.Text != "" {
|
|
if b.Len() > 0 {
|
|
b.WriteByte('\n')
|
|
}
|
|
b.WriteString(p.Text)
|
|
}
|
|
}
|
|
if b.Len() > 0 || hasMedia {
|
|
return b.String(), hasMedia
|
|
}
|
|
}
|
|
return "", hasMedia
|
|
}
|
|
|
|
// zenError writes a dialect-correct error. Anthropic callers (Claude Code) get the
|
|
// Anthropic error envelope; OpenAI callers get the OpenAI one.
|
|
func (c *ApiController) zenError(dialect, msg string, status int) {
|
|
if dialect == "anthropic" {
|
|
c.respondAnthropicError(anthropicErrorTypeForStatus(status), msg, status)
|
|
return
|
|
}
|
|
body, _ := json.Marshal(map[string]any{
|
|
"error": map[string]any{"message": msg, "type": "api_error", "code": status},
|
|
})
|
|
c.SetHeader("Content-Type", "application/json")
|
|
_ = c.Bytes(status, body)
|
|
}
|