Files
ai/controllers/failover.go
zeekay feb7995277 maker: an OpenAI model is asked of OpenAI before OpenRouter, and a drained account is said on the bus
OpenRouter out of credit answered 503 for every openai/* SKU and Enso's
OpenAI rungs, though the platform holds an OpenAI account that lists them.
The resale family now names makers by prefix: Anthropic's, as before, and
OpenAI's, which speaks the request's own dialect (native_chat.go). A chat
goes as written under OpenAI's id, its ceiling as max_completion_tokens,
streamed with its usage; 429 insufficient_quota and 402 rest the key, any
refusal of the request goes to the reseller, and the answer bills its cost
at OpenAI's price with the platform's fee, under OpenRouter's 5.5%.

An account that cannot pay (a maker's key, an OpenRouter key, a route row)
is published by name through object.PublishDrained, once per ten minutes
while it stays so, for the host's bus.
2026-10-16 22:51:47 -07:00

433 lines
18 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
metric "github.com/luxfi/metric"
iam "github.com/hanzoai/ai/internal/iam"
"github.com/hanzoai/ai/log"
"github.com/hanzoai/ai/model"
"github.com/hanzoai/ai/object"
)
// errPartiallyWritten stops a retry that can no longer be made safely: bytes are
// already on the wire, so replaying the call would duplicate the client's
// output. Deliberately NOT transient — retryTransient sees a permanent error and
// stops immediately rather than sleeping first.
var errPartiallyWritten = errors.New("response partially written; cannot retry")
// served names the provider that answered: our route label for it, the host it
// answered from, and the row whose credential was spent. Three projections of
// ONE fact travelling together, so a caller cannot record the label while losing
// the address — or, worse, bill against a row that did not serve.
//
// A failed attempt carries a name and nothing else: nothing answered, and saying
// nothing is the honest form of that.
type served struct {
name string
origin string
// row is the provider that actually spent a credential. providerBYO must
// read THIS and not the row auth resolved before the request moved: a
// customer's own key is only "bring your own" if it is the key that paid.
row *object.Provider
}
// ask is one text completion waiting to be served: what to send, where the
// answer goes, and who has already refused it.
//
// A struct rather than eleven positional arguments because two of them are
// []*model.RawMessage — history and knowledge — and in money code a silent
// argument swap between two same-typed slices is not a hazard worth keeping for
// the sake of a shorter call.
type ask struct {
ctx context.Context
route *modelRoute
org string // ledger org, so a customer's own provider row wins where it exists
model string // caller-facing model name, for the honest error
// primary is the row auth already resolved for route.providerName. Reusing
// it keeps the first attempt on exactly the credential the auth path chose;
// re-resolving by name would quietly swap a customer's connected key for
// the platform's.
primary *object.Provider
question string
history []*model.RawMessage
knowledge []*model.RawMessage
lang string
writer io.Writer
// prompt is the prompt's token count, already measured by the caller. Only a
// raced attempt reads it: a loser is billed for the prompt the vendor did
// accept plus whatever it produced before losing, and re-counting it here
// would be a second answer to a question the caller already asked.
prompt int
// fan carries this request into fast mode: several providers at once, first
// answer wins. Nil is the cascade, which is what every request that does not
// ask for fast mode gets.
fan *fan
// sent reports whether any byte has reached the client. Once it has, this
// request is committed and can never be moved.
sent func() bool
// prior are providers that already refused this request elsewhere — the
// family pipe, typically. They are not asked again and they count against
// the attempt cap.
prior []attempt
}
// serve offers the completion to each capable provider in turn and returns the
// first real answer.
//
// The loop is the whole failover policy in one place:
//
// provider fault -> record it, demote the vendor, offer it to the next one
// request fault -> stop; every vendor refuses this identically
// nobody left -> exhausted(), which names who was asked and why they said no
//
// It returns every attempt made, successful or not, so the caller can record
// the ones that failed. A failover nobody can see is half a fix: the empty
// account that caused it stays empty.
func (a ask) serve() (*model.ModelResult, served, []attempt, error) {
// Fast mode asks several at once; everything else takes them in turn. One
// entry point either way, so a caller sets a number rather than choosing a
// code path — and a fan too narrow to race falls back here by itself.
if a.fan != nil && a.fan.n > 1 {
return a.race()
}
return a.cascade()
}
// cascade is the failover loop itself: one provider at a time, in order.
func (a ask) cascade() (*model.ModelResult, served, []attempt, error) {
tried := a.prior
for _, c := range candidates(a.org, a.route, a.prior, FreeOnlyFor(a.context())) {
// The caller hung up. Offering the request to another vendor would spend
// money answering an empty room, and would file a healthy vendor's name
// against the client's disconnect.
if err := a.context().Err(); err != nil {
return nil, served{}, tried, err
}
res, row, err := a.call(c)
if err == nil {
if len(tried) > 0 {
log.Warn("failover: model=%s served by %s after %d refusal(s) — %s",
a.model, c.provider, len(tried), reasons(tried))
}
// The row travels with the answer because it is the row whose
// credential was spent, and that is what decides whether this call
// was the customer's own key or ours.
// No mark: this path builds the answer from the provider SDK rather
// than relaying an envelope, so nothing disclosed an upstream and the
// host we dialled is the whole answer.
return res, served{c.provider, originOf(row, nil), row}, tried, nil
}
at := attempt{
provider: c.provider,
upstream: c.upstream,
origin: originOf(row, nil),
status: upstreamHTTPStatus(err),
fault: faultOf(err),
err: err,
}
tried = append(tried, at)
announce(a.model, at)
if d := cooled.rest(a.org, c.provider, err); d > 0 {
log.Warn("failover: demoting provider=%s for %s after status=%d", c.provider, d, at.status)
}
// The request is wrong, not the vendor. Every other vendor refuses it
// the same way, so stop and hand the caller the real reason.
if at.fault == faultRequest {
return nil, served{name: c.provider}, tried, err
}
// Bytes reached the client during that attempt. Nothing can be replayed
// now; report what we have rather than duplicating output.
if a.sent != nil && a.sent() {
log.Warn("failover: model=%s provider=%s failed after the stream opened — cannot move: %v",
a.model, c.provider, err)
return nil, served{name: c.provider}, tried, err
}
}
return nil, served{}, tried, exhausted(a.model, tried)
}
// call offers the completion to ONE provider, riding out a transient refusal
// (a 429 "overloaded", a 503, a timeout) rather than giving up on it.
//
// A 429 does not mean "this cannot be served", it means "ask again shortly" —
// and we are better placed to do that than a client who can neither pick a
// different vendor nor stagger itself against every other client doing the
// same. We wait it out here; when waiting stops helping, serve() moves on.
func (a ask) call(c candidate) (*model.ModelResult, *object.Provider, error) {
var res *model.ModelResult
var row *object.Provider
err := retryTransient(a.context(), currentRetryPolicy(), func() error {
// Committed: bytes are on the wire and this can no longer be replayed.
if a.sent != nil && a.sent() {
return errPartiallyWritten
}
// Discard whatever a failed attempt left in the writer. Both writers
// APPEND to an internal buffer, so without this the next provider's
// answer is served glued to the dead one's half-sentence — a silent
// corruption that reads like a model going mad.
if r, ok := a.writer.(interface{ Reset() }); ok {
r.Reset()
}
var e error
res, row, e = callProvider(a.org, rowFor(a.primary, c), c, a.question, a.writer, a.history, a.knowledge, a.lang)
if e != nil && isTransientError(e) {
log.Warn("retry: provider=%s is busy (%v) — holding the request rather than bouncing it to the client", c.provider, e)
}
return wrapUpstreamError(e)
})
return res, row, err
}
// context returns the caller's context, so a retry never outlives the request
// that asked for it. Background would make retryPolicy.waitBeforeRetry's
// deadline check vacuous and let a dead client's request keep sleeping.
func (a ask) context() context.Context {
if a.ctx != nil {
return a.ctx
}
return context.Background()
}
// rowFor returns the row auth resolved when it IS candidate c's provider, and nil
// for any other candidate so that one is resolved for the org.
//
// It asks the ROW's name, never the route's. routeForPrompt may move a request to a
// route led by a different vendor, and handing that route's first candidate the row
// auth resolved would send the prompt to the wrong vendor, under a model id meant
// for another one.
func rowFor(primary *object.Provider, c candidate) *object.Provider {
if primary != nil && primary.Name == c.provider {
return primary
}
return nil
}
// The conditions supplyRefused counts. They label the metric and name the branch
// that records them, so the reason an operator reads and the reason the code
// took are one value rather than two spellings of one.
const (
reasonFunding = "funding" // 402 — this account with the vendor is spent
reasonCredential = "credential" // 401 — our key is dead or revoked
reasonCeiling = "ceiling" // our own cash breaker is holding
reasonRefused = "refused" // everything else a vendor answers
)
// supplyRefused counts the refusals that are OURS. The caller is told 503 and owes
// nothing, so nobody on their side has a reason to report it: without this, an
// empty vendor account is visible only to whoever thinks to tail a log, and a log
// is read after somebody has already noticed.
//
// Labelled by provider and reason because "top this vendor up", "this key is dead"
// and "our own ceiling is holding" are three different jobs, and a single number
// cannot say which one is running. Both label sets are bounded — the providers are
// the route table's, the reasons are the four above.
//
// It registers into the same DefaultRegistry that GET /v1/metrics serves
// (object.MetricsHandler), which is the seam vmagent already scrapes. Alert on
// rate(cloud_supply_refused[5m]) > 0.
var supplyRefused = metric.NewCounterVec(metric.CounterOpts{
Name: "cloud_supply_refused",
Help: "Requests refused because our own supply could not serve them, by provider and reason",
}, []string{"provider", "reason"})
// announce puts a refusal where someone will see it. Level is chosen by who has
// to act on it: a busy vendor is routine, an empty account needs a human with a
// credit card, and a rejected credential needs one now.
//
// The counter carries the same fact for anything that watches rather than reads.
// A log line is evidence after the fact and only for whoever thinks to look; the
// metric is what can raise a hand before a customer does.
func announce(model string, a attempt) {
switch {
case a.status == http.StatusUnauthorized:
supplyRefused.WithLabelValues(a.provider, reasonCredential).Inc()
log.Error("provider=%s rejected our credential (401) serving model=%s — this key is dead or revoked; the request FAILED OVER, so customers are being served and only this vendor is idle. Rotate it: %v",
a.provider, model, a.err)
case a.status == http.StatusPaymentRequired:
supplyRefused.WithLabelValues(a.provider, reasonFunding).Inc()
emptied(a.provider, "", model, a.status)
log.Error("provider=%s is out of money (402) serving model=%s — top it up; failing over: %v",
a.provider, model, a.err)
default:
supplyRefused.WithLabelValues(a.provider, reasonRefused).Inc()
log.Warn("failover: model=%s provider=%s refused status=%d fault=%s: %v",
model, a.provider, a.status, a.fault, a.err)
}
}
// drainedEvery is how often one drained account is said again while it stays drained:
// once when it is found, and again each drainedEvery after, so a reader who missed
// the first still hears it and a busy route does not say it on every request.
const drainedEvery = 10 * time.Minute
var (
drainedMu sync.Mutex
drainedSaid = map[string]time.Time{}
)
// emptied says that one of our accounts with vendor answered it cannot pay — on the
// host's bus (object.PublishSpent), where the SuperAdmin's feed hears it — once per
// drainedEvery for each account. account is the key's NAME, never the key; "" when
// the refusal speaks for every account of the vendor at once.
func emptied(vendor, account, model string, status int) {
now := keyNow()
id := vendor + "\x00" + account
drainedMu.Lock()
last, said := drainedSaid[id]
if said && now.Sub(last) < drainedEvery {
drainedMu.Unlock()
return
}
drainedSaid[id] = now
drainedMu.Unlock()
object.PublishDrained(context.Background(), object.Drained{Vendor: vendor, Account: account, Model: model, Status: status, At: now})
}
// reasons renders the refusals so far for one log line.
func reasons(tried []attempt) string {
var out strings.Builder
for i, a := range tried {
if i > 0 {
out.WriteString("; ")
}
out.WriteString(fmt.Sprintf("%s status=%d", a.provider, a.status))
}
return out.String()
}
// callProvider asks one provider row for a completion and reports the row that
// answered. The row is resolved FOR THE ORG, so a customer that connected its
// own key spends its own key — on a fallback exactly as on the primary.
// Resolving globally here is how a request billed as "bring your own" comes to
// be served on the platform's credential.
//
// Indirected through a var so the failover loop can be exercised against
// scripted providers without a live DB or a live upstream, the same way
// orgAutoRoutingLookup is.
var callProvider = func(
org string,
row *object.Provider,
c candidate,
question string,
writer io.Writer,
history []*model.RawMessage,
knowledge []*model.RawMessage,
lang string,
) (*model.ModelResult, *object.Provider, error) {
if row == nil {
var err error
row, err = object.GetModelProviderByNameForOrg(org, c.provider)
if err != nil {
return nil, nil, err
}
}
if row == nil {
// GetModelProviderByNameForOrg returns nil for BOTH a missing provider
// and one an admin switched off. Either way this vendor cannot serve and
// the next candidate must be tried, which is how a route's fallbacks stay
// honoured when the primary is toggled off from /v1/admin/ai/providers.
//
return nil, nil, unavailable(c.provider)
}
p := *row // copy: SubType is per-request, the row may be shared
p.SubType = c.upstream
modelProvider, err := p.GetModelProvider(lang)
if err != nil {
return nil, &p, err
}
result, err := modelProvider.QueryText(question, writer, history, "", knowledge, nil, lang)
return result, &p, err
}
// unavailable is how a provider row that is missing or switched off refuses.
//
// It carries a STATUS rather than a word for the classifier to find. This error
// is ours: re-reading our own prose to recover what we already knew is how
// "unavailable" came to be a substring the classifier trusted, where it also
// matched a VENDOR saying "unavailable for your account tier" — a different
// refusal with a different answer.
//
// Named so the cascade tests can assert the value production emits instead of a
// string that merely resembles it.
func unavailable(provider string) error {
return &apiError{status: http.StatusServiceUnavailable,
msg: fmt.Sprintf("provider %q is unavailable (disabled or not configured)", provider)}
}
// recordRefusals writes one usage row per provider that refused, so a vendor
// that stopped serving appears in the ledger the team already reads rather than
// only in a log line nobody tails. That absence is the second half of the
// outage this exists to prevent: the failover worked, nobody noticed, and the
// account stayed empty.
//
// The rows bill nothing: recordUsage is not called here, and no budget hold is
// touched. The money for this request is settled ONCE, on the record the caller
// writes for the outcome — served or failed — and an attempt that refused before
// producing a token adds nothing to it. The status is its own word — "failover",
// not "error" — so dashboards that count failures keep counting the same thing and
// a refused ATTEMPT on a request that ultimately succeeded is not filed as a
// failed request.
//
// A PROVIDER BEATEN IN A RACE NEVER ARRIVES HERE, and must not: it was cancelled
// mid-generation, so the vendor produced tokens and charged us, and this row
// would file a real charge as a free refusal. ask.race keeps them out of the
// attempts it returns and settles them through fan.bill instead — one ledger
// path each, so neither rule can be applied to the other's case.
// recordRefusals logs every vendor that refused. Its caller runs inside a stream
// writer, where the request context is already released, so it reads the
// request's outliving parts from a snapshot.
func recordRefusals(snap snapshot, model string, tried []attempt, user *iam.User, premium, stream bool, requestId string, start time.Time) {
if user == nil {
return
}
for _, a := range tried {
rec := &usageRecord{
Owner: snap.org,
Model: model,
Provider: a.provider,
Origin: a.origin,
Premium: premium,
Stream: stream,
Status: "failover",
ErrorMsg: a.err.Error(),
ClientIP: snap.ip,
RequestID: requestId,
}
rec.bind(snap.ctx, user)
recordTrace(snap.ctx, rec, start)
}
}