Files
ai/controllers/failover_loop_test.go
zooqueenandHanzo Dev 4cbf1f81a5 Paid lane: each call holds its worst case against the day, its org's share and its plan
A paid-lane call is seated once, by the balance gate (controllers.Seat), and
holds its estimate — prompt and completion ceiling at the most of list price
and cost, twice for fast mode, media at unit price — from admission until its
usage record settles it. The hold must fit PAID_LANE_DAILY, the paying org's
share of it (PAID_LANE_ORG_SHARE, default 0.25) and, for a plan-paid call, the
plan class's LimitGrant.Spend less the payer's holds in flight. A call that
does not fit is on the free lane: a chat for a third-party model is answered by
the free model, anything else only the paid lane serves is refused 429
paid_lane_full. Each org's settled spend per UTC day is kept in the store
(object.PaidDay), so a restart keeps the day. PAID_LANE_DAILY parses with
bounds; Inf and NaN close the lane.

Every ZAP handler asks the HTTP handlers' lane decision (paying via zapLane,
Jev in decide): a ZAP request holds no seat, so it never reaches a vendor that
spends Hanzo's money, with the switch on or off.

Race losers and video jobs settle the seat their request was admitted on; a
family is sent X-Hanzo-Spend only on a seat, capped at what it holds. A row an
org holds for itself is served off the paid lane while the switch is on, is
never handed to the free model and counts nothing against the day. With the
switch off no lane is named and a program past its plan's usage gets its 402,
as before. The test hook on the lane is gone; tests seat their requests.

Co-authored-by: Hanzo Dev <[email protected]>
2026-10-08 06:11:02 -07:00

874 lines
32 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"bufio"
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/hanzoai/ai/model"
"github.com/hanzoai/ai/object"
)
// fleet scripts a set of providers: each name answers with an error, or serves.
// It records the order it was asked in, which is the property most of these
// tests are really about.
type fleet struct {
answer map[string]error // nil entry ⇒ that provider serves
asked []string
// wrote is text a provider emits into the writer before failing, to prove a
// dead provider's half-sentence never reaches the next one's answer.
wrote map[string]string
}
func (f *fleet) install(t *testing.T) {
t.Helper()
saved := callProvider
t.Cleanup(func() { callProvider = saved })
callProvider = func(org string, row *object.Provider, c candidate, question string,
w io.Writer, history, knowledge []*model.RawMessage, lang string,
) (*model.ModelResult, *object.Provider, error) {
f.asked = append(f.asked, c.provider)
if s := f.wrote[c.provider]; s != "" {
_, _ = w.Write([]byte(s))
}
if err, ok := f.answer[c.provider]; ok && err != nil {
return nil, &object.Provider{Owner: "admin", Name: c.provider}, err
}
return &model.ModelResult{ResponseTokenCount: 7},
&object.Provider{Owner: "admin", Name: c.provider}, nil
}
}
// buffer is the minimum a writer must be for the loop: it accumulates, and it
// can be told to forget. Mirrors OpenAIWriter/AnthropicWriter, which both
// APPEND on every Write.
type buffer struct {
text string
sent bool // set once bytes have reached the "client"
}
func (b *buffer) Write(p []byte) (int, error) { b.text += string(p); return len(p), nil }
func (b *buffer) Reset() { b.text = "" }
func quick(t *testing.T) {
t.Helper()
savedPolicy, savedConfig := defaultRetryPolicy, globalModelConfig
t.Cleanup(func() { defaultRetryPolicy, globalModelConfig = savedPolicy, savedConfig })
// One try per provider and no sleeping: these tests are about which vendor
// is asked, not about how patiently each one is waited on (retry_test covers
// that). The config must be cleared as well as the default, because
// currentRetryPolicy reads models.yaml FIRST — a config another test in this
// package left loaded would otherwise reinstate the real backoff.
globalModelConfig = nil
defaultRetryPolicy = retryPolicy{attempts: 1, base: time.Millisecond, max: time.Millisecond}
}
// distinct counts the VENDORS asked, which is what the cap bounds. The same
// vendor asked twice by the same-provider retry is one vendor.
func distinct(asked []string) int {
seen := map[string]bool{}
for _, a := range asked {
seen[a] = true
}
return len(seen)
}
func newAsk(r *modelRoute, w io.Writer, sent func() bool) ask {
return ask{route: r, model: "test-model", question: "hi", writer: w, sent: sent, ctx: paidSeat(context.Background())}
}
// THE OUTAGE. One vendor's account is empty; the product must not go dark.
func TestPaymentRequiredFailsOverAndTheNextProviderServes(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{answer: map[string]error{
"enso": apiErr(402, "Insufficient credits."),
}}
f.install(t)
var w buffer
res, by, tried, err := newAsk(route("enso", "do-ai"), &w, nil).serve()
if err != nil {
t.Fatalf("a 402 from one vendor must not fail the request: %v", err)
}
if res == nil || by.name != "do-ai" {
t.Fatalf("served by %q, want do-ai", by.name)
}
if strings.Join(f.asked, ",") != "enso,do-ai" {
t.Errorf("asked %v, want [enso do-ai]", f.asked)
}
if len(tried) != 1 || tried[0].provider != "enso" || tried[0].status != 402 {
t.Fatalf("the refusal must be reported so the empty account gets noticed, got %+v", tried)
}
if tried[0].fault != faultProvider {
t.Errorf("a 402 is the vendor's fault, got %s", tried[0].fault)
}
}
// The opposite guard: a broken request must cost exactly one upstream call.
func TestBadRequestDoesNotFailOver(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{answer: map[string]error{
"do-ai": apiErr(400, "invalid 'messages': empty"),
}}
f.install(t)
var w buffer
_, _, tried, err := newAsk(route("do-ai", "anthropic", "fireworks"), &w, nil).serve()
if err == nil {
t.Fatal("a 400 must surface, not be papered over")
}
if len(f.asked) != 1 {
t.Errorf("asked %v — a malformed request fails identically everywhere; retrying it "+
"turns one error into %d and bills for each", f.asked, len(f.asked))
}
if !strings.Contains(err.Error(), "invalid 'messages'") {
t.Errorf("the caller must get the REAL reason, got %v", err)
}
if len(tried) != 1 || tried[0].fault != faultRequest {
t.Errorf("want one request-fault attempt, got %+v", tried)
}
}
// A DEAD CREDENTIAL FAILS OVER, AND IS STILL LOUD. Those are two jobs, and this
// used to do the second by refusing to do the first.
//
// The refusal is a fact about the VENDOR — our key will not open it — and the
// caller's own credential was checked at our edge long before any vendor was
// dialled, so nothing about their request is wrong. announce() counts
// cloud_supply_refused{reason="credential"} and logs at ERROR whichever way the
// request goes, so surfacing costs the failover nothing.
//
// Measured, the old coupling cost exactly what it was meant to prevent:
// DO_AI_API_KEY died and 93 routes answered 401 to CUSTOMERS for days — louder
// than any alert and aimed at the wrong people — while the metric that was meant
// to raise a hand had been sitting there the whole time.
func TestUnauthorizedFailsOverAndIsStillLoud(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{answer: map[string]error{"do-ai": apiErr(401, "invalid api key")}}
f.install(t)
var w buffer
_, served, tried, err := newAsk(route("do-ai", "anthropic"), &w, nil).serve()
if err != nil {
t.Fatalf("a dead key of ours must not become the customer's error: %v", err)
}
if served.name != "anthropic" {
t.Errorf("served by %q, want anthropic — the request must reach a vendor that can answer", served.name)
}
if len(f.asked) != 2 {
t.Errorf("asked %v — the dead vendor is tried once, then the request moves", f.asked)
}
if len(tried) != 1 || tried[0].status != 401 {
t.Errorf("the 401 must be RECORDED as an attempt, not swallowed: %+v", tried)
}
// Rested like an empty account: both need a human, and neither clears itself
// in seconds, so the fleet must not return to it every few seconds.
if !cooled.cooling(credential{"", "do-ai"}) {
t.Error("a vendor whose key is dead must be demoted, or every request pays its 401 first")
}
}
// The point of demotion: the second request must not pay the dead vendor's
// round trip all over again.
func TestDemotedProviderIsSkippedWhileCoolingDown(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{answer: map[string]error{"enso": apiErr(402, "Insufficient credits.")}}
f.install(t)
var w1 buffer
if _, _, _, err := newAsk(route("enso", "do-ai"), &w1, nil).serve(); err != nil {
t.Fatalf("first request: %v", err)
}
if !cooled.cooling(credential{"", "enso"}) {
t.Fatal("a vendor that answered 402 must be demoted")
}
f.asked = nil
var w2 buffer
_, by, _, err := newAsk(route("enso", "do-ai"), &w2, nil).serve()
if err != nil {
t.Fatalf("second request: %v", err)
}
if by.name != "do-ai" {
t.Fatalf("served by %q, want do-ai", by.name)
}
if strings.Join(f.asked, ",") != "do-ai" {
t.Errorf("asked %v, want [do-ai] only — without the cooldown every single request "+
"pays the empty vendor's latency before getting a real answer", f.asked)
}
}
// The loop must file its demotion against the account that earned it.
//
// This is the WRITE side of the property TestOneTenantsEmptyAccountIsNotAnothersOutage
// asserts on the READ side. Both are needed: with only one, dropping the org at
// the other end restores the cross-tenant leak with every other test still green.
func TestTheLoopDemotesTheAccountNotTheVendor(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{answer: map[string]error{"enso": apiErr(402, "Insufficient credits.")}}
f.install(t)
var w buffer
a := newAsk(route("enso", "do-ai"), &w, nil)
a.org = "org-a"
if _, _, _, err := a.serve(); err != nil {
t.Fatalf("serve: %v", err)
}
if !cooled.cooling(credential{"org-a", "enso"}) {
t.Error("the org whose key answered 402 must be resting — otherwise its next request " +
"pays the empty account's round trip all over again")
}
if cooled.cooling(credential{"org-b", "enso"}) {
t.Error("another tenant inherited the penalty; its request spends a different credential")
}
if cooled.cooling(credential{"", "enso"}) {
t.Error("the refusal was filed against the VENDOR rather than the account that earned it, " +
"which is the whole cross-tenant leak")
}
}
// Both sites that learn from a refusal learn the same thing, so both are asked
// the same question here. A refusal rests the ACCOUNT that answered it; no other
// tenant and no other vendor inherits anything.
func TestARefusalRestsOneAccount(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
if d := cooled.rest("org-a", "enso", apiErr(402, "Insufficient credits.")); d != coolBroke {
t.Fatalf("an empty account rests %v, want %v", d, coolBroke)
}
if !cooled.cooling(credential{"org-a", "enso"}) {
t.Error("the account that refused must be resting")
}
if cooled.cooling(credential{"org-b", "enso"}) {
t.Error("another tenant inherited it")
}
if cooled.cooling(credential{"org-a", "do-ai"}) {
t.Error("another vendor inherited it")
}
// A DEAD KEY RESTS AS LONG AS AN EMPTY ACCOUNT. It is the same operational
// class — one needs a human with a credit card, the other a human with a new
// key — and neither clears itself in seconds, so a short rest would put every
// request back on a vendor that cannot answer.
if d := cooled.rest("org-a", "do-ai", apiErr(401, "invalid api key")); d != coolBroke {
t.Errorf("a 401 rested the vendor for %v, want %v", d, coolBroke)
}
if !cooled.cooling(credential{"org-a", "do-ai"}) {
t.Error("the vendor whose key is dead must be resting")
}
// And it is still ONE account's problem: another tenant's credential for the
// same vendor is untouched.
if cooled.cooling(credential{"org-b", "do-ai"}) {
t.Error("another tenant inherited our dead key")
}
}
// breaks is an http.ResponseWriter that delivers `ok` writes and then fails —
// the shape of an upstream dying mid-answer, after the client already has bytes.
type breaks struct {
*httptest.ResponseRecorder
ok int
}
func breaksAfter(ok int) *breaks { return &breaks{httptest.NewRecorder(), ok} }
// tears delivers PART of a write and then reports the failure — a connection that
// broke mid-event. n > 0 with a non-nil error is the case the flag has to get
// right: those bytes are on the wire whatever the error says about the rest.
type tears struct{ *httptest.ResponseRecorder }
func (t *tears) Write(p []byte) (int, error) {
n, _ := t.ResponseRecorder.Write(p[:len(p)/2])
return n, errors.New("connection reset by peer")
}
func (b *breaks) Write(p []byte) (int, error) {
if b.ok <= 0 {
return 0, errors.New("connection reset by peer")
}
b.ok--
return b.ResponseRecorder.Write(p)
}
// A8. StreamSent is the whole movability decision, and it was set at the END of
// Write — after message_start and content_block_start had already gone out.
//
// So there was a window, 185 measured bytes wide, in which the client held the
// opening of one vendor's answer and the loop still believed the request could be
// offered to another. It would then have received the whole of a second vendor's
// answer glued to the first one's first breath.
func TestTheAnthropicWriterSaysSentAsSoonAsBytesGoOut(t *testing.T) {
// Deliver message_start and content_block_start, then fail — exactly the gap.
rec := breaksAfter(2)
w := &AnthropicWriter{
Writer: bufio.NewWriter(rec),
RequestID: "req-1",
Stream: true,
Cleaner: *NewCleaner(6),
Model: "claude-x",
}
_, err := w.Write([]byte("2 + 2 = 4"))
if err == nil {
t.Fatal("the scripted writer must fail on the delta")
}
if rec.Body.Len() == 0 {
t.Fatal("nothing was written; this test proves nothing")
}
if !w.StreamSent {
t.Errorf("%d bytes reached the client and StreamSent is false — the loop would move "+
"this request and the customer would read two vendors' answers spliced together",
rec.Body.Len())
}
if !strings.Contains(rec.Body.String(), "message_start") {
t.Errorf("expected the header events on the wire, got:\n%s", rec.Body.String())
}
}
// A half-written event is still a written event. The bytes are gone; whether the
// call that sent them also returned an error changes nothing about that.
func TestAPartialWriteCountsAsSent(t *testing.T) {
rec := &tears{httptest.NewRecorder()}
w := &AnthropicWriter{
Writer: bufio.NewWriter(rec),
RequestID: "req-1",
Stream: true,
Cleaner: *NewCleaner(6),
Model: "claude-x",
}
if _, err := w.Write([]byte("2 + 2 = 4")); err == nil {
t.Fatal("the scripted writer must report the break")
}
if rec.Body.Len() == 0 {
t.Fatal("the scripted writer delivered nothing; this test proves nothing")
}
if !w.StreamSent {
t.Errorf("%d bytes are on the wire after a failed write and StreamSent is false — "+
"reading the error instead of the byte count is how a half-sent answer "+
"gets a second vendor's answer appended to it", rec.Body.Len())
}
}
// The other half: a writer that has sent nothing is still movable, or a single
// transient failure on the first byte would strand every request it touched.
func TestTheAnthropicWriterIsMovableUntilItWrites(t *testing.T) {
rec := breaksAfter(0)
w := &AnthropicWriter{
Writer: bufio.NewWriter(rec),
RequestID: "req-1",
Stream: true,
Cleaner: *NewCleaner(6),
Model: "claude-x",
}
if _, err := w.Write([]byte("2 + 2 = 4")); err == nil {
t.Fatal("the scripted writer must fail on the first event")
}
if rec.Body.Len() != 0 {
t.Fatalf("the scripted writer delivered %d bytes; it must deliver none", rec.Body.Len())
}
if w.StreamSent {
t.Error("nothing reached the client, so this request is still movable — saying " +
"otherwise turns one transient failure into a refusal we could have routed around")
}
}
// pipeFamily drives the real family pipe against a family answering `status`,
// with a funded reservation, and reports what came back and what the ledger says.
func pipeFamily(t *testing.T, subject string, status int, body string) (refused []attempt, avail int64) {
t.Helper()
family := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(status)
_, _ = w.Write([]byte(body))
}))
defer family.Close()
fam := &modelFamily{
name: "enso", prefix: "enso",
providerFn: func() *object.Provider {
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: family.URL}
},
}
object.GlobalBalanceLedger.SetBalance(subject, 1000)
hold, ok := reserveBudget(subject, 100)
if !ok {
t.Fatal("reserve against a funded subject must succeed")
}
if a, _ := object.GlobalBalanceLedger.Available(subject); a != 900 {
t.Fatalf("available after reserve = %d, want 900", a)
}
req := []byte(`{"model":"enso-flash","messages":[{"role":"user","content":"hi"}]}`)
c := visit("POST", "/v1/chat/completions")
refused = c.pipeToFamily(fam, "chat/completions", "openai", "enso-flash", req, false, 0,
"org-a", nil, false, hold, time.Now())
avail, _ = object.GlobalBalanceLedger.Available(subject)
return refused, avail
}
// A1. What the pipe returns and what it owes the reservation are the same fact,
// so they are asserted together.
//
// nil means the request is OVER: nothing downstream will run, so the cents must
// be back. A refusal means the request is still MOVING: the cents must still be
// held, because whichever provider ends up serving it has to be billed and
// settle only fires once.
//
// The leak this pins: a 400 from an embeddings family returned nil, and nothing
// anywhere released the hold. The customer's balance simply stayed short.
func TestTheHoldFollowsTheAnswer(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
t.Run("a refusal nobody else can fix ends the request and gives the cents back", func(t *testing.T) {
refused, avail := pipeFamily(t, "holdtest/req-fault", http.StatusBadRequest,
`{"error":{"message":"invalid 'messages': empty"}}`)
if refused != nil {
t.Fatalf("a 400 is the request's fault; no other vendor changes it, got %+v", refused)
}
if avail != 1000 {
t.Errorf("available = %d, want 1000 — the request is over and nothing downstream "+
"will ever settle this hold, so the customer is short by the reservation", avail)
}
})
t.Run("a refusal somebody else can fix keeps the cents held", func(t *testing.T) {
refused, avail := pipeFamily(t, "holdtest/provider-fault", http.StatusPaymentRequired,
`{"error":{"message":"Insufficient credits."}}`)
if len(refused) != 1 {
t.Fatalf("a 402 is the vendor's fault and the request must stay movable, got %+v", refused)
}
if avail != 900 {
t.Errorf("available = %d, want 900 — releasing here is one-shot, and whichever "+
"provider serves this next would then be billed nothing", avail)
}
})
// A caller who hangs up is also an ending — there is nobody left to serve, so
// the request is not offered anywhere else. That makes it a nil, and a nil
// owes the cents back. It is the one ending that reaches nil through the
// refusal path rather than around it, which is exactly how it was missed.
t.Run("a caller who hung up ends the request and gives the cents back", func(t *testing.T) {
const subject = "holdtest/hangup"
fam := &modelFamily{
name: "enso", prefix: "enso",
providerFn: func() *object.Provider {
// Nothing listens here: the dial fails and the pipe reaches the
// refusal path, where it finds the caller already gone.
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: "http://127.0.0.1:1"}
},
}
object.GlobalBalanceLedger.SetBalance(subject, 1000)
hold, ok := reserveBudget(subject, 100)
if !ok {
t.Fatal("reserve must succeed")
}
gone, hangUp := context.WithCancel(context.Background())
hangUp()
req := []byte(`{"model":"enso-flash","messages":[{"role":"user","content":"hi"}]}`)
c := visit(http.MethodPost, "/v1/chat/completions")
// Nobody is listening, and the handler learns that the way it does in
// production: off the request it was handed.
c.SetContext(gone)
if refused := c.pipeToFamily(fam, "chat/completions", "openai", "enso-flash", req, false, 0,
"org-a", nil, false, hold, time.Now()); refused != nil {
t.Fatalf("nobody is listening; the request must not be offered elsewhere, got %+v", refused)
}
if avail, _ := object.GlobalBalanceLedger.Available(subject); avail != 1000 {
t.Errorf("available = %d, want 1000 — the request is over and nothing will settle "+
"this hold, so a client that disconnects leaves its own balance short", avail)
}
})
t.Run("a served request settles its real cost", func(t *testing.T) {
refused, avail := pipeFamily(t, "holdtest/served", http.StatusOK,
`{"id":"x","object":"chat.completion","choices":[],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
if refused != nil {
t.Fatalf("a 200 is served, got %+v", refused)
}
if avail < 900 || avail > 1000 {
t.Errorf("available = %d, want the hold released and the real cost debited", avail)
}
})
}
// The family pipe is the OTHER site that learns from a refusal, and it is the
// one that served the outage. Driven end to end against a family answering the
// measured 402, so the org has to survive the whole call rather than only the
// helper it eventually reaches.
func TestTheFamilyPipeRestsTheCallersAccount(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
family := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusPaymentRequired)
_, _ = w.Write([]byte(`{"error":{"message":"Insufficient credits."}}`))
}))
defer family.Close()
fam := &modelFamily{
name: "enso", prefix: "enso",
providerFn: func() *object.Provider {
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: family.URL}
},
}
body := []byte(`{"model":"enso-flash","messages":[{"role":"user","content":"hi"}]}`)
c := visit("POST", "/v1/chat/completions")
refused := c.pipeToFamily(fam, "chat/completions", "openai", "enso-flash", body, false, 0,
"org-a", nil, false, nil, time.Now())
if len(refused) != 1 || refused[0].status != 402 {
t.Fatalf("the pipe must hand back the refusal so the request can move, got %+v", refused)
}
if !cooled.cooling(credential{"org-a", "enso"}) {
t.Error("the calling org's account must be resting after its 402")
}
if cooled.cooling(credential{"org-b", "enso"}) {
t.Error("a different tenant inherited the family's 402 — it spends a different credential")
}
if cooled.cooling(credential{"", "enso"}) {
t.Error("the 402 was filed against the family rather than against the account that earned it")
}
}
// Demotion is a preference, never a veto: a model whose only vendor is resting
// must still be attempted rather than refused.
func TestCoolingProviderIsStillTriedWhenItIsTheOnlyOne(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
cooled.demote(credential{"", "do-ai"}, time.Minute)
f := &fleet{}
f.install(t)
var w buffer
_, by, _, err := newAsk(route("do-ai"), &w, nil).serve()
if err != nil {
t.Fatalf("a resting vendor that is the only one must still be asked: %v", err)
}
if by.name != "do-ai" || len(f.asked) != 1 {
t.Errorf("asked %v, served by %q", f.asked, by.name)
}
}
// When everyone refuses the client gets one honest error naming who was asked
// and why the last one said no.
func TestAllRefusedYieldsAnHonestError(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{answer: map[string]error{
"enso": apiErr(402, "Insufficient credits."),
"do-ai": apiErr(429, "Platform overloaded"),
"fireworks": apiErr(503, "upstream is down for maintenance"),
}}
f.install(t)
var w buffer
_, _, tried, err := newAsk(route("enso", "do-ai", "fireworks"), &w, nil).serve()
if err == nil {
t.Fatal("every vendor refused; that must be an error, not an empty answer")
}
if len(tried) != 3 {
t.Fatalf("want 3 recorded refusals, got %d", len(tried))
}
msg := err.Error()
for _, leak := range []string{"enso", "402", "do-ai", "429", "fireworks", "503", "maintenance"} {
if strings.Contains(msg, leak) {
t.Errorf("error %q names %q — who refused and why is ours, in the log", msg, leak)
}
}
if got := statusOf(err); got != 503 {
t.Errorf("status = %d, want 503", got)
}
}
// The cap is what keeps a generous models.yaml from turning a bad afternoon into
// a minute-long request.
func TestAttemptCapHolds(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{answer: map[string]error{
"p1": apiErr(503, "down"), "p2": apiErr(503, "down"), "p3": apiErr(503, "down"),
"p4": apiErr(503, "down"), "p5": apiErr(503, "down"),
}}
f.install(t)
var w buffer
_, _, tried, err := newAsk(route("p1", "p2", "p3", "p4", "p5"), &w, nil).serve()
if err == nil {
t.Fatal("everyone was down; want an error")
}
if n := distinct(f.asked); n != maxProviders {
t.Errorf("asked %d vendors (%v), want %d — the walk must be bounded by the constant, "+
"not by however many fallbacks an operator typed", n, f.asked, maxProviders)
}
if len(tried) != maxProviders {
t.Errorf("recorded %d attempts, want %d", len(tried), maxProviders)
}
}
// A vendor that already refused elsewhere in this request (the family pipe) is
// not asked again, and it spends one of the three.
func TestPriorRefusalsAreCarried(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{}
f.install(t)
prior := []attempt{{provider: "enso", status: 402, fault: faultProvider,
err: errors.New("Insufficient credits.")}}
var w buffer
a := newAsk(route("enso", "do-ai"), &w, nil)
a.prior = prior
_, by, tried, err := a.serve()
if err != nil {
t.Fatalf("the alternate must serve: %v", err)
}
if by.name != "do-ai" || strings.Join(f.asked, ",") != "do-ai" {
t.Errorf("asked %v, served by %q — the family already refused; asking it again is waste", f.asked, by.name)
}
if len(tried) != 1 || tried[0].provider != "enso" {
t.Errorf("the earlier refusal must travel with the result so it gets recorded, got %+v", tried)
}
}
// A family model with no alternate declared: the honest answer names the vendor
// and the reason rather than forwarding a 402 that blames the customer.
func TestFamilyRefusalWithNoAlternateIsHonest(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{}
f.install(t)
prior := []attempt{{provider: "enso", status: 402, fault: faultProvider,
err: errors.New("Insufficient credits.")}}
var w buffer
a := newAsk(route("enso"), &w, nil) // family passthrough: primary only, no fallbacks
a.prior = prior
_, _, _, err := a.serve()
if err == nil {
t.Fatal("nobody could serve this; want an error")
}
if len(f.asked) != 0 {
t.Errorf("asked %v — the only vendor already refused", f.asked)
}
if strings.Contains(err.Error(), "Insufficient credits") || strings.Contains(err.Error(), "(402)") {
t.Errorf("error %q quotes the vendor — the caller is told only that the model is unavailable", err.Error())
}
if got := statusOf(err); got == 402 {
t.Error("the upstream's 402 must not reach the customer as a 402")
}
}
// ── the streaming decision ──────────────────────────────────────────────────
// Once a byte has reached the client the request is committed. Moving it would
// emit a second answer after the first one already started.
func TestNoFailoverOnceTheStreamHasOpened(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
var w buffer
f := &fleet{
answer: map[string]error{"do-ai": apiErr(500, "died mid-answer")},
wrote: map[string]string{"do-ai": "The answer is"},
}
f.install(t)
// The writer reports that bytes are on the wire as soon as it holds any —
// exactly what OpenAIWriter.StreamSent means.
_, _, tried, err := newAsk(route("do-ai", "anthropic"), &w, func() bool { return w.text != "" }).serve()
if err == nil {
t.Fatal("want the failure surfaced, not a second answer stitched on")
}
if len(f.asked) != 1 {
t.Errorf("asked %v — failing over after the first token duplicates output, "+
"which is worse than not failing over at all", f.asked)
}
if strings.Contains(strings.Join(f.asked, ","), "anthropic") {
t.Error("the alternate must NOT be asked once the client is already reading")
}
// The two assertions above are not enough on their own, and this test used to
// stop there. Removing the guard in serve() leaves them BOTH green, because the
// guard inside call() still refuses to dial a second vendor — but the loop has
// already entered that iteration, so it records an attempt against a vendor
// nobody contacted, and recordRefusals then writes a ledger row blaming it. It
// also replaces the real reason with our own "cannot retry".
if len(tried) != 1 || tried[0].provider != "do-ai" {
t.Errorf("tried = %+v, want exactly the one vendor that actually answered — "+
"a refusal recorded against a vendor we never dialled is a ledger row "+
"accusing an innocent party", tried)
}
if !strings.Contains(err.Error(), "died mid-answer") {
t.Errorf("err = %v, want the upstream's real reason. Once it reads as our own "+
"\"response partially written\", whoever debugs this is looking at us "+
"instead of at the vendor that died", err)
}
}
// The same rule applies to asking the SAME vendor again, which is the case the
// cross-provider guard above cannot see: a 429 or 503 is transient, so the retry
// loop would normally ask once more — but not after tokens have gone out. This
// is the guard inside the retry, and it is the one that stops a client reading
// the first half of an answer twice.
func TestNoSameProviderRetryOnceTheStreamHasOpened(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
// Two tries per vendor: without the guard the transient 503 below earns a
// second call, and the second call writes the same tokens again.
defaultRetryPolicy = retryPolicy{attempts: 2, base: time.Millisecond, max: time.Millisecond}
var w buffer
f := &fleet{
answer: map[string]error{"do-ai": apiErr(503, "died mid-answer")},
wrote: map[string]string{"do-ai": "Hello"},
}
f.install(t)
_, _, _, err := newAsk(route("do-ai"), &w, func() bool { return w.text != "" }).serve()
if err == nil {
t.Fatal("want the failure surfaced")
}
if len(f.asked) != 1 {
t.Errorf("do-ai was asked %d times — retrying a vendor that already emitted tokens "+
"sends the client the same partial answer twice", len(f.asked))
}
if w.text != "Hello" {
t.Errorf("client would receive %q, want %q — a replayed attempt duplicated output", w.text, "Hello")
}
}
// The other half of the streaming rule: while nothing has been flushed, the
// buffered bytes of a dead attempt must be discarded, or the next provider's
// answer is served glued to the dead one's half-sentence.
func TestFailedAttemptLeavesNothingInTheWriter(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{
answer: map[string]error{"do-ai": apiErr(503, "died after three tokens")},
wrote: map[string]string{"do-ai": "The capital of France is Berl", "anthropic": "The capital of France is Paris."},
}
f.install(t)
var w buffer
// sent stays false: nothing was flushed to the client, so the request is
// still movable — this is the non-streaming (buffered) case.
_, by, _, err := newAsk(route("do-ai", "anthropic"), &w, func() bool { return w.sent }).serve()
if err != nil {
t.Fatalf("the alternate must serve: %v", err)
}
if by.name != "anthropic" {
t.Fatalf("served by %q, want anthropic", by.name)
}
if w.text != "The capital of France is Paris." {
t.Errorf("client would receive %q — a failed attempt's partial text must never be "+
"prepended to the answer that actually served", w.text)
}
}
// A client that hung up must not be answered at somebody else's expense.
func TestNoFailoverAfterTheCallerHangsUp(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{answer: map[string]error{"do-ai": apiErr(503, "down")}}
f.install(t)
gone, hangUp := context.WithCancel(context.Background())
var w buffer
a := newAsk(route("do-ai", "anthropic"), &w, nil)
a.ctx = gone
hangUp()
_, _, _, err := a.serve()
if err == nil {
t.Fatal("want the cancellation surfaced")
}
if len(f.asked) != 0 {
t.Errorf("asked %v — nobody is listening; every one of those calls is money spent "+
"on an answer that goes nowhere", f.asked)
}
}
// The row that ANSWERED travels back, because that is the row whose credential
// was spent and the one the BYO fee decision must read.
func TestServedCarriesTheRowThatSpentTheCredential(t *testing.T) {
cooled.forget()
t.Cleanup(cooled.forget)
quick(t)
f := &fleet{answer: map[string]error{"do-ai": apiErr(429, "busy")}}
f.install(t)
var w buffer
_, by, _, err := newAsk(route("do-ai", "anthropic"), &w, nil).serve()
if err != nil {
t.Fatalf("unexpected: %v", err)
}
if by.row == nil {
t.Fatal("no row came back — billing would attribute the call to whatever auth resolved first")
}
if by.row.Name != "anthropic" {
t.Errorf("row = %q, want anthropic — the fallback served, so the fallback's row is the one that paid", by.row.Name)
}
}