mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
874 lines
32 KiB
Go
874 lines
32 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/model"
|
|
"github.com/hanzoai/ai/object"
|
|
)
|
|
|
|
// fleet scripts a set of providers: each name answers with an error, or serves.
|
|
// It records the order it was asked in, which is the property most of these
|
|
// tests are really about.
|
|
type fleet struct {
|
|
answer map[string]error // nil entry ⇒ that provider serves
|
|
asked []string
|
|
// wrote is text a provider emits into the writer before failing, to prove a
|
|
// dead provider's half-sentence never reaches the next one's answer.
|
|
wrote map[string]string
|
|
}
|
|
|
|
func (f *fleet) install(t *testing.T) {
|
|
t.Helper()
|
|
saved := callProvider
|
|
t.Cleanup(func() { callProvider = saved })
|
|
callProvider = func(org string, row *object.Provider, c candidate, question string,
|
|
w io.Writer, history, knowledge []*model.RawMessage, lang string,
|
|
) (*model.ModelResult, *object.Provider, error) {
|
|
f.asked = append(f.asked, c.provider)
|
|
if s := f.wrote[c.provider]; s != "" {
|
|
_, _ = w.Write([]byte(s))
|
|
}
|
|
if err, ok := f.answer[c.provider]; ok && err != nil {
|
|
return nil, &object.Provider{Owner: "admin", Name: c.provider}, err
|
|
}
|
|
return &model.ModelResult{ResponseTokenCount: 7},
|
|
&object.Provider{Owner: "admin", Name: c.provider}, nil
|
|
}
|
|
}
|
|
|
|
// buffer is the minimum a writer must be for the loop: it accumulates, and it
|
|
// can be told to forget. Mirrors OpenAIWriter/AnthropicWriter, which both
|
|
// APPEND on every Write.
|
|
type buffer struct {
|
|
text string
|
|
sent bool // set once bytes have reached the "client"
|
|
}
|
|
|
|
func (b *buffer) Write(p []byte) (int, error) { b.text += string(p); return len(p), nil }
|
|
func (b *buffer) Reset() { b.text = "" }
|
|
|
|
func quick(t *testing.T) {
|
|
t.Helper()
|
|
savedPolicy, savedConfig := defaultRetryPolicy, globalModelConfig
|
|
t.Cleanup(func() { defaultRetryPolicy, globalModelConfig = savedPolicy, savedConfig })
|
|
// One try per provider and no sleeping: these tests are about which vendor
|
|
// is asked, not about how patiently each one is waited on (retry_test covers
|
|
// that). The config must be cleared as well as the default, because
|
|
// currentRetryPolicy reads models.yaml FIRST — a config another test in this
|
|
// package left loaded would otherwise reinstate the real backoff.
|
|
globalModelConfig = nil
|
|
defaultRetryPolicy = retryPolicy{attempts: 1, base: time.Millisecond, max: time.Millisecond}
|
|
}
|
|
|
|
// distinct counts the VENDORS asked, which is what the cap bounds. The same
|
|
// vendor asked twice by the same-provider retry is one vendor.
|
|
func distinct(asked []string) int {
|
|
seen := map[string]bool{}
|
|
for _, a := range asked {
|
|
seen[a] = true
|
|
}
|
|
return len(seen)
|
|
}
|
|
|
|
func newAsk(r *modelRoute, w io.Writer, sent func() bool) ask {
|
|
return ask{route: r, model: "test-model", question: "hi", writer: w, sent: sent}
|
|
}
|
|
|
|
// THE OUTAGE. One vendor's account is empty; the product must not go dark.
|
|
func TestPaymentRequiredFailsOverAndTheNextProviderServes(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{
|
|
"enso": apiErr(402, "Insufficient credits."),
|
|
}}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
res, by, tried, err := newAsk(route("enso", "do-ai"), &w, nil).serve()
|
|
if err != nil {
|
|
t.Fatalf("a 402 from one vendor must not fail the request: %v", err)
|
|
}
|
|
if res == nil || by.name != "do-ai" {
|
|
t.Fatalf("served by %q, want do-ai", by.name)
|
|
}
|
|
if strings.Join(f.asked, ",") != "enso,do-ai" {
|
|
t.Errorf("asked %v, want [enso do-ai]", f.asked)
|
|
}
|
|
if len(tried) != 1 || tried[0].provider != "enso" || tried[0].status != 402 {
|
|
t.Fatalf("the refusal must be reported so the empty account gets noticed, got %+v", tried)
|
|
}
|
|
if tried[0].fault != faultProvider {
|
|
t.Errorf("a 402 is the vendor's fault, got %s", tried[0].fault)
|
|
}
|
|
}
|
|
|
|
// The opposite guard: a broken request must cost exactly one upstream call.
|
|
func TestBadRequestDoesNotFailOver(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{
|
|
"do-ai": apiErr(400, "invalid 'messages': empty"),
|
|
}}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
_, _, tried, err := newAsk(route("do-ai", "anthropic", "fireworks"), &w, nil).serve()
|
|
if err == nil {
|
|
t.Fatal("a 400 must surface, not be papered over")
|
|
}
|
|
if len(f.asked) != 1 {
|
|
t.Errorf("asked %v — a malformed request fails identically everywhere; retrying it "+
|
|
"turns one error into %d and bills for each", f.asked, len(f.asked))
|
|
}
|
|
if !strings.Contains(err.Error(), "invalid 'messages'") {
|
|
t.Errorf("the caller must get the REAL reason, got %v", err)
|
|
}
|
|
if len(tried) != 1 || tried[0].fault != faultRequest {
|
|
t.Errorf("want one request-fault attempt, got %+v", tried)
|
|
}
|
|
}
|
|
|
|
// A DEAD CREDENTIAL FAILS OVER, AND IS STILL LOUD. Those are two jobs, and this
|
|
// used to do the second by refusing to do the first.
|
|
//
|
|
// The refusal is a fact about the VENDOR — our key will not open it — and the
|
|
// caller's own credential was checked at our edge long before any vendor was
|
|
// dialled, so nothing about their request is wrong. announce() counts
|
|
// cloud_supply_refused{reason="credential"} and logs at ERROR whichever way the
|
|
// request goes, so surfacing costs the failover nothing.
|
|
//
|
|
// Measured, the old coupling cost exactly what it was meant to prevent:
|
|
// DO_AI_API_KEY died and 93 routes answered 401 to CUSTOMERS for days — louder
|
|
// than any alert and aimed at the wrong people — while the metric that was meant
|
|
// to raise a hand had been sitting there the whole time.
|
|
func TestUnauthorizedFailsOverAndIsStillLoud(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{"do-ai": apiErr(401, "invalid api key")}}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
_, served, tried, err := newAsk(route("do-ai", "anthropic"), &w, nil).serve()
|
|
if err != nil {
|
|
t.Fatalf("a dead key of ours must not become the customer's error: %v", err)
|
|
}
|
|
if served.name != "anthropic" {
|
|
t.Errorf("served by %q, want anthropic — the request must reach a vendor that can answer", served.name)
|
|
}
|
|
if len(f.asked) != 2 {
|
|
t.Errorf("asked %v — the dead vendor is tried once, then the request moves", f.asked)
|
|
}
|
|
if len(tried) != 1 || tried[0].status != 401 {
|
|
t.Errorf("the 401 must be RECORDED as an attempt, not swallowed: %+v", tried)
|
|
}
|
|
// Rested like an empty account: both need a human, and neither clears itself
|
|
// in seconds, so the fleet must not return to it every few seconds.
|
|
if !cooled.cooling(credential{"", "do-ai"}) {
|
|
t.Error("a vendor whose key is dead must be demoted, or every request pays its 401 first")
|
|
}
|
|
}
|
|
|
|
// The point of demotion: the second request must not pay the dead vendor's
|
|
// round trip all over again.
|
|
func TestDemotedProviderIsSkippedWhileCoolingDown(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{"enso": apiErr(402, "Insufficient credits.")}}
|
|
f.install(t)
|
|
|
|
var w1 buffer
|
|
if _, _, _, err := newAsk(route("enso", "do-ai"), &w1, nil).serve(); err != nil {
|
|
t.Fatalf("first request: %v", err)
|
|
}
|
|
if !cooled.cooling(credential{"", "enso"}) {
|
|
t.Fatal("a vendor that answered 402 must be demoted")
|
|
}
|
|
|
|
f.asked = nil
|
|
var w2 buffer
|
|
_, by, _, err := newAsk(route("enso", "do-ai"), &w2, nil).serve()
|
|
if err != nil {
|
|
t.Fatalf("second request: %v", err)
|
|
}
|
|
if by.name != "do-ai" {
|
|
t.Fatalf("served by %q, want do-ai", by.name)
|
|
}
|
|
if strings.Join(f.asked, ",") != "do-ai" {
|
|
t.Errorf("asked %v, want [do-ai] only — without the cooldown every single request "+
|
|
"pays the empty vendor's latency before getting a real answer", f.asked)
|
|
}
|
|
}
|
|
|
|
// The loop must file its demotion against the account that earned it.
|
|
//
|
|
// This is the WRITE side of the property TestOneTenantsEmptyAccountIsNotAnothersOutage
|
|
// asserts on the READ side. Both are needed: with only one, dropping the org at
|
|
// the other end restores the cross-tenant leak with every other test still green.
|
|
func TestTheLoopDemotesTheAccountNotTheVendor(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{"enso": apiErr(402, "Insufficient credits.")}}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
a := newAsk(route("enso", "do-ai"), &w, nil)
|
|
a.org = "org-a"
|
|
if _, _, _, err := a.serve(); err != nil {
|
|
t.Fatalf("serve: %v", err)
|
|
}
|
|
|
|
if !cooled.cooling(credential{"org-a", "enso"}) {
|
|
t.Error("the org whose key answered 402 must be resting — otherwise its next request " +
|
|
"pays the empty account's round trip all over again")
|
|
}
|
|
if cooled.cooling(credential{"org-b", "enso"}) {
|
|
t.Error("another tenant inherited the penalty; its request spends a different credential")
|
|
}
|
|
if cooled.cooling(credential{"", "enso"}) {
|
|
t.Error("the refusal was filed against the VENDOR rather than the account that earned it, " +
|
|
"which is the whole cross-tenant leak")
|
|
}
|
|
}
|
|
|
|
// Both sites that learn from a refusal learn the same thing, so both are asked
|
|
// the same question here. A refusal rests the ACCOUNT that answered it; no other
|
|
// tenant and no other vendor inherits anything.
|
|
func TestARefusalRestsOneAccount(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
|
|
if d := cooled.rest("org-a", "enso", apiErr(402, "Insufficient credits.")); d != coolBroke {
|
|
t.Fatalf("an empty account rests %v, want %v", d, coolBroke)
|
|
}
|
|
if !cooled.cooling(credential{"org-a", "enso"}) {
|
|
t.Error("the account that refused must be resting")
|
|
}
|
|
if cooled.cooling(credential{"org-b", "enso"}) {
|
|
t.Error("another tenant inherited it")
|
|
}
|
|
if cooled.cooling(credential{"org-a", "do-ai"}) {
|
|
t.Error("another vendor inherited it")
|
|
}
|
|
|
|
// A DEAD KEY RESTS AS LONG AS AN EMPTY ACCOUNT. It is the same operational
|
|
// class — one needs a human with a credit card, the other a human with a new
|
|
// key — and neither clears itself in seconds, so a short rest would put every
|
|
// request back on a vendor that cannot answer.
|
|
if d := cooled.rest("org-a", "do-ai", apiErr(401, "invalid api key")); d != coolBroke {
|
|
t.Errorf("a 401 rested the vendor for %v, want %v", d, coolBroke)
|
|
}
|
|
if !cooled.cooling(credential{"org-a", "do-ai"}) {
|
|
t.Error("the vendor whose key is dead must be resting")
|
|
}
|
|
// And it is still ONE account's problem: another tenant's credential for the
|
|
// same vendor is untouched.
|
|
if cooled.cooling(credential{"org-b", "do-ai"}) {
|
|
t.Error("another tenant inherited our dead key")
|
|
}
|
|
}
|
|
|
|
// breaks is an http.ResponseWriter that delivers `ok` writes and then fails —
|
|
// the shape of an upstream dying mid-answer, after the client already has bytes.
|
|
type breaks struct {
|
|
*httptest.ResponseRecorder
|
|
ok int
|
|
}
|
|
|
|
func breaksAfter(ok int) *breaks { return &breaks{httptest.NewRecorder(), ok} }
|
|
|
|
// tears delivers PART of a write and then reports the failure — a connection that
|
|
// broke mid-event. n > 0 with a non-nil error is the case the flag has to get
|
|
// right: those bytes are on the wire whatever the error says about the rest.
|
|
type tears struct{ *httptest.ResponseRecorder }
|
|
|
|
func (t *tears) Write(p []byte) (int, error) {
|
|
n, _ := t.ResponseRecorder.Write(p[:len(p)/2])
|
|
return n, errors.New("connection reset by peer")
|
|
}
|
|
|
|
func (b *breaks) Write(p []byte) (int, error) {
|
|
if b.ok <= 0 {
|
|
return 0, errors.New("connection reset by peer")
|
|
}
|
|
b.ok--
|
|
return b.ResponseRecorder.Write(p)
|
|
}
|
|
|
|
// A8. StreamSent is the whole movability decision, and it was set at the END of
|
|
// Write — after message_start and content_block_start had already gone out.
|
|
//
|
|
// So there was a window, 185 measured bytes wide, in which the client held the
|
|
// opening of one vendor's answer and the loop still believed the request could be
|
|
// offered to another. It would then have received the whole of a second vendor's
|
|
// answer glued to the first one's first breath.
|
|
func TestTheAnthropicWriterSaysSentAsSoonAsBytesGoOut(t *testing.T) {
|
|
// Deliver message_start and content_block_start, then fail — exactly the gap.
|
|
rec := breaksAfter(2)
|
|
w := &AnthropicWriter{
|
|
Writer: bufio.NewWriter(rec),
|
|
RequestID: "req-1",
|
|
Stream: true,
|
|
Cleaner: *NewCleaner(6),
|
|
Model: "claude-x",
|
|
}
|
|
|
|
_, err := w.Write([]byte("2 + 2 = 4"))
|
|
if err == nil {
|
|
t.Fatal("the scripted writer must fail on the delta")
|
|
}
|
|
if rec.Body.Len() == 0 {
|
|
t.Fatal("nothing was written; this test proves nothing")
|
|
}
|
|
if !w.StreamSent {
|
|
t.Errorf("%d bytes reached the client and StreamSent is false — the loop would move "+
|
|
"this request and the customer would read two vendors' answers spliced together",
|
|
rec.Body.Len())
|
|
}
|
|
if !strings.Contains(rec.Body.String(), "message_start") {
|
|
t.Errorf("expected the header events on the wire, got:\n%s", rec.Body.String())
|
|
}
|
|
}
|
|
|
|
// A half-written event is still a written event. The bytes are gone; whether the
|
|
// call that sent them also returned an error changes nothing about that.
|
|
func TestAPartialWriteCountsAsSent(t *testing.T) {
|
|
rec := &tears{httptest.NewRecorder()}
|
|
w := &AnthropicWriter{
|
|
Writer: bufio.NewWriter(rec),
|
|
RequestID: "req-1",
|
|
Stream: true,
|
|
Cleaner: *NewCleaner(6),
|
|
Model: "claude-x",
|
|
}
|
|
if _, err := w.Write([]byte("2 + 2 = 4")); err == nil {
|
|
t.Fatal("the scripted writer must report the break")
|
|
}
|
|
if rec.Body.Len() == 0 {
|
|
t.Fatal("the scripted writer delivered nothing; this test proves nothing")
|
|
}
|
|
if !w.StreamSent {
|
|
t.Errorf("%d bytes are on the wire after a failed write and StreamSent is false — "+
|
|
"reading the error instead of the byte count is how a half-sent answer "+
|
|
"gets a second vendor's answer appended to it", rec.Body.Len())
|
|
}
|
|
}
|
|
|
|
// The other half: a writer that has sent nothing is still movable, or a single
|
|
// transient failure on the first byte would strand every request it touched.
|
|
func TestTheAnthropicWriterIsMovableUntilItWrites(t *testing.T) {
|
|
rec := breaksAfter(0)
|
|
w := &AnthropicWriter{
|
|
Writer: bufio.NewWriter(rec),
|
|
RequestID: "req-1",
|
|
Stream: true,
|
|
Cleaner: *NewCleaner(6),
|
|
Model: "claude-x",
|
|
}
|
|
if _, err := w.Write([]byte("2 + 2 = 4")); err == nil {
|
|
t.Fatal("the scripted writer must fail on the first event")
|
|
}
|
|
if rec.Body.Len() != 0 {
|
|
t.Fatalf("the scripted writer delivered %d bytes; it must deliver none", rec.Body.Len())
|
|
}
|
|
if w.StreamSent {
|
|
t.Error("nothing reached the client, so this request is still movable — saying " +
|
|
"otherwise turns one transient failure into a refusal we could have routed around")
|
|
}
|
|
}
|
|
|
|
// pipeFamily drives the real family pipe against a family answering `status`,
|
|
// with a funded reservation, and reports what came back and what the ledger says.
|
|
func pipeFamily(t *testing.T, subject string, status int, body string) (refused []attempt, avail int64) {
|
|
t.Helper()
|
|
family := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(status)
|
|
_, _ = w.Write([]byte(body))
|
|
}))
|
|
defer family.Close()
|
|
|
|
fam := &modelFamily{
|
|
name: "enso", prefix: "enso",
|
|
providerFn: func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: family.URL}
|
|
},
|
|
}
|
|
|
|
object.GlobalBalanceLedger.SetBalance(subject, 1000)
|
|
hold, ok := reserveBudget(subject, 100)
|
|
if !ok {
|
|
t.Fatal("reserve against a funded subject must succeed")
|
|
}
|
|
if a, _ := object.GlobalBalanceLedger.Available(subject); a != 900 {
|
|
t.Fatalf("available after reserve = %d, want 900", a)
|
|
}
|
|
|
|
req := []byte(`{"model":"enso-flash","messages":[{"role":"user","content":"hi"}]}`)
|
|
c := visit("POST", "/v1/chat/completions")
|
|
|
|
refused = c.pipeToFamily(fam, "chat/completions", "openai", "enso-flash", req, false,
|
|
"org-a", nil, false, hold, time.Now())
|
|
avail, _ = object.GlobalBalanceLedger.Available(subject)
|
|
return refused, avail
|
|
}
|
|
|
|
// A1. What the pipe returns and what it owes the reservation are the same fact,
|
|
// so they are asserted together.
|
|
//
|
|
// nil means the request is OVER: nothing downstream will run, so the cents must
|
|
// be back. A refusal means the request is still MOVING: the cents must still be
|
|
// held, because whichever provider ends up serving it has to be billed and
|
|
// settle only fires once.
|
|
//
|
|
// The leak this pins: a 400 from an embeddings family returned nil, and nothing
|
|
// anywhere released the hold. The customer's balance simply stayed short.
|
|
func TestTheHoldFollowsTheAnswer(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
|
|
t.Run("a refusal nobody else can fix ends the request and gives the cents back", func(t *testing.T) {
|
|
refused, avail := pipeFamily(t, "holdtest/req-fault", http.StatusBadRequest,
|
|
`{"error":{"message":"invalid 'messages': empty"}}`)
|
|
if refused != nil {
|
|
t.Fatalf("a 400 is the request's fault; no other vendor changes it, got %+v", refused)
|
|
}
|
|
if avail != 1000 {
|
|
t.Errorf("available = %d, want 1000 — the request is over and nothing downstream "+
|
|
"will ever settle this hold, so the customer is short by the reservation", avail)
|
|
}
|
|
})
|
|
|
|
t.Run("a refusal somebody else can fix keeps the cents held", func(t *testing.T) {
|
|
refused, avail := pipeFamily(t, "holdtest/provider-fault", http.StatusPaymentRequired,
|
|
`{"error":{"message":"Insufficient credits."}}`)
|
|
if len(refused) != 1 {
|
|
t.Fatalf("a 402 is the vendor's fault and the request must stay movable, got %+v", refused)
|
|
}
|
|
if avail != 900 {
|
|
t.Errorf("available = %d, want 900 — releasing here is one-shot, and whichever "+
|
|
"provider serves this next would then be billed nothing", avail)
|
|
}
|
|
})
|
|
|
|
// A caller who hangs up is also an ending — there is nobody left to serve, so
|
|
// the request is not offered anywhere else. That makes it a nil, and a nil
|
|
// owes the cents back. It is the one ending that reaches nil through the
|
|
// refusal path rather than around it, which is exactly how it was missed.
|
|
t.Run("a caller who hung up ends the request and gives the cents back", func(t *testing.T) {
|
|
const subject = "holdtest/hangup"
|
|
fam := &modelFamily{
|
|
name: "enso", prefix: "enso",
|
|
providerFn: func() *object.Provider {
|
|
// Nothing listens here: the dial fails and the pipe reaches the
|
|
// refusal path, where it finds the caller already gone.
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: "http://127.0.0.1:1"}
|
|
},
|
|
}
|
|
object.GlobalBalanceLedger.SetBalance(subject, 1000)
|
|
hold, ok := reserveBudget(subject, 100)
|
|
if !ok {
|
|
t.Fatal("reserve must succeed")
|
|
}
|
|
|
|
gone, hangUp := context.WithCancel(context.Background())
|
|
hangUp()
|
|
req := []byte(`{"model":"enso-flash","messages":[{"role":"user","content":"hi"}]}`)
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
// Nobody is listening, and the handler learns that the way it does in
|
|
// production: off the request it was handed.
|
|
c.SetContext(gone)
|
|
|
|
if refused := c.pipeToFamily(fam, "chat/completions", "openai", "enso-flash", req, false,
|
|
"org-a", nil, false, hold, time.Now()); refused != nil {
|
|
t.Fatalf("nobody is listening; the request must not be offered elsewhere, got %+v", refused)
|
|
}
|
|
if avail, _ := object.GlobalBalanceLedger.Available(subject); avail != 1000 {
|
|
t.Errorf("available = %d, want 1000 — the request is over and nothing will settle "+
|
|
"this hold, so a client that disconnects leaves its own balance short", avail)
|
|
}
|
|
})
|
|
|
|
t.Run("a served request settles its real cost", func(t *testing.T) {
|
|
refused, avail := pipeFamily(t, "holdtest/served", http.StatusOK,
|
|
`{"id":"x","object":"chat.completion","choices":[],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
|
|
if refused != nil {
|
|
t.Fatalf("a 200 is served, got %+v", refused)
|
|
}
|
|
if avail < 900 || avail > 1000 {
|
|
t.Errorf("available = %d, want the hold released and the real cost debited", avail)
|
|
}
|
|
})
|
|
}
|
|
|
|
// The family pipe is the OTHER site that learns from a refusal, and it is the
|
|
// one that served the outage. Driven end to end against a family answering the
|
|
// measured 402, so the org has to survive the whole call rather than only the
|
|
// helper it eventually reaches.
|
|
func TestTheFamilyPipeRestsTheCallersAccount(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
|
|
family := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusPaymentRequired)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"Insufficient credits."}}`))
|
|
}))
|
|
defer family.Close()
|
|
|
|
fam := &modelFamily{
|
|
name: "enso", prefix: "enso",
|
|
providerFn: func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: family.URL}
|
|
},
|
|
}
|
|
|
|
body := []byte(`{"model":"enso-flash","messages":[{"role":"user","content":"hi"}]}`)
|
|
c := visit("POST", "/v1/chat/completions")
|
|
|
|
refused := c.pipeToFamily(fam, "chat/completions", "openai", "enso-flash", body, false,
|
|
"org-a", nil, false, nil, time.Now())
|
|
|
|
if len(refused) != 1 || refused[0].status != 402 {
|
|
t.Fatalf("the pipe must hand back the refusal so the request can move, got %+v", refused)
|
|
}
|
|
if !cooled.cooling(credential{"org-a", "enso"}) {
|
|
t.Error("the calling org's account must be resting after its 402")
|
|
}
|
|
if cooled.cooling(credential{"org-b", "enso"}) {
|
|
t.Error("a different tenant inherited the family's 402 — it spends a different credential")
|
|
}
|
|
if cooled.cooling(credential{"", "enso"}) {
|
|
t.Error("the 402 was filed against the family rather than against the account that earned it")
|
|
}
|
|
}
|
|
|
|
// Demotion is a preference, never a veto: a model whose only vendor is resting
|
|
// must still be attempted rather than refused.
|
|
func TestCoolingProviderIsStillTriedWhenItIsTheOnlyOne(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
cooled.demote(credential{"", "do-ai"}, time.Minute)
|
|
f := &fleet{}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
_, by, _, err := newAsk(route("do-ai"), &w, nil).serve()
|
|
if err != nil {
|
|
t.Fatalf("a resting vendor that is the only one must still be asked: %v", err)
|
|
}
|
|
if by.name != "do-ai" || len(f.asked) != 1 {
|
|
t.Errorf("asked %v, served by %q", f.asked, by.name)
|
|
}
|
|
}
|
|
|
|
// When everyone refuses the client gets one honest error naming who was asked
|
|
// and why the last one said no.
|
|
func TestAllRefusedYieldsAnHonestError(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{
|
|
"enso": apiErr(402, "Insufficient credits."),
|
|
"do-ai": apiErr(429, "Platform overloaded"),
|
|
"fireworks": apiErr(503, "upstream is down for maintenance"),
|
|
}}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
_, _, tried, err := newAsk(route("enso", "do-ai", "fireworks"), &w, nil).serve()
|
|
if err == nil {
|
|
t.Fatal("every vendor refused; that must be an error, not an empty answer")
|
|
}
|
|
if len(tried) != 3 {
|
|
t.Fatalf("want 3 recorded refusals, got %d", len(tried))
|
|
}
|
|
msg := err.Error()
|
|
for _, want := range []string{"enso", "402", "do-ai", "429", "fireworks", "503", "maintenance"} {
|
|
if !strings.Contains(msg, want) {
|
|
t.Errorf("error %q omits %q — the last reason is the one that tells an operator what to fix", msg, want)
|
|
}
|
|
}
|
|
if got := statusOf(err); got != 503 {
|
|
t.Errorf("status = %d, want 503", got)
|
|
}
|
|
}
|
|
|
|
// The cap is what keeps a generous models.yaml from turning a bad afternoon into
|
|
// a minute-long request.
|
|
func TestAttemptCapHolds(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{
|
|
"p1": apiErr(503, "down"), "p2": apiErr(503, "down"), "p3": apiErr(503, "down"),
|
|
"p4": apiErr(503, "down"), "p5": apiErr(503, "down"),
|
|
}}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
_, _, tried, err := newAsk(route("p1", "p2", "p3", "p4", "p5"), &w, nil).serve()
|
|
if err == nil {
|
|
t.Fatal("everyone was down; want an error")
|
|
}
|
|
if n := distinct(f.asked); n != maxProviders {
|
|
t.Errorf("asked %d vendors (%v), want %d — the walk must be bounded by the constant, "+
|
|
"not by however many fallbacks an operator typed", n, f.asked, maxProviders)
|
|
}
|
|
if len(tried) != maxProviders {
|
|
t.Errorf("recorded %d attempts, want %d", len(tried), maxProviders)
|
|
}
|
|
}
|
|
|
|
// A vendor that already refused elsewhere in this request (the family pipe) is
|
|
// not asked again, and it spends one of the three.
|
|
func TestPriorRefusalsAreCarried(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{}
|
|
f.install(t)
|
|
|
|
prior := []attempt{{provider: "enso", status: 402, fault: faultProvider,
|
|
err: errors.New("Insufficient credits.")}}
|
|
|
|
var w buffer
|
|
a := newAsk(route("enso", "do-ai"), &w, nil)
|
|
a.prior = prior
|
|
_, by, tried, err := a.serve()
|
|
if err != nil {
|
|
t.Fatalf("the alternate must serve: %v", err)
|
|
}
|
|
if by.name != "do-ai" || strings.Join(f.asked, ",") != "do-ai" {
|
|
t.Errorf("asked %v, served by %q — the family already refused; asking it again is waste", f.asked, by.name)
|
|
}
|
|
if len(tried) != 1 || tried[0].provider != "enso" {
|
|
t.Errorf("the earlier refusal must travel with the result so it gets recorded, got %+v", tried)
|
|
}
|
|
}
|
|
|
|
// A family model with no alternate declared: the honest answer names the vendor
|
|
// and the reason rather than forwarding a 402 that blames the customer.
|
|
func TestFamilyRefusalWithNoAlternateIsHonest(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{}
|
|
f.install(t)
|
|
|
|
prior := []attempt{{provider: "enso", status: 402, fault: faultProvider,
|
|
err: errors.New("Insufficient credits.")}}
|
|
|
|
var w buffer
|
|
a := newAsk(route("enso"), &w, nil) // family passthrough: primary only, no fallbacks
|
|
a.prior = prior
|
|
_, _, _, err := a.serve()
|
|
if err == nil {
|
|
t.Fatal("nobody could serve this; want an error")
|
|
}
|
|
if len(f.asked) != 0 {
|
|
t.Errorf("asked %v — the only vendor already refused", f.asked)
|
|
}
|
|
if !strings.Contains(err.Error(), "Insufficient credits") || !strings.Contains(err.Error(), "enso") {
|
|
t.Errorf("error %q must name the vendor and quote its reason", err.Error())
|
|
}
|
|
if got := statusOf(err); got == 402 {
|
|
t.Error("the upstream's 402 must not reach the customer as a 402")
|
|
}
|
|
}
|
|
|
|
// ── the streaming decision ──────────────────────────────────────────────────
|
|
|
|
// Once a byte has reached the client the request is committed. Moving it would
|
|
// emit a second answer after the first one already started.
|
|
func TestNoFailoverOnceTheStreamHasOpened(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
var w buffer
|
|
f := &fleet{
|
|
answer: map[string]error{"do-ai": apiErr(500, "died mid-answer")},
|
|
wrote: map[string]string{"do-ai": "The answer is"},
|
|
}
|
|
f.install(t)
|
|
|
|
// The writer reports that bytes are on the wire as soon as it holds any —
|
|
// exactly what OpenAIWriter.StreamSent means.
|
|
_, _, tried, err := newAsk(route("do-ai", "anthropic"), &w, func() bool { return w.text != "" }).serve()
|
|
if err == nil {
|
|
t.Fatal("want the failure surfaced, not a second answer stitched on")
|
|
}
|
|
if len(f.asked) != 1 {
|
|
t.Errorf("asked %v — failing over after the first token duplicates output, "+
|
|
"which is worse than not failing over at all", f.asked)
|
|
}
|
|
if strings.Contains(strings.Join(f.asked, ","), "anthropic") {
|
|
t.Error("the alternate must NOT be asked once the client is already reading")
|
|
}
|
|
|
|
// The two assertions above are not enough on their own, and this test used to
|
|
// stop there. Removing the guard in serve() leaves them BOTH green, because the
|
|
// guard inside call() still refuses to dial a second vendor — but the loop has
|
|
// already entered that iteration, so it records an attempt against a vendor
|
|
// nobody contacted, and recordRefusals then writes a ledger row blaming it. It
|
|
// also replaces the real reason with our own "cannot retry".
|
|
if len(tried) != 1 || tried[0].provider != "do-ai" {
|
|
t.Errorf("tried = %+v, want exactly the one vendor that actually answered — "+
|
|
"a refusal recorded against a vendor we never dialled is a ledger row "+
|
|
"accusing an innocent party", tried)
|
|
}
|
|
if !strings.Contains(err.Error(), "died mid-answer") {
|
|
t.Errorf("err = %v, want the upstream's real reason. Once it reads as our own "+
|
|
"\"response partially written\", whoever debugs this is looking at us "+
|
|
"instead of at the vendor that died", err)
|
|
}
|
|
}
|
|
|
|
// The same rule applies to asking the SAME vendor again, which is the case the
|
|
// cross-provider guard above cannot see: a 429 or 503 is transient, so the retry
|
|
// loop would normally ask once more — but not after tokens have gone out. This
|
|
// is the guard inside the retry, and it is the one that stops a client reading
|
|
// the first half of an answer twice.
|
|
func TestNoSameProviderRetryOnceTheStreamHasOpened(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
// Two tries per vendor: without the guard the transient 503 below earns a
|
|
// second call, and the second call writes the same tokens again.
|
|
defaultRetryPolicy = retryPolicy{attempts: 2, base: time.Millisecond, max: time.Millisecond}
|
|
|
|
var w buffer
|
|
f := &fleet{
|
|
answer: map[string]error{"do-ai": apiErr(503, "died mid-answer")},
|
|
wrote: map[string]string{"do-ai": "Hello"},
|
|
}
|
|
f.install(t)
|
|
|
|
_, _, _, err := newAsk(route("do-ai"), &w, func() bool { return w.text != "" }).serve()
|
|
if err == nil {
|
|
t.Fatal("want the failure surfaced")
|
|
}
|
|
if len(f.asked) != 1 {
|
|
t.Errorf("do-ai was asked %d times — retrying a vendor that already emitted tokens "+
|
|
"sends the client the same partial answer twice", len(f.asked))
|
|
}
|
|
if w.text != "Hello" {
|
|
t.Errorf("client would receive %q, want %q — a replayed attempt duplicated output", w.text, "Hello")
|
|
}
|
|
}
|
|
|
|
// The other half of the streaming rule: while nothing has been flushed, the
|
|
// buffered bytes of a dead attempt must be discarded, or the next provider's
|
|
// answer is served glued to the dead one's half-sentence.
|
|
func TestFailedAttemptLeavesNothingInTheWriter(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{
|
|
answer: map[string]error{"do-ai": apiErr(503, "died after three tokens")},
|
|
wrote: map[string]string{"do-ai": "The capital of France is Berl", "anthropic": "The capital of France is Paris."},
|
|
}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
// sent stays false: nothing was flushed to the client, so the request is
|
|
// still movable — this is the non-streaming (buffered) case.
|
|
_, by, _, err := newAsk(route("do-ai", "anthropic"), &w, func() bool { return w.sent }).serve()
|
|
if err != nil {
|
|
t.Fatalf("the alternate must serve: %v", err)
|
|
}
|
|
if by.name != "anthropic" {
|
|
t.Fatalf("served by %q, want anthropic", by.name)
|
|
}
|
|
if w.text != "The capital of France is Paris." {
|
|
t.Errorf("client would receive %q — a failed attempt's partial text must never be "+
|
|
"prepended to the answer that actually served", w.text)
|
|
}
|
|
}
|
|
|
|
// A client that hung up must not be answered at somebody else's expense.
|
|
func TestNoFailoverAfterTheCallerHangsUp(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{"do-ai": apiErr(503, "down")}}
|
|
f.install(t)
|
|
|
|
gone, hangUp := context.WithCancel(context.Background())
|
|
var w buffer
|
|
a := newAsk(route("do-ai", "anthropic"), &w, nil)
|
|
a.ctx = gone
|
|
hangUp()
|
|
|
|
_, _, _, err := a.serve()
|
|
if err == nil {
|
|
t.Fatal("want the cancellation surfaced")
|
|
}
|
|
if len(f.asked) != 0 {
|
|
t.Errorf("asked %v — nobody is listening; every one of those calls is money spent "+
|
|
"on an answer that goes nowhere", f.asked)
|
|
}
|
|
}
|
|
|
|
// The row that ANSWERED travels back, because that is the row whose credential
|
|
// was spent and the one the BYO fee decision must read.
|
|
func TestServedCarriesTheRowThatSpentTheCredential(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{"do-ai": apiErr(429, "busy")}}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
_, by, _, err := newAsk(route("do-ai", "anthropic"), &w, nil).serve()
|
|
if err != nil {
|
|
t.Fatalf("unexpected: %v", err)
|
|
}
|
|
if by.row == nil {
|
|
t.Fatal("no row came back — billing would attribute the call to whatever auth resolved first")
|
|
}
|
|
if by.row.Name != "anthropic" {
|
|
t.Errorf("row = %q, want anthropic — the fallback served, so the fallback's row is the one that paid", by.row.Name)
|
|
}
|
|
}
|