mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
A vendor answered 402 and the whole product went quiet. Five upstream credentials were configured and funded; none was tried, because the model was served by the family pipe, which had no notion of an alternate at all, and the failover the QueryText path did have only engaged for routes that declared fallbacks. The missing idea was fault attribution. A refusal is either about the VENDOR (out of money, busy, broken, unreachable — somebody else can serve this) or about the REQUEST (malformed, unknown model, our credential rejected — everybody refuses it identically). Everything else follows from that one value, so it is one value, faultOf, and not a substring list re-derived at each branch. It reads the typed status first and consults text only where no status exists. 401 and plain 403 move to the request side: routing around our own dead key drains traffic onto whoever still works and nobody learns until the bill. The narrow "this account may not use this model" 403 stays a vendor fault — that is a fact about a catalogue, not a credential, and the do-ai to anthropic fallback exists for it. A vendor that refuses is demoted: sorted to the back for 15s when busy, 5min when out of money, so the next request does not pay its round trip first. Demotion reorders, never removes — a model whose only vendor is resting is still asked, and expiry re-probes a recovered one for free. The walk is capped at three vendors, the width of a declared route, so worst-case latency is a property of the constant rather than of whatever an operator typed into models.yaml. Every refusal is announced and written to the usage ledger as its own status, because a failover nobody can see leaves the empty account empty — that was the second half of the outage. When everyone refuses the client gets 503 naming who was asked and quoting the last reason, never the upstream's own 402, which would tell the customer they owe money. Streaming: a request is movable only until the first byte reaches the client. The family pipe reads the status before writing its header, so a streaming request is as movable as a buffered one at exactly that point; after it, nothing moves. Both writers accumulate, so a failed attempt is now cleared before the next one runs — otherwise the fallback's answer was served glued to the dead vendor's half sentence. Two money bugs found in the path while here: failover resolved providers globally, so a customer's connected key was silently replaced by the platform's while the call was still billed as bring-your-own at 1%; and the family pipe's transport-error path settled the budget hold, which is one-shot, so a request that later succeeded would never have been charged. Both resolved by carrying the row that actually spent a credential.
185 lines
6.2 KiB
Go
185 lines
6.2 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// fast policy: exercise the logic without sleeping through a real backoff.
|
|
var testPolicy = retryPolicy{attempts: 3, base: time.Millisecond, max: 2 * time.Millisecond}
|
|
|
|
func TestIsTransientError_SeparatesWaitFromCascade(t *testing.T) {
|
|
// Waiting fixes these: the provider is busy now, fine in a moment.
|
|
transient := []string{
|
|
"429 Too Many Requests",
|
|
`{"message":"Platform overloaded. Please try again later."}`,
|
|
"503 service unavailable",
|
|
"context deadline exceeded",
|
|
}
|
|
for _, m := range transient {
|
|
if !isTransientError(errors.New(m)) {
|
|
t.Errorf("should be transient (worth waiting for): %q", m)
|
|
}
|
|
}
|
|
|
|
// Waiting fixes NONE of these — the provider will never serve this request.
|
|
// They must cascade immediately; sleeping first only burns the deadline.
|
|
permanent := []string{
|
|
"403 this model is not available for your account",
|
|
"insufficient_quota",
|
|
"402 payment required",
|
|
}
|
|
for _, m := range permanent {
|
|
err := errors.New(m)
|
|
if isTransientError(err) {
|
|
t.Errorf("must NOT be transient (waiting cannot help): %q", m)
|
|
}
|
|
// ...but they ARE worth trying a different provider for.
|
|
if faultOf(err) != faultProvider {
|
|
t.Errorf("should still cascade to another provider: %q", m)
|
|
}
|
|
}
|
|
|
|
// 401 is the exception, and the exception is the point. Waiting cannot fix a
|
|
// rejected credential — and neither can a different vendor, because the
|
|
// broken thing is OUR key for THIS one. Cascading it would drain traffic onto
|
|
// whoever still works while the dead key goes unnoticed until the bill
|
|
// arrives, so it is a request fault: surfaced, never routed around.
|
|
unauthorized := errors.New("401 unauthorized")
|
|
if isTransientError(unauthorized) {
|
|
t.Error("401 must not be transient")
|
|
}
|
|
if faultOf(unauthorized) != faultRequest {
|
|
t.Error("401 must NOT cascade — a dead credential of ours has to surface, not hide behind another vendor's")
|
|
}
|
|
}
|
|
|
|
func TestRetryTransient_RidesOutABusyProvider(t *testing.T) {
|
|
// The exact case that sent a 429 to the client before: busy, then fine.
|
|
calls := 0
|
|
err := retryTransient(context.Background(), testPolicy, func() error {
|
|
calls++
|
|
if calls < 3 {
|
|
return errors.New("429 Platform overloaded. Please try again later.")
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("a provider that recovers must succeed, got %v", err)
|
|
}
|
|
if calls != 3 {
|
|
t.Errorf("want 3 attempts (2 refusals then success), got %d", calls)
|
|
}
|
|
}
|
|
|
|
func TestRetryTransient_DoesNotSleepOnPermanentFailure(t *testing.T) {
|
|
// A 403 must abort on the FIRST attempt: retrying an entitlement error is
|
|
// pure latency, and the client is waiting.
|
|
calls := 0
|
|
start := time.Now()
|
|
slow := retryPolicy{attempts: 5, base: 200 * time.Millisecond, max: time.Second}
|
|
err := retryTransient(context.Background(), slow, func() error {
|
|
calls++
|
|
return errors.New("403 this model is not available for your account")
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected the 403 to be returned")
|
|
}
|
|
if calls != 1 {
|
|
t.Errorf("a permanent failure must not be retried: %d attempts", calls)
|
|
}
|
|
if elapsed := time.Since(start); elapsed > 100*time.Millisecond {
|
|
t.Errorf("must not back off on a permanent failure, slept %v", elapsed)
|
|
}
|
|
}
|
|
|
|
func TestRetryTransient_GivesUpAfterAttempts(t *testing.T) {
|
|
calls := 0
|
|
err := retryTransient(context.Background(), testPolicy, func() error {
|
|
calls++
|
|
return errors.New("429 overloaded")
|
|
})
|
|
if err == nil {
|
|
t.Fatal("a provider that never recovers must surface its error")
|
|
}
|
|
if calls != testPolicy.attempts {
|
|
t.Errorf("want exactly %d attempts, got %d", testPolicy.attempts, calls)
|
|
}
|
|
}
|
|
|
|
func TestRetryTransient_StopsWhenClientIsGone(t *testing.T) {
|
|
// Holding a request open is bounded by the caller's deadline: when the
|
|
// client disconnects, we stop waiting instead of pinning a goroutine and a
|
|
// (possibly 250K-token) prompt in memory.
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
calls := 0
|
|
slow := retryPolicy{attempts: 5, base: 2 * time.Second, max: 5 * time.Second}
|
|
|
|
start := time.Now()
|
|
err := retryTransient(ctx, slow, func() error {
|
|
calls++
|
|
cancel() // client goes away during the first attempt
|
|
return errors.New("429 overloaded")
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected the last error")
|
|
}
|
|
if calls != 1 {
|
|
t.Errorf("must abandon the request once the client is gone, got %d attempts", calls)
|
|
}
|
|
if elapsed := time.Since(start); elapsed > time.Second {
|
|
t.Errorf("must not sleep out the backoff after cancellation, took %v", elapsed)
|
|
}
|
|
}
|
|
|
|
func TestRetryTransient_CannotRetryACommittedStream(t *testing.T) {
|
|
// Once bytes are on the wire the request is not replayable. errPartiallyWritten
|
|
// is deliberately non-transient so this stops at once rather than backing off.
|
|
calls := 0
|
|
err := retryTransient(context.Background(), testPolicy, func() error {
|
|
calls++
|
|
return errPartiallyWritten
|
|
})
|
|
if !errors.Is(err, errPartiallyWritten) {
|
|
t.Fatalf("want errPartiallyWritten, got %v", err)
|
|
}
|
|
if calls != 1 {
|
|
t.Errorf("a partially-written response must never be retried, got %d attempts", calls)
|
|
}
|
|
}
|
|
|
|
func TestRetryPolicy_BackoffGrowsAndIsCapped(t *testing.T) {
|
|
p := retryPolicy{attempts: 6, base: 100 * time.Millisecond, max: 400 * time.Millisecond}
|
|
for n := 0; n < 6; n++ {
|
|
d := p.backoff(n)
|
|
if d <= 0 || d > p.max {
|
|
t.Errorf("backoff(%d)=%v must be in (0, max=%v]", n, d, p.max)
|
|
}
|
|
}
|
|
// Jitter: the wait must not be identical every time, or every client that
|
|
// got a 429 together returns together and re-creates the overload.
|
|
seen := map[time.Duration]bool{}
|
|
for i := 0; i < 40; i++ {
|
|
seen[p.backoff(2)] = true
|
|
}
|
|
if len(seen) == 1 {
|
|
t.Error("backoff must be jittered — an un-jittered retry reassembles the thundering herd")
|
|
}
|
|
}
|