mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
A vendor answered 402 and the whole product went quiet. Five upstream credentials were configured and funded; none was tried, because the model was served by the family pipe, which had no notion of an alternate at all, and the failover the QueryText path did have only engaged for routes that declared fallbacks. The missing idea was fault attribution. A refusal is either about the VENDOR (out of money, busy, broken, unreachable — somebody else can serve this) or about the REQUEST (malformed, unknown model, our credential rejected — everybody refuses it identically). Everything else follows from that one value, so it is one value, faultOf, and not a substring list re-derived at each branch. It reads the typed status first and consults text only where no status exists. 401 and plain 403 move to the request side: routing around our own dead key drains traffic onto whoever still works and nobody learns until the bill. The narrow "this account may not use this model" 403 stays a vendor fault — that is a fact about a catalogue, not a credential, and the do-ai to anthropic fallback exists for it. A vendor that refuses is demoted: sorted to the back for 15s when busy, 5min when out of money, so the next request does not pay its round trip first. Demotion reorders, never removes — a model whose only vendor is resting is still asked, and expiry re-probes a recovered one for free. The walk is capped at three vendors, the width of a declared route, so worst-case latency is a property of the constant rather than of whatever an operator typed into models.yaml. Every refusal is announced and written to the usage ledger as its own status, because a failover nobody can see leaves the empty account empty — that was the second half of the outage. When everyone refuses the client gets 503 naming who was asked and quoting the last reason, never the upstream's own 402, which would tell the customer they owe money. Streaming: a request is movable only until the first byte reaches the client. The family pipe reads the status before writing its header, so a streaming request is as movable as a buffered one at exactly that point; after it, nothing moves. Both writers accumulate, so a failed attempt is now cleared before the next one runs — otherwise the fallback's answer was served glued to the dead vendor's half sentence. Two money bugs found in the path while here: failover resolved providers globally, so a customer's connected key was silently replaced by the platform's while the call was still billed as bring-your-own at 1%; and the family pipe's transport-error path settled the budget hold, which is one-shot, so a request that later succeeded would never have been charged. Both resolved by carrying the row that actually spent a credential.
104 lines
3.0 KiB
Go
104 lines
3.0 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"os"
|
|
"testing"
|
|
)
|
|
|
|
// The status taxonomy lives in fault_test.go — faultOf's table is the single
|
|
// place it is asserted, so it cannot drift between two half-copies.
|
|
|
|
func TestFallbacksInModelConfig(t *testing.T) {
|
|
yaml := `
|
|
version: 1
|
|
services:
|
|
pricing_url: "https://pricing.hanzo.ai"
|
|
cache:
|
|
pricing_ttl: "1h"
|
|
features:
|
|
live_mode: false
|
|
premium_gate: true
|
|
default_pricing:
|
|
input_per_million: 1.00
|
|
output_per_million: 4.00
|
|
models:
|
|
claude-opus-4-6:
|
|
provider: do-ai
|
|
upstream: anthropic-claude-opus-4.6
|
|
fallbacks:
|
|
- provider: anthropic
|
|
upstream: claude-opus-4-6-20250514
|
|
- provider: anthropic-backup
|
|
upstream: claude-opus-4-6-20250514
|
|
pricing: { input: 15.00, output: 75.00 }
|
|
gpt-4o:
|
|
provider: do-ai
|
|
upstream: openai-gpt-4o
|
|
pricing: { input: 2.50, output: 10.00 }
|
|
`
|
|
|
|
dir := t.TempDir()
|
|
path := dir + "/models.yaml"
|
|
if err := writeYAML(t, path, yaml); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
mc := &ModelConfig{
|
|
routes: make(map[string]modelRoute),
|
|
pricing: make(map[string]modelPrice),
|
|
stopCh: make(chan struct{}),
|
|
}
|
|
if err := mc.loadFromFile(path); err != nil {
|
|
t.Fatalf("loadFromFile failed: %v", err)
|
|
}
|
|
|
|
// claude-opus-4-6 should have 2 fallbacks
|
|
route := mc.ResolveRoute("claude-opus-4-6")
|
|
if route == nil {
|
|
t.Fatal("expected route for claude-opus-4-6")
|
|
}
|
|
if route.providerName != "do-ai" {
|
|
t.Errorf("primary provider = %q, want do-ai", route.providerName)
|
|
}
|
|
if len(route.fallbacks) != 2 {
|
|
t.Fatalf("expected 2 fallbacks, got %d", len(route.fallbacks))
|
|
}
|
|
if route.fallbacks[0].providerName != "anthropic" {
|
|
t.Errorf("fallback[0].providerName = %q, want anthropic", route.fallbacks[0].providerName)
|
|
}
|
|
if route.fallbacks[0].upstreamModel != "claude-opus-4-6-20250514" {
|
|
t.Errorf("fallback[0].upstreamModel = %q, want claude-opus-4-6-20250514", route.fallbacks[0].upstreamModel)
|
|
}
|
|
if route.fallbacks[1].providerName != "anthropic-backup" {
|
|
t.Errorf("fallback[1].providerName = %q, want anthropic-backup", route.fallbacks[1].providerName)
|
|
}
|
|
|
|
// gpt-4o should have no fallbacks
|
|
route = mc.ResolveRoute("gpt-4o")
|
|
if route == nil {
|
|
t.Fatal("expected route for gpt-4o")
|
|
}
|
|
if len(route.fallbacks) != 0 {
|
|
t.Errorf("expected 0 fallbacks for gpt-4o, got %d", len(route.fallbacks))
|
|
}
|
|
}
|
|
|
|
func writeYAML(t *testing.T, path string, content string) error {
|
|
t.Helper()
|
|
return os.WriteFile(path, []byte(content), 0o644)
|
|
}
|