mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
The paid lane was one switch for every caller. FreeOnly stays the platform's switch; under it each request is put on a lane by LaneFilter, after the balance gate, from the grant the host's usage policy left on it. A priced call that a paid plan's included usage pays for, or the payer's prepaid or granted credit past it, is served on the paid lane; every other caller is on the free lane: no plan, a free model, a model's free daily cap, a policy that said nothing, and a request that never passed the filter. Every read on the request path asks FreeOnlyFor(ctx); candidates, openrouterTail and paying take its answer. PAID_LANE_DAILY (USD, default 10, 0 closes) bounds what all paid-lane calls spend in a UTC day, counted from each usage record at the most of its price and cost. Past it payers are on the free lane, and a chat for a third-party model is answered by the free model in limited mode. A paid plan's chat past its included usage (plan_allowance_used, model_cap) is answered in limited mode on any client. Responses carry X-Hanzo-Lane (paid|free) and X-Hanzo-Lane-Reason (paid_lane_ceiling). Co-authored-by: Hanzo Dev <[email protected]>
781 lines
35 KiB
Go
781 lines
35 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
openai "github.com/hanzoai/go-openai"
|
|
|
|
"github.com/hanzoai/ai/object"
|
|
)
|
|
|
|
// apiErr builds the typed error a provider HTTP failure arrives as, so these
|
|
// tests exercise the same path production does rather than a string that merely
|
|
// looks like one.
|
|
func apiErr(status int, msg string) error {
|
|
return &openai.APIError{HTTPStatusCode: status, Message: msg}
|
|
}
|
|
|
|
// The taxonomy IS the design. Everything else — whether to cascade, whether to
|
|
// demote, what the client is told — reads off this one function, so it gets the
|
|
// exhaustive table.
|
|
func TestFaultOf(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
err error
|
|
want fault
|
|
why string
|
|
}{
|
|
// ── the outage: a vendor whose account is empty ──────────────────
|
|
{"402 typed", apiErr(402, "Insufficient credits."), faultProvider,
|
|
"the measured outage: enso-flash, 402, whole request died"},
|
|
{"402 untyped text", errors.New(`status code 402, Payment Required, "Insufficient credits."`), faultProvider,
|
|
"the same refusal from a client that stringifies its errors"},
|
|
{"insufficient_quota", errors.New("You exceeded your current quota, code: insufficient_quota"), faultProvider,
|
|
"OpenAI's spelling of the same thing"},
|
|
{"insufficient balance", errors.New("Insufficient Balance"), faultProvider, "DeepSeek's spelling"},
|
|
{"billing hard limit", errors.New("Billing hard limit has been reached"), faultProvider, "OpenAI cap"},
|
|
|
|
// ── busy / broken vendors ────────────────────────────────────────
|
|
{"429 typed", apiErr(429, "Platform overloaded"), faultProvider, "ask again shortly"},
|
|
{"500 typed", apiErr(500, "internal"), faultProvider, "vendor broke"},
|
|
{"502 typed", apiErr(502, "bad gateway"), faultProvider, "vendor broke"},
|
|
{"503 typed", apiErr(503, "unavailable"), faultProvider, "vendor broke"},
|
|
{"504 typed", apiErr(504, "gateway timeout"), faultProvider, "vendor broke"},
|
|
{"408 typed", apiErr(408, "request timeout"), faultProvider, "vendor gave up on itself"},
|
|
{"599 typed", apiErr(599, "who knows"), faultProvider, "any 5xx is the vendor's end"},
|
|
{"connection refused", errors.New("dial tcp 10.0.0.1:443: connection refused"), faultProvider, "never arrived"},
|
|
{"connection reset", errors.New("read tcp: connection reset by peer"), faultProvider, "never completed"},
|
|
{"no such host", errors.New("lookup api.example.com: no such host"), faultProvider, "never resolved"},
|
|
{"deadline exceeded", errors.New("context deadline exceeded"), faultProvider, "never answered"},
|
|
{"unexpected EOF", errors.New("unexpected EOF"), faultProvider, "closed mid-response"},
|
|
{"provider disabled", unavailable("do-ai"), faultProvider,
|
|
"an admin switched it off; the alternates must still be honoured. The VALUE the loop " +
|
|
"raises, not a string resembling it — it carries a 503 so nothing has to read our own prose"},
|
|
|
|
// ── a substring of an unrelated token must not classify ──────────
|
|
// Every one of these was measured against the old substring matcher.
|
|
{"a date in a model id is not a payment code", errors.New("model claude-3-sonnet-20240229 not found"), faultRequest,
|
|
`"402" lies inside "2024_022_9", so a typo'd model id read as an empty account: three ` +
|
|
"vendors times three retries is nine upstream calls, plus a five-minute demotion of a healthy vendor"},
|
|
{"a token count is not a status", errors.New("you requested 8503 tokens, which exceeds the limit"), faultRequest,
|
|
`"503" lies inside "8503" — a request-size fact read as a broken vendor`},
|
|
{"a longer number that STARTS with a status is not that status", errors.New("request 402913 was rejected"), faultRequest,
|
|
`"402" begins a token here, so only checking the LEFT edge still reads an id as an empty account. ` +
|
|
"A status code is exact at both ends"},
|
|
{"geofence is not an eof", errors.New("request blocked by geofence policy"), faultRequest,
|
|
`"eof" lies inside "geofence", which would read any policy refusal as a dropped connection`},
|
|
{"a tier refusal is not our own switched-off row", errors.New("this model is unavailable for your account tier"), faultRequest,
|
|
`"unavailable" was in the transport list only because our OWN disabled-row error used the word. ` +
|
|
"If a vendor is observed using this phrasing for a catalogue gate it belongs in absentText — " +
|
|
"a deliberate data change, not an accident of overlap"},
|
|
|
|
// ── the vendor does not stock the item ───────────────────────────
|
|
{"403 agreement gate typed", apiErr(403, "this model is not available for your account"), faultProvider,
|
|
"DO's agreement gate — the do-ai to anthropic fallback exists for exactly this"},
|
|
{"403 agreement gate untyped", errors.New("error, status code: 403, message: this model is not available for your account"), faultProvider,
|
|
"same refusal, no status to read"},
|
|
{"no access to model", errors.New("your account does not have access to model xyz"), faultProvider, "same fact, other wording"},
|
|
|
|
// ── OUR mistake: must NOT cascade ────────────────────────────────
|
|
{"401 typed", apiErr(401, "invalid api key"), faultProvider,
|
|
"our credential is dead, which is a fact about THIS vendor — the caller's own was checked at our edge. announce() counts it as reasonCredential and logs at ERROR either way, so failing over costs the alert nothing and saves the request"},
|
|
{"401 untyped", errors.New("HTTP 401 Unauthorized"), faultRequest,
|
|
"NO status, so this is the text talking, and the text is a guess. A vendor quotes the request back, so treating the word as a credential fact lets a prompt about being unauthorized move itself to another vendor — the trap moneyText is already narrow for. The status path is where 401 is a fact"},
|
|
{"403 plain", apiErr(403, "you do not have permission to use this key"), faultProvider,
|
|
"a credential fact rather than a catalogue one, and both are facts about this vendor; the request is fine and another vendor can serve it"},
|
|
{"403 insufficient permissions", apiErr(403, "insufficient permissions for this operation"), faultProvider,
|
|
"still not `insufficient funds` — the point of that trap was never to stop the cascade but to keep this out of broke(), which it still is: it demotes nobody and spends no money leg"},
|
|
{"400 bad request", apiErr(400, "invalid 'messages': empty"), faultRequest, "malformed everywhere"},
|
|
{"404 unknown model", apiErr(404, "model not found: gpt-99"), faultRequest,
|
|
"a bad route entry we need to SEE, not paper over"},
|
|
{"413 too large", apiErr(413, "payload too large"), faultRequest, "the request is the problem"},
|
|
{"422 content filtered", apiErr(422, "content was filtered"), faultRequest, "refused everywhere"},
|
|
{"409 unknown 4xx", apiErr(409, "conflict"), faultRequest,
|
|
"unrecognised 4xx fails closed: we do not spend at three vendors to learn what it was"},
|
|
{"context length exceeded", errors.New("This model's maximum context length is 8192 tokens"), faultRequest,
|
|
"a request-size fact; every vendor refuses it identically"},
|
|
{"unrecognised", errors.New("something went sideways"), faultRequest,
|
|
"the default is stop, not spend"},
|
|
{"partially written", errPartiallyWritten, faultRequest,
|
|
"bytes are on the wire; nothing can be moved now"},
|
|
{"nil", nil, faultRequest, "no failure, nothing to attribute"},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if got := faultOf(tc.err); got != tc.want {
|
|
t.Errorf("faultOf(%v) = %s, want %s\n because: %s", tc.err, got, tc.want, tc.why)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The status must survive being wrapped. wrapUpstreamError restates an SDK error
|
|
// as *apiError, and if the reader only knew one of the two spellings the
|
|
// taxonomy would silently degrade to substring guessing after wrapping.
|
|
func TestStatusSurvivesWrapping(t *testing.T) {
|
|
raw := apiErr(402, "Insufficient credits.")
|
|
if got := upstreamHTTPStatus(raw); got != 402 {
|
|
t.Fatalf("raw SDK error status = %d, want 402", got)
|
|
}
|
|
wrapped := wrapUpstreamError(raw)
|
|
if got := upstreamHTTPStatus(wrapped); got != 402 {
|
|
t.Errorf("wrapped status = %d, want 402 — the taxonomy would fall back to text", got)
|
|
}
|
|
if faultOf(wrapped) != faultProvider {
|
|
t.Error("a wrapped 402 must still be a provider fault")
|
|
}
|
|
}
|
|
|
|
// A vendor out of money is unwell for everything and rests a long time. A vendor
|
|
// that simply does not carry one model is perfectly healthy and must not be
|
|
// pushed down the queue for the models it serves fine.
|
|
func TestCooldownFor(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
err error
|
|
want time.Duration
|
|
}{
|
|
{"402 rests long", apiErr(402, "Insufficient credits."), coolBroke},
|
|
{"quota rests long", errors.New("insufficient_quota"), coolBroke},
|
|
{"429 rests briefly", apiErr(429, "overloaded"), coolBusy},
|
|
{"503 rests briefly", apiErr(503, "unavailable"), coolBusy},
|
|
{"transport rests briefly", errors.New("connection refused"), coolBusy},
|
|
{"absent model does NOT rest", apiErr(403, "this model is not available for your account"), 0},
|
|
// A dead key needs a human exactly as an empty account does; resting it
|
|
// briefly would put the fleet back on it every few seconds for as long as
|
|
// it stays dead.
|
|
{"401 rests LONG", apiErr(401, "invalid api key"), coolBroke},
|
|
{"400 does NOT rest", apiErr(400, "bad request"), 0},
|
|
|
|
// The status is the evidence; the text is read only where the status is
|
|
// absent or genuinely two-valued.
|
|
{"429 saying quota rests long", apiErr(429, "You exceeded your current quota"), coolBroke},
|
|
{"a 503 quoting the caller does NOT rest long", apiErr(503, `upstream said: {"prompt":"what does insufficient balance mean?"}`), coolBusy},
|
|
{"a 500 quoting a payment code does NOT rest long", apiErr(500, "internal error handling invoice 402"), coolBusy},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if got := cooldownFor(tc.err); got != tc.want {
|
|
t.Errorf("cooldownFor(%v) = %v, want %v", tc.err, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
|
|
// The table above names the constants on BOTH sides, so it survives either of
|
|
// them being changed to anything at all — including to each other, which would
|
|
// silently delete the distinction the two exist to draw. The magnitudes are
|
|
// therefore pinned here, and the ordering with them: two clocks, because a busy
|
|
// vendor clears itself in seconds and an empty account waits for a human with a
|
|
// credit card.
|
|
if coolBusy != 15*time.Second {
|
|
t.Errorf("coolBusy = %v, want 15s — long enough to matter, short enough that a "+
|
|
"vendor which recovers is re-probed on the next request", coolBusy)
|
|
}
|
|
if coolBroke != 5*time.Minute {
|
|
t.Errorf("coolBroke = %v, want 5m — an empty account does not refill itself", coolBroke)
|
|
}
|
|
if coolBroke <= coolBusy {
|
|
t.Errorf("coolBroke (%v) must outlast coolBusy (%v), or the two kinds of refusal "+
|
|
"are one kind and the distinction buys nothing", coolBroke, coolBusy)
|
|
}
|
|
}
|
|
|
|
func TestCooldownExpires(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
|
|
if cooled.cooling(credential{"", "do-ai"}) {
|
|
t.Fatal("a provider nobody demoted must not be cooling")
|
|
}
|
|
cooled.demote(credential{"", "do-ai"}, 50*time.Millisecond)
|
|
if !cooled.cooling(credential{"", "do-ai"}) {
|
|
t.Fatal("a demoted provider must be cooling")
|
|
}
|
|
time.Sleep(60 * time.Millisecond)
|
|
if cooled.cooling(credential{"", "do-ai"}) {
|
|
t.Error("the cooldown must expire on its own — nothing else re-probes a recovered vendor")
|
|
}
|
|
}
|
|
|
|
// One tenant's empty account must not cost another tenant a vendor.
|
|
//
|
|
// The measured leak: tenant A connected its own openai key, that key answered
|
|
// 402, and openai was demoted for EVERYONE. Because the queue is also capped,
|
|
// the demotion did not merely reorder tenant B's queue — it pushed openai past
|
|
// the cap and REMOVED a vendor B's own healthy key could have served from.
|
|
//
|
|
// The assertion is on B's queue rather than on the map, so it survives any
|
|
// future change to how the penalty is stored.
|
|
func TestOneTenantsEmptyAccountIsNotAnothersOutage(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
|
|
r := route("openai", "anthropic", "fireworks")
|
|
|
|
// Tenant A's own connected openai account is empty.
|
|
cooled.demote(credential{"org-a", "openai"}, coolBroke)
|
|
|
|
got := names(candidates("org-b", r, nil, false))
|
|
want := []string{"openai", "anthropic", "fireworks"}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Errorf("tenant B's queue = %v, want %v — A's empty BYO account is a fact about "+
|
|
"A's credential, not about the vendor, and B does not hold it", got, want)
|
|
}
|
|
if len(got) == 0 || got[0] != "openai" {
|
|
t.Errorf("openai is not first for B (queue %v): a tenant that never refused anything "+
|
|
"must see its declared route untouched", got)
|
|
}
|
|
|
|
// The penalty is real for the tenant that earned it — otherwise this test
|
|
// would also pass against a cooldown that simply stopped working.
|
|
if a := names(candidates("org-a", r, nil, false)); len(a) == 0 || a[0] == "openai" {
|
|
t.Errorf("tenant A's queue = %v, want openai sorted back — A's own key IS empty", a)
|
|
}
|
|
}
|
|
|
|
// The mirror: our platform key is what every org without its own key spends, so
|
|
// a refusal on it is scoped to the org that observed it and each org pays at
|
|
// most one round trip to learn the same thing. What must never happen is the
|
|
// lesson jumping to a tenant whose request would spend a DIFFERENT credential.
|
|
func TestACooldownNamesTheOrgThatEarnedIt(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
|
|
cooled.demote(credential{"org-a", "openai"}, time.Minute)
|
|
|
|
if !cooled.cooling(credential{"org-a", "openai"}) {
|
|
t.Error("the org that refused must be cooling")
|
|
}
|
|
if cooled.cooling(credential{"org-b", "openai"}) {
|
|
t.Error("a different org must NOT inherit it")
|
|
}
|
|
if cooled.cooling(credential{"org-a", "anthropic"}) {
|
|
t.Error("a different vendor must NOT inherit it")
|
|
}
|
|
if cooled.cooling(credential{"", "openai"}) {
|
|
t.Error("the org-less bucket must NOT inherit it")
|
|
}
|
|
}
|
|
|
|
func route(primary string, fallbacks ...string) *modelRoute {
|
|
r := &modelRoute{providerName: primary, upstreamModel: "up-" + primary}
|
|
for _, f := range fallbacks {
|
|
r.fallbacks = append(r.fallbacks, modelRouteFallback{providerName: f, upstreamModel: "up-" + f})
|
|
}
|
|
return r
|
|
}
|
|
|
|
func names(cs []candidate) []string {
|
|
out := make([]string, 0, len(cs))
|
|
for _, c := range cs {
|
|
out = append(out, c.provider)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestCandidatesOrder(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
|
|
t.Run("declared order is preserved when everyone is healthy", func(t *testing.T) {
|
|
cooled.forget()
|
|
got := names(candidates("", route("do-ai", "anthropic", "fireworks"), nil, false))
|
|
want := []string{"do-ai", "anthropic", "fireworks"}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Errorf("order = %v, want %v — an operator's intent must survive", got, want)
|
|
}
|
|
})
|
|
|
|
t.Run("a cooling provider sorts to the back, it is not removed", func(t *testing.T) {
|
|
cooled.forget()
|
|
cooled.demote(credential{"", "do-ai"}, time.Minute)
|
|
got := names(candidates("", route("do-ai", "anthropic", "fireworks"), nil, false))
|
|
want := []string{"anthropic", "fireworks", "do-ai"}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Errorf("order = %v, want %v", got, want)
|
|
}
|
|
})
|
|
|
|
t.Run("when everyone is cooling the queue is still offered", func(t *testing.T) {
|
|
cooled.forget()
|
|
for _, p := range []string{"do-ai", "anthropic"} {
|
|
cooled.demote(credential{"", p}, time.Minute)
|
|
}
|
|
got := names(candidates("", route("do-ai", "anthropic"), nil, false))
|
|
if len(got) != 2 {
|
|
t.Fatalf("got %v — a model whose every vendor is resting must still be attempted; "+
|
|
"demotion is a latency preference, never an authorization", got)
|
|
}
|
|
if got[0] != "do-ai" {
|
|
t.Errorf("with everyone equal the declared order stands, got %v", got)
|
|
}
|
|
})
|
|
|
|
t.Run("a provider that already refused this request is dropped", func(t *testing.T) {
|
|
cooled.forget()
|
|
prior := []attempt{{provider: "enso", err: errors.New("402")}}
|
|
got := names(candidates("", route("enso", "do-ai"), prior, false))
|
|
if strings.Join(got, ",") != "do-ai" {
|
|
t.Errorf("got %v, want [do-ai] — asking a vendor twice in one request is waste", got)
|
|
}
|
|
})
|
|
|
|
// The numbers here are literal on purpose. Asserting against maxProviders
|
|
// asserts nothing: the constant is on both sides, so it survives being
|
|
// changed to any value and the bound stops being pinned at all.
|
|
t.Run("a route longer than the cap is cut to three", func(t *testing.T) {
|
|
cooled.forget()
|
|
got := names(candidates("", route("p1", "p2", "p3", "p4", "p5"), nil, false))
|
|
want := []string{"p1", "p2", "p3"}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Errorf("offered %v, want %v — three is the width of a declared route, and an "+
|
|
"unbounded walk turns one slow afternoon into 15 upstream calls", got, want)
|
|
}
|
|
})
|
|
|
|
// A7: the family pipe is not part of the declared route, so its refusal must
|
|
// not cost the route one of its own places. It used to, which meant a model
|
|
// served by the family could never reach Fallback2 — the operator declared
|
|
// three alternates and got two, on exactly the requests the alternates existed for.
|
|
t.Run("a refusal from outside the route does not spend a place in it", func(t *testing.T) {
|
|
cooled.forget()
|
|
prior := []attempt{{provider: "enso", err: errors.New("402")}}
|
|
got := names(candidates("", route("p1", "p2", "p3"), prior, false))
|
|
want := []string{"p1", "p2", "p3"}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Errorf("offered %v after the family refused, want %v — a declared route is exactly "+
|
|
"as wide as the cap, so charging the family to it deletes the last alternate", got, want)
|
|
}
|
|
})
|
|
|
|
t.Run("the cap still holds when something refused outside the route", func(t *testing.T) {
|
|
cooled.forget()
|
|
prior := []attempt{{provider: "enso", err: errors.New("402")}}
|
|
got := names(candidates("", route("p1", "p2", "p3", "p4", "p5"), prior, false))
|
|
if len(got) != 3 {
|
|
t.Errorf("offered %v, want 3 — the route walk stays bounded whatever happened before it", got)
|
|
}
|
|
})
|
|
|
|
t.Run("no route means no candidates", func(t *testing.T) {
|
|
if got := candidates("", nil, nil, false); got != nil {
|
|
t.Errorf("candidates(nil) = %v, want nil", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
// When everyone refuses, the client must be told who was asked and why the last
|
|
// one said no — and must NOT be told they are out of money when it is OUR
|
|
// account that is empty.
|
|
func TestExhausted(t *testing.T) {
|
|
tried := []attempt{
|
|
{provider: "enso", status: 402, err: errors.New("Insufficient credits.")},
|
|
{provider: "do-ai", status: 429, err: errors.New("Platform overloaded")},
|
|
}
|
|
err := exhausted("enso-flash", tried)
|
|
|
|
if got := statusOf(err); got != http.StatusServiceUnavailable {
|
|
t.Errorf("status = %d, want 503 — forwarding the upstream's 402 tells the CUSTOMER "+
|
|
"they owe money, which is false and sends them to fix a bill that is not theirs", got)
|
|
}
|
|
msg := err.Error()
|
|
if !strings.Contains(msg, `"enso-flash"`) || !strings.Contains(msg, "unavailable") {
|
|
t.Errorf("message %q, want the model named and said to be unavailable", msg)
|
|
}
|
|
for _, leak := range []string{"(402)", "do-ai", "429", "Platform overloaded"} {
|
|
if strings.Contains(msg, leak) {
|
|
t.Errorf("message %q names %q — who refused and why stays in the log", msg, leak)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A7-chain. exhausted() quotes the last refusal verbatim, so whatever
|
|
// upstreamErrorMessage returns is published to the customer. It used to return the
|
|
// whole raw body when it could not find error.message — which put the vendor's
|
|
// name and our buy price back into an answer, through the one path the envelope
|
|
// does not sit on.
|
|
func TestARefusalDoesNotQuoteTheUpstreamsBodyBack(t *testing.T) {
|
|
// An aggregator's 402, in a shape upstreamErrorMessage cannot read, carrying
|
|
// exactly what envelope.go strips from a successful answer.
|
|
body := []byte(`{"errors":[{"reason":"Insufficient credits."}],` +
|
|
`"provider":"GMICloud","usage":{"cost":0.00123,` +
|
|
`"cost_details":{"upstream_inference_cost":0.00098},"is_byok":false}}`)
|
|
|
|
said := upstreamErrorMessage(body)
|
|
for _, tell := range []string{"GMICloud", "cost", "0.00123", "upstream_inference_cost", "is_byok"} {
|
|
if strings.Contains(said, tell) {
|
|
t.Errorf("upstreamErrorMessage discloses %q: %s", tell, said)
|
|
}
|
|
}
|
|
|
|
// And the whole way out: the refusal a customer actually reads.
|
|
err := exhausted("enso-flash", []attempt{
|
|
{provider: "enso", status: 402, err: &apiError{status: 402, msg: said}},
|
|
})
|
|
for _, tell := range []string{"GMICloud", "0.00123", "upstream_inference_cost", "is_byok"} {
|
|
if strings.Contains(err.Error(), tell) {
|
|
t.Errorf("the 503 a customer reads discloses %q:\n%s", tell, err.Error())
|
|
}
|
|
}
|
|
if statusOf(err) != http.StatusServiceUnavailable {
|
|
t.Errorf("status = %d, want 503", statusOf(err))
|
|
}
|
|
|
|
// Still diagnostic where the upstream said something we can read.
|
|
for _, shape := range []string{
|
|
`{"error":{"message":"Insufficient credits."}}`,
|
|
`{"error":"Insufficient credits."}`,
|
|
`{"message":"Insufficient credits."}`,
|
|
`{"detail":"Insufficient credits."}`,
|
|
} {
|
|
if got := upstreamErrorMessage([]byte(shape)); got != "Insufficient credits." {
|
|
t.Errorf("upstreamErrorMessage(%s) = %q, want the reason — dropping the body must "+
|
|
"not cost the diagnostic", shape, got)
|
|
}
|
|
}
|
|
if got := upstreamErrorMessage(nil); got != "upstream error" {
|
|
t.Errorf("upstreamErrorMessage(nil) = %q", got)
|
|
}
|
|
}
|
|
|
|
func TestExhaustedWithNobodyToAsk(t *testing.T) {
|
|
err := exhausted("ghost-model", nil)
|
|
if got := statusOf(err); got != http.StatusServiceUnavailable {
|
|
t.Errorf("status = %d, want 503", got)
|
|
}
|
|
if !strings.Contains(err.Error(), "not available") {
|
|
t.Errorf("message = %q, want it to say the model is not available", err.Error())
|
|
}
|
|
}
|
|
|
|
// A 402 must never reach the client as a 402. Locks the one mapping that stops
|
|
// our empty vendor account from being reported as the customer's empty wallet.
|
|
func TestUpstreamPaymentIsNotTheCustomersProblem(t *testing.T) {
|
|
err := exhausted("enso-flash", []attempt{
|
|
{provider: "enso", status: 402, err: apiErr(402, "Insufficient credits.")},
|
|
})
|
|
if statusOf(err) == http.StatusPaymentRequired {
|
|
t.Fatal("an upstream 402 was forwarded to the customer as a 402")
|
|
}
|
|
}
|
|
|
|
func TestFaultString(t *testing.T) {
|
|
if fmt.Sprint(faultProvider) != "provider" || fmt.Sprint(faultRequest) != "request" {
|
|
t.Errorf("fault names must read plainly in logs, got %q and %q", faultProvider, faultRequest)
|
|
}
|
|
}
|
|
|
|
// ── the proxied surfaces ─────────────────────────────────────────────────────
|
|
|
|
// proxied drives the real relay — the path a tool call or an image takes — against an
|
|
// upstream answering status with body, and reports what the caller was told plus
|
|
// whether the upstream was actually dialled.
|
|
//
|
|
// The reach matters as much as the status: a proxy that refused before sending
|
|
// anything would answer 503 too, and pass a test that only reads the code while
|
|
// serving nobody.
|
|
func proxied(t *testing.T, status int, body string) (c *ApiController, asked bool) {
|
|
t.Helper()
|
|
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
asked = true
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_, _ = w.Write([]byte(body))
|
|
}))
|
|
defer up.Close()
|
|
|
|
req := &openai.ChatCompletionRequest{
|
|
Model: "glm-5.2",
|
|
Messages: []openai.ChatCompletionMessage{{Role: "user", Content: "hi"}},
|
|
Tools: []openai.Tool{{Type: openai.ToolTypeFunction,
|
|
Function: &openai.FunctionDefinition{Name: "now"}}},
|
|
}
|
|
prov := &object.Provider{Owner: "admin", Name: "do-ai", Type: "DigitalOcean",
|
|
ProviderUrl: up.URL, ClientSecret: "k", SubType: "glm-5.2-upstream"}
|
|
|
|
c = visit(http.MethodPost, "/v1/chat/completions")
|
|
forwarded(t, c, prov, req, nil)
|
|
return c, asked
|
|
}
|
|
|
|
// A request carrying tools or images skips the failover pipeline — that pipeline is
|
|
// text-only and would drop both — and is proxied to the vendor raw. The proxy
|
|
// relayed whatever status came back, so the one rule exhausted() exists to hold
|
|
// was simply absent on the surface hanzo.chat and @hanzo/dev use most.
|
|
func TestAProxiedVendorRefusalIsNotTheCallersBill(t *testing.T) {
|
|
c, asked := proxied(t, http.StatusPaymentRequired, `{"error":{"message":"Insufficient credits."}}`)
|
|
if !asked {
|
|
t.Fatal("the upstream was never dialled; this test would pass on a proxy that serves nobody")
|
|
}
|
|
if answered(c) == http.StatusPaymentRequired {
|
|
t.Errorf("the caller was handed the vendor's 402 — OUR account with that vendor is empty, " +
|
|
"and a client that acts on 402 goes to fix a bill that is not theirs")
|
|
}
|
|
if answered(c) != http.StatusServiceUnavailable {
|
|
t.Errorf("status=%d, want 503 — the true statement is that we could not serve this", answered(c))
|
|
}
|
|
if got := codeOf(exhausted("glm-5.2", nil)); !strings.Contains(sent(c), got) {
|
|
t.Errorf("body %q does not carry %q, so a client cannot tell this from its own empty wallet",
|
|
sent(c), got)
|
|
}
|
|
}
|
|
|
|
// A refused call is not a served one. The proxy read usage off the error body,
|
|
// found none, tokenized it, and settled — so the customer paid for the round trip
|
|
// that told them our vendor was broke.
|
|
func TestAProxiedRefusalIsNotBilled(t *testing.T) {
|
|
const subject = "prox/refused"
|
|
object.GlobalBalanceLedger.SetBalance(subject, 1000)
|
|
hold, ok := reserveBudget(subject, 100)
|
|
if !ok {
|
|
t.Fatal("reserve against a funded subject must succeed")
|
|
}
|
|
|
|
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusPaymentRequired)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"Insufficient credits."}}`))
|
|
}))
|
|
defer up.Close()
|
|
|
|
req := &openai.ChatCompletionRequest{
|
|
Model: "glm-5.2",
|
|
Messages: []openai.ChatCompletionMessage{{Role: "user", Content: strings.Repeat("hi ", 200)}},
|
|
Tools: []openai.Tool{{Type: openai.ToolTypeFunction,
|
|
Function: &openai.FunctionDefinition{Name: "now"}}},
|
|
}
|
|
prov := &object.Provider{Owner: "admin", Name: "do-ai", Type: "DigitalOcean",
|
|
ProviderUrl: up.URL, ClientSecret: "k", SubType: "glm-5.2-upstream"}
|
|
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
forwarded(t, c, prov, req, hold)
|
|
|
|
hold.settle(0) // the call site's fail-safe; a real settle already won if one happened
|
|
if avail, _ := object.GlobalBalanceLedger.Available(subject); avail != 1000 {
|
|
t.Errorf("available = %d, want 1000 — the customer was charged for a call our vendor refused", avail)
|
|
}
|
|
}
|
|
|
|
// The reverse case, and the one a fix here can silently eat: our OWN spend gate
|
|
// relayed back by a service that fronts for us wears the same 402 and means the
|
|
// opposite thing. It is the caller's debt and has to reach them.
|
|
func TestAFrontedBillingNoticeStillReachesTheCaller(t *testing.T) {
|
|
notice := object.InsufficientBalance("api.hanzo.ai", "org-a", "request cost")
|
|
c, asked := proxied(t, http.StatusPaymentRequired, string(notice.ErrorJSON()))
|
|
if !asked {
|
|
t.Fatal("the upstream was never dialled")
|
|
}
|
|
if answered(c) != http.StatusPaymentRequired {
|
|
t.Errorf("status=%d, want 402 — this refusal is our own gate speaking through a "+
|
|
"service that fronts for us, and the debt really is the caller's", answered(c))
|
|
}
|
|
}
|
|
|
|
// scrape reads the text exposition through the SAME handler GET /v1/metrics
|
|
// serves, so what this asserts is what a collector would actually pull — not
|
|
// merely that a package variable moved.
|
|
func scrape(t *testing.T) string {
|
|
t.Helper()
|
|
exposition := httptest.NewRecorder()
|
|
object.MetricsHandler().ServeHTTP(exposition, httptest.NewRequest(http.MethodGet, "/v1/metrics", nil))
|
|
if exposition.Code != http.StatusOK {
|
|
t.Fatalf("metrics scrape = %d, want 200", exposition.Code)
|
|
}
|
|
return exposition.Body.String()
|
|
}
|
|
|
|
// reading is the value of one series in a text exposition, or 0 when it is absent.
|
|
func reading(t *testing.T, body, series string) float64 {
|
|
t.Helper()
|
|
for line := range strings.SplitSeq(body, "\n") {
|
|
rest, ok := strings.CutPrefix(strings.TrimSpace(line), series)
|
|
if !ok || !strings.HasPrefix(rest, " ") {
|
|
continue
|
|
}
|
|
var v float64
|
|
if _, err := fmt.Sscanf(strings.TrimSpace(rest), "%g", &v); err != nil {
|
|
t.Fatalf("series %q has an unreadable value %q: %v", series, rest, err)
|
|
}
|
|
return v
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// An empty vendor account is invisible from the outside: the caller is told 503
|
|
// and owes nothing, so nobody on their side has a reason to report it. The only
|
|
// evidence used to be a log line, and a log line is read after somebody already
|
|
// noticed. This is the reading a collector can pull and an alert can watch.
|
|
func TestAVendorOutOfMoneyReachesAnOperator(t *testing.T) {
|
|
// A counter is process-wide and every other test in this package feeds the same
|
|
// registry, so what is asserted is the DELTA this call produced.
|
|
const (
|
|
funding = `cloud_supply_refused{provider="do-ai",reason="funding"}`
|
|
generic = `cloud_supply_refused{provider="do-ai",reason="refused"}`
|
|
)
|
|
was := scrape(t)
|
|
wasFunding, wasGeneric := reading(t, was, funding), reading(t, was, generic)
|
|
|
|
if _, asked := proxied(t, http.StatusPaymentRequired,
|
|
`{"error":{"message":"Insufficient credits."}}`); !asked {
|
|
t.Fatal("the upstream was never dialled")
|
|
}
|
|
|
|
now := scrape(t)
|
|
if got := reading(t, now, funding) - wasFunding; got != 1 {
|
|
t.Errorf("%s moved by %v, want 1 — a vendor whose account is spent must be visible to a "+
|
|
"collector, not only to whoever thinks to tail the log", funding, got)
|
|
}
|
|
// The reason has to be the one an operator acts on. Counting this as a generic
|
|
// refusal buries "top this account up" among rate limits and timeouts.
|
|
if got := reading(t, now, generic) - wasGeneric; got != 0 {
|
|
t.Errorf("the 402 also moved %s by %v — the condition that needs a human with a card "+
|
|
"must not be filed as a busy afternoon", generic, got)
|
|
}
|
|
}
|
|
|
|
// A refusal the vendor is not to blame for keeps its own status: every vendor
|
|
// answers a malformed body the same way, so 503 would send the caller to wait out
|
|
// a condition only they can clear.
|
|
func TestAProxiedRequestFaultKeepsItsStatus(t *testing.T) {
|
|
c, asked := proxied(t, http.StatusBadRequest, `{"error":{"message":"invalid 'messages': empty"}}`)
|
|
if !asked {
|
|
t.Fatal("the upstream was never dialled")
|
|
}
|
|
if answered(c) != http.StatusBadRequest {
|
|
t.Errorf("status=%d, want 400 — the request itself is wrong and the caller can fix it", answered(c))
|
|
}
|
|
}
|
|
|
|
// catalogOf points the family at a stub catalog carrying exactly these ids and
|
|
// warms it, so a test can ask what a route falls back to with no live vendor.
|
|
func catalogOf(t *testing.T, ids ...string) {
|
|
t.Helper()
|
|
var b strings.Builder
|
|
b.WriteString(`{"data":[`)
|
|
for i, id := range ids {
|
|
if i > 0 {
|
|
b.WriteString(",")
|
|
}
|
|
fmt.Fprintf(&b, `{"id":%q,"context_length":8192,"architecture":{"input_modalities":["text"]},`+
|
|
`"pricing":{"prompt":"0.000001","completion":"0.000002"}}`, id)
|
|
}
|
|
b.WriteString(`]}`)
|
|
withOpenRouter(t, b.String())
|
|
openrouterFam.fresh() // discover once, so lookup() answers from the snapshot
|
|
}
|
|
|
|
// EVERY ROUTE ENDS SOMEWHERE. The route table declares 121 routes and no
|
|
// `fallbacks:` at all, so before the derived tail a vendor that could not serve
|
|
// ended the request — which is how one dead DO key answered 93 models' worth of
|
|
// customers with a 401.
|
|
func TestOpenRouterTailIsDerived(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
|
|
t.Run("the same model on OpenRouter, when its catalog carries it", func(t *testing.T) {
|
|
catalogOf(t, "openai/gpt-4o")
|
|
r := route("do-ai")
|
|
r.upstreamModel = "openai-gpt-4o"
|
|
got := names(candidates("", r, nil, false))
|
|
if len(got) < 2 || got[1] != "openrouter" {
|
|
t.Fatalf("got %v — do-ai must fall over to openrouter", got)
|
|
}
|
|
})
|
|
|
|
t.Run("a guess the catalog does not confirm is not offered", func(t *testing.T) {
|
|
catalogOf(t, "anthropic/claude-3.5-sonnet") // NOT what do-ai spells
|
|
r := route("do-ai")
|
|
r.upstreamModel = "anthropic-claude-4.5-sonnet"
|
|
for _, c := range candidates("", r, nil, false) {
|
|
if c.provider == "openrouter" && c.upstream == "anthropic/claude-4.5-sonnet" {
|
|
t.Fatal("a derived id must be confirmed by the catalog, never guessed onto the wire")
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("openrouter's own routes get no second tail", func(t *testing.T) {
|
|
catalogOf(t, "openai/gpt-4o")
|
|
r := route("openrouter")
|
|
r.upstreamModel = "openai/gpt-4o"
|
|
if got := names(candidates("", r, nil, false)); len(got) != 1 {
|
|
t.Fatalf("got %v — the family's own spared already offers the pool", got)
|
|
}
|
|
})
|
|
|
|
t.Run("an unconfigured OpenRouter adds nothing", func(t *testing.T) {
|
|
r := route("do-ai")
|
|
r.upstreamModel = "openai-gpt-4o"
|
|
if got := names(candidates("", r, nil, false)); len(got) != 1 {
|
|
t.Fatalf("got %v — a deployment without OpenRouter must route exactly as before", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
// ran is the money's question and faultOf is the routing one, and they are NOT
|
|
// the same question. These are the two cases where they disagree most, so if
|
|
// ran ever collapses back into faultOf this is what notices.
|
|
//
|
|
// A partial write is the important half. Bytes on the wire mean a vendor
|
|
// produced them, and it stops the cascade (request fault) — so a reader that
|
|
// took "does not fail over" to mean "did no work" would refuse to bill an answer
|
|
// the customer actually received and we were actually invoiced for.
|
|
func TestRanAndFaultOfAnswerDifferentQuestions(t *testing.T) {
|
|
for _, c := range []struct {
|
|
what string
|
|
err error
|
|
ran bool
|
|
failsOver bool
|
|
}{
|
|
{"a call that succeeded", nil, true, false},
|
|
{"a stream that broke with bytes already sent", errPartiallyWritten, true, false},
|
|
{"a dead key of ours", apiErr(401, "invalid api key"), false, true},
|
|
{"an account with no money", apiErr(402, "insufficient credits"), false, true},
|
|
{"a prompt longer than the context", apiErr(413, "payload too large"), false, false},
|
|
{"the vendor broke while running it", apiErr(500, "internal server error"), true, true},
|
|
{"nothing was ever dialled", errors.New("failed to get model provider: unsupported type"), false, false},
|
|
} {
|
|
if got := ran(c.err); got != c.ran {
|
|
t.Errorf("%s: ran = %v, want %v", c.what, got, c.ran)
|
|
}
|
|
if got := faultOf(c.err) == faultProvider; got != c.failsOver {
|
|
t.Errorf("%s: fails over = %v, want %v", c.what, got, c.failsOver)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A malformed body is described in the caller's terms, never by the Go type it
|
|
// decodes into.
|
|
func TestAParseProblemNamesNoGoType(t *testing.T) {
|
|
var v struct {
|
|
Messages []struct {
|
|
Content []struct{ Text string } `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
err := json.Unmarshal([]byte(`{"messages":[{"content":5}]}`), &v)
|
|
got := parseProblem(err)
|
|
if strings.Contains(got, "Go") || strings.Contains(got, "struct") || !strings.Contains(got, "array") {
|
|
t.Fatalf("parseProblem = %q", got)
|
|
}
|
|
}
|