mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
Owner, 2026-10-26: "enso and enso-auto are the same, and even the other enso/zen models literally never stop working. They just fall to free models and keep GOING." One rule, controllers.Fall (floor.go): a chat on /v1/chat/completions, /v1/messages or /v1/responses, streamed or whole, naming an Enso or Zen id that cannot be served for a reason that is not the caller's is served as the same request by the id's floor, Floor(model): enso-free for an Enso id, zen-free for a Zen id, read from the family's freeName. Every refusal point asks it: - the balance gate: an empty or unreadable wallet and every plan refusal of the asked model, for any credential, an API key included; the floor is asked again from the top, so what it meets itself stands; - the handlers' own gate and reservation, chat and messages: a 402, an unreadable balance, our cash breaker, an estimate the balance cannot cover; - the family pipe, before a byte reached the caller: 5xx, 429, 402 from our vendor account, a timeout, an unreachable family, our family's masked "could not serve this request", an unpriced route, a paid lane closed to the caller. The floor is always asked once. What stands: 401, 400, 403, a key's own budget or models, the free allowance, an unreadable policy, and any refusal the floor meets. A floor never falls, so a request moves at most once; when the floor fails too its own status answers, said as the floor's. The answer is the floor's: X-Hanzo-Fallback and X-Hanzo-Fallback-Reason, the body's model, X-Hanzo-Served; it bills nothing and carries no plan spend. A forced tool goes to the floor as "required" over that tool, then as "auto" with a sentence asking for the call (X-Hanzo-Fallback- Tools). A stream that breaks off after its first bytes ends with an error event and is never asked again. Counted in cloud_fallback. Router ids keep their free SKU under the same exclusions: a key's own limit now stands for them too. Testing: go test ./controllers/ ./routers/ (floor_test.go, filter_balance_floor_test.go; rates, spare, limits and lane tests updated where the owner's rule replaces the old naming and refusals). Risk: a paid plan past its usage on an Enso or Zen id is now answered by the floor rather than refused 429; answers that fell are named and billed as the floor's. How to revert: git revert this commit.
1509 lines
63 KiB
Go
1509 lines
63 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/decimal"
|
|
|
|
"github.com/hanzoai/ai/object"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
)
|
|
|
|
// A listing with both kinds of SKU in it, deliberately serialized worst-first so
|
|
// the ordering assertions below cannot pass by accident.
|
|
const spareListing = `{"data":[
|
|
{"id":"vendor/small:free","context_length":8000,"pricing":{"prompt":"0","completion":"0"},"architecture":{"output_modalities":["text"]}},
|
|
{"id":"vendor/paid-a","context_length":128000,"pricing":{"prompt":"0.000002","completion":"0.000008"},"architecture":{"output_modalities":["text"]}},
|
|
{"id":"vendor/big:free","context_length":1000000,"pricing":{"prompt":"0","completion":"0"},"architecture":{"output_modalities":["text"]}},
|
|
{"id":"vendor/paid-b","context_length":64000,"pricing":{"prompt":"0.0000005","completion":"0.0000015"},"architecture":{"output_modalities":["text"]}},
|
|
{"id":"vendor/music:free","context_length":1048576,"pricing":{"prompt":"0","completion":"0"},"architecture":{"output_modalities":["text","audio"]}},
|
|
{"id":"vendor/silent:free","context_length":999999,"pricing":{"prompt":"0","completion":"0"}}
|
|
]}`
|
|
|
|
// The catalog is the vendor's whole listing, and the spare routes are a reading of
|
|
// the same body — so what we publish is what they serve, and the floors a SKU
|
|
// carries are decided by the one fact that matters: its price. A priced SKU takes
|
|
// both floors and can never be handed out as a remedy; a free one takes neither and
|
|
// bills nothing however a request reaches it.
|
|
func TestTheCatalogIsTheListingAndPriceDecidesTheFloors(t *testing.T) {
|
|
catalog, err := openrouterCatalog([]byte(spareListing))
|
|
if err != nil {
|
|
t.Fatalf("catalog: %v", err)
|
|
}
|
|
spares := openrouterSpare([]byte(spareListing))
|
|
|
|
listed := map[string]zenModel{}
|
|
for _, m := range catalog {
|
|
listed[m.ID] = m
|
|
}
|
|
for _, id := range []string{
|
|
"vendor/paid-a", "vendor/paid-b",
|
|
"vendor/small:free", "vendor/big:free", "vendor/music:free", "vendor/silent:free",
|
|
} {
|
|
if _, ok := listed[id]; !ok {
|
|
t.Errorf("%q is advertised by the vendor and absent from the catalog", id)
|
|
}
|
|
}
|
|
if len(listed) != 6 {
|
|
t.Errorf("catalog holds %d SKUs, want the whole listing", len(listed))
|
|
}
|
|
|
|
for id, m := range listed {
|
|
free := m.Base.In.IsZero() && m.Base.Out.IsZero()
|
|
switch {
|
|
case free && (m.MinTier != "" || m.Funding != ""):
|
|
t.Errorf("%q costs nothing and carries floors (tier=%q funding=%q) — it would refuse a caller over money never spent", id, m.MinTier, m.Funding)
|
|
case !free && (m.MinTier != "paid" || m.Funding != "prepaid"):
|
|
t.Errorf("%q spends real cash and carries tier=%q funding=%q", id, m.MinTier, m.Funding)
|
|
}
|
|
}
|
|
|
|
for _, s := range spares {
|
|
m, ok := listed[s]
|
|
if !ok {
|
|
t.Errorf("%q is a spare the catalog does not hold — two discoveries, not two readings", s)
|
|
continue
|
|
}
|
|
if !m.Base.In.IsZero() || !m.Base.Out.IsZero() {
|
|
t.Errorf("%q is a spare with a price — a downgrade that bills is not a downgrade", s)
|
|
}
|
|
}
|
|
|
|
// Widest first, because the request was already sized for the SKU it asked
|
|
// for: a spare that cannot hold the prompt is a second refusal, not a
|
|
// fallback. Then by id, so the answer is a property of the listing and not of
|
|
// the order the vendor happened to serialize it in.
|
|
want := []string{"vendor/big:free", "vendor/small:free"}
|
|
if strings.Join(spares, ",") != strings.Join(want, ",") {
|
|
t.Errorf("spares = %v, want %v", spares, want)
|
|
}
|
|
}
|
|
|
|
// FREE IS NOT THE SAME QUESTION AS USABLE, and the live listing is what taught it:
|
|
// the two WIDEST free routes OpenRouter advertises are google/lyria-3-*, which
|
|
// declare out:["text","audio"] because they are music models. Ordered by context
|
|
// alone they sat at the top of the fallback list, so a chat request refused for
|
|
// money would have been handed to a music generator — a wrong answer where an
|
|
// honest error belonged, which is worse than the outage it was fixing.
|
|
func TestASpareMustAnswerInTextAndNothingElse(t *testing.T) {
|
|
for _, id := range openrouterSpare([]byte(spareListing)) {
|
|
switch id {
|
|
case "vendor/music:free":
|
|
t.Error("a model that also answers in audio is a spare — a chat request would be handed to a music generator")
|
|
case "vendor/silent:free":
|
|
t.Error("a model that declares no output modality is a spare — this is the one place that must not guess")
|
|
}
|
|
}
|
|
}
|
|
|
|
// A vendor whose listing we could not read gets no fallback invented for it.
|
|
func TestAnUnreadableListingHasNoSpares(t *testing.T) {
|
|
if got := openrouterSpare([]byte("not json")); len(got) != 0 {
|
|
t.Errorf("spares = %v, want none", got)
|
|
}
|
|
if got := openrouterSpare([]byte(`{"data":[{"id":" ","pricing":{"prompt":"0","completion":"0"}}]}`)); len(got) != 0 {
|
|
t.Errorf("spares = %v, want none — a nameless SKU is not a route", got)
|
|
}
|
|
}
|
|
|
|
// The one wiring that makes any of this reachable: OpenRouter declares its
|
|
// dialect's reading of "free", so the family knows what it still serves when the
|
|
// account is empty.
|
|
func TestOpenRouterDeclaresItsSpareRoutes(t *testing.T) {
|
|
if openrouterFam.spare == nil {
|
|
t.Fatal("openrouter declares no spare routes — a 402 from it would be final")
|
|
}
|
|
}
|
|
|
|
// Discovery carries the spare routes: ONE fetch, both projections, and the family
|
|
// prices a spare at nothing because the vendor does.
|
|
func TestDiscoveryCarriesTheSpareRoutes(t *testing.T) {
|
|
vendor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/v1/models" {
|
|
t.Errorf("discovery asked for %q", r.URL.Path)
|
|
}
|
|
_, _ = w.Write([]byte(spareListing))
|
|
}))
|
|
defer vendor.Close()
|
|
|
|
fam := &modelFamily{
|
|
name: "openrouter", prefix: "openrouter/",
|
|
decode: openrouterCatalog,
|
|
spare: openrouterSpare,
|
|
providerFn: func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "openrouter", Type: "OpenRouter", ProviderUrl: vendor.URL}
|
|
},
|
|
}
|
|
if err := fam.refresh(); err != nil {
|
|
t.Fatalf("refresh: %v", err)
|
|
}
|
|
|
|
if got := fam.spareRoutes(); len(got) != 2 || got[0] != "vendor/big:free" {
|
|
t.Errorf("spareRoutes = %v", got)
|
|
}
|
|
if !fam.isSpare("vendor/big:free") || !fam.isSpare("VENDOR/BIG:FREE") {
|
|
t.Error("a discovered free route is not recognised as a spare")
|
|
}
|
|
if fam.isSpare("vendor/paid-a") {
|
|
t.Error("a priced SKU reads as a spare — it would bill at nothing")
|
|
}
|
|
// A spare is a catalog entry the vendor prices at nothing, so a caller may name
|
|
// it and a refusal for money may land on it, and both bill the same zero.
|
|
spare, ok := fam.lookup("vendor/big:free")
|
|
if !ok {
|
|
t.Fatal("a free route the vendor advertises is missing from the catalog")
|
|
}
|
|
if !spare.Base.In.IsZero() || !spare.Base.Out.IsZero() {
|
|
t.Error("a spare route carries a price")
|
|
}
|
|
if spare.MinTier != "" || spare.Funding != "" {
|
|
t.Errorf("a free route carries floors (tier=%q funding=%q) — a free plan cannot reach it", spare.MinTier, spare.Funding)
|
|
}
|
|
}
|
|
|
|
// refuses is a family that answers `status`/`body` for anything but `free`, and
|
|
// answers `free` normally. It records every SKU it was asked for, which is what
|
|
// makes "the fallback did not fire" an assertion rather than an absence.
|
|
type refuses struct {
|
|
status int
|
|
body string
|
|
free string
|
|
asked []string
|
|
}
|
|
|
|
func (f *refuses) serve(t *testing.T) *httptest.Server {
|
|
t.Helper()
|
|
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var in struct {
|
|
Model string `json:"model"`
|
|
}
|
|
body := make([]byte, r.ContentLength)
|
|
_, _ = r.Body.Read(body)
|
|
_ = json.Unmarshal(body, &in)
|
|
f.asked = append(f.asked, in.Model)
|
|
|
|
if in.Model == f.free {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"gen-1","model":"` + f.free + `","choices":[{"index":0,"message":{"role":"assistant","content":"ok"}}],"usage":{"prompt_tokens":3,"completion_tokens":1,"total_tokens":4}}`))
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(f.status)
|
|
_, _ = w.Write([]byte(f.body))
|
|
}))
|
|
}
|
|
|
|
// pipe drives the real relay against that family and returns what the client got.
|
|
func (f *refuses) pipe(t *testing.T, fam *modelFamily, sku string) (*ApiController, []attempt) {
|
|
t.Helper()
|
|
return f.pipeSince(t, fam, sku, time.Now())
|
|
}
|
|
|
|
// pipeSince is pipe with the request's own clock, for the rules that read it.
|
|
func (f *refuses) pipeSince(t *testing.T, fam *modelFamily, sku string, start time.Time) (*ApiController, []attempt) {
|
|
t.Helper()
|
|
body := []byte(`{"model":"` + sku + `","messages":[{"role":"user","content":"2+2?"}]}`)
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody(body)
|
|
out := c.pipeToFamily(fam, "chat/completions", "openai", sku, body, false, 0, "acme", nil, false, nil, start)
|
|
return c, out
|
|
}
|
|
|
|
// restore returns fam to the state it holds now once the test ends.
|
|
//
|
|
// It saves the FIELDS, not the family. A modelFamily carries the lock guarding its
|
|
// discovery cache, and a lock may not be copied: `saved := *fam` duplicates it, so
|
|
// the restored family would guard itself with a different lock than the one every
|
|
// reader already holds a reference to. Maps and slices are carried by reference
|
|
// here exactly as a struct copy carried them — a test that replaces one is undone,
|
|
// a test that writes through one was never undone either.
|
|
func restore(t *testing.T, fam *modelFamily) {
|
|
t.Helper()
|
|
fam.mu.RLock()
|
|
name, provider, prefix, owner := fam.name, fam.typ, fam.prefix, fam.owner
|
|
urlKey, keyKey, freeName := fam.urlKey, fam.keyKey, fam.freeName
|
|
providerFn, decode, spare, terms := fam.providerFn, fam.decode, fam.spare, fam.terms
|
|
windows, byID, ids, spares := fam.windows, fam.byID, fam.ids, fam.spares
|
|
fetchedAt, loaded, named, paid := fam.fetchedAt, fam.loaded, fam.named, fam.paid
|
|
makers := fam.makers
|
|
fam.mu.RUnlock()
|
|
|
|
t.Cleanup(func() {
|
|
fam.mu.Lock()
|
|
defer fam.mu.Unlock()
|
|
fam.name, fam.typ, fam.prefix, fam.owner = name, provider, prefix, owner
|
|
fam.urlKey, fam.keyKey, fam.freeName = urlKey, keyKey, freeName
|
|
fam.providerFn, fam.decode, fam.spare, fam.terms = providerFn, decode, spare, terms
|
|
fam.windows, fam.byID, fam.ids, fam.spares = windows, byID, ids, spares
|
|
fam.fetchedAt, fam.loaded, fam.named, fam.paid = fetchedAt, loaded, named, paid
|
|
fam.makers = makers
|
|
})
|
|
}
|
|
|
|
// spareFamily points the PLATFORM free pool at a test vendor and returns it as the
|
|
// family under test. The pool is the real openrouter family — one pool for every
|
|
// family is the whole design — so a test configures that one rather than a stand-in
|
|
// no production path would consult.
|
|
//
|
|
// The snapshot is set directly: what is under test is what the pipe DOES with a
|
|
// spare, not how discovery found one (TestDiscoveryCarriesTheSpareRoutes covers
|
|
// that), and refreshing would put a second round trip inside every case below.
|
|
func spareFamily(t *testing.T, url, free string, also ...string) *modelFamily {
|
|
t.Helper()
|
|
fam := freeFamily()
|
|
restore(t, fam)
|
|
fam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "openrouter", Type: "OpenRouter", ProviderUrl: url}
|
|
}
|
|
fam.spares = []string{free}
|
|
fam.byID = map[string]zenModel{free: {ID: free}}
|
|
fam.ids = []string{free}
|
|
// Routes a case may NAME but that are not spares. A route the family can price
|
|
// is what makes a terms assertion about that route rather than about an id
|
|
// discovery never found — the two read the same today and would stop reading the
|
|
// same the moment either rule moved.
|
|
for _, id := range also {
|
|
m := zenModel{ID: id}
|
|
if !strings.HasSuffix(id, ":free") {
|
|
m.Base.In, m.Base.Out = decimal.New(3, 0), decimal.New(15, 0)
|
|
}
|
|
fam.byID[id] = m
|
|
fam.ids = append(fam.ids, id)
|
|
}
|
|
fam.loaded = true
|
|
fam.fetchedAt = time.Now()
|
|
return fam
|
|
}
|
|
|
|
// otherFamily is a family whose vendor is reached through a service of its own and
|
|
// which declares no free route — enso and zen, the two the pool exists for. Its
|
|
// refusals must reach the SAME pool as everyone else's.
|
|
func otherFamily(t *testing.T, url string) *modelFamily {
|
|
t.Helper()
|
|
return &modelFamily{
|
|
name: "enso", prefix: "enso", owner: "hanzo", freeName: "enso-free",
|
|
providerFn: func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: url}
|
|
},
|
|
loaded: true, fetchedAt: time.Now(),
|
|
}
|
|
}
|
|
|
|
// A brand's free door is answered by the platform pool, so it stands whether or not
|
|
// that brand's own service is configured: the pool is what serves it, and the vendor
|
|
// is not. zen-free and enso-free are ours in front of one pool, not routes their
|
|
// vendors publish, so an unreachable vendor has nothing to say about them.
|
|
func TestAFreeDoorStandsWithoutItsVendor(t *testing.T) {
|
|
spareFamily(t, "http://vendor.invalid", "free-route")
|
|
fam := otherFamily(t, "") // no base URL: the brand's own service is unreachable
|
|
|
|
if fam.enabled() {
|
|
t.Fatal("want a family whose own service is unconfigured")
|
|
}
|
|
if !fam.serves(fam.freeName) {
|
|
t.Errorf("%q refused while the pool holds routes", fam.freeName)
|
|
}
|
|
}
|
|
|
|
// And it closes when there is nothing behind it. A door onto an empty pool is a
|
|
// refusal that arrives later, so it is refused here instead.
|
|
func TestAFreeDoorClosesOnAnEmptyPool(t *testing.T) {
|
|
pool := freeFamily()
|
|
restore(t, pool)
|
|
pool.spares, pool.loaded, pool.fetchedAt = nil, true, time.Now()
|
|
restore(t, engineFam)
|
|
engineFam.providerFn = func() *object.Provider { return nil }
|
|
|
|
fam := otherFamily(t, "")
|
|
if fam.serves(fam.freeName) {
|
|
t.Errorf("%q opened onto an empty pool", fam.freeName)
|
|
}
|
|
}
|
|
|
|
// A NAMED THIRD-PARTY MODEL IS SERVED AS ITSELF OR NOT AT ALL. Whatever its vendor
|
|
// answers — out of credit, broken, busy, a malformed body, our own balance gate — the
|
|
// request is never moved to another model: the refusal reaches the caller, or is
|
|
// handed back so the same model can be tried at another vendor, and nothing else ever
|
|
// answers in its name. Only a Hanzo SKU or the pool's own name is ours to route.
|
|
//
|
|
// Each case drives the REAL relay against a real HTTP vendor, and the assertion is on
|
|
// what the vendor was ASKED — so "it did not move" is a fact about the wire.
|
|
func TestANamedThirdPartyRouteIsNeverMoved(t *testing.T) {
|
|
const free = "vendor/big:free"
|
|
const sku = "vendor/small:free"
|
|
const spent = `{"error":{"message":"Insufficient credits. Add more using https://openrouter.ai/settings/credits","code":402,"metadata":{"limit_source":"openrouter_credits"}}}`
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
status int
|
|
body string
|
|
}{
|
|
{"out of credit", 402, spent},
|
|
{"payment required, bare", 402, `{"error":{"message":"payment required"}}`},
|
|
{"malformed request", 400, `{"error":{"message":"messages: invalid role"}}`},
|
|
{"model not carried", 404, `{"error":{"message":"No endpoints found for vendor/paid-a."}}`},
|
|
{"content refused", 422, `{"error":{"message":"content policy violation"}}`},
|
|
{"credential rejected", 401, `{"error":{"message":"unauthorized"}}`},
|
|
{"forbidden", 403, `{"error":{"message":"forbidden"}}`},
|
|
{"rate limited", 429, `{"error":{"message":"rate limit exceeded"}}`},
|
|
{"vendor broken", 500, `{"error":{"message":"internal server error"}}`},
|
|
{"bad gateway", 502, `{"error":{"message":"bad gateway"}}`},
|
|
{"gateway timeout", 504, `{"error":{"message":"gateway timeout"}}`},
|
|
{"our own balance gate", 402, `{"error":{"message":"Insufficient balance. Add credits to your wallet at https://hanzo.ai/pay","type":"billing_error","code":"insufficient_balance"}}`},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
cooled.forget()
|
|
forgetKeys()
|
|
fake := &refuses{status: tc.status, body: tc.body, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
fam := spareFamily(t, vendor.URL, free, sku)
|
|
|
|
c, refusedBy := fake.pipe(t, fam, sku)
|
|
|
|
if len(fake.asked) != 1 || fake.asked[0] != sku {
|
|
t.Fatalf("asked=%v, want only the model the caller named", fake.asked)
|
|
}
|
|
if refusedBy == nil && sent(c) == "" {
|
|
t.Error("the refusal vanished")
|
|
}
|
|
if strings.Contains(sent(c), free) || strings.Contains(sent(c), "enso-free") {
|
|
t.Errorf("a %d was answered by another model:\n%s", tc.status, sent(c))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Terms are not traded for an answer: a route bought under `deny` that its vendor
|
|
// cannot serve never reaches a borrowed free route, which keeps what it carries.
|
|
// With no stand-in of our own configured, the refusal stands as a supply refusal.
|
|
func TestTermsAreNotTradedForAnAnswer(t *testing.T) {
|
|
const free = "vendor/big:free"
|
|
const paid = "vendor/paid-a"
|
|
restore(t, engineFam)
|
|
engineFam.urlKey = "TEST_ENGINE_URL_UNSET"
|
|
engineFam.providerFn = nil
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
status int
|
|
body string
|
|
}{
|
|
{"out of credit", 402, `{"error":{"message":"Insufficient credits. Add more using https://openrouter.ai/settings/credits","code":402}}`},
|
|
{"vendor broken", 500, `{"error":{"message":"internal server error"}}`},
|
|
{"bad gateway", 502, `{"error":{"message":"bad gateway"}}`},
|
|
{"gateway timeout", 504, `{"error":{"message":"gateway timeout"}}`},
|
|
{"every account busy", 429, `{"error":{"message":"rate limit exceeded"}}`},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
cooled.forget()
|
|
forgetKeys()
|
|
fake := &refuses{status: tc.status, body: tc.body, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
fam := spareFamily(t, vendor.URL, free, paid)
|
|
if word, stated := fam.collection(paid); !stated || word != collectionDeny {
|
|
t.Fatalf("%q is bought under (%q, stated=%v), want %q", paid, word, stated, collectionDeny)
|
|
}
|
|
|
|
c, refusedBy := fake.pipe(t, fam, paid)
|
|
|
|
for _, m := range fake.asked {
|
|
if m == free {
|
|
t.Errorf("asked=%v — a route bought under %q was offered to a free one", fake.asked, collectionDeny)
|
|
}
|
|
}
|
|
if strings.Contains(sent(c), free) {
|
|
t.Errorf("the free route answered a %q route:\n%s", collectionDeny, sent(c))
|
|
}
|
|
if refusedBy == nil && sent(c) == "" {
|
|
t.Error("the refusal vanished")
|
|
}
|
|
})
|
|
}
|
|
|
|
// A route bought under `allow` does not move either: a named model is never
|
|
// answered by another.
|
|
t.Run("allow does not move a named model", func(t *testing.T) {
|
|
cooled.forget()
|
|
const alsoFree = "vendor/small:free"
|
|
fake := &refuses{status: 402, body: `{"error":{"message":"Insufficient credits."}}`, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
fam := spareFamily(t, vendor.URL, free, alsoFree)
|
|
if word, stated := fam.collection(alsoFree); !stated || word != collectionAllow {
|
|
t.Fatalf("%q is bought under (%q, stated=%v), want %q", alsoFree, word, stated, collectionAllow)
|
|
}
|
|
if _, _ = fake.pipe(t, fam, alsoFree); len(fake.asked) != 1 {
|
|
t.Fatalf("asked=%v — a named route was moved", fake.asked)
|
|
}
|
|
})
|
|
}
|
|
|
|
// zenPipeHeader bounds ONE attempt and spareTries bounds how many we borrow.
|
|
// Neither bounds their product, and the product is what a person waits through: a
|
|
// request expected to take ~25s was measured at 237s, and one at 540s before the
|
|
// connection was reset. A four-minute hang is indistinguishable from a hung browser,
|
|
// and the person watching it cannot tell which.
|
|
//
|
|
// So the walk reads the caller's clock too. Once a request has already spent a whole
|
|
// attempt's worth of time, the next try is a longer wait before the same refusal.
|
|
func TestTheWalkIsBoundedByTheCallersClockAndNotOnlyByItsCount(t *testing.T) {
|
|
cooled.forget()
|
|
const free = "vendor/big:free"
|
|
const first = "vendor/small:free"
|
|
door := func(url string) *modelFamily {
|
|
fam := spareFamily(t, url, free)
|
|
fam.spares = []string{first, free}
|
|
fam.byID[first] = zenModel{ID: first}
|
|
return fam
|
|
}
|
|
fake := &refuses{status: 402, body: `{"error":{"message":"Insufficient credits."}}`, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
|
|
// A request that has already been running longer than one attempt is allowed to
|
|
// take. Nothing here is slow; what is under test is the rule, not the wait.
|
|
spent := time.Now().Add(-zenPipeHeader - time.Second)
|
|
c, refusedBy := fake.pipeSince(t, door(vendor.URL), freeID, spent)
|
|
|
|
if len(fake.asked) != 0 {
|
|
t.Errorf("asked=%v — the walk started an attempt on a request that had already spent its time", fake.asked)
|
|
}
|
|
// And it is a refusal, not silence: the caller gets an answer they can act on.
|
|
if refusedBy == nil && sent(c) == "" {
|
|
t.Error("the refusal vanished")
|
|
}
|
|
|
|
// The same request on a fresh clock still walks — otherwise this asserts that
|
|
// the walk is broken rather than that it is bounded.
|
|
cooled.forget()
|
|
fresh := &refuses{status: 402, body: `{"error":{"message":"Insufficient credits."}}`, free: free}
|
|
freshVendor := fresh.serve(t)
|
|
defer freshVendor.Close()
|
|
if _, _ = fresh.pipe(t, door(freshVendor.URL), freeID); len(fresh.asked) != 2 {
|
|
t.Errorf("asked=%v — a request with time left did not walk the pool", fresh.asked)
|
|
}
|
|
}
|
|
|
|
// The terms the caller is told are the terms the answer was served under, and after
|
|
// the floor above a substitution can only ever be `allow` for `allow`.
|
|
func TestTheHeaderStatesTheTermsThatServed(t *testing.T) {
|
|
cooled.forget()
|
|
const free = "vendor/big:free"
|
|
fake := &refuses{status: 402, body: `{"error":{"message":"Insufficient credits."}}`, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
|
|
c, _ := fake.pipe(t, spareFamily(t, vendor.URL, free), freeID)
|
|
|
|
if got := string(c.Fiber().Response().Header.Peek(headerCollection)); got != collectionAllow {
|
|
t.Errorf("%s = %q, want %q — the spare that answered is served under %q",
|
|
headerCollection, got, collectionAllow, collectionAllow)
|
|
}
|
|
}
|
|
|
|
// The free alias is the same rule from the other side: the caller names an id
|
|
// no vendor holds, the pool picks a route, and the answer comes back wearing the id
|
|
// that was named rather than the route that was chosen.
|
|
func TestTheFreeAliasAnswersInTheNameItWasCalledBy(t *testing.T) {
|
|
cooled.forget()
|
|
const free = "vendor/big:free"
|
|
const alias = "enso-free"
|
|
fake := &refuses{status: 500, body: `{"error":{"message":"unused"}}`, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
|
|
spareFamily(t, vendor.URL, free)
|
|
enso := otherFamily(t, vendor.URL)
|
|
|
|
c, _ := fake.pipe(t, enso, alias)
|
|
|
|
if len(fake.asked) != 1 || fake.asked[0] != free {
|
|
t.Fatalf("asked=%v — the pool did not choose a route", fake.asked)
|
|
}
|
|
var got map[string]any
|
|
if err := json.Unmarshal([]byte(sent(c)), &got); err != nil {
|
|
t.Fatalf("not JSON: %v\n%s", err, sent(c))
|
|
}
|
|
if got["model"] != alias {
|
|
t.Errorf("model = %v, want %q — the id the caller called", got["model"], alias)
|
|
}
|
|
if got["model"] == free {
|
|
t.Errorf("the envelope names %q, the route the pool chose", free)
|
|
}
|
|
}
|
|
|
|
// A spare bills nothing, because the vendor charges nothing for it — that is the
|
|
// whole reason it can answer while the account is empty. Charging retail for a
|
|
// downgrade the customer did not choose is taking money for the outage.
|
|
func TestASpareRouteIsBilledAtNothing(t *testing.T) {
|
|
fam := &modelFamily{name: "openrouter", spares: []string{"vendor/big:free"}, loaded: true, fetchedAt: time.Now()}
|
|
c := visit(http.MethodPost, "/v1/x")
|
|
w := whence{ledger: c.billingOrg(nil), ip: c.Fiber().IP(), ctx: c.Context()}
|
|
|
|
if nano := recordFamilyUsage(w, fam, "vendor/big:free", "vendor/paid-a", nil, &mark{}, nil, false, false, "r1", tokens{fresh: 1000, completion: 1000}, serving{}, time.Now(), nil, "success", ""); nano != 0 {
|
|
t.Errorf("a spare route billed %d nano", nano)
|
|
}
|
|
|
|
// And the zero has to hold at every reader of the money, not only in the hold.
|
|
// The DEBIT, the warehouse row and the span each ask usageCostNano, and a price
|
|
// table asked about a SKU it has never seen answers with its conservative
|
|
// default — which is the right guess for an unknown model and exactly the wrong
|
|
// one here, because it bills a customer for the fallback they were handed.
|
|
free := &usageRecord{Model: "vendor/big:free", Free: true, Status: "success", PromptTokens: 1000, CompletionTokens: 1000}
|
|
if got := usageCostNano(free); got != 0 {
|
|
t.Errorf("a free route costs %d nano — the customer would be billed for the outage", got)
|
|
}
|
|
if got := providerCostNano(free); got == nil || *got != 0 {
|
|
t.Errorf("a free route has COGS of %v nano, want a known 0", got)
|
|
}
|
|
if recordUnpriced(free) {
|
|
t.Error("a free route reads as unpriced — its price is zero, which is a price")
|
|
}
|
|
// The same record without the fact is what the table would have charged, so the
|
|
// assertion above is not vacuous.
|
|
guessed := *free
|
|
guessed.Free = false
|
|
if usageCostNano(&guessed) == 0 {
|
|
t.Skip("the price table happens to price this SKU at zero; the guard is untestable here")
|
|
}
|
|
}
|
|
|
|
// A priced family SKU the family says it answered from a free rung bills nothing;
|
|
// the same SKU answered by its own arm bills its price.
|
|
func TestAFamilySKUServedByTheFreeLaneIsBilledAtNothing(t *testing.T) {
|
|
enso := otherFamily(t, "http://enso.invalid")
|
|
enso.byID = map[string]zenModel{"enso-flash": {ID: "enso-flash", Base: zenTier{In: decimal.New(1, 0), Out: decimal.New(2, 0)}}}
|
|
enso.ids = []string{"enso-flash"}
|
|
c := visit(http.MethodPost, "/v1/x")
|
|
w := whence{ledger: c.billingOrg(nil), ip: c.Fiber().IP(), ctx: c.Context()}
|
|
use := tokens{fresh: 1_000_000, completion: 1_000_000}
|
|
|
|
if nano := recordFamilyUsage(w, enso, "enso-flash", "", nil, &mark{}, nil, true, false, "r1", use, servingOf(http.Header{servedHeader: {"vendor/big:free"}, freeHeader: {"true"}}), time.Now(), nil, "success", ""); nano != 0 {
|
|
t.Errorf("an answer the free lane wrote billed %d nano", nano)
|
|
}
|
|
if nano := recordFamilyUsage(w, enso, "enso-flash", "", nil, &mark{}, nil, true, false, "r2", use, serving{arm: "vendor/flash"}, time.Now(), nil, "success", ""); nano != 3_000_000_000 {
|
|
t.Errorf("an answer the SKU's own arm wrote billed %d nano, want 3000000000 ($3)", nano)
|
|
}
|
|
}
|
|
|
|
// The ledger and the span each carry the fallback, and they carry it as data a
|
|
// query can filter on rather than as a sentence in a log line.
|
|
func TestAFallbackIsVisibleInTheLedgerAndTheSpan(t *testing.T) {
|
|
row := &usageRecord{
|
|
Model: "vendor/big:free", Requested: "vendor/paid-a",
|
|
Provider: "openrouter", Owner: "acme", Status: "success",
|
|
PromptTokens: 3, CompletionTokens: 1, TotalTokens: 4,
|
|
}
|
|
|
|
// The ledger: `model` is what answered, `requested` what was wanted, and
|
|
// `WHERE requested != ''` is the query for every downgraded generation.
|
|
values := cloudUsageValues(row, time.Now())
|
|
at := -1
|
|
for i, col := range object.CloudUsageColumns {
|
|
if col == "requested" {
|
|
at = i
|
|
}
|
|
}
|
|
if at < 0 {
|
|
t.Fatal("the ledger has no requested column")
|
|
}
|
|
if values[at] != "vendor/paid-a" {
|
|
t.Errorf("ledger requested = %v, want the SKU the caller named", values[at])
|
|
}
|
|
|
|
// The span: the two standard model attributes stop agreeing, and one
|
|
// attribute states the cause so a reader does not have to infer it.
|
|
got := map[string]string{}
|
|
for _, a := range buildGenAISpanFields(row, 0, 0, new(float64(0)), nil, nil, false).attrs {
|
|
if a.Value.Type() == attribute.STRING {
|
|
got[string(a.Key)] = a.Value.AsString()
|
|
}
|
|
}
|
|
if got[attrGenAIRequestModel] != "vendor/paid-a" || got[attrGenAIResponseModel] != "vendor/big:free" {
|
|
t.Errorf("span models: request=%q response=%q — they must say what was asked and what answered",
|
|
got[attrGenAIRequestModel], got[attrGenAIResponseModel])
|
|
}
|
|
if got[attrFallback] != fallbackSpent {
|
|
t.Errorf("span fallback = %q, want %q", got[attrFallback], fallbackSpent)
|
|
}
|
|
|
|
// An ordinary generation carries none of it: absent IS the signal, so the
|
|
// attribute stays a filter rather than something to interpret.
|
|
plain := &usageRecord{Model: "vendor/paid-a", Provider: "openrouter", Status: "success"}
|
|
for _, a := range buildGenAISpanFields(plain, 0, 0, new(float64(0)), nil, nil, false).attrs {
|
|
if string(a.Key) == attrFallback {
|
|
t.Error("an ordinary generation is tagged as a fallback")
|
|
}
|
|
}
|
|
if v := cloudUsageValues(plain, time.Now())[at]; v != "" {
|
|
t.Errorf("ledger requested = %v on an ordinary row, want empty", v)
|
|
}
|
|
}
|
|
|
|
// A family that declares no spare routes is unchanged: its refusal is final and
|
|
// is handed on for another VENDOR to answer, exactly as before.
|
|
func TestAFamilyWithNoSpareIsUnchanged(t *testing.T) {
|
|
cooled.forget()
|
|
fake := &refuses{status: 402, body: `{"error":{"message":"Insufficient credits."}}`, free: "unused"}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
|
|
fam := &modelFamily{
|
|
name: "enso", prefix: "enso",
|
|
providerFn: func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: vendor.URL}
|
|
},
|
|
loaded: true, fetchedAt: time.Now(),
|
|
}
|
|
_, refusedBy := fake.pipe(t, fam, "enso-4")
|
|
|
|
if len(fake.asked) != 1 {
|
|
t.Errorf("asked=%v, want one attempt — this family has nothing to fall back to", fake.asked)
|
|
}
|
|
if len(refusedBy) != 1 || refusedBy[0].status != http.StatusPaymentRequired {
|
|
t.Fatalf("refusal = %v, want the 402 handed on to the next vendor", refusedBy)
|
|
}
|
|
// And it is never relayed to the customer as a 402: exhausted() answers 503,
|
|
// because "we could not serve this" is true and "you owe money" is not.
|
|
if s := upstreamHTTPStatus(exhausted("enso-4", refusedBy)); s != http.StatusServiceUnavailable {
|
|
t.Errorf("every-vendor-refused status = %d, want 503", s)
|
|
}
|
|
}
|
|
|
|
// The vendor's own free router is not in the pool: the pool is the set of routes we
|
|
// can say something about, and a router is precisely the one we cannot. Measured
|
|
// live, one request to it was served by a coding model and the next by a
|
|
// content-safety classifier that replied "User Safety: safe" to a chat prompt —
|
|
// choosing it would delegate the choice the pool exists to make.
|
|
//
|
|
// It stays in the catalog: curation narrows what we fall back to, never what we
|
|
// publish.
|
|
func TestTheVendorsOwnFreeRouterIsNotInThePool(t *testing.T) {
|
|
listing := `{"data":[
|
|
{"id":"vendor/wide:free","context_length":1000000,"pricing":{"prompt":"0","completion":"0"},"architecture":{"output_modalities":["text"]}},
|
|
{"id":"openrouter/free","context_length":200000,"pricing":{"prompt":"0","completion":"0"},"architecture":{"output_modalities":["text"]}},
|
|
{"id":"vendor/narrow:free","context_length":8000,"pricing":{"prompt":"0","completion":"0"},"architecture":{"output_modalities":["text"]}}
|
|
]}`
|
|
got := openrouterSpare([]byte(listing))
|
|
want := []string{"vendor/wide:free", "vendor/narrow:free"}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Errorf("spares = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
// A free route is free because the vendor keeps what it carried; a priced one pays
|
|
// instead and keeps nothing. One fact, stated to the vendor in its own dialect and
|
|
// reported to the caller in ours, so the two can never be different words about the
|
|
// same call.
|
|
func TestTermsFollowThePrice(t *testing.T) {
|
|
read := func(b []byte) string {
|
|
var m struct {
|
|
Provider struct {
|
|
DataCollection string `json:"data_collection"`
|
|
Order []string `json:"order"`
|
|
} `json:"provider"`
|
|
Model string `json:"model"`
|
|
}
|
|
if err := json.Unmarshal(b, &m); err != nil {
|
|
t.Fatalf("body is not JSON after terms: %v", err)
|
|
}
|
|
if m.Model != "vendor/x" {
|
|
t.Errorf("terms rewrote the model to %q", m.Model)
|
|
}
|
|
return m.Provider.DataCollection
|
|
}
|
|
body := []byte(`{"model":"vendor/x","messages":[]}`)
|
|
if got := read(openrouterTerms(body, zenModel{}, true, "")); got != collectionAllow {
|
|
t.Errorf("free route data_collection = %q, want %q", got, collectionAllow)
|
|
}
|
|
if got := read(openrouterTerms(body, zenModel{}, false, "")); got != collectionDeny {
|
|
t.Errorf("priced route data_collection = %q, want %q", got, collectionDeny)
|
|
}
|
|
|
|
// A caller's own vendor preferences survive: terms state one field, not the object.
|
|
kept := openrouterTerms([]byte(`{"model":"vendor/x","provider":{"order":["a","b"]}}`), zenModel{}, true, "")
|
|
var m struct {
|
|
Provider struct {
|
|
DataCollection string `json:"data_collection"`
|
|
Order []string `json:"order"`
|
|
} `json:"provider"`
|
|
}
|
|
if err := json.Unmarshal(kept, &m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(m.Provider.Order) != 2 || m.Provider.DataCollection != collectionAllow {
|
|
t.Errorf("a caller's other vendor preferences were dropped: %+v", m.Provider)
|
|
}
|
|
|
|
// A body we cannot read goes out as it came in rather than mangled.
|
|
if got := string(openrouterTerms([]byte("not json"), zenModel{}, true, "")); got != "not json" {
|
|
t.Errorf("unreadable body was rewritten to %q", got)
|
|
}
|
|
}
|
|
|
|
// The listing says what a model answers WITH, because a price cannot.
|
|
//
|
|
// The free lineup carries music models and a content-safety classifier beside the
|
|
// chat models, all at zero. A client choosing a free model for a chat turn filtering
|
|
// on price alone eventually picks the music generator — the same mistake the spare
|
|
// list makes structurally impossible for itself, made possible again one layer up
|
|
// unless the catalog says what each route produces.
|
|
func TestTheListingSaysWhatAModelAnswersWith(t *testing.T) {
|
|
fam := &modelFamily{name: "openrouter", prefix: "openrouter/", owner: "openrouter"}
|
|
fam.byID = map[string]zenModel{
|
|
"vendor/chat": {ID: "vendor/chat", Outputs: []string{"text"}},
|
|
"vendor/music": {ID: "vendor/music", Outputs: []string{"text", "audio"}},
|
|
"vendor/mute": {ID: "vendor/mute"},
|
|
}
|
|
fam.ids = []string{"vendor/chat", "vendor/music", "vendor/mute"}
|
|
fam.loaded = true
|
|
fam.fetchedAt = time.Now()
|
|
|
|
byID := indexModels(fam.mergeModels(nil))
|
|
if got := byID["vendor/chat"].Outputs; len(got) != 1 || got[0] != "text" {
|
|
t.Errorf("a text model lists outputs %v", got)
|
|
}
|
|
if got := byID["vendor/music"].Outputs; len(got) != 2 {
|
|
t.Errorf("a model that also answers in audio lists outputs %v — a chat client cannot avoid it", got)
|
|
}
|
|
if got := byID["vendor/mute"].Outputs; len(got) != 0 {
|
|
t.Errorf("a model advertising nothing lists outputs %v, want none rather than a guess", got)
|
|
}
|
|
}
|
|
|
|
// What the vendor advertises reaches the listing unchanged.
|
|
func TestOutputsComeFromTheVendorListing(t *testing.T) {
|
|
cat, err := openrouterCatalog([]byte(spareListing))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
by := map[string]zenModel{}
|
|
for _, m := range cat {
|
|
by[m.ID] = m
|
|
}
|
|
if got := by["vendor/music:free"].Outputs; len(got) != 2 || got[1] != "audio" {
|
|
t.Errorf("music model outputs = %v, want the vendor's own [text audio]", got)
|
|
}
|
|
if got := by["vendor/silent:free"].Outputs; len(got) != 0 {
|
|
t.Errorf("a SKU advertising no modality carries %v, want none", got)
|
|
}
|
|
}
|
|
|
|
// THE CASE THE POOL EXISTS FOR. enso and zen reach their vendor through a service
|
|
// of their own and declare no free route, so a refusal from either used to end the
|
|
// request — a 402 with nowhere to go, while free routes elsewhere were answering.
|
|
//
|
|
// A spare is now a route AND the family that carries it, so the refusal crosses to
|
|
// whoever still has one. The vendor that ran out of money is rarely the vendor
|
|
// holding a free route.
|
|
func TestARefusalCrossesToTheFamilyThatStillHasAFreeRoute(t *testing.T) {
|
|
cooled.forget()
|
|
const free = "vendor/big:free"
|
|
fake := &refuses{status: 402, body: `{"error":{"message":"Insufficient credits."}}`, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
|
|
// The pool lives on the free family and points at the same test vendor; the
|
|
// family under test is one that has no free route of its own.
|
|
spareFamily(t, vendor.URL, free)
|
|
enso := otherFamily(t, vendor.URL)
|
|
|
|
c, refusedBy := fake.pipe(t, enso, "enso-flash")
|
|
|
|
if len(fake.asked) < 2 {
|
|
t.Fatalf("asked=%v — the refusal did not cross to the free family", fake.asked)
|
|
}
|
|
if fake.asked[0] != "enso-flash" {
|
|
t.Errorf("first ask was %q, want the SKU the caller named", fake.asked[0])
|
|
}
|
|
if fake.asked[1] != free {
|
|
t.Errorf("second ask was %q, want the pool's route", fake.asked[1])
|
|
}
|
|
if refusedBy != nil {
|
|
t.Errorf("the request was handed on after it had already been served: %v", refusedBy)
|
|
}
|
|
// It crossed families and came back named for what answered: the asked model's
|
|
// floor, never the vendor's route behind it (floor.go).
|
|
if strings.Contains(sent(c), free) {
|
|
t.Errorf("the answer names the free family's route:\n%s", sent(c))
|
|
}
|
|
if !strings.Contains(sent(c), `"model":"enso-free"`) || header(c, FallbackHeader) != "enso-free" {
|
|
t.Errorf("the answer does not name the floor that wrote it:\n%s", sent(c))
|
|
}
|
|
}
|
|
|
|
// A family's free id is served by choosing from the pool, and it is a route the
|
|
// caller may name outright rather than only reach by being refused.
|
|
func TestAFamilysFreeIdIsServedFromThePool(t *testing.T) {
|
|
cooled.forget()
|
|
const free = "vendor/big:free"
|
|
fake := &refuses{status: 500, body: `{"error":{"message":"unused"}}`, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
|
|
spareFamily(t, vendor.URL, free)
|
|
enso := otherFamily(t, vendor.URL)
|
|
|
|
c, refusedBy := fake.pipe(t, enso, "enso-free")
|
|
|
|
if len(fake.asked) != 1 || fake.asked[0] != free {
|
|
t.Fatalf("asked=%v, want exactly the pool's route — a free id is not a model any vendor holds", fake.asked)
|
|
}
|
|
if refusedBy != nil {
|
|
t.Errorf("serving a free id handed the request on: %v", refusedBy)
|
|
}
|
|
// The pool's route wrote the answer and the answer does not name it — which id
|
|
// it DOES wear is TestTheFreeAliasAnswersInTheNameItWasCalledBy.
|
|
if strings.Contains(sent(c), free) {
|
|
t.Errorf("the answer names the route the pool chose:\n%s", sent(c))
|
|
}
|
|
}
|
|
|
|
// The free id is listed, priced at nothing, and absent when the pool is empty — a
|
|
// listed model that answers nothing is worse than an absent one.
|
|
func TestTheFreeIdIsListedOnlyWhileThePoolCanServeIt(t *testing.T) {
|
|
fam := freeFamily()
|
|
restore(t, fam)
|
|
fam.spares = []string{"vendor/a:free"}
|
|
fam.byID = map[string]zenModel{"vendor/a:free": {ID: "vendor/a:free", MaxCtx: 128000}}
|
|
fam.ids = []string{"vendor/a:free"}
|
|
fam.loaded, fam.fetchedAt = true, time.Now()
|
|
|
|
enso := &modelFamily{name: "enso", prefix: "enso", owner: "hanzo", freeName: "enso-free", loaded: true, fetchedAt: time.Now()}
|
|
byID := indexModels(enso.mergeModels(nil))
|
|
got, ok := byID["enso-free"]
|
|
if !ok {
|
|
t.Fatal("the family's free id is not listed while the pool can serve it")
|
|
}
|
|
// The listing prices a row from the billing lookup, which reads the family's own.
|
|
if p, ok := enso.modelPrice("enso-free"); got.Premium || !ok || p.InputPerMillion != 0 || p.OutputPerMillion != 0 {
|
|
t.Errorf("the free id lists as premium=%v, priced %+v ok=%v", got.Premium, p, ok)
|
|
}
|
|
if got.ContextWindow != 128000 {
|
|
t.Errorf("context = %d, want the narrowest the pool can hold", got.ContextWindow)
|
|
}
|
|
if len(got.Outputs) != 1 || got.Outputs[0] != "text" {
|
|
t.Errorf("outputs = %v, want text", got.Outputs)
|
|
}
|
|
|
|
fam.spares = nil
|
|
if _, ok := indexModels(enso.mergeModels(nil))["enso-free"]; ok {
|
|
t.Error("the free id is listed with an empty pool — it would answer nothing")
|
|
}
|
|
}
|
|
|
|
// A classifier answers in text and does not hold a conversation. Measured live:
|
|
// nemotron content-safety replied "User Safety: safe" to a chat turn.
|
|
func TestThePoolHoldsOnlyRoutesAChatTurnCanGoTo(t *testing.T) {
|
|
listing := `{"data":[
|
|
{"id":"nvidia/nemotron-3.5-content-safety:free","context_length":128000,"pricing":{"prompt":"0","completion":"0"},"architecture":{"output_modalities":["text"]}},
|
|
{"id":"vendor/guard-2:free","context_length":128000,"pricing":{"prompt":"0","completion":"0"},"architecture":{"output_modalities":["text"]}},
|
|
{"id":"vendor/chat:free","context_length":64000,"pricing":{"prompt":"0","completion":"0"},"architecture":{"output_modalities":["text"]}}
|
|
]}`
|
|
got := openrouterSpare([]byte(listing))
|
|
if len(got) != 1 || got[0] != "vendor/chat:free" {
|
|
t.Errorf("pool = %v, want only the route a chat turn can go to", got)
|
|
}
|
|
// The catalog still lists them: they are real models somebody may want by name.
|
|
cat, err := openrouterCatalog([]byte(listing))
|
|
if err != nil || len(cat) != 3 {
|
|
t.Errorf("catalog = %d SKUs, want all 3 — curation narrows the FALLBACK, not the listing", len(cat))
|
|
}
|
|
}
|
|
|
|
// The family's free id costs nothing, and everything that reads a price agrees.
|
|
//
|
|
// It is served from the pool, which is the routes that cost nothing, so its price is
|
|
// a stated zero rather than an absent one. That is what keeps the balance gate from
|
|
// asking a caller to fund a call that bills nothing — the whole point of having a
|
|
// free id — and what keeps it from listing as a model somebody's plan cannot afford.
|
|
func TestTheFreeIdCostsNothingEverywhere(t *testing.T) {
|
|
fam := freeFamily()
|
|
restore(t, fam)
|
|
fam.spares = []string{"vendor/a:free"}
|
|
fam.byID = map[string]zenModel{"vendor/a:free": {ID: "vendor/a:free", MaxCtx: 128000}}
|
|
fam.ids = []string{"vendor/a:free"}
|
|
fam.loaded, fam.fetchedAt = true, time.Now()
|
|
|
|
restore(t, ensoFam)
|
|
ensoFam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: "http://enso.invalid"}
|
|
}
|
|
ensoFam.byID = map[string]zenModel{}
|
|
ensoFam.loaded, ensoFam.fetchedAt = true, time.Now()
|
|
|
|
p, ok := ensoFam.modelPrice("enso-free")
|
|
if !ok {
|
|
t.Fatal("the free id has no price — the balance gate would ask a caller to fund it")
|
|
}
|
|
if p.InputPerMillion != 0 || p.OutputPerMillion != 0 {
|
|
t.Errorf("the free id is priced %+v", p)
|
|
}
|
|
if r := ensoFam.passthroughRoute("enso-free"); r == nil {
|
|
t.Fatal("the free id does not route")
|
|
} else if r.premium {
|
|
t.Error("the free id routes as premium — it reads as a model a plan cannot afford")
|
|
}
|
|
}
|
|
|
|
// The pool holds routes that hold a conversation, so an embeddings request does not
|
|
// walk it: a chat model does not embed, and asking three of them spends the caller's
|
|
// time to be told so three times.
|
|
func TestAnEmbeddingsRequestDoesNotWalkTheChatPool(t *testing.T) {
|
|
cooled.forget()
|
|
const free = "vendor/big:free"
|
|
fake := &refuses{status: 402, body: `{"error":{"message":"Insufficient credits."}}`, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
spareFamily(t, vendor.URL, free)
|
|
enso := otherFamily(t, vendor.URL)
|
|
|
|
body := []byte(`{"model":"enso-embed","input":"x"}`)
|
|
c := visit(http.MethodPost, "/v1/embeddings")
|
|
c.Fiber().Request().SetBody(body)
|
|
c.pipeToFamily(enso, "embeddings", "openai", "enso-embed", body, false, 0, "acme", nil, false, nil, time.Now())
|
|
|
|
if len(fake.asked) != 1 {
|
|
t.Errorf("asked=%v — an embeddings refusal walked the chat pool", fake.asked)
|
|
}
|
|
}
|
|
|
|
// The platform's own name for the pool. Every surface can send `free` and get a free
|
|
// answer without knowing or caring which family holds the routes — the branded names
|
|
// are the same pool wearing a brand, this is the same pool wearing none.
|
|
func TestThePlatformPublishesOneUnbrandedFreeId(t *testing.T) {
|
|
fam := freeFamily()
|
|
restore(t, fam)
|
|
fam.spares = []string{"vendor/a:free"}
|
|
fam.byID = map[string]zenModel{"vendor/a:free": {ID: "vendor/a:free", MaxCtx: 64000}}
|
|
fam.ids = []string{"vendor/a:free"}
|
|
fam.loaded, fam.fetchedAt = true, time.Now()
|
|
fam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "openrouter", Type: "OpenRouter", ProviderUrl: "http://or.invalid"}
|
|
}
|
|
|
|
if !fam.frontDoor(freeID) {
|
|
t.Error("the family holding the pool does not answer to the platform's free id")
|
|
}
|
|
if !fam.serves(freeID) {
|
|
t.Error("the platform free id does not route")
|
|
}
|
|
if p, ok := fam.modelPrice(freeID); !ok || p.InputPerMillion != 0 || p.OutputPerMillion != 0 {
|
|
t.Errorf("the platform free id is priced %+v ok=%v", p, ok)
|
|
}
|
|
|
|
byID := indexModels(fam.mergeModels(nil))
|
|
got, ok := byID[freeID]
|
|
if !ok {
|
|
t.Fatal("the platform free id is not listed")
|
|
}
|
|
if got.OwnedBy != "hanzo" {
|
|
t.Errorf("owned_by = %q, want hanzo — it is our product, not the vendor's", got.OwnedBy)
|
|
}
|
|
if got.Premium {
|
|
t.Errorf("the platform free id lists as premium")
|
|
}
|
|
|
|
// A family that does not hold the pool publishes only its own brand.
|
|
enso := &modelFamily{name: "enso", owner: "hanzo", freeName: "enso-free"}
|
|
if enso.frontDoor(freeID) {
|
|
t.Error("a family that does not hold the pool answers to the platform free id")
|
|
}
|
|
}
|
|
|
|
// Our own compute is in the pool, and it is LAST. It cannot run out of a vendor's
|
|
// money, which is why it is there; it is the weakest answer, which is why it is last.
|
|
func TestOurOwnComputeIsThePoolsLastResort(t *testing.T) {
|
|
fam := freeFamily()
|
|
restore(t, fam)
|
|
fam.spares = []string{"vendor/a:free", "vendor/b:free"}
|
|
fam.loaded, fam.fetchedAt = true, time.Now()
|
|
|
|
restore(t, engineFam)
|
|
|
|
// Unconfigured: the pool is the borrowed routes and nothing else.
|
|
engineFam.urlKey = "TEST_ENGINE_URL_UNSET"
|
|
if got := freeRoutes(); len(got) != 2 {
|
|
t.Errorf("pool = %d routes with no engine configured, want the 2 borrowed ones", len(got))
|
|
}
|
|
|
|
// Configured: it joins, at the end.
|
|
t.Setenv("TEST_ENGINE_URL", "http://engine.invalid")
|
|
engineFam.urlKey = "TEST_ENGINE_URL"
|
|
got := freeRoutes()
|
|
if len(got) != 3 {
|
|
t.Fatalf("pool = %d routes, want the 2 borrowed plus our own", len(got))
|
|
}
|
|
if got[2].fam != engineFam || got[2].id != engineModel {
|
|
t.Errorf("last route = %v/%s, want our own compute", got[2].fam.name, got[2].id)
|
|
}
|
|
for _, r := range got[:2] {
|
|
if r.fam == engineFam {
|
|
t.Error("our own compute is ahead of a borrowed route — it is the weakest answer, not the first")
|
|
}
|
|
}
|
|
}
|
|
|
|
// Every route in the pool bills nothing, including our own — a route is in the pool
|
|
// BECAUSE it costs nothing, so charging for one would charge for the very thing that
|
|
// makes it a fallback. Read from the pool, not from any one family's list, because
|
|
// the pool now spans families.
|
|
func TestEveryPoolRouteBillsNothing(t *testing.T) {
|
|
fam := freeFamily()
|
|
restore(t, fam)
|
|
fam.spares = []string{"vendor/a:free"}
|
|
fam.loaded, fam.fetchedAt = true, time.Now()
|
|
|
|
restore(t, engineFam)
|
|
t.Setenv("TEST_ENGINE_URL", "http://engine.invalid")
|
|
engineFam.urlKey = "TEST_ENGINE_URL"
|
|
|
|
if !inPool("vendor/a:free") {
|
|
t.Error("a borrowed free route is not read as free")
|
|
}
|
|
if !inPool(engineModel) {
|
|
t.Error("our own compute is not read as free — an answer from it would be billed")
|
|
}
|
|
if inPool("vendor/paid") {
|
|
t.Error("a route outside the pool reads as free")
|
|
}
|
|
}
|
|
|
|
// The try budget bounds the BORROWED routes, not our own.
|
|
//
|
|
// Each borrowed try is a round trip spent waiting on somebody else, which is what
|
|
// the budget is for. Our own compute is not borrowed and is last, so spending the
|
|
// budget on borrowed routes must not make it unreachable — that would leave the one
|
|
// route no vendor can withdraw as the one route never asked. Measured: a vendor whose
|
|
// data policy refuses every free route answers 404 on all of them, and there are more
|
|
// of them than the budget.
|
|
func TestTheBudgetBoundsBorrowedRoutesNotOurOwn(t *testing.T) {
|
|
cooled.forget()
|
|
fam := freeFamily()
|
|
restore(t, fam)
|
|
restore(t, engineFam)
|
|
|
|
// More borrowed routes than the budget, and every one of them refuses.
|
|
borrowed := []string{"v/a:free", "v/b:free", "v/c:free", "v/d:free", "v/e:free"}
|
|
asked := map[string]int{}
|
|
vendor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var in struct {
|
|
Model string `json:"model"`
|
|
}
|
|
b := make([]byte, r.ContentLength)
|
|
_, _ = r.Body.Read(b)
|
|
_ = json.Unmarshal(b, &in)
|
|
asked[in.Model]++
|
|
if in.Model == engineModel {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"1","model":"` + engineModel + `","choices":[{"index":0,"message":{"role":"assistant","content":"ok"}}]}`))
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNotFound)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"No endpoints available matching your data policy"}}`))
|
|
}))
|
|
defer vendor.Close()
|
|
|
|
fam.spares = borrowed
|
|
fam.loaded, fam.fetchedAt = true, time.Now()
|
|
fam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "openrouter", Type: "OpenRouter", ProviderUrl: vendor.URL}
|
|
}
|
|
t.Setenv("TEST_ENGINE_URL", vendor.URL)
|
|
engineFam.urlKey = "TEST_ENGINE_URL"
|
|
engineFam.providerFn = nil
|
|
|
|
enso := otherFamily(t, vendor.URL)
|
|
body := []byte(`{"model":"enso-free","messages":[{"role":"user","content":"hi"}]}`)
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody(body)
|
|
c.pipeToFamily(enso, "chat/completions", "openai", "enso-free", body, false, 0, "acme", nil, false, nil, time.Now())
|
|
|
|
if asked[engineModel] == 0 {
|
|
t.Fatalf("our own compute was never asked: %v — the budget made the one unwithdrawable route unreachable", asked)
|
|
}
|
|
borrowedTries := 0
|
|
for _, id := range borrowed {
|
|
borrowedTries += asked[id]
|
|
}
|
|
if borrowedTries > spareTries {
|
|
t.Errorf("asked %d borrowed routes, budget is %d", borrowedTries, spareTries)
|
|
}
|
|
}
|
|
|
|
// Settling a debit must not wait on a vendor.
|
|
//
|
|
// The free check walks the pool, and the pool refreshes itself from the vendor on
|
|
// a TTL. Reading it that way on the BILLING path put a network round trip, with its
|
|
// own timeout, between a request that had already succeeded and its debit — so a
|
|
// slow vendor made every debit slow and an unreachable one stalled the ledger. The
|
|
// snapshot answers the same question without asking anybody.
|
|
func TestBillingReadsTheSnapshotAndNeverTheVendor(t *testing.T) {
|
|
fam := freeFamily()
|
|
restore(t, fam)
|
|
|
|
asked := 0
|
|
vendor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
asked++
|
|
_, _ = w.Write([]byte(`{"data":[]}`))
|
|
}))
|
|
defer vendor.Close()
|
|
|
|
fam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "openrouter", Type: "OpenRouter", ProviderUrl: vendor.URL}
|
|
}
|
|
fam.spares = []string{"vendor/a:free"}
|
|
fam.loaded = true
|
|
fam.fetchedAt = time.Now().Add(-time.Hour) // STALE: a refreshing read would dial out
|
|
|
|
if !inPool("vendor/a:free") {
|
|
t.Error("a route in the snapshot is not read as free")
|
|
}
|
|
if inPool("vendor/paid") {
|
|
t.Error("a route outside the pool reads as free")
|
|
}
|
|
if asked != 0 {
|
|
t.Errorf("the billing path dialled the vendor %d time(s) — a debit must not wait on one", asked)
|
|
}
|
|
}
|
|
|
|
// A priced third-party route its vendor cannot serve is refused as itself: no Enso
|
|
// SKU, no free route and none of our own compute answers in its name.
|
|
func TestAPricedThirdPartyRouteIsNeverAnsweredByAnother(t *testing.T) {
|
|
cooled.forget()
|
|
forgetKeys()
|
|
const free = "vendor/big:free"
|
|
const paid = "vendor/paid-a"
|
|
t.Setenv("OPENROUTER_API_KEY", "k1")
|
|
t.Setenv("OPENROUTER_API_KEY_2", "")
|
|
t.Setenv("OPENROUTER_API_KEY_3", "")
|
|
|
|
fake := &refuses{status: http.StatusPaymentRequired, body: `{"error":{"message":"Insufficient credits.","code":402}}`, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
fam := spareFamily(t, vendor.URL, free, paid)
|
|
|
|
var asked []string
|
|
enso := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var in struct {
|
|
Model string `json:"model"`
|
|
}
|
|
b, _ := io.ReadAll(r.Body)
|
|
_ = json.Unmarshal(b, &in)
|
|
asked = append(asked, in.Model)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"1","model":"` + in.Model + `","choices":[{"index":0,"message":{"role":"assistant","content":"ok"}}]}`))
|
|
}))
|
|
defer enso.Close()
|
|
restore(t, ensoFam)
|
|
ensoFam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: enso.URL}
|
|
}
|
|
ensoFam.byID = map[string]zenModel{"enso-pro": {ID: "enso-pro"}, "enso-flash": {ID: "enso-flash"}}
|
|
ensoFam.ids = []string{"enso-flash", "enso-pro"}
|
|
ensoFam.loaded, ensoFam.fetchedAt = true, time.Now()
|
|
|
|
c, out := fake.pipe(t, fam, paid)
|
|
if len(asked) != 0 {
|
|
t.Fatalf("enso asked %v for a named third-party model", asked)
|
|
}
|
|
if len(fake.asked) != 1 || fake.asked[0] != paid {
|
|
t.Fatalf("vendor asked %v, want only %q", fake.asked, paid)
|
|
}
|
|
if out == nil && sent(c) == "" {
|
|
t.Fatal("the refusal vanished")
|
|
}
|
|
if strings.Contains(sent(c), "enso-pro") || strings.Contains(sent(c), `"ok"`) {
|
|
t.Fatalf("another model answered in its name: %s", sent(c))
|
|
}
|
|
}
|
|
|
|
// The route that just refused is not offered the same request again.
|
|
//
|
|
// The pool holds the free routes, and a free route that fell over is IN it — so
|
|
// without the skip the walk would open by asking the one vendor already known to
|
|
// be unable to answer, spending a whole round trip of the caller's time to be
|
|
// told a second time.
|
|
func TestTheRouteThatRefusedIsNotAskedAgain(t *testing.T) {
|
|
cooled.forget()
|
|
const free = "vendor/big:free"
|
|
|
|
asked := map[string]int{}
|
|
vendor := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var in struct {
|
|
Model string `json:"model"`
|
|
}
|
|
b := make([]byte, r.ContentLength)
|
|
_, _ = r.Body.Read(b)
|
|
_ = json.Unmarshal(b, &in)
|
|
asked[in.Model]++
|
|
if in.Model == engineModel {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"1","model":"` + engineModel + `","choices":[{"index":0,"message":{"role":"assistant","content":"ok"}}]}`))
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"internal server error"}}`))
|
|
}))
|
|
defer vendor.Close()
|
|
|
|
fam := spareFamily(t, vendor.URL, free)
|
|
restore(t, engineFam)
|
|
t.Setenv("TEST_ENGINE_URL", vendor.URL)
|
|
engineFam.urlKey = "TEST_ENGINE_URL"
|
|
engineFam.providerFn = nil
|
|
|
|
body := []byte(`{"model":"` + freeID + `","messages":[{"role":"user","content":"2+2?"}]}`)
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody(body)
|
|
c.pipeToFamily(fam, "chat/completions", "openai", freeID, body, false, 0, "acme", nil, false, nil, time.Now())
|
|
|
|
if asked[free] != 1 {
|
|
t.Errorf("the refusing route was asked %d times, want 1 — the walk offered it its own request back", asked[free])
|
|
}
|
|
if asked[engineModel] == 0 {
|
|
t.Errorf("the walk stopped at the route that had already refused: %v", asked)
|
|
}
|
|
}
|
|
|
|
// A route the transport cannot reach at all is stepped over, not the end of the
|
|
// walk. A pool whose borrowed vendor is unreachable still reaches our own.
|
|
func TestAnUnreachableRouteIsSteppedOver(t *testing.T) {
|
|
cooled.forget()
|
|
|
|
askedEngine := 0
|
|
engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
askedEngine++
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"1","model":"` + engineModel + `","choices":[{"index":0,"message":{"role":"assistant","content":"ok"}}]}`))
|
|
}))
|
|
defer engine.Close()
|
|
|
|
// The vendor being asked answers, and refuses with its own failure.
|
|
refuser := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusBadGateway)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"bad gateway"}}`))
|
|
}))
|
|
defer refuser.Close()
|
|
|
|
// The borrowed pool points at a port nothing is listening on, so every borrowed
|
|
// route fails in the transport rather than with a status.
|
|
free := freeFamily()
|
|
restore(t, free)
|
|
free.spares = []string{"v/a:free", "v/b:free"}
|
|
free.loaded, free.fetchedAt = true, time.Now()
|
|
free.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "openrouter", Type: "OpenRouter", ProviderUrl: "http://127.0.0.1:1"}
|
|
}
|
|
|
|
restore(t, engineFam)
|
|
t.Setenv("TEST_ENGINE_URL", engine.URL)
|
|
engineFam.urlKey = "TEST_ENGINE_URL"
|
|
engineFam.providerFn = nil
|
|
|
|
fam := otherFamily(t, refuser.URL)
|
|
body := []byte(`{"model":"enso","messages":[{"role":"user","content":"2+2?"}]}`)
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody(body)
|
|
c.pipeToFamily(fam, "chat/completions", "openai", "enso", body, false, 0, "acme", nil, false, nil, time.Now())
|
|
|
|
if askedEngine == 0 {
|
|
t.Error("an unreachable borrowed route ended the walk; our own route is the one that cannot be withdrawn")
|
|
}
|
|
}
|
|
|
|
// A family whose own catalog publishes its free id serves it as a SKU — its catalog's
|
|
// rungs, fallbacks and vision — and lists it once, from its catalog. Without it the
|
|
// id is the platform pool's.
|
|
func TestAFamilyThatPublishesItsFreeIDServesItItself(t *testing.T) {
|
|
spareFamily(t, "http://vendor.invalid", "free-route")
|
|
fam := otherFamily(t, "http://enso.invalid")
|
|
if !fam.frontDoor("enso-free") {
|
|
t.Fatal("a catalog that does not publish enso-free must leave it to the pool")
|
|
}
|
|
fam.byID = map[string]zenModel{"enso-free": {ID: "enso-free"}}
|
|
fam.ids = []string{"enso-free"}
|
|
if fam.frontDoor("enso-free") {
|
|
t.Error("a catalog that publishes enso-free must serve it itself, not the pool")
|
|
}
|
|
if !fam.serves("enso-free") {
|
|
t.Error("the family no longer serves the free id its catalog publishes")
|
|
}
|
|
for _, n := range fam.freeNames() {
|
|
if n.id == "enso-free" {
|
|
t.Error("enso-free is listed as a pool door while the catalog lists it too")
|
|
}
|
|
}
|
|
}
|
|
|
|
// A stand-in bills at no more than the model the caller named would have cost for
|
|
// the same tokens, and never above the hold.
|
|
func TestAStandInCostsNoMoreThanTheModelNamed(t *testing.T) {
|
|
enso := otherFamily(t, "http://enso.invalid")
|
|
enso.byID = map[string]zenModel{"enso-pro": {ID: "enso-pro", Base: zenTier{In: decimal.New(4, 0), Out: decimal.New(13, 0)}}}
|
|
enso.ids = []string{"enso-pro"}
|
|
resale := otherFamily(t, "http://vendor.invalid")
|
|
resale.byID = map[string]zenModel{
|
|
"vendor/cheap": {ID: "vendor/cheap", Base: zenTier{In: decimal.New(1, 0), Out: decimal.New(1, 0)}},
|
|
"vendor/dear": {ID: "vendor/dear", Base: zenTier{In: decimal.New(15, 0), Out: decimal.New(75, 0)}},
|
|
}
|
|
c := visit(http.MethodPost, "/v1/x")
|
|
w := whence{ledger: c.billingOrg(nil), ip: c.Fiber().IP(), ctx: c.Context(), asked: resale}
|
|
use := tokens{fresh: 1_000_000, completion: 1_000_000}
|
|
|
|
if nano := recordFamilyUsage(w, enso, "enso-pro", "vendor/cheap", nil, &mark{}, nil, true, false, "r1", use, serving{}, time.Now(), nil, "success", ""); nano != 2_000_000_000 {
|
|
t.Errorf("enso-pro standing in for a $1/$1 model billed %d nano, want $2 — the named model's price", nano)
|
|
}
|
|
if nano := recordFamilyUsage(w, enso, "enso-pro", "vendor/dear", nil, &mark{}, nil, true, false, "r2", use, serving{}, time.Now(), nil, "success", ""); nano != 17_000_000_000 {
|
|
t.Errorf("enso-pro standing in for a $15/$75 model billed %d nano, want $17 — its own price", nano)
|
|
}
|
|
// An answer that cost more than its hold was estimated at is billed in full:
|
|
// the hold is an estimate, never a price.
|
|
hold := &budgetHold{est: 6}
|
|
if nano := recordFamilyUsage(w, resale, "vendor/dear", "", nil, &mark{}, nil, true, false, "r3", use, serving{}, time.Now(), hold, "success", ""); nano != 90_000_000_000 {
|
|
t.Errorf("billed %d nano for a $15/$75 answer held at 6 cents, want $90", nano)
|
|
}
|
|
}
|
|
|
|
// A Hanzo family that cannot be reached at all moves its request to the pool like a
|
|
// refusal: its SKU is ours to route.
|
|
func TestAnUnreachableFamilyIsAskedOnceAndItsCallerStillAnswered(t *testing.T) {
|
|
cooled.forget()
|
|
forgetKeys()
|
|
const free = "vendor/big:free"
|
|
dead := httptest.NewServer(http.NotFoundHandler())
|
|
deadURL := dead.URL
|
|
dead.Close()
|
|
|
|
fake := &refuses{status: http.StatusServiceUnavailable, body: `{}`, free: free}
|
|
pool := fake.serve(t)
|
|
defer pool.Close()
|
|
spareFamily(t, pool.URL, free)
|
|
gone := otherFamily(t, deadURL)
|
|
body := []byte(`{"model":"enso-flash","messages":[{"role":"user","content":"2+2?"}]}`)
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody(body)
|
|
if out := c.pipeToFamily(gone, "chat/completions", "openai", "enso-flash", body, false, 0, "acme", nil, false, nil, time.Now()); out != nil {
|
|
t.Fatalf("attempts=%+v, want the pool to answer an unreachable family", out)
|
|
}
|
|
if !strings.Contains(sent(c), `"ok"`) {
|
|
t.Fatalf("body %s, want the pool's answer", sent(c))
|
|
}
|
|
}
|
|
|
|
// With the paid lane off, a priced third-party route is never sent and nothing
|
|
// answers in its place: it is refused, plainly. A free route is sent as ever.
|
|
func TestWithThePaidLaneOffAPricedRouteIsNeverSent(t *testing.T) {
|
|
cooled.forget()
|
|
forgetKeys()
|
|
FreeOnly = func() bool { return true }
|
|
t.Cleanup(func() { FreeOnly = func() bool { return false } })
|
|
const free = "vendor/big:free"
|
|
const paid = "vendor/paid-a"
|
|
t.Setenv("OPENROUTER_API_KEY", "k1")
|
|
t.Setenv("OPENROUTER_API_KEY_2", "")
|
|
t.Setenv("OPENROUTER_API_KEY_3", "")
|
|
|
|
fake := &refuses{status: http.StatusOK, body: `{}`, free: free}
|
|
vendor := fake.serve(t)
|
|
defer vendor.Close()
|
|
fam := spareFamily(t, vendor.URL, free, paid)
|
|
|
|
var asked []string
|
|
enso := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var in struct {
|
|
Model string `json:"model"`
|
|
}
|
|
b, _ := io.ReadAll(r.Body)
|
|
_ = json.Unmarshal(b, &in)
|
|
asked = append(asked, in.Model)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"1","model":"` + in.Model + `","choices":[{"index":0,"message":{"role":"assistant","content":"ok"}}]}`))
|
|
}))
|
|
defer enso.Close()
|
|
restore(t, ensoFam)
|
|
ensoFam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: enso.URL}
|
|
}
|
|
ensoFam.loaded, ensoFam.fetchedAt = true, time.Now()
|
|
|
|
c, out := fake.pipe(t, fam, paid)
|
|
if out != nil {
|
|
t.Fatalf("attempts=%+v, want the refusal answered here", out)
|
|
}
|
|
if len(fake.asked) != 0 || len(asked) != 0 {
|
|
t.Fatalf("vendor asked %v, enso asked %v — nothing may be sent for a priced third-party route with the paid lane off", fake.asked, asked)
|
|
}
|
|
if !strings.Contains(sent(c), "third-party models are not being served") {
|
|
t.Fatalf("body %s, want the plain reason", sent(c))
|
|
}
|
|
|
|
// A free route is sent as ever.
|
|
fake.asked = nil
|
|
if _, out := fake.pipe(t, fam, free); out != nil || len(fake.asked) != 1 || fake.asked[0] != free {
|
|
t.Fatalf("free route: attempts=%+v asked=%v, want it sent once", out, fake.asked)
|
|
}
|
|
}
|
|
|
|
// A priced route asks for the fastest provider OpenRouter has at no more than the
|
|
// SKU's retail; a free route, a per-call price and a caller's own choice are left
|
|
// alone.
|
|
func TestOpenRouterTermsRouteByLatencyUnderRetail(t *testing.T) {
|
|
priced := zenModel{Base: zenTier{In: decimal.MustParse("0.039035"), Out: decimal.MustParse("0.17935")}}
|
|
read := func(b []byte) (sort string, max map[string]float64) {
|
|
var m struct {
|
|
Provider struct {
|
|
Sort string `json:"sort"`
|
|
MaxPrice map[string]float64 `json:"max_price"`
|
|
} `json:"provider"`
|
|
}
|
|
if err := json.Unmarshal(b, &m); err != nil {
|
|
t.Fatalf("body is not JSON after terms: %v", err)
|
|
}
|
|
return m.Provider.Sort, m.Provider.MaxPrice
|
|
}
|
|
body := []byte(`{"model":"openai/gpt-oss-120b","messages":[]}`)
|
|
if s, max := read(openrouterTerms(body, priced, false, "")); s != "latency" || max["prompt"] != 0.039035 || max["completion"] != 0.17935 {
|
|
t.Errorf("priced route: sort %q max_price %v, want latency under 0.039035/0.17935", s, max)
|
|
}
|
|
if s, max := read(openrouterTerms(body, priced, true, "")); s != "" || max != nil {
|
|
t.Errorf("free route: sort %q max_price %v, want neither", s, max)
|
|
}
|
|
variable := priced
|
|
variable.Variable = true
|
|
if s, max := read(openrouterTerms(body, variable, false, "")); s != "" || max != nil {
|
|
t.Errorf("per-call price: sort %q max_price %v, want neither", s, max)
|
|
}
|
|
if s, max := read(openrouterTerms(body, zenModel{}, false, "")); s != "" || max != nil {
|
|
t.Errorf("unpriced SKU: sort %q max_price %v, want neither", s, max)
|
|
}
|
|
own := []byte(`{"model":"openai/gpt-oss-120b","provider":{"sort":"throughput","max_price":{"prompt":1}}}`)
|
|
if s, max := read(openrouterTerms(own, priced, false, "")); s != "throughput" || max["prompt"] != 1 || len(max) != 1 {
|
|
t.Errorf("caller's own choice: sort %q max_price %v, want it kept", s, max)
|
|
}
|
|
}
|