Files
ai/controllers/listing_test.go
zeekay 1569f5288b usage: one prompt on every path, and cache writes billed as writes
prompt_tokens meant two things. The family and relay paths recorded the
part of the prompt read fresh, the table path and the providers the
whole prompt, so a cached enso-auto row read cache_read > prompt. No
path recorded a cache write: Anthropic's cache_creation_input_tokens
was folded into the fresh read and billed at the input rate, though a
write is priced above it, and OpenRouter's cache_write_tokens was not
read at all. cache_write_tokens was 0 on every row.

A record's prompt is now every input token, with the cache reads and
writes parts of it, whatever shape the upstream reported: OpenAI's
cached_tokens inside prompt_tokens, OpenRouter's cache_write_tokens
beside them, Anthropic's two counts beside input_tokens. Pricing bills
the input rate on the part read fresh and each cache part at its own
rate: a family SKU's cache_read and cache_write (a resold model's
input_cache_read and input_cache_write plus the vendor's fee), else the
input rate. The Anthropic provider and the OpenAI adapter over
Anthropic now read both cache counts, and the providers' settles pass
them.
2026-10-09 03:59:41 -07:00

424 lines
16 KiB
Go

// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
package controllers
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/hanzoai/ai/object"
"github.com/hanzoai/decimal"
"github.com/zap-proto/zip"
)
// vendor is an OpenRouter double whose list a test changes between syncs.
type vendor struct {
mu sync.Mutex
body string
hits atomic.Int32
}
func (v *vendor) set(body string) {
v.mu.Lock()
v.body = body
v.mu.Unlock()
}
// listingVendor points the OpenRouter family at a vendor double serving body, with a
// store of its own and a recorder for what the bus is told.
func listingVendor(t *testing.T, body string) (*vendor, *[]object.ListingEvent) {
t.Helper()
v := &vendor{body: body}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/models" {
http.NotFound(w, r)
return
}
v.hits.Add(1)
v.mu.Lock()
defer v.mu.Unlock()
_, _ = io.WriteString(w, v.body)
}))
t.Cleanup(srv.Close)
withFamily(t, openrouterFam, "")
t.Setenv(openrouterFam.urlKey, srv.URL)
var mu sync.Mutex
told := &[]object.ListingEvent{}
object.SetListingPublisher(func(_ context.Context, evs []object.ListingEvent) {
mu.Lock()
defer mu.Unlock()
*told = append(*told, evs...)
})
t.Cleanup(func() { object.SetListingPublisher(nil) })
return v, told
}
// syncNow runs a sync as the next interval's would.
func syncNow(t *testing.T) *synced {
t.Helper()
s, err := openrouterFam.syncEvery(context.Background(), openrouterFam.provider(), 0, time.Now())
if err != nil {
t.Fatal(err)
}
if s == nil {
t.Fatal("the sync did not take its interval")
}
return s
}
const (
haiku = `{"id":"anthropic/claude-haiku-5.5","name":"Anthropic: Claude Haiku 5.5","created":1791397883,` +
`"context_length":1000000,"architecture":{"input_modalities":["text","image","file"],"output_modalities":["text"]},` +
`"pricing":{"prompt":"0.0000001","completion":"0.0000005","web_search":"0.01","input_cache_read":"0.00000001",` +
`"input_cache_write":"0.000000125","overrides":[{"min_prompt_tokens":100000,"prompt":"0.0000005","completion":"0.0000025","input_cache_read":"0.00000005"}]},` +
`"top_provider":{"context_length":1000000,"max_completion_tokens":64000,"is_moderated":false},` +
`"supported_parameters":["tools","reasoning","structured_outputs"],"expiration_date":null}`
gemma = `{"id":"google/gemma-4-31b-it:free","context_length":262144,` +
`"architecture":{"input_modalities":["text","image"],"output_modalities":["text"]},` +
`"pricing":{"prompt":"0","completion":"0"},"top_provider":{"max_completion_tokens":8192},` +
`"supported_parameters":["tools"],"expiration_date":"2026-12-31"}`
laguna = `{"id":"poolside/laguna-s-2.1","context_length":131072,` +
`"architecture":{"input_modalities":["text"],"output_modalities":["text"]},` +
`"pricing":{"prompt":"0.0000003","completion":"0.0000012"}}`
)
func listOf(items ...string) string { return `{"data":[` + strings.Join(items, ",") + `]}` }
// A sync keeps every model the vendor lists, each entry exactly as listed, and the
// family serves the store's list: what OpenRouter lists is what we list, priced at
// its rates plus the host's fee, its free models marked free.
func TestASyncKeepsEveryModelAsListed(t *testing.T) {
_, told := listingVendor(t, listOf(haiku, gemma, laguna))
fee(t, "20")
s := syncNow(t)
if s.listed != 3 || s.count(object.ListingNew) != 3 {
t.Fatalf("synced %d with %d new, want 3 and 3", s.listed, s.count(object.ListingNew))
}
rows, err := object.Listings("openrouter")
if err != nil || len(rows) != 3 {
t.Fatalf("store holds %d rows (%v), want 3", len(rows), err)
}
for _, r := range rows {
want := map[string]string{"anthropic/claude-haiku-5.5": haiku, "google/gemma-4-31b-it:free": gemma, "poolside/laguna-s-2.1": laguna}[r.ID]
if r.Item != want {
t.Errorf("%s kept as\n %s\nlisted as\n %s", r.ID, r.Item, want)
}
if r.Free != (r.ID == "google/gemma-4-31b-it:free") {
t.Errorf("%s free = %v", r.ID, r.Free)
}
}
if len(*told) != 0 {
t.Errorf("a sync that seeded the store told the bus %d changes, want none", len(*told))
}
openrouterFam.mu.Lock()
openrouterFam.loaded = false
openrouterFam.mu.Unlock()
for _, id := range []string{"anthropic/claude-haiku-5.5", "google/gemma-4-31b-it:free", "poolside/laguna-s-2.1"} {
if !openrouterFam.serves(id) {
t.Errorf("%s is listed and not served", id)
}
}
m, _ := openrouterFam.lookup("anthropic/claude-haiku-5.5")
if m.MaxOut != 64000 || m.Card == nil || m.Card.Rates["web_search"] != "0.012" || m.Card.Rates["input_cache_write"] != "0.00000015" {
t.Errorf("haiku lost its contract: max out %d, card %+v", m.MaxOut, m.Card)
}
if g, _ := openrouterFam.lookup("google/gemma-4-31b-it:free"); g.Expires != "2026-12-31" || g.priced() {
t.Errorf("gemma expires %q priced %v", g.Expires, g.priced())
}
}
// What a sync reads changing is recorded and told: a new model, a repricing, a
// model turned from free to paid and back, a model gone and back. A gone model is
// served no longer, and its row stays.
func TestASyncRecordsWhatChanged(t *testing.T) {
v, told := listingVendor(t, listOf(haiku, gemma, laguna))
syncNow(t)
repriced := strings.Replace(laguna, `"prompt":"0.0000003"`, `"prompt":"0.00000035"`, 1)
paid := strings.Replace(gemma, `"prompt":"0","completion":"0"`, `"prompt":"0.0000001","completion":"0.0000002"`, 1)
v.set(listOf(paid, repriced))
s := syncNow(t)
kinds := map[string]string{}
for _, e := range s.events {
kinds[e.ID] = e.Kind
}
want := map[string]string{"google/gemma-4-31b-it:free": object.ListingPaid, "poolside/laguna-s-2.1": object.ListingPrice, "anthropic/claude-haiku-5.5": object.ListingGone}
for id, k := range want {
if kinds[id] != k {
t.Errorf("%s: %q, want %q", id, kinds[id], k)
}
}
if len(s.events) != 3 {
t.Errorf("%d changes, want 3: %+v", len(s.events), s.events)
}
openrouterFam.mu.Lock()
openrouterFam.loaded = false
openrouterFam.mu.Unlock()
if openrouterFam.serves("anthropic/claude-haiku-5.5") {
t.Error("a model the vendor dropped is still served")
}
rows, _ := object.Listings("openrouter")
gone := false
for _, r := range rows {
if r.ID == "anthropic/claude-haiku-5.5" {
gone = r.Gone != ""
}
}
if !gone {
t.Error("a dropped model's row was not kept, marked gone")
}
v.set(listOf(haiku, gemma, repriced))
s = syncNow(t)
kinds = map[string]string{}
for _, e := range s.events {
kinds[e.ID] = e.Kind
}
if kinds["anthropic/claude-haiku-5.5"] != object.ListingBack || kinds["google/gemma-4-31b-it:free"] != object.ListingFree || len(s.events) != 2 {
t.Errorf("the third sync read %+v", s.events)
}
if len(*told) != 3+2 {
t.Errorf("the bus was told %d changes, want the 5 after the seeding sync", len(*told))
}
evs, err := object.ListingEvents("openrouter", "google/gemma-4-31b-it:free", 10)
if err != nil || len(evs) != 3 || evs[0].Kind != object.ListingFree || evs[2].Kind != object.ListingNew {
t.Fatalf("gemma's history %+v (%v), want free, paid, new", evs, err)
}
if !strings.Contains(evs[1].Now, `"prompt":"0.0000001"`) || !strings.Contains(evs[1].Was, `"prompt":"0"`) {
t.Errorf("a turn to paid states its prices: was %s now %s", evs[1].Was, evs[1].Now)
}
}
// A price the vendor spells differently is the same price: no change is read.
func TestASpellingIsNoChange(t *testing.T) {
v, _ := listingVendor(t, listOf(laguna))
syncNow(t)
v.set(listOf(strings.Replace(laguna, `"prompt":"0.0000003"`, `"prompt":"0.00000030"`, 1)))
if s := syncNow(t); len(s.events) != 0 {
t.Errorf("a respelled price read as %+v", s.events)
}
}
// A list of nothing is a list gone wrong: it marks nothing gone.
func TestAnEmptyListChangesNothing(t *testing.T) {
v, _ := listingVendor(t, listOf(laguna))
syncNow(t)
v.set(`{"data":[]}`)
if _, err := openrouterFam.syncEvery(context.Background(), openrouterFam.provider(), 0, time.Now()); err == nil {
t.Fatal("an empty list synced")
}
rows, _ := object.Listings("openrouter")
if len(rows) != 1 || rows[0].Gone != "" {
t.Errorf("an empty list changed the store: %+v", rows)
}
}
// One process syncs an interval; the rest read what it wrote. A sync that fails
// gives the interval back.
func TestOneProcessSyncsAnInterval(t *testing.T) {
v, _ := listingVendor(t, listOf(laguna))
now := time.Now()
if s, err := openrouterFam.syncEvery(context.Background(), openrouterFam.provider(), listingEvery, now); err != nil || s == nil {
t.Fatalf("the first process did not sync: %v", err)
}
if s, err := openrouterFam.syncEvery(context.Background(), openrouterFam.provider(), listingEvery, now.Add(time.Minute)); err != nil || s != nil {
t.Fatalf("a second process synced the same interval: %v", err)
}
if got := v.hits.Load(); got != 1 {
t.Errorf("the vendor was asked %d times in one interval", got)
}
v.set(`{"data":[]}`)
later := now.Add(listingEvery + time.Minute)
if _, err := openrouterFam.syncEvery(context.Background(), openrouterFam.provider(), listingEvery, later); err == nil {
t.Fatal("a failing sync reported none")
}
v.set(listOf(laguna))
if s, err := openrouterFam.syncEvery(context.Background(), openrouterFam.provider(), listingEvery, later.Add(time.Second)); err != nil || s == nil {
t.Fatalf("a failed sync kept its interval: %v", err)
}
}
// The list and its history are public, in the vendor's own shape: the free lane
// reads ?free=1 in place of the vendor's list.
func TestTheListingIsServedAsTheVendorListsIt(t *testing.T) {
listingVendor(t, listOf(haiku, gemma, laguna))
syncNow(t)
status, body := fetched(t, "/v1/models/vendors/:vendor", "/v1/models/vendors/openrouter?free=1", (*ApiController).ListListing)
var got struct {
Vendor string `json:"vendor"`
Synced string `json:"synced"`
Data []json.RawMessage `json:"data"`
}
if err := json.Unmarshal([]byte(body), &got); err != nil || status != http.StatusOK {
t.Fatalf("answered %d %s", status, body)
}
if got.Vendor != "openrouter" || got.Synced == "" || len(got.Data) != 1 || string(got.Data[0]) != gemma {
t.Errorf("free listing %+v", got)
}
_, body = fetched(t, "/v1/models/vendors/:vendor/events", "/v1/models/vendors/openrouter/events", (*ApiController).ListListingEvents)
var evs struct {
Data []object.ListingEvent `json:"data"`
}
if err := json.Unmarshal([]byte(body), &evs); err != nil || len(evs.Data) != 3 {
t.Errorf("history answered %s", body)
}
if status, _ := fetched(t, "/v1/models/vendors/:vendor", "/v1/models/vendors/nobody", (*ApiController).ListListing); status != http.StatusNotFound {
t.Errorf("an unlisted vendor answered %d", status)
}
}
// fetched answers a GET of target through a route of pattern, so the handler reads
// its path parameters as it does when served.
func fetched(t *testing.T, pattern, target string, h func(*ApiController)) (int, string) {
t.Helper()
app := zip.New(zip.Config{DisableStartupMessage: true})
app.Raw(http.MethodGet, pattern, func(c *zip.Ctx) error {
h(&ApiController{Ctx: c})
return nil
})
res, err := app.Test(httptest.NewRequest(http.MethodGet, target, nil), zip.TestConfig{Timeout: 5 * time.Second})
if err != nil {
t.Fatal(err)
}
b, _ := io.ReadAll(res.Body)
return res.StatusCode, string(b)
}
// /v1/models lists a resold model with every rate its vendor states, each times the
// fee exactly, its conditional rates beside them, and says whether it is free.
func TestTheCatalogPublishesEveryRateExactly(t *testing.T) {
listingVendor(t, listOf(haiku, gemma))
fee(t, "20")
syncNow(t)
openrouterFam.mu.Lock()
openrouterFam.loaded = false
openrouterFam.mu.Unlock()
openrouterFam.fresh()
byID := map[string]modelInfo{}
for _, m := range openrouterFam.mergeModels(nil) {
byID[m.ID] = m
}
h := byID["anthropic/claude-haiku-5.5"]
h.Pricing = pricingInfo(openrouterFam.modelPrice(h.ID))
if h.MaxOutputTokens != 64000 || h.Pricing == nil || h.Pricing.Prompt != "0.00000012" || h.Pricing.Completion != "0.0000006" ||
h.Pricing.Rates["input_cache_read"] != "0.000000012" || h.Pricing.Rates["web_search"] != "0.012" || len(h.Pricing.Overrides) != 1 {
t.Fatalf("haiku listed as %+v %+v", h, h.Pricing)
}
if o := h.Pricing.Overrides[0]; o["prompt"] != "0.0000006" || fmt.Sprint(o["min_prompt_tokens"]) != "100000" {
t.Errorf("haiku's override listed as %v", o)
}
g := byID["google/gemma-4-31b-it:free"]
if g.Expires != "2026-12-31" || g.Premium {
t.Errorf("gemma listed as %+v", g)
}
}
// A resold SKU bills at the tier its prompt reaches, the dearer side of an
// override's line, and a cached token at the cache rate. A discount by hours of the
// day is never billed, so no call bills below the base.
func TestAnOverrideBillsTheDearerTier(t *testing.T) {
var w openrouterWireModel
body := strings.Replace(haiku, `"overrides":[`, `"overrides":[{"utc_start":0,"utc_end":800,"prompt":"0.00000001"},`, 1)
if err := json.Unmarshal([]byte(body), &w); err != nil {
t.Fatal(err)
}
m := w.model(decimal.MustParse("1.2"))
if len(m.Tiers) != 2 || m.Tiers[0].MaxCtx != 99999 {
t.Fatalf("tiers %+v", m.Tiers)
}
if got := m.bill(nil, tokens{fresh: 99999, cached: 0, completion: 0}); got != nanoUp(decimal.MustParse("0.00000012").Mul(decimal.New(99999, 0))) {
t.Errorf("a prompt under the line billed %d", got)
}
if got := m.bill(nil, tokens{fresh: 100000, cached: 0, completion: 0}); got != nanoUp(decimal.MustParse("0.0000006").Mul(decimal.New(100000, 0))) {
t.Errorf("a prompt at the line billed %d", got)
}
if got := m.bill(nil, tokens{fresh: 0, cached: 1000, completion: 0}); got != nanoUp(decimal.MustParse("0.000000012").Mul(decimal.New(1000, 0))) {
t.Errorf("a thousand cached tokens billed %d", got)
}
}
// The free view is the free models listed now, each with what the last day did with
// it and which of our accounts served it.
func TestTheFreeViewCountsEachAccount(t *testing.T) {
listingVendor(t, listOf(haiku, gemma))
syncNow(t)
now := time.Now()
for _, c := range []struct {
account string
ok bool
}{{"OPENROUTER_API_KEY", true}, {"OPENROUTER_API_KEY", true}, {"OPENROUTER_API_KEY_2", false}, {"OPENROUTER_API_KEY_2", true}} {
if err := object.CountServed("openrouter", "google/gemma-4-31b-it:free", c.account, now, c.ok); err != nil {
t.Fatal(err)
}
}
if err := object.CountServed("openrouter", "google/gemma-4-31b-it:free", "OPENROUTER_API_KEY", now.Add(-25*time.Hour), false); err != nil {
t.Fatal(err)
}
set, err := freeSet("openrouter", now)
if err != nil || len(set) != 1 {
t.Fatalf("free set %+v (%v), want gemma alone", set, err)
}
g := set[0]
if g.ID != "google/gemma-4-31b-it:free" || g.Answered != 3 || g.Failed != 1 || g.Rate == nil || *g.Rate != 0.75 || len(g.Accounts) != 2 {
t.Fatalf("gemma's day %+v", g)
}
if g.Accounts[1] != (freeAccount{Account: "OPENROUTER_API_KEY_2", Answered: 1, Failed: 1}) {
t.Errorf("the second account's day %+v", g.Accounts[1])
}
}
// Each free request ai sends counts toward the account it went on: refused on one,
// answered on the next.
func TestAFreeRequestCountsTheAccountItWentOn(t *testing.T) {
listingVendor(t, listOf(gemma))
syncNow(t)
t.Setenv("OPENROUTER_API_KEY", "k-one")
t.Setenv("OPENROUTER_API_KEY_2", "k-two")
do := func(r *http.Request) (*http.Response, error) {
status := http.StatusOK
if r.Header.Get("Authorization") == "Bearer k-one" {
status = http.StatusTooManyRequests
}
return &http.Response{StatusCode: status, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("{}"))}, nil
}
r, _ := http.NewRequest(http.MethodPost, "http://vendor/v1/chat/completions", strings.NewReader(`{"model":"google/gemma-4-31b-it:free"}`))
count := openrouterFam.counting(r, do)
for _, k := range []string{"k-one", "k-two"} {
rr := r.Clone(context.Background())
rr.Header.Set("Authorization", "Bearer "+k)
if _, err := count(rr); err != nil {
t.Fatal(err)
}
}
var rows []object.Served
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
rows, _ = object.ServedSince("openrouter", time.Now().Add(-time.Hour))
if len(rows) == 2 {
break
}
}
got := map[string][2]int64{}
for _, s := range rows {
got[s.Account] = [2]int64{s.Answered, s.Failed}
}
if got["OPENROUTER_API_KEY"] != [2]int64{0, 1} || got["OPENROUTER_API_KEY_2"] != [2]int64{1, 0} {
t.Errorf("served %+v", rows)
}
}