Files
ai/controllers/native_test.go
zeekay a3cd3b4309 fee: a call bills what its arms cost and the platform's fee; Enso is quoted from, typical and up to
A call through the gateway bills the price of the arm that answered and the
platform's fee on top, and nothing else: 5.5% on Hanzo's accounts, 1% on the
org's own key, both the host's settings (object.Fee: hanzo, hanzo_byok). A
vendor's fee is part of the cost: a call OpenRouter's credit bought cost its
stated price plus 5.5%, and the platform's fee goes on that.

- Enso answers bill their cost of record (X-Hanzo-Cogs and the buying
  vendor's fee) times the platform's fee; a maker's answer the same; a resold
  OpenRouter SKU its vendor price times both fees. An answer that states no
  cost bills the SKU's listed ceiling. A call billed below its cost writes
  loss_nano and a warning.
- hanzo.cloud_usage gains reasoning, reasoning_tokens, served, vendor, arms
  and loss_nano: each Enso row names the arm that answered, its rate at the
  time and the tokens it was paid for, and each arm that failed first.
- rates.go prices every Enso SKU at each depth: from (its cheapest arm), up to
  (its dearest, a long prompt included), and typical (its measured cost a
  token over the last day, or its last 50 calls), the arms read from the
  engine's admin catalog. /v1/models lists the typical rate with from, up_to,
  fee and window beside it; Prices, Price and Blend serve the rest. The
  board's ledger counts answers, failovers, OpenRouter-bought answers, fee
  and losses by model and vendor.
2026-10-15 12:33:32 -07:00

927 lines
40 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"slices"
"strconv"
"strings"
"sync"
"testing"
"time"
iam "github.com/hanzoai/ai/internal/iam"
"github.com/hanzoai/ai/object"
"github.com/hanzoai/decimal"
)
// claudeSKU is the SKU the cases ask for, as the resale catalog spells it, and
// claudeID what its maker calls it.
const (
claudeSKU = "anthropic/claude-haiku-5.5"
claudeID = "claude-haiku-5-5"
)
// maker is a model vendor's own API on httptest: a list of models for each key,
// and a Messages endpoint that answers each key as the case says and remembers
// what it was sent.
type maker struct {
*httptest.Server
mu sync.Mutex
// lists is the status GET /v1/models answers a key with, and refuse what
// POST /v1/messages answers it with; a key in neither is answered 200.
lists map[string]int
refuse map[string]turnedAway
// events is the stream a 200 answers with; nil for one that says "ok".
events []string
// whole answers a request for a stream with one JSON body.
whole bool
asked []string // the key of each Messages call, in order
bodies []string
heads []http.Header
read []string // the key of each list read
}
// turnedAway is what the maker answers a key with in place of an answer.
type turnedAway struct {
status int
kind string
message string
after string // Retry-After
}
// makerModels is what every key lists: one model that thinks adaptively and takes
// no sampling parameter, one dated release that thinks within a budget.
const makerModels = `{"data":[
{"type":"model","id":"claude-haiku-5-5","max_tokens":128000,"capabilities":{
"image_input":{"supported":true},"structured_outputs":{"supported":true},
"thinking":{"supported":true,"types":{"enabled":{"supported":false},"adaptive":{"supported":true}}},
"effort":{"supported":true,"low":{"supported":true},"medium":{"supported":true},"high":{"supported":true},"xhigh":{"supported":false},"max":{"supported":true}}}},
{"type":"model","id":"claude-haiku-4-5-20251001","max_tokens":64000,"capabilities":{
"image_input":{"supported":true},"structured_outputs":{"supported":false},
"thinking":{"supported":true,"types":{"enabled":{"supported":true},"adaptive":{"supported":false}}},
"effort":{"supported":false}}},
{"type":"model","id":"claude-haiku-4-5-20250101","max_tokens":64000,"capabilities":{}},
{"type":"model","id":"claude-free-1","max_tokens":4096,"capabilities":{}}
],"has_more":false,"last_id":"claude-free-1"}`
// okEvents is the stream of an answer that says "ok": 1,000 prompt tokens read
// fresh, 3,000 read from the cache, 2,000 written to it, and 500 written out.
var okEvents = []string{
`{"type":"message_start","message":{"id":"msg_vendor","type":"message","role":"assistant","model":"claude-haiku-5-5","content":[],"usage":{"input_tokens":1000,"cache_creation_input_tokens":2000,"cache_read_input_tokens":3000,"output_tokens":1}}}`,
`{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
`{"type":"ping"}`,
`{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}`,
`{"type":"content_block_stop","index":0}`,
`{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":500}}`,
`{"type":"message_stop"}`,
}
func (m *maker) serve(w http.ResponseWriter, r *http.Request) {
key := r.Header.Get("x-api-key")
if r.URL.Path == "/v1/models" {
m.mu.Lock()
m.read = append(m.read, key)
status := m.lists[key]
m.mu.Unlock()
w.Header().Set("Content-Type", "application/json")
if status != 0 && status != http.StatusOK {
w.WriteHeader(status)
_, _ = io.WriteString(w, `{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}`)
return
}
_, _ = io.WriteString(w, makerModels)
return
}
b, _ := io.ReadAll(r.Body)
var in struct {
Stream bool `json:"stream"`
}
_ = json.Unmarshal(b, &in)
m.mu.Lock()
m.asked = append(m.asked, key)
m.bodies = append(m.bodies, string(b))
m.heads = append(m.heads, r.Header.Clone())
no, refused := m.refuse[key]
events := m.events
if m.whole {
in.Stream = false
}
m.mu.Unlock()
if refused {
w.Header().Set("Content-Type", "application/json")
if no.after != "" {
w.Header().Set("Retry-After", no.after)
}
w.WriteHeader(no.status)
fmt.Fprintf(w, `{"type":"error","error":{"type":%q,"message":%q}}`, no.kind, no.message)
return
}
if events == nil {
events = okEvents
}
if !in.Stream {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"id":"msg_vendor","type":"message","role":"assistant","model":"claude-haiku-5-5","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":1000,"cache_creation_input_tokens":2000,"cache_read_input_tokens":3000,"output_tokens":500}}`)
return
}
w.Header().Set("Content-Type", "text/event-stream")
for _, e := range events {
var typ struct {
Type string `json:"type"`
}
_ = json.Unmarshal([]byte(e), &typ)
fmt.Fprintf(w, "event: %s\ndata: %s\n\n", typ.Type, e)
}
}
func (m *maker) calls() []string {
m.mu.Lock()
defer m.mu.Unlock()
return slices.Clone(m.asked)
}
// sentBody is the body of the maker's i-th Messages call, decoded.
func (m *maker) sentBody(t *testing.T, i int) map[string]json.RawMessage {
t.Helper()
m.mu.Lock()
defer m.mu.Unlock()
if i >= len(m.bodies) {
t.Fatalf("the maker was sent %d calls, want more than %d", len(m.bodies), i)
}
return decoded(t, []byte(m.bodies[i]))
}
// makerAt starts a maker, makes it Anthropic's address with keys under the
// reserved names in order, and reads what each key lists.
func makerAt(t *testing.T, m *maker, keys ...string) *maker {
t.Helper()
m.Server = httptest.NewServer(http.HandlerFunc(m.serve))
t.Cleanup(m.Close)
t.Setenv("ANTHROPIC_URL", m.URL)
for i, name := range object.AnthropicKeys {
key := ""
if i < len(keys) {
key = keys[i]
}
t.Setenv(name, key)
}
t.Setenv(object.AnthropicKeyList, "")
forgetKeys()
cooled.forget()
anthropicNative.forget()
t.Cleanup(func() {
forgetKeys()
cooled.forget()
anthropicNative.forget()
})
anthropicNative.refresh()
return m
}
// resale points the resale family at a vendor that answers each of its keys as
// the case says, listing the Claude SKU at the vendor's price plus a 5.5% fee, a free
// route for the pool, and whatever else a case names.
func resale(t *testing.T, a *accounts, also ...string) *modelFamily {
t.Helper()
restore(t, engineFam)
engineFam.urlKey = "TEST_ENGINE_URL_UNSET"
engineFam.providerFn = nil
t.Setenv("OPENROUTER_API_KEY", "k1")
t.Setenv("OPENROUTER_API_KEY_2", "")
t.Setenv("OPENROUTER_API_KEY_3", "")
fam := spareFamily(t, a.serve(t).URL, "vendor/big:free", also...)
// A maker's answer bills what it cost, the platform's fee on top (fee.go).
fees(t, map[string]string{"hanzo": "5.5"})
// $0.10 in, $0.50 out, $0.01 a cached read and $0.125 a cache write a million
// tokens at the vendor, each times 1.055.
tier := zenTier{
MaxCtx: 1_000_000,
In: decimal.MustParse("0.1055"), Out: decimal.MustParse("0.5275"),
CacheRead: decimal.MustParse("0.01055"), CacheWrite: decimal.MustParse("0.131875"),
}
fam.byID[claudeSKU] = zenModel{ID: claudeSKU, Margin: decimal.MustParse("1.055"), Base: tier, Tiers: []zenTier{tier}}
fam.ids = append(fam.ids, claudeSKU)
return fam
}
// chat asks fam for one chat completion of sku through the pipe and returns the
// controller, the refusals the pipe handed back, and what the caller was sent.
func chat(t *testing.T, c *ApiController, fam *modelFamily, sku, body string, stream bool, user *iam.User) ([]attempt, string) {
t.Helper()
c.Fiber().Request().SetBody([]byte(body))
out := c.pipeToFamily(fam, "chat/completions", "openai", sku, []byte(body), stream, 0, "acme", user, false, nil, time.Now())
return out, drain(t, c)
}
func hello(sku string) string {
return `{"model":"` + sku + `","messages":[{"role":"user","content":"2+2?"}]}`
}
// A Claude model is asked of its maker before the reseller that lists it, on each
// key in the order the pool names them: a key the vendor refuses hands the same
// request to the next, and the reseller is never asked once one answers. The
// caller is answered as the model it named.
func TestAClaudeModelIsAskedOfItsMakerOnEveryKeyInOrder(t *testing.T) {
or := &accounts{}
fam := resale(t, or)
m := makerAt(t, &maker{refuse: map[string]turnedAway{
"a1": {status: 401, kind: "authentication_error", message: "invalid x-api-key"},
"a2": {status: 429, kind: "rate_limit_error", message: "slow down", after: "30"},
}}, "a1", "a2", "a3")
c := visit(http.MethodPost, "/v1/chat/completions")
out, answer := chat(t, c, fam, claudeSKU, hello(claudeSKU), false, nil)
if out != nil || !strings.Contains(answer, `"content":"ok"`) {
t.Fatalf("attempts=%+v answer=%s — the third key should have served", out, answer)
}
if got := m.calls(); !slices.Equal(got, []string{"a1", "a2", "a3"}) {
t.Fatalf("the maker's keys were asked %v, want [a1 a2 a3]", got)
}
if got := or.calls(); len(got) != 0 {
t.Fatalf("the reseller was asked %v after the maker answered", got)
}
if got := header(c, servedHeader); got != claudeSKU {
t.Errorf("%s = %q, want the model asked for", servedHeader, got)
}
var whole struct {
Model string `json:"model"`
Usage struct {
Prompt int `json:"prompt_tokens"`
Details struct {
Cached int `json:"cached_tokens"`
Written int `json:"cache_write_tokens"`
} `json:"prompt_tokens_details"`
Completion int `json:"completion_tokens"`
} `json:"usage"`
}
if err := json.Unmarshal([]byte(strings.TrimSpace(answer)), &whole); err != nil {
t.Fatalf("the answer is not one chat completion: %v: %s", err, answer)
}
if whole.Model != claudeSKU || whole.Usage.Prompt != 6000 || whole.Usage.Details.Cached != 3000 || whole.Usage.Details.Written != 2000 || whole.Usage.Completion != 500 {
t.Errorf("answer = %+v, want the model asked for and a prompt of 6000 with 3000 read from the cache and 2000 written to it", whole)
}
// What each key is asked for is the vendor's own id, with the vendor's
// headers, and never another vendor's field.
for i := range m.calls() {
b := m.sentBody(t, i)
if string(b["model"]) != `"`+claudeID+`"` || b["provider"] != nil {
t.Errorf("call %d was sent model %s provider %s", i, b["model"], b["provider"])
}
if v := m.heads[i].Get("anthropic-version"); v != anthropicVersion {
t.Errorf("call %d carried anthropic-version %q", i, v)
}
if m.heads[i].Get("Authorization") != "" {
t.Errorf("call %d carried an Authorization header", i)
}
}
// The refused key sits out; the limited one until the vendor's word, no longer.
accts := anthropicNative.accounts()
if !cooling(accts[0].Key, false) || !cooling(accts[1].Key, false) || cooling(accts[2].Key, false) {
t.Errorf("after 401, 429 and an answer the keys sit out %v %v %v, want true true false",
cooling(accts[0].Key, false), cooling(accts[1].Key, false), cooling(accts[2].Key, false))
}
}
// A key the vendor refused as a credential, or for want of credit, is not asked
// again until nativeDead has passed — and is asked again then. A rate-limited key
// is back when the vendor said it would be, and never later than nativeBusy.
func TestARefusedKeySitsOutItsTimeAndIsAskedAgain(t *testing.T) {
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
keyNow = func() time.Time { return now }
t.Cleanup(func() { keyNow = time.Now })
or := &accounts{}
fam := resale(t, or)
m := makerAt(t, &maker{refuse: map[string]turnedAway{
"a1": {status: 401, kind: "authentication_error", message: "invalid x-api-key"},
"a2": {status: 400, kind: "invalid_request_error", message: "Your credit balance is too low to access the API."},
"a3": {status: 429, kind: "rate_limit_error", message: "slow down", after: "600"},
}}, "a1", "a2", "a3")
ask := func() []string {
t.Helper()
m.mu.Lock()
m.asked = nil
m.mu.Unlock()
or.reset()
c := visit(http.MethodPost, "/v1/chat/completions")
if out, answer := chat(t, c, fam, claudeSKU, hello(claudeSKU), false, nil); out != nil || !strings.Contains(answer, `"ok"`) {
t.Fatalf("attempts=%+v answer=%s — the reseller should have served", out, answer)
}
if got := or.calls(); !slices.Equal(got, []string{"k1"}) {
t.Fatalf("the reseller was asked %v, want once", got)
}
return m.calls()
}
if got := ask(); !slices.Equal(got, []string{"a1", "a2", "a3"}) {
t.Fatalf("asked %v, want every key once", got)
}
if got := ask(); len(got) != 0 {
t.Fatalf("asked %v while every key sits out", got)
}
// A minute on, the limited key is back whatever the vendor said; the refused
// two are not.
now = now.Add(nativeBusy + time.Second)
if got := ask(); !slices.Equal(got, []string{"a3"}) {
t.Fatalf("asked %v a minute on, want the rate-limited key alone", got)
}
now = now.Add(nativeDead)
if got := ask(); !slices.Equal(got, []string{"a1", "a2", "a3"}) {
t.Fatalf("asked %v once the wait was over, want every key probed again", got)
}
// A key refused as a credential is back as soon as its list reads again.
m.mu.Lock()
m.refuse = nil
m.mu.Unlock()
anthropicNative.refresh()
if got := askOnce(t, m, fam); !slices.Equal(got, []string{"a1"}) {
t.Fatalf("asked %v after the keys' lists read again, want the first key to serve", got)
}
}
// A key's list reading again says the vendor knows the key, and nothing of what the
// key may do or its account can pay: a key refused permission, and one resting for
// want of credit, rest on through it, and the next key serves. Asking them again
// every time the lists were read would cost a refused call each five minutes.
func TestAKeyOutOfCreditOrRefusedPermissionRestsThoughItsListReads(t *testing.T) {
or := &accounts{}
fam := resale(t, or)
m := makerAt(t, &maker{refuse: map[string]turnedAway{
"a1": {status: 400, kind: "invalid_request_error", message: "Your credit balance is too low to access the API."},
"a2": {status: 403, kind: "permission_error", message: "not allowed"},
}}, "a1", "a2", "a3")
if got := askOnce(t, m, fam); !slices.Equal(got, []string{"a1", "a2", "a3"}) {
t.Fatalf("asked %v, want the two keys that cannot serve and then the next", got)
}
anthropicNative.refresh()
if got := askOnce(t, m, fam); !slices.Equal(got, []string{"a3"}) {
t.Fatalf("asked %v after the lists read again, want the serving key alone", got)
}
if got := or.calls(); len(got) != 0 {
t.Fatalf("the reseller was asked %v while a key of the maker's served", got)
}
}
// Only the vendor's own sentence benches a key for credit. A validation error that
// quotes the caller's request back may hold the same words, and is the request's
// own: one key hears it, none rests, and the reseller answers. Read as the
// account's, one crafted request would bench every key of the pool for all callers.
func TestAVendorsErrorQuotingTheCallerBenchesNoKey(t *testing.T) {
for _, tc := range []struct {
status int
kind, message string
want bool
}{
{402, "billing_error", "Payment required.", true},
{400, "billing_error", "anything", true},
{400, "invalid_request_error", "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing.", true},
{400, "invalid_request_error", " your credit balance is too low", true},
{400, "invalid_request_error", "You have reached your specified API usage limits. You will regain access on 2026-11-01 at 00:00 UTC.", true},
{400, "invalid_request_error", "messages: You have reached your specified API usage limits", false},
{400, "invalid_request_error", "messages.0.content.0: Input tag 'Your credit balance is too low' found using 'type' does not match", false},
{400, "invalid_request_error", "tools.0.custom.name: credit balance", false},
{429, "rate_limit_error", "Your credit balance is too low", false},
} {
if got := unfunded(tc.status, tc.kind, tc.message); got != tc.want {
t.Errorf("unfunded(%d, %q, %q) = %v, want %v", tc.status, tc.kind, tc.message, got, tc.want)
}
}
or := &accounts{}
fam := resale(t, or)
m := makerAt(t, &maker{refuse: map[string]turnedAway{
"a1": {status: 400, kind: "invalid_request_error", message: "messages.0.content.0: Input tag 'your credit balance is too low' found using 'type'"},
"a2": {status: 400, kind: "invalid_request_error", message: "messages.0.content.0: Input tag 'your credit balance is too low' found using 'type'"},
}}, "a1", "a2")
c := visit(http.MethodPost, "/v1/chat/completions")
if out, answer := chat(t, c, fam, claudeSKU, hello(claudeSKU), false, nil); out != nil || !strings.Contains(answer, `"ok"`) {
t.Fatalf("attempts=%+v answer=%s — the reseller should have answered", out, answer)
}
if got := m.calls(); !slices.Equal(got, []string{"a1"}) {
t.Fatalf("the maker's keys were asked %v, want one: the refusal is the request's own", got)
}
for _, a := range anthropicNative.accounts() {
if cooling(a.Key, false) {
t.Fatalf("key %s rests after an error that quoted the caller", a.Name)
}
}
}
// askOnce asks once and returns the maker's keys asked, for an answer the maker gives.
func askOnce(t *testing.T, m *maker, fam *modelFamily) []string {
t.Helper()
m.mu.Lock()
m.asked = nil
m.mu.Unlock()
c := visit(http.MethodPost, "/v1/chat/completions")
if out, answer := chat(t, c, fam, claudeSKU, hello(claudeSKU), false, nil); out != nil || !strings.Contains(answer, `"ok"`) {
t.Fatalf("attempts=%+v answer=%s", out, answer)
}
return m.calls()
}
// A key the vendor refuses when its list is read is never asked for an answer,
// and one whose list was never read is not asked either.
func TestAKeyIsAskedOnlyForWhatItLists(t *testing.T) {
or := &accounts{}
fam := resale(t, or)
m := makerAt(t, &maker{lists: map[string]int{"a1": 401, "a2": 500}}, "a1", "a2", "a3")
if got := askOnce(t, m, fam); !slices.Equal(got, []string{"a3"}) {
t.Fatalf("asked %v, want only the key whose list was read", got)
}
n := anthropicNative
accts := n.accounts()
for sku, want := range map[string]string{
claudeSKU: claudeID,
"anthropic/claude-haiku-4.5": "claude-haiku-4-5-20251001", // the newest release of a dated id
"anthropic/claude-haiku-5.5:batch": "", // the reseller's own variant
"anthropic/claude-opus-9": "", // not listed
"openai/gpt-6": "", // another vendor's
} {
got, ok := n.find(accts[2], sku)
if got.ID != want || ok != (want != "") {
t.Errorf("find(%q) = %q %v, want %q", sku, got.ID, ok, want)
}
}
if !n.serves(claudeSKU) || n.serves("anthropic/claude-opus-9") {
t.Error("serves disagrees with what the keys list")
}
if _, ok := n.find(accts[0], claudeSKU); ok {
t.Error("a key the vendor refused lists a model")
}
if c := mustFind(t, accts[2], claudeSKU); !c.Adaptive || c.Budget || !c.Vision || !c.Schema || c.Out != 128000 || !c.Efforts["max"] || c.Efforts["xhigh"] {
t.Errorf("the adaptive model read as %+v", c)
}
if c := mustFind(t, accts[2], "anthropic/claude-haiku-4.5"); c.Adaptive || !c.Budget || c.Schema || len(c.Efforts) != 0 {
t.Errorf("the budget model read as %+v", c)
}
}
func mustFind(t *testing.T, a object.Account, sku string) claude {
t.Helper()
c, ok := anthropicNative.find(a, sku)
if !ok {
t.Fatalf("%s is not listed", sku)
}
return c
}
// When no key of the maker's answers, the reseller is asked for the same model,
// under its own id for it, and its answer is the caller's. A refusal that is the
// request's own — one every key would give — goes to the reseller after one key.
func TestWhenNoKeyAnswersTheResellerIsAskedForTheSameModel(t *testing.T) {
or := &accounts{}
fam := resale(t, or)
m := makerAt(t, &maker{refuse: map[string]turnedAway{
"a1": {status: 529, kind: "overloaded_error", message: "Overloaded"},
"a2": {status: 500, kind: "api_error", message: "Internal server error"},
}}, "a1", "a2")
c := visit(http.MethodPost, "/v1/chat/completions")
out, answer := chat(t, c, fam, claudeSKU, hello(claudeSKU), false, nil)
if out != nil || !strings.Contains(answer, `"ok"`) {
t.Fatalf("attempts=%+v answer=%s — the reseller should have served", out, answer)
}
if got := m.calls(); !slices.Equal(got, []string{"a1", "a2"}) {
t.Fatalf("the maker's keys were asked %v, want both", got)
}
if got := or.calls(); !slices.Equal(got, []string{"k1"}) {
t.Fatalf("the reseller was asked %v, want once", got)
}
if b := decoded(t, []byte(or.bodies[0])); string(b["model"]) != `"`+claudeSKU+`"` {
t.Fatalf("the reseller was asked for %s, want the model named", b["model"])
}
if got := header(c, servedHeader); got != claudeSKU {
t.Errorf("%s = %q, want the model asked for", servedHeader, got)
}
// Neither an overloaded vendor nor a failing one benches a key.
for _, a := range anthropicNative.accounts() {
if cooling(a.Key, false) {
t.Errorf("key %s sits out after a 5xx", a.Name)
}
}
// The request's own refusal: one key hears it, then the reseller.
m.mu.Lock()
m.asked, m.refuse = nil, map[string]turnedAway{"a1": {status: 400, kind: "invalid_request_error", message: "max_tokens: too large"}}
m.mu.Unlock()
or.reset()
if out, answer := chat(t, visit(http.MethodPost, "/v1/chat/completions"), fam, claudeSKU, hello(claudeSKU), false, nil); out != nil || !strings.Contains(answer, `"ok"`) {
t.Fatalf("attempts=%+v answer=%s", out, answer)
}
if got := m.calls(); !slices.Equal(got, []string{"a1"}) {
t.Fatalf("the maker's keys were asked %v for a refusal every key would give, want one", got)
}
if got := or.calls(); !slices.Equal(got, []string{"k1"}) {
t.Fatalf("the reseller was asked %v, want once", got)
}
// An answer that ended having said nothing is the request's own too: every
// key would buy the same silence, so one is asked and then the reseller.
m.mu.Lock()
m.asked, m.refuse = nil, nil
m.events = []string{okEvents[0], okEvents[5], okEvents[6]}
m.mu.Unlock()
or.reset()
if out, answer := chat(t, visit(http.MethodPost, "/v1/chat/completions"), fam, claudeSKU, hello(claudeSKU), false, nil); out != nil || !strings.Contains(answer, `"ok"`) {
t.Fatalf("attempts=%+v answer=%s", out, answer)
}
if got, sold := m.calls(), or.calls(); !slices.Equal(got, []string{"a1"}) || !slices.Equal(sold, []string{"k1"}) {
t.Fatalf("an empty answer: the maker was asked %v and the reseller %v, want one key and then the reseller", got, sold)
}
// An answer that is not the stream asked for is not relayed as one.
m.mu.Lock()
m.asked, m.refuse, m.events, m.whole = nil, nil, nil, true
m.mu.Unlock()
or.reset()
if out, answer := chat(t, visit(http.MethodPost, "/v1/chat/completions"), fam, claudeSKU, hello(claudeSKU), false, nil); out != nil || !strings.Contains(answer, `"ok"`) {
t.Fatalf("attempts=%+v answer=%s", out, answer)
}
if got, sold := m.calls(), or.calls(); !slices.Equal(got, []string{"a1"}) || !slices.Equal(sold, []string{"k1"}) {
t.Fatalf("a whole answer to a stream: the maker was asked %v and the reseller %v, want one key and then the reseller", got, sold)
}
}
// A Claude model nobody could serve is refused as itself: the maker and the
// reseller are each asked for that model and no other, nothing is written under a
// 200, and the caller is told 503 providers_exhausted. No free route, Enso SKU or
// compute of ours answers in its name.
func TestAClaudeModelNobodyServesIsRefusedNeverStoodInFor(t *testing.T) {
or := &accounts{status: map[string]int{"k1": http.StatusBadGateway}}
fam := resale(t, or)
m := makerAt(t, &maker{refuse: map[string]turnedAway{
"a1": {status: 529, kind: "overloaded_error", message: "Overloaded"},
}}, "a1")
for _, stream := range []bool{false, true} {
m.mu.Lock()
m.asked, m.bodies = nil, nil
m.mu.Unlock()
or.reset()
cooled.forget()
c := visit(http.MethodPost, "/v1/chat/completions")
out, answer := chat(t, c, fam, claudeSKU, hello(claudeSKU), stream, nil)
if len(out) != 1 || out[0].fault != faultProvider || answer != "" || answered(c) != http.StatusOK && answered(c) != 0 {
t.Fatalf("stream=%v: attempts=%+v answer=%q status=%d — want one refusal handed back and nothing written", stream, out, answer, answered(c))
}
var ae *apiError
if err := exhausted(claudeSKU, out); !errors.As(err, &ae) || ae.status != http.StatusServiceUnavailable || ae.code != codeExhausted {
t.Fatalf("stream=%v: the caller is told %+v, want 503 %s", stream, ae, codeExhausted)
}
if got := m.calls(); !slices.Equal(got, []string{"a1"}) {
t.Fatalf("stream=%v: the maker was asked %v", stream, got)
}
for _, b := range append(slices.Clone(m.bodies), or.bodies...) {
var in struct {
Model string `json:"model"`
}
_ = json.Unmarshal([]byte(b), &in)
if in.Model != claudeSKU && in.Model != claudeID {
t.Fatalf("stream=%v: a vendor was asked for %q in place of %s", stream, in.Model, claudeSKU)
}
}
// Nothing follows the family on the route either: the model has no
// alternate, and the pool is not one.
if queue := candidates("acme", fam.passthroughRoute(claudeSKU), out, false); len(queue) != 0 {
t.Fatalf("stream=%v: the route offers %+v after the refusal", stream, queue)
}
}
}
// The free lane never reaches a maker. A request not seated on the paid lane, a
// platform whose paid switch is off, a route the family charges nothing for and
// the free pool's own door are each served, or refused, without the maker being
// asked — though every one of its keys is ready and lists the model.
func TestTheFreeLaneNeverReachesAMaker(t *testing.T) {
const gratis = "anthropic/claude-free-1"
or := &accounts{}
fam := resale(t, or)
fam.byID[gratis] = zenModel{ID: gratis}
fam.ids = append(fam.ids, gratis)
m := makerAt(t, &maker{}, "a1", "a2")
if !anthropicNative.serves(claudeSKU) || !anthropicNative.serves(gratis) {
t.Fatal("the maker lists neither model: the case proves nothing")
}
unasked := func(what string) {
t.Helper()
if got := m.calls(); len(got) != 0 {
t.Fatalf("%s: the maker was asked %v", what, got)
}
}
// No seat on the paid lane: a priced Claude model is refused, not sent.
c := unseated(visit(http.MethodPost, "/v1/chat/completions"))
if out, _ := chat(t, c, fam, claudeSKU, hello(claudeSKU), false, nil); out != nil || answered(c) < 400 {
t.Fatalf("an unseated caller got attempts=%+v status=%d, want a refusal", out, answered(c))
}
unasked("an unseated caller")
// The platform's paid switch off.
FreeOnly = func() bool { return true }
c = visit(http.MethodPost, "/v1/chat/completions")
if out, _ := chat(t, c, fam, claudeSKU, hello(claudeSKU), false, nil); out != nil || answered(c) < 400 {
t.Fatalf("with the paid lane off: attempts=%+v status=%d, want a refusal", out, answered(c))
}
FreeOnly = func() bool { return false }
unasked("the paid lane off")
// A route the family charges nothing for, by a seated caller.
or.reset()
c = visit(http.MethodPost, "/v1/chat/completions")
if out, answer := chat(t, c, fam, gratis, hello(gratis), false, nil); out != nil || !strings.Contains(answer, `"ok"`) {
t.Fatalf("a free route: attempts=%+v answer=%s", out, answer)
}
unasked("a free route")
if got := or.calls(); !slices.Equal(got, []string{"k1"}) {
t.Fatalf("a free route asked the family's vendor %v, want once", got)
}
// The pool's own door, seated or not.
for _, seat := range []bool{true, false} {
c = visit(http.MethodPost, "/v1/chat/completions")
if !seat {
c = unseated(c)
}
if out, answer := chat(t, c, fam, freeID, hello(freeID), false, nil); out != nil || !strings.Contains(answer, `"ok"`) {
t.Fatalf("the pool, seated=%v: attempts=%+v answer=%s", seat, out, answer)
}
unasked("the free pool")
}
}
// A maker's answer bills the catalog's price and nothing else: the tokens it read
// fresh, read from its cache and wrote to it, each at the SKU's own rate. What it
// cost is those tokens at the vendor's price, the family's fee taken off, so the
// margin of a call the maker served is on its row.
func TestAMakersAnswerBillsTheCatalogPriceAndStatesWhatItCost(t *testing.T) {
or := &accounts{}
fam := resale(t, or)
makerAt(t, &maker{}, "a1")
got := ledger(t)
alice := &iam.User{Owner: "acme", Name: "alice"}
// 1,000 fresh, 3,000 cached, 2,000 written, 500 out:
// vendor: (1000×0.10 + 3000×0.01 + 2000×0.125 + 500×0.50) / 1e6 = $0.00063
// caller: that × 1.055, the platform's fee = $0.00066465
const cost, price = "0.00063", "0.00066465"
for _, stream := range []bool{true, false} {
c := visit(http.MethodPost, "/v1/chat/completions")
if out, answer := chat(t, c, fam, claudeSKU, hello(claudeSKU), stream, alice); out != nil || !strings.Contains(answer, "ok") {
t.Fatalf("stream=%v: attempts=%+v answer=%s", stream, out, answer)
}
}
// The vendor's own dialect, whole and streamed, is priced the same.
for _, stream := range []bool{false, true} {
body := `{"model":"` + claudeSKU + `","max_tokens":64,"stream":` + strconv.FormatBool(stream) + `,"messages":[{"role":"user","content":"2+2?"}]}`
c := visit(http.MethodPost, "/v1/messages")
c.Fiber().Request().SetBody([]byte(body))
if out := c.pipeToFamily(fam, "messages", "anthropic", claudeSKU, []byte(body), stream, 64, "acme", alice, false, nil, time.Now()); out != nil {
t.Fatalf("messages stream=%v refused: %+v", stream, out)
}
if answer := drain(t, c); !strings.Contains(answer, "ok") || !strings.Contains(answer, claudeSKU) || strings.Contains(answer, "msg_vendor") {
t.Fatalf("messages stream=%v answered %s, want the vendor's answer under our id and the model asked for", stream, answer)
}
}
ev := got()
if len(ev) != 4 {
t.Fatalf("%d debits, want one an answer: %+v", len(ev), ev)
}
for i, e := range ev {
if e.USD != price || e.CostUSD != cost || e.Model != claudeSKU || e.Provider != "openrouter" {
t.Errorf("debit %d = $%s at a cost of $%s for %s via %s, want $%s at $%s for %s",
i, e.USD, e.CostUSD, e.Model, e.Provider, price, cost, claudeSKU)
}
}
if got := or.calls(); len(got) != 0 {
t.Errorf("the reseller was asked %v", got)
}
}
// An hour-long cache write costs the vendor's hour rate, which the catalog does
// not list: the call bills what it cost times the platform's fee, never the
// five-minute rate.
func TestAnHourLongCacheWriteNeverBillsBelowItsCost(t *testing.T) {
or := &accounts{}
fam := resale(t, or)
hour := slices.Clone(okEvents)
hour[0] = `{"type":"message_start","message":{"id":"msg_vendor","role":"assistant","model":"claude-haiku-5-5","usage":{"input_tokens":1000,"cache_creation_input_tokens":2000,"cache_read_input_tokens":0,"output_tokens":1,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2000}}}}`
makerAt(t, &maker{events: hour}, "a1")
got := ledger(t)
alice := &iam.User{Owner: "acme", Name: "alice"}
body := `{"model":"` + claudeSKU + `","max_tokens":64,"stream":true,"messages":[{"role":"user","content":"2+2?"}]}`
c := visit(http.MethodPost, "/v1/messages")
c.Fiber().Request().SetBody([]byte(body))
if out := c.pipeToFamily(fam, "messages", "anthropic", claudeSKU, []byte(body), true, 64, "acme", alice, false, nil, time.Now()); out != nil {
t.Fatalf("refused: %+v", out)
}
_ = drain(t, c)
ev := got()
if len(ev) != 1 {
t.Fatalf("%d debits: %+v", len(ev), ev)
}
billed, _ := usdNanos(ev[0].USD)
spent, _ := usdNanos(ev[0].CostUSD)
// 2,000 tokens written for an hour cost 2 × the input rate: $0.0004, and the
// call $0.00075 with its fresh prompt and its answer.
if spent < 750_000 || billed < spent || billed*1000 < spent*1055 {
t.Fatalf("billed $%s at a cost of $%s, want the hour's cost (at least $0.00075) and the fee on top", ev[0].USD, ev[0].CostUSD)
}
}
// An answer a maker served says where it came from: the maker's host and the place
// in its pool of the key that answered, and the same account on its span. Any other
// answer of the family's says the family's own address.
func TestAnAnswerSaysWhichOfTheMakersKeysServedIt(t *testing.T) {
t.Setenv("ANTHROPIC_URL", "https://api.maker.example")
prov := &object.Provider{Owner: "admin", Name: "openrouter", Type: "OpenRouter", ProviderUrl: "https://reseller.example/api"}
h := http.Header{}
h.Set(armHeader, claudeID)
h.Set(providerHeader, "anthropic")
h.Set(keyHeader, "ANTHROPIC_API_KEY_2")
made := func(h http.Header) serving {
sv := servingOf(h)
sv.maker = anthropicNative
return sv
}
if origin, via := reached(prov, nil, made(h)); origin != "api.maker.example#2" || via != "anthropic#2" {
t.Errorf("a maker's answer came from %q via %q, want api.maker.example#2 via anthropic#2", origin, via)
}
h.Set(keyHeader, object.AnthropicKeyList+"#3")
if origin, via := reached(prov, nil, made(h)); origin != "api.maker.example#6" || via != "anthropic#6" {
t.Errorf("a key of the list came from %q via %q, want place 6: after the three names", origin, via)
}
if origin, via := reached(prov, nil, servingOf(http.Header{})); origin != "reseller.example" || via != "" {
t.Errorf("the reseller's answer came from %q via %q, want its own host and no account", origin, via)
}
// That a maker served an answer is the pipe's own knowledge. A reseller's
// address answering with a maker's headers stays the reseller, and a cost in
// its headers is never the floor of a bill.
h.Set(cogsHeader, "9")
if origin, via := reached(prov, nil, servingOf(h)); via != "" || origin != "reseller.example" {
t.Errorf("a maker's headers alone read as %q via %q, want the reseller's own address", origin, via)
}
fam := resale(t, &accounts{})
w := whence{ledger: "acme", ctx: context.Background(), asked: fam}
spent := tokens{fresh: 1000, completion: 500, reported: true}
sku, _ := fam.lookup(claudeSKU)
if nano := recordFamilyUsage(w, fam, claudeSKU, "", prov, &mark{}, nil, true, false, "r", spent, servingOf(h), time.Now(), nil, "success", ""); nano != sku.retailNano(spent) {
t.Errorf("a $9 cost in a reseller's headers billed %d nano, want the catalog price %d", nano, sku.retailNano(spent))
}
if nano := recordFamilyUsage(w, fam, claudeSKU, "", prov, &mark{}, nil, true, false, "r", spent, made(h), time.Now(), nil, "success", ""); nano < 9_000_000_000 {
t.Errorf("a maker's stated cost of $9 billed %d nano, want no less than the cost and the fee", nano)
}
// Each key that refused first is an arm of the chain, read back as the family
// service's are.
missed := []miss{
{upstream: claudeID, provider: "anthropic", reason: "key 1: 401 authentication_error"},
{upstream: claudeID, provider: "anthropic", reason: "key 2: 529 overloaded_error"},
}
if got := arms(chain(missed)); !slices.Equal(got, missed) {
t.Errorf("the chain read back as %+v, want %+v", got, missed)
}
rows := failoverRows(whence{ledger: "acme", ctx: context.Background()}, openrouterFam, claudeSKU, chain(missed), &iam.User{Owner: "acme", Name: "alice"}, true, false, "req-1")
if len(rows) != 2 || rows[0].ErrorMsg != claudeID+": key 1: 401 authentication_error" || rows[0].Vendor != "anthropic" || rows[0].Status != "failover" {
t.Errorf("failover rows = %+v", rows)
}
}
// The pool is the reserved names in order, then each line of the list; a key's
// place is its name's whatever the names before it hold, and one key held twice
// is one account.
func TestTheMakersPoolIsItsNamesThenItsList(t *testing.T) {
t.Setenv("ANTHROPIC_API_KEY", "")
t.Setenv("ANTHROPIC_API_KEY_2", "second")
t.Setenv("ANTHROPIC_API_KEY_3", "third")
t.Setenv(object.AnthropicKeyList, "fourth\n\n third \nfifth\n")
var got []string
for _, a := range anthropicNative.accounts() {
got = append(got, fmt.Sprintf("%d %s %s", a.Place, a.Name, a.Key))
}
want := []string{"2 ANTHROPIC_API_KEY_2 second", "3 ANTHROPIC_API_KEY_3 third", "4 ANTHROPIC_API_KEYS#1 fourth", "6 ANTHROPIC_API_KEYS#3 fifth"}
if !slices.Equal(got, want) {
t.Fatalf("accounts = %q, want %q", got, want)
}
for _, a := range anthropicNative.accounts() {
if p := anthropicNative.place(a.Name); p != a.Place {
t.Errorf("place(%s) = %d, want %d", a.Name, p, a.Place)
}
}
// With no address the maker is never asked, whatever keys are held.
t.Setenv("ANTHROPIC_URL", "")
if resp, missed := anthropicNative.send(context.Background(), nativeCall{path: "chat/completions", sku: claudeSKU, body: []byte(hello(claudeSKU)), start: time.Now()}); resp != nil || missed != nil {
t.Fatal("a maker with no address was asked")
}
}
// talk is a long answer its maker is still writing: the stream opens with the
// prompt's count and an output of one token, as the vendor's does, and never
// reaches the count that closes it.
func talk() []string {
events := []string{
`{"type":"message_start","message":{"id":"msg_vendor","role":"assistant","model":"claude-haiku-5-5","usage":{"input_tokens":1000,"output_tokens":1}}}`,
`{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
}
for i := 0; i < 400; i++ {
events = append(events, `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"forty characters of an answer, give or t"}}`)
}
return events
}
// taker is a caller that takes so many writes of an answer and goes away, keeping
// what it took.
type taker struct {
left int
took bytes.Buffer
}
func (k *taker) Write(p []byte) (int, error) {
if k.left <= 0 {
return 0, errors.New("client gone")
}
k.left--
return k.took.Write(p)
}
// A stream its caller hangs up on is billed for what was written, in either
// dialect, and says what it cost as far as it got. The vendor's stream opens with
// an output count of one token; billed as that, a caller who streamed a long
// answer and dropped the connection before its end paid for the prompt and one
// token while the vendor billed every token it wrote.
func TestAStreamItsCallerHangsUpOnIsBilledForWhatWasWritten(t *testing.T) {
or := &accounts{}
fam := resale(t, or)
makerAt(t, &maker{events: talk()}, "a1")
got := ledger(t)
alice := &iam.User{Owner: "acme", Name: "alice"}
// The prompt and one token of answer: what the stream's opening count says.
sku, _ := fam.lookup(claudeSKU)
least := sku.retailNano(tokens{fresh: 1000, completion: 1})
const piece = "forty characters of an answer, give or t"
for i, dialect := range []string{"anthropic", "openai"} {
path, body := "messages", `{"model":"`+claudeSKU+`","max_tokens":8000,"stream":true,"messages":[{"role":"user","content":"Go on."}]}`
if dialect == "openai" {
path = "chat/completions"
}
c := visit(http.MethodPost, "/v1/"+path)
c.Fiber().Request().SetBody([]byte(body))
if out := c.pipeToFamily(fam, path, dialect, claudeSKU, []byte(body), true, 8000, "acme", alice, false, nil, time.Now()); out != nil {
t.Fatalf("%s: refused: %+v", dialect, out)
}
// The caller takes some of the answer and goes away: the write to it fails,
// and the server closes the stream it was writing, as it does for a
// connection that is gone. The stream's writer settles on its own goroutine.
caller := &taker{left: 120}
_ = c.Fiber().Response().BodyWriteTo(bufio.NewWriterSize(caller, 16))
_ = c.Fiber().Response().CloseBodyStream()
for deadline := time.Now().Add(5 * time.Second); len(got()) != i+1 && time.Now().Before(deadline); {
time.Sleep(5 * time.Millisecond)
}
ev := got()
if len(ev) != i+1 {
t.Fatalf("%s: %d debits, want one a stream: %+v", dialect, len(ev), ev)
}
billed, _ := usdNanos(ev[i].USD)
spent, stated := usdNanos(ev[i].CostUSD)
// What the caller took is the least that was written: it is on the bill, at
// four characters a token.
took := strings.Count(caller.took.String(), piece) * len(piece)
if floor := sku.retailNano(tokens{fresh: 1000, completion: took / 4}); took == 0 || billed < floor || billed <= least {
t.Errorf("%s: a caller who took %d characters and hung up was billed $%s; the prompt and one token is $%s, and the text it took $%s",
dialect, took, ev[i].USD, nanoUSD(&least), nanoUSD(&floor))
}
if !stated || spent <= 0 || billed*1000 < spent*1055 {
t.Errorf("%s: billed $%s at a stated cost of $%q, want the cost of what was written and the fee on top", dialect, ev[i].USD, ev[i].CostUSD)
}
}
}