mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-21 02:36:49 +00:00
1104 lines
45 KiB
Go
1104 lines
45 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
"github.com/hanzoai/ai/object"
|
|
)
|
|
|
|
// A plan covers Hanzo SKUs: Enso's and Zen's, and the pool's own name, which Enso
|
|
// answers. Every other model is a third-party one.
|
|
func TestFamilyOfNamesHanzoSKUsOnly(t *testing.T) {
|
|
for model, want := range map[string]string{
|
|
"enso": "enso", "enso-pro": "enso", "Enso-Flash": "enso", "hanzo/enso": "enso", "free": "enso",
|
|
"zen5": "zen", "zen6-flash": "zen",
|
|
"anthropic/claude-opus-5.5": "", "openai/gpt-6-sol": "", "vendor/small:free": "", "": "",
|
|
} {
|
|
if got := FamilyOf(model); got != want {
|
|
t.Errorf("FamilyOf(%q) = %q, want %q", model, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A family's stated cost reads in nano-dollars, rounded up; a missing, malformed or
|
|
// negative figure is no figure at all.
|
|
func TestAStatedCostReadsInNanos(t *testing.T) {
|
|
for in, want := range map[string]int64{"0.0025": 2_500_000, "0.0000000001": 1, "0": 0, "12.5": 12_500_000_000} {
|
|
if got, ok := usdNanos(in); !ok || got != want {
|
|
t.Errorf("usdNanos(%q) = %d, %t; want %d", in, got, ok, want)
|
|
}
|
|
}
|
|
for _, bad := range []string{"", "lots", "-1", "1,2"} {
|
|
if _, ok := usdNanos(bad); ok {
|
|
t.Errorf("%q read as a cost", bad)
|
|
}
|
|
}
|
|
if got := spendUSD(50_000_000); got != "0.050000000" {
|
|
t.Errorf("spendUSD = %q", got)
|
|
}
|
|
}
|
|
|
|
// committed answers the way a family answers a request its plan opened a paid rung
|
|
// for: 200 at once with the most the request can cost (bound, none when empty) and
|
|
// its cost declared as a trailer, then body, then the cost (none when cost is empty).
|
|
func committed(w http.ResponseWriter, sse bool, bound, body, cost string) {
|
|
if sse {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
} else {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
}
|
|
if bound != "" {
|
|
w.Header().Set(costBoundHeader, bound)
|
|
}
|
|
w.Header().Set("Trailer", costHeader)
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = io.WriteString(w, body)
|
|
if cost != "" {
|
|
w.Header().Set(costHeader, cost)
|
|
}
|
|
}
|
|
|
|
// cutAfter writes a committed answer's head and body chunks by hand and drops the
|
|
// connection before the body ends: no trailer ever arrives.
|
|
func cutAfter(w http.ResponseWriter, sse bool, bound string, chunks ...string) {
|
|
conn, rw, err := w.(http.Hijacker).Hijack()
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer conn.Close()
|
|
ct := "application/json"
|
|
if sse {
|
|
ct = "text/event-stream"
|
|
}
|
|
head := "HTTP/1.1 200 OK\r\nContent-Type: " + ct + "\r\nTrailer: " + costHeader + "\r\nTransfer-Encoding: chunked\r\n"
|
|
if bound != "" {
|
|
head += costBoundHeader + ": " + bound + "\r\n"
|
|
}
|
|
_, _ = rw.WriteString(head + "\r\n")
|
|
for _, c := range chunks {
|
|
_, _ = fmt.Fprintf(rw, "%x\r\n%s\r\n", len(c), c)
|
|
}
|
|
_ = rw.Flush()
|
|
}
|
|
|
|
// paidSSE is a paid rung's whole answer on a committed stream: the paid marker, the
|
|
// words, the usage and the end.
|
|
const paidSSE = ": keep-alive\n\n: paid\n\n" +
|
|
`data: {"id":"x","model":"enso-pro","choices":[{"delta":{"content":"ok"}}]}` + "\n\n" +
|
|
`data: {"id":"x","choices":[],"usage":{"prompt_tokens":1000,"completion_tokens":2000}}` + "\n\ndata: [DONE]\n\n"
|
|
|
|
// freeSSE is a free rung's whole answer on a committed stream: no paid marker.
|
|
const freeSSE = ": keep-alive\n\n" +
|
|
`data: {"id":"x","model":"enso-pro","choices":[{"delta":{"content":"ok"}}]}` + "\n\n" +
|
|
`data: {"id":"x","choices":[],"usage":{"prompt_tokens":1000,"completion_tokens":2000}}` + "\n\ndata: [DONE]\n\n"
|
|
|
|
// planEnso stands up an Enso service that records the spend each request carried and
|
|
// answers it committed, from a paid rung when paid is set, at cost.
|
|
func planEnso(t *testing.T, paid bool, cost string) (*httptest.Server, func() []string) {
|
|
t.Helper()
|
|
var mu sync.Mutex
|
|
var spends []string
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
mu.Lock()
|
|
spends = append(spends, r.Header.Get(spendHeader))
|
|
mu.Unlock()
|
|
body := freeSSE
|
|
if paid {
|
|
body = paidSSE
|
|
}
|
|
committed(w, true, "0.004", body, cost)
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
restore(t, ensoFam)
|
|
ensoFam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: srv.URL}
|
|
}
|
|
ensoFam.byID = map[string]zenModel{"enso-pro": {ID: "enso-pro", Plan: true}}
|
|
ensoFam.ids = []string{"enso-pro"}
|
|
ensoFam.loaded, ensoFam.fetchedAt = true, time.Now()
|
|
return srv, func() []string { mu.Lock(); defer mu.Unlock(); return append([]string(nil), spends...) }
|
|
}
|
|
|
|
// A request its plan covers carries what the plan holds to Enso, and settles against
|
|
// the plan at what Enso states the answer cost: the usage record says the plan covered
|
|
// it, so no wallet is debited and no free allowance counted. A free rung's answer
|
|
// costs nothing.
|
|
func TestAPlanCoveredFamilyCallCarriesSpendAndSettlesItsCost(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
paid bool
|
|
cost string
|
|
want int64
|
|
}{
|
|
{"a paid rung answered", true, "0.0025", 2_500_000},
|
|
{"a free rung answered", false, "0", 0},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
cooled.forget()
|
|
_, spends := planEnso(t, tc.paid, tc.cost)
|
|
|
|
var mu sync.Mutex
|
|
var events []object.UsageEvent
|
|
prev := object.UsageRecorder()
|
|
object.SetUsageRecorder(func(_ context.Context, u object.UsageEvent) error {
|
|
mu.Lock()
|
|
events = append(events, u)
|
|
mu.Unlock()
|
|
return nil
|
|
})
|
|
t.Cleanup(func() { object.SetUsageRecorder(prev) })
|
|
|
|
var settled []int64
|
|
grant := &object.LimitGrant{Plan: "max-20x", Pays: object.PaysPlan, Spend: 50_000_000, Settle: func(n int64) {
|
|
mu.Lock()
|
|
settled = append(settled, n)
|
|
mu.Unlock()
|
|
}}
|
|
body := []byte(`{"model":"enso-pro","messages":[{"role":"user","content":"hi"}]}`)
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody(body)
|
|
Cover(c.Ctx, grant)
|
|
user := &iam.User{Owner: "acme", Name: "ann"}
|
|
if out := c.pipeToFamily(ensoFam, "chat/completions", "openai", "enso-pro", body, false, 0, "acme", user, false, nil, time.Now()); out != nil {
|
|
t.Fatalf("attempts=%+v, want Enso to answer", out)
|
|
}
|
|
drain(t, c)
|
|
|
|
if got := spends(); len(got) != 1 || got[0] != "0.050000000" {
|
|
t.Fatalf("Enso was sent spend %v, want the plan's hold in USD", got)
|
|
}
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if len(settled) == 0 || settled[0] != tc.want {
|
|
t.Fatalf("settled %v, want %d first", settled, tc.want)
|
|
}
|
|
for _, n := range settled[1:] {
|
|
if n != 0 {
|
|
t.Fatalf("settled %v: a later settle charged more", settled)
|
|
}
|
|
}
|
|
if len(events) != 1 || !events[0].Plan {
|
|
t.Fatalf("usage events %+v, want one marked as the plan's", events)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Without a grant, or with one holding nothing, no spend travels: the family answers
|
|
// from its free rungs.
|
|
func TestNoSpendTravelsWithoutAHold(t *testing.T) {
|
|
for name, grant := range map[string]*object.LimitGrant{
|
|
"no plan": nil,
|
|
"a plan holding none": {Plan: "max-20x", Pays: object.PaysPlan, Spend: 0, Settle: func(int64) {}},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
cooled.forget()
|
|
_, spends := planEnso(t, false, "0")
|
|
body := []byte(`{"model":"enso-pro","messages":[{"role":"user","content":"hi"}]}`)
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody(body)
|
|
Cover(c.Ctx, grant)
|
|
c.pipeToFamily(ensoFam, "chat/completions", "openai", "enso-pro", body, false, 0, "acme", nil, false, nil, time.Now())
|
|
drain(t, c)
|
|
if got := spends(); len(got) != 1 || got[0] != "" {
|
|
t.Fatalf("Enso was sent spend %v, want none", got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The spend travels to a Hanzo family only: when Enso cannot answer and the pool
|
|
// does, the pool's vendor is never told what the plan holds.
|
|
func TestTheSpendNeverReachesAnotherFamily(t *testing.T) {
|
|
cooled.forget()
|
|
forgetKeys()
|
|
const free = "vendor/big:free"
|
|
var poolSpend []string
|
|
pool := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
poolSpend = append(poolSpend, r.Header.Get(spendHeader))
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"id":"1","model":"` + free + `","choices":[{"index":0,"message":{"role":"assistant","content":"ok"}}]}`))
|
|
}))
|
|
defer pool.Close()
|
|
spareFamily(t, pool.URL, free)
|
|
|
|
down := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"down"}}`))
|
|
}))
|
|
defer down.Close()
|
|
restore(t, ensoFam)
|
|
ensoFam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: down.URL}
|
|
}
|
|
ensoFam.loaded, ensoFam.fetchedAt = true, time.Now()
|
|
|
|
body := []byte(`{"model":"enso-flash","messages":[{"role":"user","content":"hi"}]}`)
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody(body)
|
|
Cover(c.Ctx, &object.LimitGrant{Plan: "max-20x", Pays: object.PaysPlan, Spend: 50_000_000, Settle: func(int64) {}})
|
|
c.pipeToFamily(ensoFam, "chat/completions", "openai", "enso-flash", body, false, 0, "acme", nil, false, nil, time.Now())
|
|
if len(poolSpend) == 0 {
|
|
t.Fatal("the pool was never asked — this test proves nothing")
|
|
}
|
|
for _, s := range poolSpend {
|
|
if s != "" {
|
|
t.Fatalf("the pool's vendor was sent the plan's spend %q", s)
|
|
}
|
|
}
|
|
}
|
|
|
|
// enso stands up an Enso service that answers every request with handler.
|
|
func enso(t *testing.T, handler http.HandlerFunc) {
|
|
t.Helper()
|
|
srv := httptest.NewServer(handler)
|
|
t.Cleanup(srv.Close)
|
|
restore(t, ensoFam)
|
|
ensoFam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: srv.URL}
|
|
}
|
|
ensoFam.byID = map[string]zenModel{"enso-pro": {ID: "enso-pro", Plan: true}}
|
|
ensoFam.ids = []string{"enso-pro"}
|
|
ensoFam.loaded, ensoFam.fetchedAt = true, time.Now()
|
|
}
|
|
|
|
// drain runs a streamed answer's writer to its end, which is where it settles.
|
|
func drain(t *testing.T, c *ApiController) string {
|
|
t.Helper()
|
|
s := toStream()
|
|
if err := c.Fiber().Response().BodyWriteTo(s.w); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = s.w.Flush()
|
|
return s.buf.String()
|
|
}
|
|
|
|
// pipeCovered sends one covered enso-pro chat request through the family pipe, asked
|
|
// whole, and answers what its grant was settled with.
|
|
func pipeCovered(t *testing.T, spend int64) []int64 { return pipeCoveredAs(t, spend, "openai") }
|
|
|
|
// pipeCoveredAs is pipeCovered in dialect: "openai" (a chat completion, which ai asks
|
|
// of the family as a stream), "openai-stream" (one the caller streams) or "anthropic"
|
|
// (a message, asked whole).
|
|
func pipeCoveredAs(t *testing.T, spend int64, dialect string) []int64 {
|
|
t.Helper()
|
|
settled, _ := coveredCall(t, spend, dialect)
|
|
return settled
|
|
}
|
|
|
|
// coveredCall is pipeCoveredAs that also answers what the caller was sent: the body,
|
|
// with the status ahead of it when it is not 200.
|
|
func coveredCall(t *testing.T, spend int64, dialect string) ([]int64, string) {
|
|
t.Helper()
|
|
settled, sent, _ := planCall(t, &spend, dialect, time.Now())
|
|
return settled, sent
|
|
}
|
|
|
|
// planCall sends one enso-pro request through the family pipe, covered by a grant of spend
|
|
// when spend is set, as if it began at start, and answers what settled, what the caller
|
|
// was sent, and how long the caller waited for its status: pipeToFamily returning is
|
|
// the moment the status and the first bytes can leave.
|
|
func planCall(t *testing.T, spend *int64, dialect string, start time.Time) ([]int64, string, time.Duration) {
|
|
t.Helper()
|
|
cooled.forget()
|
|
var mu sync.Mutex
|
|
var settled []int64
|
|
prev := object.UsageRecorder()
|
|
object.SetUsageRecorder(func(context.Context, object.UsageEvent) error { return nil })
|
|
t.Cleanup(func() { object.SetUsageRecorder(prev) })
|
|
path, apiPath := "/v1/chat/completions", "chat/completions"
|
|
body := []byte(`{"model":"enso-pro","messages":[{"role":"user","content":"hi"}]}`)
|
|
stream := dialect == "openai-stream"
|
|
if stream {
|
|
dialect = "openai"
|
|
}
|
|
if dialect == "anthropic" {
|
|
path, apiPath = "/v1/messages", "messages"
|
|
body = []byte(`{"model":"enso-pro","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}`)
|
|
}
|
|
c := visit(http.MethodPost, path)
|
|
c.Fiber().Request().SetBody(body)
|
|
if spend != nil {
|
|
Cover(c.Ctx, &object.LimitGrant{Plan: "max-20x", Pays: object.PaysPlan, Spend: *spend, Settle: func(n int64) { mu.Lock(); settled = append(settled, n); mu.Unlock() }})
|
|
}
|
|
began := time.Now()
|
|
c.pipeToFamily(ensoFam, apiPath, dialect, "enso-pro", body, stream, 0, "acme", &iam.User{Owner: "acme", Name: "ann"}, false, nil, start)
|
|
waited := time.Since(began)
|
|
sent := drain(t, c)
|
|
if st := c.Fiber().Response().StatusCode(); st != http.StatusOK {
|
|
sent = fmt.Sprintf("%d %s", st, sent)
|
|
}
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
return append([]int64(nil), settled...), sent, waited
|
|
}
|
|
|
|
// charged is what settles add up to, and whether the first of them is all of it.
|
|
func charged(got []int64) (sum int64, once bool) {
|
|
for _, n := range got {
|
|
sum += n
|
|
}
|
|
return sum, len(got) > 0 && got[0] == sum
|
|
}
|
|
|
|
// A covered request its family refuses, with nothing served, settles at nothing as
|
|
// soon as it ends: its hold never waits out its keeper.
|
|
func TestARefusedCoveredRequestSettlesAtNothing(t *testing.T) {
|
|
enso(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
_, _ = io.WriteString(w, `{"error":{"message":"bad request"}}`)
|
|
})
|
|
got := pipeCovered(t, 50_000_000)
|
|
if len(got) == 0 {
|
|
t.Fatal("a refused covered request was never settled")
|
|
}
|
|
for _, n := range got {
|
|
if n != 0 {
|
|
t.Fatalf("settled %v, want nothing charged", got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Once a paid rung was tried, no later dispatch of the request carries the plan's
|
|
// spend: one request buys at most one paid answer. Only a Hanzo family is ever sent
|
|
// it, only for a SKU its family lists as plan-capable, and only while the grant holds
|
|
// something.
|
|
func TestThePlansSpendIsNeverSentTwice(t *testing.T) {
|
|
restore(t, ensoFam)
|
|
ensoFam.byID = map[string]zenModel{"enso-pro": {ID: "enso-pro", Plan: true}, "enso-flash": {ID: "enso-flash"}}
|
|
ensoFam.ids = []string{"enso-pro", "enso-flash"}
|
|
ensoFam.loaded, ensoFam.fetchedAt = true, time.Now()
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
if got := spendFor(c.Ctx, ensoFam, "enso-pro"); got != "" {
|
|
t.Fatalf("no grant sent spend %q", got)
|
|
}
|
|
Cover(c.Ctx, &object.LimitGrant{Plan: "max-20x", Pays: object.PaysPlan, Spend: 1_000_000_000, Settle: func(int64) {}})
|
|
if got := spendFor(c.Ctx, ensoFam, "enso-pro"); got != "1.000000000" {
|
|
t.Fatalf("a covered request sent spend %q, want the hold", got)
|
|
}
|
|
if got := spendFor(c.Ctx, ensoFam, "enso-flash"); got != "" {
|
|
t.Fatalf("a SKU its family does not list as plan-capable was sent spend %q", got)
|
|
}
|
|
if got := spendFor(c.Ctx, freeFamily(), "vendor/big:free"); got != "" {
|
|
t.Fatalf("a borrowed family was sent spend %q", got)
|
|
}
|
|
markOf(c.Ctx).tried.Store(true)
|
|
if got := spendFor(c.Ctx, ensoFam, "enso-pro"); got != "" {
|
|
t.Fatalf("after a paid rung was tried the request still sends spend %q", got)
|
|
}
|
|
}
|
|
|
|
// A family says which SKUs it opens plan rungs for in its listing, and ai says it
|
|
// reads the cost a committed answer states (TE: trailers) with every spend it sends.
|
|
// A SKU listed without it, as every listing from a family that predates the committed
|
|
// answer is, is sent no spend: a plan opens no paid rung until both sides run it.
|
|
func TestASpendTravelsOnlyBetweenSidesThatSpeakTheCommittedAnswer(t *testing.T) {
|
|
var got struct{ spend, te string }
|
|
for _, tc := range []struct {
|
|
name string
|
|
plan bool
|
|
wantSpend, want string
|
|
}{
|
|
{"a SKU listed as plan-capable", true, "1.000000000", "trailers"},
|
|
{"a SKU listed without it", false, "", ""},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
enso(t, func(w http.ResponseWriter, r *http.Request) {
|
|
got.spend, got.te = r.Header.Get(spendHeader), r.Header.Get("TE")
|
|
committed(w, true, "0.004", freeSSE, "0")
|
|
})
|
|
ensoFam.byID = map[string]zenModel{"enso-pro": {ID: "enso-pro", Plan: tc.plan}}
|
|
pipeCovered(t, 1_000_000_000)
|
|
if got.spend != tc.wantSpend || got.te != tc.want {
|
|
t.Errorf("Enso was sent spend %q and TE %q, want %q and %q", got.spend, got.te, tc.wantSpend, tc.want)
|
|
}
|
|
})
|
|
}
|
|
var listed struct {
|
|
Data []zenWireModel `json:"data"`
|
|
}
|
|
if err := json.Unmarshal([]byte(`{"data":[{"id":"enso-pro","plan":true},{"id":"enso-flash"}]}`), &listed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !listed.Data[0].model().Plan || listed.Data[1].model().Plan {
|
|
t.Errorf("listing read as plan %t/%t, want true/false", listed.Data[0].model().Plan, listed.Data[1].model().Plan)
|
|
}
|
|
}
|
|
|
|
// ensoAt points the Enso family at handler and shortens the wait for a family's
|
|
// headers to wait, so a test can outlast it.
|
|
func ensoAt(t *testing.T, wait time.Duration, handler http.HandlerFunc) {
|
|
t.Helper()
|
|
srv := httptest.NewServer(handler)
|
|
t.Cleanup(srv.Close)
|
|
restore(t, ensoFam)
|
|
ensoFam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Type: "Enso", ProviderUrl: srv.URL}
|
|
}
|
|
ensoFam.byID = map[string]zenModel{"enso-pro": {ID: "enso-pro", Plan: true}}
|
|
ensoFam.ids = []string{"enso-pro"}
|
|
ensoFam.loaded, ensoFam.fetchedAt = true, time.Now()
|
|
prev := zenPipeClient
|
|
zenPipeClient = &http.Client{Transport: &http.Transport{ResponseHeaderTimeout: wait}}
|
|
t.Cleanup(func() { zenPipeClient = prev })
|
|
}
|
|
|
|
// A dispatch that carried the plan's spend and got no answer back — its headers came
|
|
// after ai stopped waiting, or the connection was cut — is the request's one paid
|
|
// attempt: no later dispatch carries the spend again. It is charged nothing, since a
|
|
// family states a paid rung's rate and bound the moment that rung's upstream accepts.
|
|
func TestASpendWithNoAnswerIsTheOnePaidAttempt(t *testing.T) {
|
|
cases := map[string]func(http.ResponseWriter, *http.Request){
|
|
"headers past the wait": func(w http.ResponseWriter, r *http.Request) {
|
|
time.Sleep(300 * time.Millisecond)
|
|
committed(w, true, "0.004", paidSSE, "0.0025")
|
|
},
|
|
"connection cut": func(w http.ResponseWriter, r *http.Request) {
|
|
conn, _, err := w.(http.Hijacker).Hijack()
|
|
if err == nil {
|
|
_ = conn.Close()
|
|
}
|
|
},
|
|
}
|
|
for name, answer := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
var asked, spent atomic.Int32
|
|
ensoAt(t, 100*time.Millisecond, func(w http.ResponseWriter, r *http.Request) {
|
|
asked.Add(1)
|
|
if r.Header.Get(spendHeader) != "" {
|
|
spent.Add(1)
|
|
}
|
|
answer(w, r)
|
|
})
|
|
for _, n := range pipeCovered(t, 1_000_000_000) {
|
|
if n != 0 {
|
|
t.Errorf("a dispatch with no answer was charged %d", n)
|
|
}
|
|
}
|
|
if spent.Load() != 1 {
|
|
t.Errorf("enso was sent the spend %d time(s) over %d dispatch(es), want once", spent.Load(), asked.Load())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A dispatch that never carried the spend and got no answer charges the plan
|
|
// nothing: no paid rung could have been asked.
|
|
func TestNoSpendNoAnswerSettlesNothing(t *testing.T) {
|
|
ensoAt(t, 100*time.Millisecond, func(w http.ResponseWriter, r *http.Request) {
|
|
conn, _, err := w.(http.Hijacker).Hijack()
|
|
if err == nil {
|
|
_ = conn.Close()
|
|
}
|
|
})
|
|
for _, n := range pipeCovered(t, 0) {
|
|
if n != 0 {
|
|
t.Fatalf("a request that sent no spend was charged %d", n)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A committed answer cut before its end states no cost: it is charged the most its
|
|
// family said the request can cost once a paid rung's upstream accepted it — the paid
|
|
// marker on a stream, a tab ahead of a whole answer — and nothing before that.
|
|
// A family that states no bound is charged at most unstatedNanos, never what its family
|
|
// has left.
|
|
func TestACutAnswerSettlesItsBoundOnceAPaidRungAccepted(t *testing.T) {
|
|
const left = 10_000_000_000 // $10 the family has left this period
|
|
words := `data: {"id":"x","choices":[{"delta":{"content":"ok"}}]}` + "\n\n"
|
|
for _, tc := range []struct {
|
|
name, dialect string
|
|
answer http.HandlerFunc
|
|
want int64
|
|
}{
|
|
{"a stream cut after the paid marker", "openai", func(w http.ResponseWriter, r *http.Request) {
|
|
cutAfter(w, true, "0.004", ": keep-alive\n\n", ": paid\n\n", words)
|
|
}, 4_000_000},
|
|
{"a stream cut before any paid rung accepted", "openai", func(w http.ResponseWriter, r *http.Request) {
|
|
cutAfter(w, true, "0.004", ": keep-alive\n\n", words)
|
|
}, 0},
|
|
{"a caller's stream cut after the paid marker", "openai-stream", func(w http.ResponseWriter, r *http.Request) {
|
|
cutAfter(w, true, "0.004", ": keep-alive\n\n", ": paid\n\n", words)
|
|
}, 4_000_000},
|
|
{"a caller's stream cut before any paid rung accepted", "openai-stream", func(w http.ResponseWriter, r *http.Request) {
|
|
cutAfter(w, true, "0.004", ": keep-alive\n\n", words)
|
|
}, 0},
|
|
{"a whole answer cut after a paid rung accepted", "anthropic", func(w http.ResponseWriter, r *http.Request) {
|
|
cutAfter(w, false, "0.004", " \t", `{"id":"msg_1",`)
|
|
}, 4_000_000},
|
|
{"a bound past what is left", "openai", func(w http.ResponseWriter, r *http.Request) {
|
|
cutAfter(w, true, "75", ": paid\n\n", words)
|
|
}, left},
|
|
{"no bound stated", "openai", func(w http.ResponseWriter, r *http.Request) {
|
|
cutAfter(w, true, "", ": paid\n\n", words)
|
|
}, unstatedNanos},
|
|
{"a malformed bound", "openai", func(w http.ResponseWriter, r *http.Request) {
|
|
cutAfter(w, true, "lots", ": paid\n\n", words)
|
|
}, unstatedNanos},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
enso(t, tc.answer)
|
|
got := pipeCoveredAs(t, left, tc.dialect)
|
|
if sum, once := charged(got); sum != tc.want || (tc.want > 0 && !once) {
|
|
t.Errorf("settled %v, want %d first and once", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A committed answer that ends whole is charged the cost its family states, never past
|
|
// what the family has left, whatever the paid marker said; one that ends whole with no
|
|
// stated cost is charged as a cut one.
|
|
func TestAWholeAnswerSettlesItsStatedCost(t *testing.T) {
|
|
const left = 1_000_000_000 // $1
|
|
for _, tc := range []struct {
|
|
name, dialect, body, cost string
|
|
want int64
|
|
}{
|
|
{"a paid stream", "openai", paidSSE, "0.0025", 2_500_000},
|
|
{"a free stream", "openai", freeSSE, "0", 0},
|
|
{"a caller's paid stream", "openai-stream", paidSSE, "0.0025", 2_500_000},
|
|
{"a cost past what is left", "openai", paidSSE, "7.5", left},
|
|
{"a paid whole answer", "anthropic", ` {"id":"msg_1","type":"message","role":"assistant","model":"enso-pro","content":[{"type":"text","text":"ok"}],"usage":{"input_tokens":10,"output_tokens":5}}`, "0.0001", 100_000},
|
|
{"a free whole answer", "anthropic", ` {"id":"msg_1","type":"message","role":"assistant","model":"enso-pro","content":[{"type":"text","text":"ok"}],"usage":{"input_tokens":10,"output_tokens":5}}`, "0", 0},
|
|
{"a paid stream that states no cost", "openai", paidSSE, "", 4_000_000},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
enso(t, func(w http.ResponseWriter, r *http.Request) {
|
|
committed(w, tc.dialect != "anthropic", "0.004", tc.body, tc.cost)
|
|
})
|
|
got := pipeCoveredAs(t, left, tc.dialect)
|
|
if sum, once := charged(got); sum != tc.want || (tc.want > 0 && !once) {
|
|
t.Errorf("settled %v, want %d first and once", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// freePool stands up the free pool answering every request with "from-pool", in the
|
|
// shape it was asked for, and counts the requests it was sent.
|
|
func freePool(t *testing.T) *atomic.Int32 {
|
|
t.Helper()
|
|
cooled.forget()
|
|
forgetKeys()
|
|
var asked atomic.Int32
|
|
const free = "vendor/big:free"
|
|
pool := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
asked.Add(1)
|
|
b, _ := io.ReadAll(r.Body)
|
|
switch {
|
|
case strings.HasSuffix(r.URL.Path, "/messages"):
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = io.WriteString(w, `{"id":"msg_p","type":"message","role":"assistant","model":"`+free+`","content":[{"type":"text","text":"from-pool"}],"stop_reason":"end_turn","usage":{"input_tokens":3,"output_tokens":2}}`)
|
|
case strings.Contains(string(b), `"stream":true`):
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = io.WriteString(w, `data: {"id":"p","model":"`+free+`","choices":[{"delta":{"content":"from-pool"}}]}`+"\n\n"+
|
|
`data: {"id":"p","choices":[],"usage":{"prompt_tokens":3,"completion_tokens":2}}`+"\n\ndata: [DONE]\n\n")
|
|
default:
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = io.WriteString(w, `{"id":"p","model":"`+free+`","choices":[{"index":0,"message":{"role":"assistant","content":"from-pool"},"finish_reason":"stop"}]}`)
|
|
}
|
|
}))
|
|
t.Cleanup(pool.Close)
|
|
spareFamily(t, pool.URL, free)
|
|
return &asked
|
|
}
|
|
|
|
// A committed answer that fails before it says anything is a refusal like any other
|
|
// family's: it falls to the free pool, exactly as it would for a caller no plan
|
|
// covers, and it is charged what its family states it cost — nothing when no paid
|
|
// rung accepted it, whatever bound it was committed under. Late or whole, it is the
|
|
// same: a covered caller never sees an error a caller with no plan would not.
|
|
func TestACommittedFailureFallsToThePool(t *testing.T) {
|
|
prev := openWait
|
|
openWait = 50 * time.Millisecond
|
|
t.Cleanup(func() { openWait = prev })
|
|
fail := `data: {"error":{"message":"this model is temporarily unavailable","code":503}}` + "\n\n"
|
|
late := func(w http.ResponseWriter, sse bool, tail, cost string) {
|
|
if sse {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
} else {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
}
|
|
w.Header().Set(costBoundHeader, "0.75")
|
|
w.Header().Set("Trailer", costHeader)
|
|
w.WriteHeader(http.StatusOK)
|
|
for i := 0; i < 6; i++ { // the family's keep-alives while it walks, past openWait
|
|
if sse {
|
|
_, _ = io.WriteString(w, ": keep-alive\n\n")
|
|
} else {
|
|
_, _ = io.WriteString(w, " ")
|
|
}
|
|
w.(http.Flusher).Flush()
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
_, _ = io.WriteString(w, tail)
|
|
w.Header().Set(costHeader, cost)
|
|
}
|
|
for _, tc := range []struct {
|
|
name, dialect string
|
|
answer http.HandlerFunc
|
|
want int64
|
|
}{
|
|
{"a stream that fails before any paid rung", "openai", func(w http.ResponseWriter, r *http.Request) {
|
|
committed(w, true, "0.75", ": keep-alive\n\n"+fail, "0")
|
|
}, 0},
|
|
{"a caller's stream that fails before any paid rung", "openai-stream", func(w http.ResponseWriter, r *http.Request) {
|
|
committed(w, true, "0.75", ": keep-alive\n\n"+fail, "0")
|
|
}, 0},
|
|
{"a stream that fails past openWait", "openai-stream", func(w http.ResponseWriter, r *http.Request) {
|
|
late(w, true, fail, "0")
|
|
}, 0},
|
|
{"a stream that ends empty", "openai", func(w http.ResponseWriter, r *http.Request) {
|
|
committed(w, true, "0.75", ": keep-alive\n\ndata: [DONE]\n\n", "0")
|
|
}, 0},
|
|
{"a paid rung that accepted, then failed", "openai-stream", func(w http.ResponseWriter, r *http.Request) {
|
|
committed(w, true, "0.75", ": paid\n\n"+fail, "0.004")
|
|
}, 4_000_000},
|
|
{"a paid rung that accepted, then failed, its cost lost", "openai-stream", func(w http.ResponseWriter, r *http.Request) {
|
|
cutAfter(w, true, "0.004", ": paid\n\n", fail)
|
|
}, 4_000_000},
|
|
{"a whole answer that is an error object", "anthropic", func(w http.ResponseWriter, r *http.Request) {
|
|
late(w, false, `{"type":"error","error":{"type":"api_error","message":"this model is temporarily unavailable"}}`, "0")
|
|
}, 0},
|
|
{"a whole answer that is an error object, a paid rung having failed", "anthropic", func(w http.ResponseWriter, r *http.Request) {
|
|
committed(w, false, "0.75", ` {"error":{"message":"the model returned an unusable answer","type":"zen_error","code":502}}`, "0.004")
|
|
}, 4_000_000},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
asked := freePool(t)
|
|
enso(t, tc.answer)
|
|
got, sent := coveredCall(t, 1_000_000_000, tc.dialect)
|
|
if !strings.Contains(sent, "from-pool") || strings.Contains(sent, "temporarily unavailable") || strings.Contains(sent, "unusable") {
|
|
t.Errorf("the caller was sent %q (pool asked %d), want the free pool's answer", sent, asked.Load())
|
|
}
|
|
if sum, once := charged(got); sum != tc.want || (tc.want > 0 && !once) {
|
|
t.Errorf("settled %v, want %d first and once", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A committed answer that has not begun by the time ai stops waiting for a family is
|
|
// abandoned exactly as a family that sent no headers would be: the family is hung up
|
|
// on, which stops its walk before any paid rung, the free pool answers, and nothing is
|
|
// charged. Once a paid rung accepted, ai waits for its answer however long it takes.
|
|
func TestACommittedAnswerThatNeverBeginsFallsToThePool(t *testing.T) {
|
|
prev := commitWait
|
|
commitWait = 100 * time.Millisecond
|
|
t.Cleanup(func() { commitWait = prev })
|
|
hungUp := make(chan struct{}, 1)
|
|
asked := freePool(t)
|
|
enso(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.Header().Set(costBoundHeader, "0.75")
|
|
w.Header().Set("Trailer", costHeader)
|
|
w.WriteHeader(http.StatusOK)
|
|
for {
|
|
if _, err := io.WriteString(w, ": keep-alive\n\n"); err != nil {
|
|
hungUp <- struct{}{}
|
|
return
|
|
}
|
|
w.(http.Flusher).Flush()
|
|
select {
|
|
case <-r.Context().Done():
|
|
hungUp <- struct{}{}
|
|
return
|
|
case <-time.After(20 * time.Millisecond):
|
|
}
|
|
}
|
|
})
|
|
got, sent := coveredCall(t, 1_000_000_000, "openai-stream")
|
|
if !strings.Contains(sent, "from-pool") {
|
|
t.Errorf("the caller was sent %q (pool asked %d), want the free pool's answer", sent, asked.Load())
|
|
}
|
|
if sum, _ := charged(got); sum != 0 {
|
|
t.Errorf("settled %v, want nothing", got)
|
|
}
|
|
select {
|
|
case <-hungUp:
|
|
case <-time.After(2 * time.Second):
|
|
t.Error("the family was never hung up on: its walk goes on")
|
|
}
|
|
|
|
slow := `data: {"id":"x","choices":[{"delta":{"content":"paid words"}}]}` + "\n\n"
|
|
enso(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.Header().Set(costBoundHeader, "0.75")
|
|
w.Header().Set("Trailer", costHeader)
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = io.WriteString(w, ": paid\n\n")
|
|
w.(http.Flusher).Flush()
|
|
time.Sleep(300 * time.Millisecond) // the paid rung thinks past commitWait
|
|
_, _ = io.WriteString(w, slow+"data: [DONE]\n\n")
|
|
w.Header().Set(costHeader, "0.002")
|
|
})
|
|
got, sent = coveredCall(t, 1_000_000_000, "openai-stream")
|
|
if !strings.Contains(sent, "paid words") {
|
|
t.Errorf("the caller was sent %q, want the paid rung's answer", sent)
|
|
}
|
|
if sum, _ := charged(got); sum != 2_000_000 {
|
|
t.Errorf("settled %v, want the stated 2000000", got)
|
|
}
|
|
}
|
|
|
|
// A committed refusal the caller caused — a request no vendor would serve — reaches
|
|
// the caller with its status, as it would with no plan, and charges nothing.
|
|
func TestACommittedRefusalTheCallerCausedReachesTheCaller(t *testing.T) {
|
|
asked := freePool(t)
|
|
enso(t, func(w http.ResponseWriter, r *http.Request) {
|
|
committed(w, true, "0.75", ": keep-alive\n\n"+`data: {"error":{"message":"messages must not be empty","code":400}}`+"\n\n", "0")
|
|
})
|
|
got, sent := coveredCall(t, 1_000_000_000, "openai")
|
|
if !strings.HasPrefix(sent, "400 ") || asked.Load() != 0 {
|
|
t.Errorf("the caller was sent %q and the pool asked %d times; want the 400 and no pool", sent, asked.Load())
|
|
}
|
|
if sum, _ := charged(got); sum != 0 {
|
|
t.Errorf("settled %v, want nothing", got)
|
|
}
|
|
}
|
|
|
|
// slowCommit answers a committed stream that keeps its host waiting for think — its
|
|
// family walking its rungs, keep-alives meanwhile — then ends with tail and states
|
|
// cost (none when empty). paid writes the paid marker first.
|
|
func slowCommit(think time.Duration, paid bool, tail, cost string) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.Header().Set(costBoundHeader, "0.004")
|
|
w.Header().Set("Trailer", costHeader)
|
|
w.WriteHeader(http.StatusOK)
|
|
if paid {
|
|
_, _ = io.WriteString(w, ": paid\n\n")
|
|
}
|
|
for end := time.Now().Add(think); time.Now().Before(end); time.Sleep(20 * time.Millisecond) {
|
|
if _, err := io.WriteString(w, ": keep-alive\n\n"); err != nil {
|
|
return
|
|
}
|
|
w.(http.Flusher).Flush()
|
|
}
|
|
_, _ = io.WriteString(w, tail)
|
|
if cost != "" {
|
|
w.Header().Set(costHeader, cost)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A covered caller is answered on the clock an uncovered one is: its status and
|
|
// keep-alives go out once openWait passes, however long the committed answer takes
|
|
// to begin, and the answer follows in the same response. With no free route to answer
|
|
// a refusal from, the request is held instead, so a refusal still goes back to its
|
|
// route's other providers.
|
|
func TestACoveredCallerIsAnsweredOnTheUncoveredClock(t *testing.T) {
|
|
prev := openWait
|
|
openWait = 50 * time.Millisecond
|
|
t.Cleanup(func() { openWait = prev })
|
|
const think = 400 * time.Millisecond
|
|
words := `data: {"id":"x","model":"enso-pro","choices":[{"delta":{"content":"paid words"}}]}` + "\n\ndata: [DONE]\n\n"
|
|
for _, d := range []string{"openai-stream", "openai"} {
|
|
t.Run(d, func(t *testing.T) {
|
|
freePool(t)
|
|
enso(t, slowCommit(think, true, words, "0.002"))
|
|
spend := int64(1_000_000_000)
|
|
got, sent, waited := planCall(t, &spend, d, time.Now())
|
|
if waited >= think/2 {
|
|
t.Errorf("the covered caller waited %v for its status, past openWait", waited.Round(time.Millisecond))
|
|
}
|
|
if !strings.Contains(sent, "paid words") {
|
|
t.Errorf("the caller was sent %q, want the answer", sent)
|
|
}
|
|
if sum, once := charged(got); sum != 2_000_000 || !once {
|
|
t.Errorf("settled %v, want the stated 2000000 once", got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A committed answer that refuses after the caller's answer opened — its family's
|
|
// rungs all failed, or a paid rung accepted and then failed — is answered from the free
|
|
// pool inside that open answer, never with the error, and charged what its family
|
|
// states: nothing when no paid rung accepted it. One ai stops waiting for (commitWait)
|
|
// is hung up on and answered the same way, whenever the request began.
|
|
func TestARefusalAfterTheAnswerOpenedIsAnsweredFromThePool(t *testing.T) {
|
|
prevOpen, prevCommit := openWait, commitWait
|
|
openWait, commitWait = 50*time.Millisecond, 2*time.Second
|
|
t.Cleanup(func() { openWait, commitWait = prevOpen, prevCommit })
|
|
fail := `data: {"error":{"message":"this model is temporarily unavailable","code":503}}` + "\n\n"
|
|
for _, tc := range []struct {
|
|
name, dialect string
|
|
answer http.HandlerFunc
|
|
wait time.Duration
|
|
want int64
|
|
}{
|
|
{"every rung failed", "openai-stream", slowCommit(300*time.Millisecond, false, fail, "0"), 0, 0},
|
|
{"every rung failed, asked whole", "openai", slowCommit(300*time.Millisecond, false, fail, "0"), 0, 0},
|
|
{"a paid rung accepted, then failed", "openai-stream", slowCommit(300*time.Millisecond, true, fail, "0.004"), 0, 4_000_000},
|
|
{"ai stopped waiting before any paid rung", "openai-stream", slowCommit(time.Hour, false, "", ""), 200 * time.Millisecond, 0},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if tc.wait > 0 {
|
|
commitWait = tc.wait
|
|
t.Cleanup(func() { commitWait = 2 * time.Second })
|
|
}
|
|
asked := freePool(t)
|
|
enso(t, tc.answer)
|
|
spend := int64(1_000_000_000)
|
|
// The request began long ago: the pool still answers inside an open answer.
|
|
got, sent, waited := planCall(t, &spend, tc.dialect, time.Now().Add(-3*time.Minute))
|
|
if waited >= 200*time.Millisecond {
|
|
t.Errorf("the covered caller waited %v for its status, past openWait", waited.Round(time.Millisecond))
|
|
}
|
|
if !strings.Contains(sent, "from-pool") || strings.Contains(sent, "temporarily unavailable") {
|
|
t.Errorf("the caller was sent %q (pool asked %d), want the free pool's answer", sent, asked.Load())
|
|
}
|
|
if sum, once := charged(got); sum != tc.want || (tc.want > 0 && !once) {
|
|
t.Errorf("settled %v, want %d", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A covered request its family refuses without committing, handed back to the route's
|
|
// other providers, settles its grant before it goes: what it owes, else nothing. No
|
|
// hold outlives it.
|
|
func TestARefusalHandedBackSettlesTheGrant(t *testing.T) {
|
|
enso(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_, _ = io.WriteString(w, `{"error":{"message":"this model is temporarily unavailable","code":503}}`)
|
|
})
|
|
for _, d := range []string{"openai-stream", "openai"} {
|
|
spend := int64(1_000_000_000)
|
|
got, _, _ := planCall(t, &spend, d, time.Now())
|
|
if len(got) == 0 {
|
|
t.Errorf("%s: the request was handed back with its grant unsettled", d)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A whole committed answer cut before it ends is charged its bound only when a paid
|
|
// rung accepted it, which a whole answer says with a tab ahead of its body.
|
|
func TestAWholeCommitIsPaidOnlyPastItsTab(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name, lead string
|
|
want int64
|
|
}{
|
|
{"cut before any paid rung", " ", 0},
|
|
{"cut after a paid rung accepted", " \t", 4_000_000},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
freePool(t)
|
|
enso(t, func(w http.ResponseWriter, r *http.Request) {
|
|
cutAfter(w, false, "0.004", tc.lead, " ")
|
|
})
|
|
got := pipeCoveredAs(t, 1_000_000_000, "anthropic")
|
|
if sum, _ := charged(got); sum != tc.want {
|
|
t.Errorf("settled %v, want %d", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A covered request with no free route to answer a refusal from is held until its
|
|
// committed answer begins or refuses, and a refusal goes back to its route's other
|
|
// providers, as an uncovered caller's does.
|
|
func TestACoveredRequestWithNoFreeRouteIsHeld(t *testing.T) {
|
|
prev := openWait
|
|
openWait = 50 * time.Millisecond
|
|
t.Cleanup(func() { openWait = prev })
|
|
restore(t, freeFamily())
|
|
freeFamily().spares, freeFamily().loaded, freeFamily().fetchedAt = nil, true, time.Now()
|
|
fail := `data: {"error":{"message":"this model is temporarily unavailable","code":503}}` + "\n\n"
|
|
enso(t, slowCommit(200*time.Millisecond, false, fail, "0"))
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
body := []byte(`{"model":"enso-pro","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
|
|
c.Fiber().Request().SetBody(body)
|
|
var got []int64
|
|
Cover(c.Ctx, &object.LimitGrant{Plan: "max-20x", Pays: object.PaysPlan, Spend: 1_000_000_000, Settle: func(n int64) { got = append(got, n) }})
|
|
out := c.pipeToFamily(ensoFam, "chat/completions", "openai", "enso-pro", body, true, 0, "acme", nil, false, nil, time.Now())
|
|
if out == nil {
|
|
t.Fatalf("the refusal was not handed back to the route; the caller was sent %q", drain(t, c))
|
|
}
|
|
if sum, _ := charged(got); sum != 0 {
|
|
t.Errorf("settled %v, want nothing", got)
|
|
}
|
|
}
|
|
|
|
// heard records when each write of a streamed answer reached the caller.
|
|
type heard struct {
|
|
mu sync.Mutex
|
|
buf strings.Builder
|
|
times []time.Time
|
|
}
|
|
|
|
func (g *heard) Write(p []byte) (int, error) {
|
|
g.mu.Lock()
|
|
defer g.mu.Unlock()
|
|
g.times = append(g.times, time.Now())
|
|
g.buf.Write(p)
|
|
return len(p), nil
|
|
}
|
|
|
|
// silence is the longest gap between two writes the caller saw.
|
|
func (g *heard) silence() time.Duration {
|
|
var m time.Duration
|
|
for i := 1; i < len(g.times); i++ {
|
|
m = max(m, g.times[i].Sub(g.times[i-1]))
|
|
}
|
|
return m
|
|
}
|
|
|
|
// openCall sends one covered enso-pro request whose committed answer opens (openWait
|
|
// and heartbeat shortened) and answers what the caller got and the longest silence.
|
|
func openCall(t *testing.T, path, apiPath, dialect string, body []byte, stream bool) (string, time.Duration) {
|
|
t.Helper()
|
|
cooled.forget()
|
|
prev := object.UsageRecorder()
|
|
object.SetUsageRecorder(func(context.Context, object.UsageEvent) error { return nil })
|
|
t.Cleanup(func() { object.SetUsageRecorder(prev) })
|
|
c := visit(http.MethodPost, path)
|
|
c.Fiber().Request().SetBody(body)
|
|
Cover(c.Ctx, &object.LimitGrant{Plan: "max-20x", Pays: object.PaysPlan, Spend: 1_000_000_000, Settle: func(int64) {}})
|
|
c.pipeToFamily(ensoFam, apiPath, dialect, "enso-pro", body, stream, 0, "acme", &iam.User{Owner: "acme", Name: "ann"}, false, nil, time.Now())
|
|
g := &heard{}
|
|
_ = c.Fiber().Response().BodyWriteTo(g)
|
|
return g.buf.String(), g.silence()
|
|
}
|
|
|
|
// opened shortens openWait and heartbeat so a committed answer opens, and keeps alive,
|
|
// within a test's time.
|
|
func opened(t *testing.T) {
|
|
t.Helper()
|
|
prevOpen, prevBeat := openWait, heartbeat
|
|
openWait, heartbeat = 50*time.Millisecond, 40*time.Millisecond
|
|
t.Cleanup(func() { openWait, heartbeat = prevOpen, prevBeat })
|
|
}
|
|
|
|
// poolSays stands up the free pool answering every request with handler.
|
|
func poolSays(t *testing.T, handler http.HandlerFunc) {
|
|
t.Helper()
|
|
cooled.forget()
|
|
forgetKeys()
|
|
srv := httptest.NewServer(handler)
|
|
t.Cleanup(srv.Close)
|
|
spareFamily(t, srv.URL, "vendor/big:free")
|
|
}
|
|
|
|
var (
|
|
chatWhole = []byte(`{"model":"enso-pro","messages":[{"role":"user","content":"hi"}]}`)
|
|
chatStream = []byte(`{"model":"enso-pro","stream":true,"messages":[{"role":"user","content":"hi"}]}`)
|
|
msgStream = []byte(`{"model":"enso-pro","stream":true,"max_tokens":100,"messages":[{"role":"user","content":"hi"}]}`)
|
|
unavail = `data: {"error":{"message":"this model is temporarily unavailable","code":503}}` + "\n\n"
|
|
)
|
|
|
|
// An open answer the free pool cannot fill ends in the caller's dialect's error: a
|
|
// JSON error frame on a chat stream, an Anthropic error event on a message stream, an
|
|
// error object on a whole answer — never Go's words, never an empty completion.
|
|
func TestAnOpenAnswerThePoolCannotFillEndsInTheDialectsError(t *testing.T) {
|
|
opened(t)
|
|
prev := commitWait
|
|
commitWait = 300 * time.Millisecond
|
|
t.Cleanup(func() { commitWait = prev })
|
|
poolSays(t, func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_, _ = io.WriteString(w, `{"error":{"message":"busy"}}`)
|
|
})
|
|
for _, tc := range []struct {
|
|
name, path, api, dialect string
|
|
body []byte
|
|
stream bool
|
|
check func(string) bool
|
|
}{
|
|
{"chat stream", "/v1/chat/completions", "chat/completions", "openai", chatStream, true, func(s string) bool {
|
|
return strings.Contains(s, `data: {"error":{`) && !strings.Contains(s, "data: the family")
|
|
}},
|
|
{"chat whole", "/v1/chat/completions", "chat/completions", "openai", chatWhole, false, func(s string) bool {
|
|
return strings.HasPrefix(strings.TrimSpace(s), `{"error":{`)
|
|
}},
|
|
{"message stream", "/v1/messages", "messages", "anthropic", msgStream, true, func(s string) bool {
|
|
return strings.Contains(s, "event: error\ndata: {") && strings.Contains(s, `"type":"error"`)
|
|
}},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
enso(t, slowCommit(time.Hour, false, "", ""))
|
|
sent, _ := openCall(t, tc.path, tc.api, tc.dialect, tc.body, tc.stream)
|
|
if !tc.check(sent) {
|
|
t.Errorf("the caller was sent %q", sent)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The free pool's answer given whole, inside an open answer, is the stream it adds up
|
|
// to in the caller's dialect: each tool call its own, a whole Anthropic message as its
|
|
// events. A commit's own event lines ahead of its refusal never reach the caller.
|
|
func TestAWholePoolAnswerIsStreamedInTheCallersDialect(t *testing.T) {
|
|
opened(t)
|
|
whole := `{"id":"p","model":"vendor/big:free","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":null,"tool_calls":[` +
|
|
`{"id":"a","type":"function","function":{"name":"read","arguments":"{\"f\":1}"}},` +
|
|
`{"id":"b","type":"function","function":{"name":"write","arguments":"{\"g\":2}"}}]}}]}`
|
|
poolSays(t, func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = io.WriteString(w, whole)
|
|
})
|
|
enso(t, slowCommit(200*time.Millisecond, false, unavail, "0"))
|
|
sent, _ := openCall(t, "/v1/chat/completions", "chat/completions", "openai", chatWhole, false)
|
|
if strings.Contains(sent, "readwrite") || !strings.Contains(sent, `"name":"read"`) || !strings.Contains(sent, `"name":"write"`) {
|
|
t.Errorf("the two tool calls arrived as %s", strings.TrimSpace(sent))
|
|
}
|
|
|
|
freePool(t) // answers /messages with a whole Anthropic message
|
|
fail := "event: error\n" + `data: {"type":"error","error":{"type":"overloaded_error","message":"this model is temporarily unavailable"}}` + "\n\n"
|
|
enso(t, slowCommit(200*time.Millisecond, false, fail, "0"))
|
|
sent, _ = openCall(t, "/v1/messages", "messages", "anthropic", msgStream, true)
|
|
if !strings.Contains(sent, "from-pool") || strings.Contains(sent, "event: error") || !strings.Contains(sent, "event: message_stop") {
|
|
t.Errorf("the caller was sent %q", sent)
|
|
}
|
|
}
|
|
|
|
// While the free pool works on an open stream's answer, the caller keeps hearing
|
|
// keep-alives.
|
|
func TestThePoolKeepsTheCallerAlive(t *testing.T) {
|
|
opened(t)
|
|
const slow = 600 * time.Millisecond
|
|
poolSays(t, func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
time.Sleep(slow)
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = io.WriteString(w, `data: {"id":"p","choices":[{"delta":{"content":"from-pool"}}]}`+"\n\ndata: [DONE]\n\n")
|
|
})
|
|
enso(t, slowCommit(200*time.Millisecond, false, unavail, "0"))
|
|
sent, gap := openCall(t, "/v1/chat/completions", "chat/completions", "openai", chatStream, true)
|
|
if !strings.Contains(sent, "from-pool") || gap >= slow/2 {
|
|
t.Errorf("the caller heard nothing for %v (pool answered: %t)", gap.Round(time.Millisecond), strings.Contains(sent, "from-pool"))
|
|
}
|
|
}
|