mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
ai reads the paying org's published capabilities from the host
(object.SetCapabilities; in cloud, the train app over the plane) and
sends them to the decision service as X-Org-Capabilities, a JSON array
of {name, sha256, base_sha256, form, route}. Kai fetches each head by
its sha256 and answers the org's questions by it.
A held answer is keyed by that set, so a publish, a republish or a
withdrawal is never answered from what the org held before; the
capability-tracking guess it replaces is gone. A set that cannot be read
refuses the call 503 rather than answer the org by the base.
861 lines
34 KiB
Go
861 lines
34 KiB
Go
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
|
||
//
|
||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||
// you may not use this file except in compliance with the License.
|
||
// You may obtain a copy of the License at
|
||
//
|
||
// http://www.apache.org/licenses/LICENSE-2.0
|
||
//
|
||
// Unless required by applicable law or agreed to in writing, software
|
||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
// See the License for the specific language governing permissions and
|
||
// limitations under the License.
|
||
|
||
package controllers
|
||
|
||
import (
|
||
"bytes"
|
||
"compress/gzip"
|
||
"compress/zlib"
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"runtime"
|
||
"slices"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/andybalholm/brotli"
|
||
"github.com/klauspost/compress/zstd"
|
||
"github.com/luxfi/zap"
|
||
"github.com/valyala/fasthttp"
|
||
|
||
"github.com/hanzoai/ai/object"
|
||
"github.com/hanzoai/ai/util"
|
||
)
|
||
|
||
// A decision service that cannot be reached is a 502 that names no address: where
|
||
// the service lives is ours to know.
|
||
func TestDecisionRefusalNamesNoAddress(t *testing.T) {
|
||
setupDecisions(t)
|
||
t.Setenv("KAI_URL", "http://127.0.0.1:1")
|
||
status, body := driveDecisions(t, "Bearer "+decisionsKey, decisionBody)
|
||
if status != http.StatusBadGateway || strings.Contains(body, "127.0.0.1") || strings.Contains(body, "http") {
|
||
t.Fatalf("unreachable => %d %s", status, body)
|
||
}
|
||
}
|
||
|
||
// One org asking more than its share of the cache evicts its own answers, never
|
||
// another org's: acme's held answer survives globex filling the cache.
|
||
func TestDecisionCacheKeepsEachOrgsShare(t *testing.T) {
|
||
fake, _ := setupDecisions(t)
|
||
fake.answer = heldAnswer
|
||
kai := object.KaiProvider()
|
||
ctx := context.Background()
|
||
recall(ctx, kai, "kai", "acme", "", "r", []byte(decisionBody))
|
||
recall(ctx, kai, "kai", "acme", "", "r", []byte(decisionBody))
|
||
for i := 0; i < decisionMax; i++ {
|
||
b := strings.Replace(decisionBody, "twice", fmt.Sprintf("twice %d", i), 1)
|
||
recall(ctx, kai, "kai", "globex", "", "r", []byte(b))
|
||
}
|
||
before, _, _ := fake.seen()
|
||
recall(ctx, kai, "kai", "acme", "", "r", []byte(decisionBody))
|
||
if after, _, _ := fake.seen(); after != before {
|
||
t.Fatalf("globex's %d requests evicted acme's held answer", decisionMax)
|
||
}
|
||
decisionCache.mu.Lock()
|
||
defer decisionCache.mu.Unlock()
|
||
if n := decisionCache.orgs["globex"].n; n > decisionMax/decisionShare {
|
||
t.Fatalf("globex holds %d answers, over its share of %d", n, decisionMax/decisionShare)
|
||
}
|
||
}
|
||
|
||
// gatewayCall is a MsgType 200 request carrying headers and a body, as the ZAP
|
||
// gateway sends one.
|
||
func gatewayCall(t *testing.T, path string, headers map[string]string, body string) *zap.Message {
|
||
t.Helper()
|
||
h, _ := json.Marshal(headers)
|
||
b := zap.NewBuilder(len(body) + len(h) + 256)
|
||
obj := b.StartObject(40)
|
||
obj.SetText(0, http.MethodPost)
|
||
obj.SetText(8, path)
|
||
obj.SetBytes(16, h)
|
||
obj.SetBytes(24, []byte(body))
|
||
obj.FinishAsRoot()
|
||
msg, err := zap.Parse(b.FinishWithFlags(object.MsgTypeHTTPRequest << 8))
|
||
if err != nil {
|
||
t.Fatalf("build gateway request: %v", err)
|
||
}
|
||
return msg
|
||
}
|
||
|
||
// Both doors pay from, and scope handles by, the org a JWT asked to act in under
|
||
// X-Org-Id, and both refuse an org its signed membership does not cover.
|
||
func TestDecisionDoorsAgreeUnderOrgSwitch(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
var mu sync.Mutex
|
||
var seen []string
|
||
fake.serve = func(_ string, body []byte) (int, string) {
|
||
var b struct{ Observe, Handle string }
|
||
_ = json.Unmarshal(body, &b)
|
||
mu.Lock()
|
||
seen = append(seen, b.Observe+b.Handle)
|
||
mu.Unlock()
|
||
return http.StatusOK, decisionAnswer
|
||
}
|
||
tok := mintUsageJWTWithOrgs(t, "beta", "bob", "beta", "acme")
|
||
observe := strings.Replace(decisionBody, `}}}`, `}},"observe":"s1"}`, 1)
|
||
if status, body, _ := drive(t, "Bearer "+tok, observe, map[string]string{"X-Org-Id": "acme"}); status != 200 {
|
||
t.Fatalf("HTTP => %d %s", status, body)
|
||
}
|
||
msg, err := gateway(nil)(context.Background(), "", gatewayCall(t, decisionsPath,
|
||
map[string]string{"Authorization": "Bearer " + tok, "X-Org-Id": "acme", "X-Request-Id": "zap-7"}, `{"model":"kai","handle":"s1"}`))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if st := msg.Root().Uint32(object.GatewayRespStatus); st != 200 {
|
||
t.Fatalf("ZAP => %d %s", st, msg.Root().Bytes(object.GatewayRespBody))
|
||
}
|
||
var h map[string]string
|
||
if json.Unmarshal(msg.Root().Bytes(object.GatewayRespHeaders), &h); h["X-Request-Id"] != "zap-7" || fake.rid != "zap-7" {
|
||
t.Fatalf("the gateway's request id: answered %q, sent %q", h["X-Request-Id"], fake.rid)
|
||
}
|
||
settled(t)
|
||
mu.Lock()
|
||
if strings.Join(seen, " ") != "acme/s1 acme/s1" {
|
||
t.Fatalf("the service was sent %v; both doors must name acme's handle", seen)
|
||
}
|
||
mu.Unlock()
|
||
if len(*events) != 2 || (*events)[0].Namespace != "acme" || (*events)[1].Namespace != "acme" {
|
||
t.Fatalf("debits = %+v; both doors must pay from acme", *events)
|
||
}
|
||
|
||
msg, _ = gateway(nil)(context.Background(), "", gatewayCall(t, decisionsPath,
|
||
map[string]string{"Authorization": "Bearer " + tok, "X-Org-Id": "globex"}, decisionBody))
|
||
if st := msg.Root().Uint32(object.GatewayRespStatus); st != http.StatusForbidden {
|
||
t.Fatalf("ZAP in an org outside the membership => %d, want 403", st)
|
||
}
|
||
}
|
||
|
||
// A debit the ledger does not take is tried again, each try bounded, and Settled
|
||
// waits long enough for every try: the answer went out, so the debit must land.
|
||
func TestDecisionDebitOutlivesAFailingLedger(t *testing.T) {
|
||
_, _ = setupDecisions(t)
|
||
prevTimeout, prevBackoff := usageTimeout, settleBackoff
|
||
usageTimeout, settleBackoff = 100*time.Millisecond, 10*time.Millisecond
|
||
t.Cleanup(func() { usageTimeout, settleBackoff = prevTimeout, prevBackoff })
|
||
if SettleBudget() <= usageTimeout*settleTries {
|
||
t.Fatalf("SettleBudget %v does not cover %d tries of %v", SettleBudget(), settleTries, usageTimeout)
|
||
}
|
||
|
||
var tries atomic.Int32
|
||
var landed atomic.Int32
|
||
object.SetUsageRecorder(func(ctx context.Context, _ object.UsageEvent) error {
|
||
if dl, ok := ctx.Deadline(); !ok || time.Until(dl) > usageTimeout {
|
||
t.Errorf("a debit's hand-off is not bounded by usageTimeout")
|
||
}
|
||
if tries.Add(1) == 1 {
|
||
<-ctx.Done() // the ledger does not answer
|
||
return ctx.Err()
|
||
}
|
||
landed.Add(1)
|
||
return nil
|
||
})
|
||
if status, _, _ := drive(t, "Bearer "+decisionsKey, decisionBody, nil); status != 200 {
|
||
t.Fatalf("status %d", status)
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), SettleBudget())
|
||
defer cancel()
|
||
if err := Settled(ctx); err != nil {
|
||
t.Fatalf("Settled gave up inside its budget: %v", err)
|
||
}
|
||
if tries.Load() != 2 || landed.Load() != 1 {
|
||
t.Fatalf("tries = %d, landed = %d; want the timed-out debit tried again and landed once", tries.Load(), landed.Load())
|
||
}
|
||
}
|
||
|
||
// However many answers go out at once, the debits behind them run on a fixed set
|
||
// of settlers, and one past the queue is filed where it was answered, not dropped.
|
||
func TestDecisionSettlersAreBounded(t *testing.T) {
|
||
_, _ = setupDecisions(t)
|
||
release := make(chan struct{})
|
||
var landed atomic.Int32
|
||
object.SetUsageRecorder(func(context.Context, object.UsageEvent) error {
|
||
<-release
|
||
landed.Add(1)
|
||
return nil
|
||
})
|
||
before := runtime.NumGoroutine()
|
||
rec := func() *usageRecord {
|
||
return &usageRecord{Owner: decisionsOrg, Model: "kai", Provider: object.KaiName, PromptTokens: 1, DecisionCount: 1, Status: "success", RequestID: "r"}
|
||
}
|
||
const n = settleWorkers + settleDepth
|
||
for i := 0; i < n; i++ {
|
||
settleAfter(context.Background(), rec(), time.Now())
|
||
}
|
||
if grew := runtime.NumGoroutine() - before; grew > settleWorkers+8 {
|
||
t.Fatalf("%d debits in flight grew %d goroutines; the settlers are %d", n, grew, settleWorkers)
|
||
}
|
||
inline := make(chan struct{})
|
||
go func() {
|
||
settleAfter(context.Background(), rec(), time.Now())
|
||
close(inline)
|
||
}()
|
||
select {
|
||
case <-inline:
|
||
t.Fatal("a debit past a full queue returned before it was filed")
|
||
case <-time.After(50 * time.Millisecond):
|
||
}
|
||
close(release)
|
||
<-inline
|
||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||
defer cancel()
|
||
if err := Settled(ctx); err != nil {
|
||
t.Fatalf("%d debits did not land: %v", n+1, err)
|
||
}
|
||
if got := landed.Load(); got != n+1 {
|
||
t.Fatalf("landed %d of %d debits", got, n+1)
|
||
}
|
||
}
|
||
|
||
// A decision costs a fraction of a cent and the ledger holds cents, so each call
|
||
// holds at least one and its spend counts in nano: a balance of one cent pays for
|
||
// what it covers and no more.
|
||
func TestJevCannotOverdrawACent(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
fake.answer = `{"model":"typesafe/jev-1.13","answers":{"is_bug":{"type":"noul","noul":0.5}},"usage":{"input_tokens":110000,"output_tokens":0,"cost":0.00462}}`
|
||
object.SetBalanceReader(balReader(1, nil))
|
||
user, _ := providerKeyBillingUser(&object.Provider{Owner: decisionsOrg})
|
||
subject := user.PayerSubject(decisionsOrg)
|
||
object.GlobalBalanceLedger.SetBalance(subject, 1)
|
||
t.Cleanup(func() { object.GlobalBalanceLedger.SetBalance(subject, 0) })
|
||
asked := strings.Replace(decisionBody, `"model":"kai"`, `"model":"typesafe/jev-1.13"`, 1)
|
||
ok := 0
|
||
for i := 0; i < 20; i++ {
|
||
if status, _ := driveDecisions(t, "Bearer "+decisionsKey, asked); status == http.StatusOK {
|
||
ok++
|
||
}
|
||
}
|
||
total := 0.0
|
||
for _, e := range *events {
|
||
f, _ := strconv.ParseFloat(e.USD, 64)
|
||
total += f
|
||
}
|
||
if ok == 0 || total > 0.01 {
|
||
t.Fatalf("a one-cent balance admitted %d Jev calls billing $%.5f", ok, total)
|
||
}
|
||
}
|
||
|
||
// A model asked in any case is priced, filed and forwarded as its route's id.
|
||
func TestDecisionCaseVariantsPriceAsTheirRoute(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
useCatalog(t, "../conf/models.yaml")
|
||
fake.answer = `{"model":"x","answers":{"q":{"type":"noul","noul":0.5}},"usage":{"input_tokens":1000000,"output_tokens":0}}`
|
||
want := map[string]string{"KAI": "0.021", "Kai": "0.021", "TYPESAFE/JEV-1.13": "0.042", "~TypeSafe/Jev-Latest": "0.042"}
|
||
for model, usd := range want {
|
||
body := strings.Replace(decisionBody, `"model":"kai"`, `"model":"`+model+`"`, 1)
|
||
if status, out := driveDecisions(t, "Bearer "+decisionsKey, body); status != 200 {
|
||
t.Fatalf("%s => %d %s", model, status, out)
|
||
}
|
||
_, _, sent := fake.seen()
|
||
e := (*events)[len(*events)-1]
|
||
if e.Model != strings.ToLower(model) || e.USD != usd || top(t, string(sent), "model") != `"`+strings.ToLower(model)+`"` {
|
||
t.Fatalf("%s: debit %s at $%s per 1M, forwarded %s", model, e.Model, e.USD, top(t, string(sent), "model"))
|
||
}
|
||
}
|
||
}
|
||
|
||
// The gateway reads the body the way the service does, key for key: a second
|
||
// spelling of a field, in any case, or a key given twice, is refused before
|
||
// anything is priced or sent.
|
||
func TestDecisionBodyIsReadKeyForKey(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
seedOther(t)
|
||
for _, body := range []string{
|
||
strings.Replace(decisionBody, `"model":"kai"`, `"model":"typesafe/jev-1.13","MODEL":"kai"`, 1),
|
||
strings.Replace(decisionBody, `"model":"kai"`, `"Model":"kai"`, 1),
|
||
`{"model":"kai","handle":"s1","handle":null}`,
|
||
`{"model":"kai","Handle":"s1"}`,
|
||
`{"model":"kai","HANDLE":"s1","handle":null}`,
|
||
`{"model":"kai","ſtate":"x","questions":{"q":{"type":"noul"}}}`,
|
||
`{"model":"kai","state":"x","questions":{"a":{"type":"noul"}},"questions":{"b":{"type":"noul"}}}`,
|
||
`{"model":"kai","state":"x","questions":{"q":{"type":"noul"}}} {"model":"typesafe/jev-1.13"}`,
|
||
} {
|
||
if status, out := driveDecisions(t, "Bearer "+otherKey, body); status != http.StatusBadRequest {
|
||
t.Errorf("%s => %d %s", body, status, out)
|
||
}
|
||
}
|
||
if calls, _, _ := fake.seen(); calls != 0 || len(*events) != 0 {
|
||
t.Fatalf("an ambiguous body reached the service %d time(s) and was billed %d", calls, len(*events))
|
||
}
|
||
}
|
||
|
||
// The usage row files under the id the caller was answered under, and keeps an id
|
||
// of its own: the caller's finds their bill, and never becomes the row's key.
|
||
func TestDecisionRowCarriesTheCallersRequestID(t *testing.T) {
|
||
_, _ = setupDecisions(t)
|
||
var mu sync.Mutex
|
||
var posted []map[string]any
|
||
commerce := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
b, _ := io.ReadAll(r.Body)
|
||
var m map[string]any
|
||
_ = json.Unmarshal(b, &m)
|
||
mu.Lock()
|
||
posted = append(posted, m)
|
||
mu.Unlock()
|
||
w.WriteHeader(http.StatusOK)
|
||
}))
|
||
t.Cleanup(commerce.Close)
|
||
object.SetUsageRecorder(nil)
|
||
prevQueue := billingQueue
|
||
billingQueue = util.NewBillingQueue(commerce.URL, "t")
|
||
t.Cleanup(func() { billingQueue.Shutdown(); billingQueue = prevQueue })
|
||
|
||
if status, _, _ := drive(t, "Bearer "+decisionsKey, decisionBody, map[string]string{"X-Request-Id": "req-42"}); status != 200 {
|
||
t.Fatalf("status %d", status)
|
||
}
|
||
deadline := time.Now().Add(5 * time.Second)
|
||
for {
|
||
mu.Lock()
|
||
n := len(posted)
|
||
mu.Unlock()
|
||
if n > 0 || time.Now().After(deadline) {
|
||
break
|
||
}
|
||
time.Sleep(10 * time.Millisecond)
|
||
}
|
||
mu.Lock()
|
||
defer mu.Unlock()
|
||
if len(posted) != 1 || posted[0]["clientRequestId"] != "req-42" || posted[0]["requestId"] == "req-42" || posted[0]["requestId"] == "" {
|
||
t.Fatalf("usage posted = %v", posted)
|
||
}
|
||
row := cloudUsageValues(&usageRecord{RequestID: "own", ClientRequestID: "req-42"}, time.Now())
|
||
if row[0] != "own" {
|
||
t.Fatalf("the row's own id is %v, want the one it minted", row[0])
|
||
}
|
||
shown := false
|
||
for _, v := range row {
|
||
if v == "req-42" {
|
||
shown = true
|
||
}
|
||
}
|
||
if !shown {
|
||
t.Fatal("the row does not carry the caller's request id")
|
||
}
|
||
}
|
||
|
||
// A recorder failure is an error recordUsage returns, not only a log line.
|
||
func TestRecordUsageReturnsTheLedgersRefusal(t *testing.T) {
|
||
_, _ = setupDecisions(t)
|
||
refusal := errors.New("ledger down")
|
||
object.SetUsageRecorder(func(context.Context, object.UsageEvent) error { return refusal })
|
||
rec := &usageRecord{Owner: decisionsOrg, Model: "kai", Provider: object.KaiName, PromptTokens: 10, DecisionCount: 1, Status: "success", RequestID: "r"}
|
||
if err := recordUsage(rec); !errors.Is(err, refusal) {
|
||
t.Fatalf("recordUsage = %v, want the recorder's refusal", err)
|
||
}
|
||
}
|
||
|
||
// A body past the bound the gateway and the service share is refused by the gateway
|
||
// in the service's shape — 422 request_too_long, under a request id — and nothing is
|
||
// sent.
|
||
func TestDecisionBodyPastTheBound(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
big := `{"model":"kai","state":"` + strings.Repeat("x", decisionBodyBytes) + `","questions":{"q":{"type":"noul"}}}`
|
||
status, body, c := drive(t, "Bearer "+decisionsKey, big, nil)
|
||
var native struct {
|
||
Error struct{ Code, Message string } `json:"error"`
|
||
}
|
||
if status != 422 || json.Unmarshal([]byte(body), &native) != nil || native.Error.Code != "request_too_long" || replied(c, "X-Request-Id") == "" {
|
||
t.Fatalf("/v1/decisions => %d %s", status, body)
|
||
}
|
||
// The bound costs a length and is the one thing asked before the credential.
|
||
if status, _ := driveDecisions(t, "Bearer sk-nobody-issued-this", big); status != 422 {
|
||
t.Fatalf("an unauthenticated oversized body => %d, want 422", status)
|
||
}
|
||
gw, _ := lookupGatewayHandler(decisionsPath)
|
||
msg, _ := gw(context.Background(), "Bearer "+decisionsKey, []byte(big))
|
||
if st := msg.Root().Uint32(object.GatewayRespStatus); st != 422 {
|
||
t.Fatalf("ZAP => %d", st)
|
||
}
|
||
if calls, _, _ := fake.seen(); calls != 0 || len(*events) != 0 {
|
||
t.Fatalf("an oversized body reached the service %d time(s), billed %d", calls, len(*events))
|
||
}
|
||
}
|
||
|
||
// Kai's versioned id — kai- and the first 12 lowercase hex of the served weights'
|
||
// sha256 — is Kai: priced and filed as kai, sent to the service as asked. Anything
|
||
// else of that shape is an unknown model and is sent nowhere.
|
||
func TestKaiVersionedID(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
const version = "kai-0834a74f2d14"
|
||
asked := strings.Replace(decisionBody, `"model":"kai"`, `"model":"`+version+`"`, 1)
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, asked); status != 200 {
|
||
t.Fatalf("%s => %d %s", version, status, body)
|
||
}
|
||
_, _, sent := fake.seen()
|
||
if string(sent) != asked {
|
||
t.Fatalf("the service was sent %s, want the body as asked", sent)
|
||
}
|
||
e := (*events)[len(*events)-1]
|
||
if e.Model != "kai" || e.USD != nanoToUSD(42*21) {
|
||
t.Fatalf("debit %s at $%s, want kai at 42 tokens × $0.021/M", e.Model, e.USD)
|
||
}
|
||
calls, _, _ := fake.seen()
|
||
for _, model := range []string{"kai-0834a74f2d1", "kai-0834a74f2d145", "kai-0834A74F2D14", "kai-zzzzzzzzzzzz", "KAI-0834a74f2d14", "kai-", "kai-0834a74f2d1g"} {
|
||
body := strings.Replace(decisionBody, `"model":"kai"`, `"model":"`+model+`"`, 1)
|
||
if status, out := driveDecisions(t, "Bearer "+decisionsKey, body); status != http.StatusBadRequest {
|
||
t.Errorf("%s => %d %s", model, status, out)
|
||
}
|
||
}
|
||
if now, _, _ := fake.seen(); now != calls {
|
||
t.Fatalf("a malformed versioned id reached the service %d time(s)", now-calls)
|
||
}
|
||
}
|
||
|
||
// A body is read only once its sender is known, and reading it costs one pass over
|
||
// the handful of fields a decision has: a hundred thousand keys are refused at the
|
||
// first one the service does not know, and a body behind a refused credential is
|
||
// never read at all.
|
||
func TestDecisionBodyIsCheapAndReadAfterAuth(t *testing.T) {
|
||
fake, _ := setupDecisions(t)
|
||
var sb strings.Builder
|
||
sb.WriteString(`{`)
|
||
for i := 0; i < 100_000; i++ {
|
||
if i > 0 {
|
||
sb.WriteString(",")
|
||
}
|
||
fmt.Fprintf(&sb, `"k%d":1`, i)
|
||
}
|
||
sb.WriteString(`}`)
|
||
many := sb.String()
|
||
start := time.Now()
|
||
status, body := driveDecisions(t, "Bearer "+decisionsKey, many)
|
||
if took := time.Since(start); status != http.StatusBadRequest || took > 250*time.Millisecond {
|
||
t.Fatalf("100k unknown keys => %d in %v (%s)", status, took, body)
|
||
}
|
||
|
||
var reads atomic.Int32
|
||
prev := readModel
|
||
readModel = func(body []byte) (string, string, *decisionRefusal) {
|
||
reads.Add(1)
|
||
return prev(body)
|
||
}
|
||
t.Cleanup(func() { readModel = prev })
|
||
if status, _ := driveDecisions(t, "Bearer sk-nobody-issued-this", many); status != http.StatusUnauthorized {
|
||
t.Fatalf("unauthenticated => %d, want 401", status)
|
||
}
|
||
gw, _ := lookupGatewayHandler(decisionsPath)
|
||
if msg, _ := gw(context.Background(), "Bearer sk-nobody-issued-this", []byte(many)); msg.Root().Uint32(object.GatewayRespStatus) != 401 {
|
||
t.Fatal("ZAP: unauthenticated was not 401")
|
||
}
|
||
if n := reads.Load(); n != 0 {
|
||
t.Fatalf("a body behind a refused credential was read %d time(s)", n)
|
||
}
|
||
if calls, _, _ := fake.seen(); calls != 0 {
|
||
t.Fatalf("the service saw %d call(s)", calls)
|
||
}
|
||
}
|
||
|
||
// bigAnswer is an answer billing 4M input tokens: 8.4¢ at Kai's price.
|
||
const bigAnswer = `{"id":"dec_1","model":"kai","provider":"Hanzo","answers":{"q":{"type":"noul","noul":0.5}},"usage":{"input_tokens":4000000,"output_tokens":0},"routing":{"backend":"kai","checkpoint":"k","reason":"r"},"state_hash":"s","latency_ms":1}`
|
||
|
||
// fund sets the paying org's balance, in cents.
|
||
func fund(org string, cents int64) {
|
||
user, _ := providerKeyBillingUser(&object.Provider{Owner: org})
|
||
object.GlobalBalanceLedger.SetBalance(user.PayerSubject(org), cents)
|
||
}
|
||
|
||
// A decision over a handle bills the questions observed with it, not its few-byte
|
||
// body and not the state, so it is held at its observed questions' size, then at
|
||
// what a decision over it last billed: a balance that cannot cover that is refused
|
||
// before the service is asked.
|
||
func TestHandleCallHoldsWhatItsHandleBills(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
fake.answer = bigAnswer
|
||
// The observe bills 8.4¢, its state's and its questions'; the questions alone are
|
||
// a handful of tokens.
|
||
observe := strings.Replace(decisionBody, `}}}`, `}},"observe":"big"}`, 1)
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, observe); status != 200 {
|
||
t.Fatalf("observe => %d %s", status, body)
|
||
}
|
||
fund(decisionsOrg, 1)
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, `{"model":"kai","handle":"big"}`); status != 200 {
|
||
t.Fatalf("the first decision over a handle held its state's cost: %d %s", status, body)
|
||
}
|
||
// That decision billed 8.4¢; the next is held at it, and a 1¢ balance cannot.
|
||
fund(decisionsOrg, 1)
|
||
calls, _, _ := fake.seen()
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, `{"model":"kai","handle":"big"}`); status != http.StatusPaymentRequired {
|
||
t.Fatalf("a handle billing 8.4¢ against a 1¢ balance => %d %s", status, body)
|
||
}
|
||
if now, _, _ := fake.seen(); now != calls || len(*events) != 2 {
|
||
t.Fatalf("the refused handle call reached the service or was billed")
|
||
}
|
||
// Another org's handle of the same name is its own.
|
||
if handled.cost(otherOrg, "big") != 0 || handled.cost(decisionsOrg, "big") != 4_000_000 {
|
||
t.Fatal("a handle's cost is not its own org's")
|
||
}
|
||
}
|
||
|
||
// Each org's handle costs are its own: another org remembering eight times the
|
||
// per-org bound evicts none of them, an org past the bound evicts only its own, and
|
||
// the table never holds more than handledOrgs × handledPerOrg entries.
|
||
func TestHandleCostsAreEachOrgsOwn(t *testing.T) {
|
||
fake, _ := setupDecisions(t)
|
||
fake.answer = bigAnswer
|
||
observe := strings.Replace(decisionBody, `}}}`, `}},"observe":"big"}`, 1)
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, observe); status != 200 {
|
||
t.Fatalf("observe => %d %s", status, body)
|
||
}
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, `{"model":"kai","handle":"big"}`); status != 200 {
|
||
t.Fatalf("handle => %d %s", status, body)
|
||
}
|
||
for i := 0; i < 8*handledPerOrg; i++ {
|
||
handled.note(otherOrg, fmt.Sprintf("h%d", i), 1)
|
||
}
|
||
if handled.cost(decisionsOrg, "big") != 4_000_000 {
|
||
t.Fatal("another org's handles evicted this org's")
|
||
}
|
||
fund(decisionsOrg, 1)
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, `{"model":"kai","handle":"big"}`); status != http.StatusPaymentRequired {
|
||
t.Fatalf("after another org's flood, a handle billing 8.4¢ against a 1¢ balance => %d %s", status, body)
|
||
}
|
||
handled.mu.Lock()
|
||
own := len(handled.orgs[otherOrg])
|
||
handled.mu.Unlock()
|
||
if own != handledPerOrg {
|
||
t.Fatalf("an org holds %d handle costs, want at most %d", own, handledPerOrg)
|
||
}
|
||
for i := 0; i < 2*handledOrgs; i++ {
|
||
handled.note(fmt.Sprintf("org%d", i), "h", 1)
|
||
}
|
||
handled.mu.Lock()
|
||
orgs := len(handled.orgs)
|
||
handled.mu.Unlock()
|
||
if orgs != handledOrgs {
|
||
t.Fatalf("the table holds %d orgs, want at most %d", orgs, handledOrgs)
|
||
}
|
||
}
|
||
|
||
// A record's debit carries one Ref however many times it is filed, and two records
|
||
// never share one: the key a host dedups a re-sent debit on, minted here.
|
||
func TestDebitRefIsStablePerRecord(t *testing.T) {
|
||
_, _ = setupDecisions(t)
|
||
var refs []string
|
||
fail := true
|
||
object.SetUsageRecorder(func(_ context.Context, u object.UsageEvent) error {
|
||
refs = append(refs, u.Ref)
|
||
if fail {
|
||
fail = false
|
||
return errors.New("answer lost")
|
||
}
|
||
return nil
|
||
})
|
||
rec := &usageRecord{Owner: decisionsOrg, Model: "kai", Provider: object.KaiName, PromptTokens: 10, DecisionCount: 1, Status: "success", RequestID: "same"}
|
||
_ = recordUsage(rec)
|
||
_ = recordUsage(rec)
|
||
other := *rec
|
||
other.ref = ""
|
||
_ = recordUsage(&other)
|
||
if len(refs) != 3 || refs[0] == "" || refs[0] != refs[1] || refs[2] == refs[0] {
|
||
t.Fatalf("refs = %v; want one ref across a record's retries and another for another record", refs)
|
||
}
|
||
}
|
||
|
||
// A handle id is 1 to 128 characters of A-Z, a-z, 0-9, '.', '_' and '-'. Anything
|
||
// else, a megabyte of id included, is refused 422 in the service's shape under a
|
||
// request id on both doors, before the service is asked, billed or remembered; an id at the
|
||
// bound is served and remembered under a fixed-size key.
|
||
func TestHandleIDIsBounded(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
observe := func(id string) string {
|
||
q, _ := json.Marshal(id)
|
||
return strings.Replace(decisionBody, `}}}`, `}},"observe":`+string(q)+`}`, 1)
|
||
}
|
||
var native struct {
|
||
Error struct {
|
||
Code int
|
||
Message string
|
||
} `json:"error"`
|
||
}
|
||
for i := 0; i < 48; i++ {
|
||
status, body, c := drive(t, "Bearer "+decisionsKey, observe(fmt.Sprintf("%d-", i)+strings.Repeat("A", 1<<20)), nil)
|
||
if status != 422 || json.Unmarshal([]byte(body), &native) != nil || native.Error.Code != 422 || replied(c, "X-Request-Id") == "" {
|
||
t.Fatalf("a 1 MiB id => %d %.200s", status, body)
|
||
}
|
||
}
|
||
for _, id := range []string{"", strings.Repeat("a", handleIDBytes+1), "acme/s1", "a b", "é", "a\x00b", "a%2Fb", "a\nb"} {
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, observe(id)); status != 422 {
|
||
t.Errorf("observe %q => %d %.200s", id, status, body)
|
||
}
|
||
q, _ := json.Marshal(id)
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, `{"model":"kai","handle":`+string(q)+`}`); status != 422 {
|
||
t.Errorf("handle %q => %d %.200s", id, status, body)
|
||
}
|
||
}
|
||
gw, _ := lookupGatewayHandler(decisionsPath)
|
||
msg, _ := gw(context.Background(), "Bearer "+decisionsKey, []byte(observe(strings.Repeat("A", 1<<20))))
|
||
if st := msg.Root().Uint32(object.GatewayRespStatus); st != 422 {
|
||
t.Fatalf("ZAP: a 1 MiB id => %d", st)
|
||
}
|
||
handled.mu.Lock()
|
||
held := len(handled.orgs)
|
||
handled.mu.Unlock()
|
||
if calls, _, _ := fake.seen(); calls != 0 || len(*events) != 0 || held != 0 {
|
||
t.Fatalf("refused ids reached the service %d time(s), billed %d, remembered %d", calls, len(*events), held)
|
||
}
|
||
|
||
id := strings.Repeat("Az09._-", handleIDBytes/7) + "xy"
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, observe(id)); status != 200 {
|
||
t.Fatalf("an id of %d bytes => %d %s", len(id), status, body)
|
||
}
|
||
if _, _, sent := fake.seen(); top(t, string(sent), "observe") != `"`+decisionsOrg+"/"+id+`"` {
|
||
t.Fatalf("the service was sent %s", sent)
|
||
}
|
||
if handled.cost(decisionsOrg, id) != coarseTokenEstimate([]byte(top(t, decisionBody, "questions"))) {
|
||
t.Fatal("an id at the bound was not remembered at its questions' size")
|
||
}
|
||
}
|
||
|
||
// gzipped is s as a gzip body.
|
||
func gzipped(s string) string {
|
||
var b bytes.Buffer
|
||
w, _ := gzip.NewWriterLevel(&b, gzip.BestCompression)
|
||
_, _ = w.Write([]byte(s))
|
||
_ = w.Close()
|
||
return b.String()
|
||
}
|
||
|
||
// A compressed decision body is decoded once, after its credential, and never past
|
||
// the bound: an unauthenticated caller's 64 MiB of gzip is refused 401 without being
|
||
// decoded, an authenticated one inflating past 16 MiB is 422 request_too_long, a
|
||
// coding nobody decodes is 415, and a small one is served as the
|
||
// JSON it decodes to.
|
||
func TestDecisionBodyIsDecodedOnceAfterItsCredential(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
gz := map[string]string{"Content-Encoding": "gzip"}
|
||
|
||
var reads atomic.Int32
|
||
prev := readModel
|
||
readModel = func(body []byte) (string, string, *decisionRefusal) {
|
||
reads.Add(1)
|
||
return prev(body)
|
||
}
|
||
t.Cleanup(func() { readModel = prev })
|
||
bomb := gzipped(`{"state":1,` + strings.Repeat(" ", 64<<20) + `"model":"kai"}`)
|
||
var a, b runtime.MemStats
|
||
runtime.GC()
|
||
runtime.ReadMemStats(&a)
|
||
status, body, _ := drive(t, "Bearer sk-nobody-issued-this", bomb, gz)
|
||
runtime.ReadMemStats(&b)
|
||
if status != http.StatusUnauthorized {
|
||
t.Fatalf("unauthenticated gzip => %d %s", status, body)
|
||
}
|
||
if alloc := b.TotalAlloc - a.TotalAlloc; alloc > 2<<20 {
|
||
t.Fatalf("refusing %d KiB of unauthenticated gzip allocated %d MiB: it was decoded", len(bomb)>>10, alloc>>20)
|
||
}
|
||
if n := reads.Load(); n != 0 {
|
||
t.Fatalf("a body behind a refused credential was read %d time(s)", n)
|
||
}
|
||
|
||
over := gzipped(`{"model":"kai","state":"` + strings.Repeat("x", decisionBodyBytes+1024) + `","questions":{"q":{"type":"noul"}}}`)
|
||
status, body, c := drive(t, "Bearer "+decisionsKey, over, gz)
|
||
var native struct {
|
||
Error struct{ Code, Message string } `json:"error"`
|
||
}
|
||
if status != 422 || json.Unmarshal([]byte(body), &native) != nil || native.Error.Code != "request_too_long" || replied(c, "X-Request-Id") == "" {
|
||
t.Fatalf("gzip past the bound => %d %s", status, body)
|
||
}
|
||
for coding, want := range map[string]int{"compress": 415, "gzip, gzip": 415, "gzip": 400} {
|
||
sent := gzipped(decisionBody)
|
||
if coding == "gzip" {
|
||
sent = "not gzip at all"
|
||
}
|
||
if status, body, _ := drive(t, "Bearer "+decisionsKey, sent, map[string]string{"Content-Encoding": coding}); status != want {
|
||
t.Errorf("Content-Encoding %q => %d %s, want %d", coding, status, body, want)
|
||
}
|
||
}
|
||
if calls, _, _ := fake.seen(); calls != 0 || len(*events) != 0 {
|
||
t.Fatalf("a refused body reached the service %d time(s), billed %d", calls, len(*events))
|
||
}
|
||
|
||
if status, body, _ := drive(t, "Bearer "+decisionsKey, gzipped(decisionBody), gz); status != 200 {
|
||
t.Fatalf("a gzip decision => %d %s", status, body)
|
||
}
|
||
if _, _, sent := fake.seen(); string(sent) != decisionBody || len(*events) != 1 {
|
||
t.Fatalf("the service was sent %q, billed %d", sent, len(*events))
|
||
}
|
||
}
|
||
|
||
// coded is s in coding as a client sends it, under a window of 1<<log bytes where
|
||
// the coding has one to choose.
|
||
func coded(t *testing.T, coding, s string, log int) string {
|
||
t.Helper()
|
||
var b bytes.Buffer
|
||
var w io.WriteCloser
|
||
switch coding {
|
||
case "gzip":
|
||
w, _ = gzip.NewWriterLevel(&b, gzip.BestSpeed)
|
||
case "deflate":
|
||
w, _ = zlib.NewWriterLevel(&b, zlib.BestSpeed)
|
||
case "br":
|
||
w = brotli.NewWriterOptions(&b, brotli.WriterOptions{Quality: brotli.BestSpeed, LGWin: log})
|
||
case "zstd":
|
||
zw, err := zstd.NewWriter(&b, zstd.WithWindowSize(1<<log))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
w = zw
|
||
}
|
||
if _, err := io.WriteString(w, s); err != nil || w.Close() != nil {
|
||
t.Fatalf("%s: %v", coding, err)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// windowed is a zstd frame holding s in one raw block, under a header that declares a
|
||
// window of 1<<log bytes.
|
||
func windowed(log int, s string) string {
|
||
f := []byte{0x28, 0xb5, 0x2f, 0xfd, 0x00, byte(log-10) << 3}
|
||
h := 1 | len(s)<<3 // the last block, raw, len(s) bytes
|
||
return string(append(append(f, byte(h), byte(h>>8), byte(h>>16)), s...))
|
||
}
|
||
|
||
// allocated is the heap run allocates.
|
||
func allocated(run func()) uint64 {
|
||
var a, b runtime.MemStats
|
||
runtime.GC()
|
||
runtime.ReadMemStats(&a)
|
||
run()
|
||
runtime.ReadMemStats(&b)
|
||
return b.TotalAlloc - a.TotalAlloc
|
||
}
|
||
|
||
// Every coding a decision body arrives in is read by a decoder bounded at 16 MiB,
|
||
// never at a library's defaults. A zstd frame whose header declares a window past
|
||
// the bound is refused before the window is allocated, however little it holds; at
|
||
// the bound it is served. A gzip, deflate, brotli or zstd body inflating past the
|
||
// bound is 422 request_too_long. Nothing refused reaches the service, and each
|
||
// coding's small body is served as the JSON it decodes to.
|
||
func TestDecisionBodyDecodersAreBounded(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
zst := map[string]string{"Content-Encoding": "zstd"}
|
||
past := func(status int, body string) bool {
|
||
var out struct {
|
||
Error struct{ Code string } `json:"error"`
|
||
}
|
||
return status == 422 && json.Unmarshal([]byte(body), &out) == nil && out.Error.Code == "request_too_long"
|
||
}
|
||
|
||
for _, log := range []int{28, 41} {
|
||
var status int
|
||
var body string
|
||
n := allocated(func() { status, body, _ = drive(t, "Bearer "+decisionsKey, windowed(log, decisionBody), zst) })
|
||
if !past(status, body) {
|
||
t.Fatalf("a zstd frame declaring a 2^%d-byte window => %d %s", log, status, body)
|
||
}
|
||
if n > 4<<20 {
|
||
t.Fatalf("refusing a zstd frame declaring a 2^%d-byte window allocated %d MiB", log, n>>20)
|
||
}
|
||
}
|
||
|
||
over := `{"model":"kai","state":"` + strings.Repeat("x", decisionBodyBytes) + `","questions":{"q":{"type":"noul"}}}`
|
||
for _, coding := range []string{"gzip", "deflate", "br", "zstd"} {
|
||
status, body, _ := drive(t, "Bearer "+decisionsKey, coded(t, coding, over, 22), map[string]string{"Content-Encoding": coding})
|
||
if !past(status, body) {
|
||
t.Fatalf("%s inflating past the bound => %d %s", coding, status, body)
|
||
}
|
||
}
|
||
if calls, _, _ := fake.seen(); calls != 0 || len(*events) != 0 {
|
||
t.Fatalf("a refused body reached the service %d time(s), billed %d", calls, len(*events))
|
||
}
|
||
|
||
for _, coding := range []string{"gzip", "deflate", "br", "zstd"} {
|
||
if status, body, _ := drive(t, "Bearer "+decisionsKey, coded(t, coding, decisionBody, 22), map[string]string{"Content-Encoding": coding}); status != 200 {
|
||
t.Fatalf("a %s decision => %d %s", coding, status, body)
|
||
}
|
||
if _, _, sent := fake.seen(); string(sent) != decisionBody {
|
||
t.Fatalf("%s: the service was sent %q", coding, sent)
|
||
}
|
||
}
|
||
if status, body, _ := drive(t, "Bearer "+decisionsKey, windowed(24, decisionBody), zst); status != 200 {
|
||
t.Fatalf("a zstd frame declaring a window at the bound => %d %s", status, body)
|
||
}
|
||
}
|
||
|
||
// Refusing a body that inflates past the bound costs what reading to the bound
|
||
// costs, whatever it inflates to: a body inflating to 32 times its bound, in each
|
||
// coding under a window within that bound, is refused having allocated under a
|
||
// quarter of what it inflates to — the window, and the bound read twice over, in
|
||
// chunks and then at its final size.
|
||
func TestDecodeCostsTheBound(t *testing.T) {
|
||
const limit = 1 << 20
|
||
bomb := strings.Repeat("x", 32*limit)
|
||
for _, coding := range []string{"gzip", "deflate", "br", "zstd"} {
|
||
sent := []byte(coded(t, coding, bomb, 20))
|
||
var err error
|
||
n := allocated(func() { _, err = decode(coding, sent, limit) })
|
||
if !errors.Is(err, fasthttp.ErrBodyTooLarge) {
|
||
t.Fatalf("%s inflating to %d MiB under a %d MiB bound: %v", coding, len(bomb)>>20, limit>>20, err)
|
||
}
|
||
if n > uint64(len(bomb))/4 {
|
||
t.Fatalf("%s: refusing %d MiB under a %d MiB bound allocated %d KiB", coding, len(bomb)>>20, limit>>20, n>>10)
|
||
}
|
||
}
|
||
}
|
||
|
||
// A decision model is free to an org whose route prices it at zero, and to no other:
|
||
// the balance gate refuses an empty wallet on a decision path before decoding its
|
||
// body only when none is. The org is one no other test prices under, because routes
|
||
// are cached per org for the process.
|
||
func TestADecisionModelPricedAtZeroIsFree(t *testing.T) {
|
||
routeDB(t)
|
||
if DecisionFree("gratis") {
|
||
t.Fatal("a decision model is free with no route pricing one at zero")
|
||
}
|
||
addRoute(t, &object.ModelRoute{
|
||
Owner: "gratis", ModelName: "kai", Provider: object.KaiName,
|
||
Upstream: "kai", Priced: true, InputPrice: 0, OutputPrice: 0, Enabled: true,
|
||
})
|
||
if !DecisionFree("gratis") || DecisionFree("other") {
|
||
t.Fatalf("free to gratis %v, to other %v; want true, false", DecisionFree("gratis"), DecisionFree("other"))
|
||
}
|
||
}
|
||
|
||
// The settle loop tries a debit whose answer was lost again under the same Ref, so a
|
||
// ledger that keys on it moves the money once however many tries it took.
|
||
func TestSettleRetriesCarryOneRef(t *testing.T) {
|
||
_, _ = setupDecisions(t)
|
||
prev := settleBackoff
|
||
settleBackoff = time.Millisecond
|
||
t.Cleanup(func() { settleBackoff = prev })
|
||
var mu sync.Mutex
|
||
var refs []string
|
||
object.SetUsageRecorder(func(_ context.Context, u object.UsageEvent) error {
|
||
mu.Lock()
|
||
defer mu.Unlock()
|
||
refs = append(refs, u.Ref)
|
||
if len(refs) < settleTries {
|
||
return errors.New("answer lost")
|
||
}
|
||
return nil
|
||
})
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, decisionBody); status != 200 {
|
||
t.Fatalf("decision => %d %s", status, body)
|
||
}
|
||
mu.Lock()
|
||
defer mu.Unlock()
|
||
if len(refs) != settleTries || refs[0] == "" || slices.ContainsFunc(refs, func(r string) bool { return r != refs[0] }) {
|
||
t.Fatalf("tries carried refs %v; want %d tries under one ref", refs, settleTries)
|
||
}
|
||
}
|