mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
The hold read a body at four bytes a token, but a chat model's tokenizer reads a digit, a hex character or an emoji's byte as one, and the decision service writes the body out again before the router counts it (1e15 becomes 1000000000000000.0): a 65 KB state the hold priced at 7 cents settled at 56, and a wallet went negative. A model whose output is priced is now held at a token a byte plus 1,024 for the prompt around it, and an answer is settled and debited at most its hold whoever pays, a plan included; the excess is ours, and logged. Until the service passes the state's bytes through, a number-heavy body can still cost past its hold, about $1.76 a call at worst. A token count is the service's report of what a router relayed, so each is bounded where it is priced: none below zero, output at the 4,096 a completion is asked for, input short of where count times price overflows. A reported 1e9 output tokens billed $7,260 on a 3-cent hold, and 1.3e15 wrapped to a negative debit. An answer whose usage does not read as two whole token counts (and a cost, where stated, a number of 0 or more) is refused 502 and billed nothing, whatever model answered, rather than billed at zero for the tokens it did not state. The service writes usage from integer fields, so only a broken service reaches it, and the loss is the one any refusal already accepts. Kai's and Jev's bills and holds are unchanged for every count: their output is priced at zero and the byte-rate hold and the cap apply to priced output alone.
762 lines
31 KiB
Go
762 lines
31 KiB
Go
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
|
||
//
|
||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||
// you may not use this file except in compliance with the License.
|
||
// You may obtain a copy of the License at
|
||
//
|
||
// http://www.apache.org/licenses/LICENSE-2.0
|
||
//
|
||
// Unless required by applicable law or agreed to in writing, software
|
||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
// See the License for the specific language governing permissions and
|
||
// limitations under the License.
|
||
|
||
package controllers
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"slices"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/hanzoai/ai/object"
|
||
)
|
||
|
||
// POST /v1/decisions against a fake decision service. The vehicle is an sk-
|
||
// provider key: the one credential that carries a billable principal with no
|
||
// IAM round trip, exactly as the chat money-path suite drives it.
|
||
|
||
const (
|
||
decisionsKey = "sk-decisions-funded"
|
||
decisionsOrg = "acme"
|
||
)
|
||
|
||
// decisionAnswer is a Decisions response as the service writes it.
|
||
const decisionAnswer = `{"id":"dec_1","model":"kai","provider":"Hanzo","answers":{"is_bug":{"type":"noul","noul":0.96,"answer_confidence":0.96}},"usage":{"input_tokens":42,"output_tokens":0,"cost":0.00002},"routing":{"backend":"kai","checkpoint":"hanzoai/kai","reason":"explicit model='kai'"},"state_hash":"sha256:00","latency_ms":1.5}`
|
||
|
||
const decisionBody = `{"model":"kai","state":{"message":"Payment failed twice"},"questions":{"is_bug":{"type":"noul","instructions":"Is this a bug?"}}}`
|
||
|
||
// decisionsServed is every model /v1/decisions serves, as its 400 names them.
|
||
const decisionsServed = "kai, typesafe/jev-1.13, zenlm/zen6, zenlm/zen6-flash, ~typesafe/jev-latest"
|
||
|
||
var decisionsSeq atomic.Int64
|
||
|
||
// fakeDecisions is the decision service: it records what reached it and answers
|
||
// with status, body and header — or, when answer is set, with what it says.
|
||
type fakeDecisions struct {
|
||
mu sync.Mutex
|
||
calls int
|
||
path string
|
||
body []byte
|
||
rid string
|
||
org string // X-Org-Id
|
||
set string // X-Org-Capabilities
|
||
// capture is the X-Capture header the last call carried.
|
||
capture string
|
||
status int
|
||
answer string
|
||
header map[string]string
|
||
serve func(path string, body []byte) (int, string)
|
||
}
|
||
|
||
func (f *fakeDecisions) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||
b, _ := io.ReadAll(r.Body)
|
||
f.mu.Lock()
|
||
f.calls++
|
||
f.path, f.body, f.rid, f.org = r.URL.Path, b, r.Header.Get("X-Request-Id"), r.Header.Get("X-Org-Id")
|
||
f.capture = r.Header.Get("X-Capture")
|
||
f.set = r.Header.Get("X-Org-Capabilities")
|
||
status, answer, serve := f.status, f.answer, f.serve
|
||
for k, v := range f.header {
|
||
w.Header().Set(k, v)
|
||
}
|
||
f.mu.Unlock()
|
||
if serve != nil {
|
||
status, answer = serve(r.URL.Path, b)
|
||
}
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(status)
|
||
_, _ = w.Write([]byte(answer))
|
||
}
|
||
|
||
func (f *fakeDecisions) seen() (int, string, []byte) {
|
||
f.mu.Lock()
|
||
defer f.mu.Unlock()
|
||
return f.calls, f.path, f.body
|
||
}
|
||
|
||
// settled waits for every debit filed after its reply.
|
||
func settled(t *testing.T) {
|
||
t.Helper()
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
if err := Settled(ctx); err != nil {
|
||
t.Fatalf("debits filed after their reply did not land: %v", err)
|
||
}
|
||
}
|
||
|
||
// setupDecisions installs a funded provider-key principal, a fake decision
|
||
// service at KAI_URL, and a recorder for every debit, and restores all of it.
|
||
func setupDecisions(t *testing.T) (*fakeDecisions, *[]object.UsageEvent) {
|
||
t.Helper()
|
||
forget := func() {
|
||
decisionCache.mu.Lock()
|
||
decisionCache.lru, decisionCache.m, decisionCache.orgs, decisionCache.rev, decisionCache.size = nil, nil, nil, nil, 0
|
||
decisionCache.mu.Unlock()
|
||
}
|
||
forget()
|
||
t.Cleanup(forget)
|
||
handled.mu.Lock()
|
||
handled.orgs = nil
|
||
handled.mu.Unlock()
|
||
dsn := fmt.Sprintf("file:decisions_%d?mode=memory&cache=shared", decisionsSeq.Add(1))
|
||
restore, err := object.UseMemoryDB(dsn, &object.Provider{})
|
||
if err != nil {
|
||
t.Fatalf("UseMemoryDB: %v", err)
|
||
}
|
||
t.Cleanup(restore)
|
||
if _, err := object.AddProvider(&object.Provider{
|
||
Owner: decisionsOrg, Name: "acme-openai", Category: "Model", Type: "OpenAI",
|
||
ProviderKey: decisionsKey, ProviderUrl: "http://127.0.0.1:1/v1",
|
||
ClientSecret: "upstream-secret", State: "Active",
|
||
}); err != nil {
|
||
t.Fatalf("seed provider: %v", err)
|
||
}
|
||
|
||
fake := &fakeDecisions{status: http.StatusOK, answer: decisionAnswer}
|
||
srv := httptest.NewServer(fake)
|
||
t.Cleanup(srv.Close)
|
||
t.Setenv("KAI_URL", srv.URL)
|
||
t.Setenv("KAI_API_KEY", "")
|
||
|
||
prevBalance := object.BalanceReader()
|
||
object.SetBalanceReader(balReader(100_00, nil))
|
||
t.Cleanup(func() { object.SetBalanceReader(prevBalance) })
|
||
// The in-process ledger outlives a test; each one starts both tenants funded.
|
||
for _, org := range []string{decisionsOrg, "globex"} {
|
||
user, _ := providerKeyBillingUser(&object.Provider{Owner: org})
|
||
object.GlobalBalanceLedger.SetBalance(user.PayerSubject(org), 100_00)
|
||
}
|
||
|
||
var mu sync.Mutex
|
||
events := &[]object.UsageEvent{}
|
||
prevUsage := object.UsageRecorder()
|
||
object.SetUsageRecorder(func(_ context.Context, u object.UsageEvent) error {
|
||
mu.Lock()
|
||
defer mu.Unlock()
|
||
*events = append(*events, u)
|
||
return nil
|
||
})
|
||
t.Cleanup(func() { settled(t); object.SetUsageRecorder(prevUsage) })
|
||
return fake, events
|
||
}
|
||
|
||
// driveDecisions runs the real Decisions handler, waits for the debit it filed
|
||
// after replying, and returns its status and body.
|
||
func driveDecisions(t *testing.T, authorization, body string) (int, string) {
|
||
t.Helper()
|
||
status, body, _ := drive(t, authorization, body, nil)
|
||
return status, body
|
||
}
|
||
|
||
// drive runs the Decisions handler with the given request headers, waits for the
|
||
// debit it filed after replying, and returns its status, body and reply.
|
||
func drive(t *testing.T, authorization, body string, header map[string]string) (int, string, *ApiController) {
|
||
t.Helper()
|
||
c := presenting(visit(http.MethodPost, decisionsPath), authorization)
|
||
for k, v := range header {
|
||
c.Fiber().Request().Header.Set(k, v)
|
||
}
|
||
c.Fiber().Request().SetBody([]byte(body))
|
||
status := answering(t, c, c.Decisions)
|
||
settled(t)
|
||
return status, sent(c), c
|
||
}
|
||
|
||
// replied is a header the handler set on its reply.
|
||
func replied(c *ApiController, name string) string {
|
||
return string(c.Fiber().Response().Header.Peek(name))
|
||
}
|
||
|
||
// The body reaches the service unchanged, the answer comes back unchanged, and
|
||
// the call is debited once, at $0.021 per million input tokens. Output is free.
|
||
func TestDecisionsForwardsAndMeters(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
|
||
status, body := driveDecisions(t, "Bearer "+decisionsKey, decisionBody)
|
||
if status != http.StatusOK {
|
||
t.Fatalf("status = %d, want 200 (body %s)", status, body)
|
||
}
|
||
if body != decisionAnswer {
|
||
t.Fatalf("answer changed on the way back:\n got %s\nwant %s", body, decisionAnswer)
|
||
}
|
||
calls, path, sentBody := fake.seen()
|
||
if calls != 1 || path != "/v1/decisions" {
|
||
t.Fatalf("service saw %d call(s) at %q, want 1 at /v1/decisions", calls, path)
|
||
}
|
||
if string(sentBody) != decisionBody {
|
||
t.Fatalf("body changed on the way out:\n got %s\nwant %s", sentBody, decisionBody)
|
||
}
|
||
|
||
if len(*events) != 1 {
|
||
t.Fatalf("debits = %d, want exactly 1", len(*events))
|
||
}
|
||
e := (*events)[0]
|
||
if e.Model != "kai" || e.Provider != object.KaiName {
|
||
t.Fatalf("debit names %s/%s, want kai/kai", e.Provider, e.Model)
|
||
}
|
||
if e.USD != nanoToUSD(42*21) {
|
||
t.Fatalf("debit = $%s, want 42 input tokens at $0.021/M", e.USD)
|
||
}
|
||
if e.Namespace != decisionsOrg {
|
||
t.Fatalf("debit lands on %q, want the key's org %q", e.Namespace, decisionsOrg)
|
||
}
|
||
}
|
||
|
||
// A model whose route names another id upstream reaches the service under that id,
|
||
// and the answer and the debit name the model asked for, at its own row. Its alias
|
||
// is neither listed nor a route of its own: it names that model.
|
||
func TestDecisionsServeTheModelAskedFor(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
path := filepath.Join(t.TempDir(), "models.yaml")
|
||
catalog := `version: 1
|
||
models:
|
||
hanzo/kai:
|
||
provider: kai
|
||
upstream: kai
|
||
owned_by: hanzo
|
||
outputs: [decision]
|
||
pricing: {input: 0.021, output: 0}
|
||
hanzoai/kai:
|
||
alias_of: hanzo/kai
|
||
`
|
||
if err := os.WriteFile(path, []byte(catalog), 0o600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
useCatalog(t, path)
|
||
|
||
if id, ok := Canonical("HanzoAI/Kai"); !ok || id != "hanzo/kai" {
|
||
t.Fatalf("Canonical(hanzoai/kai) = %q, %v; want hanzo/kai", id, ok)
|
||
}
|
||
if id, ok := Canonical("hanzo/kai"); ok {
|
||
t.Fatalf("hanzo/kai is not an alias, yet names %q", id)
|
||
}
|
||
if r := resolveModelRoute("hanzoai/kai"); r != nil {
|
||
t.Fatalf("an alias is a route of its own: %+v", r)
|
||
}
|
||
listed := map[string]bool{}
|
||
for _, m := range listAvailableModels() {
|
||
listed[m.ID] = true
|
||
}
|
||
if !listed["hanzo/kai"] || listed["hanzoai/kai"] {
|
||
t.Fatalf("listed hanzo/kai=%v hanzoai/kai=%v, want the model listed and its alias not", listed["hanzo/kai"], listed["hanzoai/kai"])
|
||
}
|
||
|
||
asked := strings.Replace(decisionBody, `"model":"kai"`, `"model":"hanzo/kai"`, 1)
|
||
status, body := driveDecisions(t, "Bearer "+decisionsKey, asked)
|
||
if status != http.StatusOK {
|
||
t.Fatalf("status = %d, want 200 (body %s)", status, body)
|
||
}
|
||
var answer, served map[string]json.RawMessage
|
||
if err := json.Unmarshal([]byte(body), &answer); err != nil || string(answer["model"]) != `"hanzo/kai"` {
|
||
t.Fatalf("answer names %s, want hanzo/kai (body %s)", answer["model"], body)
|
||
}
|
||
var sent, in map[string]json.RawMessage
|
||
_, _, raw := fake.seen()
|
||
if err := json.Unmarshal(raw, &sent); err != nil || string(sent["model"]) != `"kai"` {
|
||
t.Fatalf("service was sent %s, want kai", raw)
|
||
}
|
||
_ = json.Unmarshal([]byte(asked), &in)
|
||
_ = json.Unmarshal([]byte(decisionAnswer), &served)
|
||
for _, k := range []string{"state", "questions"} {
|
||
if string(sent[k]) != string(in[k]) {
|
||
t.Fatalf("%s changed on the way out: %s", k, sent[k])
|
||
}
|
||
}
|
||
for _, k := range []string{"answers", "usage", "routing"} {
|
||
if string(answer[k]) != string(served[k]) {
|
||
t.Fatalf("%s changed on the way back: %s", k, answer[k])
|
||
}
|
||
}
|
||
|
||
if len(*events) != 1 {
|
||
t.Fatalf("debits = %d, want exactly 1", len(*events))
|
||
}
|
||
if e := (*events)[0]; e.Model != "hanzo/kai" || e.USD != nanoToUSD(42*21) {
|
||
t.Fatalf("debit = %s at $%s, want hanzo/kai at 42 input tokens × $0.021/M", e.Model, e.USD)
|
||
}
|
||
}
|
||
|
||
// The row carries the answer's usage: input tokens set the price, output tokens
|
||
// do not, and the stated upstream cost is COGS.
|
||
func TestDecisionRecordMetersUsage(t *testing.T) {
|
||
cost := 0.00002
|
||
u := decisionsUsage{InputTokens: 42, OutputTokens: 3, Cost: &cost}
|
||
var answer decisionsResponse
|
||
if err := json.Unmarshal([]byte(decisionAnswer), &answer); err != nil {
|
||
t.Fatalf("the declared response does not read the service's answer: %v", err)
|
||
}
|
||
if answer.Usage.InputTokens != 42 || answer.Usage.Cost == nil || *answer.Usage.Cost != cost {
|
||
t.Fatalf("usage read from the answer = %+v", answer.Usage)
|
||
}
|
||
|
||
rec := &usageRecord{Model: "kai", Provider: object.KaiName, PromptTokens: u.InputTokens,
|
||
CompletionTokens: u.OutputTokens, DecisionCount: 1}
|
||
want := int64(u.InputTokens) * 21
|
||
if got := usageCostNano(rec); got != want {
|
||
t.Fatalf("cost = %d nano, want %d (42 input tokens at $0.021/M)", got, want)
|
||
}
|
||
rec.CompletionTokens = 1_000_000
|
||
if got := usageCostNano(rec); got != want {
|
||
t.Fatalf("output tokens moved the price to %d nano", got)
|
||
}
|
||
rec.PromptTokens = 1_000_000
|
||
rec.CompletionTokens = u.OutputTokens
|
||
if got := usageCostNano(rec); got != 21_000_000 {
|
||
t.Fatalf("a million input tokens cost %d nano, want $0.021", got)
|
||
}
|
||
rec.PromptTokens = u.InputTokens
|
||
if recordUnpriced(rec) {
|
||
t.Fatal("a decision is priced on its input tokens; it must not read as unpriced")
|
||
}
|
||
stated := usdToNano(cost)
|
||
rec.CostNanoExact = &stated
|
||
if m := usageMargin(rec); m.MarginNano == nil || *m.MarginNano != want-stated {
|
||
t.Fatalf("margin = %v, want price − stated cost", m.MarginNano)
|
||
}
|
||
}
|
||
|
||
// A decision is priced from the model price table like every model: the row the
|
||
// loaded config states, a live catalog refresh included, sets the price. Jev is a
|
||
// row like Kai's, at its list price.
|
||
func TestDecisionPriceIsTheTableRow(t *testing.T) {
|
||
prev := globalModelConfig
|
||
globalModelConfig = &ModelConfig{
|
||
routes: map[string]modelRoute{},
|
||
pricing: map[string]modelPrice{"kai": {InputPerMillion: 0.042}, "typesafe/jev-1.13": {InputPerMillion: 0.084}},
|
||
defaults: modelPrice{InputPerMillion: 1, OutputPerMillion: 4},
|
||
stopCh: make(chan struct{}),
|
||
}
|
||
t.Cleanup(func() { globalModelConfig = prev })
|
||
|
||
rec := &usageRecord{Model: "kai", Provider: object.KaiName, PromptTokens: 1000, CompletionTokens: 50, DecisionCount: 1}
|
||
if got := usageCostNano(rec); got != 1000*42 {
|
||
t.Fatalf("cost = %d nano, want 1000 input tokens at the table's $0.042/M", got)
|
||
}
|
||
if recordUnpriced(rec) {
|
||
t.Fatal("kai has a row in the table; it must not read as unpriced")
|
||
}
|
||
rec.Model = "typesafe/jev-1.13"
|
||
if got := usageCostNano(rec); got != 1000*84 {
|
||
t.Fatalf("jev = %d nano, want 1000 input tokens at the table's $0.084/M", got)
|
||
}
|
||
if recordUnpriced(rec) {
|
||
t.Fatal("jev has a row in the table; it must not read as unpriced")
|
||
}
|
||
rec.Model = "laya"
|
||
if !recordUnpriced(rec) {
|
||
t.Fatal("a decision model with no row reads as unpriced")
|
||
}
|
||
}
|
||
|
||
// The shipped catalog and the static table say one price each: Kai $0.021 and Jev
|
||
// its list $0.042 per million input tokens, output free.
|
||
func TestDecisionPricesAgree(t *testing.T) {
|
||
want := map[string]float64{"kai": 0.021, "typesafe/jev-1.13": 0.042, "~typesafe/jev-latest": 0.042}
|
||
for model, in := range want {
|
||
if p := modelPricing[model]; p.InputPerMillion != in || p.OutputPerMillion != 0 {
|
||
t.Errorf("static %s = %+v, want $%v/M in, free out", model, p, in)
|
||
}
|
||
}
|
||
useCatalog(t, "../conf/models.yaml")
|
||
for model, in := range want {
|
||
if p := getModelPrice(model); p.InputPerMillion != in || p.OutputPerMillion != 0 {
|
||
t.Errorf("models.yaml %s = %+v, want $%v/M in, free out", model, p, in)
|
||
}
|
||
rec := &usageRecord{Model: model, Provider: object.KaiName, PromptTokens: 1_000_000, DecisionCount: 1}
|
||
if got := usageCostNano(rec); got != int64(in*1e9) {
|
||
t.Errorf("%s: a million input tokens bill %d nano, want $%v", model, got, in)
|
||
}
|
||
}
|
||
// Zen answers with a completion, so both legs carry a price, the same in both.
|
||
for _, model := range []string{"zenlm/zen6", "zenlm/zen6-flash"} {
|
||
static, p := modelPricing[model], getModelPrice(model)
|
||
if p.InputPerMillion != 2.28 || p.OutputPerMillion != 7.26 || static.InputPerMillion != p.InputPerMillion || static.OutputPerMillion != p.OutputPerMillion {
|
||
t.Errorf("%s: models.yaml %+v, static %+v; want $2.28 in and $7.26 out per million in both", model, p, static)
|
||
}
|
||
}
|
||
}
|
||
|
||
// An LLM model answers a decision with a completion, and the completion is billed:
|
||
// its output tokens at the model's output price beside its input. The body reaches
|
||
// the service as written, a boolean still a boolean, under the id asked. The call is
|
||
// held at the most output an answer can report, so a balance that covers less is
|
||
// refused before the service is asked, while Kai, whose output is free, is held at
|
||
// its cent as before. Jev reports output tokens, and they bill nothing.
|
||
func TestAnLLMDecisionBillsItsCompletion(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
const zen = `{"id":"dec_3","model":"zenlm/zen6","provider":"Hanzo","answers":{"is_bug":{"type":"boolean","probability":1}},"usage":{"input_tokens":319,"output_tokens":1400},"routing":{"backend":"llm","checkpoint":"zen6","reason":"r"},"state_hash":"s","latency_ms":1}`
|
||
const jev = `{"id":"dec_2","model":"typesafe/jev-1.13","provider":"OpenRouter","answers":{"is_bug":{"type":"noul","noul":0.96}},"usage":{"input_tokens":42,"output_tokens":70,"cost":0.000019992}}`
|
||
fake.serve = func(_ string, body []byte) (int, string) {
|
||
var b struct{ Model string }
|
||
_ = json.Unmarshal(body, &b)
|
||
switch b.Model {
|
||
case "zenlm/zen6":
|
||
return http.StatusOK, zen
|
||
case "typesafe/jev-1.13":
|
||
return http.StatusOK, jev
|
||
}
|
||
return http.StatusOK, decisionAnswer
|
||
}
|
||
asked := `{"model":"zenlm/zen6","state":"I was charged twice.","questions":{"is_bug":{"type":"boolean","instructions":"Is this a bug?"}}}`
|
||
cents := (decisionCostNano("zenlm/zen6", len(asked)+decisionPromptTokens, decisionOutputTokens) + nanoPerCent - 1) / nanoPerCent
|
||
if cents < 2 {
|
||
t.Fatalf("zenlm/zen6 holds %d cents; its output ceiling must hold more than Kai's cent", cents)
|
||
}
|
||
|
||
fund(decisionsOrg, cents-1)
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, asked); status != http.StatusPaymentRequired {
|
||
t.Fatalf("zenlm/zen6 against a balance under its output ceiling => %d %s, want 402", status, body)
|
||
}
|
||
if calls, _, _ := fake.seen(); calls != 0 || len(*events) != 0 {
|
||
t.Fatalf("a refused hold reached the service (%d calls) or was billed (%d debits)", calls, len(*events))
|
||
}
|
||
fund(decisionsOrg, cents-1)
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, decisionBody); status != http.StatusOK {
|
||
t.Fatalf("kai against the same balance => %d %s, want 200: its output is free", status, body)
|
||
}
|
||
|
||
fund(decisionsOrg, cents)
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, asked); status != http.StatusOK || body != zen {
|
||
t.Fatalf("zenlm/zen6 against its output ceiling => %d %s", status, body)
|
||
}
|
||
if _, _, sent := fake.seen(); string(sent) != asked {
|
||
t.Fatalf("zenlm/zen6 was sent\n%s\nwant the body as written", sent)
|
||
}
|
||
// The hold settles at what the answer spent: 1.09¢, its output nearly all of it,
|
||
// so a whole cent leaves the balance and the rest is held against it.
|
||
user, _ := providerKeyBillingUser(&object.Provider{Owner: decisionsOrg})
|
||
if left, _ := object.GlobalBalanceLedger.Available(user.PayerSubject(decisionsOrg)); left != cents-2 {
|
||
t.Fatalf("after a 1.09¢ answer %d of %d cents are left, want %d", left, cents, cents-2)
|
||
}
|
||
|
||
fund(decisionsOrg, 100_00)
|
||
jevAsked := strings.Replace(decisionBody, `"model":"kai"`, `"model":"typesafe/jev-1.13"`, 1)
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, jevAsked); status != http.StatusOK || body != jev {
|
||
t.Fatalf("jev => %d %s", status, body)
|
||
}
|
||
|
||
want := []struct{ model, usd string }{
|
||
{"kai", nanoToUSD(42 * 21)},
|
||
{"zenlm/zen6", nanoToUSD(319*2280 + 1400*7260)},
|
||
{"typesafe/jev-1.13", nanoToUSD(42 * 42)},
|
||
}
|
||
if len(*events) != len(want) {
|
||
t.Fatalf("debits = %+v, want %d", *events, len(want))
|
||
}
|
||
for i, w := range want {
|
||
if e := (*events)[i]; e.Model != w.model || e.USD != w.usd {
|
||
t.Errorf("debit %d = %s at $%s, want %s at $%s", i, e.Model, e.USD, w.model, w.usd)
|
||
}
|
||
}
|
||
}
|
||
|
||
// zenlm/zen6 begins with "zen", and zen6, the Zen chat SKU it names, lists at $0.
|
||
// Neither makes the decision the Zen family's or free: beside a Zen family that
|
||
// lists zen6 at zero and even names zenlm/zen6 its alias, the decision id routes
|
||
// to the decision service at its own row, in no family, priced, and zen6 still
|
||
// chats through Zen, its listing no longer claiming the decision id as its slug.
|
||
func TestNoFamilyPricesOrNamesADecision(t *testing.T) {
|
||
zenServiceAt(t, false)
|
||
zenFam.mu.Lock()
|
||
zenFam.named = map[string]string{"zenlm/zen6": "zen6", "zenlm/zen6-flash": "zen6"}
|
||
zenFam.mu.Unlock()
|
||
useCatalog(t, "../conf/models.yaml")
|
||
if p, ok := familyModelPrice("zenlm/zen6"); !ok || p.InputPerMillion != 0 || p.OutputPerMillion != 0 {
|
||
t.Fatalf("the family prices zenlm/zen6 at %+v, %v; the test needs it to claim the id at zero", p, ok)
|
||
}
|
||
if r := resolveModelRoute("zen6"); r == nil || r.providerName != zenFam.name {
|
||
t.Fatalf("zen6 routes to %+v, want the Zen family", r)
|
||
}
|
||
for _, id := range []string{"zenlm/zen6", "zenlm/zen6-flash"} {
|
||
if r := resolveModelRoute(id); r == nil || r.providerName != object.KaiName || r.upstreamModel != id {
|
||
t.Errorf("%s routes to %+v, want the decision service under its own id", id, r)
|
||
}
|
||
if model, version, bad := decisionModel([]byte(`{"model":"` + strings.ToUpper(id) + `"}`)); bad != nil || model != id || version != "" {
|
||
t.Errorf("%s reads as %q %q %v", id, model, version, bad)
|
||
}
|
||
if p, ok := getModelPriceForOrgOK(id, decisionsOrg); !ok || p.InputPerMillion != 2.28 || p.OutputPerMillion != 7.26 {
|
||
t.Errorf("%s is priced %+v, %v; want its own row", id, p, ok)
|
||
}
|
||
if costsNothing(id, decisionsOrg) || FamilyOf(id) != "" || ClassOf(id) != object.ClassOurs {
|
||
t.Errorf("%s: free %v, family %q, class %q; want priced, no family, ours",
|
||
id, costsNothing(id, decisionsOrg), FamilyOf(id), ClassOf(id))
|
||
}
|
||
}
|
||
if DecisionFree(decisionsOrg) {
|
||
t.Fatal("a decision model reads free")
|
||
}
|
||
listed := map[string]modelInfo{}
|
||
for _, m := range listAvailableModels() {
|
||
listed[m.ID] = m
|
||
}
|
||
if zen6, ok := listed["zen6"]; !ok || strings.EqualFold(zen6.CanonicalSlug, "zenlm/zen6") {
|
||
t.Fatalf("zen6 listed %v with slug %q; want it listed, not claiming the decision id", ok, zen6.CanonicalSlug)
|
||
}
|
||
if m := listed["zenlm/zen6"]; m.Family != "kai" || m.Free {
|
||
t.Fatalf("zenlm/zen6 lists in family %q, free %v; want kai, priced", m.Family, m.Free)
|
||
}
|
||
}
|
||
|
||
// No credential, or one nobody issued, is 401 — and nothing reaches the service.
|
||
func TestDecisionsAuthRequired(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
for _, auth := range []string{"", "Bearer ", "Bearer sk-nobody-issued-this", "Bearer pk-publishable"} {
|
||
status, _ := driveDecisions(t, auth, decisionBody)
|
||
if status != http.StatusUnauthorized && status != http.StatusForbidden {
|
||
t.Errorf("%q => %d, want 401/403", auth, status)
|
||
}
|
||
}
|
||
if calls, _, _ := fake.seen(); calls != 0 {
|
||
t.Fatalf("service saw %d call(s) from unauthenticated callers", calls)
|
||
}
|
||
if len(*events) != 0 {
|
||
t.Fatalf("unauthenticated callers were debited %d time(s)", len(*events))
|
||
}
|
||
}
|
||
|
||
// A model this endpoint does not publish is refused in the service's error shape,
|
||
// naming what it does serve, without a call. Laya is the benchmark baseline the
|
||
// service can load; it is not published, so it is unknown here.
|
||
func TestDecisionsUnknownModel(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
for _, model := range []string{"laya", "laya-agent", "gpt-4o", "zen5", "bge-m3", "zen-scribe"} {
|
||
status, body := driveDecisions(t, "Bearer "+decisionsKey,
|
||
`{"model":"`+model+`","state":"x","questions":{"q":{"type":"noul","instructions":"?"}}}`)
|
||
if status != http.StatusBadRequest {
|
||
t.Fatalf("%s => %d, want 400", model, status)
|
||
}
|
||
want := string(decisionsFailure(400, fmt.Sprintf("unknown model %q; use one of %s", model, decisionsServed)))
|
||
if body != want {
|
||
t.Fatalf("%s refusal:\n got %s\nwant %s", model, body, want)
|
||
}
|
||
}
|
||
// A body with no model says so, in the same shape.
|
||
if status, body := driveDecisions(t, "Bearer "+decisionsKey, `{"state":"x"}`); status != 400 ||
|
||
body != `{"error":{"code":400,"message":"the request needs a 'model'"}}` {
|
||
t.Fatalf("no model => %d %s", status, body)
|
||
}
|
||
// Authentication still comes first.
|
||
if status, _ := driveDecisions(t, "Bearer sk-nobody-issued-this", `{"model":"laya"}`); status != http.StatusUnauthorized {
|
||
t.Fatalf("unauthenticated unknown model => %d, want 401", status)
|
||
}
|
||
if calls, _, _ := fake.seen(); calls != 0 {
|
||
t.Fatalf("service saw %d call(s) for models it must never be sent", calls)
|
||
}
|
||
if len(*events) != 0 {
|
||
t.Fatalf("refused calls were debited %d time(s)", len(*events))
|
||
}
|
||
}
|
||
|
||
// The service's own refusal comes back unchanged and is not charged.
|
||
func TestDecisionsServiceRefusalPassesThrough(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
fake.status = http.StatusUnprocessableEntity
|
||
fake.answer = `{"error":{"code":422,"message":"question 'c' options exceed head_max_len=16"}}`
|
||
|
||
status, body := driveDecisions(t, "Bearer "+decisionsKey, decisionBody)
|
||
if status != http.StatusUnprocessableEntity || body != fake.answer {
|
||
t.Fatalf("refusal => %d %s, want 422 %s", status, body, fake.answer)
|
||
}
|
||
if len(*events) != 0 {
|
||
t.Fatalf("a refused decision was debited %d time(s)", len(*events))
|
||
}
|
||
}
|
||
|
||
// With no decision service configured the answer is a 503 in the service's shape.
|
||
func TestDecisionsUnconfigured(t *testing.T) {
|
||
setupDecisions(t)
|
||
t.Setenv("KAI_URL", "")
|
||
status, body := driveDecisions(t, "Bearer "+decisionsKey, decisionBody)
|
||
if status != http.StatusServiceUnavailable ||
|
||
body != `{"error":{"code":503,"message":"the decision service is not configured"}}` {
|
||
t.Fatalf("unconfigured => %d %s", status, body)
|
||
}
|
||
}
|
||
|
||
// The ZAP twin: 401 with no credential, the same refusal for an unknown model, and
|
||
// the answer forwarded unchanged.
|
||
func TestZapDecisions(t *testing.T) {
|
||
fake, _ := setupDecisions(t)
|
||
if _, ok := lookupGatewayHandler("/v1/decisions"); !ok {
|
||
t.Fatal("gateway path /v1/decisions not registered")
|
||
}
|
||
if _, ok := lookupCloudHandler("decisions"); !ok {
|
||
t.Fatal("cloud method \"decisions\" not registered")
|
||
}
|
||
|
||
zapDecisions, _ := lookupCloudHandler("decisions")
|
||
msg, err := zapDecisions(context.Background(), "", []byte(decisionBody))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if status, _, _ := cloudRespStatus(t, msg); status != 401 {
|
||
t.Fatalf("no credential => %d, want 401", status)
|
||
}
|
||
|
||
msg, _ = zapDecisions(context.Background(), "Bearer "+decisionsKey, []byte(`{"model":"laya"}`))
|
||
status, body, _ := cloudRespStatus(t, msg)
|
||
if status != 400 || string(body) != string(decisionsFailure(400, `unknown model "laya"; use one of `+decisionsServed)) {
|
||
t.Fatalf("unknown model => %d %s", status, body)
|
||
}
|
||
|
||
msg, _ = zapDecisions(context.Background(), "Bearer sk-nobody-issued-this", []byte(`{"model":"laya"}`))
|
||
if status, _, _ := cloudRespStatus(t, msg); status != 401 {
|
||
t.Fatalf("unauthenticated unknown model => %d, want 401", status)
|
||
}
|
||
|
||
msg, _ = zapDecisions(context.Background(), "Bearer "+decisionsKey, []byte(decisionBody))
|
||
status, body, errText := cloudRespStatus(t, msg)
|
||
if status != 200 || string(body) != decisionAnswer {
|
||
t.Fatalf("forward => %d %s (%s)", status, body, errText)
|
||
}
|
||
if calls, _, sentBody := fake.seen(); calls != 1 || string(sentBody) != decisionBody {
|
||
t.Fatalf("service saw %d call(s), body %s", calls, sentBody)
|
||
}
|
||
}
|
||
|
||
// The shipped catalog publishes kai and the Zen ids as decision models, keeps the
|
||
// Jev ids callable and unlisted, and the static fallback agrees. A Zen id lists as
|
||
// the decision service's, priced on both legs.
|
||
func TestDecisionsCatalog(t *testing.T) {
|
||
want := []string{"kai", "typesafe/jev-1.13", "zenlm/zen6", "zenlm/zen6-flash", "~typesafe/jev-latest"}
|
||
if got := strings.Join(decisionModels(), ", "); got != decisionsServed {
|
||
t.Fatalf("decisionModels = %s, want what a 400 names: %s", got, decisionsServed)
|
||
}
|
||
if got := decisionModels(); !slices.Equal(got, want) {
|
||
t.Fatalf("static decision models = %v, want %v", got, want)
|
||
}
|
||
|
||
useCatalog(t, "../conf/models.yaml")
|
||
if got := decisionModels(); !slices.Equal(got, want) {
|
||
t.Fatalf("catalog decision models = %v, want %v", got, want)
|
||
}
|
||
listed := map[string]modelInfo{}
|
||
for _, m := range listAvailableModels() {
|
||
listed[m.ID] = m
|
||
}
|
||
kai, ok := listed["kai"]
|
||
if !ok {
|
||
t.Fatal("kai is not in /v1/models")
|
||
}
|
||
if !slices.Equal(kai.Outputs, []string{"decision"}) || kai.OwnedBy != "hanzo" {
|
||
t.Fatalf("kai lists as outputs=%v owned_by=%q, want [decision] hanzo", kai.Outputs, kai.OwnedBy)
|
||
}
|
||
for _, id := range []string{"typesafe/jev-1.13", "~typesafe/jev-latest"} {
|
||
if _, ok := listed[id]; ok {
|
||
t.Errorf("%s is listed; it is callable, not published", id)
|
||
}
|
||
}
|
||
for _, id := range []string{"zenlm/zen6", "zenlm/zen6-flash"} {
|
||
m, ok := listed[id]
|
||
if !ok || !slices.Equal(m.Outputs, []string{"decision"}) || m.OwnedBy != "hanzo" || m.Family != "kai" {
|
||
t.Errorf("%s lists as %+v, want a decision model of hanzo's in family kai", id, m)
|
||
continue
|
||
}
|
||
if m.Free || m.Pricing == nil || m.Pricing.InputPerMillion != 2.28 || m.Pricing.OutputPerMillion != 7.26 {
|
||
t.Errorf("%s lists free=%v at %+v, want $2.28 in and $7.26 out per million", id, m.Free, m.Pricing)
|
||
}
|
||
}
|
||
for _, id := range []string{"laya", "laya-multilingual", "laya-agent"} {
|
||
if r := resolveModelRoute(id); r != nil && r.providerName == object.KaiName {
|
||
t.Errorf("%s routes to the decision service; the baseline is never published", id)
|
||
}
|
||
}
|
||
}
|
||
|
||
// An answer is billed on the usage it states, so one whose usage is not two whole
|
||
// token counts (and a cost, if any, a number of 0 or more) is refused 502 and
|
||
// billed nothing, whatever model answered: no string, fraction, exponent, sign or
|
||
// count past int64 is read as some number of tokens, and the hold is given back.
|
||
func TestAnAnswerThatDoesNotSayWhatItSpentIsRefused(t *testing.T) {
|
||
fake, events := setupDecisions(t)
|
||
user, _ := providerKeyBillingUser(&object.Provider{Owner: decisionsOrg})
|
||
subject := user.PayerSubject(decisionsOrg)
|
||
asked := `{"model":"zenlm/zen6","state":"x","questions":{"q":{"type":"boolean"}}}`
|
||
answer := func(usage string) string {
|
||
return `{"id":"d","model":"zenlm/zen6","provider":"Hanzo","answers":{"q":{"type":"boolean","probability":1}}` + usage +
|
||
`,"routing":{"backend":"llm","checkpoint":"zen6","reason":"r"},"state_hash":"s","latency_ms":1}`
|
||
}
|
||
for _, usage := range []string{
|
||
``,
|
||
`,"usage":null`,
|
||
`,"usage":{"input_tokens":319}`,
|
||
`,"usage":{"input_tokens":319,"output_tokens":"4096"}`,
|
||
`,"usage":{"input_tokens":"319","output_tokens":23}`,
|
||
`,"usage":{"input_tokens":319,"output_tokens":23.5}`,
|
||
`,"usage":{"input_tokens":319,"output_tokens":1e3}`,
|
||
`,"usage":{"input_tokens":319,"output_tokens":-1}`,
|
||
`,"usage":{"input_tokens":319,"output_tokens":99999999999999999999}`,
|
||
`,"usage":{"input_tokens":319,"output_tokens":23,"cost":"0.001"}`,
|
||
`,"usage":{"input_tokens":319,"output_tokens":23,"cost":-0.001}`,
|
||
} {
|
||
fake.answer = answer(usage)
|
||
for _, model := range []string{"zenlm/zen6", "kai"} {
|
||
fund(decisionsOrg, 100_00)
|
||
body := strings.Replace(asked, `"model":"zenlm/zen6"`, `"model":"`+model+`"`, 1)
|
||
status, out := driveDecisions(t, "Bearer "+decisionsKey, body)
|
||
if status != http.StatusBadGateway || out != `{"error":{"code":502,"message":"the decision service's answer did not say what it spent"}}` {
|
||
t.Errorf("%s answering usage %q => %d %s, want 502", model, usage, status, out)
|
||
}
|
||
if left, _ := object.GlobalBalanceLedger.Available(subject); left != 100_00 {
|
||
t.Errorf("%s answering usage %q left %d of 10000 cents held", model, usage, left)
|
||
}
|
||
}
|
||
}
|
||
if len(*events) != 0 {
|
||
t.Fatalf("refused answers were debited: %+v", *events)
|
||
}
|
||
fake.answer = answer(`,"usage":{"input_tokens":319,"output_tokens":23,"cost":null}`)
|
||
if status, out := driveDecisions(t, "Bearer "+decisionsKey, asked); status != http.StatusOK || len(*events) != 1 ||
|
||
(*events)[0].USD != nanoToUSD(319*2280+23*7260) {
|
||
t.Fatalf("a usage of two counts and a null cost => %d %s, debits %+v", status, out, *events)
|
||
}
|
||
}
|
||
|
||
// With no config loaded the static table is the route table, and decisionRoute
|
||
// reads it as decisionModels does: every route to the decision service, in any
|
||
// case and spacing, and no other id.
|
||
func TestDecisionRouteReadsTheStaticTableWithNoConfig(t *testing.T) {
|
||
prev := globalModelConfig
|
||
globalModelConfig = nil
|
||
t.Cleanup(func() { globalModelConfig = prev })
|
||
if GetModelConfig() != nil {
|
||
t.Fatal("a config is still loaded")
|
||
}
|
||
ids := decisionModels()
|
||
if len(ids) != 5 {
|
||
t.Fatalf("static decision models = %v", ids)
|
||
}
|
||
for _, id := range append(ids, " ZenLM/Zen6 ", "KAI") {
|
||
if !decisionRoute(id) {
|
||
t.Errorf("%q is a route to the decision service; decisionRoute says not", id)
|
||
}
|
||
}
|
||
for _, id := range []string{"", "zen6", "zenlm/zen5", "gpt-4o", "kai-0123456789ab", "jev-latest"} {
|
||
if decisionRoute(id) {
|
||
t.Errorf("%q is no route to the decision service; decisionRoute says it is", id)
|
||
}
|
||
}
|
||
}
|