Files
ai/controllers/billing_reserve_test.go
zeekayandGitHub fcac09e8b2 enso: five models, an effort ladder, and billing by the rung that served (#126)
enso-auto, enso-flash, enso-pro, enso-max and enso-ultra are sold; nothing is
named free. The effort ladder reads router.depth: instant, low, medium, high,
max. With no effort, or auto, the router decides: it starts at Flash and goes
deeper only on its prefer table for the request's task or on Kai's effort
decision (one choice over the ladder's words, counted at 0.9 within 600 ms).

Every Enso tier is served to every caller. The gate orders its rungs by who
pays and tells the engine (X-Hanzo-Classes): ours, premium, free for a paying
caller, premium only while the wallet holds cash; free, ours at no charge for
one nothing pays for, in limited mode, never another model. Fast mode puts
premium first where cash pays it.

The engine's X-Hanzo-Class bills the answer: ours as the grant says, premium
and any fast answer from the wallet's cash at list on every plan, free
nothing. Fast bills 2x enso-auto and Flash, 3x Pro and Max, 4x Ultra.
UsageEvent.Lane and LimitAsk.Lane name the kind for the host.

Prices are the catalog's quotes: pro 3.30 / 16.50 / 0.165 / 4.125, max 4.30
/ 21.50 / 0.215 / 5.375, ultra 5.236 / 26.18 / 0.2618 / 6.545. An answer to
a router id is named for it; the pool's id and its no-cost models are never
listed, and a caller naming one is refused.

Claude-Session: https://claude.ai/code/session_015XYKLnBbrbajqj1GpRydF8
2026-11-07 20:49:12 -08:00

339 lines
15 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"strings"
"testing"
"github.com/hanzoai/ai/object"
)
// TestStreamCaptureUsageCapturesTokens is the #1 P0 assertion: a streaming
// tool-call response that includes a usage chunk yields REAL token counts to
// bill — the streamed path no longer bills $0.
func TestStreamCaptureUsageCapturesTokens(t *testing.T) {
sse := strings.Join([]string{
`data: {"id":"c1","model":"gpt-4o-mini","choices":[{"delta":{"content":"Hel"}}]}`,
`data: {"id":"c1","model":"gpt-4o-mini","choices":[{"delta":{"content":"lo"}}]}`,
`data: {"id":"c1","model":"gpt-4o-mini","choices":[{"delta":{"tool_calls":[{"function":{"arguments":"{\"x\":1}"}}]}}]}`,
`data: {"id":"c1","model":"gpt-4o-mini","choices":[],"usage":{"prompt_tokens":12,"completion_tokens":5,"total_tokens":17}}`,
"data: [DONE]",
"",
}, "\n")
var out strings.Builder
used, text := streamCaptureUsage(strings.NewReader(sse), &out, nil, false, nil, &mark{id: "chatcmpl-req1", model: "gpt-4o-mini", seller: "hanzo"})
if used.prompt() != 12 || used.completion != 5 {
t.Fatalf("captured usage = (%d,%d), want (12,5) — streamed tool call MUST capture real tokens", used.prompt(), used.completion)
}
if !strings.Contains(text, "Hello") || !strings.Contains(text, `{"x":1}`) {
t.Errorf("accumulated output = %q, want it to contain content + tool args", text)
}
// Client did NOT request usage => the forced usage-only chunk must be suppressed.
if strings.Contains(out.String(), `"usage"`) {
t.Errorf("usage-only chunk leaked to client that did not request it:\n%s", out.String())
}
// Content chunks ARE forwarded.
if !strings.Contains(out.String(), `"content":"Hel"`) {
t.Errorf("content chunks must be forwarded, got:\n%s", out.String())
}
}
// TestStreamCaptureUsageForwardsUsageWhenRequested: when the client asked for
// usage, the usage chunk is forwarded (with a fixed envelope).
func TestStreamCaptureUsageForwardsUsageWhenRequested(t *testing.T) {
sse := strings.Join([]string{
`data: {"id":"c2","model":"m","choices":[{"delta":{"content":"hi"}}]}`,
`data: {"choices":[],"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}}`,
"data: [DONE]",
"",
}, "\n")
var out strings.Builder
used, _ := streamCaptureUsage(strings.NewReader(sse), &out, nil, true, nil, &mark{id: "chatcmpl-c2", model: "m", seller: "hanzo"})
if total := used.prompt() + used.completion; total != 5 {
t.Fatalf("total=%d, want 5", total)
}
if !strings.Contains(out.String(), `"usage"`) {
t.Errorf("usage chunk must be forwarded when requested, got:\n%s", out.String())
}
// The forced bare-usage chunk got an envelope (id/object) for SDK clients.
if !strings.Contains(out.String(), `"object":"chat.completion.chunk"`) {
t.Errorf("forwarded usage chunk must carry a fixed envelope, got:\n%s", out.String())
}
}
// TestReserveSettleDebitsLedger proves the controller reserve/settle path holds
// then debits the SAME ledger the gate reads — the per-org debit for a request.
func TestReserveSettleDebitsLedger(t *testing.T) {
const subject = "billtest/u1"
object.GlobalBalanceLedger.SetBalance(subject, 1000)
hold, ok := reserveBudget(subject, 100)
if !ok || hold == nil {
t.Fatal("reserve against a funded subject must succeed")
}
if avail, _ := object.GlobalBalanceLedger.Available(subject); avail != 900 {
t.Fatalf("available after reserve = %d, want 900 (100 held)", avail)
}
hold.settle(40) // actual cost 40
if avail, _ := object.GlobalBalanceLedger.Available(subject); avail != 960 {
t.Fatalf("available after settle = %d, want 960 (debited 40, hold released)", avail)
}
// Double-settle is a no-op (idempotent).
hold.settle(40)
if avail, _ := object.GlobalBalanceLedger.Available(subject); avail != 960 {
t.Fatalf("double settle changed balance: %d, want 960", avail)
}
}
// TestReserveBudgetGatesInsufficient: a known subject without funds for the
// estimate is gated (ok=false).
func TestReserveBudgetGatesInsufficient(t *testing.T) {
const subject = "billtest/u2"
object.GlobalBalanceLedger.SetBalance(subject, 50)
if _, ok := reserveBudget(subject, 100); ok {
t.Error("reserve must fail when the estimate exceeds the balance")
}
}
// TestReserveBudgetSkipsUnknownAndAnon: a cold subject (no known balance) and an
// empty subject (widget/anon) are NOT gated — reservation only augments the
// existing balance gate, it never invents a new fail-closed path.
func TestReserveBudgetSkipsUnknownAndAnon(t *testing.T) {
if h, ok := reserveBudget("billtest/cold-unknown", 100); !ok || h == nil || h.subject != "" {
t.Error("cold/unknown subject must be skipped (ok, no real hold)")
}
if h, ok := reserveBudget("", 100); !ok || h == nil || h.subject != "" {
t.Error("empty subject (widget/anon) must be skipped")
}
}
// TestEstimateRequestCostMonotonic: more max_tokens never costs less (upper bound),
// and the estimate is non-negative.
func TestEstimateRequestCostMonotonic(t *testing.T) {
small := estimateRequestCostCents("gpt-4o-mini", 100, reserveCompletionFloor)
big := estimateRequestCostCents("gpt-4o-mini", 100, 8192)
if small < 0 || big < 0 {
t.Fatalf("estimates must be non-negative: small=%d big=%d", small, big)
}
if big < small {
t.Errorf("more max_tokens must not cost less: small=%d big=%d", small, big)
}
}
// TestFreeRouteEstimatesZero: a route priced at zero estimates zero, so an empty
// wallet still reaches it. The 1-cent floor exists to keep a real call from
// settling as free; charging it to an ESTIMATE refuses a call that will cost
// $0.00, which is the free tier locked out of the free routes published for it.
func TestFreeRouteEstimatesZero(t *testing.T) {
if est := estimateRequestCostCents(freeID, 100, 8192); est != 0 {
t.Errorf("a zero-priced route must estimate zero, got %d", est)
}
const subject = "billtest/broke"
object.GlobalBalanceLedger.SetBalance(subject, 0)
if _, ok := reserveBudget(subject, estimateRequestCostCents(freeID, 100, 8192)); !ok {
t.Error("an empty wallet must still reach a route that costs nothing")
}
// A priced route on the same empty wallet is still refused.
if _, ok := reserveBudget(subject, estimateRequestCostCents("gpt-4o", 100, 8192)); ok {
t.Error("an empty wallet must not reach a priced route")
}
}
// TestOverdrawnWalletStillRefused: zero is not a free pass. A NEGATIVE balance
// fails `balance-reserved >= 0`, so an overdrawn wallet is refused even on a
// route that costs nothing.
func TestOverdrawnWalletStillRefused(t *testing.T) {
const subject = "billtest/overdrawn"
object.GlobalBalanceLedger.SetBalance(subject, -5)
if _, ok := reserveBudget(subject, estimateRequestCostCents(freeID, 100, 8192)); ok {
t.Error("an overdrawn wallet must be refused even on a free route")
}
}
// TestClampMaxTokens locks the upstream completion ceiling: uncapped →
// reserveCompletionFloor, absurd → maxReserveCompletionTokens; a normal client cap
// is preserved unchanged (R1b).
func TestClampMaxTokens(t *testing.T) {
cases := []struct{ in, want int }{
{0, reserveCompletionFloor}, // uncapped
{-5, reserveCompletionFloor}, // negative
{500, 500}, // normal cap preserved
{reserveCompletionFloor, reserveCompletionFloor}, // at floor
{maxReserveCompletionTokens, maxReserveCompletionTokens}, // at max boundary
{maxReserveCompletionTokens + 1, maxReserveCompletionTokens}, // absurd => max, never below the floor of what was asked
{100_000_000, maxReserveCompletionTokens}, // far past it
}
for _, c := range cases {
if got := clampMaxTokens("gpt-4o-mini", c.in); got != c.want {
t.Errorf("clampMaxTokens(%d)=%d, want %d", c.in, got, c.want)
}
}
}
// A paid Hanzo tier's models always reason, and reasoning spends the ceiling before
// the answer begins: a request that names no ceiling gets the most a hold covers,
// not the floor, or a reasoning model returns `length` with no text and is billed.
// A free tier keeps the floor.
func TestAPaidTierThatReasonsGetsRoomToAnswer(t *testing.T) {
zenServiceAt(t, true)
for model, want := range map[string]int{"zen5-pro": maxReserveCompletionTokens, "zen6": reserveCompletionFloor} {
if got := clampMaxTokens(model, 0); got != want {
t.Errorf("clampMaxTokens(%q, 0) = %d, want %d", model, got, want)
}
if got := reserveCompletionTokens(model, 0); got != want {
t.Errorf("reserveCompletionTokens(%q, 0) = %d, want %d", model, got, want)
}
}
if got := clampMaxTokens("zen5-pro", 2_000); got != 2_000 {
t.Errorf("a ceiling the caller named is kept: %d", got)
}
}
// A model resold from OpenRouter's catalog is not a Hanzo tier that reasons: one
// that names no ceiling gets the floor. And no ceiling is ever above the most one
// answer may hold where the catalog states it, which the vendor would refuse.
func TestAResoldModelKeepsTheFloorAndItsOwnMost(t *testing.T) {
gpt4o := `{"id":"openai/gpt-4o","context_length":128000,` +
`"architecture":{"input_modalities":["text","image"],"output_modalities":["text"]},` +
`"pricing":{"prompt":"0.0000025","completion":"0.00001"},"top_provider":{"max_completion_tokens":16384},` +
`"supported_parameters":["tools"]}`
listingVendor(t, listOf(gpt4o, laguna))
syncNow(t)
openrouterFam.mu.Lock()
openrouterFam.loaded = false
openrouterFam.mu.Unlock()
openrouterFam.serves("openai/gpt-4o")
if m, ok := familyLookup("openai/gpt-4o"); !ok || !m.priced() || m.MaxOut != 16384 {
t.Fatalf("the resale catalog was not loaded: %+v %v", m, ok)
}
for _, c := range []struct {
model string
in, want int
}{
{"openai/gpt-4o", 0, reserveCompletionFloor},
{"openai/gpt-4o", 30_000, 16_384},
{"openai/gpt-4o", 100_000_000, 16_384},
{"openai/gpt-4o", 2_000, 2_000},
{"poolside/laguna-s-2.1", 0, reserveCompletionFloor},
{"poolside/laguna-s-2.1", 100_000_000, maxReserveCompletionTokens},
} {
if got := clampMaxTokens(c.model, c.in); got != c.want {
t.Errorf("clampMaxTokens(%q, %d) = %d, want %d", c.model, c.in, got, c.want)
}
}
}
// TestReserveCoversModelLayerCeiling: the QueryText pipeline caps completion at
// reserveCompletionFloor regardless of a low client cap, so the reservation MUST
// cover that ceiling even when the client capped lower.
func TestReserveCoversModelLayerCeiling(t *testing.T) {
if got := reserveCompletionTokens("gpt-4o-mini", 100); got < reserveCompletionFloor {
t.Fatalf("reserveCompletionTokens(100)=%d must be >= floor %d (QueryText emits up to the floor)", got, reserveCompletionFloor)
}
estLowCap := estimateRequestCostCents("gpt-4o-mini", 100, 100)
estFloor := calculateCostCents("gpt-4o-mini", 100, reserveCompletionFloor)
if estLowCap < estFloor {
t.Fatalf("a low client cap must still reserve >= the model-layer ceiling cost: got %d, floor cost %d", estLowCap, estFloor)
}
}
// TestUncappedRequestCannotOverdraft is the R1b assertion: an UNCAPPED request,
// after the max_tokens clamp at the call site, can never settle MORE than it
// reserved — so a single request cannot drive the balance negative. The prior
// 1024-token estimate under-reserved and the actual (larger) settle went negative.
func TestUncappedRequestCannotOverdraft(t *testing.T) {
const (
subject = "billtest/uncapped"
model = "gpt-4o-mini"
)
promptTokens := 100
// Controller flow: clamp the uncapped max_tokens, then reserve the estimate.
clamped := clampMaxTokens(model, 0) // uncapped
if clamped != reserveCompletionFloor {
t.Fatalf("uncapped clamp=%d, want reserveCompletionFloor=%d", clamped, reserveCompletionFloor)
}
est := estimateRequestCostCents(model, promptTokens, clamped)
// Fund EXACTLY to the reservation — the tightest no-overdraft case.
object.GlobalBalanceLedger.SetBalance(subject, est)
hold, ok := reserveBudget(subject, est)
if !ok || hold == nil {
t.Fatalf("reserve must succeed when balance == estimate (est=%d)", est)
}
// Worst-case ACTUAL: a completion at the clamped ceiling. Because the upstream
// is clamped to `clamped` tokens, the model can emit no more — actual <= reserve.
worstActual := calculateCostCents(model, promptTokens, clamped)
if worstActual > est {
t.Fatalf("worst-case actual %d exceeds reservation %d — overdraft possible", worstActual, est)
}
hold.settle(worstActual)
if avail, _ := object.GlobalBalanceLedger.Available(subject); avail < 0 {
t.Fatalf("balance went NEGATIVE within one request: %d (worstActual=%d est=%d)", avail, worstActual, est)
}
}
// A configured route's models.yaml max_output_tokens bounds the ceiling as a family
// catalog's does, however many aliases and upstream models lead to it, and a route
// whose upstream is a family's model is bounded by that model's catalog.
func TestAConfiguredRouteKeepsItsOwnMost(t *testing.T) {
gpt4o := `{"id":"openai/gpt-4o","context_length":128000,` +
`"architecture":{"input_modalities":["text"],"output_modalities":["text"]},` +
`"pricing":{"prompt":"0.0000025","completion":"0.00001"},"top_provider":{"max_completion_tokens":16384}}`
listingVendor(t, listOf(gpt4o))
syncNow(t)
openrouterFam.mu.Lock()
openrouterFam.loaded = false
openrouterFam.mu.Unlock()
openrouterFam.serves("openai/gpt-4o")
prev := globalModelConfig
t.Cleanup(func() { globalModelConfig = prev })
globalModelConfig = &ModelConfig{
routes: map[string]modelRoute{
"vendor/small": {maxOutput: 8_192},
"small": {upstreamModel: "vendor/small"},
"smaller": {upstreamModel: "small"},
"four": {upstreamModel: "openai/gpt-4o"},
"loop-a": {upstreamModel: "loop-b"},
"loop-b": {upstreamModel: "loop-a"},
},
aliases: map[string]string{"tiny": "smaller"},
pricing: map[string]modelPrice{},
stopCh: make(chan struct{}),
}
for _, c := range []struct {
model string
in, want int
}{
{"vendor/small", 30_000, 8_192},
{"small", 100_000_000, 8_192},
{"small", 2_000, 2_000},
{"smaller", 30_000, 8_192},
{"tiny", 30_000, 8_192},
{"four", 30_000, 16_384},
{"loop-a", 30_000, 30_000},
{"vendor/unbounded", 30_000, 30_000},
} {
if got := clampMaxTokens(c.model, c.in); got != c.want {
t.Errorf("clampMaxTokens(%q, %d) = %d, want %d", c.model, c.in, got, c.want)
}
}
}