Files
ai/object/balance_ledger_test.go
zeekay 531e9f423a lane: no platform ceiling on the paid lane; a payer spends what they bought
The paid lane capped every paid-lane call together at PAID_LANE_DAILY ($10 a UTC
day by default) and each org at PAID_LANE_ORG_SHARE (25% of it), and counted
wallet-paid calls against both: a customer with $1,000 prepaid was held to $2.50 a
day, and every payer together to $10. The rule is no ceiling but what a payer
prepays, or their usage. Free users get free models; payers spend their own money.

Deleted: the platform day and the org share (paidLaneDaily, orgShare, the day,
spent and per-org accounting in laneBook), the paid_day store (object.PaidDay, its
table registration, the writer, flush, drain, load and turn), Settled's drain of
it, laneSpend, PaidLaneOpen (the gate reads !FreeOnly()), and the refusals only the
day produced ("closed", "after 00:00 UTC"). paid_lane_ceiling named a platform
ceiling that no longer exists: a payer the lane has no room for is told
paid_lane_full, the code its refusal already carried, in X-Hanzo-Lane-Reason and
X-Hanzo-Usage-Reason. Stores keep an orphan paid_day table nothing reads.

Kept: the wallet hold (reserveFor), the plan class's LimitGrant.Spend less the
payer's calls in flight, quote sizing against it, 8 seats a payer, lapse and
stream-idle give-back, race losers holding the seat, BYO, and the free-model
hand-off for a payer with no room.

The replica guard. The lane was closed unless CLOUD_API_REPLICAS=1 because the day
lived in this process. No deployment sets CLOUD_API_REPLICAS: the cloud chart
injects CLOUD_REPLICAS (1 in production). So the boot assertion meant to stop a
second replica double-spending the in-pod wallet ledger never fired, and the lane
guard was the one live check; it also kept the paid lane closed in production. The
plan holds that remain are in-pod exactly as the wallet ledger is, so they take its
guard: SinglePodReplicaHint reads CLOUD_REPLICAS, and bootstrap refuses to start
when the deployment states more than one replica. A declared second replica stops
the process instead of closing the lane, and no switch is left whose only purpose
was the day.
2026-10-09 14:08:38 -07:00

182 lines
6.1 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package object
import (
"sync"
"sync/atomic"
"testing"
"time"
)
// TestSinglePodReplicaHint documents/asserts the in-pod ledger's single-pod
// invariant: the CLOUD_REPLICAS hint is parsed so the boot path can
// refuse to start at >1 replica (the in-pod reservation ledger double-spends
// across pods). Unset/garbage => ok=false (invariant logged, not enforced).
func TestSinglePodReplicaHint(t *testing.T) {
t.Setenv("CLOUD_REPLICAS", "")
if _, ok := SinglePodReplicaHint(); ok {
t.Error("unset hint must be ok=false (invariant logged, not hard-enforced)")
}
t.Setenv("CLOUD_REPLICAS", "1")
if n, ok := SinglePodReplicaHint(); !ok || n != 1 {
t.Errorf("CLOUD_REPLICAS=1 => (1,true), got (%d,%v)", n, ok)
}
t.Setenv("CLOUD_REPLICAS", "2")
if n, ok := SinglePodReplicaHint(); !ok || n != 2 {
t.Errorf("CLOUD_REPLICAS=2 => (2,true) so boot can refuse, got (%d,%v)", n, ok)
}
t.Setenv("CLOUD_REPLICAS", "garbage")
if _, ok := SinglePodReplicaHint(); ok {
t.Error("unparseable hint must be ok=false")
}
}
// TestLedgerColdSubjectCannotReserve: an unknown subject (no Commerce balance
// cached) can never be reserved against — the gate must fetch first.
func TestLedgerColdSubjectCannotReserve(t *testing.T) {
l := NewBalanceLedger(BalanceLedgerTTL)
if l.Reserve("cold", 1) {
t.Fatal("cold subject must not be reservable")
}
if _, known := l.Available("cold"); known {
t.Fatal("cold subject must be unknown")
}
}
// TestLedgerReserveGatesOnAvailable: reservations succeed only while balance
// minus outstanding holds covers the estimate.
func TestLedgerReserveGatesOnAvailable(t *testing.T) {
l := NewBalanceLedger(BalanceLedgerTTL)
l.SetBalance("acme", 100)
if !l.Reserve("acme", 60) {
t.Fatal("first reserve of 60 against 100 must pass")
}
if l.Reserve("acme", 60) {
t.Fatal("second reserve of 60 must fail (only 40 available)")
}
if !l.Reserve("acme", 40) {
t.Fatal("reserve of remaining 40 must pass")
}
if l.Reserve("acme", 1) {
t.Fatal("reserve of 1 with 0 available must fail")
}
if avail, _ := l.Available("acme"); avail != 0 {
t.Fatalf("available=%d, want 0", avail)
}
}
// TestLedgerSettleAppliesActualAndReleasesHold: settle releases the hold and
// decrements the cached balance by the real cost (closing the stale window).
func TestLedgerSettleAppliesActualAndReleasesHold(t *testing.T) {
l := NewBalanceLedger(BalanceLedgerTTL)
l.SetBalance("acme", 100)
if !l.Reserve("acme", 50) { // hold 50 (upper bound)
t.Fatal("reserve must pass")
}
if avail, _ := l.Available("acme"); avail != 50 {
t.Fatalf("available after reserve=%d, want 50", avail)
}
l.Settle("acme", 50, 10) // actual spend was only 10
// Hold released; balance reduced by the actual 10 => available 90.
if avail, _ := l.Available("acme"); avail != 90 {
t.Fatalf("available after settle=%d, want 90", avail)
}
bal, reserved, _, _ := l.Snapshot("acme")
if bal != 90 || reserved != 0 {
t.Fatalf("snapshot bal=%d reserved=%d, want 90/0", bal, reserved)
}
}
// TestLedgerStaleWindowClosedByLocalDebit: even though the cached balance is not
// refreshed from Commerce, local settles make later reserves see the reduced
// balance — the 30s stale window can no longer over-grant.
func TestLedgerStaleWindowClosedByLocalDebit(t *testing.T) {
l := NewBalanceLedger(BalanceLedgerTTL)
l.SetBalance("acme", 100)
// Spend 100 in ten settled requests of 10 each (reserve 10, settle 10).
for i := range 10 {
if !l.Reserve("acme", 10) {
t.Fatalf("reserve %d must pass", i)
}
l.Settle("acme", 10, 10)
}
// Balance is now locally 0 despite never re-fetching Commerce.
if l.Reserve("acme", 1) {
t.Fatal("after spending the full balance, no further reserve may pass within the stale window")
}
}
// TestLedgerConcurrentNoOverdraw is the core race assertion: hammer Reserve from
// many goroutines against a fixed balance and prove the sum of granted holds
// never exceeds the balance and the balance never goes negative.
func TestLedgerConcurrentNoOverdraw(t *testing.T) {
l := NewBalanceLedger(BalanceLedgerTTL)
const balance = 1000
const cost = 10 // each request costs exactly 10 => at most 100 may succeed
l.SetBalance("race", balance)
var granted int64
var wg sync.WaitGroup
for range 500 {
wg.Go(func() {
if l.Reserve("race", cost) {
atomic.AddInt64(&granted, 1)
// Settle the full actual cost (worst case == estimate).
l.Settle("race", cost, cost)
}
})
}
wg.Wait()
// At most balance/cost requests may ever be granted.
if granted > balance/cost {
t.Fatalf("overdraw: granted=%d, max allowed=%d", granted, balance/cost)
}
// Balance must be exactly drained to 0 (every grant settled cost), never negative.
bal, reserved, _, _ := l.Snapshot("race")
if bal < 0 {
t.Fatalf("balance went negative: %d", bal)
}
if reserved != 0 {
t.Fatalf("reservations leaked: reserved=%d", reserved)
}
if want := int64(balance) - granted*cost; bal != want {
t.Fatalf("balance=%d, want %d (granted=%d)", bal, want, granted)
}
}
// TestLedgerReserveThenConcurrentSettleNoNegative interleaves reserves and
// settles concurrently with -race to surface any unsynchronized access.
func TestLedgerReserveThenConcurrentSettleNoNegative(t *testing.T) {
l := NewBalanceLedger(BalanceLedgerTTL)
l.SetBalance("mix", 10000)
var wg sync.WaitGroup
for range 200 {
wg.Go(func() {
if l.Reserve("mix", 50) {
time.Sleep(time.Microsecond)
l.Settle("mix", 50, 25)
}
})
}
wg.Wait()
if bal, _, _, _ := l.Snapshot("mix"); bal < 0 {
t.Fatalf("balance negative under concurrency: %d", bal)
}
}