Files
ai/util/billing_queue.go
zeekayandHanzo Dev 0c08bd1d94 Go 1.27.1, and the code it can say more simply
Modules declare go 1.27.1 and the build images follow. go fix applied the
modernizers; omitzero is not applied, since it changes what the JSON says.

Co-authored-by: Hanzo Dev <[email protected]>
2026-09-17 01:01:10 -07:00

219 lines
6.4 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"bytes"
"fmt"
"io"
"net/http"
"sync"
"time"
"github.com/hanzoai/ai/log"
)
const (
// billingQueueSize is the capacity of the in-memory usage record buffer.
// At peak load (~100 req/s), this provides ~160 seconds of headroom.
billingQueueSize = 16384
// billingMaxRetries is the maximum number of delivery attempts per record.
billingMaxRetries = 3
// billingHTTPTimeout is the per-request timeout for Commerce API calls.
billingHTTPTimeout = 5 * time.Second
// billingShutdownTimeout is the maximum time to wait for queue drain on shutdown.
billingShutdownTimeout = 10 * time.Second
// billingWorkerCount is the number of concurrent workers draining the queue.
billingWorkerCount = 4
)
// billingBackoff returns the delay before retry attempt n (0-indexed).
// Sequence: 1s, 4s, 16s (exponential with base 4).
func billingBackoff(attempt int) time.Duration {
switch attempt {
case 0:
return 1 * time.Second
case 1:
return 4 * time.Second
default:
return 16 * time.Second
}
}
// BillingRecord holds a pre-serialized usage record ready for HTTP delivery.
// Controllers serialize once; the queue retries with the same payload bytes.
type BillingRecord struct {
Body []byte // JSON payload, serialized by the caller
RequestID string // for structured logging on failure
Org string // IAM org slug — billing key + X-Org-Id namespace scope
Model string // model name for structured logging
}
// BillingQueue is a buffered, retrying usage record delivery queue.
// Records are enqueued without blocking the HTTP handler. Background workers
// drain the queue and POST each record to Commerce with exponential backoff.
type BillingQueue struct {
endpoint string // Commerce base URL (e.g. "http://commerce:8001")
token string // Bearer token for Commerce API
ch chan *BillingRecord
wg sync.WaitGroup
stop chan struct{}
client *http.Client
}
// NewBillingQueue creates and starts a billing queue. The endpoint and token
// are resolved once at startup; if the Commerce endpoint is reconfigured at
// runtime the process must be restarted (consistent with how other config
// values are used in cloud).
func NewBillingQueue(endpoint, token string) *BillingQueue {
q := &BillingQueue{
endpoint: endpoint,
token: token,
ch: make(chan *BillingRecord, billingQueueSize),
stop: make(chan struct{}),
client: &http.Client{Timeout: billingHTTPTimeout},
}
q.wg.Add(billingWorkerCount)
for range billingWorkerCount {
go q.worker()
}
return q
}
// Enqueue adds a billing record to the delivery queue. If the queue is full,
// the record is dropped and an error is logged. This never blocks the caller.
func (q *BillingQueue) Enqueue(record *BillingRecord) {
select {
case q.ch <- record:
default:
log.Error("billing_queue: dropped record org=%s model=%s request_id=%s (queue full)",
record.Org, record.Model, record.RequestID)
}
}
// Shutdown signals workers to finish and waits for the queue to drain (up to
// billingShutdownTimeout). Returns the number of records that were still
// pending when the timeout expired.
func (q *BillingQueue) Shutdown() int {
close(q.stop)
done := make(chan struct{})
go func() {
q.wg.Wait()
close(done)
}()
select {
case <-done:
return 0
case <-time.After(billingShutdownTimeout):
remaining := len(q.ch)
log.Error("billing_queue: shutdown timed out, %d records pending", remaining)
return remaining
}
}
// worker drains the queue, delivering each record with retries.
func (q *BillingQueue) worker() {
defer q.wg.Done()
for {
select {
case record := <-q.ch:
q.deliver(record)
case <-q.stop:
// Drain remaining records before exiting.
for {
select {
case record := <-q.ch:
q.deliver(record)
default:
return
}
}
}
}
}
// deliver attempts to POST a billing record to Commerce, retrying with
// exponential backoff on transient failures.
func (q *BillingQueue) deliver(record *BillingRecord) {
// Per global rule: /v1/ only, never /api/. Commerce serves at /v1/billing/usage.
url := q.endpoint + "/v1/billing/usage"
for attempt := range billingMaxRetries {
if attempt > 0 {
delay := billingBackoff(attempt - 1)
// Respect shutdown during backoff.
select {
case <-time.After(delay):
case <-q.stop:
// Still try once more before giving up.
}
}
err := q.post(url, record.Org, record.Body)
if err == nil {
return
}
log.Warning("billing_queue: attempt %d/%d failed org=%s model=%s request_id=%s: %v",
attempt+1, billingMaxRetries, record.Org, record.Model, record.RequestID, err)
}
log.Error("billing_queue: permanently failed org=%s model=%s request_id=%s after %d attempts",
record.Org, record.Model, record.RequestID, billingMaxRetries)
}
// post sends a single HTTP POST to the Commerce billing endpoint. The org slug
// scopes the service-token call to that org's namespace via X-Org-Id, so the
// usage debit lands in the same per-org balance the credit and gate use.
// Returns nil on 2xx, a retryable error on 5xx/network errors, and a
// non-retryable error on 4xx (which will still be retried — Commerce
// should not return 4xx for valid records, so retrying is safer than dropping).
func (q *BillingQueue) post(url, org string, body []byte) error {
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
if err != nil {
return fmt.Errorf("build request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
if q.token != "" {
req.Header.Set("Authorization", "Bearer "+q.token)
}
if org != "" {
req.Header.Set("X-Org-Id", org)
}
resp, err := q.client.Do(req)
if err != nil {
return fmt.Errorf("http: %w", err)
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return nil
}
return fmt.Errorf("commerce returned %d", resp.StatusCode)
}