Files
ai/controllers/openai_api.go
zeekay d362f183b2 decisions: a chat naming Kai is told the door and the body
Kai answers POST /v1/decisions. A chat naming kai, hanzo/kai or a Kai
version reached the decision service and came back "404 upstream
error" or "not available", which said nothing about where to go.
2026-10-26 16:33:57 -07:00

2979 lines
129 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
// Portions Copyright 2025 The OpenAgent Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"bufio"
"bytes"
"context"
"crypto/subtle"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/hanzoai/account"
"github.com/hanzoai/decimal"
"github.com/zap-proto/zip"
"github.com/hanzoai/ai/address"
"github.com/hanzoai/ai/conf"
"github.com/hanzoai/ai/funding"
iam "github.com/hanzoai/ai/internal/iam"
"github.com/hanzoai/ai/log"
"github.com/hanzoai/ai/model"
"github.com/hanzoai/ai/object"
"github.com/hanzoai/ai/upstream"
"github.com/hanzoai/ai/util"
"github.com/hanzoai/go-openai"
)
// getUserBalance returns the current balance for a user by fetching from Commerce.
// Balance is mutable financial state (not identity) so it is never read from the
// JWT — always checked against the source of truth. Caching is handled by the
// router-level BalanceGate (routers/filter_balance.go); this controller-level
// call is a defense-in-depth backstop and does not maintain its own cache.
// The userId should be in "owner/name" format (e.g., "hanzo/alice").
// getUserBalance returns the available balance (in dollars) for an org. Billing
// is per-org: orgKey is the IAM org slug, used both as the balance destination
// key (?user=) and as the namespace selector (X-Org-Id), matching the gate
// and the per-org credit. Without the header, commerce's service-token path
// defaults to the "hanzo" namespace and a per-org credit is invisible.
func getUserBalance(subject, namespace string) (float64, error) {
// Native path: a co-resident host (cloud) reads the subject's wallet balance
// DIRECTLY from the in-process finance ledger — no HTTP. Cents → dollars.
if r := object.BalanceReader(); r != nil {
cents, err := r(context.Background(), subject, namespace, "usd")
if err != nil {
return 0, err
}
return float64(cents) / 100, nil
}
commerceEndpoint := conf.GetConfigString("commerceEndpoint")
if commerceEndpoint == "" {
return 0, fmt.Errorf("commerceEndpoint is not configured")
}
commerceEndpoint = strings.TrimRight(commerceEndpoint, "/")
commerceToken := conf.GetConfigString("commerceToken")
// Per global rule: /v1/ only, never /api/.
// All commerce endpoints live under /v1/.
// subject (?user=) is the per-user/per-org billing key; namespace
// (X-Org-Id) is the org. Both must match the gate and the usage debit.
reqURL := fmt.Sprintf("%s/v1/billing/balance?user=%s&currency=usd", commerceEndpoint, url.QueryEscape(subject))
client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest(http.MethodGet, reqURL, nil)
if err != nil {
return 0, fmt.Errorf("Commerce request build failed: %w", err)
}
if commerceToken != "" {
req.Header.Set("Authorization", "Bearer "+commerceToken)
}
// Scope the service-token call to this org's namespace.
req.Header.Set("X-Org-Id", namespace)
resp, err := client.Do(req)
if err != nil {
return 0, fmt.Errorf("Commerce request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return 0, fmt.Errorf("Commerce returned status %d", resp.StatusCode)
}
var result struct {
Available int64 `json:"available"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return 0, fmt.Errorf("failed to parse Commerce response: %w", err)
}
// Convert cents to dollars for backward compatibility with existing balance > 0 check
balanceDollars := float64(result.Available) / 100.0
return balanceDollars, nil
}
// isJwtToken checks if a token looks like a JWT (3 base64 segments separated by dots).
func isJwtToken(token string) bool {
parts := strings.Split(token, ".")
return len(parts) == 3 && len(parts[0]) > 10 && len(parts[1]) > 10
}
// isIAMApiKey checks if a token is an IAM-issued API key (sk- prefix) — the
// confidential half of the pair IAM mints, and the only one that authenticates.
func isIAMApiKey(token string) bool {
return strings.HasPrefix(token, "sk-")
}
// isPublishableKey checks if a token is a publishable API key (pk- prefix).
// Publishable keys are safe for client-side use and can only access read-only endpoints.
func isPublishableKey(token string) bool {
return strings.HasPrefix(token, "pk-")
}
// resolveProviderFromJwt validates a hanzo.id JWT token and returns the
// appropriate model provider for the requested model, plus the translated
// upstream model name.
//
// requested is the raw X-Org-Id the caller asked to act in ("" for none). This is
// the ONE auth path that can honor an org switch, because it is the one that
// holds the signed `orgs` claim proving membership — the ledger is resolved here,
// from those claims, rather than re-parsing the token downstream.
//
// req is the HTTP request the token arrived on, whose host verdict says whether
// the caller is the SuperAdmin (object.Verify); nil for a raw ZAP door, which has
// none.
func resolveProviderFromJwt(ctx context.Context, req *zip.Ctx, token string, requested string, requestedModel string, lang string) (*object.Provider, *iam.User, string, error) {
// Signature + issuer/audience validation (never raw iam.ParseJwtToken), so a
// token minted for a foreign app/issuer cannot authenticate a paid request.
claims, err := object.Verify(req, token)
if err != nil {
return nil, nil, "", authError("invalid access token: %s", err.Error())
}
user := &claims.User
// An explicit org the signed claim does not cover is refused, not silently
// billed to the caller's personal wallet. Unauthorized and nonexistent are one
// answer so the header cannot be used to enumerate orgs.
effective, orgErr := account.EffectiveOrg(user.Owner, claims.Orgs, requested)
if orgErr != nil {
return nil, nil, "", forbiddenError("organization %q is not available to this principal", requested)
}
ledger := account.LedgerOrg(effective, user.Owner, util.IsSuperAdmin(user))
return resolveProviderForUser(ctx, user, ledger, requestedModel, lang)
}
// resolveProviderFromIAMKey validates an IAM API key (sk-{accessKey})
// and returns the model provider + user, same as JWT path.
//
// An IAM key carries no signed `orgs` claim, so it can never switch org: it bills
// the org that owns the key, which is its home org.
func resolveProviderFromIAMKey(ctx context.Context, apiKey string, requestedModel string, lang string) (*object.Provider, *iam.User, string, error) {
// The whole token, prefix included, IS the accessKey IAM resolves.
accessKey := apiKey
user, err := getUserByAccessKey(accessKey)
if err != nil {
// IAM may return "password or code is incorrect" for service-account users
// (cloud-agent, etc.) due to a known IAM deployment quirk where the
// deployed binary handles certain user records differently. As a safe
// fallback, we check the key against the CLOUD_AGENT_KEY KMS secret
// (env CLOUD_AGENT_KEY as fallback). If it matches, we construct a
// minimal user identity and let the Commerce balance check validate
// the request as normal — so no billing bypass occurs.
if fallbackUser := tryCloudAgentKeyFallback(apiKey); fallbackUser != nil {
// Never log the API key (even masked) — owner/name identify the
// fallback identity for debugging without leaking the credential.
log.Warn("[iam-fallback] IAM returned %q; using cloud-agent fallback identity (owner=%s name=%s)",
err.Error(), fallbackUser.Owner, fallbackUser.Name)
return resolveProviderForUser(ctx, fallbackUser, fallbackUser.Owner, requestedModel, lang)
}
return nil, nil, "", authError("API key validation failed: %s", err.Error())
}
if user == nil {
return nil, nil, "", authError("invalid API key")
}
return resolveProviderForUser(ctx, user, user.Owner, requestedModel, lang)
}
// tryCloudAgentKeyFallback checks whether apiKey matches the known cloud-agent
// service key stored in KMS (secret name "CLOUD_AGENT_KEY") with an env var
// fallback. Returns a minimal *iam.User on match, nil otherwise.
// This is intentionally narrow: only the exact key stored in KMS is accepted.
func tryCloudAgentKeyFallback(apiKey string) *iam.User {
// Try KMS first
var knownKey string
if v, err := object.GetKMSSecret("CLOUD_AGENT_KEY"); err == nil && v != "" {
knownKey = strings.TrimSpace(v)
}
// Env var fallback for local dev / bootstrap
if knownKey == "" {
knownKey = strings.TrimSpace(os.Getenv("CLOUD_AGENT_KEY"))
}
// A secret is compared in constant time, as the two other places in this module
// that compare one already do: an ordinary != stops at the first differing
// byte, and how long it took is a measurement a caller can make.
if knownKey == "" || subtle.ConstantTimeCompare([]byte(apiKey), []byte(knownKey)) != 1 {
return nil
}
// This identity is assembled HERE rather than read from IAM, so it must state
// what IAM would have stated for it. Left unsaid it read as a PERSON:
// account.Payer's shape rule hands a person in the signup org a personal
// wallet, and "hanzo" IS the signup org — so every call on this key addressed
// hanzo/cloud-agent, a wallet no funding path can name, which reads $0 while
// the org's balance sits one key away.
//
// The LEDGER is named outright and the type carries only attribution. In this
// org an org account is the platform's own balance, so which account pays is a
// statement the assembling code makes on the strength of the KMS secret it just
// verified — not something inferred from a class, which is a thing rows can
// also carry. This is the same reason the provider-key and widget identities
// name theirs.
return &iam.User{
Owner: account.SignupOrg,
Name: "cloud-agent",
Type: iam.Machine, // attribution: no person behind this call
BillingAccount: account.Org(account.SignupOrg).String(),
}
}
// resolveProviderForUser is the shared logic for JWT and API key auth paths.
// Given a validated user, resolves the model route and provider.
//
// ledger is the org that PAYS for this request (account.LedgerOrg). It selects both
// the org's own BYOK provider and the wallet the balance gate reads, so a request
// billed to an org is served with that org's connected key — the two cannot name
// different tenants.
func resolveProviderForUser(ctx context.Context, user *iam.User, ledger string, requestedModel string, lang string) (*object.Provider, *iam.User, string, error) {
// Look up the model in the static routing table. A valid caller asking for
// an unknown model is a client error (400), not an auth failure.
route := resolveModelRoute(requestedModel)
if route == nil {
return nil, user, "", modelError(
"model %q is not available. Use GET /v1/models to list available models",
requestedModel,
)
}
// Fetch the provider entry that holds API keys/URLs for this upstream. Prefer
// the org's OWN custom provider (BYOK) if it configured one, else the global
// built-in provider on api.hanzo.ai. Returns a shallow copy, safe to mutate. A
// missing/unconfigured provider is a server-side misconfiguration (500).
provider, err := object.GetModelProviderByNameForOrg(ledger, route.providerName)
if err != nil {
return nil, user, "", serverError("failed to get provider %q: %s", route.providerName, err.Error())
}
if provider == nil {
return nil, user, "", serverError("provider %q not configured in database", route.providerName)
}
// Prepaid-balance gate. Extracted into enforceBalanceGate so the provider-key
// (sk-) path in authResolveProvider enforces the IDENTICAL policy — no auth
// path can drift (M1).
if gateErr := enforceBalanceGate(ctx, user, ledger, requestedModel); gateErr != nil {
return nil, user, "", gateErr
}
return provider, user, route.upstreamModel, nil
}
// enforceBalanceGate applies the prepaid-balance policy to a resolved billing
// principal and stamps user.Balance. A positive prepaid balance is required for
// EVERY model — there is no implicit free allowance and no per-model tier: a
// subject's credit is only ever what it paid or was explicitly granted (commerce
// owns the one grant endpoint). It returns a typed billingError (402) when the
// balance is not positive, a serverError (500) when the balance cannot be verified
// (fail-closed — never grant on a lookup transport error), or nil to proceed.
//
// It is the SINGLE gate shared by the JWT/IAM path (resolveProviderForUser) and
// the provider-key (sk-) path (authResolveProvider), so no auth path can drift
// (M1). There is NO exempt path: every principal is gated on a positive prepaid
// balance. subject is the per-namespace billing account the gate read, the budget
// reservation, and the usage debit all key on.
//
// ledger is the org this request SPENDS FROM — account.LedgerOrg, via c.billingOrg or
// the claims the JWT resolver already holds. It is passed rather than derived
// from user.Owner so the gate reads the SAME wallet the debit writes: keying the
// gate on the selected org and the debit on the home org would check one balance
// and drain another. An empty ledger means the caller could not resolve one, and
// falls back to the home org — the behavior before the org switch existed.
func enforceBalanceGate(ctx context.Context, user *iam.User, ledger string, requestedModel string) error {
if user == nil {
return nil
}
if ledger == "" {
ledger = user.Owner
}
orgKey := ledger // namespace (X-Org-Id): the org tenant whose ledger pays
subject := user.PayerSubject(ledger)
// A model priced at zero has nothing for this gate to refuse. Everything below
// stops a call that cannot be paid for — the caller's wallet, and our own cash
// behind it — and a route that bills zero on both sides spends neither. Refusing
// one asks somebody to add funds for a call that will cost $0.00, which is how a
// free plan came to be unable to reach the free routes we publish for it.
if costsNothing(requestedModel, ledger) {
return nil
}
// Nor does a request the plan or a free cap pays for: no wallet is asked about it.
if covered(ctx) {
return nil
}
// Cash circuit-breaker (internal/funding). Checked BEFORE the balance read
// because it is a property of OUR bank account, not of the caller's wallet: a
// caller with a perfectly good platform balance is exactly who spends our cash
// once upstream promo credit is gone. Disarmed unless a ceiling is configured,
// so this is a no-op until someone deliberately arms it.
//
// Being our bank account, it refuses with supplyError (503) and not with
// billingError (402). The caller owes nothing, so 402 would send a funded org to
// top up a balance that is already funded, and its code would put a billing
// prompt in front of them. The figures stay on our side of the line for the same
// reason the envelope keeps a buy price out of an answer: they are what we pay to
// buy inference. The counter is what an operator reads.
if cash := funding.Current(); cash.Refuse() {
supplyRefused.WithLabelValues("hanzo", reasonCeiling).Inc()
return supplyError(
"paid inference is temporarily unavailable. Your balance is not affected — " +
"nothing is owed and no action is needed. Retry shortly.",
)
}
balance, err := getUserBalance(subject, orgKey)
if err != nil {
// Balance unverifiable (Commerce down, a rejected service token, or an
// unset endpoint) → DENY. AI is prepaid: a broken or misconfigured billing
// backend must never degrade to free, ungated inference. There is no
// fail-open escape and no exempt principal.
return serverError("failed to verify account balance: %s", err.Error())
}
if balance <= 0 {
return billingError(
"model %q requires a positive balance. Your current balance is $%.2f. "+
"Add funds at https://hanzo.ai/billing",
requestedModel, balance,
)
}
user.Balance = balance
return nil
}
// iamClientCreds resolves this service's confidential-app credentials, in order:
// env vars (IAM_CLIENT_ID/IAM_CLIENT_SECRET), KMS secrets, then the router config.
func iamClientCreds() (string, string) {
clientId := conf.GetConfigString("IAM_CLIENT_ID")
clientSecret := conf.GetConfigString("IAM_CLIENT_SECRET")
// Try KMS if config values are empty or placeholders
if clientId == "" {
if v, err := object.GetKMSSecret("IAM_CLIENT_ID"); err == nil && v != "" {
clientId = v
}
}
if clientSecret == "" {
if v, err := object.GetKMSSecret("IAM_CLIENT_SECRET"); err == nil && v != "" {
clientSecret = v
}
}
return clientId, clientSecret
}
// GetUserByAccessKey resolves an sk- IAM API key to its owning user via Hanzo
// IAM. Exported so the authz filter and the balance gate (package routers)
// resolve the key path to the same verified principal as the JWT path — ONE
// credential resolver, one tenant, one billing subject, one IAM transport.
func GetUserByAccessKey(accessKey string) (*iam.User, error) {
return getUserByAccessKey(accessKey)
}
// PublishableOrg resolves a publishable pk- to the org that holds it. Exported so
// the router's tenant resolver answers a page key the same way the controller
// does — ONE credential resolver, one tenant, one IAM transport, exactly as
// GetUserByAccessKey is exported for the secret half.
func PublishableOrg(accessKey string) (string, error) { return publishableOrg(accessKey) }
// The floor a key clears before this process will ask IAM about it. IAM mints
// "{pk|sk}-{live|test}-{random}" (iam internal/keys.Mint), so anything outside
// this alphabet and length is not a key this estate ever issued and needs no round
// trip to refuse. The floor is deliberately wider than the mint: it exists to drop
// junk, not to re-implement the key format, and a key whose shape is plausible is
// still decided by IAM.
//
// The two halves carry their own prefix rather than sharing one pattern, so the
// publishable resolver still refuses a secret locally and the secret resolver still
// refuses a publishable — each answers about its own kind of key, and asking the
// wrong one is a question this process can settle without IAM.
const keyBody = `-[A-Za-z0-9_-]{8,128}$`
var (
publishableKeyShape = regexp.MustCompile(`^pk` + keyBody)
secretKeyShape = regexp.MustCompile(`^sk` + keyBody)
)
const (
// How long a resolved org stays fresh, and how long a refusal is remembered.
// The refusal is shorter because a key can become valid — it was just minted,
// or IAM was briefly unreachable — and nothing should stay denied for a full
// positive lifetime over a blip.
publishableOrgTTL = 5 * time.Minute
publishableDenyTTL = 30 * time.Second
// A bound on distinct keys remembered at once. The keys arriving here are
// whatever the internet sends, so the map must not grow with them.
publishableCacheMax = 8192
)
// publishableAnswer is what the endpoint said about one key, refusals included.
type publishableAnswer struct {
org string
err error
answered time.Time
}
var (
publishableMu sync.RWMutex
publishableCache = map[string]publishableAnswer{}
)
// publishableOrg is resolveOrgFromPublishableKey with a memory, and it is what
// every caller should use.
//
// REMEMBERING THE REFUSALS IS THE POINT. A publishable key ships in the source of
// a page, so the keys presented here are whatever anyone cares to send. Asking IAM
// about each one turns a public endpoint into a load generator aimed at the single
// service every other credential path also depends on — and the way that failure
// shows up is a pk- request resolving no org, which is the one answer an attacker
// would like to induce. Two things stop it: a shape check that costs nothing, and
// a remembered "no" that costs a map read.
//
// It is also what makes the two resolvers AGREE. A single request can ask for this
// org more than once — the tenant resolver and the model resolver both do — and two
// live round trips can answer differently when IAM is flaky, leaving one request
// scoped to one org and billed to another. One answer per key, shared.
func publishableOrg(accessKey string) (string, error) {
if !publishableKeyShape.MatchString(accessKey) {
return "", authError("invalid publishable key")
}
publishableMu.RLock()
a, ok := publishableCache[accessKey]
publishableMu.RUnlock()
if ok && time.Since(a.answered) <= publishableFresh(a) {
return a.org, a.err
}
org, err := resolveOrgFromPublishableKey(accessKey)
publishableRemember(accessKey, publishableAnswer{org: org, err: err, answered: time.Now()})
return org, err
}
// publishableFresh is how long THIS answer stands.
func publishableFresh(a publishableAnswer) time.Duration {
if a.err != nil {
return publishableDenyTTL
}
return publishableOrgTTL
}
// publishableRemember stores an answer, dropping what has expired when the map
// reaches its bound and starting over if that was not enough.
//
// STARTING OVER IS INDUCIBLE, and saying otherwise would overstate what this is.
// Refusals are remembered too, so enough shape-valid keys — none of which IAM has
// to recognise — fill the bound while still fresh and force the reset, which costs
// every live key a round trip. The shape floor drops junk for free, but past it an
// attacker still buys one IAM call per distinct key. This is a latency and
// duplicate-work optimisation; the ceiling on what reaches IAM is the rate limiter,
// not this map.
func publishableRemember(accessKey string, a publishableAnswer) {
publishableMu.Lock()
defer publishableMu.Unlock()
if len(publishableCache) >= publishableCacheMax {
for k, v := range publishableCache {
if time.Since(v.answered) > publishableFresh(v) {
delete(publishableCache, k)
}
}
if len(publishableCache) >= publishableCacheMax {
publishableCache = map[string]publishableAnswer{}
}
}
publishableCache[accessKey] = a
}
// resolveOrgFromPublishableKey resolves a publishable pk- to the ORG that holds
// it, via IAM's publishable endpoint (/v1/iam/keys/org) — the exact dual of
// getUserByAccessKey, which answers only for secret keys and refuses a pk- as
// key_wrong_door. Same confidential Basic transport, same envelope, same typed
// refusals; the answer is an org and never a person.
func resolveOrgFromPublishableKey(accessKey string) (string, error) {
iamEndpoint := conf.GetConfigString("IAM_URL")
if iamEndpoint == "" {
return "", fmt.Errorf("IAM_URL is not configured")
}
iamEndpoint = strings.TrimRight(iamEndpoint, "/")
reqURL := fmt.Sprintf("%s/v1/iam/keys/org?accessKey=%s", iamEndpoint, url.QueryEscape(accessKey))
clientId, clientSecret := iamClientCreds()
if clientId == "" || clientSecret == "" {
return "", fmt.Errorf("IAM client credentials are not configured")
}
client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest(http.MethodGet, reqURL, nil)
if err != nil {
return "", fmt.Errorf("IAM request build failed: %w", err)
}
req.SetBasicAuth(clientId, clientSecret)
resp, err := client.Do(req)
if err != nil {
// The key travels in the query string, so a transport failure carries it:
// *url.Error prints the whole URL. Redacted before it becomes an error value,
// because from here it is logged and wrapped further.
return "", fmt.Errorf("IAM request failed: %s", object.RedactKeys(err.Error()))
}
defer resp.Body.Close()
var result struct {
Status string `json:"status"`
Msg string `json:"msg"`
Code string `json:"code"`
Data struct {
Org string `json:"org"`
} `json:"data"`
}
decodeErr := json.NewDecoder(resp.Body).Decode(&result)
if decodeErr == nil && (result.Code != "" || result.Status != "ok") {
return "", keyRefusal(result.Code, result.Msg, accessKey)
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("IAM returned status %d and named no reason", resp.StatusCode)
}
if decodeErr != nil {
return "", fmt.Errorf("failed to parse IAM response: %w", decodeErr)
}
if result.Data.Org == "" {
return "", authError("publishable key resolved to no org")
}
return result.Data.Org, nil
}
// getUserByAccessKey looks up a user by their IAM API key via Hanzo IAM.
func getUserByAccessKey(accessKey string) (*iam.User, error) {
// The dual of the floor publishableOrg applies to a pk-, and it is here rather
// than at any one call site because this is the ONE place a secret key is
// resolved: the controller, the balance gate and the authz filter all arrive
// through it, and a floor at one door is a floor nobody else stands behind.
//
// It refuses in IAM's OWN words. A holder who presents a retired spelling, a
// typo, or another vendor's key needs the one sentence that ends the problem —
// mint a new key — and which side of the wire decided that is no business of
// theirs. Saying it here also leaves nothing to probe: a shape this estate never
// mints and a key it minted and revoked answer identically, so the floor cannot
// be used to learn the format.
if !secretKeyShape.MatchString(accessKey) {
return nil, keyRefusal("key_unknown", "", accessKey)
}
iamEndpoint := conf.GetConfigString("IAM_URL")
if iamEndpoint == "" {
return nil, fmt.Errorf("IAM_URL is not configured")
}
iamEndpoint = strings.TrimRight(iamEndpoint, "/")
data, err := answerFor(iamEndpoint, accessKey, func() (json.RawMessage, bool, error) {
return askForKey(iamEndpoint, accessKey)
})
if err != nil {
return nil, err
}
var u iam.User
if err := json.Unmarshal(data, &u); err != nil {
return nil, fmt.Errorf("failed to parse IAM response: %w", err)
}
return &u, nil
}
// askForKey asks IAM which principal a secret key speaks for: the principal's JSON,
// or IAM's refusal. final says the answer is about the key, so it may be held; a
// fault in reaching or reading IAM is not.
func askForKey(iamEndpoint, accessKey string) (json.RawMessage, bool, error) {
// Per global rule: /v1/ only, never /api/. A path under /api/ is intercepted
// by the @hanzo/id SPA ingress and returns HTML, which broke API-key
// resolution once ("invalid character '<'").
//
// This is the SECRET-key endpoint, and it is its own route rather than a CRUD
// read. Resolving a key used to ride on the user read as
// `get-user?accessKey=` — an authentication boundary reached through a verb
// whose target was a credential rather than the owner/name that read
// authorizes on. It resolves an sk- to the full principal; the publishable
// pk- has a separate endpoint (resolveOrgFromPublishableKey) that yields an org
// and never a person, so the browser-safe disclosure and this one are never
// behind a single authorization decision.
//
// The segments name things — a key, its principal — and the method says the
// verb. IAM refuses a new verb-noun address outright; the one sibling that
// still reads as one (resolve-key, below) is a frozen spelling live
// consumers hard-code, and that list only ever shrinks.
reqURL := fmt.Sprintf("%s/v1/iam/keys/principal?accessKey=%s", iamEndpoint, url.QueryEscape(accessKey))
// Auth is client_secret_basic (RFC 6749 §2.3.1) — the ONE transport IAM reads
// to establish a confidential-APP principal. Key resolution is a
// credential-disclosure boundary gated on `p.App != ""` holding CapKeyResolve,
// and IAM derives p.App ONLY from Basic credentials: query params yield no
// principal at all (401 "authentication required"), and a client_credentials
// bearer resolves to a plain org-scoped principal with an EMPTY App, which the
// gate refuses with "auth:Unauthorized operation". Basic is also why no token
// cache is needed — there is no token.
clientId, clientSecret := iamClientCreds()
if clientId == "" || clientSecret == "" {
return nil, false, fmt.Errorf("IAM client credentials are not configured")
}
client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest(http.MethodGet, reqURL, nil)
if err != nil {
return nil, false, fmt.Errorf("IAM request build failed: %w", err)
}
req.SetBasicAuth(clientId, clientSecret)
resp, err := client.Do(req)
if err != nil {
// The key travels in the query string, so a transport failure carries it:
// *url.Error prints the whole URL. Redacted before it becomes an error value,
// because from here it is logged and wrapped further.
return nil, false, fmt.Errorf("IAM request failed: %s", object.RedactKeys(err.Error()))
}
defer resp.Body.Close()
// IAM states its reason in the BODY (code + msg), not the status line. This
// used to return on a non-200 before reading it, so `key_unknown` — "the
// entity does not exist" — reached the caller as a bare "IAM returned status
// 400". Those read alike and mean opposite things: one says this key is not
// there, the other says key validation itself is broken. Every gateway key in
// the cluster once failed this way, and the bare status sent the search
// toward a contract break for hours when IAM had named the cause in the first
// reply. Decode first, judge second.
var result struct {
Status string `json:"status"`
Msg string `json:"msg"`
Code string `json:"code"` // WHY, when IAM refused (iam store.KeyFailure)
Data json.RawMessage `json:"data"`
}
decodeErr := json.NewDecoder(resp.Body).Decode(&result)
// A refusal IAM named is a refusal we can relay, whatever the status line. It is
// about the key only when it carries a code.
if decodeErr == nil && (result.Code != "" || result.Status != "ok") {
return nil, result.Code != "", keyRefusal(result.Code, result.Msg, accessKey)
}
if resp.StatusCode != http.StatusOK {
return nil, false, fmt.Errorf("IAM returned status %d and named no reason", resp.StatusCode)
}
if decodeErr != nil {
return nil, false, fmt.Errorf("failed to parse IAM response: %w", decodeErr)
}
// ok-with-nobody. IAM said the request succeeded and named no user, which this
// used to relay as (nil, nil) — no error, no principal — so the caller fell to
// its bare "invalid API key" with nothing behind it. That message is the one
// shape a holder cannot act on and an operator cannot trace: it looks identical
// whether the key is wrong, the envelope changed, or IAM answered about someone
// who no longer exists. A key that resolves to nobody IS unusable, so refuse it
// — but say which of the two things happened, and keep the redacted prefix so
// the line names the credential without disclosing it.
if len(result.Data) == 0 || string(result.Data) == "null" {
return nil, true, authError(
"API key %s resolved to no user — IAM accepted the lookup and returned nothing. "+
"The key may have been deleted, or its owner removed. Mint a new one at "+KeysURL(""),
keyHint(accessKey))
}
var u iam.User
if err := json.Unmarshal(result.Data, &u); err != nil {
return nil, false, fmt.Errorf("failed to parse IAM response: %w", err)
}
return result.Data, true, nil
}
// KeysURL is where a holder mints a key: the API-keys page on the console of the
// brand host belongs to, "" for a brand that runs none. It is spelled ONCE because
// every refusal below names it and three copies of an address drift the moment the
// page moves. It moved already: these messages sent people to cloud.hanzo.ai/keys,
// which answers 404 (cloud.hanzo.ai is the product site; the console is its own
// host, and its key surface is /api-keys). A refusal that names a cure the holder
// cannot reach is worse than one that names none. The key refusals below hold no
// request and pass "", the default brand.
func KeysURL(host string) string {
if c := brandDefs[brandFromHost(host)].console; c != "" {
return "https://" + c + "/api-keys"
}
return ""
}
// keyRefusal turns IAM's refusal into something the holder can ACT on.
//
// IAM answers every unresolvable key with one sentence — "the entity does not exist"
// — and this function used to relay it verbatim, so a user whose key had simply been
// revoked was told their entity was gone and went looking for a deleted organization
// instead of minting a new key. The reason now rides beside that sentence as a code
// (iam internal/store/apikey.go), and each code has exactly one cure.
//
// The key is named by PREFIX only. A holder needs to know WHICH of their keys failed;
// nobody needs the rest of it, and this string reaches logs and error bodies.
func keyRefusal(code, msg, key string) error {
switch code {
case "key_unknown":
// IT SAYS WHAT IS KNOWN, WHICH IS THAT THE KEY DOES NOT RESOLVE.
//
// This code carries no history. It is the same answer for a key that was
// revoked, one that was replaced, and one that never resolved from the
// moment it was issued — and the third really happens: a service account's
// credential was written to a column nothing reads, so it answered this
// code for its entire life (iam internal/serviceaccounts). Saying "it was
// revoked or replaced" turned that into a hunt for a revocation nobody
// performed, on keys that had never worked once.
//
// The cure is the same either way, so the sentence keeps it and drops the
// story it cannot support.
return fmt.Errorf("API key %s does not resolve — mint a new one at "+
KeysURL(""), keyHint(key))
case "key_wrong_door":
return fmt.Errorf("API key %s is not a secret key. A publishable pk- key "+
"identifies an org for ingest and cannot authenticate a request; "+
"use your secret (sk-) key", keyHint(key))
case "key_expired":
return fmt.Errorf("API key %s has expired — mint a new one at "+
KeysURL(""), keyHint(key))
case "key_not_publishable":
return fmt.Errorf("API key %s is not a publishable key", keyHint(key))
case "key_foreign_user", "key_dangling_user":
// Not the holder's doing, and not something they can fix. Say only that it
// was refused; the code carries the detail to whoever reads the logs.
return fmt.Errorf("API key %s was refused (%s)", keyHint(key), code)
}
// An IAM that sent no code, or one this build does not know: relay what it said
// rather than inventing a cause.
return fmt.Errorf("IAM error: %s", msg)
}
// keyHint names a key by its prefix and nothing else — enough to tell WHICH key
// failed, useless to anyone who reads it. The ONE way a key is rendered outside its
// own use.
func keyHint(key string) string {
key = strings.TrimSpace(key)
const shown = 9 // "sk-" + 6
if len(key) <= shown {
return "…"
}
return key[:shown] + "…"
}
// ── Usage tracking ──────────────────────────────────────────────────────────
// usageRecord mirrors IAM's UsageRecord for JSON serialization.
type usageRecord struct {
Owner string `json:"owner"`
User string `json:"user"`
Organization string `json:"organization"`
Model string `json:"model"`
Provider string `json:"provider"`
// PromptTokens is EVERY input token on every path; CacheReadTokens and
// CacheWriteTokens are parts of it, read from and written to the upstream's
// prompt cache, each billed at its own rate (tokenNanoAt).
PromptTokens int `json:"promptTokens"`
CompletionTokens int `json:"completionTokens"`
TotalTokens int `json:"totalTokens"`
CacheReadTokens int `json:"cacheReadTokens,omitempty"`
CacheWriteTokens int `json:"cacheWriteTokens,omitempty"`
Cost float64 `json:"cost"`
Currency string `json:"currency"`
Premium bool `json:"premium"`
Stream bool `json:"stream"`
Status string `json:"status"`
ErrorMsg string `json:"errorMsg"`
ClientIP string `json:"clientIp"`
RequestID string `json:"requestId"`
// ref names this record's debit for the ledger (object.UsageEvent.Ref): minted
// the first time the record is filed and kept, so filing it again — a retry after
// a lost answer — is the same debit.
ref string
// ClientRequestID is the X-Request-Id the caller was answered under, where the
// surface answers with one. It is what the row's request_id column shows, so a
// caller's id finds their bill; RequestID stays the row's own, minted here, and is
// never a value a caller chose.
ClientRequestID string `json:"clientRequestId,omitempty"`
// plan is the grant of the plan that covers this call (Cover), nil when none
// does: the call debits no wallet and counts against no free allowance. planNanos
// is what its paid upstream cost, settled against the plan's budget (planCost).
plan *object.LimitGrant
planNanos int64
// cash says the host's grant lets only the wallet's cash pay for this call: the
// model's policy refuses granted credit (object.UsageEvent.Cash).
cash bool
// key is the API key the call arrived on (object.UsageEvent.Key), set by bind.
key string
// seat is the call's seat on the paid lane (lane.go), set by bind: what it spent
// settles it, against the platform's paid-lane day and its org's share.
seat *seat
// Requested is the model the caller ASKED for, set only when a different route
// answered — today, when a vendor's account was spent and it served the request
// from a route it charges nothing for. Empty on every ordinary call, so
// non-empty IS the fallback flag, and the pair (Requested, Model) says both what
// was wanted and what arrived.
//
// It exists because a downgrade nobody can see is a lie about what the user got:
// the answer is real, it is just not the model they picked, and every reader of
// this record — the ledger, the span, whoever is looking at a bad answer — needs
// to be able to tell those two situations apart.
Requested string `json:"requested,omitempty"`
// Free states that the route that answered is priced at nothing BY THE VENDOR
// — a spare route, the kind that keeps answering while an account is empty.
//
// It is a separate fact from Requested, and keeping them apart is the point:
// one says the caller did not get what they asked for, the other says what the
// answer cost. A price table asked about a SKU it has never seen answers with
// its conservative default, which is the right guess for an unknown model and
// exactly the wrong one here — it would bill a customer for the fallback they
// were given because a vendor of ours ran out of money.
Free bool `json:"free,omitempty"`
// ImageCount is the number of images generated. Image models bill per image,
// not per token: when > 0, recordUsage bills via imageCostCents instead of
// the token-based cost. Placed last so the token-field alignment above is
// unchanged.
ImageCount int `json:"imageCount,omitempty"`
// VideoCount is the number of videos generated. Like images, video models
// bill per unit, not per token: when > 0, recordUsage bills via
// videoCostCents. Mutually exclusive with ImageCount in practice (a call is
// either an image or a video generation, never both).
VideoCount int `json:"videoCount,omitempty"`
// DecisionCount is the number of decisions answered (POST /v1/decisions), one
// per call. When > 0, usageCostNano prices the call by decisionCostNano: its
// input and output tokens at the model's price.
DecisionCount int `json:"decisionCount,omitempty"`
// AudioSeconds is the DURATION of audio a speech-to-text call consumed, and
// AudioChars the number of characters a text-to-speech call synthesized.
// Audio bills per unit like images and video, but the unit differs by
// DIRECTION — the market prices transcription per minute and synthesis per
// million characters — so the two travel separately rather than collapsing
// into one "audio units" field that means different things per row.
//
// A speech model that bills TOKENS instead (an ASR pipeline decoding through
// an LM) needs no field here: it fills PromptTokens/CompletionTokens like any
// other token-billed call. The record carries what was actually consumed.
//
// Both were previously discarded at the emit sites, which is why audio rows
// billed 0: the quantity never reached the record, so the token math it fell
// through to had nothing to multiply.
AudioSeconds float64 `json:"audioSeconds,omitempty"`
AudioChars int `json:"audioChars,omitempty"`
// BYO marks a call executed against a customer-connected third-party account
// (the caller's org supplied its own provider key via /v1/ai/connections),
// rather than a Hanzo-served provider bought with Hanzo credits. When true the
// customer already paid the upstream directly, so recordUsage bills only the
// 1% platform fee (platformFeeCents) instead of the full token cost — the full
// cost is still recorded for analytics. Set at the emit site via providerBYO.
BYO bool `json:"byo,omitempty"`
// FeeCents is the Hanzo platform surcharge stamped by recordUsage
// (platformFeeCents(costCents, BYO)): the amount actually billed on a BYO
// call, 0 for a Hanzo-credits call. Persisted to the warehouse fee_cents column.
FeeCents int64 `json:"feeCents,omitempty"`
// Account is the attribution label for the account that served the call:
// "<owner>/<name>" for a BYO connection, "hanzo" for a Hanzo-served provider.
Account string `json:"account,omitempty"`
// Unpriced marks a token-billed call whose model had NO configured price in any
// source (family/route/conf/static), so it billed at the conservative default
// ($1/$4 per 1M) rather than a real rate. The debit is UNCHANGED — the flag just
// makes the guess honest: the row is queryable and the o11y span carries
// priced=false, instead of silently presenting an invented price as real. Set in
// recordUsage; image/video calls (own per-unit pricing) are never flagged.
Unpriced bool `json:"unpriced,omitempty"`
// BilledNanoExact / CostNanoExact carry an EXACT nano-USD billed amount and
// provider COGS when the biller knows them (zen's commerce Meter computes both
// per served tier). Set, they are what the call billed and cost, and the table
// recompute is not consulted.
//
// Pointers because exactness is PRESENCE, not positivity. A turn that billed
// exactly nothing — a free tier, a zero-price model — knows its amount as
// precisely as any other, and a plain int64 could not say so: 0 read as "unset"
// sent it back to the table, which invents a price for a call that had none.
BilledNanoExact *int64 `json:"billedNanoExact,omitempty"`
CostNanoExact *int64 `json:"costNanoExact,omitempty"`
// ── o11y gen_ai span enrichment (all optional, best-effort) ──────────────
// These carry the observation-of-record attribution the o11y span plane reads
// (controllers/telemetry.go emitGenAISpan). They are omitempty so a record that
// does not populate them emits no attribute — the span is honest, never a
// fabricated value, and each dimension lights up as its producer is wired.
// Session is the conversation/session id. Emitted as gen_ai.conversation.id +
// session.id, which is what turns the o11y sessions view on for this org.
Session string `json:"session,omitempty"`
// TraceID is the gen_ai span's OWN trace id, stamped by emitGenAISpan.
//
// The span plane and the spend ledger answer different questions about one call
// — what happened, and what it cost — and joining them needs an id BOTH sides
// observed. Deriving one independently on each side would produce two ids that
// merely describe the same request, which is not the same thing and cannot be
// joined on. Empty when telemetry is off: there was no span, so there is no
// trace, and saying so is better than inventing an id nothing else will carry.
TraceID string `json:"traceId,omitempty"`
// Environment is the caller's logical environment label (X-Environment). Emitted
// as deployment.environment on the span so Observe narrows by environment instead
// of defaulting to "default". Empty emits no attribute (honest, never fabricated).
Environment string `json:"environment,omitempty"`
// Project is the caller's org SUB-SCOPE (X-Project-Id). It is stamped on the
// cloud_usage ledger row + the gen_ai span, so cost/tokens/latency narrow WITHIN
// an org by project. Empty is the org's default project (whole-org view).
// Populated once in recordTrace from the request context (WithGenAIAttribution).
Project string `json:"project,omitempty"`
// APIKeyHash is a NON-reversible ref (SHA-256 hex) of the caller credential —
// never the plaintext key. Emitted on the gen_ai span (gen_ai.hanzo.api_key_hash)
// so a span correlates to a key without the store ever holding a secret.
// Populated in recordTrace from the request context.
APIKeyHash string `json:"apiKeyHash,omitempty"`
// ServedBy is where inference ran: "hanzo" (cloud), "byo-provider" (an
// org-owned provider key), or "byo-gpu" (an org's own cluster). Empty defaults
// to "hanzo" at emit — the cloud-served majority.
ServedBy string `json:"servedBy,omitempty"`
// ClusterID is the BYO-GPU cluster id when ServedBy == "byo-gpu".
ClusterID string `json:"clusterId,omitempty"`
// RoutePolicy is the enso route-policy decision that selected the model.
RoutePolicy string `json:"routePolicy,omitempty"`
// Routing is the router's whole decision (`Routing`, encoded) when one routed this
// request; emitted on the span as gen_ai.hanzo.routing. Never persisted.
Routing string `json:"-"`
// Served is the arm that generated the answer when a family names one
// (X-Hanzo-Served) — the model behind an adaptive SKU. Emitted as
// gen_ai.response.model; empty leaves response.model the SKU.
Served string `json:"served,omitempty"`
// Vendor is the provider that ran that arm ("digitalocean", "openrouter").
// Emitted as gen_ai.provider.name, ahead of Provider, which is our route label.
Vendor string `json:"vendor,omitempty"`
// Failover is the chain of arms that failed before the one that answered, as
// the family reported it. Empty when the first arm answered.
Failover string `json:"failover,omitempty"`
// Via is the vendor account an answer ran on where a model's own vendor served
// it, as the vendor and the key's place in its pool ("anthropic#2"): the name
// of a place, never a key. Empty on every other call. Emitted on the span as
// gen_ai.hanzo.via; the row says the same in Origin, as host#place.
Via string `json:"via,omitempty"`
// ReasoningTokens is the part of CompletionTokens the model spent reasoning.
ReasoningTokens int `json:"reasoningTokens,omitempty"`
// Reasoning is how hard the caller asked the model to think (depthOf): none,
// low, medium, high, max, or default when it asked nothing. Empty where the path
// does not read it.
Reasoning string `json:"reasoning,omitempty"`
// Arms is every arm a call priced at cost ran (trail), as JSON; empty on a call
// priced any other way.
Arms string `json:"arms,omitempty"`
// LossNano is what the call cost us past what it billed: 0 on every call billed
// at or above its cost. A loss is never silent; it is this.
LossNano int64 `json:"lossNano,omitempty"`
// First is the time from the start of the call to the first token of the
// answer. Zero when not measured.
First time.Duration `json:"-"`
// InputMessages / OutputMessages are the serialized prompt/completion. They are
// PII and are emitted ONLY when O11Y_GENAI_CAPTURE_MESSAGES is enabled
// (default off = redacted). Never logged, never billed — telemetry only.
InputMessages string `json:"inputMessages,omitempty"`
OutputMessages string `json:"outputMessages,omitempty"`
// Agent is the machine credential that authorized this call, "<org>/<name>",
// when the caller was not a person. An IAM application is a route, not a
// customer: it answers "which program placed the call" and never "whose spend
// is this", so it is recorded HERE and User is left to the person the
// application acts for. A row carrying an Agent and no User is a call nobody
// owns — visible as such, rather than presented as the application's own spend.
Agent string `json:"agent,omitempty"`
// Origin is the host that answered — the hostname of the serving provider's
// URL (object.Provider.Origin). Provider is our route label for the same call;
// carrying both lets a row be asked whether the label still matches the address
// the bytes went to. Empty when the serving provider declares no URL.
Origin string `json:"-"`
// Payer is the money address this call spends from — the account the balance
// gate READ, carried here rather than re-derived, so the debit cannot land
// somewhere else. Not serialized: an Account's fields are unexported by design
// (it is money; an account whose owner disagrees with its key must not exist),
// and the wire already carries the subject it resolves to.
Payer account.Account `json:"-"`
// Visitor is the lanes a public-lane call is counted on, narrowest first. Only
// the public lane puts one on the request (withVisitor); every other call is
// empty here. Not serialized — it addresses the lane's count, it does not
// describe the row.
Visitor []address.Bucket `json:"-"`
}
// reached answers whether a vendor was committed to this call — the question the money
// turns on, since a vendor that ran a request invoices us for it whatever came back.
//
// Provider is where that fact already lives. Every emit site fills it from the row that
// SERVED or REFUSED the request, and leaves it empty when the request died before any
// vendor was chosen: an unresolvable route, a candidate list that ran out, a caller who
// hung up. `served` says the same thing from the other side — a failed attempt carries
// a name and nothing else.
//
// IT IS A NAME, NOT A RECEIPT. It says a vendor was committed to this request, not that
// the vendor billed us for it — a 400 the upstream rejected on sight usually costs us
// nothing. That is the honest limit of what a record knows here, and the error it makes
// is the harmless one: a zero-cost row filed for a vendor that charged nothing, rather
// than no row at all for one that did.
func (r *usageRecord) reached() bool { return r.Provider != "" }
// answered reports that a model produced a result for this call. Only the public
// lane's own count asks it — see recordUsage.
func (r *usageRecord) answered() bool { return r.Status == "success" }
// spent reports the tokens a call produced, for a call that then failed.
//
// reached() says a vendor ran the request; this says how much of it it ran. Without
// both, a failure files a row naming a vendor and a quantity of zero, which prices a
// half-generated answer at nothing and leaves us holding the invoice for it.
//
// The pipeline's own count is authoritative whenever it filled one. Otherwise the
// prompt was already counted before the call went out — routeForPrompt sizes the route
// with it — and the partial answer is whatever reached the writer before the stream
// broke. Neither number is invented: one was measured on the way out, the other is the
// text the vendor actually sent.
func spent(res *model.ModelResult, name string, prompt int, partial string, err error) (int, int) {
if res != nil && res.TotalTokenCount > 0 {
return res.PromptTokenCount, res.ResponseTokenCount
}
completion, _ := model.GetTokenSize(name, partial)
if completion == 0 && !ran(err) {
// Nothing came back and nobody ran it: the vendor reported no meter of its
// own, no text reached the caller, and the refusal is one that arrives at
// the edge. The prompt was measured on the way out — routeForPrompt sizes
// the route with it — but measuring a prompt is not a vendor processing it,
// and there is no invoice behind this to pass on.
//
// It is the same reasoning recordUsage states for a call that never left,
// applied to the quantity rather than the row. Left unsaid, the charge
// scales with the prompt, so the larger the request the more it costs to be
// turned away.
return 0, 0
}
return prompt, completion
}
// payer answers who pays for this call. ONE rule, and it prefers the answer the gate
// already computed over deriving a second one.
//
// The fallback is for a record with no authenticated principal behind it — the
// session-scoped and self-billing surfaces. It is account.PayerOf, which is what
// every debit used to do, and it is exactly why this field exists: PayerOf sees two
// strings, so it cannot see the signed billing_account claim naming a personal
// wallet, and it cannot see that a credential is a machine. Both are inputs to who
// pays. Missing them, it confidently answers the pre-claim default — a different
// account from the one the gate read a moment earlier, on the same request.
func (r *usageRecord) payer() account.Account {
if !r.Payer.Zero() {
return r.Payer
}
return account.PayerOf(r.Owner, r.User)
}
// bind ties this record to the credential that authorized the call: the money
// address it spends from, and the name its spend is attributed to. It is the ONE
// place a record learns its principal, so the two answers are read from one
// identity and cannot disagree.
//
// THE MONEY is unchanged and is the same single expression the balance gate
// resolves with (iam.User.Payer) — one identity, one rule, one address, so gate and
// debit cannot answer differently. Nil-safe: a surface with no principal leaves the
// field unset and falls back.
//
// THE NAME follows one rule: only a person may be named, because only a person can
// owe money.
//
// - A person's credential names itself, and nothing on the request can move that.
// Otherwise any caller could send a header and put their bill on a colleague.
//
// - A machine credential names no person, because there is none behind it. It is
// recorded as the Agent, and the User becomes whoever the identity boundary
// authenticated before the machine placed the call (X-User-Id, qualified
// "<org>/<name>"). A bare name is not an identity and is refused rather than
// guessed at.
//
// - A machine that names nobody leaves User empty and says so at once. An empty
// column is a call a query can find; the application's own name in that column
// reads as attributed and is not, which is how spend with no owner reaches an
// invoice unnoticed.
//
// Naming a person never moves money: for a machine the payer resolves to the org
// whatever name the row carries, and a person's payer comes from their own
// credential. Attribution and settlement stay separate answers to separate questions.
func (r *usageRecord) bind(ctx context.Context, u *iam.User) {
if r == nil || u == nil {
return
}
r.Payer = u.Payer(r.Owner)
// The API key the call arrived on, as the boundary named it: what the host
// counts the key's own spend against.
r.key = object.GenAIAttributionFromContext(ctx).Key
r.seat = seatOf(ctx)
// The public lane's visitor, read from the request the lane alone writes it on.
r.Visitor = visitorOf(ctx)
// Who the host said pays: a covered call settles against its grant and debits no
// wallet; one sent to the wallet carries whether only cash may pay.
if g := grantFrom(ctx); g.Covered() {
if r.plan == nil {
r.plan = g
}
} else if g != nil {
r.cash = g.Cash
}
self := u.Owner + "/" + u.Name
// account.IsMachine is the ONE predicate for "is this credential a program",
// shared with the payer rule above.
if !account.IsMachine(u.Type) {
r.User = self
return
}
r.Agent = self
if p := person(ctx, u); p != "" {
r.User = p
return
}
r.User = ""
warnUnownedOnce(self)
}
// person is the person a machine credential's call is for: whoever the identity
// boundary authenticated before the machine placed it (X-User-Id, "<org>/<name>").
// "" for a person's own credential, which names itself, and for a machine that names
// nobody else.
func person(ctx context.Context, u *iam.User) string {
if u == nil || !account.IsMachine(u.Type) {
return ""
}
named := strings.TrimSpace(object.GenAIAttributionFromContext(ctx).User)
// …and the name must be SOMEONE ELSE. A caller reaching us through the identity
// boundary never chooses this header: the boundary deletes what arrived and
// rewrites it from the presenting credential's own claims, so a machine that
// came that way is handed ITS OWN subject. Copying that into the person column
// puts the application back where this whole rule exists to keep it out of, one
// hop later and harder to see.
//
// The two spellings differ in the org half — a token's `sub` is qualified by the
// registration's owner and the `owner` claim by the org it serves — so the NAME
// is what identifies the principal across them.
if _, name, ok := strings.Cut(named, "/"); ok && !strings.EqualFold(name, u.Name) {
return named
}
return ""
}
// tenant is the org a model family is told a call is for (X-Org-Id): the customer a
// program in this process stated (object.Caller), else the org of the person a
// machine credential acts for (person), else the org the request bills, else the
// caller's own. A sibling's call over the plane is the platform working for that
// person's org, and the family keeps its per-org share by it. Money is not moved by
// it: ai settles the call, and the family is told so (X-Hanzo-Fronted-By).
func tenant(ctx context.Context, org string, u *iam.User) string {
if who, ok := object.CallerOf(ctx); ok {
return who.Org
}
if o, _, _ := strings.Cut(person(ctx, u), "/"); o != "" {
return o
}
if org != "" {
return org
}
if u != nil {
return u.Owner
}
return ""
}
// payerHeader tells a Hanzo family the account a call is for (zen's Tenant wallet),
// beside the org in X-Org-Id. The family keeps its free lane's share per payer, so
// the members of the signup org, who each pay for themselves, each hold a share of
// their own rather than all of them one share of the org's.
const payerHeader = "X-Hanzo-Payer"
// payerOf is the account a model family is told a call is for, named in the order
// tenant names its org: the customer a program in this process stated, else the
// person a machine credential acts for, else the caller's own payer in org. "" when
// none resolves, and the family then counts the call against the org.
func payerOf(ctx context.Context, org string, u *iam.User) string {
if who, ok := object.CallerOf(ctx); ok {
return account.PayerOf(who.Org, who.Person).Subject()
}
if named := person(ctx, u); named != "" {
return account.PayerOf("", named).Subject()
}
if u != nil {
return u.PayerSubject(org)
}
return ""
}
// unownedWarned dedupes the unowned-spend report to once per agent, so a busy
// application logs one actionable line instead of one per request.
var unownedWarned sync.Map
// warnUnownedOnce reports spend that no person owns: an application bought
// inference and named nobody it was buying for, so the row bills the org with an
// empty user column. The fix is upstream — the caller sends X-User-Id with the
// person it authenticated — and this is the only moment the evidence still exists.
func warnUnownedOnce(agent string) {
if _, seen := unownedWarned.LoadOrStore(agent, struct{}{}); seen {
return
}
log.Error("usage: application %q is buying inference for nobody — its rows carry no user. Send X-User-Id with the person it authenticated, as \"<org>/<name>\".", agent)
}
// billingQueue is the singleton usage record delivery queue. Initialized by
// InitBillingQueue() in main.go. If nil (Commerce not configured), recordUsage
// is a no-op.
var billingQueue *util.BillingQueue
// InitBillingQueue creates the billing queue from app config. Must be called
// once during startup. Returns the queue so main.go can call Shutdown().
func InitBillingQueue() *util.BillingQueue {
endpoint := conf.GetConfigString("commerceEndpoint")
if endpoint == "" {
return nil
}
endpoint = strings.TrimRight(endpoint, "/")
token := conf.GetConfigString("commerceToken")
billingQueue = util.NewBillingQueue(endpoint, token)
return billingQueue
}
// usageCostCents is the cost of a call in cents: the exact nano cost (usageCostNano
// — per unit for image, video and audio, per token otherwise) rounded to the nearest
// cent. ONE money, reported at two precisions, rather than a second arithmetic over
// the same inputs that can reach a different answer.
//
// It could, and did. The cents path rounded a float and then raised anything left at
// zero to a whole cent, so a call the ledger charged $0.0002 for was reported as $0.01
// — the same money, fifty times over, in the column every spend view reads. Measured
// on ten days of hanzo.cloud_usage: 867 of 1477 rows carried a cent that was invented
// rather than charged, and the totals differed by $8.03 on $34.69.
//
// A call that costs less than half a cent now reports zero cents, which is what it
// costs. Nothing is lost by saying so: cost_nano and billed_nano on the same row carry
// the exact amount, and they are what the debit and the invoice read.
func usageCostCents(record *usageRecord) int64 {
return nanoToCents(usageCostNano(record))
}
// usageBilledCents is what Hanzo actually DEBITS the org ledger for a call, in
// cents, given its full provider cost: the full cost for a Hanzo-served call, or
// only the platform fee (platformFeeCents, ~1%) for a BYO call where the customer
// already paid the upstream with their own key. This is the ONE billed-amount of
// record: recordUsage debits it and emitGenAISpan reports it as
// _o11y.gen_ai.billed_cost, so Observe reconciles with the invoice — while
// _o11y.gen_ai.total_cost keeps the full provider cost for analytics.
func usageBilledCents(record *usageRecord, costCents int64) int64 {
if record.BYO {
return platformFeeCents(costCents, true)
}
return costCents
}
// shownID is the request id the row files under in its request_id column: the one
// the caller was answered under when there was one, else the row's own.
func (r *usageRecord) shownID() string {
if r.ClientRequestID != "" {
return r.ClientRequestID
}
return r.RequestID
}
// paidBy names who pays for a recorded call when it is not the wallet: the plan
// that covered it, or the platform for a free model it absorbs. "" is the wallet.
func paidBy(record *usageRecord) string {
switch {
case record.plan != nil && record.plan.Pays == object.PaysCaller:
return object.PaysCaller
case record.plan != nil:
return object.PaysPlan
case usageFree(record):
return "hanzo"
}
return ""
}
// nanoUSD renders a nano-dollar cost as exact decimal USD, "" when unknown.
func nanoUSD(nano *int64) string {
if nano == nil {
return ""
}
return decimal.New(*nano, 9).String()
}
// usageTimeout bounds one debit's hand-off to the ledger. The native recorder
// answers once the debit is durably posted, which is milliseconds; a ledger that
// has not answered in this long has failed, and the failure is logged with the
// request id instead of holding a goroutine — and, on a path that files the debit
// before its reply, the caller — for as long as the ledger stays silent. It is kept
// well inside a pod's shutdown budget, so a debit filed after its reply can be tried
// again before the process goes (settleAfter).
var usageTimeout = 5 * time.Second
// recordUsage files what a call spent, and returns the native recorder's refusal
// when the debit did not land. It is the ONE chokepoint between a request and
// the money: an exact debit on the org's wallet where this build is co-resident with
// the ledger, or the same amount on its way to Commerce where it is not.
//
// A CALL THAT REACHED A VENDOR IS BILLED, ANSWER OR NO ANSWER. The rule here used to be
// "success only", which reads as fairness and is not one: by the time a call fails the
// request has been sent, the vendor has run it, and we have been invoiced for the tokens
// it processed. Filing nothing does not undo any of that — it only removes the line that
// says it happened, and leaves the spend with us.
//
// THE TRADE IS DELIBERATE AND IT FALLS ON THE CALLER. A request they broke costs them
// what it cost us to try. Keeping our side answering is our job and stays our job; what
// a caller does to their own request is theirs. The alternative prices a failing request
// at zero, which is the one property an unbounded workload must never have — the cheapest
// call in the catalogue would be the one that breaks, and whoever sends the most of them
// would pay the least.
//
// WHAT IT COST IS THE SAME ONE COST. usageCostCents / usageBilledUSD are asked here for a
// failure exactly as for a success, so the row and its o11y gen_ai span carry the
// identical number and cannot disagree. A failure that came back with no tokens therefore
// bills nothing — not because failure is free, but because that is what the meter reads,
// and reading it is the whole point.
func recordUsage(record *usageRecord) error {
// Dense flywheel reward (HIP-510): score EVERY routed request's outcome and
// attach it to its routing decision, so the bandit learns from request-volume
// signal instead of sparse explicit thumbs. An errored request scores 0 (the arm
// is penalized for failing). Dark-by-default, async, best-effort — never slows or
// fails the request.
emitAutoRoutingReward(record)
// NOTHING WAS SPENT ON A CALL THAT NEVER LEFT, so nothing is filed for one. There is
// no invoice behind it and therefore no amount, and an amount is not a thing to
// invent because a row would look tidier with one. The call is still visible — the
// emit sites write the warehouse row and the span for it either way; what it does
// not get is a place in the money.
if !record.reached() {
if record.plan != nil {
record.plan.Settle(0)
}
return nil
}
// Calculate cost. usageCostCents is the ONE cost of record — the same value the
// o11y gen_ai span reports (emitGenAISpan), so the ledger debit and the span
// can never disagree.
costCents := usageCostCents(record)
// Honesty flag: if the model has no configured price, the cost above was billed at
// the conservative default. Mark the row (and warn once) rather than silently
// presenting the invented rate as real — the debit itself is unchanged.
if recordUnpriced(record) {
record.Unpriced = true
warnUnpricedOnce(record.Model)
}
// BYO: the customer paid the upstream directly with their own connected key,
// so Hanzo bills ONLY the 1% platform fee — not the token cost. The full
// costCents is still recorded (payload + warehouse) for analytics. On a
// Hanzo-served call (byo=false) feeCents is 0 and the token cost is billed as
// before. Stamp record.FeeCents so the warehouse writer persists the same value.
feeCents := platformFeeCents(costCents, record.BYO)
record.FeeCents = feeCents
amount := usageBilledCents(record, costCents)
// The debit MUST hit the same account the balance gate read and the starter
// credit funded: the billing SUBJECT within the org NAMESPACE.
// namespace (X-Org-Id) = record.Owner (the org)
// subject (?user=) = the payer the emit site carried from the principal
// For a personal-billing org that is "owner/name" (per-user); for a pooled org
// it is the org slug. record.Owner is the IAM `owner`; fall back to deriving
// owner+name from "owner/name" if Owner was not populated upstream.
org := record.Owner
if org == "" {
if i := strings.IndexByte(record.User, '/'); i > 0 {
org = record.User[:i]
} else {
org = record.User
}
}
subject := record.payer().Subject()
// A call the caller's plan covers settles against the plan: what it spent on
// paid upstream, against the budget the plan held for it. It debits no wallet and
// counts against no free allowance — the plan counted it when it was admitted.
if record.plan != nil {
record.plan.Settle(planUse(record))
}
// A call a program in this process pays for is told its price — the amount this
// record would debit any other caller — so the program charges that and nothing
// of its own.
if g := record.plan; g != nil && g.Pays == object.PaysCaller && g.Bill != nil {
g.Bill.Add(usageMargin(record).BilledNano)
}
// A call on the paid lane settles its seat, whoever paid: the hold taken at
// admission is given back. After the plan's settle, so the payer's next call reads
// the host's figure with this one in it before this hold is gone.
record.seat.settle()
// The public lane keeps its own count in this process, by the visitor's address
// and the site it shares, so its ceiling holds while the host is unreachable. It
// rises on an answer; the host's count was taken when the lane admitted the call.
//
// A VISITOR ON THE RECORD IS WHAT SAYS THIS IS THAT LANE. Only the lane puts one
// there, and nothing on the request can name one. The org would be the obvious
// test and is the wrong one: it is resolved from X-Org-Id, which any caller sets,
// so a bound that read it could be steered out of the lane it bounds.
if record.plan == nil && record.answered() && len(record.Visitor) > 0 && usageFree(record) {
publicCount.serve(record.Visitor, utcDay(time.Now()), publicChatDaily())
}
// Native in-proc finance debit — the ONE money path when co-resident with the
// finance ledger (hanzoai/cloud unified binary). The debit lands DIRECTLY on the
// org's SQLite wallet, the SAME account the prepaid gate reads, so a funded
// account both gates AND depletes. When the hook is installed we do NOT also
// enqueue to Commerce — that would double-bill. Standalone ai (no hook) falls
// through to the HTTP billing queue below.
if rec := object.UsageRecorder(); rec != nil {
ctx, cancel := context.WithTimeout(context.Background(), usageTimeout)
defer cancel()
if record.ref == "" {
record.ref = uuid.NewString()
}
if err := rec(ctx, object.UsageEvent{
Subject: subject,
Namespace: org,
USD: usageBilledUSD(record), // EXACT atto-precise debit, never a floored cent
Currency: "usd",
Model: record.Model,
Provider: record.Provider,
Actor: record.User,
Plan: record.plan != nil,
Cash: record.cash,
RequestID: record.RequestID,
Ref: record.ref,
Class: ClassOf(record.Model),
Units: int64(record.TotalTokens),
// The COGS the row and the span read (usageMargin): a family's or a
// vendor's own statement where one was made, else the rate table's.
CostUSD: nanoUSD(usageMargin(record).CostNano),
PaidBy: paidBy(record),
Key: record.key,
}); err != nil {
log.Error("billing: native usage record failed request_id=%s: %v", record.RequestID, err)
return err
}
return nil
}
if billingQueue == nil {
return nil
}
// The SAME facts the native event carries, in the shape Commerce reads them. Money
// alone: a free call's count was taken at admission (object.SpentFunc), so no
// writer carries one, and what a call spent does not depend on which writer a
// build has.
payload := map[string]any{
"user": subject,
"actor": record.User,
"currency": "usd",
"amount": amount,
"model": record.Model,
"provider": record.Provider,
"promptTokens": record.PromptTokens,
"completionTokens": record.CompletionTokens,
"totalTokens": record.TotalTokens,
"cacheReadTokens": record.CacheReadTokens,
"cacheWriteTokens": record.CacheWriteTokens,
"imageCount": record.ImageCount,
"videoCount": record.VideoCount,
"decisionCount": record.DecisionCount,
"audioSeconds": record.AudioSeconds,
"audioChars": record.AudioChars,
"requestId": record.RequestID,
"clientRequestId": record.ClientRequestID,
"premium": record.Premium,
"stream": record.Stream,
"status": record.Status,
"clientIp": record.ClientIP,
"byo": record.BYO,
"feeCents": feeCents,
"costCents": costCents,
"account": record.Account,
}
body, err := json.Marshal(payload)
if err != nil {
log.Error("billing: failed to marshal usage record request_id=%s: %v", record.RequestID, err)
return nil
}
billingQueue.Enqueue(&util.BillingRecord{
Body: body,
RequestID: record.RequestID,
Org: org,
Model: record.Model,
})
return nil
}
// recordTrace persists an LLM/agent trace + usage record to hanzoai/datastore
// (native datastore OLAP) over native ZAP — the ONE internal telemetry path.
//
// The datastore (datastore) is reached directly via object.DatastoreExec
// (object/datastore.go) — the datastore image serves datastore on :8123/:9000,
// not a ZAP bridge. Fire-and-forget — failures are logged inside the writer,
// never block the request.
//
// Only the spend ledger (hanzo.cloud_usage) is written here — that is the ai
// module's table, read by GetCloudUsageOverview for the console Overview. The
// per-tenant trace ledger (canonical hanzo.observations / hanzo.traces) is owned
// and populated by the o11y/insights ingestion pipeline, not this module — one
// writer per table.
//
// Separately, emitGenAISpan ships one OpenTelemetry GenAI span per call to the
// o11y OTLP collector (opt-in via OTEL_EXPORTER_OTLP_ENDPOINT). That is the
// SOURCE the o11y pipeline ingests — this module never writes the observations
// table directly. The span emit is batched/async and a no-op when telemetry is
// off, so it never blocks the request.
func recordTrace(ctx context.Context, record *usageRecord, startTime time.Time) {
// Enrich the record with the request-scoped observability attribution the
// TenantContextFilter stashed on the context (project sub-scope + a
// non-reversible credential ref). Doing it HERE — the ONE funnel every
// completion surface passes through — stamps BOTH the warehouse row
// (zapWriteUsage) and the gen_ai span (emitGenAISpan) from a single place,
// rather than threading it through ~15 emit sites. A record that already set a
// value (an explicit producer) is left untouched.
if record != nil {
attr := object.GenAIAttributionFromContext(ctx)
// Org fallback: stamp the VERIFIED tenant org only when a producer set neither
// Organization nor Owner, so real tenant traffic always carries its org (the
// o11y llmobs views drop empty-org spans) without overriding an explicit value.
if record.Organization == "" && record.Owner == "" {
record.Organization = attr.Org
}
if record.Project == "" {
record.Project = attr.Project
}
if record.Session == "" {
record.Session = attr.Session
}
if record.Environment == "" {
record.Environment = attr.Environment
}
if record.APIKeyHash == "" {
record.APIKeyHash = attr.APIKeyHash
}
}
// The span is emitted BEFORE the ledger write, because emitting it is what
// decides the trace id and the row has to carry it. The write is the async half
// (a goroutine, off the request path); the span emit is already batched and
// non-blocking, so leading with it costs the caller nothing and is the only
// order in which the two planes can share an id.
emitGenAISpan(ctx, record, startTime)
go zapWriteUsage(record, startTime)
emitGenAIEvent(ctx, record, startTime)
}
// ── API handlers ────────────────────────────────────────────────────────────
// authenticate validates a bearer credential to a real principal WITHOUT routing
// a model — the authentication half of authResolveProvider. Handlers call it on a
// request-body error so an INVALID credential is rejected (401) regardless of
// body validity: a malformed (or field-incomplete) body from an unauthenticated
// caller must never return 200/400 that confirms the endpoint or lets it be
// probed. It is invoked only on the error path, so the happy path keeps a single
// validation (authResolveProvider).
//
// It is a strict SUBSET of authResolveProvider's branches, and the omissions are
// the point: a pk- never reaches here (every generative endpoint refuses it before
// the body is read) and a run key authenticates only against the run table, so
// neither is admitted by this check. A branch missing here can only REFUSE a
// request the resolver would have taken — never admit one it would have refused —
// which is the direction a credential check is allowed to be wrong in.
func (c *ApiController) authenticate(token string) error {
return authenticateToken(token, c.GetAcceptLanguage())
}
// authenticateToken is authenticate with the request's language passed in, for a
// caller that holds no controller (a ZAP handler).
func authenticateToken(token, lang string) error {
switch {
case isJwtToken(token):
// Signature + issuer/audience validation (R3): a foreign-aud or
// wrong-issuer token is rejected here, not just signature-checked.
if _, err := object.ParseAndValidateJWT(token); err != nil {
return authError("invalid access token: %s", err.Error())
}
return nil
default:
// A secret key — see authResolveProvider for why the STORE that owns it,
// not its spelling, decides what it is.
provider, err := object.GetProviderByProviderKey(token, lang)
if err != nil {
return authError("invalid API key: %s", err.Error())
}
if provider != nil {
return nil
}
// getUserByAccessKey returns (nil, nil) for an unknown key (IAM 200 +
// data:null), so check BOTH the error AND a nil user — exactly as
// resolveProviderFromIAMKey does. Missing the nil-user case let an invalid
// key fall through to a 400 parse error instead of a 401.
user, err := getUserByAccessKey(token)
if err != nil {
// Same cloud-agent service-key fallback as resolveProviderFromIAMKey,
// so a valid service key is not falsely rejected here.
if tryCloudAgentKeyFallback(token) != nil {
return nil
}
return authError("API key validation failed: %s", err.Error())
}
if user == nil {
return authError("invalid API key")
}
return nil
}
}
// providerKeyBillingUser derives the billing identity for a provider-key (sk-)
// caller: the org that OWNS the provider row the key belongs to (and therefore
// minted the key). The sk- key is a machine credential, so it bills the OWNER
// ORG — never a per-person wallet no one funds. A provider with no owner is
// unattributable: return an auth error so the caller refuses rather than spend
// the shared upstream key for free — the invariant is that every call spending
// the shared upstream key bills someone.
//
// It NAMES that ledger rather than implying it through a class. This identity is
// synthesized HERE, from a provider row this process just read — there is no
// person and no token — so the org is a fact the calling code holds, and it says
// so in the field that carries such statements. Marking it "application" and
// letting the payer infer the org worked everywhere except the one org where it
// mattered: in the signup org an org account is the platform's own balance, so an
// inferred class was the difference between billing a tenant and spending Hanzo's
// money. A stated ledger cannot be confused with an asserted class.
func providerKeyBillingUser(provider *object.Provider) (*iam.User, error) {
if provider == nil || strings.TrimSpace(provider.Owner) == "" {
return nil, authError("provider key is not attributable to a billable owner")
}
owner := strings.TrimSpace(provider.Owner)
return &iam.User{
Owner: owner,
Type: iam.Machine, // attribution: this call has no person behind it
BillingAccount: account.Org(owner).String(),
}, nil
}
// authResolveProvider authenticates a bearer token and resolves the requested
// model to its upstream provider. It is the single auth + model-routing policy
// for every OpenAI-compatible surface (chat, embeddings, rerank): each handler
// calls it instead of re-implementing the IAM/JWT/provider-key branches.
//
// Returns the resolved provider (with KMS-resolved secret), the billed user
// (nil for provider-key auth), the upstream model id, and whether the route is
// premium. Errors are returned pre-formatted for ResponseError.
func (c *ApiController) authResolveProvider(token, requestedModel, orgId string) (provider *object.Provider, authUser *iam.User, upstreamModel string, isPremium bool, err error) {
lang := c.GetAcceptLanguage()
ctx := c.Context()
switch {
case isRunKey(token):
// Run key (hrun_...) — an autonomous run buying inference on the ledger of
// the org that started it. It resolves to a machine principal and to NO
// user, so it authenticates nobody and opens no other surface; see run.go.
// The org rides the TOKEN, never orgId, so a run cannot be pointed at
// another tenant's balance by a header.
provider, authUser, upstreamModel, err = resolveProviderFromRunKey(ctx, token, requestedModel, lang)
if err != nil {
err = wrapAuth(err)
return
}
c.Locals("recordUserId", authUser.Owner+"/run")
return
case isJwtToken(token):
// hanzo.id JWT token — full model routing + billing. Same typed-status
// contract as the IAM key path. The raw X-Org-Id goes in unvalidated: the
// resolver holds the signed `orgs` claim and is the only place allowed to
// decide whether the request may act — and pay — in that org.
provider, authUser, upstreamModel, err = resolveProviderFromJwt(ctx, c.Ctx, token, strings.TrimSpace(c.Header("X-Org-Id")), requestedModel, lang)
if err != nil {
err = wrapAuth(err)
return
}
case isPublishableKey(token):
// Publishable key (pk-...) — the read-only credential class. It reaches
// only the surfaces that do not refuse it up front (embeddings; every
// generative handler rejects pk- before calling here). IAM's publishable
// endpoint answers with the ORG that holds the key and never a person, so
// the call bills that org as a machine — the same shape as a provider
// key. This is the credential the cloud deployment documents for its
// embed client: least privilege for a read-only endpoint.
org, kerr := publishableOrg(token)
if kerr != nil {
err = wrapAuth(kerr)
return
}
machine := &iam.User{
Owner: org,
Type: iam.Machine, // attribution: no person is behind a page key
BillingAccount: account.Org(org).String(),
}
provider, authUser, upstreamModel, err = resolveProviderForUser(ctx, machine, org, requestedModel, lang)
if err != nil {
err = wrapAuth(err)
return
}
c.Locals("recordUserId", org+"/publishable")
default:
// A secret key, and the STORE that owns it decides what it is. Two
// families share the sk- spelling: an upstream vendor key, which lives in
// the provider table, and the key this estate mints, which lives in IAM.
// Both lookups are exact, so neither can claim the other's key; the order
// decides only who answers when NEITHER owns it, and IAM's refusal is the
// one that names the cure ("mint a new one at" + KeysURL) where
// a provider miss can only say "invalid".
provider, err = object.GetProviderByProviderKey(token, lang)
if err != nil {
err = authError("invalid API key: %s", err.Error())
return
}
if provider == nil {
// Not a vendor key — full model routing + billing off the IAM
// principal. resolveProviderFromIAMKey returns a typed apiError (401
// invalid key / 400 bad model / 402 balance / 500 misconfig); wrapAuth
// preserves it and 401s any untyped error.
provider, authUser, upstreamModel, err = resolveProviderFromIAMKey(ctx, token, requestedModel, lang)
if err != nil {
err = wrapAuth(err)
return
}
break
}
// Attribute + bill this vendor-key call to the org that OWNS the provider row
// (and thus minted this key). Without an authUser, BOTH reserveBudget and
// recordUsage are skipped (they gate on authUser != nil) — so an sk- key
// spending the SHARED upstream (do-ai) ran at ZERO cost and bypassed the
// balance/premium gate. Resolve the billing owner from the key's OWN
// provider row BEFORE any model-route swap below, so the debit lands on
// the key minter, not the upstream route provider we merely call. An
// unattributable provider (no owner) is refused (fail-secure).
authUser, err = providerKeyBillingUser(provider)
if err != nil {
return
}
c.Locals("recordUserId", authUser.Owner+"/provider-key")
// Apply model routing for sk- keys too. If the route points to a
// different provider than the one that owns the API key, switch to the
// route's provider so zen/fireworks models work with any key.
if route := resolveModelRouteForOrg(requestedModel, orgId); route != nil {
upstreamModel = route.upstreamModel
isPremium = route.premium
if route.providerName != provider.Name {
if routeProvider, routeErr := object.GetModelProviderByName(route.providerName); routeErr == nil && routeProvider != nil {
provider = routeProvider
}
}
}
// M1: apply the SAME prepaid-balance gate the JWT/IAM paths enforce
// (resolveProviderForUser), so an sk- provider key can never reach paid
// upstreams without a positive balance. The billed owner is authUser (the
// provider-row owner resolved just above), and a provider key carries no
// membership claim — it always bills the org that minted it.
if gateErr := enforceBalanceGate(ctx, authUser, authUser.Owner, requestedModel); gateErr != nil {
err = gateErr
return
}
return
}
// Shared post-resolution for IAM/JWT auth: record the billed user and the
// premium flag from the route table.
if authUser != nil {
c.Locals("recordUserId", authUser.Owner+"/"+authUser.Name)
}
if route := resolveModelRouteForOrg(requestedModel, orgId); route != nil {
isPremium = route.premium
}
return
}
// ChatCompletions implements the OpenAI-compatible chat completions API
// @Title ChatCompletions
// @Tag OpenAI Compatible API
// @Description OpenAI compatible chat completions API. Accepts:
// - IAM API key (sk-...) — full model routing + billing
// - hanzo.id JWT token — full model routing + billing
// - Provider API key — direct provider access
//
// @Param body body openai.ChatCompletionRequest true "The OpenAI chat request"
// @Success 200 {object} openai.ChatCompletionResponse
// @router /chat [post]
// A completion renders itself to the client. A sink, when a caller supplies one,
// renders it somewhere else instead: /v1/responses reads this same completion in a
// second dialect, and used to do it by swapping the writer out of the request
// context and reading what came back.
//
// Two functions, because a stream and a whole body are two different things and
// collapsing them is what made the old version need a status machine. `wrap`
// decorates the stream's destination as it is produced; `whole` translates the
// non-streaming answer entire, wherever it is written — answerBody, or the writer
// that assembles a whole answer from a family's stream (assembleZenStream).
type sink struct {
wrap func(io.Writer) io.Writer
whole func([]byte) ([]byte, error)
}
// shape is one whole chat completion in the dialect the request asked for: as it
// is with no sink, or translated by it.
func (s *sink) shape(chat []byte) ([]byte, error) {
if s == nil || s.whole == nil {
return chat, nil
}
return s.whole(chat)
}
// ChatCompletions implements the OpenAI-compatible chat completions API
// @Title ChatCompletions
// @Tag OpenAI Compatible API
// @Description OpenAI compatible chat completions API. Accepts:
// - IAM API key (sk-...) — full model routing + billing
// - hanzo.id JWT token — full model routing + billing
// - Provider API key — direct provider access
//
// @Param body body openai.ChatCompletionRequest true "The OpenAI chat request"
// @Success 200 {object} openai.ChatCompletionResponse
// @router /chat [post]
func (c *ApiController) ChatCompletions() { c.chatCompletions(callerBearer, nil) }
// caller says which address a completion arrived at, and therefore what is taken from
// the request and what is decided for it. It is an unexported type with no decoded
// form, so no header, body or query can produce one: the route decides it and the
// request cannot. callerPublic is reachable only from ChatCompletionsPublic, after
// that lane's ceiling has admitted the call.
type caller int
const (
callerBearer caller = iota // a credential is presented and decides everything
callerPublic // no credential; the public lane decided everything
)
// chatCompletions is the one completion pipeline. Both entry points run it; they
// differ only in who the caller is, which is a value passed in rather than a fact
// re-derived here.
func (c *ApiController) chatCompletions(from caller, to *sink) {
c.answer = to
var token string
if from == callerBearer {
// Extract Bearer token
authHeader := c.Header("Authorization")
if !strings.HasPrefix(authHeader, "Bearer ") {
c.ResponseErrorWithStatus(401, c.T("openai:Invalid API key format. Expected 'Bearer API_KEY'"))
return
}
token = strings.TrimPrefix(authHeader, "Bearer ")
// Publishable keys (pk-) cannot access completions — reject early
if isPublishableKey(token) {
c.rejectPublishableKey()
return
}
}
// Track timing for observability
requestStartTime := time.Now().UTC()
// Parse request body. Authenticate BEFORE reporting a parse error so an
// invalid credential is 401 regardless of body validity — a malformed body
// from an unauthenticated caller must not return 200. A valid credential with
// a bad body gets 400 (not 200).
var parsed chatRequest
refusal := ""
if err := json.Unmarshal(c.Body(), &parsed); err != nil {
refusal = fmt.Sprintf("Failed to parse request: %s", parseProblem(err))
} else if why := casefolded(c.Body()); why != "" {
refusal = why
} else if f := unpricedField(c.Body()); f != "" {
refusal = fmt.Sprintf("%q is not accepted: it would buy something this model's price does not cover. Remove it and send the request again.", f)
} else if parsed.N > 1 {
// n completions cost n times one, and the hold covers one. Priced by n, a
// request would be admitted on a balance a single answer fits and settle far
// past it; one answer per request is the price that was quoted.
refusal = fmt.Sprintf("n=%d is not accepted: one request is one completion here.", parsed.N)
}
if refusal != "" {
if from == callerBearer {
if authErr := c.authenticate(token); authErr != nil {
c.ResponseAuthError(authErr)
return
}
}
c.ResponseErrorWithStatus(http.StatusBadRequest, refusal)
return
}
request := parsed.ChatCompletionRequest
// A DECISION MODEL IS NOT A CHAT MODEL. Kai answers POST /v1/decisions; a chat
// naming it reached the decision service and came back "404 upstream error",
// which told the caller nothing about the right door.
if why := decisionDoor(request.Model); why != "" {
if from == callerBearer {
if authErr := c.authenticate(token); authErr != nil {
c.ResponseAuthError(authErr)
return
}
}
c.ResponseErrorWithStatus(http.StatusBadRequest, why)
return
}
// Resolve org context for per-org model routing and pricing.
orgId := c.GetOrg()
if from == callerPublic {
// THE PUBLIC LANE READS NEITHER THE MODEL NOR THE ORG FROM THE REQUEST. Both
// are overwritten before anything downstream can consult them, so naming a
// paid model or another tenant's org is not refused — it is unrepresentable.
// The assignment is here rather than in the lane because this is the last
// point at which the body could still be read.
request.Model, orgId = freeID, publicOrg
}
// A PUBLIC CALL ACTS UNDER NO AMBIENT IDENTITY. The widget lives on our own
// pages, so a visitor may well be carrying a session for this estate; every
// helper that reads one would then attribute an anonymous call to whoever is
// signed in — the routing ledger would record their name, and the wallet
// resolver would try to switch them onto an org and answer "nobody to bill".
// The lane admitted a stranger, so a stranger is who the rest of this serves.
routingUser := c.routingUserId()
principal := c.principalUser()
if from == callerPublic {
routingUser, principal = "", nil
}
// `auto`/`zen-router` resolution (below) calls the router engine — an internal
// HTTP request carrying the caller's prompt — and records a RoutingEvent, BEFORE
// authResolveProvider authenticates further down. Authenticate FIRST so an
// UNAUTHENTICATED caller can never drive that internal machinery: auth precedes
// every side effect. authenticate is the SAME credential check authResolveProvider
// runs (a strict subset — credential only, no model/balance), read-only, so it
// never rejects a request the resolver would accept. Concrete models skip this
// (resolveAutoModel is a no-op for them), so the dominant path pays nothing.
strict := c.strictAsked()
if isAutoModel(request.Model) {
if authErr := c.authenticate(token); authErr != nil {
c.ResponseAuthError(authErr)
return
}
if strict {
c.refuseStrict("route_auto", "auto-routing chooses the model; name one")
return
}
}
// RouteAuto has usually rewritten "auto" to its choice before this handler runs,
// so the body alone no longer says the model was chosen for the caller.
if _, routed := c.Locals(autoRoutedKey).(autoRouted); routed && strict {
if authErr := c.authenticate(token); authErr != nil {
c.ResponseAuthError(authErr)
return
}
c.refuseStrict("route_auto", "auto-routing chose the model; name one")
return
}
// One request id, generated once here — it is the response id (`chatcmpl-<id>`),
// the usage-ledger request_id, AND the routing-event join key, so a later reward
// (POST /v1/ai/feedback) can be tied back to THIS decision. Generated
// before routing so resolveAutoModel can stamp it on the RoutingEvent.
requestId := uuid.NewString()
// Virtual `auto`/`zen-router` model → resolve to a concrete servable model id
// BEFORE any provider/pricing/billing resolution, so the ENTIRE existing path
// (auth+routing, ModelRoute fallbacks, zen identity, balance reserve/settle,
// usage record, response `model` echo) bills and reports the model that
// actually served. The transparency header lets callers see the routed choice.
// routedTask + routingRecorded carry the routing decision to the post-response
// judge hook at the tail of this handler (the LLM-as-a-judge dense reward): set
// in whichever branch records a RoutingEvent, so the judge scores only requests
// that are actually in the routing ledger.
var routedTask string
var routingRecorded bool
if r, ok := c.Locals(autoRoutedKey).(autoRouted); ok && r.routed == request.Model && from == callerBearer {
// RouteAuto resolved `auto` ahead of the balance gate and rewrote the request
// to name the SKU; the gate priced that SKU and the RoutingEvent is already
// recorded under r.requestId. Keep that id so the response and the ledger join
// the decision, and record nothing twice.
requestId = r.requestId
routedTask, routingRecorded = r.task, true
object.GlobalTraffic.RecordTask(
Country(c.Ctx),
"",
r.task,
)
} else if routed, task, ok := resolveAutoModel(request.Model, orgId, routingUser, requestId, principal, &request, c.sloFromHeaders()); ok {
request.Model = routed
routedTask, routingRecorded = task, true
c.SetHeader(RoutedModelHeader, routed)
// Fold this request's TASK into the region's task-mix for the live-traffic
// globe — geo from the edge headers only (NO IP), aggregates only. Best-effort.
object.GlobalTraffic.RecordTask(
Country(c.Ctx),
"",
task,
)
} else if !isAutoModel(request.Model) {
// NON-auto: record the caller's explicit model selection so EVERY request is
// a rateable, trainable data point — up/down feedback for all models, and the
// ledger captures which model served which task even when the caller picked.
task := recordExplicitRouting(request.Model, orgId, routingUser, requestId, &request)
routedTask, routingRecorded = task, true
object.GlobalTraffic.RecordTask(
Country(c.Ctx),
"",
task,
)
}
// Authenticate the bearer token and resolve the requested model to its
// upstream provider, premium flag, and (for IAM/JWT auth) the billed user.
// This is the ONE auth+routing policy, shared with /v1/embeddings and
// /v1/rerank — see authResolveProvider.
var (
provider *object.Provider
authUser *iam.User
upstreamModel string
isPremium bool
err error
)
if from == callerPublic {
// No credential to authenticate. The lane already settled who is asking and
// what they may run; this settles only who runs it.
provider, authUser, upstreamModel, err = c.resolveProviderForPublic()
} else {
provider, authUser, upstreamModel, isPremium, err = c.authResolveProvider(token, request.Model, orgId)
}
if err != nil {
c.ResponseAuthError(err)
return
}
// The ledger this request spends from — the org the switcher selected when the
// signed membership claim proves it, else the caller's home org. Resolved ONCE
// here so the reservation below and every usage record at the tail of this
// handler key on the SAME wallet the gate inside authResolveProvider just read.
ledger := c.billingOrg(authUser)
if from == callerPublic {
// Cap max_tokens for a caller nobody can be billed for: the public lane
// serves a stranger, so the answer is bounded instead.
if request.MaxTokens == 0 || request.MaxTokens > publicMaxTokens {
request.MaxTokens = publicMaxTokens
}
}
if provider.Category != "Model" {
c.ResponseError(fmt.Sprintf("Provider %s is not a model provider", provider.Name))
return
}
// Set the upstream model name on the provider. For JWT/IAM key auth, this
// is the translated upstream model from the routing table. For provider
// API key auth, fall back to the request model or provider's default.
if upstreamModel != "" {
provider.SubType = upstreamModel
} else if request.Model != "" {
provider.SubType = request.Model
}
// ── Balance reservation ────────────────────────────────────────────
// Hold an upper-bound budget for this request so concurrent requests for the
// same subject can't double-spend a balance the async debit hasn't applied
// yet. The router gate is coarse (balance>0); this enforces
// balance >= estimated cost and reserves it atomically. It is settled with
// the ACTUAL cost when the request completes (deferred fail-safe release).
var hold *budgetHold
// wide is how many providers this request may be offered to AT ONCE. It is
// decided with the reservation and never after: what a race costs is N
// completions, so the hold either covers them or the request is not raced.
wide := 1
// The prompt as sent, every message of it. The reservation prices it and the
// relay sizes the route by it, so it is measured once.
measured := estimatePromptTokens(&request)
if authUser != nil {
subject := authUser.PayerSubject(ledger)
// Clamp the upstream completion ceiling BEFORE reserving so the relayed
// upstream can never emit more than we reserve — the actual settle can
// never exceed the hold (R1b). reserveCompletionTokens also covers the
// QueryText pipeline's fixed cap, which ignores max_tokens.
// The ceiling a caller named under either key is the one the hold covers; one
// named only as max_completion_tokens was otherwise lowered to the floor.
if request.MaxTokens == 0 {
request.MaxTokens = request.MaxCompletionTokens
}
// On the paid lane the ceiling is never above the one its seat was sized for.
request.MaxTokens = laneTokens(c.Context(), clampMaxTokens(request.Model, request.MaxTokens))
// The vendor bills the whole body — tools, schemas and call arguments as well
// as message text — so the hold is never priced on less than its size.
est := estimateRequestCostCents(request.Model, max(measured, len(c.Body())/4), request.MaxTokens)
var ok bool
if hold, wide, ok = c.widthFor(authUser, subject, est); !ok {
c.ResponseAuthError(billingError("%s", object.InsufficientBalance(c.Host(), ledger, "request cost").Message))
return
}
}
// Released on the way out — unless a relayed stream took it. That answer settles
// from inside its own writer, which runs after this handler has returned, and a
// release here would win the one-shot settle before the answer's cost is known.
defer func() { hold.settle(0) }()
// ── Model families (Zen, Enso) ─────────────────────
// A family model is served by its family service, which owns identity, reasoning,
// the 1M ladder, vision, the fan-out, and the upstream. ai forwards verbatim and
// meters the result; it holds no family routing of its own (hip-00NN).
//
// A family is one provider among several. When it refuses for a reason of its
// own — its account is empty, it is rate limited, it is down — it writes
// nothing and hands back the reason, and the request carries on to the
// route's declared alternates below. That is the difference between a vendor
// running out of money and the product going dark.
var familyRefused []attempt
if fam := familyForProviderType(provider.Type); fam != nil && strict {
c.refuseStrict("translation", "this model is not served in a way that keeps the request as sent")
return
}
if fam := familyForProviderType(provider.Type); fam != nil {
// The free tier is served under freeDoor, so it answers as a model of ours.
sku := request.Model
if fam == freeFamily() && strings.EqualFold(strings.TrimSpace(sku), freeID) {
fam, sku = freeDoor()
}
familyRefused = c.pipeToFamily(fam, "chat/completions", "openai", sku, c.Body(), request.Stream, clampMaxTokens(sku, request.MaxTokens), orgId, authUser, isPremium, hold, requestStartTime)
if familyRefused == nil {
if request.Stream {
hold = nil // a family's stream settles its own hold, from its writer
}
return
}
recordRefusals(c.takeSnapshot(authUser), request.Model, familyRefused, authUser, isPremium, request.Stream, requestId, requestStartTime)
}
// ── Tools and images ───────────────────────────────────────────────
// A tool call or an image the family refused stops here. What follows a family
// is its route's tail — the resale copy of the model and the free text floor —
// and neither is chosen for tools or vision: an alternate could answer the call
// without the part that made it one, and an answer shaped wrongly is worse than
// an honest refusal that names the vendor and the reason.
//
// With the paid lane off, only that floor is asked, so the same holds.
tooled := len(request.Tools) > 0 || request.ToolChoice != nil || len(request.Functions) > 0 || request.FunctionCall != nil
media := requestHasMedia(&request)
if tooled || media {
if familyRefused != nil {
c.ResponseFailure(exhausted(request.Model, familyRefused))
return
}
if shut(c.Context(), provider) {
c.ResponseFailure(laneOff(c.Context(), request.Model))
return
}
}
// ── The relay ──────────────────────────────────────────────────────
// A route that speaks OpenAI's dialect is sent the caller's own request —
// every message, every parameter, tools and images alike — and fails over
// along the route exactly as the text pipeline does (forward.go).
if relays(provider) {
question := lastUserText(&request)
// The router judge scores a text answer against the turn that asked for it,
// as it did before; a tool call or an image is not what it scores. Every
// value it reads is taken now, because it runs where a stream is finished.
var score func(string)
if routingRecorded && !tooled && !media {
agent, country, org := strings.Clone(c.Header("User-Agent")), strings.Clone(Country(c.Ctx)), strings.Clone(orgId)
sku, task := request.Model, routedTask
score = func(answer string) {
judgeRoutedResponse(agent, org, requestId, country, sku, task, question, answer)
}
}
// A strict request goes where its route is declared to go. Sized to the prompt,
// routeForPrompt may move it to another row serving another model, which the
// digests, taken without the model, would not show.
route := routeForPrompt(request.Model, orgId, measured)
if strict {
route = resolveModelRouteForOrg(request.Model, orgId)
}
if c.forward(pass{
req: &request,
body: c.Body(),
primary: provider,
route: route,
prior: familyRefused,
knowledge: c.retrieveKnowledgeIfEnabled(
question, retrievalOwner(authUser), c.retrievalStore(), c.GetAcceptLanguage()),
prompt: measured,
user: authUser,
premium: isPremium,
hold: hold,
id: requestId,
start: requestStartTime,
judge: score,
strict: strict,
}) {
hold = nil
}
return
}
// Anthropic-type rows and rows with no OpenAI-compatible address keep the older
// path until the relay speaks their dialect: a tool call or an image converted
// for Anthropic, and plain text through the QueryText pipeline below.
//
// Neither carries a response format, so one the caller named is refused rather than
// ignored: an answer that skipped the schema would read as one that followed it.
// These rows move to the Messages translator when it lands, and this goes with them.
if strict {
c.refuseStrict("translation", "this model is not served in a way that keeps the request as sent")
return
}
var format struct {
Type string `json:"type"`
}
if json.Unmarshal(parsed.ResponseFormat, &format) == nil && format.Type != "" && format.Type != "text" {
c.ResponseFailure(modelError("response_format %q is not supported for %s on this route; send the request without it", format.Type, request.Model))
return
}
if tooled || media {
if model.Upstream(provider.Type) != model.Anthropic {
c.ResponseError("No upstream endpoint configured for provider: " + provider.Name)
return
}
// The body is built from the upstream's own id; the envelope, the usage row
// and the price read the SKU.
sku := request.Model
request.Model = provider.SubType
c.proxyToolRequestAnthropic(provider, &request, sku, requestStartTime, authUser, isPremium, orgId, requestId, hold)
return
}
// Extract messages content
var question string
var systemPrompt string
history := []*model.RawMessage{}
for _, msg := range request.Messages {
// Extract text from Content or MultiContent (array-style content parts)
text := msg.Content
if text == "" && len(msg.MultiContent) > 0 {
var parts []string
for _, part := range msg.MultiContent {
if part.Type == openai.ChatMessagePartTypeText && part.Text != "" {
parts = append(parts, part.Text)
}
}
text = strings.Join(parts, "\n")
}
switch msg.Role {
case "system":
systemPrompt = text
case "user":
question = text
case "assistant":
history = append(history, &model.RawMessage{
Author: "AI",
Text: text,
})
}
}
if question == "" {
c.ResponseError(c.T("openai:No user message found in the request"))
return
}
// Combine system prompt with user question if available
if systemPrompt != "" {
question = fmt.Sprintf("System: %s\n\nUser: %s", systemPrompt, question)
}
// The OpenAI-shaped view of the stream. Its destination is bound below, where
// the body is produced.
writer := &OpenAIWriter{
Buffer: []byte{},
RequestID: requestId,
Stream: request.Stream,
Cleaner: *NewCleaner(6),
Model: request.Model,
IncludeUsage: request.StreamOptions != nil && request.StreamOptions.IncludeUsage,
}
// Optional RAG: unified retrieval path shared with the old /chat-docs route.
// Enabled when any of the following is true:
// - Request header `X-Retrieval: 1` or body field `retrieval=true`
// - Header `X-Retrieval-Store` specifies a store
knowledge := c.retrieveKnowledgeIfEnabled(
question,
retrievalOwner(authUser),
c.retrievalStore(),
c.GetAcceptLanguage(),
)
// Resolve the route for failover (may have fallback providers), sized to the
// prompt: if this prompt is bigger than the provider we would normally use
// can hold, we route it to one that can rather than refusing it. See
// routeForPrompt — a too-large prompt is a fact about the PROVIDER, not the
// model, and the fallback chain already knows who else serves it.
promptTokens, _ := model.GetTokenSize(request.Model, question)
route := routeForPrompt(request.Model, orgId, promptTokens)
// Call the model provider, cascading to the route's alternates on a refusal
// that is about the vendor rather than about this request.
//
// Every routed model takes this path, not just the ones with fallbacks
// declared. A single-provider model has no alternate to move to, but it
// still earns the rest: its vendor gets demoted when it refuses, the refusal
// is recorded where someone reads it, and "everyone refused" comes back as
// an honest 503 naming the reason instead of an upstream 402 telling the
// customer THEY are out of money.
// EVERY BYTE OF THE BODY IS WRITTEN IN HERE. A fasthttp response is produced
// inside a stream callback rather than into a writer the handler holds, so the
// completion, its bookkeeping and its final chunk all have to run within one
// call. They run in the order they already ran — this is a wrap, not a
// reordering.
//
// `answered` carries what a bare `return` used to. Those returns are now
// returns from a closure, which would fall through to the judge below instead
// of skipping it. (Not `served` — that name is a type in this package.)
// Built HERE, not inside complete: it reads the caller's IP and holds it, and
// a provider beaten in a race is billed long after this handler is done with
// the request that named it.
billRaced := c.billRaced(request.Model, authUser, isPremium, request.Stream, requestId, requestStartTime)
// What the body reads from the request, read now. A streamed body is produced
// after this handler returns and fiber has released the request, so complete
// reads this and never the controller (base.go snapshot).
snap := c.takeSnapshot(authUser)
// complete produces the whole answer into out and returns what went wrong, if
// anything. It renders nothing itself: the caller knows whether a status can
// still be sent.
//
// It settles `own`, never `hold`. A streamed answer finishes after this handler
// has returned, and the handler's deferred release would win the one settle first
// and leave the ledger blind to what the stream cost; so a stream takes the hold
// from the handler below, and complete must still be holding it when it does.
own := hold
answered := false
complete := func(out io.Writer) error {
writer.out = out
var modelResult *model.ModelResult
var actualProvider served
var tried []attempt
// Fast mode, when the reservation covered it. Nil is the cascade, which
// is what every ordinary request gets.
var race *fan
if wide > 1 {
race = &fan{
n: wide,
to: out,
fork: func(o io.Writer) io.Writer { return writer.fork(o) },
adopt: writer.adopt,
bill: billRaced,
}
}
if route != nil {
modelResult, actualProvider, tried, err = ask{
ctx: snap.ctx,
route: route,
org: ledger,
model: request.Model,
primary: provider,
question: question,
history: history,
knowledge: knowledge,
lang: snap.lang,
writer: writer,
prompt: promptTokens,
fan: race,
sent: func() bool { return writer.StreamSent },
prior: familyRefused,
}.serve()
} else {
// Model absent from the route table: call the provider auth resolved.
var modelProvider model.ModelProvider
modelProvider, err = provider.GetModelProvider(snap.lang)
if err != nil {
return serverError("Failed to get model provider: %s", err.Error())
}
modelResult, err = modelProvider.QueryText(question, writer, history, "", knowledge, nil, snap.lang)
actualProvider = served{provider.Name, provider.Origin(), provider}
}
// Every vendor that refused goes in the ledger, whether or not one of them
// eventually served. A failover nobody can see leaves the empty account
// empty. The family's own refusal is already recorded above, so skip it here.
if n := len(familyRefused); len(tried) > n {
recordRefusals(snap, request.Model, tried[n:], authUser, isPremium, request.Stream, requestId, requestStartTime)
}
if err != nil {
// Record failed usage
if authUser != nil {
errRecord := &usageRecord{
Owner: ledger,
Model: request.Model,
Provider: actualProvider.name,
Origin: actualProvider.origin,
Premium: isPremium,
Stream: request.Stream,
Status: "error",
ErrorMsg: err.Error(),
ClientIP: snap.ip,
RequestID: requestId,
}
errRecord.PromptTokens, errRecord.CompletionTokens =
spent(modelResult, request.Model, promptTokens, writer.MessageString(), err)
errRecord.TotalTokens = errRecord.PromptTokens + errRecord.CompletionTokens
errRecord.bind(snap.ctx, authUser)
errRecord.BYO, errRecord.Account = providerBYO(provider, authUser)
recordUsage(errRecord)
recordTrace(snap.ctx, errRecord, requestStartTime)
}
return err
}
// Record successful usage (actualProvider reflects which provider served the request)
if authUser != nil {
successRecord := &usageRecord{
Owner: ledger,
Organization: authUser.Owner,
Model: request.Model,
Provider: actualProvider.name,
Origin: actualProvider.origin,
PromptTokens: modelResult.PromptTokenCount,
CacheReadTokens: modelResult.CacheReadTokenCount,
CacheWriteTokens: modelResult.CacheWriteTokenCount,
CompletionTokens: modelResult.ResponseTokenCount,
TotalTokens: modelResult.TotalTokenCount,
Currency: "USD",
Premium: isPremium,
Stream: request.Stream,
Status: "success",
ClientIP: snap.ip,
RequestID: requestId,
}
successRecord.bind(snap.ctx, authUser)
// Whether this call was "bring your own key" is a property of the row
// that SPENT a credential, not of the row auth resolved before failover
// moved the request. Reading the latter is how a call served on the
// platform's key gets billed as BYO — 1% instead of the token cost, with
// the platform eating the upstream.
successRecord.BYO, successRecord.Account = providerBYO(actualProvider.row, authUser)
recordUsage(successRecord)
recordTrace(snap.ctx, successRecord, requestStartTime)
// Settle the reservation with the ACTUAL cost (this works identically for
// streaming and non-streaming non-tool responses — both have real token
// counts here from the QueryText pipeline).
own.settle(calculateCostCentsWithCache(request.Model, modelResult.PromptTokenCount, modelResult.ResponseTokenCount, modelResult.CacheReadTokenCount, modelResult.CacheWriteTokenCount))
}
// Handle response based on streaming mode
if !request.Stream {
answer := writer.MessageString()
response := openai.ChatCompletionResponse{
ID: "chatcmpl-" + requestId,
Object: "chat.completion",
Created: util.GetCurrentUnixTime(),
Model: request.Model,
Choices: []openai.ChatCompletionChoice{
{
Index: 0,
Message: openai.ChatCompletionMessage{
Role: "assistant",
Content: answer,
},
FinishReason: openai.FinishReasonStop,
},
},
Usage: openai.Usage{
PromptTokens: modelResult.PromptTokenCount,
CompletionTokens: modelResult.ResponseTokenCount,
TotalTokens: modelResult.TotalTokenCount,
},
}
jsonResponse, err := json.Marshal(response)
if err != nil {
return serverError("%s", err.Error())
}
// Only the non-streaming answer is written here, and it runs inside the
// handler, where the request is still ours.
c.answerBody(jsonResponse)
} else {
err = writer.Close(
modelResult.PromptTokenCount,
modelResult.ResponseTokenCount,
modelResult.TotalTokenCount,
)
if err != nil {
return err
}
}
answered = true
return nil
}
// LLM-as-a-judge: score THIS served (prompt, response) into a dense quality reward
// for the enso router. It reads the ANSWER, so it runs where the answer is finished
// — which for a stream is inside the callback, not after it: the callback has not
// run when SendStreamWriter returns, so out here MessageString() is still empty and
// every streamed turn would be scored on nothing. It never adds latency either way;
// the hook only spawns a goroutine after cheap gates, and is a no-op unless
// ROUTER_JUDGE_ENABLED. The prompt and response are passed transiently — never
// persisted (see router_judge.go).
agent, country := c.Header("User-Agent"), Country(c.Ctx)
judge := func() {
if answered && routingRecorded {
judgeRoutedResponse(agent, orgId, requestId, country, request.Model, routedTask, question, writer.MessageString())
}
}
if request.Stream {
// THE FIRST BYTE DECIDES THE STATUS. Handing zip the stream commits a 200, so
// the answer is produced into `unsent` first and this handler waits for its first
// byte. A request every provider refused before a byte was written is answered
// with the status that says so; one that has begun is handed to the stream with
// what it already said, and a failure after that goes out as an error event.
out := newUnsent()
var failed error
done := make(chan struct{})
hold = nil // the stream settles it: complete with the cost, or at zero if it failed
go func() {
defer close(done)
defer own.settle(0)
failed = complete(out)
judge()
}()
select {
case <-out.first:
case <-done:
}
if !out.opened() && failed != nil {
c.ResponseModelFailure(failed)
return
}
// The headers go on before the callback: the first chunk commits the status
// line, and nothing can be added to it afterwards.
c.SetHeader("Content-Type", "text/event-stream")
c.SetHeader("Cache-Control", "no-cache")
c.SetHeader("Connection", "keep-alive")
_ = c.SendStreamWriter(func(bw *bufio.Writer) {
attached := out.attach(bw)
<-done
if failed != nil && attached == nil {
streamOpenAIError(bw, failed)
}
})
return
}
// Nothing is streamed: OpenAIWriter accumulates into MessageBuf and the one JSON
// body is written by complete itself, so there is no stream to name.
if err := complete(io.Discard); err != nil {
c.ResponseModelFailure(err)
}
judge()
}
// ListModels returns the list of available models from the routing table.
//
// PUBLIC BY DESIGN, AND IT DOES NOT AUTHENTICATE — that is the whole contract, so it
// is stated here rather than left to be inferred. The catalogue is the same for
// everyone (listAvailableModels takes no principal), docs.hanzo.ai fetches it from
// the browser, and every policy layer around it says so: the authz filter lists
// "models" as public, filter_balance does not gate it, the rate limiter excludes it,
// and cloud's spend.Reachable carries /v1/models/ as "the model catalog the shell
// reads for discovery".
//
// SO THE Authorization HEADER IS NOT AN ADMISSION CHECK HERE. It is read for ONE
// thing — annotating gated SKUs with the caller's own access standing — and
// annotation degrades to nothing when there is no verified principal.
//
// A credential that is presented is verified, never merely shape-checked: a caller
// using /v1/models to ask "is my key working?" gets an honest answer, because a key
// that does not verify annotates nothing rather than reading as accepted.
//
// @Title ListModels
// @Tag OpenAI Compatible API
// @Description Returns a list of all available models. Public — no authentication.
// @Success 200 {object} object
// @router /models [get]
func (c *ApiController) ListModels() {
// Gated (limited-preview) SKUs carry the caller's access standing
// (waitlist|requested|granted), so the client can show "request access" vs
// "granted" without a second call.
//
// The ONLY use of the credential on this route, and it is best-effort:
// principalUser returns nil unless the request carries a VERIFIED principal (a
// session, a signature-checked JWT, or a key IAM resolved). So an absent, expired
// or forged credential simply yields the un-annotated public catalogue — never a
// 401, and never another caller's standing.
jsonResponse, err := modelListing(c.principalUser())
if err != nil {
c.ResponseError(err.Error())
return
}
c.SetHeader("Content-Type", "application/json")
c.Bytes(http.StatusOK, jsonResponse)
}
// proxyToolRequestAnthropic handles tool-calling requests for Claude/Anthropic
// providers by converting the OpenAI format to Anthropic Messages API format
// and converting the response back.
// sku is the model the CALLER named. request.Model already carries the upstream's
// own id by the time this runs, so it is what the outbound Anthropic body is built
// from and nothing else: the envelope, the usage row and the price all read sku,
// which is the model this call was sold as.
func (c *ApiController) proxyToolRequestAnthropic(
provider *object.Provider,
request *openai.ChatCompletionRequest,
sku string,
requestStartTime time.Time,
authUser *iam.User,
isPremium bool,
orgId string,
requestId string,
hold *budgetHold,
) {
// The same wallet ChatCompletions gated and reserved on, re-derived from the same
// credential rather than threaded, so the two cannot be given different arguments.
ledger := c.billingOrg(authUser)
baseURL := provider.ProviderUrl
if baseURL == "" {
baseURL = "https://api.anthropic.com"
}
baseURL = strings.TrimRight(baseURL, "/")
// Convert OpenAI messages to Anthropic format
var systemPrompt string
anthropicMessages := []map[string]any{}
for _, msg := range request.Messages {
if msg.Role == "system" {
systemPrompt = msg.Content
continue
}
anthropicMsg := map[string]any{
"role": msg.Role,
}
if msg.Role == "tool" {
// Tool result message
anthropicMsg["role"] = "user"
anthropicMsg["content"] = []map[string]any{
{
"type": "tool_result",
"tool_use_id": msg.ToolCallID,
"content": msg.Content,
},
}
} else if len(msg.ToolCalls) > 0 {
// Assistant message with tool calls
content := []map[string]any{}
if msg.Content != "" {
content = append(content, map[string]any{
"type": "text",
"text": msg.Content,
})
}
for _, tc := range msg.ToolCalls {
var inputObj any
_ = json.Unmarshal([]byte(tc.Function.Arguments), &inputObj)
if inputObj == nil {
inputObj = map[string]any{}
}
content = append(content, map[string]any{
"type": "tool_use",
"id": tc.ID,
"name": tc.Function.Name,
"input": inputObj,
})
}
anthropicMsg["content"] = content
} else if len(msg.MultiContent) > 0 {
content := []map[string]any{}
for _, part := range msg.MultiContent {
if part.Type == openai.ChatMessagePartTypeText {
content = append(content, map[string]any{
"type": "text",
"text": part.Text,
})
}
}
anthropicMsg["content"] = content
} else {
anthropicMsg["content"] = msg.Content
}
anthropicMessages = append(anthropicMessages, anthropicMsg)
}
// Convert OpenAI tools to Anthropic tool format
anthropicTools := []map[string]any{}
for _, tool := range request.Tools {
if tool.Type == openai.ToolTypeFunction {
anthropicTool := map[string]any{
"name": tool.Function.Name,
"description": tool.Function.Description,
}
if tool.Function.Parameters != nil {
var params any
raw, _ := json.Marshal(tool.Function.Parameters)
_ = json.Unmarshal(raw, &params)
anthropicTool["input_schema"] = params
} else {
anthropicTool["input_schema"] = map[string]any{
"type": "object",
"properties": map[string]any{},
}
}
anthropicTools = append(anthropicTools, anthropicTool)
}
}
// Build Anthropic request. Anthropic caches only a prompt the request marks, so
// the top-level cache_control caches this one up to its last block (claudeCache):
// the next turn of the conversation reads that prefix at the cache rate.
anthropicReq := map[string]any{
"model": request.Model,
"messages": anthropicMessages,
"max_tokens": 4096,
"tools": anthropicTools,
"cache_control": ephemeral,
}
if systemPrompt != "" {
anthropicReq["system"] = systemPrompt
}
if request.MaxTokens > 0 {
anthropicReq["max_tokens"] = request.MaxTokens
}
if request.Temperature > 0 {
anthropicReq["temperature"] = request.Temperature
}
// Always fetch the FULL (non-streamed) Anthropic response so the body is
// parseable JSON carrying token usage for billing. If the client requested
// streaming, the converted result is re-emitted as SSE below. Previously
// stream=true made io.ReadAll+json.Unmarshal fail on the SSE body, so streamed
// tool calls errored AND were never billed.
body, err := json.Marshal(anthropicReq)
if err != nil {
c.ResponseError(fmt.Sprintf("Failed to marshal Anthropic request: %s", err.Error()))
return
}
req, err := http.NewRequest(http.MethodPost, baseURL+"/v1/messages", bytes.NewReader(body))
if err != nil {
c.ResponseError(fmt.Sprintf("Failed to create Anthropic request: %s", err.Error()))
return
}
req.Header.Set("Content-Type", "application/json")
upstream.Authorize(req, provider)
req.Header.Set("anthropic-version", "2023-06-01")
client := &http.Client{Timeout: 120 * time.Second}
resp, err := client.Do(req)
if err != nil {
c.ResponseError(fmt.Sprintf("Anthropic request failed: %s", err.Error()))
return
}
defer resp.Body.Close()
// Read full Anthropic response
respBody, err := io.ReadAll(resp.Body)
if err != nil {
c.ResponseError(fmt.Sprintf("Failed to read Anthropic response: %s", err.Error()))
return
}
if resp.StatusCode != http.StatusOK {
c.ResponseFailure(relay(sku, provider.Name, resp.StatusCode, respBody))
return
}
// Parse Anthropic response
var anthropicResp struct {
ID string `json:"id"`
Content []struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Input json.RawMessage `json:"input,omitempty"`
} `json:"content"`
StopReason string `json:"stop_reason"`
Usage AnthropicUsage `json:"usage"`
}
if err := json.Unmarshal(respBody, &anthropicResp); err != nil {
c.ResponseError(fmt.Sprintf("Failed to parse Anthropic response: %s", err.Error()))
return
}
// The prompt as every record counts it: read fresh, read from the cache and
// written to it, each part billed at its own rate.
used := anthropicResp.Usage.billed(sku)
// Convert Anthropic response to OpenAI format
var contentText strings.Builder
var toolCalls []openai.ToolCall
toolCallIdx := 0
for _, block := range anthropicResp.Content {
switch block.Type {
case "text":
contentText.WriteString(block.Text)
case "tool_use":
tc := openai.ToolCall{
Index: &toolCallIdx,
ID: block.ID,
Type: openai.ToolTypeFunction,
Function: openai.FunctionCall{
Name: block.Name,
Arguments: string(block.Input),
},
}
toolCalls = append(toolCalls, tc)
toolCallIdx++
}
}
finishReason := openai.FinishReasonStop
if anthropicResp.StopReason == "tool_use" {
finishReason = openai.FinishReasonToolCalls
}
openaiResp := openai.ChatCompletionResponse{
ID: "chatcmpl-" + requestId,
Object: "chat.completion",
Created: util.GetCurrentUnixTime(),
Model: sku,
Choices: []openai.ChatCompletionChoice{
{
Index: 0,
Message: openai.ChatCompletionMessage{
Role: "assistant",
Content: contentText.String(),
ToolCalls: toolCalls,
},
FinishReason: finishReason,
},
},
Usage: openai.Usage{
PromptTokens: used.prompt(),
CompletionTokens: used.out,
TotalTokens: used.prompt() + used.out,
PromptTokensDetails: &openai.PromptTokensDetails{CachedTokens: used.read},
},
}
// Record usage
if authUser != nil {
successRecord := &usageRecord{
Owner: ledger,
Organization: authUser.Owner,
Model: sku,
Provider: provider.Name,
Origin: provider.Origin(),
PromptTokens: used.prompt(),
CacheReadTokens: used.read,
CacheWriteTokens: used.write,
CompletionTokens: used.out,
TotalTokens: used.prompt() + used.out,
Currency: "USD",
Premium: isPremium,
Stream: false,
Status: "success",
ClientIP: c.Fiber().IP(),
RequestID: requestId,
}
successRecord.bind(c.Context(), authUser)
successRecord.BYO, successRecord.Account = providerBYO(provider, authUser)
recordUsage(successRecord)
recordTrace(c.Context(), successRecord, requestStartTime)
}
// The same model the usage row above is priced from. recordUsage debits what
// usageCostCents makes of record.Model, so a hold settled against a different
// model would hold one number and bill another.
hold.settle(calculateCostCentsWithCache(sku, used.prompt(), used.out, used.read, used.write))
jsonResponse, err := json.Marshal(openaiResp)
if err != nil {
c.ResponseError(err.Error())
return
}
if request.Stream {
// Client asked for streaming: emit the converted completion as a single
// SSE chunk followed by [DONE], so OpenAI SDK clients consuming a stream
// still work while billing used the real (full-response) token usage.
c.SetHeader("Content-Type", "text/event-stream")
c.SetHeader("Cache-Control", "no-cache")
c.SetHeader("Connection", "keep-alive")
chunk := map[string]any{
"id": openaiResp.ID,
"object": "chat.completion.chunk",
"created": openaiResp.Created,
"model": openaiResp.Model,
"choices": []map[string]any{{
"index": 0,
"delta": map[string]any{
"role": "assistant",
"content": contentText.String(),
"tool_calls": toolCalls,
},
"finish_reason": finishReason,
}},
}
// The stream owns the connection from here: zip hands the handler a writer
// and flushes each frame as it is written, so a chunk reaches the client
// when it is produced rather than when the handler returns.
_ = c.SendStreamWriter(func(w *bufio.Writer) {
if chunkJSON, mErr := json.Marshal(chunk); mErr == nil {
_, _ = fmt.Fprintf(w, "data: %s\n\n", string(chunkJSON))
}
_, _ = fmt.Fprint(w, "data: [DONE]\n\n")
})
return
}
c.answerBody(jsonResponse)
}
// planUse is what a covered call used of its plan, in nano-dollars: its list price,
// what the caller would have paid; for a model sold at zero, what its paid upstream
// cost us (planNanos), since its list price says nothing about what it spent.
func planUse(record *usageRecord) int64 {
if billed := usageMargin(record).BilledNano; billed > 0 {
return billed
}
return record.planNanos
}