mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
A call through the gateway bills the price of the arm that answered and the platform's fee on top, and nothing else: 5.5% on Hanzo's accounts, 1% on the org's own key, both the host's settings (object.Fee: hanzo, hanzo_byok). A vendor's fee is part of the cost: a call OpenRouter's credit bought cost its stated price plus 5.5%, and the platform's fee goes on that. - Enso answers bill their cost of record (X-Hanzo-Cogs and the buying vendor's fee) times the platform's fee; a maker's answer the same; a resold OpenRouter SKU its vendor price times both fees. An answer that states no cost bills the SKU's listed ceiling. A call billed below its cost writes loss_nano and a warning. - hanzo.cloud_usage gains reasoning, reasoning_tokens, served, vendor, arms and loss_nano: each Enso row names the arm that answered, its rate at the time and the tokens it was paid for, and each arm that failed first. - rates.go prices every Enso SKU at each depth: from (its cheapest arm), up to (its dearest, a long prompt included), and typical (its measured cost a token over the last day, or its last 50 calls), the arms read from the engine's admin catalog. /v1/models lists the typical rate with from, up_to, fee and window beside it; Prices, Price and Blend serve the rest. The board's ledger counts answers, failovers, OpenRouter-bought answers, fee and losses by model and vendor.
2954 lines
128 KiB
Go
2954 lines
128 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
// Portions Copyright 2025 The OpenAgent Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"crypto/subtle"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/hanzoai/account"
|
|
"github.com/hanzoai/decimal"
|
|
|
|
"github.com/hanzoai/ai/address"
|
|
"github.com/hanzoai/ai/conf"
|
|
"github.com/hanzoai/ai/funding"
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/hanzoai/ai/model"
|
|
"github.com/hanzoai/ai/object"
|
|
"github.com/hanzoai/ai/upstream"
|
|
"github.com/hanzoai/ai/util"
|
|
"github.com/hanzoai/go-openai"
|
|
)
|
|
|
|
// getUserBalance returns the current balance for a user by fetching from Commerce.
|
|
// Balance is mutable financial state (not identity) so it is never read from the
|
|
// JWT — always checked against the source of truth. Caching is handled by the
|
|
// router-level BalanceGate (routers/filter_balance.go); this controller-level
|
|
// call is a defense-in-depth backstop and does not maintain its own cache.
|
|
// The userId should be in "owner/name" format (e.g., "hanzo/alice").
|
|
// getUserBalance returns the available balance (in dollars) for an org. Billing
|
|
// is per-org: orgKey is the IAM org slug, used both as the balance destination
|
|
// key (?user=) and as the namespace selector (X-Org-Id), matching the gate
|
|
// and the per-org credit. Without the header, commerce's service-token path
|
|
// defaults to the "hanzo" namespace and a per-org credit is invisible.
|
|
func getUserBalance(subject, namespace string) (float64, error) {
|
|
// Native path: a co-resident host (cloud) reads the subject's wallet balance
|
|
// DIRECTLY from the in-process finance ledger — no HTTP. Cents → dollars.
|
|
if r := object.BalanceReader(); r != nil {
|
|
cents, err := r(context.Background(), subject, namespace, "usd")
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return float64(cents) / 100, nil
|
|
}
|
|
commerceEndpoint := conf.GetConfigString("commerceEndpoint")
|
|
if commerceEndpoint == "" {
|
|
return 0, fmt.Errorf("commerceEndpoint is not configured")
|
|
}
|
|
commerceEndpoint = strings.TrimRight(commerceEndpoint, "/")
|
|
commerceToken := conf.GetConfigString("commerceToken")
|
|
|
|
// Per global rule: /v1/ only, never /api/.
|
|
// All commerce endpoints live under /v1/.
|
|
// subject (?user=) is the per-user/per-org billing key; namespace
|
|
// (X-Org-Id) is the org. Both must match the gate and the usage debit.
|
|
reqURL := fmt.Sprintf("%s/v1/billing/balance?user=%s¤cy=usd", commerceEndpoint, url.QueryEscape(subject))
|
|
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
req, err := http.NewRequest(http.MethodGet, reqURL, nil)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("Commerce request build failed: %w", err)
|
|
}
|
|
if commerceToken != "" {
|
|
req.Header.Set("Authorization", "Bearer "+commerceToken)
|
|
}
|
|
// Scope the service-token call to this org's namespace.
|
|
req.Header.Set("X-Org-Id", namespace)
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("Commerce request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return 0, fmt.Errorf("Commerce returned status %d", resp.StatusCode)
|
|
}
|
|
|
|
var result struct {
|
|
Available int64 `json:"available"`
|
|
}
|
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
|
return 0, fmt.Errorf("failed to parse Commerce response: %w", err)
|
|
}
|
|
|
|
// Convert cents to dollars for backward compatibility with existing balance > 0 check
|
|
balanceDollars := float64(result.Available) / 100.0
|
|
|
|
return balanceDollars, nil
|
|
}
|
|
|
|
// isJwtToken checks if a token looks like a JWT (3 base64 segments separated by dots).
|
|
func isJwtToken(token string) bool {
|
|
parts := strings.Split(token, ".")
|
|
return len(parts) == 3 && len(parts[0]) > 10 && len(parts[1]) > 10
|
|
}
|
|
|
|
// isIAMApiKey checks if a token is an IAM-issued API key (sk- prefix) — the
|
|
// confidential half of the pair IAM mints, and the only one that authenticates.
|
|
func isIAMApiKey(token string) bool {
|
|
return strings.HasPrefix(token, "sk-")
|
|
}
|
|
|
|
// isPublishableKey checks if a token is a publishable API key (pk- prefix).
|
|
// Publishable keys are safe for client-side use and can only access read-only endpoints.
|
|
func isPublishableKey(token string) bool {
|
|
return strings.HasPrefix(token, "pk-")
|
|
}
|
|
|
|
// resolveProviderFromJwt validates a hanzo.id JWT token and returns the
|
|
// appropriate model provider for the requested model, plus the translated
|
|
// upstream model name.
|
|
//
|
|
// requested is the raw X-Org-Id the caller asked to act in ("" for none). This is
|
|
// the ONE auth path that can honor an org switch, because it is the one that
|
|
// holds the signed `orgs` claim proving membership — the ledger is resolved here,
|
|
// from those claims, rather than re-parsing the token downstream.
|
|
func resolveProviderFromJwt(ctx context.Context, token string, requested string, requestedModel string, lang string) (*object.Provider, *iam.User, string, error) {
|
|
// Signature + issuer/audience validation (never raw iam.ParseJwtToken), so a
|
|
// token minted for a foreign app/issuer cannot authenticate a paid request.
|
|
claims, err := object.ParseAndValidateJWT(token)
|
|
if err != nil {
|
|
return nil, nil, "", authError("invalid access token: %s", err.Error())
|
|
}
|
|
|
|
user := &claims.User
|
|
// An explicit org the signed claim does not cover is refused, not silently
|
|
// billed to the caller's personal wallet. Unauthorized and nonexistent are one
|
|
// answer so the header cannot be used to enumerate orgs.
|
|
effective, orgErr := account.EffectiveOrg(user.Owner, claims.Orgs, requested)
|
|
if orgErr != nil {
|
|
return nil, nil, "", forbiddenError("organization %q is not available to this principal", requested)
|
|
}
|
|
ledger := account.LedgerOrg(effective, user.Owner, util.IsSuperAdmin(user))
|
|
return resolveProviderForUser(ctx, user, ledger, requestedModel, lang)
|
|
}
|
|
|
|
// resolveProviderFromIAMKey validates an IAM API key (sk-{accessKey})
|
|
// and returns the model provider + user, same as JWT path.
|
|
//
|
|
// An IAM key carries no signed `orgs` claim, so it can never switch org: it bills
|
|
// the org that owns the key, which is its home org.
|
|
func resolveProviderFromIAMKey(ctx context.Context, apiKey string, requestedModel string, lang string) (*object.Provider, *iam.User, string, error) {
|
|
// The whole token, prefix included, IS the accessKey IAM resolves.
|
|
accessKey := apiKey
|
|
|
|
user, err := getUserByAccessKey(accessKey)
|
|
if err != nil {
|
|
// IAM may return "password or code is incorrect" for service-account users
|
|
// (cloud-agent, etc.) due to a known IAM deployment quirk where the
|
|
// deployed binary handles certain user records differently. As a safe
|
|
// fallback, we check the key against the CLOUD_AGENT_KEY KMS secret
|
|
// (env CLOUD_AGENT_KEY as fallback). If it matches, we construct a
|
|
// minimal user identity and let the Commerce balance check validate
|
|
// the request as normal — so no billing bypass occurs.
|
|
if fallbackUser := tryCloudAgentKeyFallback(apiKey); fallbackUser != nil {
|
|
// Never log the API key (even masked) — owner/name identify the
|
|
// fallback identity for debugging without leaking the credential.
|
|
log.Warn("[iam-fallback] IAM returned %q; using cloud-agent fallback identity (owner=%s name=%s)",
|
|
err.Error(), fallbackUser.Owner, fallbackUser.Name)
|
|
return resolveProviderForUser(ctx, fallbackUser, fallbackUser.Owner, requestedModel, lang)
|
|
}
|
|
return nil, nil, "", authError("API key validation failed: %s", err.Error())
|
|
}
|
|
if user == nil {
|
|
return nil, nil, "", authError("invalid API key")
|
|
}
|
|
|
|
return resolveProviderForUser(ctx, user, user.Owner, requestedModel, lang)
|
|
}
|
|
|
|
// tryCloudAgentKeyFallback checks whether apiKey matches the known cloud-agent
|
|
// service key stored in KMS (secret name "CLOUD_AGENT_KEY") with an env var
|
|
// fallback. Returns a minimal *iam.User on match, nil otherwise.
|
|
// This is intentionally narrow: only the exact key stored in KMS is accepted.
|
|
func tryCloudAgentKeyFallback(apiKey string) *iam.User {
|
|
// Try KMS first
|
|
var knownKey string
|
|
if v, err := object.GetKMSSecret("CLOUD_AGENT_KEY"); err == nil && v != "" {
|
|
knownKey = strings.TrimSpace(v)
|
|
}
|
|
// Env var fallback for local dev / bootstrap
|
|
if knownKey == "" {
|
|
knownKey = strings.TrimSpace(os.Getenv("CLOUD_AGENT_KEY"))
|
|
}
|
|
// A secret is compared in constant time, as the two other places in this module
|
|
// that compare one already do: an ordinary != stops at the first differing
|
|
// byte, and how long it took is a measurement a caller can make.
|
|
if knownKey == "" || subtle.ConstantTimeCompare([]byte(apiKey), []byte(knownKey)) != 1 {
|
|
return nil
|
|
}
|
|
// This identity is assembled HERE rather than read from IAM, so it must state
|
|
// what IAM would have stated for it. Left unsaid it read as a PERSON:
|
|
// account.Payer's shape rule hands a person in the signup org a personal
|
|
// wallet, and "hanzo" IS the signup org — so every call on this key addressed
|
|
// hanzo/cloud-agent, a wallet no funding path can name, which reads $0 while
|
|
// the org's balance sits one key away.
|
|
//
|
|
// The LEDGER is named outright and the type carries only attribution. In this
|
|
// org an org account is the platform's own balance, so which account pays is a
|
|
// statement the assembling code makes on the strength of the KMS secret it just
|
|
// verified — not something inferred from a class, which is a thing rows can
|
|
// also carry. This is the same reason the provider-key and widget identities
|
|
// name theirs.
|
|
return &iam.User{
|
|
Owner: account.SignupOrg,
|
|
Name: "cloud-agent",
|
|
Type: iam.Machine, // attribution: no person behind this call
|
|
BillingAccount: account.Org(account.SignupOrg).String(),
|
|
}
|
|
}
|
|
|
|
// resolveProviderForUser is the shared logic for JWT and API key auth paths.
|
|
// Given a validated user, resolves the model route and provider.
|
|
//
|
|
// ledger is the org that PAYS for this request (account.LedgerOrg). It selects both
|
|
// the org's own BYOK provider and the wallet the balance gate reads, so a request
|
|
// billed to an org is served with that org's connected key — the two cannot name
|
|
// different tenants.
|
|
func resolveProviderForUser(ctx context.Context, user *iam.User, ledger string, requestedModel string, lang string) (*object.Provider, *iam.User, string, error) {
|
|
// Look up the model in the static routing table. A valid caller asking for
|
|
// an unknown model is a client error (400), not an auth failure.
|
|
route := resolveModelRoute(requestedModel)
|
|
if route == nil {
|
|
return nil, user, "", modelError(
|
|
"model %q is not available. Use GET /v1/models to list available models",
|
|
requestedModel,
|
|
)
|
|
}
|
|
|
|
// Fetch the provider entry that holds API keys/URLs for this upstream. Prefer
|
|
// the org's OWN custom provider (BYOK) if it configured one, else the global
|
|
// built-in provider on api.hanzo.ai. Returns a shallow copy, safe to mutate. A
|
|
// missing/unconfigured provider is a server-side misconfiguration (500).
|
|
provider, err := object.GetModelProviderByNameForOrg(ledger, route.providerName)
|
|
if err != nil {
|
|
return nil, user, "", serverError("failed to get provider %q: %s", route.providerName, err.Error())
|
|
}
|
|
if provider == nil {
|
|
return nil, user, "", serverError("provider %q not configured in database", route.providerName)
|
|
}
|
|
|
|
// Prepaid-balance gate. Extracted into enforceBalanceGate so the provider-key
|
|
// (sk-) path in authResolveProvider enforces the IDENTICAL policy — no auth
|
|
// path can drift (M1).
|
|
if gateErr := enforceBalanceGate(ctx, user, ledger, requestedModel); gateErr != nil {
|
|
return nil, user, "", gateErr
|
|
}
|
|
|
|
return provider, user, route.upstreamModel, nil
|
|
}
|
|
|
|
// enforceBalanceGate applies the prepaid-balance policy to a resolved billing
|
|
// principal and stamps user.Balance. A positive prepaid balance is required for
|
|
// EVERY model — there is no implicit free allowance and no per-model tier: a
|
|
// subject's credit is only ever what it paid or was explicitly granted (commerce
|
|
// owns the one grant endpoint). It returns a typed billingError (402) when the
|
|
// balance is not positive, a serverError (500) when the balance cannot be verified
|
|
// (fail-closed — never grant on a lookup transport error), or nil to proceed.
|
|
//
|
|
// It is the SINGLE gate shared by the JWT/IAM path (resolveProviderForUser) and
|
|
// the provider-key (sk-) path (authResolveProvider), so no auth path can drift
|
|
// (M1). There is NO exempt path: every principal is gated on a positive prepaid
|
|
// balance. subject is the per-namespace billing account the gate read, the budget
|
|
// reservation, and the usage debit all key on.
|
|
//
|
|
// ledger is the org this request SPENDS FROM — account.LedgerOrg, via c.billingOrg or
|
|
// the claims the JWT resolver already holds. It is passed rather than derived
|
|
// from user.Owner so the gate reads the SAME wallet the debit writes: keying the
|
|
// gate on the selected org and the debit on the home org would check one balance
|
|
// and drain another. An empty ledger means the caller could not resolve one, and
|
|
// falls back to the home org — the behavior before the org switch existed.
|
|
func enforceBalanceGate(ctx context.Context, user *iam.User, ledger string, requestedModel string) error {
|
|
if user == nil {
|
|
return nil
|
|
}
|
|
if ledger == "" {
|
|
ledger = user.Owner
|
|
}
|
|
orgKey := ledger // namespace (X-Org-Id): the org tenant whose ledger pays
|
|
subject := user.PayerSubject(ledger)
|
|
|
|
// A model priced at zero has nothing for this gate to refuse. Everything below
|
|
// stops a call that cannot be paid for — the caller's wallet, and our own cash
|
|
// behind it — and a route that bills zero on both sides spends neither. Refusing
|
|
// one asks somebody to add funds for a call that will cost $0.00, which is how a
|
|
// free plan came to be unable to reach the free routes we publish for it.
|
|
if costsNothing(requestedModel, ledger) {
|
|
return nil
|
|
}
|
|
// Nor does a request the plan or a free cap pays for: no wallet is asked about it.
|
|
if covered(ctx) {
|
|
return nil
|
|
}
|
|
|
|
// Cash circuit-breaker (internal/funding). Checked BEFORE the balance read
|
|
// because it is a property of OUR bank account, not of the caller's wallet: a
|
|
// caller with a perfectly good platform balance is exactly who spends our cash
|
|
// once upstream promo credit is gone. Disarmed unless a ceiling is configured,
|
|
// so this is a no-op until someone deliberately arms it.
|
|
//
|
|
// Being our bank account, it refuses with supplyError (503) and not with
|
|
// billingError (402). The caller owes nothing, so 402 would send a funded org to
|
|
// top up a balance that is already funded, and its code would put a billing
|
|
// prompt in front of them. The figures stay on our side of the line for the same
|
|
// reason the envelope keeps a buy price out of an answer: they are what we pay to
|
|
// buy inference. The counter is what an operator reads.
|
|
if cash := funding.Current(); cash.Refuse() {
|
|
supplyRefused.WithLabelValues("hanzo", reasonCeiling).Inc()
|
|
return supplyError(
|
|
"paid inference is temporarily unavailable. Your balance is not affected — " +
|
|
"nothing is owed and no action is needed. Retry shortly.",
|
|
)
|
|
}
|
|
|
|
balance, err := getUserBalance(subject, orgKey)
|
|
if err != nil {
|
|
// Balance unverifiable (Commerce down, a rejected service token, or an
|
|
// unset endpoint) → DENY. AI is prepaid: a broken or misconfigured billing
|
|
// backend must never degrade to free, ungated inference. There is no
|
|
// fail-open escape and no exempt principal.
|
|
return serverError("failed to verify account balance: %s", err.Error())
|
|
}
|
|
if balance <= 0 {
|
|
return billingError(
|
|
"model %q requires a positive balance. Your current balance is $%.2f. "+
|
|
"Add funds at https://hanzo.ai/billing",
|
|
requestedModel, balance,
|
|
)
|
|
}
|
|
|
|
user.Balance = balance
|
|
return nil
|
|
}
|
|
|
|
// iamClientCreds resolves this service's confidential-app credentials, in order:
|
|
// env vars (IAM_CLIENT_ID/IAM_CLIENT_SECRET), KMS secrets, then the router config.
|
|
func iamClientCreds() (string, string) {
|
|
clientId := conf.GetConfigString("IAM_CLIENT_ID")
|
|
clientSecret := conf.GetConfigString("IAM_CLIENT_SECRET")
|
|
|
|
// Try KMS if config values are empty or placeholders
|
|
if clientId == "" {
|
|
if v, err := object.GetKMSSecret("IAM_CLIENT_ID"); err == nil && v != "" {
|
|
clientId = v
|
|
}
|
|
}
|
|
if clientSecret == "" {
|
|
if v, err := object.GetKMSSecret("IAM_CLIENT_SECRET"); err == nil && v != "" {
|
|
clientSecret = v
|
|
}
|
|
}
|
|
return clientId, clientSecret
|
|
}
|
|
|
|
// GetUserByAccessKey resolves an sk- IAM API key to its owning user via Hanzo
|
|
// IAM. Exported so the authz filter and the balance gate (package routers)
|
|
// resolve the key path to the same verified principal as the JWT path — ONE
|
|
// credential resolver, one tenant, one billing subject, one IAM transport.
|
|
func GetUserByAccessKey(accessKey string) (*iam.User, error) {
|
|
return getUserByAccessKey(accessKey)
|
|
}
|
|
|
|
// PublishableOrg resolves a publishable pk- to the org that holds it. Exported so
|
|
// the router's tenant resolver answers a page key the same way the controller
|
|
// does — ONE credential resolver, one tenant, one IAM transport, exactly as
|
|
// GetUserByAccessKey is exported for the secret half.
|
|
func PublishableOrg(accessKey string) (string, error) { return publishableOrg(accessKey) }
|
|
|
|
// The floor a key clears before this process will ask IAM about it. IAM mints
|
|
// "{pk|sk}-{live|test}-{random}" (iam internal/keys.Mint), so anything outside
|
|
// this alphabet and length is not a key this estate ever issued and needs no round
|
|
// trip to refuse. The floor is deliberately wider than the mint: it exists to drop
|
|
// junk, not to re-implement the key format, and a key whose shape is plausible is
|
|
// still decided by IAM.
|
|
//
|
|
// The two halves carry their own prefix rather than sharing one pattern, so the
|
|
// publishable resolver still refuses a secret locally and the secret resolver still
|
|
// refuses a publishable — each answers about its own kind of key, and asking the
|
|
// wrong one is a question this process can settle without IAM.
|
|
const keyBody = `-[A-Za-z0-9_-]{8,128}$`
|
|
|
|
var (
|
|
publishableKeyShape = regexp.MustCompile(`^pk` + keyBody)
|
|
secretKeyShape = regexp.MustCompile(`^sk` + keyBody)
|
|
)
|
|
|
|
const (
|
|
// How long a resolved org stays fresh, and how long a refusal is remembered.
|
|
// The refusal is shorter because a key can become valid — it was just minted,
|
|
// or IAM was briefly unreachable — and nothing should stay denied for a full
|
|
// positive lifetime over a blip.
|
|
publishableOrgTTL = 5 * time.Minute
|
|
publishableDenyTTL = 30 * time.Second
|
|
|
|
// A bound on distinct keys remembered at once. The keys arriving here are
|
|
// whatever the internet sends, so the map must not grow with them.
|
|
publishableCacheMax = 8192
|
|
)
|
|
|
|
// publishableAnswer is what the endpoint said about one key, refusals included.
|
|
type publishableAnswer struct {
|
|
org string
|
|
err error
|
|
answered time.Time
|
|
}
|
|
|
|
var (
|
|
publishableMu sync.RWMutex
|
|
publishableCache = map[string]publishableAnswer{}
|
|
)
|
|
|
|
// publishableOrg is resolveOrgFromPublishableKey with a memory, and it is what
|
|
// every caller should use.
|
|
//
|
|
// REMEMBERING THE REFUSALS IS THE POINT. A publishable key ships in the source of
|
|
// a page, so the keys presented here are whatever anyone cares to send. Asking IAM
|
|
// about each one turns a public endpoint into a load generator aimed at the single
|
|
// service every other credential path also depends on — and the way that failure
|
|
// shows up is a pk- request resolving no org, which is the one answer an attacker
|
|
// would like to induce. Two things stop it: a shape check that costs nothing, and
|
|
// a remembered "no" that costs a map read.
|
|
//
|
|
// It is also what makes the two resolvers AGREE. A single request can ask for this
|
|
// org more than once — the tenant resolver and the model resolver both do — and two
|
|
// live round trips can answer differently when IAM is flaky, leaving one request
|
|
// scoped to one org and billed to another. One answer per key, shared.
|
|
func publishableOrg(accessKey string) (string, error) {
|
|
if !publishableKeyShape.MatchString(accessKey) {
|
|
return "", authError("invalid publishable key")
|
|
}
|
|
publishableMu.RLock()
|
|
a, ok := publishableCache[accessKey]
|
|
publishableMu.RUnlock()
|
|
if ok && time.Since(a.answered) <= publishableFresh(a) {
|
|
return a.org, a.err
|
|
}
|
|
org, err := resolveOrgFromPublishableKey(accessKey)
|
|
publishableRemember(accessKey, publishableAnswer{org: org, err: err, answered: time.Now()})
|
|
return org, err
|
|
}
|
|
|
|
// publishableFresh is how long THIS answer stands.
|
|
func publishableFresh(a publishableAnswer) time.Duration {
|
|
if a.err != nil {
|
|
return publishableDenyTTL
|
|
}
|
|
return publishableOrgTTL
|
|
}
|
|
|
|
// publishableRemember stores an answer, dropping what has expired when the map
|
|
// reaches its bound and starting over if that was not enough.
|
|
//
|
|
// STARTING OVER IS INDUCIBLE, and saying otherwise would overstate what this is.
|
|
// Refusals are remembered too, so enough shape-valid keys — none of which IAM has
|
|
// to recognise — fill the bound while still fresh and force the reset, which costs
|
|
// every live key a round trip. The shape floor drops junk for free, but past it an
|
|
// attacker still buys one IAM call per distinct key. This is a latency and
|
|
// duplicate-work optimisation; the ceiling on what reaches IAM is the rate limiter,
|
|
// not this map.
|
|
func publishableRemember(accessKey string, a publishableAnswer) {
|
|
publishableMu.Lock()
|
|
defer publishableMu.Unlock()
|
|
if len(publishableCache) >= publishableCacheMax {
|
|
for k, v := range publishableCache {
|
|
if time.Since(v.answered) > publishableFresh(v) {
|
|
delete(publishableCache, k)
|
|
}
|
|
}
|
|
if len(publishableCache) >= publishableCacheMax {
|
|
publishableCache = map[string]publishableAnswer{}
|
|
}
|
|
}
|
|
publishableCache[accessKey] = a
|
|
}
|
|
|
|
// resolveOrgFromPublishableKey resolves a publishable pk- to the ORG that holds
|
|
// it, via IAM's publishable endpoint (/v1/iam/keys/org) — the exact dual of
|
|
// getUserByAccessKey, which answers only for secret keys and refuses a pk- as
|
|
// key_wrong_door. Same confidential Basic transport, same envelope, same typed
|
|
// refusals; the answer is an org and never a person.
|
|
func resolveOrgFromPublishableKey(accessKey string) (string, error) {
|
|
iamEndpoint := conf.GetConfigString("IAM_URL")
|
|
if iamEndpoint == "" {
|
|
return "", fmt.Errorf("IAM_URL is not configured")
|
|
}
|
|
iamEndpoint = strings.TrimRight(iamEndpoint, "/")
|
|
reqURL := fmt.Sprintf("%s/v1/iam/keys/org?accessKey=%s", iamEndpoint, url.QueryEscape(accessKey))
|
|
clientId, clientSecret := iamClientCreds()
|
|
if clientId == "" || clientSecret == "" {
|
|
return "", fmt.Errorf("IAM client credentials are not configured")
|
|
}
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
req, err := http.NewRequest(http.MethodGet, reqURL, nil)
|
|
if err != nil {
|
|
return "", fmt.Errorf("IAM request build failed: %w", err)
|
|
}
|
|
req.SetBasicAuth(clientId, clientSecret)
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
// The key travels in the query string, so a transport failure carries it:
|
|
// *url.Error prints the whole URL. Redacted before it becomes an error value,
|
|
// because from here it is logged and wrapped further.
|
|
return "", fmt.Errorf("IAM request failed: %s", object.RedactKeys(err.Error()))
|
|
}
|
|
defer resp.Body.Close()
|
|
var result struct {
|
|
Status string `json:"status"`
|
|
Msg string `json:"msg"`
|
|
Code string `json:"code"`
|
|
Data struct {
|
|
Org string `json:"org"`
|
|
} `json:"data"`
|
|
}
|
|
decodeErr := json.NewDecoder(resp.Body).Decode(&result)
|
|
if decodeErr == nil && (result.Code != "" || result.Status != "ok") {
|
|
return "", keyRefusal(result.Code, result.Msg, accessKey)
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", fmt.Errorf("IAM returned status %d and named no reason", resp.StatusCode)
|
|
}
|
|
if decodeErr != nil {
|
|
return "", fmt.Errorf("failed to parse IAM response: %w", decodeErr)
|
|
}
|
|
if result.Data.Org == "" {
|
|
return "", authError("publishable key resolved to no org")
|
|
}
|
|
return result.Data.Org, nil
|
|
}
|
|
|
|
// getUserByAccessKey looks up a user by their IAM API key via Hanzo IAM.
|
|
func getUserByAccessKey(accessKey string) (*iam.User, error) {
|
|
// The dual of the floor publishableOrg applies to a pk-, and it is here rather
|
|
// than at any one call site because this is the ONE place a secret key is
|
|
// resolved: the controller, the balance gate and the authz filter all arrive
|
|
// through it, and a floor at one door is a floor nobody else stands behind.
|
|
//
|
|
// It refuses in IAM's OWN words. A holder who presents a retired spelling, a
|
|
// typo, or another vendor's key needs the one sentence that ends the problem —
|
|
// mint a new key — and which side of the wire decided that is no business of
|
|
// theirs. Saying it here also leaves nothing to probe: a shape this estate never
|
|
// mints and a key it minted and revoked answer identically, so the floor cannot
|
|
// be used to learn the format.
|
|
if !secretKeyShape.MatchString(accessKey) {
|
|
return nil, keyRefusal("key_unknown", "", accessKey)
|
|
}
|
|
|
|
iamEndpoint := conf.GetConfigString("IAM_URL")
|
|
if iamEndpoint == "" {
|
|
return nil, fmt.Errorf("IAM_URL is not configured")
|
|
}
|
|
iamEndpoint = strings.TrimRight(iamEndpoint, "/")
|
|
data, err := answerFor(iamEndpoint, accessKey, func() (json.RawMessage, bool, error) {
|
|
return askForKey(iamEndpoint, accessKey)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var u iam.User
|
|
if err := json.Unmarshal(data, &u); err != nil {
|
|
return nil, fmt.Errorf("failed to parse IAM response: %w", err)
|
|
}
|
|
return &u, nil
|
|
}
|
|
|
|
// askForKey asks IAM which principal a secret key speaks for: the principal's JSON,
|
|
// or IAM's refusal. final says the answer is about the key, so it may be held; a
|
|
// fault in reaching or reading IAM is not.
|
|
func askForKey(iamEndpoint, accessKey string) (json.RawMessage, bool, error) {
|
|
// Per global rule: /v1/ only, never /api/. A path under /api/ is intercepted
|
|
// by the @hanzo/id SPA ingress and returns HTML, which broke API-key
|
|
// resolution once ("invalid character '<'").
|
|
//
|
|
// This is the SECRET-key endpoint, and it is its own route rather than a CRUD
|
|
// read. Resolving a key used to ride on the user read as
|
|
// `get-user?accessKey=` — an authentication boundary reached through a verb
|
|
// whose target was a credential rather than the owner/name that read
|
|
// authorizes on. It resolves an sk- to the full principal; the publishable
|
|
// pk- has a separate endpoint (resolveOrgFromPublishableKey) that yields an org
|
|
// and never a person, so the browser-safe disclosure and this one are never
|
|
// behind a single authorization decision.
|
|
//
|
|
// The segments name things — a key, its principal — and the method says the
|
|
// verb. IAM refuses a new verb-noun address outright; the one sibling that
|
|
// still reads as one (resolve-key, below) is a frozen spelling live
|
|
// consumers hard-code, and that list only ever shrinks.
|
|
reqURL := fmt.Sprintf("%s/v1/iam/keys/principal?accessKey=%s", iamEndpoint, url.QueryEscape(accessKey))
|
|
|
|
// Auth is client_secret_basic (RFC 6749 §2.3.1) — the ONE transport IAM reads
|
|
// to establish a confidential-APP principal. Key resolution is a
|
|
// credential-disclosure boundary gated on `p.App != ""` holding CapKeyResolve,
|
|
// and IAM derives p.App ONLY from Basic credentials: query params yield no
|
|
// principal at all (401 "authentication required"), and a client_credentials
|
|
// bearer resolves to a plain org-scoped principal with an EMPTY App, which the
|
|
// gate refuses with "auth:Unauthorized operation". Basic is also why no token
|
|
// cache is needed — there is no token.
|
|
clientId, clientSecret := iamClientCreds()
|
|
if clientId == "" || clientSecret == "" {
|
|
return nil, false, fmt.Errorf("IAM client credentials are not configured")
|
|
}
|
|
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
req, err := http.NewRequest(http.MethodGet, reqURL, nil)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("IAM request build failed: %w", err)
|
|
}
|
|
req.SetBasicAuth(clientId, clientSecret)
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
// The key travels in the query string, so a transport failure carries it:
|
|
// *url.Error prints the whole URL. Redacted before it becomes an error value,
|
|
// because from here it is logged and wrapped further.
|
|
return nil, false, fmt.Errorf("IAM request failed: %s", object.RedactKeys(err.Error()))
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// IAM states its reason in the BODY (code + msg), not the status line. This
|
|
// used to return on a non-200 before reading it, so `key_unknown` — "the
|
|
// entity does not exist" — reached the caller as a bare "IAM returned status
|
|
// 400". Those read alike and mean opposite things: one says this key is not
|
|
// there, the other says key validation itself is broken. Every gateway key in
|
|
// the cluster once failed this way, and the bare status sent the search
|
|
// toward a contract break for hours when IAM had named the cause in the first
|
|
// reply. Decode first, judge second.
|
|
var result struct {
|
|
Status string `json:"status"`
|
|
Msg string `json:"msg"`
|
|
Code string `json:"code"` // WHY, when IAM refused (iam store.KeyFailure)
|
|
Data json.RawMessage `json:"data"`
|
|
}
|
|
decodeErr := json.NewDecoder(resp.Body).Decode(&result)
|
|
|
|
// A refusal IAM named is a refusal we can relay, whatever the status line. It is
|
|
// about the key only when it carries a code.
|
|
if decodeErr == nil && (result.Code != "" || result.Status != "ok") {
|
|
return nil, result.Code != "", keyRefusal(result.Code, result.Msg, accessKey)
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, false, fmt.Errorf("IAM returned status %d and named no reason", resp.StatusCode)
|
|
}
|
|
if decodeErr != nil {
|
|
return nil, false, fmt.Errorf("failed to parse IAM response: %w", decodeErr)
|
|
}
|
|
|
|
// ok-with-nobody. IAM said the request succeeded and named no user, which this
|
|
// used to relay as (nil, nil) — no error, no principal — so the caller fell to
|
|
// its bare "invalid API key" with nothing behind it. That message is the one
|
|
// shape a holder cannot act on and an operator cannot trace: it looks identical
|
|
// whether the key is wrong, the envelope changed, or IAM answered about someone
|
|
// who no longer exists. A key that resolves to nobody IS unusable, so refuse it
|
|
// — but say which of the two things happened, and keep the redacted prefix so
|
|
// the line names the credential without disclosing it.
|
|
if len(result.Data) == 0 || string(result.Data) == "null" {
|
|
return nil, true, authError(
|
|
"API key %s resolved to no user — IAM accepted the lookup and returned nothing. "+
|
|
"The key may have been deleted, or its owner removed. Mint a new one at "+KeysURL(""),
|
|
keyHint(accessKey))
|
|
}
|
|
var u iam.User
|
|
if err := json.Unmarshal(result.Data, &u); err != nil {
|
|
return nil, false, fmt.Errorf("failed to parse IAM response: %w", err)
|
|
}
|
|
return result.Data, true, nil
|
|
}
|
|
|
|
// KeysURL is where a holder mints a key: the API-keys page on the console of the
|
|
// brand host belongs to, "" for a brand that runs none. It is spelled ONCE because
|
|
// every refusal below names it and three copies of an address drift the moment the
|
|
// page moves. It moved already: these messages sent people to cloud.hanzo.ai/keys,
|
|
// which answers 404 (cloud.hanzo.ai is the product site; the console is its own
|
|
// host, and its key surface is /api-keys). A refusal that names a cure the holder
|
|
// cannot reach is worse than one that names none. The key refusals below hold no
|
|
// request and pass "", the default brand.
|
|
func KeysURL(host string) string {
|
|
if c := brandDefs[brandFromHost(host)].console; c != "" {
|
|
return "https://" + c + "/api-keys"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// keyRefusal turns IAM's refusal into something the holder can ACT on.
|
|
//
|
|
// IAM answers every unresolvable key with one sentence — "the entity does not exist"
|
|
// — and this function used to relay it verbatim, so a user whose key had simply been
|
|
// revoked was told their entity was gone and went looking for a deleted organization
|
|
// instead of minting a new key. The reason now rides beside that sentence as a code
|
|
// (iam internal/store/apikey.go), and each code has exactly one cure.
|
|
//
|
|
// The key is named by PREFIX only. A holder needs to know WHICH of their keys failed;
|
|
// nobody needs the rest of it, and this string reaches logs and error bodies.
|
|
func keyRefusal(code, msg, key string) error {
|
|
switch code {
|
|
case "key_unknown":
|
|
// IT SAYS WHAT IS KNOWN, WHICH IS THAT THE KEY DOES NOT RESOLVE.
|
|
//
|
|
// This code carries no history. It is the same answer for a key that was
|
|
// revoked, one that was replaced, and one that never resolved from the
|
|
// moment it was issued — and the third really happens: a service account's
|
|
// credential was written to a column nothing reads, so it answered this
|
|
// code for its entire life (iam internal/serviceaccounts). Saying "it was
|
|
// revoked or replaced" turned that into a hunt for a revocation nobody
|
|
// performed, on keys that had never worked once.
|
|
//
|
|
// The cure is the same either way, so the sentence keeps it and drops the
|
|
// story it cannot support.
|
|
return fmt.Errorf("API key %s does not resolve — mint a new one at "+
|
|
KeysURL(""), keyHint(key))
|
|
case "key_wrong_door":
|
|
return fmt.Errorf("API key %s is not a secret key. A publishable pk- key "+
|
|
"identifies an org for ingest and cannot authenticate a request; "+
|
|
"use your secret (sk-) key", keyHint(key))
|
|
case "key_expired":
|
|
return fmt.Errorf("API key %s has expired — mint a new one at "+
|
|
KeysURL(""), keyHint(key))
|
|
case "key_not_publishable":
|
|
return fmt.Errorf("API key %s is not a publishable key", keyHint(key))
|
|
case "key_foreign_user", "key_dangling_user":
|
|
// Not the holder's doing, and not something they can fix. Say only that it
|
|
// was refused; the code carries the detail to whoever reads the logs.
|
|
return fmt.Errorf("API key %s was refused (%s)", keyHint(key), code)
|
|
}
|
|
// An IAM that sent no code, or one this build does not know: relay what it said
|
|
// rather than inventing a cause.
|
|
return fmt.Errorf("IAM error: %s", msg)
|
|
}
|
|
|
|
// keyHint names a key by its prefix and nothing else — enough to tell WHICH key
|
|
// failed, useless to anyone who reads it. The ONE way a key is rendered outside its
|
|
// own use.
|
|
func keyHint(key string) string {
|
|
key = strings.TrimSpace(key)
|
|
const shown = 9 // "sk-" + 6
|
|
if len(key) <= shown {
|
|
return "…"
|
|
}
|
|
return key[:shown] + "…"
|
|
}
|
|
|
|
// ── Usage tracking ──────────────────────────────────────────────────────────
|
|
|
|
// usageRecord mirrors IAM's UsageRecord for JSON serialization.
|
|
type usageRecord struct {
|
|
Owner string `json:"owner"`
|
|
User string `json:"user"`
|
|
Organization string `json:"organization"`
|
|
Model string `json:"model"`
|
|
Provider string `json:"provider"`
|
|
// PromptTokens is EVERY input token on every path; CacheReadTokens and
|
|
// CacheWriteTokens are parts of it, read from and written to the upstream's
|
|
// prompt cache, each billed at its own rate (tokenNanoAt).
|
|
PromptTokens int `json:"promptTokens"`
|
|
CompletionTokens int `json:"completionTokens"`
|
|
TotalTokens int `json:"totalTokens"`
|
|
CacheReadTokens int `json:"cacheReadTokens,omitempty"`
|
|
CacheWriteTokens int `json:"cacheWriteTokens,omitempty"`
|
|
Cost float64 `json:"cost"`
|
|
Currency string `json:"currency"`
|
|
Premium bool `json:"premium"`
|
|
Stream bool `json:"stream"`
|
|
Status string `json:"status"`
|
|
ErrorMsg string `json:"errorMsg"`
|
|
ClientIP string `json:"clientIp"`
|
|
RequestID string `json:"requestId"`
|
|
// ref names this record's debit for the ledger (object.UsageEvent.Ref): minted
|
|
// the first time the record is filed and kept, so filing it again — a retry after
|
|
// a lost answer — is the same debit.
|
|
ref string
|
|
// ClientRequestID is the X-Request-Id the caller was answered under, where the
|
|
// surface answers with one. It is what the row's request_id column shows, so a
|
|
// caller's id finds their bill; RequestID stays the row's own, minted here, and is
|
|
// never a value a caller chose.
|
|
ClientRequestID string `json:"clientRequestId,omitempty"`
|
|
|
|
// plan is the grant of the plan that covers this call (Cover), nil when none
|
|
// does: the call debits no wallet and counts against no free allowance. planNanos
|
|
// is what its paid upstream cost, settled against the plan's budget (planCost).
|
|
plan *object.LimitGrant
|
|
planNanos int64
|
|
// cash says the host's grant lets only the wallet's cash pay for this call: the
|
|
// model's policy refuses granted credit (object.UsageEvent.Cash).
|
|
cash bool
|
|
// key is the API key the call arrived on (object.UsageEvent.Key), set by bind.
|
|
key string
|
|
// seat is the call's seat on the paid lane (lane.go), set by bind: what it spent
|
|
// settles it, against the platform's paid-lane day and its org's share.
|
|
seat *seat
|
|
|
|
// Requested is the model the caller ASKED for, set only when a different route
|
|
// answered — today, when a vendor's account was spent and it served the request
|
|
// from a route it charges nothing for. Empty on every ordinary call, so
|
|
// non-empty IS the fallback flag, and the pair (Requested, Model) says both what
|
|
// was wanted and what arrived.
|
|
//
|
|
// It exists because a downgrade nobody can see is a lie about what the user got:
|
|
// the answer is real, it is just not the model they picked, and every reader of
|
|
// this record — the ledger, the span, whoever is looking at a bad answer — needs
|
|
// to be able to tell those two situations apart.
|
|
Requested string `json:"requested,omitempty"`
|
|
|
|
// Free states that the route that answered is priced at nothing BY THE VENDOR
|
|
// — a spare route, the kind that keeps answering while an account is empty.
|
|
//
|
|
// It is a separate fact from Requested, and keeping them apart is the point:
|
|
// one says the caller did not get what they asked for, the other says what the
|
|
// answer cost. A price table asked about a SKU it has never seen answers with
|
|
// its conservative default, which is the right guess for an unknown model and
|
|
// exactly the wrong one here — it would bill a customer for the fallback they
|
|
// were given because a vendor of ours ran out of money.
|
|
Free bool `json:"free,omitempty"`
|
|
|
|
// ImageCount is the number of images generated. Image models bill per image,
|
|
// not per token: when > 0, recordUsage bills via imageCostCents instead of
|
|
// the token-based cost. Placed last so the token-field alignment above is
|
|
// unchanged.
|
|
ImageCount int `json:"imageCount,omitempty"`
|
|
|
|
// VideoCount is the number of videos generated. Like images, video models
|
|
// bill per unit, not per token: when > 0, recordUsage bills via
|
|
// videoCostCents. Mutually exclusive with ImageCount in practice (a call is
|
|
// either an image or a video generation, never both).
|
|
VideoCount int `json:"videoCount,omitempty"`
|
|
|
|
// DecisionCount is the number of decisions answered (POST /v1/decisions), one
|
|
// per call. When > 0, usageCostNano prices the call by decisionCostNano: its
|
|
// input and output tokens at the model's price.
|
|
DecisionCount int `json:"decisionCount,omitempty"`
|
|
|
|
// AudioSeconds is the DURATION of audio a speech-to-text call consumed, and
|
|
// AudioChars the number of characters a text-to-speech call synthesized.
|
|
// Audio bills per unit like images and video, but the unit differs by
|
|
// DIRECTION — the market prices transcription per minute and synthesis per
|
|
// million characters — so the two travel separately rather than collapsing
|
|
// into one "audio units" field that means different things per row.
|
|
//
|
|
// A speech model that bills TOKENS instead (an ASR pipeline decoding through
|
|
// an LM) needs no field here: it fills PromptTokens/CompletionTokens like any
|
|
// other token-billed call. The record carries what was actually consumed.
|
|
//
|
|
// Both were previously discarded at the emit sites, which is why audio rows
|
|
// billed 0: the quantity never reached the record, so the token math it fell
|
|
// through to had nothing to multiply.
|
|
AudioSeconds float64 `json:"audioSeconds,omitempty"`
|
|
AudioChars int `json:"audioChars,omitempty"`
|
|
|
|
// BYO marks a call executed against a customer-connected third-party account
|
|
// (the caller's org supplied its own provider key via /v1/ai/connections),
|
|
// rather than a Hanzo-served provider bought with Hanzo credits. When true the
|
|
// customer already paid the upstream directly, so recordUsage bills only the
|
|
// 1% platform fee (platformFeeCents) instead of the full token cost — the full
|
|
// cost is still recorded for analytics. Set at the emit site via providerBYO.
|
|
BYO bool `json:"byo,omitempty"`
|
|
|
|
// FeeCents is the Hanzo platform surcharge stamped by recordUsage
|
|
// (platformFeeCents(costCents, BYO)): the amount actually billed on a BYO
|
|
// call, 0 for a Hanzo-credits call. Persisted to the warehouse fee_cents column.
|
|
FeeCents int64 `json:"feeCents,omitempty"`
|
|
|
|
// Account is the attribution label for the account that served the call:
|
|
// "<owner>/<name>" for a BYO connection, "hanzo" for a Hanzo-served provider.
|
|
Account string `json:"account,omitempty"`
|
|
|
|
// Unpriced marks a token-billed call whose model had NO configured price in any
|
|
// source (family/route/conf/static), so it billed at the conservative default
|
|
// ($1/$4 per 1M) rather than a real rate. The debit is UNCHANGED — the flag just
|
|
// makes the guess honest: the row is queryable and the o11y span carries
|
|
// priced=false, instead of silently presenting an invented price as real. Set in
|
|
// recordUsage; image/video calls (own per-unit pricing) are never flagged.
|
|
Unpriced bool `json:"unpriced,omitempty"`
|
|
|
|
// BilledNanoExact / CostNanoExact carry an EXACT nano-USD billed amount and
|
|
// provider COGS when the biller knows them (zen's commerce Meter computes both
|
|
// per served tier). Set, they are what the call billed and cost, and the table
|
|
// recompute is not consulted.
|
|
//
|
|
// Pointers because exactness is PRESENCE, not positivity. A turn that billed
|
|
// exactly nothing — a free tier, a zero-price model — knows its amount as
|
|
// precisely as any other, and a plain int64 could not say so: 0 read as "unset"
|
|
// sent it back to the table, which invents a price for a call that had none.
|
|
BilledNanoExact *int64 `json:"billedNanoExact,omitempty"`
|
|
CostNanoExact *int64 `json:"costNanoExact,omitempty"`
|
|
|
|
// ── o11y gen_ai span enrichment (all optional, best-effort) ──────────────
|
|
// These carry the observation-of-record attribution the o11y span plane reads
|
|
// (controllers/telemetry.go emitGenAISpan). They are omitempty so a record that
|
|
// does not populate them emits no attribute — the span is honest, never a
|
|
// fabricated value, and each dimension lights up as its producer is wired.
|
|
|
|
// Session is the conversation/session id. Emitted as gen_ai.conversation.id +
|
|
// session.id, which is what turns the o11y sessions view on for this org.
|
|
Session string `json:"session,omitempty"`
|
|
// TraceID is the gen_ai span's OWN trace id, stamped by emitGenAISpan.
|
|
//
|
|
// The span plane and the spend ledger answer different questions about one call
|
|
// — what happened, and what it cost — and joining them needs an id BOTH sides
|
|
// observed. Deriving one independently on each side would produce two ids that
|
|
// merely describe the same request, which is not the same thing and cannot be
|
|
// joined on. Empty when telemetry is off: there was no span, so there is no
|
|
// trace, and saying so is better than inventing an id nothing else will carry.
|
|
TraceID string `json:"traceId,omitempty"`
|
|
// Environment is the caller's logical environment label (X-Environment). Emitted
|
|
// as deployment.environment on the span so Observe narrows by environment instead
|
|
// of defaulting to "default". Empty emits no attribute (honest, never fabricated).
|
|
Environment string `json:"environment,omitempty"`
|
|
// Project is the caller's org SUB-SCOPE (X-Project-Id). It is stamped on the
|
|
// cloud_usage ledger row + the gen_ai span, so cost/tokens/latency narrow WITHIN
|
|
// an org by project. Empty is the org's default project (whole-org view).
|
|
// Populated once in recordTrace from the request context (WithGenAIAttribution).
|
|
Project string `json:"project,omitempty"`
|
|
// APIKeyHash is a NON-reversible ref (SHA-256 hex) of the caller credential —
|
|
// never the plaintext key. Emitted on the gen_ai span (gen_ai.hanzo.api_key_hash)
|
|
// so a span correlates to a key without the store ever holding a secret.
|
|
// Populated in recordTrace from the request context.
|
|
APIKeyHash string `json:"apiKeyHash,omitempty"`
|
|
// ServedBy is where inference ran: "hanzo" (cloud), "byo-provider" (an
|
|
// org-owned provider key), or "byo-gpu" (an org's own cluster). Empty defaults
|
|
// to "hanzo" at emit — the cloud-served majority.
|
|
ServedBy string `json:"servedBy,omitempty"`
|
|
// ClusterID is the BYO-GPU cluster id when ServedBy == "byo-gpu".
|
|
ClusterID string `json:"clusterId,omitempty"`
|
|
// RoutePolicy is the enso route-policy decision that selected the model.
|
|
RoutePolicy string `json:"routePolicy,omitempty"`
|
|
// Routing is the router's whole decision (`Routing`, encoded) when one routed this
|
|
// request; emitted on the span as gen_ai.hanzo.routing. Never persisted.
|
|
Routing string `json:"-"`
|
|
// Served is the arm that generated the answer when a family names one
|
|
// (X-Hanzo-Served) — the model behind an adaptive SKU. Emitted as
|
|
// gen_ai.response.model; empty leaves response.model the SKU.
|
|
Served string `json:"served,omitempty"`
|
|
// Vendor is the provider that ran that arm ("digitalocean", "openrouter").
|
|
// Emitted as gen_ai.provider.name, ahead of Provider, which is our route label.
|
|
Vendor string `json:"vendor,omitempty"`
|
|
// Failover is the chain of arms that failed before the one that answered, as
|
|
// the family reported it. Empty when the first arm answered.
|
|
Failover string `json:"failover,omitempty"`
|
|
// Via is the vendor account an answer ran on where a model's own vendor served
|
|
// it, as the vendor and the key's place in its pool ("anthropic#2"): the name
|
|
// of a place, never a key. Empty on every other call. Emitted on the span as
|
|
// gen_ai.hanzo.via; the row says the same in Origin, as host#place.
|
|
Via string `json:"via,omitempty"`
|
|
// ReasoningTokens is the part of CompletionTokens the model spent reasoning.
|
|
ReasoningTokens int `json:"reasoningTokens,omitempty"`
|
|
// Reasoning is how hard the caller asked the model to think (depthOf): none,
|
|
// low, medium, high, max, or default when it asked nothing. Empty where the path
|
|
// does not read it.
|
|
Reasoning string `json:"reasoning,omitempty"`
|
|
// Arms is every arm a call priced at cost ran (trail), as JSON; empty on a call
|
|
// priced any other way.
|
|
Arms string `json:"arms,omitempty"`
|
|
// LossNano is what the call cost us past what it billed: 0 on every call billed
|
|
// at or above its cost. A loss is never silent; it is this.
|
|
LossNano int64 `json:"lossNano,omitempty"`
|
|
// First is the time from the start of the call to the first token of the
|
|
// answer. Zero when not measured.
|
|
First time.Duration `json:"-"`
|
|
// InputMessages / OutputMessages are the serialized prompt/completion. They are
|
|
// PII and are emitted ONLY when O11Y_GENAI_CAPTURE_MESSAGES is enabled
|
|
// (default off = redacted). Never logged, never billed — telemetry only.
|
|
InputMessages string `json:"inputMessages,omitempty"`
|
|
OutputMessages string `json:"outputMessages,omitempty"`
|
|
|
|
// Agent is the machine credential that authorized this call, "<org>/<name>",
|
|
// when the caller was not a person. An IAM application is a route, not a
|
|
// customer: it answers "which program placed the call" and never "whose spend
|
|
// is this", so it is recorded HERE and User is left to the person the
|
|
// application acts for. A row carrying an Agent and no User is a call nobody
|
|
// owns — visible as such, rather than presented as the application's own spend.
|
|
Agent string `json:"agent,omitempty"`
|
|
|
|
// Origin is the host that answered — the hostname of the serving provider's
|
|
// URL (object.Provider.Origin). Provider is our route label for the same call;
|
|
// carrying both lets a row be asked whether the label still matches the address
|
|
// the bytes went to. Empty when the serving provider declares no URL.
|
|
Origin string `json:"-"`
|
|
|
|
// Payer is the money address this call spends from — the account the balance
|
|
// gate READ, carried here rather than re-derived, so the debit cannot land
|
|
// somewhere else. Not serialized: an Account's fields are unexported by design
|
|
// (it is money; an account whose owner disagrees with its key must not exist),
|
|
// and the wire already carries the subject it resolves to.
|
|
Payer account.Account `json:"-"`
|
|
|
|
// Visitor is the lanes a public-lane call is counted on, narrowest first. Only
|
|
// the public lane puts one on the request (withVisitor); every other call is
|
|
// empty here. Not serialized — it addresses the lane's count, it does not
|
|
// describe the row.
|
|
Visitor []address.Bucket `json:"-"`
|
|
}
|
|
|
|
// reached answers whether a vendor was committed to this call — the question the money
|
|
// turns on, since a vendor that ran a request invoices us for it whatever came back.
|
|
//
|
|
// Provider is where that fact already lives. Every emit site fills it from the row that
|
|
// SERVED or REFUSED the request, and leaves it empty when the request died before any
|
|
// vendor was chosen: an unresolvable route, a candidate list that ran out, a caller who
|
|
// hung up. `served` says the same thing from the other side — a failed attempt carries
|
|
// a name and nothing else.
|
|
//
|
|
// IT IS A NAME, NOT A RECEIPT. It says a vendor was committed to this request, not that
|
|
// the vendor billed us for it — a 400 the upstream rejected on sight usually costs us
|
|
// nothing. That is the honest limit of what a record knows here, and the error it makes
|
|
// is the harmless one: a zero-cost row filed for a vendor that charged nothing, rather
|
|
// than no row at all for one that did.
|
|
func (r *usageRecord) reached() bool { return r.Provider != "" }
|
|
|
|
// answered reports that a model produced a result for this call. Only the public
|
|
// lane's own count asks it — see recordUsage.
|
|
func (r *usageRecord) answered() bool { return r.Status == "success" }
|
|
|
|
// spent reports the tokens a call produced, for a call that then failed.
|
|
//
|
|
// reached() says a vendor ran the request; this says how much of it it ran. Without
|
|
// both, a failure files a row naming a vendor and a quantity of zero, which prices a
|
|
// half-generated answer at nothing and leaves us holding the invoice for it.
|
|
//
|
|
// The pipeline's own count is authoritative whenever it filled one. Otherwise the
|
|
// prompt was already counted before the call went out — routeForPrompt sizes the route
|
|
// with it — and the partial answer is whatever reached the writer before the stream
|
|
// broke. Neither number is invented: one was measured on the way out, the other is the
|
|
// text the vendor actually sent.
|
|
func spent(res *model.ModelResult, name string, prompt int, partial string, err error) (int, int) {
|
|
if res != nil && res.TotalTokenCount > 0 {
|
|
return res.PromptTokenCount, res.ResponseTokenCount
|
|
}
|
|
completion, _ := model.GetTokenSize(name, partial)
|
|
if completion == 0 && !ran(err) {
|
|
// Nothing came back and nobody ran it: the vendor reported no meter of its
|
|
// own, no text reached the caller, and the refusal is one that arrives at
|
|
// the edge. The prompt was measured on the way out — routeForPrompt sizes
|
|
// the route with it — but measuring a prompt is not a vendor processing it,
|
|
// and there is no invoice behind this to pass on.
|
|
//
|
|
// It is the same reasoning recordUsage states for a call that never left,
|
|
// applied to the quantity rather than the row. Left unsaid, the charge
|
|
// scales with the prompt, so the larger the request the more it costs to be
|
|
// turned away.
|
|
return 0, 0
|
|
}
|
|
return prompt, completion
|
|
}
|
|
|
|
// payer answers who pays for this call. ONE rule, and it prefers the answer the gate
|
|
// already computed over deriving a second one.
|
|
//
|
|
// The fallback is for a record with no authenticated principal behind it — the
|
|
// session-scoped and self-billing surfaces. It is account.PayerOf, which is what
|
|
// every debit used to do, and it is exactly why this field exists: PayerOf sees two
|
|
// strings, so it cannot see the signed billing_account claim naming a personal
|
|
// wallet, and it cannot see that a credential is a machine. Both are inputs to who
|
|
// pays. Missing them, it confidently answers the pre-claim default — a different
|
|
// account from the one the gate read a moment earlier, on the same request.
|
|
func (r *usageRecord) payer() account.Account {
|
|
if !r.Payer.Zero() {
|
|
return r.Payer
|
|
}
|
|
return account.PayerOf(r.Owner, r.User)
|
|
}
|
|
|
|
// bind ties this record to the credential that authorized the call: the money
|
|
// address it spends from, and the name its spend is attributed to. It is the ONE
|
|
// place a record learns its principal, so the two answers are read from one
|
|
// identity and cannot disagree.
|
|
//
|
|
// THE MONEY is unchanged and is the same single expression the balance gate
|
|
// resolves with (iam.User.Payer) — one identity, one rule, one address, so gate and
|
|
// debit cannot answer differently. Nil-safe: a surface with no principal leaves the
|
|
// field unset and falls back.
|
|
//
|
|
// THE NAME follows one rule: only a person may be named, because only a person can
|
|
// owe money.
|
|
//
|
|
// - A person's credential names itself, and nothing on the request can move that.
|
|
// Otherwise any caller could send a header and put their bill on a colleague.
|
|
//
|
|
// - A machine credential names no person, because there is none behind it. It is
|
|
// recorded as the Agent, and the User becomes whoever the identity boundary
|
|
// authenticated before the machine placed the call (X-User-Id, qualified
|
|
// "<org>/<name>"). A bare name is not an identity and is refused rather than
|
|
// guessed at.
|
|
//
|
|
// - A machine that names nobody leaves User empty and says so at once. An empty
|
|
// column is a call a query can find; the application's own name in that column
|
|
// reads as attributed and is not, which is how spend with no owner reaches an
|
|
// invoice unnoticed.
|
|
//
|
|
// Naming a person never moves money: for a machine the payer resolves to the org
|
|
// whatever name the row carries, and a person's payer comes from their own
|
|
// credential. Attribution and settlement stay separate answers to separate questions.
|
|
func (r *usageRecord) bind(ctx context.Context, u *iam.User) {
|
|
if r == nil || u == nil {
|
|
return
|
|
}
|
|
r.Payer = u.Payer(r.Owner)
|
|
// The API key the call arrived on, as the boundary named it: what the host
|
|
// counts the key's own spend against.
|
|
r.key = object.GenAIAttributionFromContext(ctx).Key
|
|
r.seat = seatOf(ctx)
|
|
// The public lane's visitor, read from the request the lane alone writes it on.
|
|
r.Visitor = visitorOf(ctx)
|
|
// Who the host said pays: a covered call settles against its grant and debits no
|
|
// wallet; one sent to the wallet carries whether only cash may pay.
|
|
if g := grantFrom(ctx); g.Covered() {
|
|
if r.plan == nil {
|
|
r.plan = g
|
|
}
|
|
} else if g != nil {
|
|
r.cash = g.Cash
|
|
}
|
|
|
|
self := u.Owner + "/" + u.Name
|
|
// account.IsMachine is the ONE predicate for "is this credential a program",
|
|
// shared with the payer rule above.
|
|
if !account.IsMachine(u.Type) {
|
|
r.User = self
|
|
return
|
|
}
|
|
|
|
r.Agent = self
|
|
if p := person(ctx, u); p != "" {
|
|
r.User = p
|
|
return
|
|
}
|
|
r.User = ""
|
|
warnUnownedOnce(self)
|
|
}
|
|
|
|
// person is the person a machine credential's call is for: whoever the identity
|
|
// boundary authenticated before the machine placed it (X-User-Id, "<org>/<name>").
|
|
// "" for a person's own credential, which names itself, and for a machine that names
|
|
// nobody else.
|
|
func person(ctx context.Context, u *iam.User) string {
|
|
if u == nil || !account.IsMachine(u.Type) {
|
|
return ""
|
|
}
|
|
named := strings.TrimSpace(object.GenAIAttributionFromContext(ctx).User)
|
|
// …and the name must be SOMEONE ELSE. A caller reaching us through the identity
|
|
// boundary never chooses this header: the boundary deletes what arrived and
|
|
// rewrites it from the presenting credential's own claims, so a machine that
|
|
// came that way is handed ITS OWN subject. Copying that into the person column
|
|
// puts the application back where this whole rule exists to keep it out of, one
|
|
// hop later and harder to see.
|
|
//
|
|
// The two spellings differ in the org half — a token's `sub` is qualified by the
|
|
// registration's owner and the `owner` claim by the org it serves — so the NAME
|
|
// is what identifies the principal across them.
|
|
if _, name, ok := strings.Cut(named, "/"); ok && !strings.EqualFold(name, u.Name) {
|
|
return named
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// tenant is the org a model family is told a call is for (X-Org-Id): the customer a
|
|
// program in this process stated (object.Caller), else the org of the person a
|
|
// machine credential acts for (person), else the org the request bills, else the
|
|
// caller's own. A sibling's call over the plane is the platform working for that
|
|
// person's org, and the family keeps its per-org share by it. Money is not moved by
|
|
// it: ai settles the call, and the family is told so (X-Hanzo-Fronted-By).
|
|
func tenant(ctx context.Context, org string, u *iam.User) string {
|
|
if who, ok := object.CallerOf(ctx); ok {
|
|
return who.Org
|
|
}
|
|
if o, _, _ := strings.Cut(person(ctx, u), "/"); o != "" {
|
|
return o
|
|
}
|
|
if org != "" {
|
|
return org
|
|
}
|
|
if u != nil {
|
|
return u.Owner
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// payerHeader tells a Hanzo family the account a call is for (zen's Tenant wallet),
|
|
// beside the org in X-Org-Id. The family keeps its free lane's share per payer, so
|
|
// the members of the signup org, who each pay for themselves, each hold a share of
|
|
// their own rather than all of them one share of the org's.
|
|
const payerHeader = "X-Hanzo-Payer"
|
|
|
|
// payerOf is the account a model family is told a call is for, named in the order
|
|
// tenant names its org: the customer a program in this process stated, else the
|
|
// person a machine credential acts for, else the caller's own payer in org. "" when
|
|
// none resolves, and the family then counts the call against the org.
|
|
func payerOf(ctx context.Context, org string, u *iam.User) string {
|
|
if who, ok := object.CallerOf(ctx); ok {
|
|
return account.PayerOf(who.Org, who.Person).Subject()
|
|
}
|
|
if named := person(ctx, u); named != "" {
|
|
return account.PayerOf("", named).Subject()
|
|
}
|
|
if u != nil {
|
|
return u.PayerSubject(org)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// unownedWarned dedupes the unowned-spend report to once per agent, so a busy
|
|
// application logs one actionable line instead of one per request.
|
|
var unownedWarned sync.Map
|
|
|
|
// warnUnownedOnce reports spend that no person owns: an application bought
|
|
// inference and named nobody it was buying for, so the row bills the org with an
|
|
// empty user column. The fix is upstream — the caller sends X-User-Id with the
|
|
// person it authenticated — and this is the only moment the evidence still exists.
|
|
func warnUnownedOnce(agent string) {
|
|
if _, seen := unownedWarned.LoadOrStore(agent, struct{}{}); seen {
|
|
return
|
|
}
|
|
log.Error("usage: application %q is buying inference for nobody — its rows carry no user. Send X-User-Id with the person it authenticated, as \"<org>/<name>\".", agent)
|
|
}
|
|
|
|
// billingQueue is the singleton usage record delivery queue. Initialized by
|
|
// InitBillingQueue() in main.go. If nil (Commerce not configured), recordUsage
|
|
// is a no-op.
|
|
var billingQueue *util.BillingQueue
|
|
|
|
// InitBillingQueue creates the billing queue from app config. Must be called
|
|
// once during startup. Returns the queue so main.go can call Shutdown().
|
|
func InitBillingQueue() *util.BillingQueue {
|
|
endpoint := conf.GetConfigString("commerceEndpoint")
|
|
if endpoint == "" {
|
|
return nil
|
|
}
|
|
endpoint = strings.TrimRight(endpoint, "/")
|
|
token := conf.GetConfigString("commerceToken")
|
|
|
|
billingQueue = util.NewBillingQueue(endpoint, token)
|
|
return billingQueue
|
|
}
|
|
|
|
// usageCostCents is the cost of a call in cents: the exact nano cost (usageCostNano
|
|
// — per unit for image, video and audio, per token otherwise) rounded to the nearest
|
|
// cent. ONE money, reported at two precisions, rather than a second arithmetic over
|
|
// the same inputs that can reach a different answer.
|
|
//
|
|
// It could, and did. The cents path rounded a float and then raised anything left at
|
|
// zero to a whole cent, so a call the ledger charged $0.0002 for was reported as $0.01
|
|
// — the same money, fifty times over, in the column every spend view reads. Measured
|
|
// on ten days of hanzo.cloud_usage: 867 of 1477 rows carried a cent that was invented
|
|
// rather than charged, and the totals differed by $8.03 on $34.69.
|
|
//
|
|
// A call that costs less than half a cent now reports zero cents, which is what it
|
|
// costs. Nothing is lost by saying so: cost_nano and billed_nano on the same row carry
|
|
// the exact amount, and they are what the debit and the invoice read.
|
|
func usageCostCents(record *usageRecord) int64 {
|
|
return nanoToCents(usageCostNano(record))
|
|
}
|
|
|
|
// usageBilledCents is what Hanzo actually DEBITS the org ledger for a call, in
|
|
// cents, given its full provider cost: the full cost for a Hanzo-served call, or
|
|
// only the platform fee (platformFeeCents, ~1%) for a BYO call where the customer
|
|
// already paid the upstream with their own key. This is the ONE billed-amount of
|
|
// record: recordUsage debits it and emitGenAISpan reports it as
|
|
// _o11y.gen_ai.billed_cost, so Observe reconciles with the invoice — while
|
|
// _o11y.gen_ai.total_cost keeps the full provider cost for analytics.
|
|
func usageBilledCents(record *usageRecord, costCents int64) int64 {
|
|
if record.BYO {
|
|
return platformFeeCents(costCents, true)
|
|
}
|
|
return costCents
|
|
}
|
|
|
|
// shownID is the request id the row files under in its request_id column: the one
|
|
// the caller was answered under when there was one, else the row's own.
|
|
func (r *usageRecord) shownID() string {
|
|
if r.ClientRequestID != "" {
|
|
return r.ClientRequestID
|
|
}
|
|
return r.RequestID
|
|
}
|
|
|
|
// paidBy names who pays for a recorded call when it is not the wallet: the plan
|
|
// that covered it, or the platform for a free model it absorbs. "" is the wallet.
|
|
func paidBy(record *usageRecord) string {
|
|
switch {
|
|
case record.plan != nil && record.plan.Pays == object.PaysCaller:
|
|
return object.PaysCaller
|
|
case record.plan != nil:
|
|
return object.PaysPlan
|
|
case usageFree(record):
|
|
return "hanzo"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// nanoUSD renders a nano-dollar cost as exact decimal USD, "" when unknown.
|
|
func nanoUSD(nano *int64) string {
|
|
if nano == nil {
|
|
return ""
|
|
}
|
|
return decimal.New(*nano, 9).String()
|
|
}
|
|
|
|
// usageTimeout bounds one debit's hand-off to the ledger. The native recorder
|
|
// answers once the debit is durably posted, which is milliseconds; a ledger that
|
|
// has not answered in this long has failed, and the failure is logged with the
|
|
// request id instead of holding a goroutine — and, on a path that files the debit
|
|
// before its reply, the caller — for as long as the ledger stays silent. It is kept
|
|
// well inside a pod's shutdown budget, so a debit filed after its reply can be tried
|
|
// again before the process goes (settleAfter).
|
|
var usageTimeout = 5 * time.Second
|
|
|
|
// recordUsage files what a call spent, and returns the native recorder's refusal
|
|
// when the debit did not land. It is the ONE chokepoint between a request and
|
|
// the money: an exact debit on the org's wallet where this build is co-resident with
|
|
// the ledger, or the same amount on its way to Commerce where it is not.
|
|
//
|
|
// A CALL THAT REACHED A VENDOR IS BILLED, ANSWER OR NO ANSWER. The rule here used to be
|
|
// "success only", which reads as fairness and is not one: by the time a call fails the
|
|
// request has been sent, the vendor has run it, and we have been invoiced for the tokens
|
|
// it processed. Filing nothing does not undo any of that — it only removes the line that
|
|
// says it happened, and leaves the spend with us.
|
|
//
|
|
// THE TRADE IS DELIBERATE AND IT FALLS ON THE CALLER. A request they broke costs them
|
|
// what it cost us to try. Keeping our side answering is our job and stays our job; what
|
|
// a caller does to their own request is theirs. The alternative prices a failing request
|
|
// at zero, which is the one property an unbounded workload must never have — the cheapest
|
|
// call in the catalogue would be the one that breaks, and whoever sends the most of them
|
|
// would pay the least.
|
|
//
|
|
// WHAT IT COST IS THE SAME ONE COST. usageCostCents / usageBilledUSD are asked here for a
|
|
// failure exactly as for a success, so the row and its o11y gen_ai span carry the
|
|
// identical number and cannot disagree. A failure that came back with no tokens therefore
|
|
// bills nothing — not because failure is free, but because that is what the meter reads,
|
|
// and reading it is the whole point.
|
|
func recordUsage(record *usageRecord) error {
|
|
// Dense flywheel reward (HIP-510): score EVERY routed request's outcome and
|
|
// attach it to its routing decision, so the bandit learns from request-volume
|
|
// signal instead of sparse explicit thumbs. An errored request scores 0 (the arm
|
|
// is penalized for failing). Dark-by-default, async, best-effort — never slows or
|
|
// fails the request.
|
|
emitAutoRoutingReward(record)
|
|
|
|
// NOTHING WAS SPENT ON A CALL THAT NEVER LEFT, so nothing is filed for one. There is
|
|
// no invoice behind it and therefore no amount, and an amount is not a thing to
|
|
// invent because a row would look tidier with one. The call is still visible — the
|
|
// emit sites write the warehouse row and the span for it either way; what it does
|
|
// not get is a place in the money.
|
|
if !record.reached() {
|
|
if record.plan != nil {
|
|
record.plan.Settle(0)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Calculate cost. usageCostCents is the ONE cost of record — the same value the
|
|
// o11y gen_ai span reports (emitGenAISpan), so the ledger debit and the span
|
|
// can never disagree.
|
|
costCents := usageCostCents(record)
|
|
|
|
// Honesty flag: if the model has no configured price, the cost above was billed at
|
|
// the conservative default. Mark the row (and warn once) rather than silently
|
|
// presenting the invented rate as real — the debit itself is unchanged.
|
|
if recordUnpriced(record) {
|
|
record.Unpriced = true
|
|
warnUnpricedOnce(record.Model)
|
|
}
|
|
|
|
// BYO: the customer paid the upstream directly with their own connected key,
|
|
// so Hanzo bills ONLY the 1% platform fee — not the token cost. The full
|
|
// costCents is still recorded (payload + warehouse) for analytics. On a
|
|
// Hanzo-served call (byo=false) feeCents is 0 and the token cost is billed as
|
|
// before. Stamp record.FeeCents so the warehouse writer persists the same value.
|
|
feeCents := platformFeeCents(costCents, record.BYO)
|
|
record.FeeCents = feeCents
|
|
amount := usageBilledCents(record, costCents)
|
|
|
|
// The debit MUST hit the same account the balance gate read and the starter
|
|
// credit funded: the billing SUBJECT within the org NAMESPACE.
|
|
// namespace (X-Org-Id) = record.Owner (the org)
|
|
// subject (?user=) = the payer the emit site carried from the principal
|
|
// For a personal-billing org that is "owner/name" (per-user); for a pooled org
|
|
// it is the org slug. record.Owner is the IAM `owner`; fall back to deriving
|
|
// owner+name from "owner/name" if Owner was not populated upstream.
|
|
org := record.Owner
|
|
if org == "" {
|
|
if i := strings.IndexByte(record.User, '/'); i > 0 {
|
|
org = record.User[:i]
|
|
} else {
|
|
org = record.User
|
|
}
|
|
}
|
|
subject := record.payer().Subject()
|
|
|
|
// A call the caller's plan covers settles against the plan: what it spent on
|
|
// paid upstream, against the budget the plan held for it. It debits no wallet and
|
|
// counts against no free allowance — the plan counted it when it was admitted.
|
|
if record.plan != nil {
|
|
record.plan.Settle(planUse(record))
|
|
}
|
|
// A call on the paid lane settles its seat, whoever paid: the hold taken at
|
|
// admission is given back. After the plan's settle, so the payer's next call reads
|
|
// the host's figure with this one in it before this hold is gone.
|
|
record.seat.settle()
|
|
// The public lane keeps its own count in this process, by the visitor's address
|
|
// and the site it shares, so its ceiling holds while the host is unreachable. It
|
|
// rises on an answer; the host's count was taken when the lane admitted the call.
|
|
//
|
|
// A VISITOR ON THE RECORD IS WHAT SAYS THIS IS THAT LANE. Only the lane puts one
|
|
// there, and nothing on the request can name one. The org would be the obvious
|
|
// test and is the wrong one: it is resolved from X-Org-Id, which any caller sets,
|
|
// so a bound that read it could be steered out of the lane it bounds.
|
|
if record.plan == nil && record.answered() && len(record.Visitor) > 0 && usageFree(record) {
|
|
publicCount.serve(record.Visitor, utcDay(time.Now()), publicChatDaily())
|
|
}
|
|
|
|
// Native in-proc finance debit — the ONE money path when co-resident with the
|
|
// finance ledger (hanzoai/cloud unified binary). The debit lands DIRECTLY on the
|
|
// org's SQLite wallet, the SAME account the prepaid gate reads, so a funded
|
|
// account both gates AND depletes. When the hook is installed we do NOT also
|
|
// enqueue to Commerce — that would double-bill. Standalone ai (no hook) falls
|
|
// through to the HTTP billing queue below.
|
|
if rec := object.UsageRecorder(); rec != nil {
|
|
ctx, cancel := context.WithTimeout(context.Background(), usageTimeout)
|
|
defer cancel()
|
|
if record.ref == "" {
|
|
record.ref = uuid.NewString()
|
|
}
|
|
if err := rec(ctx, object.UsageEvent{
|
|
Subject: subject,
|
|
Namespace: org,
|
|
USD: usageBilledUSD(record), // EXACT atto-precise debit, never a floored cent
|
|
Currency: "usd",
|
|
Model: record.Model,
|
|
Provider: record.Provider,
|
|
Actor: record.User,
|
|
Plan: record.plan != nil,
|
|
Cash: record.cash,
|
|
RequestID: record.RequestID,
|
|
Ref: record.ref,
|
|
Class: ClassOf(record.Model),
|
|
Units: int64(record.TotalTokens),
|
|
// The COGS the row and the span read (usageMargin): a family's or a
|
|
// vendor's own statement where one was made, else the rate table's.
|
|
CostUSD: nanoUSD(usageMargin(record).CostNano),
|
|
PaidBy: paidBy(record),
|
|
Key: record.key,
|
|
}); err != nil {
|
|
log.Error("billing: native usage record failed request_id=%s: %v", record.RequestID, err)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
if billingQueue == nil {
|
|
return nil
|
|
}
|
|
|
|
// The SAME facts the native event carries, in the shape Commerce reads them. Money
|
|
// alone: a free call's count was taken at admission (object.SpentFunc), so no
|
|
// writer carries one, and what a call spent does not depend on which writer a
|
|
// build has.
|
|
payload := map[string]any{
|
|
"user": subject,
|
|
"actor": record.User,
|
|
"currency": "usd",
|
|
"amount": amount,
|
|
"model": record.Model,
|
|
"provider": record.Provider,
|
|
"promptTokens": record.PromptTokens,
|
|
"completionTokens": record.CompletionTokens,
|
|
"totalTokens": record.TotalTokens,
|
|
"cacheReadTokens": record.CacheReadTokens,
|
|
"cacheWriteTokens": record.CacheWriteTokens,
|
|
"imageCount": record.ImageCount,
|
|
"videoCount": record.VideoCount,
|
|
"decisionCount": record.DecisionCount,
|
|
"audioSeconds": record.AudioSeconds,
|
|
"audioChars": record.AudioChars,
|
|
"requestId": record.RequestID,
|
|
"clientRequestId": record.ClientRequestID,
|
|
"premium": record.Premium,
|
|
"stream": record.Stream,
|
|
"status": record.Status,
|
|
"clientIp": record.ClientIP,
|
|
"byo": record.BYO,
|
|
"feeCents": feeCents,
|
|
"costCents": costCents,
|
|
"account": record.Account,
|
|
}
|
|
|
|
body, err := json.Marshal(payload)
|
|
if err != nil {
|
|
log.Error("billing: failed to marshal usage record request_id=%s: %v", record.RequestID, err)
|
|
return nil
|
|
}
|
|
|
|
billingQueue.Enqueue(&util.BillingRecord{
|
|
Body: body,
|
|
RequestID: record.RequestID,
|
|
Org: org,
|
|
Model: record.Model,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// recordTrace persists an LLM/agent trace + usage record to hanzoai/datastore
|
|
// (native datastore OLAP) over native ZAP — the ONE internal telemetry path.
|
|
//
|
|
// The datastore (datastore) is reached directly via object.DatastoreExec
|
|
// (object/datastore.go) — the datastore image serves datastore on :8123/:9000,
|
|
// not a ZAP bridge. Fire-and-forget — failures are logged inside the writer,
|
|
// never block the request.
|
|
//
|
|
// Only the spend ledger (hanzo.cloud_usage) is written here — that is the ai
|
|
// module's table, read by GetCloudUsageOverview for the console Overview. The
|
|
// per-tenant trace ledger (canonical hanzo.observations / hanzo.traces) is owned
|
|
// and populated by the o11y/insights ingestion pipeline, not this module — one
|
|
// writer per table.
|
|
//
|
|
// Separately, emitGenAISpan ships one OpenTelemetry GenAI span per call to the
|
|
// o11y OTLP collector (opt-in via OTEL_EXPORTER_OTLP_ENDPOINT). That is the
|
|
// SOURCE the o11y pipeline ingests — this module never writes the observations
|
|
// table directly. The span emit is batched/async and a no-op when telemetry is
|
|
// off, so it never blocks the request.
|
|
func recordTrace(ctx context.Context, record *usageRecord, startTime time.Time) {
|
|
// Enrich the record with the request-scoped observability attribution the
|
|
// TenantContextFilter stashed on the context (project sub-scope + a
|
|
// non-reversible credential ref). Doing it HERE — the ONE funnel every
|
|
// completion surface passes through — stamps BOTH the warehouse row
|
|
// (zapWriteUsage) and the gen_ai span (emitGenAISpan) from a single place,
|
|
// rather than threading it through ~15 emit sites. A record that already set a
|
|
// value (an explicit producer) is left untouched.
|
|
if record != nil {
|
|
attr := object.GenAIAttributionFromContext(ctx)
|
|
// Org fallback: stamp the VERIFIED tenant org only when a producer set neither
|
|
// Organization nor Owner, so real tenant traffic always carries its org (the
|
|
// o11y llmobs views drop empty-org spans) without overriding an explicit value.
|
|
if record.Organization == "" && record.Owner == "" {
|
|
record.Organization = attr.Org
|
|
}
|
|
if record.Project == "" {
|
|
record.Project = attr.Project
|
|
}
|
|
if record.Session == "" {
|
|
record.Session = attr.Session
|
|
}
|
|
if record.Environment == "" {
|
|
record.Environment = attr.Environment
|
|
}
|
|
if record.APIKeyHash == "" {
|
|
record.APIKeyHash = attr.APIKeyHash
|
|
}
|
|
}
|
|
// The span is emitted BEFORE the ledger write, because emitting it is what
|
|
// decides the trace id and the row has to carry it. The write is the async half
|
|
// (a goroutine, off the request path); the span emit is already batched and
|
|
// non-blocking, so leading with it costs the caller nothing and is the only
|
|
// order in which the two planes can share an id.
|
|
emitGenAISpan(ctx, record, startTime)
|
|
go zapWriteUsage(record, startTime)
|
|
emitGenAIEvent(ctx, record, startTime)
|
|
}
|
|
|
|
// ── API handlers ────────────────────────────────────────────────────────────
|
|
|
|
// authenticate validates a bearer credential to a real principal WITHOUT routing
|
|
// a model — the authentication half of authResolveProvider. Handlers call it on a
|
|
// request-body error so an INVALID credential is rejected (401) regardless of
|
|
// body validity: a malformed (or field-incomplete) body from an unauthenticated
|
|
// caller must never return 200/400 that confirms the endpoint or lets it be
|
|
// probed. It is invoked only on the error path, so the happy path keeps a single
|
|
// validation (authResolveProvider).
|
|
//
|
|
// It is a strict SUBSET of authResolveProvider's branches, and the omissions are
|
|
// the point: a pk- never reaches here (every generative endpoint refuses it before
|
|
// the body is read) and a run key authenticates only against the run table, so
|
|
// neither is admitted by this check. A branch missing here can only REFUSE a
|
|
// request the resolver would have taken — never admit one it would have refused —
|
|
// which is the direction a credential check is allowed to be wrong in.
|
|
func (c *ApiController) authenticate(token string) error {
|
|
return authenticateToken(token, c.GetAcceptLanguage())
|
|
}
|
|
|
|
// authenticateToken is authenticate with the request's language passed in, for a
|
|
// caller that holds no controller (a ZAP handler).
|
|
func authenticateToken(token, lang string) error {
|
|
switch {
|
|
case isJwtToken(token):
|
|
// Signature + issuer/audience validation (R3): a foreign-aud or
|
|
// wrong-issuer token is rejected here, not just signature-checked.
|
|
if _, err := object.ParseAndValidateJWT(token); err != nil {
|
|
return authError("invalid access token: %s", err.Error())
|
|
}
|
|
return nil
|
|
default:
|
|
// A secret key — see authResolveProvider for why the STORE that owns it,
|
|
// not its spelling, decides what it is.
|
|
provider, err := object.GetProviderByProviderKey(token, lang)
|
|
if err != nil {
|
|
return authError("invalid API key: %s", err.Error())
|
|
}
|
|
if provider != nil {
|
|
return nil
|
|
}
|
|
// getUserByAccessKey returns (nil, nil) for an unknown key (IAM 200 +
|
|
// data:null), so check BOTH the error AND a nil user — exactly as
|
|
// resolveProviderFromIAMKey does. Missing the nil-user case let an invalid
|
|
// key fall through to a 400 parse error instead of a 401.
|
|
user, err := getUserByAccessKey(token)
|
|
if err != nil {
|
|
// Same cloud-agent service-key fallback as resolveProviderFromIAMKey,
|
|
// so a valid service key is not falsely rejected here.
|
|
if tryCloudAgentKeyFallback(token) != nil {
|
|
return nil
|
|
}
|
|
return authError("API key validation failed: %s", err.Error())
|
|
}
|
|
if user == nil {
|
|
return authError("invalid API key")
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// providerKeyBillingUser derives the billing identity for a provider-key (sk-)
|
|
// caller: the org that OWNS the provider row the key belongs to (and therefore
|
|
// minted the key). The sk- key is a machine credential, so it bills the OWNER
|
|
// ORG — never a per-person wallet no one funds. A provider with no owner is
|
|
// unattributable: return an auth error so the caller refuses rather than spend
|
|
// the shared upstream key for free — the invariant is that every call spending
|
|
// the shared upstream key bills someone.
|
|
//
|
|
// It NAMES that ledger rather than implying it through a class. This identity is
|
|
// synthesized HERE, from a provider row this process just read — there is no
|
|
// person and no token — so the org is a fact the calling code holds, and it says
|
|
// so in the field that carries such statements. Marking it "application" and
|
|
// letting the payer infer the org worked everywhere except the one org where it
|
|
// mattered: in the signup org an org account is the platform's own balance, so an
|
|
// inferred class was the difference between billing a tenant and spending Hanzo's
|
|
// money. A stated ledger cannot be confused with an asserted class.
|
|
func providerKeyBillingUser(provider *object.Provider) (*iam.User, error) {
|
|
if provider == nil || strings.TrimSpace(provider.Owner) == "" {
|
|
return nil, authError("provider key is not attributable to a billable owner")
|
|
}
|
|
owner := strings.TrimSpace(provider.Owner)
|
|
return &iam.User{
|
|
Owner: owner,
|
|
Type: iam.Machine, // attribution: this call has no person behind it
|
|
BillingAccount: account.Org(owner).String(),
|
|
}, nil
|
|
}
|
|
|
|
// authResolveProvider authenticates a bearer token and resolves the requested
|
|
// model to its upstream provider. It is the single auth + model-routing policy
|
|
// for every OpenAI-compatible surface (chat, embeddings, rerank): each handler
|
|
// calls it instead of re-implementing the IAM/JWT/provider-key branches.
|
|
//
|
|
// Returns the resolved provider (with KMS-resolved secret), the billed user
|
|
// (nil for provider-key auth), the upstream model id, and whether the route is
|
|
// premium. Errors are returned pre-formatted for ResponseError.
|
|
func (c *ApiController) authResolveProvider(token, requestedModel, orgId string) (provider *object.Provider, authUser *iam.User, upstreamModel string, isPremium bool, err error) {
|
|
lang := c.GetAcceptLanguage()
|
|
ctx := c.Context()
|
|
|
|
switch {
|
|
case isRunKey(token):
|
|
// Run key (hrun_...) — an autonomous run buying inference on the ledger of
|
|
// the org that started it. It resolves to a machine principal and to NO
|
|
// user, so it authenticates nobody and opens no other surface; see run.go.
|
|
// The org rides the TOKEN, never orgId, so a run cannot be pointed at
|
|
// another tenant's balance by a header.
|
|
provider, authUser, upstreamModel, err = resolveProviderFromRunKey(ctx, token, requestedModel, lang)
|
|
if err != nil {
|
|
err = wrapAuth(err)
|
|
return
|
|
}
|
|
c.Locals("recordUserId", authUser.Owner+"/run")
|
|
return
|
|
|
|
case isJwtToken(token):
|
|
// hanzo.id JWT token — full model routing + billing. Same typed-status
|
|
// contract as the IAM key path. The raw X-Org-Id goes in unvalidated: the
|
|
// resolver holds the signed `orgs` claim and is the only place allowed to
|
|
// decide whether the request may act — and pay — in that org.
|
|
provider, authUser, upstreamModel, err = resolveProviderFromJwt(ctx, token, strings.TrimSpace(c.Header("X-Org-Id")), requestedModel, lang)
|
|
if err != nil {
|
|
err = wrapAuth(err)
|
|
return
|
|
}
|
|
|
|
case isPublishableKey(token):
|
|
// Publishable key (pk-...) — the read-only credential class. It reaches
|
|
// only the surfaces that do not refuse it up front (embeddings; every
|
|
// generative handler rejects pk- before calling here). IAM's publishable
|
|
// endpoint answers with the ORG that holds the key and never a person, so
|
|
// the call bills that org as a machine — the same shape as a provider
|
|
// key. This is the credential the cloud deployment documents for its
|
|
// embed client: least privilege for a read-only endpoint.
|
|
org, kerr := publishableOrg(token)
|
|
if kerr != nil {
|
|
err = wrapAuth(kerr)
|
|
return
|
|
}
|
|
machine := &iam.User{
|
|
Owner: org,
|
|
Type: iam.Machine, // attribution: no person is behind a page key
|
|
BillingAccount: account.Org(org).String(),
|
|
}
|
|
provider, authUser, upstreamModel, err = resolveProviderForUser(ctx, machine, org, requestedModel, lang)
|
|
if err != nil {
|
|
err = wrapAuth(err)
|
|
return
|
|
}
|
|
c.Locals("recordUserId", org+"/publishable")
|
|
|
|
default:
|
|
// A secret key, and the STORE that owns it decides what it is. Two
|
|
// families share the sk- spelling: an upstream vendor key, which lives in
|
|
// the provider table, and the key this estate mints, which lives in IAM.
|
|
// Both lookups are exact, so neither can claim the other's key; the order
|
|
// decides only who answers when NEITHER owns it, and IAM's refusal is the
|
|
// one that names the cure ("mint a new one at" + KeysURL) where
|
|
// a provider miss can only say "invalid".
|
|
provider, err = object.GetProviderByProviderKey(token, lang)
|
|
if err != nil {
|
|
err = authError("invalid API key: %s", err.Error())
|
|
return
|
|
}
|
|
if provider == nil {
|
|
// Not a vendor key — full model routing + billing off the IAM
|
|
// principal. resolveProviderFromIAMKey returns a typed apiError (401
|
|
// invalid key / 400 bad model / 402 balance / 500 misconfig); wrapAuth
|
|
// preserves it and 401s any untyped error.
|
|
provider, authUser, upstreamModel, err = resolveProviderFromIAMKey(ctx, token, requestedModel, lang)
|
|
if err != nil {
|
|
err = wrapAuth(err)
|
|
return
|
|
}
|
|
break
|
|
}
|
|
// Attribute + bill this vendor-key call to the org that OWNS the provider row
|
|
// (and thus minted this key). Without an authUser, BOTH reserveBudget and
|
|
// recordUsage are skipped (they gate on authUser != nil) — so an sk- key
|
|
// spending the SHARED upstream (do-ai) ran at ZERO cost and bypassed the
|
|
// balance/premium gate. Resolve the billing owner from the key's OWN
|
|
// provider row BEFORE any model-route swap below, so the debit lands on
|
|
// the key minter, not the upstream route provider we merely call. An
|
|
// unattributable provider (no owner) is refused (fail-secure).
|
|
authUser, err = providerKeyBillingUser(provider)
|
|
if err != nil {
|
|
return
|
|
}
|
|
c.Locals("recordUserId", authUser.Owner+"/provider-key")
|
|
// Apply model routing for sk- keys too. If the route points to a
|
|
// different provider than the one that owns the API key, switch to the
|
|
// route's provider so zen/fireworks models work with any key.
|
|
if route := resolveModelRouteForOrg(requestedModel, orgId); route != nil {
|
|
upstreamModel = route.upstreamModel
|
|
isPremium = route.premium
|
|
if route.providerName != provider.Name {
|
|
if routeProvider, routeErr := object.GetModelProviderByName(route.providerName); routeErr == nil && routeProvider != nil {
|
|
provider = routeProvider
|
|
}
|
|
}
|
|
}
|
|
// M1: apply the SAME prepaid-balance gate the JWT/IAM paths enforce
|
|
// (resolveProviderForUser), so an sk- provider key can never reach paid
|
|
// upstreams without a positive balance. The billed owner is authUser (the
|
|
// provider-row owner resolved just above), and a provider key carries no
|
|
// membership claim — it always bills the org that minted it.
|
|
if gateErr := enforceBalanceGate(ctx, authUser, authUser.Owner, requestedModel); gateErr != nil {
|
|
err = gateErr
|
|
return
|
|
}
|
|
return
|
|
}
|
|
|
|
// Shared post-resolution for IAM/JWT auth: record the billed user and the
|
|
// premium flag from the route table.
|
|
if authUser != nil {
|
|
c.Locals("recordUserId", authUser.Owner+"/"+authUser.Name)
|
|
}
|
|
if route := resolveModelRouteForOrg(requestedModel, orgId); route != nil {
|
|
isPremium = route.premium
|
|
}
|
|
return
|
|
}
|
|
|
|
// ChatCompletions implements the OpenAI-compatible chat completions API
|
|
// @Title ChatCompletions
|
|
// @Tag OpenAI Compatible API
|
|
// @Description OpenAI compatible chat completions API. Accepts:
|
|
// - IAM API key (sk-...) — full model routing + billing
|
|
// - hanzo.id JWT token — full model routing + billing
|
|
// - Provider API key — direct provider access
|
|
//
|
|
// @Param body body openai.ChatCompletionRequest true "The OpenAI chat request"
|
|
// @Success 200 {object} openai.ChatCompletionResponse
|
|
// @router /chat [post]
|
|
// A completion renders itself to the client. A sink, when a caller supplies one,
|
|
// renders it somewhere else instead: /v1/responses reads this same completion in a
|
|
// second dialect, and used to do it by swapping the writer out of the request
|
|
// context and reading what came back.
|
|
//
|
|
// Two functions, because a stream and a whole body are two different things and
|
|
// collapsing them is what made the old version need a status machine. `wrap`
|
|
// decorates the stream's destination as it is produced; `whole` translates the
|
|
// non-streaming answer entire, wherever it is written — answerBody, or the writer
|
|
// that assembles a whole answer from a family's stream (assembleZenStream).
|
|
type sink struct {
|
|
wrap func(io.Writer) io.Writer
|
|
whole func([]byte) ([]byte, error)
|
|
}
|
|
|
|
// shape is one whole chat completion in the dialect the request asked for: as it
|
|
// is with no sink, or translated by it.
|
|
func (s *sink) shape(chat []byte) ([]byte, error) {
|
|
if s == nil || s.whole == nil {
|
|
return chat, nil
|
|
}
|
|
return s.whole(chat)
|
|
}
|
|
|
|
// ChatCompletions implements the OpenAI-compatible chat completions API
|
|
// @Title ChatCompletions
|
|
// @Tag OpenAI Compatible API
|
|
// @Description OpenAI compatible chat completions API. Accepts:
|
|
// - IAM API key (sk-...) — full model routing + billing
|
|
// - hanzo.id JWT token — full model routing + billing
|
|
// - Provider API key — direct provider access
|
|
//
|
|
// @Param body body openai.ChatCompletionRequest true "The OpenAI chat request"
|
|
// @Success 200 {object} openai.ChatCompletionResponse
|
|
// @router /chat [post]
|
|
func (c *ApiController) ChatCompletions() { c.chatCompletions(callerBearer, nil) }
|
|
|
|
// caller says which address a completion arrived at, and therefore what is taken from
|
|
// the request and what is decided for it. It is an unexported type with no decoded
|
|
// form, so no header, body or query can produce one: the route decides it and the
|
|
// request cannot. callerPublic is reachable only from ChatCompletionsPublic, after
|
|
// that lane's ceiling has admitted the call.
|
|
type caller int
|
|
|
|
const (
|
|
callerBearer caller = iota // a credential is presented and decides everything
|
|
callerPublic // no credential; the public lane decided everything
|
|
)
|
|
|
|
// chatCompletions is the one completion pipeline. Both entry points run it; they
|
|
// differ only in who the caller is, which is a value passed in rather than a fact
|
|
// re-derived here.
|
|
func (c *ApiController) chatCompletions(from caller, to *sink) {
|
|
c.answer = to
|
|
var token string
|
|
if from == callerBearer {
|
|
// Extract Bearer token
|
|
authHeader := c.Header("Authorization")
|
|
if !strings.HasPrefix(authHeader, "Bearer ") {
|
|
c.ResponseErrorWithStatus(401, c.T("openai:Invalid API key format. Expected 'Bearer API_KEY'"))
|
|
return
|
|
}
|
|
|
|
token = strings.TrimPrefix(authHeader, "Bearer ")
|
|
|
|
// Publishable keys (pk-) cannot access completions — reject early
|
|
if isPublishableKey(token) {
|
|
c.rejectPublishableKey()
|
|
return
|
|
}
|
|
}
|
|
|
|
// Track timing for observability
|
|
requestStartTime := time.Now().UTC()
|
|
|
|
// Parse request body. Authenticate BEFORE reporting a parse error so an
|
|
// invalid credential is 401 regardless of body validity — a malformed body
|
|
// from an unauthenticated caller must not return 200. A valid credential with
|
|
// a bad body gets 400 (not 200).
|
|
var parsed chatRequest
|
|
refusal := ""
|
|
if err := json.Unmarshal(c.Body(), &parsed); err != nil {
|
|
refusal = fmt.Sprintf("Failed to parse request: %s", parseProblem(err))
|
|
} else if why := casefolded(c.Body()); why != "" {
|
|
refusal = why
|
|
} else if f := unpricedField(c.Body()); f != "" {
|
|
refusal = fmt.Sprintf("%q is not accepted: it would buy something this model's price does not cover. Remove it and send the request again.", f)
|
|
} else if parsed.N > 1 {
|
|
// n completions cost n times one, and the hold covers one. Priced by n, a
|
|
// request would be admitted on a balance a single answer fits and settle far
|
|
// past it; one answer per request is the price that was quoted.
|
|
refusal = fmt.Sprintf("n=%d is not accepted: one request is one completion here.", parsed.N)
|
|
}
|
|
if refusal != "" {
|
|
if from == callerBearer {
|
|
if authErr := c.authenticate(token); authErr != nil {
|
|
c.ResponseAuthError(authErr)
|
|
return
|
|
}
|
|
}
|
|
c.ResponseErrorWithStatus(http.StatusBadRequest, refusal)
|
|
return
|
|
}
|
|
request := parsed.ChatCompletionRequest
|
|
|
|
// Resolve org context for per-org model routing and pricing.
|
|
orgId := c.GetOrg()
|
|
if from == callerPublic {
|
|
// THE PUBLIC LANE READS NEITHER THE MODEL NOR THE ORG FROM THE REQUEST. Both
|
|
// are overwritten before anything downstream can consult them, so naming a
|
|
// paid model or another tenant's org is not refused — it is unrepresentable.
|
|
// The assignment is here rather than in the lane because this is the last
|
|
// point at which the body could still be read.
|
|
request.Model, orgId = freeID, publicOrg
|
|
}
|
|
|
|
// A PUBLIC CALL ACTS UNDER NO AMBIENT IDENTITY. The widget lives on our own
|
|
// pages, so a visitor may well be carrying a session for this estate; every
|
|
// helper that reads one would then attribute an anonymous call to whoever is
|
|
// signed in — the routing ledger would record their name, and the wallet
|
|
// resolver would try to switch them onto an org and answer "nobody to bill".
|
|
// The lane admitted a stranger, so a stranger is who the rest of this serves.
|
|
routingUser := c.routingUserId()
|
|
principal := c.principalUser()
|
|
if from == callerPublic {
|
|
routingUser, principal = "", nil
|
|
}
|
|
|
|
// `auto`/`zen-router` resolution (below) calls the router engine — an internal
|
|
// HTTP request carrying the caller's prompt — and records a RoutingEvent, BEFORE
|
|
// authResolveProvider authenticates further down. Authenticate FIRST so an
|
|
// UNAUTHENTICATED caller can never drive that internal machinery: auth precedes
|
|
// every side effect. authenticate is the SAME credential check authResolveProvider
|
|
// runs (a strict subset — credential only, no model/balance), read-only, so it
|
|
// never rejects a request the resolver would accept. Concrete models skip this
|
|
// (resolveAutoModel is a no-op for them), so the dominant path pays nothing.
|
|
strict := c.strictAsked()
|
|
if isAutoModel(request.Model) {
|
|
if authErr := c.authenticate(token); authErr != nil {
|
|
c.ResponseAuthError(authErr)
|
|
return
|
|
}
|
|
if strict {
|
|
c.refuseStrict("route_auto", "auto-routing chooses the model; name one")
|
|
return
|
|
}
|
|
}
|
|
// RouteAuto has usually rewritten "auto" to its choice before this handler runs,
|
|
// so the body alone no longer says the model was chosen for the caller.
|
|
if _, routed := c.Locals(autoRoutedKey).(autoRouted); routed && strict {
|
|
if authErr := c.authenticate(token); authErr != nil {
|
|
c.ResponseAuthError(authErr)
|
|
return
|
|
}
|
|
c.refuseStrict("route_auto", "auto-routing chose the model; name one")
|
|
return
|
|
}
|
|
|
|
// One request id, generated once here — it is the response id (`chatcmpl-<id>`),
|
|
// the usage-ledger request_id, AND the routing-event join key, so a later reward
|
|
// (POST /v1/ai/feedback) can be tied back to THIS decision. Generated
|
|
// before routing so resolveAutoModel can stamp it on the RoutingEvent.
|
|
requestId := uuid.NewString()
|
|
|
|
// Virtual `auto`/`zen-router` model → resolve to a concrete servable model id
|
|
// BEFORE any provider/pricing/billing resolution, so the ENTIRE existing path
|
|
// (auth+routing, ModelRoute fallbacks, zen identity, balance reserve/settle,
|
|
// usage record, response `model` echo) bills and reports the model that
|
|
// actually served. The transparency header lets callers see the routed choice.
|
|
// routedTask + routingRecorded carry the routing decision to the post-response
|
|
// judge hook at the tail of this handler (the LLM-as-a-judge dense reward): set
|
|
// in whichever branch records a RoutingEvent, so the judge scores only requests
|
|
// that are actually in the routing ledger.
|
|
var routedTask string
|
|
var routingRecorded bool
|
|
if r, ok := c.Locals(autoRoutedKey).(autoRouted); ok && r.routed == request.Model && from == callerBearer {
|
|
// RouteAuto resolved `auto` ahead of the balance gate and rewrote the request
|
|
// to name the SKU; the gate priced that SKU and the RoutingEvent is already
|
|
// recorded under r.requestId. Keep that id so the response and the ledger join
|
|
// the decision, and record nothing twice.
|
|
requestId = r.requestId
|
|
routedTask, routingRecorded = r.task, true
|
|
object.GlobalTraffic.RecordTask(
|
|
Country(c.Ctx),
|
|
"",
|
|
r.task,
|
|
)
|
|
} else if routed, task, ok := resolveAutoModel(request.Model, orgId, routingUser, requestId, principal, &request, c.sloFromHeaders()); ok {
|
|
request.Model = routed
|
|
routedTask, routingRecorded = task, true
|
|
c.SetHeader(RoutedModelHeader, routed)
|
|
// Fold this request's TASK into the region's task-mix for the live-traffic
|
|
// globe — geo from the edge headers only (NO IP), aggregates only. Best-effort.
|
|
object.GlobalTraffic.RecordTask(
|
|
Country(c.Ctx),
|
|
"",
|
|
task,
|
|
)
|
|
} else if !isAutoModel(request.Model) {
|
|
// NON-auto: record the caller's explicit model selection so EVERY request is
|
|
// a rateable, trainable data point — up/down feedback for all models, and the
|
|
// ledger captures which model served which task even when the caller picked.
|
|
task := recordExplicitRouting(request.Model, orgId, routingUser, requestId, &request)
|
|
routedTask, routingRecorded = task, true
|
|
object.GlobalTraffic.RecordTask(
|
|
Country(c.Ctx),
|
|
"",
|
|
task,
|
|
)
|
|
}
|
|
|
|
// Authenticate the bearer token and resolve the requested model to its
|
|
// upstream provider, premium flag, and (for IAM/JWT auth) the billed user.
|
|
// This is the ONE auth+routing policy, shared with /v1/embeddings and
|
|
// /v1/rerank — see authResolveProvider.
|
|
var (
|
|
provider *object.Provider
|
|
authUser *iam.User
|
|
upstreamModel string
|
|
isPremium bool
|
|
err error
|
|
)
|
|
if from == callerPublic {
|
|
// No credential to authenticate. The lane already settled who is asking and
|
|
// what they may run; this settles only who runs it.
|
|
provider, authUser, upstreamModel, err = c.resolveProviderForPublic()
|
|
} else {
|
|
provider, authUser, upstreamModel, isPremium, err = c.authResolveProvider(token, request.Model, orgId)
|
|
}
|
|
if err != nil {
|
|
c.ResponseAuthError(err)
|
|
return
|
|
}
|
|
// The ledger this request spends from — the org the switcher selected when the
|
|
// signed membership claim proves it, else the caller's home org. Resolved ONCE
|
|
// here so the reservation below and every usage record at the tail of this
|
|
// handler key on the SAME wallet the gate inside authResolveProvider just read.
|
|
ledger := c.billingOrg(authUser)
|
|
if from == callerPublic {
|
|
// Cap max_tokens for a caller nobody can be billed for: the public lane
|
|
// serves a stranger, so the answer is bounded instead.
|
|
if request.MaxTokens == 0 || request.MaxTokens > publicMaxTokens {
|
|
request.MaxTokens = publicMaxTokens
|
|
}
|
|
}
|
|
|
|
if provider.Category != "Model" {
|
|
c.ResponseError(fmt.Sprintf("Provider %s is not a model provider", provider.Name))
|
|
return
|
|
}
|
|
|
|
// Set the upstream model name on the provider. For JWT/IAM key auth, this
|
|
// is the translated upstream model from the routing table. For provider
|
|
// API key auth, fall back to the request model or provider's default.
|
|
if upstreamModel != "" {
|
|
provider.SubType = upstreamModel
|
|
} else if request.Model != "" {
|
|
provider.SubType = request.Model
|
|
}
|
|
|
|
// ── Balance reservation ────────────────────────────────────────────
|
|
// Hold an upper-bound budget for this request so concurrent requests for the
|
|
// same subject can't double-spend a balance the async debit hasn't applied
|
|
// yet. The router gate is coarse (balance>0); this enforces
|
|
// balance >= estimated cost and reserves it atomically. It is settled with
|
|
// the ACTUAL cost when the request completes (deferred fail-safe release).
|
|
var hold *budgetHold
|
|
// wide is how many providers this request may be offered to AT ONCE. It is
|
|
// decided with the reservation and never after: what a race costs is N
|
|
// completions, so the hold either covers them or the request is not raced.
|
|
wide := 1
|
|
// The prompt as sent, every message of it. The reservation prices it and the
|
|
// relay sizes the route by it, so it is measured once.
|
|
measured := estimatePromptTokens(&request)
|
|
if authUser != nil {
|
|
subject := authUser.PayerSubject(ledger)
|
|
// Clamp the upstream completion ceiling BEFORE reserving so the relayed
|
|
// upstream can never emit more than we reserve — the actual settle can
|
|
// never exceed the hold (R1b). reserveCompletionTokens also covers the
|
|
// QueryText pipeline's fixed cap, which ignores max_tokens.
|
|
// The ceiling a caller named under either key is the one the hold covers; one
|
|
// named only as max_completion_tokens was otherwise lowered to the floor.
|
|
if request.MaxTokens == 0 {
|
|
request.MaxTokens = request.MaxCompletionTokens
|
|
}
|
|
// On the paid lane the ceiling is never above the one its seat was sized for.
|
|
request.MaxTokens = laneTokens(c.Context(), clampMaxTokens(request.Model, request.MaxTokens))
|
|
// The vendor bills the whole body — tools, schemas and call arguments as well
|
|
// as message text — so the hold is never priced on less than its size.
|
|
est := estimateRequestCostCents(request.Model, max(measured, len(c.Body())/4), request.MaxTokens)
|
|
var ok bool
|
|
if hold, wide, ok = c.widthFor(authUser, subject, est); !ok {
|
|
c.ResponseAuthError(billingError("%s", object.InsufficientBalance(c.Host(), ledger, "request cost").Message))
|
|
return
|
|
}
|
|
}
|
|
// Released on the way out — unless a relayed stream took it. That answer settles
|
|
// from inside its own writer, which runs after this handler has returned, and a
|
|
// release here would win the one-shot settle before the answer's cost is known.
|
|
defer func() { hold.settle(0) }()
|
|
|
|
// ── Model families (Zen, Enso) ─────────────────────
|
|
// A family model is served by its family service, which owns identity, reasoning,
|
|
// the 1M ladder, vision, the fan-out, and the upstream. ai forwards verbatim and
|
|
// meters the result; it holds no family routing of its own (hip-00NN).
|
|
//
|
|
// A family is one provider among several. When it refuses for a reason of its
|
|
// own — its account is empty, it is rate limited, it is down — it writes
|
|
// nothing and hands back the reason, and the request carries on to the
|
|
// route's declared alternates below. That is the difference between a vendor
|
|
// running out of money and the product going dark.
|
|
var familyRefused []attempt
|
|
if fam := familyForProviderType(provider.Type); fam != nil && strict {
|
|
c.refuseStrict("translation", "this model is not served in a way that keeps the request as sent")
|
|
return
|
|
}
|
|
if fam := familyForProviderType(provider.Type); fam != nil {
|
|
// The free tier is served under freeDoor, so it answers as a model of ours.
|
|
sku := request.Model
|
|
if fam == freeFamily() && strings.EqualFold(strings.TrimSpace(sku), freeID) {
|
|
fam, sku = freeDoor()
|
|
}
|
|
familyRefused = c.pipeToFamily(fam, "chat/completions", "openai", sku, c.Body(), request.Stream, clampMaxTokens(sku, request.MaxTokens), orgId, authUser, isPremium, hold, requestStartTime)
|
|
if familyRefused == nil {
|
|
if request.Stream {
|
|
hold = nil // a family's stream settles its own hold, from its writer
|
|
}
|
|
return
|
|
}
|
|
recordRefusals(c.takeSnapshot(authUser), request.Model, familyRefused, authUser, isPremium, request.Stream, requestId, requestStartTime)
|
|
}
|
|
|
|
// ── Tools and images ───────────────────────────────────────────────
|
|
// A tool call or an image the family refused stops here. What follows a family
|
|
// is its route's tail — the resale copy of the model and the free text floor —
|
|
// and neither is chosen for tools or vision: an alternate could answer the call
|
|
// without the part that made it one, and an answer shaped wrongly is worse than
|
|
// an honest refusal that names the vendor and the reason.
|
|
//
|
|
// With the paid lane off, only that floor is asked, so the same holds.
|
|
tooled := len(request.Tools) > 0 || request.ToolChoice != nil || len(request.Functions) > 0 || request.FunctionCall != nil
|
|
media := requestHasMedia(&request)
|
|
if tooled || media {
|
|
if familyRefused != nil {
|
|
c.ResponseFailure(exhausted(request.Model, familyRefused))
|
|
return
|
|
}
|
|
if shut(c.Context(), provider) {
|
|
c.ResponseFailure(laneOff(c.Context(), request.Model))
|
|
return
|
|
}
|
|
}
|
|
|
|
// ── The relay ──────────────────────────────────────────────────────
|
|
// A route that speaks OpenAI's dialect is sent the caller's own request —
|
|
// every message, every parameter, tools and images alike — and fails over
|
|
// along the route exactly as the text pipeline does (forward.go).
|
|
if relays(provider) {
|
|
question := lastUserText(&request)
|
|
// The router judge scores a text answer against the turn that asked for it,
|
|
// as it did before; a tool call or an image is not what it scores. Every
|
|
// value it reads is taken now, because it runs where a stream is finished.
|
|
var score func(string)
|
|
if routingRecorded && !tooled && !media {
|
|
agent, country, org := strings.Clone(c.Header("User-Agent")), strings.Clone(Country(c.Ctx)), strings.Clone(orgId)
|
|
sku, task := request.Model, routedTask
|
|
score = func(answer string) {
|
|
judgeRoutedResponse(agent, org, requestId, country, sku, task, question, answer)
|
|
}
|
|
}
|
|
// A strict request goes where its route is declared to go. Sized to the prompt,
|
|
// routeForPrompt may move it to another row serving another model, which the
|
|
// digests, taken without the model, would not show.
|
|
route := routeForPrompt(request.Model, orgId, measured)
|
|
if strict {
|
|
route = resolveModelRouteForOrg(request.Model, orgId)
|
|
}
|
|
if c.forward(pass{
|
|
req: &request,
|
|
body: c.Body(),
|
|
primary: provider,
|
|
route: route,
|
|
prior: familyRefused,
|
|
knowledge: c.retrieveKnowledgeIfEnabled(
|
|
question, retrievalOwner(authUser), c.retrievalStore(), c.GetAcceptLanguage()),
|
|
prompt: measured,
|
|
user: authUser,
|
|
premium: isPremium,
|
|
hold: hold,
|
|
id: requestId,
|
|
start: requestStartTime,
|
|
judge: score,
|
|
strict: strict,
|
|
}) {
|
|
hold = nil
|
|
}
|
|
return
|
|
}
|
|
|
|
// Anthropic-type rows and rows with no OpenAI-compatible address keep the older
|
|
// path until the relay speaks their dialect: a tool call or an image converted
|
|
// for Anthropic, and plain text through the QueryText pipeline below.
|
|
//
|
|
// Neither carries a response format, so one the caller named is refused rather than
|
|
// ignored: an answer that skipped the schema would read as one that followed it.
|
|
// These rows move to the Messages translator when it lands, and this goes with them.
|
|
if strict {
|
|
c.refuseStrict("translation", "this model is not served in a way that keeps the request as sent")
|
|
return
|
|
}
|
|
var format struct {
|
|
Type string `json:"type"`
|
|
}
|
|
if json.Unmarshal(parsed.ResponseFormat, &format) == nil && format.Type != "" && format.Type != "text" {
|
|
c.ResponseFailure(modelError("response_format %q is not supported for %s on this route; send the request without it", format.Type, request.Model))
|
|
return
|
|
}
|
|
if tooled || media {
|
|
if model.Upstream(provider.Type) != model.Anthropic {
|
|
c.ResponseError("No upstream endpoint configured for provider: " + provider.Name)
|
|
return
|
|
}
|
|
// The body is built from the upstream's own id; the envelope, the usage row
|
|
// and the price read the SKU.
|
|
sku := request.Model
|
|
request.Model = provider.SubType
|
|
c.proxyToolRequestAnthropic(provider, &request, sku, requestStartTime, authUser, isPremium, orgId, requestId, hold)
|
|
return
|
|
}
|
|
|
|
// Extract messages content
|
|
var question string
|
|
var systemPrompt string
|
|
history := []*model.RawMessage{}
|
|
|
|
for _, msg := range request.Messages {
|
|
// Extract text from Content or MultiContent (array-style content parts)
|
|
text := msg.Content
|
|
if text == "" && len(msg.MultiContent) > 0 {
|
|
var parts []string
|
|
for _, part := range msg.MultiContent {
|
|
if part.Type == openai.ChatMessagePartTypeText && part.Text != "" {
|
|
parts = append(parts, part.Text)
|
|
}
|
|
}
|
|
text = strings.Join(parts, "\n")
|
|
}
|
|
switch msg.Role {
|
|
case "system":
|
|
systemPrompt = text
|
|
case "user":
|
|
question = text
|
|
case "assistant":
|
|
history = append(history, &model.RawMessage{
|
|
Author: "AI",
|
|
Text: text,
|
|
})
|
|
}
|
|
}
|
|
|
|
if question == "" {
|
|
c.ResponseError(c.T("openai:No user message found in the request"))
|
|
return
|
|
}
|
|
|
|
// Combine system prompt with user question if available
|
|
if systemPrompt != "" {
|
|
question = fmt.Sprintf("System: %s\n\nUser: %s", systemPrompt, question)
|
|
}
|
|
|
|
// The OpenAI-shaped view of the stream. Its destination is bound below, where
|
|
// the body is produced.
|
|
writer := &OpenAIWriter{
|
|
Buffer: []byte{},
|
|
RequestID: requestId,
|
|
Stream: request.Stream,
|
|
Cleaner: *NewCleaner(6),
|
|
Model: request.Model,
|
|
IncludeUsage: request.StreamOptions != nil && request.StreamOptions.IncludeUsage,
|
|
}
|
|
|
|
// Optional RAG: unified retrieval path shared with the old /chat-docs route.
|
|
// Enabled when any of the following is true:
|
|
// - Request header `X-Retrieval: 1` or body field `retrieval=true`
|
|
// - Header `X-Retrieval-Store` specifies a store
|
|
knowledge := c.retrieveKnowledgeIfEnabled(
|
|
question,
|
|
retrievalOwner(authUser),
|
|
c.retrievalStore(),
|
|
c.GetAcceptLanguage(),
|
|
)
|
|
|
|
// Resolve the route for failover (may have fallback providers), sized to the
|
|
// prompt: if this prompt is bigger than the provider we would normally use
|
|
// can hold, we route it to one that can rather than refusing it. See
|
|
// routeForPrompt — a too-large prompt is a fact about the PROVIDER, not the
|
|
// model, and the fallback chain already knows who else serves it.
|
|
promptTokens, _ := model.GetTokenSize(request.Model, question)
|
|
route := routeForPrompt(request.Model, orgId, promptTokens)
|
|
|
|
// Call the model provider, cascading to the route's alternates on a refusal
|
|
// that is about the vendor rather than about this request.
|
|
//
|
|
// Every routed model takes this path, not just the ones with fallbacks
|
|
// declared. A single-provider model has no alternate to move to, but it
|
|
// still earns the rest: its vendor gets demoted when it refuses, the refusal
|
|
// is recorded where someone reads it, and "everyone refused" comes back as
|
|
// an honest 503 naming the reason instead of an upstream 402 telling the
|
|
// customer THEY are out of money.
|
|
// EVERY BYTE OF THE BODY IS WRITTEN IN HERE. A fasthttp response is produced
|
|
// inside a stream callback rather than into a writer the handler holds, so the
|
|
// completion, its bookkeeping and its final chunk all have to run within one
|
|
// call. They run in the order they already ran — this is a wrap, not a
|
|
// reordering.
|
|
//
|
|
// `answered` carries what a bare `return` used to. Those returns are now
|
|
// returns from a closure, which would fall through to the judge below instead
|
|
// of skipping it. (Not `served` — that name is a type in this package.)
|
|
// Built HERE, not inside complete: it reads the caller's IP and holds it, and
|
|
// a provider beaten in a race is billed long after this handler is done with
|
|
// the request that named it.
|
|
billRaced := c.billRaced(request.Model, authUser, isPremium, request.Stream, requestId, requestStartTime)
|
|
|
|
// What the body reads from the request, read now. A streamed body is produced
|
|
// after this handler returns and fiber has released the request, so complete
|
|
// reads this and never the controller (base.go snapshot).
|
|
snap := c.takeSnapshot(authUser)
|
|
|
|
// complete produces the whole answer into out and returns what went wrong, if
|
|
// anything. It renders nothing itself: the caller knows whether a status can
|
|
// still be sent.
|
|
//
|
|
// It settles `own`, never `hold`. A streamed answer finishes after this handler
|
|
// has returned, and the handler's deferred release would win the one settle first
|
|
// and leave the ledger blind to what the stream cost; so a stream takes the hold
|
|
// from the handler below, and complete must still be holding it when it does.
|
|
own := hold
|
|
answered := false
|
|
complete := func(out io.Writer) error {
|
|
writer.out = out
|
|
var modelResult *model.ModelResult
|
|
var actualProvider served
|
|
var tried []attempt
|
|
|
|
// Fast mode, when the reservation covered it. Nil is the cascade, which
|
|
// is what every ordinary request gets.
|
|
var race *fan
|
|
if wide > 1 {
|
|
race = &fan{
|
|
n: wide,
|
|
to: out,
|
|
fork: func(o io.Writer) io.Writer { return writer.fork(o) },
|
|
adopt: writer.adopt,
|
|
bill: billRaced,
|
|
}
|
|
}
|
|
|
|
if route != nil {
|
|
modelResult, actualProvider, tried, err = ask{
|
|
ctx: snap.ctx,
|
|
route: route,
|
|
org: ledger,
|
|
model: request.Model,
|
|
primary: provider,
|
|
question: question,
|
|
history: history,
|
|
knowledge: knowledge,
|
|
lang: snap.lang,
|
|
writer: writer,
|
|
prompt: promptTokens,
|
|
fan: race,
|
|
sent: func() bool { return writer.StreamSent },
|
|
prior: familyRefused,
|
|
}.serve()
|
|
} else {
|
|
// Model absent from the route table: call the provider auth resolved.
|
|
var modelProvider model.ModelProvider
|
|
modelProvider, err = provider.GetModelProvider(snap.lang)
|
|
if err != nil {
|
|
return serverError("Failed to get model provider: %s", err.Error())
|
|
}
|
|
modelResult, err = modelProvider.QueryText(question, writer, history, "", knowledge, nil, snap.lang)
|
|
actualProvider = served{provider.Name, provider.Origin(), provider}
|
|
}
|
|
|
|
// Every vendor that refused goes in the ledger, whether or not one of them
|
|
// eventually served. A failover nobody can see leaves the empty account
|
|
// empty. The family's own refusal is already recorded above, so skip it here.
|
|
if n := len(familyRefused); len(tried) > n {
|
|
recordRefusals(snap, request.Model, tried[n:], authUser, isPremium, request.Stream, requestId, requestStartTime)
|
|
}
|
|
|
|
if err != nil {
|
|
// Record failed usage
|
|
if authUser != nil {
|
|
errRecord := &usageRecord{
|
|
Owner: ledger,
|
|
Model: request.Model,
|
|
Provider: actualProvider.name,
|
|
Origin: actualProvider.origin,
|
|
Premium: isPremium,
|
|
Stream: request.Stream,
|
|
Status: "error",
|
|
ErrorMsg: err.Error(),
|
|
ClientIP: snap.ip,
|
|
RequestID: requestId,
|
|
}
|
|
errRecord.PromptTokens, errRecord.CompletionTokens =
|
|
spent(modelResult, request.Model, promptTokens, writer.MessageString(), err)
|
|
errRecord.TotalTokens = errRecord.PromptTokens + errRecord.CompletionTokens
|
|
errRecord.bind(snap.ctx, authUser)
|
|
errRecord.BYO, errRecord.Account = providerBYO(provider, authUser)
|
|
recordUsage(errRecord)
|
|
recordTrace(snap.ctx, errRecord, requestStartTime)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Record successful usage (actualProvider reflects which provider served the request)
|
|
if authUser != nil {
|
|
successRecord := &usageRecord{
|
|
Owner: ledger,
|
|
Organization: authUser.Owner,
|
|
Model: request.Model,
|
|
Provider: actualProvider.name,
|
|
Origin: actualProvider.origin,
|
|
PromptTokens: modelResult.PromptTokenCount,
|
|
CacheReadTokens: modelResult.CacheReadTokenCount,
|
|
CacheWriteTokens: modelResult.CacheWriteTokenCount,
|
|
CompletionTokens: modelResult.ResponseTokenCount,
|
|
TotalTokens: modelResult.TotalTokenCount,
|
|
Currency: "USD",
|
|
Premium: isPremium,
|
|
Stream: request.Stream,
|
|
Status: "success",
|
|
ClientIP: snap.ip,
|
|
RequestID: requestId,
|
|
}
|
|
successRecord.bind(snap.ctx, authUser)
|
|
// Whether this call was "bring your own key" is a property of the row
|
|
// that SPENT a credential, not of the row auth resolved before failover
|
|
// moved the request. Reading the latter is how a call served on the
|
|
// platform's key gets billed as BYO — 1% instead of the token cost, with
|
|
// the platform eating the upstream.
|
|
successRecord.BYO, successRecord.Account = providerBYO(actualProvider.row, authUser)
|
|
recordUsage(successRecord)
|
|
recordTrace(snap.ctx, successRecord, requestStartTime)
|
|
// Settle the reservation with the ACTUAL cost (this works identically for
|
|
// streaming and non-streaming non-tool responses — both have real token
|
|
// counts here from the QueryText pipeline).
|
|
own.settle(calculateCostCentsWithCache(request.Model, modelResult.PromptTokenCount, modelResult.ResponseTokenCount, modelResult.CacheReadTokenCount, modelResult.CacheWriteTokenCount))
|
|
}
|
|
|
|
// Handle response based on streaming mode
|
|
if !request.Stream {
|
|
answer := writer.MessageString()
|
|
|
|
response := openai.ChatCompletionResponse{
|
|
ID: "chatcmpl-" + requestId,
|
|
Object: "chat.completion",
|
|
Created: util.GetCurrentUnixTime(),
|
|
Model: request.Model,
|
|
Choices: []openai.ChatCompletionChoice{
|
|
{
|
|
Index: 0,
|
|
Message: openai.ChatCompletionMessage{
|
|
Role: "assistant",
|
|
Content: answer,
|
|
},
|
|
FinishReason: openai.FinishReasonStop,
|
|
},
|
|
},
|
|
Usage: openai.Usage{
|
|
PromptTokens: modelResult.PromptTokenCount,
|
|
CompletionTokens: modelResult.ResponseTokenCount,
|
|
TotalTokens: modelResult.TotalTokenCount,
|
|
},
|
|
}
|
|
|
|
jsonResponse, err := json.Marshal(response)
|
|
if err != nil {
|
|
return serverError("%s", err.Error())
|
|
}
|
|
|
|
// Only the non-streaming answer is written here, and it runs inside the
|
|
// handler, where the request is still ours.
|
|
c.answerBody(jsonResponse)
|
|
} else {
|
|
err = writer.Close(
|
|
modelResult.PromptTokenCount,
|
|
modelResult.ResponseTokenCount,
|
|
modelResult.TotalTokenCount,
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
answered = true
|
|
return nil
|
|
}
|
|
|
|
// LLM-as-a-judge: score THIS served (prompt, response) into a dense quality reward
|
|
// for the enso router. It reads the ANSWER, so it runs where the answer is finished
|
|
// — which for a stream is inside the callback, not after it: the callback has not
|
|
// run when SendStreamWriter returns, so out here MessageString() is still empty and
|
|
// every streamed turn would be scored on nothing. It never adds latency either way;
|
|
// the hook only spawns a goroutine after cheap gates, and is a no-op unless
|
|
// ROUTER_JUDGE_ENABLED. The prompt and response are passed transiently — never
|
|
// persisted (see router_judge.go).
|
|
agent, country := c.Header("User-Agent"), Country(c.Ctx)
|
|
judge := func() {
|
|
if answered && routingRecorded {
|
|
judgeRoutedResponse(agent, orgId, requestId, country, request.Model, routedTask, question, writer.MessageString())
|
|
}
|
|
}
|
|
|
|
if request.Stream {
|
|
// THE FIRST BYTE DECIDES THE STATUS. Handing zip the stream commits a 200, so
|
|
// the answer is produced into `unsent` first and this handler waits for its first
|
|
// byte. A request every provider refused before a byte was written is answered
|
|
// with the status that says so; one that has begun is handed to the stream with
|
|
// what it already said, and a failure after that goes out as an error event.
|
|
out := newUnsent()
|
|
var failed error
|
|
done := make(chan struct{})
|
|
hold = nil // the stream settles it: complete with the cost, or at zero if it failed
|
|
go func() {
|
|
defer close(done)
|
|
defer own.settle(0)
|
|
failed = complete(out)
|
|
judge()
|
|
}()
|
|
select {
|
|
case <-out.first:
|
|
case <-done:
|
|
}
|
|
if !out.opened() && failed != nil {
|
|
c.ResponseModelFailure(failed)
|
|
return
|
|
}
|
|
// The headers go on before the callback: the first chunk commits the status
|
|
// line, and nothing can be added to it afterwards.
|
|
c.SetHeader("Content-Type", "text/event-stream")
|
|
c.SetHeader("Cache-Control", "no-cache")
|
|
c.SetHeader("Connection", "keep-alive")
|
|
_ = c.SendStreamWriter(func(bw *bufio.Writer) {
|
|
attached := out.attach(bw)
|
|
<-done
|
|
if failed != nil && attached == nil {
|
|
streamOpenAIError(bw, failed)
|
|
}
|
|
})
|
|
return
|
|
}
|
|
|
|
// Nothing is streamed: OpenAIWriter accumulates into MessageBuf and the one JSON
|
|
// body is written by complete itself, so there is no stream to name.
|
|
if err := complete(io.Discard); err != nil {
|
|
c.ResponseModelFailure(err)
|
|
}
|
|
judge()
|
|
}
|
|
|
|
// ListModels returns the list of available models from the routing table.
|
|
//
|
|
// PUBLIC BY DESIGN, AND IT DOES NOT AUTHENTICATE — that is the whole contract, so it
|
|
// is stated here rather than left to be inferred. The catalogue is the same for
|
|
// everyone (listAvailableModels takes no principal), docs.hanzo.ai fetches it from
|
|
// the browser, and every policy layer around it says so: the authz filter lists
|
|
// "models" as public, filter_balance does not gate it, the rate limiter excludes it,
|
|
// and cloud's spend.Reachable carries /v1/models/ as "the model catalog the shell
|
|
// reads for discovery".
|
|
//
|
|
// SO THE Authorization HEADER IS NOT AN ADMISSION CHECK HERE. It is read for ONE
|
|
// thing — annotating gated SKUs with the caller's own access standing — and
|
|
// annotation degrades to nothing when there is no verified principal.
|
|
//
|
|
// A credential that is presented is verified, never merely shape-checked: a caller
|
|
// using /v1/models to ask "is my key working?" gets an honest answer, because a key
|
|
// that does not verify annotates nothing rather than reading as accepted.
|
|
//
|
|
// @Title ListModels
|
|
// @Tag OpenAI Compatible API
|
|
// @Description Returns a list of all available models. Public — no authentication.
|
|
// @Success 200 {object} object
|
|
// @router /models [get]
|
|
func (c *ApiController) ListModels() {
|
|
// Gated (limited-preview) SKUs carry the caller's access standing
|
|
// (waitlist|requested|granted), so the client can show "request access" vs
|
|
// "granted" without a second call.
|
|
//
|
|
// The ONLY use of the credential on this route, and it is best-effort:
|
|
// principalUser returns nil unless the request carries a VERIFIED principal (a
|
|
// session, a signature-checked JWT, or a key IAM resolved). So an absent, expired
|
|
// or forged credential simply yields the un-annotated public catalogue — never a
|
|
// 401, and never another caller's standing.
|
|
jsonResponse, err := modelListing(c.principalUser())
|
|
if err != nil {
|
|
c.ResponseError(err.Error())
|
|
return
|
|
}
|
|
|
|
c.SetHeader("Content-Type", "application/json")
|
|
c.Bytes(http.StatusOK, jsonResponse)
|
|
}
|
|
|
|
// proxyToolRequestAnthropic handles tool-calling requests for Claude/Anthropic
|
|
// providers by converting the OpenAI format to Anthropic Messages API format
|
|
// and converting the response back.
|
|
// sku is the model the CALLER named. request.Model already carries the upstream's
|
|
// own id by the time this runs, so it is what the outbound Anthropic body is built
|
|
// from and nothing else: the envelope, the usage row and the price all read sku,
|
|
// which is the model this call was sold as.
|
|
func (c *ApiController) proxyToolRequestAnthropic(
|
|
provider *object.Provider,
|
|
request *openai.ChatCompletionRequest,
|
|
sku string,
|
|
requestStartTime time.Time,
|
|
authUser *iam.User,
|
|
isPremium bool,
|
|
orgId string,
|
|
requestId string,
|
|
hold *budgetHold,
|
|
) {
|
|
// The same wallet ChatCompletions gated and reserved on, re-derived from the same
|
|
// credential rather than threaded, so the two cannot be given different arguments.
|
|
ledger := c.billingOrg(authUser)
|
|
|
|
baseURL := provider.ProviderUrl
|
|
if baseURL == "" {
|
|
baseURL = "https://api.anthropic.com"
|
|
}
|
|
baseURL = strings.TrimRight(baseURL, "/")
|
|
|
|
// Convert OpenAI messages to Anthropic format
|
|
var systemPrompt string
|
|
anthropicMessages := []map[string]any{}
|
|
|
|
for _, msg := range request.Messages {
|
|
if msg.Role == "system" {
|
|
systemPrompt = msg.Content
|
|
continue
|
|
}
|
|
|
|
anthropicMsg := map[string]any{
|
|
"role": msg.Role,
|
|
}
|
|
|
|
if msg.Role == "tool" {
|
|
// Tool result message
|
|
anthropicMsg["role"] = "user"
|
|
anthropicMsg["content"] = []map[string]any{
|
|
{
|
|
"type": "tool_result",
|
|
"tool_use_id": msg.ToolCallID,
|
|
"content": msg.Content,
|
|
},
|
|
}
|
|
} else if len(msg.ToolCalls) > 0 {
|
|
// Assistant message with tool calls
|
|
content := []map[string]any{}
|
|
if msg.Content != "" {
|
|
content = append(content, map[string]any{
|
|
"type": "text",
|
|
"text": msg.Content,
|
|
})
|
|
}
|
|
for _, tc := range msg.ToolCalls {
|
|
var inputObj any
|
|
_ = json.Unmarshal([]byte(tc.Function.Arguments), &inputObj)
|
|
if inputObj == nil {
|
|
inputObj = map[string]any{}
|
|
}
|
|
content = append(content, map[string]any{
|
|
"type": "tool_use",
|
|
"id": tc.ID,
|
|
"name": tc.Function.Name,
|
|
"input": inputObj,
|
|
})
|
|
}
|
|
anthropicMsg["content"] = content
|
|
} else if len(msg.MultiContent) > 0 {
|
|
content := []map[string]any{}
|
|
for _, part := range msg.MultiContent {
|
|
if part.Type == openai.ChatMessagePartTypeText {
|
|
content = append(content, map[string]any{
|
|
"type": "text",
|
|
"text": part.Text,
|
|
})
|
|
}
|
|
}
|
|
anthropicMsg["content"] = content
|
|
} else {
|
|
anthropicMsg["content"] = msg.Content
|
|
}
|
|
|
|
anthropicMessages = append(anthropicMessages, anthropicMsg)
|
|
}
|
|
|
|
// Convert OpenAI tools to Anthropic tool format
|
|
anthropicTools := []map[string]any{}
|
|
for _, tool := range request.Tools {
|
|
if tool.Type == openai.ToolTypeFunction {
|
|
anthropicTool := map[string]any{
|
|
"name": tool.Function.Name,
|
|
"description": tool.Function.Description,
|
|
}
|
|
if tool.Function.Parameters != nil {
|
|
var params any
|
|
raw, _ := json.Marshal(tool.Function.Parameters)
|
|
_ = json.Unmarshal(raw, ¶ms)
|
|
anthropicTool["input_schema"] = params
|
|
} else {
|
|
anthropicTool["input_schema"] = map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{},
|
|
}
|
|
}
|
|
anthropicTools = append(anthropicTools, anthropicTool)
|
|
}
|
|
}
|
|
|
|
// Build Anthropic request. Anthropic caches only a prompt the request marks, so
|
|
// the top-level cache_control caches this one up to its last block (claudeCache):
|
|
// the next turn of the conversation reads that prefix at the cache rate.
|
|
anthropicReq := map[string]any{
|
|
"model": request.Model,
|
|
"messages": anthropicMessages,
|
|
"max_tokens": 4096,
|
|
"tools": anthropicTools,
|
|
"cache_control": ephemeral,
|
|
}
|
|
if systemPrompt != "" {
|
|
anthropicReq["system"] = systemPrompt
|
|
}
|
|
if request.MaxTokens > 0 {
|
|
anthropicReq["max_tokens"] = request.MaxTokens
|
|
}
|
|
if request.Temperature > 0 {
|
|
anthropicReq["temperature"] = request.Temperature
|
|
}
|
|
// Always fetch the FULL (non-streamed) Anthropic response so the body is
|
|
// parseable JSON carrying token usage for billing. If the client requested
|
|
// streaming, the converted result is re-emitted as SSE below. Previously
|
|
// stream=true made io.ReadAll+json.Unmarshal fail on the SSE body, so streamed
|
|
// tool calls errored AND were never billed.
|
|
|
|
body, err := json.Marshal(anthropicReq)
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to marshal Anthropic request: %s", err.Error()))
|
|
return
|
|
}
|
|
|
|
req, err := http.NewRequest(http.MethodPost, baseURL+"/v1/messages", bytes.NewReader(body))
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to create Anthropic request: %s", err.Error()))
|
|
return
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
upstream.Authorize(req, provider)
|
|
req.Header.Set("anthropic-version", "2023-06-01")
|
|
|
|
client := &http.Client{Timeout: 120 * time.Second}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Anthropic request failed: %s", err.Error()))
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Read full Anthropic response
|
|
respBody, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to read Anthropic response: %s", err.Error()))
|
|
return
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
c.ResponseFailure(relay(sku, provider.Name, resp.StatusCode, respBody))
|
|
return
|
|
}
|
|
|
|
// Parse Anthropic response
|
|
var anthropicResp struct {
|
|
ID string `json:"id"`
|
|
Content []struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text,omitempty"`
|
|
ID string `json:"id,omitempty"`
|
|
Name string `json:"name,omitempty"`
|
|
Input json.RawMessage `json:"input,omitempty"`
|
|
} `json:"content"`
|
|
StopReason string `json:"stop_reason"`
|
|
Usage AnthropicUsage `json:"usage"`
|
|
}
|
|
if err := json.Unmarshal(respBody, &anthropicResp); err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to parse Anthropic response: %s", err.Error()))
|
|
return
|
|
}
|
|
// The prompt as every record counts it: read fresh, read from the cache and
|
|
// written to it, each part billed at its own rate.
|
|
used := anthropicResp.Usage.billed(sku)
|
|
|
|
// Convert Anthropic response to OpenAI format
|
|
var contentText strings.Builder
|
|
var toolCalls []openai.ToolCall
|
|
toolCallIdx := 0
|
|
|
|
for _, block := range anthropicResp.Content {
|
|
switch block.Type {
|
|
case "text":
|
|
contentText.WriteString(block.Text)
|
|
case "tool_use":
|
|
tc := openai.ToolCall{
|
|
Index: &toolCallIdx,
|
|
ID: block.ID,
|
|
Type: openai.ToolTypeFunction,
|
|
Function: openai.FunctionCall{
|
|
Name: block.Name,
|
|
Arguments: string(block.Input),
|
|
},
|
|
}
|
|
toolCalls = append(toolCalls, tc)
|
|
toolCallIdx++
|
|
}
|
|
}
|
|
|
|
finishReason := openai.FinishReasonStop
|
|
if anthropicResp.StopReason == "tool_use" {
|
|
finishReason = openai.FinishReasonToolCalls
|
|
}
|
|
|
|
openaiResp := openai.ChatCompletionResponse{
|
|
ID: "chatcmpl-" + requestId,
|
|
Object: "chat.completion",
|
|
Created: util.GetCurrentUnixTime(),
|
|
Model: sku,
|
|
Choices: []openai.ChatCompletionChoice{
|
|
{
|
|
Index: 0,
|
|
Message: openai.ChatCompletionMessage{
|
|
Role: "assistant",
|
|
Content: contentText.String(),
|
|
ToolCalls: toolCalls,
|
|
},
|
|
FinishReason: finishReason,
|
|
},
|
|
},
|
|
Usage: openai.Usage{
|
|
PromptTokens: used.prompt(),
|
|
CompletionTokens: used.out,
|
|
TotalTokens: used.prompt() + used.out,
|
|
PromptTokensDetails: &openai.PromptTokensDetails{CachedTokens: used.read},
|
|
},
|
|
}
|
|
|
|
// Record usage
|
|
if authUser != nil {
|
|
successRecord := &usageRecord{
|
|
Owner: ledger,
|
|
Organization: authUser.Owner,
|
|
Model: sku,
|
|
Provider: provider.Name,
|
|
Origin: provider.Origin(),
|
|
PromptTokens: used.prompt(),
|
|
CacheReadTokens: used.read,
|
|
CacheWriteTokens: used.write,
|
|
CompletionTokens: used.out,
|
|
TotalTokens: used.prompt() + used.out,
|
|
Currency: "USD",
|
|
Premium: isPremium,
|
|
Stream: false,
|
|
Status: "success",
|
|
ClientIP: c.Fiber().IP(),
|
|
RequestID: requestId,
|
|
}
|
|
successRecord.bind(c.Context(), authUser)
|
|
successRecord.BYO, successRecord.Account = providerBYO(provider, authUser)
|
|
recordUsage(successRecord)
|
|
recordTrace(c.Context(), successRecord, requestStartTime)
|
|
}
|
|
// The same model the usage row above is priced from. recordUsage debits what
|
|
// usageCostCents makes of record.Model, so a hold settled against a different
|
|
// model would hold one number and bill another.
|
|
hold.settle(calculateCostCentsWithCache(sku, used.prompt(), used.out, used.read, used.write))
|
|
|
|
jsonResponse, err := json.Marshal(openaiResp)
|
|
if err != nil {
|
|
c.ResponseError(err.Error())
|
|
return
|
|
}
|
|
|
|
if request.Stream {
|
|
// Client asked for streaming: emit the converted completion as a single
|
|
// SSE chunk followed by [DONE], so OpenAI SDK clients consuming a stream
|
|
// still work while billing used the real (full-response) token usage.
|
|
c.SetHeader("Content-Type", "text/event-stream")
|
|
c.SetHeader("Cache-Control", "no-cache")
|
|
c.SetHeader("Connection", "keep-alive")
|
|
chunk := map[string]any{
|
|
"id": openaiResp.ID,
|
|
"object": "chat.completion.chunk",
|
|
"created": openaiResp.Created,
|
|
"model": openaiResp.Model,
|
|
"choices": []map[string]any{{
|
|
"index": 0,
|
|
"delta": map[string]any{
|
|
"role": "assistant",
|
|
"content": contentText.String(),
|
|
"tool_calls": toolCalls,
|
|
},
|
|
"finish_reason": finishReason,
|
|
}},
|
|
}
|
|
// The stream owns the connection from here: zip hands the handler a writer
|
|
// and flushes each frame as it is written, so a chunk reaches the client
|
|
// when it is produced rather than when the handler returns.
|
|
_ = c.SendStreamWriter(func(w *bufio.Writer) {
|
|
if chunkJSON, mErr := json.Marshal(chunk); mErr == nil {
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", string(chunkJSON))
|
|
}
|
|
_, _ = fmt.Fprint(w, "data: [DONE]\n\n")
|
|
})
|
|
return
|
|
}
|
|
|
|
c.answerBody(jsonResponse)
|
|
}
|
|
|
|
// planUse is what a covered call used of its plan, in nano-dollars: its list price,
|
|
// what the caller would have paid; for a model sold at zero, what its paid upstream
|
|
// cost us (planNanos), since its list price says nothing about what it spent.
|
|
func planUse(record *usageRecord) int64 {
|
|
if billed := usageMargin(record).BilledNano; billed > 0 {
|
|
return billed
|
|
}
|
|
return record.planNanos
|
|
}
|