mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-21 02:36:49 +00:00
Bind a public hz_ widget key to EXACTLY the models its owner provisioned — the docs Ask AI key to enso only — so a leaked key cannot reach arbitrary or expensive models the coarse global widget allowlist happens to permit. - object.WidgetKeyModels(token): per-key model allowlist over WIDGET_KEY_MODELS (KMS-first, env fallback), mirroring WidgetKeyOwner exactly; the two share one parser (loadKeyMap), so a widget key is configured the one way everywhere. - resolveProviderFromWidgetKey: a key with an explicit binding is restricted to that set; a key with no binding falls back to the global cheap-model allowlist. Fail-secure: an enso-bound key can never fall through to other models. - Tests: object + controller gate (bound key = enso-only, cannot reach premium; unbound key falls back to global; case-insensitive; JSON + comma-list forms). Guardrail for the docs.hanzo.ai support-AI: with rate limit (per-owner-org) and the prepaid balance gate already enforced, this caps WHICH models a leaked docs key can call. No behavior change for existing widget keys (no WIDGET_KEY_MODELS).
2094 lines
81 KiB
Go
2094 lines
81 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
// Portions Copyright 2025 The OpenAgent Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/hanzoai/account"
|
|
|
|
"github.com/hanzoai/ai/conf"
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/hanzoai/ai/model"
|
|
"github.com/hanzoai/ai/object"
|
|
"github.com/hanzoai/ai/util"
|
|
"github.com/hanzoai/go-openai"
|
|
)
|
|
|
|
// getUserBalance returns the current balance for a user by fetching from Commerce.
|
|
// Balance is mutable financial state (not identity) so it is never read from the
|
|
// JWT — always checked against the source of truth. Caching is handled by the
|
|
// router-level BalanceGate (routers/filter_balance.go); this controller-level
|
|
// call is a defense-in-depth backstop and does not maintain its own cache.
|
|
// The userId should be in "owner/name" format (e.g., "hanzo/alice").
|
|
// getUserBalance returns the available balance (in dollars) for an org. Billing
|
|
// is per-org: orgKey is the IAM org slug, used both as the balance destination
|
|
// key (?user=) and as the namespace selector (X-Org-Id), matching the gate
|
|
// and the per-org credit. Without the header, commerce's service-token path
|
|
// defaults to the "hanzo" namespace and a per-org credit is invisible.
|
|
func getUserBalance(subject, namespace string) (float64, error) {
|
|
// Native path: a co-resident host (cloud) reads the subject's wallet balance
|
|
// DIRECTLY from the in-process finance ledger — no HTTP. Cents → dollars.
|
|
if r := object.BalanceReader(); r != nil {
|
|
cents, err := r(context.Background(), subject, namespace, "usd")
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return float64(cents) / 100, nil
|
|
}
|
|
commerceEndpoint := conf.GetConfigString("commerceEndpoint")
|
|
if commerceEndpoint == "" {
|
|
return 0, fmt.Errorf("commerceEndpoint is not configured")
|
|
}
|
|
commerceEndpoint = strings.TrimRight(commerceEndpoint, "/")
|
|
commerceToken := conf.GetConfigString("commerceToken")
|
|
|
|
// Per global rule: /v1/ only, never /api/.
|
|
// All commerce endpoints live under /v1/.
|
|
// subject (?user=) is the per-user/per-org billing key; namespace
|
|
// (X-Org-Id) is the org. Both must match the gate and the usage debit.
|
|
reqURL := fmt.Sprintf("%s/v1/billing/balance?user=%s¤cy=usd", commerceEndpoint, url.QueryEscape(subject))
|
|
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
req, err := http.NewRequest(http.MethodGet, reqURL, nil)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("Commerce request build failed: %w", err)
|
|
}
|
|
if commerceToken != "" {
|
|
req.Header.Set("Authorization", "Bearer "+commerceToken)
|
|
}
|
|
// Scope the service-token call to this org's namespace.
|
|
req.Header.Set("X-Org-Id", namespace)
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("Commerce request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return 0, fmt.Errorf("Commerce returned status %d", resp.StatusCode)
|
|
}
|
|
|
|
var result struct {
|
|
Available int64 `json:"available"`
|
|
}
|
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
|
return 0, fmt.Errorf("failed to parse Commerce response: %w", err)
|
|
}
|
|
|
|
// Convert cents to dollars for backward compatibility with existing balance > 0 check
|
|
balanceDollars := float64(result.Available) / 100.0
|
|
|
|
return balanceDollars, nil
|
|
}
|
|
|
|
// isJwtToken checks if a token looks like a JWT (3 base64 segments separated by dots).
|
|
func isJwtToken(token string) bool {
|
|
parts := strings.Split(token, ".")
|
|
return len(parts) == 3 && len(parts[0]) > 10 && len(parts[1]) > 10
|
|
}
|
|
|
|
// isIAMApiKey checks if a token is an IAM-issued API key (hk- prefix).
|
|
func isIAMApiKey(token string) bool {
|
|
return strings.HasPrefix(token, "hk-")
|
|
}
|
|
|
|
// isPublishableKey checks if a token is a publishable API key (pk- prefix).
|
|
// Publishable keys are safe for client-side use and can only access read-only endpoints.
|
|
func isPublishableKey(token string) bool {
|
|
return strings.HasPrefix(token, "pk-")
|
|
}
|
|
|
|
// isWidgetKey checks if a token is a widget key (hz_ prefix).
|
|
// Widget keys provide restricted access for public-facing chat widgets
|
|
// on Hanzo properties (docs.hanzo.ai, hanzo.ai). They bypass balance
|
|
// checks but are limited to non-premium models with capped tokens.
|
|
func isWidgetKey(token string) bool {
|
|
return strings.HasPrefix(token, "hz_")
|
|
}
|
|
|
|
// validateWidgetKey checks a widget key against KMS-stored valid keys.
|
|
// Resolution order: KMS secret "WIDGET_KEYS" (comma-separated list),
|
|
// then WIDGET_KEYS env var, then rejects. This replaces the former
|
|
// hardcoded "hz_widget_public" check.
|
|
func validateWidgetKey(token string) bool {
|
|
// Try KMS first
|
|
if keys, err := object.GetKMSSecret("WIDGET_KEYS"); err == nil && keys != "" {
|
|
for _, k := range strings.Split(keys, ",") {
|
|
if strings.TrimSpace(k) == token {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Env var fallback (WIDGET_KEYS=hz_widget_public,hz_other_key)
|
|
if keys := os.Getenv("WIDGET_KEYS"); keys != "" {
|
|
for _, k := range strings.Split(keys, ",") {
|
|
if strings.TrimSpace(k) == token {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// No keys configured — reject all widget tokens
|
|
return false
|
|
}
|
|
|
|
// widgetMaxTokens caps the maximum tokens per widget request to control costs.
|
|
const widgetMaxTokens = 800
|
|
|
|
// Widget tenant resolution is bound to the widget KEY (see widgetKeyOwner in
|
|
// chat_retrieval.go), never the request Origin/Referer — a forgeable header must
|
|
// not select another tenant's data.
|
|
|
|
// widgetAllowedModels defines which models widget keys can access.
|
|
// Only cheap DO-AI models are allowed to keep costs minimal.
|
|
var widgetAllowedModels = map[string]bool{
|
|
"llama-3.1-8b": true,
|
|
"llama-3.3-70b": true,
|
|
"mistral-nemo": true,
|
|
"gpt-4o-mini": true,
|
|
"deepseek-r1-distill-70b": true,
|
|
"claude-3-5-haiku": true,
|
|
"claude-haiku-4-5": true,
|
|
}
|
|
|
|
// resolveProviderFromWidgetKey authenticates a widget key request.
|
|
// Widget keys skip balance checks but are restricted to non-premium models
|
|
// and have a token cap per request.
|
|
func resolveProviderFromWidgetKey(token string, requestedModel string, lang string) (*object.Provider, string, error) {
|
|
// Validate the widget key against KMS-stored keys, with env var fallback.
|
|
if !validateWidgetKey(token) {
|
|
return nil, "", fmt.Errorf("invalid widget key")
|
|
}
|
|
|
|
// Look up the model in the routing table
|
|
route := resolveModelRoute(requestedModel)
|
|
if route == nil {
|
|
return nil, "", fmt.Errorf(
|
|
"model %q is not available for widget access",
|
|
requestedModel,
|
|
)
|
|
}
|
|
|
|
// Model allowlist. A widget key may declare an EXPLICIT per-key allowlist
|
|
// (WIDGET_KEY_MODELS) that binds it to exactly the models its owner
|
|
// provisioned — the docs key to `enso` only — so a leaked key cannot reach
|
|
// arbitrary or expensive models the coarse global allowlist permits. A key
|
|
// with no explicit binding falls back to the global cheap-model allowlist.
|
|
if !widgetKeyModelAllowed(token, requestedModel) {
|
|
return nil, "", fmt.Errorf(
|
|
"model %q is not available for this widget key. Allowed models: %s",
|
|
requestedModel, widgetKeyAllowedList(token),
|
|
)
|
|
}
|
|
|
|
provider, err := object.GetModelProviderByName(route.providerName)
|
|
if err != nil {
|
|
return nil, "", fmt.Errorf("failed to get provider %q: %s", route.providerName, err.Error())
|
|
}
|
|
if provider == nil {
|
|
return nil, "", fmt.Errorf("provider %q not configured", route.providerName)
|
|
}
|
|
|
|
return provider, route.upstreamModel, nil
|
|
}
|
|
|
|
// widgetAllowedModelsList returns a comma-separated list of widget-allowed models.
|
|
func widgetAllowedModelsList() string {
|
|
models := make([]string, 0, len(widgetAllowedModels))
|
|
for m := range widgetAllowedModels {
|
|
models = append(models, m)
|
|
}
|
|
sort.Strings(models)
|
|
return strings.Join(models, ", ")
|
|
}
|
|
|
|
// widgetKeyModelAllowed reports whether a widget key may call requestedModel. A key
|
|
// with an explicit per-key allowlist (object.WidgetKeyModels, from WIDGET_KEY_MODELS)
|
|
// is bound to EXACTLY that set — the tightest, per-tenant guardrail (the docs key to
|
|
// `enso` only). A key with no explicit binding falls back to the global cheap-model
|
|
// allowlist (widgetAllowedModels). Matching is case-insensitive.
|
|
func widgetKeyModelAllowed(token, requestedModel string) bool {
|
|
m := strings.ToLower(strings.TrimSpace(requestedModel))
|
|
if allowed := object.WidgetKeyModels(token); len(allowed) > 0 {
|
|
for _, a := range allowed {
|
|
if a == m {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
return widgetAllowedModels[m]
|
|
}
|
|
|
|
// widgetKeyAllowedList is the human-readable allowlist for a widget key — its
|
|
// explicit per-key binding if it has one, else the global widget allowlist. Used
|
|
// only to build the 4xx rejection message.
|
|
func widgetKeyAllowedList(token string) string {
|
|
if allowed := object.WidgetKeyModels(token); len(allowed) > 0 {
|
|
sort.Strings(allowed)
|
|
return strings.Join(allowed, ", ")
|
|
}
|
|
return widgetAllowedModelsList()
|
|
}
|
|
|
|
// resolveProviderFromJwt validates a hanzo.id JWT token and returns the
|
|
// appropriate model provider for the requested model, plus the translated
|
|
// upstream model name.
|
|
func resolveProviderFromJwt(token string, requestedModel string, lang string) (*object.Provider, *iam.User, string, error) {
|
|
// Signature + issuer/audience validation (never raw iam.ParseJwtToken), so a
|
|
// token minted for a foreign app/issuer cannot authenticate a paid request.
|
|
claims, err := object.ParseAndValidateJWT(token)
|
|
if err != nil {
|
|
return nil, nil, "", authError("invalid hanzo.id token: %s", err.Error())
|
|
}
|
|
|
|
user := &claims.User
|
|
return resolveProviderForUser(user, requestedModel, lang)
|
|
}
|
|
|
|
// resolveProviderFromIAMKey validates an IAM API key (hk-{accessKey})
|
|
// and returns the model provider + user, same as JWT path.
|
|
func resolveProviderFromIAMKey(apiKey string, requestedModel string, lang string) (*object.Provider, *iam.User, string, error) {
|
|
// IAM API key format: hk-{uuid}
|
|
// Look up user by accessKey via IAM API
|
|
accessKey := apiKey // the full token including hk- prefix is the accessKey
|
|
|
|
user, err := getUserByAccessKey(accessKey)
|
|
if err != nil {
|
|
// IAM may return "password or code is incorrect" for service-account users
|
|
// (cloud-agent, etc.) due to a known IAM deployment quirk where the
|
|
// deployed binary handles certain user records differently. As a safe
|
|
// fallback, we check the key against the CLOUD_AGENT_KEY KMS secret
|
|
// (env CLOUD_AGENT_KEY as fallback). If it matches, we construct a
|
|
// minimal user identity and let the Commerce balance check validate
|
|
// the request as normal — so no billing bypass occurs.
|
|
if fallbackUser := tryCloudAgentKeyFallback(apiKey); fallbackUser != nil {
|
|
// Never log the API key (even masked) — owner/name identify the
|
|
// fallback identity for debugging without leaking the credential.
|
|
log.Warn("[iam-fallback] IAM returned %q; using cloud-agent fallback identity (owner=%s name=%s)",
|
|
err.Error(), fallbackUser.Owner, fallbackUser.Name)
|
|
return resolveProviderForUser(fallbackUser, requestedModel, lang)
|
|
}
|
|
return nil, nil, "", authError("API key validation failed: %s", err.Error())
|
|
}
|
|
if user == nil {
|
|
return nil, nil, "", authError("invalid API key")
|
|
}
|
|
|
|
return resolveProviderForUser(user, requestedModel, lang)
|
|
}
|
|
|
|
// tryCloudAgentKeyFallback checks whether apiKey matches the known cloud-agent
|
|
// service key stored in KMS (secret name "CLOUD_AGENT_KEY") with an env var
|
|
// fallback. Returns a minimal *iam.User on match, nil otherwise.
|
|
// This is intentionally narrow: only the exact key stored in KMS is accepted.
|
|
func tryCloudAgentKeyFallback(apiKey string) *iam.User {
|
|
// Try KMS first
|
|
var knownKey string
|
|
if v, err := object.GetKMSSecret("CLOUD_AGENT_KEY"); err == nil && v != "" {
|
|
knownKey = strings.TrimSpace(v)
|
|
}
|
|
// Env var fallback for local dev / bootstrap
|
|
if knownKey == "" {
|
|
knownKey = strings.TrimSpace(os.Getenv("CLOUD_AGENT_KEY"))
|
|
}
|
|
if knownKey == "" || apiKey != knownKey {
|
|
return nil
|
|
}
|
|
return &iam.User{
|
|
Owner: "hanzo",
|
|
Name: "cloud-agent",
|
|
}
|
|
}
|
|
|
|
// resolveProviderForUser is the shared logic for JWT and API key auth paths.
|
|
// Given a validated user, resolves the model route and provider.
|
|
func resolveProviderForUser(user *iam.User, requestedModel string, lang string) (*object.Provider, *iam.User, string, error) {
|
|
// Look up the model in the static routing table. A valid caller asking for
|
|
// an unknown model is a client error (400), not an auth failure.
|
|
route := resolveModelRoute(requestedModel)
|
|
if route == nil {
|
|
return nil, user, "", modelError(
|
|
"model %q is not available. Use GET /v1/models to list available models",
|
|
requestedModel,
|
|
)
|
|
}
|
|
|
|
// Fetch the provider entry that holds API keys/URLs for this upstream. Prefer
|
|
// the org's OWN custom provider (BYOK) if it configured one, else the global
|
|
// built-in provider on api.hanzo.ai. Returns a shallow copy, safe to mutate. A
|
|
// missing/unconfigured provider is a server-side misconfiguration (500).
|
|
provider, err := object.GetModelProviderByNameForOrg(user.Owner, route.providerName)
|
|
if err != nil {
|
|
return nil, user, "", serverError("failed to get provider %q: %s", route.providerName, err.Error())
|
|
}
|
|
if provider == nil {
|
|
return nil, user, "", serverError("provider %q not configured in database", route.providerName)
|
|
}
|
|
|
|
// Prepaid-balance gate. Extracted into enforceBalanceGate so the provider-key
|
|
// (sk-) path in authResolveProvider enforces the IDENTICAL policy — no auth
|
|
// path can drift (M1).
|
|
if gateErr := enforceBalanceGate(user, requestedModel); gateErr != nil {
|
|
return nil, user, "", gateErr
|
|
}
|
|
|
|
return provider, user, route.upstreamModel, nil
|
|
}
|
|
|
|
// enforceBalanceGate applies the prepaid-balance policy to a resolved billing
|
|
// principal and stamps user.Balance. A positive prepaid balance is required for
|
|
// EVERY model — there is no implicit free allowance and no per-model tier: a
|
|
// subject's credit is only ever what it paid or was explicitly granted (commerce
|
|
// owns the one grant endpoint). It returns a typed billingError (402) when the
|
|
// balance is not positive, a serverError (500) when the balance cannot be verified
|
|
// (fail-closed — never grant on a lookup transport error), or nil to proceed.
|
|
//
|
|
// It is the SINGLE gate shared by the JWT/IAM path (resolveProviderForUser) and
|
|
// the provider-key (sk-) path (authResolveProvider), so no auth path can drift
|
|
// (M1). There is NO exempt path: every principal is gated on a positive prepaid
|
|
// balance. subject is the per-namespace billing account the gate read, the budget
|
|
// reservation, and the usage debit all key on.
|
|
func enforceBalanceGate(user *iam.User, requestedModel string) error {
|
|
if user == nil {
|
|
return nil
|
|
}
|
|
orgKey := user.Owner // namespace (X-Org-Id): the org tenant
|
|
subject := account.Payer(account.Credential{Owner: user.Owner, Name: user.Name, Machine: account.IsMachine(user.Type)}).Subject()
|
|
|
|
balance, err := getUserBalance(subject, orgKey)
|
|
if err != nil {
|
|
// Balance unverifiable (Commerce down, a rejected service token, or an
|
|
// unset endpoint) → DENY. AI is prepaid: a broken or misconfigured billing
|
|
// backend must never degrade to free, ungated inference. There is no
|
|
// fail-open escape and no exempt principal.
|
|
return serverError("failed to verify account balance: %s", err.Error())
|
|
}
|
|
if balance <= 0 {
|
|
return billingError(
|
|
"model %q requires a positive balance. Your current balance is $%.2f. "+
|
|
"Add funds at https://hanzo.ai/billing",
|
|
requestedModel, balance,
|
|
)
|
|
}
|
|
|
|
user.Balance = balance
|
|
return nil
|
|
}
|
|
|
|
// iamAuthQuery returns the clientId/clientSecret query string for IAM API auth.
|
|
// Credentials are resolved in order: env vars (IAM_CLIENT_ID/IAM_CLIENT_SECRET),
|
|
// KMS secrets, then Beego config (for local dev).
|
|
func iamAuthQuery() string {
|
|
clientId := conf.GetConfigString("IAM_CLIENT_ID")
|
|
clientSecret := conf.GetConfigString("IAM_CLIENT_SECRET")
|
|
|
|
// Try KMS if config values are empty or placeholders
|
|
if clientId == "" {
|
|
if v, err := object.GetKMSSecret("IAM_CLIENT_ID"); err == nil && v != "" {
|
|
clientId = v
|
|
}
|
|
}
|
|
if clientSecret == "" {
|
|
if v, err := object.GetKMSSecret("IAM_CLIENT_SECRET"); err == nil && v != "" {
|
|
clientSecret = v
|
|
}
|
|
}
|
|
|
|
if clientId != "" && clientSecret != "" {
|
|
return "&clientId=" + url.QueryEscape(clientId) + "&clientSecret=" + url.QueryEscape(clientSecret)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// GetUserByAccessKey resolves an hk- IAM API key to its owning user via Hanzo
|
|
// IAM. Exported so the authz filter (package routers) resolves the key path to
|
|
// the same verified principal as the JWT path — one credential resolver, one
|
|
// tenant, one billing subject.
|
|
func GetUserByAccessKey(accessKey string) (*iam.User, error) {
|
|
return getUserByAccessKey(accessKey)
|
|
}
|
|
|
|
// getUserByAccessKey looks up a user by their IAM API key via Hanzo IAM.
|
|
func getUserByAccessKey(accessKey string) (*iam.User, error) {
|
|
// Call IAM's get-user endpoint with accessKey query parameter
|
|
iamEndpoint := conf.GetConfigString("IAM_URL")
|
|
if iamEndpoint == "" {
|
|
return nil, fmt.Errorf("IAM_URL is not configured")
|
|
}
|
|
iamEndpoint = strings.TrimRight(iamEndpoint, "/")
|
|
|
|
// Per global rule: /v1/ only, never /api/. IAM serves at /v1/iam/get-user.
|
|
// The legacy /api/get-user path is intercepted by the @hanzo/id SPA ingress
|
|
// and returns HTML, which broke hk- API-key resolution ("invalid character
|
|
// '<'"). iamAuthQuery() appends clientId/clientSecret so the IAM
|
|
// AutoSigninFilter authenticates this service call (the accessKey lookup
|
|
// requires an authenticated caller).
|
|
reqURL := fmt.Sprintf("%s/v1/iam/get-user?accessKey=%s%s", iamEndpoint, url.QueryEscape(accessKey), iamAuthQuery())
|
|
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
req, err := http.NewRequest(http.MethodGet, reqURL, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("IAM request build failed: %w", err)
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("IAM request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("IAM returned status %d", resp.StatusCode)
|
|
}
|
|
|
|
var result struct {
|
|
Status string `json:"status"`
|
|
Msg string `json:"msg"`
|
|
Data *iam.User `json:"data"`
|
|
}
|
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
|
return nil, fmt.Errorf("failed to parse IAM response: %w", err)
|
|
}
|
|
|
|
if result.Status != "ok" {
|
|
return nil, fmt.Errorf("IAM error: %s", result.Msg)
|
|
}
|
|
|
|
return result.Data, nil
|
|
}
|
|
|
|
// ── Usage tracking ──────────────────────────────────────────────────────────
|
|
|
|
// usageRecord mirrors IAM's UsageRecord for JSON serialization.
|
|
type usageRecord struct {
|
|
Owner string `json:"owner"`
|
|
User string `json:"user"`
|
|
Organization string `json:"organization"`
|
|
Model string `json:"model"`
|
|
Provider string `json:"provider"`
|
|
PromptTokens int `json:"promptTokens"`
|
|
CompletionTokens int `json:"completionTokens"`
|
|
TotalTokens int `json:"totalTokens"`
|
|
CacheReadTokens int `json:"cacheReadTokens,omitempty"`
|
|
CacheWriteTokens int `json:"cacheWriteTokens,omitempty"`
|
|
Cost float64 `json:"cost"`
|
|
Currency string `json:"currency"`
|
|
Premium bool `json:"premium"`
|
|
Stream bool `json:"stream"`
|
|
Status string `json:"status"`
|
|
ErrorMsg string `json:"errorMsg"`
|
|
ClientIP string `json:"clientIp"`
|
|
RequestID string `json:"requestId"`
|
|
|
|
// ImageCount is the number of images generated. Image models bill per image,
|
|
// not per token: when > 0, recordUsage bills via imageCostCents instead of
|
|
// the token-based cost. Placed last so the token-field alignment above is
|
|
// unchanged.
|
|
ImageCount int `json:"imageCount,omitempty"`
|
|
|
|
// VideoCount is the number of videos generated. Like images, video models
|
|
// bill per unit, not per token: when > 0, recordUsage bills via
|
|
// videoCostCents. Mutually exclusive with ImageCount in practice (a call is
|
|
// either an image or a video generation, never both).
|
|
VideoCount int `json:"videoCount,omitempty"`
|
|
|
|
// BYO marks a call executed against a customer-connected third-party account
|
|
// (the caller's org supplied its own provider key via /v1/ai/connections),
|
|
// rather than a Hanzo-served provider bought with Hanzo credits. When true the
|
|
// customer already paid the upstream directly, so recordUsage bills only the
|
|
// 1% platform fee (platformFeeCents) instead of the full token cost — the full
|
|
// cost is still recorded for analytics. Set at the emit site via providerBYO.
|
|
BYO bool `json:"byo,omitempty"`
|
|
|
|
// FeeCents is the Hanzo platform surcharge stamped by recordUsage
|
|
// (platformFeeCents(costCents, BYO)): the amount actually billed on a BYO
|
|
// call, 0 for a Hanzo-credits call. Persisted to the warehouse fee_cents column.
|
|
FeeCents int64 `json:"feeCents,omitempty"`
|
|
|
|
// Account is the attribution label for the account that served the call:
|
|
// "<owner>/<name>" for a BYO connection, "hanzo" for a Hanzo-served provider.
|
|
Account string `json:"account,omitempty"`
|
|
|
|
// Unpriced marks a token-billed call whose model had NO configured price in any
|
|
// source (family/route/conf/static), so it billed at the conservative default
|
|
// ($1/$4 per 1M) rather than a real rate. The debit is UNCHANGED — the flag just
|
|
// makes the guess honest: the row is queryable and the o11y span carries
|
|
// priced=false, instead of silently presenting an invented price as real. Set in
|
|
// recordUsage; image/video calls (own per-unit pricing) are never flagged.
|
|
Unpriced bool `json:"unpriced,omitempty"`
|
|
|
|
// BilledNanoExact / CostNanoExact carry an EXACT nano-USD billed amount and
|
|
// provider COGS when the biller knows them (zen's commerce Meter computes both
|
|
// per served tier). When > 0 they override usageMargin's table recompute, so a
|
|
// self-billing subsystem's warehouse row and span carry its true margin instead
|
|
// of a rate-table approximation. 0 = unset → the table path (unchanged).
|
|
BilledNanoExact int64 `json:"billedNanoExact,omitempty"`
|
|
CostNanoExact int64 `json:"costNanoExact,omitempty"`
|
|
|
|
// ── o11y gen_ai span enrichment (all optional, best-effort) ──────────────
|
|
// These carry the observation-of-record attribution the o11y span plane reads
|
|
// (controllers/telemetry.go emitGenAISpan). They are omitempty so a record that
|
|
// does not populate them emits no attribute — the span is honest, never a
|
|
// fabricated value, and each dimension lights up as its producer is wired.
|
|
|
|
// Session is the conversation/session id. Emitted as gen_ai.conversation.id +
|
|
// session.id, which is what turns the o11y sessions view on for this org.
|
|
Session string `json:"session,omitempty"`
|
|
// Environment is the caller's logical environment label (X-Environment). Emitted
|
|
// as deployment.environment on the span so Observe narrows by environment instead
|
|
// of defaulting to "default". Empty emits no attribute (honest, never fabricated).
|
|
Environment string `json:"environment,omitempty"`
|
|
// Project is the caller's org SUB-SCOPE (X-Project-Id). It is stamped on the
|
|
// cloud_usage ledger row + the gen_ai span, so cost/tokens/latency narrow WITHIN
|
|
// an org by project. Empty is the org's default project (whole-org view).
|
|
// Populated once in recordTrace from the request context (WithGenAIAttribution).
|
|
Project string `json:"project,omitempty"`
|
|
// APIKeyHash is a NON-reversible ref (SHA-256 hex) of the caller credential —
|
|
// never the plaintext key. Emitted on the gen_ai span (gen_ai.hanzo.api_key_hash)
|
|
// so a span correlates to a key without the store ever holding a secret.
|
|
// Populated in recordTrace from the request context.
|
|
APIKeyHash string `json:"apiKeyHash,omitempty"`
|
|
// ServedBy is where inference ran: "hanzo" (cloud), "byo-provider" (an
|
|
// org-owned provider key), or "byo-gpu" (an org's own cluster). Empty defaults
|
|
// to "hanzo" at emit — the cloud-served majority.
|
|
ServedBy string `json:"servedBy,omitempty"`
|
|
// ClusterID is the BYO-GPU cluster id when ServedBy == "byo-gpu".
|
|
ClusterID string `json:"clusterId,omitempty"`
|
|
// RoutePolicy is the enso route-policy decision that selected the model.
|
|
RoutePolicy string `json:"routePolicy,omitempty"`
|
|
// InputMessages / OutputMessages are the serialized prompt/completion. They are
|
|
// PII and are emitted ONLY when O11Y_GENAI_CAPTURE_MESSAGES is enabled
|
|
// (default off = redacted). Never logged, never billed — telemetry only.
|
|
InputMessages string `json:"inputMessages,omitempty"`
|
|
OutputMessages string `json:"outputMessages,omitempty"`
|
|
}
|
|
|
|
// billingQueue is the singleton usage record delivery queue. Initialized by
|
|
// InitBillingQueue() in main.go. If nil (Commerce not configured), recordUsage
|
|
// is a no-op.
|
|
var billingQueue *util.BillingQueue
|
|
|
|
// InitBillingQueue creates the billing queue from app config. Must be called
|
|
// once during startup. Returns the queue so main.go can call Shutdown().
|
|
func InitBillingQueue() *util.BillingQueue {
|
|
endpoint := conf.GetConfigString("commerceEndpoint")
|
|
if endpoint == "" {
|
|
return nil
|
|
}
|
|
endpoint = strings.TrimRight(endpoint, "/")
|
|
token := conf.GetConfigString("commerceToken")
|
|
|
|
billingQueue = util.NewBillingQueue(endpoint, token)
|
|
return billingQueue
|
|
}
|
|
|
|
// usageCostCents is the authoritative billable cost of a call, in cents. Image and
|
|
// video generations bill per unit (token counts are 0 on those paths); everything
|
|
// else bills from the cache-aware token table. A record carries at most one of
|
|
// ImageCount/VideoCount. This is the ONE cost of record: recordUsage debits it and
|
|
// emitGenAISpan reports it, so the ledger and the span never diverge.
|
|
func usageCostCents(record *usageRecord) int64 {
|
|
switch {
|
|
case record.VideoCount > 0:
|
|
return videoCostCents(record.Model, record.VideoCount)
|
|
case record.ImageCount > 0:
|
|
return imageCostCents(record.Model, record.ImageCount)
|
|
default:
|
|
return calculateCostCentsWithCache(
|
|
record.Model, record.PromptTokens, record.CompletionTokens,
|
|
record.CacheReadTokens, record.CacheWriteTokens,
|
|
)
|
|
}
|
|
}
|
|
|
|
// usageBilledCents is what Hanzo actually DEBITS the org ledger for a call, in
|
|
// cents, given its full provider cost: the full cost for a Hanzo-served call, or
|
|
// only the platform fee (platformFeeCents, ~1%) for a BYO call where the customer
|
|
// already paid the upstream with their own key. This is the ONE billed-amount of
|
|
// record: recordUsage debits it and emitGenAISpan reports it as
|
|
// _o11y.gen_ai.billed_cost, so Observe reconciles with the invoice — while
|
|
// _o11y.gen_ai.total_cost keeps the full provider cost for analytics.
|
|
func usageBilledCents(record *usageRecord, costCents int64) int64 {
|
|
if record.BYO {
|
|
return platformFeeCents(costCents, true)
|
|
}
|
|
return costCents
|
|
}
|
|
|
|
// recordUsage serializes a usage record and enqueues it for reliable delivery
|
|
// to Commerce. The queue handles retries with exponential backoff.
|
|
// Only successful API calls are recorded (error status is filtered here).
|
|
func recordUsage(record *usageRecord) {
|
|
// Dense flywheel reward (HIP-510): score EVERY routed request's outcome and
|
|
// attach it to its routing decision, so the bandit learns from request-volume
|
|
// signal instead of sparse explicit thumbs. Runs BEFORE the success filter so an
|
|
// errored request scores 0 (the arm is penalized for failing). Dark-by-default,
|
|
// async, best-effort — never slows or fails the request.
|
|
emitAutoRoutingReward(record)
|
|
|
|
// Only record successful calls
|
|
if record.Status != "success" {
|
|
return
|
|
}
|
|
|
|
// Calculate cost. usageCostCents is the ONE cost of record — the same value the
|
|
// o11y gen_ai span reports (emitGenAISpan), so the ledger debit and the span
|
|
// can never disagree.
|
|
costCents := usageCostCents(record)
|
|
|
|
// Honesty flag: if the model has no configured price, the cost above was billed at
|
|
// the conservative default. Mark the row (and warn once) rather than silently
|
|
// presenting the invented rate as real — the debit itself is unchanged.
|
|
if recordUnpriced(record) {
|
|
record.Unpriced = true
|
|
warnUnpricedOnce(record.Model)
|
|
}
|
|
|
|
// BYO: the customer paid the upstream directly with their own connected key,
|
|
// so Hanzo bills ONLY the 1% platform fee — not the token cost. The full
|
|
// costCents is still recorded (payload + warehouse) for analytics. On a
|
|
// Hanzo-served call (byo=false) feeCents is 0 and the token cost is billed as
|
|
// before. Stamp record.FeeCents so the warehouse writer persists the same value.
|
|
feeCents := platformFeeCents(costCents, record.BYO)
|
|
record.FeeCents = feeCents
|
|
amount := usageBilledCents(record, costCents)
|
|
|
|
// The debit MUST hit the same account the balance gate reads and the starter
|
|
// credit funded: the billing SUBJECT within the org NAMESPACE.
|
|
// namespace (X-Org-Id) = record.Owner (the org)
|
|
// subject (?user=) = account.Payer(account.Credential{Owner: owner, Name: name}).Subject()
|
|
// For a personal-billing org that is "owner/name" (per-user); for a pooled
|
|
// org it is the org slug. record.Owner is the IAM `owner`; fall back to
|
|
// deriving owner+name from "owner/name" if Owner was not populated upstream.
|
|
org := record.Owner
|
|
if org == "" {
|
|
if i := strings.IndexByte(record.User, '/'); i > 0 {
|
|
org = record.User[:i]
|
|
} else {
|
|
org = record.User
|
|
}
|
|
}
|
|
subject := account.PayerOf(record.Owner, record.User).Subject()
|
|
|
|
// Native in-proc finance debit — the ONE money path when co-resident with the
|
|
// finance ledger (hanzoai/cloud unified binary). The debit lands DIRECTLY on the
|
|
// org's SQLite wallet, the SAME account the prepaid gate reads, so a funded
|
|
// account both gates AND depletes. When the hook is installed we do NOT also
|
|
// enqueue to Commerce — that would double-bill. Standalone ai (no hook) falls
|
|
// through to the HTTP billing queue below.
|
|
if rec := object.UsageRecorder(); rec != nil {
|
|
if err := rec(context.Background(), object.UsageEvent{
|
|
Subject: subject,
|
|
Namespace: org,
|
|
USD: usageBilledUSD(record), // EXACT atto-precise debit, never a floored cent
|
|
Currency: "usd",
|
|
Model: record.Model,
|
|
Provider: record.Provider,
|
|
RequestID: record.RequestID,
|
|
}); err != nil {
|
|
log.Error("billing: native usage record failed request_id=%s: %v", record.RequestID, err)
|
|
}
|
|
return
|
|
}
|
|
|
|
if billingQueue == nil {
|
|
return
|
|
}
|
|
|
|
payload := map[string]interface{}{
|
|
"user": subject,
|
|
"actor": record.User,
|
|
"currency": "usd",
|
|
"amount": amount,
|
|
"model": record.Model,
|
|
"provider": record.Provider,
|
|
"promptTokens": record.PromptTokens,
|
|
"completionTokens": record.CompletionTokens,
|
|
"totalTokens": record.TotalTokens,
|
|
"cacheReadTokens": record.CacheReadTokens,
|
|
"cacheWriteTokens": record.CacheWriteTokens,
|
|
"imageCount": record.ImageCount,
|
|
"videoCount": record.VideoCount,
|
|
"requestId": record.RequestID,
|
|
"premium": record.Premium,
|
|
"stream": record.Stream,
|
|
"status": record.Status,
|
|
"clientIp": record.ClientIP,
|
|
"byo": record.BYO,
|
|
"feeCents": feeCents,
|
|
"costCents": costCents,
|
|
"account": record.Account,
|
|
}
|
|
|
|
body, err := json.Marshal(payload)
|
|
if err != nil {
|
|
log.Error("billing: failed to marshal usage record request_id=%s: %v", record.RequestID, err)
|
|
return
|
|
}
|
|
|
|
billingQueue.Enqueue(&util.BillingRecord{
|
|
Body: body,
|
|
RequestID: record.RequestID,
|
|
Org: org,
|
|
Model: record.Model,
|
|
})
|
|
}
|
|
|
|
// recordTrace persists an LLM/agent trace + usage record to hanzoai/datastore
|
|
// (native datastore OLAP) over native ZAP — the ONE internal telemetry path.
|
|
//
|
|
// The datastore (datastore) is reached directly via object.DatastoreExec
|
|
// (object/datastore.go) — the datastore image serves datastore on :8123/:9000,
|
|
// not a ZAP bridge. Fire-and-forget — failures are logged inside the writer,
|
|
// never block the request.
|
|
//
|
|
// Only the spend ledger (hanzo.cloud_usage) is written here — that is the ai
|
|
// module's table, read by GetCloudUsageOverview for the console Overview. The
|
|
// per-tenant trace ledger (canonical hanzo.observations / hanzo.traces) is owned
|
|
// and populated by the o11y/insights ingestion pipeline, not this module — one
|
|
// writer per table.
|
|
//
|
|
// Separately, emitGenAISpan ships one OpenTelemetry GenAI span per call to the
|
|
// o11y OTLP collector (opt-in via OTEL_EXPORTER_OTLP_ENDPOINT). That is the
|
|
// SOURCE the o11y pipeline ingests — this module never writes the observations
|
|
// table directly. The span emit is batched/async and a no-op when telemetry is
|
|
// off, so it never blocks the request.
|
|
func recordTrace(ctx context.Context, record *usageRecord, startTime time.Time) {
|
|
// Enrich the record with the request-scoped observability attribution the
|
|
// TenantContextFilter stashed on the context (project sub-scope + a
|
|
// non-reversible credential ref). Doing it HERE — the ONE funnel every
|
|
// completion surface passes through — stamps BOTH the warehouse row
|
|
// (zapWriteUsage) and the gen_ai span (emitGenAISpan) from a single place,
|
|
// rather than threading it through ~15 emit sites. A record that already set a
|
|
// value (an explicit producer) is left untouched.
|
|
if record != nil {
|
|
attr := object.GenAIAttributionFromContext(ctx)
|
|
// Org fallback: stamp the VERIFIED tenant org only when a producer set neither
|
|
// Organization nor Owner, so real tenant traffic always carries its org (the
|
|
// o11y llmobs views drop empty-org spans) without overriding an explicit value.
|
|
if record.Organization == "" && record.Owner == "" {
|
|
record.Organization = attr.Org
|
|
}
|
|
if record.Project == "" {
|
|
record.Project = attr.Project
|
|
}
|
|
if record.Session == "" {
|
|
record.Session = attr.Session
|
|
}
|
|
if record.Environment == "" {
|
|
record.Environment = attr.Environment
|
|
}
|
|
if record.APIKeyHash == "" {
|
|
record.APIKeyHash = attr.APIKeyHash
|
|
}
|
|
}
|
|
go zapWriteUsage(record, startTime)
|
|
emitGenAISpan(ctx, record, startTime)
|
|
}
|
|
|
|
// ── API handlers ────────────────────────────────────────────────────────────
|
|
|
|
// authenticate validates a bearer credential to a real principal WITHOUT routing
|
|
// a model — the authentication half of authResolveProvider. Handlers call it on a
|
|
// request-body error so an INVALID credential is rejected (401) regardless of
|
|
// body validity: a malformed (or field-incomplete) body from an unauthenticated
|
|
// caller must never return 200/400 that confirms the endpoint or lets it be
|
|
// probed. It is invoked only on the error path, so the happy path keeps a single
|
|
// validation (authResolveProvider). It mirrors authResolveProvider's four auth
|
|
// branches EXACTLY and never grants on an unknown key (fail-secure).
|
|
func (c *ApiController) authenticate(token string) error {
|
|
switch {
|
|
case isWidgetKey(token):
|
|
if !validateWidgetKey(token) {
|
|
return authError("Widget authentication failed: invalid widget key")
|
|
}
|
|
return nil
|
|
case isIAMApiKey(token):
|
|
// getUserByAccessKey returns (nil, nil) for an unknown key (IAM 200 +
|
|
// data:null), so check BOTH the error AND a nil user — exactly as
|
|
// resolveProviderFromIAMKey does. Missing the nil-user case let an invalid
|
|
// hk- key fall through to a 400 parse error instead of a 401.
|
|
user, err := getUserByAccessKey(token)
|
|
if err != nil {
|
|
// Same cloud-agent service-key fallback as resolveProviderFromIAMKey,
|
|
// so a valid service key is not falsely rejected here.
|
|
if tryCloudAgentKeyFallback(token) != nil {
|
|
return nil
|
|
}
|
|
return authError("API key validation failed: %s", err.Error())
|
|
}
|
|
if user == nil {
|
|
return authError("invalid API key")
|
|
}
|
|
return nil
|
|
case isJwtToken(token):
|
|
// Signature + issuer/audience validation (R3): a foreign-aud or
|
|
// wrong-issuer token is rejected here, not just signature-checked.
|
|
if _, err := object.ParseAndValidateJWT(token); err != nil {
|
|
return authError("invalid hanzo.id token: %s", err.Error())
|
|
}
|
|
return nil
|
|
default:
|
|
// Provider API key (sk-...). An unresolvable key is a 401.
|
|
provider, err := object.GetProviderByProviderKey(token, c.GetAcceptLanguage())
|
|
if err != nil {
|
|
return authError("invalid API key: %s", err.Error())
|
|
}
|
|
if provider == nil {
|
|
return authError("invalid API key")
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// providerKeyBillingUser derives the billing identity for a provider-key (sk-)
|
|
// caller: the org that OWNS the provider row the key belongs to (and therefore
|
|
// minted the key). The sk- key is a machine credential, so it bills the OWNER
|
|
// ORG — Type "application" marks it M2M (account.IsMachine), so account.Payer
|
|
// resolves it to the org account for every org, never a per-person wallet no one
|
|
// funds. A provider with no owner is
|
|
// unattributable: return an auth error so the caller refuses rather than spend
|
|
// the shared upstream key for free — the invariant is that every call spending
|
|
// the shared upstream key bills someone.
|
|
func providerKeyBillingUser(provider *object.Provider) (*iam.User, error) {
|
|
if provider == nil || strings.TrimSpace(provider.Owner) == "" {
|
|
return nil, authError("provider key is not attributable to a billable owner")
|
|
}
|
|
return &iam.User{Owner: provider.Owner, Type: "application"}, nil
|
|
}
|
|
|
|
// authResolveProvider authenticates a bearer token and resolves the requested
|
|
// model to its upstream provider. It is the single auth + model-routing policy
|
|
// for every OpenAI-compatible surface (chat, embeddings, rerank): each handler
|
|
// calls it instead of re-implementing the widget/IAM/JWT/provider-key branches.
|
|
//
|
|
// Returns the resolved provider (with KMS-resolved secret), the billed user
|
|
// (nil for widget/provider-key auth), the upstream model id, whether the route
|
|
// is premium, and whether the caller used an anonymous widget key. Errors are
|
|
// returned pre-formatted for ResponseError.
|
|
func (c *ApiController) authResolveProvider(token, requestedModel, orgId string) (provider *object.Provider, authUser *iam.User, upstreamModel string, isPremium bool, isWidget bool, err error) {
|
|
lang := c.GetAcceptLanguage()
|
|
|
|
switch {
|
|
case isWidgetKey(token):
|
|
// Widget key (hz_...) — restricted, token-capped model access (isWidget
|
|
// caps MaxTokens to widgetMaxTokens downstream). It is NOT free: like an
|
|
// sk- provider key, a widget call bills the OWNER ORG that minted the key
|
|
// (widgetKeyOwner → WIDGET_KEY_OWNERS / WIDGET_DEFAULT_OWNER), so the
|
|
// balance gate + budget reservation + usage debit all engage exactly as
|
|
// for a normal principal. Type "application" marks it a machine, so
|
|
// account.Payer resolves the billing subject to the org account. An
|
|
// unattributable widget key (no owner mapping and no default owner) is a
|
|
// config error: refuse rather than spend the shared upstream for free —
|
|
// the same fail-secure invariant the sk- path enforces (every call that
|
|
// spends the shared upstream bills someone).
|
|
isWidget = true
|
|
var widgetUpstream string
|
|
provider, widgetUpstream, err = resolveProviderFromWidgetKey(token, requestedModel, lang)
|
|
if err != nil {
|
|
err = authError("Widget authentication failed: %s", err.Error())
|
|
return
|
|
}
|
|
owner := widgetKeyOwner(token)
|
|
if strings.TrimSpace(owner) == "" {
|
|
err = authError("widget key is not attributable to a billable owner")
|
|
return
|
|
}
|
|
authUser = &iam.User{Owner: owner, Type: "application"}
|
|
upstreamModel = widgetUpstream
|
|
c.Ctx.Input.SetParam("recordUserId", owner+"/widget")
|
|
log.Info("Widget key access: owner=%s, model=%s, upstream=%s", owner, requestedModel, upstreamModel)
|
|
return
|
|
|
|
case isIAMApiKey(token):
|
|
// IAM API key (hk-...) — full model routing + billing. resolveProviderFromIAMKey
|
|
// returns a typed apiError (401 invalid key / 400 bad model / 402 balance /
|
|
// 500 misconfig); wrapAuth preserves it and 401s any untyped error.
|
|
provider, authUser, upstreamModel, err = resolveProviderFromIAMKey(token, requestedModel, lang)
|
|
if err != nil {
|
|
err = wrapAuth(err)
|
|
return
|
|
}
|
|
|
|
case isJwtToken(token):
|
|
// hanzo.id JWT token — full model routing + billing. Same typed-status
|
|
// contract as the IAM key path.
|
|
provider, authUser, upstreamModel, err = resolveProviderFromJwt(token, requestedModel, lang)
|
|
if err != nil {
|
|
err = wrapAuth(err)
|
|
return
|
|
}
|
|
|
|
default:
|
|
// Provider API key (sk-...) — direct provider access. An unresolvable key
|
|
// is an auth failure (401).
|
|
provider, err = object.GetProviderByProviderKey(token, lang)
|
|
if err != nil {
|
|
err = authError("invalid API key: %s", err.Error())
|
|
return
|
|
}
|
|
if provider == nil {
|
|
err = authError("invalid API key")
|
|
return
|
|
}
|
|
// Attribute + bill this sk- call to the org that OWNS the provider row
|
|
// (and thus minted this key). Without an authUser, BOTH reserveBudget and
|
|
// recordUsage are skipped (they gate on authUser != nil) — so an sk- key
|
|
// spending the SHARED upstream (do-ai) ran at ZERO cost and bypassed the
|
|
// balance/premium gate. Resolve the billing owner from the key's OWN
|
|
// provider row BEFORE any model-route swap below, so the debit lands on
|
|
// the key minter, not the upstream route provider we merely call. An
|
|
// unattributable provider (no owner) is refused (fail-secure).
|
|
authUser, err = providerKeyBillingUser(provider)
|
|
if err != nil {
|
|
return
|
|
}
|
|
c.Ctx.Input.SetParam("recordUserId", authUser.Owner+"/provider-key")
|
|
// Apply model routing for sk- keys too. If the route points to a
|
|
// different provider than the one that owns the API key, switch to the
|
|
// route's provider so zen/fireworks models work with any key.
|
|
if route := resolveModelRouteForOrg(requestedModel, orgId); route != nil {
|
|
upstreamModel = route.upstreamModel
|
|
isPremium = route.premium
|
|
if route.providerName != provider.Name {
|
|
if routeProvider, routeErr := object.GetModelProviderByName(route.providerName); routeErr == nil && routeProvider != nil {
|
|
provider = routeProvider
|
|
}
|
|
}
|
|
}
|
|
// M1: apply the SAME prepaid-balance gate the JWT/IAM paths enforce
|
|
// (resolveProviderForUser), so an sk- provider key can never reach paid
|
|
// upstreams without a positive balance. The billed owner is authUser (the
|
|
// provider-row owner resolved just above).
|
|
if gateErr := enforceBalanceGate(authUser, requestedModel); gateErr != nil {
|
|
err = gateErr
|
|
return
|
|
}
|
|
return
|
|
}
|
|
|
|
// Shared post-resolution for IAM/JWT auth: record the billed user and the
|
|
// premium flag from the route table.
|
|
if authUser != nil {
|
|
c.Ctx.Input.SetParam("recordUserId", authUser.Owner+"/"+authUser.Name)
|
|
}
|
|
if route := resolveModelRouteForOrg(requestedModel, orgId); route != nil {
|
|
isPremium = route.premium
|
|
}
|
|
return
|
|
}
|
|
|
|
// ChatCompletions implements the OpenAI-compatible chat completions API
|
|
// @Title ChatCompletions
|
|
// @Tag OpenAI Compatible API
|
|
// @Description OpenAI compatible chat completions API. Accepts:
|
|
// - Widget key (hz_...) — restricted models, no balance check, token-capped
|
|
// - IAM API key (hk-...) — full model routing + billing
|
|
// - hanzo.id JWT token — full model routing + billing
|
|
// - Provider API key — direct provider access
|
|
//
|
|
// @Param body body openai.ChatCompletionRequest true "The OpenAI chat request"
|
|
// @Success 200 {object} openai.ChatCompletionResponse
|
|
// @router /chat [post]
|
|
func (c *ApiController) ChatCompletions() {
|
|
// Extract Bearer token
|
|
authHeader := c.Ctx.Request.Header.Get("Authorization")
|
|
if !strings.HasPrefix(authHeader, "Bearer ") {
|
|
c.ResponseErrorWithStatus(401, c.T("openai:Invalid API key format. Expected 'Bearer API_KEY'"))
|
|
return
|
|
}
|
|
|
|
token := strings.TrimPrefix(authHeader, "Bearer ")
|
|
|
|
// Publishable keys (pk-) cannot access completions — reject early
|
|
if isPublishableKey(token) {
|
|
c.Ctx.Output.SetStatus(403)
|
|
c.Ctx.Output.Header("Content-Type", "application/json")
|
|
c.Ctx.Output.Body([]byte(`{"error":{"message":"Publishable keys (pk-) can only access read-only endpoints (/api/models, /health). Use a secret key (sk-) for completions.","type":"auth_error","code":403}}`))
|
|
c.EnableRender = false
|
|
return
|
|
}
|
|
|
|
// Track timing for observability
|
|
requestStartTime := time.Now().UTC()
|
|
|
|
// Parse request body. Authenticate BEFORE reporting a parse error so an
|
|
// invalid credential is 401 regardless of body validity — a malformed body
|
|
// from an unauthenticated caller must not return 200. A valid credential with
|
|
// a bad body gets 400 (not 200).
|
|
var request openai.ChatCompletionRequest
|
|
if err := json.Unmarshal(c.Ctx.Input.RequestBody, &request); err != nil {
|
|
if authErr := c.authenticate(token); authErr != nil {
|
|
c.ResponseAuthError(authErr)
|
|
return
|
|
}
|
|
c.ResponseErrorWithStatus(http.StatusBadRequest, fmt.Sprintf("Failed to parse request: %s", err.Error()))
|
|
return
|
|
}
|
|
|
|
// Resolve org context for per-org model routing and pricing.
|
|
orgId := c.GetOrg()
|
|
|
|
// `auto`/`zen-router` resolution (below) calls the router engine — an internal
|
|
// HTTP request carrying the caller's prompt — and records a RoutingEvent, BEFORE
|
|
// authResolveProvider authenticates further down. Authenticate FIRST so an
|
|
// UNAUTHENTICATED caller can never drive that internal machinery: auth precedes
|
|
// every side effect. authenticate is the SAME credential check authResolveProvider
|
|
// runs (a strict subset — credential only, no model/balance), read-only, so it
|
|
// never rejects a request the resolver would accept. Concrete models skip this
|
|
// (resolveAutoModel is a no-op for them), so the dominant path pays nothing.
|
|
if isAutoModel(request.Model) {
|
|
if authErr := c.authenticate(token); authErr != nil {
|
|
c.ResponseAuthError(authErr)
|
|
return
|
|
}
|
|
}
|
|
|
|
// One request id, generated once here — it is the response id (`chatcmpl-<id>`),
|
|
// the usage-ledger request_id, AND the routing-event join key, so a later reward
|
|
// (POST /v1/feedback) can be tied back to THIS decision. Generated
|
|
// before routing so resolveAutoModel can stamp it on the RoutingEvent.
|
|
requestId := util.GenerateUUID()
|
|
|
|
// Virtual `auto`/`zen-router` model → resolve to a concrete servable model id
|
|
// BEFORE any provider/pricing/billing resolution, so the ENTIRE existing path
|
|
// (auth+routing, ModelRoute fallbacks, zen identity, balance reserve/settle,
|
|
// usage record, response `model` echo) bills and reports the model that
|
|
// actually served. The transparency header lets callers see the routed choice.
|
|
// routedTask + routingRecorded carry the routing decision to the post-response
|
|
// judge hook at the tail of this handler (the LLM-as-a-judge dense reward): set
|
|
// in whichever branch records a RoutingEvent, so the judge scores only requests
|
|
// that are actually in the routing ledger.
|
|
var routedTask string
|
|
var routingRecorded bool
|
|
if routed, task, ok := resolveAutoModel(request.Model, orgId, c.routingUserId(), requestId, c.principalUser(), &request, c.sloFromHeaders()); ok {
|
|
request.Model = routed
|
|
routedTask, routingRecorded = task, true
|
|
c.Ctx.ResponseWriter.Header().Set(RoutedModelHeader, routed)
|
|
// Fold this request's TASK into the region's task-mix for the live-traffic
|
|
// globe — geo from the edge headers only (NO IP), aggregates only. Best-effort.
|
|
object.GlobalTraffic.RecordTask(
|
|
c.Ctx.Request.Header.Get("CF-IPCountry"),
|
|
c.Ctx.Request.Header.Get("CF-Region-Code"),
|
|
task,
|
|
)
|
|
} else if !isAutoModel(request.Model) {
|
|
// NON-auto: record the caller's explicit model selection so EVERY request is
|
|
// a rateable, trainable data point — up/down feedback for all models, and the
|
|
// ledger captures which model served which task even when the caller picked.
|
|
task := recordExplicitRouting(request.Model, orgId, c.routingUserId(), requestId, &request)
|
|
routedTask, routingRecorded = task, true
|
|
object.GlobalTraffic.RecordTask(
|
|
c.Ctx.Request.Header.Get("CF-IPCountry"),
|
|
c.Ctx.Request.Header.Get("CF-Region-Code"),
|
|
task,
|
|
)
|
|
}
|
|
|
|
// Authenticate the bearer token and resolve the requested model to its
|
|
// upstream provider, premium flag, and (for IAM/JWT auth) the billed user.
|
|
// This is the ONE auth+routing policy, shared with /v1/embeddings and
|
|
// /v1/rerank — see authResolveProvider.
|
|
provider, authUser, upstreamModel, isPremium, isWidget, err := c.authResolveProvider(token, request.Model, orgId)
|
|
if err != nil {
|
|
c.ResponseAuthError(err)
|
|
return
|
|
}
|
|
if isWidget {
|
|
// Cap max_tokens for anonymous widget requests.
|
|
if request.MaxTokens == 0 || request.MaxTokens > widgetMaxTokens {
|
|
request.MaxTokens = widgetMaxTokens
|
|
}
|
|
}
|
|
|
|
if provider.Category != "Model" {
|
|
c.ResponseError(fmt.Sprintf("Provider %s is not a model provider", provider.Name))
|
|
return
|
|
}
|
|
|
|
// Set the upstream model name on the provider. For JWT/IAM key auth, this
|
|
// is the translated upstream model from the routing table. For provider
|
|
// API key auth, fall back to the request model or provider's default.
|
|
if upstreamModel != "" {
|
|
provider.SubType = upstreamModel
|
|
} else if request.Model != "" {
|
|
provider.SubType = request.Model
|
|
}
|
|
|
|
// ── Balance reservation ────────────────────────────────────────────
|
|
// Hold an upper-bound budget for this request so concurrent requests for the
|
|
// same subject can't double-spend a balance the async debit hasn't applied
|
|
// yet. The router gate is coarse (balance>0); this enforces
|
|
// balance >= estimated cost and reserves it atomically. It is settled with
|
|
// the ACTUAL cost when the request completes (deferred fail-safe release).
|
|
var hold *budgetHold
|
|
if authUser != nil {
|
|
subject := account.Payer(account.Credential{Owner: authUser.Owner, Name: authUser.Name}).Subject()
|
|
// Clamp the upstream completion ceiling BEFORE reserving so the proxied
|
|
// (tool/stream) upstream can never emit more than we reserve — the actual
|
|
// settle can never exceed the hold (R1b). reserveCompletionTokens also
|
|
// covers the QueryText pipeline's fixed cap, which ignores max_tokens.
|
|
request.MaxTokens = clampMaxTokens(request.MaxTokens)
|
|
est := estimateRequestCostCents(request.Model, estimatePromptTokens(&request), request.MaxTokens)
|
|
var ok bool
|
|
if hold, ok = reserveBudget(subject, est); !ok {
|
|
c.ResponseAuthError(billingError("%s", object.InsufficientBalance(authUser.Owner, "request cost").Message))
|
|
return
|
|
}
|
|
}
|
|
defer hold.settle(0)
|
|
|
|
// ── Model families (Zen, Enso) ─────────────────────
|
|
// A family model is served by its family service, which owns identity, reasoning,
|
|
// the 1M ladder, vision, the fan-out, and the upstream. ai forwards verbatim and
|
|
// meters the result; it holds no family routing of its own (hip-00NN).
|
|
if fam := familyForProviderType(provider.Type); fam != nil {
|
|
c.pipeToFamily(fam, "chat/completions", "openai", request.Model, c.Ctx.Input.RequestBody, request.Stream, orgId, authUser, isPremium, hold, requestStartTime)
|
|
return
|
|
}
|
|
|
|
// ── Tool-calling pass-through ──────────────────────────────────────
|
|
// When the request includes tools/functions, the QueryText pipeline
|
|
// cannot handle structured tool calls. Proxy the raw request directly
|
|
// to the upstream provider's OpenAI-compatible endpoint so the LLM
|
|
// receives tool definitions and can return tool_calls in the response.
|
|
if len(request.Tools) > 0 || request.ToolChoice != nil {
|
|
c.proxyToolRequest(provider, &request, requestStartTime, authUser, isPremium, orgId, hold)
|
|
return
|
|
}
|
|
|
|
// Multimodal (vision): the QueryText pipeline below is text-only and would drop
|
|
// image parts. Forward multimodal requests verbatim to the upstream (the same path
|
|
// tool-calls take), so vision-capable models actually receive the images.
|
|
if requestHasMedia(&request) {
|
|
c.proxyToolRequest(provider, &request, requestStartTime, authUser, isPremium, orgId, hold)
|
|
return
|
|
}
|
|
|
|
// Extract messages content
|
|
var question string
|
|
var systemPrompt string
|
|
history := []*model.RawMessage{}
|
|
|
|
for _, msg := range request.Messages {
|
|
// Extract text from Content or MultiContent (array-style content parts)
|
|
text := msg.Content
|
|
if text == "" && len(msg.MultiContent) > 0 {
|
|
var parts []string
|
|
for _, part := range msg.MultiContent {
|
|
if part.Type == openai.ChatMessagePartTypeText && part.Text != "" {
|
|
parts = append(parts, part.Text)
|
|
}
|
|
}
|
|
text = strings.Join(parts, "\n")
|
|
}
|
|
switch msg.Role {
|
|
case "system":
|
|
systemPrompt = text
|
|
case "user":
|
|
question = text
|
|
case "assistant":
|
|
history = append(history, &model.RawMessage{
|
|
Author: "AI",
|
|
Text: text,
|
|
})
|
|
}
|
|
}
|
|
|
|
if question == "" {
|
|
c.ResponseError(c.T("openai:No user message found in the request"))
|
|
return
|
|
}
|
|
|
|
// Combine system prompt with user question if available
|
|
if systemPrompt != "" {
|
|
question = fmt.Sprintf("System: %s\n\nUser: %s", systemPrompt, question)
|
|
}
|
|
|
|
// Setup for streaming if enabled
|
|
if request.Stream {
|
|
c.Ctx.ResponseWriter.Header().Set("Content-Type", "text/event-stream")
|
|
c.Ctx.ResponseWriter.Header().Set("Cache-Control", "no-cache")
|
|
c.Ctx.ResponseWriter.Header().Set("Connection", "keep-alive")
|
|
}
|
|
|
|
// Create custom writer for OpenAI format
|
|
writer := &OpenAIWriter{
|
|
Response: *c.Ctx.ResponseWriter,
|
|
Buffer: []byte{},
|
|
RequestID: requestId,
|
|
Stream: request.Stream,
|
|
Cleaner: *NewCleaner(6),
|
|
Model: request.Model,
|
|
IncludeUsage: request.StreamOptions != nil && request.StreamOptions.IncludeUsage,
|
|
}
|
|
|
|
// Optional RAG: unified retrieval path shared with the old /chat-docs route.
|
|
// Enabled when any of the following is true:
|
|
// - Request header `X-Retrieval: 1` or body field `retrieval=true`
|
|
// - Header `X-Retrieval-Store` specifies a store
|
|
// - Auth is a widget key AND WIDGET_RETRIEVAL=1 (auto-RAG for public widgets)
|
|
knowledge := c.retrieveKnowledgeIfEnabled(
|
|
question,
|
|
retrievalOwner(authUser, token),
|
|
c.Ctx.Request.Header.Get("X-Retrieval-Store"),
|
|
c.GetAcceptLanguage(),
|
|
)
|
|
|
|
// Resolve the route for failover (may have fallback providers), sized to the
|
|
// prompt: if this prompt is bigger than the provider we would normally use
|
|
// can hold, we route it to one that can rather than refusing it. See
|
|
// routeForPrompt — a too-large prompt is a fact about the PROVIDER, not the
|
|
// model, and the fallback chain already knows who else serves it.
|
|
promptTokens, _ := model.GetTokenSize(request.Model, question)
|
|
route := routeForPrompt(request.Model, orgId, promptTokens)
|
|
|
|
// Call the model provider with failover support
|
|
var modelResult *model.ModelResult
|
|
var actualProvider string
|
|
|
|
if route != nil && len(route.fallbacks) > 0 {
|
|
modelResult, actualProvider, err = failoverQueryText(
|
|
route, question, writer, history, knowledge,
|
|
c.GetAcceptLanguage(),
|
|
func() bool { return writer.StreamSent },
|
|
)
|
|
} else {
|
|
// No fallbacks configured — direct call (original path)
|
|
var modelProvider model.ModelProvider
|
|
modelProvider, err = provider.GetModelProvider(c.GetAcceptLanguage())
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to get model provider: %s", err.Error()))
|
|
return
|
|
}
|
|
modelResult, err = modelProvider.QueryText(question, writer, history, "", knowledge, nil, c.GetAcceptLanguage())
|
|
actualProvider = provider.Name
|
|
}
|
|
|
|
if err != nil {
|
|
// Record failed usage
|
|
if authUser != nil {
|
|
errRecord := &usageRecord{
|
|
Owner: authUser.Owner,
|
|
User: authUser.Owner + "/" + authUser.Name,
|
|
Model: request.Model,
|
|
Provider: actualProvider,
|
|
Premium: isPremium,
|
|
Stream: request.Stream,
|
|
Status: "error",
|
|
ErrorMsg: err.Error(),
|
|
ClientIP: c.Ctx.Request.RemoteAddr,
|
|
RequestID: requestId,
|
|
}
|
|
errRecord.BYO, errRecord.Account = providerBYO(provider, authUser)
|
|
recordUsage(errRecord)
|
|
recordTrace(c.Ctx.Request.Context(), errRecord, requestStartTime)
|
|
}
|
|
c.ResponseError(err.Error())
|
|
return
|
|
}
|
|
|
|
// Record successful usage (actualProvider reflects which provider served the request)
|
|
if authUser != nil {
|
|
successRecord := &usageRecord{
|
|
Owner: authUser.Owner,
|
|
User: authUser.Owner + "/" + authUser.Name,
|
|
Organization: authUser.Owner,
|
|
Model: request.Model,
|
|
Provider: actualProvider,
|
|
PromptTokens: modelResult.PromptTokenCount,
|
|
CompletionTokens: modelResult.ResponseTokenCount,
|
|
TotalTokens: modelResult.TotalTokenCount,
|
|
Currency: "USD",
|
|
Premium: isPremium,
|
|
Stream: request.Stream,
|
|
Status: "success",
|
|
ClientIP: c.Ctx.Request.RemoteAddr,
|
|
RequestID: requestId,
|
|
}
|
|
successRecord.BYO, successRecord.Account = providerBYO(provider, authUser)
|
|
recordUsage(successRecord)
|
|
recordTrace(c.Ctx.Request.Context(), successRecord, requestStartTime)
|
|
// Settle the reservation with the ACTUAL cost (this works identically for
|
|
// streaming and non-streaming non-tool responses — both have real token
|
|
// counts here from the QueryText pipeline).
|
|
hold.settle(calculateCostCentsWithCache(request.Model, modelResult.PromptTokenCount, modelResult.ResponseTokenCount, 0, 0))
|
|
}
|
|
|
|
// Handle response based on streaming mode
|
|
if !request.Stream {
|
|
answer := writer.MessageString()
|
|
|
|
response := openai.ChatCompletionResponse{
|
|
ID: "chatcmpl-" + requestId,
|
|
Object: "chat.completion",
|
|
Created: util.GetCurrentUnixTime(),
|
|
Model: request.Model,
|
|
Choices: []openai.ChatCompletionChoice{
|
|
{
|
|
Index: 0,
|
|
Message: openai.ChatCompletionMessage{
|
|
Role: "assistant",
|
|
Content: answer,
|
|
},
|
|
FinishReason: openai.FinishReasonStop,
|
|
},
|
|
},
|
|
Usage: openai.Usage{
|
|
PromptTokens: modelResult.PromptTokenCount,
|
|
CompletionTokens: modelResult.ResponseTokenCount,
|
|
TotalTokens: modelResult.TotalTokenCount,
|
|
},
|
|
}
|
|
|
|
jsonResponse, err := json.Marshal(response)
|
|
if err != nil {
|
|
c.ResponseError(err.Error())
|
|
return
|
|
}
|
|
|
|
c.Ctx.Output.Header("Content-Type", "application/json")
|
|
c.Ctx.Output.Body(jsonResponse)
|
|
} else {
|
|
err = writer.Close(
|
|
modelResult.PromptTokenCount,
|
|
modelResult.ResponseTokenCount,
|
|
modelResult.TotalTokenCount,
|
|
)
|
|
if err != nil {
|
|
c.ResponseError(err.Error())
|
|
return
|
|
}
|
|
}
|
|
// LLM-as-a-judge: score THIS served (prompt, response) into a dense quality
|
|
// reward for the enso router. Placed AFTER the reply is fully written (both
|
|
// stream and non-stream), so it never adds latency; the hook only spawns a
|
|
// goroutine after cheap gates, and is a no-op unless ROUTER_JUDGE_ENABLED. The
|
|
// prompt/response are passed transiently — never persisted (see router_judge.go).
|
|
if routingRecorded {
|
|
judgeRoutedResponse(c.Ctx.Request.UserAgent(), orgId, requestId, c.Ctx.Request.Header.Get("CF-IPCountry"), request.Model, routedTask, question, writer.MessageString())
|
|
}
|
|
c.EnableRender = false
|
|
}
|
|
|
|
// ListModels returns the list of available models from the routing table.
|
|
// Requires a valid Bearer token (JWT, hk-, pk-, sk-, or hz_ key).
|
|
// @Title ListModels
|
|
// @Tag OpenAI Compatible API
|
|
// @Description Returns a list of all available models. Requires authentication.
|
|
// @Param Authorization header string true "Bearer token"
|
|
// @Success 200 {object} object
|
|
// @Failure 401 {object} object "Unauthorized"
|
|
// @router /models [get]
|
|
func (c *ApiController) ListModels() {
|
|
// R-04 fix: require authentication for model listing.
|
|
// Accept any valid token type (JWT, IAM key, publishable key, widget key).
|
|
authHeader := c.Ctx.Request.Header.Get("Authorization")
|
|
token := ""
|
|
if strings.HasPrefix(authHeader, "Bearer ") {
|
|
token = strings.TrimPrefix(authHeader, "Bearer ")
|
|
}
|
|
hasSession := c.GetSessionUsername() != ""
|
|
if token == "" && !hasSession {
|
|
c.Ctx.Output.Header("Content-Type", "application/json")
|
|
c.Ctx.ResponseWriter.WriteHeader(401)
|
|
c.Ctx.Output.Body([]byte(`{"error":{"message":"Authentication required. Provide a Bearer token.","type":"authentication_error","code":"unauthorized"}}`))
|
|
c.EnableRender = false
|
|
return
|
|
}
|
|
|
|
// R-RED-03: Validate token format — reject obviously invalid bearer values.
|
|
// Accepted prefixes: hk- (IAM key), sk- (secret key), pk- (publishable key),
|
|
// hz_ (Hanzo token). JWTs must have 3 base64url-encoded parts.
|
|
if token != "" {
|
|
isKnownPrefix := strings.HasPrefix(token, "hk-") ||
|
|
strings.HasPrefix(token, "sk-") ||
|
|
strings.HasPrefix(token, "pk-") ||
|
|
strings.HasPrefix(token, "hz_")
|
|
isValidJWT := false
|
|
if !isKnownPrefix {
|
|
// JWT must have exactly 3 dot-separated parts, each valid base64url
|
|
parts := strings.Split(token, ".")
|
|
if len(parts) == 3 {
|
|
isValidJWT = true
|
|
for _, part := range parts {
|
|
if len(part) == 0 {
|
|
isValidJWT = false
|
|
break
|
|
}
|
|
if _, err := base64.RawURLEncoding.DecodeString(part); err != nil {
|
|
isValidJWT = false
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if !isKnownPrefix && !isValidJWT {
|
|
c.Ctx.Output.Header("Content-Type", "application/json")
|
|
c.Ctx.ResponseWriter.WriteHeader(401)
|
|
c.Ctx.Output.Body([]byte(`{"error":{"message":"Invalid token format.","type":"authentication_error","code":"unauthorized"}}`))
|
|
c.EnableRender = false
|
|
return
|
|
}
|
|
}
|
|
|
|
models := listAvailableModels()
|
|
|
|
// Annotate gated (limited-preview) SKUs with the authenticated caller's access
|
|
// standing (waitlist|requested|granted), so the client can show "request access"
|
|
// vs "granted" without a second call.
|
|
annotateModelAccess(models, c.principalUser())
|
|
|
|
response := modelListEnvelope(models)
|
|
|
|
jsonResponse, err := json.Marshal(response)
|
|
if err != nil {
|
|
c.ResponseError(err.Error())
|
|
return
|
|
}
|
|
|
|
c.Ctx.Output.Header("Content-Type", "application/json")
|
|
c.Ctx.Output.Body(jsonResponse)
|
|
c.EnableRender = false
|
|
}
|
|
|
|
// proxyToolRequest forwards an OpenAI chat completion request that contains
|
|
// tool definitions directly to the upstream provider, bypassing the QueryText
|
|
// pipeline which cannot handle structured tool calls. The raw upstream response
|
|
// (including tool_calls) is streamed back to the client.
|
|
func (c *ApiController) proxyToolRequest(
|
|
provider *object.Provider,
|
|
request *openai.ChatCompletionRequest,
|
|
requestStartTime time.Time,
|
|
authUser *iam.User,
|
|
isPremium bool,
|
|
orgId string,
|
|
hold *budgetHold,
|
|
) {
|
|
requestId := util.GenerateUUID()
|
|
|
|
// Rewrite model to upstream model name
|
|
request.Model = provider.SubType
|
|
|
|
// For Claude/Anthropic providers, convert to Anthropic Messages API format
|
|
if provider.Type == "Claude" {
|
|
c.proxyToolRequestAnthropic(provider, request, requestStartTime, authUser, isPremium, orgId, requestId, hold)
|
|
return
|
|
}
|
|
|
|
// On the streaming path FORCE the upstream to emit a final usage chunk
|
|
// (stream_options.include_usage) so streamed tool calls are billed for their
|
|
// real token counts. Without this the streamed response carried no usage and
|
|
// was debited as $0 — any funded key + a dummy tool + stream:true = free
|
|
// premium inference. Remember whether the CLIENT requested usage so the
|
|
// injected usage-only chunk can be suppressed if it did not.
|
|
clientWantsUsage := true
|
|
if request.Stream {
|
|
clientWantsUsage = request.StreamOptions != nil && request.StreamOptions.IncludeUsage
|
|
request.StreamOptions = &openai.StreamOptions{IncludeUsage: true}
|
|
}
|
|
|
|
// Determine upstream endpoint and auth
|
|
upstreamURL, apiKey, authHeader := resolveUpstreamEndpoint(provider)
|
|
if upstreamURL == "" {
|
|
c.ResponseError("No upstream endpoint configured for provider: " + provider.Name)
|
|
return
|
|
}
|
|
|
|
// Marshal the full request (tools included) for OpenAI-compatible providers
|
|
body, err := json.Marshal(request)
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to marshal request: %s", err.Error()))
|
|
return
|
|
}
|
|
|
|
// Build upstream HTTP request
|
|
req, err := http.NewRequest(http.MethodPost, upstreamURL, bytes.NewReader(body))
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to create upstream request: %s", err.Error()))
|
|
return
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
if authHeader != "" {
|
|
req.Header.Set("Authorization", authHeader)
|
|
} else if apiKey != "" {
|
|
req.Header.Set("Authorization", "Bearer "+apiKey)
|
|
}
|
|
|
|
client := &http.Client{Timeout: 120 * time.Second}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
if authUser != nil {
|
|
errRecord := &usageRecord{
|
|
Owner: authUser.Owner,
|
|
User: authUser.Owner + "/" + authUser.Name,
|
|
Model: request.Model,
|
|
Provider: provider.Name,
|
|
Premium: isPremium,
|
|
Stream: request.Stream,
|
|
Status: "error",
|
|
ErrorMsg: err.Error(),
|
|
ClientIP: c.Ctx.Request.RemoteAddr,
|
|
RequestID: requestId,
|
|
}
|
|
errRecord.BYO, errRecord.Account = providerBYO(provider, authUser)
|
|
recordUsage(errRecord)
|
|
recordTrace(c.Ctx.Request.Context(), errRecord, requestStartTime)
|
|
}
|
|
c.ResponseError(fmt.Sprintf("Upstream request failed: %s", err.Error()))
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Copy upstream response headers
|
|
for k, vals := range resp.Header {
|
|
for _, v := range vals {
|
|
c.Ctx.ResponseWriter.Header().Add(k, v)
|
|
}
|
|
}
|
|
|
|
if request.Stream {
|
|
// Stream: copy SSE events while capturing token usage for billing.
|
|
c.Ctx.ResponseWriter.Header().Set("Content-Type", "text/event-stream")
|
|
c.Ctx.ResponseWriter.Header().Set("Cache-Control", "no-cache")
|
|
c.Ctx.ResponseWriter.Header().Set("Connection", "keep-alive")
|
|
c.Ctx.ResponseWriter.WriteHeader(resp.StatusCode)
|
|
|
|
// Copy the SSE stream to the client while capturing token usage (and the
|
|
// output text for a tokenizer fallback). This is the billing-critical core
|
|
// of the streaming tool path — see streamCaptureUsage. A reasoning-inlining
|
|
// upstream (DeepSeek) also gets its leading <think></think> block stripped
|
|
// from the forwarded content; every other upstream streams unchanged.
|
|
var strip *model.ReasoningStripper
|
|
if model.InlinesReasoning(request.Model) {
|
|
strip = &model.ReasoningStripper{}
|
|
}
|
|
capPrompt, capCompletion, capTotal, completionText := streamCaptureUsage(
|
|
resp.Body, c.Ctx.ResponseWriter, c.Ctx.ResponseWriter.Flush,
|
|
clientWantsUsage, requestId, request.Model, strip,
|
|
)
|
|
|
|
// Settle billing with the REAL token usage — captured from the forced
|
|
// usage chunk, or tokenized as a fallback so a successful streamed
|
|
// response is never billed as zero.
|
|
prompt, completion, total := capPrompt, capCompletion, capTotal
|
|
if completion == 0 && total == 0 {
|
|
if pt, err := model.OpenaiNumTokensFromMessages(request.Messages, request.Model); err == nil {
|
|
prompt = pt
|
|
}
|
|
completion, _ = model.GetTokenSize(request.Model, completionText)
|
|
}
|
|
if total == 0 {
|
|
total = prompt + completion
|
|
}
|
|
actualCents := calculateCostCentsWithCache(request.Model, prompt, completion, 0, 0)
|
|
if authUser != nil {
|
|
successRecord := &usageRecord{
|
|
Owner: authUser.Owner,
|
|
User: authUser.Owner + "/" + authUser.Name,
|
|
Organization: authUser.Owner,
|
|
Model: request.Model,
|
|
Provider: provider.Name,
|
|
PromptTokens: prompt,
|
|
CompletionTokens: completion,
|
|
TotalTokens: total,
|
|
Currency: "USD",
|
|
Premium: isPremium,
|
|
Stream: true,
|
|
Status: "success",
|
|
ClientIP: c.Ctx.Request.RemoteAddr,
|
|
RequestID: requestId,
|
|
}
|
|
successRecord.BYO, successRecord.Account = providerBYO(provider, authUser)
|
|
recordUsage(successRecord)
|
|
recordTrace(c.Ctx.Request.Context(), successRecord, requestStartTime)
|
|
}
|
|
hold.settle(actualCents)
|
|
} else {
|
|
// Non-streaming: read full response, extract token counts, forward
|
|
respBody, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to read upstream response: %s", err.Error()))
|
|
return
|
|
}
|
|
|
|
// Try to extract usage for billing
|
|
var upstreamResp struct {
|
|
Usage struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
} `json:"usage"`
|
|
}
|
|
_ = json.Unmarshal(respBody, &upstreamResp)
|
|
|
|
prompt := upstreamResp.Usage.PromptTokens
|
|
completion := upstreamResp.Usage.CompletionTokens
|
|
total := upstreamResp.Usage.TotalTokens
|
|
if completion == 0 && total == 0 {
|
|
// Upstream returned no usage — tokenize so a successful tool response
|
|
// is never billed as zero.
|
|
if pt, err := model.OpenaiNumTokensFromMessages(request.Messages, request.Model); err == nil {
|
|
prompt = pt
|
|
}
|
|
completion, _ = model.GetTokenSize(request.Model, string(respBody))
|
|
}
|
|
if total == 0 {
|
|
total = prompt + completion
|
|
}
|
|
actualCents := calculateCostCentsWithCache(request.Model, prompt, completion, 0, 0)
|
|
|
|
if authUser != nil {
|
|
successRecord := &usageRecord{
|
|
Owner: authUser.Owner,
|
|
User: authUser.Owner + "/" + authUser.Name,
|
|
Organization: authUser.Owner,
|
|
Model: request.Model,
|
|
Provider: provider.Name,
|
|
PromptTokens: prompt,
|
|
CompletionTokens: completion,
|
|
TotalTokens: total,
|
|
Currency: "USD",
|
|
Premium: isPremium,
|
|
Stream: false,
|
|
Status: "success",
|
|
ClientIP: c.Ctx.Request.RemoteAddr,
|
|
RequestID: requestId,
|
|
}
|
|
successRecord.BYO, successRecord.Account = providerBYO(provider, authUser)
|
|
recordUsage(successRecord)
|
|
recordTrace(c.Ctx.Request.Context(), successRecord, requestStartTime)
|
|
}
|
|
hold.settle(actualCents)
|
|
|
|
// Strip a reasoning-inlining upstream's leading <think></think> block from
|
|
// the forwarded body (billing above already tokenized the original).
|
|
if model.InlinesReasoning(request.Model) {
|
|
respBody = stripReasoningBody(respBody)
|
|
}
|
|
c.Ctx.ResponseWriter.WriteHeader(resp.StatusCode)
|
|
c.Ctx.Output.Body(respBody)
|
|
}
|
|
c.EnableRender = false
|
|
}
|
|
|
|
// resolveUpstreamEndpoint returns the chat completions URL, API key, and
|
|
// optional full Authorization header for the given provider. It is a thin
|
|
// alias over resolveEndpointForPath so chat, embeddings, and rerank all share
|
|
// exactly one per-provider endpoint map.
|
|
func resolveUpstreamEndpoint(provider *object.Provider) (url string, apiKey string, authHeader string) {
|
|
return resolveEndpointForPath(provider, "chat/completions")
|
|
}
|
|
|
|
// resolveEndpointForPath returns the upstream URL, API key, and optional full
|
|
// Authorization header for the given provider and OpenAI-style API path
|
|
// (e.g. "chat/completions", "embeddings", "rerank"). This is the single place
|
|
// that knows each provider's base URL and auth scheme; every OpenAI-compatible
|
|
// surface is built by varying apiPath only — no per-endpoint copy of provider
|
|
// routing exists.
|
|
func resolveEndpointForPath(provider *object.Provider, apiPath string) (url string, apiKey string, authHeader string) {
|
|
apiKey = provider.ClientSecret
|
|
|
|
switch provider.Type {
|
|
case "OpenAI":
|
|
baseURL := provider.ProviderUrl
|
|
if baseURL == "" {
|
|
baseURL = "https://api.openai.com/v1"
|
|
}
|
|
baseURL = strings.TrimRight(baseURL, "/")
|
|
if !strings.HasSuffix(baseURL, "/v1") {
|
|
baseURL += "/v1"
|
|
}
|
|
return baseURL + "/" + apiPath, apiKey, ""
|
|
|
|
case "Fireworks":
|
|
return "https://api.fireworks.ai/inference/v1/" + apiPath, apiKey, ""
|
|
|
|
case "Grok":
|
|
return "https://api.x.ai/v1/" + apiPath, apiKey, ""
|
|
|
|
case "OpenRouter":
|
|
return "https://openrouter.ai/api/v1/" + apiPath, apiKey, ""
|
|
|
|
case "Moonshot":
|
|
return "https://api.moonshot.cn/v1/" + apiPath, apiKey, ""
|
|
|
|
case "Gemini":
|
|
// Gemini exposes an OpenAI-compatible surface under /v1beta/openai.
|
|
return "https://generativelanguage.googleapis.com/v1beta/openai/" + apiPath, apiKey, ""
|
|
|
|
case "Jina":
|
|
// Jina AI: OpenAI-compatible /v1/embeddings and a native /v1/rerank.
|
|
return "https://api.jina.ai/v1/" + apiPath, apiKey, ""
|
|
|
|
case "Cohere":
|
|
// Cohere v1 exposes /v1/embeddings and /v1/rerank.
|
|
return "https://api.cohere.com/v1/" + apiPath, apiKey, ""
|
|
|
|
case "Azure":
|
|
baseURL := strings.TrimRight(provider.ProviderUrl, "/")
|
|
apiVersion := provider.ApiVersion
|
|
if apiVersion == "" {
|
|
apiVersion = "2024-02-01"
|
|
}
|
|
return fmt.Sprintf("%s/openai/deployments/%s/%s?api-version=%s",
|
|
baseURL, provider.SubType, apiPath, apiVersion), "", "api-key " + apiKey
|
|
|
|
case "Local", "Ollama", "DigitalOcean":
|
|
// Local/compatible providers with custom URLs.
|
|
baseURL := strings.TrimRight(provider.ProviderUrl, "/")
|
|
if baseURL == "" {
|
|
return "", "", ""
|
|
}
|
|
if strings.HasSuffix(baseURL, "/v1") {
|
|
return baseURL + "/" + apiPath, apiKey, ""
|
|
}
|
|
return baseURL + "/v1/" + apiPath, apiKey, ""
|
|
|
|
default:
|
|
// Any other OpenAI-compatible provider with a custom URL.
|
|
if provider.ProviderUrl != "" {
|
|
baseURL := strings.TrimRight(provider.ProviderUrl, "/")
|
|
return baseURL + "/" + apiPath, apiKey, ""
|
|
}
|
|
return "", "", ""
|
|
}
|
|
}
|
|
|
|
// proxyToolRequestAnthropic handles tool-calling requests for Claude/Anthropic
|
|
// providers by converting the OpenAI format to Anthropic Messages API format
|
|
// and converting the response back.
|
|
func (c *ApiController) proxyToolRequestAnthropic(
|
|
provider *object.Provider,
|
|
request *openai.ChatCompletionRequest,
|
|
requestStartTime time.Time,
|
|
authUser *iam.User,
|
|
isPremium bool,
|
|
orgId string,
|
|
requestId string,
|
|
hold *budgetHold,
|
|
) {
|
|
apiKey := provider.ClientSecret
|
|
baseURL := provider.ProviderUrl
|
|
if baseURL == "" {
|
|
baseURL = "https://api.anthropic.com"
|
|
}
|
|
baseURL = strings.TrimRight(baseURL, "/")
|
|
|
|
// Convert OpenAI messages to Anthropic format
|
|
var systemPrompt string
|
|
anthropicMessages := []map[string]interface{}{}
|
|
|
|
for _, msg := range request.Messages {
|
|
if msg.Role == "system" {
|
|
systemPrompt = msg.Content
|
|
continue
|
|
}
|
|
|
|
anthropicMsg := map[string]interface{}{
|
|
"role": msg.Role,
|
|
}
|
|
|
|
if msg.Role == "tool" {
|
|
// Tool result message
|
|
anthropicMsg["role"] = "user"
|
|
anthropicMsg["content"] = []map[string]interface{}{
|
|
{
|
|
"type": "tool_result",
|
|
"tool_use_id": msg.ToolCallID,
|
|
"content": msg.Content,
|
|
},
|
|
}
|
|
} else if len(msg.ToolCalls) > 0 {
|
|
// Assistant message with tool calls
|
|
content := []map[string]interface{}{}
|
|
if msg.Content != "" {
|
|
content = append(content, map[string]interface{}{
|
|
"type": "text",
|
|
"text": msg.Content,
|
|
})
|
|
}
|
|
for _, tc := range msg.ToolCalls {
|
|
var inputObj interface{}
|
|
_ = json.Unmarshal([]byte(tc.Function.Arguments), &inputObj)
|
|
if inputObj == nil {
|
|
inputObj = map[string]interface{}{}
|
|
}
|
|
content = append(content, map[string]interface{}{
|
|
"type": "tool_use",
|
|
"id": tc.ID,
|
|
"name": tc.Function.Name,
|
|
"input": inputObj,
|
|
})
|
|
}
|
|
anthropicMsg["content"] = content
|
|
} else if len(msg.MultiContent) > 0 {
|
|
content := []map[string]interface{}{}
|
|
for _, part := range msg.MultiContent {
|
|
if part.Type == openai.ChatMessagePartTypeText {
|
|
content = append(content, map[string]interface{}{
|
|
"type": "text",
|
|
"text": part.Text,
|
|
})
|
|
}
|
|
}
|
|
anthropicMsg["content"] = content
|
|
} else {
|
|
anthropicMsg["content"] = msg.Content
|
|
}
|
|
|
|
anthropicMessages = append(anthropicMessages, anthropicMsg)
|
|
}
|
|
|
|
// Convert OpenAI tools to Anthropic tool format
|
|
anthropicTools := []map[string]interface{}{}
|
|
for _, tool := range request.Tools {
|
|
if tool.Type == openai.ToolTypeFunction {
|
|
anthropicTool := map[string]interface{}{
|
|
"name": tool.Function.Name,
|
|
"description": tool.Function.Description,
|
|
}
|
|
if tool.Function.Parameters != nil {
|
|
var params interface{}
|
|
raw, _ := json.Marshal(tool.Function.Parameters)
|
|
_ = json.Unmarshal(raw, ¶ms)
|
|
anthropicTool["input_schema"] = params
|
|
} else {
|
|
anthropicTool["input_schema"] = map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{},
|
|
}
|
|
}
|
|
anthropicTools = append(anthropicTools, anthropicTool)
|
|
}
|
|
}
|
|
|
|
// Build Anthropic request
|
|
anthropicReq := map[string]interface{}{
|
|
"model": request.Model,
|
|
"messages": anthropicMessages,
|
|
"max_tokens": 4096,
|
|
"tools": anthropicTools,
|
|
}
|
|
if systemPrompt != "" {
|
|
anthropicReq["system"] = systemPrompt
|
|
}
|
|
if request.MaxTokens > 0 {
|
|
anthropicReq["max_tokens"] = request.MaxTokens
|
|
}
|
|
if request.Temperature > 0 {
|
|
anthropicReq["temperature"] = request.Temperature
|
|
}
|
|
// Always fetch the FULL (non-streamed) Anthropic response so the body is
|
|
// parseable JSON carrying token usage for billing. If the client requested
|
|
// streaming, the converted result is re-emitted as SSE below. Previously
|
|
// stream=true made io.ReadAll+json.Unmarshal fail on the SSE body, so streamed
|
|
// tool calls errored AND were never billed.
|
|
|
|
body, err := json.Marshal(anthropicReq)
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to marshal Anthropic request: %s", err.Error()))
|
|
return
|
|
}
|
|
|
|
req, err := http.NewRequest(http.MethodPost, baseURL+"/v1/messages", bytes.NewReader(body))
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to create Anthropic request: %s", err.Error()))
|
|
return
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("x-api-key", apiKey)
|
|
req.Header.Set("anthropic-version", "2023-06-01")
|
|
|
|
client := &http.Client{Timeout: 120 * time.Second}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Anthropic request failed: %s", err.Error()))
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Read full Anthropic response
|
|
respBody, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to read Anthropic response: %s", err.Error()))
|
|
return
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
log.Error("[proxyToolRequest] Anthropic error %d: %s", resp.StatusCode, string(respBody))
|
|
c.Ctx.ResponseWriter.WriteHeader(resp.StatusCode)
|
|
c.Ctx.Output.Body(respBody)
|
|
c.EnableRender = false
|
|
return
|
|
}
|
|
|
|
// Parse Anthropic response
|
|
var anthropicResp struct {
|
|
ID string `json:"id"`
|
|
Content []struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text,omitempty"`
|
|
ID string `json:"id,omitempty"`
|
|
Name string `json:"name,omitempty"`
|
|
Input json.RawMessage `json:"input,omitempty"`
|
|
} `json:"content"`
|
|
StopReason string `json:"stop_reason"`
|
|
Usage struct {
|
|
InputTokens int `json:"input_tokens"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
} `json:"usage"`
|
|
}
|
|
if err := json.Unmarshal(respBody, &anthropicResp); err != nil {
|
|
c.ResponseError(fmt.Sprintf("Failed to parse Anthropic response: %s", err.Error()))
|
|
return
|
|
}
|
|
|
|
// Convert Anthropic response to OpenAI format
|
|
var contentText string
|
|
var toolCalls []openai.ToolCall
|
|
toolCallIdx := 0
|
|
|
|
for _, block := range anthropicResp.Content {
|
|
switch block.Type {
|
|
case "text":
|
|
contentText += block.Text
|
|
case "tool_use":
|
|
tc := openai.ToolCall{
|
|
Index: &toolCallIdx,
|
|
ID: block.ID,
|
|
Type: openai.ToolTypeFunction,
|
|
Function: openai.FunctionCall{
|
|
Name: block.Name,
|
|
Arguments: string(block.Input),
|
|
},
|
|
}
|
|
toolCalls = append(toolCalls, tc)
|
|
toolCallIdx++
|
|
}
|
|
}
|
|
|
|
finishReason := openai.FinishReasonStop
|
|
if anthropicResp.StopReason == "tool_use" {
|
|
finishReason = openai.FinishReasonToolCalls
|
|
}
|
|
|
|
openaiResp := openai.ChatCompletionResponse{
|
|
ID: "chatcmpl-" + requestId,
|
|
Object: "chat.completion",
|
|
Created: util.GetCurrentUnixTime(),
|
|
Model: request.Model,
|
|
Choices: []openai.ChatCompletionChoice{
|
|
{
|
|
Index: 0,
|
|
Message: openai.ChatCompletionMessage{
|
|
Role: "assistant",
|
|
Content: contentText,
|
|
ToolCalls: toolCalls,
|
|
},
|
|
FinishReason: finishReason,
|
|
},
|
|
},
|
|
Usage: openai.Usage{
|
|
PromptTokens: anthropicResp.Usage.InputTokens,
|
|
CompletionTokens: anthropicResp.Usage.OutputTokens,
|
|
TotalTokens: anthropicResp.Usage.InputTokens + anthropicResp.Usage.OutputTokens,
|
|
},
|
|
}
|
|
|
|
// Record usage
|
|
if authUser != nil {
|
|
successRecord := &usageRecord{
|
|
Owner: authUser.Owner,
|
|
User: authUser.Owner + "/" + authUser.Name,
|
|
Organization: authUser.Owner,
|
|
Model: request.Model,
|
|
Provider: provider.Name,
|
|
PromptTokens: anthropicResp.Usage.InputTokens,
|
|
CompletionTokens: anthropicResp.Usage.OutputTokens,
|
|
TotalTokens: anthropicResp.Usage.InputTokens + anthropicResp.Usage.OutputTokens,
|
|
Currency: "USD",
|
|
Premium: isPremium,
|
|
Stream: false,
|
|
Status: "success",
|
|
ClientIP: c.Ctx.Request.RemoteAddr,
|
|
RequestID: requestId,
|
|
}
|
|
successRecord.BYO, successRecord.Account = providerBYO(provider, authUser)
|
|
recordUsage(successRecord)
|
|
recordTrace(c.Ctx.Request.Context(), successRecord, requestStartTime)
|
|
}
|
|
hold.settle(calculateCostCentsWithCache(request.Model, anthropicResp.Usage.InputTokens, anthropicResp.Usage.OutputTokens, 0, 0))
|
|
|
|
jsonResponse, err := json.Marshal(openaiResp)
|
|
if err != nil {
|
|
c.ResponseError(err.Error())
|
|
return
|
|
}
|
|
|
|
if request.Stream {
|
|
// Client asked for streaming: emit the converted completion as a single
|
|
// SSE chunk followed by [DONE], so OpenAI SDK clients consuming a stream
|
|
// still work while billing used the real (full-response) token usage.
|
|
c.Ctx.ResponseWriter.Header().Set("Content-Type", "text/event-stream")
|
|
c.Ctx.ResponseWriter.Header().Set("Cache-Control", "no-cache")
|
|
c.Ctx.ResponseWriter.Header().Set("Connection", "keep-alive")
|
|
chunk := map[string]interface{}{
|
|
"id": openaiResp.ID,
|
|
"object": "chat.completion.chunk",
|
|
"created": openaiResp.Created,
|
|
"model": openaiResp.Model,
|
|
"choices": []map[string]interface{}{{
|
|
"index": 0,
|
|
"delta": map[string]interface{}{
|
|
"role": "assistant",
|
|
"content": contentText,
|
|
"tool_calls": toolCalls,
|
|
},
|
|
"finish_reason": finishReason,
|
|
}},
|
|
}
|
|
if chunkJSON, mErr := json.Marshal(chunk); mErr == nil {
|
|
_, _ = fmt.Fprintf(c.Ctx.ResponseWriter, "data: %s\n\n", string(chunkJSON))
|
|
}
|
|
_, _ = fmt.Fprint(c.Ctx.ResponseWriter, "data: [DONE]\n\n")
|
|
c.Ctx.ResponseWriter.Flush()
|
|
c.EnableRender = false
|
|
return
|
|
}
|
|
|
|
c.Ctx.Output.Header("Content-Type", "application/json")
|
|
c.Ctx.Output.Body(jsonResponse)
|
|
c.EnableRender = false
|
|
}
|