mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
prompt_tokens meant two things. The family and relay paths recorded the part of the prompt read fresh, the table path and the providers the whole prompt, so a cached enso-auto row read cache_read > prompt. No path recorded a cache write: Anthropic's cache_creation_input_tokens was folded into the fresh read and billed at the input rate, though a write is priced above it, and OpenRouter's cache_write_tokens was not read at all. cache_write_tokens was 0 on every row. A record's prompt is now every input token, with the cache reads and writes parts of it, whatever shape the upstream reported: OpenAI's cached_tokens inside prompt_tokens, OpenRouter's cache_write_tokens beside them, Anthropic's two counts beside input_tokens. Pricing bills the input rate on the part read fresh and each cache part at its own rate: a family SKU's cache_read and cache_write (a resold model's input_cache_read and input_cache_write plus the vendor's fee), else the input rate. The Anthropic provider and the OpenAI adapter over Anthropic now read both cache counts, and the providers' settles pass them.
718 lines
27 KiB
Go
718 lines
27 KiB
Go
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
// forward.go — a chat completion relayed as the caller wrote it.
|
|
//
|
|
// The text pipeline this replaces for OpenAI-compatible routes rebuilt every request
|
|
// from two of its parts: the last system message and the last user message. Earlier
|
|
// user turns were dropped without a word, assistant turns became a history the
|
|
// pipeline trimmed to fit, and every sampling parameter the caller set — temperature
|
|
// 0, a seed, a response format, stop sequences — gave way to the provider row's
|
|
// defaults, a default system prompt and a fixed token ceiling. What reached the model
|
|
// was a different conversation from the one the caller sent.
|
|
//
|
|
// A tool call or an image took a second path, which marshalled the request back out
|
|
// of go-openai's struct: `omitempty` dropped temperature 0 and top_p 0, every field
|
|
// the struct does not name was lost, and a refusal ended the request at the one
|
|
// provider it was sent to.
|
|
//
|
|
// So a request whose route speaks OpenAI's dialect is sent the caller's own body. Four
|
|
// things in it are ours to write, and only these: the model (each provider's own id
|
|
// for the SKU), the completion ceiling where the reservation holds less than the
|
|
// caller asked for, include_usage on a stream so the answer can be billed, and — when
|
|
// the caller asked for retrieval — the retrieved knowledge as a leading system
|
|
// message. This API's own fields are not sent on (ours), and a request naming a field
|
|
// that would buy something on our account the SKU's price does not cover is refused
|
|
// before it gets here (unpriced). Everything else arrives upstream as it left the caller.
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"maps"
|
|
"net/http"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/hanzoai/ai/model"
|
|
"github.com/hanzoai/ai/object"
|
|
"github.com/hanzoai/ai/upstream"
|
|
openai "github.com/hanzoai/go-openai"
|
|
)
|
|
|
|
// chatRequest is a chat request as the handler reads it. Two fields that go-openai
|
|
// types more narrowly than the API does are held raw: a response_format carrying a
|
|
// json_schema (its schema is a json.Marshaler, which nothing can be decoded into) and
|
|
// a stop given as one string. Typed, either refused a request every vendor accepts
|
|
// with a 400 before it was routed. The handler reads neither; the relay sends the
|
|
// caller's own bytes for both.
|
|
type chatRequest struct {
|
|
openai.ChatCompletionRequest
|
|
ResponseFormat json.RawMessage `json:"response_format,omitempty"`
|
|
Stop json.RawMessage `json:"stop,omitempty"`
|
|
}
|
|
|
|
// ours are this API's own request fields, not any vendor's: fast mode (fast.go) and
|
|
// retrieval (chat_retrieval.go). This handler has read them by the time the body
|
|
// leaves, and a vendor that checks its parameters — OpenAI does — refuses a name it
|
|
// does not define, so they are not sent on.
|
|
var ours = []string{"fast", "retrieval", "retrieval_store"}
|
|
|
|
// unpriced are the fields that change what a vendor charges US without changing what
|
|
// the SKU is sold for: another model (models), another endpoint (provider, transforms,
|
|
// route), a paid tool (plugins, web_search_options), a dearer queue (service_tier).
|
|
// Sent on, each would be bought on our account at a price nobody quoted the caller.
|
|
//
|
|
// A request naming one is REFUSED, on every path, before any vendor is asked. It is
|
|
// never quietly stripped: a caller who asked for another model's fallback or a web
|
|
// search and got an answer without it would read that answer as what they asked for.
|
|
var unpriced = []string{"models", "provider", "transforms", "route", "plugins", "web_search_options", "service_tier"}
|
|
|
|
// unpricedField names the first unpriced field raw carries, or "" when it carries none.
|
|
// It runs after casefolded has bounded the body's keys, and folds each key once.
|
|
func unpricedField(raw []byte) string {
|
|
var fields map[string]json.RawMessage
|
|
if json.Unmarshal(raw, &fields) != nil {
|
|
return ""
|
|
}
|
|
for k := range fields {
|
|
if unpricedFolds[fold(k)] {
|
|
return k
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// unpricedFolds is unpriced, folded once.
|
|
var unpricedFolds = func() map[string]bool {
|
|
m := make(map[string]bool, len(unpriced))
|
|
for _, u := range unpriced {
|
|
m[fold(u)] = true
|
|
}
|
|
return m
|
|
}()
|
|
|
|
// pass is one chat completion on its way through the relay: what the caller sent,
|
|
// what was decided for it, and who pays.
|
|
type pass struct {
|
|
// req is the request as this handler decided it — the SKU after auto-routing and
|
|
// the public lane, and the completion ceiling the hold was reserved for. body is
|
|
// the request as the caller WROTE it, and is what goes upstream.
|
|
req *openai.ChatCompletionRequest
|
|
body []byte
|
|
|
|
// primary is the row auth resolved for the route's own provider. route is nil for
|
|
// a model the route table does not carry, which primary serves alone.
|
|
primary *object.Provider
|
|
route *modelRoute
|
|
// prior are providers that already refused this request — the family pipe. They
|
|
// are not asked again, and their refusals are already recorded.
|
|
prior []attempt
|
|
|
|
knowledge []*model.RawMessage
|
|
// prompt is the prompt as the reservation measured it, over every message. It is
|
|
// read only when an upstream answers without saying what it used.
|
|
prompt int
|
|
|
|
user *iam.User
|
|
premium bool
|
|
hold *budgetHold
|
|
id string
|
|
start time.Time
|
|
// judge scores a finished answer for the router, or is nil for an answer it does
|
|
// not score. It is called from inside a stream writer, so it closes over values.
|
|
judge func(answer string)
|
|
// strict asks for the request to reach the route's first row exactly as sent, or be
|
|
// refused (strict.go).
|
|
strict bool
|
|
}
|
|
|
|
// relays reports whether a provider row can be sent a chat body as the caller wrote
|
|
// it: it has an OpenAI-compatible chat address, it does not speak Anthropic's own
|
|
// dialect, which the relay does not translate, and it is not a family's.
|
|
//
|
|
// A family's row is left to the family pipe. The pipe keeps back the fields that
|
|
// would buy another model or tool on our account, states the data-collection terms a
|
|
// paid SKU is sold under, and names and prices a free stand-in as what it is; a raw
|
|
// body sent to the same row directly would do none of that, and a tool or image
|
|
// request would land on the free floor answering under the SKU's name. So a route's
|
|
// OpenRouter tail and free pool are passed over here, not reached around the pipe.
|
|
func relays(p *object.Provider) bool {
|
|
return familyForProviderType(p.Type) == nil &&
|
|
model.Upstream(p.Type) != model.Anthropic && upstream.Endpoint(p, "chat/completions") != ""
|
|
}
|
|
|
|
// errUnrelayable is a candidate whose row the relay cannot send an OpenAI chat body
|
|
// to. It is not a refusal — nothing was asked — so it is passed over rather than
|
|
// recorded, and a route whose alternates are all of this kind ends at exhausted().
|
|
var errUnrelayable = errors.New("provider speaks no OpenAI chat dialect")
|
|
|
|
// dial sends one attempt to one provider: the row the candidate names, resolved for
|
|
// the org the way callProvider resolves it — so a customer that connected its own
|
|
// key spends its own key, on a fallback exactly as on the primary — and the body
|
|
// built for that row.
|
|
//
|
|
// The client is zenPipeClient's: a bound on the time to the response HEADERS and none
|
|
// on the body, because a streamed answer is long by design and a whole-request
|
|
// timeout cuts it off mid-sentence.
|
|
//
|
|
// A var so the failover loop can be driven against scripted providers, the way
|
|
// callProvider is.
|
|
var dial = func(ctx context.Context, org string, row *object.Provider, c candidate, body func(*object.Provider) []byte) (*http.Response, *object.Provider, error) {
|
|
if row == nil {
|
|
var err error
|
|
if row, err = object.GetModelProviderByNameForOrg(org, c.provider); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if row == nil {
|
|
return nil, nil, unavailable(c.provider)
|
|
}
|
|
}
|
|
p := *row // copy: SubType is per request, and Azure addresses its deployment by it
|
|
p.SubType = c.upstream
|
|
if !relays(&p) {
|
|
return nil, &p, errUnrelayable
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, upstream.Endpoint(&p, "chat/completions"), bytes.NewReader(body(&p)))
|
|
if err != nil {
|
|
return nil, &p, err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
upstream.Authorize(req, &p)
|
|
resp, err := zenPipeClient.Do(req)
|
|
return resp, &p, err
|
|
}
|
|
|
|
// draft is the caller's request as it will leave: their fields, with ours written
|
|
// over the few the relay decides. Each provider is sent it under its own id for the
|
|
// model (body).
|
|
type draft struct {
|
|
fields map[string]json.RawMessage
|
|
// ceiling is the completion ceiling the hold covers, written only when the caller
|
|
// named none — which key a vendor reads for it is a fact about the vendor.
|
|
ceiling int
|
|
}
|
|
|
|
// outbound reads the caller's body into the draft every attempt is built from.
|
|
func outbound(raw []byte, req *openai.ChatCompletionRequest, knowledge []*model.RawMessage) (draft, error) {
|
|
var fields map[string]json.RawMessage
|
|
if err := json.Unmarshal(raw, &fields); err != nil {
|
|
return draft{}, err
|
|
}
|
|
if fields == nil {
|
|
fields = map[string]json.RawMessage{}
|
|
}
|
|
for k := range fields {
|
|
for _, o := range ours {
|
|
if fold(k) == fold(o) {
|
|
delete(fields, k)
|
|
}
|
|
}
|
|
}
|
|
|
|
// THE CEILING IS LOWERED, NEVER RAISED, AND UNDER THE CALLER'S OWN KEY. The hold
|
|
// was reserved for req.MaxTokens completion tokens (clampMaxTokens, and the public
|
|
// lane's cap), so an upstream allowed more could settle past it. A caller who
|
|
// asked for less keeps what they asked for. A caller who named neither key gets
|
|
// the reserved ceiling under the key the vendor reads (body).
|
|
d := draft{fields: fields}
|
|
if limit := req.MaxTokens; limit > 0 {
|
|
named := false
|
|
for _, k := range []string{"max_tokens", "max_completion_tokens"} {
|
|
v, ok := fields[k]
|
|
if !ok {
|
|
continue
|
|
}
|
|
named = true
|
|
var n int
|
|
if json.Unmarshal(v, &n) != nil || n <= 0 || n > limit {
|
|
fields[k] = json.RawMessage(strconv.Itoa(limit))
|
|
}
|
|
}
|
|
if !named {
|
|
d.ceiling = limit
|
|
}
|
|
}
|
|
|
|
// A streamed answer is billed from the usage chunk the vendor sends only when it
|
|
// is asked to. Whether the CALLER sees that chunk is decided on the way out
|
|
// (streamCaptureUsage); every other stream option they set is theirs.
|
|
if req.Stream {
|
|
opts := map[string]json.RawMessage{}
|
|
if v, ok := fields["stream_options"]; ok {
|
|
_ = json.Unmarshal(v, &opts)
|
|
if opts == nil {
|
|
opts = map[string]json.RawMessage{}
|
|
}
|
|
}
|
|
opts["include_usage"] = json.RawMessage("true")
|
|
b, err := encode(opts)
|
|
if err != nil {
|
|
return draft{}, err
|
|
}
|
|
fields["stream_options"] = b
|
|
}
|
|
|
|
// Retrieval, when the caller asked for it, arrives as one system message ahead of
|
|
// their conversation — which is otherwise sent exactly as they wrote it.
|
|
if len(knowledge) > 0 {
|
|
var msgs []json.RawMessage
|
|
if raw, ok := fields["messages"]; ok {
|
|
if err := json.Unmarshal(raw, &msgs); err != nil {
|
|
return draft{}, err
|
|
}
|
|
}
|
|
parts := make([]string, 0, len(knowledge))
|
|
for i, k := range knowledge {
|
|
parts = append(parts, fmt.Sprintf("Knowledge %d: %s", i+1, k.Text))
|
|
}
|
|
sys, err := encode(map[string]string{"role": "system", "content": strings.Join(parts, "\n\n")})
|
|
if err != nil {
|
|
return draft{}, err
|
|
}
|
|
b, err := encode(append([]json.RawMessage{sys}, msgs...))
|
|
if err != nil {
|
|
return draft{}, err
|
|
}
|
|
fields["messages"] = b
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
// body is the draft as one provider is sent it: under that provider's own id for the
|
|
// model, and with the reserved ceiling under the key it reads when the caller named
|
|
// none. OpenAI refuses max_tokens on its reasoning models and reads
|
|
// max_completion_tokens on every chat model; the vendors that copy its dialect read
|
|
// max_tokens.
|
|
func (d draft) body(upstreamModel string, row *object.Provider) []byte {
|
|
out := make(map[string]json.RawMessage, len(d.fields)+2)
|
|
maps.Copy(out, d.fields)
|
|
out["model"] = text(upstreamModel)
|
|
if d.ceiling > 0 {
|
|
key := "max_tokens"
|
|
if model.Upstream(row.Type) == model.OpenAI {
|
|
key = "max_completion_tokens"
|
|
}
|
|
out[key] = json.RawMessage(strconv.Itoa(d.ceiling))
|
|
}
|
|
// Every value came out of json.Unmarshal or was written above, so this encodes.
|
|
b, _ := encode(out)
|
|
return b
|
|
}
|
|
|
|
// declined is an upstream's non-200 as the error the cascade reads. The status leads
|
|
// its text the way the provider SDK's own errors carry it, because that text is what
|
|
// retryTransient reads to decide a 429 or a 5xx is worth waiting out.
|
|
//
|
|
// Our own spend gate, relayed by a service that fronts for us, keeps its code and its
|
|
// words: that refusal is the caller's debt, and it reaches them as it was written.
|
|
func declined(status int, body []byte) error {
|
|
said := upstreamErrorMessage(body)
|
|
if billingNotice(body) {
|
|
return &apiError{status: status, msg: said, code: object.CodeInsufficientBalance}
|
|
}
|
|
return &apiError{status: status, msg: fmt.Sprintf("%d %s", status, said)}
|
|
}
|
|
|
|
// call offers the request to one provider, riding out a transient refusal as
|
|
// ask.call does, and hands back an answer only when it is one: a 200 whose stream has
|
|
// begun (opening), or a buffered body read to its end. Anything else comes back as
|
|
// the refusal it is, with nothing written — which is what keeps it movable.
|
|
func (p pass) call(ctx context.Context, org string, row *object.Provider, c candidate, d draft) (resp *http.Response, whole []byte, used *object.Provider, err error) {
|
|
body := func(u *object.Provider) []byte { return d.body(c.upstream, u) }
|
|
err = retryTransient(ctx, currentRetryPolicy(), func() error {
|
|
r, u, e := dial(ctx, org, row, c, body)
|
|
used = u
|
|
if e != nil {
|
|
return wrapUpstreamError(e)
|
|
}
|
|
if p.req.Stream {
|
|
r = opening(r)
|
|
}
|
|
if r.StatusCode != http.StatusOK {
|
|
b, _ := io.ReadAll(r.Body)
|
|
r.Body.Close()
|
|
return declined(r.StatusCode, b)
|
|
}
|
|
if !p.req.Stream {
|
|
b, e := io.ReadAll(r.Body)
|
|
r.Body.Close()
|
|
if e != nil {
|
|
// Cut off on the way to us. Nothing has reached the caller, so this is
|
|
// still movable.
|
|
return e
|
|
}
|
|
// A 200 whose body is an error and no answer is the vendor refusing, and is
|
|
// moved like any refusal. Delivered, it would reach the caller as a success
|
|
// and be billed at the prompt it never answered.
|
|
if refusedIn(b) {
|
|
return declined(errorStatus(b), b)
|
|
}
|
|
whole = b
|
|
}
|
|
resp = r
|
|
return nil
|
|
})
|
|
return resp, whole, used, err
|
|
}
|
|
|
|
// forward relays p to the first provider on its route that answers, and reports
|
|
// whether a stream took the hold with it — a streamed answer settles from inside its
|
|
// own writer, which runs after the handler has returned.
|
|
//
|
|
// It is the failover loop ask.cascade runs for the text pipeline, applied to a body
|
|
// rather than a question:
|
|
//
|
|
// provider fault -> record it, rest the account, offer it to the next provider
|
|
// request fault -> stop, and relay the upstream's own status
|
|
// nobody left -> exhausted()
|
|
//
|
|
// Every one of those is decided on the status, before a byte reaches the caller. A
|
|
// stream is judged by its opening frames too (opening), because a vendor that
|
|
// streams answers 200 before it has an answer. Once the answer begins, the request
|
|
// belongs to that provider and nothing after can move it.
|
|
//
|
|
// THE RELAY DOES NOT RACE. Fast mode races the text pipeline's writers, which carry
|
|
// a beaten provider's partial answer to the ledger. A relayed attempt is an HTTP
|
|
// response committed at its status, and racing several would mean reading every
|
|
// loser to its end to learn what it cost. A fast request takes the cascade here, and
|
|
// the wider reservation widthFor made for it covers the one attempt it makes.
|
|
func (c *ApiController) forward(p pass) bool {
|
|
sku := p.req.Model
|
|
d, err := outbound(p.body, p.req, p.knowledge)
|
|
if err != nil {
|
|
c.ResponseFailure(modelError("Failed to parse request: %s", parseProblem(err)))
|
|
return false
|
|
}
|
|
// What the answer reads from the request, read now: a streamed answer is billed
|
|
// after this handler returns and fiber has released the request (base.go).
|
|
snap := c.takeSnapshot(p.user)
|
|
ctx := c.Context()
|
|
|
|
queue := candidates(snap.org, p.route, p.prior, FreeOnlyFor(ctx))
|
|
if p.route == nil {
|
|
queue = []candidate{{p.primary.Name, p.primary.SubType}}
|
|
}
|
|
var proof [2]string
|
|
if p.strict {
|
|
// The route's own first row, whether or not it is resting: a strict request is
|
|
// served where it was sent or refused there, never moved.
|
|
if p.route != nil {
|
|
queue = []candidate{{p.route.providerName, p.route.upstreamModel}}
|
|
}
|
|
// Every row the relay serves sells its answers, so with the paid lane off a
|
|
// strict request is refused exactly as candidates() leaves any other with
|
|
// nobody to ask. Taking the first row directly must not step around the lane.
|
|
if shut(ctx, p.primary) {
|
|
c.ResponseFailure(laneOff(ctx, sku))
|
|
return false
|
|
}
|
|
var ok bool
|
|
if proof, ok = c.provenUnchanged(p, d, queue[0]); !ok {
|
|
return false
|
|
}
|
|
}
|
|
|
|
// With the paid lane off a third-party route is offered nobody — its own vendors
|
|
// sell their answers and nothing stands in for the model named — so it is
|
|
// refused, plainly, rather than answered by something else.
|
|
if len(queue) == 0 && FreeOnlyFor(ctx) && len(p.prior) == 0 {
|
|
c.ResponseFailure(laneOff(ctx, sku))
|
|
return false
|
|
}
|
|
tried := p.prior
|
|
by := ""
|
|
for _, cand := range queue {
|
|
// The caller hung up. Offering the request to another vendor would spend
|
|
// money answering an empty room.
|
|
if err = ctx.Err(); err != nil {
|
|
break
|
|
}
|
|
resp, whole, row, e := p.call(ctx, snap.org, rowFor(p.primary, cand), cand, d)
|
|
if e == nil {
|
|
if len(tried) > 0 {
|
|
log.Warn("failover: model=%s served by %s after %d refusal(s) — %s",
|
|
sku, cand.provider, len(tried), reasons(tried))
|
|
}
|
|
if n := len(p.prior); len(tried) > n {
|
|
recordRefusals(snap, sku, tried[n:], p.user, p.premium, p.req.Stream, p.id, p.start)
|
|
}
|
|
// The proof is stamped on an answer only: a refusal was served nothing.
|
|
if p.strict {
|
|
c.SetHeader(strictHeader, "1")
|
|
c.SetHeader(requestSha, proof[0])
|
|
c.SetHeader(upstreamSha, proof[1])
|
|
}
|
|
return c.deliver(p, cand, row, resp, whole, snap)
|
|
}
|
|
if errors.Is(e, errUnrelayable) {
|
|
log.Warn("relay: model=%s provider=%s speaks no OpenAI chat dialect and is passed over", sku, cand.provider)
|
|
continue
|
|
}
|
|
err, by = e, cand.provider
|
|
// The caller's own debt, relayed back to us. No vendor refused anything and
|
|
// none will fix it, so nobody is rested and nobody else is asked.
|
|
if codeOf(e) == object.CodeInsufficientBalance {
|
|
break
|
|
}
|
|
at := attempt{
|
|
provider: cand.provider,
|
|
upstream: cand.upstream,
|
|
origin: originOf(row, nil),
|
|
status: upstreamHTTPStatus(e),
|
|
fault: faultOf(e),
|
|
err: e,
|
|
}
|
|
tried = append(tried, at)
|
|
announce(sku, at)
|
|
if rest := cooled.rest(snap.org, cand.provider, e); rest > 0 {
|
|
log.Warn("failover: demoting provider=%s for %s after status=%d", cand.provider, rest, at.status)
|
|
}
|
|
// The request is wrong, not the vendor. Every other vendor refuses it the
|
|
// same way, so stop and hand the caller the real reason.
|
|
if at.fault == faultRequest {
|
|
break
|
|
}
|
|
err, by = nil, ""
|
|
}
|
|
if err == nil {
|
|
err = exhausted(sku, tried)
|
|
}
|
|
if n := len(p.prior); len(tried) > n {
|
|
recordRefusals(snap, sku, tried[n:], p.user, p.premium, p.req.Stream, p.id, p.start)
|
|
}
|
|
|
|
// The outcome's own row, as the text pipeline files one: named for the provider
|
|
// that ended the request, if one did, and priced at what it ran (spent).
|
|
if p.user != nil {
|
|
rec := &usageRecord{
|
|
Owner: snap.org,
|
|
Model: sku,
|
|
Provider: by,
|
|
Premium: p.premium,
|
|
Stream: p.req.Stream,
|
|
Status: "error",
|
|
ErrorMsg: err.Error(),
|
|
ClientIP: snap.ip,
|
|
RequestID: p.id,
|
|
}
|
|
rec.PromptTokens, rec.CompletionTokens = spent(nil, sku, p.prompt, "", err)
|
|
rec.TotalTokens = rec.PromptTokens + rec.CompletionTokens
|
|
rec.bind(snap.ctx, p.user)
|
|
rec.BYO, rec.Account = providerBYO(p.primary, p.user)
|
|
recordUsage(rec)
|
|
recordTrace(snap.ctx, rec, p.start)
|
|
}
|
|
|
|
// Our own spend gate keeps its 402: the debt is the caller's (relay). Every other
|
|
// failure is a model call nothing answered.
|
|
if codeOf(err) == object.CodeInsufficientBalance {
|
|
c.ResponseFailure(err)
|
|
} else {
|
|
c.ResponseModelFailure(err)
|
|
}
|
|
return false
|
|
}
|
|
|
|
// deliver hands the caller the answer one provider gave, stamped as ours, and bills
|
|
// it: to the SKU the caller asked for, at the tokens the upstream reported, with a
|
|
// cached prompt priced as cached.
|
|
func (c *ApiController) deliver(p pass, cand candidate, row *object.Provider, resp *http.Response, whole []byte, snap snapshot) bool {
|
|
sku := p.req.Model
|
|
mk := &mark{id: "chatcmpl-" + p.id, model: sku, seller: seller(row, p.user)}
|
|
|
|
// settle runs where the answer is finished — for a stream, inside its writer — so
|
|
// it reads the snapshot and p, and never the controller.
|
|
settle := func(t tokens, text string) {
|
|
if t.prompt() == 0 && t.completion == 0 {
|
|
// The upstream said nothing about what it used. The prompt was measured on
|
|
// the way in and the answer is the text it sent, so neither is invented.
|
|
t.fresh = p.prompt
|
|
t.completion, _ = model.GetTokenSize(sku, text)
|
|
}
|
|
if p.user != nil {
|
|
rec := &usageRecord{
|
|
Owner: snap.org,
|
|
Organization: p.user.Owner,
|
|
Model: sku,
|
|
Provider: cand.provider,
|
|
Origin: originOf(row, mk),
|
|
CostNanoExact: mk.cogs(),
|
|
PromptTokens: t.prompt(),
|
|
CacheReadTokens: t.cached,
|
|
CacheWriteTokens: t.written,
|
|
CompletionTokens: t.completion,
|
|
TotalTokens: t.prompt() + t.completion,
|
|
ReasoningTokens: t.reasoning,
|
|
Currency: "USD",
|
|
Premium: p.premium,
|
|
Stream: p.req.Stream,
|
|
Status: "success",
|
|
ClientIP: snap.ip,
|
|
RequestID: p.id,
|
|
}
|
|
rec.bind(snap.ctx, p.user)
|
|
// Whose key paid is a property of the row that served, not of the one
|
|
// auth resolved before the request moved.
|
|
rec.BYO, rec.Account = providerBYO(row, p.user)
|
|
recordUsage(rec)
|
|
recordTrace(snap.ctx, rec, p.start)
|
|
}
|
|
// The same model and the same split the row above is priced from, so the
|
|
// hold settles what the ledger debits.
|
|
p.hold.settle(calculateCostCentsWithCache(sku, t.prompt(), t.completion, t.cached, t.written))
|
|
if p.judge != nil {
|
|
p.judge(text)
|
|
}
|
|
}
|
|
|
|
// A reasoning-inlining upstream (DeepSeek) has its leading <think></think> block
|
|
// stripped from what is forwarded; billing counts what it sent.
|
|
var strip *model.ReasoningStripper
|
|
if model.InlinesReasoning(cand.upstream) {
|
|
strip = &model.ReasoningStripper{}
|
|
}
|
|
|
|
if p.req.Stream {
|
|
c.SetHeader("Content-Type", "text/event-stream")
|
|
c.SetHeader("Cache-Control", "no-cache")
|
|
c.SetHeader("Connection", "keep-alive")
|
|
c.Status(http.StatusOK)
|
|
wants := p.req.StreamOptions != nil && p.req.StreamOptions.IncludeUsage
|
|
// The upstream body travels into the writer, which outlives this call, and is
|
|
// closed there. The hold travels with it: released on the way out if the
|
|
// writer ends without an answer, a no-op once settle has run.
|
|
upstreamBody := resp.Body
|
|
_ = c.SendStreamWriter(func(w *bufio.Writer) {
|
|
defer upstreamBody.Close()
|
|
defer p.hold.settle(0)
|
|
settle(streamCaptureUsage(upstreamBody, w, w.Flush, wants, strip, mk))
|
|
})
|
|
return true
|
|
}
|
|
|
|
// What goes out is ours (envelope.go); what came in is what the billing reads.
|
|
out := mk.stamp(whole)
|
|
if strip != nil {
|
|
out = stripReasoningBody(out)
|
|
}
|
|
var t tokens
|
|
sniffZenUsage(whole, &t)
|
|
settle(t, answerText(whole))
|
|
c.answerBody(out)
|
|
return false
|
|
}
|
|
|
|
// answerText is what a buffered chat answer said: its content and its tool-call
|
|
// arguments, the same text a streamed answer accumulates for the tokenizer and the
|
|
// judge.
|
|
func answerText(body []byte) string {
|
|
var r struct {
|
|
Choices []struct {
|
|
Message struct {
|
|
Content string `json:"content"`
|
|
ToolCalls []struct {
|
|
Function struct {
|
|
Arguments string `json:"arguments"`
|
|
} `json:"function"`
|
|
} `json:"tool_calls"`
|
|
} `json:"message"`
|
|
} `json:"choices"`
|
|
}
|
|
if json.Unmarshal(body, &r) != nil {
|
|
return ""
|
|
}
|
|
var sb strings.Builder
|
|
for _, ch := range r.Choices {
|
|
sb.WriteString(ch.Message.Content)
|
|
for _, tc := range ch.Message.ToolCalls {
|
|
sb.WriteString(tc.Function.Arguments)
|
|
}
|
|
}
|
|
return sb.String()
|
|
}
|
|
|
|
// refusedIn reports a buffered answer that carries an error and no choices.
|
|
func refusedIn(body []byte) bool {
|
|
var f struct {
|
|
Error json.RawMessage `json:"error"`
|
|
Choices json.RawMessage `json:"choices"`
|
|
}
|
|
return json.Unmarshal(body, &f) == nil && present(f.Error) && !present(f.Choices)
|
|
}
|
|
|
|
// provenUnchanged holds a strict request to the body its first attempt would send,
|
|
// and either refuses it naming what would change or returns the two digests that
|
|
// prove nothing did.
|
|
func (c *ApiController) provenUnchanged(p pass, d draft, cand candidate) ([2]string, bool) {
|
|
if d.ceiling > 0 {
|
|
c.refuseStrict("ceiling_unset", "name max_tokens or max_completion_tokens; the gateway writes a ceiling otherwise")
|
|
return [2]string{}, false
|
|
}
|
|
invariant, field, sent, relayed, err := unchanged(p.body, d.body(cand.upstream, p.primary))
|
|
if err != nil {
|
|
c.refuseStrict("not_canonical", err.Error())
|
|
return [2]string{}, false
|
|
}
|
|
switch {
|
|
case invariant == "":
|
|
return [2]string{sent, relayed}, true
|
|
case field == "max_tokens" || field == "max_completion_tokens":
|
|
c.refuseStrict("ceiling_lowered", fmt.Sprintf("%s is above what this request may spend; ask for less", field))
|
|
case field == "messages" && len(p.knowledge) > 0:
|
|
c.refuseStrict("rag", "retrieval adds a system message; turn retrieval off")
|
|
default:
|
|
c.refuseStrict(invariant, fmt.Sprintf("%q would not reach the vendor as sent", field))
|
|
}
|
|
return [2]string{}, false
|
|
}
|
|
|
|
// ownRow reports that org has a provider row of its own under name, which is the
|
|
// customer's key and not the platform's family.
|
|
func ownRow(org, name string) bool {
|
|
row, err := object.GetModelProviderByNameForOrg(org, name)
|
|
return err == nil && row != nil && row.Owner != "admin"
|
|
}
|
|
|
|
// unpiped is the refusal for a relayed request the family pipe cannot carry as sent,
|
|
// or "". The pipe sends a family only the fields it names (familyFields) and has no
|
|
// place for retrieved knowledge; a request relying on either would be answered
|
|
// without it, and read as if it had not been.
|
|
func unpiped(p pass) string {
|
|
if len(p.knowledge) > 0 {
|
|
return "retrieval is not available for this model while it is served from the free tier"
|
|
}
|
|
var fields map[string]json.RawMessage
|
|
if json.Unmarshal(p.body, &fields) != nil {
|
|
return ""
|
|
}
|
|
allowed := familyFields["chat/completions"]
|
|
for _, k := range slices.Sorted(maps.Keys(fields)) {
|
|
if !allowed[k] && !slices.Contains(ours, k) {
|
|
return fmt.Sprintf("%q is not available for this model while it is served from the free tier; send the request without it", k)
|
|
}
|
|
}
|
|
return ""
|
|
}
|