mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
- model/local.go, volcengine.go: LocalModelProvider (do-ai) and Volcengine run content through the stripper (ReasoningStripper.Answer), as openai.go does - Finish(reason): a finished turn releases what is held; a cut (length, no finish, no [DONE]) drops an open block - reasonedStream frames released text (<=64 KiB lines) - flagged stripper searches only the new tail for </think> - Anthropic and ZAP paths read opener_eaten from the route, as forward does - markup: a paragraph continuing a quote without a mark is quoted - proveNative is the one strict check; ZAP calls it Co-authored-by: Hanzo Dev <[email protected]>
391 lines
15 KiB
Go
391 lines
15 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/hanzoai/ai/model"
|
|
"github.com/hanzoai/ai/object"
|
|
openai "github.com/hanzoai/go-openai"
|
|
)
|
|
|
|
// sseStreamChunk is the subset of an OpenAI streaming chunk the relay reads to bill
|
|
// a streamed answer: whether it carries usage (the forced include_usage chunk), and
|
|
// the delta content / tool-call arguments the tokenizer falls back to.
|
|
type sseStreamChunk struct {
|
|
Choices []struct {
|
|
Delta struct {
|
|
Content string `json:"content"`
|
|
ToolCalls []struct {
|
|
Function struct {
|
|
Arguments string `json:"arguments"`
|
|
} `json:"function"`
|
|
} `json:"tool_calls"`
|
|
} `json:"delta"`
|
|
// FinishReason is read only for whether the answer said it was done: a stream
|
|
// that ends without one, and without [DONE], was cut, not finished.
|
|
FinishReason string `json:"finish_reason"`
|
|
} `json:"choices"`
|
|
// Usage is read here only for whether it is there. Its counts are read by
|
|
// sniffZenUsage, the one reader of usage in both dialects, because the split
|
|
// between a fresh and a cached prompt token is what the price turns on.
|
|
Usage *struct{} `json:"usage"`
|
|
}
|
|
|
|
// streamCaptureUsage copies an upstream OpenAI-style SSE stream from r to w while
|
|
// capturing token usage (from the forced include_usage chunk) and accumulating
|
|
// the output text (content + tool-call arguments) for a tokenizer fallback. A
|
|
// forced usage-only chunk (usage present, no choices) is suppressed when the
|
|
// client did not request usage; otherwise its envelope is fixed up for SDK
|
|
// clients. This is the billing-critical core of the streaming relay: it
|
|
// guarantees a streamed answer yields real token counts to bill.
|
|
//
|
|
// mk is what stamps every event on the way out (envelope.go): each chunk goes out in
|
|
// our envelope, stamped from the one mark, so the id holds for the whole stream.
|
|
//
|
|
// A line may be as long as the family relay allows (relayZenStream). A scanner that
|
|
// meets a longer one stops, and everything after it — the rest of the answer and
|
|
// the usage that bills it — would be dropped without an error anyone sees.
|
|
func streamCaptureUsage(r io.Reader, w io.Writer, flush func() error, clientWantsUsage bool, strip *model.ReasoningStripper, mk *mark) (t tokens, completionText string) {
|
|
scanner := bufio.NewScanner(r)
|
|
scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
|
var sb strings.Builder
|
|
// finished is the answer saying it is done: a finish_reason or [DONE]. gone is the
|
|
// caller hanging up, after which nothing more can be told to them.
|
|
var finished, gone bool
|
|
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
// An SSE comment is a vendor keeping its connection open in its own words
|
|
// (": OPENROUTER PROCESSING"). The keep-alive goes on; the words do not.
|
|
if strings.HasPrefix(line, ":") {
|
|
line = ":"
|
|
}
|
|
|
|
if after, ok := strings.CutPrefix(line, "data: "); ok {
|
|
raw := after
|
|
if strings.TrimSpace(raw) == "[DONE]" {
|
|
finished = true
|
|
if !letGoStripped(w, flush, mk, strip, finished) || !letGo(w, flush, mk) {
|
|
gone = true
|
|
break
|
|
}
|
|
} else {
|
|
var chunk sseStreamChunk
|
|
if json.Unmarshal([]byte(raw), &chunk) == nil {
|
|
if chunk.Usage != nil {
|
|
sniffZenUsage([]byte(raw), &t)
|
|
}
|
|
for _, ch := range chunk.Choices {
|
|
if ch.FinishReason != "" {
|
|
finished = true
|
|
}
|
|
sb.WriteString(ch.Delta.Content)
|
|
for _, tc := range ch.Delta.ToolCalls {
|
|
sb.WriteString(tc.Function.Arguments)
|
|
}
|
|
}
|
|
// Strip inline reasoning from the FORWARDED content only —
|
|
// billing above already counted the original. Gated by a
|
|
// non-nil stripper (reasoning-inlining upstreams only), so
|
|
// every other stream is byte-for-byte unchanged.
|
|
if strip != nil && len(chunk.Choices) > 0 {
|
|
line = "data: " + applyReasoningStrip(raw, strip)
|
|
}
|
|
if chunk.Usage != nil && len(chunk.Choices) == 0 {
|
|
if !clientWantsUsage {
|
|
continue
|
|
}
|
|
// An SDK client reads a usage-only event as a chunk like any
|
|
// other, so give it the rest of the envelope. The id and the
|
|
// model are the stamp's to say, and it says them below.
|
|
var usageChunk map[string]any
|
|
if json.Unmarshal([]byte(raw), &usageChunk) == nil {
|
|
usageChunk["object"] = "chat.completion.chunk"
|
|
usageChunk["created"] = time.Now().Unix()
|
|
usageChunk["choices"] = []any{}
|
|
if fixed, err := json.Marshal(usageChunk); err == nil {
|
|
line = "data: " + string(fixed)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Last thing before the wire, so it covers the line whatever the rewrites
|
|
// above left it as. `[DONE]` is not an object and passes through untouched.
|
|
line = "data: " + string(mk.stamp([]byte(strings.TrimPrefix(line, "data: "))))
|
|
}
|
|
|
|
// A write that fails is the client gone: return, and the caller's close of r
|
|
// hangs up on the vendor so its model stops. The text so far is what bills.
|
|
if _, err := fmt.Fprintf(w, "%s\n", line); err != nil {
|
|
gone = true
|
|
break
|
|
}
|
|
if flush != nil && flush() != nil {
|
|
gone = true
|
|
break
|
|
}
|
|
}
|
|
// The vendor stopped before the answer said it was done: its connection was cut,
|
|
// or a line outgrew the scanner. Ended quietly here, the caller's stream closes
|
|
// clean and its SDK hands back half an answer as if it were whole. An error frame
|
|
// is what an SDK raises on, so the caller learns the answer is not complete. What
|
|
// was relayed is still what bills.
|
|
if !gone {
|
|
gone = !letGoStripped(w, flush, mk, strip, finished) || !letGo(w, flush, mk)
|
|
}
|
|
if !finished && !gone {
|
|
why := "the upstream stream ended before the answer finished"
|
|
if err := scanner.Err(); err != nil {
|
|
log.Warn("relay: upstream stream broke mid-answer: %v", err)
|
|
why = "the upstream stream broke before the answer finished"
|
|
} else {
|
|
log.Warn("relay: upstream stream ended with no finish_reason and no [DONE]")
|
|
}
|
|
body, _ := json.Marshal(map[string]any{"error": map[string]any{"message": why}})
|
|
_, _ = w.Write(openError("openai", http.StatusBadGateway, body))
|
|
if flush != nil {
|
|
_ = flush()
|
|
}
|
|
}
|
|
return t, sb.String()
|
|
}
|
|
|
|
// letGoStripped writes the answer a reasoning stripper still holds as one chunk,
|
|
// through the stamp so the lifter reads it, before the lifter's own tail.
|
|
func letGoStripped(w io.Writer, flush func() error, mk *mark, strip *model.ReasoningStripper, done bool) bool {
|
|
tail := strippedTail(strip, done)
|
|
if tail == nil {
|
|
return true
|
|
}
|
|
if _, err := fmt.Fprintf(w, "data: %s\n\n", mk.stamp(tail)); err != nil {
|
|
return false
|
|
}
|
|
return flush == nil || flush() == nil
|
|
}
|
|
|
|
// letGo writes the text a stream's lifter still holds (lift.go) as one last chunk,
|
|
// before [DONE] or the end of the stream. false when the client is gone.
|
|
func letGo(w io.Writer, flush func() error, mk *mark) bool {
|
|
rest := mk.rest()
|
|
if rest == nil {
|
|
return true
|
|
}
|
|
if _, err := fmt.Fprintf(w, "data: %s\n\n", rest); err != nil {
|
|
return false
|
|
}
|
|
return flush == nil || flush() == nil
|
|
}
|
|
|
|
const (
|
|
// reserveCompletionFloor is the completion-token cap the QueryText pipeline
|
|
// applies to EVERY model (model.ChatCompletionRequest) AND the ceiling an
|
|
// UNCAPPED request's max_tokens is clamped to. The reservation always covers
|
|
// at least this many completion tokens, so a request can never settle beyond
|
|
// its hold — closing the single-request overdraft (R1b): the prior 1024-token
|
|
// estimate under-reserved an uncapped request that then settled the ACTUAL
|
|
// (larger) spend, driving the balance negative within one request.
|
|
reserveCompletionFloor = 4096
|
|
|
|
// maxReserveCompletionTokens bounds an absurd client max_tokens so the
|
|
// reservation can't be driven arbitrarily large by a hostile request.
|
|
maxReserveCompletionTokens = 32768
|
|
)
|
|
|
|
// clampMaxTokens is the upstream completion ceiling enforced on a request for model.
|
|
// A normal client cap (0 < mt <= maxReserveCompletionTokens) is preserved as-is, and
|
|
// a larger one is lowered to maxReserveCompletionTokens, so the proxied (tool/stream)
|
|
// upstream can never emit an unbounded completion. A request that names no cap gets
|
|
// reserveCompletionFloor — except on a paid tier of Zen or Enso (thinks), whose
|
|
// models think before they answer and spend the ceiling on it: there a request that
|
|
// names none gets maxReserveCompletionTokens, or the answer is all reasoning, cut at
|
|
// `length`, and billed. Whatever the ceiling, it is never above the most one answer
|
|
// of model may hold where it is stated (mostOut; gpt-4o: 16,384), which its vendor
|
|
// would refuse. The caller MUST assign this back to request.MaxTokens before the
|
|
// upstream call.
|
|
func clampMaxTokens(model string, maxTokens int) int {
|
|
c := reserveCompletionFloor
|
|
switch {
|
|
case maxTokens > maxReserveCompletionTokens:
|
|
c = maxReserveCompletionTokens
|
|
case maxTokens > 0:
|
|
c = maxTokens
|
|
case thinks(model):
|
|
c = maxReserveCompletionTokens
|
|
}
|
|
if most := mostOut(model); most > 0 && c > most {
|
|
c = most
|
|
}
|
|
return c
|
|
}
|
|
|
|
// mostOut is the most completion tokens one answer of model may hold, where it is
|
|
// stated: a family's catalog (OpenRouter's top_provider.max_completion_tokens) or a
|
|
// configured route's models.yaml max_output_tokens, following the configuration's
|
|
// aliases and each route's upstream model, however many steps, into a family's
|
|
// catalog where one ends there. 0 when nothing on the way states it.
|
|
func mostOut(model string) int {
|
|
mc := GetModelConfig()
|
|
seen := map[string]bool{}
|
|
for m := strings.ToLower(strings.TrimSpace(model)); m != "" && !seen[m]; {
|
|
seen[m] = true
|
|
if f, ok := familyLookup(m); ok && f.MaxOut > 0 {
|
|
return f.MaxOut
|
|
}
|
|
if mc == nil {
|
|
return 0
|
|
}
|
|
n, next := mc.outStep(m)
|
|
if n > 0 {
|
|
return n
|
|
}
|
|
m = next
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// thinks reports whether model is a paid tier of Zen or Enso: priced, so served by
|
|
// models that always reason. A model resold from another vendor's catalog is not
|
|
// one. Read from the discovery snapshot, never a refresh.
|
|
func thinks(model string) bool {
|
|
for _, f := range []*modelFamily{zenFam, ensoFam} {
|
|
if m, ok := f.lookup(model); ok {
|
|
return m.priced()
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// reserveCompletionTokens is the completion-token count to RESERVE for: the larger
|
|
// of the (clamped) upstream ceiling and the QueryText pipeline's fixed cap. The
|
|
// QueryText pipeline does NOT thread the request's max_tokens (it caps at
|
|
// reserveCompletionFloor itself), so the reservation must cover that floor even
|
|
// when the client capped lower — guaranteeing actual <= reserve on EVERY path.
|
|
func reserveCompletionTokens(model string, maxTokens int) int {
|
|
if c := clampMaxTokens(model, maxTokens); c > reserveCompletionFloor {
|
|
return c
|
|
}
|
|
return reserveCompletionFloor
|
|
}
|
|
|
|
// budgetHold is a per-request reservation against the shared balance ledger. It
|
|
// holds an upper-bound estimate before the upstream call and releases it (while
|
|
// applying the ACTUAL spend) when the request settles — so concurrent requests
|
|
// for one subject can never double-spend or drive the balance negative.
|
|
type budgetHold struct {
|
|
subject string
|
|
est int64
|
|
// settled is set by the one settle that runs. Atomic, because a streamed answer
|
|
// settles from the goroutine producing it while the handler's deferred settle
|
|
// may run at the same moment.
|
|
settled atomic.Bool
|
|
}
|
|
|
|
// reserveBudget holds est cents against the caller's spendable balance BEFORE a
|
|
// paid request runs. It returns ok=false (gate the request) ONLY when the subject
|
|
// has a known ledger balance that cannot cover the estimate. When the subject is
|
|
// unknown to the ledger (billing disabled, exempt org, or balance not yet
|
|
// fetched) it does NOT gate — reservation only strengthens the existing balance
|
|
// gate, it never introduces a new fail-closed path of its own.
|
|
func reserveBudget(subject string, est int64) (*budgetHold, bool) {
|
|
if subject == "" {
|
|
return &budgetHold{}, true
|
|
}
|
|
if _, known := object.GlobalBalanceLedger.Available(subject); !known {
|
|
return &budgetHold{}, true
|
|
}
|
|
if !object.GlobalBalanceLedger.Reserve(subject, est) {
|
|
return nil, false
|
|
}
|
|
return &budgetHold{subject: subject, est: est}, true
|
|
}
|
|
|
|
// reserveFor is reserveBudget for the request on ctx: a request the plan or a free
|
|
// cap pays for holds nothing, because no wallet pays for it.
|
|
func reserveFor(ctx context.Context, subject string, est int64) (*budgetHold, bool) {
|
|
if covered(ctx) {
|
|
return &budgetHold{}, true
|
|
}
|
|
return reserveBudget(subject, est)
|
|
}
|
|
|
|
// settle releases the hold and applies the ACTUAL spend (actualCents) to the
|
|
// ledger so subsequent reads reflect it immediately. Idempotent and nil-safe, so
|
|
// a deferred fail-safe settle(0) is harmless once a real settle has run.
|
|
func (h *budgetHold) settle(actualCents int64) {
|
|
h.settleNano(actualCents * 10_000_000)
|
|
}
|
|
|
|
// settleNano is settle with the actual spend in nano-USD, for a call priced below a
|
|
// cent: the ledger carries the fraction rather than rounding it away.
|
|
func (h *budgetHold) settleNano(actualNano int64) {
|
|
if h == nil || h.subject == "" || !h.settled.CompareAndSwap(false, true) {
|
|
return
|
|
}
|
|
object.GlobalBalanceLedger.SettleNano(h.subject, h.est, actualNano)
|
|
}
|
|
|
|
// estimatePromptTokens counts the prompt tokens for a chat request (tiktoken via
|
|
// the model package), falling back to a coarse character estimate.
|
|
func estimatePromptTokens(req *openai.ChatCompletionRequest) int {
|
|
if n, err := model.OpenaiNumTokensFromMessages(req.Messages, req.Model); err == nil && n > 0 {
|
|
return n
|
|
}
|
|
chars := 0
|
|
for _, m := range req.Messages {
|
|
chars += len(m.Content)
|
|
for _, p := range m.MultiContent {
|
|
chars += len(p.Text)
|
|
}
|
|
}
|
|
return chars/4 + 1
|
|
}
|
|
|
|
// estimateRequestCostCents returns an UPPER-BOUND cost for a chat request: the
|
|
// prompt cost plus the cost of reserveCompletionTokens(maxTokens) — the worst-case
|
|
// completion the upstream can actually emit on any serving path. The balance gate
|
|
// reserves this so a request is admitted only when the spendable balance covers
|
|
// its worst case, and (with the max_tokens clamp at the call site) the actual
|
|
// settle can never exceed the reservation.
|
|
//
|
|
// A route priced at zero estimates zero, the same answer costsNothing gives the
|
|
// two wallet gates. Without this the worst case is a cent that will never be
|
|
// charged: calculateCostCents floors a non-zero token count at 1 cent so a real
|
|
// call is never billed as free, and on a $0.00 wallet that invented cent is the
|
|
// difference between admitted and refused — which put the free routes out of
|
|
// reach of the free tier they are published for. The floor still governs
|
|
// settlement, where it belongs.
|
|
//
|
|
// The price is read the way the arithmetic below reads it — no org, so the guard
|
|
// and the sum it guards can never disagree — and it fails closed like its twins:
|
|
// only a price that was FOUND and is zero estimates zero.
|
|
func estimateRequestCostCents(modelName string, promptTokens, maxTokens int) int64 {
|
|
if costsNothing(modelName, "") {
|
|
return 0
|
|
}
|
|
return calculateCostCents(modelName, promptTokens, reserveCompletionTokens(modelName, maxTokens))
|
|
}
|