mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
397 lines
16 KiB
Go
397 lines
16 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
|
|
metric "github.com/luxfi/metric"
|
|
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/hanzoai/ai/model"
|
|
"github.com/hanzoai/ai/object"
|
|
)
|
|
|
|
// errPartiallyWritten stops a retry that can no longer be made safely: bytes are
|
|
// already on the wire, so replaying the call would duplicate the client's
|
|
// output. Deliberately NOT transient — retryTransient sees a permanent error and
|
|
// stops immediately rather than sleeping first.
|
|
var errPartiallyWritten = errors.New("response partially written; cannot retry")
|
|
|
|
// served names the provider that answered: our route label for it, the host it
|
|
// answered from, and the row whose credential was spent. Three projections of
|
|
// ONE fact travelling together, so a caller cannot record the label while losing
|
|
// the address — or, worse, bill against a row that did not serve.
|
|
//
|
|
// A failed attempt carries a name and nothing else: nothing answered, and saying
|
|
// nothing is the honest form of that.
|
|
type served struct {
|
|
name string
|
|
origin string
|
|
// row is the provider that actually spent a credential. providerBYO must
|
|
// read THIS and not the row auth resolved before the request moved: a
|
|
// customer's own key is only "bring your own" if it is the key that paid.
|
|
row *object.Provider
|
|
}
|
|
|
|
// ask is one text completion waiting to be served: what to send, where the
|
|
// answer goes, and who has already refused it.
|
|
//
|
|
// A struct rather than eleven positional arguments because two of them are
|
|
// []*model.RawMessage — history and knowledge — and in money code a silent
|
|
// argument swap between two same-typed slices is not a hazard worth keeping for
|
|
// the sake of a shorter call.
|
|
type ask struct {
|
|
ctx context.Context
|
|
route *modelRoute
|
|
org string // ledger org, so a customer's own provider row wins where it exists
|
|
model string // caller-facing model name, for the honest error
|
|
|
|
// primary is the row auth already resolved for route.providerName. Reusing
|
|
// it keeps the first attempt on exactly the credential the auth path chose;
|
|
// re-resolving by name would quietly swap a customer's connected key for
|
|
// the platform's.
|
|
primary *object.Provider
|
|
|
|
question string
|
|
history []*model.RawMessage
|
|
knowledge []*model.RawMessage
|
|
lang string
|
|
|
|
writer io.Writer
|
|
// prompt is the prompt's token count, already measured by the caller. Only a
|
|
// raced attempt reads it: a loser is billed for the prompt the vendor did
|
|
// accept plus whatever it produced before losing, and re-counting it here
|
|
// would be a second answer to a question the caller already asked.
|
|
prompt int
|
|
// fan carries this request into fast mode: several providers at once, first
|
|
// answer wins. Nil is the cascade, which is what every request that does not
|
|
// ask for fast mode gets.
|
|
fan *fan
|
|
// sent reports whether any byte has reached the client. Once it has, this
|
|
// request is committed and can never be moved.
|
|
sent func() bool
|
|
// prior are providers that already refused this request elsewhere — the
|
|
// family pipe, typically. They are not asked again and they count against
|
|
// the attempt cap.
|
|
prior []attempt
|
|
}
|
|
|
|
// serve offers the completion to each capable provider in turn and returns the
|
|
// first real answer.
|
|
//
|
|
// The loop is the whole failover policy in one place:
|
|
//
|
|
// provider fault -> record it, demote the vendor, offer it to the next one
|
|
// request fault -> stop; every vendor refuses this identically
|
|
// nobody left -> exhausted(), which names who was asked and why they said no
|
|
//
|
|
// It returns every attempt made, successful or not, so the caller can record
|
|
// the ones that failed. A failover nobody can see is half a fix: the empty
|
|
// account that caused it stays empty.
|
|
func (a ask) serve() (*model.ModelResult, served, []attempt, error) {
|
|
// Fast mode asks several at once; everything else takes them in turn. One
|
|
// entry point either way, so a caller sets a number rather than choosing a
|
|
// code path — and a fan too narrow to race falls back here by itself.
|
|
if a.fan != nil && a.fan.n > 1 {
|
|
return a.race()
|
|
}
|
|
return a.cascade()
|
|
}
|
|
|
|
// cascade is the failover loop itself: one provider at a time, in order.
|
|
func (a ask) cascade() (*model.ModelResult, served, []attempt, error) {
|
|
tried := a.prior
|
|
for _, c := range candidates(a.org, a.route, a.prior) {
|
|
// The caller hung up. Offering the request to another vendor would spend
|
|
// money answering an empty room, and would file a healthy vendor's name
|
|
// against the client's disconnect.
|
|
if err := a.context().Err(); err != nil {
|
|
return nil, served{}, tried, err
|
|
}
|
|
res, row, err := a.call(c)
|
|
if err == nil {
|
|
if len(tried) > 0 {
|
|
log.Warn("failover: model=%s served by %s after %d refusal(s) — %s",
|
|
a.model, c.provider, len(tried), reasons(tried))
|
|
}
|
|
// The row travels with the answer because it is the row whose
|
|
// credential was spent, and that is what decides whether this call
|
|
// was the customer's own key or ours.
|
|
// No mark: this path builds the answer from the provider SDK rather
|
|
// than relaying an envelope, so nothing disclosed an upstream and the
|
|
// host we dialled is the whole answer.
|
|
return res, served{c.provider, originOf(row, nil), row}, tried, nil
|
|
}
|
|
|
|
at := attempt{
|
|
provider: c.provider,
|
|
upstream: c.upstream,
|
|
origin: originOf(row, nil),
|
|
status: upstreamHTTPStatus(err),
|
|
fault: faultOf(err),
|
|
err: err,
|
|
}
|
|
tried = append(tried, at)
|
|
announce(a.model, at)
|
|
|
|
if d := cooled.rest(a.org, c.provider, err); d > 0 {
|
|
log.Warn("failover: demoting provider=%s for %s after status=%d", c.provider, d, at.status)
|
|
}
|
|
|
|
// The request is wrong, not the vendor. Every other vendor refuses it
|
|
// the same way, so stop and hand the caller the real reason.
|
|
if at.fault == faultRequest {
|
|
return nil, served{name: c.provider}, tried, err
|
|
}
|
|
|
|
// Bytes reached the client during that attempt. Nothing can be replayed
|
|
// now; report what we have rather than duplicating output.
|
|
if a.sent != nil && a.sent() {
|
|
log.Warn("failover: model=%s provider=%s failed after the stream opened — cannot move: %v",
|
|
a.model, c.provider, err)
|
|
return nil, served{name: c.provider}, tried, err
|
|
}
|
|
}
|
|
return nil, served{}, tried, exhausted(a.model, tried)
|
|
}
|
|
|
|
// call offers the completion to ONE provider, riding out a transient refusal
|
|
// (a 429 "overloaded", a 503, a timeout) rather than giving up on it.
|
|
//
|
|
// A 429 does not mean "this cannot be served", it means "ask again shortly" —
|
|
// and we are better placed to do that than a client who can neither pick a
|
|
// different vendor nor stagger itself against every other client doing the
|
|
// same. We wait it out here; when waiting stops helping, serve() moves on.
|
|
func (a ask) call(c candidate) (*model.ModelResult, *object.Provider, error) {
|
|
var res *model.ModelResult
|
|
var row *object.Provider
|
|
err := retryTransient(a.context(), currentRetryPolicy(), func() error {
|
|
// Committed: bytes are on the wire and this can no longer be replayed.
|
|
if a.sent != nil && a.sent() {
|
|
return errPartiallyWritten
|
|
}
|
|
// Discard whatever a failed attempt left in the writer. Both writers
|
|
// APPEND to an internal buffer, so without this the next provider's
|
|
// answer is served glued to the dead one's half-sentence — a silent
|
|
// corruption that reads like a model going mad.
|
|
if r, ok := a.writer.(interface{ Reset() }); ok {
|
|
r.Reset()
|
|
}
|
|
var e error
|
|
res, row, e = callProvider(a.org, a.rowFor(c), c, a.question, a.writer, a.history, a.knowledge, a.lang)
|
|
if e != nil && isTransientError(e) {
|
|
log.Warn("retry: provider=%s is busy (%v) — holding the request rather than bouncing it to the client", c.provider, e)
|
|
}
|
|
return wrapUpstreamError(e)
|
|
})
|
|
return res, row, err
|
|
}
|
|
|
|
// context returns the caller's context, so a retry never outlives the request
|
|
// that asked for it. Background would make retryPolicy.waitBeforeRetry's
|
|
// deadline check vacuous and let a dead client's request keep sleeping.
|
|
func (a ask) context() context.Context {
|
|
if a.ctx != nil {
|
|
return a.ctx
|
|
}
|
|
return context.Background()
|
|
}
|
|
|
|
// rowFor returns the pre-resolved provider row when the candidate IS the one
|
|
// auth resolved, and nil otherwise so callProvider resolves it for the org.
|
|
func (a ask) rowFor(c candidate) *object.Provider {
|
|
if a.primary != nil && a.route != nil && c.provider == a.route.providerName {
|
|
return a.primary
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// The conditions supplyRefused counts. They label the metric and name the branch
|
|
// that records them, so the reason an operator reads and the reason the code
|
|
// took are one value rather than two spellings of one.
|
|
const (
|
|
reasonFunding = "funding" // 402 — this account with the vendor is spent
|
|
reasonCredential = "credential" // 401 — our key is dead or revoked
|
|
reasonCeiling = "ceiling" // our own cash breaker is holding
|
|
reasonRefused = "refused" // everything else a vendor answers
|
|
)
|
|
|
|
// supplyRefused counts the refusals that are OURS. The caller is told 503 and owes
|
|
// nothing, so nobody on their side has a reason to report it: without this, an
|
|
// empty vendor account is visible only to whoever thinks to tail a log, and a log
|
|
// is read after somebody has already noticed.
|
|
//
|
|
// Labelled by provider and reason because "top this vendor up", "this key is dead"
|
|
// and "our own ceiling is holding" are three different jobs, and a single number
|
|
// cannot say which one is running. Both label sets are bounded — the providers are
|
|
// the route table's, the reasons are the four above.
|
|
//
|
|
// It registers into the same DefaultRegistry that GET /v1/metrics serves
|
|
// (object.MetricsHandler), which is the seam vmagent already scrapes. Alert on
|
|
// rate(cloud_supply_refused[5m]) > 0.
|
|
var supplyRefused = metric.NewCounterVec(metric.CounterOpts{
|
|
Name: "cloud_supply_refused",
|
|
Help: "Requests refused because our own supply could not serve them, by provider and reason",
|
|
}, []string{"provider", "reason"})
|
|
|
|
// announce puts a refusal where someone will see it. Level is chosen by who has
|
|
// to act on it: a busy vendor is routine, an empty account needs a human with a
|
|
// credit card, and a rejected credential needs one now.
|
|
//
|
|
// The counter carries the same fact for anything that watches rather than reads.
|
|
// A log line is evidence after the fact and only for whoever thinks to look; the
|
|
// metric is what can raise a hand before a customer does.
|
|
func announce(model string, a attempt) {
|
|
switch {
|
|
case a.status == http.StatusUnauthorized:
|
|
supplyRefused.WithLabelValues(a.provider, reasonCredential).Inc()
|
|
log.Error("provider=%s rejected our credential (401) serving model=%s — this key is dead or revoked; the request FAILED OVER, so customers are being served and only this vendor is idle. Rotate it: %v",
|
|
a.provider, model, a.err)
|
|
case a.status == http.StatusPaymentRequired:
|
|
supplyRefused.WithLabelValues(a.provider, reasonFunding).Inc()
|
|
log.Error("provider=%s is out of money (402) serving model=%s — top it up; failing over: %v",
|
|
a.provider, model, a.err)
|
|
default:
|
|
supplyRefused.WithLabelValues(a.provider, reasonRefused).Inc()
|
|
log.Warn("failover: model=%s provider=%s refused status=%d fault=%s: %v",
|
|
model, a.provider, a.status, a.fault, a.err)
|
|
}
|
|
}
|
|
|
|
// reasons renders the refusals so far for one log line.
|
|
func reasons(tried []attempt) string {
|
|
out := ""
|
|
for i, a := range tried {
|
|
if i > 0 {
|
|
out += "; "
|
|
}
|
|
out += fmt.Sprintf("%s status=%d", a.provider, a.status)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// callProvider asks one provider row for a completion and reports the row that
|
|
// answered. The row is resolved FOR THE ORG, so a customer that connected its
|
|
// own key spends its own key — on a fallback exactly as on the primary.
|
|
// Resolving globally here is how a request billed as "bring your own" comes to
|
|
// be served on the platform's credential.
|
|
//
|
|
// Indirected through a var so the failover loop can be exercised against
|
|
// scripted providers without a live DB or a live upstream, the same way
|
|
// orgAutoRoutingLookup is.
|
|
var callProvider = func(
|
|
org string,
|
|
row *object.Provider,
|
|
c candidate,
|
|
question string,
|
|
writer io.Writer,
|
|
history []*model.RawMessage,
|
|
knowledge []*model.RawMessage,
|
|
lang string,
|
|
) (*model.ModelResult, *object.Provider, error) {
|
|
if row == nil {
|
|
var err error
|
|
row, err = object.GetModelProviderByNameForOrg(org, c.provider)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
}
|
|
if row == nil {
|
|
// GetModelProviderByNameForOrg returns nil for BOTH a missing provider
|
|
// and one an admin switched off. Either way this vendor cannot serve and
|
|
// the next candidate must be tried, which is how a route's fallbacks stay
|
|
// honoured when the primary is toggled off from /v1/admin/providers.
|
|
//
|
|
return nil, nil, unavailable(c.provider)
|
|
}
|
|
|
|
p := *row // copy: SubType is per-request, the row may be shared
|
|
p.SubType = c.upstream
|
|
|
|
modelProvider, err := p.GetModelProvider(lang)
|
|
if err != nil {
|
|
return nil, &p, err
|
|
}
|
|
|
|
result, err := modelProvider.QueryText(question, writer, history, "", knowledge, nil, lang)
|
|
return result, &p, err
|
|
}
|
|
|
|
// unavailable is how a provider row that is missing or switched off refuses.
|
|
//
|
|
// It carries a STATUS rather than a word for the classifier to find. This error
|
|
// is ours: re-reading our own prose to recover what we already knew is how
|
|
// "unavailable" came to be a substring the classifier trusted, where it also
|
|
// matched a VENDOR saying "unavailable for your account tier" — a different
|
|
// refusal with a different answer.
|
|
//
|
|
// Named so the cascade tests can assert the value production emits instead of a
|
|
// string that merely resembles it.
|
|
func unavailable(provider string) error {
|
|
return &apiError{status: http.StatusServiceUnavailable,
|
|
msg: fmt.Sprintf("provider %q is unavailable (disabled or not configured)", provider)}
|
|
}
|
|
|
|
// recordRefusals writes one usage row per provider that refused, so a vendor
|
|
// that stopped serving appears in the ledger the team already reads rather than
|
|
// only in a log line nobody tails. That absence is the second half of the
|
|
// outage this exists to prevent: the failover worked, nobody noticed, and the
|
|
// account stayed empty.
|
|
//
|
|
// The rows bill nothing: recordUsage is not called here, and no budget hold is
|
|
// touched. The money for this request is settled ONCE, on the record the caller
|
|
// writes for the outcome — served or failed — and an attempt that refused before
|
|
// producing a token adds nothing to it. The status is its own word — "failover",
|
|
// not "error" — so dashboards that count failures keep counting the same thing and
|
|
// a refused ATTEMPT on a request that ultimately succeeded is not filed as a
|
|
// failed request.
|
|
//
|
|
// A PROVIDER BEATEN IN A RACE NEVER ARRIVES HERE, and must not: it was cancelled
|
|
// mid-generation, so the vendor produced tokens and charged us, and this row
|
|
// would file a real charge as a free refusal. ask.race keeps them out of the
|
|
// attempts it returns and settles them through fan.bill instead — one ledger
|
|
// path each, so neither rule can be applied to the other's case.
|
|
// recordRefusals logs every vendor that refused. Its caller runs inside a stream
|
|
// writer, where the request context is already released, so it reads the
|
|
// request's outliving parts from a snapshot.
|
|
func recordRefusals(snap snapshot, model string, tried []attempt, user *iam.User, premium, stream bool, requestId string, start time.Time) {
|
|
if user == nil {
|
|
return
|
|
}
|
|
for _, a := range tried {
|
|
rec := &usageRecord{
|
|
Owner: snap.org,
|
|
Model: model,
|
|
Provider: a.provider,
|
|
Origin: a.origin,
|
|
Premium: premium,
|
|
Stream: stream,
|
|
Status: "failover",
|
|
ErrorMsg: a.err.Error(),
|
|
ClientIP: snap.ip,
|
|
RequestID: requestId,
|
|
}
|
|
rec.bind(snap.ctx, user)
|
|
recordTrace(snap.ctx, rec, start)
|
|
}
|
|
}
|