mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-21 11:09:56 +00:00
1027 lines
38 KiB
Go
1027 lines
38 KiB
Go
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
// zen_client.go — ai's thin client of the Hanzo model-family services.
|
|
//
|
|
// A model family (Zen, Enso, …) is a self-contained serving layer: it owns which
|
|
// upstream serves a SKU, the identity, the reasoning-vocabulary fold, the 1M context
|
|
// ladder, vision, and the enso-ultra fan-out. ai authenticates, meters, and forwards;
|
|
// it discovers each family from GET <family>/v1/models and pipes requests verbatim.
|
|
// A family is a VALUE — a name, a base URL, a service key, an ownership prefix — so
|
|
// zen and enso are two instances of the SAME machinery (modelFamily), not two copies.
|
|
// ai holds only configuration and the public brand, never an upstream mapping. See
|
|
// hip-00NN for the catalog, pricing, and discovery contract.
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/conf"
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/hanzoai/ai/object"
|
|
"github.com/hanzoai/ai/util"
|
|
"github.com/hanzoai/decimal"
|
|
"github.com/hanzoai/money"
|
|
)
|
|
|
|
// ── the families (values, not places) ────────────────────────────────────────
|
|
//
|
|
// Each family's address is configuration (a base URL + service key, set in the
|
|
// deployment), so this repository carries no routing detail and reads as open
|
|
// source. A family is added by adding an instance here — nothing else in ai knows a
|
|
// family name.
|
|
|
|
// flagshipWindow is the served context window (1M tokens) ai guarantees for the
|
|
// flagship Zen/Enso SKUs, independent of what a given discovery snapshot advertises.
|
|
const flagshipWindow = 1_000_000
|
|
|
|
// pinnedSKU is a family SKU ai guarantees in /v1/models at a known context window.
|
|
// When the family already serves the SKU, the pin fixes its listed window; when the
|
|
// family does not advertise it yet — a SKU ai routes ahead of discovery (e.g.
|
|
// enso-pro) — the pin also synthesizes the listing entry. Pins apply only to a
|
|
// CONFIGURED family (enabled()), so an unconfigured family lists nothing. See
|
|
// hip-00NN.
|
|
type pinnedSKU struct {
|
|
id string
|
|
contextWindow int
|
|
}
|
|
|
|
// modelFamily is one serving family as a value: its public name/brand + cold-start
|
|
// ownership prefix, the config keys that hold its address, the provider record ai
|
|
// forwards through, and a discovered-catalog snapshot refreshed from the family on a
|
|
// TTL. All of zen's original machinery is a method on this value; zen and enso are
|
|
// two instances.
|
|
type modelFamily struct {
|
|
name string // "zen" | "enso" | "openrouter" — the brand label
|
|
provider string // the object.Provider Type this family serves ("Zen" | "Enso" | "OpenRouter")
|
|
prefix string // public brand prefix for cold-start ownership (before first discovery)
|
|
owner string // public /v1/models owned_by: "zenlm" (Zen LM) | "hanzo" (Hanzo)
|
|
urlKey string // config key for the base URL ("ZEN_URL" | "ENSO_URL")
|
|
keyKey string // config key for the service key ("ZEN_API_KEY" | "ENSO_API_KEY")
|
|
providerFn func() *object.Provider // the virtual provider ai forwards through
|
|
pins []pinnedSKU // SKUs ai guarantees at a known window (applied only when enabled())
|
|
|
|
// decode turns a family's raw GET /v1/models body into discovered SKUs. nil means
|
|
// the Hanzo family wire shape (zenWireModel), which already carries every field ai
|
|
// needs. A family that publishes a DIFFERENT catalog dialect — OpenRouter states
|
|
// price per TOKEN and never states a tier or a funding class — supplies its own
|
|
// translation here, so the dialect is the ONLY thing that varies and discovery,
|
|
// TTL, listing, routing, and gating stay one implementation.
|
|
decode func([]byte) ([]zenModel, error)
|
|
|
|
// discovered catalog — a read-mostly snapshot; discovery failure keeps the last
|
|
// good snapshot so a transient blip never empties ai's model list.
|
|
mu sync.RWMutex
|
|
byID map[string]zenModel
|
|
ids []string
|
|
fetchedAt time.Time
|
|
loaded bool
|
|
}
|
|
|
|
var (
|
|
zenFam = &modelFamily{
|
|
name: "zen", provider: "Zen", prefix: "zen", owner: "zenlm", urlKey: "ZEN_URL", keyKey: "ZEN_API_KEY",
|
|
providerFn: object.ZenProvider,
|
|
pins: []pinnedSKU{
|
|
{id: "zen5", contextWindow: flagshipWindow},
|
|
{id: "zen5-coder", contextWindow: flagshipWindow},
|
|
{id: "zen5-pro", contextWindow: flagshipWindow},
|
|
},
|
|
}
|
|
ensoFam = &modelFamily{
|
|
name: "enso", provider: "Enso", prefix: "enso", owner: "hanzo", urlKey: "ENSO_URL", keyKey: "ENSO_API_KEY",
|
|
providerFn: object.EnsoProvider,
|
|
pins: []pinnedSKU{
|
|
{id: "enso", contextWindow: flagshipWindow},
|
|
{id: "enso-pro", contextWindow: flagshipWindow},
|
|
{id: "enso-ultra", contextWindow: flagshipWindow},
|
|
},
|
|
}
|
|
// modelFamilies is the discovery/route/merge iteration order — the ONE list every
|
|
// generic family helper walks. openrouter is last: the first-party families claim
|
|
// their own SKUs first, and the resale catalog answers for what is left.
|
|
modelFamilies = []*modelFamily{zenFam, ensoFam, openrouterFam}
|
|
)
|
|
|
|
// pinnedWindow returns the context window ai guarantees when listing and routing a
|
|
// SKU, or 0 when the family pins none. Pins apply only to a configured family, so a
|
|
// bare or unconfigured family reports no pinned window (and synthesizes no SKU).
|
|
func (f *modelFamily) pinnedWindow(model string) int {
|
|
if !f.enabled() {
|
|
return 0
|
|
}
|
|
m := strings.ToLower(strings.TrimSpace(model))
|
|
for _, p := range f.pins {
|
|
if strings.ToLower(p.id) == m {
|
|
return p.contextWindow
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (f *modelFamily) baseURL() string {
|
|
return strings.TrimRight(strings.TrimSpace(conf.GetConfigString(f.urlKey)), "/")
|
|
}
|
|
|
|
func (f *modelFamily) serviceKey() string {
|
|
return strings.TrimSpace(conf.GetConfigString(f.keyKey))
|
|
}
|
|
|
|
// enabled reports whether the family service is configured. When it is not, ai serves
|
|
// no model of that family (serves is false) and it is simply absent from /v1/models.
|
|
func (f *modelFamily) enabled() bool { return f.baseURL() != "" }
|
|
|
|
// familyForProviderType maps a virtual provider Type onto its family, resolved from the
|
|
// SAME modelFamilies list that discovery, listing and routing all read.
|
|
//
|
|
// It used to be a hand-written switch, which is a second list that must be kept in step
|
|
// with the first — and it was not. OpenRouter was added to modelFamilies, so its 338
|
|
// prepaid models were discovered, listed and routable, but the switch never named it. Every
|
|
// dispatch site guards on `if fam := familyForProviderType(...); fam != nil`, so a nil sent
|
|
// those requests down the generic relay — around pipeToFamily, which is where the funding
|
|
// gate lives. Listed, served, and ungated, on real cash.
|
|
//
|
|
// Deriving the lookup from the one list makes that shape unrepresentable: a family that
|
|
// exists is a family this resolves, and adding one cannot silently skip the gate.
|
|
func familyForProviderType(t string) *modelFamily {
|
|
for _, f := range modelFamilies {
|
|
if f.provider == t {
|
|
return f
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// familyServing returns the family that owns a model, if any.
|
|
func familyServing(model string) (*modelFamily, bool) {
|
|
for _, f := range modelFamilies {
|
|
if f.serves(model) {
|
|
return f, true
|
|
}
|
|
}
|
|
return nil, false
|
|
}
|
|
|
|
// familyLookup returns the discovered model for an id/alias from whichever family
|
|
// holds it.
|
|
func familyLookup(model string) (zenModel, bool) {
|
|
for _, f := range modelFamilies {
|
|
if m, ok := f.lookup(model); ok {
|
|
return m, true
|
|
}
|
|
}
|
|
return zenModel{}, false
|
|
}
|
|
|
|
// familyLookupFresh is familyLookup that first refreshes each family's catalog on
|
|
// its TTL (a no-op when warm) before reading it. lookup() alone reads the cache
|
|
// with no refresh, so on a replica whose family cache is cold or stale a real SKU
|
|
// reads as unknown — which would make a gated SKU (enso) read as non-gated on a cold
|
|
// replica and skip the access gate. Access/gating decisions must see the same catalog
|
|
// the /v1/models merge does; they go through here (FamilyModelGated).
|
|
func familyLookupFresh(model string) (zenModel, bool) {
|
|
for _, f := range modelFamilies {
|
|
f.fresh()
|
|
if m, ok := f.lookup(model); ok {
|
|
return m, true
|
|
}
|
|
}
|
|
return zenModel{}, false
|
|
}
|
|
|
|
// familyPassthroughRoute routes any family SKU to its family service.
|
|
func familyPassthroughRoute(model string) *modelRoute {
|
|
for _, f := range modelFamilies {
|
|
if r := f.passthroughRoute(model); r != nil {
|
|
return r
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// familyModelPrice returns the discovered headline price for a family model.
|
|
func familyModelPrice(model string) (modelPrice, bool) {
|
|
for _, f := range modelFamilies {
|
|
if p, ok := f.modelPrice(model); ok {
|
|
return p, true
|
|
}
|
|
}
|
|
return modelPrice{}, false
|
|
}
|
|
|
|
// mergeFamilyModels overlays every configured family's discovered lineup onto the
|
|
// base /v1/models list (each SKU wins over a same-named base entry; new SKUs append).
|
|
func mergeFamilyModels(base []modelInfo) []modelInfo {
|
|
for _, f := range modelFamilies {
|
|
base = f.mergeModels(base)
|
|
}
|
|
return base
|
|
}
|
|
|
|
// ── discovered catalog (values, not places) ──────────────────────────────────
|
|
|
|
// zenTier is one context tier's retail price ($/MTok) as EXACT decimals — the same
|
|
// values the family serves, parsed losslessly (never through float).
|
|
type zenTier struct {
|
|
MaxCtx int
|
|
In decimal.Decimal
|
|
Out decimal.Decimal
|
|
}
|
|
|
|
// zenModel is a discovered SKU: what ai needs to list, route, and bill it — the SKU's
|
|
// public contract (id, context, price ladder, vision). Not its upstream, which the
|
|
// family does not disclose (hip-00NN).
|
|
type zenModel struct {
|
|
ID string
|
|
OwnedBy string
|
|
MaxCtx int
|
|
Vision bool
|
|
Access string // "" = generally available; "waitlist" = access-gated (limited preview) — ai enforces the grant
|
|
MinTier string // "" | "free" | "trial" | "paid" — min subscription tier the family advertises for this SKU; ai enforces it (Seams A/B). "" ⇒ free (all tiers). Orthogonal to Access.
|
|
// Funding is how the SKU's usage is PAID FOR upstream: "prepaid" means every path it
|
|
// can take spends a real-cash balance; "" means credits. It is a different KIND of
|
|
// floor from MinTier and is enforced differently — see familyFundingAllowed.
|
|
Funding string
|
|
Base zenTier // headline RETAIL price (the in-window tier)
|
|
Tiers []zenTier // full ladder, ascending by MaxCtx — the billing contract
|
|
|
|
// CostIn/CostOut are the upstream COGS ($/MTok) behind Base, when the family
|
|
// discloses what the SKU costs us. Zero means undisclosed, which modelPrice
|
|
// already reads as cost == price (zero margin) — so a family that states only a
|
|
// price behaves exactly as before. A resale family (OpenRouter) states both, and
|
|
// the retail it publishes is this cost times a margin, never below it.
|
|
CostIn decimal.Decimal
|
|
CostOut decimal.Decimal
|
|
}
|
|
|
|
// gated reports whether the family SKU is access-controlled (a limited-preview SKU
|
|
// that is LISTED but callable only with a granted ModelAccess row). ai learns this
|
|
// from discovery (the family's advertised access), never a hardcode.
|
|
func (m zenModel) gated() bool { return m.Access == "waitlist" }
|
|
|
|
// minTier is the SKU's advertised minimum subscription tier on the enso ladder,
|
|
// normalized: "" ⇒ "free" (every tier may use it). One of "free" | "trial" | "paid".
|
|
// ai learns this from discovery (the family's advertised min_tier), never a hardcode;
|
|
// the tier gate (familyTierAllowed, Seams A/B) enforces it.
|
|
func (m zenModel) minTier() string {
|
|
if t := strings.ToLower(strings.TrimSpace(m.MinTier)); t != "" {
|
|
return t
|
|
}
|
|
return "free"
|
|
}
|
|
|
|
// tierFor picks the tier that serves promptTokens: the smallest whose context covers
|
|
// the prompt, else the largest. This is the family's Tier rule (hip-00NN) — cost keys
|
|
// off the SERVED tier, so a 1M-overflow bills at the pricier tier and never underbills.
|
|
func (m zenModel) tierFor(promptTokens int) zenTier {
|
|
for _, t := range m.Tiers {
|
|
if promptTokens <= t.MaxCtx {
|
|
return t
|
|
}
|
|
}
|
|
if len(m.Tiers) > 0 {
|
|
return m.Tiers[len(m.Tiers)-1]
|
|
}
|
|
return m.Base
|
|
}
|
|
|
|
var zenMillion = decimal.New(1_000_000, 0)
|
|
|
|
// costCents computes exact retail cost for token counts at the served tier and returns
|
|
// it in ai's cent-granular ledger unit. The dollar value is derived the same way the
|
|
// family derives it — money/decimal, no float, no cents flooring mid-way; only the
|
|
// final debit rounds to the cent, with a one-cent floor for any non-zero usage so a
|
|
// served call is never billed zero.
|
|
func (m zenModel) costCents(promptTokens, completionTokens int) int64 {
|
|
t := m.tierFor(promptTokens)
|
|
in := t.In.Mul(decimal.New(int64(promptTokens), 0))
|
|
out := t.Out.Mul(decimal.New(int64(completionTokens), 0))
|
|
usd := in.Add(out).Quo(zenMillion, 18) // exact dollars to 18 dp — identical to the family
|
|
cents := money.New(usd, money.USD).Minor().Int64()
|
|
if cents <= 0 && (promptTokens > 0 || completionTokens > 0) {
|
|
cents = 1
|
|
}
|
|
return cents
|
|
}
|
|
|
|
// price projects the headline tier into ai's legacy per-model price struct (float,
|
|
// used only for the balance-reservation estimate and the /v1/models display). The
|
|
// exact debit path uses costCents, never this projection.
|
|
func (m zenModel) price() (modelPrice, bool) {
|
|
in, _ := strconv.ParseFloat(m.Base.In.String(), 64)
|
|
out, _ := strconv.ParseFloat(m.Base.Out.String(), 64)
|
|
if in <= 0 && out <= 0 {
|
|
return modelPrice{}, false
|
|
}
|
|
costIn, _ := strconv.ParseFloat(m.CostIn.String(), 64)
|
|
costOut, _ := strconv.ParseFloat(m.CostOut.String(), 64)
|
|
return modelPrice{
|
|
InputPerMillion: in, OutputPerMillion: out,
|
|
CostInPerMillion: costIn, CostOutPerMillion: costOut,
|
|
}, true
|
|
}
|
|
|
|
// zenWireModel is the /v1/models item shape a family serves. Prices are JSON strings
|
|
// decoded straight into decimal (decimal.UnmarshalJSON), preserving the exact value.
|
|
// There is deliberately no upstream field to read.
|
|
type zenWireModel struct {
|
|
ID string `json:"id"`
|
|
OwnedBy string `json:"owned_by"`
|
|
Access string `json:"access"` // "" | "waitlist" — access gating advertised by the family
|
|
MinTier string `json:"min_tier"` // "" | "free" | "trial" | "paid" — min subscription tier advertised for this SKU (Seams A/B)
|
|
Funding string `json:"funding"` // "prepaid" = every path this SKU can take spends a real-cash balance
|
|
ContextWindow int `json:"context_window"`
|
|
Pricing struct {
|
|
Input decimal.Decimal `json:"input"`
|
|
Output decimal.Decimal `json:"output"`
|
|
} `json:"pricing"`
|
|
PricingTiers []struct {
|
|
MaxContext int `json:"max_context"`
|
|
Input decimal.Decimal `json:"input"`
|
|
Output decimal.Decimal `json:"output"`
|
|
} `json:"pricing_tiers"`
|
|
Capabilities struct {
|
|
Vision bool `json:"vision"`
|
|
} `json:"capabilities"`
|
|
}
|
|
|
|
func (w zenWireModel) model() zenModel {
|
|
zm := zenModel{
|
|
ID: w.ID, OwnedBy: w.OwnedBy, MaxCtx: w.ContextWindow, Vision: w.Capabilities.Vision, Access: w.Access, MinTier: w.MinTier, Funding: w.Funding,
|
|
Base: zenTier{MaxCtx: w.ContextWindow, In: w.Pricing.Input, Out: w.Pricing.Output},
|
|
}
|
|
for _, t := range w.PricingTiers {
|
|
zm.Tiers = append(zm.Tiers, zenTier{MaxCtx: t.MaxContext, In: t.Input, Out: t.Output})
|
|
}
|
|
if len(zm.Tiers) == 0 {
|
|
zm.Tiers = []zenTier{zm.Base}
|
|
}
|
|
return zm
|
|
}
|
|
|
|
// decodeCatalog translates a family's raw /v1/models body into discovered SKUs using
|
|
// the family's own dialect, defaulting to the Hanzo family wire shape.
|
|
func (f *modelFamily) decodeCatalog(body []byte) ([]zenModel, error) {
|
|
if f.decode != nil {
|
|
return f.decode(body)
|
|
}
|
|
var out struct {
|
|
Data []zenWireModel `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(body, &out); err != nil {
|
|
return nil, err
|
|
}
|
|
models := make([]zenModel, 0, len(out.Data))
|
|
for _, w := range out.Data {
|
|
models = append(models, w.model())
|
|
}
|
|
return models, nil
|
|
}
|
|
|
|
const zenCatalogTTL = 5 * time.Minute
|
|
|
|
var zenDiscoveryClient = &http.Client{Timeout: 15 * time.Second}
|
|
|
|
// refresh fetches GET <family>/v1/models and swaps in a fresh snapshot. Best-effort:
|
|
// on any error the previous snapshot stands.
|
|
func (f *modelFamily) refresh() error {
|
|
base := f.baseURL()
|
|
if base == "" {
|
|
return nil
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/v1/models", nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if k := f.serviceKey(); k != "" {
|
|
req.Header.Set("Authorization", "Bearer "+k)
|
|
}
|
|
resp, err := zenDiscoveryClient.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("%s /v1/models: status %d", f.name, resp.StatusCode)
|
|
}
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
models, err := f.decodeCatalog(body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
byID := make(map[string]zenModel, len(models))
|
|
ids := make([]string, 0, len(models))
|
|
for _, m := range models {
|
|
if strings.TrimSpace(m.ID) == "" {
|
|
continue
|
|
}
|
|
byID[strings.ToLower(m.ID)] = m
|
|
ids = append(ids, m.ID)
|
|
}
|
|
f.mu.Lock()
|
|
f.byID = byID
|
|
f.ids = ids
|
|
f.fetchedAt = time.Now()
|
|
f.loaded = true
|
|
f.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
// fresh refreshes the snapshot if stale (or never loaded) and the family is
|
|
// configured. Errors are logged, not surfaced — a stale snapshot still serves.
|
|
func (f *modelFamily) fresh() {
|
|
if !f.enabled() {
|
|
return
|
|
}
|
|
f.mu.RLock()
|
|
stale := !f.loaded || time.Since(f.fetchedAt) > zenCatalogTTL
|
|
f.mu.RUnlock()
|
|
if stale {
|
|
if err := f.refresh(); err != nil {
|
|
log.Warning("%s catalog refresh failed: %v", f.name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// lookup returns the discovered model for an id/alias, if any.
|
|
func (f *modelFamily) lookup(model string) (zenModel, bool) {
|
|
f.mu.RLock()
|
|
defer f.mu.RUnlock()
|
|
m, ok := f.byID[strings.ToLower(strings.TrimSpace(model))]
|
|
return m, ok
|
|
}
|
|
|
|
// snapshot returns the discovered family (refreshing on a TTL).
|
|
func (f *modelFamily) snapshot() []zenModel {
|
|
f.fresh()
|
|
f.mu.RLock()
|
|
defer f.mu.RUnlock()
|
|
out := make([]zenModel, 0, len(f.ids))
|
|
for _, id := range f.ids {
|
|
if m, ok := f.byID[strings.ToLower(id)]; ok {
|
|
out = append(out, m)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// serves reports whether the family owns this model. True for any SKU the discovered
|
|
// catalog resolves, and for the public brand prefix so a request routes correctly even
|
|
// before the first discovery completes (cold start). The prefix is the public brand,
|
|
// not a confidential mapping. False when the family is not configured.
|
|
func (f *modelFamily) serves(model string) bool {
|
|
if !f.enabled() {
|
|
return false
|
|
}
|
|
f.fresh()
|
|
if _, ok := f.lookup(model); ok {
|
|
return true
|
|
}
|
|
return strings.HasPrefix(strings.ToLower(strings.TrimSpace(model)), f.prefix)
|
|
}
|
|
|
|
// modelPrice returns the discovered headline price for a family model, for the
|
|
// balance-reservation estimate and the model listing. The exact debit uses
|
|
// zenModel.costCents (money/decimal), never this float projection.
|
|
func (f *modelFamily) modelPrice(model string) (modelPrice, bool) {
|
|
m, ok := f.lookup(model)
|
|
if !ok {
|
|
return modelPrice{}, false
|
|
}
|
|
return m.price()
|
|
}
|
|
|
|
// passthroughRoute routes any family model to its family provider, forwarding the SKU
|
|
// name verbatim. ai holds no SKU→upstream mapping: the family resolves the SKU, injects
|
|
// identity, folds reasoning, and picks the upstream (hip-00NN).
|
|
func (f *modelFamily) passthroughRoute(model string) *modelRoute {
|
|
if !f.serves(model) {
|
|
return nil
|
|
}
|
|
ctx := 0
|
|
if m, ok := f.lookup(model); ok {
|
|
ctx = m.MaxCtx
|
|
}
|
|
if w := f.pinnedWindow(model); w > 0 {
|
|
ctx = w // guarantee the flagship served window (and give enso-pro a window before discovery advertises it)
|
|
}
|
|
return &modelRoute{
|
|
providerName: f.name,
|
|
upstreamModel: model, // identity: the family maps the SKU to its real upstream
|
|
premium: true,
|
|
ownedBy: f.owner, // public branding: enso→"hanzo", zen→"zenlm"
|
|
contextWindow: ctx,
|
|
}
|
|
}
|
|
|
|
// mergeModels overlays this family's discovered lineup onto the base /v1/models list:
|
|
// a discovered SKU wins over any same-named base entry, new SKUs are appended, and
|
|
// each carries its served context window (from discovery, overridden by a pinned
|
|
// window for the flagship SKUs). A configured family also lists its pinned SKUs that
|
|
// discovery has not advertised yet (e.g. enso-pro) so clients see the real window.
|
|
func (f *modelFamily) mergeModels(base []modelInfo) []modelInfo {
|
|
zs := f.snapshot()
|
|
// Pinned SKUs are synthesized only for a CONFIGURED family, so a bare or
|
|
// unconfigured family lists nothing it cannot serve.
|
|
pins := f.pins
|
|
if !f.enabled() {
|
|
pins = nil
|
|
}
|
|
if len(zs) == 0 && len(pins) == 0 {
|
|
return base
|
|
}
|
|
now := time.Now().Unix()
|
|
idx := make(map[string]int, len(base))
|
|
for i, m := range base {
|
|
idx[strings.ToLower(m.ID)] = i
|
|
}
|
|
upsert := func(info modelInfo) {
|
|
k := strings.ToLower(info.ID)
|
|
if i, ok := idx[k]; ok {
|
|
base[i] = info
|
|
} else {
|
|
idx[k] = len(base)
|
|
base = append(base, info)
|
|
}
|
|
}
|
|
for _, z := range zs {
|
|
owner := z.OwnedBy
|
|
if owner == "" {
|
|
owner = f.owner // public branding default: enso→"hanzo", zen→"zenlm"
|
|
}
|
|
window := z.MaxCtx
|
|
if w := f.pinnedWindow(z.ID); w > 0 {
|
|
window = w // flagship SKUs report their guaranteed served window
|
|
}
|
|
info := modelInfo{
|
|
ID: z.ID, Object: "model", Created: now, OwnedBy: owner, Premium: true,
|
|
Pricing: pricingInfo(z.price()), ContextWindow: window,
|
|
}
|
|
// A gated SKU is LISTED but access-controlled; advertise the default standing
|
|
// ("waitlist"). ListModels upgrades this to the caller's real status when authed.
|
|
if z.gated() {
|
|
info.Access = &modelAccessInfo{State: "waitlist"}
|
|
}
|
|
upsert(info)
|
|
}
|
|
// List any pinned SKU discovery has not advertised (a SKU ai routes ahead of the
|
|
// family serving it, e.g. enso-pro) with its guaranteed window.
|
|
for _, p := range pins {
|
|
if _, ok := idx[strings.ToLower(p.id)]; ok {
|
|
continue // already listed from discovery (window pinned above)
|
|
}
|
|
upsert(modelInfo{
|
|
ID: p.id, Object: "model", Created: now, OwnedBy: f.owner,
|
|
Premium: true, ContextWindow: p.contextWindow,
|
|
})
|
|
}
|
|
sort.Slice(base, func(i, j int) bool { return base[i].ID < base[j].ID })
|
|
return base
|
|
}
|
|
|
|
// ── the pipe (IO edge) ────────────────────────────────────────────────────────
|
|
|
|
// zenPipeClient forwards inference to a family. No client-level timeout: a streamed 1M
|
|
// request is long by design, and the request context (client disconnect) bounds it. A
|
|
// response-header timeout guards against a dead family without cutting live streams.
|
|
var zenPipeClient = &http.Client{
|
|
Transport: &http.Transport{
|
|
Proxy: http.ProxyFromEnvironment,
|
|
ResponseHeaderTimeout: 120 * time.Second,
|
|
},
|
|
}
|
|
|
|
var (
|
|
zenNewline = []byte("\n")
|
|
zenDataPrefix = []byte("data:")
|
|
)
|
|
|
|
// withModel returns body with its top-level "model" field set to model, preserving
|
|
// every other field. It reconciles the forwarded body with the model ai resolved: an
|
|
// `auto`/`zen-router` request rewrites request.Model to a concrete SKU while the raw
|
|
// body still says "auto", and the family serves by the body's model field. RawMessage
|
|
// preserves all sibling fields losslessly; on any parse/marshal error the body is
|
|
// returned unchanged so the rewrite never drops a request (the family then surfaces a
|
|
// precise error on the original body).
|
|
func withModel(body []byte, model string) []byte {
|
|
var m map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &m); err != nil || m == nil {
|
|
return body
|
|
}
|
|
mb, err := json.Marshal(model)
|
|
if err != nil {
|
|
return body
|
|
}
|
|
m["model"] = mb
|
|
out, err := json.Marshal(m)
|
|
if err != nil {
|
|
return body
|
|
}
|
|
return out
|
|
}
|
|
|
|
// pipeToFamily forwards the caller's request to the family service verbatim and relays
|
|
// the response back unchanged — the family owns identity, reasoning, upstream, and
|
|
// dialect; ai only authenticates, meters, and forwards. apiPath is "messages"
|
|
// (Anthropic) or "chat/completions" (OpenAI). Billing settles from the response usage
|
|
// at the discovered exact retail price; the settle is idempotent with any deferred
|
|
// fail-safe settle at the call site.
|
|
func (c *ApiController) pipeToFamily(fam *modelFamily, apiPath, dialect, model string, rawBody []byte, stream bool, orgId string, authUser *iam.User, isPremium bool, hold *budgetHold, start time.Time) {
|
|
prov := fam.providerFn()
|
|
if prov == nil {
|
|
c.zenError(dialect, fam.name+" service is not configured", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
// Access gate (the ONE choke point): a gated SKU (enso limited preview) is
|
|
// callable only with a granted ModelAccess row. Discovery tells us which models
|
|
// are gated (zenModel.gated), so this is policy learned from data, not a hardcode.
|
|
// The deferred hold.settle(0) at the call site releases the reservation on this
|
|
// early return, so a denied request is never billed.
|
|
if !c.familyAccessAllowed(fam, model, orgId, authUser) {
|
|
c.zenError(dialect, gatedAccessMessage(model), http.StatusForbidden)
|
|
return
|
|
}
|
|
// Forward the model ai RESOLVED, not the caller's raw model id. An
|
|
// `auto`/`zen-router` request rewrote request.Model to a concrete family SKU, but
|
|
// the raw body still carries "model":"auto" — and the family serves by the body's
|
|
// model field, so it 404s on the unknown id "auto" while a direct request for the
|
|
// same SKU (body model == SKU) serves. Reconcile the forwarded body's model with
|
|
// the resolved SKU; byte-identical for a direct request (model already matches).
|
|
rawBody = withModel(rawBody, model)
|
|
reqID := util.GenerateUUID()
|
|
url := prov.ProviderUrl + "/v1/" + apiPath
|
|
req, err := http.NewRequestWithContext(c.Ctx.Request.Context(), http.MethodPost, url, bytes.NewReader(rawBody))
|
|
if err != nil {
|
|
c.zenError(dialect, "build "+fam.name+" request: "+err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
if a := c.Ctx.Request.Header.Get("Accept"); a != "" {
|
|
req.Header.Set("Accept", a)
|
|
}
|
|
if prov.ClientSecret != "" {
|
|
req.Header.Set("Authorization", "Bearer "+prov.ClientSecret)
|
|
}
|
|
// Tenant attribution: the family needs a billable tenant, and ai — which settles
|
|
// the ledger — tells the family it fronts this call so it meters without
|
|
// double-charging.
|
|
if orgId != "" {
|
|
req.Header.Set("X-Org-Id", orgId)
|
|
} else if authUser != nil {
|
|
req.Header.Set("X-Org-Id", authUser.Owner)
|
|
}
|
|
req.Header.Set("X-Hanzo-Fronted-By", "ai")
|
|
|
|
resp, err := zenPipeClient.Do(req)
|
|
if err != nil {
|
|
c.recordFamilyUsage(fam, model, authUser, isPremium, stream, reqID, 0, 0, start, hold, "error", err.Error())
|
|
c.zenError(dialect, fam.name+" request failed: "+err.Error(), http.StatusBadGateway)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
prompt, completion := 0, 0
|
|
served, respID := "", ""
|
|
if stream {
|
|
if resp.StatusCode != http.StatusOK {
|
|
b, _ := io.ReadAll(resp.Body)
|
|
c.zenError(dialect, upstreamErrorMessage(b), resp.StatusCode)
|
|
return
|
|
}
|
|
c.Ctx.ResponseWriter.Header().Set("Content-Type", "text/event-stream")
|
|
c.Ctx.ResponseWriter.Header().Set("Cache-Control", "no-cache")
|
|
c.Ctx.ResponseWriter.Header().Set("Connection", "keep-alive")
|
|
c.Ctx.ResponseWriter.WriteHeader(http.StatusOK)
|
|
prompt, completion, served, respID = c.relayZenStream(resp.Body)
|
|
} else {
|
|
b, rErr := io.ReadAll(resp.Body)
|
|
if rErr != nil {
|
|
c.zenError(dialect, "read "+fam.name+" response: "+rErr.Error(), http.StatusBadGateway)
|
|
return
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
c.zenError(dialect, upstreamErrorMessage(b), resp.StatusCode)
|
|
return
|
|
}
|
|
sniffZenUsage(b, &prompt, &completion)
|
|
served = sniffZenModel(b)
|
|
respID = sniffZenId(b)
|
|
ct := resp.Header.Get("Content-Type")
|
|
if ct == "" {
|
|
ct = "application/json"
|
|
}
|
|
c.Ctx.Output.Header("Content-Type", ct)
|
|
c.Ctx.Output.Body(b)
|
|
}
|
|
|
|
if prompt == 0 {
|
|
prompt = coarseTokenEstimate(rawBody)
|
|
}
|
|
cents := c.recordFamilyUsage(fam, model, authUser, isPremium, stream, reqID, prompt, completion, start, hold, "success", "")
|
|
// Learning ledger: record this served family call (source="family") and, when the
|
|
// engine endpoint is configured, the shadow A/B pick — off the hot path, no prompt
|
|
// text ever stored (the ledger holds none). The join key is the response id the
|
|
// CLIENT sees (respID), which the client threads back to /v1/feedback; falls back
|
|
// to the internal reqID when the family did not disclose one. See object.RoutingEvent.
|
|
c.recordFamilyRouting(model, served, respID, reqID, rawBody, orgId, authUser, prompt, completion, cents, start)
|
|
c.EnableRender = false
|
|
}
|
|
|
|
// relayZenStream copies a family's SSE response to the client verbatim and captures the
|
|
// final usage for billing. The family already emits correct dialect SSE, so ai forwards
|
|
// bytes unchanged — no translation — and only reads usage as it passes.
|
|
func (c *ApiController) relayZenStream(body io.Reader) (prompt, completion int, served, respID string) {
|
|
w := c.Ctx.ResponseWriter
|
|
sc := bufio.NewScanner(body)
|
|
sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
|
|
for sc.Scan() {
|
|
line := sc.Bytes()
|
|
_, _ = w.Write(line)
|
|
_, _ = w.Write(zenNewline)
|
|
w.Flush()
|
|
if bytes.HasPrefix(line, zenDataPrefix) {
|
|
payload := bytes.TrimSpace(line[len(zenDataPrefix):])
|
|
if len(payload) > 0 && payload[0] == '{' {
|
|
sniffZenUsage(payload, &prompt, &completion)
|
|
if served == "" {
|
|
served = sniffZenModel(payload)
|
|
}
|
|
if respID == "" {
|
|
respID = sniffZenId(payload)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// sniffZenModel reads the top-level "model" (or Anthropic's message.model) from a
|
|
// family response payload — the concrete arm/rung that served, when the family
|
|
// discloses it. "" when absent; the caller then falls back to the requested SKU.
|
|
func sniffZenModel(payload []byte) string {
|
|
var p struct {
|
|
Model string `json:"model"`
|
|
Message *struct {
|
|
Model string `json:"model"`
|
|
} `json:"message"`
|
|
}
|
|
if json.Unmarshal(payload, &p) != nil {
|
|
return ""
|
|
}
|
|
if p.Model != "" {
|
|
return p.Model
|
|
}
|
|
if p.Message != nil {
|
|
return p.Message.Model
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// sniffZenId reads the response id the CLIENT sees — OpenAI's top-level "id"
|
|
// (chatcmpl-…) or Anthropic's message id (msg_…, at "id" for messages or nested under
|
|
// "message"). This is the exact id the client threads back to /v1/feedback, so it is
|
|
// the routing-event join key. "" when absent.
|
|
func sniffZenId(payload []byte) string {
|
|
var p struct {
|
|
ID string `json:"id"`
|
|
Message *struct {
|
|
ID string `json:"id"`
|
|
} `json:"message"`
|
|
}
|
|
if json.Unmarshal(payload, &p) != nil {
|
|
return ""
|
|
}
|
|
if p.ID != "" {
|
|
return p.ID
|
|
}
|
|
if p.Message != nil {
|
|
return p.Message.ID
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// sniffZenUsage reads token usage from a family response — an SSE data payload or a
|
|
// full body — trying the Anthropic (input_tokens/output_tokens, possibly nested under
|
|
// message) and OpenAI (prompt_tokens/completion_tokens) shapes. Present fields update
|
|
// the running counts; absent ones leave them, so Anthropic's message_start (input) and
|
|
// message_delta (output) accumulate across events.
|
|
func sniffZenUsage(payload []byte, prompt, completion *int) {
|
|
var p struct {
|
|
Usage *struct {
|
|
InputTokens int `json:"input_tokens"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
} `json:"usage"`
|
|
Message *struct {
|
|
Usage *struct {
|
|
InputTokens int `json:"input_tokens"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
} `json:"usage"`
|
|
} `json:"message"`
|
|
}
|
|
if json.Unmarshal(payload, &p) != nil {
|
|
return
|
|
}
|
|
if u := p.Usage; u != nil {
|
|
if u.InputTokens > 0 {
|
|
*prompt = u.InputTokens
|
|
}
|
|
if u.PromptTokens > 0 {
|
|
*prompt = u.PromptTokens
|
|
}
|
|
if u.OutputTokens > 0 {
|
|
*completion = u.OutputTokens
|
|
}
|
|
if u.CompletionTokens > 0 {
|
|
*completion = u.CompletionTokens
|
|
}
|
|
}
|
|
if p.Message != nil && p.Message.Usage != nil {
|
|
if p.Message.Usage.InputTokens > 0 {
|
|
*prompt = p.Message.Usage.InputTokens
|
|
}
|
|
if p.Message.Usage.OutputTokens > 0 {
|
|
*completion = p.Message.Usage.OutputTokens
|
|
}
|
|
}
|
|
}
|
|
|
|
// coarseTokenEstimate is a last-resort prompt-token estimate (~4 chars/token) so a
|
|
// successful call whose usage ai could not read is still billed, never zero.
|
|
func coarseTokenEstimate(body []byte) int {
|
|
return len(body)/4 + 1
|
|
}
|
|
|
|
// recordFamilyUsage settles the budget hold at the exact discovered price and records
|
|
// the usage + trace. Billing lives in one place for both stream and buffered paths and
|
|
// for every family.
|
|
func (c *ApiController) recordFamilyUsage(fam *modelFamily, model string, authUser *iam.User, isPremium, stream bool, reqID string, prompt, completion int, start time.Time, hold *budgetHold, status, errMsg string) int64 {
|
|
var cents int64
|
|
if status == "success" {
|
|
if zm, ok := fam.lookup(model); ok {
|
|
cents = zm.costCents(prompt, completion)
|
|
} else {
|
|
cents = calculateCostCentsWithCache(model, prompt, completion, 0, 0)
|
|
}
|
|
}
|
|
hold.settle(cents)
|
|
if authUser == nil {
|
|
return cents
|
|
}
|
|
rec := &usageRecord{
|
|
Owner: authUser.Owner, User: authUser.Owner + "/" + authUser.Name, Organization: authUser.Owner,
|
|
Model: model, Provider: fam.name,
|
|
PromptTokens: prompt, CompletionTokens: completion, TotalTokens: prompt + completion,
|
|
Cost: float64(cents) / 100.0, Currency: "USD",
|
|
Premium: isPremium, Stream: stream, Status: status, ErrorMsg: errMsg,
|
|
ClientIP: c.Ctx.Request.RemoteAddr, RequestID: reqID, Account: "hanzo",
|
|
}
|
|
recordUsage(rec)
|
|
recordTrace(c.Ctx.Request.Context(), rec, start)
|
|
return cents
|
|
}
|
|
|
|
// recordFamilyRouting writes the privacy-preserving learning ledger row for a served
|
|
// family call and, when the engine endpoint is configured, the shadow A/B pick. It is
|
|
// fire-and-forget off the request hot path (exactly like AddRoutingEvent's auto-route
|
|
// callers): a collection failure never touches the served response. It NEVER stores
|
|
// prompt text — only token/cost/latency metrics, the served arm, and (for the shadow)
|
|
// the engine's own opaque feature vector + counterfactual model.
|
|
//
|
|
// The request-derived routing inputs (last user text, media flag) are extracted
|
|
// synchronously from rawBody before the goroutine spawns, so nothing reads the beego
|
|
// context off-thread. The shadow /v1/route call is hard-capped by router.DefaultTimeout.
|
|
func (c *ApiController) recordFamilyRouting(model, served, respID, reqID string, rawBody []byte, orgId string, authUser *iam.User, prompt, completion int, cents int64, start time.Time) {
|
|
owner := orgId
|
|
user := ""
|
|
if authUser != nil {
|
|
if owner == "" {
|
|
owner = authUser.Owner
|
|
}
|
|
user = authUser.Owner + "/" + authUser.Name
|
|
}
|
|
// The join key is the response id the client sees; fall back to the internal reqID
|
|
// when the family disclosed none (RecordFamilyRouting normalizes it identically to
|
|
// /v1/feedback so both sides key on the same value).
|
|
responseId := respID
|
|
if strings.TrimSpace(responseId) == "" {
|
|
responseId = reqID
|
|
}
|
|
text, hasMedia := familyRoutingText(rawBody)
|
|
endpoint := ""
|
|
if cfg := GetModelConfig(); cfg != nil {
|
|
endpoint = cfg.RouterClient(nil).Endpoint
|
|
}
|
|
in := object.FamilyRoutingInput{
|
|
Owner: owner, User: user,
|
|
RequestedModel: model, RoutedModel: served, ResponseId: responseId,
|
|
PromptTokens: prompt, CompletionTokens: completion,
|
|
CostCents: cents, LatencyMs: time.Since(start).Milliseconds(),
|
|
RouterEndpoint: endpoint, ShadowText: text,
|
|
ShadowApproxTokens: coarseTokenEstimate(rawBody), ShadowHasMedia: hasMedia,
|
|
}
|
|
// Fire-and-forget off the request hot path — the ONE shared writer (object) is
|
|
// called identically by cloud's embedded-zen meter.
|
|
go object.RecordFamilyRouting(in)
|
|
}
|
|
|
|
// familyRoutingText extracts the last user turn's text and whether the request carries
|
|
// media from a raw chat/completions or messages body — enough for the shadow router to
|
|
// classify, working for both dialects (both use messages[].role/content, content a
|
|
// string or an array of typed parts). It is a pure function of rawBody; it returns no
|
|
// prompt text to any store — only the shadow /v1/route call (which the engine already
|
|
// receives for auto routing) reads it, and only the engine's derived features persist.
|
|
func familyRoutingText(rawBody []byte) (text string, hasMedia bool) {
|
|
var body struct {
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
if json.Unmarshal(rawBody, &body) != nil {
|
|
return "", false
|
|
}
|
|
for i := len(body.Messages) - 1; i >= 0; i-- {
|
|
m := body.Messages[i]
|
|
if m.Role != "user" {
|
|
continue
|
|
}
|
|
var s string
|
|
if json.Unmarshal(m.Content, &s) == nil {
|
|
if s != "" {
|
|
return s, false
|
|
}
|
|
continue
|
|
}
|
|
var parts []struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text"`
|
|
}
|
|
if json.Unmarshal(m.Content, &parts) != nil {
|
|
continue
|
|
}
|
|
var b strings.Builder
|
|
for _, p := range parts {
|
|
if strings.Contains(p.Type, "image") || strings.Contains(p.Type, "video") {
|
|
hasMedia = true
|
|
}
|
|
if p.Text != "" {
|
|
if b.Len() > 0 {
|
|
b.WriteByte('\n')
|
|
}
|
|
b.WriteString(p.Text)
|
|
}
|
|
}
|
|
if b.Len() > 0 || hasMedia {
|
|
return b.String(), hasMedia
|
|
}
|
|
}
|
|
return "", hasMedia
|
|
}
|
|
|
|
// zenError writes a dialect-correct error. Anthropic callers (Claude Code) get the
|
|
// Anthropic error envelope; OpenAI callers get the OpenAI one.
|
|
func (c *ApiController) zenError(dialect, msg string, status int) {
|
|
if dialect == "anthropic" {
|
|
c.respondAnthropicError(anthropicErrorTypeForStatus(status), msg, status)
|
|
return
|
|
}
|
|
body, _ := json.Marshal(map[string]any{
|
|
"error": map[string]any{"message": msg, "type": "api_error", "code": status},
|
|
})
|
|
c.Ctx.ResponseWriter.Header().Set("Content-Type", "application/json")
|
|
c.Ctx.ResponseWriter.WriteHeader(status)
|
|
_, _ = c.Ctx.ResponseWriter.Write(body)
|
|
c.EnableRender = false
|
|
}
|