mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-21 02:36:49 +00:00
439 lines
19 KiB
Go
439 lines
19 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
// public_chat.go — a completion for someone with no account.
|
|
//
|
|
// Every fact a caller normally presents is decided here instead, from facts the
|
|
// request cannot state. The visitor holds no credential and is issued none: they are
|
|
// identified by the address the edge observed, they name no model, and the call is
|
|
// recorded against a reserved org that owns no data.
|
|
//
|
|
// THE CEILING IS THE SWITCH. publicChatDaily is completions per visitor per UTC day
|
|
// and zero — the default — means the lane does not exist. One number, so a deployment
|
|
// cannot arm the lane and forget the bound, which on an unauthenticated inference
|
|
// route is the only mistake that matters. Every other deployment of this binary —
|
|
// every white-label reseller — stays closed until it says otherwise.
|
|
//
|
|
// IT FAILS CLOSED, AND ALONE. The bound is counted in this process and depends on no
|
|
// other service, because a bound that asks something else a question admits the call
|
|
// whenever the answer does not arrive. The host's allowance is read as well, so a
|
|
// visitor's usage lands in the one counter that reports a free tier, but a lane that
|
|
// admitted on its silence would be unbounded exactly when the host is down.
|
|
//
|
|
// A DAY IS SPENT ON ANSWERS. Both counts are read on arrival and raised where the
|
|
// call is served, so a visitor pays for what they got and never for a route that was
|
|
// missing, a vendor that hung, or a pod being rolled.
|
|
//
|
|
// CORS is not decided here. The edge that fronts this binary owns which browser
|
|
// origins may read it, and a second opinion in this file would be a second answer to
|
|
// one question.
|
|
package controllers
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"net"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/zap-proto/zip"
|
|
|
|
"github.com/hanzoai/ai/address"
|
|
"github.com/hanzoai/ai/conf"
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/hanzoai/ai/object"
|
|
)
|
|
|
|
// publicOrg is the account a credential-less call is recorded against. It is not a
|
|
// customer and it is spelled so it can never become one: an IAM org slug starts
|
|
// alphanumeric, so no signup can mint this name and collide with the lane. cloud
|
|
// reserves the same value for the same reason (apps/tenant.Public) and refuses it at
|
|
// the mint of every tenant-scoped read, so nothing filed under it reaches real data.
|
|
const publicOrg = "$public"
|
|
|
|
// publicVisitors bounds how many distinct visitors one day holds at once. The map is
|
|
// keyed by an address a stranger influences, so without a bound it is a memory
|
|
// exhaustion primitive on an endpoint that needs no credential to reach. At the bound
|
|
// the lane stops admitting visitors it has not already seen — turning away a newcomer
|
|
// is a failure this endpoint is allowed to have.
|
|
const publicVisitors = 200_000
|
|
|
|
// publicMaxTokens caps one public answer, for the reason the lane exists at all: an
|
|
// unattributed completion cannot be billed to anyone, so it is bounded instead.
|
|
const publicMaxTokens = 800
|
|
|
|
// publicChatDaily is the ceiling and the switch at once: completions one visitor may
|
|
// take per UTC day. 0 closes the lane.
|
|
func publicChatDaily() int { return conf.GetConfigInt("PUBLIC_CHAT_DAILY") }
|
|
|
|
// publicOpen reports whether this deployment serves the lane at all.
|
|
func publicOpen() bool { return publicChatDaily() > 0 }
|
|
|
|
// ---- the count ------------------------------------------------------------
|
|
|
|
// dayCount is one UTC day of visitor counts. THE DAY IS THE WHOLE MAP, so nothing
|
|
// resets anything: when the day turns the map is dropped and every visitor is new at
|
|
// the same instant. There is no sweep, and no window in which a job has not yet run.
|
|
type dayCount struct {
|
|
mu sync.Mutex
|
|
day string
|
|
seen map[string]int
|
|
}
|
|
|
|
var publicCount = &dayCount{}
|
|
|
|
// spent reports whether visitor has already taken their day.
|
|
//
|
|
// IT READS. Asking costs a visitor nothing and a visitor at the ceiling keeps their
|
|
// count where it is, because refusals are not usage. The call itself is counted where
|
|
// it is answered (recordUsage), so a request that dies short of a model leaves this
|
|
// map untouched.
|
|
//
|
|
// A closed lane and a caller with no address are spent by definition: neither names a
|
|
// visitor this count can hold.
|
|
func (d *dayCount) spent(visitor, day string, limit int) bool {
|
|
if limit <= 0 || visitor == "" {
|
|
return true
|
|
}
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
if d.day != day || d.seen == nil {
|
|
return false // a count from another day is not this day's
|
|
}
|
|
used, known := d.seen[visitor]
|
|
if !known {
|
|
// At the bound there is no room for a newcomer, so the lane stops admitting
|
|
// visitors it has not already seen. Turning away a stranger is a failure this
|
|
// endpoint is allowed to have; growing without limit on an address a stranger
|
|
// picks is not.
|
|
return len(d.seen) >= publicVisitors
|
|
}
|
|
return used >= limit
|
|
}
|
|
|
|
// count records one call the lane SERVED.
|
|
//
|
|
// At the ceiling the count stops climbing, so what a visitor is shown is "5 of 5" and
|
|
// never "37 of 5".
|
|
//
|
|
// WHICH ALSO MEANS THE COUNT DOES NOT REPORT AN OVERSHOOT. Calls in flight for one
|
|
// visitor all read the ceiling before any of them is counted here, so all of them are
|
|
// served — the excess is the visitor's concurrency, bounded by the edge's per-IP flood
|
|
// cap ahead of this and by how long an answer takes. Generous is the direction to err
|
|
// in when the alternative is a stranger short a call they never got, and the size of
|
|
// the generosity is a number to watch rather than one this map can show.
|
|
func (d *dayCount) count(visitor, day string, limit int) {
|
|
if limit <= 0 || visitor == "" {
|
|
return
|
|
}
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
if d.seen == nil || day > d.day {
|
|
// FORWARD ONLY. The day turns when a LATER one arrives, so a clock that steps
|
|
// backwards — two calls straddling midnight, an NTP correction — cannot drop a
|
|
// map every visitor is still counted in. Counting a late call into the day
|
|
// already open is the harmless direction; wiping the open day is not.
|
|
d.day, d.seen = day, make(map[string]int)
|
|
}
|
|
used, known := d.seen[visitor]
|
|
if !known && len(d.seen) >= publicVisitors {
|
|
return
|
|
}
|
|
if used >= limit {
|
|
return
|
|
}
|
|
d.seen[visitor] = used + 1
|
|
}
|
|
|
|
// ---- whose day a call spends ----------------------------------------------
|
|
|
|
// visitorKey addresses the visitor a public call is counted for. The lane is its only
|
|
// writer, and the record of the call is its only reader (usageRecord.bind).
|
|
type visitorKey struct{}
|
|
|
|
// withVisitor returns ctx naming the visitor this call is counted for.
|
|
func withVisitor(ctx context.Context, visitor string) context.Context {
|
|
return context.WithValue(ctx, visitorKey{}, visitor)
|
|
}
|
|
|
|
// visitorOf answers the visitor a call is counted for, or "" for a caller who counts
|
|
// against their own payer — which is everyone except a stranger on this lane. The
|
|
// lane needs its own subject because a visitor's payer collapses to the reserved org,
|
|
// and counting every stranger there would spend one shared day.
|
|
func visitorOf(ctx context.Context) string {
|
|
if ctx == nil {
|
|
return ""
|
|
}
|
|
visitor, _ := ctx.Value(visitorKey{}).(string)
|
|
return visitor
|
|
}
|
|
|
|
// utcDay is the period a count belongs to. One rule, one timezone — a period that
|
|
// moved with the caller would let a traveller take two days' worth in one afternoon.
|
|
func utcDay(t time.Time) string { return t.UTC().Format("2006-01-02") }
|
|
|
|
// ---- who is asking --------------------------------------------------------
|
|
|
|
// Visitor is who a caller this estate cannot name is, and the composition is the
|
|
// whole of it: the address, then a digest of it. Empty when the request carries no
|
|
// address at all — a socket peer with nothing in front of it naming one.
|
|
//
|
|
// It is exported because the router's ceilings ask the same question this lane does.
|
|
// Two derivations of "who is this, roughly" would be two answers a caller could be
|
|
// on either side of: bounded here, unbounded there, and one of them wrong.
|
|
func Visitor(c *zip.Ctx) string {
|
|
return publicVisitor(publicAddr(c.Fiber().IP(), (&ApiController{Ctx: c}).stated()))
|
|
}
|
|
|
|
// publicVisitor is the only identity an anonymous caller has: a digest of the address
|
|
// the edge observed.
|
|
//
|
|
// THE FORWARDED ADDRESS IS BELIEVED ONLY FROM INSIDE. A caller sets X-Forwarded-For
|
|
// and CF-Connecting-IP freely, so trusting either from a public peer hands every
|
|
// request a fresh quota and the ceiling stops existing. The socket peer decides:
|
|
// reached through our own ingress the peer is private and the edge's header names the
|
|
// visitor; reached directly the peer IS the visitor and the header is ignored. A
|
|
// deployment with no edge in front therefore counts by socket — stricter, never looser.
|
|
//
|
|
// The address is hashed and never kept. It travels into a log line and a usage row,
|
|
// and an address in either is a record of who visited that nothing here needs.
|
|
func publicVisitor(addr string) string {
|
|
if addr == "" {
|
|
return ""
|
|
}
|
|
sum := sha256.Sum256([]byte(addr))
|
|
return "visitor:" + hex.EncodeToString(sum[:])[:32]
|
|
}
|
|
|
|
// publicAddr is the address a public visitor arrived from: the peer, unless the peer
|
|
// is one of our own and something in front of us named the real one.
|
|
//
|
|
// It takes the two values rather than a request because that is all it ever read.
|
|
// THE PEER DECIDES WHETHER THE STATED ADDRESS IS BELIEVED, and that is the whole
|
|
// safety property: a stranger who reached us directly has a routable peer, so their
|
|
// own claim about their address is ignored. Only a peer of ours — a socket, an
|
|
// in-cluster hop — means the address arrived from a layer entitled to state it.
|
|
func publicAddr(peer, stated string) string {
|
|
peer = strings.TrimSpace(peer)
|
|
if peer != "" && !internalAddr(peer) {
|
|
return peer
|
|
}
|
|
if stated := strings.TrimSpace(stated); stated != "" {
|
|
return stated
|
|
}
|
|
// Neither a routable peer nor a stated address: everyone who arrives this way
|
|
// shares one count, and the lane closes for all of them once it is spent.
|
|
return peer
|
|
}
|
|
|
|
// stated is who a layer in front of this process says is calling.
|
|
//
|
|
// TWO DEPLOYMENTS, ONE READ. As a subsystem the peer is the unix socket the host
|
|
// reached us over — empty, and identical for everyone — so the host resolves the
|
|
// caller and stamps it (address.Header). Served directly, nothing is in front
|
|
// but Cloudflare, which overwrites CF-Connecting-IP at its edge; X-Forwarded-For
|
|
// merely gains an entry and keeps whatever the caller put in front of it, so that
|
|
// one is never read.
|
|
//
|
|
// The host's answer wins where both are present: it is the hardened one, derived
|
|
// from the connection it can actually see.
|
|
func (c *ApiController) stated() string {
|
|
if addr := c.Header(address.Header); addr != "" {
|
|
return addr
|
|
}
|
|
return c.Header("CF-Connecting-IP")
|
|
}
|
|
|
|
// internalAddr reports whether addr is one of our own hops rather than a caller.
|
|
// Loopback and private space is what an in-cluster ingress presents; anything
|
|
// routable is a stranger, whatever it writes in a header.
|
|
func internalAddr(addr string) bool {
|
|
ip := net.ParseIP(addr)
|
|
if ip == nil {
|
|
return false
|
|
}
|
|
return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsUnspecified()
|
|
}
|
|
|
|
// ---- the lane -------------------------------------------------------------
|
|
|
|
// ChatCompletionsPublic serves one completion to a caller with no account.
|
|
//
|
|
// @Title ChatCompletionsPublic
|
|
// @Tag OpenAI Compatible API
|
|
// @Description Anonymous completion on the free pool. No credential is presented and
|
|
// none is issued; the model is the platform's free route and cannot be chosen.
|
|
// @Param body body openai.ChatCompletionRequest true "messages; any model field is ignored"
|
|
// @Success 200 {object} openai.ChatCompletionResponse
|
|
// @router /chat/public [post]
|
|
func (c *ApiController) ChatCompletionsPublic() {
|
|
if !publicOpen() {
|
|
// A closed deployment does not advertise the lane: there is nothing here to
|
|
// describe, so the honest answer is that the route does not exist.
|
|
c.publicRefuse(http.StatusNotFound, "invalid_request_error", "public_lane_closed",
|
|
"This deployment does not serve anonymous completions.")
|
|
return
|
|
}
|
|
|
|
visitor := Visitor(c.Ctx)
|
|
if visitor == "" {
|
|
c.publicRefuse(http.StatusForbidden, "invalid_request_error", "public_no_address",
|
|
"This request carries no address to count against.")
|
|
return
|
|
}
|
|
|
|
// A LANE THAT CANNOT ANSWER SAYS SO PLAINLY. The pool is asked whether it has a
|
|
// route before the call goes any further, so a visitor arriving during a
|
|
// misconfiguration reads "the free pool is down" rather than an authorization
|
|
// error thrown by provider setup. It is two map lookups, and it is the same
|
|
// question resolveProviderForPublic asks below, through the same function, so the
|
|
// two cannot drift.
|
|
if _, err := publicPoolRoute(); err != nil {
|
|
c.publicRefuse(http.StatusServiceUnavailable, "server_error", "public_pool_unavailable",
|
|
"The free pool is not available right now. Nothing was counted against you; try again.")
|
|
return
|
|
}
|
|
|
|
// BOTH COUNTS ARE READ HERE AND TAKEN NOWHERE. A day is spent on answers, so the
|
|
// call is counted where it is answered — recordUsage, which fills an allowance only
|
|
// for a call that came back — and this asks only whether the visitor is already
|
|
// out. A request that dies short of a model leaves them every call they arrived
|
|
// with.
|
|
//
|
|
// OUR OWN BOUND FIRST, and it decides. It is kept in this process and asks
|
|
// nothing, so it holds while anything else is down.
|
|
if publicCount.spent(visitor, utcDay(time.Now()), publicChatDaily()) {
|
|
c.publicSpent(visitor, "count")
|
|
return
|
|
}
|
|
|
|
// The host's allowance is the counter that REPORTS a free tier, read under a
|
|
// subject of the visitor's own — one shared subject would let a single caller
|
|
// starve every visitor's tier. It may refuse; it may not admit, which is why its
|
|
// silence is not consulted.
|
|
if spent := object.Spent(); spent != nil {
|
|
if out, err := spent(c.Context(), visitor, publicOrg); err == nil && out {
|
|
c.publicSpent(visitor, "allowance")
|
|
return
|
|
}
|
|
}
|
|
|
|
// The visitor rides on the request, because the record of the call is what counts
|
|
// it and the record has no other way to know which stranger this was.
|
|
c.SetContext(withVisitor(c.Context(), visitor))
|
|
|
|
c.chatCompletions(callerPublic, nil)
|
|
}
|
|
|
|
// publicSpent refuses a visitor who has taken their day.
|
|
func (c *ApiController) publicSpent(visitor, by string) {
|
|
log.Info("public_chat: allowance spent visitor=%s by=%s", visitor, by)
|
|
c.publicRefuse(http.StatusPaymentRequired, "insufficient_quota", "public_allowance_spent",
|
|
"You've used today's free messages. They reset at midnight UTC — sign in at https://hanzo.ai to keep going.")
|
|
}
|
|
|
|
// publicRefuse writes the house error envelope — the shape the balance gate, the
|
|
// rolling cap and every auth refusal on this surface already answer with, so a client
|
|
// reads one error and not a fourth dialect of one.
|
|
func (c *ApiController) publicRefuse(status int, kind, code, message string) {
|
|
c.SetHeader("Content-Type", "application/json")
|
|
c.SetHeader("Cache-Control", "no-store")
|
|
c.Bytes(status, publicErrorJSON(kind, code, message))
|
|
}
|
|
|
|
// publicErrorJSON renders the house error envelope.
|
|
func publicErrorJSON(kind, code, message string) []byte {
|
|
var e struct {
|
|
Error struct {
|
|
Message string `json:"message"`
|
|
Type string `json:"type"`
|
|
Code string `json:"code"`
|
|
} `json:"error"`
|
|
}
|
|
e.Error.Message, e.Error.Type, e.Error.Code = message, kind, code
|
|
b, err := json.Marshal(e)
|
|
if err != nil {
|
|
return []byte(`{"error":{"message":"public lane refused the request","type":"invalid_request_error","code":"public_lane_closed"}}`)
|
|
}
|
|
return b
|
|
}
|
|
|
|
// ---- who serves it --------------------------------------------------------
|
|
|
|
// publicPoolRoute answers whether the free pool can serve a call right now, and with
|
|
// what. It is ONE definition asked twice — once when the lane decides it can answer at
|
|
// all, once when the call is actually resolved — so the question that opens the lane
|
|
// and the question that picks the upstream can never drift apart.
|
|
//
|
|
// It fails closed on both halves. A price that is not FOUND to be zero closes the lane
|
|
// rather than guessing, and a pool with no route closes it rather than reaching for
|
|
// something billable.
|
|
//
|
|
// NO ORG GOES INTO ROUTE SELECTION, and the empty argument is the whole point.
|
|
// resolveModelRouteForOrg degrades a route when the caller's org yields a subject: that
|
|
// is the AUTO-ROUTER's preference gate, which walks a caller down to the next servable
|
|
// SKU. This lane is the direct path resolving its ONE authoritative route, and the
|
|
// resolver says so itself — "the direct call path (which resolves its authoritative
|
|
// route with orgId "" ⇒ empty subject ⇒ admit) is never degraded here". Passing $public
|
|
// degraded it to nil, and a direct path with no route left is not a downgrade, it is a
|
|
// lane that answers nothing. It refused for a reason that cannot apply: the funding
|
|
// gate asks the catalog what a family SKU costs, and the pool's id is an ALIAS
|
|
// rather than a discovered SKU, so the lookup misses and an undescribed id is refused
|
|
// by design. The pool IS the routes that cost nothing; there is no spend to protect.
|
|
func publicPoolRoute() (*modelRoute, error) {
|
|
if !ModelCostsNothing(freeID, publicOrg) {
|
|
return nil, authError("the free pool is not currently free, so the public lane is closed")
|
|
}
|
|
route := resolveFreeRoute(freeID, "")
|
|
if route == nil {
|
|
return nil, authError("the free pool has no route on this deployment")
|
|
}
|
|
return route, nil
|
|
}
|
|
|
|
// resolveFreeRoute is the route resolver, as a seam. The ARGUMENT is what this lane
|
|
// shipped wrong, so the argument is the thing worth pinning — a test can hold "no org
|
|
// reaches route selection" without standing up family discovery, which no unit test
|
|
// has.
|
|
var resolveFreeRoute = resolveModelRouteForOrg
|
|
|
|
// resolveProviderForPublic settles the model, the upstream that runs it and the
|
|
// account it is recorded against — with no credential, and without reading the request.
|
|
//
|
|
// THE MODEL IS ASSIGNED, NEVER CHOSEN. freeID is the platform's own name for the free
|
|
// pool, and publicPoolRoute has verified it costs nothing before it is served. A pool
|
|
// that stopped being free — a catalog edit, a price that failed to load — closes the
|
|
// lane rather than quietly spending on the house, which is the fail-closed rule the
|
|
// balance gate already keeps one layer up.
|
|
func (c *ApiController) resolveProviderForPublic() (provider *object.Provider, authUser *iam.User, upstream string, err error) {
|
|
route, err := publicPoolRoute()
|
|
if err != nil {
|
|
return nil, nil, "", err
|
|
}
|
|
provider, perr := object.GetModelProviderByName(route.providerName)
|
|
if perr != nil || provider == nil {
|
|
return nil, nil, "", authError("the free pool's provider is unavailable")
|
|
}
|
|
// Type "application" marks a machine, so account.Payer resolves the billing
|
|
// subject to the org rather than looking for a person who does not exist.
|
|
authUser = &iam.User{Owner: publicOrg, Type: "application"}
|
|
c.Locals("recordUserId", publicOrg+"/visitor")
|
|
return provider, authUser, route.upstreamModel, nil
|
|
}
|