mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-21 11:09:56 +00:00
Deep-review fixes on the per-org router config: 1. ALLOWLIST LEAK: RouteDecisionFor's heuristic last resort did ForTask(task, nil), relaxing the whole Known predicate — which now carries the enabled-models allowlist. So if an org's allowlist excluded every model in its prefer table for a task, the fallback routed to a DISABLED model. Add a separate Client.Allow (the hard allowlist) that the last resort relaxes servability against but NEVER the allowlist: ForTask(task, ec.Allow). nil Allow = no allowlist = historical relax-all. Test pins it. 2. DEAD-END: if the allowlist admits no model the prefer table offers at all, firstAllowedServableModel routes to the org's first servable allowlisted model instead of dead-ending auto — an allowlist always resolves to one of ITS OWN, never empty. 3. UNOWNED-FIELD CLOBBER: UpdateRouterPolicy built a fresh row + hand-listed the fields to preserve, which missed OrgSettings fields main added later (RouterOverrides, Judge*) — a router-policy write ZEROED them. Mutate the existing row instead, touching only the four owned fields, so every other field is preserved by construction (future-proof).
255 lines
9.8 KiB
Go
255 lines
9.8 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"math/rand"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// DefaultTimeout is the hard cap on the engine round-trip. Routing must add
|
|
// negligible latency to a completion, so a slow/unreachable engine falls through
|
|
// to the local heuristic well within budget.
|
|
const DefaultTimeout = 150 * time.Millisecond
|
|
|
|
// Slo is the per-request service-level objective forwarded to the engine. `0`
|
|
// fields disable the corresponding ceiling. It mirrors hanzo-router's Slo (the
|
|
// operator's budget for this request), distinct from a user's learned taste.
|
|
type Slo struct {
|
|
MaxCost float64 `json:"max_cost"` // cost cap, USD PER 1K TOKENS (per-1k); 0 disables
|
|
MaxLatencyMs int `json:"max_latency_ms"` // latency cap, milliseconds; 0 disables
|
|
}
|
|
|
|
// Client resolves a Request to a concrete model id. Strategy order:
|
|
// 1. engine — POST {Endpoint}/route and use its choice (a learned zen-router
|
|
// served OpenAI-compatibly by hanzo-engine); hard-capped by Timeout.
|
|
// 2. heuristic — Classify + Policy, a pure-Go port of the Rust rule router.
|
|
//
|
|
// Any engine error (unreachable, timeout, non-200, bad body, unservable choice)
|
|
// falls through to the heuristic, so `auto` always resolves with zero extra
|
|
// infrastructure.
|
|
type Client struct {
|
|
// Endpoint is the zen-router base URL (e.g. http://engine.hanzo.svc:3000).
|
|
// Empty disables the engine strategy — heuristic only.
|
|
Endpoint string
|
|
// Timeout hard-caps the engine round-trip (default DefaultTimeout).
|
|
Timeout time.Duration
|
|
// Policy is the heuristic task→model table (and the map for an engine reply
|
|
// that returns only a task).
|
|
Policy Policy
|
|
// HTTP is the client used for the engine call (default http.DefaultClient).
|
|
HTTP *http.Client
|
|
// Known, if set, restricts choices to models the caller can serve; a choice
|
|
// it rejects is skipped. nil accepts all.
|
|
Known func(string) bool
|
|
// Allow, if set, is the HARD allowlist (the org's enabled-models set) — a
|
|
// constraint the heuristic's last-resort fallback must NOT relax, unlike Known
|
|
// (servability), which it does relax to avoid dead-ending `auto`. So a model the
|
|
// org has DISABLED is never routed to, even when no servable preferred model
|
|
// remains; only servability is relaxed as a last resort, never the allowlist.
|
|
// nil = no allowlist (the last resort relaxes everything, the historical behavior).
|
|
Allow func(string) bool
|
|
// Explore is the epsilon exploration floor in [0,1]: the probability the
|
|
// heuristic samples a RANDOM servable NON-champion arm instead of the champion,
|
|
// so the bandit keeps collecting reward on the whole pool (and reacts to newly
|
|
// added models) rather than collapsing onto today's champion — the reward from
|
|
// an explore request trains it exactly like an exploit request. 0 = pure
|
|
// exploit. The engine strategy does its own UCB exploration, so this governs the
|
|
// heuristic path.
|
|
Explore float64
|
|
// Rand, if set, sources the exploration coin flip (injectable for deterministic
|
|
// tests); nil uses the auto-seeded package generator.
|
|
Rand *rand.Rand
|
|
}
|
|
|
|
// engineRequest is the /route wire contract sent to the engine.
|
|
type engineRequest struct {
|
|
Prompt string `json:"prompt"`
|
|
Tasks []string `json:"tasks,omitempty"`
|
|
Slo Slo `json:"slo"`
|
|
}
|
|
|
|
// engineResponse is the /route wire contract returned by the engine. Features is
|
|
// the OPTIONAL frozen-backbone embedding the engine may return alongside its
|
|
// choice; it is tolerated when absent and passed through to the routing ledger
|
|
// for training (never prompt text).
|
|
type engineResponse struct {
|
|
Model string `json:"model"`
|
|
Task string `json:"task"`
|
|
Confidence float64 `json:"confidence"`
|
|
Features []float64 `json:"features,omitempty"`
|
|
}
|
|
|
|
// Source labels which strategy produced a Decision.
|
|
const (
|
|
SourceEngine = "engine"
|
|
SourceHeuristic = "heuristic"
|
|
SourceExplore = "explore" // an epsilon-floor sample of a non-champion arm
|
|
)
|
|
|
|
// Decision is the full outcome of resolving a request: the chosen model, the
|
|
// task it was classified as, the engine confidence (0 for the heuristic), the
|
|
// Source that produced it, and the engine's optional feature vector. It carries
|
|
// exactly what the routing ledger records — no prompt text.
|
|
type Decision struct {
|
|
Model string
|
|
Task Task
|
|
Confidence float64
|
|
Source string
|
|
Features []float64
|
|
}
|
|
|
|
// Route resolves req to a concrete model id and the task it was classified as.
|
|
// It never errors: on any engine failure it returns the heuristic decision. It
|
|
// is a thin projection of RouteDecision for callers that only need the id.
|
|
func (c Client) Route(ctx context.Context, req Request, slo Slo) (model string, task Task) {
|
|
d := c.RouteDecision(ctx, req, slo)
|
|
return d.Model, d.Task
|
|
}
|
|
|
|
// RouteDecision resolves req to a full Decision (model, task, confidence, source,
|
|
// features) under the default policy — the zero RoutingPolicy, which reproduces the
|
|
// historical behavior (enso-leaning: the engine when configured, else the heuristic
|
|
// floor). It never errors. This is the zero-policy shortcut for RouteDecisionFor;
|
|
// callers holding a per-org policy call RouteDecisionFor directly.
|
|
func (c Client) RouteDecision(ctx context.Context, req Request, slo Slo) Decision {
|
|
return c.RouteDecisionFor(ctx, req, slo, RoutingPolicy{})
|
|
}
|
|
|
|
// exploreArm returns a uniformly-random NON-champion servable arm for the task with
|
|
// probability Explore, or (,false) to exploit. It needs ≥2 servable arms — with one
|
|
// (or none) there is nothing to explore toward, so it always exploits.
|
|
func (c Client) exploreArm(t Task) (string, bool) {
|
|
f, ni := rand.Float64, rand.Intn
|
|
if c.Rand != nil {
|
|
f, ni = c.Rand.Float64, c.Rand.Intn
|
|
}
|
|
if f() >= c.Explore {
|
|
return "", false
|
|
}
|
|
cands := c.Policy.Candidates(t, c.Known)
|
|
if len(cands) < 2 {
|
|
return "", false
|
|
}
|
|
return cands[1+ni(len(cands)-1)], true // index 0 is the champion; sample [1:]
|
|
}
|
|
|
|
// ObserveRequest is the online-learning feedback the gateway posts to the engine
|
|
// after a reward is joined: the request's feature vector `x` (the same vector the
|
|
// engine returned at decision time), the arm that served, the caller (keys the
|
|
// per-user LinUCB bandit), the caller org (selects the scope), and the outcome reward
|
|
// in [0,1]. No prompt text.
|
|
type ObserveRequest struct {
|
|
User string `json:"user"`
|
|
Org string `json:"org"`
|
|
Features []float64 `json:"features"`
|
|
Model string `json:"model"`
|
|
Reward float64 `json:"reward"`
|
|
}
|
|
|
|
// Observe forwards a joined reward to the engine's /route/observe endpoint so the
|
|
// learned policy updates per-user theta in-process (the fast online loop). It is
|
|
// best-effort and fire-and-forget by contract: any error (no endpoint, unreachable,
|
|
// non-2xx) is ignored — a missed online update never fails the reward write, and the
|
|
// offline corpus (the rewarded ledger) still captures the signal. Hard-capped by
|
|
// Timeout so it never hangs the caller's goroutine.
|
|
func (c Client) Observe(ctx context.Context, obs ObserveRequest) {
|
|
if c.Endpoint == "" || len(obs.Features) == 0 || obs.Model == "" {
|
|
return
|
|
}
|
|
timeout := c.Timeout
|
|
if timeout <= 0 {
|
|
timeout = DefaultTimeout
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
body, err := json.Marshal(obs)
|
|
if err != nil {
|
|
return
|
|
}
|
|
url := strings.TrimRight(c.Endpoint, "/") + "/route/observe"
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
|
if err != nil {
|
|
return
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
cli := c.HTTP
|
|
if cli == nil {
|
|
cli = http.DefaultClient
|
|
}
|
|
resp, err := cli.Do(req)
|
|
if err != nil {
|
|
return
|
|
}
|
|
_ = resp.Body.Close()
|
|
}
|
|
|
|
// routeEngine performs the constrained /route call. Returns ok=false on any
|
|
// error so the caller falls back to the heuristic.
|
|
func (c Client) routeEngine(ctx context.Context, req Request, slo Slo) (Decision, bool) {
|
|
timeout := c.Timeout
|
|
if timeout <= 0 {
|
|
timeout = DefaultTimeout
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
|
|
body, err := json.Marshal(engineRequest{Prompt: req.Text, Slo: slo})
|
|
if err != nil {
|
|
return Decision{}, false
|
|
}
|
|
url := strings.TrimRight(c.Endpoint, "/") + "/route"
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
|
if err != nil {
|
|
return Decision{}, false
|
|
}
|
|
httpReq.Header.Set("Content-Type", "application/json")
|
|
|
|
cli := c.HTTP
|
|
if cli == nil {
|
|
cli = http.DefaultClient
|
|
}
|
|
resp, err := cli.Do(httpReq)
|
|
if err != nil {
|
|
return Decision{}, false
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return Decision{}, false
|
|
}
|
|
var out engineResponse
|
|
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&out); err != nil {
|
|
return Decision{}, false
|
|
}
|
|
|
|
// Prefer an explicit, servable model; else map the returned task via the
|
|
// policy table (an engine that emits only a task label).
|
|
if out.Model != "" && (c.Known == nil || c.Known(out.Model)) {
|
|
return Decision{Model: out.Model, Task: Task(out.Task), Confidence: out.Confidence, Source: SourceEngine, Features: out.Features}, true
|
|
}
|
|
if out.Task != "" {
|
|
if m := c.Policy.ForTask(Task(out.Task), c.Known); m != "" {
|
|
return Decision{Model: m, Task: Task(out.Task), Confidence: out.Confidence, Source: SourceEngine, Features: out.Features}, true
|
|
}
|
|
}
|
|
return Decision{}, false
|
|
}
|