mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
The flywheel has no separate ML engine to route through — the trainer
retrains the prefer table instead — so `engine_share` counted consultations
of a decision service that was never built. It read 0 on every day of its
existence and told the dashboards the learned router always loses.
Replace it with what actually learns:
learned_share share of decisions whose served model is the pick of the
TUNED prefer table (trainer-published arms + admin pins,
above the static conf seed)
by_source the decision-provenance histogram behind that ratio, so a
strategy that NEVER fires is visible at a glance instead
of being inferred from a share stuck at zero
Both engine and judge swallowed every failure and returned a bare
false, which is how an unset ROUTER_ENDPOINT and a judge with no service
bearer stayed invisible while the reward ledger went empty and the
trainer's gate never cleared. Each failure path now routes through a
counter (router.Misses, controllers.JudgeMisses) and a rate-limited line
carrying the reason, and an armed judge with no token says so at boot.
Judge reasons stay content-free — the no-retention guarantee covers logs.
Co-authored-by: Hanzo Dev <[email protected]>
288 lines
11 KiB
Go
288 lines
11 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"math/rand"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/log"
|
|
)
|
|
|
|
// DefaultTimeout is the hard cap on the engine round-trip. Routing must add
|
|
// negligible latency to a completion, so a slow/unreachable engine falls through
|
|
// to the local heuristic well within budget.
|
|
const DefaultTimeout = 150 * time.Millisecond
|
|
|
|
// Slo is the per-request service-level objective forwarded to the engine. `0`
|
|
// fields disable the corresponding ceiling. It mirrors hanzo-router's Slo (the
|
|
// operator's budget for this request), distinct from a user's learned taste.
|
|
type Slo struct {
|
|
MaxCost float64 `json:"max_cost"` // cost cap, USD PER 1K TOKENS (per-1k); 0 disables
|
|
MaxLatencyMs int `json:"max_latency_ms"` // latency cap, milliseconds; 0 disables
|
|
}
|
|
|
|
// Client resolves a Request to a concrete model id. Strategy order:
|
|
// 1. engine — POST {Endpoint}/route and use its choice (a learned zen-router
|
|
// served OpenAI-compatibly by hanzo-engine); hard-capped by Timeout.
|
|
// 2. heuristic — Classify + Policy, a pure-Go port of the Rust rule router.
|
|
//
|
|
// Any engine error (unreachable, timeout, non-200, bad body, unservable choice)
|
|
// falls through to the heuristic, so `auto` always resolves with zero extra
|
|
// infrastructure.
|
|
type Client struct {
|
|
// Endpoint is the zen-router base URL (e.g. http://engine.hanzo.svc:3000).
|
|
// Empty disables the engine strategy — heuristic only.
|
|
Endpoint string
|
|
// Timeout hard-caps the engine round-trip (default DefaultTimeout).
|
|
Timeout time.Duration
|
|
// Policy is the heuristic task→model table (and the map for an engine reply
|
|
// that returns only a task).
|
|
Policy Policy
|
|
// HTTP is the client used for the engine call (default http.DefaultClient).
|
|
HTTP *http.Client
|
|
// Known, if set, restricts choices to models the caller can serve; a choice
|
|
// it rejects is skipped. nil accepts all.
|
|
Known func(string) bool
|
|
// Allow, if set, is the HARD allowlist (the org's enabled-models set) — a
|
|
// constraint the heuristic's last-resort fallback must NOT relax, unlike Known
|
|
// (servability), which it does relax to avoid dead-ending `auto`. So a model the
|
|
// org has DISABLED is never routed to, even when no servable preferred model
|
|
// remains; only servability is relaxed as a last resort, never the allowlist.
|
|
// nil = no allowlist (the last resort relaxes everything, the historical behavior).
|
|
Allow func(string) bool
|
|
// Explore is the epsilon exploration floor in [0,1]: the probability the
|
|
// heuristic samples a RANDOM servable NON-champion arm instead of the champion,
|
|
// so the bandit keeps collecting reward on the whole pool (and reacts to newly
|
|
// added models) rather than collapsing onto today's champion — the reward from
|
|
// an explore request trains it exactly like an exploit request. 0 = pure
|
|
// exploit. The engine strategy does its own UCB exploration, so this governs the
|
|
// heuristic path.
|
|
Explore float64
|
|
// Rand, if set, sources the exploration coin flip (injectable for deterministic
|
|
// tests); nil uses the auto-seeded package generator.
|
|
Rand *rand.Rand
|
|
}
|
|
|
|
// engineRequest is the /route wire contract sent to the engine.
|
|
type engineRequest struct {
|
|
Prompt string `json:"prompt"`
|
|
Tasks []string `json:"tasks,omitempty"`
|
|
Slo Slo `json:"slo"`
|
|
}
|
|
|
|
// engineResponse is the /route wire contract returned by the engine. Features is
|
|
// the OPTIONAL frozen-backbone embedding the engine may return alongside its
|
|
// choice; it is tolerated when absent and passed through to the routing ledger
|
|
// for training (never prompt text).
|
|
type engineResponse struct {
|
|
Model string `json:"model"`
|
|
Task string `json:"task"`
|
|
Confidence float64 `json:"confidence"`
|
|
Features []float64 `json:"features,omitempty"`
|
|
}
|
|
|
|
// Source labels which strategy produced a Decision.
|
|
const (
|
|
SourceEngine = "engine"
|
|
SourceHeuristic = "heuristic"
|
|
SourceExplore = "explore" // an epsilon-floor sample of a non-champion arm
|
|
)
|
|
|
|
// Decision is the full outcome of resolving a request: the chosen model, the
|
|
// task it was classified as, the engine confidence (0 for the heuristic), the
|
|
// Source that produced it, and the engine's optional feature vector. It carries
|
|
// exactly what the routing ledger records — no prompt text.
|
|
type Decision struct {
|
|
Model string
|
|
Task Task
|
|
Confidence float64
|
|
Source string
|
|
Features []float64
|
|
}
|
|
|
|
// Route resolves req to a concrete model id and the task it was classified as.
|
|
// It never errors: on any engine failure it returns the heuristic decision. It
|
|
// is a thin projection of RouteDecision for callers that only need the id.
|
|
func (c Client) Route(ctx context.Context, req Request, slo Slo) (model string, task Task) {
|
|
d := c.RouteDecision(ctx, req, slo)
|
|
return d.Model, d.Task
|
|
}
|
|
|
|
// RouteDecision resolves req to a full Decision (model, task, confidence, source,
|
|
// features) under the default policy — the zero RoutingPolicy, which reproduces the
|
|
// historical behavior (enso-leaning: the engine when configured, else the heuristic
|
|
// floor). It never errors. This is the zero-policy shortcut for RouteDecisionFor;
|
|
// callers holding a per-org policy call RouteDecisionFor directly.
|
|
func (c Client) RouteDecision(ctx context.Context, req Request, slo Slo) Decision {
|
|
return c.RouteDecisionFor(ctx, req, slo, RoutingPolicy{})
|
|
}
|
|
|
|
// exploreArm returns a uniformly-random NON-champion servable arm for the task with
|
|
// probability Explore, or (,false) to exploit. It needs ≥2 servable arms — with one
|
|
// (or none) there is nothing to explore toward, so it always exploits.
|
|
func (c Client) exploreArm(t Task) (string, bool) {
|
|
f, ni := rand.Float64, rand.Intn
|
|
if c.Rand != nil {
|
|
f, ni = c.Rand.Float64, c.Rand.Intn
|
|
}
|
|
if f() >= c.Explore {
|
|
return "", false
|
|
}
|
|
cands := c.Policy.Candidates(t, c.Known)
|
|
if len(cands) < 2 {
|
|
return "", false
|
|
}
|
|
return cands[1+ni(len(cands)-1)], true // index 0 is the champion; sample [1:]
|
|
}
|
|
|
|
// ObserveRequest is the online-learning feedback the gateway posts to the engine
|
|
// after a reward is joined: the request's feature vector `x` (the same vector the
|
|
// engine returned at decision time), the arm that served, the caller (keys the
|
|
// per-user LinUCB bandit), the caller org (selects the scope), and the outcome reward
|
|
// in [0,1]. No prompt text.
|
|
type ObserveRequest struct {
|
|
User string `json:"user"`
|
|
Org string `json:"org"`
|
|
Features []float64 `json:"features"`
|
|
Model string `json:"model"`
|
|
Reward float64 `json:"reward"`
|
|
}
|
|
|
|
// Observe forwards a joined reward to the engine's /route/observe endpoint so the
|
|
// learned policy updates per-user theta in-process (the fast online loop). It is
|
|
// best-effort and fire-and-forget by contract: any error (no endpoint, unreachable,
|
|
// non-2xx) is ignored — a missed online update never fails the reward write, and the
|
|
// offline corpus (the rewarded ledger) still captures the signal. Hard-capped by
|
|
// Timeout so it never hangs the caller's goroutine.
|
|
func (c Client) Observe(ctx context.Context, obs ObserveRequest) {
|
|
if c.Endpoint == "" || len(obs.Features) == 0 || obs.Model == "" {
|
|
return
|
|
}
|
|
timeout := c.Timeout
|
|
if timeout <= 0 {
|
|
timeout = DefaultTimeout
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
body, err := json.Marshal(obs)
|
|
if err != nil {
|
|
return
|
|
}
|
|
url := strings.TrimRight(c.Endpoint, "/") + "/route/observe"
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
|
if err != nil {
|
|
return
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
cli := c.HTTP
|
|
if cli == nil {
|
|
cli = http.DefaultClient
|
|
}
|
|
resp, err := cli.Do(req)
|
|
if err != nil {
|
|
return
|
|
}
|
|
_ = resp.Body.Close()
|
|
}
|
|
|
|
// missLogEvery bounds how often a failing engine writes a line: an endpoint that
|
|
// is down for a week must stay VISIBLE without logging once per request.
|
|
const missLogEvery = time.Minute
|
|
|
|
// misses counts engine calls that fell through to the heuristic since boot, and
|
|
// lastMissLog is the unix-nano stamp of the last line written.
|
|
var (
|
|
misses atomic.Uint64
|
|
lastMissLog atomic.Int64
|
|
)
|
|
|
|
// Misses is the number of engine calls that fell through to the heuristic since
|
|
// boot. Read it beside the routed volume: a count that tracks total traffic means
|
|
// the engine is unreachable or misconfigured, not merely losing on the merits.
|
|
func Misses() uint64 { return misses.Load() }
|
|
|
|
// miss records an engine call that could not be used and returns the zero
|
|
// Decision, so every failure path reads `return c.miss(...)`. Counting is
|
|
// unconditional; logging is rate-limited to one line per missLogEvery.
|
|
func (c Client) miss(reason string) (Decision, bool) {
|
|
n := misses.Add(1)
|
|
now := time.Now().UnixNano()
|
|
if prev := lastMissLog.Load(); now-prev >= int64(missLogEvery) && lastMissLog.CompareAndSwap(prev, now) {
|
|
log.Warning("router: engine %q unusable (%s) — serving the heuristic instead; %d fallbacks since boot", c.Endpoint, reason, n)
|
|
}
|
|
return Decision{}, false
|
|
}
|
|
|
|
// routeEngine performs the constrained /route call. Returns ok=false on any
|
|
// error so the caller falls back to the heuristic — every such exit goes through
|
|
// c.miss, which counts it and periodically says so.
|
|
func (c Client) routeEngine(ctx context.Context, req Request, slo Slo) (Decision, bool) {
|
|
timeout := c.Timeout
|
|
if timeout <= 0 {
|
|
timeout = DefaultTimeout
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
|
|
body, err := json.Marshal(engineRequest{Prompt: req.Text, Slo: slo})
|
|
if err != nil {
|
|
return c.miss("encode request: " + err.Error())
|
|
}
|
|
url := strings.TrimRight(c.Endpoint, "/") + "/route"
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
|
if err != nil {
|
|
return c.miss("build request: " + err.Error())
|
|
}
|
|
httpReq.Header.Set("Content-Type", "application/json")
|
|
|
|
cli := c.HTTP
|
|
if cli == nil {
|
|
cli = http.DefaultClient
|
|
}
|
|
resp, err := cli.Do(httpReq)
|
|
if err != nil {
|
|
return c.miss("transport: " + err.Error())
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return c.miss("HTTP " + strconv.Itoa(resp.StatusCode) + " from " + url)
|
|
}
|
|
var out engineResponse
|
|
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&out); err != nil {
|
|
return c.miss("decode reply: " + err.Error())
|
|
}
|
|
|
|
// Prefer an explicit, servable model; else map the returned task via the
|
|
// policy table (an engine that emits only a task label).
|
|
if out.Model != "" && (c.Known == nil || c.Known(out.Model)) {
|
|
return Decision{Model: out.Model, Task: Task(out.Task), Confidence: out.Confidence, Source: SourceEngine, Features: out.Features}, true
|
|
}
|
|
if out.Task != "" {
|
|
if m := c.Policy.ForTask(Task(out.Task), c.Known); m != "" {
|
|
return Decision{Model: m, Task: Task(out.Task), Confidence: out.Confidence, Source: SourceEngine, Features: out.Features}, true
|
|
}
|
|
}
|
|
return c.miss("engine chose model " + strconv.Quote(out.Model) + " / task " + strconv.Quote(out.Task) + ", neither servable")
|
|
}
|