mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
Add a virtual model id `auto` (alias `zen-router`) to the /v1 chat path. When
routing is enabled for the deployment, the requested model is resolved to a
concrete servable id BEFORE provider/pricing/billing resolution, so the request
bills and reports the model that actually served it.
- router/: pure-Go RouterClient. Two strategies: (a) engine — POST {prompt,
tasks?,slo} to {endpoint}/route (learned zen-router via hanzo-engine), hard
150ms timeout; (b) heuristic — faithful port of hanzo-router's Rust
Heuristic::classify + a task->model preference table. Any engine error falls
through to the heuristic, so auto works with zero extra infrastructure.
- controllers/auto_route.go: resolveAutoModel glue + X-Routed-Model header +
X-Max-Cost/X-Max-Latency-Ms SLO headers. One insertion point in
ChatCompletions rewrites request.Model so the entire existing path (ModelRoute
fallbacks, zen identity, reserve/settle, usage record, response echo) binds to
the resolved id.
- conf/models.yaml: router: block {enabled:false, endpoint, prefer, cost_ceiling}
with sensible defaults; env overrides ROUTER_ENABLED / ROUTER_ENDPOINT.
- Tests: classifier table tests (mirror the Rust cases), RouterClient fallback
(engine 500/timeout/unservable -> heuristic), controller resolution + the
end-to-end HTTP contract (rewrite + header + body echo).
- README: enabling, model:auto, SLO headers, transparency header, and that the
learned zen-router (hf.co/zenlm/zen-router) is the upgrade over the heuristic.
Per-org: routing enable is deployment-global (no per-org settings object yet);
the resolved id still flows through per-org ModelRoute resolution + pricing.
94 lines
3.3 KiB
Go
94 lines
3.3 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
// Package router turns a chat request into a concrete model id. It backs the
|
|
// virtual `auto`/`zen-router` model: classify the request into a coarse task,
|
|
// then pick the preferred servable model for that task. The classifier is a
|
|
// faithful port of hanzo-router's Rust Heuristic (engine/hanzo-router/
|
|
// src/classify.rs); the Client can instead consult a learned zen-router served
|
|
// by hanzo-engine, falling back to this heuristic on ANY error so `auto` works
|
|
// with zero extra infrastructure.
|
|
package router
|
|
|
|
import "strings"
|
|
|
|
// Task is the coarse task bucket a request is classified into. Values mirror the
|
|
// snake_case tags of hanzo-router's registry::Task so a learned classifier and
|
|
// this heuristic produce interchangeable labels.
|
|
type Task string
|
|
|
|
const (
|
|
TaskCode Task = "code"
|
|
TaskReasoning Task = "reasoning"
|
|
TaskMath Task = "math"
|
|
TaskCreative Task = "creative"
|
|
TaskVision Task = "vision"
|
|
TaskLongContext Task = "long_context"
|
|
TaskCheapChat Task = "cheap_chat"
|
|
TaskGeneral Task = "general"
|
|
)
|
|
|
|
// Request is the value the router reasons about — extracted from the chat
|
|
// request by the caller so classification stays pure. Mirrors classify.rs's
|
|
// Request.
|
|
type Request struct {
|
|
// Text is the concatenated user text (the last user turn is enough).
|
|
Text string
|
|
// ApproxTokens is the approximate prompt length in tokens (drives
|
|
// long-context routing).
|
|
ApproxTokens int
|
|
// HasMedia is true when the request carries image/video input.
|
|
HasMedia bool
|
|
// TaskHint is an optional explicit task override; "" skips it.
|
|
TaskHint Task
|
|
}
|
|
|
|
// Classify maps a request to a Task using keyword/length/media heuristics. This
|
|
// is a faithful port of hanzo-router's Heuristic::classify — same order, same
|
|
// keyword sets, same thresholds — so routing matches the Rust engine's rule path.
|
|
func Classify(req Request) Task {
|
|
if req.TaskHint != "" {
|
|
return req.TaskHint
|
|
}
|
|
if req.HasMedia {
|
|
return TaskVision
|
|
}
|
|
if req.ApproxTokens >= 32000 {
|
|
return TaskLongContext
|
|
}
|
|
t := strings.ToLower(req.Text)
|
|
has := func(words ...string) bool {
|
|
for _, w := range words {
|
|
if strings.Contains(t, w) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
switch {
|
|
case strings.Contains(t, "```") || has("code", "function", "bug", "compile", "stack trace", "refactor"):
|
|
return TaskCode
|
|
case has("prove", "theorem", "integral", "equation", "calculate", "solve for"):
|
|
return TaskMath
|
|
case has("why", "reason", "step by step", "analyze", "explain how", "trade-off"):
|
|
return TaskReasoning
|
|
case has("write a story", "poem", "creative", "imagine", "brainstorm"):
|
|
return TaskCreative
|
|
case req.ApproxTokens <= 16:
|
|
return TaskCheapChat
|
|
default:
|
|
return TaskGeneral
|
|
}
|
|
}
|