mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
425 lines
18 KiB
Go
425 lines
18 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/object"
|
|
"github.com/hanzoai/decimal"
|
|
)
|
|
|
|
// depthConfig loads a model table carrying the router.depth rows and the router's
|
|
// prefer table as universe declares them (values/hanzo/cloud.yaml), through the same
|
|
// file loader production uses.
|
|
func depthConfig(t *testing.T) {
|
|
t.Helper()
|
|
path := filepath.Join(t.TempDir(), "models.yaml")
|
|
const doc = `
|
|
version: 1
|
|
router:
|
|
enabled: true
|
|
depth:
|
|
enso-auto: &enso
|
|
default: enso-flash
|
|
instant: enso-flash
|
|
"off": enso-flash
|
|
none: enso-flash
|
|
minimal: enso-flash
|
|
low: enso-flash
|
|
medium: enso-pro
|
|
high: enso-max
|
|
xhigh: enso-ultra
|
|
max: enso-ultra
|
|
ENSO: *enso
|
|
prefer:
|
|
cheap_chat: [enso-flash]
|
|
creative: [enso-flash]
|
|
default: [enso-flash]
|
|
vision: [enso-flash]
|
|
code: [enso-pro]
|
|
long_context: [enso-pro]
|
|
reasoning: [enso-pro]
|
|
math: [enso-max]
|
|
models: {}
|
|
`
|
|
if err := os.WriteFile(path, []byte(doc), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prev := globalModelConfig
|
|
mc := &ModelConfig{}
|
|
if err := mc.loadFromFile(path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
globalModelConfig = mc
|
|
t.Cleanup(func() { globalModelConfig = prev })
|
|
}
|
|
|
|
// seedEnso stands in for enso discovery: the four priced tiers, and one priced SKU
|
|
// held behind a grant. The router ids are no SKUs and are not listed.
|
|
func seedEnso(t *testing.T) {
|
|
t.Helper()
|
|
price := func(in, out string, ctx int) zenTier {
|
|
return zenTier{MaxCtx: ctx, In: decimal.MustParse(in), Out: decimal.MustParse(out)}
|
|
}
|
|
saved := ensoFam.byID
|
|
ensoFam.byID = map[string]zenModel{
|
|
"enso-flash": {ID: "enso-flash", Base: price("0.33", "1.65", 1_000_000), Tiers: []zenTier{price("0.33", "1.65", 1_000_000)}},
|
|
"enso-pro": {ID: "enso-pro", Base: price("3.30", "16.50", 1_000_000), Tiers: []zenTier{price("3.30", "16.50", 1_000_000)}},
|
|
"enso-max": {ID: "enso-max", Base: price("4.30", "21.50", 1_000_000), Tiers: []zenTier{price("4.30", "21.50", 1_000_000)}},
|
|
"enso-ultra": {ID: "enso-ultra", Base: price("5.236", "26.18", 1_000_000), Tiers: []zenTier{price("5.236", "26.18", 1_000_000)}},
|
|
"enso-preview": {ID: "enso-preview", Base: price("1", "1", 1_000_000), Tiers: []zenTier{price("1", "1", 1_000_000)}, Access: "waitlist"},
|
|
}
|
|
t.Cleanup(func() { ensoFam.byID = saved })
|
|
}
|
|
|
|
// noKai answers no Kai, and files the router's decisions where a test reads them.
|
|
func noKai(t *testing.T) *[]object.RoutingEvent {
|
|
t.Helper()
|
|
prevAsk, prevSink := askKai, routingEventSink
|
|
askKai = func(context.Context, string, string, []byte) ([]byte, bool) { return nil, false }
|
|
var got []object.RoutingEvent
|
|
routingEventSink = func(e object.RoutingEvent) { got = append(got, e) }
|
|
t.Cleanup(func() { askKai, routingEventSink = prevAsk, prevSink })
|
|
return &got
|
|
}
|
|
|
|
// THE EFFORT LADDER, every value and alias, on enso-auto: a stated word names its
|
|
// tier, and a request that states none, or auto, or a word the ladder does not hold,
|
|
// is the router's; a trivial one stays at its start. Every dialect that states a
|
|
// depth reads the same.
|
|
func TestTheEffortLadderNamesEveryTier(t *testing.T) {
|
|
depthConfig(t)
|
|
seedEnso(t)
|
|
noKai(t)
|
|
for _, tc := range []struct {
|
|
body, want, by string
|
|
}{
|
|
{`{"model":"enso-auto","messages":[{"role":"user","content":"hi"}]}`, "enso-flash", "default"},
|
|
{`{"model":"enso-auto","reasoning_effort":"auto"}`, "enso-flash", "default"},
|
|
{`{"model":"enso-auto","reasoning_effort":"instant"}`, "enso-flash", "effort"},
|
|
{`{"model":"enso-auto","reasoning_effort":"none"}`, "enso-flash", "effort"},
|
|
{`{"model":"enso-auto","reasoning_effort":"minimal"}`, "enso-flash", "effort"},
|
|
{`{"model":"enso-auto","reasoning_effort":"low"}`, "enso-flash", "effort"},
|
|
{`{"model":"enso-auto","reasoning_effort":"medium"}`, "enso-pro", "effort"},
|
|
{`{"model":"enso-auto","reasoning_effort":"HIGH"}`, "enso-max", "effort"},
|
|
{`{"model":"enso-auto","reasoning_effort":"max"}`, "enso-ultra", "effort"},
|
|
{`{"model":"enso-auto","reasoning_effort":"xhigh"}`, "enso-ultra", "effort"},
|
|
{`{"model":"enso-auto","reasoning":{"effort":"auto"}}`, "enso-flash", "default"},
|
|
{`{"model":"enso-auto","reasoning":{"effort":"high"}}`, "enso-max", "effort"},
|
|
{`{"model":"enso-auto","reasoning":{"effort":"none"}}`, "enso-flash", "effort"},
|
|
{`{"model":"enso-auto","reasoning":{"enabled":false}}`, "enso-flash", "effort"},
|
|
{`{"model":"enso-auto","reasoning":{"max_tokens":32000}}`, "enso-ultra", "effort"},
|
|
{`{"model":"enso-auto","thinking":{"type":"enabled","budget_tokens":2048}}`, "enso-flash", "effort"},
|
|
{`{"model":"enso-auto","thinking":{"type":"enabled","budget_tokens":20000}}`, "enso-ultra", "effort"},
|
|
{`{"model":"enso-auto","thinking":{"type":"disabled"}}`, "enso-flash", "effort"},
|
|
{`{"model":"enso-auto","metadata":{"kai.tier":"small"}}`, "enso-flash", "effort"},
|
|
{`{"model":"enso-auto","metadata":{"kai.tier":"medium"}}`, "enso-pro", "effort"},
|
|
{`{"model":"enso-auto","metadata":{"kai.tier":"large"}}`, "enso-max", "effort"},
|
|
{`{"model":"enso-auto","metadata":{"kai.tier":"frontier"}}`, "enso-ultra", "effort"},
|
|
{`{"model":"enso-auto","metadata":{"kai.effort":"medium"}}`, "enso-pro", "effort"},
|
|
{`{"model":"enso-auto","metadata":{"kai.congested":"true"}}`, "enso-max", "effort"},
|
|
{`{"model":"enso-auto","reasoning_effort":"a word nobody defined"}`, "enso-flash", "default"},
|
|
{`not json`, "enso-flash", "default"},
|
|
} {
|
|
d, ok := RouteDepth(context.Background(), "enso-auto", []byte(tc.body), "acme", "acme/alice", "r1")
|
|
if !ok || d.Tier() != tc.want || d.p.by != tc.by {
|
|
t.Errorf("%s -> (%q by %q, %v), want %q by %q", tc.body, d.Tier(), d.p.by, ok, tc.want, tc.by)
|
|
}
|
|
}
|
|
if tier, floor, ok := DepthRoute("Enso", []byte(`{"reasoning_effort":"high"}`)); !ok || tier != "enso-max" || floor != "enso-flash" {
|
|
t.Errorf("the old name routes like enso-auto, got (%q, %q, %v)", tier, floor, ok)
|
|
}
|
|
}
|
|
|
|
// The router decides a request that left its effort to it: its table names the tier
|
|
// for the task it reads the request as, and the deeper of that and the start serves.
|
|
// A stated effort is never moved by the router. Every decision is filed for the
|
|
// router's trainer.
|
|
func TestTheRouterEscalatesOnlyWhatItJudgesNeedsMore(t *testing.T) {
|
|
depthConfig(t)
|
|
seedEnso(t)
|
|
filed := noKai(t)
|
|
for _, tc := range []struct {
|
|
say, effort, want, by, task string
|
|
}{
|
|
{"Say hi in one word.", "", "enso-flash", "default", "cheap_chat"},
|
|
{"Write a poem about the sea.", "", "enso-flash", "default", "creative"},
|
|
{"Fix this bug: the function returns nil on an empty list.", "", "enso-pro", "router", "code"},
|
|
{"Prove step by step that there are infinitely many primes p = 3 mod 4.", "", "enso-max", "router", "math"},
|
|
{"Prove step by step that there are infinitely many primes p = 3 mod 4.", "low", "enso-flash", "effort", ""},
|
|
} {
|
|
msg, _ := json.Marshal(tc.say)
|
|
body := `{"model":"enso-auto","messages":[{"role":"user","content":` + string(msg) + `}]`
|
|
if tc.effort != "" {
|
|
body += `,"reasoning_effort":"` + tc.effort + `"`
|
|
}
|
|
body += `}`
|
|
d, _ := RouteDepth(context.Background(), "enso-auto", []byte(body), "acme", "acme/alice", "r1")
|
|
if d.Tier() != tc.want || d.p.by != tc.by || d.p.task != tc.task {
|
|
t.Errorf("%q effort %q -> %q by %q task %q, want %q by %q task %q", tc.say, tc.effort, d.Tier(), d.p.by, d.p.task, tc.want, tc.by, tc.task)
|
|
}
|
|
r := d.Routing("")
|
|
if r.Chosen != tc.want || r.By != tc.by || r.Inputs.Policy != "funded" {
|
|
t.Errorf("routing = %+v", r)
|
|
}
|
|
}
|
|
if len(*filed) != 4 {
|
|
t.Fatalf("filed %d decisions, want the four the router made", len(*filed))
|
|
}
|
|
if e := (*filed)[3]; e.RequestedModel != "enso-auto" || e.RoutedModel != "enso-max" || e.Task != "math" || e.Source != "enso-router" || e.Owner != "acme" {
|
|
t.Errorf("filed %+v", e)
|
|
}
|
|
}
|
|
|
|
// Kai is asked how much reasoning a request the router decides needs, as one choice
|
|
// over the ladder's words; its answer moves the request deeper only at its floor, and
|
|
// never shallower than the router's own decision.
|
|
func TestKaiEscalatesAtItsFloorAndNeverDown(t *testing.T) {
|
|
depthConfig(t)
|
|
seedEnso(t)
|
|
noKai(t)
|
|
var asked map[string]any
|
|
answer := func(choice string, p float64) {
|
|
askKai = func(_ context.Context, org, _ string, body []byte) ([]byte, bool) {
|
|
_ = json.Unmarshal(body, &asked)
|
|
b, _ := json.Marshal(map[string]any{"answers": map[string]any{"effort": map[string]any{"type": "choice", "choice": choice, "answer_confidence": p}}})
|
|
return b, org == "acme"
|
|
}
|
|
}
|
|
route := func(say string) *Depth {
|
|
msg, _ := json.Marshal(say)
|
|
d, _ := RouteDepth(context.Background(), "enso-auto", []byte(`{"model":"enso-auto","messages":[{"role":"user","content":`+string(msg)+`}],"tools":[{"type":"function"}]}`), "acme", "acme/alice", "r1")
|
|
return d
|
|
}
|
|
|
|
answer("max", 0.95)
|
|
if d := route("Plan the migration of our billing ledger."); d.Tier() != "enso-ultra" || d.p.by != "kai" {
|
|
t.Errorf("Kai at 0.95 for max: %q by %q, want enso-ultra by kai", d.Tier(), d.p.by)
|
|
}
|
|
if asked["model"] != "kai" {
|
|
t.Errorf("asked %v, want kai", asked["model"])
|
|
}
|
|
q := asked["questions"].(map[string]any)["effort"].(map[string]any)
|
|
labels := q["criteria"].(map[string]any)
|
|
for _, w := range []string{"instant", "low", "medium", "high", "max"} {
|
|
if _, ok := labels[w]; !ok || q["type"] != "choice" {
|
|
t.Errorf("the question %v lacks the ladder's word %q", q, w)
|
|
}
|
|
}
|
|
state := asked["state"].(map[string]any)
|
|
if state["request"] != "Plan the migration of our billing ledger." || state["tools"] != float64(1) || state["turns"] != float64(1) {
|
|
t.Errorf("state = %v", state)
|
|
}
|
|
|
|
answer("max", 0.85)
|
|
if d := route("Plan the migration of our billing ledger."); d.Tier() != "enso-flash" || d.p.by != "default" {
|
|
t.Errorf("Kai under its floor moved the request: %q by %q", d.Tier(), d.p.by)
|
|
}
|
|
answer("low", 0.99)
|
|
if d := route("Prove that the square root of two is irrational."); d.Tier() != "enso-max" || d.p.by != "router" {
|
|
t.Errorf("Kai's low moved a math request off the router's tier: %q by %q", d.Tier(), d.p.by)
|
|
}
|
|
answer("deeper", 0.99)
|
|
if d := route("Plan the migration of our billing ledger."); d.Tier() != "enso-flash" {
|
|
t.Errorf("a label Kai was not asked for moved the request: %q", d.Tier())
|
|
}
|
|
}
|
|
|
|
// A Kai that has not answered within its budget leaves the router's decision: the
|
|
// request is not held past kaiBudget.
|
|
func TestASlowKaiIsNotWaitedFor(t *testing.T) {
|
|
depthConfig(t)
|
|
seedEnso(t)
|
|
noKai(t)
|
|
askKai = func(ctx context.Context, _, _ string, _ []byte) ([]byte, bool) {
|
|
<-ctx.Done()
|
|
return nil, false
|
|
}
|
|
start := time.Now()
|
|
d, _ := RouteDepth(context.Background(), "enso-auto", []byte(`{"messages":[{"role":"user","content":"Plan a trip."}]}`), "acme", "", "r1")
|
|
if took := time.Since(start); d.Tier() != "enso-flash" || took > kaiBudget+300*time.Millisecond {
|
|
t.Errorf("served %q after %s, want enso-flash within Kai's budget", d.Tier(), took)
|
|
}
|
|
}
|
|
|
|
// Only a router id is routed, and only onto a tier the family serves at a price
|
|
// without a grant; a request no gate decided for is served at the row's start.
|
|
func TestDepthRouteRoutesOnlyARouterIdOntoAPaidTier(t *testing.T) {
|
|
depthConfig(t)
|
|
seedEnso(t)
|
|
for _, model := range []string{"enso-flash", "enso-ultra", "zen6", "gpt-6-sol", ""} {
|
|
if tier, floor, ok := DepthRoute(model, []byte(`{"reasoning_effort":"high"}`)); ok {
|
|
t.Errorf("%q is no router id and was routed to (%q, %q)", model, tier, floor)
|
|
}
|
|
if got := unpaid(model); got != model {
|
|
t.Errorf("unpaid(%q) = %q, want it unchanged", model, got)
|
|
}
|
|
}
|
|
for _, model := range []string{"enso-auto", "ENSO"} {
|
|
if got := unpaid(model); got != "enso-flash" {
|
|
t.Errorf("unpaid(%q) = %q, want the row's start", model, got)
|
|
}
|
|
}
|
|
for _, sku := range []string{"enso-gone", "enso-preview"} {
|
|
globalModelConfig.mu.Lock()
|
|
globalModelConfig.router.Depth["enso-auto"]["high"] = sku
|
|
globalModelConfig.mu.Unlock()
|
|
if tier, _, ok := DepthRoute("enso-auto", []byte(`{"reasoning_effort":"high"}`)); !ok || tier != "" {
|
|
t.Errorf("a row naming %q: (%q, %v), want no tier", sku, tier, ok)
|
|
}
|
|
}
|
|
globalModelConfig = nil
|
|
if tier, floor, ok := DepthRoute("enso-auto", []byte(`{}`)); ok {
|
|
t.Errorf("no model table routed to (%q, %q)", tier, floor)
|
|
}
|
|
}
|
|
|
|
// WithModel changes the model and nothing else.
|
|
func TestWithModelChangesOnlyTheModel(t *testing.T) {
|
|
in := []byte(`{"model":"enso-auto","stream":true,"messages":[{"role":"user","content":"a <b> & c"}],"reasoning_effort":"high","n":1}`)
|
|
out, ok := WithModel(in, "enso-pro")
|
|
if !ok {
|
|
t.Fatal("a JSON object was refused")
|
|
}
|
|
var a, b map[string]any
|
|
if err := json.Unmarshal(in, &a); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := json.Unmarshal(out, &b); err != nil {
|
|
t.Fatalf("rewritten body is not JSON: %s", out)
|
|
}
|
|
if b["model"] != "enso-pro" {
|
|
t.Fatalf("model = %v", b["model"])
|
|
}
|
|
a["model"] = "enso-pro"
|
|
ja, _ := json.Marshal(a)
|
|
jb, _ := json.Marshal(b)
|
|
if string(ja) != string(jb) {
|
|
t.Errorf("fields changed:\n%s\n%s", ja, jb)
|
|
}
|
|
for _, bad := range []string{``, `null`, `[]`, `"x"`, `not json`} {
|
|
if _, ok := WithModel([]byte(bad), "enso-pro"); ok {
|
|
t.Errorf("%q was rewritten", bad)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDepthRoutingSaysWhatEnsoReadAndWhy(t *testing.T) {
|
|
depthConfig(t)
|
|
seedEnso(t)
|
|
high := []byte(`{"model":"enso","reasoning_effort":"high"}`)
|
|
|
|
r := DepthRouting("enso", high, "")
|
|
if r == nil || r.Router != "enso" || r.Chosen != "enso-max" || r.By != "effort" {
|
|
t.Fatalf("routing = %+v, want enso chose enso-max by effort", r)
|
|
}
|
|
if r.Inputs != (RoutingInputs{Depth: "high", Key: "high", Policy: "funded"}) {
|
|
t.Fatalf("inputs = %+v", r.Inputs)
|
|
}
|
|
want := []Candidate{
|
|
{Model: "enso-flash", Depths: []string{"instant", "off", "none", "minimal", "default", "low"}},
|
|
{Model: "enso-pro", Depths: []string{"medium"}},
|
|
{Model: "enso-max", Depths: []string{"high"}},
|
|
{Model: "enso-ultra", Depths: []string{"xhigh", "max"}},
|
|
}
|
|
got, _ := json.Marshal(r.Candidates)
|
|
exp, _ := json.Marshal(want)
|
|
if string(got) != string(exp) {
|
|
t.Fatalf("candidates = %s, want %s", got, exp)
|
|
}
|
|
if r.Reason != "High effort names enso-max, and your plan or balance pays for it." {
|
|
t.Fatalf("reason = %q", r.Reason)
|
|
}
|
|
|
|
// The same request nothing pays the tier for is served on its no-cost rungs.
|
|
r = DepthRouting("enso", high, "nothing pays for it (insufficient_balance)")
|
|
if r.Chosen != "enso-max" || r.Inputs.Policy != "unfunded" ||
|
|
r.Reason != "High effort names enso-max, but nothing pays for it (insufficient_balance), so enso-max serves at no charge on its no-cost rungs." {
|
|
t.Fatalf("unfunded: %+v %q", r.Inputs, r.Reason)
|
|
}
|
|
|
|
// A request stating no effort is the router's, read at its start.
|
|
r = DepthRouting("enso", []byte(`{"model":"enso"}`), "")
|
|
if r.Inputs.Depth != "default" || r.Inputs.Key != "default" || r.Chosen != "enso-flash" || r.By != "default" {
|
|
t.Fatalf("default: %+v chose %q by %q", r.Inputs, r.Chosen, r.By)
|
|
}
|
|
|
|
// A tier the family does not serve at a price is said so.
|
|
globalModelConfig.mu.Lock()
|
|
globalModelConfig.router.Depth["enso"]["max"] = "enso-preview"
|
|
globalModelConfig.mu.Unlock()
|
|
r = DepthRouting("enso", []byte(`{"reasoning_effort":"max"}`), "")
|
|
if r.Inputs.Policy != "n/a" || r.Reason != "Max effort names enso-preview, but enso-preview is not a model open to every caller." {
|
|
t.Fatalf("unserved tier: policy %q reason %q", r.Inputs.Policy, r.Reason)
|
|
}
|
|
|
|
for _, model := range []string{"enso-pro", "zen5"} {
|
|
if DepthRouting(model, high, "") != nil {
|
|
t.Fatalf("%s has no depth row: want no routing", model)
|
|
}
|
|
}
|
|
}
|
|
|
|
// /v1/models lists each router id: its start's row and price under its own id, the
|
|
// narrowest window of its tiers, and a description of where it routes. No free id is
|
|
// in the description, and a router whose start is not listed is not listed either.
|
|
func TestTheRouterIdsAreListed(t *testing.T) {
|
|
depthConfig(t)
|
|
seedEnso(t)
|
|
models := []modelInfo{
|
|
{ID: "enso-flash", Object: "model", OwnedBy: "hanzo", Premium: true, ContextWindow: 1_000_000, SupportsVision: true},
|
|
{ID: "enso-max", Object: "model", OwnedBy: "hanzo", Premium: true, ContextWindow: 1_000_000},
|
|
{ID: "enso-pro", Object: "model", OwnedBy: "hanzo", Premium: true, ContextWindow: 1_000_000},
|
|
{ID: "enso-ultra", Object: "model", OwnedBy: "hanzo", Premium: true, ContextWindow: 1_000_000},
|
|
}
|
|
out := depthListings(globalModelConfig, models)
|
|
var ids []string
|
|
byID := map[string]modelInfo{}
|
|
for _, m := range out {
|
|
ids = append(ids, m.ID)
|
|
byID[m.ID] = m
|
|
}
|
|
if got := strings.Join(ids, ","); got != "enso,enso-auto,enso-flash,enso-max,enso-pro,enso-ultra" {
|
|
t.Fatalf("listed %s", got)
|
|
}
|
|
for _, id := range []string{"enso", "enso-auto"} {
|
|
m := byID[id]
|
|
if m.Name != "Enso" || !m.Premium || !m.SupportsVision {
|
|
t.Errorf("%s = %+v, want enso-flash's row", id, m)
|
|
}
|
|
for _, says := range []string{"enso-max at high", "enso-ultra at xhigh, max", "starting at enso-flash"} {
|
|
if !strings.Contains(m.Description, says) {
|
|
t.Errorf("%s description %q does not say %q", id, m.Description, says)
|
|
}
|
|
}
|
|
if strings.Contains(m.Description, "free") {
|
|
t.Errorf("%s description names a free model: %q", id, m.Description)
|
|
}
|
|
if got := listedAs(id); got != "enso-flash" {
|
|
t.Errorf("%s is priced as %q, want enso-flash", id, got)
|
|
}
|
|
}
|
|
if out := depthListings(globalModelConfig, models[1:]); len(out) != 3 {
|
|
t.Errorf("without enso-flash listed: %d rows", len(out))
|
|
}
|
|
}
|