mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
dbx Model(s).Update() writes EVERY column (incl nil), so any writer that rebuilds a fresh OrgSettings + copies a hand-picked subset NULLs the fields it forgot. Feature 1 fixed the beego UpdateRouterPolicy but three other writers still wiped the new allowlist/dial (and more): - F1-1 (HIGH): zapUpdateRouterPolicyHandler — the LIVE gateway path (served natively over ZAP, no beego fallback) — dropped enabledModels/qualityBias on write AND wiped RouterEnabledModels/QualityBias/Strategy/Overrides/TrainingContribution. The feature was non-functional + destructive on the real customer path. Now mutate-existing + reads both new fields. - F1-2 (HIGH): the trainer's deployRouterPreferTo wiped the same set (+ Judge*/MeanField* on the '*' row) on EVERY gated deploy (routine). Now mutate-existing, merging only RouterPrefer. Test pins preservation of allowlist/dial/consent across a deploy. - /v1/update-org-settings (super-admin) was PUT-replace: a partial write (e.g. the admin auto-routing toggle sending just autoRouting) NULLed the whole row. Now PATCH-merge (unmarshal onto the existing row; absent fields keep their value; send a field to clear). The TrainingContribution setters + SeedInternalTrainingConsent were already mutate-existing (verified). Root smell — one row, many writers — mitigated by making every writer mutate.
335 lines
14 KiB
Go
335 lines
14 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"runtime"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/object"
|
|
)
|
|
|
|
// TestFlywheelDarkByDefault proves the router flywheel launches are safe to wire into
|
|
// the always-run Bootstrap (the single launch site, HIP-510): with none of their env
|
|
// flags set, StartRouterProbe and StartRouterTrainer are no-ops that spawn NO background
|
|
// goroutine. This dark-by-default gating is exactly what lets Bootstrap call them
|
|
// unconditionally without turning the flywheel on outside a configured deployment.
|
|
func TestFlywheelDarkByDefault(t *testing.T) {
|
|
t.Setenv("ROUTER_TRAIN_ENABLED", "")
|
|
t.Setenv("ROUTER_PROBE_RPH", "")
|
|
t.Setenv("ROUTER_PROBE_TOKEN", "")
|
|
|
|
// Let any async goroutines from earlier tests settle so the baseline is quiet, then
|
|
// measure tightly around the two synchronous calls: when disabled they run zero `go`
|
|
// statements, so the count cannot rise.
|
|
time.Sleep(50 * time.Millisecond)
|
|
before := runtime.NumGoroutine()
|
|
StartRouterProbe()
|
|
StartRouterTrainer()
|
|
if after := runtime.NumGoroutine(); after > before {
|
|
t.Fatalf("flywheel spawned %d goroutine(s) with env unset; must be dark by default", after-before)
|
|
}
|
|
}
|
|
|
|
// TestTrainerFitsEarlyForChurnResilience locks the fit-early invariant (HIP-510):
|
|
// the FIRST fit runs a short boot delay after start — well before a full interval —
|
|
// so a pod that lives past ~the boot delay always produces at least one completed
|
|
// verdict. Without this, frequent redeploys perpetually reset a long sleep to zero
|
|
// completed cycles (observed live: 15m interval + churn → cycles kept getting
|
|
// starved). The boot delay MUST stay far below the minimum interval for the
|
|
// guarantee to hold, and it is overridable via ROUTER_TRAIN_BOOT_DELAY.
|
|
func TestTrainerFitsEarlyForChurnResilience(t *testing.T) {
|
|
if trainBootDelay >= trainMinEvery {
|
|
t.Fatalf("boot delay %s must be << min interval %s to survive redeploy churn", trainBootDelay, trainMinEvery)
|
|
}
|
|
t.Setenv("ROUTER_TRAIN_BOOT_DELAY", "")
|
|
if got := envDuration("ROUTER_TRAIN_BOOT_DELAY", trainBootDelay); got != trainBootDelay {
|
|
t.Fatalf("default boot delay = %s, want %s", got, trainBootDelay)
|
|
}
|
|
t.Setenv("ROUTER_TRAIN_BOOT_DELAY", "5s")
|
|
if got := envDuration("ROUTER_TRAIN_BOOT_DELAY", trainBootDelay); got != 5*time.Second {
|
|
t.Fatalf("override boot delay = %s, want 5s", got)
|
|
}
|
|
}
|
|
|
|
// rewarded builds a batch of n rewarded routing events for one (task, model) arm
|
|
// at a fixed reward — the pure fit's only inputs.
|
|
func rewarded(task, model string, n int, reward float64) []*object.RoutingEvent {
|
|
out := make([]*object.RoutingEvent, n)
|
|
for i := range out {
|
|
out[i] = &object.RoutingEvent{
|
|
Task: task,
|
|
RoutedModel: model,
|
|
Reward: reward,
|
|
RewardedTime: "2026-07-16T00:00:00Z",
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// TestFitRouterHeads proves the fold: per-(task,model) mean over rewarded rows,
|
|
// unscored/blank rows ignored, reward clamped to [0,1].
|
|
func TestFitRouterHeads(t *testing.T) {
|
|
var ev []*object.RoutingEvent
|
|
ev = append(ev, rewarded("code", "zen5-coder", 3, 0.9)...)
|
|
ev = append(ev, rewarded("code", "gpt-4o", 2, 0.4)...)
|
|
// An unscored row (no RewardedTime) must not contribute.
|
|
ev = append(ev, &object.RoutingEvent{Task: "code", RoutedModel: "zen5-coder", Reward: 0.0})
|
|
// An over-range reward clamps to 1.
|
|
ev = append(ev, &object.RoutingEvent{Task: "math", RoutedModel: "zen5", Reward: 5, RewardedTime: "t"})
|
|
|
|
arms := fitRouterHeads(ev)
|
|
if a := arms["code\x00zen5-coder"]; a == nil || a.N != 3 || a.Mean() != 0.9 {
|
|
t.Errorf("code/zen5-coder: got %+v, want N=3 mean=0.9", a)
|
|
}
|
|
if a := arms["code\x00gpt-4o"]; a == nil || a.N != 2 || a.Mean() != 0.4 {
|
|
t.Errorf("code/gpt-4o: got %+v, want N=2 mean=0.4", a)
|
|
}
|
|
if a := arms["math\x00zen5"]; a == nil || a.Mean() != 1.0 {
|
|
t.Errorf("math/zen5: reward must clamp to 1.0, got %+v", a)
|
|
}
|
|
}
|
|
|
|
// TestBestArmPerTaskHonorsMinEvents proves a low-evidence arm can't win.
|
|
func TestBestArmPerTaskHonorsMinEvents(t *testing.T) {
|
|
arms := fitRouterHeads(append(
|
|
rewarded("code", "hot-new", 3, 1.0), // great but only 3 samples
|
|
rewarded("code", "steady", 50, 0.7)..., // solid evidence
|
|
))
|
|
best := bestArmPerTask(arms, 20)
|
|
if best["code"].Model != "steady" {
|
|
t.Errorf("min-events must exclude the 3-sample arm; got %q", best["code"].Model)
|
|
}
|
|
// Lower the bar and the high-mean arm wins.
|
|
if bestArmPerTask(arms, 2)["code"].Model != "hot-new" {
|
|
t.Error("with minEvents=2 the higher-mean arm should win")
|
|
}
|
|
}
|
|
|
|
// TestGateDeploysOnlyOnMarginBeat proves a task deploys only when its best arm
|
|
// beats the incumbent's mean by the margin, and the "*"/conf incumbent is read.
|
|
func TestGateDeploysOnlyOnMarginBeat(t *testing.T) {
|
|
var ev []*object.RoutingEvent
|
|
ev = append(ev, rewarded("code", "challenger", 40, 0.90)...)
|
|
ev = append(ev, rewarded("code", "incumbent", 40, 0.80)...) // beat by 0.10
|
|
ev = append(ev, rewarded("math", "challenger2", 40, 0.61)...) // barely
|
|
ev = append(ev, rewarded("math", "incumbent2", 40, 0.60)...) // beat by only 0.01
|
|
arms := fitRouterHeads(ev)
|
|
|
|
incumbent := map[string][]string{"code": {"incumbent"}, "math": {"incumbent2"}}
|
|
res := gateRouterFit(arms, incumbent, nil, 20, 0.02)
|
|
|
|
if got := res.Prefer["code"]; len(got) != 1 || got[0] != "challenger" {
|
|
t.Errorf("code: 0.10 > margin → deploy challenger, got %v", got)
|
|
}
|
|
if _, ok := res.Prefer["math"]; ok {
|
|
t.Error("math: 0.01 < 0.02 margin → must NOT deploy (keep incumbent)")
|
|
}
|
|
if !res.Passed {
|
|
t.Error("run passes when ≥1 task deploys")
|
|
}
|
|
if res.NewMean <= res.BaseMean {
|
|
t.Errorf("deployed new mean %.3f must exceed base %.3f", res.NewMean, res.BaseMean)
|
|
}
|
|
}
|
|
|
|
// TestGateFirstFitNoIncumbent proves that with no incumbent evidence a real
|
|
// winner still deploys (beats the absolute floor of margin above 0).
|
|
func TestGateFirstFitNoIncumbent(t *testing.T) {
|
|
arms := fitRouterHeads(rewarded("creative", "zen5", 30, 0.75))
|
|
res := gateRouterFit(arms, nil, nil, 20, 0.02)
|
|
if got := res.Prefer["creative"]; len(got) != 1 || got[0] != "zen5" {
|
|
t.Errorf("first fit with a clear winner must deploy, got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestGateHoldsWhenNothingClears proves an all-weak fit keeps the incumbent and
|
|
// records a non-published verdict (the honest "kept incumbent" path).
|
|
func TestGateHoldsWhenNothingClears(t *testing.T) {
|
|
arms := fitRouterHeads(rewarded("code", "meh", 5, 0.5)) // below minEvents
|
|
res := gateRouterFit(arms, map[string][]string{"code": {"incumbent"}}, nil, 20, 0.02)
|
|
if res.Passed || len(res.Prefer) != 0 {
|
|
t.Errorf("nothing clears min-events → no deploy, got passed=%v prefer=%v", res.Passed, res.Prefer)
|
|
}
|
|
if res.Note == "" {
|
|
t.Error("a held run must carry an honest note")
|
|
}
|
|
}
|
|
|
|
// TestTrainingOwnerSetIncludesInternalAndDedups proves the consent set always
|
|
// carries the reserved internal orgs, dedups, and drops "*"/blank.
|
|
func TestTrainingOwnerSetIncludesInternalAndDedups(t *testing.T) {
|
|
t.Setenv("ROUTER_TRAIN_INTERNAL_ORGS", "hanzo, probe-org , ")
|
|
got := trainingOwnerSet()
|
|
seen := map[string]int{}
|
|
for _, o := range got {
|
|
seen[o]++
|
|
}
|
|
for _, must := range []string{"admin", "hanzo", "probe-org"} {
|
|
if seen[must] == 0 {
|
|
t.Errorf("owner set must include internal org %q; got %v", must, got)
|
|
}
|
|
}
|
|
for o, n := range seen {
|
|
if n > 1 {
|
|
t.Errorf("owner %q duplicated %d times", o, n)
|
|
}
|
|
if o == "*" || o == "" {
|
|
t.Errorf("owner set must never contain %q", o)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestTrainScopePerOrgAndGlobalPartition proves the driver partitions the ledger
|
|
// by owner: the pure fit sees exactly the events for each scope. (The DB-writing
|
|
// half of trainScope is covered by the live path; here we prove the fit input.)
|
|
func TestTrainScopePerOrgAndGlobalPartition(t *testing.T) {
|
|
var all []*object.RoutingEvent
|
|
all = append(all, rewardedOwner("acme", "code", "acme-coder", 30, 0.9)...)
|
|
all = append(all, rewardedOwner("globex", "code", "globex-coder", 30, 0.8)...)
|
|
byOwner := map[string][]*object.RoutingEvent{}
|
|
for _, e := range all {
|
|
byOwner[e.Owner] = append(byOwner[e.Owner], e)
|
|
}
|
|
if len(byOwner["acme"]) != 30 || len(byOwner["globex"]) != 30 {
|
|
t.Fatalf("partition wrong: acme=%d globex=%d", len(byOwner["acme"]), len(byOwner["globex"]))
|
|
}
|
|
// acme's fit must see ONLY acme's arm (its own data trains its own policy).
|
|
arms := fitRouterHeads(byOwner["acme"])
|
|
if arms["code\x00acme-coder"] == nil || arms["code\x00globex-coder"] != nil {
|
|
t.Errorf("per-org fit leaked another org's data: %+v", arms)
|
|
}
|
|
}
|
|
|
|
// rewardedOwner is `rewarded` with an explicit owner for the partition test.
|
|
func rewardedOwner(owner, task, model string, n int, reward float64) []*object.RoutingEvent {
|
|
out := rewarded(task, model, n, reward)
|
|
for _, e := range out {
|
|
e.Owner = owner
|
|
}
|
|
return out
|
|
}
|
|
|
|
// TestTrainScopeAppendsRetrainLog is the regression guard for the retrain-timeline
|
|
// bug. The in-process trainer must append ONE RouterTrainingLog row per fit — the row
|
|
// the world.hanzo.ai "Model Improvement" panel counts as a retrain. Before the fix
|
|
// trainScope only upserted the "latest per scope" artifact meta and never appended, so
|
|
// the panel showed 0 retrains forever even while the trainer ran. A fit appends whether
|
|
// the gate passes OR misses, so one call ⇒ exactly one row.
|
|
func TestTrainScopeAppendsRetrainLog(t *testing.T) {
|
|
restore, err := object.UseMemoryDB(
|
|
"file:trainscope_retrainlog_test?mode=memory&cache=shared",
|
|
&object.RouterArtifactMeta{}, &object.RouterTrainingLog{}, &object.OrgSettings{},
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("UseMemoryDB: %v", err)
|
|
}
|
|
defer restore()
|
|
|
|
// A clean first fit for the shared-base scope with a clear winner.
|
|
if err := trainScope("*", rewarded("code", "zen5-coder", 30, 0.9), 1, 0.0); err != nil {
|
|
t.Fatalf("trainScope: %v", err)
|
|
}
|
|
|
|
rows, err := object.ListRouterTrainingLog("*", "", 0)
|
|
if err != nil {
|
|
t.Fatalf("list: %v", err)
|
|
}
|
|
if len(rows) != 1 {
|
|
t.Fatalf("want exactly one retrain row appended by trainScope, got %d", len(rows))
|
|
}
|
|
if rows[0].Owner != "*" || rows[0].Events <= 0 || rows[0].Version == "" {
|
|
t.Errorf("retrain row not populated from the fit: %+v", rows[0])
|
|
}
|
|
}
|
|
|
|
// TestSeedInternalTrainingConsent proves the boot seed flips OUR OWN reserved orgs
|
|
// to EXPLICIT training-contribution consent (so the EU per-request judge guard never
|
|
// suppresses internal traffic), is idempotent, and NEVER bulk-enables customer orgs.
|
|
func TestSeedInternalTrainingConsent(t *testing.T) {
|
|
t.Setenv("ROUTER_TRAIN_INTERNAL_ORGS", "")
|
|
restore, err := object.UseMemoryDB("file:seed_internal_consent_test?mode=memory&cache=shared", &object.OrgSettings{})
|
|
if err != nil {
|
|
t.Fatalf("UseMemoryDB: %v", err)
|
|
}
|
|
t.Cleanup(restore)
|
|
|
|
SeedInternalTrainingConsent()
|
|
|
|
// Every reserved internal org is now EXPLICITLY enabled (not merely opt-out default).
|
|
for _, org := range reservedInternalOrgs() {
|
|
if got := object.GetCachedOrgTrainingContribution(org); got != object.TrainingContributionEnabled {
|
|
t.Fatalf("internal org %q contribution = %q, want %q", org, got, object.TrainingContributionEnabled)
|
|
}
|
|
}
|
|
|
|
// A customer org is untouched — the seed never bulk-enables anyone else.
|
|
if got := object.GetCachedOrgTrainingContribution("some-customer-org"); got != object.TrainingContributionUnset {
|
|
t.Fatalf("customer org contribution = %q, want unset (never bulk-enabled)", got)
|
|
}
|
|
|
|
// Idempotent: a second run is a clean no-op and leaves the state enabled.
|
|
SeedInternalTrainingConsent()
|
|
if got := object.GetCachedOrgTrainingContribution("hanzo"); got != object.TrainingContributionEnabled {
|
|
t.Fatalf("after re-seed: hanzo = %q, want %q", got, object.TrainingContributionEnabled)
|
|
}
|
|
}
|
|
|
|
// TestDeployRouterPreferPreservesFields pins the F1-2 fix: the trainer's auto-deploy
|
|
// (deployRouterPreferTo) merges into RouterPrefer WITHOUT nulling any other field. dbx
|
|
// Update writes all columns, so the old fresh-row + preserve-list wiped the org's
|
|
// allowlist / dial / training consent on every gated deploy — silently re-opening the
|
|
// disabled-model set and resetting the cost dial.
|
|
func TestDeployRouterPreferPreservesFields(t *testing.T) {
|
|
restore, err := object.UseMemoryDB("file:deploy_prefer_preserve_test?mode=memory&cache=shared", &object.OrgSettings{})
|
|
if err != nil {
|
|
t.Fatalf("UseMemoryDB: %v", err)
|
|
}
|
|
defer restore()
|
|
|
|
bias := 0.3
|
|
if _, err := object.AddOrgSettings(&object.OrgSettings{
|
|
Owner: "acme",
|
|
RouterPrefer: object.JSONMap[[]string]{"default": {"a"}},
|
|
RouterEnabledModels: object.JSONMap[bool]{"gpt-4o": true},
|
|
RouterQualityBias: &bias,
|
|
TrainingContribution: object.TrainingContributionEnabled,
|
|
}); err != nil {
|
|
t.Fatalf("seed: %v", err)
|
|
}
|
|
|
|
if err := deployRouterPreferTo("acme", map[string][]string{"code": {"b"}}); err != nil {
|
|
t.Fatalf("deploy: %v", err)
|
|
}
|
|
|
|
got, err := object.GetOrgSettings("acme")
|
|
if err != nil || got == nil {
|
|
t.Fatalf("read: %v (nil=%v)", err, got == nil)
|
|
}
|
|
if !got.RouterEnabledModels["gpt-4o"] {
|
|
t.Fatalf("allowlist WIPED by deploy: %v", got.RouterEnabledModels)
|
|
}
|
|
if got.RouterQualityBias == nil || *got.RouterQualityBias != 0.3 {
|
|
t.Fatalf("dial WIPED by deploy: %v", got.RouterQualityBias)
|
|
}
|
|
if got.TrainingContribution != object.TrainingContributionEnabled {
|
|
t.Fatalf("training consent WIPED by deploy: %q", got.TrainingContribution)
|
|
}
|
|
if got.RouterPrefer["default"][0] != "a" || got.RouterPrefer["code"][0] != "b" {
|
|
t.Fatalf("prefer merge wrong (want default:[a]+code:[b]): %v", got.RouterPrefer)
|
|
}
|
|
}
|