Files
ai/controllers/router_trainer_test.go
zeekay 4b6fb87e42 router: fix ALL OrgSettings writers that null unowned fields (RED F1-1, F1-2)
dbx Model(s).Update() writes EVERY column (incl nil), so any writer that rebuilds a
fresh OrgSettings + copies a hand-picked subset NULLs the fields it forgot. Feature 1
fixed the beego UpdateRouterPolicy but three other writers still wiped the new
allowlist/dial (and more):

- F1-1 (HIGH): zapUpdateRouterPolicyHandler — the LIVE gateway path (served natively
  over ZAP, no beego fallback) — dropped enabledModels/qualityBias on write AND wiped
  RouterEnabledModels/QualityBias/Strategy/Overrides/TrainingContribution. The feature
  was non-functional + destructive on the real customer path. Now mutate-existing + reads
  both new fields.
- F1-2 (HIGH): the trainer's deployRouterPreferTo wiped the same set (+ Judge*/MeanField*
  on the '*' row) on EVERY gated deploy (routine). Now mutate-existing, merging only
  RouterPrefer. Test pins preservation of allowlist/dial/consent across a deploy.
- /v1/update-org-settings (super-admin) was PUT-replace: a partial write (e.g. the admin
  auto-routing toggle sending just autoRouting) NULLed the whole row. Now PATCH-merge
  (unmarshal onto the existing row; absent fields keep their value; send a field to clear).

The TrainingContribution setters + SeedInternalTrainingConsent were already mutate-existing
(verified). Root smell — one row, many writers — mitigated by making every writer mutate.
2026-07-21 12:42:12 -07:00

335 lines
14 KiB
Go

// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"runtime"
"testing"
"time"
"github.com/hanzoai/ai/object"
)
// TestFlywheelDarkByDefault proves the router flywheel launches are safe to wire into
// the always-run Bootstrap (the single launch site, HIP-510): with none of their env
// flags set, StartRouterProbe and StartRouterTrainer are no-ops that spawn NO background
// goroutine. This dark-by-default gating is exactly what lets Bootstrap call them
// unconditionally without turning the flywheel on outside a configured deployment.
func TestFlywheelDarkByDefault(t *testing.T) {
t.Setenv("ROUTER_TRAIN_ENABLED", "")
t.Setenv("ROUTER_PROBE_RPH", "")
t.Setenv("ROUTER_PROBE_TOKEN", "")
// Let any async goroutines from earlier tests settle so the baseline is quiet, then
// measure tightly around the two synchronous calls: when disabled they run zero `go`
// statements, so the count cannot rise.
time.Sleep(50 * time.Millisecond)
before := runtime.NumGoroutine()
StartRouterProbe()
StartRouterTrainer()
if after := runtime.NumGoroutine(); after > before {
t.Fatalf("flywheel spawned %d goroutine(s) with env unset; must be dark by default", after-before)
}
}
// TestTrainerFitsEarlyForChurnResilience locks the fit-early invariant (HIP-510):
// the FIRST fit runs a short boot delay after start — well before a full interval —
// so a pod that lives past ~the boot delay always produces at least one completed
// verdict. Without this, frequent redeploys perpetually reset a long sleep to zero
// completed cycles (observed live: 15m interval + churn → cycles kept getting
// starved). The boot delay MUST stay far below the minimum interval for the
// guarantee to hold, and it is overridable via ROUTER_TRAIN_BOOT_DELAY.
func TestTrainerFitsEarlyForChurnResilience(t *testing.T) {
if trainBootDelay >= trainMinEvery {
t.Fatalf("boot delay %s must be << min interval %s to survive redeploy churn", trainBootDelay, trainMinEvery)
}
t.Setenv("ROUTER_TRAIN_BOOT_DELAY", "")
if got := envDuration("ROUTER_TRAIN_BOOT_DELAY", trainBootDelay); got != trainBootDelay {
t.Fatalf("default boot delay = %s, want %s", got, trainBootDelay)
}
t.Setenv("ROUTER_TRAIN_BOOT_DELAY", "5s")
if got := envDuration("ROUTER_TRAIN_BOOT_DELAY", trainBootDelay); got != 5*time.Second {
t.Fatalf("override boot delay = %s, want 5s", got)
}
}
// rewarded builds a batch of n rewarded routing events for one (task, model) arm
// at a fixed reward — the pure fit's only inputs.
func rewarded(task, model string, n int, reward float64) []*object.RoutingEvent {
out := make([]*object.RoutingEvent, n)
for i := range out {
out[i] = &object.RoutingEvent{
Task: task,
RoutedModel: model,
Reward: reward,
RewardedTime: "2026-07-16T00:00:00Z",
}
}
return out
}
// TestFitRouterHeads proves the fold: per-(task,model) mean over rewarded rows,
// unscored/blank rows ignored, reward clamped to [0,1].
func TestFitRouterHeads(t *testing.T) {
var ev []*object.RoutingEvent
ev = append(ev, rewarded("code", "zen5-coder", 3, 0.9)...)
ev = append(ev, rewarded("code", "gpt-4o", 2, 0.4)...)
// An unscored row (no RewardedTime) must not contribute.
ev = append(ev, &object.RoutingEvent{Task: "code", RoutedModel: "zen5-coder", Reward: 0.0})
// An over-range reward clamps to 1.
ev = append(ev, &object.RoutingEvent{Task: "math", RoutedModel: "zen5", Reward: 5, RewardedTime: "t"})
arms := fitRouterHeads(ev)
if a := arms["code\x00zen5-coder"]; a == nil || a.N != 3 || a.Mean() != 0.9 {
t.Errorf("code/zen5-coder: got %+v, want N=3 mean=0.9", a)
}
if a := arms["code\x00gpt-4o"]; a == nil || a.N != 2 || a.Mean() != 0.4 {
t.Errorf("code/gpt-4o: got %+v, want N=2 mean=0.4", a)
}
if a := arms["math\x00zen5"]; a == nil || a.Mean() != 1.0 {
t.Errorf("math/zen5: reward must clamp to 1.0, got %+v", a)
}
}
// TestBestArmPerTaskHonorsMinEvents proves a low-evidence arm can't win.
func TestBestArmPerTaskHonorsMinEvents(t *testing.T) {
arms := fitRouterHeads(append(
rewarded("code", "hot-new", 3, 1.0), // great but only 3 samples
rewarded("code", "steady", 50, 0.7)..., // solid evidence
))
best := bestArmPerTask(arms, 20)
if best["code"].Model != "steady" {
t.Errorf("min-events must exclude the 3-sample arm; got %q", best["code"].Model)
}
// Lower the bar and the high-mean arm wins.
if bestArmPerTask(arms, 2)["code"].Model != "hot-new" {
t.Error("with minEvents=2 the higher-mean arm should win")
}
}
// TestGateDeploysOnlyOnMarginBeat proves a task deploys only when its best arm
// beats the incumbent's mean by the margin, and the "*"/conf incumbent is read.
func TestGateDeploysOnlyOnMarginBeat(t *testing.T) {
var ev []*object.RoutingEvent
ev = append(ev, rewarded("code", "challenger", 40, 0.90)...)
ev = append(ev, rewarded("code", "incumbent", 40, 0.80)...) // beat by 0.10
ev = append(ev, rewarded("math", "challenger2", 40, 0.61)...) // barely
ev = append(ev, rewarded("math", "incumbent2", 40, 0.60)...) // beat by only 0.01
arms := fitRouterHeads(ev)
incumbent := map[string][]string{"code": {"incumbent"}, "math": {"incumbent2"}}
res := gateRouterFit(arms, incumbent, nil, 20, 0.02)
if got := res.Prefer["code"]; len(got) != 1 || got[0] != "challenger" {
t.Errorf("code: 0.10 > margin → deploy challenger, got %v", got)
}
if _, ok := res.Prefer["math"]; ok {
t.Error("math: 0.01 < 0.02 margin → must NOT deploy (keep incumbent)")
}
if !res.Passed {
t.Error("run passes when ≥1 task deploys")
}
if res.NewMean <= res.BaseMean {
t.Errorf("deployed new mean %.3f must exceed base %.3f", res.NewMean, res.BaseMean)
}
}
// TestGateFirstFitNoIncumbent proves that with no incumbent evidence a real
// winner still deploys (beats the absolute floor of margin above 0).
func TestGateFirstFitNoIncumbent(t *testing.T) {
arms := fitRouterHeads(rewarded("creative", "zen5", 30, 0.75))
res := gateRouterFit(arms, nil, nil, 20, 0.02)
if got := res.Prefer["creative"]; len(got) != 1 || got[0] != "zen5" {
t.Errorf("first fit with a clear winner must deploy, got %v", got)
}
}
// TestGateHoldsWhenNothingClears proves an all-weak fit keeps the incumbent and
// records a non-published verdict (the honest "kept incumbent" path).
func TestGateHoldsWhenNothingClears(t *testing.T) {
arms := fitRouterHeads(rewarded("code", "meh", 5, 0.5)) // below minEvents
res := gateRouterFit(arms, map[string][]string{"code": {"incumbent"}}, nil, 20, 0.02)
if res.Passed || len(res.Prefer) != 0 {
t.Errorf("nothing clears min-events → no deploy, got passed=%v prefer=%v", res.Passed, res.Prefer)
}
if res.Note == "" {
t.Error("a held run must carry an honest note")
}
}
// TestTrainingOwnerSetIncludesInternalAndDedups proves the consent set always
// carries the reserved internal orgs, dedups, and drops "*"/blank.
func TestTrainingOwnerSetIncludesInternalAndDedups(t *testing.T) {
t.Setenv("ROUTER_TRAIN_INTERNAL_ORGS", "hanzo, probe-org , ")
got := trainingOwnerSet()
seen := map[string]int{}
for _, o := range got {
seen[o]++
}
for _, must := range []string{"admin", "hanzo", "probe-org"} {
if seen[must] == 0 {
t.Errorf("owner set must include internal org %q; got %v", must, got)
}
}
for o, n := range seen {
if n > 1 {
t.Errorf("owner %q duplicated %d times", o, n)
}
if o == "*" || o == "" {
t.Errorf("owner set must never contain %q", o)
}
}
}
// TestTrainScopePerOrgAndGlobalPartition proves the driver partitions the ledger
// by owner: the pure fit sees exactly the events for each scope. (The DB-writing
// half of trainScope is covered by the live path; here we prove the fit input.)
func TestTrainScopePerOrgAndGlobalPartition(t *testing.T) {
var all []*object.RoutingEvent
all = append(all, rewardedOwner("acme", "code", "acme-coder", 30, 0.9)...)
all = append(all, rewardedOwner("globex", "code", "globex-coder", 30, 0.8)...)
byOwner := map[string][]*object.RoutingEvent{}
for _, e := range all {
byOwner[e.Owner] = append(byOwner[e.Owner], e)
}
if len(byOwner["acme"]) != 30 || len(byOwner["globex"]) != 30 {
t.Fatalf("partition wrong: acme=%d globex=%d", len(byOwner["acme"]), len(byOwner["globex"]))
}
// acme's fit must see ONLY acme's arm (its own data trains its own policy).
arms := fitRouterHeads(byOwner["acme"])
if arms["code\x00acme-coder"] == nil || arms["code\x00globex-coder"] != nil {
t.Errorf("per-org fit leaked another org's data: %+v", arms)
}
}
// rewardedOwner is `rewarded` with an explicit owner for the partition test.
func rewardedOwner(owner, task, model string, n int, reward float64) []*object.RoutingEvent {
out := rewarded(task, model, n, reward)
for _, e := range out {
e.Owner = owner
}
return out
}
// TestTrainScopeAppendsRetrainLog is the regression guard for the retrain-timeline
// bug. The in-process trainer must append ONE RouterTrainingLog row per fit — the row
// the world.hanzo.ai "Model Improvement" panel counts as a retrain. Before the fix
// trainScope only upserted the "latest per scope" artifact meta and never appended, so
// the panel showed 0 retrains forever even while the trainer ran. A fit appends whether
// the gate passes OR misses, so one call ⇒ exactly one row.
func TestTrainScopeAppendsRetrainLog(t *testing.T) {
restore, err := object.UseMemoryDB(
"file:trainscope_retrainlog_test?mode=memory&cache=shared",
&object.RouterArtifactMeta{}, &object.RouterTrainingLog{}, &object.OrgSettings{},
)
if err != nil {
t.Fatalf("UseMemoryDB: %v", err)
}
defer restore()
// A clean first fit for the shared-base scope with a clear winner.
if err := trainScope("*", rewarded("code", "zen5-coder", 30, 0.9), 1, 0.0); err != nil {
t.Fatalf("trainScope: %v", err)
}
rows, err := object.ListRouterTrainingLog("*", "", 0)
if err != nil {
t.Fatalf("list: %v", err)
}
if len(rows) != 1 {
t.Fatalf("want exactly one retrain row appended by trainScope, got %d", len(rows))
}
if rows[0].Owner != "*" || rows[0].Events <= 0 || rows[0].Version == "" {
t.Errorf("retrain row not populated from the fit: %+v", rows[0])
}
}
// TestSeedInternalTrainingConsent proves the boot seed flips OUR OWN reserved orgs
// to EXPLICIT training-contribution consent (so the EU per-request judge guard never
// suppresses internal traffic), is idempotent, and NEVER bulk-enables customer orgs.
func TestSeedInternalTrainingConsent(t *testing.T) {
t.Setenv("ROUTER_TRAIN_INTERNAL_ORGS", "")
restore, err := object.UseMemoryDB("file:seed_internal_consent_test?mode=memory&cache=shared", &object.OrgSettings{})
if err != nil {
t.Fatalf("UseMemoryDB: %v", err)
}
t.Cleanup(restore)
SeedInternalTrainingConsent()
// Every reserved internal org is now EXPLICITLY enabled (not merely opt-out default).
for _, org := range reservedInternalOrgs() {
if got := object.GetCachedOrgTrainingContribution(org); got != object.TrainingContributionEnabled {
t.Fatalf("internal org %q contribution = %q, want %q", org, got, object.TrainingContributionEnabled)
}
}
// A customer org is untouched — the seed never bulk-enables anyone else.
if got := object.GetCachedOrgTrainingContribution("some-customer-org"); got != object.TrainingContributionUnset {
t.Fatalf("customer org contribution = %q, want unset (never bulk-enabled)", got)
}
// Idempotent: a second run is a clean no-op and leaves the state enabled.
SeedInternalTrainingConsent()
if got := object.GetCachedOrgTrainingContribution("hanzo"); got != object.TrainingContributionEnabled {
t.Fatalf("after re-seed: hanzo = %q, want %q", got, object.TrainingContributionEnabled)
}
}
// TestDeployRouterPreferPreservesFields pins the F1-2 fix: the trainer's auto-deploy
// (deployRouterPreferTo) merges into RouterPrefer WITHOUT nulling any other field. dbx
// Update writes all columns, so the old fresh-row + preserve-list wiped the org's
// allowlist / dial / training consent on every gated deploy — silently re-opening the
// disabled-model set and resetting the cost dial.
func TestDeployRouterPreferPreservesFields(t *testing.T) {
restore, err := object.UseMemoryDB("file:deploy_prefer_preserve_test?mode=memory&cache=shared", &object.OrgSettings{})
if err != nil {
t.Fatalf("UseMemoryDB: %v", err)
}
defer restore()
bias := 0.3
if _, err := object.AddOrgSettings(&object.OrgSettings{
Owner: "acme",
RouterPrefer: object.JSONMap[[]string]{"default": {"a"}},
RouterEnabledModels: object.JSONMap[bool]{"gpt-4o": true},
RouterQualityBias: &bias,
TrainingContribution: object.TrainingContributionEnabled,
}); err != nil {
t.Fatalf("seed: %v", err)
}
if err := deployRouterPreferTo("acme", map[string][]string{"code": {"b"}}); err != nil {
t.Fatalf("deploy: %v", err)
}
got, err := object.GetOrgSettings("acme")
if err != nil || got == nil {
t.Fatalf("read: %v (nil=%v)", err, got == nil)
}
if !got.RouterEnabledModels["gpt-4o"] {
t.Fatalf("allowlist WIPED by deploy: %v", got.RouterEnabledModels)
}
if got.RouterQualityBias == nil || *got.RouterQualityBias != 0.3 {
t.Fatalf("dial WIPED by deploy: %v", got.RouterQualityBias)
}
if got.TrainingContribution != object.TrainingContributionEnabled {
t.Fatalf("training consent WIPED by deploy: %q", got.TrainingContribution)
}
if got.RouterPrefer["default"][0] != "a" || got.RouterPrefer["code"][0] != "b" {
t.Fatalf("prefer merge wrong (want default:[a]+code:[b]): %v", got.RouterPrefer)
}
}