Files
ai/controllers/routing_autoreward_test.go
daf1875df0 feat(router): dense auto-reward (quality×cost) + exploration floor — turn the flywheel (HIP-510) (#109)
Enso was reward-STARVED: ~1500 routing decisions/day but ~2 rewarded events, so every
train cycle kept incumbent and no arm was ever promoted. Close the loop:

DENSE AUTO-REWARD (dominant lever): recordUsage — the one choke point every served
request crosses — now scores each routed request and attaches it to its routing
decision (controllers/routing_autoreward.go). reward = quality × cost-efficiency ∈[0,1]:
quality = succeeded(2xx)+non-empty (error→0, empty→0.2); cost-efficiency normalizes the
arm's per-1k-token price against a reference, so among arms that SUCCEED the cheaper
keeps more reward → the router converges to the cheapest arm that still succeeds, never
the cheapest garbage. object.AttachRoutingRewardIfUnset is the implicit FLOOR — a later
explicit thumbs/judge (AttachRoutingReward) always overrides it. Fires before the
success filter so failures score 0. Dark-by-default (ROUTER_AUTOREWARD_ENABLED), async.

EXPLORATION FLOOR: an epsilon of auto traffic (ROUTER_EXPLORE_EPSILON) samples a
non-champion servable arm (router Client.Explore / Policy.Candidates), so the bandit
keeps earning reward on the whole pool and reacts to new models instead of collapsing
onto the champion. Engine path keeps its own UCB. 0 = pure exploit (safe default).

Tests: implicit quality/reward + cost-penalty ordering (cheap>expensive) + dark-by-default;
explore samples non-champion, epsilon rate, single-arm exploits, zero=pure-exploit. Gate
(≥20 rewards/arm + margin) untouched — no regression can ship.

Co-authored-by: zeekay <[email protected]>
2026-07-18 17:29:38 -07:00

92 lines
3.9 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import "testing"
// TestImplicitQuality locks the per-request quality proxy: succeeded-with-output is
// full credit, an empty 2xx is weak, an error is zero — so a failing arm can never
// out-reward a working one no matter how cheap.
func TestImplicitQuality(t *testing.T) {
cases := []struct {
name string
r *usageRecord
want float64
}{
{"success with output", &usageRecord{Status: "success", CompletionTokens: 42}, 1.0},
{"success empty completion", &usageRecord{Status: "success"}, 0.2},
{"success image (per-unit)", &usageRecord{Status: "success", ImageCount: 1}, 1.0},
{"error with tokens", &usageRecord{Status: "error", CompletionTokens: 10}, 0.0},
{"non-success status", &usageRecord{Status: "timeout"}, 0.0},
}
for _, c := range cases {
if got := implicitQuality(c.r); got != c.want {
t.Errorf("%s: quality = %v, want %v", c.name, got, c.want)
}
}
}
// TestImplicitRewardErrorIsZero proves an errored request contributes reward 0 — the
// bandit penalizes an arm that fails, independent of its price.
func TestImplicitRewardErrorIsZero(t *testing.T) {
if got := implicitRoutingReward(&usageRecord{Status: "error", TotalTokens: 1000}); got != 0 {
t.Fatalf("error reward = %v, want 0", got)
}
}
// TestCostPenaltyReducesReward proves reward = quality × cost-efficiency: for the
// SAME successful request, a cheaper arm (higher reference rate ⇒ this arm looks
// cheap ⇒ small cost penalty) earns MORE reward than an expensive one. This is the
// gradient that drives the router to the cheapest arm that still succeeds.
func TestCostPenaltyReducesReward(t *testing.T) {
r := &usageRecord{Status: "success", PromptTokens: 500, CompletionTokens: 500, TotalTokens: 1000, Model: "unknown-model-for-default-rate"}
t.Setenv("ROUTER_AUTOREWARD_LAMBDA", "0.5")
// Reference rate WAY above the arm's price → arm is "cheap" → keeps ~full reward.
t.Setenv("ROUTER_AUTOREWARD_COST_REF_CENTS_PER_1K", "100000")
cheap := implicitRoutingReward(r)
// Reference rate WAY below the arm's price → arm is "expensive" → penalized to q·(1−λ).
t.Setenv("ROUTER_AUTOREWARD_COST_REF_CENTS_PER_1K", "0.0001")
expensive := implicitRoutingReward(r)
if !(cheap > expensive) {
t.Fatalf("cost penalty not applied: cheap=%v must exceed expensive=%v", cheap, expensive)
}
if cheap <= 0.9 {
t.Errorf("a cheap successful arm should keep near-full reward, got %v", cheap)
}
if expensive > 0.6 {
t.Errorf("an expensive arm should be penalized toward q·(1−λ)=0.5, got %v", expensive)
}
}
// TestNormRoutingCostNoTokens: per-unit/unknown-token calls carry no token-cost
// signal, so they add no cost penalty (normCost 0) rather than a bogus one.
func TestNormRoutingCostNoTokens(t *testing.T) {
if got := normRoutingCost(&usageRecord{Status: "success"}); got != 0 {
t.Fatalf("normCost with no tokens = %v, want 0", got)
}
}
// TestEmitAutoRewardDarkByDefault: with the flag unset, scoring is a no-op — it must
// not touch the ledger (safe to wire into the always-run recordUsage choke point).
func TestEmitAutoRewardDarkByDefault(t *testing.T) {
t.Setenv("ROUTER_AUTOREWARD_ENABLED", "")
// Must not panic and must return without attempting an attach (nil-adapter safe
// regardless, but the gate is the contract).
emitAutoRoutingReward(&usageRecord{Status: "success", Owner: "acme", RequestID: "req-1", CompletionTokens: 10})
}