mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
Modules declare go 1.27.1 and the build images follow. go fix applied the modernizers; omitzero is not applied, since it changes what the JSON says. Co-authored-by: Hanzo Dev <[email protected]>
239 lines
8.7 KiB
Go
239 lines
8.7 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"encoding/json"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/object"
|
|
)
|
|
|
|
// fixedPrices is a deterministic price index for tests: opus is the premium
|
|
// (baseline) arm, mini is cheap, local is free/unknown (excluded from cost).
|
|
func fixedPrices(model string) float64 {
|
|
switch model {
|
|
case "opus":
|
|
return 30
|
|
case "mid":
|
|
return 10
|
|
case "mini":
|
|
return 2
|
|
default:
|
|
return 0 // local / unknown → not priced
|
|
}
|
|
}
|
|
|
|
func ev(minsAgo int, task, model, source string, conf float64, reward *float64) *object.RoutingEvent {
|
|
e := &object.RoutingEvent{
|
|
CreatedTime: time.Now().UTC().Add(-time.Duration(minsAgo) * time.Minute).Format(time.RFC3339),
|
|
Task: task,
|
|
RoutedModel: model,
|
|
Source: source,
|
|
Confidence: conf,
|
|
}
|
|
if reward != nil {
|
|
e.Reward = *reward
|
|
e.RewardedTime = time.Now().UTC().Format(time.RFC3339)
|
|
}
|
|
return e
|
|
}
|
|
|
|
//go:fix inline
|
|
func f(v float64) *float64 { return new(v) }
|
|
|
|
// testTuned is a stand-in for the trainer-published prefer table: the picks the
|
|
// flywheel has LEARNED, against which learned_share is measured.
|
|
var testTuned = map[string][]string{"code": {"mini"}, "chat": {"mid"}}
|
|
|
|
func TestComputeRouterStats_Distributions(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
start := now.Add(-24 * time.Hour)
|
|
events := []*object.RoutingEvent{
|
|
ev(10, "code", "mini", "engine", 0.9, new(float64(1))),
|
|
ev(20, "code", "opus", "heuristic", 0, nil),
|
|
ev(30, "chat", "mini", "engine", 0.8, new(float64(0))),
|
|
ev(40, "chat", "mid", "engine", 0.7, nil),
|
|
}
|
|
s := computeRouterStats(events, fixedPrices, start, now, scopeOrg, "acme", true, testTuned)
|
|
|
|
if s.Window.Events != 4 {
|
|
t.Fatalf("events = %d, want 4", s.Window.Events)
|
|
}
|
|
if s.ByModel["mini"] != 2 || s.ByModel["opus"] != 1 || s.ByModel["mid"] != 1 {
|
|
t.Fatalf("by_model wrong: %+v", s.ByModel)
|
|
}
|
|
if s.ByTask["code"].Events != 2 || s.ByTask["code"].Models["mini"] != 1 || s.ByTask["code"].Models["opus"] != 1 {
|
|
t.Fatalf("by_task code wrong: %+v", s.ByTask["code"])
|
|
}
|
|
// learned share = the 2 events served by testTuned's pick for their task
|
|
// (code→mini, chat→mid) out of 4; "opus" on code and "mini" on chat are not the
|
|
// tuned pick, so they do not count.
|
|
if s.Quality.LearnedShare != 0.5 {
|
|
t.Fatalf("learned_share = %v, want 0.5", s.Quality.LearnedShare)
|
|
}
|
|
// by_source is the honest provenance histogram behind that ratio.
|
|
if s.BySource["engine"] != 3 || s.BySource["heuristic"] != 1 {
|
|
t.Fatalf("by_source wrong: %+v", s.BySource)
|
|
}
|
|
// reward rate = mean(1, 0) over the two rewarded events = 0.5
|
|
if s.Quality.RewardedEvents != 2 || s.Quality.RewardRate != 0.5 {
|
|
t.Fatalf("reward: rate=%v n=%d, want 0.5/2", s.Quality.RewardRate, s.Quality.RewardedEvents)
|
|
}
|
|
if s.Quality.ShadowAgreement != nil {
|
|
t.Fatalf("shadow_agreement should be nil until the ledger carries a shadow pick")
|
|
}
|
|
}
|
|
|
|
func TestComputeRouterStats_CostSaved(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
start := now.Add(-24 * time.Hour)
|
|
// 3 priced events routed to mini(2), mid(10), opus(30); baseline=opus(30).
|
|
// counterfactual = 30 each; routed sum = 42; saved = 90-42 = 48.
|
|
events := []*object.RoutingEvent{
|
|
ev(1, "chat", "mini", "engine", 0.9, nil),
|
|
ev(2, "reasoning", "mid", "engine", 0.9, nil),
|
|
ev(3, "reasoning", "opus", "heuristic", 0, nil),
|
|
ev(4, "chat", "local", "engine", 0.9, nil), // unpriced → excluded
|
|
}
|
|
s := computeRouterStats(events, fixedPrices, start, now, scopeOrg, "acme", true, testTuned)
|
|
if s.Cost == nil {
|
|
t.Fatal("cost should be present")
|
|
}
|
|
if s.Cost.BaselineModel != "opus" {
|
|
t.Fatalf("baseline = %q, want opus", s.Cost.BaselineModel)
|
|
}
|
|
if s.Cost.PricedEvents != 3 {
|
|
t.Fatalf("priced_events = %d, want 3 (local excluded)", s.Cost.PricedEvents)
|
|
}
|
|
// saved_pct = 48/90*100 = 53.3333
|
|
if s.Cost.SavedPct < 53.3 || s.Cost.SavedPct > 53.34 {
|
|
t.Fatalf("saved_pct = %v, want ~53.33", s.Cost.SavedPct)
|
|
}
|
|
if s.Cost.CumulativeSavedIndex != 48 {
|
|
t.Fatalf("cumulative_saved = %v, want 48", s.Cost.CumulativeSavedIndex)
|
|
}
|
|
if s.Cost.RoutedIndex == nil || *s.Cost.RoutedIndex != 14 { // (2+10+30)/3
|
|
t.Fatalf("routed_index = %v, want 14", s.Cost.RoutedIndex)
|
|
}
|
|
if s.Cost.CounterfactualIndex == nil || *s.Cost.CounterfactualIndex != 30 {
|
|
t.Fatalf("counterfactual_index = %v, want 30", s.Cost.CounterfactualIndex)
|
|
}
|
|
}
|
|
|
|
// The public platform scope must NEVER carry absolute $ levels or org identity, but
|
|
// SHOULD carry the safe ratio AND the REAL model ids (owner product decision: the
|
|
// world.hanzo.ai widget shows the true roster so users see which models win).
|
|
func TestComputeRouterStats_PublicScopeRedaction(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
start := now.Add(-24 * time.Hour)
|
|
events := []*object.RoutingEvent{
|
|
ev(1, "chat", "mini", "engine", 0.9, nil),
|
|
ev(2, "reasoning", "opus", "heuristic", 0, nil),
|
|
}
|
|
s := computeRouterStats(events, fixedPrices, start, now, scopePlatform, "acme", false, testTuned)
|
|
|
|
if s.Org != "" {
|
|
t.Fatalf("platform scope must not leak org, got %q", s.Org)
|
|
}
|
|
if s.Cost == nil {
|
|
t.Fatal("platform scope still exposes the safe cost ratio")
|
|
}
|
|
if s.Cost.RoutedIndex != nil || s.Cost.CounterfactualIndex != nil {
|
|
t.Fatalf("platform scope must drop absolute $ levels, got routed=%v cf=%v", s.Cost.RoutedIndex, s.Cost.CounterfactualIndex)
|
|
}
|
|
// Real ids now surface on the platform scope — the baseline + by_model carry the
|
|
// true model ids (opus/mini), never opaque arm-N labels.
|
|
if s.Cost.SavedPct <= 0 || s.Cost.BaselineModel != "opus" {
|
|
t.Fatalf("platform scope shows the REAL baseline id opus, got %+v", s.Cost)
|
|
}
|
|
if s.ByModel["opus"] != 1 || s.ByModel["mini"] != 1 {
|
|
t.Fatalf("expected real ids opus=1, mini=1, got %+v", s.ByModel)
|
|
}
|
|
for id := range s.ByModel {
|
|
if id == "arm-1" || id == "arm-2" || id == "arm-other" {
|
|
t.Fatalf("platform by_model must NOT carry arm-N labels anymore: %+v", s.ByModel)
|
|
}
|
|
}
|
|
if s.ByTask["reasoning"].Models["opus"] != 1 {
|
|
t.Fatalf("platform by_task must show the real id opus, got %+v", s.ByTask["reasoning"])
|
|
}
|
|
|
|
// Serialized, the redacted level fields must be ABSENT (omitempty on nil ptr).
|
|
b, _ := json.Marshal(s)
|
|
var m map[string]any
|
|
_ = json.Unmarshal(b, &m)
|
|
cost := m["cost"].(map[string]any)
|
|
if _, ok := cost["routed_index"]; ok {
|
|
t.Fatalf("routed_index must not serialize in platform scope: %s", b)
|
|
}
|
|
if _, ok := cost["counterfactual_index"]; ok {
|
|
t.Fatalf("counterfactual_index must not serialize in platform scope: %s", b)
|
|
}
|
|
if _, ok := m["org"]; ok {
|
|
t.Fatalf("org must not serialize in platform scope: %s", b)
|
|
}
|
|
}
|
|
|
|
func TestComputeRouterStats_Throughput(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
start := now.Add(-24 * time.Hour)
|
|
// two events in the most recent hour, one ~12h ago.
|
|
events := []*object.RoutingEvent{
|
|
ev(5, "chat", "mini", "engine", 0.9, nil),
|
|
ev(20, "chat", "mini", "engine", 0.9, nil),
|
|
ev(12*60, "chat", "mid", "engine", 0.9, nil),
|
|
}
|
|
s := computeRouterStats(events, fixedPrices, start, now, scopeOrg, "acme", true, testTuned)
|
|
if len(s.Throughput.PerHour) != routerStatsBuckets {
|
|
t.Fatalf("per_hour len = %d, want %d", len(s.Throughput.PerHour), routerStatsBuckets)
|
|
}
|
|
if s.Throughput.TotalWindow != 3 {
|
|
t.Fatalf("total_window = %d, want 3", s.Throughput.TotalWindow)
|
|
}
|
|
// most recent bucket holds the two fresh events.
|
|
if got := s.Throughput.PerHour[routerStatsBuckets-1]; got != 2 {
|
|
t.Fatalf("latest bucket = %d, want 2", got)
|
|
}
|
|
var sum int
|
|
for _, v := range s.Throughput.PerHour {
|
|
sum += v
|
|
}
|
|
if sum != 3 {
|
|
t.Fatalf("throughput buckets sum = %d, want 3", sum)
|
|
}
|
|
}
|
|
|
|
func TestComputeRouterStats_Empty(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
start := now.Add(-24 * time.Hour)
|
|
s := computeRouterStats(nil, fixedPrices, start, now, scopeOrg, "acme", true, testTuned)
|
|
if s.Cost != nil {
|
|
t.Fatalf("no priced events → cost nil, got %+v", s.Cost)
|
|
}
|
|
if s.Window.Events != 0 || s.Quality.RewardRate != 0 || s.Quality.LearnedShare != 0 {
|
|
t.Fatalf("empty aggregate should be all-zero, got %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestSortedModelShare(t *testing.T) {
|
|
got := sortedModelShare(map[string]int{"a": 1, "b": 3, "c": 3})
|
|
// desc by count, ties by id → b,c (both 3, b<c), then a
|
|
if len(got) != 3 || got[0] != "b" || got[1] != "c" || got[2] != "a" {
|
|
t.Fatalf("sorted share = %v, want [b c a]", got)
|
|
}
|
|
}
|