mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-21 02:36:49 +00:00
194 lines
7.5 KiB
Go
194 lines
7.5 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/object"
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
// routedWorld is routing on, a recorder for routing events, and a signed caller.
|
|
func routedWorld(t *testing.T) (token string, events *[]object.RoutingEvent) {
|
|
t.Helper()
|
|
prevCfg, prevSink, prevLookup := globalModelConfig, routingEventSink, orgAutoRoutingLookup
|
|
globalModelConfig = routerTestConfig(true)
|
|
var got []object.RoutingEvent
|
|
routingEventSink = func(e object.RoutingEvent) { got = append(got, e) }
|
|
orgAutoRoutingLookup = func(string) string { return object.AutoRoutingUnset }
|
|
t.Cleanup(func() { globalModelConfig, routingEventSink, orgAutoRoutingLookup = prevCfg, prevSink, prevLookup })
|
|
return mintUsageJWT(t, "acme", "ann"), &got
|
|
}
|
|
|
|
func modelOf(t *testing.T, body []byte) string {
|
|
t.Helper()
|
|
var b struct{ Model string }
|
|
if err := json.Unmarshal(body, &b); err != nil {
|
|
t.Fatalf("body is not JSON: %q", body)
|
|
}
|
|
return b.Model
|
|
}
|
|
|
|
// `auto` is resolved to the SKU that serves it before anything prices the request:
|
|
// the request names that SKU, says so in X-Routed-Model, and carries the decision the
|
|
// handler keeps. A caller who does not authenticate is not routed.
|
|
func TestAutoIsResolvedToItsSKUBeforeTheGate(t *testing.T) {
|
|
token, events := routedWorld(t)
|
|
for _, id := range []string{"auto", "zen-router", ""} {
|
|
*events = nil
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().Header.Set("Authorization", "Bearer "+token)
|
|
body := `{"model":"` + id + `","stream":true,"messages":[{"role":"user","content":"hello there"}]}`
|
|
if id == "" {
|
|
body = `{"stream":true,"messages":[{"role":"user","content":"hello there"}]}` // no model: the router chooses
|
|
}
|
|
c.Fiber().Request().SetBody([]byte(body))
|
|
c.RouteAuto()
|
|
got := modelOf(t, c.Body())
|
|
if got == "" || IsAutoModel(got) || resolveModelRoute(got) == nil {
|
|
t.Fatalf("%s: the request names %q, want a routed, servable SKU", id, got)
|
|
}
|
|
if h := string(c.Fiber().Response().Header.Peek(RoutedModelHeader)); h != got {
|
|
t.Errorf("%s: %s = %q, request names %q", id, RoutedModelHeader, h, got)
|
|
}
|
|
r, ok := c.Locals(autoRoutedKey).(autoRouted)
|
|
if !ok || r.routed != got || r.requestId == "" {
|
|
t.Errorf("%s: decision left for the handler = %+v", id, r)
|
|
}
|
|
var b struct{ Stream bool }
|
|
_ = json.Unmarshal(c.Body(), &b)
|
|
if !b.Stream {
|
|
t.Errorf("%s: the rewrite lost stream: %s", id, c.Body())
|
|
}
|
|
if len(*events) != 1 || (*events)[0].RoutedModel != got || (*events)[0].RequestId != r.requestId || (*events)[0].RequestedModel != id {
|
|
t.Errorf("%s: routing events %+v", id, *events)
|
|
}
|
|
}
|
|
|
|
*events = nil
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody([]byte(`{"model":"auto","messages":[{"role":"user","content":"hello"}]}`))
|
|
c.RouteAuto()
|
|
if got := modelOf(t, c.Body()); got != "auto" || len(*events) != 0 {
|
|
t.Errorf("an unauthenticated request was routed to %q (%d events)", got, len(*events))
|
|
}
|
|
|
|
c = visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().Header.Set("Authorization", "Bearer "+token)
|
|
c.Fiber().Request().SetBody([]byte(`{"model":"gpt-4o","messages":[]}`))
|
|
c.RouteAuto()
|
|
if len(*events) != 0 || c.Locals(autoRoutedKey) != nil {
|
|
t.Error("a concrete model was routed")
|
|
}
|
|
}
|
|
|
|
// A /v1/responses body, zstd or plain, is read in its own dialect and rewritten in
|
|
// it; a zstd body is expanded once so the gate reads the model it names.
|
|
func TestAResponsesRequestIsRoutedInItsOwnDialect(t *testing.T) {
|
|
token, _ := routedWorld(t)
|
|
enc, err := zstd.NewWriter(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, tc := range []struct {
|
|
name, model string
|
|
zstd bool
|
|
want string // "routed": whatever SKU the router picks
|
|
}{
|
|
{"auto, zstd", "auto", true, "routed"},
|
|
{"auto, plain", "auto", false, "routed"},
|
|
} {
|
|
body := []byte(`{"model":"` + tc.model + `","input":"hello there","stream":false}`)
|
|
c := visit(http.MethodPost, "/v1/responses")
|
|
c.Fiber().Request().Header.Set("Authorization", "Bearer "+token)
|
|
if tc.zstd {
|
|
c.Fiber().Request().Header.Set("Content-Encoding", "zstd")
|
|
body = enc.EncodeAll(body, nil)
|
|
}
|
|
c.Fiber().Request().SetBody(body)
|
|
c.RouteAuto()
|
|
if got := modelOf(t, c.Body()); IsAutoModel(got) || resolveModelRoute(got) == nil {
|
|
t.Errorf("%s: model %q, want a routed, servable SKU", tc.name, got)
|
|
}
|
|
if ce := string(c.Fiber().Request().Header.Peek("Content-Encoding")); ce != "" {
|
|
t.Errorf("%s: Content-Encoding %q left on a plain body", tc.name, ce)
|
|
}
|
|
var b struct{ Input string }
|
|
_ = json.Unmarshal(c.Body(), &b)
|
|
if b.Input != "hello there" {
|
|
t.Errorf("%s: the Responses body lost its input: %s", tc.name, c.Body())
|
|
}
|
|
}
|
|
}
|
|
|
|
// On a pod that has just started, a family SKU is priced from discovery on the first
|
|
// read: enso-auto costs nothing and enso-flash costs what the family says, rather
|
|
// than the unknown-model default that refused a free org plan_required.
|
|
func TestAFamilySKUIsPricedFromDiscoveryOnAColdPod(t *testing.T) {
|
|
var asked int
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/v1/models" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
asked++
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = io.WriteString(w, `{"data":[`+
|
|
`{"id":"enso-auto","owned_by":"hanzo","context_window":1000000,"pricing":{"input":"0","output":"0"}},`+
|
|
`{"id":"enso-flash","owned_by":"hanzo","context_window":1000000,"funding":"prepaid","pricing":{"input":"0.42","output":"1.5"}}]}`)
|
|
}))
|
|
defer srv.Close()
|
|
|
|
restore(t, ensoFam)
|
|
t.Cleanup(func() { ensoFam.mu.Lock(); ensoFam.tried = time.Time{}; ensoFam.mu.Unlock() })
|
|
ensoFam.providerFn = func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "enso", Category: "Model", Type: "Enso", State: "Active", ProviderUrl: srv.URL}
|
|
}
|
|
ensoFam.mu.Lock()
|
|
ensoFam.byID, ensoFam.ids, ensoFam.loaded, ensoFam.fetchedAt, ensoFam.tried = nil, nil, false, time.Time{}, time.Time{}
|
|
ensoFam.mu.Unlock()
|
|
|
|
if !ModelCostsNothing("enso-auto", "acme") {
|
|
t.Fatal("enso-auto on a cold pod is priced: a free org would be refused plan_required")
|
|
}
|
|
if ModelCostsNothing("enso-flash", "acme") {
|
|
t.Error("enso-flash on a cold pod costs nothing")
|
|
}
|
|
if p, ok := getModelPriceForOrgOK("enso-flash", "acme"); !ok || p.InputPerMillion != 0.42 || p.OutputPerMillion != 1.5 {
|
|
t.Errorf("enso-flash price %+v (ok=%v), want the discovered 0.42/1.50", p, ok)
|
|
}
|
|
if asked != 1 {
|
|
t.Errorf("discovery was asked %d times, want once", asked)
|
|
}
|
|
|
|
// A family that cannot be reached is asked again only after warmRetry.
|
|
srv.Close()
|
|
ensoFam.mu.Lock()
|
|
ensoFam.byID, ensoFam.ids, ensoFam.loaded, ensoFam.tried = nil, nil, false, time.Time{}
|
|
ensoFam.mu.Unlock()
|
|
start := time.Now()
|
|
_ = ModelCostsNothing("enso-auto", "acme")
|
|
_ = ModelCostsNothing("enso-auto", "acme")
|
|
if time.Since(start) > 5*time.Second {
|
|
t.Error("an unreachable family was asked on every read")
|
|
}
|
|
}
|