mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
Additive (omitempty) context_window on modelInfo, populated from the route (config path) and the family's discovered MaxCtx (enso/zen path). Lets OpenAI/Codex clients size a 1M-token enso context instead of guessing a smaller default; ai already forwarded enso verbatim with no cap, this just makes the window introspectable.
347 lines
12 KiB
Go
347 lines
12 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
// These tests pin the ADDITIVE enrichment of the /v1/models response shape
|
|
// (modelInfo): the OpenAI-compatible core stays byte-stable while provider +
|
|
// pricing are added omitempty, sourced only from data ai already holds. Both
|
|
// builders are covered — ListModels() (YAML/prod) and listAvailableModels()
|
|
// (static fallback) — so they emit the identical shape. A branded (owned_by)
|
|
// model surfaces its public owner in owned_by; the serving provider is omitted
|
|
// (publicProvider). These are the internal provider names that, for a branded
|
|
// model, must not appear as the public "provider" — see hip-00NN.
|
|
var upstreamProviders = map[string]bool{"do-ai": true, "fireworks": true, "openai-direct": true}
|
|
|
|
func indexModels(models []modelInfo) map[string]modelInfo {
|
|
m := make(map[string]modelInfo, len(models))
|
|
for _, mi := range models {
|
|
m[mi.ID] = mi
|
|
}
|
|
return m
|
|
}
|
|
|
|
func TestModelListEnvelopeSupportsOpenAIAndCodex(t *testing.T) {
|
|
raw, err := json.Marshal(modelListEnvelope([]modelInfo{{
|
|
ID: "zen4-coder", Object: "model", Created: 1, OwnedBy: "hanzo", Premium: true,
|
|
}}))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var got struct {
|
|
Object string `json:"object"`
|
|
Data []modelInfo `json:"data"`
|
|
Models []json.RawMessage `json:"models"`
|
|
}
|
|
if err := json.Unmarshal(raw, &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Object != "list" || len(got.Data) != 1 || got.Data[0].ID != "zen4-coder" {
|
|
t.Fatalf("OpenAI catalog changed: %#v", got)
|
|
}
|
|
if got.Models == nil || len(got.Models) != 0 {
|
|
t.Fatalf("Codex fallback catalog must be a present empty array: %#v", got.Models)
|
|
}
|
|
}
|
|
|
|
// jsonKeys marshals a modelInfo and returns the set of top-level JSON keys, so
|
|
// tests can assert presence/omission of omitempty fields on the real wire shape.
|
|
func jsonKeys(t *testing.T, mi modelInfo) map[string]bool {
|
|
t.Helper()
|
|
raw, err := json.Marshal(mi)
|
|
if err != nil {
|
|
t.Fatalf("marshal: %v", err)
|
|
}
|
|
var m map[string]json.RawMessage
|
|
if err := json.Unmarshal(raw, &m); err != nil {
|
|
t.Fatalf("unmarshal: %v", err)
|
|
}
|
|
keys := make(map[string]bool, len(m))
|
|
for k := range m {
|
|
keys[k] = true
|
|
}
|
|
return keys
|
|
}
|
|
|
|
// assertCoreUnchanged verifies the five OpenAI-compatible core keys are present
|
|
// and named exactly as external consumers (Claude Code, Codex, OpenAI SDK) parse
|
|
// them. A rename or drop here would break those clients.
|
|
func assertCoreUnchanged(t *testing.T, mi modelInfo) {
|
|
t.Helper()
|
|
keys := jsonKeys(t, mi)
|
|
for _, k := range []string{"id", "object", "created", "owned_by", "premium"} {
|
|
if !keys[k] {
|
|
t.Errorf("core field %q missing from JSON for %q", k, mi.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestModelInfoOmitEmpty proves the additive fields vanish when empty (so an
|
|
// existing consumer sees exactly the legacy core), while premium — part of the
|
|
// stable core — is always emitted even when false.
|
|
func TestModelInfoOmitEmpty(t *testing.T) {
|
|
keys := jsonKeys(t, modelInfo{
|
|
ID: "sample", Object: "model", Created: 1, OwnedBy: "do-ai", Premium: false,
|
|
})
|
|
for _, absent := range []string{"provider", "context_window", "pricing"} {
|
|
if keys[absent] {
|
|
t.Errorf("bare modelInfo must omit %q", absent)
|
|
}
|
|
}
|
|
if !keys["premium"] {
|
|
t.Error("premium must always be present (no omitempty)")
|
|
}
|
|
}
|
|
|
|
// TestModelInfoContextWindowSurfaced proves the additive context_window field is
|
|
// emitted (and honestly valued) once ai holds the datum — so an OpenAI-compatible
|
|
// client can size a 1M-token enso context instead of guessing a smaller default.
|
|
func TestModelInfoContextWindowSurfaced(t *testing.T) {
|
|
raw, err := json.Marshal(modelInfo{
|
|
ID: "enso", Object: "model", Created: 1, OwnedBy: "hanzo", Premium: true,
|
|
ContextWindow: 1_000_000,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var got modelInfo
|
|
if err := json.Unmarshal(raw, &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.ContextWindow != 1_000_000 {
|
|
t.Errorf("enso context_window: want 1000000, got %d", got.ContextWindow)
|
|
}
|
|
if !jsonKeys(t, modelInfo{ID: "enso", ContextWindow: 1_000_000})["context_window"] {
|
|
t.Error("context_window must be present when set")
|
|
}
|
|
}
|
|
|
|
// TestPricingInfoZeroOmitted proves the never-fabricate guard: a present but
|
|
// all-zero price yields no pricing block.
|
|
func TestPricingInfoZeroOmitted(t *testing.T) {
|
|
if pricingInfo(modelPrice{}, false) != nil {
|
|
t.Error("absent pricing must project to nil")
|
|
}
|
|
if pricingInfo(modelPrice{InputPerMillion: 0, OutputPerMillion: 0}, true) != nil {
|
|
t.Error("all-zero pricing must project to nil (no meaningless {0,0})")
|
|
}
|
|
got := pricingInfo(modelPrice{InputPerMillion: 1.25, OutputPerMillion: 5.00}, true)
|
|
if got == nil || got.Input != 1.25 || got.Output != 5.00 {
|
|
t.Errorf("real pricing must project faithfully, got %+v", got)
|
|
}
|
|
}
|
|
|
|
// TestPublicProviderBrandGate is the unit-level guard for the leak fix: branded
|
|
// models (ownedBy set) never expose the upstream provider; unbranded ones do.
|
|
func TestPublicProviderBrandGate(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
route modelRoute
|
|
want string
|
|
}{
|
|
{"unbranded passthrough", modelRoute{providerName: "do-ai"}, "do-ai"},
|
|
{"zen brand (hanzo)", modelRoute{providerName: "do-ai", ownedBy: "hanzo"}, ""},
|
|
{"fireworks brand", modelRoute{providerName: "fireworks", ownedBy: "hanzo"}, ""},
|
|
{"openai embedding", modelRoute{providerName: "openai-direct", ownedBy: "openai"}, ""},
|
|
}
|
|
for _, c := range cases {
|
|
if got := publicProvider(c.route); got != c.want {
|
|
t.Errorf("%s: publicProvider = %q, want %q", c.name, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestListModelsRichConfigPath covers the prod path: ModelConfig.ListModels()
|
|
// built from YAML. It exercises both an unbranded model (real provider exposed)
|
|
// and two branded classes (provider OMITTED, never the upstream), plus pricing
|
|
// present/omitted.
|
|
func TestListModelsRichConfigPath(t *testing.T) {
|
|
const richYAML = `
|
|
version: 1
|
|
features:
|
|
live_mode: false
|
|
models:
|
|
gpt-4o:
|
|
provider: do-ai
|
|
upstream: openai-gpt-4o
|
|
pricing: { input: 2.50, output: 10.00 }
|
|
llama-free:
|
|
provider: do-ai
|
|
upstream: some-llama
|
|
zen4:
|
|
provider: do-ai
|
|
upstream: glm-5
|
|
premium: true
|
|
owned_by: hanzo
|
|
pricing: { input: 3.00, output: 9.60 }
|
|
text-embedding-3-small:
|
|
provider: openai-direct
|
|
upstream: text-embedding-3-small
|
|
owned_by: openai
|
|
`
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "models.yaml")
|
|
if err := os.WriteFile(path, []byte(richYAML), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mc := &ModelConfig{
|
|
routes: make(map[string]modelRoute),
|
|
pricing: make(map[string]modelPrice),
|
|
stopCh: make(chan struct{}),
|
|
}
|
|
if err := mc.loadFromFile(path); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
byID := indexModels(mc.ListModels())
|
|
|
|
// (a) OpenAI core fields unchanged for a sample (unbranded) model, and the
|
|
// real serving provider IS exposed for unbranded passthroughs.
|
|
gpt, ok := byID["gpt-4o"]
|
|
if !ok {
|
|
t.Fatal("gpt-4o missing from listing")
|
|
}
|
|
if gpt.Object != "model" || gpt.OwnedBy != "do-ai" || gpt.Created == 0 {
|
|
t.Errorf("gpt-4o core wrong: %+v", gpt)
|
|
}
|
|
if gpt.Provider != "do-ai" {
|
|
t.Errorf("unbranded gpt-4o should expose provider do-ai, got %q", gpt.Provider)
|
|
}
|
|
if gpt.Pricing == nil || gpt.Pricing.Input != 2.50 || gpt.Pricing.Output != 10.00 {
|
|
t.Errorf("gpt-4o pricing want {2.50,10.00}, got %+v", gpt.Pricing)
|
|
}
|
|
assertCoreUnchanged(t, gpt)
|
|
|
|
// (b) unbranded, no pricing → provider present, pricing OMITTED.
|
|
free := byID["llama-free"]
|
|
if free.Provider != "do-ai" {
|
|
t.Errorf("llama-free provider want do-ai, got %q", free.Provider)
|
|
}
|
|
if free.Pricing != nil {
|
|
t.Errorf("llama-free pricing must be omitted, got %+v", *free.Pricing)
|
|
}
|
|
if jsonKeys(t, free)["pricing"] {
|
|
t.Error("llama-free JSON must not contain pricing")
|
|
}
|
|
|
|
// (c) BRANDED zen (owned_by hanzo): provider OMITTED (upstream do-ai never
|
|
// leaks), owner still hanzo, pricing still real.
|
|
zen, ok := byID["zen4"]
|
|
if !ok {
|
|
t.Fatal("zen4 missing from listing")
|
|
}
|
|
if zen.Provider != "" {
|
|
t.Errorf("zen4 must NOT expose a provider (leak), got %q", zen.Provider)
|
|
}
|
|
if zen.OwnedBy != "hanzo" {
|
|
t.Errorf("zen4 owned_by want hanzo, got %q", zen.OwnedBy)
|
|
}
|
|
if zen.Pricing == nil || zen.Pricing.Input != 3.00 || zen.Pricing.Output != 9.60 {
|
|
t.Errorf("zen4 pricing want {3.00,9.60}, got %+v", zen.Pricing)
|
|
}
|
|
zenKeys := jsonKeys(t, zen)
|
|
if zenKeys["provider"] {
|
|
t.Error("zen4 JSON must NOT contain provider")
|
|
}
|
|
if !zenKeys["pricing"] {
|
|
t.Error("zen4 JSON should still contain pricing")
|
|
}
|
|
|
|
// (c) BRANDED embedding (owned_by openai): provider OMITTED (openai-direct
|
|
// never leaks), owner openai.
|
|
emb, ok := byID["text-embedding-3-small"]
|
|
if !ok {
|
|
t.Fatal("text-embedding-3-small missing from listing")
|
|
}
|
|
if emb.Provider != "" {
|
|
t.Errorf("embedding must NOT expose provider (leak openai-direct), got %q", emb.Provider)
|
|
}
|
|
if emb.OwnedBy != "openai" {
|
|
t.Errorf("embedding owned_by want openai, got %q", emb.OwnedBy)
|
|
}
|
|
if jsonKeys(t, emb)["provider"] {
|
|
t.Error("embedding JSON must NOT contain provider")
|
|
}
|
|
}
|
|
|
|
// TestListModelsRichStaticPath covers the fallback path: listAvailableModels()
|
|
// reading the real static route + pricing tables. gpt-4o is unbranded (provider
|
|
// exposed + priced); glm-5 is unbranded with no static price; zen4 is branded
|
|
// (provider omitted).
|
|
func TestListModelsRichStaticPath(t *testing.T) {
|
|
if GetModelConfig() != nil {
|
|
t.Skip("global model config set by another test; static path not exercised")
|
|
}
|
|
byID := indexModels(listAvailableModels())
|
|
|
|
gpt, ok := byID["gpt-4o"]
|
|
if !ok {
|
|
t.Fatal("gpt-4o missing from static listing")
|
|
}
|
|
if gpt.Object != "model" || gpt.OwnedBy != "do-ai" {
|
|
t.Errorf("gpt-4o core wrong: %+v", gpt)
|
|
}
|
|
if gpt.Provider != "do-ai" {
|
|
t.Errorf("unbranded gpt-4o should expose provider do-ai, got %q", gpt.Provider)
|
|
}
|
|
if gpt.Pricing == nil || gpt.Pricing.Input != 2.50 || gpt.Pricing.Output != 10.00 {
|
|
t.Errorf("gpt-4o static pricing want {2.50,10.00}, got %+v", gpt.Pricing)
|
|
}
|
|
|
|
glm, ok := byID["glm-5"]
|
|
if !ok {
|
|
t.Fatal("glm-5 missing from static listing")
|
|
}
|
|
if glm.Provider != "do-ai" {
|
|
t.Errorf("unbranded glm-5 provider want do-ai, got %q", glm.Provider)
|
|
}
|
|
if glm.Pricing != nil {
|
|
t.Errorf("glm-5 has no static pricing; must be omitted, got %+v", *glm.Pricing)
|
|
}
|
|
|
|
zen, ok := byID["zen4"]
|
|
if !ok {
|
|
t.Fatal("zen4 missing from static listing")
|
|
}
|
|
if zen.Provider != "" {
|
|
t.Errorf("branded zen4 must NOT expose provider, got %q", zen.Provider)
|
|
}
|
|
if zen.OwnedBy != "hanzo" {
|
|
t.Errorf("zen4 owned_by want hanzo, got %q", zen.OwnedBy)
|
|
}
|
|
}
|
|
|
|
// TestNoStaticBrandedModels is the invariant after the Zen family moved to the
|
|
// zen service: ai's static table holds NO branded (owned_by) model. The Zen
|
|
// family — the only branded models — is discovered from zen's /v1/models and
|
|
// served through the zen provider; zen's discovery surface carries no upstream,
|
|
// so that is where the brand-leak guard now lives. A branded model reappearing
|
|
// here means a zen SKU was re-hardcoded into ai, the exact drift this removed.
|
|
func TestNoStaticBrandedModels(t *testing.T) {
|
|
if GetModelConfig() != nil {
|
|
t.Skip("global model config set by another test; static path not exercised")
|
|
}
|
|
for _, m := range listAvailableModels() {
|
|
if route := modelRoutes[m.ID]; route.ownedBy != "" {
|
|
t.Errorf("static table has branded model %q (owned_by=%s) — zen SKUs must be dynamic, not hardcoded", m.ID, route.ownedBy)
|
|
}
|
|
}
|
|
}
|