Files
ai/controllers/doai_enable_test.go
zeekay bd98d0f7f2 feat(ai/models): enable + capability-declare the full do-ai premium chat lineup
Expose the top DigitalOcean GenAI chat models to customers on /v1/models — the
Claude family (opus-4.8/4.7, sonnet-5, sonnet-4.6, haiku-4.5, fable-5), the
OpenAI family (gpt-5.6 luna/sol/terra, gpt-5.5, gpt-5.4, gpt-4o, gpt-4o-mini,
o3), and the strong open models (deepseek-v4-pro, llama-4-maverick, qwen3.5-397b,
glm-5.2, kimi-k2.6). Each was hidden or missing in conf/models.yaml (the runtime
source of truth), so /v1/models listed ~35 models and none of the premium chat
SKUs. Now 55 are listed. All route to the do-ai provider, so they meter under
do-ai unchanged.

Every model + capability verified LIVE against inference.do-ai.run:
- context_window + max_output_tokens from the DO catalog (authoritative).
- supports_vision / supports_tools set ONLY where the probe confirmed it:
  * gpt-5.6 (all three variants) REJECTS tool calls (HTTP 400) -> tools off.
  * claude-sonnet-5, deepseek-v4-pro, glm-5.2, kimi-k2.6, qwen3.5 REJECT image
    input (HTTP 400) -> vision off.
  * claude-sonnet-4-6 re-pointed to its REAL anthropic-claude-4.6-sonnet upstream
    (now served on this account; previously degraded to 4.5 while 4.6 403'd) and
    the stale lineup-test expectation + comments updated to match.

Schema: add omitempty max_output_tokens / supports_vision / supports_tools to
modelInfo (the /v1/models shape) + ModelDef (YAML) + modelRoute, wired through
ListModels and the static fallback. Additive: a legacy consumer is unaffected,
and a "no" is expressed by absence, never a fabricated false.

Router: append the newly-enabled models to the conf `prefer` lists as candidates
(zen champions stay first, so the default/cost is unchanged; explore floor and
the learned engine can now reach them). Only capability-matched ids per task —
confirmed-vision-only in `vision`, no gpt-5.6 in tool-heavy code/reasoning.

Drive-by: drop a vet-flagged self-assignment in object/finetune_k8s.go.

Verified: go build ./... clean; go vet ./controllers/ ./object/ clean;
go test ./controllers/ ./object/ -run 'Model|Provider|Catalog|Route' green, plus
new TestDOAIEnabledModelsVisibleAndCapable / TestModelInfoCapabilityFieldsOmitEmpty.

Follow-up: sync the prod cloud-api-models ConfigMap
(universe/infra/k8s/cloud/models.yaml) — bootstrap reads conf/models.yaml, but
prod mounts that ConfigMap over it.
2026-07-18 23:29:45 -07:00

133 lines
5.2 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import "testing"
// loadRealConfig loads the shipped conf/models.yaml into a LOCAL ModelConfig
// without touching the global singleton, so it never perturbs the static-path
// tests that skip when GetModelConfig() != nil.
func loadRealConfig(t *testing.T) *ModelConfig {
t.Helper()
mc := &ModelConfig{
routes: make(map[string]modelRoute),
pricing: make(map[string]modelPrice),
stopCh: make(chan struct{}),
}
if err := mc.loadFromFile("../conf/models.yaml"); err != nil {
t.Fatalf("load conf/models.yaml: %v", err)
}
return mc
}
// doaiEnabledModels is the customer-facing chat lineup enabled on the do-ai
// provider (2026-07-18), each verified live against inference.do-ai.run:
// user-facing SKU → {real catalog upstream, vision, tools}. vision/tools reflect
// the LIVE probe, so a "no" is a measured rejection, not an omission — gpt-5.6
// rejects tool calls (HTTP 400), and claude-sonnet-5 / deepseek-v4-pro / glm-5.2 /
// kimi-k2.6 / qwen3.5 reject image input, so those flags are intentionally false.
var doaiEnabledModels = []struct {
sku string
upstream string
vision bool
tools bool
}{
{"claude-opus-4-8", "anthropic-claude-opus-4.8", true, true},
{"claude-opus-4-7", "anthropic-claude-opus-4.7", true, true},
{"claude-sonnet-5", "anthropic-claude-5-sonnet", false, true},
{"claude-sonnet-4-6", "anthropic-claude-4.6-sonnet", true, true},
{"claude-haiku-4-5", "anthropic-claude-haiku-4.5", true, true},
{"claude-fable-5", "anthropic-claude-fable-5", true, true},
{"gpt-5.6-luna", "openai-gpt-5.6-luna", true, false},
{"gpt-5.6-sol", "openai-gpt-5.6-sol", true, false},
{"gpt-5.6-terra", "openai-gpt-5.6-terra", true, false},
{"gpt-5.5", "openai-gpt-5.5", true, true},
{"gpt-5.4", "openai-gpt-5.4", true, true},
{"gpt-4o", "openai-gpt-4o", true, true},
{"gpt-4o-mini", "openai-gpt-4o-mini", true, true},
{"o3", "openai-o3", true, true},
{"deepseek-v4-pro", "deepseek-v4-pro", false, true},
{"llama-4-maverick", "llama-4-maverick", true, true},
{"qwen3.5-397b", "qwen3.5-397b-a17b", false, true},
{"glm-5.2", "glm-5.2", false, true},
{"kimi-k2.6", "kimi-k2.6", false, true},
}
// TestDOAIEnabledModelsVisibleAndCapable pins that every customer-facing chat
// model enabled on do-ai (a) resolves to its real catalog upstream on the do-ai
// provider (so it meters under do-ai), (b) is VISIBLE in /v1/models (not hidden),
// and (c) surfaces its verified capabilities — context window, max output, and
// the vision/tools flags exactly as the live probe found them.
func TestDOAIEnabledModelsVisibleAndCapable(t *testing.T) {
mc := loadRealConfig(t)
listed := map[string]modelInfo{}
for _, m := range mc.ListModels() {
listed[m.ID] = m
}
for _, want := range doaiEnabledModels {
r := mc.ResolveRoute(want.sku)
if r == nil {
t.Errorf("%q did not resolve", want.sku)
continue
}
if r.providerName != "do-ai" {
t.Errorf("%q provider = %q, want do-ai (must meter under do-ai)", want.sku, r.providerName)
}
if r.upstreamModel != want.upstream {
t.Errorf("%q upstream = %q, want %q", want.sku, r.upstreamModel, want.upstream)
}
mi, ok := listed[want.sku]
if !ok {
t.Errorf("%q missing from /v1/models listing (still hidden?)", want.sku)
continue
}
if mi.ContextWindow <= 0 {
t.Errorf("%q listing has no context_window (clients cannot size context)", want.sku)
}
if mi.MaxOutputTokens <= 0 {
t.Errorf("%q listing has no max_output_tokens", want.sku)
}
if mi.SupportsVision != want.vision {
t.Errorf("%q supports_vision = %v, want %v (live probe)", want.sku, mi.SupportsVision, want.vision)
}
if mi.SupportsTools != want.tools {
t.Errorf("%q supports_tools = %v, want %v (live probe)", want.sku, mi.SupportsTools, want.tools)
}
}
}
// TestModelInfoCapabilityFieldsOmitEmpty proves the new capability enrichments
// are additive: present on the wire only when true/nonzero (so a legacy consumer
// is unaffected), and a "no" is expressed by ABSENCE, never a fabricated false.
func TestModelInfoCapabilityFieldsOmitEmpty(t *testing.T) {
bare := jsonKeys(t, modelInfo{ID: "x", Object: "model", Created: 1, OwnedBy: "do-ai"})
for _, k := range []string{"max_output_tokens", "supports_vision", "supports_tools"} {
if bare[k] {
t.Errorf("bare modelInfo must omit %q", k)
}
}
full := jsonKeys(t, modelInfo{
ID: "x", Object: "model", Created: 1, OwnedBy: "do-ai",
MaxOutputTokens: 128000, SupportsVision: true, SupportsTools: true,
})
for _, k := range []string{"max_output_tokens", "supports_vision", "supports_tools"} {
if !full[k] {
t.Errorf("modelInfo must surface %q when set", k)
}
}
}