Files
ai/controllers/decisions_versions_test.go
Kathir EswaranandGitHub 3e4cb365ac decisions: clear errors for wrong-endpoint model calls; cover both doors (#122)
/v1/decisions naming a chat model (zen6, zen6-flash) now answers 400 naming POST /v1/chat/completions, and /v1/chat/completions naming a decision model answers 400 naming POST /v1/decisions, each with a docs link, on HTTP and ZAP. Both refusals are published in the OpenAPI document. The decision-door check on chat now runs after the public lane overwrites the model, so a public caller is still served the free pool.

Fixes #118

Claude-Session: https://claude.ai/code/session_015XYKLix9PSyn6Jw2HWKprk
2026-11-12 19:25:05 -08:00

350 lines
14 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
)
// useKaiURLs boots KAI_URLS as spec, and restores what was there.
func useKaiURLs(t *testing.T, spec string) {
t.Helper()
prev := kaiURLs.Load()
t.Cleanup(func() { kaiURLs.Store(prev) })
t.Setenv("KAI_URLS", spec)
if err := InitKaiURLs(); err != nil {
t.Fatalf("KAI_URLS=%q: %v", spec, err)
}
}
// versionService is a decision service of its own, as a Kai version's deployment is.
func versionService(t *testing.T, answer string) (*fakeDecisions, string) {
t.Helper()
f := &fakeDecisions{status: http.StatusOK, answer: answer}
srv := httptest.NewServer(f)
t.Cleanup(srv.Close)
return f, srv.URL
}
// askFor is decisionBody asking for model.
func askFor(model string) string {
return strings.Replace(decisionBody, `"model":"kai"`, `"model":"`+model+`"`, 1)
}
// A version KAI_URLS names is answered by its own service, asked there for kai with
// the org, the request id and the capabilities the default path sends, and billed as
// kai at kai's price. Its answer names the version. kai, and a version KAI_URLS does
// not name, still go to KAI_URL.
func TestKaiURLsSendAVersionToItsService(t *testing.T) {
fake, events := setupDecisions(t)
fake.answer = heldAnswer
v14, at := versionService(t, heldAnswer)
useKaiURLs(t, "kai-1.4="+at+"/")
// Its members in the order the service is sent them, so the body kai-1.4's
// service is asked is byte for byte the body kai's is.
const kaiBody = `{"model":"kai","questions":{"is_bug":{"type":"noul","instructions":"Is this a bug?"}},"state":{"message":"Payment failed twice"}}`
v14Body := strings.Replace(kaiBody, `"model":"kai"`, `"model":"kai-1.4"`, 1)
status, body := driveDecisions(t, "Bearer "+decisionsKey, v14Body)
if status != http.StatusOK {
t.Fatalf("kai-1.4 => %d %s", status, body)
}
if !sameJSON([]byte(body), []byte(strings.Replace(heldAnswer, `"model":"kai"`, `"model":"kai-1.4"`, 1))) {
t.Fatalf("answer %s, want kai's answer naming kai-1.4", body)
}
calls, path, sent := v14.seen()
if calls != 1 || path != decisionsPath || string(sent) != kaiBody {
t.Fatalf("kai-1.4's service saw %d call(s) at %q with %s; want 1 at %s asking for kai", calls, path, sent, decisionsPath)
}
v14.mu.Lock()
org, rid := v14.org, v14.rid
v14.mu.Unlock()
if org != decisionsOrg || rid == "" {
t.Fatalf("kai-1.4's service was told org %q, request id %q", org, rid)
}
if n, _, _ := fake.seen(); n != 0 {
t.Fatalf("KAI_URL was asked %d time(s) for a version with a service of its own", n)
}
e := (*events)[len(*events)-1]
if e.Model != "kai" || e.Provider != "kai" || e.USD != nanoToUSD(42*21) {
t.Fatalf("debit %s/%s $%s, want kai/kai at 42 tokens × $0.021/M", e.Provider, e.Model, e.USD)
}
// kai, asked the same body by the same org, is KAI_URL's: kai-1.4's held answer
// is never given to it.
if status, body := driveDecisions(t, "Bearer "+decisionsKey, kaiBody); status != http.StatusOK || body != heldAnswer {
t.Fatalf("kai => %d %s", status, body)
}
if n, _, sent := fake.seen(); n != 1 || string(sent) != kaiBody {
t.Fatalf("KAI_URL was asked %d time(s) for kai, want 1 (%s)", n, sent)
}
// And each is held apart: asked again, each is answered from its own.
driveDecisions(t, "Bearer "+decisionsKey, v14Body)
driveDecisions(t, "Bearer "+decisionsKey, kaiBody)
if a, _, _ := v14.seen(); a != 1 {
t.Fatalf("kai-1.4's service was asked %d time(s) for a held answer", a)
}
if b, _, _ := fake.seen(); b != 1 {
t.Fatalf("KAI_URL was asked %d time(s) for a held answer", b)
}
// A version no pair names goes to KAI_URL, asked as written.
if status, body := driveDecisions(t, "Bearer "+decisionsKey, askFor("kai-1.5")); status != http.StatusOK {
t.Fatalf("kai-1.5 => %d %s", status, body)
}
if n, _, sent := fake.seen(); n != 2 || string(sent) != askFor("kai-1.5") {
t.Fatalf("KAI_URL saw %d call(s), last %s; want kai-1.5 as asked", n, sent)
}
}
// With KAI_URLS unset or blank, every decision is today's: kai and every version go
// to KAI_URL, the body as written, the answer as given, one debit at kai's price.
func TestKaiURLsBlankChangesNothing(t *testing.T) {
for _, spec := range []string{"", " "} {
fake, events := setupDecisions(t)
useKaiURLs(t, spec)
if kaiURLs.Load() != nil {
t.Fatalf("KAI_URLS=%q names versions", spec)
}
for _, model := range []string{"kai", "kai-1.4", "kai-0834a74f2d14"} {
asked := askFor(model)
status, body := driveDecisions(t, "Bearer "+decisionsKey, asked)
if status != http.StatusOK || body != decisionAnswer {
t.Fatalf("%s => %d %s", model, status, body)
}
if _, path, sent := fake.seen(); path != decisionsPath || string(sent) != asked {
t.Fatalf("%s reached KAI_URL at %q as %s", model, path, sent)
}
e := (*events)[len(*events)-1]
if e.Model != "kai" || e.Provider != "kai" || e.USD != nanoToUSD(42*21) {
t.Fatalf("%s: debit %s/%s $%s", model, e.Provider, e.Model, e.USD)
}
}
if n, _, _ := fake.seen(); n != 3 || len(*events) != 3 {
t.Fatalf("%d calls, %d debits; want 3 of each", n, len(*events))
}
}
}
// A Kai version is kai- and the weights' first 12 hex, or a release number of two
// or three parts, each a number without a leading zero.
func TestKaiVersion(t *testing.T) {
for id, want := range map[string]bool{
"kai-0834a74f2d14": true, "kai-1.4": true, "kai-1.4.2": true, "kai-0.1": true, "kai-10.20.30": true, "kai-2.0.0": true,
"kai": false, "kai-": false, "kai-1": false, "kai-1.": false, "kai-.4": false, "kai-1.4.": false, "kai-1..4": false,
"kai-1.4.2.1": false, "kai-01.4": false, "kai-1.04": false, "kai-1.4-rc1": false, "kai-v1.4": false, "KAI-1.4": false,
"kai-1.a": false, "kai-0834A74F2D14": false, "kai-0834a74f2d1": false, "zen-1.4": false, " kai-1.4": false,
} {
if got := kaiVersion(id); got != want {
t.Errorf("kaiVersion(%q) = %v, want %v", id, got, want)
}
}
for model, want := range map[string]string{"kai-1.4": "kai", "kai-0834a74f2d14": "kai", "kai": "kai", "kai-01.4": "kai-01.4", "zenlm/zen6": "zenlm/zen6", "gpt-4o": "gpt-4o"} {
if got := DecisionAs(model); got != want {
t.Errorf("DecisionAs(%q) = %q, want %q", model, got, want)
}
}
}
// /v1/models lists every version KAI_URLS names as Kai's own row under its id: owned
// by hanzo, at kai's price, class and family. Blank, it lists none.
func TestKaiURLsAreListed(t *testing.T) {
setupDecisions(t)
useKaiURLs(t, "")
if lists(t, "kai-1.4") {
t.Fatal("kai-1.4 is listed with KAI_URLS blank")
}
useKaiURLs(t, "kai-1.4=http://kai-1-4.hanzo.svc:8080,kai-1.5.1=http://kai-1-5.hanzo.svc:8080")
byID := map[string]modelInfo{}
var order []string
for _, m := range listAvailableModels() {
byID[m.ID] = m
order = append(order, m.ID)
}
kai, ok := byID["kai"]
if !ok {
t.Fatal("kai is not listed")
}
for _, id := range []string{"kai-1.4", "kai-1.5.1"} {
m, ok := byID[id]
if !ok {
t.Fatalf("%s is not listed", id)
}
if m.OwnedBy != "hanzo" || m.Object != "model" || m.Class != kai.Class || m.Family != "kai" || m.CanonicalSlug != "hanzo/"+id {
t.Fatalf("%s listed as %+v; kai is %+v", id, m, kai)
}
if m.Pricing == nil || !reflect.DeepEqual(m.Pricing, kai.Pricing) || m.Free != kai.Free {
t.Fatalf("%s priced %+v, kai %+v", id, m.Pricing, kai.Pricing)
}
}
for i := 1; i < len(order); i++ {
if order[i-1] >= order[i] {
t.Fatalf("the listing is out of order at %q, %q", order[i-1], order[i])
}
}
}
// A malformed KAI_URLS refuses the boot, saying which pair and why.
func TestKaiURLsMalformedRefusesBoot(t *testing.T) {
prev := kaiURLs.Load()
t.Cleanup(func() { kaiURLs.Store(prev) })
for spec, says := range map[string]string{
"kai-1.4": "is not id=url",
"http://kai-1-4.hanzo.svc:8080": "is not id=url",
"kai-1.4=http://a:8080,,": "entry 2",
"kai=http://kai.hanzo.svc:8080": "kai itself is KAI_URL's",
"zenlm/zen6=http://zen": "is no Kai version",
"kai-01.4=http://a": "is no Kai version",
"=http://a": "is no Kai version",
"kai-1.4=": "is not an http or https address",
"kai-1.4=kai-1-4.hanzo.svc:8080": "is not an http or https address",
"kai-1.4=ftp://a": "is not an http or https address",
"kai-1.4=http://": "is not an http or https address",
"kai-1.4=http://a?x=1": "is not an http or https address",
"kai-1.4=http://u:p@a": "carries a credential",
"kai-1.4=http://a,kai-1.4=http://b": "kai-1.4 is named twice",
"kai-1.4=http://a, kai-1.5=http://b, bogus=x": "entry 3",
} {
t.Setenv("KAI_URLS", spec)
err := InitKaiURLs()
if err == nil || !strings.HasPrefix(err.Error(), "KAI_URLS: ") || !strings.Contains(err.Error(), says) {
t.Errorf("KAI_URLS=%q => %v, want a refusal saying %q", spec, err, says)
}
if strings.Contains(spec, "u:p") && err != nil && strings.Contains(err.Error(), "u:p") {
t.Errorf("the refusal repeats the credential: %v", err)
}
}
t.Setenv("KAI_URLS", " kai-1.4 = http://kai-1-4.hanzo.svc:8080/ , kai-0834a74f2d14=https://kai-a7.hanzo.svc ")
if err := InitKaiURLs(); err != nil {
t.Fatal(err)
}
if u, ok := kaiURL("kai-1.4"); !ok || u != "http://kai-1-4.hanzo.svc:8080" {
t.Fatalf("kai-1.4 at %q, %v", u, ok)
}
if u, ok := kaiURL("kai-0834a74f2d14"); !ok || u != "https://kai-a7.hanzo.svc" {
t.Fatalf("kai-0834a74f2d14 at %q, %v", u, ok)
}
if _, ok := kaiURL("kai"); ok {
t.Fatal("kai has an address of its own")
}
}
// A chat naming a model /v1/decisions serves is told the door and the body, never
// passed to the decision service to come back "404 upstream error".
func TestAChatNamingKaiIsSentToDecisions(t *testing.T) {
setupDecisions(t)
for _, m := range []string{"kai", "kai-1.8", "kai-0834a74f2d14"} {
why := decisionDoor(m)
if !strings.Contains(why, "POST /v1/decisions") || !strings.Contains(why, `"model":"`+m+`"`) {
t.Errorf("decisionDoor(%q) = %q, want the decisions door with the model filled in", m, why)
}
}
for _, m := range []string{"enso-pro", "zen6", "gpt-4o", ""} {
if why := decisionDoor(m); why != "" {
t.Errorf("decisionDoor(%q) = %q, want no refusal for a chat model", m, why)
}
}
}
// A decision naming a chat model is told the chat door, never sent to the service
// for a completion it cannot answer.
func TestADecisionNamingChatIsToldTheChatDoor(t *testing.T) {
fake, events := setupDecisions(t)
for _, m := range []string{"zen6", "zen6-flash"} {
status, body := driveDecisions(t, "Bearer "+decisionsKey, askFor(m))
if status != http.StatusBadRequest {
t.Fatalf("%s => %d, want 400", m, status)
}
want := string(decisionsFailure(400, fmt.Sprintf(`model %q is a chat model; use POST /v1/chat/completions. Docs: GET /v1/openapi.json?operation=post_chat_completions`, m)))
if body != want {
t.Fatalf("%s refusal:\n got %s\nwant %s", m, body, want)
}
}
if calls, _, _ := fake.seen(); calls != 0 {
t.Fatalf("service saw %d call(s) for a chat model it must never be sent", calls)
}
if len(*events) != 0 {
t.Fatalf("refused calls were debited %d time(s)", len(*events))
}
if !isChatModel("zen6") || !isChatModel("zen6-flash") {
t.Fatalf("bare chat skus are chat models: zen6=%t zen6-flash=%t", isChatModel("zen6"), isChatModel("zen6-flash"))
}
if isChatModel("kai") || isChatModel("zenlm/zen6") || isChatModel("zen5") {
t.Fatalf("decisions and their ids are not chat models: kai=%t zenlm/zen6=%t zen5=%t",
isChatModel("kai"), isChatModel("zenlm/zen6"), isChatModel("zen5"))
}
}
// A chat naming a decision model is refused at both chat doors, HTTP and the ZAP
// fast path, with the message that names POST /v1/decisions and its docs; neither
// reaches the decision service or bills.
func TestAChatNamingKaiIsRefusedOnEveryWire(t *testing.T) {
fake, events := setupDecisions(t)
const prefix = `model "kai" is served at POST /v1/decisions.`
const docs = "Docs: GET /v1/openapi.json?operation=post_decisions"
want := decisionDoor("kai")
if !strings.HasPrefix(want, prefix) || !strings.HasSuffix(want, docs) {
t.Fatalf("decisionDoor(kai) = %q, want it to open %q and end %q", want, prefix, docs)
}
body := `{"model":"kai","messages":[{"role":"user","content":"hi"}]}`
c := newChatController("Bearer "+decisionsKey, body)
if status := answering(t, c, c.ChatCompletions); status != http.StatusBadRequest {
t.Fatalf("HTTP chat naming kai => %d, want 400 (body %s)", status, sent(c))
}
var got Response
if err := json.Unmarshal([]byte(sent(c)), &got); err != nil {
t.Fatalf("HTTP refusal is not the envelope: %v (%s)", err, sent(c))
}
if got.Status != "error" || got.Msg != want {
t.Fatalf("HTTP refusal: got %+v, want msg %q", got, want)
}
frame, err := zapChatHandler(context.Background(), "Bearer "+decisionsKey, []byte(body))
status, msg := cloudStatus(t, frame, err)
if status != http.StatusBadRequest || msg != want {
t.Fatalf("ZAP chat naming kai => %d %q, want 400 %q", status, msg, want)
}
if calls, _, _ := fake.seen(); calls != 0 {
t.Fatalf("service saw %d call(s) for a chat it must never be sent", calls)
}
if len(*events) != 0 {
t.Fatalf("refused chats were debited %d time(s)", len(*events))
}
}
// The ZAP decisions door reads the model as HTTP does, so a chat model is told the
// chat door there too.
func TestAZapDecisionNamingChatIsToldTheChatDoor(t *testing.T) {
fake, _ := setupDecisions(t)
r := zapDecision(context.Background(), "Bearer "+decisionsKey, []byte(askFor("zen6")))
want := string(decisionsFailure(400, `model "zen6" is a chat model; use POST /v1/chat/completions. Docs: GET /v1/openapi.json?operation=post_chat_completions`))
if r.status != http.StatusBadRequest || string(r.body) != want {
t.Fatalf("ZAP decision naming zen6 => %d %s, want 400 %s", r.status, r.body, want)
}
if calls, _, _ := fake.seen(); calls != 0 {
t.Fatalf("service saw %d call(s) for a chat model it must never be sent", calls)
}
}