mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
The engine after v1.6.5 no longer answers a forced tool choice its
upstream refuses with "the model could not serve this request": it
names the field ("this model does not take the request's tool_choice"),
or refuses up front when no rung takes a forced choice ("<id> cannot be
made to call a tool"). All three are the model asked unable to take a
request its floor may, so all three fall, within the free allowance.
The tests send the engine's own bytes (Response::deny: code, message,
type engine_error).
Testing: go test ./controllers/ -run 'TestAMaskedRefusalFalls|TestAModelThatCannotServe|TestARefusalThatIsTheCallers'
Risk: low.
How to revert: git revert this commit.
759 lines
37 KiB
Go
759 lines
37 KiB
Go
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/decimal"
|
|
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
"github.com/hanzoai/ai/object"
|
|
)
|
|
|
|
// Every Enso id falls to enso-free and every Zen id to zen-free; a free model, the
|
|
// platform's own free id and anybody else's model have no floor.
|
|
func TestTheFloorIsTheFamilysFreeModel(t *testing.T) {
|
|
for model, want := range map[string]string{
|
|
"enso": "enso-free", "enso-auto": "enso-free", "enso-flash": "enso-free", "enso-pro": "enso-free",
|
|
"enso-ultra": "enso-free", "hanzo/enso": "enso-free", "ENSO-PRO": "enso-free",
|
|
"zen6": "zen-free", "zen6-flash": "zen-free", "zen5-coder": "zen-free",
|
|
"enso-free": "", "zen-free": "", "free": "", "": "",
|
|
"anthropic/claude-sonnet-5.5": "", "gpt-6.1-sol": "",
|
|
} {
|
|
if got := Floor(model); got != want {
|
|
t.Errorf("Floor(%q) = %q, want %q", model, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Want of payment and a model that cannot answer fall; what the caller set, the
|
|
// free lane's own allowance and an unreadable policy stand.
|
|
func TestWhatFallsAndWhatStands(t *testing.T) {
|
|
for _, why := range []string{
|
|
object.CodeInsufficientBalance, object.CodeBalanceUnavailable, object.CodePaidPlan, object.CodePlanAllowance,
|
|
object.CodeModelCap, object.CodeFreePlanCap, object.CodeUsageCap, ReasonFull, codeSupply,
|
|
onTimeout, onUnreachable, onBusy, onUnfunded, onCredential, onCapacity, onUpstream, onMasked,
|
|
onUnpriced, onClosed, onMissing, onCut, onRoute,
|
|
} {
|
|
if to, ok := Fall("zen6", why); !ok || to != "zen-free" {
|
|
t.Errorf("Fall(zen6, %q) = %q %v, want zen-free", why, to, ok)
|
|
}
|
|
}
|
|
for _, why := range []string{"", "key_budget", "key_model", object.CodeAllowanceSpent, object.CodePoolBusy,
|
|
object.CodePoolExhausted, object.CodeUsageUnavailable, "invalid_request"} {
|
|
if to, ok := Fall("enso-pro", why); ok {
|
|
t.Errorf("Fall(enso-pro, %q) = %q, want the refusal to stand", why, to)
|
|
}
|
|
}
|
|
// A floor never falls, and nobody else's model has one: one hop at most.
|
|
for _, model := range []string{"enso-free", "zen-free", "anthropic/claude-sonnet-5.5"} {
|
|
if to, ok := Fall(model, object.CodeInsufficientBalance); ok {
|
|
t.Errorf("Fall(%q) = %q, want none", model, to)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Why a family could not serve, read from its error: the status first.
|
|
func TestCauseNamesWhyAFamilyCouldNotServe(t *testing.T) {
|
|
for want, err := range map[string]error{
|
|
onTimeout: fmt.Errorf("Post: %w", context.DeadlineExceeded),
|
|
onUnreachable: errors.New("dial tcp 10.0.0.1:443: connect: connection refused"),
|
|
onBusy: &apiError{status: http.StatusTooManyRequests, msg: "busy"},
|
|
onUnfunded: &apiError{status: http.StatusPaymentRequired, msg: "Insufficient credits"},
|
|
onCapacity: &apiError{status: http.StatusServiceUnavailable, msg: "this model is temporarily unavailable"},
|
|
onUpstream: &apiError{status: http.StatusBadGateway, msg: "bad gateway"},
|
|
onMasked: &apiError{status: http.StatusBadRequest, msg: "the model could not serve this request"},
|
|
} {
|
|
if got := cause(err); got != want {
|
|
t.Errorf("cause(%v) = %q, want %q", err, got, want)
|
|
}
|
|
}
|
|
// A refusal that describes the request is nobody's outage.
|
|
if got := cause(&apiError{status: http.StatusBadRequest, msg: "messages: required"}); got != "" {
|
|
t.Errorf("a malformed request reads as %q", got)
|
|
}
|
|
}
|
|
|
|
// A handler's own gate falls only for want of payment, only on a conversation, and
|
|
// only for a model with a floor; the body then names the floor and the answer says
|
|
// why.
|
|
func TestTheHandlersGateFallsOnlyForWantOfPayment(t *testing.T) {
|
|
ask := func(path, model string) *ApiController {
|
|
c := visit(http.MethodPost, path)
|
|
c.Fiber().Request().SetBody([]byte(`{"model":"` + model + `","stream":true,"tools":[{"type":"function","function":{"name":"f"}}],"messages":[{"role":"user","content":"hi"}]}`))
|
|
return c
|
|
}
|
|
unverified := &apiError{status: http.StatusInternalServerError, code: object.CodeBalanceUnavailable, msg: "failed to verify account balance"}
|
|
for name, tc := range map[string]struct {
|
|
err error
|
|
why string
|
|
}{
|
|
"an empty wallet": {billingError("requires a positive balance"), object.CodeInsufficientBalance},
|
|
"a balance it cannot read": {unverified, object.CodeBalanceUnavailable},
|
|
"our own cash breaker": {supplyError("paid inference is temporarily unavailable"), codeSupply},
|
|
"a bare 402 with no code": {&apiError{status: http.StatusPaymentRequired, msg: "x"}, object.CodeInsufficientBalance},
|
|
} {
|
|
for _, path := range []string{"/v1/chat/completions", "/v1/messages", "/v1/responses"} {
|
|
c := ask(path, "zen6")
|
|
to, ok := c.fall("zen6", tc.err)
|
|
if !ok || to != "zen-free" {
|
|
t.Fatalf("%s on %s: fall = %q %v, want zen-free", name, path, to, ok)
|
|
}
|
|
var body map[string]any
|
|
if err := json.Unmarshal(c.Body(), &body); err != nil || body["model"] != "zen-free" || body["stream"] != true || body["tools"] == nil {
|
|
t.Errorf("%s on %s: body %s, want the same request naming zen-free", name, path, c.Body())
|
|
}
|
|
if got := string(c.Fiber().Response().Header.Peek(FallbackHeader)); got != "zen-free" {
|
|
t.Errorf("%s: %s = %q", name, FallbackHeader, got)
|
|
}
|
|
if got := string(c.Fiber().Response().Header.Peek(FallbackReasonHeader)); got != tc.why {
|
|
t.Errorf("%s: %s = %q, want %q", name, FallbackReasonHeader, got, tc.why)
|
|
}
|
|
}
|
|
}
|
|
for name, tc := range map[string]struct {
|
|
path, model string
|
|
err error
|
|
}{
|
|
"a bad key": {"/v1/chat/completions", "zen6", authError("invalid API key")},
|
|
"a malformed request": {"/v1/chat/completions", "zen6", modelError("model %q is not available", "zen6")},
|
|
"an org it may not act in": {"/v1/chat/completions", "zen6", forbiddenError("forbidden")},
|
|
"an embedding": {"/v1/embeddings", "zen6", billingError("requires a positive balance")},
|
|
"somebody else's model": {"/v1/chat/completions", "anthropic/claude-sonnet-5.5", billingError("requires a positive balance")},
|
|
"the floor itself": {"/v1/chat/completions", "zen-free", billingError("requires a positive balance")},
|
|
"no refusal": {"/v1/chat/completions", "zen6", nil},
|
|
} {
|
|
c := ask(tc.path, tc.model)
|
|
before := string(c.Body())
|
|
if to, ok := c.fall(tc.model, tc.err); ok {
|
|
t.Errorf("%s: fell to %q, want the refusal to stand", name, to)
|
|
}
|
|
if string(c.Body()) != before || string(c.Fiber().Response().Header.Peek(FallbackHeader)) != "" {
|
|
t.Errorf("%s: the request was changed though it did not fall", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// floorVendor is one test vendor behind both the family under test and the free
|
|
// pool: the model asked gets status and body, the pool's route answers whole or as a
|
|
// stream, as the request asked, or as ask says when it is set.
|
|
type floorVendor struct {
|
|
status int
|
|
body string
|
|
header http.Header
|
|
ask func(map[string]json.RawMessage) (int, string)
|
|
|
|
mu sync.Mutex
|
|
asked []string
|
|
bodies []map[string]json.RawMessage
|
|
}
|
|
|
|
const floorRoute = "vendor/big:free"
|
|
|
|
func (v *floorVendor) serve(t *testing.T) *httptest.Server {
|
|
t.Helper()
|
|
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
raw, _ := io.ReadAll(r.Body)
|
|
var in map[string]json.RawMessage
|
|
_ = json.Unmarshal(raw, &in)
|
|
var model string
|
|
_ = json.Unmarshal(in["model"], &model)
|
|
var stream bool
|
|
_ = json.Unmarshal(in["stream"], &stream)
|
|
v.mu.Lock()
|
|
v.asked = append(v.asked, model)
|
|
v.bodies = append(v.bodies, in)
|
|
v.mu.Unlock()
|
|
if model != floorRoute {
|
|
for k, vs := range v.header {
|
|
w.Header()[k] = vs
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(v.status)
|
|
_, _ = io.WriteString(w, v.body)
|
|
return
|
|
}
|
|
if v.ask != nil {
|
|
st, b := v.ask(in)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(st)
|
|
_, _ = io.WriteString(w, b)
|
|
return
|
|
}
|
|
messages := strings.HasSuffix(r.URL.Path, "/messages")
|
|
switch {
|
|
case stream && messages:
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = io.WriteString(w, anthropicStream)
|
|
case stream:
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = io.WriteString(w, answerStream)
|
|
case messages:
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = io.WriteString(w, `{"id":"msg_1","type":"message","role":"assistant","model":"`+floorRoute+`","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":3,"output_tokens":1}}`)
|
|
default:
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = io.WriteString(w, `{"id":"gen-1","model":"`+floorRoute+`","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1,"total_tokens":4}}`)
|
|
}
|
|
}))
|
|
t.Cleanup(s.Close)
|
|
return s
|
|
}
|
|
|
|
func (v *floorVendor) calls() []string {
|
|
v.mu.Lock()
|
|
defer v.mu.Unlock()
|
|
return append([]string(nil), v.asked...)
|
|
}
|
|
|
|
// floorPool points the free pool at url, with one vendor account and none of our
|
|
// own compute.
|
|
func floorPool(t *testing.T, url string) {
|
|
t.Helper()
|
|
restore(t, engineFam)
|
|
engineFam.urlKey = "TEST_ENGINE_URL_UNSET"
|
|
engineFam.providerFn = nil
|
|
t.Setenv("OPENROUTER_API_KEY", "k1")
|
|
t.Setenv("OPENROUTER_API_KEY_2", "")
|
|
t.Setenv("OPENROUTER_API_KEY_3", "")
|
|
forgetKeys()
|
|
cooled.forget()
|
|
spareFamily(t, url, floorRoute)
|
|
}
|
|
|
|
// zenAt is a Zen family served at url, as otherFamily is an Enso one.
|
|
func zenAt(url string) *modelFamily {
|
|
return &modelFamily{
|
|
name: "zen", prefix: "zen", owner: "zenlm", freeName: "zen-free",
|
|
providerFn: func() *object.Provider {
|
|
return &object.Provider{Owner: "admin", Name: "zen", Type: "Zen", ProviderUrl: url}
|
|
},
|
|
loaded: true, fetchedAt: time.Now(),
|
|
}
|
|
}
|
|
|
|
// turn drives the real pipe for one request and returns the client's view of it.
|
|
func turn(t *testing.T, fam *modelFamily, model, dialect, body string, stream bool, start time.Time) (*ApiController, []attempt) {
|
|
t.Helper()
|
|
path, api := "/v1/chat/completions", "chat/completions"
|
|
if dialect == "anthropic" {
|
|
path, api = "/v1/messages", "messages"
|
|
}
|
|
c := visit(http.MethodPost, path)
|
|
c.Fiber().Request().SetBody([]byte(body))
|
|
return c, c.pipeToFamily(fam, api, dialect, model, []byte(body), stream, 0, "acme", nil, false, nil, start)
|
|
}
|
|
|
|
// AN ENSO OR ZEN MODEL THAT CANNOT SERVE IS ANSWERED BY ITS FLOOR, streamed or whole,
|
|
// for every failure that is not the caller's: the same request, one hop, named for
|
|
// what answered.
|
|
func TestAModelThatCannotServeIsAnsweredByItsFloor(t *testing.T) {
|
|
fails := map[string]struct {
|
|
status int
|
|
body string
|
|
why string
|
|
}{
|
|
"a server error": {http.StatusInternalServerError, `{"error":{"message":"boom"}}`, onUpstream},
|
|
"a bad gateway": {http.StatusBadGateway, `{"error":{"message":"bad gateway"}}`, onUpstream},
|
|
"no capacity": {http.StatusServiceUnavailable, `{"error":{"message":"this model is temporarily unavailable"}}`, onCapacity},
|
|
"a rate limit": {http.StatusTooManyRequests, `{"error":{"message":"this model is busy"}}`, onBusy},
|
|
"a spent account": {http.StatusPaymentRequired, `{"error":{"message":"Insufficient credits."}}`, onUnfunded},
|
|
"a masked refusal": {http.StatusBadRequest, `{"error":{"code":400,"message":"the model could not serve this request","type":"engine_error"}}`, onMasked},
|
|
"a gateway timeout": {http.StatusGatewayTimeout, `{"error":{"message":"upstream timed out"}}`, onTimeout},
|
|
}
|
|
for _, family := range []struct {
|
|
model, floor string
|
|
at func(string) *modelFamily
|
|
}{
|
|
{"enso-pro", "enso-free", func(u string) *modelFamily { return otherFamily(t, u) }},
|
|
{"zen6", "zen-free", zenAt},
|
|
} {
|
|
for name, f := range fails {
|
|
for _, stream := range []bool{false, true} {
|
|
label := fmt.Sprintf("%s %s stream=%v", family.model, name, stream)
|
|
v := &floorVendor{status: f.status, body: f.body}
|
|
s := v.serve(t)
|
|
floorPool(t, s.URL)
|
|
body := fmt.Sprintf(`{"model":%q,"stream":%v,"response_format":{"type":"json_object"},"messages":[{"role":"user","content":"2+2?"}]}`, family.model, stream)
|
|
c, out := turn(t, family.at(s.URL), family.model, "openai", body, stream, time.Now())
|
|
if out != nil || answered(c) != http.StatusOK {
|
|
t.Fatalf("%s: attempts=%+v status %d body %s, want the floor's answer", label, out, answered(c), sent(c))
|
|
}
|
|
got := sent(c)
|
|
if !strings.Contains(got, `"model":"`+family.floor+`"`) || strings.Contains(got, `"model":"`+family.model+`"`) || strings.Contains(got, floorRoute) {
|
|
t.Errorf("%s: body names the wrong model:\n%s", label, got)
|
|
}
|
|
if stream && !strings.Contains(got, "data: ") {
|
|
t.Errorf("%s: a streamed request was not answered as a stream:\n%s", label, got)
|
|
}
|
|
for h, want := range map[string]string{FallbackHeader: family.floor, FallbackReasonHeader: f.why, servedHeader: family.floor} {
|
|
if got := header(c, h); got != want {
|
|
t.Errorf("%s: %s = %q, want %q", label, h, got, want)
|
|
}
|
|
}
|
|
calls := v.calls()
|
|
if len(calls) != 2 || calls[0] != family.model || calls[1] != floorRoute {
|
|
t.Errorf("%s: asked %v, want the model once and then its floor", label, calls)
|
|
}
|
|
// The same request: what shapes the answer goes to the floor as sent.
|
|
if len(v.bodies) == 2 && string(v.bodies[1]["response_format"]) != `{"type":"json_object"}` {
|
|
t.Errorf("%s: response_format reached the floor as %s", label, v.bodies[1]["response_format"])
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// A model whose family could not be reached at all, or answered only after the
|
|
// request's whole wait, is still answered by its floor: one hop is always asked.
|
|
func TestAFloorIsAskedHoweverLongTheModelTookToFail(t *testing.T) {
|
|
v := &floorVendor{status: http.StatusGatewayTimeout, body: `{"error":{"message":"timed out"}}`}
|
|
s := v.serve(t)
|
|
floorPool(t, s.URL)
|
|
body := `{"model":"enso-ultra","messages":[{"role":"user","content":"2+2?"}]}`
|
|
c, out := turn(t, otherFamily(t, s.URL), "enso-ultra", "openai", body, false, time.Now().Add(-2*zenPipeHeader))
|
|
if out != nil || answered(c) != http.StatusOK || header(c, FallbackHeader) != "enso-free" {
|
|
t.Fatalf("attempts=%+v status %d fallback %q body %s, want the floor's answer", out, answered(c), header(c, FallbackHeader), sent(c))
|
|
}
|
|
|
|
dead := httptest.NewServer(http.NotFoundHandler())
|
|
deadURL := dead.URL
|
|
dead.Close()
|
|
c, out = turn(t, otherFamily(t, deadURL), "enso-ultra", "openai", body, false, time.Now())
|
|
if out != nil || answered(c) != http.StatusOK || header(c, FallbackReasonHeader) != onUnreachable {
|
|
t.Fatalf("unreachable: attempts=%+v status %d reason %q body %s", out, answered(c), header(c, FallbackReasonHeader), sent(c))
|
|
}
|
|
}
|
|
|
|
// A refusal that describes the request is the caller's: answered as it is, never by
|
|
// another model.
|
|
func TestARefusalThatIsTheCallersIsNotMoved(t *testing.T) {
|
|
for name, tc := range map[string]struct {
|
|
status int
|
|
body string
|
|
}{
|
|
"a malformed request": {http.StatusBadRequest, `{"error":{"message":"messages: at least one is required"}}`},
|
|
"an unknown model": {http.StatusNotFound, `{"error":{"message":"unknown model"}}`},
|
|
"refused content": {http.StatusUnprocessableEntity, `{"error":{"message":"content refused"}}`},
|
|
"our own billing": {http.StatusPaymentRequired, `{"error":{"message":"add funds","code":"` + object.CodeInsufficientBalance + `"}}`},
|
|
"a masked moderation": {http.StatusForbidden, `{"error":{"code":403,"message":"the model could not serve this request","type":"engine_error"}}`},
|
|
"a masked content refusal": {http.StatusUnprocessableEntity, `{"error":{"code":422,"message":"the model could not serve this request","type":"engine_error"}}`},
|
|
} {
|
|
v := &floorVendor{status: tc.status, body: tc.body}
|
|
s := v.serve(t)
|
|
floorPool(t, s.URL)
|
|
c, _ := turn(t, otherFamily(t, s.URL), "enso-pro", "openai", `{"model":"enso-pro","messages":[{"role":"user","content":"hi"}]}`, false, time.Now())
|
|
if got := v.calls(); len(got) != 1 {
|
|
t.Errorf("%s: asked %v, want the model alone", name, got)
|
|
}
|
|
if header(c, FallbackHeader) != "" || strings.Contains(sent(c), "enso-free") {
|
|
t.Errorf("%s: moved to the floor: %s %s", name, header(c, FallbackHeader), sent(c))
|
|
}
|
|
}
|
|
}
|
|
|
|
// NO LOOP: a floor that cannot serve either answers with its own refusal — its
|
|
// status, said as the floor's — and nothing is asked after it.
|
|
func TestAFloorThatFailsTooAnswersItsOwnRefusal(t *testing.T) {
|
|
v := &floorVendor{status: http.StatusServiceUnavailable, body: `{"error":{"message":"down"}}`,
|
|
ask: func(map[string]json.RawMessage) (int, string) {
|
|
return http.StatusBadGateway, `{"error":{"message":"Provider returned error from SomeVendor"}}`
|
|
}}
|
|
s := v.serve(t)
|
|
floorPool(t, s.URL)
|
|
c, out := turn(t, otherFamily(t, s.URL), "enso-pro", "openai", `{"model":"enso-pro","messages":[{"role":"user","content":"hi"}]}`, false, time.Now())
|
|
if out != nil {
|
|
t.Fatalf("attempts=%+v, want the floor's refusal answered here and nothing handed on", out)
|
|
}
|
|
if answered(c) != http.StatusBadGateway {
|
|
t.Errorf("status %d, want the floor's own 502 (%s)", answered(c), sent(c))
|
|
}
|
|
if !strings.Contains(sent(c), "enso-free is temporarily unavailable") || strings.Contains(sent(c), "SomeVendor") {
|
|
t.Errorf("body %s, want the floor named and no vendor's words", sent(c))
|
|
}
|
|
if header(c, FallbackHeader) != "enso-free" || header(c, FallbackReasonHeader) != onCapacity {
|
|
t.Errorf("fallback %q reason %q, want the floor that was tried and why", header(c, FallbackHeader), header(c, FallbackReasonHeader))
|
|
}
|
|
if got := v.calls(); len(got) != 2 || got[0] != "enso-pro" || got[1] != floorRoute {
|
|
t.Errorf("asked %v, want the model and its floor once each", got)
|
|
}
|
|
}
|
|
|
|
// A FORCED TOOL IS NEVER DROPPED. The floor is asked for it as "required" over that
|
|
// tool alone; a floor that refuses the shape is asked once more with "auto" and a
|
|
// sentence asking for the call, and the answer says which.
|
|
func TestAForcedToolIsAskedOfTheFloorAsRequiredThenAuto(t *testing.T) {
|
|
var choices []string
|
|
v := &floorVendor{status: http.StatusServiceUnavailable, body: `{"error":{"message":"down"}}`,
|
|
ask: func(in map[string]json.RawMessage) (int, string) {
|
|
choices = append(choices, string(in["tool_choice"]))
|
|
if string(in["tool_choice"]) != `"auto"` {
|
|
return http.StatusNotFound, `{"error":{"message":"No endpoints found that support the provided 'tool_choice' value."}}`
|
|
}
|
|
var msgs []map[string]any
|
|
_ = json.Unmarshal(in["messages"], &msgs)
|
|
var tools []map[string]any
|
|
_ = json.Unmarshal(in["tools"], &tools)
|
|
if len(msgs) == 0 || msgs[0]["role"] != "system" || msgs[0]["content"] != "Answer by calling the tool weather." || len(tools) != 1 {
|
|
return http.StatusBadRequest, `{"error":{"message":"not the request the floor was promised"}}`
|
|
}
|
|
return http.StatusOK, `{"id":"g","model":"x","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":null,` +
|
|
`"tool_calls":[{"id":"c1","type":"function","function":{"name":"weather","arguments":"{}"}}]}}]}`
|
|
}}
|
|
s := v.serve(t)
|
|
floorPool(t, s.URL)
|
|
body := `{"model":"zen6","tool_choice":{"type":"function","function":{"name":"weather"}},` +
|
|
`"tools":[{"type":"function","function":{"name":"weather"}},{"type":"function","function":{"name":"news"}}],` +
|
|
`"messages":[{"role":"user","content":"weather?"}]}`
|
|
c, out := turn(t, zenAt(s.URL), "zen6", "openai", body, false, time.Now())
|
|
if out != nil || answered(c) != http.StatusOK || !strings.Contains(sent(c), `"name":"weather"`) {
|
|
t.Fatalf("attempts=%+v status %d body %s, want the floor's tool call", out, answered(c), sent(c))
|
|
}
|
|
if strings.Join(choices, " ") != `"required" "auto"` {
|
|
t.Errorf("the floor was asked %v, want required then auto", choices)
|
|
}
|
|
if got := header(c, toolsHeader); got != "auto" {
|
|
t.Errorf("%s = %q, want auto", toolsHeader, got)
|
|
}
|
|
}
|
|
|
|
// relax loosens a tool choice one step in either dialect and never drops a tool the
|
|
// request could still call.
|
|
func TestRelaxLoosensAToolChoiceOneStep(t *testing.T) {
|
|
oai := `{"model":"m","tool_choice":{"type":"function","function":{"name":"a"}},"tools":[{"type":"function","function":{"name":"a"}},{"type":"function","function":{"name":"b"}}],"messages":[{"role":"user","content":"x"}]}`
|
|
b, did, ok := relax([]byte(oai), "openai", 1)
|
|
if !ok || did != "required" || !strings.Contains(string(b), `"tool_choice":"required"`) || strings.Contains(string(b), `"name":"b"`) || !strings.Contains(string(b), `"parallel_tool_calls":false`) {
|
|
t.Fatalf("openai forced → %s %q %v", b, did, ok)
|
|
}
|
|
b, did, ok = relax(b, "openai", 2)
|
|
var asked struct {
|
|
Messages []struct{ Role, Content string }
|
|
Tools []json.RawMessage
|
|
}
|
|
_ = json.Unmarshal(b, &asked)
|
|
if !ok || did != "auto" || !strings.Contains(string(b), `"tool_choice":"auto"`) || len(asked.Messages) != 2 || len(asked.Tools) != 1 ||
|
|
asked.Messages[0].Role != "system" || asked.Messages[0].Content != "Answer by calling the tool a." {
|
|
t.Fatalf("openai required → %s %q %v", b, did, ok)
|
|
}
|
|
if _, _, ok := relax(b, "openai", 2); ok {
|
|
t.Error("auto was loosened again")
|
|
}
|
|
// The sentence goes after the caller's own instructions, never ahead of them.
|
|
withSystem := `{"tool_choice":"required","tools":[{"type":"function","function":{"name":"a"}}],` +
|
|
`"messages":[{"role":"system","content":"You are terse."},{"role":"user","content":"x"}]}`
|
|
b, _, ok = relax([]byte(withSystem), "openai", 2)
|
|
var kept struct {
|
|
Messages []struct{ Role, Content string }
|
|
}
|
|
_ = json.Unmarshal(b, &kept)
|
|
if !ok || len(kept.Messages) != 2 || kept.Messages[0].Content != "You are terse.\n\nAnswer by calling the tool a." {
|
|
t.Fatalf("openai with the caller's system message → %s", b)
|
|
}
|
|
if _, _, ok := relax([]byte(`{"tool_choice":{"type":"function","function":{"name":"gone"}},"tools":[{"type":"function","function":{"name":"a"}}]}`), "openai", 1); ok {
|
|
t.Error("a choice naming a tool the request does not offer was rewritten")
|
|
}
|
|
|
|
ant := `{"model":"m","system":"be brief","tool_choice":{"type":"tool","name":"a"},"tools":[{"name":"a","input_schema":{}},{"name":"b","input_schema":{}}],"messages":[{"role":"user","content":"x"}]}`
|
|
b, did, ok = relax([]byte(ant), "anthropic", 1)
|
|
if !ok || did != "required" || !strings.Contains(string(b), `"type":"any"`) || strings.Contains(string(b), `"name":"b"`) {
|
|
t.Fatalf("anthropic forced → %s %q %v", b, did, ok)
|
|
}
|
|
b, did, ok = relax(b, "anthropic", 2)
|
|
if !ok || did != "auto" || !strings.Contains(string(b), `"tool_choice":{"type":"auto"}`) || !strings.Contains(string(b), `"system":"be brief\n\nAnswer by calling the tool a."`) {
|
|
t.Fatalf("anthropic any → %s %q %v", b, did, ok)
|
|
}
|
|
blocks := `{"tool_choice":{"type":"any"},"system":[{"type":"text","text":"be brief"}],"tools":[{"name":"a"},{"name":"b"}],"messages":[]}`
|
|
if b, _, ok := relax([]byte(blocks), "anthropic", 2); !ok || !strings.Contains(string(b), `"system":[{"type":"text","text":"be brief"},{"text":"Answer by calling one of the provided tools.","type":"text"}]`) {
|
|
t.Fatalf("anthropic system blocks → %s", b)
|
|
}
|
|
}
|
|
|
|
// An answer its floor wrote bills nothing, whatever the model asked costs; the same
|
|
// model answered as asked bills its price.
|
|
func TestAFallenAnswerBillsNothing(t *testing.T) {
|
|
enso := otherFamily(t, "http://enso.invalid")
|
|
enso.byID = map[string]zenModel{"enso-pro": {ID: "enso-pro", Base: zenTier{In: decimal.New(3, 0), Out: decimal.New(15, 0)}}}
|
|
enso.ids = []string{"enso-pro"}
|
|
c := visit(http.MethodPost, "/v1/x")
|
|
use := tokens{fresh: 1_000_000, completion: 1_000_000}
|
|
w := whence{ledger: c.billingOrg(nil), ip: c.Fiber().IP(), ctx: c.Context(), asked: enso}
|
|
if nano := recordFamilyUsage(w, enso, "enso-pro", "", nil, &mark{}, nil, true, false, "r1", use, serving{}, time.Now(), nil, "success", ""); nano != 18_000_000_000 {
|
|
t.Fatalf("enso-pro as asked billed %d nano, want $18", nano)
|
|
}
|
|
w.fell = true
|
|
hold := &budgetHold{est: 2000}
|
|
if nano := recordFamilyUsage(w, enso, "enso-pro", "enso-pro", nil, &mark{}, nil, true, false, "r2", use, serving{}, time.Now(), hold, "success", ""); nano != 0 {
|
|
t.Errorf("an answer its floor wrote billed %d nano", nano)
|
|
}
|
|
}
|
|
|
|
// A family whose own catalog fell to the floor says so in X-Hanzo-Served: the answer
|
|
// is then the floor's, in its body as in its headers, and bills nothing.
|
|
func TestAFamilysOwnFallbackIsNamedAsTheFloor(t *testing.T) {
|
|
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = io.ReadAll(r.Body)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Header().Set(servedHeader, "zen-free")
|
|
w.Header().Set(freeHeader, "true")
|
|
_, _ = io.WriteString(w, `{"id":"g","model":"zen6","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}`)
|
|
}))
|
|
t.Cleanup(s.Close)
|
|
c, out := turn(t, zenAt(s.URL), "zen6", "openai", `{"model":"zen6","messages":[{"role":"user","content":"hi"}]}`, false, time.Now())
|
|
if out != nil || answered(c) != http.StatusOK {
|
|
t.Fatalf("attempts=%+v status %d", out, answered(c))
|
|
}
|
|
if !strings.Contains(sent(c), `"model":"zen-free"`) || header(c, FallbackHeader) != "zen-free" || header(c, FallbackReasonHeader) != onRoute {
|
|
t.Errorf("body %s fallback %q reason %q, want the floor named", sent(c), header(c, FallbackHeader), header(c, FallbackReasonHeader))
|
|
}
|
|
}
|
|
|
|
// A STREAM THAT BREAKS OFF AFTER ITS FIRST BYTES IS NEVER ASKED AGAIN, and never
|
|
// ends as if it were complete: it ends with an error event in its dialect. A stream
|
|
// that ends as it should gets nothing added.
|
|
func TestAStreamThatBreaksOffEndsWithAnError(t *testing.T) {
|
|
cut := `data: {"id":"gen-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"half an ans"}}]}` + "\n\n"
|
|
to := toStream()
|
|
tk, _, _, _ := relayZenStream(to.w, strings.NewReader(cut), ourMark(), nil)
|
|
got := to.String()
|
|
if !strings.Contains(got, `"code":"upstream_cut"`) || !strings.Contains(got, "half an ans") {
|
|
t.Fatalf("a cut chat stream ended:\n%s", got)
|
|
}
|
|
if tk.completion == 0 {
|
|
t.Error("the text relayed before the cut is not billed")
|
|
}
|
|
|
|
whole := told(t, cut+"data: [DONE]\n\n", ourMark())
|
|
if strings.Contains(whole, "upstream_cut") {
|
|
t.Errorf("a complete chat stream was marked cut:\n%s", whole)
|
|
}
|
|
|
|
mk := ourMark()
|
|
mk.speaks = messageShape
|
|
anthropicCut := "event: content_block_delta\ndata: " + `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"half"}}` + "\n\n"
|
|
got = told(t, anthropicCut, mk)
|
|
if !strings.Contains(got, "event: error\n") || !strings.Contains(got, `"type":"api_error"`) {
|
|
t.Fatalf("a cut messages stream ended:\n%s", got)
|
|
}
|
|
if got := told(t, anthropicCut+"event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", mk); strings.Contains(got, "event: error") {
|
|
t.Errorf("a complete messages stream was marked cut:\n%s", got)
|
|
}
|
|
// One whose model said why it stopped is whole, though its [DONE] never came.
|
|
stopped := cut + `data: {"id":"gen-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}` + "\n\n"
|
|
if got := told(t, stopped, ourMark()); strings.Contains(got, "upstream_cut") {
|
|
t.Errorf("a stream cut after its model stopped was marked cut:\n%s", got)
|
|
}
|
|
spaced := cut + `data: {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}` + "\n\n"
|
|
if got := told(t, spaced, ourMark()); strings.Contains(got, "upstream_cut") {
|
|
t.Errorf("a stop written with a space was marked cut:\n%s", got)
|
|
}
|
|
// One the family ended with its own error is ended: nothing is added to it.
|
|
if got := told(t, cut+`data: {"error":{"message":"upstream failed"}}`+"\n\n", ourMark()); strings.Contains(got, "upstream_cut") {
|
|
t.Errorf("a stream the family ended with an error was marked cut again:\n%s", got)
|
|
}
|
|
}
|
|
|
|
// told relays body through relayZenStream and returns what the client read.
|
|
func told(t *testing.T, body string, mk *mark) string {
|
|
t.Helper()
|
|
var out strings.Builder
|
|
w := bufio.NewWriter(&out)
|
|
relayZenStream(w, strings.NewReader(body), mk, nil)
|
|
_ = w.Flush()
|
|
return out.String()
|
|
}
|
|
|
|
// A request its handler moved to its floor meets the free allowance there, as one the
|
|
// gate moved does: spent is 429, unreadable is 503, and a request that did not fall
|
|
// is never asked about.
|
|
func TestAFloorTheHandlerChoseMeetsTheFreeAllowance(t *testing.T) {
|
|
t.Cleanup(func() { object.SetSpent(nil) })
|
|
alice := &iam.User{Owner: "acme", Name: "alice"}
|
|
reads := 0
|
|
for name, tc := range map[string]struct {
|
|
standing object.Standing
|
|
err error
|
|
status int
|
|
}{
|
|
"room left": {object.Standing{Limit: 50, Used: 3}, nil, 0},
|
|
"spent": {object.Standing{Spent: true, Window: "day", Limit: 50, Used: 50}, nil, http.StatusTooManyRequests},
|
|
"cannot be read": {object.Standing{}, errors.New("store down"), http.StatusServiceUnavailable},
|
|
} {
|
|
object.SetSpent(func(context.Context, string, string) (object.Standing, error) {
|
|
reads++
|
|
return tc.standing, tc.err
|
|
})
|
|
for _, dialect := range []string{"openai", "anthropic"} {
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody([]byte(`{"model":"enso-ultra","messages":[]}`))
|
|
if !c.held(alice, dialect) {
|
|
t.Fatalf("%s %s: a request that did not fall met the allowance", name, dialect)
|
|
}
|
|
if _, ok := c.fall("enso-ultra", billingError("request cost")); !ok {
|
|
t.Fatalf("%s: did not fall", name)
|
|
}
|
|
got := c.held(alice, dialect)
|
|
if got != (tc.status == 0) || tc.status != 0 && answered(c) != tc.status {
|
|
t.Errorf("%s %s: held=%v status %d (%s), want %d", name, dialect, got, answered(c), sent(c), tc.status)
|
|
}
|
|
if tc.status == http.StatusTooManyRequests && !strings.Contains(sent(c), "Free tier") {
|
|
t.Errorf("%s %s: body %s, want the allowance's own sentence", name, dialect, sent(c))
|
|
}
|
|
}
|
|
}
|
|
if reads != 6 {
|
|
t.Errorf("the allowance was read %d times, want once a fallen request", reads)
|
|
}
|
|
}
|
|
|
|
// The Messages API falls the same way, whole and streamed: the floor answers in the
|
|
// caller's dialect, named for itself, and a forced tool is never dropped there either.
|
|
func TestTheMessagesAPIFallsToTheFloor(t *testing.T) {
|
|
for _, stream := range []bool{false, true} {
|
|
v := &floorVendor{status: http.StatusServiceUnavailable, body: `{"error":{"message":"down"}}`}
|
|
s := v.serve(t)
|
|
floorPool(t, s.URL)
|
|
body := fmt.Sprintf(`{"model":"enso-pro","stream":%v,"max_tokens":20,"tool_choice":{"type":"tool","name":"f"},`+
|
|
`"tools":[{"name":"f","input_schema":{"type":"object"}},{"name":"g","input_schema":{"type":"object"}}],"messages":[{"role":"user","content":"hi"}]}`, stream)
|
|
c, out := turn(t, otherFamily(t, s.URL), "enso-pro", "anthropic", body, stream, time.Now())
|
|
if out != nil || answered(c) != http.StatusOK {
|
|
t.Fatalf("stream=%v: attempts=%+v status %d body %s", stream, out, answered(c), sent(c))
|
|
}
|
|
if !strings.Contains(sent(c), `"model":"enso-free"`) || header(c, FallbackHeader) != "enso-free" || header(c, toolsHeader) != "required" {
|
|
t.Errorf("stream=%v: body %s fallback %q tools %q", stream, sent(c), header(c, FallbackHeader), header(c, toolsHeader))
|
|
}
|
|
if len(v.bodies) != 2 || string(v.bodies[1]["tool_choice"]) != `{"disable_parallel_tool_use":true,"type":"any"}` {
|
|
t.Errorf("stream=%v: the floor was asked %v", stream, v.bodies)
|
|
}
|
|
// A free route has room to think beside the caller's 20 tokens.
|
|
if len(v.bodies) == 2 && string(v.bodies[1]["max_tokens"]) != fmt.Sprint(20+freeRoom) {
|
|
t.Errorf("stream=%v: the floor was sent max_tokens %s, want %d", stream, v.bodies[1]["max_tokens"], 20+freeRoom)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A request that fell is said to be on the free lane, paid for by nobody, whatever
|
|
// the gate seated it on before it fell.
|
|
func TestAFallenAnswerIsOnTheFreeLane(t *testing.T) {
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.SetHeader(LaneHeader, "paid")
|
|
c.SetHeader(LaneTokensHeader, "4000")
|
|
c.SetHeader(paidByHeader, object.PaysCredits)
|
|
Fell(c.Ctx, "enso-pro", "enso-free", onCapacity)
|
|
if header(c, headerCollection) != collectionAllow {
|
|
t.Errorf("%s = %q, want the free route's terms disclosed", headerCollection, header(c, headerCollection))
|
|
}
|
|
if header(c, LaneHeader) != "free" || header(c, paidByHeader) != object.PaysFree || header(c, LaneTokensHeader) != "" {
|
|
t.Errorf("lane %q paid-by %q tokens %q, want free, free and none", header(c, LaneHeader), header(c, paidByHeader), header(c, LaneTokensHeader))
|
|
}
|
|
plain := visit(http.MethodPost, "/v1/chat/completions")
|
|
Fell(plain.Ctx, "zen6", "zen-free", onBusy)
|
|
if header(plain, LaneHeader) != "" || header(plain, paidByHeader) != "" {
|
|
t.Error("a request the gate named no lane or payer for was given one")
|
|
}
|
|
}
|
|
|
|
// A whole answer that says nothing is never a finished one: a model that spent its
|
|
// budget thinking stopped for length, and the answer says so.
|
|
func TestABlankWholeAnswerStoppedForLength(t *testing.T) {
|
|
thought := `data: {"id":"g","choices":[{"index":0,"delta":{"reasoning":"Let me think about the word"}}]}` + "\n\n"
|
|
for name, body := range map[string]string{
|
|
"no stated stop": thought + "data: [DONE]\n\n",
|
|
"a stated stop": thought + `data: {"id":"g","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}` + "\n\ndata: [DONE]\n\n",
|
|
"a stated length": thought + `data: {"id":"g","choices":[{"index":0,"delta":{},"finish_reason":"length"}]}` + "\n\ndata: [DONE]\n\n",
|
|
} {
|
|
var out strings.Builder
|
|
w := bufio.NewWriter(&out)
|
|
assembleZenStream(w, io.NopCloser(strings.NewReader(body)), nil, time.Hour, nil, nil)
|
|
if !strings.Contains(out.String(), `"finish_reason":"length"`) {
|
|
t.Errorf("%s: %s, want finish_reason length", name, out.String())
|
|
}
|
|
}
|
|
var out strings.Builder
|
|
w := bufio.NewWriter(&out)
|
|
assembleZenStream(w, io.NopCloser(strings.NewReader(`data: {"id":"g","choices":[{"index":0,"delta":{"content":"OK"}}]}`+"\n\ndata: [DONE]\n\n")), nil, time.Hour, nil, nil)
|
|
if !strings.Contains(out.String(), `"finish_reason":"stop"`) {
|
|
t.Errorf("an answer with content: %s, want stop", out.String())
|
|
}
|
|
}
|
|
|
|
// engineRefusals are what the family service (hanzo-inc/engine, Response::deny over
|
|
// relay.rs upstream_message and serve.rs) writes, byte for byte, when the model asked
|
|
// cannot take a request another model may: v1.6.5's hidden vendor refusal, and the
|
|
// field-naming and forced-choice refusals after it.
|
|
var engineRefusals = []string{
|
|
`{"error":{"code":400,"message":"the model could not serve this request","type":"engine_error"}}`,
|
|
`{"error":{"code":400,"message":"this model does not take the request's tool_choice","type":"engine_error"}}`,
|
|
`{"error":{"code":400,"message":"enso-pro cannot be made to call a tool: no model it routes to takes a forced tool_choice (\"required\" or a named tool); send tool_choice \"auto\"","type":"engine_error"}}`,
|
|
}
|
|
|
|
// A MASKED REFUSAL FALLS AND IS BOUNDED. A caller's own request can provoke one again
|
|
// and again (a forced tool an upstream cannot take), so its floor meets the free
|
|
// allowance first: spent, every such call is the allowance's 429 and the floor is never
|
|
// asked; with room left, the floor answers and the allowance counts each call.
|
|
func TestAMaskedRefusalFallsWithinTheFreeAllowance(t *testing.T) {
|
|
t.Cleanup(func() { object.SetSpent(nil) })
|
|
alice := &iam.User{Owner: "acme", Name: "alice"}
|
|
for _, said := range engineRefusals {
|
|
for _, spent := range []bool{true, false} {
|
|
reads := 0
|
|
object.SetSpent(func(context.Context, string, string) (object.Standing, error) {
|
|
reads++
|
|
return object.Standing{Spent: spent, Window: "day", Limit: 50, Used: 50}, nil
|
|
})
|
|
v := &floorVendor{status: http.StatusBadRequest, body: said}
|
|
s := v.serve(t)
|
|
floorPool(t, s.URL)
|
|
for i := 0; i < 3; i++ {
|
|
body := []byte(`{"model":"enso-pro","messages":[{"role":"user","content":"hi"}]}`)
|
|
c := visit(http.MethodPost, "/v1/chat/completions")
|
|
c.Fiber().Request().SetBody(body)
|
|
out := c.pipeToFamily(otherFamily(t, s.URL), "chat/completions", "openai", "enso-pro", body, false, 0, "acme", alice, false, nil, time.Now())
|
|
switch {
|
|
case out != nil:
|
|
t.Fatalf("spent=%v call %d: handed on %+v", spent, i, out)
|
|
case spent && (answered(c) != http.StatusTooManyRequests || !strings.Contains(sent(c), object.CodeAllowanceSpent)):
|
|
t.Errorf("spent call %d: %d %s, want the allowance's 429", i, answered(c), sent(c))
|
|
case !spent && (answered(c) != http.StatusOK || header(c, FallbackReasonHeader) != onMasked):
|
|
t.Errorf("call %d: %d reason %q %s, want the floor's answer", i, answered(c), header(c, FallbackReasonHeader), sent(c))
|
|
}
|
|
}
|
|
if reads != 3 {
|
|
t.Errorf("spent=%v: the allowance was read %d times for three masked calls, want 3", spent, reads)
|
|
}
|
|
floorAsked := 0
|
|
for _, m := range v.calls() {
|
|
if m == floorRoute {
|
|
floorAsked++
|
|
}
|
|
}
|
|
if spent && floorAsked != 0 || !spent && floorAsked != 3 {
|
|
t.Errorf("spent=%v: the floor was asked %d times", spent, floorAsked)
|
|
}
|
|
}
|
|
}
|
|
}
|