Files
zeekay b7653013dd A masked refusal is every engine sentence for a request the model cannot take
The engine after v1.6.5 no longer answers a forced tool choice its
upstream refuses with "the model could not serve this request": it
names the field ("this model does not take the request's tool_choice"),
or refuses up front when no rung takes a forced choice ("<id> cannot be
made to call a tool"). All three are the model asked unable to take a
request its floor may, so all three fall, within the free allowance.
The tests send the engine's own bytes (Response::deny: code, message,
type engine_error).

Testing: go test ./controllers/ -run 'TestAMaskedRefusalFalls|TestAModelThatCannotServe|TestARefusalThatIsTheCallers'
Risk: low.
How to revert: git revert this commit.
2026-10-26 21:08:12 -07:00

534 lines
20 KiB
Go

// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
// floor.go — an Enso or Zen model never stops.
//
// A chat request naming an Enso or Zen id that cannot be served as asked, for a
// reason that is not the caller's, is served as the same request by the free model
// of that id's family: enso-free for an Enso id, zen-free for a Zen id. That model
// is the id's floor. Fall is the one decision; every place a request can be
// refused or fail asks it — the balance gate in front of the handlers (routers),
// the handlers' own gate and reservation, and the family pipe when the family
// cannot answer before a byte reached the caller.
//
// What falls:
//
// the wallet or the plan does not pay: insufficient_balance, balance_unavailable,
// paid_plan_required, plan_allowance_used, model_cap, free_plan_cap,
// usage_cap_exceeded, paid_lane_full, and our own cash breaker (supply_unavailable);
// the model cannot answer: its vendor failed (5xx), is rate limited (429), timed
// out or could not be reached, our account with it is spent (402), our credential
// for it was refused, its route is unpriced, the paid lane is closed to the
// caller, its family is not configured, or our family answered with its masked
// "could not serve this request".
//
// What stands, as the caller's: a bad credential (401), a malformed request (400),
// a model the caller may not use (403), a limit the caller set on its own key
// (key_budget, key_model), the free lane's own allowance (allowance_spent), a
// policy that cannot be read (usage_unavailable), and any refusal the floor itself
// meets. A floor never falls, so a request falls at most once: asked → floor.
//
// What the caller is told: X-Hanzo-Fallback names the floor, X-Hanzo-Fallback-Reason
// names why, and the answer's `model` is the floor. It bills nothing.
import (
"encoding/json"
"net/http"
"strconv"
"strings"
"time"
metric "github.com/luxfi/metric"
iam "github.com/hanzoai/ai/internal/iam"
"github.com/hanzoai/ai/log"
"github.com/hanzoai/ai/object"
"github.com/zap-proto/zip"
)
// The headers a request served on its floor carries.
const (
FallbackHeader = "X-Hanzo-Fallback"
FallbackReasonHeader = "X-Hanzo-Fallback-Reason"
)
// paidByHeader says who paid for an answer (the gate's usage); a floor's is nobody.
const paidByHeader = "X-Hanzo-Paid-By"
// ReasonRoute is why a request falls when every rung of its model refused inside the
// family serving it, and the family's own catalog answered on the floor: the family
// says so by naming the floor in X-Hanzo-Served.
const ReasonRoute = "route_refused"
// The reasons a request falls when no refusal code names one: what the model
// could not do. The gate's own codes (object.Code*) name the rest.
const (
onTimeout = "timeout" // the vendor did not answer in time
onUnreachable = "unreachable" // no answer at all: a dial, a reset
onBusy = "rate_limited" // 429 from every account asked
onUnfunded = "upstream_unfunded" // our account with the vendor is spent (402)
onCredential = "credential" // the vendor refused our credential
onCapacity = "capacity" // 503: no rung or key could take it
onUpstream = "upstream_error" // any other 5xx
onMasked = "masked" // our family: the model asked cannot take the request (masks)
onUnpriced = "unpriced" // its route costs more than it bills, or cannot be read
onClosed = "lane_closed" // the paid lane is closed to the caller
onMissing = "unconfigured" // its family is not configured
onCut = "upstream_cut" // the answer broke off before a byte was relayed
onRoute = ReasonRoute
onSupply = codeSupply // our own cash breaker holds paid inference
onFull = ReasonFull // the paid lane has no room for the call
onBalance = object.CodeInsufficientBalance
)
// fallen counts the requests served on their floor, by floor and reason. It is the
// rule's own meter: an operator reads how often each family stops, and why, from
// rate(cloud_fallback[5m]) on GET /v1/metrics.
var fallen = metric.NewCounterVec(metric.CounterOpts{
Name: "cloud_fallback",
Help: "Requests served by their family's free model because the model asked could not serve them, by floor and reason",
}, []string{"floor", "reason"})
// Floor is the free model of model's Hanzo family — enso-free for an Enso id,
// zen-free for a Zen id — or "" when it has none: a third-party id, a decision
// route, or an id that already is a free model's name.
func Floor(model string) string {
f := familyNamed(FamilyOf(model))
if f == nil || f.freeName == "" {
return ""
}
m := strings.ToLower(strings.TrimSpace(model))
if m == strings.ToLower(f.freeName) || m == freeID {
return ""
}
return f.freeName
}
// falling are the reasons a request falls (see the file comment). Anything else is
// the caller's to clear and stands.
var falling = map[string]bool{
object.CodeInsufficientBalance: true,
object.CodeBalanceUnavailable: true,
object.CodePaidPlan: true,
object.CodePlanAllowance: true,
object.CodeModelCap: true,
object.CodeFreePlanCap: true,
object.CodeUsageCap: true,
onFull: true,
onSupply: true,
onTimeout: true,
onUnreachable: true,
onBusy: true,
onUnfunded: true,
onCredential: true,
onCapacity: true,
onUpstream: true,
onMasked: true,
onUnpriced: true,
onClosed: true,
onMissing: true,
onCut: true,
onRoute: true,
}
// Falls reports whether a request refused or failed for why is served on its
// floor rather than refused.
func Falls(why string) bool { return falling[why] }
// Fall is the rule: the model that serves a request for model that could not be
// served for why, and true when that is model's floor. False leaves the refusal
// standing — model has no floor, or why is the caller's.
func Fall(model, why string) (string, bool) {
to := Floor(model)
if to == "" || !Falls(why) {
return "", false
}
return to, true
}
// Fell says on the answer that it is the floor's: the model that answers and why,
// that it was served on the free lane and paid for by nobody, and counts it. Set
// before a byte of the answer is written. The gate, which asks the floor again from
// the top, then states the floor's own lane and payer over these.
func Fell(c *zip.Ctx, from, to, why string) {
c.SetHeader(FallbackHeader, to)
c.SetHeader(FallbackReasonHeader, why)
// The floor is served free, and a free route is free because its vendor may keep
// what it carried: said on every fallen answer (openrouterTerms).
c.SetHeader(headerCollection, collectionAllow)
h := &c.Fiber().Response().Header
if len(h.Peek(LaneHeader)) > 0 {
c.SetHeader(LaneHeader, "free")
}
h.Del(LaneTokensHeader)
if len(h.Peek(paidByHeader)) > 0 {
c.SetHeader(paidByHeader, object.PaysFree)
}
fallen.WithLabelValues(to, why).Inc()
log.Info("fallback: %s served by %s (%s)", from, to, why)
}
// cause names why a family could not serve a request, from the error it gave: the
// status first, the text only where there is no status, as faultOf reads them.
func cause(err error) string {
st := upstreamHTTPStatus(err)
msg := strings.ToLower(err.Error())
switch {
case st == 0 && has(msg, []string{"timeout", "deadline exceeded"}):
return onTimeout
case st == 0:
return onUnreachable
case st == http.StatusTooManyRequests:
return onBusy
case st == http.StatusPaymentRequired || broke(err, msg):
return onUnfunded
case st == http.StatusUnauthorized, st == http.StatusForbidden:
return onCredential
case st == http.StatusRequestTimeout, st == http.StatusGatewayTimeout:
return onTimeout
case st == http.StatusServiceUnavailable:
return onCapacity
case st >= 500:
return onUpstream
case masked(err):
return onMasked
}
return ""
}
// masks are the sentences our own family service (hanzo-inc/engine, relay.rs and
// serve.rs) answers when the model asked cannot take a request that another model
// may: its vendor's refusal hidden (`could not serve`), the field it named (`does
// not take the request's tool_choice`), or a forced tool choice none of its rungs
// takes (`cannot be made to call a tool`).
var masks = []string{
"the model could not serve this request",
"this model does not take the request's",
"cannot be made to call a tool",
}
// masked reports our family's refusal of a request the model asked cannot take: a
// 400 or 404 saying one of masks. A 403 or 422 so worded is a moderation or a content
// refusal, which every model would answer alike, so it stands like every other 4xx
// that describes the request.
func masked(err error) bool {
switch upstreamHTTPStatus(err) {
case http.StatusBadRequest, http.StatusNotFound:
said := strings.ToLower(err.Error())
for _, m := range masks {
if strings.Contains(said, m) {
return true
}
}
}
return false
}
// reason names why a handler's own gate refused a request, from its error: the
// code it carries, or "" for a refusal that is the caller's (401, 400, 403).
func reason(err error) string {
switch code := codeOf(err); {
case code != "":
return code
case statusOf(err) == http.StatusPaymentRequired:
return onBalance
}
return ""
}
// fall serves a request its handler's own gate refused (err) on its floor: the body
// names the floor and the answer says so (Fell). It reports the model now asked
// for, and false when the refusal stands — a path that is not a conversation, a
// model with no floor, or a refusal that is the caller's.
func (c *ApiController) fall(model string, err error) (string, bool) {
if err == nil || !ChatPath(c.Path()) {
return "", false
}
why := reason(err)
to, ok := Fall(model, why)
if !ok {
return "", false
}
body, ok := WithModel(c.Body(), to)
if !ok {
return "", false
}
c.Fiber().Request().SetBody(body)
c.Fiber().Request().Header.Del("Content-Encoding")
Fell(c.Ctx, model, to, why)
c.Locals(fellKey{}, true)
return to, true
}
// fellKey marks a request its handler's own gate moved to its floor (fall).
type fellKey struct{}
// held holds a request its handler moved to its floor to the free allowance, as the
// balance gate holds one it moved there: a floor costs nothing, so the allowance is
// its one bound, counted by the payer u's person as the gate counts it. It reports
// false when it answered the request itself, in dialect: the allowance is spent (429),
// or cannot be read (503). The unit is the admission's, as the allowance's own rule
// says (object.SpentFunc). Any other request is not asked about.
func (c *ApiController) held(u *iam.User, dialect string) bool {
if fell, _ := c.Locals(fellKey{}).(bool); !fell {
return true
}
return c.allowance(u, dialect)
}
// allowance takes one call of payer u's free allowance for a request about to be
// served on its floor, as the gate takes one for a free model. It reports false when
// it answered the request itself, in dialect: spent (429) or unreadable (503).
func (c *ApiController) allowance(u *iam.User, dialect string) bool {
spent := object.Spent()
if spent == nil || u == nil {
return true
}
org := c.billingOrg(u)
out, err := spent(zip.WithCaller(c.Context(), zip.CallerOf(c.Forward())), u.PayerSubject(org), org)
status, msg, body := 0, "", []byte(nil)
switch {
case err != nil:
log.Warn("allowance: unreadable for a fallen request org=%s: %v", org, err)
n := object.UsageUnavailable()
status, msg, body = n.Status, n.Message, n.ErrorJSON()
case out.Spent:
n := object.AllowanceSpent(c.Host(), org, out)
if wait := n.RetryAfter(time.Now()); wait > 0 {
c.SetHeader("Retry-After", strconv.FormatInt(wait, 10))
}
status, msg, body = n.Status, n.Message, n.ErrorJSON()
default:
return true
}
if dialect == "anthropic" {
c.respondAnthropicError(anthropicErrorTypeForStatus(status), msg, status)
return false
}
c.SetHeader("Content-Type", "application/json")
_ = c.Bytes(status, body)
return false
}
// toolsHeader says what a floor was asked in place of a forced tool choice it may
// not honor: "required" (the named tool, alone, must be called) or "auto" (the
// model was told in words to call a tool). Absent when the choice went as sent.
const toolsHeader = "X-Hanzo-Fallback-Tools"
// relax is body with its tool choice loosened one step, for a floor that may not
// force a tool, in the request's dialect ("openai" chat or "anthropic" messages).
// It never drops a tool the request could still call:
//
// step 1: a choice naming one tool becomes "required" ("any") over that tool alone
// — the same demand, since only that tool can then be called;
// step 2: "required" ("any") becomes "auto", with a sentence telling the model to
// answer by calling the tool (or one of the tools).
//
// did is the choice the floor is now asked ("required" or "auto"); ok is false when
// there is nothing to loosen at that step, and body is then returned as it came.
func relax(body []byte, dialect string, step int) ([]byte, string, bool) {
var m map[string]json.RawMessage
if json.Unmarshal(body, &m) != nil || m == nil || len(m["tool_choice"]) == 0 {
return body, "", false
}
choice := m["tool_choice"]
var word string
_ = json.Unmarshal(choice, &word)
var named struct {
Type string `json:"type"`
Name string `json:"name"`
Function struct {
Name string `json:"name"`
} `json:"function"`
}
_ = json.Unmarshal(choice, &named)
anthropic := dialect == "anthropic"
set := func(k string, v any) {
b, _ := json.Marshal(v)
m[k] = b
}
switch {
case step == 1 && !anthropic && named.Type == "function" && named.Function.Name != "":
if !only(m, named.Function.Name, false) {
return body, "", false
}
set("tool_choice", "required")
set("parallel_tool_calls", false)
case step == 1 && anthropic && named.Type == "tool" && named.Name != "":
if !only(m, named.Name, true) {
return body, "", false
}
set("tool_choice", map[string]any{"type": "any", "disable_parallel_tool_use": true})
case step == 2 && !anthropic && word == "required":
set("tool_choice", "auto")
var msgs []json.RawMessage
if json.Unmarshal(m["messages"], &msgs) != nil {
return body, "", false
}
set("messages", after(msgs, instruct(m, false)))
case step == 2 && anthropic && named.Type == "any":
set("tool_choice", map[string]any{"type": "auto"})
said := instruct(m, true)
var text string
var blocks []json.RawMessage
switch {
case len(m["system"]) == 0:
set("system", said)
case json.Unmarshal(m["system"], &text) == nil:
set("system", text+"\n\n"+said)
case json.Unmarshal(m["system"], &blocks) == nil:
tail, _ := json.Marshal(map[string]string{"type": "text", "text": said})
set("system", append(blocks, tail))
default:
return body, "", false
}
default:
return body, "", false
}
out, err := json.Marshal(m)
if err != nil {
return body, "", false
}
if step == 1 {
return out, "required", true
}
return out, "auto", true
}
// after is msgs with said added after the caller's own instructions: at the end of
// its last system or developer message, so nothing of ours comes ahead of what the
// caller said. A request with none is given one system message, which precedes no
// instruction of the caller's.
func after(msgs []json.RawMessage, said string) []json.RawMessage {
for i := len(msgs) - 1; i >= 0; i-- {
var turn map[string]json.RawMessage
if json.Unmarshal(msgs[i], &turn) != nil {
continue
}
var role string
_ = json.Unmarshal(turn["role"], &role)
if role != "system" && role != "developer" {
continue
}
var text string
var parts []json.RawMessage
switch {
case json.Unmarshal(turn["content"], &text) == nil:
turn["content"], _ = json.Marshal(text + "\n\n" + said)
case json.Unmarshal(turn["content"], &parts) == nil:
tail, _ := json.Marshal(map[string]string{"type": "text", "text": said})
turn["content"], _ = json.Marshal(append(parts, tail))
default:
continue
}
out := append([]json.RawMessage(nil), msgs...)
out[i], _ = json.Marshal(turn)
return out
}
lead, _ := json.Marshal(map[string]string{"role": "system", "content": said})
return append([]json.RawMessage{lead}, msgs...)
}
// only keeps, of the request's tools, the one named name, and reports whether the
// request offered it. anthropic says how a tool names itself: {"name"} there,
// {"function":{"name"}} in OpenAI's dialect.
func only(m map[string]json.RawMessage, name string, anthropic bool) bool {
var tools []json.RawMessage
if json.Unmarshal(m["tools"], &tools) != nil {
return false
}
for _, t := range tools {
if toolName(t, anthropic) == name {
b, _ := json.Marshal([]json.RawMessage{t})
m["tools"] = b
return true
}
}
return false
}
// toolName is the name one tool of a request declares.
func toolName(t json.RawMessage, anthropic bool) string {
var d struct {
Name string `json:"name"`
Function struct {
Name string `json:"name"`
} `json:"function"`
}
_ = json.Unmarshal(t, &d)
if anthropic {
return d.Name
}
return d.Function.Name
}
// instruct is the sentence that asks a model, in words, for the tool call a
// request could no longer demand: the tool by name when there is one, else any of
// them.
func instruct(m map[string]json.RawMessage, anthropic bool) string {
var tools []json.RawMessage
_ = json.Unmarshal(m["tools"], &tools)
if len(tools) == 1 {
if n := toolName(tools[0], anthropic); n != "" {
return "Answer by calling the tool " + n + "."
}
}
return "Answer by calling one of the provided tools."
}
// shaped reports a refusal that describes the request's shape — malformed (400), a
// field the route does not take (404, as OpenRouter answers a tool choice no
// endpoint supports), or refused content (422) — which a looser tool choice may
// clear.
func shaped(err error) bool {
switch upstreamHTTPStatus(err) {
case http.StatusBadRequest, http.StatusNotFound, http.StatusUnprocessableEntity:
return true
}
return false
}
// freeRoom is what a free route ai sends itself is given beyond a small output limit,
// as zen gives its free rungs: a model that reasons spends its first tokens thinking,
// and a probe's max_tokens (20, "reply OK") left it nothing to answer with, so the
// caller was handed an empty answer. A free route bills nothing.
const freeRoom = 1024
// room is body with each output limit it states under freeRoom lifted by freeRoom:
// the caller's limit stays the answer's, and the thinking before it gets its own.
func room(body []byte) []byte {
var m map[string]json.RawMessage
if json.Unmarshal(body, &m) != nil || m == nil {
return body
}
lifted := false
for _, k := range []string{"max_tokens", "max_completion_tokens", "max_output_tokens"} {
var v int
if json.Unmarshal(m[k], &v) == nil && v > 0 && v < freeRoom {
m[k], _ = json.Marshal(v + freeRoom)
lifted = true
}
}
if !lifted {
return body
}
out, err := json.Marshal(m)
if err != nil {
return body
}
return out
}