Files
ai/controllers/family_cache_test.go
zeekayandHanzo Dev 772cdef80d native: a stream its caller hangs up on bills what was written, and only the vendor's own words bench a key
Two reviews of 6f8a9405, before any release linked it.

Money:
- A stream its caller hung up on billed the prompt and one token while the
  vendor billed every token it wrote. A Messages stream opens with an output
  count of one (message_start), which sniffZenUsage took as the answer's own
  count, so the relay's floor on the text it relayed was skipped — on every
  family, not only a maker's. It is the answer's count only where the answer
  states it. A maker's stream now says what it cost when it is closed, not only
  at its last byte: claudeChat translates every line already received, tap
  counts every byte it passes, and settle closes the body before it reads the
  trailer.
- That a maker gave an answer is the pipe's own knowledge (serving.maker, set
  from the response dispatch was handed). A cost or a vendor in the headers of
  any other answer floors no bill and relabels no row.

Keys:
- An account out of credit was read by the words "credit balance" anywhere in
  the vendor's message. A validation error quotes the caller's request back, so
  one crafted request could bench every key of the pool for ten minutes. It is
  read by the vendor's own sentence from its start, by 402 and by
  billing_error; an account at the spending limit its owner set is read the
  same way and now hands on to the next key.
- A list that reads again brings back a key refused as a credential (401)
  only. A key refused permission (403) rests its time out like one out of
  credit: brought back every five minutes it cost a refused call, a log line
  and a count each time.
- A 429 that states no Retry-After rests the key five seconds.
- An answer that ended having said nothing, and a 200 that is not the stream
  asked for, are the request's own: one key hears it and the reseller answers,
  where every key was asked in turn.
- Only the route the caller named is a maker's (f == fam && s == model), so a
  route standing in for another never reaches one whatever it is priced at. A
  stream's head is waited for thirty seconds, not two minutes.

Answers:
- An answer the vendor breaks off ends finish_reason "length" and then says
  why, so the whole answer assembled from it is not taken for one that
  finished. Adaptive thinking is not asked beside a forced tool call.
- The answer a maker gave no longer holds the request it answers, which holds
  the key.

Co-authored-by: Hanzo Dev <[email protected]>
2026-10-11 10:30:44 -07:00

162 lines
8.0 KiB
Go

// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"encoding/json"
"net/http"
"strings"
"testing"
"github.com/hanzoai/decimal"
"github.com/klauspost/compress/zstd"
)
// A family answer's cached prompt tokens bill at the SKU's discovered cache_read
// price, in both dialects' usage shapes; an SKU that states no cache price bills
// them at the input rate, never at a guessed discount.
func TestCachedFamilyTokensBillAtTheCacheRate(t *testing.T) {
var wire struct {
Data []zenWireModel `json:"data"`
}
if err := json.Unmarshal([]byte(`{"data":[{"id":"enso-flash","context_window":1000000,`+
`"pricing":{"input":"0.42","output":"1.5","cache_read":"0.09"},`+
`"pricing_tiers":[{"max_context":1000000,"input":"0.42","output":"1.5","cache_read":"0.09"}]}]}`), &wire); err != nil {
t.Fatal(err)
}
zm := wire.Data[0].model()
// OpenAI: prompt_tokens includes the cached ones.
var oai tokens
sniffZenUsage([]byte(`{"usage":{"prompt_tokens":1000000,"completion_tokens":0,"prompt_tokens_details":{"cached_tokens":800000}}}`), &oai)
if oai != (tokens{fresh: 200000, cached: 800000}) {
t.Fatalf("openai usage read as %+v", oai)
}
// Anthropic: cache_read_input_tokens is in addition to input_tokens.
var ant tokens
sniffZenUsage([]byte(`{"type":"message_start","message":{"usage":{"input_tokens":200000,"cache_read_input_tokens":800000,"output_tokens":1}}}`), &ant)
if ant.fresh != 200000 || ant.cached != 800000 {
t.Fatalf("anthropic usage read as %+v", ant)
}
// 200k fresh at 0.42 + 800k cached at 0.09 = $0.084 + $0.072 = $0.156, exactly.
// Without the cache term the same answer billed $0.42.
if got := zm.retailNano(oai); got != 156_000_000 {
t.Errorf("retailNano = %d, want 156000000 ($0.156)", got)
}
p, _ := zm.price()
if p.CacheReadPerMillion != 0.09 {
t.Errorf("CacheReadPerMillion = %v, want the discovered 0.09", p.CacheReadPerMillion)
}
// The debit path reads the same price: a 1M prompt, 800k of it cached, in nano-USD.
if got := tokenNanoAt(p.InputPerMillion, p.OutputPerMillion, p.CacheReadPerMillion, p.CacheWritePerMillion, oai.prompt(), 0, oai.cached, 0); got != 156_000_000 {
t.Errorf("debit = %d nano-USD, want 156000000", got)
}
// No stated cache price: cached tokens bill at the input rate.
nocache := zenModel{Base: zenTier{MaxCtx: 1000000, In: decimal.MustParse("0.42"), Out: decimal.MustParse("1.5")}}
nocache.Tiers = []zenTier{nocache.Base}
if got := nocache.retailNano(tokens{fresh: 200000, cached: 800000, completion: 0}); got != 420_000_000 {
t.Errorf("unpriced cache billed %d nano, want 420000000 ($0.42) at the input rate", got)
}
if p, _ := nocache.price(); p.CacheReadPerMillion != 0.42 {
t.Errorf("unpriced cache reads %v, want the input rate 0.42", p.CacheReadPerMillion)
}
}
// A zstd /v1/responses request that is not routed (a concrete model) is read once:
// the server has already expanded the body, so it is not expanded again.
func TestAZstdResponsesRequestIsReadOnce(t *testing.T) {
token := mintUsageJWT(t, "acme", "ann")
enc, err := zstd.NewWriter(nil)
if err != nil {
t.Fatal(err)
}
c := visit(http.MethodPost, "/v1/responses")
c.Fiber().Request().Header.Set("Authorization", "Bearer "+token)
c.Fiber().Request().Header.Set("Content-Encoding", "zstd")
c.Fiber().Request().SetBody(enc.EncodeAll([]byte(`{"model":"no-such-model-anywhere","input":"hi"}`), nil))
c.Responses()
body := string(c.Fiber().Response().Body())
if strings.Contains(body, "decompress") || strings.Contains(body, "Failed to parse Responses") {
t.Fatalf("a zstd Responses body was refused as unreadable: %d %s", c.Fiber().Response().StatusCode(), body)
}
}
// Every usage shape is read into one prompt: every input token, the cache reads and
// writes parts of it. OpenAI counts its cached tokens inside prompt_tokens, OpenRouter
// its cache writes there too, and Anthropic counts both beside input_tokens.
func TestEveryUsageShapeCountsThePromptWhole(t *testing.T) {
for _, tc := range []struct {
name, body string
want tokens
}{
{"openai", `{"usage":{"prompt_tokens":1000,"completion_tokens":7,"prompt_tokens_details":{"cached_tokens":800}}}`,
tokens{fresh: 200, cached: 800, completion: 7, reported: true}},
{"openrouter", `{"usage":{"prompt_tokens":10000,"completion_tokens":5,"cost":0.01,"prompt_tokens_details":{"cached_tokens":6000,"cache_write_tokens":1000}}}`,
tokens{fresh: 3000, cached: 6000, written: 1000, completion: 5, reported: true}},
// A Messages stream opens with its prompt and an output of one token, before
// a word is written: the prompt is read from it, and it is not the answer's
// own count, which the event that closes the answer states.
{"anthropic, opening", `{"type":"message_start","message":{"usage":{"input_tokens":200,"cache_read_input_tokens":800,"cache_creation_input_tokens":300,"output_tokens":1}}}`,
tokens{fresh: 200, cached: 800, written: 300, completion: 1}},
{"anthropic, closing", `{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":42}}`,
tokens{completion: 42, reported: true}},
{"anthropic, whole", `{"type":"message","usage":{"input_tokens":200,"cache_read_input_tokens":800,"cache_creation_input_tokens":300,"output_tokens":42}}`,
tokens{fresh: 200, cached: 800, written: 300, completion: 42, reported: true}},
} {
var got tokens
sniffZenUsage([]byte(tc.body), &got)
if got != tc.want {
t.Errorf("%s read as %+v, want %+v", tc.name, got, tc.want)
}
}
// A record counts the same prompt, whole, on every path.
rec := servedRecord(ServedUsage{PromptTokens: 1300, CachedTokens: 800, WrittenTokens: 300, CompletionTokens: 1})
if rec.PromptTokens != 1300 || rec.CacheReadTokens != 800 || rec.CacheWriteTokens != 300 {
t.Errorf("served record prompt=%d read=%d write=%d, want 1300/800/300", rec.PromptTokens, rec.CacheReadTokens, rec.CacheWriteTokens)
}
}
// A cache write bills at the model's write rate: a resold model's is the vendor's
// input_cache_write plus the vendor's fee; a Hanzo model that states none bills a
// written token at its input rate. The table path bills the same prompt alike: the
// input rate on the part read fresh, each cache part at its own rate.
func TestACacheWriteBillsAtTheWriteRate(t *testing.T) {
fee(t, "5.5")
var w openrouterWireModel
if err := json.Unmarshal([]byte(`{"id":"anthropic/claude-x","context_length":200000,"pricing":{"prompt":"0.000003","completion":"0.000015","input_cache_read":"0.0000003","input_cache_write":"0.00000375"}}`), &w); err != nil {
t.Fatal(err)
}
markup, err := openrouterMarkup()
if err != nil {
t.Fatal(err)
}
m := w.model(markup)
// 1,000 fresh at $3.165/M, 2,000 read at $0.3165/M, 1,000 written at $3.95625/M.
if got := m.retailNano(tokens{fresh: 1000, cached: 2000, written: 1000}); got != 3_165_000+633_000+3_956_250 {
t.Errorf("resold retail = %d nano, want %d", got, 3_165_000+633_000+3_956_250)
}
ours := zenModel{Base: zenTier{MaxCtx: 1000000, In: decimal.MustParse("0.42"), Out: decimal.MustParse("1.5"), CacheRead: decimal.MustParse("0.09")}}
ours.Tiers = []zenTier{ours.Base}
if got := ours.retailNano(tokens{fresh: 1000, written: 1000}); got != 840_000 {
t.Errorf("a write with no stated rate billed %d nano, want 840000 at the input rate", got)
}
// $2/M in, $0.20/M read, $2.50/M written: 200 fresh, 800 read, 300 written.
if got := tokenNanoAt(2, 0, 0.2, 2.5, 1300, 0, 800, 300); got != 200*2000+800*200+300*2500 {
t.Errorf("table path billed %d nano, want %d", got, 200*2000+800*200+300*2500)
}
}