mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
414 lines
13 KiB
Go
414 lines
13 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/model"
|
|
"github.com/hanzoai/ai/object"
|
|
)
|
|
|
|
// A RACE COSTS N COMPLETIONS AND SETTLES ONE, and the gap between those two
|
|
// numbers is real money. These tests are about that gap: who is billed, from
|
|
// which writer, and what the caller is handed.
|
|
//
|
|
// The mechanism ships inert — nothing sets ask.fan, so every request in
|
|
// production today takes the cascade. That is exactly why the coverage has to be
|
|
// here rather than deferred to the switch being wired: the day someone sets a
|
|
// number, this is what says the money was already right.
|
|
|
|
// racers scripts providers that take TIME, which is the only way to decide a
|
|
// race. Each one emits its text a chunk at a time so a loser can be caught
|
|
// mid-sentence, which is the state the ledger has to price.
|
|
type racers struct {
|
|
// after is how long each provider waits before its first chunk. The smallest
|
|
// wins.
|
|
after map[string]time.Duration
|
|
// says is what each provider emits, in chunks, once it starts.
|
|
says map[string][]string
|
|
// fail names providers that refuse instead of answering.
|
|
fail map[string]error
|
|
|
|
mu sync.Mutex
|
|
got map[string]string // what each provider actually managed to write
|
|
seen []string
|
|
}
|
|
|
|
func (r *racers) install(t *testing.T) {
|
|
t.Helper()
|
|
r.got = map[string]string{}
|
|
saved := callProvider
|
|
t.Cleanup(func() { callProvider = saved })
|
|
callProvider = func(org string, row *object.Provider, c candidate, question string,
|
|
w io.Writer, history, knowledge []*model.RawMessage, lang string,
|
|
) (*model.ModelResult, *object.Provider, error) {
|
|
r.mu.Lock()
|
|
r.seen = append(r.seen, c.provider)
|
|
r.mu.Unlock()
|
|
|
|
if err, ok := r.fail[c.provider]; ok && err != nil {
|
|
return nil, &object.Provider{Owner: "admin", Name: c.provider}, err
|
|
}
|
|
time.Sleep(r.after[c.provider])
|
|
|
|
wrote := ""
|
|
for _, chunk := range r.says[c.provider] {
|
|
if _, err := w.Write([]byte(chunk)); err != nil {
|
|
// hedge.ErrLost. A provider that reads its write error stops
|
|
// here — mid-answer, having already been billed for what it
|
|
// produced. This is the loser the ledger must price.
|
|
break
|
|
}
|
|
wrote += chunk
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
r.mu.Lock()
|
|
r.got[c.provider] = wrote
|
|
r.mu.Unlock()
|
|
|
|
return &model.ModelResult{ResponseTokenCount: 7}, &object.Provider{Owner: "admin", Name: c.provider}, nil
|
|
}
|
|
}
|
|
|
|
// body is the response the client is reading — the destination a race writes
|
|
// its winner into, and nothing else.
|
|
type body struct {
|
|
mu sync.Mutex
|
|
text string
|
|
}
|
|
|
|
func (s *body) Write(p []byte) (int, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.text += string(p)
|
|
return len(p), nil
|
|
}
|
|
|
|
func (s *body) String() string {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.text
|
|
}
|
|
|
|
// plain is a stand-in for OpenAIWriter: it appends, it reports what it holds,
|
|
// and a fork of it starts empty. That is the whole contract race() depends on.
|
|
type plain struct {
|
|
out io.Writer
|
|
buf bytes.Buffer
|
|
}
|
|
|
|
func (p *plain) Write(b []byte) (int, error) {
|
|
n, err := p.out.Write(b)
|
|
if n > 0 {
|
|
p.buf.Write(b[:n])
|
|
}
|
|
return n, err
|
|
}
|
|
func (p *plain) MessageString() string { return p.buf.String() }
|
|
|
|
func fanOf(n int, to io.Writer, adopted **plain, billed chan<- attempt) *fan {
|
|
return &fan{
|
|
n: n,
|
|
to: to,
|
|
fork: func(out io.Writer) io.Writer { return &plain{out: out} },
|
|
adopt: func(won io.Writer) {
|
|
if w, ok := won.(*plain); ok {
|
|
*adopted = w
|
|
}
|
|
},
|
|
bill: func(at attempt) {
|
|
if billed != nil {
|
|
billed <- at
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
// waitBill takes the next ledger row, or says the bill never came. A loser is
|
|
// settled from its own goroutine LONG after the answer has gone out, so a test
|
|
// that reads without waiting is testing the scheduler.
|
|
func waitBill(t *testing.T, billed <-chan attempt) attempt {
|
|
t.Helper()
|
|
select {
|
|
case at := <-billed:
|
|
return at
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("no ledger row for the beaten provider — it burned real tokens upstream " +
|
|
"and nothing recorded them, which is fast mode costing money invisibly")
|
|
return attempt{}
|
|
}
|
|
}
|
|
|
|
// THE BILLING RULE. A raced loser was cancelled mid-generation, so the vendor
|
|
// produced tokens and charged us. It must arrive at the ledger carrying a real
|
|
// count — never as the free refusal a failover attempt is.
|
|
func TestARacedLoserIsBilledForWhatItProduced(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
r := &racers{
|
|
after: map[string]time.Duration{"enso": 0, "do-ai": 40 * time.Millisecond},
|
|
says: map[string][]string{
|
|
"enso": {"the winner answers"},
|
|
"do-ai": {"the loser ", "got this far ", "before it was cut"},
|
|
},
|
|
}
|
|
r.install(t)
|
|
|
|
var out body
|
|
var won *plain
|
|
billed := make(chan attempt, 4)
|
|
a := newAsk(route("enso", "do-ai"), nil, nil)
|
|
a.prompt = 11
|
|
a.fan = fanOf(2, &out, &won, billed)
|
|
|
|
res, by, tried, err := a.serve()
|
|
if err != nil {
|
|
t.Fatalf("a race one provider won must not fail: %v", err)
|
|
}
|
|
if res == nil || by.name != "enso" {
|
|
t.Fatalf("served by %q, want enso — the first to produce a byte wins", by.name)
|
|
}
|
|
// The beaten provider is deliberately NOT here. What the caller is handed it
|
|
// records as refusals that cost nothing, and that is the one row this must
|
|
// never become.
|
|
for _, at := range tried {
|
|
if at.provider == "do-ai" {
|
|
t.Fatal("the beaten provider came back in the attempts, where the caller " +
|
|
"files it as a free refusal — the exact row that hides what fast mode spent")
|
|
}
|
|
}
|
|
|
|
loser := waitBill(t, billed)
|
|
if loser.provider != "do-ai" {
|
|
t.Fatalf("billed %q, want the provider that lost", loser.provider)
|
|
}
|
|
if loser.prompt != 11 {
|
|
t.Errorf("loser prompt=%d, want 11: the vendor accepted and processed the prompt "+
|
|
"and charged us for it, whether or not it went on to win", loser.prompt)
|
|
}
|
|
if loser.completion <= 0 {
|
|
t.Errorf("loser completion=%d, want >0: it emitted %q before it was cancelled, and "+
|
|
"a zero here files a real upstream charge as a free refusal",
|
|
loser.completion, r.got["do-ai"])
|
|
}
|
|
}
|
|
|
|
// The counts must come from the LOSER's own writer. One writer shared across
|
|
// attempts holds exactly one answer, and it is the winner's — so a shared writer
|
|
// bills every loser for the winner's tokens, which is both wrong and invisible.
|
|
func TestEachRacerIsBilledFromItsOwnWriter(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
// The loser says far more than the winner. If the counts were read from one
|
|
// shared writer, the loser's would match the winner's instead.
|
|
r := &racers{
|
|
after: map[string]time.Duration{"enso": 0, "do-ai": 30 * time.Millisecond},
|
|
says: map[string][]string{
|
|
"enso": {"short"},
|
|
"do-ai": {strings.Repeat("a much longer answer that keeps going ", 12)},
|
|
},
|
|
}
|
|
r.install(t)
|
|
|
|
var out body
|
|
var won *plain
|
|
billed := make(chan attempt, 4)
|
|
a := newAsk(route("enso", "do-ai"), nil, nil)
|
|
a.prompt = 5
|
|
a.fan = fanOf(2, &out, &won, billed)
|
|
|
|
if _, _, _, err := a.serve(); err != nil {
|
|
t.Fatalf("serve: %v", err)
|
|
}
|
|
if won == nil {
|
|
t.Fatal("no writer was adopted, so the caller has no answer to return")
|
|
}
|
|
loser := waitBill(t, billed)
|
|
winner, _ := model.GetTokenSize("test-model", won.MessageString())
|
|
if loser.completion == winner {
|
|
t.Errorf("the loser was billed %d tokens, the same as the winner — the counts "+
|
|
"are being read from one shared writer rather than each attempt's own",
|
|
loser.completion)
|
|
}
|
|
if loser.completion <= winner {
|
|
t.Errorf("the loser said far more than the winner and was billed %d against %d",
|
|
loser.completion, winner)
|
|
}
|
|
if got := out.String(); got != "short" {
|
|
t.Errorf("the caller received %q, want only the winner's bytes: a loser's "+
|
|
"half-sentence glued to the winner's answer reads as a model losing its mind", got)
|
|
}
|
|
}
|
|
|
|
// The winner's writer becomes the caller's. Without this a non-streaming reply
|
|
// is assembled from the writer the handler has held all along — which, in a
|
|
// race, is the one writer nobody wrote to.
|
|
func TestTheCallerReadsTheWinnersAnswer(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
r := &racers{
|
|
after: map[string]time.Duration{"enso": 0, "do-ai": 50 * time.Millisecond},
|
|
says: map[string][]string{"enso": {"hello ", "from enso"}, "do-ai": {"never seen"}},
|
|
}
|
|
r.install(t)
|
|
|
|
var out body
|
|
var won *plain
|
|
a := newAsk(route("enso", "do-ai"), nil, nil)
|
|
a.fan = fanOf(2, &out, &won, nil)
|
|
|
|
if _, _, _, err := a.serve(); err != nil {
|
|
t.Fatalf("serve: %v", err)
|
|
}
|
|
if won == nil {
|
|
t.Fatal("nothing was adopted: a non-streaming reply built from the caller's own " +
|
|
"writer would be empty, and would be returned as a successful answer")
|
|
}
|
|
if got := won.MessageString(); got != "hello from enso" {
|
|
t.Errorf("adopted answer %q, want the winner's whole message", got)
|
|
}
|
|
}
|
|
|
|
// A REFUSAL STILL BILLS NOTHING. The rule the cascade has always had must
|
|
// survive the one being added beside it, or every failover in production starts
|
|
// charging customers for vendors that refused them.
|
|
func TestARefusalCarriesNoTokens(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{"enso": apiErr(402, "Insufficient credits.")}}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
_, _, tried, err := newAsk(route("enso", "do-ai"), &w, nil).serve()
|
|
if err != nil {
|
|
t.Fatalf("serve: %v", err)
|
|
}
|
|
for _, at := range tried {
|
|
if at.prompt != 0 || at.completion != 0 {
|
|
t.Errorf("provider %s refused and carries prompt=%d completion=%d, want zero both: "+
|
|
"an attempt that produced nothing must reach the ledger free",
|
|
at.provider, at.prompt, at.completion)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A fan too narrow to race takes the cascade, so a request that asks for fast
|
|
// mode on a model with one vendor still gets the retry and the demotion rather
|
|
// than a bare single call.
|
|
func TestAFanOfOneTakesTheCascade(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{"enso": apiErr(402, "Insufficient credits.")}}
|
|
f.install(t)
|
|
|
|
var out body
|
|
var won *plain
|
|
var w buffer
|
|
a := newAsk(route("enso", "do-ai"), &w, nil)
|
|
a.fan = fanOf(1, &out, &won, nil)
|
|
|
|
_, by, _, err := a.serve()
|
|
if err != nil {
|
|
t.Fatalf("a fan of one must behave exactly like no fan at all: %v", err)
|
|
}
|
|
if by.name != "do-ai" {
|
|
t.Errorf("served by %q, want do-ai — the cascade's failover must still happen", by.name)
|
|
}
|
|
if strings.Join(f.asked, ",") != "enso,do-ai" {
|
|
t.Errorf("asked %v, want the providers in turn", f.asked)
|
|
}
|
|
}
|
|
|
|
// THE MECHANISM IS INERT UNTIL SOMEBODY SETS A NUMBER. Nothing in production
|
|
// sets ask.fan today, and this is what says so: with it nil, the request takes
|
|
// the same path, in the same order, as it did before any of this existed.
|
|
func TestWithoutAFanNothingChanges(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
f := &fleet{answer: map[string]error{
|
|
"enso": apiErr(500, "boom"),
|
|
"do-ai": apiErr(500, "boom"),
|
|
}}
|
|
f.install(t)
|
|
|
|
var w buffer
|
|
_, _, tried, err := newAsk(route("enso", "do-ai", "openai"), &w, nil).serve()
|
|
if err != nil {
|
|
t.Fatalf("serve: %v", err)
|
|
}
|
|
if strings.Join(f.asked, ",") != "enso,do-ai,openai" {
|
|
t.Errorf("asked %v, want them one at a time in declared order", f.asked)
|
|
}
|
|
if len(tried) != 2 {
|
|
t.Fatalf("tried %d, want the 2 that refused", len(tried))
|
|
}
|
|
for _, at := range tried {
|
|
if at.prompt != 0 || at.completion != 0 {
|
|
t.Errorf("%s carries tokens on the unhedged path", at.provider)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Every racer refusing is not a silent empty answer. The caller must be told who
|
|
// was asked, the same way the cascade names them.
|
|
func TestEveryRacerRefusingNamesThemAll(t *testing.T) {
|
|
cooled.forget()
|
|
t.Cleanup(cooled.forget)
|
|
quick(t)
|
|
|
|
r := &racers{fail: map[string]error{
|
|
"enso": apiErr(500, "boom"),
|
|
"do-ai": apiErr(503, "down"),
|
|
}}
|
|
r.install(t)
|
|
|
|
var out body
|
|
var won *plain
|
|
a := newAsk(route("enso", "do-ai"), nil, nil)
|
|
a.fan = fanOf(2, &out, &won, nil)
|
|
|
|
res, _, _, err := a.serve()
|
|
if err == nil {
|
|
t.Fatal("every provider refused and the race reported success")
|
|
}
|
|
if res != nil {
|
|
t.Error("a result came back from a race nobody won")
|
|
}
|
|
// Who was asked is ours: the caller is told the model is unavailable.
|
|
for _, name := range []string{"enso (", "do-ai"} {
|
|
if strings.Contains(err.Error(), name) {
|
|
t.Errorf("the error names %s: %v", name, err)
|
|
}
|
|
}
|
|
}
|