mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
483 lines
20 KiB
Go
483 lines
20 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
// What the streaming transcript endpoint has to be true for, stated as the failures
|
|
// it prevents: a session's pushes reach the process that holds its window, one
|
|
// tenant cannot push audio into another's session, and the audio billed is the
|
|
// audio received — once, and all of it.
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/internal/authtest"
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
)
|
|
|
|
// ── the meter ───────────────────────────────────────────────────────────────
|
|
|
|
// TestMeterBillsTheDeltaOfSeconds is the money property, and the reason the
|
|
// per-push `duration` field is not what is metered.
|
|
//
|
|
// `seconds` is the session's cumulative total. Billing its delta means the error
|
|
// is bounded by ONE reading's rounding no matter how many pushes there were,
|
|
// where summing per-push values accumulates every rounding — and a push whose
|
|
// answer was lost is covered by the next delta rather than lost revenue.
|
|
func TestMeterBillsTheDeltaOfSeconds(t *testing.T) {
|
|
s := &session{org: "acme"}
|
|
// A real session's reported totals: 250 ms a push, and the field speech
|
|
// rounds to a millisecond.
|
|
var billed float64
|
|
for _, total := range []float64{0.25, 0.5, 0.75, 1.0, 1.25} {
|
|
billed += meterDelta(s, total)
|
|
}
|
|
if billed != 1.25 {
|
|
t.Fatalf("billed %v seconds over a 1.25 s session, want 1.25", billed)
|
|
}
|
|
// And nothing is billed twice: the same total again is no new audio.
|
|
if again := meterDelta(s, 1.25); again != 0 {
|
|
t.Errorf("a repeated total billed %v more seconds — audio would be charged twice on a retry", again)
|
|
}
|
|
}
|
|
|
|
// TestMeterBillsTinyPushes. The per-push `duration` field is rounded to three
|
|
// decimals, so a push of 14 bytes or fewer reports 0.0 while the session's own
|
|
// total keeps the true float. Metering that field serves audio free; metering the
|
|
// delta of the total does not.
|
|
func TestMeterBillsTinyPushes(t *testing.T) {
|
|
s := &session{org: "acme"}
|
|
var billed float64
|
|
const pushes = 1000
|
|
each := 0.0004 // 12.8 bytes of pcm16 — under `duration`'s rounding floor
|
|
total := 0.0
|
|
for range pushes {
|
|
total += each
|
|
billed += meterDelta(s, total)
|
|
}
|
|
want := pushes * each
|
|
if diff := billed - want; diff > 1e-9 || diff < -1e-9 {
|
|
t.Fatalf("billed %v seconds, want %v — pushes under the rounding floor are being served free", billed, want)
|
|
}
|
|
// The control: rounding each push the way `duration` does bills exactly zero,
|
|
// which is the bug this test exists to keep out.
|
|
rounded := 0.0
|
|
for range pushes {
|
|
rounded += math.Round(each*1000) / 1000
|
|
}
|
|
if rounded != 0 {
|
|
t.Fatalf("the fixture does not model the rounding trap: per-push rounding billed %v, want 0", rounded)
|
|
}
|
|
}
|
|
|
|
// TestMeterNeverRefunds. `seconds` only grows, so a SMALLER total is an upstream
|
|
// that restarted under the same id — not audio to give back. A negative debit is
|
|
// a refund nobody asked for, applied to a tenant nobody chose.
|
|
func TestMeterNeverRefunds(t *testing.T) {
|
|
s := &session{org: "acme"}
|
|
meterDelta(s, 10)
|
|
if back := meterDelta(s, 2); back != 0 {
|
|
t.Fatalf("a total that went backwards billed %v — the meter refunded", back)
|
|
}
|
|
}
|
|
|
|
// meterDelta is the arithmetic meterTranscript performs, exercised without the
|
|
// ledger behind it. It is the same expression, in one place, so a test cannot
|
|
// pass against arithmetic the handler does not do.
|
|
func meterDelta(s *session, total float64) float64 {
|
|
liveMu.Lock()
|
|
defer liveMu.Unlock()
|
|
delta := total - s.metered
|
|
if delta < 0 {
|
|
delta = 0
|
|
}
|
|
s.metered = total
|
|
return delta
|
|
}
|
|
|
|
// ── which process holds the session ─────────────────────────────────────────
|
|
|
|
// TestSessionIsPinnedToOnePod. speech.hanzo.svc is a ClusterIP and kube-proxy
|
|
// picks a backend PER CONNECTION, so a session addressed by Service name reaches
|
|
// a pod that has never heard of it about half the time at two replicas — and a
|
|
// client with a connection pool sees that intermittently, which is worse than a
|
|
// clean failure rate.
|
|
//
|
|
// So every call after the first goes to the address `open` named, and that
|
|
// address is a POD, not the Service.
|
|
func TestSessionIsPinnedToOnePod(t *testing.T) {
|
|
for _, tc := range []struct{ name, at, base, want string }{
|
|
{"the pod names itself", "http://10.244.3.17:8000", "http://speech.hanzo.svc/v1", "http://10.244.3.17:8000/v1"},
|
|
{"a trailing slash is not a second path segment", "http://10.244.3.17:8000/", "http://speech.hanzo.svc/v1", "http://10.244.3.17:8000/v1"},
|
|
// Empty is the honest answer from a single process: there is nothing to
|
|
// pin to, so the address already in hand stays correct.
|
|
{"nothing to pin to", "", "http://speech.hanzo.svc/v1", "http://speech.hanzo.svc/v1"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if got := pinned(tc.at, tc.base); got != tc.want {
|
|
t.Fatalf("pinned(%q, %q) = %q, want %q", tc.at, tc.base, got, tc.want)
|
|
}
|
|
if tc.at != "" && strings.Contains(pinned(tc.at, tc.base), "speech.hanzo.svc") {
|
|
t.Fatal("a pinned session still addresses the Service — half its pushes will reach the wrong replica")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// ── who may push into a session ─────────────────────────────────────────────
|
|
|
|
// TestASessionBelongsToThePrincipalThatOpenedIt. A transcript id is a bearer of
|
|
// nothing. Without the check a guessed id would let one tenant push audio into
|
|
// another's meter and read the transcript coming back — the id is in a URL,
|
|
// which is the least protected place a secret can live — and one member of an
|
|
// org read a colleague's dictation.
|
|
//
|
|
// The refusal is deliberately the SAME answer a missing session gets: whether an
|
|
// id exists is not a fact another principal is entitled to.
|
|
func TestASessionBelongsToThePrincipalThatOpenedIt(t *testing.T) {
|
|
id := "ats_" + here + "_owned"
|
|
liveMu.Lock()
|
|
live[id] = &session{org: "acme", user: "acme/alice", touched: time.Now(), release: func() {}}
|
|
liveMu.Unlock()
|
|
t.Cleanup(func() { forget(id) })
|
|
|
|
if mine, code, why := findNow("acme/alice", id); mine == nil {
|
|
t.Fatalf("the principal that opened the session was refused it: %d %s", code, why)
|
|
}
|
|
var refusal string
|
|
for _, who := range []string{"acme/bob", "other/alice"} {
|
|
theirs, code, why := findNow(who, id)
|
|
if theirs != nil {
|
|
t.Fatalf("SECURITY: %s reached a session acme/alice opened", who)
|
|
}
|
|
if code != 404 {
|
|
t.Errorf("%s was refused with %d, want 404 — the status says the id is real", who, code)
|
|
}
|
|
refusal = why
|
|
}
|
|
absent, _, missing := findNow("acme/alice", "ats_"+here+"_never-existed")
|
|
if absent != nil {
|
|
t.Fatal("a session that was never opened was resolved")
|
|
}
|
|
// Both refusals name only the id the caller already sent, so the two are the
|
|
// same sentence with the caller's own string in it. Comparing the sentences
|
|
// with that string removed is what says a stranger cannot tell a real id from
|
|
// an invented one.
|
|
shape := func(msg, tid string) string { return strings.ReplaceAll(msg, tid, "<id>") }
|
|
if got, want := shape(refusal, id), shape(missing, "ats_"+here+"_never-existed"); got != want {
|
|
t.Errorf("a foreign session answers %q and a missing one %q — the difference tells a stranger the id is real", got, want)
|
|
}
|
|
}
|
|
|
|
// TestAnIdFromAnotherInstanceSaysSo. The record IS the routing decision and the
|
|
// meter's running total, so it is worthless to a process that did not open the
|
|
// session — but answering "no such transcript" for one that certainly exists
|
|
// sends whoever is holding it to look for an expiry bug. Every id names the
|
|
// process that minted it, so the wrong instance can say which kind of miss it is.
|
|
func TestAnIdFromAnotherInstanceSaysSo(t *testing.T) {
|
|
_, code, refusal := findNow("acme/alice", "ats_deadbeef_from-another-replica")
|
|
if code != 409 || !strings.Contains(refusal, "another instance") {
|
|
t.Fatalf("an id minted elsewhere answers %d %q — indistinguishable from an expired session", code, refusal)
|
|
}
|
|
if here == "deadbeef" {
|
|
t.Skip("this process happens to be deadbeef; the case cannot be modelled")
|
|
}
|
|
// And an id that is simply unknown, in OUR namespace, is a plain miss.
|
|
if _, _, refusal := findNow("acme/alice", "ats_"+here+"_unknown"); strings.Contains(refusal, "another instance") {
|
|
t.Fatalf("an unknown id from this instance answers %q — the two misses are not distinguished", refusal)
|
|
}
|
|
}
|
|
|
|
// TestAnIdleSessionIsGoneOnPush. speech collects a window untouched for
|
|
// transcriptIdle, so a record here older than that names nothing — and with no
|
|
// open in between to sweep it, a push would be forwarded to a window that is not
|
|
// there and the slot held by a speaker who left. The idle bound holds on a push
|
|
// as it does on an open.
|
|
func TestAnIdleSessionIsGoneOnPush(t *testing.T) {
|
|
id := "ats_" + here + "_idle"
|
|
returned := 0
|
|
liveMu.Lock()
|
|
live[id] = &session{
|
|
org: "acme", user: "acme/alice",
|
|
touched: time.Now().Add(-2 * transcriptIdle),
|
|
release: func() { returned++ },
|
|
}
|
|
liveMu.Unlock()
|
|
|
|
if s, code, _ := findNow("acme/alice", id); s != nil || code != 404 {
|
|
t.Fatalf("a session idle for %v was resolved (code %d) — it outlives the window it names", 2*transcriptIdle, code)
|
|
}
|
|
liveMu.Lock()
|
|
_, still := live[id]
|
|
liveMu.Unlock()
|
|
if still || returned != 1 {
|
|
t.Fatalf("the idle session is still recorded (%v) or its slot came back %d times, want once", still, returned)
|
|
}
|
|
}
|
|
|
|
// TestOnlyAPushOrACloseContinuesATranscript is the shape of the request the
|
|
// host's rate ceiling exempts. The open is the one request a transcript is
|
|
// counted as, so it is never a continuation — and neither is the batch endpoint
|
|
// whose name it is a prefix of, nor a read.
|
|
func TestOnlyAPushOrACloseContinuesATranscript(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
method, path, tid string
|
|
ok bool
|
|
}{
|
|
{http.MethodPost, "/v1/audio/transcript/ats_x", "ats_x", true},
|
|
{http.MethodDelete, "/v1/audio/transcript/ats_x", "ats_x", true},
|
|
{http.MethodPost, "/v1/audio/transcript/ats_x/", "ats_x", true},
|
|
{http.MethodPost, "/v1/audio/transcript", "", false},
|
|
{http.MethodPost, "/v1/audio/transcript/", "", false},
|
|
{http.MethodGet, "/v1/audio/transcript/ats_x", "", false},
|
|
{http.MethodPost, "/v1/audio/transcriptions", "", false},
|
|
{http.MethodPost, "/v1/audio/transcriptions/ats_x", "", false},
|
|
{http.MethodPost, "/v1/chat/completions", "", false},
|
|
} {
|
|
tid, ok := continued(tc.method, tc.path)
|
|
if ok != tc.ok || (ok && tid != tc.tid) {
|
|
t.Errorf("continued(%s %s) = %q, %v; want %q, %v", tc.method, tc.path, tid, ok, tc.tid, tc.ok)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestAnUnresolvedCredentialContinuesNothing. The exemption is earned by the
|
|
// credential that opened the session; a request whose credential does not
|
|
// resolve is counted like any other, live session or not.
|
|
func TestAnUnresolvedCredentialContinuesNothing(t *testing.T) {
|
|
id := "ats_" + here + "_live"
|
|
liveMu.Lock()
|
|
live[id] = &session{org: "acme", user: "acme/alice", touched: time.Now(), release: func() {}}
|
|
liveMu.Unlock()
|
|
t.Cleanup(func() { forget(id) })
|
|
|
|
for _, auth := range []string{"", "Bearer ", "Bearer not-a-token"} {
|
|
if admitted(http.MethodPost, transcriptPath+"/"+id, auth) {
|
|
t.Errorf("credential %q continued a session it never opened", auth)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestTheOpenersCredentialContinuesItsSession is TranscriptModel end to end,
|
|
// through the same credential resolution the push handler runs: the opener's
|
|
// signed token continues the session, a colleague's does not, and a stale one or
|
|
// a read does not either.
|
|
func TestTheOpenersCredentialContinuesItsSession(t *testing.T) {
|
|
id := "ats_" + here + "_signed"
|
|
liveMu.Lock()
|
|
live[id] = &session{org: "acme", user: "acme/alice", touched: time.Now(), release: func() {}}
|
|
liveMu.Unlock()
|
|
t.Cleanup(func() { forget(id) })
|
|
|
|
alice := authtest.Bearer(t, iam.User{Owner: "acme", Name: "alice"})
|
|
bob := authtest.Bearer(t, iam.User{Owner: "acme", Name: "bob"})
|
|
path := transcriptPath + "/" + id
|
|
|
|
if !admitted(http.MethodPost, path, alice) {
|
|
t.Fatal("the opener's push was not recognised as continuing its session")
|
|
}
|
|
if !admitted(http.MethodDelete, path, alice) {
|
|
t.Fatal("the opener's close was not recognised as continuing its session")
|
|
}
|
|
if admitted(http.MethodPost, path, bob) {
|
|
t.Fatal("SECURITY: a colleague's credential continued acme/alice's session")
|
|
}
|
|
if admitted(http.MethodGet, path, alice) {
|
|
t.Fatal("a read continued a session")
|
|
}
|
|
if admitted(http.MethodPost, transcriptPath, alice) {
|
|
t.Fatal("an open was read as a continuation; it is the request a transcript is counted as")
|
|
}
|
|
}
|
|
|
|
// TestAFullWindowIsRefusedBeforeTheWire. max_seconds is the window's own bound,
|
|
// and with the host's request ceiling standing down for a session's pushes it is
|
|
// one of the three that hold them. A push into a full window is refused here,
|
|
// without forwarding audio to a pod that would refuse it too; the control is
|
|
// that a window with room left is forwarded.
|
|
func TestAFullWindowIsRefusedBeforeTheWire(t *testing.T) {
|
|
var hits atomic.Int32
|
|
pod := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
hits.Add(1)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(`{"text":"","pending":"","seconds":0}`))
|
|
}))
|
|
t.Cleanup(pod.Close)
|
|
|
|
full := &session{org: "acme", user: "acme/alice", at: pod.URL, upstream: "u1", limit: 600, metered: 600}
|
|
msg, err := pushTo(context.Background(), full, "ats_"+here+"_full", make([]byte, 8192))
|
|
if code, _ := cloudStatus(t, msg, err); code != 409 {
|
|
t.Fatalf("a push into a full window answered %d, want 409", code)
|
|
}
|
|
if n := hits.Load(); n != 0 {
|
|
t.Fatalf("a push into a full window reached the pod %d times — the bound is only speech's", n)
|
|
}
|
|
|
|
room := &session{org: "acme", user: "acme/alice", at: pod.URL, upstream: "u2", limit: 600, metered: 12}
|
|
msg, err = pushTo(context.Background(), room, "ats_"+here+"_room", make([]byte, 8192))
|
|
if code, _ := cloudStatus(t, msg, err); code != 200 || hits.Load() != 1 {
|
|
t.Fatalf("a window with room answered %d after %d forwards, want 200 after 1", code, hits.Load())
|
|
}
|
|
}
|
|
|
|
// findNow is find under the lock it requires, at the present moment.
|
|
func findNow(who, tid string) (*session, uint32, string) {
|
|
liveMu.Lock()
|
|
defer liveMu.Unlock()
|
|
return find(who, tid, time.Now())
|
|
}
|
|
|
|
// ── capacity ────────────────────────────────────────────────────────────────
|
|
|
|
// TestAnAbandonedSessionGivesItsSlotBack. A client that goes away without closing
|
|
// — a browser tab shut mid-call — leaves a session holding an admission slot. If
|
|
// only an explicit close returned it, the ceiling would shrink by one per
|
|
// abandoned meeting until the service admitted nobody, with no attacker involved.
|
|
func TestAnAbandonedSessionGivesItsSlotBack(t *testing.T) {
|
|
var mu sync.Mutex
|
|
returned := 0
|
|
id := "ats_" + here + "_abandoned"
|
|
|
|
liveMu.Lock()
|
|
live[id] = &session{
|
|
org: "acme",
|
|
touched: time.Now().Add(-2 * transcriptIdle),
|
|
release: func() { mu.Lock(); returned++; mu.Unlock() },
|
|
}
|
|
sweepTranscripts(time.Now())
|
|
_, still := live[id]
|
|
liveMu.Unlock()
|
|
|
|
if still {
|
|
t.Fatal("an abandoned session survived the sweep")
|
|
}
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if returned != 1 {
|
|
t.Fatalf("the sweep returned %d slots, want 1 — the ceiling shrinks by one per abandoned meeting", returned)
|
|
}
|
|
}
|
|
|
|
// TestAFreshSessionSurvivesTheSweep is the control: a sweep that dropped
|
|
// everything would also pass the test above.
|
|
func TestAFreshSessionSurvivesTheSweep(t *testing.T) {
|
|
id := "ats_" + here + "_fresh"
|
|
liveMu.Lock()
|
|
live[id] = &session{org: "acme", touched: time.Now(), release: func() {}}
|
|
sweepTranscripts(time.Now())
|
|
_, still := live[id]
|
|
liveMu.Unlock()
|
|
t.Cleanup(func() { forget(id) })
|
|
if !still {
|
|
t.Fatal("the sweep dropped a session that is in use")
|
|
}
|
|
}
|
|
|
|
// ── routing ─────────────────────────────────────────────────────────────────
|
|
|
|
// TestTranscriptDoesNotSwallowTranscriptions. "/v1/audio/transcript" is a literal
|
|
// prefix of "/v1/audio/transcriptions", and the gateway consults the HTTP-shaped
|
|
// registry FIRST — so a prefix rule that matched on characters rather than on path
|
|
// segments would route every batch transcription into the streaming endpoint, where it
|
|
// is a session id nobody opened. The batch endpoint is the one paying customers
|
|
// already use.
|
|
func TestTranscriptDoesNotSwallowTranscriptions(t *testing.T) {
|
|
for _, path := range []string{
|
|
"/v1/audio/transcriptions",
|
|
"/v1/audio/transcriptions/",
|
|
} {
|
|
for _, h := range lookupGatewayRoutes(path) {
|
|
if _, err := h(context.Background(), http.MethodPost, path, "", "", nil); err == nil || !errors.Is(err, errDecline) {
|
|
t.Fatalf("%s is claimed by an HTTP-shaped route — the batch endpoint is being swallowed", path)
|
|
}
|
|
}
|
|
}
|
|
// The control: the streaming paths ARE claimed, so the check above is about
|
|
// segment boundaries and not about the registry being empty.
|
|
for _, path := range []string{"/v1/audio/transcript", "/v1/audio/transcript/ats_x"} {
|
|
if len(lookupGatewayRoutes(path)) == 0 {
|
|
t.Fatalf("%s is claimed by nobody — the streaming endpoint is not registered", path)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestAbandonedSessionsDoNotShutTheDoorForGood is the deadlock.
|
|
//
|
|
// An abandoned session holds its admission slot until the sweep runs, and the
|
|
// sweep runs on open. Order it after admission and the failure is permanent: once
|
|
// the ceiling is full of abandoned sessions every open is refused BEFORE reaching
|
|
// the sweep that would free them, so the endpoint refuses with nothing running and
|
|
// no way back except a restart.
|
|
//
|
|
// The control is that the same ceiling refuses when the sessions are LIVE, so this
|
|
// is about abandonment and not about the ceiling being absent.
|
|
func TestAbandonedSessionsDoNotShutTheDoorForGood(t *testing.T) {
|
|
speechIdle(t)
|
|
var ids []string
|
|
t.Cleanup(func() {
|
|
for _, id := range ids {
|
|
forget(id)
|
|
}
|
|
})
|
|
|
|
fill := func(age time.Duration) {
|
|
for i := range speechCeiling {
|
|
release, refused := admitSpeech("acme")
|
|
if refused != nil {
|
|
t.Fatalf("could not fill the ceiling: %v", refused)
|
|
}
|
|
id := fmt.Sprintf("ats_%s_held-%d-%s", here, i, age)
|
|
ids = append(ids, id)
|
|
liveMu.Lock()
|
|
live[id] = &session{org: "acme", touched: time.Now().Add(-age), release: release}
|
|
liveMu.Unlock()
|
|
}
|
|
}
|
|
|
|
// LIVE sessions: the ceiling is full and must stay full.
|
|
fill(0)
|
|
if _, refused := admitSession("acme"); refused == nil {
|
|
t.Fatal("the ceiling admitted past itself while every session was live")
|
|
}
|
|
for _, id := range ids {
|
|
forget(id)
|
|
}
|
|
ids = nil
|
|
|
|
// ABANDONED sessions: the sweep is what gives the capacity back, and it has to
|
|
// run where an open can still reach it.
|
|
fill(2 * transcriptIdle)
|
|
release, refused := admitSession("acme")
|
|
if refused != nil {
|
|
t.Fatalf("the ceiling is still full of sessions nobody is using: %v — the endpoint never admits again", refused)
|
|
}
|
|
release()
|
|
}
|
|
|
|
// admitted is whether TranscriptModel names the session a request continues.
|
|
func admitted(method, path, auth string) bool {
|
|
_, ok := TranscriptModel(method, path, auth)
|
|
return ok
|
|
}
|