Files
ai/controllers/zap_transcript_test.go

483 lines
20 KiB
Go

// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
// What the streaming transcript endpoint has to be true for, stated as the failures
// it prevents: a session's pushes reach the process that holds its window, one
// tenant cannot push audio into another's session, and the audio billed is the
// audio received — once, and all of it.
import (
"context"
"errors"
"fmt"
"math"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/hanzoai/ai/internal/authtest"
iam "github.com/hanzoai/ai/internal/iam"
)
// ── the meter ───────────────────────────────────────────────────────────────
// TestMeterBillsTheDeltaOfSeconds is the money property, and the reason the
// per-push `duration` field is not what is metered.
//
// `seconds` is the session's cumulative total. Billing its delta means the error
// is bounded by ONE reading's rounding no matter how many pushes there were,
// where summing per-push values accumulates every rounding — and a push whose
// answer was lost is covered by the next delta rather than lost revenue.
func TestMeterBillsTheDeltaOfSeconds(t *testing.T) {
s := &session{org: "acme"}
// A real session's reported totals: 250 ms a push, and the field speech
// rounds to a millisecond.
var billed float64
for _, total := range []float64{0.25, 0.5, 0.75, 1.0, 1.25} {
billed += meterDelta(s, total)
}
if billed != 1.25 {
t.Fatalf("billed %v seconds over a 1.25 s session, want 1.25", billed)
}
// And nothing is billed twice: the same total again is no new audio.
if again := meterDelta(s, 1.25); again != 0 {
t.Errorf("a repeated total billed %v more seconds — audio would be charged twice on a retry", again)
}
}
// TestMeterBillsTinyPushes. The per-push `duration` field is rounded to three
// decimals, so a push of 14 bytes or fewer reports 0.0 while the session's own
// total keeps the true float. Metering that field serves audio free; metering the
// delta of the total does not.
func TestMeterBillsTinyPushes(t *testing.T) {
s := &session{org: "acme"}
var billed float64
const pushes = 1000
each := 0.0004 // 12.8 bytes of pcm16 — under `duration`'s rounding floor
total := 0.0
for range pushes {
total += each
billed += meterDelta(s, total)
}
want := pushes * each
if diff := billed - want; diff > 1e-9 || diff < -1e-9 {
t.Fatalf("billed %v seconds, want %v — pushes under the rounding floor are being served free", billed, want)
}
// The control: rounding each push the way `duration` does bills exactly zero,
// which is the bug this test exists to keep out.
rounded := 0.0
for range pushes {
rounded += math.Round(each*1000) / 1000
}
if rounded != 0 {
t.Fatalf("the fixture does not model the rounding trap: per-push rounding billed %v, want 0", rounded)
}
}
// TestMeterNeverRefunds. `seconds` only grows, so a SMALLER total is an upstream
// that restarted under the same id — not audio to give back. A negative debit is
// a refund nobody asked for, applied to a tenant nobody chose.
func TestMeterNeverRefunds(t *testing.T) {
s := &session{org: "acme"}
meterDelta(s, 10)
if back := meterDelta(s, 2); back != 0 {
t.Fatalf("a total that went backwards billed %v — the meter refunded", back)
}
}
// meterDelta is the arithmetic meterTranscript performs, exercised without the
// ledger behind it. It is the same expression, in one place, so a test cannot
// pass against arithmetic the handler does not do.
func meterDelta(s *session, total float64) float64 {
liveMu.Lock()
defer liveMu.Unlock()
delta := total - s.metered
if delta < 0 {
delta = 0
}
s.metered = total
return delta
}
// ── which process holds the session ─────────────────────────────────────────
// TestSessionIsPinnedToOnePod. speech.hanzo.svc is a ClusterIP and kube-proxy
// picks a backend PER CONNECTION, so a session addressed by Service name reaches
// a pod that has never heard of it about half the time at two replicas — and a
// client with a connection pool sees that intermittently, which is worse than a
// clean failure rate.
//
// So every call after the first goes to the address `open` named, and that
// address is a POD, not the Service.
func TestSessionIsPinnedToOnePod(t *testing.T) {
for _, tc := range []struct{ name, at, base, want string }{
{"the pod names itself", "http://10.244.3.17:8000", "http://speech.hanzo.svc/v1", "http://10.244.3.17:8000/v1"},
{"a trailing slash is not a second path segment", "http://10.244.3.17:8000/", "http://speech.hanzo.svc/v1", "http://10.244.3.17:8000/v1"},
// Empty is the honest answer from a single process: there is nothing to
// pin to, so the address already in hand stays correct.
{"nothing to pin to", "", "http://speech.hanzo.svc/v1", "http://speech.hanzo.svc/v1"},
} {
t.Run(tc.name, func(t *testing.T) {
if got := pinned(tc.at, tc.base); got != tc.want {
t.Fatalf("pinned(%q, %q) = %q, want %q", tc.at, tc.base, got, tc.want)
}
if tc.at != "" && strings.Contains(pinned(tc.at, tc.base), "speech.hanzo.svc") {
t.Fatal("a pinned session still addresses the Service — half its pushes will reach the wrong replica")
}
})
}
}
// ── who may push into a session ─────────────────────────────────────────────
// TestASessionBelongsToThePrincipalThatOpenedIt. A transcript id is a bearer of
// nothing. Without the check a guessed id would let one tenant push audio into
// another's meter and read the transcript coming back — the id is in a URL,
// which is the least protected place a secret can live — and one member of an
// org read a colleague's dictation.
//
// The refusal is deliberately the SAME answer a missing session gets: whether an
// id exists is not a fact another principal is entitled to.
func TestASessionBelongsToThePrincipalThatOpenedIt(t *testing.T) {
id := "ats_" + here + "_owned"
liveMu.Lock()
live[id] = &session{org: "acme", user: "acme/alice", touched: time.Now(), release: func() {}}
liveMu.Unlock()
t.Cleanup(func() { forget(id) })
if mine, code, why := findNow("acme/alice", id); mine == nil {
t.Fatalf("the principal that opened the session was refused it: %d %s", code, why)
}
var refusal string
for _, who := range []string{"acme/bob", "other/alice"} {
theirs, code, why := findNow(who, id)
if theirs != nil {
t.Fatalf("SECURITY: %s reached a session acme/alice opened", who)
}
if code != 404 {
t.Errorf("%s was refused with %d, want 404 — the status says the id is real", who, code)
}
refusal = why
}
absent, _, missing := findNow("acme/alice", "ats_"+here+"_never-existed")
if absent != nil {
t.Fatal("a session that was never opened was resolved")
}
// Both refusals name only the id the caller already sent, so the two are the
// same sentence with the caller's own string in it. Comparing the sentences
// with that string removed is what says a stranger cannot tell a real id from
// an invented one.
shape := func(msg, tid string) string { return strings.ReplaceAll(msg, tid, "<id>") }
if got, want := shape(refusal, id), shape(missing, "ats_"+here+"_never-existed"); got != want {
t.Errorf("a foreign session answers %q and a missing one %q — the difference tells a stranger the id is real", got, want)
}
}
// TestAnIdFromAnotherInstanceSaysSo. The record IS the routing decision and the
// meter's running total, so it is worthless to a process that did not open the
// session — but answering "no such transcript" for one that certainly exists
// sends whoever is holding it to look for an expiry bug. Every id names the
// process that minted it, so the wrong instance can say which kind of miss it is.
func TestAnIdFromAnotherInstanceSaysSo(t *testing.T) {
_, code, refusal := findNow("acme/alice", "ats_deadbeef_from-another-replica")
if code != 409 || !strings.Contains(refusal, "another instance") {
t.Fatalf("an id minted elsewhere answers %d %q — indistinguishable from an expired session", code, refusal)
}
if here == "deadbeef" {
t.Skip("this process happens to be deadbeef; the case cannot be modelled")
}
// And an id that is simply unknown, in OUR namespace, is a plain miss.
if _, _, refusal := findNow("acme/alice", "ats_"+here+"_unknown"); strings.Contains(refusal, "another instance") {
t.Fatalf("an unknown id from this instance answers %q — the two misses are not distinguished", refusal)
}
}
// TestAnIdleSessionIsGoneOnPush. speech collects a window untouched for
// transcriptIdle, so a record here older than that names nothing — and with no
// open in between to sweep it, a push would be forwarded to a window that is not
// there and the slot held by a speaker who left. The idle bound holds on a push
// as it does on an open.
func TestAnIdleSessionIsGoneOnPush(t *testing.T) {
id := "ats_" + here + "_idle"
returned := 0
liveMu.Lock()
live[id] = &session{
org: "acme", user: "acme/alice",
touched: time.Now().Add(-2 * transcriptIdle),
release: func() { returned++ },
}
liveMu.Unlock()
if s, code, _ := findNow("acme/alice", id); s != nil || code != 404 {
t.Fatalf("a session idle for %v was resolved (code %d) — it outlives the window it names", 2*transcriptIdle, code)
}
liveMu.Lock()
_, still := live[id]
liveMu.Unlock()
if still || returned != 1 {
t.Fatalf("the idle session is still recorded (%v) or its slot came back %d times, want once", still, returned)
}
}
// TestOnlyAPushOrACloseContinuesATranscript is the shape of the request the
// host's rate ceiling exempts. The open is the one request a transcript is
// counted as, so it is never a continuation — and neither is the batch endpoint
// whose name it is a prefix of, nor a read.
func TestOnlyAPushOrACloseContinuesATranscript(t *testing.T) {
for _, tc := range []struct {
method, path, tid string
ok bool
}{
{http.MethodPost, "/v1/audio/transcript/ats_x", "ats_x", true},
{http.MethodDelete, "/v1/audio/transcript/ats_x", "ats_x", true},
{http.MethodPost, "/v1/audio/transcript/ats_x/", "ats_x", true},
{http.MethodPost, "/v1/audio/transcript", "", false},
{http.MethodPost, "/v1/audio/transcript/", "", false},
{http.MethodGet, "/v1/audio/transcript/ats_x", "", false},
{http.MethodPost, "/v1/audio/transcriptions", "", false},
{http.MethodPost, "/v1/audio/transcriptions/ats_x", "", false},
{http.MethodPost, "/v1/chat/completions", "", false},
} {
tid, ok := continued(tc.method, tc.path)
if ok != tc.ok || (ok && tid != tc.tid) {
t.Errorf("continued(%s %s) = %q, %v; want %q, %v", tc.method, tc.path, tid, ok, tc.tid, tc.ok)
}
}
}
// TestAnUnresolvedCredentialContinuesNothing. The exemption is earned by the
// credential that opened the session; a request whose credential does not
// resolve is counted like any other, live session or not.
func TestAnUnresolvedCredentialContinuesNothing(t *testing.T) {
id := "ats_" + here + "_live"
liveMu.Lock()
live[id] = &session{org: "acme", user: "acme/alice", touched: time.Now(), release: func() {}}
liveMu.Unlock()
t.Cleanup(func() { forget(id) })
for _, auth := range []string{"", "Bearer ", "Bearer not-a-token"} {
if admitted(http.MethodPost, transcriptPath+"/"+id, auth) {
t.Errorf("credential %q continued a session it never opened", auth)
}
}
}
// TestTheOpenersCredentialContinuesItsSession is TranscriptModel end to end,
// through the same credential resolution the push handler runs: the opener's
// signed token continues the session, a colleague's does not, and a stale one or
// a read does not either.
func TestTheOpenersCredentialContinuesItsSession(t *testing.T) {
id := "ats_" + here + "_signed"
liveMu.Lock()
live[id] = &session{org: "acme", user: "acme/alice", touched: time.Now(), release: func() {}}
liveMu.Unlock()
t.Cleanup(func() { forget(id) })
alice := authtest.Bearer(t, iam.User{Owner: "acme", Name: "alice"})
bob := authtest.Bearer(t, iam.User{Owner: "acme", Name: "bob"})
path := transcriptPath + "/" + id
if !admitted(http.MethodPost, path, alice) {
t.Fatal("the opener's push was not recognised as continuing its session")
}
if !admitted(http.MethodDelete, path, alice) {
t.Fatal("the opener's close was not recognised as continuing its session")
}
if admitted(http.MethodPost, path, bob) {
t.Fatal("SECURITY: a colleague's credential continued acme/alice's session")
}
if admitted(http.MethodGet, path, alice) {
t.Fatal("a read continued a session")
}
if admitted(http.MethodPost, transcriptPath, alice) {
t.Fatal("an open was read as a continuation; it is the request a transcript is counted as")
}
}
// TestAFullWindowIsRefusedBeforeTheWire. max_seconds is the window's own bound,
// and with the host's request ceiling standing down for a session's pushes it is
// one of the three that hold them. A push into a full window is refused here,
// without forwarding audio to a pod that would refuse it too; the control is
// that a window with room left is forwarded.
func TestAFullWindowIsRefusedBeforeTheWire(t *testing.T) {
var hits atomic.Int32
pod := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"text":"","pending":"","seconds":0}`))
}))
t.Cleanup(pod.Close)
full := &session{org: "acme", user: "acme/alice", at: pod.URL, upstream: "u1", limit: 600, metered: 600}
msg, err := pushTo(context.Background(), full, "ats_"+here+"_full", make([]byte, 8192))
if code, _ := cloudStatus(t, msg, err); code != 409 {
t.Fatalf("a push into a full window answered %d, want 409", code)
}
if n := hits.Load(); n != 0 {
t.Fatalf("a push into a full window reached the pod %d times — the bound is only speech's", n)
}
room := &session{org: "acme", user: "acme/alice", at: pod.URL, upstream: "u2", limit: 600, metered: 12}
msg, err = pushTo(context.Background(), room, "ats_"+here+"_room", make([]byte, 8192))
if code, _ := cloudStatus(t, msg, err); code != 200 || hits.Load() != 1 {
t.Fatalf("a window with room answered %d after %d forwards, want 200 after 1", code, hits.Load())
}
}
// findNow is find under the lock it requires, at the present moment.
func findNow(who, tid string) (*session, uint32, string) {
liveMu.Lock()
defer liveMu.Unlock()
return find(who, tid, time.Now())
}
// ── capacity ────────────────────────────────────────────────────────────────
// TestAnAbandonedSessionGivesItsSlotBack. A client that goes away without closing
// — a browser tab shut mid-call — leaves a session holding an admission slot. If
// only an explicit close returned it, the ceiling would shrink by one per
// abandoned meeting until the service admitted nobody, with no attacker involved.
func TestAnAbandonedSessionGivesItsSlotBack(t *testing.T) {
var mu sync.Mutex
returned := 0
id := "ats_" + here + "_abandoned"
liveMu.Lock()
live[id] = &session{
org: "acme",
touched: time.Now().Add(-2 * transcriptIdle),
release: func() { mu.Lock(); returned++; mu.Unlock() },
}
sweepTranscripts(time.Now())
_, still := live[id]
liveMu.Unlock()
if still {
t.Fatal("an abandoned session survived the sweep")
}
mu.Lock()
defer mu.Unlock()
if returned != 1 {
t.Fatalf("the sweep returned %d slots, want 1 — the ceiling shrinks by one per abandoned meeting", returned)
}
}
// TestAFreshSessionSurvivesTheSweep is the control: a sweep that dropped
// everything would also pass the test above.
func TestAFreshSessionSurvivesTheSweep(t *testing.T) {
id := "ats_" + here + "_fresh"
liveMu.Lock()
live[id] = &session{org: "acme", touched: time.Now(), release: func() {}}
sweepTranscripts(time.Now())
_, still := live[id]
liveMu.Unlock()
t.Cleanup(func() { forget(id) })
if !still {
t.Fatal("the sweep dropped a session that is in use")
}
}
// ── routing ─────────────────────────────────────────────────────────────────
// TestTranscriptDoesNotSwallowTranscriptions. "/v1/audio/transcript" is a literal
// prefix of "/v1/audio/transcriptions", and the gateway consults the HTTP-shaped
// registry FIRST — so a prefix rule that matched on characters rather than on path
// segments would route every batch transcription into the streaming endpoint, where it
// is a session id nobody opened. The batch endpoint is the one paying customers
// already use.
func TestTranscriptDoesNotSwallowTranscriptions(t *testing.T) {
for _, path := range []string{
"/v1/audio/transcriptions",
"/v1/audio/transcriptions/",
} {
for _, h := range lookupGatewayRoutes(path) {
if _, err := h(context.Background(), http.MethodPost, path, "", "", nil); err == nil || !errors.Is(err, errDecline) {
t.Fatalf("%s is claimed by an HTTP-shaped route — the batch endpoint is being swallowed", path)
}
}
}
// The control: the streaming paths ARE claimed, so the check above is about
// segment boundaries and not about the registry being empty.
for _, path := range []string{"/v1/audio/transcript", "/v1/audio/transcript/ats_x"} {
if len(lookupGatewayRoutes(path)) == 0 {
t.Fatalf("%s is claimed by nobody — the streaming endpoint is not registered", path)
}
}
}
// TestAbandonedSessionsDoNotShutTheDoorForGood is the deadlock.
//
// An abandoned session holds its admission slot until the sweep runs, and the
// sweep runs on open. Order it after admission and the failure is permanent: once
// the ceiling is full of abandoned sessions every open is refused BEFORE reaching
// the sweep that would free them, so the endpoint refuses with nothing running and
// no way back except a restart.
//
// The control is that the same ceiling refuses when the sessions are LIVE, so this
// is about abandonment and not about the ceiling being absent.
func TestAbandonedSessionsDoNotShutTheDoorForGood(t *testing.T) {
speechIdle(t)
var ids []string
t.Cleanup(func() {
for _, id := range ids {
forget(id)
}
})
fill := func(age time.Duration) {
for i := range speechCeiling {
release, refused := admitSpeech("acme")
if refused != nil {
t.Fatalf("could not fill the ceiling: %v", refused)
}
id := fmt.Sprintf("ats_%s_held-%d-%s", here, i, age)
ids = append(ids, id)
liveMu.Lock()
live[id] = &session{org: "acme", touched: time.Now().Add(-age), release: release}
liveMu.Unlock()
}
}
// LIVE sessions: the ceiling is full and must stay full.
fill(0)
if _, refused := admitSession("acme"); refused == nil {
t.Fatal("the ceiling admitted past itself while every session was live")
}
for _, id := range ids {
forget(id)
}
ids = nil
// ABANDONED sessions: the sweep is what gives the capacity back, and it has to
// run where an open can still reach it.
fill(2 * transcriptIdle)
release, refused := admitSession("acme")
if refused != nil {
t.Fatalf("the ceiling is still full of sessions nobody is using: %v — the endpoint never admits again", refused)
}
release()
}
// admitted is whether TranscriptModel names the session a request continues.
func admitted(method, path, auth string) bool {
_, ok := TranscriptModel(method, path, auth)
return ok
}