Files
ai/controllers/zap_transcript.go

482 lines
18 KiB
Go

// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The growing transcript — /v1/audio/transcriptions' streaming sibling.
//
// POST /v1/audio/transcript {model, language} -> {id, chunk_ms, …}
// POST /v1/audio/transcript/{tid} <raw pcm16> -> {text, pending, seconds}
// DELETE /v1/audio/transcript/{tid} -> the settled text
//
// HTTP-SHAPED, not body-only, and it has to be: the session id is in the PATH
// (there is no room for it in a body that is raw audio), and push and close share
// that path and differ only by method. registerGatewayPath hands a handler the
// body and nothing else, so a body-only registration cannot see either.
//
// TWO THINGS THIS FILE OWNS, and they are the reasons it exists rather than the
// gateway simply forwarding:
//
// 1. WHICH PROCESS HOLDS THE SESSION. A growing transcript is a window in one
// speech replica's memory. speech.hanzo.svc is a ClusterIP, and kube-proxy
// picks a backend PER CONNECTION — so with two replicas roughly half of a
// session's pushes reach a pod that has never heard of it and answer 404,
// and a client with a connection pool sees it intermittently. `open` answers
// with the address of the pod that took the session ("at"), and every later
// call for that session goes there. A pod that has gone is a session that has
// gone, which is honest: the window was only ever in its memory.
//
// 2. WHAT IT COST. Audio is metered in seconds and the ledger lives here, not in
// speech. The quantity is the DELTA OF `seconds`, the cumulative total the
// session reports — never the per-push `duration`. Two reasons, and both are
// about not losing money quietly: a cumulative delta has bounded total error
// (only the last reading's rounding survives) where a sum of per-push values
// accumulates every one of them; and if one push's response is lost in
// flight, the next delta covers the gap, where a sum loses that audio for
// good.
package controllers
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/hanzoai/account"
"github.com/hanzoai/ai/object"
"github.com/hanzoai/ai/upstream"
"github.com/luxfi/zap"
)
const transcriptPath = "/v1/audio/transcript"
// transcriptCeiling is the largest push this endpoint forwards, matching the ceiling
// speech itself enforces (transcript.CEILING = 2 s of pcm16). Checked here as
// well because a body that is refused upstream still crossed the wire, and this
// is the endpoint that is reachable from outside the cluster.
const transcriptCeiling = 64 * 1024
// transcriptIdle is how long an untouched session is kept. speech collects its
// own at the same age, so a record outliving the window it names would only ever
// be a way to forward a push to a session that is gone.
const transcriptIdle = 30 * time.Second
func init() {
registerGatewayRoute(transcriptPath, zapTranscriptHandler)
registerCloud("audio.transcript", zapTranscriptCloudHandler)
}
// session is one open transcript: who pays for it, where it lives, and how much
// of it has already been billed.
type session struct {
org string
user string
model string
premium bool
provider string // provider row name, for the usage record
// payer is the money address this session spends from, resolved ONCE at open.
// A credential's payer is a property of the credential, not of the chunk it
// happens to be pushing, and re-resolving it per push is how a long session
// could start billing a different ledger halfway through.
payer account.Account
at string // the speech pod that holds the window
upstream string // the id THAT pod knows the session by
metered float64
touched time.Time
// release returns the admission slot this session holds. A live transcript is
// CONTINUOUS load — one open session keeps a decode worker busy for as long as
// the meeting lasts — so the slot is taken for the session, not for a push. A
// ceiling applied per push would admit any number of meetings and then refuse
// chunks in the middle of them, which is the same capacity spent to produce a
// worse answer.
release func()
}
// live is this process's open sessions. In-process because the record IS the
// routing decision and the meter's running total, and both are worthless to a
// process that did not open the session — there is nothing here another replica
// could usefully read.
//
// `here` is what makes that legible instead of mysterious. Every id this endpoint
// hands out names the process that minted it, so an id presented to a DIFFERENT
// instance is answered "opened by another instance" rather than "no such
// transcript" — the same 404 an expired session gets, which is exactly the
// confusion that would send someone hunting the wrong bug.
var (
liveMu sync.Mutex
live = map[string]*session{}
here = mintNonce()
)
func mintNonce() string {
b := make([]byte, 4)
_, _ = rand.Read(b)
return hex.EncodeToString(b)
}
// zapTranscriptHandler serves the three calls. It declines nothing: the whole
// /v1/audio/transcript subtree is this file's.
func zapTranscriptHandler(ctx context.Context, method, path, query, auth string, body []byte) (*zap.Message, error) {
if auth == "" {
return object.BuildCloudResponse(401, nil, "authentication required")
}
tid := strings.Trim(strings.TrimPrefix(path, transcriptPath), "/")
switch {
case tid == "" && strings.EqualFold(method, http.MethodPost):
return transcriptOpen(ctx, auth, body)
case tid != "" && strings.EqualFold(method, http.MethodPost):
return transcriptPush(ctx, auth, tid, body)
case tid != "" && strings.EqualFold(method, http.MethodDelete):
return transcriptClose(ctx, auth, tid)
}
return object.BuildCloudResponse(405, nil, "the transcript endpoint takes POST to open and push, DELETE to close")
}
// zapTranscriptCloudHandler is the native-cloud (MsgType 100) entry point onto the same
// three calls. A method name is not a URL, so the call it means travels in the
// body alongside its arguments.
func zapTranscriptCloudHandler(ctx context.Context, auth string, body []byte) (*zap.Message, error) {
var in struct {
ID string `json:"id"`
Close bool `json:"close"`
Audio []byte `json:"audio"`
}
if err := json.Unmarshal(body, &in); err != nil {
return object.BuildCloudResponse(400, nil, "invalid request: "+err.Error())
}
switch {
case in.ID == "":
return transcriptOpen(ctx, auth, body)
case in.Close:
return transcriptClose(ctx, auth, in.ID)
default:
return transcriptPush(ctx, auth, in.ID, in.Audio)
}
}
// transcriptOpen resolves the caller, opens a window on ONE speech pod, and
// records where it lives.
func transcriptOpen(ctx context.Context, auth string, body []byte) (*zap.Message, error) {
var req struct {
Model string `json:"model"`
Language string `json:"language"`
}
if err := json.Unmarshal(body, &req); err != nil {
return object.BuildCloudResponse(400, nil, "invalid request: "+err.Error())
}
if req.Model == "" {
return object.BuildCloudResponse(400, nil, "transcript request requires a \"model\" field")
}
provider, authUser, upstreamModel, err := zapResolveAuth(auth, req.Model)
if err != nil {
return object.BuildCloudResponse(401, nil, err.Error())
}
// The SAME prepaid gate every other audio endpoint takes. It runs at OPEN, so a
// caller with no balance is refused before a window exists to push into —
// the alternative is discovering it 250 ms at a time.
if gateErr := enforceBalanceGate(authUser, "", req.Model); gateErr != nil {
return object.BuildCloudResponse(uint32(statusOf(gateErr)), nil, gateErr.Error())
}
if provider.Type == "Zen" {
return object.BuildCloudResponse(400, nil, "model \""+req.Model+"\" does not serve the /v1/audio/transcript endpoint")
}
release, refused := admitSession(authUser.Owner)
if refused != nil {
return object.BuildCloudResponse(uint32(statusOf(refused)), nil, refused.Error())
}
if err := object.ResolveProviderSecret(provider); err != nil {
return object.BuildCloudResponse(502, nil, "provider secret: "+err.Error())
}
model := upstreamModel
if model == "" {
model = req.Model
}
ask, _ := json.Marshal(map[string]any{"model": model, "language": req.Language})
base := strings.TrimSuffix(provider.ProviderUrl, "/")
status, opened, err := transcriptCall(ctx, http.MethodPost, base+"/audio/transcript", provider, ask, "application/json")
// Every refusal from here on gives the slot back. A window that was never
// opened holds no capacity, and a slot kept for one would shrink the ceiling
// by one for the life of the process.
if err != nil {
release()
return object.BuildCloudResponse(502, nil, "open transcript: "+err.Error())
}
if status != http.StatusCreated && status != http.StatusOK {
release()
return object.BuildCloudResponse(uint32(status), nil, upstreamErrorMessage(opened))
}
var out map[string]any
if err := json.Unmarshal(opened, &out); err != nil {
release()
return object.BuildCloudResponse(502, nil, "open transcript: upstream answered "+err.Error())
}
upstream, _ := out["id"].(string)
if upstream == "" {
release()
return object.BuildCloudResponse(502, nil, "open transcript: upstream named no session")
}
at, _ := out["at"].(string)
at = pinned(at, base)
id := "ats_" + here + "_" + uuid.NewString()
isPremium := false
if route := resolveModelRoute(req.Model); route != nil {
isPremium = route.premium
}
liveMu.Lock()
live[id] = &session{
org: authUser.Owner, user: authUser.Owner + "/" + authUser.Name,
model: req.Model, premium: isPremium, provider: provider.Name,
payer: authUser.Payer(authUser.Owner),
at: at, upstream: upstream, touched: time.Now(), release: release,
}
liveMu.Unlock()
// The caller is handed OUR id and never the upstream's: the upstream id names
// a session on a pod, and the pod is not the caller's business.
out["id"] = id
delete(out, "at")
answer, _ := json.Marshal(out)
return object.BuildCloudResponse(201, answer, "")
}
// admitSession collects abandoned sessions, then takes a slot for a new one.
//
// ONE function, because the ORDER is the property. An abandoned session holds its
// slot until the sweep runs, and the sweep runs on open — so a sweep placed AFTER
// admission can never run once the ceiling is full of abandoned sessions: every
// open is refused before reaching it, and the endpoint refuses for good with
// nothing running and no way back but a restart. Written as two steps at the call
// site, that ordering is a comment; written here, it is a function a test can
// call.
func admitSession(org string) (func(), error) {
liveMu.Lock()
sweepTranscripts(time.Now())
liveMu.Unlock()
return admitSpeech(org)
}
// pinned is the address the rest of a session goes to: the pod `open` named, or
// the address already in hand when it named none.
//
// Empty is not a failure. A single upstream process has nothing to pin to, and
// the Service address is then exactly right — so the honest answer is to keep it,
// not to guess a host. Above one replica it IS a failure, and it is the
// deployment's: speech reads its own address from the downward API, and the
// manifest that omits it is what this cannot fix from here.
func pinned(at, base string) string {
if at == "" {
return base
}
return strings.TrimSuffix(at, "/") + "/v1"
}
// transcriptPush forwards audio to the pod holding the window and meters the
// audio the answer says has now been received.
func transcriptPush(ctx context.Context, auth string, tid string, pcm []byte) (*zap.Message, error) {
if len(pcm) > transcriptCeiling {
return object.BuildCloudResponse(413, nil, fmt.Sprintf("chunk is %d bytes; limit %d", len(pcm), transcriptCeiling))
}
s, msg := holder(auth, tid)
if msg != nil {
return msg, nil
}
start := time.Now().UTC()
status, answer, err := transcriptCall(ctx, http.MethodPost, s.at+"/audio/transcript/"+s.upstream, nil, pcm, "application/octet-stream")
if err != nil {
return object.BuildCloudResponse(502, nil, "push: "+err.Error())
}
if status != http.StatusOK {
// The window is gone upstream (expired, or its pod restarted), so the
// record here names nothing. Drop it rather than leave a session that can
// only ever fail.
if status == http.StatusNotFound {
forget(tid)
}
return object.BuildCloudResponse(uint32(status), nil, upstreamErrorMessage(answer))
}
return object.BuildCloudResponse(200, meterTranscript(ctx, s, tid, answer, start), "")
}
// transcriptClose settles the transcript and forgets it. Closing carries no
// audio, but it is metered the same way for the same reason: `seconds` is the
// truth, and the last push's answer may never have arrived.
func transcriptClose(ctx context.Context, auth string, tid string) (*zap.Message, error) {
s, msg := holder(auth, tid)
if msg != nil {
return msg, nil
}
start := time.Now().UTC()
status, answer, err := transcriptCall(ctx, http.MethodDelete, s.at+"/audio/transcript/"+s.upstream, nil, nil, "")
forget(tid)
if err != nil {
return object.BuildCloudResponse(502, nil, "close: "+err.Error())
}
if status != http.StatusOK {
return object.BuildCloudResponse(uint32(status), nil, upstreamErrorMessage(answer))
}
return object.BuildCloudResponse(200, meterTranscript(ctx, s, tid, answer, start), "")
}
// meterTranscript debits the audio that arrived since this session was last
// metered and returns the answer with the upstream's id replaced by ours.
//
// The delta is clamped at zero: `seconds` only grows, so a smaller reading is an
// upstream that has been restarted under the same id, and a negative debit is a
// refund nobody asked for.
func meterTranscript(ctx context.Context, s *session, tid string, answer []byte, start time.Time) []byte {
var state map[string]any
if err := json.Unmarshal(answer, &state); err != nil {
return answer
}
total, _ := state["seconds"].(float64)
liveMu.Lock()
delta := total - s.metered
if delta < 0 {
delta = 0
}
s.metered = total
s.touched = time.Now()
liveMu.Unlock()
if delta > 0 {
rec := &usageRecord{
Owner: s.org, User: s.user, Organization: s.org,
Model: s.model, Provider: s.provider, Currency: "USD",
Premium: s.premium, Status: "success",
AudioSeconds: delta,
RequestID: uuid.NewString(),
Payer: s.payer,
}
recordUsage(rec)
recordTrace(ctx, rec, start)
}
state["id"] = tid
// `duration` is the upstream's per-push figure, and it is NOT what was
// metered: it is rounded to a millisecond, so a sum of it drifts, and a push
// under 14 bytes reports zero. The answer carries what this call actually
// billed instead, so the caller and the ledger cannot disagree.
state["duration"] = delta
out, err := json.Marshal(state)
if err != nil {
return answer
}
return out
}
// holder resolves the session a call names, and refuses in a way that says which
// kind of miss it was.
//
// The ORG is checked, not merely the id: an id is a bearer of nothing, and a
// session belongs to the tenant that opened it. Without this a guessed id would
// let one tenant push audio into another's meter — and read their transcript.
func holder(auth string, tid string) (*session, *zap.Message) {
user, err := zapResolveUser(auth)
if err != nil {
msg, _ := object.BuildCloudResponse(401, nil, err.Error())
return nil, msg
}
owner := orgOf(user)
liveMu.Lock()
defer liveMu.Unlock()
s, ok := live[tid]
if !ok {
if parts := strings.SplitN(tid, "_", 3); len(parts) == 3 && parts[0] == "ats" && parts[1] != here {
msg, _ := object.BuildCloudResponse(409, nil, "this transcript was opened by another instance; open a new one")
return nil, msg
}
msg, _ := object.BuildCloudResponse(404, nil, "no transcript "+tid)
return nil, msg
}
if s.org != owner {
// Deliberately the same answer a missing session gets: whether an id
// exists is not a fact another tenant is entitled to.
msg, _ := object.BuildCloudResponse(404, nil, "no transcript "+tid)
return nil, msg
}
s.touched = time.Now()
return s, nil
}
func forget(tid string) {
liveMu.Lock()
if s, ok := live[tid]; ok {
delete(live, tid)
s.release()
}
liveMu.Unlock()
}
// sweepTranscripts drops records for sessions speech has already collected. It
// runs from open, for the reason speech sweeps there too: there is then no clock
// to run and no task to supervise.
//
// The caller holds liveMu. Admitting decides on this map and then writes it, so
// one lock spans both — taken again in here it would deadlock, and left out
// entirely it would be a map written from one request while another reads it,
// which is a runtime fatal error no recover answers for.
func sweepTranscripts(now time.Time) {
for id, s := range live {
if now.Sub(s.touched) > transcriptIdle {
delete(live, id)
// A session abandoned mid-call — the client went away without closing —
// still holds a slot until this runs. Sweeping is the only thing that
// gives it back, which is why the sweep is not optional.
s.release()
}
}
}
// transcriptClient is separate from the chat and zen pools: a push is small and
// frequent and must not wait behind a minutes-long completion, and its timeout is
// the ack's, not a generation's.
var transcriptClient = &http.Client{Timeout: 30 * time.Second}
// provider is nil for a call inside an open window: the pod was reached with the
// credential once, and the window's own address is what identifies it after that.
func transcriptCall(ctx context.Context, method, url string, provider *object.Provider, body []byte, ctype string) (int, []byte, error) {
var reader io.Reader
if body != nil {
reader = bytes.NewReader(body)
}
req, err := http.NewRequestWithContext(ctx, method, url, reader)
if err != nil {
return 0, nil, err
}
if ctype != "" {
req.Header.Set("Content-Type", ctype)
}
upstream.Authorize(req, provider)
resp, err := transcriptClient.Do(req)
if err != nil {
return 0, nil, err
}
defer func() { _ = resp.Body.Close() }()
answer, err := io.ReadAll(resp.Body)
if err != nil {
return resp.StatusCode, nil, err
}
return resp.StatusCode, answer, nil
}