Files
4fdd6c03bf feat(api): /v1/3d|/music|/foley/generations for creative models (#146)
Add ThreeDGenerations, catalog modeOutputs/endpointOf for 3d, and
music/foley generation path aliases that keep /v1/audio/* working.

Co-authored-by: zeekay <[email protected]>
2026-11-09 14:50:18 -08:00

404 lines
16 KiB
Go

// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Native ZAP handlers for the audio route-group — the strangler migration of
// audio_api.go / zen_media.go / text_to_speech.go / speech_to_text.go off the controller layer.
//
// Pure ZAP handlers (no ApiController, no http.ResponseWriter): they
// re-implement each HTTP handler against object/ + iam, mirroring zapChatHandler
// exactly — identity from the auth seam (STEP 1), org = authUser.Owner (STEP 2),
// the SAME request structs (STEP 3), the SAME balance/KMS gates (STEP 4), the
// object-layer provider primitives (STEP 5), ONE meter (recordUsage +
// recordTrace, STEP 6), the SAME response bodies (STEP 7).
//
// Routes migrated (self-registered into the dispatch registry below — this file
// only CALLS registerCloud/registerGatewayPath from its own init()):
//
// POST /v1/audio/speech -> audio.speech (OpenAI TTS)
// POST /v1/audio/transcriptions -> audio.transcribe (OpenAI STT)
// POST /v1/audio/voice -> audio.voice (Zen)
// POST /v1/audio/music -> audio.music (Zen)
// POST /v1/audio/foley -> audio.foley (Zen)
//
// The Zen media forward is the ONE media relay, zenMedia (zen_media.go), through
// zapServeZenMedia, so there is one zen-media meter, not a fork.
package controllers
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"strings"
"time"
"github.com/google/uuid"
iam "github.com/hanzoai/ai/internal/iam"
"github.com/hanzoai/ai/log"
"github.com/luxfi/zap"
"github.com/hanzoai/ai/object"
"github.com/hanzoai/ai/stt"
)
// The canonical ZAP dispatch registry (registerCloud / registerGatewayPath /
// registerGatewayRoute + the lookups) lives in zap_registry.go. This group only
// self-registers from its own init().
func init() {
registerZapAudio()
}
// registerZapAudio wires the audio group's handlers into the shared registry.
func registerZapAudio() {
registerCloud("audio.speech", zapAudioSpeechHandler)
registerCloud("audio.transcribe", zapAudioTranscribeHandler)
registerCloud("audio.voice", zapAudioVerbHandler("voice"))
registerCloud("audio.music", zapAudioVerbHandler("music"))
registerCloud("audio.foley", zapAudioVerbHandler("foley"))
registerGatewayPath("/v1/audio/speech", zapAudioSpeechHandler)
registerGatewayPath("/v1/audio/transcriptions", zapAudioTranscribeHandler)
registerGatewayPath("/v1/audio/voice", zapAudioVerbHandler("voice"))
registerGatewayPath("/v1/audio/music", zapAudioVerbHandler("music"))
registerGatewayPath("/v1/audio/foley", zapAudioVerbHandler("foley"))
registerGatewayPath("/v1/music/generations", zapAudioVerbHandler("music"))
registerGatewayPath("/v1/foley/generations", zapAudioVerbHandler("foley"))
}
// ── audio.speech — OpenAI-compatible TTS (mirrors ApiController.AudioSpeech) ─
func zapAudioSpeechHandler(ctx context.Context, auth string, body []byte) (*zap.Message, error) {
if auth == "" {
return object.BuildCloudResponse(401, nil, "authentication required")
}
var req audioSpeechRequest
if err := json.Unmarshal(body, &req); err != nil {
return object.BuildCloudResponse(400, nil, "invalid request: "+err.Error())
}
// The SAME bound the HTTP handler applies (MaxSpeechInput): one wire shape,
// one limit, so neither transport is the cheap way in.
//
// Checked with the body's other shape checks, before identity is resolved,
// for the reason the transcribe twin below refuses an unparseable body there:
// a size is not a credential oracle — it says nothing about the credential —
// and every byte of work after this point is work the caller has not yet
// earned the right to buy.
if len(req.Input) > MaxSpeechInput {
return object.BuildCloudResponse(413, nil, fmt.Sprintf(
"\"input\" exceeds the %d character limit", MaxSpeechInput))
}
// Identity + provider from the ONE auth seam (STEP 1). Auth is validated
// before the body-field checks, so an invalid credential is 401 regardless of
// body validity (never a probe-able 400), matching the HTTP handler.
provider, authUser, upstreamModel, err := zapResolveAuth(auth, req.Model)
if err != nil {
return object.BuildCloudResponse(401, nil, err.Error())
}
if req.Model == "" {
return object.BuildCloudResponse(400, nil, "audio request requires a \"model\" field")
}
if req.Input == "" {
return object.BuildCloudResponse(400, nil, "audio request requires an \"input\" field")
}
// Prepaid-balance gate — the ONE shared gate (STEP 4).
if gateErr := enforceBalanceGate(ctx, authUser, "", req.Model); gateErr != nil {
return object.BuildCloudResponse(uint32(statusOf(gateErr)), nil, gateErr.Error())
}
if err := zapLane(ctx, provider, req.Model); err != nil {
return object.BuildCloudResponse(uint32(statusOf(err)), nil, err.Error())
}
isPremium := false
if route := resolveModelRoute(req.Model); route != nil {
isPremium = route.premium
}
// Zen family: /v1/audio/speech is the OpenAI-compat alias of zen's voice verb.
if provider.Type == "Zen" {
return zapServeZenMedia("audio/voice", req.Model, body, 1, authUser, isPremium, time.Now().UTC())
}
// KMS secret + upstream-model/voice binding (STEP 4/5), mirroring the HTTP path.
if err := object.ResolveProviderSecret(provider); err != nil {
log.Error("ZAP: KMS resolve %s: %v", provider.Name, err)
}
if upstreamModel != "" {
provider.SubType = upstreamModel
} else if req.Model != "" {
provider.SubType = req.Model
}
if req.Voice != "" {
provider.Flavor = req.Voice
}
ttsProvider, err := provider.GetTextToSpeechProvider("en", req.ResponseFormat)
if err != nil {
return object.BuildCloudResponse(502, nil, err.Error())
}
startTime := time.Now().UTC()
// Admission around the UPSTREAM CALL — the SAME ceiling the HTTP handler
// takes, and the same per-org shares of it, so the two transports divide one
// capacity rather than each getting a private copy. The key is the org
// zapRecordAudioUsage bills below; this transport carries no header, so there
// is nothing here that could name a different one.
release, refused := admitSpeech(authUser.Owner)
if refused != nil {
return object.BuildCloudResponse(uint32(statusOf(refused)), nil, refused.Error())
}
defer release()
audioData, ttsResult, err := ttsProvider.QueryAudio(req.Input, ctx, "en")
spoken := audioQuantity{chars: ttsCharsOf(ttsResult, req.Input)}
if err != nil {
zapRecordAudioUsage(ctx, authUser, provider, req.Model, isPremium, audioQuantity{}, "error", err.Error(), startTime)
return object.BuildCloudResponse(502, nil, err.Error())
}
if len(audioData) == 0 {
zapRecordAudioUsage(ctx, authUser, provider, req.Model, isPremium, audioQuantity{}, "error", "empty audio data", startTime)
return object.BuildCloudResponse(502, nil, "the audio data is nil")
}
zapRecordAudioUsage(ctx, authUser, provider, req.Model, isPremium, spoken, "success", "", startTime)
// The native wire carries the audio bytes in the body; the requested format is
// the provider's, so no transcode. (audioFormat's content-type hint is applied
// by the gateway adapter when this is served over HTTP-over-ZAP.)
return object.BuildCloudResponse(200, audioData, "")
}
// zapRecordAudioUsage mirrors ApiController.recordAudioUsage (STEP 6), quantity
// and all: the ZAP transport meters the same units through the same cost
// switches, so a call bills identically whichever address it arrived at. Errors
// trace either way.
func zapRecordAudioUsage(ctx context.Context, authUser *iam.User, provider *object.Provider, userModel string, isPremium bool, qty audioQuantity, status, errMsg string, startTime time.Time) {
if authUser == nil {
return
}
rec := &usageRecord{
Owner: authUser.Owner,
Organization: authUser.Owner,
Model: userModel,
Provider: provider.Name,
Origin: provider.Origin(),
Currency: "USD",
Premium: isPremium,
Status: status,
ErrorMsg: errMsg,
AudioSeconds: qty.seconds,
AudioChars: qty.chars,
RequestID: uuid.NewString(),
}
rec.bind(ctx, authUser)
recordUsage(rec)
recordTrace(ctx, rec, startTime)
}
// ── audio.transcribe — OpenAI-compatible STT (mirrors AudioTranscriptions) ──
func zapAudioTranscribeHandler(ctx context.Context, auth string, body []byte) (*zap.Message, error) {
if auth == "" {
return object.BuildCloudResponse(401, nil, "authentication required")
}
// The SAME bound the HTTP handler applies (MaxTranscribeUpload). This
// transport is handed the body as one slice, so the bound is checked on the
// slice rather than installed on a reader — the bytes are already here, and
// the check that matters is the one that keeps them from being PARSED into
// three more copies and sent upstream.
if !fits(body) {
return object.BuildCloudResponse(413, nil, fmt.Sprintf(
"audio upload exceeds the %d MiB limit", MaxTranscribeUpload>>20))
}
form, err := parseTranscribeForm(body)
if err != nil {
return object.BuildCloudResponse(400, nil, err.Error())
}
// Identity + provider from the ONE auth seam (STEP 1). Auth is validated
// before the body-field checks, so an invalid credential is 401 regardless of
// body validity (never a probe-able 400), matching the HTTP handler.
provider, authUser, upstreamModel, err := zapResolveAuth(auth, form.model)
if err != nil {
return object.BuildCloudResponse(401, nil, err.Error())
}
if form.audio == nil {
return object.BuildCloudResponse(400, nil, "audio request requires a \"file\" part")
}
if form.model == "" {
return object.BuildCloudResponse(400, nil, "audio request requires a \"model\" field")
}
// Prepaid-balance gate — the ONE shared gate (STEP 4).
if gateErr := enforceBalanceGate(ctx, authUser, "", form.model); gateErr != nil {
return object.BuildCloudResponse(uint32(statusOf(gateErr)), nil, gateErr.Error())
}
if err := zapLane(ctx, provider, form.model); err != nil {
return object.BuildCloudResponse(uint32(statusOf(err)), nil, err.Error())
}
isPremium := false
if route := resolveModelRoute(form.model); route != nil {
isPremium = route.premium
}
// No Zen STT verb exists; a Zen-routed model cannot serve this endpoint.
if provider.Type == "Zen" {
return object.BuildCloudResponse(400, nil, "model \""+form.model+"\" does not serve the /v1/audio/transcriptions endpoint")
}
// KMS secret + upstream-model/language binding (STEP 4/5), mirroring the HTTP path.
if err := object.ResolveProviderSecret(provider); err != nil {
log.Error("ZAP: KMS resolve %s: %v", provider.Name, err)
}
if upstreamModel != "" {
provider.SubType = upstreamModel
} else {
provider.SubType = form.model
}
if form.language != "" {
provider.Flavor = form.language
}
sttProvider, err := provider.GetSpeechToTextProvider("en")
if err != nil {
return object.BuildCloudResponse(502, nil, err.Error())
}
startTime := time.Now().UTC()
// Admission around the UPSTREAM CALL — see the speech handler above.
release, refused := admitSpeech(authUser.Owner)
if refused != nil {
return object.BuildCloudResponse(uint32(statusOf(refused)), nil, refused.Error())
}
defer release()
heard, sttResult, err := sttProvider.ProcessAudio(form.audio, ctx, "en", form.timings)
if err != nil {
zapRecordAudioUsage(ctx, authUser, provider, form.model, isPremium, audioQuantity{}, "error", err.Error(), startTime)
return object.BuildCloudResponse(502, nil, err.Error())
}
zapRecordAudioUsage(ctx, authUser, provider, form.model, isPremium, audioQuantity{seconds: sttSecondsOf(sttResult)}, "success", "", startTime)
if form.responseFormat == "text" {
return object.BuildCloudResponse(200, []byte(heard.Text), "")
}
return object.BuildCloudResponse(200, transcriptionBody(form.responseFormat, heard, sttSecondsOf(sttResult)), "")
}
// transcribeForm is the decoded OpenAI transcription multipart body.
type transcribeForm struct {
model string
language string
responseFormat string
timings []stt.Timing
audio *bytes.Reader
}
// parseTranscribeForm reconstructs the OpenAI multipart form (file + model
// [+ language + response_format]) from the raw body, recovering the boundary
// from the body's own opening line exactly as parseSTTForm does — the handler
// stays on the shared (ctx, auth, body) signature. A missing "file" part is not
// an error here: auth is resolved first, so the handler reports it as a 400
// only to an authenticated caller.
func parseTranscribeForm(body []byte) (*transcribeForm, error) {
boundary := multipartBoundary(body)
if boundary == "" {
return nil, fmt.Errorf("audio request requires a multipart form body")
}
mr := multipart.NewReader(bytes.NewReader(body), boundary)
form, err := mr.ReadForm(MaxTranscribeUpload)
if err != nil {
return nil, fmt.Errorf("read multipart form: %s", err.Error())
}
f := &transcribeForm{}
first := func(key string) string {
if v := form.Value[key]; len(v) > 0 {
return v[0]
}
return ""
}
f.model = first("model")
f.language = first("language")
f.responseFormat = first("response_format")
f.timings, err = timingsOf(form.Value)
if err != nil {
return nil, err
}
if files := form.File["file"]; len(files) > 0 {
fh, err := files[0].Open()
if err != nil {
return nil, fmt.Errorf("open \"file\" part: %s", err.Error())
}
defer fh.Close()
data, err := io.ReadAll(fh)
if err != nil {
return nil, fmt.Errorf("read \"file\" part: %s", err.Error())
}
f.audio = bytes.NewReader(data)
}
return f, nil
}
// ── audio.voice / .music / .foley — Zen-native (mirrors AudioMedia) ─────────
func zapAudioVerbHandler(verb string) zapHandler {
return func(ctx context.Context, auth string, body []byte) (*zap.Message, error) {
if auth == "" {
return object.BuildCloudResponse(401, nil, "authentication required")
}
switch verb {
case "voice", "music", "foley":
default:
return object.BuildCloudResponse(404, nil, "unknown audio verb")
}
var req struct {
Model string `json:"model"`
}
if err := json.Unmarshal(body, &req); err != nil || req.Model == "" {
return object.BuildCloudResponse(400, nil, "audio request requires a \"model\" field")
}
provider, authUser, _, err := zapResolveAuth(auth, req.Model)
if err != nil {
return object.BuildCloudResponse(401, nil, err.Error())
}
if provider.Type != "Zen" {
return object.BuildCloudResponse(400, nil, "model \""+req.Model+"\" does not serve the /v1/audio/"+verb+" endpoint")
}
isPremium := false
if route := resolveModelRoute(req.Model); route != nil {
isPremium = route.premium
}
return zapServeZenMedia("audio/"+verb, req.Model, body, 1, authUser, isPremium, time.Now().UTC())
}
}
// multipartBoundary reads the boundary token off the first line of a multipart
// body. Over ZAP there is no http.Request to parse the Content-Type from, so the
// body states its own boundary and audio.transcribe reads it here.
func multipartBoundary(body []byte) string {
before, _, ok := bytes.Cut(body, []byte{'\n'})
if !ok {
return ""
}
first := strings.TrimRight(string(before), "\r")
if !strings.HasPrefix(first, "--") {
return ""
}
return strings.TrimPrefix(first, "--")
}