mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
Add ThreeDGenerations, catalog modeOutputs/endpointOf for 3d, and music/foley generation path aliases that keep /v1/audio/* working. Co-authored-by: zeekay <[email protected]>
404 lines
16 KiB
Go
404 lines
16 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
// Native ZAP handlers for the audio route-group — the strangler migration of
|
|
// audio_api.go / zen_media.go / text_to_speech.go / speech_to_text.go off the controller layer.
|
|
//
|
|
// Pure ZAP handlers (no ApiController, no http.ResponseWriter): they
|
|
// re-implement each HTTP handler against object/ + iam, mirroring zapChatHandler
|
|
// exactly — identity from the auth seam (STEP 1), org = authUser.Owner (STEP 2),
|
|
// the SAME request structs (STEP 3), the SAME balance/KMS gates (STEP 4), the
|
|
// object-layer provider primitives (STEP 5), ONE meter (recordUsage +
|
|
// recordTrace, STEP 6), the SAME response bodies (STEP 7).
|
|
//
|
|
// Routes migrated (self-registered into the dispatch registry below — this file
|
|
// only CALLS registerCloud/registerGatewayPath from its own init()):
|
|
//
|
|
// POST /v1/audio/speech -> audio.speech (OpenAI TTS)
|
|
// POST /v1/audio/transcriptions -> audio.transcribe (OpenAI STT)
|
|
// POST /v1/audio/voice -> audio.voice (Zen)
|
|
// POST /v1/audio/music -> audio.music (Zen)
|
|
// POST /v1/audio/foley -> audio.foley (Zen)
|
|
//
|
|
// The Zen media forward is the ONE media relay, zenMedia (zen_media.go), through
|
|
// zapServeZenMedia, so there is one zen-media meter, not a fork.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"mime/multipart"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/luxfi/zap"
|
|
|
|
"github.com/hanzoai/ai/object"
|
|
"github.com/hanzoai/ai/stt"
|
|
)
|
|
|
|
// The canonical ZAP dispatch registry (registerCloud / registerGatewayPath /
|
|
// registerGatewayRoute + the lookups) lives in zap_registry.go. This group only
|
|
// self-registers from its own init().
|
|
|
|
func init() {
|
|
registerZapAudio()
|
|
}
|
|
|
|
// registerZapAudio wires the audio group's handlers into the shared registry.
|
|
func registerZapAudio() {
|
|
registerCloud("audio.speech", zapAudioSpeechHandler)
|
|
registerCloud("audio.transcribe", zapAudioTranscribeHandler)
|
|
registerCloud("audio.voice", zapAudioVerbHandler("voice"))
|
|
registerCloud("audio.music", zapAudioVerbHandler("music"))
|
|
registerCloud("audio.foley", zapAudioVerbHandler("foley"))
|
|
|
|
registerGatewayPath("/v1/audio/speech", zapAudioSpeechHandler)
|
|
registerGatewayPath("/v1/audio/transcriptions", zapAudioTranscribeHandler)
|
|
registerGatewayPath("/v1/audio/voice", zapAudioVerbHandler("voice"))
|
|
registerGatewayPath("/v1/audio/music", zapAudioVerbHandler("music"))
|
|
registerGatewayPath("/v1/audio/foley", zapAudioVerbHandler("foley"))
|
|
registerGatewayPath("/v1/music/generations", zapAudioVerbHandler("music"))
|
|
registerGatewayPath("/v1/foley/generations", zapAudioVerbHandler("foley"))
|
|
}
|
|
|
|
// ── audio.speech — OpenAI-compatible TTS (mirrors ApiController.AudioSpeech) ─
|
|
|
|
func zapAudioSpeechHandler(ctx context.Context, auth string, body []byte) (*zap.Message, error) {
|
|
if auth == "" {
|
|
return object.BuildCloudResponse(401, nil, "authentication required")
|
|
}
|
|
var req audioSpeechRequest
|
|
if err := json.Unmarshal(body, &req); err != nil {
|
|
return object.BuildCloudResponse(400, nil, "invalid request: "+err.Error())
|
|
}
|
|
// The SAME bound the HTTP handler applies (MaxSpeechInput): one wire shape,
|
|
// one limit, so neither transport is the cheap way in.
|
|
//
|
|
// Checked with the body's other shape checks, before identity is resolved,
|
|
// for the reason the transcribe twin below refuses an unparseable body there:
|
|
// a size is not a credential oracle — it says nothing about the credential —
|
|
// and every byte of work after this point is work the caller has not yet
|
|
// earned the right to buy.
|
|
if len(req.Input) > MaxSpeechInput {
|
|
return object.BuildCloudResponse(413, nil, fmt.Sprintf(
|
|
"\"input\" exceeds the %d character limit", MaxSpeechInput))
|
|
}
|
|
|
|
// Identity + provider from the ONE auth seam (STEP 1). Auth is validated
|
|
// before the body-field checks, so an invalid credential is 401 regardless of
|
|
// body validity (never a probe-able 400), matching the HTTP handler.
|
|
provider, authUser, upstreamModel, err := zapResolveAuth(auth, req.Model)
|
|
if err != nil {
|
|
return object.BuildCloudResponse(401, nil, err.Error())
|
|
}
|
|
if req.Model == "" {
|
|
return object.BuildCloudResponse(400, nil, "audio request requires a \"model\" field")
|
|
}
|
|
if req.Input == "" {
|
|
return object.BuildCloudResponse(400, nil, "audio request requires an \"input\" field")
|
|
}
|
|
|
|
// Prepaid-balance gate — the ONE shared gate (STEP 4).
|
|
if gateErr := enforceBalanceGate(ctx, authUser, "", req.Model); gateErr != nil {
|
|
return object.BuildCloudResponse(uint32(statusOf(gateErr)), nil, gateErr.Error())
|
|
}
|
|
if err := zapLane(ctx, provider, req.Model); err != nil {
|
|
return object.BuildCloudResponse(uint32(statusOf(err)), nil, err.Error())
|
|
}
|
|
isPremium := false
|
|
if route := resolveModelRoute(req.Model); route != nil {
|
|
isPremium = route.premium
|
|
}
|
|
|
|
// Zen family: /v1/audio/speech is the OpenAI-compat alias of zen's voice verb.
|
|
if provider.Type == "Zen" {
|
|
return zapServeZenMedia("audio/voice", req.Model, body, 1, authUser, isPremium, time.Now().UTC())
|
|
}
|
|
|
|
// KMS secret + upstream-model/voice binding (STEP 4/5), mirroring the HTTP path.
|
|
if err := object.ResolveProviderSecret(provider); err != nil {
|
|
log.Error("ZAP: KMS resolve %s: %v", provider.Name, err)
|
|
}
|
|
if upstreamModel != "" {
|
|
provider.SubType = upstreamModel
|
|
} else if req.Model != "" {
|
|
provider.SubType = req.Model
|
|
}
|
|
if req.Voice != "" {
|
|
provider.Flavor = req.Voice
|
|
}
|
|
|
|
ttsProvider, err := provider.GetTextToSpeechProvider("en", req.ResponseFormat)
|
|
if err != nil {
|
|
return object.BuildCloudResponse(502, nil, err.Error())
|
|
}
|
|
|
|
startTime := time.Now().UTC()
|
|
// Admission around the UPSTREAM CALL — the SAME ceiling the HTTP handler
|
|
// takes, and the same per-org shares of it, so the two transports divide one
|
|
// capacity rather than each getting a private copy. The key is the org
|
|
// zapRecordAudioUsage bills below; this transport carries no header, so there
|
|
// is nothing here that could name a different one.
|
|
release, refused := admitSpeech(authUser.Owner)
|
|
if refused != nil {
|
|
return object.BuildCloudResponse(uint32(statusOf(refused)), nil, refused.Error())
|
|
}
|
|
defer release()
|
|
|
|
audioData, ttsResult, err := ttsProvider.QueryAudio(req.Input, ctx, "en")
|
|
spoken := audioQuantity{chars: ttsCharsOf(ttsResult, req.Input)}
|
|
if err != nil {
|
|
zapRecordAudioUsage(ctx, authUser, provider, req.Model, isPremium, audioQuantity{}, "error", err.Error(), startTime)
|
|
return object.BuildCloudResponse(502, nil, err.Error())
|
|
}
|
|
if len(audioData) == 0 {
|
|
zapRecordAudioUsage(ctx, authUser, provider, req.Model, isPremium, audioQuantity{}, "error", "empty audio data", startTime)
|
|
return object.BuildCloudResponse(502, nil, "the audio data is nil")
|
|
}
|
|
zapRecordAudioUsage(ctx, authUser, provider, req.Model, isPremium, spoken, "success", "", startTime)
|
|
|
|
// The native wire carries the audio bytes in the body; the requested format is
|
|
// the provider's, so no transcode. (audioFormat's content-type hint is applied
|
|
// by the gateway adapter when this is served over HTTP-over-ZAP.)
|
|
return object.BuildCloudResponse(200, audioData, "")
|
|
}
|
|
|
|
// zapRecordAudioUsage mirrors ApiController.recordAudioUsage (STEP 6), quantity
|
|
// and all: the ZAP transport meters the same units through the same cost
|
|
// switches, so a call bills identically whichever address it arrived at. Errors
|
|
// trace either way.
|
|
func zapRecordAudioUsage(ctx context.Context, authUser *iam.User, provider *object.Provider, userModel string, isPremium bool, qty audioQuantity, status, errMsg string, startTime time.Time) {
|
|
if authUser == nil {
|
|
return
|
|
}
|
|
rec := &usageRecord{
|
|
Owner: authUser.Owner,
|
|
Organization: authUser.Owner,
|
|
Model: userModel,
|
|
Provider: provider.Name,
|
|
Origin: provider.Origin(),
|
|
Currency: "USD",
|
|
Premium: isPremium,
|
|
Status: status,
|
|
ErrorMsg: errMsg,
|
|
AudioSeconds: qty.seconds,
|
|
AudioChars: qty.chars,
|
|
RequestID: uuid.NewString(),
|
|
}
|
|
rec.bind(ctx, authUser)
|
|
recordUsage(rec)
|
|
recordTrace(ctx, rec, startTime)
|
|
}
|
|
|
|
// ── audio.transcribe — OpenAI-compatible STT (mirrors AudioTranscriptions) ──
|
|
|
|
func zapAudioTranscribeHandler(ctx context.Context, auth string, body []byte) (*zap.Message, error) {
|
|
if auth == "" {
|
|
return object.BuildCloudResponse(401, nil, "authentication required")
|
|
}
|
|
// The SAME bound the HTTP handler applies (MaxTranscribeUpload). This
|
|
// transport is handed the body as one slice, so the bound is checked on the
|
|
// slice rather than installed on a reader — the bytes are already here, and
|
|
// the check that matters is the one that keeps them from being PARSED into
|
|
// three more copies and sent upstream.
|
|
if !fits(body) {
|
|
return object.BuildCloudResponse(413, nil, fmt.Sprintf(
|
|
"audio upload exceeds the %d MiB limit", MaxTranscribeUpload>>20))
|
|
}
|
|
form, err := parseTranscribeForm(body)
|
|
if err != nil {
|
|
return object.BuildCloudResponse(400, nil, err.Error())
|
|
}
|
|
|
|
// Identity + provider from the ONE auth seam (STEP 1). Auth is validated
|
|
// before the body-field checks, so an invalid credential is 401 regardless of
|
|
// body validity (never a probe-able 400), matching the HTTP handler.
|
|
provider, authUser, upstreamModel, err := zapResolveAuth(auth, form.model)
|
|
if err != nil {
|
|
return object.BuildCloudResponse(401, nil, err.Error())
|
|
}
|
|
if form.audio == nil {
|
|
return object.BuildCloudResponse(400, nil, "audio request requires a \"file\" part")
|
|
}
|
|
if form.model == "" {
|
|
return object.BuildCloudResponse(400, nil, "audio request requires a \"model\" field")
|
|
}
|
|
|
|
// Prepaid-balance gate — the ONE shared gate (STEP 4).
|
|
if gateErr := enforceBalanceGate(ctx, authUser, "", form.model); gateErr != nil {
|
|
return object.BuildCloudResponse(uint32(statusOf(gateErr)), nil, gateErr.Error())
|
|
}
|
|
if err := zapLane(ctx, provider, form.model); err != nil {
|
|
return object.BuildCloudResponse(uint32(statusOf(err)), nil, err.Error())
|
|
}
|
|
isPremium := false
|
|
if route := resolveModelRoute(form.model); route != nil {
|
|
isPremium = route.premium
|
|
}
|
|
|
|
// No Zen STT verb exists; a Zen-routed model cannot serve this endpoint.
|
|
if provider.Type == "Zen" {
|
|
return object.BuildCloudResponse(400, nil, "model \""+form.model+"\" does not serve the /v1/audio/transcriptions endpoint")
|
|
}
|
|
|
|
// KMS secret + upstream-model/language binding (STEP 4/5), mirroring the HTTP path.
|
|
if err := object.ResolveProviderSecret(provider); err != nil {
|
|
log.Error("ZAP: KMS resolve %s: %v", provider.Name, err)
|
|
}
|
|
if upstreamModel != "" {
|
|
provider.SubType = upstreamModel
|
|
} else {
|
|
provider.SubType = form.model
|
|
}
|
|
if form.language != "" {
|
|
provider.Flavor = form.language
|
|
}
|
|
|
|
sttProvider, err := provider.GetSpeechToTextProvider("en")
|
|
if err != nil {
|
|
return object.BuildCloudResponse(502, nil, err.Error())
|
|
}
|
|
|
|
startTime := time.Now().UTC()
|
|
// Admission around the UPSTREAM CALL — see the speech handler above.
|
|
release, refused := admitSpeech(authUser.Owner)
|
|
if refused != nil {
|
|
return object.BuildCloudResponse(uint32(statusOf(refused)), nil, refused.Error())
|
|
}
|
|
defer release()
|
|
|
|
heard, sttResult, err := sttProvider.ProcessAudio(form.audio, ctx, "en", form.timings)
|
|
if err != nil {
|
|
zapRecordAudioUsage(ctx, authUser, provider, form.model, isPremium, audioQuantity{}, "error", err.Error(), startTime)
|
|
return object.BuildCloudResponse(502, nil, err.Error())
|
|
}
|
|
zapRecordAudioUsage(ctx, authUser, provider, form.model, isPremium, audioQuantity{seconds: sttSecondsOf(sttResult)}, "success", "", startTime)
|
|
|
|
if form.responseFormat == "text" {
|
|
return object.BuildCloudResponse(200, []byte(heard.Text), "")
|
|
}
|
|
return object.BuildCloudResponse(200, transcriptionBody(form.responseFormat, heard, sttSecondsOf(sttResult)), "")
|
|
}
|
|
|
|
// transcribeForm is the decoded OpenAI transcription multipart body.
|
|
type transcribeForm struct {
|
|
model string
|
|
language string
|
|
responseFormat string
|
|
timings []stt.Timing
|
|
audio *bytes.Reader
|
|
}
|
|
|
|
// parseTranscribeForm reconstructs the OpenAI multipart form (file + model
|
|
// [+ language + response_format]) from the raw body, recovering the boundary
|
|
// from the body's own opening line exactly as parseSTTForm does — the handler
|
|
// stays on the shared (ctx, auth, body) signature. A missing "file" part is not
|
|
// an error here: auth is resolved first, so the handler reports it as a 400
|
|
// only to an authenticated caller.
|
|
func parseTranscribeForm(body []byte) (*transcribeForm, error) {
|
|
boundary := multipartBoundary(body)
|
|
if boundary == "" {
|
|
return nil, fmt.Errorf("audio request requires a multipart form body")
|
|
}
|
|
mr := multipart.NewReader(bytes.NewReader(body), boundary)
|
|
form, err := mr.ReadForm(MaxTranscribeUpload)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read multipart form: %s", err.Error())
|
|
}
|
|
f := &transcribeForm{}
|
|
first := func(key string) string {
|
|
if v := form.Value[key]; len(v) > 0 {
|
|
return v[0]
|
|
}
|
|
return ""
|
|
}
|
|
f.model = first("model")
|
|
f.language = first("language")
|
|
f.responseFormat = first("response_format")
|
|
f.timings, err = timingsOf(form.Value)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if files := form.File["file"]; len(files) > 0 {
|
|
fh, err := files[0].Open()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open \"file\" part: %s", err.Error())
|
|
}
|
|
defer fh.Close()
|
|
data, err := io.ReadAll(fh)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read \"file\" part: %s", err.Error())
|
|
}
|
|
f.audio = bytes.NewReader(data)
|
|
}
|
|
return f, nil
|
|
}
|
|
|
|
// ── audio.voice / .music / .foley — Zen-native (mirrors AudioMedia) ─────────
|
|
|
|
func zapAudioVerbHandler(verb string) zapHandler {
|
|
return func(ctx context.Context, auth string, body []byte) (*zap.Message, error) {
|
|
if auth == "" {
|
|
return object.BuildCloudResponse(401, nil, "authentication required")
|
|
}
|
|
switch verb {
|
|
case "voice", "music", "foley":
|
|
default:
|
|
return object.BuildCloudResponse(404, nil, "unknown audio verb")
|
|
}
|
|
var req struct {
|
|
Model string `json:"model"`
|
|
}
|
|
if err := json.Unmarshal(body, &req); err != nil || req.Model == "" {
|
|
return object.BuildCloudResponse(400, nil, "audio request requires a \"model\" field")
|
|
}
|
|
|
|
provider, authUser, _, err := zapResolveAuth(auth, req.Model)
|
|
if err != nil {
|
|
return object.BuildCloudResponse(401, nil, err.Error())
|
|
}
|
|
if provider.Type != "Zen" {
|
|
return object.BuildCloudResponse(400, nil, "model \""+req.Model+"\" does not serve the /v1/audio/"+verb+" endpoint")
|
|
}
|
|
isPremium := false
|
|
if route := resolveModelRoute(req.Model); route != nil {
|
|
isPremium = route.premium
|
|
}
|
|
return zapServeZenMedia("audio/"+verb, req.Model, body, 1, authUser, isPremium, time.Now().UTC())
|
|
}
|
|
}
|
|
|
|
// multipartBoundary reads the boundary token off the first line of a multipart
|
|
// body. Over ZAP there is no http.Request to parse the Content-Type from, so the
|
|
// body states its own boundary and audio.transcribe reads it here.
|
|
func multipartBoundary(body []byte) string {
|
|
before, _, ok := bytes.Cut(body, []byte{'\n'})
|
|
if !ok {
|
|
return ""
|
|
}
|
|
first := strings.TrimRight(string(before), "\r")
|
|
if !strings.HasPrefix(first, "--") {
|
|
return ""
|
|
}
|
|
return strings.TrimPrefix(first, "--")
|
|
}
|