mirror of
https://github.com/hanzoai/voice.git
synced 2026-10-03 10:14:33 +00:00
zeekayandHanzo Dev
03a9f42e10
Audio in, audio out, on an IAM bearer, billed to whoever spoke. speech holds the models; this holds the conversation. The wire is the OpenAI realtime protocol so existing clients work unchanged. hanzoai/go-openai-realtime already carries the event types but is a client, so it can only unmarshal the server's half; Read supplies the other direction. A browser cannot put a header on a WebSocket, so the bearer is spent on a POST and the URL carries a single-use thirty-second ticket naming a principal already verified here. It has no claims and cannot be verified elsewhere, so a copy lifted from a log buys nothing. Replies are synthesised a sentence at a time: 452 ms to first sound against 2267 ms for a whole reply, and every sentence after the first is free because synthesis outruns speech. Measured end to end at 1.15-1.55 s from the speaker stopping, of which 500 ms is the silence detector that Turn exists to replace. Turn, Mind and Meter are the seams. Hush ends a turn on silence alone, which cuts people off mid-thought and waits through gaps it should have taken; it is there so the surface works while a detector that reads intent is built. Co-authored-by: Hanzo Dev <[email protected]>
77 lines
2.8 KiB
Go
77 lines
2.8 KiB
Go
// Package voice serves a spoken conversation over one socket.
|
|
//
|
|
// The wire is the OpenAI realtime protocol, so any client already written
|
|
// against that protocol drives this surface unchanged. The event structs come
|
|
// from github.com/hanzoai/go-openai-realtime, which is a client: it marshals
|
|
// what a client sends and unmarshals what a server sends. Serving the other
|
|
// side needs the mirror of that, and only the receiving half is missing —
|
|
// hence Read below and nothing else.
|
|
package voice
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
|
|
rt "github.com/hanzoai/go-openai-realtime"
|
|
)
|
|
|
|
// Frame is a video keyframe offered alongside speech.
|
|
//
|
|
// The realtime protocol carries audio and no picture. A model that can see
|
|
// needs one, so the buffer that holds pictures is named for the buffer that
|
|
// holds sound and behaves the same way: append, and the next reply reads it.
|
|
const Frame rt.ClientEventType = "input_video_buffer.append"
|
|
|
|
// FrameEvent appends one still to the video buffer. Bytes are base64 JPEG,
|
|
// exactly as ClientEventTypeInputAudioBufferAppend carries base64 PCM.
|
|
type FrameEvent struct {
|
|
rt.EventBase
|
|
Image string `json:"image"`
|
|
}
|
|
|
|
func (FrameEvent) ClientEventType() rt.ClientEventType { return Frame }
|
|
|
|
// Read turns one JSON message from a client into the event it names.
|
|
//
|
|
// The library ships UnmarshalServerEvent because a client receives server
|
|
// events; a server receives client events, and that direction has no reader.
|
|
// The type field is the discriminator in both directions, so this is the same
|
|
// dispatch pointed the other way.
|
|
func Read(data []byte) (rt.ClientEvent, error) {
|
|
var head struct {
|
|
Type rt.ClientEventType `json:"type"`
|
|
}
|
|
if err := json.Unmarshal(data, &head); err != nil {
|
|
return nil, fmt.Errorf("event is not json: %w", err)
|
|
}
|
|
var out rt.ClientEvent
|
|
switch head.Type {
|
|
case rt.ClientEventTypeSessionUpdate:
|
|
out = &rt.SessionUpdateEvent{}
|
|
case rt.ClientEventTypeInputAudioBufferAppend:
|
|
out = &rt.InputAudioBufferAppendEvent{}
|
|
case rt.ClientEventTypeInputAudioBufferCommit:
|
|
out = &rt.InputAudioBufferCommitEvent{}
|
|
case rt.ClientEventTypeInputAudioBufferClear:
|
|
out = &rt.InputAudioBufferClearEvent{}
|
|
case rt.ClientEventTypeConversationItemCreate:
|
|
out = &rt.ConversationItemCreateEvent{}
|
|
case rt.ClientEventTypeConversationItemTruncate:
|
|
out = &rt.ConversationItemTruncateEvent{}
|
|
case rt.ClientEventTypeConversationItemDelete:
|
|
out = &rt.ConversationItemDeleteEvent{}
|
|
case rt.ClientEventTypeResponseCreate:
|
|
out = &rt.ResponseCreateEvent{}
|
|
case rt.ClientEventTypeResponseCancel:
|
|
out = &rt.ResponseCancelEvent{}
|
|
case Frame:
|
|
out = &FrameEvent{}
|
|
default:
|
|
return nil, fmt.Errorf("unknown event type %q", head.Type)
|
|
}
|
|
if err := json.Unmarshal(data, out); err != nil {
|
|
return nil, fmt.Errorf("event %q is malformed: %w", head.Type, err)
|
|
}
|
|
return out, nil
|
|
}
|