mirror of
https://github.com/hanzoai/voice.git
synced 2026-10-03 10:14:33 +00:00
Audio in, audio out, on an IAM bearer, billed to whoever spoke. speech holds the models; this holds the conversation. The wire is the OpenAI realtime protocol so existing clients work unchanged. hanzoai/go-openai-realtime already carries the event types but is a client, so it can only unmarshal the server's half; Read supplies the other direction. A browser cannot put a header on a WebSocket, so the bearer is spent on a POST and the URL carries a single-use thirty-second ticket naming a principal already verified here. It has no claims and cannot be verified elsewhere, so a copy lifted from a log buys nothing. Replies are synthesised a sentence at a time: 452 ms to first sound against 2267 ms for a whole reply, and every sentence after the first is free because synthesis outruns speech. Measured end to end at 1.15-1.55 s from the speaker stopping, of which 500 ms is the silence detector that Turn exists to replace. Turn, Mind and Meter are the seams. Hush ends a turn on silence alone, which cuts people off mid-thought and waits through gaps it should have taken; it is there so the surface works while a detector that reads intent is built. Co-authored-by: Hanzo Dev <[email protected]>
225 lines
6.4 KiB
Go
225 lines
6.4 KiB
Go
package voice
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/coder/websocket"
|
|
)
|
|
|
|
// A browser cannot put a header on a WebSocket. The API takes no options: the
|
|
// only things it sends are the URL, cookies for that origin, and a
|
|
// subprotocol list. So a bearer either goes in the query string, where it
|
|
// lands in every access log and every referrer, or it does not go at all.
|
|
//
|
|
// It does not go. Instead the caller spends its bearer on an ordinary POST,
|
|
// where a header works perfectly, and receives a ticket: a random name for a
|
|
// principal this process has already verified and is holding in memory. The
|
|
// ticket proves nothing by itself, carries no claims, cannot be verified by
|
|
// anyone else, and stops existing the moment it is used or the moment it goes
|
|
// stale. Putting THAT in a URL costs nothing.
|
|
//
|
|
// This is the pattern the estate already uses to authorise a browser onto a
|
|
// sandbox terminal socket, and copying it keeps one answer to one question.
|
|
const (
|
|
ticketLife = 30 * time.Second
|
|
ticketSize = 24
|
|
)
|
|
|
|
type held struct {
|
|
who Who
|
|
at time.Time
|
|
}
|
|
|
|
// Desk hands out tickets and takes them back. One use, then gone.
|
|
type Desk struct {
|
|
mu sync.Mutex
|
|
open map[string]held
|
|
}
|
|
|
|
func NewDesk() *Desk { return &Desk{open: map[string]held{}} }
|
|
|
|
func (d *Desk) Issue(who Who) string {
|
|
b := make([]byte, ticketSize)
|
|
if _, err := rand.Read(b); err != nil {
|
|
panic("voice: no randomness for a ticket: " + err.Error())
|
|
}
|
|
name := base64.RawURLEncoding.EncodeToString(b)
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
// Sweep on the way in: stale tickets are only worth collecting when new
|
|
// ones arrive, so there is no timer to run and nothing to supervise.
|
|
for k, v := range d.open {
|
|
if time.Since(v.at) > ticketLife {
|
|
delete(d.open, k)
|
|
}
|
|
}
|
|
d.open[name] = held{who: who, at: time.Now()}
|
|
return name
|
|
}
|
|
|
|
// Redeem spends a ticket. A second attempt with the same ticket fails, which
|
|
// is what makes a copy taken from a log or a browser history worthless.
|
|
func (d *Desk) Redeem(name string) (Who, bool) {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
h, ok := d.open[name]
|
|
if !ok {
|
|
return Who{}, false
|
|
}
|
|
delete(d.open, name)
|
|
if time.Since(h.at) > ticketLife {
|
|
return Who{}, false
|
|
}
|
|
return h.who, true
|
|
}
|
|
|
|
// Voice is the daemon.
|
|
type Voice struct {
|
|
Gate *Gate
|
|
Desk *Desk
|
|
Room *Floorspace
|
|
Speech *Speech
|
|
Log *slog.Logger
|
|
Meter Meter
|
|
|
|
// Think builds the mind and the turn detector for a conversation. It is a
|
|
// field rather than a type so the model behind a session can change
|
|
// without this package knowing what a model is.
|
|
Think func(who Who) (Mind, Turn)
|
|
|
|
Model string
|
|
Voice string
|
|
|
|
// Origins that may open a socket. A WebSocket is exempt from the
|
|
// same-origin policy, so this list is the only thing standing between a
|
|
// logged-in user and any page on the internet opening a microphone
|
|
// session as them. Empty means same-origin only.
|
|
Origins []string
|
|
}
|
|
|
|
func (v *Voice) Routes(mux *http.ServeMux) {
|
|
mux.HandleFunc("POST /v1/voice/session", v.session)
|
|
mux.HandleFunc("GET /v1/voice", v.talk)
|
|
mux.HandleFunc("GET /v1/voice/health", v.health)
|
|
}
|
|
|
|
// session spends a bearer and returns a ticket. This is the only place a
|
|
// credential is read, and it is read from a header.
|
|
func (v *Voice) session(w http.ResponseWriter, r *http.Request) {
|
|
who, err := v.Gate.Read(r.Header.Get("Authorization"), r.Header.Get("X-Org-Id"))
|
|
if err != nil {
|
|
refuse(w, http.StatusUnauthorized, "authentication required")
|
|
return
|
|
}
|
|
// Ask the doorway now rather than at the upgrade, so a caller learns it is
|
|
// full before it opens a microphone.
|
|
leave, err := v.Room.Enter(who.Org)
|
|
if err != nil {
|
|
refuse(w, http.StatusServiceUnavailable, err.Error())
|
|
return
|
|
}
|
|
leave() // the real slot is taken at the upgrade; this was only a question
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"ticket": v.Desk.Issue(who),
|
|
"expires_in": int(ticketLife.Seconds()),
|
|
"rate": Spoken,
|
|
"format": "pcm16",
|
|
"channels": 1,
|
|
})
|
|
}
|
|
|
|
func (v *Voice) talk(w http.ResponseWriter, r *http.Request) {
|
|
who, ok := v.Desk.Redeem(r.URL.Query().Get("ticket"))
|
|
if !ok {
|
|
refuse(w, http.StatusUnauthorized, "a valid ticket is required")
|
|
return
|
|
}
|
|
leave, err := v.Room.Enter(who.Org)
|
|
if err != nil {
|
|
refuse(w, http.StatusServiceUnavailable, err.Error())
|
|
return
|
|
}
|
|
defer leave()
|
|
|
|
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{
|
|
OriginPatterns: v.Origins,
|
|
Subprotocols: []string{"realtime"},
|
|
})
|
|
if err != nil {
|
|
v.Log.Warn("upgrade refused", "why", err)
|
|
return
|
|
}
|
|
defer conn.CloseNow()
|
|
|
|
mind, turn := v.Think(who)
|
|
s := &Session{
|
|
ID: "voice_" + strings.TrimRight(base64.RawURLEncoding.EncodeToString(randomish()), "="),
|
|
Who: who,
|
|
Sock: sock{conn},
|
|
Speech: v.Speech,
|
|
Mind: mind,
|
|
Turn: turn,
|
|
Meter: v.Meter,
|
|
Log: v.Log,
|
|
Model: v.Model,
|
|
Voice: v.Voice,
|
|
}
|
|
v.Log.Info("conversation opened", "id", s.ID, "org", who.Org, "user", who.User)
|
|
if err := s.Run(r.Context()); err != nil {
|
|
v.Log.Warn("conversation ended badly", "id", s.ID, "why", err)
|
|
}
|
|
v.Log.Info("conversation closed", "id", s.ID, "org", who.Org)
|
|
}
|
|
|
|
func (v *Voice) health(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"ok": true, "busy": v.Room.Busy(), "capacity": v.Room.Total,
|
|
})
|
|
}
|
|
|
|
func refuse(w http.ResponseWriter, code int, why string) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(code)
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"error": map[string]string{"message": why},
|
|
})
|
|
}
|
|
|
|
func randomish() []byte {
|
|
b := make([]byte, 12)
|
|
_, _ = rand.Read(b)
|
|
return b
|
|
}
|
|
|
|
// sock adapts the websocket to what a Session needs. Audio is binary on the
|
|
// wire when the client sends it that way and JSON otherwise; both arrive here
|
|
// as bytes and Read sorts them out.
|
|
type sock struct{ c *websocket.Conn }
|
|
|
|
// Big enough for a sentence of 24 kHz audio base64'd, small enough that a
|
|
// client cannot spend this process's memory by claiming a huge frame.
|
|
const most = 8 << 20
|
|
|
|
func (s sock) Read(ctx context.Context) ([]byte, error) {
|
|
s.c.SetReadLimit(most)
|
|
_, data, err := s.c.Read(ctx)
|
|
return data, err
|
|
}
|
|
|
|
func (s sock) Write(ctx context.Context, msg []byte) error {
|
|
return s.c.Write(ctx, websocket.MessageText, msg)
|
|
}
|
|
|
|
func (s sock) Close() error { return s.c.Close(websocket.StatusNormalClosure, "") }
|