hanzoai/go-openai-realtime
2
0
Fork 0
mirror of https://github.com/hanzoai/go-openai-realtime.git synced 2026-09-30 21:54:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add voice-voice examples and update docs

Browse Source
This commit is contained in:
WqyJh
2024-10-16 23:26:22 +08:00
parent c81f2182c4
commit 6d44980db1
20 changed files with 430 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -26,3 +26,4 @@ go.work.sum
.vscode
.DS_Store
experiments
examples/README.md
+8
View File
@@ -0,0 +1,8 @@
# Examples
## voice
- [text-only](./text-only/README.md) A text-to-text chat example
- [text-voice](./voice/text-voice/README.md) A text-to-voice chat example
- [voice-voice](./voice/voice-voice/README.md) A voice-to-voice chat example
examples/go.mod
+7 -4
View File
@@ -1,21 +1,24 @@
module github.com/WqyJh/go-openai-realtime/examples
go 1.19
go 1.22.0
toolchain go1.22.5
require (
github.com/WqyJh/go-openai-realtime v0.0.0
github.com/faiface/beep v1.1.0
github.com/gordonklaus/portaudio v0.0.0-20230709114228-aafa478834f5
github.com/sashabaranov/go-openai v1.32.0
)
require (
github.com/coder/websocket v1.8.12 // indirect
github.com/hajimehoshi/oto v0.7.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/sashabaranov/go-openai v1.32.0 // indirect
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8 // indirect
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 // indirect
golang.org/x/image v0.20.0 // indirect
golang.org/x/mobile v0.0.0-20190415191353-3e0bab5405d6 // indirect
golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756 // indirect
golang.org/x/sys v0.25.0 // indirect
)
replace github.com/WqyJh/go-openai-realtime => ../
examples/go.sum
+10 -2
View File
@@ -3,6 +3,7 @@ github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NA
github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
github.com/d4l3k/messagediff v1.2.2-0.20190829033028-7e0a312ae40b/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/faiface/beep v1.1.0 h1:A2gWP6xf5Rh7RG/p9/VAW2jRSDEGQm5sbOb38sf5d4c=
github.com/faiface/beep v1.1.0/go.mod h1:6I8p6kK2q4opL/eWb+kAkk38ehnTunWeToJB+s51sT4=
github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
@@ -10,6 +11,8 @@ github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebK
github.com/go-audio/audio v1.0.0/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs=
github.com/go-audio/riff v1.0.0/go.mod h1:l3cQwc85y79NQFCRB7TiPoNiaijp6q8Z0Uv38rVG498=
github.com/go-audio/wav v1.0.0/go.mod h1:3yoReyQOsiARkvPl3ERCi8JFjihzG6WhjYpZCf5zAWE=
github.com/gordonklaus/portaudio v0.0.0-20230709114228-aafa478834f5 h1:5AlozfqaVjGYGhms2OsdUyfdJME76E6rx5MdGpjzZpc=
github.com/gordonklaus/portaudio v0.0.0-20230709114228-aafa478834f5/go.mod h1:WY8R6YKlI2ZI3UyzFk7P6yGSuS+hFwNtEzrexRyD7Es=
github.com/hajimehoshi/go-mp3 v0.3.0/go.mod h1:qMJj/CSDxx6CGHiZeCgbiq2DSUkbK0UbtXShQcnfyMM=
github.com/hajimehoshi/oto v0.6.1/go.mod h1:0QXGEkbuJRohbJaxr7ZQSxnju7hEhseiPx2hrh6raOI=
github.com/hajimehoshi/oto v0.7.1 h1:I7maFPz5MBCwiutOrz++DLdbr4rTzBsbBuV2VpgU9kk=
@@ -26,20 +29,25 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sashabaranov/go-openai v1.32.0 h1:Yk3iE9moX3RBXxrof3OBtUBrE7qZR0zF9ebsoO4zVzI=
github.com/sashabaranov/go-openai v1.32.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8 h1:idBdZTd9UioThJp8KpM/rTSinK/ChZFBE43/WtIy8zg=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/image v0.0.0-20190220214146-31aff87c08e9/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 h1:KYGJGHOQy8oSi1fDlSpcZF0+juKwk/hEMv5SiwHogR0=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.20.0 h1:7cVCUjQwfL18gyBJOmYvptfSHS8Fb3YUDtfLIZ7Nbpw=
golang.org/x/image v0.20.0/go.mod h1:0a88To4CYVBAHp5FXJm8o7QbUl37Vd85ply1vyD8auM=
golang.org/x/mobile v0.0.0-20190415191353-3e0bab5405d6 h1:vyLBGJPIl9ZYbcQFM2USFmJBK6KI+t+z6jL0lbwjrnc=
golang.org/x/mobile v0.0.0-20190415191353-3e0bab5405d6/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756 h1:9nuHUbU8dRnRRfj9KjWUVrJeoexdbeMjttk6Oh1rD10=
golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
examples/text-only/README.md
+10
View File
@@ -1,5 +1,7 @@
# go-openai-realtime example for text-only
This is a simple example that demonstrates how to use the go-openai-realtime library to create a text-to-text chat program.
## Run
```bash
@@ -7,3 +9,11 @@ export OPENAI_API_KEY=<your openai api key>
export SOCKS_PROXY=<your socks proxy> # this optional
go run .
```
Input any text and press enter to send it to the server, then you can see the text response from the server, shown as teletypewriter effect. After that, you can see a line of text prefixed by `[full]`, which is the full response text from the server.
If you want to exit the program, input `exit` and press enter.
Screenshot:
![image](./example.png)
examples/text-only/example.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 107 KiB

examples/text-only/main.go
+1 -1
View File
@@ -88,7 +88,7 @@ func main() {
conn.SendMessage(ctx, &openairt.ResponseCreateEvent{
Response: openairt.ResponseCreateParams{
Modalities: []openairt.Modality{openairt.ModalityText},
MaxOutputTokens: openairt.Int(100),
MaxOutputTokens: 4000,
},
})
}
examples/voice/README.md
-24
View File
@@ -1,24 +0,0 @@
# go-openai-realtime example for voice
## Prerequisite
This example needs portaudio library to play audio, please install it before running this library.
On macOS:
```bash
brew install pkg-config
brew install portaudio
```
On Linux:
```bash
apt-get install portaudio19-dev
```
## Run
```bash
export OPENAI_API_KEY=<your openai api key>
export SOCKS_PROXY=<your socks proxy> # this optional
go run .
```
examples/voice/audio-player/README.md
+9
View File
@@ -0,0 +1,9 @@
# Audio Player (Experimental Code)
Play a raw PCM audio file, 24kHz sample rate, 16-bit PCM, mono.
Change the audio file path in the code to play different audio files.
```bash
go run .
```
examples/voice/audio-player/main.go
+51
View File
@@ -0,0 +1,51 @@
package main
import (
"fmt"
"io"
"os"
"time"
"github.com/WqyJh/go-openai-realtime/examples/voice/pcm"
"github.com/faiface/beep"
"github.com/faiface/beep/speaker"
)
func main() {
// Open your PCM16 audio file
file, err := os.Open("../record3/output.pcm")
if err != nil {
fmt.Println("Error opening file:", err)
return
}
defer file.Close()
// Read the entire file into memory
data, err := io.ReadAll(file)
if err != nil {
fmt.Println("Error reading file:", err)
return
}
// Define the format of your PCM data
sampleRate := beep.SampleRate(24000) // 24kHz sample rate
format := beep.Format{
SampleRate: sampleRate,
NumChannels: 1, // Mono audio
Precision: 2, // 16-bit PCM
}
// Initialize the speaker with the format
speaker.Init(sampleRate, sampleRate.N(time.Second/10))
// Create a new PCMStream
stream := pcm.NewPCMStream(data, format)
// Play the stream
done := make(chan bool)
speaker.Play(beep.Seq(stream, beep.Callback(func() {
done <- true
})))
<-done
}
examples/voice/data/20241016-1.pcm
BIN
View File
Binary file not shown.
examples/voice/data/20241016-2.pcm
BIN
View File
Binary file not shown.
examples/voice/data/README.md
+8
View File
@@ -0,0 +1,8 @@
# PCM samples
This directory contains some pcm sample files, generated by OpenAI Realtime API's `response.audio.delta` event, accumulated as a full sentence.
| File Name | Content |
| --------- | ------- |
| 20241016-1.pcm | Hi there! How can I help you today? |
| 20241016-2.pcm | Why sky is blue? |
examples/voice/recorder/recorder.go
+35
View File
@@ -0,0 +1,35 @@
package recorder
import (
"bytes"
"encoding/binary"
"io"
"github.com/gordonklaus/portaudio"
)
func Record(
inputChannels int,
sampleRate float64,
framesPerBuffer int,
) (data io.Reader, stop func(), err error) {
var buf bytes.Buffer
stream, err := portaudio.OpenDefaultStream(inputChannels, 0, sampleRate, framesPerBuffer, func(in []int16) {
for _, sample := range in {
binary.Write(&buf, binary.LittleEndian, sample)
}
})
if err != nil {
return nil, nil, err
}
err = stream.Start()
if err != nil {
return nil, nil, err
}
return &buf, func() {
stream.Stop()
stream.Close()
}, nil
}
examples/voice/text-voice/README.md
+34
View File
@@ -0,0 +1,34 @@
# go-openai-realtime example for voice
This is a simple example that demonstrates how to use the go-openai-realtime library to create a text-to-voice chat program.
## Prerequisite
This example needs portaudio library to play audio, please install it before running this library.
On macOS:
```bash
brew install pkg-config
brew install portaudio
```
On Linux:
```bash
apt-get install portaudio19-dev
```
## Run
```bash
export OPENAI_API_KEY=<your openai api key>
export SOCKS_PROXY=<your socks proxy> # this optional
go run .
```
Input any text and press enter to send it to the server, then you can hear the voice response from the server. Meanwhile, you can see the text response from the server, shown as teletypewriter effect. After that, you can see a line of text prefixed by `[full]`, which is the full response text from the server.
If you want to exit the program, input `exit` and press enter.
Screenshot:
![image](./example.png)
examples/voice/text-voice/example.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 111 KiB

examples/voice/main.go → examples/voice/text-voice/main.go
+11 -4
View File
@@ -71,6 +71,11 @@ func main() {
streamers := make(chan *pcm.PCMStream, 100)
datas := make(chan []byte, 100)
beep.NewBuffer(beep.Format{
SampleRate: beep.SampleRate(24000),
NumChannels: 1,
Precision: 2, // 16-bit PCM
})
// Audio response
audioResponseHandler := func(ctx context.Context, event openairt.ServerEvent) {
@@ -91,11 +96,12 @@ func main() {
for data := range datas {
fulldata = append(fulldata, data...)
}
// os.WriteFile(fmt.Sprintf("%s.pcm", event.(openairt.ResponseAudioDoneEvent).EventID), fulldata, 0o644)
streamer := pcm.NewPCMStream(
fulldata,
beep.Format{
SampleRate: beep.SampleRate(24000),
NumChannels: 2,
NumChannels: 1,
Precision: 2, // 16-bit PCM
},
)
@@ -111,9 +117,9 @@ func main() {
return
case streamer := <-streamers:
done := make(chan bool, 1)
speaker.Play(beep.Seq(streamer), beep.Callback(func() {
speaker.Play(beep.Seq(streamer, beep.Callback(func() {
done <- true
}))
})))
<-done // wait for done
}
}
@@ -142,6 +148,7 @@ func main() {
if s.Text() == "exit" || s.Text() == "quit" {
break
}
fmt.Println("---------------------")
conn.SendMessage(ctx, &openairt.ConversationItemCreateEvent{
Item: openairt.MessageItem{
ID: openairt.GenerateId("msg_", 10),
@@ -159,7 +166,7 @@ func main() {
conn.SendMessage(ctx, &openairt.ResponseCreateEvent{
Response: openairt.ResponseCreateParams{
Modalities: []openairt.Modality{openairt.ModalityText, openairt.ModalityAudio},
MaxOutputTokens: openairt.Int(4000),
MaxOutputTokens: 4000,
},
})
}
examples/voice/voice-voice/README.md
+34
View File
@@ -0,0 +1,34 @@
# go-openai-realtime example for voice
This is a simple example that demonstrates how to use the go-openai-realtime library to create a voice-to-voice chat program.
## Prerequisite
This example needs portaudio library to play audio, please install it before running this library.
On macOS:
```bash
brew install pkg-config
brew install portaudio
```
On Linux:
```bash
apt-get install portaudio19-dev
```
## Run
```bash
export OPENAI_API_KEY=<your openai api key>
export SOCKS_PROXY=<your socks proxy> # this optional
go run .
```
Input `start` and press enter to start recording, you can say whatever you want. After you finish speaking, input `stop` and press enter to stop recording. Then a line of text prefixed by `[question]` will be printed, which is the transcription of your voice. Then, a line of text prefixed by `[response]` will be printed, which is the response transcription from the server. Meanwhile, you can hear the audio response from the server.
If you want to exit the program, input `exit` and press enter.
Screenshot:
![image](./example.png)
examples/voice/voice-voice/example.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 98 KiB

examples/voice/voice-voice/main.go
+211
View File
@@ -0,0 +1,211 @@
package main
import (
"bufio"
"context"
"encoding/base64"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"time"
openairt "github.com/WqyJh/go-openai-realtime"
"github.com/WqyJh/go-openai-realtime/examples/voice/pcm"
"github.com/WqyJh/go-openai-realtime/examples/voice/recorder"
"github.com/faiface/beep"
"github.com/faiface/beep/speaker"
"github.com/gordonklaus/portaudio"
"github.com/sashabaranov/go-openai"
)
func main() {
socksProxy := os.Getenv("SOCKS_PROXY")
if socksProxy != "" {
proxyURL, err := url.Parse(socksProxy)
if err != nil {
log.Fatal(err)
}
http.DefaultTransport = &http.Transport{
Proxy: http.ProxyURL(proxyURL),
}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := openairt.NewClient(os.Getenv("OPENAI_API_KEY"))
conn, err := client.Connect(ctx, openairt.WithReadLimit(32768*2))
if err != nil {
log.Fatal(err)
}
defer conn.Close()
// Teletype response
responseDeltaHandler := func(ctx context.Context, event openairt.ServerEvent) {
switch event.ServerEventType() {
case openairt.ServerEventTypeResponseAudioTranscriptDelta:
// ignore
case openairt.ServerEventTypeResponseAudioTranscriptDone:
fmt.Printf("[response] %s\n", event.(openairt.ResponseAudioTranscriptDoneEvent).Transcript)
}
}
// Full response
responseHandler := func(ctx context.Context, event openairt.ServerEvent) {
switch event.ServerEventType() {
case openairt.ServerEventTypeConversationItemInputAudioTranscriptionCompleted:
question := event.(openairt.ConversationItemInputAudioTranscriptionCompletedEvent).Transcript
fmt.Printf("\n[question] %s\n", question)
case openairt.ServerEventTypeResponseDone:
fmt.Print("\n> ")
}
}
// Log handler
logHandler := func(ctx context.Context, event openairt.ServerEvent) {
// fmt.Printf("[%s]\n", event.ServerEventType())
switch event.ServerEventType() {
case openairt.ServerEventTypeError,
openairt.ServerEventTypeSessionUpdated:
// data, err := json.Marshal(event)
// if err != nil {
// log.Fatal(err)
// }
// fmt.Printf("[%s] %s\n", event.ServerEventType(), string(data))
}
}
// Init speaker
speaker.Init(24000, 24000/10)
defer speaker.Close()
streamers := make(chan *pcm.PCMStream, 100)
datas := make(chan []byte, 100)
beep.NewBuffer(beep.Format{
SampleRate: beep.SampleRate(24000),
NumChannels: 1,
Precision: 2, // 16-bit PCM
})
// Audio response
audioResponseHandler := func(ctx context.Context, event openairt.ServerEvent) {
switch event.ServerEventType() {
case openairt.ServerEventTypeResponseAudioDelta:
msg := event.(openairt.ResponseAudioDeltaEvent)
// log.Printf("audioResponseHandler: %v", delta)
data, err := base64.StdEncoding.DecodeString(msg.Delta)
if err != nil {
log.Fatal(err)
}
// os.WriteFile(fmt.Sprintf("%s.pcm", msg.EventID), data, 0o644)
// streamer.Append(data)
datas <- data
case openairt.ServerEventTypeResponseAudioDone:
fulldata := []byte{}
close(datas)
for data := range datas {
fulldata = append(fulldata, data...)
}
os.WriteFile(fmt.Sprintf("%s.pcm", event.(openairt.ResponseAudioDoneEvent).EventID), fulldata, 0o644)
streamer := pcm.NewPCMStream(
fulldata,
beep.Format{
SampleRate: beep.SampleRate(24000),
NumChannels: 1,
Precision: 2, // 16-bit PCM
},
)
datas = make(chan []byte, 100)
streamers <- streamer
}
}
go func() {
for {
select {
case <-ctx.Done():
return
case streamer := <-streamers:
done := make(chan bool, 1)
speaker.Play(beep.Seq(streamer, beep.Callback(func() {
done <- true
})))
<-done // wait for done
}
}
}()
connHandler := openairt.NewConnHandler(ctx, conn, logHandler, responseHandler, responseDeltaHandler, audioResponseHandler)
connHandler.Start()
defer connHandler.Stop()
err = conn.SendMessage(ctx, openairt.SessionUpdateEvent{
Session: openairt.ClientSession{
Modalities: []openairt.Modality{openairt.ModalityText, openairt.ModalityAudio},
Voice: openairt.VoiceShimmer,
OutputAudioFormat: openairt.AudioFormatPcm16,
InputAudioTranscription: &openairt.InputAudioTranscription{
Model: openai.Whisper1,
},
TurnDetection: nil,
},
})
if err != nil {
log.Fatal(err)
}
time.Sleep(1 * time.Second)
fmt.Println("Conversation (input 'start' to start recording)")
fmt.Println("---------------------")
fmt.Print("> ")
s := bufio.NewScanner(os.Stdin)
// Audio init
portaudio.Initialize()
defer portaudio.Terminate()
var data io.Reader
var stop func()
Loop:
for s.Scan() {
switch s.Text() {
case "exit", "quit":
break Loop
case "start":
data, stop, err = recorder.Record(1, 24000, 64)
if err != nil {
log.Fatal(err)
}
fmt.Println("Recording...(input 'stop' to stop recording)")
fmt.Print("> ")
case "stop":
if stop == nil {
log.Print("not recording")
continue
}
stop()
audioData, err := io.ReadAll(data)
if err != nil {
log.Fatal(err)
}
base64Audio := base64.StdEncoding.EncodeToString(audioData)
conn.SendMessage(ctx, openairt.InputAudioBufferAppendEvent{
Audio: base64Audio,
})
conn.SendMessage(ctx, openairt.InputAudioBufferCommitEvent{})
conn.SendMessage(ctx, openairt.ResponseCreateEvent{
Response: openairt.ResponseCreateParams{
Modalities: []openairt.Modality{openairt.ModalityText, openairt.ModalityAudio},
MaxOutputTokens: 4000,
},
})
}
}
}