mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
332 lines
11 KiB
Go
332 lines
11 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
// Portions Copyright 2025 The OpenAgent Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
|
|
"github.com/hanzoai/ai/util"
|
|
"github.com/hanzoai/go-openai"
|
|
)
|
|
|
|
// OpenAIWriter turns the upstream's event stream into OpenAI's, and writes the
|
|
// result to `out`.
|
|
//
|
|
// It is an io.Writer with an io.Writer inside it, and the inner one is a FIELD
|
|
// because the destination is a value the caller chooses, not a place to be
|
|
// reached for. /v1/responses composes its own translating writer here and reads
|
|
// the same chat stream in a second dialect; giving this type an http.ResponseWriter
|
|
// meant that caller had to reach into the request context and swap what it found.
|
|
type OpenAIWriter struct {
|
|
out io.Writer
|
|
Cleaner Cleaner
|
|
Buffer []byte
|
|
MessageBuf []byte
|
|
RequestID string
|
|
Stream bool
|
|
StreamSent bool
|
|
Model string
|
|
// IncludeUsage mirrors the request's stream_options.include_usage. Per the
|
|
// OpenAI spec the trailing empty-choices usage chunk is emitted ONLY when the
|
|
// client asks for it; sending it unconditionally breaks clients that read
|
|
// choices[0] on every chunk.
|
|
IncludeUsage bool
|
|
}
|
|
|
|
// Flush pushes what has been written as far as the destination allows, and is
|
|
// what makes the stream a stream: an event that sits in a buffer until the reply
|
|
// ends has been typed, not streamed. Best-effort, like the http.Flusher assertion
|
|
// it replaces — a destination that cannot flush (io.Discard, on the non-streaming
|
|
// path) has nothing to push.
|
|
//
|
|
// Two shapes, because both are real here: a *bufio.Writer from the stream callback
|
|
// returns an error, and a wrapper that forwards to one may not.
|
|
func (w *OpenAIWriter) Flush() {
|
|
switch f := w.out.(type) {
|
|
case interface{ Flush() error }:
|
|
_ = f.Flush()
|
|
case interface{ Flush() }:
|
|
f.Flush()
|
|
}
|
|
}
|
|
|
|
// Write processes incoming data chunks and formats them for OpenAI compatibility
|
|
func (w *OpenAIWriter) Write(p []byte) (n int, err error) {
|
|
// Parse the incoming SSE message format
|
|
var content string
|
|
var reasoning bool
|
|
|
|
if bytes.HasPrefix(p, []byte("event: message\ndata: ")) {
|
|
prefix := []byte("event: message\ndata: ")
|
|
suffix := []byte("\n\n")
|
|
content = string(bytes.TrimSuffix(bytes.TrimPrefix(p, prefix), suffix))
|
|
|
|
// Add content to message buffer
|
|
w.MessageBuf = append(w.MessageBuf, []byte(content)...)
|
|
} else if bytes.HasPrefix(p, []byte("event: reason\ndata: ")) {
|
|
// Reasoning is a FIELD, and it stays one. It is deliberately kept out of
|
|
// MessageBuf — the answer is the answer — and streamed below as
|
|
// reasoning_content, which is the same shape the upstream sent and the
|
|
// name every client that renders thinking already reads.
|
|
//
|
|
// It used to fall through to the content delta beneath this switch, which
|
|
// meant the comment here ("we don't expose reason data") was true of the
|
|
// buffered answer and false of the stream: a caller reading the same model
|
|
// non-streaming got "ok", and streaming got the whole chain-of-thought
|
|
// followed by "ok". Measured against GLM-5.1, whose upstream separates the
|
|
// two perfectly.
|
|
prefix := []byte("event: reason\ndata: ")
|
|
suffix := []byte("\n\n")
|
|
content = string(bytes.TrimSuffix(bytes.TrimPrefix(p, prefix), suffix))
|
|
reasoning = true
|
|
} else {
|
|
// If we can't parse, just store the raw bytes and attempt to clean
|
|
content = w.Cleaner.CleanString(string(p))
|
|
if content != "" {
|
|
w.MessageBuf = append(w.MessageBuf, []byte(content)...)
|
|
}
|
|
}
|
|
|
|
// Always store the original bytes
|
|
w.Buffer = append(w.Buffer, p...)
|
|
|
|
// For non-streaming, just collect the data
|
|
if !w.Stream {
|
|
return len(p), nil
|
|
}
|
|
|
|
// Skip empty content
|
|
if content == "" {
|
|
return len(p), nil
|
|
}
|
|
|
|
// OpenAI streams the assistant role on the FIRST delta chunk; clients
|
|
// (LangChain/LibreChat agents) read it to classify the streamed message.
|
|
// Omitting it breaks their parser with "Cannot read properties of undefined
|
|
// (reading 'role')" and the reply never renders.
|
|
delta := openai.ChatCompletionStreamChoiceDelta{Content: content}
|
|
if reasoning {
|
|
delta = openai.ChatCompletionStreamChoiceDelta{ReasoningContent: content}
|
|
}
|
|
if !w.StreamSent {
|
|
delta.Role = "assistant"
|
|
}
|
|
|
|
// Create SSE chunk using go-openai library structure
|
|
chunk := openai.ChatCompletionStreamResponse{
|
|
ID: "chatcmpl-" + w.RequestID,
|
|
Object: "chat.completion.chunk",
|
|
Created: util.GetCurrentUnixTime(),
|
|
Model: w.Model,
|
|
Choices: []openai.ChatCompletionStreamChoice{
|
|
{
|
|
Index: 0,
|
|
Delta: delta,
|
|
FinishReason: openai.FinishReasonNull,
|
|
},
|
|
},
|
|
}
|
|
|
|
jsonData, err := json.Marshal(chunk)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
// Out, not this writer: Write is the formatter, so writing to itself would recur.
|
|
_, err = w.out.Write([]byte(fmt.Sprintf("data: %s\n\n", jsonData)))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
w.StreamSent = true
|
|
w.Flush()
|
|
|
|
return len(p), nil
|
|
}
|
|
|
|
// MessageString returns the complete buffered message
|
|
func (w *OpenAIWriter) MessageString() string {
|
|
return string(w.MessageBuf)
|
|
}
|
|
|
|
// Reset discards what a failed attempt accumulated, so the next provider's
|
|
// answer is not served glued to the dead one's half-sentence.
|
|
//
|
|
// Write APPENDS to Buffer and MessageBuf, and one writer is shared across every
|
|
// failover attempt. A provider that emitted three tokens and then died leaves
|
|
// those three tokens in the buffer; without this the client is handed them
|
|
// followed by a complete answer from somebody else, which is indistinguishable
|
|
// from a model losing its mind and is not detectable downstream.
|
|
//
|
|
// StreamSent is NOT cleared. It records that bytes reached the CLIENT, which is
|
|
// a fact about the wire and cannot be undone — it is precisely the flag that
|
|
// forbids the retry this method prepares for.
|
|
func (w *OpenAIWriter) Reset() {
|
|
w.Buffer = w.Buffer[:0]
|
|
w.MessageBuf = w.MessageBuf[:0]
|
|
w.Cleaner = *NewCleaner(w.Cleaner.bufferSize)
|
|
}
|
|
|
|
// fork makes a writer for ONE raced attempt, over that attempt's own share of
|
|
// the stream.
|
|
//
|
|
// Everything about the DIALECT is copied — the request id, the model, whether
|
|
// this is a stream, whether the client asked for a usage chunk — because every
|
|
// attempt is answering the same request and must speak the same way. Everything
|
|
// about the ANSWER starts empty, because that is the whole reason a race cannot
|
|
// share one writer: the losers' token counts are what the ledger needs, and one
|
|
// set of buffers holds exactly one of them.
|
|
//
|
|
// StreamSent starts false for the same reason it is never cleared by Reset. It
|
|
// records that bytes reached the CLIENT, and for a fork that has not yet won the
|
|
// race, none have.
|
|
func (w *OpenAIWriter) fork(out io.Writer) *OpenAIWriter {
|
|
return &OpenAIWriter{
|
|
out: out,
|
|
Cleaner: *NewCleaner(w.Cleaner.bufferSize),
|
|
Buffer: []byte{},
|
|
RequestID: w.RequestID,
|
|
Stream: w.Stream,
|
|
Model: w.Model,
|
|
IncludeUsage: w.IncludeUsage,
|
|
}
|
|
}
|
|
|
|
// adopt takes the winning attempt's answer as this writer's own, so the handler
|
|
// that has held this writer all along reads what actually served: the message
|
|
// body for a non-streaming reply, and StreamSent for whether Close still owes
|
|
// the client a final chunk.
|
|
//
|
|
// `out` is deliberately NOT adopted. The winner wrote through the race's shared
|
|
// stream, which is spent the moment the race resolves; this writer's own
|
|
// destination is the response body, and the tail belongs there directly.
|
|
func (w *OpenAIWriter) adopt(won io.Writer) {
|
|
o, ok := won.(*OpenAIWriter)
|
|
if !ok || o == nil {
|
|
return
|
|
}
|
|
w.Buffer = o.Buffer
|
|
w.MessageBuf = o.MessageBuf
|
|
w.Cleaner = o.Cleaner
|
|
w.StreamSent = o.StreamSent
|
|
}
|
|
|
|
// Close finalizes the stream by sending completion message and DONE marker
|
|
func (w *OpenAIWriter) Close(promptTokens, completionTokens, totalTokens int) error {
|
|
if !w.Stream {
|
|
return nil
|
|
}
|
|
|
|
if w.StreamSent {
|
|
// Send final message with finish_reason
|
|
chunk := openai.ChatCompletionStreamResponse{
|
|
ID: "chatcmpl-" + w.RequestID,
|
|
Object: "chat.completion.chunk",
|
|
Created: util.GetCurrentUnixTime(),
|
|
Model: w.Model,
|
|
Choices: []openai.ChatCompletionStreamChoice{
|
|
{
|
|
Index: 0,
|
|
Delta: openai.ChatCompletionStreamChoiceDelta{}, // Empty delta
|
|
FinishReason: openai.FinishReasonStop,
|
|
},
|
|
},
|
|
}
|
|
|
|
jsonData, err := json.Marshal(chunk)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = w.out.Write([]byte(fmt.Sprintf("data: %s\n\n", jsonData)))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Send usage as a proper OpenAI SSE chunk so SDK clients (v6+) can parse
|
|
// it — but ONLY when the client opted in via stream_options.include_usage.
|
|
// This chunk's choices array is always empty; emitting it unconditionally
|
|
// crashes clients that read choices[0] on every chunk.
|
|
if w.IncludeUsage {
|
|
usageChunk := map[string]any{
|
|
"id": "chatcmpl-" + w.RequestID,
|
|
"object": "chat.completion.chunk",
|
|
"created": util.GetCurrentUnixTime(),
|
|
"model": w.Model,
|
|
"choices": []any{},
|
|
"usage": openai.Usage{
|
|
PromptTokens: promptTokens,
|
|
CompletionTokens: completionTokens,
|
|
TotalTokens: totalTokens,
|
|
},
|
|
}
|
|
|
|
usageData, err := json.Marshal(usageChunk)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = w.out.Write([]byte(fmt.Sprintf("data: %s\n\n", usageData)))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Final [DONE] marker for SSE
|
|
_, err = w.out.Write([]byte("data: [DONE]\n\n"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
w.Flush()
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// streamOpenAIError ends a stream that failed after it began: one error event,
|
|
// then the end marker. The status line went out with the first byte, so the event
|
|
// is the only place left to say what went wrong. The body is OpenAI's error object.
|
|
func streamOpenAIError(w *bufio.Writer, err error) {
|
|
body, merr := json.Marshal(map[string]map[string]string{"error": {
|
|
"message": err.Error(),
|
|
"type": openAIErrorType(statusForModelError(err)),
|
|
"code": codeOf(err),
|
|
}})
|
|
if merr != nil {
|
|
return
|
|
}
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\ndata: [DONE]\n\n", body)
|
|
_ = w.Flush()
|
|
}
|
|
|
|
// openAIErrorType is OpenAI's word for a failure with this status.
|
|
func openAIErrorType(status int) string {
|
|
switch {
|
|
case status == http.StatusTooManyRequests:
|
|
return "rate_limit_error"
|
|
case status >= 400 && status < 500:
|
|
return "invalid_request_error"
|
|
default:
|
|
return "api_error"
|
|
}
|
|
}
|