mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-21 11:09:56 +00:00
Hanzo's provider ends its stream without [DONE], and nothing else finished the response. Every delta reached the client and none of the terminal events did — no response.output_text.done, .content_part.done, .output_item.done or .completed — so a client reading the stream to spec waited for a completion that never came, retried, and reported the stream as truncated. Observed against api.hanzo.ai: 63 deltas carrying the whole answer, then the connection simply closed. Finishing on finish_reason alone was already tried and reverted, for a real reason: the usage-only chunk commonly arrives after the terminal choice chunk, and finishing early published response.completed with zero usage and made that accounting chunk impossible to apply. Both signals are now required, in whichever order they arrive: whichever of finish_reason and usage is second finishes the response. The stream closes on time and the usage is in it. [DONE] still finishes when a provider sends one, and the bridge Close stays the backstop for a stream that sends neither. Two tests, one per arrival order. The first fails without this change with the exact production symptom — the wire stops at output_text.delta. Co-authored-by: Hanzo <[email protected]> Co-authored-by: Hanzo Dev <[email protected]>
950 lines
32 KiB
Go
950 lines
32 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
package controllers
|
|
|
|
// This file implements the OpenAI Responses API as a total adapter around the
|
|
// existing Chat Completions controller. Keeping auth, routing, failover,
|
|
// metering and balance reservation in ChatCompletions avoids creating a second
|
|
// policy path while giving Responses-only clients (notably current Codex) the
|
|
// wire protocol they require.
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/hanzoai/go-openai"
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
// OpenAIResponsesRequest is the subset of POST /v1/responses used by Codex and
|
|
// OpenAI SDKs. Unknown fields are deliberately accepted for forward
|
|
// compatibility; fields that Chat Completions cannot represent are ignored.
|
|
type OpenAIResponsesRequest struct {
|
|
Model string `json:"model"`
|
|
Instructions string `json:"instructions,omitempty"`
|
|
Input json.RawMessage `json:"input"`
|
|
Tools []responsesTool `json:"tools,omitempty"`
|
|
ToolChoice json.RawMessage `json:"tool_choice,omitempty"`
|
|
ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"`
|
|
Stream bool `json:"stream,omitempty"`
|
|
MaxOutputTokens int `json:"max_output_tokens,omitempty"`
|
|
Temperature *float32 `json:"temperature,omitempty"`
|
|
TopP *float32 `json:"top_p,omitempty"`
|
|
Store bool `json:"store,omitempty"`
|
|
Metadata map[string]string `json:"metadata,omitempty"`
|
|
Reasoning json.RawMessage `json:"reasoning,omitempty"`
|
|
Text json.RawMessage `json:"text,omitempty"`
|
|
}
|
|
|
|
type responsesTool struct {
|
|
Type string `json:"type"`
|
|
Name string `json:"name,omitempty"`
|
|
Description string `json:"description,omitempty"`
|
|
Parameters json.RawMessage `json:"parameters,omitempty"`
|
|
Strict bool `json:"strict,omitempty"`
|
|
Format json.RawMessage `json:"format,omitempty"`
|
|
}
|
|
|
|
type responsesInputItem struct {
|
|
Type string `json:"type"`
|
|
ID string `json:"id,omitempty"`
|
|
Role string `json:"role,omitempty"`
|
|
Content json.RawMessage `json:"content,omitempty"`
|
|
CallID string `json:"call_id,omitempty"`
|
|
Name string `json:"name,omitempty"`
|
|
Arguments string `json:"arguments,omitempty"`
|
|
Input string `json:"input,omitempty"`
|
|
Output json.RawMessage `json:"output,omitempty"`
|
|
}
|
|
|
|
type responsesContentPart struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text,omitempty"`
|
|
ImageURL string `json:"image_url,omitempty"`
|
|
}
|
|
|
|
// Responses implements POST /v1/responses. The converted request is completed by
|
|
// the chat path, which is handed a sink saying where the answer goes: a stream is
|
|
// translated as it is produced, a whole body is translated entire.
|
|
func (c *ApiController) Responses() {
|
|
authHeader := c.Header("Authorization")
|
|
if !strings.HasPrefix(authHeader, "Bearer ") {
|
|
c.ResponseErrorWithStatus(http.StatusUnauthorized, "Invalid API key format. Expected 'Bearer API_KEY'")
|
|
return
|
|
}
|
|
token := strings.TrimPrefix(authHeader, "Bearer ")
|
|
if isPublishableKey(token) {
|
|
c.ResponseErrorWithStatus(http.StatusForbidden, "Publishable keys cannot access /v1/responses. Use a secret key.")
|
|
return
|
|
}
|
|
|
|
body := c.Body()
|
|
if strings.EqualFold(strings.TrimSpace(c.Header("Content-Encoding")), "zstd") {
|
|
decoded, err := decodeResponsesZstd(body)
|
|
if err != nil {
|
|
// Authenticate before reporting, for the reason the 400 below gives.
|
|
if authErr := c.authenticate(token); authErr != nil {
|
|
c.ResponseAuthError(authErr)
|
|
return
|
|
}
|
|
// A body over the bound is 413, not 400: it is not malformed, it is
|
|
// too big, and only one of those tells the caller to send less.
|
|
if errors.Is(err, zstd.ErrDecoderSizeExceeded) {
|
|
c.ResponseErrorWithStatus(http.StatusRequestEntityTooLarge, fmt.Sprintf(
|
|
"request body exceeds the %d MiB limit", MaxDecoded>>20))
|
|
return
|
|
}
|
|
c.ResponseErrorWithStatus(http.StatusBadRequest, "Failed to decompress zstd request: "+err.Error())
|
|
return
|
|
}
|
|
body = decoded
|
|
c.Fiber().Request().Header.Del("Content-Encoding")
|
|
}
|
|
|
|
var request OpenAIResponsesRequest
|
|
if err := json.Unmarshal(body, &request); err != nil {
|
|
if authErr := c.authenticate(token); authErr != nil {
|
|
c.ResponseAuthError(authErr)
|
|
return
|
|
}
|
|
c.ResponseErrorWithStatus(http.StatusBadRequest, "Failed to parse Responses request: "+err.Error())
|
|
return
|
|
}
|
|
if request.Model == "" {
|
|
c.ResponseErrorWithStatus(http.StatusBadRequest, "model is required")
|
|
return
|
|
}
|
|
|
|
chatRequest, toolKinds, err := responsesToChatRequest(&request)
|
|
if err != nil {
|
|
c.ResponseErrorWithStatus(http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
chatBody, err := json.Marshal(chatRequest)
|
|
if err != nil {
|
|
c.ResponseErrorWithStatus(http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
|
|
// This endpoint IS a chat completion, read in a second dialect. So it asks for
|
|
// one and says where the answer should go, rather than swapping the writer out
|
|
// of the request context and reading whatever came back through it.
|
|
//
|
|
// A stream is decorated as it is produced; a whole body is handed over entire.
|
|
// The bridge exists only on the streaming path, which is what `bridge != nil`
|
|
// means below.
|
|
var bridge *responsesBridge
|
|
to := &sink{
|
|
wrap: func(w io.Writer) io.Writer {
|
|
bridge = newResponsesBridge(w, &request, toolKinds)
|
|
return bridge
|
|
},
|
|
body: func(chat []byte) error {
|
|
out, err := openAIChatResponseToResponses(chat, &request, toolKinds)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
c.SetHeader("Content-Type", "application/json")
|
|
return c.Bytes(http.StatusOK, out)
|
|
},
|
|
}
|
|
|
|
// The chat request on the wire, because the completion reads its own body. This
|
|
// is the request being made, not a place being patched.
|
|
c.Fiber().Request().SetBody(chatBody)
|
|
c.Fiber().Request().Header.Del("Content-Length")
|
|
|
|
c.chatCompletions(callerBearer, to)
|
|
|
|
if bridge != nil {
|
|
// Logged, not answered: the events are already on the wire, so there is no
|
|
// status left to change. Losing it silently would leave a stream that simply
|
|
// stops with nothing to explain it.
|
|
if err := bridge.Close(); err != nil {
|
|
log.Error(fmt.Sprintf("Responses: closing the stream failed: %s", err.Error()))
|
|
}
|
|
}
|
|
}
|
|
|
|
// decodeResponsesZstd expands a zstd-framed Responses body, bounded by what it
|
|
// expands to. It uses a pure-Go decoder because Cloud release binaries are built
|
|
// with CGO_ENABLED=0. Current Codex clients compress large requests with zstd.
|
|
//
|
|
// The bound is MaxDecoded, the same ceiling every other body gets, and it has
|
|
// to be stated: the decoder's own default is 64 GiB. Capping the COMPRESSED
|
|
// input caps nothing that costs memory here — zstd's ratio on repetitive input
|
|
// runs to thousands to one, so the 64 MiB that reaches this function is already
|
|
// enough to ask for more heap than any pod has. The decode also runs before the
|
|
// token is validated, so whatever it allocates, a caller holding nothing but a
|
|
// syntactically-valid Bearer string can make it allocate.
|
|
//
|
|
// Overflow returns zstd.ErrDecoderSizeExceeded, which callers answer 413 —
|
|
// naming the limit, rather than letting a half-expanded body fail later as an
|
|
// unintelligible parse error.
|
|
func decodeResponsesZstd(body []byte) ([]byte, error) {
|
|
decoder, err := zstd.NewReader(nil, zstd.WithDecoderMaxMemory(uint64(MaxDecoded)))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer decoder.Close()
|
|
return decoder.DecodeAll(body, nil)
|
|
}
|
|
|
|
func responsesToChatRequest(request *OpenAIResponsesRequest) (*openai.ChatCompletionRequest, map[string]string, error) {
|
|
messages, err := responsesInputToMessages(request.Instructions, request.Input)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if len(messages) == 0 {
|
|
return nil, nil, fmt.Errorf("input must contain at least one message")
|
|
}
|
|
|
|
tools := make([]openai.Tool, 0, len(request.Tools))
|
|
toolKinds := make(map[string]string, len(request.Tools))
|
|
for _, tool := range request.Tools {
|
|
if tool.Name == "" {
|
|
continue
|
|
}
|
|
kind := tool.Type
|
|
if kind == "" {
|
|
kind = "function"
|
|
}
|
|
toolKinds[tool.Name] = kind
|
|
params := any(json.RawMessage(tool.Parameters))
|
|
if len(tool.Parameters) == 0 || string(tool.Parameters) == "null" {
|
|
if kind == "custom" {
|
|
params = map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{
|
|
"input": map[string]interface{}{"type": "string"},
|
|
},
|
|
"required": []string{"input"},
|
|
}
|
|
} else {
|
|
params = map[string]interface{}{"type": "object", "properties": map[string]interface{}{}}
|
|
}
|
|
}
|
|
tools = append(tools, openai.Tool{
|
|
Type: openai.ToolTypeFunction,
|
|
Function: &openai.FunctionDefinition{
|
|
Name: tool.Name, Description: tool.Description, Parameters: params, Strict: tool.Strict,
|
|
},
|
|
})
|
|
}
|
|
|
|
chat := &openai.ChatCompletionRequest{
|
|
Model: request.Model,
|
|
Messages: messages,
|
|
Tools: tools,
|
|
ToolChoice: responsesToolChoiceToChat(request.ToolChoice),
|
|
Stream: request.Stream,
|
|
MaxCompletionTokens: request.MaxOutputTokens,
|
|
Store: request.Store,
|
|
Metadata: request.Metadata,
|
|
}
|
|
if request.Stream {
|
|
// The existing ChatCompletions proxy only forwards its forced upstream
|
|
// usage chunk when the downstream client opted in. The Responses bridge
|
|
// is that client, and response.completed must contain real token counts.
|
|
chat.StreamOptions = &openai.StreamOptions{IncludeUsage: true}
|
|
}
|
|
if request.MaxOutputTokens > 0 {
|
|
// DO's OpenAI-compatible models currently honor max_tokens uniformly.
|
|
chat.MaxTokens = request.MaxOutputTokens
|
|
chat.MaxCompletionTokens = 0
|
|
}
|
|
if request.Temperature != nil {
|
|
chat.Temperature = *request.Temperature
|
|
}
|
|
if request.TopP != nil {
|
|
chat.TopP = *request.TopP
|
|
}
|
|
if request.ParallelToolCalls != nil {
|
|
chat.ParallelToolCalls = *request.ParallelToolCalls
|
|
}
|
|
return chat, toolKinds, nil
|
|
}
|
|
|
|
func responsesInputToMessages(instructions string, raw json.RawMessage) ([]openai.ChatCompletionMessage, error) {
|
|
messages := make([]openai.ChatCompletionMessage, 0, 8)
|
|
if instructions != "" {
|
|
messages = append(messages, openai.ChatCompletionMessage{Role: openai.ChatMessageRoleSystem, Content: instructions})
|
|
}
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return messages, nil
|
|
}
|
|
var text string
|
|
if err := json.Unmarshal(raw, &text); err == nil {
|
|
return append(messages, openai.ChatCompletionMessage{Role: openai.ChatMessageRoleUser, Content: text}), nil
|
|
}
|
|
|
|
var items []responsesInputItem
|
|
if err := json.Unmarshal(raw, &items); err != nil {
|
|
return nil, fmt.Errorf("input must be a string or an array of Responses input items: %w", err)
|
|
}
|
|
|
|
var pendingCalls []openai.ToolCall
|
|
flushCalls := func() {
|
|
if len(pendingCalls) == 0 {
|
|
return
|
|
}
|
|
messages = append(messages, openai.ChatCompletionMessage{Role: openai.ChatMessageRoleAssistant, ToolCalls: pendingCalls})
|
|
pendingCalls = nil
|
|
}
|
|
|
|
for _, item := range items {
|
|
switch item.Type {
|
|
case "function_call", "custom_tool_call", "local_shell_call":
|
|
callID := item.CallID
|
|
if callID == "" {
|
|
callID = item.ID
|
|
}
|
|
arguments := item.Arguments
|
|
if item.Type == "custom_tool_call" && arguments == "" {
|
|
encoded, _ := json.Marshal(map[string]string{"input": item.Input})
|
|
arguments = string(encoded)
|
|
}
|
|
pendingCalls = append(pendingCalls, openai.ToolCall{
|
|
ID: callID, Type: openai.ToolTypeFunction,
|
|
Function: openai.FunctionCall{Name: item.Name, Arguments: arguments},
|
|
})
|
|
case "function_call_output", "custom_tool_call_output", "local_shell_call_output":
|
|
flushCalls()
|
|
messages = append(messages, openai.ChatCompletionMessage{
|
|
Role: openai.ChatMessageRoleTool, ToolCallID: item.CallID, Content: responsesOutputText(item.Output),
|
|
})
|
|
case "message", "":
|
|
flushCalls()
|
|
role := item.Role
|
|
if role == "" {
|
|
role = openai.ChatMessageRoleUser
|
|
}
|
|
msg := openai.ChatCompletionMessage{Role: role}
|
|
parts := responsesContentParts(item.Content)
|
|
for _, part := range parts {
|
|
switch part.Type {
|
|
case "input_image":
|
|
msg.MultiContent = append(msg.MultiContent, openai.ChatMessagePart{
|
|
Type: openai.ChatMessagePartTypeImageURL,
|
|
ImageURL: &openai.ChatMessageImageURL{URL: part.ImageURL},
|
|
})
|
|
default:
|
|
if part.Text != "" {
|
|
msg.Content += part.Text
|
|
msg.MultiContent = append(msg.MultiContent, openai.ChatMessagePart{Type: openai.ChatMessagePartTypeText, Text: part.Text})
|
|
}
|
|
}
|
|
}
|
|
// go-openai marshals exactly one of Content and MultiContent. A
|
|
// message with an image keeps the parts and drops the joined text;
|
|
// text-only messages go out as a plain string.
|
|
if len(msg.MultiContent) > 0 {
|
|
hasImage := false
|
|
for _, part := range msg.MultiContent {
|
|
hasImage = hasImage || part.Type == openai.ChatMessagePartTypeImageURL
|
|
}
|
|
if hasImage {
|
|
msg.Content = ""
|
|
} else {
|
|
msg.MultiContent = nil
|
|
}
|
|
}
|
|
messages = append(messages, msg)
|
|
case "reasoning":
|
|
// Encrypted/reasoning items are provider-specific state. The serving
|
|
// model receives the visible conversation and continues normally.
|
|
default:
|
|
// Forward-compatible: ignore server-executed Responses item types.
|
|
}
|
|
}
|
|
flushCalls()
|
|
return messages, nil
|
|
}
|
|
|
|
func responsesContentParts(raw json.RawMessage) []responsesContentPart {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return nil
|
|
}
|
|
var text string
|
|
if json.Unmarshal(raw, &text) == nil {
|
|
return []responsesContentPart{{Type: "input_text", Text: text}}
|
|
}
|
|
var parts []responsesContentPart
|
|
if json.Unmarshal(raw, &parts) == nil {
|
|
return parts
|
|
}
|
|
return []responsesContentPart{{Type: "input_text", Text: string(raw)}}
|
|
}
|
|
|
|
func responsesOutputText(raw json.RawMessage) string {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return ""
|
|
}
|
|
var text string
|
|
if json.Unmarshal(raw, &text) == nil {
|
|
return text
|
|
}
|
|
parts := responsesContentParts(raw)
|
|
var out strings.Builder
|
|
for _, part := range parts {
|
|
out.WriteString(part.Text)
|
|
}
|
|
if out.Len() > 0 {
|
|
return out.String()
|
|
}
|
|
return string(raw)
|
|
}
|
|
|
|
func responsesToolChoiceToChat(raw json.RawMessage) interface{} {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return nil
|
|
}
|
|
var choice string
|
|
if json.Unmarshal(raw, &choice) == nil {
|
|
return choice
|
|
}
|
|
var choiceObj struct {
|
|
Type string `json:"type"`
|
|
Name string `json:"name"`
|
|
}
|
|
if json.Unmarshal(raw, &choiceObj) == nil && choiceObj.Name != "" {
|
|
return map[string]interface{}{
|
|
"type": "function", "function": map[string]string{"name": choiceObj.Name},
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// responsesBridge reads the chat completion this request was translated into and
|
|
// writes it back out in the Responses dialect.
|
|
//
|
|
// It is handed BYTES, and that is the whole shape of it. The chat side used to
|
|
// write through it as an http.ResponseWriter — status included — so it had to
|
|
// intercept WriteHeader, withhold a success it was not ready to send, keep a
|
|
// buffer of a body it meant to rewrite, and remember which of those it had done.
|
|
// Every one of those existed to separate a body from a status arriving on the same
|
|
// call. It now receives the two things it actually wants, separately: the stream's
|
|
// chunks as they are produced, and — through the sink in openai_api.go — the
|
|
// non-streaming body as one slice. The status is the handler's, and stays there.
|
|
type responsesBridge struct {
|
|
out io.Writer
|
|
request *OpenAIResponsesRequest
|
|
toolKinds map[string]string
|
|
lineBuf bytes.Buffer
|
|
translator *responsesStreamTranslator
|
|
mu sync.Mutex
|
|
closed bool
|
|
}
|
|
|
|
func newResponsesBridge(out io.Writer, request *OpenAIResponsesRequest, toolKinds map[string]string) *responsesBridge {
|
|
return &responsesBridge{out: out, request: request, toolKinds: toolKinds}
|
|
}
|
|
|
|
// Write takes the chat stream's SSE as it is produced and emits the Responses
|
|
// events for it. Chunks arrive on no particular boundary, so a partial last line
|
|
// is kept for the next call.
|
|
func (w *responsesBridge) Write(p []byte) (int, error) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if w.translator == nil {
|
|
w.translator = newResponsesStreamTranslator(w.emitLocked, w.request, w.toolKinds)
|
|
if err := w.translator.start(); err != nil {
|
|
return len(p), err
|
|
}
|
|
}
|
|
_, _ = w.lineBuf.Write(p)
|
|
for {
|
|
line, err := w.lineBuf.ReadString('\n')
|
|
if err != nil {
|
|
// ReadString consumed the partial line; put it back until the next write.
|
|
w.lineBuf.Reset()
|
|
_, _ = w.lineBuf.WriteString(line)
|
|
break
|
|
}
|
|
line = strings.TrimSpace(line)
|
|
if !strings.HasPrefix(line, "data:") {
|
|
continue
|
|
}
|
|
data := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
|
|
if data == "" {
|
|
continue
|
|
}
|
|
if data == "[DONE]" {
|
|
if err := w.translator.finish(); err != nil {
|
|
return len(p), err
|
|
}
|
|
continue
|
|
}
|
|
var chunk openaiStreamChunk
|
|
if err := json.Unmarshal([]byte(data), &chunk); err != nil {
|
|
continue
|
|
}
|
|
if err := w.translator.handleChunk(&chunk); err != nil {
|
|
return len(p), err
|
|
}
|
|
}
|
|
return len(p), nil
|
|
}
|
|
|
|
// Flush pushes each event as it is produced, which is what makes this a stream.
|
|
// Two shapes, for the same reason OpenAIWriter takes both.
|
|
func (w *responsesBridge) Flush() {
|
|
switch f := w.out.(type) {
|
|
case interface{ Flush() error }:
|
|
_ = f.Flush()
|
|
case interface{ Flush() }:
|
|
f.Flush()
|
|
}
|
|
}
|
|
|
|
// Close ends the event stream. Idempotent: the chat side sends [DONE], which
|
|
// finishes the translator, and the handler closes again on the way out.
|
|
func (w *responsesBridge) Close() error {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if w.closed {
|
|
return nil
|
|
}
|
|
w.closed = true
|
|
if w.translator == nil {
|
|
return nil
|
|
}
|
|
return w.translator.finish()
|
|
}
|
|
|
|
func (w *responsesBridge) emitLocked(event string, data interface{}) error {
|
|
b, err := json.Marshal(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err = fmt.Fprintf(w.out, "event: %s\ndata: %s\n\n", event, b); err != nil {
|
|
return err
|
|
}
|
|
w.Flush()
|
|
return nil
|
|
}
|
|
|
|
type responsesToolCallState struct {
|
|
Index int
|
|
OutputIndex int
|
|
ID string
|
|
CallID string
|
|
Name string
|
|
Arguments strings.Builder
|
|
Kind string
|
|
Started bool
|
|
}
|
|
|
|
type responsesStreamTranslator struct {
|
|
emit func(string, interface{}) error
|
|
request *OpenAIResponsesRequest
|
|
toolKinds map[string]string
|
|
responseID string
|
|
messageID string
|
|
createdAt int64
|
|
sequence int
|
|
started bool
|
|
finished bool
|
|
textStarted bool
|
|
textOutputIndex int
|
|
text strings.Builder
|
|
calls map[int]*responsesToolCallState
|
|
callOrder []int
|
|
nextOutput int
|
|
promptTokens int
|
|
completionTokens int
|
|
totalTokens int
|
|
sawFinishReason bool
|
|
sawUsage bool
|
|
}
|
|
|
|
func newResponsesStreamTranslator(emit func(string, interface{}) error, request *OpenAIResponsesRequest, toolKinds map[string]string) *responsesStreamTranslator {
|
|
return &responsesStreamTranslator{
|
|
emit: emit, request: request, toolKinds: toolKinds,
|
|
responseID: "resp_" + uuid.NewString(), messageID: "msg_" + uuid.NewString(),
|
|
createdAt: time.Now().Unix(), calls: map[int]*responsesToolCallState{},
|
|
}
|
|
}
|
|
|
|
func (t *responsesStreamTranslator) next() int { n := t.sequence; t.sequence++; return n }
|
|
|
|
func (t *responsesStreamTranslator) start() error {
|
|
if t.started {
|
|
return nil
|
|
}
|
|
t.started = true
|
|
return t.emit("response.created", map[string]interface{}{
|
|
"type": "response.created", "sequence_number": t.next(),
|
|
"response": t.resource("in_progress", []interface{}{}),
|
|
})
|
|
}
|
|
|
|
func (t *responsesStreamTranslator) handleChunk(chunk *openaiStreamChunk) error {
|
|
if err := t.start(); err != nil {
|
|
return err
|
|
}
|
|
if chunk.Usage != nil {
|
|
if chunk.Usage.PromptTokens > 0 {
|
|
t.promptTokens = chunk.Usage.PromptTokens
|
|
}
|
|
if chunk.Usage.CompletionTokens > 0 {
|
|
t.completionTokens = chunk.Usage.CompletionTokens
|
|
}
|
|
if chunk.Usage.TotalTokens > 0 {
|
|
t.totalTokens = chunk.Usage.TotalTokens
|
|
}
|
|
}
|
|
if chunk.Usage != nil {
|
|
t.sawUsage = true
|
|
if t.sawFinishReason {
|
|
return t.finish()
|
|
}
|
|
}
|
|
for i := range chunk.Choices {
|
|
choice := &chunk.Choices[i]
|
|
if choice.Delta.Content != "" {
|
|
if !t.textStarted {
|
|
t.textStarted = true
|
|
outputIndex := t.nextOutput
|
|
t.textOutputIndex = outputIndex
|
|
t.nextOutput++
|
|
if err := t.emit("response.output_item.added", map[string]interface{}{
|
|
"type": "response.output_item.added", "sequence_number": t.next(), "output_index": outputIndex,
|
|
"item": map[string]interface{}{"id": t.messageID, "type": "message", "role": "assistant", "status": "in_progress", "content": []interface{}{}},
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if err := t.emit("response.content_part.added", map[string]interface{}{
|
|
"type": "response.content_part.added", "sequence_number": t.next(), "output_index": outputIndex, "content_index": 0,
|
|
"part": map[string]interface{}{"type": "output_text", "text": "", "annotations": []interface{}{}},
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
t.text.WriteString(choice.Delta.Content)
|
|
if err := t.emit("response.output_text.delta", map[string]interface{}{
|
|
"type": "response.output_text.delta", "sequence_number": t.next(), "item_id": t.messageID,
|
|
"output_index": t.textOutputIndex, "content_index": 0, "delta": choice.Delta.Content,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
for j := range choice.Delta.ToolCalls {
|
|
tc := &choice.Delta.ToolCalls[j]
|
|
idx := j
|
|
if tc.Index != nil {
|
|
idx = *tc.Index
|
|
}
|
|
call := t.calls[idx]
|
|
if call == nil {
|
|
call = &responsesToolCallState{Index: idx, OutputIndex: t.nextOutput, Kind: "function"}
|
|
t.nextOutput++
|
|
t.calls[idx] = call
|
|
t.callOrder = append(t.callOrder, idx)
|
|
}
|
|
if tc.ID != "" {
|
|
call.ID, call.CallID = tc.ID, tc.ID
|
|
}
|
|
if tc.Function.Name != "" {
|
|
call.Name = tc.Function.Name
|
|
if kind := t.toolKinds[call.Name]; kind != "" {
|
|
call.Kind = kind
|
|
}
|
|
}
|
|
if !call.Started && (call.CallID != "" || call.Name != "") {
|
|
call.Started = true
|
|
if call.ID == "" {
|
|
call.ID = "fc_" + uuid.NewString()
|
|
}
|
|
if call.CallID == "" {
|
|
call.CallID = call.ID
|
|
}
|
|
if err := t.emit("response.output_item.added", map[string]interface{}{
|
|
"type": "response.output_item.added", "sequence_number": t.next(), "output_index": call.OutputIndex,
|
|
"item": call.item("in_progress", ""),
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if tc.Function.Arguments != "" {
|
|
call.Arguments.WriteString(tc.Function.Arguments)
|
|
event := "response.function_call_arguments.delta"
|
|
if call.Kind == "custom" {
|
|
event = "response.custom_tool_call_input.delta"
|
|
}
|
|
if err := t.emit(event, map[string]interface{}{
|
|
"type": event, "sequence_number": t.next(), "item_id": call.ID,
|
|
"output_index": call.OutputIndex, "delta": tc.Function.Arguments,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
// The end of the answer, in whichever order the two signals arrive.
|
|
//
|
|
// finish_reason alone is not enough: providers commonly send the
|
|
// usage-only chunk *after* the terminal choice chunk, and finishing on
|
|
// finish_reason alone published response.completed with zero usage and
|
|
// made that accounting chunk impossible to apply.
|
|
//
|
|
// Waiting only for [DONE] is not enough either, and that is what broke
|
|
// clients: an OpenAI-compatible provider is free to end the stream
|
|
// without one. When that happened the terminal events were never sent —
|
|
// response.output_text.done, .content_part.done, .output_item.done and
|
|
// .completed — so every delta arrived and the response never closed.
|
|
// A client reading the Responses stream to spec waits forever, retries,
|
|
// and reports the stream as truncated.
|
|
//
|
|
// So: whichever of the two arrives second finishes the response, and
|
|
// the bridge Close stays the backstop for a stream that sends neither.
|
|
if choice.FinishReason != "" {
|
|
t.sawFinishReason = true
|
|
if t.sawUsage {
|
|
return t.finish()
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *responsesToolCallState) finalArguments() string {
|
|
args := c.Arguments.String()
|
|
if c.Kind != "custom" {
|
|
return args
|
|
}
|
|
var wrapped struct {
|
|
Input string `json:"input"`
|
|
}
|
|
if json.Unmarshal([]byte(args), &wrapped) == nil && wrapped.Input != "" {
|
|
return wrapped.Input
|
|
}
|
|
return args
|
|
}
|
|
|
|
func (c *responsesToolCallState) item(status, arguments string) map[string]interface{} {
|
|
if c.Kind == "custom" {
|
|
return map[string]interface{}{
|
|
"id": c.ID, "type": "custom_tool_call", "call_id": c.CallID,
|
|
"name": c.Name, "input": arguments, "status": status,
|
|
}
|
|
}
|
|
return map[string]interface{}{
|
|
"id": c.ID, "type": "function_call", "call_id": c.CallID,
|
|
"name": c.Name, "arguments": arguments, "status": status,
|
|
}
|
|
}
|
|
|
|
func (t *responsesStreamTranslator) finish() error {
|
|
if t.finished {
|
|
return nil
|
|
}
|
|
if err := t.start(); err != nil {
|
|
return err
|
|
}
|
|
t.finished = true
|
|
outputs := make([]interface{}, 0, 1+len(t.calls))
|
|
if t.textStarted {
|
|
text := t.text.String()
|
|
message := map[string]interface{}{
|
|
"id": t.messageID, "type": "message", "role": "assistant", "status": "completed",
|
|
"content": []interface{}{map[string]interface{}{"type": "output_text", "text": text, "annotations": []interface{}{}}},
|
|
}
|
|
if err := t.emit("response.output_text.done", map[string]interface{}{
|
|
"type": "response.output_text.done", "sequence_number": t.next(), "item_id": t.messageID,
|
|
"output_index": t.textOutputIndex, "content_index": 0, "text": text,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if err := t.emit("response.content_part.done", map[string]interface{}{
|
|
"type": "response.content_part.done", "sequence_number": t.next(), "output_index": t.textOutputIndex, "content_index": 0,
|
|
"part": message["content"].([]interface{})[0],
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if err := t.emit("response.output_item.done", map[string]interface{}{
|
|
"type": "response.output_item.done", "sequence_number": t.next(), "output_index": t.textOutputIndex, "item": message,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
outputs = append(outputs, message)
|
|
}
|
|
for _, idx := range t.callOrder {
|
|
call := t.calls[idx]
|
|
args := call.finalArguments()
|
|
if !call.Started {
|
|
call.Started = true
|
|
if call.ID == "" {
|
|
call.ID = "fc_" + uuid.NewString()
|
|
}
|
|
if call.CallID == "" {
|
|
call.CallID = call.ID
|
|
}
|
|
}
|
|
if call.Kind != "custom" {
|
|
if err := t.emit("response.function_call_arguments.done", map[string]interface{}{
|
|
"type": "response.function_call_arguments.done", "sequence_number": t.next(), "item_id": call.ID,
|
|
"output_index": call.OutputIndex, "arguments": args,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
item := call.item("completed", args)
|
|
if err := t.emit("response.output_item.done", map[string]interface{}{
|
|
"type": "response.output_item.done", "sequence_number": t.next(), "output_index": call.OutputIndex, "item": item,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
outputs = append(outputs, item)
|
|
}
|
|
return t.emit("response.completed", map[string]interface{}{
|
|
"type": "response.completed", "sequence_number": t.next(),
|
|
"response": t.resource("completed", outputs),
|
|
})
|
|
}
|
|
|
|
func (t *responsesStreamTranslator) resource(status string, output []interface{}) responsesResource {
|
|
total := t.totalTokens
|
|
if total == 0 {
|
|
total = t.promptTokens + t.completionTokens
|
|
}
|
|
return responsesResource{
|
|
ID: t.responseID, Object: "response", CreatedAt: t.createdAt, Status: status,
|
|
Model: t.request.Model, Output: output,
|
|
Instructions: t.request.Instructions, MaxOutputTokens: t.request.MaxOutputTokens,
|
|
Metadata: t.request.Metadata, ParallelToolCalls: t.request.ParallelToolCalls,
|
|
Reasoning: jsonOrNil(t.request.Reasoning),
|
|
Store: t.request.Store, Temperature: t.request.Temperature,
|
|
Text: jsonOrNil(t.request.Text), ToolChoice: jsonOrNil(t.request.ToolChoice),
|
|
Tools: t.request.Tools, TopP: t.request.TopP,
|
|
Usage: responsesUsage{
|
|
InputTokens: t.promptTokens,
|
|
InputDetails: responsesInputDetails{},
|
|
OutputTokens: t.completionTokens,
|
|
OutputDetails: responsesOutputDetails{},
|
|
TotalTokens: total,
|
|
},
|
|
}
|
|
}
|
|
|
|
// responsesResource is one answer from /v1/responses.
|
|
//
|
|
// Error, IncompleteDetails and PreviousResponseID are `any` and carry no
|
|
// omitempty because the OpenAI Responses API sends them as an explicit null
|
|
// rather than omitting them, which is what the map this replaced did by setting
|
|
// the key to nil.
|
|
//
|
|
// Output is the one field with no shape to state: it is a union — a message, a
|
|
// tool call, a reasoning item — and each element says which it is in its own
|
|
// `type`. An empty schema is the honest answer for a union; a guessed one would
|
|
// send every client down the wrong branch.
|
|
type responsesResource struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
CreatedAt int64 `json:"created_at"`
|
|
Status string `json:"status"`
|
|
Model string `json:"model"`
|
|
Output []interface{} `json:"output"`
|
|
Error interface{} `json:"error"`
|
|
IncompleteDetails interface{} `json:"incomplete_details"`
|
|
Instructions string `json:"instructions"`
|
|
MaxOutputTokens int `json:"max_output_tokens"`
|
|
Metadata map[string]string `json:"metadata"`
|
|
ParallelToolCalls *bool `json:"parallel_tool_calls"`
|
|
PreviousResponseID interface{} `json:"previous_response_id"`
|
|
Reasoning interface{} `json:"reasoning"`
|
|
Store bool `json:"store"`
|
|
Temperature *float32 `json:"temperature"`
|
|
Text interface{} `json:"text"`
|
|
ToolChoice interface{} `json:"tool_choice"`
|
|
Tools []responsesTool `json:"tools"`
|
|
TopP *float32 `json:"top_p"`
|
|
Usage responsesUsage `json:"usage"`
|
|
}
|
|
|
|
// responsesUsage is what the answer cost.
|
|
type responsesUsage struct {
|
|
InputTokens int `json:"input_tokens"`
|
|
InputDetails responsesInputDetails `json:"input_tokens_details"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
OutputDetails responsesOutputDetails `json:"output_tokens_details"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
}
|
|
|
|
type responsesInputDetails struct {
|
|
CachedTokens int `json:"cached_tokens"`
|
|
}
|
|
|
|
type responsesOutputDetails struct {
|
|
ReasoningTokens int `json:"reasoning_tokens"`
|
|
}
|
|
|
|
func jsonOrNil(raw json.RawMessage) interface{} {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return nil
|
|
}
|
|
var value interface{}
|
|
if json.Unmarshal(raw, &value) != nil {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
func openAIChatResponseToResponses(body []byte, request *OpenAIResponsesRequest, toolKinds map[string]string) ([]byte, error) {
|
|
var response struct {
|
|
Choices []struct {
|
|
Message struct {
|
|
Content string `json:"content"`
|
|
ToolCalls []struct {
|
|
ID string `json:"id"`
|
|
Function struct{ Name, Arguments string } `json:"function"`
|
|
} `json:"tool_calls"`
|
|
} `json:"message"`
|
|
} `json:"choices"`
|
|
Usage struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
} `json:"usage"`
|
|
}
|
|
if err := json.Unmarshal(body, &response); err != nil {
|
|
return nil, fmt.Errorf("invalid upstream chat response: %w", err)
|
|
}
|
|
translator := newResponsesStreamTranslator(func(string, interface{}) error { return nil }, request, toolKinds)
|
|
translator.promptTokens = response.Usage.PromptTokens
|
|
translator.completionTokens = response.Usage.CompletionTokens
|
|
translator.totalTokens = response.Usage.TotalTokens
|
|
outputs := []interface{}{}
|
|
if len(response.Choices) > 0 {
|
|
message := response.Choices[0].Message
|
|
if message.Content != "" {
|
|
outputs = append(outputs, map[string]interface{}{
|
|
"id": translator.messageID, "type": "message", "role": "assistant", "status": "completed",
|
|
"content": []interface{}{map[string]interface{}{"type": "output_text", "text": message.Content, "annotations": []interface{}{}}},
|
|
})
|
|
}
|
|
for _, tc := range message.ToolCalls {
|
|
call := &responsesToolCallState{ID: tc.ID, CallID: tc.ID, Name: tc.Function.Name, Kind: toolKinds[tc.Function.Name]}
|
|
if call.Kind == "" {
|
|
call.Kind = "function"
|
|
}
|
|
call.Arguments.WriteString(tc.Function.Arguments)
|
|
outputs = append(outputs, call.item("completed", call.finalArguments()))
|
|
}
|
|
}
|
|
resource := translator.resource("completed", outputs)
|
|
return json.Marshal(resource)
|
|
}
|