mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
The reasoning stripper held content until </think> and never let it go, so an answer with no block (reasoning in reasoning_content, or a non-thinking reply) was dropped. It now strips only a block that begins the content, is off once reasoning_content arrives, and is flushed at finish, [DONE] and stream end on the forward, responses, Anthropic and console paths; the Anthropic translator keys on the upstream id. Lift: every streamed chunk feeds the fence tracker; a choice of a chunk with no delta is streamed and releases what it holds before its finish; nothing follows response.completed; fences and indented code follow CommonMark; a prose opener is released as soon as the JSON body shape fails, a fenced opener without its separator is released; JSON bodies reject repeated or case-variant members; allowed_tools narrows what is lifted; the family follows the route that served. Co-authored-by: Hanzo Dev <[email protected]>
1223 lines
42 KiB
Go
1223 lines
42 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
package controllers
|
|
|
|
// This file implements the OpenAI Responses API as a total adapter around the
|
|
// existing Chat Completions controller. Keeping auth, routing, failover,
|
|
// metering and balance reservation in ChatCompletions avoids creating a second
|
|
// policy path while giving Responses-only clients (notably current Codex) the
|
|
// wire protocol they require.
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/hanzoai/ai/log"
|
|
"github.com/hanzoai/go-openai"
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
// OpenAIResponsesRequest is the subset of POST /v1/responses used by Codex and
|
|
// OpenAI SDKs. Unknown fields are deliberately accepted for forward
|
|
// compatibility; fields that Chat Completions cannot represent are ignored.
|
|
type OpenAIResponsesRequest struct {
|
|
Model string `json:"model"`
|
|
Instructions string `json:"instructions,omitempty"`
|
|
Input json.RawMessage `json:"input"`
|
|
Tools []responsesTool `json:"tools,omitempty"`
|
|
ToolChoice json.RawMessage `json:"tool_choice,omitempty"`
|
|
ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"`
|
|
Stream bool `json:"stream,omitempty"`
|
|
MaxOutputTokens int `json:"max_output_tokens,omitempty"`
|
|
Temperature *float32 `json:"temperature,omitempty"`
|
|
TopP *float32 `json:"top_p,omitempty"`
|
|
Store bool `json:"store,omitempty"`
|
|
Metadata map[string]string `json:"metadata,omitempty"`
|
|
Reasoning json.RawMessage `json:"reasoning,omitempty"`
|
|
Text json.RawMessage `json:"text,omitempty"`
|
|
|
|
// spaces names, for each tool a namespace holds, its flat chat name's
|
|
// namespace and own name, so a call coming back is answered as the client
|
|
// declared it.
|
|
spaces map[string]spaced
|
|
}
|
|
|
|
// spaced is a tool inside a namespace: Codex declares each MCP server's tools
|
|
// as one `namespace` and routes a call by its namespace and name.
|
|
type spaced struct{ Namespace, Name string }
|
|
|
|
// flatName is a namespaced tool's chat function name. Chat Completions has no
|
|
// namespaces, so the two are joined, within the 64 characters a function name
|
|
// may have; the same pair always gives the same name, so a call in the history
|
|
// and a call in the answer agree.
|
|
func flatName(namespace, name string) string {
|
|
flat := namespace + "__" + strings.TrimLeft(name, "_")
|
|
if len(flat) <= 64 {
|
|
return flat
|
|
}
|
|
sum := sha256.Sum256([]byte(namespace + "\x00" + name))
|
|
return flat[:55] + "_" + hex.EncodeToString(sum[:4])
|
|
}
|
|
|
|
// unflat answers a chat call by the namespace and name its client declared.
|
|
func (r *OpenAIResponsesRequest) unflat(call *responsesToolCallState) {
|
|
if r == nil {
|
|
return
|
|
}
|
|
if s, ok := r.spaces[call.Name]; ok {
|
|
call.Namespace, call.Name = s.Namespace, s.Name
|
|
}
|
|
}
|
|
|
|
type responsesTool struct {
|
|
Type string `json:"type"`
|
|
Name string `json:"name,omitempty"`
|
|
Description string `json:"description,omitempty"`
|
|
Parameters json.RawMessage `json:"parameters,omitempty"`
|
|
Strict bool `json:"strict,omitempty"`
|
|
Format json.RawMessage `json:"format,omitempty"`
|
|
// Tools are what a `namespace` holds.
|
|
Tools []responsesTool `json:"tools,omitempty"`
|
|
}
|
|
|
|
type responsesInputItem struct {
|
|
Type string `json:"type"`
|
|
ID string `json:"id,omitempty"`
|
|
Role string `json:"role,omitempty"`
|
|
Content json.RawMessage `json:"content,omitempty"`
|
|
CallID string `json:"call_id,omitempty"`
|
|
Name string `json:"name,omitempty"`
|
|
Namespace string `json:"namespace,omitempty"`
|
|
Arguments string `json:"arguments,omitempty"`
|
|
Input string `json:"input,omitempty"`
|
|
Output json.RawMessage `json:"output,omitempty"`
|
|
}
|
|
|
|
type responsesContentPart struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text,omitempty"`
|
|
ImageURL string `json:"image_url,omitempty"`
|
|
}
|
|
|
|
// Responses implements POST /v1/responses. The converted request is completed by
|
|
// the chat path, which is handed a sink saying where the answer goes: a stream is
|
|
// translated as it is produced, a whole body is translated entire.
|
|
func (c *ApiController) Responses() {
|
|
authHeader := c.Header("Authorization")
|
|
if !strings.HasPrefix(authHeader, "Bearer ") {
|
|
c.ResponseErrorWithStatus(http.StatusUnauthorized, "Invalid API key format. Expected 'Bearer API_KEY'")
|
|
return
|
|
}
|
|
token := strings.TrimPrefix(authHeader, "Bearer ")
|
|
if isPublishableKey(token) {
|
|
c.ResponseErrorWithStatus(http.StatusForbidden, "Publishable keys cannot access /v1/responses. Use a secret key.")
|
|
return
|
|
}
|
|
|
|
// A Responses request is served as a chat completion, which is a translation, so
|
|
// one that asked to be served unchanged is refused — after its credential is read,
|
|
// like every other refusal here.
|
|
if c.strictAsked() {
|
|
if authErr := c.authenticate(token); authErr != nil {
|
|
c.ResponseAuthError(authErr)
|
|
return
|
|
}
|
|
c.refuseStrict("translation", "/v1/responses is served as a chat completion; send the request to /v1/chat/completions")
|
|
return
|
|
}
|
|
|
|
// c.Body() is the body the server already decoded (zstd included, within its body
|
|
// limit), so it is read as plain whatever Content-Encoding the caller sent.
|
|
call, err := ReadResponses(c.Body(), "")
|
|
if err != nil {
|
|
// Authenticate before reporting: whatever the body says, a caller without
|
|
// a valid key is told that first, and learns nothing about how we read it.
|
|
if authErr := c.authenticate(token); authErr != nil {
|
|
c.ResponseAuthError(authErr)
|
|
return
|
|
}
|
|
// A body over the bound is 413, not 400: it is not malformed, it is too
|
|
// big, and only one of those tells the caller to send less.
|
|
if errors.Is(err, zstd.ErrDecoderSizeExceeded) {
|
|
c.ResponseErrorWithStatus(http.StatusRequestEntityTooLarge, fmt.Sprintf(
|
|
"request body exceeds the %d MiB limit", MaxDecoded>>20))
|
|
return
|
|
}
|
|
c.ResponseErrorWithStatus(http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
|
|
// The chat request on the wire, because the completion reads its own body. This
|
|
// is the request being made, not a place being patched.
|
|
req := c.Fiber().Request()
|
|
req.Header.Del("Content-Encoding")
|
|
req.SetBody(call.Chat)
|
|
req.Header.Del("Content-Length")
|
|
|
|
// This endpoint IS a chat completion, read in a second dialect. So it asks for
|
|
// one and says what shape the answer must take, rather than swapping the writer
|
|
// out of the request context and reading whatever came back through it.
|
|
c.chatCompletions(callerBearer, c.responsesAnswer(call))
|
|
}
|
|
|
|
// ResponsesCall is a /v1/responses request read as the chat completion it asks
|
|
// for. Chat is that completion's body; Stream and Whole write its answer back in
|
|
// the Responses dialect. It is the one translation: ai serves Chat itself, and a
|
|
// host that serves chat in-process (cloud's zen) serves it there and answers
|
|
// through the same two functions.
|
|
type ResponsesCall struct {
|
|
Chat []byte
|
|
req OpenAIResponsesRequest
|
|
kinds map[string]string
|
|
}
|
|
|
|
// ReadResponses reads a Responses body, zstd-compressed when encoding says so.
|
|
// Every error is the caller's; one for a body too large to expand wraps
|
|
// zstd.ErrDecoderSizeExceeded.
|
|
func ReadResponses(body []byte, encoding string) (*ResponsesCall, error) {
|
|
if strings.EqualFold(strings.TrimSpace(encoding), "zstd") {
|
|
decoded, err := decodeResponsesZstd(body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Failed to decompress zstd request: %w", err)
|
|
}
|
|
body = decoded
|
|
}
|
|
call := &ResponsesCall{}
|
|
if err := json.Unmarshal(body, &call.req); err != nil {
|
|
return nil, fmt.Errorf("Failed to parse Responses request: %w", err)
|
|
}
|
|
if call.req.Model == "" {
|
|
return nil, errors.New("model is required")
|
|
}
|
|
chat, kinds, err := responsesToChatRequest(&call.req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if call.Chat, err = json.Marshal(chat); err != nil {
|
|
return nil, err
|
|
}
|
|
if call.Chat, err = carry(call.Chat, body); err != nil {
|
|
return nil, err
|
|
}
|
|
call.kinds = kinds
|
|
return call, nil
|
|
}
|
|
|
|
// carry puts into the chat body what go-openai's struct cannot hold, read from the
|
|
// Responses body as the caller wrote it: a temperature or top_p of zero (omitempty
|
|
// drops both, so a caller who asked for greedy sampling got the vendor's default),
|
|
// the prompt cache key (chat's own field of that name, which keeps a conversation's
|
|
// turns on the cache that holds its prefix), the structured-output format
|
|
// (text.format, chat's response_format) and the reasoning effort.
|
|
//
|
|
// previous_response_id asks for a stored conversation, and this gateway stores none.
|
|
// It is refused rather than answered: an answer without the conversation it named
|
|
// would read as one that had it.
|
|
func carry(chat, body []byte) ([]byte, error) {
|
|
var in struct {
|
|
Previous string `json:"previous_response_id"`
|
|
CacheKey json.RawMessage `json:"prompt_cache_key"`
|
|
Temperature json.RawMessage `json:"temperature"`
|
|
TopP json.RawMessage `json:"top_p"`
|
|
Text struct {
|
|
Format json.RawMessage `json:"format"`
|
|
} `json:"text"`
|
|
Reasoning struct {
|
|
Effort string `json:"effort"`
|
|
} `json:"reasoning"`
|
|
}
|
|
if err := json.Unmarshal(body, &in); err != nil {
|
|
return nil, fmt.Errorf("text and reasoning must be objects: %w", err)
|
|
}
|
|
if f := unpricedField(body); f != "" {
|
|
return nil, fmt.Errorf("%q is not accepted: it would buy something this model's price does not cover", f)
|
|
}
|
|
if in.Previous != "" {
|
|
return nil, errors.New("previous_response_id is not supported: no response is stored here, so send the whole conversation in input")
|
|
}
|
|
var out map[string]json.RawMessage
|
|
if err := json.Unmarshal(chat, &out); err != nil {
|
|
return nil, err
|
|
}
|
|
for k, v := range map[string]json.RawMessage{"temperature": in.Temperature, "top_p": in.TopP, "prompt_cache_key": in.CacheKey} {
|
|
if len(v) > 0 && string(v) != "null" {
|
|
out[k] = v
|
|
}
|
|
}
|
|
format, err := chatFormat(in.Text.Format)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if format != nil {
|
|
out["response_format"] = format
|
|
}
|
|
if in.Reasoning.Effort != "" {
|
|
out["reasoning_effort"], _ = json.Marshal(in.Reasoning.Effort)
|
|
}
|
|
return json.Marshal(out)
|
|
}
|
|
|
|
// chatFormat is a Responses text.format in chat's response_format shape, or nil for
|
|
// plain text. The Responses form holds the schema's fields beside its type; chat's
|
|
// holds them under json_schema.
|
|
func chatFormat(raw json.RawMessage) (json.RawMessage, error) {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return nil, nil
|
|
}
|
|
var f map[string]json.RawMessage
|
|
if err := json.Unmarshal(raw, &f); err != nil {
|
|
return nil, fmt.Errorf("text.format: %w", err)
|
|
}
|
|
var kind string
|
|
_ = json.Unmarshal(f["type"], &kind)
|
|
switch kind {
|
|
case "", "text":
|
|
return nil, nil
|
|
case "json_object":
|
|
return json.RawMessage(`{"type":"json_object"}`), nil
|
|
case "json_schema":
|
|
delete(f, "type")
|
|
schema, err := json.Marshal(f)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(map[string]json.RawMessage{"type": json.RawMessage(`"json_schema"`), "json_schema": schema})
|
|
}
|
|
return nil, fmt.Errorf("text.format type %q is not supported", kind)
|
|
}
|
|
|
|
// Stream writes a chat SSE stream to w as Responses events, as it is produced.
|
|
func (r *ResponsesCall) Stream(w io.Writer) io.WriteCloser {
|
|
return newResponsesBridge(w, &r.req, r.kinds)
|
|
}
|
|
|
|
// Whole translates one finished chat completion into a Responses object.
|
|
func (r *ResponsesCall) Whole(chat []byte) ([]byte, error) {
|
|
return openAIChatResponseToResponses(chat, &r.req, r.kinds)
|
|
}
|
|
|
|
// responsesAnswer is the Responses shape of a chat completion: a stream is
|
|
// decorated as it is produced, a whole body is translated entire.
|
|
func (c *ApiController) responsesAnswer(call *ResponsesCall) *sink {
|
|
return &sink{
|
|
wrap: func(w io.Writer) io.Writer { return call.Stream(w) },
|
|
whole: call.Whole,
|
|
}
|
|
}
|
|
|
|
// decodeResponsesZstd expands a zstd-framed Responses body, bounded by what it
|
|
// expands to. It uses a pure-Go decoder because Cloud release binaries are built
|
|
// with CGO_ENABLED=0. Current Codex clients compress large requests with zstd.
|
|
//
|
|
// The bound is MaxDecoded, the same ceiling every other body gets, and it has
|
|
// to be stated: the decoder's own default is 64 GiB. Capping the COMPRESSED
|
|
// input caps nothing that costs memory here — zstd's ratio on repetitive input
|
|
// runs to thousands to one, so the 64 MiB that reaches this function is already
|
|
// enough to ask for more heap than any pod has. The decode also runs before the
|
|
// token is validated, so whatever it allocates, a caller holding nothing but a
|
|
// syntactically-valid Bearer string can make it allocate.
|
|
//
|
|
// Overflow returns zstd.ErrDecoderSizeExceeded, which callers answer 413 —
|
|
// naming the limit, rather than letting a half-expanded body fail later as an
|
|
// unintelligible parse error.
|
|
func decodeResponsesZstd(body []byte) ([]byte, error) {
|
|
decoder, err := zstd.NewReader(nil, zstd.WithDecoderMaxMemory(uint64(MaxDecoded)))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer decoder.Close()
|
|
return decoder.DecodeAll(body, nil)
|
|
}
|
|
|
|
func responsesToChatRequest(request *OpenAIResponsesRequest) (*openai.ChatCompletionRequest, map[string]string, error) {
|
|
messages, err := responsesInputToMessages(request.Instructions, request.Input)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if len(messages) == 0 {
|
|
return nil, nil, fmt.Errorf("input must contain at least one message")
|
|
}
|
|
|
|
tools := make([]openai.Tool, 0, len(request.Tools))
|
|
toolKinds := make(map[string]string, len(request.Tools))
|
|
// A namespace is not a function: each tool it holds is one, named flat.
|
|
var add func(tool responsesTool, namespace string)
|
|
add = func(tool responsesTool, namespace string) {
|
|
if tool.Name == "" {
|
|
return
|
|
}
|
|
kind := tool.Type
|
|
if kind == "" {
|
|
kind = "function"
|
|
}
|
|
if kind == "namespace" {
|
|
for _, inner := range tool.Tools {
|
|
add(inner, tool.Name)
|
|
}
|
|
return
|
|
}
|
|
name := tool.Name
|
|
if namespace != "" {
|
|
name = flatName(namespace, tool.Name)
|
|
if request.spaces == nil {
|
|
request.spaces = map[string]spaced{}
|
|
}
|
|
request.spaces[name] = spaced{Namespace: namespace, Name: tool.Name}
|
|
}
|
|
toolKinds[name] = kind
|
|
params := any(json.RawMessage(tool.Parameters))
|
|
if len(tool.Parameters) == 0 || string(tool.Parameters) == "null" {
|
|
if kind == "custom" {
|
|
params = map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"input": map[string]any{"type": "string"},
|
|
},
|
|
"required": []string{"input"},
|
|
}
|
|
} else {
|
|
params = map[string]any{"type": "object", "properties": map[string]any{}}
|
|
}
|
|
}
|
|
tools = append(tools, openai.Tool{
|
|
Type: openai.ToolTypeFunction,
|
|
Function: &openai.FunctionDefinition{
|
|
Name: name, Description: tool.Description, Parameters: params, Strict: tool.Strict,
|
|
},
|
|
})
|
|
}
|
|
for _, tool := range request.Tools {
|
|
add(tool, "")
|
|
}
|
|
|
|
chat := &openai.ChatCompletionRequest{
|
|
Model: request.Model,
|
|
Messages: messages,
|
|
Tools: tools,
|
|
ToolChoice: responsesToolChoiceToChat(request.ToolChoice),
|
|
Stream: request.Stream,
|
|
MaxCompletionTokens: request.MaxOutputTokens,
|
|
Store: request.Store,
|
|
Metadata: request.Metadata,
|
|
}
|
|
if request.Stream {
|
|
// The existing ChatCompletions proxy only forwards its forced upstream
|
|
// usage chunk when the downstream client opted in. The Responses bridge
|
|
// is that client, and response.completed must contain real token counts.
|
|
chat.StreamOptions = &openai.StreamOptions{IncludeUsage: true}
|
|
}
|
|
if request.MaxOutputTokens > 0 {
|
|
// DO's OpenAI-compatible models currently honor max_tokens uniformly.
|
|
chat.MaxTokens = request.MaxOutputTokens
|
|
chat.MaxCompletionTokens = 0
|
|
}
|
|
if request.Temperature != nil {
|
|
chat.Temperature = *request.Temperature
|
|
}
|
|
if request.TopP != nil {
|
|
chat.TopP = *request.TopP
|
|
}
|
|
if request.ParallelToolCalls != nil {
|
|
chat.ParallelToolCalls = *request.ParallelToolCalls
|
|
}
|
|
return chat, toolKinds, nil
|
|
}
|
|
|
|
func responsesInputToMessages(instructions string, raw json.RawMessage) ([]openai.ChatCompletionMessage, error) {
|
|
messages := make([]openai.ChatCompletionMessage, 0, 8)
|
|
if instructions != "" {
|
|
messages = append(messages, openai.ChatCompletionMessage{Role: openai.ChatMessageRoleSystem, Content: instructions})
|
|
}
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return messages, nil
|
|
}
|
|
var text string
|
|
if err := json.Unmarshal(raw, &text); err == nil {
|
|
return append(messages, openai.ChatCompletionMessage{Role: openai.ChatMessageRoleUser, Content: text}), nil
|
|
}
|
|
|
|
var items []responsesInputItem
|
|
if err := json.Unmarshal(raw, &items); err != nil {
|
|
return nil, fmt.Errorf("input must be a string or an array of Responses input items: %w", err)
|
|
}
|
|
|
|
var pendingCalls []openai.ToolCall
|
|
flushCalls := func() {
|
|
if len(pendingCalls) == 0 {
|
|
return
|
|
}
|
|
messages = append(messages, openai.ChatCompletionMessage{Role: openai.ChatMessageRoleAssistant, ToolCalls: pendingCalls})
|
|
pendingCalls = nil
|
|
}
|
|
|
|
for _, item := range items {
|
|
switch item.Type {
|
|
case "function_call", "custom_tool_call", "local_shell_call":
|
|
callID := item.CallID
|
|
if callID == "" {
|
|
callID = item.ID
|
|
}
|
|
arguments := item.Arguments
|
|
if item.Type == "custom_tool_call" && arguments == "" {
|
|
encoded, _ := json.Marshal(map[string]string{"input": item.Input})
|
|
arguments = string(encoded)
|
|
}
|
|
name := item.Name
|
|
if item.Namespace != "" {
|
|
name = flatName(item.Namespace, item.Name)
|
|
}
|
|
pendingCalls = append(pendingCalls, openai.ToolCall{
|
|
ID: callID, Type: openai.ToolTypeFunction,
|
|
Function: openai.FunctionCall{Name: name, Arguments: arguments},
|
|
})
|
|
case "function_call_output", "custom_tool_call_output", "local_shell_call_output":
|
|
flushCalls()
|
|
messages = append(messages, openai.ChatCompletionMessage{
|
|
Role: openai.ChatMessageRoleTool, ToolCallID: item.CallID, Content: responsesOutputText(item.Output),
|
|
})
|
|
case "message", "":
|
|
flushCalls()
|
|
role := item.Role
|
|
if role == "" {
|
|
role = openai.ChatMessageRoleUser
|
|
}
|
|
msg := openai.ChatCompletionMessage{Role: role}
|
|
parts := responsesContentParts(item.Content)
|
|
for _, part := range parts {
|
|
switch part.Type {
|
|
case "input_image":
|
|
msg.MultiContent = append(msg.MultiContent, openai.ChatMessagePart{
|
|
Type: openai.ChatMessagePartTypeImageURL,
|
|
ImageURL: &openai.ChatMessageImageURL{URL: part.ImageURL},
|
|
})
|
|
default:
|
|
if part.Text != "" {
|
|
msg.Content += part.Text
|
|
msg.MultiContent = append(msg.MultiContent, openai.ChatMessagePart{Type: openai.ChatMessagePartTypeText, Text: part.Text})
|
|
}
|
|
}
|
|
}
|
|
// go-openai marshals exactly one of Content and MultiContent. A
|
|
// message with an image keeps the parts and drops the joined text;
|
|
// text-only messages go out as a plain string.
|
|
if len(msg.MultiContent) > 0 {
|
|
hasImage := false
|
|
for _, part := range msg.MultiContent {
|
|
hasImage = hasImage || part.Type == openai.ChatMessagePartTypeImageURL
|
|
}
|
|
if hasImage {
|
|
msg.Content = ""
|
|
} else {
|
|
msg.MultiContent = nil
|
|
}
|
|
}
|
|
messages = append(messages, msg)
|
|
case "reasoning":
|
|
// Encrypted/reasoning items are provider-specific state. The serving
|
|
// model receives the visible conversation and continues normally.
|
|
default:
|
|
// Forward-compatible: ignore server-executed Responses item types.
|
|
}
|
|
}
|
|
flushCalls()
|
|
return messages, nil
|
|
}
|
|
|
|
func responsesContentParts(raw json.RawMessage) []responsesContentPart {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return nil
|
|
}
|
|
var text string
|
|
if json.Unmarshal(raw, &text) == nil {
|
|
return []responsesContentPart{{Type: "input_text", Text: text}}
|
|
}
|
|
var parts []responsesContentPart
|
|
if json.Unmarshal(raw, &parts) == nil {
|
|
return parts
|
|
}
|
|
return []responsesContentPart{{Type: "input_text", Text: string(raw)}}
|
|
}
|
|
|
|
func responsesOutputText(raw json.RawMessage) string {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return ""
|
|
}
|
|
var text string
|
|
if json.Unmarshal(raw, &text) == nil {
|
|
return text
|
|
}
|
|
parts := responsesContentParts(raw)
|
|
var out strings.Builder
|
|
for _, part := range parts {
|
|
out.WriteString(part.Text)
|
|
}
|
|
if out.Len() > 0 {
|
|
return out.String()
|
|
}
|
|
return string(raw)
|
|
}
|
|
|
|
func responsesToolChoiceToChat(raw json.RawMessage) any {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return nil
|
|
}
|
|
var choice string
|
|
if json.Unmarshal(raw, &choice) == nil {
|
|
return choice
|
|
}
|
|
var choiceObj struct {
|
|
Type string `json:"type"`
|
|
Name string `json:"name"`
|
|
}
|
|
if json.Unmarshal(raw, &choiceObj) == nil && choiceObj.Name != "" {
|
|
return map[string]any{
|
|
"type": "function", "function": map[string]string{"name": choiceObj.Name},
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// responsesBridge reads the chat completion this request was translated into and
|
|
// writes it back out in the Responses dialect.
|
|
//
|
|
// It is handed BYTES, and that is the whole shape of it. The chat side used to
|
|
// write through it as an http.ResponseWriter — status included — so it had to
|
|
// intercept WriteHeader, withhold a success it was not ready to send, keep a
|
|
// buffer of a body it meant to rewrite, and remember which of those it had done.
|
|
// Every one of those existed to separate a body from a status arriving on the same
|
|
// call. It now receives the two things it actually wants, separately: the stream's
|
|
// chunks as they are produced, and — through the sink in openai_api.go — the
|
|
// non-streaming body as one slice. The status is the handler's, and stays there.
|
|
type responsesBridge struct {
|
|
out io.Writer
|
|
request *OpenAIResponsesRequest
|
|
toolKinds map[string]string
|
|
lineBuf bytes.Buffer
|
|
translator *responsesStreamTranslator
|
|
mu sync.Mutex
|
|
closed bool
|
|
}
|
|
|
|
func newResponsesBridge(out io.Writer, request *OpenAIResponsesRequest, toolKinds map[string]string) *responsesBridge {
|
|
return &responsesBridge{out: out, request: request, toolKinds: toolKinds}
|
|
}
|
|
|
|
// Write takes the chat stream's SSE as it is produced and emits the Responses
|
|
// events for it. Chunks arrive on no particular boundary, so a partial last line
|
|
// is kept for the next call.
|
|
func (w *responsesBridge) Write(p []byte) (int, error) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if w.translator == nil {
|
|
w.translator = newResponsesStreamTranslator(w.emitLocked, w.request, w.toolKinds)
|
|
if err := w.translator.start(); err != nil {
|
|
return len(p), err
|
|
}
|
|
}
|
|
_, _ = w.lineBuf.Write(p)
|
|
for {
|
|
line, err := w.lineBuf.ReadString('\n')
|
|
if err != nil {
|
|
// ReadString consumed the partial line; put it back until the next write.
|
|
w.lineBuf.Reset()
|
|
_, _ = w.lineBuf.WriteString(line)
|
|
break
|
|
}
|
|
line = strings.TrimSpace(line)
|
|
if !strings.HasPrefix(line, "data:") {
|
|
continue
|
|
}
|
|
data := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
|
|
if data == "" {
|
|
continue
|
|
}
|
|
if data == "[DONE]" {
|
|
if err := w.translator.finish(); err != nil {
|
|
return len(p), err
|
|
}
|
|
continue
|
|
}
|
|
var chunk openaiStreamChunk
|
|
if err := json.Unmarshal([]byte(data), &chunk); err != nil {
|
|
continue
|
|
}
|
|
if err := w.translator.handleChunk(&chunk); err != nil {
|
|
return len(p), err
|
|
}
|
|
}
|
|
return len(p), nil
|
|
}
|
|
|
|
// Flush pushes each event as it is produced, which is what makes this a stream.
|
|
// Two shapes, for the same reason OpenAIWriter takes both.
|
|
func (w *responsesBridge) Flush() {
|
|
switch f := w.out.(type) {
|
|
case interface{ Flush() error }:
|
|
_ = f.Flush()
|
|
case interface{ Flush() }:
|
|
f.Flush()
|
|
}
|
|
}
|
|
|
|
// Close ends the event stream. Idempotent: the chat side sends [DONE], which
|
|
// finishes the translator, and the handler closes again on the way out.
|
|
func (w *responsesBridge) Close() error {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if w.closed {
|
|
return nil
|
|
}
|
|
w.closed = true
|
|
if w.translator == nil {
|
|
return nil
|
|
}
|
|
return w.translator.finish()
|
|
}
|
|
|
|
func (w *responsesBridge) emitLocked(event string, data any) error {
|
|
b, err := json.Marshal(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err = fmt.Fprintf(w.out, "event: %s\ndata: %s\n\n", event, b); err != nil {
|
|
return err
|
|
}
|
|
w.Flush()
|
|
return nil
|
|
}
|
|
|
|
type responsesToolCallState struct {
|
|
Index int
|
|
OutputIndex int
|
|
ID string
|
|
CallID string
|
|
Name string
|
|
Namespace string
|
|
Arguments strings.Builder
|
|
Kind string
|
|
Started bool
|
|
}
|
|
|
|
type responsesStreamTranslator struct {
|
|
emit func(string, any) error
|
|
request *OpenAIResponsesRequest
|
|
toolKinds map[string]string
|
|
responseID string
|
|
messageID string
|
|
createdAt int64
|
|
sequence int
|
|
started bool
|
|
finished bool
|
|
textStarted bool
|
|
textOutputIndex int
|
|
text strings.Builder
|
|
calls map[int]*responsesToolCallState
|
|
callOrder []int
|
|
nextOutput int
|
|
promptTokens int
|
|
cachedTokens int
|
|
writtenTokens int
|
|
completionTokens int
|
|
totalTokens int
|
|
sawFinishReason bool
|
|
sawUsage bool
|
|
}
|
|
|
|
func newResponsesStreamTranslator(emit func(string, any) error, request *OpenAIResponsesRequest, toolKinds map[string]string) *responsesStreamTranslator {
|
|
return &responsesStreamTranslator{
|
|
emit: emit, request: request, toolKinds: toolKinds,
|
|
responseID: "resp_" + uuid.NewString(), messageID: "msg_" + uuid.NewString(),
|
|
createdAt: time.Now().Unix(), calls: map[int]*responsesToolCallState{},
|
|
}
|
|
}
|
|
|
|
func (t *responsesStreamTranslator) next() int { n := t.sequence; t.sequence++; return n }
|
|
|
|
func (t *responsesStreamTranslator) start() error {
|
|
if t.started {
|
|
return nil
|
|
}
|
|
t.started = true
|
|
return t.emit("response.created", map[string]any{
|
|
"type": "response.created", "sequence_number": t.next(),
|
|
"response": t.resource("in_progress", []any{}),
|
|
})
|
|
}
|
|
|
|
func (t *responsesStreamTranslator) handleChunk(chunk *openaiStreamChunk) error {
|
|
// Nothing follows response.completed: an event after it is one the client
|
|
// cannot place.
|
|
if t.finished {
|
|
return nil
|
|
}
|
|
if err := t.start(); err != nil {
|
|
return err
|
|
}
|
|
if chunk.Usage != nil {
|
|
t.count(chunk.Usage)
|
|
t.sawUsage = true
|
|
if t.sawFinishReason {
|
|
return t.finish()
|
|
}
|
|
}
|
|
for i := range chunk.Choices {
|
|
choice := &chunk.Choices[i]
|
|
if choice.Delta.Content != "" {
|
|
if !t.textStarted {
|
|
t.textStarted = true
|
|
outputIndex := t.nextOutput
|
|
t.textOutputIndex = outputIndex
|
|
t.nextOutput++
|
|
if err := t.emit("response.output_item.added", map[string]any{
|
|
"type": "response.output_item.added", "sequence_number": t.next(), "output_index": outputIndex,
|
|
"item": map[string]any{"id": t.messageID, "type": "message", "role": "assistant", "status": "in_progress", "content": []any{}},
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if err := t.emit("response.content_part.added", map[string]any{
|
|
"type": "response.content_part.added", "sequence_number": t.next(), "output_index": outputIndex, "content_index": 0,
|
|
"part": map[string]any{"type": "output_text", "text": "", "annotations": []any{}},
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
t.text.WriteString(choice.Delta.Content)
|
|
if err := t.emit("response.output_text.delta", map[string]any{
|
|
"type": "response.output_text.delta", "sequence_number": t.next(), "item_id": t.messageID,
|
|
"output_index": t.textOutputIndex, "content_index": 0, "delta": choice.Delta.Content,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
for j := range choice.Delta.ToolCalls {
|
|
tc := &choice.Delta.ToolCalls[j]
|
|
idx := j
|
|
if tc.Index != nil {
|
|
idx = *tc.Index
|
|
}
|
|
call := t.calls[idx]
|
|
if call == nil {
|
|
call = &responsesToolCallState{Index: idx, OutputIndex: t.nextOutput, Kind: "function"}
|
|
t.nextOutput++
|
|
t.calls[idx] = call
|
|
t.callOrder = append(t.callOrder, idx)
|
|
}
|
|
if tc.ID != "" {
|
|
call.ID, call.CallID = tc.ID, tc.ID
|
|
}
|
|
if tc.Function.Name != "" {
|
|
call.Name = tc.Function.Name
|
|
if kind := t.toolKinds[call.Name]; kind != "" {
|
|
call.Kind = kind
|
|
}
|
|
t.request.unflat(call)
|
|
}
|
|
if !call.Started && (call.CallID != "" || call.Name != "") {
|
|
call.Started = true
|
|
if call.ID == "" {
|
|
call.ID = "fc_" + uuid.NewString()
|
|
}
|
|
if call.CallID == "" {
|
|
call.CallID = call.ID
|
|
}
|
|
if err := t.emit("response.output_item.added", map[string]any{
|
|
"type": "response.output_item.added", "sequence_number": t.next(), "output_index": call.OutputIndex,
|
|
"item": call.item("in_progress", ""),
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if tc.Function.Arguments != "" {
|
|
call.Arguments.WriteString(tc.Function.Arguments)
|
|
event := "response.function_call_arguments.delta"
|
|
if call.Kind == "custom" {
|
|
event = "response.custom_tool_call_input.delta"
|
|
}
|
|
if err := t.emit(event, map[string]any{
|
|
"type": event, "sequence_number": t.next(), "item_id": call.ID,
|
|
"output_index": call.OutputIndex, "delta": tc.Function.Arguments,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
// The end of the answer, in whichever order the two signals arrive.
|
|
//
|
|
// finish_reason alone is not enough: providers commonly send the
|
|
// usage-only chunk *after* the terminal choice chunk, and finishing on
|
|
// finish_reason alone published response.completed with zero usage and
|
|
// made that accounting chunk impossible to apply.
|
|
//
|
|
// Waiting only for [DONE] is not enough either, and that is what broke
|
|
// clients: an OpenAI-compatible provider is free to end the stream
|
|
// without one. When that happened the terminal events were never sent —
|
|
// response.output_text.done, .content_part.done, .output_item.done and
|
|
// .completed — so every delta arrived and the response never closed.
|
|
// A client reading the Responses stream to spec waits forever, retries,
|
|
// and reports the stream as truncated.
|
|
//
|
|
// So: whichever of the two arrives second finishes the response, and
|
|
// the bridge Close stays the backstop for a stream that sends neither.
|
|
if choice.FinishReason != "" {
|
|
t.sawFinishReason = true
|
|
if t.sawUsage {
|
|
return t.finish()
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *responsesToolCallState) finalArguments() string {
|
|
args := c.Arguments.String()
|
|
if c.Kind != "custom" {
|
|
return args
|
|
}
|
|
var wrapped struct {
|
|
Input string `json:"input"`
|
|
}
|
|
if json.Unmarshal([]byte(args), &wrapped) == nil && wrapped.Input != "" {
|
|
return wrapped.Input
|
|
}
|
|
return args
|
|
}
|
|
|
|
func (c *responsesToolCallState) item(status, arguments string) map[string]any {
|
|
item := map[string]any{
|
|
"id": c.ID, "type": "function_call", "call_id": c.CallID,
|
|
"name": c.Name, "arguments": arguments, "status": status,
|
|
}
|
|
if c.Kind == "custom" {
|
|
item = map[string]any{
|
|
"id": c.ID, "type": "custom_tool_call", "call_id": c.CallID,
|
|
"name": c.Name, "input": arguments, "status": status,
|
|
}
|
|
}
|
|
if c.Namespace != "" {
|
|
item["namespace"] = c.Namespace
|
|
}
|
|
return item
|
|
}
|
|
|
|
func (t *responsesStreamTranslator) finish() error {
|
|
if t.finished {
|
|
return nil
|
|
}
|
|
if err := t.start(); err != nil {
|
|
return err
|
|
}
|
|
t.finished = true
|
|
outputs := make([]any, 0, 1+len(t.calls))
|
|
if t.textStarted {
|
|
text := t.text.String()
|
|
message := map[string]any{
|
|
"id": t.messageID, "type": "message", "role": "assistant", "status": "completed",
|
|
"content": []any{map[string]any{"type": "output_text", "text": text, "annotations": []any{}}},
|
|
}
|
|
if err := t.emit("response.output_text.done", map[string]any{
|
|
"type": "response.output_text.done", "sequence_number": t.next(), "item_id": t.messageID,
|
|
"output_index": t.textOutputIndex, "content_index": 0, "text": text,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if err := t.emit("response.content_part.done", map[string]any{
|
|
"type": "response.content_part.done", "sequence_number": t.next(), "output_index": t.textOutputIndex, "content_index": 0,
|
|
"part": message["content"].([]any)[0],
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if err := t.emit("response.output_item.done", map[string]any{
|
|
"type": "response.output_item.done", "sequence_number": t.next(), "output_index": t.textOutputIndex, "item": message,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
outputs = append(outputs, message)
|
|
}
|
|
for _, idx := range t.callOrder {
|
|
call := t.calls[idx]
|
|
args := call.finalArguments()
|
|
if !call.Started {
|
|
call.Started = true
|
|
if call.ID == "" {
|
|
call.ID = "fc_" + uuid.NewString()
|
|
}
|
|
if call.CallID == "" {
|
|
call.CallID = call.ID
|
|
}
|
|
}
|
|
if call.Kind != "custom" {
|
|
if err := t.emit("response.function_call_arguments.done", map[string]any{
|
|
"type": "response.function_call_arguments.done", "sequence_number": t.next(), "item_id": call.ID,
|
|
"output_index": call.OutputIndex, "arguments": args,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
item := call.item("completed", args)
|
|
if err := t.emit("response.output_item.done", map[string]any{
|
|
"type": "response.output_item.done", "sequence_number": t.next(), "output_index": call.OutputIndex, "item": item,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
outputs = append(outputs, item)
|
|
}
|
|
return t.emit("response.completed", map[string]any{
|
|
"type": "response.completed", "sequence_number": t.next(),
|
|
"response": t.resource("completed", outputs),
|
|
})
|
|
}
|
|
|
|
// count takes the usage a chunk or a whole answer reports. A count it leaves out
|
|
// keeps what an earlier chunk said: a usage chunk need not repeat every part.
|
|
func (t *responsesStreamTranslator) count(u *chatUsage) {
|
|
if u.PromptTokens > 0 {
|
|
t.promptTokens = u.PromptTokens
|
|
}
|
|
if u.CompletionTokens > 0 {
|
|
t.completionTokens = u.CompletionTokens
|
|
}
|
|
if u.TotalTokens > 0 {
|
|
t.totalTokens = u.TotalTokens
|
|
}
|
|
if d := u.PromptDetails; d != nil {
|
|
if d.Cached > 0 {
|
|
t.cachedTokens = d.Cached
|
|
}
|
|
if d.Written > 0 {
|
|
t.writtenTokens = d.Written
|
|
}
|
|
}
|
|
}
|
|
|
|
func (t *responsesStreamTranslator) resource(status string, output []any) responsesResource {
|
|
total := t.totalTokens
|
|
if total == 0 {
|
|
total = t.promptTokens + t.completionTokens
|
|
}
|
|
// Both are parts of the prompt, and never more than it.
|
|
cached := min(t.cachedTokens, t.promptTokens)
|
|
written := min(t.writtenTokens, t.promptTokens-cached)
|
|
return responsesResource{
|
|
ID: t.responseID, Object: "response", CreatedAt: t.createdAt, Status: status,
|
|
Model: t.request.Model, Output: output,
|
|
Instructions: t.request.Instructions, MaxOutputTokens: t.request.MaxOutputTokens,
|
|
Metadata: t.request.Metadata, ParallelToolCalls: t.request.ParallelToolCalls,
|
|
Reasoning: jsonOrNil(t.request.Reasoning),
|
|
Store: t.request.Store, Temperature: t.request.Temperature,
|
|
Text: jsonOrNil(t.request.Text), ToolChoice: jsonOrNil(t.request.ToolChoice),
|
|
Tools: t.request.Tools, TopP: t.request.TopP,
|
|
Usage: responsesUsage{
|
|
InputTokens: t.promptTokens,
|
|
InputDetails: responsesInputDetails{CachedTokens: cached, CacheWriteTokens: written},
|
|
OutputTokens: t.completionTokens,
|
|
OutputDetails: responsesOutputDetails{},
|
|
TotalTokens: total,
|
|
},
|
|
}
|
|
}
|
|
|
|
// responsesResource is one answer from /v1/responses.
|
|
//
|
|
// Error, IncompleteDetails and PreviousResponseID are `any` and carry no
|
|
// omitempty because the OpenAI Responses API sends them as an explicit null
|
|
// rather than omitting them, which is what the map this replaced did by setting
|
|
// the key to nil.
|
|
//
|
|
// Output is the one field with no shape to state: it is a union — a message, a
|
|
// tool call, a reasoning item — and each element says which it is in its own
|
|
// `type`. An empty schema is the honest answer for a union; a guessed one would
|
|
// send every client down the wrong branch.
|
|
type responsesResource struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
CreatedAt int64 `json:"created_at"`
|
|
Status string `json:"status"`
|
|
Model string `json:"model"`
|
|
Output []any `json:"output"`
|
|
Error any `json:"error"`
|
|
IncompleteDetails any `json:"incomplete_details"`
|
|
Instructions string `json:"instructions"`
|
|
MaxOutputTokens int `json:"max_output_tokens"`
|
|
Metadata map[string]string `json:"metadata"`
|
|
ParallelToolCalls *bool `json:"parallel_tool_calls"`
|
|
PreviousResponseID any `json:"previous_response_id"`
|
|
Reasoning any `json:"reasoning"`
|
|
Store bool `json:"store"`
|
|
Temperature *float32 `json:"temperature"`
|
|
Text any `json:"text"`
|
|
ToolChoice any `json:"tool_choice"`
|
|
Tools []responsesTool `json:"tools"`
|
|
TopP *float32 `json:"top_p"`
|
|
Usage responsesUsage `json:"usage"`
|
|
}
|
|
|
|
// responsesUsage is what the answer cost.
|
|
type responsesUsage struct {
|
|
InputTokens int `json:"input_tokens"`
|
|
InputDetails responsesInputDetails `json:"input_tokens_details"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
OutputDetails responsesOutputDetails `json:"output_tokens_details"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
}
|
|
|
|
// responsesInputDetails is the part of input_tokens the upstream's prompt cache
|
|
// served (cached_tokens) and the part this call wrote to it (cache_write_tokens,
|
|
// which Codex reads beside OpenAI's field).
|
|
type responsesInputDetails struct {
|
|
CachedTokens int `json:"cached_tokens"`
|
|
CacheWriteTokens int `json:"cache_write_tokens"`
|
|
}
|
|
|
|
// chatUsage is a chat completion's usage. prompt_tokens is the whole prompt, and
|
|
// prompt_tokens_details says how much of it the upstream's cache served
|
|
// (cached_tokens) and, on OpenRouter, how much this call wrote to it
|
|
// (cache_write_tokens).
|
|
type chatUsage struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
PromptDetails *struct {
|
|
Cached int `json:"cached_tokens"`
|
|
Written int `json:"cache_write_tokens"`
|
|
} `json:"prompt_tokens_details"`
|
|
}
|
|
|
|
type responsesOutputDetails struct {
|
|
ReasoningTokens int `json:"reasoning_tokens"`
|
|
}
|
|
|
|
func jsonOrNil(raw json.RawMessage) any {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return nil
|
|
}
|
|
var value any
|
|
if json.Unmarshal(raw, &value) != nil {
|
|
return nil
|
|
}
|
|
return value
|
|
}
|
|
|
|
func openAIChatResponseToResponses(body []byte, request *OpenAIResponsesRequest, toolKinds map[string]string) ([]byte, error) {
|
|
var response struct {
|
|
Choices []struct {
|
|
Message struct {
|
|
Content string `json:"content"`
|
|
ToolCalls []struct {
|
|
ID string `json:"id"`
|
|
Function struct{ Name, Arguments string } `json:"function"`
|
|
} `json:"tool_calls"`
|
|
} `json:"message"`
|
|
} `json:"choices"`
|
|
Usage chatUsage `json:"usage"`
|
|
}
|
|
if err := json.Unmarshal(body, &response); err != nil {
|
|
return nil, fmt.Errorf("invalid upstream chat response: %w", err)
|
|
}
|
|
translator := newResponsesStreamTranslator(func(string, any) error { return nil }, request, toolKinds)
|
|
translator.count(&response.Usage)
|
|
outputs := []any{}
|
|
if len(response.Choices) > 0 {
|
|
message := response.Choices[0].Message
|
|
if message.Content != "" {
|
|
outputs = append(outputs, map[string]any{
|
|
"id": translator.messageID, "type": "message", "role": "assistant", "status": "completed",
|
|
"content": []any{map[string]any{"type": "output_text", "text": message.Content, "annotations": []any{}}},
|
|
})
|
|
}
|
|
for _, tc := range message.ToolCalls {
|
|
call := &responsesToolCallState{ID: tc.ID, CallID: tc.ID, Name: tc.Function.Name, Kind: toolKinds[tc.Function.Name]}
|
|
if call.Kind == "" {
|
|
call.Kind = "function"
|
|
}
|
|
request.unflat(call)
|
|
call.Arguments.WriteString(tc.Function.Arguments)
|
|
outputs = append(outputs, call.item("completed", call.finalArguments()))
|
|
}
|
|
}
|
|
resource := translator.resource("completed", outputs)
|
|
return json.Marshal(resource)
|
|
}
|
|
|
|
// SendStreamWriter is zip's, with this request's answer shape applied to every
|
|
// stream it writes.
|
|
//
|
|
// The Responses bridge used to be installed by ONE path — the native text
|
|
// completion — while the family relay, the tool-call proxy and the vision proxy
|
|
// each opened their own stream and wrote upstream chat SSE straight to the
|
|
// client. So /v1/responses answered `chat.completion.chunk` for every model a
|
|
// family or a proxy serves, which is nearly all of them, and any request carrying
|
|
// tools: a Responses client (Hanzo Dev, the OpenAI SDK) saw no response.* event,
|
|
// dropped every chunk, and failed on "stream closed before response.completed".
|
|
//
|
|
// Here the bridge wraps the writer each path is handed, so the translation is a
|
|
// property of the request rather than of which path happened to serve it. A path
|
|
// flushes its own buffered writer after each chunk; the bridge emits and flushes
|
|
// each Responses event as the chunk arrives, so the answer still streams.
|
|
//
|
|
// On the paid lane the stream is paced by its seat (seat.paced): each chunk the
|
|
// client takes keeps the seat's hold, and a stream nobody reads gives it back.
|
|
func (c *ApiController) SendStreamWriter(fn func(*bufio.Writer)) error {
|
|
s := seatOf(c.Context())
|
|
to := c.answer
|
|
if to == nil || to.wrap == nil {
|
|
return c.Ctx.SendStreamWriter(s.paced(fn))
|
|
}
|
|
return c.Ctx.SendStreamWriter(s.paced(func(bw *bufio.Writer) {
|
|
out := to.wrap(bw)
|
|
in := bufio.NewWriter(out)
|
|
fn(in)
|
|
_ = in.Flush()
|
|
// An upstream that ends without [DONE] still gets its response.completed;
|
|
// finish is idempotent, so one that sent it is unaffected. This runs inside
|
|
// the callback because zip runs the callback after the handler returns: a
|
|
// close placed after chatCompletions ran before any byte was written.
|
|
//
|
|
// Logged, not answered: the events are already on the wire, so there is no
|
|
// status left to change.
|
|
if cl, ok := out.(io.Closer); ok {
|
|
if err := cl.Close(); err != nil {
|
|
log.Error(fmt.Sprintf("Responses: closing the stream failed: %s", err.Error()))
|
|
}
|
|
}
|
|
}))
|
|
}
|
|
|
|
// answerBody writes one whole successful answer in the shape this request asked
|
|
// for: the chat completion as is, or translated by the sink. It is the
|
|
// non-streaming half of SendStreamWriter's rule, for the same reason.
|
|
func (c *ApiController) answerBody(chat []byte) {
|
|
out, err := c.answer.shape(chat)
|
|
if err != nil {
|
|
c.ResponseError(err.Error())
|
|
return
|
|
}
|
|
c.SetHeader("Content-Type", "application/json")
|
|
c.Bytes(http.StatusOK, out)
|
|
}
|
|
|
|
// sendWhole writes ONE whole answer over time, paced by the request's seat as a
|
|
// stream is: the spaces that keep its connection open while it is assembled, then
|
|
// the answer. It is never decorated as a stream. Through SendStreamWriter a
|
|
// /v1/responses request asked whole had its assembled chat completion read by the
|
|
// Responses event bridge, which found no event in it and answered
|
|
// response.created and response.completed with nothing in between, under
|
|
// application/json. The writer shapes the answer itself (sink.shape).
|
|
func (c *ApiController) sendWhole(fn func(*bufio.Writer)) error {
|
|
return c.Ctx.SendStreamWriter(seatOf(c.Context()).paced(fn))
|
|
}
|