Files
Krish Chainaniandcopyberry 7b8b17b97a Support image references by file ID in inputs and tool outputs (#45794)
## What changed

- Accept `fileId` alongside the existing `url` form for app-server image inputs, and forward file references to the Responses API as `file_id`. Update generated schemas and client types.
- Preserve file references, image detail hints, and mixed inline/file image ordering through user-message events, thread history, and rollout migration. Retain file images when truncating tool output.
- Pass file references through image preparation without resolving them, while keeping resize-notice numbering correct. Omit them from unsupported TUI display and Guardian image context.
- Reject image-edit requests whose recent-image window includes a file reference, preventing selection of an older inline image instead.

## Testing

Add coverage for serialization, request and rollout preservation, mixed-image history ordering, incomplete ordering metadata, tool-output truncation, and rejection of unsupported image-edit selections.

GitOrigin-RevId: 6ca20a8577155cc934b720803c3b7b3bffdf972a
2026-09-15 21:19:08 +00:00

310 lines
6.8 KiB
JSON
Generated

{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"AdditionalContextEntry": {
"properties": {
"kind": {
"$ref": "#/definitions/AdditionalContextKind"
},
"value": {
"type": "string"
}
},
"required": [
"kind",
"value"
],
"type": "object"
},
"AdditionalContextKind": {
"enum": [
"untrusted",
"application"
],
"type": "string"
},
"ByteRange": {
"properties": {
"end": {
"format": "uint",
"minimum": 0.0,
"type": "integer"
},
"start": {
"format": "uint",
"minimum": 0.0,
"type": "integer"
}
},
"required": [
"end",
"start"
],
"type": "object"
},
"ImageDetail": {
"enum": [
"auto",
"low",
"high",
"original"
],
"type": "string"
},
"TextElement": {
"properties": {
"byteRange": {
"allOf": [
{
"$ref": "#/definitions/ByteRange"
}
],
"description": "Byte range in the parent `text` buffer that this element occupies."
},
"placeholder": {
"description": "Optional human-readable placeholder for the element, displayed in the UI.",
"type": [
"string",
"null"
]
}
},
"required": [
"byteRange"
],
"type": "object"
},
"UserInput": {
"oneOf": [
{
"properties": {
"text": {
"type": "string"
},
"text_elements": {
"default": [],
"description": "UI-defined spans within `text` used to render or persist special elements.",
"items": {
"$ref": "#/definitions/TextElement"
},
"type": "array"
},
"type": {
"enum": [
"text"
],
"title": "TextUserInputType",
"type": "string"
}
},
"required": [
"text",
"type"
],
"title": "TextUserInput",
"type": "object"
},
{
"anyOf": [
{
"properties": {
"url": {
"type": "string"
}
},
"required": [
"url"
],
"title": "UrlUserInput",
"type": "object"
},
{
"properties": {
"fileId": {
"type": "string"
}
},
"required": [
"fileId"
],
"title": "FileIdUserInput",
"type": "object"
}
],
"properties": {
"detail": {
"anyOf": [
{
"$ref": "#/definitions/ImageDetail"
},
{
"type": "null"
}
],
"default": null
},
"type": {
"enum": [
"image"
],
"title": "ImageUserInputType",
"type": "string"
}
},
"required": [
"type"
],
"title": "ImageUserInput",
"type": "object"
},
{
"properties": {
"detail": {
"anyOf": [
{
"$ref": "#/definitions/ImageDetail"
},
{
"type": "null"
}
],
"default": null
},
"path": {
"type": "string"
},
"type": {
"enum": [
"localImage"
],
"title": "LocalImageUserInputType",
"type": "string"
}
},
"required": [
"path",
"type"
],
"title": "LocalImageUserInput",
"type": "object"
},
{
"properties": {
"type": {
"enum": [
"audio"
],
"title": "AudioUserInputType",
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"type",
"url"
],
"title": "AudioUserInput",
"type": "object"
},
{
"properties": {
"path": {
"type": "string"
},
"type": {
"enum": [
"localAudio"
],
"title": "LocalAudioUserInputType",
"type": "string"
}
},
"required": [
"path",
"type"
],
"title": "LocalAudioUserInput",
"type": "object"
},
{
"properties": {
"name": {
"type": "string"
},
"path": {
"type": "string"
},
"type": {
"enum": [
"skill"
],
"title": "SkillUserInputType",
"type": "string"
}
},
"required": [
"name",
"path",
"type"
],
"title": "SkillUserInput",
"type": "object"
},
{
"properties": {
"name": {
"type": "string"
},
"path": {
"type": "string"
},
"type": {
"enum": [
"mention"
],
"title": "MentionUserInputType",
"type": "string"
}
},
"required": [
"name",
"path",
"type"
],
"title": "MentionUserInput",
"type": "object"
}
]
}
},
"properties": {
"clientUserMessageId": {
"type": [
"string",
"null"
]
},
"expectedTurnId": {
"description": "Required active turn id precondition. The request fails when it does not match the currently active turn.",
"type": "string"
},
"input": {
"items": {
"$ref": "#/definitions/UserInput"
},
"type": "array"
},
"threadId": {
"type": "string"
}
},
"required": [
"expectedTurnId",
"input",
"threadId"
],
"title": "TurnSteerParams",
"type": "object"
}