feat(ai): support text-to-speech capabilities (single & multi-speaker configurations) for non-Live API models (#10145)

* feat(ai): implementmulti-speaker speech config; includes tests

* add changeset, fix typo

* fix: yarn format

* fix changeset

* fix changeset typo

* remove unnecessary tests; fix interface description to match proposal

* Generate documentation

* fix: yarn format & lint fix

* add more descriptive changeset

* Adds tests for speech config to check if its correctly passed through to the API requests for both generateContent and generateContentStream methods

* yarn format

* yarn format

* Refine changeset file description

* Fix typo

* format fix

* update description BaseSpeechConfig

* update descriptions

* format doc

* yarn format

* fix @ typo over MultiSpeakerSpeechConfig

* fix: adjusted description wording for the interfaces; fix: public tags to beta

* fix: formatting using @remarks

* fix: typo

* yarn format

* run yarn docgen:all

* update SpeechConfig to beta

* update speech config to beta

* upd SingleSpeakerSpeechConfig description

* doc generation

.changeset/cool-bugs-pump.md

@@ -0,0 +1,6 @@
+---
+'@firebase/ai': minor
+'firebase': minor
+---
+
+Added support for text-to-speech capabilities (both single- and multi-speaker configurations) for non-Live API models (for example, Gemini TTS models).

common/api-review/ai.api.md

@@ -102,7 +102,7 @@ export interface AudioConversationController {
     stop: () => Promise<void>;
 }
 
-// @public
+// @beta
 export interface AudioTranscriptionConfig {
 }
 
@@ -134,6 +134,11 @@ export interface BaseParams {
     safetySettings?: SafetySetting[];
 }
 
+// @beta
+export interface BaseSpeechConfig {
+    languageCode?: string;
+}
+
 // @public
 export const BlockReason: {
     readonly SAFETY: "SAFETY";
@@ -585,6 +590,8 @@ export interface GenerationConfig {
     // @beta
     responseModalities?: ResponseModality[];
     responseSchema?: TypedSchema | SchemaRequest;
+    // @beta
+    speechConfig?: SpeechConfig;
     // (undocumented)
     stopSequences?: string[];
     // (undocumented)
@@ -1237,6 +1244,17 @@ export interface ModelParams extends BaseParams {
     tools?: Tool[];
 }
 
+// @beta
+export interface MultiSpeakerSpeechConfig extends BaseSpeechConfig {
+    multiSpeakerVoiceConfig?: MultiSpeakerVoiceConfig;
+    voiceConfig?: never;
+}
+
+// @beta
+export interface MultiSpeakerVoiceConfig {
+    speakerVoiceConfigs: SpeakerVoiceConfig[];
+}
+
 // @public
 export class NumberSchema extends Schema {
     constructor(schemaParams?: SchemaParams);
@@ -1475,16 +1493,26 @@ export interface SingleRequestOptions extends RequestOptions {
     signal?: AbortSignal;
 }
 
+// @beta
+export interface SingleSpeakerSpeechConfig extends BaseSpeechConfig {
+    multiSpeakerVoiceConfig?: never;
+    voiceConfig?: VoiceConfig;
+}
+
 // @beta
 export interface SlidingWindow {
     targetTokens?: number;
 }
 
 // @beta
-export interface SpeechConfig {
-    voiceConfig?: VoiceConfig;
+export interface SpeakerVoiceConfig {
+    speaker: string;
+    voiceConfig: VoiceConfig;
 }
 
+// @beta
+export type SpeechConfig = SingleSpeakerSpeechConfig | MultiSpeakerSpeechConfig;
+
 // @beta
 export function startAudioConversation(liveSession: LiveSession, options?: StartAudioConversationOptions): Promise<AudioConversationController>;
 

docs-devsite/_toc.yaml

@@ -26,6 +26,8 @@ toc:
     path: /docs/reference/js/ai.backend.md
   - title: BaseParams
     path: /docs/reference/js/ai.baseparams.md
+  - title: BaseSpeechConfig
+    path: /docs/reference/js/ai.basespeechconfig.md
   - title: BooleanSchema
     path: /docs/reference/js/ai.booleanschema.md
   - title: ChatSession
@@ -178,6 +180,10 @@ toc:
     path: /docs/reference/js/ai.modalitytokencount.md
   - title: ModelParams
     path: /docs/reference/js/ai.modelparams.md
+  - title: MultiSpeakerSpeechConfig
+    path: /docs/reference/js/ai.multispeakerspeechconfig.md
+  - title: MultiSpeakerVoiceConfig
+    path: /docs/reference/js/ai.multispeakervoiceconfig.md
   - title: NumberSchema
     path: /docs/reference/js/ai.numberschema.md
   - title: ObjectSchema
@@ -218,10 +224,12 @@ toc:
     path: /docs/reference/js/ai.sessionresumptionconfig.md
   - title: SingleRequestOptions
     path: /docs/reference/js/ai.singlerequestoptions.md
+  - title: SingleSpeakerSpeechConfig
+    path: /docs/reference/js/ai.singlespeakerspeechconfig.md
   - title: SlidingWindow
     path: /docs/reference/js/ai.slidingwindow.md
-  - title: SpeechConfig
-    path: /docs/reference/js/ai.speechconfig.md
+  - title: SpeakerVoiceConfig
+    path: /docs/reference/js/ai.speakervoiceconfig.md
   - title: StartAudioConversationOptions
     path: /docs/reference/js/ai.startaudioconversationoptions.md
   - title: StartChatParams

docs-devsite/ai.audiotranscriptionconfig.md

@@ -10,6 +10,9 @@ https://github.com/firebase/firebase-js-sdk
 {% endcomment %}
 
 # AudioTranscriptionConfig interface
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
 The audio transcription configuration.
 
 <b>Signature:</b>

docs-devsite/ai.basespeechconfig.md

@@ -9,33 +9,33 @@ overwritten. Changes should be made in the source code at
 https://github.com/firebase/firebase-js-sdk
 {% endcomment %}
 
-# SpeechConfig interface
+# BaseSpeechConfig interface
 > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
 > 
 
-Configures speech synthesis.
+Base configuration for speech synthesis.
 
 <b>Signature:</b>
 
 ```typescript
-export interface SpeechConfig 
+export interface BaseSpeechConfig 
 ```
 
 ## Properties
 
 |  Property | Type | Description |
 |  --- | --- | --- |
-|  [voiceConfig](./ai.speechconfig.md#speechconfigvoiceconfig) | [VoiceConfig](./ai.voiceconfig.md#voiceconfig_interface) | <b><i>(Public Preview)</i></b> Configures the voice to be used in speech synthesis. |
+|  [languageCode](./ai.basespeechconfig.md#basespeechconfiglanguagecode) | string | <b><i>(Public Preview)</i></b> IETF BCP-47 language code. |
 
-## SpeechConfig.voiceConfig
+## BaseSpeechConfig.languageCode
 
 > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
 > 
 
-Configures the voice to be used in speech synthesis.
+IETF BCP-47 language code.
 
 <b>Signature:</b>
 
 ```typescript
-voiceConfig?: VoiceConfig;
+languageCode?: string;
 ```

docs-devsite/ai.generationconfig.md

@@ -31,6 +31,7 @@ export interface GenerationConfig
 |  [responseMimeType](./ai.generationconfig.md#generationconfigresponsemimetype) | string | Output response MIME type of the generated candidate text. Supported MIME types are <code>text/plain</code> (default, text output), <code>application/json</code> (JSON response in the candidates), and <code>text/x.enum</code>. |
 |  [responseModalities](./ai.generationconfig.md#generationconfigresponsemodalities) | [ResponseModality](./ai.md#responsemodality)<!-- -->\[\] | <b><i>(Public Preview)</i></b> Generation modalities to be returned in generation responses. |
 |  [responseSchema](./ai.generationconfig.md#generationconfigresponseschema) | [TypedSchema](./ai.md#typedschema) \| [SchemaRequest](./ai.schemarequest.md#schemarequest_interface) | Output response schema of the generated candidate text. This value can be a class generated with a [Schema](./ai.schema.md#schema_class) static method like <code>Schema.string()</code> or <code>Schema.object()</code> or it can be a plain JS object matching the [SchemaRequest](./ai.schemarequest.md#schemarequest_interface) interface. <br/>Note: This only applies when the specified <code>responseMimeType</code> supports a schema; currently this is limited to <code>application/json</code> and <code>text/x.enum</code>. |
+|  [speechConfig](./ai.generationconfig.md#generationconfigspeechconfig) | [SpeechConfig](./ai.md#speechconfig) | <b><i>(Public Preview)</i></b> Configuration for speech synthesis for text-to-speech (TTS) models. |
 |  [stopSequences](./ai.generationconfig.md#generationconfigstopsequences) | string\[\] |  |
 |  [temperature](./ai.generationconfig.md#generationconfigtemperature) | number |  |
 |  [thinkingConfig](./ai.generationconfig.md#generationconfigthinkingconfig) | [ThinkingConfig](./ai.thinkingconfig.md#thinkingconfig_interface) | Configuration for "thinking" behavior of compatible Gemini models. |
@@ -128,6 +129,19 @@ Output response schema of the generated candidate text. This value can be a clas
 responseSchema?: TypedSchema | SchemaRequest;
 ```
 
+## GenerationConfig.speechConfig
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Configuration for speech synthesis for text-to-speech (TTS) models.
+
+<b>Signature:</b>
+
+```typescript
+speechConfig?: SpeechConfig;
+```
+
 ## GenerationConfig.stopSequences
 
 <b>Signature:</b>

docs-devsite/ai.livegenerationconfig.md

@@ -32,7 +32,7 @@ export interface LiveGenerationConfig
 |  [outputAudioTranscription](./ai.livegenerationconfig.md#livegenerationconfigoutputaudiotranscription) | [AudioTranscriptionConfig](./ai.audiotranscriptionconfig.md#audiotranscriptionconfig_interface) | <b><i>(Public Preview)</i></b> Enables transcription of audio input.<!-- -->When enabled, the model will respond with transcriptions of its audio output in the <code>outputTranscription</code> property in [LiveServerContent](./ai.liveservercontent.md#liveservercontent_interface) messages. Note that the transcriptions are broken up across messages, so you may only receive small amounts of text per message. For example, if the model says "How are you today?", the model may transcribe that output across three messages, broken up as "How a", "re yo", "u today?". |
 |  [presencePenalty](./ai.livegenerationconfig.md#livegenerationconfigpresencepenalty) | number | <b><i>(Public Preview)</i></b> Positive penalties. |
 |  [responseModalities](./ai.livegenerationconfig.md#livegenerationconfigresponsemodalities) | [ResponseModality](./ai.md#responsemodality)<!-- -->\[\] | <b><i>(Public Preview)</i></b> The modalities of the response. |
-|  [speechConfig](./ai.livegenerationconfig.md#livegenerationconfigspeechconfig) | [SpeechConfig](./ai.speechconfig.md#speechconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for speech synthesis. |
+|  [speechConfig](./ai.livegenerationconfig.md#livegenerationconfigspeechconfig) | [SpeechConfig](./ai.md#speechconfig) | <b><i>(Public Preview)</i></b> Configuration for speech synthesis for Live API Models. |
 |  [temperature](./ai.livegenerationconfig.md#livegenerationconfigtemperature) | number | <b><i>(Public Preview)</i></b> Controls the degree of randomness in token selection. A <code>temperature</code> value of 0 means that the highest probability tokens are always selected. In this case, responses for a given prompt are mostly deterministic, but a small amount of variation is still possible. |
 |  [topK](./ai.livegenerationconfig.md#livegenerationconfigtopk) | number | <b><i>(Public Preview)</i></b> Changes how the model selects token for output. A <code>topK</code> value of 1 means the select token is the most probable among all tokens in the model's vocabulary, while a <code>topK</code> value 3 means that the next token is selected from among the 3 most probably using probabilities sampled. Tokens are then further filtered with the highest selected <code>temperature</code> sampling. Defaults to 40 if unspecified. |
 |  [topP](./ai.livegenerationconfig.md#livegenerationconfigtopp) | number | <b><i>(Public Preview)</i></b> Changes how the model selects tokens for output. Tokens are selected from the most to least probable until the sum of their probabilities equals the <code>topP</code> value. For example, if tokens A, B, and C have probabilities of 0.3, 0.2, and 0.1 respectively and the <code>topP</code> value is 0.5, then the model will select either A or B as the next token by using the <code>temperature</code> and exclude C as a candidate. Defaults to 0.95 if unset. |
@@ -137,7 +137,7 @@ responseModalities?: ResponseModality[];
 > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
 > 
 
-Configuration for speech synthesis.
+Configuration for speech synthesis for Live API Models.
 
 <b>Signature:</b>
 

docs-devsite/ai.md

@@ -62,8 +62,9 @@ The Firebase AI Web SDK.
 |  [AI](./ai.ai.md#ai_interface) | An instance of the Firebase AI SDK.<!-- -->Do not create this instance directly. Instead, use [getAI()](./ai.md#getai_a94a413)<!-- -->. |
 |  [AIOptions](./ai.aioptions.md#aioptions_interface) | Options for initializing the AI service using [getAI()](./ai.md#getai_a94a413)<!-- -->. This allows specifying which backend to use (Gemini Developer API or Agent Platform Gemini API) and configuring its specific options (like location for Agent Platform). |
 |  [AudioConversationController](./ai.audioconversationcontroller.md#audioconversationcontroller_interface) | <b><i>(Public Preview)</i></b> A controller for managing an active audio conversation. |
-|  [AudioTranscriptionConfig](./ai.audiotranscriptionconfig.md#audiotranscriptionconfig_interface) | The audio transcription configuration. |
+|  [AudioTranscriptionConfig](./ai.audiotranscriptionconfig.md#audiotranscriptionconfig_interface) | <b><i>(Public Preview)</i></b> The audio transcription configuration. |
 |  [BaseParams](./ai.baseparams.md#baseparams_interface) | Base parameters for a number of methods. |
+|  [BaseSpeechConfig](./ai.basespeechconfig.md#basespeechconfig_interface) | <b><i>(Public Preview)</i></b> Base configuration for speech synthesis. |
 |  [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) | Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device inference is possible.<!-- -->These methods should not be called directly by the user. |
 |  [Citation](./ai.citation.md#citation_interface) | A single citation. |
 |  [CitationMetadata](./ai.citationmetadata.md#citationmetadata_interface) | Citation metadata that may be found on a [GenerateContentCandidate](./ai.generatecontentcandidate.md#generatecontentcandidate_interface)<!-- -->. |
@@ -130,6 +131,8 @@ The Firebase AI Web SDK.
 |  [LiveSessionResumptionUpdate](./ai.livesessionresumptionupdate.md#livesessionresumptionupdate_interface) | <b><i>(Public Preview)</i></b> An update of the session resumption state.<!-- -->This message is only sent if [SessionResumptionConfig](./ai.sessionresumptionconfig.md#sessionresumptionconfig_interface) was set in the session setup. |
 |  [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. |
 |  [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a)<!-- -->. |
+|  [MultiSpeakerSpeechConfig](./ai.multispeakerspeechconfig.md#multispeakerspeechconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for speech synthesis with multiple speakers. |
+|  [MultiSpeakerVoiceConfig](./ai.multispeakervoiceconfig.md#multispeakervoiceconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for multi-speaker setup. |
 |  [ObjectSchemaRequest](./ai.objectschemarequest.md#objectschemarequest_interface) | Interface for JSON parameters in a schema of [SchemaType](./ai.md#schematype) "object" when not using the <code>Schema.object()</code> helper. |
 |  [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | Encapsulates configuration for on-device inference. |
 |  [PrebuiltVoiceConfig](./ai.prebuiltvoiceconfig.md#prebuiltvoiceconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for a pre-built voice. |
@@ -147,8 +150,9 @@ The Firebase AI Web SDK.
 |  [Segment](./ai.segment.md#segment_interface) | Represents a specific segment within a [Content](./ai.content.md#content_interface) object, often used to pinpoint the exact location of text or data that grounding information refers to. |
 |  [SessionResumptionConfig](./ai.sessionresumptionconfig.md#sessionresumptionconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for the session resumption mechanism. |
 |  [SingleRequestOptions](./ai.singlerequestoptions.md#singlerequestoptions_interface) | Options that can be provided per-request. Extends the base [RequestOptions](./ai.requestoptions.md#requestoptions_interface) (like <code>timeout</code> and <code>baseUrl</code>) with request-specific controls like cancellation via <code>AbortSignal</code>.<!-- -->Options specified here will override any default [RequestOptions](./ai.requestoptions.md#requestoptions_interface) configured on a model (for example, [GenerativeModel](./ai.generativemodel.md#generativemodel_class)<!-- -->). |
+|  [SingleSpeakerSpeechConfig](./ai.singlespeakerspeechconfig.md#singlespeakerspeechconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for speech synthesis for a single speaker. |
 |  [SlidingWindow](./ai.slidingwindow.md#slidingwindow_interface) | <b><i>(Public Preview)</i></b> Configures the sliding window context compression mechanism. |
-|  [SpeechConfig](./ai.speechconfig.md#speechconfig_interface) | <b><i>(Public Preview)</i></b> Configures speech synthesis. |
+|  [SpeakerVoiceConfig](./ai.speakervoiceconfig.md#speakervoiceconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for a single speaker's voice. |
 |  [StartAudioConversationOptions](./ai.startaudioconversationoptions.md#startaudioconversationoptions_interface) | <b><i>(Public Preview)</i></b> Options for [startAudioConversation()](./ai.md#startaudioconversation_01c8e7f)<!-- -->. |
 |  [StartChatParams](./ai.startchatparams.md#startchatparams_interface) | Params for [GenerativeModel.startChat()](./ai.generativemodel.md#generativemodelstartchat)<!-- -->. |
 |  [StartTemplateChatParams](./ai.starttemplatechatparams.md#starttemplatechatparams_interface) | <b><i>(Public Preview)</i></b> Params for [TemplateGenerativeModel.startChat()](./ai.templategenerativemodel.md#templategenerativemodelstartchat)<!-- -->. |
@@ -166,7 +170,7 @@ The Firebase AI Web SDK.
 |  [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface) | Metadata for a single URL retrieved by the [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) tool. |
 |  [UsageMetadata](./ai.usagemetadata.md#usagemetadata_interface) | Usage metadata about a [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->. |
 |  [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Describes the input video content. |
-|  [VoiceConfig](./ai.voiceconfig.md#voiceconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for the voice to used in speech synthesis. |
+|  [VoiceConfig](./ai.voiceconfig.md#voiceconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for the voice to be used for speech synthesis. |
 |  [WebAttribution](./ai.webattribution.md#webattribution_interface) |  |
 |  [WebGroundingChunk](./ai.webgroundingchunk.md#webgroundingchunk_interface) | A grounding chunk from the web.<!-- -->Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". |
 
@@ -233,6 +237,7 @@ The Firebase AI Web SDK.
 |  [ResponseModality](./ai.md#responsemodality) | <b><i>(Public Preview)</i></b> Generation modalities to be returned in generation responses. |
 |  [Role](./ai.md#role) | Role is the producer of the content. |
 |  [SchemaType](./ai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) |
+|  [SpeechConfig](./ai.md#speechconfig) | <b><i>(Public Preview)</i></b> Configures speech synthesis. |
 |  [TemplateTool](./ai.md#templatetool) | <b><i>(Public Preview)</i></b> Defines a tool that a [TemplateGenerativeModel](./ai.templategenerativemodel.md#templategenerativemodel_class) can call to access external knowledge. Only function declarations are currently supported for templates. |
 |  [ThinkingLevel](./ai.md#thinkinglevel) | A preset that controls the model's "thinking" process. Use <code>ThinkingLevel.LOW</code> for faster responses on less complex tasks, and <code>ThinkingLevel.HIGH</code> for better reasoning on more complex tasks. |
 |  [Tool](./ai.md#tool) | Defines a tool that model can call to access external knowledge. |
@@ -1258,6 +1263,19 @@ Contains the list of OpenAPI data types as defined by the [OpenAPI specification
 export type SchemaType = (typeof SchemaType)[keyof typeof SchemaType];
 ```
 
+## SpeechConfig
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Configures speech synthesis.
+
+<b>Signature:</b>
+
+```typescript
+export type SpeechConfig = SingleSpeakerSpeechConfig | MultiSpeakerSpeechConfig;
+```
+
 ## TemplateTool
 
 > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.

docs-devsite/ai.multispeakerspeechconfig.md

@@ -0,0 +1,56 @@
+Project: /docs/reference/js/_project.yaml
+Book: /docs/reference/_book.yaml
+page_type: reference
+
+{% comment %}
+DO NOT EDIT THIS FILE!
+This is generated by the JS SDK team, and any local changes will be
+overwritten. Changes should be made in the source code at
+https://github.com/firebase/firebase-js-sdk
+{% endcomment %}
+
+# MultiSpeakerSpeechConfig interface
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Configuration for speech synthesis with multiple speakers.
+
+<b>Signature:</b>
+
+```typescript
+export interface MultiSpeakerSpeechConfig extends BaseSpeechConfig 
+```
+<b>Extends:</b> [BaseSpeechConfig](./ai.basespeechconfig.md#basespeechconfig_interface)
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [multiSpeakerVoiceConfig](./ai.multispeakerspeechconfig.md#multispeakerspeechconfigmultispeakervoiceconfig) | [MultiSpeakerVoiceConfig](./ai.multispeakervoiceconfig.md#multispeakervoiceconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for multi-speaker setup. |
+|  [voiceConfig](./ai.multispeakerspeechconfig.md#multispeakerspeechconfigvoiceconfig) | never | <b><i>(Public Preview)</i></b> Single-speaker voice configuration must not be set when using multiple speakers. |
+
+## MultiSpeakerSpeechConfig.multiSpeakerVoiceConfig
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Configuration for multi-speaker setup.
+
+<b>Signature:</b>
+
+```typescript
+multiSpeakerVoiceConfig?: MultiSpeakerVoiceConfig;
+```
+
+## MultiSpeakerSpeechConfig.voiceConfig
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Single-speaker voice configuration must not be set when using multiple speakers.
+
+<b>Signature:</b>
+
+```typescript
+voiceConfig?: never;
+```

docs-devsite/ai.multispeakervoiceconfig.md

@@ -0,0 +1,41 @@
+Project: /docs/reference/js/_project.yaml
+Book: /docs/reference/_book.yaml
+page_type: reference
+
+{% comment %}
+DO NOT EDIT THIS FILE!
+This is generated by the JS SDK team, and any local changes will be
+overwritten. Changes should be made in the source code at
+https://github.com/firebase/firebase-js-sdk
+{% endcomment %}
+
+# MultiSpeakerVoiceConfig interface
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Configuration for multi-speaker setup.
+
+<b>Signature:</b>
+
+```typescript
+export interface MultiSpeakerVoiceConfig 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [speakerVoiceConfigs](./ai.multispeakervoiceconfig.md#multispeakervoiceconfigspeakervoiceconfigs) | [SpeakerVoiceConfig](./ai.speakervoiceconfig.md#speakervoiceconfig_interface)<!-- -->\[\] | <b><i>(Public Preview)</i></b> All the enabled speaker voices. |
+
+## MultiSpeakerVoiceConfig.speakerVoiceConfigs
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+All the enabled speaker voices.
+
+<b>Signature:</b>
+
+```typescript
+speakerVoiceConfigs: SpeakerVoiceConfig[];
+```

docs-devsite/ai.prebuiltvoiceconfig.md

@@ -25,7 +25,7 @@ export interface PrebuiltVoiceConfig
 
 |  Property | Type | Description |
 |  --- | --- | --- |
-|  [voiceName](./ai.prebuiltvoiceconfig.md#prebuiltvoiceconfigvoicename) | string | <b><i>(Public Preview)</i></b> The voice name to use for speech synthesis.<!-- -->For a full list of names and demos of what each voice sounds like, see [Chirp 3: HD Voices](https://cloud.google.com/text-to-speech/docs/chirp3-hd)<!-- -->. |
+|  [voiceName](./ai.prebuiltvoiceconfig.md#prebuiltvoiceconfigvoicename) | string | <b><i>(Public Preview)</i></b> The voice name to use for speech synthesis. |
 
 ## PrebuiltVoiceConfig.voiceName
 

docs-devsite/ai.singlespeakerspeechconfig.md

@@ -0,0 +1,56 @@
+Project: /docs/reference/js/_project.yaml
+Book: /docs/reference/_book.yaml
+page_type: reference
+
+{% comment %}
+DO NOT EDIT THIS FILE!
+This is generated by the JS SDK team, and any local changes will be
+overwritten. Changes should be made in the source code at
+https://github.com/firebase/firebase-js-sdk
+{% endcomment %}
+
+# SingleSpeakerSpeechConfig interface
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Configuration for speech synthesis for a single speaker.
+
+<b>Signature:</b>
+
+```typescript
+export interface SingleSpeakerSpeechConfig extends BaseSpeechConfig 
+```
+<b>Extends:</b> [BaseSpeechConfig](./ai.basespeechconfig.md#basespeechconfig_interface)
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [multiSpeakerVoiceConfig](./ai.singlespeakerspeechconfig.md#singlespeakerspeechconfigmultispeakervoiceconfig) | never | <b><i>(Public Preview)</i></b> Multi-speaker configuration must not be set when using a single speaker. |
+|  [voiceConfig](./ai.singlespeakerspeechconfig.md#singlespeakerspeechconfigvoiceconfig) | [VoiceConfig](./ai.voiceconfig.md#voiceconfig_interface) | <b><i>(Public Preview)</i></b> Configures the voice to be used in speech synthesis. |
+
+## SingleSpeakerSpeechConfig.multiSpeakerVoiceConfig
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Multi-speaker configuration must not be set when using a single speaker.
+
+<b>Signature:</b>
+
+```typescript
+multiSpeakerVoiceConfig?: never;
+```
+
+## SingleSpeakerSpeechConfig.voiceConfig
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Configures the voice to be used in speech synthesis.
+
+<b>Signature:</b>
+
+```typescript
+voiceConfig?: VoiceConfig;
+```

docs-devsite/ai.speakervoiceconfig.md

@@ -0,0 +1,55 @@
+Project: /docs/reference/js/_project.yaml
+Book: /docs/reference/_book.yaml
+page_type: reference
+
+{% comment %}
+DO NOT EDIT THIS FILE!
+This is generated by the JS SDK team, and any local changes will be
+overwritten. Changes should be made in the source code at
+https://github.com/firebase/firebase-js-sdk
+{% endcomment %}
+
+# SpeakerVoiceConfig interface
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Configuration for a single speaker's voice.
+
+<b>Signature:</b>
+
+```typescript
+export interface SpeakerVoiceConfig 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [speaker](./ai.speakervoiceconfig.md#speakervoiceconfigspeaker) | string | <b><i>(Public Preview)</i></b> The name of the speaker to use (same as in prompt). |
+|  [voiceConfig](./ai.speakervoiceconfig.md#speakervoiceconfigvoiceconfig) | [VoiceConfig](./ai.voiceconfig.md#voiceconfig_interface) | <b><i>(Public Preview)</i></b> The configuration for the voice to use. |
+
+## SpeakerVoiceConfig.speaker
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+The name of the speaker to use (same as in prompt).
+
+<b>Signature:</b>
+
+```typescript
+speaker: string;
+```
+
+## SpeakerVoiceConfig.voiceConfig
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+The configuration for the voice to use.
+
+<b>Signature:</b>
+
+```typescript
+voiceConfig: VoiceConfig;
+```

docs-devsite/ai.voiceconfig.md

@@ -13,7 +13,7 @@ https://github.com/firebase/firebase-js-sdk
 > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
 > 
 
-Configuration for the voice to used in speech synthesis.
+Configuration for the voice to be used for speech synthesis.
 
 <b>Signature:</b>
 

packages/ai/src/models/generative-model.test.ts

@@ -251,6 +251,86 @@ describe('GenerativeModel', () => {
       })
     );
   });
+  it('passes single-speaker speechConfig through to generateContent', async () => {
+    const genModel = new GenerativeModel(
+      fakeAI,
+      {
+        model: 'my-model',
+        generationConfig: {
+          speechConfig: {
+            languageCode: 'en-US',
+            voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Puck' } }
+          }
+        }
+      },
+      {},
+      fakeChromeAdapter
+    );
+
+    const mockResponse = getMockResponse(
+      'vertexAI',
+      'unary-success-basic-reply-short.json'
+    );
+    const makeRequestStub = stub(request, 'makeRequest').resolves(
+      mockResponse as Response
+    );
+
+    await genModel.generateContent('Say hello!');
+
+    expect(makeRequestStub).to.be.calledWith(
+      {
+        model: 'publishers/google/models/my-model',
+        task: request.Task.GENERATE_CONTENT,
+        apiSettings: match.any,
+        stream: false,
+        singleRequestOptions: {}
+      },
+      match((value: string) => {
+        return value.includes('en-US') && value.includes('Puck');
+      })
+    );
+  });
+  it('passes single-speaker speechConfig through to generateContentStream', async () => {
+    const genModel = new GenerativeModel(
+      fakeAI,
+      {
+        model: 'my-model',
+        generationConfig: {
+          speechConfig: {
+            languageCode: 'en-US',
+            voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } }
+          }
+        }
+      },
+      {},
+      fakeChromeAdapter
+    );
+
+    const mockResponse = getMockResponseStreaming(
+      'vertexAI',
+      'streaming-success-basic-reply-short.txt'
+    );
+    const makeRequestStub = stub(request, 'makeRequest').resolves(
+      mockResponse as Response
+    );
+
+    await genModel.generateContentStream('Have a conversation.');
+
+    expect(makeRequestStub).to.be.calledWith(
+      {
+        model: 'publishers/google/models/my-model',
+        task: request.Task.STREAM_GENERATE_CONTENT,
+        apiSettings: match.any,
+        stream: true,
+        singleRequestOptions: {}
+      },
+      match((value: string) => {
+        return value.includes('en-US') && value.includes('Kore');
+      })
+    );
+
+    restore();
+  });
   it('passes base model params through to ChatSession when there are no startChatParams', async () => {
     const genModel = new GenerativeModel(
       fakeAI,

packages/ai/src/requests/request-helpers.test.ts

@@ -178,34 +178,168 @@ describe('request formatting methods', () => {
         ],
         systemInstruction: { role: 'system', parts: [{ text: 'be excited' }] }
       });
-    }),
-      it('formats fileData as part if provided as part', () => {
-        const result = formatGenerateContentInput([
-          'What is this?',
+    });
+    it('preserves SpeechConfig for single-speaker setups', () => {
+      const result = formatGenerateContentInput({
+        contents: [
           {
-            fileData: {
-              mimeType: 'image/jpeg',
-              fileUri: 'gs://sample.appspot.com/image.jpeg'
+            role: 'user',
+            parts: [{ text: 'Hello' }]
+          }
+        ],
+        generationConfig: {
+          speechConfig: {
+            languageCode: 'en-US',
+            voiceConfig: {
+              prebuiltVoiceConfig: {
+                voiceName: 'Puck'
+              }
             }
           }
-        ]);
-        expect(result).to.be.deep.equal({
-          contents: [
-            {
-              role: 'user',
-              parts: [
-                { text: 'What is this?' },
+        }
+      });
+      expect(result.generationConfig?.speechConfig).to.deep.equal({
+        languageCode: 'en-US',
+        voiceConfig: {
+          prebuiltVoiceConfig: {
+            voiceName: 'Puck'
+          }
+        }
+      });
+    });
+
+    it('preserves SpeechConfig for multi-speaker setups', () => {
+      const result = formatGenerateContentInput({
+        contents: [
+          {
+            role: 'user',
+            parts: [{ text: 'Write a dialogue.' }]
+          }
+        ],
+        generationConfig: {
+          speechConfig: {
+            languageCode: 'en-US',
+            multiSpeakerVoiceConfig: {
+              speakerVoiceConfigs: [
                 {
-                  fileData: {
-                    mimeType: 'image/jpeg',
-                    fileUri: 'gs://sample.appspot.com/image.jpeg'
+                  speaker: 'narrator',
+                  voiceConfig: {
+                    prebuiltVoiceConfig: { voiceName: 'Aoede' }
+                  }
+                },
+                {
+                  speaker: 'character',
+                  voiceConfig: {
+                    prebuiltVoiceConfig: { voiceName: 'Charon' }
                   }
                 }
               ]
             }
-          ]
-        });
+          }
+        }
       });
+      expect(result.generationConfig?.speechConfig).to.deep.equal({
+        languageCode: 'en-US',
+        multiSpeakerVoiceConfig: {
+          speakerVoiceConfigs: [
+            {
+              speaker: 'narrator',
+              voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Aoede' } }
+            },
+            {
+              speaker: 'character',
+              voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Charon' } }
+            }
+          ]
+        }
+      });
+    });
+    it('preserves SpeechConfig alongside other GenerationConfig parameters', () => {
+      const req = formatGenerateContentInput({
+        contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
+        generationConfig: {
+          temperature: 0.7,
+          maxOutputTokens: 100,
+          responseMimeType: 'audio/mp3',
+          speechConfig: {
+            voiceConfig: {
+              prebuiltVoiceConfig: {
+                voiceName: 'Puck'
+              }
+            }
+          }
+        }
+      });
+      expect(req.generationConfig?.temperature).to.equal(0.7);
+      expect(req.generationConfig?.maxOutputTokens).to.equal(100);
+      expect(req.generationConfig?.responseMimeType).to.equal('audio/mp3');
+      expect(
+        req.generationConfig?.speechConfig?.voiceConfig?.prebuiltVoiceConfig
+          ?.voiceName
+      ).to.equal('Puck');
+    });
+
+    it('serializes language code without nested voice configurations', () => {
+      const result = formatGenerateContentInput({
+        contents: [
+          {
+            role: 'user',
+            parts: [{ text: 'Hola' }]
+          }
+        ],
+        generationConfig: {
+          speechConfig: {
+            languageCode: 'es-ES'
+          }
+        }
+      });
+      expect(result.generationConfig?.speechConfig).to.deep.equal({
+        languageCode: 'es-ES'
+      });
+    });
+
+    it('safely passes through an empty speechConfig object', () => {
+      const result = formatGenerateContentInput({
+        contents: [
+          {
+            role: 'user',
+            parts: [{ text: 'Hello' }]
+          }
+        ],
+        generationConfig: {
+          speechConfig: {}
+        }
+      });
+      expect(result.generationConfig?.speechConfig).to.deep.equal({});
+    });
+
+    it('formats fileData as part if provided as part', () => {
+      const result = formatGenerateContentInput([
+        'What is this?',
+        {
+          fileData: {
+            mimeType: 'image/jpeg',
+            fileUri: 'gs://sample.appspot.com/image.jpeg'
+          }
+        }
+      ]);
+      expect(result).to.be.deep.equal({
+        contents: [
+          {
+            role: 'user',
+            parts: [
+              { text: 'What is this?' },
+              {
+                fileData: {
+                  mimeType: 'image/jpeg',
+                  fileUri: 'gs://sample.appspot.com/image.jpeg'
+                }
+              }
+            ]
+          }
+        ]
+      });
+    });
   });
   describe('createPredictRequestBody', () => {
     it('creates body with default request parameters', () => {

packages/ai/src/types/requests.ts

@@ -144,6 +144,13 @@ export interface GenerationConfig {
   topK?: number;
   presencePenalty?: number;
   frequencyPenalty?: number;
+
+  /**
+   * Configuration for speech synthesis for text-to-speech (TTS) models.
+   *
+   * @beta
+   */
+  speechConfig?: SpeechConfig;
   /**
    * Output response MIME type of the generated candidate text.
    * Supported MIME types are `text/plain` (default, text output),
@@ -196,7 +203,7 @@ export interface GenerationConfig {
  */
 export interface LiveGenerationConfig {
   /**
-   * Configuration for speech synthesis.
+   * Configuration for speech synthesis for Live API Models.
    */
   speechConfig?: SpeechConfig;
   /**
@@ -841,14 +848,14 @@ export interface ThinkingConfig {
 export interface PrebuiltVoiceConfig {
   /**
    * The voice name to use for speech synthesis.
-   *
+   * @remarks
    * For a full list of names and demos of what each voice sounds like, see {@link https://cloud.google.com/text-to-speech/docs/chirp3-hd | Chirp 3: HD Voices}.
    */
   voiceName?: string;
 }
 
 /**
- * Configuration for the voice to used in speech synthesis.
+ * Configuration for the voice to be used for speech synthesis.
  *
  * @beta
  */
@@ -861,17 +868,69 @@ export interface VoiceConfig {
 
 /**
  * Configures speech synthesis.
+ * @beta
+ */
+
+export type SpeechConfig = SingleSpeakerSpeechConfig | MultiSpeakerSpeechConfig;
+
+/**
+ * Base configuration for speech synthesis.
  *
  * @beta
  */
-export interface SpeechConfig {
-  /**
-   * Configures the voice to be used in speech synthesis.
-   */
-  voiceConfig?: VoiceConfig;
+export interface BaseSpeechConfig {
+  /** IETF BCP-47 language code. */
+  languageCode?: string;
 }
 
 /**
  * The audio transcription configuration.
+ *
+ * @beta
  */
 export interface AudioTranscriptionConfig {}
+
+/**
+ * Configuration for speech synthesis for a single speaker.
+ * @beta
+ */
+export interface SingleSpeakerSpeechConfig extends BaseSpeechConfig {
+  /** Configures the voice to be used in speech synthesis. */
+  voiceConfig?: VoiceConfig;
+  /** Multi-speaker configuration must not be set when using a single speaker. */
+  multiSpeakerVoiceConfig?: never;
+}
+
+/**
+ * Configuration for speech synthesis with multiple speakers.
+ * @beta
+ */
+
+export interface MultiSpeakerSpeechConfig extends BaseSpeechConfig {
+  /** Single-speaker voice configuration must not be set when using multiple speakers. */
+  voiceConfig?: never;
+  /** Configuration for multi-speaker setup. */
+  multiSpeakerVoiceConfig?: MultiSpeakerVoiceConfig;
+}
+
+/**
+ * Configuration for multi-speaker setup.
+ *
+ * @beta
+ */
+export interface MultiSpeakerVoiceConfig {
+  /** All the enabled speaker voices. */
+  speakerVoiceConfigs: SpeakerVoiceConfig[];
+}
+
+/**
+ * Configuration for a single speaker's voice.
+ *
+ * @beta
+ */
+export interface SpeakerVoiceConfig {
+  /** The name of the speaker to use (same as in prompt). */
+  speaker: string;
+  /** The configuration for the voice to use. */
+  voiceConfig: VoiceConfig;
+}