Files
sedanah-mandGitHub ce33864445 fix(ai): update deprecated gemini models strings" (#10043)
* update gemini models

* fix extra bracket error

* remove extra duplicate line

* fix lint error

* fix lint

* fix lint error

* fix prettier errors

* update gemini model in e2e

* revert tests

* yarn format

* Update AI integration tests and add new models

* yarn format

* chore: add minor changeset for hybrid fallback model update and revert generate-content.test

* revert old file: got rid of URL context and fixed tests

* revert cloud model change in src/constants to put in another PR

* delete changeset
2026-07-01 15:49:01 -04:00

677 lines
27 KiB
TypeScript

/**
* @license
* Copyright 2025 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import chai, { AssertionError } from 'chai';
import chaiAsPromised from 'chai-as-promised';
chai.use(chaiAsPromised);
const expect = chai.expect;
import {
BackendType,
Content,
GenerationConfig,
HarmBlockThreshold,
HarmCategory,
Language,
Modality,
Outcome,
SafetySetting,
URLRetrievalStatus,
getGenerativeModel
} from '../src';
import {
cheapestModel,
defaultAIInstance,
defaultGenerativeModel,
testConfigs
} from './constants';
import { TIMEOUT_EXPIRED_MESSAGE } from '../src/requests/request';
import { isNode } from '@firebase/util';
describe('Generate Content', function () {
this.timeout(90_000); // gemini 3.x requests using grounding and URL context can take up to 90s.
testConfigs.forEach(testConfig => {
describe(`${testConfig.toString()}`, () => {
const commonGenerationConfig: GenerationConfig = {
temperature: 0,
topP: 0,
responseMimeType: 'text/plain'
};
const commonSafetySettings: SafetySetting[] = [
{
category: HarmCategory.HARM_CATEGORY_HARASSMENT,
threshold: HarmBlockThreshold.BLOCK_NONE
},
{
category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
threshold: HarmBlockThreshold.BLOCK_NONE
},
{
category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
threshold: HarmBlockThreshold.BLOCK_NONE
},
{
category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
threshold: HarmBlockThreshold.BLOCK_NONE
}
];
const commonSystemInstruction: Content = {
role: 'system',
parts: [
{
text: 'You are a friendly and helpful assistant.'
}
]
};
it('generateContent: text input, text output', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
systemInstruction: commonSystemInstruction
});
const result = await model.generateContent(
'Where is Google headquarters located? Answer with the city name only.'
);
const response = result.response;
const trimmedText = response.text().trim();
expect(trimmedText).to.equal('Mountain View');
expect(response.usageMetadata).to.not.be.null;
expect(response.usageMetadata!.promptTokenCount).to.not.equal(0);
expect(response.usageMetadata!.candidatesTokenCount).to.not.equal(0);
expect(response.usageMetadata!.totalTokenCount).to.be.greaterThan(0);
expect(response.usageMetadata!.promptTokensDetails).to.not.be.null;
expect(response.usageMetadata!.promptTokensDetails!.length).to.equal(1);
expect(
response.usageMetadata!.promptTokensDetails![0].modality
).to.equal(Modality.TEXT);
expect(
response.usageMetadata!.promptTokensDetails![0].tokenCount
).to.be.greaterThan(0);
});
it('generateContent: google search grounding', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
tools: [{ googleSearch: {} }]
});
const result = await model.generateContent(
'What is the speed of light in a vaccuum in meters per second?'
);
const response = result.response;
const trimmedText = response.text().trim();
const groundingMetadata = response.candidates?.[0].groundingMetadata;
expect(trimmedText).to.contain('299,792,458');
expect(groundingMetadata).to.exist;
expect(groundingMetadata!.searchEntryPoint?.renderedContent).to.contain(
'div'
);
expect(
groundingMetadata!.groundingChunks
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingChunks!.forEach(groundingChunk => {
expect(groundingChunk.web).to.exist;
expect(groundingChunk.web!.uri).to.exist;
});
expect(
groundingMetadata?.groundingSupports
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingSupports!.forEach(groundingSupport => {
expect(
groundingSupport.groundingChunkIndices
).to.have.length.greaterThanOrEqual(1);
expect(groundingSupport.segment).to.exist;
expect(groundingSupport.segment?.endIndex).to.exist;
expect(groundingSupport.segment?.text).to.exist;
// Since partIndex and startIndex are commonly 0, they may be omitted from responses.
});
expect(groundingMetadata!.googleMapsWidgetContextToken).to.not.exist;
});
it('generateContent: google maps grounding prompt location', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
tools: [{ googleMaps: {} }]
});
const result = await model.generateContent(
'Where is a good place to grab a coffee near Arlington, MA?'
);
const response = result.response;
const groundingMetadata = response.candidates?.[0].groundingMetadata;
expect(groundingMetadata).to.exist;
expect(
groundingMetadata!.groundingChunks
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingChunks!.forEach(groundingChunk => {
expect(groundingChunk.maps).to.exist;
expect(groundingChunk.maps!.uri).to.exist;
expect(groundingChunk.maps!.title).to.exist;
expect(groundingChunk.maps!.placeId).to.exist;
});
expect(
groundingMetadata?.groundingSupports
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingSupports!.forEach(groundingSupport => {
expect(
groundingSupport.groundingChunkIndices
).to.have.length.greaterThanOrEqual(1);
expect(groundingSupport.segment).to.exist;
expect(groundingSupport.segment?.endIndex).to.exist;
expect(groundingSupport.segment?.text).to.exist;
});
expect(groundingMetadata!.googleMapsWidgetContextToken).to.not.exist;
});
it('generateContent: google maps grounding prompt location enableWidget true', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
tools: [{ googleMaps: { enableWidget: true } }]
});
const result = await model.generateContent(
'Where is a good place to grab a coffee in New York City?'
);
const response = result.response;
const groundingMetadata = response.candidates?.[0].groundingMetadata;
expect(groundingMetadata).to.exist;
expect(
groundingMetadata!.groundingChunks
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingChunks!.forEach(groundingChunk => {
expect(groundingChunk.maps).to.exist;
expect(groundingChunk.maps!.uri).to.exist;
expect(groundingChunk.maps!.title).to.exist;
expect(groundingChunk.maps!.placeId).to.exist;
});
expect(
groundingMetadata?.groundingSupports
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingSupports!.forEach(groundingSupport => {
expect(
groundingSupport.groundingChunkIndices
).to.have.length.greaterThanOrEqual(1);
expect(groundingSupport.segment).to.exist;
expect(groundingSupport.segment?.endIndex).to.exist;
expect(groundingSupport.segment?.text).to.exist;
});
expect(groundingMetadata!.googleMapsWidgetContextToken).to.exist;
});
it('generateContent: google maps grounding prompt location enableWidget false', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
tools: [{ googleMaps: { enableWidget: false } }]
});
const result = await model.generateContent(
'Where is a good place to grab a coffee in New York City?'
);
const response = result.response;
const groundingMetadata = response.candidates?.[0].groundingMetadata;
expect(groundingMetadata).to.exist;
expect(
groundingMetadata!.groundingChunks
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingChunks!.forEach(groundingChunk => {
expect(groundingChunk.maps).to.exist;
expect(groundingChunk.maps!.uri).to.exist;
expect(groundingChunk.maps!.title).to.exist;
expect(groundingChunk.maps!.placeId).to.exist;
});
expect(
groundingMetadata?.groundingSupports
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingSupports!.forEach(groundingSupport => {
expect(
groundingSupport.groundingChunkIndices
).to.have.length.greaterThanOrEqual(1);
expect(groundingSupport.segment).to.exist;
expect(groundingSupport.segment?.endIndex).to.exist;
expect(groundingSupport.segment?.text).to.exist;
});
expect(groundingMetadata!.googleMapsWidgetContextToken).to.not.exist;
});
it('generateContent: google maps grounding with RetrievalConfig', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
tools: [{ googleMaps: {} }],
toolConfig: {
retrievalConfig: {
latLng: {
latitude: 42.4154,
longitude: -71.1565
}
}
}
});
const result = await model.generateContent(
'Where is a good place to grab a coffee near here?'
);
const response = result.response;
const groundingMetadata = response.candidates?.[0].groundingMetadata;
expect(groundingMetadata).to.exist;
expect(
groundingMetadata!.groundingChunks
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingChunks!.forEach(groundingChunk => {
expect(groundingChunk.maps).to.exist;
expect(groundingChunk.maps!.uri).to.exist;
expect(groundingChunk.maps!.title).to.exist;
expect(groundingChunk.maps!.placeId).to.exist;
});
expect(
groundingMetadata?.groundingSupports
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingSupports!.forEach(groundingSupport => {
expect(
groundingSupport.groundingChunkIndices
).to.have.length.greaterThanOrEqual(1);
expect(groundingSupport.segment).to.exist;
expect(groundingSupport.segment?.endIndex).to.exist;
expect(groundingSupport.segment?.text).to.exist;
});
expect(groundingMetadata!.googleMapsWidgetContextToken).to.not.exist;
});
it('generateContent: google maps grounding RetrievalConfig enableWidget true', async () => {
if (testConfig.model === 'gemini-3-pro-preview') {
// Maps grounding is not supported in gemini-3-pro-preview.
return;
}
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
tools: [{ googleMaps: { enableWidget: true } }],
toolConfig: {
retrievalConfig: {
latLng: {
latitude: 42.4154,
longitude: -71.1565
}
}
}
});
const result = await model.generateContent(
'Where is a good place to grab a coffee near here?'
//'Where is the closest starbucks?'
);
const response = result.response;
const groundingMetadata = response.candidates?.[0].groundingMetadata;
expect(groundingMetadata).to.exist;
expect(
groundingMetadata!.groundingChunks
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingChunks!.forEach(groundingChunk => {
expect(groundingChunk.maps).to.exist;
expect(groundingChunk.maps!.uri).to.exist;
expect(groundingChunk.maps!.title).to.exist;
expect(groundingChunk.maps!.placeId).to.exist;
});
expect(
groundingMetadata?.groundingSupports
).to.have.length.greaterThanOrEqual(1);
groundingMetadata!.groundingSupports!.forEach(groundingSupport => {
expect(
groundingSupport.groundingChunkIndices
).to.have.length.greaterThanOrEqual(1);
expect(groundingSupport.segment).to.exist;
expect(groundingSupport.segment?.endIndex).to.exist;
expect(groundingSupport.segment?.text).to.exist;
});
expect(groundingMetadata!.googleMapsWidgetContextToken).to.exist;
});
describe('URL Context', () => {
it('generateContent: url context', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
tools: [{ urlContext: {} }]
});
const result = await model.generateContent(
'Summarize this website https://berkshirehathaway.com'
);
const response = result.response;
const urlContextMetadata =
response.candidates?.[0].urlContextMetadata;
expect(urlContextMetadata?.urlMetadata).to.exist;
expect(
urlContextMetadata?.urlMetadata.length
).to.be.greaterThanOrEqual(1);
expect(urlContextMetadata?.urlMetadata[0].retrievedUrl).to.exist;
expect(urlContextMetadata?.urlMetadata[0].retrievedUrl).to.equal(
'https://berkshirehathaway.com'
);
expect(
urlContextMetadata?.urlMetadata[0].urlRetrievalStatus
).to.equal(URLRetrievalStatus.URL_RETRIEVAL_STATUS_SUCCESS);
const usageMetadata = response.usageMetadata;
expect(usageMetadata).to.exist;
expect(usageMetadata?.toolUsePromptTokenCount).to.exist;
expect(usageMetadata?.toolUsePromptTokenCount).to.be.greaterThan(0);
});
it('generateContent: url context and google search grounding', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
tools: [{ urlContext: {} }, { googleSearch: {} }]
});
const result = await model.generateContent(
'According to https://info.cern.ch/hypertext/WWW/TheProject.html, what is the WorldWideWeb? Search the web for other definitions.'
);
const response = result.response;
const trimmedText = response.text().trim();
const urlContextMetadata =
response.candidates?.[0].urlContextMetadata;
const groundingMetadata = response.candidates?.[0].groundingMetadata;
expect(trimmedText.length).to.be.greaterThan(0);
expect(urlContextMetadata?.urlMetadata).to.exist;
expect(
urlContextMetadata?.urlMetadata.length
).to.be.greaterThanOrEqual(1);
expect(urlContextMetadata?.urlMetadata[0].retrievedUrl).to.exist;
expect(urlContextMetadata?.urlMetadata[0].retrievedUrl).to.equal(
'https://info.cern.ch/hypertext/WWW/TheProject.html'
);
expect(
urlContextMetadata?.urlMetadata[0].urlRetrievalStatus
).to.equal(URLRetrievalStatus.URL_RETRIEVAL_STATUS_SUCCESS);
expect(groundingMetadata).to.exist;
expect(groundingMetadata?.groundingChunks).to.exist;
expect(
groundingMetadata?.groundingChunks!.length
).to.be.greaterThanOrEqual(1);
expect(
groundingMetadata?.groundingSupports!.length
).to.be.greaterThanOrEqual(1);
const usageMetadata = response.usageMetadata;
expect(usageMetadata).to.exist;
expect(usageMetadata?.toolUsePromptTokenCount).to.exist;
expect(usageMetadata?.toolUsePromptTokenCount).to.be.greaterThan(0);
});
it('generateContent: url context and google search grounding without URLs in prompt', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
tools: [{ urlContext: {} }, { googleSearch: {} }]
});
const result = await model.generateContent(
'Recommend 3 books for beginners to read to learn more about the latest advancements in Quantum Computing'
);
const response = result.response;
const urlContextMetadata =
response.candidates?.[0].urlContextMetadata;
const groundingMetadata = response.candidates?.[0].groundingMetadata;
if (testConfig.ai.backend.backendType === BackendType.GOOGLE_AI) {
expect(urlContextMetadata?.urlMetadata).to.exist;
expect(
urlContextMetadata?.urlMetadata.length
).to.be.greaterThanOrEqual(1);
expect(urlContextMetadata?.urlMetadata[0].retrievedUrl).to.exist;
expect(
urlContextMetadata?.urlMetadata[0].urlRetrievalStatus
).to.equal(URLRetrievalStatus.URL_RETRIEVAL_STATUS_SUCCESS);
expect(groundingMetadata).to.exist;
expect(groundingMetadata?.groundingChunks).to.exist;
const usageMetadata = response.usageMetadata;
expect(usageMetadata).to.exist;
expect(usageMetadata?.toolUsePromptTokenCount).to.exist;
expect(usageMetadata?.toolUsePromptTokenCount).to.be.greaterThan(0);
} else {
// URL Context does not integrate with Google Search Grounding in Vertex AI
expect(urlContextMetadata?.urlMetadata).to.not.exist;
expect(groundingMetadata).to.exist;
expect(groundingMetadata?.groundingChunks).to.exist;
}
});
});
it('generateContent: code execution', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
tools: [{ codeExecution: {} }]
});
const prompt =
'What is the sum of the first 50 prime numbers? ' +
'Generate and run code for the calculation, and make sure you get all 50.';
const result = await model.generateContent(prompt);
const parts = result.response.candidates?.[0].content.parts;
expect(
parts?.some(part => part.executableCode?.language === Language.PYTHON)
).to.be.true;
expect(
parts?.some(part => part.codeExecutionResult?.outcome === Outcome.OK)
).to.be.true;
// Expect these to be truthy (!= null)
expect(parts?.some(part => part.executableCode?.code != null)).to.be
.true;
expect(parts?.some(part => part.codeExecutionResult?.output != null)).to
.be.true;
});
it('generateContentStream: text input, text output', async () => {
const model = getGenerativeModel(testConfig.ai, {
model: testConfig.model,
generationConfig: commonGenerationConfig,
safetySettings: commonSafetySettings,
systemInstruction: commonSystemInstruction
});
const result = await model.generateContentStream(
'Where is Google headquarters located? Answer with the city name only.'
);
let streamText = '';
for await (const chunk of result.stream) {
streamText += chunk.text();
}
expect(streamText.trim()).to.equal('Mountain View');
const response = await result.response;
const trimmedText = response.text().trim();
expect(trimmedText).to.equal('Mountain View');
expect(response.usageMetadata).to.be.undefined; // Note: This is incorrect behavior.
});
});
});
describe('Request Options', async () => {
const defaultAbortReason = isNode()
? 'This operation was aborted'
: 'signal is aborted without reason';
describe('unary', async () => {
it('timeout cancels request', async () => {
await expect(
defaultGenerativeModel.generateContent('hello', { timeout: 100 })
).to.be.rejectedWith(DOMException, TIMEOUT_EXPIRED_MESSAGE);
});
it('long timeout does not cancel request', async () => {
const result = await defaultGenerativeModel.generateContent('hello', {
timeout: 50_000
});
expect(result.response.text().length).to.be.greaterThan(0);
});
it('abort signal with no reason causes request to throw AbortError', async () => {
const abortController = new AbortController();
const responsePromise = defaultGenerativeModel.generateContent(
'hello',
{ signal: abortController.signal }
);
abortController.abort();
await expect(responsePromise)
.to.be.rejectedWith(DOMException, defaultAbortReason)
.and.eventually.have.property('name', 'AbortError');
});
it('abort signal with string reason causes request to throw reason string', async () => {
const abortController = new AbortController();
const responsePromise = defaultGenerativeModel.generateContent(
'hello',
{ signal: abortController.signal }
);
const reason = 'Cancelled';
abortController.abort(reason);
await expect(responsePromise).to.be.rejectedWith(reason);
});
it('abort signal with error reason causes request to throw reason error', async () => {
const abortController = new AbortController();
const responsePromise = defaultGenerativeModel.generateContent(
'hello',
{ signal: abortController.signal }
);
abortController.abort(new Error('Cancelled'));
// `fetch()` will reject with the exact object we passed to `abort()`. Since we throw a generic
// Error, we cannot differentiate between this error and other generic fetch errors, which
// we wrap in an AIError.
await expect(responsePromise)
.to.be.rejectedWith(Error, 'Cancelled')
.and.eventually.have.property('name', 'FirebaseError');
});
});
describe('streaming', async () => {
it('timeout cancels initial request', async () => {
await expect(
defaultGenerativeModel.generateContent('hello', { timeout: 50 })
).to.be.rejectedWith(DOMException, TIMEOUT_EXPIRED_MESSAGE);
});
it('timeout does not cancel request once streaming has begun', async () => {
const generativeModel = getGenerativeModel(defaultAIInstance, {
model: cheapestModel
});
// Setting a timeout that will be in the interval between the stream starting and ending.
// Since the timeout will expire once the stream has begun, it should have already been
// cleared, and so it shouldn't abort the stream.
const { stream, response } =
await generativeModel.generateContentStream(
'tell me a short story with 200 words.',
{ timeout: 1_000 }
);
// We should be able to get through the entire stream without an error being thrown
// from the async generator.
for await (const chunk of stream) {
expect(chunk.text().length).to.be.greaterThan(0);
}
expect((await response).text().length).to.be.greaterThan(0);
});
it('abort signal without reason should cancel stream with default abort reason', async () => {
const abortController = new AbortController();
const generativeModel = getGenerativeModel(defaultAIInstance, {
model: cheapestModel
});
const { stream, response } =
await generativeModel.generateContentStream(
'tell me a short story with 200 words.',
{ signal: abortController.signal }
);
// As soon as the initial request resolves and the stream starts, abort the stream.
abortController.abort();
try {
for await (const _ of stream) {
expect.fail('Expected stream to throw an error');
}
expect.fail('Expected stream to throw an error');
} catch (err) {
if ((err as Error) instanceof AssertionError) {
throw err;
}
expect(err).to.be.instanceof(DOMException);
expect((err as Error).name).to.equal('AbortError');
expect((err as Error).message).to.equal(defaultAbortReason);
}
await expect(response)
.to.be.rejectedWith(DOMException, defaultAbortReason)
.and.to.eventually.have.property('name', 'AbortError');
});
it('abort signal with reason string should cancel stream with string abort reason', async () => {
const abortController = new AbortController();
const generativeModel = getGenerativeModel(defaultAIInstance, {
model: cheapestModel
});
const { stream, response } =
await generativeModel.generateContentStream(
'tell me a short story with 200 words.',
{ signal: abortController.signal }
);
// As soon as the initial request resolves and the stream starts, abort the stream.
abortController.abort('Cancelled');
try {
for await (const _ of stream) {
expect.fail('Expected stream to throw an error');
}
expect.fail('Expected stream to throw an error');
} catch (err) {
if ((err as Error) instanceof AssertionError) {
throw err;
}
expect(err).to.equal('Cancelled');
}
await expect(response).to.be.rejectedWith('Cancelled');
});
});
});
});