From 98574c2f7d63e1b84664a8f1d00062cddecea93b Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Tue, 18 Mar 2025 13:40:48 -0400 Subject: [PATCH 01/25] Skeleton --- config/karma.base.js | 3 +- packages/vertexai/integration/index.test.ts | 81 +++++++++++++++++++++ packages/vertexai/karma.conf.js | 15 ++-- packages/vertexai/package.json | 1 + 4 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 packages/vertexai/integration/index.test.ts diff --git a/config/karma.base.js b/config/karma.base.js index fe53d3ac744..48cd939c527 100644 --- a/config/karma.base.js +++ b/config/karma.base.js @@ -59,7 +59,8 @@ const config = { // https://npmjs.org/browse/keyword/karma-preprocessor preprocessors: { 'test/**/*.ts': ['webpack', 'sourcemap'], - 'src/**/*.test.ts': ['webpack', 'sourcemap'] + 'src/**/*.test.ts': ['webpack', 'sourcemap'], + 'integration/**/*.test.ts': ['webpack', 'sourcemap'] }, mime: { 'text/x-typescript': ['ts', 'tsx'] }, diff --git a/packages/vertexai/integration/index.test.ts b/packages/vertexai/integration/index.test.ts new file mode 100644 index 00000000000..d6fe7c33a9a --- /dev/null +++ b/packages/vertexai/integration/index.test.ts @@ -0,0 +1,81 @@ +import { initializeApp } from "@firebase/app"; +import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, Modality, SafetySetting, getGenerativeModel, getVertexAI } from "../src"; +import { expect } from "chai"; + +// TODO (dlarocque): Use seperate Firebase config specifically for Vertex AI +// TODO (dlarocque): Load this from environment variables, so we can set the config as a +// secret in CI. +export const config = { + apiKey: "AIzaSyBNHCyZ-bpv-WA-HpXTmigJm2aq3z1kaH8", + authDomain: "jscore-sandbox-141b5.firebaseapp.com", + databaseURL: "https://jscore-sandbox-141b5.firebaseio.com", + projectId: "jscore-sandbox-141b5", + storageBucket: "jscore-sandbox-141b5.appspot.com", + messagingSenderId: "280127633210", + appId: "1:280127633210:web:1eb2f7e8799c4d5a46c203", + measurementId: "G-1VL38N8YFE" +}; + +initializeApp(config); +const MODEL_NAME = 'gemini-1.5-pro'; + +let generationConfig: GenerationConfig = { + temperature: 0, + topP: 0, + topK: 1, + responseMimeType: 'text/plain' +} + +let safetySettings: SafetySetting[] = [ + { + category: HarmCategory.HARM_CATEGORY_HARASSMENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + method: HarmBlockMethod.PROBABILITY + }, + { + category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + method: HarmBlockMethod.SEVERITY + }, + { + category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + }, + { + category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + }, +]; + +let systemInstruction: Content = { + role: 'system', + parts: [ + { + text: 'You are a friendly and helpful assistant.' + } + ] +}; + +describe('VertexAIService', () => { + it('CountTokens text', async () => { + const vertexAI = getVertexAI(); + const model = getGenerativeModel( + vertexAI, + { + model: MODEL_NAME, + generationConfig, + systemInstruction, + safetySettings + } + ); + + let response = await model.countTokens('Why is the sky blue?'); + + expect(response.totalTokens).to.equal(6); + expect(response.totalBillableCharacters).to.equal(16); + expect(response.promptTokensDetails).to.not.be.null; + expect(response.promptTokensDetails!.length).to.equal(1); + expect(response.promptTokensDetails![0].modality).to.equal(Modality.TEXT); + expect(response.promptTokensDetails![0].tokenCount).to.equal(6); + }); +}); \ No newline at end of file diff --git a/packages/vertexai/karma.conf.js b/packages/vertexai/karma.conf.js index 3fe2a2f9633..c316fd6a6f3 100644 --- a/packages/vertexai/karma.conf.js +++ b/packages/vertexai/karma.conf.js @@ -16,14 +16,21 @@ */ const karmaBase = require('../../config/karma.base'); - -const files = [`src/**/*.test.ts`]; +const { argv } = require('yargs'); module.exports = function (config) { const karmaConfig = { ...karmaBase, + // files to load into karma - files, + files: (() => { + if (argv.integration) { + return ['integration/**/*.test.ts']; + } else { + return ['src/**/*.test.ts']; + } + })(), + // frameworks to use // available frameworks: https://npmjs.org/browse/keyword/karma-adapter frameworks: ['mocha'] @@ -31,5 +38,3 @@ module.exports = function (config) { config.set(karmaConfig); }; - -module.exports.files = files; diff --git a/packages/vertexai/package.json b/packages/vertexai/package.json index f26aa2ec2a7..31320a15084 100644 --- a/packages/vertexai/package.json +++ b/packages/vertexai/package.json @@ -39,6 +39,7 @@ "test:ci": "yarn testsetup && node ../../scripts/run_tests_in_ci.js -s test", "test:skip-clone": "karma start", "test:browser": "yarn testsetup && karma start", + "test:integration": "karma start --integration", "api-report": "api-extractor run --local --verbose", "typings:public": "node ../../scripts/build/use_typings.js ./dist/vertexai-public.d.ts", "trusted-type-check": "tsec -p tsconfig.json --noEmit" From ae05e53e87daa1264eb1911b96d4b33f9e1a645a Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 19 Mar 2025 11:32:17 -0400 Subject: [PATCH 02/25] Integration tests --- packages/vertexai/integration/chat.test.ts | 0 .../{index.test.ts => constants.ts} | 40 ++++--------------- .../vertexai/integration/count-tokens.test.ts | 31 ++++++++++++++ .../integration/generate-content.test.ts | 37 +++++++++++++++++ 4 files changed, 75 insertions(+), 33 deletions(-) create mode 100644 packages/vertexai/integration/chat.test.ts rename packages/vertexai/integration/{index.test.ts => constants.ts} (55%) create mode 100644 packages/vertexai/integration/count-tokens.test.ts create mode 100644 packages/vertexai/integration/generate-content.test.ts diff --git a/packages/vertexai/integration/chat.test.ts b/packages/vertexai/integration/chat.test.ts new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/vertexai/integration/index.test.ts b/packages/vertexai/integration/constants.ts similarity index 55% rename from packages/vertexai/integration/index.test.ts rename to packages/vertexai/integration/constants.ts index d6fe7c33a9a..1307ae3675a 100644 --- a/packages/vertexai/integration/index.test.ts +++ b/packages/vertexai/integration/constants.ts @@ -1,6 +1,4 @@ -import { initializeApp } from "@firebase/app"; -import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, Modality, SafetySetting, getGenerativeModel, getVertexAI } from "../src"; -import { expect } from "chai"; +import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, SafetySetting } from "../src"; // TODO (dlarocque): Use seperate Firebase config specifically for Vertex AI // TODO (dlarocque): Load this from environment variables, so we can set the config as a @@ -16,17 +14,17 @@ export const config = { measurementId: "G-1VL38N8YFE" }; -initializeApp(config); -const MODEL_NAME = 'gemini-1.5-pro'; +export const MODEL_NAME = 'gemini-1.5-pro'; -let generationConfig: GenerationConfig = { +/// TODO (dlarocque): Fix the naming on these. +export const generationConfig: GenerationConfig = { temperature: 0, topP: 0, topK: 1, responseMimeType: 'text/plain' } -let safetySettings: SafetySetting[] = [ +export const safetySettings: SafetySetting[] = [ { category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, @@ -47,35 +45,11 @@ let safetySettings: SafetySetting[] = [ }, ]; -let systemInstruction: Content = { +export const systemInstruction: Content = { role: 'system', parts: [ { text: 'You are a friendly and helpful assistant.' } ] -}; - -describe('VertexAIService', () => { - it('CountTokens text', async () => { - const vertexAI = getVertexAI(); - const model = getGenerativeModel( - vertexAI, - { - model: MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings - } - ); - - let response = await model.countTokens('Why is the sky blue?'); - - expect(response.totalTokens).to.equal(6); - expect(response.totalBillableCharacters).to.equal(16); - expect(response.promptTokensDetails).to.not.be.null; - expect(response.promptTokensDetails!.length).to.equal(1); - expect(response.promptTokensDetails![0].modality).to.equal(Modality.TEXT); - expect(response.promptTokensDetails![0].tokenCount).to.equal(6); - }); -}); \ No newline at end of file +}; \ No newline at end of file diff --git a/packages/vertexai/integration/count-tokens.test.ts b/packages/vertexai/integration/count-tokens.test.ts new file mode 100644 index 00000000000..bee93cc0760 --- /dev/null +++ b/packages/vertexai/integration/count-tokens.test.ts @@ -0,0 +1,31 @@ +import { expect } from "chai"; +import { Modality, getGenerativeModel, getVertexAI } from "../src"; +import { MODEL_NAME, generationConfig, systemInstruction, safetySettings, config } from "./constants"; +import { initializeApp } from "@firebase/app"; + +describe('Count Tokens', () => { + + before(() => initializeApp(config)) + + it('CountTokens text', async () => { + const vertexAI = getVertexAI(); + const model = getGenerativeModel( + vertexAI, + { + model: MODEL_NAME, + generationConfig, + systemInstruction, + safetySettings + } + ); + + let response = await model.countTokens('Why is the sky blue?'); + + expect(response.totalTokens).to.equal(6); + expect(response.totalBillableCharacters).to.equal(16); + expect(response.promptTokensDetails).to.not.be.null; + expect(response.promptTokensDetails!.length).to.equal(1); + expect(response.promptTokensDetails![0].modality).to.equal(Modality.TEXT); + expect(response.promptTokensDetails![0].tokenCount).to.equal(6); + }); +}); \ No newline at end of file diff --git a/packages/vertexai/integration/generate-content.test.ts b/packages/vertexai/integration/generate-content.test.ts new file mode 100644 index 00000000000..444156949ff --- /dev/null +++ b/packages/vertexai/integration/generate-content.test.ts @@ -0,0 +1,37 @@ +import { expect } from "chai"; +import { getGenerativeModel, getVertexAI } from "../src"; +import { MODEL_NAME, generationConfig, systemInstruction, safetySettings, config } from "./constants"; +import { initializeApp } from "@firebase/app"; + +// Token counts are only expected to differ by at most this number of tokens. +// Set to 1 for whitespace that is not always present. +const TOKEN_COUNT_DELTA = 1; + +describe('Generate Content', () => { + + before(() => initializeApp(config)) + + it('generateContent', async () => { + const vertexAI = getVertexAI(); + const model = getGenerativeModel( + vertexAI, + { + model: MODEL_NAME, + generationConfig, + systemInstruction, + safetySettings + } + ); + + const result = await model.generateContent("Where is Google headquarters located? Answer with the city name only."); + const response = result.response; + + const trimmedText = response.text().trim(); + expect(trimmedText).to.equal('Mountain View'); + + console.log(JSON.stringify(response)); + + expect(response.usageMetadata).to.not.be.null; + expect(response.usageMetadata!.promptTokenCount).to.be.closeTo(21, TOKEN_COUNT_DELTA); + }); +}); \ No newline at end of file From d83b22e12e6f04b12040cbadf8543e2ce651459c Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 19 Mar 2025 15:42:40 -0400 Subject: [PATCH 03/25] Import firebase config from secret file --- .gitignore | 5 ++++- config/karma.base.js | 2 +- packages/vertexai/integration/chat.test.ts | 0 packages/vertexai/integration/constants.ts | 16 --------------- .../vertexai/integration/count-tokens.test.ts | 11 ++++++++-- .../integration/generate-content.test.ts | 20 ++++++++++++++----- packages/vertexai/karma.conf.js | 10 +++++++++- 7 files changed, 38 insertions(+), 26 deletions(-) delete mode 100644 packages/vertexai/integration/chat.test.ts diff --git a/.gitignore b/.gitignore index 5aaf5c0b5be..7c0943ad49e 100644 --- a/.gitignore +++ b/.gitignore @@ -103,4 +103,7 @@ vertexai-sdk-test-data mocks-lookup.ts # temp changeset output -changeset-temp.json \ No newline at end of file +changeset-temp.json + +# Vertex AI integration test Firebase project config +packages/vertexai/integration/firebase-config.ts \ No newline at end of file diff --git a/config/karma.base.js b/config/karma.base.js index 48cd939c527..79456806476 100644 --- a/config/karma.base.js +++ b/config/karma.base.js @@ -60,7 +60,7 @@ const config = { preprocessors: { 'test/**/*.ts': ['webpack', 'sourcemap'], 'src/**/*.test.ts': ['webpack', 'sourcemap'], - 'integration/**/*.test.ts': ['webpack', 'sourcemap'] + 'integration/**/*.ts': ['webpack', 'sourcemap'] }, mime: { 'text/x-typescript': ['ts', 'tsx'] }, diff --git a/packages/vertexai/integration/chat.test.ts b/packages/vertexai/integration/chat.test.ts deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/packages/vertexai/integration/constants.ts b/packages/vertexai/integration/constants.ts index 1307ae3675a..eaefefd0d48 100644 --- a/packages/vertexai/integration/constants.ts +++ b/packages/vertexai/integration/constants.ts @@ -1,26 +1,10 @@ import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, SafetySetting } from "../src"; -// TODO (dlarocque): Use seperate Firebase config specifically for Vertex AI -// TODO (dlarocque): Load this from environment variables, so we can set the config as a -// secret in CI. -export const config = { - apiKey: "AIzaSyBNHCyZ-bpv-WA-HpXTmigJm2aq3z1kaH8", - authDomain: "jscore-sandbox-141b5.firebaseapp.com", - databaseURL: "https://jscore-sandbox-141b5.firebaseio.com", - projectId: "jscore-sandbox-141b5", - storageBucket: "jscore-sandbox-141b5.appspot.com", - messagingSenderId: "280127633210", - appId: "1:280127633210:web:1eb2f7e8799c4d5a46c203", - measurementId: "G-1VL38N8YFE" -}; - export const MODEL_NAME = 'gemini-1.5-pro'; -/// TODO (dlarocque): Fix the naming on these. export const generationConfig: GenerationConfig = { temperature: 0, topP: 0, - topK: 1, responseMimeType: 'text/plain' } diff --git a/packages/vertexai/integration/count-tokens.test.ts b/packages/vertexai/integration/count-tokens.test.ts index bee93cc0760..7f26f2511cf 100644 --- a/packages/vertexai/integration/count-tokens.test.ts +++ b/packages/vertexai/integration/count-tokens.test.ts @@ -1,11 +1,12 @@ import { expect } from "chai"; import { Modality, getGenerativeModel, getVertexAI } from "../src"; -import { MODEL_NAME, generationConfig, systemInstruction, safetySettings, config } from "./constants"; +import { MODEL_NAME, generationConfig, systemInstruction, safetySettings, } from "./constants"; import { initializeApp } from "@firebase/app"; +import { FIREBASE_CONFIG } from "./firebase-config"; describe('Count Tokens', () => { - before(() => initializeApp(config)) + before(() => initializeApp(FIREBASE_CONFIG)) it('CountTokens text', async () => { const vertexAI = getVertexAI(); @@ -28,4 +29,10 @@ describe('Count Tokens', () => { expect(response.promptTokensDetails![0].modality).to.equal(Modality.TEXT); expect(response.promptTokensDetails![0].tokenCount).to.equal(6); }); + // TODO (dlarocque): Test countTokens() with the following: + // - inline data + // - public storage reference + // - private storage reference (testing auth integration) + // - count tokens + // - JSON schema }); \ No newline at end of file diff --git a/packages/vertexai/integration/generate-content.test.ts b/packages/vertexai/integration/generate-content.test.ts index 444156949ff..9e86a921ea5 100644 --- a/packages/vertexai/integration/generate-content.test.ts +++ b/packages/vertexai/integration/generate-content.test.ts @@ -1,7 +1,8 @@ import { expect } from "chai"; -import { getGenerativeModel, getVertexAI } from "../src"; -import { MODEL_NAME, generationConfig, systemInstruction, safetySettings, config } from "./constants"; +import { Modality, getGenerativeModel, getVertexAI } from "../src"; +import { MODEL_NAME, generationConfig, systemInstruction, safetySettings } from "./constants"; import { initializeApp } from "@firebase/app"; +import { FIREBASE_CONFIG } from "./firebase-config"; // Token counts are only expected to differ by at most this number of tokens. // Set to 1 for whitespace that is not always present. @@ -9,7 +10,7 @@ const TOKEN_COUNT_DELTA = 1; describe('Generate Content', () => { - before(() => initializeApp(config)) + before(() => initializeApp(FIREBASE_CONFIG)) it('generateContent', async () => { const vertexAI = getVertexAI(); @@ -29,9 +30,18 @@ describe('Generate Content', () => { const trimmedText = response.text().trim(); expect(trimmedText).to.equal('Mountain View'); - console.log(JSON.stringify(response)); - expect(response.usageMetadata).to.not.be.null; expect(response.usageMetadata!.promptTokenCount).to.be.closeTo(21, TOKEN_COUNT_DELTA); + expect(response.usageMetadata!.candidatesTokenCount).to.be.closeTo(4, TOKEN_COUNT_DELTA); + expect(response.usageMetadata!.totalTokenCount).to.be.closeTo(25, TOKEN_COUNT_DELTA*2); + expect(response.usageMetadata!.promptTokensDetails).to.not.be.null; + expect(response.usageMetadata!.promptTokensDetails!.length).to.equal(1); + expect(response.usageMetadata!.promptTokensDetails![0].modality).to.equal(Modality.TEXT); + expect(response.usageMetadata!.promptTokensDetails![0].tokenCount).to.equal(21); + expect(response.usageMetadata!.candidatesTokensDetails).to.not.be.null; + expect(response.usageMetadata!.candidatesTokensDetails!.length).to.equal(1); + expect(response.usageMetadata!.candidatesTokensDetails![0].modality).to.equal(Modality.TEXT); + expect(response.usageMetadata!.candidatesTokensDetails![0].tokenCount).to.equal(4); }); + // TODO (dlarocque): Test generateContentStream }); \ No newline at end of file diff --git a/packages/vertexai/karma.conf.js b/packages/vertexai/karma.conf.js index c316fd6a6f3..7f7fe79dff5 100644 --- a/packages/vertexai/karma.conf.js +++ b/packages/vertexai/karma.conf.js @@ -18,6 +18,11 @@ const karmaBase = require('../../config/karma.base'); const { argv } = require('yargs'); +// Validate that required environment variables are defined +if (!process.env.VERTEXAI_INTEGRATION_FIREBASE_CONFIG_JSON) { + throw new Error('VERTEXAI_INTEGRATION_FIREBASE_CONFIG_JSON is not defined in env. Set this env variable to be the JSON of a Firebase project config to run the integration tests with.') +} + module.exports = function (config) { const karmaConfig = { ...karmaBase, @@ -25,7 +30,7 @@ module.exports = function (config) { // files to load into karma files: (() => { if (argv.integration) { - return ['integration/**/*.test.ts']; + return ['integration/**']; } else { return ['src/**/*.test.ts']; } @@ -34,7 +39,10 @@ module.exports = function (config) { // frameworks to use // available frameworks: https://npmjs.org/browse/keyword/karma-adapter frameworks: ['mocha'] + }; + config.client.args.push(process.env.VERTEXAI_INTEGRATION_FIREBASE_CONFIG_JSON); + config.set(karmaConfig); }; From f78282a35269001dacae98c36ce5e9b9ba2fc0d3 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 19 Mar 2025 15:48:54 -0400 Subject: [PATCH 04/25] Validate that config file exists. --- packages/vertexai/karma.conf.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/vertexai/karma.conf.js b/packages/vertexai/karma.conf.js index 7f7fe79dff5..54e960b5b82 100644 --- a/packages/vertexai/karma.conf.js +++ b/packages/vertexai/karma.conf.js @@ -17,10 +17,13 @@ const karmaBase = require('../../config/karma.base'); const { argv } = require('yargs'); +const { existsSync } = require('fs'); -// Validate that required environment variables are defined -if (!process.env.VERTEXAI_INTEGRATION_FIREBASE_CONFIG_JSON) { - throw new Error('VERTEXAI_INTEGRATION_FIREBASE_CONFIG_JSON is not defined in env. Set this env variable to be the JSON of a Firebase project config to run the integration tests with.') +// Validate that the file that defines the Firebase config to be used in the integration tests exists. +if (argv.integration) { + if (!existsSync('integration/firebase-config.ts')) { + throw new Error(`integration/firebase-config.ts does not exist. This file must contain a Firebase config for a project with Vertex AI enabled.`) + } } module.exports = function (config) { From 0d1fb41919303206cf117a0b6776cd6e420c0da6 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 19 Mar 2025 15:50:34 -0400 Subject: [PATCH 05/25] Cleanup --- packages/vertexai/karma.conf.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/vertexai/karma.conf.js b/packages/vertexai/karma.conf.js index 54e960b5b82..a849245432f 100644 --- a/packages/vertexai/karma.conf.js +++ b/packages/vertexai/karma.conf.js @@ -19,6 +19,8 @@ const karmaBase = require('../../config/karma.base'); const { argv } = require('yargs'); const { existsSync } = require('fs'); +const files = [`src/**/*.test.ts`]; + // Validate that the file that defines the Firebase config to be used in the integration tests exists. if (argv.integration) { if (!existsSync('integration/firebase-config.ts')) { @@ -42,10 +44,9 @@ module.exports = function (config) { // frameworks to use // available frameworks: https://npmjs.org/browse/keyword/karma-adapter frameworks: ['mocha'] - }; - config.client.args.push(process.env.VERTEXAI_INTEGRATION_FIREBASE_CONFIG_JSON); - config.set(karmaConfig); }; + +module.exports.files = files; From b083d0087376bbe08e6933a0db768b0e354d523c Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 19 Mar 2025 15:59:38 -0400 Subject: [PATCH 06/25] Format --- packages/vertexai/integration/constants.ts | 38 ++++++-- .../vertexai/integration/count-tokens.test.ts | 54 +++++++---- .../integration/generate-content.test.ts | 91 +++++++++++++------ packages/vertexai/karma.conf.js | 4 +- 4 files changed, 134 insertions(+), 53 deletions(-) diff --git a/packages/vertexai/integration/constants.ts b/packages/vertexai/integration/constants.ts index eaefefd0d48..0219abc3f0c 100644 --- a/packages/vertexai/integration/constants.ts +++ b/packages/vertexai/integration/constants.ts @@ -1,12 +1,36 @@ -import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, SafetySetting } from "../src"; +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -export const MODEL_NAME = 'gemini-1.5-pro'; +import { + Content, + GenerationConfig, + HarmBlockMethod, + HarmBlockThreshold, + HarmCategory, + SafetySetting +} from '../src'; + +export const MODEL_NAME = 'gemini-1.5-pro'; export const generationConfig: GenerationConfig = { temperature: 0, topP: 0, responseMimeType: 'text/plain' -} +}; export const safetySettings: SafetySetting[] = [ { @@ -21,12 +45,12 @@ export const safetySettings: SafetySetting[] = [ }, { category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE }, { category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - }, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + } ]; export const systemInstruction: Content = { @@ -36,4 +60,4 @@ export const systemInstruction: Content = { text: 'You are a friendly and helpful assistant.' } ] -}; \ No newline at end of file +}; diff --git a/packages/vertexai/integration/count-tokens.test.ts b/packages/vertexai/integration/count-tokens.test.ts index 7f26f2511cf..bee8dbe468b 100644 --- a/packages/vertexai/integration/count-tokens.test.ts +++ b/packages/vertexai/integration/count-tokens.test.ts @@ -1,24 +1,42 @@ -import { expect } from "chai"; -import { Modality, getGenerativeModel, getVertexAI } from "../src"; -import { MODEL_NAME, generationConfig, systemInstruction, safetySettings, } from "./constants"; -import { initializeApp } from "@firebase/app"; -import { FIREBASE_CONFIG } from "./firebase-config"; +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -describe('Count Tokens', () => { +import { expect } from 'chai'; +import { Modality, getGenerativeModel, getVertexAI } from '../src'; +import { + MODEL_NAME, + generationConfig, + systemInstruction, + safetySettings +} from './constants'; +import { initializeApp } from '@firebase/app'; +import { FIREBASE_CONFIG } from './firebase-config'; - before(() => initializeApp(FIREBASE_CONFIG)) +describe('Count Tokens', () => { + before(() => initializeApp(FIREBASE_CONFIG)); it('CountTokens text', async () => { - const vertexAI = getVertexAI(); - const model = getGenerativeModel( - vertexAI, - { - model: MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings - } - ); + const vertexAI = getVertexAI(); + const model = getGenerativeModel(vertexAI, { + model: MODEL_NAME, + generationConfig, + systemInstruction, + safetySettings + }); let response = await model.countTokens('Why is the sky blue?'); @@ -35,4 +53,4 @@ describe('Count Tokens', () => { // - private storage reference (testing auth integration) // - count tokens // - JSON schema -}); \ No newline at end of file +}); diff --git a/packages/vertexai/integration/generate-content.test.ts b/packages/vertexai/integration/generate-content.test.ts index 9e86a921ea5..836e2c63587 100644 --- a/packages/vertexai/integration/generate-content.test.ts +++ b/packages/vertexai/integration/generate-content.test.ts @@ -1,47 +1,84 @@ -import { expect } from "chai"; -import { Modality, getGenerativeModel, getVertexAI } from "../src"; -import { MODEL_NAME, generationConfig, systemInstruction, safetySettings } from "./constants"; -import { initializeApp } from "@firebase/app"; -import { FIREBASE_CONFIG } from "./firebase-config"; +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { Modality, getGenerativeModel, getVertexAI } from '../src'; +import { + MODEL_NAME, + generationConfig, + systemInstruction, + safetySettings +} from './constants'; +import { initializeApp } from '@firebase/app'; +import { FIREBASE_CONFIG } from './firebase-config'; // Token counts are only expected to differ by at most this number of tokens. // Set to 1 for whitespace that is not always present. const TOKEN_COUNT_DELTA = 1; describe('Generate Content', () => { - - before(() => initializeApp(FIREBASE_CONFIG)) + before(() => initializeApp(FIREBASE_CONFIG)); it('generateContent', async () => { - const vertexAI = getVertexAI(); - const model = getGenerativeModel( - vertexAI, - { - model: MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings - } - ); + const vertexAI = getVertexAI(); + const model = getGenerativeModel(vertexAI, { + model: MODEL_NAME, + generationConfig, + systemInstruction, + safetySettings + }); - const result = await model.generateContent("Where is Google headquarters located? Answer with the city name only."); + const result = await model.generateContent( + 'Where is Google headquarters located? Answer with the city name only.' + ); const response = result.response; - + const trimmedText = response.text().trim(); expect(trimmedText).to.equal('Mountain View'); expect(response.usageMetadata).to.not.be.null; - expect(response.usageMetadata!.promptTokenCount).to.be.closeTo(21, TOKEN_COUNT_DELTA); - expect(response.usageMetadata!.candidatesTokenCount).to.be.closeTo(4, TOKEN_COUNT_DELTA); - expect(response.usageMetadata!.totalTokenCount).to.be.closeTo(25, TOKEN_COUNT_DELTA*2); + expect(response.usageMetadata!.promptTokenCount).to.be.closeTo( + 21, + TOKEN_COUNT_DELTA + ); + expect(response.usageMetadata!.candidatesTokenCount).to.be.closeTo( + 4, + TOKEN_COUNT_DELTA + ); + expect(response.usageMetadata!.totalTokenCount).to.be.closeTo( + 25, + TOKEN_COUNT_DELTA * 2 + ); expect(response.usageMetadata!.promptTokensDetails).to.not.be.null; expect(response.usageMetadata!.promptTokensDetails!.length).to.equal(1); - expect(response.usageMetadata!.promptTokensDetails![0].modality).to.equal(Modality.TEXT); - expect(response.usageMetadata!.promptTokensDetails![0].tokenCount).to.equal(21); + expect(response.usageMetadata!.promptTokensDetails![0].modality).to.equal( + Modality.TEXT + ); + expect(response.usageMetadata!.promptTokensDetails![0].tokenCount).to.equal( + 21 + ); expect(response.usageMetadata!.candidatesTokensDetails).to.not.be.null; expect(response.usageMetadata!.candidatesTokensDetails!.length).to.equal(1); - expect(response.usageMetadata!.candidatesTokensDetails![0].modality).to.equal(Modality.TEXT); - expect(response.usageMetadata!.candidatesTokensDetails![0].tokenCount).to.equal(4); + expect( + response.usageMetadata!.candidatesTokensDetails![0].modality + ).to.equal(Modality.TEXT); + expect( + response.usageMetadata!.candidatesTokensDetails![0].tokenCount + ).to.equal(4); }); // TODO (dlarocque): Test generateContentStream -}); \ No newline at end of file +}); diff --git a/packages/vertexai/karma.conf.js b/packages/vertexai/karma.conf.js index a849245432f..6eb7d5f5fcf 100644 --- a/packages/vertexai/karma.conf.js +++ b/packages/vertexai/karma.conf.js @@ -24,7 +24,9 @@ const files = [`src/**/*.test.ts`]; // Validate that the file that defines the Firebase config to be used in the integration tests exists. if (argv.integration) { if (!existsSync('integration/firebase-config.ts')) { - throw new Error(`integration/firebase-config.ts does not exist. This file must contain a Firebase config for a project with Vertex AI enabled.`) + throw new Error( + `integration/firebase-config.ts does not exist. This file must contain a Firebase config for a project with Vertex AI enabled.` + ); } } From 4d6a5ce34729e41eecad8a6a2960f47f04b2f453 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 19 Mar 2025 16:06:13 -0400 Subject: [PATCH 07/25] Lint fix --- packages/vertexai/integration/count-tokens.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vertexai/integration/count-tokens.test.ts b/packages/vertexai/integration/count-tokens.test.ts index bee8dbe468b..6d124494f38 100644 --- a/packages/vertexai/integration/count-tokens.test.ts +++ b/packages/vertexai/integration/count-tokens.test.ts @@ -38,7 +38,7 @@ describe('Count Tokens', () => { safetySettings }); - let response = await model.countTokens('Why is the sky blue?'); + const response = await model.countTokens('Why is the sky blue?'); expect(response.totalTokens).to.equal(6); expect(response.totalBillableCharacters).to.equal(16); From 3fbb6d6d53e003436c733166240e349be680024a Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 19 Mar 2025 16:09:30 -0400 Subject: [PATCH 08/25] Extend preprocessor in vertex ai karma conf --- config/karma.base.js | 1 - packages/vertexai/karma.conf.js | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/karma.base.js b/config/karma.base.js index 79456806476..f79037402ce 100644 --- a/config/karma.base.js +++ b/config/karma.base.js @@ -60,7 +60,6 @@ const config = { preprocessors: { 'test/**/*.ts': ['webpack', 'sourcemap'], 'src/**/*.test.ts': ['webpack', 'sourcemap'], - 'integration/**/*.ts': ['webpack', 'sourcemap'] }, mime: { 'text/x-typescript': ['ts', 'tsx'] }, diff --git a/packages/vertexai/karma.conf.js b/packages/vertexai/karma.conf.js index 6eb7d5f5fcf..dc9590e6d44 100644 --- a/packages/vertexai/karma.conf.js +++ b/packages/vertexai/karma.conf.js @@ -34,6 +34,11 @@ module.exports = function (config) { const karmaConfig = { ...karmaBase, + preprocessors: { + ...karmaBase.preprocessors, + 'integration/**/*.ts': ['webpack', 'sourcemap'] + }, + // files to load into karma files: (() => { if (argv.integration) { From d446c77b34e377f4b8f4c1c1c2a81c4c5ac16965 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 19 Mar 2025 16:12:01 -0400 Subject: [PATCH 09/25] format --- config/karma.base.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/karma.base.js b/config/karma.base.js index f79037402ce..fe53d3ac744 100644 --- a/config/karma.base.js +++ b/config/karma.base.js @@ -59,7 +59,7 @@ const config = { // https://npmjs.org/browse/keyword/karma-preprocessor preprocessors: { 'test/**/*.ts': ['webpack', 'sourcemap'], - 'src/**/*.test.ts': ['webpack', 'sourcemap'], + 'src/**/*.test.ts': ['webpack', 'sourcemap'] }, mime: { 'text/x-typescript': ['ts', 'tsx'] }, From fe6528b4c610c7c2d4d99a4e8d424fb9eaa1fae2 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 19 Mar 2025 16:14:14 -0400 Subject: [PATCH 10/25] Add placeholder `integration/firebase-config.ts` --- packages/vertexai/integration/firebase-config.ts | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 packages/vertexai/integration/firebase-config.ts diff --git a/packages/vertexai/integration/firebase-config.ts b/packages/vertexai/integration/firebase-config.ts new file mode 100644 index 00000000000..03a8d591c7a --- /dev/null +++ b/packages/vertexai/integration/firebase-config.ts @@ -0,0 +1,2 @@ +// PLACEHOLDER: This should be replaced with a Firebase config for a project with access to Vertex AI. +export const FIREBASE_CONFIG = {}; \ No newline at end of file From 7ad78332db4b1bbef4be6f5ea6ddd1df34620e42 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 20 Mar 2025 12:08:42 -0400 Subject: [PATCH 11/25] Format --- .../vertexai/integration/firebase-config.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/vertexai/integration/firebase-config.ts b/packages/vertexai/integration/firebase-config.ts index 03a8d591c7a..23c3bb42b05 100644 --- a/packages/vertexai/integration/firebase-config.ts +++ b/packages/vertexai/integration/firebase-config.ts @@ -1,2 +1,19 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // PLACEHOLDER: This should be replaced with a Firebase config for a project with access to Vertex AI. -export const FIREBASE_CONFIG = {}; \ No newline at end of file +export const FIREBASE_CONFIG = {}; From 9f908d20c36a4d2251adb508d53014a80eca3fb9 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 16 May 2025 11:27:29 -0400 Subject: [PATCH 12/25] Add test configs to run against each test --- packages/ai/integration/constants.ts | 70 +++++++++++ packages/ai/integration/count-tokens.test.ts | 89 +++++++++++++ packages/ai/integration/firebase-config.ts | 3 + .../ai/integration/generate-content.test.ts | 117 ++++++++++++++++++ packages/vertexai/integration/constants.ts | 63 ---------- .../vertexai/integration/count-tokens.test.ts | 56 --------- .../vertexai/integration/firebase-config.ts | 19 --- .../integration/generate-content.test.ts | 84 ------------- 8 files changed, 279 insertions(+), 222 deletions(-) create mode 100644 packages/ai/integration/constants.ts create mode 100644 packages/ai/integration/count-tokens.test.ts create mode 100644 packages/ai/integration/firebase-config.ts create mode 100644 packages/ai/integration/generate-content.test.ts delete mode 100644 packages/vertexai/integration/constants.ts delete mode 100644 packages/vertexai/integration/count-tokens.test.ts delete mode 100644 packages/vertexai/integration/firebase-config.ts delete mode 100644 packages/vertexai/integration/generate-content.test.ts diff --git a/packages/ai/integration/constants.ts b/packages/ai/integration/constants.ts new file mode 100644 index 00000000000..cdca655eeba --- /dev/null +++ b/packages/ai/integration/constants.ts @@ -0,0 +1,70 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { initializeApp } from '@firebase/app'; +import { + AI, + Backend, + BackendType, + VertexAIBackend, + getAI +} from '../src'; +import { FIREBASE_CONFIG } from './firebase-config'; + +const app = initializeApp(FIREBASE_CONFIG); + +export type ModelName = 'gemini-2.0-flash' | 'gemini-2.0-flash-exp'; + +/** + * Test config that all tests will be ran against. + */ +export type TestConfig = Readonly<{ + ai: AI; + model: ModelName; + /** This will be used to output the test config at runtime */ + toString: () => string; +}>; + +function formatConfigAsString(config: { ai: AI; model: ModelName }): string { + return `${backendNames.get(config.ai.backend.backendType)} ${config.model}`; +} + +const backends: ReadonlyArray = [ + // new GoogleAIBackend(), TODO: activate once live + new VertexAIBackend() +]; + +const backendNames: Map = new Map([ + [BackendType.GOOGLE_AI, 'Google AI'], + [BackendType.VERTEX_AI, 'Vertex AI'] +]); + +const modelNames: ReadonlyArray = [ + 'gemini-2.0-flash', + 'gemini-2.0-flash-exp' +]; + +export const testConfigs: ReadonlyArray = backends.flatMap(backend => { + return modelNames.map(modelName => { + const ai = getAI(app, { backend }); + return { + ai: getAI(app, { backend }), + model: modelName, + toString: () => formatConfigAsString({ ai, model: modelName }) + } + }) +}) \ No newline at end of file diff --git a/packages/ai/integration/count-tokens.test.ts b/packages/ai/integration/count-tokens.test.ts new file mode 100644 index 00000000000..6b356850a73 --- /dev/null +++ b/packages/ai/integration/count-tokens.test.ts @@ -0,0 +1,89 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, Modality, SafetySetting, getAI, getGenerativeModel, getVertexAI } from '../src'; +import { + testConfigs +} from './constants'; + +describe('Count Tokens', () => { + + testConfigs.forEach(testConfig => { + describe(`${testConfig.toString()}`, () => { + + it('CountTokens text', async () => { + const generationConfig: GenerationConfig = { + temperature: 0, + topP: 0, + responseMimeType: 'text/plain' + }; + + const safetySettings: SafetySetting[] = [ + { + category: HarmCategory.HARM_CATEGORY_HARASSMENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + method: HarmBlockMethod.PROBABILITY + }, + { + category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + method: HarmBlockMethod.SEVERITY + }, + { + category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + }, + { + category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + } + ]; + + const systemInstruction: Content = { + role: 'system', + parts: [ + { + text: 'You are a friendly and helpful assistant.' + } + ] + }; + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig, + systemInstruction, + safetySettings + }); + + const response = await model.countTokens('Why is the sky blue?'); + + expect(response.totalTokens).to.equal(6); + expect(response.totalBillableCharacters).to.equal(16); + expect(response.promptTokensDetails).to.not.be.null; + expect(response.promptTokensDetails!.length).to.equal(1); + expect(response.promptTokensDetails![0].modality).to.equal(Modality.TEXT); + expect(response.promptTokensDetails![0].tokenCount).to.equal(6); + }); + // TODO (dlarocque): Test countTokens() with the following: + // - inline data + // - public storage reference + // - private storage reference (testing auth integration) + // - count tokens + // - JSON schema + }); + }) +}); diff --git a/packages/ai/integration/firebase-config.ts b/packages/ai/integration/firebase-config.ts new file mode 100644 index 00000000000..f04097001ed --- /dev/null +++ b/packages/ai/integration/firebase-config.ts @@ -0,0 +1,3 @@ +import * as config from '../../../config/project.json'; + +export const FIREBASE_CONFIG = config; \ No newline at end of file diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts new file mode 100644 index 00000000000..bb85fdcde7c --- /dev/null +++ b/packages/ai/integration/generate-content.test.ts @@ -0,0 +1,117 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, Modality, SafetySetting, getGenerativeModel } from '../src'; +import { + testConfigs +} from './constants'; + +// Token counts are only expected to differ by at most this number of tokens. +// Set to 1 for whitespace that is not always present. +const TOKEN_COUNT_DELTA = 1; + +describe('Generate Content', () => { + testConfigs.forEach(testConfig => { + describe(`${testConfig.toString()}`, () => { + + it('generateContent', async () => { + const generationConfig: GenerationConfig = { + temperature: 0, + topP: 0, + responseMimeType: 'text/plain' + }; + + const safetySettings: SafetySetting[] = [ + { + category: HarmCategory.HARM_CATEGORY_HARASSMENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + method: HarmBlockMethod.PROBABILITY + }, + { + category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + method: HarmBlockMethod.SEVERITY + }, + { + category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + }, + { + category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + } + ]; + + const systemInstruction: Content = { + role: 'system', + parts: [ + { + text: 'You are a friendly and helpful assistant.' + } + ] + }; + + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig, + safetySettings, + systemInstruction + }); + + const result = await model.generateContent( + 'Where is Google headquarters located? Answer with the city name only.' + ); + const response = result.response; + + const trimmedText = response.text().trim(); + expect(trimmedText).to.equal('Mountain View'); + + expect(response.usageMetadata).to.not.be.null; + expect(response.usageMetadata!.promptTokenCount).to.be.closeTo( + 21, + TOKEN_COUNT_DELTA + ); + expect(response.usageMetadata!.candidatesTokenCount).to.be.closeTo( + 4, + TOKEN_COUNT_DELTA + ); + expect(response.usageMetadata!.totalTokenCount).to.be.closeTo( + 25, + TOKEN_COUNT_DELTA * 2 + ); + expect(response.usageMetadata!.promptTokensDetails).to.not.be.null; + expect(response.usageMetadata!.promptTokensDetails!.length).to.equal(1); + expect(response.usageMetadata!.promptTokensDetails![0].modality).to.equal( + Modality.TEXT + ); + expect(response.usageMetadata!.promptTokensDetails![0].tokenCount).to.equal( + 21 + ); + expect(response.usageMetadata!.candidatesTokensDetails).to.not.be.null; + expect(response.usageMetadata!.candidatesTokensDetails!.length).to.equal(1); + expect( + response.usageMetadata!.candidatesTokensDetails![0].modality + ).to.equal(Modality.TEXT); + expect( + response.usageMetadata!.candidatesTokensDetails![0].tokenCount + ).to.be.closeTo(4, TOKEN_COUNT_DELTA); + }); + // TODO (dlarocque): Test generateContentStream + }) + }) +}); diff --git a/packages/vertexai/integration/constants.ts b/packages/vertexai/integration/constants.ts deleted file mode 100644 index 0219abc3f0c..00000000000 --- a/packages/vertexai/integration/constants.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * @license - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Content, - GenerationConfig, - HarmBlockMethod, - HarmBlockThreshold, - HarmCategory, - SafetySetting -} from '../src'; - -export const MODEL_NAME = 'gemini-1.5-pro'; - -export const generationConfig: GenerationConfig = { - temperature: 0, - topP: 0, - responseMimeType: 'text/plain' -}; - -export const safetySettings: SafetySetting[] = [ - { - category: HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - method: HarmBlockMethod.PROBABILITY - }, - { - category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - method: HarmBlockMethod.SEVERITY - }, - { - category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE - }, - { - category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE - } -]; - -export const systemInstruction: Content = { - role: 'system', - parts: [ - { - text: 'You are a friendly and helpful assistant.' - } - ] -}; diff --git a/packages/vertexai/integration/count-tokens.test.ts b/packages/vertexai/integration/count-tokens.test.ts deleted file mode 100644 index 6d124494f38..00000000000 --- a/packages/vertexai/integration/count-tokens.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** - * @license - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { expect } from 'chai'; -import { Modality, getGenerativeModel, getVertexAI } from '../src'; -import { - MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings -} from './constants'; -import { initializeApp } from '@firebase/app'; -import { FIREBASE_CONFIG } from './firebase-config'; - -describe('Count Tokens', () => { - before(() => initializeApp(FIREBASE_CONFIG)); - - it('CountTokens text', async () => { - const vertexAI = getVertexAI(); - const model = getGenerativeModel(vertexAI, { - model: MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings - }); - - const response = await model.countTokens('Why is the sky blue?'); - - expect(response.totalTokens).to.equal(6); - expect(response.totalBillableCharacters).to.equal(16); - expect(response.promptTokensDetails).to.not.be.null; - expect(response.promptTokensDetails!.length).to.equal(1); - expect(response.promptTokensDetails![0].modality).to.equal(Modality.TEXT); - expect(response.promptTokensDetails![0].tokenCount).to.equal(6); - }); - // TODO (dlarocque): Test countTokens() with the following: - // - inline data - // - public storage reference - // - private storage reference (testing auth integration) - // - count tokens - // - JSON schema -}); diff --git a/packages/vertexai/integration/firebase-config.ts b/packages/vertexai/integration/firebase-config.ts deleted file mode 100644 index 23c3bb42b05..00000000000 --- a/packages/vertexai/integration/firebase-config.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** - * @license - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// PLACEHOLDER: This should be replaced with a Firebase config for a project with access to Vertex AI. -export const FIREBASE_CONFIG = {}; diff --git a/packages/vertexai/integration/generate-content.test.ts b/packages/vertexai/integration/generate-content.test.ts deleted file mode 100644 index 836e2c63587..00000000000 --- a/packages/vertexai/integration/generate-content.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -/** - * @license - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { expect } from 'chai'; -import { Modality, getGenerativeModel, getVertexAI } from '../src'; -import { - MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings -} from './constants'; -import { initializeApp } from '@firebase/app'; -import { FIREBASE_CONFIG } from './firebase-config'; - -// Token counts are only expected to differ by at most this number of tokens. -// Set to 1 for whitespace that is not always present. -const TOKEN_COUNT_DELTA = 1; - -describe('Generate Content', () => { - before(() => initializeApp(FIREBASE_CONFIG)); - - it('generateContent', async () => { - const vertexAI = getVertexAI(); - const model = getGenerativeModel(vertexAI, { - model: MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings - }); - - const result = await model.generateContent( - 'Where is Google headquarters located? Answer with the city name only.' - ); - const response = result.response; - - const trimmedText = response.text().trim(); - expect(trimmedText).to.equal('Mountain View'); - - expect(response.usageMetadata).to.not.be.null; - expect(response.usageMetadata!.promptTokenCount).to.be.closeTo( - 21, - TOKEN_COUNT_DELTA - ); - expect(response.usageMetadata!.candidatesTokenCount).to.be.closeTo( - 4, - TOKEN_COUNT_DELTA - ); - expect(response.usageMetadata!.totalTokenCount).to.be.closeTo( - 25, - TOKEN_COUNT_DELTA * 2 - ); - expect(response.usageMetadata!.promptTokensDetails).to.not.be.null; - expect(response.usageMetadata!.promptTokensDetails!.length).to.equal(1); - expect(response.usageMetadata!.promptTokensDetails![0].modality).to.equal( - Modality.TEXT - ); - expect(response.usageMetadata!.promptTokensDetails![0].tokenCount).to.equal( - 21 - ); - expect(response.usageMetadata!.candidatesTokensDetails).to.not.be.null; - expect(response.usageMetadata!.candidatesTokensDetails!.length).to.equal(1); - expect( - response.usageMetadata!.candidatesTokensDetails![0].modality - ).to.equal(Modality.TEXT); - expect( - response.usageMetadata!.candidatesTokensDetails![0].tokenCount - ).to.equal(4); - }); - // TODO (dlarocque): Test generateContentStream -}); From abe830cb9d185b8ce5809534ec19cb6806826d70 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 16 May 2025 11:27:29 -0400 Subject: [PATCH 13/25] Add test configs to run against each test --- packages/ai/integration/constants.ts | 70 +++++++++++ packages/ai/integration/count-tokens.test.ts | 106 ++++++++++++++++ packages/ai/integration/firebase-config.ts | 3 + .../ai/integration/generate-content.test.ts | 117 ++++++++++++++++++ packages/vertexai/integration/constants.ts | 63 ---------- .../vertexai/integration/count-tokens.test.ts | 56 --------- .../vertexai/integration/firebase-config.ts | 19 --- .../integration/generate-content.test.ts | 84 ------------- 8 files changed, 296 insertions(+), 222 deletions(-) create mode 100644 packages/ai/integration/constants.ts create mode 100644 packages/ai/integration/count-tokens.test.ts create mode 100644 packages/ai/integration/firebase-config.ts create mode 100644 packages/ai/integration/generate-content.test.ts delete mode 100644 packages/vertexai/integration/constants.ts delete mode 100644 packages/vertexai/integration/count-tokens.test.ts delete mode 100644 packages/vertexai/integration/firebase-config.ts delete mode 100644 packages/vertexai/integration/generate-content.test.ts diff --git a/packages/ai/integration/constants.ts b/packages/ai/integration/constants.ts new file mode 100644 index 00000000000..cdca655eeba --- /dev/null +++ b/packages/ai/integration/constants.ts @@ -0,0 +1,70 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { initializeApp } from '@firebase/app'; +import { + AI, + Backend, + BackendType, + VertexAIBackend, + getAI +} from '../src'; +import { FIREBASE_CONFIG } from './firebase-config'; + +const app = initializeApp(FIREBASE_CONFIG); + +export type ModelName = 'gemini-2.0-flash' | 'gemini-2.0-flash-exp'; + +/** + * Test config that all tests will be ran against. + */ +export type TestConfig = Readonly<{ + ai: AI; + model: ModelName; + /** This will be used to output the test config at runtime */ + toString: () => string; +}>; + +function formatConfigAsString(config: { ai: AI; model: ModelName }): string { + return `${backendNames.get(config.ai.backend.backendType)} ${config.model}`; +} + +const backends: ReadonlyArray = [ + // new GoogleAIBackend(), TODO: activate once live + new VertexAIBackend() +]; + +const backendNames: Map = new Map([ + [BackendType.GOOGLE_AI, 'Google AI'], + [BackendType.VERTEX_AI, 'Vertex AI'] +]); + +const modelNames: ReadonlyArray = [ + 'gemini-2.0-flash', + 'gemini-2.0-flash-exp' +]; + +export const testConfigs: ReadonlyArray = backends.flatMap(backend => { + return modelNames.map(modelName => { + const ai = getAI(app, { backend }); + return { + ai: getAI(app, { backend }), + model: modelName, + toString: () => formatConfigAsString({ ai, model: modelName }) + } + }) +}) \ No newline at end of file diff --git a/packages/ai/integration/count-tokens.test.ts b/packages/ai/integration/count-tokens.test.ts new file mode 100644 index 00000000000..47769d63844 --- /dev/null +++ b/packages/ai/integration/count-tokens.test.ts @@ -0,0 +1,106 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, Modality, SafetySetting, getAI, getGenerativeModel, getVertexAI } from '../src'; +import { + testConfigs +} from './constants'; + +describe('Count Tokens', () => { + testConfigs.forEach(testConfig => { + describe(`${testConfig.toString()}`, () => { + + it('text input', async () => { + const generationConfig: GenerationConfig = { + temperature: 0, + topP: 0, + responseMimeType: 'text/plain' + }; + + const safetySettings: SafetySetting[] = [ + { + category: HarmCategory.HARM_CATEGORY_HARASSMENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + method: HarmBlockMethod.PROBABILITY + }, + { + category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + method: HarmBlockMethod.SEVERITY + }, + { + category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + }, + { + category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + } + ]; + + const systemInstruction: Content = { + role: 'system', + parts: [ + { + text: 'You are a friendly and helpful assistant.' + } + ] + }; + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig, + systemInstruction, + safetySettings + }); + + const response = await model.countTokens('Why is the sky blue?'); + + expect(response.totalTokens).to.equal(6); + expect(response.totalBillableCharacters).to.equal(16); + expect(response.promptTokensDetails).to.not.be.null; + expect(response.promptTokensDetails!.length).to.equal(1); + expect(response.promptTokensDetails![0].modality).to.equal(Modality.TEXT); + expect(response.promptTokensDetails![0].tokenCount).to.equal(6); + }); + it('image input', async () => { + + }) + it('audio input', async () => { + + }) + it('text, image, and audio input', async () => { + + }) + it('public storage reference', async () => { + + }) + it('private storage reference', async () => { + + }) + it('schema', async () => { + + }) + // TODO (dlarocque): Test countTokens() with the following: + // - inline data + // - public storage reference + // - private storage reference (testing auth integration) + // - count tokens + // - JSON schema + }); + }) +}); diff --git a/packages/ai/integration/firebase-config.ts b/packages/ai/integration/firebase-config.ts new file mode 100644 index 00000000000..f04097001ed --- /dev/null +++ b/packages/ai/integration/firebase-config.ts @@ -0,0 +1,3 @@ +import * as config from '../../../config/project.json'; + +export const FIREBASE_CONFIG = config; \ No newline at end of file diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts new file mode 100644 index 00000000000..203f4d4f51c --- /dev/null +++ b/packages/ai/integration/generate-content.test.ts @@ -0,0 +1,117 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, Modality, SafetySetting, getGenerativeModel } from '../src'; +import { + testConfigs +} from './constants'; + +// Token counts are only expected to differ by at most this number of tokens. +// Set to 1 for whitespace that is not always present. +const TOKEN_COUNT_DELTA = 1; + +describe('Generate Content', () => { + testConfigs.forEach(testConfig => { + describe(`${testConfig.toString()}`, () => { + + it('text input, text output', async () => { + const generationConfig: GenerationConfig = { + temperature: 0, + topP: 0, + responseMimeType: 'text/plain' + }; + + const safetySettings: SafetySetting[] = [ + { + category: HarmCategory.HARM_CATEGORY_HARASSMENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + method: HarmBlockMethod.PROBABILITY + }, + { + category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + method: HarmBlockMethod.SEVERITY + }, + { + category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + }, + { + category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + } + ]; + + const systemInstruction: Content = { + role: 'system', + parts: [ + { + text: 'You are a friendly and helpful assistant.' + } + ] + }; + + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig, + safetySettings, + systemInstruction + }); + + const result = await model.generateContent( + 'Where is Google headquarters located? Answer with the city name only.' + ); + const response = result.response; + + const trimmedText = response.text().trim(); + expect(trimmedText).to.equal('Mountain View'); + + expect(response.usageMetadata).to.not.be.null; + expect(response.usageMetadata!.promptTokenCount).to.be.closeTo( + 21, + TOKEN_COUNT_DELTA + ); + expect(response.usageMetadata!.candidatesTokenCount).to.be.closeTo( + 4, + TOKEN_COUNT_DELTA + ); + expect(response.usageMetadata!.totalTokenCount).to.be.closeTo( + 25, + TOKEN_COUNT_DELTA * 2 + ); + expect(response.usageMetadata!.promptTokensDetails).to.not.be.null; + expect(response.usageMetadata!.promptTokensDetails!.length).to.equal(1); + expect(response.usageMetadata!.promptTokensDetails![0].modality).to.equal( + Modality.TEXT + ); + expect(response.usageMetadata!.promptTokensDetails![0].tokenCount).to.equal( + 21 + ); + expect(response.usageMetadata!.candidatesTokensDetails).to.not.be.null; + expect(response.usageMetadata!.candidatesTokensDetails!.length).to.equal(1); + expect( + response.usageMetadata!.candidatesTokensDetails![0].modality + ).to.equal(Modality.TEXT); + expect( + response.usageMetadata!.candidatesTokensDetails![0].tokenCount + ).to.be.closeTo(4, TOKEN_COUNT_DELTA); + }); + // TODO (dlarocque): Test generateContentStream + }) + }) +}); diff --git a/packages/vertexai/integration/constants.ts b/packages/vertexai/integration/constants.ts deleted file mode 100644 index 0219abc3f0c..00000000000 --- a/packages/vertexai/integration/constants.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * @license - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Content, - GenerationConfig, - HarmBlockMethod, - HarmBlockThreshold, - HarmCategory, - SafetySetting -} from '../src'; - -export const MODEL_NAME = 'gemini-1.5-pro'; - -export const generationConfig: GenerationConfig = { - temperature: 0, - topP: 0, - responseMimeType: 'text/plain' -}; - -export const safetySettings: SafetySetting[] = [ - { - category: HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - method: HarmBlockMethod.PROBABILITY - }, - { - category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - method: HarmBlockMethod.SEVERITY - }, - { - category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE - }, - { - category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE - } -]; - -export const systemInstruction: Content = { - role: 'system', - parts: [ - { - text: 'You are a friendly and helpful assistant.' - } - ] -}; diff --git a/packages/vertexai/integration/count-tokens.test.ts b/packages/vertexai/integration/count-tokens.test.ts deleted file mode 100644 index 6d124494f38..00000000000 --- a/packages/vertexai/integration/count-tokens.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** - * @license - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { expect } from 'chai'; -import { Modality, getGenerativeModel, getVertexAI } from '../src'; -import { - MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings -} from './constants'; -import { initializeApp } from '@firebase/app'; -import { FIREBASE_CONFIG } from './firebase-config'; - -describe('Count Tokens', () => { - before(() => initializeApp(FIREBASE_CONFIG)); - - it('CountTokens text', async () => { - const vertexAI = getVertexAI(); - const model = getGenerativeModel(vertexAI, { - model: MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings - }); - - const response = await model.countTokens('Why is the sky blue?'); - - expect(response.totalTokens).to.equal(6); - expect(response.totalBillableCharacters).to.equal(16); - expect(response.promptTokensDetails).to.not.be.null; - expect(response.promptTokensDetails!.length).to.equal(1); - expect(response.promptTokensDetails![0].modality).to.equal(Modality.TEXT); - expect(response.promptTokensDetails![0].tokenCount).to.equal(6); - }); - // TODO (dlarocque): Test countTokens() with the following: - // - inline data - // - public storage reference - // - private storage reference (testing auth integration) - // - count tokens - // - JSON schema -}); diff --git a/packages/vertexai/integration/firebase-config.ts b/packages/vertexai/integration/firebase-config.ts deleted file mode 100644 index 23c3bb42b05..00000000000 --- a/packages/vertexai/integration/firebase-config.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** - * @license - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// PLACEHOLDER: This should be replaced with a Firebase config for a project with access to Vertex AI. -export const FIREBASE_CONFIG = {}; diff --git a/packages/vertexai/integration/generate-content.test.ts b/packages/vertexai/integration/generate-content.test.ts deleted file mode 100644 index 836e2c63587..00000000000 --- a/packages/vertexai/integration/generate-content.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -/** - * @license - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { expect } from 'chai'; -import { Modality, getGenerativeModel, getVertexAI } from '../src'; -import { - MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings -} from './constants'; -import { initializeApp } from '@firebase/app'; -import { FIREBASE_CONFIG } from './firebase-config'; - -// Token counts are only expected to differ by at most this number of tokens. -// Set to 1 for whitespace that is not always present. -const TOKEN_COUNT_DELTA = 1; - -describe('Generate Content', () => { - before(() => initializeApp(FIREBASE_CONFIG)); - - it('generateContent', async () => { - const vertexAI = getVertexAI(); - const model = getGenerativeModel(vertexAI, { - model: MODEL_NAME, - generationConfig, - systemInstruction, - safetySettings - }); - - const result = await model.generateContent( - 'Where is Google headquarters located? Answer with the city name only.' - ); - const response = result.response; - - const trimmedText = response.text().trim(); - expect(trimmedText).to.equal('Mountain View'); - - expect(response.usageMetadata).to.not.be.null; - expect(response.usageMetadata!.promptTokenCount).to.be.closeTo( - 21, - TOKEN_COUNT_DELTA - ); - expect(response.usageMetadata!.candidatesTokenCount).to.be.closeTo( - 4, - TOKEN_COUNT_DELTA - ); - expect(response.usageMetadata!.totalTokenCount).to.be.closeTo( - 25, - TOKEN_COUNT_DELTA * 2 - ); - expect(response.usageMetadata!.promptTokensDetails).to.not.be.null; - expect(response.usageMetadata!.promptTokensDetails!.length).to.equal(1); - expect(response.usageMetadata!.promptTokensDetails![0].modality).to.equal( - Modality.TEXT - ); - expect(response.usageMetadata!.promptTokensDetails![0].tokenCount).to.equal( - 21 - ); - expect(response.usageMetadata!.candidatesTokensDetails).to.not.be.null; - expect(response.usageMetadata!.candidatesTokensDetails!.length).to.equal(1); - expect( - response.usageMetadata!.candidatesTokensDetails![0].modality - ).to.equal(Modality.TEXT); - expect( - response.usageMetadata!.candidatesTokensDetails![0].tokenCount - ).to.equal(4); - }); - // TODO (dlarocque): Test generateContentStream -}); From 472c452ac13d1dbb1630a5a69328229a27944f0b Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 16 May 2025 14:45:19 -0400 Subject: [PATCH 14/25] count tokens tests --- packages/ai/integration/constants.ts | 17 +-- packages/ai/integration/count-tokens.test.ts | 108 ++++++++++++++++--- 2 files changed, 102 insertions(+), 23 deletions(-) diff --git a/packages/ai/integration/constants.ts b/packages/ai/integration/constants.ts index cdca655eeba..32e833da256 100644 --- a/packages/ai/integration/constants.ts +++ b/packages/ai/integration/constants.ts @@ -27,19 +27,17 @@ import { FIREBASE_CONFIG } from './firebase-config'; const app = initializeApp(FIREBASE_CONFIG); -export type ModelName = 'gemini-2.0-flash' | 'gemini-2.0-flash-exp'; - /** * Test config that all tests will be ran against. */ export type TestConfig = Readonly<{ ai: AI; - model: ModelName; + model: string; /** This will be used to output the test config at runtime */ toString: () => string; }>; -function formatConfigAsString(config: { ai: AI; model: ModelName }): string { +function formatConfigAsString(config: { ai: AI; model: string }): string { return `${backendNames.get(config.ai.backend.backendType)} ${config.model}`; } @@ -53,9 +51,9 @@ const backendNames: Map = new Map([ [BackendType.VERTEX_AI, 'Vertex AI'] ]); -const modelNames: ReadonlyArray = [ +const modelNames: ReadonlyArray = [ 'gemini-2.0-flash', - 'gemini-2.0-flash-exp' + // 'gemini-2.0-flash-exp' ]; export const testConfigs: ReadonlyArray = backends.flatMap(backend => { @@ -67,4 +65,9 @@ export const testConfigs: ReadonlyArray = backends.flatMap(backend = toString: () => formatConfigAsString({ ai, model: modelName }) } }) -}) \ No newline at end of file +}) + +export const TINY_IMG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII='; +export const IMAGE_MIME_TYPE = 'image/png'; +export const TINY_MP3_BASE64 = 'SUQzBAAAAAAAIlRTU0UAAAAOAAADTGF2ZjYxLjcuMTAwAAAAAAAAAAAAAAD/+0DAAAAAAAAAAAAAAAAAAAAAAABJbmZvAAAADwAAAAUAAAK+AGhoaGhoaGhoaGhoaGhoaGhoaGiOjo6Ojo6Ojo6Ojo6Ojo6Ojo6OjrS0tLS0tLS0tLS0tLS0tLS0tLS02tra2tra2tra2tra2tra2tra2tr//////////////////////////wAAAABMYXZjNjEuMTkAAAAAAAAAAAAAAAAkAwYAAAAAAAACvhC6DYoAAAAAAP/7EMQAA8AAAaQAAAAgAAA0gAAABExBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV//sQxCmDwAABpAAAACAAADSAAAAEVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+xDEUwPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVf/7EMR8g8AAAaQAAAAgAAA0gAAABFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV//sQxKYDwAABpAAAACAAADSAAAAEVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVU='; +export const AUDIO_MIME_TYPE = 'audio/mpeg'; \ No newline at end of file diff --git a/packages/ai/integration/count-tokens.test.ts b/packages/ai/integration/count-tokens.test.ts index 47769d63844..773fae54ee1 100644 --- a/packages/ai/integration/count-tokens.test.ts +++ b/packages/ai/integration/count-tokens.test.ts @@ -16,10 +16,30 @@ */ import { expect } from 'chai'; -import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, Modality, SafetySetting, getAI, getGenerativeModel, getVertexAI } from '../src'; import { + Content, + GenerationConfig, + HarmBlockMethod, + HarmBlockThreshold, + HarmCategory, + Modality, + SafetySetting, + getGenerativeModel, + Part, + CountTokensRequest, + Schema, + InlineDataPart, + FileDataPart +} from '../src'; +import { + AUDIO_MIME_TYPE, + IMAGE_MIME_TYPE, + TINY_IMG_BASE64, + TINY_MP3_BASE64, testConfigs } from './constants'; +import { FIREBASE_CONFIG } from './firebase-config'; + describe('Count Tokens', () => { testConfigs.forEach(testConfig => { @@ -77,30 +97,86 @@ describe('Count Tokens', () => { expect(response.promptTokensDetails![0].modality).to.equal(Modality.TEXT); expect(response.promptTokensDetails![0].tokenCount).to.equal(6); }); + it('image input', async () => { + const model = getGenerativeModel(testConfig.ai, { model: testConfig.model }); + const imagePart: Part = { + inlineData: { + mimeType: IMAGE_MIME_TYPE, + data: TINY_IMG_BASE64 + } + }; + const response = await model.countTokens([imagePart]); + + const expectedImageTokens = 258; + expect(response.totalTokens, 'totalTokens should have correct token count').to.equal(expectedImageTokens); + expect(response.totalBillableCharacters, 'totalBillableCharacters should be undefined').to.be.undefined; // Incorrect behavior + expect(response.promptTokensDetails!.length, 'promptTokensDetails should have one entry').to.equal(1); + expect(response.promptTokensDetails![0].modality, 'modality should be IMAGE').to.equal(Modality.IMAGE); + expect(response.promptTokensDetails![0].tokenCount, 'promptTokenDetails tokenCount should be correct').to.equal(expectedImageTokens); + }); - }) it('audio input', async () => { + const model = getGenerativeModel(testConfig.ai, { model: testConfig.model }); + const audioPart: InlineDataPart = { + inlineData: { + mimeType: AUDIO_MIME_TYPE, + data: TINY_MP3_BASE64 + } + }; + + const response = await model.countTokens([audioPart]); + // This may be different on Google AI + expect(response.totalTokens, 'totalTokens is expected to be undefined').to.be.undefined; + expect(response.totalBillableCharacters, 'totalBillableCharacters should be undefined').to.be.undefined; // Incorrect behavior + expect(response.promptTokensDetails!.length, 'promptTokensDetails should have one entry').to.equal(1); + expect(response.promptTokensDetails![0].modality, 'modality should be AUDIO').to.equal(Modality.AUDIO); + expect(response.promptTokensDetails![0].tokenCount, 'promptTokenDetails tokenCount is expected to be undefined').to.be.undefined; + }); - }) it('text, image, and audio input', async () => { + const model = getGenerativeModel(testConfig.ai, { model: testConfig.model }); + const textPart: Part = { text: 'Describe these:' }; + const imagePart: Part = { inlineData: { mimeType: IMAGE_MIME_TYPE, data: TINY_IMG_BASE64 } }; + const audioPart: Part = { inlineData: { mimeType: AUDIO_MIME_TYPE, data: TINY_MP3_BASE64 } }; - }) - it('public storage reference', async () => { + const request: CountTokensRequest = { + contents: [{ role: 'user', parts: [textPart, imagePart, audioPart] }] + }; + const response = await model.countTokens(request); + + expect(response.totalTokens, 'totalTokens should have correct token count').to.equal(261); + expect(response.totalBillableCharacters, 'totalBillableCharacters should have correct count').to.equal('Describe these:'.length - 1); // For some reason it's the length-1 + + expect(response.promptTokensDetails!.length, 'promptTokensDetails should have three entries').to.equal(3); - }) - it('private storage reference', async () => { + const textDetails = response.promptTokensDetails!.find(d => d.modality === Modality.TEXT); + const visionDetails = response.promptTokensDetails!.find(d => d.modality === Modality.IMAGE); + const audioDetails = response.promptTokensDetails!.find(d => d.modality === Modality.AUDIO); - }) - it('schema', async () => { + expect(textDetails).to.deep.equal({ modality: Modality.TEXT, tokenCount: 3 }); + expect(visionDetails).to.deep.equal({ modality: Modality.IMAGE, tokenCount: 258 }); + expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO }); // Incorrect behavior because there's no tokenCount + }); - }) - // TODO (dlarocque): Test countTokens() with the following: - // - inline data - // - public storage reference - // - private storage reference (testing auth integration) - // - count tokens - // - JSON schema + it('public storage reference', async () => { + const model = getGenerativeModel(testConfig.ai, { model: testConfig.model }); + const filePart: FileDataPart = { + fileData: { + mimeType: IMAGE_MIME_TYPE, + fileUri: `gs://${FIREBASE_CONFIG.storageBucket}/images/tree.png` + } + }; + const response = await model.countTokens([filePart]); + + const expectedFileTokens = 258; + expect(response.totalTokens, 'totalTokens should have correct token count').to.equal(expectedFileTokens); + expect(response.totalBillableCharacters, 'totalBillableCharacters should be undefined').to.be.undefined; + expect(response.promptTokensDetails).to.not.be.null; + expect(response.promptTokensDetails!.length).to.equal(1); + expect(response.promptTokensDetails![0].modality).to.equal(Modality.IMAGE); + expect(response.promptTokensDetails![0].tokenCount).to.equal(expectedFileTokens); + }); }); }) }); From 681b4bd7eb017f46ea7dd8763ae1ecadb20ad68b Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 16 May 2025 14:48:16 -0400 Subject: [PATCH 15/25] Format and fix lint issues --- packages/ai/integration/constants.ts | 34 ++--- packages/ai/integration/count-tokens.test.ts | 142 +++++++++++++----- packages/ai/integration/firebase-config.ts | 19 ++- .../ai/integration/generate-content.test.ts | 34 +++-- 4 files changed, 162 insertions(+), 67 deletions(-) diff --git a/packages/ai/integration/constants.ts b/packages/ai/integration/constants.ts index 32e833da256..e2074c07851 100644 --- a/packages/ai/integration/constants.ts +++ b/packages/ai/integration/constants.ts @@ -16,13 +16,7 @@ */ import { initializeApp } from '@firebase/app'; -import { - AI, - Backend, - BackendType, - VertexAIBackend, - getAI -} from '../src'; +import { AI, Backend, BackendType, VertexAIBackend, getAI } from '../src'; import { FIREBASE_CONFIG } from './firebase-config'; const app = initializeApp(FIREBASE_CONFIG); @@ -41,7 +35,7 @@ function formatConfigAsString(config: { ai: AI; model: string }): string { return `${backendNames.get(config.ai.backend.backendType)} ${config.model}`; } -const backends: ReadonlyArray = [ +const backends: readonly Backend[] = [ // new GoogleAIBackend(), TODO: activate once live new VertexAIBackend() ]; @@ -51,23 +45,29 @@ const backendNames: Map = new Map([ [BackendType.VERTEX_AI, 'Vertex AI'] ]); -const modelNames: ReadonlyArray = [ - 'gemini-2.0-flash', +const modelNames: readonly string[] = [ + 'gemini-2.0-flash' // 'gemini-2.0-flash-exp' ]; -export const testConfigs: ReadonlyArray = backends.flatMap(backend => { +/** + * Array of test configurations that is iterated over to get full coverage + * of backends and models. Contains all combinations of backends and models. + */ +export const testConfigs: readonly TestConfig[] = backends.flatMap(backend => { return modelNames.map(modelName => { const ai = getAI(app, { backend }); return { ai: getAI(app, { backend }), model: modelName, toString: () => formatConfigAsString({ ai, model: modelName }) - } - }) -}) + }; + }); +}); -export const TINY_IMG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII='; +export const TINY_IMG_BASE64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII='; export const IMAGE_MIME_TYPE = 'image/png'; -export const TINY_MP3_BASE64 = 'SUQzBAAAAAAAIlRTU0UAAAAOAAADTGF2ZjYxLjcuMTAwAAAAAAAAAAAAAAD/+0DAAAAAAAAAAAAAAAAAAAAAAABJbmZvAAAADwAAAAUAAAK+AGhoaGhoaGhoaGhoaGhoaGhoaGiOjo6Ojo6Ojo6Ojo6Ojo6Ojo6OjrS0tLS0tLS0tLS0tLS0tLS0tLS02tra2tra2tra2tra2tra2tra2tr//////////////////////////wAAAABMYXZjNjEuMTkAAAAAAAAAAAAAAAAkAwYAAAAAAAACvhC6DYoAAAAAAP/7EMQAA8AAAaQAAAAgAAA0gAAABExBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV//sQxCmDwAABpAAAACAAADSAAAAEVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+xDEUwPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVf/7EMR8g8AAAaQAAAAgAAA0gAAABFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV//sQxKYDwAABpAAAACAAADSAAAAEVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVU='; -export const AUDIO_MIME_TYPE = 'audio/mpeg'; \ No newline at end of file +export const TINY_MP3_BASE64 = + 'SUQzBAAAAAAAIlRTU0UAAAAOAAADTGF2ZjYxLjcuMTAwAAAAAAAAAAAAAAD/+0DAAAAAAAAAAAAAAAAAAAAAAABJbmZvAAAADwAAAAUAAAK+AGhoaGhoaGhoaGhoaGhoaGhoaGiOjo6Ojo6Ojo6Ojo6Ojo6Ojo6OjrS0tLS0tLS0tLS0tLS0tLS0tLS02tra2tra2tra2tra2tra2tra2tr//////////////////////////wAAAABMYXZjNjEuMTkAAAAAAAAAAAAAAAAkAwYAAAAAAAACvhC6DYoAAAAAAP/7EMQAA8AAAaQAAAAgAAA0gAAABExBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV//sQxCmDwAABpAAAACAAADSAAAAEVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+xDEUwPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVf/7EMR8g8AAAaQAAAAgAAA0gAAABFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV//sQxKYDwAABpAAAACAAADSAAAAEVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVU='; +export const AUDIO_MIME_TYPE = 'audio/mpeg'; diff --git a/packages/ai/integration/count-tokens.test.ts b/packages/ai/integration/count-tokens.test.ts index 773fae54ee1..dbbc0da3a70 100644 --- a/packages/ai/integration/count-tokens.test.ts +++ b/packages/ai/integration/count-tokens.test.ts @@ -27,7 +27,6 @@ import { getGenerativeModel, Part, CountTokensRequest, - Schema, InlineDataPart, FileDataPart } from '../src'; @@ -40,11 +39,9 @@ import { } from './constants'; import { FIREBASE_CONFIG } from './firebase-config'; - describe('Count Tokens', () => { testConfigs.forEach(testConfig => { describe(`${testConfig.toString()}`, () => { - it('text input', async () => { const generationConfig: GenerationConfig = { temperature: 0, @@ -94,12 +91,16 @@ describe('Count Tokens', () => { expect(response.totalBillableCharacters).to.equal(16); expect(response.promptTokensDetails).to.not.be.null; expect(response.promptTokensDetails!.length).to.equal(1); - expect(response.promptTokensDetails![0].modality).to.equal(Modality.TEXT); + expect(response.promptTokensDetails![0].modality).to.equal( + Modality.TEXT + ); expect(response.promptTokensDetails![0].tokenCount).to.equal(6); }); it('image input', async () => { - const model = getGenerativeModel(testConfig.ai, { model: testConfig.model }); + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model + }); const imagePart: Part = { inlineData: { mimeType: IMAGE_MIME_TYPE, @@ -109,15 +110,32 @@ describe('Count Tokens', () => { const response = await model.countTokens([imagePart]); const expectedImageTokens = 258; - expect(response.totalTokens, 'totalTokens should have correct token count').to.equal(expectedImageTokens); - expect(response.totalBillableCharacters, 'totalBillableCharacters should be undefined').to.be.undefined; // Incorrect behavior - expect(response.promptTokensDetails!.length, 'promptTokensDetails should have one entry').to.equal(1); - expect(response.promptTokensDetails![0].modality, 'modality should be IMAGE').to.equal(Modality.IMAGE); - expect(response.promptTokensDetails![0].tokenCount, 'promptTokenDetails tokenCount should be correct').to.equal(expectedImageTokens); + expect( + response.totalTokens, + 'totalTokens should have correct token count' + ).to.equal(expectedImageTokens); + expect( + response.totalBillableCharacters, + 'totalBillableCharacters should be undefined' + ).to.be.undefined; // Incorrect behavior + expect( + response.promptTokensDetails!.length, + 'promptTokensDetails should have one entry' + ).to.equal(1); + expect( + response.promptTokensDetails![0].modality, + 'modality should be IMAGE' + ).to.equal(Modality.IMAGE); + expect( + response.promptTokensDetails![0].tokenCount, + 'promptTokenDetails tokenCount should be correct' + ).to.equal(expectedImageTokens); }); it('audio input', async () => { - const model = getGenerativeModel(testConfig.ai, { model: testConfig.model }); + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model + }); const audioPart: InlineDataPart = { inlineData: { mimeType: AUDIO_MIME_TYPE, @@ -127,40 +145,82 @@ describe('Count Tokens', () => { const response = await model.countTokens([audioPart]); // This may be different on Google AI - expect(response.totalTokens, 'totalTokens is expected to be undefined').to.be.undefined; - expect(response.totalBillableCharacters, 'totalBillableCharacters should be undefined').to.be.undefined; // Incorrect behavior - expect(response.promptTokensDetails!.length, 'promptTokensDetails should have one entry').to.equal(1); - expect(response.promptTokensDetails![0].modality, 'modality should be AUDIO').to.equal(Modality.AUDIO); - expect(response.promptTokensDetails![0].tokenCount, 'promptTokenDetails tokenCount is expected to be undefined').to.be.undefined; + expect(response.totalTokens, 'totalTokens is expected to be undefined') + .to.be.undefined; + expect( + response.totalBillableCharacters, + 'totalBillableCharacters should be undefined' + ).to.be.undefined; // Incorrect behavior + expect( + response.promptTokensDetails!.length, + 'promptTokensDetails should have one entry' + ).to.equal(1); + expect( + response.promptTokensDetails![0].modality, + 'modality should be AUDIO' + ).to.equal(Modality.AUDIO); + expect( + response.promptTokensDetails![0].tokenCount, + 'promptTokenDetails tokenCount is expected to be undefined' + ).to.be.undefined; }); it('text, image, and audio input', async () => { - const model = getGenerativeModel(testConfig.ai, { model: testConfig.model }); + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model + }); const textPart: Part = { text: 'Describe these:' }; - const imagePart: Part = { inlineData: { mimeType: IMAGE_MIME_TYPE, data: TINY_IMG_BASE64 } }; - const audioPart: Part = { inlineData: { mimeType: AUDIO_MIME_TYPE, data: TINY_MP3_BASE64 } }; + const imagePart: Part = { + inlineData: { mimeType: IMAGE_MIME_TYPE, data: TINY_IMG_BASE64 } + }; + const audioPart: Part = { + inlineData: { mimeType: AUDIO_MIME_TYPE, data: TINY_MP3_BASE64 } + }; const request: CountTokensRequest = { contents: [{ role: 'user', parts: [textPart, imagePart, audioPart] }] }; const response = await model.countTokens(request); - expect(response.totalTokens, 'totalTokens should have correct token count').to.equal(261); - expect(response.totalBillableCharacters, 'totalBillableCharacters should have correct count').to.equal('Describe these:'.length - 1); // For some reason it's the length-1 - - expect(response.promptTokensDetails!.length, 'promptTokensDetails should have three entries').to.equal(3); - - const textDetails = response.promptTokensDetails!.find(d => d.modality === Modality.TEXT); - const visionDetails = response.promptTokensDetails!.find(d => d.modality === Modality.IMAGE); - const audioDetails = response.promptTokensDetails!.find(d => d.modality === Modality.AUDIO); - - expect(textDetails).to.deep.equal({ modality: Modality.TEXT, tokenCount: 3 }); - expect(visionDetails).to.deep.equal({ modality: Modality.IMAGE, tokenCount: 258 }); + expect( + response.totalTokens, + 'totalTokens should have correct token count' + ).to.equal(261); + expect( + response.totalBillableCharacters, + 'totalBillableCharacters should have correct count' + ).to.equal('Describe these:'.length - 1); // For some reason it's the length-1 + + expect( + response.promptTokensDetails!.length, + 'promptTokensDetails should have three entries' + ).to.equal(3); + + const textDetails = response.promptTokensDetails!.find( + d => d.modality === Modality.TEXT + ); + const visionDetails = response.promptTokensDetails!.find( + d => d.modality === Modality.IMAGE + ); + const audioDetails = response.promptTokensDetails!.find( + d => d.modality === Modality.AUDIO + ); + + expect(textDetails).to.deep.equal({ + modality: Modality.TEXT, + tokenCount: 3 + }); + expect(visionDetails).to.deep.equal({ + modality: Modality.IMAGE, + tokenCount: 258 + }); expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO }); // Incorrect behavior because there's no tokenCount }); it('public storage reference', async () => { - const model = getGenerativeModel(testConfig.ai, { model: testConfig.model }); + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model + }); const filePart: FileDataPart = { fileData: { mimeType: IMAGE_MIME_TYPE, @@ -170,13 +230,23 @@ describe('Count Tokens', () => { const response = await model.countTokens([filePart]); const expectedFileTokens = 258; - expect(response.totalTokens, 'totalTokens should have correct token count').to.equal(expectedFileTokens); - expect(response.totalBillableCharacters, 'totalBillableCharacters should be undefined').to.be.undefined; + expect( + response.totalTokens, + 'totalTokens should have correct token count' + ).to.equal(expectedFileTokens); + expect( + response.totalBillableCharacters, + 'totalBillableCharacters should be undefined' + ).to.be.undefined; expect(response.promptTokensDetails).to.not.be.null; expect(response.promptTokensDetails!.length).to.equal(1); - expect(response.promptTokensDetails![0].modality).to.equal(Modality.IMAGE); - expect(response.promptTokensDetails![0].tokenCount).to.equal(expectedFileTokens); + expect(response.promptTokensDetails![0].modality).to.equal( + Modality.IMAGE + ); + expect(response.promptTokensDetails![0].tokenCount).to.equal( + expectedFileTokens + ); }); }); - }) + }); }); diff --git a/packages/ai/integration/firebase-config.ts b/packages/ai/integration/firebase-config.ts index f04097001ed..3f559e7c64d 100644 --- a/packages/ai/integration/firebase-config.ts +++ b/packages/ai/integration/firebase-config.ts @@ -1,3 +1,20 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import * as config from '../../../config/project.json'; -export const FIREBASE_CONFIG = config; \ No newline at end of file +export const FIREBASE_CONFIG = config; diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index 203f4d4f51c..149658c6232 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -16,10 +16,17 @@ */ import { expect } from 'chai'; -import { Content, GenerationConfig, HarmBlockMethod, HarmBlockThreshold, HarmCategory, Modality, SafetySetting, getGenerativeModel } from '../src'; import { - testConfigs -} from './constants'; + Content, + GenerationConfig, + HarmBlockMethod, + HarmBlockThreshold, + HarmCategory, + Modality, + SafetySetting, + getGenerativeModel +} from '../src'; +import { testConfigs } from './constants'; // Token counts are only expected to differ by at most this number of tokens. // Set to 1 for whitespace that is not always present. @@ -28,7 +35,6 @@ const TOKEN_COUNT_DELTA = 1; describe('Generate Content', () => { testConfigs.forEach(testConfig => { describe(`${testConfig.toString()}`, () => { - it('text input, text output', async () => { const generationConfig: GenerationConfig = { temperature: 0, @@ -96,14 +102,16 @@ describe('Generate Content', () => { ); expect(response.usageMetadata!.promptTokensDetails).to.not.be.null; expect(response.usageMetadata!.promptTokensDetails!.length).to.equal(1); - expect(response.usageMetadata!.promptTokensDetails![0].modality).to.equal( - Modality.TEXT - ); - expect(response.usageMetadata!.promptTokensDetails![0].tokenCount).to.equal( - 21 - ); + expect( + response.usageMetadata!.promptTokensDetails![0].modality + ).to.equal(Modality.TEXT); + expect( + response.usageMetadata!.promptTokensDetails![0].tokenCount + ).to.equal(21); expect(response.usageMetadata!.candidatesTokensDetails).to.not.be.null; - expect(response.usageMetadata!.candidatesTokensDetails!.length).to.equal(1); + expect( + response.usageMetadata!.candidatesTokensDetails!.length + ).to.equal(1); expect( response.usageMetadata!.candidatesTokensDetails![0].modality ).to.equal(Modality.TEXT); @@ -112,6 +120,6 @@ describe('Generate Content', () => { ).to.be.closeTo(4, TOKEN_COUNT_DELTA); }); // TODO (dlarocque): Test generateContentStream - }) - }) + }); + }); }); From c76931797f095bf07bc6331708226a482b826c59 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 21 May 2025 15:24:25 -0400 Subject: [PATCH 16/25] count tokens tests against both backends --- packages/ai/integration/count-tokens.test.ts | 74 ++++++++++++------- .../ai/integration/generate-content.test.ts | 2 - 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/packages/ai/integration/count-tokens.test.ts b/packages/ai/integration/count-tokens.test.ts index 5c157a008d3..bcdab441dc8 100644 --- a/packages/ai/integration/count-tokens.test.ts +++ b/packages/ai/integration/count-tokens.test.ts @@ -116,7 +116,6 @@ describe('Count Tokens', () => { } }; const response = await model.countTokens([imagePart]); - console.log(JSON.stringify(response)); if (testConfig.ai.backend.backendType === BackendType.GOOGLE_AI) { const expectedImageTokens = 259; @@ -149,19 +148,30 @@ describe('Count Tokens', () => { }; const response = await model.countTokens([audioPart]); - console.log(JSON.stringify(response)); - // This may be different on Google AI - expect(response.totalTokens).to.be.undefined; + + const textDetails = response.promptTokensDetails!.find( + d => d.modality === Modality.TEXT + ); + const audioDetails = response.promptTokensDetails!.find( + d => d.modality === Modality.AUDIO + ); + + if (testConfig.ai.backend.backendType === BackendType.GOOGLE_AI) { + expect(response.totalTokens).to.equal(6); + expect( + response.promptTokensDetails!.length, + ).to.equal(2); + expect(textDetails).to.deep.equal({ modality: Modality.TEXT, tokenCount: 1 }) + expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO, tokenCount: 5 }) + } else if (testConfig.ai.backend.backendType === BackendType.VERTEX_AI) { + expect(response.totalTokens).to.be.undefined; + expect(response.promptTokensDetails!.length).to.equal(1); // For some reason we don't get text + expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO }); // For some reason there are no tokens + } + expect( response.totalBillableCharacters, ).to.be.undefined; // Incorrect behavior - expect( - response.promptTokensDetails!.length, - ).to.equal(1); - expect( - response.promptTokensDetails![0].modality, - ).to.equal(Modality.AUDIO); - expect(response.promptTokensDetails![0].tokenCount).to.be.undefined; }); it('text, image, and audio input', async () => { @@ -180,15 +190,6 @@ describe('Count Tokens', () => { contents: [{ role: 'user', parts: [textPart, imagePart, audioPart] }] }; const response = await model.countTokens(request); - console.log(JSON.stringify(response)); - - expect(response.totalTokens).to.equal(261); - expect( - response.totalBillableCharacters, - ).to.equal('Describe these:'.length - 1); // For some reason it's the length-1 - - expect(response.promptTokensDetails!.length).to.equal(3); - const textDetails = response.promptTokensDetails!.find( d => d.modality === Modality.TEXT ); @@ -199,18 +200,39 @@ describe('Count Tokens', () => { d => d.modality === Modality.AUDIO ); - expect(textDetails).to.deep.equal({ - modality: Modality.TEXT, - tokenCount: 3 - }); + if (testConfig.ai.backend.backendType === BackendType.GOOGLE_AI) { + expect(response.totalTokens).to.equal(267); + expect(response.totalBillableCharacters).to.be.undefined; + expect(textDetails).to.deep.equal({ + modality: Modality.TEXT, + tokenCount: 4 + }); + expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO, tokenCount: 5 }); // Incorrect behavior because there's no tokenCount + } else if (testConfig.ai.backend.backendType === BackendType.VERTEX_AI) { + expect(response.totalTokens).to.equal(261); + expect(textDetails).to.deep.equal({ + modality: Modality.TEXT, + tokenCount: 3 + }); + expect( + response.totalBillableCharacters, + ).to.equal('Describe these:'.length - 1); // For some reason it's the length-1 + expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO }); // Incorrect behavior because there's no tokenCount + } + + expect(response.promptTokensDetails!.length).to.equal(3); + expect(visionDetails).to.deep.equal({ modality: Modality.IMAGE, tokenCount: 258 }); - expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO }); // Incorrect behavior because there's no tokenCount }); it('public storage reference', async () => { + // This test is not expected to pass when using Google AI. + if (testConfig.ai.backend.backendType === BackendType.GOOGLE_AI) { + return; + } const model = getGenerativeModel(testConfig.ai, { model: testConfig.model }); @@ -220,8 +242,8 @@ describe('Count Tokens', () => { fileUri: `gs://${FIREBASE_CONFIG.storageBucket}/images/tree.png` } }; + const response = await model.countTokens([filePart]); - console.log(JSON.stringify(response)); const expectedFileTokens = 258; expect(response.totalTokens).to.equal(expectedFileTokens); diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index 149658c6232..a4741e0a240 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -46,12 +46,10 @@ describe('Generate Content', () => { { category: HarmCategory.HARM_CATEGORY_HARASSMENT, threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - method: HarmBlockMethod.PROBABILITY }, { category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - method: HarmBlockMethod.SEVERITY }, { category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, From d78fc295b34a0607906e6f65576131fe848cabbf Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 21 May 2025 15:49:37 -0400 Subject: [PATCH 17/25] count tokens test cleanup --- packages/ai/integration/constants.ts | 11 ++- packages/ai/integration/count-tokens.test.ts | 83 ++++++++++++------- .../ai/integration/generate-content.test.ts | 4 +- 3 files changed, 62 insertions(+), 36 deletions(-) diff --git a/packages/ai/integration/constants.ts b/packages/ai/integration/constants.ts index 13376cb7787..583e28322e5 100644 --- a/packages/ai/integration/constants.ts +++ b/packages/ai/integration/constants.ts @@ -16,7 +16,14 @@ */ import { initializeApp } from '@firebase/app'; -import { AI, Backend, BackendType, GoogleAIBackend, VertexAIBackend, getAI } from '../src'; +import { + AI, + Backend, + BackendType, + GoogleAIBackend, + VertexAIBackend, + getAI +} from '../src'; import { FIREBASE_CONFIG } from './firebase-config'; const app = initializeApp(FIREBASE_CONFIG); @@ -51,7 +58,7 @@ const modelNames: readonly string[] = [ ]; /** - * Array of test configurations that is iterated over to get full coverage + * Array of test configurations that is iterated over to get full coverage * of backends and models. Contains all combinations of backends and models. */ export const testConfigs: readonly TestConfig[] = backends.flatMap(backend => { diff --git a/packages/ai/integration/count-tokens.test.ts b/packages/ai/integration/count-tokens.test.ts index bcdab441dc8..31dd1ae1403 100644 --- a/packages/ai/integration/count-tokens.test.ts +++ b/packages/ai/integration/count-tokens.test.ts @@ -88,17 +88,18 @@ describe('Count Tokens', () => { const response = await model.countTokens('Why is the sky blue?'); - expect(response.promptTokensDetails).to.not.be.null; + expect(response.promptTokensDetails).to.exist; expect(response.promptTokensDetails!.length).to.equal(1); expect(response.promptTokensDetails![0].modality).to.equal( Modality.TEXT ); - if (testConfig.ai.backend.backendType === BackendType.GOOGLE_AI) { expect(response.totalTokens).to.equal(7); expect(response.totalBillableCharacters).to.be.undefined; expect(response.promptTokensDetails![0].tokenCount).to.equal(7); - } else if (testConfig.ai.backend.backendType === BackendType.VERTEX_AI) { + } else if ( + testConfig.ai.backend.backendType === BackendType.VERTEX_AI + ) { expect(response.totalTokens).to.equal(6); expect(response.totalBillableCharacters).to.equal(16); expect(response.promptTokensDetails![0].tokenCount).to.equal(6); @@ -119,7 +120,17 @@ describe('Count Tokens', () => { if (testConfig.ai.backend.backendType === BackendType.GOOGLE_AI) { const expectedImageTokens = 259; - + expect(response.totalTokens).to.equal(expectedImageTokens); + expect(response.totalBillableCharacters).to.be.undefined; // Incorrect behavior + expect(response.promptTokensDetails!.length).to.equal(2); + expect(response.promptTokensDetails![0]).to.deep.equal({ + modality: Modality.TEXT, // Note: 1 unexpected text token observed for Google AI with image-only input. + tokenCount: 1 + }); + expect(response.promptTokensDetails![1]).to.deep.equal({ + modality: Modality.IMAGE, + tokenCount: 258 + }); } else if (testConfig.ai.backend.backendType === BackendType.VERTEX_AI) { const expectedImageTokens = 258; expect(response.totalTokens).to.equal(expectedImageTokens); @@ -129,9 +140,8 @@ describe('Count Tokens', () => { expect( response.promptTokensDetails!.length, ).to.equal(1); - expect( - response.promptTokensDetails![0].modality, - ).to.equal(Modality.IMAGE); + // Note: No text tokens are present for Vertex AI with image-only input. + expect(response.promptTokensDetails![0]).to.deep.equal({ modality: Modality.IMAGE, tokenCount: 258 }) expect(response.promptTokensDetails![0].tokenCount).to.equal(expectedImageTokens); } }); @@ -149,6 +159,7 @@ describe('Count Tokens', () => { const response = await model.countTokens([audioPart]); + expect(response.promptTokensDetails).to.exist; const textDetails = response.promptTokensDetails!.find( d => d.modality === Modality.TEXT ); @@ -158,20 +169,24 @@ describe('Count Tokens', () => { if (testConfig.ai.backend.backendType === BackendType.GOOGLE_AI) { expect(response.totalTokens).to.equal(6); - expect( - response.promptTokensDetails!.length, - ).to.equal(2); - expect(textDetails).to.deep.equal({ modality: Modality.TEXT, tokenCount: 1 }) - expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO, tokenCount: 5 }) - } else if (testConfig.ai.backend.backendType === BackendType.VERTEX_AI) { + expect(response.promptTokensDetails!.length).to.equal(2); + expect(textDetails).to.deep.equal({ + modality: Modality.TEXT, + tokenCount: 1 + }); + expect(audioDetails).to.deep.equal({ + modality: Modality.AUDIO, + tokenCount: 5 + }); + } else if ( + testConfig.ai.backend.backendType === BackendType.VERTEX_AI + ) { expect(response.totalTokens).to.be.undefined; - expect(response.promptTokensDetails!.length).to.equal(1); // For some reason we don't get text - expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO }); // For some reason there are no tokens + expect(response.promptTokensDetails!.length).to.equal(1); // Note: Text modality details absent for Vertex AI with audio-only input. + expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO }); // Note: Audio tokenCount is undefined for Vertex AI with audio-only input. } - expect( - response.totalBillableCharacters, - ).to.be.undefined; // Incorrect behavior + expect(response.totalBillableCharacters).to.be.undefined; // Incorrect behavior }); it('text, image, and audio input', async () => { @@ -193,12 +208,19 @@ describe('Count Tokens', () => { const textDetails = response.promptTokensDetails!.find( d => d.modality === Modality.TEXT ); - const visionDetails = response.promptTokensDetails!.find( + const imageDetails = response.promptTokensDetails!.find( d => d.modality === Modality.IMAGE ); const audioDetails = response.promptTokensDetails!.find( d => d.modality === Modality.AUDIO ); + expect(response.promptTokensDetails).to.exist; + expect(response.promptTokensDetails!.length).to.equal(3); + + expect(imageDetails).to.deep.equal({ + modality: Modality.IMAGE, + tokenCount: 258 + }); if (testConfig.ai.backend.backendType === BackendType.GOOGLE_AI) { expect(response.totalTokens).to.equal(267); @@ -207,25 +229,22 @@ describe('Count Tokens', () => { modality: Modality.TEXT, tokenCount: 4 }); - expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO, tokenCount: 5 }); // Incorrect behavior because there's no tokenCount - } else if (testConfig.ai.backend.backendType === BackendType.VERTEX_AI) { + expect(audioDetails).to.deep.equal({ + modality: Modality.AUDIO, + tokenCount: 5 + }); + } else if ( + testConfig.ai.backend.backendType === BackendType.VERTEX_AI + ) { expect(response.totalTokens).to.equal(261); expect(textDetails).to.deep.equal({ modality: Modality.TEXT, tokenCount: 3 }); - expect( - response.totalBillableCharacters, - ).to.equal('Describe these:'.length - 1); // For some reason it's the length-1 + const expectedText = 'Describe these:'; + expect(response.totalBillableCharacters).to.equal(expectedText.length - 1); // Note: BillableCharacters observed as (text length - 1) for Vertex AI. expect(audioDetails).to.deep.equal({ modality: Modality.AUDIO }); // Incorrect behavior because there's no tokenCount } - - expect(response.promptTokensDetails!.length).to.equal(3); - - expect(visionDetails).to.deep.equal({ - modality: Modality.IMAGE, - tokenCount: 258 - }); }); it('public storage reference', async () => { @@ -248,7 +267,7 @@ describe('Count Tokens', () => { const expectedFileTokens = 258; expect(response.totalTokens).to.equal(expectedFileTokens); expect(response.totalBillableCharacters).to.be.undefined; - expect(response.promptTokensDetails).to.not.be.null; + expect(response.promptTokensDetails).to.exist; expect(response.promptTokensDetails!.length).to.equal(1); expect(response.promptTokensDetails![0].modality).to.equal( Modality.IMAGE diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index a4741e0a240..645120be7d8 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -45,11 +45,11 @@ describe('Generate Content', () => { const safetySettings: SafetySetting[] = [ { category: HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE }, { category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE }, { category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, From 676e90d347e22181e2ae54dcc09f5a598ca4390e Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 22 May 2025 11:14:56 -0400 Subject: [PATCH 18/25] remaining integration tests --- packages/ai/integration/chat.test.ts | 132 +++++++++++++++++ packages/ai/integration/constants.ts | 4 + .../ai/integration/generate-content.test.ts | 136 ++++++++++++------ 3 files changed, 231 insertions(+), 41 deletions(-) create mode 100644 packages/ai/integration/chat.test.ts diff --git a/packages/ai/integration/chat.test.ts b/packages/ai/integration/chat.test.ts new file mode 100644 index 00000000000..c2a3726004f --- /dev/null +++ b/packages/ai/integration/chat.test.ts @@ -0,0 +1,132 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { + Content, + GenerationConfig, + HarmBlockThreshold, + HarmCategory, + SafetySetting, + getGenerativeModel +} from '../src'; +import { testConfigs, TOKEN_COUNT_DELTA } from './constants'; + +describe('Chat Session', () => { + testConfigs.forEach(testConfig => { + describe(`${testConfig.toString()}`, () => { + const commonGenerationConfig: GenerationConfig = { + temperature: 0, + topP: 0, + responseMimeType: 'text/plain' + }; + + const commonSafetySettings: SafetySetting[] = [ + { + category: HarmCategory.HARM_CATEGORY_HARASSMENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + }, + { + category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + }, + { + category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + }, + { + category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + } + ]; + + const commonSystemInstruction: Content = { + role: 'system', + parts: [ + { + text: 'You are a friendly and helpful assistant.' + } + ] + }; + + it('startChat and sendMessage: text input, text output', async () => { + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig: commonGenerationConfig, + safetySettings: commonSafetySettings, + systemInstruction: commonSystemInstruction + }); + + const chat = model.startChat(); + const result1 = await chat.sendMessage( + 'What is the capital of France?' + ); + const response1 = result1.response; + expect(response1.text().trim().toLowerCase()).to.include('paris'); + + let history = await chat.getHistory(); + expect(history.length).to.equal(2); + expect(history[0].role).to.equal('user'); + expect(history[0].parts[0].text).to.equal( + 'What is the capital of France?' + ); + expect(history[1].role).to.equal('model'); + expect(history[1].parts[0].text?.toLowerCase()).to.include('paris'); + + expect(response1.usageMetadata).to.not.be.null; + // Token counts can vary slightly in chat context + expect(response1.usageMetadata!.promptTokenCount).to.be.closeTo( + 15, // "What is the capital of France?" + system instruction + TOKEN_COUNT_DELTA + 2 // More variance for chat context + ); + expect(response1.usageMetadata!.candidatesTokenCount).to.be.closeTo( + 8, // "Paris" + TOKEN_COUNT_DELTA + ); + expect(response1.usageMetadata!.totalTokenCount).to.be.closeTo( + 23, // "What is the capital of France?" + system instruction + "Paris" + TOKEN_COUNT_DELTA + 3 // More variance for chat context + ); + + const result2 = await chat.sendMessage('And what about Italy?'); + const response2 = result2.response; + expect(response2.text().trim().toLowerCase()).to.include('rome'); + + history = await chat.getHistory(); + expect(history.length).to.equal(4); + expect(history[2].role).to.equal('user'); + expect(history[2].parts[0].text).to.equal('And what about Italy?'); + expect(history[3].role).to.equal('model'); + expect(history[3].parts[0].text?.toLowerCase()).to.include('rome'); + + expect(response2.usageMetadata).to.not.be.null; + expect(response2.usageMetadata!.promptTokenCount).to.be.closeTo( + 28, // History + "And what about Italy?" + system instruction + TOKEN_COUNT_DELTA + 5 // More variance for chat context with history + ); + expect(response2.usageMetadata!.candidatesTokenCount).to.be.closeTo( + 8, + TOKEN_COUNT_DELTA + ); + expect(response2.usageMetadata!.totalTokenCount).to.be.closeTo( + 36, + TOKEN_COUNT_DELTA + ); + }); + }); + }); +}); \ No newline at end of file diff --git a/packages/ai/integration/constants.ts b/packages/ai/integration/constants.ts index 583e28322e5..873da390704 100644 --- a/packages/ai/integration/constants.ts +++ b/packages/ai/integration/constants.ts @@ -78,3 +78,7 @@ export const IMAGE_MIME_TYPE = 'image/png'; export const TINY_MP3_BASE64 = 'SUQzBAAAAAAAIlRTU0UAAAAOAAADTGF2ZjYxLjcuMTAwAAAAAAAAAAAAAAD/+0DAAAAAAAAAAAAAAAAAAAAAAABJbmZvAAAADwAAAAUAAAK+AGhoaGhoaGhoaGhoaGhoaGhoaGiOjo6Ojo6Ojo6Ojo6Ojo6Ojo6OjrS0tLS0tLS0tLS0tLS0tLS0tLS02tra2tra2tra2tra2tra2tra2tr//////////////////////////wAAAABMYXZjNjEuMTkAAAAAAAAAAAAAAAAkAwYAAAAAAAACvhC6DYoAAAAAAP/7EMQAA8AAAaQAAAAgAAA0gAAABExBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV//sQxCmDwAABpAAAACAAADSAAAAEVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+xDEUwPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVf/7EMR8g8AAAaQAAAAgAAA0gAAABFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV//sQxKYDwAABpAAAACAAADSAAAAEVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVU='; export const AUDIO_MIME_TYPE = 'audio/mpeg'; + +// Token counts are only expected to differ by at most this number of tokens. +// Set to 1 for whitespace that is not always present. +export const TOKEN_COUNT_DELTA = 1; diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index 645120be7d8..22ab6557a09 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -19,62 +19,58 @@ import { expect } from 'chai'; import { Content, GenerationConfig, - HarmBlockMethod, HarmBlockThreshold, HarmCategory, Modality, SafetySetting, getGenerativeModel } from '../src'; -import { testConfigs } from './constants'; +import { testConfigs, TOKEN_COUNT_DELTA } from './constants'; -// Token counts are only expected to differ by at most this number of tokens. -// Set to 1 for whitespace that is not always present. -const TOKEN_COUNT_DELTA = 1; describe('Generate Content', () => { testConfigs.forEach(testConfig => { describe(`${testConfig.toString()}`, () => { - it('text input, text output', async () => { - const generationConfig: GenerationConfig = { - temperature: 0, - topP: 0, - responseMimeType: 'text/plain' - }; + const commonGenerationConfig: GenerationConfig = { + temperature: 0, + topP: 0, + responseMimeType: 'text/plain' + }; - const safetySettings: SafetySetting[] = [ - { - category: HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE - }, - { - category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE - }, - { - category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE - }, + const commonSafetySettings: SafetySetting[] = [ + { + category: HarmCategory.HARM_CATEGORY_HARASSMENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + }, + { + category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + }, + { + category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + }, + { + category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + } + ]; + + const commonSystemInstruction: Content = { + role: 'system', + parts: [ { - category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + text: 'You are a friendly and helpful assistant.' } - ]; - - const systemInstruction: Content = { - role: 'system', - parts: [ - { - text: 'You are a friendly and helpful assistant.' - } - ] - }; + ] + }; + it('generateContent: text input, text output', async () => { const model = getGenerativeModel(testConfig.ai, { model: testConfig.model, - generationConfig, - safetySettings, - systemInstruction + generationConfig: commonGenerationConfig, + safetySettings: commonSafetySettings, + systemInstruction: commonSystemInstruction }); const result = await model.generateContent( @@ -117,7 +113,65 @@ describe('Generate Content', () => { response.usageMetadata!.candidatesTokensDetails![0].tokenCount ).to.be.closeTo(4, TOKEN_COUNT_DELTA); }); - // TODO (dlarocque): Test generateContentStream + + it('generateContentStream: text input, text output', async () => { + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig: commonGenerationConfig, + safetySettings: commonSafetySettings, + systemInstruction: commonSystemInstruction + }); + + const result = await model.generateContentStream( + 'Where is Google headquarters located? Answer with the city name only.' + ); + + let streamText = ''; + for await (const chunk of result.stream) { + streamText += chunk.text(); + } + expect(streamText.trim()).to.equal('Mountain View'); + + const response = await result.response; + const trimmedText = response.text().trim(); + expect(trimmedText).to.equal('Mountain View'); + expect(response.usageMetadata).to.be.undefined; // Note: This is incorrect behavior. + + /* + expect(response.usageMetadata).to.exist; + expect(response.usageMetadata!.promptTokenCount).to.be.closeTo( + 21, + TOKEN_COUNT_DELTA + ); // TODO: fix promptTokenToke is undefined + // Candidate token count can be slightly different in streaming + expect(response.usageMetadata!.candidatesTokenCount).to.be.closeTo( + 4, + TOKEN_COUNT_DELTA + 1 // Allow slightly more variance for stream + ); + expect(response.usageMetadata!.totalTokenCount).to.be.closeTo( + 25, + TOKEN_COUNT_DELTA * 2 + 1 // Allow slightly more variance for stream + ); + expect(response.usageMetadata!.promptTokensDetails).to.not.be.null; + expect(response.usageMetadata!.promptTokensDetails!.length).to.equal(1); + expect( + response.usageMetadata!.promptTokensDetails![0].modality + ).to.equal(Modality.TEXT); + expect( + response.usageMetadata!.promptTokensDetails![0].tokenCount + ).to.equal(21); + expect(response.usageMetadata!.candidatesTokensDetails).to.not.be.null; + expect( + response.usageMetadata!.candidatesTokensDetails!.length + ).to.equal(1); + expect( + response.usageMetadata!.candidatesTokensDetails![0].modality + ).to.equal(Modality.TEXT); + expect( + response.usageMetadata!.candidatesTokensDetails![0].tokenCount + ).to.be.closeTo(4, TOKEN_COUNT_DELTA + 1); // Allow slightly more variance for stream + */ + }); }); }); -}); +}); \ No newline at end of file From ddfd7f5c828707360399604aefce27690b5ec70d Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 22 May 2025 11:50:21 -0400 Subject: [PATCH 19/25] revert changes to gitignore --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 7c0943ad49e..a989576ccf3 100644 --- a/.gitignore +++ b/.gitignore @@ -104,6 +104,3 @@ mocks-lookup.ts # temp changeset output changeset-temp.json - -# Vertex AI integration test Firebase project config -packages/vertexai/integration/firebase-config.ts \ No newline at end of file From 28f843cc4f0890a7ee927e0e4e0c0261f4281223 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 22 May 2025 13:52:05 -0400 Subject: [PATCH 20/25] use ci config file --- packages/ai/integration/firebase-config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ai/integration/firebase-config.ts b/packages/ai/integration/firebase-config.ts index 3f559e7c64d..2527f020530 100644 --- a/packages/ai/integration/firebase-config.ts +++ b/packages/ai/integration/firebase-config.ts @@ -15,6 +15,6 @@ * limitations under the License. */ -import * as config from '../../../config/project.json'; +import * as config from '../../../config/ci.config.json'; export const FIREBASE_CONFIG = config; From f851be42d18821bea2eddecc2c39a86b76cc2502 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 23 May 2025 11:31:50 -0400 Subject: [PATCH 21/25] import from minified @firebase/ai in integration tests --- packages/ai/integration/chat.test.ts | 2 +- packages/ai/integration/constants.ts | 2 +- packages/ai/integration/count-tokens.test.ts | 2 +- packages/ai/integration/generate-content.test.ts | 2 +- packages/ai/karma.conf.js | 9 --------- 5 files changed, 4 insertions(+), 13 deletions(-) diff --git a/packages/ai/integration/chat.test.ts b/packages/ai/integration/chat.test.ts index 6af0e7a9af9..51c1f0997d3 100644 --- a/packages/ai/integration/chat.test.ts +++ b/packages/ai/integration/chat.test.ts @@ -23,7 +23,7 @@ import { HarmCategory, SafetySetting, getGenerativeModel -} from '../src'; +} from '@firebase/ai'; import { testConfigs, TOKEN_COUNT_DELTA } from './constants'; describe('Chat Session', () => { diff --git a/packages/ai/integration/constants.ts b/packages/ai/integration/constants.ts index 873da390704..7f2b6d02779 100644 --- a/packages/ai/integration/constants.ts +++ b/packages/ai/integration/constants.ts @@ -23,7 +23,7 @@ import { GoogleAIBackend, VertexAIBackend, getAI -} from '../src'; +} from '@firebase/ai'; import { FIREBASE_CONFIG } from './firebase-config'; const app = initializeApp(FIREBASE_CONFIG); diff --git a/packages/ai/integration/count-tokens.test.ts b/packages/ai/integration/count-tokens.test.ts index 3256a9ed9d7..b351638abd8 100644 --- a/packages/ai/integration/count-tokens.test.ts +++ b/packages/ai/integration/count-tokens.test.ts @@ -30,7 +30,7 @@ import { InlineDataPart, FileDataPart, BackendType -} from '../src'; +} from '@firebase/ai'; import { AUDIO_MIME_TYPE, IMAGE_MIME_TYPE, diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index 40b63d78815..1433d4bd8a6 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -24,7 +24,7 @@ import { Modality, SafetySetting, getGenerativeModel -} from '../src'; +} from '@firebase/ai'; import { testConfigs, TOKEN_COUNT_DELTA } from './constants'; describe('Generate Content', () => { diff --git a/packages/ai/karma.conf.js b/packages/ai/karma.conf.js index dc9590e6d44..6d69c6bf3a0 100644 --- a/packages/ai/karma.conf.js +++ b/packages/ai/karma.conf.js @@ -21,15 +21,6 @@ const { existsSync } = require('fs'); const files = [`src/**/*.test.ts`]; -// Validate that the file that defines the Firebase config to be used in the integration tests exists. -if (argv.integration) { - if (!existsSync('integration/firebase-config.ts')) { - throw new Error( - `integration/firebase-config.ts does not exist. This file must contain a Firebase config for a project with Vertex AI enabled.` - ); - } -} - module.exports = function (config) { const karmaConfig = { ...karmaBase, From f327b4b532d5b086778985cfe948a9a3643cb9e7 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 23 May 2025 11:41:53 -0400 Subject: [PATCH 22/25] exclude integration from rollup --- packages/ai/karma.conf.js | 1 - packages/ai/rollup.config.js | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/ai/karma.conf.js b/packages/ai/karma.conf.js index 6d69c6bf3a0..e64048d5f6d 100644 --- a/packages/ai/karma.conf.js +++ b/packages/ai/karma.conf.js @@ -17,7 +17,6 @@ const karmaBase = require('../../config/karma.base'); const { argv } = require('yargs'); -const { existsSync } = require('fs'); const files = [`src/**/*.test.ts`]; diff --git a/packages/ai/rollup.config.js b/packages/ai/rollup.config.js index b3a1e5ebcf8..3b155335898 100644 --- a/packages/ai/rollup.config.js +++ b/packages/ai/rollup.config.js @@ -32,7 +32,12 @@ const buildPlugins = [ typescriptPlugin({ typescript, tsconfigOverride: { - exclude: [...tsconfig.exclude, '**/*.test.ts', 'test-utils'], + exclude: [ + ...tsconfig.exclude, + '**/*.test.ts', + 'test-utils', + 'integration' + ], compilerOptions: { target: 'es2017' } From 604a419f9524915b01920952c0261677d6e17aeb Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 23 May 2025 12:00:52 -0400 Subject: [PATCH 23/25] revert change to import from @firebase/ai --- packages/ai/integration/chat.test.ts | 2 +- packages/ai/integration/constants.ts | 2 +- packages/ai/integration/count-tokens.test.ts | 2 +- packages/ai/integration/generate-content.test.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/ai/integration/chat.test.ts b/packages/ai/integration/chat.test.ts index 51c1f0997d3..6af0e7a9af9 100644 --- a/packages/ai/integration/chat.test.ts +++ b/packages/ai/integration/chat.test.ts @@ -23,7 +23,7 @@ import { HarmCategory, SafetySetting, getGenerativeModel -} from '@firebase/ai'; +} from '../src'; import { testConfigs, TOKEN_COUNT_DELTA } from './constants'; describe('Chat Session', () => { diff --git a/packages/ai/integration/constants.ts b/packages/ai/integration/constants.ts index 7f2b6d02779..873da390704 100644 --- a/packages/ai/integration/constants.ts +++ b/packages/ai/integration/constants.ts @@ -23,7 +23,7 @@ import { GoogleAIBackend, VertexAIBackend, getAI -} from '@firebase/ai'; +} from '../src'; import { FIREBASE_CONFIG } from './firebase-config'; const app = initializeApp(FIREBASE_CONFIG); diff --git a/packages/ai/integration/count-tokens.test.ts b/packages/ai/integration/count-tokens.test.ts index b351638abd8..3256a9ed9d7 100644 --- a/packages/ai/integration/count-tokens.test.ts +++ b/packages/ai/integration/count-tokens.test.ts @@ -30,7 +30,7 @@ import { InlineDataPart, FileDataPart, BackendType -} from '@firebase/ai'; +} from '../src'; import { AUDIO_MIME_TYPE, IMAGE_MIME_TYPE, diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index 1433d4bd8a6..40b63d78815 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -24,7 +24,7 @@ import { Modality, SafetySetting, getGenerativeModel -} from '@firebase/ai'; +} from '../src'; import { testConfigs, TOKEN_COUNT_DELTA } from './constants'; describe('Generate Content', () => { From 9b97310e946e2d38db985b16170bfa0be5fe1461 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 23 May 2025 12:21:11 -0400 Subject: [PATCH 24/25] cleanup --- packages/ai/integration/constants.ts | 1 - .../ai/integration/generate-content.test.ts | 35 ------------------- 2 files changed, 36 deletions(-) diff --git a/packages/ai/integration/constants.ts b/packages/ai/integration/constants.ts index 873da390704..f4127e89a24 100644 --- a/packages/ai/integration/constants.ts +++ b/packages/ai/integration/constants.ts @@ -54,7 +54,6 @@ const backendNames: Map = new Map([ const modelNames: readonly string[] = [ 'gemini-2.0-flash' - // 'gemini-2.0-flash-exp' ]; /** diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index 40b63d78815..af877396cc8 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -135,41 +135,6 @@ describe('Generate Content', () => { const trimmedText = response.text().trim(); expect(trimmedText).to.equal('Mountain View'); expect(response.usageMetadata).to.be.undefined; // Note: This is incorrect behavior. - - /* - expect(response.usageMetadata).to.exist; - expect(response.usageMetadata!.promptTokenCount).to.be.closeTo( - 21, - TOKEN_COUNT_DELTA - ); // TODO: fix promptTokenToke is undefined - // Candidate token count can be slightly different in streaming - expect(response.usageMetadata!.candidatesTokenCount).to.be.closeTo( - 4, - TOKEN_COUNT_DELTA + 1 // Allow slightly more variance for stream - ); - expect(response.usageMetadata!.totalTokenCount).to.be.closeTo( - 25, - TOKEN_COUNT_DELTA * 2 + 1 // Allow slightly more variance for stream - ); - expect(response.usageMetadata!.promptTokensDetails).to.not.be.null; - expect(response.usageMetadata!.promptTokensDetails!.length).to.equal(1); - expect( - response.usageMetadata!.promptTokensDetails![0].modality - ).to.equal(Modality.TEXT); - expect( - response.usageMetadata!.promptTokensDetails![0].tokenCount - ).to.equal(21); - expect(response.usageMetadata!.candidatesTokensDetails).to.not.be.null; - expect( - response.usageMetadata!.candidatesTokensDetails!.length - ).to.equal(1); - expect( - response.usageMetadata!.candidatesTokensDetails![0].modality - ).to.equal(Modality.TEXT); - expect( - response.usageMetadata!.candidatesTokensDetails![0].tokenCount - ).to.be.closeTo(4, TOKEN_COUNT_DELTA + 1); // Allow slightly more variance for stream - */ }); }); }); From 63f721538455b450bdd9d162431eee1ee8696f4a Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 23 May 2025 12:47:12 -0400 Subject: [PATCH 25/25] format --- packages/ai/integration/constants.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/ai/integration/constants.ts b/packages/ai/integration/constants.ts index f4127e89a24..68aebf9eddc 100644 --- a/packages/ai/integration/constants.ts +++ b/packages/ai/integration/constants.ts @@ -52,9 +52,7 @@ const backendNames: Map = new Map([ [BackendType.VERTEX_AI, 'Vertex AI'] ]); -const modelNames: readonly string[] = [ - 'gemini-2.0-flash' -]; +const modelNames: readonly string[] = ['gemini-2.0-flash']; /** * Array of test configurations that is iterated over to get full coverage