diff --git a/.changeset/five-kids-grow.md b/.changeset/five-kids-grow.md new file mode 100644 index 00000000000..4303518b6ff --- /dev/null +++ b/.changeset/five-kids-grow.md @@ -0,0 +1,6 @@ +--- +'firebase': minor +'@firebase/ai': minor +--- + +Add support for Grounding with Google Search. diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 23c13688fb1..7fec37c12a8 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -485,6 +485,15 @@ export interface GoogleAIGenerateContentResponse { usageMetadata?: UsageMetadata; } +// @public +export interface GoogleSearch { +} + +// @public +export interface GoogleSearchTool { + googleSearch: GoogleSearch; +} + // @public @deprecated (undocumented) export interface GroundingAttribution { // (undocumented) @@ -497,16 +506,29 @@ export interface GroundingAttribution { web?: WebAttribution; } +// @public +export interface GroundingChunk { + web?: WebGroundingChunk; +} + // @public export interface GroundingMetadata { // @deprecated (undocumented) groundingAttributions: GroundingAttribution[]; - // (undocumented) + groundingChunks?: GroundingChunk[]; + groundingSupports?: GroundingSupport[]; + // @deprecated (undocumented) retrievalQueries?: string[]; - // (undocumented) + searchEntryPoint?: SearchEntrypoint; webSearchQueries?: string[]; } +// @public +export interface GroundingSupport { + groundingChunkIndices?: number[]; + segment?: Segment; +} + // @public export enum HarmBlockMethod { PROBABILITY = "PROBABILITY", @@ -857,14 +879,17 @@ export enum SchemaType { STRING = "string" } -// @public (undocumented) +// @public +export interface SearchEntrypoint { + renderedContent?: string; +} + +// @public export interface Segment { - // (undocumented) endIndex: number; - // (undocumented) partIndex: number; - // (undocumented) startIndex: number; + text: string; } // @public @@ -901,7 +926,7 @@ export interface TextPart { } // @public -export type Tool = FunctionDeclarationsTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool; // @public export interface ToolConfig { @@ -961,5 +986,12 @@ export interface WebAttribution { uri: string; } +// @public +export interface WebGroundingChunk { + domain?: string; + title?: string; + uri?: string; +} + ``` diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index b77a6b5910e..bf1b8978b57 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -76,10 +76,18 @@ toc: path: /docs/reference/js/ai.generativemodel.md - title: GoogleAIBackend path: /docs/reference/js/ai.googleaibackend.md + - title: GoogleSearch + path: /docs/reference/js/ai.googlesearch.md + - title: GoogleSearchTool + path: /docs/reference/js/ai.googlesearchtool.md - title: GroundingAttribution path: /docs/reference/js/ai.groundingattribution.md + - title: GroundingChunk + path: /docs/reference/js/ai.groundingchunk.md - title: GroundingMetadata path: /docs/reference/js/ai.groundingmetadata.md + - title: GroundingSupport + path: /docs/reference/js/ai.groundingsupport.md - title: ImagenGCSImage path: /docs/reference/js/ai.imagengcsimage.md - title: ImagenGenerationConfig @@ -130,6 +138,8 @@ toc: path: /docs/reference/js/ai.schemarequest.md - title: SchemaShared path: /docs/reference/js/ai.schemashared.md + - title: SearchEntrypoint + path: /docs/reference/js/ai.searchentrypoint.md - title: Segment path: /docs/reference/js/ai.segment.md - title: StartChatParams @@ -150,6 +160,8 @@ toc: path: /docs/reference/js/ai.videometadata.md - title: WebAttribution path: /docs/reference/js/ai.webattribution.md + - title: WebGroundingChunk + path: /docs/reference/js/ai.webgroundingchunk.md - title: analytics path: /docs/reference/js/analytics.md section: diff --git a/docs-devsite/ai.functiondeclaration.md b/docs-devsite/ai.functiondeclaration.md index 2a87d67ed47..6fa9e53cae9 100644 --- a/docs-devsite/ai.functiondeclaration.md +++ b/docs-devsite/ai.functiondeclaration.md @@ -15,7 +15,7 @@ Structured representation of a function declaration as defined by the [OpenAPI 3 Signature: ```typescript -export declare interface FunctionDeclaration +export interface FunctionDeclaration ``` ## Properties diff --git a/docs-devsite/ai.functiondeclarationstool.md b/docs-devsite/ai.functiondeclarationstool.md index bde785d730b..d72d9db2f53 100644 --- a/docs-devsite/ai.functiondeclarationstool.md +++ b/docs-devsite/ai.functiondeclarationstool.md @@ -15,7 +15,7 @@ A `FunctionDeclarationsTool` is a piece of code that enables the system to inter Signature: ```typescript -export declare interface FunctionDeclarationsTool +export interface FunctionDeclarationsTool ``` ## Properties diff --git a/docs-devsite/ai.googlesearch.md b/docs-devsite/ai.googlesearch.md new file mode 100644 index 00000000000..78fdef51606 --- /dev/null +++ b/docs-devsite/ai.googlesearch.md @@ -0,0 +1,21 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# GoogleSearch interface +Specifies the Google Search configuration. + +Currently, this is an empty object, but it's reserved for future configuration options. + +Signature: + +```typescript +export interface GoogleSearch +``` diff --git a/docs-devsite/ai.googlesearchtool.md b/docs-devsite/ai.googlesearchtool.md new file mode 100644 index 00000000000..49983bb8269 --- /dev/null +++ b/docs-devsite/ai.googlesearchtool.md @@ -0,0 +1,39 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# GoogleSearchTool interface +A tool that allows a Gemini model to connect to Google Search to access and incorporate up-to-date information from the web into its responses. + +Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or [Vertex AI Gemini API](https://cloud.google.com/terms/service-terms) (see Service Terms section within the Service Specific Terms). + +Signature: + +```typescript +export interface GoogleSearchTool +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [googleSearch](./ai.googlesearchtool.md#googlesearchtoolgooglesearch) | [GoogleSearch](./ai.googlesearch.md#googlesearch_interface) | Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options.When using this feature, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or [Vertex AI Gemini API](https://cloud.google.com/terms/service-terms) (see Service Terms section within the Service Specific Terms). | + +## GoogleSearchTool.googleSearch + +Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. + +When using this feature, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or [Vertex AI Gemini API](https://cloud.google.com/terms/service-terms) (see Service Terms section within the Service Specific Terms). + +Signature: + +```typescript +googleSearch: GoogleSearch; +``` diff --git a/docs-devsite/ai.groundingchunk.md b/docs-devsite/ai.groundingchunk.md new file mode 100644 index 00000000000..2b84af29d8e --- /dev/null +++ b/docs-devsite/ai.groundingchunk.md @@ -0,0 +1,35 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# GroundingChunk interface +Represents a chunk of retrieved data that supports a claim in the model's response. This is part of the grounding information provided when grounding is enabled. + +Signature: + +```typescript +export interface GroundingChunk +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [web](./ai.groundingchunk.md#groundingchunkweb) | [WebGroundingChunk](./ai.webgroundingchunk.md#webgroundingchunk_interface) | Contains details if the grounding chunk is from a web source. | + +## GroundingChunk.web + +Contains details if the grounding chunk is from a web source. + +Signature: + +```typescript +web?: WebGroundingChunk; +``` diff --git a/docs-devsite/ai.groundingmetadata.md b/docs-devsite/ai.groundingmetadata.md index 90994d9c01c..d9dc1148c53 100644 --- a/docs-devsite/ai.groundingmetadata.md +++ b/docs-devsite/ai.groundingmetadata.md @@ -10,7 +10,11 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # GroundingMetadata interface -Metadata returned to client when grounding is enabled. +Metadata returned when grounding is enabled. + +Currently, only Grounding with Google Search is supported (see [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface)). + +Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or [Vertex AI Gemini API](https://cloud.google.com/terms/service-terms) (see Service Terms section within the Service Specific Terms). Signature: @@ -23,13 +27,17 @@ export interface GroundingMetadata | Property | Type | Description | | --- | --- | --- | | [groundingAttributions](./ai.groundingmetadata.md#groundingmetadatagroundingattributions) | [GroundingAttribution](./ai.groundingattribution.md#groundingattribution_interface)\[\] | | +| [groundingChunks](./ai.groundingmetadata.md#groundingmetadatagroundingchunks) | [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface)\[\] | A list of [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) objects. Each chunk represents a piece of retrieved content (for example, from a web page). that the model used to ground its response. | +| [groundingSupports](./ai.groundingmetadata.md#groundingmetadatagroundingsupports) | [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface)\[\] | A list of [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) objects. Each object details how specific segments of the model's response are supported by the groundingChunks. | | [retrievalQueries](./ai.groundingmetadata.md#groundingmetadataretrievalqueries) | string\[\] | | -| [webSearchQueries](./ai.groundingmetadata.md#groundingmetadatawebsearchqueries) | string\[\] | | +| [searchEntryPoint](./ai.groundingmetadata.md#groundingmetadatasearchentrypoint) | [SearchEntrypoint](./ai.searchentrypoint.md#searchentrypoint_interface) | Google Search entry point for web searches. This contains an HTML/CSS snippet that must be embedded in an app to display a Google Search entry point for follow-up web searches related to a model's “Grounded Response”. | +| [webSearchQueries](./ai.groundingmetadata.md#groundingmetadatawebsearchqueries) | string\[\] | A list of web search queries that the model performed to gather the grounding information. These can be used to allow users to explore the search results themselves. | ## GroundingMetadata.groundingAttributions > Warning: This API is now obsolete. > +> Use [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) instead. > Signature: @@ -38,16 +46,53 @@ export interface GroundingMetadata groundingAttributions: GroundingAttribution[]; ``` +## GroundingMetadata.groundingChunks + +A list of [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) objects. Each chunk represents a piece of retrieved content (for example, from a web page). that the model used to ground its response. + +Signature: + +```typescript +groundingChunks?: GroundingChunk[]; +``` + +## GroundingMetadata.groundingSupports + +A list of [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) objects. Each object details how specific segments of the model's response are supported by the `groundingChunks`. + +Signature: + +```typescript +groundingSupports?: GroundingSupport[]; +``` + ## GroundingMetadata.retrievalQueries +> Warning: This API is now obsolete. +> +> Use [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) instead. +> + Signature: ```typescript retrievalQueries?: string[]; ``` +## GroundingMetadata.searchEntryPoint + +Google Search entry point for web searches. This contains an HTML/CSS snippet that must be embedded in an app to display a Google Search entry point for follow-up web searches related to a model's “Grounded Response”. + +Signature: + +```typescript +searchEntryPoint?: SearchEntrypoint; +``` + ## GroundingMetadata.webSearchQueries +A list of web search queries that the model performed to gather the grounding information. These can be used to allow users to explore the search results themselves. + Signature: ```typescript diff --git a/docs-devsite/ai.groundingsupport.md b/docs-devsite/ai.groundingsupport.md new file mode 100644 index 00000000000..67eb190497c --- /dev/null +++ b/docs-devsite/ai.groundingsupport.md @@ -0,0 +1,46 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# GroundingSupport interface +Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks. + +Signature: + +```typescript +export interface GroundingSupport +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [groundingChunkIndices](./ai.groundingsupport.md#groundingsupportgroundingchunkindices) | number\[\] | A list of indices that refer to specific [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) objects within the [GroundingMetadata.groundingChunks](./ai.groundingmetadata.md#groundingmetadatagroundingchunks) array. These referenced chunks are the sources that support the claim made in the associated segment of the response. For example, an array [1, 3, 4] means that groundingChunks[1], groundingChunks[3], and groundingChunks[4] are the retrieved content supporting this part of the response. | +| [segment](./ai.groundingsupport.md#groundingsupportsegment) | [Segment](./ai.segment.md#segment_interface) | Specifies the segment of the model's response content that this grounding support pertains to. | + +## GroundingSupport.groundingChunkIndices + +A list of indices that refer to specific [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) objects within the [GroundingMetadata.groundingChunks](./ai.groundingmetadata.md#groundingmetadatagroundingchunks) array. These referenced chunks are the sources that support the claim made in the associated `segment` of the response. For example, an array `[1, 3, 4]` means that `groundingChunks[1]`, `groundingChunks[3]`, and `groundingChunks[4]` are the retrieved content supporting this part of the response. + +Signature: + +```typescript +groundingChunkIndices?: number[]; +``` + +## GroundingSupport.segment + +Specifies the segment of the model's response content that this grounding support pertains to. + +Signature: + +```typescript +segment?: Segment; +``` diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 286c8351fd7..079321332af 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -95,8 +95,12 @@ The Firebase AI Web SDK. | [GenerateContentStreamResult](./ai.generatecontentstreamresult.md#generatecontentstreamresult_interface) | Result object returned from [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) call. Iterate over stream to get chunks as they come in and/or use the response promise to get the aggregated response when the stream is done. | | [GenerationConfig](./ai.generationconfig.md#generationconfig_interface) | Config options for content-related requests | | [GenerativeContentBlob](./ai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | +| [GoogleSearch](./ai.googlesearch.md#googlesearch_interface) | Specifies the Google Search configuration. | +| [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface) | A tool that allows a Gemini model to connect to Google Search to access and incorporate up-to-date information from the web into its responses.Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or [Vertex AI Gemini API](https://cloud.google.com/terms/service-terms) (see Service Terms section within the Service Specific Terms). | | [GroundingAttribution](./ai.groundingattribution.md#groundingattribution_interface) | | -| [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | +| [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) | Represents a chunk of retrieved data that supports a claim in the model's response. This is part of the grounding information provided when grounding is enabled. | +| [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned when grounding is enabled.Currently, only Grounding with Google Search is supported (see [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface)).Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or [Vertex AI Gemini API](https://cloud.google.com/terms/service-terms) (see Service Terms section within the Service Specific Terms). | +| [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) | Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks. | | [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | | [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | | [ImagenGenerationResponse](./ai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | @@ -116,7 +120,8 @@ The Firebase AI Web SDK. | [SchemaParams](./ai.schemaparams.md#schemaparams_interface) | Params passed to [Schema](./ai.schema.md#schema_class) static methods to create specific [Schema](./ai.schema.md#schema_class) classes. | | [SchemaRequest](./ai.schemarequest.md#schemarequest_interface) | Final format for [Schema](./ai.schema.md#schema_class) params passed to backend requests. | | [SchemaShared](./ai.schemashared.md#schemashared_interface) | Basic [Schema](./ai.schema.md#schema_class) properties shared across several Schema-related types. | -| [Segment](./ai.segment.md#segment_interface) | | +| [SearchEntrypoint](./ai.searchentrypoint.md#searchentrypoint_interface) | Google search entry point. | +| [Segment](./ai.segment.md#segment_interface) | Represents a specific segment within a [Content](./ai.content.md#content_interface) object, often used to pinpoint the exact location of text or data that grounding information refers to. | | [StartChatParams](./ai.startchatparams.md#startchatparams_interface) | Params for [GenerativeModel.startChat()](./ai.generativemodel.md#generativemodelstartchat). | | [TextPart](./ai.textpart.md#textpart_interface) | Content part interface if the part represents a text string. | | [ToolConfig](./ai.toolconfig.md#toolconfig_interface) | Tool config. This config is shared for all tools provided in the request. | @@ -124,6 +129,7 @@ The Firebase AI Web SDK. | [VertexAIOptions](./ai.vertexaioptions.md#vertexaioptions_interface) | Options when initializing the Firebase AI SDK. | | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Describes the input video content. | | [WebAttribution](./ai.webattribution.md#webattribution_interface) | | +| [WebGroundingChunk](./ai.webgroundingchunk.md#webgroundingchunk_interface) | A grounding chunk from the web.Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". | ## Variables @@ -400,7 +406,7 @@ Defines a tool that model can call to access external knowledge. Signature: ```typescript -export declare type Tool = FunctionDeclarationsTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool; ``` ## TypedSchema diff --git a/docs-devsite/ai.searchentrypoint.md b/docs-devsite/ai.searchentrypoint.md new file mode 100644 index 00000000000..db35db06a49 --- /dev/null +++ b/docs-devsite/ai.searchentrypoint.md @@ -0,0 +1,48 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# SearchEntrypoint interface +Google search entry point. + +Signature: + +```typescript +export interface SearchEntrypoint +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [renderedContent](./ai.searchentrypoint.md#searchentrypointrenderedcontent) | string | HTML/CSS snippet that must be embedded in a web page. The snippet is designed to avoid undesired interaction with the rest of the page's CSS.To ensure proper rendering and prevent CSS conflicts, it is recommended to encapsulate this renderedContent within a shadow DOM when embedding it into a webpage. See [MDN: Using shadow DOM](https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM). | + +## SearchEntrypoint.renderedContent + +HTML/CSS snippet that must be embedded in a web page. The snippet is designed to avoid undesired interaction with the rest of the page's CSS. + +To ensure proper rendering and prevent CSS conflicts, it is recommended to encapsulate this `renderedContent` within a shadow DOM when embedding it into a webpage. See [MDN: Using shadow DOM](https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM). + +Signature: + +```typescript +renderedContent?: string; +``` + +### Example + + +```javascript +const container = document.createElement('div'); +document.body.appendChild(container); +container.attachShadow({ mode: 'open' }).innerHTML = renderedContent; + +``` + diff --git a/docs-devsite/ai.segment.md b/docs-devsite/ai.segment.md index 69f4aaf8407..35db1be5e83 100644 --- a/docs-devsite/ai.segment.md +++ b/docs-devsite/ai.segment.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # Segment interface +Represents a specific segment within a [Content](./ai.content.md#content_interface) object, often used to pinpoint the exact location of text or data that grounding information refers to. Signature: @@ -21,12 +22,15 @@ export interface Segment | Property | Type | Description | | --- | --- | --- | -| [endIndex](./ai.segment.md#segmentendindex) | number | | -| [partIndex](./ai.segment.md#segmentpartindex) | number | | -| [startIndex](./ai.segment.md#segmentstartindex) | number | | +| [endIndex](./ai.segment.md#segmentendindex) | number | The zero-based end index of the segment within the specified Part, measured in UTF-8 bytes. This offset is exclusive, meaning the character at this index is not included in the segment. | +| [partIndex](./ai.segment.md#segmentpartindex) | number | The zero-based index of the [Part](./ai.md#part) object within the parts array of its parent [Content](./ai.content.md#content_interface) object. This identifies which part of the content the segment belongs to. | +| [startIndex](./ai.segment.md#segmentstartindex) | number | The zero-based start index of the segment within the specified Part, measured in UTF-8 bytes. This offset is inclusive, starting from 0 at the beginning of the part's content (e.g., Part.text). | +| [text](./ai.segment.md#segmenttext) | string | The text corresponding to the segment from the response. | ## Segment.endIndex +The zero-based end index of the segment within the specified `Part`, measured in UTF-8 bytes. This offset is exclusive, meaning the character at this index is not included in the segment. + Signature: ```typescript @@ -35,6 +39,8 @@ endIndex: number; ## Segment.partIndex +The zero-based index of the [Part](./ai.md#part) object within the `parts` array of its parent [Content](./ai.content.md#content_interface) object. This identifies which part of the content the segment belongs to. + Signature: ```typescript @@ -43,8 +49,20 @@ partIndex: number; ## Segment.startIndex +The zero-based start index of the segment within the specified `Part`, measured in UTF-8 bytes. This offset is inclusive, starting from 0 at the beginning of the part's content (e.g., `Part.text`). + Signature: ```typescript startIndex: number; ``` + +## Segment.text + +The text corresponding to the segment from the response. + +Signature: + +```typescript +text: string; +``` diff --git a/docs-devsite/ai.webgroundingchunk.md b/docs-devsite/ai.webgroundingchunk.md new file mode 100644 index 00000000000..8d4c59f7e23 --- /dev/null +++ b/docs-devsite/ai.webgroundingchunk.md @@ -0,0 +1,61 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# WebGroundingChunk interface +A grounding chunk from the web. + +Important: If using Grounding with Google Search, you are required to comply with the [Service Specific Terms](https://cloud.google.com/terms/service-terms) for "Grounding with Google Search". + +Signature: + +```typescript +export interface WebGroundingChunk +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [domain](./ai.webgroundingchunk.md#webgroundingchunkdomain) | string | The domain of the original URI from which the content was retrieved.This property is only supported in the Vertex AI Gemini API ([VertexAIBackend](./ai.vertexaibackend.md#vertexaibackend_class)). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)), this property will be undefined. | +| [title](./ai.webgroundingchunk.md#webgroundingchunktitle) | string | The title of the retrieved web page. | +| [uri](./ai.webgroundingchunk.md#webgroundingchunkuri) | string | The URI of the retrieved web page. | + +## WebGroundingChunk.domain + +The domain of the original URI from which the content was retrieved. + +This property is only supported in the Vertex AI Gemini API ([VertexAIBackend](./ai.vertexaibackend.md#vertexaibackend_class)). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)), this property will be `undefined`. + +Signature: + +```typescript +domain?: string; +``` + +## WebGroundingChunk.title + +The title of the retrieved web page. + +Signature: + +```typescript +title?: string; +``` + +## WebGroundingChunk.uri + +The URI of the retrieved web page. + +Signature: + +```typescript +uri?: string; +``` diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index af877396cc8..4dd65d6469e 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -113,6 +113,46 @@ describe('Generate Content', () => { ).to.be.closeTo(4, TOKEN_COUNT_DELTA); }); + it('generateContent: google search grounding', async () => { + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig: commonGenerationConfig, + safetySettings: commonSafetySettings, + tools: [{ googleSearch: {} }] + }); + + const result = await model.generateContent( + 'What is the speed of light in a vaccuum in meters per second?' + ); + const response = result.response; + const trimmedText = response.text().trim(); + const groundingMetadata = response.candidates?.[0].groundingMetadata; + expect(trimmedText).to.contain('299,792,458'); + expect(groundingMetadata).to.exist; + expect(groundingMetadata!.searchEntryPoint?.renderedContent).to.contain( + 'div' + ); + expect( + groundingMetadata!.groundingChunks + ).to.have.length.greaterThanOrEqual(1); + groundingMetadata!.groundingChunks!.forEach(groundingChunk => { + expect(groundingChunk.web).to.exist; + expect(groundingChunk.web!.uri).to.exist; + }); + expect( + groundingMetadata?.groundingSupports + ).to.have.length.greaterThanOrEqual(1); + groundingMetadata!.groundingSupports!.forEach(groundingSupport => { + expect( + groundingSupport.groundingChunkIndices + ).to.have.length.greaterThanOrEqual(1); + expect(groundingSupport.segment).to.exist; + expect(groundingSupport.segment?.endIndex).to.exist; + expect(groundingSupport.segment?.text).to.exist; + // Since partIndex and startIndex are commonly 0, they may be omitted from responses. + }); + }); + it('generateContentStream: text input, text output', async () => { const model = getGenerativeModel(testConfig.ai, { model: testConfig.model, diff --git a/packages/ai/src/methods/generate-content.test.ts b/packages/ai/src/methods/generate-content.test.ts index 13250fd83dd..a5e4c0d1b57 100644 --- a/packages/ai/src/methods/generate-content.test.ts +++ b/packages/ai/src/methods/generate-content.test.ts @@ -193,6 +193,53 @@ describe('generateContent()', () => { match.any ); }); + it('google search grounding', async () => { + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-google-search-grounding.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const result = await generateContent( + fakeApiSettings, + 'model', + fakeRequestParams + ); + expect(result.response.text()).to.include('The temperature is 67°F (19°C)'); + const groundingMetadata = result.response.candidates?.[0].groundingMetadata; + expect(groundingMetadata).to.not.be.undefined; + expect(groundingMetadata!.searchEntryPoint?.renderedContent).to.contain( + 'div' + ); + expect(groundingMetadata!.groundingChunks?.length).to.equal(2); + expect(groundingMetadata!.groundingChunks?.[0].web?.uri).to.contain( + 'https://vertexaisearch.cloud.google.com' + ); + expect(groundingMetadata!.groundingChunks?.[0].web?.title).to.equal( + 'accuweather.com' + ); + expect(groundingMetadata!.groundingSupports?.length).to.equal(3); + expect( + groundingMetadata!.groundingSupports?.[0].groundingChunkIndices + ).to.deep.equal([0]); + expect(groundingMetadata!.groundingSupports?.[0].segment).to.deep.equal({ + endIndex: 56, + text: 'The current weather in London, United Kingdom is cloudy.' + }); + expect(groundingMetadata!.groundingSupports?.[0].segment?.partIndex).to.be + .undefined; + expect(groundingMetadata!.groundingSupports?.[0].segment?.startIndex).to.be + .undefined; + + expect(makeRequestStub).to.be.calledWith( + 'model', + Task.GENERATE_CONTENT, + fakeApiSettings, + false, + match.any + ); + }); it('blocked prompt', async () => { const mockResponse = getMockResponse( 'vertexAI', diff --git a/packages/ai/src/requests/request.ts b/packages/ai/src/requests/request.ts index 31c5e9b8125..3fa0b33012f 100644 --- a/packages/ai/src/requests/request.ts +++ b/packages/ai/src/requests/request.ts @@ -185,6 +185,7 @@ export async function makeRequest( } if ( response.status === 403 && + errorDetails && errorDetails.some( (detail: ErrorDetails) => detail.reason === 'SERVICE_DISABLED' ) && diff --git a/packages/ai/src/requests/stream-reader.ts b/packages/ai/src/requests/stream-reader.ts index 543d1d02266..c3a35b1da4a 100644 --- a/packages/ai/src/requests/stream-reader.ts +++ b/packages/ai/src/requests/stream-reader.ts @@ -190,6 +190,8 @@ export function aggregateResponses( candidate.finishMessage; aggregatedResponse.candidates[i].safetyRatings = candidate.safetyRatings; + aggregatedResponse.candidates[i].groundingMetadata = + candidate.groundingMetadata; /** * Candidates should always have content and parts, but this handles diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index 67f45095c2a..9d497a619d4 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -165,7 +165,7 @@ export interface RequestOptions { * Defines a tool that model can call to access external knowledge. * @public */ -export declare type Tool = FunctionDeclarationsTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool; /** * Structured representation of a function declaration as defined by the @@ -176,7 +176,7 @@ export declare type Tool = FunctionDeclarationsTool; * as a Tool by the model and executed by the client. * @public */ -export declare interface FunctionDeclaration { +export interface FunctionDeclaration { /** * The name of the function to call. Must start with a letter or an * underscore. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with @@ -196,13 +196,48 @@ export declare interface FunctionDeclaration { parameters?: ObjectSchemaInterface; } +/** + * A tool that allows a Gemini model to connect to Google Search to access and incorporate + * up-to-date information from the web into its responses. + * + * Important: If using Grounding with Google Search, you are required to comply with the + * "Grounding with Google Search" usage requirements for your chosen API provider: {@link https://ai.google.dev/gemini-api/terms#grounding-with-google-search | Gemini Developer API} + * or {@link https://cloud.google.com/terms/service-terms | Vertex AI Gemini API} + * (see Service Terms section within the Service Specific Terms). + * + * @public + */ +export interface GoogleSearchTool { + /** + * Specifies the Google Search configuration. + * Currently, this is an empty object, but it's reserved for future configuration options. + * Specifies the Google Search configuration. Currently, this is an empty object, but it's + * reserved for future configuration options. + * + * When using this feature, you are required to comply with the "Grounding with Google Search" + * usage requirements for your chosen API provider: {@link https://ai.google.dev/gemini-api/terms#grounding-with-google-search | Gemini Developer API} + * or {@link https://cloud.google.com/terms/service-terms | Vertex AI Gemini API} + * (see Service Terms section within the Service Specific Terms). + */ + googleSearch: GoogleSearch; +} + +/** + * Specifies the Google Search configuration. + * + * @remarks Currently, this is an empty object, but it's reserved for future configuration options. + * + * @public + */ +export interface GoogleSearch {} + /** * A `FunctionDeclarationsTool` is a piece of code that enables the system to * interact with external systems to perform an action, or set of actions, * outside of knowledge and scope of the model. * @public */ -export declare interface FunctionDeclarationsTool { +export interface FunctionDeclarationsTool { /** * Optional. One or more function declarations * to be passed to the model along with the current user query. Model may diff --git a/packages/ai/src/types/responses.ts b/packages/ai/src/types/responses.ts index e33b8a86bd3..aa72f946ed9 100644 --- a/packages/ai/src/types/responses.ts +++ b/packages/ai/src/types/responses.ts @@ -171,36 +171,174 @@ export interface Citation { } /** - * Metadata returned to client when grounding is enabled. + * Metadata returned when grounding is enabled. + * + * Currently, only Grounding with Google Search is supported (see {@link GoogleSearchTool}). + * + * Important: If using Grounding with Google Search, you are required to comply with the + * "Grounding with Google Search" usage requirements for your chosen API provider: {@link https://ai.google.dev/gemini-api/terms#grounding-with-google-search | Gemini Developer API} + * or {@link https://cloud.google.com/terms/service-terms | Vertex AI Gemini API} + * (see Service Terms section within the Service Specific Terms). + * * @public */ export interface GroundingMetadata { + /** + * Google Search entry point for web searches. This contains an HTML/CSS snippet that must be + * embedded in an app to display a Google Search entry point for follow-up web searches related to + * a model's “Grounded Response”. + */ + searchEntryPoint?: SearchEntrypoint; + /** + * A list of {@link GroundingChunk} objects. Each chunk represents a piece of retrieved content + * (for example, from a web page). that the model used to ground its response. + */ + groundingChunks?: GroundingChunk[]; + /** + * A list of {@link GroundingSupport} objects. Each object details how specific segments of the + * model's response are supported by the `groundingChunks`. + */ + groundingSupports?: GroundingSupport[]; + /** + * A list of web search queries that the model performed to gather the grounding information. + * These can be used to allow users to explore the search results themselves. + */ webSearchQueries?: string[]; + /** + * @deprecated Use {@link GroundingSupport} instead. + */ retrievalQueries?: string[]; /** - * @deprecated + * @deprecated Use {@link GroundingChunk} instead. */ groundingAttributions: GroundingAttribution[]; } /** - * @deprecated + * Google search entry point. + * * @public */ -export interface GroundingAttribution { - segment: Segment; - confidenceScore?: number; - web?: WebAttribution; - retrievedContext?: RetrievedContextAttribution; +export interface SearchEntrypoint { + /** + * HTML/CSS snippet that must be embedded in a web page. The snippet is designed to avoid + * undesired interaction with the rest of the page's CSS. + * + * To ensure proper rendering and prevent CSS conflicts, it is recommended + * to encapsulate this `renderedContent` within a shadow DOM when embedding it + * into a webpage. See {@link https://developer.mozilla.org/en-US/docs/Web/API/Web_components/Using_shadow_DOM | MDN: Using shadow DOM}. + * + * @example + * ```javascript + * const container = document.createElement('div'); + * document.body.appendChild(container); + * container.attachShadow({ mode: 'open' }).innerHTML = renderedContent; + * ``` + */ + renderedContent?: string; +} + +/** + * Represents a chunk of retrieved data that supports a claim in the model's response. This is part + * of the grounding information provided when grounding is enabled. + * + * @public + */ +export interface GroundingChunk { + /** + * Contains details if the grounding chunk is from a web source. + */ + web?: WebGroundingChunk; } /** + * A grounding chunk from the web. + * + * Important: If using Grounding with Google Search, you are required to comply with the + * {@link https://cloud.google.com/terms/service-terms | Service Specific Terms} for "Grounding with Google Search". + * + * @public + */ +export interface WebGroundingChunk { + /** + * The URI of the retrieved web page. + */ + uri?: string; + /** + * The title of the retrieved web page. + */ + title?: string; + /** + * The domain of the original URI from which the content was retrieved. + * + * This property is only supported in the Vertex AI Gemini API ({@link VertexAIBackend}). + * When using the Gemini Developer API ({@link GoogleAIBackend}), this property will be + * `undefined`. + */ + domain?: string; +} + +/** + * Provides information about how a specific segment of the model's response is supported by the + * retrieved grounding chunks. + * + * @public + */ +export interface GroundingSupport { + /** + * Specifies the segment of the model's response content that this grounding support pertains to. + */ + segment?: Segment; + /** + * A list of indices that refer to specific {@link GroundingChunk} objects within the + * {@link GroundingMetadata.groundingChunks} array. These referenced chunks + * are the sources that support the claim made in the associated `segment` of the response. + * For example, an array `[1, 3, 4]` means that `groundingChunks[1]`, `groundingChunks[3]`, + * and `groundingChunks[4]` are the retrieved content supporting this part of the response. + */ + groundingChunkIndices?: number[]; +} + +/** + * Represents a specific segment within a {@link Content} object, often used to + * pinpoint the exact location of text or data that grounding information refers to. + * * @public */ export interface Segment { + /** + * The zero-based index of the {@link Part} object within the `parts` array + * of its parent {@link Content} object. This identifies which part of the + * content the segment belongs to. + */ partIndex: number; + /** + * The zero-based start index of the segment within the specified `Part`, + * measured in UTF-8 bytes. This offset is inclusive, starting from 0 at the + * beginning of the part's content (e.g., `Part.text`). + */ startIndex: number; + /** + * The zero-based end index of the segment within the specified `Part`, + * measured in UTF-8 bytes. This offset is exclusive, meaning the character + * at this index is not included in the segment. + */ endIndex: number; + /** + * The text corresponding to the segment from the response. + */ + text: string; +} + +/** + * @deprecated + * @public + */ +export interface GroundingAttribution { + segment: Segment; + confidenceScore?: number; + web?: WebAttribution; + retrievedContext?: RetrievedContextAttribution; } /**