Skip to content

Commit 553a05c

Browse files
maneeshtmaneesht
committed
Merge remote-tracking branch 'origin/main' into mtewani/fix-fdc-typings
2 parents 1ffb378 + dcfb3da commit 553a05c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+2760
-113
lines changed

‎.changeset/chilled-tips-judge.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
'@firebase/vertexai': minor
3+
'firebase': minor
4+
---
5+
6+
Added support for modality-based token count.

‎.changeset/violet-planets-impress.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
'firebase': minor
3+
'@firebase/vertexai': minor
4+
---
5+
6+
**Public Preview** Added support for generating images using the Imagen 3 model.

‎common/api-review/vertexai.api.md

Lines changed: 119 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ export interface CountTokensRequest {
9292

9393
// @public
9494
export interface CountTokensResponse {
95+
promptTokensDetails?: ModalityTokenCount[];
9596
totalBillableCharacters?: number;
9697
totalTokens: number;
9798
}
@@ -323,16 +324,14 @@ export interface GenerativeContentBlob {
323324
}
324325

325326
// @public
326-
export class GenerativeModel {
327+
export class GenerativeModel extends VertexAIModel {
327328
constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions);
328329
countTokens(request: CountTokensRequest | string | Array<string | Part>): Promise<CountTokensResponse>;
329330
generateContent(request: GenerateContentRequest | string | Array<string | Part>): Promise<GenerateContentResult>;
330331
generateContentStream(request: GenerateContentRequest | string | Array<string | Part>): Promise<GenerateContentStreamResult>;
331332
// (undocumented)
332333
generationConfig: GenerationConfig;
333334
// (undocumented)
334-
model: string;
335-
// (undocumented)
336335
requestOptions?: RequestOptions;
337336
// (undocumented)
338337
safetySettings: SafetySetting[];
@@ -348,6 +347,9 @@ export class GenerativeModel {
348347
// @public
349348
export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel;
350349

350+
// @beta
351+
export function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
352+
351353
// @public
352354
export function getVertexAI(app?: FirebaseApp, options?: VertexAIOptions): VertexAI;
353355

@@ -429,6 +431,90 @@ export enum HarmSeverity {
429431
HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE"
430432
}
431433

434+
// @beta
435+
export enum ImagenAspectRatio {
436+
LANDSCAPE_16x9 = "16:9",
437+
LANDSCAPE_3x4 = "3:4",
438+
PORTRAIT_4x3 = "4:3",
439+
PORTRAIT_9x16 = "9:16",
440+
SQUARE = "1:1"
441+
}
442+
443+
// @public
444+
export interface ImagenGCSImage {
445+
gcsURI: string;
446+
mimeType: string;
447+
}
448+
449+
// @beta
450+
export interface ImagenGenerationConfig {
451+
addWatermark?: boolean;
452+
aspectRatio?: ImagenAspectRatio;
453+
imageFormat?: ImagenImageFormat;
454+
negativePrompt?: string;
455+
numberOfImages?: number;
456+
}
457+
458+
// @beta
459+
export interface ImagenGenerationResponse<T extends ImagenInlineImage | ImagenGCSImage> {
460+
filteredReason?: string;
461+
images: T[];
462+
}
463+
464+
// @beta
465+
export class ImagenImageFormat {
466+
compressionQuality?: number;
467+
static jpeg(compressionQuality?: number): ImagenImageFormat;
468+
mimeType: string;
469+
static png(): ImagenImageFormat;
470+
}
471+
472+
// @beta
473+
export interface ImagenInlineImage {
474+
bytesBase64Encoded: string;
475+
mimeType: string;
476+
}
477+
478+
// @beta
479+
export class ImagenModel extends VertexAIModel {
480+
constructor(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions | undefined);
481+
generateImages(prompt: string): Promise<ImagenGenerationResponse<ImagenInlineImage>>;
482+
// @internal
483+
generateImagesGCS(prompt: string, gcsURI: string): Promise<ImagenGenerationResponse<ImagenGCSImage>>;
484+
generationConfig?: ImagenGenerationConfig;
485+
// (undocumented)
486+
requestOptions?: RequestOptions | undefined;
487+
safetySettings?: ImagenSafetySettings;
488+
}
489+
490+
// @beta
491+
export interface ImagenModelParams {
492+
generationConfig?: ImagenGenerationConfig;
493+
model: string;
494+
safetySettings?: ImagenSafetySettings;
495+
}
496+
497+
// @beta
498+
export enum ImagenPersonFilterLevel {
499+
ALLOW_ADULT = "allow_adult",
500+
ALLOW_ALL = "allow_all",
501+
BLOCK_ALL = "dont_allow"
502+
}
503+
504+
// @beta
505+
export enum ImagenSafetyFilterLevel {
506+
BLOCK_LOW_AND_ABOVE = "block_low_and_above",
507+
BLOCK_MEDIUM_AND_ABOVE = "block_medium_and_above",
508+
BLOCK_NONE = "block_none",
509+
BLOCK_ONLY_HIGH = "block_only_high"
510+
}
511+
512+
// @beta
513+
export interface ImagenSafetySettings {
514+
personFilterLevel?: ImagenPersonFilterLevel;
515+
safetyFilterLevel?: ImagenSafetyFilterLevel;
516+
}
517+
432518
// @public
433519
export interface InlineDataPart {
434520
// (undocumented)
@@ -447,6 +533,22 @@ export class IntegerSchema extends Schema {
447533
constructor(schemaParams?: SchemaParams);
448534
}
449535

536+
// @public
537+
export enum Modality {
538+
AUDIO = "AUDIO",
539+
DOCUMENT = "DOCUMENT",
540+
IMAGE = "IMAGE",
541+
MODALITY_UNSPECIFIED = "MODALITY_UNSPECIFIED",
542+
TEXT = "TEXT",
543+
VIDEO = "VIDEO"
544+
}
545+
546+
// @public
547+
export interface ModalityTokenCount {
548+
modality: Modality;
549+
tokenCount: number;
550+
}
551+
450552
// @public
451553
export interface ModelParams extends BaseParams {
452554
// (undocumented)
@@ -682,8 +784,12 @@ export interface UsageMetadata {
682784
// (undocumented)
683785
candidatesTokenCount: number;
684786
// (undocumented)
787+
candidatesTokensDetails?: ModalityTokenCount[];
788+
// (undocumented)
685789
promptTokenCount: number;
686790
// (undocumented)
791+
promptTokensDetails?: ModalityTokenCount[];
792+
// (undocumented)
687793
totalTokenCount: number;
688794
}
689795

@@ -718,6 +824,16 @@ export const enum VertexAIErrorCode {
718824
RESPONSE_ERROR = "response-error"
719825
}
720826

827+
// @public
828+
export abstract class VertexAIModel {
829+
// @internal
830+
protected constructor(vertexAI: VertexAI, modelName: string);
831+
// @internal (undocumented)
832+
protected _apiSettings: ApiSettings;
833+
readonly model: string;
834+
static normalizeModelName(modelName: string): string;
835+
}
836+
721837
// @public
722838
export interface VertexAIOptions {
723839
// (undocumented)

‎docs-devsite/_toc.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -536,10 +536,28 @@ toc:
536536
path: /docs/reference/js/vertexai.groundingattribution.md
537537
- title: GroundingMetadata
538538
path: /docs/reference/js/vertexai.groundingmetadata.md
539+
- title: ImagenGCSImage
540+
path: /docs/reference/js/vertexai.imagengcsimage.md
541+
- title: ImagenGenerationConfig
542+
path: /docs/reference/js/vertexai.imagengenerationconfig.md
543+
- title: ImagenGenerationResponse
544+
path: /docs/reference/js/vertexai.imagengenerationresponse.md
545+
- title: ImagenImageFormat
546+
path: /docs/reference/js/vertexai.imagenimageformat.md
547+
- title: ImagenInlineImage
548+
path: /docs/reference/js/vertexai.imageninlineimage.md
549+
- title: ImagenModel
550+
path: /docs/reference/js/vertexai.imagenmodel.md
551+
- title: ImagenModelParams
552+
path: /docs/reference/js/vertexai.imagenmodelparams.md
553+
- title: ImagenSafetySettings
554+
path: /docs/reference/js/vertexai.imagensafetysettings.md
539555
- title: InlineDataPart
540556
path: /docs/reference/js/vertexai.inlinedatapart.md
541557
- title: IntegerSchema
542558
path: /docs/reference/js/vertexai.integerschema.md
559+
- title: ModalityTokenCount
560+
path: /docs/reference/js/vertexai.modalitytokencount.md
543561
- title: ModelParams
544562
path: /docs/reference/js/vertexai.modelparams.md
545563
- title: NumberSchema
@@ -584,6 +602,8 @@ toc:
584602
path: /docs/reference/js/vertexai.vertexai.md
585603
- title: VertexAIError
586604
path: /docs/reference/js/vertexai.vertexaierror.md
605+
- title: VertexAIModel
606+
path: /docs/reference/js/vertexai.vertexaimodel.md
587607
- title: VertexAIOptions
588608
path: /docs/reference/js/vertexai.vertexaioptions.md
589609
- title: VideoMetadata

‎docs-devsite/vertexai.counttokensresponse.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,20 @@ export interface CountTokensResponse
2222

2323
| Property | Type | Description |
2424
| --- | --- | --- |
25+
| [promptTokensDetails](./vertexai.counttokensresponse.md#counttokensresponseprompttokensdetails) | [ModalityTokenCount](./vertexai.modalitytokencount.md#modalitytokencount_interface)<!-- -->\[\] | The breakdown, by modality, of how many tokens are consumed by the prompt. |
2526
| [totalBillableCharacters](./vertexai.counttokensresponse.md#counttokensresponsetotalbillablecharacters) | number | The total number of billable characters counted across all instances from the request. |
2627
| [totalTokens](./vertexai.counttokensresponse.md#counttokensresponsetotaltokens) | number | The total number of tokens counted across all instances from the request. |
2728

29+
## CountTokensResponse.promptTokensDetails
30+
31+
The breakdown, by modality, of how many tokens are consumed by the prompt.
32+
33+
<b>Signature:</b>
34+
35+
```typescript
36+
promptTokensDetails?: ModalityTokenCount[];
37+
```
38+
2839
## CountTokensResponse.totalBillableCharacters
2940

3041
The total number of billable characters counted across all instances from the request.

‎docs-devsite/vertexai.generativemodel.md

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ Class for generative model APIs.
1515
<b>Signature:</b>
1616

1717
```typescript
18-
export declare class GenerativeModel
18+
export declare class GenerativeModel extends VertexAIModel
1919
```
20+
<b>Extends:</b> [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class)
2021
2122
## Constructors
2223
@@ -29,7 +30,6 @@ export declare class GenerativeModel
2930
| Property | Modifiers | Type | Description |
3031
| --- | --- | --- | --- |
3132
| [generationConfig](./vertexai.generativemodel.md#generativemodelgenerationconfig) | | [GenerationConfig](./vertexai.generationconfig.md#generationconfig_interface) | |
32-
| [model](./vertexai.generativemodel.md#generativemodelmodel) | | string | |
3333
| [requestOptions](./vertexai.generativemodel.md#generativemodelrequestoptions) | | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | |
3434
| [safetySettings](./vertexai.generativemodel.md#generativemodelsafetysettings) | | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface)<!-- -->\[\] | |
3535
| [systemInstruction](./vertexai.generativemodel.md#generativemodelsysteminstruction) | | [Content](./vertexai.content.md#content_interface) | |
@@ -71,14 +71,6 @@ constructor(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: Reque
7171
generationConfig: GenerationConfig;
7272
```
7373
74-
## GenerativeModel.model
75-
76-
<b>Signature:</b>
77-
78-
```typescript
79-
model: string;
80-
```
81-
8274
## GenerativeModel.requestOptions
8375
8476
<b>Signature:</b>

‎docs-devsite/vertexai.imagengcsimage.md

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
Project: /docs/reference/js/_project.yaml
2+
Book: /docs/reference/_book.yaml
3+
page_type: reference
4+
5+
{% comment %}
6+
DO NOT EDIT THIS FILE!
7+
This is generated by the JS SDK team, and any local changes will be
8+
overwritten. Changes should be made in the source code at
9+
https://github.com/firebase/firebase-js-sdk
10+
{% endcomment %}
11+
12+
# ImagenGCSImage interface
13+
An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.
14+
15+
This feature is not available yet.
16+
17+
<b>Signature:</b>
18+
19+
```typescript
20+
export interface ImagenGCSImage
21+
```
22+
23+
## Properties
24+
25+
| Property | Type | Description |
26+
| --- | --- | --- |
27+
| [gcsURI](./vertexai.imagengcsimage.md#imagengcsimagegcsuri) | string | The URI of the file stored in a Cloud Storage for Firebase bucket. |
28+
| [mimeType](./vertexai.imagengcsimage.md#imagengcsimagemimetype) | string | The MIME type of the image; either <code>&quot;image/png&quot;</code> or <code>&quot;image/jpeg&quot;</code>.<!-- -->To request a different format, set the <code>imageFormat</code> property in your <code>[ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface)</code>. |
29+
30+
## ImagenGCSImage.gcsURI
31+
32+
The URI of the file stored in a Cloud Storage for Firebase bucket.
33+
34+
<b>Signature:</b>
35+
36+
```typescript
37+
gcsURI: string;
38+
```
39+
40+
### Example
41+
42+
`"gs://bucket-name/path/sample_0.jpg"`<!-- -->.
43+
44+
## ImagenGCSImage.mimeType
45+
46+
The MIME type of the image; either `"image/png"` or `"image/jpeg"`<!-- -->.
47+
48+
To request a different format, set the `imageFormat` property in your <code>[ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface)</code>.
49+
50+
<b>Signature:</b>
51+
52+
```typescript
53+
mimeType: string;
54+
```

0 commit comments

Comments
 (0)