fix(firestore-bigquery-export): fixed timestamp as a field-name partitioning

dackers86 · web-flow · commit b452cfb23e77 · 2024-05-15T22:04:52.000+01:00
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/package-lock.json b/firestore-bigquery-export/firestore-bigquery-change-tracker/package-lock.json
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json
@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.33",
+  "version": "1.1.34",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/alternativeProject.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/alternativeProject.test.ts
@@ -21,7 +21,7 @@ describe("Using an alternative bigquery project", () => {
     randomID = (Math.random() + 1).toString(36).substring(7);
     datasetId = `dataset_${randomID}`;
     tableId = `table_${randomID}`;
-    bq = new BigQuery();
+    bq = new BigQuery({ projectId: process.env.PROJECT_ID });
   });
   describe("has a valid alternative project id", () => {
     beforeEach(async () => {
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/checkUpdates.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/checkUpdates.test.ts
@@ -10,7 +10,7 @@ import {
 
 process.env.PROJECT_ID = "extensions-testing";
 
-const bq: BigQuery = new BigQuery();
+const bq: BigQuery = new BigQuery({ projectId: process.env.PROJECT_ID });
 const event: FirestoreDocumentChangeEvent = changeTrackerEvent({});
 let randomID: string;
 let datasetId: string;
@@ -736,6 +736,7 @@ describe("Checking updates", () => {
         }).record([event]);
 
         const raw_changelog_table = bq.dataset(datasetId).table(tableId_raw);
+
         const [metadata] = await raw_changelog_table.getMetadata();
 
         expect(
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/clustering.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/clustering.test.ts
@@ -6,7 +6,7 @@ import { deleteTable } from "../fixtures/clearTables";
 
 process.env.PROJECT_ID = "extensions-testing";
 
-const bq: BigQuery = new BigQuery();
+const bq: BigQuery = new BigQuery({ projectId: process.env.PROJECT_ID });
 const event: FirestoreDocumentChangeEvent = changeTrackerEvent({});
 let randomID: string;
 let datasetId: string;
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/e2e.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/e2e.test.ts
@@ -18,7 +18,7 @@ process.env.PROJECT_ID = "extensions-testing";
 const consoleLogSpy = jest.spyOn(logger, "log").mockImplementation();
 const consoleLogSpyWarn = jest.spyOn(logger, "warn").mockImplementation();
 
-const bq: BigQuery = new BigQuery();
+const bq: BigQuery = new BigQuery({ projectId: process.env.PROJECT_ID });
 const event: FirestoreDocumentChangeEvent = changeTrackerEvent({});
 let randomID: string;
 let datasetId: string;
@@ -267,6 +267,41 @@ describe("e2e", () => {
         );
       });
 
+      test("successfully partitions with a valid Firebase Timestamp value with `timestamp` as field name and Timestamp type", async () => {
+        const created = firestore.Timestamp.now();
+        const expectedDate = created.toDate().toISOString().substring(0, 22);
+
+        const event: FirestoreDocumentChangeEvent = changeTrackerEvent({
+          data: { created },
+        });
+
+        await changeTracker({
+          datasetId,
+          tableId,
+          timePartitioning: "DAY",
+          timePartitioningField: "timestamp",
+          timePartitioningFieldType: "TIMESTAMP",
+          timePartitioningFirestoreField: "created",
+        }).record([event]);
+
+        const [metadata] = await dataset.table(`${tableId_raw}`).getMetadata();
+
+        const [changeLogRows] = await getBigQueryTableData(
+          process.env.PROJECT_ID,
+          datasetId,
+          tableId
+        );
+
+        expect(metadata.timePartitioning).toBeDefined();
+        expect(metadata.timePartitioning.type).toEqual("DAY");
+        expect(metadata.timePartitioning.field).toEqual("timestamp");
+
+        //TODO: check data has been added successfully
+        expect(changeLogRows[0].timestamp.value).toBe(
+          BigQuery.timestamp(created.toDate()).value
+        );
+      });
+
       test("old_data is null if is not provided", async () => {
         const event: FirestoreDocumentChangeEvent = changeTrackerEvent({
           data: { foo: "foo" },
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning.test.ts
@@ -6,7 +6,6 @@ import { ChangeType, FirestoreDocumentChangeEvent } from "../..";
 import { FirestoreBigQueryEventHistoryTrackerConfig } from "../../bigquery";
 import { Partitioning } from "../../bigquery/partitioning";
 import { deleteTable } from "../fixtures/clearTables";
-import { changeTracker } from "../fixtures/changeTracker";
 
 let bq: BigQuery;
 let dataset: Dataset;
@@ -16,7 +15,7 @@ let datasetId: string;
 
 describe("processing partitions on a new table", () => {
   beforeAll(async () => {
-    bq = new BigQuery();
+    bq = new BigQuery({ projectId: process.env.PROJECT_ID });
     randomID = (Math.random() + 1).toString(36).substring(7);
     datasetId = `bq_${randomID}`;
     [dataset] = await bq.createDataset(datasetId, {
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/stresstest.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/stresstest.test.ts
@@ -7,7 +7,7 @@ import { ChangeType } from "../..";
 import { buildLatestSnapshotViewQuery } from "../../bigquery/snapshot";
 process.env.PROJECT_ID = "extensions-testing";
 
-const bq = new BigQuery();
+const bq = new BigQuery({ projectId: process.env.PROJECT_ID });
 let event: FirestoreDocumentChangeEvent;
 let randomID: string;
 let datasetId: string;
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/wildcardDocument.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/wildcardDocument.test.ts
@@ -6,7 +6,7 @@ import { deleteTable } from "../fixtures/clearTables";
 
 process.env.PROJECT_ID = "extensions-testing";
 
-const bq = new BigQuery();
+const bq = new BigQuery({ projectId: process.env.PROJECT_ID });
 let event: FirestoreDocumentChangeEvent;
 let randomID: string;
 let datasetId: string;
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/queries.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/queries.ts
@@ -9,7 +9,7 @@ export const defaultQuery = (
       LIMIT 1`;
 
 export const getBigQueryTableData = async (bqProjectId, datasetId, tableId) => {
-  const bq = new BigQuery();
+  const bq = new BigQuery({ projectId: bqProjectId });
 
   // Setup queries
   const [changeLogQuery] = await bq.createQueryJob({
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning.ts
@@ -239,41 +239,48 @@ export class Partitioning {
   }
 
   customFieldExists(fields = []) {
-    if (!fields.length) return false;
-
+    /** Extract the time partioning field name */
     const { timePartitioningField } = this.config;
 
+    /** Return based the field already exist */
     return fields.map(($) => $.name).includes(timePartitioningField);
   }
 
-  private async shouldAddPartitioningToSchema(): Promise<{
+  private async shouldAddPartitioningToSchema(fields: string[]): Promise<{
     proceed: boolean;
     message: string;
   }> {
     if (!this.isPartitioningEnabled()) {
       return { proceed: false, message: "Partitioning not enabled" };
     }
+
     if (!this.hasValidTableReference()) {
       return { proceed: false, message: "Invalid table reference" };
     }
+
     if (!this.hasValidCustomPartitionConfig()) {
       return { proceed: false, message: "Invalid partition config" };
     }
+
     if (!this.hasValidTimePartitionType()) {
       return { proceed: false, message: "Invalid partition type" };
     }
+
     if (!this.hasValidTimePartitionOption()) {
       return { proceed: false, message: "Invalid partition option" };
     }
+
     if (this.hasHourAndDatePartitionConfig()) {
       return {
         proceed: false,
         message: "Invalid partitioning and field type combination",
       };
     }
-    if (this.customFieldExists()) {
+
+    if (this.customFieldExists(fields)) {
       return { proceed: false, message: "Field already exists on schema" };
     }
+
     if (await this.isTablePartitioned()) {
       return { proceed: false, message: "Table is already partitioned" };
     }
@@ -284,7 +291,10 @@ export class Partitioning {
   }
 
   async addPartitioningToSchema(fields = []): Promise<void> {
-    const { proceed, message } = await this.shouldAddPartitioningToSchema();
+    const { proceed, message } = await this.shouldAddPartitioningToSchema(
+      fields
+    );
+
     if (!proceed) {
       functions.logger.warn(`Did not add partitioning to schema: ${message}`);
       return;
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/tsconfig.test.json b/firestore-bigquery-export/firestore-bigquery-change-tracker/tsconfig.test.json
@@ -1,7 +1,7 @@
 {
   "extends": "./tsconfig.json",
   "compilerOptions": {
-    "types": ["node", "jest"],
-    "include": ["src/**/*.ts", "src/**/*.test.ts"]
-  }
+    "types": ["node", "jest"]
+  },
+  "include": ["src/**/*.ts", "src/**/*.test.ts"]
 }
diff --git a/firestore-bigquery-export/functions/__tests__/e2e.test.ts b/firestore-bigquery-export/functions/__tests__/e2e.test.ts
@@ -2,13 +2,13 @@ import * as admin from "firebase-admin";
 import { BigQuery } from "@google-cloud/bigquery";
 
 /** Set defaults */
-const bqProjectId = "dev-extensions-testing";
+const bqProjectId = "extensions-testing";
 const datasetId = "firestore_export";
 const tableId = "bq_e2e_test_raw_changelog";
 
 /** Init resources */
 admin.initializeApp({ projectId: bqProjectId });
-const bq = new BigQuery({ projectId: "dev-extensions-testing" });
+const bq = new BigQuery({ projectId: "extensions-testing" });
 import { documentData } from "./fixtures/documentData";
 
 /***
diff --git a/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/e2e/e2e.test.ts b/firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/e2e/e2e.test.ts
@@ -2,7 +2,7 @@ import { BigQuery } from "@google-cloud/bigquery";
 import { getQueryResult, randomId, setupBQ } from "./helpers/setup";
 import executeScript from "./helpers/executeScript";
 
-const bq = new BigQuery({ projectId: "dev-extensions-testing" });
+const bq = new BigQuery({ projectId: "extensions-testing" });
 
 const datasetPrefix = "e2e_test_";
 
