fix(firestore-bigquery-export): fix occasional duplicate entries in BigQuery (#118)

Author: jmwski

.prettierignore

@@ -7,6 +7,7 @@ package-lock.json
 # generated files
 README.md
 **/functions/lib/**
+**/lib/**
 **/dist/**
 
 # extension install md files

firestore-bigquery-export/firestore-bigquery-change-tracker/package.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {

firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts

@@ -51,29 +51,38 @@ export class FirestoreBigQueryEventHistoryTracker
 
   async record(events: FirestoreDocumentChangeEvent[]) {
     await this.initialize();
-
+    const options = {
+      raw: true,
+    };
     const rows = events.map((event) => {
-      // This must match firestoreToBQTable().
       return {
-        timestamp: event.timestamp,
-        event_id: event.eventId,
-        document_name: event.documentName,
-        operation: ChangeType[event.operation],
-        data: JSON.stringify(event.data),
+        insertId: event.eventId,
+        json: {
+          timestamp: event.timestamp,
+          event_id: event.eventId,
+          document_name: event.documentName,
+          operation: ChangeType[event.operation],
+          data: JSON.stringify(event.data),
+        },
       };
     });
-    await this.insertData(rows);
+    await this.insertData(rows, options);
   }
 
   /**
    * Inserts rows of data into the BigQuery raw change log table.
    */
-  private async insertData(rows: bigquery.RowMetadata[]) {
+  private async insertData(rows: bigquery.RowMetadata[], options: object) {
+    const payload = {
+      skipInvalidRows: false,
+      ignoreUnkownValues: false,
+      rows: rows,
+    };
     try {
       const dataset = this.bq.dataset(this.config.datasetId);
       const table = dataset.table(this.rawChangeLogTableName());
       logs.dataInserting(rows.length);
-      await table.insert(rows);
+      await table.insert(payload, options);
       logs.dataInserted(rows.length);
     } catch (e) {
       // Reinitializing in case the destintation table is modified.

firestore-bigquery-export/firestore-bigquery-change-tracker/tsconfig.json

@@ -8,5 +8,6 @@
     "target": "es6"
   },
   "compileOnSave": true,
-  "include": ["src"]
+  "include": ["src"],
+  "exclude": ["**/node_modules/@types/jest/**"]
 }

firestore-bigquery-export/functions/package.json

@@ -14,7 +14,7 @@
   "main": "lib/index.js",
   "dependencies": {
     "firebase-admin": "~7.0.0",
-    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.0.0",
+    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.0.1",
     "firebase-functions": "^2.3.0",
     "sql-formatter": "^2.3.3"
   },

firestore-bigquery-export/scripts/gen-schema-view/tsconfig.json

@@ -9,5 +9,6 @@
     "types": ["node", "mocha", "chai"]
   },
   "compileOnSave": true,
-  "include": ["src"]
+  "include": ["src"],
+  "exclude": ["**/node_modules/@types/jest/**"]
 }

firestore-bigquery-export/scripts/import/package.json

@@ -24,7 +24,7 @@
   "author": "Jan Wyszynski <wyszynski@google.com>",
   "license": "Apache-2.0",
   "dependencies": {
-    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.0.0",
+    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.0.1",
     "@google-cloud/bigquery": "^2.1.0",
     "firebase-admin": "^7.1.1",
     "firebase-functions": "^2.2.1",

firestore-bigquery-export/scripts/import/tsconfig.json

@@ -8,5 +8,6 @@
     "target": "es6"
   },
   "compileOnSave": true,
-  "include": ["src"]
+  "include": ["src"],
+  "exclude": ["**/node_modules/@types/jest/**"]
 }