Merge pull request #197 from joaqcid/fix-bq-import

fix(fs-bq-import-collection): update bigquery lib and insert api usage

Author: Ashley Gau

firestore-bigquery-export/firestore-bigquery-change-tracker/package.json

@@ -22,13 +22,13 @@
   "author": "Jan Wyszynski <wyszynski@google.com>",
   "license": "Apache-2.0",
   "dependencies": {
-    "@google-cloud/bigquery": "^2.1.0",
+    "@google-cloud/bigquery": "^4.7.0",
     "firebase-admin": "^7.1.1",
     "firebase-functions": "^2.2.1",
-    "lodash": "^4.17.14",
-    "sql-formatter": "^2.3.3",
     "generate-schema": "^2.6.0",
-    "inquirer": "^6.4.0"
+    "inquirer": "^6.4.0",
+    "lodash": "^4.17.14",
+    "sql-formatter": "^2.3.3"
   },
   "devDependencies": {
     "chai": "^4.2.0",

firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts

@@ -24,7 +24,6 @@ import {
   FirestoreDocumentChangeEvent,
 } from "../tracker";
 import * as logs from "../logs";
-import { BigQuery } from "@google-cloud/bigquery";
 
 export interface FirestoreBigQueryEventHistoryTrackerConfig {
   datasetId: string;
@@ -51,9 +50,6 @@ export class FirestoreBigQueryEventHistoryTracker
 
   async record(events: FirestoreDocumentChangeEvent[]) {
     await this.initialize();
-    const options = {
-      raw: true,
-    };
     const rows = events.map((event) => {
       return {
         insertId: event.eventId,
@@ -66,23 +62,23 @@ export class FirestoreBigQueryEventHistoryTracker
         },
       };
     });
-    await this.insertData(rows, options);
+    await this.insertData(rows);
   }
 
   /**
    * Inserts rows of data into the BigQuery raw change log table.
    */
-  private async insertData(rows: bigquery.RowMetadata[], options: object) {
-    const payload = {
+  private async insertData(rows: bigquery.RowMetadata[]) {
+    const options = {
       skipInvalidRows: false,
       ignoreUnkownValues: false,
-      rows: rows,
+      raw: true,
     };
     try {
       const dataset = this.bq.dataset(this.config.datasetId);
       const table = dataset.table(this.rawChangeLogTableName());
       logs.dataInserting(rows.length);
-      await table.insert(payload, options);
+      await table.insert(rows, options);
       logs.dataInserted(rows.length);
     } catch (e) {
       // Reinitializing in case the destintation table is modified.

firestore-bigquery-export/scripts/import/package.json

@@ -25,7 +25,6 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@firebaseextensions/firestore-bigquery-change-tracker": "^1.0.1",
-    "@google-cloud/bigquery": "^2.1.0",
     "firebase-admin": "^7.1.1",
     "firebase-functions": "^2.2.1",
     "generate-schema": "^2.6.0",

firestore-bigquery-export/scripts/import/src/index.ts

@@ -16,7 +16,6 @@
  * limitations under the License.
  */
 
-import * as bigquery from "@google-cloud/bigquery";
 import * as firebase from "firebase-admin";
 import * as fs from "fs";
 import * as inquirer from "inquirer";