Merge pull request #334 from FirebasePrivate/fz/bigquery-collection-group

Add TABLE_ID parameter to BigQuery export extension

Author: fredzqm

firestore-bigquery-export/POSTINSTALL.md

@@ -13,13 +13,13 @@ You can test out this extension right away:
 1.  Query your **raw changelog table**, which should contain a single log of creating the `bigquery-mirror-test` document.
 
     ```  
-    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:COLLECTION_PATH}_raw_changelog`   
+    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:TABLE_ID}_raw_changelog`   
     ```
 
 1.  Query your **latest view**, which should return the latest change event for the only document present -- `bigquery-mirror-test`.
 
     ```  
-    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:COLLECTION_PATH}_raw_latest`
+    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:TABLE_ID}_raw_latest`
     ```
 
 1.  Delete the `bigquery-mirror-test` document from [Cloud Firestore](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data).  
@@ -28,7 +28,7 @@ The `bigquery-mirror-test` document will disappear from the **latest view** and
 1.  You can check the changelogs of a single document with this query:
 
     ```  
-    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:COLLECTION_PATH}_raw_changelog` 
+    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:TABLE_ID}_raw_changelog` 
        WHERE document_name = "bigquery-mirror-test" 
        ORDER BY TIMESTAMP ASC  
     ```
@@ -37,12 +37,12 @@ The `bigquery-mirror-test` document will disappear from the **latest view** and
 
 Whenever a document is created, updated, imported, or deleted in the specified collection, this extension sends that update to BigQuery. You can then run queries on this mirrored dataset which contains the following resources:
 
-+   **raw changelog table:** [`${param:DATASET_ID}_raw_changelog`](https://console.cloud.google.com/bigquery?project=${param:PROJECT_ID}&p=${param:PROJECT_ID}&d=${param:DATASET_ID}&t=${param:COLLECTION_PATH}_raw_changelog&page=table)
-+   **latest view:** [`${param:DATASET_ID}_raw_latest`](https://console.cloud.google.com/bigquery?project=${param:PROJECT_ID}&p=${param:PROJECT_ID}&d=${param:DATASET_ID}&t=${param:COLLECTION_PATH}_raw_latest&page=table)
++   **raw changelog table:** [`${param:TABLE_ID}_raw_changelog`](https://console.cloud.google.com/bigquery?project=${param:PROJECT_ID}&p=${param:PROJECT_ID}&d=${param:DATASET_ID}&t=${param:TABLE_ID}_raw_changelog&page=table)
++   **latest view:** [`${param:TABLE_ID}_raw_latest`](https://console.cloud.google.com/bigquery?project=${param:PROJECT_ID}&p=${param:PROJECT_ID}&d=${param:DATASET_ID}&t=${param:TABLE_ID}_raw_latest&page=table)
 
 To review the schema for these two resources, click the **Schema** tab for each resource in BigQuery.
 
-Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database.
+Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database. Or if you have the same subcollection across documents in a given collection, you can use `{wildcard}` notation to listen to all those subcollections (for example: `chats/{chatid}/posts`).
 
 ### _(Optional)_ Import existing documents
 

firestore-bigquery-export/PREINSTALL.md

@@ -7,7 +7,7 @@ The extension creates and updates a [dataset](https://cloud.google.com/bigquery/
 
 Whenever a document is created, updated, deleted, or imported in the specified collection, this extension sends that update to BigQuery. You can then run queries on this mirrored dataset.
 
-Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database.
+Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database. Or if you have the same subcollection across documents in a given collection, you can use `{wildcard}` notation to listen to all those subcollections (for example: `chats/{chatid}/posts`).
 
 ### Additional setup
 

firestore-bigquery-export/extension.yaml

@@ -85,9 +85,11 @@ params:
     type: string
     label: Collection path
     description: >-
-      What is the path of the the collection that you would like to export?
-    default: extensions
-    example: extensions
+      What is the path of the collection that you would like to export? 
+      You may use `{wildcard}` notation to match a subcollection of all documents in a collection
+      (for example: `chatrooms/{chatid}/posts`).
+    default: posts
+    example: posts
     validationRegex: "^[^/]+(/[^/]+/[^/]+)*$"
     validationErrorMessage: Must be a valid Cloud Firestore collection
     required: true
@@ -98,10 +100,25 @@ params:
     description: >-
       What ID would you like to use for your BigQuery dataset? This
       extension will create the dataset, if it doesn't already exist.
-    validationRegex: ^[a-zA-Z0-9_]+$
+    validationRegex: "^[a-zA-Z0-9_]+$"
     validationErrorMessage: >
-      Dataset IDs must be alphanumeric (plus underscores) and must be no more than
+      BigQuery dataset IDs must be alphanumeric (plus underscores) and must be no more than
       1024 characters.
-    default: exported_collection
-    example: exported_collection
+    default: firestore_export
+    example: firestore_export
+    required: true
+
+  - param: TABLE_ID
+    type: string
+    label: Table ID
+    description: >-
+      What identifying prefix would you like to use for your table and view
+      inside your BigQuery dataset? This extension will create the table and view,
+      if they don't already exist.
+    validationRegex: "^[a-zA-Z0-9_]+$"
+    validationErrorMessage: >
+      BigQuery table IDs must be alphanumeric (plus underscores) and must be no more than
+      1024 characters.
+    default: posts
+    example: posts
     required: true

firestore-bigquery-export/functions/lib/config.js

@@ -18,6 +18,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = {
     collectionPath: process.env.COLLECTION_PATH,
     datasetId: process.env.DATASET_ID,
+    tableId: process.env.TABLE_ID,
     location: process.env.LOCATION,
     initialized: false
 };

firestore-bigquery-export/functions/lib/index.js

@@ -29,7 +29,12 @@ const functions = require("firebase-functions");
 const firestore_bigquery_change_tracker_1 = require("@firebaseextensions/firestore-bigquery-change-tracker");
 const logs = require("./logs");
 const util_1 = require("./util");
-const eventTracker = new firestore_bigquery_change_tracker_1.FirestoreBigQueryEventHistoryTracker(config_1.default);
+const eventTracker = new firestore_bigquery_change_tracker_1.FirestoreBigQueryEventHistoryTracker({
+    collectionPath: config_1.default.tableId,
+    datasetId: config_1.default.datasetId,
+    initialized: false,
+    suppressWarnings: false,
+});
 logs.init();
 exports.fsexportbigquery = functions.handler.firestore.document.onWrite((change, context) => __awaiter(void 0, void 0, void 0, function* () {
     logs.start();

firestore-bigquery-export/functions/src/config.ts

@@ -17,6 +17,7 @@
 export default {
   collectionPath: process.env.COLLECTION_PATH,
   datasetId: process.env.DATASET_ID,
+  tableId: process.env.TABLE_ID,
   location: process.env.LOCATION,
   initialized: false
 };

firestore-bigquery-export/functions/src/index.ts

@@ -24,7 +24,12 @@ import {
 import * as logs from "./logs";
 import { getChangeType } from "./util";
 
-const eventTracker: FirestoreEventHistoryTracker = new FirestoreBigQueryEventHistoryTracker(config);
+const eventTracker: FirestoreEventHistoryTracker = new FirestoreBigQueryEventHistoryTracker({
+  collectionPath: config.tableId, // TODO: rename collectionPath.
+  datasetId: config.datasetId,
+  initialized: false, // TODO: Remove this config.
+  suppressWarnings: false, // TODO: Remove this config.
+});
 
 logs.init();