Merge branch 'next' into rachelsaunders-sectionsH4s

Author: rachelsaunders

delete-user-data/functions/lib/index.js

@@ -109,9 +109,6 @@ const clearStorageData = (storagePaths, uid) => __awaiter(this, void 0, void 0,
     logs.storageDeleted();
 });
 const clearFirestoreData = (firestorePaths, uid) => __awaiter(this, void 0, void 0, function* () {
-    // admin.firestore().settings({
-    //   timestampsInSnapshots: true,
-    // });
     logs.firestoreDeleting();
     const paths = extractUserPaths(firestorePaths, uid);
     const promises = paths.map((path) => __awaiter(this, void 0, void 0, function* () {

delete-user-data/functions/src/index.ts

@@ -112,9 +112,6 @@ const clearStorageData = async (storagePaths: string, uid: string) => {
 };
 
 const clearFirestoreData = async (firestorePaths: string, uid: string) => {
-  // admin.firestore().settings({
-  //   timestampsInSnapshots: true,
-  // });
   logs.firestoreDeleting();
 
   const paths = extractUserPaths(firestorePaths, uid);

firestore-bigquery-export/POSTINSTALL.md

@@ -13,13 +13,13 @@ You can test out this extension right away:
 1.  Query your **raw changelog table**, which should contain a single log of creating the `bigquery-mirror-test` document.
 
     ```  
-    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:COLLECTION_PATH}_raw_changelog`   
+    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:TABLE_ID}_raw_changelog`   
     ```
 
 1.  Query your **latest view**, which should return the latest change event for the only document present -- `bigquery-mirror-test`.
 
     ```  
-    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:COLLECTION_PATH}_raw_latest`
+    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:TABLE_ID}_raw_latest`
     ```
 
 1.  Delete the `bigquery-mirror-test` document from [Cloud Firestore](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data).  
@@ -28,7 +28,7 @@ The `bigquery-mirror-test` document will disappear from the **latest view** and
 1.  You can check the changelogs of a single document with this query:
 
     ```  
-    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:COLLECTION_PATH}_raw_changelog` 
+    SELECT * FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:TABLE_ID}_raw_changelog` 
        WHERE document_name = "bigquery-mirror-test" 
        ORDER BY TIMESTAMP ASC  
     ```
@@ -37,12 +37,12 @@ The `bigquery-mirror-test` document will disappear from the **latest view** and
 
 Whenever a document is created, updated, imported, or deleted in the specified collection, this extension sends that update to BigQuery. You can then run queries on this mirrored dataset which contains the following resources:
 
-+   **raw changelog table:** [`${param:DATASET_ID}_raw_changelog`](https://console.cloud.google.com/bigquery?project=${param:PROJECT_ID}&p=${param:PROJECT_ID}&d=${param:DATASET_ID}&t=${param:COLLECTION_PATH}_raw_changelog&page=table)
-+   **latest view:** [`${param:DATASET_ID}_raw_latest`](https://console.cloud.google.com/bigquery?project=${param:PROJECT_ID}&p=${param:PROJECT_ID}&d=${param:DATASET_ID}&t=${param:COLLECTION_PATH}_raw_latest&page=table)
++   **raw changelog table:** [`${param:TABLE_ID}_raw_changelog`](https://console.cloud.google.com/bigquery?project=${param:PROJECT_ID}&p=${param:PROJECT_ID}&d=${param:DATASET_ID}&t=${param:TABLE_ID}_raw_changelog&page=table)
++   **latest view:** [`${param:TABLE_ID}_raw_latest`](https://console.cloud.google.com/bigquery?project=${param:PROJECT_ID}&p=${param:PROJECT_ID}&d=${param:DATASET_ID}&t=${param:TABLE_ID}_raw_latest&page=table)
 
 To review the schema for these two resources, click the **Schema** tab for each resource in BigQuery.
 
-Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database.
+Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database. Or if you have the same subcollection across documents in a given collection, you can use `{wildcard}` notation to listen to all those subcollections (for example: `chats/{chatid}/posts`).
 
 ### _(Optional)_ Import existing documents
 

firestore-bigquery-export/PREINSTALL.md

@@ -6,7 +6,7 @@ The extension creates and updates a [dataset](https://cloud.google.com/bigquery/
 
 Whenever a document is created, updated, deleted, or imported in the specified collection, this extension sends that update to BigQuery. You can then run queries on this mirrored dataset.
 
-Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database.
+Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database. Or if you have the same subcollection across documents in a given collection, you can use `{wildcard}` notation to listen to all those subcollections (for example: `chats/{chatid}/posts`).
 
 #### Additional setup
 

firestore-bigquery-export/extension.yaml

@@ -85,9 +85,11 @@ params:
     type: string
     label: Collection path
     description: >-
-      What is the path of the the collection that you would like to export?
-    default: extensions
-    example: extensions
+      What is the path of the collection that you would like to export? 
+      You may use `{wildcard}` notation to match a subcollection of all documents in a collection
+      (for example: `chatrooms/{chatid}/posts`).
+    default: posts
+    example: posts
     validationRegex: "^[^/]+(/[^/]+/[^/]+)*$"
     validationErrorMessage: Must be a valid Cloud Firestore collection
     required: true
@@ -98,10 +100,25 @@ params:
     description: >-
       What ID would you like to use for your BigQuery dataset? This
       extension will create the dataset, if it doesn't already exist.
-    validationRegex: ^[a-zA-Z0-9_]+$
+    validationRegex: "^[a-zA-Z0-9_]+$"
     validationErrorMessage: >
-      Dataset IDs must be alphanumeric (plus underscores) and must be no more than
+      BigQuery dataset IDs must be alphanumeric (plus underscores) and must be no more than
       1024 characters.
-    default: exported_collection
-    example: exported_collection
+    default: firestore_export
+    example: firestore_export
+    required: true
+
+  - param: TABLE_ID
+    type: string
+    label: Table ID
+    description: >-
+      What identifying prefix would you like to use for your table and view
+      inside your BigQuery dataset? This extension will create the table and view,
+      if they don't already exist.
+    validationRegex: "^[a-zA-Z0-9_]+$"
+    validationErrorMessage: >
+      BigQuery table IDs must be alphanumeric (plus underscores) and must be no more than
+      1024 characters.
+    default: posts
+    example: posts
     required: true

firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts

@@ -25,136 +25,102 @@ import * as logs from "../logs";
 import { BigQuery } from "@google-cloud/bigquery";
 
 export interface FirestoreBigQueryEventHistoryTrackerConfig {
-  collectionPath: string;
   datasetId: string;
-  initialized: boolean;
-  suppressWarnings: boolean;
+  tableId: string;
 }
 
 /**
- * An interface to BigQuery which handles:
- * - Iniitializing the raw changelog table when the first event gets recorded.
- * - Initializing the latest view over the raw changelog.
- * - Streaming writes into the raw changelog table.
+ * An FirestoreEventHistoryTracker that exports data to BigQuery.
+ *
+ * When the first event is received, it creates necessary BigQuery resources:
+ * - Dataset: {@link FirestoreBigQueryEventHistoryTrackerConfig#datasetId}.
+ * - Table: Raw change log table {@link FirestoreBigQueryEventHistoryTracker#rawChangeLogTableName}.
+ * - View: Latest view {@link FirestoreBigQueryEventHistoryTracker#rawLatestView}.
+ * If any subsequent data export fails, it will attempt to reinitialize.
  */
 export class FirestoreBigQueryEventHistoryTracker implements FirestoreEventHistoryTracker {
   bq: bigquery.BigQuery;
-  tableName: string;
-  initialized: boolean;
-  suppressWarnings: boolean;
+  initialized: boolean = false;
 
   constructor(public config: FirestoreBigQueryEventHistoryTrackerConfig) {
     this.bq = new bigquery.BigQuery();
-    this.initialized = config.initialized;
-    this.tableName = config.collectionPath.replace(/\//g, "_");
-    this.suppressWarnings = config.suppressWarnings;
   }
 
   async record(events: FirestoreDocumentChangeEvent[]) {
-    if (!this.config.initialized) {
-      try {
-        await this.initialize(this.config.datasetId, this.tableName);
-        this.initialized = true;
-      } catch (e) {
-        logs.bigQueryErrorRecordingDocumentChange(e);
-      }
-    }
+    await this.initialize();
+
     const rows = events.map(event => {
-      return this.buildDataRow(
-        // Use the function's event ID to protect against duplicate executions
-        event.eventId,
-        event.operation,
-        event.timestamp,
-        event.documentName,
-        event.data);
+      // This must match firestoreToBQTable().
+      return {
+        timestamp: event.timestamp,
+        event_id: event.eventId,
+        document_name: event.documentName,
+        operation: ChangeType[event.operation],
+        data: JSON.stringify(event.data),
+      };
     });
-    await this.insertData(this.config.datasetId, this.tableName, rows);
+    await this.insertData(rows);
   }
 
   /**
-   * Ensure that the defined Firestore schema exists within BigQuery and
-   * contains the correct information. This is invoked for the first time when
-   * the first document change event is recorded.
+   * Inserts rows of data into the BigQuery raw change log table.
    */
-  async initialize(datasetId: string, tableName: string) {
-    const rawTable = raw(tableName);
-
-    await this.initializeDataset(datasetId);
-    await this.initializeChangelog(datasetId, rawTable);
-    await this.initializeLatestView(datasetId, rawTable);
-  };
-
-  buildDataRow(
-    eventId: string,
-    changeType: ChangeType,
-    timestamp: string,
-    document_name: string,
-    data?: Object
-  ): bigquery.RowMetadata {
-    // This must match firestoreToBQTable().
-    return {
-      timestamp: timestamp,
-      event_id: eventId,
-      document_name: document_name,
-      operation: ChangeType[changeType],
-      data: JSON.stringify(data),
-    };
-  };
+  private async insertData(rows: bigquery.RowMetadata[]) {
+    try {
+      const dataset = this.bq.dataset(this.config.datasetId);
+      const table = dataset.table(this.rawChangeLogTableName());
+      logs.dataInserting(rows.length);
+      await table.insert(rows);
+      logs.dataInserted(rows.length);
+    } catch (e) {
+      // Reinitializing in case the destintation table is modified.
+      this.initialized = false;
+      throw e;
+    }
+  }
 
   /**
-   * Insert a row of data into the BigQuery `raw` data table
+   * Creates the BigQuery resources with the expected schema for {@link FirestoreEventHistoryTracker}.
+   * After the first invokation, it skips initialization assuming these resources are still there.
    */
-  async insertData(
-    datasetId: string,
-    collectionTableName: string,
-    rows: bigquery.RowMetadata[]
-  ) {
-    const name = changeLog(raw(collectionTableName));
-    const dataset = this.bq.dataset(datasetId);
-    const table = dataset.table(name);
-    const rowCount = rows.length;
-
-    logs.dataInserting(rowCount);
-    await table.insert(rows);
-    logs.dataInserted(rowCount);
-  };
+  private async initialize() {
+    if (this.initialized) {
+      return;
+    }
+    await this.initializeDataset();
+    await this.initializeRawChangeLogTable();
+    await this.initializeLatestView();
+    this.initialized = true;
+  }
 
   /**
-   * Check that the specified dataset exists, and create it if it doesn't.
+   * Creates the specified dataset if it doesn't already exists.
    */
-  async initializeDataset(datasetId: string): Promise<bigquery.Dataset> {
-    const dataset = this.bq.dataset(datasetId);
+  private async initializeDataset() {
+    const dataset = this.bq.dataset(this.config.datasetId);
     const [datasetExists] = await dataset.exists();
     if (datasetExists) {
-      if (!this.suppressWarnings) {
-        logs.bigQueryDatasetExists(datasetId);
-      }
+      logs.bigQueryDatasetExists(this.config.datasetId);
     } else {
-      logs.bigQueryDatasetCreating(datasetId);
+      logs.bigQueryDatasetCreating(this.config.datasetId);
       await dataset.create();
-      logs.bigQueryDatasetCreated(datasetId);
+      logs.bigQueryDatasetCreated(this.config.datasetId);
     }
     return dataset;
-  };
+  }
 
   /**
-   * Check that the table exists within the specified dataset, and create it
-   * if it doesn't.  If the table does exist, validate that the BigQuery schema
-   * is correct and add any missing fields.
+   * Creates the raw change log table if it doesn't already exist.
+   * TODO: Validate that the BigQuery schema is correct if the table does exist,
    */
-  async initializeChangelog(
-    datasetId: string,
-    tableName: string,
-  ): Promise<bigquery.Table> {
-    const changelogName = changeLog(tableName);
-    const dataset = this.bq.dataset(datasetId);
-    let table = dataset.table(changelogName);
+  private async initializeRawChangeLogTable() {
+    const changelogName = this.rawChangeLogTableName();
+    const dataset = this.bq.dataset(this.config.datasetId);
+    const table = dataset.table(changelogName);
     const [tableExists] = await table.exists();
 
     if (tableExists) {
-      if (!this.suppressWarnings) {
-        logs.bigQueryTableAlreadyExists(table.id, dataset.id);
-      }
+      logs.bigQueryTableAlreadyExists(table.id, dataset.id);
     } else {
       logs.bigQueryTableCreating(changelogName);
       const options = {
@@ -169,37 +135,35 @@ export class FirestoreBigQueryEventHistoryTracker implements FirestoreEventHisto
   };
 
   /**
-   * Create a view over a table storing a change log of Firestore documents
-   * which contains only latest version of all live documents in the mirrored
-   * collection.
+   * Creates the latest snapshot view, which returns only latest operations
+   * of all existing documents over the raw change log table.
    */
-  async initializeLatestView(
-    datasetId: string,
-    tableName: string
-  ): Promise<bigquery.Table> {
-    let viewName = latest(tableName);
-    const dataset = this.bq.dataset(datasetId);
-    let view = dataset.table(viewName);
+  private async initializeLatestView() {
+    const dataset = this.bq.dataset(this.config.datasetId);
+    const view = dataset.table(this.rawLatestView());
     const [viewExists] = await view.exists();
 
     if (viewExists) {
-      if (!this.suppressWarnings) {
-        logs.bigQueryViewAlreadyExists(view.id, dataset.id);
-      }
+      logs.bigQueryViewAlreadyExists(view.id, dataset.id);
     } else {
-      const latestSnapshot = latestConsistentSnapshotView(datasetId, changeLog(tableName));
-      logs.bigQueryViewCreating(viewName, latestSnapshot.query);
+      const latestSnapshot = latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName());
+      logs.bigQueryViewCreating(this.rawLatestView(), latestSnapshot.query);
       const options = {
-        friendlyName: viewName,
+        friendlyName: this.rawLatestView(),
         view: latestSnapshot,
       };
       await view.create(options);
-      logs.bigQueryViewCreated(viewName);
+      logs.bigQueryViewCreated(this.rawLatestView());
     }
     return view;
-  };
+  }
+
+  private rawChangeLogTableName(): string {
+    return `${this.config.tableId}_raw_changelog`;
+  }
+
+  private rawLatestView(): string {
+    return `${this.config.tableId}_raw_latest`;
+  }
 }
 
-export function raw(tableName: string): string { return `${tableName}_raw`; };
-export function changeLog(tableName: string): string { return `${tableName}_changelog`; }
-export function latest(tableName: string): string { return `${tableName}_latest`; };

firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts

@@ -17,7 +17,6 @@
 import * as bigquery from "@google-cloud/bigquery";
 import * as errors from "../errors";
 import * as logs from "../logs";
-import { changeLog, raw, latest } from "../bigquery";
 import * as sqlFormatter from "sql-formatter";
 
 export type BigQueryFieldMode = "NULLABLE" | "REPEATED" | "REQUIRED";

firestore-bigquery-export/functions/lib/config.js

@@ -18,6 +18,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = {
     collectionPath: process.env.COLLECTION_PATH,
     datasetId: process.env.DATASET_ID,
+    tableId: process.env.TABLE_ID,
     location: process.env.LOCATION,
     initialized: false
 };

firestore-bigquery-export/functions/lib/index.js

@@ -29,7 +29,10 @@ const functions = require("firebase-functions");
 const firestore_bigquery_change_tracker_1 = require("@firebaseextensions/firestore-bigquery-change-tracker");
 const logs = require("./logs");
 const util_1 = require("./util");
-const eventTracker = new firestore_bigquery_change_tracker_1.FirestoreBigQueryEventHistoryTracker(config_1.default);
+const eventTracker = new firestore_bigquery_change_tracker_1.FirestoreBigQueryEventHistoryTracker({
+    tableId: config_1.default.tableId,
+    datasetId: config_1.default.datasetId,
+});
 logs.init();
 exports.fsexportbigquery = functions.handler.firestore.document.onWrite((change, context) => __awaiter(void 0, void 0, void 0, function* () {
     logs.start();