December 12, 2019 Release

Author: laurenzlong

.github/CODEOWNERS

@@ -1,5 +1,6 @@
 # All documentation changes, e.g. changelogs & readmes.
 *.md             @rachelsaunders
+*/extension.yaml @rachelsaunders
 
 # Dependencies, package.json and mono-repo files
 package.json     @Salakar @Ehesp

.github/ISSUE_TEMPLATE/bug.md

@@ -24,6 +24,7 @@ other category use one of these other channels:
 ### [REQUIRED] Step 2: Describe your configuration
 
 - Extension name: **\_** (`storage-resize-images`, `firestore-send-email`, etc)
+- Extension version: **\_**
 - Configuration values (redact info where appropriate):
   - **\_**
   - **\_**

.prettierignore

@@ -7,6 +7,7 @@ package-lock.json
 # generated files
 README.md
 **/functions/lib/**
+**/lib/**
 **/dist/**
 
 # extension install md files

firestore-bigquery-export/CHANGELOG.md

@@ -0,0 +1,3 @@
+## Version 0.1.1
+
+fixed - Fixed occasional duplicate rows created in the BigQuery backup table (issue #101).

firestore-bigquery-export/extension.yaml

@@ -15,7 +15,7 @@
 name: firestore-bigquery-export
 displayName: Export Collections to BigQuery
 specVersion: v1beta
-version: 0.1.0
+version: 0.1.1
 
 description:
   Sends realtime, incremental updates from a specified Cloud Firestore collection to BigQuery.

firestore-bigquery-export/firestore-bigquery-change-tracker/package.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {

firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts

@@ -51,29 +51,38 @@ export class FirestoreBigQueryEventHistoryTracker
 
   async record(events: FirestoreDocumentChangeEvent[]) {
     await this.initialize();
-
+    const options = {
+      raw: true,
+    };
     const rows = events.map((event) => {
-      // This must match firestoreToBQTable().
       return {
-        timestamp: event.timestamp,
-        event_id: event.eventId,
-        document_name: event.documentName,
-        operation: ChangeType[event.operation],
-        data: JSON.stringify(event.data),
+        insertId: event.eventId,
+        json: {
+          timestamp: event.timestamp,
+          event_id: event.eventId,
+          document_name: event.documentName,
+          operation: ChangeType[event.operation],
+          data: JSON.stringify(event.data),
+        },
       };
     });
-    await this.insertData(rows);
+    await this.insertData(rows, options);
   }
 
   /**
    * Inserts rows of data into the BigQuery raw change log table.
    */
-  private async insertData(rows: bigquery.RowMetadata[]) {
+  private async insertData(rows: bigquery.RowMetadata[], options: object) {
+    const payload = {
+      skipInvalidRows: false,
+      ignoreUnkownValues: false,
+      rows: rows,
+    };
     try {
       const dataset = this.bq.dataset(this.config.datasetId);
       const table = dataset.table(this.rawChangeLogTableName());
       logs.dataInserting(rows.length);
-      await table.insert(rows);
+      await table.insert(payload, options);
       logs.dataInserted(rows.length);
     } catch (e) {
       // Reinitializing in case the destintation table is modified.

firestore-bigquery-export/firestore-bigquery-change-tracker/tsconfig.json

@@ -8,5 +8,6 @@
     "target": "es6"
   },
   "compileOnSave": true,
-  "include": ["src"]
+  "include": ["src"],
+  "exclude": ["**/node_modules/@types/jest/**"]
 }

firestore-bigquery-export/functions/package.json

@@ -14,7 +14,7 @@
   "main": "lib/index.js",
   "dependencies": {
     "firebase-admin": "~7.0.0",
-    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.0.0",
+    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.0.1",
     "firebase-functions": "^2.3.0",
     "sql-formatter": "^2.3.3"
   },

firestore-bigquery-export/guides/IMPORT_EXISTING_DOCUMENTS.md

@@ -59,7 +59,7 @@ Run the import script using [`npx` (the Node Package Runner)](https://www.npmjs.
     ```
     SELECT COUNT(*) FROM
       `${PROJECT_ID}.${COLLECTION_PATH}.${COLLECTION_PATH}_raw_changelog`
-      WHERE operation = "import"
+      WHERE operation = "IMPORT"
     ```
 
     The result set will contain the number of documents in your source collection.

firestore-bigquery-export/scripts/gen-schema-view/tsconfig.json

@@ -9,5 +9,6 @@
     "types": ["node", "mocha", "chai"]
   },
   "compileOnSave": true,
-  "include": ["src"]
+  "include": ["src"],
+  "exclude": ["**/node_modules/@types/jest/**"]
 }

firestore-bigquery-export/scripts/import/package.json

@@ -24,7 +24,7 @@
   "author": "Jan Wyszynski <wyszynski@google.com>",
   "license": "Apache-2.0",
   "dependencies": {
-    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.0.0",
+    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.0.1",
     "@google-cloud/bigquery": "^2.1.0",
     "firebase-admin": "^7.1.1",
     "firebase-functions": "^2.2.1",

firestore-bigquery-export/scripts/import/src/index.ts

@@ -163,7 +163,7 @@ const run = async (): Promise<number> => {
 
   let cursorPositionFile =
     __dirname +
-    `/from-${sourceCollectionPath}-to-${projectId}\:${datasetId}\:${rawChangeLogName}`;
+    `/from-${sourceCollectionPath}-to-${projectId}_${datasetId}_${rawChangeLogName}`;
   if (await exists(cursorPositionFile)) {
     let cursorDocumentId = (await read(cursorPositionFile)).toString();
     cursor = await firebase

firestore-bigquery-export/scripts/import/tsconfig.json

@@ -8,5 +8,6 @@
     "target": "es6"
   },
   "compileOnSave": true,
-  "include": ["src"]
+  "include": ["src"],
+  "exclude": ["**/node_modules/@types/jest/**"]
 }

firestore-counter/CHANGELOG.md

@@ -0,0 +1,5 @@
+## Version 0.1.1
+
+changed - Moves the logic for monitoring the extension's workload from the existing HTTP function to a new Pub/Sub controllerCore function. Now, if called, the HTTP function triggers the new `controllerCore` function instead. This change was made to accommodate a [change in the way Google Cloud Functions handles HTTP functions](https://cloud.google.com/functions/docs/securing/managing-access#allowing_unauthenticated_function_invocation).
+
+We recommend that you edit your existing Cloud Scheduler job to instead send a message to the extension's Pub/Sub topic which triggers the new `controllerCore` function (detailed instructions provided in the [POSTINSTALL file](https://github.com/firebase/extensions/blob/master/firestore-counter/POSTINSTALL.md#set-up-a-cloud-scheduler-job). Although it's not recommended, if you leave your Cloud Scheduler job calling the HTTP function, your extension will continue to run as expected.

firestore-counter/POSTINSTALL.md

@@ -1,6 +1,6 @@
 ### Post-installation configuration
 
-Before you can use this extension, you'll need to update your security rules, set up a scheduled function, and add some code to your JavaScript app.
+Before you can use this extension, you'll need to update your security rules, set up a Cloud Scheduler job, and add some code to your JavaScript app.
 
 #### Update security rules
 
@@ -18,17 +18,23 @@ match /databases/{database}/documents/pages/{page} {
 }
 ```
 
-#### Set up a scheduled function
 
-Review the [scheduled function documentation](https://firebase.google.com/docs/functions/schedule-functions) to set up a call to `${function:controller.url}` every minute. You may need to enable some APIs in your Firebase project to use scheduled functions.
+#### Set up a Cloud Scheduler job
 
-As an example, to set up a scheduled function, you can run the following [`gcloud`](https://cloud.google.com/sdk/gcloud/) commands:
+**IMPORTANT:** Note the following about v0.1.1 of this extension:
+- **If you updated your extension from v0.1.0 to v0.1.1:**  We recommend that you edit your Cloud Scheduler job to instead send a message to the extension's Pub/Sub topic, as described in this section. Although it's not recommended, if you leave your Cloud Scheduler job calling `${function:controller.url}`, your extension will continue to run as expected. For more information about the changes for v0.1.1, refer to the [changelog](https://github.com/firebase/extensions/blob/master/firestore-counter/CHANGELOG.md).
+- **If you installed this extension for the first time at v0.1.1:** Follow the instructions as described in this section.
+
+Set up a [Cloud Scheduler job](https://cloud.google.com/scheduler/docs/quickstart) to regularly send a message to the extension's Pub/Sub topic (`${param:EXT_INSTANCE_ID}`). This Pub/Sub topic then automatically triggers the controllerCore function (`${function:controllerCore.name}`). This controllerCore function is created by the extension. It works by either aggregating shards itself or scheduling and monitoring workers to aggregate shards.
+
+As an example, to set up the required Cloud Scheduler job, you can run the following `gcloud` commands:
 
 ```
-gcloud services enable cloudscheduler.googleapis.com
-gcloud scheduler jobs create http firestore-sharded-counter-controller --schedule="* * * * *" --uri=${function:controller.url} --project=${param:PROJECT_ID}
+gcloud --project=${param:PROJECT_ID} services enable cloudscheduler.googleapis.com
+gcloud --project=${param:PROJECT_ID} scheduler jobs create pubsub ${param:EXT_INSTANCE_ID} --schedule="* * * * *" --topic=${param:EXT_INSTANCE_ID} --message-body="{}"
 ```
 
+
 #### Specify a document path and increment value in your app
 
 1.  Download and copy the [Counter SDK](https://github.com/firebase/extensions/blob/master/firestore-counter/clients/web/dist/sharded-counter.js) into your application project.
@@ -77,7 +83,7 @@ After you complete the post-installation configuration above, the process runs a
 
 1. The client SDK writes to these subcollections to distribute the write load.
 
-1. The scheduled function that you deployed sums the subcollections' values into the single `visits` field (or whichever field you configured in your master document).
+1. The controllerCore function sums the subcollections' values into the single `visits` field (or whichever field you configured in your master document).
 
 1. After each summation, the extension deletes the subcollections, leaving only the count in the master document. This is the document field to which you should listen for the count.
 

firestore-counter/PREINSTALL.md

@@ -18,7 +18,7 @@ Before installing this extension, make sure that you've [set up a Cloud Firestor
 After installing this extension, you'll need to:
 
 - Update your [database security rules](https://firebase.google.com/docs/rules).
-- Set up a [scheduled function](https://firebase.google.com/docs/functions/schedule-functions) to regularly call the controller function, which is created by this extension and monitors the extension's workload.
+- Set up a [Cloud Scheduler job](https://cloud.google.com/scheduler/docs/quickstart) to regularly call the controllerCore function, which is created by this extension. It works by either aggregating shards itself or scheduling and monitoring workers to aggregate shards.
 - Install the provided [Counter SDK](https://github.com/firebase/extensions/blob/master/firestore-counter/clients/web/src/index.ts) in your app. You can then use this library in your code to specify your document path and increment values.
 
 Detailed information for these post-installation tasks are provided after you install this extension.

firestore-counter/extension.yaml

@@ -15,7 +15,7 @@
 name: firestore-counter
 displayName: Distributed Counter
 specVersion: v1beta
-version: 0.1.0
+version: 0.1.1
 
 description:
   Records event counters at scale to accommodate high-velocity writes to Cloud Firestore.
@@ -35,14 +35,32 @@ contributors:
 
 roles:
   - role: datastore.user
-    reason: Allows the extension to aggregate Cloud Firestore counter shards.
+    reason:
+      Allows the extension to aggregate Cloud Firestore counter shards.
+  - role: pubsub.publisher
+    reason:
+      Allows the HTTPS controller function to publish a message to the extension's Pub/Sub topic,
+      which triggers the controllerCore function.
 
 resources:
-  - name: controller
+  - name: controllerCore
     type: firebaseextensions.v1beta.function
     description:
       Scheduled to run every minute.
       This function either aggregates shards itself, or it schedules and monitors workers to aggregate shards.
+    properties:
+      sourceDirectory: .
+      location: ${LOCATION}
+      maxInstances: 1
+      eventTrigger:
+        eventType: google.pubsub.topic.publish
+        resource: projects/${PROJECT_ID}/topics/${EXT_INSTANCE_ID}
+
+  - name: controller
+    type: firebaseextensions.v1beta.function
+    description:
+      Maintained for backwards compatibility.
+      This function relays a message to the extension's Pub/Sub topic to trigger the controllerCore function.
     properties:
       sourceDirectory: .
       location: ${LOCATION}
@@ -67,7 +85,7 @@ resources:
     description:
       Monitors a range of shards and aggregates them, as needed.
       There may be 0 or more worker functions running at any point in time.
-      The controller function is responsible for scheduling and monitoring these workers.
+      The controllerCore function is responsible for scheduling and monitoring these workers.
     properties:
       sourceDirectory: .
       location: ${LOCATION}
@@ -108,4 +126,8 @@ params:
       What is the path to the document where the extension can keep its internal state?
     default: _firebase_ext_/sharded_counter
     example: _firebase_ext_/sharded_counter
+    validationRegex: "^[^/]+/[^/]+(/[^/]+/[^/]+)*$"
+    validationErrorMessage:
+      Enter a document path, not a collection path. The path must have an even number of segments,
+      for example, `my_collection/doc` or `my_collection/doc/subcollection/doc`, but not `my_collection`.
     required: true

firestore-counter/functions/lib/aggregator.js

@@ -24,7 +24,7 @@ class NumericUpdate {
     /**
      * Merges numeric values from an arbitrary deep json into the NumericUpdate object.
      *  - it ignores non-numeric leaves
-     *  - if there's a type mismatch ('number' vs 'object') current data will be overriden
+     *  - if there's a type mismatch ('number' vs 'object') current data will be overridden
      * @param from An object with numeric values to merge from.
      */
     mergeFrom(from) {
@@ -33,7 +33,7 @@ class NumericUpdate {
     /**
      * Subtracts numeric values in an arbitrary deep json from the NumericUpdate object.
      *  - it ignores non-numeric leaves
-     *  - if there's a type mismatch ('number' vs 'object') current data will be overriden
+     *  - if there's a type mismatch ('number' vs 'object') current data will be overridden
      * @param from An object with numeric values to merge from.
      */
     subtractFrom(from) {

firestore-counter/functions/lib/common.js

@@ -14,15 +14,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 function isUpdatedFrequently(shard) {
     if (!shard.exists)
@@ -58,9 +49,3 @@ function queryRange(db, collectionId, start, end, limit) {
     return query;
 }
 exports.queryRange = queryRange;
-function delay(ms) {
-    return __awaiter(this, void 0, void 0, function* () {
-        return new Promise((resolve) => setTimeout(resolve, ms));
-    });
-}
-exports.delay = delay;

firestore-counter/functions/lib/controller.js

@@ -191,7 +191,7 @@ class ShardedCounterController {
                     yield t.get(this.controllerDocRef);
                 }
                 catch (err) {
-                    console.log("Failed to read controler doc " + this.controllerDocRef.path);
+                    console.log("Failed to read controller doc " + this.controllerDocRef.path);
                     throw Error("Failed to read controller doc.");
                 }
                 // Read all workers' metadata and construct sharding info based on collected stats.
@@ -201,7 +201,7 @@ class ShardedCounterController {
                 }
                 catch (err) {
                     console.log("Failed to read worker docs.", err);
-                    throw Error("Failed to reqad worker docs.");
+                    throw Error("Failed to read worker docs.");
                 }
                 let shardingInfo = yield Promise.all(query.docs.map((worker) => __awaiter(this, void 0, void 0, function* () {
                     const slice = worker.get("slice");