changed(firestore-counter): use pubsub trigger for controller function (#49)

Author: Salakar

firestore-counter/POSTINSTALL.md

@@ -1,6 +1,6 @@
 ### Post-installation configuration
 
-Before you can use this extension, you'll need to update your security rules, set up a scheduled function, and add some code to your JavaScript app.
+Before you can use this extension, you'll need to update your security rules, set up a Cloud Scheduler job, and add some code to your JavaScript app.
 
 #### Update security rules
 
@@ -18,17 +18,23 @@ match /databases/{database}/documents/pages/{page} {
 }
 ```
 
-#### Set up a scheduled function
 
-Review the [scheduled function documentation](https://firebase.google.com/docs/functions/schedule-functions) to set up a call to `${function:controller.url}` every minute. You may need to enable some APIs in your Firebase project to use scheduled functions.
+#### Set up a Cloud Scheduler job
 
-As an example, to set up a scheduled function, you can run the following [`gcloud`](https://cloud.google.com/sdk/gcloud/) commands:
+**IMPORTANT:** Note the following about v0.1.1 of this extension:
+- **If you updated your extension from v0.1.0 to v0.1.1:**  We recommend that you edit your Cloud Scheduler job to instead send a message to the extension's Pub/Sub topic, as described in this section. Although it's not recommended, if you leave your Cloud Scheduler job calling `${function:controller.url}`, your extension will continue to run as expected. For more information about the changes for v0.1.1, refer to the [changelog](https://github.com/firebase/extensions/blob/master/firestore-counter/CHANGELOG.md).
+- **If you installed this extension for the first time at v0.1.1:** Follow the instructions as described in this section.
+
+Set up a [Cloud Scheduler job](https://cloud.google.com/scheduler/docs/quickstart) to regularly send a message to the extension's Pub/Sub topic (`${param:EXT_INSTANCE_ID}`). This Pub/Sub topic then automatically triggers the controllerCore function (`${function:controllerCore.name}`). This controllerCore function is created by the extension. It works by either aggregating shards itself or scheduling and monitoring workers to aggregate shards.
+
+As an example, to set up the required Cloud Scheduler job, you can run the following `gcloud` commands:
 
 ```
-gcloud services enable cloudscheduler.googleapis.com
-gcloud scheduler jobs create http firestore-sharded-counter-controller --schedule="* * * * *" --uri=${function:controller.url} --project=${param:PROJECT_ID}
+gcloud --project=${param:PROJECT_ID} services enable cloudscheduler.googleapis.com
+gcloud --project=${param:PROJECT_ID} scheduler jobs create pubsub ${param:EXT_INSTANCE_ID} --schedule="* * * * *" --topic=${param:EXT_INSTANCE_ID} --message-body="{}"
 ```
 
+
 #### Specify a document path and increment value in your app
 
 1.  Download and copy the [Counter SDK](https://github.com/firebase/extensions/blob/master/firestore-counter/clients/web/dist/sharded-counter.js) into your application project.
@@ -77,7 +83,7 @@ After you complete the post-installation configuration above, the process runs a
 
 1. The client SDK writes to these subcollections to distribute the write load.
 
-1. The scheduled function that you deployed sums the subcollections' values into the single `visits` field (or whichever field you configured in your master document).
+1. The controllerCore function sums the subcollections' values into the single `visits` field (or whichever field you configured in your master document).
 
 1. After each summation, the extension deletes the subcollections, leaving only the count in the master document. This is the document field to which you should listen for the count.
 

firestore-counter/PREINSTALL.md

@@ -18,7 +18,7 @@ Before installing this extension, make sure that you've [set up a Cloud Firestor
 After installing this extension, you'll need to:
 
 - Update your [database security rules](https://firebase.google.com/docs/rules).
-- Set up a [scheduled function](https://firebase.google.com/docs/functions/schedule-functions) to regularly call the controller function, which is created by this extension and monitors the extension's workload.
+- Set up a [Cloud Scheduler job](https://cloud.google.com/scheduler/docs/quickstart) to regularly call the controllerCore function, which is created by this extension. It works by either aggregating shards itself or scheduling and monitoring workers to aggregate shards.
 - Install the provided [Counter SDK](https://github.com/firebase/extensions/blob/master/firestore-counter/clients/web/src/index.ts) in your app. You can then use this library in your code to specify your document path and increment values.
 
 Detailed information for these post-installation tasks are provided after you install this extension.

firestore-counter/extension.yaml

@@ -35,14 +35,32 @@ contributors:
 
 roles:
   - role: datastore.user
-    reason: Allows the extension to aggregate Cloud Firestore counter shards.
+    reason:
+      Allows the extension to aggregate Cloud Firestore counter shards.
+  - role: pubsub.publisher
+    reason:
+      Allows the HTTPS controller function to publish a message to the extension's Pub/Sub topic,
+      which triggers the controllerCore function.
 
 resources:
-  - name: controller
+  - name: controllerCore
     type: firebaseextensions.v1beta.function
     description:
       Scheduled to run every minute.
       This function either aggregates shards itself, or it schedules and monitors workers to aggregate shards.
+    properties:
+      sourceDirectory: .
+      location: ${LOCATION}
+      maxInstances: 1
+      eventTrigger:
+        eventType: google.pubsub.topic.publish
+        resource: projects/${PROJECT_ID}/topics/${EXT_INSTANCE_ID}
+
+  - name: controller
+    type: firebaseextensions.v1beta.function
+    description:
+      Maintained for backwards compatibility.
+      This function relays a message to the extension's Pub/Sub topic to trigger the controllerCore function.
     properties:
       sourceDirectory: .
       location: ${LOCATION}
@@ -67,7 +85,7 @@ resources:
     description:
       Monitors a range of shards and aggregates them, as needed.
       There may be 0 or more worker functions running at any point in time.
-      The controller function is responsible for scheduling and monitoring these workers.
+      The controllerCore function is responsible for scheduling and monitoring these workers.
     properties:
       sourceDirectory: .
       location: ${LOCATION}
@@ -108,4 +126,8 @@ params:
       What is the path to the document where the extension can keep its internal state?
     default: _firebase_ext_/sharded_counter
     example: _firebase_ext_/sharded_counter
+    validationRegex: "^[^/]+/[^/]+(/[^/]+/[^/]+)*$"
+    validationErrorMessage:
+      Enter a document path, not a collection path. The path must have an even number of segments,
+      for example, `my_collection/doc` or `my_collection/doc/subcollection/doc`, but not `my_collection`.
     required: true

firestore-counter/functions/lib/aggregator.js

@@ -24,7 +24,7 @@ class NumericUpdate {
     /**
      * Merges numeric values from an arbitrary deep json into the NumericUpdate object.
      *  - it ignores non-numeric leaves
-     *  - if there's a type mismatch ('number' vs 'object') current data will be overriden
+     *  - if there's a type mismatch ('number' vs 'object') current data will be overridden
      * @param from An object with numeric values to merge from.
      */
     mergeFrom(from) {
@@ -33,7 +33,7 @@ class NumericUpdate {
     /**
      * Subtracts numeric values in an arbitrary deep json from the NumericUpdate object.
      *  - it ignores non-numeric leaves
-     *  - if there's a type mismatch ('number' vs 'object') current data will be overriden
+     *  - if there's a type mismatch ('number' vs 'object') current data will be overridden
      * @param from An object with numeric values to merge from.
      */
     subtractFrom(from) {

firestore-counter/functions/lib/common.js

@@ -14,15 +14,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 function isUpdatedFrequently(shard) {
     if (!shard.exists)
@@ -58,9 +49,3 @@ function queryRange(db, collectionId, start, end, limit) {
     return query;
 }
 exports.queryRange = queryRange;
-function delay(ms) {
-    return __awaiter(this, void 0, void 0, function* () {
-        return new Promise((resolve) => setTimeout(resolve, ms));
-    });
-}
-exports.delay = delay;

firestore-counter/functions/lib/controller.js

@@ -191,7 +191,7 @@ class ShardedCounterController {
                     yield t.get(this.controllerDocRef);
                 }
                 catch (err) {
-                    console.log("Failed to read controler doc " + this.controllerDocRef.path);
+                    console.log("Failed to read controller doc " + this.controllerDocRef.path);
                     throw Error("Failed to read controller doc.");
                 }
                 // Read all workers' metadata and construct sharding info based on collected stats.

firestore-counter/functions/lib/index.js

@@ -26,19 +26,21 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 const functions = require("firebase-functions");
 const admin = require("firebase-admin");
+const pubsub_1 = require("@google-cloud/pubsub");
 const worker_1 = require("./worker");
 const controller_1 = require("./controller");
 admin.initializeApp();
 const firestore = admin.firestore();
 firestore.settings({ timestampsInSnapshots: true });
+let pubsub;
 const SHARDS_COLLECTION_ID = "_counter_shards_";
 const WORKERS_COLLECTION_ID = "_counter_workers_";
 /**
- * Controller is scheduled every minute. It tries to aggregate shards if
+ * The controllerCore is scheduled every minute. It tries to aggregate shards if
  * there's less than 200 of them. Otherwise it is scheduling and monitoring
  * workers to do the aggregation.
  */
-exports.controller = functions.https.onRequest((req, res) => __awaiter(void 0, void 0, void 0, function* () {
+exports.controllerCore = functions.handler.pubsub.topic.onPublish(() => __awaiter(void 0, void 0, void 0, function* () {
     const metadocRef = firestore.doc(process.env.INTERNAL_STATE_PATH);
     const controller = new controller_1.ShardedCounterController(metadocRef, SHARDS_COLLECTION_ID);
     let status = yield controller.aggregateOnce({ start: "", end: "" }, 200);
@@ -47,14 +49,26 @@ exports.controller = functions.https.onRequest((req, res) => __awaiter(void 0, v
         status === controller_1.ControllerStatus.FAILURE) {
         yield controller.rescheduleWorkers();
     }
-    res.status(200).send("OK");
+    return null;
+}));
+/**
+ * Backwards compatible HTTPS function
+ */
+exports.controller = functions.https.onRequest((req, res) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!pubsub) {
+        pubsub = new pubsub_1.PubSub();
+    }
+    yield pubsub
+        .topic(process.env.EXT_INSTANCE_ID)
+        .publish(Buffer.from(JSON.stringify({})));
+    res.status(200).send("Ok");
 }));
 /**
  * Worker is responsible for aggregation of a defined range of shards. It is controlled
  * by a worker metadata document. At the end of its run (that lasts for 45s) it writes
  * back stats that kicks off another run at the same time.
  *
- * Controller is monitoring these metadata documents to detect overload that requires
+ * ControllerCore is monitoring these metadata documents to detect overload that requires
  * resharding and to detect failed workers that need poking.
  */
 exports.worker = functions.firestore
@@ -69,7 +83,7 @@ exports.worker = functions.firestore
 /**
  * This is an additional function that is triggered for every shard write. It is
  * limited to one concurrent run at the time. This helps reduce latency for workloads
- * that are below the theshold for workers.
+ * that are below the threshold for workers.
  */
 exports.onWrite = functions.firestore
     .document("/{collection}/{**}/_counter_shards_/{shardId}")

firestore-counter/functions/lib/worker.js

@@ -44,7 +44,7 @@ const WORKER_TIMEOUT_MS = 45000;
  *
  * Workers avoid double scheduling and overruns by including their metadata documents in every
  * aggregation transaction. If metadata changes underneath, transaction fails, worker detects that
- * and terminates immmediately.
+ * and terminates immediately.
  */
 class ShardedCounterWorker {
     constructor(metadoc, shardCollection, singleRun = false) {

firestore-counter/functions/src/aggregator.ts

@@ -15,7 +15,7 @@
  */
 
 import { firestore } from "firebase-admin";
-import { FieldPath, FieldValue } from "@google-cloud/firestore";
+import { FieldValue } from "@google-cloud/firestore";
 import * as uuid from "uuid";
 
 export class NumericUpdate {
@@ -24,7 +24,7 @@ export class NumericUpdate {
   /**
    * Merges numeric values from an arbitrary deep json into the NumericUpdate object.
    *  - it ignores non-numeric leaves
-   *  - if there's a type mismatch ('number' vs 'object') current data will be overriden
+   *  - if there's a type mismatch ('number' vs 'object') current data will be overridden
    * @param from An object with numeric values to merge from.
    */
   public mergeFrom(from: { [key: string]: any }) {
@@ -33,7 +33,7 @@ export class NumericUpdate {
   /**
    * Subtracts numeric values in an arbitrary deep json from the NumericUpdate object.
    *  - it ignores non-numeric leaves
-   *  - if there's a type mismatch ('number' vs 'object') current data will be overriden
+   *  - if there's a type mismatch ('number' vs 'object') current data will be overridden
    * @param from An object with numeric values to merge from.
    */
   public subtractFrom(from: { [key: string]: any }) {

firestore-counter/functions/src/common.ts

@@ -15,9 +15,6 @@
  */
 
 import { firestore } from "firebase-admin";
-import * as path from "path";
-import { FieldPath } from "@google-cloud/firestore";
-import { ShardedCounterController } from "./controller";
 
 /**
  * Represents a document range that a single worker is responsible for.
@@ -81,7 +78,3 @@ export function queryRange(
   query = query.limit(limit);
   return query;
 }
-
-export async function delay(ms: number): Promise<void> {
-  return new Promise<void>((resolve) => setTimeout(resolve, ms));
-}