Merge pull request #466 from firebase/next

October 8, 2020 Release

Author: tinaliang

firestore-counter/CHANGELOG.md

@@ -1,3 +1,7 @@
+## Version 0.2.0
+
+feature - convert Firestore Counter to use scheduled functions (#353)
+
 ## Version 0.1.5
 
 feature - Add new Cloud Functions locations. For more information about locations and their pricing tiers, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations).

firestore-counter/POSTINSTALL.md

@@ -1,6 +1,7 @@
 ### Post-installation configuration
 
-Before you can use this extension, you'll need to update your security rules, set up a Cloud Scheduler job, and add some code to your JavaScript app.
+Before you can use this extension, you'll need to update your security rules and add some code to your JavaScript app.
+
 
 #### Update security rules
 
@@ -19,22 +20,6 @@ match /databases/{database}/documents/pages/{page} {
 ```
 
 
-#### Set up a Cloud Scheduler job
-
-**IMPORTANT:** Note the following about v0.1.1 of this extension:
-- **If you updated your extension from v0.1.0 to v0.1.1:**  We recommend that you edit your Cloud Scheduler job to instead send a message to the extension's Pub/Sub topic, as described in this section. Although it's not recommended, if you leave your Cloud Scheduler job calling `${function:controller.url}`, your extension will continue to run as expected. For more information about the changes for v0.1.1, refer to the [changelog](https://github.com/firebase/extensions/blob/master/firestore-counter/CHANGELOG.md).
-- **If you installed this extension for the first time at v0.1.1 or later:** Follow the instructions as described in this section.
-
-Set up a [Cloud Scheduler job](https://cloud.google.com/scheduler/docs/quickstart) to regularly send a message to the extension's Pub/Sub topic (`${param:EXT_INSTANCE_ID}`). This Pub/Sub topic then automatically triggers the controllerCore function (`${function:controllerCore.name}`). This controllerCore function is created by the extension. It works by either aggregating shards itself or scheduling and monitoring workers to aggregate shards.
-
-As an example, to set up the required Cloud Scheduler job, you can run the following `gcloud` commands:
-
-```
-gcloud --project=${param:PROJECT_ID} services enable cloudscheduler.googleapis.com
-gcloud --project=${param:PROJECT_ID} scheduler jobs create pubsub ${param:EXT_INSTANCE_ID} --schedule="* * * * *" --topic=${param:EXT_INSTANCE_ID} --message-body="{}"
-```
-
-
 #### Specify a document path and increment value in your web app
 
 1.  Download and copy the [compiled client sample](https://github.com/firebase/extensions/blob/master/firestore-counter/clients/web/dist/sharded-counter.js) into your application project.
@@ -75,6 +60,14 @@ gcloud --project=${param:PROJECT_ID} scheduler jobs create pubsub ${param:EXT_IN
   </html>
   ```
 
+
+#### Upgrading from v0.1.3 and earlier
+
+If you installed v0.1.3 or an earlier version of this extension, you set up a Cloud Scheduler job that either sent messages to the extension's Pub/Sub topic (`${param:EXT_INSTANCE_ID}`) or called the extension's controller function. Starting in v0.1.4, the controllerCore function (`${function:controllerCore.name}`) has a configurable schedule, so the manually-created Cloud Scheduler job is no longer required and will start to fail.
+
+Delete the old Cloud Scheduler job on the [Cloud Scheduler](https://console.cloud.google.com/cloudscheduler?project=_) page of the Cloud console.
+
+
 ### Using the extension
 
 After you complete the post-installation configuration above, the process runs as follows:

firestore-counter/PREINSTALL.md

@@ -18,7 +18,6 @@ Before installing this extension, make sure that you've [set up a Cloud Firestor
 After installing this extension, you'll need to:
 
 - Update your [database security rules](https://firebase.google.com/docs/rules).
-- Set up a [Cloud Scheduler job](https://cloud.google.com/scheduler/docs/quickstart) to regularly call the controllerCore function, which is created by this extension. It works by either aggregating shards itself or scheduling and monitoring workers to aggregate shards.
 - Use the provided [client sample](https://github.com/firebase/extensions/blob/master/firestore-counter/clients/web/src/index.ts) or your own client code to specify your document path and increment values.
 
 Detailed information for these post-installation tasks are provided after you install this extension.

firestore-counter/extension.yaml

@@ -13,12 +13,11 @@
 # limitations under the License.
 
 name: firestore-counter
-version: 0.1.5
+version: 0.2.0
 specVersion: v1beta
 
 displayName: Distributed Counter
-description:
-  Records event counters at scale to accommodate high-velocity writes to Cloud Firestore.
+description: Records event counters at scale to accommodate high-velocity writes to Cloud Firestore.
 
 license: Apache-2.0
 
@@ -36,48 +35,29 @@ contributors:
   - authorName: Invertase
     email: oss@invertase.io
     url: https://github.com/invertase
-    
 
 billingRequired: true
 
 roles:
   - role: datastore.user
-    reason:
-      Allows the extension to aggregate Cloud Firestore counter shards.
-  - role: pubsub.publisher
-    reason:
-      Allows the HTTPS controller function to publish a message to the extension's Pub/Sub topic,
-      which triggers the controllerCore function.
+    reason: Allows the extension to aggregate Cloud Firestore counter shards.
+  - role: cloudscheduler.admin
+    reason: Allows the extension to create a new Cloud Scheduler function.
 
 resources:
   - name: controllerCore
     type: firebaseextensions.v1beta.function
-    description:
-      Scheduled to run every minute.
-      This function either aggregates shards itself, or it schedules and monitors workers to aggregate shards.
+    description: This scheduled function either aggregates shards itself, or it schedules and monitors workers to aggregate shards.
     properties:
       location: ${LOCATION}
       runtime: nodejs10
       maxInstances: 1
-      eventTrigger:
-        eventType: google.pubsub.topic.publish
-        resource: projects/${PROJECT_ID}/topics/${EXT_INSTANCE_ID}
-
-  - name: controller
-    type: firebaseextensions.v1beta.function
-    description:
-      Maintained for backwards compatibility.
-      This function relays a message to the extension's Pub/Sub topic to trigger the controllerCore function.
-    properties:
-      location: ${LOCATION}
-      runtime: nodejs10
-      maxInstances: 1
-      httpsTrigger: {}
+      scheduleTrigger:
+        schedule: "every ${SCHEDULE_FREQUENCY} minutes"
 
   - name: onWrite
     type: firebaseextensions.v1beta.function
-    description:
-      Listens for changes on counter shards that may need aggregating. This function is limited to max 1 instance.
+    description: Listens for changes on counter shards that may need aggregating. This function is limited to max 1 instance.
     properties:
       location: ${LOCATION}
       runtime: nodejs10
@@ -89,8 +69,7 @@ resources:
 
   - name: worker
     type: firebaseextensions.v1beta.function
-    description:
-      Monitors a range of shards and aggregates them, as needed.
+    description: Monitors a range of shards and aggregates them, as needed.
       There may be 0 or more worker functions running at any point in time.
       The controllerCore function is responsible for scheduling and monitoring these workers.
     properties:
@@ -163,3 +142,12 @@ params:
       for example, `my_collection/doc` or `my_collection/doc/subcollection/doc`, but not `my_collection`.
     default: _firebase_ext_/sharded_counter
     required: true
+
+  - param: SCHEDULE_FREQUENCY
+    label: Frequency for controllerCore function to be run
+    description: >-
+      In minutes, how often should the function to aggregate shards be run?
+    validationRegex: "^[1-9][0-9]*$"
+    validationErrorMessage: The number of minutes must be an integer value greater than zero.
+    default: "1"
+    required: true

firestore-counter/functions/README.md

@@ -26,7 +26,6 @@ Before installing this extension, make sure that you've [set up a Cloud Firestor
 After installing this extension, you'll need to:
 
 - Update your [database security rules](https://firebase.google.com/docs/rules).
-- Set up a [Cloud Scheduler job](https://cloud.google.com/scheduler/docs/quickstart) to regularly call the controllerCore function, which is created by this extension. It works by either aggregating shards itself or scheduling and monitoring workers to aggregate shards.
 - Use the provided [client sample](https://github.com/firebase/extensions/blob/master/firestore-counter/clients/web/src/index.ts) or your own client code to specify your document path and increment values.
 
 Detailed information for these post-installation tasks are provided after you install this extension.
@@ -49,13 +48,13 @@ To install an extension, your project must be on the [Blaze (pay as you go) plan
 
 * Document path for internal state: What is the path to the document where the extension can keep its internal state?
 
+* Frequency for controllerCore function to be run: In minutes, how often should the function to aggregate shards be run?
 
 
-**Cloud Functions:**
 
-* **controllerCore:** Scheduled to run every minute. This function either aggregates shards itself, or it schedules and monitors workers to aggregate shards.
+**Cloud Functions:**
 
-* **controller:** Maintained for backwards compatibility. This function relays a message to the extension's Pub/Sub topic to trigger the controllerCore function.
+* **controllerCore:** This scheduled function either aggregates shards itself, or it schedules and monitors workers to aggregate shards.
 
 * **onWrite:** Listens for changes on counter shards that may need aggregating. This function is limited to max 1 instance.
 
@@ -71,4 +70,4 @@ This extension will operate with the following project IAM roles:
 
 * datastore.user (Reason: Allows the extension to aggregate Cloud Firestore counter shards.)
 
-* pubsub.publisher (Reason: Allows the HTTPS controller function to publish a message to the extension's Pub/Sub topic, which triggers the controllerCore function.)
+* cloudscheduler.admin (Reason: Allows the extension to create a new Cloud Scheduler function.)

firestore-counter/functions/lib/index.js

@@ -15,24 +15,21 @@
  * limitations under the License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.onWrite = exports.worker = exports.controller = exports.controllerCore = void 0;
+exports.onWrite = exports.worker = exports.controllerCore = void 0;
 const functions = require("firebase-functions");
 const admin = require("firebase-admin");
-const pubsub_1 = require("@google-cloud/pubsub");
 const worker_1 = require("./worker");
 const controller_1 = require("./controller");
 admin.initializeApp();
 const firestore = admin.firestore();
 firestore.settings({ timestampsInSnapshots: true });
-let pubsub;
 const SHARDS_COLLECTION_ID = "_counter_shards_";
-const WORKERS_COLLECTION_ID = "_counter_workers_";
 /**
- * The controllerCore is scheduled every minute. It tries to aggregate shards if
+ * The controllerCore is scheduled to run automatically. It tries to aggregate shards if
  * there's less than 200 of them. Otherwise it is scheduling and monitoring
  * workers to do the aggregation.
  */
-exports.controllerCore = functions.handler.pubsub.topic.onPublish(async () => {
+exports.controllerCore = functions.handler.pubsub.schedule.onRun(async () => {
     const metadocRef = firestore.doc(process.env.INTERNAL_STATE_PATH);
     const controller = new controller_1.ShardedCounterController(metadocRef, SHARDS_COLLECTION_ID);
     let status = await controller.aggregateOnce({ start: "", end: "" }, 200);
@@ -43,18 +40,6 @@ exports.controllerCore = functions.handler.pubsub.topic.onPublish(async () => {
     }
     return null;
 });
-/**
- * Backwards compatible HTTPS function
- */
-exports.controller = functions.handler.https.onRequest(async (req, res) => {
-    if (!pubsub) {
-        pubsub = new pubsub_1.PubSub();
-    }
-    await pubsub
-        .topic(process.env.EXT_INSTANCE_ID)
-        .publish(Buffer.from(JSON.stringify({})));
-    res.status(200).send("Ok");
-});
 /**
  * Worker is responsible for aggregation of a defined range of shards. It is controlled
  * by a worker metadata document. At the end of its run (that lasts for 45s) it writes

firestore-counter/functions/package.json

@@ -5,7 +5,6 @@
   "license": "Apache-2.0",
   "description": "Auto-scalable counters for your app.",
   "dependencies": {
-    "@google-cloud/pubsub": "^1.1.5",
     "deep-equal": "^1.0.1",
     "firebase-admin": "^8.3.0",
     "firebase-functions": "^3.9.0",

firestore-counter/functions/src/index.ts

@@ -16,25 +16,21 @@
 
 import * as functions from "firebase-functions";
 import * as admin from "firebase-admin";
-import { PubSub } from "@google-cloud/pubsub";
 import { ShardedCounterWorker } from "./worker";
 import { ShardedCounterController, ControllerStatus } from "./controller";
 
 admin.initializeApp();
 const firestore = admin.firestore();
 firestore.settings({ timestampsInSnapshots: true });
 
-let pubsub;
-
 const SHARDS_COLLECTION_ID = "_counter_shards_";
-const WORKERS_COLLECTION_ID = "_counter_workers_";
 
 /**
- * The controllerCore is scheduled every minute. It tries to aggregate shards if
+ * The controllerCore is scheduled to run automatically. It tries to aggregate shards if
  * there's less than 200 of them. Otherwise it is scheduling and monitoring
  * workers to do the aggregation.
  */
-export const controllerCore = functions.handler.pubsub.topic.onPublish(
+export const controllerCore = functions.handler.pubsub.schedule.onRun(
   async () => {
     const metadocRef = firestore.doc(process.env.INTERNAL_STATE_PATH);
     const controller = new ShardedCounterController(
@@ -53,21 +49,6 @@ export const controllerCore = functions.handler.pubsub.topic.onPublish(
   }
 );
 
-/**
- * Backwards compatible HTTPS function
- */
-export const controller = functions.handler.https.onRequest(
-  async (req, res) => {
-    if (!pubsub) {
-      pubsub = new PubSub();
-    }
-    await pubsub
-      .topic(process.env.EXT_INSTANCE_ID)
-      .publish(Buffer.from(JSON.stringify({})));
-    res.status(200).send("Ok");
-  }
-);
-
 /**
  * Worker is responsible for aggregation of a defined range of shards. It is controlled
  * by a worker metadata document. At the end of its run (that lasts for 45s) it writes