Merge pull request #485 from firebase/next

Co-authored-by: Russell Wheatley <russellwheatley85@gmail.com>
Co-authored-by: Kevin Cheung <kevinthecheung@users.noreply.github.com>
Co-authored-by: moog16 <moog16@users.noreply.github.com>

Author: 4 people

.gitignore

@@ -1,6 +1,7 @@
 .vscode
 node_modules/
 *.env
+!install-params.env
 firebase-debug.log
 .DS_Store
 mods-test-data/key.json

delete-user-data/README.md

@@ -1,5 +1,7 @@
 # Delete User Data
 
+**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**)
+
 **Description**: Deletes data keyed on a userId from Cloud Firestore, Realtime Database, and/or Cloud Storage when a user deletes their account.
 
 

firestore-bigquery-export/extension.yaml

@@ -52,7 +52,6 @@ resources:
       Listens for document changes in your specified Cloud Firestore collection,
       then exports the changes into BigQuery.
     properties:
-      sourceDirectory: .
       location: ${param:LOCATION}
       runtime: nodejs10
       eventTrigger:

firestore-bigquery-export/firestore-bigquery-change-tracker/package.json

@@ -24,7 +24,7 @@
   "dependencies": {
     "@google-cloud/bigquery": "^4.7.0",
     "firebase-admin": "^7.1.1",
-    "firebase-functions": "^2.2.1",
+    "firebase-functions": "^3.9.0",
     "generate-schema": "^2.6.0",
     "inquirer": "^6.4.0",
     "lodash": "^4.17.14",
@@ -33,7 +33,7 @@
   },
   "devDependencies": {
     "@types/traverse": "^0.6.32",
-    "typescript": "^3.4.5",
+    "typescript": "^3.6.3",
     "rimraf": "^2.6.3",
     "nyc": "^14.0.0",
     "jest": "^24.9.0",

firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts

@@ -141,12 +141,6 @@ export const dataInsertRetried = (rowCount: number) => {
   );
 };
 
-export const dataInsertRetried = (rowCount: number) => {
-  console.log(
-    `Retried to insert ${rowCount} row(s) of data into BigQuery (ignoring uknown columns)`
-  );
-};
-
 export const dataInserting = (rowCount: number) => {
   logger.log(`Inserting ${rowCount} row(s) of data into BigQuery`);
 };

firestore-bigquery-export/firestore-bigquery-change-tracker/tsconfig.json

@@ -1,13 +1,9 @@
 {
+  "extends": "../../tsconfig.json",
   "compilerOptions": {
-    "lib": ["es6"],
-    "module": "commonjs",
-    "noImplicitReturns": true,
     "outDir": "lib",
-    "sourceMap": false,
-    "target": "es6",
-    "types": ["node", "jest", "chai"]
+    "types": ["node", "jest", "chai"],
+    "target": "es2018"
   },
-  "compileOnSave": true,
   "include": ["src"]
 }

firestore-bigquery-export/functions/__tests__/__mocks__/console.ts

@@ -0,0 +1,5 @@
+export const mockConsoleLog = jest.spyOn(console, "log").mockImplementation();
+
+export const mockConsoleError = jest
+  .spyOn(console, "error")
+  .mockImplementation();

firestore-bigquery-export/functions/__tests__/__mocks__/firestore.ts

@@ -0,0 +1,36 @@
+import * as functionsTestInit from "firebase-functions-test";
+
+export const snapshot = (
+  input = { input: "hello" },
+  path = "translations/id1"
+) => {
+  let functionsTest = functionsTestInit();
+  return functionsTest.firestore.makeDocumentSnapshot(input, path);
+};
+
+export const mockDocumentSnapshotFactory = (documentSnapshot) => {
+  return jest.fn().mockImplementation(() => {
+    return {
+      exists: true,
+      get: documentSnapshot.get.bind(documentSnapshot),
+      ref: { path: documentSnapshot.ref.path },
+    };
+  })();
+};
+
+export const makeChange = (before, after) => {
+  let functionsTest = functionsTestInit();
+  return functionsTest.makeChange(before, after);
+};
+
+export const mockFirestoreTransaction = jest.fn().mockImplementation(() => {
+  return (transactionHandler) => {
+    transactionHandler({
+      update(ref, field, data) {
+        mockFirestoreUpdate(field, data);
+      },
+    });
+  };
+});
+
+export const mockFirestoreUpdate = jest.fn();

firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap

@@ -0,0 +1,41 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`extension config config loaded from environment variables 1`] = `
+Object {
+  "collectionPath": undefined,
+  "datasetId": undefined,
+  "initialized": false,
+  "location": "europe-west2",
+  "tableId": undefined,
+}
+`;
+
+exports[`extension config config.datasetId param exists 1`] = `
+Object {
+  "default": "firestore_export",
+  "description": "What ID would you like to use for your BigQuery dataset? This extension will create the dataset, if it doesn't already exist.",
+  "example": "firestore_export",
+  "label": "Dataset ID",
+  "param": "DATASET_ID",
+  "required": true,
+  "type": "string",
+  "validationErrorMessage": "BigQuery dataset IDs must be alphanumeric (plus underscores) and must be no more than 1024 characters.
+",
+  "validationRegex": "^[a-zA-Z0-9_]+$",
+}
+`;
+
+exports[`extension config config.tableId param exists 1`] = `
+Object {
+  "default": "posts",
+  "description": "What identifying prefix would you like to use for your table and view inside your BigQuery dataset? This extension will create the table and view, if they don't already exist.",
+  "example": "posts",
+  "label": "Table ID",
+  "param": "TABLE_ID",
+  "required": true,
+  "type": "string",
+  "validationErrorMessage": "BigQuery table IDs must be alphanumeric (plus underscores) and must be no more than 1024 characters.
+",
+  "validationRegex": "^[a-zA-Z0-9_]+$",
+}
+`;

firestore-bigquery-export/functions/__tests__/config.test.ts

@@ -0,0 +1,99 @@
+import { readFileSync } from "fs";
+
+import { resolve as pathResolve } from "path";
+
+import * as yaml from "js-yaml";
+import mockedEnv from "mocked-env";
+import config from "../src/config";
+
+let restoreEnv;
+let extensionYaml;
+let extensionParams;
+
+const environment = {
+  LOCATION: "us-central1",
+  DATASET_ID: "my_dataset",
+  TABLE_ID: "my_table",
+};
+
+describe("extension config", () => {
+  beforeAll(() => {
+    extensionYaml = yaml.safeLoad(
+      readFileSync(pathResolve(__dirname, "../../extension.yaml"), "utf8")
+    );
+
+    extensionParams = extensionYaml.params.reduce((obj, param) => {
+      obj[param.param] = param;
+      return obj;
+    }, {});
+  });
+
+  beforeEach(() => {
+    restoreEnv = mockedEnv(environment);
+  });
+
+  afterEach(() => restoreEnv());
+
+  test("config loaded from environment variables", () => {
+    const functionsConfig = config;
+
+    expect(functionsConfig).toMatchSnapshot({});
+  });
+
+  // DATASET_ID
+  describe("config.datasetId", () => {
+    test("param exists", () => {
+      const extensionParam = extensionParams["DATASET_ID"];
+      expect(extensionParam).toMatchSnapshot();
+    });
+
+    describe("validationRegex", () => {
+      test("does not allow empty strings", () => {
+        const { validationRegex } = extensionParams["DATASET_ID"];
+        expect(Boolean("".match(new RegExp(validationRegex)))).toBeFalsy();
+      });
+
+      test("does not allow spaces", () => {
+        const { validationRegex } = extensionParams["DATASET_ID"];
+        expect(
+          Boolean("foo bar,".match(new RegExp(validationRegex)))
+        ).toBeFalsy();
+      });
+
+      test("allows a alphanumeric underscore ids", () => {
+        const { validationRegex } = extensionParams["DATASET_ID"];
+        expect(
+          Boolean("my_dataset".match(new RegExp(validationRegex)))
+        ).toBeTruthy();
+      });
+    });
+  });
+
+  // TABLE_ID
+  describe("config.tableId", () => {
+    test("param exists", () => {
+      const extensionParam = extensionParams["TABLE_ID"];
+      expect(extensionParam).toMatchSnapshot();
+    });
+
+    describe("validationRegex", () => {
+      test("does not allow empty strings", () => {
+        const { validationRegex } = extensionParams["TABLE_ID"];
+        expect(Boolean("".match(new RegExp(validationRegex)))).toBeFalsy();
+      });
+      test("does not allow spaces", () => {
+        const { validationRegex } = extensionParams["TABLE_ID"];
+        expect(
+          Boolean("foo bar,".match(new RegExp(validationRegex)))
+        ).toBeFalsy();
+      });
+
+      test("allows a alphanumeric underscore ids", () => {
+        const { validationRegex } = extensionParams["TABLE_ID"];
+        expect(
+          Boolean("my_table".match(new RegExp(validationRegex)))
+        ).toBeTruthy();
+      });
+    });
+  });
+});

firestore-bigquery-export/functions/__tests__/functions.test.ts

@@ -0,0 +1,121 @@
+import * as functionsTestInit from "../node_modules/firebase-functions-test";
+import mockedEnv from "../node_modules/mocked-env";
+
+import { mockConsoleLog } from "./__mocks__/console";
+
+import config from "../src/config";
+
+jest.mock("@firebaseextensions/firestore-bigquery-change-tracker", () => ({
+  FirestoreBigQueryEventHistoryTracker: jest.fn(() => {
+    return {
+      record: jest.fn(() => {}),
+    };
+  }),
+  ChangeType: {
+    DELETE: 2,
+    UPDATE: 1,
+    CREATE: 0,
+  },
+}));
+
+jest.mock("../src/logs", () => ({
+  start: jest.fn(() =>
+    console.log("Started execution of extension with configuration", config)
+  ),
+  init: jest.fn(() => {}),
+  error: jest.fn(() => {}),
+  complete: jest.fn(() => console.log("Completed execution of extension")),
+}));
+
+const defaultEnvironment = {
+  PROJECT_ID: "fake-project",
+  DATASET_ID: "my_ds_id",
+  TABLE_ID: "my_id",
+  COLLECTION_PATH: "example",
+};
+
+export const mockExport = (document, data) => {
+  const ref = require("../src/index").fsexportbigquery;
+  let functionsTest = functionsTestInit();
+
+  const wrapped = functionsTest.wrap(ref);
+  return wrapped(document, data);
+};
+
+export const mockedFirestoreBigQueryEventHistoryTracker = () => {};
+
+let restoreEnv;
+let functionsTest = functionsTestInit();
+
+describe("extension", () => {
+  beforeEach(() => {
+    restoreEnv = mockedEnv(defaultEnvironment);
+  });
+
+  test("functions are exported", () => {
+    const exportedFunctions = jest.requireActual("../src");
+    expect(exportedFunctions.fsexportbigquery).toBeInstanceOf(Function);
+  });
+
+  describe("functions.fsexportbigquery", () => {
+    let functionsConfig;
+    let callResult;
+
+    beforeEach(async () => {
+      jest.resetModules();
+      functionsTest = functionsTestInit();
+
+      functionsConfig = config;
+    });
+
+    test("functions runs with a deletion", async () => {
+      const beforeSnapshot = { foo: "bar" };
+      const afterSnapshot = { foo: "bars" };
+
+      const documentChange = functionsTest.makeChange(
+        beforeSnapshot,
+        afterSnapshot
+      );
+
+      const callResult = await mockExport(documentChange, {
+        resource: {
+          name: "test",
+        },
+      });
+
+      expect(callResult).toBeUndefined();
+
+      expect(mockConsoleLog).toBeCalledWith(
+        "Started execution of extension with configuration",
+        functionsConfig
+      );
+
+      expect(mockConsoleLog).toBeCalledWith("Completed execution of extension");
+    });
+
+    test("function runs with updated data", async () => {
+      const beforeSnapshot = { foo: "bar" };
+      const afterSnapshot = { foo: "bars", exists: true };
+
+      const documentChange = functionsTest.makeChange(
+        beforeSnapshot,
+        afterSnapshot
+      );
+
+      const callResult = await mockExport(documentChange, {
+        resource: {
+          name: "test",
+        },
+      });
+
+      expect(callResult).toBeUndefined();
+
+      expect(mockConsoleLog).toBeCalledWith(
+        "Started execution of extension with configuration",
+        functionsConfig
+      );
+
+      expect(mockConsoleLog).toBeCalledWith("Completed execution of extension");
+    });
+  });
+});