Add --non-interactive flag to views generator script (#75)

* Add --no-interactive flag to views generator script

* address review feedback

* small fixes

* address latest review feedback round

* rename util -> schema-loader-utils, let -> const

Author: jmwski

firestore-bigquery-export/scripts/gen-schema-view/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firebaseextensions/fs-bq-schema-views",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Generate strongly-typed BigQuery Views based on raw JSON",
   "main": "./lib/index.js",
   "repository": {
@@ -26,19 +26,23 @@
   "author": "Jan Wyszynski <wyszynski@google.com>",
   "license": "Apache-2.0",
   "dependencies": {
+    "commander": "3.0.2",
+    "glob": "7.1.5",
     "@google-cloud/bigquery": "^2.1.0",
+    "fs-find": "^0.4.0",
     "firebase-admin": "^7.1.1",
     "firebase-functions": "^2.2.1",
     "sql-formatter": "^2.3.3",
     "generate-schema": "^2.6.0",
-    "@firebaseextensions/firestore-bigquery-change-tracker": "^0.1.3",
     "inquirer": "^6.4.0"
   },
   "devDependencies": {
+    "@types/mocha": "^5.2.5",
     "rimraf": "^2.6.3",
     "nyc": "^14.0.0",
-    "mocha": "^5.0.5",
-    "typescript": "^3.4.5",
+    "mocha": "^5.2.0",
+    "mocha-typescript": "*",
+    "typescript": "^3.5.2",
     "ts-node": "^7.0.1"
   }
 }

firestore-bigquery-export/scripts/gen-schema-view/src/index.ts

@@ -17,16 +17,13 @@
  */
 
 import * as bigquery from "@google-cloud/bigquery";
+import * as program from "commander";
 import * as firebase from "firebase-admin";
 import * as inquirer from "inquirer";
-import * as path from "path";
 
-import { existsSync, readdirSync } from "fs";
+import { FirestoreBigQuerySchemaViewFactory, FirestoreSchema } from "./schema";
 
-import {
-  FirestoreBigQuerySchemaViewFactory,
-  FirestoreSchema,
-} from "./schema";
+import { readSchemas } from "./schema-loader-utils";
 
 const BIGQUERY_VALID_CHARACTERS = /^[a-zA-Z0-9_]+$/;
 const FIRESTORE_VALID_CHARACTERS = /^[^\/]+$/;
@@ -41,86 +38,139 @@ const validateInput = (value: any, name: string, regex: RegExp) => {
   return true;
 };
 
+function collect(value, previous) {
+  return previous.concat([value]);
+}
+
+program
+  .name("gen-schema-views")
+  .option(
+    "--non-interactive",
+    "Parse all input from command line flags instead of prompting the caller.",
+    false
+  )
+  .option(
+    "-P, --project <project>",
+    "Firebase Project ID for project containing Cloud Firestore database."
+  )
+  .option(
+    "-d, --dataset <dataset>",
+    "The ID of the BigQuery dataset containing a raw Cloud Firestore document changelog."
+  )
+  .option(
+    "-t, --table-name-prefix <table-name-prefix>",
+    "A common prefix for the names of all views generated by this script."
+  )
+  .option(
+    "-f, --schema-files <schema-files>",
+    "A collection of files from which to read schemas.",
+    collect,
+    []
+  );
+
 const questions = [
   {
     message: "What is your Firebase project ID?",
-    name: "projectId",
+    name: "project",
     type: "input",
     validate: (value) =>
       validateInput(value, "project ID", FIRESTORE_VALID_CHARACTERS),
   },
   {
     message:
       "What is the ID of the BigQuery dataset the raw changelog lives in? (The dataset and the raw changelog must already exist!)",
-    name: "datasetId",
+    name: "dataset",
     type: "input",
     validate: (value) =>
-      validateInput(value, "dataset", BIGQUERY_VALID_CHARACTERS),
+      validateInput(value, "dataset ID", BIGQUERY_VALID_CHARACTERS),
   },
   {
     message:
-      "What is the name of the Cloud Firestore Collection that you would like to generate a schema view for?",
-    name: "collectionName",
+      "What is the name of the Cloud Firestore collection for which you want to generate a schema view?",
+    name: "tableNamePrefix",
     type: "input",
     validate: (value) =>
-      validateInput(value, "dataset", BIGQUERY_VALID_CHARACTERS),
+      validateInput(value, "table name prefix", BIGQUERY_VALID_CHARACTERS),
   },
   {
     message:
-      "Have you installed all your desired schemas in ./schemas/*.json?",
-    name: "confirmed",
-    type: "confirm",
-  }
+      "Where should this script look for schema definitions? (Enter a comma-separated list of, optionally globbed, paths to files or directories).",
+    name: "schemaFiles",
+    type: "input",
+  },
 ];
 
-async function run(): Promise<number> {
-  const schemaDirectory = [process.cwd(), "schemas"].join('/');
-  const schemaDirExists = existsSync(schemaDirectory);
-  if (!schemaDirExists) {
-    console.log(`Expected directory "${schemaDirectory}" not found!`);
-    process.exit(1);
-  }
+interface CliConfig {
+  projectId: string;
+  datasetId: string;
+  tableNamePrefix: string;
+  schemas: { [schemaName: string]: FirestoreSchema };
+}
 
-  const {
-    projectId,
-    datasetId,
-    collectionName,
-    confirmed
-  } = await inquirer.prompt(questions);
+async function run(): Promise<number> {
+  // Get all configuration options via inquirer prompt or commander flags.
+  const config: CliConfig = await parseConfig();
 
   // Set project ID so it can be used in BigQuery intialization
-  process.env.PROJECT_ID = projectId;
-  // BigQuery aactually requires this variable to set the project correctly.
-  process.env.GOOGLE_CLOUD_PROJECT = projectId;
+  process.env.PROJECT_ID = config.projectId;
+  // BigQuery actually requires this variable to set the project correctly.
+  process.env.GOOGLE_CLOUD_PROJECT = config.projectId;
 
   // Initialize Firebase
   firebase.initializeApp({
     credential: firebase.credential.applicationDefault(),
-    databaseURL: `https://${projectId}.firebaseio.com`,
+    databaseURL: `https://${config.projectId}.firebaseio.com`,
   });
 
   // @ts-ignore string not assignable to enum
-  const schemas: { [schemaName: string]: FirestoreSchema} = readSchemas(schemaDirectory);
-  if (Object.keys(schemas).length === 0) {
-    console.log(`Found no schemas in ${schemaDirectory}!`);
+  if (Object.keys(config.schemas).length === 0) {
+    console.log(`No schema files found!`);
   }
   const viewFactory = new FirestoreBigQuerySchemaViewFactory();
-
-  for (const schemaName in schemas) {
-    await viewFactory.initializeSchemaViewResources(datasetId, collectionName, schemaName, schemas[schemaName]);
+  for (const schemaName in config.schemas) {
+    await viewFactory.initializeSchemaViewResources(
+      config.datasetId,
+      config.tableNamePrefix,
+      schemaName,
+      config.schemas[schemaName]
+    );
   }
   return 0;
-};
+}
 
-function readSchemas(directory: string): { [schemaName: string]: FirestoreSchema } {
-  let results = {};
-  let files = readdirSync(directory);
-  let schemaNames = files.map(fileName => path.basename(fileName).split('.').slice(0, -1).join('.').replace(/-/g,'_'));
-  for (var i = 0; i < files.length; i++) {
-    const schema: FirestoreSchema = require([directory, files[i]].join('/'));
-    results[schemaNames[i]] = schema;
+async function parseConfig(): Promise<CliConfig> {
+  program.parse(process.argv);
+  if (program.nonInteractive) {
+    if (
+      program.project === undefined ||
+      program.dataset === undefined ||
+      program.tableNamePrefix === undefined ||
+      program.schemaFiles.length === 0
+    ) {
+      program.outputHelp();
+      process.exit(1);
+    }
+    return {
+      projectId: program.project,
+      datasetId: program.dataset,
+      tableNamePrefix: program.tableNamePrefix,
+      schemas: readSchemas(program.schemaFiles)
+    };
   }
-  return results;
+  const {
+    project,
+    dataset,
+    tableNamePrefix,
+    schemaFiles,
+  } = await inquirer.prompt(questions);
+  return {
+    projectId: project,
+    datasetId: dataset,
+    tableNamePrefix: tableNamePrefix,
+    schemas: readSchemas(
+      schemaFiles.split(",").map((schemaFileName) => schemaFileName.trim()),
+    ),
+  };
 }
 
 run()

firestore-bigquery-export/scripts/gen-schema-view/src/schema-loader-utils.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as glob from "glob";
+import * as path from "path";
+
+import { FirestoreSchema } from "./schema";
+
+import { existsSync, readdirSync, lstatSync } from "fs";
+
+export function readSchemas(
+  globs: string[]
+): { [schemaName: string]: FirestoreSchema } {
+  let schemas = {};
+  const expanded = expandGlobs(globs);
+  for (var i = 0; i < expanded.length; i++) {
+    const dirent = resolveFilePath(expanded[i]);
+    const stats = lstatSync(dirent);
+    if (stats.isDirectory()) {
+      const directorySchemas = readSchemasFromDirectory(dirent);
+      for (const schemaName in directorySchemas) {
+        if (schemas.hasOwnProperty(schemaName)) {
+          warnDuplicateSchemaName(schemaName);
+        }
+        schemas[schemaName] = directorySchemas[schemaName];
+      }
+    } else {
+      const schemaName = filePathToSchemaName(dirent);
+      if (schemas.hasOwnProperty(schemaName)) {
+        warnDuplicateSchemaName(schemaName);
+      }
+      schemas[schemaName] = readSchemaFromFile(dirent);
+    }
+  }
+  return schemas;
+}
+
+function warnDuplicateSchemaName(schemaName: string) {
+  console.log(
+    `Found multiple schema files named ${schemaName}! Only the last one will be used to create a schema view!`
+  );
+}
+
+function resolveFilePath(filePath: string): string {
+  if (filePath.startsWith(".") || !filePath.startsWith("/")) {
+    return [process.cwd(), filePath].join("/");
+  }
+  return filePath;
+}
+
+function expandGlobs(globs: string[]): string[] {
+  let results = [];
+  for (var i = 0; i < globs.length; i++) {
+    const globResults = glob.sync(globs[i]);
+    results = results.concat(globResults);
+  }
+  return results;
+}
+
+function readSchemasFromDirectory(
+  directory: string
+): { [schemaName: string]: FirestoreSchema } {
+  let results = {};
+  const files = readdirSync(directory);
+  const schemaNames = files.map((fileName) => filePathToSchemaName(fileName));
+  for (var i = 0; i < files.length; i++) {
+    const schema: FirestoreSchema = readSchemaFromFile(
+      [directory, files[i]].join("/")
+    );
+    results[schemaNames[i]] = schema;
+  }
+  return results;
+}
+
+function readSchemaFromFile(file: string): FirestoreSchema {
+  return require(file);
+}
+
+export function filePathToSchemaName(filePath: string): string {
+  return path
+    .basename(filePath)
+    .split(".")
+    .slice(0, -1)
+    .join(".")
+    .replace(/-/g, "_");
+}

firestore-bigquery-export/scripts/gen-schema-view/src/schema.ts

@@ -72,14 +72,14 @@ export class FirestoreBigQuerySchemaViewFactory {
    */
   async initializeSchemaViewResources(
     datasetId: string,
-    collectionName: string,
+    tableNamePrefix: string,
     schemaName: string,
     firestoreSchema: FirestoreSchema,
   ): Promise<bigquery.Table> {
-    const rawChangeLogTableName = changeLog(raw(collectionName));
-    const latestRawViewName = latest(raw(collectionName));
-    const changeLogSchemaViewName = changeLog(schema(collectionName, schemaName));
-    const latestSchemaViewName = latest(schema(collectionName, schemaName));
+    const rawChangeLogTableName = changeLog(raw(tableNamePrefix));
+    const latestRawViewName = latest(raw(tableNamePrefix));
+    const changeLogSchemaViewName = changeLog(schema(tableNamePrefix, schemaName));
+    const latestSchemaViewName = latest(schema(tableNamePrefix, schemaName));
     const dataset = this.bq.dataset(datasetId);
 
     const udfNames = Object.keys(udfs);
@@ -181,7 +181,7 @@ export const buildSchemaViewQuery = (
 }
 
 /**
- * Given a firestore schema which may contain values for any type present
+ * Given a Cloud Firestore schema which may contain values for any type present
  * in the Firestore document proto, return a list of clauses that may be
  * used to extract schema values from a JSON string and convert them into
  * the corresponding BigQuery type.
@@ -302,7 +302,7 @@ const processLeafField = (
     const longitude = jsonExtract(dataFieldName, extractPrefix, field, `._longitude`, transformer);
      /*
       * We return directly from this branch because it's the only one that
-      * generate multiple selector clauses.
+      * generates multiple selector clauses.
       */
     fieldNameToSelector[qualifyFieldName(prefix, field.name)] =
       `${firestoreGeopoint(

firestore-bigquery-export/scripts/gen-schema-view/src/test/fixtures/schema-files/deep-directory/1/2/leaf-schemas-1.json

@@ -0,0 +1 @@
+{}

firestore-bigquery-export/scripts/gen-schema-view/src/test/fixtures/schema-files/deep-directory/1/2/leaf-schemas-2.json

@@ -0,0 +1 @@
+{}

firestore-bigquery-export/scripts/gen-schema-view/src/test/fixtures/schema-files/deep-directory/1/2/leaf-schemas-3.json

@@ -0,0 +1 @@
+{}

firestore-bigquery-export/scripts/gen-schema-view/src/test/fixtures/schema-files/deep-directory/1/2/leaf-schemas-4.json

@@ -0,0 +1 @@
+{}

firestore-bigquery-export/scripts/gen-schema-view/src/test/fixtures/schema-files/deep-directory/1/2/leaf-schemas-5.json

@@ -0,0 +1 @@
+{}

firestore-bigquery-export/scripts/gen-schema-view/src/test/fixtures/schema-files/deep-directory/1/other-schemas-1.json

@@ -0,0 +1 @@
+{}