Merge pull request #54 from mcode/standalone-mcode-client

Standalone mcode client

Author: Dtphelan1

.gitignore

@@ -1,3 +1,7 @@
 node_modules/
 .vscode/
 .DS_Store
+output/
+logs/
+config/*.json
+!config/mcode-csv-config.example.json

config/mcode-csv-config.example.json

@@ -0,0 +1,79 @@
+{
+  "patientIdCsvPath": "./data/patient-mrns.csv",
+  "commonExtractorArgs": {},
+  "notificationInfo": {
+    "host": "smtp.example.com",
+    "port": 587,
+    "from": "sender@example.com",
+    "to": [
+      "demo@example.com",
+      "test@example.com"
+    ]
+  },
+  "extractors": [
+    {
+      "label": "condition",
+      "type": "CSVConditionExtractor",
+      "constructorArgs": {
+        "filePath": "./data/condition-information.csv"
+      }
+    },
+    {
+      "label": "patient",
+      "type": "CSVPatientExtractor",
+      "constructorArgs": {
+        "filePath": "./data/patient-information.csv"
+      }
+    },
+    {
+      "label": "cancerDiseaseStatus",
+      "type": "CSVCancerDiseaseStatusExtractor",
+      "constructorArgs": {
+        "filePath": "./data/cancer-disease-status-information.csv"
+      }
+    },
+    {
+      "label": "clinicalTrialInformation",
+      "type": "CSVClinicalTrialInformationExtractor",
+      "constructorArgs": {
+        "filePath": "./data/clinical-trial-information.csv",
+        "clinicalSiteID": "example-site-id"
+      }
+    },
+    {
+      "label": "treatmentPlanChange",
+      "type": "CSVTreatmentPlanChangeExtractor",
+      "constructorArgs": {
+        "filePath": "./data/treatment-plan-change-information.csv"
+      }
+    },
+    {
+      "label": "staging",
+      "type": "CSVStagingExtractor",
+      "constructorArgs": {
+        "filePath": "./data/staging-information.csv"
+      }
+    },
+    {
+      "label": "cancerRelatedMedication",
+      "type": "CSVCancerRelatedMedicationExtractor",
+      "constructorArgs": {
+        "filePath": "./data/cancer-related-medication-information.csv"
+      }
+    },
+    {
+      "label": "genericObservations",
+      "type": "CSVObservationExtractor",
+      "constructorArgs": {
+        "filePath": "./data/observation-information.csv"
+      }
+    },
+    {
+      "label": "genericProcedures",
+      "type": "CSVProcedureExtractor",
+      "constructorArgs": {
+        "filePath": "./data/procedure-information.csv"
+      }
+    }
+  ]
+}

docs/config.example.json

@@ -28,8 +28,8 @@
       }
     }
   ],
-  "auth": {
-    "tokenEndpoint": "http://example.com/oauth2/token",
+  "webServiceAuthConfig": {
+    "url": "http://example.com/oauth2/token",
     "clientId": "client_id",
     "jwk": {}
   },

package-lock.json

@@ -20,11 +20,13 @@
   "license": "Apache-2.0",
   "dependencies": {
     "axios": "^0.19.0",
+    "commander": "^6.2.0",
     "csv-parse": "^4.8.8",
     "fhir-crud-client": "^1.2.1",
     "fhirpath": "^2.1.5",
     "lodash": "^4.17.19",
     "moment": "^2.26.0",
+    "nodemailer": "^6.4.14",
     "sha.js": "^2.4.9",
     "winston": "^3.2.1"
   },

package.json

@@ -0,0 +1,67 @@
+const path = require('path');
+const moment = require('moment');
+const fs = require('fs');
+const logger = require('../helpers/logger');
+
+// Sort Log records by `dateRun`, more recent (larger) dates to least recent (smaller)
+function logSorter(a, b) {
+  return moment(b.dateRun) - moment(a.dateRun);
+}
+
+// Common wrapper function for checking if a value is a date
+function isDate(date) {
+  return moment(date).isValid();
+}
+
+// Create a new instance of a Log record based on a `to` and `from` date
+function createLogObject(fromDate, toDate) {
+  const now = moment();
+  return {
+    dateRun: now,
+    toDate: toDate || now,
+    fromDate,
+  };
+}
+
+// Provides an interface for loading existing logs of client RunInstances,
+// getting the most recent runs, and returning the most recent run
+class RunInstanceLogger {
+  constructor(pathToLogFile) {
+    this.logPath = pathToLogFile;
+    try {
+      this.logs = JSON.parse(fs.readFileSync(path.resolve(this.logPath)));
+      // Sort logs on load
+      this.logs.sort(logSorter);
+    } catch (err) {
+      logger.error(`FATAL-Could not parse the logPath provided: ${this.logPath}`);
+      process.exit(1);
+    }
+  }
+
+  // Get the most recent run performed and logged
+  getMostRecentToDate() {
+    // Filter logs by client
+    const filteredLogs = this.logs.filter((l) => l.client === this.client);
+    // Sorting of logs is maintained over load and update; this should be safe
+    return filteredLogs[0] && filteredLogs[0].toDate;
+  }
+
+  // Calling this adds a new Log record to our local object and to the file on disk
+  addRun(fromDate, toDate) {
+    logger.info('Logging successful run information to records');
+    // If fromDate isn't valid, or if toDate is both defined and invalid, we can't properly log
+    if (!isDate(fromDate) || (toDate && !isDate(toDate))) {
+      logger.error(`Trying to add a run to RunInstance logger, but toDate and fromDate are not valid dates: to ${toDate} and from ${fromDate}`);
+      logger.error('Log failed');
+      return;
+    }
+    const log = createLogObject(fromDate, toDate);
+    this.logs.push(log);
+    this.logs.sort(logSorter);
+    fs.writeFileSync(this.logPath, JSON.stringify(this.logs));
+  }
+}
+
+module.exports = {
+  RunInstanceLogger,
+};

src/cli/RunInstanceLogger.js

@@ -0,0 +1,128 @@
+const parse = require('csv-parse/lib/sync');
+const fs = require('fs');
+const path = require('path');
+const moment = require('moment');
+const logger = require('../helpers/logger');
+const { RunInstanceLogger } = require('./RunInstanceLogger');
+const { sendEmailNotification, zipErrors } = require('./emailNotifications');
+const { extractDataForPatients } = require('./mcodeExtraction');
+
+function getConfig(pathToConfig) {
+  // Checks pathToConfig points to valid JSON file
+  const fullPath = path.resolve(pathToConfig);
+  try {
+    return JSON.parse(fs.readFileSync(fullPath));
+  } catch (err) {
+    throw new Error(`The provided filepath to a configuration file ${pathToConfig}, full path ${fullPath} did not point to a valid JSON file.`);
+  }
+}
+
+function checkInputAndConfig(config, fromDate, toDate) {
+  // Check input args and needed config variables based on client being used
+  const { patientIdCsvPath } = config;
+
+  // Check if `fromDate` is a valid date
+  if (fromDate && !moment(fromDate).isValid()) {
+    throw new Error('-f/--from-date is not a valid date.');
+  }
+
+  // Check if `toDate` is a valid date
+  if (toDate && !moment(toDate).isValid()) {
+    throw new Error('-t/--to-date is not a valid date.');
+  }
+
+  // Check if there is a path to the MRN CSV within our config JSON
+  if (!patientIdCsvPath) {
+    throw new Error('patientIdCsvPath is required in config file');
+  }
+}
+
+function checkLogFile(pathToLogs) {
+  // Check that the given log file exists
+  try {
+    const logFileContent = JSON.parse(fs.readFileSync(pathToLogs));
+    if (!Array.isArray(logFileContent)) throw new Error('Log file needs to be an array.');
+  } catch (err) {
+    logger.error(`The provided filepath to a LogFile, ${pathToLogs}, did not point to a valid JSON file. Create a json file with an empty array at this location.`);
+    throw new Error(err.message);
+  }
+}
+
+// Use previous runs to infer a valid fromDate if none was provided
+function getEffectiveFromDate(fromDate, runLogger) {
+  if (fromDate) return fromDate;
+
+  // Use the most recent ToDate
+  logger.info('No fromDate was provided, inferring an effectiveFromDate');
+  const effectiveFromDate = runLogger.getMostRecentToDate();
+  logger.info(`effectiveFromDate: ${effectiveFromDate}`);
+  if (!effectiveFromDate) {
+    throw new Error('no valid fromDate was supplied, and there are no log records from which we could pull a fromDate');
+  }
+
+  return effectiveFromDate;
+}
+
+async function mcodeApp(Client, fromDate, toDate, pathToConfig, pathToRunLogs, debug, allEntries) {
+  try {
+    if (debug) logger.level = 'debug';
+    // Don't require a run-logs file if we are extracting all-entries. Only required when using --entries-filter.
+    if (!allEntries) checkLogFile(pathToRunLogs);
+    const config = getConfig(pathToConfig);
+    checkInputAndConfig(config, fromDate, toDate);
+
+    // Create and initialize client
+    const mcodeClient = new Client(config);
+    await mcodeClient.init();
+
+    // Parse CSV for list of patient mrns
+    const patientIdsCsvPath = path.resolve(config.patientIdCsvPath);
+    const patientIds = parse(fs.readFileSync(patientIdsCsvPath, 'utf8'), { columns: true }).map((row) => row.mrn);
+
+    // Get RunInstanceLogger for recording new runs and inferring dates from previous runs
+    const runLogger = allEntries ? null : new RunInstanceLogger(pathToRunLogs);
+    const effectiveFromDate = allEntries ? null : getEffectiveFromDate(fromDate, runLogger);
+    const effectiveToDate = allEntries ? null : toDate;
+
+    // Extract the data
+    logger.info(`Extracting data for ${patientIds.length} patients`);
+    const { extractedData, successfulExtraction, totalExtractionErrors } = await extractDataForPatients(patientIds, mcodeClient, effectiveFromDate, effectiveToDate);
+
+    // If we have notification information, send an emailNotification
+    const { notificationInfo } = config;
+    if (notificationInfo) {
+      const notificationErrors = zipErrors(totalExtractionErrors);
+      await sendEmailNotification(notificationInfo, notificationErrors, debug);
+    }
+    // A run is successful and should be logged when both extraction finishes without fatal errors
+    // and messages are posted without fatal errors
+    if (!allEntries && effectiveFromDate) {
+      const successCondition = successfulExtraction;
+      if (successCondition) {
+        runLogger.addRun(effectiveFromDate, effectiveToDate);
+      }
+    }
+
+    // Finally, save the data to disk
+    const outputPath = './output';
+    if (!fs.existsSync(outputPath)) {
+      logger.info(`Creating directory ${outputPath}`);
+      fs.mkdirSync(outputPath);
+    }
+    // For each bundle in our extractedData, write it to our output directory
+    extractedData.forEach((bundle, i) => {
+      const outputFile = path.join(outputPath, `mcode-extraction-patient-${i + 1}.json`);
+      logger.debug(`Logging mCODE output to ${outputFile}`);
+      fs.writeFileSync(outputFile, JSON.stringify(bundle), 'utf8');
+    });
+    logger.info(`Successfully logged ${extractedData.length} mCODE bundle(S) to ${outputPath}`);
+  } catch (e) {
+    logger.error(e.message);
+    logger.debug(e.stack);
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  mcodeApp,
+};

src/cli/app.js

@@ -0,0 +1,26 @@
+const path = require('path');
+const program = require('commander');
+const { MCODEClient } = require('../client/MCODEClient');
+const { mcodeApp } = require('./app');
+
+const defaultPathToConfig = path.join('config', 'csv.config.json');
+const defaultPathToRunLogs = path.join('logs', 'run-logs.json');
+
+program
+  .usage('[options]')
+  .option('-f --from-date <date>', 'The earliest date and time to search')
+  .option('-t --to-date <date>', 'The latest date and time to search')
+  .option('-e, --entries-filter', 'Flag to indicate to filter data by date')
+  .option('-c --config-filepath <path>', 'Specify relative path to config to use:', defaultPathToConfig)
+  .option('-r --run-log-filepath <path>', 'Specify relative path to log file of previous runs:', defaultPathToRunLogs)
+  .option('-d, --debug', 'output extra debugging information')
+  .parse(process.argv);
+
+const {
+  fromDate, toDate, configFilepath, runLogFilepath, debug, entriesFilter,
+} = program;
+
+// Flag to extract allEntries, or just to use to-from dates
+const allEntries = !entriesFilter;
+
+mcodeApp(MCODEClient, fromDate, toDate, configFilepath, runLogFilepath, debug, allEntries);