implemented CSVURLModule; updated CSVextractors to support new arg

Dtphelan1 · Dtphelan1 · commit 7b894b098a16 · 2021-07-02T14:31:02.000-04:00
diff --git a/src/client/BaseClient.js b/src/client/BaseClient.js
@@ -25,32 +25,37 @@ class BaseClient {
   }
 
   // Given an extractor configuration, initialize all the necessary extractors
-  initializeExtractors(extractorConfig, commonExtractorArgs) {
-    let allExtractorsValid = true;
-
+  async initializeExtractors(extractorConfig, commonExtractorArgs) {
+    // Loop to initialize the extractors
     extractorConfig.forEach((curExtractorConfig) => {
       const { label, type, constructorArgs } = curExtractorConfig;
       logger.debug(`Initializing ${label} extractor with type ${type}`);
       const ExtractorClass = this.extractorClasses[type];
-
       try {
         const newExtractor = new ExtractorClass({ ...commonExtractorArgs, ...constructorArgs });
-
-        if (newExtractor.validate) {
-          const isExtractorValid = newExtractor.validate();
-          allExtractorsValid = (allExtractorsValid && isExtractorValid);
-          if (isExtractorValid) {
-            logger.debug(`Extractor ${label} PASSED CSV validation`);
-          } else {
-            logger.debug(`Extractor ${label} FAILED CSV validation`);
-          }
-        }
-
         this.extractors.push(newExtractor);
       } catch (e) {
         throw new Error(`Unable to initialize ${label} extractor with type ${type}: ${e.message}`);
       }
     });
+    // For validation, we are looping over extractors and performing an async operation on each.
+    // We need to loop without forEach (since forEach is sequential).
+    // Using Reduce to compute the validity of all extractors
+    const allExtractorsValid = await this.extractors.reduce(async (curExtractorsValid, curExtractor) => {
+      const { name } = curExtractor.constructor;
+
+      if (curExtractor.validate) {
+        logger.debug(`Validating ${name}`);
+        const isExtractorValid = await curExtractor.validate();
+        if (isExtractorValid) {
+          logger.debug(`Extractor ${name} PASSED CSV validation`);
+        } else {
+          logger.warn(`Extractor ${name} FAILED CSV validation`);
+        }
+        return (curExtractorsValid && isExtractorValid);
+      }
+      return curExtractorsValid;
+    }, true);
 
     if (allExtractorsValid) {
       logger.info('Validation succeeded');
diff --git a/src/extractors/BaseCSVExtractor.js b/src/extractors/BaseCSVExtractor.js
@@ -1,25 +1,25 @@
-const path = require('path');
-const { Extractor } = require('./Extractor');
-const { CSVModule } = require('../modules');
-const { validateCSV } = require('../helpers/csvValidator');
 const logger = require('../helpers/logger');
+const { Extractor } = require('./Extractor');
+const { CSVFileModule, CSVURLModule } = require('../modules');
 
 class BaseCSVExtractor extends Extractor {
-  constructor({ filePath, csvSchema, unalterableColumns }) {
+  constructor({ filePath, url, csvSchema, unalterableColumns }) {
     super();
     this.unalterableColumns = unalterableColumns || [];
     this.csvSchema = csvSchema;
-    this.filePath = path.resolve(filePath);
-    this.csvModule = new CSVModule(this.filePath, this.unalterableColumns);
+    if (filePath) {
+      this.filePath = filePath;
+      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
+    } else if (url) {
+      this.url = url;
+      this.csvModule = new CSVURLModule(this.url, this.unalterableColumns);
+    } else {
+      throw new Error('Trying to instantiate a CSVExtractor without a filePath or url');
+    }
   }
 
-  validate() {
-    if (this.csvSchema) {
-      logger.info(`Validating CSV file for ${this.filePath}`);
-      return validateCSV(this.filePath, this.csvSchema, this.csvModule.data);
-    }
-    logger.warn(`No CSV schema provided for ${this.filePath}`);
-    return true;
+  async validate() {
+    return this.csvModule.validate(this.csvSchema);
   }
 }
 
diff --git a/src/extractors/CSVAdverseEventExtractor.js b/src/extractors/CSVAdverseEventExtractor.js
@@ -69,8 +69,8 @@ function formatData(adverseEventData, patientId) {
 }
 
 class CSVAdverseEventExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath });
+  constructor({ filePath, url }) {
+    super({ filePath, url });
   }
 
   async getAdverseEventData(mrn) {
diff --git a/src/extractors/CSVCancerDiseaseStatusExtractor.js b/src/extractors/CSVCancerDiseaseStatusExtractor.js
@@ -8,8 +8,8 @@ const logger = require('../helpers/logger');
 const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');
 
 class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
-  constructor({ filePath, implementation }) {
-    super({ filePath, csvSchema: CSVCancerDiseaseStatusSchema });
+  constructor({ filePath, url, implementation }) {
+    super({ filePath, url, csvSchema: CSVCancerDiseaseStatusSchema });
     this.implementation = implementation;
   }
 
diff --git a/src/extractors/CSVCancerRelatedMedicationExtractor.js b/src/extractors/CSVCancerRelatedMedicationExtractor.js
@@ -46,8 +46,8 @@ function formatData(medicationData, patientId) {
 }
 
 class CSVCancerRelatedMedicationExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath });
+  constructor({ filePath, url }) {
+    super({ filePath, url });
   }
 
   async getMedicationData(mrn) {
diff --git a/src/extractors/CSVClinicalTrialInformationExtractor.js b/src/extractors/CSVClinicalTrialInformationExtractor.js
@@ -8,8 +8,8 @@ const { CSVClinicalTrialInformationSchema } = require('../helpers/schemas/csv');
 
 
 class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
-  constructor({ filePath, clinicalSiteID, clinicalSiteSystem }) {
-    super({ filePath, csvSchema: CSVClinicalTrialInformationSchema });
+  constructor({ filePath, url, clinicalSiteID, clinicalSiteSystem }) {
+    super({ filePath, url, csvSchema: CSVClinicalTrialInformationSchema });
     if (!clinicalSiteID) logger.warn(`${this.constructor.name} expects a value for clinicalSiteID but got ${clinicalSiteID}`);
     this.clinicalSiteID = clinicalSiteID;
     this.clinicalSiteSystem = clinicalSiteSystem;
diff --git a/src/extractors/CSVConditionExtractor.js b/src/extractors/CSVConditionExtractor.js
@@ -49,8 +49,8 @@ function formatData(conditionData, patientId) {
 }
 
 class CSVConditionExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath, csvSchema: CSVConditionSchema });
+  constructor({ filePath, url }) {
+    super({ filePath, url, csvSchema: CSVConditionSchema });
   }
 
   async getConditionData(mrn) {
diff --git a/src/extractors/CSVObservationExtractor.js b/src/extractors/CSVObservationExtractor.js
@@ -42,8 +42,8 @@ function formatData(observationData, patientId) {
 }
 
 class CSVObservationExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath });
+  constructor({ filePath, url }) {
+    super({ filePath, url });
   }
 
   async getObservationData(mrn) {
diff --git a/src/extractors/CSVPatientExtractor.js b/src/extractors/CSVPatientExtractor.js
@@ -54,10 +54,15 @@ function joinAndReformatData(patientData) {
 }
 
 class CSVPatientExtractor extends BaseCSVExtractor {
-  constructor({ filePath, mask = [] }) {
+  constructor({ filePath, url, mask = [] }) {
     // Define CSV Columns whose values should never be altered
     const unalterableColumns = ['familyName', 'givenName'];
-    super({ filePath, csvSchema: CSVPatientSchema, unalterableColumns });
+    super({
+      filePath,
+      url,
+      // csvSchema: CSVPatientSchema,
+      unalterableColumns,
+    });
     this.mask = mask;
   }
 
diff --git a/src/extractors/CSVProcedureExtractor.js b/src/extractors/CSVProcedureExtractor.js
@@ -48,8 +48,8 @@ function formatData(procedureData, patientId) {
 }
 
 class CSVProcedureExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath });
+  constructor({ filePath, url }) {
+    super({ filePath, url });
   }
 
   async getProcedureData(mrn) {
diff --git a/src/extractors/CSVStagingExtractor.js b/src/extractors/CSVStagingExtractor.js
@@ -63,8 +63,8 @@ function formatStagingData(stagingData, categoryIds, patientId) {
 }
 
 class CSVStagingExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath });
+  constructor({ filePath, url }) {
+    super({ filePath, url });
   }
 
   async getStagingData(mrn) {
diff --git a/src/extractors/CSVTreatmentPlanChangeExtractor.js b/src/extractors/CSVTreatmentPlanChangeExtractor.js
@@ -70,8 +70,8 @@ function formatData(tpcData, patientId) {
 }
 
 class CSVTreatmentPlanChangeExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath, csvSchema: CSVTreatmentPlanChangeSchema });
+  constructor({ filePath, url }) {
+    super({ filePath, url, csvSchema: CSVTreatmentPlanChangeSchema });
   }
 
   async getTPCData(mrn, fromDate, toDate) {
diff --git a/src/helpers/csvParsingUtils.js b/src/helpers/csvParsingUtils.js
@@ -0,0 +1,43 @@
+const logger = require('./logger');
+
+// The standard string normalizer function
+function stringNormalizer(str) {
+  return str.toLowerCase();
+}
+
+// For translating null/nil-like values into empty strings
+function normalizeEmptyValues(data, unalterableColumns = []) {
+  const EMPTY_VALUES = ['null', 'nil'].map(stringNormalizer);
+  const normalizedUnalterableColumns = unalterableColumns.map(stringNormalizer);
+  // Flag tracking if empty values were normalized or not.
+  let wasEmptyNormalized = false;
+  const newData = data.map((row, i) => {
+    const newRow = { ...row };
+    // Filter out unalterable columns
+    const columnsToNormalize = Object.keys(row).filter(
+      (col) => !normalizedUnalterableColumns.includes(stringNormalizer(col)),
+    );
+    columnsToNormalize.forEach((col) => {
+      const value = newRow[col];
+      // If the value for this row-col combo is a value that should be empty, replace it
+      if (EMPTY_VALUES.includes(stringNormalizer(value))) {
+        logger.debug(
+          `NULL/NIL values '${value}' found in row-${i}, col-${col}`,
+        );
+        wasEmptyNormalized = true;
+        newRow[col] = '';
+      }
+    });
+    return newRow;
+  });
+
+  if (wasEmptyNormalized) {
+    logger.warn('NULL/NIL values found and replaced with empty-strings');
+  }
+  return newData;
+}
+
+module.exports = {
+  stringNormalizer,
+  normalizeEmptyValues,
+};
diff --git a/src/helpers/csvValidator.js b/src/helpers/csvValidator.js
@@ -1,7 +1,9 @@
 const _ = require('lodash');
 const logger = require('./logger');
 
-function validateCSV(pathToCSVFile, csvSchema, csvData) {
+// Validates csvData against the csvSchema
+// Uses the csvFileIdentifier in logs for readability
+function validateCSV(csvFileIdentifier, csvSchema, csvData) {
   let isValid = true;
 
   // Check headers
@@ -10,17 +12,17 @@ function validateCSV(pathToCSVFile, csvSchema, csvData) {
   const fileDiff = _.difference(headers, csvSchema.headers.map((h) => h.name.toLowerCase()));
 
   if (fileDiff.length > 0) {
-    logger.warn(`Found extra column(s) in CSV ${pathToCSVFile}: "${fileDiff.join(',')}"`);
+    logger.warn(`Found extra column(s) in CSV ${csvFileIdentifier}: "${fileDiff.join(',')}"`);
   }
 
   if (schemaDiff.length > 0) {
     schemaDiff.forEach((sd) => {
       const headerSchema = csvSchema.headers.find((h) => h.name.toLowerCase() === sd);
       if (headerSchema.required) {
-        logger.error(`Column ${sd} is marked as required but is missing in CSV ${pathToCSVFile}`);
+        logger.error(`Column ${sd} is marked as required but is missing in CSV ${csvFileIdentifier}`);
         isValid = false;
       } else {
-        logger.warn(`Column ${sd} is missing in CSV ${pathToCSVFile}`);
+        logger.warn(`Column ${sd} is missing in CSV ${csvFileIdentifier}`);
       }
     });
   }
@@ -31,7 +33,7 @@ function validateCSV(pathToCSVFile, csvSchema, csvData) {
       const schema = csvSchema.headers.find((h) => h.name === key);
 
       if (schema && schema.required && !value) {
-        logger.error(`Column ${key} marked as required but missing value in row ${i + 1} column ${j + 1} in CSV ${pathToCSVFile}`);
+        logger.error(`Column ${key} marked as required but missing value in row ${i + 1} column ${j + 1} in CSV ${csvFileIdentifier}`);
         isValid = false;
       }
     });
diff --git a/src/index.js b/src/index.js
@@ -35,7 +35,7 @@ const {
   MCODERadiationProcedureExtractor,
   MCODESurgicalProcedureExtractor,
 } = require('./extractors');
-const { BaseFHIRModule, CSVModule } = require('./modules');
+const { BaseFHIRModule, CSVFileModule, CSVURLModule } = require('./modules');
 const { getEthnicityDisplay,
   getPatientName,
   getRaceCodesystem,
@@ -85,7 +85,8 @@ module.exports = {
   CSVCancerRelatedMedicationExtractor,
   CSVClinicalTrialInformationExtractor,
   CSVConditionExtractor,
-  CSVModule,
+  CSVFileModule,
+  CSVURLModule,
   CSVPatientExtractor,
   CSVObservationExtractor,
   CSVProcedureExtractor,
diff --git a/src/modules/CSVFileModule.js b/src/modules/CSVFileModule.js
@@ -2,52 +2,23 @@ const fs = require('fs');
 const moment = require('moment');
 const parse = require('csv-parse/lib/sync');
 const logger = require('../helpers/logger');
+const { validateCSV } = require('../helpers/csvValidator');
+const { stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
 
-// The standard string normalizer function
-function stringNormalizer(str) {
-  return str.toLowerCase();
-}
-
-// For translating null/nil-like values into empty strings
-function normalizeEmptyValues(data, unalterableColumns = []) {
-  const EMPTY_VALUES = ['null', 'nil'].map(stringNormalizer);
-  const normalizedUnalterableColumns = unalterableColumns.map(stringNormalizer);
-  // Flag tracking if empty values were normalized or not.
-  let wasEmptyNormalized = false;
-  const newData = data.map((row, i) => {
-    const newRow = { ...row };
-    // Filter out unalterable columns
-    const columnsToNormalize = Object.keys(row).filter((col) => !normalizedUnalterableColumns.includes(stringNormalizer(col)));
-    columnsToNormalize.forEach((col) => {
-      const value = newRow[col];
-      // If the value for this row-col combo is a value that should be empty, replace it
-      if (EMPTY_VALUES.includes(stringNormalizer(value))) {
-        logger.debug(`NULL/NIL values '${value}' found in row-${i}, col-${col}`);
-        wasEmptyNormalized = true;
-        newRow[col] = '';
-      }
-    });
-    return newRow;
-  });
-
-  if (wasEmptyNormalized) {
-    logger.warn('NULL/NIL values found and replaced with empty-strings');
-  }
-  return newData;
-}
-
-class CSVModule {
+class CSVFileModule {
   constructor(csvFilePath, unalterableColumns) {
     // Parse then normalize the data
     const parsedData = parse(fs.readFileSync(csvFilePath), {
       columns: (header) => header.map((column) => stringNormalizer(column)),
       bom: true,
     });
+    this.filePath = csvFilePath;
+
     this.data = normalizeEmptyValues(parsedData, unalterableColumns);
   }
 
   async get(key, value, fromDate, toDate) {
-    logger.debug(`Get csvModule info by key '${key}'`);
+    logger.debug(`Get csvFileModule info by key '${key}'`);
     // return all rows if key and value aren't provided
     if (!key && !value) return this.data;
     let result = this.data.filter((d) => d[stringNormalizer(key)] === value);
@@ -62,8 +33,18 @@ class CSVModule {
     if (result.length === 0) logger.warn('No data for patient within specified time range');
     return result;
   }
+
+  async validate(csvSchema) {
+    if (csvSchema) {
+      logger.info(`Validating CSV file for ${this.filePath}`);
+      return validateCSV(this.filePath, csvSchema, this.data);
+    }
+    logger.warn(`No CSV schema provided for ${this.filePath}`);
+    return true;
+  }
 }
 
+
 module.exports = {
-  CSVModule,
+  CSVFileModule,
 };
diff --git a/src/modules/CSVURLModule.js b/src/modules/CSVURLModule.js
diff --git a/src/modules/index.js b/src/modules/index.js
diff --git a/test/client/BaseClient.test.js b/test/client/BaseClient.test.js
diff --git a/test/helpers/csvParsingUtils.test.js b/test/helpers/csvParsingUtils.test.js
diff --git a/test/modules/CSVFileModule.test.js b/test/modules/CSVFileModule.test.js
diff --git a/test/modules/CSVModule.test.js b/test/modules/CSVModule.test.js

Original file line number	Diff line number	Diff line change
`@@ -69,8 +69,8 @@ function formatData(adverseEventData, patientId) {`
`69`	`69`	`}`
`70`	`70`
`71`	`71`	`class CSVAdverseEventExtractor extends BaseCSVExtractor {`
`72`		`- constructor({ filePath }) {`
`73`		`- super({ filePath });`
	`72`	`+ constructor({ filePath, url }) {`
	`73`	`+ super({ filePath, url });`
`74`	`74`	`}`
`75`	`75`
`76`	`76`	`async getAdverseEventData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -8,8 +8,8 @@ const logger = require('../helpers/logger');`
`8`	`8`	`const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');`
`9`	`9`
`10`	`10`	`class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {`
`11`		`- constructor({ filePath, implementation }) {`
`12`		`- super({ filePath, csvSchema: CSVCancerDiseaseStatusSchema });`
	`11`	`+ constructor({ filePath, url, implementation }) {`
	`12`	`+ super({ filePath, url, csvSchema: CSVCancerDiseaseStatusSchema });`
`13`	`13`	`this.implementation = implementation;`
`14`	`14`	`}`
`15`	`15`
Original file line number	Diff line number	Diff line change
`@@ -46,8 +46,8 @@ function formatData(medicationData, patientId) {`
`46`	`46`	`}`
`47`	`47`
`48`	`48`	`class CSVCancerRelatedMedicationExtractor extends BaseCSVExtractor {`
`49`		`- constructor({ filePath }) {`
`50`		`- super({ filePath });`
	`49`	`+ constructor({ filePath, url }) {`
	`50`	`+ super({ filePath, url });`
`51`	`51`	`}`
`52`	`52`
`53`	`53`	`async getMedicationData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -49,8 +49,8 @@ function formatData(conditionData, patientId) {`
`49`	`49`	`}`
`50`	`50`
`51`	`51`	`class CSVConditionExtractor extends BaseCSVExtractor {`
`52`		`- constructor({ filePath }) {`
`53`		`- super({ filePath, csvSchema: CSVConditionSchema });`
	`52`	`+ constructor({ filePath, url }) {`
	`53`	`+ super({ filePath, url, csvSchema: CSVConditionSchema });`
`54`	`54`	`}`
`55`	`55`
`56`	`56`	`async getConditionData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -42,8 +42,8 @@ function formatData(observationData, patientId) {`
`42`	`42`	`}`
`43`	`43`
`44`	`44`	`class CSVObservationExtractor extends BaseCSVExtractor {`
`45`		`- constructor({ filePath }) {`
`46`		`- super({ filePath });`
	`45`	`+ constructor({ filePath, url }) {`
	`46`	`+ super({ filePath, url });`
`47`	`47`	`}`
`48`	`48`
`49`	`49`	`async getObservationData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -48,8 +48,8 @@ function formatData(procedureData, patientId) {`
`48`	`48`	`}`
`49`	`49`
`50`	`50`	`class CSVProcedureExtractor extends BaseCSVExtractor {`
`51`		`- constructor({ filePath }) {`
`52`		`- super({ filePath });`
	`51`	`+ constructor({ filePath, url }) {`
	`52`	`+ super({ filePath, url });`
`53`	`53`	`}`
`54`	`54`
`55`	`55`	`async getProcedureData(mrn) {`