Merge pull request #91 from mcode/csv-validation

mgramigna · web-flow · commit 1bbbbe40b18e · 2021-03-17T10:06:41.000-04:00
CSV Validation
diff --git a/docs/CSV_Templates.xlsx b/docs/CSV_Templates.xlsx
diff --git a/src/client/BaseClient.js b/src/client/BaseClient.js
@@ -26,17 +26,37 @@ class BaseClient {
 
   // Given an extractor configuration, initialize all the necessary extractors
   initializeExtractors(extractorConfig, commonExtractorArgs) {
+    let allExtractorsValid = true;
+
     extractorConfig.forEach((curExtractorConfig) => {
       const { label, type, constructorArgs } = curExtractorConfig;
       logger.debug(`Initializing ${label} extractor with type ${type}`);
       const ExtractorClass = this.extractorClasses[type];
+
       try {
         const newExtractor = new ExtractorClass({ ...commonExtractorArgs, ...constructorArgs });
+
+        if (newExtractor.validate) {
+          const isExtractorValid = newExtractor.validate();
+          allExtractorsValid = (allExtractorsValid && isExtractorValid);
+          if (isExtractorValid) {
+            logger.debug(`Extractor ${label} PASSED CSV validation`);
+          } else {
+            logger.debug(`Extractor ${label} FAILED CSV validation`);
+          }
+        }
+
         this.extractors.push(newExtractor);
       } catch (e) {
-        throw new Error(`Unable to initialize ${label} extractor with type ${type}`);
+        throw new Error(`Unable to initialize ${label} extractor with type ${type}: ${e.message}`);
       }
     });
+
+    if (allExtractorsValid) {
+      logger.info('Validation succeeded');
+    } else {
+      throw new Error('Error occurred during CSV validation');
+    }
   }
 
   // NOTE: Async because in other clients that extend this, we need async helper functions (ex. auth)
diff --git a/src/extractors/BaseCSVExtractor.js b/src/extractors/BaseCSVExtractor.js
@@ -0,0 +1,27 @@
+const path = require('path');
+const { Extractor } = require('./Extractor');
+const { CSVModule } = require('../modules');
+const { validateCSV } = require('../helpers/csvValidator');
+const logger = require('../helpers/logger');
+
+class BaseCSVExtractor extends Extractor {
+  constructor({ filePath, csvSchema }) {
+    super();
+    this.csvSchema = csvSchema;
+    this.filePath = path.resolve(filePath);
+    this.csvModule = new CSVModule(this.filePath);
+  }
+
+  validate() {
+    if (this.csvSchema) {
+      logger.info(`Validating CSV file for ${this.filePath}`);
+      return validateCSV(this.filePath, this.csvSchema, this.csvModule.data);
+    }
+    logger.warn(`No CSV schema provided for ${this.filePath}`);
+    return true;
+  }
+}
+
+module.exports = {
+  BaseCSVExtractor,
+};
diff --git a/src/extractors/CSVAdverseEventExtractor.js b/src/extractors/CSVAdverseEventExtractor.js
@@ -1,7 +1,5 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const { Extractor } = require('./Extractor');
 const logger = require('../helpers/logger');
 const { formatDateTime } = require('../helpers/dateUtils');
 
@@ -53,15 +51,14 @@ function formatData(adverseEventData) {
   });
 }
 
-class CSVAdverseEventExtractor extends Extractor {
+class CSVAdverseEventExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.CSVModule = new CSVModule(path.resolve(filePath));
+    super({ filePath });
   }
 
   async getAdverseEventData(mrn) {
     logger.debug('Getting Adverse Event Data');
-    return this.CSVModule.get('mrn', mrn);
+    return this.csvModule.get('mrn', mrn);
   }
 
   async get({ mrn }) {
diff --git a/src/extractors/CSVCancerDiseaseStatusExtractor.js b/src/extractors/CSVCancerDiseaseStatusExtractor.js
@@ -1,14 +1,14 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { formatDateTime } = require('../helpers/dateUtils');
 const { getDiseaseStatusDisplay, getDiseaseStatusEvidenceDisplay } = require('../helpers/diseaseStatusUtils');
 const { generateMcodeResources } = require('../templates');
 const { getEmptyBundle } = require('../helpers/fhirUtils');
 const logger = require('../helpers/logger');
+const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');
 
-class CSVCancerDiseaseStatusExtractor {
+class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
   constructor({ filePath, implementation }) {
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath, csvSchema: CSVCancerDiseaseStatusSchema });
     this.implementation = implementation;
   }
 
diff --git a/src/extractors/CSVCancerRelatedMedicationExtractor.js b/src/extractors/CSVCancerRelatedMedicationExtractor.js
@@ -1,7 +1,5 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const { Extractor } = require('./Extractor');
 const logger = require('../helpers/logger');
 const { formatDateTime } = require('../helpers/dateUtils');
 
@@ -35,10 +33,9 @@ function formatData(medicationData) {
   });
 }
 
-class CSVCancerRelatedMedicationExtractor extends Extractor {
+class CSVCancerRelatedMedicationExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath });
   }
 
   async getMedicationData(mrn) {
diff --git a/src/extractors/CSVClinicalTrialInformationExtractor.js b/src/extractors/CSVClinicalTrialInformationExtractor.js
@@ -1,9 +1,8 @@
-const path = require('path');
-const { Extractor } = require('./Extractor');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { firstEntryInBundle, getBundleResourcesByType } = require('../helpers/fhirUtils');
 const { generateMcodeResources } = require('../templates');
 const logger = require('../helpers/logger');
+const { CSVClinicalTrialInformationSchema } = require('../helpers/schemas/csv');
 
 function getPatientId(context) {
   const patientInContext = getBundleResourcesByType(context, 'Patient', {}, true);
@@ -16,10 +15,9 @@ function getPatientId(context) {
   return undefined;
 }
 
-class CSVClinicalTrialInformationExtractor extends Extractor {
+class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
   constructor({ filePath, clinicalSiteID }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath, csvSchema: CSVClinicalTrialInformationSchema });
     if (!clinicalSiteID) logger.warn(`${this.constructor.name} expects a value for clinicalSiteID but got ${clinicalSiteID}`);
     this.clinicalSiteID = clinicalSiteID;
   }
diff --git a/src/extractors/CSVConditionExtractor.js b/src/extractors/CSVConditionExtractor.js
@@ -1,9 +1,8 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const { Extractor } = require('./Extractor');
 const logger = require('../helpers/logger');
 const { formatDateTime } = require('../helpers/dateUtils');
+const { CSVConditionSchema } = require('../helpers/schemas/csv');
 
 // Formats data to be passed into template-friendly format
 function formatData(conditionData) {
@@ -37,10 +36,9 @@ function formatData(conditionData) {
   });
 }
 
-class CSVConditionExtractor extends Extractor {
+class CSVConditionExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath, csvSchema: CSVConditionSchema });
   }
 
   async getConditionData(mrn) {
diff --git a/src/extractors/CSVObservationExtractor.js b/src/extractors/CSVObservationExtractor.js
@@ -1,7 +1,5 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const { Extractor } = require('./Extractor');
 const logger = require('../helpers/logger');
 const { formatDateTime } = require('../helpers/dateUtils');
 
@@ -32,10 +30,9 @@ function formatData(observationData) {
   });
 }
 
-class CSVObservationExtractor extends Extractor {
+class CSVObservationExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath });
   }
 
   async getObservationData(mrn) {
diff --git a/src/extractors/CSVPatientExtractor.js b/src/extractors/CSVPatientExtractor.js
@@ -1,11 +1,10 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
 const { generateMcodeResources } = require('../templates');
-const { Extractor } = require('./Extractor');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { getEthnicityDisplay,
   getRaceCodesystem,
   getRaceDisplay } = require('../helpers/patientUtils');
 const logger = require('../helpers/logger');
+const { CSVPatientSchema } = require('../helpers/schemas/csv');
 
 function joinAndReformatData(patientData) {
   logger.debug('Reformatting patient data from CSV into template format');
@@ -39,10 +38,9 @@ function joinAndReformatData(patientData) {
   };
 }
 
-class CSVPatientExtractor extends Extractor {
+class CSVPatientExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath, csvSchema: CSVPatientSchema });
   }
 
   async getPatientData(mrn) {
diff --git a/src/extractors/CSVProcedureExtractor.js b/src/extractors/CSVProcedureExtractor.js
@@ -1,7 +1,5 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const { Extractor } = require('./Extractor');
 const logger = require('../helpers/logger');
 const { formatDateTime } = require('../helpers/dateUtils');
 
@@ -35,10 +33,9 @@ function formatData(procedureData) {
   });
 }
 
-class CSVProcedureExtractor extends Extractor {
+class CSVProcedureExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath });
   }
 
   async getProcedureData(mrn) {
diff --git a/src/extractors/CSVStagingExtractor.js b/src/extractors/CSVStagingExtractor.js
@@ -1,6 +1,4 @@
-const path = require('path');
-const { Extractor } = require('./Extractor');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { firstEntryInBundle } = require('../helpers/fhirUtils');
 const { generateMcodeResources } = require('../templates');
 const logger = require('../helpers/logger');
@@ -51,10 +49,9 @@ function formatStagingData(stagingData, categoryIds) {
   };
 }
 
-class CSVStagingExtractor extends Extractor {
+class CSVStagingExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath });
   }
 
   async getStagingData(mrn) {
diff --git a/src/extractors/CSVTreatmentPlanChangeExtractor.js b/src/extractors/CSVTreatmentPlanChangeExtractor.js
@@ -1,11 +1,10 @@
-const path = require('path');
 const _ = require('lodash');
-const { Extractor } = require('./Extractor');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { formatDate } = require('../helpers/dateUtils');
 const { generateMcodeResources } = require('../templates');
 const { getEmptyBundle } = require('../helpers/fhirUtils');
 const logger = require('../helpers/logger');
+const { CSVTreatmentPlanChangeSchema } = require('../helpers/schemas/csv');
 
 // Formats data to be passed into template-friendly format
 function formatData(tpcData) {
@@ -68,10 +67,9 @@ function formatData(tpcData) {
   return [combinedData];
 }
 
-class CSVTreatmentPlanChangeExtractor extends Extractor {
+class CSVTreatmentPlanChangeExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath, csvSchema: CSVTreatmentPlanChangeSchema });
   }
 
   async getTPCData(mrn, fromDate, toDate) {
diff --git a/src/helpers/csvValidator.js b/src/helpers/csvValidator.js
@@ -0,0 +1,45 @@
+const _ = require('lodash');
+const logger = require('./logger');
+
+function validateCSV(pathToCSVFile, csvSchema, csvData) {
+  let isValid = true;
+
+  // Check headers
+  const headers = Object.keys(csvData[0]);
+  const schemaDiff = _.difference(csvSchema.headers.map((h) => h.name), headers);
+  const fileDiff = _.difference(headers, csvSchema.headers.map((h) => h.name));
+
+  if (fileDiff.length > 0) {
+    logger.warn(`Found extra column(s) in CSV ${pathToCSVFile}: "${fileDiff.join(',')}"`);
+  }
+
+  if (schemaDiff.length > 0) {
+    schemaDiff.forEach((sd) => {
+      const headerSchema = csvSchema.headers.find((h) => h.name === sd);
+      if (headerSchema.required) {
+        logger.error(`Column ${sd} is marked as required but is missing in CSV ${pathToCSVFile}`);
+        isValid = false;
+      } else {
+        logger.warn(`Column ${sd} is missing in CSV ${pathToCSVFile}`);
+      }
+    });
+  }
+
+  // Check values
+  csvData.forEach((row, i) => {
+    Object.entries(row).forEach(([key, value], j) => {
+      const schema = csvSchema.headers.find((h) => h.name === key);
+
+      if (schema && schema.required && !value) {
+        logger.error(`Column ${key} marked as required but missing value in row ${i + 1} column ${j + 1} in CSV ${pathToCSVFile}`);
+        isValid = false;
+      }
+    });
+  });
+
+  return isValid;
+}
+
+module.exports = {
+  validateCSV,
+};
diff --git a/src/helpers/schemas/csv.js b/src/helpers/schemas/csv.js
diff --git a/test/cli/mcodeExtraction.test.js b/test/cli/mcodeExtraction.test.js
diff --git a/test/extractors/CSVAdverseEventExtractor.test.js b/test/extractors/CSVAdverseEventExtractor.test.js
diff --git a/test/helpers/csvValidator.test.js b/test/helpers/csvValidator.test.js
