CSV Validation

* Include schemas and tools for validating CSVs
* Add BaseCSVExtractor class that all CSV extractors extend
* Update excel sheet to indicate all fields used
* Update tests to use proper calls

Author: Matthew Gramigna

docs/CSV_Templates.xlsx

@@ -22,6 +22,7 @@
     "ajv": "^6.12.6",
     "antlr4": "4.8.0",
     "commander": "^6.2.0",
+    "csv-file-validator": "^1.10.1",
     "csv-parse": "^4.8.8",
     "fhir-crud-client": "^1.2.2",
     "fhirpath": "2.1.5",

package-lock.json

@@ -25,18 +25,30 @@ class BaseClient {
   }
 
   // Given an extractor configuration, initialize all the necessary extractors
-  initializeExtractors(extractorConfig, commonExtractorArgs) {
-    extractorConfig.forEach((curExtractorConfig) => {
+  async initializeExtractors(extractorConfig, commonExtractorArgs) {
+    const extractorInits = extractorConfig.map(async (curExtractorConfig) => {
       const { label, type, constructorArgs } = curExtractorConfig;
       logger.debug(`Initializing ${label} extractor with type ${type}`);
       const ExtractorClass = this.extractorClasses[type];
+
       try {
         const newExtractor = new ExtractorClass({ ...commonExtractorArgs, ...constructorArgs });
-        this.extractors.push(newExtractor);
+
+        if (newExtractor.validate) {
+          await newExtractor.validate();
+        }
+
+        return newExtractor;
       } catch (e) {
         throw new Error(`Unable to initialize ${label} extractor with type ${type}`);
       }
     });
+
+    await Promise.all(extractorInits).then((extractors) => {
+      this.extractors.push(...extractors);
+    });
+
+    logger.info('Validation succeeded');
   }
 
   // NOTE: Async because in other clients that extend this, we need async helper functions (ex. auth)

package.json

@@ -0,0 +1,27 @@
+const path = require('path');
+const { Extractor } = require('./Extractor');
+const { CSVModule } = require('../modules');
+const { validateCSV } = require('../helpers/csvValidator');
+const logger = require('../helpers/logger');
+
+class BaseCSVExtractor extends Extractor {
+  constructor({ filePath, csvSchema }) {
+    super();
+    this.csvSchema = csvSchema;
+    this.filePath = path.resolve(filePath);
+    this.csvModule = new CSVModule(this.filePath);
+  }
+
+  async validate() {
+    if (this.csvSchema) {
+      logger.info(`Validating CSV file for ${this.filePath}`);
+      await validateCSV(this.filePath, this.csvSchema);
+    } else {
+      logger.warn(`No CSV schema provided for ${this.filePath}`);
+    }
+  }
+}
+
+module.exports = {
+  BaseCSVExtractor,
+};

src/client/BaseClient.js

@@ -1,7 +1,5 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const { Extractor } = require('./Extractor');
 const logger = require('../helpers/logger');
 const { formatDateTime } = require('../helpers/dateUtils');
 
@@ -53,15 +51,14 @@ function formatData(adverseEventData) {
   });
 }
 
-class CSVAdverseEventExtractor extends Extractor {
+class CSVAdverseEventExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.CSVModule = new CSVModule(path.resolve(filePath));
+    super({ filePath });
   }
 
   async getAdverseEventData(mrn) {
     logger.debug('Getting Adverse Event Data');
-    return this.CSVModule.get('mrn', mrn);
+    return this.csvModule.get('mrn', mrn);
   }
 
   async get({ mrn }) {

src/extractors/BaseCSVExtractor.js

@@ -1,14 +1,14 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { formatDateTime } = require('../helpers/dateUtils');
 const { getDiseaseStatusDisplay, getDiseaseStatusEvidenceDisplay } = require('../helpers/diseaseStatusUtils');
 const { generateMcodeResources } = require('../templates');
 const { getEmptyBundle } = require('../helpers/fhirUtils');
 const logger = require('../helpers/logger');
+const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');
 
-class CSVCancerDiseaseStatusExtractor {
+class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
   constructor({ filePath, implementation }) {
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath, csvSchema: CSVCancerDiseaseStatusSchema });
     this.implementation = implementation;
   }
 

src/extractors/CSVAdverseEventExtractor.js

@@ -1,7 +1,5 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const { Extractor } = require('./Extractor');
 const logger = require('../helpers/logger');
 const { formatDateTime } = require('../helpers/dateUtils');
 
@@ -35,10 +33,9 @@ function formatData(medicationData) {
   });
 }
 
-class CSVCancerRelatedMedicationExtractor extends Extractor {
+class CSVCancerRelatedMedicationExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath });
   }
 
   async getMedicationData(mrn) {

src/extractors/CSVCancerDiseaseStatusExtractor.js

@@ -1,9 +1,8 @@
-const path = require('path');
-const { Extractor } = require('./Extractor');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { firstEntryInBundle, getBundleResourcesByType } = require('../helpers/fhirUtils');
 const { generateMcodeResources } = require('../templates');
 const logger = require('../helpers/logger');
+const { CSVClinicalTrialInformationSchema } = require('../helpers/schemas/csv');
 
 function getPatientId(context) {
   const patientInContext = getBundleResourcesByType(context, 'Patient', {}, true);
@@ -16,10 +15,9 @@ function getPatientId(context) {
   return undefined;
 }
 
-class CSVClinicalTrialInformationExtractor extends Extractor {
+class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
   constructor({ filePath, clinicalSiteID }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath, csvSchema: CSVClinicalTrialInformationSchema });
     if (!clinicalSiteID) logger.warn(`${this.constructor.name} expects a value for clinicalSiteID but got ${clinicalSiteID}`);
     this.clinicalSiteID = clinicalSiteID;
   }

src/extractors/CSVCancerRelatedMedicationExtractor.js

@@ -1,9 +1,8 @@
-const path = require('path');
-const { CSVModule } = require('../modules');
+const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const { Extractor } = require('./Extractor');
 const logger = require('../helpers/logger');
 const { formatDateTime } = require('../helpers/dateUtils');
+const { CSVConditionSchema } = require('../helpers/schemas/csv');
 
 // Formats data to be passed into template-friendly format
 function formatData(conditionData) {
@@ -37,10 +36,9 @@ function formatData(conditionData) {
   });
 }
 
-class CSVConditionExtractor extends Extractor {
+class CSVConditionExtractor extends BaseCSVExtractor {
   constructor({ filePath }) {
-    super();
-    this.csvModule = new CSVModule(path.resolve(filePath));
+    super({ filePath, csvSchema: CSVConditionSchema });
   }
 
   async getConditionData(mrn) {