Merge pull request #175 from mcode/config-csv-options

dmendelowitz · web-flow · commit 7155039de0d2 · 2022-01-21T09:48:08.000-05:00
Passthrough csv-parse options supported in the config
diff --git a/README.md b/README.md
@@ -199,6 +199,22 @@ cat -v <file.csv>
 
 If there is an unexpected symbol at the beginning of the file, then there may be a byte order marker that needs to be removed.
 
+#### Troubleshooting Additional Errors
+The mCODE Extraction Framework uses the node `csv-parse` library to parse specified CSV files. [Parsing options for the `csv-parse` library](https://csv.js.org/parse/options/) can be included in the configuration file within the `commonExtractorArgs.csvParse.options` section. For example, the following configuration will pass the `to` option to the `csv-parse` module, causing the mCODE Extraction Framework to only read CSV files up to the specified line number: 
+
+```
+"commonExtractorArgs": {
+    "dataDirectory": "/Users/*****/Documents/dataDirectory",
+    "csvParse": {
+      "options": {
+        "to": 3
+      }
+    }
+  },
+```
+
+**Note:** The mCODE Extraction Framework enables the `bom`, `skip_empty_lines`, and `skip_lines_with_empty_values` options by default, including these options in the configuration file will cause these default options to be overwritten.
+
 ## Terminology and Architecture
 
 This framework consists of three key components: Extractors, Modules and Templates. Below is, in order:
diff --git a/src/application/app.js b/src/application/app.js
@@ -31,7 +31,8 @@ async function mcodeApp(Client, fromDate, toDate, config, pathToRunLogs, debug,
 
   // Parse CSV for list of patient mrns
   const dataDirectory = config.commonExtractorArgs && config.commonExtractorArgs.dataDirectory;
-  const patientIds = parsePatientIds(config.patientIdCsvPath, dataDirectory);
+  const parserOptions = config.commonExtractorArgs && config.commonExtractorArgs.csvParse && config.commonExtractorArgs.csvParse.options ? config.commonExtractorArgs.csvParse.options : {};
+  const patientIds = parsePatientIds(config.patientIdCsvPath, dataDirectory, parserOptions);
 
   // Get RunInstanceLogger for recording new runs and inferring dates from previous runs
   const runLogger = allEntries ? null : new RunInstanceLogger(pathToRunLogs);
diff --git a/src/extractors/BaseCSVExtractor.js b/src/extractors/BaseCSVExtractor.js
@@ -6,26 +6,27 @@ const logger = require('../helpers/logger');
 
 class BaseCSVExtractor extends Extractor {
   constructor({
-    filePath, url, fileName, dataDirectory, csvSchema, unalterableColumns,
+    filePath, url, fileName, dataDirectory, csvSchema, unalterableColumns, csvParse,
   }) {
     super();
     this.unalterableColumns = unalterableColumns || [];
     this.csvSchema = csvSchema;
+    this.parserOptions = csvParse && csvParse.options ? csvParse.options : {};
     if (url) {
       logger.debug('Found url argument; creating a CSVURLModule with the provided url');
       this.url = url;
-      this.csvModule = new CSVURLModule(this.url, this.unalterableColumns);
+      this.csvModule = new CSVURLModule(this.url, this.unalterableColumns, this.parserOptions);
     } else if (fileName && dataDirectory) {
       if (!path.isAbsolute(dataDirectory)) throw new Error('dataDirectory is not an absolutePath, it needs to be.');
       this.filePath = path.join(dataDirectory, fileName);
       logger.debug(
         'Found fileName and dataDirectory arguments; creating a CSVFileModule with the provided dataDirectory and fileName',
       );
-      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
+      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns, this.parserOptions);
     } else if (filePath) {
       logger.debug('Found filePath argument; creating a CSVFileModule with the provided filePath');
       this.filePath = filePath;
-      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
+      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns, this.parserOptions);
     } else {
       logger.debug(
         'Could not instantiate a CSVExtractor with the provided constructor args',
diff --git a/src/extractors/CSVAdverseEventExtractor.js b/src/extractors/CSVAdverseEventExtractor.js
@@ -69,8 +69,10 @@ function formatData(adverseEventData, patientId) {
 }
 
 class CSVAdverseEventExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getAdverseEventData(mrn) {
diff --git a/src/extractors/CSVCTCAdverseEventExtractor.js b/src/extractors/CSVCTCAdverseEventExtractor.js
@@ -109,8 +109,10 @@ function formatData(adverseEventData, patientId) {
 }
 
 class CSVCTCAdverseEventExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getAdverseEventData(mrn) {
diff --git a/src/extractors/CSVCancerDiseaseStatusExtractor.js b/src/extractors/CSVCancerDiseaseStatusExtractor.js
@@ -9,9 +9,9 @@ const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');
 
 class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
   constructor({
-    filePath, url, fileName, dataDirectory, implementation,
+    filePath, url, fileName, dataDirectory, csvParse, implementation,
   }) {
-    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVCancerDiseaseStatusSchema });
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVCancerDiseaseStatusSchema, csvParse });
     this.implementation = implementation;
   }
 
diff --git a/src/extractors/CSVCancerRelatedMedicationAdministrationExtractor.js b/src/extractors/CSVCancerRelatedMedicationAdministrationExtractor.js
@@ -46,8 +46,10 @@ function formatData(medicationData, patientId) {
 }
 
 class CSVCancerRelatedMedicationAdministrationExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getMedicationData(mrn) {
diff --git a/src/extractors/CSVCancerRelatedMedicationRequestExtractor.js b/src/extractors/CSVCancerRelatedMedicationRequestExtractor.js
@@ -48,8 +48,10 @@ function formatData(medicationData, patientId) {
 }
 
 class CSVCancerRelatedMedicationRequestExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getMedicationData(mrn) {
diff --git a/src/extractors/CSVClinicalTrialInformationExtractor.js b/src/extractors/CSVClinicalTrialInformationExtractor.js
@@ -9,9 +9,9 @@ const { CSVClinicalTrialInformationSchema } = require('../helpers/schemas/csv');
 
 class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
   constructor({
-    filePath, url, fileName, dataDirectory, clinicalSiteID, clinicalSiteSystem,
+    filePath, url, fileName, dataDirectory, csvParse, clinicalSiteID, clinicalSiteSystem,
   }) {
-    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVClinicalTrialInformationSchema });
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVClinicalTrialInformationSchema, csvParse });
     if (!clinicalSiteID) logger.warn(`${this.constructor.name} expects a value for clinicalSiteID but got ${clinicalSiteID}`);
     this.clinicalSiteID = clinicalSiteID;
     this.clinicalSiteSystem = clinicalSiteSystem;
diff --git a/src/extractors/CSVConditionExtractor.js b/src/extractors/CSVConditionExtractor.js
@@ -49,8 +49,10 @@ function formatData(conditionData, patientId) {
 }
 
 class CSVConditionExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema, csvParse });
   }
 
   async getConditionData(mrn) {
diff --git a/src/extractors/CSVObservationExtractor.js b/src/extractors/CSVObservationExtractor.js
@@ -42,8 +42,10 @@ function formatData(observationData, patientId) {
 }
 
 class CSVObservationExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getObservationData(mrn) {
diff --git a/src/extractors/CSVPatientExtractor.js b/src/extractors/CSVPatientExtractor.js
@@ -55,7 +55,7 @@ function joinAndReformatData(patientData) {
 
 class CSVPatientExtractor extends BaseCSVExtractor {
   constructor({
-    filePath, url, fileName, dataDirectory, mask = [],
+    filePath, url, fileName, dataDirectory, mask = [], csvParse,
   }) {
     // Define CSV Columns whose values should never be altered
     const unalterableColumns = ['familyName', 'givenName'];
@@ -66,6 +66,7 @@ class CSVPatientExtractor extends BaseCSVExtractor {
       dataDirectory,
       csvSchema: CSVPatientSchema,
       unalterableColumns,
+      csvParse,
     });
     this.mask = mask;
   }
diff --git a/src/extractors/CSVProcedureExtractor.js b/src/extractors/CSVProcedureExtractor.js
@@ -48,8 +48,10 @@ function formatData(procedureData, patientId) {
 }
 
 class CSVProcedureExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getProcedureData(mrn) {
diff --git a/src/extractors/CSVStagingExtractor.js b/src/extractors/CSVStagingExtractor.js
@@ -63,8 +63,10 @@ function formatStagingData(stagingData, categoryIds, patientId) {
 }
 
 class CSVStagingExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getStagingData(mrn) {
diff --git a/src/extractors/CSVTreatmentPlanChangeExtractor.js b/src/extractors/CSVTreatmentPlanChangeExtractor.js
@@ -70,8 +70,10 @@ function formatData(tpcData, patientId) {
 }
 
 class CSVTreatmentPlanChangeExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVTreatmentPlanChangeSchema });
+  constructor({
+    filePath, url, fileName, dataDirectory, csvParse,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVTreatmentPlanChangeSchema, csvParse });
   }
 
   async getTPCData(mrn, fromDate, toDate) {
diff --git a/src/helpers/appUtils.js b/src/helpers/appUtils.js
@@ -25,11 +25,12 @@ function getPatientIdCSVData(patientIdCsvPath, dataDirectory) {
  *
  * @param {string} patientIdCsvPath filePath to the CSV content to be parsed to get IDs
  * @param {string} dataDirectory optional argument for if a dataDirectory was specified by the config
+ * @param {object} parserOptions options for the csv-parse module
  * @returns array of parsed IDs from the CSV
  */
-function parsePatientIds(patientIdCsvPath, dataDirectory) {
+function parsePatientIds(patientIdCsvPath, dataDirectory, parserOptions) {
   const csvData = getPatientIdCSVData(patientIdCsvPath, dataDirectory);
-  return csvParse(csvData).map((row) => {
+  return csvParse(csvData, parserOptions).map((row) => {
     if (!row.mrn) {
       throw new Error(`${patientIdCsvPath} has no "mrn" column`);
     }
diff --git a/src/helpers/schemas/config.schema.json b/src/helpers/schemas/config.schema.json
@@ -40,6 +40,17 @@
         "requestHeaders": {
           "title": "Request Headers",
           "type": "object"
+        },
+        "csvParse" : {
+          "title": "CSV Parse",
+          "type": "object",
+          "properties": {
+            "options": {
+              "titles": "Options",
+              "description": "Options to be passed to the \"CSV Parse\" module used for parsing CSV files.",
+              "type": "object"
+            }
+          }
         }
       }
     },
diff --git a/src/modules/CSVFileModule.js b/src/modules/CSVFileModule.js
@@ -5,9 +5,9 @@ const { validateCSV } = require('../helpers/csvValidator');
 const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
 
 class CSVFileModule {
-  constructor(csvFilePath, unalterableColumns) {
+  constructor(csvFilePath, unalterableColumns, parserOptions) {
     // Parse then normalize the data
-    const parsedData = csvParse(fs.readFileSync(csvFilePath));
+    const parsedData = csvParse(fs.readFileSync(csvFilePath), parserOptions);
     this.filePath = csvFilePath;
     this.data = normalizeEmptyValues(parsedData, unalterableColumns);
   }
diff --git a/src/modules/CSVURLModule.js b/src/modules/CSVURLModule.js
@@ -5,10 +5,11 @@ const { validateCSV } = require('../helpers/csvValidator');
 const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
 
 class CSVURLModule {
-  constructor(url, unalterableColumns) {
+  constructor(url, unalterableColumns, parserOptions) {
     this.unalterableColumns = unalterableColumns;
     this.url = url;
     this.data = undefined;
+    this.parserOptions = parserOptions;
   }
 
   // Ensures that this.data contains normalized CSV data fetched from the module's url
@@ -24,7 +25,7 @@ class CSVURLModule {
         });
       logger.debug('Web request successful');
       // Parse then normalize the data
-      const parsedData = csvParse(csvData);
+      const parsedData = csvParse(csvData, this.parserOptions);
       logger.debug('CSV Data parsing successful');
       this.data = normalizeEmptyValues(parsedData, this.unalterableColumns);
     }

Original file line number	Diff line number	Diff line change
`@@ -69,8 +69,10 @@ function formatData(adverseEventData, patientId) {`
`69`	`69`	`}`
`70`	`70`
`71`	`71`	`class CSVAdverseEventExtractor extends BaseCSVExtractor {`
`72`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`73`		`- super({ filePath, url, fileName, dataDirectory });`
	`72`	`+ constructor({`
	`73`	`+ filePath, url, fileName, dataDirectory, csvParse,`
	`74`	`+ }) {`
	`75`	`+ super({ filePath, url, fileName, dataDirectory, csvParse });`
`74`	`76`	`}`
`75`	`77`
`76`	`78`	`async getAdverseEventData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -109,8 +109,10 @@ function formatData(adverseEventData, patientId) {`
`109`	`109`	`}`
`110`	`110`
`111`	`111`	`class CSVCTCAdverseEventExtractor extends BaseCSVExtractor {`
`112`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`113`		`- super({ filePath, url, fileName, dataDirectory });`
	`112`	`+ constructor({`
	`113`	`+ filePath, url, fileName, dataDirectory, csvParse,`
	`114`	`+ }) {`
	`115`	`+ super({ filePath, url, fileName, dataDirectory, csvParse });`
`114`	`116`	`}`
`115`	`117`
`116`	`118`	`async getAdverseEventData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -46,8 +46,10 @@ function formatData(medicationData, patientId) {`
`46`	`46`	`}`
`47`	`47`
`48`	`48`	`class CSVCancerRelatedMedicationAdministrationExtractor extends BaseCSVExtractor {`
`49`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`50`		`- super({ filePath, url, fileName, dataDirectory });`
	`49`	`+ constructor({`
	`50`	`+ filePath, url, fileName, dataDirectory, csvParse,`
	`51`	`+ }) {`
	`52`	`+ super({ filePath, url, fileName, dataDirectory, csvParse });`
`51`	`53`	`}`
`52`	`54`
`53`	`55`	`async getMedicationData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -48,8 +48,10 @@ function formatData(medicationData, patientId) {`
`48`	`48`	`}`
`49`	`49`
`50`	`50`	`class CSVCancerRelatedMedicationRequestExtractor extends BaseCSVExtractor {`
`51`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`52`		`- super({ filePath, url, fileName, dataDirectory });`
	`51`	`+ constructor({`
	`52`	`+ filePath, url, fileName, dataDirectory, csvParse,`
	`53`	`+ }) {`
	`54`	`+ super({ filePath, url, fileName, dataDirectory, csvParse });`
`53`	`55`	`}`
`54`	`56`
`55`	`57`	`async getMedicationData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -49,8 +49,10 @@ function formatData(conditionData, patientId) {`
`49`	`49`	`}`
`50`	`50`
`51`	`51`	`class CSVConditionExtractor extends BaseCSVExtractor {`
`52`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`53`		`- super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema });`
	`52`	`+ constructor({`
	`53`	`+ filePath, url, fileName, dataDirectory, csvParse,`
	`54`	`+ }) {`
	`55`	`+ super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema, csvParse });`
`54`	`56`	`}`
`55`	`57`
`56`	`58`	`async getConditionData(mrn) {`