Initial csvParse option in the config

julianxcarter · julianxcarter · commit 16bc7eaff73e · 2022-01-19T12:08:25.000-05:00
diff --git a/config/csv.config.example.json b/config/csv.config.example.json
@@ -1,7 +1,12 @@
 {
   "patientIdCsvPath": "Users/YourAccount/absolute/path/to/patient-mrns.csv",
   "commonExtractorArgs": {
-    "dataDirectory": "Users/YourAccount/absolute/path/to/data/directory"
+    "dataDirectory": "Users/YourAccount/absolute/path/to/data/directory",
+    "csvParse": {
+      "options": {
+        "bom": true
+      }
+    }
   },
   "notificationInfo": {
     "host": "smtp.example.com",
diff --git a/src/extractors/BaseCSVExtractor.js b/src/extractors/BaseCSVExtractor.js
@@ -6,26 +6,27 @@ const logger = require('../helpers/logger');
 
 class BaseCSVExtractor extends Extractor {
   constructor({
-    filePath, url, fileName, dataDirectory, csvSchema, unalterableColumns,
+    filePath, url, fileName, dataDirectory, csvSchema, unalterableColumns, csvParse
   }) {
     super();
     this.unalterableColumns = unalterableColumns || [];
     this.csvSchema = csvSchema;
+    this.parserOptions = csvParse && csvParse.options ? csvParse.options : {};
     if (url) {
       logger.debug('Found url argument; creating a CSVURLModule with the provided url');
       this.url = url;
-      this.csvModule = new CSVURLModule(this.url, this.unalterableColumns);
+      this.csvModule = new CSVURLModule(this.url, this.unalterableColumns, this.parserOptions);
     } else if (fileName && dataDirectory) {
       if (!path.isAbsolute(dataDirectory)) throw new Error('dataDirectory is not an absolutePath, it needs to be.');
       this.filePath = path.join(dataDirectory, fileName);
       logger.debug(
         'Found fileName and dataDirectory arguments; creating a CSVFileModule with the provided dataDirectory and fileName',
       );
-      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
+      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns, this.parserOptions);
     } else if (filePath) {
       logger.debug('Found filePath argument; creating a CSVFileModule with the provided filePath');
       this.filePath = filePath;
-      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
+      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns, this.parserOptions);
     } else {
       logger.debug(
         'Could not instantiate a CSVExtractor with the provided constructor args',
diff --git a/src/extractors/CSVAdverseEventExtractor.js b/src/extractors/CSVAdverseEventExtractor.js
@@ -69,8 +69,8 @@ function formatData(adverseEventData, patientId) {
 }
 
 class CSVAdverseEventExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({ filePath, url, fileName, dataDirectory, csvParse }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getAdverseEventData(mrn) {
diff --git a/src/extractors/CSVCTCAdverseEventExtractor.js b/src/extractors/CSVCTCAdverseEventExtractor.js
@@ -109,8 +109,8 @@ function formatData(adverseEventData, patientId) {
 }
 
 class CSVCTCAdverseEventExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({ filePath, url, fileName, dataDirectory, csvParse }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getAdverseEventData(mrn) {
diff --git a/src/extractors/CSVCancerDiseaseStatusExtractor.js b/src/extractors/CSVCancerDiseaseStatusExtractor.js
@@ -9,9 +9,9 @@ const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');
 
 class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
   constructor({
-    filePath, url, fileName, dataDirectory, implementation,
+    filePath, url, fileName, dataDirectory, csvParse, implementation
   }) {
-    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVCancerDiseaseStatusSchema });
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVCancerDiseaseStatusSchema, csvParse });
     this.implementation = implementation;
   }
 
diff --git a/src/extractors/CSVCancerRelatedMedicationAdministrationExtractor.js b/src/extractors/CSVCancerRelatedMedicationAdministrationExtractor.js
@@ -46,8 +46,8 @@ function formatData(medicationData, patientId) {
 }
 
 class CSVCancerRelatedMedicationAdministrationExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({ filePath, url, fileName, dataDirectory, csvParse }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getMedicationData(mrn) {
diff --git a/src/extractors/CSVCancerRelatedMedicationRequestExtractor.js b/src/extractors/CSVCancerRelatedMedicationRequestExtractor.js
@@ -48,8 +48,8 @@ function formatData(medicationData, patientId) {
 }
 
 class CSVCancerRelatedMedicationRequestExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({ filePath, url, fileName, dataDirectory, csvParse }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getMedicationData(mrn) {
diff --git a/src/extractors/CSVClinicalTrialInformationExtractor.js b/src/extractors/CSVClinicalTrialInformationExtractor.js
@@ -9,7 +9,7 @@ const { CSVClinicalTrialInformationSchema } = require('../helpers/schemas/csv');
 
 class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
   constructor({
-    filePath, url, fileName, dataDirectory, clinicalSiteID, clinicalSiteSystem,
+    filePath, url, fileName, dataDirectory, csvParse, clinicalSiteID, clinicalSiteSystem
   }) {
     super({ filePath, url, fileName, dataDirectory, csvSchema: CSVClinicalTrialInformationSchema });
     if (!clinicalSiteID) logger.warn(`${this.constructor.name} expects a value for clinicalSiteID but got ${clinicalSiteID}`);
diff --git a/src/extractors/CSVConditionExtractor.js b/src/extractors/CSVConditionExtractor.js
@@ -49,8 +49,8 @@ function formatData(conditionData, patientId) {
 }
 
 class CSVConditionExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema });
+  constructor({ filePath, url, fileName, dataDirectory, csvParse }) {
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema, csvParse });
   }
 
   async getConditionData(mrn) {
diff --git a/src/extractors/CSVObservationExtractor.js b/src/extractors/CSVObservationExtractor.js
@@ -42,8 +42,8 @@ function formatData(observationData, patientId) {
 }
 
 class CSVObservationExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({ filePath, url, fileName, dataDirectory, csvParse }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getObservationData(mrn) {
diff --git a/src/extractors/CSVPatientExtractor.js b/src/extractors/CSVPatientExtractor.js
@@ -55,7 +55,7 @@ function joinAndReformatData(patientData) {
 
 class CSVPatientExtractor extends BaseCSVExtractor {
   constructor({
-    filePath, url, fileName, dataDirectory, mask = [],
+    filePath, url, fileName, dataDirectory, mask = [], csvParse
   }) {
     // Define CSV Columns whose values should never be altered
     const unalterableColumns = ['familyName', 'givenName'];
@@ -66,6 +66,7 @@ class CSVPatientExtractor extends BaseCSVExtractor {
       dataDirectory,
       csvSchema: CSVPatientSchema,
       unalterableColumns,
+      csvParse
     });
     this.mask = mask;
   }
diff --git a/src/extractors/CSVProcedureExtractor.js b/src/extractors/CSVProcedureExtractor.js
@@ -48,8 +48,8 @@ function formatData(procedureData, patientId) {
 }
 
 class CSVProcedureExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({ filePath, url, fileName, dataDirectory, csvParse }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getProcedureData(mrn) {
diff --git a/src/extractors/CSVStagingExtractor.js b/src/extractors/CSVStagingExtractor.js
@@ -63,8 +63,8 @@ function formatStagingData(stagingData, categoryIds, patientId) {
 }
 
 class CSVStagingExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory });
+  constructor({ filePath, url, fileName, dataDirectory, csvParse }) {
+    super({ filePath, url, fileName, dataDirectory, csvParse });
   }
 
   async getStagingData(mrn) {
diff --git a/src/extractors/CSVTreatmentPlanChangeExtractor.js b/src/extractors/CSVTreatmentPlanChangeExtractor.js
@@ -70,8 +70,8 @@ function formatData(tpcData, patientId) {
 }
 
 class CSVTreatmentPlanChangeExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, fileName, dataDirectory }) {
-    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVTreatmentPlanChangeSchema });
+  constructor({ filePath, url, fileName, dataDirectory, csvParse }) {
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVTreatmentPlanChangeSchema, csvParse });
   }
 
   async getTPCData(mrn, fromDate, toDate) {
diff --git a/src/modules/CSVFileModule.js b/src/modules/CSVFileModule.js
@@ -5,9 +5,9 @@ const { validateCSV } = require('../helpers/csvValidator');
 const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
 
 class CSVFileModule {
-  constructor(csvFilePath, unalterableColumns) {
+  constructor(csvFilePath, unalterableColumns, parserOptions) {
     // Parse then normalize the data
-    const parsedData = csvParse(fs.readFileSync(csvFilePath));
+    const parsedData = csvParse(fs.readFileSync(csvFilePath), parserOptions);
     this.filePath = csvFilePath;
     this.data = normalizeEmptyValues(parsedData, unalterableColumns);
   }
diff --git a/src/modules/CSVURLModule.js b/src/modules/CSVURLModule.js
@@ -5,10 +5,11 @@ const { validateCSV } = require('../helpers/csvValidator');
 const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
 
 class CSVURLModule {
-  constructor(url, unalterableColumns) {
+  constructor(url, unalterableColumns, parserOptions) {
     this.unalterableColumns = unalterableColumns;
     this.url = url;
     this.data = undefined;
+    this.parserOptions = parserOptions;
   }
 
   // Ensures that this.data contains normalized CSV data fetched from the module's url
@@ -24,7 +25,7 @@ class CSVURLModule {
         });
       logger.debug('Web request successful');
       // Parse then normalize the data
-      const parsedData = csvParse(csvData);
+      const parsedData = csvParse(csvData, this.parserOptions);
       logger.debug('CSV Data parsing successful');
       this.data = normalizeEmptyValues(parsedData, this.unalterableColumns);
     }

Original file line number	Diff line number	Diff line change
`@@ -69,8 +69,8 @@ function formatData(adverseEventData, patientId) {`
`69`	`69`	`}`
`70`	`70`
`71`	`71`	`class CSVAdverseEventExtractor extends BaseCSVExtractor {`
`72`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`73`		`- super({ filePath, url, fileName, dataDirectory });`
	`72`	`+ constructor({ filePath, url, fileName, dataDirectory, csvParse }) {`
	`73`	`+ super({ filePath, url, fileName, dataDirectory, csvParse });`
`74`	`74`	`}`
`75`	`75`
`76`	`76`	`async getAdverseEventData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -109,8 +109,8 @@ function formatData(adverseEventData, patientId) {`
`109`	`109`	`}`
`110`	`110`
`111`	`111`	`class CSVCTCAdverseEventExtractor extends BaseCSVExtractor {`
`112`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`113`		`- super({ filePath, url, fileName, dataDirectory });`
	`112`	`+ constructor({ filePath, url, fileName, dataDirectory, csvParse }) {`
	`113`	`+ super({ filePath, url, fileName, dataDirectory, csvParse });`
`114`	`114`	`}`
`115`	`115`
`116`	`116`	`async getAdverseEventData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -46,8 +46,8 @@ function formatData(medicationData, patientId) {`
`46`	`46`	`}`
`47`	`47`
`48`	`48`	`class CSVCancerRelatedMedicationAdministrationExtractor extends BaseCSVExtractor {`
`49`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`50`		`- super({ filePath, url, fileName, dataDirectory });`
	`49`	`+ constructor({ filePath, url, fileName, dataDirectory, csvParse }) {`
	`50`	`+ super({ filePath, url, fileName, dataDirectory, csvParse });`
`51`	`51`	`}`
`52`	`52`
`53`	`53`	`async getMedicationData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -48,8 +48,8 @@ function formatData(medicationData, patientId) {`
`48`	`48`	`}`
`49`	`49`
`50`	`50`	`class CSVCancerRelatedMedicationRequestExtractor extends BaseCSVExtractor {`
`51`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`52`		`- super({ filePath, url, fileName, dataDirectory });`
	`51`	`+ constructor({ filePath, url, fileName, dataDirectory, csvParse }) {`
	`52`	`+ super({ filePath, url, fileName, dataDirectory, csvParse });`
`53`	`53`	`}`
`54`	`54`
`55`	`55`	`async getMedicationData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -49,8 +49,8 @@ function formatData(conditionData, patientId) {`
`49`	`49`	`}`
`50`	`50`
`51`	`51`	`class CSVConditionExtractor extends BaseCSVExtractor {`
`52`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`53`		`- super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema });`
	`52`	`+ constructor({ filePath, url, fileName, dataDirectory, csvParse }) {`
	`53`	`+ super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema, csvParse });`
`54`	`54`	`}`
`55`	`55`
`56`	`56`	`async getConditionData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -42,8 +42,8 @@ function formatData(observationData, patientId) {`
`42`	`42`	`}`
`43`	`43`
`44`	`44`	`class CSVObservationExtractor extends BaseCSVExtractor {`
`45`		`- constructor({ filePath, url, fileName, dataDirectory }) {`
`46`		`- super({ filePath, url, fileName, dataDirectory });`
	`45`	`+ constructor({ filePath, url, fileName, dataDirectory, csvParse }) {`
	`46`	`+ super({ filePath, url, fileName, dataDirectory, csvParse });`
`47`	`47`	`}`
`48`	`48`
`49`	`49`	`async getObservationData(mrn) {`