Initial addition of data-directory argument to config

Dtphelan1 · Dtphelan1 · commit e840e30b0124 · 2021-12-14T15:26:04.000-05:00
diff --git a/config/csv.config.example.json b/config/csv.config.example.json
@@ -1,6 +1,8 @@
 {
-  "patientIdCsvPath": "./data/patient-mrns.csv",
-  "commonExtractorArgs": {},
+  "patientIdCsvPath": "~/absolute/path/to/patient-mrns.csv",
+  "commonExtractorArgs": {
+    "dataDirectory": "~/absolute/path/to/data/directory"
+  },
   "notificationInfo": {
     "host": "smtp.example.com",
     "port": 587,
@@ -16,85 +18,85 @@
       "label": "patient",
       "type": "CSVPatientExtractor",
       "constructorArgs": {
-        "filePath": "./data/patient-information.csv"
+        "fileName": "patient-information.csv"
       }
     },
     {
       "label": "condition",
       "type": "CSVConditionExtractor",
       "constructorArgs": {
-        "filePath": "./data/condition-information.csv"
+        "fileName": "condition-information.csv"
       }
     },
     {
       "label": "cancerDiseaseStatus",
       "type": "CSVCancerDiseaseStatusExtractor",
       "constructorArgs": {
-        "filePath": "./data/cancer-disease-status-information.csv"
+        "fileName": "cancer-disease-status-information.csv"
       }
     },
     {
       "label": "clinicalTrialInformation",
       "type": "CSVClinicalTrialInformationExtractor",
       "constructorArgs": {
-        "filePath": "./data/clinical-trial-information.csv",
+        "fileName": "clinical-trial-information.csv",
         "clinicalSiteID": "example-site-id"
       }
     },
     {
       "label": "treatmentPlanChange",
       "type": "CSVTreatmentPlanChangeExtractor",
       "constructorArgs": {
-        "filePath": "./data/treatment-plan-change-information.csv"
+        "fileName": "treatment-plan-change-information.csv"
       }
     },
     {
       "label": "staging",
       "type": "CSVStagingExtractor",
       "constructorArgs": {
-        "filePath": "./data/staging-information.csv"
+        "fileName": "staging-information.csv"
       }
     },
     {
       "label": "cancerRelatedMedicationAdministration",
       "type": "CSVCancerRelatedMedicationAdministrationExtractor",
       "constructorArgs": {
-        "filePath": "./data/cancer-related-medication-administration-information.csv"
+        "fileName": "cancer-related-medication-administration-information.csv"
       }
     },
     {
       "label": "cancerRelatedMedicationRequest",
       "type": "CSVCancerRelatedMedicationRequestExtractor",
       "constructorArgs": {
-        "filePath": "./data/cancer-related-medication-request-information.csv"
+        "fileName": "cancer-related-medication-request-information.csv"
       }
     },
     {
       "label": "genericObservations",
       "type": "CSVObservationExtractor",
       "constructorArgs": {
-        "filePath": "./data/observation-information.csv"
+        "fileName": "observation-information.csv"
       }
     },
     {
       "label": "genericProcedures",
       "type": "CSVProcedureExtractor",
       "constructorArgs": {
-        "filePath": "./data/procedure-information.csv"
+        "fileName": "procedure-information.csv"
       }
     },
     {
       "label": "adverseEvent",
       "type": "CSVAdverseEventExtractor",
       "constructorArgs": {
-        "filePath": "./test/sample-client-data/adverse-event-information.csv"
+        "fileName": "adverse-event-information.csv"
       }
     },
     {
       "label": "ctcAdverseEvent",
       "type": "CSVCTCAdverseEventExtractor",
       "constructorArgs": {
-        "filePath": "./test/sample-client-data/ctc-adverse-event-information.csv"
+        "fileName": "ctc-adverse-event-information.csv"
       }
     }
   ]
diff --git a/src/extractors/BaseCSVExtractor.js b/src/extractors/BaseCSVExtractor.js
@@ -1,19 +1,35 @@
+const path = require('path');
 const { Extractor } = require('./Extractor');
 const { CSVFileModule, CSVURLModule } = require('../modules');
+const logger = require('../helpers/logger');
+
 
 class BaseCSVExtractor extends Extractor {
-  constructor({ filePath, url, csvSchema, unalterableColumns }) {
+  constructor({
+    filePath, url, fileName, dataDirectory, csvSchema, unalterableColumns,
+  }) {
     super();
     this.unalterableColumns = unalterableColumns || [];
     this.csvSchema = csvSchema;
     if (url) {
+      logger.debug('Found url argument; creating a CSVURLModule with the provided url');
       this.url = url;
       this.csvModule = new CSVURLModule(this.url, this.unalterableColumns);
+    } else if (fileName && dataDirectory) {
+      this.filePath = path.join(dataDirectory, fileName);
+      logger.debug(
+        'Found fileName and dataDirectory arguments; creating a CSVFileModule with the provided dataDirectory and fileName',
+      );
+      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
     } else if (filePath) {
+      logger.debug('Found filePath argument; creating a CSVFileModule with the provided filePath');
       this.filePath = filePath;
       this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
     } else {
-      throw new Error('Trying to instantiate a CSVExtractor without a filePath or url');
+      logger.debug(
+        'Could not instantiate a CSVExtractor with the provided constructor args',
+      );
+      throw new Error('Trying to instantiate a CSVExtractor without a filePath, url, or fileName+dataDirectory combination');
     }
   }
 
diff --git a/src/extractors/CSVAdverseEventExtractor.js b/src/extractors/CSVAdverseEventExtractor.js
@@ -69,8 +69,8 @@ function formatData(adverseEventData, patientId) {
 }
 
 class CSVAdverseEventExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url }) {
-    super({ filePath, url });
+  constructor({ filePath, url, fileName, dataDirectory }) {
+    super({ filePath, url, fileName, dataDirectory });
   }
 
   async getAdverseEventData(mrn) {
diff --git a/src/extractors/CSVCTCAdverseEventExtractor.js b/src/extractors/CSVCTCAdverseEventExtractor.js
@@ -95,8 +95,8 @@ function formatData(adverseEventData, patientId) {
 }
 
 class CSVCTCAdverseEventExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url }) {
-    super({ filePath, url });
+  constructor({ filePath, url, fileName, dataDirectory }) {
+    super({ filePath, url, fileName, dataDirectory });
   }
 
   async getAdverseEventData(mrn) {
diff --git a/src/extractors/CSVCancerDiseaseStatusExtractor.js b/src/extractors/CSVCancerDiseaseStatusExtractor.js
@@ -8,8 +8,10 @@ const logger = require('../helpers/logger');
 const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');
 
 class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, implementation }) {
-    super({ filePath, url, csvSchema: CSVCancerDiseaseStatusSchema });
+  constructor({
+    filePath, url, fileName, dataDirectory, implementation,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVCancerDiseaseStatusSchema });
     this.implementation = implementation;
   }
 
diff --git a/src/extractors/CSVCancerRelatedMedicationAdministrationExtractor.js b/src/extractors/CSVCancerRelatedMedicationAdministrationExtractor.js
@@ -46,8 +46,8 @@ function formatData(medicationData, patientId) {
 }
 
 class CSVCancerRelatedMedicationAdministrationExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url }) {
-    super({ filePath, url });
+  constructor({ filePath, url, fileName, dataDirectory }) {
+    super({ filePath, url, fileName, dataDirectory });
   }
 
   async getMedicationData(mrn) {
diff --git a/src/extractors/CSVCancerRelatedMedicationRequestExtractor.js b/src/extractors/CSVCancerRelatedMedicationRequestExtractor.js
@@ -48,8 +48,8 @@ function formatData(medicationData, patientId) {
 }
 
 class CSVCancerRelatedMedicationRequestExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url }) {
-    super({ filePath, url });
+  constructor({ filePath, url, fileName, dataDirectory }) {
+    super({ filePath, url, fileName, dataDirectory });
   }
 
   async getMedicationData(mrn) {
diff --git a/src/extractors/CSVClinicalTrialInformationExtractor.js b/src/extractors/CSVClinicalTrialInformationExtractor.js
@@ -8,8 +8,10 @@ const { CSVClinicalTrialInformationSchema } = require('../helpers/schemas/csv');
 
 
 class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, clinicalSiteID, clinicalSiteSystem }) {
-    super({ filePath, url, csvSchema: CSVClinicalTrialInformationSchema });
+  constructor({
+    filePath, url, fileName, dataDirectory, clinicalSiteID, clinicalSiteSystem,
+  }) {
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVClinicalTrialInformationSchema });
     if (!clinicalSiteID) logger.warn(`${this.constructor.name} expects a value for clinicalSiteID but got ${clinicalSiteID}`);
     this.clinicalSiteID = clinicalSiteID;
     this.clinicalSiteSystem = clinicalSiteSystem;
diff --git a/src/extractors/CSVConditionExtractor.js b/src/extractors/CSVConditionExtractor.js
@@ -49,8 +49,8 @@ function formatData(conditionData, patientId) {
 }
 
 class CSVConditionExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url }) {
-    super({ filePath, url, csvSchema: CSVConditionSchema });
+  constructor({ filePath, url, fileName, dataDirectory }) {
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema });
   }
 
   async getConditionData(mrn) {
diff --git a/src/extractors/CSVObservationExtractor.js b/src/extractors/CSVObservationExtractor.js
@@ -42,8 +42,8 @@ function formatData(observationData, patientId) {
 }
 
 class CSVObservationExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url }) {
-    super({ filePath, url });
+  constructor({ filePath, url, fileName, dataDirectory }) {
+    super({ filePath, url, fileName, dataDirectory });
   }
 
   async getObservationData(mrn) {
diff --git a/src/extractors/CSVPatientExtractor.js b/src/extractors/CSVPatientExtractor.js
@@ -54,12 +54,16 @@ function joinAndReformatData(patientData) {
 }
 
 class CSVPatientExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url, mask = [] }) {
+  constructor({
+    filePath, url, fileName, dataDirectory, mask = [],
+  }) {
     // Define CSV Columns whose values should never be altered
     const unalterableColumns = ['familyName', 'givenName'];
     super({
       filePath,
       url,
+      fileName,
+      dataDirectory,
       csvSchema: CSVPatientSchema,
       unalterableColumns,
     });
diff --git a/src/extractors/CSVProcedureExtractor.js b/src/extractors/CSVProcedureExtractor.js
@@ -48,8 +48,8 @@ function formatData(procedureData, patientId) {
 }
 
 class CSVProcedureExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url }) {
-    super({ filePath, url });
+  constructor({ filePath, url, fileName, dataDirectory }) {
+    super({ filePath, url, fileName, dataDirectory });
   }
 
   async getProcedureData(mrn) {
diff --git a/src/extractors/CSVStagingExtractor.js b/src/extractors/CSVStagingExtractor.js
@@ -63,8 +63,8 @@ function formatStagingData(stagingData, categoryIds, patientId) {
 }
 
 class CSVStagingExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url }) {
-    super({ filePath, url });
+  constructor({ filePath, url, fileName, dataDirectory }) {
+    super({ filePath, url, fileName, dataDirectory });
   }
 
   async getStagingData(mrn) {
diff --git a/src/extractors/CSVTreatmentPlanChangeExtractor.js b/src/extractors/CSVTreatmentPlanChangeExtractor.js
@@ -70,8 +70,8 @@ function formatData(tpcData, patientId) {
 }
 
 class CSVTreatmentPlanChangeExtractor extends BaseCSVExtractor {
-  constructor({ filePath, url }) {
-    super({ filePath, url, csvSchema: CSVTreatmentPlanChangeSchema });
+  constructor({ filePath, url, fileName, dataDirectory }) {
+    super({ filePath, url, fileName, dataDirectory, csvSchema: CSVTreatmentPlanChangeSchema });
   }
 
   async getTPCData(mrn, fromDate, toDate) {
diff --git a/src/helpers/schemas/config.schema.json b/src/helpers/schemas/config.schema.json
@@ -32,6 +32,11 @@
           "type": "string",
           "format": "uri"
         },
+        "dataDirectory": {
+          "title": "Data Directory",
+          "description": "Absolute path for the directory containing any data files used by extractors.",
+          "type": "string"
+        },
         "requestHeaders": {
           "title": "Request Headers",
           "type": "object"
@@ -110,6 +115,17 @@
       "properties": {
         "filePath": {
           "title": "File Path",
+          "description": "An absolute path to a data file from which data is to be extracted",
+          "type": "string"
+        },
+        "fileName": {
+          "title": "File Name",
+          "description": "The name of a file, with the appropriate file suffix, from which data is to be extracted. This fileName is combined with the extractor's dataDictionary to create a valid path.",
+          "type": "string"
+        },
+        "dataDirectory": {
+          "title": "Data Directory",
+          "description": "Absolute path for the directory containing any data files used by extractors. Extractors can infer this value from the global commonExtractorArgs or define their own.",
           "type": "string"
         },
         "url": {

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,8 @@`
`1`	`1`	`{`
`2`		`- "patientIdCsvPath": "./data/patient-mrns.csv",`
`3`		`- "commonExtractorArgs": {},`
	`2`	`+ "patientIdCsvPath": "~/absolute/path/to/patient-mrns.csv",`
	`3`	`+ "commonExtractorArgs": {`
	`4`	`+ "dataDirectory": "~/absolute/path/to/data/directory"`
	`5`	`+ },`
`4`	`6`	`"notificationInfo": {`
`5`	`7`	`"host": "smtp.example.com",`
`6`	`8`	`"port": 587,`
`@@ -16,85 +18,85 @@`
`16`	`18`	`"label": "patient",`
`17`	`19`	`"type": "CSVPatientExtractor",`
`18`	`20`	`"constructorArgs": {`
`19`		`- "filePath": "./data/patient-information.csv"`
	`21`	`+ "fileName": "patient-information.csv"`
`20`	`22`	`}`
`21`	`23`	`},`
`22`	`24`	`{`
`23`	`25`	`"label": "condition",`
`24`	`26`	`"type": "CSVConditionExtractor",`
`25`	`27`	`"constructorArgs": {`
`26`		`- "filePath": "./data/condition-information.csv"`
	`28`	`+ "fileName": "condition-information.csv"`
`27`	`29`	`}`
`28`	`30`	`},`
`29`	`31`	`{`
`30`	`32`	`"label": "cancerDiseaseStatus",`
`31`	`33`	`"type": "CSVCancerDiseaseStatusExtractor",`
`32`	`34`	`"constructorArgs": {`
`33`		`- "filePath": "./data/cancer-disease-status-information.csv"`
	`35`	`+ "fileName": "cancer-disease-status-information.csv"`
`34`	`36`	`}`
`35`	`37`	`},`
`36`	`38`	`{`
`37`	`39`	`"label": "clinicalTrialInformation",`
`38`	`40`	`"type": "CSVClinicalTrialInformationExtractor",`
`39`	`41`	`"constructorArgs": {`
`40`		`- "filePath": "./data/clinical-trial-information.csv",`
	`42`	`+ "fileName": "clinical-trial-information.csv",`
`41`	`43`	`"clinicalSiteID": "example-site-id"`
`42`	`44`	`}`
`43`	`45`	`},`
`44`	`46`	`{`
`45`	`47`	`"label": "treatmentPlanChange",`
`46`	`48`	`"type": "CSVTreatmentPlanChangeExtractor",`
`47`	`49`	`"constructorArgs": {`
`48`		`- "filePath": "./data/treatment-plan-change-information.csv"`
	`50`	`+ "fileName": "treatment-plan-change-information.csv"`
`49`	`51`	`}`
`50`	`52`	`},`
`51`	`53`	`{`
`52`	`54`	`"label": "staging",`
`53`	`55`	`"type": "CSVStagingExtractor",`
`54`	`56`	`"constructorArgs": {`
`55`		`- "filePath": "./data/staging-information.csv"`
	`57`	`+ "fileName": "staging-information.csv"`
`56`	`58`	`}`
`57`	`59`	`},`
`58`	`60`	`{`
`59`	`61`	`"label": "cancerRelatedMedicationAdministration",`
`60`	`62`	`"type": "CSVCancerRelatedMedicationAdministrationExtractor",`
`61`	`63`	`"constructorArgs": {`
`62`		`- "filePath": "./data/cancer-related-medication-administration-information.csv"`
	`64`	`+ "fileName": "cancer-related-medication-administration-information.csv"`
`63`	`65`	`}`
`64`	`66`	`},`
`65`	`67`	`{`
`66`	`68`	`"label": "cancerRelatedMedicationRequest",`
`67`	`69`	`"type": "CSVCancerRelatedMedicationRequestExtractor",`
`68`	`70`	`"constructorArgs": {`
`69`		`- "filePath": "./data/cancer-related-medication-request-information.csv"`
	`71`	`+ "fileName": "cancer-related-medication-request-information.csv"`
`70`	`72`	`}`
`71`	`73`	`},`
`72`	`74`	`{`
`73`	`75`	`"label": "genericObservations",`
`74`	`76`	`"type": "CSVObservationExtractor",`
`75`	`77`	`"constructorArgs": {`
`76`		`- "filePath": "./data/observation-information.csv"`
	`78`	`+ "fileName": "observation-information.csv"`
`77`	`79`	`}`
`78`	`80`	`},`
`79`	`81`	`{`
`80`	`82`	`"label": "genericProcedures",`
`81`	`83`	`"type": "CSVProcedureExtractor",`
`82`	`84`	`"constructorArgs": {`
`83`		`- "filePath": "./data/procedure-information.csv"`
	`85`	`+ "fileName": "procedure-information.csv"`
`84`	`86`	`}`
`85`	`87`	`},`
`86`	`88`	`{`
`87`	`89`	`"label": "adverseEvent",`
`88`	`90`	`"type": "CSVAdverseEventExtractor",`
`89`	`91`	`"constructorArgs": {`
`90`		`- "filePath": "./test/sample-client-data/adverse-event-information.csv"`
	`92`	`+ "fileName": "adverse-event-information.csv"`
`91`	`93`	`}`
`92`	`94`	`},`
`93`	`95`	`{`
`94`	`96`	`"label": "ctcAdverseEvent",`
`95`	`97`	`"type": "CSVCTCAdverseEventExtractor",`
`96`	`98`	`"constructorArgs": {`
`97`		`- "filePath": "./test/sample-client-data/ctc-adverse-event-information.csv"`
	`99`	`+ "fileName": "ctc-adverse-event-information.csv"`
`98`	`100`	`}`
`99`	`101`	`}`
`100`	`102`	`]`
Original file line number	Diff line number	Diff line change
`@@ -69,8 +69,8 @@ function formatData(adverseEventData, patientId) {`
`69`	`69`	`}`
`70`	`70`
`71`	`71`	`class CSVAdverseEventExtractor extends BaseCSVExtractor {`
`72`		`- constructor({ filePath, url }) {`
`73`		`- super({ filePath, url });`
	`72`	`+ constructor({ filePath, url, fileName, dataDirectory }) {`
	`73`	`+ super({ filePath, url, fileName, dataDirectory });`
`74`	`74`	`}`
`75`	`75`
`76`	`76`	`async getAdverseEventData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -95,8 +95,8 @@ function formatData(adverseEventData, patientId) {`
`95`	`95`	`}`
`96`	`96`
`97`	`97`	`class CSVCTCAdverseEventExtractor extends BaseCSVExtractor {`
`98`		`- constructor({ filePath, url }) {`
`99`		`- super({ filePath, url });`
	`98`	`+ constructor({ filePath, url, fileName, dataDirectory }) {`
	`99`	`+ super({ filePath, url, fileName, dataDirectory });`
`100`	`100`	`}`
`101`	`101`
`102`	`102`	`async getAdverseEventData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -46,8 +46,8 @@ function formatData(medicationData, patientId) {`
`46`	`46`	`}`
`47`	`47`
`48`	`48`	`class CSVCancerRelatedMedicationAdministrationExtractor extends BaseCSVExtractor {`
`49`		`- constructor({ filePath, url }) {`
`50`		`- super({ filePath, url });`
	`49`	`+ constructor({ filePath, url, fileName, dataDirectory }) {`
	`50`	`+ super({ filePath, url, fileName, dataDirectory });`
`51`	`51`	`}`
`52`	`52`
`53`	`53`	`async getMedicationData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -48,8 +48,8 @@ function formatData(medicationData, patientId) {`
`48`	`48`	`}`
`49`	`49`
`50`	`50`	`class CSVCancerRelatedMedicationRequestExtractor extends BaseCSVExtractor {`
`51`		`- constructor({ filePath, url }) {`
`52`		`- super({ filePath, url });`
	`51`	`+ constructor({ filePath, url, fileName, dataDirectory }) {`
	`52`	`+ super({ filePath, url, fileName, dataDirectory });`
`53`	`53`	`}`
`54`	`54`
`55`	`55`	`async getMedicationData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -49,8 +49,8 @@ function formatData(conditionData, patientId) {`
`49`	`49`	`}`
`50`	`50`
`51`	`51`	`class CSVConditionExtractor extends BaseCSVExtractor {`
`52`		`- constructor({ filePath, url }) {`
`53`		`- super({ filePath, url, csvSchema: CSVConditionSchema });`
	`52`	`+ constructor({ filePath, url, fileName, dataDirectory }) {`
	`53`	`+ super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema });`
`54`	`54`	`}`
`55`	`55`
`56`	`56`	`async getConditionData(mrn) {`
Original file line number	Diff line number	Diff line change
`@@ -42,8 +42,8 @@ function formatData(observationData, patientId) {`
`42`	`42`	`}`
`43`	`43`
`44`	`44`	`class CSVObservationExtractor extends BaseCSVExtractor {`
`45`		`- constructor({ filePath, url }) {`
`46`		`- super({ filePath, url });`
	`45`	`+ constructor({ filePath, url, fileName, dataDirectory }) {`
	`46`	`+ super({ filePath, url, fileName, dataDirectory });`
`47`	`47`	`}`
`48`	`48`
`49`	`49`	`async getObservationData(mrn) {`