Skip to content

Commit d81800d

Browse files
authored
Merge pull request #170 from mcode/data-directory-config
CSVExtractors support dataDirectory & fileName arguments
2 parents 0bd315d + 45fab60 commit d81800d

18 files changed

+133
-43
lines changed

README.md

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,11 @@ Examples files for these extractor can be found in the [`test/sample-client-data
9292

9393
After exporting your CSV files to the `data` directory, kickstart the creation of a configuration file by renaming the provided `csv.config.example.json` to `csv.config.json`. Then, ensure the following configuration parameters are properly set:
9494

95-
1. `patientIdCsvPath` should provide a file path to a CSV file containing MRN's for relevant patients;
96-
2. For each extractor, `filePath:` should provide a file path to a CSV file containing that corresponding extractor's data;
95+
1. `patientIdCsvPath` should correspond to an absolute file path to a CSV file containing MRN's for relevant patients;
96+
2. `commonExtractorArgs.dataDirectory` should correspond to an absolute path to the dataDirectory containing all your exported CSV files;
97+
3. For each extractor, `fileName` should correspond to the file name this extractor should be reading from. Note: combining the `dataDirectory` above and `fileName` should resolve to a file on disk containing this corresponding extractor's data;
98+
99+
**Note**: Previous versions of the MEF suggested using a `filePath` property for each extractor; while this property should still work without issue, the recommended approach is to use a common dataDirectory for all CSV files and to have each Extractor call out the name of the CSV file they need.
97100

98101
For instructions on setting up an email notification trigger whenever an error is encountered in extraction, see the [Email Notification](#Email-Notification) section below.
99102

@@ -143,7 +146,7 @@ To mask a property, provide an array of the properties to mask in the `construct
143146
"label": "patient",
144147
"type": "CSVPatientExtractor",
145148
"constructorArgs": {
146-
"filePath": "./data/patient-information.csv"
149+
"fileName": "patient-information.csv"
147150
"mask": ["address", "birthDate"]
148151
}
149152
}
@@ -156,7 +159,7 @@ Alternatively, providing a string with a value of `all` in the `constructorArgs`
156159
"label": "patient",
157160
"type": "CSVPatientExtractor",
158161
"constructorArgs": {
159-
"filePath": "./data/patient-information.csv"
162+
"fileName": "patient-information.csv"
160163
"mask": "all"
161164
}
162165
}

config/csv.config.example.json

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
{
2-
"patientIdCsvPath": "./data/patient-mrns.csv",
3-
"commonExtractorArgs": {},
2+
"patientIdCsvPath": "Users/YourAccount/absolute/path/to/patient-mrns.csv",
3+
"commonExtractorArgs": {
4+
"dataDirectory": "Users/YourAccount/absolute/path/to/data/directory"
5+
},
46
"notificationInfo": {
57
"host": "smtp.example.com",
68
"port": 587,
@@ -16,85 +18,85 @@
1618
"label": "patient",
1719
"type": "CSVPatientExtractor",
1820
"constructorArgs": {
19-
"filePath": "./data/patient-information.csv"
21+
"fileName": "patient-information.csv"
2022
}
2123
},
2224
{
2325
"label": "condition",
2426
"type": "CSVConditionExtractor",
2527
"constructorArgs": {
26-
"filePath": "./data/condition-information.csv"
28+
"fileName": "condition-information.csv"
2729
}
2830
},
2931
{
3032
"label": "cancerDiseaseStatus",
3133
"type": "CSVCancerDiseaseStatusExtractor",
3234
"constructorArgs": {
33-
"filePath": "./data/cancer-disease-status-information.csv"
35+
"fileName": "cancer-disease-status-information.csv"
3436
}
3537
},
3638
{
3739
"label": "clinicalTrialInformation",
3840
"type": "CSVClinicalTrialInformationExtractor",
3941
"constructorArgs": {
40-
"filePath": "./data/clinical-trial-information.csv",
42+
"fileName": "clinical-trial-information.csv",
4143
"clinicalSiteID": "example-site-id"
4244
}
4345
},
4446
{
4547
"label": "treatmentPlanChange",
4648
"type": "CSVTreatmentPlanChangeExtractor",
4749
"constructorArgs": {
48-
"filePath": "./data/treatment-plan-change-information.csv"
50+
"fileName": "treatment-plan-change-information.csv"
4951
}
5052
},
5153
{
5254
"label": "staging",
5355
"type": "CSVStagingExtractor",
5456
"constructorArgs": {
55-
"filePath": "./data/staging-information.csv"
57+
"fileName": "staging-information.csv"
5658
}
5759
},
5860
{
5961
"label": "cancerRelatedMedicationAdministration",
6062
"type": "CSVCancerRelatedMedicationAdministrationExtractor",
6163
"constructorArgs": {
62-
"filePath": "./data/cancer-related-medication-administration-information.csv"
64+
"fileName": "cancer-related-medication-administration-information.csv"
6365
}
6466
},
6567
{
6668
"label": "cancerRelatedMedicationRequest",
6769
"type": "CSVCancerRelatedMedicationRequestExtractor",
6870
"constructorArgs": {
69-
"filePath": "./data/cancer-related-medication-request-information.csv"
71+
"fileName": "cancer-related-medication-request-information.csv"
7072
}
7173
},
7274
{
7375
"label": "genericObservations",
7476
"type": "CSVObservationExtractor",
7577
"constructorArgs": {
76-
"filePath": "./data/observation-information.csv"
78+
"fileName": "observation-information.csv"
7779
}
7880
},
7981
{
8082
"label": "genericProcedures",
8183
"type": "CSVProcedureExtractor",
8284
"constructorArgs": {
83-
"filePath": "./data/procedure-information.csv"
85+
"fileName": "procedure-information.csv"
8486
}
8587
},
8688
{
8789
"label": "adverseEvent",
8890
"type": "CSVAdverseEventExtractor",
8991
"constructorArgs": {
90-
"filePath": "./test/sample-client-data/adverse-event-information.csv"
92+
"fileName": "adverse-event-information.csv"
9193
}
9294
},
9395
{
9496
"label": "ctcAdverseEvent",
9597
"type": "CSVCTCAdverseEventExtractor",
9698
"constructorArgs": {
97-
"filePath": "./test/sample-client-data/ctc-adverse-event-information.csv"
99+
"fileName": "ctc-adverse-event-information.csv"
98100
}
99101
}
100102
]

src/extractors/BaseCSVExtractor.js

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,36 @@
1+
const path = require('path');
12
const { Extractor } = require('./Extractor');
23
const { CSVFileModule, CSVURLModule } = require('../modules');
4+
const logger = require('../helpers/logger');
5+
36

47
class BaseCSVExtractor extends Extractor {
5-
constructor({ filePath, url, csvSchema, unalterableColumns }) {
8+
constructor({
9+
filePath, url, fileName, dataDirectory, csvSchema, unalterableColumns,
10+
}) {
611
super();
712
this.unalterableColumns = unalterableColumns || [];
813
this.csvSchema = csvSchema;
914
if (url) {
15+
logger.debug('Found url argument; creating a CSVURLModule with the provided url');
1016
this.url = url;
1117
this.csvModule = new CSVURLModule(this.url, this.unalterableColumns);
18+
} else if (fileName && dataDirectory) {
19+
if (!path.isAbsolute(dataDirectory)) throw new Error('dataDirectory is not an absolutePath, it needs to be.');
20+
this.filePath = path.join(dataDirectory, fileName);
21+
logger.debug(
22+
'Found fileName and dataDirectory arguments; creating a CSVFileModule with the provided dataDirectory and fileName',
23+
);
24+
this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
1225
} else if (filePath) {
26+
logger.debug('Found filePath argument; creating a CSVFileModule with the provided filePath');
1327
this.filePath = filePath;
1428
this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
1529
} else {
16-
throw new Error('Trying to instantiate a CSVExtractor without a filePath or url');
30+
logger.debug(
31+
'Could not instantiate a CSVExtractor with the provided constructor args',
32+
);
33+
throw new Error('Trying to instantiate a CSVExtractor without a valid filePath, url, or fileName+dataDirectory combination');
1734
}
1835
}
1936

src/extractors/CSVAdverseEventExtractor.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,8 @@ function formatData(adverseEventData, patientId) {
6969
}
7070

7171
class CSVAdverseEventExtractor extends BaseCSVExtractor {
72-
constructor({ filePath, url }) {
73-
super({ filePath, url });
72+
constructor({ filePath, url, fileName, dataDirectory }) {
73+
super({ filePath, url, fileName, dataDirectory });
7474
}
7575

7676
async getAdverseEventData(mrn) {

src/extractors/CSVCTCAdverseEventExtractor.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,8 @@ function formatData(adverseEventData, patientId) {
9595
}
9696

9797
class CSVCTCAdverseEventExtractor extends BaseCSVExtractor {
98-
constructor({ filePath, url }) {
99-
super({ filePath, url });
98+
constructor({ filePath, url, fileName, dataDirectory }) {
99+
super({ filePath, url, fileName, dataDirectory });
100100
}
101101

102102
async getAdverseEventData(mrn) {

src/extractors/CSVCancerDiseaseStatusExtractor.js

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ const logger = require('../helpers/logger');
88
const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');
99

1010
class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
11-
constructor({ filePath, url, implementation }) {
12-
super({ filePath, url, csvSchema: CSVCancerDiseaseStatusSchema });
11+
constructor({
12+
filePath, url, fileName, dataDirectory, implementation,
13+
}) {
14+
super({ filePath, url, fileName, dataDirectory, csvSchema: CSVCancerDiseaseStatusSchema });
1315
this.implementation = implementation;
1416
}
1517

src/extractors/CSVCancerRelatedMedicationAdministrationExtractor.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ function formatData(medicationData, patientId) {
4646
}
4747

4848
class CSVCancerRelatedMedicationAdministrationExtractor extends BaseCSVExtractor {
49-
constructor({ filePath, url }) {
50-
super({ filePath, url });
49+
constructor({ filePath, url, fileName, dataDirectory }) {
50+
super({ filePath, url, fileName, dataDirectory });
5151
}
5252

5353
async getMedicationData(mrn) {

src/extractors/CSVCancerRelatedMedicationRequestExtractor.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ function formatData(medicationData, patientId) {
4848
}
4949

5050
class CSVCancerRelatedMedicationRequestExtractor extends BaseCSVExtractor {
51-
constructor({ filePath, url }) {
52-
super({ filePath, url });
51+
constructor({ filePath, url, fileName, dataDirectory }) {
52+
super({ filePath, url, fileName, dataDirectory });
5353
}
5454

5555
async getMedicationData(mrn) {

src/extractors/CSVClinicalTrialInformationExtractor.js

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ const { CSVClinicalTrialInformationSchema } = require('../helpers/schemas/csv');
88

99

1010
class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
11-
constructor({ filePath, url, clinicalSiteID, clinicalSiteSystem }) {
12-
super({ filePath, url, csvSchema: CSVClinicalTrialInformationSchema });
11+
constructor({
12+
filePath, url, fileName, dataDirectory, clinicalSiteID, clinicalSiteSystem,
13+
}) {
14+
super({ filePath, url, fileName, dataDirectory, csvSchema: CSVClinicalTrialInformationSchema });
1315
if (!clinicalSiteID) logger.warn(`${this.constructor.name} expects a value for clinicalSiteID but got ${clinicalSiteID}`);
1416
this.clinicalSiteID = clinicalSiteID;
1517
this.clinicalSiteSystem = clinicalSiteSystem;

src/extractors/CSVConditionExtractor.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@ function formatData(conditionData, patientId) {
4949
}
5050

5151
class CSVConditionExtractor extends BaseCSVExtractor {
52-
constructor({ filePath, url }) {
53-
super({ filePath, url, csvSchema: CSVConditionSchema });
52+
constructor({ filePath, url, fileName, dataDirectory }) {
53+
super({ filePath, url, fileName, dataDirectory, csvSchema: CSVConditionSchema });
5454
}
5555

5656
async getConditionData(mrn) {

0 commit comments

Comments
 (0)