Merge pull request #105 from mcode/csv-module-graceful-fail-on-filter

Csv module graceful fail on filter

Author: jafeltra

src/cli/app.js

@@ -99,7 +99,11 @@ async function mcodeApp(Client, fromDate, toDate, pathToConfig, pathToRunLogs, d
     const { notificationInfo } = config;
     if (notificationInfo) {
       const notificationErrors = zipErrors(totalExtractionErrors);
-      await sendEmailNotification(notificationInfo, notificationErrors, debug);
+      try {
+        await sendEmailNotification(notificationInfo, notificationErrors, debug);
+      } catch (e) {
+        logger.error(e.message);
+      }
     }
     // A run is successful and should be logged when both extraction finishes without fatal errors
     // and messages are posted without fatal errors
@@ -115,8 +119,14 @@ async function mcodeApp(Client, fromDate, toDate, pathToConfig, pathToRunLogs, d
     const patientConfig = config.extractors.find((e) => e.type === 'CSVPatientExtractor');
     if (patientConfig && ('constructorArgs' in patientConfig && 'mask' in patientConfig.constructorArgs)) {
       if (patientConfig.constructorArgs.mask.includes('mrn')) {
-        extractedData.forEach((bundle) => {
-          maskMRN(bundle);
+        extractedData.forEach((bundle, i) => {
+          // NOTE: This may fail to mask MRN-related properties on non-patient resources
+          //       Need to investigate further.
+          try {
+            maskMRN(bundle);
+          } catch (e) {
+            logger.error(`Bundle ${i + 1}: ${e.message}`);
+          }
         });
       }
     }

src/extractors/CSVPatientExtractor.js

@@ -1,9 +1,11 @@
+const _ = require('lodash');
 const { generateMcodeResources } = require('../templates');
 const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { getEthnicityDisplay,
   getRaceCodesystem,
   getRaceDisplay,
   maskPatientData } = require('../helpers/patientUtils');
+const { getEmptyBundle } = require('../helpers/fhirUtils');
 const logger = require('../helpers/logger');
 const { CSVPatientSchema } = require('../helpers/schemas/csv');
 
@@ -55,6 +57,10 @@ class CSVPatientExtractor extends BaseCSVExtractor {
   async get({ mrn }) {
     // 1. Get all relevant data and do necessary post-processing
     const patientData = await this.getPatientData(mrn);
+    if (_.isEmpty(patientData)) {
+      logger.warn('No patient data found for this patient');
+      return getEmptyBundle();
+    }
 
     // 2. Format data for research study and research subject
     const packagedPatientData = joinAndReformatData(patientData);

src/modules/CSVModule.js

@@ -13,7 +13,10 @@ class CSVModule {
     // return all rows if key and value aren't provided
     if (!key && !value) return this.data;
     let result = this.data.filter((d) => d[key] === value);
-    if (result.length === 0) throw new ReferenceError(`CSV Record with provided key '${key}' and value was not found`);
+    if (result.length === 0) {
+      logger.warn(`CSV Record with provided key '${key}' and value was not found`);
+      return result;
+    }
 
     // If fromDate and toDate is provided, filter out all results that fall outside that timespan
     if (fromDate && moment(fromDate).isValid()) result = result.filter((r) => !(r.dateRecorded && moment(fromDate).isAfter(r.dateRecorded)));

test/cli/fixtures/example-clinical-trial-info.csv

@@ -0,0 +1,4 @@
+mrn,trialSubjectID,enrollmentStatus,trialResearchID,trialStatus,trialResearchSystem
+123,subjectId-1,potential-candidate,researchId-1,approved,system-1
+456,subjectId-2,on-study-intervention,researchId-1,completed,system-2
+789,subjectId-3,on-study-observation,researchId-2,active,

test/cli/mcodeExtraction.test.js

@@ -43,7 +43,7 @@ describe('mcodeExtraction', () => {
       expect(extractedData).toHaveLength(0);
     });
 
-    it('should result in a successful extraction when non fatal errors are encountered in the client\'s get method', async () => {
+    it('should succeed in extraction when CSV files do not have data for all patients', async () => {
       const testConfig = {
         extractors: [
           {
@@ -69,10 +69,34 @@ describe('mcodeExtraction', () => {
 
       const { extractedData, successfulExtraction, totalExtractionErrors } = await extractDataForPatients(testPatientIds, testClient, testFromDate, testToDate);
       expect(successfulExtraction).toEqual(true);
+      // Should have data for 3 patients and 0 errors
+      expect(extractedData).toHaveLength(3);
+      const flatErrors = flattenErrorValues(totalExtractionErrors);
+      expect(flatErrors).toHaveLength(0);
+    });
+    it('should result in a successful extraction when non fatal errors are encountered in the client\'s get method', async () => {
+      const testConfig = {
+        extractors: [
+          // Should fail when this extractor is run without patient data in context
+          {
+            label: 'CTI',
+            type: 'CSVClinicalTrialInformationExtractor',
+            constructorArgs: {
+              filePath: path.join(__dirname, './fixtures/example-clinical-trial-info.csv'),
+            },
+          },
+        ],
+      };
 
+      const testClient = new MCODEClient(testConfig);
+      await testClient.init();
+
+      const { extractedData, successfulExtraction, totalExtractionErrors } = await extractDataForPatients(testPatientIds, testClient, testFromDate, testToDate);
+      expect(successfulExtraction).toEqual(true);
+      // Should have three (empty) bundles for patients and an error for each patient when extracting CTI
       expect(extractedData).toHaveLength(3);
       const flatErrors = flattenErrorValues(totalExtractionErrors);
-      expect(flatErrors).toHaveLength(1);
+      expect(flatErrors).toHaveLength(3);
     });
   });
 });

test/modules/CSVModule.test.js

@@ -31,10 +31,7 @@ test('Returns data with recordedDate before specified to date', async () => {
   expect(data).toHaveLength(1);
 });
 
-test('Invalid MRN', async () => {
-  try {
-    await csvModule.get('mrn', INVALID_MRN);
-  } catch (e) {
-    expect(e).toEqual(ReferenceError('CSV Record with provided key \'mrn\' and value was not found'));
-  }
+test('Should return an empty array when key-value pair does not exist', async () => {
+  const data = await csvModule.get('mrn', INVALID_MRN);
+  expect(data).toEqual([]);
 });

test/sample-client-data/cancer-disease-status-information.csv

@@ -1,5 +1,4 @@
 mrn,conditionId,diseaseStatusCode,diseaseStatusText,dateOfObservation,evidence,observationStatus,dateRecorded
 123,conditionId-1,268910001,responding,2019-12-02,363679005|252416005,preliminary,2020-01-10
-456,conditionId-2,359746009,stable,2020-01-12,363679005,amended,2020-01-12
 789,conditionId-2,709137006,not evaluated,2020-04-22,,final,2020-06-10
 123,conditionId-1,not-asked,,2020-01-12,,,

test/sample-client-data/patient-mrns.csv

@@ -2,3 +2,4 @@ mrn
 123
 456
 789
+1011