Merge pull request #108 from mcode/csv-extractor-helpful-error-messages

Numerous Extractor Updates

Author: jafeltra

config/csv.config.example.json

@@ -13,17 +13,17 @@
   },
   "extractors": [
     {
-      "label": "condition",
-      "type": "CSVConditionExtractor",
+      "label": "patient",
+      "type": "CSVPatientExtractor",
       "constructorArgs": {
-        "filePath": "./data/condition-information.csv"
+        "filePath": "./data/patient-information.csv"
       }
     },
     {
-      "label": "patient",
-      "type": "CSVPatientExtractor",
+      "label": "condition",
+      "type": "CSVConditionExtractor",
       "constructorArgs": {
-        "filePath": "./data/patient-information.csv"
+        "filePath": "./data/condition-information.csv"
       }
     },
     {

src/extractors/CSVAdverseEventExtractor.js

@@ -1,19 +1,21 @@
 const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const logger = require('../helpers/logger');
+const { getEmptyBundle } = require('../helpers/fhirUtils');
+const { getPatientFromContext } = require('../helpers/contextUtils');
 const { formatDateTime } = require('../helpers/dateUtils');
+const logger = require('../helpers/logger');
 
 // Formats data to be passed into template-friendly format
-function formatData(adverseEventData) {
+function formatData(adverseEventData, patientId) {
   logger.debug('Reformatting adverse event data from CSV into template format');
   return adverseEventData.map((data) => {
     const {
-      mrn, adverseEventId, adverseEventCode, adverseEventCodeSystem, adverseEventDisplayText, suspectedCauseId, suspectedCauseType, seriousness, seriousnessCodeSystem, seriousnessDisplayText,
+      adverseEventId, adverseEventCode, adverseEventCodeSystem, adverseEventDisplayText, suspectedCauseId, suspectedCauseType, seriousness, seriousnessCodeSystem, seriousnessDisplayText,
       category, categoryCodeSystem, categoryDisplayText, severity, actuality, studyId, effectiveDate, recordedDate,
     } = data;
 
-    if (!(mrn && adverseEventCode && effectiveDate)) {
-      throw new Error('The adverse event is missing an expected attribute. Adverse event code, mrn, and effective date are all required.');
+    if (!(adverseEventCode && effectiveDate)) {
+      throw new Error('The adverse event is missing an expected attribute. Adverse event code and effective date are all required.');
     }
 
     const categoryCodes = category.split('|');
@@ -27,7 +29,7 @@ function formatData(adverseEventData) {
 
     return {
       ...(adverseEventId && { id: adverseEventId }),
-      subjectId: mrn,
+      subjectId: patientId,
       code: adverseEventCode,
       system: !adverseEventCodeSystem ? 'http://snomed.info/sct' : adverseEventCodeSystem,
       display: adverseEventDisplayText,
@@ -61,10 +63,18 @@ class CSVAdverseEventExtractor extends BaseCSVExtractor {
     return this.csvModule.get('mrn', mrn);
   }
 
-  async get({ mrn }) {
+  async get({ mrn, context }) {
     const adverseEventData = await this.getAdverseEventData(mrn);
-    const formattedData = formatData(adverseEventData);
+    if (adverseEventData.length === 0) {
+      logger.warn('No adverse event data found for patient');
+      return getEmptyBundle();
+    }
+    const patientId = getPatientFromContext(context).id;
+
+    // Reformat data
+    const formattedData = formatData(adverseEventData, patientId);
 
+    // Fill templates
     return generateMcodeResources('AdverseEvent', formattedData);
   }
 }

src/extractors/CSVCancerDiseaseStatusExtractor.js

@@ -2,6 +2,7 @@ const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { formatDateTime } = require('../helpers/dateUtils');
 const { getDiseaseStatusDisplay, getDiseaseStatusEvidenceDisplay } = require('../helpers/diseaseStatusUtils');
 const { generateMcodeResources } = require('../templates');
+const { getPatientFromContext } = require('../helpers/contextUtils');
 const { getEmptyBundle } = require('../helpers/fhirUtils');
 const logger = require('../helpers/logger');
 const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');
@@ -12,12 +13,12 @@ class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
     this.implementation = implementation;
   }
 
-  joinAndReformatData(arrOfDiseaseStatusData) {
+  joinAndReformatData(arrOfDiseaseStatusData, patientId) {
     logger.debug('Reformatting disease status data from CSV into template format');
     // Check the shape of the data
     arrOfDiseaseStatusData.forEach((record) => {
-      if (!record.mrn || !record.conditionId || !record.diseaseStatusCode || !record.dateOfObservation) {
-        throw new Error('DiseaseStatusData missing an expected property: mrn, conditionId, diseaseStatusCode, and dateOfObservation are required.');
+      if (!record.conditionId || !record.diseaseStatusCode || !record.dateOfObservation) {
+        throw new Error('DiseaseStatusData missing an expected property: conditionId, diseaseStatusCode, and dateOfObservation are required.');
       }
     });
     const evidenceDelimiter = '|';
@@ -29,7 +30,7 @@ class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
         display: record.diseaseStatusText ? record.diseaseStatusText : getDiseaseStatusDisplay(record.diseaseStatusCode, this.implementation),
       },
       subject: {
-        id: record.mrn,
+        id: patientId,
       },
       condition: {
         id: record.conditionId,
@@ -47,16 +48,17 @@ class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
     return this.csvModule.get('mrn', mrn, fromDate, toDate);
   }
 
-  async get({ mrn, fromDate, toDate }) {
+  async get({ mrn, context, fromDate, toDate }) {
     // 1. Get all relevant data and do necessary post-processing
     const diseaseStatusData = await this.getDiseaseStatusData(mrn, fromDate, toDate);
     if (diseaseStatusData.length === 0) {
       logger.warn('No disease status data found for patient');
       return getEmptyBundle();
     }
+    const patientId = getPatientFromContext(context).id;
 
     // 2. Format data for research study and research subject
-    const packagedDiseaseStatusData = this.joinAndReformatData(diseaseStatusData);
+    const packagedDiseaseStatusData = this.joinAndReformatData(diseaseStatusData, patientId);
 
     // 3. Generate FHIR Resources
     const resources = generateMcodeResources('CancerDiseaseStatus', packagedDiseaseStatusData);

src/extractors/CSVCancerRelatedMedicationExtractor.js

@@ -1,24 +1,26 @@
 const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const logger = require('../helpers/logger');
+const { getPatientFromContext } = require('../helpers/contextUtils');
+const { getEmptyBundle } = require('../helpers/fhirUtils');
 const { formatDateTime } = require('../helpers/dateUtils');
+const logger = require('../helpers/logger');
 
 
-function formatData(medicationData) {
+function formatData(medicationData, patientId) {
   logger.debug('Reformatting cancer-related medication data from CSV into template format');
 
   return medicationData.map((medication) => {
     const {
-      mrn, medicationId, code, codeSystem, displayText, startDate, endDate, treatmentReasonCode, treatmentReasonCodeSystem, treatmentReasonDisplayText, treatmentIntent, status,
+      medicationId, code, codeSystem, displayText, startDate, endDate, treatmentReasonCode, treatmentReasonCodeSystem, treatmentReasonDisplayText, treatmentIntent, status,
     } = medication;
 
-    if (!(mrn && code && codeSystem && status)) {
-      throw new Error('The cancer-related medication is missing an expected element; mrn, code, code system, and status are all required values.');
+    if (!(code && codeSystem && status)) {
+      throw new Error('The cancer-related medication is missing an expected element; code, code system, and status are all required values.');
     }
 
     return {
-      mrn,
       ...(medicationId && { id: medicationId }),
+      subjectId: patientId,
       code,
       codeSystem,
       displayText,
@@ -43,10 +45,18 @@ class CSVCancerRelatedMedicationExtractor extends BaseCSVExtractor {
     return this.csvModule.get('mrn', mrn);
   }
 
-  async get({ mrn }) {
+  async get({ mrn, context }) {
     const medicationData = await this.getMedicationData(mrn);
-    const formattedData = formatData(medicationData);
+    if (medicationData.length === 0) {
+      logger.warn('No medication data found for patient');
+      return getEmptyBundle();
+    }
+    const patientId = getPatientFromContext(context).id;
+
+    // Reformat data
+    const formattedData = formatData(medicationData, patientId);
 
+    // Fill templates
     return generateMcodeResources('CancerRelatedMedication', formattedData);
   }
 }

src/extractors/CSVClinicalTrialInformationExtractor.js

@@ -1,19 +1,11 @@
+const _ = require('lodash');
 const { BaseCSVExtractor } = require('./BaseCSVExtractor');
-const { firstEntryInBundle, getBundleResourcesByType } = require('../helpers/fhirUtils');
+const { firstEntryInBundle, getEmptyBundle } = require('../helpers/fhirUtils');
+const { getPatientFromContext } = require('../helpers/contextUtils');
 const { generateMcodeResources } = require('../templates');
 const logger = require('../helpers/logger');
 const { CSVClinicalTrialInformationSchema } = require('../helpers/schemas/csv');
 
-function getPatientId(context) {
-  const patientInContext = getBundleResourcesByType(context, 'Patient', {}, true);
-  if (patientInContext) {
-    logger.debug('Patient resource found in context.');
-    return patientInContext.id;
-  }
-
-  logger.debug('No patient resource found in context.');
-  return undefined;
-}
 
 class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
   constructor({ filePath, clinicalSiteID, clinicalSiteSystem }) {
@@ -23,15 +15,15 @@ class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
     this.clinicalSiteSystem = clinicalSiteSystem;
   }
 
-  joinClinicalTrialData(patientId, clinicalTrialData) {
+  joinClinicalTrialData(clinicalTrialData, patientId) {
     logger.debug('Reformatting clinical trial data from CSV into template format');
     const {
       trialSubjectID, enrollmentStatus, trialResearchID, trialStatus, trialResearchSystem,
     } = clinicalTrialData;
     const { clinicalSiteID, clinicalSiteSystem } = this;
 
-    if (!(patientId && clinicalSiteID && trialSubjectID && enrollmentStatus && trialResearchID && trialStatus)) {
-      throw new Error('Clinical trial missing an expected property: patientId, clinicalSiteID, trialSubjectID, enrollmentStatus, trialResearchID, and trialStatus are required.');
+    if (!(clinicalSiteID && trialSubjectID && enrollmentStatus && trialResearchID && trialStatus)) {
+      throw new Error('Clinical trial missing an expected property: clinicalSiteID, trialSubjectID, enrollmentStatus, trialResearchID, and trialStatus are required.');
     }
 
     // Need separate data objects for ResearchSubject and ResearchStudy so that they get different resource ids
@@ -61,11 +53,15 @@ class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
   }
 
   async get({ mrn, context }) {
-    const patientId = getPatientId(context) || mrn;
     const clinicalTrialData = await this.getClinicalTrialData(mrn);
+    if (_.isEmpty(clinicalTrialData)) {
+      logger.warn('No clinicalTrial record found for patient');
+      return getEmptyBundle();
+    }
+    const patientId = getPatientFromContext(context).id;
 
     // Format data for research study and research subject
-    const formattedData = this.joinClinicalTrialData(patientId, clinicalTrialData);
+    const formattedData = this.joinClinicalTrialData(clinicalTrialData, patientId);
     const { formattedDataSubject, formattedDataStudy } = formattedData;
 
     // Generate ResearchSubject and ResearchStudy resources and combine into one bundle to return

src/extractors/CSVConditionExtractor.js

@@ -1,24 +1,26 @@
 const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const logger = require('../helpers/logger');
+const { getPatientFromContext } = require('../helpers/contextUtils');
+const { getEmptyBundle } = require('../helpers/fhirUtils');
 const { formatDateTime } = require('../helpers/dateUtils');
 const { CSVConditionSchema } = require('../helpers/schemas/csv');
+const logger = require('../helpers/logger');
 
 // Formats data to be passed into template-friendly format
-function formatData(conditionData) {
+function formatData(conditionData, patientId) {
   logger.debug('Reformatting condition data from CSV into template format');
   return conditionData.map((data) => {
     const {
-      mrn, conditionId, codeSystem, code, displayName, category, dateOfDiagnosis, clinicalStatus, verificationStatus, bodySite, laterality, histology,
+      conditionId, codeSystem, code, displayName, category, dateOfDiagnosis, clinicalStatus, verificationStatus, bodySite, laterality, histology,
     } = data;
 
-    if (!(conditionId && mrn && codeSystem && code && category)) {
-      throw new Error('The condition is missing an expected attribute. Condition id, mrn, code system, code, and category are all required.');
+    if (!(conditionId && codeSystem && code && category)) {
+      throw new Error('The condition is missing an expected attribute. Condition id, code system, code, and category are all required.');
     }
     return {
       id: conditionId,
       subject: {
-        id: mrn,
+        id: patientId,
       },
       code: {
         code,
@@ -46,10 +48,18 @@ class CSVConditionExtractor extends BaseCSVExtractor {
     return this.csvModule.get('mrn', mrn);
   }
 
-  async get({ mrn }) {
+  async get({ mrn, context }) {
     const conditionData = await this.getConditionData(mrn);
-    const formattedData = formatData(conditionData);
+    if (conditionData.length === 0) {
+      logger.warn('No condition data found for patient');
+      return getEmptyBundle();
+    }
+    const patientId = getPatientFromContext(context).id;
+
+    // Reformat data
+    const formattedData = formatData(conditionData, patientId);
 
+    // Fill templates
     return generateMcodeResources('Condition', formattedData);
   }
 }

src/extractors/CSVObservationExtractor.js

@@ -1,22 +1,24 @@
 const { BaseCSVExtractor } = require('./BaseCSVExtractor');
 const { generateMcodeResources } = require('../templates');
-const logger = require('../helpers/logger');
+const { getPatientFromContext } = require('../helpers/contextUtils');
+const { getEmptyBundle } = require('../helpers/fhirUtils');
 const { formatDateTime } = require('../helpers/dateUtils');
+const logger = require('../helpers/logger');
 
-function formatData(observationData) {
+function formatData(observationData, patientId) {
   logger.debug('Reformatting observation data from CSV into template format');
   return observationData.map((data) => {
     const {
-      mrn, observationId, status, code, codeSystem, displayName, value, valueCodeSystem, effectiveDate, bodySite, laterality,
+      observationId, status, code, codeSystem, displayName, value, valueCodeSystem, effectiveDate, bodySite, laterality,
     } = data;
 
-    if (!mrn || !observationId || !status || !code || !codeSystem || !value || !effectiveDate) {
-      throw new Error('The observation is missing an expected attribute. Observation id, mrn, status, code, code system, value, and effective date are all required.');
+    if (!observationId || !status || !code || !codeSystem || !value || !effectiveDate) {
+      throw new Error('The observation is missing an expected attribute. Observation id, status, code, code system, value, and effective date are all required.');
     }
 
     return {
       id: observationId,
-      subjectId: mrn,
+      subjectId: patientId,
       status,
       code,
       system: codeSystem,
@@ -40,10 +42,18 @@ class CSVObservationExtractor extends BaseCSVExtractor {
     return this.csvModule.get('mrn', mrn);
   }
 
-  async get({ mrn }) {
+  async get({ mrn, context }) {
     const observationData = await this.getObservationData(mrn);
-    const formattedData = formatData(observationData);
+    if (observationData.length === 0) {
+      logger.warn('No observation data found for patient');
+      return getEmptyBundle();
+    }
+    const patientId = getPatientFromContext(context).id;
+
+    // Reformat data
+    const formattedData = formatData(observationData, patientId);
 
+    // Fill template
     return generateMcodeResources('Observation', formattedData);
   }
 }

src/extractors/CSVPatientExtractor.js

@@ -6,8 +6,8 @@ const { getEthnicityDisplay,
   getRaceDisplay,
   maskPatientData } = require('../helpers/patientUtils');
 const { getEmptyBundle } = require('../helpers/fhirUtils');
-const logger = require('../helpers/logger');
 const { CSVPatientSchema } = require('../helpers/schemas/csv');
+const logger = require('../helpers/logger');
 
 function joinAndReformatData(patientData) {
   logger.debug('Reformatting patient data from CSV into template format');