Merge pull request #58 from mcode/empty-data-standard

Dtphelan1 · web-flow · commit b7c29bbf775b · 2020-11-30T12:23:52.000-05:00
Standardized empty data approach
diff --git a/src/extractors/CSVCancerRelatedMedicationExtractor.js b/src/extractors/CSVCancerRelatedMedicationExtractor.js
@@ -23,13 +23,13 @@ function formatData(medicationData) {
       ...(medicationId && { id: medicationId }),
       code,
       codeSystem,
-      displayText: !displayText ? null : displayText,
+      displayText,
       startDate: !startDate ? null : formatDateTime(startDate),
       endDate: !endDate ? null : formatDateTime(endDate),
-      treatmentReasonCode: !treatmentReasonCode ? null : treatmentReasonCode,
-      treatmentReasonCodeSystem: !treatmentReasonCodeSystem ? null : treatmentReasonCodeSystem,
-      treatmentReasonDisplayText: !treatmentReasonDisplayText ? null : treatmentReasonDisplayText,
-      treatmentIntent: !treatmentIntent ? null : treatmentIntent,
+      treatmentReasonCode,
+      treatmentReasonCodeSystem,
+      treatmentReasonDisplayText,
+      treatmentIntent,
       status,
     };
   });
diff --git a/src/extractors/CSVConditionExtractor.js b/src/extractors/CSVConditionExtractor.js
@@ -28,11 +28,11 @@ function formatData(conditionData) {
       },
       category: category.split('|'),
       dateOfDiagnosis: !dateOfDiagnosis ? null : formatDateTime(dateOfDiagnosis),
-      clinicalStatus: !clinicalStatus ? null : clinicalStatus,
-      verificationStatus: !verificationStatus ? null : verificationStatus,
+      clinicalStatus,
+      verificationStatus,
       bodySite: !bodySite ? null : bodySite.split('|'),
-      laterality: !laterality ? null : laterality,
-      histology: !histology ? null : histology,
+      laterality,
+      histology,
     };
   });
 }
diff --git a/src/extractors/CSVObservationExtractor.js b/src/extractors/CSVObservationExtractor.js
@@ -22,9 +22,9 @@ function formatData(observationData) {
       status,
       code,
       system: codeSystem,
-      display: !displayName ? null : displayName,
+      display: displayName,
       valueCode: value,
-      valueCodeSystem: !valueCodeSystem ? null : valueCodeSystem,
+      valueCodeSystem,
       effectiveDateTime: formatDateTime(effectiveDate),
       bodySite,
       laterality,
diff --git a/src/extractors/CSVProcedureExtractor.js b/src/extractors/CSVProcedureExtractor.js
@@ -22,15 +22,15 @@ function formatData(procedureData) {
       status,
       code,
       system: codeSystem,
-      display: displayName || null,
-      reasonCode: reasonCode || null,
-      reasonCodeSystem: reasonCodeSystem || null,
-      reasonDisplayName: reasonDisplayName || null,
+      display: displayName,
+      reasonCode,
+      reasonCodeSystem,
+      reasonDisplayName,
       conditionId,
-      bodySite: bodySite || null,
-      laterality: laterality || null,
+      bodySite,
+      laterality,
       effectiveDateTime: formatDateTime(effectiveDate),
-      treatmentIntent: treatmentIntent || null,
+      treatmentIntent,
     };
   });
 }
diff --git a/src/templates/ResourceGenerator.js b/src/templates/ResourceGenerator.js
@@ -37,6 +37,22 @@ function generateResourceId(data) {
   return shajs('sha256').update(JSON.stringify(data)).digest('hex');
 }
 
+// Ensures that empty data in the resource object carries a null value, rather than being undefined or an empty string
+function cleanEmptyData(data, depth = 0) {
+  const cleanData = data;
+  const MAX_DEPTH = 50;
+  Object.keys(cleanData).forEach((key) => {
+    if (typeof cleanData[key] === 'object' && cleanData[key]) {
+      if (depth < MAX_DEPTH) cleanEmptyData(cleanData[key], depth + 1);
+      else logger.warn('Maximum depth of 50 was reached while cleaning empty data on a resource, resource may not be cleaned entirely');
+    }
+    if (cleanData[key] === '' || cleanData[key] === undefined) {
+      cleanData[key] = null;
+    }
+  });
+  return cleanData;
+}
+
 // Augment a data object with an ID if it doesn't have one
 function dataWithId(data) {
   return { id: generateResourceId(data), ...data };
@@ -56,10 +72,10 @@ function fillAndBundleTemplate(template, data) {
 
 function generateMcodeResources(mcodeProfileID, data) {
   logger.debug(`Generating FHIR resource for ${mcodeProfileID} data element`);
-
   const template = loadFhirTemplate(mcodeProfileID);
+  const cleanData = _.isArray(data) ? data.map((d) => cleanEmptyData(d)) : cleanEmptyData(data);
   if (!template) throw new Error(`No matching profile for ${mcodeProfileID} found`);
-  return fillAndBundleTemplate(template, data);
+  return fillAndBundleTemplate(template, cleanData);
 }
 
 module.exports = {
diff --git a/test/extractors/CSVCancerDiseaseStatusExtractor.test.js b/test/extractors/CSVCancerDiseaseStatusExtractor.test.js
@@ -30,17 +30,18 @@ describe('CSVCancerDiseaseStatusExtractor', () => {
       // Test that valid data works fine
       expect(csvCancerDiseaseStatusExtractor.joinAndReformatData(exampleCSVDiseaseStatusModuleResponse)).toEqual(expect.anything());
 
-      // Test all required properties are
-      delete localData[0].evidence; // Evidence is not required and will not throw an error
-      delete localData[0].observationStatus; // Observation Status is not required and will not throw an error
+      localData[0].evidence = ''; // Evidence is not required and will not throw an error
+      localData[0].observationStatus = ''; // Observation Status is not required and will not throw an error
 
       // Only including required properties is valid
       expect(csvCancerDiseaseStatusExtractor.joinAndReformatData(localData)).toEqual(expect.anything());
 
+      const requiredProperties = ['mrn', 'conditionId', 'diseaseStatusCode', 'dateOfObservation'];
+
       // Removing each required property should throw an error
-      Object.keys(localData[0]).forEach((key) => {
+      requiredProperties.forEach((key) => {
         const clonedData = _.cloneDeep(localData);
-        delete clonedData[0][key];
+        clonedData[0][key] = '';
         expect(() => csvCancerDiseaseStatusExtractor.joinAndReformatData(clonedData)).toThrow(new Error(expectedErrorString));
       });
     });
diff --git a/test/templates/carePlanWithReview.test.js b/test/templates/carePlanWithReview.test.js
@@ -12,6 +12,9 @@ describe('JavaScript render CarePlan template', () => {
       effectiveDate: '2020-01-23',
       hasChanged: 'false',
       mrn: 'abc-def',
+      reasonCode: null,
+      reasonDisplayText: null,
+      name: null,
     };
 
     const generatedCarePlan = carePlanWithReviewTemplate(CARE_PLAN_VALID_DATA);
@@ -61,6 +64,7 @@ describe('JavaScript render CarePlan template', () => {
       effectiveDateTime: '2020-01-23T09:07:00Z',
       effectiveDate: '2020-01-23',
       mrn: 'abc-def',
+      haschanged: null,
     };
 
     expect(() => carePlanWithReviewTemplate(INVALID_DATA)).toThrow(Error);
diff --git a/test/templates/medication.test.js b/test/templates/medication.test.js
@@ -36,7 +36,7 @@ const MEDICATION_MINIMAL_DATA = {
 
 
 const MEDICATION_INVALID_DATA = {
-  // Omitting 'mrn', 'code', 'codesystem', fields which are required
+  // Omitting 'mrn', 'code', 'codesystem', and 'status' fields which are required
   mrn: null,
   code: null,
   codeSystem: null,
diff --git a/test/templates/patient.test.js b/test/templates/patient.test.js
@@ -12,6 +12,18 @@ describe('JavaScript Render Patient', () => {
       familyName: 'Patient',
       givenName: 'Test',
       gender: 'female',
+      birthsex: null,
+      dateOfBirth: null,
+      language: null,
+      addressLine: null,
+      city: null,
+      state: null,
+      zip: null,
+      raceCodesystem: null,
+      raceCode: null,
+      raceText: null,
+      ethnicityCode: null,
+      ethnicityText: null,
     };
     const generatedPatient = patientTemplate(PATIENT_VALID_DATA);
     expect(generatedPatient).toEqual(basicPatient);
@@ -77,6 +89,7 @@ describe('JavaScript Render Patient', () => {
       familyName: 'Patient',
       givenName: 'Test',
       gender: 'female',
+      mrn: null,
     };
 
     expect(() => patientTemplate(PATIENT_INVALID_DATA)).toThrow(Error);
diff --git a/test/templates/procedure.test.js b/test/templates/procedure.test.js
@@ -13,6 +13,14 @@ describe('JavaScript render Procedure template', () => {
       status: 'completed',
       code: '152198000',
       system: 'http://snomed.info/sct',
+      display: null,
+      reasonCode: null,
+      reasonCodeSystem: null,
+      reasonDisplayName: null,
+      conditionId: null,
+      bodySite: null,
+      laterality: null,
+      treatmentIntent: null,
     };
 
     const generatedProcedure = procedureTemplate(PROCEDURE_MINIMAL_DATA);
@@ -75,6 +83,7 @@ describe('JavaScript render Procedure template', () => {
       effectiveDateTime: '2020-01-01',
       code: '152198000',
       system: 'http://snomed.info/sct',
+      status: null,
     };
 
     expect(() => procedureTemplate(INVALID_DATA)).toThrow(Error);
diff --git a/test/templates/researchStudy.test.js b/test/templates/researchStudy.test.js
@@ -12,6 +12,7 @@ const VALID_DATA = {
 const INVALID_DATA = {
   // Omitting 'trialResearchID' field which is required
   trialStatus: 'completed',
+  trialResearchID: null,
 };
 
 describe('test ResearchStudy template', () => {
diff --git a/test/templates/researchSubject.test.js b/test/templates/researchSubject.test.js
@@ -15,6 +15,7 @@ const INVALID_DATA = {
   enrollmentStatus: 'candidate',
   trialResearchID: 'rs1',
   patientId: 'mCODEPatient1',
+  trialSubjectID: null,
 };
 
 describe('test ResearchSubject template', () => {
diff --git a/test/templates/resourceGenerator.test.js b/test/templates/resourceGenerator.test.js
@@ -4,6 +4,7 @@ const resourceGenerator = rewire('../../src/templates/ResourceGenerator.js');
 
 // Use Rewire to access private methods
 const generateResourceId = resourceGenerator.__get__('generateResourceId');
+const cleanEmptyData = resourceGenerator.__get__('cleanEmptyData');
 
 // Some data to feed into the generateResourceId function
 const MOCK_DATA_1 = {
@@ -23,3 +24,100 @@ describe('generateResourceId', () => {
     expect(generateResourceId(MOCK_DATA_1)).not.toEqual(generateResourceId(MOCK_DATA_2));
   });
 });
+
+describe('cleanEmptyData', () => {
+  test('Should convert empty string and undefined values to null values', () => {
+    const testObject = {
+      id: 'example-id',
+      code: 'example-code',
+      display: 'example-display',
+      system: 'example-system',
+      status: '',
+      evidence: undefined,
+    };
+
+    const cleanData = {
+      id: 'example-id',
+      code: 'example-code',
+      display: 'example-display',
+      system: 'example-system',
+      status: null,
+      evidence: null,
+    };
+    expect(cleanEmptyData(testObject)).toStrictEqual(cleanData);
+  });
+
+  test('Should convert empty string and undefined values within object properties to null values', () => {
+    const testObject = {
+      id: 'example-id',
+      code: {
+        code: 'example-code',
+        display: '',
+        system: {
+          system: undefined,
+          code: '',
+        },
+      },
+      status: '',
+      evidence: undefined,
+    };
+
+    const cleanData = {
+      id: 'example-id',
+      code: {
+        code: 'example-code',
+        display: null,
+        system: {
+          system: null,
+          code: null,
+        },
+      },
+      status: null,
+      evidence: null,
+    };
+    expect(cleanEmptyData(testObject)).toStrictEqual(cleanData);
+  });
+
+  test('Should only be called recursively a maximum of 50 times', () => {
+    const testObject = {
+      id: 'example-id',
+      code: {
+        code: 'example-code',
+        display: '',
+        system: undefined,
+      },
+      status: '',
+      nestedObject: {
+        status: '',
+        nestedObject: {},
+      },
+    };
+
+    // Call cleanEmptyData with the recursive ceiling having been reached
+    cleanEmptyData(testObject, 50);
+
+    // The first object in the nest should have all of its properties cleaned
+    expect(testObject.status).toStrictEqual(null);
+
+    // Objects in the nest beyond the 50th call should still have non-standard properties
+    expect(testObject.nestedObject.status).toStrictEqual('');
+  });
+
+  test('Handles circular references without crashing', () => {
+    const testObject = {
+      id: 'example-id',
+      code: {
+        code: 'example-code',
+        display: '',
+        system: undefined,
+      },
+      status: '',
+      circular: {},
+    };
+
+    // Creating a circular reference
+    testObject.circular = testObject;
+
+    expect(() => cleanEmptyData(testObject)).not.toThrow();
+  });
+});
diff --git a/test/templates/staging.test.js b/test/templates/staging.test.js
@@ -14,6 +14,7 @@ describe('JavaScript render Staging template', () => {
       effectiveDateTime: '2020-01-01',
       stageGroup: '3C',
       type: 'Clinical',
+      categoryIds: null,
     };
 
     const generatedStaging = stagingTemplate(STAGING_CLINICAL_MINIMAL_DATA);
@@ -29,6 +30,7 @@ describe('JavaScript render Staging template', () => {
       effectiveDateTime: '2020-01-01',
       stageGroup: '3C',
       type: 'Pathologic',
+      categoryIds: null,
     };
 
     const generatedStaging = stagingTemplate(STAGING_CLINICAL_MINIMAL_DATA);
@@ -76,6 +78,7 @@ describe('JavaScript render Staging template', () => {
       subjectId: 'example-mrn',
       conditionId: 'example-condition-id',
       type: 'Clinical',
+      effectiveDateTime: null,
     };
 
     expect(() => stagingTemplate(INVALID_DATA)).toThrow(Error);
