added null-nil empty-value normalizer

Author: Dtphelan1

src/extractors/BaseCSVExtractor.js

@@ -5,11 +5,12 @@ const { validateCSV } = require('../helpers/csvValidator');
 const logger = require('../helpers/logger');
 
 class BaseCSVExtractor extends Extractor {
-  constructor({ filePath, csvSchema }) {
+  constructor({ filePath, csvSchema, unalterableAttributes }) {
     super();
+    this.unalterableAttributes = unalterableAttributes || [];
     this.csvSchema = csvSchema;
     this.filePath = path.resolve(filePath);
-    this.csvModule = new CSVModule(this.filePath);
+    this.csvModule = new CSVModule(this.filePath, this.unalterableAttributes);
   }
 
   validate() {

src/extractors/CSVPatientExtractor.js

@@ -55,7 +55,9 @@ function joinAndReformatData(patientData) {
 
 class CSVPatientExtractor extends BaseCSVExtractor {
   constructor({ filePath, mask = [] }) {
-    super({ filePath, csvSchema: CSVPatientSchema });
+    // Define CSV attributes whose values should never be altered
+    const unalterableAttributes = ['familyName', 'givenName'];
+    super({ filePath, csvSchema: CSVPatientSchema, unalterableAttributes });
     this.mask = mask;
   }
 

src/modules/CSVModule.js

@@ -3,9 +3,32 @@ const moment = require('moment');
 const parse = require('csv-parse/lib/sync');
 const logger = require('../helpers/logger');
 
+function normalizeEmptyValues(data, unalterableAttributes = []) {
+  const EMPTY_VALUES = ['null', 'nil'];
+
+  return data.map((row) => {
+    const newRow = { ...row };
+    // Filter out unalterable attributes
+    const attributesToNormalize = Object.keys(row).filter((attr) => !unalterableAttributes.includes(attr));
+    attributesToNormalize.forEach((attr) => {
+      const value = newRow[attr];
+      // If the value for this row-attr combo is a value that should be empty, replace it
+      if (EMPTY_VALUES.includes(value.toLowerCase())) {
+        newRow[attr] = '';
+      }
+    });
+    return newRow;
+  });
+}
+
 class CSVModule {
-  constructor(csvFilePath) {
-    this.data = parse(fs.readFileSync(csvFilePath), { columns: (header) => header.map((column) => column.toLowerCase()), bom: true });
+  constructor(csvFilePath, unalterableAttributes) {
+    // Parse then normalize the data
+    const parsedData = parse(fs.readFileSync(csvFilePath), {
+      columns: (header) => header.map((column) => column.toLowerCase()),
+      bom: true,
+    });
+    this.data = normalizeEmptyValues(parsedData, unalterableAttributes);
   }
 
   async get(key, value, fromDate, toDate) {

test/modules/CSVModule.test.js

@@ -1,48 +1,83 @@
 const path = require('path');
+const rewire = require('rewire');
 const { CSVModule } = require('../../src/modules');
 const exampleResponse = require('./fixtures/csv-response.json');
 
+const CSVModuleRewired = rewire('../../src/modules/CSVModule.js');
+const normalizeEmptyValues = CSVModuleRewired.__get__('normalizeEmptyValues');
+
 const INVALID_MRN = 'INVALID MRN';
 const csvModule = new CSVModule(path.join(__dirname, './fixtures/example-csv.csv'));
 const csvModuleWithBOMs = new CSVModule(path.join(__dirname, './fixtures/example-csv-bom.csv'));
 
-test('Reads data from CSV', async () => {
-  const data = await csvModule.get('mrn', 'example-mrn-1');
-  expect(data).toEqual(exampleResponse);
-});
 
-test('Reads data from CSV with a Byte Order Mark', async () => {
-  const data = await csvModuleWithBOMs.get('mrn', 'example-mrn-1');
-  expect(data).toEqual(exampleResponse);
-});
+describe('CSVModule', () => {
+  describe('get', () => {
+    test('Reads data from CSV', async () => {
+      const data = await csvModule.get('mrn', 'example-mrn-1');
+      expect(data).toEqual(exampleResponse);
+    });
 
-test('Returns multiple rows', async () => {
-  const data = await csvModule.get('mrn', 'example-mrn-2');
-  expect(data).toHaveLength(2);
-});
+    test('Reads data from CSV with a Byte Order Mark', async () => {
+      const data = await csvModuleWithBOMs.get('mrn', 'example-mrn-1');
+      expect(data).toEqual(exampleResponse);
+    });
 
-test('Returns all rows when both key and value are undefined', async () => {
-  const data = await csvModule.get();
-  expect(data).toHaveLength(csvModule.data.length);
-  expect(data).toEqual(csvModule.data);
-});
+    test('Returns multiple rows', async () => {
+      const data = await csvModule.get('mrn', 'example-mrn-2');
+      expect(data).toHaveLength(2);
+    });
 
-test('Returns data with recordedDate after specified from date', async () => {
-  const data = await csvModule.get('mrn', 'example-mrn-2', '2020-05-01');
-  expect(data).toHaveLength(1);
-});
+    test('Returns all rows when both key and value are undefined', async () => {
+      const data = await csvModule.get();
+      expect(data).toHaveLength(csvModule.data.length);
+      expect(data).toEqual(csvModule.data);
+    });
 
-test('Returns data with recordedDate before specified to date', async () => {
-  const data = await csvModule.get('mrn', 'example-mrn-2', null, '2020-05-01');
-  expect(data).toHaveLength(1);
-});
+    test('Returns data with recordedDate after specified from date', async () => {
+      const data = await csvModule.get('mrn', 'example-mrn-2', '2020-05-01');
+      expect(data).toHaveLength(1);
+    });
 
-test('Should return an empty array when key-value pair does not exist', async () => {
-  const data = await csvModule.get('mrn', INVALID_MRN);
-  expect(data).toEqual([]);
-});
+    test('Returns data with recordedDate before specified to date', async () => {
+      const data = await csvModule.get('mrn', 'example-mrn-2', null, '2020-05-01');
+      expect(data).toHaveLength(1);
+    });
+
+    test('Should return an empty array when key-value pair does not exist', async () => {
+      const data = await csvModule.get('mrn', INVALID_MRN);
+      expect(data).toEqual([]);
+    });
+
+    test('Should return proper value regardless of key casing', async () => {
+      const data = await csvModule.get('mRN', 'example-mrn-1');
+      expect(data).toEqual(exampleResponse);
+    });
+  });
+
+  describe('normalizeEmptyValues', () => {
+    it('Should turn "null" values into empty strings, regardless of case', () => {
+      const data = [{ key: 'null' }, { key: 'NULL' }, { key: 'nuLL' }];
+      const normalizedData = normalizeEmptyValues(data);
+      normalizedData.forEach((d) => {
+        expect(d.key).toBe('');
+      });
+    });
+
+    it('Should turn "nil" values into empty strings, regardless of case', () => {
+      const data = [{ key: 'nil' }, { key: 'NIL' }, { key: 'NIl' }];
+      const normalizedData = normalizeEmptyValues(data);
+      normalizedData.forEach((d) => {
+        expect(d.key).toBe('');
+      });
+    });
 
-test('Should return proper value regardless of key casing', async () => {
-  const data = await csvModule.get('mRN', 'example-mrn-1');
-  expect(data).toEqual(exampleResponse);
+    it('Should leave all other values uneffected, regardless of case', () => {
+      const data = [{ key: 'anything' }, { key: 'any' }, { key: 'thing' }];
+      const normalizedData = normalizeEmptyValues(data);
+      normalizedData.forEach((d) => {
+        expect(d.key).not.toBe('');
+      });
+    });
+  });
 });