single point of truth for csv parsing

Author: Dtphelan1

src/helpers/appUtils.js

@@ -1,6 +1,6 @@
 const fs = require('fs');
 const path = require('path');
-const parse = require('csv-parse/lib/sync');
+const { csvParse } = require('./csvParsingUtils');
 
 /**
  * Parses a provided CSV with MRN column into string array of IDs
@@ -11,10 +11,7 @@ const parse = require('csv-parse/lib/sync');
 function parsePatientIds(pathToCSV) {
   // Parse CSV for list of patient IDs
   const patientIdsCsvPath = path.resolve(pathToCSV);
-  const patientIds = parse(fs.readFileSync(patientIdsCsvPath, 'utf8'), {
-    columns: (header) => header.map((column) => column.toLowerCase()),
-    bom: true,
-  }).map((row) => {
+  const patientIds = csvParse(fs.readFileSync(patientIdsCsvPath, 'utf8')).map((row) => {
     if (!row.mrn) {
       throw new Error(`${pathToCSV} has no "mrn" column`);
     }

src/helpers/csvParsingUtils.js

@@ -1,3 +1,4 @@
+const parse = require('csv-parse/lib/sync');
 const logger = require('./logger');
 
 // The standard string normalizer function
@@ -38,7 +39,27 @@ function normalizeEmptyValues(data, unalterableColumns = []) {
   return newData;
 }
 
+// Default options for CSV parsing
+const DEFAULT_OPTIONS = {
+  columns: (header) => header.map((column) => stringNormalizer(column)),
+  // https://csv.js.org/parse/options/bom/
+  bom: true,
+  // https://csv.js.org/parse/options/skip_empty_lines/
+  skip_empty_lines: true,
+  // NOTE: This will skip any records with empty values, not just skip the empty values themselves
+  // NOTE-2: The name of the flag changed from v4 (what we use) to v5 (what is documented)
+  // https://csv.js.org/parse/options/skip_records_with_empty_values/
+  skip_lines_with_empty_values: true,
+};
+
+// Common utility for parsing CSV files
+function csvParse(csvData, options = {}) {
+  return parse(csvData, { ...DEFAULT_OPTIONS, ...options });
+}
+
+
 module.exports = {
   stringNormalizer,
   normalizeEmptyValues,
+  csvParse,
 };

src/modules/CSVFileModule.js

@@ -1,25 +1,13 @@
 const fs = require('fs');
 const moment = require('moment');
-const parse = require('csv-parse/lib/sync');
 const logger = require('../helpers/logger');
 const { validateCSV } = require('../helpers/csvValidator');
-const { stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
+const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
 
 class CSVFileModule {
   constructor(csvFilePath, unalterableColumns) {
     // Parse then normalize the data
-    const parsedData = parse(fs.readFileSync(csvFilePath), {
-      columns: (header) => header.map((column) => stringNormalizer(column)),
-      // https://csv.js.org/parse/options/bom/
-      bom: true,
-      // https://csv.js.org/parse/options/skip_empty_lines/
-      skip_empty_lines: true,
-      // NOTE: This will skip any records with empty values, not just skip the empty values themselves
-      // NOTE-2: The name of the flag changed from v4 (what we use) to v5 (what is documented)
-      // https://csv.js.org/parse/options/skip_records_with_empty_values/
-      skip_lines_with_empty_values: true,
-    });
-
+    const parsedData = csvParse(fs.readFileSync(csvFilePath));
     this.filePath = csvFilePath;
     this.data = normalizeEmptyValues(parsedData, unalterableColumns);
   }

src/modules/CSVURLModule.js

@@ -1,9 +1,8 @@
 const axios = require('axios');
 const moment = require('moment');
-const parse = require('csv-parse/lib/sync');
 const logger = require('../helpers/logger');
 const { validateCSV } = require('../helpers/csvValidator');
-const { stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
+const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
 
 class CSVURLModule {
   constructor(url, unalterableColumns) {
@@ -25,10 +24,7 @@ class CSVURLModule {
         });
       logger.debug('Web request successful');
       // Parse then normalize the data
-      const parsedData = parse(csvData, {
-        columns: (header) => header.map((column) => stringNormalizer(column)),
-        bom: true,
-      });
+      const parsedData = csvParse(csvData);
       logger.debug('CSV Data parsing successful');
       this.data = normalizeEmptyValues(parsedData, this.unalterableColumns);
     }