Merge pull request #185 from mcode/allow-empty-csv

Allowing CSVs with no rows + adding trimming to CSV parsing

Author: Dtphelan1

src/helpers/csvParsingUtils.js

@@ -44,6 +44,8 @@ const DEFAULT_OPTIONS = {
   columns: (header) => header.map((column) => stringNormalizer(column)),
   // https://csv.js.org/parse/options/bom/
   bom: true,
+  // https://csv.js.org/parse/options/trim/
+  trim: true,
   // https://csv.js.org/parse/options/skip_empty_lines/
   skip_empty_lines: true,
   // NOTE: This will skip any records with empty values, not just skip the empty values themselves
@@ -57,9 +59,17 @@ function csvParse(csvData, options = {}) {
   return parse(csvData, { ...DEFAULT_OPTIONS, ...options });
 }
 
+function getCSVHeader(csvData) {
+  return parse(csvData, {
+    bom: true,
+    trim: true,
+    to: 1,
+  })[0].map((h) => h.toLowerCase());
+}
 
 module.exports = {
   stringNormalizer,
   normalizeEmptyValues,
   csvParse,
+  getCSVHeader,
 };

src/helpers/csvValidator.js

@@ -3,11 +3,10 @@ const logger = require('./logger');
 
 // Validates csvData against the csvSchema
 // Uses the csvFileIdentifier in logs for readability
-function validateCSV(csvFileIdentifier, csvSchema, csvData) {
+function validateCSV(csvFileIdentifier, csvSchema, csvData, headers) {
   let isValid = true;
 
   // Check headers
-  const headers = Object.keys(csvData[0]).map((h) => h.toLowerCase());
   const schemaDiff = _.difference(csvSchema.headers.map((h) => h.name.toLowerCase()), headers);
   const fileDiff = _.difference(headers, csvSchema.headers.map((h) => h.name.toLowerCase()));
 

src/modules/CSVFileModule.js

@@ -2,14 +2,16 @@ const fs = require('fs');
 const moment = require('moment');
 const logger = require('../helpers/logger');
 const { validateCSV } = require('../helpers/csvValidator');
-const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
+const { csvParse, stringNormalizer, normalizeEmptyValues, getCSVHeader } = require('../helpers/csvParsingUtils');
 
 class CSVFileModule {
   constructor(csvFilePath, unalterableColumns, parserOptions) {
     // Parse then normalize the data
-    const parsedData = csvParse(fs.readFileSync(csvFilePath), parserOptions);
+    const csvData = fs.readFileSync(csvFilePath);
+    const parsedData = csvParse(csvData, parserOptions);
     this.filePath = csvFilePath;
     this.data = normalizeEmptyValues(parsedData, unalterableColumns);
+    this.header = getCSVHeader(csvData);
   }
 
   async get(key, value, fromDate, toDate) {
@@ -32,7 +34,7 @@ class CSVFileModule {
   async validate(csvSchema) {
     if (csvSchema) {
       logger.info(`Validating CSV file for ${this.filePath}`);
-      return validateCSV(this.filePath, csvSchema, this.data);
+      return validateCSV(this.filePath, csvSchema, this.data, this.header);
     }
     logger.warn(`No CSV schema provided for ${this.filePath}`);
     return true;

src/modules/CSVURLModule.js

@@ -2,7 +2,7 @@ const axios = require('axios');
 const moment = require('moment');
 const logger = require('../helpers/logger');
 const { validateCSV } = require('../helpers/csvValidator');
-const { csvParse, stringNormalizer, normalizeEmptyValues } = require('../helpers/csvParsingUtils');
+const { csvParse, stringNormalizer, normalizeEmptyValues, getCSVHeader } = require('../helpers/csvParsingUtils');
 
 class CSVURLModule {
   constructor(url, unalterableColumns, parserOptions) {
@@ -28,6 +28,7 @@ class CSVURLModule {
       const parsedData = csvParse(csvData, this.parserOptions);
       logger.debug('CSV Data parsing successful');
       this.data = normalizeEmptyValues(parsedData, this.unalterableColumns);
+      this.header = getCSVHeader(csvData);
     }
   }
 
@@ -55,7 +56,7 @@ class CSVURLModule {
 
     if (csvSchema) {
       this.data = normalizeEmptyValues(this.data, this.unalterableColumns);
-      return validateCSV(this.url, csvSchema, this.data);
+      return validateCSV(this.url, csvSchema, this.data, this.header);
     }
     logger.warn(`No CSV schema provided for data found at ${this.url}`);
     return true;

test/helpers/csvValidator.test.js

@@ -83,26 +83,30 @@ const schema = {
 
 describe('csvValidator', () => {
   test('simple data validates', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA)).toBe(true);
+    expect(validateCSV('', schema, SIMPLE_DATA, Object.keys(SIMPLE_DATA[0]))).toBe(true);
   });
 
   test('data missing required value does not validate', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_REQUIRED_VALUE)).toBe(false);
+    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_REQUIRED_VALUE, Object.keys(SIMPLE_DATA_MISSING_REQUIRED_VALUE[0]).map((h) => h.toLowerCase()))).toBe(false);
   });
 
   test('data missing required header does not validate', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_HEADER)).toBe(false);
+    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_HEADER, Object.keys(SIMPLE_DATA_MISSING_HEADER[0]).map((h) => h.toLowerCase()))).toBe(false);
   });
 
   test('data with erroneous column should still validate', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA_EXTRA_COLUMNS)).toBe(true);
+    expect(validateCSV('', schema, SIMPLE_DATA_EXTRA_COLUMNS, Object.keys(SIMPLE_DATA_EXTRA_COLUMNS[0]).map((h) => h.toLowerCase()))).toBe(true);
   });
 
   test('data missing an optional column should still validate', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_OPTIONAL_COLUMN)).toBe(true);
+    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_OPTIONAL_COLUMN, Object.keys(SIMPLE_DATA_MISSING_OPTIONAL_COLUMN[0]).map((h) => h.toLowerCase()))).toBe(true);
   });
 
   test('data with different casing in the column header should still validate', () => {
-    expect(validateCSV('', schema, SIMPLE_DATA_DIFFERENT_CASING)).toBe(true);
+    expect(validateCSV('', schema, SIMPLE_DATA_DIFFERENT_CASING, Object.keys(SIMPLE_DATA_DIFFERENT_CASING[0]).map((h) => h.toLowerCase()))).toBe(true);
+  });
+
+  test('data with only the header but no rows should still validate', () => {
+    expect(validateCSV('', schema, [], ['header1', 'header2', 'header3'])).toBe(true);
   });
 });