Update to switch to custom implementation of validation with tests

Author: Matthew Gramigna

package-lock.json

@@ -22,7 +22,6 @@
     "ajv": "^6.12.6",
     "antlr4": "4.8.0",
     "commander": "^6.2.0",
-    "csv-file-validator": "^1.10.1",
     "csv-parse": "^4.8.8",
     "fhir-crud-client": "^1.2.2",
     "fhirpath": "2.1.5",

package.json

@@ -25,8 +25,10 @@ class BaseClient {
   }
 
   // Given an extractor configuration, initialize all the necessary extractors
-  async initializeExtractors(extractorConfig, commonExtractorArgs) {
-    const extractorInits = extractorConfig.map(async (curExtractorConfig) => {
+  initializeExtractors(extractorConfig, commonExtractorArgs) {
+    let allExtractorsValid = true;
+
+    extractorConfig.forEach((curExtractorConfig) => {
       const { label, type, constructorArgs } = curExtractorConfig;
       logger.debug(`Initializing ${label} extractor with type ${type}`);
       const ExtractorClass = this.extractorClasses[type];
@@ -35,20 +37,26 @@ class BaseClient {
         const newExtractor = new ExtractorClass({ ...commonExtractorArgs, ...constructorArgs });
 
         if (newExtractor.validate) {
-          await newExtractor.validate();
+          const isExtractorValid = newExtractor.validate();
+          allExtractorsValid = (allExtractorsValid && isExtractorValid);
+          if (isExtractorValid) {
+            logger.debug(`Extractor ${label} PASSED CSV validation`);
+          } else {
+            logger.debug(`Extractor ${label} FAILED CSV validation`);
+          }
         }
 
-        return newExtractor;
+        this.extractors.push(newExtractor);
       } catch (e) {
-        throw new Error(`Unable to initialize ${label} extractor with type ${type}`);
+        throw new Error(`Unable to initialize ${label} extractor with type ${type}: ${e.message}`);
       }
     });
 
-    await Promise.all(extractorInits).then((extractors) => {
-      this.extractors.push(...extractors);
-    });
-
-    logger.info('Validation succeeded');
+    if (allExtractorsValid) {
+      logger.info('Validation succeeded');
+    } else {
+      throw new Error('Error occurred during CSV validation');
+    }
   }
 
   // NOTE: Async because in other clients that extend this, we need async helper functions (ex. auth)

src/client/BaseClient.js

@@ -12,13 +12,13 @@ class BaseCSVExtractor extends Extractor {
     this.csvModule = new CSVModule(this.filePath);
   }
 
-  async validate() {
+  validate() {
     if (this.csvSchema) {
       logger.info(`Validating CSV file for ${this.filePath}`);
-      await validateCSV(this.filePath, this.csvSchema);
-    } else {
-      logger.warn(`No CSV schema provided for ${this.filePath}`);
+      return validateCSV(this.filePath, this.csvSchema, this.csvModule.data);
     }
+    logger.warn(`No CSV schema provided for ${this.filePath}`);
+    return true;
   }
 }
 

src/extractors/BaseCSVExtractor.js

@@ -1,37 +1,43 @@
 const _ = require('lodash');
-const fs = require('fs');
-const validate = require('csv-file-validator');
-const parse = require('csv-parse/lib/sync');
 const logger = require('./logger');
 
-async function validateCSV(pathToCSVFile, csvSchema) {
-  const csv = fs.readFileSync(pathToCSVFile, { columns: true, encoding: 'utf8' });
+function validateCSV(pathToCSVFile, csvSchema, csvData) {
+  let isValid = true;
 
-  // Use CSV parser to determine actual number of columns in file
-  const csvJson = parse(csv);
+  // Check headers
+  const headers = Object.keys(csvData[0]);
+  const schemaDiff = _.difference(csvSchema.headers.map((h) => h.name), headers);
+  const fileDiff = _.difference(headers, csvSchema.headers.map((h) => h.name));
 
-  if (csvJson[0].length !== csvSchema.headers.length) {
-    // Report which erroneous columns exist in provided CSV
-    const difference = _.difference(csvJson[0], csvSchema.headers.map((h) => h.name));
+  if (fileDiff.length > 0) {
+    logger.warn(`Found extra column(s) in CSV ${pathToCSVFile}: "${fileDiff.join(',')}"`);
+  }
 
-    logger.error(`Validation error in CSV ${pathToCSVFile}: found extra column(s) "${difference}"`);
-    process.exit(1);
+  if (schemaDiff.length > 0) {
+    schemaDiff.forEach((sd) => {
+      const headerSchema = csvSchema.headers.find((h) => h.name === sd);
+      if (headerSchema.required) {
+        logger.error(`Column ${sd} is marked as required but is missing in CSV ${pathToCSVFile}`);
+        isValid = false;
+      } else {
+        logger.warn(`Column ${sd} is missing in CSV ${pathToCSVFile}`);
+      }
+    });
   }
 
-  try {
-    const { inValidMessages } = await validate(csv, csvSchema);
+  // Check values
+  csvData.forEach((row, i) => {
+    Object.entries(row).forEach(([key, value], j) => {
+      const schema = csvSchema.headers.find((h) => h.name === key);
 
-    if (inValidMessages.length > 0) {
-      inValidMessages.forEach((errorMsg) => {
-        logger.error(`Validation error in CSV ${pathToCSVFile}: ${errorMsg}`);
-      });
+      if (schema && schema.required && !value) {
+        logger.error(`Column ${key} marked as required but missing value in row ${i + 1} column ${j + 1} in CSV ${pathToCSVFile}`);
+        isValid = false;
+      }
+    });
+  });
 
-      process.exit(1);
-    }
-  } catch (e) {
-    logger.error(`Error occurred during CSV validation: ${e.message}`);
-    process.exit(1);
-  }
+  return isValid;
 }
 
 module.exports = {

src/helpers/csvValidator.js

@@ -0,0 +1,96 @@
+const { validateCSV } = require('../../src/helpers/csvValidator');
+
+const SIMPLE_DATA = [
+  {
+    header1: '1',
+    header2: '2',
+    header3: '3',
+  },
+  {
+    header1: '4',
+    header2: '',
+    header3: '6',
+  },
+];
+
+const SIMPLE_DATA_MISSING_REQUIRED_VALUE = [
+  {
+    header1: '',
+    header2: '2',
+    header3: '3',
+  },
+  {
+    header1: '4',
+    header2: '2',
+    header3: '3',
+  },
+];
+
+const SIMPLE_DATA_MISSING_HEADER = [
+  {
+    wrongHeader1: '1',
+    header2: '2',
+    header3: '3',
+  },
+  {
+    wrongHeader1: '4',
+    header2: '2',
+    header3: '3',
+  },
+];
+
+const SIMPLE_DATA_EXTRA_COLUMNS = [
+  {
+    header1: '1',
+    header2: '2',
+    header3: '3',
+    header4: '4',
+  },
+  {
+    header1: '5',
+    header2: '6',
+    header3: '7',
+    header4: '',
+  },
+];
+
+const SIMPLE_DATA_MISSING_OPTIONAL_COLUMN = [
+  {
+    header1: '1',
+    header2: '2',
+  },
+  {
+    header1: '3',
+    header2: '4',
+  },
+];
+
+const schema = {
+  headers: [
+    { name: 'header1', required: true },
+    { name: 'header2' },
+    { name: 'header3' },
+  ],
+};
+
+describe('csvValidator', () => {
+  test('simple data validates', () => {
+    expect(validateCSV('', schema, SIMPLE_DATA)).toBe(true);
+  });
+
+  test('data missing required value does not validate', () => {
+    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_REQUIRED_VALUE)).toBe(false);
+  });
+
+  test('data missing required header does not validate', () => {
+    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_HEADER)).toBe(false);
+  });
+
+  test('data with erroneous column should still validate', () => {
+    expect(validateCSV('', schema, SIMPLE_DATA_EXTRA_COLUMNS)).toBe(true);
+  });
+
+  test('data missing an optional column should still validate', () => {
+    expect(validateCSV('', schema, SIMPLE_DATA_MISSING_OPTIONAL_COLUMN)).toBe(true);
+  });
+});