Merge pull request #141 from mcode/validate-url-csv-module

Better logging in the CSVURLModule

Author: dmendelowitz

src/client/BaseClient.js

@@ -43,19 +43,23 @@ class BaseClient {
     // Using Reduce to compute the validity of all extractors
     const allExtractorsValid = await this.extractors.reduce(async (curExtractorsValid, curExtractor) => {
       const { name } = curExtractor.constructor;
-
       if (curExtractor.validate) {
         logger.debug(`Validating ${name}`);
-        const isExtractorValid = await curExtractor.validate();
-        if (isExtractorValid) {
-          logger.debug(`Extractor ${name} PASSED CSV validation`);
-        } else {
-          logger.warn(`Extractor ${name} FAILED CSV validation`);
+        try {
+          const isExtractorValid = await curExtractor.validate();
+          if (isExtractorValid) {
+            logger.debug(`Extractor ${name} PASSED CSV validation`);
+          } else {
+            logger.warn(`Extractor ${name} FAILED CSV validation`);
+          }
+          return ((await curExtractorsValid) && isExtractorValid);
+        } catch (e) {
+          logger.warn(`Extractor ${name} could not validate. Encountered the following error: ${e.message}`);
+          return false;
         }
-        return (curExtractorsValid && isExtractorValid);
       }
       return curExtractorsValid;
-    }, true);
+    }, Promise.resolve(true));
 
     if (allExtractorsValid) {
       logger.info('Validation succeeded');

src/helpers/csvParsingUtils.js

@@ -7,6 +7,7 @@ function stringNormalizer(str) {
 
 // For translating null/nil-like values into empty strings
 function normalizeEmptyValues(data, unalterableColumns = []) {
+  logger.debug('Checking for empty CSV values to normalize');
   const EMPTY_VALUES = ['null', 'nil'].map(stringNormalizer);
   const normalizedUnalterableColumns = unalterableColumns.map(stringNormalizer);
   // Flag tracking if empty values were normalized or not.

src/modules/CSVURLModule.js

@@ -16,12 +16,20 @@ class CSVURLModule {
   // If data is already cached, this function does nothing
   async fillDataCache() {
     if (!this.data) {
-      const csvData = await axios.get(this.url).then((res) => res.data);
+      logger.debug('Filling the data cache of CSVURLModule');
+      const csvData = await axios.get(this.url)
+        .then((res) => res.data)
+        .catch((e) => {
+          logger.error('Error occurred when making a connection to this url');
+          throw e;
+        });
+      logger.debug('Web request successful');
       // Parse then normalize the data
       const parsedData = parse(csvData, {
         columns: (header) => header.map((column) => stringNormalizer(column)),
         bom: true,
       });
+      logger.debug('CSV Data parsing successful');
       this.data = normalizeEmptyValues(parsedData, this.unalterableColumns);
     }
   }