@@ -3,27 +3,34 @@ const moment = require('moment');
3
3
const parse = require ( 'csv-parse/lib/sync' ) ;
4
4
const logger = require ( '../helpers/logger' ) ;
5
5
6
+ // The standard string normalizer function
7
+ function stringNormalizer ( str ) {
8
+ return str . toLowerCase ( ) ;
9
+ }
10
+
11
+ // For translating null/nil-like values into empty strings
6
12
function normalizeEmptyValues ( data , unalterableColumns = [ ] ) {
7
- const EMPTY_VALUES = [ 'null' , 'nil' ] . map ( ( v ) => v . toLowerCase ( ) ) ;
8
- // Flag tracking if data was normalized or not.
9
- let wasNormalized = false ;
13
+ const EMPTY_VALUES = [ 'null' , 'nil' ] . map ( stringNormalizer ) ;
14
+ const normalizedUnalterableColumns = unalterableColumns . map ( stringNormalizer ) ;
15
+ // Flag tracking if empty values were normalized or not.
16
+ let wasEmptyNormalized = false ;
10
17
const newData = data . map ( ( row , i ) => {
11
18
const newRow = { ...row } ;
12
19
// Filter out unalterable columns
13
- const columnsToNormalize = Object . keys ( row ) . filter ( ( col ) => ! unalterableColumns . includes ( col ) ) ;
20
+ const columnsToNormalize = Object . keys ( row ) . filter ( ( col ) => ! normalizedUnalterableColumns . includes ( stringNormalizer ( col ) ) ) ;
14
21
columnsToNormalize . forEach ( ( col ) => {
15
22
const value = newRow [ col ] ;
16
23
// If the value for this row-col combo is a value that should be empty, replace it
17
- if ( EMPTY_VALUES . includes ( value . toLowerCase ( ) ) ) {
24
+ if ( EMPTY_VALUES . includes ( stringNormalizer ( value ) ) ) {
18
25
logger . debug ( `NULL/NIL values '${ value } ' found in row-${ i } , col-${ col } ` ) ;
19
- wasNormalized = true ;
26
+ wasEmptyNormalized = true ;
20
27
newRow [ col ] = '' ;
21
28
}
22
29
} ) ;
23
30
return newRow ;
24
31
} ) ;
25
32
26
- if ( wasNormalized ) {
33
+ if ( wasEmptyNormalized ) {
27
34
logger . warn ( 'NULL/NIL values found and replaced with empty-strings' ) ;
28
35
}
29
36
return newData ;
@@ -33,7 +40,7 @@ class CSVModule {
33
40
constructor ( csvFilePath , unalterableColumns ) {
34
41
// Parse then normalize the data
35
42
const parsedData = parse ( fs . readFileSync ( csvFilePath ) , {
36
- columns : ( header ) => header . map ( ( column ) => column . toLowerCase ( ) ) ,
43
+ columns : ( header ) => header . map ( ( column ) => stringNormalizer ( column ) ) ,
37
44
bom : true ,
38
45
} ) ;
39
46
this . data = normalizeEmptyValues ( parsedData , unalterableColumns ) ;
@@ -43,7 +50,7 @@ class CSVModule {
43
50
logger . debug ( `Get csvModule info by key '${ key } '` ) ;
44
51
// return all rows if key and value aren't provided
45
52
if ( ! key && ! value ) return this . data ;
46
- let result = this . data . filter ( ( d ) => d [ key . toLowerCase ( ) ] === value ) ;
53
+ let result = this . data . filter ( ( d ) => d [ stringNormalizer ( key ) ] === value ) ;
47
54
if ( result . length === 0 ) {
48
55
logger . warn ( `CSV Record with provided key '${ key } ' and value was not found` ) ;
49
56
return result ;
0 commit comments