Deal with DSV files missing columns in certain rows

Author: mmarx

nemo/src/io/formats/dsv/reader.rs

@@ -76,6 +76,7 @@ impl<T: BufRead> DsvReader<T> {
             .has_headers(self.ignore_headers)
             .double_quote(true)
             .quoting(self.quoting)
+            .flexible(true)
             .from_reader(self.read)
     }
 
@@ -107,6 +108,11 @@ impl<T: BufRead> DsvReader<T> {
         let mut line_count: u64 = 0;
         let mut drop_count: u64 = 0;
         for row in dsv_reader.records().flatten() {
+            if row.len() != expected_file_arity {
+                drop_count += 1;
+                continue;
+            }
+
             for (idx, value) in row.iter().enumerate() {
                 if idx >= expected_file_arity || skip[idx] {
                     continue;

resources/testcases/data-formats/dsv/missing-column.rls

@@ -0,0 +1,8 @@
+@import noHeader :- csv{ resource="sources/missing-columns.csv", ignore_headers=true } .
+@import header :- csv{ resource="sources/missing-columns.csv", ignore_headers=false } .
+
+withHeader(?x, ?y) :- header(?x, ?y) .
+withoutHeader(?x, ?y) :- noHeader(?x, ?y) .
+
+@export withHeader :- csv{} .
+@export withoutHeader :- csv{} .

resources/testcases/data-formats/dsv/missing-column/withHeader.csv

@@ -0,0 +1 @@
+23,42

resources/testcases/data-formats/dsv/missing-column/withoutHeader.csv

@@ -0,0 +1 @@
+23,42

resources/testcases/data-formats/dsv/sources/missing-columns.csv

@@ -0,0 +1,3 @@
+column
+23, 42
+1337