Add support for missing columns in CSV (#163)

* Increase chunk size

* Add support for not checking column correctness

* Fix for test format

desktop/panel/file.test.js

@@ -193,4 +193,45 @@ for (const subprocessName of RUNNERS) {
       }
     );
   }
+
+  const regressions = [
+    {
+      file: path.join('testdata', 'csv', 'missing_columns.csv'),
+      expected: [
+        { a: '3', b: '4' },
+        { a: '1', b: '2' },
+        { a: '5', b: '6' },
+      ],
+      name: 'missing columns in csv',
+    },
+  ];
+
+  for (const regression of regressions) {
+    describe('regression: ' + regression.name, () => {
+      test('correctness', () => {
+        const fp = new FilePanelInfo({
+          name: regression.file,
+        });
+
+        const panels = [fp];
+
+        return withSavedPanels(
+          panels,
+          (project) => {
+            // Grab result
+            const value = JSON.parse(
+              fs
+                .readFileSync(
+                  getProjectResultsFile(project.projectName) + fp.id
+                )
+                .toString()
+            );
+
+            expect(value).toStrictEqual(regression.expected);
+          },
+          { evalPanels: true, subprocessName }
+        );
+      });
+    });
+  }
 }

runner/file.go

@@ -82,6 +82,8 @@ func withJSONArrayOutWriterFile(out io.Writer, cb func(w *JSONArrayWriter) error
 func transformCSV(in io.Reader, out io.Writer, delimiter rune) error {
 	r := csv.NewReader(in)
 	r.Comma = delimiter
+	r.ReuseRecord = true
+	r.FieldsPerRecord = -1
 
 	return withJSONArrayOutWriterFile(out, func(w *JSONArrayWriter) error {
 		isHeader := true

testdata/csv/missing_columns.csv

@@ -0,0 +1,4 @@
+a,b
+3,4
+1,2,3
+5,6