feat: Add support for flattening arrays and change transforms argumen…

…ts to an object. (#432)
zemirco · Oct 19, 2019 · 916e448 · 916e448
1 parent fae7166
commit 916e448
Show file tree

Hide file tree

Showing 10 changed files with 142 additions and 43 deletions.
diff --git a/README.md b/README.md
@@ -74,9 +74,10 @@ Options:
   -b, --with-bom                       Includes BOM character at the beginning of the CSV.
   -p, --pretty                         Print output as a pretty table. Use only when printing to console.
   -u, --unwind <paths>                 Creates multiple rows from a single JSON document similar to MongoDB unwind.
-  -B, --unwind-blank                   When unwinding, blank out instead of repeating data.
-  -F, --flatten                        Flatten nested objects.
-  -S, --flatten-separator <separator>  Flattened keys separator. Defaults to '.'.
+  -B, --unwind-blank                   When unwinding, blank out instead of repeating data. Defaults to false.
+  -F, --flatten-objects                Flatten nested objects. Defaults to false.
+  -F, --flatten-arrays                 Flatten nested arrays. Defaults to false.
+  -S, --flatten-separator <separator>  Flattened keys separator. Defaults to '.'. (default: ".")
   -h, --help                           output usage information
 ```
 
@@ -394,30 +395,35 @@ const { transforms: { unwind, flatten } } = require('json2csv');
 
 The unwind transform deconstructs an array field from the input item to output a row for each element. Is's similar to MongoDB's $unwind aggregation.
 
-The transform needs to be instantiated and takes 2 arguments:
+The transform needs to be instantiated and takes an options object as arguments containing:
 - `paths` - Array of String, list the paths to the fields to be unwound. It's mandatory and should not be empty.
-- `blank` - Boolean, unwind using blank values instead of repeating data. Defaults to `false`.
+- `blankOut` - Boolean, unwind using blank values instead of repeating data. Defaults to `false`.
 
 ```js
 // Default
-unwind(['fieldToUnwind']);
+unwind({ paths: ['fieldToUnwind'] });
 
 // Blanking out repeated data
-unwind(['fieldToUnwind'], true);
+unwind({ paths: ['fieldToUnwind'], blankOut: true });
 ```
 
 ##### Flatten
 Flatten nested javascript objects into a single level object.
 
-The transform needs to be instantiated and takes 1 argument:
+The transform needs to be instantiated and takes an options object as arguments containing:
+- `objects` - Boolean, whether to flatten JSON objects or not. Defaults to `true`.
+- `arrays`- Boolean, whether to flatten Arrays or not. Defaults to `false`.
 - `separator` - String, separator to use between nested JSON keys when flattening a field. Defaults to `.`.
 
 ```js
 // Default
 flatten();
 
 // Custom separator '__'
-flatten('_');
+flatten({ separator: '_' });
+
+// Flatten only arrays
+flatten({ objects: false, arrays: true });
 ```
 
 ### Javascript module examples

diff --git a/bin/json2csv.js b/bin/json2csv.js
@@ -40,10 +40,11 @@ program
   .option('-b, --with-bom', 'Includes BOM character at the beginning of the CSV.')
   .option('-p, --pretty', 'Print output as a pretty table. Use only when printing to console.')
   // Built-in transforms
-  .option('-u, --unwind <paths>', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.')
-  .option('-B, --unwind-blank', 'When unwinding, blank out instead of repeating data.')
-  .option('-F, --flatten', 'Flatten nested objects.')
-  .option('-S, --flatten-separator <separator>', 'Flattened keys separator. Defaults to \'.\'.')
+  .option('--unwind <paths>', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.')
+  .option('--unwind-blank', 'When unwinding, blank out instead of repeating data. Defaults to false.', false)
+  .option('--flatten-objects', 'Flatten nested objects. Defaults to false.', false)
+  .option('--flatten-arrays', 'Flatten nested arrays. Defaults to false.', false)
+  .option('--flatten-separator <separator>', 'Flattened keys separator. Defaults to \'.\'.', '.')
   .parse(process.argv);
 
 function makePathAbsolute(filePath) {
@@ -136,8 +137,20 @@ async function processStream(config, opts) {
     const config = Object.assign({}, program.config ? require(program.config) : {}, program);
 
     const transforms = [];
-    if (config.unwind) transforms.push(unwind(config.unwind.split(','), config.unwindBlank || false));
-    if (config.flatten) transforms.push(flatten(config.flattenSeparator || '.'));
+    if (config.unwind) {
+      transforms.push(unwind({
+        paths: config.unwind.split(','),
+        blankOut: config.unwindBlank
+      }));
+    }
+
+    if (config.flattenObjects || config.flattenArrays) {
+      transforms.push(flatten({
+        objects: config.flattenObjects,
+        arrays: config.flattenArrays,
+        separator: config.flattenSeparator
+      }));
+    }
 
     const opts = {
       transforms,

diff --git a/lib/transforms/flatten.js b/lib/transforms/flatten.js
@@ -4,22 +4,28 @@
  * @param {String} separator Separator to be used as the flattened field name
  * @returns {Object => Object} Flattened object
  */
-function flatten(separator = '.') {
+function flatten({ objects = true, arrays = false, separator = '.' } = {}) {
   function step (obj, flatDataRow, currentPath) {
     Object.keys(obj).forEach((key) => {
       const newPath = currentPath ? `${currentPath}${separator}${key}` : key;
       const value = obj[key];
 
-      if (typeof value !== 'object'
-        || value === null
-        || Array.isArray(value)
-        || Object.prototype.toString.call(value.toJSON) === '[object Function]'
-        || !Object.keys(value).length) {
-        flatDataRow[newPath] = value;
+      if (objects
+        && typeof value === 'object'
+        && value !== null
+        && !Array.isArray(value)
+        && Object.prototype.toString.call(value.toJSON) !== '[object Function]'
+        && Object.keys(value).length) {
+        step(value, flatDataRow, newPath);
         return;
       }
 
-      step(value, flatDataRow, newPath);
+      if (arrays && Array.isArray(value)) {
+        step(value, flatDataRow, newPath);
+        return;
+      }
+
+      flatDataRow[newPath] = value;
     });
 
     return flatDataRow;

diff --git a/lib/transforms/unwind.js b/lib/transforms/unwind.js
@@ -8,7 +8,7 @@ const { setProp, flattenReducer } = require('../utils');
  * @param {String[]} unwindPaths The paths as strings to be used to deconstruct the array
  * @returns {Object => Array} Array of objects containing all rows after unwind of chosen paths
 */
-function unwind(paths, blankOut = false) {
+function unwind({ paths = [], blankOut = false } = {}) {
   function unwindReducer(rows, unwindPath) {
     return rows
       .map(row => {

diff --git a/test/CLI.js b/test/CLI.js
@@ -777,7 +777,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   });
 
   testRunner.add('should support flattening deep JSON using the flatten transform', (t) => {
-    const opts = '--flatten';
+    const opts = '--flatten-objects';
 
     exec(`${cli} -i "${getFixturePath('/json/deepJSON.json')}" ${opts}`, (err, stdout, stderr) => {
       t.notOk(stderr);
@@ -786,9 +786,19 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
       t.end();
     });
   });
+  testRunner.add('should support flattening JSON with nested arrays using the flatten transform', (t) => {
+    const opts = '--flatten-objects --flatten-arrays';
+
+    exec(`${cli} -i "${getFixturePath('/json/flattenArrays.json')}" ${opts}`, (err, stdout, stderr) => {
+      t.notOk(stderr);
+      const csv = stdout;
+      t.equal(csv, csvFixtures.flattenedArrays);
+      t.end();
+    });
+  });
 
   testRunner.add('should support custom flatten separator using the flatten transform', (t) => {
-    const opts = '--flatten --flatten-separator __';
+    const opts = '--flatten-objects --flatten-separator __';
 
     exec(`${cli} -i "${getFixturePath('/json/deepJSON.json')}" ${opts}`, (err, stdout, stderr) => {
       t.notOk(stderr);
@@ -799,7 +809,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   });
 
   testRunner.add('should support multiple transforms and honor the order in which they are declared', (t) => {
-    const opts = '--unwind items --flatten';
+    const opts = '--unwind items --flatten-objects';
 
     exec(`${cli} -i "${getFixturePath('/json/unwindAndFlatten.json')}" ${opts}`, (err, stdout, stderr) => {
       t.notOk(stderr);

diff --git a/test/JSON2CSVAsyncParser.js b/test/JSON2CSVAsyncParser.js
@@ -1131,7 +1131,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
   testRunner.add('should support unwinding an object into multiple rows using the unwind transform', async (t) => {
     const opts = {
       fields: ['carModel', 'price', 'colors'],
-      transforms: [unwind(['colors'])],
+      transforms: [unwind({ paths: ['colors'] })],
     };
     const parser = new AsyncParser(opts);
 
@@ -1148,7 +1148,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
   testRunner.add('should support multi-level unwind using the unwind transform', async (t) => {
     const opts = {
       fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'],
-      transforms: [unwind(['extras.items', 'extras.items.items'])],
+      transforms: [unwind({ paths: ['extras.items', 'extras.items.items'] })],
     };
     const parser = new AsyncParser(opts);
 
@@ -1165,7 +1165,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
   testRunner.add('should support unwind and blank out repeated data using the unwind transform', async (t) => {
     const opts = {
       fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'],
-      transforms: [unwind(['extras.items', 'extras.items.items'], true)],
+      transforms: [unwind({ paths: ['extras.items', 'extras.items.items'], blankOut: true })],
     };
     const parser = new AsyncParser(opts);
 
@@ -1195,9 +1195,25 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
     t.end();
   });
 
+  testRunner.add('should support flattening JSON with nested arrays using the flatten transform', async (t) => {
+    const opts = {
+      transforms: [flatten({ arrays: true })],
+    };
+    const parser = new AsyncParser(opts);
+
+    try {
+      const csv = await parser.fromInput(jsonFixtures.flattenArrays()).promise();
+      t.equal(csv, csvFixtures.flattenedArrays);
+    } catch(err) {
+      t.fail(err.message);
+    }
+
+    t.end();
+  });
+
   testRunner.add('should support custom flatten separator using the flatten transform', async (t) => {
     const opts = {
-      transforms: [flatten('__')],
+      transforms: [flatten({ separator: '__' })],
     };
     const parser = new AsyncParser(opts);
 
@@ -1213,7 +1229,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
 
   testRunner.add('should support multiple transforms and honor the order in which they are declared', async (t) => {
     const opts = {
-      transforms: [unwind(['items']), flatten()],
+      transforms: [unwind({ paths: ['items'] }), flatten()],
     };
     const parser = new AsyncParser(opts);
 

diff --git a/test/JSON2CSVParser.js b/test/JSON2CSVParser.js
@@ -672,7 +672,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   testRunner.add('should support unwinding an object into multiple rows using the unwind transform', (t) => {
     const opts = {
       fields: ['carModel', 'price', 'colors'],
-      transforms: [unwind(['colors'])],
+      transforms: [unwind({ paths: ['colors'] })],
     };
 
     const parser = new Json2csvParser(opts);
@@ -685,7 +685,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   testRunner.add('should support multi-level unwind using the unwind transform', (t) => {
     const opts = {
       fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'],
-      transforms: [unwind(['extras.items', 'extras.items.items'])],
+      transforms: [unwind({ paths: ['extras.items', 'extras.items.items'] })],
     };
 
     const parser = new Json2csvParser(opts);
@@ -698,7 +698,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
   testRunner.add('should support unwind and blank out repeated data using the unwind transform', (t) => {
     const opts = {
       fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'],
-      transforms: [unwind(['extras.items', 'extras.items.items'], true)],
+      transforms: [unwind({ paths: ['extras.items', 'extras.items.items'], blankOut: true })],
     };
 
     const parser = new Json2csvParser(opts);
@@ -733,9 +733,21 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
     t.end();
   });
 
+  testRunner.add('should support flattening JSON with nested arrays using the flatten transform', (t) => {
+    const opts = {
+      transforms: [flatten({ arrays: true })],
+    };
+
+    const parser = new Json2csvParser(opts);
+    const csv = parser.parse(jsonFixtures.flattenArrays);
+
+    t.equal(csv, csvFixtures.flattenedArrays);
+    t.end();
+  });
+
   testRunner.add('should support custom flatten separator using the flatten transform', (t) => {
     const opts = {
-      transforms: [flatten('__')],
+      transforms: [flatten({ separator: '__' })],
     };
 
     const parser = new Json2csvParser(opts);
@@ -747,7 +759,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
 
   testRunner.add('should support multiple transforms and honor the order in which they are declared', (t) => {
     const opts = {
-      transforms: [unwind('items'), flatten()],
+      transforms: [unwind({ paths: ['items'] }), flatten()],
     };
 
     const parser = new Json2csvParser(opts);

diff --git a/test/JSON2CSVTransform.js b/test/JSON2CSVTransform.js
@@ -1126,7 +1126,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
   testRunner.add('should support unwinding an object into multiple rows using the unwind transform', (t) => {
     const opts = {
       fields: ['carModel', 'price', 'colors'],
-      transforms: [unwind(['colors'])],
+      transforms: [unwind({ paths: ['colors'] })],
     };
 
     const transform = new Json2csvTransform(opts);
@@ -1148,7 +1148,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
   testRunner.add('should support multi-level unwind using the unwind transform', (t) => {
     const opts = {
       fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'],
-      transforms: [unwind(['extras.items', 'extras.items.items'])],
+      transforms: [unwind({ paths: ['extras.items', 'extras.items.items'] })],
     };
 
     const transform = new Json2csvTransform(opts);
@@ -1172,7 +1172,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
   testRunner.add('should support unwind and blank out repeated data using the unwind transform', (t) => {
     const opts = {
       fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'],
-      transforms: [unwind(['extras.items', 'extras.items.items'], true)],
+      transforms: [unwind({ paths: ['extras.items', 'extras.items.items'], blankOut: true })],
     };
 
     const transform = new Json2csvTransform(opts);
@@ -1212,9 +1212,30 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
       });
   });
 
+  testRunner.add('should support flattening JSON with nested arrays using the flatten transform', (t) => {
+    const opts = {
+      transforms: [flatten({ arrays: true })],
+    };
+
+    const transform = new Json2csvTransform(opts);
+    const processor = jsonFixtures.flattenArrays().pipe(transform);
+
+    let csv = '';
+    processor
+      .on('data', chunk => (csv += chunk.toString()))
+      .on('end', () => {
+        t.equal(csv, csvFixtures.flattenedArrays);
+        t.end();
+      })
+      .on('error', err => {
+        t.fail(err.message);
+        t.end();  
+      });
+  });
+
   testRunner.add('should support custom flatten separator using the flatten transform', (t) => {
     const opts = {
-      transforms: [flatten('__')],
+      transforms: [flatten({ separator: '__' })],
     };
 
     const transform = new Json2csvTransform(opts);
@@ -1235,7 +1256,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
 
   testRunner.add('should support multiple transforms and honor the order in which they are declared', (t) => {
     const opts = {
-      transforms: [unwind(['items']), flatten()],
+      transforms: [unwind({ paths: ['items'] }), flatten()],
     };
 
     const transform = new Json2csvTransform(opts);
@@ -1254,7 +1275,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
       });
   });
 
-  testRunner.add('should support custom transforms', async (t) => {
+  testRunner.add('should support custom transforms', (t) => {
     const opts = {
       transforms: [row => ({
         model: row.carModel,

diff --git a/test/fixtures/csv/flattenedArrays.csv b/test/fixtures/csv/flattenedArrays.csv
@@ -0,0 +1,3 @@
+"name","age","friends.0.name","friends.0.age","friends.1.name","friends.1.age"
+"Jack",39,"Oliver",40,"Harry",50
+"Thomas",40,"Harry",35,,
diff --git a/test/fixtures/json/flattenArrays.json b/test/fixtures/json/flattenArrays.json
@@ -0,0 +1,12 @@
+[
+  {
+    "name": "Jack",
+    "age": 39,
+    "friends": [{ "name": "Oliver", "age": 40 }, { "name": "Harry", "age": 50 }]
+  },
+  {
+    "name": "Thomas",
+    "age": 40,
+    "friends": [{ "name": "Harry", "age": 35 }]
+  }
+]