From 916e44838734169e9f9c8cc537a65590d7d75151 Mon Sep 17 00:00:00 2001 From: Juanjo Diaz Date: Sat, 19 Oct 2019 15:58:31 +0200 Subject: [PATCH] feat: Add support for flattening arrays and change transforms arguments to an object. (#432) --- README.md | 24 +++++++++++-------- bin/json2csv.js | 25 +++++++++++++++----- lib/transforms/flatten.js | 22 +++++++++++------- lib/transforms/unwind.js | 2 +- test/CLI.js | 16 ++++++++++--- test/JSON2CSVAsyncParser.js | 26 +++++++++++++++++---- test/JSON2CSVParser.js | 22 ++++++++++++++---- test/JSON2CSVTransform.js | 33 ++++++++++++++++++++++----- test/fixtures/csv/flattenedArrays.csv | 3 +++ test/fixtures/json/flattenArrays.json | 12 ++++++++++ 10 files changed, 142 insertions(+), 43 deletions(-) create mode 100644 test/fixtures/csv/flattenedArrays.csv create mode 100644 test/fixtures/json/flattenArrays.json diff --git a/README.md b/README.md index ef111ce1..93b57961 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,10 @@ Options: -b, --with-bom Includes BOM character at the beginning of the CSV. -p, --pretty Print output as a pretty table. Use only when printing to console. -u, --unwind Creates multiple rows from a single JSON document similar to MongoDB unwind. - -B, --unwind-blank When unwinding, blank out instead of repeating data. - -F, --flatten Flatten nested objects. - -S, --flatten-separator Flattened keys separator. Defaults to '.'. + -B, --unwind-blank When unwinding, blank out instead of repeating data. Defaults to false. + -F, --flatten-objects Flatten nested objects. Defaults to false. + -F, --flatten-arrays Flatten nested arrays. Defaults to false. + -S, --flatten-separator Flattened keys separator. Defaults to '.'. (default: ".") -h, --help output usage information ``` @@ -394,22 +395,24 @@ const { transforms: { unwind, flatten } } = require('json2csv'); The unwind transform deconstructs an array field from the input item to output a row for each element. Is's similar to MongoDB's $unwind aggregation. -The transform needs to be instantiated and takes 2 arguments: +The transform needs to be instantiated and takes an options object as arguments containing: - `paths` - Array of String, list the paths to the fields to be unwound. It's mandatory and should not be empty. -- `blank` - Boolean, unwind using blank values instead of repeating data. Defaults to `false`. +- `blankOut` - Boolean, unwind using blank values instead of repeating data. Defaults to `false`. ```js // Default -unwind(['fieldToUnwind']); +unwind({ paths: ['fieldToUnwind'] }); // Blanking out repeated data -unwind(['fieldToUnwind'], true); +unwind({ paths: ['fieldToUnwind'], blankOut: true }); ``` ##### Flatten Flatten nested javascript objects into a single level object. -The transform needs to be instantiated and takes 1 argument: +The transform needs to be instantiated and takes an options object as arguments containing: +- `objects` - Boolean, whether to flatten JSON objects or not. Defaults to `true`. +- `arrays`- Boolean, whether to flatten Arrays or not. Defaults to `false`. - `separator` - String, separator to use between nested JSON keys when flattening a field. Defaults to `.`. ```js @@ -417,7 +420,10 @@ The transform needs to be instantiated and takes 1 argument: flatten(); // Custom separator '__' -flatten('_'); +flatten({ separator: '_' }); + +// Flatten only arrays +flatten({ objects: false, arrays: true }); ``` ### Javascript module examples diff --git a/bin/json2csv.js b/bin/json2csv.js index 2a1158c6..0769a794 100755 --- a/bin/json2csv.js +++ b/bin/json2csv.js @@ -40,10 +40,11 @@ program .option('-b, --with-bom', 'Includes BOM character at the beginning of the CSV.') .option('-p, --pretty', 'Print output as a pretty table. Use only when printing to console.') // Built-in transforms - .option('-u, --unwind ', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.') - .option('-B, --unwind-blank', 'When unwinding, blank out instead of repeating data.') - .option('-F, --flatten', 'Flatten nested objects.') - .option('-S, --flatten-separator ', 'Flattened keys separator. Defaults to \'.\'.') + .option('--unwind ', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.') + .option('--unwind-blank', 'When unwinding, blank out instead of repeating data. Defaults to false.', false) + .option('--flatten-objects', 'Flatten nested objects. Defaults to false.', false) + .option('--flatten-arrays', 'Flatten nested arrays. Defaults to false.', false) + .option('--flatten-separator ', 'Flattened keys separator. Defaults to \'.\'.', '.') .parse(process.argv); function makePathAbsolute(filePath) { @@ -136,8 +137,20 @@ async function processStream(config, opts) { const config = Object.assign({}, program.config ? require(program.config) : {}, program); const transforms = []; - if (config.unwind) transforms.push(unwind(config.unwind.split(','), config.unwindBlank || false)); - if (config.flatten) transforms.push(flatten(config.flattenSeparator || '.')); + if (config.unwind) { + transforms.push(unwind({ + paths: config.unwind.split(','), + blankOut: config.unwindBlank + })); + } + + if (config.flattenObjects || config.flattenArrays) { + transforms.push(flatten({ + objects: config.flattenObjects, + arrays: config.flattenArrays, + separator: config.flattenSeparator + })); + } const opts = { transforms, diff --git a/lib/transforms/flatten.js b/lib/transforms/flatten.js index 11a3cfc2..a7dcf1b8 100644 --- a/lib/transforms/flatten.js +++ b/lib/transforms/flatten.js @@ -4,22 +4,28 @@ * @param {String} separator Separator to be used as the flattened field name * @returns {Object => Object} Flattened object */ -function flatten(separator = '.') { +function flatten({ objects = true, arrays = false, separator = '.' } = {}) { function step (obj, flatDataRow, currentPath) { Object.keys(obj).forEach((key) => { const newPath = currentPath ? `${currentPath}${separator}${key}` : key; const value = obj[key]; - if (typeof value !== 'object' - || value === null - || Array.isArray(value) - || Object.prototype.toString.call(value.toJSON) === '[object Function]' - || !Object.keys(value).length) { - flatDataRow[newPath] = value; + if (objects + && typeof value === 'object' + && value !== null + && !Array.isArray(value) + && Object.prototype.toString.call(value.toJSON) !== '[object Function]' + && Object.keys(value).length) { + step(value, flatDataRow, newPath); return; } - step(value, flatDataRow, newPath); + if (arrays && Array.isArray(value)) { + step(value, flatDataRow, newPath); + return; + } + + flatDataRow[newPath] = value; }); return flatDataRow; diff --git a/lib/transforms/unwind.js b/lib/transforms/unwind.js index fa70f021..79b8c38d 100644 --- a/lib/transforms/unwind.js +++ b/lib/transforms/unwind.js @@ -8,7 +8,7 @@ const { setProp, flattenReducer } = require('../utils'); * @param {String[]} unwindPaths The paths as strings to be used to deconstruct the array * @returns {Object => Array} Array of objects containing all rows after unwind of chosen paths */ -function unwind(paths, blankOut = false) { +function unwind({ paths = [], blankOut = false } = {}) { function unwindReducer(rows, unwindPath) { return rows .map(row => { diff --git a/test/CLI.js b/test/CLI.js index fb7d8388..132cea01 100644 --- a/test/CLI.js +++ b/test/CLI.js @@ -777,7 +777,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); testRunner.add('should support flattening deep JSON using the flatten transform', (t) => { - const opts = '--flatten'; + const opts = '--flatten-objects'; exec(`${cli} -i "${getFixturePath('/json/deepJSON.json')}" ${opts}`, (err, stdout, stderr) => { t.notOk(stderr); @@ -786,9 +786,19 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { t.end(); }); }); + testRunner.add('should support flattening JSON with nested arrays using the flatten transform', (t) => { + const opts = '--flatten-objects --flatten-arrays'; + + exec(`${cli} -i "${getFixturePath('/json/flattenArrays.json')}" ${opts}`, (err, stdout, stderr) => { + t.notOk(stderr); + const csv = stdout; + t.equal(csv, csvFixtures.flattenedArrays); + t.end(); + }); + }); testRunner.add('should support custom flatten separator using the flatten transform', (t) => { - const opts = '--flatten --flatten-separator __'; + const opts = '--flatten-objects --flatten-separator __'; exec(`${cli} -i "${getFixturePath('/json/deepJSON.json')}" ${opts}`, (err, stdout, stderr) => { t.notOk(stderr); @@ -799,7 +809,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); testRunner.add('should support multiple transforms and honor the order in which they are declared', (t) => { - const opts = '--unwind items --flatten'; + const opts = '--unwind items --flatten-objects'; exec(`${cli} -i "${getFixturePath('/json/unwindAndFlatten.json')}" ${opts}`, (err, stdout, stderr) => { t.notOk(stderr); diff --git a/test/JSON2CSVAsyncParser.js b/test/JSON2CSVAsyncParser.js index 4ce4e71b..6f7da6f3 100644 --- a/test/JSON2CSVAsyncParser.js +++ b/test/JSON2CSVAsyncParser.js @@ -1131,7 +1131,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = testRunner.add('should support unwinding an object into multiple rows using the unwind transform', async (t) => { const opts = { fields: ['carModel', 'price', 'colors'], - transforms: [unwind(['colors'])], + transforms: [unwind({ paths: ['colors'] })], }; const parser = new AsyncParser(opts); @@ -1148,7 +1148,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = testRunner.add('should support multi-level unwind using the unwind transform', async (t) => { const opts = { fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'], - transforms: [unwind(['extras.items', 'extras.items.items'])], + transforms: [unwind({ paths: ['extras.items', 'extras.items.items'] })], }; const parser = new AsyncParser(opts); @@ -1165,7 +1165,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = testRunner.add('should support unwind and blank out repeated data using the unwind transform', async (t) => { const opts = { fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'], - transforms: [unwind(['extras.items', 'extras.items.items'], true)], + transforms: [unwind({ paths: ['extras.items', 'extras.items.items'], blankOut: true })], }; const parser = new AsyncParser(opts); @@ -1195,9 +1195,25 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = t.end(); }); + testRunner.add('should support flattening JSON with nested arrays using the flatten transform', async (t) => { + const opts = { + transforms: [flatten({ arrays: true })], + }; + const parser = new AsyncParser(opts); + + try { + const csv = await parser.fromInput(jsonFixtures.flattenArrays()).promise(); + t.equal(csv, csvFixtures.flattenedArrays); + } catch(err) { + t.fail(err.message); + } + + t.end(); + }); + testRunner.add('should support custom flatten separator using the flatten transform', async (t) => { const opts = { - transforms: [flatten('__')], + transforms: [flatten({ separator: '__' })], }; const parser = new AsyncParser(opts); @@ -1213,7 +1229,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = testRunner.add('should support multiple transforms and honor the order in which they are declared', async (t) => { const opts = { - transforms: [unwind(['items']), flatten()], + transforms: [unwind({ paths: ['items'] }), flatten()], }; const parser = new AsyncParser(opts); diff --git a/test/JSON2CSVParser.js b/test/JSON2CSVParser.js index 8040cd5f..d7177bc9 100644 --- a/test/JSON2CSVParser.js +++ b/test/JSON2CSVParser.js @@ -672,7 +672,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { testRunner.add('should support unwinding an object into multiple rows using the unwind transform', (t) => { const opts = { fields: ['carModel', 'price', 'colors'], - transforms: [unwind(['colors'])], + transforms: [unwind({ paths: ['colors'] })], }; const parser = new Json2csvParser(opts); @@ -685,7 +685,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { testRunner.add('should support multi-level unwind using the unwind transform', (t) => { const opts = { fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'], - transforms: [unwind(['extras.items', 'extras.items.items'])], + transforms: [unwind({ paths: ['extras.items', 'extras.items.items'] })], }; const parser = new Json2csvParser(opts); @@ -698,7 +698,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { testRunner.add('should support unwind and blank out repeated data using the unwind transform', (t) => { const opts = { fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'], - transforms: [unwind(['extras.items', 'extras.items.items'], true)], + transforms: [unwind({ paths: ['extras.items', 'extras.items.items'], blankOut: true })], }; const parser = new Json2csvParser(opts); @@ -733,9 +733,21 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { t.end(); }); + testRunner.add('should support flattening JSON with nested arrays using the flatten transform', (t) => { + const opts = { + transforms: [flatten({ arrays: true })], + }; + + const parser = new Json2csvParser(opts); + const csv = parser.parse(jsonFixtures.flattenArrays); + + t.equal(csv, csvFixtures.flattenedArrays); + t.end(); + }); + testRunner.add('should support custom flatten separator using the flatten transform', (t) => { const opts = { - transforms: [flatten('__')], + transforms: [flatten({ separator: '__' })], }; const parser = new Json2csvParser(opts); @@ -747,7 +759,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { testRunner.add('should support multiple transforms and honor the order in which they are declared', (t) => { const opts = { - transforms: [unwind('items'), flatten()], + transforms: [unwind({ paths: ['items'] }), flatten()], }; const parser = new Json2csvParser(opts); diff --git a/test/JSON2CSVTransform.js b/test/JSON2CSVTransform.js index aad092f8..d519c023 100644 --- a/test/JSON2CSVTransform.js +++ b/test/JSON2CSVTransform.js @@ -1126,7 +1126,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = testRunner.add('should support unwinding an object into multiple rows using the unwind transform', (t) => { const opts = { fields: ['carModel', 'price', 'colors'], - transforms: [unwind(['colors'])], + transforms: [unwind({ paths: ['colors'] })], }; const transform = new Json2csvTransform(opts); @@ -1148,7 +1148,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = testRunner.add('should support multi-level unwind using the unwind transform', (t) => { const opts = { fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'], - transforms: [unwind(['extras.items', 'extras.items.items'])], + transforms: [unwind({ paths: ['extras.items', 'extras.items.items'] })], }; const transform = new Json2csvTransform(opts); @@ -1172,7 +1172,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = testRunner.add('should support unwind and blank out repeated data using the unwind transform', (t) => { const opts = { fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'], - transforms: [unwind(['extras.items', 'extras.items.items'], true)], + transforms: [unwind({ paths: ['extras.items', 'extras.items.items'], blankOut: true })], }; const transform = new Json2csvTransform(opts); @@ -1212,9 +1212,30 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = }); }); + testRunner.add('should support flattening JSON with nested arrays using the flatten transform', (t) => { + const opts = { + transforms: [flatten({ arrays: true })], + }; + + const transform = new Json2csvTransform(opts); + const processor = jsonFixtures.flattenArrays().pipe(transform); + + let csv = ''; + processor + .on('data', chunk => (csv += chunk.toString())) + .on('end', () => { + t.equal(csv, csvFixtures.flattenedArrays); + t.end(); + }) + .on('error', err => { + t.fail(err.message); + t.end(); + }); + }); + testRunner.add('should support custom flatten separator using the flatten transform', (t) => { const opts = { - transforms: [flatten('__')], + transforms: [flatten({ separator: '__' })], }; const transform = new Json2csvTransform(opts); @@ -1235,7 +1256,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = testRunner.add('should support multiple transforms and honor the order in which they are declared', (t) => { const opts = { - transforms: [unwind(['items']), flatten()], + transforms: [unwind({ paths: ['items'] }), flatten()], }; const transform = new Json2csvTransform(opts); @@ -1254,7 +1275,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = }); }); - testRunner.add('should support custom transforms', async (t) => { + testRunner.add('should support custom transforms', (t) => { const opts = { transforms: [row => ({ model: row.carModel, diff --git a/test/fixtures/csv/flattenedArrays.csv b/test/fixtures/csv/flattenedArrays.csv new file mode 100644 index 00000000..526813b8 --- /dev/null +++ b/test/fixtures/csv/flattenedArrays.csv @@ -0,0 +1,3 @@ +"name","age","friends.0.name","friends.0.age","friends.1.name","friends.1.age" +"Jack",39,"Oliver",40,"Harry",50 +"Thomas",40,"Harry",35,, \ No newline at end of file diff --git a/test/fixtures/json/flattenArrays.json b/test/fixtures/json/flattenArrays.json new file mode 100644 index 00000000..f4fea5ff --- /dev/null +++ b/test/fixtures/json/flattenArrays.json @@ -0,0 +1,12 @@ +[ + { + "name": "Jack", + "age": 39, + "friends": [{ "name": "Oliver", "age": 40 }, { "name": "Harry", "age": 50 }] + }, + { + "name": "Thomas", + "age": 40, + "friends": [{ "name": "Harry", "age": 35 }] + } +] \ No newline at end of file