From 6b2183bdb9b5d7cf6614f56bfb120e2474618072 Mon Sep 17 00:00:00 2001 From: Juanjo Diaz Date: Mon, 19 Aug 2019 17:58:47 +0300 Subject: [PATCH] feat: replace fields config by a global config (#338) --- README.md | 8 ++- bin/json2csv.js | 65 +++++++++---------- test/CLI.js | 20 +++--- .../fields/emptyRowDefaultValues.json | 26 ++++---- test/fixtures/fields/fancyfields.js | 34 +++++----- test/fixtures/fields/fieldNames.json | 16 +++-- test/fixtures/fields/functionNoStringify.js | 12 ++-- .../fields/functionStringifyByDefault.js | 10 +-- test/fixtures/fields/nested.json | 34 +++++----- .../fields/overriddenDefaultValue.json | 12 ++-- .../fields/overriddenDefaultValue2.js | 30 +++++---- 11 files changed, 143 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index f100144d..15aa2c01 100644 --- a/README.md +++ b/README.md @@ -50,17 +50,19 @@ By default, the above script will get the latest release of json2csv. You can al `json2csv` can be called from the command line if installed globally (using the `-g` flag). -```sh +```bash Usage: json2csv [options] + Options: + -V, --version output the version number + -c, --config Specify a file with a valid JSON configuration. -i, --input Path and name of the incoming json file. Defaults to stdin. -o, --output [output] Path and name of the resulting csv file. Defaults to stdout. -n, --ndjson Treat the input as NewLine-Delimited JSON. -s, --no-streaming Process the whole JSON array in memory instead of doing it line by line. -f, --fields List of fields to process. Defaults to field auto-detection. - -c, --fields-config File with a fields configuration as a JSON array. -u, --unwind Creates multiple rows from a single JSON document similar to MongoDB unwind. -B, --unwind-blank When unwinding, blank out instead of repeating data. -F, --flatten Flatten nested objects. @@ -83,6 +85,8 @@ If no output `-o` is specified the result is printed to the console standard out If no fields `-f` or `-c` are passed the fields of the first element are used since json2csv CLI process the items one at a time. You can use the `--no-streaming` flag to load the entire JSON in memory and get all the headers. However, keep in mind that this is slower and requires much more memory. Use `-p` to show the result as a table in the console. +Any option pass through a config file `-c` will be overriden if a specific flag is passed as well. For example, the fields option of the config will be overriden if the fields flag `-f` is used. + ### CLI examples All examples use this example [input file](https://github.com/zemirco/json2csv/blob/master/test/fixtures/json/default.json). diff --git a/bin/json2csv.js b/bin/json2csv.js index 1f95f856..7cecde6a 100755 --- a/bin/json2csv.js +++ b/bin/json2csv.js @@ -18,10 +18,10 @@ program .version(pkg.version) .option('-i, --input ', 'Path and name of the incoming json file. Defaults to stdin.') .option('-o, --output [output]', 'Path and name of the resulting csv file. Defaults to stdout.') + .option('-c, --config ', 'Specify a file with a valid JSON configuration.') .option('-n, --ndjson', 'Treat the input as NewLine-Delimited JSON.') .option('-s, --no-streaming', 'Process the whole JSON array in memory instead of doing it line by line.') .option('-f, --fields ', 'List of fields to process. Defaults to field auto-detection.') - .option('-c, --fields-config ', 'File with a fields configuration as a JSON array.') .option('-u, --unwind ', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.') .option('-B, --unwind-blank', 'When unwinding, blank out instead of repeating data.') .option('-F, --flatten', 'Flatten nested objects.') @@ -46,8 +46,10 @@ function makePathAbsolute(filePath) { const inputPath = makePathAbsolute(program.input); const outputPath = makePathAbsolute(program.output); -const fieldsConfigPath = makePathAbsolute(program.fieldsConfig); +const configPath = makePathAbsolute(program.config); +if (program.fields) program.fields = program.fields.split(','); +if (program.unwind) program.unwind = program.unwind.split(','); program.delimiter = program.delimiter || ','; program.eol = program.eol || os.EOL; @@ -59,14 +61,10 @@ process.stdout.on('error', (error) => { } }); -function getFields() { - if (fieldsConfigPath) { - return require(fieldsConfigPath); - } - - return program.fields - ? program.fields.split(',') - : undefined; +function getConfigFromFile() { + return configPath + ? require(configPath) + : {}; } function getInputStream() { @@ -152,25 +150,27 @@ function processOutput(csv) { Promise.resolve() .then(() => { + const config = Object.assign({}, getConfigFromFile(), program); + const opts = { - fields: getFields(), - unwind: program.unwind ? program.unwind.split(',') : [], - unwindBlank: program.unwindBlank, - flatten: program.flatten, - flattenSeparator: program.flattenSeparator, - defaultValue: program.defaultValue, - quote: program.quote, - doubleQuote: program.doubleQuote, - delimiter: program.delimiter, - eol: program.eol, - excelStrings: program.excelStrings, - header: program.header, - includeEmptyRows: program.includeEmptyRows, - withBOM: program.withBom + fields: config.fields, + unwind: config.unwind, + unwindBlank: config.unwindBlank, + flatten: config.flatten, + flattenSeparator: config.flattenSeparator, + defaultValue: config.defaultValue, + quote: config.quote, + doubleQuote: config.doubleQuote, + delimiter: config.delimiter, + eol: config.eol, + excelStrings: config.excelStrings, + header: config.header, + includeEmptyRows: config.includeEmptyRows, + withBOM: config.withBom }; - if (!program.streaming) { - return getInput() + if (!config.streaming) { + return getInput(config.ndjson) .then(input => new JSON2CSVParser(opts).parse(input)) .then(processOutput); } @@ -179,7 +179,7 @@ Promise.resolve() const input = getInputStream(); const stream = input.pipe(transform); - if (program.output) { + if (config.output) { const outputStream = fs.createWriteStream(outputPath, { encoding: 'utf8' }); const output = stream.pipe(outputStream); return new Promise((resolve, reject) => { @@ -190,7 +190,7 @@ Promise.resolve() }); } - if (!program.pretty) { + if (!config.pretty) { const output = stream.pipe(process.stdout); return new Promise((resolve, reject) => { input.on('error', reject); @@ -205,18 +205,17 @@ Promise.resolve() input.on('error', reject); stream.on('error', reject); let csv = ''; - const table = new TablePrinter(program); + const table = new TablePrinter(config); stream .on('data', chunk => { csv += chunk.toString(); - const index = csv.lastIndexOf(program.eol); + const index = csv.lastIndexOf(config.eol); let lines = csv.substring(0, index); csv = csv.substring(index + 1); if (lines) { table.push(lines); } - }) .on('end', () => { table.end(csv); @@ -230,8 +229,8 @@ Promise.resolve() err = new Error('Invalid input file. (' + err.message + ')'); } else if (outputPath && err.message.includes(outputPath)) { err = new Error('Invalid output file. (' + err.message + ')'); - } else if (fieldsConfigPath && err.message.includes(fieldsConfigPath)) { - err = new Error('Invalid fields config file. (' + err.message + ')'); + } else if (configPath && err.message.indexOf(configPath) !== -1) { + err = new Error('Invalid config file. (' + err.message + ')'); } // eslint-disable-next-line no-console console.error(err); diff --git a/test/CLI.js b/test/CLI.js index 34ff7fe6..41ad76f2 100644 --- a/test/CLI.js +++ b/test/CLI.js @@ -168,10 +168,10 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { // }); testRunner.add('should error on invalid fields config file path', (t) => { - const opts = ' --fields-config ' + getFixturePath('/fields2/fieldNames.json'); + const opts = ' --config ' + getFixturePath('/fields2/fieldNames.json'); child_process.exec(cli + '-i ' + getFixturePath('/json/default.json') + opts, (err, stdout, stderr) => { - t.ok(stderr.includes('Invalid fields config file.')); + t.ok(stderr.indexOf('Invalid config file.') !== -1); t.end(); }); }); @@ -221,7 +221,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); testRunner.add('should name columns as specified in \'fields\' property', (t) => { - const opts = ' --fields-config ' + getFixturePath('/fields/fieldNames.json'); + const opts = ' --config ' + getFixturePath('/fields/fieldNames.json'); child_process.exec(cli + '-i ' + getFixturePath('/json/default.json') + opts, (err, stdout, stderr) => { t.notOk(stderr); @@ -232,7 +232,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); testRunner.add('should support nested properties selectors', (t) => { - const opts = ' --fields-config ' + getFixturePath('/fields/nested.json'); + const opts = ' --config ' + getFixturePath('/fields/nested.json'); child_process.exec(cli + '-i ' + getFixturePath('/json/nested.json') + opts, (err, stdout, stderr) => { t.notOk(stderr); @@ -254,7 +254,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); testRunner.add('field.value function should stringify results by default', (t) => { - const opts = ' --fields-config ' + getFixturePath('/fields/functionStringifyByDefault.js'); + const opts = ' --config ' + getFixturePath('/fields/functionStringifyByDefault.js'); child_process.exec(cli + '-i ' + getFixturePath('/json/functionStringifyByDefault.json') + opts, (err, stdout, stderr) => { t.notOk(stderr); @@ -265,7 +265,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); testRunner.add('field.value function should not stringify if stringify is selected to false', (t) => { - const opts = ' --fields-config ' + getFixturePath('/fields/functionNoStringify.js'); + const opts = ' --config ' + getFixturePath('/fields/functionNoStringify.js'); child_process.exec(cli + '-i ' + getFixturePath('/json/functionNoStringify.json') + opts, (err, stdout, stderr) => { t.notOk(stderr); @@ -276,7 +276,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); testRunner.add('should process different combinations in fields option', (t) => { - const opts = ' --fields-config ' + getFixturePath('/fields/fancyfields.js') + const opts = ' --config ' + getFixturePath('/fields/fancyfields.js') + ' --default-value NULL'; child_process.exec(cli + '-i ' + getFixturePath('/json/fancyfields.json') + opts, (err, stdout, stderr) => { @@ -371,7 +371,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); testRunner.add('should override \'options.defaultValue\' with \'field.defaultValue\'', (t) => { - const opts = ' --fields-config ' + getFixturePath('/fields/overriddenDefaultValue.json') + const opts = ' --config ' + getFixturePath('/fields/overriddenDefaultValue.json') + ' --default-value ""'; child_process.exec(cli + '-i ' + getFixturePath('/json/overriddenDefaultValue.json') + opts, (err, stdout, stderr) => { @@ -383,7 +383,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); testRunner.add('should use \'options.defaultValue\' when no \'field.defaultValue\'', (t) => { - const opts = ' --fields-config ' + getFixturePath('/fields/overriddenDefaultValue2.js') + const opts = ' --config ' + getFixturePath('/fields/overriddenDefaultValue2.js') + ' --default-value ""'; child_process.exec(cli + '-i ' + getFixturePath('/json/overriddenDefaultValue.json') + opts, (err, stdout, stderr) => { @@ -612,7 +612,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { }); testRunner.add('should include empty rows when options.includeEmptyRows is true, with default values', (t) => { - const opts = ' --fields-config ' + getFixturePath('/fields/emptyRowDefaultValues.json') + const opts = ' --config ' + getFixturePath('/fields/emptyRowDefaultValues.json') + ' --default-value NULL' + ' --include-empty-rows'; diff --git a/test/fixtures/fields/emptyRowDefaultValues.json b/test/fixtures/fields/emptyRowDefaultValues.json index 08a1f117..da371f36 100644 --- a/test/fixtures/fields/emptyRowDefaultValues.json +++ b/test/fixtures/fields/emptyRowDefaultValues.json @@ -1,12 +1,14 @@ -[ - { - "value": "carModel" - }, - { - "value": "price", - "default": 1 - }, - { - "value": "color" - } -] \ No newline at end of file +{ + "fields": [ + { + "value": "carModel" + }, + { + "value": "price", + "default": 1 + }, + { + "value": "color" + } + ] +} \ No newline at end of file diff --git a/test/fixtures/fields/fancyfields.js b/test/fixtures/fields/fancyfields.js index eeccbfe8..d07e657e 100644 --- a/test/fixtures/fields/fancyfields.js +++ b/test/fixtures/fields/fancyfields.js @@ -1,16 +1,18 @@ -module.exports = [{ - label: 'PATH1', - value: 'path1' -}, { - label: 'PATH1+PATH2', - value: row => row.path1+row.path2 -}, { - label: 'NEST1', - value: 'bird.nest1' -}, -'bird.nest2', -{ - label: 'nonexistent', - value: 'fake.path', - default: 'col specific default value' -}]; \ No newline at end of file +module.exports = { + fields: [{ + label: 'PATH1', + value: 'path1' + }, { + label: 'PATH1+PATH2', + value: row => row.path1+row.path2 + }, { + label: 'NEST1', + value: 'bird.nest1' + }, + 'bird.nest2', + { + label: 'nonexistent', + value: 'fake.path', + default: 'col specific default value' + }] +}; \ No newline at end of file diff --git a/test/fixtures/fields/fieldNames.json b/test/fixtures/fields/fieldNames.json index eef28bf3..6552d37e 100644 --- a/test/fixtures/fields/fieldNames.json +++ b/test/fixtures/fields/fieldNames.json @@ -1,7 +1,9 @@ -[{ - "label": "Car Model", - "value": "carModel" -},{ - "label": "Price USD", - "value": "price" -}] \ No newline at end of file +{ + "fields": [{ + "label": "Car Model", + "value": "carModel" + },{ + "label": "Price USD", + "value": "price" + }] +} \ No newline at end of file diff --git a/test/fixtures/fields/functionNoStringify.js b/test/fixtures/fields/functionNoStringify.js index a774d15b..511ff164 100644 --- a/test/fixtures/fields/functionNoStringify.js +++ b/test/fixtures/fields/functionNoStringify.js @@ -1,5 +1,7 @@ -module.exports = [{ - label: 'Value1', - value: row => row.value1.toLocaleString(), - stringify: false -}]; \ No newline at end of file +module.exports = { + fields: [{ + label: 'Value1', + value: row => row.value1.toLocaleString(), + stringify: false + }] +}; \ No newline at end of file diff --git a/test/fixtures/fields/functionStringifyByDefault.js b/test/fixtures/fields/functionStringifyByDefault.js index 9b1de355..0c796ed5 100644 --- a/test/fixtures/fields/functionStringifyByDefault.js +++ b/test/fixtures/fields/functionStringifyByDefault.js @@ -1,4 +1,6 @@ -module.exports = [{ - label: 'Value1', - value: row => row.value1.toLocaleString() -}]; \ No newline at end of file +module.exports = { + fields: [{ + label: 'Value1', + value: row => row.value1.toLocaleString() + }] +}; \ No newline at end of file diff --git a/test/fixtures/fields/nested.json b/test/fixtures/fields/nested.json index 5d8ca064..550b2d8a 100644 --- a/test/fixtures/fields/nested.json +++ b/test/fixtures/fields/nested.json @@ -1,16 +1,18 @@ -[{ - "label": "Make", - "value": "car.make" -},{ - "label": "Model", - "value": "car.model" -},{ - "label": "Price", - "value": "price" -},{ - "label": "Color", - "value": "color" -},{ - "label": "Year", - "value": "car.ye.ar" -}] \ No newline at end of file +{ + "fields": [{ + "label": "Make", + "value": "car.make" + },{ + "label": "Model", + "value": "car.model" + },{ + "label": "Price", + "value": "price" + },{ + "label": "Color", + "value": "color" + },{ + "label": "Year", + "value": "car.ye.ar" + }] +} \ No newline at end of file diff --git a/test/fixtures/fields/overriddenDefaultValue.json b/test/fixtures/fields/overriddenDefaultValue.json index 41e3994f..f4c3ae1a 100644 --- a/test/fixtures/fields/overriddenDefaultValue.json +++ b/test/fixtures/fields/overriddenDefaultValue.json @@ -1,5 +1,7 @@ -[ - { "value": "carModel" }, - { "value": "price", "default": 1 }, - { "value": "color" } -] \ No newline at end of file +{ + "fields": [ + { "value": "carModel" }, + { "value": "price", "default": 1 }, + { "value": "color" } + ] +} \ No newline at end of file diff --git a/test/fixtures/fields/overriddenDefaultValue2.js b/test/fixtures/fields/overriddenDefaultValue2.js index ab9d2656..9c986375 100644 --- a/test/fixtures/fields/overriddenDefaultValue2.js +++ b/test/fixtures/fields/overriddenDefaultValue2.js @@ -1,14 +1,16 @@ -module.exports = [ - { - value: 'carModel' - }, - { - label: 'price', - value: row => row.price, - default: 1 - }, - { - label: 'color', - value: row => row.color - } -]; \ No newline at end of file +module.exports = { + fields: [ + { + value: 'carModel' + }, + { + label: 'price', + value: row => row.price, + default: 1 + }, + { + label: 'color', + value: row => row.color + } + ] +}; \ No newline at end of file