feat: Add support for transforms (#431)

BREAKING CHANGE: module no longer takes `unwind`, `unwindBlank`, `flatten` or the `flattenSeparator` options, instead see the new `transforms` option. CLI options are unchanged from the callers side, but use the built in transforms under the hood. * Add support for transforms * Add documentation about transforms
zemirco · Oct 18, 2019 · f1d04d0 · f1d04d0
1 parent f7dd7eb
commit f1d04d0
Show file tree

Hide file tree

Showing 13 changed files with 739 additions and 606 deletions.
diff --git a/README.md b/README.md
diff --git a/bin/json2csv.js b/bin/json2csv.js
@@ -17,6 +17,7 @@ const writeFile = promisify(writeFileOrig);
 const isAbsolutePath = promisify(isAbsolute);
 const joinPath = promisify(join);
 
+const { unwind, flatten } = json2csv.transforms;
 const JSON2CSVParser = json2csv.Parser;
 const Json2csvTransform = json2csv.Transform;
 
@@ -28,20 +29,21 @@ program
   .option('-n, --ndjson', 'Treat the input as NewLine-Delimited JSON.')
   .option('-s, --no-streaming', 'Process the whole JSON array in memory instead of doing it line by line.')
   .option('-f, --fields <fields>', 'List of fields to process. Defaults to field auto-detection.')
-  .option('-u, --unwind <paths>', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.')
-  .option('-B, --unwind-blank', 'When unwinding, blank out instead of repeating data.')
-  .option('-F, --flatten', 'Flatten nested objects.')
-  .option('-S, --flatten-separator <separator>', 'Flattened keys separator. Defaults to \'.\'.')
   .option('-v, --default-value [defaultValue]', 'Default value to use for missing fields.')
   .option('-q, --quote [quote]', 'Character(s) to use as quote mark. Defaults to \'"\'.')
   .option('-Q, --escaped-quote [escapedQuote]', 'Character(s) to use as a escaped quote. Defaults to a double `quote`, \'""\'.')
-  .option('-d, --delimiter [delimiter]', 'Character(s) to use as delimiter. Defaults to \',\'.')
-  .option('-e, --eol [eol]', 'Character(s) to use as End-of-Line for separating rows. Defaults to \'\\n\'.')
+  .option('-d, --delimiter [delimiter]', 'Character(s) to use as delimiter. Defaults to \',\'.', ',')
+  .option('-e, --eol [eol]', 'Character(s) to use as End-of-Line for separating rows. Defaults to \'\\n\'.', os.EOL)
   .option('-E, --excel-strings','Wraps string data to force Excel to interpret it as string even if it contains a number.')
   .option('-H, --no-header', 'Disable the column name header.')
   .option('-a, --include-empty-rows', 'Includes empty rows in the resulting CSV output.')
   .option('-b, --with-bom', 'Includes BOM character at the beginning of the CSV.')
   .option('-p, --pretty', 'Print output as a pretty table. Use only when printing to console.')
+  // Built-in transforms
+  .option('-u, --unwind <paths>', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.')
+  .option('-B, --unwind-blank', 'When unwinding, blank out instead of repeating data.')
+  .option('-F, --flatten', 'Flatten nested objects.')
+  .option('-S, --flatten-separator <separator>', 'Flattened keys separator. Defaults to \'.\'.')
   .parse(process.argv);
 
 function makePathAbsolute(filePath) {
@@ -54,11 +56,6 @@ program.input = makePathAbsolute(program.input);
 program.output = makePathAbsolute(program.output);
 program.config = makePathAbsolute(program.config);
 
-if (program.fields) program.fields = program.fields.split(',');
-if (program.unwind) program.unwind = program.unwind.split(',');
-program.delimiter = program.delimiter || ',';
-program.eol = program.eol || os.EOL;
-
 // don't fail if piped to e.g. head
 /* istanbul ignore next */
 process.stdout.on('error', (error) => {
@@ -137,13 +134,16 @@ async function processStream(config, opts) {
 (async (program) => {
   try {
     const config = Object.assign({}, program.config ? require(program.config) : {}, program);
+
+    const transforms = [];
+    if (config.unwind) transforms.push(unwind(config.unwind.split(','), config.unwindBlank || false));
+    if (config.flatten) transforms.push(flatten(config.flattenSeparator || '.'));
 
     const opts = {
-      fields: config.fields,
-      unwind: config.unwind,
-      unwindBlank: config.unwindBlank,
-      flatten: config.flatten,
-      flattenSeparator: config.flattenSeparator,
+      transforms,
+      fields: config.fields
+        ? (Array.isArray(config.fields) ? config.fields : config.fields.split(','))
+        : config.fields,
       defaultValue: config.defaultValue,
       quote: config.quote,
       escapedQuote: config.escapedQuote,

diff --git a/lib/JSON2CSVBase.js b/lib/JSON2CSVBase.js
@@ -2,12 +2,11 @@
 
 const os = require('os');
 const lodashGet = require('lodash.get');
-const { getProp, setProp, fastJoin, flattenReducer } = require('./utils');
+const { getProp, fastJoin, flattenReducer } = require('./utils');
 
 class JSON2CSVBase {
   constructor(opts) {
     this.opts = this.preprocessOpts(opts);
-    this.preprocessRow = this.memoizePreprocessRow();
   }
 
   /**
@@ -18,14 +17,13 @@ class JSON2CSVBase {
    */
   preprocessOpts(opts) {
     const processedOpts = Object.assign({}, opts);
-    processedOpts.unwind = !Array.isArray(processedOpts.unwind)
-      ? (processedOpts.unwind ? [processedOpts.unwind] : [])
-      : processedOpts.unwind
+    processedOpts.transforms = !Array.isArray(processedOpts.transforms)
+      ? (processedOpts.transforms ? [processedOpts.transforms] : [])
+      : processedOpts.transforms
     processedOpts.delimiter = processedOpts.delimiter || ',';
-    processedOpts.flattenSeparator = processedOpts.flattenSeparator || '.';
     processedOpts.eol = processedOpts.eol || os.EOL;
     processedOpts.quote = typeof processedOpts.quote === 'string'
-      ? opts.quote
+      ? processedOpts.quote
       : '"';
     processedOpts.escapedQuote = typeof processedOpts.escapedQuote === 'string'
       ? processedOpts.escapedQuote
@@ -100,39 +98,16 @@ class JSON2CSVBase {
     );
   }
 
-  memoizePreprocessRow() {
-    if (this.opts.unwind && this.opts.unwind.length) {
-      if (this.opts.flatten) {
-        return function (row) {
-          return this.unwindData(row, this.opts.unwind)
-            .map(row => this.flatten(row, this.opts.flattenSeparator));
-        };
-      }
-
-      return function (row) {
-        return this.unwindData(row, this.opts.unwind);
-      };
-    }
-
-    if (this.opts.flatten) {
-      return function (row) {
-        return [this.flatten(row, this.opts.flattenSeparator)];
-      };
-    }
-
-    return function (row) {
-      return [row];
-    };
-  }
-
   /**
-   * Preprocess each object according to the give opts (unwind, flatten, etc.).
-   * The actual body of the function is dynamically set on the constructor by the
-   *  `memoizePreprocessRow` method after parsing the options.
-   *
+   * Preprocess each object according to the given transforms (unwind, flatten, etc.).
    * @param {Object} row JSON object to be converted in a CSV row
    */
-  preprocessRow() {}
+  preprocessRow(row) {
+    return this.opts.transforms.reduce((rows, transform) =>
+      rows.map(row => transform(row)).reduce(flattenReducer, []),
+      [row]
+    );
+  }
 
   /**
    * Create the content of a specific CSV row
@@ -206,75 +181,6 @@ class JSON2CSVBase {
 
     return value;
   }
-
-  /**
-   * Performs the flattening of a data row recursively
-   *
-   * @param {Object} dataRow Original JSON object
-   * @param {String} separator Separator to be used as the flattened field name
-   * @returns {Object} Flattened object
-   */
-  flatten(dataRow, separator) {
-    function step (obj, flatDataRow, currentPath) {
-      Object.keys(obj).forEach((key) => {
-        const value = obj[key];
-
-        const newPath = currentPath
-          ? `${currentPath}${separator}${key}`
-          : key;
-
-        if (typeof value !== 'object'
-          || value === null
-          || Array.isArray(value)
-          || Object.prototype.toString.call(value.toJSON) === '[object Function]'
-          || !Object.keys(value).length) {
-          flatDataRow[newPath] = value;
-          return;
-        }
-
-        step(value, flatDataRow, newPath);
-      });
-
-      return flatDataRow;
-    }
-
-    return step(dataRow, {});
-  }
-
-  /**
-   * Performs the unwind recursively in specified sequence
-   *
-   * @param {Object} dataRow Original JSON object
-   * @param {String[]} unwindPaths The paths as strings to be used to deconstruct the array
-   * @returns {Array} Array of objects containing all rows after unwind of chosen paths
-  */
-  unwindData(dataRow, unwindPaths) {
-    const unwind = (rows, unwindPath) => {
-      return rows
-        .map(row => {
-          const unwindArray = lodashGet(row, unwindPath);
-
-          if (!Array.isArray(unwindArray)) {
-            return row;
-          }
-
-          if (!unwindArray.length) {
-            return setProp(row, unwindPath, undefined);
-          }
-
-          return unwindArray.map((unwindRow, index) => {
-            const clonedRow = (this.opts.unwindBlank && index > 0)
-              ? {}
-              : row;
-
-            return setProp(clonedRow, unwindPath, unwindRow);
-          });
-        })
-        .reduce(flattenReducer, []);
-    };
-
-    return unwindPaths.reduce(unwind, [dataRow]);
-  }
 }
 
 module.exports = JSON2CSVBase;
diff --git a/lib/JSON2CSVParser.js b/lib/JSON2CSVParser.js
@@ -57,9 +57,7 @@ class JSON2CSVParser extends JSON2CSVBase {
       throw new Error('Data should not be empty or the "fields" option should be included');
     }
 
-    if ((!this.opts.unwind || !this.opts.unwind.length) && !this.opts.flatten) {
-      return processedData;
-    }
+    if (this.opts.transforms.length === 0) return processedData;
 
     return processedData
       .map(row => this.preprocessRow(row))

diff --git a/lib/JSON2CSVTransform.js b/lib/JSON2CSVTransform.js
@@ -13,7 +13,6 @@ class JSON2CSVTransform extends Transform {
     Object.getOwnPropertyNames(JSON2CSVBase.prototype)
       .forEach(key => (this[key] = JSON2CSVBase.prototype[key]));
     this.opts = this.preprocessOpts(opts);
-    this.preprocessRow = this.memoizePreprocessRow();
 
     this._data = '';
     this._hasWritten = false;

diff --git a/lib/json2csv.js b/lib/json2csv.js
@@ -4,6 +4,8 @@ const { Readable } = require('stream');
 const JSON2CSVParser = require('./JSON2CSVParser');
 const JSON2CSVAsyncParser = require('./JSON2CSVAsyncParser');
 const JSON2CSVTransform = require('./JSON2CSVTransform');
+const flatten = require('./transforms/flatten');
+const unwind = require('./transforms/unwind');
 
 module.exports.Parser = JSON2CSVParser;
 module.exports.AsyncParser = JSON2CSVAsyncParser;
@@ -35,3 +37,8 @@ module.exports.parseAsync = (data, opts, transformOpts) => {
     return Promise.reject(err);
   }
 };
+
+module.exports.transforms = {
+  flatten,
+  unwind,
+};
diff --git a/lib/transforms/flatten.js b/lib/transforms/flatten.js
@@ -0,0 +1,31 @@
+/**
+ * Performs the flattening of a data row recursively
+ *
+ * @param {String} separator Separator to be used as the flattened field name
+ * @returns {Object => Object} Flattened object
+ */
+function flatten(separator = '.') {
+  function step (obj, flatDataRow, currentPath) {
+    Object.keys(obj).forEach((key) => {
+      const newPath = currentPath ? `${currentPath}${separator}${key}` : key;
+      const value = obj[key];
+
+      if (typeof value !== 'object'
+        || value === null
+        || Array.isArray(value)
+        || Object.prototype.toString.call(value.toJSON) === '[object Function]'
+        || !Object.keys(value).length) {
+        flatDataRow[newPath] = value;
+        return;
+      }
+
+      step(value, flatDataRow, newPath);
+    });
+
+    return flatDataRow;
+  }
+
+  return dataRow => step(dataRow, {});
+}
+
+module.exports = flatten;
diff --git a/lib/transforms/unwind.js b/lib/transforms/unwind.js
@@ -0,0 +1,40 @@
+
+const lodashGet = require('lodash.get');
+const { setProp, flattenReducer } = require('../utils');
+
+/**
+ * Performs the unwind recursively in specified sequence
+ *
+ * @param {String[]} unwindPaths The paths as strings to be used to deconstruct the array
+ * @returns {Object => Array} Array of objects containing all rows after unwind of chosen paths
+*/
+function unwind(paths, blankOut = false) {
+  function unwindReducer(rows, unwindPath) {
+    return rows
+      .map(row => {
+        const unwindArray = lodashGet(row, unwindPath);
+
+        if (!Array.isArray(unwindArray)) {
+          return row;
+        }
+
+        if (!unwindArray.length) {
+          return setProp(row, unwindPath, undefined);
+        }
+
+        return unwindArray.map((unwindRow, index) => {
+          const clonedRow = (blankOut && index > 0)
+            ? {}
+            : row;
+
+          return setProp(clonedRow, unwindPath, unwindRow);
+        });
+      })
+      .reduce(flattenReducer, []);
+  }
+
+  paths = Array.isArray(paths) ? paths : (paths ? [paths] : []);
+  return dataRow => paths.reduce(unwindReducer, [dataRow]);
+}
+
+module.exports = unwind;