feat: Improves the unwind transform so it unwinds all unwindable fiel…

…ds if … (#434) * Improves the unwind transform so it unwinds all unwindable fields if no paths are provided * docs: fix unwind opt
zemirco · Dec 14, 2019 · ec1f301 · ec1f301
1 parent c00a85b
commit ec1f301
Show file tree

Hide file tree

Showing 10 changed files with 119 additions and 35 deletions.
diff --git a/README.md b/README.md
@@ -73,7 +73,7 @@ Options:
   -a, --include-empty-rows            Includes empty rows in the resulting CSV output.
   -b, --with-bom                      Includes BOM character at the beginning of the CSV.
   -p, --pretty                        Print output as a pretty table. Use only when printing to console.
-  --unwind <paths>                    Creates multiple rows from a single JSON document similar to MongoDB unwind.
+  --unwind [paths]                    Creates multiple rows from a single JSON document similar to MongoDB unwind.
   --unwind-blank                      When unwinding, blank out instead of repeating data. Defaults to false. (default: false)
   --flatten-objects                   Flatten nested objects. Defaults to false. (default: false)
   --flatten-arrays                    Flatten nested arrays. Defaults to false. (default: false)

diff --git a/bin/json2csv.js b/bin/json2csv.js
@@ -40,7 +40,7 @@ program
   .option('-b, --with-bom', 'Includes BOM character at the beginning of the CSV.')
   .option('-p, --pretty', 'Print output as a pretty table. Use only when printing to console.')
   // Built-in transforms
-  .option('--unwind <paths>', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.')
+  .option('--unwind [paths]', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.')
   .option('--unwind-blank', 'When unwinding, blank out instead of repeating data. Defaults to false.', false)
   .option('--flatten-objects', 'Flatten nested objects. Defaults to false.', false)
   .option('--flatten-arrays', 'Flatten nested arrays. Defaults to false.', false)
@@ -139,7 +139,7 @@ async function processStream(config, opts) {
     const transforms = [];
     if (config.unwind) {
       transforms.push(unwind({
-        paths: config.unwind.split(','),
+        paths: config.unwind === true ? undefined : config.unwind.split(','),
         blankOut: config.unwindBlank
       }));
     }

diff --git a/lib/transforms/unwind.js b/lib/transforms/unwind.js
@@ -2,13 +2,36 @@
 const lodashGet = require('lodash.get');
 const { setProp, flattenReducer } = require('../utils');
 
+function getUnwindablePaths(obj, currentPath) {
+  return Object.keys(obj).reduce((unwindablePaths, key) => {
+    const newPath = currentPath ? `${currentPath}.${key}` : key;
+    const value = obj[key];
+
+    if (typeof value === 'object'
+      && value !== null
+      && !Array.isArray(value)
+      && Object.prototype.toString.call(value.toJSON) !== '[object Function]'
+      && Object.keys(value).length) {
+      unwindablePaths = unwindablePaths.concat(getUnwindablePaths(value, newPath));
+    } else if (Array.isArray(value)) {
+      unwindablePaths.push(newPath);
+      unwindablePaths = unwindablePaths.concat(value
+        .map(arrObj => getUnwindablePaths(arrObj, newPath))
+        .reduce(flattenReducer, [])
+        .filter((item, index, arr) => arr.indexOf(item) !== index));
+    }
+
+    return unwindablePaths;
+  }, []);
+}
+
 /**
  * Performs the unwind recursively in specified sequence
  *
  * @param {String[]} unwindPaths The paths as strings to be used to deconstruct the array
  * @returns {Object => Array} Array of objects containing all rows after unwind of chosen paths
 */
-function unwind({ paths = [], blankOut = false } = {}) {
+function unwind({ paths = undefined, blankOut = false } = {}) {
   function unwindReducer(rows, unwindPath) {
     return rows
       .map(row => {
@@ -33,8 +56,8 @@ function unwind({ paths = [], blankOut = false } = {}) {
       .reduce(flattenReducer, []);
   }
 
-  paths = Array.isArray(paths) ? paths : (paths ? [paths] : []);
-  return dataRow => paths.reduce(unwindReducer, [dataRow]);
+  paths = Array.isArray(paths) ? paths : (paths ? [paths] : undefined);
+  return dataRow => (paths || getUnwindablePaths(dataRow)).reduce(unwindReducer, [dataRow]);
 }
 
 module.exports = unwind;
diff --git a/test/CLI.js b/test/CLI.js
@@ -741,7 +741,19 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
 
   // Preprocessing
 
-  testRunner.add('should support unwinding an object into multiple rows using the unwind transform', (t) => {
+  testRunner.add('should unwind all unwindable fields using the unwind transform', (t) => {
+    const opts = '--fields carModel,price,extras.items.name,extras.items.color,extras.items.items.position,extras.items.items.color'
+      + ' --unwind';
+
+    exec(`${cli} -i "${getFixturePath('/json/unwind2.json')}" ${opts}`, (err, stdout, stderr) => {
+      t.notOk(stderr);
+      const csv = stdout;
+      t.equal(csv, csvFixtures.unwind2);
+      t.end();
+    });
+  });
+
+  testRunner.add('should support unwinding specific fields using the unwind transform', (t) => {
     const opts = '--unwind colors';
 
     exec(`${cli} -i "${getFixturePath('/json/unwind.json')}" ${opts}`, (err, stdout, stderr) => {
@@ -786,6 +798,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
       t.end();
     });
   });
+
   testRunner.add('should support flattening JSON with nested arrays using the flatten transform', (t) => {
     const opts = '--flatten-objects --flatten-arrays';
 

diff --git a/test/JSON2CSVAsyncParser.js b/test/JSON2CSVAsyncParser.js
@@ -1128,7 +1128,24 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
 
   // Transforms
 
-  testRunner.add('should support unwinding an object into multiple rows using the unwind transform', async (t) => {
+  testRunner.add('should unwind all unwindable fields using the unwind transform', async (t) => {
+    const opts = {
+      fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'],
+      transforms: [unwind()],
+    };
+    const parser = new AsyncParser(opts);
+
+    try {
+      const csv = await parser.fromInput(jsonFixtures.unwind2()).promise();
+      t.equal(csv, csvFixtures.unwind2);
+    } catch(err) {
+      t.fail(err.message);
+    }
+
+    t.end();
+  });
+
+  testRunner.add('should support unwinding specific fields using the unwind transform', async (t) => {
     const opts = {
       fields: ['carModel', 'price', 'colors'],
       transforms: [unwind({ paths: ['colors'] })],

diff --git a/test/JSON2CSVParser.js b/test/JSON2CSVParser.js
@@ -669,7 +669,20 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
 
   // Transforms
 
-  testRunner.add('should support unwinding an object into multiple rows using the unwind transform', (t) => {
+  testRunner.add('should unwind all unwindable fields using the unwind transform', (t) => {
+    const opts = {
+      fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'],
+      transforms: [unwind()],
+    };
+
+    const parser = new Json2csvParser(opts);
+    const csv = parser.parse(jsonFixtures.unwind2);
+
+    t.equal(csv, csvFixtures.unwind2);
+    t.end();
+  });
+
+  testRunner.add('should support unwinding specific fields using the unwind transform', (t) => {
     const opts = {
       fields: ['carModel', 'price', 'colors'],
       transforms: [unwind({ paths: ['colors'] })],
@@ -708,7 +721,6 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
     t.end();
   });
 
-
   testRunner.add('should support flattening deep JSON using the flatten transform', (t) => {
     const opts = {
       transforms: [flatten()],

diff --git a/test/JSON2CSVTransform.js b/test/JSON2CSVTransform.js
@@ -1123,7 +1123,29 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
 
   // Transform
 
-  testRunner.add('should support unwinding an object into multiple rows using the unwind transform', (t) => {
+  testRunner.add('should unwind all unwindable fields using the unwind transform', (t) => {
+    const opts = {
+      fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'],
+      transforms: [unwind()],
+    };
+
+    const transform = new Json2csvTransform(opts);
+    const processor = jsonFixtures.unwind2().pipe(transform);
+
+    let csv = '';
+    processor
+      .on('data', chunk => (csv += chunk.toString()))
+      .on('end', () => {
+        t.equal(csv, csvFixtures.unwind2);
+        t.end();
+      })
+      .on('error', err => {
+        t.fail(err.message);
+        t.end();  
+      });
+  });
+
+  testRunner.add('should support unwinding specific fields using the unwind transform', (t) => {
     const opts = {
       fields: ['carModel', 'price', 'colors'],
       transforms: [unwind({ paths: ['colors'] })],
@@ -1167,8 +1189,6 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
       });
   });
 
-
-
   testRunner.add('should support unwind and blank out repeated data using the unwind transform', (t) => {
     const opts = {
       fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'],

diff --git a/test/fixtures/csv/unwind2.csv b/test/fixtures/csv/unwind2.csv
@@ -1,7 +1,7 @@
 "carModel","price","extras.items.name","extras.items.color","extras.items.items.position","extras.items.items.color"
-"BMW",15000,"airbag","white",,
-"BMW",15000,"dashboard","black",,
 "Porsche",30000,"airbag",,"left","white"
 "Porsche",30000,"airbag",,"right","gray"
 "Porsche",30000,"dashboard",,"left","gray"
-"Porsche",30000,"dashboard",,"right","black"
+"Porsche",30000,"dashboard",,"right","black"
+"BMW",15000,"airbag","white",,
+"BMW",15000,"dashboard","black",,
diff --git a/test/fixtures/csv/unwind2Blank.csv b/test/fixtures/csv/unwind2Blank.csv
@@ -1,7 +1,7 @@
 "carModel","price","extras.items.name","extras.items.color","extras.items.items.position","extras.items.items.color"
-"BMW",15000,"airbag","white",,
-,,"dashboard","black",,
 "Porsche",30000,"airbag",,"left","white"
 ,,,,"right","gray"
 ,,"dashboard",,"left","gray"
-,,,,"right","black"
+,,,,"right","black"
+"BMW",15000,"airbag","white",,
+,,"dashboard","black",,
diff --git a/test/fixtures/json/unwind2.json b/test/fixtures/json/unwind2.json
@@ -1,19 +1,4 @@
-[
-    {
-        "carModel": "BMW",
-        "price": 15000,
-        "extras": {
-          "items": [
-              {
-                  "name": "airbag",
-                  "color": "white"
-              }, {
-                  "name": "dashboard",
-                  "color": "black"
-              }
-          ]
-        }
-    }, {
+[{
         "carModel": "Porsche",
         "price": 30000,
         "extras": {
@@ -44,5 +29,19 @@
               }
           ]
         }
+    }, {
+        "carModel": "BMW",
+        "price": 15000,
+        "extras": {
+          "items": [
+              {
+                  "name": "airbag",
+                  "color": "white"
+              }, {
+                  "name": "dashboard",
+                  "color": "black"
+              }
+          ]
+        }
     }
 ]