diff --git a/README.md b/README.md index 9db2fc8b..1622cb0d 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Options: -a, --include-empty-rows Includes empty rows in the resulting CSV output. -b, --with-bom Includes BOM character at the beginning of the CSV. -p, --pretty Print output as a pretty table. Use only when printing to console. - --unwind Creates multiple rows from a single JSON document similar to MongoDB unwind. + --unwind [paths] Creates multiple rows from a single JSON document similar to MongoDB unwind. --unwind-blank When unwinding, blank out instead of repeating data. Defaults to false. (default: false) --flatten-objects Flatten nested objects. Defaults to false. (default: false) --flatten-arrays Flatten nested arrays. Defaults to false. (default: false) diff --git a/bin/json2csv.js b/bin/json2csv.js index 88d12a08..1cb9debc 100755 --- a/bin/json2csv.js +++ b/bin/json2csv.js @@ -40,7 +40,7 @@ program .option('-b, --with-bom', 'Includes BOM character at the beginning of the CSV.') .option('-p, --pretty', 'Print output as a pretty table. Use only when printing to console.') // Built-in transforms - .option('--unwind ', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.') + .option('--unwind [paths]', 'Creates multiple rows from a single JSON document similar to MongoDB unwind.') .option('--unwind-blank', 'When unwinding, blank out instead of repeating data. Defaults to false.', false) .option('--flatten-objects', 'Flatten nested objects. Defaults to false.', false) .option('--flatten-arrays', 'Flatten nested arrays. Defaults to false.', false) @@ -139,7 +139,7 @@ async function processStream(config, opts) { const transforms = []; if (config.unwind) { transforms.push(unwind({ - paths: config.unwind.split(','), + paths: config.unwind === true ? undefined : config.unwind.split(','), blankOut: config.unwindBlank })); } diff --git a/lib/transforms/unwind.js b/lib/transforms/unwind.js index 79b8c38d..51289b1a 100644 --- a/lib/transforms/unwind.js +++ b/lib/transforms/unwind.js @@ -2,13 +2,36 @@ const lodashGet = require('lodash.get'); const { setProp, flattenReducer } = require('../utils'); +function getUnwindablePaths(obj, currentPath) { + return Object.keys(obj).reduce((unwindablePaths, key) => { + const newPath = currentPath ? `${currentPath}.${key}` : key; + const value = obj[key]; + + if (typeof value === 'object' + && value !== null + && !Array.isArray(value) + && Object.prototype.toString.call(value.toJSON) !== '[object Function]' + && Object.keys(value).length) { + unwindablePaths = unwindablePaths.concat(getUnwindablePaths(value, newPath)); + } else if (Array.isArray(value)) { + unwindablePaths.push(newPath); + unwindablePaths = unwindablePaths.concat(value + .map(arrObj => getUnwindablePaths(arrObj, newPath)) + .reduce(flattenReducer, []) + .filter((item, index, arr) => arr.indexOf(item) !== index)); + } + + return unwindablePaths; + }, []); +} + /** * Performs the unwind recursively in specified sequence * * @param {String[]} unwindPaths The paths as strings to be used to deconstruct the array * @returns {Object => Array} Array of objects containing all rows after unwind of chosen paths */ -function unwind({ paths = [], blankOut = false } = {}) { +function unwind({ paths = undefined, blankOut = false } = {}) { function unwindReducer(rows, unwindPath) { return rows .map(row => { @@ -33,8 +56,8 @@ function unwind({ paths = [], blankOut = false } = {}) { .reduce(flattenReducer, []); } - paths = Array.isArray(paths) ? paths : (paths ? [paths] : []); - return dataRow => paths.reduce(unwindReducer, [dataRow]); + paths = Array.isArray(paths) ? paths : (paths ? [paths] : undefined); + return dataRow => (paths || getUnwindablePaths(dataRow)).reduce(unwindReducer, [dataRow]); } module.exports = unwind; \ No newline at end of file diff --git a/test/CLI.js b/test/CLI.js index 132cea01..3a1ad57b 100644 --- a/test/CLI.js +++ b/test/CLI.js @@ -741,7 +741,19 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { // Preprocessing - testRunner.add('should support unwinding an object into multiple rows using the unwind transform', (t) => { + testRunner.add('should unwind all unwindable fields using the unwind transform', (t) => { + const opts = '--fields carModel,price,extras.items.name,extras.items.color,extras.items.items.position,extras.items.items.color' + + ' --unwind'; + + exec(`${cli} -i "${getFixturePath('/json/unwind2.json')}" ${opts}`, (err, stdout, stderr) => { + t.notOk(stderr); + const csv = stdout; + t.equal(csv, csvFixtures.unwind2); + t.end(); + }); + }); + + testRunner.add('should support unwinding specific fields using the unwind transform', (t) => { const opts = '--unwind colors'; exec(`${cli} -i "${getFixturePath('/json/unwind.json')}" ${opts}`, (err, stdout, stderr) => { @@ -786,6 +798,7 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { t.end(); }); }); + testRunner.add('should support flattening JSON with nested arrays using the flatten transform', (t) => { const opts = '--flatten-objects --flatten-arrays'; diff --git a/test/JSON2CSVAsyncParser.js b/test/JSON2CSVAsyncParser.js index 6f7da6f3..3a0d82a7 100644 --- a/test/JSON2CSVAsyncParser.js +++ b/test/JSON2CSVAsyncParser.js @@ -1128,7 +1128,24 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = // Transforms - testRunner.add('should support unwinding an object into multiple rows using the unwind transform', async (t) => { + testRunner.add('should unwind all unwindable fields using the unwind transform', async (t) => { + const opts = { + fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'], + transforms: [unwind()], + }; + const parser = new AsyncParser(opts); + + try { + const csv = await parser.fromInput(jsonFixtures.unwind2()).promise(); + t.equal(csv, csvFixtures.unwind2); + } catch(err) { + t.fail(err.message); + } + + t.end(); + }); + + testRunner.add('should support unwinding specific fields using the unwind transform', async (t) => { const opts = { fields: ['carModel', 'price', 'colors'], transforms: [unwind({ paths: ['colors'] })], diff --git a/test/JSON2CSVParser.js b/test/JSON2CSVParser.js index d7177bc9..ac0b2f12 100644 --- a/test/JSON2CSVParser.js +++ b/test/JSON2CSVParser.js @@ -669,7 +669,20 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { // Transforms - testRunner.add('should support unwinding an object into multiple rows using the unwind transform', (t) => { + testRunner.add('should unwind all unwindable fields using the unwind transform', (t) => { + const opts = { + fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'], + transforms: [unwind()], + }; + + const parser = new Json2csvParser(opts); + const csv = parser.parse(jsonFixtures.unwind2); + + t.equal(csv, csvFixtures.unwind2); + t.end(); + }); + + testRunner.add('should support unwinding specific fields using the unwind transform', (t) => { const opts = { fields: ['carModel', 'price', 'colors'], transforms: [unwind({ paths: ['colors'] })], @@ -708,7 +721,6 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { t.end(); }); - testRunner.add('should support flattening deep JSON using the flatten transform', (t) => { const opts = { transforms: [flatten()], diff --git a/test/JSON2CSVTransform.js b/test/JSON2CSVTransform.js index d519c023..b1240283 100644 --- a/test/JSON2CSVTransform.js +++ b/test/JSON2CSVTransform.js @@ -1123,7 +1123,29 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = // Transform - testRunner.add('should support unwinding an object into multiple rows using the unwind transform', (t) => { + testRunner.add('should unwind all unwindable fields using the unwind transform', (t) => { + const opts = { + fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'], + transforms: [unwind()], + }; + + const transform = new Json2csvTransform(opts); + const processor = jsonFixtures.unwind2().pipe(transform); + + let csv = ''; + processor + .on('data', chunk => (csv += chunk.toString())) + .on('end', () => { + t.equal(csv, csvFixtures.unwind2); + t.end(); + }) + .on('error', err => { + t.fail(err.message); + t.end(); + }); + }); + + testRunner.add('should support unwinding specific fields using the unwind transform', (t) => { const opts = { fields: ['carModel', 'price', 'colors'], transforms: [unwind({ paths: ['colors'] })], @@ -1167,8 +1189,6 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = }); }); - - testRunner.add('should support unwind and blank out repeated data using the unwind transform', (t) => { const opts = { fields: ['carModel', 'price', 'extras.items.name', 'extras.items.color', 'extras.items.items.position', 'extras.items.items.color'], diff --git a/test/fixtures/csv/unwind2.csv b/test/fixtures/csv/unwind2.csv index 26858168..319c56a3 100644 --- a/test/fixtures/csv/unwind2.csv +++ b/test/fixtures/csv/unwind2.csv @@ -1,7 +1,7 @@ "carModel","price","extras.items.name","extras.items.color","extras.items.items.position","extras.items.items.color" -"BMW",15000,"airbag","white",, -"BMW",15000,"dashboard","black",, "Porsche",30000,"airbag",,"left","white" "Porsche",30000,"airbag",,"right","gray" "Porsche",30000,"dashboard",,"left","gray" -"Porsche",30000,"dashboard",,"right","black" \ No newline at end of file +"Porsche",30000,"dashboard",,"right","black" +"BMW",15000,"airbag","white",, +"BMW",15000,"dashboard","black",, \ No newline at end of file diff --git a/test/fixtures/csv/unwind2Blank.csv b/test/fixtures/csv/unwind2Blank.csv index ea0d82e0..17092013 100644 --- a/test/fixtures/csv/unwind2Blank.csv +++ b/test/fixtures/csv/unwind2Blank.csv @@ -1,7 +1,7 @@ "carModel","price","extras.items.name","extras.items.color","extras.items.items.position","extras.items.items.color" -"BMW",15000,"airbag","white",, -,,"dashboard","black",, "Porsche",30000,"airbag",,"left","white" ,,,,"right","gray" ,,"dashboard",,"left","gray" -,,,,"right","black" \ No newline at end of file +,,,,"right","black" +"BMW",15000,"airbag","white",, +,,"dashboard","black",, \ No newline at end of file diff --git a/test/fixtures/json/unwind2.json b/test/fixtures/json/unwind2.json index fa3cc447..266d6a57 100644 --- a/test/fixtures/json/unwind2.json +++ b/test/fixtures/json/unwind2.json @@ -1,19 +1,4 @@ -[ - { - "carModel": "BMW", - "price": 15000, - "extras": { - "items": [ - { - "name": "airbag", - "color": "white" - }, { - "name": "dashboard", - "color": "black" - } - ] - } - }, { +[{ "carModel": "Porsche", "price": 30000, "extras": { @@ -44,5 +29,19 @@ } ] } + }, { + "carModel": "BMW", + "price": 15000, + "extras": { + "items": [ + { + "name": "airbag", + "color": "white" + }, { + "name": "dashboard", + "color": "black" + } + ] + } } ]