From 6b3d88fccd588f7dcd2069e59ecc5bcc1a024a1f Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Fri, 14 May 2021 15:27:42 -0700 Subject: [PATCH] Fix csv parsing function, switch json to csv library Signed-off-by: Joshua Li --- kibana-reports/package.json | 3 +- .../server/routes/utils/dataReportHelpers.ts | 35 ++++---------- kibana-reports/yarn.lock | 46 +++++++++++++------ 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/kibana-reports/package.json b/kibana-reports/package.json index 46e8fd79..0da194db 100644 --- a/kibana-reports/package.json +++ b/kibana-reports/package.json @@ -30,7 +30,7 @@ "jest-fetch-mock": "^3.0.3", "jquery": "^3.5.0", "jsdom": "13.1.0", - "json-2-csv": "^3.7.6", + "json2csv": "^5.0.6", "puppeteer-core": "^1.19.0", "react-addons-test-utils": "^15.6.2", "react-id-generator": "^3.0.1", @@ -50,6 +50,7 @@ "@types/dompurify": "^2.0.4", "@types/enzyme-adapter-react-16": "^1.0.6", "@types/jsdom": "^16.2.3", + "@types/json2csv": "^5.0.2", "@types/puppeteer-core": "^2.0.0", "@types/react": "^16.9.36", "@types/react-addons-test-utils": "^0.14.25", diff --git a/kibana-reports/server/routes/utils/dataReportHelpers.ts b/kibana-reports/server/routes/utils/dataReportHelpers.ts index 296a2a25..f26fe42a 100644 --- a/kibana-reports/server/routes/utils/dataReportHelpers.ts +++ b/kibana-reports/server/routes/utils/dataReportHelpers.ts @@ -17,7 +17,8 @@ import { DATA_REPORT_CONFIG } from './constants'; import esb from 'elastic-builder'; import moment from 'moment'; -import converter from 'json-2-csv'; +import { parseAsync, transforms as Transforms } from 'json2csv'; +import _ from 'lodash'; export var metaData = { saved_search_id: null, @@ -168,7 +169,7 @@ export const getEsData = (arrayHits, report, params) => { } delete data['fields']; if (report._source.fields_exist === true) { - let result = traverse(data, report._source.selectedFields); + let result = traverse(data._source, report._source.selectedFields); hits.push(params.excel ? sanitize(result) : result); } else { hits.push(params.excel ? sanitize(data) : data); @@ -185,34 +186,16 @@ export const getEsData = (arrayHits, report, params) => { //Convert the data to Csv format export const convertToCSV = async (dataset) => { - let convertedData: any = []; - const options = { - delimiter: { field: ',', eol: '\n' }, - emptyFieldValue: ' ', - }; - await converter.json2csvAsync(dataset[0], options).then((csv) => { - convertedData = csv; - }); - return convertedData; + const transforms = [Transforms.flatten()]; + return parseAsync(dataset[0], { transforms }); }; //Return only the selected fields function traverse(data, keys, result = {}) { - for (let k of Object.keys(data)) { - if (keys.includes(k)) { - result = Object.assign({}, result, { - [k]: data[k], - }); - continue; - } - if ( - data[k] && - typeof data[k] === 'object' && - Object.keys(data[k]).length > 0 - ) { - result = traverse(data[k], keys, result); - } - } + keys.forEach((key) => { + const value = _.get(data, key, undefined); + if (value !== undefined) result[key] = value; + }); return result; } diff --git a/kibana-reports/yarn.lock b/kibana-reports/yarn.lock index 07b6daeb..2e2e6cb1 100644 --- a/kibana-reports/yarn.lock +++ b/kibana-reports/yarn.lock @@ -641,6 +641,13 @@ "@types/parse5" "*" "@types/tough-cookie" "*" +"@types/json2csv@^5.0.2": + version "5.0.2" + resolved "https://registry.yarnpkg.com/@types/json2csv/-/json2csv-5.0.2.tgz#64e0b5c9e46f3f8a55dd6670624c2a62a553ce1d" + integrity sha512-B9f75qwJxu+JLlUoFUmyM5g31fMOqoPEHd9bvWIwitWLeCI5e+KfsLTvPq1GuFh3OIM9Upc5vYkfOJDE/GMMQA== + dependencies: + "@types/node" "*" + "@types/node@*": version "14.6.3" resolved "https://registry.yarnpkg.com/@types/node/-/node-14.6.3.tgz#cc4f979548ca4d8e7b90bc0180052ab99ee64224" @@ -1836,6 +1843,11 @@ commander@^5.1.0: resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae" integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg== +commander@^6.1.0: + version "6.2.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" + integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== + common-tags@^1.8.0: version "1.8.0" resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.0.tgz#8e3153e542d4a39e9b10554434afaaf98956a937" @@ -2146,11 +2158,6 @@ decode-uri-component@^0.2.0: resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU= -deeks@2.2.6: - version "2.2.6" - resolved "https://registry.yarnpkg.com/deeks/-/deeks-2.2.6.tgz#7fc4c07c4251e1deb7e919fd14734fea151dc8b2" - integrity sha512-KlfWy96oUxxfAEC8q8QVoik++6yWsTYT9Cm5S6nUddLciZaOy7WDfcFbVCyeHKFO44ptWxtz7sQd9Upm4SGf4g== - deep-is@~0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" @@ -2212,7 +2219,7 @@ diffie-hellman@^5.0.0: miller-rabin "^4.0.0" randombytes "^2.0.0" -doc-path@2.0.4, doc-path@2.1.2: +doc-path@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/doc-path/-/doc-path-2.1.2.tgz#08344d188619a6ff7c6631e462362dd720e4e81f" integrity sha512-saM17czrIb4jYLsS5728OKbCa/WQ3xVctkGiMixOHz3X1VYsRn/Q5xPMxE1A5WN+XHHLWak34mMMjmAKRgMLeA== @@ -3839,14 +3846,6 @@ jsesc@^2.5.1: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== -json-2-csv@^3.7.6: - version "3.7.6" - resolved "https://registry.yarnpkg.com/json-2-csv/-/json-2-csv-3.7.6.tgz#2a88b323b8986f31dcd4ef785b3bfb3521d42899" - integrity sha512-um9bT7CgI77f/m5G/V5uRKz9UruFQFijD5Fh+EAnr5QowOJpdIk1ya37KSq1MxjBnJgvmpqCnI7ZjsvyTYiW0w== - dependencies: - deeks "2.2.6" - doc-path "2.0.4" - json-parse-better-errors@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" @@ -3872,6 +3871,15 @@ json-stringify-safe@~5.0.1: resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= +json2csv@^5.0.6: + version "5.0.6" + resolved "https://registry.yarnpkg.com/json2csv/-/json2csv-5.0.6.tgz#590e0e1b9579e59baa53bda0c0d840f4d8009687" + integrity sha512-0/4Lv6IenJV0qj2oBdgPIAmFiKKnh8qh7bmLFJ+/ZZHLjSeiL3fKKGX3UryvKPbxFbhV+JcYo9KUC19GJ/Z/4A== + dependencies: + commander "^6.1.0" + jsonparse "^1.3.1" + lodash.get "^4.4.2" + json5@2.x, json5@^2.1.2: version "2.1.3" resolved "https://registry.yarnpkg.com/json5/-/json5-2.1.3.tgz#c9b0f7fa9233bfe5807fe66fcf3a5617ed597d43" @@ -3895,6 +3903,11 @@ jsonfile@^6.0.1: optionalDependencies: graceful-fs "^4.1.6" +jsonparse@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" + integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA= + jsprim@^1.2.2: version "1.4.1" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" @@ -4055,6 +4068,11 @@ lodash.foreach@^4.3.0: resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53" integrity sha1-Gmo16s5AEoDH8G3d7DUWWrJ+PlM= +lodash.get@^4.4.2: + version "4.4.2" + resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" + integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk= + lodash.has@^4.5.2: version "4.5.2" resolved "https://registry.yarnpkg.com/lodash.has/-/lodash.has-4.5.2.tgz#d19f4dc1095058cccbe2b0cdf4ee0fe4aa37c862"