Fix csv parsing function, switch json to csv library

Signed-off-by: Joshua Li <joshuali925@gmail.com>

kibana-reports/package.json

@@ -30,7 +30,7 @@
     "jest-fetch-mock": "^3.0.3",
     "jquery": "^3.5.0",
     "jsdom": "13.1.0",
-    "json-2-csv": "^3.7.6",
+    "json2csv": "^5.0.6",
     "puppeteer-core": "^1.19.0",
     "react-addons-test-utils": "^15.6.2",
     "react-id-generator": "^3.0.1",
@@ -50,6 +50,7 @@
     "@types/dompurify": "^2.0.4",
     "@types/enzyme-adapter-react-16": "^1.0.6",
     "@types/jsdom": "^16.2.3",
+    "@types/json2csv": "^5.0.2",
     "@types/puppeteer-core": "^2.0.0",
     "@types/react": "^16.9.36",
     "@types/react-addons-test-utils": "^0.14.25",

kibana-reports/server/routes/utils/dataReportHelpers.ts

@@ -17,7 +17,8 @@ import { DATA_REPORT_CONFIG } from './constants';
 
 import esb from 'elastic-builder';
 import moment from 'moment';
-import converter from 'json-2-csv';
+import { parseAsync,  transforms as Transforms } from 'json2csv';
+import _ from 'lodash';
 
 export var metaData = {
   saved_search_id: <string>null,
@@ -168,7 +169,7 @@ export const getEsData = (arrayHits, report, params) => {
       }
       delete data['fields'];
       if (report._source.fields_exist === true) {
-        let result = traverse(data, report._source.selectedFields);
+        let result = traverse(data._source, report._source.selectedFields);
         hits.push(params.excel ? sanitize(result) : result);
       } else {
         hits.push(params.excel ? sanitize(data) : data);
@@ -185,34 +186,16 @@ export const getEsData = (arrayHits, report, params) => {
 
 //Convert the data to Csv format
 export const convertToCSV = async (dataset) => {
-  let convertedData: any = [];
-  const options = {
-    delimiter: { field: ',', eol: '\n' },
-    emptyFieldValue: ' ',
-  };
-  await converter.json2csvAsync(dataset[0], options).then((csv) => {
-    convertedData = csv;
-  });
-  return convertedData;
+  const transforms = [Transforms.flatten()];
+  return parseAsync(dataset[0], { transforms });
 };
 
 //Return only the selected fields
 function traverse(data, keys, result = {}) {
-  for (let k of Object.keys(data)) {
-    if (keys.includes(k)) {
-      result = Object.assign({}, result, {
-        [k]: data[k],
-      });
-      continue;
-    }
-    if (
-      data[k] &&
-      typeof data[k] === 'object' &&
-      Object.keys(data[k]).length > 0
-    ) {
-      result = traverse(data[k], keys, result);
-    }
-  }
+  keys.forEach((key) => {
+    const value = _.get(data, key, undefined);
+    if (value !== undefined) result[key] = value;
+  });
   return result;
 }
 

kibana-reports/yarn.lock

@@ -641,6 +641,13 @@
     "@types/parse5" "*"
     "@types/tough-cookie" "*"
 
+"@types/json2csv@^5.0.2":
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/@types/json2csv/-/json2csv-5.0.2.tgz#64e0b5c9e46f3f8a55dd6670624c2a62a553ce1d"
+  integrity sha512-B9f75qwJxu+JLlUoFUmyM5g31fMOqoPEHd9bvWIwitWLeCI5e+KfsLTvPq1GuFh3OIM9Upc5vYkfOJDE/GMMQA==
+  dependencies:
+    "@types/node" "*"
+
 "@types/node@*":
   version "14.6.3"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.6.3.tgz#cc4f979548ca4d8e7b90bc0180052ab99ee64224"
@@ -1836,6 +1843,11 @@ commander@^5.1.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae"
   integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==
 
+commander@^6.1.0:
+  version "6.2.1"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c"
+  integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==
+
 common-tags@^1.8.0:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.0.tgz#8e3153e542d4a39e9b10554434afaaf98956a937"
@@ -2146,11 +2158,6 @@ decode-uri-component@^0.2.0:
   resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545"
   integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=
 
-deeks@2.2.6:
-  version "2.2.6"
-  resolved "https://registry.yarnpkg.com/deeks/-/deeks-2.2.6.tgz#7fc4c07c4251e1deb7e919fd14734fea151dc8b2"
-  integrity sha512-KlfWy96oUxxfAEC8q8QVoik++6yWsTYT9Cm5S6nUddLciZaOy7WDfcFbVCyeHKFO44ptWxtz7sQd9Upm4SGf4g==
-
 deep-is@~0.1.3:
   version "0.1.3"
   resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
@@ -2212,7 +2219,7 @@ diffie-hellman@^5.0.0:
     miller-rabin "^4.0.0"
     randombytes "^2.0.0"
 
-doc-path@2.0.4, doc-path@2.1.2:
+doc-path@2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/doc-path/-/doc-path-2.1.2.tgz#08344d188619a6ff7c6631e462362dd720e4e81f"
   integrity sha512-saM17czrIb4jYLsS5728OKbCa/WQ3xVctkGiMixOHz3X1VYsRn/Q5xPMxE1A5WN+XHHLWak34mMMjmAKRgMLeA==
@@ -3839,14 +3846,6 @@ jsesc@^2.5.1:
   resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4"
   integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==
 
-json-2-csv@^3.7.6:
-  version "3.7.6"
-  resolved "https://registry.yarnpkg.com/json-2-csv/-/json-2-csv-3.7.6.tgz#2a88b323b8986f31dcd4ef785b3bfb3521d42899"
-  integrity sha512-um9bT7CgI77f/m5G/V5uRKz9UruFQFijD5Fh+EAnr5QowOJpdIk1ya37KSq1MxjBnJgvmpqCnI7ZjsvyTYiW0w==
-  dependencies:
-    deeks "2.2.6"
-    doc-path "2.0.4"
-
 json-parse-better-errors@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9"
@@ -3872,6 +3871,15 @@ json-stringify-safe@~5.0.1:
   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
   integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=
 
+json2csv@^5.0.6:
+  version "5.0.6"
+  resolved "https://registry.yarnpkg.com/json2csv/-/json2csv-5.0.6.tgz#590e0e1b9579e59baa53bda0c0d840f4d8009687"
+  integrity sha512-0/4Lv6IenJV0qj2oBdgPIAmFiKKnh8qh7bmLFJ+/ZZHLjSeiL3fKKGX3UryvKPbxFbhV+JcYo9KUC19GJ/Z/4A==
+  dependencies:
+    commander "^6.1.0"
+    jsonparse "^1.3.1"
+    lodash.get "^4.4.2"
+
 json5@2.x, json5@^2.1.2:
   version "2.1.3"
   resolved "https://registry.yarnpkg.com/json5/-/json5-2.1.3.tgz#c9b0f7fa9233bfe5807fe66fcf3a5617ed597d43"
@@ -3895,6 +3903,11 @@ jsonfile@^6.0.1:
   optionalDependencies:
     graceful-fs "^4.1.6"
 
+jsonparse@^1.3.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280"
+  integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=
+
 jsprim@^1.2.2:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"
@@ -4055,6 +4068,11 @@ lodash.foreach@^4.3.0:
   resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53"
   integrity sha1-Gmo16s5AEoDH8G3d7DUWWrJ+PlM=
 
+lodash.get@^4.4.2:
+  version "4.4.2"
+  resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
+  integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=
+
 lodash.has@^4.5.2:
   version "4.5.2"
   resolved "https://registry.yarnpkg.com/lodash.has/-/lodash.has-4.5.2.tgz#d19f4dc1095058cccbe2b0cdf4ee0fe4aa37c862"