Fix csv parsing function (opensearch-project#53)

* Fix csv parsing function Signed-off-by: Joshua Li <joshuali925@gmail.com> * Flatten nested results for csv Signed-off-by: Joshua Li <joshuali925@gmail.com> * Add test case for nested fields Signed-off-by: Joshua Li <joshuali925@gmail.com>
kavilla · May 25, 2021 · b80ad2a · b80ad2a
1 parent 2067dd3
commit b80ad2a
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 22 deletions.
diff --git a/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/server/routes/utils/__tests__/savedSearchReportHelper.test.ts
@@ -249,6 +249,31 @@ describe('test create saved search report', () => {
     );
   }, 20000);
 
+  test('create report for data set with nested fields', async () => {
+    const hits = [
+      hit({
+        'geoip.country_iso_code': 'GB',
+        'geoip.location': { lon: -0.1, lat: 51.5 },
+      }),
+      hit({
+        'geoip.country_iso_code': 'US',
+        'geoip.city_name': 'New York',
+        'geoip.location': { lon: -74, lat: 40.8 },
+      }),
+    ];
+    const client = mockOpenSearchClient(
+      hits,
+      '"geoip.country_iso_code", "geoip.city_name", "geoip.location"'
+    );
+    const { dataUrl } = await createSavedSearchReport(input, client);
+
+    expect(dataUrl).toEqual(
+      'geoip.country_iso_code,geoip.location.lon,geoip.location.lat,geoip.city_name\n' +
+        'GB,-0.1,51.5, \n' +
+        'US,-74,40.8,New York'
+    );
+  }, 20000);
+
   test('create report by sanitizing data set for Excel', async () => {
     const hits = [
       hit({ category: 'c1', customer_gender: '=Male' }),
@@ -312,7 +337,10 @@ test('create report for data set contains null field value', async () => {
 /**
  * Mock Elasticsearch client and return different mock objects based on endpoint and parameters.
  */
-function mockOpenSearchClient(mockHits: Array<{ _source: any }>) {
+function mockOpenSearchClient(
+  mockHits: Array<{ _source: any }>,
+  columns = '"category", "customer_gender"'
+) {
   let call = 0;
   const client = jest.fn();
   client.callAsInternalUser = jest
@@ -323,7 +351,7 @@ function mockOpenSearchClient(mockHits: Array<{ _source: any }>) {
           return {
             _source: params.id.startsWith('index-pattern:')
               ? mockIndexPattern()
-              : mockSavedSearch(),
+              : mockSavedSearch(columns),
           };
         case 'indices.getSettings':
           return mockIndexSettings();
@@ -357,9 +385,9 @@ function mockOpenSearchClient(mockHits: Array<{ _source: any }>) {
 }
 
 /**
- * Mock a saved search for opensearch_dashboards_sample_data_ecommerce with 2 selected fields: category and customer_gender.
+ * Mock a saved search for opensearch_dashboards_sample_data_ecommerce with 2 default selected fields: category and customer_gender.
  */
-function mockSavedSearch() {
+function mockSavedSearch(columns = '"category", "customer_gender"') {
   return JSON.parse(`
   {
     "type": "search",
@@ -368,10 +396,7 @@ function mockSavedSearch() {
       "title": "Show category and gender",
       "description": "",
       "hits": 0,
-      "columns": [
-        "category",
-        "customer_gender"
-      ],
+      "columns": [ ${columns} ],
       "sort": [],
       "version": 1,
       "opensearchDashboardsSavedObjectMeta": {

diff --git a/server/routes/utils/dataReportHelpers.ts b/server/routes/utils/dataReportHelpers.ts
@@ -29,6 +29,7 @@ import { DATA_REPORT_CONFIG } from './constants';
 import esb from 'elastic-builder';
 import moment from 'moment';
 import converter from 'json-2-csv';
+import _ from 'lodash';
 
 export var metaData = {
   saved_search_id: <string>null,
@@ -181,7 +182,7 @@ export const getOpenSearchData = (arrayHits, report, params) => {
       }
       delete data['fields'];
       if (report._source.fields_exist === true) {
-        let result = traverse(data, report._source.selectedFields);
+        let result = traverse(data._source, report._source.selectedFields);
         hits.push(params.excel ? sanitize(result) : result);
       } else {
         hits.push(params.excel ? sanitize(data) : data);
@@ -208,26 +209,39 @@ export const convertToCSV = async (dataset) => {
   return convertedData;
 };
 
-//Return only the selected fields
-function traverse(data, keys, result = {}) {
-  for (let k of Object.keys(data)) {
-    if (keys.includes(k)) {
-      result = Object.assign({}, result, {
-        [k]: data[k],
-      });
-      continue;
-    }
+function flattenHits(hits, result = {}, prefix = '') {
+  for (const [key, value] of Object.entries(hits)) {
+    if (!hits.hasOwnProperty(key)) continue;
     if (
-      data[k] &&
-      typeof data[k] === 'object' &&
-      Object.keys(data[k]).length > 0
+      value != null &&
+      typeof value === 'object' &&
+      !Array.isArray(value) &&
+      Object.keys(value).length > 0
     ) {
-      result = traverse(data[k], keys, result);
+      flattenHits(value, result, prefix + key + '.');
+    } else {
+      result[prefix + key] = value;
     }
   }
   return result;
 }
 
+//Return only the selected fields
+function traverse(data, keys, result = {}) {
+  data = flattenHits(data);
+  const sourceKeys = Object.keys(data);
+  keys.forEach((key) => {
+    const value = _.get(data, key, undefined);
+    if (value !== undefined) result[key] = value;
+    else {
+      Object.keys(data)
+        .filter((sourceKey) => sourceKey.startsWith(key + '.'))
+        .forEach((sourceKey) => (result[sourceKey] = data[sourceKey]));
+    }
+  });
+  return result;
+}
+
 /**
  * Escape special characters if field value prefixed with.
  * This is intend to avoid CSV injection in Microsoft Excel.