fix csv missing fields issue and empty csv on _source fields

opensearch-project · Nov 1, 2021 · 343a778 · 343a778
1 parent 56008de
commit 343a778
Show file tree

Hide file tree

Showing 6 changed files with 119 additions and 38 deletions.
diff --git a/.github/workflows/dashboards-reports-test-and-build-workflow.yml b/.github/workflows/dashboards-reports-test-and-build-workflow.yml
@@ -5,7 +5,7 @@ on: [pull_request, push]
 env:
   PLUGIN_NAME: reportsDashboards
   ARTIFACT_NAME: reports-dashboards
-  OPENSEARCH_VERSION: '1.x'
+  OPENSEARCH_VERSION: '1.1'
   OPENSEARCH_PLUGIN_VERSION: 1.1.0.0
 
 jobs:

diff --git a/dashboards-reports/server/routes/lib/createReport.ts b/dashboards-reports/server/routes/lib/createReport.ts
@@ -69,7 +69,8 @@ export const createReport = async (
   // @ts-ignore
   const timezone = request.query.timezone;
   // @ts-ignore
-  const dateFormat = request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat;
+  const dateFormat =
+    request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat;
   const {
     basePath,
     serverInfo: { protocol, port, hostname },
@@ -97,7 +98,8 @@ export const createReport = async (
         report,
         opensearchClient,
         dateFormat,
-        isScheduledTask
+        isScheduledTask,
+        logger
       );
     } else {
       // report source can only be one of [saved search, visualization, dashboard, notebook]

diff --git a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts
@@ -27,6 +27,7 @@
 import 'regenerator-runtime/runtime';
 import { createSavedSearchReport } from '../savedSearchReportHelper';
 import { reportSchema } from '../../../model';
+import mockLogger from '../../../../test/__mocks__/loggerMock';
 
 /**
  * The mock and sample input for saved search export function.
@@ -54,14 +55,16 @@ const input = {
       configIds: [],
       title: 'title',
       textDescription: 'text description',
-      htmlDescription: 'html description'
+      htmlDescription: 'html description',
     },
     trigger: {
       trigger_type: 'On demand',
     },
   },
 };
 
+const mockDateFormat = 'date_hour_minute_second_fraction';
+
 /**
  * Max result window size in OpenSearch index settings.
  */
@@ -86,22 +89,34 @@ describe('test create saved search report', () => {
   test('create report with expected file name extension', async () => {
     const csvReport = await createSavedSearchReport(
       input,
-      mockOpenSearchClient([])
+      mockOpenSearchClient([]),
+      mockDateFormat,
+      undefined,
+      mockLogger
     );
     expect(csvReport.fileName).toContain('.csv');
 
     input.report_definition.report_params.core_params.report_format = 'xlsx';
     const xlsxReport = await createSavedSearchReport(
       input,
-      mockOpenSearchClient([])
+      mockOpenSearchClient([]),
+      mockDateFormat,
+      undefined,
+      mockLogger
     );
     expect(xlsxReport.fileName).toContain('.xlsx');
   }, 20000);
 
   test('create report for empty data set', async () => {
     const hits: Array<{ _source: any }> = [];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
     expect(dataUrl).toEqual('');
   }, 20000);
 
@@ -114,7 +129,13 @@ describe('test create saved search report', () => {
       hit({ category: 'c5', customer_gender: 'Male' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -141,7 +162,13 @@ describe('test create saved search report', () => {
       hit({ category: 'c11', customer_gender: 'Male' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -171,7 +198,13 @@ describe('test create saved search report', () => {
       hit({ category: 'c5', customer_gender: 'Male' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual('category,customer_gender\n' + 'c1,Male');
   }, 20000);
@@ -193,7 +226,13 @@ describe('test create saved search report', () => {
       hit({ category: 'c10', customer_gender: 'Female' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -219,7 +258,13 @@ describe('test create saved search report', () => {
       hit({ category: 'c6', customer_gender: 'Female' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -239,7 +284,13 @@ describe('test create saved search report', () => {
       hit({ category: ',,c3', customer_gender: 'Male,,,' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -265,7 +316,13 @@ describe('test create saved search report', () => {
       hits,
       '"geoip.country_iso_code", "geoip.city_name", "geoip.location"'
     );
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'geoip.country_iso_code,geoip.location.lon,geoip.location.lat,geoip.city_name\n' +
@@ -283,7 +340,13 @@ describe('test create saved search report', () => {
       hit({ category: ',,,@c5', customer_gender: 'Male' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -307,7 +370,13 @@ describe('test create saved search report', () => {
       hit({ category: ',,,@c5', customer_gender: 'Male' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -327,7 +396,13 @@ test('create report for data set contains null field value', async () => {
     hit({ category: 'c3', customer_gender: null }),
   ];
   const client = mockOpenSearchClient(hits);
-  const { dataUrl } = await createSavedSearchReport(input, client);
+  const { dataUrl } = await createSavedSearchReport(
+    input,
+    client,
+    mockDateFormat,
+    undefined,
+    mockLogger
+  );
 
   expect(dataUrl).toEqual(
     'category,customer_gender\n' + 'c1,Ma\n' + 'c2,le\n' + 'c3, '

diff --git a/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts
@@ -28,18 +28,9 @@ import 'regenerator-runtime/runtime';
 import { createVisualReport } from '../visual_report/visualReportHelper';
 import { Logger } from '../../../../../../src/core/server';
 import { ReportParamsSchemaType, reportSchema } from '../../../model';
+import mockLogger from '../../../../test/__mocks__/loggerMock';
 
-const mockLogger: Logger = {
-  info: jest.fn(),
-  trace: jest.fn(),
-  warn: jest.fn(),
-  debug: jest.fn(),
-  error: jest.fn(),
-  fatal: jest.fn(),
-  log: jest.fn(),
-  get: jest.fn(),
-};
-
+const mockHeader = { mockKey: 'mockValue' };
 const input = {
   query_url: '/app/dashboards#/view/7adfa750-4c81-11e8-b3d7-01146121b73d',
   time_from: 1343576635300,
@@ -84,7 +75,8 @@ describe('test create visual report', () => {
     const { dataUrl, fileName } = await createVisualReport(
       reportParams as ReportParamsSchemaType,
       mockHtmlPath,
-      mockLogger
+      mockLogger,
+      mockHeader
     );
     expect(fileName).toContain(`${reportParams.report_name}`);
     expect(fileName).toContain('.png');
@@ -99,7 +91,8 @@ describe('test create visual report', () => {
     const { dataUrl, fileName } = await createVisualReport(
       reportParams as ReportParamsSchemaType,
       mockHtmlPath,
-      mockLogger
+      mockLogger,
+      mockHeader
     );
     expect(fileName).toContain(`${reportParams.report_name}`);
     expect(fileName).toContain('.pdf');

diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts
@@ -49,14 +49,17 @@ export var metaData = {
 // Get the selected columns by the user.
 export const getSelectedFields = async (columns) => {
   const selectedFields = [];
+  let fields_exist = false;
   for (let column of columns) {
     if (column !== '_source') {
-      metaData.fields_exist = true;
+      fields_exist = true;
       selectedFields.push(column);
     } else {
+      fields_exist = false;
       selectedFields.push('_source');
     }
   }
+  metaData.fields_exist = fields_exist;
   metaData.selectedFields = selectedFields;
 };
 
@@ -191,7 +194,7 @@ export const getOpenSearchData = (arrayHits, report, params, dateFormat: string)
       }
       delete data['fields'];
       if (report._source.fields_exist === true) {
-        let result = traverse(data._source, report._source.selectedFields);
+        let result = traverse(data, report._source.selectedFields);
         hits.push(params.excel ? sanitize(result) : result);
       } else {
         hits.push(params.excel ? sanitize(data) : data);
@@ -229,7 +232,7 @@ function flattenHits(hits, result = {}, prefix = '') {
     ) {
       flattenHits(value, result, prefix + key + '.');
     } else {
-      result[prefix + key] = value;
+      result[prefix.replace(/^_source\./, '') + key] = value;
     }
   }
   return result;

diff --git a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts
@@ -34,6 +34,7 @@ import {
 import {
   ILegacyClusterClient,
   ILegacyScopedClusterClient,
+  Logger,
 } from '../../../../../src/core/server';
 import { getFileName, callCluster } from './helpers';
 import { CreateReportResultType } from './types';
@@ -49,18 +50,20 @@ export async function createSavedSearchReport(
   report: any,
   client: ILegacyClusterClient | ILegacyScopedClusterClient,
   dateFormat: string,
-  isScheduledTask: boolean = true
+  isScheduledTask: boolean = true,
+  logger: Logger
 ): Promise<CreateReportResultType> {
   const params = report.report_definition.report_params;
   const reportFormat = params.core_params.report_format;
   const reportName = params.report_name;
 
-  await populateMetaData(client, report, isScheduledTask);
+  await populateMetaData(client, report, isScheduledTask, logger);
   const data = await generateReportData(
     client,
     params.core_params,
     dateFormat,
-    isScheduledTask
+    isScheduledTask,
+    logger
   );
 
   const curTime = new Date();
@@ -81,7 +84,8 @@ export async function createSavedSearchReport(
 async function populateMetaData(
   client: ILegacyClusterClient | ILegacyScopedClusterClient,
   report: any,
-  isScheduledTask: boolean
+  isScheduledTask: boolean,
+  logger: Logger
 ) {
   metaData.saved_search_id =
     report.report_definition.report_params.core_params.saved_search_id;
@@ -144,7 +148,8 @@ async function generateReportData(
   client: ILegacyClusterClient | ILegacyScopedClusterClient,
   params: any,
   dateFormat: string,
-  isScheduledTask: boolean
+  isScheduledTask: boolean,
+  logger: Logger
 ) {
   let opensearchData: any = {};
   const arrayHits: any = [];
@@ -159,6 +164,9 @@ async function generateReportData(
   }
 
   const reqBody = buildRequestBody(buildQuery(report, 0));
+  logger.info(
+    `[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}`
+  );
   if (total > maxResultSize) {
     await getOpenSearchDataByScroll();
   } else {