Fixing exceptions export format (#114920) (#114949)

### Summary

Fixing exceptions export format and adding integration tests for it.

Co-authored-by: Yara Tercero <yctercero@users.noreply.github.com>

packages/kbn-securitysolution-list-api/src/api/index.ts

@@ -558,7 +558,7 @@ export const exportExceptionList = async ({
   signal,
 }: ExportExceptionListProps): Promise<Blob> =>
   http.fetch<Blob>(`${EXCEPTION_LIST_URL}/_export`, {
-    method: 'GET',
+    method: 'POST',
     query: { id, list_id: listId, namespace_type: namespaceType },
     signal,
   });

packages/kbn-securitysolution-utils/src/index.ts

@@ -7,3 +7,4 @@
  */
 
 export * from './add_remove_id_to_item';
+export * from './transform_data_to_ndjson';

packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.test.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { transformDataToNdjson } from './';
+
+export const ANCHOR_DATE = '2020-02-20T03:57:54.037Z';
+
+const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE) => ({
+  author: [],
+  id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
+  created_at: new Date(anchorDate).toISOString(),
+  updated_at: new Date(anchorDate).toISOString(),
+  created_by: 'elastic',
+  description: 'some description',
+  enabled: true,
+  false_positives: ['false positive 1', 'false positive 2'],
+  from: 'now-6m',
+  immutable: false,
+  name: 'Query with a rule id',
+  query: 'user.name: root or user.name: admin',
+  references: ['test 1', 'test 2'],
+  severity: 'high',
+  severity_mapping: [],
+  updated_by: 'elastic_kibana',
+  tags: ['some fake tag 1', 'some fake tag 2'],
+  to: 'now',
+  type: 'query',
+  threat: [],
+  version: 1,
+  status: 'succeeded',
+  status_date: '2020-02-22T16:47:50.047Z',
+  last_success_at: '2020-02-22T16:47:50.047Z',
+  last_success_message: 'succeeded',
+  output_index: '.siem-signals-default',
+  max_signals: 100,
+  risk_score: 55,
+  risk_score_mapping: [],
+  language: 'kuery',
+  rule_id: 'query-rule-id',
+  interval: '5m',
+  exceptions_list: [],
+});
+
+describe('transformDataToNdjson', () => {
+  test('if rules are empty it returns an empty string', () => {
+    const ruleNdjson = transformDataToNdjson([]);
+    expect(ruleNdjson).toEqual('');
+  });
+
+  test('single rule will transform with new line ending character for ndjson', () => {
+    const rule = getRulesSchemaMock();
+    const ruleNdjson = transformDataToNdjson([rule]);
+    expect(ruleNdjson.endsWith('\n')).toBe(true);
+  });
+
+  test('multiple rules will transform with two new line ending characters for ndjson', () => {
+    const result1 = getRulesSchemaMock();
+    const result2 = getRulesSchemaMock();
+    result2.id = 'some other id';
+    result2.rule_id = 'some other id';
+    result2.name = 'Some other rule';
+
+    const ruleNdjson = transformDataToNdjson([result1, result2]);
+    // this is how we count characters in JavaScript :-)
+    const count = ruleNdjson.split('\n').length - 1;
+    expect(count).toBe(2);
+  });
+
+  test('you can parse two rules back out without errors', () => {
+    const result1 = getRulesSchemaMock();
+    const result2 = getRulesSchemaMock();
+    result2.id = 'some other id';
+    result2.rule_id = 'some other id';
+    result2.name = 'Some other rule';
+
+    const ruleNdjson = transformDataToNdjson([result1, result2]);
+    const ruleStrings = ruleNdjson.split('\n');
+    const reParsed1 = JSON.parse(ruleStrings[0]);
+    const reParsed2 = JSON.parse(ruleStrings[1]);
+    expect(reParsed1).toEqual(result1);
+    expect(reParsed2).toEqual(result2);
+  });
+});

packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const transformDataToNdjson = (data: unknown[]): string => {
+  if (data.length !== 0) {
+    const dataString = data.map((item) => JSON.stringify(item)).join('\n');
+    return `${dataString}\n`;
+  } else {
+    return '';
+  }
+};

x-pack/plugins/lists/public/exceptions/api.test.ts

@@ -759,7 +759,7 @@ describe('Exceptions Lists API', () => {
       });
 
       expect(httpMock.fetch).toHaveBeenCalledWith('/api/exception_lists/_export', {
-        method: 'GET',
+        method: 'POST',
         query: {
           id: 'some-id',
           list_id: 'list-id',

x-pack/plugins/lists/server/routes/export_exception_list_route.ts

@@ -6,6 +6,7 @@
  */
 
 import { transformError } from '@kbn/securitysolution-es-utils';
+import { transformDataToNdjson } from '@kbn/securitysolution-utils';
 import { exportExceptionListQuerySchema } from '@kbn/securitysolution-io-ts-list-types';
 import { EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants';
 
@@ -14,7 +15,7 @@ import type { ListsPluginRouter } from '../types';
 import { buildRouteValidation, buildSiemResponse, getExceptionListClient } from './utils';
 
 export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
-  router.get(
+  router.post(
     {
       options: {
         tags: ['access:lists-read'],
@@ -26,6 +27,7 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
     },
     async (context, request, response) => {
       const siemResponse = buildSiemResponse(response);
+
       try {
         const { id, list_id: listId, namespace_type: namespaceType } = request.query;
         const exceptionLists = getExceptionListClient(context);
@@ -37,11 +39,10 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
 
         if (exceptionList == null) {
           return siemResponse.error({
-            body: `list_id: ${listId} does not exist`,
+            body: `exception list with list_id: ${listId} does not exist`,
             statusCode: 400,
           });
         } else {
-          const { exportData: exportList } = getExport([exceptionList]);
           const listItems = await exceptionLists.findExceptionListItem({
             filter: undefined,
             listId,
@@ -51,19 +52,15 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
             sortField: 'exception-list.created_at',
             sortOrder: 'desc',
           });
+          const exceptionItems = listItems?.data ?? [];
 
-          const { exportData: exportListItems, exportDetails } = getExport(listItems?.data ?? []);
-
-          const responseBody = [
-            exportList,
-            exportListItems,
-            { exception_list_items_details: exportDetails },
-          ];
+          const { exportData } = getExport([exceptionList, ...exceptionItems]);
+          const { exportDetails } = getExportDetails(exceptionItems);
 
           // TODO: Allow the API to override the name of the file to export
           const fileName = exceptionList.list_id;
           return response.ok({
-            body: transformDataToNdjson(responseBody),
+            body: `${exportData}${exportDetails}`,
             headers: {
               'Content-Disposition': `attachment; filename="${fileName}"`,
               'Content-Type': 'application/ndjson',
@@ -81,24 +78,23 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
   );
 };
 
-const transformDataToNdjson = (data: unknown[]): string => {
-  if (data.length !== 0) {
-    const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n');
-    return `${dataString}\n`;
-  } else {
-    return '';
-  }
-};
-
 export const getExport = (
   data: unknown[]
 ): {
   exportData: string;
-  exportDetails: string;
 } => {
   const ndjson = transformDataToNdjson(data);
+
+  return { exportData: ndjson };
+};
+
+export const getExportDetails = (
+  items: unknown[]
+): {
+  exportDetails: string;
+} => {
   const exportDetails = JSON.stringify({
-    exported_count: data.length,
+    exported_list_items_count: items.length,
   });
-  return { exportData: ndjson, exportDetails: `${exportDetails}\n` };
+  return { exportDetails: `${exportDetails}\n` };
 };

x-pack/plugins/security_solution/cypress/downloads/test_exception_list.ndjson

@@ -0,0 +1,2 @@
+{"_version":"WzQyNjA0LDFd","created_at":"2021-10-14T01:30:22.034Z","created_by":"elastic","description":"Test exception list description","id":"4c65a230-2c8e-11ec-be1c-2bbdec602f88","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"b04983b4-1617-441c-bb6c-c729281fa2e9","type":"detection","updated_at":"2021-10-14T01:30:22.036Z","updated_by":"elastic","version":1}
+{"exported_list_items_count":0}

x-pack/plugins/security_solution/cypress/integration/exceptions/exceptions_table.spec.ts

@@ -81,7 +81,7 @@ describe('Exceptions Table', () => {
 
     cy.wait('@export').then(({ response }) =>
       cy
-        .wrap(response?.body!)
+        .wrap(response?.body)
         .should('eql', expectedExportedExceptionList(this.exceptionListResponse))
     );
   });

x-pack/plugins/security_solution/cypress/objects/exception.ts

@@ -41,6 +41,5 @@ export const expectedExportedExceptionList = (
   exceptionListResponse: Cypress.Response<ExceptionListItemSchema>
 ): string => {
   const jsonrule = exceptionListResponse.body;
-
-  return `"{\\"_version\\":\\"${jsonrule._version}\\",\\"created_at\\":\\"${jsonrule.created_at}\\",\\"created_by\\":\\"elastic\\",\\"description\\":\\"${jsonrule.description}\\",\\"id\\":\\"${jsonrule.id}\\",\\"immutable\\":false,\\"list_id\\":\\"test_exception_list\\",\\"name\\":\\"Test exception list\\",\\"namespace_type\\":\\"single\\",\\"os_types\\":[],\\"tags\\":[],\\"tie_breaker_id\\":\\"${jsonrule.tie_breaker_id}\\",\\"type\\":\\"detection\\",\\"updated_at\\":\\"${jsonrule.updated_at}\\",\\"updated_by\\":\\"elastic\\",\\"version\\":1}\\n"\n""\n{"exception_list_items_details":"{\\"exported_count\\":0}\\n"}\n`;
+  return `{"_version":"${jsonrule._version}","created_at":"${jsonrule.created_at}","created_by":"elastic","description":"${jsonrule.description}","id":"${jsonrule.id}","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"${jsonrule.tie_breaker_id}","type":"detection","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","version":1}\n{"exported_list_items_count":0}\n`;
 };

x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.ts

@@ -5,11 +5,12 @@
  * 2.0.
  */
 
+import { transformDataToNdjson } from '@kbn/securitysolution-utils';
+
 import { RulesClient } from '../../../../../alerting/server';
 import { getNonPackagedRules } from './get_existing_prepackaged_rules';
 import { getExportDetailsNdjson } from './get_export_details_ndjson';
 import { transformAlertsToRules } from '../routes/rules/utils';
-import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';
 
 export const getExportAll = async (
   rulesClient: RulesClient,

x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.ts

@@ -6,13 +6,13 @@
  */
 
 import { chunk } from 'lodash';
+import { transformDataToNdjson } from '@kbn/securitysolution-utils';
 
 import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
 import { RulesClient } from '../../../../../alerting/server';
 import { getExportDetailsNdjson } from './get_export_details_ndjson';
 import { isAlertType } from '../rules/types';
 import { transformAlertToRule } from '../routes/rules/utils';
-import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';
 import { INTERNAL_RULE_ID_KEY } from '../../../../common/constants';
 import { findRules } from './find_rules';
 

x-pack/plugins/security_solution/server/lib/telemetry/sender.ts

@@ -8,10 +8,11 @@
 import { cloneDeep } from 'lodash';
 import axios from 'axios';
 import { URL } from 'url';
+import { transformDataToNdjson } from '@kbn/securitysolution-utils';
+
 import { Logger } from 'src/core/server';
 import { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server';
 import { UsageCounter } from 'src/plugins/usage_collection/server';
-import { transformDataToNdjson } from '../../utils/read_stream/create_stream_from_ndjson';
 import {
   TaskManagerSetupContract,
   TaskManagerStartContract,

x-pack/plugins/security_solution/server/lib/timeline/routes/timelines/export_timelines/helpers.ts

@@ -6,6 +6,7 @@
  */
 
 import { omit } from 'lodash/fp';
+import { transformDataToNdjson } from '@kbn/securitysolution-utils';
 
 import {
   ExportedTimelines,
@@ -15,8 +16,6 @@ import {
 import { NoteSavedObject } from '../../../../../../common/types/timeline/note';
 import { PinnedEventSavedObject } from '../../../../../../common/types/timeline/pinned_event';
 
-import { transformDataToNdjson } from '../../../../../utils/read_stream/create_stream_from_ndjson';
-
 import { FrameworkRequest } from '../../../../framework';
 import * as noteLib from '../../../saved_object/notes';
 import * as pinnedEventLib from '../../../saved_object/pinned_events';

x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.ts

@@ -48,12 +48,3 @@ export const createLimitStream = (limit: number): Transform => {
     },
   });
 };
-
-export const transformDataToNdjson = (data: unknown[]): string => {
-  if (data.length !== 0) {
-    const dataString = data.map((rule) => JSON.stringify(rule)).join('\n');
-    return `${dataString}\n`;
-  } else {
-    return '';
-  }
-};