From 4dccbcad33d2411147a9d5b7ce6a58f473437703 Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Tue, 15 Dec 2020 19:50:31 -0700 Subject: [PATCH] [SecuritySolution][Detections] Resolves referential integrity issues when deleting value lists (#85925) ## Summary Resolves https://github.com/elastic/kibana/issues/77324, https://github.com/elastic/kibana/issues/77325, resolves https://github.com/elastic/kibana/issues/77325, and resolves https://github.com/elastic/kibana/issues/81302 This PR addresses referential integrity issues when deleting value lists. Previously when deleting value lists, any references in Exception Lists/Items would be left behind. This PR introduces a new confirmation modal when deleting value lists that are referenced in either space aware (`simple`) or space `agnostic` exception lists. Also includes: * Fixed Lists plugin `quick_start.sh` as it was using endpoint exception list + value lists (unsupported) * Adds `quick_start_value_list_references.sh` to create exception lists/items, value lists, and references to easily test * Add support to `findExceptionList` for searching for both `simple` and `agnostic` list types * Two new query params have been added to the `deleteListRoute` * `ignoreReferences` (default:false) when true, maintains pre-7.11 behavior of deleting value list without performing any additional checks. * NOTE: As written, this becomes an API breaking change as existing existing calls to the same API will `409` conflict if references exist. cc @jmikell821 @DonNateR * `deleteReferences` (default:false) to perform dry run and identify referenced exception lists/items ## Testing To test, run `quick_start_value_list_references.sh` and it will create all the necessary resources/references to easily exercise the above functionality. The below diagram details the resources created and how the references are wired up. > Creates three different exception lists and value lists, and associates as > below to test referential integrity functionality. > > NOTE: Endpoint lists don't support value lists, and are not tested here > > EL: Exception list > ELI Exception list Item > VL: Value list > > EL1 EL2 (Agnostic) EL3 > | | | > ELI1 ELI2 ELI3 > |\ /| | > | \ / | | > | \ / | | > | \ / | | > | \/ | | > | /\ | | > | / \ | | > | / \ | | > | / \ | | > |/ \| | > VL1 VL2 VL3 VL4 > ips.txt ip_range.txt text.txt hosts.txt > Corner cases to be aware of: * An exception item may have multiple value list entries -- only referenced value list entries should be removed * There is no API for removing individual entries. If all entries are references the entire item is deleted. If only some entries are references, the item is updated via a `PUT` (no `PATCH` support for exception items) * It's not possible via the UI to create a space agnostic list that has value list exception items (only agnostic endpoint exception lists can be created and they do not support value lists). Please use above script to exercise this behavior. Additional notes: * Once the Exception List table is introduced (https://github.com/elastic/kibana/pull/85465), we can add an enhancement for deeplinking to exception lists from the reference error modal. * The `deleteListRoute` response has been updated to include the responses from the reference checks to provide maximum flexibility * There is no bulk API for deleting exception list items, and so they are iterated over via the `deleteExceptionListItem` API. ##### Reference error modal

##### Overflow example

### Checklist Delete any items that are not applicable to this PR. - [X] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md) - [X] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [X] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) ### For maintainers - [X] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- x-pack/plugins/lists/common/constants.mock.ts | 1 + .../lists/common/format_errors.test.ts | 188 ++++++++++++++++++ x-pack/plugins/lists/common/format_errors.ts | 31 +++ .../request/delete_list_schema.mock.ts | 2 + .../schemas/request/delete_list_schema.ts | 19 +- .../default_string_boolean_false.test.ts | 101 ++++++++++ .../types/default_string_boolean_false.ts | 32 +++ x-pack/plugins/lists/common/test_utils.ts | 50 +++++ x-pack/plugins/lists/public/lists/api.test.ts | 8 +- x-pack/plugins/lists/public/lists/api.ts | 8 +- x-pack/plugins/lists/public/lists/types.ts | 2 + .../lists/server/routes/delete_list_route.ts | 177 ++++++++++++++++- .../new/exception_list_item_with_list.json | 12 +- .../exception_list_detection_1.json | 7 + .../exception_list_detection_2_agnostic.json | 8 + .../exception_list_detection_3.json | 7 + ...xception_list_item_1_multi_value_list.json | 35 ++++ .../exception_list_item_1_non_value_list.json | 28 +++ ...xception_list_item_2_multi_value_list.json | 36 ++++ ...ception_list_item_3_single_value_list.json | 26 +++ .../lists/server/scripts/patch_list.sh | 2 +- .../lists/server/scripts/patch_list_item.sh | 2 +- .../server/scripts/post_endpoint_list.sh | 2 +- .../server/scripts/post_endpoint_list_item.sh | 2 +- .../server/scripts/post_exception_list.sh | 2 +- .../scripts/post_exception_list_item.sh | 2 +- .../plugins/lists/server/scripts/post_list.sh | 2 +- .../lists/server/scripts/post_list_item.sh | 2 +- .../scripts/post_x_exception_list_items.sh | 2 +- .../quick_start_value_list_references.sh | 47 +++++ .../server/scripts/update_endpoint_item.sh | 2 +- .../server/scripts/update_exception_list.sh | 2 +- .../scripts/update_exception_list_item.sh | 2 +- .../lists/server/scripts/update_list.sh | 2 +- .../lists/server/scripts/update_list_item.sh | 2 +- .../exception_lists/exception_list_client.ts | 26 ++- .../exception_list_client_types.ts | 10 +- .../exception_lists/find_exception_list.ts | 25 ++- .../find_exception_list_items.ts | 37 +++- .../default_string_boolean_false.test.ts | 2 +- .../value_lists_management_modal/modal.tsx | 82 +++++++- .../reference_error_modal/index.tsx | 7 + .../reference_error_modal.tsx | 89 +++++++++ .../translations.ts | 28 +++ .../security_and_spaces/tests/delete_lists.ts | 161 ++++++++++++++- 45 files changed, 1267 insertions(+), 53 deletions(-) create mode 100644 x-pack/plugins/lists/common/format_errors.test.ts create mode 100644 x-pack/plugins/lists/common/format_errors.ts create mode 100644 x-pack/plugins/lists/common/schemas/types/default_string_boolean_false.test.ts create mode 100644 x-pack/plugins/lists/common/schemas/types/default_string_boolean_false.ts create mode 100644 x-pack/plugins/lists/common/test_utils.ts create mode 100644 x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_1.json create mode 100644 x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_2_agnostic.json create mode 100644 x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_3.json create mode 100644 x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_1_multi_value_list.json create mode 100644 x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_1_non_value_list.json create mode 100644 x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_2_multi_value_list.json create mode 100644 x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_3_single_value_list.json create mode 100755 x-pack/plugins/lists/server/scripts/quick_start_value_list_references.sh create mode 100644 x-pack/plugins/security_solution/public/detections/components/value_lists_management_modal/reference_error_modal/index.tsx create mode 100644 x-pack/plugins/security_solution/public/detections/components/value_lists_management_modal/reference_error_modal/reference_error_modal.tsx diff --git a/x-pack/plugins/lists/common/constants.mock.ts b/x-pack/plugins/lists/common/constants.mock.ts index 5385a116b29bc..d64a20e8d29de 100644 --- a/x-pack/plugins/lists/common/constants.mock.ts +++ b/x-pack/plugins/lists/common/constants.mock.ts @@ -41,6 +41,7 @@ export const NESTED_FIELD = 'parent.field'; // Exception List specific export const ID = 'uuid_here'; export const ITEM_ID = 'some-list-item-id'; +export const DETECTION_TYPE = 'detection'; export const ENDPOINT_TYPE = 'endpoint'; export const FIELD = 'host.name'; export const OPERATOR = 'included'; diff --git a/x-pack/plugins/lists/common/format_errors.test.ts b/x-pack/plugins/lists/common/format_errors.test.ts new file mode 100644 index 0000000000000..47b4aa19b911f --- /dev/null +++ b/x-pack/plugins/lists/common/format_errors.test.ts @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +import { formatErrors } from './format_errors'; + +describe('utils', () => { + test('returns an empty error message string if there are no errors', () => { + const errors: t.Errors = []; + const output = formatErrors(errors); + expect(output).toEqual([]); + }); + + test('returns a single error message if given one', () => { + const validationError: t.ValidationError = { + context: [], + message: 'some error', + value: 'Some existing error', + }; + const errors: t.Errors = [validationError]; + const output = formatErrors(errors); + expect(output).toEqual(['some error']); + }); + + test('returns a two error messages if given two', () => { + const validationError1: t.ValidationError = { + context: [], + message: 'some error 1', + value: 'Some existing error 1', + }; + const validationError2: t.ValidationError = { + context: [], + message: 'some error 2', + value: 'Some existing error 2', + }; + const errors: t.Errors = [validationError1, validationError2]; + const output = formatErrors(errors); + expect(output).toEqual(['some error 1', 'some error 2']); + }); + + test('it filters out duplicate error messages', () => { + const validationError1: t.ValidationError = { + context: [], + message: 'some error 1', + value: 'Some existing error 1', + }; + const validationError2: t.ValidationError = { + context: [], + message: 'some error 1', + value: 'Some existing error 1', + }; + const errors: t.Errors = [validationError1, validationError2]; + const output = formatErrors(errors); + expect(output).toEqual(['some error 1']); + }); + + test('will use message before context if it is set', () => { + const context: t.Context = ([{ key: 'some string key' }] as unknown) as t.Context; + const validationError1: t.ValidationError = { + context, + message: 'I should be used first', + value: 'Some existing error 1', + }; + const errors: t.Errors = [validationError1]; + const output = formatErrors(errors); + expect(output).toEqual(['I should be used first']); + }); + + test('will use context entry of a single string', () => { + const context: t.Context = ([{ key: 'some string key' }] as unknown) as t.Context; + const validationError1: t.ValidationError = { + context, + value: 'Some existing error 1', + }; + const errors: t.Errors = [validationError1]; + const output = formatErrors(errors); + expect(output).toEqual(['Invalid value "Some existing error 1" supplied to "some string key"']); + }); + + test('will use two context entries of two strings', () => { + const context: t.Context = ([ + { key: 'some string key 1' }, + { key: 'some string key 2' }, + ] as unknown) as t.Context; + const validationError1: t.ValidationError = { + context, + value: 'Some existing error 1', + }; + const errors: t.Errors = [validationError1]; + const output = formatErrors(errors); + expect(output).toEqual([ + 'Invalid value "Some existing error 1" supplied to "some string key 1,some string key 2"', + ]); + }); + + test('will filter out and not use any strings of numbers', () => { + const context: t.Context = ([ + { key: '5' }, + { key: 'some string key 2' }, + ] as unknown) as t.Context; + const validationError1: t.ValidationError = { + context, + value: 'Some existing error 1', + }; + const errors: t.Errors = [validationError1]; + const output = formatErrors(errors); + expect(output).toEqual([ + 'Invalid value "Some existing error 1" supplied to "some string key 2"', + ]); + }); + + test('will filter out and not use null', () => { + const context: t.Context = ([ + { key: null }, + { key: 'some string key 2' }, + ] as unknown) as t.Context; + const validationError1: t.ValidationError = { + context, + value: 'Some existing error 1', + }; + const errors: t.Errors = [validationError1]; + const output = formatErrors(errors); + expect(output).toEqual([ + 'Invalid value "Some existing error 1" supplied to "some string key 2"', + ]); + }); + + test('will filter out and not use empty strings', () => { + const context: t.Context = ([ + { key: '' }, + { key: 'some string key 2' }, + ] as unknown) as t.Context; + const validationError1: t.ValidationError = { + context, + value: 'Some existing error 1', + }; + const errors: t.Errors = [validationError1]; + const output = formatErrors(errors); + expect(output).toEqual([ + 'Invalid value "Some existing error 1" supplied to "some string key 2"', + ]); + }); + + test('will use a name context if it cannot find a keyContext', () => { + const context: t.Context = ([ + { key: '' }, + { key: '', type: { name: 'someName' } }, + ] as unknown) as t.Context; + const validationError1: t.ValidationError = { + context, + value: 'Some existing error 1', + }; + const errors: t.Errors = [validationError1]; + const output = formatErrors(errors); + expect(output).toEqual(['Invalid value "Some existing error 1" supplied to "someName"']); + }); + + test('will return an empty string if name does not exist but type does', () => { + const context: t.Context = ([{ key: '' }, { key: '', type: {} }] as unknown) as t.Context; + const validationError1: t.ValidationError = { + context, + value: 'Some existing error 1', + }; + const errors: t.Errors = [validationError1]; + const output = formatErrors(errors); + expect(output).toEqual(['Invalid value "Some existing error 1" supplied to ""']); + }); + + test('will stringify an error value', () => { + const context: t.Context = ([ + { key: '' }, + { key: 'some string key 2' }, + ] as unknown) as t.Context; + const validationError1: t.ValidationError = { + context, + value: { foo: 'some error' }, + }; + const errors: t.Errors = [validationError1]; + const output = formatErrors(errors); + expect(output).toEqual([ + 'Invalid value "{"foo":"some error"}" supplied to "some string key 2"', + ]); + }); +}); diff --git a/x-pack/plugins/lists/common/format_errors.ts b/x-pack/plugins/lists/common/format_errors.ts new file mode 100644 index 0000000000000..4e1f5e4796152 --- /dev/null +++ b/x-pack/plugins/lists/common/format_errors.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { isObject } from 'lodash/fp'; + +export const formatErrors = (errors: t.Errors): string[] => { + const err = errors.map((error) => { + if (error.message != null) { + return error.message; + } else { + const keyContext = error.context + .filter( + (entry) => entry.key != null && !Number.isInteger(+entry.key) && entry.key.trim() !== '' + ) + .map((entry) => entry.key) + .join(','); + + const nameContext = error.context.find((entry) => entry.type?.name?.length > 0); + const suppliedValue = + keyContext !== '' ? keyContext : nameContext != null ? nameContext.type.name : ''; + const value = isObject(error.value) ? JSON.stringify(error.value) : error.value; + return `Invalid value "${value}" supplied to "${suppliedValue}"`; + } + }); + + return [...new Set(err)]; +}; diff --git a/x-pack/plugins/lists/common/schemas/request/delete_list_schema.mock.ts b/x-pack/plugins/lists/common/schemas/request/delete_list_schema.mock.ts index bc0fb7c479c50..6559dfbec2181 100644 --- a/x-pack/plugins/lists/common/schemas/request/delete_list_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/request/delete_list_schema.mock.ts @@ -9,5 +9,7 @@ import { LIST_ID } from '../../constants.mock'; import { DeleteListSchema } from './delete_list_schema'; export const getDeleteListSchemaMock = (): DeleteListSchema => ({ + deleteReferences: false, id: LIST_ID, + ignoreReferences: true, }); diff --git a/x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts index 630c77bf80dd2..285d5546a2b2b 100644 --- a/x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts +++ b/x-pack/plugins/lists/common/schemas/request/delete_list_schema.ts @@ -8,12 +8,21 @@ import * as t from 'io-ts'; import { id } from '../common/schemas'; import { RequiredKeepUndefined } from '../../types'; +import { DefaultStringBooleanFalse } from '../types/default_string_boolean_false'; -export const deleteListSchema = t.exact( - t.type({ - id, - }) -); +export const deleteListSchema = t.intersection([ + t.exact( + t.type({ + id, + }) + ), + t.exact( + t.partial({ + deleteReferences: DefaultStringBooleanFalse, + ignoreReferences: DefaultStringBooleanFalse, + }) + ), +]); export type DeleteListSchema = RequiredKeepUndefined>; export type DeleteListSchemaEncoded = t.OutputOf; diff --git a/x-pack/plugins/lists/common/schemas/types/default_string_boolean_false.test.ts b/x-pack/plugins/lists/common/schemas/types/default_string_boolean_false.test.ts new file mode 100644 index 0000000000000..310e401a8e8b4 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/types/default_string_boolean_false.test.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; + +import { foldLeftRight, getPaths } from '../../test_utils'; + +import { DefaultStringBooleanFalse } from './default_string_boolean_false'; + +describe('default_string_boolean_false', () => { + test('it should validate a boolean false', () => { + const payload = false; + const decoded = DefaultStringBooleanFalse.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should validate a boolean true', () => { + const payload = true; + const decoded = DefaultStringBooleanFalse.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should not validate a number', () => { + const payload = 5; + const decoded = DefaultStringBooleanFalse.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "5" supplied to "DefaultStringBooleanFalse"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should return a default false', () => { + const payload = null; + const decoded = DefaultStringBooleanFalse.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(false); + }); + + test('it should return a default false when given a string of "false"', () => { + const payload = 'false'; + const decoded = DefaultStringBooleanFalse.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(false); + }); + + test('it should return a default true when given a string of "true"', () => { + const payload = 'true'; + const decoded = DefaultStringBooleanFalse.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(true); + }); + + test('it should return a default true when given a string of "TruE"', () => { + const payload = 'TruE'; + const decoded = DefaultStringBooleanFalse.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(true); + }); + + test('it should not work with a string of junk "junk"', () => { + const payload = 'junk'; + const decoded = DefaultStringBooleanFalse.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "junk" supplied to "DefaultStringBooleanFalse"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should not work with an empty string', () => { + const payload = ''; + const decoded = DefaultStringBooleanFalse.decode(payload); + const message = pipe(decoded, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "" supplied to "DefaultStringBooleanFalse"', + ]); + expect(message.schema).toEqual({}); + }); +}); diff --git a/x-pack/plugins/lists/common/schemas/types/default_string_boolean_false.ts b/x-pack/plugins/lists/common/schemas/types/default_string_boolean_false.ts new file mode 100644 index 0000000000000..eba805048fe3a --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/types/default_string_boolean_false.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { Either } from 'fp-ts/lib/Either'; + +/** + * Types the DefaultStringBooleanFalse as: + * - If a string this will convert the string to a boolean + * - If null or undefined, then a default false will be set + */ +export const DefaultStringBooleanFalse = new t.Type( + 'DefaultStringBooleanFalse', + t.boolean.is, + (input, context): Either => { + if (input == null) { + return t.success(false); + } else if (typeof input === 'string' && input.toLowerCase() === 'true') { + return t.success(true); + } else if (typeof input === 'string' && input.toLowerCase() === 'false') { + return t.success(false); + } else { + return t.boolean.validate(input, context); + } + }, + t.identity +); + +export type DefaultStringBooleanFalseC = typeof DefaultStringBooleanFalse; diff --git a/x-pack/plugins/lists/common/test_utils.ts b/x-pack/plugins/lists/common/test_utils.ts new file mode 100644 index 0000000000000..4735bab94e6a8 --- /dev/null +++ b/x-pack/plugins/lists/common/test_utils.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { fold } from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/pipeable'; + +import { formatErrors } from './format_errors'; + +interface Message { + errors: t.Errors; + schema: T | {}; +} + +const onLeft = (errors: t.Errors): Message => { + return { errors, schema: {} }; +}; + +const onRight = (schema: T): Message => { + return { + errors: [], + schema, + }; +}; + +export const foldLeftRight = fold(onLeft, onRight); + +/** + * Convenience utility to keep the error message handling within tests to be + * very concise. + * @param validation The validation to get the errors from + */ +export const getPaths = (validation: t.Validation): string[] => { + return pipe( + validation, + fold( + (errors) => formatErrors(errors), + () => ['no errors'] + ) + ); +}; + +/** + * Convenience utility to remove text appended to links by EUI + */ +export const removeExternalLinkText = (str: string): string => + str.replace(/\(opens in a new tab or window\)/g, ''); diff --git a/x-pack/plugins/lists/public/lists/api.test.ts b/x-pack/plugins/lists/public/lists/api.test.ts index d79dc86802399..6e36dda9e1231 100644 --- a/x-pack/plugins/lists/public/lists/api.test.ts +++ b/x-pack/plugins/lists/public/lists/api.test.ts @@ -41,7 +41,11 @@ describe('Value Lists API', () => { it('DELETEs specifying the id as a query parameter', async () => { const abortCtrl = new AbortController(); - const payload: ApiPayload = { id: 'list-id' }; + const payload: ApiPayload = { + deleteReferences: false, + id: 'list-id', + ignoreReferences: true, + }; await deleteList({ http: httpMock, ...payload, @@ -52,7 +56,7 @@ describe('Value Lists API', () => { '/api/lists', expect.objectContaining({ method: 'DELETE', - query: { id: 'list-id' }, + query: { deleteReferences: false, id: 'list-id', ignoreReferences: true }, }) ); }); diff --git a/x-pack/plugins/lists/public/lists/api.ts b/x-pack/plugins/lists/public/lists/api.ts index 81410aaa06ece..0612e05edeb80 100644 --- a/x-pack/plugins/lists/public/lists/api.ts +++ b/x-pack/plugins/lists/public/lists/api.ts @@ -129,23 +129,27 @@ const importListWithValidation = async ({ export { importListWithValidation as importList }; const deleteList = async ({ + deleteReferences = false, http, id, + ignoreReferences = false, signal, }: ApiParams & DeleteListSchemaEncoded): Promise => http.fetch(LIST_URL, { method: 'DELETE', - query: { id }, + query: { deleteReferences, id, ignoreReferences }, signal, }); const deleteListWithValidation = async ({ + deleteReferences, http, id, + ignoreReferences, signal, }: DeleteListParams): Promise => pipe( - { id }, + { deleteReferences, id, ignoreReferences }, (payload) => fromEither(validateEither(deleteListSchema, payload)), chain((payload) => tryCatch(() => deleteList({ http, signal, ...payload }), toError)), chain((response) => fromEither(validateEither(listSchema, response))), diff --git a/x-pack/plugins/lists/public/lists/types.ts b/x-pack/plugins/lists/public/lists/types.ts index 95a21820536e4..e772143bfe5c8 100644 --- a/x-pack/plugins/lists/public/lists/types.ts +++ b/x-pack/plugins/lists/public/lists/types.ts @@ -26,7 +26,9 @@ export interface ImportListParams extends ApiParams { } export interface DeleteListParams extends ApiParams { + deleteReferences?: boolean; id: string; + ignoreReferences?: boolean; } export interface ExportListParams extends ApiParams { diff --git a/x-pack/plugins/lists/server/routes/delete_list_route.ts b/x-pack/plugins/lists/server/routes/delete_list_route.ts index f87645b79fc75..9562a83b7c31c 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_list_route.ts @@ -9,9 +9,18 @@ import { IRouter } from 'kibana/server'; import { LIST_URL } from '../../common/constants'; import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { validate } from '../../common/shared_imports'; -import { deleteListSchema, listSchema } from '../../common/schemas'; +import { + EntriesArray, + ExceptionListItemSchema, + FoundExceptionListSchema, + deleteListSchema, + exceptionListItemSchema, + listSchema, +} from '../../common/schemas'; +import { getSavedObjectType } from '../services/exception_lists/utils'; +import { ExceptionListClient } from '../services/exception_lists/exception_list_client'; -import { getListClient } from '.'; +import { getExceptionListClient, getListClient } from '.'; export const deleteListRoute = (router: IRouter): void => { router.delete( @@ -28,7 +37,68 @@ export const deleteListRoute = (router: IRouter): void => { const siemResponse = buildSiemResponse(response); try { const lists = getListClient(context); - const { id } = request.query; + const exceptionLists = getExceptionListClient(context); + const { id, deleteReferences, ignoreReferences } = request.query; + let deleteExceptionItemResponses; + + // ignoreReferences=true maintains pre-7.11 behavior of deleting value list without performing any additional checks + if (!ignoreReferences) { + const referencedExceptionListItems = await exceptionLists.findValueListExceptionListItems( + { + page: 1, + perPage: 10000, + sortField: undefined, + sortOrder: undefined, + valueListId: id, + } + ); + + if (referencedExceptionListItems?.data?.length) { + // deleteReferences=false to perform dry run and identify referenced exception lists/items + if (deleteReferences) { + // Delete referenced exception list items + // TODO: Create deleteListItems to delete in batch + deleteExceptionItemResponses = await Promise.all( + referencedExceptionListItems.data.map(async (listItem) => { + // Ensure only the single entry is deleted as there could be a separate value list referenced that is okay to keep // TODO: Add API to delete single entry + // @ts-ignore inline way of verifying entry type is EntryList? + const remainingEntries = listItem.entries.filter((e) => e?.list?.id !== id); + if (remainingEntries.length === 0) { + // All entries reference value list specified in request, delete entire exception list item + return deleteExceptionListItem(exceptionLists, listItem); + } else { + // Contains more entries than just value list specified in request , patch (doesn't exist yet :) exception list item to remove entry + return updateExceptionListItems(exceptionLists, listItem, remainingEntries); + } + }) + ); + } else { + const referencedExceptionLists = await getReferencedExceptionLists( + exceptionLists, + referencedExceptionListItems.data + ); + const refError = `Value list '${id}' is referenced in existing exception list(s)`; + const references = referencedExceptionListItems.data.map((item) => ({ + exception_item: item, + exception_list: referencedExceptionLists.data.find( + (l) => l.list_id === item.list_id + ), + })); + + return siemResponse.error({ + body: { + error: { + message: refError, + references, + value_list_id: id, + }, + }, + statusCode: 409, + }); + } + } + } + const deleted = await lists.deleteList({ id }); if (deleted == null) { return siemResponse.error({ @@ -40,7 +110,11 @@ export const deleteListRoute = (router: IRouter): void => { if (errors != null) { return siemResponse.error({ body: errors, statusCode: 500 }); } else { - return response.ok({ body: validated ?? {} }); + return response.ok({ + body: validated ?? { + deleteItemResponses: deleteExceptionItemResponses, + }, + }); } } } catch (err) { @@ -53,3 +127,98 @@ export const deleteListRoute = (router: IRouter): void => { } ); }; + +/** + * Fetches ExceptionLists for given ExceptionListItems + * @param exceptionLists ExceptionListClient + * @param exceptionListItems ExceptionListItemSchema[] + */ +const getReferencedExceptionLists = async ( + exceptionLists: ExceptionListClient, + exceptionListItems: ExceptionListItemSchema[] +): Promise => { + const filter = exceptionListItems + .map( + (item) => + `${getSavedObjectType({ + namespaceType: item.namespace_type, + })}.attributes.list_id: ${item.list_id}` + ) + .join(' OR '); + return exceptionLists.findExceptionList({ + filter: `(${filter})`, + page: 1, + perPage: 10000, + sortField: undefined, + sortOrder: undefined, + }); +}; + +/** + * Adapted from deleteExceptionListItemRoute + * @param exceptionLists ExceptionListClient + * @param listItem ExceptionListItemSchema + */ +const deleteExceptionListItem = async ( + exceptionLists: ExceptionListClient, + listItem: ExceptionListItemSchema +): Promise => { + const deletedExceptionListItem = await exceptionLists.deleteExceptionListItem({ + id: listItem.id, + itemId: listItem.item_id, + namespaceType: listItem.namespace_type, + }); + if (deletedExceptionListItem == null) { + return { + body: `list item with id: "${listItem.id}" not found`, + statusCode: 404, + }; + } else { + const [validated, errors] = validate(deletedExceptionListItem, exceptionListItemSchema); + if (errors != null) { + return { body: errors, statusCode: 500 }; + } else { + return { body: validated ?? {} }; + } + } +}; + +/** + * Adapted from updateExceptionListItemRoute + * @param exceptionLists ExceptionListClient + * @param listItem ExceptionListItemSchema + * @param remainingEntries EntriesArray + */ +const updateExceptionListItems = async ( + exceptionLists: ExceptionListClient, + listItem: ExceptionListItemSchema, + remainingEntries: EntriesArray +): Promise => { + const updateExceptionListItem = await exceptionLists.updateExceptionListItem({ + _version: listItem._version, + comments: listItem.comments, + description: listItem.description, + entries: remainingEntries, + id: listItem.id, + itemId: listItem.item_id, + meta: listItem.meta, + name: listItem.name, + namespaceType: listItem.namespace_type, + osTypes: listItem.os_types, + tags: listItem.tags, + type: listItem.type, + }); + if (updateExceptionListItem == null) { + return { + body: `exception list item id: "${listItem.item_id}" does not exist`, + statusCode: 404, + }; + } else { + const [validated, errors] = validate(updateExceptionListItem, exceptionListItemSchema); + if (errors != null) { + return { body: errors, statusCode: 500 }; + } else { + return { body: validated ?? {} }; + } + } +}; diff --git a/x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item_with_list.json b/x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item_with_list.json index d0756b990aad0..23c8b626d9945 100644 --- a/x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item_with_list.json +++ b/x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item_with_list.json @@ -1,19 +1,13 @@ { - "list_id": "endpoint_list", - "item_id": "endpoint_list_item_lg_val_list", + "list_id": "simple_list", + "item_id": "simple_list_item_lg_val_list", "tags": ["user added string for a tag", "malware"], "type": "simple", "description": "This is a sample exception list item with a large value list included", - "name": "Sample Endpoint Exception List Item with large value list", + "name": "Sample Simple List Item with large value list", "os_types": ["windows"], "comments": [], "entries": [ - { - "field": "event.module", - "operator": "excluded", - "type": "match_any", - "value": ["suricata"] - }, { "field": "source.ip", "operator": "excluded", diff --git a/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_1.json b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_1.json new file mode 100644 index 0000000000000..bb1ecce66d0a2 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_1.json @@ -0,0 +1,7 @@ +{ + "list_id": "detection_list_1", + "tags": ["user added string for a tag", "malware"], + "type": "detection", + "description": "This is a sample detection type exception list", + "name": "Detection Exception List (1)" +} diff --git a/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_2_agnostic.json b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_2_agnostic.json new file mode 100644 index 0000000000000..4bd299eac1fc8 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_2_agnostic.json @@ -0,0 +1,8 @@ +{ + "list_id": "detection_list_2", + "tags": ["user added string for a tag", "malware"], + "type": "detection", + "description": "This is a sample agnostic detection type exception list", + "name": "Detection Exception List (2)", + "namespace_type": "agnostic" +} diff --git a/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_3.json b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_3.json new file mode 100644 index 0000000000000..da7f7c37f8cdf --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_detection_3.json @@ -0,0 +1,7 @@ +{ + "list_id": "detection_list_3", + "tags": ["user added string for a tag", "malware"], + "type": "detection", + "description": "This is a sample detection type exception list", + "name": "Detection Exception List (3)" +} diff --git a/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_1_multi_value_list.json b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_1_multi_value_list.json new file mode 100644 index 0000000000000..3927e36a29675 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_1_multi_value_list.json @@ -0,0 +1,35 @@ +{ + "list_id": "detection_list_1", + "item_id": "simple_list_item_two_value_lists", + "tags": [ + "user added string for a tag", + "malware" + ], + "type": "simple", + "description": "This is a sample exception list item with a large value list included", + "name": "Simple List Item with ip value list", + "os_types": [ + "windows" + ], + "comments": [], + "entries": [ + { + "field": "source.ip", + "operator": "excluded", + "type": "list", + "list": { + "id": "ips.txt", + "type": "ip" + } + }, + { + "field": "source.ip", + "operator": "excluded", + "type": "list", + "list": { + "id": "ip_range.txt", + "type": "ip_range" + } + } + ] +} diff --git a/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_1_non_value_list.json b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_1_non_value_list.json new file mode 100644 index 0000000000000..9fe2a05e3787e --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_1_non_value_list.json @@ -0,0 +1,28 @@ +{ + "list_id": "detection_list_1", + "item_id": "simple_list_item_two_non-value_list", + "tags": [ + "user added string for a tag", + "malware" + ], + "type": "simple", + "description": "This is a sample exception list item with two non-value list entries", + "name": "Sample Detection Exception List Item", + "os_types": [ + "windows" + ], + "comments": [], + "entries": [ + { + "field": "actingProcess.file.signer", + "operator": "excluded", + "type": "exists" + }, + { + "field": "host.name", + "operator": "included", + "type": "match_any", + "value": ["some host", "another host"] + } + ] +} diff --git a/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_2_multi_value_list.json b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_2_multi_value_list.json new file mode 100644 index 0000000000000..9fcc2be6af332 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_2_multi_value_list.json @@ -0,0 +1,36 @@ +{ + "list_id": "detection_list_2", + "item_id": "simple_list_item_two_value_lists", + "tags": [ + "user added string for a tag", + "malware" + ], + "type": "simple", + "description": "This is a sample exception list item with a large value list included", + "name": "Simple List Item with ip value list", + "namespace_type": "agnostic", + "os_types": [ + "windows" + ], + "comments": [], + "entries": [ + { + "field": "source.ip", + "operator": "excluded", + "type": "list", + "list": { + "id": "ips.txt", + "type": "ip" + } + }, + { + "field": "source.ip", + "operator": "excluded", + "type": "list", + "list": { + "id": "ip_range.txt", + "type": "ip_range" + } + } + ] +} diff --git a/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_3_single_value_list.json b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_3_single_value_list.json new file mode 100644 index 0000000000000..8235e9275095a --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/exception_lists/new/references/exception_list_item_3_single_value_list.json @@ -0,0 +1,26 @@ +{ + "list_id": "detection_list_3", + "item_id": "simple_list_item_one_value_list", + "tags": [ + "user added string for a tag", + "malware" + ], + "type": "simple", + "description": "This is a sample exception list item with a large value list included", + "name": "Simple List Item with ip value list", + "os_types": [ + "windows" + ], + "comments": [], + "entries": [ + { + "field": "source.ip", + "operator": "excluded", + "type": "list", + "list": { + "id": "text.txt", + "type": "keyword" + } + } + ] +} diff --git a/x-pack/plugins/lists/server/scripts/patch_list.sh b/x-pack/plugins/lists/server/scripts/patch_list.sh index 3a517a52dbd21..d78bfc29f5fcf 100755 --- a/x-pack/plugins/lists/server/scripts/patch_list.sh +++ b/x-pack/plugins/lists/server/scripts/patch_list.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/patch_list_item.sh b/x-pack/plugins/lists/server/scripts/patch_list_item.sh index 82470f0aba533..c837b8494a5d7 100755 --- a/x-pack/plugins/lists/server/scripts/patch_list_item.sh +++ b/x-pack/plugins/lists/server/scripts/patch_list_item.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/post_endpoint_list.sh b/x-pack/plugins/lists/server/scripts/post_endpoint_list.sh index e0b179f443547..86e99b13ad2f0 100755 --- a/x-pack/plugins/lists/server/scripts/post_endpoint_list.sh +++ b/x-pack/plugins/lists/server/scripts/post_endpoint_list.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/post_endpoint_list_item.sh b/x-pack/plugins/lists/server/scripts/post_endpoint_list_item.sh index 8235a2ec06eb7..f0d31251b5be9 100755 --- a/x-pack/plugins/lists/server/scripts/post_endpoint_list_item.sh +++ b/x-pack/plugins/lists/server/scripts/post_endpoint_list_item.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/post_exception_list.sh b/x-pack/plugins/lists/server/scripts/post_exception_list.sh index 84a775ffcf7f1..40465f0464891 100755 --- a/x-pack/plugins/lists/server/scripts/post_exception_list.sh +++ b/x-pack/plugins/lists/server/scripts/post_exception_list.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/post_exception_list_item.sh b/x-pack/plugins/lists/server/scripts/post_exception_list_item.sh index 6cee54b1a6148..073240d47dfe0 100755 --- a/x-pack/plugins/lists/server/scripts/post_exception_list_item.sh +++ b/x-pack/plugins/lists/server/scripts/post_exception_list_item.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/post_list.sh b/x-pack/plugins/lists/server/scripts/post_list.sh index e0e442164535c..5d4612557412d 100755 --- a/x-pack/plugins/lists/server/scripts/post_list.sh +++ b/x-pack/plugins/lists/server/scripts/post_list.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/post_list_item.sh b/x-pack/plugins/lists/server/scripts/post_list_item.sh index 49f9759c7879c..530abc4ddf1c5 100755 --- a/x-pack/plugins/lists/server/scripts/post_list_item.sh +++ b/x-pack/plugins/lists/server/scripts/post_list_item.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/post_x_exception_list_items.sh b/x-pack/plugins/lists/server/scripts/post_x_exception_list_items.sh index 8e29e96f884a5..88d56b78ff581 100755 --- a/x-pack/plugins/lists/server/scripts/post_x_exception_list_items.sh +++ b/x-pack/plugins/lists/server/scripts/post_x_exception_list_items.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/quick_start_value_list_references.sh b/x-pack/plugins/lists/server/scripts/quick_start_value_list_references.sh new file mode 100755 index 0000000000000..0731bfce5c89a --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/quick_start_value_list_references.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# +# Creates three different exception lists and value lists, and associates as +# below to test referential integrity functionality. +# +# NOTE: Endpoint lists don't support value lists, and are not tested here +# +# EL: Exception list +# ELI Exception list Item +# VL: Value list +# +# +# EL1 EL2 (Agnostic) EL3 +# | | | +# ELI1 ELI2 ELI3 +# |\ /| | +# | \ / | | +# | \ / | | +# | \ / | | +# | \/ | | +# | /\ | | +# | / \ | | +# | / \ | | +# | / \ | | +# |/ \| | +# VL1 VL2 VL3 VL4 +# ips.txt ip_range.txt text.txt hosts.txt + +./hard_reset.sh && \ +# Create value lists +./import_list_items_by_filename.sh ip ./lists/files/ips.txt && \ +./import_list_items_by_filename.sh ip_range ./lists/files/ip_range.txt && \ +./import_list_items_by_filename.sh keyword ./lists/files/text.txt && \ +./import_list_items_by_filename.sh keyword ./lists/files/hosts.txt && \ +# Create exception lists 1, 2 (agnostic), 3 +./post_exception_list.sh ./exception_lists/new/references/exception_list_detection_1.json && \ +./post_exception_list.sh ./exception_lists/new/references/exception_list_detection_2_agnostic.json && \ +./post_exception_list.sh ./exception_lists/new/references/exception_list_detection_3.json && \ +# Create exception list items with value lists +./post_exception_list_item.sh ./exception_lists/new/references/exception_list_item_1_multi_value_list.json && \ +./post_exception_list_item.sh ./exception_lists/new/references/exception_list_item_2_multi_value_list.json && \ +./post_exception_list_item.sh ./exception_lists/new/references/exception_list_item_3_single_value_list.json && \ +# Create exception list items (non value lists, to ensure they're not deleted on cleanup) +./post_exception_list_item.sh ./exception_lists/new/references/exception_list_item_1_non_value_list.json diff --git a/x-pack/plugins/lists/server/scripts/update_endpoint_item.sh b/x-pack/plugins/lists/server/scripts/update_endpoint_item.sh index 4a6ca3881a323..5ef237e4402cb 100755 --- a/x-pack/plugins/lists/server/scripts/update_endpoint_item.sh +++ b/x-pack/plugins/lists/server/scripts/update_endpoint_item.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/update_exception_list.sh b/x-pack/plugins/lists/server/scripts/update_exception_list.sh index d7523a0804a89..4d3372bde85b5 100755 --- a/x-pack/plugins/lists/server/scripts/update_exception_list.sh +++ b/x-pack/plugins/lists/server/scripts/update_exception_list.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/update_exception_list_item.sh b/x-pack/plugins/lists/server/scripts/update_exception_list_item.sh index 029bfcdabee3e..28869b3f3ca0b 100755 --- a/x-pack/plugins/lists/server/scripts/update_exception_list_item.sh +++ b/x-pack/plugins/lists/server/scripts/update_exception_list_item.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/update_list.sh b/x-pack/plugins/lists/server/scripts/update_list.sh index 4d93544d568a8..73df67112f626 100755 --- a/x-pack/plugins/lists/server/scripts/update_list.sh +++ b/x-pack/plugins/lists/server/scripts/update_list.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/scripts/update_list_item.sh b/x-pack/plugins/lists/server/scripts/update_list_item.sh index e9915f905b971..24b3d67f2c891 100755 --- a/x-pack/plugins/lists/server/scripts/update_list_item.sh +++ b/x-pack/plugins/lists/server/scripts/update_list_item.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts index 9747c58d1cd0f..88e0914cf345e 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts @@ -27,6 +27,7 @@ import { FindExceptionListItemOptions, FindExceptionListOptions, FindExceptionListsItemOptions, + FindValueListExceptionListsItems, GetEndpointListItemOptions, GetExceptionListItemOptions, GetExceptionListOptions, @@ -44,7 +45,10 @@ import { deleteExceptionList } from './delete_exception_list'; import { deleteExceptionListItem, deleteExceptionListItemById } from './delete_exception_list_item'; import { findExceptionListItem } from './find_exception_list_item'; import { findExceptionList } from './find_exception_list'; -import { findExceptionListsItem } from './find_exception_list_items'; +import { + findExceptionListsItem, + findValueListExceptionListItems, +} from './find_exception_list_items'; import { createEndpointList } from './create_endpoint_list'; import { createEndpointTrustedAppsList } from './create_endpoint_trusted_apps_list'; @@ -139,7 +143,7 @@ export class ExceptionListClient { }; /** - * This is the same as "updateListItem" except it applies specifically to the endpoint list and will + * This is the same as "updateExceptionListItem" except it applies specifically to the endpoint list and will * auto-call the "createEndpointList" for you so that you have the best chance of the endpoint * being there if it did not exist before. If the list did not exist before, then creating it here will still cause a * return of null but at least the list exists again. @@ -410,6 +414,24 @@ export class ExceptionListClient { }); }; + public findValueListExceptionListItems = async ({ + perPage, + page, + sortField, + sortOrder, + valueListId, + }: FindValueListExceptionListsItems): Promise => { + const { savedObjectsClient } = this; + return findValueListExceptionListItems({ + page, + perPage, + savedObjectsClient, + sortField, + sortOrder, + valueListId, + }); + }; + public findExceptionList = async ({ filter, perPage, diff --git a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts index 1fef2da5d975e..74dc1e215f479 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts @@ -196,8 +196,16 @@ export interface FindExceptionListsItemOptions { sortOrder: SortOrderOrUndefined; } +export interface FindValueListExceptionListsItems { + valueListId: Id; + perPage: PerPageOrUndefined; + page: PageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; +} + export interface FindExceptionListOptions { - namespaceType: NamespaceType; + namespaceType?: NamespaceType; filter: FilterOrUndefined; perPage: PerPageOrUndefined; page: PageOrUndefined; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts index 84cc7ba2f1021..a30e843d853a5 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts @@ -16,12 +16,16 @@ import { SortFieldOrUndefined, SortOrderOrUndefined, } from '../../../common/schemas'; -import { SavedObjectType } from '../../saved_objects'; +import { + SavedObjectType, + exceptionListAgnosticSavedObjectType, + exceptionListSavedObjectType, +} from '../../saved_objects'; import { getSavedObjectType, transformSavedObjectsToFoundExceptionList } from './utils'; interface FindExceptionListOptions { - namespaceType: NamespaceType; + namespaceType?: NamespaceType; savedObjectsClient: SavedObjectsClientContract; filter: FilterOrUndefined; perPage: PerPageOrUndefined; @@ -39,7 +43,9 @@ export const findExceptionList = async ({ sortField, sortOrder, }: FindExceptionListOptions): Promise => { - const savedObjectType = getSavedObjectType({ namespaceType }); + const savedObjectType: SavedObjectType[] = namespaceType + ? [getSavedObjectType({ namespaceType })] + : [exceptionListSavedObjectType, exceptionListAgnosticSavedObjectType]; const savedObjectsFindResponse = await savedObjectsClient.find({ filter: getExceptionListFilter({ filter, savedObjectType }), page, @@ -56,11 +62,18 @@ export const getExceptionListFilter = ({ savedObjectType, }: { filter: FilterOrUndefined; - savedObjectType: SavedObjectType; + savedObjectType: SavedObjectType[]; }): string => { + const savedObjectTypeFilter = `(${savedObjectType + .map((sot) => `${sot}.attributes.list_type: list`) + .join(' OR ')})`; if (filter == null) { - return `${savedObjectType}.attributes.list_type: list`; + return savedObjectTypeFilter; } else { - return `${savedObjectType}.attributes.list_type: list AND ${filter}`; + if (Array.isArray(savedObjectType)) { + return `${savedObjectTypeFilter} AND ${filter}`; + } else { + return `${savedObjectType}.attributes.list_type: list AND ${filter}`; + } } }; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts index 47a0d809cce67..5a21d7a12b020 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts @@ -11,12 +11,17 @@ import { NonEmptyStringArrayDecoded } from '../../../common/schemas/types/non_em import { ExceptionListSoSchema, FoundExceptionListItemSchema, + Id, PageOrUndefined, PerPageOrUndefined, SortFieldOrUndefined, SortOrderOrUndefined, } from '../../../common/schemas'; -import { SavedObjectType } from '../../saved_objects'; +import { + SavedObjectType, + exceptionListAgnosticSavedObjectType, + exceptionListSavedObjectType, +} from '../../saved_objects'; import { getSavedObjectTypes, transformSavedObjectsToFoundExceptionListItem } from './utils'; import { getExceptionList } from './get_exception_list'; @@ -92,3 +97,33 @@ export const getExceptionListsItemFilter = ({ } }, ''); }; + +interface FindValueListExceptionListsItems { + valueListId: Id; + savedObjectsClient: SavedObjectsClientContract; + perPage: PerPageOrUndefined; + page: PageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; +} + +export const findValueListExceptionListItems = async ({ + valueListId, + savedObjectsClient, + page, + perPage, + sortField, + sortOrder, +}: FindValueListExceptionListsItems): Promise => { + const savedObjectsFindResponse = await savedObjectsClient.find({ + filter: `(exception-list.attributes.list_type: item AND exception-list.attributes.entries.list.id:${valueListId}) OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.entries.list.id:${valueListId}) `, + page, + perPage, + sortField, + sortOrder, + type: [exceptionListSavedObjectType, exceptionListAgnosticSavedObjectType], + }); + return transformSavedObjectsToFoundExceptionListItem({ + savedObjectsFindResponse, + }); +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_string_boolean_false.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_string_boolean_false.test.ts index dcb264d77b14b..a8c1bb98b22fd 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_string_boolean_false.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_string_boolean_false.test.ts @@ -75,7 +75,7 @@ describe('default_string_boolean_false', () => { expect(message.schema).toEqual(true); }); - test('it should not work with a strong of junk "junk"', () => { + test('it should not work with a string of junk "junk"', () => { const payload = 'junk'; const decoded = DefaultStringBooleanFalse.decode(payload); const message = pipe(decoded, foldLeftRight); diff --git a/x-pack/plugins/security_solution/public/detections/components/value_lists_management_modal/modal.tsx b/x-pack/plugins/security_solution/public/detections/components/value_lists_management_modal/modal.tsx index 4921a98b38bd1..f0e47fcd5c104 100644 --- a/x-pack/plugins/security_solution/public/detections/components/value_lists_management_modal/modal.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/value_lists_management_modal/modal.tsx @@ -5,6 +5,7 @@ */ import React, { useCallback, useEffect, useState } from 'react'; +import { isEmpty } from 'lodash/fp'; import { EuiBasicTable, EuiButton, @@ -32,12 +33,27 @@ import * as i18n from './translations'; import { buildColumns } from './table_helpers'; import { ValueListsForm } from './form'; import { AutoDownload } from './auto_download'; +import { ReferenceErrorModal } from './reference_error_modal'; interface ValueListsModalProps { onClose: () => void; showModal: boolean; } +interface ReferenceModalState { + contentText: string; + exceptionListReferences: string[]; + isLoading: boolean; + valueListId: string; +} + +const referenceModalInitialState: ReferenceModalState = { + contentText: '', + exceptionListReferences: [], + isLoading: false, + valueListId: '', +}; + export const ValueListsModalComponent: React.FC = ({ onClose, showModal, @@ -47,24 +63,42 @@ export const ValueListsModalComponent: React.FC = ({ const [cursor, setCursor] = useCursor({ pageIndex, pageSize }); const { http } = useKibana().services; const { start: findLists, ...lists } = useFindLists(); - const { start: deleteList, result: deleteResult } = useDeleteList(); + const { start: deleteList, result: deleteResult, error: deleteError } = useDeleteList(); const [deletingListIds, setDeletingListIds] = useState([]); const [exportingListIds, setExportingListIds] = useState([]); const [exportDownload, setExportDownload] = useState<{ name?: string; blob?: Blob }>({}); const { addError, addSuccess } = useAppToasts(); + const [showReferenceErrorModal, setShowReferenceErrorModal] = useState(false); + const [referenceModalState, setReferenceModalState] = useState( + referenceModalInitialState + ); const fetchLists = useCallback(() => { findLists({ cursor, http, pageIndex: pageIndex + 1, pageSize }); }, [cursor, http, findLists, pageIndex, pageSize]); const handleDelete = useCallback( - ({ id }: { id: string }) => { + ({ + deleteReferences, + id, + }: { + deleteReferences?: boolean; + id: string; + ignoreReferences?: boolean; + }) => { setDeletingListIds([...deletingListIds, id]); - deleteList({ http, id }); + deleteList({ deleteReferences, http, id }); }, [deleteList, deletingListIds, http] ); + const handleReferenceDelete = useCallback(async () => { + setShowReferenceErrorModal(false); + deleteList({ deleteReferences: true, http, id: referenceModalState.valueListId }); + setReferenceModalState(referenceModalInitialState); + setDeletingListIds([]); + }, [deleteList, http, referenceModalState.valueListId]); + useEffect(() => { if (deleteResult != null) { setDeletingListIds((ids) => [...ids.filter((id) => id !== deleteResult.id)]); @@ -72,6 +106,26 @@ export const ValueListsModalComponent: React.FC = ({ } }, [deleteResult, fetchLists]); + useEffect(() => { + if (!isEmpty(deleteError)) { + const references: string[] = + // @ts-ignore-next-line deleteError response unknown message.error.references + deleteError?.body?.message?.error?.references?.map( + // @ts-ignore-next-line response not typed + (ref) => ref?.exception_list.name + ) ?? []; + const uniqueExceptionListReferences = Array.from(new Set(references)); + setShowReferenceErrorModal(true); + setReferenceModalState({ + contentText: i18n.referenceErrorMessage(uniqueExceptionListReferences.length), + exceptionListReferences: uniqueExceptionListReferences, + isLoading: false, + // @ts-ignore-next-line deleteError response unknown + valueListId: deleteError?.body?.message?.error?.value_list_id, + }); + } + }, [deleteError]); + const handleExport = useCallback( async ({ id }: { id: string }) => { try { @@ -126,6 +180,17 @@ export const ValueListsModalComponent: React.FC = ({ } }, [lists.loading, lists.result, setCursor]); + const handleCloseReferenceErrorModal = useCallback(() => { + setDeletingListIds([]); + setShowReferenceErrorModal(false); + setReferenceModalState({ + contentText: '', + exceptionListReferences: [], + isLoading: false, + valueListId: '', + }); + }, []); + if (!showModal) { return null; } @@ -173,6 +238,17 @@ export const ValueListsModalComponent: React.FC = ({ + theme.eui.euiScrollBar}; + width: ${({ theme }) => theme.eui.euiScrollBar}; + } + + &::-webkit-scrollbar-thumb { + background-clip: content-box; + background-color: ${({ theme }) => rgba(theme.eui.euiColorDarkShade, 0.5)}; + border: ${({ theme }) => theme.eui.euiScrollBarCorner} solid transparent; + } + + &::-webkit-scrollbar-corner, + &::-webkit-scrollbar-track { + background-color: transparent; + } +`; + +interface ReferenceErrorModalProps { + cancelText: string; + confirmText: string; + contentText: string; + onCancel: () => void; + onClose: () => void; + onConfirm: () => void; + references: string[]; + showModal: boolean; + titleText: string; +} + +export const ReferenceErrorModalComponent: React.FC = ({ + cancelText, + confirmText, + contentText, + onClose, + onCancel, + onConfirm, + references = [], + showModal, + titleText, +}) => { + if (!showModal) { + return null; + } + + return ( + + +

{contentText}

+ + + {references.map((r, index) => ( + + ))} + + + + + ); +}; + +ReferenceErrorModalComponent.displayName = 'ReferenceErrorModalComponent'; + +export const ReferenceErrorModal = React.memo(ReferenceErrorModalComponent); + +ReferenceErrorModal.displayName = 'ReferenceErrorModal'; diff --git a/x-pack/plugins/security_solution/public/detections/components/value_lists_management_modal/translations.ts b/x-pack/plugins/security_solution/public/detections/components/value_lists_management_modal/translations.ts index 992c696121485..2dfdc8e45d5ce 100644 --- a/x-pack/plugins/security_solution/public/detections/components/value_lists_management_modal/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/value_lists_management_modal/translations.ts @@ -167,3 +167,31 @@ export const TEXT_RADIO = i18n.translate( defaultMessage: 'Text', } ); + +export const REFERENCE_MODAL_TITLE = i18n.translate( + 'xpack.securitySolution.lists.referenceModalTitle', + { + defaultMessage: 'Remove value list', + } +); + +export const REFERENCE_MODAL_CANCEL_BUTTON = i18n.translate( + 'xpack.securitySolution.lists.referenceModalCancelButton', + { + defaultMessage: 'Cancel', + } +); + +export const REFERENCE_MODAL_CONFIRM_BUTTON = i18n.translate( + 'xpack.securitySolution.lists.referenceModalDeleteButton', + { + defaultMessage: 'Remove value list', + } +); + +export const referenceErrorMessage = (referenceCount: number) => + i18n.translate('xpack.securitySolution.lists.referenceModalDescription', { + defaultMessage: + 'This value list is associated with ({referenceCount}) exception {referenceCount, plural, =1 {list} other {lists}}. Removing this list will remove all exception items that reference this value list.', + values: { referenceCount }, + }); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts index 3703e1b6ca306..28913da325b98 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts @@ -7,19 +7,32 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { LIST_URL } from '../../../../plugins/lists/common/constants'; +import { + EXCEPTION_LIST_ITEM_URL, + EXCEPTION_LIST_URL, + LIST_ITEM_URL, + LIST_URL, +} from '../../../../plugins/lists/common/constants'; -import { getCreateMinimalListSchemaMock } from '../../../../plugins/lists/common/schemas/request/create_list_schema.mock'; +import { + getCreateMinimalListSchemaMock, + getCreateMinimalListSchemaMockWithoutId, +} from '../../../../plugins/lists/common/schemas/request/create_list_schema.mock'; import { createListsIndex, + deleteAllExceptions, deleteListsIndex, removeListServerGeneratedProperties, } from '../../utils'; import { getListResponseMockWithoutAutoGeneratedValues } from '../../../../plugins/lists/common/schemas/response/list_schema.mock'; +import { getCreateExceptionListMinimalSchemaMock } from '../../../../plugins/lists/common/schemas/request/create_exception_list_schema.mock'; +import { getCreateExceptionListItemMinimalSchemaMockWithoutId } from '../../../../plugins/lists/common/schemas/request/create_exception_list_item_schema.mock'; +import { DETECTION_TYPE, LIST_ID } from '../../../../plugins/lists/common/constants.mock'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); + const es = getService('es'); describe('delete_lists', () => { describe('deleting lists', () => { @@ -54,7 +67,7 @@ export default ({ getService }: FtrProviderContext) => { const { body: bodyWithCreatedList } = await supertest .post(LIST_URL) .set('kbn-xsrf', 'true') - .send(getCreateMinimalListSchemaMock()) + .send(getCreateMinimalListSchemaMockWithoutId()) .expect(200); // delete that list by its auto-generated id @@ -78,6 +91,148 @@ export default ({ getService }: FtrProviderContext) => { status_code: 404, }); }); + + describe('deleting lists referenced in exceptions', () => { + afterEach(async () => { + await deleteAllExceptions(es); + }); + + it('should return an error when deleting a list referenced within an exception list item', async () => { + // create a list + const { body: valueListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create an exception list + const { body: exceptionListBody } = await supertest + .post(EXCEPTION_LIST_URL) + .set('kbn-xsrf', 'true') + .send({ ...getCreateExceptionListMinimalSchemaMock(), type: DETECTION_TYPE }) + .expect(200); + + // create an exception list item referencing value list + await supertest + .post(EXCEPTION_LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send({ + ...getCreateExceptionListItemMinimalSchemaMockWithoutId(), + list_id: exceptionListBody.list_id, + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'list', + list: { id: valueListBody.id, type: 'keyword' }, + }, + ], + }) + .expect(200); + + // try to delete that list by its auto-generated id + await supertest + .delete(`${LIST_URL}?id=${valueListBody.id}`) + .set('kbn-xsrf', 'true') + .expect(409); + + // really delete that list by its auto-generated id + await supertest + .delete(`${LIST_URL}?id=${valueListBody.id}&ignoreReferences=true`) + .set('kbn-xsrf', 'true') + .expect(200); + }); + + // Tests in development + it.skip('should delete a single list referenced within an exception list item if ignoreReferences=true', async () => { + // create a list + const { body: valueListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create an exception list + const { body: exceptionListBody } = await supertest + .post(EXCEPTION_LIST_URL) + .set('kbn-xsrf', 'true') + .send({ ...getCreateExceptionListMinimalSchemaMock(), type: DETECTION_TYPE }) + .expect(200); + + // create an exception list item referencing value list + await supertest + .post(EXCEPTION_LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send({ + ...getCreateExceptionListItemMinimalSchemaMockWithoutId(), + list_id: exceptionListBody.list_id, + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'list', + list: { id: valueListBody.id, type: 'keyword' }, + }, + ], + }) + .expect(200); + + // delete that list by its auto-generated id and ignoreReferences + supertest + .delete(`${LIST_URL}?id=${valueListBody.id}&ignoreReferences=true`) + .set('kbn-xsrf', 'true') + .expect(409); + }); + + // Tests in development + it.skip('should delete a single list referenced within an exception list item and referenced exception list items if deleteReferences=true', async () => { + // create a list + const { body: valueListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create an exception list + const { body: exceptionListBody } = await supertest + .post(EXCEPTION_LIST_URL) + .set('kbn-xsrf', 'true') + .send({ ...getCreateExceptionListMinimalSchemaMock(), type: DETECTION_TYPE }) + .expect(200); + + // create an exception list item referencing value list + await supertest + .post(EXCEPTION_LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send({ + ...getCreateExceptionListItemMinimalSchemaMockWithoutId(), + list_id: exceptionListBody.list_id, + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'list', + list: { id: valueListBody.id, type: 'keyword' }, + }, + ], + }) + .expect(200); + + // delete that list by its auto-generated id and delete referenced list items + const deleteListBody = await supertest + .delete(`${LIST_URL}?id=${valueListBody.id}&ignoreReferences=true`) + .set('kbn-xsrf', 'true'); + + const bodyToCompare = removeListServerGeneratedProperties(deleteListBody.body); + expect(bodyToCompare).to.eql(getListResponseMockWithoutAutoGeneratedValues()); + + await supertest + .get(`${LIST_ITEM_URL}/_find?list_id=${LIST_ID}`) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + }); + }); }); }); };