From 1a0a6ed98e3c95bcf91d74cef082b228cca0210d Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 26 Jan 2021 16:37:35 -0500 Subject: [PATCH 1/3] First pass at switching rules to depend on fields instead of _source --- .../signals/__mocks__/es_results.ts | 11 +++++ .../signals/build_events_query.test.ts | 41 ++++++++++++++++--- .../signals/build_events_query.ts | 6 +++ .../bulk_create_threshold_signals.test.ts | 2 +- .../signals/bulk_create_threshold_signals.ts | 6 +-- .../create_set_to_filter_against.test.ts | 8 ++-- .../filters/create_set_to_filter_against.ts | 2 +- .../signals/filters/filter_events.test.ts | 10 ++--- .../signals/filters/filter_events.ts | 4 +- .../signals/find_threshold_signals.ts | 6 +++ .../build_threat_mapping_filter.mock.ts | 11 +++++ .../build_threat_mapping_filter.test.ts | 18 ++++++-- .../build_threat_mapping_filter.ts | 9 ++-- .../signals/threat_mapping/get_threat_list.ts | 6 +++ .../detection_engine/signals/utils.test.ts | 25 +++++++---- .../server/lib/machine_learning/index.ts | 6 +++ 16 files changed, 135 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 6718ff2d1f15f..8be31aeacf3c3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -169,6 +169,12 @@ export const sampleDocWithSortId = ( ip: destIp ?? '127.0.0.1', }, }, + fields: { + someKey: ['someValue'], + '@timestamp': ['2020-04-20T21:27:45+0000'], + 'source.ip': [ip ?? '127.0.0.1'], + 'destination.ip': [destIp ?? '127.0.0.1'], + }, sort: ['1234567891111'], }); @@ -188,6 +194,11 @@ export const sampleDocNoSortId = ( ip: ip ?? '127.0.0.1', }, }, + fields: { + someKey: ['someValue'], + '@timestamp': ['2020-04-20T21:27:45+0000'], + 'source.ip': [ip ?? '127.0.0.1'], + }, sort: [], }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts index b6793af22f1b9..2dec059a67ac4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts @@ -55,7 +55,12 @@ describe('create_signals', () => { ], }, }, - + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [ { '@timestamp': { @@ -114,7 +119,12 @@ describe('create_signals', () => { ], }, }, - + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [ { '@timestamp': { @@ -174,7 +184,12 @@ describe('create_signals', () => { ], }, }, - + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [ { '@timestamp': { @@ -235,7 +250,12 @@ describe('create_signals', () => { ], }, }, - + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [ { '@timestamp': { @@ -295,7 +315,12 @@ describe('create_signals', () => { ], }, }, - + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [ { '@timestamp': { @@ -357,6 +382,12 @@ describe('create_signals', () => { ], }, }, + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], aggregations: { tags: { terms: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 5957b4b671bd9..0e9e0b2e17b96 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -88,6 +88,12 @@ export const buildEventsSearchQuery = ({ ], }, }, + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], ...(aggregations ? { aggregations } : {}), sort: [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts index 022c07defc9c1..71bd37b73a08e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -66,7 +66,7 @@ describe('transformThresholdResultsToEcs', () => { _id, _index: 'test', _source: { - '@timestamp': '2020-04-20T21:27:45+0000', + '@timestamp': ['2020-04-20T21:27:45+0000'], threshold_result: { count: 1, value: '127.0.0.1', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 438f08656a90f..da99aeaec7895 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -74,7 +74,7 @@ const getTransformedHits = ( } const source = { - '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), + '@timestamp': get(timestampOverride ?? '@timestamp', hit.fields), threshold_result: { count: totalResults, value: ruleId, @@ -103,10 +103,10 @@ const getTransformedHits = ( } const source = { - '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), + '@timestamp': get(timestampOverride ?? '@timestamp', hit.fields), threshold_result: { count: docCount, - value: get(threshold.field, hit._source), + value: key, }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts index 0ac09713cc8a6..5b543cfe97833 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts @@ -61,9 +61,9 @@ describe('createSetToFilterAgainst', () => { expect(listClient.searchListItemByValues).toHaveBeenCalledWith({ listId: 'list-123', type: 'ip', - value: ['1.1.1.1'], + value: [['1.1.1.1']], }); - expect([...field]).toEqual([JSON.stringify('1.1.1.1')]); + expect([...field]).toEqual([JSON.stringify(['1.1.1.1'])]); }); test('it returns 2 fields if the list returns 2 items', async () => { @@ -80,9 +80,9 @@ describe('createSetToFilterAgainst', () => { expect(listClient.searchListItemByValues).toHaveBeenCalledWith({ listId: 'list-123', type: 'ip', - value: ['1.1.1.1', '2.2.2.2'], + value: [['1.1.1.1'], ['2.2.2.2']], }); - expect([...field]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]); + expect([...field]).toEqual([JSON.stringify(['1.1.1.1']), JSON.stringify(['2.2.2.2'])]); }); test('it returns 0 fields if the field does not match up to a valid field within the event', async () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts index c546654676c83..df07bf5bc3322 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts @@ -30,7 +30,7 @@ export const createSetToFilterAgainst = async ({ buildRuleMessage, }: CreateSetToFilterAgainstOptions): Promise> => { const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => { - const valueField = get(field, searchResultItem._source); + const valueField = searchResultItem.fields ? searchResultItem.fields[field] : undefined; if (valueField != null) { acc.add(valueField); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts index 6a045f6694da1..93ef10894f856 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts @@ -39,7 +39,7 @@ describe('filterEvents', () => { { field: 'source.ip', operator: 'included', - matchedSet: new Set([JSON.stringify('1.1.1.1')]), + matchedSet: new Set([JSON.stringify(['1.1.1.1'])]), }, ]; const field = filterEvents({ @@ -55,7 +55,7 @@ describe('filterEvents', () => { { field: 'source.ip', operator: 'excluded', - matchedSet: new Set([JSON.stringify('1.1.1.1')]), + matchedSet: new Set([JSON.stringify(['1.1.1.1'])]), }, ]; const field = filterEvents({ @@ -71,7 +71,7 @@ describe('filterEvents', () => { { field: 'madeup.nonexistent', // field does not exist operator: 'included', - matchedSet: new Set([JSON.stringify('1.1.1.1')]), + matchedSet: new Set([JSON.stringify(['1.1.1.1'])]), }, ]; const field = filterEvents({ @@ -87,12 +87,12 @@ describe('filterEvents', () => { { field: 'source.ip', operator: 'included', - matchedSet: new Set([JSON.stringify('1.1.1.1')]), + matchedSet: new Set([JSON.stringify(['1.1.1.1'])]), }, { field: 'source.ip', operator: 'excluded', - matchedSet: new Set([JSON.stringify('1.1.1.1')]), + matchedSet: new Set([JSON.stringify(['1.1.1.1'])]), }, ]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts index e8667510da686..75fe1428cb5a7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash/fp'; import { SearchResponse } from '../../../types'; +import { getFieldFromHit } from '../utils'; import { FilterEventsOptions } from './types'; /** @@ -21,7 +21,7 @@ export const filterEvents = ({ return events.filter((item) => { return fieldAndSetTuples .map((tuple) => { - const eventItem = get(tuple.field, item._source); + const eventItem = item.fields ? item.fields[tuple.field] : undefined; if (eventItem == null) { return true; } else if (tuple.operator === 'included') { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index d52c2f5253711..163fc2bc5b1b1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -68,6 +68,12 @@ export const findThresholdSignals = async ({ }, }, ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], size: 1, }, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts index b1fab34d66ab8..6be4b0f9b8f34 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts @@ -80,6 +80,7 @@ export const getThreatListSearchResponseMock = (): SearchResponse ({ }, }); +export const getThreatListItemFieldsMock = () => ({ + '@timestamp': ['2020-09-09T21:59:13Z'], + 'host.name': ['host-1'], + 'host.ip': ['192.168.0.0.1'], + 'source.ip': ['127.0.0.1'], + 'source.port': [1], + 'destination.ip': ['127.0.0.1'], + 'destination.port': [1], +}); + export const getFilterThreatMapping = (): ThreatMapping => [ { entries: [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts index 8eed838fc9680..bc80ca9ba122b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts @@ -132,10 +132,16 @@ describe('build_threat_mapping_filter', () => { }, ], threatListItem: { - '@timestamp': '2020-09-09T21:59:13Z', - host: { - name: 'host-1', - // since ip is missing this entire AND clause should be dropped + _source: { + '@timestamp': '2020-09-09T21:59:13Z', + host: { + name: 'host-1', + // since ip is missing this entire AND clause should be dropped + }, + }, + fields: { + '@timestamp': ['2020-09-09T21:59:13Z'], + 'host.name': ['host-1'], }, }, }); @@ -176,6 +182,10 @@ describe('build_threat_mapping_filter', () => { name: 'host-1', }, }, + fields: { + '@timestamp': ['2020-09-09T21:59:13Z'], + 'host.name': ['host-1'], + }, }, }); expect(item).toEqual([ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts index 294d97e0bf2f1..aa43d6ce59bec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts @@ -54,7 +54,8 @@ export const filterThreatMapping = ({ threatMapping .map((threatMap) => { const atLeastOneItemMissingInThreatList = threatMap.entries.some((entry) => { - return get(entry.value, threatListItem._source) == null; + const itemValue = get(entry.value, threatListItem.fields); + return itemValue == null || itemValue.length !== 1; }); if (atLeastOneItemMissingInThreatList) { return { ...threatMap, entries: [] }; @@ -69,15 +70,15 @@ export const createInnerAndClauses = ({ threatListItem, }: CreateInnerAndClausesOptions): BooleanFilter[] => { return threatMappingEntries.reduce((accum, threatMappingEntry) => { - const value = get(threatMappingEntry.value, threatListItem._source); - if (value != null) { + const value = get(threatMappingEntry.value, threatListItem.fields); + if (value != null && value.length === 1) { // These values could be potentially 10k+ large so mutating the array intentionally accum.push({ bool: { should: [ { match: { - [threatMappingEntry.field]: value, + [threatMappingEntry.field]: value[0], }, }, ], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts index aba3f6f69d706..a47ddc26d85bb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts @@ -54,6 +54,12 @@ export const getThreatList = async ({ const response: SearchResponse = await callCluster('search', { body: { query: queryFilter, + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], search_after: searchAfter, sort: getSortWithTieBreaker({ sortField, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index ac004a7fff1fa..f4e53e83442ce 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -1072,6 +1072,9 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a non-existent @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; + } const { lastLookBackDate } = createSearchAfterReturnTypeFromResponse({ searchResult, timestampOverride: undefined, @@ -1082,6 +1085,9 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a null @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; + } const { lastLookBackDate } = createSearchAfterReturnTypeFromResponse({ searchResult, timestampOverride: undefined, @@ -1092,6 +1098,9 @@ describe('utils', () => { test('It will not set an invalid date time stamp from an invalid @timestamp string', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid'; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid']; + } const { lastLookBackDate } = createSearchAfterReturnTypeFromResponse({ searchResult, timestampOverride: undefined, @@ -1104,6 +1113,9 @@ describe('utils', () => { test('It returns undefined if the search result contains a null timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; + } const date = lastValidDate({ searchResult, timestampOverride: undefined }); expect(date).toEqual(undefined); }); @@ -1111,6 +1123,9 @@ describe('utils', () => { test('It returns undefined if the search result contains a undefined timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; + } const date = lastValidDate({ searchResult, timestampOverride: undefined }); expect(date).toEqual(undefined); }); @@ -1118,13 +1133,9 @@ describe('utils', () => { test('It returns undefined if the search result contains an invalid string value', () => { const searchResult = sampleDocSearchResultsNoSortId(); (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid value'; - const date = lastValidDate({ searchResult, timestampOverride: undefined }); - expect(date).toEqual(undefined); - }); - - test('It returns correct date time stamp if the search result contains an invalid string value', () => { - const searchResult = sampleDocSearchResultsNoSortId(); - (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid value'; + if (searchResult.hits.hits[0].fields != null) { + (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid value']; + } const date = lastValidDate({ searchResult, timestampOverride: undefined }); expect(date).toEqual(undefined); }); diff --git a/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts b/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts index c8bf6790ae9b2..b3d3bf939e687 100644 --- a/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts +++ b/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts @@ -59,6 +59,12 @@ export const getAnomalies = async ( })?.query, }, }, + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], sort: [{ record_score: { order: 'desc' } }], }, }, From 2ff44e129b13316c45137b4416e0485c46f56030 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 26 Jan 2021 20:08:26 -0500 Subject: [PATCH 2/3] Fix tests --- .../signals/__mocks__/es_results.ts | 4 ++-- .../filters/create_field_and_set_tuples.test.ts | 8 ++++---- .../filters/create_set_to_filter_against.ts | 1 - .../signals/filters/filter_events.ts | 1 - .../filters/filter_events_against_list.test.ts | 16 ++++++++-------- .../signals/search_after_bulk_create.test.ts | 14 +++++++------- .../signals/single_bulk_create.test.ts | 14 ++------------ 7 files changed, 23 insertions(+), 35 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 1963a8fbbfe0e..60f027bc83874 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -168,8 +168,8 @@ export const sampleDocWithSortId = ( fields: { someKey: ['someValue'], '@timestamp': ['2020-04-20T21:27:45+0000'], - 'source.ip': [ip ?? '127.0.0.1'], - 'destination.ip': [destIp ?? '127.0.0.1'], + 'source.ip': ip ? (Array.isArray(ip) ? ip : [ip]) : ['127.0.0.1'], + 'destination.ip': destIp ? (Array.isArray(destIp) ? destIp : [destIp]) : ['127.0.0.1'], }, sort: ['1234567891111'], }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts index 9192eeb35d0e8..42eb5ee23a3dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts @@ -119,7 +119,7 @@ describe('filterEventsAgainstList', () => { exceptionItem, buildRuleMessage, }); - expect([...matchedSet]).toEqual([JSON.stringify('1.1.1.1')]); + expect([...matchedSet]).toEqual([JSON.stringify(['1.1.1.1'])]); }); test('it returns two matched sets as a JSON.stringify() set from the "events"', async () => { @@ -132,7 +132,7 @@ describe('filterEventsAgainstList', () => { exceptionItem, buildRuleMessage, }); - expect([...matchedSet]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]); + expect([...matchedSet]).toEqual([JSON.stringify(['1.1.1.1']), JSON.stringify(['2.2.2.2'])]); }); test('it returns an array as a set as a JSON.stringify() array from the "events"', async () => { @@ -281,7 +281,7 @@ describe('filterEventsAgainstList', () => { exceptionItem, buildRuleMessage, }); - expect([...matchedSet1]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]); - expect([...matchedSet2]).toEqual([JSON.stringify('3.3.3.3'), JSON.stringify('5.5.5.5')]); + expect([...matchedSet1]).toEqual([JSON.stringify(['1.1.1.1']), JSON.stringify(['2.2.2.2'])]); + expect([...matchedSet2]).toEqual([JSON.stringify(['3.3.3.3']), JSON.stringify(['5.5.5.5'])]); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts index df07bf5bc3322..876acdb395089 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash/fp'; import { CreateSetToFilterAgainstOptions } from './types'; /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts index 75fe1428cb5a7..98a6c36f887f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts @@ -5,7 +5,6 @@ */ import { SearchResponse } from '../../../types'; -import { getFieldFromHit } from '../utils'; import { FilterEventsOptions } from './types'; /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts index eb6e905c03038..f66def0947e53 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts @@ -161,12 +161,12 @@ describe('filterEventsAgainstList', () => { // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, - { ...getSearchListItemResponseMock(), value: '4.4.4.4' }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, + { ...getSearchListItemResponseMock(), value: ['4.4.4.4'] }, ]); // this call represents an exception list with a value list containing ['6.6.6.6'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '6.6.6.6' }, + { ...getSearchListItemResponseMock(), value: ['6.6.6.6'] }, ]); const res = await filterEventsAgainstList({ @@ -223,11 +223,11 @@ describe('filterEventsAgainstList', () => { // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, ]); // this call represents an exception list with a value list containing ['6.6.6.6'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '6.6.6.6' }, + { ...getSearchListItemResponseMock(), value: ['6.6.6.6'] }, ]); const res = await filterEventsAgainstList({ @@ -282,11 +282,11 @@ describe('filterEventsAgainstList', () => { // this call represents an exception list with a value list containing ['2.2.2.2'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, ]); // this call represents an exception list with a value list containing ['4.4.4.4'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getSearchListItemResponseMock(), value: '4.4.4.4' }, + { ...getSearchListItemResponseMock(), value: ['4.4.4.4'] }, ]); const res = await filterEventsAgainstList({ @@ -364,7 +364,7 @@ describe('filterEventsAgainstList', () => { // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4'] (listClient.searchListItemByValues as jest.Mock).mockResolvedValue([ - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, ]); const res = await filterEventsAgainstList({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index b0459e1c225d6..48c5bc59c0122 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -309,9 +309,9 @@ describe('searchAfterAndBulkCreate', () => { test('should return success when all search results are in the allowlist and with sortId present', async () => { const searchListItems: SearchListItemArraySchema = [ - { ...getSearchListItemResponseMock(), value: '1.1.1.1' }, - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, - { ...getSearchListItemResponseMock(), value: '3.3.3.3' }, + { ...getSearchListItemResponseMock(), value: ['1.1.1.1'] }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, + { ...getSearchListItemResponseMock(), value: ['3.3.3.3'] }, ]; listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems); const sampleParams = sampleRuleAlertParams(30); @@ -373,10 +373,10 @@ describe('searchAfterAndBulkCreate', () => { test('should return success when all search results are in the allowlist and no sortId present', async () => { const searchListItems: SearchListItemArraySchema = [ - { ...getSearchListItemResponseMock(), value: '1.1.1.1' }, - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, - { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: ['1.1.1.1'] }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, + { ...getSearchListItemResponseMock(), value: ['2.2.2.2'] }, ]; listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts index eeeda6561892d..9e94ae1592888 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -315,19 +315,9 @@ describe('singleBulkCreate', () => { }); test('filter duplicate rules will return back search responses if they do not have a signal and will NOT filter the source out', () => { - const ancestors = sampleDocWithAncestors(); - ancestors.hits.hits[0]._source = { '@timestamp': '2020-04-20T21:27:45+0000' }; + const ancestors = sampleDocSearchResultsNoSortId(); const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', ancestors); - expect(filtered).toEqual([ - { - _index: 'myFakeSignalIndex', - _type: 'doc', - _score: 100, - _version: 1, - _id: 'e1e08ddc-5e37-49ff-a258-5393aa44435a', - _source: { '@timestamp': '2020-04-20T21:27:45+0000' }, - }, - ]); + expect(filtered).toEqual(ancestors.hits.hits); }); test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own numeric signal type', () => { From 8d8b62aedc9a99c5e8117f0355bbdc2cae4e62aa Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 26 Jan 2021 21:31:05 -0500 Subject: [PATCH 3/3] Change operator: excluded logic so missing fields are allowlisted --- .../detection_engine/signals/filters/filter_events.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts index 98a6c36f887f8..0196f0f3e3c49 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts @@ -21,12 +21,16 @@ export const filterEvents = ({ return fieldAndSetTuples .map((tuple) => { const eventItem = item.fields ? item.fields[tuple.field] : undefined; - if (eventItem == null) { - return true; - } else if (tuple.operator === 'included') { + if (tuple.operator === 'included') { + if (eventItem == null) { + return true; + } // only create a signal if the event is not in the value list return !tuple.matchedSet.has(JSON.stringify(eventItem)); } else if (tuple.operator === 'excluded') { + if (eventItem == null) { + return false; + } // only create a signal if the event is in the value list return tuple.matchedSet.has(JSON.stringify(eventItem)); } else {