diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts index 6ffbf4e4c8d4c..1b0417cf59bc2 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts @@ -48,6 +48,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -130,6 +132,8 @@ export const addPrepackagedRulesSchema = t.intersection([ threat_query, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults to "undefined" if not set during decode + items_per_search, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts index a4f002b589ef5..1b6a8d6f27762 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts @@ -1702,5 +1702,23 @@ describe('create rules schema', () => { expect(getPaths(left(message.errors))).toEqual([]); expect(message.schema).toEqual(expected); }); + + test('You can set a threat query, index, mapping, filters, concurrent_searches, items_per_search with a when creating a rule', () => { + const payload: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + concurrent_searches: 10, + items_per_search: 10, + }; + const decoded = createRulesSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected: CreateRulesSchemaDecoded = { + ...getCreateThreatMatchRulesSchemaDecodedMock(), + concurrent_searches: 10, + items_per_search: 10, + }; + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts index d8e7614fcb840..2fe52bbe470a5 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts @@ -49,6 +49,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -126,6 +128,8 @@ export const createRulesSchema = t.intersection([ threat_filters, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults "undefined" if not set during decode + items_per_search, // defaults "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts index 75ad92578318c..a78b41cd0da18 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts @@ -125,4 +125,36 @@ describe('create_rules_type_dependents', () => { const errors = createRuleValidateTypeDependents(schema); expect(errors).toEqual([]); }); + + test('validates that both "items_per_search" and "concurrent_searches" works when together', () => { + const schema: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + concurrent_searches: 10, + items_per_search: 10, + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([]); + }); + + test('does NOT validate when only "items_per_search" is present', () => { + const schema: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + items_per_search: 10, + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([ + 'when "items_per_search" exists, "concurrent_searches" must also exist', + ]); + }); + + test('does NOT validate when only "concurrent_searches" is present', () => { + const schema: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + concurrent_searches: 10, + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([ + 'when "concurrent_searches" exists, "items_per_search" must also exist', + ]); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts index c2a41005ebf4d..c93b0f0b14f6a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts @@ -110,17 +110,23 @@ export const validateThreshold = (rule: CreateRulesSchema): string[] => { export const validateThreatMapping = (rule: CreateRulesSchema): string[] => { let errors: string[] = []; if (isThreatMatchRule(rule.type)) { - if (!rule.threat_mapping) { + if (rule.threat_mapping == null) { errors = ['when "type" is "threat_match", "threat_mapping" is required', ...errors]; } else if (rule.threat_mapping.length === 0) { errors = ['threat_mapping" must have at least one element', ...errors]; } - if (!rule.threat_query) { + if (rule.threat_query == null) { errors = ['when "type" is "threat_match", "threat_query" is required', ...errors]; } - if (!rule.threat_index) { + if (rule.threat_index == null) { errors = ['when "type" is "threat_match", "threat_index" is required', ...errors]; } + if (rule.concurrent_searches == null && rule.items_per_search != null) { + errors = ['when "items_per_search" exists, "concurrent_searches" must also exist', ...errors]; + } + if (rule.concurrent_searches != null && rule.items_per_search == null) { + errors = ['when "concurrent_searches" exists, "items_per_search" must also exist', ...errors]; + } } return errors; }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts index 852394b74767b..4f28c46923865 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts @@ -55,6 +55,8 @@ import { } from '../common/schemas'; import { threat_index, + items_per_search, + concurrent_searches, threat_query, threat_filters, threat_mapping, @@ -149,6 +151,8 @@ export const importRulesSchema = t.intersection([ threat_query, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults to "undefined" if not set during decode + items_per_search, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts index f4dce5c7ac05f..45fcfbaa3c76a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts @@ -50,6 +50,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -109,6 +111,8 @@ export const patchRulesSchema = t.exact( threat_filters, threat_mapping, threat_language, + concurrent_searches, + items_per_search, }) ); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts index b0cd8b1c53688..5d759fc12cd52 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts @@ -51,6 +51,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -134,6 +136,8 @@ export const updateRulesSchema = t.intersection([ threat_filters, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults to "undefined" if not set during decode + items_per_search, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts index 82675768a11b7..3508526e182d7 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts @@ -762,9 +762,9 @@ describe('rules_schema', () => { expect(fields).toEqual(expected); }); - test('should return 5 fields for a rule of type "threat_match"', () => { + test('should return 8 fields for a rule of type "threat_match"', () => { const fields = addThreatMatchFields({ type: 'threat_match' }); - expect(fields.length).toEqual(6); + expect(fields.length).toEqual(8); }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index e85beddf0e51e..0f7d04763a36f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -63,6 +63,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -144,6 +146,8 @@ export const dependentRulesSchema = t.partial({ threat_filters, threat_index, threat_query, + concurrent_searches, + items_per_search, threat_mapping, threat_language, }); @@ -282,6 +286,12 @@ export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.exact(t.partial({ threat_language: dependentRulesSchema.props.threat_language })), t.exact(t.partial({ threat_filters: dependentRulesSchema.props.threat_filters })), t.exact(t.partial({ saved_id: dependentRulesSchema.props.saved_id })), + t.exact(t.partial({ concurrent_searches: dependentRulesSchema.props.concurrent_searches })), + t.exact( + t.partial({ + items_per_search: dependentRulesSchema.props.items_per_search, + }) + ), ]; } else { return []; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts index 63d593ea84e67..d8f61e4309b17 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts @@ -5,6 +5,8 @@ */ import { + concurrent_searches, + items_per_search, ThreatMapping, threatMappingEntries, ThreatMappingEntries, @@ -33,7 +35,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual(payload); }); - test('it should NOT validate an extra entry item', () => { + test('it should fail validation with an extra entry item', () => { const payload: ThreatMappingEntries & Array<{ extra: string }> = [ { field: 'field.one', @@ -50,7 +52,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate a non string', () => { + test('it should fail validation with a non string', () => { const payload = ([ { field: 5, @@ -66,7 +68,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate a wrong type', () => { + test('it should fail validation with a wrong type', () => { const payload = ([ { field: 'field.one', @@ -107,7 +109,7 @@ describe('threat_mapping', () => { }); }); - test('it should NOT validate an extra key', () => { + test('it should fail validate with an extra key', () => { const payload: ThreatMapping & Array<{ extra: string }> = [ { entries: [ @@ -129,7 +131,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate an extra inner entry', () => { + test('it should fail validate with an extra inner entry', () => { const payload: ThreatMapping & Array<{ entries: Array<{ extra: string }> }> = [ { entries: [ @@ -151,7 +153,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate an extra inner entry with the wrong data type', () => { + test('it should fail validate with an extra inner entry with the wrong data type', () => { const payload = ([ { entries: [ @@ -173,4 +175,48 @@ describe('threat_mapping', () => { ]); expect(message.schema).toEqual({}); }); + + test('it should fail validation when concurrent_searches is < 0', () => { + const payload = -1; + const decoded = concurrent_searches.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "-1" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should fail validation when concurrent_searches is 0', () => { + const payload = 0; + const decoded = concurrent_searches.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "0" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should fail validation when items_per_search is 0', () => { + const payload = 0; + const decoded = items_per_search.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "0" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should fail validation when items_per_search is < 0', () => { + const payload = -1; + const decoded = items_per_search.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "-1" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts index a1be6485f596b..dec8ddd000132 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts @@ -9,6 +9,7 @@ import * as t from 'io-ts'; import { language } from '../common/schemas'; import { NonEmptyString } from './non_empty_string'; +import { PositiveIntegerGreaterThanZero } from './positive_integer_greater_than_zero'; export const threat_query = t.string; export type ThreatQuery = t.TypeOf; @@ -55,3 +56,13 @@ export const threat_language = t.union([language, t.undefined]); export type ThreatLanguage = t.TypeOf; export const threatLanguageOrUndefined = t.union([threat_language, t.undefined]); export type ThreatLanguageOrUndefined = t.TypeOf; + +export const concurrent_searches = PositiveIntegerGreaterThanZero; +export type ConcurrentSearches = t.TypeOf; +export const concurrentSearchesOrUndefined = t.union([concurrent_searches, t.undefined]); +export type ConcurrentSearchesOrUndefined = t.TypeOf; + +export const items_per_search = PositiveIntegerGreaterThanZero; +export type ItemsPerSearch = t.TypeOf; +export const itemsPerSearchOrUndefined = t.union([items_per_search, t.undefined]); +export type ItemsPerSearchOrUndefined = t.TypeOf; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx index 164b1df8463e6..221963767caad 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx @@ -95,7 +95,7 @@ export const THREAT_MATCH_INDEX_HELPER_TEXT = i18n.translate( export const THREAT_MATCH_REQUIRED = i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customThreatQueryFieldRequiredError', { - defaultMessage: 'At least one threat match is required.', + defaultMessage: 'At least one indicator match is required.', } ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 94b820344b37c..773e84d9c88fc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -407,6 +407,8 @@ export const getResult = (): RuleAlertType => ({ note: '# Investigative notes', version: 1, exceptionsList: getListArrayMock(), + concurrentSearches: undefined, + itemsPerSearch: undefined, }, createdAt: new Date('2019-12-13T16:40:33.400Z'), updatedAt: new Date('2019-12-13T16:40:33.400Z'), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index 8c7a19869ce18..aa409580df965 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -102,6 +102,8 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threat_mapping: threatMapping, threat_query: threatQuery, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, threshold, throttle, timestamp_override: timestampOverride, @@ -193,6 +195,8 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threatQuery, threatIndex, threatLanguage, + concurrentSearches, + itemsPerSearch, threshold, timestampOverride, references, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 6ba7bc78fbded..97c05b4626ddc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -85,6 +85,8 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, throttle, timestamp_override: timestampOverride, to, @@ -182,6 +184,8 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts index 7cbcf25590921..688036c59c8ff 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -169,6 +169,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, threshold, timestamp_override: timestampOverride, to, @@ -235,6 +237,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, @@ -284,6 +288,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, references, note, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts index 4c310774ec72b..7dfb4daa1a0a2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts @@ -97,6 +97,8 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, timestamp_override: timestampOverride, throttle, references, @@ -162,6 +164,8 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts index dbdcd9844c0a7..aadb13ef54e72 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts @@ -83,6 +83,8 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, timestamp_override: timestampOverride, throttle, references, @@ -161,6 +163,8 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts index b93b3f319193f..f4a31c2bb456d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts @@ -102,6 +102,8 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, throttle, timestamp_override: timestampOverride, references, @@ -174,6 +176,8 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts index ea19fed5d6668..7ad525b67f7aa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -86,6 +86,8 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, throttle, timestamp_override: timestampOverride, references, @@ -163,6 +165,8 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index fb4ba855f6536..7360dc77aac22 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -151,6 +151,8 @@ export const transformAlertToRule = ( threat_query: alert.params.threatQuery, threat_mapping: alert.params.threatMapping, threat_language: alert.params.threatLanguage, + concurrent_searches: alert.params.concurrentSearches, + items_per_search: alert.params.itemsPerSearch, throttle: ruleActions?.ruleThrottle || 'no_actions', timestamp_override: alert.params.timestampOverride, note: alert.params.note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts index 271b1043ea568..68199c531a2fe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts @@ -43,6 +43,8 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ threatFilters: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, threatQuery: undefined, threatIndex: undefined, threshold: undefined, @@ -94,6 +96,8 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ threatMapping: undefined, threatQuery: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, threshold: undefined, timestampOverride: undefined, to: 'now', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index 776882d0f8494..3c814ce7e6606 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -46,6 +46,8 @@ export const createRules = async ({ threatFilters, threatIndex, threatLanguage, + concurrentSearches, + itemsPerSearch, threatQuery, threatMapping, threshold, @@ -96,6 +98,8 @@ export const createRules = async ({ threatFilters, threatIndex, threatQuery, + concurrentSearches, + itemsPerSearch, threatMapping, threatLanguage, timestampOverride, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts index 0a43c652234d0..4c01318f02cde 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -51,6 +51,8 @@ export const installPrepackagedRules = ( threat_filters: threatFilters, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, threat_query: threatQuery, threat_index: threatIndex, threshold, @@ -103,6 +105,8 @@ export const installPrepackagedRules = ( threatFilters, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, threatQuery, threatIndex, threshold, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts index ef7cd35f28f1b..60f1d599470e3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts @@ -154,6 +154,8 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, timestampOverride: undefined, to: 'now', type: 'query', @@ -203,6 +205,8 @@ export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({ threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, timestampOverride: undefined, to: 'now', type: 'machine_learning', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts index 1982dcf9dd9b6..22b2593283696 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts @@ -49,6 +49,8 @@ export const patchRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, @@ -97,6 +99,8 @@ export const patchRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, @@ -141,6 +145,8 @@ export const patchRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index fb4763a982f43..f6ab3fb0c3ed2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -92,6 +92,8 @@ import { ThreatMappingOrUndefined, ThreatFiltersOrUndefined, ThreatLanguageOrUndefined, + ConcurrentSearchesOrUndefined, + ItemsPerSearchOrUndefined, } from '../../../../common/detection_engine/schemas/types/threat_mapping'; import { AlertsClient, PartialAlert } from '../../../../../alerts/server'; @@ -234,6 +236,8 @@ export interface CreateRulesOptions { threatIndex: ThreatIndexOrUndefined; threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; threatLanguage: ThreatLanguageOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; @@ -284,6 +288,8 @@ export interface UpdateRulesOptions { threatIndex: ThreatIndexOrUndefined; threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; threatLanguage: ThreatLanguageOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; @@ -327,6 +333,8 @@ export interface PatchRulesOptions { severityMapping: SeverityMappingOrUndefined; tags: TagsOrUndefined; threat: ThreatOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; threshold: ThresholdOrUndefined; threatFilters: ThreatFiltersOrUndefined; threatIndex: ThreatIndexOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts index c685c4198c119..3d4b27b74c0af 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts @@ -52,6 +52,8 @@ export const updatePrepackagedRules = async ( threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, timestamp_override: timestampOverride, references, version, @@ -107,6 +109,8 @@ export const updatePrepackagedRules = async ( threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, references, version, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts index a33651580ef22..34be0f6ad843d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts @@ -49,6 +49,8 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ threatMapping: undefined, threatLanguage: undefined, timestampOverride: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: 'now', type: 'query', references: ['http://www.example.com'], @@ -99,6 +101,8 @@ export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({ threatMapping: undefined, threatLanguage: undefined, timestampOverride: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: 'now', type: 'machine_learning', references: ['http://www.example.com'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts index 3da921ed47f26..5168affca5c62 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts @@ -50,6 +50,8 @@ export const updateRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, @@ -99,6 +101,8 @@ export const updateRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts index 654383ff97c7a..8555af424ecd7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts @@ -60,6 +60,8 @@ describe('utils', () => { threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: undefined, timestampOverride: undefined, type: undefined, @@ -108,6 +110,8 @@ describe('utils', () => { threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: undefined, timestampOverride: undefined, type: undefined, @@ -158,6 +162,8 @@ describe('utils', () => { threatLanguage: undefined, to: undefined, timestampOverride: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, type: undefined, references: undefined, version: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts index a9a100543b528..83d9e3fd3e59f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts @@ -43,6 +43,8 @@ import { } from '../../../../common/detection_engine/schemas/common/schemas'; import { PartialFilter } from '../types'; import { + ConcurrentSearchesOrUndefined, + ItemsPerSearchOrUndefined, ListArrayOrUndefined, ThreatFiltersOrUndefined, ThreatIndexOrUndefined, @@ -98,6 +100,8 @@ export interface UpdateProperties { threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; threatLanguage: ThreatLanguageOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: ToOrUndefined; type: TypeOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh index 23c1914387c44..4807afd71e8d2 100755 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh @@ -12,7 +12,7 @@ set -e # Adds port mock data to a threat list for testing. # Example: ./create_threat_data.sh -# Example: ./create_threat_data.sh 1000 2000 +# Example: ./create_threat_data.sh 1 500 START=${1:-1} END=${2:-1000} @@ -22,7 +22,7 @@ do { curl -s -k \ -H "Content-Type: application/json" \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ - -X PUT ${ELASTICSEARCH_URL}/mock-threat-list/_doc/$i \ + -X PUT ${ELASTICSEARCH_URL}/mock-threat-list-1/_doc/$i \ --data " { \"@timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json new file mode 100644 index 0000000000000..c573db7fbca35 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json @@ -0,0 +1,32 @@ +{ + "concurrent_searches": 10, + "items_per_search": 10, + "index": ["auditbeat-*", "endgame-*", "filebeat-*", "logs-*", "packetbeat-*", "winlogbeat-*"], + "name": "Indicator Match Concurrent Searches", + "description": "Does 100 Concurrent searches with 10 items per search", + "rule_id": "indicator_concurrent_search", + "risk_score": 1, + "severity": "high", + "type": "threat_match", + "query": "*:*", + "tags": ["concurrent_searches_test", "from_script"], + "threat_index": ["mock-threat-list-1"], + "threat_language": "kuery", + "threat_query": "*:*", + "threat_mapping": [ + { + "entries": [ + { + "field": "source.port", + "type": "mapping", + "value": "source.port" + }, + { + "field": "source.ip", + "type": "mapping", + "value": "source.ip" + } + ] + } + ] +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 4559a658c9583..92e6b9562d970 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -68,6 +68,8 @@ export const sampleRuleAlertParams = ( threat: undefined, version: 1, exceptionsList: getListArrayMock(), + concurrentSearches: undefined, + itemsPerSearch: undefined, }); export const sampleRuleSO = (): SavedObject => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index cfe71f66395b0..50e740e81830f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -54,6 +54,8 @@ const signalSchema = schema.object({ threatQuery: schema.maybe(schema.string()), threatMapping: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threatLanguage: schema.maybe(schema.string()), + concurrentSearches: schema.maybe(schema.number()), + itemsPerSearch: schema.maybe(schema.number()), }); /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 415abc9d995fb..dc68e3949eb36 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -504,7 +504,7 @@ describe('rules_notification_alert_type', () => { await alert.executor(payload); expect(logger.error).toHaveBeenCalled(); expect(logger.error.mock.calls[0][0]).toContain( - 'An error occurred during rule execution: message: "Threat Match rule is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' + 'An error occurred during rule execution: message: "Indicator match is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' ); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index a0d5c833b208c..1d2b1c23f868f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -119,6 +119,8 @@ export const signalRulesAlertType = ({ timestampOverride, type, exceptionsList, + concurrentSearches, + itemsPerSearch, } = params; const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); @@ -360,7 +362,7 @@ export const signalRulesAlertType = ({ ) { throw new Error( [ - 'Threat Match rule is missing threatQuery and/or threatIndex and/or threatMapping:', + 'Indicator match is missing threatQuery and/or threatIndex and/or threatMapping:', `threatQuery: "${threatQuery}"`, `threatIndex: "${threatIndex}"`, `threatMapping: "${threatMapping}"`, @@ -403,6 +405,8 @@ export const signalRulesAlertType = ({ threatLanguage, buildRuleMessage, threatIndex, + concurrentSearches: concurrentSearches ?? 1, + itemsPerSearch: itemsPerSearch ?? 9000, }); } else if (type === 'query' || type === 'saved_query') { const inputIndex = await getInputIndex(services, version, index); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts index 85d172b3631a9..8eed838fc9680 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts @@ -19,28 +19,32 @@ import { } from './build_threat_mapping_filter'; import { getThreatMappingMock, - getThreatListSearchResponseMock, getThreatListItemMock, getThreatMappingFilterMock, getFilterThreatMapping, getThreatMappingFiltersShouldMock, getThreatMappingFilterShouldMock, + getThreatListSearchResponseMock, } from './build_threat_mapping_filter.mock'; -import { BooleanFilter } from './types'; +import { BooleanFilter, ThreatListItem } from './types'; describe('build_threat_mapping_filter', () => { describe('buildThreatMappingFilter', () => { test('it should throw if given a chunk over 1024 in size', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; expect(() => - buildThreatMappingFilter({ threatMapping, threatList, chunkSize: 1025 }) + buildThreatMappingFilter({ + threatMapping, + threatList, + chunkSize: 1025, + }) ).toThrow('chunk sizes cannot exceed 1024 in size'); }); test('it should NOT throw if given a chunk under 1024 in size', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; expect(() => buildThreatMappingFilter({ threatMapping, threatList, chunkSize: 1023 }) ).not.toThrow(); @@ -48,30 +52,30 @@ describe('build_threat_mapping_filter', () => { test('it should create the correct entries when using the default mocks', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const filter = buildThreatMappingFilter({ threatMapping, threatList }); expect(filter).toEqual(getThreatMappingFilterMock()); }); test('it should not mutate the original threatMapping', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; buildThreatMappingFilter({ threatMapping, threatList }); expect(threatMapping).toEqual(getThreatMappingMock()); }); test('it should not mutate the original threatListItem', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; buildThreatMappingFilter({ threatMapping, threatList }); - expect(threatList).toEqual(getThreatListSearchResponseMock()); + expect(threatList).toEqual(getThreatListSearchResponseMock().hits.hits); }); }); describe('filterThreatMapping', () => { test('it should not remove any entries when using the default mocks', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const item = filterThreatMapping({ threatMapping, threatListItem }); const expected = getFilterThreatMapping(); @@ -80,7 +84,7 @@ describe('build_threat_mapping_filter', () => { test('it should only give one filtered element if only 1 element is defined', () => { const [firstElement] = getThreatMappingMock(); // get only the first element - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const item = filterThreatMapping({ threatMapping: [firstElement], threatListItem }); const [firstElementFilter] = getFilterThreatMapping(); // get only the first element to compare @@ -89,7 +93,7 @@ describe('build_threat_mapping_filter', () => { test('it should not mutate the original threatMapping', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; filterThreatMapping({ threatMapping, @@ -100,13 +104,13 @@ describe('build_threat_mapping_filter', () => { test('it should not mutate the original threatListItem', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; filterThreatMapping({ threatMapping, threatListItem, }); - expect(threatListItem).toEqual(getThreatListItemMock()); + expect(threatListItem).toEqual(getThreatListSearchResponseMock().hits.hits[0]); }); test('it should remove the entire "AND" clause if one of the pieces of data is missing from the list', () => { @@ -166,9 +170,11 @@ describe('build_threat_mapping_filter', () => { }, ], threatListItem: { - '@timestamp': '2020-09-09T21:59:13Z', - host: { - name: 'host-1', + _source: { + '@timestamp': '2020-09-09T21:59:13Z', + host: { + name: 'host-1', + }, }, }, }); @@ -189,7 +195,7 @@ describe('build_threat_mapping_filter', () => { describe('createInnerAndClauses', () => { test('it should return two clauses given a single entry', () => { const [{ entries: threatMappingEntries }] = getThreatMappingMock(); // get the first element - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); const { bool: { @@ -219,7 +225,7 @@ describe('build_threat_mapping_filter', () => { type: 'mapping', }, ]; - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); const { bool: { @@ -248,7 +254,7 @@ describe('build_threat_mapping_filter', () => { type: 'mapping', }, ]; - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); const { bool: { @@ -275,7 +281,7 @@ describe('build_threat_mapping_filter', () => { type: 'mapping', }, ]; - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); expect(innerClause).toEqual([]); }); @@ -284,27 +290,31 @@ describe('build_threat_mapping_filter', () => { describe('createAndOrClauses', () => { test('it should return all clauses given the entries', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createAndOrClauses({ threatMapping, threatListItem }); expect(innerClause).toEqual(getThreatMappingFilterShouldMock()); }); test('it should filter out data from entries that do not have mappings', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = { ...getThreatListItemMock(), foo: 'bar' }; + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; + threatListItem._source = { + ...getThreatListSearchResponseMock().hits.hits[0]._source, + foo: 'bar', + }; const innerClause = createAndOrClauses({ threatMapping, threatListItem }); expect(innerClause).toEqual(getThreatMappingFilterShouldMock()); }); test('it should return an empty boolean given an empty array', () => { - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createAndOrClauses({ threatMapping: [], threatListItem }); expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } }); }); test('it should return an empty boolean clause given an empty object for a threat list item', () => { const threatMapping = getThreatMappingMock(); - const innerClause = createAndOrClauses({ threatMapping, threatListItem: {} }); + const innerClause = createAndOrClauses({ threatMapping, threatListItem: { _source: {} } }); expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } }); }); }); @@ -312,7 +322,7 @@ describe('build_threat_mapping_filter', () => { describe('buildEntriesMappingFilter', () => { test('it should return all clauses given the entries', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const mapping = buildEntriesMappingFilter({ threatMapping, threatList, @@ -326,8 +336,7 @@ describe('build_threat_mapping_filter', () => { test('it should return empty "should" given an empty threat list', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); - threatList.hits.hits = []; + const threatList: ThreatListItem[] = []; const mapping = buildEntriesMappingFilter({ threatMapping, threatList, @@ -340,7 +349,7 @@ describe('build_threat_mapping_filter', () => { }); test('it should return empty "should" given an empty threat mapping', () => { - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const mapping = buildEntriesMappingFilter({ threatMapping: [], threatList, @@ -374,7 +383,7 @@ describe('build_threat_mapping_filter', () => { }, ], ]; - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const mapping = buildEntriesMappingFilter({ threatMapping, threatList, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts index 346f156a9ec33..294d97e0bf2f1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts @@ -53,9 +53,9 @@ export const filterThreatMapping = ({ }: FilterThreatMappingOptions): ThreatMapping => threatMapping .map((threatMap) => { - const atLeastOneItemMissingInThreatList = threatMap.entries.some( - (entry) => get(entry.value, threatListItem) == null - ); + const atLeastOneItemMissingInThreatList = threatMap.entries.some((entry) => { + return get(entry.value, threatListItem._source) == null; + }); if (atLeastOneItemMissingInThreatList) { return { ...threatMap, entries: [] }; } else { @@ -69,7 +69,7 @@ export const createInnerAndClauses = ({ threatListItem, }: CreateInnerAndClausesOptions): BooleanFilter[] => { return threatMappingEntries.reduce((accum, threatMappingEntry) => { - const value = get(threatMappingEntry.value, threatListItem); + const value = get(threatMappingEntry.value, threatListItem._source); if (value != null) { // These values could be potentially 10k+ large so mutating the array intentionally accum.push({ @@ -114,24 +114,21 @@ export const buildEntriesMappingFilter = ({ threatList, chunkSize, }: BuildEntriesMappingFilterOptions): BooleanFilter => { - const combinedShould = threatList.hits.hits.reduce( - (accum, threatListSearchItem) => { - const filteredEntries = filterThreatMapping({ - threatMapping, - threatListItem: threatListSearchItem._source, - }); - const queryWithAndOrClause = createAndOrClauses({ - threatMapping: filteredEntries, - threatListItem: threatListSearchItem._source, - }); - if (queryWithAndOrClause.bool.should.length !== 0) { - // These values can be 10k+ large, so using a push here for performance - accum.push(queryWithAndOrClause); - } - return accum; - }, - [] - ); + const combinedShould = threatList.reduce((accum, threatListSearchItem) => { + const filteredEntries = filterThreatMapping({ + threatMapping, + threatListItem: threatListSearchItem, + }); + const queryWithAndOrClause = createAndOrClauses({ + threatMapping: filteredEntries, + threatListItem: threatListSearchItem, + }); + if (queryWithAndOrClause.bool.should.length !== 0) { + // These values can be 10k+ large, so using a push here for performance + accum.push(queryWithAndOrClause); + } + return accum; + }, []); const should = splitShouldClauses({ should: combinedShould, chunkSize }); return { bool: { should, minimum_should_match: 1 } }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 037f91240edfa..43fb759d07620 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -4,13 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getThreatList } from './get_threat_list'; import { buildThreatMappingFilter } from './build_threat_mapping_filter'; import { getFilter } from '../get_filter'; import { searchAfterAndBulkCreate } from '../search_after_bulk_create'; -import { CreateThreatSignalOptions, ThreatSignalResults } from './types'; -import { combineResults } from './utils'; +import { CreateThreatSignalOptions } from './types'; +import { SearchAfterAndBulkCreateReturnType } from '../types'; export const createThreatSignal = async ({ threatMapping, @@ -41,28 +40,11 @@ export const createThreatSignal = async ({ refresh, tags, throttle, - threatFilters, - threatQuery, - threatLanguage, buildRuleMessage, - threatIndex, name, currentThreatList, currentResult, -}: CreateThreatSignalOptions): Promise => { - const threatList = await getThreatList({ - callCluster: services.callCluster, - exceptionItems, - query: threatQuery, - language: threatLanguage, - threatFilters, - index: threatIndex, - searchAfter: currentThreatList.hits.hits[currentThreatList.hits.hits.length - 1].sort, - sortField: undefined, - sortOrder: undefined, - listClient, - }); - +}: CreateThreatSignalOptions): Promise => { const threatFilter = buildThreatMappingFilter({ threatMapping, threatList: currentThreatList, @@ -71,7 +53,12 @@ export const createThreatSignal = async ({ if (threatFilter.query.bool.should.length === 0) { // empty threat list and we do not want to return everything as being // a hit so opt to return the existing result. - return { threatList, results: currentResult }; + logger.debug( + buildRuleMessage( + 'Indicator items are empty after filtering for missing data, returning without attempting a match' + ) + ); + return currentResult; } else { const esFilter = await getFilter({ type, @@ -83,7 +70,13 @@ export const createThreatSignal = async ({ index: inputIndex, lists: exceptionItems, }); - const newResult = await searchAfterAndBulkCreate({ + + logger.debug( + buildRuleMessage( + `${threatFilter.query.bool.should.length} indicator items are being checked for existence of matches` + ) + ); + const result = await searchAfterAndBulkCreate({ gap, previousStartedAt, listClient, @@ -110,7 +103,15 @@ export const createThreatSignal = async ({ throttle, buildRuleMessage, }); - const results = combineResults(currentResult, newResult); - return { threatList, results }; + logger.debug( + buildRuleMessage( + `${ + threatFilter.query.bool.should.length + } items have completed match checks and the total times to search were ${ + result.searchAfterTimes.length !== 0 ? result.searchAfterTimes : '(unknown) ' + }ms` + ) + ); + return result; } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 8be76dc8caf0f..e90c45d40de95 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -4,11 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getThreatList } from './get_threat_list'; +import chunk from 'lodash/fp/chunk'; +import { getThreatList, getThreatListCount } from './get_threat_list'; import { CreateThreatSignalsOptions } from './types'; import { createThreatSignal } from './create_threat_signal'; import { SearchAfterAndBulkCreateReturnType } from '../types'; +import { combineConcurrentResults } from './utils'; export const createThreatSignals = async ({ threatMapping, @@ -45,7 +47,12 @@ export const createThreatSignals = async ({ buildRuleMessage, threatIndex, name, + concurrentSearches, + itemsPerSearch, }: CreateThreatSignalsOptions): Promise => { + logger.debug(buildRuleMessage('Indicator matching rule starting')); + const perPage = concurrentSearches * itemsPerSearch; + let results: SearchAfterAndBulkCreateReturnType = { success: true, bulkCreateTimes: [], @@ -55,6 +62,16 @@ export const createThreatSignals = async ({ errors: [], }; + let threatListCount = await getThreatListCount({ + callCluster: services.callCluster, + exceptionItems, + threatFilters, + query: threatQuery, + language: threatLanguage, + index: threatIndex, + }); + logger.debug(buildRuleMessage(`Total indicator items: ${threatListCount}`)); + let threatList = await getThreatList({ callCluster: services.callCluster, exceptionItems, @@ -66,47 +83,89 @@ export const createThreatSignals = async ({ searchAfter: undefined, sortField: undefined, sortOrder: undefined, + logger, + buildRuleMessage, + perPage, }); - while (threatList.hits.hits.length !== 0 && results.createdSignalsCount <= params.maxSignals) { - ({ threatList, results } = await createThreatSignal({ - threatMapping, - query, - inputIndex, - type, - filters, - language, - savedId, - services, + while (threatList.hits.hits.length !== 0) { + const chunks = chunk(itemsPerSearch, threatList.hits.hits); + logger.debug(buildRuleMessage(`${chunks.length} concurrent indicator searches are starting.`)); + const concurrentSearchesPerformed = chunks.map>( + (slicedChunk) => + createThreatSignal({ + threatMapping, + query, + inputIndex, + type, + filters, + language, + savedId, + services, + exceptionItems, + gap, + previousStartedAt, + listClient, + logger, + eventsTelemetry, + alertId, + outputIndex, + params, + searchAfterSize, + actions, + createdBy, + createdAt, + updatedBy, + updatedAt, + interval, + enabled, + tags, + refresh, + throttle, + buildRuleMessage, + name, + currentThreatList: slicedChunk, + currentResult: results, + }) + ); + const searchesPerformed = await Promise.all(concurrentSearchesPerformed); + results = combineConcurrentResults(results, searchesPerformed); + threatListCount -= threatList.hits.hits.length; + logger.debug( + buildRuleMessage( + `Concurrent indicator match searches completed with ${results.createdSignalsCount} signals found`, + `search times of ${results.searchAfterTimes}ms,`, + `bulk create times ${results.bulkCreateTimes}ms,`, + `all successes are ${results.success}` + ) + ); + if (results.createdSignalsCount >= params.maxSignals) { + logger.debug( + buildRuleMessage( + `Indicator match has reached its max signals count ${params.maxSignals}. Additional indicator items not checked are ${threatListCount}` + ) + ); + break; + } + logger.debug(buildRuleMessage(`Indicator items left to check are ${threatListCount}`)); + + threatList = await getThreatList({ + callCluster: services.callCluster, exceptionItems, - gap, - previousStartedAt, - listClient, - logger, - eventsTelemetry, - alertId, - outputIndex, - params, - searchAfterSize, - actions, - createdBy, - createdAt, - updatedBy, - updatedAt, - interval, - enabled, - tags, - refresh, - throttle, + query: threatQuery, + language: threatLanguage, threatFilters, - threatQuery, + index: threatIndex, + searchAfter: threatList.hits.hits[threatList.hits.hits.length - 1].sort, + sortField: undefined, + sortOrder: undefined, + listClient, buildRuleMessage, - threatIndex, - threatLanguage, - name, - currentThreatList: threatList, - currentResult: results, - })); + logger, + perPage, + }); } + + logger.debug(buildRuleMessage('Indicator matching rule has completed')); return results; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts index 3147eb1705168..aba3f6f69d706 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts @@ -10,6 +10,7 @@ import { GetSortWithTieBreakerOptions, GetThreatListOptions, SortWithTieBreaker, + ThreatListCountOptions, ThreatListItem, } from './types'; @@ -30,6 +31,8 @@ export const getThreatList = async ({ exceptionItems, threatFilters, listClient, + buildRuleMessage, + logger, }: GetThreatListOptions): Promise> => { const calculatedPerPage = perPage ?? MAX_PER_PAGE; if (calculatedPerPage > 10000) { @@ -43,6 +46,11 @@ export const getThreatList = async ({ exceptionItems ); + logger.debug( + buildRuleMessage( + `Querying the indicator items from the index: "${index}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items` + ) + ); const response: SearchResponse = await callCluster('search', { body: { query: queryFilter, @@ -58,6 +66,8 @@ export const getThreatList = async ({ index, size: calculatedPerPage, }); + + logger.debug(buildRuleMessage(`Retrieved indicator items of size: ${response.hits.hits.length}`)); return response; }; @@ -89,3 +99,30 @@ export const getSortWithTieBreaker = ({ } } }; + +export const getThreatListCount = async ({ + callCluster, + query, + language, + threatFilters, + index, + exceptionItems, +}: ThreatListCountOptions): Promise => { + const queryFilter = getQueryFilter( + query, + language ?? 'kuery', + threatFilters, + index, + exceptionItems + ); + const response: { + count: number; + } = await callCluster('count', { + body: { + query: queryFilter, + }, + ignoreUnavailable: true, + index, + }); + return response.count; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 0078cf1b3c64f..2e32a4e682403 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -5,7 +5,6 @@ */ import { Duration } from 'moment'; -import { SearchResponse } from 'elasticsearch'; import { ListClient } from '../../../../../../lists/server'; import { Type, @@ -17,6 +16,8 @@ import { ThreatMappingEntries, ThreatIndex, ThreatLanguageOrUndefined, + ConcurrentSearches, + ItemsPerSearch, } from '../../../../../common/detection_engine/schemas/types/threat_mapping'; import { PartialFilter, RuleTypeParams } from '../../types'; import { AlertServices } from '../../../../../../alerts/server'; @@ -62,6 +63,8 @@ export interface CreateThreatSignalsOptions { threatIndex: ThreatIndex; threatLanguage: ThreatLanguageOrUndefined; name: string; + concurrentSearches: ConcurrentSearches; + itemsPerSearch: ItemsPerSearch; } export interface CreateThreatSignalOptions { @@ -93,24 +96,15 @@ export interface CreateThreatSignalOptions { tags: string[]; refresh: false | 'wait_for'; throttle: string; - threatFilters: PartialFilter[]; - threatQuery: ThreatQuery; buildRuleMessage: BuildRuleMessage; - threatIndex: ThreatIndex; - threatLanguage: ThreatLanguageOrUndefined; name: string; - currentThreatList: SearchResponse; + currentThreatList: ThreatListItem[]; currentResult: SearchAfterAndBulkCreateReturnType; } -export interface ThreatSignalResults { - threatList: SearchResponse; - results: SearchAfterAndBulkCreateReturnType; -} - export interface BuildThreatMappingFilterOptions { threatMapping: ThreatMapping; - threatList: SearchResponse; + threatList: ThreatListItem[]; chunkSize?: number; } @@ -131,7 +125,7 @@ export interface CreateAndOrClausesOptions { export interface BuildEntriesMappingFilterOptions { threatMapping: ThreatMapping; - threatList: SearchResponse; + threatList: ThreatListItem[]; chunkSize: number; } @@ -156,6 +150,17 @@ export interface GetThreatListOptions { threatFilters: PartialFilter[]; exceptionItems: ExceptionListItemSchema[]; listClient: ListClient; + buildRuleMessage: BuildRuleMessage; + logger: Logger; +} + +export interface ThreatListCountOptions { + callCluster: ILegacyScopedClusterClient['callAsCurrentUser']; + query: string; + language: ThreatLanguageOrUndefined; + threatFilters: PartialFilter[]; + index: string[]; + exceptionItems: ExceptionListItemSchema[]; } export interface GetSortWithTieBreakerOptions { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 27593b40b0c8f..840d64381c793 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -6,7 +6,13 @@ import { SearchAfterAndBulkCreateReturnType } from '../types'; -import { calculateAdditiveMax, combineResults } from './utils'; +import { + calculateAdditiveMax, + calculateMax, + calculateMaxLookBack, + combineConcurrentResults, + combineResults, +} from './utils'; describe('utils', () => { describe('calculateAdditiveMax', () => { @@ -156,4 +162,383 @@ describe('utils', () => { ); }); }); + + describe('calculateMax', () => { + test('it should return 0 for two empty arrays', () => { + const max = calculateMax([], []); + expect(max).toEqual('0'); + }); + + test('it should return 5 for two arrays with the numbers 5', () => { + const max = calculateMax(['5'], ['5']); + expect(max).toEqual('5'); + }); + + test('it should return 5 for two arrays with second array having just 5', () => { + const max = calculateMax([], ['5']); + expect(max).toEqual('5'); + }); + + test('it should return 5 for two arrays with first array having just 5', () => { + const max = calculateMax(['5'], []); + expect(max).toEqual('5'); + }); + + test('it should return 10 for the max of the two arrays when the max of each array is 10', () => { + const max = calculateMax(['3', '5', '1'], ['3', '5', '10']); + expect(max).toEqual('10'); + }); + + test('it should return 10 for the max of the two arrays when the max of the first is 10', () => { + const max = calculateMax(['3', '5', '10'], ['3', '5', '1']); + expect(max).toEqual('10'); + }); + }); + + describe('calculateMaxLookBack', () => { + test('it should return null if both are null', () => { + const max = calculateMaxLookBack(null, null); + expect(max).toEqual(null); + }); + + test('it should return undefined if both are undefined', () => { + const max = calculateMaxLookBack(undefined, undefined); + expect(max).toEqual(undefined); + }); + + test('it should return null if both one is null and other other is undefined', () => { + const max = calculateMaxLookBack(undefined, null); + expect(max).toEqual(null); + }); + + test('it should return null if both one is null and other other is undefined with flipped arguments', () => { + const max = calculateMaxLookBack(null, undefined); + expect(max).toEqual(null); + }); + + test('it should return a date time if one argument is null', () => { + const max = calculateMaxLookBack(null, new Date('2020-09-16T03:34:32.390Z')); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is null with flipped arguments', () => { + const max = calculateMaxLookBack(new Date('2020-09-16T03:34:32.390Z'), null); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is undefined', () => { + const max = calculateMaxLookBack(new Date('2020-09-16T03:34:32.390Z'), undefined); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is undefined with flipped arguments', () => { + const max = calculateMaxLookBack(undefined, new Date('2020-09-16T03:34:32.390Z')); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time that is larger than the other', () => { + const max = calculateMaxLookBack( + new Date('2020-10-16T03:34:32.390Z'), + new Date('2020-09-16T03:34:32.390Z') + ); + expect(max).toEqual(new Date('2020-10-16T03:34:32.390Z')); + }); + + test('it should return a date time that is larger than the other with arguments flipped', () => { + const max = calculateMaxLookBack( + new Date('2020-09-16T03:34:32.390Z'), + new Date('2020-10-16T03:34:32.390Z') + ); + expect(max).toEqual(new Date('2020-10-16T03:34:32.390Z')); + }); + }); + + describe('combineConcurrentResults', () => { + test('it should use the maximum found if given an empty array for newResults', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['30'], // max value from existingResult.searchAfterTimes + bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, []); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should work with empty arrays for searchAfterTimes and bulkCreateTimes', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: [], + bulkCreateTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 0, + errors: [], + }; + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['30'], // max value from existingResult.searchAfterTimes + bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should get the max of two new results and then combine the result with an existingResult correctly', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should get the max of two new results and then combine the result with an existingResult correctly when the results are flipped around', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult2, newResult1]); // two array elements are flipped + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should return the max date correctly if one date contains a null', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: null, + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should combine two results with success set to "true" if both are "true"', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.success).toEqual(true); + }); + + test('it should combine two results with success set to "false" if one of them is "false"', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.success).toEqual(false); + }); + + test('it should use the latest date if it is set in the new result', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.lastLookBackDate?.toISOString()).toEqual('2020-09-16T03:34:32.390Z'); + }); + + test('it should combine the searchAfterTimes and the bulkCreateTimes', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual( + expect.objectContaining({ + searchAfterTimes: ['60'], + bulkCreateTimes: ['50'], + }) + ); + }); + + test('it should combine errors together without duplicates', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: ['error 1', 'error 2', 'error 3'], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: ['error 4', 'error 1', 'error 3', 'error 5'], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual( + expect.objectContaining({ + errors: ['error 1', 'error 2', 'error 3', 'error 4', 'error 5'], + }) + ); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 401a4a1acb065..d6c91fad6d9cb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -19,6 +19,41 @@ export const calculateAdditiveMax = (existingTimers: string[], newTimers: string return [String(numericNewTimerMax + numericExistingTimerMax)]; }; +/** + * Given two timers this will take the max of each and then get the max from each. + * Max(Max(timer_array_1), Max(timer_array_2)) + * @param existingTimers String array of existing timers + * @param newTimers String array of new timers. + * @returns String array of the new maximum between the two timers + */ +export const calculateMax = (existingTimers: string[], newTimers: string[]): string => { + const numericNewTimerMax = Math.max(0, ...newTimers.map((time) => +time)); + const numericExistingTimerMax = Math.max(0, ...existingTimers.map((time) => +time)); + return String(Math.max(numericNewTimerMax, numericExistingTimerMax)); +}; + +/** + * Given two dates this will return the larger of the two unless one of them is null + * or undefined. If both one or the other is null/undefined it will return the newDate. + * If there is a mix of "undefined" and "null", this will prefer to set it to "null" as having + * a higher value than "undefined" + * @param existingDate The existing date which can be undefined or null or a date + * @param newDate The new date which can be undefined or null or a date + */ +export const calculateMaxLookBack = ( + existingDate: Date | null | undefined, + newDate: Date | null | undefined +): Date | null | undefined => { + const newDateValue = newDate === null ? 1 : newDate === undefined ? 0 : newDate.valueOf(); + const existingDateValue = + existingDate === null ? 1 : existingDate === undefined ? 0 : existingDate.valueOf(); + if (newDateValue >= existingDateValue) { + return newDate; + } else { + return existingDate; + } +}; + /** * Combines two results together and returns the results combined * @param currentResult The current result to combine with a newResult @@ -38,3 +73,39 @@ export const combineResults = ( createdSignalsCount: currentResult.createdSignalsCount + newResult.createdSignalsCount, errors: [...new Set([...currentResult.errors, ...newResult.errors])], }); + +/** + * Combines two results together and returns the results combined + * @param currentResult The current result to combine with a newResult + * @param newResult The new result to combine + */ +export const combineConcurrentResults = ( + currentResult: SearchAfterAndBulkCreateReturnType, + newResult: SearchAfterAndBulkCreateReturnType[] +): SearchAfterAndBulkCreateReturnType => { + const maxedNewResult = newResult.reduce( + (accum, item) => { + const maxSearchAfterTime = calculateMax(accum.searchAfterTimes, item.searchAfterTimes); + const maxBulkCreateTimes = calculateMax(accum.bulkCreateTimes, item.bulkCreateTimes); + const lastLookBackDate = calculateMaxLookBack(accum.lastLookBackDate, item.lastLookBackDate); + return { + success: accum.success && item.success, + searchAfterTimes: [maxSearchAfterTime], + bulkCreateTimes: [maxBulkCreateTimes], + lastLookBackDate, + createdSignalsCount: accum.createdSignalsCount + item.createdSignalsCount, + errors: [...new Set([...accum.errors, ...item.errors])], + }; + }, + { + success: true, + searchAfterTimes: [], + bulkCreateTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 0, + errors: [], + } + ); + + return combineResults(currentResult, maxedNewResult); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index cf4d989c1f4c8..5cac76e2b0c01 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -44,6 +44,8 @@ import { ThreatQueryOrUndefined, ThreatMappingOrUndefined, ThreatLanguageOrUndefined, + ConcurrentSearchesOrUndefined, + ItemsPerSearchOrUndefined, } from '../../../common/detection_engine/schemas/types/threat_mapping'; import { LegacyCallAPIOptions } from '../../../../../../src/core/server'; @@ -93,6 +95,8 @@ export interface RuleTypeParams { references: References; version: Version; exceptionsList: ListArrayOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; } // eslint-disable-next-line @typescript-eslint/no-explicit-any