From 0ce8db9c18b4841d903874157cda526d4547f5b4 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 5 Aug 2021 02:09:31 -0400 Subject: [PATCH] [Security Solution][RAC] Flatten alert fields (#107581) * incremental changes * No more type errors * Type guards * Begin adding tests * Flatten * Reduce scope of branch * Remove extraneous argument to filter_duplicate_signals --- .../src/technical_field_names.ts | 11 +- .../helper/get_alert_annotations.test.tsx | 4 + x-pack/plugins/rule_registry/README.md | 1 - .../common/assets/field_maps/ecs_field_map.ts | 2 +- .../field_maps/technical_rule_field_map.ts | 4 +- .../server/routes/get_alert_by_id.test.ts | 2 + .../utils/create_lifecycle_executor.test.ts | 13 + .../server/utils/create_lifecycle_executor.ts | 5 + .../rule_registry_log_client.ts | 11 +- .../create_security_rule_type_factory.ts | 4 +- .../factories/utils/build_alert.test.ts | 419 ++++++++++++++++++ .../rule_types/factories/utils/build_alert.ts | 182 ++++---- .../factories/utils/build_bulk_body.ts | 35 +- .../factories/utils/flatten_with_prefix.ts | 20 + .../rule_types/factories/wrap_hits_factory.ts | 39 +- .../rule_types/field_maps/alerts.ts | 15 + .../rule_types/field_maps/field_names.ts | 13 + .../signals/__mocks__/es_results.ts | 16 + .../signals/filter_duplicate_signals.ts | 24 +- .../lib/detection_engine/signals/types.ts | 8 + .../lib/detection_engine/signals/utils.ts | 32 +- 21 files changed, 710 insertions(+), 150 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts diff --git a/packages/kbn-rule-data-utils/src/technical_field_names.ts b/packages/kbn-rule-data-utils/src/technical_field_names.ts index 0df1ad79f6702..a29c1023caf67 100644 --- a/packages/kbn-rule-data-utils/src/technical_field_names.ts +++ b/packages/kbn-rule-data-utils/src/technical_field_names.ts @@ -76,6 +76,12 @@ const ALERT_RULE_UPDATED_AT = `${ALERT_RULE_NAMESPACE}.updated_at` as const; const ALERT_RULE_UPDATED_BY = `${ALERT_RULE_NAMESPACE}.updated_by` as const; const ALERT_RULE_VERSION = `${ALERT_RULE_NAMESPACE}.version` as const; +const namespaces = { + KIBANA_NAMESPACE, + ALERT_NAMESPACE, + ALERT_RULE_NAMESPACE, +}; + const fields = { CONSUMERS, ECS_VERSION, @@ -146,6 +152,8 @@ export { ALERT_EVALUATION_THRESHOLD, ALERT_EVALUATION_VALUE, ALERT_ID, + ALERT_NAMESPACE, + ALERT_RULE_NAMESPACE, ALERT_OWNER, ALERT_CONSUMERS, ALERT_PRODUCER, @@ -191,6 +199,7 @@ export { ECS_VERSION, EVENT_ACTION, EVENT_KIND, + KIBANA_NAMESPACE, RULE_CATEGORY, RULE_CONSUMERS, RULE_ID, @@ -202,4 +211,4 @@ export { VERSION, }; -export type TechnicalRuleDataFieldName = ValuesType; +export type TechnicalRuleDataFieldName = ValuesType; diff --git a/x-pack/plugins/apm/public/components/shared/charts/helper/get_alert_annotations.test.tsx b/x-pack/plugins/apm/public/components/shared/charts/helper/get_alert_annotations.test.tsx index 01a8293163106..4f3ac31517020 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/helper/get_alert_annotations.test.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/helper/get_alert_annotations.test.tsx @@ -11,10 +11,12 @@ import { ALERT_EVALUATION_VALUE, ALERT_ID, ALERT_PRODUCER, + ALERT_OWNER, ALERT_SEVERITY_LEVEL, ALERT_START, ALERT_STATUS, ALERT_UUID, + SPACE_IDS, } from '@kbn/rule-data-utils'; import { ValuesType } from 'utility-types'; import { EuiTheme } from '../../../../../../../../src/plugins/kibana_react/common'; @@ -32,6 +34,7 @@ const theme = ({ eui: { euiColorDanger, euiColorWarning }, } as unknown) as EuiTheme; const alert: Alert = { + [SPACE_IDS]: ['space-id'], 'rule.id': ['apm.transaction_duration'], [ALERT_EVALUATION_VALUE]: [2057657.39], 'service.name': ['frontend-rum'], @@ -42,6 +45,7 @@ const alert: Alert = { 'transaction.type': ['page-load'], [ALERT_PRODUCER]: ['apm'], [ALERT_UUID]: ['af2ae371-df79-4fca-b0eb-a2dbd9478180'], + [ALERT_OWNER]: ['apm'], 'rule.uuid': ['82e0ee40-c2f4-11eb-9a42-a9da66a1722f'], 'event.action': ['active'], '@timestamp': ['2021-06-01T16:16:05.183Z'], diff --git a/x-pack/plugins/rule_registry/README.md b/x-pack/plugins/rule_registry/README.md index 19e276aeec489..16e4b8f3e01e6 100644 --- a/x-pack/plugins/rule_registry/README.md +++ b/x-pack/plugins/rule_registry/README.md @@ -124,7 +124,6 @@ The following fields are defined in the technical field component template and s - `rule.uuid`: the saved objects id of the rule. - `rule.name`: the name of the rule (as specified by the user). - `rule.category`: the name of the rule type (as defined by the rule type producer) -- `kibana.alert.producer`: the producer of the rule type. Usually a Kibana plugin. e.g., `APM`. - `kibana.alert.owner`: the feature which produced the alert. Usually a Kibana feature id like `apm`, `siem`... - `kibana.alert.id`: the id of the alert, that is unique within the context of the rule execution it was created in. E.g., for a rule that monitors latency for all services in all environments, this might be `opbeans-java:production`. - `kibana.alert.uuid`: the unique identifier for the alert during its lifespan. If an alert recovers (or closes), this identifier is re-generated when it is opened again. diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts index ff81e05851f7e..859070bd498e3 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts @@ -2148,7 +2148,7 @@ export const ecsFieldMap = { 'rule.id': { type: 'keyword', array: false, - required: false, + required: true, }, 'rule.license': { type: 'keyword', diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts index ebe7c88f756b2..11e572260d133 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts @@ -20,9 +20,9 @@ export const technicalRuleFieldMap = { Fields.RULE_CATEGORY, Fields.TAGS ), - [Fields.ALERT_OWNER]: { type: 'keyword' }, + [Fields.ALERT_OWNER]: { type: 'keyword', required: true }, [Fields.ALERT_PRODUCER]: { type: 'keyword' }, - [Fields.SPACE_IDS]: { type: 'keyword', array: true }, + [Fields.SPACE_IDS]: { type: 'keyword', array: true, required: true }, [Fields.ALERT_UUID]: { type: 'keyword' }, [Fields.ALERT_ID]: { type: 'keyword' }, [Fields.ALERT_START]: { type: 'date' }, diff --git a/x-pack/plugins/rule_registry/server/routes/get_alert_by_id.test.ts b/x-pack/plugins/rule_registry/server/routes/get_alert_by_id.test.ts index d89eb305545e8..073a48248f89a 100644 --- a/x-pack/plugins/rule_registry/server/routes/get_alert_by_id.test.ts +++ b/x-pack/plugins/rule_registry/server/routes/get_alert_by_id.test.ts @@ -13,6 +13,7 @@ import { CONSUMERS, ECS_VERSION, RULE_ID, + SPACE_IDS, TIMESTAMP, VERSION, } from '@kbn/rule-data-utils'; @@ -33,6 +34,7 @@ const getMockAlert = (): ParsedTechnicalFields => ({ [ALERT_OWNER]: 'apm', [ALERT_STATUS]: 'open', [ALERT_RULE_RISK_SCORE]: 20, + [SPACE_IDS]: ['fake-space-id'], [ALERT_RULE_SEVERITY]: 'warning', }); diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts index 80b75b8c74732..037efadabd8de 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts @@ -23,6 +23,9 @@ import { ALERT_STATUS, EVENT_ACTION, EVENT_KIND, + RULE_ID, + ALERT_OWNER, + SPACE_IDS, } from '../../common/technical_rule_data_field_names'; import { createRuleDataClientMock } from '../rule_data_client/create_rule_data_client_mock'; import { createLifecycleExecutor } from './create_lifecycle_executor'; @@ -128,12 +131,16 @@ describe('createLifecycleExecutor', () => { { fields: { [ALERT_ID]: 'TEST_ALERT_0', + [ALERT_OWNER]: 'CONSUMER', + [RULE_ID]: 'RULE_TYPE_ID', labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc }, }, { fields: { [ALERT_ID]: 'TEST_ALERT_1', + [ALERT_OWNER]: 'CONSUMER', + [RULE_ID]: 'RULE_TYPE_ID', labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc }, }, @@ -222,6 +229,9 @@ describe('createLifecycleExecutor', () => { fields: { '@timestamp': '', [ALERT_ID]: 'TEST_ALERT_0', + [ALERT_OWNER]: 'CONSUMER', + [RULE_ID]: 'RULE_TYPE_ID', + [SPACE_IDS]: ['fake-space-id'], labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must show up in the written doc }, }, @@ -229,6 +239,9 @@ describe('createLifecycleExecutor', () => { fields: { '@timestamp': '', [ALERT_ID]: 'TEST_ALERT_1', + [ALERT_OWNER]: 'CONSUMER', + [RULE_ID]: 'RULE_TYPE_ID', + [SPACE_IDS]: ['fake-space-id'], labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc }, }, diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts index 50ac8afb945b4..23ae24cb91bc4 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts @@ -30,6 +30,7 @@ import { EVENT_ACTION, EVENT_KIND, ALERT_OWNER, + RULE_ID, RULE_UUID, TIMESTAMP, SPACE_IDS, @@ -154,6 +155,8 @@ export const createLifecycleExecutor = ( currentAlerts[id] = { ...fields, [ALERT_ID]: id, + [RULE_ID]: rule.ruleTypeId, + [ALERT_OWNER]: rule.consumer, }; return alertInstanceFactory(id); }, @@ -226,6 +229,8 @@ export const createLifecycleExecutor = ( alertsDataMap[alertId] = { ...fields, [ALERT_ID]: alertId, + [RULE_ID]: rule.ruleTypeId, + [ALERT_OWNER]: rule.consumer, }; }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts index 362a9985bf85c..e2c5c98702f12 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts @@ -6,7 +6,14 @@ */ import { estypes } from '@elastic/elasticsearch'; -import { EVENT_ACTION, EVENT_KIND, RULE_ID, SPACE_IDS, TIMESTAMP } from '@kbn/rule-data-utils'; +import { + ALERT_OWNER, + EVENT_ACTION, + EVENT_KIND, + RULE_ID, + SPACE_IDS, + TIMESTAMP, +} from '@kbn/rule-data-utils'; import { once } from 'lodash/fp'; import moment from 'moment'; import { RuleDataClient } from '../../../../../../rule_registry/server'; @@ -221,6 +228,7 @@ export class RuleRegistryLogClient implements IRuleRegistryLogClient { [getMetricField(metric)]: value, [RULE_ID]: ruleId, [TIMESTAMP]: new Date().toISOString(), + [ALERT_OWNER]: 'siem', }, namespace ); @@ -244,6 +252,7 @@ export class RuleRegistryLogClient implements IRuleRegistryLogClient { [RULE_STATUS_SEVERITY]: statusSeverityDict[newStatus], [RULE_STATUS]: newStatus, [TIMESTAMP]: new Date().toISOString(), + [ALERT_OWNER]: 'siem', }, namespace ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts index f48fc564a3f63..168502d120b8f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts @@ -191,8 +191,10 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = ({ ); const wrapHits = wrapHitsFactory({ - ruleSO, + logger, mergeStrategy, + ruleSO, + spaceId, }); for (const tuple of tuples) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts new file mode 100644 index 0000000000000..f9874478e7a5d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts @@ -0,0 +1,419 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ALERT_OWNER, + ALERT_RULE_NAMESPACE, + ALERT_STATUS, + ALERT_WORKFLOW_STATUS, + SPACE_IDS, +} from '@kbn/rule-data-utils'; + +import { sampleDocNoSortIdWithTimestamp } from '../../../signals/__mocks__/es_results'; +import { flattenWithPrefix } from './flatten_with_prefix'; +import { + buildAlert, + buildParent, + buildAncestors, + additionalAlertFields, + removeClashes, +} from './build_alert'; +import { Ancestor, SignalSourceHit } from '../../../signals/types'; +import { + getRulesSchemaMock, + ANCHOR_DATE, +} from '../../../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; +import { + ALERT_ANCESTORS, + ALERT_ORIGINAL_EVENT, + ALERT_ORIGINAL_TIME, +} from '../../field_maps/field_names'; +import { SERVER_APP_ID } from '../../../../../../common/constants'; + +type SignalDoc = SignalSourceHit & { + _source: Required['_source'] & { '@timestamp': string }; +}; + +const SPACE_ID = 'space'; + +describe('buildAlert', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('it builds an alert as expected without original_event if event does not exist', () => { + const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + delete doc._source.event; + const rule = getRulesSchemaMock(); + const alert = { + ...buildAlert([doc], rule, SPACE_ID), + ...additionalAlertFields(doc), + }; + const timestamp = alert['@timestamp']; + const expected = { + '@timestamp': timestamp, + [SPACE_IDS]: [SPACE_ID], + [ALERT_OWNER]: SERVER_APP_ID, + [ALERT_ANCESTORS]: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z', + [ALERT_STATUS]: 'open', + [ALERT_WORKFLOW_STATUS]: 'open', + ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(ANCHOR_DATE).toISOString(), + updated_at: new Date(ANCHOR_DATE).toISOString(), + created_by: 'elastic', + description: 'some description', + enabled: true, + false_positives: ['false positive 1', 'false positive 2'], + from: 'now-6m', + immutable: false, + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + references: ['test 1', 'test 2'], + severity: 'high', + severity_mapping: [], + updated_by: 'elastic_kibana', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), + }), + 'kibana.alert.depth': 1, + }; + expect(alert).toEqual(expected); + }); + + test('it builds an alert as expected with original_event if is present', () => { + const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + doc._source.event = { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }; + const rule = getRulesSchemaMock(); + const alert = { + ...buildAlert([doc], rule, SPACE_ID), + ...additionalAlertFields(doc), + }; + const timestamp = alert['@timestamp']; + const expected = { + '@timestamp': timestamp, + [SPACE_IDS]: [SPACE_ID], + [ALERT_OWNER]: SERVER_APP_ID, + [ALERT_ANCESTORS]: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z', + [ALERT_ORIGINAL_EVENT]: { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }, + [ALERT_STATUS]: 'open', + [ALERT_WORKFLOW_STATUS]: 'open', + ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(ANCHOR_DATE).toISOString(), + updated_at: new Date(ANCHOR_DATE).toISOString(), + created_by: 'elastic', + description: 'some description', + enabled: true, + false_positives: ['false positive 1', 'false positive 2'], + from: 'now-6m', + immutable: false, + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + references: ['test 1', 'test 2'], + severity: 'high', + severity_mapping: [], + updated_by: 'elastic_kibana', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), + }), + 'kibana.alert.depth': 1, + }; + expect(alert).toEqual(expected); + }); + + test('it builds an ancestor correctly if the parent does not exist', () => { + const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + doc._source.event = { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }; + const parent = buildParent(doc); + const expected: Ancestor = { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }; + expect(parent).toEqual(expected); + }); + + test('it builds an ancestor correctly if the parent does exist', () => { + const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + doc._source.event = { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }; + doc._source.signal = { + parents: [ + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + depth: 1, + rule: { + id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', + }, + }; + const parent = buildParent(doc); + const expected: Ancestor = { + rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }; + expect(parent).toEqual(expected); + }); + + test('it builds an alert ancestor correctly if the parent does not exist', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc: SignalDoc = { + ...sampleDoc, + _source: { + ...sampleDoc._source, + '@timestamp': new Date().toISOString(), + }, + }; + doc._source.event = { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }; + const ancestor = buildAncestors(doc); + const expected: Ancestor[] = [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ]; + expect(ancestor).toEqual(expected); + }); + + test('it builds an alert ancestor correctly if the parent does exist', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc: SignalDoc = { + ...sampleDoc, + _source: { + ...sampleDoc._source, + '@timestamp': new Date().toISOString(), + }, + }; + doc._source.event = { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }; + doc._source.signal = { + parents: [ + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + rule: { + id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', + }, + depth: 1, + }; + const ancestors = buildAncestors(doc); + const expected: Ancestor[] = [ + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ]; + expect(ancestors).toEqual(expected); + }); + + describe('removeClashes', () => { + test('it will call renameClashes with a regular doc and not mutate it if it does not have a signal clash', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc: SignalDoc = { + ...sampleDoc, + _source: { + ...sampleDoc._source, + '@timestamp': new Date().toISOString(), + }, + }; + const output = removeClashes(doc); + expect(output).toBe(doc); // reference check + }); + + test('it will call renameClashes with a regular doc and not change anything', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc: SignalDoc = { + ...sampleDoc, + _source: { + ...sampleDoc._source, + '@timestamp': new Date().toISOString(), + }, + }; + const output = removeClashes(doc); + expect(output).toEqual(doc); // deep equal check + }); + + test('it will remove a "signal" numeric clash', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: 127, + }, + } as unknown) as SignalDoc; + const output = removeClashes(doc); + const timestamp = output._source['@timestamp']; + expect(output).toEqual({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + '@timestamp': timestamp, + }, + }); + }); + + test('it will remove a "signal" object clash', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { child_1: { child_2: 'Test nesting' } }, + }, + } as unknown) as SignalDoc; + const output = removeClashes(doc); + const timestamp = output._source['@timestamp']; + expect(output).toEqual({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + '@timestamp': timestamp, + }, + }); + }); + + test('it will not remove a "signal" if that is signal is one of our signals', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { rule: { id: '123' } }, + }, + } as unknown) as SignalDoc; + const output = removeClashes(doc); + const timestamp = output._source['@timestamp']; + const expected = { + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { rule: { id: '123' } }, + '@timestamp': timestamp, + }, + }; + expect(output).toEqual(expected); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index fbd033a7f4ec4..641b37cb54bc4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -5,124 +5,113 @@ * 2.0. */ -import { ALERT_STATUS, ALERT_WORKFLOW_STATUS } from '@kbn/rule-data-utils'; -import { SearchTypes } from '../../../../../../common/detection_engine/types'; +import { + ALERT_OWNER, + ALERT_RULE_NAMESPACE, + ALERT_STATUS, + ALERT_WORKFLOW_STATUS, + SPACE_IDS, +} from '@kbn/rule-data-utils'; import { RulesSchema } from '../../../../../../common/detection_engine/schemas/response/rules_schema'; import { isEventTypeSignal } from '../../../signals/build_event_type_signal'; +import { Ancestor, BaseSignalHit, SimpleHit } from '../../../signals/types'; import { - Ancestor, - BaseSignalHit, - SignalHit, - SignalSourceHit, - ThresholdResult, -} from '../../../signals/types'; -import { getValidDateFromDoc } from '../../../signals/utils'; + getField, + getValidDateFromDoc, + isWrappedRACAlert, + isWrappedSignalHit, +} from '../../../signals/utils'; import { invariant } from '../../../../../../common/utils/invariant'; -import { DEFAULT_MAX_SIGNALS } from '../../../../../../common/constants'; +import { RACAlert } from '../../types'; +import { flattenWithPrefix } from './flatten_with_prefix'; +import { + ALERT_ANCESTORS, + ALERT_DEPTH, + ALERT_ORIGINAL_EVENT, + ALERT_ORIGINAL_TIME, +} from '../../field_maps/field_names'; +import { SERVER_APP_ID } from '../../../../../../common/constants'; /** - * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child - * signal's `signal.parents` array. - * @param doc The parent signal or event + * Takes an event document and extracts the information needed for the corresponding entry in the child + * alert's ancestors array. + * @param doc The parent event */ -export const buildParent = (doc: BaseSignalHit): Ancestor => { - if (doc._source?.signal != null) { - return { - rule: doc._source?.signal.rule.id, - id: doc._id, - type: 'signal', - index: doc._index, - // We first look for signal.depth and use that if it exists. If it doesn't exist, this should be a pre-7.10 signal - // and should have signal.parent.depth instead. signal.parent.depth in this case is treated as equivalent to signal.depth. - depth: doc._source?.signal.depth ?? doc._source?.signal.parent?.depth ?? 1, - }; - } else { - return { - id: doc._id, - type: 'event', - index: doc._index, - depth: 0, - }; +export const buildParent = (doc: SimpleHit): Ancestor => { + const isSignal: boolean = isWrappedSignalHit(doc) || isWrappedRACAlert(doc); + const parent: Ancestor = { + id: doc._id, + type: isSignal ? 'signal' : 'event', + index: doc._index, + depth: isSignal ? getField(doc, 'signal.depth') ?? 1 : 0, + }; + if (isSignal) { + parent.rule = getField(doc, 'signal.rule.id'); } + return parent; }; /** - * Takes a parent signal or event document with N ancestors and adds the parent document to the ancestry array, + * Takes a parent event document with N ancestors and adds the parent document to the ancestry array, * creating an array of N+1 ancestors. - * @param doc The parent signal/event for which to extend the ancestry. + * @param doc The parent event for which to extend the ancestry. */ -export const buildAncestors = (doc: BaseSignalHit): Ancestor[] => { +export const buildAncestors = (doc: SimpleHit): Ancestor[] => { const newAncestor = buildParent(doc); - const existingAncestors = doc._source?.signal?.ancestors; - if (existingAncestors != null) { - return [...existingAncestors, newAncestor]; - } else { - return [newAncestor]; - } + const existingAncestors: Ancestor[] = getField(doc, 'signal.ancestors') ?? []; + return [...existingAncestors, newAncestor]; }; /** - * This removes any signal name clashes such as if a source index has + * This removes any alert name clashes such as if a source index has * "signal" but is not a signal object we put onto the object. If this * is our "signal object" then we don't want to remove it. * @param doc The source index doc to a signal. */ -export const removeClashes = (doc: BaseSignalHit): BaseSignalHit => { - invariant(doc._source, '_source field not found'); - const { signal, ...noSignal } = doc._source; - if (signal == null || isEventTypeSignal(doc)) { - return doc; - } else { - return { - ...doc, - _source: { ...noSignal }, - }; +export const removeClashes = (doc: SimpleHit) => { + if (isWrappedSignalHit(doc)) { + invariant(doc._source, '_source field not found'); + const { signal, ...noSignal } = doc._source; + if (signal == null || isEventTypeSignal(doc)) { + return doc; + } else { + return { + ...doc, + _source: { ...noSignal }, + }; + } } + return doc; }; /** - * Builds the `signal.*` fields that are common across all signals. - * @param docs The parent signals/events of the new signal to be built. - * @param rule The rule that is generating the new signal. + * Builds the `kibana.alert.*` fields that are common across all alerts. + * @param docs The parent alerts/events of the new alert to be built. + * @param rule The rule that is generating the new alert. */ -export const buildAlert = (doc: SignalSourceHit, rule: RulesSchema) => { - const removedClashes = removeClashes(doc); - const parent = buildParent(removedClashes); - const ancestors = buildAncestors(removedClashes); - const immutable = doc._source?.signal?.rule.immutable ? 'true' : 'false'; - - const source = doc._source as SignalHit; - const signal = source?.signal; - const signalRule = signal?.rule; +export const buildAlert = ( + docs: SimpleHit[], + rule: RulesSchema, + spaceId: string | null | undefined +): RACAlert => { + const removedClashes = docs.map(removeClashes); + const parents = removedClashes.map(buildParent); + const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; + const ancestors = removedClashes.reduce( + (acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), + [] + ); - return { - 'kibana.alert.ancestors': ancestors as object[], + return ({ + '@timestamp': new Date().toISOString(), + [ALERT_OWNER]: SERVER_APP_ID, + [SPACE_IDS]: spaceId != null ? [spaceId] : [], + [ALERT_ANCESTORS]: ancestors, [ALERT_STATUS]: 'open', [ALERT_WORKFLOW_STATUS]: 'open', - 'kibana.alert.depth': parent.depth, - 'kibana.alert.rule.false_positives': signalRule?.false_positives ?? [], - 'kibana.alert.rule.id': rule.id, - 'kibana.alert.rule.immutable': immutable, - 'kibana.alert.rule.index': signalRule?.index ?? [], - 'kibana.alert.rule.language': signalRule?.language ?? 'kuery', - 'kibana.alert.rule.max_signals': signalRule?.max_signals ?? DEFAULT_MAX_SIGNALS, - 'kibana.alert.rule.query': signalRule?.query ?? '*:*', - 'kibana.alert.rule.saved_id': signalRule?.saved_id ?? '', - 'kibana.alert.rule.threat_index': signalRule?.threat_index, - 'kibana.alert.rule.threat_indicator_path': signalRule?.threat_indicator_path, - 'kibana.alert.rule.threat_language': signalRule?.threat_language, - 'kibana.alert.rule.threat_mapping.field': '', // TODO - 'kibana.alert.rule.threat_mapping.value': '', // TODO - 'kibana.alert.rule.threat_mapping.type': '', // TODO - 'kibana.alert.rule.threshold.field': signalRule?.threshold?.field, - 'kibana.alert.rule.threshold.value': signalRule?.threshold?.value, - 'kibana.alert.rule.threshold.cardinality.field': '', // TODO - 'kibana.alert.rule.threshold.cardinality.value': 0, // TODO - }; -}; - -const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is ThresholdResult => { - return typeof thresholdResult === 'object'; + [ALERT_DEPTH]: depth, + ...flattenWithPrefix(ALERT_RULE_NAMESPACE, rule), + } as unknown) as RACAlert; }; /** @@ -131,17 +120,16 @@ const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is Thr * @param doc The parent signal/event of the new signal to be built. */ export const additionalAlertFields = (doc: BaseSignalHit) => { - const thresholdResult = doc._source?.threshold_result; - if (thresholdResult != null && !isThresholdResult(thresholdResult)) { - throw new Error(`threshold_result failed to validate: ${thresholdResult}`); - } const originalTime = getValidDateFromDoc({ doc, timestampOverride: undefined, }); - return { - 'kibana.alert.original_time': originalTime != null ? originalTime.toISOString() : undefined, - 'kibana.alert.original_event': doc._source?.event ?? undefined, - 'kibana.alert.threshold_result': thresholdResult, + const additionalFields: Record = { + [ALERT_ORIGINAL_TIME]: originalTime != null ? originalTime.toISOString() : undefined, }; + const event = doc._source?.event; + if (event != null) { + additionalFields[ALERT_ORIGINAL_EVENT] = event; + } + return additionalFields; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts index 8c868ece26ceb..ca5857e0ee395 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts @@ -6,14 +6,21 @@ */ import { SavedObject } from 'src/core/types'; +import { BaseHit } from '../../../../../../common/detection_engine/types'; import type { ConfigType } from '../../../../../config'; -import { buildRuleWithOverrides } from '../../../signals/build_rule'; +import { buildRuleWithOverrides, buildRuleWithoutOverrides } from '../../../signals/build_rule'; import { getMergeStrategy } from '../../../signals/source_fields_merging/strategies'; -import { AlertAttributes, SignalSourceHit } from '../../../signals/types'; +import { AlertAttributes, SignalSource, SignalSourceHit } from '../../../signals/types'; import { RACAlert } from '../../types'; import { additionalAlertFields, buildAlert } from './build_alert'; import { filterSource } from './filter_source'; +const isSourceDoc = ( + hit: SignalSourceHit +): hit is BaseHit<{ '@timestamp': string; _source: SignalSource }> => { + return hit._source != null; +}; + /** * Formats the search_after result for insertion into the signals index. We first create a * "best effort" merged "fields" with the "_source" object, then build the signal object, @@ -24,17 +31,25 @@ import { filterSource } from './filter_source'; * @returns The body that can be added to a bulk call for inserting the signal. */ export const buildBulkBody = ( + spaceId: string | null | undefined, ruleSO: SavedObject, doc: SignalSourceHit, - mergeStrategy: ConfigType['alertMergeStrategy'] + mergeStrategy: ConfigType['alertMergeStrategy'], + applyOverrides: boolean ): RACAlert => { const mergedDoc = getMergeStrategy(mergeStrategy)({ doc }); - const rule = buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {}); + const rule = applyOverrides + ? buildRuleWithOverrides(ruleSO, mergedDoc._source ?? {}) + : buildRuleWithoutOverrides(ruleSO); const filteredSource = filterSource(mergedDoc); - return { - ...filteredSource, - ...buildAlert(mergedDoc, rule), - ...additionalAlertFields(mergedDoc), - '@timestamp': new Date().toISOString(), - }; + if (isSourceDoc(mergedDoc)) { + return { + ...filteredSource, + ...buildAlert([mergedDoc], rule, spaceId), + ...additionalAlertFields(mergedDoc), + '@timestamp': new Date().toISOString(), + }; + } + + throw Error('Error building alert from source document.'); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts new file mode 100644 index 0000000000000..d472dc5885e57 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SearchTypes } from '../../../../../../common/detection_engine/types'; + +export const flattenWithPrefix = ( + prefix: string, + obj: Record +): Record => { + return Object.keys(obj).reduce((acc: Record, key) => { + return { + ...acc, + [`${prefix}.${key}`]: obj[key], + }; + }, {}); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts index 620e599e7a499..0b00b2f656379 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { Logger } from 'kibana/server'; + import { SearchAfterAndBulkCreateParams, SignalSourceHit, WrapHits } from '../../signals/types'; import { buildBulkBody } from './utils/build_bulk_body'; import { generateId } from '../../signals/utils'; @@ -13,24 +15,33 @@ import type { ConfigType } from '../../../../config'; import { WrappedRACAlert } from '../types'; export const wrapHitsFactory = ({ - ruleSO, + logger, mergeStrategy, + ruleSO, + spaceId, }: { + logger: Logger; ruleSO: SearchAfterAndBulkCreateParams['ruleSO']; mergeStrategy: ConfigType['alertMergeStrategy']; + spaceId: string | null | undefined; }): WrapHits => (events) => { - const wrappedDocs: WrappedRACAlert[] = events.flatMap((doc) => [ - { - _index: '', - _id: generateId( - doc._index, - doc._id, - String(doc._version), - ruleSO.attributes.params.ruleId ?? '' - ), - _source: buildBulkBody(ruleSO, doc as SignalSourceHit, mergeStrategy), - }, - ]); + try { + const wrappedDocs: WrappedRACAlert[] = events.flatMap((doc) => [ + { + _index: '', + _id: generateId( + doc._index, + doc._id, + String(doc._version), + ruleSO.attributes.params.ruleId ?? '' + ), + _source: buildBulkBody(spaceId, ruleSO, doc as SignalSourceHit, mergeStrategy, true), + }, + ]); - return filterDuplicateSignals(ruleSO.id, wrappedDocs, true); + return filterDuplicateSignals(ruleSO.id, wrappedDocs, true); + } catch (error) { + logger.error(error); + return []; + } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts index 244905329f8ca..7ab998fe16074 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts @@ -43,6 +43,21 @@ export const alertsFieldMap: FieldMap = { array: false, required: true, }, + 'kibana.alert.group': { + type: 'object', + array: false, + required: false, + }, + 'kibana.alert.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'kibana.alert.group.index': { + type: 'keyword', + array: false, + required: false, + }, 'kibana.alert.original_event': { type: 'object', array: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts new file mode 100644 index 0000000000000..41b7e6b02c9c6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ALERT_NAMESPACE } from '@kbn/rule-data-utils'; + +export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors`; +export const ALERT_DEPTH = `${ALERT_NAMESPACE}.depth`; +export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event`; +export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time`; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 5f4a9f5f7a422..ed93c41035dca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -168,6 +168,22 @@ export const sampleDocNoSortId = ( sort: [], }); +export const sampleDocNoSortIdWithTimestamp = ( + someUuid: string = sampleIdGuid, + ip?: string +): SignalSourceHit & { + _source: Required['_source'] & { '@timestamp': string }; +} => { + const doc = sampleDocNoSortId(someUuid, ip); + return { + ...doc, + _source: { + ...doc._source, + '@timestamp': new Date().toISOString(), + }, + }; +}; + export const sampleDocSeverity = (severity?: unknown, fieldName?: string): SignalSourceHit => { const doc = { _index: 'myFakeSignalIndex', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts index fb562a2d11f0a..460cf6894a73c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts @@ -8,37 +8,21 @@ import { WrappedRACAlert } from '../rule_types/types'; import { Ancestor, SimpleHit, WrappedSignalHit } from './types'; -const isWrappedSignalHit = ( - signals: SimpleHit[], - isRuleRegistryEnabled: boolean -): signals is WrappedSignalHit[] => { - return !isRuleRegistryEnabled; -}; - -const isWrappedRACAlert = ( - signals: SimpleHit[], - isRuleRegistryEnabled: boolean -): signals is WrappedRACAlert[] => { - return isRuleRegistryEnabled; -}; - export const filterDuplicateSignals = ( ruleId: string, signals: SimpleHit[], isRuleRegistryEnabled: boolean ) => { - if (isWrappedSignalHit(signals, isRuleRegistryEnabled)) { - return signals.filter( + if (!isRuleRegistryEnabled) { + return (signals as WrappedSignalHit[]).filter( (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId) ); - } else if (isWrappedRACAlert(signals, isRuleRegistryEnabled)) { - return signals.filter( + } else { + return (signals as WrappedRACAlert[]).filter( (doc) => !(doc._source['kibana.alert.ancestors'] as Ancestor[]).some( (ancestor) => ancestor.rule === ruleId ) ); - } else { - return signals; } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 8088742b32f7e..6cbe0d1a52704 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -33,6 +33,8 @@ import { BuildRuleMessage } from './rule_messages'; import { TelemetryEventsSender } from '../../telemetry/sender'; import { RuleParams } from '../schemas/rule_schemas'; import { GenericBulkCreateResponse } from './bulk_create_factory'; +import { EcsFieldMap } from '../../../../../rule_registry/common/assets/field_maps/ecs_field_map'; +import { TypeOfFieldMap } from '../../../../../rule_registry/common/field_map'; // used for gap detection code // eslint-disable-next-line @typescript-eslint/naming-convention @@ -167,6 +169,12 @@ export interface GetResponse { _source: SearchTypes; } +export type EventHit = Exclude, '@timestamp'> & { + '@timestamp': string; + [key: string]: SearchTypes; +}; +export type WrappedEventHit = BaseHit; + export type SignalSearchResponse = estypes.SearchResponse; export type SignalSourceHit = estypes.SearchHit; export type WrappedSignalHit = BaseHit; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index cb1bf9d774359..72ac4f6d0f550 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -5,18 +5,20 @@ * 2.0. */ import { createHash } from 'crypto'; +import { chunk, get, isEmpty, partition } from 'lodash'; import moment from 'moment'; import uuidv5 from 'uuid/v5'; + import dateMath from '@elastic/datemath'; import type { estypes } from '@elastic/elasticsearch'; -import { chunk, isEmpty, partition } from 'lodash'; import { ApiResponse, Context } from '@elastic/elasticsearch/lib/Transport'; - +import { ALERT_ID } from '@kbn/rule-data-utils'; import type { ListArray, ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import { MAX_EXCEPTION_LIST_SIZE } from '@kbn/securitysolution-list-constants'; import { hasLargeValueList } from '@kbn/securitysolution-list-utils'; import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils'; import { ElasticsearchClient } from '@kbn/securitysolution-es-utils'; + import { TimestampOverrideOrUndefined, Privilege, @@ -39,6 +41,8 @@ import { RuleRangeTuple, BaseSignalHit, SignalSourceHit, + SimpleHit, + WrappedEventHit, } from './types'; import { BuildRuleMessage } from './rule_messages'; import { ShardError } from '../../types'; @@ -52,6 +56,8 @@ import { ThreatRuleParams, ThresholdRuleParams, } from '../schemas/rule_schemas'; +import { WrappedRACAlert } from '../rule_types/types'; +import { SearchTypes } from '../../../../common/detection_engine/types'; interface SortExceptionsReturn { exceptionsWithValueLists: ExceptionListItemSchema[]; @@ -928,3 +934,25 @@ export const buildChunkedOrFilter = (field: string, values: string[], chunkSize: }) .join(' OR '); }; + +export const isWrappedEventHit = (event: SimpleHit): event is WrappedEventHit => { + return !isWrappedSignalHit(event) && !isWrappedRACAlert(event); +}; + +export const isWrappedSignalHit = (event: SimpleHit): event is WrappedSignalHit => { + return (event as WrappedSignalHit)?._source?.signal != null; +}; + +export const isWrappedRACAlert = (event: SimpleHit): event is WrappedRACAlert => { + return (event as WrappedRACAlert)?._source?.[ALERT_ID] != null; +}; + +export const getField = (event: SimpleHit, field: string): T | undefined => { + if (isWrappedRACAlert(event)) { + return event._source, field.replace('signal', 'kibana.alert') as T; // TODO: handle special cases + } else if (isWrappedSignalHit(event)) { + return get(event._source, field) as T; + } else if (isWrappedEventHit(event)) { + return get(event._source, field) as T; + } +};