From 23edb41739b1f9b80e0a9b416618124b27c34ff1 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Mon, 9 Dec 2019 20:57:38 -0700 Subject: [PATCH] [SIEM][Detection Engine] Utilizes native alert tags ## Summary * Changes out the params of tags to use the native alert tags. * Updated unit tests * Updated examples Tests are: Post a query with a tag ```sh ./post_rule.sh ./rules/queries/query_with_tags.json ``` Filter by that tag: ```sh ./find_rule_by_filter.sh "alert.attributes.tags:tag_1" ``` Update a query with a tag: ```sh ./update_rule.sh ./rules/updates/update_tags.json ``` ### Checklist Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR. ~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~ ~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~ ~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~ - [ ] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios ~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~ ### For maintainers ~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~ - [ ] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process) --- .../alerts/__mocks__/es_results.ts | 3 +- .../detection_engine/alerts/create_rules.ts | 3 +- .../alerts/rules_alert_type.ts | 3 +- .../lib/detection_engine/alerts/types.ts | 2 +- .../detection_engine/alerts/update_rules.ts | 4 +- .../lib/detection_engine/alerts/utils.test.ts | 37 ++++++++++++++----- .../lib/detection_engine/alerts/utils.ts | 15 +++++++- .../lib/detection_engine/routes/utils.ts | 2 +- .../scripts/find_rule_by_filter.sh | 1 + .../rules/queries/query_with_tags.json | 12 ++++++ .../scripts/rules/updates/update_tags.json | 4 ++ 11 files changed, 65 insertions(+), 21 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/query_with_tags.json create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/updates/update_tags.json diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts index 435a8d9bf8688..4c113544e6e21 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -22,7 +22,6 @@ export const sampleRuleAlertParams = ( index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], type: 'query', from: 'now-6m', - tags: ['some fake tag'], to: 'now', severity: 'high', query: 'user.name: root or user.name: admin', @@ -277,7 +276,7 @@ export const sampleRule = (): Partial => { references: ['http://www.example.com', 'https://ww.example.com'], severity: 'high', updated_by: 'elastic', - tags: [], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_rules.ts index e673bb116e1dd..4cbf3756f58ac 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_rules.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_rules.ts @@ -37,7 +37,7 @@ export const createRules = async ({ return alertsClient.create({ data: { name, - tags: [], + tags, alertTypeId: SIGNALS_ID, params: { description, @@ -55,7 +55,6 @@ export const createRules = async ({ maxSignals, riskScore, severity, - tags, threats, to, type, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/rules_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/rules_alert_type.ts index 488c34c945b48..9823d8b3b9bea 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/rules_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/rules_alert_type.ts @@ -46,7 +46,6 @@ export const rulesAlertType = ({ maxSignals: schema.number({ defaultValue: DEFAULT_MAX_SIGNALS }), riskScore: schema.number(), severity: schema.string(), - tags: schema.arrayOf(schema.string(), { defaultValue: [] }), threats: schema.nullable(schema.arrayOf(schema.object({}, { allowUnknowns: true }))), to: schema.string(), type: schema.string(), @@ -70,6 +69,7 @@ export const rulesAlertType = ({ // TODO: Remove this hard extraction of name once this is fixed: https://github.com/elastic/kibana/issues/50522 const savedObject = await services.savedObjectsClient.get('alert', alertId); const name: string = savedObject.attributes.name; + const tags: string[] = savedObject.attributes.tags; const createdBy: string = savedObject.attributes.createdBy; const updatedBy: string = savedObject.attributes.updatedBy; @@ -134,6 +134,7 @@ export const rulesAlertType = ({ interval, enabled, pageSize: searchAfterSize, + tags, }); if (bulkIndexResult) { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts index 7f61765f5532c..c9d265ebffacd 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts @@ -148,7 +148,7 @@ export interface ReadRuleByRuleId { ruleId: string; } -export type RuleTypeParams = Omit; +export type RuleTypeParams = Omit; export type RuleAlertType = Alert & { id: string; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_rules.ts index a2d49b88a8f64..2eaa05ae2fa6a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_rules.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_rules.ts @@ -99,7 +99,6 @@ export const updateRules = async ({ maxSignals, riskScore, severity, - tags, threats, to, type, @@ -112,11 +111,10 @@ export const updateRules = async ({ } else if (!rule.enabled && enabled) { await alertsClient.enable({ id: rule.id }); } - return alertsClient.update({ id: rule.id, data: { - tags: [], + tags: tags != null ? tags : [], name: calculateName({ updatedName: name, originalName: rule.name }), interval: calculateInterval(interval, rule.interval), actions, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index 83bf509fa7a93..41052ab4bbb15 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -68,6 +68,7 @@ describe('utils', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], }); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; @@ -102,7 +103,7 @@ describe('utils', () => { query: 'user.name: root or user.name: admin', references: ['http://google.com'], severity: 'high', - tags: ['some fake tag'], + tags: ['some fake tag 1', 'some fake tag 2'], type: 'query', to: 'now', enabled: true, @@ -131,6 +132,7 @@ describe('utils', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], }); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; @@ -174,7 +176,7 @@ describe('utils', () => { query: 'user.name: root or user.name: admin', references: ['http://google.com'], severity: 'high', - tags: ['some fake tag'], + tags: ['some fake tag 1', 'some fake tag 2'], type: 'query', to: 'now', enabled: true, @@ -202,6 +204,7 @@ describe('utils', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], }); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; @@ -244,7 +247,7 @@ describe('utils', () => { query: 'user.name: root or user.name: admin', references: ['http://google.com'], severity: 'high', - tags: ['some fake tag'], + tags: ['some fake tag 1', 'some fake tag 2'], type: 'query', to: 'now', enabled: true, @@ -270,6 +273,7 @@ describe('utils', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], }); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; @@ -307,7 +311,7 @@ describe('utils', () => { query: 'user.name: root or user.name: admin', references: ['http://google.com'], severity: 'high', - tags: ['some fake tag'], + tags: ['some fake tag 1', 'some fake tag 2'], type: 'query', to: 'now', enabled: true, @@ -448,6 +452,7 @@ describe('utils', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(successfulsingleBulkCreate).toEqual(true); }); @@ -475,6 +480,7 @@ describe('utils', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(successfulsingleBulkCreate).toEqual(true); }); @@ -494,6 +500,7 @@ describe('utils', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(successfulsingleBulkCreate).toEqual(true); }); @@ -513,6 +520,7 @@ describe('utils', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(mockLogger.error).toHaveBeenCalled(); expect(successfulsingleBulkCreate).toEqual(true); @@ -583,6 +591,7 @@ describe('utils', () => { enabled: true, pageSize: 1, filter: undefined, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(mockService.callCluster).toHaveBeenCalledTimes(0); expect(result).toEqual(true); @@ -634,6 +643,7 @@ describe('utils', () => { enabled: true, pageSize: 1, filter: undefined, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(mockService.callCluster).toHaveBeenCalledTimes(5); expect(result).toEqual(true); @@ -656,6 +666,7 @@ describe('utils', () => { enabled: true, pageSize: 1, filter: undefined, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(mockLogger.error).toHaveBeenCalled(); expect(result).toEqual(false); @@ -685,6 +696,7 @@ describe('utils', () => { enabled: true, pageSize: 1, filter: undefined, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(mockLogger.error).toHaveBeenCalled(); expect(result).toEqual(false); @@ -714,6 +726,7 @@ describe('utils', () => { enabled: true, pageSize: 1, filter: undefined, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(result).toEqual(true); }); @@ -745,6 +758,7 @@ describe('utils', () => { enabled: true, pageSize: 1, filter: undefined, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(result).toEqual(true); }); @@ -776,6 +790,7 @@ describe('utils', () => { enabled: true, pageSize: 1, filter: undefined, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(result).toEqual(true); }); @@ -809,6 +824,7 @@ describe('utils', () => { enabled: true, pageSize: 1, filter: undefined, + tags: ['some fake tag 1', 'some fake tag 2'], }); expect(result).toEqual(false); }); @@ -884,7 +900,7 @@ describe('utils', () => { references: ['http://www.example.com', 'https://ww.example.com'], severity: 'high', updated_by: 'elastic', - tags: [], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', }, @@ -937,7 +953,7 @@ describe('utils', () => { references: ['http://www.example.com', 'https://ww.example.com'], severity: 'high', updated_by: 'elastic', - tags: [], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', }, @@ -968,6 +984,7 @@ describe('utils', () => { createdBy: 'elastic', updatedBy: 'elastic', interval: 'some interval', + tags: ['some fake tag 1', 'some fake tag 2'], }); const expected: Partial = { created_by: 'elastic', @@ -988,7 +1005,7 @@ describe('utils', () => { risk_score: 50, rule_id: 'rule-1', severity: 'high', - tags: ['some fake tag'], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', updated_by: 'elastic', @@ -1018,6 +1035,7 @@ describe('utils', () => { createdBy: 'elastic', updatedBy: 'elastic', interval: 'some interval', + tags: ['some fake tag 1', 'some fake tag 2'], }); const expected: Partial = { created_by: 'elastic', @@ -1038,7 +1056,7 @@ describe('utils', () => { risk_score: 50, rule_id: 'rule-1', severity: 'high', - tags: ['some fake tag'], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', updated_by: 'elastic', @@ -1057,6 +1075,7 @@ describe('utils', () => { createdBy: 'elastic', updatedBy: 'elastic', interval: 'some interval', + tags: ['some fake tag 1', 'some fake tag 2'], }); const expected: Partial = { created_by: 'elastic', @@ -1077,7 +1096,7 @@ describe('utils', () => { risk_score: 50, rule_id: 'rule-1', severity: 'high', - tags: ['some fake tag'], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', updated_by: 'elastic', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index a7668f47b614c..1787aa3a3081b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -26,6 +26,7 @@ interface BuildRuleParams { createdBy: string; updatedBy: string; interval: string; + tags: string[]; } export const buildRule = ({ @@ -36,6 +37,7 @@ export const buildRule = ({ createdBy, updatedBy, interval, + tags, }: BuildRuleParams): Partial => { return pickBy((value: unknown) => value != null, { id, @@ -56,7 +58,7 @@ export const buildRule = ({ query: ruleParams.query, references: ruleParams.references, severity: ruleParams.severity, - tags: ruleParams.tags, + tags, type: ruleParams.type, to: ruleParams.to, enabled, @@ -94,6 +96,7 @@ interface BuildBulkBodyParams { updatedBy: string; interval: string; enabled: boolean; + tags: string[]; } export const buildEventTypeSignal = (doc: SignalSourceHit): object => { @@ -114,6 +117,7 @@ export const buildBulkBody = ({ updatedBy, interval, enabled, + tags, }: BuildBulkBodyParams): SignalHit => { const rule = buildRule({ ruleParams, @@ -123,6 +127,7 @@ export const buildBulkBody = ({ createdBy, updatedBy, interval, + tags, }); const signal = buildSignal(doc, rule); const event = buildEventTypeSignal(doc); @@ -147,6 +152,7 @@ interface SingleBulkCreateParams { updatedBy: string; interval: string; enabled: boolean; + tags: string[]; } export const generateId = ( @@ -172,6 +178,7 @@ export const singleBulkCreate = async ({ updatedBy, interval, enabled, + tags, }: SingleBulkCreateParams): Promise => { if (someResult.hits.hits.length === 0) { return true; @@ -197,7 +204,7 @@ export const singleBulkCreate = async ({ ), }, }, - buildBulkBody({ doc, ruleParams, id, name, createdBy, updatedBy, interval, enabled }), + buildBulkBody({ doc, ruleParams, id, name, createdBy, updatedBy, interval, enabled, tags }), ]); const time1 = performance.now(); const firstResult: BulkResponse = await services.callCluster('bulk', { @@ -291,6 +298,7 @@ interface SearchAfterAndBulkCreateParams { enabled: boolean; pageSize: number; filter: unknown; + tags: string[]; } // search_after through documents and re-index using bulk endpoint. @@ -308,6 +316,7 @@ export const searchAfterAndBulkCreate = async ({ interval, enabled, pageSize, + tags, }: SearchAfterAndBulkCreateParams): Promise => { if (someResult.hits.hits.length === 0) { return true; @@ -326,6 +335,7 @@ export const searchAfterAndBulkCreate = async ({ updatedBy, interval, enabled, + tags, }); const totalHits = typeof someResult.hits.total === 'number' ? someResult.hits.total : someResult.hits.total.value; @@ -385,6 +395,7 @@ export const searchAfterAndBulkCreate = async ({ updatedBy, interval, enabled, + tags, }); logger.debug('finished next bulk index'); } catch (exc) { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts index 0d15fa1faa78f..6df4174e628b3 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts @@ -52,7 +52,7 @@ export const transformAlertToRule = (alert: RuleAlertType): Partial