[Detections][EQL] EQL rule execution in detection engine (#77419)

* First draft of EQL rules in detection engine

* Reorganize functions to separate files

* Start adding eventCategoryOverride option for EQL rules

* Add building block alerts for each event within sequence

* Use eql instead of eql_query for rule type

* Remove unused imports

* Fix tests

* Add basic tests for buildEqlSearchRequest

* Add rulesSchema tests for eql

* Add buildSignalFromSequence test

* Add threat rule fields to buildRuleWithoutOverrides

* Fix buildSignalFromSequence typecheck error

* Add more tests

* Add tests for wrapBuildingBlock and generateSignalId

* Use isEqlRule function and fix import error

* delete frank

* Move sequence interface to types.ts

* Fix import

* Remove EQL execution placeholder, add back language to eql rule type

* allow no indices for eql search

* Fix unit tests for language update

* Fix buildEqlSearchRequest tests

* Replace signal.child with signal.group

* remove unused import

* Move sequence signal group building to separate testable function

* Unbork the merge conflict resolution

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { getQueryFilter, buildExceptionFilter } from './get_query_filter';
+import { getQueryFilter, buildExceptionFilter, buildEqlSearchRequest } from './get_query_filter';
 import { Filter, EsQueryConfig } from 'src/plugins/data/public';
 import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock';
 
@@ -1085,4 +1085,138 @@ describe('get_filter', () => {
       });
     });
   });
+
+  describe('buildEqlSearchRequest', () => {
+    test('should build a basic request with time range', () => {
+      const request = buildEqlSearchRequest(
+        'process where true',
+        ['testindex1', 'testindex2'],
+        'now-5m',
+        'now',
+        100,
+        undefined,
+        [],
+        undefined
+      );
+      expect(request).toEqual({
+        method: 'POST',
+        path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`,
+        body: {
+          size: 100,
+          query: 'process where true',
+          filter: {
+            range: {
+              '@timestamp': {
+                gte: 'now-5m',
+                lte: 'now',
+              },
+            },
+          },
+        },
+      });
+    });
+
+    test('should build a request with timestamp and event category overrides', () => {
+      const request = buildEqlSearchRequest(
+        'process where true',
+        ['testindex1', 'testindex2'],
+        'now-5m',
+        'now',
+        100,
+        'event.ingested',
+        [],
+        'event.other_category'
+      );
+      expect(request).toEqual({
+        method: 'POST',
+        path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`,
+        event_category_field: 'event.other_category',
+        body: {
+          size: 100,
+          query: 'process where true',
+          filter: {
+            range: {
+              'event.ingested': {
+                gte: 'now-5m',
+                lte: 'now',
+              },
+            },
+          },
+        },
+      });
+    });
+
+    test('should build a request with exceptions', () => {
+      const request = buildEqlSearchRequest(
+        'process where true',
+        ['testindex1', 'testindex2'],
+        'now-5m',
+        'now',
+        100,
+        undefined,
+        [getExceptionListItemSchemaMock()],
+        undefined
+      );
+      expect(request).toEqual({
+        method: 'POST',
+        path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`,
+        body: {
+          size: 100,
+          query: 'process where true',
+          filter: {
+            range: {
+              '@timestamp': {
+                gte: 'now-5m',
+                lte: 'now',
+              },
+            },
+            bool: {
+              must_not: {
+                bool: {
+                  should: [
+                    {
+                      bool: {
+                        filter: [
+                          {
+                            nested: {
+                              path: 'some.parentField',
+                              query: {
+                                bool: {
+                                  minimum_should_match: 1,
+                                  should: [
+                                    {
+                                      match_phrase: {
+                                        'some.parentField.nested.field': 'some value',
+                                      },
+                                    },
+                                  ],
+                                },
+                              },
+                              score_mode: 'none',
+                            },
+                          },
+                          {
+                            bool: {
+                              minimum_should_match: 1,
+                              should: [
+                                {
+                                  match_phrase: {
+                                    'some.not.nested.field': 'some value',
+                                  },
+                                },
+                              ],
+                            },
+                          },
+                        ],
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+          },
+        },
+      });
+    });
+  });
 });

x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts

@@ -17,7 +17,12 @@ import {
   CreateExceptionListItemSchema,
 } from '../../../lists/common/schemas';
 import { buildExceptionListQueries } from './build_exceptions_query';
-import { Query as QueryString, Language, Index } from './schemas/common/schemas';
+import {
+  Query as QueryString,
+  Language,
+  Index,
+  TimestampOverrideOrUndefined,
+} from './schemas/common/schemas';
 
 export const getQueryFilter = (
   query: QueryString,
@@ -67,6 +72,78 @@ export const getQueryFilter = (
   return buildEsQuery(indexPattern, initialQuery, enabledFilters, config);
 };
 
+interface EqlSearchRequest {
+  method: string;
+  path: string;
+  body: object;
+  event_category_field?: string;
+}
+
+export const buildEqlSearchRequest = (
+  query: string,
+  index: string[],
+  from: string,
+  to: string,
+  size: number,
+  timestampOverride: TimestampOverrideOrUndefined,
+  exceptionLists: ExceptionListItemSchema[],
+  eventCategoryOverride: string | undefined
+): EqlSearchRequest => {
+  const timestamp = timestampOverride ?? '@timestamp';
+  const indexPattern: IIndexPattern = {
+    fields: [],
+    title: index.join(),
+  };
+  const config: EsQueryConfig = {
+    allowLeadingWildcards: true,
+    queryStringOptions: { analyze_wildcard: true },
+    ignoreFilterIfFieldNotInIndex: false,
+    dateFormatTZ: 'Zulu',
+  };
+  const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists });
+  let exceptionFilter: Filter | undefined;
+  if (exceptionQueries.length > 0) {
+    // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
+    // allowing us to make 1024-item chunks of exception list items.
+    // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
+    // very conservative value.
+    exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024);
+  }
+  const indexString = index.join();
+  const baseRequest = {
+    method: 'POST',
+    path: `/${indexString}/_eql/search?allow_no_indices=true`,
+    body: {
+      size,
+      query,
+      filter: {
+        range: {
+          [timestamp]: {
+            gte: from,
+            lte: to,
+          },
+        },
+        bool:
+          exceptionFilter !== undefined
+            ? {
+                must_not: {
+                  bool: exceptionFilter?.query.bool,
+                },
+              }
+            : undefined,
+      },
+    },
+  };
+  if (eventCategoryOverride) {
+    return {
+      ...baseRequest,
+      event_category_field: eventCategoryOverride,
+    };
+  } else {
+    return baseRequest;
+  }
+};
+
 export const buildExceptionFilter = (
   exceptionQueries: Query[],
   indexPattern: IIndexPattern,

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -40,6 +40,12 @@ export type Enabled = t.TypeOf<typeof enabled>;
 export const enabledOrUndefined = t.union([enabled, t.undefined]);
 export type EnabledOrUndefined = t.TypeOf<typeof enabledOrUndefined>;
 
+export const event_category_override = t.string;
+export type EventCategoryOverride = t.TypeOf<typeof event_category_override>;
+
+export const eventCategoryOverrideOrUndefined = t.union([event_category_override, t.undefined]);
+export type EventCategoryOverrideOrUndefined = t.TypeOf<typeof eventCategoryOverrideOrUndefined>;
+
 export const false_positives = t.array(t.string);
 export type FalsePositives = t.TypeOf<typeof false_positives>;
 

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -44,6 +44,7 @@ import {
   Author,
   RiskScoreMapping,
   SeverityMapping,
+  event_category_override,
 } from '../common/schemas';
 import {
   threat_index,
@@ -96,6 +97,7 @@ export const addPrepackagedRulesSchema = t.intersection([
       author: DefaultStringArray, // defaults to empty array of strings if not set during decode
       building_block_type, // defaults to undefined if not set during decode
       enabled: DefaultBooleanFalse, // defaults to false if not set during decode
+      event_category_override, // defaults to "undefined" if not set during decode
       false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts

@@ -45,6 +45,7 @@ import {
   Author,
   RiskScoreMapping,
   SeverityMapping,
+  event_category_override,
 } from '../common/schemas';
 import {
   threat_index,
@@ -88,6 +89,7 @@ export const createRulesSchema = t.intersection([
       author: DefaultStringArray, // defaults to empty array of strings if not set during decode
       building_block_type, // defaults to undefined if not set during decode
       enabled: DefaultBooleanTrue, // defaults to true if not set during decode
+      event_category_override, // defaults to "undefined" if not set during decode
       false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts

@@ -51,6 +51,7 @@ import {
   Author,
   RiskScoreMapping,
   SeverityMapping,
+  event_category_override,
 } from '../common/schemas';
 import {
   threat_index,
@@ -107,6 +108,7 @@ export const importRulesSchema = t.intersection([
       author: DefaultStringArray, // defaults to empty array of strings if not set during decode
       building_block_type, // defaults to undefined if not set during decode
       enabled: DefaultBooleanTrue, // defaults to true if not set during decode
+      event_category_override, // defaults to "undefined" if not set during decode
       false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts

@@ -46,6 +46,7 @@ import {
   timestamp_override,
   risk_score_mapping,
   severity_mapping,
+  event_category_override,
 } from '../common/schemas';
 import { listArrayOrUndefined } from '../types/lists';
 
@@ -65,6 +66,7 @@ export const patchRulesSchema = t.exact(
     actions,
     anomaly_threshold,
     enabled,
+    event_category_override,
     false_positives,
     filters,
     from,

x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts

@@ -47,6 +47,7 @@ import {
   Author,
   RiskScoreMapping,
   SeverityMapping,
+  event_category_override,
 } from '../common/schemas';
 
 import {
@@ -90,6 +91,7 @@ export const updateRulesSchema = t.intersection([
       author: DefaultStringArray, // defaults to empty array of strings if not set during decode
       building_block_type, // defaults to undefined if not set during decode
       enabled: DefaultBooleanTrue, // defaults to true if not set during decode
+      event_category_override,
       false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts

@@ -52,7 +52,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem
   severity: 'high',
   severity_mapping: [],
   updated_by: 'elastic_kibana',
-  tags: [],
+  tags: ['some fake tag 1', 'some fake tag 2'],
   to: 'now',
   type: 'query',
   threat: [],
@@ -61,7 +61,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem
   status_date: '2020-02-22T16:47:50.047Z',
   last_success_at: '2020-02-22T16:47:50.047Z',
   last_success_message: 'succeeded',
-  output_index: '.siem-signals-hassanabad-frank-default',
+  output_index: '.siem-signals-default',
   max_signals: 100,
   risk_score: 55,
   risk_score_mapping: [],
@@ -110,3 +110,12 @@ export const getThreatMatchingSchemaMock = (anchorDate: string = ANCHOR_DATE): R
     ],
   };
 };
+
+export const getRulesEqlSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => {
+  return {
+    ...getRulesSchemaMock(anchorDate),
+    language: 'eql',
+    type: 'eql',
+    query: 'process where true',
+  };
+};