diff --git a/packages/kbn-es/src/serverless_resources/project_roles/security/roles.yml b/packages/kbn-es/src/serverless_resources/project_roles/security/roles.yml index e47cc78eadc33..3c118688f6429 100644 --- a/packages/kbn-es/src/serverless_resources/project_roles/security/roles.yml +++ b/packages/kbn-es/src/serverless_resources/project_roles/security/roles.yml @@ -35,6 +35,7 @@ viewer: - '.fleet-actions*' - 'risk-score.risk-score-*' - '.asset-criticality.asset-criticality-*' + - '.ml-anomalies-*' privileges: - read applications: @@ -100,6 +101,10 @@ editor: - 'read' - 'write' allow_restricted_indices: false + - names: + - '.ml-anomalies-*' + privileges: + - read applications: - application: 'kibana-.kibana' privileges: @@ -154,6 +159,7 @@ t1_analyst: - '.fleet-actions*' - risk-score.risk-score-* - .asset-criticality.asset-criticality-* + - '.ml-anomalies-*' privileges: - read applications: @@ -201,6 +207,7 @@ t2_analyst: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - '.ml-anomalies-*' privileges: - read - names: @@ -262,6 +269,7 @@ t3_analyst: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - '.ml-anomalies-*' privileges: - read applications: @@ -331,6 +339,7 @@ threat_intelligence_analyst: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - '.ml-anomalies-*' privileges: - read applications: @@ -389,6 +398,7 @@ rule_author: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - '.ml-anomalies-*' privileges: - read applications: @@ -453,6 +463,7 @@ soc_manager: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - '.ml-anomalies-*' privileges: - read applications: @@ -513,6 +524,7 @@ detections_admin: - metrics-endpoint.metadata_current_* - .fleet-agents* - .fleet-actions* + - '.ml-anomalies-*' privileges: - read - names: @@ -570,6 +582,10 @@ platform_engineer: privileges: - read - write + - names: + - '.ml-anomalies-*' + privileges: + - read applications: - application: 'kibana-.kibana' privileges: @@ -620,6 +636,7 @@ endpoint_operations_analyst: - .lists* - .items* - risk-score.risk-score-* + - '.ml-anomalies-*' privileges: - read - names: @@ -710,6 +727,10 @@ endpoint_policy_manager: - read - write - manage + - names: + - '.ml-anomalies-*' + privileges: + - read applications: - application: 'kibana-.kibana' privileges: diff --git a/x-pack/plugins/alerting/server/integration_tests/__snapshots__/serverless_upgrade_and_rollback_checks.test.ts.snap b/x-pack/plugins/alerting/server/integration_tests/__snapshots__/serverless_upgrade_and_rollback_checks.test.ts.snap index 456b7e00dd1ed..932daa1fed69d 100644 --- a/x-pack/plugins/alerting/server/integration_tests/__snapshots__/serverless_upgrade_and_rollback_checks.test.ts.snap +++ b/x-pack/plugins/alerting/server/integration_tests/__snapshots__/serverless_upgrade_and_rollback_checks.test.ts.snap @@ -7127,6 +7127,52 @@ Object { }, Object { "properties": Object { + "alertSuppression": Object { + "additionalProperties": false, + "properties": Object { + "duration": Object { + "additionalProperties": false, + "properties": Object { + "unit": Object { + "enum": Array [ + "s", + "m", + "h", + ], + "type": "string", + }, + "value": Object { + "minimum": 1, + "type": "integer", + }, + }, + "required": Array [ + "value", + "unit", + ], + "type": "object", + }, + "groupBy": Object { + "items": Object { + "type": "string", + }, + "maxItems": 3, + "minItems": 1, + "type": "array", + }, + "missingFieldsStrategy": Object { + "enum": Array [ + "doNotSuppress", + "suppress", + ], + "type": "string", + }, + }, + "required": Array [ + "groupBy", + ], + "type": "object", + }, "anomalyThreshold": Object { "minimum": 0, "type": "integer", diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts index b7435c7dd86e8..a22886b287c7f 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts @@ -1272,6 +1272,7 @@ describe('rules schema', () => { { ruleType: 'saved_query', ruleMock: getCreateSavedQueryRulesSchemaMock() }, { ruleType: 'eql', ruleMock: getCreateEqlRuleSchemaMock() }, { ruleType: 'new_terms', ruleMock: getCreateNewTermsRulesSchemaMock() }, + { ruleType: 'machine_learning', ruleMock: getCreateMachineLearningRulesSchemaMock() }, ]; cases.forEach(({ ruleType, ruleMock }) => { diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index 9bb1b26fafd95..83bf6778ec3e3 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -468,14 +468,25 @@ export const MachineLearningRuleRequiredFields = z.object({ machine_learning_job_id: MachineLearningJobId, }); +export type MachineLearningRuleOptionalFields = z.infer; +export const MachineLearningRuleOptionalFields = z.object({ + alert_suppression: AlertSuppression.optional(), +}); + export type MachineLearningRulePatchFields = z.infer; -export const MachineLearningRulePatchFields = MachineLearningRuleRequiredFields.partial(); +export const MachineLearningRulePatchFields = MachineLearningRuleRequiredFields.partial().merge( + MachineLearningRuleOptionalFields +); export type MachineLearningRuleResponseFields = z.infer; -export const MachineLearningRuleResponseFields = MachineLearningRuleRequiredFields; +export const MachineLearningRuleResponseFields = MachineLearningRuleRequiredFields.merge( + MachineLearningRuleOptionalFields +); export type MachineLearningRuleCreateFields = z.infer; -export const MachineLearningRuleCreateFields = MachineLearningRuleRequiredFields; +export const MachineLearningRuleCreateFields = MachineLearningRuleRequiredFields.merge( + MachineLearningRuleOptionalFields +); export type MachineLearningRule = z.infer; export const MachineLearningRule = SharedResponseProps.merge(MachineLearningRuleResponseFields); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml index de424af505c1f..4ade72c15fbb9 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml @@ -686,18 +686,27 @@ components: - machine_learning_job_id - anomaly_threshold + MachineLearningRuleOptionalFields: + type: object + properties: + alert_suppression: + $ref: './common_attributes.schema.yaml#/components/schemas/AlertSuppression' + MachineLearningRulePatchFields: allOf: - $ref: '#/components/schemas/MachineLearningRuleRequiredFields' x-modify: partial + - $ref: '#/components/schemas/MachineLearningRuleOptionalFields' MachineLearningRuleResponseFields: allOf: - $ref: '#/components/schemas/MachineLearningRuleRequiredFields' + - $ref: '#/components/schemas/MachineLearningRuleOptionalFields' MachineLearningRuleCreateFields: allOf: - $ref: '#/components/schemas/MachineLearningRuleRequiredFields' + - $ref: '#/components/schemas/MachineLearningRuleOptionalFields' MachineLearningRule: allOf: diff --git a/x-pack/plugins/security_solution/common/detection_engine/constants.ts b/x-pack/plugins/security_solution/common/detection_engine/constants.ts index 54c81cf93568f..8e06f46f1f46d 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/constants.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/constants.ts @@ -47,6 +47,7 @@ export const SUPPRESSIBLE_ALERT_RULES: Type[] = [ 'new_terms', 'threat_match', 'eql', + 'machine_learning', ]; export const SUPPRESSIBLE_ALERT_RULES_GA: Type[] = ['saved_query', 'query']; diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts index 2e5ac39936fa3..a4db006a67463 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -236,9 +236,7 @@ describe('Alert Suppression Rules', () => { expect(isSuppressibleAlertRule('threat_match')).toBe(true); expect(isSuppressibleAlertRule('new_terms')).toBe(true); expect(isSuppressibleAlertRule('eql')).toBe(true); - - // Rule types that don't support alert suppression: - expect(isSuppressibleAlertRule('machine_learning')).toBe(false); + expect(isSuppressibleAlertRule('machine_learning')).toBe(true); }); test('should return false for an unknown rule type', () => { @@ -273,9 +271,7 @@ describe('Alert Suppression Rules', () => { expect(isSuppressionRuleConfiguredWithDuration('threat_match')).toBe(true); expect(isSuppressionRuleConfiguredWithDuration('new_terms')).toBe(true); expect(isSuppressionRuleConfiguredWithDuration('eql')).toBe(true); - - // Rule types that don't support alert suppression: - expect(isSuppressionRuleConfiguredWithDuration('machine_learning')).toBe(false); + expect(isSuppressionRuleConfiguredWithDuration('machine_learning')).toBe(true); }); test('should return false for an unknown rule type', () => { @@ -294,9 +290,7 @@ describe('Alert Suppression Rules', () => { expect(isSuppressionRuleConfiguredWithGroupBy('threat_match')).toBe(true); expect(isSuppressionRuleConfiguredWithGroupBy('new_terms')).toBe(true); expect(isSuppressionRuleConfiguredWithGroupBy('eql')).toBe(true); - - // Rule types that don't support alert suppression: - expect(isSuppressionRuleConfiguredWithGroupBy('machine_learning')).toBe(false); + expect(isSuppressionRuleConfiguredWithGroupBy('machine_learning')).toBe(true); }); test('should return false for a threshold rule type', () => { @@ -320,9 +314,7 @@ describe('Alert Suppression Rules', () => { expect(isSuppressionRuleConfiguredWithMissingFields('threat_match')).toBe(true); expect(isSuppressionRuleConfiguredWithMissingFields('new_terms')).toBe(true); expect(isSuppressionRuleConfiguredWithMissingFields('eql')).toBe(true); - - // Rule types that don't support alert suppression: - expect(isSuppressionRuleConfiguredWithMissingFields('machine_learning')).toBe(false); + expect(isSuppressionRuleConfiguredWithMissingFields('machine_learning')).toBe(true); }); test('should return false for a threshold rule type', () => { diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 0a7558515226f..66b5f4bd948a1 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -175,6 +175,11 @@ export const allowedExperimentalValues = Object.freeze({ */ riskEnginePrivilegesRouteEnabled: true, + /** + * Enables alerts suppression for machine learning rules + */ + alertSuppressionForMachineLearningRuleEnabled: false, + /** * Enables experimental Experimental S1 integration data to be available in Analyzer */ diff --git a/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_ml_rule_config.ts b/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_ml_rule_config.ts new file mode 100644 index 0000000000000..86551ad64b43a --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_ml_rule_config.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMemo } from 'react'; +import type { DataViewFieldBase } from '@kbn/es-query'; + +import { getTermsAggregationFields } from '../../../../detection_engine/rule_creation_ui/components/step_define_rule/utils'; +import { useRuleFields } from '../../../../detection_engine/rule_management/logic/use_rule_fields'; +import type { BrowserField } from '../../../containers/source'; +import { useMlCapabilities } from './use_ml_capabilities'; +import { useMlRuleValidations } from './use_ml_rule_validations'; +import { hasMlAdminPermissions } from '../../../../../common/machine_learning/has_ml_admin_permissions'; +import { hasMlLicense } from '../../../../../common/machine_learning/has_ml_license'; + +export interface UseMlRuleConfigReturn { + hasMlAdminPermissions: boolean; + hasMlLicense: boolean; + mlFields: DataViewFieldBase[]; + mlFieldsLoading: boolean; + mlSuppressionFields: BrowserField[]; + noMlJobsStarted: boolean; + someMlJobsStarted: boolean; +} + +/** + * This hook is used to retrieve the various configurations and status needed for creating/editing an ML Rule in the Detection Engine UI. It composes several other ML hooks. + * + * @param machineLearningJobId The ID(s) of the ML job to retrieve the configuration for + * + * @returns {UseMlRuleConfigReturn} An object containing the various configurations and statuses needed for creating/editing an ML Rule + * + */ +export const useMLRuleConfig = ({ + machineLearningJobId, +}: { + machineLearningJobId: string[]; +}): UseMlRuleConfigReturn => { + const mlCapabilities = useMlCapabilities(); + const { someJobsStarted: someMlJobsStarted, noJobsStarted: noMlJobsStarted } = + useMlRuleValidations({ machineLearningJobId }); + const { loading: mlFieldsLoading, fields: mlFields } = useRuleFields({ + machineLearningJobId, + }); + const mlSuppressionFields = useMemo( + () => getTermsAggregationFields(mlFields as BrowserField[]), + [mlFields] + ); + + return { + hasMlAdminPermissions: hasMlAdminPermissions(mlCapabilities), + hasMlLicense: hasMlLicense(mlCapabilities), + mlFields, + mlFieldsLoading, + mlSuppressionFields, + noMlJobsStarted, + someMlJobsStarted, + }; +}; diff --git a/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_ml_rule_validations.test.ts b/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_ml_rule_validations.test.ts new file mode 100644 index 0000000000000..6f14d6fe2a736 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_ml_rule_validations.test.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { TestProviders } from '../../../mock'; +import { buildMockJobsSummary, getJobsSummaryResponseMock } from '../../ml_popover/api.mock'; +import { useInstalledSecurityJobs } from './use_installed_security_jobs'; + +import { useMlRuleValidations } from './use_ml_rule_validations'; + +jest.mock('./use_installed_security_jobs'); + +describe('useMlRuleValidations', () => { + const machineLearningJobId = ['test_job', 'test_job_2']; + + beforeEach(() => { + (useInstalledSecurityJobs as jest.Mock).mockReturnValue({ + loading: true, + jobs: [], + }); + }); + + it('returns loading state from inner hook', () => { + const { result, rerender } = renderHook(() => useMlRuleValidations({ machineLearningJobId }), { + wrapper: TestProviders, + }); + expect(result.current).toEqual(expect.objectContaining({ loading: true })); + + (useInstalledSecurityJobs as jest.Mock).mockReturnValueOnce({ + loading: false, + jobs: [], + }); + + rerender(); + + expect(result.current).toEqual(expect.objectContaining({ loading: false })); + }); + + it('returns no jobs started when no jobs are started', () => { + const { result } = renderHook(() => useMlRuleValidations({ machineLearningJobId }), { + wrapper: TestProviders, + }); + + expect(result.current).toEqual( + expect.objectContaining({ noJobsStarted: true, someJobsStarted: false }) + ); + }); + + it('returns some jobs started when some jobs are started', () => { + (useInstalledSecurityJobs as jest.Mock).mockReturnValueOnce({ + loading: false, + jobs: getJobsSummaryResponseMock([ + buildMockJobsSummary({ + id: machineLearningJobId[0], + jobState: 'opened', + datafeedState: 'started', + }), + buildMockJobsSummary({ + id: machineLearningJobId[1], + }), + ]), + }); + + const { result } = renderHook(() => useMlRuleValidations({ machineLearningJobId }), { + wrapper: TestProviders, + }); + + expect(result.current).toEqual( + expect.objectContaining({ noJobsStarted: false, someJobsStarted: true }) + ); + }); + + it('returns neither "no jobs started" nor "some jobs started" when all jobs are started', () => { + (useInstalledSecurityJobs as jest.Mock).mockReturnValueOnce({ + loading: false, + jobs: getJobsSummaryResponseMock([ + buildMockJobsSummary({ + id: machineLearningJobId[0], + jobState: 'opened', + datafeedState: 'started', + }), + buildMockJobsSummary({ + id: machineLearningJobId[1], + jobState: 'opened', + datafeedState: 'started', + }), + ]), + }); + + const { result } = renderHook(() => useMlRuleValidations({ machineLearningJobId }), { + wrapper: TestProviders, + }); + + expect(result.current).toEqual( + expect.objectContaining({ noJobsStarted: false, someJobsStarted: false }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_ml_rule_validations.ts b/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_ml_rule_validations.ts new file mode 100644 index 0000000000000..81897c5d29b82 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_ml_rule_validations.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isJobStarted } from '../../../../../common/machine_learning/helpers'; +import { useInstalledSecurityJobs } from './use_installed_security_jobs'; + +export interface UseMlRuleValidationsParams { + machineLearningJobId: string[] | undefined; +} + +export interface UseMlRuleValidationsReturn { + loading: boolean; + noJobsStarted: boolean; + someJobsStarted: boolean; +} + +/** + * Hook to encapsulate some of our validation checks for ML rules. + * + * @param machineLearningJobId the ML Job IDs of the rule + * @returns validation state about the rule, relative to its ML jobs. + */ +export const useMlRuleValidations = ({ + machineLearningJobId, +}: UseMlRuleValidationsParams): UseMlRuleValidationsReturn => { + const { jobs: installedJobs, loading } = useInstalledSecurityJobs(); + const ruleMlJobs = installedJobs.filter((installedJob) => + (machineLearningJobId ?? []).includes(installedJob.id) + ); + const numberOfRuleMlJobsStarted = ruleMlJobs.filter((job) => + isJobStarted(job.jobState, job.datafeedState) + ).length; + const noMlJobsStarted = numberOfRuleMlJobsStarted === 0; + const someMlJobsStarted = !noMlJobsStarted && numberOfRuleMlJobsStarted !== ruleMlJobs.length; + + return { loading, noJobsStarted: noMlJobsStarted, someJobsStarted: someMlJobsStarted }; +}; diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts b/x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts index 2000db1807cbf..fdd9d66ebaf90 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts @@ -100,6 +100,16 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [ }, ]; +export const getJobsSummaryResponseMock = (additionalJobs: MlSummaryJob[]): MlSummaryJob[] => [ + ...mockJobsSummaryResponse, + ...additionalJobs, +]; + +export const buildMockJobsSummary = (overrides: Partial): MlSummaryJob => ({ + ...mockJobsSummaryResponse[0], + ...overrides, +}); + export const mockGetModuleResponse: Module[] = [ { id: 'security_linux_v3', diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx b/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx index 8d0b63d8b32fe..567d7e038b5ad 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx @@ -6,6 +6,7 @@ */ import type { MlSummaryJob } from '@kbn/ml-plugin/public'; +import { isSecurityJob } from '../../../../../common/machine_learning/is_security_job'; import type { AugmentedSecurityJobFields, Module, @@ -111,13 +112,11 @@ export const getInstalledJobs = ( moduleJobs: SecurityJob[], compatibleModuleIds: string[] ): SecurityJob[] => - jobSummaryData - .filter(({ groups }) => groups.includes('siem') || groups.includes('security')) - .map((jobSummary) => ({ - ...jobSummary, - ...getAugmentedFields(jobSummary.id, moduleJobs, compatibleModuleIds), - isInstalled: true, - })); + jobSummaryData.filter(isSecurityJob).map((jobSummary) => ({ + ...jobSummary, + ...getAugmentedFields(jobSummary.id, moduleJobs, compatibleModuleIds), + isInstalled: true, + })); /** * Combines installed jobs + moduleSecurityJobs that don't overlap and sorts by name asc diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx index 8695041697120..f5a7e39634359 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx @@ -14,7 +14,6 @@ import { buildListItems, getDescriptionItem, } from '.'; -import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import { FilterManager, UI_SETTINGS } from '@kbn/data-plugin/public'; import type { Filter } from '@kbn/es-query'; @@ -575,7 +574,6 @@ describe('description_step', () => { }); describe('alert suppression', () => { - const ruleTypesWithoutSuppression: Type[] = ['machine_learning']; const suppressionFields = { groupByDuration: { unit: 'm', @@ -587,23 +585,6 @@ describe('description_step', () => { suppressionMissingFields: 'suppress', }; describe('groupByDuration', () => { - ruleTypesWithoutSuppression.forEach((ruleType) => { - test(`should be empty if rule is ${ruleType}`, () => { - const result: ListItems[] = getDescriptionItem( - 'groupByDuration', - 'label', - { - ruleType, - ...suppressionFields, - }, - mockFilterManager, - mockLicenseService - ); - - expect(result).toEqual([]); - }); - }); - ['query', 'saved_query'].forEach((ruleType) => { test(`should be empty if groupByFields empty for ${ruleType} rule`, () => { const result: ListItems[] = getDescriptionItem( @@ -686,22 +667,21 @@ describe('description_step', () => { }); describe('groupByFields', () => { - [...ruleTypesWithoutSuppression, 'threshold'].forEach((ruleType) => { - test(`should be empty if rule is ${ruleType}`, () => { - const result: ListItems[] = getDescriptionItem( - 'groupByFields', - 'label', - { - ruleType, - ...suppressionFields, - }, - mockFilterManager, - mockLicenseService - ); + test(`should be empty if rule type is 'threshold'`, () => { + const result: ListItems[] = getDescriptionItem( + 'groupByFields', + 'label', + { + ruleType: 'threshold', + ...suppressionFields, + }, + mockFilterManager, + mockLicenseService + ); - expect(result).toEqual([]); - }); + expect(result).toEqual([]); }); + ['query', 'saved_query'].forEach((ruleType) => { test(`should return item for ${ruleType} rule`, () => { const result: ListItems[] = getDescriptionItem( @@ -720,22 +700,21 @@ describe('description_step', () => { }); describe('suppressionMissingFields', () => { - [...ruleTypesWithoutSuppression, 'threshold'].forEach((ruleType) => { - test(`should be empty if rule is ${ruleType}`, () => { - const result: ListItems[] = getDescriptionItem( - 'suppressionMissingFields', - 'label', - { - ruleType, - ...suppressionFields, - }, - mockFilterManager, - mockLicenseService - ); + test(`should be empty if rule type is 'threshold'`, () => { + const result: ListItems[] = getDescriptionItem( + 'suppressionMissingFields', + 'label', + { + ruleType: 'threshold', + ...suppressionFields, + }, + mockFilterManager, + mockLicenseService + ); - expect(result).toEqual([]); - }); + expect(result).toEqual([]); }); + ['query', 'saved_query'].forEach((ruleType) => { test(`should return item for ${ruleType} rule`, () => { const result: ListItems[] = getDescriptionItem( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 98fe3bae27f5e..df6152c7069df 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -36,9 +36,6 @@ import { FormattedMessage } from '@kbn/i18n-react'; import { useSetFieldValueWithCallback } from '../../../../common/utils/use_set_field_value_cb'; import { useRuleFromTimeline } from '../../../../detections/containers/detection_engine/rules/use_rule_from_timeline'; import { isMlRule } from '../../../../../common/machine_learning/helpers'; -import { hasMlAdminPermissions } from '../../../../../common/machine_learning/has_ml_admin_permissions'; -import { hasMlLicense } from '../../../../../common/machine_learning/has_ml_license'; -import { useMlCapabilities } from '../../../../common/components/ml/hooks/use_ml_capabilities'; import type { EqlOptionsSelected, FieldsEqlOptions } from '../../../../../common/search_strategy'; import { filterRuleFieldsForType, getStepDataDataSource } from '../../pages/rule_creation/helpers'; import type { @@ -105,6 +102,7 @@ import { useAllEsqlRuleFields } from '../../hooks'; import { useAlertSuppression } from '../../../rule_management/logic/use_alert_suppression'; import { AiAssistant } from '../ai_assistant'; import { RelatedIntegrations } from '../../../rule_creation/components/related_integrations'; +import { useMLRuleConfig } from '../../../../common/components/ml/hooks/use_ml_rule_config'; const CommonUseField = getUseField({ component: Field }); @@ -169,41 +167,53 @@ const IntendedRuleTypeEuiFormRow = styled(RuleTypeEuiFormRow)` // eslint-disable-next-line complexity const StepDefineRuleComponent: FC = ({ - isLoading, - isUpdateView = false, - kibanaDataViews, - indicesConfig, - threatIndicesConfig, + browserFields, + dataSourceType, defaultSavedQuery, + enableThresholdSuppression, form, - optionsSelected, - setOptionsSelected, + groupByFields, + index, indexPattern, + indicesConfig, isIndexPatternLoading, - browserFields, + isLoading, isQueryBarValid, + isUpdateView = false, + kibanaDataViews, + optionsSelected, + queryBarSavedId, + queryBarTitle, + ruleType, setIsQueryBarValid, setIsThreatQueryBarValid, - ruleType, - index, - threatIndex, - groupByFields, - dataSourceType, + setOptionsSelected, shouldLoadQueryDynamically, - queryBarTitle, - queryBarSavedId, + threatIndex, + threatIndicesConfig, thresholdFields, - enableThresholdSuppression, }) => { const queryClient = useQueryClient(); const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(ruleType); - const mlCapabilities = useMlCapabilities(); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); const [indexModified, setIndexModified] = useState(false); const [threatIndexModified, setThreatIndexModified] = useState(false); const license = useLicense(); + const [{ machineLearningJobId }] = useFormData({ + form, + watch: ['machineLearningJobId'], + }); + const { + hasMlAdminPermissions, + hasMlLicense, + mlFieldsLoading, + mlSuppressionFields, + noMlJobsStarted, + someMlJobsStarted, + } = useMLRuleConfig({ machineLearningJobId }); + const esqlQueryRef = useRef(undefined); const isAlertSuppressionLicenseValid = license.isAtLeast(MINIMUM_LICENSE_FOR_SUPPRESSION); @@ -474,6 +484,24 @@ const StepDefineRuleComponent: FC = ({ isEqlSequenceQuery(queryBar?.query?.query as string) && groupByFields.length === 0; + const isSuppressionGroupByDisabled = + !isAlertSuppressionLicenseValid || + areSuppressionFieldsDisabledBySequence || + isEsqlSuppressionLoading || + (isMlRule(ruleType) && (noMlJobsStarted || mlFieldsLoading || !mlSuppressionFields.length)); + + const suppressionGroupByDisabledText = areSuppressionFieldsDisabledBySequence + ? i18n.EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP + : isMlRule(ruleType) && noMlJobsStarted + ? i18n.MACHINE_LEARNING_SUPPRESSION_DISABLED_LABEL + : alertSuppressionUpsellingMessage; + + const suppressionGroupByFields = isEsqlRule(ruleType) + ? esqlSuppressionFields + : isMlRule(ruleType) + ? mlSuppressionFields + : termsAggregationFields; + /** * Component that allows selection of suppression intervals disabled: * - if suppression license is not valid(i.e. less than platinum) @@ -868,10 +896,10 @@ const StepDefineRuleComponent: FC = ({ () => ({ describedByIds: ['detectionEngineStepDefineRuleType'], isUpdateView, - hasValidLicense: hasMlLicense(mlCapabilities), - isMlAdmin: hasMlAdminPermissions(mlCapabilities), + hasValidLicense: hasMlLicense, + isMlAdmin: hasMlAdminPermissions, }), - [isUpdateView, mlCapabilities] + [hasMlAdminPermissions, hasMlLicense, isUpdateView] ); return ( @@ -1078,22 +1106,22 @@ const StepDefineRuleComponent: FC = ({ } > - + <> + + {someMlJobsStarted && ( + + {i18n.MACHINE_LEARNING_SUPPRESSION_INCOMPLETE_LABEL} + + )} + >(): (( fields: T ) => T) => { + const isAlertSuppressionForMachineLearningRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForMachineLearningRuleEnabled' + ); const isAlertSuppressionForEsqlRuleEnabled = useIsExperimentalFeatureEnabled( 'alertSuppressionForEsqlRuleEnabled' ); @@ -23,7 +26,8 @@ export const useExperimentalFeatureFieldsTransform = { const isSuppressionDisabled = - isEsqlRule(fields.ruleType) && !isAlertSuppressionForEsqlRuleEnabled; + (isMlRule(fields.ruleType) && !isAlertSuppressionForMachineLearningRuleEnabled) || + (isEsqlRule(fields.ruleType) && !isAlertSuppressionForEsqlRuleEnabled); // reset any alert suppression values hidden behind feature flag if (isSuppressionDisabled) { @@ -38,7 +42,7 @@ export const useExperimentalFeatureFieldsTransform = { expect(result).toEqual(expected); }); + + it('returns suppression fields for machine_learning rules', () => { + const mockStepData: DefineStepRule = { + ...mockData, + ruleType: 'machine_learning', + machineLearningJobId: ['some_jobert_id'], + anomalyThreshold: 44, + groupByFields: ['event.type'], + groupByRadioSelection: GroupByOptions.PerTimePeriod, + groupByDuration: { value: 10, unit: 'm' }, + }; + const result = formatDefineStepData(mockStepData); + + const expected: DefineStepRuleJson = { + machine_learning_job_id: ['some_jobert_id'], + anomaly_threshold: 44, + type: 'machine_learning', + alert_suppression: { + group_by: ['event.type'], + duration: { value: 10, unit: 'm' }, + missing_fields_strategy: 'suppress', + }, + }; + + expect(result).toEqual(expect.objectContaining(expected)); + }); }); describe('formatScheduleStepData', () => { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts index f281b3b6b4a2b..8cda58eeeb541 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts @@ -439,6 +439,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep ? { anomaly_threshold: ruleFields.anomalyThreshold, machine_learning_job_id: ruleFields.machineLearningJobId, + ...alertSuppressionFields, } : isThresholdFields(ruleFields) ? { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx index d12a5ff97d50a..fb00b73e88ffd 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx @@ -37,18 +37,38 @@ describe('useAlertSuppression', () => { expect(result.current.isSuppressionEnabled).toBe(false); }); - it('should return isSuppressionEnabled false if ES|QL Feature Flag is disabled', () => { - const { result } = renderHook(() => useAlertSuppression('esql')); + describe('ML rules', () => { + it('is true if the feature flag is enabled', () => { + jest + .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') + .mockReset() + .mockReturnValue(true); + const { result } = renderHook(() => useAlertSuppression('machine_learning')); - expect(result.current.isSuppressionEnabled).toBe(false); + expect(result.current.isSuppressionEnabled).toBe(true); + }); + + it('is false if the feature flag is disabled', () => { + const { result } = renderHook(() => useAlertSuppression('machine_learning')); + + expect(result.current.isSuppressionEnabled).toBe(false); + }); }); - it('should return isSuppressionEnabled true if ES|QL Feature Flag is enabled', () => { - jest - .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') - .mockImplementation((flag) => flag === 'alertSuppressionForEsqlRuleEnabled'); - const { result } = renderHook(() => useAlertSuppression('esql')); + describe('ES|QL rules', () => { + it('should return isSuppressionEnabled false if ES|QL Feature Flag is disabled', () => { + const { result } = renderHook(() => useAlertSuppression('esql')); + + expect(result.current.isSuppressionEnabled).toBe(false); + }); + + it('should return isSuppressionEnabled true if ES|QL Feature Flag is enabled', () => { + jest + .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') + .mockImplementation((flag) => flag === 'alertSuppressionForEsqlRuleEnabled'); + const { result } = renderHook(() => useAlertSuppression('esql')); - expect(result.current.isSuppressionEnabled).toBe(true); + expect(result.current.isSuppressionEnabled).toBe(true); + }); }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx index 1c9f139633c8c..6d0ecefe8345d 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx @@ -6,7 +6,7 @@ */ import { useCallback } from 'react'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; -import { isSuppressibleAlertRule } from '../../../../common/detection_engine/utils'; +import { isMlRule, isSuppressibleAlertRule } from '../../../../common/detection_engine/utils'; import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; export interface UseAlertSuppressionReturn { @@ -14,6 +14,9 @@ export interface UseAlertSuppressionReturn { } export const useAlertSuppression = (ruleType: Type | undefined): UseAlertSuppressionReturn => { + const isAlertSuppressionForMachineLearningRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForMachineLearningRuleEnabled' + ); const isAlertSuppressionForEsqlRuleEnabled = useIsExperimentalFeatureEnabled( 'alertSuppressionForEsqlRuleEnabled' ); @@ -27,8 +30,16 @@ export const useAlertSuppression = (ruleType: Type | undefined): UseAlertSuppres return isSuppressibleAlertRule(ruleType) && isAlertSuppressionForEsqlRuleEnabled; } + if (isMlRule(ruleType)) { + return isSuppressibleAlertRule(ruleType) && isAlertSuppressionForMachineLearningRuleEnabled; + } + return isSuppressibleAlertRule(ruleType); - }, [ruleType, isAlertSuppressionForEsqlRuleEnabled]); + }, [ + isAlertSuppressionForEsqlRuleEnabled, + isAlertSuppressionForMachineLearningRuleEnabled, + ruleType, + ]); return { isSuppressionEnabled: isSuppressionEnabledForRuleType(), diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_fields.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_fields.ts new file mode 100644 index 0000000000000..c0f34c5502f94 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_fields.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DataViewFieldBase } from '@kbn/es-query'; + +import { useRuleIndices } from './use_rule_indices'; +import { useFetchIndex } from '../../../common/containers/source'; + +interface UseRuleFieldParams { + machineLearningJobId?: string[]; + indexPattern?: string[]; +} + +interface UseRuleFieldsReturn { + loading: boolean; + fields: DataViewFieldBase[]; +} + +export const useRuleFields = ({ + machineLearningJobId, + indexPattern, +}: UseRuleFieldParams): UseRuleFieldsReturn => { + const { ruleIndices } = useRuleIndices(machineLearningJobId, indexPattern); + const [ + loading, + { + indexPatterns: { fields }, + }, + ] = useFetchIndex(ruleIndices); + + return { loading, fields }; +}; diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index c1465be7e67e0..b88ca5ff6ab83 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -30,6 +30,7 @@ import { TIMESTAMP, } from '@kbn/rule-data-utils'; +import type { Type as RuleType } from '@kbn/securitysolution-io-ts-alerting-types'; import { lastValueFrom } from 'rxjs'; import type { EcsSecurityExtension as Ecs } from '@kbn/securitysolution-ecs'; import type { DataTableModel } from '@kbn/securitysolution-data-table'; @@ -42,7 +43,13 @@ import { ALERT_NEW_TERMS, ALERT_RULE_INDICES, } from '../../../../common/field_maps/field_names'; -import { isEqlRule, isEsqlRule } from '../../../../common/detection_engine/utils'; +import { + isEqlRule, + isEsqlRule, + isMlRule, + isNewTermsRule, + isThresholdRule, +} from '../../../../common/detection_engine/utils'; import type { TimelineResult } from '../../../../common/api/timeline'; import { TimelineId } from '../../../../common/types/timeline'; import { TimelineStatus, TimelineType } from '../../../../common/api/timeline'; @@ -266,31 +273,16 @@ export const isEqlAlertWithGroupId = (ecsData: Ecs): boolean => { return isEql && groupId?.length > 0; }; -export const isThresholdAlert = (ecsData: Ecs): boolean => { - const ruleType = getField(ecsData, ALERT_RULE_TYPE); - return ( - ruleType === 'threshold' || - (Array.isArray(ruleType) && ruleType.length > 0 && ruleType[0] === 'threshold') - ); -}; - -export const isEqlAlert = (ecsData: Ecs): boolean => { +const getRuleType = (ecsData: Ecs): RuleType | undefined => { const ruleType = getField(ecsData, ALERT_RULE_TYPE); - return isEqlRule(ruleType) || (Array.isArray(ruleType) && isEqlRule(ruleType[0])); + return Array.isArray(ruleType) ? ruleType[0] : ruleType; }; -export const isEsqlAlert = (ecsData: Ecs): boolean => { - const ruleType = getField(ecsData, ALERT_RULE_TYPE); - return isEsqlRule(ruleType) || (Array.isArray(ruleType) && isEsqlRule(ruleType[0])); -}; - -export const isNewTermsAlert = (ecsData: Ecs): boolean => { - const ruleType = getField(ecsData, ALERT_RULE_TYPE); - return ( - ruleType === 'new_terms' || - (Array.isArray(ruleType) && ruleType.length > 0 && ruleType[0] === 'new_terms') - ); -}; +const isNewTermsAlert = (ecsData: Ecs): boolean => isNewTermsRule(getRuleType(ecsData)); +const isEsqlAlert = (ecsData: Ecs): boolean => isEsqlRule(getRuleType(ecsData)); +const isEqlAlert = (ecsData: Ecs): boolean => isEqlRule(getRuleType(ecsData)); +const isThresholdAlert = (ecsData: Ecs): boolean => isThresholdRule(getRuleType(ecsData)); +const isMlAlert = (ecsData: Ecs): boolean => isMlRule(getRuleType(ecsData)); const isSuppressedAlert = (ecsData: Ecs): boolean => { return getField(ecsData, ALERT_SUPPRESSION_DOCS_COUNT) != null; @@ -1035,7 +1027,12 @@ export const sendAlertToTimelineAction = async ({ getExceptionFilter ); // The Query field should remain unpopulated with the suppressed EQL/ES|QL alert. - } else if (isSuppressedAlert(ecsData) && !isEqlAlert(ecsData) && !isEsqlAlert(ecsData)) { + } else if ( + isSuppressedAlert(ecsData) && + !isEqlAlert(ecsData) && + !isEsqlAlert(ecsData) && + !isMlAlert(ecsData) + ) { return createSuppressedTimeline( ecsData, createTimeline, @@ -1106,7 +1103,12 @@ export const sendAlertToTimelineAction = async ({ } else if (isNewTermsAlert(ecsData)) { return createNewTermsTimeline(ecsData, createTimeline, noteContent, {}, getExceptionFilter); // The Query field should remain unpopulated with the suppressed EQL/ES|QL alert. - } else if (isSuppressedAlert(ecsData) && !isEqlAlert(ecsData) && !isEsqlAlert(ecsData)) { + } else if ( + isSuppressedAlert(ecsData) && + !isEqlAlert(ecsData) && + !isEsqlAlert(ecsData) && + !isMlAlert(ecsData) + ) { return createSuppressedTimeline(ecsData, createTimeline, noteContent, {}, getExceptionFilter); } else { let { dataProviders, filters } = buildTimelineDataProviderOrFilter( diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/serverless/es_serverless_resources/roles.yml b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/serverless/es_serverless_resources/roles.yml index 3bc3320b96026..c94d4a9a31d8e 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/serverless/es_serverless_resources/roles.yml +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/serverless/es_serverless_resources/roles.yml @@ -53,6 +53,7 @@ viewer: - ".fleet-actions*" - "risk-score.risk-score-*" - ".asset-criticality.asset-criticality-*" + - ".ml-anomalies-*" privileges: - read applications: @@ -119,6 +120,10 @@ editor: - "read" - "write" allow_restricted_indices: false + - names: + - ".ml-anomalies-*" + privileges: + - read applications: - application: "kibana-.kibana" privileges: @@ -174,6 +179,7 @@ t1_analyst: - ".fleet-actions*" - risk-score.risk-score-* - .asset-criticality.asset-criticality-* + - ".ml-anomalies-*" privileges: - read applications: @@ -222,6 +228,7 @@ t2_analyst: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read - names: @@ -284,6 +291,7 @@ t3_analyst: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read applications: @@ -349,6 +357,7 @@ threat_intelligence_analyst: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read applications: @@ -408,6 +417,7 @@ rule_author: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read applications: @@ -473,6 +483,7 @@ soc_manager: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read applications: @@ -534,6 +545,7 @@ detections_admin: - metrics-endpoint.metadata_current_* - .fleet-agents* - .fleet-actions* + - ".ml-anomalies-*" privileges: - read - names: @@ -592,6 +604,10 @@ platform_engineer: privileges: - read - write + - names: + - ".ml-anomalies-*" + privileges: + - read applications: - application: "kibana-.kibana" privileges: @@ -643,6 +659,7 @@ endpoint_operations_analyst: - .lists* - .items* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read - names: @@ -711,6 +728,7 @@ endpoint_policy_manager: - packetbeat-* - winlogbeat-* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read - names: diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts index 537d7b6abaf8a..5df02371befa2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts @@ -110,6 +110,51 @@ describe('rule_converters', () => { }); }); + describe('machine learning rules', () => { + test('should accept machine learning params when existing rule type is machine learning', () => { + const patchParams = { + anomaly_threshold: 5, + }; + const rule = getMlRuleParams(); + const patchedParams = patchTypeSpecificSnakeToCamel(patchParams, rule); + expect(patchedParams).toEqual( + expect.objectContaining({ + anomalyThreshold: 5, + }) + ); + }); + + test('should reject invalid machine learning params when existing rule type is machine learning', () => { + const patchParams = { + anomaly_threshold: 'invalid', + } as PatchRuleRequestBody; + const rule = getMlRuleParams(); + expect(() => patchTypeSpecificSnakeToCamel(patchParams, rule)).toThrowError( + 'anomaly_threshold: Expected number, received string' + ); + }); + + it('accepts suppression params', () => { + const patchParams = { + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress' as const, + }, + }; + const rule = getMlRuleParams(); + const patchedParams = patchTypeSpecificSnakeToCamel(patchParams, rule); + + expect(patchedParams).toEqual( + expect.objectContaining({ + alertSuppression: { + groupBy: ['agent.name'], + missingFieldsStrategy: 'suppress', + }, + }) + ); + }); + }); + test('should accept threat match params when existing rule type is threat match', () => { const patchParams = { threat_indicator_path: 'my.indicator', @@ -298,29 +343,6 @@ describe('rule_converters', () => { ); }); - test('should accept machine learning params when existing rule type is machine learning', () => { - const patchParams = { - anomaly_threshold: 5, - }; - const rule = getMlRuleParams(); - const patchedParams = patchTypeSpecificSnakeToCamel(patchParams, rule); - expect(patchedParams).toEqual( - expect.objectContaining({ - anomalyThreshold: 5, - }) - ); - }); - - test('should reject invalid machine learning params when existing rule type is machine learning', () => { - const patchParams = { - anomaly_threshold: 'invalid', - } as PatchRuleRequestBody; - const rule = getMlRuleParams(); - expect(() => patchTypeSpecificSnakeToCamel(patchParams, rule)).toThrowError( - 'anomaly_threshold: Expected number, received string' - ); - }); - test('should accept new terms params when existing rule type is new terms', () => { const patchParams = { new_terms_fields: ['event.new_field'], @@ -344,6 +366,7 @@ describe('rule_converters', () => { ); }); }); + describe('typeSpecificCamelToSnake', () => { describe('EQL', () => { test('should accept EQL params when existing rule type is EQL', () => { @@ -396,6 +419,54 @@ describe('rule_converters', () => { ); }); }); + + describe('machine learning rules', () => { + it('accepts normal params', () => { + const params = { + anomalyThreshold: 74, + machineLearningJobId: ['job-1'], + }; + const ruleParams = { ...getMlRuleParams(), ...params }; + const transformedParams = typeSpecificCamelToSnake(ruleParams); + expect(transformedParams).toEqual( + expect.objectContaining({ + anomaly_threshold: 74, + machine_learning_job_id: ['job-1'], + }) + ); + }); + + it('accepts suppression params', () => { + const params = { + anomalyThreshold: 74, + machineLearningJobId: ['job-1'], + alertSuppression: { + groupBy: ['event.type'], + duration: { + value: 10, + unit: 'm', + } as AlertSuppressionDuration, + missingFieldsStrategy: 'suppress' as AlertSuppressionMissingFieldsStrategy, + }, + }; + const ruleParams = { ...getMlRuleParams(), ...params }; + const transformedParams = typeSpecificCamelToSnake(ruleParams); + expect(transformedParams).toEqual( + expect.objectContaining({ + anomaly_threshold: 74, + machine_learning_job_id: ['job-1'], + alert_suppression: { + group_by: ['event.type'], + duration: { + value: 10, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }) + ); + }); + }); }); describe('commonParamsCamelToSnake', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts index 7aac52dfe52c4..db815f32fb5ed 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts @@ -191,6 +191,7 @@ export const typeSpecificSnakeToCamel = ( type: params.type, anomalyThreshold: params.anomaly_threshold, machineLearningJobId: normalizeMachineLearningJobIds(params.machine_learning_job_id), + alertSuppression: convertAlertSuppressionToCamel(params.alert_suppression), }; } case 'new_terms': { @@ -338,6 +339,8 @@ const patchMachineLearningParams = ( machineLearningJobId: params.machine_learning_job_id ? normalizeMachineLearningJobIds(params.machine_learning_job_id) : existingRule.machineLearningJobId, + alertSuppression: + convertAlertSuppressionToCamel(params.alert_suppression) ?? existingRule.alertSuppression, }; }; @@ -706,6 +709,7 @@ export const typeSpecificCamelToSnake = ( type: params.type, anomaly_threshold: params.anomalyThreshold, machine_learning_job_id: params.machineLearningJobId, + alert_suppression: convertAlertSuppressionToSnake(params.alertSuppression), }; } case 'new_terms': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index 48637e898dda3..b3000edf895dc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -268,6 +268,7 @@ export const MachineLearningSpecificRuleParams = z.object({ type: z.literal('machine_learning'), anomalyThreshold: AnomalyThreshold, machineLearningJobId: z.array(z.string()), + alertSuppression: AlertSuppressionCamel.optional(), }); export type MachineLearningRuleParams = BaseRuleParams & MachineLearningSpecificRuleParams; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts index ca0edac6fca4e..2d38b16e94b5f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts @@ -11,13 +11,15 @@ import { DEFAULT_APP_CATEGORIES } from '@kbn/core-application-common'; import { SERVER_APP_ID } from '../../../../../common/constants'; import { MachineLearningRuleParams } from '../../rule_schema'; +import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active'; import { mlExecutor } from './ml'; -import type { CreateRuleOptions, SecurityAlertType } from '../types'; +import type { CreateRuleOptions, SecurityAlertType, WrapSuppressedHits } from '../types'; +import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts'; export const createMlAlertType = ( createOptions: CreateRuleOptions ): SecurityAlertType => { - const { ml } = createOptions; + const { experimentalFeatures, ml, licensing } = createOptions; return { id: ML_RULE_TYPE_ID, name: 'Machine Learning Rule', @@ -56,11 +58,39 @@ export const createMlAlertType = ( wrapHits, exceptionFilter, unprocessedExceptions, + mergeStrategy, + alertTimestampOverride, + publicBaseUrl, + alertWithSuppression, + primaryTimestamp, + secondaryTimestamp, }, services, + spaceId, state, } = execOptions; + const isAlertSuppressionActive = await getIsAlertSuppressionActive({ + alertSuppression: completeRule.ruleParams.alertSuppression, + isFeatureDisabled: !experimentalFeatures.alertSuppressionForMachineLearningRuleEnabled, + licensing, + }); + + const wrapSuppressedHits: WrapSuppressedHits = (events, buildReasonMessage) => + wrapSuppressedAlerts({ + events, + spaceId, + completeRule, + mergeStrategy, + indicesToQuery: [], + buildReasonMessage, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, + }); + const result = await mlExecutor({ completeRule, tuple, @@ -72,6 +102,11 @@ export const createMlAlertType = ( wrapHits, exceptionFilter, unprocessedExceptions, + wrapSuppressedHits, + alertTimestampOverride, + alertWithSuppression, + isAlertSuppressionActive, + experimentalFeatures, }); return { ...result, state }; }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts index c357a7e077bb2..59a0204ef9545 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.test.ts @@ -9,6 +9,7 @@ import dateMath from '@kbn/datemath'; import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks'; import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; import { mlExecutor } from './ml'; +import type { ExperimentalFeatures } from '../../../../../common'; import { getCompleteRuleMock, getMlRuleParams } from '../../rule_schema/mocks'; import { getListClientMock } from '@kbn/lists-plugin/server/services/lists/list_client.mock'; import { findMlSignals } from './find_ml_signals'; @@ -21,6 +22,7 @@ jest.mock('./find_ml_signals'); jest.mock('./bulk_create_ml_signals'); describe('ml_executor', () => { + let mockExperimentalFeatures: jest.Mocked; let jobsSummaryMock: jest.Mock; let forceStartDatafeedsMock: jest.Mock; let stopDatafeedsMock: jest.Mock; @@ -37,6 +39,7 @@ describe('ml_executor', () => { const listClient = getListClientMock(); beforeEach(() => { + mockExperimentalFeatures = {} as jest.Mocked; jobsSummaryMock = jest.fn(); mlMock = mlPluginServerMock.createSetupContract(); mlMock.jobServiceProvider.mockReturnValue({ @@ -59,7 +62,7 @@ describe('ml_executor', () => { }); (bulkCreateMlSignals as jest.Mock).mockResolvedValue({ success: true, - bulkCreateDuration: 0, + bulkCreateDuration: 21, createdItemsCount: 0, errors: [], createdItems: [], @@ -80,6 +83,11 @@ describe('ml_executor', () => { wrapHits: jest.fn(), exceptionFilter: undefined, unprocessedExceptions: [], + wrapSuppressedHits: jest.fn(), + alertTimestampOverride: undefined, + alertWithSuppression: jest.fn(), + isAlertSuppressionActive: true, + experimentalFeatures: mockExperimentalFeatures, }) ).rejects.toThrow('ML plugin unavailable during rule execution'); }); @@ -97,6 +105,11 @@ describe('ml_executor', () => { wrapHits: jest.fn(), exceptionFilter: undefined, unprocessedExceptions: [], + wrapSuppressedHits: jest.fn(), + alertTimestampOverride: undefined, + alertWithSuppression: jest.fn(), + isAlertSuppressionActive: true, + experimentalFeatures: mockExperimentalFeatures, }); expect(ruleExecutionLogger.warn).toHaveBeenCalled(); expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain( @@ -125,6 +138,11 @@ describe('ml_executor', () => { wrapHits: jest.fn(), exceptionFilter: undefined, unprocessedExceptions: [], + wrapSuppressedHits: jest.fn(), + alertTimestampOverride: undefined, + alertWithSuppression: jest.fn(), + isAlertSuppressionActive: true, + experimentalFeatures: mockExperimentalFeatures, }); expect(ruleExecutionLogger.warn).toHaveBeenCalled(); expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain( @@ -149,9 +167,49 @@ describe('ml_executor', () => { wrapHits: jest.fn(), exceptionFilter: undefined, unprocessedExceptions: [], + wrapSuppressedHits: jest.fn(), + alertTimestampOverride: undefined, + alertWithSuppression: jest.fn(), + isAlertSuppressionActive: true, + experimentalFeatures: mockExperimentalFeatures, }); expect(result.userError).toEqual(true); expect(result.success).toEqual(false); expect(result.errors).toEqual(['my_test_job_name missing']); }); + + it('returns some timing information as part of the result', async () => { + // ensure our mock corresponds to the job that the rule uses + jobsSummaryMock.mockResolvedValue( + mlCompleteRule.ruleParams.machineLearningJobId.map((jobId) => ({ + id: jobId, + jobState: 'opened', + datafeedState: 'started', + })) + ); + + const result = await mlExecutor({ + completeRule: mlCompleteRule, + tuple, + ml: mlMock, + services: alertServices, + ruleExecutionLogger, + listClient, + bulkCreate: jest.fn(), + wrapHits: jest.fn(), + exceptionFilter: undefined, + unprocessedExceptions: [], + wrapSuppressedHits: jest.fn(), + alertTimestampOverride: undefined, + alertWithSuppression: jest.fn(), + isAlertSuppressionActive: true, + experimentalFeatures: mockExperimentalFeatures, + }); + + expect(result).toEqual( + expect.objectContaining({ + bulkCreateTimes: expect.arrayContaining([expect.any(Number)]), + }) + ); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts index 641a9dab05cb2..4b7de9b27a667 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/ml.ts @@ -8,6 +8,7 @@ /* eslint require-atomic-updates: ["error", { "allowProperties": true }] */ import type { KibanaRequest } from '@kbn/core/server'; +import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import type { AlertInstanceContext, @@ -17,11 +18,12 @@ import type { import type { ListClient } from '@kbn/lists-plugin/server'; import type { Filter } from '@kbn/es-query'; import { isJobStarted } from '../../../../../common/machine_learning/helpers'; +import type { ExperimentalFeatures } from '../../../../../common/experimental_features'; import type { CompleteRule, MachineLearningRuleParams } from '../../rule_schema'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; import { filterEventsAgainstList } from '../utils/large_list_filters/filter_events_against_list'; import { findMlSignals } from './find_ml_signals'; -import type { BulkCreate, RuleRangeTuple, WrapHits } from '../types'; +import type { BulkCreate, RuleRangeTuple, WrapHits, WrapSuppressedHits } from '../types'; import { addToSearchAfterReturn, createErrorsFromShard, @@ -33,6 +35,26 @@ import type { SetupPlugins } from '../../../../plugin'; import { withSecuritySpan } from '../../../../utils/with_security_span'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import type { AnomalyResults } from '../../../machine_learning'; +import { bulkCreateSuppressedAlertsInMemory } from '../utils/bulk_create_suppressed_alerts_in_memory'; +import { buildReasonMessageForMlAlert } from '../utils/reason_formatters'; + +interface MachineLearningRuleExecutorParams { + completeRule: CompleteRule; + tuple: RuleRangeTuple; + ml: SetupPlugins['ml']; + listClient: ListClient; + services: RuleExecutorServices; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + bulkCreate: BulkCreate; + wrapHits: WrapHits; + exceptionFilter: Filter | undefined; + unprocessedExceptions: ExceptionListItemSchema[]; + wrapSuppressedHits: WrapSuppressedHits; + alertTimestampOverride: Date | undefined; + alertWithSuppression: SuppressedAlertService; + isAlertSuppressionActive: boolean; + experimentalFeatures: ExperimentalFeatures; +} export const mlExecutor = async ({ completeRule, @@ -45,18 +67,12 @@ export const mlExecutor = async ({ wrapHits, exceptionFilter, unprocessedExceptions, -}: { - completeRule: CompleteRule; - tuple: RuleRangeTuple; - ml: SetupPlugins['ml']; - listClient: ListClient; - services: RuleExecutorServices; - ruleExecutionLogger: IRuleExecutionLogForExecutors; - bulkCreate: BulkCreate; - wrapHits: WrapHits; - exceptionFilter: Filter | undefined; - unprocessedExceptions: ExceptionListItemSchema[]; -}) => { + isAlertSuppressionActive, + wrapSuppressedHits, + alertTimestampOverride, + alertWithSuppression, + experimentalFeatures, +}: MachineLearningRuleExecutorParams) => { const result = createSearchAfterReturnType(); const ruleParams = completeRule.ruleParams; @@ -120,6 +136,7 @@ export const mlExecutor = async ({ return result; } + // TODO we add the max_signals warning _before_ filtering the anomalies against the exceptions list. Is that correct? if ( anomalyResults.hits.total && typeof anomalyResults.hits.total !== 'number' && @@ -140,17 +157,36 @@ export const mlExecutor = async ({ ruleExecutionLogger.debug(`Found ${anomalyCount} signals from ML anomalies`); } - const createResult = await bulkCreateMlSignals({ - anomalyHits: filteredAnomalyHits, - completeRule, - services, - ruleExecutionLogger, - id: completeRule.alertId, - signalsIndex: ruleParams.outputIndex, - bulkCreate, - wrapHits, - }); - addToSearchAfterReturn({ current: result, next: createResult }); + if (anomalyCount && isAlertSuppressionActive) { + await bulkCreateSuppressedAlertsInMemory({ + enrichedEvents: filteredAnomalyHits, + toReturn: result, + wrapHits, + bulkCreate, + services, + buildReasonMessage: buildReasonMessageForMlAlert, + ruleExecutionLogger, + tuple, + alertSuppression: completeRule.ruleParams.alertSuppression, + wrapSuppressedHits, + alertTimestampOverride, + alertWithSuppression, + experimentalFeatures, + }); + } else { + const createResult = await bulkCreateMlSignals({ + anomalyHits: filteredAnomalyHits, + completeRule, + services, + ruleExecutionLogger, + id: completeRule.alertId, + signalsIndex: ruleParams.outputIndex, + bulkCreate, + wrapHits, + }); + addToSearchAfterReturn({ current: result, next: createResult }); + } + const shardFailures = anomalyResults._shards.failures ?? []; const searchErrors = createErrorsFromShard({ errors: shardFailures, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 31aa1797234bf..8f7a50b195e4f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -37,7 +37,7 @@ import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server'; import type { RuleResponseAction } from '../../../../common/api/detection_engine/model/rule_response_actions'; import type { ConfigType } from '../../../config'; import type { SetupPlugins } from '../../../plugin'; -import type { CompleteRule, EqlRuleParams, RuleParams, ThreatRuleParams } from '../rule_schema'; +import type { CompleteRule, RuleParams } from '../rule_schema'; import type { ExperimentalFeatures } from '../../../../common/experimental_features'; import type { ITelemetryEventsSender } from '../../telemetry/sender'; import type { IRuleExecutionLogForExecutors, IRuleMonitoringService } from '../rule_monitoring'; @@ -401,5 +401,3 @@ export interface OverrideBodyQuery { _source?: estypes.SearchSourceConfig; fields?: estypes.Fields; } - -export type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 89328f176567d..70fee20116fc4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -9,14 +9,19 @@ import objectHash from 'object-hash'; import { TIMESTAMP } from '@kbn/rule-data-utils'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; -import type { RuleWithInMemorySuppression, SignalSourceHit } from '../types'; +import type { SignalSourceHit } from '../types'; import type { BaseFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import type { ConfigType } from '../../../../config'; -import type { CompleteRule } from '../../rule_schema'; +import type { + CompleteRule, + EqlRuleParams, + MachineLearningRuleParams, + ThreatRuleParams, +} from '../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { buildBulkBody } from '../factories/utils/build_bulk_body'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; @@ -24,6 +29,8 @@ import { generateId } from './utils'; import type { BuildReasonMessage } from './reason_formatters'; +type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; + /** * wraps suppressed alerts * creates instanceId hash, which is used to search on time interval alerts diff --git a/x-pack/test/common/utils/security_solution/detections_response/delete_all_anomalies.ts b/x-pack/test/common/utils/security_solution/detections_response/delete_all_anomalies.ts new file mode 100644 index 0000000000000..1f9df710c5d5d --- /dev/null +++ b/x-pack/test/common/utils/security_solution/detections_response/delete_all_anomalies.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ToolingLog } from '@kbn/tooling-log'; +import type { Client } from '@elastic/elasticsearch'; + +import { countDownTest } from './count_down_test'; + +export const deleteAllAnomalies = async ( + log: ToolingLog, + es: Client, + index: string[] = ['.ml-anomalies-*'] +): Promise => { + await countDownTest( + async () => { + await es.deleteByQuery({ + index, + body: { + query: { + match_all: {}, + }, + }, + refresh: true, + }); + return { + passed: true, + }; + }, + 'deleteAllAnomalies', + log + ); +}; diff --git a/x-pack/test/common/utils/security_solution/detections_response/index.ts b/x-pack/test/common/utils/security_solution/detections_response/index.ts index d6a06f8e57797..43c2a54900c15 100644 --- a/x-pack/test/common/utils/security_solution/detections_response/index.ts +++ b/x-pack/test/common/utils/security_solution/detections_response/index.ts @@ -7,6 +7,7 @@ export * from './rules'; export * from './alerts'; +export * from './delete_all_anomalies'; export * from './count_down_test'; export * from './route_with_namespace'; export * from './wait_for'; diff --git a/x-pack/test/functional/es_archives/security_solution/anomalies/mappings.json b/x-pack/test/functional/es_archives/security_solution/anomalies/mappings.json index 484e0f3fc9aa0..56a26b937a49b 100644 --- a/x-pack/test/functional/es_archives/security_solution/anomalies/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/anomalies/mappings.json @@ -2,22 +2,21 @@ "type": "index", "value": { "aliases": { - ".ml-anomalies-.write-linux_anomalous_network_activity_ecs": { + ".ml-anomalies-.write-v3_linux_anomalous_network_activity": { "is_hidden": true }, - ".ml-anomalies-linux_anomalous_network_activity_ecs": { + ".ml-anomalies-v3_linux_anomalous_network_activity": { "filter": { "term": { "job_id": { - "boost": 1, - "value": "linux_anomalous_network_activity_ecs" + "value": "v3_linux_anomalous_network_activity" } } }, "is_hidden": true } }, - "index": ".ml-anomalies-custom-linux_anomalous_network_activity_ecs", + "index": ".ml-anomalies-custom-v3_linux_anomalous_network_activity", "mappings": { "_meta": { "version": "8.0.0" diff --git a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts index 7256432174e3c..a47e43bd426e6 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts @@ -84,6 +84,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s 'riskScoringPersistence', 'riskScoringRoutesEnabled', 'bulkCustomHighlightedFieldsEnabled', + 'alertSuppressionForMachineLearningRuleEnabled', 'manualRuleRunEnabled', ])}`, '--xpack.task_manager.poll_interval=1000', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts index 76c73ff71cc18..825d6a0e5833b 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts @@ -19,6 +19,7 @@ export default createTestConfig({ ])}`, // See tests within the file "ignore_fields.ts" which use these values in "alertIgnoreFields" `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'bulkCustomHighlightedFieldsEnabled', + 'alertSuppressionForMachineLearningRuleEnabled', 'alertSuppressionForEsqlRuleEnabled', ])}`, ], diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts index 3ea2c4e6c9359..5d0e8f4db4061 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts @@ -14,6 +14,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./esql')); loadTestFile(require.resolve('./esql_suppression')); loadTestFile(require.resolve('./machine_learning')); + loadTestFile(require.resolve('./machine_learning_alert_suppression')); loadTestFile(require.resolve('./new_terms')); loadTestFile(require.resolve('./new_terms_alert_suppression')); loadTestFile(require.resolve('./saved_query')); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts index 3fb077df86a38..5d73249e576f4 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts @@ -151,7 +151,7 @@ export default ({ getService }: FtrProviderContext) => { [SPACE_IDS]: ['default'], [ALERT_SEVERITY]: 'critical', [ALERT_RISK_SCORE]: 50, - [ALERT_RULE_PARAMETERS]: { + [ALERT_RULE_PARAMETERS]: expect.objectContaining({ anomaly_threshold: 30, author: [], description: 'Test ML rule description', @@ -174,7 +174,7 @@ export default ({ getService }: FtrProviderContext) => { to: 'now', type: 'machine_learning', version: 1, - }, + }), [ALERT_DEPTH]: 1, [ALERT_REASON]: `event with process store, by root on mothra created critical alert Test ML rule.`, [ALERT_ORIGINAL_TIME]: expect.any(String), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning_alert_suppression.ts new file mode 100644 index 0000000000000..b29ce8abbb8ef --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning_alert_suppression.ts @@ -0,0 +1,1106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { expect } from 'expect'; + +import { + MachineLearningRuleCreateProps, + RuleExecutionStatusEnum, +} from '@kbn/security-solution-plugin/common/api/detection_engine'; +import type { Anomaly } from '@kbn/security-solution-plugin/server/lib/machine_learning'; +import { + ALERT_LAST_DETECTED, + ALERT_START, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_TERMS, + TIMESTAMP, +} from '@kbn/rule-data-utils'; +import { ALERT_ORIGINAL_TIME } from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { DETECTION_ENGINE_SIGNALS_STATUS_URL as DETECTION_ENGINE_ALERTS_STATUS_URL } from '@kbn/security-solution-plugin/common/constants'; +import { EsArchivePathBuilder } from '../../../../../../es_archive_path_builder'; +import { FtrProviderContext } from '../../../../../../ftr_provider_context'; +import { + dataGeneratorFactory, + executeSetupModuleRequest, + forceStartDatafeeds, + getAlerts, + getOpenAlerts, + getPreviewAlerts, + patchRule, + previewRule, + previewRuleWithExceptionEntries, + setAlertStatus, +} from '../../../../utils'; +import { + createRule, + deleteAllAlerts, + deleteAllAnomalies, + deleteAllRules, +} from '../../../../../../../common/utils/security_solution'; +import { deleteAllExceptions } from '../../../../../lists_and_exception_lists/utils'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + const config = getService('config'); + + const isServerless = config.get('serverless'); + const dataPathBuilder = new EsArchivePathBuilder(isServerless); + const auditbeatArchivePath = dataPathBuilder.getPath('auditbeat/hosts'); + + const { indexListOfDocuments } = dataGeneratorFactory({ + es, + index: '.ml-anomalies-custom-v3_linux_anomalous_network_activity', + log, + }); + + const mlModuleName = 'security_linux_v3'; + const mlJobId = 'v3_linux_anomalous_network_activity'; + const baseRuleProps: MachineLearningRuleCreateProps = { + name: 'Test ML rule', + description: 'Test ML rule description', + risk_score: 50, + severity: 'critical', + type: 'machine_learning', + anomaly_threshold: 40, + machine_learning_job_id: mlJobId, + from: '1900-01-01T00:00:00.000Z', + rule_id: 'ml-rule-id', + }; + let ruleProps: MachineLearningRuleCreateProps; + const baseAnomaly: Partial = { + is_interim: false, + record_score: 43, // exceeds anomaly_threshold above + result_type: 'record', + job_id: mlJobId, + 'user.name': ['root'], + }; + + // The tests described in this file rely on the + // 'alertSuppressionForMachineLearningRuleEnabled' feature flag, and are thus + // skipped in MKI + describe('@ess @serverless @skipInServerlessMKI Machine Learning Detection Rule - Alert Suppression', () => { + describe('with an active ML Job', () => { + before(async () => { + // Order is critical here: auditbeat data must be loaded before attempting to start the ML job, + // as the job looks for certain indices on start + await esArchiver.load(auditbeatArchivePath); + await executeSetupModuleRequest({ module: mlModuleName, rspCode: 200, supertest }); + await forceStartDatafeeds({ jobId: mlJobId, rspCode: 200, supertest }); + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/anomalies'); + await deleteAllAnomalies(log, es); + }); + + after(async () => { + await esArchiver.load(auditbeatArchivePath); + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/anomalies'); + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + }); + + afterEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + await deleteAllAnomalies(log, es); + }); + + describe('with per-execution suppression duration', () => { + beforeEach(() => { + ruleProps = { + ...baseRuleProps, + alert_suppression: { + group_by: ['user.name'], + missing_fields_strategy: 'suppress', + }, + }; + }); + + it('performs no suppression if a single alert is generated', async () => { + const timestamp = new Date().toISOString(); + const anomaly = { + ...baseAnomaly, + timestamp, + }; + await indexListOfDocuments([anomaly]); + const createdRule = await createRule(supertest, log, ruleProps); + const alerts = await getAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [{ field: 'user.name', value: ['root'] }], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('suppresses alerts within a single execution', async () => { + const timestamp = new Date().toISOString(); + const anomaly = { + ...baseAnomaly, + timestamp, + }; + await indexListOfDocuments([anomaly, anomaly]); + + const createdRule = await createRule(supertest, log, { + ...ruleProps, + from: timestamp, + }); + + const alerts = await getAlerts(supertest, log, es, createdRule); + expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }) + ); + }); + + it('deduplicates previously suppressed alerts if rule has overlapping execution windows', async () => { + const firstTimestamp = new Date().toISOString(); + const firstAnomaly = { + ...baseAnomaly, + timestamp: firstTimestamp, + }; + await indexListOfDocuments([firstAnomaly]); + + const createdRule = await createRule(supertest, log, { + ...ruleProps, + from: firstTimestamp, + }); + const alerts = await getAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + // suppression boundaries equal to original event time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + const secondTimestamp = new Date().toISOString(); + const secondAnomaly = { + ...baseAnomaly, + timestamp: secondTimestamp, + }; + + // Add more anomalies, then disable and re-enable to trigger another + // rule run. The second anomaly should trigger an update to the + // existing alert without changing the timestamp + await indexListOfDocuments([secondAnomaly, secondAnomaly]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + new Date() + ); + + expect(secondAlerts.hits.hits).toHaveLength(2); + expect(secondAlerts.hits.hits[1]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // 1 of the two new anomalies was suppressed on this execution + }) + ); + }); + }); + + describe('with interval suppression duration', () => { + beforeEach(() => { + ruleProps = { + ...baseRuleProps, + alert_suppression: { + duration: { + value: 300, + unit: 'm', + }, + group_by: ['user.name'], + missing_fields_strategy: 'suppress', + }, + }; + }); + + it('performs no suppression if a single alert is generated', async () => { + const timestamp = new Date().toISOString(); + const anomaly = { + ...baseAnomaly, + timestamp, + }; + await indexListOfDocuments([anomaly]); + const createdRule = await createRule(supertest, log, ruleProps); + const alerts = await getAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [{ field: 'user.name', value: ['root'] }], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('suppresses alerts across two executions', async () => { + const firstTimestamp = new Date().toISOString(); + const firstAnomaly = { + ...baseAnomaly, + timestamp: firstTimestamp, + }; + await indexListOfDocuments([firstAnomaly]); + + const createdRule = await createRule(supertest, log, { + ...ruleProps, + from: firstTimestamp, + }); + const alerts = await getAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + // suppression boundaries equal to original event time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + const secondTimestamp = new Date().toISOString(); + const secondAnomaly = { + ...baseAnomaly, + timestamp: secondTimestamp, + }; + + // Add more anomalies, then disable and re-enable to trigger another + // rule run. The second anomaly should trigger an update to the + // existing alert without changing the timestamp + await indexListOfDocuments([secondAnomaly, secondAnomaly]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + new Date() + ); + + expect(secondAlerts.hits.hits).toHaveLength(1); + expect(secondAlerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, // 1 of the two new anomalies was suppressed on this execution + }) + ); + }); + + describe('with anomalies spanning multiple rule execution windows', () => { + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + const thirdTimestamp = '2020-10-28T06:45:00.000Z'; + const afterThirdTimestamp = '2020-10-28T07:00:00.000Z'; + + beforeEach(async () => { + const firstAnomaly = { + ...baseAnomaly, + timestamp: firstTimestamp, + }; + const secondAnomaly = { + ...baseAnomaly, + timestamp: secondTimestamp, + }; + const thirdAnomaly = { + ...baseAnomaly, + timestamp: thirdTimestamp, + }; + + await indexListOfDocuments([ + firstAnomaly, + firstAnomaly, + secondAnomaly, + secondAnomaly, + thirdAnomaly, + ]); + }); + + it('suppresses alerts across three executions', async () => { + const rule = { ...ruleProps, interval: '30m' }; + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date(afterThirdTimestamp), + invocationCount: 3, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: afterThirdTimestamp, + [ALERT_START]: '2020-10-28T06:00:00.000Z', + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: thirdTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 4, // in total 4 alert got suppressed: 1 from the first run, 2 from the second, 1 from the third + }) + ); + }); + + it('suppresses alerts across multiple, sparse executions', async () => { + const fifthTimestamp = '2020-10-28T07:45:00.000Z'; + const afterFifthTimestamp = '2020-10-28T08:00:00.000Z'; + const fifthAnomaly = { ...baseAnomaly, timestamp: fifthTimestamp }; + // no anomaly for fourth execution + await indexListOfDocuments([fifthAnomaly]); + + const rule = { ...ruleProps, interval: '30m' }; + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date(afterFifthTimestamp), + invocationCount: 5, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: afterFifthTimestamp, + [ALERT_START]: '2020-10-28T06:00:00.000Z', + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: fifthTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 5, // in total 5 alerts were suppressed: 1 from the first run, 2 from the second, 1 from the third run, none from the fourth, and one from the fifth. + }) + ); + }); + }); + + it('suppresses alerts on multiple fields', async () => { + const timestamp = new Date().toISOString(); + const anomaly = { + ...baseAnomaly, + timestamp, + 'process.name': ['auditbeat'], + }; + await indexListOfDocuments([anomaly, anomaly]); + + const rule = { + ...ruleProps, + alert_suppression: { + ...ruleProps.alert_suppression, + group_by: ['user.name', 'process.name'], + }, + }; + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date(timestamp), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + { + field: 'process.name', + value: ['auditbeat'], + }, + ], + [TIMESTAMP]: timestamp, + [ALERT_START]: timestamp, + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }) + ); + }); + + it('suppresses alerts with missing fields, if configured to do so', async () => { + const timestamp = new Date().toISOString(); + const anomaly = { + ...baseAnomaly, + timestamp, + 'host.name': ['relevant'], + }; + const anomalyWithoutSuppressionField = { + ...baseAnomaly, + timestamp, + }; + await indexListOfDocuments([anomaly, anomaly, anomalyWithoutSuppressionField]); + + const rule = { + ...ruleProps, + alert_suppression: { + ...ruleProps.alert_suppression, + group_by: ['host.name'], + }, + }; + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date(timestamp), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_SUPPRESSION_DOCS_COUNT], + }); + + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, + }, + ], + [TIMESTAMP]: timestamp, + [ALERT_START]: timestamp, + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + expect(previewAlerts[1]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['relevant'], + }, + ], + [TIMESTAMP]: timestamp, + [ALERT_START]: timestamp, + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // the anomaly without `host.name` is not represented here + }) + ); + }); + + it('does not suppress alerts with missing fields, if not configured to do so', async () => { + const rule = { + ...ruleProps, + alert_suppression: { + ...ruleProps.alert_suppression, + group_by: ['host.name'], + missing_fields_strategy: 'doNotSuppress' as const, + }, + }; + const timestamp = new Date().toISOString(); + const anomaly = { + ...baseAnomaly, + timestamp, + 'host.name': ['relevant'], + }; + const anomalyWithoutSuppressionField = { + ...baseAnomaly, + timestamp, + 'user.name': ['irrelevant'], + }; + await indexListOfDocuments([ + anomaly, + anomaly, + anomalyWithoutSuppressionField, + anomalyWithoutSuppressionField, + ]); + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date(timestamp), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + + expect(previewAlerts.length).toEqual(3); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + 'user.name': ['irrelevant'], + [TIMESTAMP]: timestamp, + [ALERT_START]: timestamp, + }) + ); + + expect(previewAlerts[0]._source).toEqual( + expect.not.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: expect.anything(), + [ALERT_ORIGINAL_TIME]: expect.anything(), + [ALERT_SUPPRESSION_START]: expect.anything(), + [ALERT_SUPPRESSION_END]: expect.anything(), + [ALERT_SUPPRESSION_DOCS_COUNT]: expect.anything(), + }) + ); + + expect(previewAlerts[1]._source).toEqual( + expect.objectContaining({ + 'user.name': ['irrelevant'], + [TIMESTAMP]: timestamp, + [ALERT_START]: timestamp, + }) + ); + expect(previewAlerts[1]._source).toEqual( + expect.not.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: expect.anything(), + [ALERT_ORIGINAL_TIME]: expect.anything(), + [ALERT_SUPPRESSION_START]: expect.anything(), + [ALERT_SUPPRESSION_END]: expect.anything(), + [ALERT_SUPPRESSION_DOCS_COUNT]: expect.anything(), + }) + ); + + expect(previewAlerts[2]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['relevant'], + }, + ], + [TIMESTAMP]: timestamp, + [ALERT_START]: timestamp, + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // the anomaly without `host.name` is not represented here + }) + ); + }); + + it('does not suppress into a closed alert', async () => { + const firstTimestamp = new Date().toISOString(); + const firstAnomaly = { + ...baseAnomaly, + timestamp: firstTimestamp, + }; + await indexListOfDocuments([firstAnomaly]); + + const createdRule = await createRule(supertest, log, { + ...ruleProps, + from: firstTimestamp, + }); + const alerts = await getAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + const alertId = alerts.hits.hits[0]._id!; + + // close generated alert + await supertest + .post(DETECTION_ENGINE_ALERTS_STATUS_URL) + .set('kbn-xsrf', 'true') + .send(setAlertStatus({ alertIds: [alertId], status: 'closed' })) + .expect(200); + + const secondTimestamp = new Date().toISOString(); + const secondAnomaly = { + ...baseAnomaly, + timestamp: secondTimestamp, + }; + + // Add more anomalies, then disable and re-enable to trigger another + // rule run. The second anomalies should create a new alert, since the existing alert is closed. + await indexListOfDocuments([secondAnomaly, secondAnomaly]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + new Date() + ); + + expect(secondAlerts.hits.hits).toHaveLength(1); + expect(secondAlerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }) + ); + }); + + it('does not suppress into an unsuppressed alert', async () => { + const firstTimestamp = new Date().toISOString(); + const firstAnomaly = { + ...baseAnomaly, + timestamp: firstTimestamp, + }; + await indexListOfDocuments([firstAnomaly]); + + const ruleWithoutSuppression = { ...ruleProps, alert_suppression: undefined }; + const createdRule = await createRule(supertest, log, { + ...ruleWithoutSuppression, + from: firstTimestamp, + }); + const alerts = await getAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + + // update the rule to include suppression + await patchRule(supertest, log, { + id: createdRule.id, + alert_suppression: ruleProps.alert_suppression, + }); + + const secondTimestamp = new Date().toISOString(); + const secondAnomaly = { + ...baseAnomaly, + timestamp: secondTimestamp, + }; + + // Add more anomalies, then disable and re-enable to trigger another + // rule run. The second anomalies should create a new suppressed alert, since the original was not suppressed. + await indexListOfDocuments([secondAnomaly, secondAnomaly, secondAnomaly]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + new Date() + ); + + expect(secondAlerts.hits.hits).toHaveLength(2); + // assert that the first alert does not have suppression fields + expect(secondAlerts.hits.hits[0]._source).toEqual( + expect.not.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: expect.anything(), + [ALERT_ORIGINAL_TIME]: expect.anything(), + [ALERT_SUPPRESSION_START]: expect.anything(), + [ALERT_SUPPRESSION_END]: expect.anything(), + [ALERT_SUPPRESSION_DOCS_COUNT]: expect.anything(), + }) + ); + + expect(secondAlerts.hits.hits[1]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }) + ); + }); + + it('suppresses alerts that would be _created_ within the suppression duration window, even if the original anomalies were outside that suppression duration window', async () => { + const rule = { + ...ruleProps, + interval: '30m', + alert_suppression: { + ...ruleProps.alert_suppression, + duration: { + value: 1, + unit: 'm', + }, + }, + } as MachineLearningRuleCreateProps; + const firstTimestamp = '2020-10-28T06:00:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + const firstAnomaly = { ...baseAnomaly, timestamp: firstTimestamp }; + const secondAnomaly = { ...baseAnomaly, timestamp: secondTimestamp }; + await indexListOfDocuments([firstAnomaly, secondAnomaly]); + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date(secondTimestamp), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [TIMESTAMP]: secondTimestamp, + [ALERT_LAST_DETECTED]: secondTimestamp, + [ALERT_START]: secondTimestamp, + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }) + ); + }); + + it('does not suppress across multiple runs if the suppression interval is less than the rule interval ', async () => { + const rule = { + ...ruleProps, + interval: '5m', + alert_suppression: { + ...ruleProps.alert_suppression, + duration: { + value: 1, + unit: 'm', + }, + }, + } as MachineLearningRuleCreateProps; + const firstTimestamp = '2020-10-28T06:00:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + const firstAnomaly = { ...baseAnomaly, timestamp: firstTimestamp }; + const secondAnomaly = { ...baseAnomaly, timestamp: secondTimestamp }; + await indexListOfDocuments([firstAnomaly, secondAnomaly]); + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date(secondTimestamp), + invocationCount: 3, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + expect(previewAlerts[1]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('suppresses alerts within a single execution', async () => { + const timestamp = new Date().toISOString(); + const anomaly = { + ...baseAnomaly, + timestamp, + }; + await indexListOfDocuments([anomaly, anomaly]); + + const createdRule = await createRule(supertest, log, { + ...ruleProps, + from: timestamp, + }); + + const alerts = await getAlerts(supertest, log, es, createdRule); + expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }) + ); + }); + + it('deduplicates previously suppressed alerts if rule has overlapping execution windows', async () => { + const firstTimestamp = new Date().toISOString(); + const firstAnomaly = { + ...baseAnomaly, + timestamp: firstTimestamp, + }; + await indexListOfDocuments([firstAnomaly]); + + const createdRule = await createRule(supertest, log, { + ...ruleProps, + from: firstTimestamp, + }); + const alerts = await getAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + // suppression boundaries equal to original event time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + const secondTimestamp = new Date().toISOString(); + const secondAnomaly = { + ...baseAnomaly, + timestamp: secondTimestamp, + }; + + // Add more anomalies, then disable and re-enable to trigger another + // rule run. The second anomaly should trigger an update to the + // existing alert without changing the timestamp + await indexListOfDocuments([secondAnomaly, secondAnomaly]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + new Date() + ); + + expect(secondAlerts.hits.hits).toHaveLength(1); + expect(secondAlerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, // both new anomalies were suppressed into the original + }) + ); + }); + + it('suppresses alerts with array field values', async () => { + const timestamp = new Date().toISOString(); + const anomaly = { + ...baseAnomaly, + 'user.name': ['host1', 'host2'], + timestamp, + }; + await indexListOfDocuments([anomaly, anomaly]); + + const createdRule = await createRule(supertest, log, { + ...ruleProps, + from: timestamp, + }); + + const alerts = await getAlerts(supertest, log, es, createdRule); + expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['host1', 'host2'], + }, + ], + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }) + ); + }); + + describe('with exceptions', () => { + beforeEach(async () => { + await deleteAllExceptions(supertest, log); + }); + + it('applies exceptions before suppression', async () => { + const timestamp = new Date().toISOString(); + const anomaly = { + ...baseAnomaly, + timestamp, + }; + const anomalyWithExceptionField = { + ...anomaly, + 'process.name': ['auditbeat'], + }; + await indexListOfDocuments([anomaly, anomalyWithExceptionField]); + + const { previewId } = await previewRuleWithExceptionEntries({ + supertest, + rule: ruleProps, + log, + timeframeEnd: new Date(timestamp), + entries: [ + [ + { + field: 'process.name', + operator: 'included', + type: 'match', + value: 'auditbeat', + }, + ], + ], + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'user.name', + value: ['root'], + }, + ], + [TIMESTAMP]: timestamp, + [ALERT_START]: timestamp, + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // the anomaly with the exception field was not suppressed but omitted due to the exception + }) + ); + }); + }); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/machine_learning/machine_learning_setup.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/machine_learning/machine_learning_setup.ts index a9b9bf1c8ce5b..fa0c6fa4f78b5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/machine_learning/machine_learning_setup.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/machine_learning/machine_learning_setup.ts @@ -6,6 +6,7 @@ */ import type SuperTest from 'supertest'; +import { ML_GROUP_ID } from '@kbn/security-solution-plugin/common/constants'; import { getCommonRequestHeader } from '../../../../../functional/services/ml/common_api'; export const executeSetupModuleRequest = async ({ @@ -22,7 +23,7 @@ export const executeSetupModuleRequest = async ({ .set(getCommonRequestHeader('1')) .send({ prefix: '', - groups: ['auditbeat'], + groups: [ML_GROUP_ID], indexPatternName: 'auditbeat-*', startDatafeed: false, useDedicatedIndex: true, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/remove_server_generated_properties_including_rule_id.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/remove_server_generated_properties_including_rule_id.ts index 1b57b5663ec23..176ce575a6457 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/remove_server_generated_properties_including_rule_id.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/remove_server_generated_properties_including_rule_id.ts @@ -7,7 +7,10 @@ import type { RuleResponse } from '@kbn/security-solution-plugin/common/api/detection_engine'; -import { removeServerGeneratedProperties } from './remove_server_generated_properties'; +import { + removeServerGeneratedProperties, + type RuleWithoutServerGeneratedProperties, +} from './remove_server_generated_properties'; /** * This will remove server generated properties such as date times, etc... including the rule_id @@ -15,9 +18,8 @@ import { removeServerGeneratedProperties } from './remove_server_generated_prope */ export const removeServerGeneratedPropertiesIncludingRuleId = ( rule: RuleResponse -): Partial => { +): Omit => { const ruleWithRemovedProperties = removeServerGeneratedProperties(rule); - // eslint-disable-next-line @typescript-eslint/naming-convention - const { rule_id, ...additionalRuledIdRemoved } = ruleWithRemovedProperties; + const { rule_id: _, ...additionalRuledIdRemoved } = ruleWithRemovedProperties; return additionalRuledIdRemoved; }; diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index 6e65ab15324a6..092fe4b79d38f 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -47,6 +47,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'alertSuppressionForEsqlRuleEnabled', 'bulkCustomHighlightedFieldsEnabled', + 'alertSuppressionForMachineLearningRuleEnabled', 'manualRuleRunEnabled', ])}`, // mock cloud to enable the guided onboarding tour in e2e tests diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts index d6f23687cf418..946e0190bc1f8 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts @@ -15,6 +15,7 @@ import { login } from '../../../../tasks/login'; import { visit } from '../../../../tasks/navigation'; import { ALERT_SUPPRESSION_FIELDS_INPUT, + MACHINE_LEARNING_TYPE, THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX, } from '../../../../screens/create_new_rule'; import { CREATE_RULE_URL } from '../../../../urls/navigation'; @@ -22,7 +23,7 @@ import { CREATE_RULE_URL } from '../../../../urls/navigation'; describe( 'Detection rules, Alert Suppression for Essentials tier', { - // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled + // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled, alertSuppressionForMachineLearningRuleEnabled tags: ['@serverless', '@skipInServerlessMKI'], env: { ftrConfig: { @@ -35,6 +36,7 @@ describe( kbnServerArgs: [ `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'alertSuppressionForEsqlRuleEnabled', + 'alertSuppressionForMachineLearningRuleEnabled', ])}`, ], }, @@ -60,6 +62,9 @@ describe( selectEsqlRuleType(); cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).should('be.enabled'); + + // ML Rules require Complete tier + cy.get(MACHINE_LEARNING_TYPE).get('button').should('be.disabled'); }); } ); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts index 1f86d6d0dd789..a4e7a7dabb5fe 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts @@ -8,6 +8,7 @@ import { THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX, ALERT_SUPPRESSION_DURATION_INPUT, + MACHINE_LEARNING_TYPE, } from '../../../../screens/create_new_rule'; import { @@ -52,6 +53,9 @@ describe( selectEsqlRuleType(); openSuppressionFieldsTooltipAndCheckLicense(); + // ML Rules require Platinum license + cy.get(MACHINE_LEARNING_TYPE).get('button').should('be.disabled'); + selectThresholdRuleType(); cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).should('be.disabled'); cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).parent().trigger('mouseover'); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule_suppression.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule_suppression.cy.ts new file mode 100644 index 0000000000000..befa75fce93ff --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/machine_learning_rule_suppression.cy.ts @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getMachineLearningRule } from '../../../../objects/rule'; +import { TOOLTIP } from '../../../../screens/common'; +import { + ALERT_SUPPRESSION_FIELDS, + ALERT_SUPPRESSION_FIELDS_INPUT, +} from '../../../../screens/create_new_rule'; +import { + DEFINITION_DETAILS, + DETAILS_TITLE, + SUPPRESS_BY_DETAILS, + SUPPRESS_FOR_DETAILS, + SUPPRESS_MISSING_FIELD, +} from '../../../../screens/rule_details'; +import { + executeSetupModuleRequest, + forceStartDatafeeds, + forceStopAndCloseJob, +} from '../../../../support/machine_learning'; +import { + continueFromDefineStep, + fillAlertSuppressionFields, + fillDefineMachineLearningRule, + selectMachineLearningRuleType, + selectAlertSuppressionPerInterval, + setAlertSuppressionDuration, + selectDoNotSuppressForMissingFields, + skipScheduleRuleAction, + fillAboutRuleMinimumAndContinue, + createRuleWithoutEnabling, +} from '../../../../tasks/create_new_rule'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; +import { getDetails } from '../../../../tasks/rule_details'; +import { CREATE_RULE_URL } from '../../../../urls/navigation'; + +describe( + 'Machine Learning Detection Rules - Creation', + { + // Skipped in MKI as tests depend on feature flag alertSuppressionForMachineLearningRuleEnabled + tags: ['@ess', '@serverless', '@skipInServerlessMKI'], + env: { + ftrConfig: { + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForMachineLearningRuleEnabled', + ])}`, + ], + }, + }, + }, + () => { + let mlRule: ReturnType; + const jobId = 'v3_linux_anomalous_network_activity'; + const suppressByFields = ['by_field_name', 'by_field_value']; + + beforeEach(() => { + login(); + visit(CREATE_RULE_URL); + }); + + describe('with Alert Suppression', () => { + describe('when no ML jobs have run', () => { + before(() => { + const machineLearningJobIds = ([] as string[]).concat( + getMachineLearningRule().machine_learning_job_id + ); + // ensure no ML jobs are started before the suite + machineLearningJobIds.forEach((j) => forceStopAndCloseJob({ jobId: j })); + }); + + beforeEach(() => { + mlRule = getMachineLearningRule(); + selectMachineLearningRuleType(); + fillDefineMachineLearningRule(mlRule); + }); + + it('disables the suppression fields and displays a message', () => { + cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).should('be.disabled'); + cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).realHover(); + cy.get(TOOLTIP).should( + 'contain.text', + 'To enable alert suppression, start relevant Machine Learning jobs.' + ); + }); + }); + + describe('when ML jobs have run', () => { + before(() => { + cy.task('esArchiverLoad', { archiveName: '../auditbeat/hosts', type: 'ftr' }); + executeSetupModuleRequest({ moduleName: 'security_linux_v3' }); + forceStartDatafeeds({ jobIds: [jobId] }); + cy.task('esArchiverLoad', { archiveName: 'anomalies', type: 'ftr' }); + }); + + after(() => { + cy.task('esArchiverUnload', { archiveName: 'anomalies', type: 'ftr' }); + cy.task('esArchiverUnload', { archiveName: '../auditbeat/hosts', type: 'ftr' }); + }); + + describe('when not all jobs are running', () => { + beforeEach(() => { + mlRule = getMachineLearningRule(); + selectMachineLearningRuleType(); + fillDefineMachineLearningRule(mlRule); + }); + + it('displays a warning message on the suppression fields', () => { + cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).should('be.enabled'); + cy.get(ALERT_SUPPRESSION_FIELDS).should( + 'contain.text', + 'This list of fields might be incomplete as some Machine Learning jobs are not running. Start all relevant jobs for a complete list.' + ); + }); + }); + + describe('when all jobs are running', () => { + beforeEach(() => { + mlRule = getMachineLearningRule({ machine_learning_job_id: [jobId] }); + selectMachineLearningRuleType(); + fillDefineMachineLearningRule(mlRule); + }); + + it('allows a rule with per-execution suppression to be created and displayed', () => { + fillAlertSuppressionFields(suppressByFields); + continueFromDefineStep(); + + // ensures details preview works correctly + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', suppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + }); + + fillAboutRuleMinimumAndContinue(mlRule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', suppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + }); + }); + + it('allows a rule with interval suppression to be created and displayed', () => { + fillAlertSuppressionFields(suppressByFields); + selectAlertSuppressionPerInterval(); + setAlertSuppressionDuration(45, 'm'); + selectDoNotSuppressForMissingFields(); + continueFromDefineStep(); + + // ensures details preview works correctly + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', suppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '45m'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + }); + + fillAboutRuleMinimumAndContinue(mlRule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', suppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '45m'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + }); + }); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/machine_learning_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/machine_learning_rule.cy.ts new file mode 100644 index 0000000000000..5e6cd673070ba --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/machine_learning_rule.cy.ts @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getMachineLearningRule } from '../../../../objects/rule'; +import { + ALERT_SUPPRESSION_DURATION_INPUT, + ALERT_SUPPRESSION_FIELDS, + ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS, +} from '../../../../screens/create_new_rule'; +import { + DEFINITION_DETAILS, + DETAILS_TITLE, + SUPPRESS_BY_DETAILS, + SUPPRESS_FOR_DETAILS, + SUPPRESS_MISSING_FIELD, +} from '../../../../screens/rule_details'; +import { + executeSetupModuleRequest, + forceStartDatafeeds, + forceStopAndCloseJob, +} from '../../../../support/machine_learning'; +import { editFirstRule } from '../../../../tasks/alerts_detection_rules'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; +import { createRule } from '../../../../tasks/api_calls/rules'; +import { + clearAlertSuppressionFields, + fillAlertSuppressionFields, + selectAlertSuppressionPerInterval, + selectAlertSuppressionPerRuleExecution, + setAlertSuppressionDuration, +} from '../../../../tasks/create_new_rule'; +import { saveEditedRule } from '../../../../tasks/edit_rule'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; +import { assertDetailsNotExist, getDetails } from '../../../../tasks/rule_details'; +import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; + +describe( + 'Machine Learning Detection Rules - Editing', + { + // Skipping in MKI as it depends on feature flag alertSuppressionForMachineLearningRuleEnabled + tags: ['@ess', '@serverless', '@skipInServerlessMKI'], + env: { + ftrConfig: { + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForMachineLearningRuleEnabled', + ])}`, + ], + }, + }, + }, + () => { + let mlRule: ReturnType; + const suppressByFields = ['by_field_name', 'by_field_value']; + const jobId = 'v3_linux_anomalous_network_activity'; + + before(() => { + const machineLearningJobIds = ([] as string[]).concat( + getMachineLearningRule().machine_learning_job_id + ); + // ensure no ML jobs are started before the test + machineLearningJobIds.forEach((j) => forceStopAndCloseJob({ jobId: j })); + }); + + beforeEach(() => { + login(); + deleteAlertsAndRules(); + cy.task('esArchiverLoad', { archiveName: '../auditbeat/hosts', type: 'ftr' }); + executeSetupModuleRequest({ moduleName: 'security_linux_v3' }); + forceStartDatafeeds({ jobIds: [jobId] }); + cy.task('esArchiverLoad', { archiveName: 'anomalies', type: 'ftr' }); + }); + + describe('without Alert Suppression', () => { + beforeEach(() => { + mlRule = getMachineLearningRule({ machine_learning_job_id: [jobId] }); + createRule(mlRule); + visit(RULES_MANAGEMENT_URL); + editFirstRule(); + }); + + it('allows editing of a rule to add suppression configuration', () => { + fillAlertSuppressionFields(suppressByFields); + selectAlertSuppressionPerInterval(); + setAlertSuppressionDuration(2, 'h'); + + saveEditedRule(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', suppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '2h'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + }); + }); + }); + + describe('with Alert Suppression', () => { + beforeEach(() => { + mlRule = { + ...getMachineLearningRule({ machine_learning_job_id: [jobId] }), + alert_suppression: { + group_by: suppressByFields, + duration: { value: 360, unit: 's' }, + missing_fields_strategy: 'doNotSuppress', + }, + }; + + createRule(mlRule); + visit(RULES_MANAGEMENT_URL); + editFirstRule(); + }); + + it('allows editing of a rule to change its suppression configuration', () => { + // check saved suppression settings + cy.get(ALERT_SUPPRESSION_DURATION_INPUT) + .eq(0) + .should('be.enabled') + .should('have.value', 360); + cy.get(ALERT_SUPPRESSION_DURATION_INPUT) + .eq(1) + .should('be.enabled') + .should('have.value', 's'); + + cy.get(ALERT_SUPPRESSION_FIELDS).should('contain', suppressByFields.join('')); + cy.get(ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS).should('be.checked'); + + // set new duration first to overcome some flaky racing conditions during form save + setAlertSuppressionDuration(2, 'h'); + selectAlertSuppressionPerRuleExecution(); + + saveEditedRule(); + + // check execution duration has changed + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + }); + }); + + it('allows editing of a rule to remove suppression configuration', () => { + // check saved suppression settings + cy.get(ALERT_SUPPRESSION_DURATION_INPUT) + .eq(0) + .should('be.enabled') + .should('have.value', 360); + cy.get(ALERT_SUPPRESSION_DURATION_INPUT) + .eq(1) + .should('be.enabled') + .should('have.value', 's'); + + cy.get(ALERT_SUPPRESSION_FIELDS).should('contain', suppressByFields.join('')); + cy.get(ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS).should('be.checked'); + + // set new duration first to overcome some flaky racing conditions during form save + setAlertSuppressionDuration(2, 'h'); + + clearAlertSuppressionFields(); + saveEditedRule(); + + // check suppression is now absent + cy.get(DEFINITION_DETAILS).within(() => { + assertDetailsNotExist(SUPPRESS_FOR_DETAILS); + assertDetailsNotExist(SUPPRESS_BY_DETAILS); + }); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts index 4b4a9542ff1bc..fca78851ddf03 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts @@ -220,9 +220,17 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () type: 'machine_learning', anomaly_threshold: 65, machine_learning_job_id: ['auth_high_count_logon_events', 'auth_high_count_logon_fails'], + alert_suppression: { + group_by: ['host.name'], + duration: { unit: 'm', value: 5 }, + missing_fields_strategy: 'suppress', + }, }), ['security-rule.query', 'security-rule.language'] - ) as typeof CUSTOM_QUERY_INDEX_PATTERN_RULE; + ) as Omit< + ReturnType, + 'security-rule.query' | 'security-rule.language' + >; const THRESHOLD_RULE_INDEX_PATTERN = createRuleAssetSavedObject({ name: 'Threshold index pattern rule', @@ -500,24 +508,30 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () }); it('Machine learning rule properties', function () { - clickAddElasticRulesButton(); - - openRuleInstallPreview(MACHINE_LEARNING_RULE['security-rule'].name); - - assertCommonPropertiesShown(commonProperties); - const { + name, + alert_suppression: alertSuppression, anomaly_threshold: anomalyThreshold, machine_learning_job_id: machineLearningJobIds, } = MACHINE_LEARNING_RULE['security-rule'] as { + name: string; anomaly_threshold: number; machine_learning_job_id: string[]; + alert_suppression: AlertSuppression; }; + + clickAddElasticRulesButton(); + openRuleInstallPreview(name); + + assertCommonPropertiesShown(commonProperties); + assertMachineLearningPropertiesShown( anomalyThreshold, machineLearningJobIds, this.mlModules ); + + assertAlertSuppressionPropertiesShown(alertSuppression); }); it('Threshold rule properties', () => { diff --git a/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts b/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts index e562a693865e3..5fb869cebc29f 100644 --- a/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts +++ b/x-pack/test/security_solution_cypress/cypress/support/machine_learning.ts @@ -5,8 +5,72 @@ * 2.0. */ +import { ML_GROUP_ID } from '@kbn/security-solution-plugin/common/constants'; import { rootRequest } from '../tasks/api_calls/common'; +/** + * + * Calls the internal ML Module API to set up a module, which installs the jobs + * contained in that module. + * @param moduleName the name of the ML module to set up + * @returns the response from the setup module request + */ +export const executeSetupModuleRequest = ({ moduleName }: { moduleName: string }) => + rootRequest({ + headers: { + 'elastic-api-version': 1, + }, + method: 'POST', + url: `/internal/ml/modules/setup/${moduleName}`, + failOnStatusCode: true, + body: { + prefix: '', + groups: [ML_GROUP_ID], + indexPatternName: 'auditbeat-*', + startDatafeed: false, + useDedicatedIndex: true, + applyToAllSpaces: true, + }, + }); + +/** + * + * Calls the internal ML Jobs API to force start the datafeeds for the given job IDs. Necessary to get them in the "started" state for the purposes of the detection engine + * @param jobIds the job IDs for which to force start datafeeds + * @returns the response from the force start datafeeds request + */ +export const forceStartDatafeeds = ({ jobIds }: { jobIds: string[] }) => + rootRequest({ + headers: { + 'elastic-api-version': 1, + }, + method: 'POST', + url: '/internal/ml/jobs/force_start_datafeeds', + failOnStatusCode: true, + body: { + datafeedIds: jobIds.map((jobId) => `datafeed-${jobId}`), + start: new Date().getUTCMilliseconds(), + }, + }); + +/** + * Calls the internal ML Jobs API to stop the datafeeds for the given job IDs. + * @param jobIds the job IDs for which to stop datafeeds + * @returns the response from the stop datafeeds request + */ +export const stopDatafeeds = ({ jobIds }: { jobIds: string[] }) => + rootRequest({ + headers: { + 'elastic-api-version': 1, + }, + method: 'POST', + url: '/internal/ml/jobs/stop_datafeeds', + failOnStatusCode: true, + body: { + datafeedIds: jobIds.map((jobId) => `datafeed-${jobId}`), + }, + }); + /** * Calls the internal ML Jobs API to force stop the datafeed of, and force close * the job with the given ID. diff --git a/x-pack/test/security_solution_cypress/serverless_config.ts b/x-pack/test/security_solution_cypress/serverless_config.ts index ebdd5d1b333c9..b9f153028e5c8 100644 --- a/x-pack/test/security_solution_cypress/serverless_config.ts +++ b/x-pack/test/security_solution_cypress/serverless_config.ts @@ -37,6 +37,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'alertSuppressionForEsqlRuleEnabled', 'bulkCustomHighlightedFieldsEnabled', + 'alertSuppressionForMachineLearningRuleEnabled', 'manualRuleRunEnabled', ])}`, ], diff --git a/x-pack/test_serverless/shared/lib/security/kibana_roles/project_controller_security_roles.yml b/x-pack/test_serverless/shared/lib/security/kibana_roles/project_controller_security_roles.yml index 9f3220959c486..5f86c7a23ca27 100644 --- a/x-pack/test_serverless/shared/lib/security/kibana_roles/project_controller_security_roles.yml +++ b/x-pack/test_serverless/shared/lib/security/kibana_roles/project_controller_security_roles.yml @@ -34,6 +34,7 @@ viewer: - ".fleet-actions*" - "risk-score.risk-score-*" - ".asset-criticality.asset-criticality-*" + - ".ml-anomalies-*" privileges: - read applications: @@ -100,6 +101,10 @@ editor: - "read" - "write" allow_restricted_indices: false + - names: + - ".ml-anomalies-*" + privileges: + - read applications: - application: "kibana-.kibana" privileges: @@ -155,6 +160,7 @@ t1_analyst: - ".fleet-actions*" - risk-score.risk-score-* - .asset-criticality.asset-criticality-* + - ".ml-anomalies-*" privileges: - read applications: @@ -203,6 +209,7 @@ t2_analyst: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read - names: @@ -265,6 +272,7 @@ t3_analyst: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read applications: @@ -330,6 +338,7 @@ threat_intelligence_analyst: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read applications: @@ -389,6 +398,7 @@ rule_author: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read applications: @@ -454,6 +464,7 @@ soc_manager: - .fleet-agents* - .fleet-actions* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read applications: @@ -515,6 +526,7 @@ detections_admin: - metrics-endpoint.metadata_current_* - .fleet-agents* - .fleet-actions* + - ".ml-anomalies-*" privileges: - read - names: @@ -573,6 +585,10 @@ platform_engineer: privileges: - read - write + - names: + - ".ml-anomalies-*" + privileges: + - read applications: - application: "kibana-.kibana" privileges: @@ -624,6 +640,7 @@ endpoint_operations_analyst: - .lists* - .items* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read - names: @@ -692,6 +709,7 @@ endpoint_policy_manager: - packetbeat-* - winlogbeat-* - risk-score.risk-score-* + - ".ml-anomalies-*" privileges: - read - names: