[RAM][Security Solution][Alerts] Support the ability to trigger a rul…

…e action per alert generated (#153611) (#155384)

## Summary

These changes enable triggering of "per-alert" actions.

Closes #153611

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Ying <ying.mao@elastic.co>
Co-authored-by: Xavier Mouligneau <xavier.mouligneau@elastic.co>

x-pack/plugins/alerting/server/alert/alert.test.ts

@@ -661,27 +661,33 @@ describe('resetPendingRecoveredCount', () => {
 
 describe('isFilteredOut', () => {
   const summarizedAlerts = {
-    all: { count: 1, data: [{ kibana: { alert: { instance: { id: 1 } } } }] },
+    all: { count: 1, data: [{ kibana: { alert: { uuid: '1' } } }] },
     new: { count: 0, data: [] },
     ongoing: { count: 0, data: [] },
     recovered: { count: 0, data: [] },
   };
 
   test('returns false if summarizedAlerts is null', () => {
     const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1', {
-      meta: { pendingRecoveredCount: 3 },
+      meta: { pendingRecoveredCount: 3, uuid: '1' },
     });
     expect(alert.isFilteredOut(null)).toBe(false);
   });
-  test('returns false if the alert is in summarizedAlerts', () => {
+  test('returns false if the alert with same ID is in summarizedAlerts', () => {
     const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1', {
-      meta: { pendingRecoveredCount: 3 },
+      meta: { pendingRecoveredCount: 3, uuid: 'no' },
     });
-    expect(alert.isFilteredOut(null)).toBe(false);
+    expect(alert.isFilteredOut(summarizedAlerts)).toBe(false);
   });
-  test('returns true if the alert is not in summarizedAlerts', () => {
+  test('returns false if the alert with same UUID is in summarizedAlerts', () => {
     const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('2', {
-      meta: { pendingRecoveredCount: 3 },
+      meta: { pendingRecoveredCount: 3, uuid: '1' },
+    });
+    expect(alert.isFilteredOut(summarizedAlerts)).toBe(false);
+  });
+  test('returns true if the alert with same UUID or ID is not in summarizedAlerts', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('2', {
+      meta: { pendingRecoveredCount: 3, uuid: '3' },
     });
     expect(alert.isFilteredOut(summarizedAlerts)).toBe(true);
   });

x-pack/plugins/alerting/server/alert/alert.ts

@@ -7,7 +7,7 @@
 
 import { v4 as uuidV4 } from 'uuid';
 import { get, isEmpty } from 'lodash';
-import { ALERT_INSTANCE_ID, ALERT_RULE_UUID } from '@kbn/rule-data-utils';
+import { ALERT_UUID } from '@kbn/rule-data-utils';
 import { CombinedSummarizedAlerts } from '../types';
 import {
   AlertInstanceMeta,
@@ -271,15 +271,29 @@ export class Alert<
     this.meta.pendingRecoveredCount = 0;
   }
 
+  /**
+   * Checks whether this alert exists in the given alert summary
+   */
   isFilteredOut(summarizedAlerts: CombinedSummarizedAlerts | null) {
     if (summarizedAlerts === null) {
       return false;
     }
 
+    // We check the alert UUID against both the alert ID and the UUID here
+    // The framework generates a UUID for each new reported alert.
+    // For lifecycle rule types, this UUID is written out in the ALERT_UUID field
+    // so we can compare ALERT_UUID to getUuid()
+    // For persistence rule types, the executor generates its own UUID which is a SHA
+    // of the alert data and stores it in the ALERT_UUID and uses it as the alert ID
+    // before reporting the alert back to the framework. The framework then generates
+    // another UUID that is never persisted. For these alerts, we want to compare
+    // ALERT_UUID to getId()
+    //
+    // Related issue: https://github.com/elastic/kibana/issues/144862
+
     return !summarizedAlerts.all.data.some(
       (alert) =>
-        get(alert, ALERT_INSTANCE_ID) === this.getId() ||
-        get(alert, ALERT_RULE_UUID) === this.getId()
+        get(alert, ALERT_UUID) === this.getId() || get(alert, ALERT_UUID) === this.getUuid()
     );
   }
 }

x-pack/plugins/alerting/server/rules_client/lib/get_alert_from_raw.ts

@@ -165,8 +165,7 @@ export function getPartialRuleFromRaw<Params extends RuleTypeParams>(
   // Need the `rule` object to build a URL
   if (!excludeFromPublicApi) {
     const viewInAppRelativeUrl =
-      ruleType.getViewInAppRelativeUrl &&
-      ruleType.getViewInAppRelativeUrl({ rule: rule as Rule<Params> });
+      ruleType.getViewInAppRelativeUrl && ruleType.getViewInAppRelativeUrl({ rule });
     if (viewInAppRelativeUrl) {
       rule.viewInAppRelativeUrl = viewInAppRelativeUrl;
     }

x-pack/plugins/alerting/server/saved_objects/migrations/8.8/index.ts

@@ -10,6 +10,8 @@ import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-p
 import { v4 as uuidv4 } from 'uuid';
 import { createEsoMigration, isDetectionEngineAADRuleType, pipeMigrations } from '../utils';
 import { RawRule } from '../../../types';
+import { transformToAlertThrottle } from '../../../rules_client/lib/siem_legacy_actions/transform_to_alert_throttle';
+import { transformToNotifyWhen } from '../../../rules_client/lib/siem_legacy_actions/transform_to_notify_when';
 
 function addRevision(doc: SavedObjectUnsanitizedDoc<RawRule>): SavedObjectUnsanitizedDoc<RawRule> {
   return {
@@ -42,9 +44,38 @@ function addActionUuid(
   };
 }
 
+function addSecuritySolutionActionsFrequency(
+  doc: SavedObjectUnsanitizedDoc<RawRule>
+): SavedObjectUnsanitizedDoc<RawRule> {
+  if (isDetectionEngineAADRuleType(doc)) {
+    const {
+      attributes: { throttle, actions },
+    } = doc;
+
+    return {
+      ...doc,
+      attributes: {
+        ...doc.attributes,
+        actions: actions
+          ? actions.map((action) => ({
+              ...action,
+              // Till now SIEM worked without action level frequencies. Instead rule level `throttle` and `notifyWhen` used
+              frequency: action.frequency ?? {
+                summary: true,
+                notifyWhen: transformToNotifyWhen(throttle) ?? 'onActiveAlert',
+                throttle: transformToAlertThrottle(throttle),
+              },
+            }))
+          : [],
+      },
+    };
+  }
+  return doc;
+}
+
 export const getMigrations880 = (encryptedSavedObjects: EncryptedSavedObjectsPluginSetup) =>
   createEsoMigration(
     encryptedSavedObjects,
     (doc: SavedObjectUnsanitizedDoc<RawRule>): doc is SavedObjectUnsanitizedDoc<RawRule> => true,
-    pipeMigrations(addActionUuid, addRevision)
+    pipeMigrations(addActionUuid, addRevision, addSecuritySolutionActionsFrequency)
   );

x-pack/plugins/alerting/server/saved_objects/migrations/index.test.ts

@@ -2668,6 +2668,52 @@ describe('successful migrations', () => {
       const migratedAlert880 = migration880(rule, migrationContext);
       expect(migratedAlert880.attributes.revision).toEqual(2);
     });
+
+    describe('migrate actions frequency for Security Solution ', () => {
+      test('Add frequency when throttle is null', () => {
+        const migration880 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[
+          '8.8.0'
+        ];
+
+        const rule = getMockData({ alertTypeId: ruleTypeMappings.eql });
+        const migratedAlert880 = migration880(rule, migrationContext);
+        expect(migratedAlert880.attributes.actions[0].frequency.summary).toEqual(true);
+        expect(migratedAlert880.attributes.actions[0].frequency.notifyWhen).toEqual(
+          'onActiveAlert'
+        );
+        expect(migratedAlert880.attributes.actions[0].frequency.throttle).toEqual(null);
+      });
+
+      test('Add frequency when throttle is 1h', () => {
+        const migration880 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[
+          '8.8.0'
+        ];
+
+        const rule = getMockData({ alertTypeId: ruleTypeMappings.eql, throttle: '1h' });
+        const migratedAlert880 = migration880(rule, migrationContext);
+        expect(migratedAlert880.attributes.actions[0].frequency.summary).toEqual(true);
+        expect(migratedAlert880.attributes.actions[0].frequency.notifyWhen).toEqual(
+          'onThrottleInterval'
+        );
+        expect(migratedAlert880.attributes.actions[0].frequency.throttle).toEqual('1h');
+      });
+
+      test('Do not migrate action when alert does NOT belong to security solution', () => {
+        const migration880 = getMigrations(encryptedSavedObjectsSetup, {}, isPreconfigured)[
+          '8.8.0'
+        ];
+
+        const rule = getMockData();
+        const migratedAlert880 = migration880(rule, migrationContext);
+        const updatedActions = migratedAlert880.attributes.actions.map(
+          (action: { [x: string]: unknown; uuid: unknown }) => {
+            const { uuid, ...updatedAction } = action;
+            return updatedAction;
+          }
+        );
+        expect(updatedActions).toEqual(rule.attributes.actions);
+      });
+    });
   });
 
   describe('Metrics Inventory Threshold rule', () => {

x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts

@@ -1366,7 +1366,7 @@ describe('Execution Handler', () => {
     getSummarizedAlertsMock.mockResolvedValue({
       new: {
         count: 1,
-        data: [{ ...mockAAD, kibana: { alert: { instance: { id: '1' } } } }],
+        data: [{ ...mockAAD, kibana: { alert: { uuid: '1' } } }],
       },
       ongoing: {
         count: 0,

x-pack/plugins/alerting/server/types.ts

@@ -169,7 +169,8 @@ export interface CombinedSummarizedAlerts extends SummarizedAlerts {
 }
 export type GetSummarizedAlertsFn = (opts: GetSummarizedAlertsFnOpts) => Promise<SummarizedAlerts>;
 export interface GetViewInAppRelativeUrlFnOpts<Params extends RuleTypeParams> {
-  rule: Omit<SanitizedRule<Params>, 'viewInAppRelativeUrl'>;
+  rule: Pick<SanitizedRule<Params>, 'id'> &
+    Omit<Partial<SanitizedRule<Params>>, 'viewInAppRelativeUrl'>;
   // Optional time bounds
   start?: number;
   end?: number;

x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts

@@ -12,19 +12,24 @@ import { chunk, partition } from 'lodash';
 import {
   ALERT_INSTANCE_ID,
   ALERT_LAST_DETECTED,
+  ALERT_NAMESPACE,
   ALERT_START,
   ALERT_SUPPRESSION_DOCS_COUNT,
   ALERT_SUPPRESSION_END,
   ALERT_UUID,
   ALERT_WORKFLOW_STATUS,
+  TIMESTAMP,
   VERSION,
 } from '@kbn/rule-data-utils';
+import { mapKeys, snakeCase } from 'lodash/fp';
 import { getCommonAlertFields } from './get_common_alert_fields';
 import { CreatePersistenceRuleTypeWrapper } from './persistence_types';
 import { errorAggregator } from './utils';
-import { createGetSummarizedAlertsFn } from './create_get_summarized_alerts_fn';
+import { AlertDocument, createGetSummarizedAlertsFn } from './create_get_summarized_alerts_fn';
 import { AlertWithSuppressionFields870 } from '../../common/schemas/8.7.0';
 
+export const ALERT_GROUP_INDEX = `${ALERT_NAMESPACE}.group.index` as const;
+
 const augmentAlerts = <T>({
   alerts,
   options,
@@ -56,7 +61,7 @@ const mapAlertsToBulkCreate = <T>(alerts: Array<{ _id: string; _source: T }>) =>
 };
 
 export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper =
-  ({ logger, ruleDataClient }) =>
+  ({ logger, ruleDataClient, formatAlert }) =>
   (type) => {
     return {
       ...type,
@@ -162,17 +167,40 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                   return { createdAlerts: [], errors: {}, alertsWereTruncated };
                 }
 
-                return {
-                  createdAlerts: augmentedAlerts
-                    .map((alert, idx) => {
-                      const responseItem = response.body.items[idx].create;
-                      return {
-                        _id: responseItem?._id ?? '',
-                        _index: responseItem?._index ?? '',
-                        ...alert._source,
-                      };
+                const createdAlerts = augmentedAlerts
+                  .map((alert, idx) => {
+                    const responseItem = response.body.items[idx].create;
+                    return {
+                      _id: responseItem?._id ?? '',
+                      _index: responseItem?._index ?? '',
+                      ...alert._source,
+                    };
+                  })
+                  .filter((_, idx) => response.body.items[idx].create?.status === 201)
+                  // Security solution's EQL rule consists of building block alerts which should be filtered out.
+                  // Building block alerts have additional "kibana.alert.group.index" attribute which is absent for the root alert.
+                  .filter((alert) => !Object.keys(alert).includes(ALERT_GROUP_INDEX));
+
+                createdAlerts.forEach((alert) =>
+                  options.services.alertFactory
+                    .create(alert._id)
+                    .scheduleActions(type.defaultActionGroupId, {
+                      rule: mapKeys(snakeCase, {
+                        ...options.params,
+                        name: options.rule.name,
+                        id: options.rule.id,
+                      }),
+                      results_link: type.getViewInAppRelativeUrl?.({
+                        rule: { ...options.rule, params: options.params },
+                        start: Date.parse(alert[TIMESTAMP]),
+                        end: Date.parse(alert[TIMESTAMP]),
+                      }),
+                      alerts: [alert],
                     })
-                    .filter((_, idx) => response.body.items[idx].create?.status === 201),
+                );
+
+                return {
+                  createdAlerts,
                   errors: errorAggregator(response.body, [409]),
                   alertsWereTruncated,
                 };
@@ -327,21 +355,44 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                   return { createdAlerts: [], errors: {} };
                 }
 
-                return {
-                  createdAlerts: augmentedAlerts
-                    .map((alert, idx) => {
-                      const responseItem =
-                        bulkResponse.body.items[idx + duplicateAlerts.length].create;
-                      return {
-                        _id: responseItem?._id ?? '',
-                        _index: responseItem?._index ?? '',
-                        ...alert._source,
-                      };
+                const createdAlerts = augmentedAlerts
+                  .map((alert, idx) => {
+                    const responseItem =
+                      bulkResponse.body.items[idx + duplicateAlerts.length].create;
+                    return {
+                      _id: responseItem?._id ?? '',
+                      _index: responseItem?._index ?? '',
+                      ...alert._source,
+                    };
+                  })
+                  .filter(
+                    (_, idx) =>
+                      bulkResponse.body.items[idx + duplicateAlerts.length].create?.status === 201
+                  )
+                  // Security solution's EQL rule consists of building block alerts which should be filtered out.
+                  // Building block alerts have additional "kibana.alert.group.index" attribute which is absent for the root alert.
+                  .filter((alert) => !Object.keys(alert).includes(ALERT_GROUP_INDEX));
+
+                createdAlerts.forEach((alert) =>
+                  options.services.alertFactory
+                    .create(alert._id)
+                    .scheduleActions(type.defaultActionGroupId, {
+                      rule: mapKeys(snakeCase, {
+                        ...options.params,
+                        name: options.rule.name,
+                        id: options.rule.id,
+                      }),
+                      results_link: type.getViewInAppRelativeUrl?.({
+                        rule: { ...options.rule, params: options.params },
+                        start: Date.parse(alert[TIMESTAMP]),
+                        end: Date.parse(alert[TIMESTAMP]),
+                      }),
+                      alerts: [alert],
                     })
-                    .filter(
-                      (_, idx) =>
-                        bulkResponse.body.items[idx + duplicateAlerts.length].create?.status === 201
-                    ),
+                );
+
+                return {
+                  createdAlerts,
                   errors: errorAggregator(bulkResponse.body, [409]),
                 };
               } else {
@@ -358,6 +409,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
         ruleDataClient,
         useNamespace: true,
         isLifecycleAlert: false,
+        formatAlert: formatAlert as (alert: AlertDocument) => AlertDocument,
       })(),
     };
   };

x-pack/plugins/rule_registry/server/utils/persistence_types.ts

@@ -90,11 +90,7 @@ export type PersistenceAlertType<
 export type CreatePersistenceRuleTypeWrapper = (options: {
   ruleDataClient: IRuleDataClient;
   logger: Logger;
-}) => <
-  TParams extends RuleTypeParams,
-  TState extends RuleTypeState,
-  TInstanceContext extends AlertInstanceContext = {},
-  TActionGroupIds extends string = never
->(
-  type: PersistenceAlertType<TParams, TState, TInstanceContext, TActionGroupIds>
-) => RuleType<TParams, TParams, TState, AlertInstanceState, TInstanceContext, TActionGroupIds>;
+  formatAlert?: (alert: unknown) => unknown;
+}) => <TParams extends RuleTypeParams, TState extends RuleTypeState>(
+  type: PersistenceAlertType<TParams, TState, AlertInstanceContext, 'default'>
+) => RuleType<TParams, TParams, TState, AlertInstanceState, AlertInstanceContext, 'default'>;

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx

@@ -565,6 +565,7 @@ const CreateRulePageComponent: React.FC = () => {
                             setForm={setFormHook}
                             onSubmit={() => submitStep(RuleStep.ruleActions)}
                             actionMessageParams={actionMessageParams}
+                            summaryActionMessageParams={actionMessageParams}
                             // We need a key to make this component remount when edit/view mode is toggled
                             // https://github.com/elastic/kibana/pull/132834#discussion_r881705566
                             key={isShouldRerenderStep(RuleStep.ruleActions, activeStep)}

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx

@@ -310,6 +310,7 @@ const EditRulePageComponent: FC = () => {
                   defaultValues={actionsStep.data}
                   setForm={setFormHook}
                   actionMessageParams={actionMessageParams}
+                  summaryActionMessageParams={actionMessageParams}
                   ruleType={rule?.type}
                 />
               )}