Add circuit breaker for max number of actions by connector type (elas…

…tic#128319)

* connectorTypeOverrides key in kibana.yml can create a connector type specific action config.

* Update docs and docker allowed keys

docs/settings/alert-action-settings.asciidoc

@@ -211,13 +211,26 @@ For example, `20m`, `24h`, `7d`, `1w`. Default: `5m`.
 `xpack.alerting.rules.run.ruleTypeOverrides`::
 Overrides the configs under `xpack.alerting.rules.run` for the rule type with the given ID. List the rule identifier and its settings in an array of objects.
 +
---
 For example:
-```
+[source,yaml]
+--
 xpack.alerting.rules.run:
     timeout: '5m'
     ruleTypeOverrides:
         - id: '.index-threshold'
           timeout: '15m'
-```
---
+--
+
+`xpack.alerting.rules.run.actions.connectorTypeOverrides`::
+Overrides the configs under `xpack.alerting.rules.run.actions` for the connector type with the given ID. List the connector type identifier and its settings in an array of objects.
++
+For example:
+[source,yaml]
+--
+xpack.alerting.rules.run:
+    actions:
+        max: 10
+        connectorTypeOverrides:
+            - id: '.server-log'
+              max: 5
+--

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -211,6 +211,7 @@ kibana_vars=(
     xpack.alerting.rules.minimumScheduleInterval.value
     xpack.alerting.rules.minimumScheduleInterval.enforce
     xpack.alerting.rules.run.actions.max
+    xpack.alerting.rules.run.actions.connectorTypeOverrides
     xpack.alerts.healthCheck.interval
     xpack.alerts.invalidateApiKeysTask.interval
     xpack.alerts.invalidateApiKeysTask.removalDelay

x-pack/plugins/alerting/common/rule_task_instance.ts

@@ -22,7 +22,7 @@ const ruleExecutionMetricsSchema = t.partial({
   esSearchDurationMs: t.number,
 });
 
-const alertExecutionStore = t.partial({
+const alertExecutionMetrics = t.partial({
   numberOfTriggeredActions: t.number,
   numberOfGeneratedActions: t.number,
   triggeredActionsStatus: t.string,
@@ -32,7 +32,7 @@ export type RuleExecutionMetrics = t.TypeOf<typeof ruleExecutionMetricsSchema>;
 export type RuleTaskState = t.TypeOf<typeof ruleStateSchema>;
 export type RuleExecutionState = RuleTaskState & {
   metrics: RuleExecutionMetrics;
-  alertExecutionStore: t.TypeOf<typeof alertExecutionStore>;
+  alertExecutionMetrics: t.TypeOf<typeof alertExecutionMetrics>;
 };
 
 export const ruleParamsSchema = t.intersection([

x-pack/plugins/alerting/server/config.ts

@@ -14,6 +14,11 @@ const ruleTypeSchema = schema.object({
   timeout: schema.maybe(schema.string({ validate: validateDurationSchema })),
 });
 
+const connectorTypeSchema = schema.object({
+  id: schema.string(),
+  max: schema.maybe(schema.number({ max: 100000 })),
+});
+
 const rulesSchema = schema.object({
   minimumScheduleInterval: schema.object({
     value: schema.string({
@@ -36,6 +41,7 @@ const rulesSchema = schema.object({
     timeout: schema.maybe(schema.string({ validate: validateDurationSchema })),
     actions: schema.object({
       max: schema.number({ defaultValue: 100000, max: 100000 }),
+      connectorTypeOverrides: schema.maybe(schema.arrayOf(connectorTypeSchema)),
     }),
     ruleTypeOverrides: schema.maybe(schema.arrayOf(ruleTypeSchema)),
   }),
@@ -59,5 +65,6 @@ export const configSchema = schema.object({
 
 export type AlertingConfig = TypeOf<typeof configSchema>;
 export type RulesConfig = TypeOf<typeof rulesSchema>;
-export type RuleTypeConfig = Omit<RulesConfig, 'ruleTypeOverrides' | 'minimumScheduleInterval'>;
 export type AlertingRulesConfig = Pick<AlertingConfig['rules'], 'minimumScheduleInterval'>;
+export type ActionsConfig = RulesConfig['run']['actions'];
+export type ActionTypeConfig = Omit<ActionsConfig, 'connectorTypeOverrides'>;

x-pack/plugins/alerting/server/lib/alert_execution_store.test.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AlertExecutionStore } from './alert_execution_store';
+import { ActionsCompletion } from '../task_runner/types';
+
+describe('AlertExecutionStore', () => {
+  const alertExecutionStore = new AlertExecutionStore();
+  const testConnectorId = 'test-connector-id';
+
+  // Getter Setter
+  test('returns the default values if there is no change', () => {
+    expect(alertExecutionStore.getTriggeredActionsStatus()).toBe(ActionsCompletion.COMPLETE);
+    expect(alertExecutionStore.getNumberOfTriggeredActions()).toBe(0);
+    expect(alertExecutionStore.getNumberOfGeneratedActions()).toBe(0);
+    expect(alertExecutionStore.getStatusByConnectorType('any')).toBe(undefined);
+  });
+
+  test('sets and returns numberOfTriggeredActions', () => {
+    alertExecutionStore.setNumberOfTriggeredActions(5);
+    expect(alertExecutionStore.getNumberOfTriggeredActions()).toBe(5);
+  });
+
+  test('sets and returns numberOfGeneratedActions', () => {
+    alertExecutionStore.setNumberOfGeneratedActions(15);
+    expect(alertExecutionStore.getNumberOfGeneratedActions()).toBe(15);
+  });
+
+  test('sets and returns triggeredActionsStatusByConnectorType', () => {
+    alertExecutionStore.setTriggeredActionsStatusByConnectorType({
+      actionTypeId: testConnectorId,
+      status: ActionsCompletion.PARTIAL,
+    });
+    expect(
+      alertExecutionStore.getStatusByConnectorType(testConnectorId).triggeredActionsStatus
+    ).toBe(ActionsCompletion.PARTIAL);
+    expect(alertExecutionStore.getTriggeredActionsStatus()).toBe(ActionsCompletion.PARTIAL);
+  });
+
+  // increment
+  test('increments numberOfTriggeredActions by 1', () => {
+    alertExecutionStore.incrementNumberOfTriggeredActions();
+    expect(alertExecutionStore.getNumberOfTriggeredActions()).toBe(6);
+  });
+
+  test('increments incrementNumberOfGeneratedActions by x', () => {
+    alertExecutionStore.incrementNumberOfGeneratedActions(2);
+    expect(alertExecutionStore.getNumberOfGeneratedActions()).toBe(17);
+  });
+
+  test('increments numberOfTriggeredActionsByConnectorType by 1', () => {
+    alertExecutionStore.incrementNumberOfTriggeredActionsByConnectorType(testConnectorId);
+    expect(
+      alertExecutionStore.getStatusByConnectorType(testConnectorId).numberOfTriggeredActions
+    ).toBe(1);
+  });
+
+  test('increments NumberOfGeneratedActionsByConnectorType by 1', () => {
+    alertExecutionStore.incrementNumberOfGeneratedActionsByConnectorType(testConnectorId);
+    expect(
+      alertExecutionStore.getStatusByConnectorType(testConnectorId).numberOfGeneratedActions
+    ).toBe(1);
+  });
+
+  // Checker
+  test('checks if it has reached the executable actions limit', () => {
+    expect(alertExecutionStore.hasReachedTheExecutableActionsLimit({ default: { max: 10 } })).toBe(
+      false
+    );
+
+    expect(alertExecutionStore.hasReachedTheExecutableActionsLimit({ default: { max: 5 } })).toBe(
+      true
+    );
+  });
+
+  test('checks if it has reached the executable actions limit by connector type', () => {
+    alertExecutionStore.incrementNumberOfTriggeredActionsByConnectorType(testConnectorId);
+    alertExecutionStore.incrementNumberOfTriggeredActionsByConnectorType(testConnectorId);
+    alertExecutionStore.incrementNumberOfTriggeredActionsByConnectorType(testConnectorId);
+    alertExecutionStore.incrementNumberOfTriggeredActionsByConnectorType(testConnectorId);
+    alertExecutionStore.incrementNumberOfTriggeredActionsByConnectorType(testConnectorId);
+
+    expect(
+      alertExecutionStore.hasReachedTheExecutableActionsLimitByConnectorType({
+        actionsConfigMap: {
+          default: { max: 20 },
+          [testConnectorId]: {
+            max: 5,
+          },
+        },
+        actionTypeId: testConnectorId,
+      })
+    ).toBe(true);
+
+    expect(
+      alertExecutionStore.hasReachedTheExecutableActionsLimitByConnectorType({
+        actionsConfigMap: {
+          default: { max: 20 },
+          [testConnectorId]: {
+            max: 8,
+          },
+        },
+        actionTypeId: testConnectorId,
+      })
+    ).toBe(false);
+  });
+
+  test('checks if a connector type it has already reached the executable actions limit', () => {
+    expect(alertExecutionStore.hasConnectorTypeReachedTheLimit(testConnectorId)).toBe(true);
+  });
+});

x-pack/plugins/alerting/server/lib/alert_execution_store.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { set } from 'lodash';
+import { ActionsConfigMap } from './get_actions_config_map';
+import { ActionsCompletion } from '../task_runner/types';
+
+interface State {
+  numberOfTriggeredActions: number;
+  numberOfGeneratedActions: number;
+  connectorTypes: {
+    [key: string]: {
+      triggeredActionsStatus: ActionsCompletion;
+      numberOfTriggeredActions: number;
+      numberOfGeneratedActions: number;
+    };
+  };
+}
+
+export class AlertExecutionStore {
+  private state: State = {
+    numberOfTriggeredActions: 0,
+    numberOfGeneratedActions: 0,
+    connectorTypes: {},
+  };
+
+  // Getters
+  public getTriggeredActionsStatus = () => {
+    const hasPartial = Object.values(this.state.connectorTypes).some(
+      (connectorType) => connectorType?.triggeredActionsStatus === ActionsCompletion.PARTIAL
+    );
+    return hasPartial ? ActionsCompletion.PARTIAL : ActionsCompletion.COMPLETE;
+  };
+  public getNumberOfTriggeredActions = () => {
+    return this.state.numberOfTriggeredActions;
+  };
+  public getNumberOfGeneratedActions = () => {
+    return this.state.numberOfGeneratedActions;
+  };
+  public getStatusByConnectorType = (actionTypeId: string) => {
+    return this.state.connectorTypes[actionTypeId];
+  };
+
+  // Setters
+  public setNumberOfTriggeredActions = (numberOfTriggeredActions: number) => {
+    this.state.numberOfTriggeredActions = numberOfTriggeredActions;
+  };
+
+  public setNumberOfGeneratedActions = (numberOfGeneratedActions: number) => {
+    this.state.numberOfGeneratedActions = numberOfGeneratedActions;
+  };
+
+  public setTriggeredActionsStatusByConnectorType = ({
+    actionTypeId,
+    status,
+  }: {
+    actionTypeId: string;
+    status: ActionsCompletion;
+  }) => {
+    set(this.state, `connectorTypes["${actionTypeId}"].triggeredActionsStatus`, status);
+  };
+
+  // Checkers
+  public hasReachedTheExecutableActionsLimit = (actionsConfigMap: ActionsConfigMap): boolean =>
+    this.state.numberOfTriggeredActions >= actionsConfigMap.default.max;
+
+  public hasReachedTheExecutableActionsLimitByConnectorType = ({
+    actionsConfigMap,
+    actionTypeId,
+  }: {
+    actionsConfigMap: ActionsConfigMap;
+    actionTypeId: string;
+  }): boolean => {
+    const numberOfTriggeredActionsByConnectorType =
+      this.state.connectorTypes[actionTypeId]?.numberOfTriggeredActions || 0;
+    const executableActionsLimitByConnectorType =
+      actionsConfigMap[actionTypeId]?.max || actionsConfigMap.default.max;
+
+    return numberOfTriggeredActionsByConnectorType >= executableActionsLimitByConnectorType;
+  };
+
+  public hasConnectorTypeReachedTheLimit = (actionTypeId: string) =>
+    this.state.connectorTypes[actionTypeId]?.triggeredActionsStatus === ActionsCompletion.PARTIAL;
+
+  // Incrementer
+  public incrementNumberOfTriggeredActions = () => {
+    this.state.numberOfTriggeredActions++;
+  };
+
+  public incrementNumberOfGeneratedActions = (incrementBy: number) => {
+    this.state.numberOfGeneratedActions += incrementBy;
+  };
+
+  public incrementNumberOfTriggeredActionsByConnectorType = (actionTypeId: string) => {
+    const currentVal = this.state.connectorTypes[actionTypeId]?.numberOfTriggeredActions || 0;
+    set(this.state, `connectorTypes["${actionTypeId}"].numberOfTriggeredActions`, currentVal + 1);
+  };
+  public incrementNumberOfGeneratedActionsByConnectorType = (actionTypeId: string) => {
+    const currentVal = this.state.connectorTypes[actionTypeId]?.numberOfGeneratedActions || 0;
+    set(this.state, `connectorTypes["${actionTypeId}"].numberOfGeneratedActions`, currentVal + 1);
+  };
+}

x-pack/plugins/alerting/server/lib/get_actions_config_map.test.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getActionsConfigMap } from './get_actions_config_map';
+
+const connectorTypeId = 'test-connector-type-id';
+const actionsConfig = {
+  max: 1000,
+};
+
+const actionsConfigWithConnectorType = {
+  ...actionsConfig,
+  connectorTypeOverrides: [
+    {
+      id: connectorTypeId,
+      max: 20,
+    },
+  ],
+};
+
+describe('get actions config map', () => {
+  test('returns the default actions config', () => {
+    expect(getActionsConfigMap(actionsConfig)).toEqual({
+      default: {
+        max: 1000,
+      },
+    });
+  });
+
+  test('applies the connector type specific config', () => {
+    expect(getActionsConfigMap(actionsConfigWithConnectorType)).toEqual({
+      default: {
+        max: 1000,
+      },
+      [connectorTypeId]: {
+        max: 20,
+      },
+    });
+  });
+});

x-pack/plugins/alerting/server/lib/get_actions_config_map.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { omit } from 'lodash';
+import { ActionsConfig, ActionTypeConfig } from '../config';
+
+export interface ActionsConfigMap {
+  default: ActionTypeConfig;
+  [key: string]: ActionTypeConfig;
+}
+
+export const getActionsConfigMap = (actionsConfig: ActionsConfig): ActionsConfigMap => {
+  const configsByConnectorType = actionsConfig.connectorTypeOverrides?.reduce(
+    (config, configByConnectorType) => {
+      return { ...config, [configByConnectorType.id]: omit(configByConnectorType, 'id') };
+    },
+    {}
+  );
+  return {
+    default: omit(actionsConfig, 'connectorTypeOverrides'),
+    ...configsByConnectorType,
+  };
+};