Adds escalation step to notify a team

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## v1.3.106 (2024-02-14)
+
+### Fixed
+
+- Add escalation to notify entire Grafana team ([#3477](https://github.com/grafana/oncall/pull/3477))
+
 ## v1.3.105 (2024-02-13)
 
 ### Fixed

docs/sources/configure/escalation-chains-and-routes/index.md

@@ -80,6 +80,7 @@ need a larger time interval, use multiple wait steps in a row.
 * `Notify users` - send a notification to a user or a group of users.
 * `Notify users from on-call schedule` - send a notification to a user or a group of users
 from an on-call schedule.
+* `Notify all users from a team` - send a notification to all users in a team.
 * `Resolve incident automatically` - resolve the alert group right now with status
 `Resolved automatically`.
 * `Notify whole slack channel` - send a notification to the users in the slack channel.

engine/apps/alerts/escalation_snapshot/escalation_snapshot_mixin.py

@@ -67,6 +67,7 @@ def build_raw_escalation_snapshot(self) -> dict:
                     'wait_delay': None,
                     'notify_schedule': None,
                     'notify_to_group': None,
+                    'notify_to_team_members': None,
                     'passed_last_time': None,
                     'escalation_counter': 0,
                     'last_notified_user': None,
@@ -84,6 +85,7 @@ def build_raw_escalation_snapshot(self) -> dict:
                     'wait_delay': '00:05:00',
                     'notify_schedule': None,
                     'notify_to_group': None,
+                    'notify_to_team_members': None,
                     'passed_last_time': None,
                     'escalation_counter': 0,
                     'last_notified_user': None,

engine/apps/alerts/escalation_snapshot/serializers/escalation_policy_snapshot.py

@@ -83,6 +83,7 @@ class Meta:
             "custom_webhook",
             "notify_schedule",
             "notify_to_group",
+            "notify_to_team_members",
             "escalation_counter",
             "passed_last_time",
             "pause_escalation",

engine/apps/alerts/escalation_snapshot/snapshot_classes/escalation_policy_snapshot.py

@@ -15,6 +15,7 @@
     custom_webhook_result,
     notify_all_task,
     notify_group_task,
+    notify_team_members_task,
     notify_user_task,
     resolve_by_last_step_task,
 )
@@ -41,6 +42,7 @@ class EscalationPolicySnapshot:
         "custom_webhook",
         "notify_schedule",
         "notify_to_group",
+        "notify_to_team_members",
         "escalation_counter",
         "passed_last_time",
         "pause_escalation",
@@ -72,6 +74,7 @@ def __init__(
         escalation_counter,
         passed_last_time,
         pause_escalation,
+        notify_to_team_members=None,
     ):
         self.id = id
         self.order = order
@@ -87,6 +90,7 @@ def __init__(
         self.custom_webhook = custom_webhook
         self.notify_schedule = notify_schedule
         self.notify_to_group = notify_to_group
+        self.notify_to_team_members = notify_to_team_members
         self.escalation_counter = escalation_counter  # used for STEP_REPEAT_ESCALATION_N_TIMES
         self.passed_last_time = passed_last_time  # used for building escalation plan
         self.pause_escalation = pause_escalation  # used for STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW
@@ -124,6 +128,8 @@ def execute(self, alert_group: "AlertGroup", reason) -> StepExecutionResultData:
             EscalationPolicy.STEP_FINAL_RESOLVE: self._escalation_step_resolve,
             EscalationPolicy.STEP_NOTIFY_GROUP: self._escalation_step_notify_user_group,
             EscalationPolicy.STEP_NOTIFY_GROUP_IMPORTANT: self._escalation_step_notify_user_group,
+            EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS: self._escalation_step_notify_team_members,
+            EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT: self._escalation_step_notify_team_members,
             EscalationPolicy.STEP_NOTIFY_SCHEDULE: self._escalation_step_notify_on_call_schedule,
             EscalationPolicy.STEP_NOTIFY_SCHEDULE_IMPORTANT: self._escalation_step_notify_on_call_schedule,
             EscalationPolicy.STEP_TRIGGER_CUSTOM_BUTTON: self._escalation_step_trigger_custom_button,
@@ -358,6 +364,34 @@ def _escalation_step_notify_user_group(self, alert_group: "AlertGroup", reason:
             tasks.append(notify_group)
         self._execute_tasks(tasks)
 
+    def _escalation_step_notify_team_members(self, alert_group: "AlertGroup", reason: str) -> None:
+        tasks = []
+
+        if self.notify_to_team_members is None:
+            log_record = AlertGroupLogRecord(
+                type=AlertGroupLogRecord.TYPE_ESCALATION_FAILED,
+                alert_group=alert_group,
+                reason=reason,
+                escalation_policy=self.escalation_policy,
+                escalation_error_code=AlertGroupLogRecord.ERROR_ESCALATION_NOTIFY_TEAM_MEMBERS_STEP_IS_NOT_CONFIGURED,
+                escalation_policy_step=self.step,
+            )
+            log_record.save()
+        else:
+            notify_team_members = notify_team_members_task.signature(
+                args=(
+                    self.notify_to_team_members.pk,
+                    alert_group.pk,
+                ),
+                kwargs={
+                    "reason": reason,
+                    "important": self.step == EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT,
+                },
+                immutable=True,
+            )
+            tasks.append(notify_team_members)
+        self._execute_tasks(tasks)
+
     def _escalation_step_notify_if_time(self, alert_group: "AlertGroup", _reason: str) -> StepExecutionResultData:
         eta = None
 

engine/apps/alerts/migrations/0045_add_notify_to_team.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.19 on 2023-07-05 16:19
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('alerts', '0044_alertreceivechannel_alertmanager_v2_backup_templates_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='escalationpolicy',
+            name='notify_to_team_members',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='escalation_policies', to='user_management.Team'),
+        ),
+    ]

engine/apps/alerts/models/alert_group_log_record.py

@@ -159,7 +159,8 @@ class AlertGroupLogRecord(models.Model):
         ERROR_ESCALATION_NOTIFY_IN_SLACK,
         ERROR_ESCALATION_NOTIFY_IF_NUM_ALERTS_IN_WINDOW_STEP_IS_NOT_CONFIGURED,
         ERROR_ESCALATION_TRIGGER_CUSTOM_WEBHOOK_ERROR,
-    ) = range(18)
+        ERROR_ESCALATION_NOTIFY_TEAM_MEMBERS_STEP_IS_NOT_CONFIGURED,
+    ) = range(19)
 
     type = models.IntegerField(choices=TYPE_CHOICES)
 
@@ -519,6 +520,8 @@ def rendered_log_line_action(self, for_slack=False, html=False, substitute_autho
                 result += 'skipped escalation step "Notify Schedule" because it is not configured'
             elif self.escalation_error_code == AlertGroupLogRecord.ERROR_ESCALATION_NOTIFY_GROUP_STEP_IS_NOT_CONFIGURED:
                 result += 'skipped escalation step "Notify Group" because it is not configured'
+            elif self.escalation_error_code == AlertGroupLogRecord.ERROR_ESCALATION_NOTIFY_TEAM_MEMBERS_STEP_IS_NOT_CONFIGURED:
+                result += 'skipped escalation step "Notify Team Members" because it is not configured'
             elif (
                 self.escalation_error_code
                 == AlertGroupLogRecord.ERROR_ESCALATION_TRIGGER_CUSTOM_BUTTON_STEP_IS_NOT_CONFIGURED

engine/apps/alerts/models/escalation_policy.py

@@ -45,7 +45,9 @@ class EscalationPolicy(OrderedModel):
         STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT,
         STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW,
         STEP_TRIGGER_CUSTOM_WEBHOOK,
-    ) = range(17)
+        STEP_NOTIFY_TEAM_MEMBERS,
+        STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT,
+    ) = range(19)
 
     # Must be the same order as previous
     STEP_CHOICES = (
@@ -66,6 +68,8 @@ class EscalationPolicy(OrderedModel):
         (STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT, "Notify multiple Users (Important)"),
         (STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW, "Continue escalation if >X alerts per Y minutes"),
         (STEP_TRIGGER_CUSTOM_WEBHOOK, "Trigger Webhook"),
+        (STEP_NOTIFY_TEAM_MEMBERS, "Notify all users in a Team"),
+        (STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT, "Notify all users in a Team (Important)"),
     )
 
     # Ordered step choices available for internal api.
@@ -74,6 +78,7 @@ class EscalationPolicy(OrderedModel):
         # Common
         STEP_WAIT,
         STEP_NOTIFY_MULTIPLE_USERS,
+        STEP_NOTIFY_TEAM_MEMBERS,
         STEP_NOTIFY_SCHEDULE,
         STEP_FINAL_RESOLVE,
         # Slack
@@ -100,6 +105,8 @@ class EscalationPolicy(OrderedModel):
         STEP_NOTIFY_USERS_QUEUE,
         STEP_NOTIFY_IF_TIME,
         STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW,
+        STEP_NOTIFY_TEAM_MEMBERS,
+        STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT,
         STEP_NOTIFY_MULTIPLE_USERS,
         STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT,
         STEP_TRIGGER_CUSTOM_BUTTON,
@@ -113,6 +120,7 @@ class EscalationPolicy(OrderedModel):
         # Common steps
         STEP_WAIT: ("Wait {{wait_delay}} minute(s)", "Wait"),
         STEP_NOTIFY_MULTIPLE_USERS: ("Start {{importance}} notification for {{users}}", "Notify users"),
+        STEP_NOTIFY_TEAM_MEMBERS: ("Start {{importance}} notification for {{team}} team members", "Notify all team members"),
         STEP_NOTIFY_SCHEDULE: (
             "Start {{importance}} notification for schedule {{schedule}}",
             "Notify users from on-call schedule",
@@ -157,24 +165,28 @@ class EscalationPolicy(OrderedModel):
         STEP_NOTIFY_GROUP: STEP_NOTIFY_GROUP_IMPORTANT,
         STEP_NOTIFY_SCHEDULE: STEP_NOTIFY_SCHEDULE_IMPORTANT,
         STEP_NOTIFY_MULTIPLE_USERS: STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT,
+        STEP_NOTIFY_TEAM_MEMBERS: STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT,
     }
     IMPORTANT_TO_DEFAULT_STEP_MAPPING = {
         STEP_NOTIFY_GROUP_IMPORTANT: STEP_NOTIFY_GROUP,
         STEP_NOTIFY_SCHEDULE_IMPORTANT: STEP_NOTIFY_SCHEDULE,
         STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT: STEP_NOTIFY_MULTIPLE_USERS,
+        STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT: STEP_NOTIFY_TEAM_MEMBERS,
     }
 
     # Default steps are just usual version of important steps. E.g. notify group - notify group important
     DEFAULT_STEPS_SET = {
         STEP_NOTIFY_GROUP,
         STEP_NOTIFY_SCHEDULE,
         STEP_NOTIFY_MULTIPLE_USERS,
+        STEP_NOTIFY_TEAM_MEMBERS,
     }
 
     IMPORTANT_STEPS_SET = {
         STEP_NOTIFY_GROUP_IMPORTANT,
         STEP_NOTIFY_SCHEDULE_IMPORTANT,
         STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT,
+        STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT,
     }
 
     SLACK_INTEGRATION_REQUIRED_STEPS = [
@@ -187,6 +199,7 @@ class EscalationPolicy(OrderedModel):
         STEP_WAIT,
         STEP_NOTIFY_SCHEDULE,
         STEP_NOTIFY_MULTIPLE_USERS,
+        STEP_NOTIFY_TEAM_MEMBERS,
         STEP_NOTIFY_USERS_QUEUE,
         STEP_NOTIFY_GROUP,
         STEP_FINAL_RESOLVE,
@@ -213,6 +226,8 @@ class EscalationPolicy(OrderedModel):
         STEP_NOTIFY_USERS_QUEUE: "notify_person_next_each_time",
         STEP_NOTIFY_MULTIPLE_USERS: "notify_persons",
         STEP_NOTIFY_MULTIPLE_USERS_IMPORTANT: "notify_persons",
+        STEP_NOTIFY_TEAM_MEMBERS: "notify_team_members",
+        STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT: "notify_team_members",
         STEP_NOTIFY_IF_TIME: "notify_if_time_from_to",
         STEP_NOTIFY_IF_NUM_ALERTS_IN_TIME_WINDOW: "notify_if_num_alerts_in_window",
         STEP_REPEAT_ESCALATION_N_TIMES: "repeat_escalation",
@@ -244,6 +259,14 @@ class EscalationPolicy(OrderedModel):
 
     step = models.IntegerField(choices=STEP_CHOICES, default=None, null=True)
 
+    notify_to_team_members = models.ForeignKey(
+        "user_management.Team",
+        on_delete=models.SET_NULL,
+        related_name="escalation_policies",
+        default=None,
+        null=True,
+    )
+
     notify_to_group = models.ForeignKey(
         "slack.SlackUserGroup",
         on_delete=models.SET_NULL,
@@ -368,6 +391,10 @@ def insight_logs_serialized(self):
             if self.notify_to_group:
                 result["user_group"] = self.notify_to_group.name
                 result["user_group_id"] = self.notify_to_group.public_primary_key
+        elif self.step in [EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS, EscalationPolicy.STEP_NOTIFY_TEAM_MEMBERS_IMPORTANT]:
+            if self.notify_to_team_members:
+                result["team"] = self.notify_to_team_members.name
+                result["team_id"] = self.notify_to_team_members.public_primary_key
         elif self.step in [EscalationPolicy.STEP_NOTIFY_SCHEDULE, EscalationPolicy.STEP_NOTIFY_SCHEDULE_IMPORTANT]:
             if self.notify_schedule:
                 result["on-call_schedule"] = self.notify_schedule.insight_logs_verbal

engine/apps/alerts/tasks/__init__.py

@@ -17,6 +17,7 @@
 from .notify_group import notify_group_task  # noqa: F401
 from .notify_ical_schedule_shift import notify_ical_schedule_shift  # noqa: F401
 from .notify_user import notify_user_task  # noqa: F401
+from .notify_team_members import notify_team_members_task  # noqa: F401
 from .resolve_alert_group_by_source_if_needed import resolve_alert_group_by_source_if_needed  # noqa: F401
 from .resolve_by_last_step import resolve_by_last_step_task  # noqa: F401
 from .send_alert_group_signal import send_alert_group_signal  # noqa: F401

engine/apps/alerts/tasks/notify_team_members.py

@@ -0,0 +1,39 @@
+
+from django.conf import settings
+from django.db import transaction
+
+from common.custom_celery_tasks import shared_dedicated_queue_retry_task
+
+from .task_logger import task_logger
+from .notify_user import notify_user_task
+
+@shared_dedicated_queue_retry_task(
+    autoretry_for=(Exception,), retry_backoff=True, max_retries=1 if settings.DEBUG else None
+)
+def notify_team_members_task(
+    team_pk,
+    alert_group_pk,
+    **kwargs # kwargs to pass through to notify_user_task.apply_async
+):
+    from apps.user_management.models import Team
+
+
+    try:
+        team = Team.objects.filter(pk=team_pk).first()
+    except Team.DoesNotExist:
+        return f"notify_team_members_task: team {team_pk} doesn't exist"
+
+
+    for user in team.users.all():
+        try:
+            if user.is_notification_allowed:
+                task_logger.debug(f"notify_team_members_task: notifying {user.pk}")
+                notify_user_task.apply_async(
+                    args=(
+                        user.pk,
+                        alert_group_pk,
+                    ),
+                    kwargs=kwargs
+                )
+        except:
+                task_logger.info(f"notify_team_members_task: user {user.pk} failed")

engine/apps/alerts/tests/test_escalation_snapshot.py

@@ -40,6 +40,7 @@ def test_raw_escalation_snapshot(escalation_snapshot_test_setup):
                 "last_notified_user": None,
                 "notify_schedule": None,
                 "notify_to_group": None,
+                "notify_to_team_members": None,
                 "from_time": None,
                 "to_time": None,
                 "num_alerts_in_window": None,
@@ -59,6 +60,7 @@ def test_raw_escalation_snapshot(escalation_snapshot_test_setup):
                 "last_notified_user": None,
                 "notify_schedule": None,
                 "notify_to_group": None,
+                "notify_to_team_members": None,
                 "from_time": None,
                 "to_time": None,
                 "num_alerts_in_window": None,
@@ -78,6 +80,7 @@ def test_raw_escalation_snapshot(escalation_snapshot_test_setup):
                 "last_notified_user": None,
                 "notify_schedule": None,
                 "notify_to_group": None,
+                "notify_to_team_members": None,
                 "from_time": notify_if_time_step.from_time.isoformat(),
                 "to_time": notify_if_time_step.to_time.isoformat(),
                 "num_alerts_in_window": None,