Added PodEvicted default event

helm/robusta/values.yaml

@@ -176,6 +176,15 @@ builtinPlaybooks:
   - image_pull_backoff_reporter: {}
 
 # playbooks for non-prometheus based monitoring that use prometheus for enrichment
+- name: "PodEvicted"
+  triggers:
+  - on_pod_evicted: {}
+  actions:
+  - on_pod_evicted_enricher: {}
+  - pod_events_enricher: {}
+  - enrich_pod_with_node_events: {}
+  - logs_enricher: {}
+
 - name: "PodOOMKill"
   triggers:
   - on_pod_oom_killed:

playbooks/robusta_playbooks/event_enrichments.py

@@ -272,6 +272,34 @@ def pod_events_enricher(event: PodEvent, params: EventEnricherParams):
         )
 
 
+@action
+def enrich_pod_with_node_events(event: PodEvent, params: EventEnricherParams):
+    """
+    Given a Kubernetes pod, fetch related events in the near past for its node
+    """
+    pod = event.get_pod()
+    node: Node = Node.readNode(pod.spec.nodeName).obj
+    if not node:
+        logging.error(f"cannot run pods_node_events_enricher on alert with no node object: {event}")
+        return
+
+    events_table_block = get_resource_events_table(
+        "*Node events:*",
+        node.kind,
+        node.metadata.name,
+        node.metadata.namespace,
+        included_types=params.included_types,
+        max_events=params.max_events,
+    )
+    if events_table_block:
+        event.add_enrichment(
+            [events_table_block],
+            {SlackAnnotations.ATTACHMENT: True},
+            enrichment_type=EnrichmentType.k8s_events,
+            title="Node Events",
+        )
+
+
 @action
 def deployment_events_enricher(event: DeploymentEvent, params: ExtendedEventEnricherParams):
     """

playbooks/robusta_playbooks/node_enrichments.py

@@ -2,8 +2,11 @@
 from typing import List
 
 from hikaru.model.rel_1_26 import Pod, PodList
+from robusta_playbooks.playbook_utils import pod_row
+
 from robusta.api import (
     BaseBlock,
+    EnrichmentType,
     FileBlock,
     Finding,
     FindingSeverity,
@@ -19,19 +22,9 @@
     TableBlock,
     action,
     create_node_graph_enrichment,
-    EnrichmentType
 )
 
 
-def pod_row(pod: Pod) -> List[str]:
-    ready_condition = [condition.status for condition in pod.status.conditions if condition.type == "Ready"]
-    return [
-        pod.metadata.namespace,
-        pod.metadata.name,
-        ready_condition[0] if ready_condition else "Unknown",
-    ]
-
-
 def has_resource_request(pod: Pod, resource_type: str) -> bool:
     for container in pod.spec.containers:
         try:
@@ -85,7 +78,7 @@ def node_running_pods_enricher(event: NodeEvent):
 
     effected_pods_rows = [pod_row(pod) for pod in pod_list.items]
     block_list.append(
-        TableBlock(effected_pods_rows, ["namespace", "name", "ready"], table_name=f"Pods running on the node")
+        TableBlock(effected_pods_rows, ["namespace", "name", "ready"], table_name="Pods running on the node")
     )
     event.add_enrichment(block_list)
 
@@ -127,7 +120,7 @@ def node_status_enricher(event: NodeEvent):
         logging.error(f"node_status_enricher was called on event without node : {event}")
         return
 
-    logging.info(f"node_status_enricher is depricated, use status_enricher instead")
+    logging.info("node_status_enricher is depricated, use status_enricher instead")
 
     event.add_enrichment(
         [
@@ -154,8 +147,9 @@ def node_dmesg_enricher(event: NodeEvent, params: PodRunningParams):
     )
     if exec_result:
         event.add_enrichment(
-            [FileBlock(f"dmesg.log", exec_result.encode())], enrichment_type=EnrichmentType.text_file,
-            title="DMESG Info"
+            [FileBlock("dmesg.log", exec_result.encode())],
+            enrichment_type=EnrichmentType.text_file,
+            title="DMESG Info",
         )
 
 
@@ -189,8 +183,9 @@ def node_health_watcher(event: NodeChangeEvent):
         subject=KubeObjFindingSubject(event.obj),
     )
     event.add_finding(finding)
-    event.add_enrichment([KubernetesDiffBlock([], event.old_obj,
-                                              event.obj, event.obj.metadata.name, kind=event.obj.kind)])
+    event.add_enrichment(
+        [KubernetesDiffBlock([], event.old_obj, event.obj, event.obj.metadata.name, kind=event.obj.kind)]
+    )
     node_status_enricher(event)
 
 

playbooks/robusta_playbooks/playbook_utils.py

@@ -0,0 +1,12 @@
+from typing import List
+
+from hikaru.model.rel_1_26 import Pod
+
+
+def pod_row(pod: Pod) -> List[str]:
+    ready_condition = [condition.status for condition in pod.status.conditions if condition.type == "Ready"]
+    return [
+        pod.metadata.namespace,
+        pod.metadata.name,
+        ready_condition[0] if ready_condition else "Unknown",
+    ]

playbooks/robusta_playbooks/pod_evicted_enrichments.py

@@ -0,0 +1,80 @@
+import logging
+from typing import List
+
+from hikaru.model.rel_1_26 import Node, PodList
+from robusta_playbooks.playbook_utils import pod_row
+
+from robusta.api import (
+    BaseBlock,
+    EnrichmentType,
+    Finding,
+    FindingSeverity,
+    PodEvent,
+    PodFindingSubject,
+    TableBlock,
+    action,
+)
+
+
+@action
+def on_pod_evicted_enricher(event: PodEvent):
+    """
+    Retrieves pod and node information for an OOMKilled pod
+    """
+    pod = event.get_pod()
+    if not pod:
+        logging.error(f"cannot run on_pod_evicted_enricher on event with no pod: {event}")
+        return
+
+    try:
+        node = Node.readNode(pod.spec.nodeName).obj
+    except Exception as e:
+        logging.error(f"Failed to read pod's node information: {e}")
+        return
+
+    finding = Finding(
+        title=f"Pod {pod.metadata.name} in namespace {pod.metadata.namespace} was Evicted",
+        aggregation_key="PodEvictedTriggered",
+        severity=FindingSeverity.HIGH,
+        subject=PodFindingSubject(pod),
+    )
+
+    node: Node = Node.readNode(pod.spec.nodeName).obj
+    node_labels = [("Node Name", pod.spec.nodeName)]
+    node_info_block = TableBlock(
+        [[k, v] for k, v in node_labels],
+        headers=["Field", "Value"],
+        table_name="*Node general info:*",
+    )
+    node_status_block = TableBlock(
+        [[condition.type, condition.status] for condition in node.status.conditions],
+        headers=["Type", "Status"],
+        table_name="*Node status details:*",
+    )
+
+    allocatable_resources_block = TableBlock(
+        [[resource, value] for resource, value in node.status.allocatable.items()],
+        headers=["Resource", "Value"],
+        table_name="*Node Allocatable Resources:*",
+    )
+
+    finding.add_enrichment(
+        [node_info_block, node_status_block, allocatable_resources_block],
+        enrichment_type=EnrichmentType.node_info,
+        title="Node Info",
+    )
+
+    event.add_finding(finding)
+
+    try:
+        pod_list = PodList.listPodForAllNamespaces(field_selector=f"spec.nodeName={node.metadata.name}").obj
+    except Exception as e:
+        logging.error(f"Failed to list pods for node {node.metadata.name}: {e}")
+        return
+
+    effected_pods_rows = [pod_row(pod) for pod in pod_list.items]
+    block_list: List[BaseBlock] = []
+    block_list.append(
+        TableBlock(effected_pods_rows, ["namespace", "name", "ready"], table_name="Pods running on the node")
+    )
+    event.add_enrichment(block_list)