From 46652d67187faa5c237c4e5fe0e1399fb7200fd5 Mon Sep 17 00:00:00 2001 From: Surya Date: Tue, 9 Apr 2024 17:27:43 +0000 Subject: [PATCH 1/5] Add error log for missing CSI annotations in controllerCleanupPod --- internal/monitor/controller.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/monitor/controller.go b/internal/monitor/controller.go index 5cfb18b..ae60b4e 100644 --- a/internal/monitor/controller.go +++ b/internal/monitor/controller.go @@ -262,6 +262,14 @@ func (cm *PodMonitorType) controllerCleanupPod(pod *v1.Pod, node *v1.Node, reaso if cm.SkipArrayConnectionValidation && taintnoexec { log.WithFields(fields).Info("SkipArrayConnectionValidation is set and taintnoexec is true- proceeding") } else { + if err != nil { + log.Errorf("%s", err.Error()) + if strings.Contains(err.Error(), "Could not determine CSI NodeID for node") { + // This error is returned if CSI annotations could not be fetched for the node + log.WithFields(fields).Error("Aborting pod cleanup because volume host connectivity could not be validated due to missing CSI annotations") + return false + } + } log.WithFields(fields).Info("Aborting pod cleanup because array still connected and/or recently did I/O") if err = K8sAPI.CreateEvent(podmon, pod, k8sapi.EventTypeWarning, reason, "podmon aborted pod cleanup %s array connected or recent I/O", From b97cbd696e963a24c5e9799ff3fa9e82ad016ffa Mon Sep 17 00:00:00 2001 From: Surya Date: Wed, 10 Apr 2024 08:06:14 +0000 Subject: [PATCH 2/5] update logs with k8s events --- internal/monitor/controller.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/internal/monitor/controller.go b/internal/monitor/controller.go index ae60b4e..2beec5d 100644 --- a/internal/monitor/controller.go +++ b/internal/monitor/controller.go @@ -263,12 +263,21 @@ func (cm *PodMonitorType) controllerCleanupPod(pod *v1.Pod, node *v1.Node, reaso log.WithFields(fields).Info("SkipArrayConnectionValidation is set and taintnoexec is true- proceeding") } else { if err != nil { - log.Errorf("%s", err.Error()) + log.WithFields(fields).Error("Aborting pod cleanup due to error: %s", err.Error()) if strings.Contains(err.Error(), "Could not determine CSI NodeID for node") { - // This error is returned if CSI annotations could not be fetched for the node - log.WithFields(fields).Error("Aborting pod cleanup because volume host connectivity could not be validated due to missing CSI annotations") - return false + if err = K8sAPI.CreateEvent(podmon, pod, k8sapi.EventTypeWarning, reason, + "podmon aborted pod cleanup %s due to missing CSI annotations", + string(pod.ObjectMeta.UID), node.ObjectMeta.Name); err != nil { + log.Errorf("Failed to send %s event: %s", reason, err.Error()) + } + } else { + if err = K8sAPI.CreateEvent(podmon, pod, k8sapi.EventTypeWarning, reason, + "podmon aborted pod cleanup %s due to error while validating volume host connectivity", + string(pod.ObjectMeta.UID), node.ObjectMeta.Name); err != nil { + log.Errorf("Failed to send %s event: %s", reason, err.Error()) + } } + return false } log.WithFields(fields).Info("Aborting pod cleanup because array still connected and/or recently did I/O") if err = K8sAPI.CreateEvent(podmon, pod, k8sapi.EventTypeWarning, reason, From fd61a780d23075579344d07def36400f2d1bacfe Mon Sep 17 00:00:00 2001 From: Surya Date: Wed, 10 Apr 2024 09:57:39 +0000 Subject: [PATCH 3/5] fix formatting directive --- internal/monitor/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/monitor/controller.go b/internal/monitor/controller.go index 2beec5d..a83dbe2 100644 --- a/internal/monitor/controller.go +++ b/internal/monitor/controller.go @@ -263,7 +263,7 @@ func (cm *PodMonitorType) controllerCleanupPod(pod *v1.Pod, node *v1.Node, reaso log.WithFields(fields).Info("SkipArrayConnectionValidation is set and taintnoexec is true- proceeding") } else { if err != nil { - log.WithFields(fields).Error("Aborting pod cleanup due to error: %s", err.Error()) + log.WithFields(fields).Error("Aborting pod cleanup due to error: ", err.Error()) if strings.Contains(err.Error(), "Could not determine CSI NodeID for node") { if err = K8sAPI.CreateEvent(podmon, pod, k8sapi.EventTypeWarning, reason, "podmon aborted pod cleanup %s due to missing CSI annotations", From aaf19d237a2c372d5a5471633efe2d2417da07a5 Mon Sep 17 00:00:00 2001 From: Surya Date: Wed, 10 Apr 2024 10:34:08 +0000 Subject: [PATCH 4/5] UT changes --- internal/monitor/features/controller.feature | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/monitor/features/controller.feature b/internal/monitor/features/controller.feature index 17045dd..6cb15c4 100644 --- a/internal/monitor/features/controller.feature +++ b/internal/monitor/features/controller.feature @@ -22,7 +22,7 @@ Feature: Controller Monitor | "node1" | 2 | "DeleteVolumeAttachment" | "node1" | "false" | "Couldn't delete VolumeAttachment" | | "node1" | 2 | "DeletePod" | "node1" | "false" | "Delete pod failed" | | "node1" | 2 | "ControllerUnpublishVolume" | "node1" | "false" | "errors calling ControllerUnpublishVolume to fence" | - | "node1" | 2 | "ValidateVolumeHostConnectivity" | "node1" | "false" | "Aborting pod cleanup because array still connected" | + | "node1" | 2 | "ValidateVolumeHostConnectivity" | "node1" | "false" | "Aborting pod cleanup due to error" | | "node1" | 2 | "CreateEvent" | "node1" | "true" | "Successfully cleaned up pod" | @controller-mode @@ -52,8 +52,8 @@ Feature: Controller Monitor | "node1" | 2 | "NotReady" | "false" | "noexec" | "CreateEvent" | "Updated" | "true" | "false" | "Successfully cleaned up pod" | | "node1" | 2 | "CrashLoop" | "false" | "noexec" | "CreateEvent" | "Updated" | "true" | "false" | "Successfully cleaned up pod" | | "node1" | 2 | "Initialized" | "false" | "noexec" | "CreateEvent" | "Updated" | "true" | "false" | "Successfully cleaned up pod" | - | "node1" | 2 | "NotReady" | "false" | "nosched" | "NoAnnotation" | "Updated" | "false" | "false" | "Aborting pod cleanup because array still connected" | - | "node1" | 2 | "NotReady" | "false" | "nosched" | "BadCSINode" | "Updated" | "false" | "false" | "Aborting pod cleanup because array still connected" | + | "node1" | 2 | "NotReady" | "false" | "nosched" | "NoAnnotation" | "Updated" | "false" | "false" | "Aborting pod cleanup due to error" | + | "node1" | 2 | "NotReady" | "false" | "nosched" | "BadCSINode" | "Updated" | "false" | "false" | "Aborting pod cleanup due to error" | @controller-mode Scenario Outline: test controllerModePodHandler skipping validate volume with a CSIExtensionNotPresent error From 32b76766818fa4e21cffe811d5587b7a8218930f Mon Sep 17 00:00:00 2001 From: Surya Date: Wed, 10 Apr 2024 16:30:23 +0000 Subject: [PATCH 5/5] change error log to info log --- internal/monitor/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/monitor/controller.go b/internal/monitor/controller.go index a83dbe2..af8d917 100644 --- a/internal/monitor/controller.go +++ b/internal/monitor/controller.go @@ -263,7 +263,7 @@ func (cm *PodMonitorType) controllerCleanupPod(pod *v1.Pod, node *v1.Node, reaso log.WithFields(fields).Info("SkipArrayConnectionValidation is set and taintnoexec is true- proceeding") } else { if err != nil { - log.WithFields(fields).Error("Aborting pod cleanup due to error: ", err.Error()) + log.WithFields(fields).Info("Aborting pod cleanup due to error: ", err.Error()) if strings.Contains(err.Error(), "Could not determine CSI NodeID for node") { if err = K8sAPI.CreateEvent(podmon, pod, k8sapi.EventTypeWarning, reason, "podmon aborted pod cleanup %s due to missing CSI annotations",