Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add error log and abort pod cleanup k8s event in controllerCleanupPod #196

Merged
merged 5 commits into from
Apr 12, 2024
Merged

Add error log and abort pod cleanup k8s event in controllerCleanupPod #196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
46652d6
Add error log for missing CSI annotations in controllerCleanupPod
suryagupta4 Apr 9, 2024
b97cbd6
update logs with k8s events
suryagupta4 Apr 10, 2024
fd61a78
fix formatting directive
suryagupta4 Apr 10, 2024
aaf19d2
UT changes
suryagupta4 Apr 10, 2024
32b7676
change error log to info log
suryagupta4 Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions internal/monitor/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,23 @@ func (cm *PodMonitorType) controllerCleanupPod(pod *v1.Pod, node *v1.Node, reaso
if cm.SkipArrayConnectionValidation && taintnoexec {
log.WithFields(fields).Info("SkipArrayConnectionValidation is set and taintnoexec is true- proceeding")
} else {
if err != nil {
log.WithFields(fields).Info("Aborting pod cleanup due to error: ", err.Error())
if strings.Contains(err.Error(), "Could not determine CSI NodeID for node") {
if err = K8sAPI.CreateEvent(podmon, pod, k8sapi.EventTypeWarning, reason,
"podmon aborted pod cleanup %s due to missing CSI annotations",
string(pod.ObjectMeta.UID), node.ObjectMeta.Name); err != nil {
log.Errorf("Failed to send %s event: %s", reason, err.Error())
}
} else {
if err = K8sAPI.CreateEvent(podmon, pod, k8sapi.EventTypeWarning, reason,
"podmon aborted pod cleanup %s due to error while validating volume host connectivity",
string(pod.ObjectMeta.UID), node.ObjectMeta.Name); err != nil {
log.Errorf("Failed to send %s event: %s", reason, err.Error())
}
}
return false
}
log.WithFields(fields).Info("Aborting pod cleanup because array still connected and/or recently did I/O")
if err = K8sAPI.CreateEvent(podmon, pod, k8sapi.EventTypeWarning, reason,
"podmon aborted pod cleanup %s array connected or recent I/O",
Expand Down
6 changes: 3 additions & 3 deletions internal/monitor/features/controller.feature
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Feature: Controller Monitor
| "node1" | 2 | "DeleteVolumeAttachment" | "node1" | "false" | "Couldn't delete VolumeAttachment" |
| "node1" | 2 | "DeletePod" | "node1" | "false" | "Delete pod failed" |
| "node1" | 2 | "ControllerUnpublishVolume" | "node1" | "false" | "errors calling ControllerUnpublishVolume to fence" |
| "node1" | 2 | "ValidateVolumeHostConnectivity" | "node1" | "false" | "Aborting pod cleanup because array still connected" |
| "node1" | 2 | "ValidateVolumeHostConnectivity" | "node1" | "false" | "Aborting pod cleanup due to error" |
| "node1" | 2 | "CreateEvent" | "node1" | "true" | "Successfully cleaned up pod" |

@controller-mode
Expand Down Expand Up @@ -52,8 +52,8 @@ Feature: Controller Monitor
| "node1" | 2 | "NotReady" | "false" | "noexec" | "CreateEvent" | "Updated" | "true" | "false" | "Successfully cleaned up pod" |
| "node1" | 2 | "CrashLoop" | "false" | "noexec" | "CreateEvent" | "Updated" | "true" | "false" | "Successfully cleaned up pod" |
| "node1" | 2 | "Initialized" | "false" | "noexec" | "CreateEvent" | "Updated" | "true" | "false" | "Successfully cleaned up pod" |
| "node1" | 2 | "NotReady" | "false" | "nosched" | "NoAnnotation" | "Updated" | "false" | "false" | "Aborting pod cleanup because array still connected" |
| "node1" | 2 | "NotReady" | "false" | "nosched" | "BadCSINode" | "Updated" | "false" | "false" | "Aborting pod cleanup because array still connected" |
| "node1" | 2 | "NotReady" | "false" | "nosched" | "NoAnnotation" | "Updated" | "false" | "false" | "Aborting pod cleanup due to error" |
| "node1" | 2 | "NotReady" | "false" | "nosched" | "BadCSINode" | "Updated" | "false" | "false" | "Aborting pod cleanup due to error" |

@controller-mode
Scenario Outline: test controllerModePodHandler skipping validate volume with a CSIExtensionNotPresent error
Expand Down
Loading