Release leader for life lock in case pod is preempted.

Signed-off-by: Jagpreet Singh Tamber <jagpreetstamber@gmail.com>

leader/leader.go

@@ -179,8 +179,8 @@ func Become(ctx context.Context, lockName string, opts ...Option) error {
 					log.Info("Leader pod has been deleted, waiting for garbage collection to remove the lock.")
 				case err != nil:
 					return err
-				case isPodEvicted(*leaderPod) && leaderPod.GetDeletionTimestamp() == nil:
-					log.Info("Operator pod with leader lock has been evicted.", "leader", leaderPod.Name)
+				case (isPodEvicted(*leaderPod) || isPodPreempted(*leaderPod)) && leaderPod.GetDeletionTimestamp() == nil:
+					log.Info("Operator pod with leader lock has been evicted or preempted.", "leader", leaderPod.Name)
 					log.Info("Deleting evicted leader.")
 					// Pod may not delete immediately, continue with backoff
 					err := config.Client.Delete(ctx, leaderPod)
@@ -241,6 +241,12 @@ func isPodEvicted(pod corev1.Pod) bool {
 	return podFailed && podEvicted
 }
 
+func isPodPreempted(pod corev1.Pod) bool {
+	podFailed := pod.Status.Phase == corev1.PodFailed
+	podPreempted := pod.Status.Reason == "Preempting"
+	return podFailed && podPreempted
+}
+
 // getPod returns a Pod object that corresponds to the pod in which the code
 // is currently running.
 // It expects the environment variable POD_NAME to be set by the downwards API.

leader/leader_test.go

@@ -108,6 +108,31 @@ var _ = Describe("Leader election", func() {
 			Expect(isPodEvicted(*leaderPod)).To(BeTrue())
 		})
 	})
+	Describe("isPodPreempted", func() {
+		var (
+			leaderPod *corev1.Pod
+		)
+		BeforeEach(func() {
+			leaderPod = &corev1.Pod{}
+		})
+		It("should return false with an empty status", func() {
+			Expect(isPodPreempted(*leaderPod)).To(BeFalse())
+		})
+		It("should return false if reason is incorrect", func() {
+			leaderPod.Status.Phase = corev1.PodFailed
+			leaderPod.Status.Reason = "invalid"
+			Expect(isPodPreempted(*leaderPod)).To(BeFalse())
+		})
+		It("should return false if pod is in the wrong phase", func() {
+			leaderPod.Status.Phase = corev1.PodRunning
+			Expect(isPodPreempted(*leaderPod)).To(BeFalse())
+		})
+		It("should return true when Phase and Reason are set", func() {
+			leaderPod.Status.Phase = corev1.PodFailed
+			leaderPod.Status.Reason = "Preempting"
+			Expect(isPodPreempted(*leaderPod)).To(BeTrue())
+		})
+	})
 	Describe("myOwnerRef", func() {
 		var (
 			client crclient.Client