Skip to content

Commit

Permalink
leader: check the node status for the leader-election (operator-frame…
Browse files Browse the repository at this point in the history
  • Loading branch information
silverbullet49 committed Aug 5, 2020
1 parent 30fdafc commit 240286a
Show file tree
Hide file tree
Showing 2 changed files with 101 additions and 0 deletions.
38 changes: 38 additions & 0 deletions leader/leader.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ func Become(ctx context.Context, lockName string, opts ...Option) error {
log.Info("Leader lock configmap owner reference must be a pod.", "OwnerReference", existingOwners[0])
default:
leaderPod := &corev1.Pod{}
leaderNode := &corev1.Node{}
key = crclient.ObjectKey{Namespace: ns, Name: existingOwners[0].Name}
err = config.Client.Get(ctx, key, leaderPod)
switch {
Expand All @@ -179,6 +180,24 @@ func Become(ctx context.Context, lockName string, opts ...Option) error {
if err != nil {
log.Error(err, "Leader pod could not be deleted.")
}
case getNode(ctx, config.Client, leaderPod.Spec.NodeName, leaderNode) == nil && isNotReadyNode(*leaderNode):
log.Info("the status of the node where operator pod with leader lock was running has been 'notReady'")
log.Info("Deleting the leader.")

//Mark the termainating status to the leader
err := config.Client.Delete(ctx, leaderPod)
if err != nil {
log.Error(err, "Leader pod could not be deleted.")
} else {
err := config.Client.Delete(ctx, existing)
switch {
case err == nil:
case apierrors.IsNotFound(err):
log.Info("cm has been deleted by prior operator.")
case err != nil:
return err
}
}

default:
log.Info("Not the leader. Waiting.")
Expand Down Expand Up @@ -267,3 +286,22 @@ func getPod(ctx context.Context, client crclient.Client, ns string) (*corev1.Pod

return pod, nil
}

func getNode(ctx context.Context, client crclient.Client, nodeName string, node *corev1.Node) error {
key := crclient.ObjectKey{Namespace: "", Name: nodeName}
err := client.Get(ctx, key, node)
if err != nil {
log.Error(err, "Failed to get Node", "Node.Name", nodeName)
return err
}
return nil
}

func isNotReadyNode(node corev1.Node) bool {
for _, condition := range node.Status.Conditions {
if condition.Type == corev1.NodeReady && condition.Status != corev1.ConditionTrue {
return true
}
}
return false
}
63 changes: 63 additions & 0 deletions leader/leader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,4 +209,67 @@ var _ = Describe("Leader election", func() {
Expect(pod.TypeMeta.Kind).To(Equal("Pod"))
})
})

Describe("getNode", func() {
var (
client crclient.Client
)
BeforeEach(func() {
client = fake.NewFakeClient(
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "mynode",
},
},
)
})
It("should return an error if no node is found", func() {
node := corev1.Node{}
err := getNode(context.TODO(), client, "", &node)
Expect(err).ShouldNot(BeNil())
})
It("should return the node with the given name", func() {
node := corev1.Node{}
err := getNode(context.TODO(), client, "mynode", &node)
Expect(err).Should(BeNil())
Expect(node.TypeMeta.APIVersion).To(Equal("v1"))
Expect(node.TypeMeta.Kind).To(Equal("Node"))
})
})

Describe("isNotReadyNode", func() {
var (
leaderNode *corev1.Node
)
BeforeEach(func() {
leaderNode = &corev1.Node{
Status: corev1.NodeStatus{
Conditions: make([]corev1.NodeCondition, 1),
},
}
})
It("should return false with an empty status", func() {
Expect(isNotReadyNode(*leaderNode)).To(Equal(false))
})
It("should return false if type is incorrect", func() {
leaderNode.Status.Conditions[0].Type = corev1.NodeMemoryPressure
leaderNode.Status.Conditions[0].Status = corev1.ConditionFalse
Expect(isNotReadyNode(*leaderNode)).To(Equal(false))
})
It("should return false if NodeReady's status is true", func() {
leaderNode.Status.Conditions[0].Type = corev1.NodeReady
leaderNode.Status.Conditions[0].Status = corev1.ConditionTrue
Expect(isNotReadyNode(*leaderNode)).To(Equal(false))
})
It("should return true when Type is set and Status is set to false", func() {
leaderNode.Status.Conditions[0].Type = corev1.NodeReady
leaderNode.Status.Conditions[0].Status = corev1.ConditionFalse
Expect(isNotReadyNode(*leaderNode)).To(Equal(true))
})
It("should return true when Type is set and Status is set to unknown", func() {
leaderNode.Status.Conditions[0].Type = corev1.NodeReady
leaderNode.Status.Conditions[0].Status = corev1.ConditionUnknown
Expect(isNotReadyNode(*leaderNode)).To(Equal(true))
})
})
})

0 comments on commit 240286a

Please sign in to comment.