feat: Add reboot required annotation

Signed-off-by: Timo Haas <haastimo@gmx.de>

cmd/kured/main.go

@@ -30,13 +30,13 @@ import (
 	"github.com/google/shlex"
 
 	shoutrrr "github.com/containrrr/shoutrrr"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/kubereboot/kured/pkg/alerts"
 	"github.com/kubereboot/kured/pkg/daemonsetlock"
 	"github.com/kubereboot/kured/pkg/delaytick"
 	"github.com/kubereboot/kured/pkg/taints"
 	"github.com/kubereboot/kured/pkg/timewindow"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
@@ -95,6 +95,8 @@ const (
 	KuredRebootInProgressAnnotation string = "weave.works/kured-reboot-in-progress"
 	// KuredMostRecentRebootNeededAnnotation is the canonical string value for the kured most-recent-reboot-needed annotation
 	KuredMostRecentRebootNeededAnnotation string = "weave.works/kured-most-recent-reboot-needed"
+	// KuredRebootRequiredAnnotation is the canonical string value for the kured reboot-required annotation
+	KuredRebootRequiredAnnotation string = "weave.works/reboot-required"
 	// EnvPrefix The environment variable prefix of all environment variables bound to our command line flags.
 	EnvPrefix = "KURED"
 )
@@ -566,13 +568,23 @@ func addNodeAnnotations(client *kubernetes.Clientset, nodeID string, annotations
 }
 
 func deleteNodeAnnotation(client *kubernetes.Clientset, nodeID, key string) error {
+
+	node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
+	if err != nil {
+		log.Errorf("Error deleting node annotation %s via k8s API: %v", key, err)
+		return err
+	}
+	if _, exists := node.Annotations[key]; !exists {
+		return nil
+	}
+
 	log.Infof("Deleting node %s annotation %s", nodeID, key)
 
 	// JSON Patch takes as path input a JSON Pointer, defined in RFC6901
 	// So we replace all instances of "/" with "~1" as per:
 	// https://tools.ietf.org/html/rfc6901#section-3
 	patch := []byte(fmt.Sprintf("[{\"op\":\"remove\",\"path\":\"/metadata/annotations/%s\"}]", strings.ReplaceAll(key, "/", "~1")))
-	_, err := client.CoreV1().Nodes().Patch(context.TODO(), nodeID, types.JSONPatchType, patch, metav1.PatchOptions{})
+	_, err = client.CoreV1().Nodes().Patch(context.TODO(), nodeID, types.JSONPatchType, patch, metav1.PatchOptions{})
 	if err != nil {
 		log.Errorf("Error deleting node annotation %s via k8s API: %v", key, err)
 		return err
@@ -627,6 +639,12 @@ func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []s
 	source := rand.NewSource(time.Now().UnixNano())
 	tick := delaytick.New(source, 1*time.Minute)
 	for range tick {
+		if annotateNodes && !rebootRequired(sentinelCommand) {
+			err := deleteNodeAnnotation(client, nodeID, KuredRebootRequiredAnnotation)
+			if err != nil {
+				continue
+			}
+		}
 		if holding(lock, &nodeMeta) {
 			node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{})
 			if err != nil {
@@ -696,6 +714,16 @@ func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []s
 		}
 		log.Infof("Reboot required")
 
+		if annotateNodes {
+			// Annotate this node to indicate that "I want to be rebooted!"
+			// so that other node maintenance tools can move stateful workloads(in their respective maintenance windows) away
+			annotations := map[string]string{KuredRebootRequiredAnnotation: "true"}
+			err := addNodeAnnotations(client, nodeID, annotations)
+			if err != nil {
+				continue
+			}
+		}
+
 		var blockCheckers []RebootBlocker
 		if prometheusURL != "" {
 			blockCheckers = append(blockCheckers, PrometheusBlockingChecker{promClient: promClient, filter: alertFilter, firingOnly: alertFiringOnly})