WIP

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

pkg/etcd/snapshot.go

@@ -796,6 +796,7 @@ func (e *ETCD) ReconcileSnapshotData(ctx context.Context) error {
 		}
 	}
 
+	e.config.Runtime.K3s.K3s().V1().ETCDSnapshotFile().EnqueueAfter(startKey, 5*time.Second)
 	return nil
 }
 

pkg/etcd/snapshot_controller.go

@@ -5,23 +5,27 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	apisv1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
 	controllersv1 "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io/v1"
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	controllerv1 "github.com/rancher/wrangler/v2/pkg/generated/controllers/core/v1"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/util/retry"
 
 	"github.com/sirupsen/logrus"
 )
 
 const (
 	pruneStepSize = 4
+	startKey      = "_start_"
 )
 
 var (
@@ -33,6 +37,7 @@ type etcdSnapshotHandler struct {
 	etcd       *ETCD
 	snapshots  controllersv1.ETCDSnapshotFileController
 	configmaps controllerv1.ConfigMapController
+	once       *sync.Once
 }
 
 func registerSnapshotHandlers(ctx context.Context, etcd *ETCD) {
@@ -42,6 +47,7 @@ func registerSnapshotHandlers(ctx context.Context, etcd *ETCD) {
 		etcd:       etcd,
 		snapshots:  snapshots,
 		configmaps: etcd.config.Runtime.Core.Core().V1().ConfigMap(),
+		once:       &sync.Once{},
 	}
 
 	logrus.Infof("Starting managed etcd snapshot ConfigMap controller")
@@ -50,6 +56,15 @@ func registerSnapshotHandlers(ctx context.Context, etcd *ETCD) {
 }
 
 func (e *etcdSnapshotHandler) sync(key string, esf *apisv1.ETCDSnapshotFile) (*apisv1.ETCDSnapshotFile, error) {
+	if key == startKey {
+		go e.once.Do(func() {
+			if err := e.pruneMissing(); err != nil {
+				logrus.Errorf("Failed to reconcile snapshot ConfigMap data: %v", err)
+			}
+		})
+		return esf, nil
+	}
+
 	sf := snapshotFile{}
 	sf.fromETCDSnapshotFile(esf)
 	m, err := marshalSnapshotFile(sf)
@@ -131,6 +146,71 @@ func (e *etcdSnapshotHandler) onRemove(key string, esf *apisv1.ETCDSnapshotFile)
 	return esf, nil
 }
 
+func (e *etcdSnapshotHandler) pruneMissing() error {
+	logrus.Infof("Reconciling snapshot ConfigMap data to prune entries from missing nodes")
+
+	snapshotConfigMap, err := e.configmaps.Get(metav1.NamespaceSystem, snapshotConfigMapName, metav1.GetOptions{})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrap(err, "failed to get snapshot ConfigMap")
+	}
+
+	// Get a list of all etcd nodes currently in the cluster.
+	// We will use this list to prune local entries for any node that does not exist.
+	nodes := e.config.Runtime.Core.Core().V1().Node()
+	etcdSelector := labels.Set{util.ETCDRoleLabelKey: "true"}
+	nodeList, err := nodes.List(metav1.ListOptions{LabelSelector: etcdSelector.String()})
+	if err != nil {
+		return err
+	}
+
+	// Get a list of s3 snapshots.
+	// We will use this list to prune s3 entries, if any are found.
+	s3SnapshotNames := map[string]bool{}
+	selector := labels.Set{labelStorageNode: "s3"}
+	esfList, err := e.snapshots.List(metav1.ListOptions{LabelSelector: selector.String()})
+	if err != nil {
+		return err
+	}
+
+	for _, esf := range esfList.Items {
+		s3SnapshotNames[esf.Name] = true
+	}
+	if len(s3SnapshotNames) > 0 {
+		logrus.Infof("Reconciling snapshot ConfigMap data to prune entries from S3")
+	}
+
+	cleanData := map[string]string{}
+	for key := range snapshotConfigMap.Data {
+		switch {
+		case strings.HasPrefix(key, "s3-"):
+			if len(s3SnapshotNames) == 0 {
+				// no s3 snapshots found, keep them all
+				cleanData[key] = snapshotConfigMap.Data[key]
+			} else if _, ok := s3SnapshotNames[key]; ok {
+				// have snapshots in S3 and this is one of them, so keep it
+				cleanData[key] = snapshotConfigMap.Data[key]
+			}
+		case strings.HasPrefix(key, "local-"):
+			for _, node := range nodeList.Items {
+				if strings.HasPrefix(key, "local-"+node.Name) {
+					// TODO: match node names in snapshot name
+					cleanData[key] = snapshotConfigMap.Data[key]
+				}
+			}
+		default:
+			// Don't know what this is for, so keep it
+			cleanData[key] = snapshotConfigMap.Data[key]
+		}
+	}
+	snapshotConfigMap.Data = cleanData
+
+	// TODO: save configmap
+	return nil
+}
+
 // pruneConfigMap drops the oldest entries from the configMap.
 // Note that the actual snapshot files are not removed, just the entries that track them in the configmap.
 func pruneConfigMap(snapshotConfigMap *v1.ConfigMap, pruneCount int) error {