[0.1] Delete machines

pkg/controller/cluster/BUILD.bazel

@@ -11,12 +11,17 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/apis/cluster/v1alpha1:go_default_library",
+        "//pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library",
         "//pkg/controller/error:go_default_library",
         "//pkg/util:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//vendor/k8s.io/client-go/tools/pager:go_default_library",
         "//vendor/k8s.io/client-go/util/retry:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
         "//vendor/sigs.k8s.io/controller-runtime/pkg/client:go_default_library",

pkg/controller/cluster/cluster_controller.go

@@ -18,15 +18,21 @@ package cluster
 
 import (
 	"context"
+	"time"
 
 	"github.com/pkg/errors"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	errorag "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/client-go/tools/pager"
 	"k8s.io/client-go/util/retry"
 	"k8s.io/klog"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 	clusterv1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
+	"sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1"
 	controllerError "sigs.k8s.io/cluster-api/pkg/controller/error"
 	"sigs.k8s.io/cluster-api/pkg/util"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -37,15 +43,30 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
+const deleteRequeueAfter = 5 * time.Second
+
 var DefaultActuator Actuator
 
 func AddWithActuator(mgr manager.Manager, actuator Actuator) error {
-	return add(mgr, newReconciler(mgr, actuator))
+	reconciler, err := newReconciler(mgr, actuator)
+	if err != nil {
+		return err
+	}
+
+	return add(mgr, reconciler)
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, actuator Actuator) reconcile.Reconciler {
-	return &ReconcileCluster{Client: mgr.GetClient(), scheme: mgr.GetScheme(), actuator: actuator}
+func newReconciler(mgr manager.Manager, actuator Actuator) (reconcile.Reconciler, error) {
+	cclient, err := v1alpha1.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	return &ReconcileCluster{
+		Client:        mgr.GetClient(),
+		clusterClient: cclient,
+		scheme:        mgr.GetScheme(),
+		actuator:      actuator}, nil
 }
 
 // add adds a new Controller to mgr with r as the reconcile.Reconciler
@@ -70,8 +91,10 @@ var _ reconcile.Reconciler = &ReconcileCluster{}
 // ReconcileCluster reconciles a Cluster object
 type ReconcileCluster struct {
 	client.Client
-	scheme   *runtime.Scheme
-	actuator Actuator
+	// TODO: remove this once the controller-runtime Client has pagination support
+	clusterClient v1alpha1.ClusterV1alpha1Interface
+	scheme        *runtime.Scheme
+	actuator      Actuator
 }
 
 // +kubebuilder:rbac:groups=cluster.k8s.io,resources=clusters,verbs=get;list;watch;create;update;patch;delete
@@ -85,7 +108,7 @@ func (r *ReconcileCluster) Reconcile(request reconcile.Request) (reconcile.Resul
 			return reconcile.Result{}, nil
 		}
 		// Error reading the object - requeue the request.
-		return reconcile.Result{}, err
+		return reconcile.Result{}, errors.WithStack(err)
 	}
 
 	name := cluster.Name
@@ -107,7 +130,7 @@ func (r *ReconcileCluster) Reconcile(request reconcile.Request) (reconcile.Resul
 		if len(cluster.Finalizers) > finalizerCount {
 			if err := r.Update(context.Background(), cluster); err != nil {
 				klog.Infof("Failed to add finalizer to cluster %q: %v", name, err)
-				return reconcile.Result{}, err
+				return reconcile.Result{}, errors.WithStack(err)
 			}
 
 			// Since adding the finalizer updates the object return to avoid later update issues.
@@ -119,18 +142,58 @@ func (r *ReconcileCluster) Reconcile(request reconcile.Request) (reconcile.Resul
 	if !cluster.ObjectMeta.DeletionTimestamp.IsZero() {
 		// no-op if finalizer has been removed.
 		if !util.Contains(cluster.ObjectMeta.Finalizers, clusterv1.ClusterFinalizer) {
-			klog.Infof("reconciling cluster object %v causes a no-op as there is no finalizer.", name)
+			klog.Infof("Reconciling cluster object %v causes a no-op as there is no finalizer.", name)
 			return reconcile.Result{}, nil
 		}
 
-		klog.Infof("reconciling cluster object %v triggers delete.", name)
+		children, err := r.listChildren(context.Background(), cluster)
+		if err != nil {
+			klog.Errorf("Failed to list dependent objects of cluster %s/%s: %v", cluster.ObjectMeta.Namespace, cluster.ObjectMeta.Name, err)
+			return reconcile.Result{}, err
+		}
+
+		if len(children) > 0 {
+			klog.Infof("Deleting cluster %s: %d children still exist, will requeue", name, len(children))
+
+			var errList []error
+
+			for _, child := range children {
+				accessor, err := meta.Accessor(child)
+				if err != nil {
+					klog.Errorf("cluster %s: couldn't create accessor for %T: %v", name, child, err)
+					continue
+				}
+
+				if accessor.GetDeletionTimestamp() != nil {
+					continue
+				}
+
+				gvk := child.GetObjectKind().GroupVersionKind().String()
+
+				klog.V(4).Infof("Deleting cluster %s: Deleting %s %s", name, gvk, accessor.GetName())
+				if err := r.Delete(context.Background(), child, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil {
+					err = errors.Wrapf(err, "deleting cluster %s: failed to delete %s %s", name, gvk, accessor.GetName())
+					klog.Errorf("Deletion error: %v", err)
+					errList = append(errList, err)
+				}
+			}
+
+			if len(errList) > 0 {
+				return reconcile.Result{}, errorag.NewAggregate(errList)
+
+			}
+
+			return reconcile.Result{Requeue: true, RequeueAfter: deleteRequeueAfter}, nil
+		}
+
+		klog.Infof("Reconciling cluster object %v triggers delete.", name)
 		if err := r.actuator.Delete(cluster); err != nil {
 			klog.Errorf("Error deleting cluster object %v; %v", name, err)
-			return reconcile.Result{}, err
+			return reconcile.Result{}, errors.WithStack(err)
 		}
 
 		// Remove finalizer on successful deletion.
-		klog.Infof("cluster object %v deletion successful, removing finalizer.", name)
+		klog.Infof("Cluster object %v deletion successful, removing finalizer.", name)
 		err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
 			// It's possible the actuator's Delete call modified the cluster. We can't guarantee that every provider
 			// updated the in memory cluster object with the latest copy of the cluster, so try to get a fresh copy.
@@ -151,20 +214,83 @@ func (r *ReconcileCluster) Reconcile(request reconcile.Request) (reconcile.Resul
 		})
 		if err != nil {
 			klog.Errorf("Error removing finalizer from cluster object %v; %v", name, err)
-			return reconcile.Result{}, err
+			return reconcile.Result{}, errors.WithStack(err)
 		}
 
 		return reconcile.Result{}, nil
 	}
 
-	klog.Infof("reconciling cluster object %v triggers idempotent reconcile.", name)
+	klog.Infof("Reconciling cluster object %v triggers idempotent reconcile.", name)
 	if err := r.actuator.Reconcile(cluster); err != nil {
 		if requeueErr, ok := errors.Cause(err).(controllerError.HasRequeueAfterError); ok {
 			klog.Infof("Actuator returned requeue-after error: %v", requeueErr)
 			return reconcile.Result{Requeue: true, RequeueAfter: requeueErr.GetRequeueAfter()}, nil
 		}
 		klog.Errorf("Error reconciling cluster object %v; %v", name, err)
-		return reconcile.Result{}, err
+		return reconcile.Result{}, errors.WithStack(err)
 	}
 	return reconcile.Result{}, nil
 }
+
+// listChildren returns a list of Deployments, Sets, and Machines than have an ownerref to the given cluster
+func (r *ReconcileCluster) listChildren(ctx context.Context, cluster *clusterv1.Cluster) ([]runtime.Object, error) {
+	var children []runtime.Object
+
+	ns := cluster.GetNamespace()
+	opts := metav1.ListOptions{
+		LabelSelector: labels.FormatLabels(
+			map[string]string{clusterv1.MachineClusterLabelName: cluster.GetName()},
+		),
+	}
+
+	dfunc := func(ctx context.Context, m metav1.ListOptions) (runtime.Object, error) {
+		return r.clusterClient.MachineDeployments(ns).List(m)
+	}
+	sfunc := func(ctx context.Context, m metav1.ListOptions) (runtime.Object, error) {
+		return r.clusterClient.MachineSets(ns).List(m)
+	}
+	mfunc := func(ctx context.Context, m metav1.ListOptions) (runtime.Object, error) {
+		return r.clusterClient.Machines(ns).List(m)
+	}
+
+	eachFunc := func(o runtime.Object) error {
+
+		acc, err := meta.Accessor(o)
+		if err != nil {
+			klog.Errorf("cluster %s: couldn't create accessor for %T: %v", cluster.Name, o, err)
+			return nil
+		}
+
+		if util.PointsTo(acc.GetOwnerReferences(), &cluster.ObjectMeta) {
+			children = append(children, o)
+		}
+
+		return nil
+	}
+
+	deployments, err := pager.New(dfunc).List(ctx, opts)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to list MachineDeployments in %s/%s", ns, cluster.Name)
+	}
+	if err := meta.EachListItem(deployments, eachFunc); err != nil {
+		return nil, errors.Wrapf(err, "couldn't iterate MachinesDeployments for cluster %s/%s", ns, cluster.Name)
+	}
+
+	sets, err := pager.New(sfunc).List(ctx, opts)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to list MachineSets in %s/%s", ns, cluster.Name)
+	}
+	if err := meta.EachListItem(sets, eachFunc); err != nil {
+		return nil, errors.Wrapf(err, "couldn't iterate MachineSets for cluster %s/%s", ns, cluster.Name)
+	}
+
+	machines, err := pager.New(mfunc).List(ctx, opts)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to list Machines in %s/%s", ns, cluster.Name)
+	}
+	if err := meta.EachListItem(machines, eachFunc); err != nil {
+		return nil, errors.Wrapf(err, "couldn't iterate Machines for cluster %s/%s", ns, cluster.Name)
+	}
+
+	return children, nil
+}

pkg/controller/cluster/cluster_reconciler_test.go

@@ -55,7 +55,11 @@ func TestReconcile(t *testing.T) {
 	c = mgr.GetClient()
 
 	a := newTestActuator()
-	recFn, requests := SetupTestReconcile(newReconciler(mgr, a))
+	r, err := newReconciler(mgr, a)
+	if err != nil {
+		t.Fatalf("Couldn't create controller: %v", err)
+	}
+	recFn, requests := SetupTestReconcile(r)
 	if err := add(mgr, recFn); err != nil {
 		t.Fatalf("error adding controller to manager: %v", err)
 	}

pkg/util/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "//pkg/apis/cluster/v1alpha1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/json:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
@@ -25,4 +26,8 @@ go_test(
     name = "go_default_test",
     srcs = ["util_test.go"],
     embed = [":go_default_library"],
+    deps = [
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+    ],
 )

pkg/util/util.go

@@ -30,6 +30,7 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/yaml"
@@ -312,4 +313,16 @@ func decodeClusterV1Kinds(decoder *yaml.YAMLOrJSONDecoder, kind string) ([][]byt
 	}
 
 	return outs, nil
+
+}
+
+// Returns true if any of the owner references point to the given target
+func PointsTo(refs []metav1.OwnerReference, target *metav1.ObjectMeta) bool {
+	for _, ref := range refs {
+		if ref.UID == target.UID {
+			return true
+		}
+	}
+
+	return false
 }

pkg/util/util_test.go

@@ -20,6 +20,9 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 const validCluster = `
@@ -334,3 +337,56 @@ func createTempFile(contents string) (string, error) {
 	f.WriteString(contents)
 	return f.Name(), nil
 }
+
+func TestPointsTo(t *testing.T) {
+	targetID := "fri3ndsh1p"
+
+	meta := metav1.ObjectMeta{
+		UID: types.UID(targetID),
+	}
+
+	tests := []struct {
+		name     string
+		refIDs   []string
+		expected bool
+	}{
+		{
+			name: "empty owner list",
+		},
+		{
+			name:   "single wrong owner ref",
+			refIDs: []string{"m4g1c"},
+		},
+		{
+			name:     "single right owner ref",
+			refIDs:   []string{targetID},
+			expected: true,
+		},
+		{
+			name:   "multiple wrong refs",
+			refIDs: []string{"m4g1c", "h4rm0ny"},
+		},
+		{
+			name:     "multiple refs one right",
+			refIDs:   []string{"m4g1c", targetID},
+			expected: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			pointer := &metav1.ObjectMeta{}
+
+			for _, ref := range test.refIDs {
+				pointer.OwnerReferences = append(pointer.OwnerReferences, metav1.OwnerReference{
+					UID: types.UID(ref),
+				})
+			}
+
+			result := PointsTo(pointer.OwnerReferences, &meta)
+			if result != test.expected {
+				t.Errorf("expected %v, got %v", test.expected, result)
+			}
+		})
+	}
+}