diff --git a/cmd/clusterctl/client/cluster/ownergraph.go b/cmd/clusterctl/client/cluster/ownergraph.go index 487cd7de8cc5..dd3b33b0cf45 100644 --- a/cmd/clusterctl/client/cluster/ownergraph.go +++ b/cmd/clusterctl/client/cluster/ownergraph.go @@ -56,7 +56,7 @@ func GetOwnerGraph(ctx context.Context, namespace, kubeconfigPath string) (Owner } // graph.Discovery can not be used here as it will use the latest APIVersion for ownerReferences - not those - // present in the object 'metadata.ownerReferences`. + // present in the object `metadata.ownerReferences`. owners, err := discoverOwnerGraph(ctx, namespace, graph) if err != nil { return OwnerGraph{}, errors.Wrap(err, "failed to discovery ownerGraph types") @@ -65,7 +65,7 @@ func GetOwnerGraph(ctx context.Context, namespace, kubeconfigPath string) (Owner } func discoverOwnerGraph(ctx context.Context, namespace string, o *objectGraph) (OwnerGraph, error) { - selectors := []client.ListOption{} + var selectors []client.ListOption if namespace != "" { selectors = append(selectors, client.InNamespace(namespace)) } @@ -107,7 +107,7 @@ func discoverOwnerGraph(ctx context.Context, namespace string, o *objectGraph) ( continue } // Exclude the default service account from the owner graph. - // This Secret is not longer generated by default in Kubernetes 1.24+. + // This Secret is no longer generated by default in Kubernetes 1.24+. // This is not a CAPI related Secret, so it can be ignored. if obj.GetKind() == "Secret" && strings.Contains(obj.GetName(), "default-token") { continue diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index 0dc2fc495efe..6f04ed3f1038 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -181,3 +181,36 @@ var _ = Describe("When following the Cluster API quick-start with dualstack and } }) }) + +var _ = Describe("When following the Cluster API quick-start check finalizers resilience after deletion", func() { + QuickStartSpec(ctx, func() QuickStartSpecInput { + return QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) { + // This check ensures that finalizers are resilient - i.e. correctly re-reconciled - when removed. + framework.ValidateFinalizersResilience(ctx, proxy, namespace, clusterName) + }, + } + }) +}) + +var _ = Describe("When following the Cluster API quick-start with ClusterClass check finalizers resilience after deletion [ClusterClass]", func() { + QuickStartSpec(ctx, func() QuickStartSpecInput { + return QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: pointer.String("topology"), + PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) { + // This check ensures that finalizers are resilient - i.e. correctly re-reconciled - when removed. + framework.ValidateFinalizersResilience(ctx, proxy, namespace, clusterName) + }, + } + }) +}) diff --git a/test/framework/finalizers_helpers.go b/test/framework/finalizers_helpers.go new file mode 100644 index 000000000000..a4ab71e5469e --- /dev/null +++ b/test/framework/finalizers_helpers.go @@ -0,0 +1,110 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "fmt" + "sigs.k8s.io/cluster-api/util/patch" + "time" + + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + clusterctlcluster "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ValidateFinalizersResilience checks that expected finalizers are in place, deletes them, and verifies that expected finalizers are properly added again. +func ValidateFinalizersResilience(ctx context.Context, proxy ClusterProxy, namespace, clusterName string) { + // Get the initial list of finalizers + initialFinalizers := getFinalizersFromAllObjs(ctx, proxy, namespace) + + clusterKey := client.ObjectKey{Namespace: namespace, Name: clusterName} + + // Removes all the finalizers. + // We are testing the worst-case scenario, i.e. all finalizers are deleted, even when reconciliation is paused. + // Deletion of the finalizers can either happen from a system generated attack or by mistake, which is an edge case. + // The reconciliation loop should be able to recover from this, by adding the required finalizers back. + setClusterPause(ctx, proxy.GetClient(), clusterKey, true) + + // Once all Clusters are paused remove the finalizers from all objects in the graph. + removeFinalizers(ctx, proxy, namespace) + + // Unpause the cluster. + setClusterPause(ctx, proxy.GetClient(), clusterKey, false) + + // Annotate the clusterClass, if one is in use, to speed up reconciliation. This ensures ClusterClass finalizers + // are re-reconciled before asserting the owner reference graph. + forceClusterClassReconcile(ctx, proxy.GetClient(), clusterKey) + + // Check that the finalizers are as expected after further reconciliations. + assertFinalizersExist(ctx, proxy, namespace, initialFinalizers) +} + +func getFinalizersFromAllObjs(ctx context.Context, proxy ClusterProxy, namespace string) map[string][]string { + finalizers := map[string][]string{} + graph, err := clusterctlcluster.GetOwnerGraph(ctx, proxy.GetKubeconfigPath(), namespace) + Expect(err).To(BeNil()) + for _, v := range graph { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(v.Object.APIVersion) + obj.SetKind(v.Object.Kind) + err := proxy.GetClient().Get(ctx, client.ObjectKey{Namespace: v.Object.Namespace, Name: v.Object.Name}, obj) + Expect(err).To(BeNil()) + if obj.GetFinalizers() != nil { + finalizers[fmt.Sprintf("%s/%s", v.Object.Kind, client.ObjectKey{Name: v.Object.Name, Namespace: v.Object.Namespace}.String())] = obj.GetFinalizers() + } + } + return finalizers +} + +func removeFinalizers(ctx context.Context, proxy ClusterProxy, namespace string) { + graph, err := clusterctlcluster.GetOwnerGraph(ctx, namespace, proxy.GetKubeconfigPath()) + Expect(err).ToNot(HaveOccurred()) + for _, object := range graph { + ref := object.Object + obj := new(unstructured.Unstructured) + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + obj.SetName(ref.Name) + + Expect(proxy.GetClient().Get(ctx, client.ObjectKey{Namespace: namespace, Name: object.Object.Name}, obj)).To(Succeed()) + helper, err := patch.NewHelper(obj, proxy.GetClient()) + Expect(err).ToNot(HaveOccurred()) + obj.SetFinalizers([]string{}) + Expect(helper.Patch(ctx, obj)).To(Succeed()) + } +} + +func assertFinalizersExist(ctx context.Context, proxy ClusterProxy, namespace string, initialFinalizers map[string][]string) { + Eventually(func() { + afterDeleteFinalizers := getFinalizersFromAllObjs(ctx, proxy, namespace) + + missing := map[string][]string{} + for k, v := range initialFinalizers { + if _, ok := afterDeleteFinalizers[k]; !ok { + missing[k] = v + } + } + missingString := "" + for k, v := range missing { + missingString = fmt.Sprintf("%s\n%s %s", missingString, k, v) + } + Expect(len(missing)).To(Equal(0), missingString) + }).WithTimeout(5 * time.Minute).WithPolling(2 * time.Second).Should(Succeed()) +}