Skip to content

Commit

Permalink
own
Browse files Browse the repository at this point in the history
  • Loading branch information
@killianmuldoon
killianmuldoon committed Mar 15, 2023
1 parent a32f66e commit 44c7c80
Show file tree
Hide file tree
Showing 4 changed files with 162 additions and 33 deletions.
95 changes: 66 additions & 29 deletions cmd/clusterctl/client/cluster/ownergraph.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ import (
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/controller-runtime/pkg/client"

clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
)

// OwnerGraph contains a graph with all the objects considered by clusterctl move as nodes and the OwnerReference relationship
Expand All @@ -32,22 +36,6 @@ type OwnerGraphNode struct {
Owners []metav1.OwnerReference
}

func nodeToOwnerRef(n *node, attributes ownerReferenceAttributes) metav1.OwnerReference {
ref := metav1.OwnerReference{
Name: n.identity.Name,
APIVersion: n.identity.APIVersion,
Kind: n.identity.Kind,
UID: n.identity.UID,
}
if attributes.BlockOwnerDeletion != nil {
ref.BlockOwnerDeletion = attributes.BlockOwnerDeletion
}
if attributes.Controller != nil {
ref.Controller = attributes.Controller
}
return ref
}

// GetOwnerGraph returns a graph with all the objects considered by clusterctl move as nodes and the OwnerReference relationship between those objects as edges.
// NOTE: this data structure is exposed to allow implementation of E2E tests verifying that CAPI can properly rebuild its
// own owner references; there is no guarantee about the stability of this API. Using this test with providers may require
Expand All @@ -64,20 +52,69 @@ func GetOwnerGraph(namespace, kubeconfigPath string) (OwnerGraph, error) {
return OwnerGraph{}, errors.Wrap(err, "failed to retrieve discovery types")
}

// Discovery the object graph for the selected types:
// - Nodes are defined the Kubernetes objects (Clusters, Machines etc.) identified during the discovery process.
// - Edges are derived by the OwnerReferences between nodes.
if err := graph.Discovery(namespace); err != nil {
return OwnerGraph{}, errors.Wrap(err, "failed to discover the object graph")
owners, err := discoverOwnerGraph(namespace, graph)
if err != nil {
return OwnerGraph{}, errors.Wrap(err, "failed to discovery ownerGraph types")
}
return owners, nil
}

func discoverOwnerGraph(namespace string, o *objectGraph) (OwnerGraph, error) {
selectors := []client.ListOption{}
if namespace != "" {
selectors = append(selectors, client.InNamespace(namespace))
}
owners := OwnerGraph{}
// Using getMoveNodes here ensures only objects that are part of the Cluster are added to the OwnerGraph.
for _, v := range graph.getMoveNodes() {
n := OwnerGraphNode{Object: v.identity, Owners: []metav1.OwnerReference{}}
for owner, attributes := range v.owners {
n.Owners = append(n.Owners, nodeToOwnerRef(owner, attributes))
ownerGraph := OwnerGraph{}

discoveryBackoff := newReadBackoff()
for _, discoveryType := range o.types {
typeMeta := discoveryType.typeMeta
objList := new(unstructured.UnstructuredList)

if err := retryWithExponentialBackoff(discoveryBackoff, func() error {
return getObjList(o.proxy, typeMeta, selectors, objList)
}); err != nil {
return nil, err
}

// if we are discovering Secrets, also secrets from the providers namespace should be included.
if discoveryType.typeMeta.GetObjectKind().GroupVersionKind().GroupKind() == corev1.SchemeGroupVersion.WithKind("SecretList").GroupKind() {
providers, err := o.providerInventory.List()
if err != nil {
return nil, err
}
for _, p := range providers.Items {
if p.Type == string(clusterctlv1.InfrastructureProviderType) {
providerNamespaceSelector := []client.ListOption{client.InNamespace(p.Namespace)}
providerNamespaceSecretList := new(unstructured.UnstructuredList)
if err := retryWithExponentialBackoff(discoveryBackoff, func() error {
return getObjList(o.proxy, typeMeta, providerNamespaceSelector, providerNamespaceSecretList)
}); err != nil {
return nil, err
}
objList.Items = append(objList.Items, providerNamespaceSecretList.Items...)
}
}
}

for i := range objList.Items {
obj := objList.Items[i]
ownerGraph = addNodeToOwnerGraph(ownerGraph, obj)
}
owners[string(v.identity.UID)] = n
}
return owners, nil
return ownerGraph, nil
}

func addNodeToOwnerGraph(graph OwnerGraph, obj unstructured.Unstructured) OwnerGraph {
// write code to add a node to the ownerGraph
graph[string(obj.GetUID())] = OwnerGraphNode{
Owners: obj.GetOwnerReferences(),
Object: corev1.ObjectReference{
APIVersion: obj.GetAPIVersion(),
Kind: obj.GetKind(),
Name: obj.GetName(),
Namespace: obj.GetNamespace(),
},
}
return graph
}
4 changes: 2 additions & 2 deletions test/e2e/clusterctl_upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ type ClusterctlUpgradeSpecInput struct {
ControlPlaneWaiters clusterctl.ControlPlaneWaiters
PreInit func(managementClusterProxy framework.ClusterProxy)
PreUpgrade func(managementClusterProxy framework.ClusterProxy)
PostUpgrade func(managementClusterProxy framework.ClusterProxy)
PostUpgrade func(managementClusterProxy framework.ClusterProxy, namespace, clusterName string)
// PreCleanupManagementCluster hook can be used for extra steps that might be required from providers, for example, remove conflicting service (such as DHCP) running on
// the target management cluster and run it on bootstrap (before the latter resumes LCM) if both clusters share the same LAN
PreCleanupManagementCluster func(managementClusterProxy framework.ClusterProxy)
Expand Down Expand Up @@ -432,7 +432,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg

if input.PostUpgrade != nil {
By("Running Post-upgrade steps against the management cluster")
input.PostUpgrade(managementClusterProxy)
input.PostUpgrade(managementClusterProxy, testNamespace.Name, managementClusterName)
}

// After the upgrade check that there were no unexpected rollouts.
Expand Down
83 changes: 81 additions & 2 deletions test/e2e/clusterctl_upgrade_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ package e2e

import (
. "github.com/onsi/ginkgo/v2"

"sigs.k8s.io/cluster-api/test/framework"
)

var _ = Describe("When testing clusterctl upgrades (v0.3=>current)", func() {
var _ = Describe("When testing clusterctl upgrades (v0.3=>current) [PR-Blocking]", func() {
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
Expand All @@ -39,6 +41,17 @@ var _ = Describe("When testing clusterctl upgrades (v0.3=>current)", func() {
UpgradeClusterctlVariables: map[string]string{
"CLUSTER_TOPOLOGY": "false",
},
// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
framework.ValidateOwnerReferencesOnUpdate(ctx, proxy, namespace,
framework.CoreTypeOwnerReferenceAssertion,
framework.ExpOwnerReferenceAssertions,
framework.DockerInfraOwnerReferenceAssertions,
framework.KubeadmBootstrapOwnerReferenceAssertions,
framework.KubeadmControlPlaneOwnerReferenceAssertions,
framework.KubernetesReferenceAssertions,
)
},
}
})
})
Expand All @@ -53,7 +66,18 @@ var _ = Describe("When testing clusterctl upgrades (v0.4=>current)", func() {
SkipCleanup: skipCleanup,
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.8/clusterctl-{OS}-{ARCH}",
InitWithProvidersContract: "v1alpha4",
InitWithKubernetesVersion: "v1.23.13",
InitWithKubernetesVersion: "v1.25.3",
// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
framework.ValidateOwnerReferencesOnUpdate(ctx, proxy, namespace,
framework.CoreTypeOwnerReferenceAssertion,
framework.ExpOwnerReferenceAssertions,
framework.DockerInfraOwnerReferenceAssertions,
framework.KubeadmBootstrapOwnerReferenceAssertions,
framework.KubeadmControlPlaneOwnerReferenceAssertions,
framework.KubernetesReferenceAssertions,
)
},
}
})
})
Expand All @@ -78,6 +102,17 @@ var _ = Describe("When testing clusterctl upgrades (v1.0=>current)", func() {
// try to deploy the latest version of our test-extension from docker.yaml.
InitWithRuntimeExtensionProviders: []string{},
InitWithKubernetesVersion: "v1.23.13",
// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
framework.ValidateOwnerReferencesOnUpdate(ctx, proxy, namespace,
framework.CoreTypeOwnerReferenceAssertion,
framework.ExpOwnerReferenceAssertions,
framework.DockerInfraOwnerReferenceAssertions,
framework.KubeadmBootstrapOwnerReferenceAssertions,
framework.KubeadmControlPlaneOwnerReferenceAssertions,
framework.KubernetesReferenceAssertions,
)
},
}
})
})
Expand All @@ -102,6 +137,17 @@ var _ = Describe("When testing clusterctl upgrades (v1.2=>current)", func() {
// try to deploy the latest version of our test-extension from docker.yaml.
InitWithRuntimeExtensionProviders: []string{},
InitWithKubernetesVersion: "v1.26.0",
// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
framework.ValidateOwnerReferencesOnUpdate(ctx, proxy, namespace,
framework.CoreTypeOwnerReferenceAssertion,
framework.ExpOwnerReferenceAssertions,
framework.DockerInfraOwnerReferenceAssertions,
framework.KubeadmBootstrapOwnerReferenceAssertions,
framework.KubeadmControlPlaneOwnerReferenceAssertions,
framework.KubernetesReferenceAssertions,
)
},
}
})
})
Expand All @@ -127,6 +173,17 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (v1.2=>cur
InitWithRuntimeExtensionProviders: []string{},
InitWithKubernetesVersion: "v1.26.0",
WorkloadFlavor: "topology",
// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
framework.ValidateOwnerReferencesOnUpdate(ctx, proxy, namespace,
framework.CoreTypeOwnerReferenceAssertion,
framework.ExpOwnerReferenceAssertions,
framework.DockerInfraOwnerReferenceAssertions,
framework.KubeadmBootstrapOwnerReferenceAssertions,
framework.KubeadmControlPlaneOwnerReferenceAssertions,
framework.KubernetesReferenceAssertions,
)
},
}
})
})
Expand All @@ -142,6 +199,17 @@ var _ = Describe("When testing clusterctl upgrades (v1.3=>current)", func() {
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.0/clusterctl-{OS}-{ARCH}",
InitWithProvidersContract: "v1beta1",
InitWithKubernetesVersion: "v1.26.0",
// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
framework.ValidateOwnerReferencesOnUpdate(ctx, proxy, namespace,
framework.CoreTypeOwnerReferenceAssertion,
framework.ExpOwnerReferenceAssertions,
framework.DockerInfraOwnerReferenceAssertions,
framework.KubeadmBootstrapOwnerReferenceAssertions,
framework.KubeadmControlPlaneOwnerReferenceAssertions,
framework.KubernetesReferenceAssertions,
)
},
}
})
})
Expand All @@ -158,6 +226,17 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (v1.3=>cur
InitWithProvidersContract: "v1beta1",
InitWithKubernetesVersion: "v1.26.0",
WorkloadFlavor: "topology",
// This check ensures that ownerReference apiVersions are updated for all types after the upgrade.
PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
framework.ValidateOwnerReferencesOnUpdate(ctx, proxy, namespace,
framework.CoreTypeOwnerReferenceAssertion,
framework.ExpOwnerReferenceAssertions,
framework.DockerInfraOwnerReferenceAssertions,
framework.KubeadmBootstrapOwnerReferenceAssertions,
framework.KubeadmControlPlaneOwnerReferenceAssertions,
framework.KubernetesReferenceAssertions,
)
},
}
})
})
13 changes: 13 additions & 0 deletions test/framework/ownerreference_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,16 @@ import (
clusterctlcluster "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
runtimev1 "sigs.k8s.io/cluster-api/exp/runtime/api/v1alpha1"
"sigs.k8s.io/cluster-api/util/patch"
)

// ValidateOwnerReferencesResilience checks that expected owner references are in place, deletes them, and verifies that expect owner references are properly rebuilt.
func ValidateOwnerReferencesOnUpdate(ctx context.Context, proxy ClusterProxy, namespace string, assertFuncs ...map[string]func(reference []metav1.OwnerReference) error) {
// Check that the ownerReferences are as expected on the first iteration.
AssertOwnerReferences(namespace, proxy.GetKubeconfigPath(), assertFuncs...)
}

// ValidateOwnerReferencesResilience checks that expected owner references are in place, deletes them, and verifies that expect owner references are properly rebuilt.
func ValidateOwnerReferencesResilience(ctx context.Context, proxy ClusterProxy, namespace, clusterName string, assertFuncs ...map[string]func(reference []metav1.OwnerReference) error) {
// Check that the ownerReferences are as expected on the first iteration.
Expand Down Expand Up @@ -105,12 +112,14 @@ var (
machineSetKind = "MachineSet"
machineDeploymentKind = "MachineDeployment"
machineHealthCheckKind = "MachineHealthCheck"
extensionConfigKind = "ExtensionConfig"

clusterClassGVK = clusterv1.GroupVersion.WithKind(clusterClassKind)
clusterGVK = clusterv1.GroupVersion.WithKind(clusterKind)
machineDeploymentGVK = clusterv1.GroupVersion.WithKind(machineDeploymentKind)
machineSetGVK = clusterv1.GroupVersion.WithKind(machineSetKind)
machineGVK = clusterv1.GroupVersion.WithKind(machineKind)
extensionConfigGVK = runtimev1.GroupVersion.WithKind(extensionConfigKind)
)

// CoreTypeOwnerReferenceAssertion maps Cluster API core types to functions which return an error if the passed
Expand Down Expand Up @@ -140,6 +149,10 @@ var CoreTypeOwnerReferenceAssertion = map[string]func([]metav1.OwnerReference) e
// MachineHealthChecks must be owned by the Cluster.
return hasExactOwnersByGVK(owners, []schema.GroupVersionKind{clusterGVK})
},
extensionConfigKind: func(owners []metav1.OwnerReference) error {
// ExtensionConfig should have no owners.
return hasExactOwnersByGVK(owners, []schema.GroupVersionKind{})
},
}

// Kind and GVK for types in the exp package.
Expand Down

0 comments on commit 44c7c80

Please sign in to comment.