Skip to content

Commit

Permalink
Merge pull request #9965 from k8s-infra-cherrypick-robot/cherry-pick-…
Browse files Browse the repository at this point in the history 
…9963-to-release-1.6

[release-1.6] 🌱 Make etcd and CoreDNS optional in upgrade and self-hosted tests
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Jan 8, 2024
2 parents 1677c6a + 3c48cd8 commit eb46d08
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 33 deletions.
20 changes: 14 additions & 6 deletions test/e2e/cluster_upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
controlPlaneMachineCount int64
workerMachineCount int64

etcdVersionUpgradeTo string
coreDNSVersionUpgradeTo string

clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
kubetestConfigFilePath string
)
Expand All @@ -98,8 +101,6 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust

Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeFrom))
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeTo))
Expect(input.E2EConfig.Variables).To(HaveKey(EtcdVersionUpgradeTo))
Expect(input.E2EConfig.Variables).To(HaveKey(CoreDNSVersionUpgradeTo))

Expect(input.E2EConfig.Variables).To(HaveKey(kubetestConfigurationVariable), "% spec requires a %s variable to be defined in the config file", specName, kubetestConfigurationVariable)
kubetestConfigFilePath = input.E2EConfig.GetVariable(kubetestConfigurationVariable)
Expand All @@ -117,6 +118,13 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
workerMachineCount = *input.WorkerMachineCount
}

if input.E2EConfig.HasVariable(EtcdVersionUpgradeTo) {
etcdVersionUpgradeTo = input.E2EConfig.GetVariable(EtcdVersionUpgradeTo)
}
if input.E2EConfig.HasVariable(CoreDNSVersionUpgradeTo) {
coreDNSVersionUpgradeTo = input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo)
}

// Setup a Namespace where to host objects for this spec and create a watcher for the Namespace events.
namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
Expand Down Expand Up @@ -158,8 +166,8 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
ClusterProxy: input.BootstrapClusterProxy,
Cluster: clusterResources.Cluster,
ControlPlane: clusterResources.ControlPlane,
EtcdImageTag: input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
DNSImageTag: input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
EtcdImageTag: etcdVersionUpgradeTo,
DNSImageTag: coreDNSVersionUpgradeTo,
MachineDeployments: clusterResources.MachineDeployments,
MachinePools: clusterResources.MachinePools,
KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
Expand Down Expand Up @@ -189,8 +197,8 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
ClusterProxy: input.BootstrapClusterProxy,
Cluster: clusterResources.Cluster,
ControlPlane: clusterResources.ControlPlane,
EtcdImageTag: input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
DNSImageTag: input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
EtcdImageTag: etcdVersionUpgradeTo,
DNSImageTag: coreDNSVersionUpgradeTo,
KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
UpgradeMachineTemplate: upgradeCPMachineTemplateTo,
WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
Expand Down
2 changes: 0 additions & 2 deletions test/e2e/cluster_upgrade_runtimesdk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,6 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl

Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeFrom))
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeTo))
Expect(input.E2EConfig.Variables).To(HaveKey(EtcdVersionUpgradeTo))
Expect(input.E2EConfig.Variables).To(HaveKey(CoreDNSVersionUpgradeTo))

if input.ControlPlaneMachineCount == nil {
controlPlaneMachineCount = 1
Expand Down
20 changes: 14 additions & 6 deletions test/e2e/self_hosted.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,9 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
controlPlaneMachineCount int64
workerMachineCount int64

etcdVersionUpgradeTo string
coreDNSVersionUpgradeTo string

kubernetesVersion string
)

Expand All @@ -113,10 +116,15 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
} else {
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeFrom))
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeTo))
Expect(input.E2EConfig.Variables).To(HaveKey(EtcdVersionUpgradeTo))
Expect(input.E2EConfig.Variables).To(HaveKey(CoreDNSVersionUpgradeTo))

kubernetesVersion = input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom)

if input.E2EConfig.HasVariable(EtcdVersionUpgradeTo) {
etcdVersionUpgradeTo = input.E2EConfig.GetVariable(EtcdVersionUpgradeTo)
}
if input.E2EConfig.HasVariable(CoreDNSVersionUpgradeTo) {
coreDNSVersionUpgradeTo = input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo)
}
}

// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
Expand Down Expand Up @@ -314,8 +322,8 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
ClusterProxy: selfHostedClusterProxy,
Cluster: clusterResources.Cluster,
ControlPlane: clusterResources.ControlPlane,
EtcdImageTag: input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
DNSImageTag: input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
EtcdImageTag: etcdVersionUpgradeTo,
DNSImageTag: coreDNSVersionUpgradeTo,
MachineDeployments: clusterResources.MachineDeployments,
MachinePools: clusterResources.MachinePools,
KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
Expand Down Expand Up @@ -345,8 +353,8 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
ClusterProxy: selfHostedClusterProxy,
Cluster: clusterResources.Cluster,
ControlPlane: clusterResources.ControlPlane,
EtcdImageTag: input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
DNSImageTag: input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
EtcdImageTag: etcdVersionUpgradeTo,
DNSImageTag: coreDNSVersionUpgradeTo,
KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
UpgradeMachineTemplate: upgradeCPMachineTemplateTo,
WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
Expand Down
4 changes: 2 additions & 2 deletions test/framework/cluster_topology_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,11 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC

input.Cluster.Spec.Topology.Version = input.KubernetesUpgradeVersion
for i, variable := range input.Cluster.Spec.Topology.Variables {
if variable.Name == "etcdImageTag" {
if input.EtcdImageTag != "" && variable.Name == "etcdImageTag" {
// NOTE: strconv.Quote is used to produce a valid JSON string.
input.Cluster.Spec.Topology.Variables[i].Value = apiextensionsv1.JSON{Raw: []byte(strconv.Quote(input.EtcdImageTag))}
}
if variable.Name == "coreDNSImageTag" {
if input.DNSImageTag != "" && variable.Name == "coreDNSImageTag" {
// NOTE: strconv.Quote is used to produce a valid JSON string.
input.Cluster.Spec.Topology.Variables[i].Value = apiextensionsv1.JSON{Raw: []byte(strconv.Quote(input.DNSImageTag))}
}
Expand Down
40 changes: 23 additions & 17 deletions test/framework/controlplane_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,8 +326,6 @@ func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeCont
Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeControlPlaneAndWaitForUpgrade")
Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling UpgradeControlPlaneAndWaitForUpgrade")
Expect(input.KubernetesUpgradeVersion).ToNot(BeNil(), "Invalid argument. input.KubernetesUpgradeVersion can't be empty when calling UpgradeControlPlaneAndWaitForUpgrade")
Expect(input.EtcdImageTag).ToNot(BeNil(), "Invalid argument. input.EtcdImageTag can't be empty when calling UpgradeControlPlaneAndWaitForUpgrade")
Expect(input.DNSImageTag).ToNot(BeNil(), "Invalid argument. input.DNSImageTag can't be empty when calling UpgradeControlPlaneAndWaitForUpgrade")

mgmtClient := input.ClusterProxy.GetClient()

Expand All @@ -348,8 +346,12 @@ func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeCont
input.ControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local = new(bootstrapv1.LocalEtcd)
}

input.ControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local.ImageMeta.ImageTag = input.EtcdImageTag
input.ControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.DNS.ImageMeta.ImageTag = input.DNSImageTag
if input.EtcdImageTag != "" {
input.ControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local.ImageMeta.ImageTag = input.EtcdImageTag
}
if input.DNSImageTag != "" {
input.ControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.DNS.ImageMeta.ImageTag = input.DNSImageTag
}

Eventually(func() error {
return patchHelper.Patch(ctx, input.ControlPlane)
Expand All @@ -371,20 +373,24 @@ func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeCont
KubernetesVersion: input.KubernetesUpgradeVersion,
}, input.WaitForKubeProxyUpgrade...)

log.Logf("Waiting for CoreDNS to have the upgraded image tag")
WaitForDNSUpgrade(ctx, WaitForDNSUpgradeInput{
Getter: workloadClient,
DNSVersion: input.DNSImageTag,
}, input.WaitForDNSUpgrade...)
if input.DNSImageTag != "" {
log.Logf("Waiting for CoreDNS to have the upgraded image tag")
WaitForDNSUpgrade(ctx, WaitForDNSUpgradeInput{
Getter: workloadClient,
DNSVersion: input.DNSImageTag,
}, input.WaitForDNSUpgrade...)
}

log.Logf("Waiting for etcd to have the upgraded image tag")
lblSelector, err := labels.Parse("component=etcd")
Expect(err).ToNot(HaveOccurred())
WaitForPodListCondition(ctx, WaitForPodListConditionInput{
Lister: workloadClient,
ListOptions: &client.ListOptions{LabelSelector: lblSelector},
Condition: EtcdImageTagCondition(input.EtcdImageTag, int(*input.ControlPlane.Spec.Replicas)),
}, input.WaitForEtcdUpgrade...)
if input.EtcdImageTag != "" {
log.Logf("Waiting for etcd to have the upgraded image tag")
lblSelector, err := labels.Parse("component=etcd")
Expect(err).ToNot(HaveOccurred())
WaitForPodListCondition(ctx, WaitForPodListConditionInput{
Lister: workloadClient,
ListOptions: &client.ListOptions{LabelSelector: lblSelector},
Condition: EtcdImageTagCondition(input.EtcdImageTag, int(*input.ControlPlane.Spec.Replicas)),
}, input.WaitForEtcdUpgrade...)
}
}

// controlPlaneMachineOptions returns a set of ListOptions that allows to get all machine objects belonging to control plane.
Expand Down

0 comments on commit eb46d08

Please sign in to comment.