From 6d06697c7778f010f242f6bed6e1854c74ed21dc Mon Sep 17 00:00:00 2001 From: Lennart Jern Date: Fri, 10 May 2024 10:48:11 +0300 Subject: [PATCH 1/3] E2E: Increase timeouts to stabilize CI We are changing CI infrastructure. An unfortunate side-effect is that some steps are taking longer than they used to. Because of this we now see some timeouts. This commit is for increasing the relevant timeouts to make the CI stable again. Signed-off-by: Lennart Jern --- config/overlays/e2e-release-0.4/kustomization.yaml | 2 +- config/overlays/e2e-release-0.5/kustomization.yaml | 2 +- config/overlays/e2e-release-0.6/kustomization.yaml | 2 +- .../e2e-release-24.0-with-inspector/kustomization.yaml | 8 ++++---- .../overlays/e2e-release-24.1/kustomization.yaml | 8 ++++---- .../overlays/e2e-with-inspector/kustomization.yaml | 6 +++--- test/e2e/config/ironic.yaml | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/config/overlays/e2e-release-0.4/kustomization.yaml b/config/overlays/e2e-release-0.4/kustomization.yaml index b0759b25a4..10c4705ec6 100644 --- a/config/overlays/e2e-release-0.4/kustomization.yaml +++ b/config/overlays/e2e-release-0.4/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: baremetal-operator-system resources: -- https://github.com/metal3-io/baremetal-operator/config/tls?ref=release-0.4 +- https://github.com/metal3-io/baremetal-operator/config/tls?ref=release-0.4&timeout=120s components: - ../../components/basic-auth configMapGenerator: diff --git a/config/overlays/e2e-release-0.5/kustomization.yaml b/config/overlays/e2e-release-0.5/kustomization.yaml index 7a37de3785..5c26ca29fa 100644 --- a/config/overlays/e2e-release-0.5/kustomization.yaml +++ b/config/overlays/e2e-release-0.5/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: baremetal-operator-system resources: -- https://github.com/metal3-io/baremetal-operator/config/overlays/basic-auth_tls?ref=release-0.5 +- https://github.com/metal3-io/baremetal-operator/config/overlays/basic-auth_tls?ref=release-0.5&timeout=120s configMapGenerator: - name: ironic behavior: create diff --git a/config/overlays/e2e-release-0.6/kustomization.yaml b/config/overlays/e2e-release-0.6/kustomization.yaml index 1e7b25240d..f78cb6c1d8 100644 --- a/config/overlays/e2e-release-0.6/kustomization.yaml +++ b/config/overlays/e2e-release-0.6/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: baremetal-operator-system resources: -- https://github.com/metal3-io/baremetal-operator/config/overlays/basic-auth_tls?ref=release-0.6 +- https://github.com/metal3-io/baremetal-operator/config/overlays/basic-auth_tls?ref=release-0.6&timeout=120s configMapGenerator: - name: ironic behavior: create diff --git a/ironic-deployment/overlays/e2e-release-24.0-with-inspector/kustomization.yaml b/ironic-deployment/overlays/e2e-release-24.0-with-inspector/kustomization.yaml index 1d6ccb25b8..2f06326fb7 100644 --- a/ironic-deployment/overlays/e2e-release-24.0-with-inspector/kustomization.yaml +++ b/ironic-deployment/overlays/e2e-release-24.0-with-inspector/kustomization.yaml @@ -2,12 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: baremetal-operator-system resources: -- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.5 -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5 +- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.5&timeout=120s +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5&timeout=120s components: -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5 -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5 +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5&timeout=120s +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5&timeout=120s configMapGenerator: - envs: diff --git a/ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml b/ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml index f52e3da9c2..1936f375f6 100644 --- a/ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml +++ b/ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml @@ -2,12 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: baremetal-operator-system resources: -- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.6 -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.6 +- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.6&timeout=120s +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.6&timeout=120s components: -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.6 -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.6 +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.6&timeout=120s +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.6&timeout=120s configMapGenerator: - envs: diff --git a/ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml b/ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml index 39a9fadecc..d1009417ca 100644 --- a/ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml +++ b/ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml @@ -3,11 +3,11 @@ kind: Kustomization namespace: baremetal-operator-system resources: - ../../../config/namespace -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5 +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5&timeout=120s components: -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5 -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5 +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5&timeout=120s +- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5&timeout=120s configMapGenerator: - envs: diff --git a/test/e2e/config/ironic.yaml b/test/e2e/config/ironic.yaml index 9568cf8aeb..6c275faece 100644 --- a/test/e2e/config/ironic.yaml +++ b/test/e2e/config/ironic.yaml @@ -42,7 +42,7 @@ intervals: external-inspection/wait-available: ["1m", "1s"] default/wait-inspecting: ["2m", "2s"] default/wait-available: ["10m", "1s"] - default/wait-deployment: ["5m", "1s"] + default/wait-deployment: ["10m", "1s"] default/wait-namespace-deleted: ["10m", "1s"] ironic/wait-deployment: ["10m", "2s"] default/wait-power-state: ["10m", "100ms"] From 975bdc30da8e20dcc7f6f34eea3721328f72aade Mon Sep 17 00:00:00 2001 From: Lennart Jern Date: Tue, 14 May 2024 08:39:39 +0300 Subject: [PATCH 2/3] E2E: Wait for baremetal-operator-system namespace to be deleted Signed-off-by: Lennart Jern --- test/e2e/upgrade_test.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index 95a9f63c7b..33778f6d85 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -214,7 +214,7 @@ func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeCl DeferCleanup(func() { By(fmt.Sprintf("Removing Ironic kustomization %s from the upgrade cluster", initIronicKustomization)) - BuildAndRemoveKustomization(ctx, initIronicKustomization, upgradeClusterProxy) + cleanupBaremetalOperatorSystem(ctx, upgradeClusterProxy, initIronicKustomization) }) } if input.DeployBMO { @@ -233,7 +233,7 @@ func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeCl Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { By(fmt.Sprintf("Removing BMO kustomization %s from the upgrade cluster", initBMOKustomization)) - BuildAndRemoveKustomization(ctx, initBMOKustomization, upgradeClusterProxy) + cleanupBaremetalOperatorSystem(ctx, upgradeClusterProxy, initBMOKustomization) }) } @@ -302,7 +302,7 @@ func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeCl Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { By(fmt.Sprintf("Removing %s kustomization %s from the upgrade cluster", input.UpgradeEntityName, upgradeKustomization)) - BuildAndRemoveKustomization(ctx, upgradeKustomization, upgradeClusterProxy) + cleanupBaremetalOperatorSystem(ctx, upgradeClusterProxy, upgradeKustomization) }) By(fmt.Sprintf("Waiting for %s update to rollout", input.UpgradeEntityName)) @@ -415,3 +415,14 @@ var _ = Describe("Upgrade", Label("optional", "upgrade"), func() { cleanup(ctx, upgradeClusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) }) }) + +// cleanupBaremetalOperatorSystem removes the kustomization from the cluster and waits for the +// baremetal-operator-system namespace to be deleted. +func cleanupBaremetalOperatorSystem(ctx context.Context, clusterProxy framework.ClusterProxy, kustomization string) { + BuildAndRemoveKustomization(ctx, kustomization, clusterProxy) + // We need to ensure that the namespace actually gets deleted. + WaitForNamespaceDeleted(ctx, WaitForNamespaceDeletedInput{ + Getter: clusterProxy.GetClient(), + Namespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "baremetal-operator-system"}}, + }, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) +} From ce09ad62dfbc2826105777e59b328463f9956f61 Mon Sep 17 00:00:00 2001 From: Lennart Jern Date: Tue, 14 May 2024 14:38:39 +0300 Subject: [PATCH 3/3] E2E: Implement flake attempts This adds a function for retrying function calls that are flaky and makes use of it for kustomizations. Signed-off-by: Lennart Jern --- test/e2e/common.go | 13 +++++++++ test/e2e/e2e_suite_test.go | 41 ++++++++++++++------------ test/e2e/upgrade_test.go | 60 +++++++++++++++++++++----------------- 3 files changed, 68 insertions(+), 46 deletions(-) diff --git a/test/e2e/common.go b/test/e2e/common.go index 5e81185711..a20ef790b2 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -438,3 +438,16 @@ func AnnotateBmh(ctx context.Context, client client.Client, host metal3api.BareM func Logf(format string, a ...interface{}) { fmt.Fprintf(GinkgoWriter, "INFO: "+format+"\n", a...) } + +// FlakeAttempt retries the given function up to attempts times. +func FlakeAttempt(attempts int, f func() error) error { + var err error + for i := 0; i < attempts; i++ { + err = f() + if err == nil { + return nil + } + Logf("Attempt %d failed: %v", i+1, err) + } + return err +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index d5da72296b..dc71cfb178 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -128,32 +128,35 @@ var _ = SynchronizedBeforeSuite(func() []byte { if e2eConfig.GetVariable("DEPLOY_IRONIC") != "false" { // Install Ironic By("Installing Ironic") - err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ - Kustomization: e2eConfig.GetVariable("IRONIC_KUSTOMIZATION"), - ClusterProxy: clusterProxy, - WaitForDeployment: true, - WatchDeploymentLogs: true, - DeploymentName: "ironic", - DeploymentNamespace: bmoIronicNamespace, - LogPath: filepath.Join(artifactFolder, "logs", bmoIronicNamespace), - WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + err := FlakeAttempt(2, func() error { + return BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: e2eConfig.GetVariable("IRONIC_KUSTOMIZATION"), + ClusterProxy: clusterProxy, + WaitForDeployment: true, + WatchDeploymentLogs: true, + DeploymentName: "ironic", + DeploymentNamespace: bmoIronicNamespace, + LogPath: filepath.Join(artifactFolder, "logs", bmoIronicNamespace), + WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + }) }) Expect(err).NotTo(HaveOccurred()) - } if e2eConfig.GetVariable("DEPLOY_BMO") != "false" { // Install BMO By("Installing BMO") - err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ - Kustomization: e2eConfig.GetVariable("BMO_KUSTOMIZATION"), - ClusterProxy: clusterProxy, - WaitForDeployment: true, - WatchDeploymentLogs: true, - DeploymentName: "baremetal-operator-controller-manager", - DeploymentNamespace: bmoIronicNamespace, - LogPath: filepath.Join(artifactFolder, "logs", bmoIronicNamespace), - WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + err := FlakeAttempt(2, func() error { + return BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: e2eConfig.GetVariable("BMO_KUSTOMIZATION"), + ClusterProxy: clusterProxy, + WaitForDeployment: true, + WatchDeploymentLogs: true, + DeploymentName: "baremetal-operator-controller-manager", + DeploymentNamespace: bmoIronicNamespace, + LogPath: filepath.Join(artifactFolder, "logs", bmoIronicNamespace), + WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + }) }) Expect(err).NotTo(HaveOccurred()) } diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index 33778f6d85..cd02daeb9b 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -200,15 +200,17 @@ func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeCl if input.DeployIronic { // Install Ironic By(fmt.Sprintf("Installing Ironic from kustomization %s on the upgrade cluster", initIronicKustomization)) - err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ - Kustomization: initIronicKustomization, - ClusterProxy: upgradeClusterProxy, - WaitForDeployment: true, - WatchDeploymentLogs: true, - DeploymentName: "ironic", - DeploymentNamespace: bmoIronicNamespace, - LogPath: filepath.Join(testCaseArtifactFolder, "logs", "init-ironic"), - WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + err := FlakeAttempt(2, func() error { + return BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: initIronicKustomization, + ClusterProxy: upgradeClusterProxy, + WaitForDeployment: true, + WatchDeploymentLogs: true, + DeploymentName: "ironic", + DeploymentNamespace: bmoIronicNamespace, + LogPath: filepath.Join(testCaseArtifactFolder, "logs", "init-ironic"), + WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + }) }) Expect(err).NotTo(HaveOccurred()) @@ -220,15 +222,17 @@ func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeCl if input.DeployBMO { // Install BMO By(fmt.Sprintf("Installing BMO from %s on the upgrade cluster", initBMOKustomization)) - err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ - Kustomization: initBMOKustomization, - ClusterProxy: upgradeClusterProxy, - WaitForDeployment: true, - WatchDeploymentLogs: true, - DeploymentName: "baremetal-operator-controller-manager", - DeploymentNamespace: bmoIronicNamespace, - LogPath: filepath.Join(testCaseArtifactFolder, "logs", "init-bmo"), - WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + err := FlakeAttempt(2, func() error { + return BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: initBMOKustomization, + ClusterProxy: upgradeClusterProxy, + WaitForDeployment: true, + WatchDeploymentLogs: true, + DeploymentName: "baremetal-operator-controller-manager", + DeploymentNamespace: bmoIronicNamespace, + LogPath: filepath.Join(testCaseArtifactFolder, "logs", "init-bmo"), + WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + }) }) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { @@ -289,15 +293,17 @@ func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeCl deploy, err := clientSet.AppsV1().Deployments(bmoIronicNamespace).Get(ctx, upgradeDeploymentName, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) upgradeKustomization := input.UpgradeEntityKustomization - err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ - Kustomization: upgradeKustomization, - ClusterProxy: upgradeClusterProxy, - WaitForDeployment: false, - WatchDeploymentLogs: true, - DeploymentName: upgradeDeploymentName, - DeploymentNamespace: bmoIronicNamespace, - LogPath: filepath.Join(testCaseArtifactFolder, "logs", "bmo-upgrade-main"), - WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + err = FlakeAttempt(2, func() error { + return BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: upgradeKustomization, + ClusterProxy: upgradeClusterProxy, + WaitForDeployment: false, + WatchDeploymentLogs: true, + DeploymentName: upgradeDeploymentName, + DeploymentNamespace: bmoIronicNamespace, + LogPath: filepath.Join(testCaseArtifactFolder, "logs", "bmo-upgrade-main"), + WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), + }) }) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() {