🌱 E2E: Increase timeouts to stabilize CI

config/overlays/e2e-release-0.4/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: baremetal-operator-system
 resources:
-- https://github.com/metal3-io/baremetal-operator/config/tls?ref=release-0.4
+- https://github.com/metal3-io/baremetal-operator/config/tls?ref=release-0.4&timeout=120s
 components:
 - ../../components/basic-auth
 configMapGenerator:

config/overlays/e2e-release-0.5/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: baremetal-operator-system
 resources:
-- https://github.com/metal3-io/baremetal-operator/config/overlays/basic-auth_tls?ref=release-0.5
+- https://github.com/metal3-io/baremetal-operator/config/overlays/basic-auth_tls?ref=release-0.5&timeout=120s
 configMapGenerator:
 - name: ironic
   behavior: create

config/overlays/e2e-release-0.6/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: baremetal-operator-system
 resources:
-- https://github.com/metal3-io/baremetal-operator/config/overlays/basic-auth_tls?ref=release-0.6
+- https://github.com/metal3-io/baremetal-operator/config/overlays/basic-auth_tls?ref=release-0.6&timeout=120s
 configMapGenerator:
 - name: ironic
   behavior: create

ironic-deployment/overlays/e2e-release-24.0-with-inspector/kustomization.yaml

@@ -2,12 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: baremetal-operator-system
 resources:
-- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.5
-- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5
+- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.5&timeout=120s
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5&timeout=120s
 
 components:
-- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5
-- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5&timeout=120s
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5&timeout=120s
 
 configMapGenerator:
 - envs:

ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml

@@ -2,12 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: baremetal-operator-system
 resources:
-- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.6
-- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.6
+- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.6&timeout=120s
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.6&timeout=120s
 
 components:
-- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.6
-- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.6
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.6&timeout=120s
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.6&timeout=120s
 
 configMapGenerator:
 - envs:

ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml

@@ -3,11 +3,11 @@ kind: Kustomization
 namespace: baremetal-operator-system
 resources:
 - ../../../config/namespace
-- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5&timeout=120s
 
 components:
-- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5
-- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5&timeout=120s
+- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5&timeout=120s
 
 configMapGenerator:
 - envs:

test/e2e/common.go

@@ -438,3 +438,16 @@ func AnnotateBmh(ctx context.Context, client client.Client, host metal3api.BareM
 func Logf(format string, a ...interface{}) {
 	fmt.Fprintf(GinkgoWriter, "INFO: "+format+"\n", a...)
 }
+
+// FlakeAttempt retries the given function up to attempts times.
+func FlakeAttempt(attempts int, f func() error) error {
+	var err error
+	for i := 0; i < attempts; i++ {
+		err = f()
+		if err == nil {
+			return nil
+		}
+		Logf("Attempt %d failed: %v", i+1, err)
+	}
+	return err
+}

test/e2e/config/ironic.yaml

@@ -42,7 +42,7 @@ intervals:
   external-inspection/wait-available: ["1m", "1s"]
   default/wait-inspecting: ["2m", "2s"]
   default/wait-available: ["10m", "1s"]
-  default/wait-deployment: ["5m", "1s"]
+  default/wait-deployment: ["10m", "1s"]
   default/wait-namespace-deleted: ["10m", "1s"]
   ironic/wait-deployment: ["10m", "2s"]
   default/wait-power-state: ["10m", "100ms"]

test/e2e/e2e_suite_test.go

@@ -128,32 +128,35 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	if e2eConfig.GetVariable("DEPLOY_IRONIC") != "false" {
 		// Install Ironic
 		By("Installing Ironic")
-		err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
-			Kustomization:       e2eConfig.GetVariable("IRONIC_KUSTOMIZATION"),
-			ClusterProxy:        clusterProxy,
-			WaitForDeployment:   true,
-			WatchDeploymentLogs: true,
-			DeploymentName:      "ironic",
-			DeploymentNamespace: bmoIronicNamespace,
-			LogPath:             filepath.Join(artifactFolder, "logs", bmoIronicNamespace),
-			WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
+		err := FlakeAttempt(2, func() error {
+			return BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
+				Kustomization:       e2eConfig.GetVariable("IRONIC_KUSTOMIZATION"),
+				ClusterProxy:        clusterProxy,
+				WaitForDeployment:   true,
+				WatchDeploymentLogs: true,
+				DeploymentName:      "ironic",
+				DeploymentNamespace: bmoIronicNamespace,
+				LogPath:             filepath.Join(artifactFolder, "logs", bmoIronicNamespace),
+				WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
+			})
 		})
 		Expect(err).NotTo(HaveOccurred())
-
 	}
 
 	if e2eConfig.GetVariable("DEPLOY_BMO") != "false" {
 		// Install BMO
 		By("Installing BMO")
-		err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
-			Kustomization:       e2eConfig.GetVariable("BMO_KUSTOMIZATION"),
-			ClusterProxy:        clusterProxy,
-			WaitForDeployment:   true,
-			WatchDeploymentLogs: true,
-			DeploymentName:      "baremetal-operator-controller-manager",
-			DeploymentNamespace: bmoIronicNamespace,
-			LogPath:             filepath.Join(artifactFolder, "logs", bmoIronicNamespace),
-			WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
+		err := FlakeAttempt(2, func() error {
+			return BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
+				Kustomization:       e2eConfig.GetVariable("BMO_KUSTOMIZATION"),
+				ClusterProxy:        clusterProxy,
+				WaitForDeployment:   true,
+				WatchDeploymentLogs: true,
+				DeploymentName:      "baremetal-operator-controller-manager",
+				DeploymentNamespace: bmoIronicNamespace,
+				LogPath:             filepath.Join(artifactFolder, "logs", bmoIronicNamespace),
+				WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
+			})
 		})
 		Expect(err).NotTo(HaveOccurred())
 	}

test/e2e/upgrade_test.go

@@ -200,40 +200,44 @@ func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeCl
 	if input.DeployIronic {
 		// Install Ironic
 		By(fmt.Sprintf("Installing Ironic from kustomization %s on the upgrade cluster", initIronicKustomization))
-		err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
-			Kustomization:       initIronicKustomization,
-			ClusterProxy:        upgradeClusterProxy,
-			WaitForDeployment:   true,
-			WatchDeploymentLogs: true,
-			DeploymentName:      "ironic",
-			DeploymentNamespace: bmoIronicNamespace,
-			LogPath:             filepath.Join(testCaseArtifactFolder, "logs", "init-ironic"),
-			WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
+		err := FlakeAttempt(2, func() error {
+			return BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
+				Kustomization:       initIronicKustomization,
+				ClusterProxy:        upgradeClusterProxy,
+				WaitForDeployment:   true,
+				WatchDeploymentLogs: true,
+				DeploymentName:      "ironic",
+				DeploymentNamespace: bmoIronicNamespace,
+				LogPath:             filepath.Join(testCaseArtifactFolder, "logs", "init-ironic"),
+				WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
+			})
 		})
 		Expect(err).NotTo(HaveOccurred())
 
 		DeferCleanup(func() {
 			By(fmt.Sprintf("Removing Ironic kustomization %s from the upgrade cluster", initIronicKustomization))
-			BuildAndRemoveKustomization(ctx, initIronicKustomization, upgradeClusterProxy)
+			cleanupBaremetalOperatorSystem(ctx, upgradeClusterProxy, initIronicKustomization)
 		})
 	}
 	if input.DeployBMO {
 		// Install BMO
 		By(fmt.Sprintf("Installing BMO from %s on the upgrade cluster", initBMOKustomization))
-		err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
-			Kustomization:       initBMOKustomization,
-			ClusterProxy:        upgradeClusterProxy,
-			WaitForDeployment:   true,
-			WatchDeploymentLogs: true,
-			DeploymentName:      "baremetal-operator-controller-manager",
-			DeploymentNamespace: bmoIronicNamespace,
-			LogPath:             filepath.Join(testCaseArtifactFolder, "logs", "init-bmo"),
-			WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
+		err := FlakeAttempt(2, func() error {
+			return BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
+				Kustomization:       initBMOKustomization,
+				ClusterProxy:        upgradeClusterProxy,
+				WaitForDeployment:   true,
+				WatchDeploymentLogs: true,
+				DeploymentName:      "baremetal-operator-controller-manager",
+				DeploymentNamespace: bmoIronicNamespace,
+				LogPath:             filepath.Join(testCaseArtifactFolder, "logs", "init-bmo"),
+				WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
+			})
 		})
 		Expect(err).NotTo(HaveOccurred())
 		DeferCleanup(func() {
 			By(fmt.Sprintf("Removing BMO kustomization %s from the upgrade cluster", initBMOKustomization))
-			BuildAndRemoveKustomization(ctx, initBMOKustomization, upgradeClusterProxy)
+			cleanupBaremetalOperatorSystem(ctx, upgradeClusterProxy, initBMOKustomization)
 		})
 	}
 
@@ -289,20 +293,22 @@ func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeCl
 	deploy, err := clientSet.AppsV1().Deployments(bmoIronicNamespace).Get(ctx, upgradeDeploymentName, metav1.GetOptions{})
 	Expect(err).NotTo(HaveOccurred())
 	upgradeKustomization := input.UpgradeEntityKustomization
-	err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
-		Kustomization:       upgradeKustomization,
-		ClusterProxy:        upgradeClusterProxy,
-		WaitForDeployment:   false,
-		WatchDeploymentLogs: true,
-		DeploymentName:      upgradeDeploymentName,
-		DeploymentNamespace: bmoIronicNamespace,
-		LogPath:             filepath.Join(testCaseArtifactFolder, "logs", "bmo-upgrade-main"),
-		WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
+	err = FlakeAttempt(2, func() error {
+		return BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
+			Kustomization:       upgradeKustomization,
+			ClusterProxy:        upgradeClusterProxy,
+			WaitForDeployment:   false,
+			WatchDeploymentLogs: true,
+			DeploymentName:      upgradeDeploymentName,
+			DeploymentNamespace: bmoIronicNamespace,
+			LogPath:             filepath.Join(testCaseArtifactFolder, "logs", "bmo-upgrade-main"),
+			WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
+		})
 	})
 	Expect(err).NotTo(HaveOccurred())
 	DeferCleanup(func() {
 		By(fmt.Sprintf("Removing %s kustomization %s from the upgrade cluster", input.UpgradeEntityName, upgradeKustomization))
-		BuildAndRemoveKustomization(ctx, upgradeKustomization, upgradeClusterProxy)
+		cleanupBaremetalOperatorSystem(ctx, upgradeClusterProxy, upgradeKustomization)
 	})
 
 	By(fmt.Sprintf("Waiting for %s update to rollout", input.UpgradeEntityName))
@@ -415,3 +421,14 @@ var _ = Describe("Upgrade", Label("optional", "upgrade"), func() {
 		cleanup(ctx, upgradeClusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...)
 	})
 })
+
+// cleanupBaremetalOperatorSystem removes the kustomization from the cluster and waits for the
+// baremetal-operator-system namespace to be deleted.
+func cleanupBaremetalOperatorSystem(ctx context.Context, clusterProxy framework.ClusterProxy, kustomization string) {
+	BuildAndRemoveKustomization(ctx, kustomization, clusterProxy)
+	// We need to ensure that the namespace actually gets deleted.
+	WaitForNamespaceDeleted(ctx, WaitForNamespaceDeletedInput{
+		Getter:    clusterProxy.GetClient(),
+		Namespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "baremetal-operator-system"}},
+	}, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...)
+}