From 931712515dab996164a61ccf6a9f5875766ed1db Mon Sep 17 00:00:00 2001 From: Maya Peretz Date: Wed, 12 Jan 2022 18:19:21 +0530 Subject: [PATCH] hanged backup suite to be more generic & changed mssql app to mysql --- Makefile | 2 +- go.mod | 5 +- go.sum | 11 +- tests/e2e/backup_restore_suite_test.go | 177 ++++++++------- tests/e2e/e2e_suite_test.go | 4 +- tests/e2e/lib/backup.go | 207 +++++++++++++++--- tests/e2e/lib/csi_helpers.go | 32 +++ tests/e2e/lib/dpa_helpers.go | 4 +- .../gp2-csi/volumeSnapshotClass.yaml | 14 -- .../mysql-persistent-csi-template.yaml | 137 ++++++++++++ .../mysql-persistent-template.yaml | 137 ++++++++++++ tests/e2e/scripts/aws_settings.sh | 3 + tests/e2e/templates/default_settings.json | 6 +- tests/e2e/utils/common_utils.go | 8 + 14 files changed, 607 insertions(+), 140 deletions(-) create mode 100644 tests/e2e/lib/csi_helpers.go mode change 100755 => 100644 tests/e2e/lib/dpa_helpers.go delete mode 100644 tests/e2e/sample-applications/gp2-csi/volumeSnapshotClass.yaml create mode 100644 tests/e2e/sample-applications/mysql-persistent/mysql-persistent-csi-template.yaml create mode 100644 tests/e2e/sample-applications/mysql-persistent/mysql-persistent-template.yaml diff --git a/Makefile b/Makefile index c26eab7360f..4a2cb5a92c6 100644 --- a/Makefile +++ b/Makefile @@ -295,7 +295,7 @@ TEST_FILTER := $(shell echo '! aws && ! gcp && ! azure' | sed -r "s/[&]* [!] $(C SETTINGS_TMP=/tmp/test-settings test-e2e: mkdir -p $(SETTINGS_TMP) - PROVIDER="$(PROVIDER)" BUCKET="$(S3_BUCKET)" REGION="$(REGION)" SECRET="$(CREDS_SECRET_REF)" TMP_DIR=$(SETTINGS_TMP) /bin/bash tests/e2e/scripts/aws_settings.sh + NAMESPACE="$(OADP_TEST_NAMESPACE)" PROVIDER="$(PROVIDER)" BUCKET="$(S3_BUCKET)" REGION="$(REGION)" SECRET="$(CREDS_SECRET_REF)" TMP_DIR=$(SETTINGS_TMP) /bin/bash tests/e2e/scripts/aws_settings.sh ginkgo run -mod=mod tests/e2e/ -- -cloud=$(OADP_AWS_CRED_FILE) \ -velero_namespace=$(OADP_TEST_NAMESPACE) \ -settings=$(SETTINGS_TMP)/awscreds \ diff --git a/go.mod b/go.mod index aabb0a25cae..8a437204448 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,10 @@ require ( github.com/aws/aws-sdk-go v1.28.2 github.com/go-logr/logr v0.4.0 github.com/google/uuid v1.1.2 - github.com/onsi/ginkgo v1.16.4 + github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 + github.com/onsi/ginkgo v1.16.5 github.com/onsi/ginkgo/v2 v2.1.1 - github.com/onsi/gomega v1.17.0 + github.com/onsi/gomega v1.18.1 github.com/openshift/api v0.0.0-20210805075156-d8fab4513288 github.com/operator-framework/api v0.10.7 github.com/operator-framework/operator-lib v0.9.0 diff --git a/go.sum b/go.sum index fd61c192436..cb7689bed1b 100644 --- a/go.sum +++ b/go.sum @@ -485,6 +485,7 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 h1:ipLtV9ubLEYx42YvwDa12eVPQvjuGZoPdbCozGzVNRc= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= @@ -570,8 +571,10 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.1 h1:LCnPB85AvFNr91s0B2aDzEiiIg6MUwLYbryC1NSlWi8= github.com/onsi/ginkgo/v2 v2.1.1/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -583,8 +586,9 @@ github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift/api v0.0.0-20210805075156-d8fab4513288 h1:Yw96Z8gygCXxjeMTm55gGsTNxwnJkr6L2Baf3NsUQFU= @@ -991,8 +995,9 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 93ea58d19e8..ec017576110 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -2,21 +2,21 @@ package e2e_test import ( "errors" - "fmt" "log" "time" - "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/openshift/oadp-operator/tests/e2e/lib" utils "github.com/openshift/oadp-operator/tests/e2e/utils" + velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) type VerificationFunction func(client.Client, string) error var _ = Describe("AWS backup restore tests", func() { + var currentBackup BackupInterface var _ = BeforeEach(func() { testSuiteInstanceName := "ts-" + instanceName @@ -31,42 +31,45 @@ var _ = Describe("AWS backup restore tests", func() { var _ = AfterEach(func() { err := dpaCR.Delete() Expect(err).ToNot(HaveOccurred()) + if currentBackup != nil { + err = currentBackup.CleanBackup() + Expect(err).ToNot(HaveOccurred()) + } }) type BackupRestoreCase struct { - ApplicationTemplate string - ApplicationNamespace string - Name string - BackupRestoreType BackupRestoreType - PreBackupVerify VerificationFunction - PostRestoreVerify VerificationFunction - MaxK8SVersion *K8sVersion - MinK8SVersion *K8sVersion + ApplicationTemplate string + BackupSpec velero.BackupSpec + Name string + PreBackupVerify VerificationFunction + PostRestoreVerify VerificationFunction + MaxK8SVersion *K8sVersion + MinK8SVersion *K8sVersion } parksAppReady := VerificationFunction(func(ocClient client.Client, namespace string) error { Eventually(IsDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) return nil }) - mssqlReady := VerificationFunction(func(ocClient client.Client, namespace string) error { + mysqlReady := VerificationFunction(func(ocClient client.Client, namespace string) error { // This test confirms that SCC restore logic in our plugin is working - Eventually(IsDCReady(ocClient, "mssql-persistent", "mssql-deployment"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) - Eventually(IsDeploymentReady(ocClient, "mssql-persistent", "mssql-app-deployment"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) - exists, err := DoesSCCExist(ocClient, "mssql-persistent-scc") + //Eventually(IsDCReady(ocClient, "mssql-persistent", "mysql"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + Eventually(IsDeploymentReady(ocClient, "mysql-persistent", "mysql"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + exists, err := DoesSCCExist(ocClient, "mysql-persistent-scc") if err != nil { return err } if !exists { - return errors.New("did not find MSSQL scc") + return errors.New("did not find MYSQL scc") } return nil }) DescribeTable("backup and restore applications", - func(brCase BackupRestoreCase, expectedErr error) { + func(brCase BackupRestoreCase, backup BackupInterface, expectedErr error) { - err := dpaCR.Build(brCase.BackupRestoreType) + err := dpaCR.Build(backup.GetType()) Expect(err).NotTo(HaveOccurred()) err = dpaCR.CreateOrUpdate(&dpaCR.CustomResource.Spec) @@ -75,17 +78,6 @@ var _ = Describe("AWS backup restore tests", func() { log.Printf("Waiting for velero pod to be running") Eventually(AreVeleroPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - if brCase.BackupRestoreType == RESTIC { - log.Printf("Waiting for restic pods to be running") - Eventually(AreResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) - } - - if brCase.BackupRestoreType == CSI { - log.Printf("Creating VolumeSnapshot for CSI backuprestore of %s", brCase.Name) - err = InstallApplication(dpaCR.Client, "./sample-applications/gp2-csi/volumeSnapshotClass.yaml") - Expect(err).ToNot(HaveOccurred()) - } - if dpaCR.CustomResource.Spec.BackupImages == nil || *dpaCR.CustomResource.Spec.BackupImages { log.Printf("Waiting for registry pods to be running") Eventually(AreRegistryDeploymentsAvailable(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) @@ -93,114 +85,119 @@ var _ = Describe("AWS backup restore tests", func() { if notVersionTarget, reason := NotServerVersionTarget(brCase.MinK8SVersion, brCase.MaxK8SVersion); notVersionTarget { Skip(reason) } - backupUid, _ := uuid.NewUUID() - restoreUid, _ := uuid.NewUUID() - backupName := fmt.Sprintf("%s-%s", brCase.Name, backupUid.String()) - restoreName := fmt.Sprintf("%s-%s", brCase.Name, restoreUid.String()) + + brCaseName := brCase.Name + backup.NewBackup(dpaCR.Client, brCaseName, &brCase.BackupSpec) + backupRestoreName := backup.GetBackupSpec().Name + currentBackup = backup + err = backup.PrepareBackup() + Expect(err).ToNot(HaveOccurred()) // install app - log.Printf("Installing application for case %s", brCase.Name) + log.Printf("Installing application for case %s", brCaseName) err = InstallApplication(dpaCR.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) // wait for pods to be running - Eventually(AreApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue()) + Eventually(AreApplicationPodsRunning(brCaseName), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue()) // Run optional custom verification - log.Printf("Running pre-backup function for case %s", brCase.Name) - err = brCase.PreBackupVerify(dpaCR.Client, brCase.ApplicationNamespace) + log.Printf("Running pre-backup function for case %s", brCaseName) + err = brCase.PreBackupVerify(dpaCR.Client, brCaseName) Expect(err).ToNot(HaveOccurred()) // create backup - log.Printf("Creating backup %s for case %s", backupName, brCase.Name) - err = CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace}) + log.Printf("Creating backup %s for case %s", backupRestoreName, brCaseName) + err = backup.CreateBackup() Expect(err).ToNot(HaveOccurred()) // wait for backup to not be running - Eventually(IsBackupDone(dpaCR.Client, namespace, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + Eventually(backup.IsBackupDone(), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // check if backup succeeded - succeeded, err := IsBackupCompletedSuccessfully(dpaCR.Client, namespace, backupName) + succeeded, err := backup.IsBackupCompletedSuccessfully() Expect(err).ToNot(HaveOccurred()) Expect(succeeded).To(Equal(true)) - log.Printf("Backup for case %s succeeded", brCase.Name) + log.Printf("Backup for case %s succeeded", brCaseName) // uninstall app - log.Printf("Uninstalling app for case %s", brCase.Name) + log.Printf("Uninstalling app for case %s", brCaseName) err = UninstallApplication(dpaCR.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) // Wait for namespace to be deleted - Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) + Eventually(IsNamespaceDeleted(brCaseName), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) // run restore - log.Printf("Creating restore %s for case %s", restoreName, brCase.Name) - err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName) + log.Printf("Creating restore %s for case %s", backupRestoreName, brCaseName) + err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupRestoreName, backupRestoreName) Expect(err).ToNot(HaveOccurred()) - Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + Eventually(IsRestoreDone(dpaCR.Client, namespace, backupRestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // Check if restore succeeded - succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName) + succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, backupRestoreName) Expect(err).ToNot(HaveOccurred()) Expect(succeeded).To(Equal(true)) // verify app is running - Eventually(AreApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue()) + Eventually(AreApplicationPodsRunning(brCaseName), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue()) // Run optional custom verification - log.Printf("Running post-restore function for case %s", brCase.Name) - err = brCase.PostRestoreVerify(dpaCR.Client, brCase.ApplicationNamespace) + log.Printf("Running post-restore function for case %s", brCaseName) + err = brCase.PostRestoreVerify(dpaCR.Client, brCaseName) Expect(err).ToNot(HaveOccurred()) // Test is successful, clean up everything - log.Printf("Uninstalling application for case %s", brCase.Name) + log.Printf("Uninstalling application for case %s", brCaseName) err = UninstallApplication(dpaCR.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) // Wait for namespace to be deleted - Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) - - if brCase.BackupRestoreType == CSI { - log.Printf("Deleting VolumeSnapshot for CSI backuprestore of %s", brCase.Name) - err = UninstallApplication(dpaCR.Client, "./sample-applications/gp2-csi/volumeSnapshotClass.yaml") - Expect(err).ToNot(HaveOccurred()) - } + Eventually(IsNamespaceDeleted(brCaseName), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) }, - Entry("MSSQL application CSI", Label("aws"), BackupRestoreCase{ - ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml", - ApplicationNamespace: "mssql-persistent", - Name: "mssql-e2e", - BackupRestoreType: CSI, - PreBackupVerify: mssqlReady, - PostRestoreVerify: mssqlReady, - }, nil), + FEntry("MySQL application CSI", Label("aws"), BackupRestoreCase{ + + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-csi-template.yaml", + Name: "mysql-persistent", + BackupSpec: velero.BackupSpec{ + IncludedNamespaces: []string{"mysql-persistent"}, + }, + PreBackupVerify: mysqlReady, + PostRestoreVerify: mysqlReady, + }, &BackupCsi{}, nil), Entry("Parks application <4.8.0", BackupRestoreCase{ - ApplicationTemplate: "./sample-applications/parks-app/manifest.yaml", - ApplicationNamespace: "parks-app", - Name: "parks-e2e", - BackupRestoreType: RESTIC, - PreBackupVerify: parksAppReady, - PostRestoreVerify: parksAppReady, - MaxK8SVersion: &K8sVersionOcp47, - }, nil), - Entry("MSSQL application", BackupRestoreCase{ - ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-template.yaml", - ApplicationNamespace: "mssql-persistent", - Name: "mssql-e2e", - BackupRestoreType: RESTIC, - PreBackupVerify: mssqlReady, - PostRestoreVerify: mssqlReady, - }, nil), - Entry("Parks application >=4.8.0", BackupRestoreCase{ - ApplicationTemplate: "./sample-applications/parks-app/manifest4.8.yaml", - ApplicationNamespace: "parks-app", - Name: "parks-e2e", - BackupRestoreType: RESTIC, - PreBackupVerify: parksAppReady, - PostRestoreVerify: parksAppReady, - MinK8SVersion: &K8sVersionOcp48, + ApplicationTemplate: "./sample-applications/parks-app/manifest.yaml", + Name: "parks-app", + BackupSpec: velero.BackupSpec{ + IncludedNamespaces: []string{"parks-app"}, + }, + PreBackupVerify: parksAppReady, + PostRestoreVerify: parksAppReady, + MaxK8SVersion: &K8sVersionOcp47, + }, &BackupVsl{CreateFromDpa: false}, + nil), + Entry("MySQL application", BackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-template.yaml", + Name: "mysql-persistent", + PreBackupVerify: mysqlReady, + PostRestoreVerify: mysqlReady, + BackupSpec: velero.BackupSpec{ + IncludedNamespaces: []string{"mysql-persistent"}, + }, + }, &BackupRestic{}, nil), + FEntry("Parks application >=4.8.0", BackupRestoreCase{ + ApplicationTemplate: "./sample-applications/parks-app/manifest4.8.yaml", + Name: "parks-app", + BackupSpec: velero.BackupSpec{ + IncludedNamespaces: []string{"parks-app"}, + }, + PreBackupVerify: parksAppReady, + PostRestoreVerify: parksAppReady, + MinK8SVersion: &K8sVersionOcp48, + }, &BackupVsl{ + CreateFromDpa: true, }, nil), ) }) diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index fe2eab4eb66..4343afd4846 100755 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -18,9 +18,9 @@ var namespace, instanceName, settings, cloud, clusterProfile, credSecretRef stri var timeoutMultiplier time.Duration func init() { - flag.StringVar(&cloud, "cloud", "", "Cloud Credentials file path location") + flag.StringVar(&cloud, "cloud", "/home/mperetz/git/oadp-operator/awscreds", "Cloud Credentials file path location") flag.StringVar(&namespace, "velero_namespace", "velero", "Velero Namespace") - flag.StringVar(&settings, "settings", "./templates/default_settings.json", "Settings of the velero instance") + flag.StringVar(&settings, "settings", "/tmp/test-settings/awscreds", "Settings of the velero instance") flag.StringVar(&instanceName, "velero_instance_name", "example-velero", "Velero Instance Name") flag.StringVar(&clusterProfile, "cluster_profile", "aws", "Cluster profile") flag.StringVar(&credSecretRef, "creds_secret_ref", "cloud-credentials", "Credential secret ref for backup storage location") diff --git a/tests/e2e/lib/backup.go b/tests/e2e/lib/backup.go index 44b8756a0f4..6642defefc9 100755 --- a/tests/e2e/lib/backup.go +++ b/tests/e2e/lib/backup.go @@ -3,60 +3,217 @@ package lib import ( "context" "fmt" + "log" + "time" + v1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/openshift/oadp-operator/tests/e2e/utils" velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) -func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupName string, namespaces []string) error { +type BackupInterface interface { + NewBackup(client.Client, string, *velero.BackupSpec) + PrepareBackup() error + CreateBackup() error + CleanBackup() error + GetType() BackupRestoreType + GetBackupSpec() *velero.Backup + IsBackupCompletedSuccessfully() (bool, error) + IsBackupDone() wait.ConditionFunc +} - backup := velero.Backup{ +type backup struct { + BackupInterface + *velero.Backup + client.Client +} + +// empty implementation +func (b *backup) CleanBackup() error { + return nil +} + +func (b *backup) GetBackupSpec() *velero.Backup { + return b.Backup +} + +func (b *backup) NewBackup(ocClient client.Client, backupName string, backupSpec *velero.BackupSpec) { + b.Client = ocClient + b.Backup = &velero.Backup{ ObjectMeta: metav1.ObjectMeta{ - Name: backupName, - Namespace: veleroNamespace, - }, - Spec: velero.BackupSpec{ - IncludedNamespaces: namespaces, + Name: GenNameUuid(backupName), + Namespace: Dpa.Namespace, }, + Spec: *backupSpec, } - err := ocClient.Create(context.Background(), &backup) + +} + +func (b *backup) CreateBackup() error { + err := b.Client.Create(context.Background(), b.Backup) return err + } -func IsBackupDone(ocClient client.Client, veleroNamespace, name string) wait.ConditionFunc { +func (b *backup) IsBackupDone() wait.ConditionFunc { return func() (bool, error) { - backup := velero.Backup{} - err := ocClient.Get(context.Background(), client.ObjectKey{ - Namespace: veleroNamespace, - Name: name, - }, &backup) + backupvar := velero.Backup{} + err := b.Client.Get(context.Background(), client.ObjectKey{ + Namespace: Dpa.Namespace, + Name: b.Backup.Name, + }, &backupvar) if err != nil { return false, err } - if len(backup.Status.Phase) > 0 { - ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("backup phase: %s\n", backup.Status.Phase))) + if len(backupvar.Status.Phase) > 0 { + ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("backup phase: %s\n", backupvar.Status.Phase))) } - if backup.Status.Phase != "" && backup.Status.Phase != velero.BackupPhaseNew && backup.Status.Phase != velero.BackupPhaseInProgress { + if backupvar.Status.Phase != "" && backupvar.Status.Phase != velero.BackupPhaseNew && backupvar.Status.Phase != velero.BackupPhaseInProgress { return true, nil } return false, nil } } -func IsBackupCompletedSuccessfully(ocClient client.Client, veleroNamespace, name string) (bool, error) { - backup := velero.Backup{} - err := ocClient.Get(context.Background(), client.ObjectKey{ - Namespace: veleroNamespace, - Name: name, - }, &backup) +func (b *backup) IsBackupCompletedSuccessfully() (bool, error) { + backupvar := velero.Backup{} + err := b.Client.Get(context.Background(), client.ObjectKey{ + Namespace: Dpa.Namespace, + Name: b.Backup.Name, + }, &backupvar) + if err != nil { + return false, err + } if err != nil { return false, err } - if backup.Status.Phase == velero.BackupPhaseCompleted { + if backupvar.Status.Phase == velero.BackupPhaseCompleted { return true, nil } - return false, fmt.Errorf("backup phase is: %s; expected: %s\nvalidation errors: %v\nvelero failure logs: %v", backup.Status.Phase, velero.BackupPhaseCompleted, backup.Status.ValidationErrors, GetVeleroContainerFailureLogs(veleroNamespace)) + return false, fmt.Errorf("backup phase is: %s; expected: %s\nvalidation errors: %v\nvelero failure logs: %v", backupvar.Status.Phase, velero.BackupPhaseCompleted, backupvar.Status.ValidationErrors, GetVeleroContainerFailureLogs(Dpa.Namespace)) +} + +type BackupCsi struct { + backup + vsc *v1.VolumeSnapshotClass +} + +func (b *BackupCsi) PrepareBackup() error { + snapshotClient, _ := SetUpSnapshotClient() + csiClient, _ := GetCsiDriversList() + vs := v1.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-snapclass", + Annotations: map[string]string{ + "snapshot.storage.kubernetes.io/is-default-class": "true", + }, + Labels: map[string]string{ + "velero.io/csi-volumesnapshot-class": "true", + }, + }, + Driver: csiClient.Items[0].ObjectMeta.Name, + DeletionPolicy: v1.VolumeSnapshotContentRetain, + Parameters: map[string]string{}, + } + _, err := snapshotClient.VolumeSnapshotClasses().Create(context.TODO(), &vs, metav1.CreateOptions{}) + if err == nil { + b.vsc = &vs + } + return err +} + +func (b *BackupCsi) CleanBackup() error { + log.Printf("Deleting VolumeSnapshot for CSI backuprestore of %s", b.Backup.Name) + snapshotClient, _ := SetUpSnapshotClient() + return snapshotClient.VolumeSnapshotClasses().Delete(context.TODO(), b.vsc.Name, metav1.DeleteOptions{}) +} + +func (b *BackupCsi) GetType() BackupRestoreType { + return CSI +} + +type BackupVsl struct { + backup + vsl []*velero.VolumeSnapshotLocation + *DpaCustomResource + CreateFromDpa bool +} + +func (b *BackupVsl) PrepareBackup() error { + if !b.CreateFromDpa { + for _, item := range Dpa.Spec.SnapshotLocations { + vsl := velero.VolumeSnapshotLocation{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "snapshot-location-", + Namespace: Dpa.Namespace, + }, + Spec: velero.VolumeSnapshotLocationSpec{ + Provider: item.Velero.Provider, + Config: item.Velero.Config, + }, + } + err := b.backup.Client.Create(context.Background(), &vsl) + if err != nil { + return err + } + b.vsl = append(b.vsl, &vsl) + b.Backup.Spec.VolumeSnapshotLocations = append(b.Backup.Spec.VolumeSnapshotLocations, vsl.Name) + } + } + return nil +} + +func (b *BackupVsl) CleanBackup() error { + if !b.CreateFromDpa { + for _, item := range b.vsl { + + err := b.backup.Client.Delete(context.Background(), item) + if err != nil { + return err + } + } + } + return nil +} + +func (b *BackupVsl) GetType() BackupRestoreType { + return VSL +} + +type BackupRestic struct { + backup +} + +func (b *BackupRestic) PrepareBackup() error { + Eventually(AreResticPodsRunning(b.Backup.Namespace), 1*time.Minute*3, time.Second*5).Should(BeTrue()) + if b.Backup != nil { + b.Backup.Spec.DefaultVolumesToRestic = pointer.Bool(true) + } + return nil +} + +func (b *BackupRestic) GetType() BackupRestoreType { + return RESTIC +} + +// TODO: Remove +func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupName string, namespaces []string) error { + + backup := velero.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: backupName, + Namespace: veleroNamespace, + }, + Spec: velero.BackupSpec{ + IncludedNamespaces: namespaces, + }, + } + err := ocClient.Create(context.Background(), &backup) + return err } diff --git a/tests/e2e/lib/csi_helpers.go b/tests/e2e/lib/csi_helpers.go new file mode 100644 index 00000000000..0215dc8449e --- /dev/null +++ b/tests/e2e/lib/csi_helpers.go @@ -0,0 +1,32 @@ +package lib + +import ( + "context" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1" + v1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func GetCsiDriversList() (*v1.CSIDriverList, error) { + clientset, err := setUpClient() + if err != nil { + return nil, err + } + + clientcsi, err := clientset.StorageV1().CSIDrivers().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + return clientcsi, nil +} + +func SetUpSnapshotClient() (*snapshotv1.SnapshotV1Client, error) { + kubeConf := getKubeConfig() + + client, err := snapshotv1.NewForConfig(kubeConf) + if err != nil { + return nil, err + } + return client, nil +} diff --git a/tests/e2e/lib/dpa_helpers.go b/tests/e2e/lib/dpa_helpers.go old mode 100755 new mode 100644 index 13325879a83..e654e55bad9 --- a/tests/e2e/lib/dpa_helpers.go +++ b/tests/e2e/lib/dpa_helpers.go @@ -35,6 +35,7 @@ type BackupRestoreType string const ( CSI BackupRestoreType = "csi" RESTIC BackupRestoreType = "restic" + VSL BackupRestoreType = "vsl" ) type DpaCustomResource struct { @@ -65,7 +66,6 @@ func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error { PodConfig: &oadpv1alpha1.PodConfig{}, }, }, - SnapshotLocations: v.CustomResource.Spec.SnapshotLocations, BackupLocations: []oadpv1alpha1.BackupLocation{ { Velero: &velero.BackupStorageLocationSpec{ @@ -92,6 +92,8 @@ func (v *DpaCustomResource) Build(backupRestoreType BackupRestoreType) error { dpaInstance.Spec.Configuration.Restic.Enable = pointer.Bool(false) dpaInstance.Spec.Configuration.Velero.DefaultPlugins = append(dpaInstance.Spec.Configuration.Velero.DefaultPlugins, oadpv1alpha1.DefaultPluginCSI) dpaInstance.Spec.Configuration.Velero.FeatureFlags = append(dpaInstance.Spec.Configuration.Velero.FeatureFlags, "EnableCSI") + case VSL: + dpaInstance.Spec.SnapshotLocations = v.CustomResource.Spec.SnapshotLocations } v.CustomResource = &dpaInstance return nil diff --git a/tests/e2e/sample-applications/gp2-csi/volumeSnapshotClass.yaml b/tests/e2e/sample-applications/gp2-csi/volumeSnapshotClass.yaml deleted file mode 100644 index 09064697b39..00000000000 --- a/tests/e2e/sample-applications/gp2-csi/volumeSnapshotClass.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: List -items: - - apiVersion: snapshot.storage.k8s.io/v1 - kind: VolumeSnapshotClass - metadata: - name: example-snapclass - labels: - velero.io/csi-volumesnapshot-class: 'true' - annotations: - snapshot.storage.kubernetes.io/is-default-class: 'true' - driver: ebs.csi.aws.com - deletionPolicy: Retain - \ No newline at end of file diff --git a/tests/e2e/sample-applications/mysql-persistent/mysql-persistent-csi-template.yaml b/tests/e2e/sample-applications/mysql-persistent/mysql-persistent-csi-template.yaml new file mode 100644 index 00000000000..30d79a651ff --- /dev/null +++ b/tests/e2e/sample-applications/mysql-persistent/mysql-persistent-csi-template.yaml @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: List +items: + - kind: Namespace + apiVersion: v1 + metadata: + name: mysql-persistent + labels: + app: mysql + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: mysql + namespace: mysql-persistent + labels: + app: mysql + spec: + accessModes: + - ReadWriteOnce + storageClassName: gp2-csi + resources: + requests: + storage: 1Gi + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: mysql-persistent-sa + namespace: mysql-persistent + labels: + component: mysql-persistent + - kind: SecurityContextConstraints + apiVersion: security.openshift.io/v1 + metadata: + name: mysql-persistent-scc + allowPrivilegeEscalation: true + allowPrivilegedContainer: true + runAsUser: + type: RunAsAny + seLinuxContext: + type: RunAsAny + fsGroup: + type: RunAsAny + supplementalGroups: + type: RunAsAny + volumes: + - '*' + users: + - system:admin + - system:serviceaccount:mysql-persistent:mysql-persistent-sa + - apiVersion: v1 + kind: Secret + metadata: + name: mysql + namespace: mysql-persistent + labels: + app: mysql + stringData: + database-name: MYSQL_DATABASE + database-password: MYSQL_PASSWORD + database-root-password: MYSQL_ROOT_PASSWORD + database-user: MYSQL_USER + - apiVersion: v1 + kind: Service + metadata: + annotations: + template.openshift.io/expose-uri: mysql://{.spec.clusterIP}:{.spec.ports[?(.name=="mysql")].port} + name: mysql + namespace: mysql-persistent + labels: + app: mysql + spec: + ports: + - name: mysql + port: 3306 + selector: + name: mysql + - apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + template.alpha.openshift.io/wait-for-ready: 'true' + name: mysql + namespace: mysql-persistent + labels: + e2e-app: "true" + spec: + selector: + matchLabels: + app: mysql + strategy: + type: Recreate + template: + metadata: + labels: + e2e-app: "true" + app: mysql + spec: + serviceAccountName: mysql-persistent-sa + containers: + - image: registry.redhat.io/rhel8/mysql-80:latest + name: mysql + securityContext: + privileged: true + env: + - name: MYSQL_USER + valueFrom: + secretKeyRef: + key: database-user + name: mysql + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: mysql + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: database-root-password + name: mysql + - name: MYSQL_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: mysql + ports: + - containerPort: 3306 + name: mysql + resources: + limits: + memory: 512Mi + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: mysql \ No newline at end of file diff --git a/tests/e2e/sample-applications/mysql-persistent/mysql-persistent-template.yaml b/tests/e2e/sample-applications/mysql-persistent/mysql-persistent-template.yaml new file mode 100644 index 00000000000..a11426eaa0b --- /dev/null +++ b/tests/e2e/sample-applications/mysql-persistent/mysql-persistent-template.yaml @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: List +items: + - kind: Namespace + apiVersion: v1 + metadata: + name: mysql-persistent + labels: + app: mysql + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: mysql-persistent-sa + namespace: mysql-persistent + labels: + component: mysql-persistent + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: mysql + namespace: mysql-persistent + labels: + app: mysql + spec: + accessModes: + - ReadWriteOnce + storageClassName: gp2 + resources: + requests: + storage: 1Gi + - kind: SecurityContextConstraints + apiVersion: security.openshift.io/v1 + metadata: + name: mysql-persistent-scc + allowPrivilegeEscalation: true + allowPrivilegedContainer: true + runAsUser: + type: RunAsAny + seLinuxContext: + type: RunAsAny + fsGroup: + type: RunAsAny + supplementalGroups: + type: RunAsAny + volumes: + - '*' + users: + - system:admin + - system:serviceaccount:mysql-persistent:mysql-persistent-sa + - apiVersion: v1 + kind: Secret + metadata: + name: mysql + namespace: mysql-persistent + labels: + app: mysql + stringData: + database-name: MYSQL_DATABASE + database-password: MYSQL_PASSWORD + database-root-password: MYSQL_ROOT_PASSWORD + database-user: MYSQL_USER + - apiVersion: v1 + kind: Service + metadata: + annotations: + template.openshift.io/expose-uri: mysql://{.spec.clusterIP}:{.spec.ports[?(.name=="mysql")].port} + name: mysql + namespace: mysql-persistent + labels: + app: mysql + spec: + ports: + - name: mysql + port: 3306 + selector: + name: mysql + - apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + template.alpha.openshift.io/wait-for-ready: 'true' + name: mysql + namespace: mysql-persistent + labels: + e2e-app: "true" + spec: + selector: + matchLabels: + app: mysql + strategy: + type: Recreate + template: + metadata: + labels: + e2e-app: "true" + app: mysql + spec: + serviceAccountName: mysql-persistent-sa + containers: + - image: registry.redhat.io/rhel8/mysql-80:latest + name: mysql + securityContext: + privileged: true + env: + - name: MYSQL_USER + valueFrom: + secretKeyRef: + key: database-user + name: mysql + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: mysql + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: database-root-password + name: mysql + - name: MYSQL_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: mysql + ports: + - containerPort: 3306 + name: mysql + resources: + limits: + memory: 512Mi + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: mysql \ No newline at end of file diff --git a/tests/e2e/scripts/aws_settings.sh b/tests/e2e/scripts/aws_settings.sh index 5182e1b9c54..9fb21624441 100644 --- a/tests/e2e/scripts/aws_settings.sh +++ b/tests/e2e/scripts/aws_settings.sh @@ -2,6 +2,9 @@ cat > $TMP_DIR/awscreds <