Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Changed backup suite to be more generic & changed mssql app to mysql #565

Closed
wants to merge 1 commit into from
Closed

Changed backup suite to be more generic & changed mssql app to mysql #565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ SETTINGS_TMP=/tmp/test-settings

test-e2e-setup:
mkdir -p $(SETTINGS_TMP)
PROVIDER="$(CLUSTER_TYPE)" BUCKET="$(OADP_BUCKET)" BSL_REGION="$(BSL_REGION)" SECRET="$(CREDS_SECRET_REF)" TMP_DIR=$(SETTINGS_TMP) \
NAMESPACE="$(OADP_TEST_NAMESPACE)" PROVIDER="$(CLUSTER_TYPE)" BUCKET="$(OADP_BUCKET)" BSL_REGION="$(BSL_REGION)" SECRET="$(CREDS_SECRET_REF)" TMP_DIR=$(SETTINGS_TMP) \
VSL_REGION="$(VSL_REGION)" BSL_AWS_PROFILE="$(BSL_AWS_PROFILE)" BSL_REGION="$(BSL_REGION)" /bin/bash "tests/e2e/scripts/$(CLUSTER_TYPE)_settings.sh"

test-e2e: test-e2e-setup
Expand All @@ -347,6 +347,7 @@ test-e2e: test-e2e-setup
-azure_resource_file=$(AZURE_RESOURCE_FILE) \
-provider=$(CLUSTER_TYPE) \
-creds_secret_ref=$(CREDS_SECRET_REF)
-csi_driver=$(CSI_DRIVER)

test-e2e-cleanup:
rm -rf $(SETTINGS_TMP)
34 changes: 22 additions & 12 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,35 @@ module github.com/openshift/oadp-operator
go 1.16

require (
github.com/Azure-Samples/azure-sdk-for-go-samples v0.0.0-20210506191746-b49c4162aa1d
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0
github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc
github.com/aws/aws-sdk-go v1.28.2
github.com/go-logr/logr v0.4.0
github.com/go-logr/logr v1.2.0
github.com/google/uuid v1.1.2
github.com/onsi/ginkgo v1.16.4
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0
github.com/onsi/ginkgo v1.16.5
github.com/onsi/ginkgo/v2 v2.1.1
github.com/onsi/gomega v1.17.0
github.com/openshift/api v0.0.0-20210805075156-d8fab4513288
github.com/onsi/gomega v1.18.1
github.com/openshift/api v0.0.0-20211209135129-c58d9f695577
github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3
github.com/operator-framework/api v0.10.7
github.com/operator-framework/operator-lib v0.9.0
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.51.2
github.com/vmware-tanzu/velero v1.7.0 // TODO: Update this to a pinned version
k8s.io/api v0.22.2
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
k8s.io/api v0.23.0
k8s.io/apiextensions-apiserver v0.22.2
k8s.io/apimachinery v0.22.2
k8s.io/client-go v0.22.2
k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a
k8s.io/apimachinery v0.23.0
k8s.io/client-go v0.23.0
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
sigs.k8s.io/controller-runtime v0.10.3
)

replace (
github.com/go-logr/logr v1.2.0 => github.com/go-logr/logr v0.4.0
github.com/openshift/api v0.0.0-20211209135129-c58d9f695577 => github.com/openshift/api v0.0.0-20210805075156-d8fab4513288
k8s.io/api v0.23.0 => k8s.io/api v0.22.2
k8s.io/apimachinery v0.23.0 => k8s.io/apimachinery v0.22.2
k8s.io/client-go v0.23.0 => k8s.io/client-go v0.22.2
k8s.io/klog/v2 v2.30.0 => k8s.io/klog/v2 v2.9.0
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b => k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a

)
140 changes: 31 additions & 109 deletions go.sum
View file
Open in desktop

Large diffs are not rendered by default.

177 changes: 89 additions & 88 deletions tests/e2e/backup_restore_suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,27 @@ package e2e_test

import (
"errors"
"fmt"
"log"
"time"

"github.com/google/uuid"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "github.com/openshift/oadp-operator/tests/e2e/lib"
utils "github.com/openshift/oadp-operator/tests/e2e/utils"
velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type VerificationFunction func(client.Client, string) error

var _ = Describe("AWS backup restore tests", func() {
var currentBackup BackupInterface

var _ = BeforeEach(func() {
testSuiteInstanceName := "ts-" + instanceName

dpaCR.Name = testSuiteInstanceName
var err error

credData, err := utils.ReadFile(credFile)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -31,42 +33,45 @@ var _ = Describe("AWS backup restore tests", func() {
var _ = AfterEach(func() {
err := dpaCR.Delete()
Expect(err).ToNot(HaveOccurred())
if currentBackup != nil {
err = currentBackup.CleanBackup()
Expect(err).NotTo(HaveOccurred())
}

})

type BackupRestoreCase struct {
ApplicationTemplate string
ApplicationNamespace string
Name string
BackupRestoreType BackupRestoreType
PreBackupVerify VerificationFunction
PostRestoreVerify VerificationFunction
MaxK8SVersion *K8sVersion
MinK8SVersion *K8sVersion
ApplicationTemplate string
BackupSpec velero.BackupSpec
Name string
PreBackupVerify VerificationFunction
PostRestoreVerify VerificationFunction
MaxK8SVersion *K8sVersion
MinK8SVersion *K8sVersion
}

parksAppReady := VerificationFunction(func(ocClient client.Client, namespace string) error {
Eventually(IsDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
return nil
})
mssqlReady := VerificationFunction(func(ocClient client.Client, namespace string) error {
mysqlReady := VerificationFunction(func(ocClient client.Client, namespace string) error {
// This test confirms that SCC restore logic in our plugin is working
Eventually(IsDCReady(ocClient, "mssql-persistent", "mssql-deployment"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
Eventually(IsDeploymentReady(ocClient, "mssql-persistent", "mssql-app-deployment"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
exists, err := DoesSCCExist(ocClient, "mssql-persistent-scc")
//Eventually(IsDCReady(ocClient, "mssql-persistent", "mysql"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
Eventually(IsDeploymentReady(ocClient, "mysql-persistent", "mysql"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue())
exists, err := DoesSCCExist(ocClient, "mysql-persistent-scc")
if err != nil {
return err
}
if !exists {
return errors.New("did not find MSSQL scc")
return errors.New("did not find MYSQL scc")
}
return nil
})

DescribeTable("backup and restore applications",
func(brCase BackupRestoreCase, expectedErr error) {
func(brCase BackupRestoreCase, backup BackupInterface, expectedErr error) {

err := dpaCR.Build(brCase.BackupRestoreType)
err := dpaCR.Build(backup.GetType())
Expect(err).NotTo(HaveOccurred())

err = dpaCR.CreateOrUpdate(&dpaCR.CustomResource.Spec)
Expand All @@ -75,132 +80,128 @@ var _ = Describe("AWS backup restore tests", func() {
log.Printf("Waiting for velero pod to be running")
Eventually(AreVeleroPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue())

if brCase.BackupRestoreType == RESTIC {
log.Printf("Waiting for restic pods to be running")
Eventually(AreResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue())
}

if brCase.BackupRestoreType == CSI {
log.Printf("Creating VolumeSnapshot for CSI backuprestore of %s", brCase.Name)
err = InstallApplication(dpaCR.Client, "./sample-applications/gp2-csi/volumeSnapshotClass.yaml")
Expect(err).ToNot(HaveOccurred())
}

if dpaCR.CustomResource.Spec.BackupImages == nil || *dpaCR.CustomResource.Spec.BackupImages {
log.Printf("Waiting for registry pods to be running")
Eventually(AreRegistryDeploymentsAvailable(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue())
}
if notVersionTarget, reason := NotServerVersionTarget(brCase.MinK8SVersion, brCase.MaxK8SVersion); notVersionTarget {
Skip(reason)
}
backupUid, _ := uuid.NewUUID()
restoreUid, _ := uuid.NewUUID()
backupName := fmt.Sprintf("%s-%s", brCase.Name, backupUid.String())
restoreName := fmt.Sprintf("%s-%s", brCase.Name, restoreUid.String())

brCaseName := brCase.Name
backup.NewBackup(dpaCR.Client, brCaseName, &brCase.BackupSpec)
backupRestoreName := backup.GetBackupSpec().Name
err = backup.PrepareBackup()
Expect(err).ToNot(HaveOccurred())
currentBackup = backup

// install app
log.Printf("Installing application for case %s", brCase.Name)
log.Printf("Installing application for case %s", brCaseName)
err = InstallApplication(dpaCR.Client, brCase.ApplicationTemplate)
Expect(err).ToNot(HaveOccurred())
// wait for pods to be running
Eventually(AreApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue())
Eventually(AreApplicationPodsRunning(brCaseName), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue())

// Run optional custom verification
log.Printf("Running pre-backup function for case %s", brCase.Name)
err = brCase.PreBackupVerify(dpaCR.Client, brCase.ApplicationNamespace)
log.Printf("Running pre-backup function for case %s", brCaseName)
err = brCase.PreBackupVerify(dpaCR.Client, brCaseName)
Expect(err).ToNot(HaveOccurred())

// create backup
log.Printf("Creating backup %s for case %s", backupName, brCase.Name)
err = CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace})
log.Printf("Creating backup %s for case %s", backupRestoreName, brCaseName)
err = backup.CreateBackup()
Expect(err).ToNot(HaveOccurred())

// wait for backup to not be running
Eventually(IsBackupDone(dpaCR.Client, namespace, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue())
Eventually(backup.IsBackupDone(), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue())
Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{}))

// check if backup succeeded
succeeded, err := IsBackupCompletedSuccessfully(dpaCR.Client, namespace, backupName)
succeeded, err := backup.IsBackupCompletedSuccessfully()
Expect(err).ToNot(HaveOccurred())
Expect(succeeded).To(Equal(true))
log.Printf("Backup for case %s succeeded", brCase.Name)
log.Printf("Backup for case %s succeeded", brCaseName)

// uninstall app
log.Printf("Uninstalling app for case %s", brCase.Name)
log.Printf("Uninstalling app for case %s", brCaseName)
err = UninstallApplication(dpaCR.Client, brCase.ApplicationTemplate)
Expect(err).ToNot(HaveOccurred())

// Wait for namespace to be deleted
Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue())
Eventually(IsNamespaceDeleted(brCaseName), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue())

// run restore
log.Printf("Creating restore %s for case %s", restoreName, brCase.Name)
err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName)
log.Printf("Creating restore %s for case %s", backupRestoreName, brCaseName)
err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupRestoreName, backupRestoreName)
Expect(err).ToNot(HaveOccurred())
Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue())
Eventually(IsRestoreDone(dpaCR.Client, namespace, backupRestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue())
Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{}))

// Check if restore succeeded
succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName)
succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, backupRestoreName)
Expect(err).ToNot(HaveOccurred())
Expect(succeeded).To(Equal(true))

// verify app is running
Eventually(AreApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue())
Eventually(AreApplicationPodsRunning(brCaseName), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue())

// Run optional custom verification
log.Printf("Running post-restore function for case %s", brCase.Name)
err = brCase.PostRestoreVerify(dpaCR.Client, brCase.ApplicationNamespace)
log.Printf("Running post-restore function for case %s", brCaseName)
err = brCase.PostRestoreVerify(dpaCR.Client, brCaseName)
Expect(err).ToNot(HaveOccurred())

// Test is successful, clean up everything
log.Printf("Uninstalling application for case %s", brCase.Name)
log.Printf("Uninstalling application for case %s", brCaseName)
err = UninstallApplication(dpaCR.Client, brCase.ApplicationTemplate)
Expect(err).ToNot(HaveOccurred())

// Wait for namespace to be deleted
Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue())

if brCase.BackupRestoreType == CSI {
log.Printf("Deleting VolumeSnapshot for CSI backuprestore of %s", brCase.Name)
err = UninstallApplication(dpaCR.Client, "./sample-applications/gp2-csi/volumeSnapshotClass.yaml")
Expect(err).ToNot(HaveOccurred())
}
Eventually(IsNamespaceDeleted(brCaseName), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue())

},
Entry("MSSQL application CSI", Label("aws"), BackupRestoreCase{
ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-csi-template.yaml",
ApplicationNamespace: "mssql-persistent",
Name: "mssql-e2e",
BackupRestoreType: CSI,
PreBackupVerify: mssqlReady,
PostRestoreVerify: mssqlReady,
Entry("MySQL application CSI", Label("aws"), BackupRestoreCase{

ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-template.yaml",
Name: "mysql-persistent",
BackupSpec: velero.BackupSpec{
IncludedNamespaces: []string{"mysql-persistent"},
},
PreBackupVerify: mysqlReady,
PostRestoreVerify: mysqlReady,
}, &BackupCsi{
DriverName: csi_driver,
}, nil),
Entry("MySQL application", BackupRestoreCase{
ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-template.yaml",
Name: "mysql-persistent",
PreBackupVerify: mysqlReady,
PostRestoreVerify: mysqlReady,
BackupSpec: velero.BackupSpec{
IncludedNamespaces: []string{"mysql-persistent"},
},
}, &BackupRestic{}, nil),
Entry("Parks application <4.8.0", BackupRestoreCase{
ApplicationTemplate: "./sample-applications/parks-app/manifest.yaml",
ApplicationNamespace: "parks-app",
Name: "parks-e2e",
BackupRestoreType: RESTIC,
PreBackupVerify: parksAppReady,
PostRestoreVerify: parksAppReady,
MaxK8SVersion: &K8sVersionOcp47,
}, nil),
Entry("MSSQL application", BackupRestoreCase{
ApplicationTemplate: "./sample-applications/mssql-persistent/mssql-persistent-template.yaml",
ApplicationNamespace: "mssql-persistent",
Name: "mssql-e2e",
BackupRestoreType: RESTIC,
PreBackupVerify: mssqlReady,
PostRestoreVerify: mssqlReady,
}, nil),
ApplicationTemplate: "./sample-applications/parks-app/manifest.yaml",
Name: "parks-app",
BackupSpec: velero.BackupSpec{
IncludedNamespaces: []string{"parks-app"},
},
PreBackupVerify: parksAppReady,
PostRestoreVerify: parksAppReady,
MaxK8SVersion: &K8sVersionOcp47,
}, &BackupVsl{CreateFromDpa: false},
nil),
Entry("Parks application >=4.8.0", BackupRestoreCase{
ApplicationTemplate: "./sample-applications/parks-app/manifest4.8.yaml",
ApplicationNamespace: "parks-app",
Name: "parks-e2e",
BackupRestoreType: RESTIC,
PreBackupVerify: parksAppReady,
PostRestoreVerify: parksAppReady,
MinK8SVersion: &K8sVersionOcp48,
ApplicationTemplate: "./sample-applications/parks-app/manifest4.8.yaml",
Name: "parks-app",
BackupSpec: velero.BackupSpec{
IncludedNamespaces: []string{"parks-app"},
},
PreBackupVerify: parksAppReady,
PostRestoreVerify: parksAppReady,
MinK8SVersion: &K8sVersionOcp48,
}, &BackupVsl{
CreateFromDpa: true,
}, nil),
)
})
Loading