Skip to content

Commit

Permalink
tests for using multiple Rook/Ceph cluster environment
Browse files Browse the repository at this point in the history 
Signed-off-by: Yuji Ito <llamerada.jp@gmail.com>
  • Loading branch information
@llamerada-jp
llamerada-jp committed Jul 5, 2024
1 parent 38016c6 commit d41cd5f
Show file tree
Hide file tree
Showing 7 changed files with 470 additions and 105 deletions.
3 changes: 2 additions & 1 deletion test/e2e/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ launch-rook-ceph: create-loop-dev
> testdata/persistentvolumes.yaml
$(KUBECTL) apply -f testdata/persistentvolumes.yaml
$(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \
--namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values-cluster.yaml --wait rook-ceph-cluster rook-ceph-cluster
--namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values-cluster.yaml \
--wait rook-ceph-cluster rook-ceph-cluster
$(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \
--create-namespace --namespace $(CEPH_CLUSTER2_NAMESPACE) -f testdata/values-cluster.yaml \
--set cephClusterSpec.dataDirHostPath=/var/lib/rook2 \
Expand Down
125 changes: 41 additions & 84 deletions test/e2e/backup_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package e2e

import (
"encoding/json"
"fmt"
"strings"

Expand All @@ -19,11 +18,9 @@ const (
)

type backupTest struct {
poolName string
storageClassName1 string
storageClassName2 string
tenantNamespace1 string
tenantNamespace2 string
poolName string
storageClassName string
tenantNamespace string

pvcName1 string
pvcName2 string
Expand All @@ -34,11 +31,9 @@ type backupTest struct {

func backupTestSuite() {
test := &backupTest{
poolName: util.GetUniqueName("pool-"),
storageClassName1: util.GetUniqueName("sc-"),
storageClassName2: util.GetUniqueName("sc-"),
tenantNamespace1: util.GetUniqueName("ns-"),
tenantNamespace2: util.GetUniqueName("ns-"),
poolName: util.GetUniqueName("pool-"),
storageClassName: util.GetUniqueName("sc-"),
tenantNamespace: util.GetUniqueName("ns-"),

pvcName1: "rbd-pvc1",
pvcName2: "rbd-pvc2",
Expand All @@ -53,74 +48,47 @@ func backupTestSuite() {
}

func (test *backupTest) setupEnv() {
It("creating common resources", func() {
for _, ns := range []string{test.tenantNamespace1, test.tenantNamespace2} {
err := createNamespace(ns)
Expect(err).NotTo(HaveOccurred())
}
It("setting up the test environment", func() {
fmt.Fprintf(GinkgoWriter, "%+v\n", *test)
})

err := applyRBDPoolAndSCTemplate(cephCluster1Namespace, test.poolName, test.storageClassName1)
It("creating common resources", func() {
err := createNamespace(test.tenantNamespace)
Expect(err).NotTo(HaveOccurred())

err = applyRBDPoolAndSCTemplate(cephCluster2Namespace, test.poolName, test.storageClassName2)
err = applyRBDPoolAndSCTemplate(cephCluster1Namespace, test.poolName, test.storageClassName)
Expect(err).NotTo(HaveOccurred())

for _, name := range []string{test.pvcName1, test.pvcName2} {
By(fmt.Sprintf("Creating PVC, PV and RBD image (%s)", name))
err = applyPVCTemplate(test.tenantNamespace1, name, test.storageClassName1)
Expect(err).NotTo(HaveOccurred())

pvName, err := getPVFromPVC(test.tenantNamespace1, name)
Expect(err).NotTo(HaveOccurred())

imageName, err := getImageNameFromPVName(pvName)
err = applyPVCTemplate(test.tenantNamespace, name, test.storageClassName)
Expect(err).NotTo(HaveOccurred())

// Create a new RBD image in cephNamespace2 with the same name as imageName.
Eventually(func() error {
return createRBDImage(cephCluster2Namespace, test.poolName, imageName, 100)
}).Should(Succeed())
}
})

It("waiting for mantle-controller to get ready", func() {
Eventually(func() error {
return checkDeploymentReady(cephCluster1Namespace, "mantle-controller")
}).Should(Succeed())
})
}

func (test *backupTest) teardownEnv() {
It("deleting MantleBackups", func() {
for _, mantleBackup := range []string{test.mantleBackupName1, test.mantleBackupName2, test.mantleBackupName3} {
_, _, _ = kubectl("delete", "-n", test.tenantNamespace1, "mantlebackup", mantleBackup)
}
It("delete resources in the namespace: "+test.tenantNamespace, func() {
err := deleteNamespacedResource(test.tenantNamespace, "mantlebackup")
Expect(err).NotTo(HaveOccurred())
err = deleteNamespacedResource(test.tenantNamespace, "pvc")
Expect(err).NotTo(HaveOccurred())
})

It("deleting PVCs", func() {
for _, pvc := range []string{test.pvcName1, test.pvcName2} {
_, _, _ = kubectl("delete", "-n", test.tenantNamespace1, "pvc", pvc)
}

By("Deleting RBD images in " + cephCluster2Namespace)
stdout, _, err := kubectl("exec", "-n", cephCluster2Namespace, "deploy/rook-ceph-tools", "--",
"rbd", "ls", test.poolName, "--format=json")
if err == nil {
imageNames := []string{}
if err := json.Unmarshal(stdout, &imageNames); err == nil {
for _, imageName := range imageNames {
_, _, _ = kubectl("exec", "-n", cephCluster2Namespace, "deploy/rook-ceph-tools", "--",
"rbd", "rm", test.poolName+"/"+imageName)
}
}
}
It("delete namespace: "+test.tenantNamespace, func() {
_, stderr, err := kubectl("delete", "namespace", test.tenantNamespace)
Expect(err).NotTo(HaveOccurred(), string(stderr))
})

It("deleting common resources", func() {
_, _, _ = kubectl("delete", "sc", test.storageClassName1, "--wait=false")
_, _, _ = kubectl("delete", "sc", test.storageClassName2, "--wait=false")
_, _, _ = kubectl("delete", "-n", cephCluster1Namespace, "cephblockpool", test.poolName, "--wait=false")
_, _, _ = kubectl("delete", "-n", cephCluster2Namespace, "cephblockpool", test.poolName, "--wait=false")
It("clean up the SCs and RBD pools", func() {
_, stderr, err := kubectl("delete", "sc", test.storageClassName)
Expect(err).NotTo(HaveOccurred(), string(stderr))

err = removeAllRBDImageAndSnap(cephCluster1Namespace, test.poolName)
Expect(err).NotTo(HaveOccurred())

_, _, err = kubectl("delete", "-n", cephCluster1Namespace, "cephblockpool", test.poolName, "--wait=false")
Expect(err).NotTo(HaveOccurred())
})
}

Expand All @@ -129,10 +97,10 @@ func (test *backupTest) testCase1() {

createMantleBackupAndGetImage := func(mantleBackupName string) string {
By("Creating MantleBackup")
err := applyMantleBackupTemplate(test.tenantNamespace1, test.pvcName1, mantleBackupName)
err := applyMantleBackupTemplate(test.tenantNamespace, test.pvcName1, mantleBackupName)
Expect(err).NotTo(HaveOccurred())

pvName, err := getPVFromPVC(test.tenantNamespace1, test.pvcName1)
pvName, err := getPVFromPVC(test.tenantNamespace, test.pvcName1)
Expect(err).NotTo(HaveOccurred())

By("Waiting for RBD snapshot to be created")
Expand All @@ -146,17 +114,6 @@ func (test *backupTest) testCase1() {
return checkSnapshotExist(cephCluster1Namespace, test.poolName, imageName, mantleBackupName)
}).Should(Succeed())

By("Checking that the mantle-controller deployed for a certain Rook/Ceph cluster (i.e., " +
cephCluster1Namespace + ") doesn't create a snapshot for a MantleBackup for a different Rook/Ceph cluster (i.e., " +
cephCluster2Namespace + ")")
Consistently(func() error {
err := checkSnapshotExist(cephCluster2Namespace, test.poolName, imageName, mantleBackupName)
if err == nil {
return fmt.Errorf("a wrong snapshot exists. snapshotName: %s", mantleBackupName)
}
return nil
}).Should(Succeed())

return imageName
}

Expand All @@ -170,10 +127,10 @@ func (test *backupTest) testCase1() {

It("should create MantleBackups resources for different PVCs", func() {
By("Creating a third MantleBackup for the other PVC")
err := applyMantleBackupTemplate(test.tenantNamespace1, test.pvcName2, test.mantleBackupName3)
err := applyMantleBackupTemplate(test.tenantNamespace, test.pvcName2, test.mantleBackupName3)
Expect(err).NotTo(HaveOccurred())

pvName, err := getPVFromPVC(test.tenantNamespace1, test.pvcName2)
pvName, err := getPVFromPVC(test.tenantNamespace, test.pvcName2)
Expect(err).NotTo(HaveOccurred())

By("Waiting for RBD snapshot to be created")
Expand All @@ -189,12 +146,12 @@ func (test *backupTest) testCase1() {

It("should not delete MantleBackup resource when delete backup target PVC", func() {
By("Deleting backup target PVC")
_, _, err := kubectl("-n", test.tenantNamespace1, "delete", "pvc", test.pvcName2)
_, _, err := kubectl("-n", test.tenantNamespace, "delete", "pvc", test.pvcName2)
Expect(err).NotTo(HaveOccurred())

By("Checking backup target PVC deletion")
Eventually(func() error {
stdout, stderr, err := kubectl("-n", test.tenantNamespace1, "get", "pvc", test.pvcName2)
stdout, stderr, err := kubectl("-n", test.tenantNamespace, "get", "pvc", test.pvcName2)
if err != nil {
if strings.Contains(string(stderr), kubectlIsNotFoundMessage) {
return nil
Expand All @@ -205,7 +162,7 @@ func (test *backupTest) testCase1() {
}).Should(Succeed())

By("Checking that the status.conditions of the MantleBackup resource remain \"Bound\"")
stdout, _, err := kubectl("-n", test.tenantNamespace1, "get", "mantlebackup", test.mantleBackupName3, "-o", "json")
stdout, _, err := kubectl("-n", test.tenantNamespace, "get", "mantlebackup", test.mantleBackupName3, "-o", "json")
Expect(err).NotTo(HaveOccurred())
var backup mantlev1.MantleBackup
err = yaml.Unmarshal(stdout, &backup)
Expand All @@ -216,7 +173,7 @@ func (test *backupTest) testCase1() {

It("should delete MantleBackup resource", func() {
By("Delete MantleBackup")
_, _, err := kubectl("-n", test.tenantNamespace1, "delete", "mantlebackup", test.mantleBackupName1, "--wait=false")
_, _, err := kubectl("-n", test.tenantNamespace, "delete", "mantlebackup", test.mantleBackupName1, "--wait=false")
Expect(err).NotTo(HaveOccurred())

By("Waiting for RBD snapshot to be deleted")
Expand All @@ -231,7 +188,7 @@ func (test *backupTest) testCase1() {

By("Checking MantleBackup resource deletion")
Eventually(func() error {
stdout, stderr, err := kubectl("-n", test.tenantNamespace1, "get", "mantlebackup", test.mantleBackupName1)
stdout, stderr, err := kubectl("-n", test.tenantNamespace, "get", "mantlebackup", test.mantleBackupName1)
if err != nil {
if strings.Contains(string(stderr), kubectlIsNotFoundMessage) {
return nil
Expand All @@ -244,12 +201,12 @@ func (test *backupTest) testCase1() {

It("should delete MantleBackup resource when backup target PVC is missing", func() {
By("Deleting MantleBackup resource")
_, _, err := kubectl("-n", test.tenantNamespace1, "delete", "mantlebackup", test.mantleBackupName3)
_, _, err := kubectl("-n", test.tenantNamespace, "delete", "mantlebackup", test.mantleBackupName3)
Expect(err).NotTo(HaveOccurred())

By("Checking MantleBackup resource deletion")
Eventually(func() error {
stdout, stderr, err := kubectl("-n", test.tenantNamespace1, "get", "mantlebackup", test.mantleBackupName3)
stdout, stderr, err := kubectl("-n", test.tenantNamespace, "get", "mantlebackup", test.mantleBackupName3)
if err != nil {
if strings.Contains(string(stderr), kubectlIsNotFoundMessage) {
return nil
Expand Down
Loading

0 comments on commit d41cd5f

Please sign in to comment.