From cfebab97d91e9bdacc91a379c8e5d180bc0ad697 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 28 Oct 2021 18:04:07 +0530 Subject: [PATCH] e2e: add tests using different accessModes and volumeModes for rbd-nbd Add tests for RWX and ROX accessModes for Block and FileSystem Mode PVCs. Fixes: #2262 Signed-off-by: Prasanna Kumar Kalever --- e2e/rbd.go | 334 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 334 insertions(+) diff --git a/e2e/rbd.go b/e2e/rbd.go index 0978df39179d..a7cc6d4f5960 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -33,6 +33,7 @@ var ( rbdDirPath = "../deploy/rbd/kubernetes/" examplePath = "../examples/" rbdExamplePath = examplePath + "/rbd/" + e2eTemplatesPath = "../e2e/templates/" rbdDeploymentName = "csi-rbdplugin-provisioner" rbdDaemonsetName = "csi-rbdplugin" defaultRBDPool = "replicapool" @@ -59,6 +60,8 @@ var ( appBlockSmartClonePath = rbdExamplePath + "block-pod-clone.yaml" appEphemeralPath = rbdExamplePath + "pod-ephemeral.yaml" snapshotPath = rbdExamplePath + "snapshot.yaml" + deployFSAppPath = e2eTemplatesPath + "rbd-fs-deployment.yaml" + deployBlockAppPath = e2eTemplatesPath + "rbd-block-deployment.yaml" defaultCloneCount = 10 nbdMapOptions = "nbd:debug-rbd=20" @@ -576,6 +579,337 @@ var _ = Describe("RBD", func() { } }) + // NOTE: RWX is restricted for FileSystem VolumeMode at ceph-csi, + // see pull#261 for more details. + By("Create RWX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + // Storage class with rbd-nbd mounter + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{ + "mounter": "rbd-nbd", + "mapOptions": nbdMapOptions, + "cephLogStrategy": e2eDefaultCephLogStrategy, + }, + deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + pvc, err := loadPVC(rawPvcPath) + if err != nil { + e2elog.Failf("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany} + + app, err := loadAppDeployment(deployBlockAppPath) + if err != nil { + e2elog.Failf("failed to load application deployment: %v", err) + } + app.Namespace = f.UniqueName + + err = createPVCAndDeploymentApp(f, "", pvc, app, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application: %v", err) + } + + err = waitForDeploymentComplete(f.ClientSet, app.Name, app.Namespace, deployTimeout) + if err != nil { + e2elog.Failf("timeout waiting for deployment to be in running state: %v", err) + } + + devPath := app.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath + cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath) + + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", app.Name), + } + podList, err := f.PodClientNS(app.Namespace).List(context.TODO(), opt) + if err != nil { + e2elog.Failf("get pod list failed: %v", err) + } + if len(podList.Items) != int(*app.Spec.Replicas) { + e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *app.Spec.Replicas) + } + for _, pod := range podList.Items { + _, _, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, app.Namespace) + if err != nil { + e2elog.Failf("command %q failed: %v", cmd, err) + } + } + + err = deletePVCAndDeploymentApp(f, "", pvc, app) + if err != nil { + e2elog.Failf("failed to delete PVC and application: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + // validate images in trash + err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout) + if err != nil { + e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + }) + + By("Create ROX+FS Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + // Storage class with rbd-nbd mounter + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{ + "mounter": "rbd-nbd", + "mapOptions": nbdMapOptions, + "cephLogStrategy": e2eDefaultCephLogStrategy, + }, + deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + + // create PVC and bind it to an app + pvc, err := loadPVC(pvcPath) + if err != nil { + e2elog.Failf("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + app, err := loadApp(appPath) + if err != nil { + e2elog.Failf("failed to load application: %v", err) + } + app.Namespace = f.UniqueName + err = createPVCAndApp("", f, pvc, app, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 1, defaultRBDPool) + err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete application: %v", err) + } + + // create clone PVC as ROX + pvcClone, err := loadPVC(pvcSmartClonePath) + if err != nil { + e2elog.Failf("failed to load PVC: %v", err) + } + pvcClone.Spec.DataSource.Name = pvc.Name + pvcClone.Namespace = f.UniqueName + pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany} + appClone, err := loadAppDeployment(deployFSAppPath) + if err != nil { + e2elog.Failf("failed to load application deployment: %v", err) + } + appClone.Namespace = f.UniqueName + appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name + appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true + err = createPVCAndDeploymentApp(f, "", pvcClone, appClone, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application: %v", err) + } + + err = waitForDeploymentComplete(f.ClientSet, appClone.Name, appClone.Namespace, deployTimeout) + if err != nil { + e2elog.Failf("timeout waiting for deployment to be in running state: %v", err) + } + + // validate created backend rbd images + validateRBDImageCount(f, 3, defaultRBDPool) + + filePath := appClone.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + cmd := fmt.Sprintf("echo 'Hello World' > %s", filePath) + + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", appClone.Name), + } + podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt) + if err != nil { + e2elog.Failf("get pod list failed: %v", err) + } + if len(podList.Items) != int(*appClone.Spec.Replicas) { + e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas) + } + for _, pod := range podList.Items { + var stdErr string + _, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace) + if err != nil { + e2elog.Logf("command %q failed: %v", cmd, err) + } + readOnlyErr := fmt.Sprintf("cannot create %s: Read-only file system", filePath) + if !strings.Contains(stdErr, readOnlyErr) { + e2elog.Failf(stdErr) + } + } + + err = deletePVCAndDeploymentApp(f, "", pvcClone, appClone) + if err != nil { + e2elog.Failf("failed to delete PVC and application: %v", err) + } + // delete parent pvc + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete PVC: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + // validate images in trash + err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout) + if err != nil { + e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + }) + + By("Create ROX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + // Storage class with rbd-nbd mounter + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{ + "mounter": "rbd-nbd", + "mapOptions": nbdMapOptions, + "cephLogStrategy": e2eDefaultCephLogStrategy, + }, + deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + + // create PVC and bind it to an app + pvc, err := loadPVC(rawPvcPath) + if err != nil { + e2elog.Failf("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + app, err := loadApp(rawAppPath) + if err != nil { + e2elog.Failf("failed to load application: %v", err) + } + app.Namespace = f.UniqueName + err = createPVCAndApp("", f, pvc, app, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 1, defaultRBDPool) + err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete application: %v", err) + } + + // create clone PVC as ROX + pvcClone, err := loadPVC(pvcBlockSmartClonePath) + if err != nil { + e2elog.Failf("failed to load PVC: %v", err) + } + pvcClone.Spec.DataSource.Name = pvc.Name + pvcClone.Namespace = f.UniqueName + pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany} + volumeMode := v1.PersistentVolumeBlock + pvcClone.Spec.VolumeMode = &volumeMode + appClone, err := loadAppDeployment(deployBlockAppPath) + if err != nil { + e2elog.Failf("failed to load application deployment: %v", err) + } + appClone.Namespace = f.UniqueName + appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name + appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true + err = createPVCAndDeploymentApp(f, "", pvcClone, appClone, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application: %v", err) + } + + err = waitForDeploymentComplete(f.ClientSet, appClone.Name, appClone.Namespace, deployTimeout) + if err != nil { + e2elog.Failf("timeout waiting for deployment to be in running state: %v", err) + } + + // validate created backend rbd images + validateRBDImageCount(f, 3, defaultRBDPool) + + devPath := appClone.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath + cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath) + + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", appClone.Name), + } + podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt) + if err != nil { + e2elog.Failf("get pod list failed: %v", err) + } + if len(podList.Items) != int(*appClone.Spec.Replicas) { + e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas) + } + for _, pod := range podList.Items { + var stdErr string + _, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace) + if err != nil { + e2elog.Logf("command %q failed: %v", cmd, err) + } + readOnlyErr := fmt.Sprintf("'%s': Operation not permitted", devPath) + if !strings.Contains(stdErr, readOnlyErr) { + e2elog.Failf(stdErr) + } + } + err = deletePVCAndDeploymentApp(f, "", pvcClone, appClone) + if err != nil { + e2elog.Failf("failed to delete PVC and application: %v", err) + } + // delete parent pvc + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete PVC: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + // validate images in trash + err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout) + if err != nil { + e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + }) + By("perform IO on rbd-nbd volume after nodeplugin restart", func() { err := deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil {