Skip to content

Commit

Permalink
e2e: add tests using different accessModes and volumeModes for rbd-nbd
Browse files Browse the repository at this point in the history 
Add tests for RWX and ROX accessModes for Block and FileSystem Mode
PVCs.

Fixes: #2262
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
  • Loading branch information
Prasanna Kumar Kalever committed Oct 29, 2021
1 parent 2c738ec commit b92ed1a
Showing 1 changed file with 370 additions and 0 deletions.
370 changes: 370 additions & 0 deletions e2e/rbd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ var (
appSmartClonePath = rbdExamplePath + "pod-clone.yaml"
appBlockSmartClonePath = rbdExamplePath + "block-pod-clone.yaml"
snapshotPath = rbdExamplePath + "snapshot.yaml"
rxAppPath = rbdExamplePath + "fs-pod-rx.yaml"
rxRawAppPath = rbdExamplePath + "block-pod-rx.yaml"
defaultCloneCount = 10

nbdMapOptions = "debug-rbd=20"
Expand Down Expand Up @@ -545,6 +547,374 @@ var _ = Describe("RBD", func() {
}
})

// NOTE: RWX is restricted for FileSystem VolumeMode at ceph-csi,
// see pull#261 for more details.
By("Create RWX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
// Storage class with rbd-nbd mounter
err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
map[string]string{
"mounter": "rbd-nbd",
"mapOptions": nbdMapOptions,
"cephLogStrategy": e2eDefaultCephLogStrategy,
},
deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
pvc, err := loadPVC(pvcPath)
if err != nil {
e2elog.Failf("failed to load PVC: %v", err)
}
pvc.Namespace = f.UniqueName
pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}
volumeMode := v1.PersistentVolumeBlock
pvc.Spec.VolumeMode = &volumeMode

app, err := loadAppDeployment(rxRawAppPath)
if err != nil {
e2elog.Failf("failed to load application deployment: %v", err)
}
app.Namespace = f.UniqueName
app.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name

err = createPVCAndDeploymentApp("", f, pvc, app, deployTimeout)
if err != nil {
e2elog.Failf("failed to create PVC and application: %v", err)
}

err = waitForDeploymentComplete(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
}

devPath := app.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath
cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath)

opt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("app=%s", app.Name),
}
podList, err := f.PodClientNS(app.Namespace).List(context.TODO(), opt)
if err != nil {
e2elog.Failf("get pod list failed: %v", err)
}
if len(podList.Items) != int(*app.Spec.Replicas) {
e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *app.Spec.Replicas)
}
for _, pod := range podList.Items {
_, _, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, app.Namespace)
if err != nil {
e2elog.Failf("command %q failed: %v", cmd, err)
}
}

err = deletePVCAndDeploymentApp("", f, pvc, app)
if err != nil {
e2elog.Failf("failed to delete PVC and application: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
})

By("Create ROX+FS Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
// Storage class with rbd-nbd mounter
err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
map[string]string{
"mounter": "rbd-nbd",
"mapOptions": nbdMapOptions,
"cephLogStrategy": e2eDefaultCephLogStrategy,
},
deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
// snapshot beta is only supported from v1.17+
err = createRBDSnapshotClass(f)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
defer func() {
err = deleteRBDSnapshotClass()
if err != nil {
e2elog.Failf("failed to delete VolumeSnapshotClass: %v", err)
}
}()

// create PVC and bind it to an app
pvc, err := loadPVC(pvcPath)
if err != nil {
e2elog.Failf("failed to load PVC: %v", err)
}

pvc.Namespace = f.UniqueName
app, err := loadApp(appPath)
if err != nil {
e2elog.Failf("failed to load application: %v", err)
}
app.Namespace = f.UniqueName
err = createPVCAndApp("", f, pvc, app, deployTimeout)
if err != nil {
e2elog.Failf("failed to create PVC and application: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 1, defaultRBDPool)
// delete pod as we should not create snapshot for in-use pvc
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete application: %v", err)
}

snap := getSnapshot(snapshotPath)
snap.Namespace = f.UniqueName
snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name

err = createSnapshot(&snap, deployTimeout)
if err != nil {
e2elog.Failf("failed to create snapshot: %v", err)
}
// validate created backend rbd images
// parent PVC + snapshot
totalImages := 2
validateRBDImageCount(f, totalImages, defaultRBDPool)
pvcClone, err := loadPVC(pvcClonePath)
if err != nil {
e2elog.Failf("failed to load PVC: %v", err)
}
// create clone PVC as ROX
pvcClone.Namespace = f.UniqueName
pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}

appClone, err := loadAppDeployment(rxAppPath)
if err != nil {
e2elog.Failf("failed to load application deployment: %v", err)
}
appClone.Namespace = f.UniqueName
appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name

err = createPVCAndDeploymentApp("", f, pvcClone, appClone, deployTimeout)
if err != nil {
e2elog.Failf("failed to create PVC and application: %v", err)
}

err = waitForDeploymentComplete(appClone.Name, appClone.Namespace, f.ClientSet, deployTimeout)
if err != nil {
e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
}

// validate created backend rbd images
// parent pvc+ snapshot + clone
totalImages = 3
validateRBDImageCount(f, totalImages, defaultRBDPool)

filePath := appClone.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
cmd := fmt.Sprintf("echo 'Hello World' > %s", filePath)

opt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("app=%s", appClone.Name),
}
podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt)
if err != nil {
e2elog.Failf("get pod list failed: %v", err)
}
if len(podList.Items) != int(*appClone.Spec.Replicas) {
e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas)
}
for _, pod := range podList.Items {
var stdErr string
_, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace)
if err != nil {
e2elog.Logf("command %q failed: %v", cmd, err)
}
readOnlyErr := fmt.Sprintf("cannot create %s: Read-only file system", filePath)
if !strings.Contains(stdErr, readOnlyErr) {
e2elog.Failf(stdErr)
}
}

err = deletePVCAndDeploymentApp("", f, pvcClone, appClone)
if err != nil {
e2elog.Failf("failed to delete PVC and application: %v", err)
}
// delete snapshot
err = deleteSnapshot(&snap, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete snapshot: %v", err)
}
// delete parent pvc
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete PVC: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
})

By("Create ROX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
// Storage class with rbd-nbd mounter
err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
map[string]string{
"mounter": "rbd-nbd",
"mapOptions": nbdMapOptions,
"cephLogStrategy": e2eDefaultCephLogStrategy,
},
deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
// snapshot beta is only supported from v1.17+
err = createRBDSnapshotClass(f)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
defer func() {
err = deleteRBDSnapshotClass()
if err != nil {
e2elog.Failf("failed to delete VolumeSnapshotClass: %v", err)
}
}()

// create PVC and bind it to an app
pvc, err := loadPVC(pvcPath)
if err != nil {
e2elog.Failf("failed to load PVC: %v", err)
}

pvc.Namespace = f.UniqueName
app, err := loadApp(appPath)
if err != nil {
e2elog.Failf("failed to load application: %v", err)
}
app.Namespace = f.UniqueName
err = createPVCAndApp("", f, pvc, app, deployTimeout)
if err != nil {
e2elog.Failf("failed to create PVC and application: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 1, defaultRBDPool)
// delete pod as we should not create snapshot for in-use pvc
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete application: %v", err)
}

snap := getSnapshot(snapshotPath)
snap.Namespace = f.UniqueName
snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name

err = createSnapshot(&snap, deployTimeout)
if err != nil {
e2elog.Failf("failed to create snapshot: %v", err)
}
// validate created backend rbd images
// parent PVC + snapshot
totalImages := 2
validateRBDImageCount(f, totalImages, defaultRBDPool)
pvcClone, err := loadPVC(pvcClonePath)
if err != nil {
e2elog.Failf("failed to load PVC: %v", err)
}

// create clone PVC as ROX
pvcClone.Namespace = f.UniqueName
pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}
volumeMode := v1.PersistentVolumeBlock
pvcClone.Spec.VolumeMode = &volumeMode

appClone, err := loadAppDeployment(rxRawAppPath)
if err != nil {
e2elog.Failf("failed to load application deployment: %v", err)
}
appClone.Namespace = f.UniqueName
appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name

err = createPVCAndDeploymentApp("", f, pvcClone, appClone, deployTimeout)
if err != nil {
e2elog.Failf("failed to create PVC and application: %v", err)
}

err = waitForDeploymentComplete(appClone.Name, appClone.Namespace, f.ClientSet, deployTimeout)
if err != nil {
e2elog.Failf("timeout waiting for deployment to be in running state: %v", err)
}

// validate created backend rbd images
// parent pvc+ snapshot + clone
totalImages = 3
validateRBDImageCount(f, totalImages, defaultRBDPool)

devPath := appClone.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath
cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath)

opt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("app=%s", appClone.Name),
}
podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt)
if err != nil {
e2elog.Failf("get pod list failed: %v", err)
}
if len(podList.Items) != int(*appClone.Spec.Replicas) {
e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas)
}
for _, pod := range podList.Items {
var stdErr string
_, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace)
if err != nil {
e2elog.Logf("command %q failed: %v", cmd, err)
}
readOnlyErr := fmt.Sprintf("dd: error writing '%s': Operation not permitted", devPath)
if !strings.Contains(stdErr, readOnlyErr) {
e2elog.Failf(stdErr)
}
}
err = deletePVCAndDeploymentApp("", f, pvcClone, appClone)
if err != nil {
e2elog.Failf("failed to delete PVC and application: %v", err)
}
// delete snapshot
err = deleteSnapshot(&snap, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete snapshot: %v", err)
}
// delete parent pvc
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete PVC: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
})

By("perform IO on rbd-nbd volume after nodeplugin restart", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
Expand Down

0 comments on commit b92ed1a

Please sign in to comment.