diff --git a/e2e/cephfs.go b/e2e/cephfs.go index dbbc1d2e48c..b340969dae9 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -282,7 +282,7 @@ var _ = Describe("cephfs", func() { appEphemeralPath := cephFSExamplePath + "pod-ephemeral.yaml" By("checking provisioner deployment is running", func() { - err := waitForDeploymentComplete(cephFSDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) + err := waitForDeploymentComplete(f.ClientSet, cephFSDeploymentName, cephCSINamespace, deployTimeout) if err != nil { e2elog.Failf("timeout waiting for deployment %s: %v", cephFSDeploymentName, err) } diff --git a/e2e/deployment.go b/e2e/deployment.go new file mode 100644 index 00000000000..b70bf190451 --- /dev/null +++ b/e2e/deployment.go @@ -0,0 +1,172 @@ +/* +Copyright 2021 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "errors" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util" + "k8s.io/kubernetes/test/e2e/framework" + e2elog "k8s.io/kubernetes/test/e2e/framework/log" +) + +// execCommandInPodWithName run command in pod using podName. +func execCommandInPodWithName( + f *framework.Framework, + cmdString, + podName, + containerName, + nameSpace string) (string, string, error) { + cmd := []string{"/bin/sh", "-c", cmdString} + podOpt := framework.ExecOptions{ + Command: cmd, + PodName: podName, + Namespace: nameSpace, + ContainerName: containerName, + Stdin: nil, + CaptureStdout: true, + CaptureStderr: true, + PreserveWhitespace: true, + } + + return f.ExecWithOptions(podOpt) +} + +// loadAppDeployment loads the deployment app config and return deployment +// object. +func loadAppDeployment(path string) (*appsv1.Deployment, error) { + deploy := appsv1.Deployment{} + if err := unmarshal(path, &deploy); err != nil { + return nil, err + } + + return &deploy, nil +} + +// createDeploymentApp creates the deployment object and waits for it to be in +// Available state. +func createDeploymentApp(clientSet kubernetes.Interface, app *appsv1.Deployment, deployTimeout int) error { + _, err := clientSet.AppsV1().Deployments(app.Namespace).Create(context.TODO(), app, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create deploy: %w", err) + } + + return waitForDeploymentInAvailableState(clientSet, app.Name, app.Namespace, deployTimeout) +} + +// deleteDeploymentApp deletes the deployment object. +func deleteDeploymentApp(clientSet kubernetes.Interface, name, ns string, deployTimeout int) error { + timeout := time.Duration(deployTimeout) * time.Minute + err := clientSet.AppsV1().Deployments(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete deployment: %w", err) + } + start := time.Now() + e2elog.Logf("Waiting for deployment %q to be deleted", name) + + return wait.PollImmediate(poll, timeout, func() (bool, error) { + _, err := clientSet.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + if isRetryableAPIError(err) { + return false, nil + } + if apierrs.IsNotFound(err) { + return true, nil + } + e2elog.Logf("%q deployment to be deleted (%d seconds elapsed)", name, int(time.Since(start).Seconds())) + + return false, fmt.Errorf("failed to get deployment: %w", err) + } + + return false, nil + }) +} + +// waitForDeploymentInAvailableState wait for deployment to be in Available state. +func waitForDeploymentInAvailableState(clientSet kubernetes.Interface, name, ns string, deployTimeout int) error { + timeout := time.Duration(deployTimeout) * time.Minute + start := time.Now() + e2elog.Logf("Waiting up to %q to be in Available state", name) + + return wait.PollImmediate(poll, timeout, func() (bool, error) { + d, err := clientSet.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + if isRetryableAPIError(err) { + return false, nil + } + e2elog.Logf("%q deployment to be Available (%d seconds elapsed)", name, int(time.Since(start).Seconds())) + + return false, err + } + cond := deploymentutil.GetDeploymentCondition(d.Status, appsv1.DeploymentAvailable) + + return cond != nil, nil + }) +} + +// Waits for the deployment to complete. +func waitForDeploymentComplete(clientSet kubernetes.Interface, name, ns string, deployTimeout int) error { + var ( + deployment *appsv1.Deployment + reason string + err error + ) + timeout := time.Duration(deployTimeout) * time.Minute + err = wait.PollImmediate(poll, timeout, func() (bool, error) { + deployment, err = clientSet.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + if isRetryableAPIError(err) { + return false, nil + } + e2elog.Logf("deployment error: %v", err) + + return false, err + } + + // TODO need to check rolling update + + // When the deployment status and its underlying resources reach the + // desired state, we're done + if deployment.Status.Replicas == deployment.Status.ReadyReplicas { + return true, nil + } + e2elog.Logf( + "deployment status: expected replica count %d running replica count %d", + deployment.Status.Replicas, + deployment.Status.ReadyReplicas) + reason = fmt.Sprintf("deployment status: %#v", deployment.Status.String()) + + return false, nil + }) + + if errors.Is(err, wait.ErrWaitTimeout) { + err = fmt.Errorf("%s", reason) + } + if err != nil { + return fmt.Errorf("error waiting for deployment %q status to match desired state: %w", name, err) + } + + return nil +} diff --git a/e2e/pod.go b/e2e/pod.go index a8242bd6a6e..9f0127d0933 100644 --- a/e2e/pod.go +++ b/e2e/pod.go @@ -7,7 +7,6 @@ import ( "strings" "time" - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,52 +72,6 @@ func waitForDaemonSets(name, ns string, c kubernetes.Interface, t int) error { }) } -// Waits for the deployment to complete. - -func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) error { - var ( - deployment *appsv1.Deployment - reason string - err error - ) - timeout := time.Duration(t) * time.Minute - err = wait.PollImmediate(poll, timeout, func() (bool, error) { - deployment, err = c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - if isRetryableAPIError(err) { - return false, nil - } - e2elog.Logf("deployment error: %v", err) - - return false, err - } - - // TODO need to check rolling update - - // When the deployment status and its underlying resources reach the - // desired state, we're done - if deployment.Status.Replicas == deployment.Status.ReadyReplicas { - return true, nil - } - e2elog.Logf( - "deployment status: expected replica count %d running replica count %d", - deployment.Status.Replicas, - deployment.Status.ReadyReplicas) - reason = fmt.Sprintf("deployment status: %#v", deployment.Status.String()) - - return false, nil - }) - - if errors.Is(err, wait.ErrWaitTimeout) { - err = fmt.Errorf("%s", reason) - } - if err != nil { - return fmt.Errorf("error waiting for deployment %q status to match expectation: %w", name, err) - } - - return nil -} - func findPodAndContainerName(f *framework.Framework, ns, cn string, opt *metav1.ListOptions) (string, string, error) { podList, err := f.PodClientNS(ns).List(context.TODO(), *opt) if err != nil { diff --git a/e2e/rbd.go b/e2e/rbd.go index e5ab04e644c..a7cc6d4f596 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -33,6 +33,7 @@ var ( rbdDirPath = "../deploy/rbd/kubernetes/" examplePath = "../examples/" rbdExamplePath = examplePath + "/rbd/" + e2eTemplatesPath = "../e2e/templates/" rbdDeploymentName = "csi-rbdplugin-provisioner" rbdDaemonsetName = "csi-rbdplugin" defaultRBDPool = "replicapool" @@ -59,6 +60,8 @@ var ( appBlockSmartClonePath = rbdExamplePath + "block-pod-clone.yaml" appEphemeralPath = rbdExamplePath + "pod-ephemeral.yaml" snapshotPath = rbdExamplePath + "snapshot.yaml" + deployFSAppPath = e2eTemplatesPath + "rbd-fs-deployment.yaml" + deployBlockAppPath = e2eTemplatesPath + "rbd-block-deployment.yaml" defaultCloneCount = 10 nbdMapOptions = "nbd:debug-rbd=20" @@ -254,7 +257,7 @@ var _ = Describe("RBD", func() { deployVault(f.ClientSet, deployTimeout) // wait for provisioner deployment - err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) + err = waitForDeploymentComplete(f.ClientSet, rbdDeploymentName, cephCSINamespace, deployTimeout) if err != nil { e2elog.Failf("timeout waiting for deployment %s: %v", rbdDeploymentName, err) } @@ -576,6 +579,337 @@ var _ = Describe("RBD", func() { } }) + // NOTE: RWX is restricted for FileSystem VolumeMode at ceph-csi, + // see pull#261 for more details. + By("Create RWX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + // Storage class with rbd-nbd mounter + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{ + "mounter": "rbd-nbd", + "mapOptions": nbdMapOptions, + "cephLogStrategy": e2eDefaultCephLogStrategy, + }, + deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + pvc, err := loadPVC(rawPvcPath) + if err != nil { + e2elog.Failf("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + pvc.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany} + + app, err := loadAppDeployment(deployBlockAppPath) + if err != nil { + e2elog.Failf("failed to load application deployment: %v", err) + } + app.Namespace = f.UniqueName + + err = createPVCAndDeploymentApp(f, "", pvc, app, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application: %v", err) + } + + err = waitForDeploymentComplete(f.ClientSet, app.Name, app.Namespace, deployTimeout) + if err != nil { + e2elog.Failf("timeout waiting for deployment to be in running state: %v", err) + } + + devPath := app.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath + cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath) + + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", app.Name), + } + podList, err := f.PodClientNS(app.Namespace).List(context.TODO(), opt) + if err != nil { + e2elog.Failf("get pod list failed: %v", err) + } + if len(podList.Items) != int(*app.Spec.Replicas) { + e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *app.Spec.Replicas) + } + for _, pod := range podList.Items { + _, _, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, app.Namespace) + if err != nil { + e2elog.Failf("command %q failed: %v", cmd, err) + } + } + + err = deletePVCAndDeploymentApp(f, "", pvc, app) + if err != nil { + e2elog.Failf("failed to delete PVC and application: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + // validate images in trash + err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout) + if err != nil { + e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + }) + + By("Create ROX+FS Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + // Storage class with rbd-nbd mounter + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{ + "mounter": "rbd-nbd", + "mapOptions": nbdMapOptions, + "cephLogStrategy": e2eDefaultCephLogStrategy, + }, + deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + + // create PVC and bind it to an app + pvc, err := loadPVC(pvcPath) + if err != nil { + e2elog.Failf("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + app, err := loadApp(appPath) + if err != nil { + e2elog.Failf("failed to load application: %v", err) + } + app.Namespace = f.UniqueName + err = createPVCAndApp("", f, pvc, app, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 1, defaultRBDPool) + err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete application: %v", err) + } + + // create clone PVC as ROX + pvcClone, err := loadPVC(pvcSmartClonePath) + if err != nil { + e2elog.Failf("failed to load PVC: %v", err) + } + pvcClone.Spec.DataSource.Name = pvc.Name + pvcClone.Namespace = f.UniqueName + pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany} + appClone, err := loadAppDeployment(deployFSAppPath) + if err != nil { + e2elog.Failf("failed to load application deployment: %v", err) + } + appClone.Namespace = f.UniqueName + appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name + appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true + err = createPVCAndDeploymentApp(f, "", pvcClone, appClone, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application: %v", err) + } + + err = waitForDeploymentComplete(f.ClientSet, appClone.Name, appClone.Namespace, deployTimeout) + if err != nil { + e2elog.Failf("timeout waiting for deployment to be in running state: %v", err) + } + + // validate created backend rbd images + validateRBDImageCount(f, 3, defaultRBDPool) + + filePath := appClone.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + cmd := fmt.Sprintf("echo 'Hello World' > %s", filePath) + + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", appClone.Name), + } + podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt) + if err != nil { + e2elog.Failf("get pod list failed: %v", err) + } + if len(podList.Items) != int(*appClone.Spec.Replicas) { + e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas) + } + for _, pod := range podList.Items { + var stdErr string + _, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace) + if err != nil { + e2elog.Logf("command %q failed: %v", cmd, err) + } + readOnlyErr := fmt.Sprintf("cannot create %s: Read-only file system", filePath) + if !strings.Contains(stdErr, readOnlyErr) { + e2elog.Failf(stdErr) + } + } + + err = deletePVCAndDeploymentApp(f, "", pvcClone, appClone) + if err != nil { + e2elog.Failf("failed to delete PVC and application: %v", err) + } + // delete parent pvc + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete PVC: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + // validate images in trash + err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout) + if err != nil { + e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + }) + + By("Create ROX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + // Storage class with rbd-nbd mounter + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{ + "mounter": "rbd-nbd", + "mapOptions": nbdMapOptions, + "cephLogStrategy": e2eDefaultCephLogStrategy, + }, + deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + + // create PVC and bind it to an app + pvc, err := loadPVC(rawPvcPath) + if err != nil { + e2elog.Failf("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + app, err := loadApp(rawAppPath) + if err != nil { + e2elog.Failf("failed to load application: %v", err) + } + app.Namespace = f.UniqueName + err = createPVCAndApp("", f, pvc, app, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 1, defaultRBDPool) + err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete application: %v", err) + } + + // create clone PVC as ROX + pvcClone, err := loadPVC(pvcBlockSmartClonePath) + if err != nil { + e2elog.Failf("failed to load PVC: %v", err) + } + pvcClone.Spec.DataSource.Name = pvc.Name + pvcClone.Namespace = f.UniqueName + pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany} + volumeMode := v1.PersistentVolumeBlock + pvcClone.Spec.VolumeMode = &volumeMode + appClone, err := loadAppDeployment(deployBlockAppPath) + if err != nil { + e2elog.Failf("failed to load application deployment: %v", err) + } + appClone.Namespace = f.UniqueName + appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name + appClone.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true + err = createPVCAndDeploymentApp(f, "", pvcClone, appClone, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC and application: %v", err) + } + + err = waitForDeploymentComplete(f.ClientSet, appClone.Name, appClone.Namespace, deployTimeout) + if err != nil { + e2elog.Failf("timeout waiting for deployment to be in running state: %v", err) + } + + // validate created backend rbd images + validateRBDImageCount(f, 3, defaultRBDPool) + + devPath := appClone.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath + cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath) + + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", appClone.Name), + } + podList, err := f.PodClientNS(appClone.Namespace).List(context.TODO(), opt) + if err != nil { + e2elog.Failf("get pod list failed: %v", err) + } + if len(podList.Items) != int(*appClone.Spec.Replicas) { + e2elog.Failf("podlist contains %d items, expected %d items", len(podList.Items), *appClone.Spec.Replicas) + } + for _, pod := range podList.Items { + var stdErr string + _, stdErr, err = execCommandInPodWithName(f, cmd, pod.Name, pod.Spec.Containers[0].Name, appClone.Namespace) + if err != nil { + e2elog.Logf("command %q failed: %v", cmd, err) + } + readOnlyErr := fmt.Sprintf("'%s': Operation not permitted", devPath) + if !strings.Contains(stdErr, readOnlyErr) { + e2elog.Failf(stdErr) + } + } + err = deletePVCAndDeploymentApp(f, "", pvcClone, appClone) + if err != nil { + e2elog.Failf("failed to delete PVC and application: %v", err) + } + // delete parent pvc + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete PVC: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + // validate images in trash + err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout) + if err != nil { + e2elog.Failf("failed to validate rbd images in pool %s trash: %v", rbdOptions(defaultRBDPool), err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + }) + By("perform IO on rbd-nbd volume after nodeplugin restart", func() { err := deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -2414,7 +2748,7 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("timeout waiting for daemonset pods: %v", err) } - err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) + err = waitForDeploymentComplete(f.ClientSet, rbdDeploymentName, cephCSINamespace, deployTimeout) if err != nil { e2elog.Failf("timeout waiting for deployment to be in running state: %v", err) } diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index 08661c792f9..6d6950fb43a 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -997,7 +997,7 @@ func recreateCSIRBDPods(f *framework.Framework) error { if err != nil { return fmt.Errorf("timeout waiting for daemonset pods: %w", err) } - err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) + err = waitForDeploymentComplete(f.ClientSet, rbdDeploymentName, cephCSINamespace, deployTimeout) if err != nil { return fmt.Errorf("timeout waiting for deployment to be in running state: %w", err) } diff --git a/e2e/templates/rbd-block-deployment.yaml b/e2e/templates/rbd-block-deployment.yaml new file mode 100644 index 00000000000..0fcf2a5df6f --- /dev/null +++ b/e2e/templates/rbd-block-deployment.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pod-block-rx-volume + labels: + app: pod-block-rx-volume +spec: + replicas: 3 + selector: + matchLabels: + app: pod-block-rx-volume + template: + metadata: + labels: + app: pod-block-rx-volume + spec: + containers: + - name: centos + image: quay.io/centos/centos:latest + imagePullPolicy: IfNotPresent + command: ["/bin/sleep", "infinity"] + volumeDevices: + - name: data + devicePath: /dev/xvda + volumes: + - name: data + persistentVolumeClaim: + claimName: raw-block-pvc + readOnly: false diff --git a/e2e/templates/rbd-fs-deployment.yaml b/e2e/templates/rbd-fs-deployment.yaml new file mode 100644 index 00000000000..8ee3cbb9c33 --- /dev/null +++ b/e2e/templates/rbd-fs-deployment.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pod-fs-rx-volume + labels: + app: pod-fs-rx-volume +spec: + replicas: 3 + selector: + matchLabels: + app: pod-fs-rx-volume + template: + metadata: + labels: + app: pod-fs-rx-volume + spec: + containers: + - name: web-server + image: docker.io/library/nginx:latest + imagePullPolicy: IfNotPresent + volumeMounts: + - name: mypvc + mountPath: /var/lib/www/html + volumes: + - name: mypvc + persistentVolumeClaim: + claimName: rbd-pvc + readOnly: false diff --git a/e2e/upgrade-cephfs.go b/e2e/upgrade-cephfs.go index e01926c6b34..236675841cb 100644 --- a/e2e/upgrade-cephfs.go +++ b/e2e/upgrade-cephfs.go @@ -148,7 +148,7 @@ var _ = Describe("CephFS Upgrade Testing", func() { Context("Cephfs Upgrade Test", func() { It("Cephfs Upgrade Test", func() { By("checking provisioner deployment is running", func() { - err = waitForDeploymentComplete(cephFSDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) + err = waitForDeploymentComplete(f.ClientSet, cephFSDeploymentName, cephCSINamespace, deployTimeout) if err != nil { e2elog.Failf("timeout waiting for deployment %s: %v", cephFSDeploymentName, err) } @@ -241,7 +241,7 @@ var _ = Describe("CephFS Upgrade Testing", func() { } deployCephfsPlugin() - err = waitForDeploymentComplete(cephFSDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) + err = waitForDeploymentComplete(f.ClientSet, cephFSDeploymentName, cephCSINamespace, deployTimeout) if err != nil { e2elog.Failf("timeout waiting for upgraded deployment %s: %v", cephFSDeploymentName, err) } diff --git a/e2e/upgrade-rbd.go b/e2e/upgrade-rbd.go index 67c45ab1055..2db1e419a31 100644 --- a/e2e/upgrade-rbd.go +++ b/e2e/upgrade-rbd.go @@ -165,7 +165,7 @@ var _ = Describe("RBD Upgrade Testing", func() { appPath := rbdExamplePath + "pod.yaml" By("checking provisioner deployment is running", func() { - err := waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) + err := waitForDeploymentComplete(f.ClientSet, rbdDeploymentName, cephCSINamespace, deployTimeout) if err != nil { e2elog.Failf("timeout waiting for deployment %s: %v", rbdDeploymentName, err) } @@ -260,7 +260,7 @@ var _ = Describe("RBD Upgrade Testing", func() { deployRBDPlugin() - err = waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) + err = waitForDeploymentComplete(f.ClientSet, rbdDeploymentName, cephCSINamespace, deployTimeout) if err != nil { e2elog.Failf("timeout waiting for upgraded deployment %s: %v", rbdDeploymentName, err) } diff --git a/e2e/utils.go b/e2e/utils.go index 58a13853e43..9af1a42a602 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -15,6 +15,7 @@ import ( "time" snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + appsv1 "k8s.io/api/apps/v1" batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" scv1 "k8s.io/api/storage/v1" @@ -177,6 +178,50 @@ func createPVCAndApp( return err } +// createPVCAndDeploymentApp creates pvc and deployment, if name is not empty +// same will be set as pvc and app name. +func createPVCAndDeploymentApp( + f *framework.Framework, + name string, + pvc *v1.PersistentVolumeClaim, + app *appsv1.Deployment, + pvcTimeout int) error { + if name != "" { + pvc.Name = name + app.Name = name + app.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name + } + err := createPVCAndvalidatePV(f.ClientSet, pvc, pvcTimeout) + if err != nil { + return err + } + err = createDeploymentApp(f.ClientSet, app, deployTimeout) + + return err +} + +// DeletePVCAndDeploymentApp deletes pvc and deployment, if name is not empty +// same will be set as pvc and app name. +func deletePVCAndDeploymentApp( + f *framework.Framework, + name string, + pvc *v1.PersistentVolumeClaim, + app *appsv1.Deployment) error { + if name != "" { + pvc.Name = name + app.Name = name + app.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name + } + + err := deleteDeploymentApp(f.ClientSet, app.Name, app.Namespace, deployTimeout) + if err != nil { + return err + } + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + + return err +} + // deletePVCAndApp delete pvc and pod // if name is not empty same will be set as pvc and app name. func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {