ci: add support for VM_DRIVER=podman to scripts/minikube.sh

e2e/deployment.go

@@ -124,6 +124,9 @@ func waitForDeploymentInAvailableState(clientSet kubernetes.Interface, name, ns
 			if isRetryableAPIError(err) {
 				return false, nil
 			}
+			if apierrs.IsNotFound(err) {
+				return false, nil
+			}
 			e2elog.Logf("%q deployment to be Available (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
 
 			return false, err
@@ -390,6 +393,12 @@ func waitForContainersArgsUpdate(
 ) error {
 	e2elog.Logf("waiting for deployment updates %s/%s", ns, deploymentName)
 
+	// wait for the deployment to be available
+	err := waitForDeploymentInAvailableState(c, deploymentName, ns, deployTimeout)
+	if err != nil {
+		return fmt.Errorf("deployment %s/%s did not become available yet: %w", ns, deploymentName, err)
+	}
+
 	// Scale down to 0.
 	scale, err := c.AppsV1().Deployments(ns).GetScale(context.TODO(), deploymentName, metav1.GetOptions{})
 	if err != nil {

e2e/e2e_test.go

@@ -39,6 +39,7 @@ func init() {
 	flag.BoolVar(&deployNFS, "deploy-nfs", false, "deploy nfs csi driver")
 	flag.BoolVar(&testCephFS, "test-cephfs", true, "test cephFS csi driver")
 	flag.BoolVar(&testRBD, "test-rbd", true, "test rbd csi driver")
+	flag.BoolVar(&testNBD, "test-nbd", false, "test rbd csi driver with rbd-nbd mounter")
 	flag.BoolVar(&testNFS, "test-nfs", false, "test nfs csi driver")
 	flag.BoolVar(&helmTest, "helm-test", false, "tests running on deployment via helm")
 	flag.BoolVar(&upgradeTesting, "upgrade-testing", false, "perform upgrade testing")

e2e/pvc.go

@@ -284,21 +284,29 @@ func deletePVCAndValidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai
 			int(time.Since(start).Seconds()))
 		pvc, err = c.CoreV1().PersistentVolumeClaims(nameSpace).Get(context.TODO(), name, metav1.GetOptions{})
 		if err == nil {
+			e2elog.Logf("PVC %s (status: %s) has not been deleted yet, rechecking...", name, pvc.Status)
+
 			return false, nil
 		}
 		if isRetryableAPIError(err) {
+			e2elog.Logf("failed to verify deletion of PVC %s (status: %s): %v", name, pvc.Status, err)
+
 			return false, nil
 		}
 		if !apierrs.IsNotFound(err) {
 			return false, fmt.Errorf("get on deleted PVC %v failed with error other than \"not found\": %w", name, err)
 		}
 
 		// Examine the pv.ClaimRef and UID. Expect nil values.
-		_, err = c.CoreV1().PersistentVolumes().Get(context.TODO(), pv.Name, metav1.GetOptions{})
+		oldPV, err := c.CoreV1().PersistentVolumes().Get(context.TODO(), pv.Name, metav1.GetOptions{})
 		if err == nil {
+			e2elog.Logf("PV %s (status: %s) has not been deleted yet, rechecking...", pv.Name, oldPV.Status)
+
 			return false, nil
 		}
 		if isRetryableAPIError(err) {
+			e2elog.Logf("failed to verify deletion of PV %s (status: %s): %v", pv.Name, oldPV.Status, err)
+
 			return false, nil
 		}
 		if !apierrs.IsNotFound(err) {

e2e/rbd.go

@@ -1047,6 +1047,12 @@ var _ = Describe("RBD", func() {
 			})
 
 			By("create a PVC and bind it to an app using rbd-nbd mounter", func() {
+				if !testNBD {
+					e2elog.Logf("skipping NBD test")
+
+					return
+				}
+
 				err := deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1083,6 +1089,12 @@ var _ = Describe("RBD", func() {
 			})
 
 			By("Resize rbd-nbd PVC and check application directory size", func() {
+				if !testNBD {
+					e2elog.Logf("skipping NBD test")
+
+					return
+				}
+
 				if util.CheckKernelSupport(kernelRelease, nbdResizeSupport) {
 					err := deleteResource(rbdExamplePath + "storageclass.yaml")
 					if err != nil {
@@ -1290,6 +1302,12 @@ var _ = Describe("RBD", func() {
 
 			By("create PVC with journaling,fast-diff image-features and bind it to an app using rbd-nbd mounter",
 				func() {
+					if !testNBD {
+						e2elog.Logf("skipping NBD test")
+
+						return
+					}
+
 					if util.CheckKernelSupport(kernelRelease, fastDiffSupport) {
 						err := deleteResource(rbdExamplePath + "storageclass.yaml")
 						if err != nil {
@@ -1330,6 +1348,12 @@ var _ = Describe("RBD", func() {
 			// NOTE: RWX is restricted for FileSystem VolumeMode at ceph-csi,
 			// see pull#261 for more details.
 			By("Create RWX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
+				if !testNBD {
+					e2elog.Logf("skipping NBD test")
+
+					return
+				}
+
 				err := deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1415,6 +1439,12 @@ var _ = Describe("RBD", func() {
 			})
 
 			By("Create ROX+FS Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
+				if !testNBD {
+					e2elog.Logf("skipping NBD test")
+
+					return
+				}
+
 				err := deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1540,6 +1570,12 @@ var _ = Describe("RBD", func() {
 			})
 
 			By("Create ROX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() {
+				if !testNBD {
+					e2elog.Logf("skipping NBD test")
+
+					return
+				}
+
 				err := deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1666,6 +1702,12 @@ var _ = Describe("RBD", func() {
 			})
 
 			By("perform IO on rbd-nbd volume after nodeplugin restart", func() {
+				if !testNBD {
+					e2elog.Logf("skipping NBD test")
+
+					return
+				}
+
 				err := deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1830,6 +1872,12 @@ var _ = Describe("RBD", func() {
 			})
 
 			By("create a PVC and bind it to an app using rbd-nbd mounter with encryption", func() {
+				if !testNBD {
+					e2elog.Logf("skipping NBD test")
+
+					return
+				}
+
 				err := deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -2199,6 +2247,12 @@ var _ = Describe("RBD", func() {
 			By(
 				"create a PVC and Bind it to an app with journaling/exclusive-lock image-features and rbd-nbd mounter",
 				func() {
+					if !testNBD {
+						e2elog.Logf("skipping NBD test")
+
+						return
+					}
+
 					err := deleteResource(rbdExamplePath + "storageclass.yaml")
 					if err != nil {
 						e2elog.Failf("failed to delete storageclass: %v", err)

e2e/utils.go

@@ -85,6 +85,7 @@ var (
 	deployNFS        bool
 	testCephFS       bool
 	testRBD          bool
+	testNBD          bool
 	testNFS          bool
 	helmTest         bool
 	upgradeTesting   bool

scripts/install-helm.sh

@@ -136,7 +136,7 @@ install() {
         mkdir -p ${TEMP}
         # shellcheck disable=SC2021
         dist=$(echo "${dist}" | tr "[A-Z]" "[a-z]")
-        wget "https://get.helm.sh/helm-${HELM_VERSION}-${dist}-${arch}.tar.gz" -O "${TEMP}/helm.tar.gz"
+        wget "https://get.helm.sh/helm-${HELM_VERSION}-${dist}-${arch}.tar.gz" -O "${TEMP}/helm.tar.gz" || exit 1
         tar -C "${TEMP}" -zxvf "${TEMP}/helm.tar.gz"
     fi
     echo "Helm install successful"

scripts/minikube.sh

@@ -22,7 +22,7 @@ function copy_image_to_cluster() {
     if [ -z "$(${CONTAINER_CMD} images -q "${build_image}")" ]; then
         ${CONTAINER_CMD} pull "${build_image}"
     fi
-    if [[ "${VM_DRIVER}" == "none" ]]; then
+    if [[ "${VM_DRIVER}" == "none" ]] || [[ "${VM_DRIVER}" == "podman" ]]; then
         ${CONTAINER_CMD} tag "${build_image}" "${final_image}"
         return
     fi
@@ -139,6 +139,36 @@ function validate_sidecar() {
 done
 }
 
+# install_podman_wrapper creates /usr/bin/podman.wrapper which adds /sys
+# filesystem mount points when a privileged container is started. This makes it
+# possible to map RBD devices in the container that minikube creates when
+# VM_DRIVER=podman is used.
+function install_podman_wrapper() {
+    if [[ -e /usr/bin/podman.wrapper ]]
+    then
+        return
+    fi
+
+    # disabled single quoted check, the script should be created as is
+    # shellcheck disable=SC2016
+    echo '#!/bin/sh
+if [[ "${1}" = run ]]
+then
+    if (echo "${@}" | grep -q privileged)
+    then
+        shift
+        exec /usr/bin/podman.real run -v /sys:/sys:rw -v /dev:/dev:rw --systemd=true "${@}"
+    fi
+fi
+
+exec /usr/bin/podman.real "${@}"
+' > /usr/bin/podman.wrapper
+    chmod +x /usr/bin/podman.wrapper
+
+    mv /usr/bin/podman /usr/bin/podman.real
+    ln -s podman.wrapper /usr/bin/podman
+}
+
 # Storage providers and the default storage class is not needed for Ceph-CSI
 # testing. In order to reduce resources and potential conflicts between storage
 # plugins, disable them.
@@ -185,7 +215,7 @@ K8S_FEATURE_GATES=${K8S_FEATURE_GATES:-""}
 # kubelet.resolv-conf needs to point to a file, not a symlink
 # the default minikube VM has /etc/resolv.conf -> /run/systemd/resolve/resolv.conf
 RESOLV_CONF='/run/systemd/resolve/resolv.conf'
-if [[ "${VM_DRIVER}" == "none" ]] && [[ ! -e "${RESOLV_CONF}" ]]; then
+if { [[ "${VM_DRIVER}" == "none" ]] || [[ "${VM_DRIVER}" == "podman" ]]; } && [[ ! -e "${RESOLV_CONF}" ]]; then
 	# in case /run/systemd/resolve/resolv.conf does not exist, use the
 	# standard /etc/resolv.conf (with symlink resolved)
 	RESOLV_CONF="$(readlink -f /etc/resolv.conf)"
@@ -216,6 +246,8 @@ up)
     if [[ "${VM_DRIVER}" == "none" ]]; then
         mkdir -p "$HOME"/.kube "$HOME"/.minikube
         install_kubectl
+    elif [[ "${VM_DRIVER}" == "podman" ]]; then
+        install_podman_wrapper
     fi
 
     disable_storage_addons
@@ -234,11 +266,14 @@ up)
 
     # create a link so the default dataDirHostPath will work for this
     # environment
-    if [[ "${VM_DRIVER}" != "none" ]]; then
+    if [[ "${VM_DRIVER}" != "none" ]] && [[ "${VM_DRIVER}" != "podman" ]]; then
         wait_for_ssh
         # shellcheck disable=SC2086
         ${minikube} ssh "sudo mkdir -p /mnt/${DISK}/var/lib/rook;sudo ln -s /mnt/${DISK}/var/lib/rook /var/lib/rook"
     fi
+    if [[ "${VM_DRIVER}" = "podman" ]]; then
+        ${minikube} ssh "sudo mount -oremount,rw /sys"
+    fi
     ${minikube} kubectl -- cluster-info
     ;;
 down)