fix(pool-mgmt): Removing stale CVR/CSP when dataset/pool does not exist

cmd/cstor-pool-mgmt/pool/pool.go

@@ -345,13 +345,18 @@ func DeletePool(poolName string) error {
 	deletePoolStr := []string{"destroy", poolName}
 	stdoutStderr, err := RunnerVar.RunCombinedOutput(zpool.PoolOperator, deletePoolStr...)
 	if err != nil {
-		klog.Errorf("Unable to delete pool: %v", string(stdoutStderr))
+		// If pool is missing then do not return error
+		if strings.Contains(string(stdoutStderr), "no such pool") {
+			klog.Infof("Assuming pool deletion successful for error: %v", string(stdoutStderr))
+			return nil
+		}
+		klog.Errorf("Unable to delete pool : %v", string(stdoutStderr))
 		alertlog.Logger.Errorw("",
 			"eventcode", "cstor.pool.delete.failure",
 			"msg", "Failed to delete CStor pool",
 			"rname", poolName,
 		)
-		return err
+		return errors.Wrapf(err, "failed to delete pool.. %s", string(stdoutStderr))
 	}
 	alertlog.Logger.Infow("",
 		"eventcode", "cstor.pool.delete.success",

cmd/cstor-pool-mgmt/volumereplica/volumereplica.go

@@ -448,7 +448,7 @@ func DeleteVolume(fullVolName string) error {
 	stdoutStderr, err := RunnerVar.RunCombinedOutput(VolumeReplicaOperator, deleteVolStr...)
 	if err != nil {
 		// If volume is missing then do not return error
-		if strings.Contains(err.Error(), "dataset does not exist") {
+		if strings.Contains(string(stdoutStderr), "dataset does not exist") {
 			klog.Infof("Assuming volume deletion successful for error: %v", string(stdoutStderr))
 			return nil
 		}
@@ -458,7 +458,7 @@ func DeleteVolume(fullVolName string) error {
 			"msg", "Failed to delete CStor volume",
 			"rname", fullVolName,
 		)
-		return err
+		return errors.Wrapf(err, "failed to delete volume.. %s", string(stdoutStderr))
 	}
 	alertlog.Logger.Infow("",
 		"eventcode", "cstor.volume.delete.success",

tests/admission/admission_test.go

@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
+	cvr "github.com/openebs/maya/pkg/cstor/volumereplica/v1alpha1"
 	pvc "github.com/openebs/maya/pkg/kubernetes/persistentvolumeclaim/v1alpha1"
 	snap "github.com/openebs/maya/pkg/kubernetes/snapshot/v1alpha1"
 	sc "github.com/openebs/maya/pkg/kubernetes/storageclass/v1alpha1"
@@ -131,7 +132,7 @@ var _ = Describe("[cstor] TEST ADMISSION SERVER VALIDATION", func() {
 				nsName,
 			)
 			cvrLabel := pvLabel + pvcObj.Spec.VolumeName
-			cvrCount := ops.GetCstorVolumeReplicaCountEventually(openebsNamespace, cvrLabel, cstor.ReplicaCount)
+			cvrCount := ops.GetCstorVolumeReplicaCountEventually(openebsNamespace, cvrLabel, cstor.ReplicaCount, cvr.IsHealthy())
 			Expect(cvrCount).To(Equal(true), "while checking cstorvolume replica count")
 
 			By("verifying pvc status as bound")

tests/csi/cstor/volume/utils.go

@@ -27,6 +27,7 @@ import (
 	cspc "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1"
 	poolspec "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1/cstorpoolspecs"
 	rgrp "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1/raidgroups"
+	cvr "github.com/openebs/maya/pkg/cstor/volumereplica/v1alpha1"
 	container "github.com/openebs/maya/pkg/kubernetes/container/v1alpha1"
 	deploy "github.com/openebs/maya/pkg/kubernetes/deployment/appsv1/v1alpha1"
 	pvc "github.com/openebs/maya/pkg/kubernetes/persistentvolumeclaim/v1alpha1"
@@ -295,7 +296,7 @@ func verifyTargetPodCount(count int) {
 
 func verifyCstorVolumeReplicaCount(count int) {
 	targetVolumeLabel := pvLabel + pvcObj.Spec.VolumeName
-	isReqCVRCount := ops.GetCstorVolumeReplicaCountEventually(openebsNamespace, targetVolumeLabel, count)
+	isReqCVRCount := ops.GetCstorVolumeReplicaCountEventually(openebsNamespace, targetVolumeLabel, count, cvr.IsHealthy())
 	Expect(isReqCVRCount).To(Equal(true), "while checking cstorvolume replica count")
 }
 

tests/cstor/clone/clone_test.go

@@ -31,6 +31,7 @@ import (
 	spc "github.com/openebs/maya/pkg/storagepoolclaim/v1alpha1"
 	"github.com/openebs/maya/tests/cstor"
 
+	cvr "github.com/openebs/maya/pkg/cstor/volumereplica/v1alpha1"
 	container "github.com/openebs/maya/pkg/kubernetes/container/v1alpha1"
 	volume "github.com/openebs/maya/pkg/kubernetes/volume/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
@@ -138,7 +139,7 @@ var _ = Describe("[cstor] TEST VOLUME CLONE PROVISIONING", func() {
 				nsObj.Name,
 			)
 			cvrLabel := pvLabel + pvcObj.Spec.VolumeName
-			cvrCount := ops.GetCstorVolumeReplicaCountEventually(openebsNamespace, cvrLabel, cstor.ReplicaCount)
+			cvrCount := ops.GetCstorVolumeReplicaCountEventually(openebsNamespace, cvrLabel, cstor.ReplicaCount, cvr.IsHealthy())
 			Expect(cvrCount).To(Equal(true), "while checking cstorvolume replica count")
 
 			By("verifying pvc status as bound")

tests/cstor/node-selector/volume_selector_test.go

@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
+	cvr "github.com/openebs/maya/pkg/cstor/volumereplica/v1alpha1"
 	pvc "github.com/openebs/maya/pkg/kubernetes/persistentvolumeclaim/v1alpha1"
 	sc "github.com/openebs/maya/pkg/kubernetes/storageclass/v1alpha1"
 	spc "github.com/openebs/maya/pkg/storagepoolclaim/v1alpha1"
@@ -132,7 +133,7 @@ var _ = Describe("[cstor] TEST VOLUME PROVISIONING USING TARGET AND REPLICA SELE
 
 				By("verifying cstorvolume replica count")
 				cvrLabel := pvLabel + pvcObj.Spec.VolumeName
-				cvrCount := ops.GetCstorVolumeReplicaCountEventually(openebsNamespace, cvrLabel, cstor.ReplicaCount)
+				cvrCount := ops.GetCstorVolumeReplicaCountEventually(openebsNamespace, cvrLabel, cstor.ReplicaCount, cvr.IsHealthy())
 				Expect(cvrCount).To(Equal(true), "while checking cstorvolume replica count")
 
 				By("verifying pvc status as bound")

tests/cstor/pool/negative/provision_without_pool.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2019 The OpenEBS Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package negative
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/openebs/maya/cmd/cstor-pool-mgmt/pool"
+	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
+	"github.com/openebs/maya/tests"
+	"github.com/openebs/maya/tests/cstor"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	types "k8s.io/apimachinery/pkg/types"
+)
+
+var _ = Describe("[cstor] [-ve] TEST PROVISION WITHOUT POOL", func() {
+	var (
+		err error
+	)
+
+	BeforeEach(func() {
+		By("creating storagepoolclaim")
+		spcConfig := &tests.SPCConfig{
+			Name:               spcName,
+			DiskType:           "sparse",
+			PoolCount:          cstor.PoolCount,
+			IsOverProvisioning: false,
+			PoolType:           "striped",
+		}
+		ops.Config = spcConfig
+		spcObj = ops.BuildAndCreateSPC()
+		By("Creating SPC, Desired Number of CSP Should Be Created", func() {
+			ops.VerifyDesiredCSPCount(spcObj, cstor.PoolCount)
+		})
+	})
+
+	AfterEach(func() {
+		By("deleting storagepoolclaim")
+
+		err = ops.SPCClient.Delete(spcObj.Name, &metav1.DeleteOptions{})
+		Expect(err).To(BeNil(), "while deleting the storagepoolclaim {%s}", spcObj.Name)
+
+		cspCount := ops.GetCSPCount(getLabelSelector(spcObj))
+		Expect(cspCount).To(Equal(0), "stale CSP")
+	})
+
+	When("creating storagepoolclaim without", func() {
+		It("Testing SPC deletion if pool does not exist", func() {
+			spcLabel := getLabelSelector(spcObj)
+			poolPodList, err := ops.PodClient.
+				WithNamespace(openebsNamespace).
+				List(metav1.ListOptions{LabelSelector: spcLabel})
+			Expect(err).To(BeNil())
+			Expect(len(poolPodList.Items)).Should(BeNumerically(">=", 1))
+
+			tPod := poolPodList.Items[0]
+			var puid string
+			for _, e := range tPod.Spec.Containers[0].Env {
+				if e.Name == "OPENEBS_IO_CSTOR_ID" {
+					puid = e.Value
+				}
+			}
+			Expect(len(puid)).Should(BeNumerically(">=", 1))
+
+			cspList, err := ops.CSPClient.List(metav1.ListOptions{LabelSelector: string(apis.StoragePoolClaimCPK) + "=" + spcObj.Name})
+			Expect(err).To(BeNil())
+			Expect(len(cspList.Items)).Should(BeNumerically(">=", 1))
+			var cspObj apis.CStorPool
+			for _, l := range cspList.Items {
+				if l.GetUID() == types.UID(puid) {
+					cspObj = l
+					break
+				}
+			}
+
+			poolName := string(pool.PoolPrefix) + string(cspObj.ObjectMeta.UID)
+			zfsCmd := "zpool destroy -f " + poolName
+			cmd := []string{"/bin/bash", "-c", zfsCmd}
+
+			opts := tests.NewOptions().
+				WithPodName(tPod.Name).
+				WithNamespace(openebsNamespace).
+				WithContainer("cstor-pool").
+				WithCommand(cmd...)
+
+			out, err := ops.ExecPod(opts)
+			Expect(err).To(BeNil())
+			Expect(len(out)).Should(BeNumerically("==", 0))
+		})
+	})
+})
+
+func getLabelSelector(spc *apis.StoragePoolClaim) string {
+	return string(apis.StoragePoolClaimCPK) + "=" + spc.Name
+}

tests/cstor/snapshot/snapshot_test.go

@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
+	cvr "github.com/openebs/maya/pkg/cstor/volumereplica/v1alpha1"
 	pvc "github.com/openebs/maya/pkg/kubernetes/persistentvolumeclaim/v1alpha1"
 	snap "github.com/openebs/maya/pkg/kubernetes/snapshot/v1alpha1"
 	sc "github.com/openebs/maya/pkg/kubernetes/storageclass/v1alpha1"
@@ -131,7 +132,7 @@ var _ = Describe("[cstor] TEST SNAPSHOT PROVISIONING", func() {
 				nsObj.Name,
 			)
 			cvrLabel := pvLabel + pvcObj.Spec.VolumeName
-			cvrCount := ops.GetCstorVolumeReplicaCountEventually(openebsNamespace, cvrLabel, cstor.ReplicaCount)
+			cvrCount := ops.GetCstorVolumeReplicaCountEventually(openebsNamespace, cvrLabel, cstor.ReplicaCount, cvr.IsHealthy())
 			Expect(cvrCount).To(Equal(true), "while checking cstorvolume replica count")
 
 			By("verifying pvc status as bound")