From 051c223b8881b89e9a43eb0577122f52ab0e911b Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Mon, 18 Nov 2019 17:59:12 +0530 Subject: [PATCH 1/5] chore(cStorVolume_bdd): add negative test cases for cStorVolumeProvisioning This PR does the following changes 1. This PR provides the capability to get debug build for the cstor-pool-mgmt. 2. Debug build has an injection server running on port 8080 on all the cstor-pool-mgmgt which can be used to inject the errors for CR's and ZFS operations for negative path testing. **Perform the following steps to run the negative tests on your local machine:** 1. Navigate to tests/cstor/volume using cd. 2. In your command line run the following command to execute negative BDD CMD: ginkgo -v -focus="\[Cstor Volume Provisioning Negative\]" -- -kubeconfig=/home/sai/.kube/config -cstor-maxpools=3 -cstor-replicas=3 Signed-off-by: mittachaitu --- GNUmakefile | 13 ++ .../pool-controller/run_pool_controller.go | 5 + .../controller/replica-controller/handler.go | 4 + .../volumereplica/volumereplica.go | 13 ++ pkg/debug/debug.go | 49 ++++- pkg/debug/release.go | 42 ++++ pkg/debug/types.go | 113 +++++++++-- pkg/kubernetes/service/v1alpha1/build.go | 15 ++ tests/cstor/volume/provision_bulk_test.go | 2 +- tests/cstor/volume/provision_negative_test.go | 186 ++++++++++++++++++ tests/cstor/volume/provision_test.go | 3 +- tests/cstor/volume/suite_test.go | 7 + tests/operations.go | 29 +++ 13 files changed, 461 insertions(+), 20 deletions(-) create mode 100644 tests/cstor/volume/provision_negative_test.go diff --git a/GNUmakefile b/GNUmakefile index b8c4d310b5..1279aedc7d 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -108,6 +108,7 @@ VOLUME_MGMT=cstor-volume-mgmt EXPORTER=maya-exporter CSPC_OPERATOR=cspc-operator CSPC_OPERATOR_DEBUG=cspc-operator-debug +CSP_OPERATOR_DEBUG=cstor-pool-mgmt-debug # Specify the date o build @@ -311,6 +312,18 @@ pool-mgmt-image: cstor-pool-mgmt @cd buildscripts/cstor-pool-mgmt && sudo docker build -t ${HUB_USER}/${CSTOR_POOL_MGMT_REPO_NAME}:${IMAGE_TAG} --build-arg BASE_IMAGE=${CSTOR_BASE_IMAGE} --build-arg BUILD_DATE=${BUILD_DATE} . --no-cache @rm buildscripts/cstor-pool-mgmt/${POOL_MGMT} +#Use this to build debug image of cstor-pool-mgmt +.PHONY: pool-mgmt-debug-image +pool-mgmt-debug-image: + @echo "----------------------------" + @echo -n "--> cstor-pool-mgmt debug image " + @echo "${HUB_USER}/${CSTOR_POOL_MGMT_REPO_NAME}:inject" + @echo "----------------------------" + @PNAME=${CSP_OPERATOR_DEBUG} CTLNAME=${POOL_MGMT} BUILD_TAG="-tags=debug" sh -c "'$(PWD)/buildscripts/build.sh'" + @cp bin/${CSP_OPERATOR_DEBUG}/${POOL_MGMT} buildscripts/${CSP_OPERATOR_DEBUG}/ + @cd buildscripts/${CSP_OPERATOR_DEBUG} && sudo docker build -t ${HUB_USER}/${CSTOR_POOL_MGMT_REPO_NAME}:inject --build-arg BASE_IMAGE=${CSTOR_BASE_IMAGE} --build-arg BUILD_DATE=${BUILD_DATE} . --no-cache + @rm buildscripts/${CSP_OPERATOR_DEBUG}/${POOL_MGMT} + #Use this to build cspi-mgmt .PHONY: cspi-mgmt cspi-mgmt: diff --git a/cmd/cstor-pool-mgmt/controller/pool-controller/run_pool_controller.go b/cmd/cstor-pool-mgmt/controller/pool-controller/run_pool_controller.go index 6fb2c0d539..f130fc2a03 100644 --- a/cmd/cstor-pool-mgmt/controller/pool-controller/run_pool_controller.go +++ b/cmd/cstor-pool-mgmt/controller/pool-controller/run_pool_controller.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/openebs/maya/cmd/cstor-pool-mgmt/controller/common" + "github.com/openebs/maya/pkg/debug" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" @@ -34,6 +35,10 @@ func (c *CStorPoolController) Run(threadiness int, stopCh <-chan struct{}) error defer runtime.HandleCrash() defer c.workqueue.ShutDown() + // Start server only in debug build + debug.LogBuildDetails() + debug.StartInjectionServer() + // Start the informer factories to begin populating the informer caches klog.Info("Starting CStorPool controller") diff --git a/cmd/cstor-pool-mgmt/controller/replica-controller/handler.go b/cmd/cstor-pool-mgmt/controller/replica-controller/handler.go index 657fe2b231..78fda2201a 100644 --- a/cmd/cstor-pool-mgmt/controller/replica-controller/handler.go +++ b/cmd/cstor-pool-mgmt/controller/replica-controller/handler.go @@ -27,6 +27,7 @@ import ( "github.com/openebs/maya/cmd/cstor-pool-mgmt/volumereplica" apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" clientset "github.com/openebs/maya/pkg/client/generated/clientset/versioned" + "github.com/openebs/maya/pkg/debug" merrors "github.com/openebs/maya/pkg/errors/v1alpha1" pkg_errors "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -146,6 +147,9 @@ func (c *CStorVolumeReplicaController) syncHandler( // need to update cvr before returning this error if err != nil { + if debug.EI.IsCVRUpdateErrorInjected() { + return merrors.Errorf("CVR update error via injection") + } _, err1 := c.clientset. OpenebsV1alpha1(). CStorVolumeReplicas(cvrGot.Namespace). diff --git a/cmd/cstor-pool-mgmt/volumereplica/volumereplica.go b/cmd/cstor-pool-mgmt/volumereplica/volumereplica.go index e152851473..edb60ad812 100644 --- a/cmd/cstor-pool-mgmt/volumereplica/volumereplica.go +++ b/cmd/cstor-pool-mgmt/volumereplica/volumereplica.go @@ -27,6 +27,7 @@ import ( "encoding/json" apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" + "github.com/openebs/maya/pkg/debug" "github.com/openebs/maya/pkg/hash" "github.com/openebs/maya/pkg/util" zfs "github.com/openebs/maya/pkg/zfs/cmd/v1alpha1" @@ -183,6 +184,10 @@ func CreateVolumeReplica(cStorVolumeReplica *apis.CStorVolumeReplica, fullVolNam isClone := cStorVolumeReplica.Labels[string(apis.CloneEnableKEY)] == "true" snapName := "" + if debug.EI.IsZFSCreateErrorInjected() { + return errors.New("ZFS create error via injection") + } + if isClone { srcVolume := cStorVolumeReplica.Annotations[string(apis.SourceVolumeKey)] snapName = cStorVolumeReplica.Annotations[string(apis.SnapshotNameKey)] @@ -413,6 +418,10 @@ func buildVolumeRestoreCommand(poolName, fullVolName, restoreSrc string) []strin // GetVolumes returns the slice of volumes. func GetVolumes() ([]string, error) { + if debug.EI.IsZFSGetErrorInjected() { + return []string{}, errors.New("ZFS get error via injection") + } + volStrCmd := []string{"get", "-Hp", "name", "-o", "name"} volnameByte, err := RunnerVar.RunStdoutPipe(VolumeReplicaOperator, volStrCmd...) if err != nil || string(volnameByte) == "" { @@ -431,6 +440,10 @@ func GetVolumes() ([]string, error) { // DeleteVolume deletes the specified volume. func DeleteVolume(fullVolName string) error { + if debug.EI.IsZFSDeleteErrorInjected() { + return errors.New("ZFS delete error via injection") + } + deleteVolStr := []string{"destroy", "-R", fullVolName} stdoutStderr, err := RunnerVar.RunCombinedOutput(VolumeReplicaOperator, deleteVolStr...) if err != nil { diff --git a/pkg/debug/debug.go b/pkg/debug/debug.go index 717d033145..8332760876 100644 --- a/pkg/debug/debug.go +++ b/pkg/debug/debug.go @@ -22,10 +22,11 @@ import ( "encoding/json" "fmt" "io/ioutil" - "k8s.io/klog" "log" "net/http" "time" + + "k8s.io/klog" ) // LogBuildDetails logs the build details when the cspc-operator starts. @@ -287,3 +288,49 @@ func (ei *ErrorInjection) IsDeploymentPatchErrorInjected() bool { } return false } + +// IsZFSGetErrorInjected returns true if error is injected for ZFS get command +func (ei *ErrorInjection) IsZFSGetErrorInjected() bool { + if ei.ZFSError.CRUDErrorInjection.InjectGetError == Inject { + return true + } + return false +} + +// IsZFSDeleteErrorInjected returns true if error is injected for ZFS delete command +func (ei *ErrorInjection) IsZFSDeleteErrorInjected() bool { + if ei.ZFSError.CRUDErrorInjection.InjectDeleteError == Inject { + return true + } + return false +} + +// IsZFSCreateErrorInjected returns true if error is injected for ZFS create command +func (ei *ErrorInjection) IsZFSCreateErrorInjected() bool { + if ei.ZFSError.CRUDErrorInjection.InjectCreateError == Inject { + return true + } + return false +} + +// IsCVRCreateErrorInjected returns true if error is injected for CVR create +// command +func (ei *ErrorInjection) IsCVRCreateErrorInjected() bool { + return false +} + +// IsCVRDeleteErrorInjected returns true if error is injected for CVR delete +// command +func (ei *ErrorInjection) IsCVRDeleteErrorInjected() bool { + return false +} + +// IsCVRGetErrorInjected returns true if error is injected for CVR get command +func (ei *ErrorInjection) IsCVRGetErrorInjected() bool { + return false +} + +// IsCVRUpdateErrorInjected returns true if error is injected for CVR update command +func (ei *ErrorInjection) IsCVRUpdateErrorInjected() bool { + return false +} diff --git a/pkg/debug/release.go b/pkg/debug/release.go index 4b2ec6ab02..8225ff28c5 100644 --- a/pkg/debug/release.go +++ b/pkg/debug/release.go @@ -128,3 +128,45 @@ func (ei *ErrorInjection) IsDeploymentUpdateErrorInjected() bool { func (ei *ErrorInjection) IsDeploymentPatchErrorInjected() bool { return false } + +// IsZFSGetErrorInjected is production alternative for the same function that +// exists in debug build +func (ei *ErrorInjection) IsZFSGetErrorInjected() bool { + return false +} + +// IsZFSCreateErrorInjected is production alternative for the same function that +// exists in debug build +func (ei *ErrorInjection) IsZFSCreateErrorInjected() bool { + return false +} + +// IsZFSDeleteErrorInjected is production alternative for the same function that +// exists in debug build +func (ei *ErrorInjection) IsZFSDeleteErrorInjected() bool { + return false +} + +// IsCVRCreateErrorInjected is production alternative for the same function that +// exists in debug build +func (ei *ErrorInjection) IsCVRCreateErrorInjected() bool { + return false +} + +// IsCVRDeleteErrorInjected is production alternative for the same function that +// exists in debug build +func (ei *ErrorInjection) IsCVRDeleteErrorInjected() bool { + return false +} + +// IsCVRGetErrorInjected is production alternative for the same function that +// exists in debug build +func (ei *ErrorInjection) IsCVRGetErrorInjected() bool { + return false +} + +// IsCVRUpdateErrorInjected is production alternative for the same function that +// exists in debug build +func (ei *ErrorInjection) IsCVRUpdateErrorInjected() bool { + return false +} diff --git a/pkg/debug/types.go b/pkg/debug/types.go index 3974d0ddb1..39e2d2b7e1 100644 --- a/pkg/debug/types.go +++ b/pkg/debug/types.go @@ -41,38 +41,54 @@ var EI = &ErrorInjection{} // ErrorInjection schema to inject errors. type ErrorInjection struct { - CSPIError CSPIErrorInjection `json:"cspi_error"` - CSPCError CSPCErrorInjection `json:"cspc_error"` - DeploymentError DeploymentErrorInjection `json:"deployment_error"` + CSPIError CSPIErrorInjection `json:"cspiError"` + CSPCError CSPCErrorInjection `json:"cspcError"` + DeploymentError DeploymentErrorInjection `json:"deploymentError"` + ZFSError ZFSErrorInjection `json:"zfsError"` + CVRError CVRErrorInjection `json:"cvrError"` } // CSPIErrorInjection is used to inject errors for CSPI related operations. type CSPIErrorInjection struct { - CRUDErrorInjection CRUDErrorInjection `json:"crud_error_injection"` - ErrorPercentage ErrorPercentageThreshold `json:"error_percentage"` + CRUDErrorInjection CRUDErrorInjection `json:"crudErrorInjection"` + ErrorPercentage ErrorPercentageThreshold `json:"errorPercentage"` } // CSPCErrorInjection is used to inject errors for CSPC related operations. type CSPCErrorInjection struct { - CRUDErrorInjection CRUDErrorInjection `json:"crud_error_injection"` - ErrorPercentage ErrorPercentageThreshold `json:"error_percentage"` + CRUDErrorInjection CRUDErrorInjection `json:"crudErrorInjection"` + ErrorPercentage ErrorPercentageThreshold `json:"errorPercentage"` } // DeploymentErrorInjection is used to inject errors for CSPC related operations. type DeploymentErrorInjection struct { - CRUDErrorInjection CRUDErrorInjection `json:"crud_error_injection"` - ErrorPercentage ErrorPercentageThreshold `json:"error_percentage"` + CRUDErrorInjection CRUDErrorInjection `json:"crudErrorInjection"` + ErrorPercentage ErrorPercentageThreshold `json:"errorPercentage"` +} + +// ZFSErrorInjection is used to inject errors for Volume Replica related +// operations. +type ZFSErrorInjection struct { + CRUDErrorInjection CRUDErrorInjection `json:"crudErrorInjection"` + ErrorPercentage ErrorPercentageThreshold `json:"errorPercentage"` +} + +// CVRErrorInjection is used to inject errors in API calls for Volume Replica +// related operations +type CVRErrorInjection struct { + CRUDErrorInjection CRUDErrorInjection `json:"crudErrorInjection"` + ErrorPercentage ErrorPercentageThreshold `json:"errorPercentage"` } // CRUDErrorInjection is used to inject CRUD errors. type CRUDErrorInjection struct { - InjectDeleteCollectionError string `json:"inject_delete_collection_error"` - InjectDeleteError string `json:"inject_delete_error"` - InjectListError string `json:"inject_list_error"` - InjectGetError string `json:"inject_get_error"` - InjectCreateError string `json:"inject_create_error"` - InjectUpdateError string `json:"inject_update_error"` - InjectPatchError string `json:"inject_patch_error"` + InjectDeleteCollectionError string `json:"injectDeleteCollectionError"` + InjectDeleteError string `json:"injectDeleteError"` + InjectListError string `json:"injectListError"` + InjectGetError string `json:"injectGetError"` + InjectCreateError string `json:"injectCreateError"` + InjectUpdateError string `json:"injectUpdateError"` + InjectPatchError string `json:"injectPatchError"` } // ErrorPercentageThreshold is the threshold value above which the error will not be injected. @@ -139,6 +155,29 @@ func (ei *ErrorInjection) WithDeploymentThreshold(threshold int) *ErrorInjection return ei } +// WithZFSThreshold injects ZFS errors depending on passed error threshold value +func (ei *ErrorInjection) WithZFSThreshold(threshold int) *ErrorInjection { + ei.ZFSError.ErrorPercentage.Threshold = threshold + if GetRandomErrorPercentage() > threshold { + ei.WithZFSCreateError(Inject). + WithZFSDeleteError(Inject). + WithZFSGetError(Inject) + } + return ei +} + +// WithCVRThreshold injects CVR error depending on passed error threshold value +func (ei *ErrorInjection) WithCVRThreshold(threshold int) *ErrorInjection { + ei.CVRError.ErrorPercentage.Threshold = threshold + if GetRandomErrorPercentage() > threshold { + ei.WithCVRCreateError(Inject). + WithCVRDeleteError(Inject). + WithCVRGetError(Inject). + WithCVRUpdateError(Inject) + } + return ei +} + // WithCSPCDeleteCollectionError injects/ejects CSPC delete collection error. func (ei *ErrorInjection) WithCSPCDeleteCollectionError(ejectOrInject string) *ErrorInjection { ei.CSPCError.CRUDErrorInjection.InjectDeleteCollectionError = ejectOrInject @@ -264,3 +303,45 @@ func (ei *ErrorInjection) WithDeploymentPatchError(ejectOrInject string) *ErrorI ei.DeploymentError.CRUDErrorInjection.InjectPatchError = ejectOrInject return ei } + +// WithZFSGetError injects/ejects ZFS get error. +func (ei *ErrorInjection) WithZFSGetError(ejectOrInject string) *ErrorInjection { + ei.ZFSError.CRUDErrorInjection.InjectGetError = ejectOrInject + return ei +} + +// WithZFSCreateError injects/ejects ZFS create error. +func (ei *ErrorInjection) WithZFSCreateError(ejectOrInject string) *ErrorInjection { + ei.ZFSError.CRUDErrorInjection.InjectCreateError = ejectOrInject + return ei +} + +// WithZFSDeleteError injects/ejects ZFS delete error. +func (ei *ErrorInjection) WithZFSDeleteError(ejectOrInject string) *ErrorInjection { + ei.ZFSError.CRUDErrorInjection.InjectDeleteError = ejectOrInject + return ei +} + +// WithCVRCreateError injects/ejects CVR create error. +func (ei *ErrorInjection) WithCVRCreateError(ejectOrInject string) *ErrorInjection { + ei.CVRError.CRUDErrorInjection.InjectCreateError = ejectOrInject + return ei +} + +// WithCVRGetError injects/ejects CVR get error. +func (ei *ErrorInjection) WithCVRGetError(ejectOrInject string) *ErrorInjection { + ei.CVRError.CRUDErrorInjection.InjectGetError = ejectOrInject + return ei +} + +// WithCVRDeleteError injects/ejects CVR delete error. +func (ei *ErrorInjection) WithCVRDeleteError(ejectOrInject string) *ErrorInjection { + ei.CVRError.CRUDErrorInjection.InjectDeleteError = ejectOrInject + return ei +} + +// WithCVRUpdateError injects/ejects CVR update error. +func (ei *ErrorInjection) WithCVRUpdateError(ejectOrInject string) *ErrorInjection { + ei.CVRError.CRUDErrorInjection.InjectUpdateError = ejectOrInject + return ei +} diff --git a/pkg/kubernetes/service/v1alpha1/build.go b/pkg/kubernetes/service/v1alpha1/build.go index 008c5bb3c1..215b76f582 100644 --- a/pkg/kubernetes/service/v1alpha1/build.go +++ b/pkg/kubernetes/service/v1alpha1/build.go @@ -240,6 +240,21 @@ func (b *Builder) WithPorts(ports []corev1.ServicePort) *Builder { return b } +// WithType sets the Type field of Service with provided arguments +func (b *Builder) WithType(svcType corev1.ServiceType) *Builder { + if len(svcType) == 0 { + b.errs = append( + b.errs, + errors. + New("failed to build service object: missing service type"), + ) + return b + } + + b.service.object.Spec.Type = svcType + return b +} + // Build returns the Service API instance func (b *Builder) Build() (*corev1.Service, error) { if len(b.errs) > 0 { diff --git a/tests/cstor/volume/provision_bulk_test.go b/tests/cstor/volume/provision_bulk_test.go index 55d2690cdf..9e4f4eea7e 100644 --- a/tests/cstor/volume/provision_bulk_test.go +++ b/tests/cstor/volume/provision_bulk_test.go @@ -32,7 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var _ = Describe("[cstor] TEST BULK VOLUME PROVISIONING", func() { +var _ = Describe("[Cstor Volume Provisioning Positive] TEST BULK VOLUME PROVISIONING", func() { var ( err error pvcNamePrefix = "cstor-volume-claim-" diff --git a/tests/cstor/volume/provision_negative_test.go b/tests/cstor/volume/provision_negative_test.go new file mode 100644 index 0000000000..fd2a83c089 --- /dev/null +++ b/tests/cstor/volume/provision_negative_test.go @@ -0,0 +1,186 @@ +package volume + +import ( + "strconv" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" + "github.com/openebs/maya/pkg/debug" + sc "github.com/openebs/maya/pkg/kubernetes/storageclass/v1alpha1" + "github.com/openebs/maya/tests" + "github.com/openebs/maya/tests/cstor" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + // auth plugins + + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" +) + +var _ = Describe("[Cstor Volume Provisioning Negative] Volume Provisioning", func() { + When("SPC is created", func() { + It("cStor Pools Should be Provisioned ", func() { + By("Build And Create StoragePoolClaim object") + // Populate configurations and create + spcConfig := &tests.SPCConfig{ + Name: spcName, + DiskType: "sparse", + PoolCount: cstor.PoolCount, + IsOverProvisioning: false, + PoolType: "striped", + } + ops.Config = spcConfig + spcObj = ops.BuildAndCreateSPC() + By("Creating SPC, Desired Number of CSP Should Be Created", func() { + ops.VerifyDesiredCSPCount(spcObj, cstor.PoolCount) + }) + By("Build And Create StorageClass", buildAndCreateSC) + }) + }) + When("Service is Applied", func() { + It("Should Create Service", func() { + var nodeIP string + var err error + By("Creating Service To Inject Errors During Volume Provisioning") + spcLabel := SPCLabel + "=" + spcObj.Name + poolPodList, err = ops.PodClient. + WithNamespace(openebsNamespace). + List(metav1.ListOptions{LabelSelector: spcLabel}) + Expect(err).To(BeNil()) + + Expect(len(poolPodList.Items)).Should(BeNumerically(">=", 1), + "Pool Pods are not yet created") + servicePort := []corev1.ServicePort{ + corev1.ServicePort{ + Name: "http", + Port: int32(8080), + Protocol: "TCP", + NodePort: int32(30031), + }, + } + nodeName := poolPodList.Items[0].Spec.NodeSelector[hostLabel] + nodeObj, err := ops.NodeClient.Get(nodeName, metav1.GetOptions{}) + Expect(err).To(BeNil()) + + //GetNode Ip + for _, address := range nodeObj.Status.Addresses { + if address.Type == corev1.NodeExternalIP { + nodeIP = address.Address + break + } + } + Expect(nodeIP).NotTo(BeEmpty()) + hostIpPort = nodeIP + ":30031" + serviceConfig := &tests.ServiceConfig{ + Name: svcName, + Namespace: openebsNamespace, + Selectors: poolPodList.Items[0].Labels, + ServicePort: servicePort, + } + ops.Config = serviceConfig + serviceObj = ops.BuildAndCreateService() + }) + }) + + When("PersistentVolumeClaim Is Created", func() { + It("Volume Should be Created and Provisioned", func() { + // Populate PVC configurations + pvcConfig := &tests.PVCConfig{ + Name: pvcName, + Namespace: nsObj.Name, + SCName: scObj.Name, + Capacity: "5G", + AccessModes: accessModes, + } + ops.Config = pvcConfig + + //Injecting Errors During CVRUpdate and ZFS Creation Time + injectError := debug.NewClient(hostIpPort) + err := injectError.PostInject( + debug.NewErrorInjection(). + WithZFSCreateError(debug.Inject). + WithCVRUpdateError(debug.Inject)) + Expect(err).To(BeNil()) + + pvcObj = ops.BuildAndCreatePVC() + // n-1 Replicas should be Healthy + By("Creating PVC, Desired Number of CVR Should Be Created", func() { + ops.VerifyVolumeStatus(pvcObj, cstor.ReplicaCount-1) + }) + + // GetLatest PVC object + pvcObj, err = ops.PVCClient. + WithNamespace(nsObj.Name). + Get(pvcObj.Name, metav1.GetOptions{}) + Expect(err).To(BeNil()) + }) + }) + + When("PVC Created During Error Injection", func() { + It("All Volume Replicas Should Not Be Healthy", func() { + By("Verify Volume Status after Injecting Errors", func() { + command := "zfs get guid | grep " + pvcObj.Spec.VolumeName + " | awk '{print $3}'" + _ = ops.ExecuteCMDEventually( + &poolPodList.Items[0], + "cstor-pool-mgmt", command, false) + + // Eject the ZFSCreate error + injectError := debug.NewClient(hostIpPort) + err := injectError.PostInject( + debug.NewErrorInjection(). + WithZFSCreateError(debug.Eject). + WithCVRUpdateError(debug.Inject)) + Expect(err).To(BeNil()) + + //After ejecting the ZFS Create volume dataset should be created + zvolGUID := ops.ExecuteCMDEventually( + &poolPodList.Items[0], + "cstor-pool-mgmt", command, true) + Expect(zvolGUID).NotTo(BeEmpty()) + // CVR Update will error due to error injection. So replica count + // should be n-1 + ops.VerifyVolumeStatus(pvcObj, cstor.ReplicaCount-1) + + //Eject CVRUpdate error then CVR should become Healthy + err = injectError.PostInject( + debug.NewErrorInjection(). + WithCVRUpdateError(debug.Eject)) + Expect(err).To(BeNil()) + + ops.VerifyVolumeStatus(pvcObj, cstor.ReplicaCount) + }) + }) + }) + + When("CleanUp Negative Volume Provisioned Resources", func() { + It("Should Delete All The Resources Related To Test", func() { + By("Delete persistentVolumeClaim then volume resources should be deleted", func() { + ops.DeleteVolumeResources(pvcObj, scObj) + }) + By("Delete StoragePoolClaim then pool related resources should be deleted", func() { + ops.DeleteStoragePoolClaim(spcObj.Name) + }) + }) + }) +}) + +func buildAndCreateSC() { + var err error + casConfig := strings.Replace( + openebsCASConfigValue, "$spcName", spcObj.Name, 1) + casConfig = strings.Replace( + casConfig, "$count", strconv.Itoa(cstor.ReplicaCount), 1) + annotations[string(apis.CASTypeKey)] = string(apis.CstorVolume) + annotations[string(apis.CASConfigKey)] = casConfig + scObj, err = sc.NewBuilder(). + WithGenerateName(scName). + WithAnnotations(annotations). + WithProvisioner(openebsProvisioner).Build() + Expect(err).ShouldNot(HaveOccurred(), "while building storageclass obj for storageclass {%s}", scName) + + By("creating storageclass") + scObj, err = ops.SCClient.Create(scObj) + Expect(err).To(BeNil(), "while creating storageclass", scName) +} diff --git a/tests/cstor/volume/provision_test.go b/tests/cstor/volume/provision_test.go index 56d7e18fa0..2fb3524f92 100644 --- a/tests/cstor/volume/provision_test.go +++ b/tests/cstor/volume/provision_test.go @@ -31,7 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var _ = Describe("[cstor] TEST VOLUME PROVISIONING", func() { +var _ = Describe("[Cstor Volume Provisioning Positive] TEST VOLUME PROVISIONING", func() { var ( err error pvcName = "cstor-volume-claim" @@ -189,5 +189,4 @@ var _ = Describe("[cstor] TEST VOLUME PROVISIONING", func() { Expect(cvCount).To(Equal(true), "while checking cstorvolume count") }) }) - }) diff --git a/tests/cstor/volume/suite_test.go b/tests/cstor/volume/suite_test.go index 881b1ef301..dbf1841d2e 100644 --- a/tests/cstor/volume/suite_test.go +++ b/tests/cstor/volume/suite_test.go @@ -52,9 +52,16 @@ var ( targetLabel = "openebs.io/target=cstor-target" pvLabel = "openebs.io/persistent-volume=" pvcLabel = "openebs.io/persistent-volume-claim=" + SPCLabel = "openebs.io/storage-pool-claim" accessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} capacity = "5G" annotations = map[string]string{} + svcName = "volume-provision-svc-injector" + serviceObj *corev1.Service + pvcName = "volume-claim1" + hostIpPort string + hostLabel = "kubernetes.io/hostname" + poolPodList *corev1.PodList ) func TestSource(t *testing.T) { diff --git a/tests/operations.go b/tests/operations.go index 94bbdd8c2f..09cadf18b4 100644 --- a/tests/operations.go +++ b/tests/operations.go @@ -133,6 +133,14 @@ type CVRConfig struct { ReplicaID string } +// ServiceConfig provides config to create Service +type ServiceConfig struct { + Name string + Namespace string + Selectors map[string]string + ServicePort []corev1.ServicePort +} + var ( pvLabel = "openebs.io/persistent-volume=" poolLabel = "openebs.io/storagepoolclaim=" @@ -253,6 +261,9 @@ func (ops *Operations) withDefaults() { if ops.BDCClient == nil { ops.BDCClient = bdc.NewKubeClient(bdc.WithKubeConfigPath(ops.KubeConfigPath)) } + if ops.SVCClient == nil { + ops.SVCClient = svc.NewKubeClient(svc.WithKubeConfigPath(ops.KubeConfigPath)) + } } // VerifyOpenebs verify running state of required openebs control plane components @@ -1140,6 +1151,24 @@ func (ops *Operations) BuildAndCreateCVR() *apis.CStorVolumeReplica { return cvrObj } +// BuildAndCreateService builds and creates Service in cluster +func (ops *Operations) BuildAndCreateService() *corev1.Service { + svcConfig := ops.Config.(*ServiceConfig) + buildSVCObj, err := svc.NewBuilder(). + WithGenerateName(svcConfig.Name). + WithNamespace(svcConfig.Namespace). + WithSelectorsNew(svcConfig.Selectors). + WithPorts(svcConfig.ServicePort). + WithType(corev1.ServiceTypeNodePort). + Build() + Expect(err).To(BeNil()) + svcObj, err := ops.SVCClient. + WithNamespace(svcConfig.Namespace). + Create(buildSVCObj) + Expect(err).To(BeNil()) + return svcObj +} + // DeletePersistentVolumeClaim deletes PVC from cluster based on provided // argument func (ops *Operations) DeletePersistentVolumeClaim(name, namespace string) { From e505911eed057b9d62df9e6532aa2b71bdf30e78 Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Mon, 18 Nov 2019 18:45:13 +0530 Subject: [PATCH 2/5] This commit deletes the service created during deleting the test related resources Signed-off-by: mittachaitu --- tests/cstor/volume/provision_negative_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/cstor/volume/provision_negative_test.go b/tests/cstor/volume/provision_negative_test.go index fd2a83c089..e1098c8b9f 100644 --- a/tests/cstor/volume/provision_negative_test.go +++ b/tests/cstor/volume/provision_negative_test.go @@ -156,6 +156,8 @@ var _ = Describe("[Cstor Volume Provisioning Negative] Volume Provisioning", fun When("CleanUp Negative Volume Provisioned Resources", func() { It("Should Delete All The Resources Related To Test", func() { + err := ops.SVCClient.Delete(serviceObj.Name, &metav1.DeleteOptions{}) + Expect(err).To(BeNil()) By("Delete persistentVolumeClaim then volume resources should be deleted", func() { ops.DeleteVolumeResources(pvcObj, scObj) }) From fa976ce18fcc3e97b24d32141cc93a15c4716f07 Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Mon, 18 Nov 2019 19:04:24 +0530 Subject: [PATCH 3/5] This commit adds the license header to new file Signed-off-by: mittachaitu --- tests/cstor/volume/provision_negative_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/cstor/volume/provision_negative_test.go b/tests/cstor/volume/provision_negative_test.go index e1098c8b9f..6dae7ced0d 100644 --- a/tests/cstor/volume/provision_negative_test.go +++ b/tests/cstor/volume/provision_negative_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 The OpenEBS Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package volume import ( From 01db15ea9c5a7d0988227345498be359db8aace9 Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Mon, 18 Nov 2019 19:12:40 +0530 Subject: [PATCH 4/5] This commit adds docker files related to the cstor-pool-mgmgt debug build Signed-off-by: mittachaitu --- buildscripts/cstor-pool-mgmt-debug/Dockerfile | 30 +++++++++++++++++++ .../cstor-pool-mgmt-debug/entrypoint.sh | 8 +++++ 2 files changed, 38 insertions(+) create mode 100644 buildscripts/cstor-pool-mgmt-debug/Dockerfile create mode 100644 buildscripts/cstor-pool-mgmt-debug/entrypoint.sh diff --git a/buildscripts/cstor-pool-mgmt-debug/Dockerfile b/buildscripts/cstor-pool-mgmt-debug/Dockerfile new file mode 100644 index 0000000000..23d2b986a1 --- /dev/null +++ b/buildscripts/cstor-pool-mgmt-debug/Dockerfile @@ -0,0 +1,30 @@ +# +# This Dockerfile builds a recent cstor-pool-mgmt-debug using the latest binary from +# cstor-pool-mgmt releases. +# + +#openebs/cstor-base is the image that contains cstor related binaries and +#libraries - zpool, zfs, zrepl +#FROM openebs/cstor-base:ci +ARG BASE_IMAGE +FROM $BASE_IMAGE + +COPY cstor-pool-mgmt /usr/local/bin/ +COPY entrypoint.sh /usr/local/bin/ + +RUN echo '#!/bin/bash\nif [ $# -lt 1 ]; then\n\techo "argument missing"\n\texit 1\nfi\neval "$*"\n' >> /usr/local/bin/execute.sh + +RUN chmod +x /usr/local/bin/execute.sh +RUN apt install netcat -y +RUN chmod +x /usr/local/bin/entrypoint.sh + +ARG BUILD_DATE +LABEL org.label-schema.name="cstor-pool-mgmt" +LABEL org.label-schema.description="OpenEBS" +LABEL org.label-schema.url="http://www.openebs.io/" +LABEL org.label-schema.vcs-url="https://github.com/openebs/maya" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$BUILD_DATE + +ENTRYPOINT entrypoint.sh +EXPOSE 7676 diff --git a/buildscripts/cstor-pool-mgmt-debug/entrypoint.sh b/buildscripts/cstor-pool-mgmt-debug/entrypoint.sh new file mode 100644 index 0000000000..49dea71818 --- /dev/null +++ b/buildscripts/cstor-pool-mgmt-debug/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -ex + +rm /usr/local/bin/zrepl +exec /usr/local/bin/cstor-pool-mgmt start +exec service ssh start +exec service rsyslog start From d3457846d2ebb3face3657d31a07693b65f4fe01 Mon Sep 17 00:00:00 2001 From: mittachaitu Date: Tue, 19 Nov 2019 10:57:17 +0530 Subject: [PATCH 5/5] This commit fixes the issue reported by Codacy Signed-off-by: mittachaitu --- buildscripts/cstor-pool-mgmt-debug/Dockerfile | 2 +- tests/cstor/volume/provision_negative_test.go | 6 +++--- tests/cstor/volume/suite_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/buildscripts/cstor-pool-mgmt-debug/Dockerfile b/buildscripts/cstor-pool-mgmt-debug/Dockerfile index 23d2b986a1..d901fa5708 100644 --- a/buildscripts/cstor-pool-mgmt-debug/Dockerfile +++ b/buildscripts/cstor-pool-mgmt-debug/Dockerfile @@ -12,7 +12,7 @@ FROM $BASE_IMAGE COPY cstor-pool-mgmt /usr/local/bin/ COPY entrypoint.sh /usr/local/bin/ -RUN echo '#!/bin/bash\nif [ $# -lt 1 ]; then\n\techo "argument missing"\n\texit 1\nfi\neval "$*"\n' >> /usr/local/bin/execute.sh +RUN printf '#!/bin/bash\nif [ $# -lt 1 ]; then\n\techo "argument missing"\n\texit 1\nfi\neval "$*"\n' >> /usr/local/bin/execute.sh RUN chmod +x /usr/local/bin/execute.sh RUN apt install netcat -y diff --git a/tests/cstor/volume/provision_negative_test.go b/tests/cstor/volume/provision_negative_test.go index 6dae7ced0d..36acbd2fe8 100644 --- a/tests/cstor/volume/provision_negative_test.go +++ b/tests/cstor/volume/provision_negative_test.go @@ -88,7 +88,7 @@ var _ = Describe("[Cstor Volume Provisioning Negative] Volume Provisioning", fun } } Expect(nodeIP).NotTo(BeEmpty()) - hostIpPort = nodeIP + ":30031" + hostIPPort = nodeIP + ":30031" serviceConfig := &tests.ServiceConfig{ Name: svcName, Namespace: openebsNamespace, @@ -113,7 +113,7 @@ var _ = Describe("[Cstor Volume Provisioning Negative] Volume Provisioning", fun ops.Config = pvcConfig //Injecting Errors During CVRUpdate and ZFS Creation Time - injectError := debug.NewClient(hostIpPort) + injectError := debug.NewClient(hostIPPort) err := injectError.PostInject( debug.NewErrorInjection(). WithZFSCreateError(debug.Inject). @@ -143,7 +143,7 @@ var _ = Describe("[Cstor Volume Provisioning Negative] Volume Provisioning", fun "cstor-pool-mgmt", command, false) // Eject the ZFSCreate error - injectError := debug.NewClient(hostIpPort) + injectError := debug.NewClient(hostIPPort) err := injectError.PostInject( debug.NewErrorInjection(). WithZFSCreateError(debug.Eject). diff --git a/tests/cstor/volume/suite_test.go b/tests/cstor/volume/suite_test.go index dbf1841d2e..0799ea109d 100644 --- a/tests/cstor/volume/suite_test.go +++ b/tests/cstor/volume/suite_test.go @@ -59,7 +59,7 @@ var ( svcName = "volume-provision-svc-injector" serviceObj *corev1.Service pvcName = "volume-claim1" - hostIpPort string + hostIPPort string hostLabel = "kubernetes.io/hostname" poolPodList *corev1.PodList )