feat(localpv-device): allow local pv device on select devices (#1648)

Ref: openebs/openebs#2916

This PR makes use of the Label Selector capability
added in the BDC. More details: openebs/openebs#2921

The BDC Label Selector feature makes it possible for
administrators to group a set of block devices and use
them for creating Local PV (device).

To use the Label Selector feature with Local PV (Devices)
the administrator is expected to group(or pool) a set of
block devices by assigning them the label:
 openebs.io/block-device-tag=< tag-x >.

The `<tag-x>` can be passed to Local PV storage class
via cas annotations `BlockDeviceTag`. If the value is present, then Local PV
device provisioner will set the following additional selector
on the BDC:
`openebs.io/block-device-tag=< tag-x >`

Signed-off-by: kmova <kiran.mova@mayadata.io>

cmd/provisioner-localpv/app/config.go

@@ -37,12 +37,47 @@ const (
 	//KeyPVStorageType defines if the PV should be backed
 	// a hostpath ( sub directory or a storage device)
 	KeyPVStorageType = "StorageType"
+
 	//KeyPVBasePath defines base directory for hostpath volumes
 	// can be configured via the StorageClass annotations.
 	KeyPVBasePath = "BasePath"
+
 	//KeyPVFSType defines filesystem type to be used with devices
 	// and can be configured via the StorageClass annotations.
 	KeyPVFSType = "FSType"
+
+	//KeyBDTag defines the value for the Block Device Tag
+	//label selector configured via the StorageClass annotations.
+	//User can group block devices across nodes by setting the
+	//label on block devices as:
+	//  openebs.io/block-device-tag=<tag-value>
+	//
+	//The <tag-value> used above can be passsed to the
+	//Local PV device provisioner via the StorageClass
+	//CAS annotations, to specify that Local PV (device)
+	//should only make use of those block devices that
+	//tagged with the given <tag-value>.
+	//
+	//Example: Local PV device StorageClass for picking devices
+	//labeled as: openebs.io/block-device-tag=tag-x
+	//will be as follows
+	//
+	// kind: StorageClass
+	// metadata:
+	//   name: openebs-device-tag-x
+	//   annotations:
+	//     openebs.io/cas-type: local
+	//     cas.openebs.io/config: |
+	//       - name: StorageType
+	//         value: "device"
+	//       - name: BlockDeviceTag
+	//         value: "tag-x"
+	// provisioner: openebs.io/local
+	// volumeBindingMode: WaitForFirstConsumer
+	// reclaimPolicy: Delete
+	//
+	KeyBDTag = "BlockDeviceTag"
+
 	//KeyPVRelativePath defines the alternate folder name under the BasePath
 	// By default, the pv name will be used as the folder name.
 	// KeyPVBasePath can be useful for providing the same underlying folder
@@ -139,6 +174,20 @@ func (c *VolumeConfig) GetFSType() string {
 	return fsType
 }
 
+//GetBDTagValue returns the block device tag
+//value configured in StorageClass.
+//
+//Default is "", no device tag will be set and any
+//available block device (without labelled with tag)
+//can be used for creating Local PV(device).
+func (c *VolumeConfig) GetBDTagValue() string {
+	bdTagValue := c.getValue(KeyBDTag)
+	if len(strings.TrimSpace(bdTagValue)) == 0 {
+		return ""
+	}
+	return bdTagValue
+}
+
 //GetPath returns a valid PV path based on the configuration
 // or an error. The Path is constructed using the following rules:
 // If AbsolutePath is specified return it. (Future)

cmd/provisioner-localpv/app/helper_blockdevice.go

@@ -65,6 +65,10 @@ type HelperBlockDeviceOptions struct {
 	bdcName string
 	//  volumeMode of PVC
 	volumeMode corev1.PersistentVolumeMode
+
+	//bdTagValue is the value passed for
+	// BlockDeviceTag via StorageClass config
+	bdTagValue string
 }
 
 // validate checks that the required fields to create BDC
@@ -120,14 +124,20 @@ func (p *Provisioner) createBlockDeviceClaim(blkDevOpts *HelperBlockDeviceOption
 		return nil
 	}
 
-	bdcObj, err := blockdeviceclaim.NewBuilder().
+	bdcObjBuilder := blockdeviceclaim.NewBuilder().
 		WithNamespace(p.namespace).
 		WithName(bdcName).
 		WithHostName(blkDevOpts.nodeHostname).
 		WithCapacity(blkDevOpts.capacity).
 		WithFinalizer(LocalPVFinalizer).
-		WithBlockVolumeMode(blkDevOpts.volumeMode).
-		Build()
+		WithBlockVolumeMode(blkDevOpts.volumeMode)
+
+	// If bdTagValue is configure, set it on the BDC
+	if len(blkDevOpts.bdTagValue) > 0 {
+		bdcObjBuilder.WithBlockDeviceTag(blkDevOpts.bdTagValue)
+	}
+
+	bdcObj, err := bdcObjBuilder.Build()
 
 	if err != nil {
 		//TODO : Need to relook at this error

cmd/provisioner-localpv/app/provisioner_blockdevice.go

@@ -45,6 +45,7 @@ func (p *Provisioner) ProvisionBlockDevice(opts pvController.VolumeOptions, volu
 		name:         name,
 		capacity:     capacity.String(),
 		volumeMode:   *opts.PVC.Spec.VolumeMode,
+		bdTagValue:   volumeConfig.GetBDTagValue(),
 	}
 
 	path, blkPath, err := p.getBlockDevicePath(blkDevOpts)

pkg/blockdeviceclaim/v1alpha1/build.go

@@ -31,8 +31,13 @@ const (
 
 	// StoragePoolKindCSPC holds the value of CStorPoolCluster
 	StoragePoolKindCSPC = "CStorPoolCluster"
+
 	// APIVersion holds the value of OpenEBS version
 	APIVersion = "openebs.io/v1alpha1"
+
+	// bdTagKey defines the label selector key
+	// used for grouping block devices using a tag.
+	bdTagKey = "openebs.io/block-device-tag"
 )
 
 // Builder is the builder object for BlockDeviceClaim
@@ -323,3 +328,28 @@ func (b *Builder) WithBlockVolumeMode(mode corev1.PersistentVolumeMode) *Builder
 
 	return b
 }
+
+// WithBlockDeviceTag appends (or creates) the BDC Label Selector
+// by setting the provided value to the fixed key
+// openebs.io/block-device-tag
+// This will enable the NDM to pick only devices that
+// match the node (hostname) and block device tag value.
+func (b *Builder) WithBlockDeviceTag(bdTagValue string) *Builder {
+	if len(bdTagValue) == 0 {
+		b.errs = append(
+			b.errs,
+			errors.New("failed to build BDC object: missing block device tag value"),
+		)
+		return b
+	}
+
+	if b.BDC.Object.Spec.Selector == nil {
+		b.BDC.Object.Spec.Selector = &metav1.LabelSelector{}
+	}
+	if b.BDC.Object.Spec.Selector.MatchLabels == nil {
+		b.BDC.Object.Spec.Selector.MatchLabels = map[string]string{}
+	}
+
+	b.BDC.Object.Spec.Selector.MatchLabels[bdTagKey] = bdTagValue
+	return b
+}

pkg/blockdeviceclaim/v1alpha1/build_test.go

@@ -206,16 +206,46 @@ func TestBuildWithCapacity(t *testing.T) {
 	}
 }
 
+func TestBuilderWithBlockDeviceTag(t *testing.T) {
+	tests := map[string]struct {
+		tag       string
+		expectErr bool
+	}{
+		"Test Builder with tag": {
+			tag:       "test",
+			expectErr: false,
+		},
+		"Test Builder without tag": {
+			tag:       "",
+			expectErr: true,
+		},
+	}
+	for name, mock := range tests {
+		name, mock := name, mock
+		t.Run(name, func(t *testing.T) {
+			b := NewBuilder().WithBlockDeviceTag(mock.tag)
+			if mock.expectErr && len(b.errs) == 0 {
+				t.Fatalf("Test %q failed: expected error not to be nil", name)
+			}
+			if !mock.expectErr && len(b.errs) > 0 {
+				t.Fatalf("Test %q failed: expected error to be nil", name)
+			}
+		})
+	}
+}
+
 func TestBuild(t *testing.T) {
 	tests := map[string]struct {
 		name        string
 		capacity    string
+		tagValue    string
 		expectedBDC *apis.BlockDeviceClaim
 		expectedErr bool
 	}{
 		"BDC with correct details": {
 			name:     "BDC1",
 			capacity: "10Ti",
+			tagValue: "",
 			expectedBDC: &apis.BlockDeviceClaim{
 				ObjectMeta: metav1.ObjectMeta{Name: "BDC1"},
 				Spec: apis.DeviceClaimSpec{
@@ -228,17 +258,48 @@ func TestBuild(t *testing.T) {
 			},
 			expectedErr: false,
 		},
+		"BDC with correct details, including device pool": {
+			name:     "BDC1",
+			capacity: "10Ti",
+			tagValue: "test",
+			expectedBDC: &apis.BlockDeviceClaim{
+				ObjectMeta: metav1.ObjectMeta{Name: "BDC1"},
+				Spec: apis.DeviceClaimSpec{
+					Resources: apis.DeviceClaimResources{
+						Requests: corev1.ResourceList{
+							corev1.ResourceName(ndm.ResourceStorage): fakeCapacity("10Ti"),
+						},
+					},
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{
+							bdTagKey: "test",
+						},
+					},
+				},
+			},
+			expectedErr: false,
+		},
 		"BDC with error": {
 			name:        "",
 			capacity:    "500Gi",
+			tagValue:    "test",
 			expectedBDC: nil,
 			expectedErr: true,
 		},
 	}
 	for name, mock := range tests {
 		name, mock := name, mock
 		t.Run(name, func(t *testing.T) {
-			bdcObj, err := NewBuilder().WithName(mock.name).WithCapacity(mock.capacity).Build()
+			bdcObjBuilder := NewBuilder().
+				WithName(mock.name).
+				WithCapacity(mock.capacity)
+
+			if len(mock.tagValue) > 0 {
+				bdcObjBuilder.WithBlockDeviceTag(mock.tagValue)
+			}
+
+			bdcObj, err := bdcObjBuilder.Build()
+
 			if mock.expectedErr && err == nil {
 				t.Fatalf("Test %q failed: expected error not to be nil", name)
 			}

tests/localpv/README.md

@@ -0,0 +1,40 @@
+# Local PV Provisioner BDD
+
+Local PV Provisioner BDD tests are developed using ginkgo & gomega libraries.
+
+## How to run the tests?
+
+### Pre-requisites
+
+- Install Ginkgo and Gomega on your development machine. 
+  ```
+  $ go get github.com/onsi/ginkgo/ginkgo
+  $ go get github.com/onsi/gomega/...
+  ```
+- Get your Kubernetes Cluster ready and make sure you can run 
+  kubectl from your development machine. 
+  Note down the path to the `kubeconfig` file used by kubectl 
+  to access your cluster.  Example: /home/<user>/.kube/config
+
+- (Optional) Set the KUBECONFIG environment variable on your 
+  development machine to point to the kubeconfig file. 
+  Example: KUBECONFIG=/home/<user>/.kube/config
+
+  If you do not set this ENV, you will have to pass the file 
+  to the ginkgo CLI
+
+- Some of the tests require block devices (that are not mounted)
+  to be available in the cluster.
+
+- Install required OpenEBS components. 
+  Example: `kubectl apply -f openebs-operator.yaml`
+
+### Run tests
+
+- Run the tests by being in the localpv tests folder. 
+  `$ cd $GOPATH/src/github.com/openebs/maya/tests/localpv/`
+  `$ ginkgo -v --`
+
+  In case the KUBECONFIG env is not configured, you can run:
+  `$ ginkgo -v -- -kubeconfig=/path/to/kubeconfig`
+

tests/localpv/suite_test.go

@@ -16,6 +16,7 @@ package localpv
 import (
 	"flag"
 
+	"os"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
@@ -45,7 +46,7 @@ func TestSource(t *testing.T) {
 }
 
 func init() {
-	flag.StringVar(&kubeConfigPath, "kubeconfig", "", "path to kubeconfig to invoke kubernetes API calls")
+	flag.StringVar(&kubeConfigPath, "kubeconfig", os.Getenv("KUBECONFIG"), "path to kubeconfig to invoke kubernetes API calls")
 }
 
 var ops *tests.Operations