Signing commit

dell-csi-helm-installer/myvalues.yaml

@@ -0,0 +1,31 @@
+kubeletConfigDir: /var/lib/kubelet
+nodeFCPortsFilterFile: /etc/fc-ports-filter
+externalAccess: 
+imagePullPolicy: Always
+controller:
+   controllerCount: 2
+   volumeNamePrefix: csivol
+   snapshot:
+      enabled: true
+      snapNamePrefix: csisnap
+   resizer:
+      enabled: true
+   healthMonitor:
+      enabled: true
+      volumeHealthMonitorInterval: 60s
+   replication:
+      enabled: false
+      image: amaas-eos-mw1.cec.lab.emc.com:5028/dell-csi-replicator:v0.0.1.025
+      replicationContextPrefix: "powerstore"
+      replicationPrefix: "replication.storage.dell.com"   
+   nodeSelector:
+   tolerations:
+node:
+   nodeNamePrefix: csi-node
+   nodeIDPath: /etc/machine-id
+   healthMonitor:
+      enabled: true
+   nodeSelector:
+   tolerations:
+connection:
+   enableCHAP: false

dell-csi-helm-installer/verify-csi-powerstore.sh

@@ -17,6 +17,7 @@ function verify-csi-powerstore() {
   verify_alpha_snap_resources
   verify_optional_replication_requirements
   verify_iscsi_installation
+  verify_nvme_installation
   verify_helm_3
 }
 

dell-csi-helm-installer/verify.sh

@@ -95,6 +95,34 @@ function header() {
   echo "|- Kubernetes Version: ${kMajorVersion}.${kMinorVersion}"
 }
 
+# Check if the NVMe client is installed
+function verify_nvme_installation() {
+  if [ ${NODE_VERIFY} -eq 0 ]; then
+    return
+  fi
+
+  log smart_step "Verifying NVMe installation" "$1"
+
+  error=0
+  for node in $MINION_NODES; do
+    # check if the NVMe client is installed
+    run_command ssh ${NODEUSER}@"${node}" "cat /etc/nvme/hostnqn" >/dev/null 2>&1
+    rv=$?
+    if [ $rv -ne 0 ]; then
+      error=1
+      found_warning "Either NVMe client was not found on node: $node or not able to verify"
+    fi
+    run_command ssh ${NODEUSER}@"${node}" lsmod | grep nvme_tcp &>/dev/null
+    rv=$?
+    if [ $rv -ne 0 ]; then
+      error=1
+      found_warning "Either NVMe module is not loaded on node: $node or not able to verify"
+    fi
+  done
+
+  check_error error
+}
+
 # Check if the iSCSI client is installed
 function verify_iscsi_installation() {
   if [ ${NODE_VERIFY} -eq 0 ]; then

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/dell/gocsi v1.5.0
 	github.com/dell/gofsutil v1.6.0
 	github.com/dell/goiscsi v1.2.0
-	github.com/dell/gopowerstore v1.6.1-0.20211223095101-c47391fc979f
+	github.com/dell/gopowerstore v1.6.1-0.20220120132720-96c2320b69fe
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/golang/mock v1.4.4
 	github.com/golang/protobuf v1.5.2
@@ -38,6 +38,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/dell/gonvme v0.0.0-20220119211702-1dd63f6211d2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-logr/logr v0.4.0 // indirect
 	github.com/go-openapi/errors v0.19.8 // indirect

go.sum

@@ -170,8 +170,12 @@ github.com/dell/gofsutil v1.6.0/go.mod h1:98Wpcg7emz4iGgY16fd4MKpnal2SX2hBiwP5gh
 github.com/dell/goiscsi v1.1.0/go.mod h1:MfuMjbKWsh/MOb0VDW20C+LFYRIOfWKGiAxWkeM5TKo=
 github.com/dell/goiscsi v1.2.0 h1:ocQs4pz2Fw2vr73RVAQBKwpN468SK4TZHPLhU7/FB9A=
 github.com/dell/goiscsi v1.2.0/go.mod h1:MfuMjbKWsh/MOb0VDW20C+LFYRIOfWKGiAxWkeM5TKo=
+github.com/dell/gonvme v0.0.0-20220119211702-1dd63f6211d2 h1:DmgJgOZGH5z5cxmLnrxgqTenTQ6tY3cNdZYkekNYGaw=
+github.com/dell/gonvme v0.0.0-20220119211702-1dd63f6211d2/go.mod h1:JXLmgxE/2dsiXqW5E+kH5Zo11lOjCuHyiHu92xqR9O0=
 github.com/dell/gopowerstore v1.6.1-0.20211223095101-c47391fc979f h1:F+aAuMlcTUV/F6eWYoM94+I6MR94gVE3g8ZY65DGzbE=
 github.com/dell/gopowerstore v1.6.1-0.20211223095101-c47391fc979f/go.mod h1:0ziQJ1iuZYDV+P53ua+VeH+rIylYT9WNjGSI/7aPly0=
+github.com/dell/gopowerstore v1.6.1-0.20220120132720-96c2320b69fe h1:GC4KihepEV4ctfHoZXM+o2wd807fsBGzLrXsl1bHk6Q=
+github.com/dell/gopowerstore v1.6.1-0.20220120132720-96c2320b69fe/go.mod h1:0ziQJ1iuZYDV+P53ua+VeH+rIylYT9WNjGSI/7aPly0=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=

pkg/common/common.go

@@ -78,6 +78,8 @@ const (
 	FcTransport TransportType = "FC"
 	// ISCSITransport indicates that ISCSI is chosen as a SCSI transport protocol
 	ISCSITransport TransportType = "ISCSI"
+	// NVMETransport indicates that NVMe is chosen as a transport protocol
+	NVMETransport TransportType = "NVME"
 	// AutoDetectTransport indicates that SCSI transport protocol would be detected automatically
 	AutoDetectTransport TransportType = "AUTO"
 	// NoneTransport indicates that no SCSI transport protocol needed

pkg/node/node.go

@@ -40,6 +40,7 @@ import (
 	csictx "github.com/dell/gocsi/context"
 	"github.com/dell/gofsutil"
 	"github.com/dell/goiscsi"
+	"github.com/dell/gonvme"
 	"github.com/dell/gopowerstore"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc/codes"
@@ -69,11 +70,13 @@ type Service struct {
 	iscsiConnector ISCSIConnector
 	fcConnector    FcConnector
 	iscsiLib       goiscsi.ISCSIinterface
+	nvmeLib        *gonvme.NVMeTCP
 
 	opts   Opts
 	nodeID string
 
 	useFC                  bool
+	useISCSI               bool
 	initialized            bool
 	reusedHost             bool
 	isHealthMonitorEnabled bool
@@ -89,17 +92,21 @@ func (s *Service) Init() error {
 
 	s.initConnectors()
 
+	NVMeOpts := make(map[string]string)
+	NVMeOpts["chrootDirectory"] = s.opts.NodeChrootPath
+	s.nvmeLib = gonvme.NewNVMeTCP(NVMeOpts)
+
 	err := s.updateNodeID()
 	if err != nil {
 		return fmt.Errorf("can't update node id: %s", err.Error())
 	}
 
-	iscsiInitiators, fcInitiators, err := s.getInitiators()
+	iscsiInitiators, fcInitiators, nvmeInitiators, err := s.getInitiators()
 	if err != nil {
 		return fmt.Errorf("can't get initiators of the node: %s", err.Error())
 	}
 
-	if len(iscsiInitiators) == 0 && len(fcInitiators) == 0 {
+	if len(iscsiInitiators) == 0 && len(fcInitiators) == 0 && len(nvmeInitiators) == 0 {
 		return nil
 	}
 
@@ -112,24 +119,38 @@ func (s *Service) Init() error {
 		var initiators []string
 
 		switch arr.BlockProtocol {
+		case common.NVMETransport:
+			if len(nvmeInitiators) == 0 {
+				return fmt.Errorf("NVMe transport was requested but NVMe initiator is not available")
+			}
+			s.useISCSI = false
+			s.useFC = false
 		case common.ISCSITransport:
 			if len(iscsiInitiators) == 0 {
 				return fmt.Errorf("iSCSI transport was requested but iSCSI initiator is not available")
 			}
+			s.useISCSI = true
 			s.useFC = false
 		case common.FcTransport:
 			if len(fcInitiators) == 0 {
 				return fmt.Errorf("FC transport was requested but FC initiator is not available")
 			}
 			s.useFC = true
+			s.useISCSI = false
 		default:
 			s.useFC = len(fcInitiators) > 0
+			s.useISCSI = len(iscsiInitiators) > 0
 		}
 
 		if s.useFC {
 			initiators = fcInitiators
-		} else {
+			log.Infof("FC Protocol is requested")
+		} else if s.useISCSI {
 			initiators = iscsiInitiators
+			log.Infof("iSCSI Protocol is requested")
+		} else {
+			initiators = nvmeInitiators
+			log.Infof("NVMe Protocol is requested")
 		}
 
 		err = s.setupHost(initiators, arr.GetClient(), arr.GetIP())
@@ -988,13 +1009,26 @@ func (s *Service) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest)
 					continue
 				}
 
-				_, err = s.iscsiLib.DiscoverTargets(infoList[0].Portal, false)
-				if err != nil {
-					log.Error("couldn't discover targets")
-					continue
-				}
+				if !s.useFC && s.useISCSI {
+					_, err = s.iscsiLib.DiscoverTargets(infoList[0].Portal, false)
+					if err != nil {
+						log.Error("couldn't discover targets")
+						continue
+					}
+
+					resp.AccessibleTopology.Segments[common.Name+"/"+arr.GetIP()+"-iscsi"] = "true"
+
+				} else {
+					log.Infof("Discovering NVMe targets is requested")
+					nvmeIP := strings.Split(infoList[0].Portal, ":")
+					_, err := s.nvmeLib.DiscoverNVMeTCPTargets(nvmeIP[0], false)
 
-				resp.AccessibleTopology.Segments[common.Name+"/"+arr.GetIP()+"-iscsi"] = "true"
+					if err != nil {
+						log.Error("couldn't discover NVMe targets")
+						continue
+					}
+					resp.AccessibleTopology.Segments[common.Name+"/"+arr.GetIP()+"-nvme"] = "true"
+				}
 			}
 		}
 	}
@@ -1040,11 +1074,12 @@ func (s *Service) updateNodeID() error {
 	return nil
 }
 
-func (s *Service) getInitiators() ([]string, []string, error) {
+func (s *Service) getInitiators() ([]string, []string, []string, error) {
 	ctx := context.Background()
 
 	var iscsiAvailable bool
 	var fcAvailable bool
+	var nvmeAvailable bool
 
 	iscsiInitiators, err := s.iscsiConnector.GetInitiatorName(ctx)
 	if err != nil {
@@ -1066,12 +1101,23 @@ func (s *Service) getInitiators() ([]string, []string, error) {
 		fcAvailable = true
 	}
 
-	if !iscsiAvailable && !fcAvailable {
+	nvmeInitiators, err := s.nvmeLib.GetInitiators("")
+
+	if err != nil {
+		log.Error("nodeStartup could not get Initiator NQNs")
+	} else if len(nvmeInitiators) == 0 {
+		log.Error("NVMe initiators not found on node")
+	} else {
+		log.Error("NVMe initiators found on node")
+		nvmeAvailable = true
+	}
+
+	if !iscsiAvailable && !fcAvailable && !nvmeAvailable {
 		// If we haven't found any initiators we still can use NFS
-		log.Info("FC and iSCSI initiators not found on node")
+		log.Info("FC, iSCSI and NVMe initiators not found on node")
 	}
 
-	return iscsiInitiators, fcInitiators, nil
+	return iscsiInitiators, fcInitiators, nvmeInitiators, nil
 }
 
 func (s *Service) getNodeFCPorts(ctx context.Context) ([]string, error) {
@@ -1239,13 +1285,16 @@ func (s *Service) buildInitiatorsArray(initiators []string) []gopowerstore.Initi
 	var portType gopowerstore.InitiatorProtocolTypeEnum
 	if s.useFC {
 		portType = gopowerstore.InitiatorProtocolTypeEnumFC
-	} else {
+	} else if s.useISCSI {
 		portType = gopowerstore.InitiatorProtocolTypeEnumISCSI
+	} else {
+		portType = gopowerstore.InitiatorProtocolTypeEnumNVME
 	}
+
 	initiatorsReq := make([]gopowerstore.InitiatorCreateModify, len(initiators))
 	for i, iqn := range initiators {
 		iqn := iqn
-		if !s.useFC && s.opts.EnableCHAP {
+		if !s.useFC && s.useISCSI && s.opts.EnableCHAP {
 			initiatorsReq[i] = gopowerstore.InitiatorCreateModify{
 				ChapSinglePassword: &s.opts.CHAPPassword,
 				ChapSingleUsername: &s.opts.CHAPUsername,