From 9e98707cd53a80474f92094718d97d83476c1245 Mon Sep 17 00:00:00 2001 From: Qiaowei Ren Date: Tue, 19 Mar 2019 20:14:36 +0800 Subject: [PATCH] contrib: add NVMeoF support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch will add NVMeoF support. The NVMe over Fabrics (NVMeOF) is an emerging technology, and it gives data centers unprecedented access to NVMe SSD storage. Like iSCSI, NVMeОF is a network protocol used to communicate between a host and a storage system over a network (aka fabric). Signed-off-by: Shixin Yang Signed-off-by: Wenjie Liu Signed-off-by: Qiaowei Ren --- contrib/connector/connector.go | 3 + contrib/connector/nvmeof/nvmeof.go | 40 +++ contrib/connector/nvmeof/nvmeof_helper.go | 199 +++++++++++++++ contrib/drivers/lvm/lvm.go | 40 ++- contrib/drivers/lvm/targets/nvmeof.go | 293 +++++++++++++++++++++ contrib/drivers/lvm/targets/targets.go | 50 +++- pkg/controller/controller.go | 8 +- script/devsds/lib/lvm.sh | 113 +++++++- test/e2e/connector/connector.go | 10 +- test/e2e/e2ef_test.go | 298 +++++++++++++++++++++- 10 files changed, 1022 insertions(+), 32 deletions(-) create mode 100644 contrib/connector/nvmeof/nvmeof.go create mode 100644 contrib/connector/nvmeof/nvmeof_helper.go create mode 100755 contrib/drivers/lvm/targets/nvmeof.go diff --git a/contrib/connector/connector.go b/contrib/connector/connector.go index 13bb7abdd..d54cb5b5b 100644 --- a/contrib/connector/connector.go +++ b/contrib/connector/connector.go @@ -30,6 +30,9 @@ const ( Iqn = "iqn" RbdDriver = "rbd" + + NvmeofDriver = "nvmeof" + Nqn = "nqn" ) // Connector implementation diff --git a/contrib/connector/nvmeof/nvmeof.go b/contrib/connector/nvmeof/nvmeof.go new file mode 100644 index 000000000..590e8f937 --- /dev/null +++ b/contrib/connector/nvmeof/nvmeof.go @@ -0,0 +1,40 @@ +// Copyright (c) 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvmeof + +import ( + "github.com/opensds/opensds/contrib/connector" +) + +type Nvmeof struct{} + +func init() { + connector.RegisterConnector(connector.NvmeofDriver, &Nvmeof{}) + connector.ExecCmd("modprobe", "nvme-rdma") +} + +func (nof *Nvmeof) Attach(conn map[string]interface{}) (string, error) { + return Connect(conn) +} + +func (nof *Nvmeof) Detach(conn map[string]interface{}) error { + NvmeofCon := ParseNvmeofConnectInfo(conn) + return DisConnect(NvmeofCon.Nqn) +} + +// GetInitiatorInfo implementation +func (nof *Nvmeof) GetInitiatorInfo() (string, error) { + return getInitiatorInfo() +} diff --git a/contrib/connector/nvmeof/nvmeof_helper.go b/contrib/connector/nvmeof/nvmeof_helper.go new file mode 100644 index 000000000..7eda124ff --- /dev/null +++ b/contrib/connector/nvmeof/nvmeof_helper.go @@ -0,0 +1,199 @@ +// Copyright (c) 2019 Intel Corporation, Ltd. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvmeof + +import ( + "errors" + "log" + "strings" + "time" + + "github.com/mitchellh/mapstructure" + "github.com/opensds/opensds/contrib/connector" +) + +const ( + iniNvmePrefix = "nqn.ini." +) + +// ConnectorInfo define +type ConnectorInfo struct { + Nqn string `mapstructure:"targetNQN"` //NVMe subsystem name to the volume to be connected + TgtPort string `mapstructure:"targetPort"` //NVMe target port that hosts the nqn sybsystem + TgtPortal string `mapstructure:"targetIP"` //NVMe target ip that hosts the nqn sybsystem + TranType string `mapstructure:"transporType "` // Nvme transport type + HostNqn string `mapstructure:"hostNqn"` // host nqn +} + +////////////////////////////////////////////////////////////////////////////////////////// +// Refer some codes from: https://github.intel.com/yingxinc/cinder-rsd-os-brick // +////////////////////////////////////////////////////////////////////////////////////////// + +// GetInitiator returns all the Nvmeof UUID +func GetInitiator() ([]string, error) { + res, err := connector.ExecCmd("dmidecode") + nqns := []string{} + if err != nil { + log.Printf("Unable to execute dmidecode,Error encountered gathering Nvmeof UUID: %v\n", err) + return nqns, nil + } + + lines := strings.Split(string(res), "\n") + for _, l := range lines { + if strings.Contains(l, "UUID:") { + tmp := iniNvmePrefix + strings.Split(l, ":")[1] + nqns = append(nqns, tmp) + log.Printf("Found the following nqns: %s", nqns) + return nqns, nil + } + } + log.Println("can not find any nqn initiator") + return nqns, errors.New("can not find any nqn initiator") +} + +func getInitiatorInfo() (string, error) { + + initiators, err := GetInitiator() + if err != nil { + return "", err + } + + if len(initiators) == 0 { + return "", errors.New("no nqn found") + } + + if len(initiators) > 1 { + return "", errors.New("the number of nqn is wrong") + } + + hostName, err := connector.GetHostName() + if err != nil { + return "", errors.New("can not get hostname") + } + + info := initiators[0] + "." + hostName + return info, nil +} + +// GetNvmeDevice get all the nvme devices +func GetNvmeDevice() (map[string]int, error) { + nvmeDevice := make(map[string]int) + pattern := "/dev/nvme" + Npath, err := connector.ExecCmd("nvme", "list") + if err != nil { + return nvmeDevice, err + } + log.Println("nvme list succeed") + lines := strings.Split(string(Npath), "\n") + for _, l := range lines { + if strings.Contains(l, pattern) { + name := strings.Split(l, " ")[0] + nvmeDevice[name] = 1 + } + } + return nvmeDevice, err +} + +// GetNvmeSubsystems :list connected target name +func GetNvmeSubsystems() (map[string]int, error) { + nqn := make(map[string]int) + res, err := connector.ExecCmd("nvme", "list-subsys") + if err != nil { + return nqn, err + } + + lines := strings.Split(string(res), "\n") + for _, l := range lines { + if strings.Contains(l, "NQN=") { + name := strings.Split(l, "NQN=")[1] + nqn[name] = 1 + } + } + + log.Printf("Found the following NQN: %s", res) + return nqn, nil +} + +// Discovery NVMe-OF target +func Discovery(connMap map[string]interface{}) error { + conn := ParseNvmeofConnectInfo(connMap) + targetip := conn.TgtPortal + targetport := conn.TgtPort + info, err := connector.ExecCmd("nvme", "discover", "-t", "rdma", "-a", targetip, "-s", targetport) + if err != nil { + log.Println("Error encountered in send targets:%s, err: %v", string(info), err) + return err + } + return nil +} + +// Connect NVMe-OF Target ,return the new target device path in this node +func Connect(connMap map[string]interface{}) (string, error) { + CurrentNvmeDevice, _ := GetNvmeDevice() + conn := ParseNvmeofConnectInfo(connMap) + connNqn := conn.Nqn + targetPortal := conn.TgtPortal + port := conn.TgtPort + nvmeTransportType := "rdma" + log.Printf("conn information: ", connNqn, ",", targetPortal, ",", port) + + _, err := connector.ExecCmd("nvme", "connect", "-t", + nvmeTransportType, "-n", connNqn, "-a", targetPortal, "-s", port) + if err != nil { + log.Println("Failed to connect to NVMe nqn :", connNqn) + return "", err + } + + for retry := 0; retry < 10; retry++ { + allNvmeDevices, _ := GetNvmeDevice() + for p, _ := range allNvmeDevices { + if _, ok := CurrentNvmeDevice[p]; !ok { + log.Printf("NVMe device to be connected to is : %v", p) + return p, nil + } + time.Sleep(time.Second) + } + } + return "", errors.New("could not connect volume: Timeout after 10s") +} + +// DisConnect nvme device by name +func DisConnect(nqn string) error { + currentNvmeNames, err := GetNvmeSubsystems() + if err != nil { + log.Println("can not get nvme device") + return err + } + if _, ok := currentNvmeNames[nqn]; !ok { + log.Println("Trying to disconnect nqn" + nqn + + "is not connected.") + return errors.New("device path not found ") + } + + _, err = connector.ExecCmd("nvme", "disconnect", "-n", nqn) + if err != nil { + log.Println("could not disconnect nvme nqn ： ", nqn) + return err + } + log.Println(" disconnect nvme nqn ： ", nqn) + return nil +} + +// ParseNvmeofConnectInfo decode +func ParseNvmeofConnectInfo(connectInfo map[string]interface{}) *ConnectorInfo { + var con ConnectorInfo + mapstructure.Decode(connectInfo, &con) + return &con +} diff --git a/contrib/drivers/lvm/lvm.go b/contrib/drivers/lvm/lvm.go index 4236380d3..9d37004dc 100755 --- a/contrib/drivers/lvm/lvm.go +++ b/contrib/drivers/lvm/lvm.go @@ -42,6 +42,9 @@ const ( snapshotPrefix = "_snapshot-" blocksize = 4096 sizeShiftBit = 30 + opensdsnvmepool = "opensds-nvmegroup" + nvmeofAccess = "nvmeof" + iscsiAccess = "iscsi" ) const ( @@ -249,7 +252,11 @@ func (d *Driver) InitializeConnection(opt *pb.CreateVolumeAttachmentOpts) (*mode if d.conf.EnableChapAuth { chapAuth = []string{utils.RandSeqWithAlnum(20), utils.RandSeqWithAlnum(16)} } - t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir) + + // create target according to the pool's access protocol + accPro := opt.AccessProtocol + log.Info("accpro:", accPro) + t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir, accPro) expt, err := t.CreateExport(opt.GetVolumeId(), lvPath, hostIP, initiator, chapAuth) if err != nil { log.Error("Failed to initialize connection of logic volume:", err) @@ -257,15 +264,16 @@ func (d *Driver) InitializeConnection(opt *pb.CreateVolumeAttachmentOpts) (*mode } return &model.ConnectionInfo{ - DriverVolumeType: ISCSIProtocol, + DriverVolumeType: accPro, ConnectionData: expt, }, nil } func (d *Driver) TerminateConnection(opt *pb.DeleteVolumeAttachmentOpts) error { - t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir) + accPro := opt.AccessProtocol + t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir, accPro) if err := t.RemoveExport(opt.GetVolumeId()); err != nil { - log.Error("Failed to initialize connection of logic volume:", err) + log.Error("failed to terminate connection of logic volume:", err) return err } return nil @@ -406,6 +414,11 @@ func (d *Driver) CreateSnapshot(opt *pb.CreateVolumeSnapshotOpts) (snap *model.V metadata := map[string]string{KLvsPath: lvsPath} if bucket, ok := opt.Metadata["bucket"]; ok { + //nvmet right now can not support snap volume serve as nvme target + if vg == opensdsnvmepool { + log.Infof("nvmet right now can not support snap volume serve as nvme target") + log.Infof("still store in nvme pool but initialize connection by iscsi protocol") + } mountPoint, info, err := d.AttachSnapshot(opt.GetId(), lvsPath) if err != nil { d.cli.Delete(snapName, vg) @@ -524,7 +537,13 @@ func (d *Driver) InitializeSnapshotConnection(opt *pb.CreateSnapshotAttachmentOp chapAuth = []string{utils.RandSeqWithAlnum(20), utils.RandSeqWithAlnum(16)} } - t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir) + accPro := opt.AccessProtocol + if accPro == nvmeofAccess { + log.Infof("nvmet right now can not support snap volume serve as nvme target") + log.Infof("still create snapshot connection by iscsi") + accPro = iscsiAccess + } + t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir, accPro) data, err := t.CreateExport(opt.GetSnapshotId(), lvsPath, hostIP, initiator, chapAuth) if err != nil { log.Error("Failed to initialize snapshot connection of logic volume:", err) @@ -532,13 +551,20 @@ func (d *Driver) InitializeSnapshotConnection(opt *pb.CreateSnapshotAttachmentOp } return &model.ConnectionInfo{ - DriverVolumeType: ISCSIProtocol, + DriverVolumeType: accPro, ConnectionData: data, }, nil } func (d *Driver) TerminateSnapshotConnection(opt *pb.DeleteSnapshotAttachmentOpts) error { - t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir) + accPro := opt.AccessProtocol + if accPro == nvmeofAccess{ + log.Infof("nvmet right now can not support snap volume serve as nvme target") + log.Infof("still create snapshot connection by iscsi") + accPro = iscsiAccess + } + log.Info("terminate snapshot conn") + t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir, accPro) if err := t.RemoveExport(opt.GetSnapshotId()); err != nil { log.Error("Failed to terminate snapshot connection of logic volume:", err) return err diff --git a/contrib/drivers/lvm/targets/nvmeof.go b/contrib/drivers/lvm/targets/nvmeof.go new file mode 100755 index 000000000..65c8122f4 --- /dev/null +++ b/contrib/drivers/lvm/targets/nvmeof.go @@ -0,0 +1,293 @@ +// Copyright (c) 2019 Intel Corporation, Ltd. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package targets + +import ( + "bytes" + "errors" + "io" + "os" + "os/exec" + "strconv" + "strings" + + log "github.com/golang/glog" + "github.com/opensds/opensds/pkg/utils" +) + +const ( + opensdsNvmeofPrefix = "opensds-Nvmeof" + NvmetDir = "/sys/kernel/config/nvmet" +) + +type NvmeofTarget interface { + CreateNvmeofTarget(volId, tgtIqn, path, hostIp, initiator string, chapAuth []string) error + GetNvmeofTarget(iqn string) int + RemoveNvmeofTarget(volId, iqn string) error +} + +func NewNvmeofTarget(bip, tgtConfDir string) NvmeofTarget { + return &NvmeoftgtTarget{ + TgtConfDir: tgtConfDir, + BindIp: bip, + } +} + +type NvmeoftgtTarget struct { + BindIp string + TgtConfDir string +} + +func (t *NvmeoftgtTarget) init() { + t.execCmd("modprobe", "nvmet") + t.execCmd("modprobe", "nvmet-rdma") +} + +func (t *NvmeoftgtTarget) getTgtConfPath(volId string) string { + return NvmetDir + "/" + opensdsNvmeofPrefix + volId +} + +func (t *NvmeoftgtTarget) CreateNvmeofTarget(volId, tgtNqn, path, hostIp, initiator string, chapAuth []string) error { + + if exist, _ := utils.PathExists(NvmetDir); !exist { + os.MkdirAll(NvmetDir, 0755) + } + sysdir := NvmetDir + "/subsystems/" + tgtNqn + if exist, _ := utils.PathExists(sysdir); !exist { + os.MkdirAll(sysdir, 0755) + } + + var err error + if initiator == "ALL" { + // echo 1 > attr_allow_any_host + attrfile := sysdir + "/attr_allow_any_host" + content := "1" + err = t.WriteWithIo(attrfile, content) + if err != nil { + log.Errorf("can not set attr_allow_any_host ") + return err + } + } else { + // allow specific initiators to connect to this target + var initiatorInfo = "initiator:" + hostIp + ":" + initiator + hostpath := NvmetDir + "/hosts" + if exist, _ := utils.PathExists(hostpath); !exist { + os.MkdirAll(hostpath, 0755) + } + + hostDir := hostpath + "/" + initiatorInfo + if exist, _ := utils.PathExists(hostDir); !exist { + os.MkdirAll(hostDir, 0755) + } + + // create symbolic link of host + hostsys := sysdir + "/allowed_hosts/" + _, err = t.execCmd("ln", "-s", hostDir, hostsys) + if err != nil { + log.Errorf("Fail to create host link: " + initiatorInfo) + return err + } + } + + // get volume namespaceid + namespaceid := t.Getnamespaceid(volId) + if namespaceid == "" { + return errors.New("null namesapce") + } + namespace := sysdir + "/namespaces/" + namespaceid + if exist, _ := utils.PathExists(namespace); !exist { + os.MkdirAll(namespace, 0755) + } + + // volid as device path + devpath := namespace + "/device_path" + err = t.WriteWithIo(devpath, path) + if err != nil { + log.Errorf("Fail to set device path") + return err + } + + enablepath := namespace + "/enable" + err = t.WriteWithIo(enablepath, "1") + if err != nil { + log.Errorf("Fail to set device path") + return err + } + + //create port + portid := 1 + portspath := NvmetDir + "/ports/" + strconv.Itoa(portid) + if exist, _ := utils.PathExists(portspath); !exist { + //log.Errorf(portspath) + os.MkdirAll(portspath, 0755) + } + + // get target ip + ippath := portspath + "/addr_traddr" + ip, err := t.execCmd("hostname", "-I") + if err != nil { + log.Errorf("fail to get target ipv4 address") + return err + } + // Set nvmeof parameters, rightnow only supports rdma + // if built on virtual machine the return string ip may contain several ip addresses + ip = strings.Split(ip, " ")[0] + err = t.WriteWithIo(ippath, ip) + if err != nil { + log.Errorf("Fail to set target ip") + return err + } + + trtypepath := portspath + "/addr_trtype" + err = t.WriteWithIo(trtypepath, "rdma") + if err != nil { + log.Errorf("Fail to set rdma type") + return err + } + + trsvcidpath := portspath + "/addr_trsvcid" + err = t.WriteWithIo(trsvcidpath, "4420") + if err != nil { + log.Errorf("Fail to set ip port") + return err + } + + adrfampath := portspath + "/addr_adrfam" + err = t.WriteWithIo(adrfampath, "ipv4") + if err != nil { + log.Errorf("Fail to set ip family") + return err + } + + // create a soft link + portssub := portspath + "/subsystems/" + tgtNqn + _, err = t.execCmd("ln", "-s", sysdir, portssub) + if err != nil { + log.Errorf("Fail to create link") + return err + } + + // check + info, err := t.execCmd("dmesg", `|grep "enabling port"`) + if err != nil || info == "" { + log.Errorf("nvme target is not listening on the port") + return err + } + log.Info("create nvme target") + return nil +} + +func (t *NvmeoftgtTarget) GetNvmeofTarget(nqn string) int { + _, err := t.execCmd("cd", "/sys/kernel/config/nvmet/subsystems") + if err != nil { + log.Errorf("Fail to exec to enter nvme target dir:%v", err) + return -1 + } + _, err = t.execCmd("cd", nqn) + if err != nil { + log.Errorf("Fail to exec to display nvme target :%v", err) + return -1 + } + return 0 +} + +func (t *NvmeoftgtTarget) RemoveNvmeofTarget(volId, nqn string) error { + log.Info("removing target",nqn) + tgtConfPath := NvmetDir + "/subsystems/" + nqn + if exist, _ := utils.PathExists(tgtConfPath); !exist { + log.Warningf("Volume path %s does not exist, nothing to remove.", tgtConfPath) + return nil + } + + // port's link has to be removed first or the subsystem cannot be removed + portpath := NvmetDir + "/ports/1/subsystems/" + nqn + info, err := t.execCmd("rm", "-f", portpath) + if err != nil { + log.Errorf("can not rm port") + log.Errorf(info) + return err + } + + // remove namespace + ns := t.Getnamespaceid(volId) + if ns == "" { + log.Errorf("can not find volume %s's namespace",volId) + return errors.New("null namespace") + } + naspPath := NvmetDir + "/subsystems/" + nqn + "/namespaces/" + ns + info, err = t.execCmd("rmdir", naspPath) + if err != nil { + log.Errorf("can not rm nasp") + return err + } + + // remove namespaces ; if it allows all initiators ,then this dir should be empty + // if it allow specific hosts ,then here remove all the hosts + cmd := "rm -f " + NvmetDir + "/subsystems/" + nqn + "/allowed_hosts/" + "*" + info, err = t.execBash(cmd) + if err != nil { + log.Errorf("can not rm allowed hosts") + log.Errorf(info) + return err + } + + // remove subsystem + syspath := NvmetDir + "/subsystems/" + nqn + info, err = t.execCmd("rmdir", syspath) + if err != nil { + log.Errorf("can not rm subsys") + return err + } + return nil +} + +func (*NvmeoftgtTarget) execCmd(name string, cmd ...string) (string, error) { + ret, err := exec.Command(name, cmd...).Output() + if err != nil { + log.Errorf("error info: %v", err) + } + return string(ret), err +} + +func (*NvmeoftgtTarget) execBash(name string) (string, error) { + ret, err := exec.Command("/bin/sh", "-c", name).Output() + if err != nil { + log.Errorf("error info in sh %v ", err) + } + return string(ret), err +} + +func (*NvmeoftgtTarget) WriteWithIo(name, content string) error { + fileObj, err := os.OpenFile(name, os.O_RDWR, 0644) + if err != nil { + log.Errorf("Failed to open the file %v", err) + return err + } + if _, err := io.WriteString(fileObj, content); err == nil { + log.Infof("Successful appending to the file with os.OpenFile and io.WriteString.%s", content) + return nil + } + return err +} + +func (t *NvmeoftgtTarget) Getnamespaceid(volId string) string { + var buffer bytes.Buffer + for _, rune := range volId { + if rune >= '0' && rune <= '9' { + buffer.WriteRune(rune) + } + } + return buffer.String()[0:2] +} diff --git a/contrib/drivers/lvm/targets/targets.go b/contrib/drivers/lvm/targets/targets.go index f0979543a..3471ecaaa 100755 --- a/contrib/drivers/lvm/targets/targets.go +++ b/contrib/drivers/lvm/targets/targets.go @@ -15,7 +15,10 @@ package targets const ( - iscsiTgtPrefix = "iqn.2017-10.io.opensds:" + iscsiTgtPrefix = "iqn.2017-10.io.opensds:" + nvmeofTgtPrefix = "nqn.2019-01.com.opensds:nvme:" + iscsiAccess = "iscsi" + nvmeofAccess = "nvmeof" ) // Target is an interface for exposing some operations of different targets, @@ -26,10 +29,18 @@ type Target interface { RemoveExport(volId string) error } -// NewTarget method creates a new iscsi target. -func NewTarget(bip string, tgtConfDir string) Target { - return &iscsiTarget{ - ISCSITarget: NewISCSITarget(bip, tgtConfDir), +// NewTarget method creates a new target based on its type. +func NewTarget(bip string, tgtConfDir string, access string) Target { + if access == iscsiAccess { + return &iscsiTarget{ + ISCSITarget: NewISCSITarget(bip, tgtConfDir), + } + } else if access == nvmeofAccess { + return &nvmeofTarget{ + NvmeofTarget: NewNvmeofTarget(bip, tgtConfDir), + } + } else { + return nil } } @@ -62,3 +73,32 @@ func (t *iscsiTarget) RemoveExport(volId string) error { tgtIqn := iscsiTgtPrefix + volId return t.RemoveISCSITarget(volId, tgtIqn) } + +type nvmeofTarget struct { + NvmeofTarget +} + +func (t *nvmeofTarget) CreateExport(volId, path, hostIp, initiator string, chapAuth []string) (map[string]interface{}, error) { + tgtNqn := nvmeofTgtPrefix + volId + if err := t.CreateNvmeofTarget(volId, tgtNqn, path, hostIp, initiator, chapAuth); err != nil { + return nil, err + } + conn := map[string]interface{}{ + "targetDiscovered": true, + "targetNQN": tgtNqn, + "targetIP": t.NvmeofTarget.(*NvmeoftgtTarget).BindIp , + "targetPort": "4420", + "discard": false, + } + if len(chapAuth) == 2 { + conn["authMethod"] = "chap" + conn["authUserName"] = chapAuth[0] + conn["authPassword"] = chapAuth[1] + } + return conn, nil +} + +func (t *nvmeofTarget) RemoveExport(volId string) error { + tgtNqn := nvmeofTgtPrefix + volId + return t.RemoveNvmeofTarget(volId, tgtNqn) +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 70ce07f30..96247d217 100755 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -139,10 +139,10 @@ func (c *Controller) CreateVolume(contx context.Context, opt *pb.CreateVolumeOpt db.UpdateVolumeStatus(ctx, db.C, opt.Id, model.VolumeError) return pb.GenericResponseError(err), err } - if opt.PoolId == "" { - opt.PoolId = polInfo.Id - opt.PoolName = polInfo.Name - } + // whether specify a pool or not, opt's poolid and pool name should be + // assigned by polInfo + opt.PoolId = polInfo.Id + opt.PoolName = polInfo.Name dockInfo, err := db.C.GetDock(ctx, polInfo.DockId) if err != nil { diff --git a/script/devsds/lib/lvm.sh b/script/devsds/lib/lvm.sh index 0956594a9..7eb312856 100755 --- a/script/devsds/lib/lvm.sh +++ b/script/devsds/lib/lvm.sh @@ -23,6 +23,9 @@ set +o xtrace # Name of the lvm volume groups to use/create for iscsi volumes VOLUME_GROUP_NAME=${VOLUME_GROUP_NAME:-opensds-volumes} DEFAULT_VOLUME_GROUP_NAME=$VOLUME_GROUP_NAME-default + +#Name of lvm nvme volume group to use/create for nvme volumes +NVME_VOLUME_GROUP_NAME=$VOLUME_GROUP_NAME-nvme # Backing file name is of the form $VOLUME_GROUP$BACKING_FILE_SUFFIX BACKING_FILE_SUFFIX=-backing-file # Default volume size @@ -30,13 +33,33 @@ VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-20G} LVM_DIR=$OPT_DIR/lvm DATA_DIR=$LVM_DIR mkdir -p $LVM_DIR +# nvme device +LVM_DEVICE=/dev/nvme0n1 osds::lvm::pkg_install(){ - sudo apt-get install -y lvm2 tgt open-iscsi + sudo apt-get install -y lvm2 tgt open-iscsi ibverbs-utils } osds::lvm::pkg_uninstall(){ - sudo apt-get purge -y lvm2 tgt open-iscsi + sudo apt-get purge -y lvm2 tgt open-iscsi ibvverbs-utils +} + +osds::lvm::nvmeofpkginstall(){ + # nvme-cli utility for nvmeof initiator + sudo wget https://github.com/linux-nvme/nvme-cli/archive/v1.7.tar.gz -O /opt/nvmecli-1.7.tar.gz + sudo tar -zxvf /opt/nvmecli-1.7.tar.gz -C /opt/ + cd /opt/nvme-cli-1.7 && sudo make && sudo make install + # nvme kernel + sudo modprobe nvmet + sudo modprobe nvme-rdma + sudo modprobe nvmet-rdma +} + +osds::lvm::nvmeofpkguninstall(){ + sudo nvme disconnect-all + sudo modprobe -r nvme-rdma + sudo modprobe -r nvmet-rdma + sudo modprobe -r nvmet } osds::lvm::create_volume_group(){ @@ -57,6 +80,37 @@ osds::lvm::create_volume_group(){ fi } +osds::lvm::create_nvme_vg(){ + local vg=$1 + local size=$2 + cap=$(parted $LVM_DEVICE unit GB print free | grep 'Free Space' | tail -n1 | awk '{print $3}') + if [ cap > '$size' ];then + # Only create if the file doesn't already exists + # create volume group and prepare kernel module + sudo mkdir -p $DATA_DIR/$vg + sudo mount $LVM_DEVICE $DATA_DIR/$vg + local backing_file=$DATA_DIR/$vg/$vg$BACKING_FILE_SUFFIX + if ! sudo vgs $vg; then + # Only create if the file doesn't already exists + [[ -f $backing_file ]] || truncate -s $size $backing_file + local vg_dev + vg_dev=`sudo losetup -f --show $backing_file` + + # Only create physical volume if it doesn't already exist + if ! sudo pvs $vg_dev; then + sudo pvcreate $vg_dev + fi + + # Only create volume group if it doesn't already exist + if ! sudo vgs $vg; then + sudo vgcreate $vg $vg_dev + fi + fi + else + echo "disk $LVM_DEVICE does not have enough space" + fi +} + osds::lvm::set_configuration(){ cat > $OPENSDS_DRIVER_CONFIG_DIR/lvm.yaml << OPENSDS_LVM_CONFIG_DOC tgtBindIp: $HOST_IP @@ -88,6 +142,26 @@ config_path = /etc/opensds/driver/lvm.yaml OPENSDS_LVM_GLOBAL_CONFIG_DOC } +osds::lvm::set_nvme_configuration(){ +cat >> $OPENSDS_DRIVER_CONFIG_DIR/lvm.yaml << OPENSDS_LVM_CONFIG_DOC + + $NVME_VOLUME_GROUP_NAME: + diskType: NL-SAS + availabilityZone: default + extras: + dataStorage: + provisioningPolicy: Thin + isSpaceEfficient: false + ioConnectivity: + accessProtocol: nvmeof + maxIOPS: 7000000 + maxBWS: 600 + advanced: + diskType: SSD + latency: 20us +OPENSDS_LVM_CONFIG_DOC +} + osds::lvm::remove_volumes() { local vg=$1 @@ -126,6 +200,20 @@ osds::lvm::clean_volume_group() { fi } +osds::lvm::clean_nvme_volume_group(){ + local nvmevg=$1 + echo "nvme pool ${nvmevg}" + osds::lvm::remove_volumes $nvmevg + osds::lvm::remove_volume_group $nvmevg + # if there is no logical volume left, it's safe to attempt a cleanup + # of the backing file + if [[ -z "$(sudo lvs --noheadings -o lv_name $nvmevg 2>/dev/null)" ]]; then + osds::lvm::clean_backing_file $DATA_DIR/$nvmevg/$nvmevg$BACKING_FILE_SUFFIX + fi + sleep 3 + sudo umount $DATA_DIR/$nvmevg +} + # osds::lvm::clean_lvm_filter() Remove the filter rule set in set_lvm_filter() @@ -172,12 +260,32 @@ osds::lvm::install() { # Remove volumes that already exist. osds::lvm::remove_volumes $vg osds::lvm::set_configuration + + # check nvmeof prerequisites + local nvmevg=$NVME_VOLUME_GROUP_NAME + if [[ -e "$LVM_DEVICE" ]]; then + phys_port_cnt=$(ibv_devinfo |grep -Eow hca_id |wc -l) + echo "The actual quantity of RDMA ports is $phys_port_cnt" + if [[ "$phys_port_cnt" < '1' ]]; then + echo "RDMA card not found" + else + osds::lvm::create_nvme_vg $nvmevg $size + osds::lvm::nvmeofpkginstall + # remove volumes that already exist + osds::lvm::remove_volumes $nvmevg + osds::lvm::set_nvme_configuration + fi + fi osds::lvm::set_lvm_filter } osds::lvm::cleanup(){ osds::lvm::clean_volume_group $DEFAULT_VOLUME_GROUP_NAME osds::lvm::clean_lvm_filter + local nvmevg=$NVME_VOLUME_GROUP_NAME + if vgs $nvmevg ; then + osds::lvm::clean_nvme_volume_group $nvmevg + fi } osds::lvm::uninstall(){ @@ -186,6 +294,7 @@ osds::lvm::uninstall(){ osds::lvm::uninstall_purge(){ echo osds::lvm::pkg_uninstall + echo osds::lvm::nvmeofpkguninstall } # Restore xtrace diff --git a/test/e2e/connector/connector.go b/test/e2e/connector/connector.go index 595917631..f40ef03cb 100644 --- a/test/e2e/connector/connector.go +++ b/test/e2e/connector/connector.go @@ -21,10 +21,12 @@ import ( "github.com/opensds/opensds/contrib/connector" _ "github.com/opensds/opensds/contrib/connector/iscsi" + _ "github.com/opensds/opensds/contrib/connector/nvmeof" ) const ( - iscsiProtocol = "iscsi" + iscsiProtocol = "iscsi" + nvmeofProtocol = "nvmeof" attachCommand = "attach" detachCommand = "detach" @@ -41,17 +43,19 @@ func main() { os.Exit(-1) } + accPro := os.Args[3] switch os.Args[1] { case attachCommand: - dev, err := connector.NewConnector(iscsiProtocol).Attach(connData) + dev, err := connector.NewConnector(accPro).Attach(connData) if err != nil { fmt.Println("Failed to attach volume to the host:", err) os.Exit(-1) } fmt.Println("Got device:", dev) break + case detachCommand: - if err := connector.NewConnector(iscsiProtocol).Detach(connData); err != nil { + if err := connector.NewConnector(accPro).Detach(connData); err != nil { fmt.Println("Failed to detach volume to the host:", err) os.Exit(-1) } diff --git a/test/e2e/e2ef_test.go b/test/e2e/e2ef_test.go index 0d5be8bf1..0260c2f99 100644 --- a/test/e2e/e2ef_test.go +++ b/test/e2e/e2ef_test.go @@ -29,6 +29,11 @@ import ( "github.com/opensds/opensds/pkg/utils/constants" ) +const ( + nvmepool = "opensds-volumes-nvme" + defaultgroup = "opensds-volumes-default" +) + var u *client.Client //init Create Profile @@ -446,8 +451,9 @@ func TestVolumeAttach(t *testing.T) { t.Error("Failed to marshal connection data:", err) return } + accPro := getatt.AccessProtocol output, err = execCmd("sudo", "./volume-connector", - "attach", string(conn)) + "attach", string(conn), accPro) if err != nil { t.Error("Failed to attach volume:", output, err) return @@ -456,7 +462,30 @@ func TestVolumeAttach(t *testing.T) { t.Log("Volume attach success!") } -//Test Case:Volume Detach +//Test Case:Delete Attachment +func TestDeleteAttach(t *testing.T) { + attc, err := PrepareAttachment(t) + if err != nil { + t.Error("Prepare Attachment Fail!", err) + return + } + defer DeleteVolume(attc.VolumeId) + err = u.DeleteVolumeAttachment(attc.Id, nil) + if err != nil { + t.Error("Delete Attachment Fail", err) + return + } + _, err = u.GetVolumeAttachment(attc.Id) + t.Log("err:", err) + if strings.Contains(err.Error(), "can't find") { + t.Log("Delete attachment Success") + return + } else { + t.Error("Delete Attachment Fail!", err) + } +} + +//Test Case:Delete Attachment func TestVolumeDetach(t *testing.T) { attc, err := PrepareAttachment(t) if err != nil { @@ -476,6 +505,7 @@ func TestVolumeDetach(t *testing.T) { } t.Log("Begin to Scan volume:") + t.Log("getatt.Accessprotocol", getatt.AccessProtocol) t.Log("getatt.Metadata", getatt.ConnectionData) //execute bin file @@ -486,8 +516,9 @@ func TestVolumeDetach(t *testing.T) { } // attach first, then detach + accPro := getatt.AccessProtocol output, err := execCmd("sudo", "./volume-connector", - "attach", string(conn)) + "attach", string(conn), accPro) if err != nil { t.Error("Failed to attach volume:", output, err) return @@ -496,7 +527,7 @@ func TestVolumeDetach(t *testing.T) { t.Log(output) output, err = execCmd("sudo", "./volume-connector", - "detach", string(conn)) + "detach", string(conn), accPro) if err != nil { t.Error("Failed to detach volume:", output, err) return @@ -505,29 +536,220 @@ func TestVolumeDetach(t *testing.T) { t.Log("Volume Detach Success!") } -//Test Case:Delete Attachment -func TestDeleteAttach(t *testing.T) { - attc, err := PrepareAttachment(t) +//Test for nvmeof connection +func TestNvmeofAttachIssues(t *testing.T) { + // pool list get nvme pool + pols, err := u.ListPools() if err != nil { - t.Error("Prepare Attachment Fail!", err) + t.Error("list pools failed:", err) + return + } + polId := "" + for _, p := range pols { + if p.Name == nvmepool { + polId = p.Id + t.Log("nvme pool id is: ", polId) + break + } + } + if polId == "" { + t.Log("no nvme pool ") + return + } + //PrepareNvmeVolume() + err = CreateNvmeofAttach(t) + if err != nil { + t.Error("create nvmeof attachment fail", err) + return + } + err = ListNvmeofAttachment(t) + if err != nil { + t.Error("list nvmeof attachment fail", err) + return + } + err = ShowNvmeofAttachDetail(t) + if err != nil { + t.Error("show nvmeof attachment fail", err) + return + } + err = NvmeofVolumeAttach(t) + if err != nil { + t.Error("connect nvmeof attachment fail", err) return } + + err = DeleteNvmeofAttach(t) + if err != nil { + t.Error("delete nvmeof attachment fail", err) + return + } + + t.Log("nvmeof attach issues success") +} + +func CreateNvmeofAttach(t *testing.T) error { + vol, err := PrepareNvmeVolume() + if err != nil { + t.Error("Prepare nvme Volume Fail", err) + return err + } + defer DeleteVolume(vol.Id) + var body = &model.VolumeAttachmentSpec{ + VolumeId: vol.Id, + HostInfo: model.HostInfo{}, + } + attc, err := u.CreateVolumeAttachment(body) + if err != nil { + t.Error("create nvmeof volume attachment failed:", err) + return err + } + defer DeleteAttachment(attc.Id) + attrs, _ := json.MarshalIndent(attc, "", " ") + t.Log(string(attrs)) + t.Log("Create nvmeof Volume Attachment Success") + return nil +} + +func ListNvmeofAttachment(t *testing.T) error { + attc, err := PrepareNvmeofAttachment(t) + if err != nil { + t.Error("Prepare nvmeof Attachment Fail!", err) + return err + } + defer DeleteVolume(attc.VolumeId) + defer DeleteAttachment(attc.Id) + atts, err := u.ListVolumeAttachments() + if err != nil { + t.Error("List nvmeof Attachment Error!", err) + return err + } + attli, _ := json.MarshalIndent(atts, "", " ") + t.Log(string(attli)) + t.Log("List nvmeof Attachment Success!") + return nil +} + +func ShowNvmeofAttachDetail(t *testing.T) error { + attc, err := PrepareNvmeofAttachment(t) + if err != nil { + t.Error("Prepare Attachment Fail!", err) + return err + } + defer DeleteVolume(attc.VolumeId) + defer DeleteAttachment(attc.Id) + + getatt, err := u.GetVolumeAttachment(attc.Id) + if err != nil || getatt.Status != "available" { + t.Error("Get Volume Attachment Detail Fail!", err) + return err + } + t.Log("Get Volume Attachment Detail Success") + return nil +} + +func DeleteNvmeofAttach(t *testing.T) error { + attc, err := PrepareNvmeofAttachment(t) + if err != nil { + t.Error("Prepare Attachment Fail!", err) + return err + } defer DeleteVolume(attc.VolumeId) err = u.DeleteVolumeAttachment(attc.Id, nil) if err != nil { - t.Error("Delete Attachment Fail", err) - return + t.Error("Delete nvme Attachment Fail", err) + return err } _, err = u.GetVolumeAttachment(attc.Id) t.Log("err:", err) if strings.Contains(err.Error(), "can't find") { t.Log("Delete attachment Success") - return + return nil } else { t.Error("Delete Attachment Fail!", err) + return err } } +//Test Case:Nvmeof Volume Attach +func NvmeofVolumeAttach(t *testing.T) error { + attc, err := PrepareNvmeofAttachment(t) + if err != nil { + t.Error("Prepare Attachment Fail:", err) + return err + } + defer DeleteVolume(attc.VolumeId) + defer DeleteAttachment(attc.Id) + + getatt, err := u.GetVolumeAttachment(attc.Id) + if err != nil || getatt.Status != "available" { + t.Errorf("attachment(%s) is not available: %v", attc.Id, err) + return err + } + + t.Log("Begin to Scan Volume:") + t.Log("getatt.AccessProtocol", getatt.AccessProtocol) + t.Log("getatt.Metadata", getatt.ConnectionData) + + output, _ := execCmd("/bin/bash", "-c", "ps -ef") + t.Log(output) + //execute bin file + conn, err := json.Marshal(&getatt.ConnectionData) + if err != nil { + t.Error("Failed to marshal connection data:", err) + return err + } + accPro := getatt.AccessProtocol + output, err = execCmd("sudo", "./volume-connector", + "attach", string(conn), accPro) + if err != nil { + t.Error("Failed to attach volume:", output, err) + return err + } + t.Log(output) + t.Log("Nvmeof Volume attach yoyo success!") + // detach it + err = NvmeofVolumeDetach(t, attc) + if err != nil { + t.Error("detach failed") + return err + } + return nil +} + +//Test Case:Delete Attachment +func NvmeofVolumeDetach(t *testing.T, attc *model.VolumeAttachmentSpec) error { + defer DeleteVolume(attc.VolumeId) + defer DeleteAttachment(attc.Id) + + getatt, err := u.GetVolumeAttachment(attc.Id) + if err != nil || getatt.Status != "available" { + t.Errorf("attachment(%s) is not available: %v", attc.Id, err) + return err + } + + t.Log("Begin to Scan volume:") + t.Log("getatt.AccessProtocol", getatt.AccessProtocol) + t.Log("getatt.Metadata", getatt.ConnectionData) + + //execute bin file + conn, err := json.Marshal(&getatt.ConnectionData) + if err != nil { + t.Error("Failed to marshal connection data:", err) + return err + } + accPro := getatt.AccessProtocol + output, err := execCmd("sudo", "./volume-connector", + "detach", string(conn), accPro) + if err != nil { + t.Error("Failed to detach volume:", attc.VolumeId, output, err) + return err + } + t.Log(output) + t.Log("Volume Detach Success!") + return nil +} + +// cmd func execCmd(name string, arg ...string) (string, error) { fmt.Printf("Command: %s %s:\n", name, strings.Join(arg, " ")) info, err := exec.Command(name, arg...).CombinedOutput() @@ -556,6 +778,28 @@ func PrepareAttachment(t *testing.T) (*model.VolumeAttachmentSpec, error) { return attc, nil } +// prepare nvmeof attachment +func PrepareNvmeofAttachment(t *testing.T) (*model.VolumeAttachmentSpec, error) { + vol, err := PrepareNvmeVolume() + if err != nil { + t.Error("Prepare nvmeof Volume Fail", err) + return nil, err + } + + var body = &model.VolumeAttachmentSpec{ + VolumeId: vol.Id, + HostInfo: model.HostInfo{}, + } + attc, err := u.CreateVolumeAttachment(body) + if err != nil { + t.Error("prepare volume attachment failed:", err) + return nil, err + } + + t.Log("prepare nvmeof Volume Attachment Success") + return attc, nil +} + //delete attachment func DeleteAttachment(attId string) error { err := u.DeleteVolumeAttachment(attId, nil) @@ -625,6 +869,38 @@ func PrepareVolume() (*model.VolumeSpec, error) { return create, nil } +//nvme volume is essential for nvmeof attachment ,so the volume should be created in nvme pool +func PrepareNvmeVolume() (*model.VolumeSpec, error) { + // get poolid + pols, err := u.ListPools() + if err != nil { + return nil, err + } + polId := "" + for _, p := range pols { + if p.Name == nvmepool { + polId = p.Id + break + } + } + if polId == "" { + return nil, nil + } + + //create volume in specified nvme pool + var volbody = &model.VolumeSpec{ + Name: "nvme flowTest", + Description: "This a test for nvme flow", + Size: int64(1), + PoolId: polId, + } + create, err := u.CreateVolume(volbody) + if err != nil { + return nil, err + } + return create, nil +} + //Delete volume after test func DeleteVolume(volId string) error { err := u.DeleteVolume(volId, nil)