diff --git a/contrib/connector/connector.go b/contrib/connector/connector.go index 13bb7abdd..d54cb5b5b 100644 --- a/contrib/connector/connector.go +++ b/contrib/connector/connector.go @@ -30,6 +30,9 @@ const ( Iqn = "iqn" RbdDriver = "rbd" + + NvmeofDriver = "nvmeof" + Nqn = "nqn" ) // Connector implementation diff --git a/contrib/connector/nvmeof/nvmeof.go b/contrib/connector/nvmeof/nvmeof.go new file mode 100644 index 000000000..590e8f937 --- /dev/null +++ b/contrib/connector/nvmeof/nvmeof.go @@ -0,0 +1,40 @@ +// Copyright (c) 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvmeof + +import ( + "github.com/opensds/opensds/contrib/connector" +) + +type Nvmeof struct{} + +func init() { + connector.RegisterConnector(connector.NvmeofDriver, &Nvmeof{}) + connector.ExecCmd("modprobe", "nvme-rdma") +} + +func (nof *Nvmeof) Attach(conn map[string]interface{}) (string, error) { + return Connect(conn) +} + +func (nof *Nvmeof) Detach(conn map[string]interface{}) error { + NvmeofCon := ParseNvmeofConnectInfo(conn) + return DisConnect(NvmeofCon.Nqn) +} + +// GetInitiatorInfo implementation +func (nof *Nvmeof) GetInitiatorInfo() (string, error) { + return getInitiatorInfo() +} diff --git a/contrib/connector/nvmeof/nvmeof_helper.go b/contrib/connector/nvmeof/nvmeof_helper.go new file mode 100644 index 000000000..7eda124ff --- /dev/null +++ b/contrib/connector/nvmeof/nvmeof_helper.go @@ -0,0 +1,199 @@ +// Copyright (c) 2019 Intel Corporation, Ltd. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvmeof + +import ( + "errors" + "log" + "strings" + "time" + + "github.com/mitchellh/mapstructure" + "github.com/opensds/opensds/contrib/connector" +) + +const ( + iniNvmePrefix = "nqn.ini." +) + +// ConnectorInfo define +type ConnectorInfo struct { + Nqn string `mapstructure:"targetNQN"` //NVMe subsystem name to the volume to be connected + TgtPort string `mapstructure:"targetPort"` //NVMe target port that hosts the nqn sybsystem + TgtPortal string `mapstructure:"targetIP"` //NVMe target ip that hosts the nqn sybsystem + TranType string `mapstructure:"transporType "` // Nvme transport type + HostNqn string `mapstructure:"hostNqn"` // host nqn +} + +////////////////////////////////////////////////////////////////////////////////////////// +// Refer some codes from: https://github.intel.com/yingxinc/cinder-rsd-os-brick // +////////////////////////////////////////////////////////////////////////////////////////// + +// GetInitiator returns all the Nvmeof UUID +func GetInitiator() ([]string, error) { + res, err := connector.ExecCmd("dmidecode") + nqns := []string{} + if err != nil { + log.Printf("Unable to execute dmidecode,Error encountered gathering Nvmeof UUID: %v\n", err) + return nqns, nil + } + + lines := strings.Split(string(res), "\n") + for _, l := range lines { + if strings.Contains(l, "UUID:") { + tmp := iniNvmePrefix + strings.Split(l, ":")[1] + nqns = append(nqns, tmp) + log.Printf("Found the following nqns: %s", nqns) + return nqns, nil + } + } + log.Println("can not find any nqn initiator") + return nqns, errors.New("can not find any nqn initiator") +} + +func getInitiatorInfo() (string, error) { + + initiators, err := GetInitiator() + if err != nil { + return "", err + } + + if len(initiators) == 0 { + return "", errors.New("no nqn found") + } + + if len(initiators) > 1 { + return "", errors.New("the number of nqn is wrong") + } + + hostName, err := connector.GetHostName() + if err != nil { + return "", errors.New("can not get hostname") + } + + info := initiators[0] + "." + hostName + return info, nil +} + +// GetNvmeDevice get all the nvme devices +func GetNvmeDevice() (map[string]int, error) { + nvmeDevice := make(map[string]int) + pattern := "/dev/nvme" + Npath, err := connector.ExecCmd("nvme", "list") + if err != nil { + return nvmeDevice, err + } + log.Println("nvme list succeed") + lines := strings.Split(string(Npath), "\n") + for _, l := range lines { + if strings.Contains(l, pattern) { + name := strings.Split(l, " ")[0] + nvmeDevice[name] = 1 + } + } + return nvmeDevice, err +} + +// GetNvmeSubsystems :list connected target name +func GetNvmeSubsystems() (map[string]int, error) { + nqn := make(map[string]int) + res, err := connector.ExecCmd("nvme", "list-subsys") + if err != nil { + return nqn, err + } + + lines := strings.Split(string(res), "\n") + for _, l := range lines { + if strings.Contains(l, "NQN=") { + name := strings.Split(l, "NQN=")[1] + nqn[name] = 1 + } + } + + log.Printf("Found the following NQN: %s", res) + return nqn, nil +} + +// Discovery NVMe-OF target +func Discovery(connMap map[string]interface{}) error { + conn := ParseNvmeofConnectInfo(connMap) + targetip := conn.TgtPortal + targetport := conn.TgtPort + info, err := connector.ExecCmd("nvme", "discover", "-t", "rdma", "-a", targetip, "-s", targetport) + if err != nil { + log.Println("Error encountered in send targets:%s, err: %v", string(info), err) + return err + } + return nil +} + +// Connect NVMe-OF Target ,return the new target device path in this node +func Connect(connMap map[string]interface{}) (string, error) { + CurrentNvmeDevice, _ := GetNvmeDevice() + conn := ParseNvmeofConnectInfo(connMap) + connNqn := conn.Nqn + targetPortal := conn.TgtPortal + port := conn.TgtPort + nvmeTransportType := "rdma" + log.Printf("conn information: ", connNqn, ",", targetPortal, ",", port) + + _, err := connector.ExecCmd("nvme", "connect", "-t", + nvmeTransportType, "-n", connNqn, "-a", targetPortal, "-s", port) + if err != nil { + log.Println("Failed to connect to NVMe nqn :", connNqn) + return "", err + } + + for retry := 0; retry < 10; retry++ { + allNvmeDevices, _ := GetNvmeDevice() + for p, _ := range allNvmeDevices { + if _, ok := CurrentNvmeDevice[p]; !ok { + log.Printf("NVMe device to be connected to is : %v", p) + return p, nil + } + time.Sleep(time.Second) + } + } + return "", errors.New("could not connect volume: Timeout after 10s") +} + +// DisConnect nvme device by name +func DisConnect(nqn string) error { + currentNvmeNames, err := GetNvmeSubsystems() + if err != nil { + log.Println("can not get nvme device") + return err + } + if _, ok := currentNvmeNames[nqn]; !ok { + log.Println("Trying to disconnect nqn" + nqn + + "is not connected.") + return errors.New("device path not found ") + } + + _, err = connector.ExecCmd("nvme", "disconnect", "-n", nqn) + if err != nil { + log.Println("could not disconnect nvme nqn : ", nqn) + return err + } + log.Println(" disconnect nvme nqn : ", nqn) + return nil +} + +// ParseNvmeofConnectInfo decode +func ParseNvmeofConnectInfo(connectInfo map[string]interface{}) *ConnectorInfo { + var con ConnectorInfo + mapstructure.Decode(connectInfo, &con) + return &con +} diff --git a/contrib/drivers/lvm/lvm.go b/contrib/drivers/lvm/lvm.go index 4236380d3..9d37004dc 100755 --- a/contrib/drivers/lvm/lvm.go +++ b/contrib/drivers/lvm/lvm.go @@ -42,6 +42,9 @@ const ( snapshotPrefix = "_snapshot-" blocksize = 4096 sizeShiftBit = 30 + opensdsnvmepool = "opensds-nvmegroup" + nvmeofAccess = "nvmeof" + iscsiAccess = "iscsi" ) const ( @@ -249,7 +252,11 @@ func (d *Driver) InitializeConnection(opt *pb.CreateVolumeAttachmentOpts) (*mode if d.conf.EnableChapAuth { chapAuth = []string{utils.RandSeqWithAlnum(20), utils.RandSeqWithAlnum(16)} } - t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir) + + // create target according to the pool's access protocol + accPro := opt.AccessProtocol + log.Info("accpro:", accPro) + t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir, accPro) expt, err := t.CreateExport(opt.GetVolumeId(), lvPath, hostIP, initiator, chapAuth) if err != nil { log.Error("Failed to initialize connection of logic volume:", err) @@ -257,15 +264,16 @@ func (d *Driver) InitializeConnection(opt *pb.CreateVolumeAttachmentOpts) (*mode } return &model.ConnectionInfo{ - DriverVolumeType: ISCSIProtocol, + DriverVolumeType: accPro, ConnectionData: expt, }, nil } func (d *Driver) TerminateConnection(opt *pb.DeleteVolumeAttachmentOpts) error { - t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir) + accPro := opt.AccessProtocol + t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir, accPro) if err := t.RemoveExport(opt.GetVolumeId()); err != nil { - log.Error("Failed to initialize connection of logic volume:", err) + log.Error("failed to terminate connection of logic volume:", err) return err } return nil @@ -406,6 +414,11 @@ func (d *Driver) CreateSnapshot(opt *pb.CreateVolumeSnapshotOpts) (snap *model.V metadata := map[string]string{KLvsPath: lvsPath} if bucket, ok := opt.Metadata["bucket"]; ok { + //nvmet right now can not support snap volume serve as nvme target + if vg == opensdsnvmepool { + log.Infof("nvmet right now can not support snap volume serve as nvme target") + log.Infof("still store in nvme pool but initialize connection by iscsi protocol") + } mountPoint, info, err := d.AttachSnapshot(opt.GetId(), lvsPath) if err != nil { d.cli.Delete(snapName, vg) @@ -524,7 +537,13 @@ func (d *Driver) InitializeSnapshotConnection(opt *pb.CreateSnapshotAttachmentOp chapAuth = []string{utils.RandSeqWithAlnum(20), utils.RandSeqWithAlnum(16)} } - t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir) + accPro := opt.AccessProtocol + if accPro == nvmeofAccess { + log.Infof("nvmet right now can not support snap volume serve as nvme target") + log.Infof("still create snapshot connection by iscsi") + accPro = iscsiAccess + } + t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir, accPro) data, err := t.CreateExport(opt.GetSnapshotId(), lvsPath, hostIP, initiator, chapAuth) if err != nil { log.Error("Failed to initialize snapshot connection of logic volume:", err) @@ -532,13 +551,20 @@ func (d *Driver) InitializeSnapshotConnection(opt *pb.CreateSnapshotAttachmentOp } return &model.ConnectionInfo{ - DriverVolumeType: ISCSIProtocol, + DriverVolumeType: accPro, ConnectionData: data, }, nil } func (d *Driver) TerminateSnapshotConnection(opt *pb.DeleteSnapshotAttachmentOpts) error { - t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir) + accPro := opt.AccessProtocol + if accPro == nvmeofAccess{ + log.Infof("nvmet right now can not support snap volume serve as nvme target") + log.Infof("still create snapshot connection by iscsi") + accPro = iscsiAccess + } + log.Info("terminate snapshot conn") + t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir, accPro) if err := t.RemoveExport(opt.GetSnapshotId()); err != nil { log.Error("Failed to terminate snapshot connection of logic volume:", err) return err diff --git a/contrib/drivers/lvm/targets/nvmeof.go b/contrib/drivers/lvm/targets/nvmeof.go new file mode 100755 index 000000000..65c8122f4 --- /dev/null +++ b/contrib/drivers/lvm/targets/nvmeof.go @@ -0,0 +1,293 @@ +// Copyright (c) 2019 Intel Corporation, Ltd. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package targets + +import ( + "bytes" + "errors" + "io" + "os" + "os/exec" + "strconv" + "strings" + + log "github.com/golang/glog" + "github.com/opensds/opensds/pkg/utils" +) + +const ( + opensdsNvmeofPrefix = "opensds-Nvmeof" + NvmetDir = "/sys/kernel/config/nvmet" +) + +type NvmeofTarget interface { + CreateNvmeofTarget(volId, tgtIqn, path, hostIp, initiator string, chapAuth []string) error + GetNvmeofTarget(iqn string) int + RemoveNvmeofTarget(volId, iqn string) error +} + +func NewNvmeofTarget(bip, tgtConfDir string) NvmeofTarget { + return &NvmeoftgtTarget{ + TgtConfDir: tgtConfDir, + BindIp: bip, + } +} + +type NvmeoftgtTarget struct { + BindIp string + TgtConfDir string +} + +func (t *NvmeoftgtTarget) init() { + t.execCmd("modprobe", "nvmet") + t.execCmd("modprobe", "nvmet-rdma") +} + +func (t *NvmeoftgtTarget) getTgtConfPath(volId string) string { + return NvmetDir + "/" + opensdsNvmeofPrefix + volId +} + +func (t *NvmeoftgtTarget) CreateNvmeofTarget(volId, tgtNqn, path, hostIp, initiator string, chapAuth []string) error { + + if exist, _ := utils.PathExists(NvmetDir); !exist { + os.MkdirAll(NvmetDir, 0755) + } + sysdir := NvmetDir + "/subsystems/" + tgtNqn + if exist, _ := utils.PathExists(sysdir); !exist { + os.MkdirAll(sysdir, 0755) + } + + var err error + if initiator == "ALL" { + // echo 1 > attr_allow_any_host + attrfile := sysdir + "/attr_allow_any_host" + content := "1" + err = t.WriteWithIo(attrfile, content) + if err != nil { + log.Errorf("can not set attr_allow_any_host ") + return err + } + } else { + // allow specific initiators to connect to this target + var initiatorInfo = "initiator:" + hostIp + ":" + initiator + hostpath := NvmetDir + "/hosts" + if exist, _ := utils.PathExists(hostpath); !exist { + os.MkdirAll(hostpath, 0755) + } + + hostDir := hostpath + "/" + initiatorInfo + if exist, _ := utils.PathExists(hostDir); !exist { + os.MkdirAll(hostDir, 0755) + } + + // create symbolic link of host + hostsys := sysdir + "/allowed_hosts/" + _, err = t.execCmd("ln", "-s", hostDir, hostsys) + if err != nil { + log.Errorf("Fail to create host link: " + initiatorInfo) + return err + } + } + + // get volume namespaceid + namespaceid := t.Getnamespaceid(volId) + if namespaceid == "" { + return errors.New("null namesapce") + } + namespace := sysdir + "/namespaces/" + namespaceid + if exist, _ := utils.PathExists(namespace); !exist { + os.MkdirAll(namespace, 0755) + } + + // volid as device path + devpath := namespace + "/device_path" + err = t.WriteWithIo(devpath, path) + if err != nil { + log.Errorf("Fail to set device path") + return err + } + + enablepath := namespace + "/enable" + err = t.WriteWithIo(enablepath, "1") + if err != nil { + log.Errorf("Fail to set device path") + return err + } + + //create port + portid := 1 + portspath := NvmetDir + "/ports/" + strconv.Itoa(portid) + if exist, _ := utils.PathExists(portspath); !exist { + //log.Errorf(portspath) + os.MkdirAll(portspath, 0755) + } + + // get target ip + ippath := portspath + "/addr_traddr" + ip, err := t.execCmd("hostname", "-I") + if err != nil { + log.Errorf("fail to get target ipv4 address") + return err + } + // Set nvmeof parameters, rightnow only supports rdma + // if built on virtual machine the return string ip may contain several ip addresses + ip = strings.Split(ip, " ")[0] + err = t.WriteWithIo(ippath, ip) + if err != nil { + log.Errorf("Fail to set target ip") + return err + } + + trtypepath := portspath + "/addr_trtype" + err = t.WriteWithIo(trtypepath, "rdma") + if err != nil { + log.Errorf("Fail to set rdma type") + return err + } + + trsvcidpath := portspath + "/addr_trsvcid" + err = t.WriteWithIo(trsvcidpath, "4420") + if err != nil { + log.Errorf("Fail to set ip port") + return err + } + + adrfampath := portspath + "/addr_adrfam" + err = t.WriteWithIo(adrfampath, "ipv4") + if err != nil { + log.Errorf("Fail to set ip family") + return err + } + + // create a soft link + portssub := portspath + "/subsystems/" + tgtNqn + _, err = t.execCmd("ln", "-s", sysdir, portssub) + if err != nil { + log.Errorf("Fail to create link") + return err + } + + // check + info, err := t.execCmd("dmesg", `|grep "enabling port"`) + if err != nil || info == "" { + log.Errorf("nvme target is not listening on the port") + return err + } + log.Info("create nvme target") + return nil +} + +func (t *NvmeoftgtTarget) GetNvmeofTarget(nqn string) int { + _, err := t.execCmd("cd", "/sys/kernel/config/nvmet/subsystems") + if err != nil { + log.Errorf("Fail to exec to enter nvme target dir:%v", err) + return -1 + } + _, err = t.execCmd("cd", nqn) + if err != nil { + log.Errorf("Fail to exec to display nvme target :%v", err) + return -1 + } + return 0 +} + +func (t *NvmeoftgtTarget) RemoveNvmeofTarget(volId, nqn string) error { + log.Info("removing target",nqn) + tgtConfPath := NvmetDir + "/subsystems/" + nqn + if exist, _ := utils.PathExists(tgtConfPath); !exist { + log.Warningf("Volume path %s does not exist, nothing to remove.", tgtConfPath) + return nil + } + + // port's link has to be removed first or the subsystem cannot be removed + portpath := NvmetDir + "/ports/1/subsystems/" + nqn + info, err := t.execCmd("rm", "-f", portpath) + if err != nil { + log.Errorf("can not rm port") + log.Errorf(info) + return err + } + + // remove namespace + ns := t.Getnamespaceid(volId) + if ns == "" { + log.Errorf("can not find volume %s's namespace",volId) + return errors.New("null namespace") + } + naspPath := NvmetDir + "/subsystems/" + nqn + "/namespaces/" + ns + info, err = t.execCmd("rmdir", naspPath) + if err != nil { + log.Errorf("can not rm nasp") + return err + } + + // remove namespaces ; if it allows all initiators ,then this dir should be empty + // if it allow specific hosts ,then here remove all the hosts + cmd := "rm -f " + NvmetDir + "/subsystems/" + nqn + "/allowed_hosts/" + "*" + info, err = t.execBash(cmd) + if err != nil { + log.Errorf("can not rm allowed hosts") + log.Errorf(info) + return err + } + + // remove subsystem + syspath := NvmetDir + "/subsystems/" + nqn + info, err = t.execCmd("rmdir", syspath) + if err != nil { + log.Errorf("can not rm subsys") + return err + } + return nil +} + +func (*NvmeoftgtTarget) execCmd(name string, cmd ...string) (string, error) { + ret, err := exec.Command(name, cmd...).Output() + if err != nil { + log.Errorf("error info: %v", err) + } + return string(ret), err +} + +func (*NvmeoftgtTarget) execBash(name string) (string, error) { + ret, err := exec.Command("/bin/sh", "-c", name).Output() + if err != nil { + log.Errorf("error info in sh %v ", err) + } + return string(ret), err +} + +func (*NvmeoftgtTarget) WriteWithIo(name, content string) error { + fileObj, err := os.OpenFile(name, os.O_RDWR, 0644) + if err != nil { + log.Errorf("Failed to open the file %v", err) + return err + } + if _, err := io.WriteString(fileObj, content); err == nil { + log.Infof("Successful appending to the file with os.OpenFile and io.WriteString.%s", content) + return nil + } + return err +} + +func (t *NvmeoftgtTarget) Getnamespaceid(volId string) string { + var buffer bytes.Buffer + for _, rune := range volId { + if rune >= '0' && rune <= '9' { + buffer.WriteRune(rune) + } + } + return buffer.String()[0:2] +} diff --git a/contrib/drivers/lvm/targets/targets.go b/contrib/drivers/lvm/targets/targets.go index f0979543a..3471ecaaa 100755 --- a/contrib/drivers/lvm/targets/targets.go +++ b/contrib/drivers/lvm/targets/targets.go @@ -15,7 +15,10 @@ package targets const ( - iscsiTgtPrefix = "iqn.2017-10.io.opensds:" + iscsiTgtPrefix = "iqn.2017-10.io.opensds:" + nvmeofTgtPrefix = "nqn.2019-01.com.opensds:nvme:" + iscsiAccess = "iscsi" + nvmeofAccess = "nvmeof" ) // Target is an interface for exposing some operations of different targets, @@ -26,10 +29,18 @@ type Target interface { RemoveExport(volId string) error } -// NewTarget method creates a new iscsi target. -func NewTarget(bip string, tgtConfDir string) Target { - return &iscsiTarget{ - ISCSITarget: NewISCSITarget(bip, tgtConfDir), +// NewTarget method creates a new target based on its type. +func NewTarget(bip string, tgtConfDir string, access string) Target { + if access == iscsiAccess { + return &iscsiTarget{ + ISCSITarget: NewISCSITarget(bip, tgtConfDir), + } + } else if access == nvmeofAccess { + return &nvmeofTarget{ + NvmeofTarget: NewNvmeofTarget(bip, tgtConfDir), + } + } else { + return nil } } @@ -62,3 +73,32 @@ func (t *iscsiTarget) RemoveExport(volId string) error { tgtIqn := iscsiTgtPrefix + volId return t.RemoveISCSITarget(volId, tgtIqn) } + +type nvmeofTarget struct { + NvmeofTarget +} + +func (t *nvmeofTarget) CreateExport(volId, path, hostIp, initiator string, chapAuth []string) (map[string]interface{}, error) { + tgtNqn := nvmeofTgtPrefix + volId + if err := t.CreateNvmeofTarget(volId, tgtNqn, path, hostIp, initiator, chapAuth); err != nil { + return nil, err + } + conn := map[string]interface{}{ + "targetDiscovered": true, + "targetNQN": tgtNqn, + "targetIP": t.NvmeofTarget.(*NvmeoftgtTarget).BindIp , + "targetPort": "4420", + "discard": false, + } + if len(chapAuth) == 2 { + conn["authMethod"] = "chap" + conn["authUserName"] = chapAuth[0] + conn["authPassword"] = chapAuth[1] + } + return conn, nil +} + +func (t *nvmeofTarget) RemoveExport(volId string) error { + tgtNqn := nvmeofTgtPrefix + volId + return t.RemoveNvmeofTarget(volId, tgtNqn) +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 70ce07f30..96247d217 100755 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -139,10 +139,10 @@ func (c *Controller) CreateVolume(contx context.Context, opt *pb.CreateVolumeOpt db.UpdateVolumeStatus(ctx, db.C, opt.Id, model.VolumeError) return pb.GenericResponseError(err), err } - if opt.PoolId == "" { - opt.PoolId = polInfo.Id - opt.PoolName = polInfo.Name - } + // whether specify a pool or not, opt's poolid and pool name should be + // assigned by polInfo + opt.PoolId = polInfo.Id + opt.PoolName = polInfo.Name dockInfo, err := db.C.GetDock(ctx, polInfo.DockId) if err != nil { diff --git a/script/devsds/lib/lvm.sh b/script/devsds/lib/lvm.sh index 0956594a9..7eb312856 100755 --- a/script/devsds/lib/lvm.sh +++ b/script/devsds/lib/lvm.sh @@ -23,6 +23,9 @@ set +o xtrace # Name of the lvm volume groups to use/create for iscsi volumes VOLUME_GROUP_NAME=${VOLUME_GROUP_NAME:-opensds-volumes} DEFAULT_VOLUME_GROUP_NAME=$VOLUME_GROUP_NAME-default + +#Name of lvm nvme volume group to use/create for nvme volumes +NVME_VOLUME_GROUP_NAME=$VOLUME_GROUP_NAME-nvme # Backing file name is of the form $VOLUME_GROUP$BACKING_FILE_SUFFIX BACKING_FILE_SUFFIX=-backing-file # Default volume size @@ -30,13 +33,33 @@ VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-20G} LVM_DIR=$OPT_DIR/lvm DATA_DIR=$LVM_DIR mkdir -p $LVM_DIR +# nvme device +LVM_DEVICE=/dev/nvme0n1 osds::lvm::pkg_install(){ - sudo apt-get install -y lvm2 tgt open-iscsi + sudo apt-get install -y lvm2 tgt open-iscsi ibverbs-utils } osds::lvm::pkg_uninstall(){ - sudo apt-get purge -y lvm2 tgt open-iscsi + sudo apt-get purge -y lvm2 tgt open-iscsi ibvverbs-utils +} + +osds::lvm::nvmeofpkginstall(){ + # nvme-cli utility for nvmeof initiator + sudo wget https://github.com/linux-nvme/nvme-cli/archive/v1.7.tar.gz -O /opt/nvmecli-1.7.tar.gz + sudo tar -zxvf /opt/nvmecli-1.7.tar.gz -C /opt/ + cd /opt/nvme-cli-1.7 && sudo make && sudo make install + # nvme kernel + sudo modprobe nvmet + sudo modprobe nvme-rdma + sudo modprobe nvmet-rdma +} + +osds::lvm::nvmeofpkguninstall(){ + sudo nvme disconnect-all + sudo modprobe -r nvme-rdma + sudo modprobe -r nvmet-rdma + sudo modprobe -r nvmet } osds::lvm::create_volume_group(){ @@ -57,6 +80,37 @@ osds::lvm::create_volume_group(){ fi } +osds::lvm::create_nvme_vg(){ + local vg=$1 + local size=$2 + cap=$(parted $LVM_DEVICE unit GB print free | grep 'Free Space' | tail -n1 | awk '{print $3}') + if [ cap > '$size' ];then + # Only create if the file doesn't already exists + # create volume group and prepare kernel module + sudo mkdir -p $DATA_DIR/$vg + sudo mount $LVM_DEVICE $DATA_DIR/$vg + local backing_file=$DATA_DIR/$vg/$vg$BACKING_FILE_SUFFIX + if ! sudo vgs $vg; then + # Only create if the file doesn't already exists + [[ -f $backing_file ]] || truncate -s $size $backing_file + local vg_dev + vg_dev=`sudo losetup -f --show $backing_file` + + # Only create physical volume if it doesn't already exist + if ! sudo pvs $vg_dev; then + sudo pvcreate $vg_dev + fi + + # Only create volume group if it doesn't already exist + if ! sudo vgs $vg; then + sudo vgcreate $vg $vg_dev + fi + fi + else + echo "disk $LVM_DEVICE does not have enough space" + fi +} + osds::lvm::set_configuration(){ cat > $OPENSDS_DRIVER_CONFIG_DIR/lvm.yaml << OPENSDS_LVM_CONFIG_DOC tgtBindIp: $HOST_IP @@ -88,6 +142,26 @@ config_path = /etc/opensds/driver/lvm.yaml OPENSDS_LVM_GLOBAL_CONFIG_DOC } +osds::lvm::set_nvme_configuration(){ +cat >> $OPENSDS_DRIVER_CONFIG_DIR/lvm.yaml << OPENSDS_LVM_CONFIG_DOC + + $NVME_VOLUME_GROUP_NAME: + diskType: NL-SAS + availabilityZone: default + extras: + dataStorage: + provisioningPolicy: Thin + isSpaceEfficient: false + ioConnectivity: + accessProtocol: nvmeof + maxIOPS: 7000000 + maxBWS: 600 + advanced: + diskType: SSD + latency: 20us +OPENSDS_LVM_CONFIG_DOC +} + osds::lvm::remove_volumes() { local vg=$1 @@ -126,6 +200,20 @@ osds::lvm::clean_volume_group() { fi } +osds::lvm::clean_nvme_volume_group(){ + local nvmevg=$1 + echo "nvme pool ${nvmevg}" + osds::lvm::remove_volumes $nvmevg + osds::lvm::remove_volume_group $nvmevg + # if there is no logical volume left, it's safe to attempt a cleanup + # of the backing file + if [[ -z "$(sudo lvs --noheadings -o lv_name $nvmevg 2>/dev/null)" ]]; then + osds::lvm::clean_backing_file $DATA_DIR/$nvmevg/$nvmevg$BACKING_FILE_SUFFIX + fi + sleep 3 + sudo umount $DATA_DIR/$nvmevg +} + # osds::lvm::clean_lvm_filter() Remove the filter rule set in set_lvm_filter() @@ -172,12 +260,32 @@ osds::lvm::install() { # Remove volumes that already exist. osds::lvm::remove_volumes $vg osds::lvm::set_configuration + + # check nvmeof prerequisites + local nvmevg=$NVME_VOLUME_GROUP_NAME + if [[ -e "$LVM_DEVICE" ]]; then + phys_port_cnt=$(ibv_devinfo |grep -Eow hca_id |wc -l) + echo "The actual quantity of RDMA ports is $phys_port_cnt" + if [[ "$phys_port_cnt" < '1' ]]; then + echo "RDMA card not found" + else + osds::lvm::create_nvme_vg $nvmevg $size + osds::lvm::nvmeofpkginstall + # remove volumes that already exist + osds::lvm::remove_volumes $nvmevg + osds::lvm::set_nvme_configuration + fi + fi osds::lvm::set_lvm_filter } osds::lvm::cleanup(){ osds::lvm::clean_volume_group $DEFAULT_VOLUME_GROUP_NAME osds::lvm::clean_lvm_filter + local nvmevg=$NVME_VOLUME_GROUP_NAME + if vgs $nvmevg ; then + osds::lvm::clean_nvme_volume_group $nvmevg + fi } osds::lvm::uninstall(){ @@ -186,6 +294,7 @@ osds::lvm::uninstall(){ osds::lvm::uninstall_purge(){ echo osds::lvm::pkg_uninstall + echo osds::lvm::nvmeofpkguninstall } # Restore xtrace diff --git a/test/e2e/connector/connector.go b/test/e2e/connector/connector.go index 595917631..f40ef03cb 100644 --- a/test/e2e/connector/connector.go +++ b/test/e2e/connector/connector.go @@ -21,10 +21,12 @@ import ( "github.com/opensds/opensds/contrib/connector" _ "github.com/opensds/opensds/contrib/connector/iscsi" + _ "github.com/opensds/opensds/contrib/connector/nvmeof" ) const ( - iscsiProtocol = "iscsi" + iscsiProtocol = "iscsi" + nvmeofProtocol = "nvmeof" attachCommand = "attach" detachCommand = "detach" @@ -41,17 +43,19 @@ func main() { os.Exit(-1) } + accPro := os.Args[3] switch os.Args[1] { case attachCommand: - dev, err := connector.NewConnector(iscsiProtocol).Attach(connData) + dev, err := connector.NewConnector(accPro).Attach(connData) if err != nil { fmt.Println("Failed to attach volume to the host:", err) os.Exit(-1) } fmt.Println("Got device:", dev) break + case detachCommand: - if err := connector.NewConnector(iscsiProtocol).Detach(connData); err != nil { + if err := connector.NewConnector(accPro).Detach(connData); err != nil { fmt.Println("Failed to detach volume to the host:", err) os.Exit(-1) } diff --git a/test/e2e/e2ef_test.go b/test/e2e/e2ef_test.go index 0d5be8bf1..0260c2f99 100644 --- a/test/e2e/e2ef_test.go +++ b/test/e2e/e2ef_test.go @@ -29,6 +29,11 @@ import ( "github.com/opensds/opensds/pkg/utils/constants" ) +const ( + nvmepool = "opensds-volumes-nvme" + defaultgroup = "opensds-volumes-default" +) + var u *client.Client //init Create Profile @@ -446,8 +451,9 @@ func TestVolumeAttach(t *testing.T) { t.Error("Failed to marshal connection data:", err) return } + accPro := getatt.AccessProtocol output, err = execCmd("sudo", "./volume-connector", - "attach", string(conn)) + "attach", string(conn), accPro) if err != nil { t.Error("Failed to attach volume:", output, err) return @@ -456,7 +462,30 @@ func TestVolumeAttach(t *testing.T) { t.Log("Volume attach success!") } -//Test Case:Volume Detach +//Test Case:Delete Attachment +func TestDeleteAttach(t *testing.T) { + attc, err := PrepareAttachment(t) + if err != nil { + t.Error("Prepare Attachment Fail!", err) + return + } + defer DeleteVolume(attc.VolumeId) + err = u.DeleteVolumeAttachment(attc.Id, nil) + if err != nil { + t.Error("Delete Attachment Fail", err) + return + } + _, err = u.GetVolumeAttachment(attc.Id) + t.Log("err:", err) + if strings.Contains(err.Error(), "can't find") { + t.Log("Delete attachment Success") + return + } else { + t.Error("Delete Attachment Fail!", err) + } +} + +//Test Case:Delete Attachment func TestVolumeDetach(t *testing.T) { attc, err := PrepareAttachment(t) if err != nil { @@ -476,6 +505,7 @@ func TestVolumeDetach(t *testing.T) { } t.Log("Begin to Scan volume:") + t.Log("getatt.Accessprotocol", getatt.AccessProtocol) t.Log("getatt.Metadata", getatt.ConnectionData) //execute bin file @@ -486,8 +516,9 @@ func TestVolumeDetach(t *testing.T) { } // attach first, then detach + accPro := getatt.AccessProtocol output, err := execCmd("sudo", "./volume-connector", - "attach", string(conn)) + "attach", string(conn), accPro) if err != nil { t.Error("Failed to attach volume:", output, err) return @@ -496,7 +527,7 @@ func TestVolumeDetach(t *testing.T) { t.Log(output) output, err = execCmd("sudo", "./volume-connector", - "detach", string(conn)) + "detach", string(conn), accPro) if err != nil { t.Error("Failed to detach volume:", output, err) return @@ -505,29 +536,220 @@ func TestVolumeDetach(t *testing.T) { t.Log("Volume Detach Success!") } -//Test Case:Delete Attachment -func TestDeleteAttach(t *testing.T) { - attc, err := PrepareAttachment(t) +//Test for nvmeof connection +func TestNvmeofAttachIssues(t *testing.T) { + // pool list get nvme pool + pols, err := u.ListPools() if err != nil { - t.Error("Prepare Attachment Fail!", err) + t.Error("list pools failed:", err) + return + } + polId := "" + for _, p := range pols { + if p.Name == nvmepool { + polId = p.Id + t.Log("nvme pool id is: ", polId) + break + } + } + if polId == "" { + t.Log("no nvme pool ") + return + } + //PrepareNvmeVolume() + err = CreateNvmeofAttach(t) + if err != nil { + t.Error("create nvmeof attachment fail", err) + return + } + err = ListNvmeofAttachment(t) + if err != nil { + t.Error("list nvmeof attachment fail", err) + return + } + err = ShowNvmeofAttachDetail(t) + if err != nil { + t.Error("show nvmeof attachment fail", err) + return + } + err = NvmeofVolumeAttach(t) + if err != nil { + t.Error("connect nvmeof attachment fail", err) return } + + err = DeleteNvmeofAttach(t) + if err != nil { + t.Error("delete nvmeof attachment fail", err) + return + } + + t.Log("nvmeof attach issues success") +} + +func CreateNvmeofAttach(t *testing.T) error { + vol, err := PrepareNvmeVolume() + if err != nil { + t.Error("Prepare nvme Volume Fail", err) + return err + } + defer DeleteVolume(vol.Id) + var body = &model.VolumeAttachmentSpec{ + VolumeId: vol.Id, + HostInfo: model.HostInfo{}, + } + attc, err := u.CreateVolumeAttachment(body) + if err != nil { + t.Error("create nvmeof volume attachment failed:", err) + return err + } + defer DeleteAttachment(attc.Id) + attrs, _ := json.MarshalIndent(attc, "", " ") + t.Log(string(attrs)) + t.Log("Create nvmeof Volume Attachment Success") + return nil +} + +func ListNvmeofAttachment(t *testing.T) error { + attc, err := PrepareNvmeofAttachment(t) + if err != nil { + t.Error("Prepare nvmeof Attachment Fail!", err) + return err + } + defer DeleteVolume(attc.VolumeId) + defer DeleteAttachment(attc.Id) + atts, err := u.ListVolumeAttachments() + if err != nil { + t.Error("List nvmeof Attachment Error!", err) + return err + } + attli, _ := json.MarshalIndent(atts, "", " ") + t.Log(string(attli)) + t.Log("List nvmeof Attachment Success!") + return nil +} + +func ShowNvmeofAttachDetail(t *testing.T) error { + attc, err := PrepareNvmeofAttachment(t) + if err != nil { + t.Error("Prepare Attachment Fail!", err) + return err + } + defer DeleteVolume(attc.VolumeId) + defer DeleteAttachment(attc.Id) + + getatt, err := u.GetVolumeAttachment(attc.Id) + if err != nil || getatt.Status != "available" { + t.Error("Get Volume Attachment Detail Fail!", err) + return err + } + t.Log("Get Volume Attachment Detail Success") + return nil +} + +func DeleteNvmeofAttach(t *testing.T) error { + attc, err := PrepareNvmeofAttachment(t) + if err != nil { + t.Error("Prepare Attachment Fail!", err) + return err + } defer DeleteVolume(attc.VolumeId) err = u.DeleteVolumeAttachment(attc.Id, nil) if err != nil { - t.Error("Delete Attachment Fail", err) - return + t.Error("Delete nvme Attachment Fail", err) + return err } _, err = u.GetVolumeAttachment(attc.Id) t.Log("err:", err) if strings.Contains(err.Error(), "can't find") { t.Log("Delete attachment Success") - return + return nil } else { t.Error("Delete Attachment Fail!", err) + return err } } +//Test Case:Nvmeof Volume Attach +func NvmeofVolumeAttach(t *testing.T) error { + attc, err := PrepareNvmeofAttachment(t) + if err != nil { + t.Error("Prepare Attachment Fail:", err) + return err + } + defer DeleteVolume(attc.VolumeId) + defer DeleteAttachment(attc.Id) + + getatt, err := u.GetVolumeAttachment(attc.Id) + if err != nil || getatt.Status != "available" { + t.Errorf("attachment(%s) is not available: %v", attc.Id, err) + return err + } + + t.Log("Begin to Scan Volume:") + t.Log("getatt.AccessProtocol", getatt.AccessProtocol) + t.Log("getatt.Metadata", getatt.ConnectionData) + + output, _ := execCmd("/bin/bash", "-c", "ps -ef") + t.Log(output) + //execute bin file + conn, err := json.Marshal(&getatt.ConnectionData) + if err != nil { + t.Error("Failed to marshal connection data:", err) + return err + } + accPro := getatt.AccessProtocol + output, err = execCmd("sudo", "./volume-connector", + "attach", string(conn), accPro) + if err != nil { + t.Error("Failed to attach volume:", output, err) + return err + } + t.Log(output) + t.Log("Nvmeof Volume attach yoyo success!") + // detach it + err = NvmeofVolumeDetach(t, attc) + if err != nil { + t.Error("detach failed") + return err + } + return nil +} + +//Test Case:Delete Attachment +func NvmeofVolumeDetach(t *testing.T, attc *model.VolumeAttachmentSpec) error { + defer DeleteVolume(attc.VolumeId) + defer DeleteAttachment(attc.Id) + + getatt, err := u.GetVolumeAttachment(attc.Id) + if err != nil || getatt.Status != "available" { + t.Errorf("attachment(%s) is not available: %v", attc.Id, err) + return err + } + + t.Log("Begin to Scan volume:") + t.Log("getatt.AccessProtocol", getatt.AccessProtocol) + t.Log("getatt.Metadata", getatt.ConnectionData) + + //execute bin file + conn, err := json.Marshal(&getatt.ConnectionData) + if err != nil { + t.Error("Failed to marshal connection data:", err) + return err + } + accPro := getatt.AccessProtocol + output, err := execCmd("sudo", "./volume-connector", + "detach", string(conn), accPro) + if err != nil { + t.Error("Failed to detach volume:", attc.VolumeId, output, err) + return err + } + t.Log(output) + t.Log("Volume Detach Success!") + return nil +} + +// cmd func execCmd(name string, arg ...string) (string, error) { fmt.Printf("Command: %s %s:\n", name, strings.Join(arg, " ")) info, err := exec.Command(name, arg...).CombinedOutput() @@ -556,6 +778,28 @@ func PrepareAttachment(t *testing.T) (*model.VolumeAttachmentSpec, error) { return attc, nil } +// prepare nvmeof attachment +func PrepareNvmeofAttachment(t *testing.T) (*model.VolumeAttachmentSpec, error) { + vol, err := PrepareNvmeVolume() + if err != nil { + t.Error("Prepare nvmeof Volume Fail", err) + return nil, err + } + + var body = &model.VolumeAttachmentSpec{ + VolumeId: vol.Id, + HostInfo: model.HostInfo{}, + } + attc, err := u.CreateVolumeAttachment(body) + if err != nil { + t.Error("prepare volume attachment failed:", err) + return nil, err + } + + t.Log("prepare nvmeof Volume Attachment Success") + return attc, nil +} + //delete attachment func DeleteAttachment(attId string) error { err := u.DeleteVolumeAttachment(attId, nil) @@ -625,6 +869,38 @@ func PrepareVolume() (*model.VolumeSpec, error) { return create, nil } +//nvme volume is essential for nvmeof attachment ,so the volume should be created in nvme pool +func PrepareNvmeVolume() (*model.VolumeSpec, error) { + // get poolid + pols, err := u.ListPools() + if err != nil { + return nil, err + } + polId := "" + for _, p := range pols { + if p.Name == nvmepool { + polId = p.Id + break + } + } + if polId == "" { + return nil, nil + } + + //create volume in specified nvme pool + var volbody = &model.VolumeSpec{ + Name: "nvme flowTest", + Description: "This a test for nvme flow", + Size: int64(1), + PoolId: polId, + } + create, err := u.CreateVolume(volbody) + if err != nil { + return nil, err + } + return create, nil +} + //Delete volume after test func DeleteVolume(volId string) error { err := u.DeleteVolume(volId, nil)