Bdev ext api

include/spdk/nvme.h

@@ -2545,6 +2545,90 @@ int spdk_nvme_ns_cmd_writev_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qp
 				    spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
 				    uint16_t apptag_mask, uint16_t apptag);
 
+/**
+ * Callback used to get a Memory Key per IO request
+ *
+ * pd is input parameter and should point to a memory domain
+ * mkey is an output value
+ */
+typedef int (*spdk_nvme_ns_cmd_io_get_mkey)(void *cb_arg, void *address, size_t length, void *pd,
+		uint32_t *mkey);
+
+enum spdk_nvme_ns_cmd_ext_io_opts_mem_types {
+	/** Memory in IO request belongs to another memory domain and it is described by Memory Key.
+	 * If this value is set then \b mkey structure in spdk_nvme_ns_cmd_ext_io_opts_mem_type contains
+	 * a callback and its argument that can be used to get a Memory Key */
+	SPDK_NVME_NS_CMD_EXT_IO_OPTS_MEM_TYPE_MEMORY_KEY = 0,
+};
+
+struct spdk_nvme_ns_cmd_ext_io_opts_mem_type {
+	/** This value determines which part of union should be used. Provides extensibility for this structure */
+	enum spdk_nvme_ns_cmd_ext_io_opts_mem_types type;
+	union {
+		struct {
+			spdk_nvme_ns_cmd_io_get_mkey get_mkey_cb;
+		} mkey;
+	} u;
+};
+
+enum spdk_nvme_ns_cmd_ext_io_opts_flags {
+	/** This flag determines the type of memory passed in IO request.
+	 * Refer to \ref spdk_nvme_ns_cmd_ext_io_opts_mem_types for more information.
+	 * If this flag is set in spdk_nvme_ns_cmd_ext_io_opts then \b mem_type member of
+	 * \b spdk_nvme_ns_cmd_ext_io_opts should point to a structure that describes memory buffer */
+	SPDK_NVME_NS_CMD_EXT_IO_OPTS_MEM_TYPE = 1u << 0,
+};
+
+/**
+ * Structure with optional IO request parameters
+ */
+struct spdk_nvme_ns_cmd_ext_io_opts {
+	/** Combination of bits defined in \b enum spdk_nvme_ns_cmd_ext_io_opts_flags */
+	uint64_t flags;
+	/** Describes type of the memory used in IO request
+	 * This structure must be filled by the user if \b SPDK_NVME_NS_CMD_EXT_IO_OPTS_MEM_TYPE bit is set
+	 * in \b flags member. Used by RDMA transport, other transports ignore this extension */
+	struct spdk_nvme_ns_cmd_ext_io_opts_mem_type *mem_type;
+};
+
+/**
+ * Submit a write I/O to the specified NVMe namespace.
+ *
+ * The command is submitted to a qpair allocated by spdk_nvme_ctrlr_alloc_io_qpair().
+ * The user must ensure that only one thread submits I/O on a given qpair at any
+ * given time.
+ *
+ * \param ns NVMe namespace to submit the write I/O
+ * \param qpair I/O queue pair to submit the request
+ * \param lba starting LBA to write the data
+ * \param lba_count length (in sectors) for the write operation
+ * \param cb_fn callback function to invoke when the I/O is completed
+ * \param cb_arg argument to pass to the callback function
+ * \param io_flags set flags, defined in nvme_spec.h, for this I/O
+ * \param reset_sgl_fn callback function to reset scattered payload
+ * \param next_sge_fn callback function to iterate each scattered
+ * payload memory segment
+ * \param metadata virtual address pointer to the metadata payload, the length
+ * of metadata is specified by spdk_nvme_ns_get_md_size()
+ * \param apptag_mask application tag mask.
+ * \param apptag application tag to use end-to-end protection information.
+ * \param opts Optional structure with extended IO request options.
+ *
+ * \return 0 if successfully submitted, negated errnos on the following error conditions:
+ * -EINVAL: The request is malformed.
+ * -ENOMEM: The request cannot be allocated.
+ * -ENXIO: The qpair is failed at the transport level.
+ * -EFAULT: Invalid address was specified as part of payload.  cb_fn is also called
+ *          with error status including dnr=1 in this case.
+ */
+int spdk_nvme_ns_cmd_writev_with_md_ext(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+					uint64_t lba, uint32_t lba_count,
+					spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
+					spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
+					spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
+					uint16_t apptag_mask, uint16_t apptag,
+					struct spdk_nvme_ns_cmd_ext_io_opts *opts);
+
 /**
  * Submit a write I/O to the specified NVMe namespace.
  *
@@ -2725,6 +2809,43 @@ int spdk_nvme_ns_cmd_readv_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpa
 				   spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
 				   uint16_t apptag_mask, uint16_t apptag);
 
+/**
+ * Submit a read I/O to the specified NVMe namespace.
+ *
+ * The command is submitted to a qpair allocated by spdk_nvme_ctrlr_alloc_io_qpair().
+ * The user must ensure that only one thread submits I/O on a given qpair at any given time.
+ *
+ * \param ns NVMe namespace to submit the read I/O
+ * \param qpair I/O queue pair to submit the request
+ * \param lba starting LBA to read the data
+ * \param lba_count length (in sectors) for the read operation
+ * \param cb_fn callback function to invoke when the I/O is completed
+ * \param cb_arg argument to pass to the callback function
+ * \param io_flags set flags, defined in nvme_spec.h, for this I/O
+ * \param reset_sgl_fn callback function to reset scattered payload
+ * \param next_sge_fn callback function to iterate each scattered
+ * payload memory segment
+ * \param metadata virtual address pointer to the metadata payload, the length
+ *	           of metadata is specified by spdk_nvme_ns_get_md_size()
+ * \param apptag_mask application tag mask.
+ * \param apptag application tag to use end-to-end protection information.
+ * \param opts Optional structure with extended IO request options.
+ *
+ * \return 0 if successfully submitted, negated errnos on the following error conditions:
+ * -EINVAL: The request is malformed.
+ * -ENOMEM: The request cannot be allocated.
+ * -ENXIO: The qpair is failed at the transport level.
+ * -EFAULT: Invalid address was specified as part of payload.  cb_fn is also called
+ *          with error status including dnr=1 in this case.
+ */
+int spdk_nvme_ns_cmd_readv_with_md_ext(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+				       uint64_t lba, uint32_t lba_count,
+				       spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
+				       spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
+				       spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
+				       uint16_t apptag_mask, uint16_t apptag,
+				       struct spdk_nvme_ns_cmd_ext_io_opts *opts);
+
 /**
  * Submits a read I/O to the specified NVMe namespace.
  *

lib/nvme/nvme_internal.h

@@ -214,12 +214,21 @@ struct nvme_payload {
 	spdk_nvme_req_reset_sgl_cb reset_sgl_fn;
 	spdk_nvme_req_next_sge_cb next_sge_fn;
 
+	/**
+	 * Function to be used to get an mkey for scattered payload
+	 */
+	spdk_nvme_ns_cmd_io_get_mkey get_sge_mkey;
+
 	/**
 	 * If reset_sgl_fn == NULL, this is a contig payload, and contig_or_cb_arg contains the
 	 * virtual memory address of a single virtually contiguous buffer.
 	 *
 	 * If reset_sgl_fn != NULL, this is a SGL payload, and contig_or_cb_arg contains the
 	 * cb_arg that will be passed to the SGL callback functions.
+	 *
+	 * If get_sgl_mkey != NULL, this is a SGL payload, and contig_or_cb_arg contains the
+	 * cb_arg that will be passed to the get_sge_mkey callback function. Moreover data returned
+	 * by next_sge_fn and get_sgl_mkey belongs to another memory domain and can not be accessed by SPDK
 	 */
 	void *contig_or_cb_arg;
 

lib/nvme/nvme_ns_cmd.c

@@ -31,6 +31,7 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <spdk/nvme.h>
 #include "nvme_internal.h"
 
 static inline struct nvme_request *_nvme_ns_cmd_rw(struct spdk_nvme_ns *ns,
@@ -709,6 +710,52 @@ spdk_nvme_ns_cmd_readv_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *
 	}
 }
 
+int
+spdk_nvme_ns_cmd_readv_with_md_ext(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+				   uint64_t lba, uint32_t lba_count,
+				   spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
+				   spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
+				   spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
+				   uint16_t apptag_mask, uint16_t apptag,
+				   struct spdk_nvme_ns_cmd_ext_io_opts *opts)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	if (reset_sgl_fn == NULL || next_sge_fn == NULL) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, metadata);
+
+	if (opts) {
+		if (opts->flags & SPDK_NVME_NS_CMD_EXT_IO_OPTS_MEM_TYPE) {
+			if (opts->mem_type->type != SPDK_NVME_NS_CMD_EXT_IO_OPTS_MEM_TYPE_MEMORY_KEY) {
+				SPDK_ERRLOG("Unknown memory type %d\n", opts->mem_type->type);
+				return -EINVAL;
+			}
+			payload.get_sge_mkey = opts->mem_type->u.mkey.get_mkey_cb;
+		}
+	}
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ,
+			      io_flags, apptag_mask, apptag, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+}
+
 int
 spdk_nvme_ns_cmd_write(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
 		       void *buffer, uint64_t lba,
@@ -836,6 +883,53 @@ spdk_nvme_ns_cmd_writev_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair
 	}
 }
 
+int
+spdk_nvme_ns_cmd_writev_with_md_ext(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+				    uint64_t lba, uint32_t lba_count,
+				    spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags,
+				    spdk_nvme_req_reset_sgl_cb reset_sgl_fn,
+				    spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata,
+				    uint16_t apptag_mask, uint16_t apptag,
+				    struct spdk_nvme_ns_cmd_ext_io_opts *opts)
+{
+	struct nvme_request *req;
+	struct nvme_payload payload;
+
+	if (!_is_io_flags_valid(io_flags)) {
+		return -EINVAL;
+	}
+
+	if (reset_sgl_fn == NULL || next_sge_fn == NULL) {
+		return -EINVAL;
+	}
+
+	payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, metadata);
+
+	if (opts) {
+		if (opts->flags & SPDK_NVME_NS_CMD_EXT_IO_OPTS_MEM_TYPE) {
+			if (opts->mem_type->type != SPDK_NVME_NS_CMD_EXT_IO_OPTS_MEM_TYPE_MEMORY_KEY) {
+				SPDK_ERRLOG("Unknown memory type %d\n", opts->mem_type->type);
+				return -EINVAL;
+			}
+			payload.get_sge_mkey = opts->mem_type->u.mkey.get_mkey_cb;
+		}
+	}
+
+	req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE,
+			      io_flags, apptag_mask, apptag, true);
+	if (req != NULL) {
+		return nvme_qpair_submit_request(qpair, req);
+	} else if (nvme_ns_check_request_length(lba_count,
+						ns->sectors_per_max_io,
+						ns->sectors_per_stripe,
+						qpair->ctrlr->opts.io_queue_requests)) {
+		return -EINVAL;
+	} else {
+		return -ENOMEM;
+	}
+
+}
+
 int
 spdk_nvme_ns_cmd_write_zeroes(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
 			      uint64_t lba, uint32_t lba_count,

lib/nvme/nvme_rdma.c

@@ -1564,7 +1564,7 @@ nvme_rdma_build_sgl_request(struct nvme_rdma_qpair *rqpair,
 	uint32_t remaining_size;
 	uint32_t sge_length;
 	int rc, max_num_sgl, num_sgl_desc;
-	uint32_t rkey = 0;
+	uint32_t mkey = 0;
 
 	assert(req->payload_size != 0);
 	assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL);
@@ -1590,12 +1590,21 @@ nvme_rdma_build_sgl_request(struct nvme_rdma_qpair *rqpair,
 			return -1;
 		}
 
-		if (spdk_unlikely(!nvme_rdma_get_key(rqpair->mr_map->map, virt_addr, sge_length,
-						     NVME_RDMA_MR_RKEY, &rkey))) {
-			return -1;
+		if (req->payload.get_sge_mkey) {
+			rc = req->payload.get_sge_mkey(req->payload.contig_or_cb_arg, virt_addr, sge_length,
+						       rqpair->rdma_qp->qp->pd, &mkey);
+			if (spdk_unlikely(rc)) {
+				SPDK_ERRLOG("Memory translation failed, rc %d\n", rc);
+				return -1;
+			}
+		} else {
+			if (spdk_unlikely(!nvme_rdma_get_key(rqpair->mr_map->map, virt_addr, sge_length,
+							     NVME_RDMA_MR_RKEY, &mkey))) {
+				return -1;
+			}
 		}
 
-		cmd->sgl[num_sgl_desc].keyed.key = rkey;
+		cmd->sgl[num_sgl_desc].keyed.key = mkey;
 		cmd->sgl[num_sgl_desc].keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK;
 		cmd->sgl[num_sgl_desc].keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS;
 		cmd->sgl[num_sgl_desc].keyed.length = sge_length;
@@ -1664,7 +1673,7 @@ nvme_rdma_build_sgl_inline_request(struct nvme_rdma_qpair *rqpair,
 				   struct spdk_nvme_rdma_req *rdma_req)
 {
 	struct nvme_request *req = rdma_req->req;
-	uint32_t lkey = 0;
+	uint32_t mkey = 0;
 	uint32_t length;
 	void *virt_addr;
 	int rc;
@@ -1689,14 +1698,23 @@ nvme_rdma_build_sgl_inline_request(struct nvme_rdma_qpair *rqpair,
 		length = req->payload_size;
 	}
 
-	if (spdk_unlikely(!nvme_rdma_get_key(rqpair->mr_map->map, virt_addr, length,
-					     NVME_RDMA_MR_LKEY, &lkey))) {
-		return -1;
+	if (req->payload.get_sge_mkey) {
+		rc = req->payload.get_sge_mkey(req->payload.contig_or_cb_arg, virt_addr, length,
+					       rqpair->rdma_qp->qp->pd, &mkey);
+		if (spdk_unlikely(rc)) {
+			SPDK_ERRLOG("Memory translation failed, rc %d\n", rc);
+			return -1;
+		}
+	} else {
+		if (spdk_unlikely(!nvme_rdma_get_key(rqpair->mr_map->map, virt_addr, length,
+						     NVME_RDMA_MR_LKEY, &mkey))) {
+			return -1;
+		}
 	}
 
 	rdma_req->send_sgl[1].addr = (uint64_t)virt_addr;
 	rdma_req->send_sgl[1].length = length;
-	rdma_req->send_sgl[1].lkey = lkey;
+	rdma_req->send_sgl[1].lkey = mkey;
 
 	rdma_req->send_wr.num_sge = 2;