RDMA SRQ implementation for NVMe-oF target

.gitignore

@@ -18,7 +18,7 @@ tags
 cscope.out
 dpdk-*
 CUnit-Memory-Dump.xml
-config.h
+include/spdk/config.h
 CONFIG.local
 *VC.db
 .vscode

CHANGELOG.md

@@ -4,6 +4,10 @@
 
 ### nvme
 
+spdk_nvme_ctrlr_cmd_security_send() and spdk_nvme_ctrlr_cmd_security_receive()
+were added to support sending or receiving security protocol data to or from
+nvme controller.
+
 spdk_nvme_ns_get_extended_sector_size() was added.  This function includes
 the metadata size per sector (if any).  spdk_nvme_ns_get_sector_size() still
 returns only the data size per sector, not including metadata.
@@ -22,6 +26,25 @@ does not at this time confer any SPDK ABI compatibility claims.
 
 spdk_bdev_alias_del_all() was added to delete all alias from block device.
 
+A new virtual bdev module has been added to perform at rest data encryption using the DPDK CryptoDev
+Framework.  The module initially uses a software AESNI CBC cipher with experimental support for the
+Intel QAT hardware accelerator also currently implemented with support for CBC cipher. Future work
+may include additional ciphers as well as consideration for authentication. NOTE: this module is
+currently marked as experimental.  Do not use in production.
+
+The RAID virtual bdev module is now always enabled by default.  The configure --with-raid and
+--without-raid options are now ignored and deprecated and will be removed in the next release.
+
+### Environment Abstraction Layer and Event Framework
+
+The size parameter of spdk_mem_map_translate is now a pointer. This allows the
+function to report back the actual size of the translation relative to the original
+request made by the user.
+
+A new structure spdk_mem_map_ops has been introduced to hold memory map related
+callbacks. This structure is now passed as the second argument of spdk_mem_map_alloc
+in lieu of the notify callback.
+
 ### iscsi
 
 Parameter names of `set_iscsi_options` and `get_iscsi_global_params` RPC
@@ -40,6 +63,14 @@ specified. This default has been removed. Users must now explicitly specify
 the location of this file to load CHAP shared secrets from a file, or use
 the related iSCSI RPC methods to add them at runtime.
 
+### NVMe-oF Target
+New `configure` options `--with-rdma-srq` and `--without-rdma-srq` [default],
+provide the capability to enable or disable shared receive queue support
+in RDMA transport for NVMe-oF target.
+Size of shared receive queue is defined by transport configuration file parameter
+`MaxSRQDepth` and `nvmf_create_transport` RPC method parameter `max_srq_depth`.
+Default size is 4096.
+
 ### Miscellaneous
 
 The ReactorMask config file parameter has been deprecated.  Users should
@@ -50,6 +81,30 @@ Default config file pathnames have been removed from iscsi_tgt, nvmf_tgt
 and vhost.  Config file pathnames may now only be specified using the
 -c command line option.
 
+Users may no longer set DPDK_DIR in their environment to specify the
+location of the DPDK installation used to build SPDK.  Using DPDK_DIR
+has not been the documented nor recommended way to specify the DPDK
+location for several releases, but removing it ensures no unexpected
+surprises for users who may have DPDK_DIR defined for other reasons.
+Users should just use the "configure" script to specify the DPDK
+location before building SPDK.
+
+Although we know that many developers still use Python 2 we are officially
+switching to Python3 with requirement that all new code must be valid also
+for Python 2 up to the EOL which is year 2020.
+
+Invoking interpreter explicitly is forbidden for executable scripts. There
+is no need to use syntax like "python ./scripts/rpc.py". All executable
+scripts must contain proper shebang pointing to the right interpreter.
+Scripts without shebang musn't be executable.
+
+A Python script has been added to enable conversion of old INI config file
+to new JSON-RPC config file format. This script can be found at
+scripts/config_converter.py. Example how this script can be used:
+~~~{.sh}
+cat old_format.ini | scripts/config_converter.py > new_json_format.json
+~~~
+
 ## v18.07:
 
 ### bdev

CONFIG

@@ -37,8 +37,8 @@ CONFIG_PREFIX?=/usr/local
 # Build with debug logging. Turn off for performance testing and normal usage
 CONFIG_DEBUG?=n
 
-# Show backtrace when logging message at level <= lvl (ERROR, WARN, NOTICE, DEBUG)
-#CONFIG_LOG_BACKTRACE?=lvl
+# Build with support of backtrace printing in log messages. Requires libunwind.
+#CONFIG_LOG_BACKTRACE?=n
 
 # Treat warnings as errors (fail the build on any warning).
 CONFIG_WERROR?=n
@@ -66,9 +66,7 @@ CONFIG_TESTS?=y
 CONFIG_ENV?=$(SPDK_ROOT_DIR)/lib/env_dpdk
 
 # This directory should contain 'include' and 'lib' directories for your DPDK
-# installation. Alternatively you can specify this on the command line
-# with 'make DPDK_DIR=/path/to/dpdk'. This is only a valid entry
-# when using the default SPDK environment library.
+# installation.
 CONFIG_DPDK_DIR?=$(SPDK_ROOT_DIR)/dpdk/build
 
 # Build SPDK FIO plugin. Requires FIO_SOURCE_DIR set to a valid
@@ -102,9 +100,6 @@ CONFIG_VPP?=n
 # Requires libiscsi development libraries.
 CONFIG_ISCSI_INITIATOR?=n
 
-# Build with raid
-CONFIG_RAID?=n
-
 # Enable the dependencies for building the crypto vbdev
 CONFIG_CRYPTO?=n
 

Makefile

@@ -41,7 +41,7 @@ DIRS-$(CONFIG_SHARED) += shared_lib
 DIRS-y += examples app include
 DIRS-$(CONFIG_TESTS) += test
 
-.PHONY: all clean $(DIRS-y) config.h CONFIG.local mk/cc.mk cc_version cxx_version
+.PHONY: all clean $(DIRS-y) include/spdk/config.h CONFIG.local mk/cc.mk cc_version cxx_version
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 ifeq ($(CURDIR)/dpdk/build,$(CONFIG_DPDK_DIR))
@@ -52,33 +52,42 @@ endif
 endif
 endif
 
+ifeq ($(CONFIG_SHARED),y)
+LIB = shared_lib
+else
+LIB = lib
+endif
+
 all: $(DIRS-y)
 clean: $(DIRS-y)
 	$(Q)rm -f mk/cc.mk
-	$(Q)rm -f config.h
+	$(Q)rm -f include/spdk/config.h
 
 install: all
 	$(Q)echo "Installed to $(DESTDIR)$(CONFIG_PREFIX)"
 
 shared_lib: lib
 lib: $(DPDKBUILD)
-app: lib
-test: lib
-examples: lib
+app: $(LIB)
+test: $(LIB)
+examples: $(LIB)
 pkgdep:
 	sh ./scripts/pkgdep.sh
 
-$(DIRS-y): mk/cc.mk config.h
+$(DIRS-y): mk/cc.mk include/spdk/config.h
 
 mk/cc.mk:
 	$(Q)scripts/detect_cc.sh --cc=$(CC) --cxx=$(CXX) --lto=$(CONFIG_LTO) > $@.tmp; \
 	cmp -s $@.tmp $@ || mv $@.tmp $@ ; \
 	rm -f $@.tmp
 
-config.h: CONFIG CONFIG.local scripts/genconfig.py
+include/spdk/config.h: CONFIG CONFIG.local scripts/genconfig.py
 	$(Q)PYCMD=$$(cat PYTHON_COMMAND 2>/dev/null) ; \
 	test -z "$$PYCMD" && PYCMD=python ; \
-	$$PYCMD scripts/genconfig.py $(MAKEFLAGS) > $@.tmp; \
+	echo "#ifndef SPDK_CONFIG_H" > $@.tmp; \
+	echo "#define SPDK_CONFIG_H" >> $@.tmp; \
+	$$PYCMD scripts/genconfig.py $(MAKEFLAGS) >> $@.tmp; \
+	echo "#endif /* SPDK_CONFIG_H */" >> $@.tmp; \
 	cmp -s $@.tmp $@ || mv $@.tmp $@ ; \
 	rm -f $@.tmp
 

README.md

@@ -132,8 +132,10 @@ make CONFIG_RDMA=y
 ~~~
 
 Users may wish to use a version of DPDK different from the submodule included
-in the SPDK repository.  To specify an alternate DPDK installation, run
-configure with the --with-dpdk option.  For example:
+in the SPDK repository.  Note, this includes the ability to build not only
+from DPDK sources, but also just with the includes and libraries
+installed via the dpdk and dpdk-devel packages.  To specify an alternate DPDK
+installation, run configure with the --with-dpdk option.  For example:
 
 Linux:
 

app/spdk_tgt/spdk_tgt.c

@@ -33,6 +33,7 @@
 
 #include "spdk/stdinc.h"
 
+#include "spdk/config.h"
 #include "spdk/env.h"
 #include "spdk/event.h"
 #include "spdk/vhost.h"

autobuild.sh

@@ -23,14 +23,20 @@ echo "** END ** Info for Hostname: $HOSTNAME"
 
 timing_enter autobuild
 
-./configure $config_params
-
 timing_enter check_format
 if [ $SPDK_RUN_CHECK_FORMAT -eq 1 ]; then
 	./scripts/check_format.sh
 fi
 timing_exit check_format
 
+$MAKE $MAKEFLAGS clean
+if [ $SPDK_BUILD_SHARED_OBJECT -eq 1 ]; then
+	./configure $config_params --with-shared
+	$MAKE $MAKEFLAGS
+	$MAKE $MAKEFLAGS clean
+	report_test_completion "shared_object_build"
+fi
+
 scanbuild=''
 make_timing_label='make'
 if [ $SPDK_RUN_SCANBUILD -eq 1 ] && hash scan-build; then
@@ -53,10 +59,10 @@ if [ $SPDK_RUN_UBSAN -eq 1 ]; then
 fi
 
 echo $scanbuild
-$MAKE $MAKEFLAGS clean
 
 timing_enter "$make_timing_label"
 fail=0
+./configure $config_params
 time $scanbuild $MAKE $MAKEFLAGS || fail=1
 if [ $fail -eq 1 ]; then
 	if [ -d $out/scan-build-tmp ]; then

autotest.sh

@@ -102,6 +102,10 @@ if [ $SPDK_TEST_BLOCKDEV -eq 1 ]; then
 	fi
 fi
 
+if [ $SPDK_TEST_JSON -eq 1 ]; then
+	run_test suite test/config_converter/test_converter.sh
+fi
+
 if [ $SPDK_TEST_EVENT -eq 1 ]; then
 	run_test suite test/event/event.sh
 fi
@@ -140,6 +144,7 @@ fi
 if [ $SPDK_TEST_NVMF -eq 1 ]; then
 	run_test suite ./test/nvmf/nvmf.sh
 	run_test suite ./test/nvmf/nvmfjson/json_config.sh
+	run_test suite ./test/spdkcli/nvmf.sh
 fi
 
 if [ $SPDK_TEST_VHOST -eq 1 ]; then
@@ -209,6 +214,7 @@ if [ $SPDK_TEST_LVOL -eq 1 ]; then
 	test_cases+="700,701,702,750,751,752,753,754,755,756,757,758,759,"
 	test_cases+="800,801,802,803,804,10000"
 	run_test suite ./test/lvol/lvol.sh --test-cases=$test_cases
+	run_test suite ./test/blobstore/blob_io_wait/blob_io_wait.sh
 	report_test_completion "lvol"
 	timing_exit lvol
 fi

configure

@@ -16,8 +16,7 @@ function usage()
 	echo " --prefix=path             Configure installation prefix (default: /usr/local)"
 	echo ""
 	echo " --enable-debug            Configure for debug builds"
-	echo " --enable-log-bt=lvl       Show backtrace using libunwind when logging message at level <= lvl."
-	echo "                           Valid values are: ERROR, WARN, NOTICE, DEBUG."
+	echo " --enable-log-bt           Enable support of backtrace printing in SPDK logs (requires libunwind)."
 	echo " --enable-werror           Treat compiler warnings as errors"
 	echo " --enable-asan             Enable address sanitizer"
 	echo " --enable-ubsan            Enable undefined behavior sanitizer"
@@ -51,12 +50,12 @@ function usage()
 	echo "                           No path required."
 	echo " rdma                      [disabled]"
 	echo "                           No path required."
+	echo " rdma-srq                  [disabled]"
+	echo "                           No path required."
 	echo " shared                    Required to build spdk shared libraries."
 	echo "                           No path required."
 	echo " iscsi-initiator           [disabled]"
 	echo "                           No path required."
-	echo " raid                      [disabled]"
-	echo "                           No path required."
 	echo " vtune                     Required to profile I/O under Intel VTune Amplifier XE."
 	echo "                           example: /opt/intel/vtune_amplifier_xe_version"
 	echo ""
@@ -93,8 +92,11 @@ for i in "$@"; do
 		--disable-debug)
 			CONFIG_DEBUG=n
 			;;
-		--enable-log-bt=*)
-			CONFIG_LOG_BACKTRACE=${i#*=}
+		--enable-log-bt)
+			CONFIG_LOG_BACKTRACE=y
+			;;
+		--disable-log-bt)
+			CONFIG_LOG_BACKTRACE=n
 			;;
 		--enable-asan)
 			CONFIG_ASAN=y
@@ -148,18 +150,23 @@ for i in "$@"; do
 			CONFIG_RBD=n
 			;;
 		--with-raid)
-			CONFIG_RAID=y
-			echo "Warning: the RAID module does not support multiple iovecs per request and will not work with vhost"
+			echo "--with-raid option ignored and is now deprecated.  RAID module is always enabled."
 			;;
 		--without-raid)
-			CONFIG_RAID=n
+			echo "--without-raid option ignored and is now deprecated.  RAID module is always enabled."
 			;;
 		--with-rdma)
 			CONFIG_RDMA=y
 			;;
 		--without-rdma)
 			CONFIG_RDMA=n
 			;;
+		--with-rdma-srq)
+			CONFIG_RDMA_SRQ=y
+			;;
+		--without-rdma-srq)
+			CONFIG_RDMA_SRQ=n
+			;;
 		--with-shared)
 			CONFIG_SHARED=y
 			;;
@@ -247,15 +254,6 @@ for i in "$@"; do
 	esac
 done
 
-case "$CONFIG_LOG_BACKTRACE" in
-	""|ERROR|WARN|NOTICE|DEBUG)
-		;;
-	*)
-		echo "Invalid '--enable-log-bt=$CONFIG_LOG_BACKTRACE' option. Valid levels are ERROR, WARN, NOTICE or DEBUG"
-		exit 1
-		;;
-esac
-
 if [ -z "$CONFIG_ENV" ]; then
 	if [ -z "$CONFIG_DPDK_DIR" ]; then
 		rootdir=$(readlink -f $(dirname $0))
@@ -339,6 +337,7 @@ than or equal to 4.14 will see significantly reduced performance.
 fi
 
 if [[ "$CONFIG_CRYPTO" = "y" ]]; then
+	echo NOTE: Crypto is currently marked as experimental.  Do not use in production.
 	set +e
 	nasm_ver=$(nasm -v | sed 's/[^0-9]*//g' | awk '{print substr ($0, 0, 5)}')
 	ipsec="$(find /usr -name intel-ipsec-mb.h 2>/dev/null)"
@@ -429,6 +428,9 @@ fi
 if [ -n "$CONFIG_RDMA_SEND_WITH_INVAL" ]; then
 	echo "CONFIG_RDMA_SEND_WITH_INVAL?=$CONFIG_RDMA_SEND_WITH_INVAL" >> CONFIG.local
 fi
+if [ -n "$CONFIG_RDMA_SRQ" ]; then
+	echo "CONFIG_RDMA_SRQ?=$CONFIG_RDMA_SRQ" >> CONFIG.local
+fi
 if [ -n "$CONFIG_SHARED" ]; then
 	echo "CONFIG_SHARED?=$CONFIG_SHARED" >> CONFIG.local
 fi
@@ -438,9 +440,6 @@ fi
 if [ -n "$CONFIG_RBD" ]; then
 	echo "CONFIG_RBD?=$CONFIG_RBD" >> CONFIG.local
 fi
-if [ -n "$CONFIG_RAID" ]; then
-	echo "CONFIG_RAID?=$CONFIG_RAID" >> CONFIG.local
-fi
 if [ -n "$CONFIG_VTUNE" ]; then
 	echo "CONFIG_VTUNE?=$CONFIG_VTUNE" >> CONFIG.local
 fi
@@ -462,21 +461,6 @@ if [ -n "$DESTDIR" ]; then
 	echo "DESTDIR?=$DESTDIR" >> CONFIG.local
 fi
 
-python_command=
-for interpreter in python3 python2 python ; do
-	type $interpreter > /dev/null 2>&1 || continue
-	python_command=$interpreter
-	break
-done
-if [[ -z "$python_command" ]] ; then
-	echo "Could not find python interpreter! Bailing out."
-	exit 1
-fi
-
-rm -f PYTHON_COMMAND
-echo $python_command > PYTHON_COMMAND
-$python_command scripts/genconfig.py > config.h
-
 echo "done."
 if [[ "$OSTYPE" == "freebsd"* ]]; then
 	echo "Type 'gmake' to build."