ICP: Improve AES-GCM performance

Currently SIMD accelerated AES-GCM performance is limited by two
factors:

a. The need to disable preemption and interrupts and save the FPU
state before using it and to do the reverse when done. Due to the
way the code is organized (see (b) below) we have to pay this price
twice for each 16 byte GCM block processed.

b. Most processing is done in C, operating on single GCM blocks.
The use of SIMD instructions is limited to the AES encryption of the
counter block (AES-NI) and the Galois multiplication (PCLMULQDQ).
This leads to the FPU not being fully utilized for crypto
operations.

To solve (a) we do crypto processing in larger chunks while owning
the FPU. An `icp_gcm_avx_chunk_size` module parameter was introduced
to make this chunk size tweakable. It defaults to 32 KiB. This step
alone roughly doubles performance. (b) is tackled by porting and
using the highly optimized openssl AES-GCM assembler routines, which
do all the processing (CTR, AES, GMULT) in a single routine. Both
steps together result in up to 32x reduction of the time spend in
the en/decryption routines, leading up to approximately 12x
throughput increase for large (128 KiB) blocks.

Lastly, this commit changes the default encryption algorithm from
AES-CCM to AES-GCM when setting the `encryption=on` property.

Reviewed-By: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-By: Jason King <jason.king@joyent.com>
Reviewed-By: Tom Caputi <tcaputi@datto.com>
Reviewed-By: Richard Laager <rlaager@wiktel.com>
Signed-off-by: Attila Fülöp <attila@fueloep.org>
Closes openzfs#9749

COPYRIGHT

@@ -20,6 +20,10 @@ notable exceptions and their respective licenses include:
   * AES Implementation: module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl
   * PBKDF2 Implementation: lib/libzfs/THIRDPARTYLICENSE.openssl
   * SPL Implementation: module/spl/THIRDPARTYLICENSE.gplv2
+  * GCM Implementaion: module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.cryptogams
+  * GCM Implementaion: module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.openssl
+  * GHASH Implementaion: module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.cryptogams
+  * GHASH Implementaion: module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.openssl
 
 This product includes software developed by the OpenSSL Project for use
 in the OpenSSL Toolkit (http://www.openssl.org/)

config/toolchain-simd.m4

@@ -23,6 +23,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VL
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE
 			;;
 	esac
 ])
@@ -401,3 +402,23 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ], [
 		AC_MSG_RESULT([no])
 	])
 ])
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE], [
+	AC_MSG_CHECKING([whether host toolchain supports MOVBE])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		void main()
+		{
+			__asm__ __volatile__("movbe 0(%eax), %eax");
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_MOVBE], 1, [Define if host toolchain supports MOVBE])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])

include/linux/simd_x86.h

@@ -382,7 +382,8 @@ typedef enum cpuid_inst_sets {
 	AVX512ER,
 	AVX512VL,
 	AES,
-	PCLMULQDQ
+	PCLMULQDQ,
+    MOVBE
 } cpuid_inst_sets_t;
 
 /*
@@ -406,6 +407,7 @@ typedef struct cpuid_feature_desc {
 #define	_AVX512VL_BIT		(1U << 31) /* if used also check other levels */
 #define	_AES_BIT		(1U << 25)
 #define	_PCLMULQDQ_BIT		(1U << 1)
+#define _MOVBE_BIT          (1U << 22)
 
 /*
  * Descriptions of supported instruction sets
@@ -433,6 +435,7 @@ static const cpuid_feature_desc_t cpuid_features[] = {
 	[AVX512VL]	= {7U, 0U, _AVX512ER_BIT,	EBX	},
 	[AES]		= {1U, 0U, _AES_BIT,		ECX	},
 	[PCLMULQDQ]	= {1U, 0U, _PCLMULQDQ_BIT,	ECX	},
+    [MOVBE]     = {1U, 0U, _MOVBE_BIT,      ECX },
 };
 
 /*
@@ -505,6 +508,7 @@ CPUID_FEATURE_CHECK(avx512er, AVX512ER);
 CPUID_FEATURE_CHECK(avx512vl, AVX512VL);
 CPUID_FEATURE_CHECK(aes, AES);
 CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ);
+CPUID_FEATURE_CHECK(movbe, MOVBE);
 
 #endif /* !defined(_KERNEL) */
 
@@ -719,6 +723,23 @@ zfs_pclmulqdq_available(void)
 #endif
 }
 
+/*
+ * Check if MOVBE instruction is available
+ */
+static inline boolean_t
+zfs_movbe_available(void)
+{
+#if defined(_KERNEL)
+#if defined(X86_FEATURE_MOVBE)
+       return (!!boot_cpu_has(X86_FEATURE_MOVBE));
+#else
+       return (B_FALSE);
+#endif
+#elif !defined(_KERNEL)
+    return (__cpuid_has_movbe());
+#endif
+}
+
 /*
  * AVX-512 family of instruction sets:
  *

include/sys/zio.h

@@ -118,7 +118,7 @@ enum zio_encrypt {
 	ZIO_CRYPT_FUNCTIONS
 };
 
-#define	ZIO_CRYPT_ON_VALUE	ZIO_CRYPT_AES_256_CCM
+#define	ZIO_CRYPT_ON_VALUE	ZIO_CRYPT_AES_256_GCM
 #define	ZIO_CRYPT_DEFAULT	ZIO_CRYPT_OFF
 
 /* macros defining encryption lengths */

lib/libicp/Makefile.am

@@ -20,6 +20,8 @@ ASM_SOURCES_AS = \
 	asm-x86_64/aes/aes_amd64.S \
 	asm-x86_64/aes/aes_aesni.S \
 	asm-x86_64/modes/gcm_pclmulqdq.S \
+	asm-x86_64/modes/aesni-gcm-x86_64.S \
+	asm-x86_64/modes/ghash-x86_64.S \
 	asm-x86_64/sha1/sha1-x86_64.S \
 	asm-x86_64/sha2/sha256_impl.S \
 	asm-x86_64/sha2/sha512_impl.S

man/man8/zfs.8

@@ -1440,7 +1440,7 @@ Selecting
 .Sy encryption Ns = Ns Sy on
 when creating a dataset indicates that the default encryption suite will be
 selected, which is currently
-.Sy aes-256-ccm .
+.Sy aes-256-gcm .
 In order to provide consistent data protection, encryption must be specified at
 dataset creation time and it cannot be changed afterwards.
 .Pp

module/icp/Makefile.in

@@ -13,6 +13,16 @@ ASM_SOURCES += asm-x86_64/modes/gcm_pclmulqdq.o
 ASM_SOURCES += asm-x86_64/sha1/sha1-x86_64.o
 ASM_SOURCES += asm-x86_64/sha2/sha256_impl.o
 ASM_SOURCES += asm-x86_64/sha2/sha512_impl.o
+ASM_SOURCES += asm-x86_64/aes/aeskey.o
+ASM_SOURCES += asm-x86_64/aes/aes_amd64.o
+ASM_SOURCES += asm-x86_64/aes/aes_aesni.o
+ASM_SOURCES += asm-x86_64/modes/gcm_pclmulqdq.o
+ASM_SOURCES += asm-x86_64/modes/aesni-gcm-x86_64.o
+ASM_SOURCES += asm-x86_64/modes/ghash-x86_64.o
+ASM_SOURCES += asm-x86_64/sha1/sha1-x86_64.o
+ASM_SOURCES += asm-x86_64/sha2/sha256_impl.o
+ASM_SOURCES += asm-x86_64/sha2/sha512_impl.o
+
 endif
 
 ifeq ($(TARGET_ASM_DIR), asm-i386)
@@ -72,6 +82,13 @@ $(MODULE)-$(CONFIG_X86) += algs/modes/gcm_pclmulqdq.o
 $(MODULE)-$(CONFIG_X86) += algs/aes/aes_impl_aesni.o
 $(MODULE)-$(CONFIG_X86) += algs/aes/aes_impl_x86-64.o
 
+# Suppress objtool "can't find jump dest instruction at" warnings.  They
+# are caused by the constants which are defined in the text section of the
+# assembly file using .byte instructions (e.g. bswap_mask).  The objtool
+# utility tries to interpret them as opcodes and obviously fails doing so.
+OBJECT_FILES_NON_STANDARD_aesni-gcm-x86_64.o := y
+OBJECT_FILES_NON_STANDARD_ghash-x86_64.o := y
+
 ICP_DIRS = \
 	api \
 	core \