SHA2 reworking and API for iterating over multiple implementations

cmd/ztest.c

@@ -135,6 +135,7 @@
 #include <libnvpair.h>
 #include <libzutil.h>
 #include <sys/crypto/icp.h>
+#include <sys/zfs_impl.h>
 #if (__GLIBC__ && !__UCLIBC__)
 #include <execinfo.h> /* for backtrace() */
 #endif
@@ -6410,6 +6411,7 @@ ztest_blake3(ztest_ds_t *zd, uint64_t id)
 	int i, *ptr;
 	uint32_t size;
 	BLAKE3_CTX ctx;
+	const zfs_impl_t *blake3 = zfs_impl_get_ops("blake3");
 
 	size = ztest_random_blocksize();
 	buf = umem_alloc(size, UMEM_NOFAIL);
@@ -6434,7 +6436,7 @@ ztest_blake3(ztest_ds_t *zd, uint64_t id)
 		void *res2 = &zc_res2;
 
 		/* BLAKE3_KEY_LEN = 32 */
-		VERIFY0(blake3_impl_setname("generic"));
+		VERIFY0(blake3->setname("generic"));
 		templ = abd_checksum_blake3_tmpl_init(&salt);
 		Blake3_InitKeyed(&ctx, salt_ptr);
 		Blake3_Update(&ctx, buf, size);
@@ -6443,7 +6445,7 @@ ztest_blake3(ztest_ds_t *zd, uint64_t id)
 		ZIO_CHECKSUM_BSWAP(&zc_ref2);
 		abd_checksum_blake3_tmpl_free(templ);
 
-		VERIFY0(blake3_impl_setname("cycle"));
+		VERIFY0(blake3->setname("cycle"));
 		while (run_count-- > 0) {
 
 			/* Test current implementation */

config/always-arch.m4

@@ -22,6 +22,9 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_ARCH], [
 	aarch64*)
 		TARGET_CPU=aarch64
 		;;
+	armv*)
+		TARGET_CPU=arm
+		;;
 	sparc64)
 		TARGET_CPU=sparc64
 		;;
@@ -31,7 +34,8 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_ARCH], [
 	esac
 
 	AM_CONDITIONAL([TARGET_CPU_AARCH64], test $TARGET_CPU = aarch64)
-	AM_CONDITIONAL([TARGET_CPU_X86_64], test $TARGET_CPU = x86_64)
+	AM_CONDITIONAL([TARGET_CPU_X86_64],  test $TARGET_CPU = x86_64)
 	AM_CONDITIONAL([TARGET_CPU_POWERPC], test $TARGET_CPU = powerpc)
 	AM_CONDITIONAL([TARGET_CPU_SPARC64], test $TARGET_CPU = sparc64)
+	AM_CONDITIONAL([TARGET_CPU_ARM],     test $TARGET_CPU = arm)
 ])

include/Makefile.am

@@ -75,6 +75,7 @@ COMMON_H = \
 	sys/rrwlock.h \
 	sys/sa.h \
 	sys/sa_impl.h \
+	sys/sha2.h \
 	sys/skein.h \
 	sys/spa.h \
 	sys/spa_checkpoint.h \
@@ -124,6 +125,7 @@ COMMON_H = \
 	sys/zfs_delay.h \
 	sys/zfs_file.h \
 	sys/zfs_fuid.h \
+	sys/zfs_impl.h \
 	sys/zfs_project.h \
 	sys/zfs_quota.h \
 	sys/zfs_racct.h \

include/os/freebsd/Makefile.am

@@ -51,6 +51,8 @@ noinst_HEADERS = \
 	%D%/spl/sys/sid.h \
 	%D%/spl/sys/sig.h \
 	%D%/spl/sys/simd.h \
+	%D%/spl/sys/simd_aarch64.h \
+	%D%/spl/sys/simd_arm.h \
 	%D%/spl/sys/simd_powerpc.h \
 	%D%/spl/sys/simd_x86.h \
 	%D%/spl/sys/spl_condvar.h \
@@ -79,7 +81,6 @@ noinst_HEADERS = \
 	%D%/spl/sys/zone.h \
 	\
 	%D%/zfs/sys/freebsd_crypto.h \
-	%D%/zfs/sys/sha2.h \
 	%D%/zfs/sys/vdev_os.h \
 	%D%/zfs/sys/zfs_bootenv_os.h \
 	%D%/zfs/sys/zfs_context_os.h \

include/os/freebsd/spl/sys/mod_os.h

@@ -97,6 +97,12 @@
 #define	blake3_param_set_args(var) \
     CTLTYPE_STRING, NULL, 0, blake3_param, "A"
 
+#define	sha256_param_set_args(var) \
+    CTLTYPE_STRING, NULL, 0, sha256_param, "A"
+
+#define	sha512_param_set_args(var) \
+    CTLTYPE_STRING, NULL, 0, sha512_param, "A"
+
 #include <sys/kernel.h>
 #define	module_init(fn) \
 static void \

include/os/freebsd/spl/sys/simd.h

@@ -32,6 +32,12 @@
 #if defined(__amd64__) || defined(__i386__)
 #include <sys/simd_x86.h>
 
+#elif defined(__arm__)
+#include <sys/simd_arm.h>
+
+#elif defined(__aarch64__)
+#include <sys/simd_aarch64.h>
+
 #elif defined(__powerpc__)
 #include <sys/simd_powerpc.h>
 

include/os/freebsd/spl/sys/simd_aarch64.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * SIMD support:
+ *
+ * Following functions should be called to determine whether CPU feature
+ * is supported. All functions are usable in kernel and user space.
+ * If a SIMD algorithm is using more than one instruction set
+ * all relevant feature test functions should be called.
+ *
+ * Supported features:
+ *   zfs_neon_available()
+ *   zfs_sha256_available()
+ *   zfs_sha512_available()
+ */
+
+#ifndef _FREEBSD_SIMD_AARCH64_H
+#define	_FREEBSD_SIMD_AARCH64_H
+
+#include <sys/types.h>
+#include <machine/elf.h>
+
+#define	kfpu_allowed()		1
+#define	kfpu_initialize(tsk)	do {} while (0)
+#define	kfpu_begin()		do {} while (0)
+#define	kfpu_end()		do {} while (0)
+#define	kfpu_init()		(0)
+#define	kfpu_fini()		do {} while (0)
+
+/*
+ * Check if NEON is available
+ */
+static inline boolean_t
+zfs_neon_available(void)
+{
+	return (elf_hwcap & HWCAP_FP);
+}
+
+/*
+ * Check if SHA256 is available
+ */
+static inline boolean_t
+zfs_sha256_available(void)
+{
+	return (elf_hwcap & HWCAP_SHA2);
+}
+
+/*
+ * Check if SHA512 is available
+ */
+static inline boolean_t
+zfs_sha512_available(void)
+{
+	return (elf_hwcap & HWCAP_SHA512);
+}
+
+#endif /* _FREEBSD_SIMD_AARCH64_H */

include/os/freebsd/spl/sys/simd_arm.h

@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * SIMD support:
+ *
+ * Following functions should be called to determine whether CPU feature
+ * is supported. All functions are usable in kernel and user space.
+ * If a SIMD algorithm is using more than one instruction set
+ * all relevant feature test functions should be called.
+ *
+ * Supported features:
+ *   zfs_neon_available()
+ *   zfs_sha256_available()
+ */
+
+#ifndef _FREEBSD_SIMD_ARM_H
+#define	_FREEBSD_SIMD_ARM_H
+
+#include <sys/types.h>
+#include <machine/elf.h>
+
+#define	kfpu_allowed()		1
+#define	kfpu_initialize(tsk)	do {} while (0)
+#define	kfpu_begin()		do {} while (0)
+#define	kfpu_end()		do {} while (0)
+#define	kfpu_init()		(0)
+#define	kfpu_fini()		do {} while (0)
+
+/*
+ * Check if NEON is available
+ */
+static inline boolean_t
+zfs_neon_available(void)
+{
+	return (elf_hwcap & HWCAP_NEON);
+}
+
+/*
+ * Check if SHA256 is available
+ */
+static inline boolean_t
+zfs_sha256_available(void)
+{
+	return (elf_hwcap2 & HWCAP2_SHA2);
+}
+
+#endif /* _FREEBSD_SIMD_ARM_H */

include/os/freebsd/spl/sys/simd_powerpc.h

@@ -1,38 +1,32 @@
 /*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
+ * Copyright (C) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
+ * All rights reserved.
  *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or https://opensource.org/licenses/CDDL-1.0.
- * See the License for the specific language governing permissions
- * and limitations under the License.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
  *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
  *
- * CDDL HEADER END
- */
-
-/*
- * Copyright (C) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
+ * $FreeBSD$
  */
 
 /*
- * USER API:
- *
- * Kernel fpu methods:
- *	kfpu_allowed()
- *	kfpu_begin()
- *	kfpu_end()
- *	kfpu_init()
- *	kfpu_fini()
- *
  * SIMD support:
  *
  * Following functions should be called to determine whether CPU feature

include/os/freebsd/spl/sys/simd_x86.h

@@ -173,6 +173,19 @@ zfs_avx2_available(void)
 	return (has_avx2 && __ymm_enabled());
 }
 
+/*
+ * Check if SHA_NI instruction set is available
+ */
+static inline boolean_t
+zfs_shani_available(void)
+{
+	boolean_t has_shani;
+
+	has_shani = (cpu_stdext_feature & CPUID_STDEXT_SHA) != 0;
+
+	return (has_shani && __ymm_enabled());
+}
+
 /*
  * AVX-512 family of instruction sets:
  *