Add FreeBSD support to OpenZFS

Add the FreeBSD platform code to the OpenZFS repository.  As of this
commit the source can be compiled and tested on FreeBSD 11 and 12.
Subsequent commits are now required to compile on FreeBSD and Linux.
Additionally, they must pass the ZFS Test Suite on FreeBSD which is
being run by the CI.  As of this commit 1230 tests pass on FreeBSD
and there are no unexpected failures.

Reviewed-by: Sean Eric Fagan <sef@ixsystems.com>
Reviewed-by: Jorgen Lundman <lundman@lundman.net>
Reviewed-by: Richard Laager <rlaager@wiktel.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Co-authored-by: Ryan Moeller <ryan@iXsystems.com>
Signed-off-by: Matt Macy <mmacy@FreeBSD.org>
Signed-off-by: Ryan Moeller <ryan@iXsystems.com>
Closes openzfs#898 
Closes openzfs#8987

.github/suppressions.txt

@@ -1,3 +1,6 @@
 preprocessorErrorDirective:./module/zfs/vdev_raidz_math_avx512f.c:243
 preprocessorErrorDirective:./module/zfs/vdev_raidz_math_sse2.c:266
-
+uninitvar:module/os/freebsd/zfs/vdev_geom.c
+uninitvar:module/os/freebsd/zfs/zfs_vfsops.c
+uninitvar:module/os/freebsd/spl/spl_zone.c
+uninitvar:lib/libzutil/os/freebsd/zutil_import_os.c

.gitignore

@@ -62,4 +62,9 @@ cscope.*
 *.patch
 *.orig
 *.log
+*.tmp
 venv
+
+*.so
+*.so.debug
+*.so.full

Makefile.am

@@ -104,8 +104,9 @@ commitcheck:
 	fi
 
 cstyle:
-	@find ${top_srcdir} -name build -prune -o -name '*.[hc]' \
-		! -name 'zfs_config.*' ! -name '*.mod.c' -type f \
+	@find ${top_srcdir} -name build -prune -o -type f -name '*.[hc]' \
+		! -name 'zfs_config.*' ! -name '*.mod.c' \
+		! -name 'opt_global.h' ! -name '*_if*.h' \
 		-exec ${top_srcdir}/scripts/cstyle.pl -cpP {} \+
 
 filter_executable = -exec test -x '{}' \; -print

README.md

@@ -19,8 +19,6 @@ This repository contains the code for running OpenZFS on Linux and FreeBSD.
 Full documentation for installing OpenZFS on your favorite Linux distribution can
 be found at the [ZoL Site](https://zfsonlinux.org/).
 
-FreeBSD support is a work in progress.  See the [PR](https://github.com/openzfs/zfs/pull/8987).
-
 # Contribute & Develop
 
 We have a separate document with [contribution guidelines](./.github/CONTRIBUTING.md).
@@ -34,3 +32,4 @@ For more details see the NOTICE, LICENSE and COPYRIGHT files; `UCRL-CODE-235197`
 
 # Supported Kernels
   * The `META` file contains the officially recognized supported Linux kernel versions.
+  * Supported FreeBSD versions are 12-STABLE and 13-CURRENT.

cmd/Makefile.am

@@ -1,10 +1,10 @@
 SUBDIRS  = zfs zpool zdb zhack zinject zstream zstreamdump ztest
-SUBDIRS += fsck_zfs vdev_id raidz_test zgenhostid
+SUBDIRS += fsck_zfs vdev_id raidz_test
 
 if USING_PYTHON
 SUBDIRS += arcstat arc_summary dbufstat
 endif
 
 if BUILD_LINUX
-SUBDIRS += mount_zfs zed zvol_id zvol_wait
+SUBDIRS += mount_zfs zed zgenhostid zvol_id zvol_wait
 endif

cmd/zpool/Makefile.am

@@ -11,6 +11,10 @@ zpool_SOURCES = \
 	zpool_util.h \
 	zpool_vdev.c
 
+if BUILD_FREEBSD
+zpool_SOURCES += os/freebsd/zpool_vdev_os.c
+endif
+
 if BUILD_LINUX
 zpool_SOURCES += os/linux/zpool_vdev_os.c
 endif
@@ -20,6 +24,9 @@ zpool_LDADD = \
 	$(top_builddir)/lib/libuutil/libuutil.la \
 	$(top_builddir)/lib/libzfs/libzfs.la
 
+if BUILD_FREEBSD
+zpool_LDADD += -L/usr/local/lib -lintl -lgeom
+endif
 zpool_LDADD += -lm $(LIBBLKID)
 
 zpoolconfdir = $(sysconfdir)/zfs/zpool.d

cmd/zpool/os/freebsd/zpool_vdev_os.c

@@ -0,0 +1,113 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2016, 2017 Intel Corporation.
+ * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
+ */
+
+/*
+ * Functions to convert between a list of vdevs and an nvlist representing the
+ * configuration.  Each entry in the list can be one of:
+ *
+ * 	Device vdevs
+ * 		disk=(path=..., devid=...)
+ * 		file=(path=...)
+ *
+ * 	Group vdevs
+ * 		raidz[1|2]=(...)
+ * 		mirror=(...)
+ *
+ * 	Hot spares
+ *
+ * While the underlying implementation supports it, group vdevs cannot contain
+ * other group vdevs.  All userland verification of devices is contained within
+ * this file.  If successful, the nvlist returned can be passed directly to the
+ * kernel; we've done as much verification as possible in userland.
+ *
+ * Hot spares are a special case, and passed down as an array of disk vdevs, at
+ * the same level as the root of the vdev tree.
+ *
+ * The only function exported by this file is 'make_root_vdev'.  The
+ * function performs several passes:
+ *
+ * 	1. Construct the vdev specification.  Performs syntax validation and
+ *         makes sure each device is valid.
+ * 	2. Check for devices in use.  Using libdiskmgt, makes sure that no
+ *         devices are also in use.  Some can be overridden using the 'force'
+ *         flag, others cannot.
+ * 	3. Check for replication errors if the 'force' flag is not specified.
+ *         validates that the replication level is consistent across the
+ *         entire pool.
+ * 	4. Call libzfs to label any whole disks with an EFI label.
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libintl.h>
+#include <libnvpair.h>
+#include <libzutil.h>
+#include <limits.h>
+#include <sys/spa.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <paths.h>
+#include <sys/stat.h>
+#include <sys/disk.h>
+#include <sys/mntent.h>
+#include <libgeom.h>
+
+#include "zpool_util.h"
+#include <sys/zfs_context.h>
+
+int
+check_device(const char *name, boolean_t force, boolean_t isspare,
+    boolean_t iswholedisk)
+{
+	char path[MAXPATHLEN];
+
+	if (strncmp(name, _PATH_DEV, sizeof (_PATH_DEV) - 1) != 0)
+		snprintf(path, sizeof (path), "%s%s", _PATH_DEV, name);
+	else
+		strlcpy(path, name, sizeof (path));
+
+	return (check_file(path, force, isspare));
+}
+
+boolean_t
+check_sector_size_database(char *path, int *sector_size)
+{
+	return (0);
+}
+
+void
+zpool_vdev_enable_file(struct stat64 *statbuf, boolean_t *wholedisk)
+{
+	if (S_ISCHR(statbuf->st_mode)) {
+		statbuf->st_mode &= ~S_IFCHR;
+		statbuf->st_mode |= S_IFBLK;
+		*wholedisk = B_FALSE;
+	}
+}

cmd/zpool/zpool_vdev.c

@@ -941,6 +941,10 @@ make_disks(zpool_handle_t *zhp, nvlist_t *nv)
 		if (fd == -1) {
 			if (errno == EBUSY)
 				is_exclusive = 1;
+#ifdef __FreeBSD__
+			if (errno == EPERM)
+				is_exclusive = 1;
+#endif
 		} else {
 			(void) close(fd);
 		}

config/Rules.am

@@ -14,26 +14,48 @@ DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/lib/libspl/include/os/linux
 endif
 
+if BUILD_FREEBSD
+DEFAULT_INCLUDES += \
+	-I$(top_srcdir)/lib/libspl/include/os/freebsd
+endif
+
 AM_LIBTOOLFLAGS = --silent
 
 AM_CFLAGS  = -std=gnu99 -Wall -Wstrict-prototypes -fno-strict-aliasing
 AM_CFLAGS += $(NO_OMIT_FRAME_POINTER)
 AM_CFLAGS += $(DEBUG_CFLAGS)
 AM_CFLAGS += $(ASAN_CFLAGS)
-AM_CFLAGS += $(CODE_COVERAGE_CFLAGS)
+AM_CFLAGS += $(CODE_COVERAGE_CFLAGS) $(NO_FORMAT_ZERO_LENGTH)
+if BUILD_FREEBSD
+AM_CFLAGS += -fPIC -Werror -Wno-unknown-pragmas -Wno-enum-conversion
+AM_CFLAGS += -include $(top_srcdir)/include/os/freebsd/spl/sys/ccompile.h
+AM_CFLAGS += -I/usr/include -I/usr/local/include
+AM_CFLAGS += -D_MACHINE_ENDIAN_H_
+endif
 
 AM_CPPFLAGS  = -D_GNU_SOURCE
 AM_CPPFLAGS += -D_REENTRANT
 AM_CPPFLAGS += -D_FILE_OFFSET_BITS=64
 AM_CPPFLAGS += -D_LARGEFILE64_SOURCE
 AM_CPPFLAGS += -DHAVE_LARGE_STACKS=1
-AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\"
 AM_CPPFLAGS += -DLIBEXECDIR=\"$(libexecdir)\"
 AM_CPPFLAGS += -DRUNSTATEDIR=\"$(runstatedir)\"
 AM_CPPFLAGS += -DSBINDIR=\"$(sbindir)\"
 AM_CPPFLAGS += -DSYSCONFDIR=\"$(sysconfdir)\"
 AM_CPPFLAGS += $(DEBUG_CPPFLAGS)
 AM_CPPFLAGS += $(CODE_COVERAGE_CPPFLAGS)
+if BUILD_LINUX
+AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\"
+endif
+if BUILD_FREEBSD
+AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-freebsd-user\"
+endif
 
 AM_LDFLAGS  = $(DEBUG_LDFLAGS)
 AM_LDFLAGS += $(ASAN_LDFLAGS)
+
+if BUILD_FREEBSD
+AM_LDFLAGS += -fstack-protector-strong -shared
+AM_LDFLAGS += -Wl,-x -Wl,--fatal-warnings -Wl,--warn-shared-textrel
+AM_LDFLAGS += -lm
+endif

config/always-arch.m4

@@ -17,7 +17,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_ARCH], [
 	i?86)
 		TARGET_CPU=i386
 		;;
-	x86_64)
+	amd64|x86_64)
 		TARGET_CPU=x86_64
 		;;
 	powerpc*)

config/always-compiler-options.m4

@@ -87,6 +87,27 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION], [
 	AC_SUBST([NO_FORMAT_TRUNCATION])
 ])
 
+dnl #
+dnl # Check if gcc supports -Wno-format-truncation option.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_ZERO_LENGTH], [
+	AC_MSG_CHECKING([whether $CC supports -Wno-format-zero-length])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Werror -Wno-format-zero-length"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		NO_FORMAT_ZERO_LENGTH=-Wno-format-zero-length
+		AC_MSG_RESULT([yes])
+	], [
+		NO_FORMAT_ZERO_LENGTH=
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([NO_FORMAT_ZERO_LENGTH])
+])
+
 
 dnl #
 dnl # Check if gcc supports -Wno-bool-compare option.

config/kernel.m4

@@ -2,29 +2,31 @@ dnl #
 dnl # Default ZFS kernel configuration
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [
-	dnl # Setup the kernel build environment.
-	ZFS_AC_KERNEL
-	ZFS_AC_QAT
-
-	dnl # Sanity checks for module building and CONFIG_* defines
-	ZFS_AC_KERNEL_TEST_MODULE
-	ZFS_AC_KERNEL_CONFIG_DEFINED
-
-	dnl # Sequential ZFS_LINUX_TRY_COMPILE tests
-	ZFS_AC_KERNEL_FPU_HEADER
-	ZFS_AC_KERNEL_WAIT_QUEUE_ENTRY_T
-	ZFS_AC_KERNEL_MISC_MINOR
-	ZFS_AC_KERNEL_DECLARE_EVENT_CLASS
-
-	dnl # Parallel ZFS_LINUX_TEST_SRC / ZFS_LINUX_TEST_RESULT tests
-	ZFS_AC_KERNEL_TEST_SRC
-	ZFS_AC_KERNEL_TEST_RESULT
-
-	AS_IF([test "$LINUX_OBJ" != "$LINUX"], [
-		KERNEL_MAKE="$KERNEL_MAKE O=$LINUX_OBJ"
-	])
+	AM_COND_IF([BUILD_LINUX], [
+		dnl # Setup the kernel build environment.
+		ZFS_AC_KERNEL
+		ZFS_AC_QAT
+
+		dnl # Sanity checks for module building and CONFIG_* defines
+		ZFS_AC_KERNEL_TEST_MODULE
+		ZFS_AC_KERNEL_CONFIG_DEFINED
+
+		dnl # Sequential ZFS_LINUX_TRY_COMPILE tests
+		ZFS_AC_KERNEL_FPU_HEADER
+		ZFS_AC_KERNEL_WAIT_QUEUE_ENTRY_T
+		ZFS_AC_KERNEL_MISC_MINOR
+		ZFS_AC_KERNEL_DECLARE_EVENT_CLASS
+
+		dnl # Parallel ZFS_LINUX_TEST_SRC / ZFS_LINUX_TEST_RESULT tests
+		ZFS_AC_KERNEL_TEST_SRC
+		ZFS_AC_KERNEL_TEST_RESULT
+
+		AS_IF([test "$LINUX_OBJ" != "$LINUX"], [
+			KERNEL_MAKE="$KERNEL_MAKE O=$LINUX_OBJ"
+		])
 
-	AC_SUBST(KERNEL_MAKE)
+		AC_SUBST(KERNEL_MAKE)
+	])
 ])
 
 dnl #

config/toolchain-simd.m4

@@ -3,7 +3,7 @@ dnl # Checks if host toolchain supports SIMD instructions
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [
 	case "$host_cpu" in
-		x86_64 | x86 | i686)
+		amd64 | x86_64 | x86 | i686)
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE2
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE3

config/user.m4

@@ -4,25 +4,26 @@ dnl #
 AC_DEFUN([ZFS_AC_CONFIG_USER], [
 	ZFS_AC_CONFIG_USER_GETTEXT
 	ZFS_AC_CONFIG_USER_MOUNT_HELPER
-	ZFS_AC_CONFIG_USER_UDEV
-	ZFS_AC_CONFIG_USER_SYSTEMD
 	ZFS_AC_CONFIG_USER_SYSVINIT
 	ZFS_AC_CONFIG_USER_DRACUT
 	ZFS_AC_CONFIG_USER_ZLIB
-	ZFS_AC_CONFIG_USER_LIBUUID
+	AM_COND_IF([BUILD_LINUX], [
+		ZFS_AC_CONFIG_USER_UDEV
+		ZFS_AC_CONFIG_USER_SYSTEMD
+		ZFS_AC_CONFIG_USER_LIBUUID
+		ZFS_AC_CONFIG_USER_LIBBLKID
+	])
 	ZFS_AC_CONFIG_USER_LIBTIRPC
-	ZFS_AC_CONFIG_USER_LIBBLKID
 	ZFS_AC_CONFIG_USER_LIBUDEV
 	ZFS_AC_CONFIG_USER_LIBSSL
 	ZFS_AC_CONFIG_USER_LIBAIO
 	ZFS_AC_CONFIG_USER_RUNSTATEDIR
 	ZFS_AC_CONFIG_USER_MAKEDEV_IN_SYSMACROS
 	ZFS_AC_CONFIG_USER_MAKEDEV_IN_MKDEV
 	ZFS_AC_CONFIG_USER_ZFSEXEC
-
 	ZFS_AC_TEST_FRAMEWORK
 
-	AC_CHECK_FUNCS([mlockall strlcat strlcpy])
+	AC_CHECK_FUNCS([issetugid mlockall strlcat strlcpy])
 ])
 
 dnl #