Introduce ZSTD compression to ZFS

Makefile.am

@@ -104,7 +104,8 @@ commitcheck:
 	fi
 
 cstyle:
-	@find ${top_srcdir} -name build -prune -o -name '*.[hc]' \
+	@find ${top_srcdir} \( -name build -o -path ./contrib/zstd \) -prune \
+		-o -name '*.[hc]' \
 		! -name 'zfs_config.*' ! -name '*.mod.c' -type f \
 		-exec ${top_srcdir}/scripts/cstyle.pl -cpP {} \+
 

cmd/dbufstat/dbufstat

@@ -343,7 +343,7 @@ def get_compstring(c):
                     "ZIO_COMPRESS_GZIP_6",  "ZIO_COMPRESS_GZIP_7",
                     "ZIO_COMPRESS_GZIP_8",  "ZIO_COMPRESS_GZIP_9",
                     "ZIO_COMPRESS_ZLE",     "ZIO_COMPRESS_LZ4",
-                    "ZIO_COMPRESS_FUNCTION"]
+                    "ZIO_COMPRESS_ZSTD",    "ZIO_COMPRESS_FUNCTION"]
 
     # If "-rr" option is used, don't convert to string representation
     if raw > 1:

cmd/zdb/zdb.c

@@ -6888,9 +6888,9 @@ zdb_decompress_block(abd_t *pabd, void *buf, void *lbuf, uint64_t lsize,
 			VERIFY0(random_get_pseudo_bytes(lbuf2, lsize));
 
 			if (zio_decompress_data(*cfuncp, pabd,
-			    lbuf, psize, lsize) == 0 &&
+			    lbuf, psize, lsize, NULL) == 0 &&
 			    zio_decompress_data(*cfuncp, pabd,
-			    lbuf2, psize, lsize) == 0 &&
+			    lbuf2, psize, lsize, NULL) == 0 &&
 			    bcmp(lbuf, lbuf2, lsize) == 0)
 				break;
 		}

config/ax_code_coverage.m4

@@ -222,7 +222,7 @@ CODE_COVERAGE_GENHTML_OPTIONS ?= $(CODE_COVERAGE_GENHTML_OPTIONS_DEFAULT)
 
 # Add any folders you want to ignore here
 # Ignore tmp and tests themselves
-CODE_COVERAGE_IGNORE_PATTERN ?= "/tmp/*" "*/tests/*"  
+CODE_COVERAGE_IGNORE_PATTERN ?= "/tmp/*" "*/tests/*" "*/contrib/zstd/*"
 
 GITIGNOREFILES ?=
 GITIGNOREFILES += $(CODE_COVERAGE_OUTPUT_FILE) $(CODE_COVERAGE_OUTPUT_DIRECTORY)

configure.ac

@@ -98,6 +98,10 @@ AC_CONFIG_FILES([
 	contrib/pyzfs/Makefile
 	contrib/pyzfs/setup.py
 	contrib/zcp/Makefile
+	contrib/zstd/Makefile
+	contrib/zstd/common/Makefile
+	contrib/zstd/compress/Makefile
+	contrib/zstd/decompress/Makefile
 	etc/Makefile
 	etc/default/Makefile
 	etc/init.d/Makefile
@@ -124,6 +128,7 @@ AC_CONFIG_FILES([
 	include/sys/fs/Makefile
 	include/sys/lua/Makefile
 	include/sys/sysevent/Makefile
+	include/sys/zstd/Makefile
 	lib/Makefile
 	lib/libavl/Makefile
 	lib/libefi/Makefile
@@ -152,6 +157,7 @@ AC_CONFIG_FILES([
 	lib/libzfs/libzfs_core.pc
 	lib/libzfs_core/Makefile
 	lib/libzpool/Makefile
+	lib/libzstd/Makefile
 	lib/libzutil/Makefile
 	man/Makefile
 	man/man1/Makefile
@@ -170,6 +176,7 @@ AC_CONFIG_FILES([
 	module/unicode/Makefile
 	module/zcommon/Makefile
 	module/zfs/Makefile
+	module/zstd/Makefile
 	rpm/Makefile
 	rpm/generic/Makefile
 	rpm/generic/zfs-dkms.spec

contrib/Makefile.am

@@ -1,2 +1,2 @@
-SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp
-DIST_SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp
+SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp zstd
+DIST_SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp zstd

contrib/zstd/.gitignore

@@ -0,0 +1 @@
+*.o.cmd

contrib/zstd/Makefile.am

@@ -0,0 +1,2 @@
+SUBDIRS = common compress decompress
+EXTRA_DIST = common compress decompress zstd.h

contrib/zstd/README.md

@@ -0,0 +1,24 @@
+# ZSTD Contrib Library Manual
+
+## Introduction
+
+This `contrib` contains the ZSTD library used in ZFS. It is heavily cut-down by
+dropping any unneeded files but otherwise is intentionally unmodified. Please do
+not alter these files in any way, besides upgrading to a newer ZSTD release.
+
+## Updating ZSTD
+
+To update ZSTD the following steps need to be taken:
+
+1. Grab the latest release of [ZSTD](https://github.com/facebook/zstd/releases).
+2. Replace (not merge) `common`, `compress`, `decompress` and `zstd.h` with the
+   new versions from `lib/`.
+3. Make sure any newly required files and/or folders are also included.
+   1. Add an empty file `Makefile.am` inside any new folder.
+   2. Add them to `AC_CONFIG_FILES` in `configure.ac` accordingly.
+   3. Make sure new files/folders are listed in
+      - `contrib/zstd/Makefile.am`
+      - `lib/libzstd/Makefile.am`
+      - `module/zstd/Makefile.in`
+      - this README
+4. Update the version `ZFS_MODULE_VERSION("x.y.z")` in `module/zstd/zstd.c`.

contrib/zstd/THIRDPARTYLICENSE

@@ -0,0 +1,30 @@
+BSD License
+
+For Zstandard software
+
+Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+ * Neither the name Facebook nor the names of its contributors may be used to
+   endorse or promote products derived from this software without specific
+   prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.