adaptive compression by @RubenKelevra

Signed-off-by: RubenKelevra <ruben@vfn-nrw.de>

cmd/dbufstat/dbufstat

@@ -335,15 +335,15 @@ def get_typestring(t):
 
 
 def get_compstring(c):
-    comp_strings = ["ZIO_COMPRESS_INHERIT", "ZIO_COMPRESS_ON",
-                    "ZIO_COMPRESS_OFF",     "ZIO_COMPRESS_LZJB",
-                    "ZIO_COMPRESS_EMPTY",   "ZIO_COMPRESS_GZIP_1",
-                    "ZIO_COMPRESS_GZIP_2",  "ZIO_COMPRESS_GZIP_3",
-                    "ZIO_COMPRESS_GZIP_4",  "ZIO_COMPRESS_GZIP_5",
-                    "ZIO_COMPRESS_GZIP_6",  "ZIO_COMPRESS_GZIP_7",
-                    "ZIO_COMPRESS_GZIP_8",  "ZIO_COMPRESS_GZIP_9",
-                    "ZIO_COMPRESS_ZLE",     "ZIO_COMPRESS_LZ4",
-                    "ZIO_COMPRESS_AUTO",    "ZIO_COMPRESS_FUNCTION"]
+    comp_strings = ["ZIO_COMPRESS_INHERIT",  "ZIO_COMPRESS_ON",
+                    "ZIO_COMPRESS_OFF",      "ZIO_COMPRESS_LZJB",
+                    "ZIO_COMPRESS_EMPTY",    "ZIO_COMPRESS_GZIP_1",
+                    "ZIO_COMPRESS_GZIP_2",   "ZIO_COMPRESS_GZIP_3",
+                    "ZIO_COMPRESS_GZIP_4",   "ZIO_COMPRESS_GZIP_5",
+                    "ZIO_COMPRESS_GZIP_6",   "ZIO_COMPRESS_GZIP_7",
+                    "ZIO_COMPRESS_GZIP_8",   "ZIO_COMPRESS_GZIP_9",
+                    "ZIO_COMPRESS_ZLE",      "ZIO_COMPRESS_LZ4",
+                    "ZIO_COMPRESS_ADAPTIVE", "ZIO_COMPRESS_FUNCTION"]
 
     # If "-rr" option is used, don't convert to string representation
     if raw > 1:

include/sys/Makefile.am

@@ -17,6 +17,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/cityhash.h \
 	$(top_srcdir)/include/sys/dataset_kstats.h \
 	$(top_srcdir)/include/sys/compress_auto.h \
+	$(top_srcdir)/include/sys/compress_adaptive.h \
 	$(top_srcdir)/include/sys/dbuf.h \
 	$(top_srcdir)/include/sys/ddt.h \
 	$(top_srcdir)/include/sys/dmu.h \

include/sys/compress_auto.h → include/sys/compress_adaptive.h

@@ -19,20 +19,20 @@
  * CDDL HEADER END
  */
 
-#ifndef _SYS_AUTO_COMPRESS_H
-#define	_SYS_AUTO_COMPRESS_H
+#ifndef _SYS_ADAPTIVE_COMPRESS_H
+#define	_SYS_ADAPTIVE_COMPRESS_H
 
-#define	COMPRESS_AUTO_LEVELS 3
+#define	COMPRESS_ADAPTIVE_LEVELS 10
 
 #include <sys/spa.h>
 #include <sys/zio.h>
 
-size_t compress_auto(zio_t *zio, abd_t *src, void *dst,
+size_t compress_adaptive(zio_t *zio, abd_t *src, void *dst,
 	size_t s_len, enum zio_compress *c);
 
 void compress_calc_avg_without_zero(uint64_t act, uint64_t *res, int n);
 
 uint64_t compress_calc_Bps(uint64_t byte, hrtime_t delay);
 
 
-#endif /* _SYS_AUTO_COMPRESS_H */
+#endif /* _SYS_ADAPTIVE_COMPRESS_H */

include/sys/dsl_dataset.h

@@ -485,6 +485,7 @@ void dsl_dataset_activate_redaction(dsl_dataset_t *ds, uint64_t *redact_snaps,
     uint64_t num_redact_snaps, dmu_tx_t *tx);
 
 int dsl_dataset_activate_compress_auto(const char *ddname);
+int dsl_dataset_activate_compress_adaptive(const char *ddname);
 
 void dsl_dataset_activate_feature(uint64_t dsobj,
     spa_feature_t f, dmu_tx_t *tx);

include/sys/zio.h

@@ -38,7 +38,7 @@
 #include <sys/avl.h>
 #include <sys/fs/zfs.h>
 #include <sys/zio_impl.h>
-#include <sys/compress_auto.h>
+#include <sys/compress_adaptive.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -515,10 +515,11 @@ struct zio {
 	/* Taskq dispatching state */
 	taskq_ent_t	io_tqent;
 
-	uint8_t		io_compress_level; //stored in pio
+	/* stored in pio */
+	uint8_t		io_compress_level;
+	boolean_t	io_compress_adaptive_exploring;
 
-	boolean_t	io_compress_auto_exploring; //stored in pio
-	uint64_t	io_compress_auto_Bps[COMPRESS_AUTO_LEVELS];
+	uint64_t	io_compress_adaptive_Bps[COMPRESS_ADAPTIVE_LEVELS];
 };
 
 extern int zio_bookmark_compare(const void *, const void *);

include/sys/zio_compress.h

@@ -51,7 +51,7 @@ enum zio_compress {
 	ZIO_COMPRESS_GZIP_9,
 	ZIO_COMPRESS_ZLE,
 	ZIO_COMPRESS_LZ4,
-	ZIO_COMPRESS_AUTO,
+	ZIO_COMPRESS_ADAPTIVE,
 	ZIO_COMPRESS_FUNCTIONS
 };
 

include/zfeature_common.h

@@ -75,6 +75,7 @@ typedef enum spa_feature {
 	SPA_FEATURE_LOG_SPACEMAP,
 	SPA_FEATURE_LIVELIST,
 	SPA_FEATURE_COMPRESS_AUTO,
+	SPA_FEATURE_COMPRESS_ADAPTIVE,
 	SPA_FEATURES
 } spa_feature_t;
 

lib/libzpool/Makefile.am

@@ -50,6 +50,7 @@ KERNEL_C = \
 	bqueue.c \
 	cityhash.c \
 	compress_auto.c \
+	compress_adaptive.c \
 	dbuf.c \
 	dbuf_stats.c \
 	ddt.c \

man/man5/zpool-features.5

@@ -258,28 +258,28 @@ returned to the \fBenabled\fR state when all bookmarks with these fields are des
 .sp
 .ne 2
 .na
-\fB\fBcompress_auto\fR\fR
+\fB\fBcompress_adaptive\fR\fR
 .ad
 .RS 4n
 .TS
 l l .
-GUID	org.zfsonlinux:compress_auto
+GUID	org.zfsonlinux:compress_adaptive
 READ\-ONLY COMPATIBLE	yes
 DEPENDENCIES	lz4_compress,extensible_dataset
 .TE
 
-\fBcompress_auto\fR chooses the compression algorithm out lz4 and gzip[1-9] 
-to perform the optimal write throughput.
+\fBcompress_adaptive\fR chooses the compression algorithm out of lz4 and gzip[1-9] 
+to perform with optimal write throughput.
 
-When the \fBcompress_auto\fR feature is set to \fBenabled\fR, the
-administrator can turn on \fBcompress_auto\fR on any dataset on the
+When the \fBcompress_adaptive\fR feature is set to \fBenabled\fR, the
+administrator can turn on \fBcompress_adaptive\fR on any dataset on the
 pool using the \fBzfs\fR(8) command. Please note that doing so will
-immediately activate the \fB\fR feature on the underlying
+immediately activate the \fBcompress_adaptive\fR feature on the underlying
 pool.
 
 This feature becomes \fBactive\fR as soon as it is used on one dataset and will 
 return to being \fBenabled\fR once all filesystems that have ever had their compression set to
-+\fBcompress_auto\fR are destroyed.
++\fBcompress_adaptive\fR are destroyed.
 .RE
 
 .sp
@@ -641,12 +641,16 @@ giving approximately 10% better compression ratio.
 
 When the \fBlz4_compress\fR feature is set to \fBenabled\fR, the
 administrator can turn on \fBlz4\fR compression on any dataset on the
-pool using the zfs(8) command. Please note that doing so will
+pool using the \fBzfs\fR(8) command. Please note that doing so will
 immediately activate the \fBlz4_compress\fR feature on the underlying
-pool using the zfs(8) command. Also, all newly written metadata
+pool using the \fBzfs\fR(1M) command. Also, all newly written metadata
 will be compressed with \fBlz4\fR algorithm. Since this feature is not
 read-only compatible, this operation will render the pool unimportable
-on systems without support for the \fBlz4_compress\fR feature.
+on systems without support for the \fBlz4_compress\fR feature. Booting
+off of \fBlz4\fR-compressed root pools is supported.
+
+This feature becomes \fBactive\fR as soon as it is enabled and will
+never return to being \fBenabled\fR.
 
 Booting off of \fBlz4\fR-compressed root pools is supported.
 

man/man8/zfs.8

@@ -158,9 +158,6 @@ Creates snapshots with the given names.
 .It Xr zfs-rollback 8
 Roll back the given dataset to a previous snapshot.
 .It Xo
-.Sy compression Ns = Ns Sy on Ns | Ns Sy off Ns | Ns Sy auto Ns | Ns Sy gzip Ns | Ns
-.Sy gzip- Ns Em N Ns | Ns Sy lz4 Ns | Ns Sy lzjb Ns | Ns Sy zle
->>>>>>> adaptive compression: inital version of autocompress as implemented by n1kl
 .Xr zfs-hold 8 /
 .Xr zfs-release 8
 .Xc
@@ -200,6 +197,64 @@ Creates a bookmark of the given snapshot.
 Bookmarks mark the point in time when the snapshot was created, and can be used
 as the incremental source for a
 .Nm zfs Cm send
+.Sy compression Ns = Ns Sy on Ns | Ns Sy off Ns | Ns Sy adaptive Ns | Ns Sy gzip Ns | Ns
+.Sy gzip- Ns Em N Ns | Ns Sy lz4 Ns | Ns Sy lzjb Ns | Ns Sy zle
+Controls the compression algorithm used for this dataset.
+.Pp
+Setting compression to
+.Sy on
+indicates that the current default compression algorithm should be used.
+The default balances compression and decompression speed, with compression ratio
+and is expected to work well on a wide variety of workloads.
+Unlike all other settings for this property,
+.Sy on
+does not select a fixed compression type.
+As new compression algorithms are added to ZFS and enabled on a pool, the
+default compression algorithm may change.
+The current default compression algorithm is either
+.Sy lzjb
+or, if the
+.Sy lz4_compress
+feature is enabled,
+.Sy lz4 .
+.Pp
+If compression is set to
+.Sy adaptive
+the optimal compression algorithm out of lz4 and gzip-[1-9] will be choosen based on system load and write performance of the pool's devices, to maintain optimal write speed. To use this compression mode, the
+.Sy compress_adaptive
+feature flag needs to be enabled. See
+.Xr zpool-features 5
+for details on ZFS feature flags and the
+.Sy compress_adaptive
+feature.
+.Pp
+The
+.Sy lz4
+compression algorithm is a high-performance replacement for the
+.Sy lzjb
+algorithm.
+It features significantly faster compression and decompression, as well as a
+moderately higher compression ratio than
+.Sy lzjb ,
+but can only be used on pools with the
+.Sy lz4_compress
+feature set to
+.Sy enabled .
+See
+.Xr zpool-features 5
+for details on ZFS feature flags and the
+.Sy lz4_compress
+feature.
+.Pp
+The
+.Sy lzjb
+compression algorithm is optimized for performance while providing decent data
+compression.
+.Pp
+The
+.Sy gzip
+compression algorithm uses the same compression as the
+.Xr gzip 1
 command.
 .It Xr zfs-redact 8
 Generate a new redaction bookmark.

module/zcommon/zfeature_common.c

@@ -275,16 +275,16 @@ zpool_feature_init(void)
 	    ZFEATURE_FLAG_ACTIVATE_ON_ENABLE, ZFEATURE_TYPE_BOOLEAN, NULL);
 
 	{
-	static const spa_feature_t compress_auto_deps[] = {
+	static const spa_feature_t compress_adaptive_deps[] = {
 			SPA_FEATURE_LZ4_COMPRESS,
 			SPA_FEATURE_EXTENSIBLE_DATASET,
 			SPA_FEATURE_NONE
 	};
-	zfeature_register(SPA_FEATURE_COMPRESS_AUTO,
-	    "org.zfsonlinux:compress_auto", "compress_auto",
-	    "auto compression algorithm support.",
+	zfeature_register(SPA_FEATURE_COMPRESS_ADAPTIVE,
+	    "org.zfsonlinux:compress_adaptive", "compress_adaptive",
+	    "adaptive compression algorithm selection support.",
 	    ZFEATURE_FLAG_PER_DATASET | ZFEATURE_FLAG_READONLY_COMPAT,
-	    compress_auto_deps);
+	    compress_adaptive_deps);
 	}
 
 	zfeature_register(SPA_FEATURE_MULTI_VDEV_CRASH_DUMP,

module/zcommon/zfs_prop.c

@@ -125,7 +125,7 @@ zfs_prop_init(void)
 		{ "gzip-9",	ZIO_COMPRESS_GZIP_9 },
 		{ "zle",	ZIO_COMPRESS_ZLE },
 		{ "lz4",	ZIO_COMPRESS_LZ4 },
-		{ "auto",	ZIO_COMPRESS_AUTO },
+		{ "adaptive",	ZIO_COMPRESS_ADAPTIVE },
 		{ NULL }
 	};
 
@@ -323,7 +323,7 @@ zfs_prop_init(void)
 	zprop_register_index(ZFS_PROP_COMPRESSION, "compression",
 	    ZIO_COMPRESS_DEFAULT, PROP_INHERIT,
 	    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
-	    "on | off | auto | lzjb | gzip | gzip-[1-9] | zle | lz4 ",
+	    "on | off | adaptive | lzjb | gzip | gzip-[1-9] | zle | lz4",
 	    "COMPRESS", compress_table);
 	zprop_register_index(ZFS_PROP_SNAPDIR, "snapdir", ZFS_SNAPDIR_HIDDEN,
 	    PROP_INHERIT, ZFS_TYPE_FILESYSTEM,

module/zfs/Makefile.in

@@ -22,6 +22,7 @@ $(MODULE)-objs += blkptr.o
 $(MODULE)-objs += bplist.o
 $(MODULE)-objs += bpobj.o
 $(MODULE)-objs += compress_auto.o
+$(MODULE)-objs += compress_adaptive.o
 $(MODULE)-objs += bptree.o
 $(MODULE)-objs += btree.o
 $(MODULE)-objs += bqueue.o