autocompress

cmd/dbufstat/dbufstat.py

@@ -303,15 +303,16 @@ def get_typestring(t):
 
 
 def get_compstring(c):
-    comp_strings = ["ZIO_COMPRESS_INHERIT", "ZIO_COMPRESS_ON",
-                    "ZIO_COMPRESS_OFF",     "ZIO_COMPRESS_LZJB",
-                    "ZIO_COMPRESS_EMPTY",   "ZIO_COMPRESS_GZIP_1",
-                    "ZIO_COMPRESS_GZIP_2",  "ZIO_COMPRESS_GZIP_3",
-                    "ZIO_COMPRESS_GZIP_4",  "ZIO_COMPRESS_GZIP_5",
-                    "ZIO_COMPRESS_GZIP_6",  "ZIO_COMPRESS_GZIP_7",
-                    "ZIO_COMPRESS_GZIP_8",  "ZIO_COMPRESS_GZIP_9",
-                    "ZIO_COMPRESS_ZLE",     "ZIO_COMPRESS_LZ4",
-                    "ZIO_COMPRESS_FUNCTION"]
+    comp_strings = ["ZIO_COMPRESS_INHERIT",  "ZIO_COMPRESS_ON",
+                    "ZIO_COMPRESS_OFF",      "ZIO_COMPRESS_LZJB",
+                    "ZIO_COMPRESS_EMPTY",    "ZIO_COMPRESS_GZIP_1",
+                    "ZIO_COMPRESS_GZIP_2",   "ZIO_COMPRESS_GZIP_3",
+                    "ZIO_COMPRESS_GZIP_4",   "ZIO_COMPRESS_GZIP_5",
+                    "ZIO_COMPRESS_GZIP_6",   "ZIO_COMPRESS_GZIP_7",
+                    "ZIO_COMPRESS_GZIP_8",   "ZIO_COMPRESS_GZIP_9",
+                    "ZIO_COMPRESS_ZLE",      "ZIO_COMPRESS_LZ4",
+                    "ZIO_COMPRESS_FUNCTION", "ZIO_COMPRESS_AUTO",
+                    "ZIO_COMPRESS_META_FUNCTIONS"]
 
     # If "-rr" option is used, don't convert to string representation
     if raw > 1:

cmd/zdb/zdb.c

@@ -66,7 +66,7 @@
 #include <zfs_comutil.h>
 #include <libzfs.h>
 
-#define	ZDB_COMPRESS_NAME(idx) ((idx) < ZIO_COMPRESS_FUNCTIONS ?	\
+#define	ZDB_COMPRESS_NAME(idx) ((idx) < ZIO_COMPRESS_META_FUNCTIONS ?	\
 	zio_compress_table[(idx)].ci_name : "UNKNOWN")
 #define	ZDB_CHECKSUM_NAME(idx) ((idx) < ZIO_CHECKSUM_FUNCTIONS ?	\
 	zio_checksum_table[(idx)].ci_name : "UNKNOWN")

include/sys/Makefile.am

@@ -11,6 +11,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/bpobj.h \
 	$(top_srcdir)/include/sys/bptree.h \
 	$(top_srcdir)/include/sys/bqueue.h \
+	$(top_srcdir)/include/sys/compress_auto.h \
 	$(top_srcdir)/include/sys/dbuf.h \
 	$(top_srcdir)/include/sys/ddt.h \
 	$(top_srcdir)/include/sys/dmu.h \

include/sys/compress_auto.h

@@ -0,0 +1,34 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+#ifndef _SYS_AUTO_COMPRESS_H
+#define	_SYS_AUTO_COMPRESS_H
+
+#define	COMPRESS_AUTO_LEVELS 10
+
+#include <sys/spa.h>
+#include <sys/zio.h>
+
+size_t compress_auto(zio_t *zio, enum zio_compress *c, abd_t *src, void *dst,
+	size_t s_len);
+void compress_auto_calc_avg_nozero(uint64_t act, uint64_t *res, int n);
+
+#endif /* _SYS_AUTO_COMPRESS_H */

include/sys/dsl_dataset.h

@@ -342,6 +342,8 @@ boolean_t dsl_dataset_is_zapified(dsl_dataset_t *ds);
 boolean_t dsl_dataset_has_resume_receive_state(dsl_dataset_t *ds);
 int dsl_dataset_rollback(const char *fsname, void *owner, nvlist_t *result);
 
+int dsl_dataset_activate_compress_auto(const char *ddname);
+
 void dsl_dataset_deactivate_feature(uint64_t dsobj,
     spa_feature_t f, dmu_tx_t *tx);
 

include/sys/fs/zfs.h

@@ -895,6 +895,9 @@ typedef struct vdev_stat_ex {
 	uint64_t vsx_agg_histo[ZIO_PRIORITY_NUM_QUEUEABLE]
 	    [VDEV_RQ_HISTO_BUCKETS];
 
+
+	uint64_t vsx_diskBps[ZIO_TYPES];
+
 } vdev_stat_ex_t;
 
 /*

include/sys/vdev_impl.h

@@ -108,6 +108,7 @@ struct vdev_cache {
 
 typedef struct vdev_queue_class {
 	uint32_t	vqc_active;
+	uint64_t	vqc_queued_size;
 
 	/*
 	 * Sorted by offset or timestamp, depending on if the queue is

include/sys/zio.h

@@ -38,6 +38,7 @@
 #include <sys/avl.h>
 #include <sys/fs/zfs.h>
 #include <sys/zio_impl.h>
+#include <sys/compress_auto.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -464,6 +465,13 @@ struct zio {
 
 	/* Taskq dispatching state */
 	taskq_ent_t	io_tqent;
+
+	uint8_t		io_compress_level;
+	zio_t		*io_temp_parent;
+
+	boolean_t	io_compress_auto_exploring;
+	uint64_t	io_compress_auto_Bps[COMPRESS_AUTO_LEVELS];
+	hrtime_t	io_compress_auto_delay;
 };
 
 extern int zio_bookmark_compare(const void *, const void *);

include/sys/zio_compress.h

@@ -51,7 +51,9 @@ enum zio_compress {
 	ZIO_COMPRESS_GZIP_9,
 	ZIO_COMPRESS_ZLE,
 	ZIO_COMPRESS_LZ4,
-	ZIO_COMPRESS_FUNCTIONS
+	ZIO_COMPRESS_FUNCTIONS,
+	ZIO_COMPRESS_AUTO,
+	ZIO_COMPRESS_META_FUNCTIONS
 };
 
 /* Common signature for all zio compress functions. */
@@ -78,7 +80,7 @@ typedef const struct zio_compress_info {
 	zio_decompress_func_t		*ci_decompress;
 } zio_compress_info_t;
 
-extern zio_compress_info_t zio_compress_table[ZIO_COMPRESS_FUNCTIONS];
+extern zio_compress_info_t zio_compress_table[ZIO_COMPRESS_META_FUNCTIONS];
 
 /*
  * lz4 compression init & free

include/zfeature_common.h

@@ -57,6 +57,7 @@ typedef enum spa_feature {
 	SPA_FEATURE_SKEIN,
 	SPA_FEATURE_EDONR,
 	SPA_FEATURE_USEROBJ_ACCOUNTING,
+	SPA_FEATURE_COMPRESS_AUTO,
 	SPA_FEATURES
 } spa_feature_t;
 

lib/libzpool/Makefile.am

@@ -40,6 +40,7 @@ KERNEL_C = \
 	bpobj.c \
 	bptree.c \
 	bqueue.c \
+	compress_auto.c \
 	dbuf.c \
 	dbuf_stats.c \
 	ddt.c \

man/man5/zpool-features.5

@@ -255,6 +255,33 @@ This feature becomes \fBactive\fR as soon as it is enabled and will
 never return to being \fBenabled\fB.
 .RE
 
+.sp
+.ne 2
+.na
+\fB\fBcompress_auto\fR\fR
+.ad
+.RS 4n
+.TS
+l l .
+GUID	org.zfsonlinux:compress_auto
+READ\-ONLY COMPATIBLE	yes
+DEPENDENCIES	lz4_compress,extensible_dataset
+.TE
+
+\fBcompress_auto\fR chooses the compression algorithm out lz4 and gzip[1-9] 
+to perform the optimal write throughput.
+
+When the \fBcompress_auto\fR feature is set to \fBenabled\fR, the
+administrator can turn on \fBcompress_auto\fR on any dataset on the
+pool using the \fBzfs\fR(8) command. Please note that doing so will
+immediately activate the \fB\fR feature on the underlying
+pool.
+
+This feature becomes \fBactive\fR as soon as it is used on one dataset and will 
+return to being \fBenabled\fR once all filesystems that have ever had their compression set to
++\fBcompress_auto\fR are destroyed.
+.RE
+
 .sp
 .ne 2
 .na

man/man8/zfs.8

@@ -901,7 +901,7 @@ Changing this property affects only newly-written data.
 .sp
 .ne 2
 .na
-\fB\fBcompression\fR=\fBoff\fR | \fBon\fR | \fBlzjb\fR | \fBlz4\fR |
+\fB\fBcompression\fR=\fBoff\fR | \fBon\fR | \fBauto\fR | \fBlzjb\fR | \fBlz4\fR |
 \fBgzip\fR | \fBgzip-\fR\fIN\fR | \fBzle\fR\fR
 .ad
 .sp .6

module/zcommon/zfs_prop.c

@@ -116,6 +116,7 @@ zfs_prop_init(void)
 		{ "gzip-9",	ZIO_COMPRESS_GZIP_9 },
 		{ "zle",	ZIO_COMPRESS_ZLE },
 		{ "lz4",	ZIO_COMPRESS_LZ4 },
+		{ "auto",	ZIO_COMPRESS_AUTO },
 		{ NULL }
 	};
 
@@ -267,8 +268,8 @@ zfs_prop_init(void)
 	zprop_register_index(ZFS_PROP_COMPRESSION, "compression",
 	    ZIO_COMPRESS_DEFAULT, PROP_INHERIT,
 	    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
-	    "on | off | lzjb | gzip | gzip-[1-9] | zle | lz4", "COMPRESS",
-	    compress_table);
+	    "on | off | auto | lzjb | gzip | gzip-[1-9] | zle | lz4 ",
+	    "COMPRESS", compress_table);
 	zprop_register_index(ZFS_PROP_SNAPDIR, "snapdir", ZFS_SNAPDIR_HIDDEN,
 	    PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
 	    "hidden | visible", "SNAPDIR", snapdir_table);

module/zfs/Makefile.in

@@ -12,6 +12,7 @@ $(MODULE)-objs += arc.o
 $(MODULE)-objs += blkptr.o
 $(MODULE)-objs += bplist.o
 $(MODULE)-objs += bpobj.o
+$(MODULE)-objs += compress_auto.o
 $(MODULE)-objs += dbuf.o
 $(MODULE)-objs += dbuf_stats.o
 $(MODULE)-objs += bptree.o