autocompress

cmd/dbufstat/dbufstat.py

@@ -336,7 +336,7 @@ def get_compstring(c):
                     "ZIO_COMPRESS_GZIP_6",  "ZIO_COMPRESS_GZIP_7",
                     "ZIO_COMPRESS_GZIP_8",  "ZIO_COMPRESS_GZIP_9",
                     "ZIO_COMPRESS_ZLE",     "ZIO_COMPRESS_LZ4",
-                    "ZIO_COMPRESS_FUNCTION"]
+                    "ZIO_COMPRESS_AUTO",    "ZIO_COMPRESS_FUNCTION"]
 
     # If "-rr" option is used, don't convert to string representation
     if raw > 1:

include/sys/Makefile.am

@@ -11,6 +11,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/bpobj.h \
 	$(top_srcdir)/include/sys/bptree.h \
 	$(top_srcdir)/include/sys/bqueue.h \
+	$(top_srcdir)/include/sys/compress_auto.h \
 	$(top_srcdir)/include/sys/dbuf.h \
 	$(top_srcdir)/include/sys/ddt.h \
 	$(top_srcdir)/include/sys/dmu.h \

include/sys/compress_auto.h

@@ -0,0 +1,38 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+#ifndef _SYS_AUTO_COMPRESS_H
+#define	_SYS_AUTO_COMPRESS_H
+
+#define	COMPRESS_AUTO_LEVELS 3
+
+#include <sys/spa.h>
+#include <sys/zio.h>
+
+size_t compress_auto(zio_t *zio, abd_t *src, void *dst,
+	size_t s_len, enum zio_compress *c);
+
+void compress_calc_avg_without_zero(uint64_t act, uint64_t *res, int n);
+
+uint64_t compress_calc_Bps(uint64_t byte, hrtime_t delay);
+
+
+#endif /* _SYS_AUTO_COMPRESS_H */

include/sys/dsl_dataset.h

@@ -358,6 +358,8 @@ boolean_t dsl_dataset_has_resume_receive_state(dsl_dataset_t *ds);
 int dsl_dataset_rollback(const char *fsname, const char *tosnap, void *owner,
     nvlist_t *result);
 
+int dsl_dataset_activate_compress_auto(const char *ddname);
+
 void dsl_dataset_activate_feature(uint64_t dsobj,
     spa_feature_t f, dmu_tx_t *tx);
 void dsl_dataset_deactivate_feature(uint64_t dsobj,

include/sys/fs/zfs.h

@@ -968,6 +968,9 @@ typedef struct vdev_stat_ex {
 	uint64_t vsx_agg_histo[ZIO_PRIORITY_NUM_QUEUEABLE]
 	    [VDEV_RQ_HISTO_BUCKETS];
 
+
+	uint64_t vsx_diskBps[ZIO_TYPES];
+
 } vdev_stat_ex_t;
 
 /*

include/sys/vdev_impl.h

@@ -108,6 +108,7 @@ struct vdev_cache {
 
 typedef struct vdev_queue_class {
 	uint32_t	vqc_active;
+	uint64_t	vqc_queued_size;
 
 	/*
 	 * Sorted by offset or timestamp, depending on if the queue is

include/sys/zio.h

@@ -38,6 +38,7 @@
 #include <sys/avl.h>
 #include <sys/fs/zfs.h>
 #include <sys/zio_impl.h>
+#include <sys/compress_auto.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -494,6 +495,11 @@ struct zio {
 
 	/* Taskq dispatching state */
 	taskq_ent_t	io_tqent;
+
+	uint8_t		io_compress_level; //stored in pio
+
+	boolean_t	io_compress_auto_exploring; //stored in pio
+	uint64_t	io_compress_auto_Bps[COMPRESS_AUTO_LEVELS];
 };
 
 extern int zio_bookmark_compare(const void *, const void *);

include/sys/zio_compress.h

@@ -51,6 +51,7 @@ enum zio_compress {
 	ZIO_COMPRESS_GZIP_9,
 	ZIO_COMPRESS_ZLE,
 	ZIO_COMPRESS_LZ4,
+	ZIO_COMPRESS_AUTO,
 	ZIO_COMPRESS_FUNCTIONS
 };
 

include/zfeature_common.h

@@ -58,6 +58,7 @@ typedef enum spa_feature {
 	SPA_FEATURE_EDONR,
 	SPA_FEATURE_USEROBJ_ACCOUNTING,
 	SPA_FEATURE_ENCRYPTION,
+	SPA_FEATURE_COMPRESS_AUTO,
 	SPA_FEATURES
 } spa_feature_t;
 

lib/libzpool/Makefile.am

@@ -41,6 +41,7 @@ KERNEL_C = \
 	bpobj.c \
 	bptree.c \
 	bqueue.c \
+	compress_auto.c \
 	dbuf.c \
 	dbuf_stats.c \
 	ddt.c \

man/man5/zpool-features.5

@@ -255,6 +255,33 @@ This feature becomes \fBactive\fR as soon as it is enabled and will
 never return to being \fBenabled\fB.
 .RE
 
+.sp
+.ne 2
+.na
+\fB\fBcompress_auto\fR\fR
+.ad
+.RS 4n
+.TS
+l l .
+GUID	org.zfsonlinux:compress_auto
+READ\-ONLY COMPATIBLE	yes
+DEPENDENCIES	lz4_compress,extensible_dataset
+.TE
+
+\fBcompress_auto\fR chooses the compression algorithm out lz4 and gzip[1-9] 
+to perform the optimal write throughput.
+
+When the \fBcompress_auto\fR feature is set to \fBenabled\fR, the
+administrator can turn on \fBcompress_auto\fR on any dataset on the
+pool using the \fBzfs\fR(8) command. Please note that doing so will
+immediately activate the \fB\fR feature on the underlying
+pool.
+
+This feature becomes \fBactive\fR as soon as it is used on one dataset and will 
+return to being \fBenabled\fR once all filesystems that have ever had their compression set to
++\fBcompress_auto\fR are destroyed.
+.RE
+
 .sp
 .ne 2
 .na

man/man8/zfs.8

@@ -1091,7 +1091,7 @@ for more information on these algorithms.
 .Pp
 Changing this property affects only newly-written data.
 .It Xo
-.Sy compression Ns = Ns Sy on Ns | Ns Sy off Ns | Ns Sy gzip Ns | Ns
+.Sy compression Ns = Ns Sy on Ns | Ns Sy off Ns | Ns Sy auto Ns | Ns Sy gzip Ns | Ns
 .Sy gzip- Ns Em N Ns | Ns Sy lz4 Ns | Ns Sy lzjb Ns | Ns Sy zle
 .Xc
 Controls the compression algorithm used for this dataset.

module/zcommon/zfeature_common.c

@@ -193,6 +193,19 @@ zpool_feature_init(void)
 	    "LZ4 compression algorithm support.",
 	    ZFEATURE_FLAG_ACTIVATE_ON_ENABLE, NULL);
 
+	{
+	static const spa_feature_t compress_auto_deps[] = {
+			SPA_FEATURE_LZ4_COMPRESS,
+			SPA_FEATURE_EXTENSIBLE_DATASET,
+			SPA_FEATURE_NONE
+	};
+	zfeature_register(SPA_FEATURE_COMPRESS_AUTO,
+	    "org.zfsonlinux:compress_auto", "compress_auto",
+	    "auto compression algorithm support.",
+	    ZFEATURE_FLAG_PER_DATASET | ZFEATURE_FLAG_READONLY_COMPAT,
+	    compress_auto_deps);
+	}
+
 	zfeature_register(SPA_FEATURE_MULTI_VDEV_CRASH_DUMP,
 	    "com.joyent:multi_vdev_crash_dump", "multi_vdev_crash_dump",
 	    "Crash dumps to multiple vdev pools.",

module/zcommon/zfs_prop.c

@@ -117,6 +117,7 @@ zfs_prop_init(void)
 		{ "gzip-9",	ZIO_COMPRESS_GZIP_9 },
 		{ "zle",	ZIO_COMPRESS_ZLE },
 		{ "lz4",	ZIO_COMPRESS_LZ4 },
+		{ "auto",	ZIO_COMPRESS_AUTO },
 		{ NULL }
 	};
 
@@ -304,8 +305,8 @@ zfs_prop_init(void)
 	zprop_register_index(ZFS_PROP_COMPRESSION, "compression",
 	    ZIO_COMPRESS_DEFAULT, PROP_INHERIT,
 	    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
-	    "on | off | lzjb | gzip | gzip-[1-9] | zle | lz4", "COMPRESS",
-	    compress_table);
+	    "on | off | auto | lzjb | gzip | gzip-[1-9] | zle | lz4 ",
+	    "COMPRESS", compress_table);
 	zprop_register_index(ZFS_PROP_SNAPDIR, "snapdir", ZFS_SNAPDIR_HIDDEN,
 	    PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
 	    "hidden | visible", "SNAPDIR", snapdir_table);

module/zfs/Makefile.in

@@ -12,6 +12,7 @@ $(MODULE)-objs += arc.o
 $(MODULE)-objs += blkptr.o
 $(MODULE)-objs += bplist.o
 $(MODULE)-objs += bpobj.o
+$(MODULE)-objs += compress_auto.o
 $(MODULE)-objs += dbuf.o
 $(MODULE)-objs += dbuf_stats.o
 $(MODULE)-objs += bptree.o