Add adaptive compression

- This Commit rebases and squashes openzfs#7560 - Adds tests for Adaptive compression Co-authored-by: n1kl (bunge) <n1kl@users.noreply.github.com> Co-authored-by: Ruben Kelevra <RubenKelevra@users.noreply.github.com> Co-authored-by: Andy Lavr <andy.lavr@gmail.com> Co-authored-by: Sebastian Gottschall <s.gottschall@dd-wrt.com> Co-authored-by: Kjeld Schouten-Lebbing <kjeld@schouten-lebbing.nl> Signed-off-by: n1kl (bunge) <n1kl@users.noreply.github.com> Signed-off-by: RubenKelevra <ruben@vfn-nrw.de> Signed-off-by: Andy Lavr <andy.lavr@gmail.com> Signed-off-by: Sebastian Gottschall <s.gottschall@dd-wrt.com> Signed-off-by: Kjeld Schouten-Lebbing <kjeld@schouten-lebbing.nl>
PrivatePuffin · Oct 7, 2020 · 33128cb · 33128cb
1 parent 266d112
commit 33128cb
Show file tree

Hide file tree

Showing 25 changed files with 4,910 additions and 288 deletions.
diff --git a/cmd/dbufstat/dbufstat.in b/cmd/dbufstat/dbufstat.in
@@ -343,7 +343,8 @@ def get_compstring(c):
                     "ZIO_COMPRESS_GZIP_6",  "ZIO_COMPRESS_GZIP_7",
                     "ZIO_COMPRESS_GZIP_8",  "ZIO_COMPRESS_GZIP_9",
                     "ZIO_COMPRESS_ZLE",     "ZIO_COMPRESS_LZ4",
-                    "ZIO_COMPRESS_ZSTD",    "ZIO_COMPRESS_FUNCTION"]
+                    "ZIO_COMPRESS_ZSTD",    "ZIO_COMPRESS_ADAPTIVE",
+                    "ZIO_COMPRESS_FUNCTION"]
 
     # If "-rr" option is used, don't convert to string representation
     if raw > 1:

diff --git a/include/sys/Makefile.am b/include/sys/Makefile.am
@@ -15,6 +15,7 @@ COMMON_H = \
 	bptree.h \
 	btree.h \
 	bqueue.h \
+	compress_adaptive.h \
 	dataset_kstats.h \
 	dbuf.h \
 	ddt.h \

diff --git a/include/sys/compress_adaptive.h b/include/sys/compress_adaptive.h
@@ -0,0 +1,38 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+#ifndef _SYS_ADAPTIVE_COMPRESS_H
+#define	_SYS_ADAPTIVE_COMPRESS_H
+
+#define	COMPRESS_ADAPTIVE_LEVELS 10
+
+#include <sys/spa.h>
+#include <sys/zio.h>
+
+size_t compress_adaptive(zio_t *zio, abd_t *src, void *dst,
+	size_t s_len, enum zio_compress *c, uint8_t level);
+
+void compress_calc_avg_without_zero(uint64_t act, uint64_t *res, int n);
+
+uint64_t compress_calc_Bps(uint64_t byte, hrtime_t delay);
+
+
+#endif /* _SYS_ADAPTIVE_COMPRESS_H */
diff --git a/include/sys/dsl_dataset.h b/include/sys/dsl_dataset.h
@@ -488,6 +488,8 @@ boolean_t dsl_dataset_get_uint64_array_feature(dsl_dataset_t *ds,
 void dsl_dataset_activate_redaction(dsl_dataset_t *ds, uint64_t *redact_snaps,
     uint64_t num_redact_snaps, dmu_tx_t *tx);
 
+int dsl_dataset_activate_compress_adaptive( const char *ddname);
+
 #ifdef ZFS_DEBUG
 #define	dprintf_ds(ds, fmt, ...) do { \
 	if (zfs_flags & ZFS_DEBUG_DPRINTF) { \

diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h
@@ -1130,6 +1130,9 @@ typedef struct vdev_stat_ex {
 	uint64_t vsx_agg_histo[ZIO_PRIORITY_NUM_QUEUEABLE]
 	    [VDEV_RQ_HISTO_BUCKETS];
 
+
+	uint64_t vsx_diskBps[ZIO_TYPES];
+
 } vdev_stat_ex_t;
 
 /*

diff --git a/include/sys/vdev_impl.h b/include/sys/vdev_impl.h
@@ -132,6 +132,7 @@ struct vdev_cache {
 
 typedef struct vdev_queue_class {
 	uint32_t	vqc_active;
+	uint64_t	vqc_queued_size;
 
 	/*
 	 * Sorted by offset or timestamp, depending on if the queue is

diff --git a/include/sys/zio.h b/include/sys/zio.h
@@ -41,6 +41,7 @@
 #include <sys/avl.h>
 #include <sys/fs/zfs.h>
 #include <sys/zio_impl.h>
+#include <sys/compress_adaptive.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -527,6 +528,12 @@ struct zio {
 
 	/* Taskq dispatching state */
 	taskq_ent_t	io_tqent;
+
+	/* stored in pio */
+	uint8_t		io_compress_level;
+	boolean_t	io_compress_adaptive_exploring;
+
+	uint64_t	io_compress_adaptive_Bps[COMPRESS_ADAPTIVE_LEVELS];
 };
 
 enum blk_verify_flag {

diff --git a/include/sys/zio_compress.h b/include/sys/zio_compress.h
@@ -54,6 +54,7 @@ enum zio_compress {
 	ZIO_COMPRESS_ZLE,
 	ZIO_COMPRESS_LZ4,
 	ZIO_COMPRESS_ZSTD,
+	ZIO_COMPRESS_ADAPTIVE,
 	ZIO_COMPRESS_FUNCTIONS
 };
 

diff --git a/include/zfeature_common.h b/include/zfeature_common.h
@@ -76,6 +76,7 @@ typedef enum spa_feature {
 	SPA_FEATURE_LIVELIST,
 	SPA_FEATURE_DEVICE_REBUILD,
 	SPA_FEATURE_ZSTD_COMPRESS,
+	SPA_FEATURE_COMPRESS_ADAPTIVE,
 	SPA_FEATURES
 } spa_feature_t;
 

diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am
@@ -62,6 +62,7 @@ KERNEL_C = \
 	btree.c \
 	bqueue.c \
 	cityhash.c \
+	compress_adaptive.c \
 	dbuf.c \
 	dbuf_stats.c \
 	ddt.c \

diff --git a/man/man5/zpool-features.5 b/man/man5/zpool-features.5
@@ -282,8 +282,38 @@ healing resilver when the time to verify the checksums is included.  However,
 unless there is additional pool damage no checksum errors should be reported
 by the scrub.  This feature is incompatible with raidz configurations.
 
+
 This feature becomes \fBactive\fR while a sequential resilver is in progress,
 and returns to \fBenabled\fR when the resilver completes.
+
+.RE
+
+.sp
+.ne 2
+.na
+\fB\fBcompress_adaptive\fR\fR
+.ad
+.RS 4n
+.TS
+l l .
+GUID	org.zfsonlinux:compress_adaptive
+READ\-ONLY COMPATIBLE	yes
+DEPENDENCIES	lz4_compress,extensible_dataset
+.TE
+
+\fBcompress_adaptive\fR chooses the compression algorithm out of lz4 and gzip[1-9] 
+to perform with optimal write throughput.
+
+When the \fBcompress_adaptive\fR feature is set to \fBenabled\fR, the
+administrator can turn on \fBcompress_adaptive\fR on any dataset on the
+pool using the \fBzfs\fR(8) command. Please note that doing so will
+immediately activate the \fBcompress_adaptive\fR feature on the underlying
+pool.
+
+This feature becomes \fBactive\fR as soon as it is used on one dataset and will 
+return to being \fBenabled\fR once all filesystems that have ever had their compression set to
++\fBcompress_adaptive\fR are destroyed.
+
 .RE
 
 .sp
@@ -298,7 +328,6 @@ GUID	com.delphix:device_removal
 READ\-ONLY COMPATIBLE	no
 DEPENDENCIES	none
 .TE
-
 This feature enables the \fBzpool remove\fR subcommand to remove top-level
 vdevs, evacuating them to reduce the total size of the pool.