Multi ABD Type

Adding the mutli abd type, which allows for
linear and scatter abd's to be chained together
into a single abd.

As part of this PR, the ABD code has been
reorganized so the OS dependant ABD code has
been split into their own independent files.
The shared ABD code is now under:
module/zfs/abd.c
With the independent OS code in:
module/os/linux/zfs/abd_os.c
module/os/freebsd/zfs/abd_os.c

Signed-off-by: Brian Atkinson <batkinson@lanl.gov>
Co-authored-by: Mark Maybee <mmaybee@cray.com>
Co-authored-by: Brian Atkinson <batkinson@lanl.gov>

include/sys/abd.h

@@ -36,19 +36,29 @@ extern "C" {
 #endif
 
 typedef enum abd_flags {
-	ABD_FLAG_LINEAR	= 1 << 0,	/* is buffer linear (or scattered)? */
-	ABD_FLAG_OWNER	= 1 << 1,	/* does it own its data buffers? */
-	ABD_FLAG_META	= 1 << 2,	/* does this represent FS metadata? */
-	ABD_FLAG_MULTI_ZONE  = 1 << 3,	/* pages split over memory zones */
-	ABD_FLAG_MULTI_CHUNK = 1 << 4,	/* pages split over multiple chunks */
-	ABD_FLAG_LINEAR_PAGE = 1 << 5,	/* linear but allocd from page */
+	ABD_FLAG_LINEAR		= 1 << 0, /* is buffer linear (or scattered)? */
+	ABD_FLAG_OWNER		= 1 << 1, /* does it own its data buffers? */
+	ABD_FLAG_META		= 1 << 2, /* does this represent FS metadata? */
+	ABD_FLAG_MULTI_ZONE  	= 1 << 3, /* pages split over memory zones */
+	ABD_FLAG_MULTI_CHUNK 	= 1 << 4, /* pages split over multiple chunks */
+	ABD_FLAG_LINEAR_PAGE 	= 1 << 5, /* linear but allocd from page */
+	ABD_FLAG_MULTI_LIST	= 1 << 6, /* mult ABDs chained together */
+	ABD_FLAG_MULTI_FREE	= 1 << 7, /* mult ABD is responsible for mem */
+	ABD_FLAG_ZEROS		= 1 << 8, /* ABD for zero-filled buffer */
 } abd_flags_t;
 
+typedef enum abd_stats_op {
+	ABDSTAT_INCR, /* Increase abdstat values */
+	ABDSTAT_DECR  /* Decrease abdstat values */
+} abd_stats_op_t;
+
 typedef struct abd {
 	abd_flags_t	abd_flags;
 	uint_t		abd_size;	/* excludes scattered abd_offset */
+	list_node_t	abd_multi_link;
 	struct abd	*abd_parent;
 	zfs_refcount_t	abd_children;
+	kmutex_t	abd_mtx;
 	union {
 		struct abd_scatter {
 			uint_t		abd_offset;
@@ -64,20 +74,51 @@ typedef struct abd {
 			void		*abd_buf;
 			struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
 		} abd_linear;
+		struct abd_multi {
+			list_t abd_chain;
+		} abd_multi;
 	} abd_u;
 } abd_t;
 
+#if defined(_KERNEL)
+unsigned int abd_bio_map_off(struct bio *, abd_t *, unsigned int, size_t);
+unsigned long abd_nr_pages_off(abd_t *, unsigned int, size_t);
+#endif
+
+struct scatterlist; /* forward declartion */
+
+struct abd_iter {
+	/* public interface */
+	void		*iter_mapaddr;	/* addr corresponding to iter_pos */
+	size_t		iter_mapsize;	/* length of data valid at mapaddr */
+
+	/* private */
+	abd_t		*iter_abd;	/* ABD being iterated through */
+	size_t		iter_pos;
+	size_t		iter_offset;	/* offset in current sg/abd_buf, */
+					/* abd_offset included */
+	struct scatterlist *iter_sg;	/* current sg */
+};
+
 typedef int abd_iter_func_t(void *buf, size_t len, void *private);
 typedef int abd_iter_func2_t(void *bufa, void *bufb, size_t len, void *private);
 
 extern int zfs_abd_scatter_enabled;
+extern abd_t *abd_zero_scatter;
 
 static inline boolean_t
 abd_is_linear(abd_t *abd)
 {
 	return ((abd->abd_flags & ABD_FLAG_LINEAR) != 0 ? B_TRUE : B_FALSE);
 }
 
+static inline boolean_t
+abd_is_multi(abd_t *abd)
+{
+	return ((abd->abd_flags & ABD_FLAG_MULTI_LIST) != 0 ? B_TRUE :
+	    B_FALSE);
+}
+
 static inline boolean_t
 abd_is_linear_page(abd_t *abd)
 {
@@ -91,13 +132,16 @@ abd_is_linear_page(abd_t *abd)
 
 abd_t *abd_alloc(size_t, boolean_t);
 abd_t *abd_alloc_linear(size_t, boolean_t);
+abd_t *abd_alloc_multi(void);
 abd_t *abd_alloc_for_io(size_t, boolean_t);
 abd_t *abd_alloc_sametype(abd_t *, size_t);
+void abd_add_child(abd_t *, abd_t *, boolean_t);
+abd_t *abd_find_child_off(abd_t *, size_t *);
 void abd_free(abd_t *);
 abd_t *abd_get_offset(abd_t *, size_t);
 abd_t *abd_get_offset_size(abd_t *, size_t, size_t);
+abd_t *abd_get_zeros(size_t);
 abd_t *abd_get_from_buf(void *, size_t);
-void abd_put(abd_t *);
 
 /*
  * Conversion to and from a normal buffer
@@ -124,12 +168,7 @@ void abd_copy_to_buf_off(void *, abd_t *, size_t, size_t);
 int abd_cmp(abd_t *, abd_t *);
 int abd_cmp_buf_off(abd_t *, const void *, size_t, size_t);
 void abd_zero_off(abd_t *, size_t, size_t);
-
-#if defined(_KERNEL)
-unsigned int abd_scatter_bio_map_off(struct bio *, abd_t *, unsigned int,
-		size_t);
-unsigned long abd_nr_pages_off(abd_t *, unsigned int, size_t);
-#endif
+void abd_verify(abd_t *);
 
 void abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
 	ssize_t csize, ssize_t dsize, const unsigned parity,
@@ -174,13 +213,51 @@ abd_zero(abd_t *abd, size_t size)
 	abd_zero_off(abd, 0, size);
 }
 
+/*
+ * OS specific functions
+ */
+
+void abd_put(abd_t *);
+abd_t *abd_alloc_struct(size_t size);
+void abd_free_struct(abd_t *abd);
+void abd_alloc_pages(abd_t *abd, size_t size);
+void abd_free_pages(abd_t *abd);
+boolean_t abd_size_alloc_linear(size_t size);
+void abd_update_scatter_stats(abd_t *abd, abd_stats_op_t op);
+void abd_update_linear_stats(abd_t *abd, abd_stats_op_t op);
+void abd_verify_scatter(abd_t *abd);
+void abd_free_linear_page(abd_t *abd);
+abd_t *abd_get_offset_impl(abd_t *sabd, size_t off, size_t size);
+void abd_enter_critical(unsigned long flags);
+void abd_exit_critical(unsigned long flags);
+/* OS specific abd_iter functions */
+void abd_iter_init(struct abd_iter  *aiter, abd_t *abd);
+boolean_t abd_iter_at_end(struct abd_iter *aiter);
+void abd_iter_advance(struct abd_iter *aiter, size_t amount);
+void abd_iter_map(struct abd_iter *aiter);
+void abd_iter_unmap(struct abd_iter *aiter);
+
 /*
  * Module lifecycle
+ * Defined in each specific OS's abd.c
  */
 
 void abd_init(void);
 void abd_fini(void);
 
+/*
+ * Helper macros
+ */
+#define	ABDSTAT(stat)		(abd_stats.stat.value.ui64)
+#define	ABDSTAT_INCR(stat, val) \
+	atomic_add_64(&abd_stats.stat.value.ui64, (val))
+#define	ABDSTAT_BUMP(stat)	ABDSTAT_INCR(stat, 1)
+#define	ABDSTAT_BUMPDOWN(stat)	ABDSTAT_INCR(stat, -1)
+
+#define	ABD_SCATTER(abd)	(abd->abd_u.abd_scatter)
+#define	ABD_LINEAR_BUF(abd)	(abd->abd_u.abd_linear.abd_buf)
+#define	ABD_MULTI(abd)		(abd->abd_u.abd_multi)
+
 #ifdef __cplusplus
 }
 #endif

lib/libzpool/Makefile.am

@@ -39,6 +39,7 @@ KERNEL_C = \
 	zpool_prop.c \
 	zprop_common.c \
 	abd.c \
+	abd_os.c \
 	aggsum.c \
 	arc.c \
 	arc_os.c \

module/Makefile.bsd

@@ -127,7 +127,7 @@ SRCS+= spl_atomic.c
 .endif
 
 #os/freebsd/zfs
-SRCS+=	abd.c \
+SRCS+=	abd_os.c \
 	crypto_os.c \
 	dmu_os.c \
 	hkdf.c \
@@ -169,7 +169,8 @@ SRCS+=	zfeature_common.c \
 	zprop_common.c
 
 #zfs
-SRCS+=	aggsum.c \
+SRCS+=	abd.c \
+	aggsum.c \
 	arc.c \
 	arc_os.c \
 	blkptr.c \