Revert "SA spill block cache"

The SA spill_cache was originally introduced to avoid the need to
perform large kmem or vmem allocations.  Instead a small dedicated
cache of preallocated SA buffers was kept.

This solution was viable while the maximum block size was limited
to 128K.  But with the planned increase of the maximum block size
to 16M callers need to migrate to the zio_buf_alloc().  However,
they should be aware this interface is expected to change again
once the zio buffers are fully backed by scatter-gather lists.

Alternately, if the callers know these buffers will never be large
or be infrequently accessed they may kmem_alloc() or vmem_alloc()
the needed temporary space.

This change has the additional benegit of bringing the code back
inline with the upstream Illumos source.

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>

include/sys/sa.h

@@ -150,8 +150,6 @@ int sa_replace_all_by_template_locked(sa_handle_t *, sa_bulk_attr_t *,
 boolean_t sa_enabled(objset_t *);
 void sa_cache_init(void);
 void sa_cache_fini(void);
-void *sa_spill_alloc(int);
-void sa_spill_free(void *);
 int sa_set_sa_object(objset_t *, uint64_t);
 int sa_hdrsize(void *);
 void sa_handle_lock(sa_handle_t *);

module/zfs/sa.c

@@ -202,7 +202,6 @@ sa_attr_type_t sa_dummy_zpl_layout[] = { 0 };
 
 static int sa_legacy_attr_count = 16;
 static kmem_cache_t *sa_cache = NULL;
-static kmem_cache_t *spill_cache = NULL;
 
 /*ARGSUSED*/
 static int
@@ -234,30 +233,13 @@ sa_cache_init(void)
 	sa_cache = kmem_cache_create("sa_cache",
 	    sizeof (sa_handle_t), 0, sa_cache_constructor,
 	    sa_cache_destructor, NULL, NULL, NULL, 0);
-	spill_cache = kmem_cache_create("spill_cache",
-	    SPA_MAXBLOCKSIZE, 0, NULL, NULL, NULL, NULL, NULL, 0);
 }
 
 void
 sa_cache_fini(void)
 {
 	if (sa_cache)
 		kmem_cache_destroy(sa_cache);
-
-	if (spill_cache)
-		kmem_cache_destroy(spill_cache);
-}
-
-void *
-sa_spill_alloc(int flags)
-{
-	return (kmem_cache_alloc(spill_cache, flags));
-}
-
-void
-sa_spill_free(void *obj)
-{
-	kmem_cache_free(spill_cache, obj);
 }
 
 static int
@@ -1672,6 +1654,7 @@ sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
 	void *old_data[2];
 	int bonus_attr_count = 0;
 	int bonus_data_size = 0;
+	int spill_data_size = 0;
 	int spill_attr_count = 0;
 	int error;
 	uint16_t length;
@@ -1701,8 +1684,8 @@ sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
 	/* Bring spill buffer online if it isn't currently */
 
 	if ((error = sa_get_spill(hdl)) == 0) {
-		ASSERT3U(hdl->sa_spill->db_size, <=, SPA_MAXBLOCKSIZE);
-		old_data[1] = sa_spill_alloc(KM_SLEEP);
+		spill_data_size = hdl->sa_spill->db_size;
+		old_data[1] = zio_buf_alloc(spill_data_size);
 		bcopy(hdl->sa_spill->db_data, old_data[1],
 		    hdl->sa_spill->db_size);
 		spill_attr_count =
@@ -1787,7 +1770,7 @@ sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
 	if (old_data[0])
 		kmem_free(old_data[0], bonus_data_size);
 	if (old_data[1])
-		sa_spill_free(old_data[1]);
+		zio_buf_free(old_data[1], spill_data_size);
 	kmem_free(attr_desc, sizeof (sa_bulk_attr_t) * attr_count);
 
 	return (error);
@@ -2077,8 +2060,6 @@ EXPORT_SYMBOL(sa_replace_all_by_template_locked);
 EXPORT_SYMBOL(sa_enabled);
 EXPORT_SYMBOL(sa_cache_init);
 EXPORT_SYMBOL(sa_cache_fini);
-EXPORT_SYMBOL(sa_spill_alloc);
-EXPORT_SYMBOL(sa_spill_free);
 EXPORT_SYMBOL(sa_set_sa_object);
 EXPORT_SYMBOL(sa_hdrsize);
 EXPORT_SYMBOL(sa_handle_lock);

module/zfs/zfs_sa.c

@@ -205,13 +205,13 @@ zfs_sa_get_xattr(znode_t *zp)
 			return (error);
 	}
 
-	obj = sa_spill_alloc(KM_SLEEP);
+	obj = zio_buf_alloc(size);
 
 	error = sa_lookup(zp->z_sa_hdl, SA_ZPL_DXATTR(zsb), obj, size);
 	if (error == 0)
 		error = nvlist_unpack(obj, size, &zp->z_xattr_cached, KM_SLEEP);
 
-	sa_spill_free(obj);
+	zio_buf_free(obj, size);
 
 	return (error);
 }
@@ -233,7 +233,7 @@ zfs_sa_set_xattr(znode_t *zp)
 	if (error)
 		goto out;
 
-	obj = sa_spill_alloc(KM_SLEEP);
+	obj = zio_buf_alloc(size);
 
 	error = nvlist_pack(zp->z_xattr_cached, &obj, &size,
 	    NV_ENCODE_XDR, KM_SLEEP);
@@ -256,7 +256,7 @@ zfs_sa_set_xattr(znode_t *zp)
 			dmu_tx_commit(tx);
 	}
 out_free:
-	sa_spill_free(obj);
+	zio_buf_free(obj, size);
 out:
 	return (error);
 }

module/zfs/zio.c

@@ -3393,6 +3393,10 @@ EXPORT_SYMBOL(zio_handle_fault_injection);
 EXPORT_SYMBOL(zio_handle_device_injection);
 EXPORT_SYMBOL(zio_handle_label_injection);
 EXPORT_SYMBOL(zio_type_name);
+EXPORT_SYMBOL(zio_buf_alloc);
+EXPORT_SYMBOL(zio_data_buf_alloc);
+EXPORT_SYMBOL(zio_buf_free);
+EXPORT_SYMBOL(zio_data_buf_free);
 
 module_param(zio_bulk_flags, int, 0644);
 MODULE_PARM_DESC(zio_bulk_flags, "Additional flags to pass to bulk buffers");