Implement persistent L2ARC as a zpool property

Signed-off-by: George Amanakis <gamanakis@gmail.com>

include/sys/arc.h

@@ -300,7 +300,7 @@ void arc_fini(void);
  * Level 2 ARC
  */
 
-void l2arc_add_vdev(spa_t *spa, vdev_t *vd, boolean_t rebuild);
+void l2arc_add_vdev(spa_t *spa, vdev_t *vd);
 void l2arc_remove_vdev(vdev_t *vd);
 boolean_t l2arc_vdev_present(vdev_t *vd);
 void l2arc_init(void);

include/sys/fs/zfs.h

@@ -246,6 +246,7 @@ typedef enum {
 	ZPOOL_PROP_CHECKPOINT,
 	ZPOOL_PROP_LOAD_GUID,
 	ZPOOL_PROP_AUTOTRIM,
+	ZPOOL_PROP_L2CACHE_PERSISTENT,
 	ZPOOL_NUM_PROPS
 } zpool_prop_t;
 
@@ -687,7 +688,6 @@ typedef struct zpool_load_policy {
 #define	ZPOOL_CONFIG_PHYS_PATH		"phys_path"
 #define	ZPOOL_CONFIG_IS_LOG		"is_log"
 #define	ZPOOL_CONFIG_L2CACHE		"l2cache"
-#define	ZPOOL_CONFIG_L2CACHE_PERSISTENT	"l2cache_persistent"
 #define	ZPOOL_CONFIG_HOLE_ARRAY		"hole_array"
 #define	ZPOOL_CONFIG_VDEV_CHILDREN	"vdev_children"
 #define	ZPOOL_CONFIG_IS_HOLE		"is_hole"

include/sys/spa_impl.h

@@ -412,6 +412,7 @@ struct spa {
 	list_t		spa_leaf_list;		/* list of leaf vdevs */
 	uint64_t	spa_leaf_list_gen;	/* track leaf_list changes */
 	uint32_t	spa_hostid;		/* cached system hostid */
+	uint64_t	spa_l2cache_persistent; /* persistent L2ARC */
 
 	/* synchronization for threads in spa_wait */
 	kmutex_t	spa_activities_lock;

man/man8/zpool.8

@@ -520,8 +520,9 @@ If a read error is encountered on a cache device, that read I/O is reissued to
 the original storage pool device, which might be part of a mirrored or raidz
 configuration.
 .Pp
-The content of the cache devices is considered volatile, as is the case with
-other system caches.
+The content of the cache devices can be made non-volatile, see the
+.Sy l2cache_persistent
+zpool property.
 .Ss Pool checkpoint
 Before starting critical procedures that include destructive actions (e.g
 .Nm zfs Cm destroy
@@ -910,6 +911,11 @@ See
 .Xr spl-module-parameters 5
 for additional details.  The default value is
 .Sy off .
+.It Sy l2cache_persistent Ns = Ns Sy on Ns | Ns Sy off
+Controls whether the L2ARC is rebuilt at reboot and thus made persistent.
+When writing buffers to L2ARC, we periodically add some metadata to make sure
+we can pick them up after reboot to rebuilt the L2ARC, thus reducing the impact
+that any downtime has on the performance of storage systems with large caches.
 .It Sy version Ns = Ns Ar version
 The current on-disk version of the pool.
 This can be increased, but never decreased.

module/zcommon/zpool_prop.c

@@ -123,6 +123,10 @@ zpool_prop_init(void)
 	zprop_register_index(ZPOOL_PROP_MULTIHOST, "multihost", 0,
 	    PROP_DEFAULT, ZFS_TYPE_POOL, "on | off", "MULTIHOST",
 	    boolean_table);
+	zprop_register_index(ZPOOL_PROP_L2CACHE_PERSISTENT,
+	    "l2cache_persistent", 0,
+	    PROP_DEFAULT, ZFS_TYPE_POOL, "on | off", "L2CACHE_PERSISTENT",
+	    boolean_table);
 
 	/* default index properties */
 	zprop_register_index(ZPOOL_PROP_FAILUREMODE, "failmode",

module/zfs/arc.c

@@ -865,20 +865,6 @@ static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
 static void l2arc_read_done(zio_t *);
 
-/*
- * Performance tuning of L2ARC persistence:
- *
- * l2arc_rebuild_enabled : Controls whether L2ARC device adds (either at
- *		pool import or when adding one manually later) will attempt
- *		to rebuild L2ARC buffer contents. In special circumstances,
- *		the administrator may want to set this to B_FALSE, if they
- *		are having trouble importing a pool or attaching an L2ARC
- *		device (e.g. the L2ARC device is slow to read in stored log
- *		metadata, or the metadata has become somehow
- *		fragmented/unusable).
- */
-boolean_t l2arc_rebuild_enabled = B_TRUE;
-
 /* L2ARC persistence rebuild control routines. */
 static void l2arc_dev_rebuild_start(l2arc_dev_t *dev);
 static int l2arc_rebuild(l2arc_dev_t *dev);
@@ -8823,7 +8809,7 @@ l2arc_vdev_get(vdev_t *vd)
  * we should attempt a persistent L2ARC rebuild.
  */
 void
-l2arc_add_vdev(spa_t *spa, vdev_t *vd, boolean_t rebuild)
+l2arc_add_vdev(spa_t *spa, vdev_t *vd)
 {
 	l2arc_dev_t *adddev;
 
@@ -8865,7 +8851,7 @@ l2arc_add_vdev(spa_t *spa, vdev_t *vd, boolean_t rebuild)
 	mutex_enter(&l2arc_dev_mtx);
 	list_insert_head(l2arc_dev_list, adddev);
 	atomic_inc_64(&l2arc_ndev);
-	if (rebuild && l2arc_rebuild_enabled &&
+	if (spa->spa_l2cache_persistent &&
 	    adddev->l2ad_end - adddev->l2ad_start > L2ARC_PERSIST_MIN_SIZE) {
 		/*
 		 * Just mark the device as pending for a rebuild. We won't
@@ -8897,11 +8883,10 @@ l2arc_remove_vdev(vdev_t *vd)
 	 * Cancel any ongoing or scheduled rebuild (race protection with
 	 * l2arc_spa_rebuild_start provided via l2arc_dev_mtx).
 	 */
-	if (remdev->l2ad_rebuild == B_TRUE &&
-	    remdev->l2ad_rebuild_began == B_TRUE) {
+	if (remdev->l2ad_rebuild_began == B_TRUE) {
 		remdev->l2ad_rebuild_cancel = B_TRUE;
 		mutex_enter(&l2arc_rebuild_thr_lock);
-		while (remdev->l2ad_rebuild == B_TRUE)
+		while (remdev->l2ad_rebuild_began == B_TRUE)
 			cv_wait(&l2arc_rebuild_thr_cv, &l2arc_rebuild_thr_lock);
 		mutex_exit(&l2arc_rebuild_thr_lock);
 	}
@@ -9022,7 +9007,10 @@ l2arc_spa_rebuild_start(spa_t *spa)
 			/* Don't attempt a rebuild if the vdev is UNAVAIL */
 			continue;
 		}
-		if (dev->l2ad_rebuild && !dev->l2ad_rebuild_cancel) {
+		if ((dev->l2ad_rebuild =
+		    (spa->spa_l2cache_persistent &&
+		    dev->l2ad_end - dev->l2ad_start >
+		    L2ARC_PERSIST_MIN_SIZE)) && !dev->l2ad_rebuild_cancel) {
 #ifdef	_KERNEL
 			(void) thread_create(NULL, 0,
 			    (void (*)(void *))l2arc_dev_rebuild_start, dev,
@@ -9047,7 +9035,6 @@ l2arc_dev_rebuild_start(l2arc_dev_t *dev)
 		dev->l2ad_rebuild_began = B_TRUE;
 		(void) l2arc_rebuild(dev);
 		dev->l2ad_rebuild_began = B_FALSE;
-		dev->l2ad_rebuild = B_FALSE;
 	}
 
 	thread_exit();
@@ -9193,7 +9180,7 @@ l2arc_rebuild(l2arc_dev_t *dev)
 		for (;;) {
 			if (dev->l2ad_rebuild_cancel) {
 				mutex_enter(&l2arc_rebuild_thr_lock);
-				dev->l2ad_rebuild = B_FALSE;
+				dev->l2ad_rebuild_began = B_FALSE;
 				cv_signal(&l2arc_rebuild_thr_cv);
 				mutex_exit(&l2arc_rebuild_thr_lock);
 				err = SET_ERROR(ECANCELED);

module/zfs/spa.c

@@ -1894,14 +1894,8 @@ spa_load_l2cache(spa_t *spa)
 
 			(void) vdev_validate_aux(vd);
 
-			if (!vdev_is_dead(vd)) {
-				boolean_t do_rebuild = B_FALSE;
-
-				(void) nvlist_lookup_boolean_value(l2cache[i],
-				    ZPOOL_CONFIG_L2CACHE_PERSISTENT,
-				    &do_rebuild);
-				l2arc_add_vdev(spa, vd, do_rebuild);
-			}
+			if (!vdev_is_dead(vd))
+				l2arc_add_vdev(spa, vd);
 		}
 	}
 
@@ -4101,6 +4095,8 @@ spa_ld_get_props(spa_t *spa)
 		spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand);
 		spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost);
 		spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim);
+		spa_prop_find(spa, ZPOOL_PROP_L2CACHE_PERSISTENT,
+		    &spa->spa_l2cache_persistent);
 		spa->spa_autoreplace = (autoreplace != 0);
 	}
 
@@ -5883,6 +5879,8 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
 	spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND);
 	spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST);
 	spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM);
+	spa->spa_l2cache_persistent =
+	    zpool_prop_default_numeric(ZPOOL_PROP_L2CACHE_PERSISTENT);
 
 	if (props != NULL) {
 		spa_configfile_set(spa, props, B_FALSE);
@@ -8562,6 +8560,11 @@ spa_sync_props(void *arg, dmu_tx_t *tx)
 			case ZPOOL_PROP_MULTIHOST:
 				spa->spa_multihost = intval;
 				break;
+			case ZPOOL_PROP_L2CACHE_PERSISTENT:
+				spa->spa_l2cache_persistent = intval;
+				spa_async_request(spa,
+				    SPA_ASYNC_L2CACHE_REBUILD);
+				break;
 			default:
 				break;
 			}

module/zfs/vdev.c

@@ -2286,12 +2286,7 @@ vdev_reopen(vdev_t *vd)
 		if (vdev_readable(vd) && vdev_writeable(vd) &&
 		    vd->vdev_aux == &spa->spa_l2cache &&
 		    !l2arc_vdev_present(vd)) {
-			/*
-			 * When reopening we can assume persistent L2ARC is
-			 * supported, since we've already opened the device
-			 * in the past and prepended an L2ARC uberblock.
-			 */
-			l2arc_add_vdev(spa, vd, B_TRUE);
+			l2arc_add_vdev(spa, vd);
 		}
 	} else {
 		(void) vdev_validate(vd);

module/zfs/vdev_label.c

@@ -508,11 +508,6 @@ vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats,
 		}
 	}
 
-	if (flags & VDEV_CONFIG_L2CACHE)
-		/* indicate that we support L2ARC persistency */
-		VERIFY(nvlist_add_boolean_value(nv,
-		    ZPOOL_CONFIG_L2CACHE_PERSISTENT, B_TRUE) == 0);
-
 	if (vd->vdev_dtl_sm != NULL) {
 		fnvlist_add_uint64(nv, ZPOOL_CONFIG_DTL,
 		    space_map_object(vd->vdev_dtl_sm));

tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg

@@ -57,6 +57,7 @@ typeset -a properties=(
     "leaked"
     "multihost"
     "autotrim"
+    "l2cache_persistent"
     "feature@async_destroy"
     "feature@empty_bpobj"
     "feature@lz4_compress"