From e02d8f051a19961d47b0b61093c031f192f74592 Mon Sep 17 00:00:00 2001 From: George Amanakis Date: Fri, 15 Nov 2019 22:52:49 -0500 Subject: [PATCH] Implement persistent L2ARC as a zpool property Signed-off-by: George Amanakis --- include/sys/arc.h | 2 +- include/sys/fs/zfs.h | 2 +- include/sys/spa_impl.h | 1 + man/man8/zpool.8 | 10 ++++-- module/zcommon/zpool_prop.c | 4 +++ module/zfs/arc.c | 31 ++++++------------- module/zfs/spa.c | 19 +++++++----- module/zfs/vdev.c | 7 +---- module/zfs/vdev_label.c | 5 --- .../cli_root/zpool_get/zpool_get.cfg | 1 + 10 files changed, 37 insertions(+), 45 deletions(-) diff --git a/include/sys/arc.h b/include/sys/arc.h index 30b6468746aa..0d17b4e467d7 100644 --- a/include/sys/arc.h +++ b/include/sys/arc.h @@ -300,7 +300,7 @@ void arc_fini(void); * Level 2 ARC */ -void l2arc_add_vdev(spa_t *spa, vdev_t *vd, boolean_t rebuild); +void l2arc_add_vdev(spa_t *spa, vdev_t *vd); void l2arc_remove_vdev(vdev_t *vd); boolean_t l2arc_vdev_present(vdev_t *vd); void l2arc_init(void); diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h index 9689a76841fd..21cd96865392 100644 --- a/include/sys/fs/zfs.h +++ b/include/sys/fs/zfs.h @@ -246,6 +246,7 @@ typedef enum { ZPOOL_PROP_CHECKPOINT, ZPOOL_PROP_LOAD_GUID, ZPOOL_PROP_AUTOTRIM, + ZPOOL_PROP_L2CACHE_PERSISTENT, ZPOOL_NUM_PROPS } zpool_prop_t; @@ -687,7 +688,6 @@ typedef struct zpool_load_policy { #define ZPOOL_CONFIG_PHYS_PATH "phys_path" #define ZPOOL_CONFIG_IS_LOG "is_log" #define ZPOOL_CONFIG_L2CACHE "l2cache" -#define ZPOOL_CONFIG_L2CACHE_PERSISTENT "l2cache_persistent" #define ZPOOL_CONFIG_HOLE_ARRAY "hole_array" #define ZPOOL_CONFIG_VDEV_CHILDREN "vdev_children" #define ZPOOL_CONFIG_IS_HOLE "is_hole" diff --git a/include/sys/spa_impl.h b/include/sys/spa_impl.h index c83252bcafec..c157c0fa7cbe 100644 --- a/include/sys/spa_impl.h +++ b/include/sys/spa_impl.h @@ -412,6 +412,7 @@ struct spa { list_t spa_leaf_list; /* list of leaf vdevs */ uint64_t spa_leaf_list_gen; /* track leaf_list changes */ uint32_t spa_hostid; /* cached system hostid */ + uint64_t spa_l2cache_persistent; /* persistent L2ARC */ /* synchronization for threads in spa_wait */ kmutex_t spa_activities_lock; diff --git a/man/man8/zpool.8 b/man/man8/zpool.8 index df30b7ca05c8..f63c4c6852e0 100644 --- a/man/man8/zpool.8 +++ b/man/man8/zpool.8 @@ -520,8 +520,9 @@ If a read error is encountered on a cache device, that read I/O is reissued to the original storage pool device, which might be part of a mirrored or raidz configuration. .Pp -The content of the cache devices is considered volatile, as is the case with -other system caches. +The content of the cache devices can be made non-volatile, see the +.Sy l2cache_persistent +zpool property. .Ss Pool checkpoint Before starting critical procedures that include destructive actions (e.g .Nm zfs Cm destroy @@ -910,6 +911,11 @@ See .Xr spl-module-parameters 5 for additional details. The default value is .Sy off . +.It Sy l2cache_persistent Ns = Ns Sy on Ns | Ns Sy off +Controls whether the L2ARC is rebuilt at reboot and thus made persistent. +When writing buffers to L2ARC, we periodically add some metadata to make sure +we can pick them up after reboot to rebuilt the L2ARC, thus reducing the impact +that any downtime has on the performance of storage systems with large caches. .It Sy version Ns = Ns Ar version The current on-disk version of the pool. This can be increased, but never decreased. diff --git a/module/zcommon/zpool_prop.c b/module/zcommon/zpool_prop.c index 155d557f464a..4180ca389a6e 100644 --- a/module/zcommon/zpool_prop.c +++ b/module/zcommon/zpool_prop.c @@ -123,6 +123,10 @@ zpool_prop_init(void) zprop_register_index(ZPOOL_PROP_MULTIHOST, "multihost", 0, PROP_DEFAULT, ZFS_TYPE_POOL, "on | off", "MULTIHOST", boolean_table); + zprop_register_index(ZPOOL_PROP_L2CACHE_PERSISTENT, + "l2cache_persistent", 0, + PROP_DEFAULT, ZFS_TYPE_POOL, "on | off", "L2CACHE_PERSISTENT", + boolean_table); /* default index properties */ zprop_register_index(ZPOOL_PROP_FAILUREMODE, "failmode", diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 74b5af687881..562ecb7dd7a0 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -865,20 +865,6 @@ static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags); static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *); static void l2arc_read_done(zio_t *); -/* - * Performance tuning of L2ARC persistence: - * - * l2arc_rebuild_enabled : Controls whether L2ARC device adds (either at - * pool import or when adding one manually later) will attempt - * to rebuild L2ARC buffer contents. In special circumstances, - * the administrator may want to set this to B_FALSE, if they - * are having trouble importing a pool or attaching an L2ARC - * device (e.g. the L2ARC device is slow to read in stored log - * metadata, or the metadata has become somehow - * fragmented/unusable). - */ -boolean_t l2arc_rebuild_enabled = B_TRUE; - /* L2ARC persistence rebuild control routines. */ static void l2arc_dev_rebuild_start(l2arc_dev_t *dev); static int l2arc_rebuild(l2arc_dev_t *dev); @@ -8823,7 +8809,7 @@ l2arc_vdev_get(vdev_t *vd) * we should attempt a persistent L2ARC rebuild. */ void -l2arc_add_vdev(spa_t *spa, vdev_t *vd, boolean_t rebuild) +l2arc_add_vdev(spa_t *spa, vdev_t *vd) { l2arc_dev_t *adddev; @@ -8865,7 +8851,7 @@ l2arc_add_vdev(spa_t *spa, vdev_t *vd, boolean_t rebuild) mutex_enter(&l2arc_dev_mtx); list_insert_head(l2arc_dev_list, adddev); atomic_inc_64(&l2arc_ndev); - if (rebuild && l2arc_rebuild_enabled && + if (spa->spa_l2cache_persistent && adddev->l2ad_end - adddev->l2ad_start > L2ARC_PERSIST_MIN_SIZE) { /* * Just mark the device as pending for a rebuild. We won't @@ -8897,11 +8883,10 @@ l2arc_remove_vdev(vdev_t *vd) * Cancel any ongoing or scheduled rebuild (race protection with * l2arc_spa_rebuild_start provided via l2arc_dev_mtx). */ - if (remdev->l2ad_rebuild == B_TRUE && - remdev->l2ad_rebuild_began == B_TRUE) { + if (remdev->l2ad_rebuild_began == B_TRUE) { remdev->l2ad_rebuild_cancel = B_TRUE; mutex_enter(&l2arc_rebuild_thr_lock); - while (remdev->l2ad_rebuild == B_TRUE) + while (remdev->l2ad_rebuild_began == B_TRUE) cv_wait(&l2arc_rebuild_thr_cv, &l2arc_rebuild_thr_lock); mutex_exit(&l2arc_rebuild_thr_lock); } @@ -9022,7 +9007,10 @@ l2arc_spa_rebuild_start(spa_t *spa) /* Don't attempt a rebuild if the vdev is UNAVAIL */ continue; } - if (dev->l2ad_rebuild && !dev->l2ad_rebuild_cancel) { + if ((dev->l2ad_rebuild = + (spa->spa_l2cache_persistent && + dev->l2ad_end - dev->l2ad_start > + L2ARC_PERSIST_MIN_SIZE)) && !dev->l2ad_rebuild_cancel) { #ifdef _KERNEL (void) thread_create(NULL, 0, (void (*)(void *))l2arc_dev_rebuild_start, dev, @@ -9047,7 +9035,6 @@ l2arc_dev_rebuild_start(l2arc_dev_t *dev) dev->l2ad_rebuild_began = B_TRUE; (void) l2arc_rebuild(dev); dev->l2ad_rebuild_began = B_FALSE; - dev->l2ad_rebuild = B_FALSE; } thread_exit(); @@ -9193,7 +9180,7 @@ l2arc_rebuild(l2arc_dev_t *dev) for (;;) { if (dev->l2ad_rebuild_cancel) { mutex_enter(&l2arc_rebuild_thr_lock); - dev->l2ad_rebuild = B_FALSE; + dev->l2ad_rebuild_began = B_FALSE; cv_signal(&l2arc_rebuild_thr_cv); mutex_exit(&l2arc_rebuild_thr_lock); err = SET_ERROR(ECANCELED); diff --git a/module/zfs/spa.c b/module/zfs/spa.c index 50d6d204f004..fd03e1d3308a 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -1894,14 +1894,8 @@ spa_load_l2cache(spa_t *spa) (void) vdev_validate_aux(vd); - if (!vdev_is_dead(vd)) { - boolean_t do_rebuild = B_FALSE; - - (void) nvlist_lookup_boolean_value(l2cache[i], - ZPOOL_CONFIG_L2CACHE_PERSISTENT, - &do_rebuild); - l2arc_add_vdev(spa, vd, do_rebuild); - } + if (!vdev_is_dead(vd)) + l2arc_add_vdev(spa, vd); } } @@ -4101,6 +4095,8 @@ spa_ld_get_props(spa_t *spa) spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost); spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim); + spa_prop_find(spa, ZPOOL_PROP_L2CACHE_PERSISTENT, + &spa->spa_l2cache_persistent); spa->spa_autoreplace = (autoreplace != 0); } @@ -5883,6 +5879,8 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST); spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM); + spa->spa_l2cache_persistent = + zpool_prop_default_numeric(ZPOOL_PROP_L2CACHE_PERSISTENT); if (props != NULL) { spa_configfile_set(spa, props, B_FALSE); @@ -8562,6 +8560,11 @@ spa_sync_props(void *arg, dmu_tx_t *tx) case ZPOOL_PROP_MULTIHOST: spa->spa_multihost = intval; break; + case ZPOOL_PROP_L2CACHE_PERSISTENT: + spa->spa_l2cache_persistent = intval; + spa_async_request(spa, + SPA_ASYNC_L2CACHE_REBUILD); + break; default: break; } diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index 8eb2ec56b52d..421a76ae4554 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -2286,12 +2286,7 @@ vdev_reopen(vdev_t *vd) if (vdev_readable(vd) && vdev_writeable(vd) && vd->vdev_aux == &spa->spa_l2cache && !l2arc_vdev_present(vd)) { - /* - * When reopening we can assume persistent L2ARC is - * supported, since we've already opened the device - * in the past and prepended an L2ARC uberblock. - */ - l2arc_add_vdev(spa, vd, B_TRUE); + l2arc_add_vdev(spa, vd); } } else { (void) vdev_validate(vd); diff --git a/module/zfs/vdev_label.c b/module/zfs/vdev_label.c index 29e44e5f180e..6bb3c3c68072 100644 --- a/module/zfs/vdev_label.c +++ b/module/zfs/vdev_label.c @@ -508,11 +508,6 @@ vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats, } } - if (flags & VDEV_CONFIG_L2CACHE) - /* indicate that we support L2ARC persistency */ - VERIFY(nvlist_add_boolean_value(nv, - ZPOOL_CONFIG_L2CACHE_PERSISTENT, B_TRUE) == 0); - if (vd->vdev_dtl_sm != NULL) { fnvlist_add_uint64(nv, ZPOOL_CONFIG_DTL, space_map_object(vd->vdev_dtl_sm)); diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg index 9dea1e2cd7a5..0b285ea70414 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg @@ -57,6 +57,7 @@ typeset -a properties=( "leaked" "multihost" "autotrim" + "l2cache_persistent" "feature@async_destroy" "feature@empty_bpobj" "feature@lz4_compress"