From 72d3020f0a32e78dc964fedf53f6154342871e6c Mon Sep 17 00:00:00 2001 From: Mark Maybee Date: Wed, 12 May 2021 09:28:08 -0600 Subject: [PATCH] Rename "noalloc" vdev property to "allocating" Add support for read-only "removing" vdev property Simplify get vdev prop interface to always require pool name Refactor space management for non-allocating devices Add ability to remove user vdev property by setting value to "" --- cmd/zhack/zhack.c | 4 +-- cmd/zpool/zpool_iter.c | 5 ++-- cmd/zpool/zpool_main.c | 57 +++++++++++++++-------------------- include/sys/fs/zfs.h | 13 ++++---- include/sys/spa_impl.h | 1 + lib/libzfs/libzfs_pool.c | 40 ++++++++++++++++--------- module/zcommon/zpool_prop.c | 13 ++++---- module/zfs/spa_misc.c | 22 +++++++------- module/zfs/vdev.c | 47 ++++++++++++++++++++--------- module/zfs/vdev_removal.c | 59 ++++++++++++++++++++++++++++--------- module/zfs/zfs_ioctl.c | 12 ++++---- 11 files changed, 162 insertions(+), 111 deletions(-) diff --git a/cmd/zhack/zhack.c b/cmd/zhack/zhack.c index d87c1ca8fa8f..4f28f9c8b6e6 100644 --- a/cmd/zhack/zhack.c +++ b/cmd/zhack/zhack.c @@ -474,7 +474,6 @@ static int zhack_do_zap(int argc, char **argv) { spa_t *spa; - objset_t *os; uint64_t obj; char *target; @@ -493,9 +492,8 @@ zhack_do_zap(int argc, char **argv) obj = strtoull(argv[1], NULL, 0); zhack_spa_open(target, B_TRUE, FTAG, &spa); - os = spa->spa_meta_objset; - dump_obj(os, obj, argv[1]); + dump_obj(spa->spa_meta_objset, obj, argv[1]); spa_close(spa, FTAG); diff --git a/cmd/zpool/zpool_iter.c b/cmd/zpool/zpool_iter.c index 6b8e035b6fa4..ba1f884308cf 100644 --- a/cmd/zpool/zpool_iter.c +++ b/cmd/zpool/zpool_iter.c @@ -304,8 +304,9 @@ for_each_vdev_cb(zpool_handle_t *zhp, nvlist_t *nv, pool_vdev_iter_f func, if (nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) != 0) return (ret); - /* Don't run our function on root vdevs */ - if (strcmp(type, VDEV_TYPE_ROOT) != 0) { + /* Don't run our function on root or indirect vdevs */ + if ((strcmp(type, VDEV_TYPE_ROOT) != 0) && + (strcmp(type, VDEV_TYPE_INDIRECT) != 0)) { ret |= func(zhp, nv, data); } diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index 71252bee2667..a9f69bafacc0 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -331,7 +331,7 @@ static zpool_command_t command_table[] = { #define VDEV_ALLOC_CLASS_LOGS "logs" static zpool_command_t *current_command; -static zfs_type_t current_prop_type = ZFS_TYPE_POOL; +static zfs_type_t current_prop_type = (ZFS_TYPE_POOL | ZFS_TYPE_VDEV); static char history_str[HIS_MAX_RECORD_LEN]; static boolean_t log_history = B_TRUE; static uint_t timestamp_fmt = NODATE; @@ -488,7 +488,7 @@ print_prop_cb(int prop, void *cb) } /* - * Callback routine that will print out a pool property value. + * Callback routine that will print out a vdev property value. */ static int print_vdev_prop_cb(int prop, void *cb) @@ -540,6 +540,7 @@ usage(boolean_t requested) } if (current_command != NULL && + current_prop_type != (ZFS_TYPE_POOL | ZFS_TYPE_VDEV) && ((strcmp(current_command->name, "set") == 0) || (strcmp(current_command->name, "get") == 0) || (strcmp(current_command->name, "list") == 0))) { @@ -826,7 +827,7 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props, (!zpool_prop_feature(propname) || zpool_prop_vdev(propname))) { (void) fprintf(stderr, gettext("property '%s' is " - "not a valid pool property\n"), propname); + "not a valid pool or vdev property\n"), propname); return (2); } @@ -9886,6 +9887,7 @@ get_callback_vdev(zpool_handle_t *zhp, char *vdevname, void *data) zprop_list_t *pl; for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) { + char *prop_name; /* * Skip the special fake placeholder. This will also skip * over the name property when 'all' is specified. @@ -9896,21 +9898,14 @@ get_callback_vdev(zpool_handle_t *zhp, char *vdevname, void *data) if (pl->pl_prop == ZPROP_INVAL) { /* User Properties */ - if (zpool_get_vdev_prop(zhp, vdevname, pl->pl_prop, - pl->pl_user_prop, value, sizeof (value), &srctype, - cbp->cb_literal) == 0) { - zprop_print_one_property( - vdevname, cbp, pl->pl_user_prop, value, - srctype, NULL, NULL); - } + prop_name = pl->pl_user_prop; } else { - if (zpool_get_vdev_prop(zhp, vdevname, pl->pl_prop, - NULL, value, sizeof (value), &srctype, - cbp->cb_literal) != 0) - continue; - - zprop_print_one_property( - vdevname, cbp, vdev_prop_to_name(pl->pl_prop), + prop_name = (char *)vdev_prop_to_name(pl->pl_prop); + } + if (zpool_get_vdev_prop(zhp, vdevname, pl->pl_prop, + prop_name, value, sizeof (value), &srctype, + cbp->cb_literal) == 0) { + zprop_print_one_property(vdevname, cbp, prop_name, value, srctype, NULL, NULL); } } @@ -9956,20 +9951,20 @@ get_callback(zpool_handle_t *zhp, void *data) int vid; if (cbp->cb_type == ZFS_TYPE_VDEV) { - if (strcmp(cbp->cb_vdevs.cb_names[0], "all") == 0) { + if (strcmp(cbp->cb_vdevs.cb_names[0], "all-vdevs") == 0) { for_each_vdev(zhp, get_callback_vdev_width_cb, data); for_each_vdev(zhp, get_callback_vdev_cb, data); } else { + /* Adjust column widths for vdev properties */ for (vid = 0; vid < cbp->cb_vdevs.cb_names_count; vid++) { - /* Adjust column widths for vdev properties */ vdev_expand_proplist(zhp, cbp->cb_vdevs.cb_names[vid], &cbp->cb_proplist); } + /* Display the properties */ for (vid = 0; vid < cbp->cb_vdevs.cb_names_count; vid++) { - /* Display the properties */ get_callback_vdev(zhp, cbp->cb_vdevs.cb_names[vid], data); } @@ -10133,24 +10128,18 @@ zpool_do_get(int argc, char **argv) /* No args, so just print the defaults. */ } else if (are_all_pools(argc, argv)) { /* All the args are pool names */ - } else if (are_vdevs_in_pool(argc, argv, NULL, &cb.cb_vdevs)) { - /* All the args are vdevs */ - cb.cb_vdevs.cb_names = argv; - cb.cb_vdevs.cb_names_count = argc; - cb.cb_type = ZFS_TYPE_VDEV; - argc = 0; /* No pools to process */ } else if (are_all_pools(1, argv)) { /* The first arg is a pool name */ - if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0], - &cb.cb_vdevs) || (strcmp(argv[1], "all") == 0)) { - /* ...and the rest are vdev names */ + if ((argc == 2 && strcmp(argv[1], "all-vdevs") == 0) || + are_vdevs_in_pool(argc - 1, argv + 1, argv[0], + &cb.cb_vdevs)) { + /* ... and the rest are vdev names */ cb.cb_vdevs.cb_names = argv + 1; cb.cb_vdevs.cb_names_count = argc - 1; cb.cb_type = ZFS_TYPE_VDEV; argc = 1; /* One pool to process */ } else { - fprintf(stderr, gettext("Expected either a list of ")); - fprintf(stderr, gettext("pools, or list of vdevs in")); + fprintf(stderr, gettext("Expected a list of vdevs in")); fprintf(stderr, " \"%s\", ", argv[0]); fprintf(stderr, gettext("but got:\n")); error_list_unresolved_vdevs(argc - 1, argv + 1, @@ -10161,11 +10150,11 @@ zpool_do_get(int argc, char **argv) } } else { /* - * The args don't make sense. The first arg isn't a pool name, - * nor are all the args vdevs. + * The first arg isn't a pool name, */ - fprintf(stderr, gettext("Unable to parse pools/vdevs list.\n")); + fprintf(stderr, gettext("missing pool name.\n")); fprintf(stderr, "\n"); + usage(B_FALSE); return (1); } diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h index 8d8229f8d16f..5d28db0a6df3 100644 --- a/include/sys/fs/zfs.h +++ b/include/sys/fs/zfs.h @@ -347,7 +347,8 @@ typedef enum { VDEV_PROP_BYTES_FREE, VDEV_PROP_BYTES_CLAIM, VDEV_PROP_BYTES_TRIM, - VDEV_PROP_NOALLOC, + VDEV_PROP_REMOVING, + VDEV_PROP_ALLOCATING, VDEV_NUM_PROPS } vdev_prop_t; @@ -1152,7 +1153,6 @@ typedef struct vdev_stat { uint64_t vs_checksum_errors; /* checksum errors */ uint64_t vs_initialize_errors; /* initializing errors */ uint64_t vs_self_healed; /* self-healed bytes */ - uint64_t vs_noalloc; /* allocations halted? */ uint64_t vs_scan_removing; /* removing? */ uint64_t vs_scan_processed; /* scan processed bytes */ uint64_t vs_fragmentation; /* device fragmentation */ @@ -1173,6 +1173,7 @@ typedef struct vdev_stat { uint64_t vs_configured_ashift; /* TLV vdev_ashift */ uint64_t vs_logical_ashift; /* vdev_logical_ashift */ uint64_t vs_physical_ashift; /* vdev_physical_ashift */ + uint64_t vs_noalloc; /* allocations halted? */ } vdev_stat_t; /* BEGIN CSTYLED */ @@ -1577,14 +1578,14 @@ typedef enum { /* * The following are names used when invoking ZFS_IOC_VDEV_GET_PROP. */ -#define ZPOOL_VDEV_GET_PROPS_VDEV "vdevprops_get_vdev" -#define ZPOOL_VDEV_GET_PROPS_PROPS "vdevprops_get_props" +#define ZPOOL_VDEV_PROPS_GET_VDEV "vdevprops_get_vdev" +#define ZPOOL_VDEV_PROPS_GET_PROPS "vdevprops_get_props" /* * The following are names used when invoking ZFS_IOC_VDEV_SET_PROP. */ -#define ZPOOL_VDEV_SET_PROPS_VDEV "vdevprops_set_vdev" -#define ZPOOL_VDEV_SET_PROPS_PROPS "vdevprops_set_props" +#define ZPOOL_VDEV_PROPS_SET_VDEV "vdevprops_set_vdev" +#define ZPOOL_VDEV_PROPS_SET_PROPS "vdevprops_set_props" /* * The following are names used when invoking ZFS_IOC_WAIT_FS. diff --git a/include/sys/spa_impl.h b/include/sys/spa_impl.h index bc88cfa15e8e..982dab1de253 100644 --- a/include/sys/spa_impl.h +++ b/include/sys/spa_impl.h @@ -305,6 +305,7 @@ struct spa { uint64_t spa_missing_tvds; /* unopenable tvds on load */ uint64_t spa_missing_tvds_allowed; /* allow loading spa? */ + uint64_t spa_nonallocating_dspace; spa_removing_phys_t spa_removing_phys; spa_vdev_removal_t *spa_vdev_removal; diff --git a/lib/libzfs/libzfs_pool.c b/lib/libzfs/libzfs_pool.c index f7fd78524b7b..8175e86344be 100644 --- a/lib/libzfs/libzfs_pool.c +++ b/lib/libzfs/libzfs_pool.c @@ -5074,8 +5074,8 @@ zpool_get_vdev_prop_value(nvlist_t *nvprop, vdev_prop_t prop, char *prop_name, verify(nvlist_lookup_string(nv, ZPROP_VALUE, &strval) == 0); } else { - src = ZPROP_SRC_NONE; - strval = "-"; + /* user prop not found */ + return (-1); } (void) strlcpy(buf, strval, len); if (srctype) @@ -5230,7 +5230,8 @@ zpool_get_vdev_prop(zpool_handle_t *zhp, const char *vdevname, vdev_prop_t prop, char *prop_name, char *buf, size_t len, zprop_source_t *srctype, boolean_t literal) { - nvlist_t *tgt, *reqnvl, *reqprops, *retprops; + nvlist_t *tgt, *reqnvl, *reqprops; + nvlist_t *retprops = NULL; uint64_t vdev_guid; boolean_t avail_spare, l2cache; char errbuf[1024]; @@ -5244,8 +5245,12 @@ zpool_get_vdev_prop(zpool_handle_t *zhp, const char *vdevname, vdev_prop_t prop, } if ((tgt = zpool_find_vdev(zhp, vdevname, &avail_spare, &l2cache, - NULL)) == NULL) - return (-1); + NULL)) == NULL) { + (void) snprintf(errbuf, sizeof (errbuf), + dgettext(TEXT_DOMAIN, "can not find %s in %s"), + vdevname, zhp->zpool_name); + return (zfs_error(zhp->zpool_hdl, EZFS_NODEVICE, errbuf)); + } verify(nvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID, &vdev_guid) == 0); @@ -5254,11 +5259,18 @@ zpool_get_vdev_prop(zpool_handle_t *zhp, const char *vdevname, vdev_prop_t prop, if (nvlist_alloc(&reqprops, NV_UNIQUE_NAME, 0) != 0) return (no_memory(zhp->zpool_hdl)); - fnvlist_add_uint64(reqnvl, ZPOOL_VDEV_GET_PROPS_VDEV, vdev_guid); + fnvlist_add_uint64(reqnvl, ZPOOL_VDEV_PROPS_GET_VDEV, vdev_guid); - if (prop != VDEV_PROP_INVAL && prop_name == NULL) - prop_name = (char *)vdev_prop_to_name(prop); + if (prop != VDEV_PROP_INVAL) { + /* prop_name overrides prop value */ + if (prop_name != NULL) + prop = vdev_name_to_prop(prop_name); + else + prop_name = (char *)vdev_prop_to_name(prop); + } + if (prop != VDEV_PROP_INVAL) + assert(prop < VDEV_NUM_PROPS); assert(prop_name != NULL); if (nvlist_add_uint64(reqprops, prop_name, prop) != 0) { nvlist_free(reqnvl); @@ -5266,7 +5278,7 @@ zpool_get_vdev_prop(zpool_handle_t *zhp, const char *vdevname, vdev_prop_t prop, return (no_memory(zhp->zpool_hdl)); } - fnvlist_add_nvlist(reqnvl, ZPOOL_VDEV_GET_PROPS_PROPS, reqprops); + fnvlist_add_nvlist(reqnvl, ZPOOL_VDEV_PROPS_GET_PROPS, reqprops); (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, "cannot get vdev property %s from %s in %s"), @@ -5301,7 +5313,7 @@ zpool_get_all_vdev_props(zpool_handle_t *zhp, const char *vdevname, uint64_t vdev_guid; boolean_t avail_spare, l2cache; char errbuf[1024]; - int ret = -1; + int ret; verify(zhp != NULL); if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) { @@ -5323,7 +5335,7 @@ zpool_get_all_vdev_props(zpool_handle_t *zhp, const char *vdevname, if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0) return (no_memory(zhp->zpool_hdl)); - fnvlist_add_uint64(nvl, ZPOOL_VDEV_GET_PROPS_VDEV, vdev_guid); + fnvlist_add_uint64(nvl, ZPOOL_VDEV_PROPS_GET_VDEV, vdev_guid); ret = lzc_get_vdev_prop(zhp->zpool_name, nvl, outnvl); @@ -5343,7 +5355,7 @@ int zpool_set_vdev_prop(zpool_handle_t *zhp, const char *vdevname, const char *propname, const char *propval) { - int ret = -1; + int ret; char errbuf[1024]; vdev_prop_t vprop; nvlist_t *nvl = NULL; @@ -5380,7 +5392,7 @@ zpool_set_vdev_prop(zpool_handle_t *zhp, const char *vdevname, if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0) return (no_memory(zhp->zpool_hdl)); - fnvlist_add_uint64(nvl, ZPOOL_VDEV_SET_PROPS_VDEV, vdev_guid); + fnvlist_add_uint64(nvl, ZPOOL_VDEV_PROPS_SET_VDEV, vdev_guid); if (nvlist_add_string(props, propname, propval) != 0) { nvlist_free(props); @@ -5399,7 +5411,7 @@ zpool_set_vdev_prop(zpool_handle_t *zhp, const char *vdevname, nvlist_free(props); props = realprops; - fnvlist_add_nvlist(nvl, ZPOOL_VDEV_SET_PROPS_PROPS, props); + fnvlist_add_nvlist(nvl, ZPOOL_VDEV_PROPS_SET_PROPS, props); ret = lzc_set_vdev_prop(zhp->zpool_name, nvl, &outnvl); diff --git a/module/zcommon/zpool_prop.c b/module/zcommon/zpool_prop.c index ad924b15a061..2d9b85b7c70e 100644 --- a/module/zcommon/zpool_prop.c +++ b/module/zcommon/zpool_prop.c @@ -359,12 +359,16 @@ vdev_prop_init(void) PROP_READONLY, ZFS_TYPE_VDEV, "", "CLAIMBYTE"); zprop_register_number(VDEV_PROP_BYTES_TRIM, "trim_bytes", 0, PROP_READONLY, ZFS_TYPE_VDEV, "", "TRIMBYTE"); + /* XXX - register this as an index (yes | no) type? */ + zprop_register_number(VDEV_PROP_REMOVING, "removing", 0, + PROP_READONLY, ZFS_TYPE_VDEV, "", "REMOVING"); /* default numeric properties */ /* default index (boolean) properties */ - zprop_register_index(VDEV_PROP_NOALLOC, "noalloc", 0, - PROP_DEFAULT, ZFS_TYPE_VDEV, "on | off", "NOALLOC", boolean_table); + zprop_register_index(VDEV_PROP_ALLOCATING, "allocating", 1, + PROP_DEFAULT, ZFS_TYPE_VDEV, "on | off", "ALLOCATING", + boolean_table); /* default index properties */ @@ -400,10 +404,7 @@ vdev_prop_user(const char *name) foundsep = B_TRUE; } - if (!foundsep) - return (B_FALSE); - - return (B_TRUE); + return (foundsep); } /* diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index f86f755a46ec..e08a6c34d4d2 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -1836,13 +1836,15 @@ spa_update_dspace(spa_t *spa) { spa->spa_dspace = metaslab_class_get_dspace(spa_normal_class(spa)) + ddt_get_dedup_dspace(spa); - if (spa->spa_vdev_removal != NULL) { + if (spa->spa_nonallocating_dspace > 0) { /* - * We can't allocate from the removing device, so subtract - * its size if it was included in dspace (i.e. if this is a - * normal-class vdev, not special/dedup). This prevents the - * DMU/DSL from filling up the (now smaller) pool while we - * are in the middle of removing the device. + * Subtract the space from all the non-allocating vdevs that + * contribute to dspace. This provides a stable/consistent + * value for the available space in the pool (i.e., if + * the data on non-allocating drives is re-written the + * available free space does not fluctuate oddly). It also + * prevents the DMU/DSL from filling up the (now smaller) + * pool if we are in the middle of removing a device. * * Note that the DMU/DSL doesn't actually know or care * how much space is allocated (it does its own tracking @@ -1852,12 +1854,8 @@ spa_update_dspace(spa_t *spa) * device). */ spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER); - vdev_t *vd = - vdev_lookup_top(spa, spa->spa_vdev_removal->svr_vdev_id); - if (vd->vdev_mg->mg_class == spa_normal_class(spa)) { - spa->spa_dspace -= spa_deflate(spa) ? - vd->vdev_stat.vs_dspace : vd->vdev_stat.vs_space; - } + ASSERT3U(spa->spa_dspace, >, spa->spa_nonallocating_dspace); + spa->spa_dspace -= spa->spa_nonallocating_dspace; spa_config_exit(spa, SCL_VDEV, FTAG); } } diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index 18a1d6017db6..767cb81ea8b8 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -1513,6 +1513,10 @@ vdev_metaslab_init(vdev_t *vd, uint64_t txg) metaslab_group_activate(vd->vdev_mg); if (vd->vdev_log_mg != NULL) metaslab_group_activate(vd->vdev_log_mg); + } else if (vd->vdev_noalloc) { + /* track non-allocating vdev space */ + spa->spa_nonallocating_dspace += spa_deflate(spa) ? + vd->vdev_stat.vs_dspace : vd->vdev_stat.vs_space; } if (txg == 0) @@ -5435,14 +5439,14 @@ vdev_sync_props(void *arg, dmu_tx_t *tx) uint64_t vdev_guid; nvlist_t *nvprops; - if (nvlist_lookup_uint64(nvp, ZPOOL_VDEV_SET_PROPS_VDEV, + if (nvlist_lookup_uint64(nvp, ZPOOL_VDEV_PROPS_SET_VDEV, &vdev_guid) != 0) return; if ((vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE)) == NULL) return; - (void) nvlist_lookup_nvlist(nvp, ZPOOL_VDEV_SET_PROPS_PROPS, &nvprops); + (void) nvlist_lookup_nvlist(nvp, ZPOOL_VDEV_PROPS_SET_PROPS, &nvprops); mutex_enter(&spa->spa_props_lock); @@ -5469,8 +5473,14 @@ vdev_sync_props(void *arg, dmu_tx_t *tx) case VDEV_PROP_INVAL: if (vdev_prop_user(propname)) { strval = fnvpair_value_string(elem); - VERIFY0(zap_update(mos, objid, propname, - 1, strlen(strval) + 1, strval, tx)); + if (strlen(strval) == 0) { + /* remove the property if value == "" */ + (void) zap_remove(mos, objid, propname, + tx); + } else { + VERIFY0(zap_update(mos, objid, propname, + 1, strlen(strval) + 1, strval, tx)); + } spa_history_log_internal(spa, "vdev set", tx, "vdev_guid=%llu: %s=%s", (u_longlong_t)vdev_guid, nvpair_name(elem), @@ -5526,11 +5536,11 @@ vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) ASSERT(vd != NULL); - if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_SET_PROPS_VDEV, + if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_SET_VDEV, &vdev_guid) != 0) return (SET_ERROR(EINVAL)); - (void) nvlist_lookup_nvlist(innvl, ZPOOL_VDEV_SET_PROPS_PROPS, + (void) nvlist_lookup_nvlist(innvl, ZPOOL_VDEV_PROPS_SET_PROPS, &nvprops); #if 0 @@ -5575,14 +5585,14 @@ vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) vdev_config_dirty(vd->vdev_top); spa_config_exit(spa, SCL_CONFIG, FTAG); break; - case VDEV_PROP_NOALLOC: + case VDEV_PROP_ALLOCATING: intval = fnvpair_value_uint64(elem); - if (intval == vd->vdev_noalloc) - return (0); /* noop */ + if (intval != vd->vdev_noalloc) + break; if (intval == 1) - error = spa_vdev_noalloc(spa, vdev_guid); - else error = spa_vdev_alloc(spa, vdev_guid); + else + error = spa_vdev_noalloc(spa, vdev_guid); break; default: /* Most processing is done in vdev_sync_props */ @@ -5619,11 +5629,11 @@ vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) ASSERT(vd != NULL); - if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_GET_PROPS_VDEV, + if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_GET_VDEV, &vdev_guid) != 0) return (SET_ERROR(EINVAL)); - (void) nvlist_lookup_nvlist(innvl, ZPOOL_VDEV_GET_PROPS_PROPS, + (void) nvlist_lookup_nvlist(innvl, ZPOOL_VDEV_PROPS_GET_PROPS, &nvprops); /* @@ -5885,13 +5895,22 @@ vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) vd->vdev_stat.vs_bytes[ZIO_TYPE_IOCTL], ZPROP_SRC_NONE); continue; + case VDEV_PROP_REMOVING: + vdev_prop_add_list(outnvl, propname, NULL, + vd->vdev_removing, ZPROP_SRC_NONE); + continue; /* Numeric Properites */ - case VDEV_PROP_NOALLOC: + case VDEV_PROP_ALLOCATING: src = ZPROP_SRC_LOCAL; strval = NULL; err = zap_lookup(mos, objid, nvpair_name(elem), sizeof (uint64_t), 1, &intval); + if (err == ENOENT) + intval = + vdev_prop_default_numeric(prop); + else if (err) + break; if (intval == vdev_prop_default_numeric(prop)) src = ZPROP_SRC_DEFAULT; vdev_prop_add_list(outnvl, propname, strval, diff --git a/module/zfs/vdev_removal.c b/module/zfs/vdev_removal.c index 9ed1e6763a44..9f51a5e0535e 100644 --- a/module/zfs/vdev_removal.c +++ b/module/zfs/vdev_removal.c @@ -171,9 +171,20 @@ static void vdev_activate(vdev_t *vd) { metaslab_group_t *mg = vd->vdev_mg; - metaslab_group_activate(mg); + spa_t *spa = vd->vdev_spa; + ASSERT(!vd->vdev_islog); + ASSERT(vd->vdev_noalloc); + + metaslab_group_activate(mg); metaslab_group_activate(vd->vdev_log_mg); + + ASSERT3U(spa->spa_nonallocating_dspace, >=, spa_deflate(spa) ? + vd->vdev_stat.vs_dspace : vd->vdev_stat.vs_space); + + spa->spa_nonallocating_dspace -= spa_deflate(spa) ? + vd->vdev_stat.vs_dspace : vd->vdev_stat.vs_space; + vd->vdev_noalloc = B_FALSE; } @@ -1208,6 +1219,15 @@ vdev_remove_complete(spa_t *spa) zfs_dbgmsg("finishing device removal for vdev %llu in txg %llu", vd->vdev_id, txg); + ASSERT3U(0, <, spa_deflate(spa) ? + vd->vdev_stat.vs_dspace : vd->vdev_stat.vs_space); + ASSERT3U(spa->spa_nonallocating_dspace, >=, spa_deflate(spa) ? + vd->vdev_stat.vs_dspace : vd->vdev_stat.vs_space); + + /* the vdev is no longer part of the dspace */ + spa->spa_nonallocating_dspace -= spa_deflate(spa) ? + vd->vdev_stat.vs_dspace : vd->vdev_stat.vs_space; + /* * Discard allocation state. */ @@ -1632,7 +1652,7 @@ vdev_prop_noalloc(vdev_t *vd) spa_t *spa = vd->vdev_spa; objset_t *mos = spa->spa_meta_objset; uint64_t objid; - uint64_t noalloc = 0; + uint64_t allocating = 1; int err = 0; ASSERT(vd != NULL); @@ -1652,8 +1672,8 @@ vdev_prop_noalloc(vdev_t *vd) mutex_enter(&spa->spa_props_lock); - err = zap_lookup(mos, objid, vdev_prop_to_name(VDEV_PROP_NOALLOC), - sizeof (uint64_t), 1, &noalloc); + err = zap_lookup(mos, objid, vdev_prop_to_name(VDEV_PROP_ALLOCATING), + sizeof (uint64_t), 1, &allocating); mutex_exit(&spa->spa_props_lock); @@ -1661,7 +1681,7 @@ vdev_prop_noalloc(vdev_t *vd) return (B_FALSE); } - return (noalloc > 0); + return (allocating == 0); } /* ARGSUSED */ @@ -2025,6 +2045,11 @@ spa_vdev_remove_top_check(vdev_t *vd) if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REMOVAL)) return (SET_ERROR(ENOTSUP)); + /* + * This device is already being removed + */ + if (vd->vdev_removing) + return (SET_ERROR(EALREADY)); metaslab_class_t *mc = vd->vdev_mg->mg_class; metaslab_class_t *normal = spa_normal_class(spa); @@ -2047,13 +2072,15 @@ spa_vdev_remove_top_check(vdev_t *vd) /* available space in the pool's normal class */ uint64_t available = dsl_dir_space_available( spa->spa_dsl_pool->dp_root_dir, NULL, 0, B_TRUE); - if (available < - vd->vdev_stat.vs_dspace + spa_get_slop_space(spa)) { + /* non-allocating vdevs have already been taken out of space */ + if (!vd->vdev_noalloc) + available -= vd->vdev_stat.vs_dspace; + if (available < spa_get_slop_space(spa)) { /* - * This is a normal device. There has to be enough free - * space to remove the device and leave double the - * "slop" space (i.e. we must leave at least 3% of the - * pool free, in addition to the normal slop space). + * There has to be enough free space to remove the + * device and leave double the "slop" space (i.e. + * we must leave at least 3% of the pool free, in + * addition to the normal slop space). */ return (SET_ERROR(ENOSPC)); } @@ -2158,7 +2185,7 @@ spa_vdev_alloc(spa_t *spa, uint64_t guid) error = SET_ERROR(ENOENT); else if (vd->vdev_mg == NULL) error = SET_ERROR(ENOTSUP); - else + else if (!vd->vdev_removing) vdev_activate(vd); if (error == 0) { @@ -2177,6 +2204,8 @@ vdev_passivate(vdev_t *vd, uint64_t *txg) spa_t *spa = vd->vdev_spa; int error; + ASSERT(!vd->vdev_noalloc); + vdev_t *rvd = spa->spa_root_vdev; metaslab_group_t *mg = vd->vdev_mg; metaslab_class_t *normal = spa_normal_class(spa); @@ -2236,6 +2265,8 @@ vdev_passivate(vdev_t *vd, uint64_t *txg) return (error); } + spa->spa_nonallocating_dspace += spa_deflate(spa) ? + vd->vdev_stat.vs_dspace : vd->vdev_stat.vs_space; vd->vdev_noalloc = B_TRUE; return (0); @@ -2334,7 +2365,7 @@ spa_vdev_noalloc(spa_t *spa, uint64_t guid) { vdev_t *vd; uint64_t txg; - int error; + int error = 0; ASSERT(!MUTEX_HELD(&spa_namespace_lock)); ASSERT(spa_writeable(spa)); @@ -2349,7 +2380,7 @@ spa_vdev_noalloc(spa_t *spa, uint64_t guid) error = SET_ERROR(ENOENT); else if (vd->vdev_mg == NULL) error = SET_ERROR(ENOTSUP); - else + else if (!vd->vdev_noalloc) error = vdev_passivate(vd, &txg); if (error == 0) { diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c index b4e434b94440..9708e41f57cf 100644 --- a/module/zfs/zfs_ioctl.c +++ b/module/zfs/zfs_ioctl.c @@ -2990,8 +2990,8 @@ zfs_ioc_pool_get_props(zfs_cmd_t *zc) * outnvl: propname -> error code (int32) */ static const zfs_ioc_key_t zfs_keys_vdev_set_props[] = { - {ZPOOL_VDEV_SET_PROPS_VDEV, DATA_TYPE_UINT64, 0}, - {ZPOOL_VDEV_SET_PROPS_PROPS, DATA_TYPE_NVLIST, 0} + {ZPOOL_VDEV_PROPS_SET_VDEV, DATA_TYPE_UINT64, 0}, + {ZPOOL_VDEV_PROPS_SET_PROPS, DATA_TYPE_NVLIST, 0} }; static int @@ -3003,7 +3003,7 @@ zfs_ioc_vdev_set_props(const char *poolname, nvlist_t *innvl, nvlist_t *outnvl) uint64_t vdev_guid; /* Early validation */ - if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_SET_PROPS_VDEV, + if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_SET_VDEV, &vdev_guid) != 0) return (SET_ERROR(EINVAL)); @@ -3033,8 +3033,8 @@ zfs_ioc_vdev_set_props(const char *poolname, nvlist_t *innvl, nvlist_t *outnvl) * outnvl: propname -> value */ static const zfs_ioc_key_t zfs_keys_vdev_get_props[] = { - {ZPOOL_VDEV_GET_PROPS_VDEV, DATA_TYPE_UINT64, 0}, - {ZPOOL_VDEV_GET_PROPS_PROPS, DATA_TYPE_NVLIST, ZK_OPTIONAL} + {ZPOOL_VDEV_PROPS_GET_VDEV, DATA_TYPE_UINT64, 0}, + {ZPOOL_VDEV_PROPS_GET_PROPS, DATA_TYPE_NVLIST, ZK_OPTIONAL} }; static int @@ -3046,7 +3046,7 @@ zfs_ioc_vdev_get_props(const char *poolname, nvlist_t *innvl, nvlist_t *outnvl) uint64_t vdev_guid; /* Early validation */ - if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_GET_PROPS_VDEV, + if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_GET_VDEV, &vdev_guid) != 0) return (SET_ERROR(EINVAL));