Fix for hotplug issues

ZED relies on udev to match vdev guids when device is removed. However,
udev does not contain the correct blkid information for the vdev due to
which the vdev is failed to match when detached and we have to rely
fault handler to make device unavailable. This PR allow vdev to trigger
a Disk Change event whenever a new vdev is added to sync blkid
information with udev.

This PR also change the device state to REMOVED whenever the device is
unplugged instead of UNAVAIL.

When vdev is unplugged, zfs_retire_recv() is not able to match

Signed-off-by: Ameer Hamza <ahamza@ixsystems.com>

cmd/zed/agents/zfs_retire.c

@@ -323,6 +323,9 @@ zfs_retire_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
 	boolean_t is_disk;
 	vdev_aux_t aux;
 	uint64_t state = 0;
+	int l2arc = 0;
+	vdev_stat_t *vs;
+	unsigned int c;
 
 	fmd_hdl_debug(hdl, "zfs_retire_recv: '%s'", class);
 
@@ -351,13 +354,33 @@ zfs_retire_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
 
 		devname = zpool_vdev_name(NULL, zhp, vdev, B_FALSE);
 
-		/* Can't replace l2arc with a spare: offline the device */
-		if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
-		    &devtype) == 0 && strcmp(devtype, VDEV_TYPE_L2CACHE) == 0) {
-			fmd_hdl_debug(hdl, "zpool_vdev_offline '%s'", devname);
-			zpool_vdev_offline(zhp, devname, B_TRUE);
-		} else if (!fmd_prop_get_int32(hdl, "spare_on_remove") ||
-		    replace_with_spare(hdl, zhp, vdev) == B_FALSE) {
+		nvlist_lookup_uint64_array(vdev, ZPOOL_CONFIG_VDEV_STATS,
+		    (uint64_t **)&vs, &c);
+
+		/*
+		 * If state removed is requested for already removed vdev,
+		 * its a loopback event from spa_async_remove(). Just
+		 * ignore it.
+		 */
+		if ((vs->vs_state == VDEV_STATE_REMOVED) &&
+		    (state == VDEV_STATE_REMOVED)) {
+			return;
+		}
+
+		l2arc = (nvlist_lookup_string(nvl,
+		    FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE, &devtype) == 0 &&
+		    strcmp(devtype, VDEV_TYPE_L2CACHE) == 0);
+
+		/* Remove the vdev since device is unplugged */
+		if (l2arc || (strcmp(class, "resource.fs.zfs.removed") == 0)) {
+			int status = zpool_vdev_remove_wanted(zhp, devname);
+			fmd_hdl_debug(hdl, "zpool_vdev_remove_wanted '%s'"
+			    ", ret:%d", devname, status);
+		}
+
+		/* Replace the vdev with a spare if its not a l2arc */
+		if (!l2arc && (!fmd_prop_get_int32(hdl, "spare_on_remove") ||
+		    replace_with_spare(hdl, zhp, vdev) == B_FALSE)) {
 			/* Could not handle with spare */
 			fmd_hdl_debug(hdl, "no spare for '%s'", devname);
 		}

cmd/ztest.c

@@ -1165,7 +1165,7 @@ ztest_kill(ztest_shared_t *zs)
 	 * See comment above spa_write_cachefile().
 	 */
 	mutex_enter(&spa_namespace_lock);
-	spa_write_cachefile(ztest_spa, B_FALSE, B_FALSE);
+	spa_write_cachefile(ztest_spa, B_FALSE, B_FALSE, B_FALSE);
 	mutex_exit(&spa_namespace_lock);
 
 	(void) raise(SIGKILL);

include/libzfs.h

@@ -308,6 +308,7 @@ _LIBZFS_H int zpool_vdev_indirect_size(zpool_handle_t *, const char *,
     uint64_t *);
 _LIBZFS_H int zpool_vdev_split(zpool_handle_t *, char *, nvlist_t **,
     nvlist_t *, splitflags_t);
+_LIBZFS_H int zpool_vdev_remove_wanted(zpool_handle_t *, const char *);
 
 _LIBZFS_H int zpool_vdev_fault(zpool_handle_t *, uint64_t, vdev_aux_t);
 _LIBZFS_H int zpool_vdev_degrade(zpool_handle_t *, uint64_t, vdev_aux_t);

include/os/linux/Makefile.am

@@ -71,6 +71,7 @@ kernel_spl_sys_HEADERS = \
 	%D%/spl/sys/kmem_cache.h \
 	%D%/spl/sys/kstat.h \
 	%D%/spl/sys/list.h \
+	%D%/spl/sys/misc_spl.h \
 	%D%/spl/sys/mod_os.h \
 	%D%/spl/sys/mutex.h \
 	%D%/spl/sys/param.h \

include/os/linux/spl/sys/misc_spl.h

@@ -0,0 +1,30 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or https://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+#ifndef _OS_LINUX_SPL_MISC_H
+#define	_OS_LINUX_SPL_MISC_H
+
+#include <linux/kobject.h>
+#include <linux/genhd.h>
+
+extern void spl_signal_kobj_evt(struct block_device *bdev);
+
+#endif

include/sys/spa.h

@@ -838,7 +838,7 @@ extern kmutex_t spa_namespace_lock;
 #define	SPA_CONFIG_UPDATE_POOL	0
 #define	SPA_CONFIG_UPDATE_VDEVS	1
 
-extern void spa_write_cachefile(spa_t *, boolean_t, boolean_t);
+extern void spa_write_cachefile(spa_t *, boolean_t, boolean_t, boolean_t);
 extern void spa_config_load(void);
 extern nvlist_t *spa_all_configs(uint64_t *);
 extern void spa_config_set(spa_t *spa, nvlist_t *config);

include/sys/vdev.h

@@ -148,6 +148,7 @@ extern int vdev_degrade(spa_t *spa, uint64_t guid, vdev_aux_t aux);
 extern int vdev_online(spa_t *spa, uint64_t guid, uint64_t flags,
     vdev_state_t *);
 extern int vdev_offline(spa_t *spa, uint64_t guid, uint64_t flags);
+extern int vdev_remove_wanted(spa_t *spa, uint64_t guid);
 extern void vdev_clear(spa_t *spa, vdev_t *vd);
 
 extern boolean_t vdev_is_dead(vdev_t *vd);
@@ -190,6 +191,8 @@ typedef enum vdev_config_flag {
 	VDEV_CONFIG_MISSING = 1 << 4
 } vdev_config_flag_t;
 
+extern void vdev_post_kobj_evt(vdev_t *vd);
+extern void vdev_clear_kobj_evt(vdev_t *vd);
 extern void vdev_top_config_generate(spa_t *spa, nvlist_t *config);
 extern nvlist_t *vdev_config_generate(spa_t *spa, vdev_t *vd,
     boolean_t getstats, vdev_config_flag_t flags);

include/sys/vdev_impl.h

@@ -69,6 +69,7 @@ extern uint32_t zfs_vdev_async_write_max_active;
  * Virtual device operations
  */
 typedef int	vdev_init_func_t(spa_t *spa, nvlist_t *nv, void **tsd);
+typedef void	vdev_kobj_post_evt_func_t(vdev_t *vd);
 typedef void	vdev_fini_func_t(vdev_t *vd);
 typedef int	vdev_open_func_t(vdev_t *vd, uint64_t *size, uint64_t *max_size,
     uint64_t *ashift, uint64_t *pshift);
@@ -123,6 +124,7 @@ typedef const struct vdev_ops {
 	vdev_config_generate_func_t	*vdev_op_config_generate;
 	vdev_nparity_func_t		*vdev_op_nparity;
 	vdev_ndisks_func_t		*vdev_op_ndisks;
+	vdev_kobj_post_evt_func_t	*vdev_op_kobj_evt_post;
 	char				vdev_op_type[16];
 	boolean_t			vdev_op_leaf;
 } vdev_ops_t;
@@ -436,6 +438,7 @@ struct vdev {
 	boolean_t	vdev_isl2cache;	/* was a l2cache device		*/
 	boolean_t	vdev_copy_uberblocks;  /* post expand copy uberblocks */
 	boolean_t	vdev_resilver_deferred;  /* resilver deferred */
+	boolean_t	vdev_kobj_flag; /* kobj event record */
 	vdev_queue_t	vdev_queue;	/* I/O deadline schedule queue	*/
 	vdev_cache_t	vdev_cache;	/* physical block cache		*/
 	spa_aux_vdev_t	*vdev_aux;	/* for l2cache and spares vdevs	*/

include/sys/zfs_context.h

@@ -50,6 +50,7 @@ extern "C" {
 #include <sys/kmem.h>
 #include <sys/kmem_cache.h>
 #include <sys/vmem.h>
+#include <sys/misc_spl.h>
 #include <sys/taskq.h>
 #include <sys/param.h>
 #include <sys/disp.h>

lib/libzfs/libzfs.abi

@@ -548,6 +548,7 @@
     <elf-symbol name='zpool_vdev_path_to_guid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_vdev_remove' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_vdev_remove_cancel' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zpool_vdev_remove_wanted' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_vdev_split' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_wait_status' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -2302,6 +2303,7 @@
       <underlying-type type-id='9cac1fee'/>
       <enumerator name='ZPROP_CONT' value='-2'/>
       <enumerator name='ZPROP_INVAL' value='-1'/>
+      <enumerator name='ZPROP_USERPROP' value='-1'/>
       <enumerator name='ZFS_PROP_TYPE' value='0'/>
       <enumerator name='ZFS_PROP_CREATION' value='1'/>
       <enumerator name='ZFS_PROP_USED' value='2'/>
@@ -3150,6 +3152,7 @@
     <enum-decl name='vdev_prop_t' naming-typedef-id='5aa5c90c' id='1573bec8'>
       <underlying-type type-id='9cac1fee'/>
       <enumerator name='VDEV_PROP_INVAL' value='-1'/>
+      <enumerator name='VDEV_PROP_USERPROP' value='-1'/>
       <enumerator name='VDEV_PROP_NAME' value='0'/>
       <enumerator name='VDEV_PROP_CAPACITY' value='1'/>
       <enumerator name='VDEV_PROP_STATE' value='2'/>
@@ -3485,6 +3488,11 @@
       <parameter type-id='c19b74c3' name='istmp'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='zpool_vdev_remove_wanted' mangled-name='zpool_vdev_remove_wanted' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_remove_wanted'>
+      <parameter type-id='4c81de99' name='zhp'/>
+      <parameter type-id='80f4b756' name='path'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='zpool_vdev_fault' mangled-name='zpool_vdev_fault' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_fault'>
       <parameter type-id='4c81de99' name='zhp'/>
       <parameter type-id='9c313c2d' name='guid'/>
@@ -3750,7 +3758,7 @@
     </class-decl>
     <typedef-decl name='sendflags_t' type-id='f6aa15be' id='945467e6'/>
     <typedef-decl name='snapfilter_cb_t' type-id='d2a5e211' id='3d3ffb69'/>
-    <class-decl name='recvflags' size-in-bits='416' is-struct='yes' visibility='default' id='34a384dc'>
+    <class-decl name='recvflags' size-in-bits='448' is-struct='yes' visibility='default' id='34a384dc'>
       <data-member access='public' layout-offset-in-bits='0'>
         <var-decl name='verbose' type-id='c19b74c3' visibility='default'/>
       </data-member>
@@ -3790,6 +3798,9 @@
       <data-member access='public' layout-offset-in-bits='384'>
         <var-decl name='forceunmount' type-id='c19b74c3' visibility='default'/>
       </data-member>
+      <data-member access='public' layout-offset-in-bits='416'>
+        <var-decl name='heal' type-id='c19b74c3' visibility='default'/>
+      </data-member>
     </class-decl>
     <typedef-decl name='recvflags_t' type-id='34a384dc' id='9e59d1d4'/>
     <pointer-type-def type-id='f20fbd51' size-in-bits='64' id='a3681dea'/>
@@ -3903,16 +3914,17 @@
       <enumerator name='ZPOOL_ERRATA_ZOL_8308_ENCRYPTION' value='4'/>
     </enum-decl>
     <typedef-decl name='zpool_errata_t' type-id='d9abbf54' id='688c495b'/>
+    <pointer-type-def type-id='80f4b756' size-in-bits='64' id='7d3cd834'/>
     <pointer-type-def type-id='688c495b' size-in-bits='64' id='cec6f2e4'/>
     <function-decl name='zpool_get_status' mangled-name='zpool_get_status' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_status'>
       <parameter type-id='4c81de99' name='zhp'/>
-      <parameter type-id='9b23c9ad' name='msgid'/>
+      <parameter type-id='7d3cd834' name='msgid'/>
       <parameter type-id='cec6f2e4' name='errata'/>
       <return type-id='d3dd6294'/>
     </function-decl>
     <function-decl name='zpool_import_status' mangled-name='zpool_import_status' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_import_status'>
       <parameter type-id='5ce45b60' name='config'/>
-      <parameter type-id='9b23c9ad' name='msgid'/>
+      <parameter type-id='7d3cd834' name='msgid'/>
       <parameter type-id='cec6f2e4' name='errata'/>
       <return type-id='d3dd6294'/>
     </function-decl>
@@ -4032,8 +4044,8 @@
       <return type-id='48b5725f'/>
     </function-decl>
     <function-decl name='libzfs_envvar_is_set' mangled-name='libzfs_envvar_is_set' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_envvar_is_set'>
-      <parameter type-id='26a90f95' name='envvar'/>
-      <return type-id='95e97e5e'/>
+      <parameter type-id='80f4b756' name='envvar'/>
+      <return type-id='c19b74c3'/>
     </function-decl>
     <function-decl name='libzfs_init' mangled-name='libzfs_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='libzfs_init'>
       <return type-id='b0382bb3'/>
@@ -4102,15 +4114,15 @@
       <return type-id='95e97e5e'/>
     </function-decl>
     <function-decl name='color_start' mangled-name='color_start' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='color_start'>
-      <parameter type-id='26a90f95' name='color'/>
+      <parameter type-id='80f4b756' name='color'/>
       <return type-id='48b5725f'/>
     </function-decl>
     <function-decl name='color_end' mangled-name='color_end' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='color_end'>
       <return type-id='48b5725f'/>
     </function-decl>
     <function-decl name='printf_color' mangled-name='printf_color' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='printf_color'>
-      <parameter type-id='26a90f95' name='color'/>
-      <parameter type-id='26a90f95' name='format'/>
+      <parameter type-id='80f4b756' name='color'/>
+      <parameter type-id='80f4b756' name='format'/>
       <parameter is-variadic='yes'/>
       <return type-id='95e97e5e'/>
     </function-decl>
@@ -4123,7 +4135,7 @@
   <abi-instr address-size='64' path='lib/libzfs/os/linux/libzfs_mount_os.c' language='LANG_C99'>
     <pointer-type-def type-id='7359adad' size-in-bits='64' id='1d2c2b85'/>
     <function-decl name='zfs_parse_mount_options' mangled-name='zfs_parse_mount_options' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_parse_mount_options'>
-      <parameter type-id='26a90f95' name='mntopts'/>
+      <parameter type-id='80f4b756' name='mntopts'/>
       <parameter type-id='1d2c2b85' name='mntflags'/>
       <parameter type-id='1d2c2b85' name='zfsflags'/>
       <parameter type-id='95e97e5e' name='sloppy'/>
@@ -4771,8 +4783,8 @@
     </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='module/zcommon/zfeature_common.c' language='LANG_C99'>
-    <array-type-def dimensions='1' type-id='83f29ca2' size-in-bits='16576' id='9d5e9e2e'>
-      <subrange length='37' type-id='7359adad' id='ae666bde'/>
+    <array-type-def dimensions='1' type-id='83f29ca2' size-in-bits='16576' id='d95b2b0b'>
+      <subrange length='37' type-id='7359adad' id='aa6426fb'/>
     </array-type-def>
     <enum-decl name='spa_feature' id='33ecb627'>
       <underlying-type type-id='9cac1fee'/>
@@ -4872,7 +4884,7 @@
     <qualified-type-def type-id='3eee3342' const='yes' id='0c1d5bbb'/>
     <pointer-type-def type-id='0c1d5bbb' size-in-bits='64' id='a3372543'/>
     <pointer-type-def type-id='d6618c78' size-in-bits='64' id='a8425263'/>
-    <var-decl name='spa_feature_table' type-id='9d5e9e2e' mangled-name='spa_feature_table' visibility='default' elf-symbol-id='spa_feature_table'/>
+    <var-decl name='spa_feature_table' type-id='d95b2b0b' mangled-name='spa_feature_table' visibility='default' elf-symbol-id='spa_feature_table'/>
     <var-decl name='zfeature_checks_disable' type-id='c19b74c3' mangled-name='zfeature_checks_disable' visibility='default' elf-symbol-id='zfeature_checks_disable'/>
     <function-decl name='zfeature_is_valid_guid' mangled-name='zfeature_is_valid_guid' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfeature_is_valid_guid'>
       <parameter type-id='80f4b756' name='name'/>
@@ -4935,7 +4947,7 @@
     </function-decl>
     <function-decl name='zfs_special_devs' mangled-name='zfs_special_devs' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_special_devs'>
       <parameter type-id='5ce45b60' name='nv'/>
-      <parameter type-id='26a90f95' name='type'/>
+      <parameter type-id='80f4b756' name='type'/>
       <return type-id='c19b74c3'/>
     </function-decl>
     <function-decl name='zpool_get_load_policy' mangled-name='zpool_get_load_policy' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_load_policy'>
@@ -5013,7 +5025,7 @@
     <typedef-decl name='zfs_deleg_note_t' type-id='729d4547' id='4613c173'/>
     <class-decl name='zfs_deleg_perm_tab' size-in-bits='128' is-struct='yes' visibility='default' id='5aa05c1f'>
       <data-member access='public' layout-offset-in-bits='0'>
-        <var-decl name='z_perm' type-id='26a90f95' visibility='default'/>
+        <var-decl name='z_perm' type-id='80f4b756' visibility='default'/>
       </data-member>
       <data-member access='public' layout-offset-in-bits='64'>
         <var-decl name='z_note' type-id='4613c173' visibility='default'/>
@@ -5455,7 +5467,6 @@
       </data-member>
     </class-decl>
     <typedef-decl name='zprop_desc_t' type-id='bbff5e4b' id='ffa52b96'/>
-    <pointer-type-def type-id='80f4b756' size-in-bits='64' id='7d3cd834'/>
     <qualified-type-def type-id='64636ce3' const='yes' id='072f7953'/>
     <pointer-type-def type-id='072f7953' size-in-bits='64' id='c8bc397b'/>
     <pointer-type-def type-id='ffa52b96' size-in-bits='64' id='76c8174b'/>

lib/libzfs/libzfs_pool.c

@@ -3073,6 +3073,39 @@ zpool_vdev_offline(zpool_handle_t *zhp, const char *path, boolean_t istmp)
 	}
 }
 
+/*
+ * Remove the specified vdev. Called from zed on udev remove event.
+ */
+int
+zpool_vdev_remove_wanted(zpool_handle_t *zhp, const char *path)
+{
+	zfs_cmd_t zc = {"\0"};
+	char errbuf[ERRBUFLEN];
+	nvlist_t *tgt;
+	boolean_t avail_spare, l2cache;
+	libzfs_handle_t *hdl = zhp->zpool_hdl;
+
+	(void) snprintf(errbuf, sizeof (errbuf),
+	    dgettext(TEXT_DOMAIN, "cannot remove %s"), path);
+
+	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
+	if ((tgt = zpool_find_vdev(zhp, path, &avail_spare, &l2cache,
+	    NULL)) == NULL)
+		return (zfs_error(hdl, EZFS_NODEVICE, errbuf));
+
+	zc.zc_guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);
+
+	if (avail_spare)
+		return (zfs_error(hdl, EZFS_ISSPARE, errbuf));
+
+	zc.zc_cookie = VDEV_STATE_REMOVED;
+
+	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
+		return (0);
+
+	return (zpool_standard_error(hdl, errno, errbuf));
+}
+
 /*
  * Mark the given vdev faulted.
  */