From 1b376d176ead7651ffde83d319edcc1bcc65da55 Mon Sep 17 00:00:00 2001 From: Matthew Macy Date: Wed, 5 Aug 2020 10:19:51 -0700 Subject: [PATCH] FreeBSD: Add support for lockless lookup Authored-by: mjg Reviewed-by: Ryan Moeller Signed-off-by: Matt Macy Closes #10657 --- include/os/freebsd/spl/sys/ccompat.h | 15 +++ include/os/freebsd/zfs/sys/zfs_znode_impl.h | 1 + module/os/freebsd/spl/spl_vfs.c | 3 + module/os/freebsd/zfs/zfs_acl.c | 3 + module/os/freebsd/zfs/zfs_vfsops.c | 3 + module/os/freebsd/zfs/zfs_vnops.c | 58 +++++++++++ module/os/freebsd/zfs/zfs_znode.c | 105 +++++++++++++++++--- 7 files changed, 176 insertions(+), 12 deletions(-) diff --git a/include/os/freebsd/spl/sys/ccompat.h b/include/os/freebsd/spl/sys/ccompat.h index 6c87676354fc..59abe921dba9 100644 --- a/include/os/freebsd/spl/sys/ccompat.h +++ b/include/os/freebsd/spl/sys/ccompat.h @@ -55,6 +55,21 @@ #else #define getnewvnode_reserve_() getnewvnode_reserve(1) #endif + +#if __FreeBSD_version < 1300102 +#define ASSERT_VOP_IN_SEQC(zp) +#define MNTK_FPLOOKUP 0 +#define vn_seqc_write_begin(vp) +#define vn_seqc_write_end(vp) + +#ifndef VFS_SMR_DECLARE +#define VFS_SMR_DECLARE +#endif +#ifndef VFS_SMR_ZONE_SET +#define VFS_SMR_ZONE_SET(zone) +#endif +#endif + struct hlist_node { struct hlist_node *next, **pprev; }; diff --git a/include/os/freebsd/zfs/sys/zfs_znode_impl.h b/include/os/freebsd/zfs/sys/zfs_znode_impl.h index c0430467572c..b1fe4aa86816 100644 --- a/include/os/freebsd/zfs/sys/zfs_znode_impl.h +++ b/include/os/freebsd/zfs/sys/zfs_znode_impl.h @@ -99,6 +99,7 @@ extern minor_t zfsdev_minor_alloc(void); #define ZTOV(ZP) ((ZP)->z_vnode) #define ZTOI(ZP) ((ZP)->z_vnode) #define VTOZ(VP) ((struct znode *)(VP)->v_data) +#define VTOZ_SMR(VP) ((znode_t *)vn_load_v_data_smr(VP)) #define ITOZ(VP) ((struct znode *)(VP)->v_data) #define zhold(zp) vhold(ZTOV((zp))) #define zrele(zp) vrele(ZTOV((zp))) diff --git a/module/os/freebsd/spl/spl_vfs.c b/module/os/freebsd/spl/spl_vfs.c index 6d308adb43ff..991a11fe2baf 100644 --- a/module/os/freebsd/spl/spl_vfs.c +++ b/module/os/freebsd/spl/spl_vfs.c @@ -161,6 +161,7 @@ mount_snapshot(kthread_t *td, vnode_t **vpp, const char *fstype, char *fspath, vput(vp); return (error); } + vn_seqc_write_begin(vp); VOP_UNLOCK1(vp); /* @@ -213,6 +214,7 @@ mount_snapshot(kthread_t *td, vnode_t **vpp, const char *fstype, char *fspath, VI_LOCK(vp); vp->v_iflag &= ~VI_MOUNT; VI_UNLOCK(vp); + vn_seqc_write_end(vp); vput(vp); vfs_unbusy(mp); vfs_freeopts(mp->mnt_optnew); @@ -248,6 +250,7 @@ mount_snapshot(kthread_t *td, vnode_t **vpp, const char *fstype, char *fspath, vfs_event_signal(NULL, VQ_MOUNT, 0); if (VFS_ROOT(mp, LK_EXCLUSIVE, &mvp)) panic("mount: lost mount"); + vn_seqc_write_end(vp); VOP_UNLOCK1(vp); #if __FreeBSD_version >= 1300048 vfs_op_exit(mp); diff --git a/module/os/freebsd/zfs/zfs_acl.c b/module/os/freebsd/zfs/zfs_acl.c index 07fa42fc819b..1c1a053b5280 100644 --- a/module/os/freebsd/zfs/zfs_acl.c +++ b/module/os/freebsd/zfs/zfs_acl.c @@ -1144,6 +1144,7 @@ zfs_acl_chown_setattr(znode_t *zp) if (zp->z_zfsvfs->z_replay == B_FALSE) ASSERT_VOP_ELOCKED(ZTOV(zp), __func__); ASSERT(MUTEX_HELD(&zp->z_acl_lock)); + ASSERT_VOP_IN_SEQC(ZTOV(zp)); if ((error = zfs_acl_node_read(zp, B_TRUE, &aclp, B_FALSE)) == 0) zp->z_mode = zfs_mode_compute(zp->z_mode, aclp, @@ -1171,6 +1172,8 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx) int count = 0; zfs_acl_phys_t acl_phys; + ASSERT_VOP_IN_SEQC(ZTOV(zp)); + mode = zp->z_mode; mode = zfs_mode_compute(mode, aclp, &zp->z_pflags, diff --git a/module/os/freebsd/zfs/zfs_vfsops.c b/module/os/freebsd/zfs/zfs_vfsops.c index cf397ca297a6..78968ed8d08b 100644 --- a/module/os/freebsd/zfs/zfs_vfsops.c +++ b/module/os/freebsd/zfs/zfs_vfsops.c @@ -1192,6 +1192,9 @@ zfs_domount(vfs_t *vfsp, char *osname) vfsp->mnt_kern_flag |= MNTK_NOMSYNC; vfsp->mnt_kern_flag |= MNTK_VMSETSIZE_BUG; +#if defined(_KERNEL) && !defined(KMEM_DEBUG) + vfsp->mnt_kern_flag |= MNTK_FPLOOKUP; +#endif /* * The fsid is 64 bits, composed of an 8-bit fs type, which * separates our fsid from any other filesystem types, and a diff --git a/module/os/freebsd/zfs/zfs_vnops.c b/module/os/freebsd/zfs/zfs_vnops.c index 742b29506660..a87235111942 100644 --- a/module/os/freebsd/zfs/zfs_vnops.c +++ b/module/os/freebsd/zfs/zfs_vnops.c @@ -39,6 +39,9 @@ #include #include #include +#if __FreeBSD_version >= 1300102 +#include +#endif #include #include #include @@ -91,6 +94,8 @@ #define VN_OPEN_INVFS 0x0 #endif +VFS_SMR_DECLARE; + #if __FreeBSD_version >= 1300047 #define vm_page_wire_lock(pp) #define vm_page_wire_unlock(pp) @@ -3932,6 +3937,7 @@ zfs_rename_(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp, char *snm = scnp->cn_nameptr; char *tnm = tcnp->cn_nameptr; int error = 0; + bool want_seqc_end __maybe_unused = false; /* Reject renames across filesystems. */ if ((*svpp)->v_mount != tdvp->v_mount || @@ -4075,6 +4081,15 @@ zfs_rename_(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp, } } + vn_seqc_write_begin(*svpp); + vn_seqc_write_begin(sdvp); + if (*tvpp != NULL) + vn_seqc_write_begin(*tvpp); + if (tdvp != *tvpp) + vn_seqc_write_begin(tdvp); +#if __FreeBSD_version >= 1300102 + want_seqc_end = true; +#endif vnevent_rename_src(*svpp, sdvp, scnp->cn_nameptr, ct); if (tzp) vnevent_rename_dest(*tvpp, tdvp, tnm, ct); @@ -4161,10 +4176,20 @@ zfs_rename_(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp, unlockout: /* all 4 vnodes are locked, ZFS_ENTER called */ ZFS_EXIT(zfsvfs); + if (want_seqc_end) { + vn_seqc_write_end(*svpp); + vn_seqc_write_end(sdvp); + if (*tvpp != NULL) + vn_seqc_write_end(*tvpp); + if (tdvp != *tvpp) + vn_seqc_write_end(tdvp); + want_seqc_end = false; + } VOP_UNLOCK1(*svpp); VOP_UNLOCK1(sdvp); out: /* original two vnodes are locked */ + MPASS(!want_seqc_end); if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) zil_commit(zilog, 0); @@ -5196,6 +5221,33 @@ zfs_freebsd_write(struct vop_write_args *ap) ap->a_cred)); } +#if __FreeBSD_version >= 1300102 +/* + * VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see + * the comment above cache_fplookup for details. + */ +static int +zfs_freebsd_fplookup_vexec(struct vop_fplookup_vexec_args *v) +{ + vnode_t *vp; + znode_t *zp; + uint64_t pflags; + + vp = v->a_vp; + zp = VTOZ_SMR(vp); + if (__predict_false(zp == NULL)) + return (EAGAIN); + pflags = atomic_load_64(&zp->z_pflags); + if (pflags & ZFS_AV_QUARANTINED) + return (EAGAIN); + if (pflags & ZFS_XATTR) + return (EAGAIN); + if ((pflags & ZFS_NO_EXECS_DENIED) == 0) + return (EAGAIN); + return (0); +} +#endif + #ifndef _SYS_SYSPROTO_H_ struct vop_access_args { struct vnode *a_vp; @@ -6483,6 +6535,9 @@ struct vop_vector zfs_vnodeops = { .vop_need_inactive = zfs_freebsd_need_inactive, #endif .vop_reclaim = zfs_freebsd_reclaim, +#if __FreeBSD_version >= 1300102 + .vop_fplookup_vexec = zfs_freebsd_fplookup_vexec, +#endif .vop_access = zfs_freebsd_access, .vop_allocate = VOP_EINVAL, .vop_lookup = zfs_cache_lookup, @@ -6537,6 +6592,9 @@ VFS_VOP_VECTOR_REGISTER(zfs_vnodeops); struct vop_vector zfs_fifoops = { .vop_default = &fifo_specops, .vop_fsync = zfs_freebsd_fsync, +#if __FreeBSD_version >= 1300102 + .vop_fplookup_vexec = zfs_freebsd_fplookup_vexec, +#endif .vop_access = zfs_freebsd_access, .vop_getattr = zfs_freebsd_getattr, .vop_inactive = zfs_freebsd_inactive, diff --git a/module/os/freebsd/zfs/zfs_znode.c b/module/os/freebsd/zfs/zfs_znode.c index a44870612152..76e24b1bdf51 100644 --- a/module/os/freebsd/zfs/zfs_znode.c +++ b/module/os/freebsd/zfs/zfs_znode.c @@ -97,7 +97,13 @@ SYSCTL_INT(_debug_sizeof, OID_AUTO, znode, CTLFLAG_RD, */ krwlock_t zfsvfs_lock; +#if defined(_KERNEL) && !defined(KMEM_DEBUG) && \ + __FreeBSD_version >= 1300102 +#define _ZFS_USE_SMR +static uma_zone_t znode_uma_zone; +#else static kmem_cache_t *znode_cache = NULL; +#endif extern struct vop_vector zfs_vnodeops; extern struct vop_vector zfs_fifoops; @@ -169,6 +175,53 @@ zfs_znode_cache_destructor(void *buf, void *arg) ASSERT(zp->z_acl_cached == NULL); } + +#ifdef _ZFS_USE_SMR +VFS_SMR_DECLARE; + +static int +zfs_znode_cache_constructor_smr(void *mem, int size __unused, void *private, + int flags) +{ + + return (zfs_znode_cache_constructor(mem, private, flags)); +} + +static void +zfs_znode_cache_destructor_smr(void *mem, int size __unused, void *private) +{ + + zfs_znode_cache_destructor(mem, private); +} + +void +zfs_znode_init(void) +{ + /* + * Initialize zcache + */ + rw_init(&zfsvfs_lock, NULL, RW_DEFAULT, NULL); + ASSERT(znode_uma_zone == NULL); + znode_uma_zone = uma_zcreate("zfs_znode_cache", + sizeof (znode_t), zfs_znode_cache_constructor_smr, + zfs_znode_cache_destructor_smr, NULL, NULL, 0, 0); + VFS_SMR_ZONE_SET(znode_uma_zone); +} + +static znode_t * +zfs_znode_alloc_kmem(int flags) +{ + + return (uma_zalloc_smr(znode_uma_zone, flags)); +} + +static void +zfs_znode_free_kmem(znode_t *zp) +{ + + uma_zfree_smr(znode_uma_zone, zp); +} +#else void zfs_znode_init(void) { @@ -180,18 +233,40 @@ zfs_znode_init(void) znode_cache = kmem_cache_create("zfs_znode_cache", sizeof (znode_t), 0, zfs_znode_cache_constructor, zfs_znode_cache_destructor, NULL, NULL, NULL, 0); - // kmem_cache_set_move(znode_cache, zfs_znode_move); } +static znode_t * +zfs_znode_alloc_kmem(int flags) +{ + + return (kmem_cache_alloc(znode_cache, flags)); +} + +static void +zfs_znode_free_kmem(znode_t *zp) +{ + + kmem_cache_free(znode_cache, zp); +} +#endif + void zfs_znode_fini(void) { /* * Cleanup zcache */ - if (znode_cache) +#ifdef _ZFS_USE_SMR + if (znode_uma_zone) { + uma_zdestroy(znode_uma_zone); + znode_uma_zone = NULL; + } +#else + if (znode_cache) { kmem_cache_destroy(znode_cache); - znode_cache = NULL; + znode_cache = NULL; + } +#endif rw_destroy(&zfsvfs_lock); } @@ -211,7 +286,7 @@ zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx) vattr.va_uid = crgetuid(kcred); vattr.va_gid = crgetgid(kcred); - sharezp = kmem_cache_alloc(znode_cache, KM_SLEEP); + sharezp = zfs_znode_alloc_kmem(KM_SLEEP); ASSERT(!POINTER_IS_VALID(sharezp->z_zfsvfs)); sharezp->z_moved = 0; sharezp->z_unlinked = 0; @@ -230,7 +305,7 @@ zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx) zfs_acl_ids_free(&acl_ids); sa_handle_destroy(sharezp->z_sa_hdl); - kmem_cache_free(znode_cache, sharezp); + zfs_znode_free_kmem(sharezp); return (error); } @@ -349,7 +424,12 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz, int count = 0; int error; - zp = kmem_cache_alloc(znode_cache, KM_SLEEP); + zp = zfs_znode_alloc_kmem(KM_SLEEP); + +#ifndef _ZFS_USE_SMR + KASSERT((zfsvfs->z_parent->z_vfs->mnt_kern_flag & MNTK_FPLOOKUP) == 0, + ("%s: fast path lookup enabled without smr", __func__)); +#endif #if __FreeBSD_version >= 1300076 KASSERT(curthread->td_vp_reserved != NULL, @@ -360,7 +440,7 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz, #endif error = getnewvnode("zfs", zfsvfs->z_parent->z_vfs, &zfs_vnodeops, &vp); if (error != 0) { - kmem_cache_free(znode_cache, zp); + zfs_znode_free_kmem(zp); return (NULL); } zp->z_vnode = vp; @@ -416,7 +496,7 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz, sa_handle_destroy(zp->z_sa_hdl); zfs_vnode_forget(vp); zp->z_vnode = NULL; - kmem_cache_free(znode_cache, zp); + zfs_znode_free_kmem(zp); return (NULL); } @@ -763,6 +843,8 @@ zfs_xvattr_set(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx) xoap = xva_getxoptattr(xvap); ASSERT(xoap); + ASSERT_VOP_IN_SEQC(ZTOV(zp)); + if (XVA_ISSET_REQ(xvap, XAT_CREATETIME)) { uint64_t times[2]; ZFS_TIME_ENCODE(&xoap->xoa_createtime, times); @@ -1191,8 +1273,7 @@ zfs_znode_free(znode_t *zp) zp->z_acl_cached = NULL; } - kmem_cache_free(znode_cache, zp); - + zfs_znode_free_kmem(zp); } void @@ -1628,7 +1709,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP); - rootzp = kmem_cache_alloc(znode_cache, KM_SLEEP); + rootzp = zfs_znode_alloc_kmem(KM_SLEEP); ASSERT(!POINTER_IS_VALID(rootzp->z_zfsvfs)); rootzp->z_moved = 0; rootzp->z_unlinked = 0; @@ -1672,7 +1753,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) POINTER_INVALIDATE(&rootzp->z_zfsvfs); sa_handle_destroy(rootzp->z_sa_hdl); - kmem_cache_free(znode_cache, rootzp); + zfs_znode_free_kmem(rootzp); /* * Create shares directory