From 4a21ec056016bb5738721543af933a8eed2308fb Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Mon, 6 Apr 2020 19:07:35 -0400 Subject: [PATCH 001/133] ZTS: Fix non-portable date format The delegate tests use `date(1)` to generate snapshot names, using the format '%F-%T-%N' to get nanosecond resolution (since multiple snapshots may be taken in the same second). '%N' is not portable, and causes tests to fail on FreeBSD. Since the only purpose these timestamps serve is to create a unique name, simply use $RANDOM instead. Reviewed-by: John Kennedy Reviewed-by: Brian Behlendorf Signed-off-by: Ryan Moeller Closes #10170 --- .../delegate/delegate_common.kshlib | 34 +++++++++---------- .../functional/delegate/zfs_allow_009_neg.ksh | 1 - 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib b/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib index 073a39f835f..e39b015b21b 100644 --- a/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib +++ b/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib @@ -379,7 +379,7 @@ function verify_send typeset dtst=$3 typeset oldval - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset snap=$dtst@snap.$stamp typeset -i ret=1 @@ -408,7 +408,7 @@ function verify_fs_receive typeset fs=$3 typeset dtst - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset newfs=$fs/newfs.$stamp typeset newvol=$fs/newvol.$stamp typeset bak_user=$TEST_BASE_DIR/bak.$user.$stamp @@ -480,7 +480,7 @@ function verify_userprop typeset perm=$2 typeset dtst=$3 - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM user_run $user zfs set "$user:ts=$stamp" $dtst zpool sync ${dtst%%/*} @@ -565,7 +565,7 @@ function verify_fs_create typeset perm=$2 typeset fs=$3 - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset newfs=$fs/nfs.$stamp typeset newvol=$fs/nvol.$stamp @@ -693,7 +693,7 @@ function verify_fs_snapshot typeset perm=$2 typeset fs=$3 - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset snap=$fs@snap.$stamp typeset mntpt=$(get_prop mountpoint $fs) @@ -737,7 +737,7 @@ function verify_fs_rollback typeset fs=$3 typeset oldval - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset snap=$fs@snap.$stamp typeset mntpt=$(get_prop mountpoint $fs) @@ -770,7 +770,7 @@ function verify_fs_clone typeset perm=$2 typeset fs=$3 - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset basefs=${fs%/*} typeset snap=$fs@snap.$stamp typeset clone=$basefs/cfs.$stamp @@ -815,7 +815,7 @@ function verify_fs_rename typeset perm=$2 typeset fs=$3 - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset basefs=${fs%/*} typeset snap=$fs@snap.$stamp typeset renamefs=$basefs/nfs.$stamp @@ -898,7 +898,7 @@ function verify_fs_mount typeset perm=$2 typeset fs=$3 - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset mntpt=$(get_prop mountpoint $fs) typeset newmntpt=$TEST_BASE_DIR/mnt.$stamp @@ -966,7 +966,7 @@ function verify_fs_mountpoint typeset perm=$2 typeset fs=$3 - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset mntpt=$(get_prop mountpoint $fs) typeset newmntpt=$TEST_BASE_DIR/mnt.$stamp @@ -1005,7 +1005,7 @@ function verify_fs_promote typeset perm=$2 typeset fs=$3 - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset basefs=${fs%/*} typeset snap=$fs@snap.$stamp typeset clone=$basefs/cfs.$stamp @@ -1061,7 +1061,7 @@ function verify_fs_canmount typeset fs=$3 typeset oldval - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM if ! ismounted $fs ; then set -A modes "on" "off" @@ -1372,7 +1372,7 @@ function verify_vol_snapshot typeset perm=$2 typeset vol=$3 - typeset stamp=${perm}.${user}.$(date +'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset basevol=${vol%/*} typeset snap=$vol@snap.$stamp @@ -1397,7 +1397,7 @@ function verify_vol_rollback typeset perm=$2 typeset vol=$3 - typeset stamp=${perm}.${user}.$(date+'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset basevol=${vol%/*} typeset snap=$vol@snap.$stamp @@ -1432,7 +1432,7 @@ function verify_vol_clone typeset perm=$2 typeset vol=$3 - typeset stamp=${perm}.${user}.$(date+'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset basevol=${vol%/*} typeset snap=$vol@snap.$stamp typeset clone=$basevol/cvol.$stamp @@ -1478,7 +1478,7 @@ function verify_vol_rename typeset perm=$2 typeset vol=$3 - typeset stamp=${perm}.${user}.$(date+'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset basevol=${vol%/*} typeset snap=$vol@snap.$stamp typeset clone=$basevol/cvol.$stamp @@ -1525,7 +1525,7 @@ function verify_vol_promote typeset perm=$2 typeset vol=$3 - typeset stamp=${perm}.${user}.$(date+'%F-%T-%N') + typeset stamp=${perm}.${user}.$RANDOM typeset basevol=${vol%/*} typeset snap=$vol@snap.$stamp typeset clone=$basevol/cvol.$stamp diff --git a/tests/zfs-tests/tests/functional/delegate/zfs_allow_009_neg.ksh b/tests/zfs-tests/tests/functional/delegate/zfs_allow_009_neg.ksh index 45fdb5b8569..a6f12244ce2 100755 --- a/tests/zfs-tests/tests/functional/delegate/zfs_allow_009_neg.ksh +++ b/tests/zfs-tests/tests/functional/delegate/zfs_allow_009_neg.ksh @@ -51,7 +51,6 @@ longset="set123456789012345678901234567890123456789012345678901234567890123" for dtst in $DATASETS ; do log_mustnot eval "zfs allow -s @$longset $dtst" # Create non-existent permission set - typeset timestamp=$(date +'%F-%R:%S') log_mustnot zfs allow -s @non-existent $dtst log_mustnot zfs allow $STAFF "atime,created,mounted" $dtst log_mustnot zfs allow $dtst $TESTPOOL From 2a15c6aab40e854808f2d910ee998cd405769dfa Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 8 Apr 2020 01:04:40 +0800 Subject: [PATCH 002/133] libzfs_pool: Remove unused check for ENOTBLK Commit 379ca9c removed the check on aux devices to be block devices also changing zfs_ioctl(hdl, ZFS_IOC_VDEV_ADD, ...) and zfs_ioctl(hdl, ZFS_IOC_POOL_CREATE, ...) to never set ENOTBLK. This change removes the dangling check for ENOTBLK that will never trigger. Reviewed-by: Brian Behlendorf Reported-by: Richard Elling Signed-off-by: Alex John Closes #10173 --- lib/libzfs/libzfs_pool.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/libzfs/libzfs_pool.c b/lib/libzfs/libzfs_pool.c index e0610a631e0..06c85f145f8 100644 --- a/lib/libzfs/libzfs_pool.c +++ b/lib/libzfs/libzfs_pool.c @@ -1351,11 +1351,6 @@ zpool_create(libzfs_handle_t *hdl, const char *pool, nvlist_t *nvroot, "one or more devices is out of space")); return (zfs_error(hdl, EZFS_BADDEV, msg)); - case ENOTBLK: - zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "cache device must be a disk or disk slice")); - return (zfs_error(hdl, EZFS_BADDEV, msg)); - default: return (zpool_standard_error(hdl, errno, msg)); } @@ -1543,12 +1538,6 @@ zpool_add(zpool_handle_t *zhp, nvlist_t *nvroot) (void) zfs_error(hdl, EZFS_BADVERSION, msg); break; - case ENOTBLK: - zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "cache device must be a disk or disk slice")); - (void) zfs_error(hdl, EZFS_BADDEV, msg); - break; - default: (void) zpool_standard_error(hdl, errno, msg); } From 7e3df9db128722143734a9459771365ea19c1c40 Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Tue, 7 Apr 2020 13:06:22 -0400 Subject: [PATCH 003/133] Finish refactoring for ZFS_MODULE_PARAM_CALL Linux and FreeBSD have different parameters for tunable proc handler. This has prevented FreeBSD from implementing the ZFS_MODULE_PARAM_CALL macro. To complete the sharing of ZFS_MODULE_PARAM_CALL declarations, create per-platform definitions of the parameter list, ZFS_MODULE_PARAM_ARGS. With the declarations wired up we discovered an incorrect scope prefix for spa_slop_shift, so this is now fixed. Reviewed-by: Brian Behlendorf Signed-off-by: Ryan Moeller Closes #10179 --- include/os/linux/kernel/linux/mod_compat.h | 4 +++- include/sys/arc_impl.h | 4 ++-- include/sys/spa.h | 8 ++++---- include/sys/zfs_context.h | 1 + module/zfs/spa_misc.c | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/include/os/linux/kernel/linux/mod_compat.h b/include/os/linux/kernel/linux/mod_compat.h index a0f618d14c1..1a53c66e522 100644 --- a/include/os/linux/kernel/linux/mod_compat.h +++ b/include/os/linux/kernel/linux/mod_compat.h @@ -134,12 +134,14 @@ enum scope_prefix_types { * spa_slop_shift */ /* BEGIN CSTYLED */ -#define ZFS_MODULE_PARAM_CALL(scope_prefix, name_prefix, name, setfunc, getfunc, perm, desc) \ +#define ZFS_MODULE_PARAM_CALL(scope_prefix, name_prefix, name, setfunc, getfunc, perm, desc) \ CTASSERT_GLOBAL((sizeof (scope_prefix) == sizeof (enum scope_prefix_types))); \ module_param_call(name_prefix ## name, setfunc, getfunc, &name_prefix ## name, perm); \ MODULE_PARM_DESC(name_prefix ## name, desc) /* END CSTYLED */ +#define ZFS_MODULE_PARAM_ARGS const char *buf, zfs_kernel_param_t *kp + #define ZFS_MODULE_DESCRIPTION(s) MODULE_DESCRIPTION(s) #define ZFS_MODULE_AUTHOR(s) MODULE_AUTHOR(s) #define ZFS_MODULE_LICENSE(s) MODULE_LICENSE(s) diff --git a/include/sys/arc_impl.h b/include/sys/arc_impl.h index c55640f8ba4..d4167945f42 100644 --- a/include/sys/arc_impl.h +++ b/include/sys/arc_impl.h @@ -614,8 +614,8 @@ extern uint64_t arc_free_memory(void); extern int64_t arc_available_memory(void); extern void arc_tuning_update(void); -extern int param_set_arc_long(const char *buf, zfs_kernel_param_t *kp); -extern int param_set_arc_int(const char *buf, zfs_kernel_param_t *kp); +extern int param_set_arc_long(ZFS_MODULE_PARAM_ARGS); +extern int param_set_arc_int(ZFS_MODULE_PARAM_ARGS); #ifdef __cplusplus } diff --git a/include/sys/spa.h b/include/sys/spa.h index 780119e9af9..e1fab3c6977 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -1179,10 +1179,10 @@ extern void spa_notify_waiters(spa_t *spa); extern void spa_wake_waiters(spa_t *spa); /* module param call functions */ -int param_set_deadman_ziotime(const char *val, zfs_kernel_param_t *kp); -int param_set_deadman_synctime(const char *val, zfs_kernel_param_t *kp); -int param_set_slop_shift(const char *buf, zfs_kernel_param_t *kp); -int param_set_deadman_failmode(const char *val, zfs_kernel_param_t *kp); +int param_set_deadman_ziotime(ZFS_MODULE_PARAM_ARGS); +int param_set_deadman_synctime(ZFS_MODULE_PARAM_ARGS); +int param_set_slop_shift(ZFS_MODULE_PARAM_ARGS); +int param_set_deadman_failmode(ZFS_MODULE_PARAM_ARGS); #ifdef ZFS_DEBUG #define dprintf_bp(bp, fmt, ...) do { \ diff --git a/include/sys/zfs_context.h b/include/sys/zfs_context.h index 39c778b1e04..00642175d6b 100644 --- a/include/sys/zfs_context.h +++ b/include/sys/zfs_context.h @@ -198,6 +198,7 @@ typedef struct zfs_kernel_param { } zfs_kernel_param_t; #define ZFS_MODULE_PARAM(scope_prefix, name_prefix, name, type, perm, desc) +#define ZFS_MODULE_PARAM_ARGS void #define ZFS_MODULE_PARAM_CALL(scope_prefix, name_prefix, name, setfunc, \ getfunc, perm, desc) diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index 6f0783468c4..6a43071119e 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -2891,5 +2891,5 @@ ZFS_MODULE_PARAM(zfs, zfs_, special_class_metadata_reserve_pct, INT, ZMOD_RW, "free space available"); /* END CSTYLED */ -ZFS_MODULE_PARAM_CALL(spa, spa_, slop_shift, param_set_slop_shift, +ZFS_MODULE_PARAM_CALL(zfs_spa, spa_, slop_shift, param_set_slop_shift, param_get_int, ZMOD_RW, "Reserved free space in pool"); From 01c4f2bf2933319e876e8f695e97d0719a9db0ed Mon Sep 17 00:00:00 2001 From: Matthew Macy Date: Wed, 8 Apr 2020 10:30:27 -0700 Subject: [PATCH 004/133] Use vn_io_fault_uiomove on FreeBSD to avoid potential deadlock Added to prevent a possible deadlock, the following comments from FreeBSD explain the issue. The comment describing vn_io_fault_uiomove: /* * Helper function to perform the requested uiomove operation using * the held pages for io->uio_iov[0].iov_base buffer instead of * copyin/copyout. Access to the pages with uiomove_fromphys() * instead of iov_base prevents page faults that could occur due to * pmap_collect() invalidating the mapping created by * vm_fault_quick_hold_pages(), or pageout daemon, page laundry or * object cleanup revoking the write access from page mappings. * * Filesystems specified MNTK_NO_IOPF shall use vn_io_fault_uiomove() * instead of plain uiomove(). */ This used for vn_io_fault which has the following motivation: /* * The vn_io_fault() is a wrapper around vn_read() and vn_write() to * prevent the following deadlock: * * Assume that the thread A reads from the vnode vp1 into userspace * buffer buf1 backed by the pages of vnode vp2. If a page in buf1 is * currently not resident, then system ends up with the call chain * vn_read() -> VOP_READ(vp1) -> uiomove() -> [Page Fault] -> * vm_fault(buf1) -> vnode_pager_getpages(vp2) -> VOP_GETPAGES(vp2) * which establishes lock order vp1->vn_lock, then vp2->vn_lock. * If, at the same time, thread B reads from vnode vp2 into buffer buf2 * backed by the pages of vnode vp1, and some page in buf2 is not * resident, we get a reversed order vp2->vn_lock, then vp1->vn_lock. * * To prevent the lock order reversal and deadlock, vn_io_fault() does * not allow page faults to happen during VOP_READ() or VOP_WRITE(). * Instead, it first tries to do the whole range i/o with pagefaults * disabled. If all pages in the i/o buffer are resident and mapped, * VOP will succeed (ignoring the genuine filesystem errors). * Otherwise, we get back EFAULT, and vn_io_fault() falls back to do * i/o in chunks, with all pages in the chunk prefaulted and held * using vm_fault_quick_hold_pages(). * * Filesystems using this deadlock avoidance scheme should use the * array of the held pages from uio, saved in the curthread->td_ma, * instead of doing uiomove(). A helper function * vn_io_fault_uiomove() converts uiomove request into * uiomove_fromphys() over td_ma array. * * Since vnode locks do not cover the whole i/o anymore, rangelocks * make the current i/o request atomic with respect to other i/os and * truncations. */ Reviewed-by: Brian Behlendorf Signed-off-by: Matt Macy Closes #10177 --- module/zfs/dmu.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index aa392b177c8..6eb935720f4 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -1359,8 +1359,13 @@ dmu_read_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size) XUIOSTAT_BUMP(xuiostat_rbuf_copied); } else #endif +#ifdef __FreeBSD__ + err = vn_io_fault_uiomove((char *)db->db_data + bufoff, + tocpy, uio); +#else err = uiomove((char *)db->db_data + bufoff, tocpy, UIO_READ, uio); +#endif if (err) break; @@ -1459,9 +1464,13 @@ dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx) * to lock the pages in memory, so that uiomove won't * block. */ +#ifdef __FreeBSD__ + err = vn_io_fault_uiomove((char *)db->db_data + bufoff, + tocpy, uio); +#else err = uiomove((char *)db->db_data + bufoff, tocpy, UIO_WRITE, uio); - +#endif if (tocpy == db->db_size) dmu_buf_fill_done(db, tx); From 68dde63d138882860109178f4ab0b7b1b1bb3cfd Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Thu, 9 Apr 2020 09:16:46 -0700 Subject: [PATCH 005/133] Linux 5.7 compat: blk_alloc_queue() Commit https://github.com/torvalds/linux/commit/3d745ea5 simplified the blk_alloc_queue() interface by updating it to take the request queue as an argument. Add a wrapper function which accepts the new arguments and internally uses the available interfaces. Other minor changes include increasing the Linux-Maximum to 5.6 now that 5.6 has been released. It was not bumped to 5.7 because this release has not yet been finalized and is still subject to change. Added local 'struct zvol_state_os *zso' variable to zvol_alloc. Reviewed-by: George Melikov Reviewed-by: Tony Hutter Signed-off-by: Brian Behlendorf Closes #10181 Closes #10187 --- META | 2 +- config/kernel-make-request-fn.m4 | 67 +++++++++++++------ include/os/linux/kernel/linux/blkdev_compat.h | 14 ++++ module/os/linux/zfs/zvol_os.c | 48 ++++++------- 4 files changed, 87 insertions(+), 44 deletions(-) diff --git a/META b/META index 39828e0a109..49d8bd73d51 100644 --- a/META +++ b/META @@ -6,5 +6,5 @@ Release: 1 Release-Tags: relext License: CDDL Author: OpenZFS on Linux -Linux-Maximum: 5.4 +Linux-Maximum: 5.6 Linux-Minimum: 3.10 diff --git a/config/kernel-make-request-fn.m4 b/config/kernel-make-request-fn.m4 index 5183176f58b..609926c1b79 100644 --- a/config/kernel-make-request-fn.m4 +++ b/config/kernel-make-request-fn.m4 @@ -17,38 +17,67 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MAKE_REQUEST_FN], [ ],[ blk_queue_make_request(NULL, &make_request); ]) + + ZFS_LINUX_TEST_SRC([blk_alloc_queue_request_fn], [ + #include + blk_qc_t make_request(struct request_queue *q, + struct bio *bio) { return (BLK_QC_T_NONE); } + ],[ + struct request_queue *q __attribute__ ((unused)); + q = blk_alloc_queue(make_request, NUMA_NO_NODE); + ]) ]) AC_DEFUN([ZFS_AC_KERNEL_MAKE_REQUEST_FN], [ dnl # - dnl # Linux 3.2 API Change - dnl # make_request_fn returns void. + dnl # Linux 5.7 API Change + dnl # blk_alloc_queue() expects request function. dnl # - AC_MSG_CHECKING([whether make_request_fn() returns void]) - ZFS_LINUX_TEST_RESULT([make_request_fn_void], [ + AC_MSG_CHECKING([whether blk_alloc_queue() expects request function]) + ZFS_LINUX_TEST_RESULT([blk_alloc_queue_request_fn], [ AC_MSG_RESULT(yes) - AC_DEFINE(MAKE_REQUEST_FN_RET, void, + + dnl # Checked as part of the blk_alloc_queue_request_fn test + AC_MSG_CHECKING([whether make_request_fn() returns blk_qc_t]) + AC_MSG_RESULT(yes) + + AC_DEFINE(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN, 1, + [blk_alloc_queue() expects request function]) + AC_DEFINE(MAKE_REQUEST_FN_RET, blk_qc_t, [make_request_fn() return type]) - AC_DEFINE(HAVE_MAKE_REQUEST_FN_RET_VOID, 1, - [Noting that make_request_fn() returns void]) + AC_DEFINE(HAVE_MAKE_REQUEST_FN_RET_QC, 1, + [Noting that make_request_fn() returns blk_qc_t]) ],[ - AC_MSG_RESULT(no) - dnl # - dnl # Linux 4.4 API Change - dnl # make_request_fn returns blk_qc_t. + dnl # Linux 3.2 API Change + dnl # make_request_fn returns void. dnl # - AC_MSG_CHECKING( - [whether make_request_fn() returns blk_qc_t]) - ZFS_LINUX_TEST_RESULT([make_request_fn_blk_qc_t], [ + AC_MSG_CHECKING([whether make_request_fn() returns void]) + ZFS_LINUX_TEST_RESULT([make_request_fn_void], [ AC_MSG_RESULT(yes) - AC_DEFINE(MAKE_REQUEST_FN_RET, blk_qc_t, + AC_DEFINE(MAKE_REQUEST_FN_RET, void, [make_request_fn() return type]) - AC_DEFINE(HAVE_MAKE_REQUEST_FN_RET_QC, 1, - [Noting that make_request_fn() ] - [returns blk_qc_t]) + AC_DEFINE(HAVE_MAKE_REQUEST_FN_RET_VOID, 1, + [Noting that make_request_fn() returns void]) ],[ - ZFS_LINUX_TEST_ERROR([make_request_fn]) + AC_MSG_RESULT(no) + + dnl # + dnl # Linux 4.4 API Change + dnl # make_request_fn returns blk_qc_t. + dnl # + AC_MSG_CHECKING( + [whether make_request_fn() returns blk_qc_t]) + ZFS_LINUX_TEST_RESULT([make_request_fn_blk_qc_t], [ + AC_MSG_RESULT(yes) + AC_DEFINE(MAKE_REQUEST_FN_RET, blk_qc_t, + [make_request_fn() return type]) + AC_DEFINE(HAVE_MAKE_REQUEST_FN_RET_QC, 1, + [Noting that make_request_fn() ] + [returns blk_qc_t]) + ],[ + ZFS_LINUX_TEST_ERROR([make_request_fn]) + ]) ]) ]) ]) diff --git a/include/os/linux/kernel/linux/blkdev_compat.h b/include/os/linux/kernel/linux/blkdev_compat.h index 3ba14ef4045..b46a6e40afe 100644 --- a/include/os/linux/kernel/linux/blkdev_compat.h +++ b/include/os/linux/kernel/linux/blkdev_compat.h @@ -487,4 +487,18 @@ blk_generic_end_io_acct(struct request_queue *q, int rw, #endif } +static inline struct request_queue * +blk_generic_alloc_queue(make_request_fn make_request, int node_id) +{ +#if defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN) + return (blk_alloc_queue(make_request, node_id)); +#else + struct request_queue *q = blk_alloc_queue(GFP_KERNEL); + if (q != NULL) + blk_queue_make_request(q, make_request); + + return (q); +#endif +} + #endif /* _ZFS_BLKDEV_H */ diff --git a/module/os/linux/zfs/zvol_os.c b/module/os/linux/zfs/zvol_os.c index 9439954b8ac..fe45d76a62a 100644 --- a/module/os/linux/zfs/zvol_os.c +++ b/module/os/linux/zfs/zvol_os.c @@ -754,6 +754,7 @@ static zvol_state_t * zvol_alloc(dev_t dev, const char *name) { zvol_state_t *zv; + struct zvol_state_os *zso; uint64_t volmode; if (dsl_prop_get_integer(name, "volmode", &volmode, NULL) != 0) @@ -766,39 +767,38 @@ zvol_alloc(dev_t dev, const char *name) return (NULL); zv = kmem_zalloc(sizeof (zvol_state_t), KM_SLEEP); - zv->zv_zso = kmem_zalloc(sizeof (struct zvol_state_os), KM_SLEEP); + zso = kmem_zalloc(sizeof (struct zvol_state_os), KM_SLEEP); + zv->zv_zso = zso; list_link_init(&zv->zv_next); - mutex_init(&zv->zv_state_lock, NULL, MUTEX_DEFAULT, NULL); - zv->zv_zso->zvo_queue = blk_alloc_queue(GFP_ATOMIC); - if (zv->zv_zso->zvo_queue == NULL) + zso->zvo_queue = blk_generic_alloc_queue(zvol_request, NUMA_NO_NODE); + if (zso->zvo_queue == NULL) goto out_kmem; - blk_queue_make_request(zv->zv_zso->zvo_queue, zvol_request); - blk_queue_set_write_cache(zv->zv_zso->zvo_queue, B_TRUE, B_TRUE); + blk_queue_set_write_cache(zso->zvo_queue, B_TRUE, B_TRUE); /* Limit read-ahead to a single page to prevent over-prefetching. */ - blk_queue_set_read_ahead(zv->zv_zso->zvo_queue, 1); + blk_queue_set_read_ahead(zso->zvo_queue, 1); /* Disable write merging in favor of the ZIO pipeline. */ - blk_queue_flag_set(QUEUE_FLAG_NOMERGES, zv->zv_zso->zvo_queue); + blk_queue_flag_set(QUEUE_FLAG_NOMERGES, zso->zvo_queue); - zv->zv_zso->zvo_disk = alloc_disk(ZVOL_MINORS); - if (zv->zv_zso->zvo_disk == NULL) + zso->zvo_disk = alloc_disk(ZVOL_MINORS); + if (zso->zvo_disk == NULL) goto out_queue; - zv->zv_zso->zvo_queue->queuedata = zv; - zv->zv_zso->zvo_dev = dev; + zso->zvo_queue->queuedata = zv; + zso->zvo_dev = dev; zv->zv_open_count = 0; strlcpy(zv->zv_name, name, MAXNAMELEN); zfs_rangelock_init(&zv->zv_rangelock, NULL, NULL); rw_init(&zv->zv_suspend_lock, NULL, RW_DEFAULT, NULL); - zv->zv_zso->zvo_disk->major = zvol_major; - zv->zv_zso->zvo_disk->events = DISK_EVENT_MEDIA_CHANGE; + zso->zvo_disk->major = zvol_major; + zso->zvo_disk->events = DISK_EVENT_MEDIA_CHANGE; if (volmode == ZFS_VOLMODE_DEV) { /* @@ -808,27 +808,27 @@ zvol_alloc(dev_t dev, const char *name) * and suppresses partition scanning (GENHD_FL_NO_PART_SCAN) * setting gendisk->flags accordingly. */ - zv->zv_zso->zvo_disk->minors = 1; + zso->zvo_disk->minors = 1; #if defined(GENHD_FL_EXT_DEVT) - zv->zv_zso->zvo_disk->flags &= ~GENHD_FL_EXT_DEVT; + zso->zvo_disk->flags &= ~GENHD_FL_EXT_DEVT; #endif #if defined(GENHD_FL_NO_PART_SCAN) - zv->zv_zso->zvo_disk->flags |= GENHD_FL_NO_PART_SCAN; + zso->zvo_disk->flags |= GENHD_FL_NO_PART_SCAN; #endif } - zv->zv_zso->zvo_disk->first_minor = (dev & MINORMASK); - zv->zv_zso->zvo_disk->fops = &zvol_ops; - zv->zv_zso->zvo_disk->private_data = zv; - zv->zv_zso->zvo_disk->queue = zv->zv_zso->zvo_queue; - snprintf(zv->zv_zso->zvo_disk->disk_name, DISK_NAME_LEN, "%s%d", + zso->zvo_disk->first_minor = (dev & MINORMASK); + zso->zvo_disk->fops = &zvol_ops; + zso->zvo_disk->private_data = zv; + zso->zvo_disk->queue = zso->zvo_queue; + snprintf(zso->zvo_disk->disk_name, DISK_NAME_LEN, "%s%d", ZVOL_DEV_NAME, (dev & MINORMASK)); return (zv); out_queue: - blk_cleanup_queue(zv->zv_zso->zvo_queue); + blk_cleanup_queue(zso->zvo_queue); out_kmem: - kmem_free(zv->zv_zso, sizeof (struct zvol_state_os)); + kmem_free(zso, sizeof (struct zvol_state_os)); kmem_free(zv, sizeof (zvol_state_t)); return (NULL); } From 8b27e08ed8c6d854ee3c398773407dfac24d7578 Mon Sep 17 00:00:00 2001 From: Matthew Macy Date: Thu, 9 Apr 2020 09:59:31 -0700 Subject: [PATCH 006/133] Add separate field for indicating that spa is in middle of split By default it's not possible to open a device already owned by an active vdev. It's necessary to make an exception to this for vdev split. The FreeBSD platform code will make an exception if spa_is splitting is set to to true. Reviewed-by: Brian Behlendorf Reviewed-by: Ryan Moeller Signed-off-by: Matt Macy Closes #10178 --- include/sys/spa_impl.h | 1 + module/zfs/spa.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/include/sys/spa_impl.h b/include/sys/spa_impl.h index e8806dda906..6481d539779 100644 --- a/include/sys/spa_impl.h +++ b/include/sys/spa_impl.h @@ -216,6 +216,7 @@ struct spa { spa_load_state_t spa_load_state; /* current load operation */ boolean_t spa_indirect_vdevs_loaded; /* mappings loaded? */ boolean_t spa_trust_config; /* do we trust vdev tree? */ + boolean_t spa_is_splitting; /* in the middle of a split? */ spa_config_source_t spa_config_source; /* where config comes from? */ uint64_t spa_import_flags; /* import specific flags */ spa_taskqs_t spa_zio_taskq[ZIO_TYPES][ZIO_TASKQ_TYPES]; diff --git a/module/zfs/spa.c b/module/zfs/spa.c index 72a54ebcef2..0d4646c1516 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -7477,6 +7477,7 @@ spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, list_destroy(&vd_trim_list); newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT; + newspa->spa_is_splitting = B_TRUE; /* create the new pool from the disks of the original pool */ error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE); @@ -7554,6 +7555,7 @@ spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, spa_history_log_internal(newspa, "split", NULL, "from pool %s", spa_name(spa)); + newspa->spa_is_splitting = B_FALSE; kmem_free(vml, children * sizeof (vdev_t *)); /* if we're not going to mount the filesystems in userland, export */ From 36a6e2335c45212f2609269bcee3004908ac6bcb Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Thu, 9 Apr 2020 18:39:48 -0400 Subject: [PATCH 007/133] Don't ignore zfs_arc_max below allmem/32 Set arc_c_min before arc_c_max so that when zfs_arc_min is set lower than the default allmem/32 zfs_arc_max can also be set lower. Add warning messages when tunables are being ignored. Reviewed-by: Brian Behlendorf Signed-off-by: Ryan Moeller Closes #10157 Closes #10158 --- include/sys/arc_impl.h | 2 +- module/os/linux/zfs/arc_os.c | 4 ++-- module/zfs/arc.c | 40 +++++++++++++++++++++++++----------- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/include/sys/arc_impl.h b/include/sys/arc_impl.h index d4167945f42..2468a4aac75 100644 --- a/include/sys/arc_impl.h +++ b/include/sys/arc_impl.h @@ -612,7 +612,7 @@ extern void arc_prune_async(int64_t); extern int arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg); extern uint64_t arc_free_memory(void); extern int64_t arc_available_memory(void); -extern void arc_tuning_update(void); +extern void arc_tuning_update(boolean_t); extern int param_set_arc_long(ZFS_MODULE_PARAM_ARGS); extern int param_set_arc_int(ZFS_MODULE_PARAM_ARGS); diff --git a/module/os/linux/zfs/arc_os.c b/module/os/linux/zfs/arc_os.c index bff0f05179f..e34d4ae0808 100644 --- a/module/os/linux/zfs/arc_os.c +++ b/module/os/linux/zfs/arc_os.c @@ -377,7 +377,7 @@ param_set_arc_long(const char *buf, zfs_kernel_param_t *kp) if (error < 0) return (SET_ERROR(error)); - arc_tuning_update(); + arc_tuning_update(B_TRUE); return (0); } @@ -391,7 +391,7 @@ param_set_arc_int(const char *buf, zfs_kernel_param_t *kp) if (error < 0) return (SET_ERROR(error)); - arc_tuning_update(); + arc_tuning_update(B_TRUE); return (0); } diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 9c5ee9829ca..b5d17431ce2 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -4558,7 +4558,7 @@ arc_adjust_cb_check(void *arg, zthr_t *zthr) * their actual internal variable counterparts. Without this, * changing those module params at runtime would have no effect. */ - arc_tuning_update(); + arc_tuning_update(B_FALSE); /* * This is necessary in order to keep the kstat information @@ -6900,6 +6900,14 @@ arc_state_multilist_index_func(multilist_t *ml, void *obj) multilist_get_num_sublists(ml)); } +#define WARN_IF_TUNING_IGNORED(tuning, value, do_warn) do { \ + if ((do_warn) && (tuning) && ((tuning) != (value))) { \ + cmn_err(CE_WARN, \ + "ignoring tunable %s (using %llu instead)", \ + (#tuning), (value)); \ + } \ +} while (0) + /* * Called during module initialization and periodically thereafter to * apply reasonable changes to the exposed performance tunings. Can also be @@ -6908,11 +6916,20 @@ arc_state_multilist_index_func(multilist_t *ml, void *obj) * values will be applied. */ void -arc_tuning_update(void) +arc_tuning_update(boolean_t verbose) { uint64_t allmem = arc_all_memory(); unsigned long limit; + /* Valid range: 32M - */ + if ((zfs_arc_min) && (zfs_arc_min != arc_c_min) && + (zfs_arc_min >= 2ULL << SPA_MAXBLOCKSHIFT) && + (zfs_arc_min <= arc_c_max)) { + arc_c_min = zfs_arc_min; + arc_c = MAX(arc_c, arc_c_min); + } + WARN_IF_TUNING_IGNORED(zfs_arc_min, arc_c_min, verbose); + /* Valid range: 64M - */ if ((zfs_arc_max) && (zfs_arc_max != arc_c_max) && (zfs_arc_max >= 64 << 20) && (zfs_arc_max < allmem) && @@ -6925,14 +6942,7 @@ arc_tuning_update(void) if (arc_dnode_size_limit > arc_meta_limit) arc_dnode_size_limit = arc_meta_limit; } - - /* Valid range: 32M - */ - if ((zfs_arc_min) && (zfs_arc_min != arc_c_min) && - (zfs_arc_min >= 2ULL << SPA_MAXBLOCKSHIFT) && - (zfs_arc_min <= arc_c_max)) { - arc_c_min = zfs_arc_min; - arc_c = MAX(arc_c, arc_c_min); - } + WARN_IF_TUNING_IGNORED(zfs_arc_max, arc_c_max, verbose); /* Valid range: 16M - */ if ((zfs_arc_meta_min) && (zfs_arc_meta_min != arc_meta_min) && @@ -6944,6 +6954,7 @@ arc_tuning_update(void) if (arc_dnode_size_limit < arc_meta_min) arc_dnode_size_limit = arc_meta_min; } + WARN_IF_TUNING_IGNORED(zfs_arc_meta_min, arc_meta_min, verbose); /* Valid range: - */ limit = zfs_arc_meta_limit ? zfs_arc_meta_limit : @@ -6952,6 +6963,7 @@ arc_tuning_update(void) (limit >= arc_meta_min) && (limit <= arc_c_max)) arc_meta_limit = limit; + WARN_IF_TUNING_IGNORED(zfs_arc_meta_limit, arc_meta_limit, verbose); /* Valid range: - */ limit = zfs_arc_dnode_limit ? zfs_arc_dnode_limit : @@ -6960,6 +6972,8 @@ arc_tuning_update(void) (limit >= arc_meta_min) && (limit <= arc_meta_limit)) arc_dnode_size_limit = limit; + WARN_IF_TUNING_IGNORED(zfs_arc_dnode_limit, arc_dnode_size_limit, + verbose); /* Valid range: 1 - N */ if (zfs_arc_grow_retry) @@ -6989,11 +7003,13 @@ arc_tuning_update(void) if ((zfs_arc_lotsfree_percent >= 0) && (zfs_arc_lotsfree_percent <= 100)) arc_lotsfree_percent = zfs_arc_lotsfree_percent; + WARN_IF_TUNING_IGNORED(zfs_arc_lotsfree_percent, arc_lotsfree_percent, + verbose); /* Valid range: 0 - */ if ((zfs_arc_sys_free) && (zfs_arc_sys_free != arc_sys_free)) arc_sys_free = MIN(MAX(zfs_arc_sys_free, 0), allmem); - + WARN_IF_TUNING_IGNORED(zfs_arc_sys_free, arc_sys_free, verbose); } static void @@ -7183,7 +7199,7 @@ arc_init(void) arc_dnode_size_limit = (percent * arc_meta_limit) / 100; /* Apply user specified tunings */ - arc_tuning_update(); + arc_tuning_update(B_TRUE); /* if kmem_flags are set, lets try to use less memory */ if (kmem_debugging()) From 77f6826b83b7e27f0996f6d192202c36f65e41fd Mon Sep 17 00:00:00 2001 From: George Amanakis Date: Fri, 10 Apr 2020 13:33:35 -0400 Subject: [PATCH 008/133] Persistent L2ARC This commit makes the L2ARC persistent across reboots. We implement a light-weight persistent L2ARC metadata structure that allows L2ARC contents to be recovered after a reboot. This significantly eases the impact a reboot has on read performance on systems with large caches. Reviewed-by: Matthew Ahrens Reviewed-by: George Wilson Reviewed-by: Ryan Moeller Reviewed-by: Brian Behlendorf Co-authored-by: Saso Kiselkov Co-authored-by: Jorgen Lundman Co-authored-by: George Amanakis Ported-by: Yuxuan Shui Signed-off-by: George Amanakis Closes #925 Closes #1823 Closes #2672 Closes #3744 Closes #9582 --- cmd/zdb/zdb.c | 229 ++- configure.ac | 1 + include/sys/arc.h | 4 + include/sys/arc_impl.h | 305 +++- include/sys/fs/zfs.h | 5 + include/sys/spa.h | 1 + lib/libzfs/libzfs_import.c | 31 +- man/man5/zfs-module-parameters.5 | 51 +- man/man8/zdb.8 | 16 +- man/man8/zpool-labelclear.8 | 5 +- man/man8/zpoolconcepts.8 | 24 +- module/os/linux/zfs/zfs_sysfs.c | 3 +- module/zfs/arc.c | 1430 ++++++++++++++++- module/zfs/spa.c | 13 + module/zfs/vdev.c | 19 +- tests/runfiles/linux.run | 6 + tests/zfs-tests/include/tunables.cfg | 2 + tests/zfs-tests/tests/functional/Makefile.am | 1 + .../functional/persist_l2arc/Makefile.am | 15 + .../functional/persist_l2arc/cleanup.ksh | 31 + .../persist_l2arc/persist_l2arc.cfg | 37 + .../persist_l2arc/persist_l2arc_001_pos.ksh | 106 ++ .../persist_l2arc/persist_l2arc_002_pos.ksh | 112 ++ .../persist_l2arc/persist_l2arc_003_neg.ksh | 87 + .../persist_l2arc/persist_l2arc_004_pos.ksh | 101 ++ .../persist_l2arc/persist_l2arc_005_pos.ksh | 108 ++ .../persist_l2arc/persist_l2arc_006_pos.ksh | 98 ++ .../persist_l2arc/persist_l2arc_007_pos.ksh | 95 ++ .../persist_l2arc/persist_l2arc_008_pos.ksh | 143 ++ .../tests/functional/persist_l2arc/setup.ksh | 29 + 30 files changed, 3020 insertions(+), 88 deletions(-) create mode 100644 tests/zfs-tests/tests/functional/persist_l2arc/Makefile.am create mode 100755 tests/zfs-tests/tests/functional/persist_l2arc/cleanup.ksh create mode 100644 tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc.cfg create mode 100755 tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_001_pos.ksh create mode 100755 tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_002_pos.ksh create mode 100755 tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_003_neg.ksh create mode 100755 tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_004_pos.ksh create mode 100755 tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_005_pos.ksh create mode 100755 tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_006_pos.ksh create mode 100755 tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh create mode 100755 tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_008_pos.ksh create mode 100755 tests/zfs-tests/tests/functional/persist_l2arc/setup.ksh diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index e9e801b11a3..dab0d8b68f8 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -62,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -3474,6 +3475,216 @@ print_label_header(zdb_label_t *label, int l) label->header_printed = B_TRUE; } +static void +print_l2arc_header(void) +{ + (void) printf("------------------------------------\n"); + (void) printf("L2ARC device header\n"); + (void) printf("------------------------------------\n"); +} + +static void +print_l2arc_log_blocks(void) +{ + (void) printf("------------------------------------\n"); + (void) printf("L2ARC device log blocks\n"); + (void) printf("------------------------------------\n"); +} + +static void +dump_l2arc_log_entries(uint64_t log_entries, + l2arc_log_ent_phys_t *le, int i) +{ + for (int j = 0; j < log_entries; j++) { + dva_t dva = le[j].le_dva; + (void) printf("lb[%4d]\tle[%4d]\tDVA asize: %llu, " + "vdev: %llu, offset: %llu\n", i, j + 1, + (u_longlong_t)DVA_GET_ASIZE(&dva), + (u_longlong_t)DVA_GET_VDEV(&dva), + (u_longlong_t)DVA_GET_OFFSET(&dva)); + (void) printf("|\t\t\t\tbirth: %llu\n", + (u_longlong_t)le[j].le_birth); + (void) printf("|\t\t\t\tlsize: %llu\n", + (u_longlong_t)L2BLK_GET_LSIZE((&le[j])->le_prop)); + (void) printf("|\t\t\t\tpsize: %llu\n", + (u_longlong_t)L2BLK_GET_PSIZE((&le[j])->le_prop)); + (void) printf("|\t\t\t\tcompr: %llu\n", + (u_longlong_t)L2BLK_GET_COMPRESS((&le[j])->le_prop)); + (void) printf("|\t\t\t\ttype: %llu\n", + (u_longlong_t)L2BLK_GET_TYPE((&le[j])->le_prop)); + (void) printf("|\t\t\t\tprotected: %llu\n", + (u_longlong_t)L2BLK_GET_PROTECTED((&le[j])->le_prop)); + (void) printf("|\t\t\t\tprefetch: %llu\n", + (u_longlong_t)L2BLK_GET_PREFETCH((&le[j])->le_prop)); + (void) printf("|\t\t\t\taddress: %llu\n", + (u_longlong_t)le[j].le_daddr); + (void) printf("|\n"); + } + (void) printf("\n"); +} + +static void +dump_l2arc_log_blkptr(l2arc_log_blkptr_t lbps) +{ + (void) printf("|\t\tdaddr: %llu\n", (u_longlong_t)lbps.lbp_daddr); + (void) printf("|\t\tpayload_asize: %llu\n", + (u_longlong_t)lbps.lbp_payload_asize); + (void) printf("|\t\tpayload_start: %llu\n", + (u_longlong_t)lbps.lbp_payload_start); + (void) printf("|\t\tlsize: %llu\n", + (u_longlong_t)L2BLK_GET_LSIZE((&lbps)->lbp_prop)); + (void) printf("|\t\tpsize: %llu\n", + (u_longlong_t)L2BLK_GET_PSIZE((&lbps)->lbp_prop)); + (void) printf("|\t\tcompralgo: %llu\n", + (u_longlong_t)L2BLK_GET_COMPRESS((&lbps)->lbp_prop)); + (void) printf("|\t\tcksumalgo: %llu\n", + (u_longlong_t)L2BLK_GET_CHECKSUM((&lbps)->lbp_prop)); + (void) printf("|\n\n"); +} + +static void +dump_l2arc_log_blocks(int fd, l2arc_dev_hdr_phys_t l2dhdr) +{ + l2arc_log_blk_phys_t this_lb; + uint64_t psize; + l2arc_log_blkptr_t lbps[2]; + abd_t *abd; + zio_cksum_t cksum; + int i = 0, failed = 0; + l2arc_dev_t dev; + + print_l2arc_log_blocks(); + bcopy((&l2dhdr)->dh_start_lbps, lbps, sizeof (lbps)); + + dev.l2ad_evict = l2dhdr.dh_evict; + dev.l2ad_start = l2dhdr.dh_start; + dev.l2ad_end = l2dhdr.dh_end; + + if (l2dhdr.dh_start_lbps[0].lbp_daddr == 0) { + /* no log blocks to read */ + (void) printf("No log blocks to read\n"); + (void) printf("\n"); + return; + } else { + dev.l2ad_hand = lbps[0].lbp_daddr + + L2BLK_GET_PSIZE((&lbps[0])->lbp_prop); + } + + dev.l2ad_first = !!(l2dhdr.dh_flags & L2ARC_DEV_HDR_EVICT_FIRST); + + for (;;) { + if (!l2arc_log_blkptr_valid(&dev, &lbps[0])) + break; + + psize = L2BLK_GET_PSIZE((&lbps[0])->lbp_prop); + if (pread64(fd, &this_lb, psize, lbps[0].lbp_daddr) != psize) { + (void) printf("Error while reading next log block\n\n"); + break; + } + + fletcher_4_native_varsize(&this_lb, psize, &cksum); + if (!ZIO_CHECKSUM_EQUAL(cksum, lbps[0].lbp_cksum)) { + failed++; + (void) printf("Invalid cksum\n"); + dump_l2arc_log_blkptr(lbps[0]); + break; + } + + switch (L2BLK_GET_COMPRESS((&lbps[0])->lbp_prop)) { + case ZIO_COMPRESS_OFF: + break; + case ZIO_COMPRESS_LZ4: + abd = abd_alloc_for_io(psize, B_TRUE); + abd_copy_from_buf_off(abd, &this_lb, 0, psize); + zio_decompress_data(L2BLK_GET_COMPRESS( + (&lbps[0])->lbp_prop), abd, &this_lb, + psize, sizeof (this_lb)); + abd_free(abd); + break; + default: + break; + } + + if (this_lb.lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC)) + byteswap_uint64_array(&this_lb, psize); + + if (this_lb.lb_magic != L2ARC_LOG_BLK_MAGIC) { + (void) printf("Invalid log block magic\n\n"); + break; + } + + i++; + if (dump_opt['l'] > 1) { + (void) printf("lb[%4d]\tmagic: %llu\n", i, + (u_longlong_t)this_lb.lb_magic); + dump_l2arc_log_blkptr(lbps[0]); + } + + if (dump_opt['l'] > 2) + dump_l2arc_log_entries(l2dhdr.dh_log_blk_ent, + this_lb.lb_entries, i); + + if (l2arc_range_check_overlap(lbps[1].lbp_daddr, + lbps[0].lbp_daddr, dev.l2ad_evict) && !dev.l2ad_first) + break; + + lbps[0] = lbps[1]; + lbps[1] = this_lb.lb_prev_lbp; + } + + (void) printf("log_blk_count:\t %d with valid cksum\n", i); + (void) printf("\t\t %d with invalid cksum\n\n", failed); +} + +static void +dump_l2arc_header(int fd) +{ + l2arc_dev_hdr_phys_t l2dhdr; + int error = B_FALSE; + + if (pread64(fd, &l2dhdr, sizeof (l2dhdr), + VDEV_LABEL_START_SIZE) != sizeof (l2dhdr)) { + error = B_TRUE; + } else { + if (l2dhdr.dh_magic == BSWAP_64(L2ARC_DEV_HDR_MAGIC)) + byteswap_uint64_array(&l2dhdr, sizeof (l2dhdr)); + + if (l2dhdr.dh_magic != L2ARC_DEV_HDR_MAGIC) + error = B_TRUE; + } + + if (error) { + (void) printf("L2ARC device header not found\n\n"); + } else if (!dump_opt['q']) { + print_l2arc_header(); + + (void) printf(" magic: %llu\n", + (u_longlong_t)l2dhdr.dh_magic); + (void) printf(" version: %llu\n", + (u_longlong_t)l2dhdr.dh_version); + (void) printf(" pool_guid: %llu\n", + (u_longlong_t)l2dhdr.dh_spa_guid); + (void) printf(" flags: %llu\n", + (u_longlong_t)l2dhdr.dh_flags); + (void) printf(" start_lbps[0]: %llu\n", + (u_longlong_t) + l2dhdr.dh_start_lbps[0].lbp_daddr); + (void) printf(" start_lbps[1]: %llu\n", + (u_longlong_t) + l2dhdr.dh_start_lbps[1].lbp_daddr); + (void) printf(" log_blk_ent: %llu\n", + (u_longlong_t)l2dhdr.dh_log_blk_ent); + (void) printf(" start: %llu\n", + (u_longlong_t)l2dhdr.dh_start); + (void) printf(" end: %llu\n", + (u_longlong_t)l2dhdr.dh_end); + (void) printf(" evict: %llu\n\n", + (u_longlong_t)l2dhdr.dh_evict); + + dump_l2arc_log_blocks(fd, l2dhdr); + } +} + static void dump_config_from_label(zdb_label_t *label, size_t buflen, int l) { @@ -3639,10 +3850,11 @@ dump_label(const char *dev) { char path[MAXPATHLEN]; zdb_label_t labels[VDEV_LABELS]; - uint64_t psize, ashift; + uint64_t psize, ashift, l2cache; struct stat64 statbuf; boolean_t config_found = B_FALSE; boolean_t error = B_FALSE; + boolean_t read_l2arc_header = B_FALSE; avl_tree_t config_tree; avl_tree_t uberblock_tree; void *node, *cookie; @@ -3735,6 +3947,15 @@ dump_label(const char *dev) if (nvlist_size(config, &size, NV_ENCODE_XDR) != 0) size = buflen; + /* If the device is a cache device clear the header. */ + if (!read_l2arc_header) { + if (nvlist_lookup_uint64(config, + ZPOOL_CONFIG_POOL_STATE, &l2cache) == 0 && + l2cache == POOL_STATE_L2CACHE) { + read_l2arc_header = B_TRUE; + } + } + fletcher_4_native_varsize(buf, size, &cksum); rec = cksum_record_insert(&config_tree, &cksum, l); @@ -3785,6 +4006,12 @@ dump_label(const char *dev) nvlist_free(label->config_nv); } + /* + * Dump the L2ARC header, if existent. + */ + if (read_l2arc_header) + dump_l2arc_header(fd); + cookie = NULL; while ((node = avl_destroy_nodes(&config_tree, &cookie)) != NULL) umem_free(node, sizeof (cksum_record_t)); diff --git a/configure.ac b/configure.ac index 370a1970ffd..8604cdaa57f 100644 --- a/configure.ac +++ b/configure.ac @@ -336,6 +336,7 @@ AC_CONFIG_FILES([ tests/zfs-tests/tests/functional/no_space/Makefile tests/zfs-tests/tests/functional/nopwrite/Makefile tests/zfs-tests/tests/functional/online_offline/Makefile + tests/zfs-tests/tests/functional/persist_l2arc/Makefile tests/zfs-tests/tests/functional/pool_checkpoint/Makefile tests/zfs-tests/tests/functional/pool_names/Makefile tests/zfs-tests/tests/functional/poolversion/Makefile diff --git a/include/sys/arc.h b/include/sys/arc.h index 75c483918ce..f500e1d45ee 100644 --- a/include/sys/arc.h +++ b/include/sys/arc.h @@ -310,10 +310,14 @@ void arc_fini(void); void l2arc_add_vdev(spa_t *spa, vdev_t *vd); void l2arc_remove_vdev(vdev_t *vd); boolean_t l2arc_vdev_present(vdev_t *vd); +void l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen); +boolean_t l2arc_range_check_overlap(uint64_t bottom, uint64_t top, + uint64_t check); void l2arc_init(void); void l2arc_fini(void); void l2arc_start(void); void l2arc_stop(void); +void l2arc_spa_rebuild_start(spa_t *spa); #ifndef _KERNEL extern boolean_t arc_watch; diff --git a/include/sys/arc_impl.h b/include/sys/arc_impl.h index 2468a4aac75..928b7232556 100644 --- a/include/sys/arc_impl.h +++ b/include/sys/arc_impl.h @@ -20,9 +20,10 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2013 by Delphix. All rights reserved. - * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. - * Copyright 2013 Nexenta Systems, Inc. All rights reserved. + * Copyright (c) 2013, Delphix. All rights reserved. + * Copyright (c) 2013, Saso Kiselkov. All rights reserved. + * Copyright (c) 2013, Nexenta Systems, Inc. All rights reserved. + * Copyright (c) 2020, George Amanakis. All rights reserved. */ #ifndef _SYS_ARC_IMPL_H @@ -176,6 +177,218 @@ typedef struct l1arc_buf_hdr { abd_t *b_pabd; } l1arc_buf_hdr_t; +typedef enum l2arc_dev_hdr_flags_t { + L2ARC_DEV_HDR_EVICT_FIRST = (1 << 0) /* mirror of l2ad_first */ +} l2arc_dev_hdr_flags_t; + +/* + * Pointer used in persistent L2ARC (for pointing to log blocks). + */ +typedef struct l2arc_log_blkptr { + /* + * Offset of log block within the device, in bytes + */ + uint64_t lbp_daddr; + /* + * Aligned payload size (in bytes) of the log block + */ + uint64_t lbp_payload_asize; + /* + * Offset in bytes of the first buffer in the payload + */ + uint64_t lbp_payload_start; + /* + * lbp_prop has the following format: + * * logical size (in bytes) + * * physical (compressed) size (in bytes) + * * compression algorithm (we always LZ4-compress l2arc logs) + * * checksum algorithm (used for lbp_cksum) + */ + uint64_t lbp_prop; + zio_cksum_t lbp_cksum; /* checksum of log */ +} l2arc_log_blkptr_t; + +/* + * The persistent L2ARC device header. + * Byte order of magic determines whether 64-bit bswap of fields is necessary. + */ +typedef struct l2arc_dev_hdr_phys { + uint64_t dh_magic; /* L2ARC_DEV_HDR_MAGIC */ + uint64_t dh_version; /* Persistent L2ARC version */ + + /* + * Global L2ARC device state and metadata. + */ + uint64_t dh_spa_guid; + uint64_t dh_vdev_guid; + uint64_t dh_log_blk_ent; /* entries per log blk */ + uint64_t dh_evict; /* evicted offset in bytes */ + uint64_t dh_flags; /* l2arc_dev_hdr_flags_t */ + /* + * Used in zdb.c for determining if a log block is valid, in the same + * way that l2arc_rebuild() does. + */ + uint64_t dh_start; + uint64_t dh_end; + + /* + * Start of log block chain. [0] -> newest log, [1] -> one older (used + * for initiating prefetch). + */ + l2arc_log_blkptr_t dh_start_lbps[2]; + const uint64_t dh_pad[34]; /* pad to 512 bytes */ + zio_eck_t dh_tail; +} l2arc_dev_hdr_phys_t; +CTASSERT_GLOBAL(sizeof (l2arc_dev_hdr_phys_t) == SPA_MINBLOCKSIZE); + +/* + * A single ARC buffer header entry in a l2arc_log_blk_phys_t. + */ +typedef struct l2arc_log_ent_phys { + dva_t le_dva; /* dva of buffer */ + uint64_t le_birth; /* birth txg of buffer */ + /* + * le_prop has the following format: + * * logical size (in bytes) + * * physical (compressed) size (in bytes) + * * compression algorithm + * * object type (used to restore arc_buf_contents_t) + * * protected status (used for encryption) + * * prefetch status (used in l2arc_read_done()) + */ + uint64_t le_prop; + uint64_t le_daddr; /* buf location on l2dev */ + /* + * We pad the size of each entry to a power of 2 so that the size of + * l2arc_log_blk_phys_t is power-of-2 aligned with SPA_MINBLOCKSHIFT, + * because of the L2ARC_SET_*SIZE macros. + */ + const uint64_t le_pad[3]; /* pad to 64 bytes */ +} l2arc_log_ent_phys_t; + +#define L2ARC_LOG_BLK_MAX_ENTRIES (1022) + +/* + * A log block of up to 1022 ARC buffer log entries, chained into the + * persistent L2ARC metadata linked list. Byte order of magic determines + * whether 64-bit bswap of fields is necessary. + */ +typedef struct l2arc_log_blk_phys { + uint64_t lb_magic; /* L2ARC_LOG_BLK_MAGIC */ + /* + * There are 2 chains (headed by dh_start_lbps[2]), and this field + * points back to the previous block in this chain. We alternate + * which chain we append to, so they are time-wise and offset-wise + * interleaved, but that is an optimization rather than for + * correctness. + */ + l2arc_log_blkptr_t lb_prev_lbp; /* pointer to prev log block */ + /* + * Pad header section to 128 bytes + */ + uint64_t lb_pad[7]; + /* Payload */ + l2arc_log_ent_phys_t lb_entries[L2ARC_LOG_BLK_MAX_ENTRIES]; +} l2arc_log_blk_phys_t; /* 64K total */ + +/* + * The size of l2arc_log_blk_phys_t has to be power-of-2 aligned with + * SPA_MINBLOCKSHIFT because of L2BLK_SET_*SIZE macros. + */ +CTASSERT_GLOBAL(IS_P2ALIGNED(sizeof (l2arc_log_blk_phys_t), + 1ULL << SPA_MINBLOCKSHIFT)); +CTASSERT_GLOBAL(sizeof (l2arc_log_blk_phys_t) >= SPA_MINBLOCKSIZE); +CTASSERT_GLOBAL(sizeof (l2arc_log_blk_phys_t) <= SPA_MAXBLOCKSIZE); + +/* + * These structures hold in-flight abd buffers for log blocks as they're being + * written to the L2ARC device. + */ +typedef struct l2arc_lb_abd_buf { + abd_t *abd; + list_node_t node; +} l2arc_lb_abd_buf_t; + +/* + * These structures hold pointers to log blocks present on the L2ARC device. + */ +typedef struct l2arc_lb_ptr_buf { + l2arc_log_blkptr_t *lb_ptr; + list_node_t node; +} l2arc_lb_ptr_buf_t; + +/* Macros for setting fields in le_prop and lbp_prop */ +#define L2BLK_GET_LSIZE(field) \ + BF64_GET_SB((field), 0, SPA_LSIZEBITS, SPA_MINBLOCKSHIFT, 1) +#define L2BLK_SET_LSIZE(field, x) \ + BF64_SET_SB((field), 0, SPA_LSIZEBITS, SPA_MINBLOCKSHIFT, 1, x) +#define L2BLK_GET_PSIZE(field) \ + BF64_GET_SB((field), 16, SPA_PSIZEBITS, SPA_MINBLOCKSHIFT, 1) +#define L2BLK_SET_PSIZE(field, x) \ + BF64_SET_SB((field), 16, SPA_PSIZEBITS, SPA_MINBLOCKSHIFT, 1, x) +#define L2BLK_GET_COMPRESS(field) \ + BF64_GET((field), 32, SPA_COMPRESSBITS) +#define L2BLK_SET_COMPRESS(field, x) \ + BF64_SET((field), 32, SPA_COMPRESSBITS, x) +#define L2BLK_GET_PREFETCH(field) BF64_GET((field), 39, 1) +#define L2BLK_SET_PREFETCH(field, x) BF64_SET((field), 39, 1, x) +#define L2BLK_GET_CHECKSUM(field) BF64_GET((field), 40, 8) +#define L2BLK_SET_CHECKSUM(field, x) BF64_SET((field), 40, 8, x) +#define L2BLK_GET_TYPE(field) BF64_GET((field), 48, 8) +#define L2BLK_SET_TYPE(field, x) BF64_SET((field), 48, 8, x) +#define L2BLK_GET_PROTECTED(field) BF64_GET((field), 56, 1) +#define L2BLK_SET_PROTECTED(field, x) BF64_SET((field), 56, 1, x) + +#define PTR_SWAP(x, y) \ + do { \ + void *tmp = (x);\ + x = y; \ + y = tmp; \ + _NOTE(CONSTCOND)\ + } while (0) + +#define L2ARC_DEV_HDR_MAGIC 0x5a46534341434845LLU /* ASCII: "ZFSCACHE" */ +#define L2ARC_LOG_BLK_MAGIC 0x4c4f47424c4b4844LLU /* ASCII: "LOGBLKHD" */ + +/* + * L2ARC Internals + */ +typedef struct l2arc_dev { + vdev_t *l2ad_vdev; /* vdev */ + spa_t *l2ad_spa; /* spa */ + uint64_t l2ad_hand; /* next write location */ + uint64_t l2ad_start; /* first addr on device */ + uint64_t l2ad_end; /* last addr on device */ + boolean_t l2ad_first; /* first sweep through */ + boolean_t l2ad_writing; /* currently writing */ + kmutex_t l2ad_mtx; /* lock for buffer list */ + list_t l2ad_buflist; /* buffer list */ + list_node_t l2ad_node; /* device list node */ + zfs_refcount_t l2ad_alloc; /* allocated bytes */ + /* + * Persistence-related stuff + */ + l2arc_dev_hdr_phys_t *l2ad_dev_hdr; /* persistent device header */ + uint64_t l2ad_dev_hdr_asize; /* aligned hdr size */ + l2arc_log_blk_phys_t l2ad_log_blk; /* currently open log block */ + int l2ad_log_ent_idx; /* index into cur log blk */ + /* Number of bytes in current log block's payload */ + uint64_t l2ad_log_blk_payload_asize; + /* + * Offset (in bytes) of the first buffer in current log block's + * payload. + */ + uint64_t l2ad_log_blk_payload_start; + /* Flag indicating whether a rebuild is scheduled or is going on */ + boolean_t l2ad_rebuild; + boolean_t l2ad_rebuild_cancel; + boolean_t l2ad_rebuild_began; + uint64_t l2ad_log_entries; /* entries per log blk */ + uint64_t l2ad_evict; /* evicted offset in bytes */ + /* List of pointers to log blocks present in the L2ARC device */ + list_t l2ad_lbptr_list; +} l2arc_dev_t; + /* * Encrypted blocks will need to be stored encrypted on the L2ARC * disk as they appear in the main pool. In order for this to work we @@ -206,32 +419,19 @@ typedef struct arc_buf_hdr_crypt { uint8_t b_mac[ZIO_DATA_MAC_LEN]; } arc_buf_hdr_crypt_t; -typedef struct l2arc_dev { - vdev_t *l2ad_vdev; /* vdev */ - spa_t *l2ad_spa; /* spa */ - uint64_t l2ad_hand; /* next write location */ - uint64_t l2ad_start; /* first addr on device */ - uint64_t l2ad_end; /* last addr on device */ - boolean_t l2ad_first; /* first sweep through */ - boolean_t l2ad_writing; /* currently writing */ - kmutex_t l2ad_mtx; /* lock for buffer list */ - list_t l2ad_buflist; /* buffer list */ - list_node_t l2ad_node; /* device list node */ - zfs_refcount_t l2ad_alloc; /* allocated bytes */ -} l2arc_dev_t; - typedef struct l2arc_buf_hdr { /* protected by arc_buf_hdr mutex */ l2arc_dev_t *b_dev; /* L2ARC device */ uint64_t b_daddr; /* disk address, offset byte */ uint32_t b_hits; - list_node_t b_l2node; } l2arc_buf_hdr_t; typedef struct l2arc_write_callback { l2arc_dev_t *l2wcb_dev; /* device info */ arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ + /* in-flight list of log blocks */ + list_t l2wcb_abd_list; } l2arc_write_callback_t; struct arc_buf_hdr { @@ -532,6 +732,71 @@ typedef struct arc_stats { kstat_named_t arcstat_l2_psize; /* Not updated directly; only synced in arc_kstat_update. */ kstat_named_t arcstat_l2_hdr_size; + /* + * Number of L2ARC log blocks written. These are used for restoring the + * L2ARC. Updated during writing of L2ARC log blocks. + */ + kstat_named_t arcstat_l2_log_blk_writes; + /* + * Moving average of the physical size of the L2ARC log blocks, in + * bytes. Updated during L2ARC rebuild and during writing of L2ARC + * log blocks. + */ + kstat_named_t arcstat_l2_log_blk_avg_size; + /* + * Moving average of the physical size of L2ARC restored data, in bytes, + * to the physical size of their metadata in ARC, in bytes. + * Updated during L2ARC rebuild and during writing of L2ARC log blocks. + */ + kstat_named_t arcstat_l2_data_to_meta_ratio; + /* + * Number of times the L2ARC rebuild was successful for an L2ARC device. + */ + kstat_named_t arcstat_l2_rebuild_success; + /* + * Number of times the L2ARC rebuild failed because the device header + * was in an unsupported format or corrupted. + */ + kstat_named_t arcstat_l2_rebuild_abort_unsupported; + /* + * Number of times the L2ARC rebuild failed because of IO errors + * while reading a log block. + */ + kstat_named_t arcstat_l2_rebuild_abort_io_errors; + /* + * Number of times the L2ARC rebuild failed because of IO errors when + * reading the device header. + */ + kstat_named_t arcstat_l2_rebuild_abort_dh_errors; + /* + * Number of L2ARC log blocks which failed to be restored due to + * checksum errors. + */ + kstat_named_t arcstat_l2_rebuild_abort_cksum_lb_errors; + /* + * Number of times the L2ARC rebuild was aborted due to low system + * memory. + */ + kstat_named_t arcstat_l2_rebuild_abort_lowmem; + /* Logical size of L2ARC restored data, in bytes. */ + kstat_named_t arcstat_l2_rebuild_size; + /* + * Number of L2ARC log entries (buffers) that were successfully + * restored in ARC. + */ + kstat_named_t arcstat_l2_rebuild_bufs; + /* + * Number of L2ARC log entries (buffers) already cached in ARC. These + * were not restored again. + */ + kstat_named_t arcstat_l2_rebuild_bufs_precached; + /* Physical size of L2ARC restored data, in bytes. */ + kstat_named_t arcstat_l2_rebuild_psize; + /* + * Number of L2ARC log blocks that were restored successfully. Each + * log block may hold up to L2ARC_LOG_BLK_MAX_ENTRIES buffers. + */ + kstat_named_t arcstat_l2_rebuild_log_blks; kstat_named_t arcstat_memory_throttle_count; kstat_named_t arcstat_memory_direct_count; kstat_named_t arcstat_memory_indirect_count; @@ -617,6 +882,10 @@ extern void arc_tuning_update(boolean_t); extern int param_set_arc_long(ZFS_MODULE_PARAM_ARGS); extern int param_set_arc_int(ZFS_MODULE_PARAM_ARGS); +/* used in zdb.c */ +boolean_t l2arc_log_blkptr_valid(l2arc_dev_t *dev, + const l2arc_log_blkptr_t *lbp); + #ifdef __cplusplus } #endif diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h index 477356aa77d..f5aced0da7c 100644 --- a/include/sys/fs/zfs.h +++ b/include/sys/fs/zfs.h @@ -573,6 +573,11 @@ typedef enum zfs_key_location { #define ZPL_VERSION_USERSPACE ZPL_VERSION_4 #define ZPL_VERSION_SA ZPL_VERSION_5 +/* Persistent L2ARC version */ +#define L2ARC_PERSISTENT_VERSION_1 1ULL +#define L2ARC_PERSISTENT_VERSION L2ARC_PERSISTENT_VERSION_1 +#define L2ARC_PERSISTENT_VERSION_STRING "1" + /* Rewind policy information */ #define ZPOOL_NO_REWIND 1 /* No policy - default behavior */ #define ZPOOL_NEVER_REWIND 2 /* Do not search for best txg or rewind */ diff --git a/include/sys/spa.h b/include/sys/spa.h index e1fab3c6977..6e844f5ee8c 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -787,6 +787,7 @@ extern int bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx); #define SPA_ASYNC_INITIALIZE_RESTART 0x100 #define SPA_ASYNC_TRIM_RESTART 0x200 #define SPA_ASYNC_AUTOTRIM_RESTART 0x400 +#define SPA_ASYNC_L2CACHE_REBUILD 0x800 /* * Controls the behavior of spa_vdev_remove(). diff --git a/lib/libzfs/libzfs_import.c b/lib/libzfs/libzfs_import.c index 88c4c645bc6..6c5f6183697 100644 --- a/lib/libzfs/libzfs_import.c +++ b/lib/libzfs/libzfs_import.c @@ -38,6 +38,7 @@ #include #include #include +#include /* * Returns true if the named pool matches the given GUID. @@ -146,8 +147,10 @@ zpool_clear_label(int fd) struct stat64 statbuf; int l; vdev_label_t *label; + l2arc_dev_hdr_phys_t *l2dhdr; uint64_t size; - int labels_cleared = 0; + int labels_cleared = 0, header_cleared = 0; + boolean_t clear_l2arc_header = B_FALSE; if (fstat64_blk(fd, &statbuf) == -1) return (0); @@ -157,8 +160,13 @@ zpool_clear_label(int fd) if ((label = calloc(1, sizeof (vdev_label_t))) == NULL) return (-1); + if ((l2dhdr = calloc(1, sizeof (l2arc_dev_hdr_phys_t))) == NULL) { + free(label); + return (-1); + } + for (l = 0; l < VDEV_LABELS; l++) { - uint64_t state, guid; + uint64_t state, guid, l2cache; nvlist_t *config; if (pread64(fd, label, sizeof (vdev_label_t), @@ -185,6 +193,15 @@ zpool_clear_label(int fd) continue; } + /* If the device is a cache device clear the header. */ + if (!clear_l2arc_header) { + if (nvlist_lookup_uint64(config, + ZPOOL_CONFIG_POOL_STATE, &l2cache) == 0 && + l2cache == POOL_STATE_L2CACHE) { + clear_l2arc_header = B_TRUE; + } + } + nvlist_free(config); /* @@ -202,7 +219,17 @@ zpool_clear_label(int fd) } } + /* Clear the L2ARC header. */ + if (clear_l2arc_header) { + memset(l2dhdr, 0, sizeof (l2arc_dev_hdr_phys_t)); + if (pwrite64(fd, l2dhdr, sizeof (l2arc_dev_hdr_phys_t), + VDEV_LABEL_START_SIZE) == sizeof (l2arc_dev_hdr_phys_t)) { + header_cleared++; + } + } + free(label); + free(l2dhdr); if (labels_cleared == 0) return (-1); diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5 index a7623ff27d5..40666c8f324 100644 --- a/man/man5/zfs-module-parameters.5 +++ b/man/man5/zfs-module-parameters.5 @@ -87,7 +87,7 @@ Default value: \fB10\fR%. .ad .RS 12n Set the size of the dbuf cache, \fBdbuf_cache_max_bytes\fR, to a log2 fraction -of the target arc size. +of the target ARC size. .sp Default value: \fB5\fR. .RE @@ -99,7 +99,7 @@ Default value: \fB5\fR. .ad .RS 12n Set the size of the dbuf metadata cache, \fBdbuf_metadata_cache_max_bytes\fR, -to a log2 fraction of the target arc size. +to a log2 fraction of the target ARC size. .sp Default value: \fB6\fR. .RE @@ -179,7 +179,10 @@ Default value: \fB1\fR. .ad .RS 12n How far through the ARC lists to search for L2ARC cacheable content, expressed -as a multiplier of \fBl2arc_write_max\fR +as a multiplier of \fBl2arc_write_max\fR. +ARC persistence across reboots can be achieved with persistent L2ARC by setting +this parameter to \fB0\fR allowing the full length of ARC lists to be searched +for cacheable content. .sp Default value: \fB2\fR. .RE @@ -203,7 +206,7 @@ Default value: \fB200\fR%. .ad .RS 12n Do not write buffers to L2ARC if they were prefetched but not used by -applications +applications. .sp Use \fB1\fR for yes (default) and \fB0\fR to disable. .RE @@ -214,7 +217,7 @@ Use \fB1\fR for yes (default) and \fB0\fR to disable. \fBl2arc_norw\fR (int) .ad .RS 12n -No reads during writes +No reads during writes. .sp Use \fB1\fR for yes and \fB0\fR for no (default). .RE @@ -237,11 +240,41 @@ Default value: \fB8,388,608\fR. \fBl2arc_write_max\fR (ulong) .ad .RS 12n -Max write bytes per interval +Max write bytes per interval. .sp Default value: \fB8,388,608\fR. .RE +.sp +.ne 2 +.na +\fBl2arc_rebuild_enabled\fR (int) +.ad +.RS 12n +Rebuild the L2ARC when importing a pool (persistent L2ARC). This can be +disabled if there are problems importing a pool or attaching an L2ARC device +(e.g. the L2ARC device is slow in reading stored log metadata, or the metadata +has become somehow fragmented/unusable). +.sp +Use \fB1\fR for yes (default) and \fB0\fR for no. +.RE + +.sp +.ne 2 +.na +\fBl2arc_rebuild_blocks_min_l2size\fR (ulong) +.ad +.RS 12n +Min size (in bytes) of an L2ARC device required in order to write log blocks +in it. The log blocks are used upon importing the pool to rebuild +the L2ARC (persistent L2ARC). Rationale: for L2ARC devices less than 1GB, the +amount of data l2arc_evict() evicts is significant compared to the amount of +restored L2ARC data. In this case do not write log blocks in L2ARC in order not +to waste space. +.sp +Default value: \fB1,073,741,824\fR (1GB). +.RE + .sp .ne 2 .na @@ -614,7 +647,7 @@ Default value: \fB1\fR. .ad .RS 12n Sets the maximum number of bytes to consume during pool import to the log2 -fraction of the target arc size. +fraction of the target ARC size. .sp Default value: \fB4\fR. .RE @@ -963,7 +996,7 @@ Default value: \fB1\fR. \fBzfs_arc_min\fR (ulong) .ad .RS 12n -Min arc size of ARC in bytes. If set to 0 then arc_c_min will default to +Min size of ARC in bytes. If set to 0 then arc_c_min will default to consuming the larger of 32M or 1/32 of total system memory. .sp Default value: \fB0\fR. @@ -1088,7 +1121,7 @@ Default value: \fB0\fR. Percent of pagecache to reclaim arc to This tunable allows ZFS arc to play more nicely with the kernel's LRU -pagecache. It can guarantee that the arc size won't collapse under scanning +pagecache. It can guarantee that the ARC size won't collapse under scanning pressure on the pagecache, yet still allows arc to be reclaimed down to zfs_arc_min if necessary. This value is specified as percent of pagecache size (as measured by NR_FILE_PAGES) where that percent may exceed 100. This diff --git a/man/man8/zdb.8 b/man/man8/zdb.8 index 8506d54780f..3915be3f8ef 100644 --- a/man/man8/zdb.8 +++ b/man/man8/zdb.8 @@ -212,18 +212,24 @@ If specified multiple times, display counts of each intent log transaction type. Examine the checkpointed state of the pool. Note, the on disk format of the pool is not reverted to the checkpointed state. .It Fl l Ar device -Read the vdev labels from the specified device. +Read the vdev labels and L2ARC header from the specified device. .Nm Fl l will return 0 if valid label was found, 1 if error occurred, and 2 if no valid -labels were found. Each unique configuration is displayed only once. +labels were found. The presence of L2ARC header is indicated by a specific +sequence (L2ARC_DEV_HDR_MAGIC). Each unique configuration is displayed only +once. .It Fl ll Ar device -In addition display label space usage stats. +In addition display label space usage stats. If a valid L2ARC header was found +also display the properties of log blocks used for restoring L2ARC contents +(persistent L2ARC). .It Fl lll Ar device -Display every configuration, unique or not. +Display every configuration, unique or not. If a valid L2ARC header was found +also display the properties of log entries in log blocks used for restoring +L2ARC contents (persistent L2ARC). .Pp If the .Fl q -option is also specified, don't print the labels. +option is also specified, don't print the labels or the L2ARC header. .Pp If the .Fl u diff --git a/man/man8/zpool-labelclear.8 b/man/man8/zpool-labelclear.8 index 52638bdf70a..ef6b92e82c9 100644 --- a/man/man8/zpool-labelclear.8 +++ b/man/man8/zpool-labelclear.8 @@ -48,7 +48,10 @@ .Xc Removes ZFS label information from the specified .Ar device . -The +If the +.Ar device +is a cache device, it also removes the L2ARC header +(persistent L2ARC). The .Ar device must not be part of an active pool configuration. .Bl -tag -width Ds diff --git a/man/man8/zpoolconcepts.8 b/man/man8/zpoolconcepts.8 index 60845eef20c..c2fc349710d 100644 --- a/man/man8/zpoolconcepts.8 +++ b/man/man8/zpoolconcepts.8 @@ -323,8 +323,28 @@ If a read error is encountered on a cache device, that read I/O is reissued to the original storage pool device, which might be part of a mirrored or raidz configuration. .Pp -The content of the cache devices is considered volatile, as is the case with -other system caches. +The content of the cache devices is persistent across reboots and restored +asynchronously when importing the pool in L2ARC (persistent L2ARC). +This can be disabled by setting +.Sy l2arc_rebuild_enabled = 0 . +For cache devices smaller than 1GB we do not write the metadata structures +required for rebuilding the L2ARC in order not to waste space. This can be +changed with +.Sy l2arc_rebuild_blocks_min_l2size . +The cache device header (512 bytes) is updated even if no metadata structures +are written. Setting +.Sy l2arc_headroom = 0 +will result in scanning the full-length ARC lists for cacheable content to be +written in L2ARC (persistent ARC). If a cache device is added with +.Nm zpool Cm add +its label and header will be overwritten and its contents are not going to be +restored in L2ARC, even if the device was previously part of the pool. If a +cache device is onlined with +.Nm zpool Cm online +its contents will be restored in L2ARC. This is useful in case of memory pressure +where the contents of the cache device are not fully restored in L2ARC. +The user can off/online the cache device when there is less memory pressure +in order to fully restore its contents to L2ARC. .Ss Pool checkpoint Before starting critical procedures that include destructive actions (e.g .Nm zfs Cm destroy diff --git a/module/os/linux/zfs/zfs_sysfs.c b/module/os/linux/zfs/zfs_sysfs.c index bb7f3b69a66..fb7c6898736 100644 --- a/module/os/linux/zfs/zfs_sysfs.c +++ b/module/os/linux/zfs/zfs_sysfs.c @@ -353,13 +353,14 @@ pool_property_show(struct kobject *kobj, struct attribute *attr, char *buf) * This list is intended for kernel features that don't have a pool feature * association or that extend existing user kernel interfaces. * - * A user processes can easily check if the running zfs kernel module + * A user process can easily check if the running zfs kernel module * supports the new feature. */ static const char *zfs_kernel_features[] = { /* --> Add new kernel features here */ "com.delphix:vdev_initialize", "org.zfsonlinux:vdev_trim", + "org.openzfs:l2arc_persistent", }; #define KERNEL_FEATURE_COUNT ARRAY_SIZE(zfs_kernel_features) diff --git a/module/zfs/arc.c b/module/zfs/arc.c index b5d17431ce2..74bfbfc7029 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -21,10 +21,11 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2018, Joyent, Inc. - * Copyright (c) 2011, 2019 by Delphix. All rights reserved. - * Copyright (c) 2014 by Saso Kiselkov. All rights reserved. - * Copyright 2017 Nexenta Systems, Inc. All rights reserved. + * Copyright (c) 2011, 2019, Delphix. All rights reserved. + * Copyright (c) 2014, Saso Kiselkov. All rights reserved. + * Copyright (c) 2017, Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2019, loli10K . All rights reserved. + * Copyright (c) 2020, George Amanakis. All rights reserved. */ /* @@ -528,6 +529,20 @@ arc_stats_t arc_stats = { { "l2_size", KSTAT_DATA_UINT64 }, { "l2_asize", KSTAT_DATA_UINT64 }, { "l2_hdr_size", KSTAT_DATA_UINT64 }, + { "l2_log_blk_writes", KSTAT_DATA_UINT64 }, + { "l2_log_blk_avg_size", KSTAT_DATA_UINT64 }, + { "l2_data_to_meta_ratio", KSTAT_DATA_UINT64 }, + { "l2_rebuild_success", KSTAT_DATA_UINT64 }, + { "l2_rebuild_unsupported", KSTAT_DATA_UINT64 }, + { "l2_rebuild_io_errors", KSTAT_DATA_UINT64 }, + { "l2_rebuild_dh_errors", KSTAT_DATA_UINT64 }, + { "l2_rebuild_cksum_lb_errors", KSTAT_DATA_UINT64 }, + { "l2_rebuild_lowmem", KSTAT_DATA_UINT64 }, + { "l2_rebuild_size", KSTAT_DATA_UINT64 }, + { "l2_rebuild_bufs", KSTAT_DATA_UINT64 }, + { "l2_rebuild_bufs_precached", KSTAT_DATA_UINT64 }, + { "l2_rebuild_psize", KSTAT_DATA_UINT64 }, + { "l2_rebuild_log_blks", KSTAT_DATA_UINT64 }, { "memory_throttle_count", KSTAT_DATA_UINT64 }, { "memory_direct_count", KSTAT_DATA_UINT64 }, { "memory_indirect_count", KSTAT_DATA_UINT64 }, @@ -582,6 +597,24 @@ arc_stats_t arc_stats = { } \ } +/* + * This macro allows us to use kstats as floating averages. Each time we + * update this kstat, we first factor it and the update value by + * ARCSTAT_AVG_FACTOR to shrink the new value's contribution to the overall + * average. This macro assumes that integer loads and stores are atomic, but + * is not safe for multiple writers updating the kstat in parallel (only the + * last writer's update will remain). + */ +#define ARCSTAT_F_AVG_FACTOR 3 +#define ARCSTAT_F_AVG(stat, value) \ + do { \ + uint64_t x = ARCSTAT(stat); \ + x = x - x / ARCSTAT_F_AVG_FACTOR + \ + (value) / ARCSTAT_F_AVG_FACTOR; \ + ARCSTAT(stat) = x; \ + _NOTE(CONSTCOND) \ + } while (0) + kstat_t *arc_ksp; static arc_state_t *arc_anon; static arc_state_t *arc_mru_ghost; @@ -805,6 +838,9 @@ static kmutex_t l2arc_feed_thr_lock; static kcondvar_t l2arc_feed_thr_cv; static uint8_t l2arc_thread_exit; +static kmutex_t l2arc_rebuild_thr_lock; +static kcondvar_t l2arc_rebuild_thr_cv; + static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, void *); static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, void *); static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, void *); @@ -816,6 +852,7 @@ static void arc_hdr_alloc_abd(arc_buf_hdr_t *, boolean_t); static void arc_access(arc_buf_hdr_t *, kmutex_t *); static boolean_t arc_is_overflowing(void); static void arc_buf_watch(arc_buf_t *); +static l2arc_dev_t *l2arc_vdev_get(vdev_t *vd); static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *); static uint32_t arc_bufc_to_flags(arc_buf_contents_t); @@ -825,6 +862,58 @@ static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags); static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *); static void l2arc_read_done(zio_t *); +/* + * Performance tuning of L2ARC persistence: + * + * l2arc_rebuild_enabled : A ZFS module parameter that controls whether adding + * an L2ARC device (either at pool import or later) will attempt + * to rebuild L2ARC buffer contents. + * l2arc_rebuild_blocks_min_l2size : A ZFS module parameter that controls + * whether log blocks are written to the L2ARC device. If the L2ARC + * device is less than 1GB, the amount of data l2arc_evict() + * evicts is significant compared to the amount of restored L2ARC + * data. In this case do not write log blocks in L2ARC in order + * not to waste space. + */ +int l2arc_rebuild_enabled = B_TRUE; +unsigned long l2arc_rebuild_blocks_min_l2size = 1024 * 1024 * 1024; + +/* L2ARC persistence rebuild control routines. */ +void l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen); +static void l2arc_dev_rebuild_start(l2arc_dev_t *dev); +static int l2arc_rebuild(l2arc_dev_t *dev); + +/* L2ARC persistence read I/O routines. */ +static int l2arc_dev_hdr_read(l2arc_dev_t *dev); +static int l2arc_log_blk_read(l2arc_dev_t *dev, + const l2arc_log_blkptr_t *this_lp, const l2arc_log_blkptr_t *next_lp, + l2arc_log_blk_phys_t *this_lb, l2arc_log_blk_phys_t *next_lb, + zio_t *this_io, zio_t **next_io); +static zio_t *l2arc_log_blk_fetch(vdev_t *vd, + const l2arc_log_blkptr_t *lp, l2arc_log_blk_phys_t *lb); +static void l2arc_log_blk_fetch_abort(zio_t *zio); + +/* L2ARC persistence block restoration routines. */ +static void l2arc_log_blk_restore(l2arc_dev_t *dev, + const l2arc_log_blk_phys_t *lb, uint64_t lb_psize, uint64_t lb_daddr); +static void l2arc_hdr_restore(const l2arc_log_ent_phys_t *le, + l2arc_dev_t *dev); + +/* L2ARC persistence write I/O routines. */ +static void l2arc_dev_hdr_update(l2arc_dev_t *dev); +static void l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, + l2arc_write_callback_t *cb); + +/* L2ARC persistence auxilliary routines. */ +boolean_t l2arc_log_blkptr_valid(l2arc_dev_t *dev, + const l2arc_log_blkptr_t *lbp); +static boolean_t l2arc_log_blk_insert(l2arc_dev_t *dev, + const arc_buf_hdr_t *ab); +boolean_t l2arc_range_check_overlap(uint64_t bottom, + uint64_t top, uint64_t check); +static void l2arc_blk_fetch_done(zio_t *zio); +static inline uint64_t + l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev); /* * We use Cityhash for this. It's fast, and has good hash properties without @@ -1583,6 +1672,42 @@ arc_buf_try_copy_decompressed_data(arc_buf_t *buf) return (copied); } +/* + * Allocates an ARC buf header that's in an evicted & L2-cached state. + * This is used during l2arc reconstruction to make empty ARC buffers + * which circumvent the regular disk->arc->l2arc path and instead come + * into being in the reverse order, i.e. l2arc->arc. + */ +arc_buf_hdr_t * +arc_buf_alloc_l2only(size_t size, arc_buf_contents_t type, l2arc_dev_t *dev, + dva_t dva, uint64_t daddr, int32_t psize, uint64_t birth, + enum zio_compress compress, boolean_t protected, boolean_t prefetch) +{ + arc_buf_hdr_t *hdr; + + ASSERT(size != 0); + hdr = kmem_cache_alloc(hdr_l2only_cache, KM_SLEEP); + hdr->b_birth = birth; + hdr->b_type = type; + hdr->b_flags = 0; + arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L2HDR); + HDR_SET_LSIZE(hdr, size); + HDR_SET_PSIZE(hdr, psize); + arc_hdr_set_compress(hdr, compress); + if (protected) + arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED); + if (prefetch) + arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); + hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa); + + hdr->b_dva = dva; + + hdr->b_l2hdr.b_dev = dev; + hdr->b_l2hdr.b_daddr = daddr; + + return (hdr); +} + /* * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t. */ @@ -7463,6 +7588,103 @@ arc_fini(void) * * These three functions determine what to write, how much, and how quickly * to send writes. + * + * L2ARC persistence: + * + * When writing buffers to L2ARC, we periodically add some metadata to + * make sure we can pick them up after reboot, thus dramatically reducing + * the impact that any downtime has on the performance of storage systems + * with large caches. + * + * The implementation works fairly simply by integrating the following two + * modifications: + * + * *) When writing to the L2ARC, we occasionally write a "l2arc log block", + * which is an additional piece of metadata which describes what's been + * written. This allows us to rebuild the arc_buf_hdr_t structures of the + * main ARC buffers. There are 2 linked-lists of log blocks headed by + * dh_start_lbps[2]. We alternate which chain we append to, so they are + * time-wise and offset-wise interleaved, but that is an optimization rather + * than for correctness. The log block also includes a pointer to the + * previous block in its chain. + * + * *) We reserve SPA_MINBLOCKSIZE of space at the start of each L2ARC device + * for our header bookkeeping purposes. This contains a device header, + * which contains our top-level reference structures. We update it each + * time we write a new log block, so that we're able to locate it in the + * L2ARC device. If this write results in an inconsistent device header + * (e.g. due to power failure), we detect this by verifying the header's + * checksum and simply fail to reconstruct the L2ARC after reboot. + * + * Implementation diagram: + * + * +=== L2ARC device (not to scale) ======================================+ + * | ___two newest log block pointers__.__________ | + * | / \dh_start_lbps[1] | + * | / \ \dh_start_lbps[0]| + * |.___/__. V V | + * ||L2 dev|....|lb |bufs |lb |bufs |lb |bufs |lb |bufs |lb |---(empty)---| + * || hdr| ^ /^ /^ / / | + * |+------+ ...--\-------/ \-----/--\------/ / | + * | \--------------/ \--------------/ | + * +======================================================================+ + * + * As can be seen on the diagram, rather than using a simple linked list, + * we use a pair of linked lists with alternating elements. This is a + * performance enhancement due to the fact that we only find out the + * address of the next log block access once the current block has been + * completely read in. Obviously, this hurts performance, because we'd be + * keeping the device's I/O queue at only a 1 operation deep, thus + * incurring a large amount of I/O round-trip latency. Having two lists + * allows us to fetch two log blocks ahead of where we are currently + * rebuilding L2ARC buffers. + * + * On-device data structures: + * + * L2ARC device header: l2arc_dev_hdr_phys_t + * L2ARC log block: l2arc_log_blk_phys_t + * + * L2ARC reconstruction: + * + * When writing data, we simply write in the standard rotary fashion, + * evicting buffers as we go and simply writing new data over them (writing + * a new log block every now and then). This obviously means that once we + * loop around the end of the device, we will start cutting into an already + * committed log block (and its referenced data buffers), like so: + * + * current write head__ __old tail + * \ / + * V V + * <--|bufs |lb |bufs |lb | |bufs |lb |bufs |lb |--> + * ^ ^^^^^^^^^___________________________________ + * | \ + * <> may overwrite this blk and/or its bufs --' + * + * When importing the pool, we detect this situation and use it to stop + * our scanning process (see l2arc_rebuild). + * + * There is one significant caveat to consider when rebuilding ARC contents + * from an L2ARC device: what about invalidated buffers? Given the above + * construction, we cannot update blocks which we've already written to amend + * them to remove buffers which were invalidated. Thus, during reconstruction, + * we might be populating the cache with buffers for data that's not on the + * main pool anymore, or may have been overwritten! + * + * As it turns out, this isn't a problem. Every arc_read request includes + * both the DVA and, crucially, the birth TXG of the BP the caller is + * looking for. So even if the cache were populated by completely rotten + * blocks for data that had been long deleted and/or overwritten, we'll + * never actually return bad data from the cache, since the DVA with the + * birth TXG uniquely identify a block in space and time - once created, + * a block is immutable on disk. The worst thing we have done is wasted + * some time and memory at l2arc rebuild to reconstruct outdated ARC + * entries that will get dropped from the l2arc as it is being updated + * with new blocks. + * + * L2ARC buffers that have been evicted by l2arc_evict() ahead of the write + * hand are not restored. This is done by saving the offset (in bytes) + * l2arc_evict() has evicted to in the L2ARC device header and taking it + * into account when restoring buffers. */ static boolean_t @@ -7508,10 +7730,12 @@ l2arc_write_size(l2arc_dev_t *dev) * iteration can occur. */ dev_size = dev->l2ad_end - dev->l2ad_start; - if (size >= dev_size) { + if ((size + l2arc_log_blk_overhead(size, dev)) >= dev_size) { cmn_err(CE_NOTE, "l2arc_write_max or l2arc_write_boost " - "exceeds the size of the cache device (guid %llu), " - "resetting them to the default (%d)", + "plus the overhead of log blocks (persistent L2ARC, " + "%llu bytes) exceeds the size of the cache device " + "(guid %llu), resetting them to the default (%d)", + l2arc_log_blk_overhead(size, dev), dev->l2ad_vdev->vdev_guid, L2ARC_WRITE_SIZE); size = l2arc_write_max = l2arc_write_boost = L2ARC_WRITE_SIZE; @@ -7584,10 +7808,10 @@ l2arc_dev_get_next(void) else if (next == first) break; - } while (vdev_is_dead(next->l2ad_vdev)); + } while (vdev_is_dead(next->l2ad_vdev) || next->l2ad_rebuild); /* if we were unable to find any usable vdevs, return NULL */ - if (vdev_is_dead(next->l2ad_vdev)) + if (vdev_is_dead(next->l2ad_vdev) || next->l2ad_rebuild) next = NULL; l2arc_dev_last = next; @@ -7636,12 +7860,14 @@ l2arc_do_free_on_write(void) static void l2arc_write_done(zio_t *zio) { - l2arc_write_callback_t *cb; - l2arc_dev_t *dev; - list_t *buflist; - arc_buf_hdr_t *head, *hdr, *hdr_prev; - kmutex_t *hash_lock; - int64_t bytes_dropped = 0; + l2arc_write_callback_t *cb; + l2arc_lb_abd_buf_t *abd_buf; + l2arc_lb_ptr_buf_t *lb_ptr_buf; + l2arc_dev_t *dev; + list_t *buflist; + arc_buf_hdr_t *head, *hdr, *hdr_prev; + kmutex_t *hash_lock; + int64_t bytes_dropped = 0; cb = zio->io_private; ASSERT3P(cb, !=, NULL); @@ -7738,12 +7964,33 @@ l2arc_write_done(zio_t *zio) mutex_exit(hash_lock); } + /* + * Free the allocated abd buffers for writing the log blocks. + * If the zio failed reclaim the allocated space and remove the + * pointers to these log blocks from the log block pointer list + * of the L2ARC device. + */ + while ((abd_buf = list_remove_tail(&cb->l2wcb_abd_list)) != NULL) { + abd_free(abd_buf->abd); + zio_buf_free(abd_buf, sizeof (*abd_buf)); + if (zio->io_error != 0) { + lb_ptr_buf = list_remove_head(&dev->l2ad_lbptr_list); + bytes_dropped += + L2BLK_GET_PSIZE((lb_ptr_buf->lb_ptr)->lbp_prop); + kmem_free(lb_ptr_buf->lb_ptr, + sizeof (l2arc_log_blkptr_t)); + kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t)); + } + } + list_destroy(&cb->l2wcb_abd_list); + atomic_inc_64(&l2arc_writes_done); list_remove(buflist, head); ASSERT(!HDR_HAS_L1HDR(head)); kmem_cache_free(hdr_l2only_cache, head); mutex_exit(&dev->l2ad_mtx); + ASSERT(dev->l2ad_vdev != NULL); vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0); l2arc_do_free_on_write(); @@ -8028,9 +8275,32 @@ l2arc_sublist_lock(int list_num) return (multilist_sublist_lock(ml, idx)); } +/* + * Calculates the maximum overhead of L2ARC metadata log blocks for a given + * L2ARC write size. l2arc_evict and l2arc_write_buffers need to include this + * overhead in processing to make sure there is enough headroom available + * when writing buffers. + */ +static inline uint64_t +l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev) +{ + if (dev->l2ad_dev_hdr->dh_log_blk_ent == 0) { + return (0); + } else { + uint64_t log_entries = write_sz >> SPA_MINBLOCKSHIFT; + + uint64_t log_blocks = (log_entries + + dev->l2ad_dev_hdr->dh_log_blk_ent - 1) / + dev->l2ad_dev_hdr->dh_log_blk_ent; + + return (vdev_psize_to_asize(dev->l2ad_vdev, + sizeof (l2arc_log_blk_phys_t)) * log_blocks); + } +} + /* * Evict buffers from the device write hand to the distance specified in - * bytes. This distance may span populated buffers, it may span nothing. + * bytes. This distance may span populated buffers, it may span nothing. * This is clearing a region on the L2ARC device ready for writing. * If the 'all' boolean is set, every buffer is evicted. */ @@ -8042,19 +8312,25 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) kmutex_t *hash_lock; uint64_t taddr; boolean_t rerun; + l2arc_lb_ptr_buf_t *lb_ptr_buf, *lb_ptr_buf_prev; buflist = &dev->l2ad_buflist; + /* + * We need to add in the worst case scenario of log block overhead. + */ + distance += l2arc_log_blk_overhead(distance, dev); + top: rerun = B_FALSE; if (dev->l2ad_hand >= (dev->l2ad_end - distance)) { /* * When there is no space to accomodate upcoming writes, - * evict to the end. Then bump the write hand to the start - * and iterate. This iteration does not happen indefinitely - * as we make sure in l2arc_write_size() that when l2ad_hand - * is reset, the write size does not exceed the end of the - * device. + * evict to the end. Then bump the write and evict hands + * to the start and iterate. This iteration does not + * happen indefinitely as we make sure in + * l2arc_write_size() that when the write hand is reset, + * the write size does not exceed the end of the device. */ rerun = B_TRUE; taddr = dev->l2ad_end; @@ -8064,16 +8340,57 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist, uint64_t, taddr, boolean_t, all); + /* + * This check has to be placed after deciding whether to iterate + * (rerun). + */ if (!all && dev->l2ad_first) { /* - * This is the first sweep through the device. There is + * This is the first sweep through the device. There is * nothing to evict. */ goto out; } + /* + * When rebuilding L2ARC we retrieve the evict hand from the header of + * the device. Of note, l2arc_evict() does not actually delete buffers + * from the cache device, but keeping track of the evict hand will be + * useful when TRIM is implemented. + */ + dev->l2ad_evict = MAX(dev->l2ad_evict, taddr); + retry: mutex_enter(&dev->l2ad_mtx); + /* + * We have to account for evicted log blocks. Run vdev_space_update() + * on log blocks whose offset (in bytes) is before the evicted offset + * (in bytes) by searching in the list of pointers to log blocks + * present in the L2ARC device. + */ + for (lb_ptr_buf = list_tail(&dev->l2ad_lbptr_list); lb_ptr_buf; + lb_ptr_buf = lb_ptr_buf_prev) { + + lb_ptr_buf_prev = list_prev(&dev->l2ad_lbptr_list, lb_ptr_buf); + + /* + * We don't worry about log blocks left behind (ie + * lbp_daddr + psize < l2ad_hand) because l2arc_write_buffers() + * will never write more than l2arc_evict() evicts. + */ + if (!all && l2arc_log_blkptr_valid(dev, lb_ptr_buf->lb_ptr)) { + break; + } else { + vdev_space_update(dev->l2ad_vdev, + -L2BLK_GET_PSIZE( + (lb_ptr_buf->lb_ptr)->lbp_prop), 0, 0); + list_remove(&dev->l2ad_lbptr_list, lb_ptr_buf); + kmem_free(lb_ptr_buf->lb_ptr, + sizeof (l2arc_log_blkptr_t)); + kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t)); + } + } + for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) { hdr_prev = list_prev(buflist, hdr); @@ -8105,7 +8422,7 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) ASSERT(!HDR_L2_WRITING(hdr)); ASSERT(!HDR_L2_WRITE_HEAD(hdr)); - if (!all && (hdr->b_l2hdr.b_daddr >= taddr || + if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict || hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) { /* * We've evicted to the target address, @@ -8144,12 +8461,17 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) mutex_exit(&dev->l2ad_mtx); out: - if (rerun) { + /* + * We need to check if we evict all buffers, otherwise we may iterate + * unnecessarily. + */ + if (!all && rerun) { /* * Bump device hand to the device start if it is approaching the * end. l2arc_evict() has already evicted ahead for this case. */ dev->l2ad_hand = dev->l2ad_start; + dev->l2ad_evict = dev->l2ad_start; dev->l2ad_first = B_FALSE; goto top; } @@ -8272,6 +8594,17 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize, return (ret); } +static void +l2arc_blk_fetch_done(zio_t *zio) +{ + l2arc_read_callback_t *cb; + + cb = zio->io_private; + if (cb->l2rcb_abd != NULL) + abd_put(cb->l2rcb_abd); + kmem_free(cb, sizeof (l2arc_read_callback_t)); +} + /* * Find and write ARC buffers to the L2ARC device. * @@ -8281,17 +8614,18 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize, * state between calls to this function. * * Returns the number of bytes actually written (which may be smaller than - * the delta by which the device hand has changed due to alignment). + * the delta by which the device hand has changed due to alignment and the + * writing of log blocks). */ static uint64_t l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) { - arc_buf_hdr_t *hdr, *hdr_prev, *head; - uint64_t write_asize, write_psize, write_lsize, headroom; - boolean_t full; - l2arc_write_callback_t *cb; - zio_t *pio, *wzio; - uint64_t guid = spa_load_guid(spa); + arc_buf_hdr_t *hdr, *hdr_prev, *head; + uint64_t write_asize, write_psize, write_lsize, headroom; + boolean_t full; + l2arc_write_callback_t *cb = NULL; + zio_t *pio, *wzio; + uint64_t guid = spa_load_guid(spa); ASSERT3P(dev->l2ad_vdev, !=, NULL); @@ -8343,7 +8677,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) } passed_sz += HDR_GET_LSIZE(hdr); - if (passed_sz > headroom) { + if (l2arc_headroom != 0 && passed_sz > headroom) { /* * Searched too far. */ @@ -8443,6 +8777,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) sizeof (l2arc_write_callback_t), KM_SLEEP); cb->l2wcb_dev = dev; cb->l2wcb_head = head; + list_create(&cb->l2wcb_abd_list, + sizeof (l2arc_lb_abd_buf_t), + offsetof(l2arc_lb_abd_buf_t, node)); pio = zio_root(spa, l2arc_write_done, cb, ZIO_FLAG_CANFAIL); } @@ -8477,6 +8814,14 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) mutex_exit(hash_lock); + /* + * Append buf info to current log and commit if full. + * arcstat_l2_{size,asize} kstats are updated + * internally. + */ + if (l2arc_log_blk_insert(dev, hdr)) + l2arc_log_blk_commit(dev, pio, cb); + zio_nowait(wzio); } @@ -8491,6 +8836,13 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) ASSERT0(write_lsize); ASSERT(!HDR_HAS_L1HDR(head)); kmem_cache_free(hdr_l2only_cache, head); + + /* + * Although we did not write any buffers l2ad_evict may + * have advanced. + */ + l2arc_dev_hdr_update(dev); + return (0); } @@ -8500,6 +8852,8 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) ARCSTAT_INCR(arcstat_l2_lsize, write_lsize); ARCSTAT_INCR(arcstat_l2_psize, write_psize); + l2arc_dev_hdr_update(dev); + dev->l2ad_writing = B_TRUE; (void) zio_wait(pio); dev->l2ad_writing = B_FALSE; @@ -8611,7 +8965,17 @@ l2arc_feed_thread(void *unused) boolean_t l2arc_vdev_present(vdev_t *vd) { - l2arc_dev_t *dev; + return (l2arc_vdev_get(vd) != NULL); +} + +/* + * Returns the l2arc_dev_t associated with a particular vdev_t or NULL if + * the vdev_t isn't an L2ARC device. + */ +static l2arc_dev_t * +l2arc_vdev_get(vdev_t *vd) +{ + l2arc_dev_t *dev; mutex_enter(&l2arc_dev_mtx); for (dev = list_head(l2arc_dev_list); dev != NULL; @@ -8621,7 +8985,7 @@ l2arc_vdev_present(vdev_t *vd) } mutex_exit(&l2arc_dev_mtx); - return (dev != NULL); + return (dev); } /* @@ -8631,22 +8995,29 @@ l2arc_vdev_present(vdev_t *vd) void l2arc_add_vdev(spa_t *spa, vdev_t *vd) { - l2arc_dev_t *adddev; + l2arc_dev_t *adddev; + uint64_t l2dhdr_asize; ASSERT(!l2arc_vdev_present(vd)); /* * Create a new l2arc device entry. */ - adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP); + adddev = vmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP); adddev->l2ad_spa = spa; adddev->l2ad_vdev = vd; - adddev->l2ad_start = VDEV_LABEL_START_SIZE; + /* leave extra size for an l2arc device header */ + l2dhdr_asize = adddev->l2ad_dev_hdr_asize = + MAX(sizeof (*adddev->l2ad_dev_hdr), 1 << vd->vdev_ashift); + adddev->l2ad_start = VDEV_LABEL_START_SIZE + l2dhdr_asize; adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd); + ASSERT3U(adddev->l2ad_start, <, adddev->l2ad_end); adddev->l2ad_hand = adddev->l2ad_start; + adddev->l2ad_evict = adddev->l2ad_start; adddev->l2ad_first = B_TRUE; adddev->l2ad_writing = B_FALSE; list_link_init(&adddev->l2ad_node); + adddev->l2ad_dev_hdr = kmem_zalloc(l2dhdr_asize, KM_SLEEP); mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL); /* @@ -8656,6 +9027,13 @@ l2arc_add_vdev(spa_t *spa, vdev_t *vd) list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node)); + /* + * This is a list of pointers to log blocks that are still present + * on the device. + */ + list_create(&adddev->l2ad_lbptr_list, sizeof (l2arc_lb_ptr_buf_t), + offsetof(l2arc_lb_ptr_buf_t, node)); + vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand); zfs_refcount_create(&adddev->l2ad_alloc); @@ -8666,6 +9044,89 @@ l2arc_add_vdev(spa_t *spa, vdev_t *vd) list_insert_head(l2arc_dev_list, adddev); atomic_inc_64(&l2arc_ndev); mutex_exit(&l2arc_dev_mtx); + + /* + * Decide if vdev is eligible for L2ARC rebuild + */ + l2arc_rebuild_vdev(adddev->l2ad_vdev, B_FALSE); +} + +void +l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen) +{ + l2arc_dev_t *dev = NULL; + l2arc_dev_hdr_phys_t *l2dhdr; + uint64_t l2dhdr_asize; + spa_t *spa; + int err; + boolean_t rebuild = B_TRUE; + + dev = l2arc_vdev_get(vd); + ASSERT3P(dev, !=, NULL); + spa = dev->l2ad_spa; + l2dhdr = dev->l2ad_dev_hdr; + l2dhdr_asize = dev->l2ad_dev_hdr_asize; + + /* + * The L2ARC has to hold at least the payload of one log block for + * them to be restored (persistent L2ARC). The payload of a log block + * depends on the amount of its log entries. We always write log blocks + * with 1022 entries. How many of them are committed or restored depends + * on the size of the L2ARC device. Thus the maximum payload of + * one log block is 1022 * SPA_MAXBLOCKSIZE = 16GB. If the L2ARC device + * is less than that, we reduce the amount of committed and restored + * log entries per block so as to enable persistence. + */ + if (dev->l2ad_end < l2arc_rebuild_blocks_min_l2size) { + dev->l2ad_log_entries = 0; + } else { + dev->l2ad_log_entries = MIN((dev->l2ad_end - + dev->l2ad_start) >> SPA_MAXBLOCKSHIFT, + L2ARC_LOG_BLK_MAX_ENTRIES); + } + + /* + * Read the device header, if an error is returned do not rebuild L2ARC. + */ + if ((err = l2arc_dev_hdr_read(dev)) != 0) + rebuild = B_FALSE; + + if (rebuild && l2dhdr->dh_log_blk_ent > 0) { + /* + * If we are onlining a cache device (vdev_reopen) that was + * still present (l2arc_vdev_present()) and rebuild is enabled, + * we should evict all ARC buffers and pointers to log blocks + * and reclaim their space before restoring its contents to + * L2ARC. + */ + if (reopen) { + if (!l2arc_rebuild_enabled) { + return; + } else { + l2arc_evict(dev, 0, B_TRUE); + /* start a new log block */ + dev->l2ad_log_ent_idx = 0; + dev->l2ad_log_blk_payload_asize = 0; + dev->l2ad_log_blk_payload_start = 0; + } + } + /* + * Just mark the device as pending for a rebuild. We won't + * be starting a rebuild in line here as it would block pool + * import. Instead spa_load_impl will hand that off to an + * async task which will call l2arc_spa_rebuild_start. + */ + dev->l2ad_rebuild = B_TRUE; + } else if (!rebuild && spa_writeable(spa)) { + /* + * The boolean rebuild is false if reading the device header + * returned an error. In this case create a new header. We + * zero out the memory holding the header to reset + * dh_start_lbps. + */ + bzero(l2dhdr, l2dhdr_asize); + l2arc_dev_hdr_update(dev); + } } /* @@ -8674,24 +9135,29 @@ l2arc_add_vdev(spa_t *spa, vdev_t *vd) void l2arc_remove_vdev(vdev_t *vd) { - l2arc_dev_t *dev, *nextdev, *remdev = NULL; + l2arc_dev_t *remdev = NULL; /* * Find the device by vdev */ - mutex_enter(&l2arc_dev_mtx); - for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) { - nextdev = list_next(l2arc_dev_list, dev); - if (vd == dev->l2ad_vdev) { - remdev = dev; - break; - } - } + remdev = l2arc_vdev_get(vd); ASSERT3P(remdev, !=, NULL); + /* + * Cancel any ongoing or scheduled rebuild. + */ + mutex_enter(&l2arc_rebuild_thr_lock); + if (remdev->l2ad_rebuild_began == B_TRUE) { + remdev->l2ad_rebuild_cancel = B_TRUE; + while (remdev->l2ad_rebuild == B_TRUE) + cv_wait(&l2arc_rebuild_thr_cv, &l2arc_rebuild_thr_lock); + } + mutex_exit(&l2arc_rebuild_thr_lock); + /* * Remove device from global list */ + mutex_enter(&l2arc_dev_mtx); list_remove(l2arc_dev_list, remdev); l2arc_dev_last = NULL; /* may have been invalidated */ atomic_dec_64(&l2arc_ndev); @@ -8702,9 +9168,12 @@ l2arc_remove_vdev(vdev_t *vd) */ l2arc_evict(remdev, 0, B_TRUE); list_destroy(&remdev->l2ad_buflist); + ASSERT(list_is_empty(&remdev->l2ad_lbptr_list)); + list_destroy(&remdev->l2ad_lbptr_list); mutex_destroy(&remdev->l2ad_mtx); zfs_refcount_destroy(&remdev->l2ad_alloc); - kmem_free(remdev, sizeof (l2arc_dev_t)); + kmem_free(remdev->l2ad_dev_hdr, remdev->l2ad_dev_hdr_asize); + vmem_free(remdev, sizeof (l2arc_dev_t)); } void @@ -8717,6 +9186,8 @@ l2arc_init(void) mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL); + mutex_init(&l2arc_rebuild_thr_lock, NULL, MUTEX_DEFAULT, NULL); + cv_init(&l2arc_rebuild_thr_cv, NULL, CV_DEFAULT, NULL); mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL); mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL); @@ -8741,6 +9212,8 @@ l2arc_fini(void) mutex_destroy(&l2arc_feed_thr_lock); cv_destroy(&l2arc_feed_thr_cv); + mutex_destroy(&l2arc_rebuild_thr_lock); + cv_destroy(&l2arc_rebuild_thr_cv); mutex_destroy(&l2arc_dev_mtx); mutex_destroy(&l2arc_free_on_write_mtx); @@ -8772,6 +9245,865 @@ l2arc_stop(void) mutex_exit(&l2arc_feed_thr_lock); } +/* + * Punches out rebuild threads for the L2ARC devices in a spa. This should + * be called after pool import from the spa async thread, since starting + * these threads directly from spa_import() will make them part of the + * "zpool import" context and delay process exit (and thus pool import). + */ +void +l2arc_spa_rebuild_start(spa_t *spa) +{ + ASSERT(MUTEX_HELD(&spa_namespace_lock)); + + /* + * Locate the spa's l2arc devices and kick off rebuild threads. + */ + for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { + l2arc_dev_t *dev = + l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]); + if (dev == NULL) { + /* Don't attempt a rebuild if the vdev is UNAVAIL */ + continue; + } + mutex_enter(&l2arc_rebuild_thr_lock); + if (dev->l2ad_rebuild && !dev->l2ad_rebuild_cancel) { + dev->l2ad_rebuild_began = B_TRUE; + (void) thread_create(NULL, 0, + (void (*)(void *))l2arc_dev_rebuild_start, + dev, 0, &p0, TS_RUN, minclsyspri); + } + mutex_exit(&l2arc_rebuild_thr_lock); + } +} + +/* + * Main entry point for L2ARC rebuilding. + */ +static void +l2arc_dev_rebuild_start(l2arc_dev_t *dev) +{ + VERIFY(!dev->l2ad_rebuild_cancel); + VERIFY(dev->l2ad_rebuild); + (void) l2arc_rebuild(dev); + mutex_enter(&l2arc_rebuild_thr_lock); + dev->l2ad_rebuild_began = B_FALSE; + dev->l2ad_rebuild = B_FALSE; + mutex_exit(&l2arc_rebuild_thr_lock); + + thread_exit(); +} + +/* + * This function implements the actual L2ARC metadata rebuild. It: + * starts reading the log block chain and restores each block's contents + * to memory (reconstructing arc_buf_hdr_t's). + * + * Operation stops under any of the following conditions: + * + * 1) We reach the end of the log block chain. + * 2) We encounter *any* error condition (cksum errors, io errors) + */ +static int +l2arc_rebuild(l2arc_dev_t *dev) +{ + vdev_t *vd = dev->l2ad_vdev; + spa_t *spa = vd->vdev_spa; + int i = 0, err = 0; + l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr; + l2arc_log_blk_phys_t *this_lb, *next_lb; + zio_t *this_io = NULL, *next_io = NULL; + l2arc_log_blkptr_t lbps[2]; + l2arc_lb_ptr_buf_t *lb_ptr_buf; + boolean_t lock_held; + + this_lb = vmem_zalloc(sizeof (*this_lb), KM_SLEEP); + next_lb = vmem_zalloc(sizeof (*next_lb), KM_SLEEP); + + /* + * We prevent device removal while issuing reads to the device, + * then during the rebuilding phases we drop this lock again so + * that a spa_unload or device remove can be initiated - this is + * safe, because the spa will signal us to stop before removing + * our device and wait for us to stop. + */ + spa_config_enter(spa, SCL_L2ARC, vd, RW_READER); + lock_held = B_TRUE; + + /* + * Retrieve the persistent L2ARC device state. + */ + dev->l2ad_evict = MAX(l2dhdr->dh_evict, dev->l2ad_start); + dev->l2ad_hand = MAX(l2dhdr->dh_start_lbps[0].lbp_daddr + + L2BLK_GET_PSIZE((&l2dhdr->dh_start_lbps[0])->lbp_prop), + dev->l2ad_start); + dev->l2ad_first = !!(l2dhdr->dh_flags & L2ARC_DEV_HDR_EVICT_FIRST); + + /* + * In case the zfs module parameter l2arc_rebuild_enabled is false + * we do not start the rebuild process. + */ + if (!l2arc_rebuild_enabled) + goto out; + + /* Prepare the rebuild process */ + bcopy(l2dhdr->dh_start_lbps, lbps, sizeof (lbps)); + + /* Start the rebuild process */ + for (;;) { + if (!l2arc_log_blkptr_valid(dev, &lbps[0])) + break; + + if ((err = l2arc_log_blk_read(dev, &lbps[0], &lbps[1], + this_lb, next_lb, this_io, &next_io)) != 0) + goto out; + + /* + * Our memory pressure valve. If the system is running low + * on memory, rather than swamping memory with new ARC buf + * hdrs, we opt not to rebuild the L2ARC. At this point, + * however, we have already set up our L2ARC dev to chain in + * new metadata log blocks, so the user may choose to offline/ + * online the L2ARC dev at a later time (or re-import the pool) + * to reconstruct it (when there's less memory pressure). + */ + if (arc_reclaim_needed()) { + ARCSTAT_BUMP(arcstat_l2_rebuild_abort_lowmem); + cmn_err(CE_NOTE, "System running low on memory, " + "aborting L2ARC rebuild."); + err = SET_ERROR(ENOMEM); + goto out; + } + + spa_config_exit(spa, SCL_L2ARC, vd); + lock_held = B_FALSE; + + /* + * Now that we know that the next_lb checks out alright, we + * can start reconstruction from this log block. + */ + l2arc_log_blk_restore(dev, this_lb, + L2BLK_GET_PSIZE((&lbps[0])->lbp_prop), + lbps[0].lbp_daddr); + i++; + + /* + * log block restored, include its pointer in the list of + * pointers to log blocks present in the L2ARC device. + */ + lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP); + lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t), + KM_SLEEP); + bcopy(&lbps[0], lb_ptr_buf->lb_ptr, + sizeof (l2arc_log_blkptr_t)); + mutex_enter(&dev->l2ad_mtx); + list_insert_tail(&dev->l2ad_lbptr_list, lb_ptr_buf); + mutex_exit(&dev->l2ad_mtx); + vdev_space_update(vd, + L2BLK_GET_PSIZE((&lbps[0])->lbp_prop), 0, 0); + + /* + * Protection against loops of log blocks: + * + * l2ad_hand l2ad_evict + * V V + * l2ad_start |=======================================| l2ad_end + * -----|||----|||---|||----||| + * (3) (2) (1) (0) + * ---|||---|||----|||---||| + * (7) (6) (5) (4) + * + * In this situation the pointer of log block (4) passes + * l2arc_log_blkptr_valid() but the log block should not be + * restored as it is overwritten by the payload of log block + * (0). Only log blocks (0)-(3) should be restored. We check + * whether l2ad_evict lies in between the next log block + * offset (lbps[1].lbp_daddr) and the present log block offset + * (lbps[0].lbp_daddr). If true and this isn't the first pass, + * we are looping from the beginning and we should stop. + */ + if (l2arc_range_check_overlap(lbps[1].lbp_daddr, + lbps[0].lbp_daddr, dev->l2ad_evict) && !dev->l2ad_first) + goto out; + + for (;;) { + mutex_enter(&l2arc_rebuild_thr_lock); + if (dev->l2ad_rebuild_cancel) { + dev->l2ad_rebuild = B_FALSE; + cv_signal(&l2arc_rebuild_thr_cv); + mutex_exit(&l2arc_rebuild_thr_lock); + err = SET_ERROR(ECANCELED); + goto out; + } + mutex_exit(&l2arc_rebuild_thr_lock); + if (spa_config_tryenter(spa, SCL_L2ARC, vd, + RW_READER)) { + lock_held = B_TRUE; + break; + } + /* + * L2ARC config lock held by somebody in writer, + * possibly due to them trying to remove us. They'll + * likely to want us to shut down, so after a little + * delay, we check l2ad_rebuild_cancel and retry + * the lock again. + */ + delay(1); + } + + /* + * Continue with the next log block. + */ + lbps[0] = lbps[1]; + lbps[1] = this_lb->lb_prev_lbp; + PTR_SWAP(this_lb, next_lb); + this_io = next_io; + next_io = NULL; + } + + if (this_io != NULL) + l2arc_log_blk_fetch_abort(this_io); +out: + if (next_io != NULL) + l2arc_log_blk_fetch_abort(next_io); + vmem_free(this_lb, sizeof (*this_lb)); + vmem_free(next_lb, sizeof (*next_lb)); + + if (!l2arc_rebuild_enabled) { + zfs_dbgmsg("L2ARC rebuild disabled"); + } else if (err == 0 && i > 0) { + ARCSTAT_BUMP(arcstat_l2_rebuild_success); + zfs_dbgmsg("L2ARC successfully rebuilt, " + "restored %d blocks", i); + } else if (err != 0) { + zfs_dbgmsg("L2ARC rebuild aborted, " + "restored %d blocks", i); + } + + if (lock_held) + spa_config_exit(spa, SCL_L2ARC, vd); + + return (err); +} + +/* + * Attempts to read the device header on the provided L2ARC device and writes + * it to `hdr'. On success, this function returns 0, otherwise the appropriate + * error code is returned. + */ +static int +l2arc_dev_hdr_read(l2arc_dev_t *dev) +{ + int err; + uint64_t guid; + l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr; + const uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize; + abd_t *abd; + + guid = spa_guid(dev->l2ad_vdev->vdev_spa); + + abd = abd_get_from_buf(l2dhdr, l2dhdr_asize); + + err = zio_wait(zio_read_phys(NULL, dev->l2ad_vdev, + VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, + ZIO_CHECKSUM_LABEL, NULL, NULL, ZIO_PRIORITY_ASYNC_READ, + ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL | + ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY | + ZIO_FLAG_SPECULATIVE, B_FALSE)); + + abd_put(abd); + + if (err != 0) { + ARCSTAT_BUMP(arcstat_l2_rebuild_abort_dh_errors); + zfs_dbgmsg("L2ARC IO error (%d) while reading device header, " + "vdev guid: %llu", err, dev->l2ad_vdev->vdev_guid); + return (err); + } + + if (l2dhdr->dh_magic == BSWAP_64(L2ARC_DEV_HDR_MAGIC)) + byteswap_uint64_array(l2dhdr, sizeof (*l2dhdr)); + + if (l2dhdr->dh_magic != L2ARC_DEV_HDR_MAGIC || + l2dhdr->dh_spa_guid != guid || + l2dhdr->dh_vdev_guid != dev->l2ad_vdev->vdev_guid || + l2dhdr->dh_version != L2ARC_PERSISTENT_VERSION || + l2dhdr->dh_log_blk_ent != dev->l2ad_log_entries || + l2dhdr->dh_end != dev->l2ad_end || + !l2arc_range_check_overlap(dev->l2ad_start, dev->l2ad_end, + l2dhdr->dh_evict)) { + /* + * Attempt to rebuild a device containing no actual dev hdr + * or containing a header from some other pool or from another + * version of persistent L2ARC. + */ + ARCSTAT_BUMP(arcstat_l2_rebuild_abort_unsupported); + return (SET_ERROR(ENOTSUP)); + } + + return (0); +} + +/* + * Reads L2ARC log blocks from storage and validates their contents. + * + * This function implements a simple fetcher to make sure that while + * we're processing one buffer the L2ARC is already fetching the next + * one in the chain. + * + * The arguments this_lp and next_lp point to the current and next log block + * address in the block chain. Similarly, this_lb and next_lb hold the + * l2arc_log_blk_phys_t's of the current and next L2ARC blk. + * + * The `this_io' and `next_io' arguments are used for block fetching. + * When issuing the first blk IO during rebuild, you should pass NULL for + * `this_io'. This function will then issue a sync IO to read the block and + * also issue an async IO to fetch the next block in the block chain. The + * fetched IO is returned in `next_io'. On subsequent calls to this + * function, pass the value returned in `next_io' from the previous call + * as `this_io' and a fresh `next_io' pointer to hold the next fetch IO. + * Prior to the call, you should initialize your `next_io' pointer to be + * NULL. If no fetch IO was issued, the pointer is left set at NULL. + * + * On success, this function returns 0, otherwise it returns an appropriate + * error code. On error the fetching IO is aborted and cleared before + * returning from this function. Therefore, if we return `success', the + * caller can assume that we have taken care of cleanup of fetch IOs. + */ +static int +l2arc_log_blk_read(l2arc_dev_t *dev, + const l2arc_log_blkptr_t *this_lbp, const l2arc_log_blkptr_t *next_lbp, + l2arc_log_blk_phys_t *this_lb, l2arc_log_blk_phys_t *next_lb, + zio_t *this_io, zio_t **next_io) +{ + int err = 0; + zio_cksum_t cksum; + abd_t *abd = NULL; + + ASSERT(this_lbp != NULL && next_lbp != NULL); + ASSERT(this_lb != NULL && next_lb != NULL); + ASSERT(next_io != NULL && *next_io == NULL); + ASSERT(l2arc_log_blkptr_valid(dev, this_lbp)); + + /* + * Check to see if we have issued the IO for this log block in a + * previous run. If not, this is the first call, so issue it now. + */ + if (this_io == NULL) { + this_io = l2arc_log_blk_fetch(dev->l2ad_vdev, this_lbp, + this_lb); + } + + /* + * Peek to see if we can start issuing the next IO immediately. + */ + if (l2arc_log_blkptr_valid(dev, next_lbp)) { + /* + * Start issuing IO for the next log block early - this + * should help keep the L2ARC device busy while we + * decompress and restore this log block. + */ + *next_io = l2arc_log_blk_fetch(dev->l2ad_vdev, next_lbp, + next_lb); + } + + /* Wait for the IO to read this log block to complete */ + if ((err = zio_wait(this_io)) != 0) { + ARCSTAT_BUMP(arcstat_l2_rebuild_abort_io_errors); + zfs_dbgmsg("L2ARC IO error (%d) while reading log block, " + "offset: %llu, vdev guid: %llu", err, this_lbp->lbp_daddr, + dev->l2ad_vdev->vdev_guid); + goto cleanup; + } + + /* Make sure the buffer checks out */ + fletcher_4_native(this_lb, + L2BLK_GET_PSIZE((this_lbp)->lbp_prop), NULL, &cksum); + if (!ZIO_CHECKSUM_EQUAL(cksum, this_lbp->lbp_cksum)) { + ARCSTAT_BUMP(arcstat_l2_rebuild_abort_cksum_lb_errors); + zfs_dbgmsg("L2ARC log block cksum failed, offset: %llu, " + "vdev guid: %llu, l2ad_hand: %llu, l2ad_evict: %llu", + this_lbp->lbp_daddr, dev->l2ad_vdev->vdev_guid, + dev->l2ad_hand, dev->l2ad_evict); + err = SET_ERROR(ECKSUM); + goto cleanup; + } + + /* Now we can take our time decoding this buffer */ + switch (L2BLK_GET_COMPRESS((this_lbp)->lbp_prop)) { + case ZIO_COMPRESS_OFF: + break; + case ZIO_COMPRESS_LZ4: + abd = abd_alloc_for_io(L2BLK_GET_PSIZE( + (this_lbp)->lbp_prop), B_TRUE); + abd_copy_from_buf_off(abd, this_lb, 0, + L2BLK_GET_PSIZE((this_lbp)->lbp_prop)); + if ((err = zio_decompress_data( + L2BLK_GET_COMPRESS((this_lbp)->lbp_prop), + abd, this_lb, L2BLK_GET_PSIZE((this_lbp)->lbp_prop), + sizeof (*this_lb))) != 0) { + err = SET_ERROR(EINVAL); + goto cleanup; + } + break; + default: + err = SET_ERROR(EINVAL); + goto cleanup; + } + if (this_lb->lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC)) + byteswap_uint64_array(this_lb, sizeof (*this_lb)); + if (this_lb->lb_magic != L2ARC_LOG_BLK_MAGIC) { + err = SET_ERROR(EINVAL); + goto cleanup; + } +cleanup: + /* Abort an in-flight fetch I/O in case of error */ + if (err != 0 && *next_io != NULL) { + l2arc_log_blk_fetch_abort(*next_io); + *next_io = NULL; + } + if (abd != NULL) + abd_free(abd); + return (err); +} + +/* + * Restores the payload of a log block to ARC. This creates empty ARC hdr + * entries which only contain an l2arc hdr, essentially restoring the + * buffers to their L2ARC evicted state. This function also updates space + * usage on the L2ARC vdev to make sure it tracks restored buffers. + */ +static void +l2arc_log_blk_restore(l2arc_dev_t *dev, const l2arc_log_blk_phys_t *lb, + uint64_t lb_psize, uint64_t lb_daddr) +{ + uint64_t size = 0, psize = 0; + uint64_t log_entries = dev->l2ad_dev_hdr->dh_log_blk_ent; + + for (int i = log_entries - 1; i >= 0; i--) { + /* + * Restore goes in the reverse temporal direction to preserve + * correct temporal ordering of buffers in the l2ad_buflist. + * l2arc_hdr_restore also does a list_insert_tail instead of + * list_insert_head on the l2ad_buflist: + * + * LIST l2ad_buflist LIST + * HEAD <------ (time) ------ TAIL + * direction +-----+-----+-----+-----+-----+ direction + * of l2arc <== | buf | buf | buf | buf | buf | ===> of rebuild + * fill +-----+-----+-----+-----+-----+ + * ^ ^ + * | | + * | | + * l2arc_fill_thread l2arc_rebuild + * places new bufs here restores bufs here + * + * This also works when the restored bufs get evicted at any + * point during the rebuild. + */ + size += L2BLK_GET_LSIZE((&lb->lb_entries[i])->le_prop); + psize += L2BLK_GET_PSIZE((&lb->lb_entries[i])->le_prop); + l2arc_hdr_restore(&lb->lb_entries[i], dev); + } + + /* + * Record rebuild stats: + * size Logical size of restored buffers in the L2ARC + * psize Physical size of restored buffers in the L2ARC + */ + ARCSTAT_INCR(arcstat_l2_rebuild_size, size); + ARCSTAT_INCR(arcstat_l2_rebuild_psize, psize); + ARCSTAT_INCR(arcstat_l2_rebuild_bufs, log_entries); + ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_size, lb_psize); + ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio, psize / lb_psize); + ARCSTAT_BUMP(arcstat_l2_rebuild_log_blks); +} + +/* + * Restores a single ARC buf hdr from a log entry. The ARC buffer is put + * into a state indicating that it has been evicted to L2ARC. + */ +static void +l2arc_hdr_restore(const l2arc_log_ent_phys_t *le, l2arc_dev_t *dev) +{ + arc_buf_hdr_t *hdr, *exists; + kmutex_t *hash_lock; + arc_buf_contents_t type = L2BLK_GET_TYPE((le)->le_prop); + uint64_t asize; + + /* + * Do all the allocation before grabbing any locks, this lets us + * sleep if memory is full and we don't have to deal with failed + * allocations. + */ + hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type, + dev, le->le_dva, le->le_daddr, + L2BLK_GET_PSIZE((le)->le_prop), le->le_birth, + L2BLK_GET_COMPRESS((le)->le_prop), + L2BLK_GET_PROTECTED((le)->le_prop), + L2BLK_GET_PREFETCH((le)->le_prop)); + asize = vdev_psize_to_asize(dev->l2ad_vdev, + L2BLK_GET_PSIZE((le)->le_prop)); + + /* + * vdev_space_update() has to be called before arc_hdr_destroy() to + * avoid underflow since the latter also calls the former. + */ + vdev_space_update(dev->l2ad_vdev, asize, 0, 0); + + ARCSTAT_INCR(arcstat_l2_lsize, HDR_GET_LSIZE(hdr)); + ARCSTAT_INCR(arcstat_l2_psize, HDR_GET_PSIZE(hdr)); + + mutex_enter(&dev->l2ad_mtx); + list_insert_tail(&dev->l2ad_buflist, hdr); + (void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr); + mutex_exit(&dev->l2ad_mtx); + + exists = buf_hash_insert(hdr, &hash_lock); + if (exists) { + /* Buffer was already cached, no need to restore it. */ + arc_hdr_destroy(hdr); + /* + * If the buffer is already cached, check whether it has + * L2ARC metadata. If not, enter them and update the flag. + * This is important is case of onlining a cache device, since + * we previously evicted all L2ARC metadata from ARC. + */ + if (!HDR_HAS_L2HDR(exists)) { + arc_hdr_set_flags(exists, ARC_FLAG_HAS_L2HDR); + exists->b_l2hdr.b_dev = dev; + exists->b_l2hdr.b_daddr = le->le_daddr; + mutex_enter(&dev->l2ad_mtx); + list_insert_tail(&dev->l2ad_buflist, exists); + (void) zfs_refcount_add_many(&dev->l2ad_alloc, + arc_hdr_size(exists), exists); + mutex_exit(&dev->l2ad_mtx); + vdev_space_update(dev->l2ad_vdev, asize, 0, 0); + ARCSTAT_INCR(arcstat_l2_lsize, HDR_GET_LSIZE(exists)); + ARCSTAT_INCR(arcstat_l2_psize, HDR_GET_PSIZE(exists)); + } + ARCSTAT_BUMP(arcstat_l2_rebuild_bufs_precached); + } + + mutex_exit(hash_lock); +} + +/* + * Starts an asynchronous read IO to read a log block. This is used in log + * block reconstruction to start reading the next block before we are done + * decoding and reconstructing the current block, to keep the l2arc device + * nice and hot with read IO to process. + * The returned zio will contain a newly allocated memory buffers for the IO + * data which should then be freed by the caller once the zio is no longer + * needed (i.e. due to it having completed). If you wish to abort this + * zio, you should do so using l2arc_log_blk_fetch_abort, which takes + * care of disposing of the allocated buffers correctly. + */ +static zio_t * +l2arc_log_blk_fetch(vdev_t *vd, const l2arc_log_blkptr_t *lbp, + l2arc_log_blk_phys_t *lb) +{ + uint32_t psize; + zio_t *pio; + l2arc_read_callback_t *cb; + + psize = L2BLK_GET_PSIZE((lbp)->lbp_prop); + ASSERT(psize <= sizeof (l2arc_log_blk_phys_t)); + cb = kmem_zalloc(sizeof (l2arc_read_callback_t), KM_SLEEP); + cb->l2rcb_abd = abd_get_from_buf(lb, psize); + pio = zio_root(vd->vdev_spa, l2arc_blk_fetch_done, cb, + ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | + ZIO_FLAG_DONT_RETRY); + (void) zio_nowait(zio_read_phys(pio, vd, lbp->lbp_daddr, psize, + cb->l2rcb_abd, ZIO_CHECKSUM_OFF, NULL, NULL, + ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL | + ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY, B_FALSE)); + + return (pio); +} + +/* + * Aborts a zio returned from l2arc_log_blk_fetch and frees the data + * buffers allocated for it. + */ +static void +l2arc_log_blk_fetch_abort(zio_t *zio) +{ + (void) zio_wait(zio); +} + +/* + * Creates a zio to update the device header on an l2arc device. The zio is + * initiated as a child of `pio'. + */ +static void +l2arc_dev_hdr_update(l2arc_dev_t *dev) +{ + l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr; + const uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize; + abd_t *abd; + int err; + + l2dhdr->dh_magic = L2ARC_DEV_HDR_MAGIC; + l2dhdr->dh_version = L2ARC_PERSISTENT_VERSION; + l2dhdr->dh_spa_guid = spa_guid(dev->l2ad_vdev->vdev_spa); + l2dhdr->dh_vdev_guid = dev->l2ad_vdev->vdev_guid; + l2dhdr->dh_log_blk_ent = dev->l2ad_log_entries; + l2dhdr->dh_evict = dev->l2ad_evict; + l2dhdr->dh_start = dev->l2ad_start; + l2dhdr->dh_end = dev->l2ad_end; + l2dhdr->dh_flags = 0; + if (dev->l2ad_first) + l2dhdr->dh_flags |= L2ARC_DEV_HDR_EVICT_FIRST; + + abd = abd_get_from_buf(l2dhdr, l2dhdr_asize); + + err = zio_wait(zio_write_phys(NULL, dev->l2ad_vdev, + VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, ZIO_CHECKSUM_LABEL, NULL, + NULL, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE)); + + abd_put(abd); + + if (err != 0) { + zfs_dbgmsg("L2ARC IO error (%d) while writing device header, " + "vdev guid: %llu", err, dev->l2ad_vdev->vdev_guid); + } +} + +/* + * Commits a log block to the L2ARC device. This routine is invoked from + * l2arc_write_buffers when the log block fills up. + * This function allocates some memory to temporarily hold the serialized + * buffer to be written. This is then released in l2arc_write_done. + */ +static void +l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb) +{ + l2arc_log_blk_phys_t *lb = &dev->l2ad_log_blk; + l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr; + uint64_t psize, asize; + zio_t *wzio; + l2arc_lb_abd_buf_t *abd_buf; + uint8_t *tmpbuf; + l2arc_lb_ptr_buf_t *lb_ptr_buf; + + VERIFY3S(dev->l2ad_log_ent_idx, ==, l2dhdr->dh_log_blk_ent); + + tmpbuf = zio_buf_alloc(sizeof (*lb)); + abd_buf = zio_buf_alloc(sizeof (*abd_buf)); + abd_buf->abd = abd_get_from_buf(lb, sizeof (*lb)); + lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP); + lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t), KM_SLEEP); + + /* link the buffer into the block chain */ + lb->lb_prev_lbp = l2dhdr->dh_start_lbps[1]; + lb->lb_magic = L2ARC_LOG_BLK_MAGIC; + + /* try to compress the buffer */ + list_insert_tail(&cb->l2wcb_abd_list, abd_buf); + psize = zio_compress_data(ZIO_COMPRESS_LZ4, + abd_buf->abd, tmpbuf, sizeof (*lb)); + + /* a log block is never entirely zero */ + ASSERT(psize != 0); + asize = vdev_psize_to_asize(dev->l2ad_vdev, psize); + ASSERT(asize <= sizeof (*lb)); + + /* + * Update the start log block pointer in the device header to point + * to the log block we're about to write. + */ + l2dhdr->dh_start_lbps[1] = l2dhdr->dh_start_lbps[0]; + l2dhdr->dh_start_lbps[0].lbp_daddr = dev->l2ad_hand; + l2dhdr->dh_start_lbps[0].lbp_payload_asize = + dev->l2ad_log_blk_payload_asize; + l2dhdr->dh_start_lbps[0].lbp_payload_start = + dev->l2ad_log_blk_payload_start; + _NOTE(CONSTCOND) + L2BLK_SET_LSIZE( + (&l2dhdr->dh_start_lbps[0])->lbp_prop, sizeof (*lb)); + L2BLK_SET_PSIZE( + (&l2dhdr->dh_start_lbps[0])->lbp_prop, asize); + L2BLK_SET_CHECKSUM( + (&l2dhdr->dh_start_lbps[0])->lbp_prop, + ZIO_CHECKSUM_FLETCHER_4); + if (asize < sizeof (*lb)) { + /* compression succeeded */ + bzero(tmpbuf + psize, asize - psize); + L2BLK_SET_COMPRESS( + (&l2dhdr->dh_start_lbps[0])->lbp_prop, + ZIO_COMPRESS_LZ4); + } else { + /* compression failed */ + bcopy(lb, tmpbuf, sizeof (*lb)); + L2BLK_SET_COMPRESS( + (&l2dhdr->dh_start_lbps[0])->lbp_prop, + ZIO_COMPRESS_OFF); + } + + /* checksum what we're about to write */ + fletcher_4_native(tmpbuf, asize, NULL, + &l2dhdr->dh_start_lbps[0].lbp_cksum); + + abd_put(abd_buf->abd); + + /* perform the write itself */ + abd_buf->abd = abd_get_from_buf(tmpbuf, sizeof (*lb)); + abd_take_ownership_of_buf(abd_buf->abd, B_TRUE); + wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand, + asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL, + ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE); + DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, zio_t *, wzio); + (void) zio_nowait(wzio); + + dev->l2ad_hand += asize; + /* + * Include the committed log block's pointer in the list of pointers + * to log blocks present in the L2ARC device. + */ + bcopy(&l2dhdr->dh_start_lbps[0], lb_ptr_buf->lb_ptr, + sizeof (l2arc_log_blkptr_t)); + mutex_enter(&dev->l2ad_mtx); + list_insert_head(&dev->l2ad_lbptr_list, lb_ptr_buf); + mutex_exit(&dev->l2ad_mtx); + vdev_space_update(dev->l2ad_vdev, asize, 0, 0); + + /* bump the kstats */ + ARCSTAT_INCR(arcstat_l2_write_bytes, asize); + ARCSTAT_BUMP(arcstat_l2_log_blk_writes); + ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_size, asize); + ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio, + dev->l2ad_log_blk_payload_asize / asize); + + /* start a new log block */ + dev->l2ad_log_ent_idx = 0; + dev->l2ad_log_blk_payload_asize = 0; + dev->l2ad_log_blk_payload_start = 0; +} + +/* + * Validates an L2ARC log block address to make sure that it can be read + * from the provided L2ARC device. + */ +boolean_t +l2arc_log_blkptr_valid(l2arc_dev_t *dev, const l2arc_log_blkptr_t *lbp) +{ + uint64_t psize = L2BLK_GET_PSIZE((lbp)->lbp_prop); + uint64_t end = lbp->lbp_daddr + psize - 1; + uint64_t start = lbp->lbp_payload_start; + boolean_t evicted = B_FALSE; + + /* + * A log block is valid if all of the following conditions are true: + * - it fits entirely (including its payload) between l2ad_start and + * l2ad_end + * - it has a valid size + * - neither the log block itself nor part of its payload was evicted + * by l2arc_evict(): + * + * l2ad_hand l2ad_evict + * | | lbp_daddr + * | start | | end + * | | | | | + * V V V V V + * l2ad_start ============================================ l2ad_end + * --------------------------|||| + * ^ ^ + * | log block + * payload + */ + + evicted = + l2arc_range_check_overlap(start, end, dev->l2ad_hand) || + l2arc_range_check_overlap(start, end, dev->l2ad_evict) || + l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, start) || + l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, end); + + return (start >= dev->l2ad_start && end <= dev->l2ad_end && + psize > 0 && psize <= sizeof (l2arc_log_blk_phys_t) && + (!evicted || dev->l2ad_first)); +} + +/* + * Inserts ARC buffer header `hdr' into the current L2ARC log block on + * the device. The buffer being inserted must be present in L2ARC. + * Returns B_TRUE if the L2ARC log block is full and needs to be committed + * to L2ARC, or B_FALSE if it still has room for more ARC buffers. + */ +static boolean_t +l2arc_log_blk_insert(l2arc_dev_t *dev, const arc_buf_hdr_t *hdr) +{ + l2arc_log_blk_phys_t *lb = &dev->l2ad_log_blk; + l2arc_log_ent_phys_t *le; + l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr; + + if (l2dhdr->dh_log_blk_ent == 0) + return (B_FALSE); + + int index = dev->l2ad_log_ent_idx++; + + ASSERT3S(index, <, l2dhdr->dh_log_blk_ent); + ASSERT(HDR_HAS_L2HDR(hdr)); + + le = &lb->lb_entries[index]; + bzero(le, sizeof (*le)); + le->le_dva = hdr->b_dva; + le->le_birth = hdr->b_birth; + le->le_daddr = hdr->b_l2hdr.b_daddr; + if (index == 0) + dev->l2ad_log_blk_payload_start = le->le_daddr; + L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr)); + L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr)); + L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr)); + L2BLK_SET_TYPE((le)->le_prop, hdr->b_type); + L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr))); + L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr))); + + dev->l2ad_log_blk_payload_asize += vdev_psize_to_asize(dev->l2ad_vdev, + HDR_GET_PSIZE(hdr)); + + return (dev->l2ad_log_ent_idx == l2dhdr->dh_log_blk_ent); +} + +/* + * Checks whether a given L2ARC device address sits in a time-sequential + * range. The trick here is that the L2ARC is a rotary buffer, so we can't + * just do a range comparison, we need to handle the situation in which the + * range wraps around the end of the L2ARC device. Arguments: + * bottom -- Lower end of the range to check (written to earlier). + * top -- Upper end of the range to check (written to later). + * check -- The address for which we want to determine if it sits in + * between the top and bottom. + * + * The 3-way conditional below represents the following cases: + * + * bottom < top : Sequentially ordered case: + * --------+-------------------+ + * | (overlap here?) | + * L2ARC dev V V + * |---------------============--------------| + * + * bottom > top: Looped-around case: + * --------+------------------+ + * | (overlap here?) | + * L2ARC dev V V + * |===============---------------===========| + * ^ ^ + * | (or here?) | + * +---------------+--------- + * + * top == bottom : Just a single address comparison. + */ +boolean_t +l2arc_range_check_overlap(uint64_t bottom, uint64_t top, uint64_t check) +{ + if (bottom < top) + return (bottom <= check && check <= top); + else if (bottom > top) + return (check <= top || bottom <= check); + else + return (check == top); +} + EXPORT_SYMBOL(arc_buf_size); EXPORT_SYMBOL(arc_write); EXPORT_SYMBOL(arc_read); @@ -8861,6 +10193,12 @@ ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_again, INT, ZMOD_RW, ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, norw, INT, ZMOD_RW, "No reads during writes"); +ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_enabled, INT, ZMOD_RW, + "Rebuild the L2ARC when importing a pool"); + +ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_blocks_min_l2size, ULONG, ZMOD_RW, + "Min size in bytes to write rebuild log blocks in L2ARC"); + ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, lotsfree_percent, param_set_arc_int, param_get_int, ZMOD_RW, "System free memory I/O throttle in bytes"); diff --git a/module/zfs/spa.c b/module/zfs/spa.c index 0d4646c1516..aface90afc6 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -4860,6 +4860,8 @@ spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport) } spa_import_progress_remove(spa_guid(spa)); + spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD); + spa_load_note(spa, "LOADED"); return (0); @@ -7985,6 +7987,17 @@ spa_async_thread(void *arg) mutex_exit(&spa_namespace_lock); } + /* + * Kick off L2 cache rebuilding. + */ + if (tasks & SPA_ASYNC_L2CACHE_REBUILD) { + mutex_enter(&spa_namespace_lock); + spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER); + l2arc_spa_rebuild_start(spa); + spa_config_exit(spa, SCL_L2ARC, FTAG); + mutex_exit(&spa_namespace_lock); + } + /* * Let the world know that we're done. */ diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index fb0d02eea5b..59147ce31d3 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -2279,9 +2279,22 @@ vdev_reopen(vdev_t *vd) if (vd->vdev_aux) { (void) vdev_validate_aux(vd); if (vdev_readable(vd) && vdev_writeable(vd) && - vd->vdev_aux == &spa->spa_l2cache && - !l2arc_vdev_present(vd)) - l2arc_add_vdev(spa, vd); + vd->vdev_aux == &spa->spa_l2cache) { + /* + * When reopening we can assume the device label has + * already the attribute l2cache_persistent, since we've + * opened the device in the past and updated the label. + * In case the vdev is present we should evict all ARC + * buffers and pointers to log blocks and reclaim their + * space before restoring its contents to L2ARC. + */ + if (l2arc_vdev_present(vd)) { + l2arc_rebuild_vdev(vd, B_TRUE); + } else { + l2arc_add_vdev(spa, vd); + } + spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD); + } } else { (void) vdev_validate(vd); } diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 897a6a95582..61df6d4208c 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -164,3 +164,9 @@ tags = ['functional', 'user_namespace'] tests = ['groupspace_001_pos', 'groupspace_002_pos', 'groupspace_003_pos', 'userquota_013_pos', 'userspace_003_pos'] tags = ['functional', 'userquota'] + +[tests/functional/persist_l2arc:Linux] +tests = ['persist_l2arc_001_pos', 'persist_l2arc_002_pos', + 'persist_l2arc_003_neg', 'persist_l2arc_004_pos', 'persist_l2arc_005_pos', + 'persist_l2arc_006_pos', 'persist_l2arc_007_pos', 'persist_l2arc_008_pos'] +tags = ['functional', 'persist_l2arc'] diff --git a/tests/zfs-tests/include/tunables.cfg b/tests/zfs-tests/include/tunables.cfg index 62d335abebc..680fcf42cb2 100644 --- a/tests/zfs-tests/include/tunables.cfg +++ b/tests/zfs-tests/include/tunables.cfg @@ -36,6 +36,8 @@ INITIALIZE_CHUNK_SIZE initialize_chunk_size zfs_initialize_chunk_size INITIALIZE_VALUE initialize_value zfs_initialize_value KEEP_LOG_SPACEMAPS_AT_EXPORT keep_log_spacemaps_at_export zfs_keep_log_spacemaps_at_export L2ARC_NOPREFETCH l2arc.noprefetch l2arc_noprefetch +L2ARC_REBUILD_BLOCKS_MIN_L2SIZE UNSUPPORTED l2arc_rebuild_blocks_min_l2size +L2ARC_REBUILD_ENABLED UNSUPPORTED l2arc_rebuild_enabled L2ARC_WRITE_BOOST l2arc.write_boost l2arc_write_boost L2ARC_WRITE_MAX l2arc.write_max l2arc_write_max LIVELIST_CONDENSE_NEW_ALLOC livelist.condense.new_alloc zfs_livelist_condense_new_alloc diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am index bd484b2da79..776222f085f 100644 --- a/tests/zfs-tests/tests/functional/Makefile.am +++ b/tests/zfs-tests/tests/functional/Makefile.am @@ -45,6 +45,7 @@ SUBDIRS = \ no_space \ nopwrite \ online_offline \ + persist_l2arc \ pool_checkpoint \ pool_names \ poolversion \ diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/Makefile.am b/tests/zfs-tests/tests/functional/persist_l2arc/Makefile.am new file mode 100644 index 00000000000..14a43de9c17 --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/Makefile.am @@ -0,0 +1,15 @@ +pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/persist_l2arc +dist_pkgdata_SCRIPTS = \ + cleanup.ksh \ + setup.ksh \ + persist_l2arc_001_pos.ksh \ + persist_l2arc_002_pos.ksh \ + persist_l2arc_003_neg.ksh \ + persist_l2arc_004_pos.ksh \ + persist_l2arc_005_pos.ksh \ + persist_l2arc_006_pos.ksh \ + persist_l2arc_007_pos.ksh \ + persist_l2arc_008_pos.ksh + +dist_pkgdata_DATA = \ + persist_l2arc.cfg diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/cleanup.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/cleanup.ksh new file mode 100755 index 00000000000..828de386250 --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/cleanup.ksh @@ -0,0 +1,31 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg + +verify_runnable "global" + +if datasetexists $TESTPOOL ; then + log_must zpool destroy -f $TESTPOOL +fi + +log_must rm -rf $VDIR + +log_pass diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc.cfg b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc.cfg new file mode 100644 index 00000000000..60bb2463760 --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc.cfg @@ -0,0 +1,37 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +export SIZE=1G +export VDIR=$TESTDIR/disk.persist_l2arc +export VDEV="$VDIR/a" +export VDEV_CACHE="$VDIR/b" + +# fio options +export DIRECTORY=/$TESTPOOL +export NUMJOBS=4 +export RUNTIME=30 +export PERF_RANDSEED=1234 +export PERF_COMPPERCENT=66 +export PERF_COMPCHUNK=0 +export BLOCKSIZE=128K +export SYNC_TYPE=0 +export DIRECT=1 diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_001_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_001_pos.ksh new file mode 100755 index 00000000000..b202fac40ac --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_001_pos.ksh @@ -0,0 +1,106 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg + +# +# DESCRIPTION: +# Persistent L2ARC with an unencrypted ZFS file system succeeds +# +# STRATEGY: +# 1. Create pool with a cache device. +# 2. Export and re-import pool without writing any data. +# 3. Create a random file in that pool and random read for 30 sec. +# 4. Export pool. +# 5. Read the amount of log blocks written from the header of the +# L2ARC device. +# 6. Import pool. +# 7. Read the amount of log blocks rebuilt in arcstats and compare to +# (4). +# 8. Check if the labels of the L2ARC device are intact. +# +# * We can predict the minimum bytes of L2ARC restored if we subtract +# from the effective size of the cache device the bytes l2arc_evict() +# evicts: +# l2: L2ARC device size - VDEV_LABEL_START_SIZE - l2ad_dev_hdr_asize +# wr_sz: l2arc_write_max + l2arc_write_boost (worst case) +# blk_overhead: wr_sz / SPA_MINBLOCKSIZE / (l2 / SPA_MAXBLOCKSIZE) * +# sizeof (l2arc_log_blk_phys_t) +# min restored size: l2 - (wr_sz + blk_overhead) +# + +verify_runnable "global" + +log_assert "Persistent L2ARC with an unencrypted ZFS file system succeeds." + +function cleanup +{ + if poolexists $TESTPOOL ; then + destroy_pool $TESTPOOL + fi + + log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch + log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \ + $rebuild_blocks_min_l2size +} +log_onexit cleanup + +# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches +typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH) +typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE) +log_must set_tunable32 L2ARC_NOPREFETCH 0 +log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0 + +typeset fill_mb=800 +typeset cache_sz=$(( floor($fill_mb / 2) )) +export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M + +log_must truncate -s ${cache_sz}M $VDEV_CACHE + +log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE + +log_must zpool export $TESTPOOL +log_must zpool import -d $VDIR $TESTPOOL + +log_must fio $FIO_SCRIPTS/mkfiles.fio +log_must fio $FIO_SCRIPTS/random_reads.fio + +log_must zpool export $TESTPOOL + +typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \ + awk '{print $2}') + +typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks) + +log_must zpool import -d $VDIR $TESTPOOL + +sleep 2 + +typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks) + +log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start )) +log_must test $l2_dh_log_blk -gt 0 + +log_must zdb -lq $VDEV_CACHE + +log_must zpool destroy -f $TESTPOOL + +log_pass "Persistent L2ARC with an unencrypted ZFS file system succeeds." diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_002_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_002_pos.ksh new file mode 100755 index 00000000000..ae219e01a4e --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_002_pos.ksh @@ -0,0 +1,112 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg +. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib + +# +# DESCRIPTION: +# Persistent L2ARC with an encrypted ZFS file system succeeds +# +# STRATEGY: +# 1. Create pool with a cache device. +# 2. Create a an encrypted ZFS file system. +# 3. Create a random file in the encyrpted file system and random +# read for 30 sec. +# 4. Export pool. +# 5. Read the amount of log blocks written from the header of the +# L2ARC device. +# 5. Import pool. +# 6. Mount the encypted ZFS file system. +# 7. Read the amount of log blocks rebuilt in arcstats and compare to +# (5). +# 8. Check if the labels of the L2ARC device are intact. +# +# * We can predict the minimum bytes of L2ARC restored if we subtract +# from the effective size of the cache device the bytes l2arc_evict() +# evicts: +# l2: L2ARC device size - VDEV_LABEL_START_SIZE - l2ad_dev_hdr_asize +# wr_sz: l2arc_write_max + l2arc_write_boost (worst case) +# blk_overhead: wr_sz / SPA_MINBLOCKSIZE / (l2 / SPA_MAXBLOCKSIZE) * +# sizeof (l2arc_log_blk_phys_t) +# min restored size: l2 - (wr_sz + blk_overhead) +# + +verify_runnable "global" + +log_assert "Persistent L2ARC with an encrypted ZFS file system succeeds." + +function cleanup +{ + if poolexists $TESTPOOL ; then + destroy_pool $TESTPOOL + fi + + log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch + log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \ + $rebuild_blocks_min_l2size +} +log_onexit cleanup + +# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches +typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH) +typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE) +log_must set_tunable32 L2ARC_NOPREFETCH 0 +log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0 + +typeset fill_mb=800 +typeset cache_sz=$(( floor($fill_mb / 2) )) +export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M + +log_must truncate -s ${cache_sz}M $VDEV_CACHE + +log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE + +log_must eval "echo $PASSPHRASE | zfs create -o encryption=on" \ + "-o keyformat=passphrase $TESTPOOL/$TESTFS1" + +log_must fio $FIO_SCRIPTS/mkfiles.fio +log_must fio $FIO_SCRIPTS/random_reads.fio + +log_must zpool export $TESTPOOL + +sleep 2 + +typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \ + awk '{print $2}') + +typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks) + +log_must zpool import -d $VDIR $TESTPOOL +log_must eval "echo $PASSPHRASE | zfs mount -l $TESTPOOL/$TESTFS1" + +sleep 2 + +typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks) + +log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start )) +log_must test $l2_dh_log_blk -gt 0 + +log_must zdb -lq $VDEV_CACHE + +log_must zpool destroy -f $TESTPOOL + +log_pass "Persistent L2ARC with an encrypted ZFS file system succeeds." diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_003_neg.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_003_neg.ksh new file mode 100755 index 00000000000..7824dfe8f1b --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_003_neg.ksh @@ -0,0 +1,87 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg + +# +# DESCRIPTION: +# Persistent L2ARC fails as expected when L2ARC_REBUILD_ENABLED = 0 +# +# STRATEGY: +# 1. Set L2ARC_REBUILD_ENABLED = 0 +# 2. Create pool with a cache device. +# 3. Create a random file in that pool and random read for 30 sec. +# 4. Export pool. +# 5. Import pool. +# 6. Check in zpool iostat if the cache device has space allocated. +# 7. Read the file written in (2) and check if l2_hits in +# /proc/spl/kstat/zfs/arcstats increased. +# + +verify_runnable "global" + +log_assert "Persistent L2ARC fails as expected when L2ARC_REBUILD_ENABLED = 0." + +function cleanup +{ + if poolexists $TESTPOOL ; then + destroy_pool $TESTPOOL + fi + + log_must set_tunable32 L2ARC_REBUILD_ENABLED $rebuild_enabled + log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch +} +log_onexit cleanup + +# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches +typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH) +log_must set_tunable32 L2ARC_NOPREFETCH 0 + +# disable L2ARC rebuild +typeset rebuild_enabled=$(get_tunable L2ARC_REBUILD_ENABLED) +log_must set_tunable32 L2ARC_REBUILD_ENABLED 0 + +typeset fill_mb=800 +typeset cache_sz=$(( 2 * $fill_mb )) +export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M + +log_must truncate -s ${cache_sz}M $VDEV_CACHE + +log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE + +log_must fio $FIO_SCRIPTS/mkfiles.fio +log_must fio $FIO_SCRIPTS/random_reads.fio + +log_must zpool export $TESTPOOL + +typeset l2_success_start=$(get_arcstat l2_rebuild_success) + +log_must zpool import -d $VDIR $TESTPOOL +log_mustnot test "$(zpool iostat -Hpv $TESTPOOL $VDEV_CACHE | awk '{print $2}')" -gt 80000000 + +typeset l2_success_end=$(get_arcstat l2_rebuild_success) + +log_mustnot test $l2_success_end -gt $l2_success_start + +log_must zpool destroy -f $TESTPOOL +log_must set_tunable32 L2ARC_REBUILD_ENABLED $rebuild_enabled + +log_pass "Persistent L2ARC fails as expected when L2ARC_REBUILD_ENABLED = 0." diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_004_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_004_pos.ksh new file mode 100755 index 00000000000..6620131d182 --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_004_pos.ksh @@ -0,0 +1,101 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg + +# +# DESCRIPTION: +# Persistent L2ARC restores all written log blocks +# +# STRATEGY: +# 1. Create pool with a cache device. +# 2. Create a random file in that pool, smaller than the cache device +# and random read for 30 sec. +# 3. Export pool. +# 4. Read amount of log blocks written. +# 5. Import pool. +# 6. Read amount of log blocks built. +# 7. Compare the two amounts +# 8. Read the file written in (2) and check if l2_hits in +# /proc/spl/kstat/zfs/arcstats increased. +# 9. Check if the labels of the L2ARC device are intact. +# + +verify_runnable "global" + +log_assert "Persistent L2ARC restores all written log blocks." + +function cleanup +{ + if poolexists $TESTPOOL ; then + destroy_pool $TESTPOOL + fi + + log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch +} +log_onexit cleanup + +# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches +typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH) +log_must set_tunable32 L2ARC_NOPREFETCH 0 + +typeset fill_mb=800 +typeset cache_sz=$(( 2 * $fill_mb )) +export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M + +log_must truncate -s ${cache_sz}M $VDEV_CACHE + +typeset log_blk_start=$(get_arcstat l2_log_blk_writes) + +log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE + +log_must fio $FIO_SCRIPTS/mkfiles.fio +log_must fio $FIO_SCRIPTS/random_reads.fio + +log_must zpool export $TESTPOOL + +sleep 2 + +typeset log_blk_end=$(get_arcstat l2_log_blk_writes) + +typeset log_blk_rebuild_start=$(get_arcstat l2_rebuild_log_blks) + +log_must zpool import -d $VDIR $TESTPOOL + +typeset l2_hits_start=$(get_arcstat l2_hits) + +export RUNTIME=10 +log_must fio $FIO_SCRIPTS/random_reads.fio + +typeset l2_hits_end=$(get_arcstat l2_hits) + +typeset log_blk_rebuild_end=$(get_arcstat l2_rebuild_log_blks) + +log_must test $(( $log_blk_rebuild_end - $log_blk_rebuild_start )) -eq \ + $(( $log_blk_end - $log_blk_start )) + +log_must test $l2_hits_end -gt $l2_hits_start + +log_must zdb -lq $VDEV_CACHE + +log_must zpool destroy -f $TESTPOOL + +log_pass "Persistent L2ARC restores all written log blocks." diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_005_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_005_pos.ksh new file mode 100755 index 00000000000..b2cad9d1fc1 --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_005_pos.ksh @@ -0,0 +1,108 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg +. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib + +# +# DESCRIPTION: +# Persistent L2ARC restores all written log blocks with encryption +# +# STRATEGY: +# 1. Create pool with a cache device. +# 2. Create a an encrypted ZFS file system. +# 3. Create a random file in the entrypted file system, +# smaller than the cache device, and random read for 30 sec. +# 4. Export pool. +# 5. Read amount of log blocks written. +# 6. Import pool. +# 7. Mount the encypted ZFS file system. +# 8. Read amount of log blocks built. +# 9. Compare the two amounts +# 10. Read the file written in (3) and check if l2_hits in +# /proc/spl/kstat/zfs/arcstats increased. +# 11. Check if the labels of the L2ARC device are intact. +# + +verify_runnable "global" + +log_assert "Persistent L2ARC restores all written log blocks with encryption." + +function cleanup +{ + if poolexists $TESTPOOL ; then + destroy_pool $TESTPOOL + fi + + log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch +} +log_onexit cleanup + +# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches +typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH) +log_must set_tunable32 L2ARC_NOPREFETCH 0 + +typeset fill_mb=800 +typeset cache_sz=$(( 2 * $fill_mb )) +export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M + +log_must truncate -s ${cache_sz}M $VDEV_CACHE + +typeset log_blk_start=$(get_arcstat l2_log_blk_writes) + +log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE + +log_must eval "echo $PASSPHRASE | zfs create -o encryption=on" \ + "-o keyformat=passphrase $TESTPOOL/$TESTFS1" + +log_must fio $FIO_SCRIPTS/mkfiles.fio +log_must fio $FIO_SCRIPTS/random_reads.fio + +log_must zpool export $TESTPOOL + +sleep 2 + +typeset log_blk_end=$(get_arcstat l2_log_blk_writes) + +typeset log_blk_rebuild_start=$(get_arcstat l2_rebuild_log_blks) + +log_must zpool import -d $VDIR $TESTPOOL +log_must eval "echo $PASSPHRASE | zfs mount -l $TESTPOOL/$TESTFS1" + +typeset l2_hits_start=$(get_arcstat l2_hits) + +export RUNTIME=10 +log_must fio $FIO_SCRIPTS/random_reads.fio + +typeset l2_hits_end=$(get_arcstat l2_hits) + +typeset log_blk_rebuild_end=$(get_arcstat l2_rebuild_log_blks) + +log_must test $(( $log_blk_rebuild_end - $log_blk_rebuild_start )) -eq \ + $(( $log_blk_end - $log_blk_start )) + +log_must test $l2_hits_end -gt $l2_hits_start + +log_must zdb -lq $VDEV_CACHE + +log_must zpool destroy -f $TESTPOOL + +log_pass "Persistent L2ARC restores all written log blocks with encryption." diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_006_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_006_pos.ksh new file mode 100755 index 00000000000..55e9f9585c0 --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_006_pos.ksh @@ -0,0 +1,98 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg + +# +# DESCRIPTION: +# Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not +# present. +# +# STRATEGY: +# 1. Create pool with a cache device. +# 2. Create a random file in that pool and random read for 30 sec. +# 3. Read the amount of log blocks written from the header of the +# L2ARC device. +# 4. Offline the L2ARC device and export pool. +# 5. Import pool and online the L2ARC device. +# 6. Read the amount of log blocks rebuilt in arcstats and compare to +# (3). +# 7. Check if the labels of the L2ARC device are intact. +# + +verify_runnable "global" + +log_assert "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not present." + +function cleanup +{ + if poolexists $TESTPOOL ; then + destroy_pool $TESTPOOL + fi + + log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch + log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \ + $rebuild_blocks_min_l2size +} +log_onexit cleanup + +# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches +typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH) +typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE) +log_must set_tunable32 L2ARC_NOPREFETCH 0 +log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0 + +typeset fill_mb=800 +typeset cache_sz=$(( floor($fill_mb / 2) )) +export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M + +log_must truncate -s ${cache_sz}M $VDEV_CACHE + +log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE + +log_must fio $FIO_SCRIPTS/mkfiles.fio +log_must fio $FIO_SCRIPTS/random_reads.fio + +log_must zpool offline $TESTPOOL $VDEV_CACHE +log_must zpool export $TESTPOOL + +sleep 5 + +typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks) + +typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \ + awk '{print $2}') + +log_must zpool import -d $VDIR $TESTPOOL +log_must zpool online $TESTPOOL $VDEV_CACHE + +sleep 5 + +typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks) + +log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start )) +log_must test $l2_dh_log_blk -gt 0 + +log_must zdb -lq $VDEV_CACHE + +log_must zpool destroy -f $TESTPOOL + +log_pass "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not present." diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh new file mode 100755 index 00000000000..e3c983be847 --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh @@ -0,0 +1,95 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg + +# +# DESCRIPTION: +# Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present. +# +# STRATEGY: +# 1. Create pool with a cache device. +# 2. Create a random file in that pool and random read for 30 sec. +# 3. Read the amount of log blocks written from the header of the +# L2ARC device. +# 4. Offline the L2ARC device. +# 5. Online the L2ARC device. +# 6. Read the amount of log blocks rebuilt in arcstats and compare to +# (3). +# 7. Check if the labels of the L2ARC device are intact. +# + +verify_runnable "global" + +log_assert "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present." + +function cleanup +{ + if poolexists $TESTPOOL ; then + destroy_pool $TESTPOOL + fi + + log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch + log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \ + $rebuild_blocks_min_l2size +} +log_onexit cleanup + +# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches +typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH) +typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE) +log_must set_tunable32 L2ARC_NOPREFETCH 0 +log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0 + +typeset fill_mb=800 +typeset cache_sz=$(( floor($fill_mb / 2) )) +export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M + +log_must truncate -s ${cache_sz}M $VDEV_CACHE + +log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE + +log_must fio $FIO_SCRIPTS/mkfiles.fio +log_must fio $FIO_SCRIPTS/random_reads.fio + +log_must zpool offline $TESTPOOL $VDEV_CACHE + +sleep 5 + +typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks) + +typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \ + awk '{print $2}') + +log_must zpool online $TESTPOOL $VDEV_CACHE + +sleep 5 + +typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks) + +log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start )) +log_must test $l2_dh_log_blk -gt 0 + +log_must zdb -lq $VDEV_CACHE + +log_must zpool destroy -f $TESTPOOL + +log_pass "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present." diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_008_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_008_pos.ksh new file mode 100755 index 00000000000..a64bd94d316 --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_008_pos.ksh @@ -0,0 +1,143 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg + +# +# DESCRIPTION: +# Off/onlining an L2ARC device restores all written blocks, vdev present. +# +# STRATEGY: +# 1. Create pool with a cache device. +# 2. Create a random file in that pool and random read for 30 sec. +# 3. Read the amount of log blocks written from the header of the +# L2ARC device. +# 4. Offline the L2ARC device. +# 5. Online the L2ARC device. +# 6. Read the amount of log blocks rebuilt in arcstats and compare to +# (3). +# 7. Create another random file in that pool and random read for 30 sec. +# 8. Read the amount of log blocks written from the header of the +# L2ARC device. +# 9. Offline the L2ARC device. +# 10. Online the L2ARC device. +# 11. Read the amount of log blocks rebuilt in arcstats and compare to +# (7). +# 12. Check if the amount of log blocks on the cache device has +# increased. +# 13. Export the pool. +# 14. Read the amount of log blocks on the cache device. +# 15. Import the pool. +# 16. Read the amount of log blocks rebuilt in arcstats and compare to +# (14). +# 17. Check if the labels of the L2ARC device are intact. +# + +verify_runnable "global" + +log_assert "Off/onlining an L2ARC device restores all written blocks , vdev present." + +function cleanup +{ + if poolexists $TESTPOOL ; then + destroy_pool $TESTPOOL + fi + + log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch +} +log_onexit cleanup + +# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches +typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH) +log_must set_tunable32 L2ARC_NOPREFETCH 0 + +typeset fill_mb=400 +typeset cache_sz=$(( 3 * $fill_mb )) +export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M + +log_must truncate -s ${cache_sz}M $VDEV_CACHE + +log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE + +log_must fio $FIO_SCRIPTS/mkfiles.fio +log_must fio $FIO_SCRIPTS/random_reads.fio + +log_must zpool offline $TESTPOOL $VDEV_CACHE + +sleep 2 + +typeset l2_dh_log_blk1=$(zdb -l $VDEV_CACHE | grep log_blk_count | \ + awk '{print $2}') + +typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks) + +log_must zpool online $TESTPOOL $VDEV_CACHE + +sleep 5 + +typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks) + +log_must test $l2_dh_log_blk1 -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start )) +log_must test $l2_dh_log_blk1 -gt 0 + +log_must fio $FIO_SCRIPTS/mkfiles.fio +log_must fio $FIO_SCRIPTS/random_reads.fio + +log_must zpool offline $TESTPOOL $VDEV_CACHE + +sleep 2 + +typeset l2_dh_log_blk2=$(zdb -l $VDEV_CACHE | grep log_blk_count | \ + awk '{print $2}') + +typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks) + +log_must zpool online $TESTPOOL $VDEV_CACHE + +sleep 5 + +typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks) + +log_must test $l2_dh_log_blk2 -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start )) + +log_must test $l2_dh_log_blk2 -gt $l2_dh_log_blk1 + +log_must zpool export $TESTPOOL + +typeset l2_dh_log_blk3=$(zdb -l $VDEV_CACHE | grep log_blk_count | \ + awk '{print $2}') + +typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks) + +log_must zpool import -d $VDIR $TESTPOOL + +sleep 5 + +typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks) + +log_must test $l2_dh_log_blk3 -eq $(( $l2_rebuild_log_blk_end - $l2_rebuild_log_blk_start )) +log_must test $l2_dh_log_blk3 -gt 0 + +log_must zdb -lq $VDEV_CACHE + +log_must zpool destroy -f $TESTPOOL + +log_pass "Off/onlining an L2ARC device restores all written blocks, vdev present." diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/setup.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/setup.ksh new file mode 100755 index 00000000000..ef95c84cdd6 --- /dev/null +++ b/tests/zfs-tests/tests/functional/persist_l2arc/setup.ksh @@ -0,0 +1,29 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, George Amanakis. All rights reserved. +# + +. $STF_SUITE/tests/functional/persist_l2arc/persist_l2arc.cfg + +verify_runnable "global" + +log_must rm -rf $VDIR +log_must mkdir -p $VDIR +log_must mkfile $SIZE $VDEV + +log_pass From c618f87cd2e96438468a391246d63ba1803f35c8 Mon Sep 17 00:00:00 2001 From: Matthew Ahrens Date: Fri, 10 Apr 2020 10:39:55 -0700 Subject: [PATCH 009/133] Add `zstream redup` command to convert deduplicated send streams Deduplicated send and receive is deprecated. To ease migration to the new dedup-send-less world, the commit adds a `zstream redup` utility to convert deduplicated send streams to normal streams, so that they can continue to be received indefinitely. The new `zstream` command also replaces the functionality of `zstreamdump`, by way of the `zstream dump` subcommand. The `zstreamdump` command is replaced by a shell script which invokes `zstream dump`. The way that `zstream redup` works under the hood is that as we read the send stream, we build up a hash table which maps from ` -> `. Whenever we see a WRITE record, we add a new entry to the hash table, which indicates where in the stream file to find the WRITE record for this block. (The key is `drr_toguid, drr_object, drr_offset`.) For entries other than WRITE_BYREF, we pass them through unchanged (except for the running checksum, which is recalculated). For WRITE_BYREF records, we change them to WRITE records. We find the referenced WRITE record by looking in the hash table (for the record with key `drr_refguid, drr_refobject, drr_refoffset`), and then reading the record header and payload from the specified offset in the stream file. This is why the stream can not be a pipe. The found WRITE record replaces the WRITE_BYREF record, with its `drr_toguid`, `drr_object`, and `drr_offset` fields changed to be the same as the WRITE_BYREF's (i.e. we are writing the same logical block, but with the data supplied by the previous WRITE record). This algorithm requires memory proportional to the number of WRITE records (same as `zfs send -D`), but the size per WRITE record is relatively low (40 bytes, vs. 72 for `zfs send -D`). A 1TB send stream with 8KB blocks (`recordsize=8k`) would use around 5GB of RAM to "redup". Reviewed-by: Jorgen Lundman Reviewed-by: Paul Dagnelie Reviewed-by: Brian Behlendorf Signed-off-by: Matthew Ahrens Closes #10124 Closes #10156 --- cmd/Makefile.am | 2 +- cmd/zstream/.gitignore | 1 + cmd/zstream/Makefile.am | 13 + cmd/zstream/zstream.c | 61 +++ cmd/zstream/zstream.h | 35 ++ .../zstreamdump.c => zstream/zstream_dump.c} | 45 +- cmd/zstream/zstream_redup.c | 468 ++++++++++++++++++ cmd/zstreamdump/Makefile.am | 11 +- cmd/zstreamdump/zstreamdump | 3 + configure.ac | 1 + lib/libzfs/libzfs_sendrecv.c | 5 +- man/man8/Makefile.am | 1 + man/man8/zstream.8 | 101 ++++ tests/zfs-tests/include/commands.cfg | 1 + .../zfs_receive/zfs_receive_013_pos.ksh | 2 + .../tests/functional/rsend/send-cD.ksh | 14 +- 16 files changed, 728 insertions(+), 36 deletions(-) create mode 100644 cmd/zstream/.gitignore create mode 100644 cmd/zstream/Makefile.am create mode 100644 cmd/zstream/zstream.c create mode 100644 cmd/zstream/zstream.h rename cmd/{zstreamdump/zstreamdump.c => zstream/zstream_dump.c} (97%) create mode 100644 cmd/zstream/zstream_redup.c create mode 100755 cmd/zstreamdump/zstreamdump create mode 100644 man/man8/zstream.8 diff --git a/cmd/Makefile.am b/cmd/Makefile.am index 90270209b28..2078bc13b3c 100644 --- a/cmd/Makefile.am +++ b/cmd/Makefile.am @@ -1,4 +1,4 @@ -SUBDIRS = zfs zpool zdb zhack zinject zstreamdump ztest +SUBDIRS = zfs zpool zdb zhack zinject zstream zstreamdump ztest SUBDIRS += fsck_zfs vdev_id raidz_test zgenhostid if USING_PYTHON diff --git a/cmd/zstream/.gitignore b/cmd/zstream/.gitignore new file mode 100644 index 00000000000..fd1240d55c4 --- /dev/null +++ b/cmd/zstream/.gitignore @@ -0,0 +1 @@ +zstream diff --git a/cmd/zstream/Makefile.am b/cmd/zstream/Makefile.am new file mode 100644 index 00000000000..892e1583072 --- /dev/null +++ b/cmd/zstream/Makefile.am @@ -0,0 +1,13 @@ +include $(top_srcdir)/config/Rules.am + +sbin_PROGRAMS = zstream + +zstream_SOURCES = \ + zstream.c \ + zstream.h \ + zstream_dump.c \ + zstream_redup.c + +zstream_LDADD = \ + $(top_builddir)/lib/libnvpair/libnvpair.la \ + $(top_builddir)/lib/libzfs/libzfs.la diff --git a/cmd/zstream/zstream.c b/cmd/zstream/zstream.c new file mode 100644 index 00000000000..95578c97c25 --- /dev/null +++ b/cmd/zstream/zstream.c @@ -0,0 +1,61 @@ +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2020 by Delphix. All rights reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "zstream.h" + +void +zstream_usage(void) +{ + (void) fprintf(stderr, + "usage: zstream command args ...\n" + "Available commands are:\n" + "\n" + "\tzstream dump [-vCd] FILE\n" + "\t... | zstream dump [-vCd]\n" + "\n" + "\tzstream redup [-v] FILE | ...\n"); + exit(1); +} + +int +main(int argc, char *argv[]) +{ + if (argc < 2) + zstream_usage(); + + char *subcommand = argv[1]; + + if (strcmp(subcommand, "dump") == 0) { + return (zstream_do_dump(argc - 1, argv + 1)); + } else if (strcmp(subcommand, "redup") == 0) { + return (zstream_do_redup(argc - 1, argv + 1)); + } else { + zstream_usage(); + } +} diff --git a/cmd/zstream/zstream.h b/cmd/zstream/zstream.h new file mode 100644 index 00000000000..5a7f4bce9a9 --- /dev/null +++ b/cmd/zstream/zstream.h @@ -0,0 +1,35 @@ +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2020 by Delphix. All rights reserved. + */ + +#ifndef _ZSTREAM_H +#define _ZSTREAM_H + +#ifdef __cplusplus +extern "C" { +#endif + +extern int zstream_do_redup(int, char *[]); +extern int zstream_do_dump(int, char *[]); +extern void zstream_usage(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _ZSTREAM_H */ diff --git a/cmd/zstreamdump/zstreamdump.c b/cmd/zstream/zstream_dump.c similarity index 97% rename from cmd/zstreamdump/zstreamdump.c rename to cmd/zstream/zstream_dump.c index ad3cefafc64..62a1d8272cb 100644 --- a/cmd/zstreamdump/zstreamdump.c +++ b/cmd/zstream/zstream_dump.c @@ -42,6 +42,7 @@ #include #include #include +#include "zstream.h" /* * If dump mode is enabled, the number of bytes to print per line @@ -58,17 +59,6 @@ FILE *send_stream = 0; boolean_t do_byteswap = B_FALSE; boolean_t do_cksum = B_TRUE; -static void -usage(void) -{ - (void) fprintf(stderr, "usage: zstreamdump [-v] [-C] [-d] < file\n"); - (void) fprintf(stderr, "\t -v -- verbose\n"); - (void) fprintf(stderr, "\t -C -- suppress checksum verification\n"); - (void) fprintf(stderr, "\t -d -- dump contents of blocks modified, " - "implies verbose\n"); - exit(1); -} - static void * safe_malloc(size_t size) { @@ -215,7 +205,7 @@ sprintf_bytes(char *str, uint8_t *buf, uint_t buf_len) } int -main(int argc, char *argv[]) +zstream_do_dump(int argc, char *argv[]) { char *buf = safe_malloc(SPA_MAXBLOCKSIZE); uint64_t drr_record_count[DRR_NUMTYPES] = { 0 }; @@ -273,26 +263,39 @@ main(int argc, char *argv[]) case ':': (void) fprintf(stderr, "missing argument for '%c' option\n", optopt); - usage(); + zstream_usage(); break; case '?': (void) fprintf(stderr, "invalid option '%c'\n", optopt); - usage(); + zstream_usage(); break; } } - if (isatty(STDIN_FILENO)) { - (void) fprintf(stderr, - "Error: Backup stream can not be read " - "from a terminal.\n" - "You must redirect standard input.\n"); - exit(1); + if (argc > optind) { + const char *filename = argv[optind]; + send_stream = fopen(filename, "r"); + if (send_stream == NULL) { + (void) fprintf(stderr, + "Error while opening file '%s': %s\n", + filename, strerror(errno)); + exit(1); + } + } else { + if (isatty(STDIN_FILENO)) { + (void) fprintf(stderr, + "Error: The send stream is a binary format " + "and can not be read from a\n" + "terminal. Standard input must be redirected, " + "or a file must be\n" + "specified as a command-line argument.\n"); + exit(1); + } + send_stream = stdin; } fletcher_4_init(); - send_stream = stdin; while (read_hdr(drr, &zc)) { /* diff --git a/cmd/zstream/zstream_redup.c b/cmd/zstream/zstream_redup.c new file mode 100644 index 00000000000..6720cfd1201 --- /dev/null +++ b/cmd/zstream/zstream_redup.c @@ -0,0 +1,468 @@ +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2020 by Delphix. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "zfs_fletcher.h" +#include "zstream.h" + + +#define MAX_RDT_PHYSMEM_PERCENT 20 +#define SMALLEST_POSSIBLE_MAX_RDT_MB 128 + +typedef struct redup_entry { + struct redup_entry *rde_next; + uint64_t rde_guid; + uint64_t rde_object; + uint64_t rde_offset; + uint64_t rde_stream_offset; +} redup_entry_t; + +typedef struct redup_table { + redup_entry_t **redup_hash_array; + umem_cache_t *ddecache; + uint64_t ddt_count; + int numhashbits; +} redup_table_t; + +int +highbit64(uint64_t i) +{ + if (i == 0) + return (0); + + return (NBBY * sizeof (uint64_t) - __builtin_clzll(i)); +} + +static void * +safe_calloc(size_t n) +{ + void *rv = calloc(1, n); + if (rv == NULL) { + fprintf(stderr, + "Error: could not allocate %u bytes of memory\n", + (int)n); + exit(1); + } + return (rv); +} + +/* + * Safe version of fread(), exits on error. + */ +static int +sfread(void *buf, size_t size, FILE *fp) +{ + int rv = fread(buf, size, 1, fp); + if (rv == 0 && ferror(fp)) { + (void) fprintf(stderr, "Error while reading file: %s\n", + strerror(errno)); + exit(1); + } + return (rv); +} + +/* + * Safe version of pread(), exits on error. + */ +static void +spread(int fd, void *buf, size_t count, off_t offset) +{ + ssize_t err = pread(fd, buf, count, offset); + if (err == -1) { + (void) fprintf(stderr, + "Error while reading file: %s\n", + strerror(errno)); + exit(1); + } else if (err != count) { + (void) fprintf(stderr, + "Error while reading file: short read\n"); + exit(1); + } +} + +static int +dump_record(dmu_replay_record_t *drr, void *payload, int payload_len, + zio_cksum_t *zc, int outfd) +{ + assert(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum) + == sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t)); + fletcher_4_incremental_native(drr, + offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), zc); + if (drr->drr_type != DRR_BEGIN) { + assert(ZIO_CHECKSUM_IS_ZERO(&drr->drr_u. + drr_checksum.drr_checksum)); + drr->drr_u.drr_checksum.drr_checksum = *zc; + } + fletcher_4_incremental_native(&drr->drr_u.drr_checksum.drr_checksum, + sizeof (zio_cksum_t), zc); + if (write(outfd, drr, sizeof (*drr)) == -1) + return (errno); + if (payload_len != 0) { + fletcher_4_incremental_native(payload, payload_len, zc); + if (write(outfd, payload, payload_len) == -1) + return (errno); + } + return (0); +} + +static void +rdt_insert(redup_table_t *rdt, + uint64_t guid, uint64_t object, uint64_t offset, uint64_t stream_offset) +{ + uint64_t ch = cityhash4(guid, object, offset, 0); + uint64_t hashcode = BF64_GET(ch, 0, rdt->numhashbits); + redup_entry_t **rdepp; + + rdepp = &(rdt->redup_hash_array[hashcode]); + redup_entry_t *rde = umem_cache_alloc(rdt->ddecache, UMEM_NOFAIL); + rde->rde_next = *rdepp; + rde->rde_guid = guid; + rde->rde_object = object; + rde->rde_offset = offset; + rde->rde_stream_offset = stream_offset; + *rdepp = rde; + rdt->ddt_count++; +} + +static void +rdt_lookup(redup_table_t *rdt, + uint64_t guid, uint64_t object, uint64_t offset, + uint64_t *stream_offsetp) +{ + uint64_t ch = cityhash4(guid, object, offset, 0); + uint64_t hashcode = BF64_GET(ch, 0, rdt->numhashbits); + + for (redup_entry_t *rde = rdt->redup_hash_array[hashcode]; + rde != NULL; rde = rde->rde_next) { + if (rde->rde_guid == guid && + rde->rde_object == object && + rde->rde_offset == offset) { + *stream_offsetp = rde->rde_stream_offset; + return; + } + } + assert(!"could not find expected redup table entry"); +} + +/* + * Convert a dedup stream (generated by "zfs send -D") to a + * non-deduplicated stream. The entire infd will be converted, including + * any substreams in a stream package (generated by "zfs send -RD"). The + * infd must be seekable. + */ +static void +zfs_redup_stream(int infd, int outfd, boolean_t verbose) +{ + int bufsz = SPA_MAXBLOCKSIZE; + dmu_replay_record_t thedrr = { 0 }; + dmu_replay_record_t *drr = &thedrr; + redup_table_t rdt; + zio_cksum_t stream_cksum; + uint64_t numbuckets; + uint64_t num_records = 0; + uint64_t num_write_byref_records = 0; + +#ifdef _ILP32 + uint64_t max_rde_size = SMALLEST_POSSIBLE_MAX_RDT_MB << 20; +#else + uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); + uint64_t max_rde_size = + MAX((physmem * MAX_RDT_PHYSMEM_PERCENT) / 100, + SMALLEST_POSSIBLE_MAX_RDT_MB << 20); +#endif + + numbuckets = max_rde_size / (sizeof (redup_entry_t)); + + /* + * numbuckets must be a power of 2. Increase number to + * a power of 2 if necessary. + */ + if (!ISP2(numbuckets)) + numbuckets = 1ULL << highbit64(numbuckets); + + rdt.redup_hash_array = + safe_calloc(numbuckets * sizeof (redup_entry_t *)); + rdt.ddecache = umem_cache_create("rde", sizeof (redup_entry_t), 0, + NULL, NULL, NULL, NULL, NULL, 0); + rdt.numhashbits = highbit64(numbuckets) - 1; + + char *buf = safe_calloc(bufsz); + FILE *ofp = fdopen(infd, "r"); + long offset = ftell(ofp); + while (sfread(drr, sizeof (*drr), ofp) != 0) { + num_records++; + + /* + * We need to regenerate the checksum. + */ + if (drr->drr_type != DRR_BEGIN) { + bzero(&drr->drr_u.drr_checksum.drr_checksum, + sizeof (drr->drr_u.drr_checksum.drr_checksum)); + } + + uint64_t payload_size = 0; + switch (drr->drr_type) { + case DRR_BEGIN: + { + struct drr_begin *drrb = &drr->drr_u.drr_begin; + int fflags; + ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); + + assert(drrb->drr_magic == DMU_BACKUP_MAGIC); + + /* clear the DEDUP feature flag for this stream */ + fflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo); + fflags &= ~(DMU_BACKUP_FEATURE_DEDUP | + DMU_BACKUP_FEATURE_DEDUPPROPS); + DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags); + + int sz = drr->drr_payloadlen; + if (sz != 0) { + if (sz > bufsz) { + free(buf); + buf = safe_calloc(sz); + bufsz = sz; + } + (void) sfread(buf, sz, ofp); + } + payload_size = sz; + break; + } + + case DRR_END: + { + struct drr_end *drre = &drr->drr_u.drr_end; + /* + * Use the recalculated checksum, unless this is + * the END record of a stream package, which has + * no checksum. + */ + if (!ZIO_CHECKSUM_IS_ZERO(&drre->drr_checksum)) + drre->drr_checksum = stream_cksum; + break; + } + + case DRR_OBJECT: + { + struct drr_object *drro = &drr->drr_u.drr_object; + + if (drro->drr_bonuslen > 0) { + payload_size = DRR_OBJECT_PAYLOAD_SIZE(drro); + (void) sfread(buf, payload_size, ofp); + } + break; + } + + case DRR_SPILL: + { + struct drr_spill *drrs = &drr->drr_u.drr_spill; + payload_size = DRR_SPILL_PAYLOAD_SIZE(drrs); + (void) sfread(buf, payload_size, ofp); + break; + } + + case DRR_WRITE_BYREF: + { + struct drr_write_byref drrwb = + drr->drr_u.drr_write_byref; + + num_write_byref_records++; + + /* + * Look up in hash table by drrwb->drr_refguid, + * drr_refobject, drr_refoffset. Replace this + * record with the found WRITE record, but with + * drr_object,drr_offset,drr_toguid replaced with ours. + */ + uint64_t stream_offset; + rdt_lookup(&rdt, drrwb.drr_refguid, + drrwb.drr_refobject, drrwb.drr_refoffset, + &stream_offset); + + spread(infd, drr, sizeof (*drr), stream_offset); + + assert(drr->drr_type == DRR_WRITE); + struct drr_write *drrw = &drr->drr_u.drr_write; + assert(drrw->drr_toguid == drrwb.drr_refguid); + assert(drrw->drr_object == drrwb.drr_refobject); + assert(drrw->drr_offset == drrwb.drr_refoffset); + + payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); + spread(infd, buf, payload_size, + stream_offset + sizeof (*drr)); + + drrw->drr_toguid = drrwb.drr_toguid; + drrw->drr_object = drrwb.drr_object; + drrw->drr_offset = drrwb.drr_offset; + break; + } + + case DRR_WRITE: + { + struct drr_write *drrw = &drr->drr_u.drr_write; + payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); + (void) sfread(buf, payload_size, ofp); + + rdt_insert(&rdt, drrw->drr_toguid, + drrw->drr_object, drrw->drr_offset, offset); + break; + } + + case DRR_WRITE_EMBEDDED: + { + struct drr_write_embedded *drrwe = + &drr->drr_u.drr_write_embedded; + payload_size = + P2ROUNDUP((uint64_t)drrwe->drr_psize, 8); + (void) sfread(buf, payload_size, ofp); + break; + } + + case DRR_FREEOBJECTS: + case DRR_FREE: + case DRR_OBJECT_RANGE: + break; + + default: + (void) fprintf(stderr, "INVALID record type 0x%x\n", + drr->drr_type); + /* should never happen, so assert */ + assert(B_FALSE); + } + + if (feof(ofp)) { + fprintf(stderr, "Error: unexpected end-of-file\n"); + exit(1); + } + if (ferror(ofp)) { + fprintf(stderr, "Error while reading file: %s\n", + strerror(errno)); + exit(1); + } + + /* + * We need to recalculate the checksum, and it needs to be + * initially zero to do that. BEGIN records don't have + * a checksum. + */ + if (drr->drr_type != DRR_BEGIN) { + bzero(&drr->drr_u.drr_checksum.drr_checksum, + sizeof (drr->drr_u.drr_checksum.drr_checksum)); + } + if (dump_record(drr, buf, payload_size, + &stream_cksum, outfd) != 0) + break; + if (drr->drr_type == DRR_END) { + /* + * Typically the END record is either the last + * thing in the stream, or it is followed + * by a BEGIN record (which also zeros the checksum). + * However, a stream package ends with two END + * records. The last END record's checksum starts + * from zero. + */ + ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); + } + offset = ftell(ofp); + } + + if (verbose) { + char mem_str[16]; + zfs_nicenum(rdt.ddt_count * sizeof (redup_entry_t), + mem_str, sizeof (mem_str)); + fprintf(stderr, "converted stream with %llu total records, " + "including %llu dedup records, using %sB memory.\n", + (long long)num_records, + (long long)num_write_byref_records, + mem_str); + } + + umem_cache_destroy(rdt.ddecache); + free(rdt.redup_hash_array); + free(buf); + (void) fclose(ofp); +} + +int +zstream_do_redup(int argc, char *argv[]) +{ + boolean_t verbose = B_FALSE; + char c; + + while ((c = getopt(argc, argv, "v")) != -1) { + switch (c) { + case 'v': + verbose = B_TRUE; + break; + case '?': + (void) fprintf(stderr, "invalid option '%c'\n", + optopt); + zstream_usage(); + break; + } + } + + argc -= optind; + argv += optind; + + if (argc != 1) + zstream_usage(); + + const char *filename = argv[0]; + + if (isatty(STDOUT_FILENO)) { + (void) fprintf(stderr, + "Error: Stream can not be written to a terminal.\n" + "You must redirect standard output.\n"); + return (1); + } + + int fd = open(filename, O_RDONLY); + if (fd == -1) { + (void) fprintf(stderr, + "Error while opening file '%s': %s\n", + filename, strerror(errno)); + exit(1); + } + + fletcher_4_init(); + zfs_redup_stream(fd, STDOUT_FILENO, verbose); + fletcher_4_fini(); + + close(fd); + + return (0); +} diff --git a/cmd/zstreamdump/Makefile.am b/cmd/zstreamdump/Makefile.am index 1f5cd4d9ff8..2c04d851315 100644 --- a/cmd/zstreamdump/Makefile.am +++ b/cmd/zstreamdump/Makefile.am @@ -1,10 +1 @@ -include $(top_srcdir)/config/Rules.am - -sbin_PROGRAMS = zstreamdump - -zstreamdump_SOURCES = \ - zstreamdump.c - -zstreamdump_LDADD = \ - $(top_builddir)/lib/libnvpair/libnvpair.la \ - $(top_builddir)/lib/libzfs/libzfs.la +dist_sbin_SCRIPTS = zstreamdump diff --git a/cmd/zstreamdump/zstreamdump b/cmd/zstreamdump/zstreamdump new file mode 100755 index 00000000000..fbf02ee687f --- /dev/null +++ b/cmd/zstreamdump/zstreamdump @@ -0,0 +1,3 @@ +#!/bin/sh + +zstream dump "$@" diff --git a/configure.ac b/configure.ac index 8604cdaa57f..7522940d2c3 100644 --- a/configure.ac +++ b/configure.ac @@ -80,6 +80,7 @@ AC_CONFIG_FILES([ cmd/zhack/Makefile cmd/zinject/Makefile cmd/zpool/Makefile + cmd/zstream/Makefile cmd/zstreamdump/Makefile cmd/ztest/Makefile cmd/zvol_id/Makefile diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 138d1ba08ec..43a39e789c5 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -61,6 +61,7 @@ #include "zfs_prop.h" #include "zfs_fletcher.h" #include "libzfs_impl.h" +#include #include #include #include @@ -5518,9 +5519,7 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, } /* Holds feature is set once in the compound stream header. */ - boolean_t holds = (DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) & - DMU_BACKUP_FEATURE_HOLDS); - if (holds) + if (featureflags & DMU_BACKUP_FEATURE_HOLDS) flags->holds = B_TRUE; if (strchr(drrb->drr_toname, '@') == NULL) { diff --git a/man/man8/Makefile.am b/man/man8/Makefile.am index 8239c2157d1..b7d26570e95 100644 --- a/man/man8/Makefile.am +++ b/man/man8/Makefile.am @@ -78,6 +78,7 @@ dist_man_MANS = \ zpool-trim.8 \ zpool-upgrade.8 \ zpool-wait.8 \ + zstream.8 \ zstreamdump.8 nodist_man_MANS = \ diff --git a/man/man8/zstream.8 b/man/man8/zstream.8 new file mode 100644 index 00000000000..1c4d3fa9a45 --- /dev/null +++ b/man/man8/zstream.8 @@ -0,0 +1,101 @@ +.\" +.\" CDDL HEADER START +.\" +.\" The contents of this file are subject to the terms of the +.\" Common Development and Distribution License (the "License"). +.\" You may not use this file except in compliance with the License. +.\" +.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +.\" or http://www.opensolaris.org/os/licensing. +.\" See the License for the specific language governing permissions +.\" and limitations under the License. +.\" +.\" When distributing Covered Code, include this CDDL HEADER in each +.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE. +.\" If applicable, add the following below this CDDL HEADER, with the +.\" fields enclosed by brackets "[]" replaced with your own identifying +.\" information: Portions Copyright [yyyy] [name of copyright owner] +.\" +.\" CDDL HEADER END +.\" +.\" +.\" Copyright (c) 2020 by Delphix. All rights reserved. +.Dd March 25, 2020 +.Dt ZSTREAM 8 +.Os Linux +.Sh NAME +.Nm zstream +.Nd manipulate zfs send streams +.Sh SYNOPSIS +.Nm +.Cm dump +.Op Fl Cvd +.Op Ar file +.Nm +.Cm redup +.Op Fl v +.Ar file +.Sh DESCRIPTION +.sp +.LP +The +.Sy zstream +utility manipulates zfs send streams, which are the output of the +.Sy zfs send +command. +.Bl -tag -width "" +.It Xo +.Nm +.Cm dump +.Op Fl Cvd +.Op Ar file +.Xc +Print information about the specified send stream, including headers and +record counts. +The send stream may either be in the specified +.Ar file , +or provided on standard input. +.Bl -tag -width "-D" +.It Fl C +Suppress the validation of checksums. +.It Fl v +Verbose. +Print metadata for each record. +.It Fl d +Dump data contained in each record. +Implies verbose. +.El +.It Xo +.Nm +.Cm redup +.Op Fl v +.Ar file +.Xc +Deduplicated send streams can be generated by using the +.Nm zfs Cm send Fl D +command. +The ability to send deduplicated send streams is deprecated. +In the future, the ability to receive a deduplicated send stream with +.Nm zfs Cm receive +will be removed. +However, deduplicated send streams can still be received by utilizing +.Nm zstream Cm redup . +.Pp +The +.Nm zstream Cm redup +command is provided a +.Ar file +containing a deduplicated send stream, and outputs an equivalent +non-deduplicated send stream on standard output. +Therefore, a deduplicated send stream can be received by running: +.Bd -literal +# zstream redup DEDUP_STREAM_FILE | zfs receive ... +.Ed +.Bl -tag -width "-D" +.It Fl v +Verbose. +Print summary of converted records. +.Sh SEE ALSO +.Xr zfs 8 , +.Xr zfs-send 8 , +.Xr zfs-receive 8 diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg index 4498f1a5303..cf65313ac51 100644 --- a/tests/zfs-tests/include/commands.cfg +++ b/tests/zfs-tests/include/commands.cfg @@ -182,6 +182,7 @@ export ZFS_FILES='zdb dbufstat zed zgenhostid + zstream zstreamdump' export ZFSTEST_FILES='btree_test diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh index 5d7a7043b11..be8f49809d4 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh @@ -67,6 +67,8 @@ zfs snapshot $src_fs@snap3 log_must eval "zfs send -D -R $src_fs@snap3 > $streamfile" log_must eval "zfs receive -v $dst_fs < $streamfile" +log_must zfs destroy -r $dst_fs +log_must eval "zstream redup $streamfile | zfs receive -v $dst_fs" cleanup diff --git a/tests/zfs-tests/tests/functional/rsend/send-cD.ksh b/tests/zfs-tests/tests/functional/rsend/send-cD.ksh index 97db0a7c212..fcbec2d9e9b 100755 --- a/tests/zfs-tests/tests/functional/rsend/send-cD.ksh +++ b/tests/zfs-tests/tests/functional/rsend/send-cD.ksh @@ -64,14 +64,26 @@ typeset size0=$(stat_size $stream0) typeset size1=$(stat_size $stream1) within_percent $size0 $size1 90 || log_fail "$size0 and $size1" -# Finally, make sure the receive works correctly. +# make sure the receive works correctly. log_must eval "zfs send -D -c -i snap0 $sendfs@snap1 >$inc" log_must eval "zfs recv -d $recvfs <$stream0" log_must eval "zfs recv -d $recvfs <$inc" cmp_ds_cont $sendfs $recvfs +# check receive with redup. +log_must zfs destroy -r $recvfs +log_must zfs create -o compress=lz4 $recvfs +log_must eval "zstream redup $stream0 | zfs recv -d $recvfs" +log_must eval "zstream redup $inc | zfs recv -d $recvfs" +cmp_ds_cont $sendfs $recvfs + # The size of the incremental should be the same as the initial send. typeset size2=$(stat_size $inc) within_percent $size0 $size2 90 || log_fail "$size0 and $size1" +# The redup'ed size should be 4x +typeset size3=$(zstream redup $inc | wc -c) +let size4=size0*4 +within_percent $size4 $size3 90 || log_fail "$size4 and $size3" + log_pass "The -c and -D flags do not interfere with each other" From 808084825492721571d617e6c670dcf307ddffcd Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 10 Apr 2020 21:10:09 -0700 Subject: [PATCH 010/133] Minor `zstream redup` command fixes * Fix uninitialized variable in `zstream redup` command. The 'rdt.ddt_count' variable is uninitialized because it was allocated from the stack and not globally. Initialize it. This was reported by gcc when compiling with debugging enabled. zstream_redup.c:157:16: error: 'rdt.ddt_count' may be used uninitialized in this function [-Werror=maybe-uninitialized] * Remove the cmd/zstreamdump/.gitignore file. It's no longer needed now that the zstreamdump command is a script. Reviewed-by: Matthew Ahrens Signed-off-by: Brian Behlendorf Closes #10192 --- cmd/zstream/zstream_redup.c | 1 + cmd/zstreamdump/.gitignore | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 cmd/zstreamdump/.gitignore diff --git a/cmd/zstream/zstream_redup.c b/cmd/zstream/zstream_redup.c index 6720cfd1201..9ed8df337f0 100644 --- a/cmd/zstream/zstream_redup.c +++ b/cmd/zstream/zstream_redup.c @@ -218,6 +218,7 @@ zfs_redup_stream(int infd, int outfd, boolean_t verbose) rdt.ddecache = umem_cache_create("rde", sizeof (redup_entry_t), 0, NULL, NULL, NULL, NULL, NULL, 0); rdt.numhashbits = highbit64(numbuckets) - 1; + rdt.ddt_count = 0; char *buf = safe_calloc(bufsz); FILE *ofp = fdopen(infd, "r"); diff --git a/cmd/zstreamdump/.gitignore b/cmd/zstreamdump/.gitignore deleted file mode 100644 index ca44a529eba..00000000000 --- a/cmd/zstreamdump/.gitignore +++ /dev/null @@ -1 +0,0 @@ -zstreamdump From 20f287855a61a64a90e01edfd551ff28f7abdb5c Mon Sep 17 00:00:00 2001 From: Matthew Ahrens Date: Fri, 10 Apr 2020 21:14:01 -0700 Subject: [PATCH 011/133] zvol_write() can use dmu_tx_hold_write_by_dnode() We can improve the performance of writes to zvols by using dmu_tx_hold_write_by_dnode() instead of dmu_tx_hold_write(). This reduces lock contention on the first block of the dnode object, and also reduces the amount of CPU needed. The benefit will be highest with multi-threaded async writes (i.e. writes that don't call zil_commit()). Reviewed-by: Jorgen Lundman Reviewed-by: Brian Behlendorf Reviewed-by: Tony Nguyen Signed-off-by: Matthew Ahrens Closes #10184 --- module/os/linux/zfs/zvol_os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/os/linux/zfs/zvol_os.c b/module/os/linux/zfs/zvol_os.c index fe45d76a62a..286c39d5ce8 100644 --- a/module/os/linux/zfs/zvol_os.c +++ b/module/os/linux/zfs/zvol_os.c @@ -143,7 +143,7 @@ zvol_write(void *arg) if (bytes > volsize - off) /* don't write past the end */ bytes = volsize - off; - dmu_tx_hold_write(tx, ZVOL_OBJ, off, bytes); + dmu_tx_hold_write_by_dnode(tx, zv->zv_dn, off, bytes); /* This will only fail for ENOSPC */ error = dmu_tx_assign(tx, TXG_WAIT); From c602b35cf7091e5afd192c9a943433379e21f567 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 14 Apr 2020 01:50:41 +0800 Subject: [PATCH 012/133] ZTS: Fix and change testcase cache_010_neg Commit 379ca9c removed the requirement on aux devices to be block devices only but the test case cache_010_neg was not updated, making it fail consistently. This change changes the test to check that cache devices _can_ be anything that presents a block interface. The testcase is renamed to cache_010_pos and the exceptions for known failure removed from the test runner. Reviewed-by: Ryan Moeller Reviewed-by: Brian Behlendorf Reported-by: Richard Elling Signed-off-by: Alex John Closes #10172 --- tests/runfiles/common.run | 2 +- tests/test-runner/bin/zts-report.py | 1 - .../tests/functional/cache/Makefile.am | 2 +- .../{cache_010_neg.ksh => cache_010_pos.ksh} | 50 ++++++++++--------- .../tests/functional/cache/setup.ksh | 4 -- 5 files changed, 29 insertions(+), 30 deletions(-) rename tests/zfs-tests/tests/functional/cache/{cache_010_neg.ksh => cache_010_pos.ksh} (64%) diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index af720ad9b2d..ccf03af89dc 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -49,7 +49,7 @@ post = [tests/functional/cache] tests = ['cache_001_pos', 'cache_002_pos', 'cache_003_pos', 'cache_004_neg', 'cache_005_neg', 'cache_006_pos', 'cache_007_neg', 'cache_008_neg', - 'cache_009_pos', 'cache_010_neg', 'cache_011_pos', 'cache_012_pos'] + 'cache_009_pos', 'cache_010_pos', 'cache_011_pos', 'cache_012_pos'] tags = ['functional', 'cache'] [tests/functional/cachefile] diff --git a/tests/test-runner/bin/zts-report.py b/tests/test-runner/bin/zts-report.py index 92fc5f6d6e1..7fc84fcecf9 100755 --- a/tests/test-runner/bin/zts-report.py +++ b/tests/test-runner/bin/zts-report.py @@ -191,7 +191,6 @@ maybe = { 'alloc_class/alloc_class_012_pos': ['FAIL', '9142'], 'alloc_class/alloc_class_013_pos': ['FAIL', '9142'], - 'cache/cache_010_neg': ['FAIL', known_reason], 'chattr/setup': ['SKIP', exec_reason], 'cli_root/zdb/zdb_006_pos': ['FAIL', known_reason], 'cli_root/zfs_get/zfs_get_004_pos': ['FAIL', known_reason], diff --git a/tests/zfs-tests/tests/functional/cache/Makefile.am b/tests/zfs-tests/tests/functional/cache/Makefile.am index 406a9281709..f28130ee9e4 100644 --- a/tests/zfs-tests/tests/functional/cache/Makefile.am +++ b/tests/zfs-tests/tests/functional/cache/Makefile.am @@ -11,7 +11,7 @@ dist_pkgdata_SCRIPTS = \ cache_007_neg.ksh \ cache_008_neg.ksh \ cache_009_pos.ksh \ - cache_010_neg.ksh \ + cache_010_pos.ksh \ cache_011_pos.ksh \ cache_012_pos.ksh diff --git a/tests/zfs-tests/tests/functional/cache/cache_010_neg.ksh b/tests/zfs-tests/tests/functional/cache/cache_010_pos.ksh similarity index 64% rename from tests/zfs-tests/tests/functional/cache/cache_010_neg.ksh rename to tests/zfs-tests/tests/functional/cache/cache_010_pos.ksh index 1d0683b8587..1d9fc5a8922 100755 --- a/tests/zfs-tests/tests/functional/cache/cache_010_neg.ksh +++ b/tests/zfs-tests/tests/functional/cache/cache_010_pos.ksh @@ -34,12 +34,12 @@ # # DESCRIPTION: -# Verify cache device must be a block device. +# Verify that cache devices can be block devices, files or character devices # # STRATEGY: # 1. Create a pool # 2. Add different object as cache -# 3. Verify character devices and files fail +# 3. Verify character devices and files pass # verify_runnable "global" @@ -50,51 +50,55 @@ function cleanup_testenv if [[ -n $lofidev ]]; then if is_linux; then losetup -d $lofidev + elif is_freebsd; then + mdconfig -du ${lofidev#md} else lofiadm -d $lofidev fi fi } -log_assert "Cache device can only be block devices." +log_assert "Verify cache devices can be disk, file, lofi device or any " \ + "device that presents a block interface" + +verify_disk_count "$DISKS" 2 log_onexit cleanup_testenv TESTVOL=testvol1$$ dsk1=${DISKS%% *} log_must zpool create $TESTPOOL ${DISKS#$dsk1} -# Add nomal ${DEV_RDSKDIR} device +# Add normal ${DEV_RDSKDIR} device log_must zpool add $TESTPOOL cache \ - ${DEV_RDSKDIR}/${dsk1}${SLICE_PREFIX}${SLICE0} + ${DEV_RDSKDIR}/${dsk1} +log_must zpool remove $TESTPOOL ${DEV_RDSKDIR}/${dsk1} + + +# Add provided disk +log_must zpool add $TESTPOOL cache $dsk1 log_must verify_cache_device $TESTPOOL $dsk1 'ONLINE' +log_must zpool remove $TESTPOOL $dsk1 # Add normal file -log_mustnot zpool add $TESTPOOL cache $VDEV2 +log_must zpool add $TESTPOOL cache $VDEV +ldev=$(random_get $VDEV) +log_must verify_cache_device $TESTPOOL $ldev 'ONLINE' -# Add /dev/rlofi device (allowed under Linux) +# Add loop back device if is_linux; then lofidev=$(losetup -f) - lofidev=${lofidev##*/} log_must losetup $lofidev ${VDEV2%% *} - log_must zpool add $TESTPOOL cache $lofidev - log_must zpool remove $TESTPOOL $lofidev - log_must losetup -d $lofidev - lofidev="" + lofidev=${lofidev##*/} +elif is_freebsd; then + lofidev=$(mdconfig -a ${VDEV2%% *}) else lofidev=${VDEV2%% *} log_must lofiadm -a $lofidev lofidev=$(lofiadm $lofidev) - log_mustnot zpool add $TESTPOOL cache "/dev/rlofi/${lofidev#/dev/lofi/}" - log_must lofiadm -d $lofidev - lofidev="" fi -# Add /dev/zvol/rdsk device (allowed under Linux) -if ! is_linux; then - log_must zpool create $TESTPOOL2 $VDEV2 - log_must zfs create -V $SIZE $TESTPOOL2/$TESTVOL - log_mustnot zpool add $TESTPOOL cache \ - ${ZVOL_RDEVDIR}/$TESTPOOL2/$TESTVOL -fi +log_must zpool add $TESTPOOL cache $lofidev +log_must verify_cache_device $TESTPOOL $lofidev 'ONLINE' -log_pass "Cache device can only be block devices." +log_pass "Verify cache devices can be disk, file, lofi device or any " \ + "device that presents a block interface" diff --git a/tests/zfs-tests/tests/functional/cache/setup.ksh b/tests/zfs-tests/tests/functional/cache/setup.ksh index d5da5d9bb06..0493637fcca 100755 --- a/tests/zfs-tests/tests/functional/cache/setup.ksh +++ b/tests/zfs-tests/tests/functional/cache/setup.ksh @@ -34,10 +34,6 @@ verify_runnable "global" -if ! is_physical_device $LDEV; then - log_unsupported "Only physical disk could be cache device" -fi - log_must rm -rf $VDIR $VDIR2 log_must mkdir -p $VDIR $VDIR2 log_must mkfile $SIZE $VDEV $VDEV2 From 791e480c6ac7a3e33adc5270b3fc99015f9f0c15 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Mon, 13 Apr 2020 10:51:44 -0700 Subject: [PATCH 013/133] Disable user space reference tracking The memory and cpu cost of reference count tracking with the current implementation is significant. For this reason it has always been disabled by default for the kmods. Apply this same default to user space so ztest doesn't always incur this performance penalty. Our intention is to re-enable this by default for ztest once the code has been optimized. Since we expect to at some point provide a FUSE implementation we wouldn't want this enabled by default for libzpool. Reviewed-by: Ryan Moeller Signed-off-by: Brian Behlendorf Closes #10189 --- module/zfs/refcount.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/module/zfs/refcount.c b/module/zfs/refcount.c index a612b2f40c0..6c9c305edaa 100644 --- a/module/zfs/refcount.c +++ b/module/zfs/refcount.c @@ -26,11 +26,12 @@ #include #include -#ifdef _KERNEL -int reference_tracking_enable = FALSE; /* runs out of memory too easily */ -#else -int reference_tracking_enable = TRUE; -#endif +/* + * Reference count tracking is disabled by default. It's memory requirements + * are reasonable, however as implemented it consumes a significant amount of + * cpu time. Until its performance is improved it should be manually enabled. + */ +int reference_tracking_enable = FALSE; int reference_history = 3; /* tunable */ #ifdef ZFS_DEBUG From 75c62019f3938e7bc81becb4fb2d5b5eb523e79a Mon Sep 17 00:00:00 2001 From: Joao Carlos Mendes Luis Date: Mon, 13 Apr 2020 14:54:41 -0300 Subject: [PATCH 014/133] Fix allocation errors, detected using ASAN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test for VDEV_TYPE_INDIRECT is done after a memory allocation, and could return from function without freeing it. Since we don't need that allocation yet, just postpone it. Add a missing free() when buffer is no longer needed. Reviewed-by: Matthew Ahrens Reviewed-by: Brian Behlendorf Signed-off-by: João Carlos Mendes Luís Closes #10193 --- cmd/zpool/zpool_main.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index b5f2f9b0230..e8e94cd94eb 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -4395,11 +4395,11 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv, uint64_t tdelta; double scale; - calcvs = safe_malloc(sizeof (*calcvs)); - if (strcmp(name, VDEV_TYPE_INDIRECT) == 0) return (ret); + calcvs = safe_malloc(sizeof (*calcvs)); + if (oldnv != NULL) { verify(nvlist_lookup_uint64_array(oldnv, ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0); @@ -7387,6 +7387,7 @@ print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs) ", (copy is slow, no estimated time)\n")); } } + free(vdev_name); if (prs->prs_mapping_memory > 0) { char mem_buf[7]; From 9f0a21e6411aa0bac23fba0ddb220342a48c7cc7 Mon Sep 17 00:00:00 2001 From: Matthew Macy Date: Tue, 14 Apr 2020 11:36:28 -0700 Subject: [PATCH 015/133] Add FreeBSD support to OpenZFS Add the FreeBSD platform code to the OpenZFS repository. As of this commit the source can be compiled and tested on FreeBSD 11 and 12. Subsequent commits are now required to compile on FreeBSD and Linux. Additionally, they must pass the ZFS Test Suite on FreeBSD which is being run by the CI. As of this commit 1230 tests pass on FreeBSD and there are no unexpected failures. Reviewed-by: Sean Eric Fagan Reviewed-by: Jorgen Lundman Reviewed-by: Richard Laager Reviewed-by: Brian Behlendorf Co-authored-by: Ryan Moeller Signed-off-by: Matt Macy Signed-off-by: Ryan Moeller Closes #898 Closes #8987 --- .github/suppressions.txt | 5 +- .gitignore | 5 + Makefile.am | 5 +- README.md | 3 +- cmd/Makefile.am | 4 +- cmd/zpool/Makefile.am | 7 + cmd/zpool/os/freebsd/zpool_vdev_os.c | 113 + cmd/zpool/zpool_vdev.c | 4 + config/Rules.am | 26 +- config/always-arch.m4 | 2 +- config/always-compiler-options.m4 | 21 + config/kernel.m4 | 46 +- config/toolchain-simd.m4 | 2 +- config/user.m4 | 13 +- config/zfs-build.m4 | 38 +- configure.ac | 10 + contrib/Makefile.am | 5 +- etc/Makefile.am | 5 +- include/os/Makefile.am | 3 + include/os/freebsd/Makefile.am | 1 + include/os/freebsd/linux/Makefile.am | 5 + include/os/freebsd/linux/compiler.h | 101 + include/os/freebsd/linux/types.h | 80 + include/os/freebsd/spl/Makefile.am | 1 + include/os/freebsd/spl/acl/Makefile.am | 4 + include/os/freebsd/spl/acl/acl_common.h | 68 + include/os/freebsd/spl/rpc/Makefile.am | 8 + include/os/freebsd/spl/rpc/xdr.h | 71 + include/os/freebsd/spl/sys/Makefile.am | 72 + include/os/freebsd/spl/sys/acl.h | 216 + include/os/freebsd/spl/sys/acl_impl.h | 61 + include/os/freebsd/spl/sys/atomic.h | 188 + include/os/freebsd/spl/sys/byteorder.h | 93 + include/os/freebsd/spl/sys/callb.h | 213 + include/os/freebsd/spl/sys/ccompile.h | 372 + include/os/freebsd/spl/sys/cmn_err.h | 100 + include/os/freebsd/spl/sys/condvar.h | 170 + include/os/freebsd/spl/sys/console.h | 48 + include/os/freebsd/spl/sys/cred.h | 188 + include/os/freebsd/spl/sys/ctype.h | 45 + include/os/freebsd/spl/sys/debug.h | 168 + include/os/freebsd/spl/sys/dirent.h | 45 + include/os/freebsd/spl/sys/disp.h | 36 + include/os/freebsd/spl/sys/dkio.h | 495 ++ include/os/freebsd/spl/sys/endian.h | 42 + include/os/freebsd/spl/sys/extdirent.h | 73 + include/os/freebsd/spl/sys/file.h | 50 + include/os/freebsd/spl/sys/freebsd_rwlock.h | 34 + include/os/freebsd/spl/sys/inttypes.h | 1 + include/os/freebsd/spl/sys/isa_defs.h | 688 ++ include/os/freebsd/spl/sys/kmem.h | 102 + include/os/freebsd/spl/sys/kmem_cache.h | 49 + include/os/freebsd/spl/sys/kstat.h | 206 + include/os/freebsd/spl/sys/list.h | 67 + include/os/freebsd/spl/sys/list_impl.h | 53 + include/os/freebsd/spl/sys/lock.h | 40 + include/os/freebsd/spl/sys/misc.h | 56 + include/os/freebsd/spl/sys/mod_os.h | 97 + include/os/freebsd/spl/sys/mode.h | 1 + include/os/freebsd/spl/sys/mount.h | 41 + include/os/freebsd/spl/sys/mutex.h | 74 + include/os/freebsd/spl/sys/param.h | 38 + include/os/freebsd/spl/sys/policy.h | 77 + include/os/freebsd/spl/sys/proc.h | 120 + include/os/freebsd/spl/sys/processor.h | 63 + include/os/freebsd/spl/sys/procfs_list.h | 64 + include/os/freebsd/spl/sys/random.h | 48 + include/os/freebsd/spl/sys/rwlock.h | 97 + include/os/freebsd/spl/sys/sdt.h | 45 + include/os/freebsd/spl/sys/sid.h | 84 + include/os/freebsd/spl/sys/sig.h | 65 + include/os/freebsd/spl/sys/simd.h | 43 + include/os/freebsd/spl/sys/simd_x86.h | 296 + include/os/freebsd/spl/sys/spl_condvar.h | 81 + include/os/freebsd/spl/sys/string.h | 39 + include/os/freebsd/spl/sys/strings.h | 1 + include/os/freebsd/spl/sys/sunddi.h | 67 + include/os/freebsd/spl/sys/sysmacros.h | 404 + include/os/freebsd/spl/sys/systeminfo.h | 34 + include/os/freebsd/spl/sys/systm.h | 43 + include/os/freebsd/spl/sys/taskq.h | 114 + include/os/freebsd/spl/sys/thread.h | 34 + include/os/freebsd/spl/sys/time.h | 96 + include/os/freebsd/spl/sys/timer.h | 38 + include/os/freebsd/spl/sys/trace.h | 1 + include/os/freebsd/spl/sys/trace_zfs.h | 1 + include/os/freebsd/spl/sys/types.h | 97 + include/os/freebsd/spl/sys/types32.h | 37 + include/os/freebsd/spl/sys/uio.h | 77 + include/os/freebsd/spl/sys/uuid.h | 99 + include/os/freebsd/spl/sys/vfs.h | 127 + include/os/freebsd/spl/sys/vm.h | 57 + include/os/freebsd/spl/sys/vmsystm.h | 34 + include/os/freebsd/spl/sys/vnode.h | 204 + include/os/freebsd/spl/sys/vnode_impl.h | 268 + include/os/freebsd/spl/sys/zmod.h | 68 + include/os/freebsd/spl/sys/zone.h | 68 + include/os/freebsd/zfs/Makefile.am | 1 + include/os/freebsd/zfs/sys/Makefile.am | 14 + include/os/freebsd/zfs/sys/freebsd_crypto.h | 98 + include/os/freebsd/zfs/sys/sha2.h | 200 + include/os/freebsd/zfs/sys/vdev_os.h | 30 + include/os/freebsd/zfs/sys/zfs_context_os.h | 92 + include/os/freebsd/zfs/sys/zfs_ctldir.h | 65 + include/os/freebsd/zfs/sys/zfs_dir.h | 74 + include/os/freebsd/zfs/sys/zfs_ioctl_compat.h | 677 ++ include/os/freebsd/zfs/sys/zfs_vfsops.h | 172 + include/os/freebsd/zfs/sys/zfs_vnops.h | 56 + include/os/freebsd/zfs/sys/zfs_znode_impl.h | 182 + include/os/freebsd/zfs/sys/zpl.h | 1 + lib/Makefile.am | 9 +- lib/libnvpair/Makefile.am | 7 + lib/libspl/Makefile.am | 9 + lib/libspl/include/os/Makefile.am | 4 + lib/libspl/include/os/freebsd/Makefile.am | 1 + lib/libspl/include/os/freebsd/sys/Makefile.am | 12 + lib/libspl/include/os/freebsd/sys/byteorder.h | 311 + lib/libspl/include/os/freebsd/sys/file.h | 42 + lib/libspl/include/os/freebsd/sys/mnttab.h | 85 + lib/libspl/include/os/freebsd/sys/mount.h | 108 + lib/libspl/include/os/freebsd/sys/param.h | 70 + lib/libspl/include/os/freebsd/sys/stat.h | 71 + lib/libspl/include/os/freebsd/sys/sysmacros.h | 1 + lib/libspl/include/os/freebsd/sys/uio.h | 98 + lib/libspl/include/os/freebsd/sys/vfs.h | 37 + .../include/os/freebsd/sys/zfs_context_os.h | 38 + lib/libspl/os/freebsd/getexecname.c | 70 + lib/libspl/os/freebsd/gethostid.c | 36 + lib/libspl/os/freebsd/getmntany.c | 67 + lib/libspl/os/freebsd/mnttab.c | 216 + lib/libuutil/Makefile.am | 4 + lib/libzfs/Makefile.am | 15 +- lib/libzfs/libzfs_util.c | 2 +- lib/libzfs/os/freebsd/libzfs_compat.c | 323 + lib/libzfs/os/freebsd/libzfs_fsshare.c | 406 + lib/libzfs/os/freebsd/libzfs_ioctl_compat.c | 432 ++ lib/libzfs/os/freebsd/libzfs_zmount.c | 147 + lib/libzfs_core/Makefile.am | 6 +- lib/libzfs_core/libzfs_core_compat.h | 47 + lib/libzpool/Makefile.am | 5 + lib/libzutil/Makefile.am | 13 +- lib/libzutil/os/freebsd/zutil_compat.c | 74 + .../os/freebsd/zutil_device_path_os.c | 132 + lib/libzutil/os/freebsd/zutil_import_os.c | 239 + module/.gitignore | 10 + module/Makefile.bsd | 381 + module/Makefile.in | 27 +- module/os/freebsd/spl/acl_common.c | 1731 +++++ module/os/freebsd/spl/callb.c | 372 + module/os/freebsd/spl/list.c | 245 + module/os/freebsd/spl/sha224.h | 96 + module/os/freebsd/spl/sha256.h | 99 + module/os/freebsd/spl/sha256c.c | 378 + module/os/freebsd/spl/sha384.h | 96 + module/os/freebsd/spl/sha512.h | 101 + module/os/freebsd/spl/sha512c.c | 508 ++ module/os/freebsd/spl/sha512t.h | 143 + module/os/freebsd/spl/spl_acl.c | 222 + module/os/freebsd/spl/spl_atomic.c | 138 + module/os/freebsd/spl/spl_cmn_err.c | 74 + module/os/freebsd/spl/spl_dtrace.c | 37 + module/os/freebsd/spl/spl_kmem.c | 351 + module/os/freebsd/spl/spl_kstat.c | 321 + module/os/freebsd/spl/spl_misc.c | 107 + module/os/freebsd/spl/spl_policy.c | 429 ++ module/os/freebsd/spl/spl_procfs_list.c | 79 + module/os/freebsd/spl/spl_string.c | 106 + module/os/freebsd/spl/spl_sunddi.c | 74 + module/os/freebsd/spl/spl_sysevent.c | 259 + module/os/freebsd/spl/spl_taskq.c | 329 + module/os/freebsd/spl/spl_uio.c | 92 + module/os/freebsd/spl/spl_vfs.c | 278 + module/os/freebsd/spl/spl_vm.c | 71 + module/os/freebsd/spl/spl_zlib.c | 268 + module/os/freebsd/spl/spl_zone.c | 265 + module/os/freebsd/zfs/abd.c | 1134 +++ module/os/freebsd/zfs/arc_os.c | 245 + module/os/freebsd/zfs/crypto_os.c | 613 ++ module/os/freebsd/zfs/dmu_os.c | 346 + module/os/freebsd/zfs/hkdf.c | 102 + module/os/freebsd/zfs/kmod_core.c | 404 + module/os/freebsd/zfs/spa_os.c | 280 + module/os/freebsd/zfs/spa_stats.c | 114 + module/os/freebsd/zfs/sysctl_os.c | 699 ++ module/os/freebsd/zfs/vdev_file.c | 326 + module/os/freebsd/zfs/vdev_geom.c | 1195 +++ module/os/freebsd/zfs/vdev_label_os.c | 74 + module/os/freebsd/zfs/zfs_acl.c | 2738 +++++++ module/os/freebsd/zfs/zfs_ctldir.c | 1345 ++++ module/os/freebsd/zfs/zfs_debug.c | 254 + module/os/freebsd/zfs/zfs_dir.c | 961 +++ module/os/freebsd/zfs/zfs_file_os.c | 309 + module/os/freebsd/zfs/zfs_fuid_os.c | 52 + module/os/freebsd/zfs/zfs_ioctl_os.c | 194 + module/os/freebsd/zfs/zfs_onexit_os.c | 70 + module/os/freebsd/zfs/zfs_vfsops.c | 2448 ++++++ module/os/freebsd/zfs/zfs_vnops.c | 6533 +++++++++++++++++ module/os/freebsd/zfs/zfs_znode.c | 1987 +++++ module/os/freebsd/zfs/zio_crypt.c | 1882 +++++ module/os/freebsd/zfs/zvol_os.c | 1476 ++++ scripts/zfs-tests.sh | 2 +- tests/runfiles/Makefile.am | 1 + tests/runfiles/common.run | 6 + tests/runfiles/freebsd.run | 0 tests/runfiles/linux.run | 6 - tests/test-runner/bin/zts-report.py | 5 +- tests/zfs-tests/include/libtest.shlib | 18 +- tests/zfs-tests/include/tunables.cfg | 4 +- .../zpool_import_missing_003_pos.ksh | 4 +- 209 files changed, 46201 insertions(+), 81 deletions(-) create mode 100644 cmd/zpool/os/freebsd/zpool_vdev_os.c create mode 100644 include/os/freebsd/Makefile.am create mode 100644 include/os/freebsd/linux/Makefile.am create mode 100644 include/os/freebsd/linux/compiler.h create mode 100644 include/os/freebsd/linux/types.h create mode 100644 include/os/freebsd/spl/Makefile.am create mode 100644 include/os/freebsd/spl/acl/Makefile.am create mode 100644 include/os/freebsd/spl/acl/acl_common.h create mode 100644 include/os/freebsd/spl/rpc/Makefile.am create mode 100644 include/os/freebsd/spl/rpc/xdr.h create mode 100644 include/os/freebsd/spl/sys/Makefile.am create mode 100644 include/os/freebsd/spl/sys/acl.h create mode 100644 include/os/freebsd/spl/sys/acl_impl.h create mode 100644 include/os/freebsd/spl/sys/atomic.h create mode 100644 include/os/freebsd/spl/sys/byteorder.h create mode 100644 include/os/freebsd/spl/sys/callb.h create mode 100644 include/os/freebsd/spl/sys/ccompile.h create mode 100644 include/os/freebsd/spl/sys/cmn_err.h create mode 100644 include/os/freebsd/spl/sys/condvar.h create mode 100644 include/os/freebsd/spl/sys/console.h create mode 100644 include/os/freebsd/spl/sys/cred.h create mode 100644 include/os/freebsd/spl/sys/ctype.h create mode 100644 include/os/freebsd/spl/sys/debug.h create mode 100644 include/os/freebsd/spl/sys/dirent.h create mode 100644 include/os/freebsd/spl/sys/disp.h create mode 100644 include/os/freebsd/spl/sys/dkio.h create mode 100644 include/os/freebsd/spl/sys/endian.h create mode 100644 include/os/freebsd/spl/sys/extdirent.h create mode 100644 include/os/freebsd/spl/sys/file.h create mode 100644 include/os/freebsd/spl/sys/freebsd_rwlock.h create mode 100644 include/os/freebsd/spl/sys/inttypes.h create mode 100644 include/os/freebsd/spl/sys/isa_defs.h create mode 100644 include/os/freebsd/spl/sys/kmem.h create mode 100644 include/os/freebsd/spl/sys/kmem_cache.h create mode 100644 include/os/freebsd/spl/sys/kstat.h create mode 100644 include/os/freebsd/spl/sys/list.h create mode 100644 include/os/freebsd/spl/sys/list_impl.h create mode 100644 include/os/freebsd/spl/sys/lock.h create mode 100644 include/os/freebsd/spl/sys/misc.h create mode 100644 include/os/freebsd/spl/sys/mod_os.h create mode 100644 include/os/freebsd/spl/sys/mode.h create mode 100644 include/os/freebsd/spl/sys/mount.h create mode 100644 include/os/freebsd/spl/sys/mutex.h create mode 100644 include/os/freebsd/spl/sys/param.h create mode 100644 include/os/freebsd/spl/sys/policy.h create mode 100644 include/os/freebsd/spl/sys/proc.h create mode 100644 include/os/freebsd/spl/sys/processor.h create mode 100644 include/os/freebsd/spl/sys/procfs_list.h create mode 100644 include/os/freebsd/spl/sys/random.h create mode 100644 include/os/freebsd/spl/sys/rwlock.h create mode 100644 include/os/freebsd/spl/sys/sdt.h create mode 100644 include/os/freebsd/spl/sys/sid.h create mode 100644 include/os/freebsd/spl/sys/sig.h create mode 100644 include/os/freebsd/spl/sys/simd.h create mode 100644 include/os/freebsd/spl/sys/simd_x86.h create mode 100644 include/os/freebsd/spl/sys/spl_condvar.h create mode 100644 include/os/freebsd/spl/sys/string.h create mode 100644 include/os/freebsd/spl/sys/strings.h create mode 100644 include/os/freebsd/spl/sys/sunddi.h create mode 100644 include/os/freebsd/spl/sys/sysmacros.h create mode 100644 include/os/freebsd/spl/sys/systeminfo.h create mode 100644 include/os/freebsd/spl/sys/systm.h create mode 100644 include/os/freebsd/spl/sys/taskq.h create mode 100644 include/os/freebsd/spl/sys/thread.h create mode 100644 include/os/freebsd/spl/sys/time.h create mode 100644 include/os/freebsd/spl/sys/timer.h create mode 100644 include/os/freebsd/spl/sys/trace.h create mode 100644 include/os/freebsd/spl/sys/trace_zfs.h create mode 100644 include/os/freebsd/spl/sys/types.h create mode 100644 include/os/freebsd/spl/sys/types32.h create mode 100644 include/os/freebsd/spl/sys/uio.h create mode 100644 include/os/freebsd/spl/sys/uuid.h create mode 100644 include/os/freebsd/spl/sys/vfs.h create mode 100644 include/os/freebsd/spl/sys/vm.h create mode 100644 include/os/freebsd/spl/sys/vmsystm.h create mode 100644 include/os/freebsd/spl/sys/vnode.h create mode 100644 include/os/freebsd/spl/sys/vnode_impl.h create mode 100644 include/os/freebsd/spl/sys/zmod.h create mode 100644 include/os/freebsd/spl/sys/zone.h create mode 100644 include/os/freebsd/zfs/Makefile.am create mode 100644 include/os/freebsd/zfs/sys/Makefile.am create mode 100644 include/os/freebsd/zfs/sys/freebsd_crypto.h create mode 100644 include/os/freebsd/zfs/sys/sha2.h create mode 100644 include/os/freebsd/zfs/sys/vdev_os.h create mode 100644 include/os/freebsd/zfs/sys/zfs_context_os.h create mode 100644 include/os/freebsd/zfs/sys/zfs_ctldir.h create mode 100644 include/os/freebsd/zfs/sys/zfs_dir.h create mode 100644 include/os/freebsd/zfs/sys/zfs_ioctl_compat.h create mode 100644 include/os/freebsd/zfs/sys/zfs_vfsops.h create mode 100644 include/os/freebsd/zfs/sys/zfs_vnops.h create mode 100644 include/os/freebsd/zfs/sys/zfs_znode_impl.h create mode 100644 include/os/freebsd/zfs/sys/zpl.h create mode 100644 lib/libspl/include/os/freebsd/Makefile.am create mode 100644 lib/libspl/include/os/freebsd/sys/Makefile.am create mode 100644 lib/libspl/include/os/freebsd/sys/byteorder.h create mode 100644 lib/libspl/include/os/freebsd/sys/file.h create mode 100644 lib/libspl/include/os/freebsd/sys/mnttab.h create mode 100644 lib/libspl/include/os/freebsd/sys/mount.h create mode 100644 lib/libspl/include/os/freebsd/sys/param.h create mode 100644 lib/libspl/include/os/freebsd/sys/stat.h create mode 100644 lib/libspl/include/os/freebsd/sys/sysmacros.h create mode 100644 lib/libspl/include/os/freebsd/sys/uio.h create mode 100644 lib/libspl/include/os/freebsd/sys/vfs.h create mode 100644 lib/libspl/include/os/freebsd/sys/zfs_context_os.h create mode 100644 lib/libspl/os/freebsd/getexecname.c create mode 100644 lib/libspl/os/freebsd/gethostid.c create mode 100644 lib/libspl/os/freebsd/getmntany.c create mode 100644 lib/libspl/os/freebsd/mnttab.c create mode 100644 lib/libzfs/os/freebsd/libzfs_compat.c create mode 100644 lib/libzfs/os/freebsd/libzfs_fsshare.c create mode 100644 lib/libzfs/os/freebsd/libzfs_ioctl_compat.c create mode 100644 lib/libzfs/os/freebsd/libzfs_zmount.c create mode 100644 lib/libzfs_core/libzfs_core_compat.h create mode 100644 lib/libzutil/os/freebsd/zutil_compat.c create mode 100644 lib/libzutil/os/freebsd/zutil_device_path_os.c create mode 100644 lib/libzutil/os/freebsd/zutil_import_os.c create mode 100644 module/Makefile.bsd create mode 100644 module/os/freebsd/spl/acl_common.c create mode 100644 module/os/freebsd/spl/callb.c create mode 100644 module/os/freebsd/spl/list.c create mode 100644 module/os/freebsd/spl/sha224.h create mode 100644 module/os/freebsd/spl/sha256.h create mode 100644 module/os/freebsd/spl/sha256c.c create mode 100644 module/os/freebsd/spl/sha384.h create mode 100644 module/os/freebsd/spl/sha512.h create mode 100644 module/os/freebsd/spl/sha512c.c create mode 100644 module/os/freebsd/spl/sha512t.h create mode 100644 module/os/freebsd/spl/spl_acl.c create mode 100644 module/os/freebsd/spl/spl_atomic.c create mode 100644 module/os/freebsd/spl/spl_cmn_err.c create mode 100644 module/os/freebsd/spl/spl_dtrace.c create mode 100644 module/os/freebsd/spl/spl_kmem.c create mode 100644 module/os/freebsd/spl/spl_kstat.c create mode 100644 module/os/freebsd/spl/spl_misc.c create mode 100644 module/os/freebsd/spl/spl_policy.c create mode 100644 module/os/freebsd/spl/spl_procfs_list.c create mode 100644 module/os/freebsd/spl/spl_string.c create mode 100644 module/os/freebsd/spl/spl_sunddi.c create mode 100644 module/os/freebsd/spl/spl_sysevent.c create mode 100644 module/os/freebsd/spl/spl_taskq.c create mode 100644 module/os/freebsd/spl/spl_uio.c create mode 100644 module/os/freebsd/spl/spl_vfs.c create mode 100644 module/os/freebsd/spl/spl_vm.c create mode 100644 module/os/freebsd/spl/spl_zlib.c create mode 100644 module/os/freebsd/spl/spl_zone.c create mode 100644 module/os/freebsd/zfs/abd.c create mode 100644 module/os/freebsd/zfs/arc_os.c create mode 100644 module/os/freebsd/zfs/crypto_os.c create mode 100644 module/os/freebsd/zfs/dmu_os.c create mode 100644 module/os/freebsd/zfs/hkdf.c create mode 100644 module/os/freebsd/zfs/kmod_core.c create mode 100644 module/os/freebsd/zfs/spa_os.c create mode 100644 module/os/freebsd/zfs/spa_stats.c create mode 100644 module/os/freebsd/zfs/sysctl_os.c create mode 100644 module/os/freebsd/zfs/vdev_file.c create mode 100644 module/os/freebsd/zfs/vdev_geom.c create mode 100644 module/os/freebsd/zfs/vdev_label_os.c create mode 100644 module/os/freebsd/zfs/zfs_acl.c create mode 100644 module/os/freebsd/zfs/zfs_ctldir.c create mode 100644 module/os/freebsd/zfs/zfs_debug.c create mode 100644 module/os/freebsd/zfs/zfs_dir.c create mode 100644 module/os/freebsd/zfs/zfs_file_os.c create mode 100644 module/os/freebsd/zfs/zfs_fuid_os.c create mode 100644 module/os/freebsd/zfs/zfs_ioctl_os.c create mode 100644 module/os/freebsd/zfs/zfs_onexit_os.c create mode 100644 module/os/freebsd/zfs/zfs_vfsops.c create mode 100644 module/os/freebsd/zfs/zfs_vnops.c create mode 100644 module/os/freebsd/zfs/zfs_znode.c create mode 100644 module/os/freebsd/zfs/zio_crypt.c create mode 100644 module/os/freebsd/zfs/zvol_os.c create mode 100644 tests/runfiles/freebsd.run diff --git a/.github/suppressions.txt b/.github/suppressions.txt index f9508a24b4a..b28514e674b 100644 --- a/.github/suppressions.txt +++ b/.github/suppressions.txt @@ -1,3 +1,6 @@ preprocessorErrorDirective:./module/zfs/vdev_raidz_math_avx512f.c:243 preprocessorErrorDirective:./module/zfs/vdev_raidz_math_sse2.c:266 - +uninitvar:module/os/freebsd/zfs/vdev_geom.c +uninitvar:module/os/freebsd/zfs/zfs_vfsops.c +uninitvar:module/os/freebsd/spl/spl_zone.c +uninitvar:lib/libzutil/os/freebsd/zutil_import_os.c diff --git a/.gitignore b/.gitignore index 57867bfc6ea..056bbb8f08c 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,9 @@ cscope.* *.patch *.orig *.log +*.tmp venv + +*.so +*.so.debug +*.so.full diff --git a/Makefile.am b/Makefile.am index 4c0b541ccd4..101b38ac335 100644 --- a/Makefile.am +++ b/Makefile.am @@ -104,8 +104,9 @@ commitcheck: fi cstyle: - @find ${top_srcdir} -name build -prune -o -name '*.[hc]' \ - ! -name 'zfs_config.*' ! -name '*.mod.c' -type f \ + @find ${top_srcdir} -name build -prune -o -type f -name '*.[hc]' \ + ! -name 'zfs_config.*' ! -name '*.mod.c' \ + ! -name 'opt_global.h' ! -name '*_if*.h' \ -exec ${top_srcdir}/scripts/cstyle.pl -cpP {} \+ filter_executable = -exec test -x '{}' \; -print diff --git a/README.md b/README.md index ff8a0e85130..9c6ed752336 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,6 @@ This repository contains the code for running OpenZFS on Linux and FreeBSD. Full documentation for installing OpenZFS on your favorite Linux distribution can be found at the [ZoL Site](https://zfsonlinux.org/). -FreeBSD support is a work in progress. See the [PR](https://github.com/openzfs/zfs/pull/8987). - # Contribute & Develop We have a separate document with [contribution guidelines](./.github/CONTRIBUTING.md). @@ -34,3 +32,4 @@ For more details see the NOTICE, LICENSE and COPYRIGHT files; `UCRL-CODE-235197` # Supported Kernels * The `META` file contains the officially recognized supported Linux kernel versions. + * Supported FreeBSD versions are 12-STABLE and 13-CURRENT. diff --git a/cmd/Makefile.am b/cmd/Makefile.am index 2078bc13b3c..6b152e848e2 100644 --- a/cmd/Makefile.am +++ b/cmd/Makefile.am @@ -1,10 +1,10 @@ SUBDIRS = zfs zpool zdb zhack zinject zstream zstreamdump ztest -SUBDIRS += fsck_zfs vdev_id raidz_test zgenhostid +SUBDIRS += fsck_zfs vdev_id raidz_test if USING_PYTHON SUBDIRS += arcstat arc_summary dbufstat endif if BUILD_LINUX -SUBDIRS += mount_zfs zed zvol_id zvol_wait +SUBDIRS += mount_zfs zed zgenhostid zvol_id zvol_wait endif diff --git a/cmd/zpool/Makefile.am b/cmd/zpool/Makefile.am index b9e221c1f7c..7b25726f498 100644 --- a/cmd/zpool/Makefile.am +++ b/cmd/zpool/Makefile.am @@ -11,6 +11,10 @@ zpool_SOURCES = \ zpool_util.h \ zpool_vdev.c +if BUILD_FREEBSD +zpool_SOURCES += os/freebsd/zpool_vdev_os.c +endif + if BUILD_LINUX zpool_SOURCES += os/linux/zpool_vdev_os.c endif @@ -20,6 +24,9 @@ zpool_LDADD = \ $(top_builddir)/lib/libuutil/libuutil.la \ $(top_builddir)/lib/libzfs/libzfs.la +if BUILD_FREEBSD +zpool_LDADD += -L/usr/local/lib -lintl -lgeom +endif zpool_LDADD += -lm $(LIBBLKID) zpoolconfdir = $(sysconfdir)/zfs/zpool.d diff --git a/cmd/zpool/os/freebsd/zpool_vdev_os.c b/cmd/zpool/os/freebsd/zpool_vdev_os.c new file mode 100644 index 00000000000..4a8d9272d30 --- /dev/null +++ b/cmd/zpool/os/freebsd/zpool_vdev_os.c @@ -0,0 +1,113 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2018 by Delphix. All rights reserved. + * Copyright (c) 2016, 2017 Intel Corporation. + * Copyright 2016 Igor Kozhukhov . + */ + +/* + * Functions to convert between a list of vdevs and an nvlist representing the + * configuration. Each entry in the list can be one of: + * + * Device vdevs + * disk=(path=..., devid=...) + * file=(path=...) + * + * Group vdevs + * raidz[1|2]=(...) + * mirror=(...) + * + * Hot spares + * + * While the underlying implementation supports it, group vdevs cannot contain + * other group vdevs. All userland verification of devices is contained within + * this file. If successful, the nvlist returned can be passed directly to the + * kernel; we've done as much verification as possible in userland. + * + * Hot spares are a special case, and passed down as an array of disk vdevs, at + * the same level as the root of the vdev tree. + * + * The only function exported by this file is 'make_root_vdev'. The + * function performs several passes: + * + * 1. Construct the vdev specification. Performs syntax validation and + * makes sure each device is valid. + * 2. Check for devices in use. Using libdiskmgt, makes sure that no + * devices are also in use. Some can be overridden using the 'force' + * flag, others cannot. + * 3. Check for replication errors if the 'force' flag is not specified. + * validates that the replication level is consistent across the + * entire pool. + * 4. Call libzfs to label any whole disks with an EFI label. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zpool_util.h" +#include + +int +check_device(const char *name, boolean_t force, boolean_t isspare, + boolean_t iswholedisk) +{ + char path[MAXPATHLEN]; + + if (strncmp(name, _PATH_DEV, sizeof (_PATH_DEV) - 1) != 0) + snprintf(path, sizeof (path), "%s%s", _PATH_DEV, name); + else + strlcpy(path, name, sizeof (path)); + + return (check_file(path, force, isspare)); +} + +boolean_t +check_sector_size_database(char *path, int *sector_size) +{ + return (0); +} + +void +zpool_vdev_enable_file(struct stat64 *statbuf, boolean_t *wholedisk) +{ + if (S_ISCHR(statbuf->st_mode)) { + statbuf->st_mode &= ~S_IFCHR; + statbuf->st_mode |= S_IFBLK; + *wholedisk = B_FALSE; + } +} diff --git a/cmd/zpool/zpool_vdev.c b/cmd/zpool/zpool_vdev.c index bb49211dc8e..a11fdd33c56 100644 --- a/cmd/zpool/zpool_vdev.c +++ b/cmd/zpool/zpool_vdev.c @@ -941,6 +941,10 @@ make_disks(zpool_handle_t *zhp, nvlist_t *nv) if (fd == -1) { if (errno == EBUSY) is_exclusive = 1; +#ifdef __FreeBSD__ + if (errno == EPERM) + is_exclusive = 1; +#endif } else { (void) close(fd); } diff --git a/config/Rules.am b/config/Rules.am index 83fbf4ca052..168cecea2cf 100644 --- a/config/Rules.am +++ b/config/Rules.am @@ -14,26 +14,48 @@ DEFAULT_INCLUDES += \ -I$(top_srcdir)/lib/libspl/include/os/linux endif +if BUILD_FREEBSD +DEFAULT_INCLUDES += \ + -I$(top_srcdir)/lib/libspl/include/os/freebsd +endif + AM_LIBTOOLFLAGS = --silent AM_CFLAGS = -std=gnu99 -Wall -Wstrict-prototypes -fno-strict-aliasing AM_CFLAGS += $(NO_OMIT_FRAME_POINTER) AM_CFLAGS += $(DEBUG_CFLAGS) AM_CFLAGS += $(ASAN_CFLAGS) -AM_CFLAGS += $(CODE_COVERAGE_CFLAGS) +AM_CFLAGS += $(CODE_COVERAGE_CFLAGS) $(NO_FORMAT_ZERO_LENGTH) +if BUILD_FREEBSD +AM_CFLAGS += -fPIC -Werror -Wno-unknown-pragmas -Wno-enum-conversion +AM_CFLAGS += -include $(top_srcdir)/include/os/freebsd/spl/sys/ccompile.h +AM_CFLAGS += -I/usr/include -I/usr/local/include +AM_CFLAGS += -D_MACHINE_ENDIAN_H_ +endif AM_CPPFLAGS = -D_GNU_SOURCE AM_CPPFLAGS += -D_REENTRANT AM_CPPFLAGS += -D_FILE_OFFSET_BITS=64 AM_CPPFLAGS += -D_LARGEFILE64_SOURCE AM_CPPFLAGS += -DHAVE_LARGE_STACKS=1 -AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\" AM_CPPFLAGS += -DLIBEXECDIR=\"$(libexecdir)\" AM_CPPFLAGS += -DRUNSTATEDIR=\"$(runstatedir)\" AM_CPPFLAGS += -DSBINDIR=\"$(sbindir)\" AM_CPPFLAGS += -DSYSCONFDIR=\"$(sysconfdir)\" AM_CPPFLAGS += $(DEBUG_CPPFLAGS) AM_CPPFLAGS += $(CODE_COVERAGE_CPPFLAGS) +if BUILD_LINUX +AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\" +endif +if BUILD_FREEBSD +AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-freebsd-user\" +endif AM_LDFLAGS = $(DEBUG_LDFLAGS) AM_LDFLAGS += $(ASAN_LDFLAGS) + +if BUILD_FREEBSD +AM_LDFLAGS += -fstack-protector-strong -shared +AM_LDFLAGS += -Wl,-x -Wl,--fatal-warnings -Wl,--warn-shared-textrel +AM_LDFLAGS += -lm +endif diff --git a/config/always-arch.m4 b/config/always-arch.m4 index eb8839b97d7..25e8c963a4b 100644 --- a/config/always-arch.m4 +++ b/config/always-arch.m4 @@ -17,7 +17,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_ARCH], [ i?86) TARGET_CPU=i386 ;; - x86_64) + amd64|x86_64) TARGET_CPU=x86_64 ;; powerpc*) diff --git a/config/always-compiler-options.m4 b/config/always-compiler-options.m4 index ca8b6bfccd1..a8412331798 100644 --- a/config/always-compiler-options.m4 +++ b/config/always-compiler-options.m4 @@ -87,6 +87,27 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION], [ AC_SUBST([NO_FORMAT_TRUNCATION]) ]) +dnl # +dnl # Check if gcc supports -Wno-format-truncation option. +dnl # +AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_ZERO_LENGTH], [ + AC_MSG_CHECKING([whether $CC supports -Wno-format-zero-length]) + + saved_flags="$CFLAGS" + CFLAGS="$CFLAGS -Werror -Wno-format-zero-length" + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [ + NO_FORMAT_ZERO_LENGTH=-Wno-format-zero-length + AC_MSG_RESULT([yes]) + ], [ + NO_FORMAT_ZERO_LENGTH= + AC_MSG_RESULT([no]) + ]) + + CFLAGS="$saved_flags" + AC_SUBST([NO_FORMAT_ZERO_LENGTH]) +]) + dnl # dnl # Check if gcc supports -Wno-bool-compare option. diff --git a/config/kernel.m4 b/config/kernel.m4 index c29de349418..8cbf4aee989 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -2,29 +2,31 @@ dnl # dnl # Default ZFS kernel configuration dnl # AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [ - dnl # Setup the kernel build environment. - ZFS_AC_KERNEL - ZFS_AC_QAT - - dnl # Sanity checks for module building and CONFIG_* defines - ZFS_AC_KERNEL_TEST_MODULE - ZFS_AC_KERNEL_CONFIG_DEFINED - - dnl # Sequential ZFS_LINUX_TRY_COMPILE tests - ZFS_AC_KERNEL_FPU_HEADER - ZFS_AC_KERNEL_WAIT_QUEUE_ENTRY_T - ZFS_AC_KERNEL_MISC_MINOR - ZFS_AC_KERNEL_DECLARE_EVENT_CLASS - - dnl # Parallel ZFS_LINUX_TEST_SRC / ZFS_LINUX_TEST_RESULT tests - ZFS_AC_KERNEL_TEST_SRC - ZFS_AC_KERNEL_TEST_RESULT - - AS_IF([test "$LINUX_OBJ" != "$LINUX"], [ - KERNEL_MAKE="$KERNEL_MAKE O=$LINUX_OBJ" - ]) + AM_COND_IF([BUILD_LINUX], [ + dnl # Setup the kernel build environment. + ZFS_AC_KERNEL + ZFS_AC_QAT + + dnl # Sanity checks for module building and CONFIG_* defines + ZFS_AC_KERNEL_TEST_MODULE + ZFS_AC_KERNEL_CONFIG_DEFINED + + dnl # Sequential ZFS_LINUX_TRY_COMPILE tests + ZFS_AC_KERNEL_FPU_HEADER + ZFS_AC_KERNEL_WAIT_QUEUE_ENTRY_T + ZFS_AC_KERNEL_MISC_MINOR + ZFS_AC_KERNEL_DECLARE_EVENT_CLASS + + dnl # Parallel ZFS_LINUX_TEST_SRC / ZFS_LINUX_TEST_RESULT tests + ZFS_AC_KERNEL_TEST_SRC + ZFS_AC_KERNEL_TEST_RESULT + + AS_IF([test "$LINUX_OBJ" != "$LINUX"], [ + KERNEL_MAKE="$KERNEL_MAKE O=$LINUX_OBJ" + ]) - AC_SUBST(KERNEL_MAKE) + AC_SUBST(KERNEL_MAKE) + ]) ]) dnl # diff --git a/config/toolchain-simd.m4 b/config/toolchain-simd.m4 index e86eb7f17a0..1153cd6941a 100644 --- a/config/toolchain-simd.m4 +++ b/config/toolchain-simd.m4 @@ -3,7 +3,7 @@ dnl # Checks if host toolchain supports SIMD instructions dnl # AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [ case "$host_cpu" in - x86_64 | x86 | i686) + amd64 | x86_64 | x86 | i686) ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE2 ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_SSE3 diff --git a/config/user.m4 b/config/user.m4 index 3d97e9a418c..b69412fda1e 100644 --- a/config/user.m4 +++ b/config/user.m4 @@ -4,14 +4,16 @@ dnl # AC_DEFUN([ZFS_AC_CONFIG_USER], [ ZFS_AC_CONFIG_USER_GETTEXT ZFS_AC_CONFIG_USER_MOUNT_HELPER - ZFS_AC_CONFIG_USER_UDEV - ZFS_AC_CONFIG_USER_SYSTEMD ZFS_AC_CONFIG_USER_SYSVINIT ZFS_AC_CONFIG_USER_DRACUT ZFS_AC_CONFIG_USER_ZLIB - ZFS_AC_CONFIG_USER_LIBUUID + AM_COND_IF([BUILD_LINUX], [ + ZFS_AC_CONFIG_USER_UDEV + ZFS_AC_CONFIG_USER_SYSTEMD + ZFS_AC_CONFIG_USER_LIBUUID + ZFS_AC_CONFIG_USER_LIBBLKID + ]) ZFS_AC_CONFIG_USER_LIBTIRPC - ZFS_AC_CONFIG_USER_LIBBLKID ZFS_AC_CONFIG_USER_LIBUDEV ZFS_AC_CONFIG_USER_LIBSSL ZFS_AC_CONFIG_USER_LIBAIO @@ -19,10 +21,9 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [ ZFS_AC_CONFIG_USER_MAKEDEV_IN_SYSMACROS ZFS_AC_CONFIG_USER_MAKEDEV_IN_MKDEV ZFS_AC_CONFIG_USER_ZFSEXEC - ZFS_AC_TEST_FRAMEWORK - AC_CHECK_FUNCS([mlockall strlcat strlcpy]) + AC_CHECK_FUNCS([issetugid mlockall strlcat strlcpy]) ]) dnl # diff --git a/config/zfs-build.m4 b/config/zfs-build.m4 index 2ee9b8eb942..016c0fc0953 100644 --- a/config/zfs-build.m4 +++ b/config/zfs-build.m4 @@ -157,6 +157,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [ ZFS_AC_CONFIG_ALWAYS_CC_NO_BOOL_COMPARE ZFS_AC_CONFIG_ALWAYS_CC_FRAME_LARGER_THAN ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION + ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_ZERO_LENGTH ZFS_AC_CONFIG_ALWAYS_CC_NO_OMIT_FRAME_POINTER ZFS_AC_CONFIG_ALWAYS_CC_NO_IPA_SRA ZFS_AC_CONFIG_ALWAYS_CC_ASAN @@ -173,13 +174,6 @@ AC_DEFUN([ZFS_AC_CONFIG], [ dnl # Remove the previous build test directory. rm -Rf build - AC_ARG_VAR([TEST_JOBS], - [simultaneous jobs during configure (defaults to $(nproc))]) - if test "x$ac_cv_env_TEST_JOBS_set" != "xset"; then - TEST_JOBS=$(nproc) - fi - AC_SUBST(TEST_JOBS) - ZFS_CONFIG=all AC_ARG_WITH([config], AS_HELP_STRING([--with-config=CONFIG], @@ -197,6 +191,16 @@ AC_DEFUN([ZFS_AC_CONFIG], [ ZFS_AC_CONFIG_ALWAYS + + AM_COND_IF([BUILD_LINUX], [ + AC_ARG_VAR([TEST_JOBS], + [simultaneous jobs during configure (defaults to $(nproc))]) + if test "x$ac_cv_env_TEST_JOBS_set" != "xset"; then + TEST_JOBS=$(nproc) + fi + AC_SUBST(TEST_JOBS) + ]) + case "$ZFS_CONFIG" in kernel) ZFS_AC_CONFIG_KERNEL ;; user) ZFS_AC_CONFIG_USER ;; @@ -405,7 +409,7 @@ dnl # Using the VENDOR tag from config.guess set the default dnl # package type for 'make pkg': (rpm | deb | tgz) dnl # AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ - AC_MSG_CHECKING([linux distribution]) + AC_MSG_CHECKING([os distribution]) if test -f /etc/toss-release ; then VENDOR=toss ; elif test -f /etc/fedora-release ; then @@ -428,6 +432,8 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ VENDOR=debian ; elif test -f /etc/alpine-release ; then VENDOR=alpine ; + elif test -f /bin/freebsd-version ; then + VENDOR=freebsd ; else VENDOR= ; fi @@ -447,13 +453,17 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ lunar) DEFAULT_PACKAGE=tgz ;; ubuntu) DEFAULT_PACKAGE=deb ;; debian) DEFAULT_PACKAGE=deb ;; + freebsd) DEFAULT_PACKAGE=pkg ;; *) DEFAULT_PACKAGE=rpm ;; esac AC_MSG_RESULT([$DEFAULT_PACKAGE]) AC_SUBST(DEFAULT_PACKAGE) - DEFAULT_INIT_DIR=$sysconfdir/init.d AC_MSG_CHECKING([default init directory]) + case "$VENDOR" in + freebsd) DEFAULT_INIT_DIR=$sysconfdir/rc.d ;; + *) DEFAULT_INIT_DIR=$sysconfdir/init.d;; + esac AC_MSG_RESULT([$DEFAULT_INIT_DIR]) AC_SUBST(DEFAULT_INIT_DIR) @@ -470,6 +480,7 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ lunar) DEFAULT_INIT_SCRIPT=lunar ;; ubuntu) DEFAULT_INIT_SCRIPT=lsb ;; debian) DEFAULT_INIT_SCRIPT=lsb ;; + freebsd) DEFAULT_INIT_SCRIPT=freebsd;; *) DEFAULT_INIT_SCRIPT=lsb ;; esac AC_MSG_RESULT([$DEFAULT_INIT_SCRIPT]) @@ -485,6 +496,7 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ sles) DEFAULT_INITCONF_DIR=/etc/sysconfig ;; ubuntu) DEFAULT_INITCONF_DIR=/etc/default ;; debian) DEFAULT_INITCONF_DIR=/etc/default ;; + freebsd) DEFAULT_INITCONF_DIR=$sysconfdir/rc.conf.d;; *) DEFAULT_INITCONF_DIR=/etc/default ;; esac AC_MSG_RESULT([$DEFAULT_INITCONF_DIR]) @@ -506,7 +518,9 @@ dnl # Default ZFS package configuration dnl # AC_DEFUN([ZFS_AC_PACKAGE], [ ZFS_AC_DEFAULT_PACKAGE - ZFS_AC_RPM - ZFS_AC_DPKG - ZFS_AC_ALIEN + AS_IF([test x$VENDOR != xfreebsd], [ + ZFS_AC_RPM + ZFS_AC_DPKG + ZFS_AC_ALIEN + ]) ]) diff --git a/configure.ac b/configure.ac index 7522940d2c3..902108f3649 100644 --- a/configure.ac +++ b/configure.ac @@ -109,6 +109,14 @@ AC_CONFIG_FILES([ etc/zfs/Makefile include/Makefile include/os/Makefile + include/os/freebsd/Makefile + include/os/freebsd/linux/Makefile + include/os/freebsd/spl/Makefile + include/os/freebsd/spl/acl/Makefile + include/os/freebsd/spl/rpc/Makefile + include/os/freebsd/spl/sys/Makefile + include/os/freebsd/zfs/Makefile + include/os/freebsd/zfs/sys/Makefile include/os/linux/Makefile include/os/linux/kernel/Makefile include/os/linux/kernel/linux/Makefile @@ -138,6 +146,8 @@ AC_CONFIG_FILES([ lib/libspl/include/ia32/Makefile lib/libspl/include/ia32/sys/Makefile lib/libspl/include/os/Makefile + lib/libspl/include/os/freebsd/Makefile + lib/libspl/include/os/freebsd/sys/Makefile lib/libspl/include/os/linux/Makefile lib/libspl/include/os/linux/sys/Makefile lib/libspl/include/rpc/Makefile diff --git a/contrib/Makefile.am b/contrib/Makefile.am index 9f34fd8354d..1486b28d3cd 100644 --- a/contrib/Makefile.am +++ b/contrib/Makefile.am @@ -1,2 +1,5 @@ -SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp +SUBDIRS = bash_completion.d pyzfs zcp +if BUILD_LINUX +SUBDIRS += bpftrace dracut initramfs +endif DIST_SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp diff --git a/etc/Makefile.am b/etc/Makefile.am index 67ef94a2017..ac71da9445d 100644 --- a/etc/Makefile.am +++ b/etc/Makefile.am @@ -1,2 +1,5 @@ -SUBDIRS = default zfs sudoers.d $(ZFS_INIT_SYSTEMD) $(ZFS_INIT_SYSV) $(ZFS_MODULE_LOAD) +SUBDIRS = zfs sudoers.d +if BUILD_LINUX +SUBDIRS += default $(ZFS_INIT_SYSTEMD) $(ZFS_INIT_SYSV) $(ZFS_MODULE_LOAD) +endif DIST_SUBDIRS = default init.d zfs systemd modules-load.d sudoers.d diff --git a/include/os/Makefile.am b/include/os/Makefile.am index 09c0beec475..7eab1abde98 100644 --- a/include/os/Makefile.am +++ b/include/os/Makefile.am @@ -1,3 +1,6 @@ if BUILD_LINUX SUBDIRS = linux endif +if BUILD_FREEBSD +SUBDIRS = freebsd +endif diff --git a/include/os/freebsd/Makefile.am b/include/os/freebsd/Makefile.am new file mode 100644 index 00000000000..3c87d4a0e79 --- /dev/null +++ b/include/os/freebsd/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = linux spl zfs diff --git a/include/os/freebsd/linux/Makefile.am b/include/os/freebsd/linux/Makefile.am new file mode 100644 index 00000000000..936cf21319b --- /dev/null +++ b/include/os/freebsd/linux/Makefile.am @@ -0,0 +1,5 @@ +KERNEL_H = \ + $(top_srcdir)/include/os/freebsd/linux/compiler.h \ + $(top_srcdir)/include/os/freebsd/linux/types.h + +EXTRA_DIST = $(KERNEL_H) diff --git a/include/os/freebsd/linux/compiler.h b/include/os/freebsd/linux/compiler.h new file mode 100644 index 00000000000..d76050378e8 --- /dev/null +++ b/include/os/freebsd/linux/compiler.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iXsystems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_COMPILER_H_ +#define _LINUX_COMPILER_H_ + +#include + +#define __user +#define __kernel +#define __safe +#define __force +#define __nocast +#define __iomem +#define __chk_user_ptr(x) ((void)0) +#define __chk_io_ptr(x) ((void)0) +#define __builtin_warning(x, y...) (1) +#define __acquires(x) +#define __releases(x) +#define __acquire(x) do { } while (0) +#define __release(x) do { } while (0) +#define __cond_lock(x, c) (c) +#define __bitwise +#define __devinitdata +#define __deprecated +#define __init +#define __initconst +#define __devinit +#define __devexit +#define __exit +#define __rcu +#define __percpu +#define __weak __weak_symbol +#define __malloc +#define ___stringify(...) #__VA_ARGS__ +#define __stringify(...) ___stringify(__VA_ARGS__) +#define __attribute_const__ __attribute__((__const__)) +#undef __always_inline +#define __always_inline inline +#define noinline __noinline +#define ____cacheline_aligned __aligned(CACHE_LINE_SIZE) + +#ifndef _KERNEL +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#endif +#define typeof(x) __typeof(x) + +#define uninitialized_var(x) x = x +#define __maybe_unused __unused +#define __always_unused __unused +#define __must_check __result_use_check + +#define __printf(a, b) __printflike(a, b) + +#define barrier() __asm__ __volatile__("": : :"memory") +#define smp_rmb() rmb() +#define ___PASTE(a, b) a##b +#define __PASTE(a, b) ___PASTE(a, b) + +#define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x)) + +#define WRITE_ONCE(x, v) do { \ + barrier(); \ + ACCESS_ONCE(x) = (v); \ + barrier(); \ +} while (0) + +#define lockless_dereference(p) READ_ONCE(p) + +#define _AT(T, X) ((T)(X)) + +#endif /* _LINUX_COMPILER_H_ */ diff --git a/include/os/freebsd/linux/types.h b/include/os/freebsd/linux/types.h new file mode 100644 index 00000000000..301163c034f --- /dev/null +++ b/include/os/freebsd/linux/types.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iXsystems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_TYPES_H_ +#define _LINUX_TYPES_H_ + +#include +#include +#include +#include + + +#ifndef __bitwise__ +#ifdef __CHECKER__ +#define __bitwise__ __attribute__((bitwise)) +#else +#define __bitwise__ +#endif +#endif + +typedef uint16_t __le16; +typedef uint16_t __be16; +typedef uint32_t __le32; +typedef uint32_t __be32; +typedef uint64_t __le64; +typedef uint64_t __be64; + +typedef unsigned gfp_t; +typedef uint64_t loff_t; +typedef vm_paddr_t resource_size_t; +typedef uint16_t __bitwise__ __sum16; +typedef unsigned long pgoff_t; +typedef unsigned __poll_t; + +typedef uint64_t u64; +typedef u64 phys_addr_t; + +typedef size_t __kernel_size_t; + +#define DECLARE_BITMAP(n, bits) \ + unsigned long n[howmany(bits, sizeof (long) * 8)] + +typedef unsigned long irq_hw_number_t; + +struct rcu_head { + void *raw[2]; +} __aligned(sizeof (void *)); + +typedef void (*rcu_callback_t)(struct rcu_head *head); +typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); +typedef int linux_task_fn_t(void *data); + +#endif /* _LINUX_TYPES_H_ */ diff --git a/include/os/freebsd/spl/Makefile.am b/include/os/freebsd/spl/Makefile.am new file mode 100644 index 00000000000..b321825cb77 --- /dev/null +++ b/include/os/freebsd/spl/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = acl rpc sys diff --git a/include/os/freebsd/spl/acl/Makefile.am b/include/os/freebsd/spl/acl/Makefile.am new file mode 100644 index 00000000000..65a03ea1789 --- /dev/null +++ b/include/os/freebsd/spl/acl/Makefile.am @@ -0,0 +1,4 @@ +KERNEL_H = \ + $(top_srcdir)/include/os/freebsd/spl/acl/acl_common.h + +EXTRA_DIST = $(KERNEL_H) diff --git a/include/os/freebsd/spl/acl/acl_common.h b/include/os/freebsd/spl/acl/acl_common.h new file mode 100644 index 00000000000..00a2a9dfe73 --- /dev/null +++ b/include/os/freebsd/spl/acl/acl_common.h @@ -0,0 +1,68 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright 2011 Nexenta Systems, Inc. All rights reserved. + */ + +#ifndef _ACL_COMMON_H +#define _ACL_COMMON_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct trivial_acl { + uint32_t allow0; /* allow mask for bits only in owner */ + uint32_t deny1; /* deny mask for bits not in owner */ + uint32_t deny2; /* deny mask for bits not in group */ + uint32_t owner; /* allow mask matching mode */ + uint32_t group; /* allow mask matching mode */ + uint32_t everyone; /* allow mask matching mode */ +} trivial_acl_t; + +extern int acltrivial(const char *); +extern void adjust_ace_pair(ace_t *pair, mode_t mode); +extern void adjust_ace_pair_common(void *, size_t, size_t, mode_t); +extern int ace_trivial(ace_t *acep, int aclcnt); +extern int ace_trivial_common(void *, int, + uint64_t (*walk)(void *, uint64_t, int aclcnt, uint16_t *, uint16_t *, + uint32_t *mask)); +#if !defined(_KERNEL) +extern acl_t *acl_alloc(acl_type_t); +extern void acl_free(acl_t *aclp); +extern int acl_translate(acl_t *aclp, int target_flavor, boolean_t isdir, + uid_t owner, gid_t group); +#endif /* !_KERNEL */ +int cmp2acls(void *a, void *b); +int acl_trivial_create(mode_t mode, boolean_t isdir, ace_t **acl, int *count); +void acl_trivial_access_masks(mode_t mode, boolean_t isdir, + trivial_acl_t *masks); + +#ifdef __cplusplus +} +#endif + +#endif /* _ACL_COMMON_H */ diff --git a/include/os/freebsd/spl/rpc/Makefile.am b/include/os/freebsd/spl/rpc/Makefile.am new file mode 100644 index 00000000000..266a3b759ca --- /dev/null +++ b/include/os/freebsd/spl/rpc/Makefile.am @@ -0,0 +1,8 @@ +COMMON_H = + +KERNEL_H = \ + $(top_srcdir)/include/os/freebsd/spl/rpc/xdr.h + +USER_H = + +EXTRA_DIST = $(COMMON_H) $(KERNEL_H) $(USER_H) diff --git a/include/os/freebsd/spl/rpc/xdr.h b/include/os/freebsd/spl/rpc/xdr.h new file mode 100644 index 00000000000..b4df2c1ea04 --- /dev/null +++ b/include/os/freebsd/spl/rpc/xdr.h @@ -0,0 +1,71 @@ +/* + * Sun RPC is a product of Sun Microsystems, Inc. and is provided for + * unrestricted use provided that this legend is included on all tape + * media and as a part of the software program in whole or part. Users + * may copy or modify Sun RPC without charge, but are not authorized + * to license or distribute it to anyone else except as part of a product or + * program developed by the user. + * + * SUN RPC IS PROVIDED AS IS WITH NO WARRANTIES OF ANY KIND INCLUDING THE + * WARRANTIES OF DESIGN, MERCHANTIBILITY AND FITNESS FOR A PARTICULAR + * PURPOSE, OR ARISING FROM A COURSE OF DEALING, USAGE OR TRADE PRACTICE. + * + * Sun RPC is provided with no support and without any obligation on the + * part of Sun Microsystems, Inc. to assist in its use, correction, + * modification or enhancement. + * + * SUN MICROSYSTEMS, INC. SHALL HAVE NO LIABILITY WITH RESPECT TO THE + * INFRINGEMENT OF COPYRIGHTS, TRADE SECRETS OR ANY PATENTS BY SUN RPC + * OR ANY PART THEREOF. + * + * In no event will Sun Microsystems, Inc. be liable for any lost revenue + * or profits or other special, indirect and consequential damages, even if + * Sun has been advised of the possibility of such damages. + * + * Sun Microsystems, Inc. + * 2550 Garcia Avenue + * Mountain View, California 94043 + */ + +#ifndef _OPENSOLARIS_RPC_XDR_H_ +#define _OPENSOLARIS_RPC_XDR_H_ + +#include +#include_next + +#ifndef _KERNEL + +#include + +/* + * Taken from sys/xdr/xdr_mem.c. + * + * FreeBSD's userland XDR doesn't implement control method (only the kernel), + * but OpenSolaris nvpair still depend on it, so we have to implement it here. + */ +static __inline bool_t +xdrmem_control(XDR *xdrs, int request, void *info) +{ + xdr_bytesrec *xptr; + + switch (request) { + case XDR_GET_BYTES_AVAIL: + xptr = (xdr_bytesrec *)info; + xptr->xc_is_last_record = TRUE; + xptr->xc_num_avail = xdrs->x_handy; + return (TRUE); + default: + assert(!"unexpected request"); + } + return (FALSE); +} + +#undef XDR_CONTROL +#define XDR_CONTROL(xdrs, req, op) \ + (((xdrs)->x_ops->x_control == NULL) ? \ + xdrmem_control((xdrs), (req), (op)) : \ + (*(xdrs)->x_ops->x_control)(xdrs, req, op)) + +#endif /* !_KERNEL */ + +#endif /* !_OPENSOLARIS_RPC_XDR_H_ */ diff --git a/include/os/freebsd/spl/sys/Makefile.am b/include/os/freebsd/spl/sys/Makefile.am new file mode 100644 index 00000000000..29a39cacf4d --- /dev/null +++ b/include/os/freebsd/spl/sys/Makefile.am @@ -0,0 +1,72 @@ +KERNEL_H = \ + $(top_srcdir)/include/os/freebsd/spl/sys/acl_impl.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/acl.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/atomic.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/byteorder.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/callb.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/ccompile.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/cmn_err.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/condvar.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/console.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/cred.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/ctype.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/debug.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/dirent.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/disp.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/dkio.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/endian.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/extdirent.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/file.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/freebsd_rwlock.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/inttypes.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/isa_defs.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/kmem_cache.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/kmem.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/kstat.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/list_impl.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/list.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/lock.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/Makefile.am \ + $(top_srcdir)/include/os/freebsd/spl/sys/misc.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/mod_os.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/mode.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/mount.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/mutex.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/param.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/policy.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/proc.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/processor.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/procfs_list.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/random.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/rwlock.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/sdt.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/sid.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/sig.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/simd_x86.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/simd.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/spl_condvar.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/string.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/strings.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/sunddi.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/sysmacros.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/systeminfo.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/systm.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/taskq.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/thread.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/time.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/timer.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/trace_zfs.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/trace.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/types.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/types32.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/uio.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/uuid.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/vfs.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/vm.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/vmsystm.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/vnode_impl.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/vnode.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/zmod.h \ + $(top_srcdir)/include/os/freebsd/spl/sys/zone.h + +EXTRA_DIST = $(KERNEL_H) diff --git a/include/os/freebsd/spl/sys/acl.h b/include/os/freebsd/spl/sys/acl.h new file mode 100644 index 00000000000..ee50b0a1836 --- /dev/null +++ b/include/os/freebsd/spl/sys/acl.h @@ -0,0 +1,216 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2014 Garrett D'Amore + * + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + * Copyright 2017 RackTop Systems. + */ + +#ifndef _SYS_ACL_H +#define _SYS_ACL_H + +#include +#include + +/* + * When compiling OpenSolaris kernel code, this file is included instead of the + * FreeBSD one. Include the original sys/acl.h as well. + */ +#undef _SYS_ACL_H +#include_next +#define _SYS_ACL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_ACL_ENTRIES (1024) /* max entries of each type */ +typedef struct { + int a_type; /* the type of ACL entry */ + uid_t a_id; /* the entry in -uid or gid */ + o_mode_t a_perm; /* the permission field */ +} aclent_t; + +typedef struct ace { + uid_t a_who; /* uid or gid */ + uint32_t a_access_mask; /* read,write,... */ + uint16_t a_flags; /* see below */ + uint16_t a_type; /* allow or deny */ +} ace_t; + +/* + * The following are Defined types for an aclent_t. + */ +#define USER_OBJ (0x01) /* object owner */ +#define USER (0x02) /* additional users */ +#define GROUP_OBJ (0x04) /* owning group of the object */ +#define GROUP (0x08) /* additional groups */ +#define CLASS_OBJ (0x10) /* file group class and mask entry */ +#define OTHER_OBJ (0x20) /* other entry for the object */ +#define ACL_DEFAULT (0x1000) /* default flag */ +/* default object owner */ +#define DEF_USER_OBJ (ACL_DEFAULT | USER_OBJ) +/* default additional users */ +#define DEF_USER (ACL_DEFAULT | USER) +/* default owning group */ +#define DEF_GROUP_OBJ (ACL_DEFAULT | GROUP_OBJ) +/* default additional groups */ +#define DEF_GROUP (ACL_DEFAULT | GROUP) +/* default mask entry */ +#define DEF_CLASS_OBJ (ACL_DEFAULT | CLASS_OBJ) +/* default other entry */ +#define DEF_OTHER_OBJ (ACL_DEFAULT | OTHER_OBJ) + +/* + * The following are defined for ace_t. + */ +#define ACE_READ_DATA 0x00000001 +#define ACE_LIST_DIRECTORY 0x00000001 +#define ACE_WRITE_DATA 0x00000002 +#define ACE_ADD_FILE 0x00000002 +#define ACE_APPEND_DATA 0x00000004 +#define ACE_ADD_SUBDIRECTORY 0x00000004 +#define ACE_READ_NAMED_ATTRS 0x00000008 +#define ACE_WRITE_NAMED_ATTRS 0x00000010 +#define ACE_EXECUTE 0x00000020 +#define ACE_DELETE_CHILD 0x00000040 +#define ACE_READ_ATTRIBUTES 0x00000080 +#define ACE_WRITE_ATTRIBUTES 0x00000100 +#define ACE_DELETE 0x00010000 +#define ACE_READ_ACL 0x00020000 +#define ACE_WRITE_ACL 0x00040000 +#define ACE_WRITE_OWNER 0x00080000 +#define ACE_SYNCHRONIZE 0x00100000 + +#define ACE_FILE_INHERIT_ACE 0x0001 +#define ACE_DIRECTORY_INHERIT_ACE 0x0002 +#define ACE_NO_PROPAGATE_INHERIT_ACE 0x0004 +#define ACE_INHERIT_ONLY_ACE 0x0008 +#define ACE_SUCCESSFUL_ACCESS_ACE_FLAG 0x0010 +#define ACE_FAILED_ACCESS_ACE_FLAG 0x0020 +#define ACE_IDENTIFIER_GROUP 0x0040 +#define ACE_INHERITED_ACE 0x0080 +#define ACE_OWNER 0x1000 +#define ACE_GROUP 0x2000 +#define ACE_EVERYONE 0x4000 + +#define ACE_ACCESS_ALLOWED_ACE_TYPE 0x0000 +#define ACE_ACCESS_DENIED_ACE_TYPE 0x0001 +#define ACE_SYSTEM_AUDIT_ACE_TYPE 0x0002 +#define ACE_SYSTEM_ALARM_ACE_TYPE 0x0003 + +#define ACL_AUTO_INHERIT 0x0001 +#define ACL_PROTECTED 0x0002 +#define ACL_DEFAULTED 0x0004 +#define ACL_FLAGS_ALL (ACL_AUTO_INHERIT|ACL_PROTECTED| \ + ACL_DEFAULTED) + +/* + * These are only applicable in a CIFS context. + */ +#define ACE_ACCESS_ALLOWED_COMPOUND_ACE_TYPE 0x04 +#define ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE 0x05 +#define ACE_ACCESS_DENIED_OBJECT_ACE_TYPE 0x06 +#define ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE 0x07 +#define ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE 0x08 +#define ACE_ACCESS_ALLOWED_CALLBACK_ACE_TYPE 0x09 +#define ACE_ACCESS_DENIED_CALLBACK_ACE_TYPE 0x0A +#define ACE_ACCESS_ALLOWED_CALLBACK_OBJECT_ACE_TYPE 0x0B +#define ACE_ACCESS_DENIED_CALLBACK_OBJECT_ACE_TYPE 0x0C +#define ACE_SYSTEM_AUDIT_CALLBACK_ACE_TYPE 0x0D +#define ACE_SYSTEM_ALARM_CALLBACK_ACE_TYPE 0x0E +#define ACE_SYSTEM_AUDIT_CALLBACK_OBJECT_ACE_TYPE 0x0F +#define ACE_SYSTEM_ALARM_CALLBACK_OBJECT_ACE_TYPE 0x10 + +#define ACE_ALL_TYPES 0x001F + +typedef struct ace_object { + uid_t a_who; /* uid or gid */ + uint32_t a_access_mask; /* read,write,... */ + uint16_t a_flags; /* see below */ + uint16_t a_type; /* allow or deny */ + uint8_t a_obj_type[16]; /* obj type */ + uint8_t a_inherit_obj_type[16]; /* inherit obj */ +} ace_object_t; + +#define ACE_ALL_PERMS (ACE_READ_DATA|ACE_LIST_DIRECTORY|ACE_WRITE_DATA| \ + ACE_ADD_FILE|ACE_APPEND_DATA|ACE_ADD_SUBDIRECTORY|ACE_READ_NAMED_ATTRS| \ + ACE_WRITE_NAMED_ATTRS|ACE_EXECUTE|ACE_DELETE_CHILD|ACE_READ_ATTRIBUTES| \ + ACE_WRITE_ATTRIBUTES|ACE_DELETE|ACE_READ_ACL|ACE_WRITE_ACL| \ + ACE_WRITE_OWNER|ACE_SYNCHRONIZE) + +#define ACE_ALL_WRITE_PERMS (ACE_WRITE_DATA|ACE_APPEND_DATA| \ + ACE_WRITE_ATTRIBUTES|ACE_WRITE_NAMED_ATTRS|ACE_WRITE_ACL| \ + ACE_WRITE_OWNER|ACE_DELETE|ACE_DELETE_CHILD) + +#define ACE_READ_PERMS (ACE_READ_DATA|ACE_READ_ACL|ACE_READ_ATTRIBUTES| \ + ACE_READ_NAMED_ATTRS) + +#define ACE_WRITE_PERMS (ACE_WRITE_DATA|ACE_APPEND_DATA|ACE_WRITE_ATTRIBUTES| \ + ACE_WRITE_NAMED_ATTRS) + +#define ACE_MODIFY_PERMS (ACE_READ_DATA|ACE_LIST_DIRECTORY|ACE_WRITE_DATA| \ + ACE_ADD_FILE|ACE_APPEND_DATA|ACE_ADD_SUBDIRECTORY|ACE_READ_NAMED_ATTRS| \ + ACE_WRITE_NAMED_ATTRS|ACE_EXECUTE|ACE_DELETE_CHILD|ACE_READ_ATTRIBUTES| \ + ACE_WRITE_ATTRIBUTES|ACE_DELETE|ACE_READ_ACL|ACE_SYNCHRONIZE) +/* + * The following flags are supported by both NFSv4 ACLs and ace_t. + */ +#define ACE_NFSV4_SUP_FLAGS (ACE_FILE_INHERIT_ACE | \ + ACE_DIRECTORY_INHERIT_ACE | \ + ACE_NO_PROPAGATE_INHERIT_ACE | \ + ACE_INHERIT_ONLY_ACE | \ + ACE_INHERITED_ACE | \ + ACE_IDENTIFIER_GROUP) + +#define ACE_TYPE_FLAGS (ACE_OWNER|ACE_GROUP|ACE_EVERYONE| \ + ACE_IDENTIFIER_GROUP) +#define ACE_INHERIT_FLAGS (ACE_FILE_INHERIT_ACE| ACL_INHERITED_ACE| \ + ACE_DIRECTORY_INHERIT_ACE|ACE_NO_PROPAGATE_INHERIT_ACE|ACE_INHERIT_ONLY_ACE) + +/* cmd args to acl(2) for aclent_t */ +#define GETACL 1 +#define SETACL 2 +#define GETACLCNT 3 + +/* cmd's to manipulate ace acls. */ +#define ACE_GETACL 4 +#define ACE_SETACL 5 +#define ACE_GETACLCNT 6 + +/* minimal acl entries from GETACLCNT */ +#define MIN_ACL_ENTRIES 4 + +extern void aces_from_acl(ace_t *aces, int *nentries, const struct acl *aclp); +extern int acl_from_aces(struct acl *aclp, const ace_t *aces, int nentries); +extern void ksort(caddr_t, int, int, int (*)(void *, void *)); +extern int cmp2acls(void *, void *); + +extern int acl(const char *path, int cmd, int cnt, void *buf); +extern int facl(int fd, int cmd, int cnt, void *buf); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_ACL_H */ diff --git a/include/os/freebsd/spl/sys/acl_impl.h b/include/os/freebsd/spl/sys/acl_impl.h new file mode 100644 index 00000000000..8718f5bcf63 --- /dev/null +++ b/include/os/freebsd/spl/sys/acl_impl.h @@ -0,0 +1,61 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_ACL_IMPL_H +#define _SYS_ACL_IMPL_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * acl flags + * + * ACL_AUTO_INHERIT, ACL_PROTECTED and ACL_DEFAULTED + * flags can also be stored in this field. + */ +#define ACL_IS_TRIVIAL 0x10000 +#define ACL_IS_DIR 0x20000 + +typedef enum acl_type { + ACLENT_T = 0, + ACE_T = 1 +} zfs_acl_type_t; + +struct acl_info { + zfs_acl_type_t acl_type; /* style of acl */ + int acl_cnt; /* number of acl entries */ + int acl_entry_size; /* sizeof acl entry */ + int acl_flags; /* special flags about acl */ + void *acl_aclp; /* the acl */ +}; + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_ACL_IMPL_H */ diff --git a/include/os/freebsd/spl/sys/atomic.h b/include/os/freebsd/spl/sys/atomic.h new file mode 100644 index 00000000000..e283c6c0e3f --- /dev/null +++ b/include/os/freebsd/spl/sys/atomic.h @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_ATOMIC_H_ +#define _OPENSOLARIS_SYS_ATOMIC_H_ + +#include +#include + +#define casptr(_a, _b, _c) \ + atomic_cmpset_ptr((volatile uintptr_t *)(_a), \ + (uintptr_t)(_b), \ + (uintptr_t)(_c)) +#define cas32 atomic_cmpset_32 +#define atomic_sub_64 atomic_subtract_64 + +#if defined(__i386__) || defined(KLD_MODULE) +#define I386_HAVE_ATOMIC64 +#endif + +#if !defined(__LP64__) && !defined(__mips_n32) && \ + !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) +extern void atomic_add_64(volatile uint64_t *target, int64_t delta); +extern void atomic_dec_64(volatile uint64_t *target); +#endif +#ifndef __sparc64__ +#if defined(__LP64__) || defined(__mips_n32) || \ + defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64) + +#define membar_producer() wmb() + +static __inline uint64_t +atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) +{ + +#ifdef __i386__ + atomic_fcmpset_64(target, &cmp, newval); +#else + atomic_fcmpset_long(target, &cmp, newval); +#endif + return (cmp); +} + +static __inline uint32_t +atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) +{ + + atomic_fcmpset_int(target, &cmp, newval); + return (cmp); +} + +static __inline uint64_t +atomic_add_64_nv(volatile uint64_t *target, int64_t delta) +{ + uint64_t prev; + + prev = atomic_fetchadd_long(target, delta); + + return (prev + delta); +} + +#else +extern uint32_t atomic_cas_32(volatile uint32_t *target, uint32_t cmp, + uint32_t newval); +extern uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, + uint64_t newval); +extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta); +extern void membar_producer(void); +#endif +#endif +extern uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value); + +#if defined(__sparc64__) || defined(__powerpc__) || defined(__arm__) || \ + defined(__mips__) || defined(__aarch64__) || defined(__riscv) +extern void atomic_or_8(volatile uint8_t *target, uint8_t value); +#else +static __inline void +atomic_or_8(volatile uint8_t *target, uint8_t value) +{ + atomic_set_8(target, value); +} +#endif + +static __inline uint32_t +atomic_add_32_nv(volatile uint32_t *target, int32_t delta) +{ + return (atomic_fetchadd_32(target, delta) + delta); +} + +static __inline uint32_t +atomic_add_int_nv(volatile uint32_t *target, int delta) +{ + return (atomic_add_32_nv(target, delta)); +} + +static __inline void +atomic_dec_32(volatile uint32_t *target) +{ + atomic_subtract_32(target, 1); +} + +static __inline uint32_t +atomic_dec_32_nv(volatile uint32_t *target) +{ + return (atomic_fetchadd_32(target, -1) - 1); +} + +#if defined(__LP64__) || defined(__mips_n32) || \ + defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64) +static __inline void +atomic_dec_64(volatile uint64_t *target) +{ + atomic_subtract_64(target, 1); +} +#endif + +static __inline void +atomic_inc_32(volatile uint32_t *target) +{ + atomic_add_32(target, 1); +} + +static __inline uint32_t +atomic_inc_32_nv(volatile uint32_t *target) +{ + return (atomic_add_32_nv(target, 1)); +} + +static __inline void +atomic_inc_64(volatile uint64_t *target) +{ + atomic_add_64(target, 1); +} + +static __inline uint64_t +atomic_inc_64_nv(volatile uint64_t *target) +{ + return (atomic_add_64_nv(target, 1)); +} + +static __inline uint64_t +atomic_dec_64_nv(volatile uint64_t *target) +{ + return (atomic_add_64_nv(target, -1)); +} + +#if !defined(COMPAT_32BIT) && defined(__LP64__) +static __inline void * +atomic_cas_ptr(volatile void *target, void *cmp, void *newval) +{ + return ((void *)atomic_cas_64((volatile uint64_t *)target, + (uint64_t)cmp, (uint64_t)newval)); +} +#else +static __inline void * +atomic_cas_ptr(volatile void *target, void *cmp, void *newval) +{ + return ((void *)atomic_cas_32((volatile uint32_t *)target, + (uint32_t)cmp, (uint32_t)newval)); +} +#endif /* !defined(COMPAT_32BIT) && defined(__LP64__) */ + +#endif /* !_OPENSOLARIS_SYS_ATOMIC_H_ */ diff --git a/include/os/freebsd/spl/sys/byteorder.h b/include/os/freebsd/spl/sys/byteorder.h new file mode 100644 index 00000000000..79ae848c725 --- /dev/null +++ b/include/os/freebsd/spl/sys/byteorder.h @@ -0,0 +1,93 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ + +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +/* + * University Copyright- Copyright (c) 1982, 1986, 1988 + * The Regents of the University of California + * All Rights Reserved + * + * University Acknowledgment- Portions of this document are derived from + * software developed by the University of California, Berkeley, and its + * contributors. + */ + +#ifndef _OPENSOLARIS_SYS_BYTEORDER_H_ +#define _OPENSOLARIS_SYS_BYTEORDER_H_ + +#include + +/* + * Macros to reverse byte order + */ +#define BSWAP_8(x) ((x) & 0xff) +#define BSWAP_16(x) ((BSWAP_8(x) << 8) | BSWAP_8((x) >> 8)) +#define BSWAP_32(x) ((BSWAP_16(x) << 16) | BSWAP_16((x) >> 16)) +#define BSWAP_64(x) ((BSWAP_32(x) << 32) | BSWAP_32((x) >> 32)) + +#define BMASK_8(x) ((x) & 0xff) +#define BMASK_16(x) ((x) & 0xffff) +#define BMASK_32(x) ((x) & 0xffffffff) +#define BMASK_64(x) (x) + +/* + * Macros to convert from a specific byte order to/from native byte order + */ +#if BYTE_ORDER == BIG_ENDIAN +#define BE_8(x) BMASK_8(x) +#define BE_16(x) BMASK_16(x) +#define BE_32(x) BMASK_32(x) +#define BE_64(x) BMASK_64(x) +#define LE_8(x) BSWAP_8(x) +#define LE_16(x) BSWAP_16(x) +#define LE_32(x) BSWAP_32(x) +#define LE_64(x) BSWAP_64(x) +#else +#define LE_8(x) BMASK_8(x) +#define LE_16(x) BMASK_16(x) +#define LE_32(x) BMASK_32(x) +#define LE_64(x) BMASK_64(x) +#define BE_8(x) BSWAP_8(x) +#define BE_16(x) BSWAP_16(x) +#define BE_32(x) BSWAP_32(x) +#define BE_64(x) BSWAP_64(x) +#endif + +#if BYTE_ORDER == BIG_ENDIAN +#define htonll(x) BMASK_64(x) +#define ntohll(x) BMASK_64(x) +#else +#define htonll(x) BSWAP_64(x) +#define ntohll(x) BSWAP_64(x) +#endif + +#define BE_IN32(xa) htonl(*((uint32_t *)(void *)(xa))) + +#endif /* _OPENSOLARIS_SYS_BYTEORDER_H_ */ diff --git a/include/os/freebsd/spl/sys/callb.h b/include/os/freebsd/spl/sys/callb.h new file mode 100644 index 00000000000..ed9ed8cd88a --- /dev/null +++ b/include/os/freebsd/spl/sys/callb.h @@ -0,0 +1,213 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CALLB_H +#define _SYS_CALLB_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * definitions of callback classes (c_class) + * + * Callbacks belong in the same class if (1) their callback routines + * do the same kind of processing (ideally, using the same callback function) + * and (2) they can/should be executed at the same time in a cpr + * suspend/resume operation. + * + * Note: The DAEMON class, in particular, is for stopping kernel threads + * and nothing else. The CALLB_* macros below should be used to deal + * with kernel threads, and the callback function should be callb_generic_cpr. + * Another idiosyncrasy of the DAEMON class is that if a suspend operation + * fails, some of the callback functions may be called with the RESUME + * code which were never called with SUSPEND. Not a problem currently, + * but see bug 4201851. + */ +#define CB_CL_CPR_DAEMON 0 +#define CB_CL_CPR_VM 1 +#define CB_CL_CPR_CALLOUT 2 +#define CB_CL_CPR_OBP 3 +#define CB_CL_CPR_FB 4 +#define CB_CL_PANIC 5 +#define CB_CL_CPR_RPC 6 +#define CB_CL_CPR_PROMPRINTF 7 +#define CB_CL_UADMIN 8 +#define CB_CL_CPR_PM 9 +#define CB_CL_HALT 10 +#define CB_CL_CPR_DMA 11 +#define CB_CL_CPR_POST_USER 12 +#define CB_CL_UADMIN_PRE_VFS 13 +#define CB_CL_MDBOOT CB_CL_UADMIN +#define CB_CL_ENTER_DEBUGGER 14 +#define CB_CL_CPR_POST_KERNEL 15 +#define CB_CL_CPU_DEEP_IDLE 16 +#define NCBCLASS 17 /* CHANGE ME if classes are added/removed */ + +/* + * CB_CL_CPR_DAEMON class specific definitions are given below: + */ + +/* + * code for CPR callb_execute_class + */ +#define CB_CODE_CPR_CHKPT 0 +#define CB_CODE_CPR_RESUME 1 + +typedef void * callb_id_t; +/* + * Per kernel thread structure for CPR daemon callbacks. + * Must be protected by either a existing lock in the daemon or + * a new lock created for such a purpose. + */ +typedef struct callb_cpr { + kmutex_t *cc_lockp; /* lock to protect this struct */ + char cc_events; /* various events for CPR */ + callb_id_t cc_id; /* callb id address */ + kcondvar_t cc_callb_cv; /* cv for callback waiting */ + kcondvar_t cc_stop_cv; /* cv to checkpoint block */ +} callb_cpr_t; + +/* + * cc_events definitions + */ +#define CALLB_CPR_START 1 /* a checkpoint request's started */ +#define CALLB_CPR_SAFE 2 /* thread is safe for CPR */ +#define CALLB_CPR_ALWAYS_SAFE 4 /* thread is ALWAYS safe for CPR */ + +/* + * Used when checking that all kernel threads are stopped. + */ +#define CALLB_MAX_RETRY 3 /* when waiting for kthread to sleep */ +#define CALLB_THREAD_DELAY 10 /* ticks allowed to reach sleep */ +#define CPR_KTHREAD_TIMEOUT_SEC 90 /* secs before callback times out -- */ + /* due to pwr mgmt of disks, make -- */ + /* big enough for worst spinup time */ + +/* + * + * CALLB_CPR_INIT macro is used by kernel threads to add their entry to + * the callback table and perform other initialization. It automatically + * adds the thread as being in the callback class CB_CL_CPR_DAEMON. + * + * cp - ptr to the callb_cpr_t structure for this kernel thread + * + * lockp - pointer to mutex protecting the callb_cpr_t stuct + * + * func - pointer to the callback function for this kernel thread. + * It has the prototype boolean_t (void *arg, int code) + * where: arg - ptr to the callb_cpr_t structure + * code - not used for this type of callback + * returns: B_TRUE if successful; B_FALSE if unsuccessful. + * + * name - a string giving the name of the kernel thread + * + * Note: lockp is the lock to protect the callb_cpr_t (cp) structure + * later on. No lock held is needed for this initialization. + */ +#define CALLB_CPR_INIT(cp, lockp, func, name) { \ + strlcpy(curthread->td_name, (name), \ + sizeof (curthread->td_name)); \ + bzero((caddr_t)(cp), sizeof (callb_cpr_t)); \ + (cp)->cc_lockp = lockp; \ + (cp)->cc_id = callb_add(func, (void *)(cp), \ + CB_CL_CPR_DAEMON, name); \ + cv_init(&(cp)->cc_callb_cv, NULL, CV_DEFAULT, NULL); \ + cv_init(&(cp)->cc_stop_cv, NULL, CV_DEFAULT, NULL); \ + } + +#ifndef __lock_lint +#define CALLB_CPR_ASSERT(cp) ASSERT(MUTEX_HELD((cp)->cc_lockp)); +#else +#define CALLB_CPR_ASSERT(cp) +#endif +/* + * Some threads (like the idle threads) do not adhere to the callback + * protocol and are always considered safe. Such threads must never exit. + * They register their presence by calling this macro during their + * initialization. + * + * Args: + * t - thread pointer of the client kernel thread + * name - a string giving the name of the kernel thread + */ +#define CALLB_CPR_INIT_SAFE(t, name) { \ + (void) callb_add_thread(callb_generic_cpr_safe, \ + (void *) &callb_cprinfo_safe, CB_CL_CPR_DAEMON, \ + name, t); \ + } +/* + * The lock to protect cp's content must be held before + * calling the following two macros. + * + * Any code region between CALLB_CPR_SAFE_BEGIN and CALLB_CPR_SAFE_END + * is safe for checkpoint/resume. + */ +#define CALLB_CPR_SAFE_BEGIN(cp) { \ + CALLB_CPR_ASSERT(cp) \ + (cp)->cc_events |= CALLB_CPR_SAFE; \ + if ((cp)->cc_events & CALLB_CPR_START) \ + cv_signal(&(cp)->cc_callb_cv); \ + } +#define CALLB_CPR_SAFE_END(cp, lockp) { \ + CALLB_CPR_ASSERT(cp) \ + while ((cp)->cc_events & CALLB_CPR_START) \ + cv_wait(&(cp)->cc_stop_cv, lockp); \ + (cp)->cc_events &= ~CALLB_CPR_SAFE; \ + } +/* + * cv_destroy is nop right now but may be needed in the future. + */ +#define CALLB_CPR_EXIT(cp) { \ + CALLB_CPR_ASSERT(cp) \ + (cp)->cc_events |= CALLB_CPR_SAFE; \ + if ((cp)->cc_events & CALLB_CPR_START) \ + cv_signal(&(cp)->cc_callb_cv); \ + mutex_exit((cp)->cc_lockp); \ + (void) callb_delete((cp)->cc_id); \ + cv_destroy(&(cp)->cc_callb_cv); \ + cv_destroy(&(cp)->cc_stop_cv); \ + } + +extern callb_cpr_t callb_cprinfo_safe; +extern callb_id_t callb_add(boolean_t (*)(void *, int), void *, int, char *); +extern callb_id_t callb_add_thread(boolean_t (*)(void *, int), + void *, int, char *, kthread_id_t); +extern int callb_delete(callb_id_t); +extern void callb_execute(callb_id_t, int); +extern void *callb_execute_class(int, int); +extern boolean_t callb_generic_cpr(void *, int); +extern boolean_t callb_generic_cpr_safe(void *, int); +extern boolean_t callb_is_stopped(kthread_id_t, caddr_t *); +extern void callb_lock_table(void); +extern void callb_unlock_table(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CALLB_H */ diff --git a/include/os/freebsd/spl/sys/ccompile.h b/include/os/freebsd/spl/sys/ccompile.h new file mode 100644 index 00000000000..0bb0e637cdd --- /dev/null +++ b/include/os/freebsd/spl/sys/ccompile.h @@ -0,0 +1,372 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CCOMPILE_H +#define _SYS_CCOMPILE_H + +/* + * This file contains definitions designed to enable different compilers + * to be used harmoniously on Solaris systems. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Allow for version tests for compiler bugs and features. + */ +#if defined(__GNUC__) +#define __GNUC_VERSION \ + (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) +#else +#define __GNUC_VERSION 0 +#endif + +#if defined(__ATTRIBUTE_IMPLEMENTED) || defined(__GNUC__) + +/* + * analogous to lint's PRINTFLIKEn + */ +#define __sun_attr___PRINTFLIKE__(__n) \ + __attribute__((__format__(printf, __n, (__n)+1))) +#define __sun_attr___VPRINTFLIKE__(__n) \ + __attribute__((__format__(printf, __n, 0))) + +/* + * Handle the kernel printf routines that can take '%b' too + */ +#if __GNUC_VERSION < 30402 +/* + * XX64 at least this doesn't work correctly yet with 3.4.1 anyway! + */ +#define __sun_attr___KPRINTFLIKE__ __sun_attr___PRINTFLIKE__ +#define __sun_attr___KVPRINTFLIKE__ __sun_attr___VPRINTFLIKE__ +#else +#define __sun_attr___KPRINTFLIKE__(__n) \ + __attribute__((__format__(cmn_err, __n, (__n)+1))) +#define __sun_attr___KVPRINTFLIKE__(__n) \ + __attribute__((__format__(cmn_err, __n, 0))) +#endif + +/* + * This one's pretty obvious -- the function never returns + */ +#define __sun_attr___noreturn__ __attribute__((__noreturn__)) + + +/* + * This is an appropriate label for functions that do not + * modify their arguments, e.g. strlen() + */ +#define __sun_attr___pure__ __attribute__((__pure__)) + +/* + * This is a stronger form of __pure__. Can be used for functions + * that do not modify their arguments and don't depend on global + * memory. + */ +#define __sun_attr___const__ __attribute__((__const__)) + +/* + * structure packing like #pragma pack(1) + */ +#define __sun_attr___packed__ __attribute__((__packed__)) + +#define ___sun_attr_inner(__a) __sun_attr_##__a +#define __sun_attr__(__a) ___sun_attr_inner __a + +#else /* __ATTRIBUTE_IMPLEMENTED || __GNUC__ */ + +#define __sun_attr__(__a) + +#endif /* __ATTRIBUTE_IMPLEMENTED || __GNUC__ */ + +/* + * Shorthand versions for readability + */ + +#define __PRINTFLIKE(__n) __sun_attr__((__PRINTFLIKE__(__n))) +#define __VPRINTFLIKE(__n) __sun_attr__((__VPRINTFLIKE__(__n))) +#define __KPRINTFLIKE(__n) __sun_attr__((__KPRINTFLIKE__(__n))) +#define __KVPRINTFLIKE(__n) __sun_attr__((__KVPRINTFLIKE__(__n))) +#ifdef _KERNEL +#define __NORETURN __sun_attr__((__noreturn__)) +#endif +#define __CONST __sun_attr__((__const__)) +#define __PURE __sun_attr__((__pure__)) + +#if (defined(ZFS_DEBUG) || !defined(NDEBUG))&& !defined(DEBUG) +#define DEBUG +#endif +#define EXPORT_SYMBOL(x) +#define MODULE_AUTHOR(s) +#define MODULE_DESCRIPTION(s) +#define MODULE_LICENSE(s) +#define module_param(a, b, c) +#define module_param_call(a, b, c, d, e) +#define module_param_named(a, b, c, d) +#define MODULE_PARM_DESC(a, b) +#define asm __asm +#ifdef ZFS_DEBUG +#undef NDEBUG +#endif + +#ifndef EINTEGRITY +#define EINTEGRITY 97 /* EINTEGRITY is new in 13 */ +#endif + +/* + * These are bespoke errnos used in ZFS. We map them to their closest FreeBSD + * equivalents. This gives us more useful error messages from strerror(3). + */ +#define ECKSUM EINTEGRITY +#define EFRAGS ENOSPC + +/* Similar for ENOACTIVE */ +#define ENOTACTIVE ECANCELED + +#define EREMOTEIO EREMOTE +#define ECHRNG ENXIO +#define ETIME ETIMEDOUT + +#define O_LARGEFILE 0 +#define O_RSYNC 0 +#define O_DSYNC 0 + +#define KMALLOC_MAX_SIZE MAXPHYS + +#ifdef _KERNEL +typedef unsigned long long u_longlong_t; +typedef long long longlong_t; + +#include +typedef void zfs_kernel_param_t; +#define param_set_charp(a, b) (0) +#define ATTR_UID AT_UID +#define ATTR_GID AT_GID +#define ATTR_MODE AT_MODE +#define ATTR_XVATTR AT_XVATTR +#define ATTR_CTIME AT_CTIME +#define ATTR_MTIME AT_MTIME +#define ATTR_ATIME AT_ATIME +#define vmem_free zfs_kmem_free +#define vmem_zalloc(size, flags) zfs_kmem_alloc(size, flags | M_ZERO) +#define vmem_alloc zfs_kmem_alloc +#define MUTEX_NOLOCKDEP 0 +#define RW_NOLOCKDEP 0 + + +#if __FreeBSD_version < 1300051 +#define vm_page_valid(m) (m)->valid = VM_PAGE_BITS_ALL +#define vm_page_do_sunbusy(m) +#define vm_page_none_valid(m) ((m)->valid == 0) +#else +#define vm_page_do_sunbusy(m) vm_page_sunbusy(m) +#endif + +#if __FreeBSD_version < 1300074 +#define VOP_UNLOCK1(x) VOP_UNLOCK(x, 0) +#else +#define VOP_UNLOCK1(x) VOP_UNLOCK(x) +#endif + +#if __FreeBSD_version < 1300064 +#define VN_IS_DOOMED(vp) ((vp)->v_iflag & VI_DOOMED) +#endif + +#if __FreeBSD_version < 1300068 +#define VFS_VOP_VECTOR_REGISTER(x) +#endif + +#if __FreeBSD_version >= 1300076 +#define getnewvnode_reserve_() getnewvnode_reserve() +#else +#define getnewvnode_reserve_() getnewvnode_reserve(1) +#endif + +struct hlist_node { + struct hlist_node *next, **pprev; +}; + +struct hlist_head { + struct hlist_node *first; +}; + +typedef struct { + volatile int counter; +} atomic_t; + + /* BEGIN CSTYLED */ +#define hlist_for_each(p, head) \ + for (p = (head)->first; p; p = (p)->next) + +#define hlist_entry(ptr, type, field) container_of(ptr, type, field) + +#define container_of(ptr, type, member) \ +({ \ + const __typeof(((type *)0)->member) *__p = (ptr); \ + (type *)((uintptr_t)__p - offsetof(type, member)); \ +}) + /* END CSTYLED */ + +static inline void +hlist_add_head(struct hlist_node *n, struct hlist_head *h) +{ + n->next = h->first; + if (h->first != NULL) + h->first->pprev = &n->next; + WRITE_ONCE(h->first, n); + n->pprev = &h->first; +} + +static inline void +hlist_del(struct hlist_node *n) +{ + WRITE_ONCE(*(n->pprev), n->next); + if (n->next != NULL) + n->next->pprev = n->pprev; +} + /* BEGIN CSTYLED */ +#define READ_ONCE(x) ({ \ + __typeof(x) __var = ({ \ + barrier(); \ + ACCESS_ONCE(x); \ + }); \ + barrier(); \ + __var; \ +}) + +#define HLIST_HEAD_INIT { } +#define HLIST_HEAD(name) struct hlist_head name = HLIST_HEAD_INIT +#define INIT_HLIST_HEAD(head) (head)->first = NULL + +#define INIT_HLIST_NODE(node) \ + do { \ + (node)->next = NULL; \ + (node)->pprev = NULL; \ + } while (0) + +/* END CSTYLED */ +static inline int +atomic_read(const atomic_t *v) +{ + return (READ_ONCE(v->counter)); +} + +static inline int +atomic_inc(atomic_t *v) +{ + return (atomic_fetchadd_int(&v->counter, 1) + 1); +} + +static inline int +atomic_dec(atomic_t *v) +{ + return (atomic_fetchadd_int(&v->counter, -1) - 1); +} + +#else +typedef long loff_t; +typedef long rlim64_t; +typedef int bool_t; +typedef int enum_t; +#define __init +#define __exit +#define FALSE 0 +#define TRUE 1 + /* + * XXX We really need to consolidate on standard + * error codes in the common code + */ +#define ENOSTR ENOTCONN +#define ENODATA EINVAL + + +#define __XSI_VISIBLE 1000 +#define __BSD_VISIBLE 1 +#define __POSIX_VISIBLE 201808 +#define ARRAY_SIZE(a) (sizeof (a) / sizeof (a[0])) +#define open64 open +#define pwrite64 pwrite +#define ftruncate64 ftruncate +#define lseek64 lseek +#define pread64 pread +#define stat64 stat +#define lstat64 lstat +#define statfs64 statfs +#define readdir64 readdir +#define dirent64 dirent +#define P2ALIGN(x, align) ((x) & -(align)) +#define P2CROSS(x, y, align) (((x) ^ (y)) > (align) - 1) +#define P2ROUNDUP(x, align) ((((x) - 1) | ((align) - 1)) + 1) +#define P2PHASE(x, align) ((x) & ((align) - 1)) +#define P2NPHASE(x, align) (-(x) & ((align) - 1)) +#define ISP2(x) (((x) & ((x) - 1)) == 0) +#define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) +#define P2BOUNDARY(off, len, align) \ + (((off) ^ ((off) + (len) - 1)) > (align) - 1) + +/* + * Typed version of the P2* macros. These macros should be used to ensure + * that the result is correctly calculated based on the data type of (x), + * which is passed in as the last argument, regardless of the data + * type of the alignment. For example, if (x) is of type uint64_t, + * and we want to round it up to a page boundary using "PAGESIZE" as + * the alignment, we can do either + * + * P2ROUNDUP(x, (uint64_t)PAGESIZE) + * or + * P2ROUNDUP_TYPED(x, PAGESIZE, uint64_t) + */ +#define P2ALIGN_TYPED(x, align, type) \ + ((type)(x) & -(type)(align)) +#define P2PHASE_TYPED(x, align, type) \ + ((type)(x) & ((type)(align) - 1)) +#define P2NPHASE_TYPED(x, align, type) \ + (-(type)(x) & ((type)(align) - 1)) +#define P2ROUNDUP_TYPED(x, align, type) \ + ((((type)(x) - 1) | ((type)(align) - 1)) + 1) +#define P2END_TYPED(x, align, type) \ + (-(~(type)(x) & -(type)(align))) +#define P2PHASEUP_TYPED(x, align, phase, type) \ + ((type)(phase) - (((type)(phase) - (type)(x)) & -(type)(align))) +#define P2CROSS_TYPED(x, y, align, type) \ + (((type)(x) ^ (type)(y)) > (type)(align) - 1) +#define P2SAMEHIGHBIT_TYPED(x, y, type) \ + (((type)(x) ^ (type)(y)) < ((type)(x) & (type)(y))) + +#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +#define RLIM64_INFINITY RLIM_INFINITY +#define ERESTART EAGAIN +#define ABS(a) ((a) < 0 ? -(a) : (a)) + +#endif +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CCOMPILE_H */ diff --git a/include/os/freebsd/spl/sys/cmn_err.h b/include/os/freebsd/spl/sys/cmn_err.h new file mode 100644 index 00000000000..a75471f647e --- /dev/null +++ b/include/os/freebsd/spl/sys/cmn_err.h @@ -0,0 +1,100 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + + +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CMN_ERR_H +#define _SYS_CMN_ERR_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#if !defined(_ASM) +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Common error handling severity levels */ + +#define CE_CONT 0 /* continuation */ +#define CE_NOTE 1 /* notice */ +#define CE_WARN 2 /* warning */ +#define CE_PANIC 3 /* panic */ +#define CE_IGNORE 4 /* print nothing */ + +#ifndef _ASM + +/*PRINTFLIKE2*/ +extern void cmn_err(int, const char *, ...) + __KPRINTFLIKE(2); +#pragma rarely_called(cmn_err) + +extern void vzcmn_err(zoneid_t, int, const char *, __va_list) + __KVPRINTFLIKE(3); +#pragma rarely_called(vzcmn_err) + +extern void vcmn_err(int, const char *, __va_list) + __KVPRINTFLIKE(2); +#pragma rarely_called(vcmn_err) + +/*PRINTFLIKE3*/ +extern void zcmn_err(zoneid_t, int, const char *, ...) + __KPRINTFLIKE(3); +#pragma rarely_called(zcmn_err) + +extern void vzprintf(zoneid_t, const char *, __va_list) + __KVPRINTFLIKE(2); +#pragma rarely_called(vzprintf) + +/*PRINTFLIKE2*/ +extern void zprintf(zoneid_t, const char *, ...) + __KPRINTFLIKE(2); +#pragma rarely_called(zprintf) + +extern void vuprintf(const char *, __va_list) + __KVPRINTFLIKE(1); +#pragma rarely_called(vuprintf) + +/*PRINTFLIKE1*/ +extern void panic(const char *, ...) + __KPRINTFLIKE(1) __NORETURN; +#pragma rarely_called(panic) + +extern void vpanic(const char *, __va_list) + __KVPRINTFLIKE(1) __NORETURN; +#pragma rarely_called(vpanic) + +#endif /* !_ASM */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CMN_ERR_H */ diff --git a/include/os/freebsd/spl/sys/condvar.h b/include/os/freebsd/spl/sys/condvar.h new file mode 100644 index 00000000000..b21940166c5 --- /dev/null +++ b/include/os/freebsd/spl/sys/condvar.h @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * Copyright (c) 2013 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_CONDVAR_H_ +#define _OPENSOLARIS_SYS_CONDVAR_H_ + +#include + +#include +#include +#include +#include + +static __inline sbintime_t +zfs_nstosbt(int64_t _ns) +{ + sbintime_t sb = 0; + +#ifdef KASSERT + KASSERT(_ns >= 0, ("Negative values illegal for nstosbt: %jd", _ns)); +#endif + if (_ns >= SBT_1S) { + sb = (_ns / 1000000000) * SBT_1S; + _ns = _ns % 1000000000; + } + /* 9223372037 = ceil(2^63 / 1000000000) */ + sb += ((_ns * 9223372037ull) + 0x7fffffff) >> 31; + return (sb); +} + + +typedef struct cv kcondvar_t; +#define CALLOUT_FLAG_ABSOLUTE C_ABSOLUTE + +typedef enum { + CV_DEFAULT, + CV_DRIVER +} kcv_type_t; + +#define zfs_cv_init(cv, name, type, arg) do { \ + const char *_name; \ + ASSERT((type) == CV_DEFAULT); \ + for (_name = #cv; *_name != '\0'; _name++) { \ + if (*_name >= 'a' && *_name <= 'z') \ + break; \ + } \ + if (*_name == '\0') \ + _name = #cv; \ + cv_init((cv), _name); \ +} while (0) +#define cv_init(cv, name, type, arg) zfs_cv_init(cv, name, type, arg) + + +static inline int +cv_wait_sig(kcondvar_t *cvp, kmutex_t *mp) +{ + + return (_cv_wait_sig(cvp, &(mp)->lock_object) == 0); +} + +static inline int +cv_timedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t timo) +{ + int rc; + + timo -= ddi_get_lbolt(); + if (timo <= 0) + return (-1); + rc = _cv_timedwait_sbt((cvp), &(mp)->lock_object, \ + tick_sbt * (timo), 0, C_HARDCLOCK); + if (rc == EWOULDBLOCK) + return (-1); + return (1); +} + +static inline int +cv_timedwait_sig(kcondvar_t *cvp, kmutex_t *mp, clock_t timo) +{ + int rc; + + timo -= ddi_get_lbolt(); + if (timo <= 0) + return (-1); + rc = _cv_timedwait_sig_sbt(cvp, &(mp)->lock_object, \ + tick_sbt * (timo), 0, C_HARDCLOCK); + if (rc == EWOULDBLOCK) + return (-1); + if (rc == EINTR || rc == ERESTART) + return (0); + + return (1); +} + +#define cv_timedwait_io cv_timedwait +#define cv_timedwait_sig_io cv_timedwait_sig + +static inline clock_t +cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, hrtime_t res, + int flag) +{ + hrtime_t hrtime; + int rc; + + ASSERT(tim >= res); + + hrtime = gethrtime(); + if (flag == 0) + tim += hrtime; + + if (hrtime >= tim) + return (tim - hrtime); + rc = cv_timedwait_sbt(cvp, mp, zfs_nstosbt(tim), + zfs_nstosbt(res), C_ABSOLUTE); + + KASSERT(rc == EWOULDBLOCK || rc == 0, ("unexpected rc value %d", rc)); + return (tim - gethrtime()); +} + +static inline clock_t +cv_timedwait_sig_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, + hrtime_t res, int flag) +{ + sbintime_t sbt; + hrtime_t hrtime; + int rc; + + ASSERT(tim >= res); + + hrtime = gethrtime(); + if (flag == 0) + tim += hrtime; + + if (hrtime >= tim) + return (tim - hrtime); + + sbt = zfs_nstosbt(tim); + rc = cv_timedwait_sig_sbt(cvp, mp, sbt, zfs_nstosbt(res), C_ABSOLUTE); + + KASSERT(rc == EWOULDBLOCK || rc == EINTR || rc == ERESTART || + rc == 0, ("unexpected rc value %d", rc)); + return (tim - gethrtime()); +} + +#endif /* _OPENSOLARIS_SYS_CONDVAR_H_ */ diff --git a/include/os/freebsd/spl/sys/console.h b/include/os/freebsd/spl/sys/console.h new file mode 100644 index 00000000000..abf3db75676 --- /dev/null +++ b/include/os/freebsd/spl/sys/console.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SPL_CONSOLE_H +#define _SPL_CONSOLE_H + +static inline void +console_vprintf(const char *fmt, va_list args) +{ + vprintf(fmt, args); +} + +static inline void +console_printf(const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + console_vprintf(fmt, args); + va_end(args); +} + +#endif /* _SPL_CONSOLE_H */ diff --git a/include/os/freebsd/spl/sys/cred.h b/include/os/freebsd/spl/sys/cred.h new file mode 100644 index 00000000000..e32910e0efa --- /dev/null +++ b/include/os/freebsd/spl/sys/cred.h @@ -0,0 +1,188 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +/* + * Portions of this source code were derived from Berkeley 4.3 BSD + * under license from the Regents of the University of California. + */ + +#ifndef _SYS_CRED_H +#define _SYS_CRED_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * The credential is an opaque kernel private data structure defined in + * . + */ + +typedef struct ucred cred_t; + +#define CRED() curthread->td_ucred +#define kcred (thread0.td_ucred) + +#define KUID_TO_SUID(x) (x) +#define KGID_TO_SGID(x) (x) +#define crgetuid(cred) ((cred)->cr_uid) +#define crgetruid(cred) ((cred)->cr_ruid) +#define crgetgid(cred) ((cred)->cr_gid) +#define crgetgroups(cred) ((cred)->cr_groups) +#define crgetngroups(cred) ((cred)->cr_ngroups) +#define crgetsid(cred, i) (NULL) + +struct proc; /* cred.h is included in proc.h */ +struct prcred; +struct ksid; +struct ksidlist; +struct credklpd; +struct credgrp; + +struct auditinfo_addr; /* cred.h is included in audit.h */ + +extern int ngroups_max; +/* + * kcred is used when you need all privileges. + */ + +extern void cred_init(void); +extern void crfree(cred_t *); +extern cred_t *cralloc(void); /* all but ref uninitialized */ +extern cred_t *cralloc_ksid(void); /* cralloc() + ksid alloc'ed */ +extern cred_t *crget(void); /* initialized */ +extern void crcopy_to(cred_t *, cred_t *); +extern cred_t *crdup(cred_t *); +extern void crdup_to(cred_t *, cred_t *); +extern cred_t *crgetcred(void); +extern void crset(struct proc *, cred_t *); +extern void crset_zone_privall(cred_t *); +extern int supgroupmember(gid_t, const cred_t *); +extern int hasprocperm(const cred_t *, const cred_t *); +extern int prochasprocperm(struct proc *, struct proc *, const cred_t *); +extern int crcmp(const cred_t *, const cred_t *); +extern cred_t *zone_kcred(void); + +extern gid_t crgetrgid(const cred_t *); +extern gid_t crgetsgid(const cred_t *); + +#define crgetzoneid(x) (0) +extern projid_t crgetprojid(const cred_t *); + +extern cred_t *crgetmapped(const cred_t *); + + +extern const struct auditinfo_addr *crgetauinfo(const cred_t *); +extern struct auditinfo_addr *crgetauinfo_modifiable(cred_t *); + +extern uint_t crgetref(const cred_t *); + +extern const gid_t *crgetggroups(const struct credgrp *); + + +/* + * Sets real, effective and/or saved uid/gid; + * -1 argument accepted as "no change". + */ +extern int crsetresuid(cred_t *, uid_t, uid_t, uid_t); +extern int crsetresgid(cred_t *, gid_t, gid_t, gid_t); + +/* + * Sets real, effective and saved uids/gids all to the same + * values. Both values must be non-negative and <= MAXUID + */ +extern int crsetugid(cred_t *, uid_t, gid_t); + +/* + * Functions to handle the supplemental group list. + */ +extern struct credgrp *crgrpcopyin(int, gid_t *); +extern void crgrprele(struct credgrp *); +extern void crsetcredgrp(cred_t *, struct credgrp *); + +/* + * Private interface for setting zone association of credential. + */ +struct zone; +extern void crsetzone(cred_t *, struct zone *); +extern struct zone *crgetzone(const cred_t *); + +/* + * Private interface for setting project id in credential. + */ +extern void crsetprojid(cred_t *, projid_t); + +/* + * Private interface for nfs. + */ +extern cred_t *crnetadjust(cred_t *); + +/* + * Private interface for procfs. + */ +extern void cred2prcred(const cred_t *, struct prcred *); + +/* + * Private interfaces for Rampart Trusted Solaris. + */ +struct ts_label_s; +extern struct ts_label_s *crgetlabel(const cred_t *); +extern boolean_t crisremote(const cred_t *); + +/* + * Private interfaces for ephemeral uids. + */ +#define VALID_UID(id, zn) \ + ((id) <= MAXUID || valid_ephemeral_uid((zn), (id))) + +#define VALID_GID(id, zn) \ + ((id) <= MAXUID || valid_ephemeral_gid((zn), (id))) + +extern boolean_t valid_ephemeral_uid(struct zone *, uid_t); +extern boolean_t valid_ephemeral_gid(struct zone *, gid_t); + +extern int eph_uid_alloc(struct zone *, int, uid_t *, int); +extern int eph_gid_alloc(struct zone *, int, gid_t *, int); + +extern void crsetsid(cred_t *, struct ksid *, int); +extern void crsetsidlist(cred_t *, struct ksidlist *); + +extern struct ksidlist *crgetsidlist(const cred_t *); + +extern int crsetpriv(cred_t *, ...); + +extern struct credklpd *crgetcrklpd(const cred_t *); +extern void crsetcrklpd(cred_t *, struct credklpd *); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CRED_H */ diff --git a/include/os/freebsd/spl/sys/ctype.h b/include/os/freebsd/spl/sys/ctype.h new file mode 100644 index 00000000000..f225858072a --- /dev/null +++ b/include/os/freebsd/spl/sys/ctype.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + +#ifndef _SPL_SYS_CTYPE_H_ +#define _SPL_SYS_CTYPE_H_ +#include_next + +#define isalnum(ch) (isalpha(ch) || isdigit(ch)) +#define iscntrl(C) (uchar(C) <= 0x1f || uchar(C) == 0x7f) +#define isgraph(C) ((C) >= 0x21 && (C) <= 0x7E) +/* BEGIN CSTYLED */ +#define ispunct(C) \ + (((C) >= 0x21 && (C) <= 0x2F) || \ + ((C) >= 0x3A && (C) <= 0x40) || \ + ((C) >= 0x5B && (C) <= 0x60) || \ + ((C) >= 0x7B && (C) <= 0x7E)) +/* END CSTYLED */ + +#endif diff --git a/include/os/freebsd/spl/sys/debug.h b/include/os/freebsd/spl/sys/debug.h new file mode 100644 index 00000000000..2751f57801f --- /dev/null +++ b/include/os/freebsd/spl/sys/debug.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* + * Available Solaris debug functions. All of the ASSERT() macros will be + * compiled out when NDEBUG is defined, this is the default behavior for + * the SPL. To enable assertions use the --enable-debug with configure. + * The VERIFY() functions are never compiled out and cannot be disabled. + * + * PANIC() - Panic the node and print message. + * ASSERT() - Assert X is true, if not panic. + * ASSERT3B() - Assert boolean X OP Y is true, if not panic. + * ASSERT3S() - Assert signed X OP Y is true, if not panic. + * ASSERT3U() - Assert unsigned X OP Y is true, if not panic. + * ASSERT3P() - Assert pointer X OP Y is true, if not panic. + * ASSERT0() - Assert value is zero, if not panic. + * VERIFY() - Verify X is true, if not panic. + * VERIFY3B() - Verify boolean X OP Y is true, if not panic. + * VERIFY3S() - Verify signed X OP Y is true, if not panic. + * VERIFY3U() - Verify unsigned X OP Y is true, if not panic. + * VERIFY3P() - Verify pointer X OP Y is true, if not panic. + * VERIFY0() - Verify value is zero, if not panic. + */ + +#ifndef _SPL_DEBUG_H +#define _SPL_DEBUG_H + + +/* + * Common DEBUG functionality. + */ +int spl_panic(const char *file, const char *func, int line, + const char *fmt, ...); +void spl_dumpstack(void); + +#ifndef expect +#define expect(expr, value) (__builtin_expect((expr), (value))) +#endif +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + +/* BEGIN CSTYLED */ +#define PANIC(fmt, a...) \ + spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a) + +#define VERIFY(cond) \ + (void) (unlikely(!(cond)) && \ + spl_panic(__FILE__, __FUNCTION__, __LINE__, \ + "%s", "VERIFY(" #cond ") failed\n")) + +#define VERIFY3B(LEFT, OP, RIGHT) do { \ + boolean_t _verify3_left = (boolean_t)(LEFT); \ + boolean_t _verify3_right = (boolean_t)(RIGHT); \ + if (!(_verify3_left OP _verify3_right)) \ + spl_panic(__FILE__, __FUNCTION__, __LINE__, \ + "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ + "failed (%d " #OP " %d)\n", \ + (boolean_t) (_verify3_left), \ + (boolean_t) (_verify3_right)); \ + } while (0) + +#define VERIFY3S(LEFT, OP, RIGHT) do { \ + int64_t _verify3_left = (int64_t)(LEFT); \ + int64_t _verify3_right = (int64_t)(RIGHT); \ + if (!(_verify3_left OP _verify3_right)) \ + spl_panic(__FILE__, __FUNCTION__, __LINE__, \ + "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ + "failed (%lld " #OP " %lld)\n", \ + (long long) (_verify3_left), \ + (long long) (_verify3_right)); \ + } while (0) + +#define VERIFY3U(LEFT, OP, RIGHT) do { \ + uint64_t _verify3_left = (uint64_t)(LEFT); \ + uint64_t _verify3_right = (uint64_t)(RIGHT); \ + if (!(_verify3_left OP _verify3_right)) \ + spl_panic(__FILE__, __FUNCTION__, __LINE__, \ + "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ + "failed (%llu " #OP " %llu)\n", \ + (unsigned long long) (_verify3_left), \ + (unsigned long long) (_verify3_right)); \ + } while (0) + +#define VERIFY3P(LEFT, OP, RIGHT) do { \ + uintptr_t _verify3_left = (uintptr_t)(LEFT); \ + uintptr_t _verify3_right = (uintptr_t)(RIGHT); \ + if (!(_verify3_left OP _verify3_right)) \ + spl_panic(__FILE__, __FUNCTION__, __LINE__, \ + "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ + "failed (%px " #OP " %px)\n", \ + (void *) (_verify3_left), \ + (void *) (_verify3_right)); \ + } while (0) + +#define VERIFY0(RIGHT) do { \ + int64_t _verify3_left = (int64_t)(0); \ + int64_t _verify3_right = (int64_t)(RIGHT); \ + if (!(_verify3_left == _verify3_right)) \ + spl_panic(__FILE__, __FUNCTION__, __LINE__, \ + "VERIFY3(0 == " #RIGHT ") " \ + "failed (0 == %lld)\n", \ + (long long) (_verify3_right)); \ + } while (0) +#define CTASSERT_GLOBAL(x) CTASSERT(x) + +/* + * Debugging disabled (--disable-debug) + */ +#ifdef NDEBUG + +#define ASSERT(x) ((void)0) +#define ASSERT3B(x,y,z) ((void)0) +#define ASSERT3S(x,y,z) ((void)0) +#define ASSERT3U(x,y,z) ((void)0) +#define ASSERT3P(x,y,z) ((void)0) +#define ASSERT0(x) ((void)0) +#define IMPLY(A, B) ((void)0) +#define EQUIV(A, B) ((void)0) + +/* + * Debugging enabled (--enable-debug) + */ +#else + +#define ASSERT3B VERIFY3B +#define ASSERT3S VERIFY3S +#define ASSERT3U VERIFY3U +#define ASSERT3P VERIFY3P +#define ASSERT0 VERIFY0 +#define ASSERT VERIFY +#define IMPLY(A, B) \ + ((void)(((!(A)) || (B)) || \ + spl_panic(__FILE__, __FUNCTION__, __LINE__, \ + "(" #A ") implies (" #B ")"))) +#define EQUIV(A, B) \ + ((void)((!!(A) == !!(B)) || \ + spl_panic(__FILE__, __FUNCTION__, __LINE__, \ + "(" #A ") is equivalent to (" #B ")"))) +/* END CSTYLED */ + +#endif /* NDEBUG */ + +#endif /* SPL_DEBUG_H */ diff --git a/include/os/freebsd/spl/sys/dirent.h b/include/os/freebsd/spl/sys/dirent.h new file mode 100644 index 00000000000..2403766a427 --- /dev/null +++ b/include/os/freebsd/spl/sys/dirent.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_DIRENT_H_ +#define _OPENSOLARIS_SYS_DIRENT_H_ + +#include + +#include_next + +typedef struct dirent dirent64_t; +typedef ino_t ino64_t; + +#define dirent64 dirent + +#define d_ino d_fileno + +#define DIRENT64_RECLEN(len) _GENERIC_DIRLEN(len) + +#endif /* !_OPENSOLARIS_SYS_DIRENT_H_ */ diff --git a/include/os/freebsd/spl/sys/disp.h b/include/os/freebsd/spl/sys/disp.h new file mode 100644 index 00000000000..2be1b76e433 --- /dev/null +++ b/include/os/freebsd/spl/sys/disp.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2013 Andriy Gapon + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_DISP_H_ +#define _OPENSOLARIS_SYS_DISP_H_ + +#include + +#define kpreempt(x) kern_yield(PRI_USER) + +#endif /* _OPENSOLARIS_SYS_DISP_H_ */ diff --git a/include/os/freebsd/spl/sys/dkio.h b/include/os/freebsd/spl/sys/dkio.h new file mode 100644 index 00000000000..4e9ded4a978 --- /dev/null +++ b/include/os/freebsd/spl/sys/dkio.h @@ -0,0 +1,495 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _OPENSOLARIS_SYS_DKIO_H_ +#define _OPENSOLARIS_SYS_DKIO_H_ + +#include /* Needed for NDKMAP define */ + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(_SUNOS_VTOC_16) +#define NDKMAP 16 /* # of logical partitions */ +#define DK_LABEL_LOC 1 /* location of disk label */ +#elif defined(_SUNOS_VTOC_8) +#define NDKMAP 8 /* # of logical partitions */ +#define DK_LABEL_LOC 0 /* location of disk label */ +#else +#error "No VTOC format defined." +#endif + +/* + * Structures and definitions for disk io control commands + */ + +/* + * Structures used as data by ioctl calls. + */ + +#define DK_DEVLEN 16 /* device name max length, including */ + /* unit # & NULL (ie - "xyc1") */ + +/* + * Used for controller info + */ +struct dk_cinfo { + char dki_cname[DK_DEVLEN]; /* controller name (no unit #) */ + ushort_t dki_ctype; /* controller type */ + ushort_t dki_flags; /* flags */ + ushort_t dki_cnum; /* controller number */ + uint_t dki_addr; /* controller address */ + uint_t dki_space; /* controller bus type */ + uint_t dki_prio; /* interrupt priority */ + uint_t dki_vec; /* interrupt vector */ + char dki_dname[DK_DEVLEN]; /* drive name (no unit #) */ + uint_t dki_unit; /* unit number */ + uint_t dki_slave; /* slave number */ + ushort_t dki_partition; /* partition number */ + ushort_t dki_maxtransfer; /* max. transfer size in DEV_BSIZE */ +}; + +/* + * Controller types + */ +#define DKC_UNKNOWN 0 +#define DKC_CDROM 1 /* CD-ROM, SCSI or otherwise */ +#define DKC_WDC2880 2 +#define DKC_XXX_0 3 /* unassigned */ +#define DKC_XXX_1 4 /* unassigned */ +#define DKC_DSD5215 5 +#define DKC_ACB4000 7 +#define DKC_MD21 8 +#define DKC_XXX_2 9 /* unassigned */ +#define DKC_NCRFLOPPY 10 +#define DKC_SMSFLOPPY 12 +#define DKC_SCSI_CCS 13 /* SCSI CCS compatible */ +#define DKC_INTEL82072 14 /* native floppy chip */ +#define DKC_MD 16 /* meta-disk (virtual-disk) driver */ +#define DKC_INTEL82077 19 /* 82077 floppy disk controller */ +#define DKC_DIRECT 20 /* Intel direct attached device i.e. IDE */ +#define DKC_PCMCIA_MEM 21 /* PCMCIA memory disk-like type */ +#define DKC_PCMCIA_ATA 22 /* PCMCIA AT Attached type */ +#define DKC_VBD 23 /* virtual block device */ + +/* + * Sun reserves up through 1023 + */ + +#define DKC_CUSTOMER_BASE 1024 + +/* + * Flags + */ +#define DKI_BAD144 0x01 /* use DEC std 144 bad sector fwding */ +#define DKI_MAPTRK 0x02 /* controller does track mapping */ +#define DKI_FMTTRK 0x04 /* formats only full track at a time */ +#define DKI_FMTVOL 0x08 /* formats only full volume at a time */ +#define DKI_FMTCYL 0x10 /* formats only full cylinders at a time */ +#define DKI_HEXUNIT 0x20 /* unit number is printed as 3 hex digits */ +#define DKI_PCMCIA_PFD 0x40 /* PCMCIA pseudo-floppy memory card */ + +/* + * partition headers: section 1 + * Returned in struct dk_allmap by ioctl DKIOC[SG]APART (dkio(7I)) + */ +struct dk_map { + uint64_t dkl_cylno; /* starting cylinder */ + uint64_t dkl_nblk; /* number of blocks; if == 0, */ + /* partition is undefined */ +}; + +/* + * Used for all partitions + */ +struct dk_allmap { + struct dk_map dka_map[NDKMAP]; +}; + +#if defined(_SYSCALL32) +struct dk_allmap32 { + struct dk_map32 dka_map[NDKMAP]; +}; +#endif /* _SYSCALL32 */ + +/* + * Definition of a disk's geometry + */ +struct dk_geom { + unsigned short dkg_ncyl; /* # of data cylinders */ + unsigned short dkg_acyl; /* # of alternate cylinders */ + unsigned short dkg_bcyl; /* cyl offset (for fixed head area) */ + unsigned short dkg_nhead; /* # of heads */ + unsigned short dkg_obs1; /* obsolete */ + unsigned short dkg_nsect; /* # of data sectors per track */ + unsigned short dkg_intrlv; /* interleave factor */ + unsigned short dkg_obs2; /* obsolete */ + unsigned short dkg_obs3; /* obsolete */ + unsigned short dkg_apc; /* alternates per cyl (SCSI only) */ + unsigned short dkg_rpm; /* revolutions per minute */ + unsigned short dkg_pcyl; /* # of physical cylinders */ + unsigned short dkg_write_reinstruct; /* # sectors to skip, writes */ + unsigned short dkg_read_reinstruct; /* # sectors to skip, reads */ + unsigned short dkg_extra[7]; /* for compatible expansion */ +}; + +/* + * These defines are for historic compatibility with old drivers. + */ +#define dkg_bhead dkg_obs1 /* used to be head offset */ +#define dkg_gap1 dkg_obs2 /* used to be gap1 */ +#define dkg_gap2 dkg_obs3 /* used to be gap2 */ + +/* + * Disk io control commands + * Warning: some other ioctls with the DIOC prefix exist elsewhere. + * The Generic DKIOC numbers are from 0 - 50. + * The Floppy Driver uses 51 - 100. + * The Hard Disk (except SCSI) 101 - 106. (these are obsolete) + * The CDROM Driver 151 - 200. + * The USCSI ioctl 201 - 250. + */ +#define DKIOC (0x04 << 8) + +/* + * The following ioctls are generic in nature and need to be + * supported as appropriate by all disk drivers + */ +#define DKIOCGGEOM (DKIOC|1) /* Get geometry */ +#define DKIOCINFO (DKIOC|3) /* Get info */ +#define DKIOCEJECT (DKIOC|6) /* Generic 'eject' */ +#define DKIOCGVTOC (DKIOC|11) /* Get VTOC */ +#define DKIOCSVTOC (DKIOC|12) /* Set VTOC & Write to Disk */ + +/* + * Disk Cache Controls. These ioctls should be supported by + * all disk drivers. + * + * DKIOCFLUSHWRITECACHE when used from user-mode ignores the ioctl + * argument, but it should be passed as NULL to allow for future + * reinterpretation. From user-mode, this ioctl request is synchronous. + * + * When invoked from within the kernel, the arg can be NULL to indicate + * a synchronous request or can be the address of a struct dk_callback + * to request an asynchronous callback when the flush request is complete. + * In this case, the flag to the ioctl must include FKIOCTL and the + * dkc_callback field of the pointed to struct must be non-null or the + * request is made synchronously. + * + * In the callback case: if the ioctl returns 0, a callback WILL be performed. + * If the ioctl returns non-zero, a callback will NOT be performed. + * NOTE: In some cases, the callback may be done BEFORE the ioctl call + * returns. The caller's locking strategy should be prepared for this case. + */ +#define DKIOCFLUSHWRITECACHE (DKIOC|34) /* flush cache to phys medium */ + +struct dk_callback { + void (*dkc_callback)(void *dkc_cookie, int error); + void *dkc_cookie; + int dkc_flag; +}; + +/* bit flag definitions for dkc_flag */ +#define FLUSH_VOLATILE 0x1 /* Bit 0: if set, only flush */ + /* volatile cache; otherwise, flush */ + /* volatile and non-volatile cache */ + +#define DKIOCGETWCE (DKIOC|36) /* Get current write cache */ + /* enablement status */ +#define DKIOCSETWCE (DKIOC|37) /* Enable/Disable write cache */ + +/* + * The following ioctls are used by Sun drivers to communicate + * with their associated format routines. Support of these ioctls + * is not required of foreign drivers + */ +#define DKIOCSGEOM (DKIOC|2) /* Set geometry */ +#define DKIOCSAPART (DKIOC|4) /* Set all partitions */ +#define DKIOCGAPART (DKIOC|5) /* Get all partitions */ +#define DKIOCG_PHYGEOM (DKIOC|32) /* get physical geometry */ +#define DKIOCG_VIRTGEOM (DKIOC|33) /* get virtual geometry */ + +/* + * The following ioctl's are removable media support + */ +#define DKIOCLOCK (DKIOC|7) /* Generic 'lock' */ +#define DKIOCUNLOCK (DKIOC|8) /* Generic 'unlock' */ +#define DKIOCSTATE (DKIOC|13) /* Inquire insert/eject state */ +#define DKIOCREMOVABLE (DKIOC|16) /* is media removable */ + + +/* + * ioctl for hotpluggable devices + */ +#define DKIOCHOTPLUGGABLE (DKIOC|35) /* is hotpluggable */ + +/* + * Ioctl to force driver to re-read the alternate partition and rebuild + * the internal defect map. + */ +#define DKIOCADDBAD (DKIOC|20) /* Re-read the alternate map (IDE) */ +#define DKIOCGETDEF (DKIOC|21) /* read defect list (IDE) */ + +/* + * Used by applications to get disk defect information from IDE + * drives. + */ +#ifdef _SYSCALL32 +struct defect_header32 { + int head; + caddr32_t buffer; +}; +#endif /* _SYSCALL32 */ + +struct defect_header { + int head; + caddr_t buffer; +}; + +#define DKIOCPARTINFO (DKIOC|22) /* Get partition or slice parameters */ + +/* + * Used by applications to get partition or slice information + */ +#ifdef _SYSCALL32 +struct part_info32 { + uint32_t p_start; + int p_length; +}; +#endif /* _SYSCALL32 */ + +struct part_info { + uint64_t p_start; + int p_length; +}; + +/* The following ioctls are for Optical Memory Device */ +#define DKIOC_EBP_ENABLE (DKIOC|40) /* enable by pass erase on write */ +#define DKIOC_EBP_DISABLE (DKIOC|41) /* disable by pass erase on write */ + +/* + * This state enum is the argument passed to the DKIOCSTATE ioctl. + */ +enum dkio_state { DKIO_NONE, DKIO_EJECTED, DKIO_INSERTED, DKIO_DEV_GONE }; + +#define DKIOCGMEDIAINFO (DKIOC|42) /* get information about the media */ + +/* + * ioctls to read/write mboot info. + */ +#define DKIOCGMBOOT (DKIOC|43) /* get mboot info */ +#define DKIOCSMBOOT (DKIOC|44) /* set mboot info */ + +/* + * ioctl to get the device temperature. + */ +#define DKIOCGTEMPERATURE (DKIOC|45) /* get temperature */ + +/* + * Used for providing the temperature. + */ + +struct dk_temperature { + uint_t dkt_flags; /* Flags */ + short dkt_cur_temp; /* Current disk temperature */ + short dkt_ref_temp; /* reference disk temperature */ +}; + +#define DKT_BYPASS_PM 0x1 +#define DKT_INVALID_TEMP 0xFFFF + + +/* + * Media types or profiles known + */ +#define DK_UNKNOWN 0x00 /* Media inserted - type unknown */ + + +/* + * SFF 8090 Specification Version 3, media types 0x01 - 0xfffe are retained to + * maintain compatibility with SFF8090. The following define the + * optical media type. + */ +#define DK_REMOVABLE_DISK 0x02 /* Removable Disk */ +#define DK_MO_ERASABLE 0x03 /* MO Erasable */ +#define DK_MO_WRITEONCE 0x04 /* MO Write once */ +#define DK_AS_MO 0x05 /* AS MO */ +#define DK_CDROM 0x08 /* CDROM */ +#define DK_CDR 0x09 /* CD-R */ +#define DK_CDRW 0x0A /* CD-RW */ +#define DK_DVDROM 0x10 /* DVD-ROM */ +#define DK_DVDR 0x11 /* DVD-R */ +#define DK_DVDRAM 0x12 /* DVD_RAM or DVD-RW */ + +/* + * Media types for other rewritable magnetic media + */ +#define DK_FIXED_DISK 0x10001 /* Fixed disk SCSI or otherwise */ +#define DK_FLOPPY 0x10002 /* Floppy media */ +#define DK_ZIP 0x10003 /* IOMEGA ZIP media */ +#define DK_JAZ 0x10004 /* IOMEGA JAZ media */ + +#define DKIOCSETEFI (DKIOC|17) /* Set EFI info */ +#define DKIOCGETEFI (DKIOC|18) /* Get EFI info */ + +#define DKIOCPARTITION (DKIOC|9) /* Get partition info */ + +/* + * Ioctls to get/set volume capabilities related to Logical Volume Managers. + * They include the ability to get/set capabilities and to issue a read to a + * specific underlying device of a replicated device. + */ + +#define DKIOCGETVOLCAP (DKIOC | 25) /* Get volume capabilities */ +#define DKIOCSETVOLCAP (DKIOC | 26) /* Set volume capabilities */ +#define DKIOCDMR (DKIOC | 27) /* Issue a directed read */ + +typedef uint_t volcapinfo_t; + +typedef uint_t volcapset_t; + +#define DKV_ABR_CAP 0x00000001 /* Support Appl.Based Recovery */ +#define DKV_DMR_CAP 0x00000002 /* Support Directed Mirror Read */ + +typedef struct volcap { + volcapinfo_t vc_info; /* Capabilities available */ + volcapset_t vc_set; /* Capabilities set */ +} volcap_t; + +#define VOL_SIDENAME 256 + +typedef struct vol_directed_rd { + int vdr_flags; + offset_t vdr_offset; + size_t vdr_nbytes; + size_t vdr_bytesread; + void *vdr_data; + int vdr_side; + char vdr_side_name[VOL_SIDENAME]; +} vol_directed_rd_t; + +#define DKV_SIDE_INIT (-1) +#define DKV_DMR_NEXT_SIDE 0x00000001 +#define DKV_DMR_DONE 0x00000002 +#define DKV_DMR_ERROR 0x00000004 +#define DKV_DMR_SUCCESS 0x00000008 +#define DKV_DMR_SHORT 0x00000010 + +#ifdef _MULTI_DATAMODEL +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif +typedef struct vol_directed_rd32 { + int32_t vdr_flags; + offset_t vdr_offset; /* 64-bit element on 32-bit alignment */ + size32_t vdr_nbytes; + size32_t vdr_bytesread; + caddr32_t vdr_data; + int32_t vdr_side; + char vdr_side_name[VOL_SIDENAME]; +} vol_directed_rd32_t; +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif +#endif /* _MULTI_DATAMODEL */ + +/* + * The ioctl is used to fetch disk's device type, vendor ID, + * model number/product ID, firmware revision and serial number together. + * + * Currently there are two device types - DKD_ATA_TYPE which means the + * disk is driven by cmdk/ata or dad/uata driver, and DKD_SCSI_TYPE + * which means the disk is driven by sd/scsi hba driver. + */ +#define DKIOC_GETDISKID (DKIOC|46) + +/* These two labels are for dkd_dtype of dk_disk_id_t */ +#define DKD_ATA_TYPE 0x01 /* ATA disk or legacy mode SATA disk */ +#define DKD_SCSI_TYPE 0x02 /* SCSI disk or native mode SATA disk */ + +#define DKD_ATA_MODEL 40 /* model number length */ +#define DKD_ATA_FWVER 8 /* firmware revision length */ +#define DKD_ATA_SERIAL 20 /* serial number length */ + +#define DKD_SCSI_VENDOR 8 /* vendor ID length */ +#define DKD_SCSI_PRODUCT 16 /* product ID length */ +#define DKD_SCSI_REVLEVEL 4 /* revision level length */ +#define DKD_SCSI_SERIAL 12 /* serial number length */ + +/* + * The argument type for DKIOC_GETDISKID ioctl. + */ +typedef struct dk_disk_id { + uint_t dkd_dtype; + union { + struct { + char dkd_amodel[DKD_ATA_MODEL]; /* 40 bytes */ + char dkd_afwver[DKD_ATA_FWVER]; /* 8 bytes */ + char dkd_aserial[DKD_ATA_SERIAL]; /* 20 bytes */ + } ata_disk_id; + struct { + char dkd_svendor[DKD_SCSI_VENDOR]; /* 8 bytes */ + char dkd_sproduct[DKD_SCSI_PRODUCT]; /* 16 bytes */ + char dkd_sfwver[DKD_SCSI_REVLEVEL]; /* 4 bytes */ + char dkd_sserial[DKD_SCSI_SERIAL]; /* 12 bytes */ + } scsi_disk_id; + } disk_id; +} dk_disk_id_t; + +/* + * The ioctl is used to update the firmware of device. + */ +#define DKIOC_UPDATEFW (DKIOC|47) + +/* The argument type for DKIOC_UPDATEFW ioctl */ +typedef struct dk_updatefw { + caddr_t dku_ptrbuf; /* pointer to firmware buf */ + uint_t dku_size; /* firmware buf length */ + uint8_t dku_type; /* firmware update type */ +} dk_updatefw_t; + +#ifdef _SYSCALL32 +typedef struct dk_updatefw_32 { + caddr32_t dku_ptrbuf; /* pointer to firmware buf */ + uint_t dku_size; /* firmware buf length */ + uint8_t dku_type; /* firmware update type */ +} dk_updatefw_32_t; +#endif /* _SYSCALL32 */ + +/* + * firmware update type - temporary or permanent use + */ +#define FW_TYPE_TEMP 0x0 /* temporary use */ +#define FW_TYPE_PERM 0x1 /* permanent use */ + + +#ifdef __cplusplus +} +#endif + +#endif /* _OPENSOLARIS_SYS_DKIO_H_ */ diff --git a/include/os/freebsd/spl/sys/endian.h b/include/os/freebsd/spl/sys/endian.h new file mode 100644 index 00000000000..4de4b8829c5 --- /dev/null +++ b/include/os/freebsd/spl/sys/endian.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + +#ifndef _SPL_SYS_ENDIAN_H_ +#define _SPL_SYS_ENDIAN_H_ + +#undef _MACHINE_ENDIAN_H_ +#include_next + +#if BYTE_ORDER == LITTLE_ENDIAN +#undef _BIG_ENDIAN +#undef BIG_ENDIAN +#define BIG_ENDIAN 4321 +#endif + +#endif /* _SPL_SYS_ENDIAN_H_ */ diff --git a/include/os/freebsd/spl/sys/extdirent.h b/include/os/freebsd/spl/sys/extdirent.h new file mode 100644 index 00000000000..65ba11f345d --- /dev/null +++ b/include/os/freebsd/spl/sys/extdirent.h @@ -0,0 +1,73 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_EXTDIRENT_H +#define _SYS_EXTDIRENT_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +/* + * Extended file-system independent directory entry. This style of + * dirent provides additional informational flag bits for each + * directory entry. This dirent will be returned instead of the + * standard dirent if a VOP_READDIR() requests dirent flags via + * V_RDDIR_ENTFLAGS, and if the file system supports the flags. + */ +typedef struct edirent { + ino64_t ed_ino; /* "inode number" of entry */ + off64_t ed_off; /* offset of disk directory entry */ + uint32_t ed_eflags; /* per-entry flags */ + unsigned short ed_reclen; /* length of this record */ + char ed_name[1]; /* name of file */ +} edirent_t; + +#define EDIRENT_RECLEN(namelen) \ + ((offsetof(edirent_t, ed_name[0]) + 1 + (namelen) + 7) & ~ 7) +#define EDIRENT_NAMELEN(reclen) \ + ((reclen) - (offsetof(edirent_t, ed_name[0]))) + +/* + * Extended entry flags + * Extended entries include a bitfield of extra information + * regarding that entry. + */ +#define ED_CASE_CONFLICT 0x10 /* Disconsidering case, entry is not unique */ + +/* + * Extended flags accessor function + */ +#define ED_CASE_CONFLICTS(x) ((x)->ed_eflags & ED_CASE_CONFLICT) +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_EXTDIRENT_H */ diff --git a/include/os/freebsd/spl/sys/file.h b/include/os/freebsd/spl/sys/file.h new file mode 100644 index 00000000000..10a82c20485 --- /dev/null +++ b/include/os/freebsd/spl/sys/file.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_FILE_H_ +#define _OPENSOLARIS_SYS_FILE_H_ + +#include_next + +#define FKIOCTL 0x80000000 /* ioctl addresses are from kernel */ + +typedef struct file file_t; + +#include + +static __inline file_t * +getf_caps(int fd, cap_rights_t *rightsp) +{ + struct file *fp; + + if (fget(curthread, fd, rightsp, &fp) == 0) + return (fp); + return (NULL); +} + +#endif /* !_OPENSOLARIS_SYS_FILE_H_ */ diff --git a/include/os/freebsd/spl/sys/freebsd_rwlock.h b/include/os/freebsd/spl/sys/freebsd_rwlock.h new file mode 100644 index 00000000000..b760f8cf23d --- /dev/null +++ b/include/os/freebsd/spl/sys/freebsd_rwlock.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2013 EMC Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_FREEBSD_RWLOCK_H_ +#define _OPENSOLARIS_SYS_FREEBSD_RWLOCK_H_ + +#include_next + +#endif diff --git a/include/os/freebsd/spl/sys/inttypes.h b/include/os/freebsd/spl/sys/inttypes.h new file mode 100644 index 00000000000..651685d3047 --- /dev/null +++ b/include/os/freebsd/spl/sys/inttypes.h @@ -0,0 +1 @@ +/* do not delete */ diff --git a/include/os/freebsd/spl/sys/isa_defs.h b/include/os/freebsd/spl/sys/isa_defs.h new file mode 100644 index 00000000000..a9d1a4e1fd6 --- /dev/null +++ b/include/os/freebsd/spl/sys/isa_defs.h @@ -0,0 +1,688 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_ISA_DEFS_H +#define _SYS_ISA_DEFS_H + +/* + * This header file serves to group a set of well known defines and to + * set these for each instruction set architecture. These defines may + * be divided into two groups; characteristics of the processor and + * implementation choices for Solaris on a processor. + * + * Processor Characteristics: + * + * _LITTLE_ENDIAN / _BIG_ENDIAN: + * The natural byte order of the processor. A pointer to an int points + * to the least/most significant byte of that int. + * + * _STACK_GROWS_UPWARD / _STACK_GROWS_DOWNWARD: + * The processor specific direction of stack growth. A push onto the + * stack increases/decreases the stack pointer, so it stores data at + * successively higher/lower addresses. (Stackless machines ignored + * without regrets). + * + * _LONG_LONG_HTOL / _LONG_LONG_LTOH: + * A pointer to a long long points to the most/least significant long + * within that long long. + * + * _BIT_FIELDS_HTOL / _BIT_FIELDS_LTOH: + * The C compiler assigns bit fields from the high/low to the low/high end + * of an int (most to least significant vs. least to most significant). + * + * _IEEE_754: + * The processor (or supported implementations of the processor) + * supports the ieee-754 floating point standard. No other floating + * point standards are supported (or significant). Any other supported + * floating point formats are expected to be cased on the ISA processor + * symbol. + * + * _CHAR_IS_UNSIGNED / _CHAR_IS_SIGNED: + * The C Compiler implements objects of type `char' as `unsigned' or + * `signed' respectively. This is really an implementation choice of + * the compiler writer, but it is specified in the ABI and tends to + * be uniform across compilers for an instruction set architecture. + * Hence, it has the properties of a processor characteristic. + * + * _CHAR_ALIGNMENT / _SHORT_ALIGNMENT / _INT_ALIGNMENT / _LONG_ALIGNMENT / + * _LONG_LONG_ALIGNMENT / _DOUBLE_ALIGNMENT / _LONG_DOUBLE_ALIGNMENT / + * _POINTER_ALIGNMENT / _FLOAT_ALIGNMENT: + * The ABI defines alignment requirements of each of the primitive + * object types. Some, if not all, may be hardware requirements as + * well. The values are expressed in "byte-alignment" units. + * + * _MAX_ALIGNMENT: + * The most stringent alignment requirement as specified by the ABI. + * Equal to the maximum of all the above _XXX_ALIGNMENT values. + * + * _ALIGNMENT_REQUIRED: + * True or false (1 or 0) whether or not the hardware requires the ABI + * alignment. + * + * _LONG_LONG_ALIGNMENT_32 + * The 32-bit ABI supported by a 64-bit kernel may have different + * alignment requirements for primitive object types. The value of this + * identifier is expressed in "byte-alignment" units. + * + * _HAVE_CPUID_INSN + * This indicates that the architecture supports the 'cpuid' + * instruction as defined by Intel. (Intel allows other vendors + * to extend the instruction for their own purposes.) + * + * + * Implementation Choices: + * + * _ILP32 / _LP64: + * This specifies the compiler data type implementation as specified in + * the relevant ABI. The choice between these is strongly influenced + * by the underlying hardware, but is not absolutely tied to it. + * Currently only two data type models are supported: + * + * _ILP32: + * Int/Long/Pointer are 32 bits. This is the historical UNIX + * and Solaris implementation. Due to its historical standing, + * this is the default case. + * + * _LP64: + * Long/Pointer are 64 bits, Int is 32 bits. This is the chosen + * implementation for 64-bit ABIs such as SPARC V9. + * + * _I32LPx: + * A compilation environment where 'int' is 32-bit, and + * longs and pointers are simply the same size. + * + * In all cases, Char is 8 bits and Short is 16 bits. + * + * _SUNOS_VTOC_8 / _SUNOS_VTOC_16 / _SVR4_VTOC_16: + * This specifies the form of the disk VTOC (or label): + * + * _SUNOS_VTOC_8: + * This is a VTOC form which is upwardly compatible with the + * SunOS 4.x disk label and allows 8 partitions per disk. + * + * _SUNOS_VTOC_16: + * In this format the incore vtoc image matches the ondisk + * version. It allows 16 slices per disk, and is not + * compatible with the SunOS 4.x disk label. + * + * Note that these are not the only two VTOC forms possible and + * additional forms may be added. One possible form would be the + * SVr4 VTOC form. The symbol for that is reserved now, although + * it is not implemented. + * + * _SVR4_VTOC_16: + * This VTOC form is compatible with the System V Release 4 + * VTOC (as implemented on the SVr4 Intel and 3b ports) with + * 16 partitions per disk. + * + * + * _DMA_USES_PHYSADDR / _DMA_USES_VIRTADDR + * This describes the type of addresses used by system DMA: + * + * _DMA_USES_PHYSADDR: + * This type of DMA, used in the x86 implementation, + * requires physical addresses for DMA buffers. The 24-bit + * addresses used by some legacy boards is the source of the + * "low-memory" (<16MB) requirement for some devices using DMA. + * + * _DMA_USES_VIRTADDR: + * This method of DMA allows the use of virtual addresses for + * DMA transfers. + * + * _FIRMWARE_NEEDS_FDISK / _NO_FDISK_PRESENT + * This indicates the presence/absence of an fdisk table. + * + * _FIRMWARE_NEEDS_FDISK + * The fdisk table is required by system firmware. If present, + * it allows a disk to be subdivided into multiple fdisk + * partitions, each of which is equivalent to a separate, + * virtual disk. This enables the co-existence of multiple + * operating systems on a shared hard disk. + * + * _NO_FDISK_PRESENT + * If the fdisk table is absent, it is assumed that the entire + * media is allocated for a single operating system. + * + * _HAVE_TEM_FIRMWARE + * Defined if this architecture has the (fallback) option of + * using prom_* calls for doing I/O if a suitable kernel driver + * is not available to do it. + * + * _DONT_USE_1275_GENERIC_NAMES + * Controls whether or not device tree node names should + * comply with the IEEE 1275 "Generic Names" Recommended + * Practice. With _DONT_USE_GENERIC_NAMES, device-specific + * names identifying the particular device will be used. + * + * __i386_COMPAT + * This indicates whether the i386 ABI is supported as a *non-native* + * mode for the platform. When this symbol is defined: + * - 32-bit xstat-style system calls are enabled + * - 32-bit xmknod-style system calls are enabled + * - 32-bit system calls use i386 sizes -and- alignments + * + * Note that this is NOT defined for the i386 native environment! + * + * __x86 + * This is ONLY a synonym for defined(__i386) || defined(__amd64) + * which is useful only insofar as these two architectures share + * common attributes. Analogous to __sparc. + * + * _PSM_MODULES + * This indicates whether or not the implementation uses PSM + * modules for processor support, reading /etc/mach from inside + * the kernel to extract a list. + * + * _RTC_CONFIG + * This indicates whether or not the implementation uses /etc/rtc_config + * to configure the real-time clock in the kernel. + * + * _UNIX_KRTLD + * This indicates that the implementation uses a dynamically + * linked unix + krtld to form the core kernel image at boot + * time, or (in the absence of this symbol) a prelinked kernel image. + * + * _OBP + * This indicates the firmware interface is OBP. + * + * _SOFT_HOSTID + * This indicates that the implementation obtains the hostid + * from the file /etc/hostid, rather than from hardware. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * The following set of definitions characterize Solaris on AMD's + * 64-bit systems. + */ +#if defined(__x86_64) || defined(__amd64) + +#if !defined(__amd64) +#define __amd64 /* preferred guard */ +#endif + +#if !defined(__x86) +#define __x86 +#endif + +/* + * Define the appropriate "processor characteristics" + */ +#define _STACK_GROWS_DOWNWARD +#define _LONG_LONG_LTOH +#define _BIT_FIELDS_LTOH +#define _IEEE_754 +#define _CHAR_IS_SIGNED +#define _BOOL_ALIGNMENT 1 +#define _CHAR_ALIGNMENT 1 +#define _SHORT_ALIGNMENT 2 +#define _INT_ALIGNMENT 4 +#define _FLOAT_ALIGNMENT 4 +#define _FLOAT_COMPLEX_ALIGNMENT 4 +#define _LONG_ALIGNMENT 8 +#define _LONG_LONG_ALIGNMENT 8 +#define _DOUBLE_ALIGNMENT 8 +#define _DOUBLE_COMPLEX_ALIGNMENT 8 +#define _LONG_DOUBLE_ALIGNMENT 16 +#define _LONG_DOUBLE_COMPLEX_ALIGNMENT 16 +#define _POINTER_ALIGNMENT 8 +#define _MAX_ALIGNMENT 16 +#define _ALIGNMENT_REQUIRED 1 + +/* + * Different alignment constraints for the i386 ABI in compatibility mode + */ +#define _LONG_LONG_ALIGNMENT_32 4 + +/* + * Define the appropriate "implementation choices". + */ +#if !defined(_LP64) +#error "_LP64 not defined" +#endif +#if !defined(_I32LPx) +#define _I32LPx +#endif +#define _MULTI_DATAMODEL +#define _SUNOS_VTOC_16 +#define _DMA_USES_PHYSADDR +#define _FIRMWARE_NEEDS_FDISK +#define __i386_COMPAT +#define _PSM_MODULES +#define _RTC_CONFIG +#define _SOFT_HOSTID +#define _DONT_USE_1275_GENERIC_NAMES +#define _HAVE_CPUID_INSN + +/* + * The feature test macro __i386 is generic for all processors implementing + * the Intel 386 instruction set or a superset of it. Specifically, this + * includes all members of the 386, 486, and Pentium family of processors. + */ +#elif defined(__i386) || defined(__i386__) + +#if !defined(__i386) +#define __i386 +#endif + +#if !defined(__x86) +#define __x86 +#endif + +/* + * Define the appropriate "processor characteristics" + */ +#define _STACK_GROWS_DOWNWARD +#define _LONG_LONG_LTOH +#define _BIT_FIELDS_LTOH +#define _IEEE_754 +#define _CHAR_IS_SIGNED +#define _BOOL_ALIGNMENT 1 +#define _CHAR_ALIGNMENT 1 +#define _SHORT_ALIGNMENT 2 +#define _INT_ALIGNMENT 4 +#define _FLOAT_ALIGNMENT 4 +#define _FLOAT_COMPLEX_ALIGNMENT 4 +#define _LONG_ALIGNMENT 4 +#define _LONG_LONG_ALIGNMENT 4 +#define _DOUBLE_ALIGNMENT 4 +#define _DOUBLE_COMPLEX_ALIGNMENT 4 +#define _LONG_DOUBLE_ALIGNMENT 4 +#define _LONG_DOUBLE_COMPLEX_ALIGNMENT 4 +#define _POINTER_ALIGNMENT 4 +#define _MAX_ALIGNMENT 4 +#define _ALIGNMENT_REQUIRED 0 + +#define _LONG_LONG_ALIGNMENT_32 _LONG_LONG_ALIGNMENT + +/* + * Define the appropriate "implementation choices". + */ +#if !defined(_ILP32) +#define _ILP32 +#endif +#if !defined(_I32LPx) +#define _I32LPx +#endif +#define _SUNOS_VTOC_16 +#define _DMA_USES_PHYSADDR +#define _FIRMWARE_NEEDS_FDISK +#define _PSM_MODULES +#define _RTC_CONFIG +#define _SOFT_HOSTID +#define _DONT_USE_1275_GENERIC_NAMES +#define _HAVE_CPUID_INSN + +#elif defined(__aarch64__) + +/* + * Define the appropriate "processor characteristics" + */ +#define _STACK_GROWS_DOWNWARD +#define _LONG_LONG_LTOH +#define _BIT_FIELDS_LTOH +#define _IEEE_754 +#define _CHAR_IS_UNSIGNED +#define _BOOL_ALIGNMENT 1 +#define _CHAR_ALIGNMENT 1 +#define _SHORT_ALIGNMENT 2 +#define _INT_ALIGNMENT 4 +#define _FLOAT_ALIGNMENT 4 +#define _FLOAT_COMPLEX_ALIGNMENT 4 +#define _LONG_ALIGNMENT 8 +#define _LONG_LONG_ALIGNMENT 8 +#define _DOUBLE_ALIGNMENT 8 +#define _DOUBLE_COMPLEX_ALIGNMENT 8 +#define _LONG_DOUBLE_ALIGNMENT 16 +#define _LONG_DOUBLE_COMPLEX_ALIGNMENT 16 +#define _POINTER_ALIGNMENT 8 +#define _MAX_ALIGNMENT 16 +#define _ALIGNMENT_REQUIRED 1 + +#define _LONG_LONG_ALIGNMENT_32 _LONG_LONG_ALIGNMENT + +/* + * Define the appropriate "implementation choices" + */ +#if !defined(_LP64) +#error "_LP64 not defined" +#endif +#define _SUNOS_VTOC_16 +#define _DMA_USES_PHYSADDR +#define _FIRMWARE_NEEDS_FDISK +#define _PSM_MODULES +#define _RTC_CONFIG +#define _DONT_USE_1275_GENERIC_NAMES +#define _HAVE_CPUID_INSN + +#elif defined(__riscv) + +/* + * Define the appropriate "processor characteristics" + */ +#define _STACK_GROWS_DOWNWARD +#define _LONG_LONG_LTOH +#define _BIT_FIELDS_LTOH +#define _IEEE_754 +#define _CHAR_IS_UNSIGNED +#define _BOOL_ALIGNMENT 1 +#define _CHAR_ALIGNMENT 1 +#define _SHORT_ALIGNMENT 2 +#define _INT_ALIGNMENT 4 +#define _FLOAT_ALIGNMENT 4 +#define _FLOAT_COMPLEX_ALIGNMENT 4 +#define _LONG_ALIGNMENT 8 +#define _LONG_LONG_ALIGNMENT 8 +#define _DOUBLE_ALIGNMENT 8 +#define _DOUBLE_COMPLEX_ALIGNMENT 8 +#define _LONG_DOUBLE_ALIGNMENT 16 +#define _LONG_DOUBLE_COMPLEX_ALIGNMENT 16 +#define _POINTER_ALIGNMENT 8 +#define _MAX_ALIGNMENT 16 +#define _ALIGNMENT_REQUIRED 1 + +#define _LONG_LONG_ALIGNMENT_32 _LONG_LONG_ALIGNMENT + +/* + * Define the appropriate "implementation choices" + */ +#if !defined(_LP64) +#define _LP64 +#endif +#define _SUNOS_VTOC_16 +#define _DMA_USES_PHYSADDR +#define _FIRMWARE_NEEDS_FDISK +#define _PSM_MODULES +#define _RTC_CONFIG +#define _DONT_USE_1275_GENERIC_NAMES +#define _HAVE_CPUID_INSN + +#elif defined(__arm__) + +/* + * Define the appropriate "processor characteristics" + */ +#define _STACK_GROWS_DOWNWARD +#define _LONG_LONG_LTOH +#define _BIT_FIELDS_LTOH +#define _IEEE_754 +#define _CHAR_IS_SIGNED +#define _BOOL_ALIGNMENT 1 +#define _CHAR_ALIGNMENT 1 +#define _SHORT_ALIGNMENT 2 +#define _INT_ALIGNMENT 4 +#define _FLOAT_ALIGNMENT 4 +#define _FLOAT_COMPLEX_ALIGNMENT 4 +#define _LONG_ALIGNMENT 4 +#define _LONG_LONG_ALIGNMENT 4 +#define _DOUBLE_ALIGNMENT 4 +#define _DOUBLE_COMPLEX_ALIGNMENT 4 +#define _LONG_DOUBLE_ALIGNMENT 4 +#define _LONG_DOUBLE_COMPLEX_ALIGNMENT 4 +#define _POINTER_ALIGNMENT 4 +#define _MAX_ALIGNMENT 4 +#define _ALIGNMENT_REQUIRED 0 + +#define _LONG_LONG_ALIGNMENT_32 _LONG_LONG_ALIGNMENT + +/* + * Define the appropriate "implementation choices". + */ +#if !defined(_ILP32) +#define _ILP32 +#endif +#if !defined(_I32LPx) +#define _I32LPx +#endif +#define _SUNOS_VTOC_16 +#define _DMA_USES_PHYSADDR +#define _FIRMWARE_NEEDS_FDISK +#define _PSM_MODULES +#define _RTC_CONFIG +#define _DONT_USE_1275_GENERIC_NAMES +#define _HAVE_CPUID_INSN + +#elif defined(__mips__) + +/* + * Define the appropriate "processor characteristics" + */ +#define _STACK_GROWS_DOWNWARD +#define _LONG_LONG_LTOH +#define _BIT_FIELDS_LTOH +#define _IEEE_754 +#define _CHAR_IS_SIGNED +#define _BOOL_ALIGNMENT 1 +#define _CHAR_ALIGNMENT 1 +#define _SHORT_ALIGNMENT 2 +#define _INT_ALIGNMENT 4 +#define _FLOAT_ALIGNMENT 4 +#define _FLOAT_COMPLEX_ALIGNMENT 4 +#if defined(__mips_n64) +#define _LONG_ALIGNMENT 8 +#define _LONG_LONG_ALIGNMENT 8 +#define _DOUBLE_ALIGNMENT 8 +#define _DOUBLE_COMPLEX_ALIGNMENT 8 +#define _LONG_DOUBLE_ALIGNMENT 8 +#define _LONG_DOUBLE_COMPLEX_ALIGNMENT 8 +#define _POINTER_ALIGNMENT 8 +#define _MAX_ALIGNMENT 8 +#define _ALIGNMENT_REQUIRED 0 + +#define _LONG_LONG_ALIGNMENT_32 _INT_ALIGNMENT +/* + * Define the appropriate "implementation choices". + */ +#if !defined(_LP64) +#error "_LP64 not defined" +#endif +#else +#define _LONG_ALIGNMENT 4 +#define _LONG_LONG_ALIGNMENT 4 +#define _DOUBLE_ALIGNMENT 4 +#define _DOUBLE_COMPLEX_ALIGNMENT 4 +#define _LONG_DOUBLE_ALIGNMENT 4 +#define _LONG_DOUBLE_COMPLEX_ALIGNMENT 4 +#define _POINTER_ALIGNMENT 4 +#define _MAX_ALIGNMENT 4 +#define _ALIGNMENT_REQUIRED 0 + +#define _LONG_LONG_ALIGNMENT_32 _LONG_LONG_ALIGNMENT + +/* + * Define the appropriate "implementation choices". + */ +#if !defined(_ILP32) +#define _ILP32 +#endif +#if !defined(_I32LPx) +#define _I32LPx +#endif +#endif +#define _SUNOS_VTOC_16 +#define _DMA_USES_PHYSADDR +#define _FIRMWARE_NEEDS_FDISK +#define _PSM_MODULES +#define _RTC_CONFIG +#define _DONT_USE_1275_GENERIC_NAMES +#define _HAVE_CPUID_INSN + +#elif defined(__powerpc__) + +#if defined(__BIG_ENDIAN__) +#define _BIT_FIELDS_HTOL +#else +#define _BIT_FIELDS_LTOH +#endif + +/* + * The following set of definitions characterize the Solaris on SPARC systems. + * + * The symbol __sparc indicates any of the SPARC family of processor + * architectures. This includes SPARC V7, SPARC V8 and SPARC V9. + * + * The symbol __sparcv8 indicates the 32-bit SPARC V8 architecture as defined + * by Version 8 of the SPARC Architecture Manual. (SPARC V7 is close enough + * to SPARC V8 for the former to be subsumed into the latter definition.) + * + * The symbol __sparcv9 indicates the 64-bit SPARC V9 architecture as defined + * by Version 9 of the SPARC Architecture Manual. + * + * The symbols __sparcv8 and __sparcv9 are mutually exclusive, and are only + * relevant when the symbol __sparc is defined. + */ +/* + * XXX Due to the existence of 5110166, "defined(__sparcv9)" needs to be added + * to support backwards builds. This workaround should be removed in s10_71. + */ +#elif defined(__sparc) || defined(__sparcv9) || defined(__sparc__) +#if !defined(__sparc) +#define __sparc +#endif + +/* + * You can be 32-bit or 64-bit, but not both at the same time. + */ +#if defined(__sparcv8) && defined(__sparcv9) +#error "SPARC Versions 8 and 9 are mutually exclusive choices" +#endif + +/* + * Existing compilers do not set __sparcv8. Years will transpire before + * the compilers can be depended on to set the feature test macro. In + * the interim, we'll set it here on the basis of historical behaviour; + * if you haven't asked for SPARC V9, then you must've meant SPARC V8. + */ +#if !defined(__sparcv9) && !defined(__sparcv8) +#define __sparcv8 +#endif + +/* + * Define the appropriate "processor characteristics" shared between + * all Solaris on SPARC systems. + */ +#define _STACK_GROWS_DOWNWARD +#define _LONG_LONG_HTOL +#define _BIT_FIELDS_HTOL +#define _IEEE_754 +#define _CHAR_IS_SIGNED +#define _BOOL_ALIGNMENT 1 +#define _CHAR_ALIGNMENT 1 +#define _SHORT_ALIGNMENT 2 +#define _INT_ALIGNMENT 4 +#define _FLOAT_ALIGNMENT 4 +#define _FLOAT_COMPLEX_ALIGNMENT 4 +#define _LONG_LONG_ALIGNMENT 8 +#define _DOUBLE_ALIGNMENT 8 +#define _DOUBLE_COMPLEX_ALIGNMENT 8 +#define _ALIGNMENT_REQUIRED 1 + +/* + * Define the appropriate "implementation choices" shared between versions. + */ +#define _SUNOS_VTOC_8 +#define _DMA_USES_VIRTADDR +#define _NO_FDISK_PRESENT +#define _HAVE_TEM_FIRMWARE +#define _OBP + +/* + * The following set of definitions characterize the implementation of + * 32-bit Solaris on SPARC V8 systems. + */ +#if defined(__sparcv8) + +/* + * Define the appropriate "processor characteristics" + */ +#define _LONG_ALIGNMENT 4 +#define _LONG_DOUBLE_ALIGNMENT 8 +#define _LONG_DOUBLE_COMPLEX_ALIGNMENT 8 +#define _POINTER_ALIGNMENT 4 +#define _MAX_ALIGNMENT 8 + +#define _LONG_LONG_ALIGNMENT_32 _LONG_LONG_ALIGNMENT + +/* + * Define the appropriate "implementation choices" + */ +#define _ILP32 +#if !defined(_I32LPx) +#define _I32LPx +#endif + +/* + * The following set of definitions characterize the implementation of + * 64-bit Solaris on SPARC V9 systems. + */ +#elif defined(__sparcv9) + +/* + * Define the appropriate "processor characteristics" + */ +#define _LONG_ALIGNMENT 8 +#define _LONG_DOUBLE_ALIGNMENT 16 +#define _LONG_DOUBLE_COMPLEX_ALIGNMENT 16 +#define _POINTER_ALIGNMENT 8 +#define _MAX_ALIGNMENT 16 + +#define _LONG_LONG_ALIGNMENT_32 _LONG_LONG_ALIGNMENT + +/* + * Define the appropriate "implementation choices" + */ +#if !defined(_LP64) +#error "_LP64 not defined" +#endif +#if !defined(_I32LPx) +#define _I32LPx +#endif +#define _MULTI_DATAMODEL + +#else +#error "unknown SPARC version" +#endif + +/* + * #error is strictly ansi-C, but works as well as anything for K&R systems. + */ +#else +#error "ISA not supported" +#endif + +#if defined(_ILP32) && defined(_LP64) +#error "Both _ILP32 and _LP64 are defined" +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_ISA_DEFS_H */ diff --git a/include/os/freebsd/spl/sys/kmem.h b/include/os/freebsd/spl/sys/kmem.h new file mode 100644 index 00000000000..cb61603d7de --- /dev/null +++ b/include/os/freebsd/spl/sys/kmem.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_KMEM_H_ +#define _OPENSOLARIS_SYS_KMEM_H_ + +#include +#include +#include +#include + +#include +#include +#include + +MALLOC_DECLARE(M_SOLARIS); + +#define POINTER_IS_VALID(p) (!((uintptr_t)(p) & 0x3)) +#define POINTER_INVALIDATE(pp) (*(pp) = (void *)((uintptr_t)(*(pp)) | 0x1)) + +#define KM_SLEEP M_WAITOK +#define KM_PUSHPAGE M_WAITOK +#define KM_NOSLEEP M_NOWAIT +#define KM_NODEBUG M_NODUMP +#define KM_NORMALPRI 0 +#define KMC_NODEBUG UMA_ZONE_NODUMP +#define KMC_NOTOUCH 0 + +typedef struct vmem vmem_t; + +extern char *kmem_asprintf(const char *, ...); +extern char *kmem_vasprintf(const char *fmt, va_list ap); + +typedef struct kmem_cache { + char kc_name[32]; +#if !defined(KMEM_DEBUG) + uma_zone_t kc_zone; +#else + size_t kc_size; +#endif + int (*kc_constructor)(void *, void *, int); + void (*kc_destructor)(void *, void *); + void *kc_private; +} kmem_cache_t; + +extern uint64_t spl_kmem_cache_inuse(kmem_cache_t *cache); +extern uint64_t spl_kmem_cache_entry_size(kmem_cache_t *cache); + +void *zfs_kmem_alloc(size_t size, int kmflags); +void zfs_kmem_free(void *buf, size_t size); +uint64_t kmem_size(void); +kmem_cache_t *kmem_cache_create(char *name, size_t bufsize, size_t align, + int (*constructor)(void *, void *, int), void (*destructor)(void *, void *), + void (*reclaim)(void *) __unused, void *private, vmem_t *vmp, int cflags); +void kmem_cache_destroy(kmem_cache_t *cache); +void *kmem_cache_alloc(kmem_cache_t *cache, int flags); +void kmem_cache_free(kmem_cache_t *cache, void *buf); +boolean_t kmem_cache_reap_active(void); +void kmem_cache_reap_soon(kmem_cache_t *); +void kmem_reap(void); +int kmem_debugging(void); +void *calloc(size_t n, size_t s); + + +#define kmem_cache_reap_now kmem_cache_reap_soon +#define freemem vm_free_count() +#define minfree vm_cnt.v_free_min +#define heap_arena kernel_arena +#define zio_arena NULL +#define kmem_alloc(size, kmflags) zfs_kmem_alloc((size), (kmflags)) +#define kmem_zalloc(size, kmflags) \ + zfs_kmem_alloc((size), (kmflags) | M_ZERO) +#define kmem_free(buf, size) zfs_kmem_free((buf), (size)) +#define vmem_qcache_reap(ptr) ((void)0) + + +#endif /* _OPENSOLARIS_SYS_KMEM_H_ */ diff --git a/include/os/freebsd/spl/sys/kmem_cache.h b/include/os/freebsd/spl/sys/kmem_cache.h new file mode 100644 index 00000000000..d8e0349e4ca --- /dev/null +++ b/include/os/freebsd/spl/sys/kmem_cache.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + +#ifndef _SPL_KMEM_CACHE_H +#define _SPL_KMEM_CACHE_H + +#include + +/* kmem move callback return values */ +typedef enum kmem_cbrc { + KMEM_CBRC_YES = 0, /* Object moved */ + KMEM_CBRC_NO = 1, /* Object not moved */ + KMEM_CBRC_LATER = 2, /* Object not moved, try again later */ + KMEM_CBRC_DONT_NEED = 3, /* Neither object is needed */ + KMEM_CBRC_DONT_KNOW = 4, /* Object unknown */ +} kmem_cbrc_t; + +extern void spl_kmem_cache_set_move(kmem_cache_t *, + kmem_cbrc_t (*)(void *, void *, size_t, void *)); + +#define kmem_cache_set_move(skc, move) spl_kmem_cache_set_move(skc, move) + +#endif diff --git a/include/os/freebsd/spl/sys/kstat.h b/include/os/freebsd/spl/sys/kstat.h new file mode 100644 index 00000000000..740f24b7072 --- /dev/null +++ b/include/os/freebsd/spl/sys/kstat.h @@ -0,0 +1,206 @@ +/* + * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. + * Copyright (C) 2007 The Regents of the University of California. + * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). + * Written by Brian Behlendorf . + * UCRL-CODE-235197 + * + * This file is part of the SPL, Solaris Porting Layer. + * For details, see . + * + * The SPL is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * The SPL is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with the SPL. If not, see . + */ + +#ifndef _SPL_KSTAT_H +#define _SPL_KSTAT_H +#include +struct list_head {}; +#include +#include + +#define KSTAT_STRLEN 255 +#define KSTAT_RAW_MAX (128*1024) + +/* + * For reference valid classes are: + * disk, tape, net, controller, vm, kvm, hat, streams, kstat, misc + */ + +#define KSTAT_TYPE_RAW 0 /* can be anything; ks_ndata >= 1 */ +#define KSTAT_TYPE_NAMED 1 /* name/value pair; ks_ndata >= 1 */ +#define KSTAT_TYPE_INTR 2 /* interrupt stats; ks_ndata == 1 */ +#define KSTAT_TYPE_IO 3 /* I/O stats; ks_ndata == 1 */ +#define KSTAT_TYPE_TIMER 4 /* event timer; ks_ndata >= 1 */ +#define KSTAT_NUM_TYPES 5 + +#define KSTAT_DATA_CHAR 0 +#define KSTAT_DATA_INT32 1 +#define KSTAT_DATA_UINT32 2 +#define KSTAT_DATA_INT64 3 +#define KSTAT_DATA_UINT64 4 +#define KSTAT_DATA_LONG 5 +#define KSTAT_DATA_ULONG 6 +#define KSTAT_DATA_STRING 7 +#define KSTAT_NUM_DATAS 8 + +#define KSTAT_INTR_HARD 0 +#define KSTAT_INTR_SOFT 1 +#define KSTAT_INTR_WATCHDOG 2 +#define KSTAT_INTR_SPURIOUS 3 +#define KSTAT_INTR_MULTSVC 4 +#define KSTAT_NUM_INTRS 5 + +#define KSTAT_FLAG_VIRTUAL 0x01 +#define KSTAT_FLAG_VAR_SIZE 0x02 +#define KSTAT_FLAG_WRITABLE 0x04 +#define KSTAT_FLAG_PERSISTENT 0x08 +#define KSTAT_FLAG_DORMANT 0x10 +#define KSTAT_FLAG_INVALID 0x20 +#define KSTAT_FLAG_LONGSTRINGS 0x40 +#define KSTAT_FLAG_NO_HEADERS 0x80 + +#define KS_MAGIC 0x9d9d9d9d + +/* Dynamic updates */ +#define KSTAT_READ 0 +#define KSTAT_WRITE 1 + +struct kstat_s; +typedef struct kstat_s kstat_t; + +typedef int kid_t; /* unique kstat id */ +typedef int kstat_update_t(struct kstat_s *, int); /* dynamic update cb */ + +typedef struct kstat_module { + char ksm_name[KSTAT_STRLEN+1]; /* module name */ + struct list_head ksm_module_list; /* module linkage */ + struct list_head ksm_kstat_list; /* list of kstat entries */ + struct proc_dir_entry *ksm_proc; /* proc entry */ +} kstat_module_t; + +typedef struct kstat_raw_ops { + int (*headers)(char *buf, size_t size); + int (*data)(char *buf, size_t size, void *data); + void *(*addr)(kstat_t *ksp, loff_t index); +} kstat_raw_ops_t; + +struct kstat_s { + int ks_magic; /* magic value */ + kid_t ks_kid; /* unique kstat ID */ + hrtime_t ks_crtime; /* creation time */ + hrtime_t ks_snaptime; /* last access time */ + char ks_module[KSTAT_STRLEN+1]; /* provider module name */ + int ks_instance; /* provider module instance */ + char ks_name[KSTAT_STRLEN+1]; /* kstat name */ + char ks_class[KSTAT_STRLEN+1]; /* kstat class */ + uchar_t ks_type; /* kstat data type */ + uchar_t ks_flags; /* kstat flags */ + void *ks_data; /* kstat type-specific data */ + uint_t ks_ndata; /* # of data records */ + size_t ks_data_size; /* size of kstat data section */ + kstat_update_t *ks_update; /* dynamic updates */ + void *ks_private; /* private data */ + kmutex_t ks_private_lock; /* kstat private data lock */ + kmutex_t *ks_lock; /* kstat data lock */ + struct list_head ks_list; /* kstat linkage */ + kstat_module_t *ks_owner; /* kstat module linkage */ + kstat_raw_ops_t ks_raw_ops; /* ops table for raw type */ + char *ks_raw_buf; /* buf used for raw ops */ + size_t ks_raw_bufsize; /* size of raw ops buffer */ + struct sysctl_ctx_list ks_sysctl_ctx; + struct sysctl_oid *ks_sysctl_root; + +}; + +typedef struct kstat_named_s { + char name[KSTAT_STRLEN]; /* name of counter */ + uchar_t data_type; /* data type */ + union { + char c[16]; /* 128-bit int */ + int32_t i32; /* 32-bit signed int */ + uint32_t ui32; /* 32-bit unsigned int */ + int64_t i64; /* 64-bit signed int */ + uint64_t ui64; /* 64-bit unsigned int */ + long l; /* native signed long */ + ulong_t ul; /* native unsigned long */ + struct { + union { + char *ptr; /* NULL-term string */ + char __pad[8]; /* 64-bit padding */ + } addr; + uint32_t len; /* # bytes for strlen + '\0' */ + } string; + } value; +} kstat_named_t; + +#define KSTAT_NAMED_STR_PTR(knptr) ((knptr)->value.string.addr.ptr) +#define KSTAT_NAMED_STR_BUFLEN(knptr) ((knptr)->value.string.len) + +typedef struct kstat_intr { + uint_t intrs[KSTAT_NUM_INTRS]; +} kstat_intr_t; + +typedef struct kstat_io { + u_longlong_t nread; /* number of bytes read */ + u_longlong_t nwritten; /* number of bytes written */ + uint_t reads; /* number of read operations */ + uint_t writes; /* number of write operations */ + hrtime_t wtime; /* cumulative wait (pre-service) time */ + hrtime_t wlentime; /* cumulative wait len*time product */ + hrtime_t wlastupdate; /* last time wait queue changed */ + hrtime_t rtime; /* cumulative run (service) time */ + hrtime_t rlentime; /* cumulative run length*time product */ + hrtime_t rlastupdate; /* last time run queue changed */ + uint_t wcnt; /* count of elements in wait state */ + uint_t rcnt; /* count of elements in run state */ +} kstat_io_t; + +typedef struct kstat_timer { + char name[KSTAT_STRLEN+1]; /* event name */ + u_longlong_t num_events; /* number of events */ + hrtime_t elapsed_time; /* cumulative elapsed time */ + hrtime_t min_time; /* shortest event duration */ + hrtime_t max_time; /* longest event duration */ + hrtime_t start_time; /* previous event start time */ + hrtime_t stop_time; /* previous event stop time */ +} kstat_timer_t; + +int spl_kstat_init(void); +void spl_kstat_fini(void); + +extern void __kstat_set_raw_ops(kstat_t *ksp, + int (*headers)(char *buf, size_t size), + int (*data)(char *buf, size_t size, void *data), + void* (*addr)(kstat_t *ksp, loff_t index)); + +extern kstat_t *__kstat_create(const char *ks_module, int ks_instance, + const char *ks_name, const char *ks_class, uchar_t ks_type, + uint_t ks_ndata, uchar_t ks_flags); + +extern void __kstat_install(kstat_t *ksp); +extern void __kstat_delete(kstat_t *ksp); +extern void kstat_waitq_enter(kstat_io_t *); +extern void kstat_waitq_exit(kstat_io_t *); +extern void kstat_runq_enter(kstat_io_t *); +extern void kstat_runq_exit(kstat_io_t *); + +#define kstat_set_raw_ops(k, h, d, a) \ + __kstat_set_raw_ops(k, h, d, a) +#define kstat_create(m, i, n, c, t, s, f) \ + __kstat_create(m, i, n, c, t, s, f) + +#define kstat_install(k) __kstat_install(k) +#define kstat_delete(k) __kstat_delete(k) + +#endif /* _SPL_KSTAT_H */ diff --git a/include/os/freebsd/spl/sys/list.h b/include/os/freebsd/spl/sys/list.h new file mode 100644 index 00000000000..8339b6226d1 --- /dev/null +++ b/include/os/freebsd/spl/sys/list.h @@ -0,0 +1,67 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_LIST_H +#define _SYS_LIST_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct list_node list_node_t; +typedef struct list list_t; + +void list_create(list_t *, size_t, size_t); +void list_destroy(list_t *); + +void list_insert_after(list_t *, void *, void *); +void list_insert_before(list_t *, void *, void *); +void list_insert_head(list_t *, void *); +void list_insert_tail(list_t *, void *); +void list_remove(list_t *, void *); +void *list_remove_head(list_t *); +void *list_remove_tail(list_t *); +void list_move_tail(list_t *, list_t *); + +void *list_head(list_t *); +void *list_tail(list_t *); +void *list_next(list_t *, void *); +void *list_prev(list_t *, void *); +int list_is_empty(list_t *); + +void list_link_init(list_node_t *); +void list_link_replace(list_node_t *, list_node_t *); + +int list_link_active(list_node_t *); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_LIST_H */ diff --git a/include/os/freebsd/spl/sys/list_impl.h b/include/os/freebsd/spl/sys/list_impl.h new file mode 100644 index 00000000000..9c42f883202 --- /dev/null +++ b/include/os/freebsd/spl/sys/list_impl.h @@ -0,0 +1,53 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2003 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_LIST_IMPL_H +#define _SYS_LIST_IMPL_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct list_node { + struct list_node *list_next; + struct list_node *list_prev; +}; + +struct list { + size_t list_size; + size_t list_offset; + struct list_node list_head; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_LIST_IMPL_H */ diff --git a/include/os/freebsd/spl/sys/lock.h b/include/os/freebsd/spl/sys/lock.h new file mode 100644 index 00000000000..7d5dc26abc7 --- /dev/null +++ b/include/os/freebsd/spl/sys/lock.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_LOCK_H_ +#define _OPENSOLARIS_SYS_LOCK_H_ + +#include_next + +#define LO_ALLMASK (LO_INITIALIZED | LO_WITNESS | LO_QUIET | \ + LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE | \ + LO_DUPOK | LO_CLASSMASK | LO_NOPROFILE) +#define LO_EXPECTED (LO_INITIALIZED | LO_WITNESS | LO_RECURSABLE | \ + LO_SLEEPABLE | LO_UPGRADABLE | LO_DUPOK | (2 << LO_CLASSSHIFT)) + +#endif /* _OPENSOLARIS_SYS_LOCK_H_ */ diff --git a/include/os/freebsd/spl/sys/misc.h b/include/os/freebsd/spl/sys/misc.h new file mode 100644 index 00000000000..e39bb07b2f4 --- /dev/null +++ b/include/os/freebsd/spl/sys/misc.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_MISC_H_ +#define _OPENSOLARIS_SYS_MISC_H_ + +#include + +#define MAXUID UID_MAX + +#define _ACL_ACLENT_ENABLED 0x1 +#define _ACL_ACE_ENABLED 0x2 + +#define _FIOFFS (INT_MIN) +#define _FIOGDIO (INT_MIN+1) +#define _FIOSDIO (INT_MIN+2) + +#define _FIO_SEEK_DATA FIOSEEKDATA +#define _FIO_SEEK_HOLE FIOSEEKHOLE + +struct opensolaris_utsname { + char *sysname; + char *nodename; + char *release; + char version[32]; + char *machine; +}; + +extern char hw_serial[11]; + +#endif /* _OPENSOLARIS_SYS_MISC_H_ */ diff --git a/include/os/freebsd/spl/sys/mod_os.h b/include/os/freebsd/spl/sys/mod_os.h new file mode 100644 index 00000000000..d81d927fd6e --- /dev/null +++ b/include/os/freebsd/spl/sys/mod_os.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SPL_MOD_H +#define _SPL_MOD_H + +#include + +#define ZFS_MODULE_DESCRIPTION(s) +#define ZFS_MODULE_AUTHOR(s) +#define ZFS_MODULE_LICENSE(s) +#define ZFS_MODULE_VERSION(s) + +#define EXPORT_SYMBOL(x) +#define module_param(a, b, c) +#define MODULE_PARM_DESC(a, b) + +#define ZMOD_RW CTLFLAG_RWTUN +#define ZMOD_RD CTLFLAG_RDTUN + +/* BEGIN CSTYLED */ +#define ZFS_MODULE_PARAM(scope_prefix, name_prefix, name, type, perm, desc) \ + SYSCTL_DECL(_vfs_ ## scope_prefix); \ + SYSCTL_##type(_vfs_ ## scope_prefix, OID_AUTO, name, perm, &name_prefix ## name, 0, desc) + +#define ZFS_MODULE_PARAM_ARGS SYSCTL_HANDLER_ARGS + +#define ZFS_MODULE_PARAM_CALL_IMPL(parent, name, perm, args, desc) \ + SYSCTL_DECL(parent); \ + SYSCTL_PROC(parent, OID_AUTO, name, perm | args, desc) + +#define ZFS_MODULE_PARAM_CALL(scope_prefix, name_prefix, name, func, _, perm, desc) \ + ZFS_MODULE_PARAM_CALL_IMPL(_vfs_ ## scope_prefix, name, perm, func ## _args(name_prefix ## name), desc) + +#define param_set_arc_long_args(var) \ + CTLTYPE_ULONG, &var, 0, param_set_arc_long, "LU" + +#define param_set_arc_int_args(var) \ + CTLTYPE_INT, &var, 0, param_set_arc_int, "I" + +#define param_set_deadman_failmode_args(var) \ + CTLTYPE_STRING, NULL, 0, param_set_deadman_failmode, "A" + +#define param_set_deadman_synctime_args(var) \ + CTLTYPE_ULONG, NULL, 0, param_set_deadman_synctime, "LU" + +#define param_set_deadman_ziotime_args(var) \ + CTLTYPE_ULONG, NULL, 0, param_set_deadman_ziotime, "LU" + +#define param_set_slop_shift_args(var) \ + CTLTYPE_INT, &var, 0, param_set_slop_shift, "I" + +#include +#define module_init(fn) \ +static void \ +wrap_ ## fn(void *dummy __unused) \ +{ \ + fn(); \ +} \ +SYSINIT(zfs_ ## fn, SI_SUB_LAST, SI_ORDER_FIRST, wrap_ ## fn, NULL) + + +#define module_exit(fn) \ +static void \ +wrap_ ## fn(void *dummy __unused) \ +{ \ + fn(); \ +} \ +SYSUNINIT(zfs_ ## fn, SI_SUB_LAST, SI_ORDER_FIRST, wrap_ ## fn, NULL) +/* END CSTYLED */ + +#endif /* SPL_MOD_H */ diff --git a/include/os/freebsd/spl/sys/mode.h b/include/os/freebsd/spl/sys/mode.h new file mode 100644 index 00000000000..651685d3047 --- /dev/null +++ b/include/os/freebsd/spl/sys/mode.h @@ -0,0 +1 @@ +/* do not delete */ diff --git a/include/os/freebsd/spl/sys/mount.h b/include/os/freebsd/spl/sys/mount.h new file mode 100644 index 00000000000..4732d283bb9 --- /dev/null +++ b/include/os/freebsd/spl/sys/mount.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_MOUNT_H_ +#define _OPENSOLARIS_SYS_MOUNT_H_ + +#include +#include_next +#include + +#define MS_FORCE MNT_FORCE +#define MS_REMOUNT MNT_UPDATE + +typedef struct fid fid_t; + +#endif /* !_OPENSOLARIS_SYS_MOUNT_H_ */ diff --git a/include/os/freebsd/spl/sys/mutex.h b/include/os/freebsd/spl/sys/mutex.h new file mode 100644 index 00000000000..ef556872a2b --- /dev/null +++ b/include/os/freebsd/spl/sys/mutex.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_MUTEX_H_ +#define _OPENSOLARIS_SYS_MUTEX_H_ + +typedef struct sx kmutex_t; + +#include +#include +#include_next +#include_next +#include +#include + +typedef enum { + MUTEX_DEFAULT = 6 /* kernel default mutex */ +} kmutex_type_t; + +#define MUTEX_HELD(x) (mutex_owned(x)) +#define MUTEX_NOT_HELD(x) (!mutex_owned(x) || panicstr) + +#ifndef OPENSOLARIS_WITNESS +#define MUTEX_FLAGS (SX_DUPOK | SX_NEW | SX_NOWITNESS) +#else +#define MUTEX_FLAGS (SX_DUPOK | SX_NEW) +#endif + +#define mutex_init(lock, desc, type, arg) do { \ + const char *_name; \ + ASSERT((type) == 0 || (type) == MUTEX_DEFAULT); \ + KASSERT(((lock)->lock_object.lo_flags & LO_ALLMASK) != \ + LO_EXPECTED, ("lock %s already initialized", #lock)); \ + for (_name = #lock; *_name != '\0'; _name++) { \ + if (*_name >= 'a' && *_name <= 'z') \ + break; \ + } \ + if (*_name == '\0') \ + _name = #lock; \ + sx_init_flags((lock), _name, MUTEX_FLAGS); \ +} while (0) +#define mutex_destroy(lock) sx_destroy(lock) +#define mutex_enter(lock) sx_xlock(lock) +#define mutex_enter_nested(lock, type) sx_xlock(lock) +#define mutex_tryenter(lock) sx_try_xlock(lock) +#define mutex_exit(lock) sx_xunlock(lock) +#define mutex_owned(lock) sx_xlocked(lock) +#define mutex_owner(lock) sx_xholder(lock) +#endif /* _OPENSOLARIS_SYS_MUTEX_H_ */ diff --git a/include/os/freebsd/spl/sys/param.h b/include/os/freebsd/spl/sys/param.h new file mode 100644 index 00000000000..01392dd6720 --- /dev/null +++ b/include/os/freebsd/spl/sys/param.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2007 John Birrell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef _COMPAT_OPENSOLARIS_SYS_PARAM_H_ +#define _COMPAT_OPENSOLARIS_SYS_PARAM_H_ + +#include_next + +#define PAGESIZE PAGE_SIZE +#define ptob(x) ((uint64_t)(x) << PAGE_SHIFT) + +#endif diff --git a/include/os/freebsd/spl/sys/policy.h b/include/os/freebsd/spl/sys/policy.h new file mode 100644 index 00000000000..6637202025e --- /dev/null +++ b/include/os/freebsd/spl/sys/policy.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $ $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_POLICY_H_ +#define _OPENSOLARIS_SYS_POLICY_H_ + +#include +#include +#include +struct mount; +struct vattr; + +int secpolicy_nfs(cred_t *cr); +int secpolicy_zfs(cred_t *crd); +int secpolicy_sys_config(cred_t *cr, int checkonly); +int secpolicy_zinject(cred_t *cr); +int secpolicy_fs_unmount(cred_t *cr, struct mount *vfsp); +int secpolicy_basic_link(vnode_t *vp, cred_t *cr); +int secpolicy_vnode_owner(vnode_t *vp, cred_t *cr, uid_t owner); +int secpolicy_vnode_chown(vnode_t *vp, cred_t *cr, uid_t owner); +int secpolicy_vnode_stky_modify(cred_t *cr); +int secpolicy_vnode_remove(vnode_t *vp, cred_t *cr); +int secpolicy_vnode_access(cred_t *cr, vnode_t *vp, uid_t owner, + accmode_t accmode); +int secpolicy_vnode_access2(cred_t *cr, vnode_t *vp, uid_t owner, + accmode_t curmode, accmode_t wantmode); +int secpolicy_vnode_any_access(cred_t *cr, vnode_t *vp, uid_t owner); +int secpolicy_vnode_setdac(vnode_t *vp, cred_t *cr, uid_t owner); +int secpolicy_vnode_setattr(cred_t *cr, vnode_t *vp, struct vattr *vap, + const struct vattr *ovap, int flags, + int unlocked_access(void *, int, cred_t *), void *node); +int secpolicy_vnode_create_gid(cred_t *cr); +int secpolicy_vnode_setids_setgids(vnode_t *vp, cred_t *cr, gid_t gid); +int secpolicy_vnode_setid_retain(vnode_t *vp, cred_t *cr, + boolean_t issuidroot); +void secpolicy_setid_clear(struct vattr *vap, vnode_t *vp, cred_t *cr); +int secpolicy_setid_setsticky_clear(vnode_t *vp, struct vattr *vap, + const struct vattr *ovap, cred_t *cr); +int secpolicy_fs_owner(struct mount *vfsp, cred_t *cr); +int secpolicy_fs_mount(cred_t *cr, vnode_t *mvp, struct mount *vfsp); +void secpolicy_fs_mount_clearopts(cred_t *cr, struct mount *vfsp); +int secpolicy_xvattr(vnode_t *vp, xvattr_t *xvap, uid_t owner, cred_t *cr, + vtype_t vtype); +int secpolicy_smb(cred_t *cr); + + +#if __FreeBSD_version >= 1300005 +#define spl_priv_check_cred(a, b) priv_check_cred((a), (b)) +#else +#define spl_priv_check_cred(a, b) priv_check_cred((a), (b), 0) +#endif +#endif /* _OPENSOLARIS_SYS_POLICY_H_ */ diff --git a/include/os/freebsd/spl/sys/proc.h b/include/os/freebsd/spl/sys/proc.h new file mode 100644 index 00000000000..fca833018ea --- /dev/null +++ b/include/os/freebsd/spl/sys/proc.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_PROC_H_ +#define _OPENSOLARIS_SYS_PROC_H_ + +#include +#include +#include_next +#include +#include +#include +#include +#include +#include +#include +#include + + +#define CPU curcpu +#define minclsyspri PRIBIO +#define defclsyspri minclsyspri +#define maxclsyspri PVM +#define max_ncpus (mp_maxid + 1) +#define boot_max_ncpus (mp_maxid + 1) + +#define TS_RUN 0 + +#define p0 proc0 + +#define t_tid td_tid + +typedef short pri_t; +typedef struct thread _kthread; +typedef struct thread kthread_t; +typedef struct thread *kthread_id_t; +typedef struct proc proc_t; + +extern struct proc *zfsproc; + +struct thread_wrap { + void *tw_arg; + void (*tw_proc)(void*); +}; + +static __inline void +solthread_wrapper(void *arg) +{ + struct thread_wrap *tw = arg; + + tw->tw_proc(tw->tw_arg); + free(tw, M_SOLARIS); + kthread_exit(); +} + +static __inline kthread_t * +do_thread_create(caddr_t stk, size_t stksize, void (*proc)(void *), void *arg, + size_t len, proc_t *pp, int state, pri_t pri) +{ + kthread_t *td = NULL; + int error; + struct thread_wrap *tw; + + /* + * Be sure there are no surprises. + */ + ASSERT(stk == NULL); + ASSERT(len == 0); + ASSERT(state == TS_RUN); + tw = malloc(sizeof (*tw), M_SOLARIS, M_WAITOK); + tw->tw_proc = proc; + tw->tw_arg = arg; + + error = kproc_kthread_add(solthread_wrapper, tw, &zfsproc, &td, + RFSTOPPED, stksize / PAGE_SIZE, "zfskern", "solthread %p", proc); + if (error == 0) { + thread_lock(td); + sched_prio(td, pri); + sched_add(td, SRQ_BORING); +#if __FreeBSD_version < 1300068 + thread_unlock(td); +#endif + } else { + free(tw, M_SOLARIS); + } + return (td); +} + +#define thread_create(stk, stksize, proc, arg, len, pp, state, pri) \ + do_thread_create(stk, stksize, proc, arg, len, pp, state, pri) +#define thread_exit() kthread_exit() + +int uread(proc_t *, void *, size_t, uintptr_t); +int uwrite(proc_t *, void *, size_t, uintptr_t); +#endif /* _OPENSOLARIS_SYS_PROC_H_ */ diff --git a/include/os/freebsd/spl/sys/processor.h b/include/os/freebsd/spl/sys/processor.h new file mode 100644 index 00000000000..53149840f21 --- /dev/null +++ b/include/os/freebsd/spl/sys/processor.h @@ -0,0 +1,63 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T + * All Rights Reserved + * + */ + +/* + * Copyright 2014 Garrett D'Amore + * + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_PROCESSOR_H +#define _SYS_PROCESSOR_H + +#include +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Definitions for p_online, processor_info & lgrp system calls. + */ + +/* + * Type for an lgrpid + */ +typedef uint16_t lgrpid_t; + +/* + * Type for processor name (CPU number). + */ +typedef int processorid_t; +typedef int chipid_t; + +#define getcpuid() curcpu + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_PROCESSOR_H */ diff --git a/include/os/freebsd/spl/sys/procfs_list.h b/include/os/freebsd/spl/sys/procfs_list.h new file mode 100644 index 00000000000..5d623c369c4 --- /dev/null +++ b/include/os/freebsd/spl/sys/procfs_list.h @@ -0,0 +1,64 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2018 by Delphix. All rights reserved. + */ + +#ifndef _SPL_PROCFS_LIST_H +#define _SPL_PROCFS_LIST_H + +#include +#include + + +/* + * procfs list manipulation + */ + +struct seq_file { }; +void seq_printf(struct seq_file *m, const char *fmt, ...); + +typedef struct procfs_list { + void *pl_private; + kmutex_t pl_lock; + list_t pl_list; + uint64_t pl_next_id; + size_t pl_node_offset; +} procfs_list_t; + +typedef struct procfs_list_node { + list_node_t pln_link; + uint64_t pln_id; +} procfs_list_node_t; + +void procfs_list_install(const char *module, + const char *name, + mode_t mode, + procfs_list_t *procfs_list, + int (*show)(struct seq_file *f, void *p), + int (*show_header)(struct seq_file *f), + int (*clear)(procfs_list_t *procfs_list), + size_t procfs_list_node_off); +void procfs_list_uninstall(procfs_list_t *procfs_list); +void procfs_list_destroy(procfs_list_t *procfs_list); +void procfs_list_add(procfs_list_t *procfs_list, void *p); + +#endif /* _SPL_PROCFS_LIST_H */ diff --git a/include/os/freebsd/spl/sys/random.h b/include/os/freebsd/spl/sys/random.h new file mode 100644 index 00000000000..b3c9115f530 --- /dev/null +++ b/include/os/freebsd/spl/sys/random.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_RANDOM_H_ +#define _OPENSOLARIS_SYS_RANDOM_H_ + +#include_next + +static inline int +random_get_bytes(uint8_t *p, size_t s) +{ + arc4rand(p, (int)s, 0); + return (0); +} + +static inline int +random_get_pseudo_bytes(uint8_t *p, size_t s) +{ + arc4rand(p, (int)s, 0); + return (0); +} + +#endif /* !_OPENSOLARIS_SYS_RANDOM_H_ */ diff --git a/include/os/freebsd/spl/sys/rwlock.h b/include/os/freebsd/spl/sys/rwlock.h new file mode 100644 index 00000000000..10107a9bee8 --- /dev/null +++ b/include/os/freebsd/spl/sys/rwlock.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_RWLOCK_H_ +#define _OPENSOLARIS_SYS_RWLOCK_H_ + +#include +#include +#include +#include + +typedef enum { + RW_DEFAULT = 4 /* kernel default rwlock */ +} krw_type_t; + + +typedef enum { + RW_NONE = 0, + RW_WRITER = 1, + RW_READER = 2 +} krw_t; + +typedef struct sx krwlock_t; + +#ifndef OPENSOLARIS_WITNESS +#define RW_FLAGS (SX_DUPOK | SX_NOWITNESS) +#else +#define RW_FLAGS (SX_DUPOK) +#endif + +#define RW_READ_HELD(x) (rw_read_held((x))) +#define RW_WRITE_HELD(x) (rw_write_held((x))) +#define RW_LOCK_HELD(x) (rw_lock_held((x))) +#define RW_ISWRITER(x) (rw_iswriter(x)) +/* BEGIN CSTYLED */ +#define rw_init(lock, desc, type, arg) do { \ + const char *_name; \ + ASSERT((type) == 0 || (type) == RW_DEFAULT); \ + KASSERT(((lock)->lock_object.lo_flags & LO_ALLMASK) != \ + LO_EXPECTED, ("lock %s already initialized", #lock)); \ + bzero((lock), sizeof(struct sx)); \ + for (_name = #lock; *_name != '\0'; _name++) { \ + if (*_name >= 'a' && *_name <= 'z') \ + break; \ + } \ + if (*_name == '\0') \ + _name = #lock; \ + sx_init_flags((lock), _name, RW_FLAGS); \ +} while (0) +#define rw_destroy(lock) sx_destroy(lock) +#define rw_enter(lock, how) do { \ + if ((how) == RW_READER) \ + sx_slock(lock); \ + else /* if ((how) == RW_WRITER) */ \ + sx_xlock(lock); \ + } while (0) + +#define rw_tryenter(lock, how) \ + ((how) == RW_READER ? sx_try_slock(lock) : sx_try_xlock(lock)) +#define rw_exit(lock) sx_unlock(lock) +#define rw_downgrade(lock) sx_downgrade(lock) +#define rw_tryupgrade(lock) sx_try_upgrade(lock) +#define rw_read_held(lock) \ + ((lock)->sx_lock != SX_LOCK_UNLOCKED && \ + ((lock)->sx_lock & SX_LOCK_SHARED)) +#define rw_write_held(lock) sx_xlocked(lock) +#define rw_lock_held(lock) (rw_read_held(lock) || rw_write_held(lock)) +#define rw_iswriter(lock) sx_xlocked(lock) +#define rw_owner(lock) sx_xholder(lock) + +/* END CSTYLED */ +#endif /* _OPENSOLARIS_SYS_RWLOCK_H_ */ diff --git a/include/os/freebsd/spl/sys/sdt.h b/include/os/freebsd/spl/sys/sdt.h new file mode 100644 index 00000000000..496fc58d7c7 --- /dev/null +++ b/include/os/freebsd/spl/sys/sdt.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_SDT_H_ +#define _OPENSOLARIS_SYS_SDT_H_ + +#include_next +/* BEGIN CSTYLED */ +#ifdef KDTRACE_HOOKS +SDT_PROBE_DECLARE(sdt, , , set__error); + +#define SET_ERROR(err) \ + ((sdt_sdt___set__error->id ? \ + (*sdt_probe_func)(sdt_sdt___set__error->id, \ + (uintptr_t)err, 0, 0, 0, 0) : 0), err) +#else +#define SET_ERROR(err) (err) +#endif + +#endif /* _OPENSOLARIS_SYS_SDT_H_ */ diff --git a/include/os/freebsd/spl/sys/sid.h b/include/os/freebsd/spl/sys/sid.h new file mode 100644 index 00000000000..18b6834250d --- /dev/null +++ b/include/os/freebsd/spl/sys/sid.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_SID_H_ +#define _OPENSOLARIS_SYS_SID_H_ + +typedef struct ksiddomain { + char *kd_name; /* Domain part of SID */ + uint_t kd_len; +} ksiddomain_t; +typedef void ksid_t; + +static __inline ksiddomain_t * +ksid_lookupdomain(const char *domain) +{ + ksiddomain_t *kd; + size_t len; + + len = strlen(domain) + 1; + kd = kmem_alloc(sizeof (*kd), KM_SLEEP); + kd->kd_len = (uint_t)len; + kd->kd_name = kmem_alloc(len, KM_SLEEP); + strcpy(kd->kd_name, domain); + return (kd); +} + +static __inline void +ksiddomain_rele(ksiddomain_t *kd) +{ + + kmem_free(kd->kd_name, kd->kd_len); + kmem_free(kd, sizeof (*kd)); +} + +static __inline uint_t +ksid_getid(ksid_t *ks) +{ + + panic("%s has been unexpectedly called", __func__); +} + +static __inline const char * +ksid_getdomain(ksid_t *ks) +{ + + panic("%s has been unexpectedly called", __func__); +} + +static __inline uint_t +ksid_getrid(ksid_t *ks) +{ + + panic("%s has been unexpectedly called", __func__); +} + +#define kidmap_getsidbyuid(zone, uid, sid_prefix, rid) (1) +#define kidmap_getsidbygid(zone, gid, sid_prefix, rid) (1) + +#endif /* _OPENSOLARIS_SYS_SID_H_ */ diff --git a/include/os/freebsd/spl/sys/sig.h b/include/os/freebsd/spl/sys/sig.h new file mode 100644 index 00000000000..426a9e827ec --- /dev/null +++ b/include/os/freebsd/spl/sys/sig.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2008 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_SIG_H_ +#define _OPENSOLARIS_SYS_SIG_H_ + +#include_next +#include +#include +#include +#include +#include +#include + +#define FORREAL 0 +#define JUSTLOOKING 1 + +static __inline int +issig(int why) +{ + struct thread *td = curthread; + struct proc *p; + int sig; + + ASSERT(why == FORREAL || why == JUSTLOOKING); + if (SIGPENDING(td)) { + if (why == JUSTLOOKING) + return (1); + p = td->td_proc; + PROC_LOCK(p); + mtx_lock(&p->p_sigacts->ps_mtx); + sig = cursig(td); + mtx_unlock(&p->p_sigacts->ps_mtx); + PROC_UNLOCK(p); + if (sig != 0) + return (1); + } + return (0); +} +#endif /* _OPENSOLARIS_SYS_SIG_H_ */ diff --git a/include/os/freebsd/spl/sys/simd.h b/include/os/freebsd/spl/sys/simd.h new file mode 100644 index 00000000000..53503e83891 --- /dev/null +++ b/include/os/freebsd/spl/sys/simd.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + +#ifndef _FREEBSD_SIMD_H +#define _FREEBSD_SIMD_H +#if defined(__amd64__) || defined(__i386__) +#include +#else + +#define kfpu_allowed() 0 +#define kfpu_initialize(tsk) do {} while (0) +#define kfpu_begin() do {} while (0) +#define kfpu_end() do {} while (0) +#define kfpu_init() (0) +#define kfpu_fini() do {} while (0) +#endif +#endif diff --git a/include/os/freebsd/spl/sys/simd_x86.h b/include/os/freebsd/spl/sys/simd_x86.h new file mode 100644 index 00000000000..da846fcbe6c --- /dev/null +++ b/include/os/freebsd/spl/sys/simd_x86.h @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include + +#define kfpu_init() (0) +#define kfpu_fini() do {} while (0) +#define kfpu_allowed() 1 +#define kfpu_initialize(tsk) do {} while (0) + +#define kfpu_begin() { \ + critical_enter(); \ + fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); \ +} + +#define kfpu_end() \ + { \ + fpu_kern_leave(curthread, NULL); \ + critical_exit(); \ + } + +/* + * Check if OS supports AVX and AVX2 by checking XCR0 + * Only call this function if CPUID indicates that AVX feature is + * supported by the CPU, otherwise it might be an illegal instruction. + */ +static inline uint64_t +xgetbv(uint32_t index) +{ + uint32_t eax, edx; + /* xgetbv - instruction byte code */ + __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0" + : "=a" (eax), "=d" (edx) + : "c" (index)); + + return ((((uint64_t)edx)<<32) | (uint64_t)eax); +} + + +/* + * Detect register set support + */ +static inline boolean_t +__simd_state_enabled(const uint64_t state) +{ + boolean_t has_osxsave; + uint64_t xcr0; + + has_osxsave = !!(cpu_feature2 & CPUID2_OSXSAVE); + + if (!has_osxsave) + return (B_FALSE); + + xcr0 = xgetbv(0); + return ((xcr0 & state) == state); +} + +#define _XSTATE_SSE_AVX (0x2 | 0x4) +#define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX) + +#define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX) +#define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512) + + +/* + * Check if SSE instruction set is available + */ +static inline boolean_t +zfs_sse_available(void) +{ + return (!!(cpu_feature & CPUID_SSE)); +} + +/* + * Check if SSE2 instruction set is available + */ +static inline boolean_t +zfs_sse2_available(void) +{ + return (!!(cpu_feature & CPUID_SSE2)); +} + +/* + * Check if SSE3 instruction set is available + */ +static inline boolean_t +zfs_sse3_available(void) +{ + return (!!(cpu_feature2 & CPUID2_SSE3)); +} + +/* + * Check if SSSE3 instruction set is available + */ +static inline boolean_t +zfs_ssse3_available(void) +{ + return (!!(cpu_feature2 & CPUID2_SSSE3)); +} + +/* + * Check if SSE4.1 instruction set is available + */ +static inline boolean_t +zfs_sse4_1_available(void) +{ + return (!!(cpu_feature2 & CPUID2_SSE41)); +} + +/* + * Check if SSE4.2 instruction set is available + */ +static inline boolean_t +zfs_sse4_2_available(void) +{ + return (!!(cpu_feature2 & CPUID2_SSE42)); +} + +/* + * Check if AVX instruction set is available + */ +static inline boolean_t +zfs_avx_available(void) +{ + boolean_t has_avx; + + has_avx = !!(cpu_feature2 & CPUID2_AVX); + + return (has_avx && __ymm_enabled()); +} + +/* + * Check if AVX2 instruction set is available + */ +static inline boolean_t +zfs_avx2_available(void) +{ + boolean_t has_avx2; + + has_avx2 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX2); + + return (has_avx2 && __ymm_enabled()); +} + +/* + * AVX-512 family of instruction sets: + * + * AVX512F Foundation + * AVX512CD Conflict Detection Instructions + * AVX512ER Exponential and Reciprocal Instructions + * AVX512PF Prefetch Instructions + * + * AVX512BW Byte and Word Instructions + * AVX512DQ Double-word and Quadword Instructions + * AVX512VL Vector Length Extensions + * + * AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4) + * AVX512VBMI Vector Byte Manipulation Instructions + */ + + +/* Check if AVX512F instruction set is available */ +static inline boolean_t +zfs_avx512f_available(void) +{ + boolean_t has_avx512; + + has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F); + + return (has_avx512 && __zmm_enabled()); +} + +/* Check if AVX512CD instruction set is available */ +static inline boolean_t +zfs_avx512cd_available(void) +{ + boolean_t has_avx512; + + has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && + !!(cpu_stdext_feature & CPUID_STDEXT_AVX512CD); + + return (has_avx512 && __zmm_enabled()); +} + +/* Check if AVX512ER instruction set is available */ +static inline boolean_t +zfs_avx512er_available(void) +{ + boolean_t has_avx512; + + has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && + !!(cpu_stdext_feature & CPUID_STDEXT_AVX512CD); + + return (has_avx512 && __zmm_enabled()); +} + +/* Check if AVX512PF instruction set is available */ +static inline boolean_t +zfs_avx512pf_available(void) +{ + boolean_t has_avx512; + + has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && + !!(cpu_stdext_feature & CPUID_STDEXT_AVX512PF); + + return (has_avx512 && __zmm_enabled()); +} + +/* Check if AVX512BW instruction set is available */ +static inline boolean_t +zfs_avx512bw_available(void) +{ + boolean_t has_avx512 = B_FALSE; + + has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512BW); + + return (has_avx512 && __zmm_enabled()); +} + +/* Check if AVX512DQ instruction set is available */ +static inline boolean_t +zfs_avx512dq_available(void) +{ + boolean_t has_avx512; + + has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && + !!(cpu_stdext_feature & CPUID_STDEXT_AVX512DQ); + + return (has_avx512 && __zmm_enabled()); +} + +/* Check if AVX512VL instruction set is available */ +static inline boolean_t +zfs_avx512vl_available(void) +{ + boolean_t has_avx512; + + has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && + !!(cpu_stdext_feature & CPUID_STDEXT_AVX512VL); + + return (has_avx512 && __zmm_enabled()); +} + +/* Check if AVX512IFMA instruction set is available */ +static inline boolean_t +zfs_avx512ifma_available(void) +{ + boolean_t has_avx512; + + has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && + !!(cpu_stdext_feature & CPUID_STDEXT_AVX512IFMA); + + return (has_avx512 && __zmm_enabled()); +} + +/* Check if AVX512VBMI instruction set is available */ +static inline boolean_t +zfs_avx512vbmi_available(void) +{ + boolean_t has_avx512; + + has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && + !!(cpu_stdext_feature & CPUID_STDEXT_BMI1); + + return (has_avx512 && __zmm_enabled()); +} diff --git a/include/os/freebsd/spl/sys/spl_condvar.h b/include/os/freebsd/spl/sys/spl_condvar.h new file mode 100644 index 00000000000..7405f647d59 --- /dev/null +++ b/include/os/freebsd/spl/sys/spl_condvar.h @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2000 Jake Burkholder . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SPL_SYS_CONDVAR_H_ +#define _SPL_SYS_CONDVAR_H_ + +#ifndef LOCORE +#include + +struct lock_object; +struct thread; + +TAILQ_HEAD(cv_waitq, thread); + +/* + * Condition variable. The waiters count is protected by the mutex that + * protects the condition; that is, the mutex that is passed to cv_wait*() + * and is held across calls to cv_signal() and cv_broadcast(). It is an + * optimization to avoid looking up the sleep queue if there are no waiters. + */ +struct cv { + const char *cv_description; + int cv_waiters; +}; + +void cv_init(struct cv *cvp, const char *desc); +void cv_destroy(struct cv *cvp); + +void _cv_wait(struct cv *cvp, struct lock_object *lock); +void _cv_wait_unlock(struct cv *cvp, struct lock_object *lock); +int _cv_wait_sig(struct cv *cvp, struct lock_object *lock); +int _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, + sbintime_t sbt, sbintime_t pr, int flags); +int _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock, + sbintime_t sbt, sbintime_t pr, int flags); + +void cv_signal(struct cv *cvp); +void cv_broadcastpri(struct cv *cvp, int pri); + +#define cv_wait(cvp, lock) \ + _cv_wait((cvp), &(lock)->lock_object) +#define cv_wait_unlock(cvp, lock) \ + _cv_wait_unlock((cvp), &(lock)->lock_object) +#define cv_timedwait_sbt(cvp, lock, sbt, pr, flags) \ + _cv_timedwait_sbt((cvp), &(lock)->lock_object, (sbt), (pr), (flags)) +#define cv_timedwait_sig_sbt(cvp, lock, sbt, pr, flags) \ + _cv_timedwait_sig_sbt((cvp), &(lock)->lock_object, (sbt), (pr), (flags)) + +#define cv_broadcast(cvp) cv_broadcastpri(cvp, 0) + +#define cv_wmesg(cvp) ((cvp)->cv_description) + +#endif /* !LOCORE */ +#endif /* _SYS_CONDVAR_H_ */ diff --git a/include/os/freebsd/spl/sys/string.h b/include/os/freebsd/spl/sys/string.h new file mode 100644 index 00000000000..859b40285a9 --- /dev/null +++ b/include/os/freebsd/spl/sys/string.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_STRING_H_ +#define _OPENSOLARIS_SYS_STRING_H_ + +#include + +char *strpbrk(const char *, const char *); +void strident_canon(char *, size_t); +void kmem_strfree(char *); +char *kmem_strdup(const char *s); + +#endif /* _OPENSOLARIS_SYS_STRING_H_ */ diff --git a/include/os/freebsd/spl/sys/strings.h b/include/os/freebsd/spl/sys/strings.h new file mode 100644 index 00000000000..651685d3047 --- /dev/null +++ b/include/os/freebsd/spl/sys/strings.h @@ -0,0 +1 @@ +/* do not delete */ diff --git a/include/os/freebsd/spl/sys/sunddi.h b/include/os/freebsd/spl/sys/sunddi.h new file mode 100644 index 00000000000..bb76cd9641c --- /dev/null +++ b/include/os/freebsd/spl/sys/sunddi.h @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. + * Copyright (C) 2007 The Regents of the University of California. + * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). + * Written by Brian Behlendorf . + * UCRL-CODE-235197 + * + * This file is part of the SPL, Solaris Porting Layer. + * For details, see . + * + * The SPL is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * The SPL is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License along + * with the SPL. If not, see . + */ + +#ifndef _SPL_SUNDDI_H +#define _SPL_SUNDDI_H + +#include +#include +#include +#include +#include + +typedef int ddi_devid_t; + +#define DDI_DEV_T_NONE ((dev_t)-1) +#define DDI_DEV_T_ANY ((dev_t)-2) +#define DI_MAJOR_T_UNKNOWN ((major_t)0) + +#define DDI_PROP_DONTPASS 0x0001 +#define DDI_PROP_CANSLEEP 0x0002 + +#define DDI_SUCCESS 0 +#define DDI_FAILURE -1 + +#define ddi_prop_lookup_string(x1, x2, x3, x4, x5) (*x5 = NULL) +#define ddi_prop_free(x) (void)0 +#define ddi_root_node() (void)0 + +extern int ddi_strtoul(const char *, char **, int, unsigned long *); +extern int ddi_strtol(const char *, char **, int, long *); +extern int ddi_strtoull(const char *, char **, int, unsigned long long *); +extern int ddi_strtoll(const char *, char **, int, long long *); + +extern int ddi_copyin(const void *from, void *to, size_t len, int flags); +extern int ddi_copyout(const void *from, void *to, size_t len, int flags); +extern void ddi_sysevent_init(void); + + +int ddi_soft_state_init(void **statep, size_t size, size_t nitems); +void ddi_soft_state_fini(void **statep); + +void *ddi_get_soft_state(void *state, int item); +int ddi_soft_state_zalloc(void *state, int item); +void ddi_soft_state_free(void *state, int item); + +#endif /* SPL_SUNDDI_H */ diff --git a/include/os/freebsd/spl/sys/sysmacros.h b/include/os/freebsd/spl/sys/sysmacros.h new file mode 100644 index 00000000000..5afca10447e --- /dev/null +++ b/include/os/freebsd/spl/sys/sysmacros.h @@ -0,0 +1,404 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + + +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_SYSMACROS_H +#define _SYS_SYSMACROS_H + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Some macros for units conversion + */ +/* + * Disk blocks (sectors) and bytes. + */ +#define dtob(DD) ((DD) << DEV_BSHIFT) +#define btod(BB) (((BB) + DEV_BSIZE - 1) >> DEV_BSHIFT) +#define btodt(BB) ((BB) >> DEV_BSHIFT) +#define lbtod(BB) (((offset_t)(BB) + DEV_BSIZE - 1) >> DEV_BSHIFT) + +/* common macros */ +#ifndef MIN +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif +#ifndef MAX +#define MAX(a, b) ((a) < (b) ? (b) : (a)) +#endif +#ifndef ABS +#define ABS(a) ((a) < 0 ? -(a) : (a)) +#endif +#ifndef SIGNOF +#define SIGNOF(a) ((a) < 0 ? -1 : (a) > 0) +#endif +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(a) (sizeof (a) / sizeof (a[0])) +#endif +#ifndef DIV_ROUND_UP +#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +#endif + +#define boot_ncpus mp_ncpus +#define kpreempt_disable() critical_enter() +#define kpreempt_enable() critical_exit() +#define CPU_SEQID curcpu +#define is_system_labeled() 0 +/* + * Convert a single byte to/from binary-coded decimal (BCD). + */ +extern unsigned char byte_to_bcd[256]; +extern unsigned char bcd_to_byte[256]; + +#define BYTE_TO_BCD(x) byte_to_bcd[(x) & 0xff] +#define BCD_TO_BYTE(x) bcd_to_byte[(x) & 0xff] + +/* + * WARNING: The device number macros defined here should not be used by device + * drivers or user software. Device drivers should use the device functions + * defined in the DDI/DKI interface (see also ddi.h). Application software + * should make use of the library routines available in makedev(3). A set of + * new device macros are provided to operate on the expanded device number + * format supported in SVR4. Macro versions of the DDI device functions are + * provided for use by kernel proper routines only. Macro routines bmajor(), + * major(), minor(), emajor(), eminor(), and makedev() will be removed or + * their definitions changed at the next major release following SVR4. + */ + +#define O_BITSMAJOR 7 /* # of SVR3 major device bits */ +#define O_BITSMINOR 8 /* # of SVR3 minor device bits */ +#define O_MAXMAJ 0x7f /* SVR3 max major value */ +#define O_MAXMIN 0xff /* SVR3 max minor value */ + + +#define L_BITSMAJOR32 14 /* # of SVR4 major device bits */ +#define L_BITSMINOR32 18 /* # of SVR4 minor device bits */ +#define L_MAXMAJ32 0x3fff /* SVR4 max major value */ +#define L_MAXMIN32 0x3ffff /* MAX minor for 3b2 software drivers. */ + /* For 3b2 hardware devices the minor is */ + /* restricted to 256 (0-255) */ + +#ifdef _LP64 +#define L_BITSMAJOR 32 /* # of major device bits in 64-bit Solaris */ +#define L_BITSMINOR 32 /* # of minor device bits in 64-bit Solaris */ +#define L_MAXMAJ 0xfffffffful /* max major value */ +#define L_MAXMIN 0xfffffffful /* max minor value */ +#else +#define L_BITSMAJOR L_BITSMAJOR32 +#define L_BITSMINOR L_BITSMINOR32 +#define L_MAXMAJ L_MAXMAJ32 +#define L_MAXMIN L_MAXMIN32 +#endif + +/* + * These are versions of the kernel routines for compressing and + * expanding long device numbers that don't return errors. + */ +#if (L_BITSMAJOR32 == L_BITSMAJOR) && (L_BITSMINOR32 == L_BITSMINOR) + +#define DEVCMPL(x) (x) +#define DEVEXPL(x) (x) + +#else + +#define DEVCMPL(x) \ + (dev32_t)((((x) >> L_BITSMINOR) > L_MAXMAJ32 || \ + ((x) & L_MAXMIN) > L_MAXMIN32) ? NODEV32 : \ + ((((x) >> L_BITSMINOR) << L_BITSMINOR32) | ((x) & L_MAXMIN32))) + +#define DEVEXPL(x) \ + (((x) == NODEV32) ? NODEV : \ + makedevice(((x) >> L_BITSMINOR32) & L_MAXMAJ32, (x) & L_MAXMIN32)) + +#endif /* L_BITSMAJOR32 ... */ + +/* convert to old (SVR3.2) dev format */ + +#define cmpdev(x) \ + (o_dev_t)((((x) >> L_BITSMINOR) > O_MAXMAJ || \ + ((x) & L_MAXMIN) > O_MAXMIN) ? NODEV : \ + ((((x) >> L_BITSMINOR) << O_BITSMINOR) | ((x) & O_MAXMIN))) + +/* convert to new (SVR4) dev format */ + +#define expdev(x) \ + (dev_t)(((dev_t)(((x) >> O_BITSMINOR) & O_MAXMAJ) << L_BITSMINOR) | \ + ((x) & O_MAXMIN)) + +/* + * Macro for checking power of 2 address alignment. + */ +#define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) + +/* + * Macros for counting and rounding. + */ +#define howmany(x, y) (((x)+((y)-1))/(y)) +#define roundup(x, y) ((((x)+((y)-1))/(y))*(y)) + +/* + * Macro to determine if value is a power of 2 + */ +#define ISP2(x) (((x) & ((x) - 1)) == 0) + +/* + * Macros for various sorts of alignment and rounding. The "align" must + * be a power of 2. Often times it is a block, sector, or page. + */ + +/* + * return x rounded down to an align boundary + * eg, P2ALIGN(1200, 1024) == 1024 (1*align) + * eg, P2ALIGN(1024, 1024) == 1024 (1*align) + * eg, P2ALIGN(0x1234, 0x100) == 0x1200 (0x12*align) + * eg, P2ALIGN(0x5600, 0x100) == 0x5600 (0x56*align) + */ +#define P2ALIGN(x, align) ((x) & -(align)) + +/* + * return x % (mod) align + * eg, P2PHASE(0x1234, 0x100) == 0x34 (x-0x12*align) + * eg, P2PHASE(0x5600, 0x100) == 0x00 (x-0x56*align) + */ +#define P2PHASE(x, align) ((x) & ((align) - 1)) + +/* + * return how much space is left in this block (but if it's perfectly + * aligned, return 0). + * eg, P2NPHASE(0x1234, 0x100) == 0xcc (0x13*align-x) + * eg, P2NPHASE(0x5600, 0x100) == 0x00 (0x56*align-x) + */ +#define P2NPHASE(x, align) (-(x) & ((align) - 1)) + +/* + * return x rounded up to an align boundary + * eg, P2ROUNDUP(0x1234, 0x100) == 0x1300 (0x13*align) + * eg, P2ROUNDUP(0x5600, 0x100) == 0x5600 (0x56*align) + */ +#define P2ROUNDUP(x, align) (-(-(x) & -(align))) + +/* + * return the ending address of the block that x is in + * eg, P2END(0x1234, 0x100) == 0x12ff (0x13*align - 1) + * eg, P2END(0x5600, 0x100) == 0x56ff (0x57*align - 1) + */ +#define P2END(x, align) (-(~(x) & -(align))) + +/* + * return x rounded up to the next phase (offset) within align. + * phase should be < align. + * eg, P2PHASEUP(0x1234, 0x100, 0x10) == 0x1310 (0x13*align + phase) + * eg, P2PHASEUP(0x5600, 0x100, 0x10) == 0x5610 (0x56*align + phase) + */ +#define P2PHASEUP(x, align, phase) ((phase) - (((phase) - (x)) & -(align))) + +/* + * return TRUE if adding len to off would cause it to cross an align + * boundary. + * eg, P2BOUNDARY(0x1234, 0xe0, 0x100) == TRUE (0x1234 + 0xe0 == 0x1314) + * eg, P2BOUNDARY(0x1234, 0x50, 0x100) == FALSE (0x1234 + 0x50 == 0x1284) + */ +#define P2BOUNDARY(off, len, align) \ + (((off) ^ ((off) + (len) - 1)) > (align) - 1) + +/* + * Return TRUE if they have the same highest bit set. + * eg, P2SAMEHIGHBIT(0x1234, 0x1001) == TRUE (the high bit is 0x1000) + * eg, P2SAMEHIGHBIT(0x1234, 0x3010) == FALSE (high bit of 0x3010 is 0x2000) + */ +#define P2SAMEHIGHBIT(x, y) (((x) ^ (y)) < ((x) & (y))) + +/* + * Typed version of the P2* macros. These macros should be used to ensure + * that the result is correctly calculated based on the data type of (x), + * which is passed in as the last argument, regardless of the data + * type of the alignment. For example, if (x) is of type uint64_t, + * and we want to round it up to a page boundary using "PAGESIZE" as + * the alignment, we can do either + * P2ROUNDUP(x, (uint64_t)PAGESIZE) + * or + * P2ROUNDUP_TYPED(x, PAGESIZE, uint64_t) + */ +#define P2ALIGN_TYPED(x, align, type) \ + ((type)(x) & -(type)(align)) +#define P2PHASE_TYPED(x, align, type) \ + ((type)(x) & ((type)(align) - 1)) +#define P2NPHASE_TYPED(x, align, type) \ + (-(type)(x) & ((type)(align) - 1)) +#define P2ROUNDUP_TYPED(x, align, type) \ + (-(-(type)(x) & -(type)(align))) +#define P2END_TYPED(x, align, type) \ + (-(~(type)(x) & -(type)(align))) +#define P2PHASEUP_TYPED(x, align, phase, type) \ + ((type)(phase) - (((type)(phase) - (type)(x)) & -(type)(align))) +#define P2CROSS_TYPED(x, y, align, type) \ + (((type)(x) ^ (type)(y)) > (type)(align) - 1) +#define P2SAMEHIGHBIT_TYPED(x, y, type) \ + (((type)(x) ^ (type)(y)) < ((type)(x) & (type)(y))) + +/* + * Macros to atomically increment/decrement a variable. mutex and var + * must be pointers. + */ +#define INCR_COUNT(var, mutex) mutex_enter(mutex), (*(var))++, mutex_exit(mutex) +#define DECR_COUNT(var, mutex) mutex_enter(mutex), (*(var))--, mutex_exit(mutex) + +/* + * Macros to declare bitfields - the order in the parameter list is + * Low to High - that is, declare bit 0 first. We only support 8-bit bitfields + * because if a field crosses a byte boundary it's not likely to be meaningful + * without reassembly in its nonnative endianness. + */ +#if defined(_BIT_FIELDS_LTOH) +#define DECL_BITFIELD2(_a, _b) \ + uint8_t _a, _b +#define DECL_BITFIELD3(_a, _b, _c) \ + uint8_t _a, _b, _c +#define DECL_BITFIELD4(_a, _b, _c, _d) \ + uint8_t _a, _b, _c, _d +#define DECL_BITFIELD5(_a, _b, _c, _d, _e) \ + uint8_t _a, _b, _c, _d, _e +#define DECL_BITFIELD6(_a, _b, _c, _d, _e, _f) \ + uint8_t _a, _b, _c, _d, _e, _f +#define DECL_BITFIELD7(_a, _b, _c, _d, _e, _f, _g) \ + uint8_t _a, _b, _c, _d, _e, _f, _g +#define DECL_BITFIELD8(_a, _b, _c, _d, _e, _f, _g, _h) \ + uint8_t _a, _b, _c, _d, _e, _f, _g, _h +#elif defined(_BIT_FIELDS_HTOL) +#define DECL_BITFIELD2(_a, _b) \ + uint8_t _b, _a +#define DECL_BITFIELD3(_a, _b, _c) \ + uint8_t _c, _b, _a +#define DECL_BITFIELD4(_a, _b, _c, _d) \ + uint8_t _d, _c, _b, _a +#define DECL_BITFIELD5(_a, _b, _c, _d, _e) \ + uint8_t _e, _d, _c, _b, _a +#define DECL_BITFIELD6(_a, _b, _c, _d, _e, _f) \ + uint8_t _f, _e, _d, _c, _b, _a +#define DECL_BITFIELD7(_a, _b, _c, _d, _e, _f, _g) \ + uint8_t _g, _f, _e, _d, _c, _b, _a +#define DECL_BITFIELD8(_a, _b, _c, _d, _e, _f, _g, _h) \ + uint8_t _h, _g, _f, _e, _d, _c, _b, _a +#else +#error One of _BIT_FIELDS_LTOH or _BIT_FIELDS_HTOL must be defined +#endif /* _BIT_FIELDS_LTOH */ + +#if !defined(_KMEMUSER) && !defined(offsetof) + +/* avoid any possibility of clashing with version */ + +#define offsetof(s, m) ((size_t)(&(((s *)0)->m))) +#endif + +/* + * Find highest one bit set. + * Returns bit number + 1 of highest bit that is set, otherwise returns 0. + * High order bit is 31 (or 63 in _LP64 kernel). + */ +static __inline int +highbit(ulong_t i) +{ +#if defined(HAVE_INLINE_FLSL) + return (flsl(i)); +#else + int h = 1; + + if (i == 0) + return (0); +#ifdef _LP64 + if (i & 0xffffffff00000000ul) { + h += 32; i >>= 32; + } +#endif + if (i & 0xffff0000) { + h += 16; i >>= 16; + } + if (i & 0xff00) { + h += 8; i >>= 8; + } + if (i & 0xf0) { + h += 4; i >>= 4; + } + if (i & 0xc) { + h += 2; i >>= 2; + } + if (i & 0x2) { + h += 1; + } + return (h); +#endif +} + +/* + * Find highest one bit set. + * Returns bit number + 1 of highest bit that is set, otherwise returns 0. + */ +static __inline int +highbit64(uint64_t i) +{ +#if defined(HAVE_INLINE_FLSLL) + return (flsll(i)); +#else + int h = 1; + + if (i == 0) + return (0); + if (i & 0xffffffff00000000ULL) { + h += 32; i >>= 32; + } + if (i & 0xffff0000) { + h += 16; i >>= 16; + } + if (i & 0xff00) { + h += 8; i >>= 8; + } + if (i & 0xf0) { + h += 4; i >>= 4; + } + if (i & 0xc) { + h += 2; i >>= 2; + } + if (i & 0x2) { + h += 1; + } + return (h); +#endif +} + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_SYSMACROS_H */ diff --git a/include/os/freebsd/spl/sys/systeminfo.h b/include/os/freebsd/spl/sys/systeminfo.h new file mode 100644 index 00000000000..4028cd7cc6f --- /dev/null +++ b/include/os/freebsd/spl/sys/systeminfo.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_SYSTEMINFO_H_ +#define _SYS_SYSTEMINFO_H_ + +#define HW_HOSTID_LEN 11 + +#endif /* !_SYS_SYSTEMINFO_H_ */ diff --git a/include/os/freebsd/spl/sys/systm.h b/include/os/freebsd/spl/sys/systm.h new file mode 100644 index 00000000000..53cc6f52717 --- /dev/null +++ b/include/os/freebsd/spl/sys/systm.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_SYSTM_H_ +#define _OPENSOLARIS_SYS_SYSTM_H_ + +#include +#include_next + +#include + +#define PAGESIZE PAGE_SIZE +#define PAGEOFFSET (PAGESIZE - 1) +#define PAGEMASK (~PAGEOFFSET) + +#define delay(x) pause("soldelay", (x)) + +#endif /* _OPENSOLARIS_SYS_SYSTM_H_ */ diff --git a/include/os/freebsd/spl/sys/taskq.h b/include/os/freebsd/spl/sys/taskq.h new file mode 100644 index 00000000000..aaa435f2f37 --- /dev/null +++ b/include/os/freebsd/spl/sys/taskq.h @@ -0,0 +1,114 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_TASKQ_H +#define _SYS_TASKQ_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define TASKQ_NAMELEN 31 + +struct taskqueue; +struct taskq { + struct taskqueue *tq_queue; +}; + +typedef struct taskq taskq_t; +typedef uintptr_t taskqid_t; +typedef void (task_func_t)(void *); + +typedef struct taskq_ent { + struct task tqent_task; + task_func_t *tqent_func; + void *tqent_arg; + struct timeout_task tqent_timeout_task; + int tqent_type; + int tqent_gen; +} taskq_ent_t; + +struct proc; + +/* + * Public flags for taskq_create(): bit range 0-15 + */ +#define TASKQ_PREPOPULATE 0x0001 /* Prepopulate with threads and data */ +#define TASKQ_CPR_SAFE 0x0002 /* Use CPR safe protocol */ +#define TASKQ_DYNAMIC 0x0004 /* Use dynamic thread scheduling */ +#define TASKQ_THREADS_CPU_PCT 0x0008 /* number of threads as % of ncpu */ +#define TASKQ_DC_BATCH 0x0010 /* Taskq uses SDC in batch mode */ + +/* + * Flags for taskq_dispatch. TQ_SLEEP/TQ_NOSLEEP should be same as + * KM_SLEEP/KM_NOSLEEP. + */ +#define TQ_SLEEP 0x00 /* Can block for memory */ +#define TQ_NOSLEEP 0x01 /* cannot block for memory; may fail */ +#define TQ_NOQUEUE 0x02 /* Do not enqueue if can't dispatch */ +#define TQ_NOALLOC 0x04 /* cannot allocate memory; may fail */ +#define TQ_FRONT 0x08 /* Put task at the front of the queue */ + +#define TASKQID_INVALID ((taskqid_t)0) + +#define taskq_init_ent(x) +extern taskq_t *system_taskq; +/* Global dynamic task queue for long delay */ +extern taskq_t *system_delay_taskq; + +extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t); +extern taskqid_t taskq_dispatch_delay(taskq_t *, task_func_t, void *, + uint_t, clock_t); +extern void taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t, + taskq_ent_t *); +extern int taskq_empty_ent(taskq_ent_t *); +taskq_t *taskq_create(const char *, int, pri_t, int, int, uint_t); +taskq_t *taskq_create_instance(const char *, int, int, pri_t, int, int, uint_t); +taskq_t *taskq_create_proc(const char *, int, pri_t, int, int, + struct proc *, uint_t); +taskq_t *taskq_create_sysdc(const char *, int, int, int, + struct proc *, uint_t, uint_t); +void nulltask(void *); +extern void taskq_destroy(taskq_t *); +extern void taskq_wait_id(taskq_t *, taskqid_t); +extern void taskq_wait_outstanding(taskq_t *, taskqid_t); +extern void taskq_wait(taskq_t *); +extern int taskq_cancel_id(taskq_t *, taskqid_t); +extern int taskq_member(taskq_t *, kthread_t *); +extern taskq_t *taskq_of_curthread(void); +void taskq_suspend(taskq_t *); +int taskq_suspended(taskq_t *); +void taskq_resume(taskq_t *); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_TASKQ_H */ diff --git a/include/os/freebsd/spl/sys/thread.h b/include/os/freebsd/spl/sys/thread.h new file mode 100644 index 00000000000..4fb1a542f55 --- /dev/null +++ b/include/os/freebsd/spl/sys/thread.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SPL_THREAD_H_ +#define _SPL_THREAD_H_ + +#define getcomm() curthread->td_name +#define getpid() curthread->td_tid +#endif diff --git a/include/os/freebsd/spl/sys/time.h b/include/os/freebsd/spl/sys/time.h new file mode 100644 index 00000000000..fbc679aacf9 --- /dev/null +++ b/include/os/freebsd/spl/sys/time.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_TIME_H_ +#define _OPENSOLARIS_SYS_TIME_H_ +#pragma once +#include_next +#include +#ifndef _SYS_KERNEL_H_ +extern int hz; +#endif + +#define SEC 1 +#define MILLISEC 1000UL +#define MICROSEC 1000000UL +#define NANOSEC 1000000000UL +#define TIME_MAX LLONG_MAX + +#define MSEC2NSEC(m) ((hrtime_t)(m) * (NANOSEC / MILLISEC)) +#define NSEC2MSEC(n) ((n) / (NANOSEC / MILLISEC)) + +#define USEC2NSEC(m) ((hrtime_t)(m) * (NANOSEC / MICROSEC)) +#define NSEC2USEC(n) ((n) / (NANOSEC / MICROSEC)) + +#define NSEC2SEC(n) ((n) / (NANOSEC / SEC)) +#define SEC2NSEC(m) ((hrtime_t)(m) * (NANOSEC / SEC)) + +typedef longlong_t hrtime_t; + +#if defined(__i386__) || defined(__powerpc__) +#define TIMESPEC_OVERFLOW(ts) \ + ((ts)->tv_sec < INT32_MIN || (ts)->tv_sec > INT32_MAX) +#else +#define TIMESPEC_OVERFLOW(ts) \ + ((ts)->tv_sec < INT64_MIN || (ts)->tv_sec > INT64_MAX) +#endif + +#define SEC_TO_TICK(sec) ((sec) * hz) +#define NSEC_TO_TICK(nsec) ((nsec) / (NANOSEC / hz)) + +static __inline hrtime_t +gethrtime(void) +{ + struct timespec ts; + hrtime_t nsec; + + nanouptime(&ts); + nsec = ((hrtime_t)ts.tv_sec * NANOSEC) + ts.tv_nsec; + return (nsec); +} + +#define gethrestime_sec() (time_second) +#define gethrestime(ts) getnanotime(ts) +#define gethrtime_waitfree() gethrtime() + +extern int nsec_per_tick; /* nanoseconds per clock tick */ + +#define ddi_get_lbolt64() \ + (int64_t)(((getsbinuptime() >> 16) * hz) >> 16) +#define ddi_get_lbolt() (clock_t)ddi_get_lbolt64() + +#else + +static __inline hrtime_t +gethrtime(void) +{ + struct timespec ts; + clock_gettime(CLOCK_UPTIME, &ts); + return (((u_int64_t)ts.tv_sec) * NANOSEC + ts.tv_nsec); +} +#endif /* !_OPENSOLARIS_SYS_TIME_H_ */ diff --git a/include/os/freebsd/spl/sys/timer.h b/include/os/freebsd/spl/sys/timer.h new file mode 100644 index 00000000000..d4694bb7c09 --- /dev/null +++ b/include/os/freebsd/spl/sys/timer.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SPL_TIMER_H_ +#define _SPL_TIMER_H_ +#define ddi_time_after(a, b) ((a) > (b)) +#define ddi_time_after64(a, b) ((a) > (b)) +#define usleep_range(wakeup, wakeupepsilon) \ + pause_sbt("usleep_range", ustosbt(wakeup), \ + ustosbt(wakeupepsilon - wakeup), 0) + +#define schedule() pause("schedule", 1) +#endif diff --git a/include/os/freebsd/spl/sys/trace.h b/include/os/freebsd/spl/sys/trace.h new file mode 100644 index 00000000000..d9639d27b60 --- /dev/null +++ b/include/os/freebsd/spl/sys/trace.h @@ -0,0 +1 @@ +/* keep me */ diff --git a/include/os/freebsd/spl/sys/trace_zfs.h b/include/os/freebsd/spl/sys/trace_zfs.h new file mode 100644 index 00000000000..d9639d27b60 --- /dev/null +++ b/include/os/freebsd/spl/sys/trace_zfs.h @@ -0,0 +1 @@ +/* keep me */ diff --git a/include/os/freebsd/spl/sys/types.h b/include/os/freebsd/spl/sys/types.h new file mode 100644 index 00000000000..ed5e8ef80bf --- /dev/null +++ b/include/os/freebsd/spl/sys/types.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SPL_SYS_TYPES_H_ +#define _SPL_SYS_TYPES_H_ + +#pragma once +/* + * This is a bag of dirty hacks to keep things compiling. + */ + +#include + +typedef int64_t clock_t; +#define _CLOCK_T_DECLARED + +#include_next +#include +#include + +#define MAXNAMELEN 256 + +typedef struct timespec timestruc_t; +typedef struct timespec timespec_t; +typedef struct timespec inode_timespec_t; +/* BEGIN CSTYLED */ +typedef u_int uint_t; +typedef u_char uchar_t; +typedef u_short ushort_t; +typedef u_long ulong_t; +typedef u_int minor_t; +/* END CSTYLED */ +#ifndef _OFF64_T_DECLARED +#define _OFF64_T_DECLARED +typedef off_t off64_t; +#endif +typedef id_t taskid_t; +typedef id_t projid_t; +typedef id_t poolid_t; +typedef id_t zoneid_t; +typedef id_t ctid_t; +typedef mode_t o_mode_t; +typedef uint64_t pgcnt_t; + +#define B_FALSE 0 +#define B_TRUE 1 + +typedef short index_t; +typedef off_t offset_t; +#ifndef _PTRDIFF_T_DECLARED +typedef __ptrdiff_t ptrdiff_t; /* pointer difference */ +#define _PTRDIFF_T_DECLARED +#endif +typedef int64_t rlim64_t; +typedef int major_t; + +#else +#ifdef NEED_SOLARIS_BOOLEAN +#if defined(__XOPEN_OR_POSIX) +typedef enum { _B_FALSE, _B_TRUE } boolean_t; +#else +typedef enum { B_FALSE, B_TRUE } boolean_t; +#endif /* defined(__XOPEN_OR_POSIX) */ +#endif + +typedef u_longlong_t u_offset_t; +typedef u_longlong_t len_t; + +typedef longlong_t diskaddr_t; + + +#endif /* !_OPENSOLARIS_SYS_TYPES_H_ */ diff --git a/include/os/freebsd/spl/sys/types32.h b/include/os/freebsd/spl/sys/types32.h new file mode 100644 index 00000000000..907b667e5d8 --- /dev/null +++ b/include/os/freebsd/spl/sys/types32.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SPL_TYPES32_H +#define _SPL_TYPES32_H + +typedef uint32_t caddr32_t; +typedef int32_t daddr32_t; +typedef int32_t time32_t; +typedef uint32_t size32_t; + +#endif /* _SPL_TYPES32_H */ diff --git a/include/os/freebsd/spl/sys/uio.h b/include/os/freebsd/spl/sys/uio.h new file mode 100644 index 00000000000..fe5e24b99d1 --- /dev/null +++ b/include/os/freebsd/spl/sys/uio.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2010 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_UIO_H_ +#define _OPENSOLARIS_SYS_UIO_H_ + +#include_next +#include +#include + + + +#define uio_loffset uio_offset + +typedef struct uio uio_t; +typedef struct iovec iovec_t; +typedef enum uio_seg uio_seg_t; + +typedef enum xuio_type { + UIOTYPE_ASYNCIO, + UIOTYPE_ZEROCOPY +} xuio_type_t; + +typedef struct xuio { + uio_t xu_uio; + + /* Extended uio fields */ + enum xuio_type xu_type; /* What kind of uio structure? */ + union { + struct { + int xu_zc_rw; + void *xu_zc_priv; + } xu_zc; + } xu_ext; +} xuio_t; + +#define XUIO_XUZC_PRIV(xuio) xuio->xu_ext.xu_zc.xu_zc_priv +#define XUIO_XUZC_RW(xuio) xuio->xu_ext.xu_zc.xu_zc_rw + +static __inline int +zfs_uiomove(void *cp, size_t n, enum uio_rw dir, uio_t *uio) +{ + + ASSERT(uio->uio_rw == dir); + return (uiomove(cp, (int)n, uio)); +} +#define uiomove(cp, n, dir, uio) zfs_uiomove((cp), (n), (dir), (uio)) + +int uiocopy(void *p, size_t n, enum uio_rw rw, struct uio *uio, size_t *cbytes); +void uioskip(uio_t *uiop, size_t n); + +#endif /* !_OPENSOLARIS_SYS_UIO_H_ */ diff --git a/include/os/freebsd/spl/sys/uuid.h b/include/os/freebsd/spl/sys/uuid.h new file mode 100644 index 00000000000..26d46e8d621 --- /dev/null +++ b/include/os/freebsd/spl/sys/uuid.h @@ -0,0 +1,99 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_UUID_H +#define _SYS_UUID_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * The copyright in this file is taken from the original Leach + * & Salz UUID specification, from which this implementation + * is derived. + */ + +/* + * Copyright (c) 1990- 1993, 1996 Open Software Foundation, Inc. + * Copyright (c) 1989 by Hewlett-Packard Company, Palo Alto, Ca. & + * Digital Equipment Corporation, Maynard, Mass. Copyright (c) 1998 + * Microsoft. To anyone who acknowledges that this file is provided + * "AS IS" without any express or implied warranty: permission to use, + * copy, modify, and distribute this file for any purpose is hereby + * granted without fee, provided that the above copyright notices and + * this notice appears in all source code copies, and that none of the + * names of Open Software Foundation, Inc., Hewlett-Packard Company, + * or Digital Equipment Corporation be used in advertising or + * publicity pertaining to distribution of the software without + * specific, written prior permission. Neither Open Software + * Foundation, Inc., Hewlett-Packard Company, Microsoft, nor Digital + * Equipment Corporation makes any representations about the + * suitability of this software for any purpose. + */ + +#include +#include + +typedef struct { + uint8_t nodeID[6]; +} uuid_node_t; + +/* + * The uuid type used throughout when referencing uuids themselves + */ +typedef struct uuid { + uint32_t time_low; + uint16_t time_mid; + uint16_t time_hi_and_version; + uint8_t clock_seq_hi_and_reserved; + uint8_t clock_seq_low; + uint8_t node_addr[6]; +} uuid_t; + +#define UUID_PRINTABLE_STRING_LENGTH 37 + +/* + * Convert a uuid to/from little-endian format + */ +#define UUID_LE_CONVERT(dest, src) \ +{ \ + (dest) = (src); \ + (dest).time_low = LE_32((dest).time_low); \ + (dest).time_mid = LE_16((dest).time_mid); \ + (dest).time_hi_and_version = LE_16((dest).time_hi_and_version); \ +} + +static __inline int +uuid_is_null(const caddr_t uuid) +{ + return (0); +} +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_UUID_H */ diff --git a/include/os/freebsd/spl/sys/vfs.h b/include/os/freebsd/spl/sys/vfs.h new file mode 100644 index 00000000000..a432f6c5673 --- /dev/null +++ b/include/os/freebsd/spl/sys/vfs.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_VFS_H_ +#define _OPENSOLARIS_SYS_VFS_H_ + +#include +#include +#include + +#define rootdir rootvnode + +struct thread; +struct vnode; +typedef struct mount vfs_t; + +typedef int umode_t; + +#define vfs_flag mnt_flag +#define vfs_data mnt_data +#define vfs_count mnt_ref +#define vfs_fsid mnt_stat.f_fsid +#define vfs_bsize mnt_stat.f_bsize +#define vfs_resource mnt_stat.f_mntfromname + +#define v_flag v_vflag +#define v_vfsp v_mount + +#define VFS_RDONLY MNT_RDONLY +#define VFS_NOSETUID MNT_NOSUID +#define VFS_NOEXEC MNT_NOEXEC + +#define fs_vscan(vp, cr, async) (0) + +#define VROOT VV_ROOT + +#define XU_NGROUPS 16 + +/* + * Structure defining a mount option for a filesystem. + * option names are found in mntent.h + */ +typedef struct mntopt { + char *mo_name; /* option name */ + char **mo_cancel; /* list of options cancelled by this one */ + char *mo_arg; /* argument string for this option */ + int mo_flags; /* flags for this mount option */ + void *mo_data; /* filesystem specific data */ +} mntopt_t; + +/* + * Flags that apply to mount options + */ + +#define MO_SET 0x01 /* option is set */ +#define MO_NODISPLAY 0x02 /* option not listed in mnttab */ +#define MO_HASVALUE 0x04 /* option takes a value */ +#define MO_IGNORE 0x08 /* option ignored by parser */ +#define MO_DEFAULT MO_SET /* option is on by default */ +#define MO_TAG 0x10 /* flags a tag set by user program */ +#define MO_EMPTY 0x20 /* empty space in option table */ + +#define VFS_NOFORCEOPT 0x01 /* honor MO_IGNORE (don't set option) */ +#define VFS_DISPLAY 0x02 /* Turn off MO_NODISPLAY bit for opt */ +#define VFS_NODISPLAY 0x04 /* Turn on MO_NODISPLAY bit for opt */ +#define VFS_CREATEOPT 0x08 /* Create the opt if it's not there */ + +/* + * Structure holding mount option strings for the mounted file system. + */ +typedef struct mntopts { + uint_t mo_count; /* number of entries in table */ + mntopt_t *mo_list; /* list of mount options */ +} mntopts_t; + +void vfs_setmntopt(vfs_t *vfsp, const char *name, const char *arg, + int flags __unused); +void vfs_clearmntopt(vfs_t *vfsp, const char *name); +int vfs_optionisset(const vfs_t *vfsp, const char *opt, char **argp); +int mount_snapshot(kthread_t *td, vnode_t **vpp, const char *fstype, + char *fspath, char *fspec, int fsflags); + +typedef uint64_t vfs_feature_t; + +#define VFSFT_XVATTR 0x100000001 /* Supports xvattr for attrs */ +#define VFSFT_CASEINSENSITIVE 0x100000002 /* Supports case-insensitive */ +#define VFSFT_NOCASESENSITIVE 0x100000004 /* NOT case-sensitive */ +#define VFSFT_DIRENTFLAGS 0x100000008 /* Supports dirent flags */ +#define VFSFT_ACLONCREATE 0x100000010 /* Supports ACL on create */ +#define VFSFT_ACEMASKONACCESS 0x100000020 /* Can use ACEMASK for access */ +#define VFSFT_SYSATTR_VIEWS 0x100000040 /* Supports sysattr view i/f */ +#define VFSFT_ACCESS_FILTER 0x100000080 /* dirents filtered by access */ +#define VFSFT_REPARSE 0x100000100 /* Supports reparse point */ +#define VFSFT_ZEROCOPY_SUPPORTED 0x100000200 + /* Support loaning /returning cache buffer */ + +#define vfs_set_feature(vfsp, feature) do { } while (0) +#define vfs_clear_feature(vfsp, feature) do { } while (0) +#define vfs_has_feature(vfsp, feature) (0) + +#include +#endif /* _OPENSOLARIS_SYS_VFS_H_ */ diff --git a/include/os/freebsd/spl/sys/vm.h b/include/os/freebsd/spl/sys/vm.h new file mode 100644 index 00000000000..07ee6bc191a --- /dev/null +++ b/include/os/freebsd/spl/sys/vm.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2013 EMC Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_VM_H_ +#define _OPENSOLARIS_SYS_VM_H_ + +#include + +extern const int zfs_vm_pagerret_bad; +extern const int zfs_vm_pagerret_error; +extern const int zfs_vm_pagerret_ok; +extern const int zfs_vm_pagerput_sync; +extern const int zfs_vm_pagerput_inval; + +void zfs_vmobject_assert_wlocked(vm_object_t object); +void zfs_vmobject_wlock(vm_object_t object); +void zfs_vmobject_wunlock(vm_object_t object); + +static inline caddr_t +zfs_map_page(vm_page_t pp, struct sf_buf **sfp) +{ + *sfp = sf_buf_alloc(pp, 0); + return ((caddr_t)sf_buf_kva(*sfp)); +} + +static inline void +zfs_unmap_page(struct sf_buf *sf) +{ + sf_buf_free(sf); +} + +#endif /* _OPENSOLARIS_SYS_VM_H_ */ diff --git a/include/os/freebsd/spl/sys/vmsystm.h b/include/os/freebsd/spl/sys/vmsystm.h new file mode 100644 index 00000000000..0db34bbe438 --- /dev/null +++ b/include/os/freebsd/spl/sys/vmsystm.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SPL_VMSYSTM_H_ +#define _SPL_VMSYSTM_H_ + +#define xcopyout copyout + +#endif diff --git a/include/os/freebsd/spl/sys/vnode.h b/include/os/freebsd/spl/sys/vnode.h new file mode 100644 index 00000000000..e330bc079f6 --- /dev/null +++ b/include/os/freebsd/spl/sys/vnode.h @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_VNODE_H_ +#define _OPENSOLARIS_SYS_VNODE_H_ + +struct vnode; +struct vattr; +struct xucred; + +typedef struct flock flock64_t; +typedef struct vnode vnode_t; +typedef struct vattr vattr_t; +typedef enum vtype vtype_t; + +#include +#include +#include_next +#include +enum symfollow { NO_FOLLOW = NOFOLLOW }; + +#define NOCRED ((struct ucred *)0) /* no credential available */ +#define F_FREESP 11 /* Free file space */ + +#include +#include +#include_next +#include +#include +#include +#include_next +#include +#include +#include + +typedef struct vop_vector vnodeops_t; +#define VOP_FID VOP_VPTOFH +#define vop_fid vop_vptofh +#define vop_fid_args vop_vptofh_args +#define a_fid a_fhp + +#define IS_XATTRDIR(dvp) (0) + +#define v_count v_usecount + +#define rootvfs (rootvnode == NULL ? NULL : rootvnode->v_mount) + +static __inline int +vn_is_readonly(vnode_t *vp) +{ + return (vp->v_mount->mnt_flag & MNT_RDONLY); +} +#define vn_vfswlock(vp) (0) +#define vn_vfsunlock(vp) do { } while (0) +#define vn_ismntpt(vp) \ + ((vp)->v_type == VDIR && (vp)->v_mountedhere != NULL) +#define vn_mountedvfs(vp) ((vp)->v_mountedhere) +#define vn_has_cached_data(vp) \ + ((vp)->v_object != NULL && \ + (vp)->v_object->resident_page_count > 0) +#define vn_exists(vp) do { } while (0) +#define vn_invalid(vp) do { } while (0) +#define vn_renamepath(tdvp, svp, tnm, lentnm) do { } while (0) +#define vn_free(vp) do { } while (0) +#define vn_matchops(vp, vops) ((vp)->v_op == &(vops)) + +#define VN_HOLD(v) vref(v) +#define VN_RELE(v) vrele(v) +#define VN_URELE(v) vput(v) + +#define vnevent_create(vp, ct) do { } while (0) +#define vnevent_link(vp, ct) do { } while (0) +#define vnevent_remove(vp, dvp, name, ct) do { } while (0) +#define vnevent_rmdir(vp, dvp, name, ct) do { } while (0) +#define vnevent_rename_src(vp, dvp, name, ct) do { } while (0) +#define vnevent_rename_dest(vp, dvp, name, ct) do { } while (0) +#define vnevent_rename_dest_dir(vp, ct) do { } while (0) + +#define specvp(vp, rdev, type, cr) (VN_HOLD(vp), (vp)) +#define MANDLOCK(vp, mode) (0) + +/* + * We will use va_spare is place of Solaris' va_mask. + * This field is initialized in zfs_setattr(). + */ +#define va_mask va_spare +/* TODO: va_fileid is shorter than va_nodeid !!! */ +#define va_nodeid va_fileid +/* TODO: This field needs conversion! */ +#define va_nblocks va_bytes +#define va_blksize va_blocksize +#define va_seq va_gen + +#define MAXOFFSET_T OFF_MAX +#define EXCL 0 + +#define FCREAT O_CREAT +#define FTRUNC O_TRUNC +#define FEXCL O_EXCL +#define FDSYNC FFSYNC +#define FRSYNC FFSYNC +#define FSYNC FFSYNC +#define FOFFMAX 0x00 +#define FIGNORECASE 0x00 + +/* + * Attributes of interest to the caller of setattr or getattr. + */ +#define AT_MODE 0x00002 +#define AT_UID 0x00004 +#define AT_GID 0x00008 +#define AT_FSID 0x00010 +#define AT_NODEID 0x00020 +#define AT_NLINK 0x00040 +#define AT_SIZE 0x00080 +#define AT_ATIME 0x00100 +#define AT_MTIME 0x00200 +#define AT_CTIME 0x00400 +#define AT_RDEV 0x00800 +#define AT_BLKSIZE 0x01000 +#define AT_NBLOCKS 0x02000 +/* 0x04000 */ /* unused */ +#define AT_SEQ 0x08000 +/* + * If AT_XVATTR is set then there are additional bits to process in + * the xvattr_t's attribute bitmap. If this is not set then the bitmap + * MUST be ignored. Note that this bit must be set/cleared explicitly. + * That is, setting AT_ALL will NOT set AT_XVATTR. + */ +#define AT_XVATTR 0x10000 + +#define AT_ALL (AT_MODE|AT_UID|AT_GID|AT_FSID|AT_NODEID|\ + AT_NLINK|AT_SIZE|AT_ATIME|AT_MTIME|AT_CTIME|\ + AT_RDEV|AT_BLKSIZE|AT_NBLOCKS|AT_SEQ) + +#define AT_STAT (AT_MODE|AT_UID|AT_GID|AT_FSID|AT_NODEID|AT_NLINK|\ + AT_SIZE|AT_ATIME|AT_MTIME|AT_CTIME|AT_RDEV) + +#define AT_TIMES (AT_ATIME|AT_MTIME|AT_CTIME) + +#define AT_NOSET (AT_NLINK|AT_RDEV|AT_FSID|AT_NODEID|\ + AT_BLKSIZE|AT_NBLOCKS|AT_SEQ) + +static __inline void +vattr_init_mask(vattr_t *vap) +{ + + vap->va_mask = 0; + + if (vap->va_uid != (uid_t)VNOVAL) + vap->va_mask |= AT_UID; + if (vap->va_gid != (gid_t)VNOVAL) + vap->va_mask |= AT_GID; + if (vap->va_size != (u_quad_t)VNOVAL) + vap->va_mask |= AT_SIZE; + if (vap->va_atime.tv_sec != VNOVAL) + vap->va_mask |= AT_ATIME; + if (vap->va_mtime.tv_sec != VNOVAL) + vap->va_mask |= AT_MTIME; + if (vap->va_mode != (uint16_t)VNOVAL) + vap->va_mask |= AT_MODE; + if (vap->va_flags != VNOVAL) + vap->va_mask |= AT_XVATTR; +} + +#define RLIM64_INFINITY 0 + +static __inline int +vn_rename(char *from, char *to, enum uio_seg seg) +{ + + ASSERT(seg == UIO_SYSSPACE); + + return (kern_renameat(curthread, AT_FDCWD, from, AT_FDCWD, to, seg)); +} + +#include + +#endif /* _OPENSOLARIS_SYS_VNODE_H_ */ diff --git a/include/os/freebsd/spl/sys/vnode_impl.h b/include/os/freebsd/spl/sys/vnode_impl.h new file mode 100644 index 00000000000..78de740d991 --- /dev/null +++ b/include/os/freebsd/spl/sys/vnode_impl.h @@ -0,0 +1,268 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright 2017 RackTop Systems. + */ + +/* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +/* + * University Copyright- Copyright (c) 1982, 1986, 1988 + * The Regents of the University of California + * All Rights Reserved + * + * University Acknowledgment- Portions of this document are derived from + * software developed by the University of California, Berkeley, and its + * contributors. + */ + +#ifndef _SYS_VNODE_IMPL_H +#define _SYS_VNODE_IMPL_H + + +#define IS_DEVVP(vp) \ + ((vp)->v_type == VCHR || (vp)->v_type == VBLK || (vp)->v_type == VFIFO) + +#define V_XATTRDIR 0x0000 /* attribute unnamed directory */ + +#define AV_SCANSTAMP_SZ 32 /* length of anti-virus scanstamp */ + +/* + * The xvattr structure is really a variable length structure that + * is made up of: + * - The classic vattr_t (xva_vattr) + * - a 32 bit quantity (xva_mapsize) that specifies the size of the + * attribute bitmaps in 32 bit words. + * - A pointer to the returned attribute bitmap (needed because the + * previous element, the requested attribute bitmap) is variable lenth. + * - The requested attribute bitmap, which is an array of 32 bit words. + * Callers use the XVA_SET_REQ() macro to set the bits corresponding to + * the attributes that are being requested. + * - The returned attribute bitmap, which is an array of 32 bit words. + * File systems that support optional attributes use the XVA_SET_RTN() + * macro to set the bits corresponding to the attributes that are being + * returned. + * - The xoptattr_t structure which contains the attribute values + * + * xva_mapsize determines how many words in the attribute bitmaps. + * Immediately following the attribute bitmaps is the xoptattr_t. + * xva_getxoptattr() is used to get the pointer to the xoptattr_t + * section. + */ + +#define XVA_MAPSIZE 3 /* Size of attr bitmaps */ +#define XVA_MAGIC 0x78766174 /* Magic # for verification */ + +/* + * The xvattr structure is an extensible structure which permits optional + * attributes to be requested/returned. File systems may or may not support + * optional attributes. They do so at their own discretion but if they do + * support optional attributes, they must register the VFSFT_XVATTR feature + * so that the optional attributes can be set/retrived. + * + * The fields of the xvattr structure are: + * + * xva_vattr - The first element of an xvattr is a legacy vattr structure + * which includes the common attributes. If AT_XVATTR is set in the va_mask + * then the entire structure is treated as an xvattr. If AT_XVATTR is not + * set, then only the xva_vattr structure can be used. + * + * xva_magic - 0x78766174 (hex for "xvat"). Magic number for verification. + * + * xva_mapsize - Size of requested and returned attribute bitmaps. + * + * xva_rtnattrmapp - Pointer to xva_rtnattrmap[]. We need this since the + * size of the array before it, xva_reqattrmap[], could change which means + * the location of xva_rtnattrmap[] could change. This will allow unbundled + * file systems to find the location of xva_rtnattrmap[] when the sizes change. + * + * xva_reqattrmap[] - Array of requested attributes. Attributes are + * represented by a specific bit in a specific element of the attribute + * map array. Callers set the bits corresponding to the attributes + * that the caller wants to get/set. + * + * xva_rtnattrmap[] - Array of attributes that the file system was able to + * process. Not all file systems support all optional attributes. This map + * informs the caller which attributes the underlying file system was able + * to set/get. (Same structure as the requested attributes array in terms + * of each attribute corresponding to specific bits and array elements.) + * + * xva_xoptattrs - Structure containing values of optional attributes. + * These values are only valid if the corresponding bits in xva_reqattrmap + * are set and the underlying file system supports those attributes. + */ + + + +/* + * Attribute bits used in the extensible attribute's (xva's) attribute + * bitmaps. Note that the bitmaps are made up of a variable length number + * of 32-bit words. The convention is to use XAT{n}_{attrname} where "n" + * is the element in the bitmap (starting at 1). This convention is for + * the convenience of the maintainer to keep track of which element each + * attribute belongs to. + * + * NOTE THAT CONSUMERS MUST *NOT* USE THE XATn_* DEFINES DIRECTLY. CONSUMERS + * MUST USE THE XAT_* DEFINES. + */ +#define XAT0_INDEX 0LL /* Index into bitmap for XAT0 attrs */ +#define XAT0_CREATETIME 0x00000001 /* Create time of file */ +#define XAT0_ARCHIVE 0x00000002 /* Archive */ +#define XAT0_SYSTEM 0x00000004 /* System */ +#define XAT0_READONLY 0x00000008 /* Readonly */ +#define XAT0_HIDDEN 0x00000010 /* Hidden */ +#define XAT0_NOUNLINK 0x00000020 /* Nounlink */ +#define XAT0_IMMUTABLE 0x00000040 /* immutable */ +#define XAT0_APPENDONLY 0x00000080 /* appendonly */ +#define XAT0_NODUMP 0x00000100 /* nodump */ +#define XAT0_OPAQUE 0x00000200 /* opaque */ +#define XAT0_AV_QUARANTINED 0x00000400 /* anti-virus quarantine */ +#define XAT0_AV_MODIFIED 0x00000800 /* anti-virus modified */ +#define XAT0_AV_SCANSTAMP 0x00001000 /* anti-virus scanstamp */ +#define XAT0_REPARSE 0x00002000 /* FS reparse point */ +#define XAT0_GEN 0x00004000 /* object generation number */ +#define XAT0_OFFLINE 0x00008000 /* offline */ +#define XAT0_SPARSE 0x00010000 /* sparse */ + +/* Support for XAT_* optional attributes */ +#define XVA_MASK 0xffffffff /* Used to mask off 32 bits */ +#define XVA_SHFT 32 /* Used to shift index */ + +/* + * Used to pry out the index and attribute bits from the XAT_* attributes + * defined below. Note that we're masking things down to 32 bits then + * casting to uint32_t. + */ +#define XVA_INDEX(attr) ((uint32_t)(((attr) >> XVA_SHFT) & XVA_MASK)) +#define XVA_ATTRBIT(attr) ((uint32_t)((attr) & XVA_MASK)) + +/* + * The following defines present a "flat namespace" so that consumers don't + * need to keep track of which element belongs to which bitmap entry. + * + * NOTE THAT THESE MUST NEVER BE OR-ed TOGETHER + */ +#define XAT_CREATETIME ((XAT0_INDEX << XVA_SHFT) | XAT0_CREATETIME) +#define XAT_ARCHIVE ((XAT0_INDEX << XVA_SHFT) | XAT0_ARCHIVE) +#define XAT_SYSTEM ((XAT0_INDEX << XVA_SHFT) | XAT0_SYSTEM) +#define XAT_READONLY ((XAT0_INDEX << XVA_SHFT) | XAT0_READONLY) +#define XAT_HIDDEN ((XAT0_INDEX << XVA_SHFT) | XAT0_HIDDEN) +#define XAT_NOUNLINK ((XAT0_INDEX << XVA_SHFT) | XAT0_NOUNLINK) +#define XAT_IMMUTABLE ((XAT0_INDEX << XVA_SHFT) | XAT0_IMMUTABLE) +#define XAT_APPENDONLY ((XAT0_INDEX << XVA_SHFT) | XAT0_APPENDONLY) +#define XAT_NODUMP ((XAT0_INDEX << XVA_SHFT) | XAT0_NODUMP) +#define XAT_OPAQUE ((XAT0_INDEX << XVA_SHFT) | XAT0_OPAQUE) +#define XAT_AV_QUARANTINED ((XAT0_INDEX << XVA_SHFT) | XAT0_AV_QUARANTINED) +#define XAT_AV_MODIFIED ((XAT0_INDEX << XVA_SHFT) | XAT0_AV_MODIFIED) +#define XAT_AV_SCANSTAMP ((XAT0_INDEX << XVA_SHFT) | XAT0_AV_SCANSTAMP) +#define XAT_REPARSE ((XAT0_INDEX << XVA_SHFT) | XAT0_REPARSE) +#define XAT_GEN ((XAT0_INDEX << XVA_SHFT) | XAT0_GEN) +#define XAT_OFFLINE ((XAT0_INDEX << XVA_SHFT) | XAT0_OFFLINE) +#define XAT_SPARSE ((XAT0_INDEX << XVA_SHFT) | XAT0_SPARSE) + +/* + * The returned attribute map array (xva_rtnattrmap[]) is located past the + * requested attribute map array (xva_reqattrmap[]). Its location changes + * when the array sizes change. We use a separate pointer in a known location + * (xva_rtnattrmapp) to hold the location of xva_rtnattrmap[]. This is + * set in xva_init() + */ +#define XVA_RTNATTRMAP(xvap) ((xvap)->xva_rtnattrmapp) + +#define MODEMASK 07777 /* mode bits plus permission bits */ +#define PERMMASK 00777 /* permission bits */ + +/* + * VOP_ACCESS flags + */ +#define V_ACE_MASK 0x1 /* mask represents NFSv4 ACE permissions */ + +/* + * Flags for vnode operations. + */ +enum rm { RMFILE, RMDIRECTORY }; /* rm or rmdir (remove) */ +enum create { CRCREAT, CRMKNOD, CRMKDIR }; /* reason for create */ + +/* + * Structure used by various vnode operations to determine + * the context (pid, host, identity) of a caller. + * + * The cc_caller_id is used to identify one or more callers who invoke + * operations, possibly on behalf of others. For example, the NFS + * server could have it's own cc_caller_id which can be detected by + * vnode/vfs operations or (FEM) monitors on those operations. New + * caller IDs are generated by fs_new_caller_id(). + */ +typedef struct caller_context { + pid_t cc_pid; /* Process ID of the caller */ + int cc_sysid; /* System ID, used for remote calls */ + u_longlong_t cc_caller_id; /* Identifier for (set of) caller(s) */ + ulong_t cc_flags; +} caller_context_t; + +struct taskq; + +/* + * Flags for VOP_LOOKUP + * + * Defined in file.h, but also possible, FIGNORECASE and FSEARCH + * + */ +#define LOOKUP_DIR 0x01 /* want parent dir vp */ +#define LOOKUP_XATTR 0x02 /* lookup up extended attr dir */ +#define CREATE_XATTR_DIR 0x04 /* Create extended attr dir */ +#define LOOKUP_HAVE_SYSATTR_DIR 0x08 /* Already created virtual GFS dir */ + +/* + * Flags for VOP_READDIR + */ +#define V_RDDIR_ENTFLAGS 0x01 /* request dirent flags */ +#define V_RDDIR_ACCFILTER 0x02 /* filter out inaccessible dirents */ + +/* + * Public vnode manipulation functions. + */ + +void vn_rele_async(struct vnode *vp, struct taskq *taskq); + +#define VN_RELE_ASYNC(vp, taskq) { \ + vn_rele_async(vp, taskq); \ +} + +/* + * Flags to VOP_SETATTR/VOP_GETATTR. + */ +#define ATTR_UTIME 0x01 /* non-default utime(2) request */ +#define ATTR_EXEC 0x02 /* invocation from exec(2) */ +#define ATTR_COMM 0x04 /* yield common vp attributes */ +#define ATTR_HINT 0x08 /* information returned will be `hint' */ +#define ATTR_REAL 0x10 /* yield attributes of the real vp */ +#define ATTR_NOACLCHECK 0x20 /* Don't check ACL when checking permissions */ +#define ATTR_TRIGGER 0x40 /* Mount first if vnode is a trigger mount */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_VNODE_H */ diff --git a/include/os/freebsd/spl/sys/zmod.h b/include/os/freebsd/spl/sys/zmod.h new file mode 100644 index 00000000000..ba0267203ce --- /dev/null +++ b/include/os/freebsd/spl/sys/zmod.h @@ -0,0 +1,68 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _ZMOD_H +#define _ZMOD_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * zmod - RFC-1950-compatible decompression routines + * + * This file provides the public interfaces to zmod, an in-kernel RFC 1950 + * decompression library. More information about the implementation of these + * interfaces can be found in the usr/src/uts/common/zmod/ directory. + */ + +#define Z_OK 0 +#define Z_STREAM_END 1 +#define Z_NEED_DICT 2 +#define Z_ERRNO (-1) +#define Z_STREAM_ERROR (-2) +#define Z_DATA_ERROR (-3) +#define Z_MEM_ERROR (-4) +#define Z_BUF_ERROR (-5) +#define Z_VERSION_ERROR (-6) + +#define Z_NO_COMPRESSION 0 +#define Z_BEST_SPEED 1 +#define Z_BEST_COMPRESSION 9 +#define Z_DEFAULT_COMPRESSION (-1) + +extern int z_uncompress(void *, size_t *, const void *, size_t); +extern int z_compress(void *, size_t *, const void *, size_t); +extern int z_compress_level(void *, size_t *, const void *, size_t, int); +extern const char *z_strerror(int); + +#ifdef __cplusplus +} +#endif + +#endif /* _ZMOD_H */ diff --git a/include/os/freebsd/spl/sys/zone.h b/include/os/freebsd/spl/sys/zone.h new file mode 100644 index 00000000000..71a28adaf26 --- /dev/null +++ b/include/os/freebsd/spl/sys/zone.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _OPENSOLARIS_SYS_ZONE_H_ +#define _OPENSOLARIS_SYS_ZONE_H_ + +/* + * Macros to help with zone visibility restrictions. + */ + +#define GLOBAL_ZONEID 0 + +/* + * Is thread in the global zone? + */ +#define INGLOBALZONE(p) in_globalzone((p)) + + +extern boolean_t in_globalzone(struct proc *); + +/* + * Attach the given dataset to the given jail. + */ +extern int zone_dataset_attach(struct ucred *, const char *, int); + +/* + * Detach the given dataset to the given jail. + */ +extern int zone_dataset_detach(struct ucred *, const char *, int); + +/* + * Returns true if the named pool/dataset is visible in the current zone. + */ +extern int zone_dataset_visible(const char *, int *); + +/* + * Safely get the hostid of the specified zone (defaults to machine's hostid + * if the specified zone doesn't emulate a hostid). Passing NULL retrieves + * the global zone's (i.e., physical system's) hostid. + */ +extern uint32_t zone_get_hostid(void *); + +#endif /* !_OPENSOLARIS_SYS_ZONE_H_ */ diff --git a/include/os/freebsd/zfs/Makefile.am b/include/os/freebsd/zfs/Makefile.am new file mode 100644 index 00000000000..081839c48c8 --- /dev/null +++ b/include/os/freebsd/zfs/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = sys diff --git a/include/os/freebsd/zfs/sys/Makefile.am b/include/os/freebsd/zfs/sys/Makefile.am new file mode 100644 index 00000000000..1dec5ef5ff5 --- /dev/null +++ b/include/os/freebsd/zfs/sys/Makefile.am @@ -0,0 +1,14 @@ +KERNEL_H = \ + $(top_srcdir)/include/os/freebsd/zfs/sys/freebsd_crypto.h \ + $(top_srcdir)/include/os/freebsd/zfs/sys/sha2.h \ + $(top_srcdir)/include/os/freebsd/zfs/sys/vdev_os.h \ + $(top_srcdir)/include/os/freebsd/zfs/sys/zfs_context_os.h \ + $(top_srcdir)/include/os/freebsd/zfs/sys/zfs_ctldir.h \ + $(top_srcdir)/include/os/freebsd/zfs/sys/zfs_dir.h \ + $(top_srcdir)/include/os/freebsd/zfs/sys/zfs_ioctl_compat.h \ + $(top_srcdir)/include/os/freebsd/zfs/sys/zfs_vfsops.h \ + $(top_srcdir)/include/os/freebsd/zfs/sys/zfs_vnops.h \ + $(top_srcdir)/include/os/freebsd/zfs/sys/zfs_znode_impl.h \ + $(top_srcdir)/include/os/freebsd/zfs/sys/zpl.h + +EXTRA_DIST = $(KERNEL_H) diff --git a/include/os/freebsd/zfs/sys/freebsd_crypto.h b/include/os/freebsd/zfs/sys/freebsd_crypto.h new file mode 100644 index 00000000000..08e058d6aff --- /dev/null +++ b/include/os/freebsd/zfs/sys/freebsd_crypto.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2018 Sean Eric Fagan + * Portions Copyright (c) 2005-2011 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Portions of this file were taken from GELI's implementation of hmac. + * + * $FreeBSD$ + */ + +#ifndef _ZFS_FREEBSD_CRYPTO_H +#define _ZFS_FREEBSD_CRYPTO_H + +#include +#include +#include +#include +#include + +#define SUN_CKM_AES_CCM "CKM_AES_CCM" +#define SUN_CKM_AES_GCM "CKM_AES_GCM" +#define SUN_CKM_SHA512_HMAC "CKM_SHA512_HMAC" + +#define CRYPTO_KEY_RAW 1 + +#define CRYPTO_BITS2BYTES(n) ((n) == 0 ? 0 : (((n) - 1) >> 3) + 1) +#define CRYPTO_BYTES2BITS(n) ((n) << 3) + +struct zio_crypt_info; + +typedef struct freebsd_crypt_session { + struct mtx fs_lock; + crypto_session_t fs_sid; + boolean_t fs_done; +} freebsd_crypt_session_t; + +/* + * Unused types to minimize code differences. + */ +typedef void *crypto_mechanism_t; +typedef void *crypto_ctx_template_t; +/* + * Unlike the ICP crypto_key type, this only + * supports (the equivalent of + * CRYPTO_KEY_RAW). + */ +typedef struct crypto_key { + int ck_format; /* Unused, but minimizes code diff */ + void *ck_data; + size_t ck_length; +} crypto_key_t; + +typedef struct hmac_ctx { + SHA512_CTX innerctx; + SHA512_CTX outerctx; +} *crypto_context_t; + +/* + * The only algorithm ZFS uses for hashing is SHA512_HMAC. + */ +void crypto_mac(const crypto_key_t *key, const void *in_data, + size_t in_data_size, void *out_data, size_t out_data_size); +void crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *key); +void crypto_mac_update(struct hmac_ctx *ctx, const void *data, + size_t data_size); +void crypto_mac_final(struct hmac_ctx *ctx, void *out_data, + size_t out_data_size); + +int freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, + struct zio_crypt_info *, crypto_key_t *); +void freebsd_crypt_freesession(freebsd_crypt_session_t *sessp); + +int freebsd_crypt_uio(boolean_t, freebsd_crypt_session_t *, + struct zio_crypt_info *, uio_t *, crypto_key_t *, uint8_t *, + size_t, size_t); + +#endif /* _ZFS_FREEBSD_CRYPTO_H */ diff --git a/include/os/freebsd/zfs/sys/sha2.h b/include/os/freebsd/zfs/sys/sha2.h new file mode 100644 index 00000000000..9d848e1fc2d --- /dev/null +++ b/include/os/freebsd/zfs/sys/sha2.h @@ -0,0 +1,200 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +/* Copyright 2013 Saso Kiselkov. All rights reserved. */ + +#ifndef _SYS_SHA2_H +#define _SYS_SHA2_H + +#include /* for uint_* */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define SHA2_HMAC_MIN_KEY_LEN 1 /* SHA2-HMAC min key length in bytes */ +#define SHA2_HMAC_MAX_KEY_LEN INT_MAX /* SHA2-HMAC max key length in bytes */ + +#define SHA256_DIGEST_LENGTH 32 /* SHA256 digest length in bytes */ +#define SHA384_DIGEST_LENGTH 48 /* SHA384 digest length in bytes */ +#define SHA512_DIGEST_LENGTH 64 /* SHA512 digest length in bytes */ + +/* Truncated versions of SHA-512 according to FIPS-180-4, section 5.3.6 */ +#define SHA512_224_DIGEST_LENGTH 28 /* SHA512/224 digest length */ +#define SHA512_256_DIGEST_LENGTH 32 /* SHA512/256 digest length */ + +#define SHA256_HMAC_BLOCK_SIZE 64 /* SHA256-HMAC block size */ +#define SHA512_HMAC_BLOCK_SIZE 128 /* SHA512-HMAC block size */ + +#define SHA256 0 +#define SHA256_HMAC 1 +#define SHA256_HMAC_GEN 2 +#define SHA384 3 +#define SHA384_HMAC 4 +#define SHA384_HMAC_GEN 5 +#define SHA512 6 +#define SHA512_HMAC 7 +#define SHA512_HMAC_GEN 8 +#define SHA512_224 9 +#define SHA512_256 10 + +/* + * SHA2 context. + * The contents of this structure are a private interface between the + * Init/Update/Final calls of the functions defined below. + * Callers must never attempt to read or write any of the fields + * in this structure directly. + */ + +#include +#include +#include +#include +typedef struct { + uint32_t algotype; /* Algorithm Type */ + union { + SHA256_CTX SHA256_ctx; + SHA384_CTX SHA384_ctx; + SHA512_CTX SHA512_ctx; + }; +} SHA2_CTX; + +extern void SHA256Init(SHA256_CTX *); + +extern void SHA256Update(SHA256_CTX *, const void *, size_t); + +extern void SHA256Final(void *, SHA256_CTX *); + +extern void SHA384Init(SHA384_CTX *); + +extern void SHA384Update(SHA384_CTX *, const void *, size_t); + +extern void SHA384Final(void *, SHA384_CTX *); + +extern void SHA512Init(SHA512_CTX *); + +extern void SHA512Update(SHA512_CTX *, const void *, size_t); + +extern void SHA512Final(void *, SHA512_CTX *); + + +static inline void +SHA2Init(uint64_t mech, SHA2_CTX *c) +{ + switch (mech) { + case SHA256: + SHA256_Init(&c->SHA256_ctx); + break; + case SHA384: + SHA384_Init(&c->SHA384_ctx); + break; + case SHA512: + SHA512_Init(&c->SHA512_ctx); + break; + case SHA512_256: + SHA512_256_Init(&c->SHA512_ctx); + break; + default: + panic("unknown mechanism %lu", mech); + } + c->algotype = (uint32_t)mech; +} + +static inline void +SHA2Update(SHA2_CTX *c, const void *p, size_t s) +{ + switch (c->algotype) { + case SHA256: + SHA256_Update(&c->SHA256_ctx, p, s); + break; + case SHA384: + SHA384_Update(&c->SHA384_ctx, p, s); + break; + case SHA512: + SHA512_Update(&c->SHA512_ctx, p, s); + break; + case SHA512_256: + SHA512_256_Update(&c->SHA512_ctx, p, s); + break; + default: + panic("unknown mechanism %d", c->algotype); + } +} + +static inline void +SHA2Final(void *p, SHA2_CTX *c) +{ + switch (c->algotype) { + case SHA256: + SHA256_Final(p, &c->SHA256_ctx); + break; + case SHA384: + SHA384_Final(p, &c->SHA384_ctx); + break; + case SHA512: + SHA512_Final(p, &c->SHA512_ctx); + break; + case SHA512_256: + SHA512_256_Final(p, &c->SHA512_ctx); + break; + default: + panic("unknown mechanism %d", c->algotype); + } +} + +#ifdef _SHA2_IMPL +/* + * The following types/functions are all private to the implementation + * of the SHA2 functions and must not be used by consumers of the interface + */ + +/* + * List of support mechanisms in this module. + * + * It is important to note that in the module, division or modulus calculations + * are used on the enumerated type to determine which mechanism is being used; + * therefore, changing the order or additional mechanisms should be done + * carefully + */ +typedef enum sha2_mech_type { + SHA256_MECH_INFO_TYPE, /* SUN_CKM_SHA256 */ + SHA256_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC */ + SHA256_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC_GENERAL */ + SHA384_MECH_INFO_TYPE, /* SUN_CKM_SHA384 */ + SHA384_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA384_HMAC */ + SHA384_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA384_HMAC_GENERAL */ + SHA512_MECH_INFO_TYPE, /* SUN_CKM_SHA512 */ + SHA512_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA512_HMAC */ + SHA512_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA512_HMAC_GENERAL */ + SHA512_224_MECH_INFO_TYPE, /* SUN_CKM_SHA512_224 */ + SHA512_256_MECH_INFO_TYPE /* SUN_CKM_SHA512_256 */ +} sha2_mech_type_t; + +#endif /* _SHA2_IMPL */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_SHA2_H */ diff --git a/include/os/freebsd/zfs/sys/vdev_os.h b/include/os/freebsd/zfs/sys/vdev_os.h new file mode 100644 index 00000000000..e2780fdbb67 --- /dev/null +++ b/include/os/freebsd/zfs/sys/vdev_os.h @@ -0,0 +1,30 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + */ + +#ifndef _SYS_VDEV_OS_H +#define _SYS_VDEV_OS_H + +extern int vdev_label_write_pad2(vdev_t *vd, const char *buf, size_t size); +#endif diff --git a/include/os/freebsd/zfs/sys/zfs_context_os.h b/include/os/freebsd/zfs/sys/zfs_context_os.h new file mode 100644 index 00000000000..529ba720454 --- /dev/null +++ b/include/os/freebsd/zfs/sys/zfs_context_os.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef ZFS_CONTEXT_OS_H_ +#define ZFS_CONTEXT_OS_H_ + +#include +#include +#include +#include_next +#include +#include +#include +#include + +#define cv_wait_io(cv, mp) cv_wait(cv, mp) +#define cv_wait_io_sig(cv, mp) cv_wait_sig(cv, mp) + +#define cond_resched() kern_yield(PRI_USER) + +#define taskq_create_sysdc(a, b, d, e, p, dc, f) \ + (taskq_create(a, b, maxclsyspri, d, e, f)) + +#define tsd_create(keyp, destructor) do { \ + *(keyp) = osd_thread_register((destructor)); \ + KASSERT(*(keyp) > 0, ("cannot register OSD")); \ +} while (0) + +#define tsd_destroy(keyp) osd_thread_deregister(*(keyp)) +#define tsd_get(key) osd_thread_get(curthread, (key)) +#define tsd_set(key, value) osd_thread_set(curthread, (key), (value)) +#define fm_panic panic + +#define cond_resched() kern_yield(PRI_USER) +extern int zfs_debug_level; +extern struct mtx zfs_debug_mtx; +#define ZFS_LOG(lvl, ...) do { \ + if (((lvl) & 0xff) <= zfs_debug_level) { \ + mtx_lock(&zfs_debug_mtx); \ + printf("%s:%u[%d]: ", \ + __func__, __LINE__, (lvl)); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + if ((lvl) & 0x100) \ + kdb_backtrace(); \ + mtx_unlock(&zfs_debug_mtx); \ + } \ +} while (0) + +#define MSEC_TO_TICK(msec) ((msec) / (MILLISEC / hz)) +extern int hz; +extern int tick; +typedef int fstrans_cookie_t; +#define spl_fstrans_mark() (0) +#define spl_fstrans_unmark(x) (x = 0) +#define signal_pending(x) SIGPENDING(x) +#define current curthread +#define thread_join(x) +#define cv_wait_io(cv, mp) cv_wait(cv, mp) +typedef struct opensolaris_utsname utsname_t; +extern utsname_t *utsname(void); +extern int spa_import_rootpool(const char *name); +#else +#if BYTE_ORDER != BIG_ENDIAN +#undef _BIG_ENDIAN +#endif +#endif diff --git a/include/os/freebsd/zfs/sys/zfs_ctldir.h b/include/os/freebsd/zfs/sys/zfs_ctldir.h new file mode 100644 index 00000000000..28a026603f0 --- /dev/null +++ b/include/os/freebsd/zfs/sys/zfs_ctldir.h @@ -0,0 +1,65 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + */ + +#ifndef _ZFS_CTLDIR_H +#define _ZFS_CTLDIR_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define ZFS_CTLDIR_NAME ".zfs" + +#define zfs_has_ctldir(zdp) \ + ((zdp)->z_id == (zdp)->z_zfsvfs->z_root && \ + ((zdp)->z_zfsvfs->z_ctldir != NULL)) +#define zfs_show_ctldir(zdp) \ + (zfs_has_ctldir(zdp) && \ + ((zdp)->z_zfsvfs->z_show_ctldir)) + +void zfsctl_create(zfsvfs_t *); +void zfsctl_destroy(zfsvfs_t *); +int zfsctl_root(zfsvfs_t *, int, vnode_t **); +void zfsctl_init(void); +void zfsctl_fini(void); +boolean_t zfsctl_is_node(vnode_t *); +int zfsctl_snapshot_unmount(char *snapname, int flags); +int zfsctl_rename_snapshot(const char *from, const char *to); +int zfsctl_destroy_snapshot(const char *snapname, int force); +int zfsctl_umount_snapshots(vfs_t *, int, cred_t *); + +int zfsctl_lookup_objset(vfs_t *vfsp, uint64_t objsetid, zfsvfs_t **zfsvfsp); + +#define ZFSCTL_INO_ROOT 0x1 +#define ZFSCTL_INO_SNAPDIR 0x2 + +#ifdef __cplusplus +} +#endif + +#endif /* _ZFS_CTLDIR_H */ diff --git a/include/os/freebsd/zfs/sys/zfs_dir.h b/include/os/freebsd/zfs/sys/zfs_dir.h new file mode 100644 index 00000000000..f6f8ab5c4e6 --- /dev/null +++ b/include/os/freebsd/zfs/sys/zfs_dir.h @@ -0,0 +1,74 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2010 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_FS_ZFS_DIR_H +#define _SYS_FS_ZFS_DIR_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* zfs_dirent_lock() flags */ +#define ZNEW 0x0001 /* entry should not exist */ +#define ZEXISTS 0x0002 /* entry should exist */ +#define ZSHARED 0x0004 /* shared access (zfs_dirlook()) */ +#define ZXATTR 0x0008 /* we want the xattr dir */ +#define ZRENAMING 0x0010 /* znode is being renamed */ +#define ZCILOOK 0x0020 /* case-insensitive lookup requested */ +#define ZCIEXACT 0x0040 /* c-i requires c-s match (rename) */ +#define ZHAVELOCK 0x0080 /* z_name_lock is already held */ + +/* mknode flags */ +#define IS_ROOT_NODE 0x01 /* create a root node */ +#define IS_XATTR 0x02 /* create an extended attribute node */ + +extern int zfs_dirent_lookup(znode_t *, const char *, znode_t **, int); +extern int zfs_link_create(znode_t *, const char *, znode_t *, dmu_tx_t *, int); +extern int zfs_link_destroy(znode_t *, const char *, znode_t *, dmu_tx_t *, int, + boolean_t *); +#if 0 +extern int zfs_dirlook(vnode_t *, const char *, vnode_t **, int); +#else +extern int zfs_dirlook(znode_t *, const char *name, znode_t **); +#endif +extern void zfs_mknode(znode_t *, vattr_t *, dmu_tx_t *, cred_t *, + uint_t, znode_t **, zfs_acl_ids_t *); +extern void zfs_rmnode(znode_t *); +extern boolean_t zfs_dirempty(znode_t *); +extern void zfs_unlinked_add(znode_t *, dmu_tx_t *); +extern void zfs_unlinked_drain(zfsvfs_t *zfsvfs); +extern int zfs_sticky_remove_access(znode_t *, znode_t *, cred_t *cr); +extern int zfs_get_xattrdir(znode_t *, znode_t **, cred_t *, int); +extern int zfs_make_xattrdir(znode_t *, vattr_t *, znode_t **, cred_t *); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_FS_ZFS_DIR_H */ diff --git a/include/os/freebsd/zfs/sys/zfs_ioctl_compat.h b/include/os/freebsd/zfs/sys/zfs_ioctl_compat.h new file mode 100644 index 00000000000..a8db8bee520 --- /dev/null +++ b/include/os/freebsd/zfs/sys/zfs_ioctl_compat.h @@ -0,0 +1,677 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2014 Xin Li . All rights reserved. + * Copyright 2013 Martin Matuska . All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_ZFS_IOCTL_COMPAT_H +#define _SYS_ZFS_IOCTL_COMPAT_H + +#include +#include +#include +#include +#include + +#ifdef _KERNEL +#include +#endif /* _KERNEL */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Backwards ioctl compatibility + */ + +/* ioctl versions for vfs.zfs.version.ioctl */ +#define ZFS_IOCVER_UNDEF -1 +#define ZFS_IOCVER_NONE 0 +#define ZFS_IOCVER_DEADMAN 1 +#define ZFS_IOCVER_LZC 2 +#define ZFS_IOCVER_ZCMD 3 +#define ZFS_IOCVER_EDBP 4 +#define ZFS_IOCVER_RESUME 5 +#define ZFS_IOCVER_INLANES 6 +#define ZFS_IOCVER_PAD 7 +#define ZFS_IOCVER_FREEBSD ZFS_IOCVER_PAD +#define ZFS_IOCVER_ZOF 15 + +/* compatibility conversion flag */ +#define ZFS_CMD_COMPAT_NONE 0 +#define ZFS_CMD_COMPAT_V15 1 +#define ZFS_CMD_COMPAT_V28 2 +#define ZFS_CMD_COMPAT_DEADMAN 3 +#define ZFS_CMD_COMPAT_LZC 4 +#define ZFS_CMD_COMPAT_ZCMD 5 +#define ZFS_CMD_COMPAT_EDBP 6 +#define ZFS_CMD_COMPAT_RESUME 7 +#define ZFS_CMD_COMPAT_INLANES 8 + +#define ZFS_IOC_COMPAT_PASS 254 +#define ZFS_IOC_COMPAT_FAIL 255 + +#define ZFS_IOCREQ(ioreq) ((ioreq) & 0xff) + +typedef struct zfs_iocparm { + uint32_t zfs_ioctl_version; + uint64_t zfs_cmd; + uint64_t zfs_cmd_size; +} zfs_iocparm_t; + +typedef struct zinject_record_v15 { + uint64_t zi_objset; + uint64_t zi_object; + uint64_t zi_start; + uint64_t zi_end; + uint64_t zi_guid; + uint32_t zi_level; + uint32_t zi_error; + uint64_t zi_type; + uint32_t zi_freq; + uint32_t zi_failfast; +} zinject_record_v15_t; + +typedef struct zfs_cmd_v15 { + char zc_name[MAXPATHLEN]; + char zc_value[MAXPATHLEN]; + char zc_string[MAXNAMELEN]; + uint64_t zc_guid; + uint64_t zc_nvlist_conf; /* really (char *) */ + uint64_t zc_nvlist_conf_size; + uint64_t zc_nvlist_src; /* really (char *) */ + uint64_t zc_nvlist_src_size; + uint64_t zc_nvlist_dst; /* really (char *) */ + uint64_t zc_nvlist_dst_size; + uint64_t zc_cookie; + uint64_t zc_objset_type; + uint64_t zc_perm_action; + uint64_t zc_history; /* really (char *) */ + uint64_t zc_history_len; + uint64_t zc_history_offset; + uint64_t zc_obj; + zfs_share_t zc_share; + uint64_t zc_jailid; + dmu_objset_stats_t zc_objset_stats; + struct drr_begin zc_begin_record; + zinject_record_v15_t zc_inject_record; +} zfs_cmd_v15_t; + +typedef struct zinject_record_v28 { + uint64_t zi_objset; + uint64_t zi_object; + uint64_t zi_start; + uint64_t zi_end; + uint64_t zi_guid; + uint32_t zi_level; + uint32_t zi_error; + uint64_t zi_type; + uint32_t zi_freq; + uint32_t zi_failfast; + char zi_func[MAXNAMELEN]; + uint32_t zi_iotype; + int32_t zi_duration; + uint64_t zi_timer; +} zinject_record_v28_t; + +typedef struct zfs_cmd_v28 { + char zc_name[MAXPATHLEN]; + char zc_value[MAXPATHLEN * 2]; + char zc_string[MAXNAMELEN]; + char zc_top_ds[MAXPATHLEN]; + uint64_t zc_guid; + uint64_t zc_nvlist_conf; /* really (char *) */ + uint64_t zc_nvlist_conf_size; + uint64_t zc_nvlist_src; /* really (char *) */ + uint64_t zc_nvlist_src_size; + uint64_t zc_nvlist_dst; /* really (char *) */ + uint64_t zc_nvlist_dst_size; + uint64_t zc_cookie; + uint64_t zc_objset_type; + uint64_t zc_perm_action; + uint64_t zc_history; /* really (char *) */ + uint64_t zc_history_len; + uint64_t zc_history_offset; + uint64_t zc_obj; + uint64_t zc_iflags; /* internal to zfs(7fs) */ + zfs_share_t zc_share; + uint64_t zc_jailid; + dmu_objset_stats_t zc_objset_stats; + struct drr_begin zc_begin_record; + zinject_record_v28_t zc_inject_record; + boolean_t zc_defer_destroy; + boolean_t zc_temphold; + uint64_t zc_action_handle; + int zc_cleanup_fd; + uint8_t zc_simple; + uint8_t zc_pad[3]; /* alignment */ + uint64_t zc_sendobj; + uint64_t zc_fromobj; + uint64_t zc_createtxg; + zfs_stat_t zc_stat; +} zfs_cmd_v28_t; + +typedef struct zinject_record_deadman { + uint64_t zi_objset; + uint64_t zi_object; + uint64_t zi_start; + uint64_t zi_end; + uint64_t zi_guid; + uint32_t zi_level; + uint32_t zi_error; + uint64_t zi_type; + uint32_t zi_freq; + uint32_t zi_failfast; + char zi_func[MAXNAMELEN]; + uint32_t zi_iotype; + int32_t zi_duration; + uint64_t zi_timer; + uint32_t zi_cmd; + uint32_t zi_pad; +} zinject_record_deadman_t; + +typedef struct zfs_cmd_deadman { + char zc_name[MAXPATHLEN]; + char zc_value[MAXPATHLEN * 2]; + char zc_string[MAXNAMELEN]; + char zc_top_ds[MAXPATHLEN]; + uint64_t zc_guid; + uint64_t zc_nvlist_conf; /* really (char *) */ + uint64_t zc_nvlist_conf_size; + uint64_t zc_nvlist_src; /* really (char *) */ + uint64_t zc_nvlist_src_size; + uint64_t zc_nvlist_dst; /* really (char *) */ + uint64_t zc_nvlist_dst_size; + uint64_t zc_cookie; + uint64_t zc_objset_type; + uint64_t zc_perm_action; + uint64_t zc_history; /* really (char *) */ + uint64_t zc_history_len; + uint64_t zc_history_offset; + uint64_t zc_obj; + uint64_t zc_iflags; /* internal to zfs(7fs) */ + zfs_share_t zc_share; + uint64_t zc_jailid; + dmu_objset_stats_t zc_objset_stats; + struct drr_begin zc_begin_record; + /* zc_inject_record doesn't change in libzfs_core */ + zinject_record_deadman_t zc_inject_record; + boolean_t zc_defer_destroy; + boolean_t zc_temphold; + uint64_t zc_action_handle; + int zc_cleanup_fd; + uint8_t zc_simple; + uint8_t zc_pad[3]; /* alignment */ + uint64_t zc_sendobj; + uint64_t zc_fromobj; + uint64_t zc_createtxg; + zfs_stat_t zc_stat; +} zfs_cmd_deadman_t; + +typedef struct zfs_cmd_zcmd { + char zc_name[MAXPATHLEN]; /* name of pool or dataset */ + uint64_t zc_nvlist_src; /* really (char *) */ + uint64_t zc_nvlist_src_size; + uint64_t zc_nvlist_dst; /* really (char *) */ + uint64_t zc_nvlist_dst_size; + boolean_t zc_nvlist_dst_filled; /* put an nvlist in dst? */ + int zc_pad2; + + /* + * The following members are for legacy ioctls which haven't been + * converted to the new method. + */ + uint64_t zc_history; /* really (char *) */ + char zc_value[MAXPATHLEN * 2]; + char zc_string[MAXNAMELEN]; + uint64_t zc_guid; + uint64_t zc_nvlist_conf; /* really (char *) */ + uint64_t zc_nvlist_conf_size; + uint64_t zc_cookie; + uint64_t zc_objset_type; + uint64_t zc_perm_action; + uint64_t zc_history_len; + uint64_t zc_history_offset; + uint64_t zc_obj; + uint64_t zc_iflags; /* internal to zfs(7fs) */ + zfs_share_t zc_share; + uint64_t zc_jailid; + dmu_objset_stats_t zc_objset_stats; + struct drr_begin zc_begin_record; + zinject_record_deadman_t zc_inject_record; + boolean_t zc_defer_destroy; + boolean_t zc_temphold; + uint64_t zc_action_handle; + int zc_cleanup_fd; + uint8_t zc_simple; + uint8_t zc_pad[3]; /* alignment */ + uint64_t zc_sendobj; + uint64_t zc_fromobj; + uint64_t zc_createtxg; + zfs_stat_t zc_stat; +} zfs_cmd_zcmd_t; + +typedef struct zfs_cmd_edbp { + char zc_name[MAXPATHLEN]; /* name of pool or dataset */ + uint64_t zc_nvlist_src; /* really (char *) */ + uint64_t zc_nvlist_src_size; + uint64_t zc_nvlist_dst; /* really (char *) */ + uint64_t zc_nvlist_dst_size; + boolean_t zc_nvlist_dst_filled; /* put an nvlist in dst? */ + int zc_pad2; + + /* + * The following members are for legacy ioctls which haven't been + * converted to the new method. + */ + uint64_t zc_history; /* really (char *) */ + char zc_value[MAXPATHLEN * 2]; + char zc_string[MAXNAMELEN]; + uint64_t zc_guid; + uint64_t zc_nvlist_conf; /* really (char *) */ + uint64_t zc_nvlist_conf_size; + uint64_t zc_cookie; + uint64_t zc_objset_type; + uint64_t zc_perm_action; + uint64_t zc_history_len; + uint64_t zc_history_offset; + uint64_t zc_obj; + uint64_t zc_iflags; /* internal to zfs(7fs) */ + zfs_share_t zc_share; + uint64_t zc_jailid; + dmu_objset_stats_t zc_objset_stats; + struct drr_begin zc_begin_record; + zinject_record_deadman_t zc_inject_record; + uint32_t zc_defer_destroy; + uint32_t zc_flags; + uint64_t zc_action_handle; + int zc_cleanup_fd; + uint8_t zc_simple; + uint8_t zc_pad[3]; /* alignment */ + uint64_t zc_sendobj; + uint64_t zc_fromobj; + uint64_t zc_createtxg; + zfs_stat_t zc_stat; +} zfs_cmd_edbp_t; + +typedef struct zfs_cmd_resume { + char zc_name[MAXPATHLEN]; /* name of pool or dataset */ + uint64_t zc_nvlist_src; /* really (char *) */ + uint64_t zc_nvlist_src_size; + uint64_t zc_nvlist_dst; /* really (char *) */ + uint64_t zc_nvlist_dst_size; + boolean_t zc_nvlist_dst_filled; /* put an nvlist in dst? */ + int zc_pad2; + + /* + * The following members are for legacy ioctls which haven't been + * converted to the new method. + */ + uint64_t zc_history; /* really (char *) */ + char zc_value[MAXPATHLEN * 2]; + char zc_string[MAXNAMELEN]; + uint64_t zc_guid; + uint64_t zc_nvlist_conf; /* really (char *) */ + uint64_t zc_nvlist_conf_size; + uint64_t zc_cookie; + uint64_t zc_objset_type; + uint64_t zc_perm_action; + uint64_t zc_history_len; + uint64_t zc_history_offset; + uint64_t zc_obj; + uint64_t zc_iflags; /* internal to zfs(7fs) */ + zfs_share_t zc_share; + uint64_t zc_jailid; + dmu_objset_stats_t zc_objset_stats; + dmu_replay_record_t zc_begin_record; + zinject_record_deadman_t zc_inject_record; + uint32_t zc_defer_destroy; + uint32_t zc_flags; + uint64_t zc_action_handle; + int zc_cleanup_fd; + uint8_t zc_simple; + boolean_t zc_resumable; + uint64_t zc_sendobj; + uint64_t zc_fromobj; + uint64_t zc_createtxg; + zfs_stat_t zc_stat; +} zfs_cmd_resume_t; + +typedef struct zfs_cmd_inlanes { + char zc_name[MAXPATHLEN]; /* name of pool or dataset */ + uint64_t zc_nvlist_src; /* really (char *) */ + uint64_t zc_nvlist_src_size; + uint64_t zc_nvlist_dst; /* really (char *) */ + uint64_t zc_nvlist_dst_size; + boolean_t zc_nvlist_dst_filled; /* put an nvlist in dst? */ + int zc_pad2; + + /* + * The following members are for legacy ioctls which haven't been + * converted to the new method. + */ + uint64_t zc_history; /* really (char *) */ + char zc_value[MAXPATHLEN * 2]; + char zc_string[MAXNAMELEN]; + uint64_t zc_guid; + uint64_t zc_nvlist_conf; /* really (char *) */ + uint64_t zc_nvlist_conf_size; + uint64_t zc_cookie; + uint64_t zc_objset_type; + uint64_t zc_perm_action; + uint64_t zc_history_len; + uint64_t zc_history_offset; + uint64_t zc_obj; + uint64_t zc_iflags; /* internal to zfs(7fs) */ + zfs_share_t zc_share; + uint64_t zc_jailid; + dmu_objset_stats_t zc_objset_stats; + dmu_replay_record_t zc_begin_record; + zinject_record_t zc_inject_record; + uint32_t zc_defer_destroy; + uint32_t zc_flags; + uint64_t zc_action_handle; + int zc_cleanup_fd; + uint8_t zc_simple; + boolean_t zc_resumable; + uint64_t zc_sendobj; + uint64_t zc_fromobj; + uint64_t zc_createtxg; + zfs_stat_t zc_stat; +} zfs_cmd_inlanes_t; + +#ifdef _KERNEL +/* + * Note: this struct must have the same layout in 32-bit and 64-bit, so + * that 32-bit processes (like /sbin/zfs) can pass it to the 64-bit + * kernel. Therefore, we add padding to it so that no "hidden" padding + * is automatically added on 64-bit (but not on 32-bit). + */ +typedef struct zfs_cmd_legacy { + char zc_name[MAXPATHLEN]; /* name of pool or dataset */ + uint64_t zc_nvlist_src; /* really (char *) */ + uint64_t zc_nvlist_src_size; + uint64_t zc_nvlist_dst; /* really (char *) */ + uint64_t zc_nvlist_dst_size; + boolean_t zc_nvlist_dst_filled; /* put an nvlist in dst? */ + int zc_pad2; + + /* + * The following members are for legacy ioctls which haven't been + * converted to the new method. + */ + uint64_t zc_history; /* really (char *) */ + char zc_value[MAXPATHLEN * 2]; + char zc_string[MAXNAMELEN]; + uint64_t zc_guid; + uint64_t zc_nvlist_conf; /* really (char *) */ + uint64_t zc_nvlist_conf_size; + uint64_t zc_cookie; + uint64_t zc_objset_type; + uint64_t zc_perm_action; + uint64_t zc_history_len; + uint64_t zc_history_offset; + uint64_t zc_obj; + uint64_t zc_iflags; /* internal to zfs(7fs) */ + zfs_share_t zc_share; + uint64_t zc_jailid; + + dmu_objset_stats_t zc_objset_stats; + uint64_t zc_freebsd_drr_pad; + struct drr_begin zc_begin_record; + zinject_record_t zc_inject_record; + uint32_t zc_defer_destroy; + uint32_t zc_flags; + uint64_t zc_action_handle; + int zc_cleanup_fd; + uint8_t zc_simple; + uint8_t zc_pad3[3]; + boolean_t zc_resumable; + uint32_t zc_pad4; + uint64_t zc_sendobj; + uint64_t zc_fromobj; + uint64_t zc_createtxg; + zfs_stat_t zc_stat; +} zfs_cmd_legacy_t; + +unsigned static long zfs_ioctl_bsd12_to_zof[] = { + ZFS_IOC_POOL_CREATE, /* 0x00 */ + ZFS_IOC_POOL_DESTROY, /* 0x01 */ + ZFS_IOC_POOL_IMPORT, /* 0x02 */ + ZFS_IOC_POOL_EXPORT, /* 0x03 */ + ZFS_IOC_POOL_CONFIGS, /* 0x04 */ + ZFS_IOC_POOL_STATS, /* 0x05 */ + ZFS_IOC_POOL_TRYIMPORT, /* 0x06 */ + ZFS_IOC_POOL_SCAN, /* 0x07 */ + ZFS_IOC_POOL_FREEZE, /* 0x08 */ + ZFS_IOC_POOL_UPGRADE, /* 0x09 */ + ZFS_IOC_POOL_GET_HISTORY, /* 0x0a */ + ZFS_IOC_VDEV_ADD, /* 0x0b */ + ZFS_IOC_VDEV_REMOVE, /* 0x0c */ + ZFS_IOC_VDEV_SET_STATE, /* 0x0d */ + ZFS_IOC_VDEV_ATTACH, /* 0x0e */ + ZFS_IOC_VDEV_DETACH, /* 0x0f */ + ZFS_IOC_VDEV_SETPATH, /* 0x10 */ + ZFS_IOC_VDEV_SETFRU, /* 0x11 */ + ZFS_IOC_OBJSET_STATS, /* 0x12 */ + ZFS_IOC_OBJSET_ZPLPROPS, /* 0x13 */ + ZFS_IOC_DATASET_LIST_NEXT, /* 0x14 */ + ZFS_IOC_SNAPSHOT_LIST_NEXT, /* 0x15 */ + ZFS_IOC_SET_PROP, /* 0x16 */ + ZFS_IOC_CREATE, /* 0x17 */ + ZFS_IOC_DESTROY, /* 0x18 */ + ZFS_IOC_ROLLBACK, /* 0x19 */ + ZFS_IOC_RENAME, /* 0x1a */ + ZFS_IOC_RECV, /* 0x1b */ + ZFS_IOC_SEND, /* 0x1c */ + ZFS_IOC_INJECT_FAULT, /* 0x1d */ + ZFS_IOC_CLEAR_FAULT, /* 0x1e */ + ZFS_IOC_INJECT_LIST_NEXT, /* 0x1f */ + ZFS_IOC_ERROR_LOG, /* 0x20 */ + ZFS_IOC_CLEAR, /* 0x21 */ + ZFS_IOC_PROMOTE, /* 0x22 */ + /* start of mismatch */ + ZFS_IOC_DESTROY_SNAPS, /* 0x23:0x3b */ + ZFS_IOC_SNAPSHOT, /* 0x24:0x23 */ + ZFS_IOC_DSOBJ_TO_DSNAME, /* 0x25:0x24 */ + ZFS_IOC_OBJ_TO_PATH, /* 0x26:0x25 */ + ZFS_IOC_POOL_SET_PROPS, /* 0x27:0x26 */ + ZFS_IOC_POOL_GET_PROPS, /* 0x28:0x27 */ + ZFS_IOC_SET_FSACL, /* 0x29:0x28 */ + ZFS_IOC_GET_FSACL, /* 0x30:0x29 */ + ZFS_IOC_SHARE, /* 0x2b:0x2a */ + ZFS_IOC_INHERIT_PROP, /* 0x2c:0x2b */ + ZFS_IOC_SMB_ACL, /* 0x2d:0x2c */ + ZFS_IOC_USERSPACE_ONE, /* 0x2e:0x2d */ + ZFS_IOC_USERSPACE_MANY, /* 0x2f:0x2e */ + ZFS_IOC_USERSPACE_UPGRADE, /* 0x30:0x2f */ + ZFS_IOC_HOLD, /* 0x31:0x30 */ + ZFS_IOC_RELEASE, /* 0x32:0x31 */ + ZFS_IOC_GET_HOLDS, /* 0x33:0x32 */ + ZFS_IOC_OBJSET_RECVD_PROPS, /* 0x34:0x33 */ + ZFS_IOC_VDEV_SPLIT, /* 0x35:0x34 */ + ZFS_IOC_NEXT_OBJ, /* 0x36:0x35 */ + ZFS_IOC_DIFF, /* 0x37:0x36 */ + ZFS_IOC_TMP_SNAPSHOT, /* 0x38:0x37 */ + ZFS_IOC_OBJ_TO_STATS, /* 0x39:0x38 */ + ZFS_IOC_JAIL, /* 0x3a:0xc2 */ + ZFS_IOC_UNJAIL, /* 0x3b:0xc3 */ + ZFS_IOC_POOL_REGUID, /* 0x3c:0x3c */ + ZFS_IOC_SPACE_WRITTEN, /* 0x3d:0x39 */ + ZFS_IOC_SPACE_SNAPS, /* 0x3e:0x3a */ + ZFS_IOC_SEND_PROGRESS, /* 0x3f:0x3e */ + ZFS_IOC_POOL_REOPEN, /* 0x40:0x3d */ + ZFS_IOC_LOG_HISTORY, /* 0x41:0x3f */ + ZFS_IOC_SEND_NEW, /* 0x42:0x40 */ + ZFS_IOC_SEND_SPACE, /* 0x43:0x41 */ + ZFS_IOC_CLONE, /* 0x44:0x42 */ + ZFS_IOC_BOOKMARK, /* 0x45:0x43 */ + ZFS_IOC_GET_BOOKMARKS, /* 0x46:0x44 */ + ZFS_IOC_DESTROY_BOOKMARKS, /* 0x47:0x45 */ + ZFS_IOC_NEXTBOOT, /* 0x48:0xc1 */ + ZFS_IOC_CHANNEL_PROGRAM, /* 0x49:0x48 */ + ZFS_IOC_REMAP, /* 0x4a:0x4c */ + ZFS_IOC_POOL_CHECKPOINT, /* 0x4b:0x4d */ + ZFS_IOC_POOL_DISCARD_CHECKPOINT, /* 0x4c:0x4e */ + ZFS_IOC_POOL_INITIALIZE, /* 0x4d:0x4f */ +}; + +unsigned static long zfs_ioctl_v15_to_v28[] = { + 0, /* 0 ZFS_IOC_POOL_CREATE */ + 1, /* 1 ZFS_IOC_POOL_DESTROY */ + 2, /* 2 ZFS_IOC_POOL_IMPORT */ + 3, /* 3 ZFS_IOC_POOL_EXPORT */ + 4, /* 4 ZFS_IOC_POOL_CONFIGS */ + 5, /* 5 ZFS_IOC_POOL_STATS */ + 6, /* 6 ZFS_IOC_POOL_TRYIMPORT */ + 7, /* 7 ZFS_IOC_POOL_SCRUB */ + 8, /* 8 ZFS_IOC_POOL_FREEZE */ + 9, /* 9 ZFS_IOC_POOL_UPGRADE */ + 10, /* 10 ZFS_IOC_POOL_GET_HISTORY */ + 11, /* 11 ZFS_IOC_VDEV_ADD */ + 12, /* 12 ZFS_IOC_VDEV_REMOVE */ + 13, /* 13 ZFS_IOC_VDEV_SET_STATE */ + 14, /* 14 ZFS_IOC_VDEV_ATTACH */ + 15, /* 15 ZFS_IOC_VDEV_DETACH */ + 16, /* 16 ZFS_IOC_VDEV_SETPATH */ + 18, /* 17 ZFS_IOC_OBJSET_STATS */ + 19, /* 18 ZFS_IOC_OBJSET_ZPLPROPS */ + 20, /* 19 ZFS_IOC_DATASET_LIST_NEXT */ + 21, /* 20 ZFS_IOC_SNAPSHOT_LIST_NEXT */ + 22, /* 21 ZFS_IOC_SET_PROP */ + ZFS_IOC_COMPAT_PASS, /* 22 ZFS_IOC_CREATE_MINOR */ + ZFS_IOC_COMPAT_PASS, /* 23 ZFS_IOC_REMOVE_MINOR */ + 23, /* 24 ZFS_IOC_CREATE */ + 24, /* 25 ZFS_IOC_DESTROY */ + 25, /* 26 ZFS_IOC_ROLLBACK */ + 26, /* 27 ZFS_IOC_RENAME */ + 27, /* 28 ZFS_IOC_RECV */ + 28, /* 29 ZFS_IOC_SEND */ + 29, /* 30 ZFS_IOC_INJECT_FAULT */ + 30, /* 31 ZFS_IOC_CLEAR_FAULT */ + 31, /* 32 ZFS_IOC_INJECT_LIST_NEXT */ + 32, /* 33 ZFS_IOC_ERROR_LOG */ + 33, /* 34 ZFS_IOC_CLEAR */ + 34, /* 35 ZFS_IOC_PROMOTE */ + 35, /* 36 ZFS_IOC_DESTROY_SNAPS */ + 36, /* 37 ZFS_IOC_SNAPSHOT */ + 37, /* 38 ZFS_IOC_DSOBJ_TO_DSNAME */ + 38, /* 39 ZFS_IOC_OBJ_TO_PATH */ + 39, /* 40 ZFS_IOC_POOL_SET_PROPS */ + 40, /* 41 ZFS_IOC_POOL_GET_PROPS */ + 41, /* 42 ZFS_IOC_SET_FSACL */ + 42, /* 43 ZFS_IOC_GET_FSACL */ + ZFS_IOC_COMPAT_PASS, /* 44 ZFS_IOC_ISCSI_PERM_CHECK */ + 43, /* 45 ZFS_IOC_SHARE */ + 44, /* 46 ZFS_IOC_IHNERIT_PROP */ + 58, /* 47 ZFS_IOC_JAIL */ + 59, /* 48 ZFS_IOC_UNJAIL */ + 45, /* 49 ZFS_IOC_SMB_ACL */ + 46, /* 50 ZFS_IOC_USERSPACE_ONE */ + 47, /* 51 ZFS_IOC_USERSPACE_MANY */ + 48, /* 52 ZFS_IOC_USERSPACE_UPGRADE */ + 17, /* 53 ZFS_IOC_SETFRU */ +}; + +#else /* KERNEL */ +unsigned static long zfs_ioctl_v28_to_v15[] = { + 0, /* 0 ZFS_IOC_POOL_CREATE */ + 1, /* 1 ZFS_IOC_POOL_DESTROY */ + 2, /* 2 ZFS_IOC_POOL_IMPORT */ + 3, /* 3 ZFS_IOC_POOL_EXPORT */ + 4, /* 4 ZFS_IOC_POOL_CONFIGS */ + 5, /* 5 ZFS_IOC_POOL_STATS */ + 6, /* 6 ZFS_IOC_POOL_TRYIMPORT */ + 7, /* 7 ZFS_IOC_POOL_SCAN */ + 8, /* 8 ZFS_IOC_POOL_FREEZE */ + 9, /* 9 ZFS_IOC_POOL_UPGRADE */ + 10, /* 10 ZFS_IOC_POOL_GET_HISTORY */ + 11, /* 11 ZFS_IOC_VDEV_ADD */ + 12, /* 12 ZFS_IOC_VDEV_REMOVE */ + 13, /* 13 ZFS_IOC_VDEV_SET_STATE */ + 14, /* 14 ZFS_IOC_VDEV_ATTACH */ + 15, /* 15 ZFS_IOC_VDEV_DETACH */ + 16, /* 16 ZFS_IOC_VDEV_SETPATH */ + 53, /* 17 ZFS_IOC_VDEV_SETFRU */ + 17, /* 18 ZFS_IOC_OBJSET_STATS */ + 18, /* 19 ZFS_IOC_OBJSET_ZPLPROPS */ + 19, /* 20 ZFS_IOC_DATASET_LIST_NEXT */ + 20, /* 21 ZFS_IOC_SNAPSHOT_LIST_NEXT */ + 21, /* 22 ZFS_IOC_SET_PROP */ + 24, /* 23 ZFS_IOC_CREATE */ + 25, /* 24 ZFS_IOC_DESTROY */ + 26, /* 25 ZFS_IOC_ROLLBACK */ + 27, /* 26 ZFS_IOC_RENAME */ + 28, /* 27 ZFS_IOC_RECV */ + 29, /* 28 ZFS_IOC_SEND */ + 30, /* 39 ZFS_IOC_INJECT_FAULT */ + 31, /* 30 ZFS_IOC_CLEAR_FAULT */ + 32, /* 31 ZFS_IOC_INJECT_LIST_NEXT */ + 33, /* 32 ZFS_IOC_ERROR_LOG */ + 34, /* 33 ZFS_IOC_CLEAR */ + 35, /* 34 ZFS_IOC_PROMOTE */ + 36, /* 35 ZFS_IOC_DESTROY_SNAPS */ + 37, /* 36 ZFS_IOC_SNAPSHOT */ + 38, /* 37 ZFS_IOC_DSOBJ_TO_DSNAME */ + 39, /* 38 ZFS_IOC_OBJ_TO_PATH */ + 40, /* 39 ZFS_IOC_POOL_SET_PROPS */ + 41, /* 40 ZFS_IOC_POOL_GET_PROPS */ + 42, /* 41 ZFS_IOC_SET_FSACL */ + 43, /* 42 ZFS_IOC_GET_FSACL */ + 45, /* 43 ZFS_IOC_SHARE */ + 46, /* 44 ZFS_IOC_IHNERIT_PROP */ + 49, /* 45 ZFS_IOC_SMB_ACL */ + 50, /* 46 ZFS_IOC_USERSPACE_ONE */ + 51, /* 47 ZFS_IOC_USERSPACE_MANY */ + 52, /* 48 ZFS_IOC_USERSPACE_UPGRADE */ + ZFS_IOC_COMPAT_FAIL, /* 49 ZFS_IOC_HOLD */ + ZFS_IOC_COMPAT_FAIL, /* 50 ZFS_IOC_RELEASE */ + ZFS_IOC_COMPAT_FAIL, /* 51 ZFS_IOC_GET_HOLDS */ + ZFS_IOC_COMPAT_FAIL, /* 52 ZFS_IOC_OBJSET_RECVD_PROPS */ + ZFS_IOC_COMPAT_FAIL, /* 53 ZFS_IOC_VDEV_SPLIT */ + ZFS_IOC_COMPAT_FAIL, /* 54 ZFS_IOC_NEXT_OBJ */ + ZFS_IOC_COMPAT_FAIL, /* 55 ZFS_IOC_DIFF */ + ZFS_IOC_COMPAT_FAIL, /* 56 ZFS_IOC_TMP_SNAPSHOT */ + ZFS_IOC_COMPAT_FAIL, /* 57 ZFS_IOC_OBJ_TO_STATS */ + 47, /* 58 ZFS_IOC_JAIL */ + 48, /* 59 ZFS_IOC_UNJAIL */ +}; +#endif /* ! _KERNEL */ + +#ifdef _KERNEL +int zfs_ioctl_compat_pre(zfs_cmd_t *, int *, const int); +void zfs_ioctl_compat_post(zfs_cmd_t *, const int, const int); +nvlist_t *zfs_ioctl_compat_innvl(zfs_cmd_t *, nvlist_t *, const int, + const int); +nvlist_t *zfs_ioctl_compat_outnvl(zfs_cmd_t *, nvlist_t *, const int, + const int); +#endif /* _KERNEL */ +void zfs_cmd_compat_get(zfs_cmd_t *, caddr_t, const int); +void zfs_cmd_compat_put(zfs_cmd_t *, caddr_t, const int, const int); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_ZFS_IOCTL_COMPAT_H */ diff --git a/include/os/freebsd/zfs/sys/zfs_vfsops.h b/include/os/freebsd/zfs/sys/zfs_vfsops.h new file mode 100644 index 00000000000..d17b8033029 --- /dev/null +++ b/include/os/freebsd/zfs/sys/zfs_vfsops.h @@ -0,0 +1,172 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011 Pawel Jakub Dawidek . + * All rights reserved. + */ + +#ifndef _SYS_FS_ZFS_VFSOPS_H +#define _SYS_FS_ZFS_VFSOPS_H + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct zfsvfs zfsvfs_t; +struct znode; + +struct zfsvfs { + vfs_t *z_vfs; /* generic fs struct */ + zfsvfs_t *z_parent; /* parent fs */ + objset_t *z_os; /* objset reference */ + uint64_t z_flags; /* super_block flags */ + uint64_t z_root; /* id of root znode */ + uint64_t z_unlinkedobj; /* id of unlinked zapobj */ + uint64_t z_max_blksz; /* maximum block size for files */ + uint64_t z_fuid_obj; /* fuid table object number */ + uint64_t z_fuid_size; /* fuid table size */ + avl_tree_t z_fuid_idx; /* fuid tree keyed by index */ + avl_tree_t z_fuid_domain; /* fuid tree keyed by domain */ + krwlock_t z_fuid_lock; /* fuid lock */ + boolean_t z_fuid_loaded; /* fuid tables are loaded */ + boolean_t z_fuid_dirty; /* need to sync fuid table ? */ + struct zfs_fuid_info *z_fuid_replay; /* fuid info for replay */ + zilog_t *z_log; /* intent log pointer */ + uint_t z_acl_mode; /* acl chmod/mode behavior */ + uint_t z_acl_inherit; /* acl inheritance behavior */ + zfs_case_t z_case; /* case-sense */ + boolean_t z_utf8; /* utf8-only */ + int z_norm; /* normalization flags */ + boolean_t z_atime; /* enable atimes mount option */ + boolean_t z_unmounted; /* unmounted */ + rrmlock_t z_teardown_lock; + krwlock_t z_teardown_inactive_lock; + list_t z_all_znodes; /* all vnodes in the fs */ + uint64_t z_nr_znodes; /* number of znodes in the fs */ + kmutex_t z_znodes_lock; /* lock for z_all_znodes */ + struct zfsctl_root *z_ctldir; /* .zfs directory pointer */ + boolean_t z_show_ctldir; /* expose .zfs in the root dir */ + boolean_t z_issnap; /* true if this is a snapshot */ + boolean_t z_vscan; /* virus scan on/off */ + boolean_t z_use_fuids; /* version allows fuids */ + boolean_t z_replay; /* set during ZIL replay */ + boolean_t z_use_sa; /* version allow system attributes */ + boolean_t z_xattr_sa; /* allow xattrs to be stores as SA */ + boolean_t z_use_namecache; /* make use of FreeBSD name cache */ + uint8_t z_xattr; /* xattr type in use */ + uint64_t z_version; /* ZPL version */ + uint64_t z_shares_dir; /* hidden shares dir */ + kmutex_t z_lock; + uint64_t z_userquota_obj; + uint64_t z_groupquota_obj; + uint64_t z_userobjquota_obj; + uint64_t z_groupobjquota_obj; + uint64_t z_projectquota_obj; + uint64_t z_projectobjquota_obj; + uint64_t z_replay_eof; /* New end of file - replay only */ + sa_attr_type_t *z_attr_table; /* SA attr mapping->id */ +#define ZFS_OBJ_MTX_SZ 64 + kmutex_t z_hold_mtx[ZFS_OBJ_MTX_SZ]; /* znode hold locks */ + struct task z_unlinked_drain_task; +}; + +#define ZSB_XATTR 0x0001 /* Enable user xattrs */ +/* + * Normal filesystems (those not under .zfs/snapshot) have a total + * file ID size limited to 12 bytes (including the length field) due to + * NFSv2 protocol's limitation of 32 bytes for a filehandle. For historical + * reasons, this same limit is being imposed by the Solaris NFSv3 implementation + * (although the NFSv3 protocol actually permits a maximum of 64 bytes). It + * is not possible to expand beyond 12 bytes without abandoning support + * of NFSv2. + * + * For normal filesystems, we partition up the available space as follows: + * 2 bytes fid length (required) + * 6 bytes object number (48 bits) + * 4 bytes generation number (32 bits) + * + * We reserve only 48 bits for the object number, as this is the limit + * currently defined and imposed by the DMU. + */ +typedef struct zfid_short { + uint16_t zf_len; + uint8_t zf_object[6]; /* obj[i] = obj >> (8 * i) */ + uint8_t zf_gen[4]; /* gen[i] = gen >> (8 * i) */ +} zfid_short_t; + +/* + * Filesystems under .zfs/snapshot have a total file ID size of 22[*] bytes + * (including the length field). This makes files under .zfs/snapshot + * accessible by NFSv3 and NFSv4, but not NFSv2. + * + * For files under .zfs/snapshot, we partition up the available space + * as follows: + * 2 bytes fid length (required) + * 6 bytes object number (48 bits) + * 4 bytes generation number (32 bits) + * 6 bytes objset id (48 bits) + * 4 bytes[**] currently just zero (32 bits) + * + * We reserve only 48 bits for the object number and objset id, as these are + * the limits currently defined and imposed by the DMU. + * + * [*] 20 bytes on FreeBSD to fit into the size of struct fid. + * [**] 2 bytes on FreeBSD for the above reason. + */ +typedef struct zfid_long { + zfid_short_t z_fid; + uint8_t zf_setid[6]; /* obj[i] = obj >> (8 * i) */ + uint8_t zf_setgen[2]; /* gen[i] = gen >> (8 * i) */ +} zfid_long_t; + +#define SHORT_FID_LEN (sizeof (zfid_short_t) - sizeof (uint16_t)) +#define LONG_FID_LEN (sizeof (zfid_long_t) - sizeof (uint16_t)) + +extern uint_t zfs_fsyncer_key; +extern int zfs_super_owner; + +extern int zfs_suspend_fs(zfsvfs_t *zfsvfs); +extern int zfs_resume_fs(zfsvfs_t *zfsvfs, struct dsl_dataset *ds); +extern int zfs_end_fs(zfsvfs_t *zfsvfs, struct dsl_dataset *ds); +extern int zfs_set_version(zfsvfs_t *zfsvfs, uint64_t newvers); +extern int zfsvfs_create(const char *name, boolean_t readonly, zfsvfs_t **zfvp); +extern int zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os); +extern void zfsvfs_free(zfsvfs_t *zfsvfs); +extern int zfs_check_global_label(const char *dsname, const char *hexsl); +extern boolean_t zfs_is_readonly(zfsvfs_t *zfsvfs); +extern int zfs_get_temporary_prop(struct dsl_dataset *ds, zfs_prop_t zfs_prop, + uint64_t *val, char *setpoint); +extern int zfs_busy(void); +extern void zfsvfs_update_fromname(const char *oldname, const char *newname); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_FS_ZFS_VFSOPS_H */ diff --git a/include/os/freebsd/zfs/sys/zfs_vnops.h b/include/os/freebsd/zfs/sys/zfs_vnops.h new file mode 100644 index 00000000000..6237372b905 --- /dev/null +++ b/include/os/freebsd/zfs/sys/zfs_vnops.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_ZFS_VNOPS_H_ +#define _SYS_ZFS_VNOPS_H_ +int dmu_write_pages(objset_t *os, uint64_t object, uint64_t offset, + uint64_t size, struct vm_page **ppa, dmu_tx_t *tx); +int dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count, + int *rbehind, int *rahead, int last_size); +extern int zfs_remove(znode_t *dzp, char *name, cred_t *cr, int flags); +extern int zfs_mkdir(znode_t *dzp, char *dirname, vattr_t *vap, + znode_t **zpp, cred_t *cr, int flags, vsecattr_t *vsecp); +extern int zfs_rmdir(znode_t *dzp, char *name, znode_t *cwd, + cred_t *cr, int flags); +extern int zfs_setattr(znode_t *zp, vattr_t *vap, int flag, cred_t *cr); +extern int zfs_rename(znode_t *sdzp, char *snm, znode_t *tdzp, + char *tnm, cred_t *cr, int flags); +extern int zfs_symlink(znode_t *dzp, const char *name, vattr_t *vap, + const char *link, znode_t **zpp, cred_t *cr, int flags); +extern int zfs_link(znode_t *tdzp, znode_t *sp, + char *name, cred_t *cr, int flags); +extern int zfs_space(znode_t *zp, int cmd, struct flock *bfp, int flag, + offset_t offset, cred_t *cr); +extern int zfs_create(znode_t *dzp, char *name, vattr_t *vap, int excl, + int mode, znode_t **zpp, cred_t *cr, int flag, vsecattr_t *vsecp); +extern int zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, + cred_t *cr); +extern int zfs_write_simple(znode_t *zp, const void *data, size_t len, + loff_t pos, size_t *resid); + +#endif diff --git a/include/os/freebsd/zfs/sys/zfs_znode_impl.h b/include/os/freebsd/zfs/sys/zfs_znode_impl.h new file mode 100644 index 00000000000..c0430467572 --- /dev/null +++ b/include/os/freebsd/zfs/sys/zfs_znode_impl.h @@ -0,0 +1,182 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. + * Copyright (c) 2014 Integros [integros.com] + * Copyright 2016 Nexenta Systems, Inc. All rights reserved. + */ + +#ifndef _FREEBSD_ZFS_SYS_ZNODE_IMPL_H +#define _FREEBSD_ZFS_SYS_ZNODE_IMPL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Directory entry locks control access to directory entries. + * They are used to protect creates, deletes, and renames. + * Each directory znode has a mutex and a list of locked names. + */ +#define ZNODE_OS_FIELDS \ + struct zfsvfs *z_zfsvfs; \ + vnode_t *z_vnode; \ + uint64_t z_uid; \ + uint64_t z_gid; \ + uint64_t z_gen; \ + uint64_t z_atime[2]; \ + uint64_t z_links; + +#define ZFS_LINK_MAX UINT64_MAX + +/* + * ZFS minor numbers can refer to either a control device instance or + * a zvol. Depending on the value of zss_type, zss_data points to either + * a zvol_state_t or a zfs_onexit_t. + */ +enum zfs_soft_state_type { + ZSST_ZVOL, + ZSST_CTLDEV +}; + +typedef struct zfs_soft_state { + enum zfs_soft_state_type zss_type; + void *zss_data; +} zfs_soft_state_t; + +extern minor_t zfsdev_minor_alloc(void); + +/* + * Range locking rules + * -------------------- + * 1. When truncating a file (zfs_create, zfs_setattr, zfs_space) the whole + * file range needs to be locked as RL_WRITER. Only then can the pages be + * freed etc and zp_size reset. zp_size must be set within range lock. + * 2. For writes and punching holes (zfs_write & zfs_space) just the range + * being written or freed needs to be locked as RL_WRITER. + * Multiple writes at the end of the file must coordinate zp_size updates + * to ensure data isn't lost. A compare and swap loop is currently used + * to ensure the file size is at least the offset last written. + * 3. For reads (zfs_read, zfs_get_data & zfs_putapage) just the range being + * read needs to be locked as RL_READER. A check against zp_size can then + * be made for reading beyond end of file. + */ + +/* + * Convert between znode pointers and vnode pointers + */ +#define ZTOV(ZP) ((ZP)->z_vnode) +#define ZTOI(ZP) ((ZP)->z_vnode) +#define VTOZ(VP) ((struct znode *)(VP)->v_data) +#define ITOZ(VP) ((struct znode *)(VP)->v_data) +#define zhold(zp) vhold(ZTOV((zp))) +#define zrele(zp) vrele(ZTOV((zp))) + +#define ZTOZSB(zp) ((zp)->z_zfsvfs) +#define ITOZSB(vp) (VTOZ(vp)->z_zfsvfs) +#define ZTOTYPE(zp) (ZTOV(zp)->v_type) +#define ZTOGID(zp) ((zp)->z_gid) +#define ZTOUID(zp) ((zp)->z_uid) +#define ZTONLNK(zp) ((zp)->z_links) +#define Z_ISBLK(type) ((type) == VBLK) +#define Z_ISCHR(type) ((type) == VCHR) +#define Z_ISLNK(type) ((type) == VLNK) + + +/* Called on entry to each ZFS vnode and vfs operation */ +#define ZFS_ENTER(zfsvfs) \ + { \ + rrm_enter_read(&(zfsvfs)->z_teardown_lock, FTAG); \ + if ((zfsvfs)->z_unmounted) { \ + ZFS_EXIT(zfsvfs); \ + return (EIO); \ + } \ + } + +/* Must be called before exiting the vop */ +#define ZFS_EXIT(zfsvfs) rrm_exit(&(zfsvfs)->z_teardown_lock, FTAG) + +/* Verifies the znode is valid */ +#define ZFS_VERIFY_ZP(zp) \ + if ((zp)->z_sa_hdl == NULL) { \ + ZFS_EXIT((zp)->z_zfsvfs); \ + return (EIO); \ + } \ + +/* + * Macros for dealing with dmu_buf_hold + */ +#define ZFS_OBJ_HASH(obj_num) ((obj_num) & (ZFS_OBJ_MTX_SZ - 1)) +#define ZFS_OBJ_MUTEX(zfsvfs, obj_num) \ + (&(zfsvfs)->z_hold_mtx[ZFS_OBJ_HASH(obj_num)]) +#define ZFS_OBJ_HOLD_ENTER(zfsvfs, obj_num) \ + mutex_enter(ZFS_OBJ_MUTEX((zfsvfs), (obj_num))) +#define ZFS_OBJ_HOLD_TRYENTER(zfsvfs, obj_num) \ + mutex_tryenter(ZFS_OBJ_MUTEX((zfsvfs), (obj_num))) +#define ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num) \ + mutex_exit(ZFS_OBJ_MUTEX((zfsvfs), (obj_num))) + +/* Encode ZFS stored time values from a struct timespec */ +#define ZFS_TIME_ENCODE(tp, stmp) \ +{ \ + (stmp)[0] = (uint64_t)(tp)->tv_sec; \ + (stmp)[1] = (uint64_t)(tp)->tv_nsec; \ +} + +/* Decode ZFS stored time values to a struct timespec */ +#define ZFS_TIME_DECODE(tp, stmp) \ +{ \ + (tp)->tv_sec = (time_t)(stmp)[0]; \ + (tp)->tv_nsec = (long)(stmp)[1]; \ +} +#define ZFS_ACCESSTIME_STAMP(zfsvfs, zp) \ + if ((zfsvfs)->z_atime && !((zfsvfs)->z_vfs->vfs_flag & VFS_RDONLY)) \ + zfs_tstamp_update_setup_ext(zp, ACCESSED, NULL, NULL, B_FALSE); + +extern void zfs_tstamp_update_setup_ext(struct znode *, + uint_t, uint64_t [2], uint64_t [2], boolean_t have_tx); +extern void zfs_znode_free(struct znode *); + +extern zil_get_data_t zfs_get_data; +extern zil_replay_func_t *zfs_replay_vector[TX_MAX_TYPE]; +extern int zfsfstype; + +extern int zfs_znode_parent_and_name(struct znode *zp, struct znode **dzpp, + char *buf); + +#ifdef __cplusplus +} +#endif + +#endif /* _FREEBSD_SYS_FS_ZFS_ZNODE_H */ diff --git a/include/os/freebsd/zfs/sys/zpl.h b/include/os/freebsd/zfs/sys/zpl.h new file mode 100644 index 00000000000..fb2b4e02d44 --- /dev/null +++ b/include/os/freebsd/zfs/sys/zpl.h @@ -0,0 +1 @@ +/* Don't remove */ diff --git a/lib/Makefile.am b/lib/Makefile.am index 8dff773df40..4f59aa359c4 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -1,6 +1,13 @@ # NB: GNU Automake Manual, Chapter 8.3.5: Libtool Convenience Libraries # These six libraries are intermediary build components. -SUBDIRS = libavl libefi libicp libshare libspl libtpool libzutil libunicode +SUBDIRS = libavl libicp libshare libspl libtpool + +if BUILD_LINUX +SUBDIRS += libefi +endif + +# libzutil depends on libefi if present +SUBDIRS += libzutil libunicode # These four libraries, which are installed as the final build product, # incorporate the six convenience libraries given above. diff --git a/lib/libnvpair/Makefile.am b/lib/libnvpair/Makefile.am index 6626b6d0548..984ca520c0a 100644 --- a/lib/libnvpair/Makefile.am +++ b/lib/libnvpair/Makefile.am @@ -24,7 +24,14 @@ nodist_libnvpair_la_SOURCES = \ $(USER_C) \ $(KERNEL_C) +if BUILD_FREEBSD +libnvpair_la_LIBADD = $(LIBTIRPC_LIBS) -L/usr/local/lib -lintl +libnvpair_la_LDFLAGS = -version-info 3:0:0 +else libnvpair_la_LIBADD = $(LIBTIRPC_LIBS) libnvpair_la_LDFLAGS = -version-info 1:1:0 +endif + + EXTRA_DIST = $(USER_C) diff --git a/lib/libspl/Makefile.am b/lib/libspl/Makefile.am index 3101b5fc50d..77d8aba0ed0 100644 --- a/lib/libspl/Makefile.am +++ b/lib/libspl/Makefile.am @@ -42,6 +42,15 @@ USER_C += \ os/linux/getmntany.c endif +if BUILD_FREEBSD +USER_C += \ + os/freebsd/getexecname.c \ + os/freebsd/gethostid.c \ + os/freebsd/getmntany.c \ + os/freebsd/mnttab.c + +endif + USER_ASM = atomic.S nodist_libspl_la_SOURCES = \ diff --git a/lib/libspl/include/os/Makefile.am b/lib/libspl/include/os/Makefile.am index 09c0beec475..7b362e02ad5 100644 --- a/lib/libspl/include/os/Makefile.am +++ b/lib/libspl/include/os/Makefile.am @@ -1,3 +1,7 @@ +if BUILD_FREEBSD +SUBDIRS = freebsd +endif + if BUILD_LINUX SUBDIRS = linux endif diff --git a/lib/libspl/include/os/freebsd/Makefile.am b/lib/libspl/include/os/freebsd/Makefile.am new file mode 100644 index 00000000000..081839c48c8 --- /dev/null +++ b/lib/libspl/include/os/freebsd/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = sys diff --git a/lib/libspl/include/os/freebsd/sys/Makefile.am b/lib/libspl/include/os/freebsd/sys/Makefile.am new file mode 100644 index 00000000000..896c9387129 --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/Makefile.am @@ -0,0 +1,12 @@ +libspldir = $(includedir)/libspl/sys +libspl_HEADERS = \ + $(top_srcdir)/lib/libspl/include/os/freebsd/sys/byteorder.h \ + $(top_srcdir)/lib/libspl/include/os/freebsd/sys/file.h \ + $(top_srcdir)/lib/libspl/include/os/freebsd/sys/mnttab.h \ + $(top_srcdir)/lib/libspl/include/os/freebsd/sys/mount.h \ + $(top_srcdir)/lib/libspl/include/os/freebsd/sys/param.h \ + $(top_srcdir)/lib/libspl/include/os/freebsd/sys/stat.h \ + $(top_srcdir)/lib/libspl/include/os/freebsd/sys/sysmacros.h \ + $(top_srcdir)/lib/libspl/include/os/freebsd/sys/uio.h \ + $(top_srcdir)/lib/libspl/include/os/freebsd/sys/vfs.h \ + $(top_srcdir)/lib/libspl/include/os/freebsd/sys/zfs_context_os.h diff --git a/lib/libspl/include/os/freebsd/sys/byteorder.h b/lib/libspl/include/os/freebsd/sys/byteorder.h new file mode 100644 index 00000000000..74649cc4e0a --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/byteorder.h @@ -0,0 +1,311 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +/* + * University Copyright- Copyright (c) 1982, 1986, 1988 + * The Regents of the University of California + * All Rights Reserved + * + * University Acknowledgment- Portions of this document are derived from + * software developed by the University of California, Berkeley, and its + * contributors. + */ + +#ifndef _SYS_BYTEORDER_H +#define _SYS_BYTEORDER_H + +/* + * XXX FIXME + * on FreeBSD _BIG_ENDIAN is defined on all architectures so we have + * to exclude _MACHINE_ENDIAN_H_ and define the bulk of it here + */ + +#include +#include + +/* + * Define the order of 32-bit words in 64-bit words. + */ +#define _QUAD_HIGHWORD 1 +#define _QUAD_LOWWORD 0 + +/* + * Definitions for byte order, according to byte significance from low + * address to high. + */ +#undef _LITTLE_ENDIAN +/* LSB first: i386, vax */ +#define _LITTLE_ENDIAN 1234 +/* LSB first in word, MSW first in long */ +#define _PDP_ENDIAN 3412 + +#define _BYTE_ORDER _LITTLE_ENDIAN + +/* + * Deprecated variants that don't have enough underscores to be useful in more + * strict namespaces. + */ +#if __BSD_VISIBLE +#define LITTLE_ENDIAN _LITTLE_ENDIAN +#define PDP_ENDIAN _PDP_ENDIAN +#define BYTE_ORDER _BYTE_ORDER +#endif + +#define __bswap16_gen(x) (__uint16_t)((x) << 8 | (x) >> 8) +#define __bswap32_gen(x) \ + (((__uint32_t)__bswap16((x) & 0xffff) << 16) | __bswap16((x) >> 16)) +#define __bswap64_gen(x) \ + (((__uint64_t)__bswap32((x) & 0xffffffff) << 32) | __bswap32((x) >> 32)) + +#ifdef __GNUCLIKE_BUILTIN_CONSTANT_P +#define __bswap16(x) \ + ((__uint16_t)(__builtin_constant_p(x) ? \ + __bswap16_gen((__uint16_t)(x)) : __bswap16_var(x))) +#define __bswap32(x) \ + (__builtin_constant_p(x) ? \ + __bswap32_gen((__uint32_t)(x)) : __bswap32_var(x)) +#define __bswap64(x) \ + (__builtin_constant_p(x) ? \ + __bswap64_gen((__uint64_t)(x)) : __bswap64_var(x)) +#else +/* XXX these are broken for use in static initializers. */ +#define __bswap16(x) __bswap16_var(x) +#define __bswap32(x) __bswap32_var(x) +#define __bswap64(x) __bswap64_var(x) +#endif + +/* These are defined as functions to avoid multiple evaluation of x. */ + +static __inline __uint16_t +__bswap16_var(__uint16_t _x) +{ + + return (__bswap16_gen(_x)); +} + +static __inline __uint32_t +__bswap32_var(__uint32_t _x) +{ + +#ifdef __GNUCLIKE_ASM + __asm("bswap %0" : "+r" (_x)); + return (_x); +#else + return (__bswap32_gen(_x)); +#endif +} +#define __htonl(x) __bswap32(x) +#define __htons(x) __bswap16(x) +#define __ntohl(x) __bswap32(x) +#define __ntohs(x) __bswap16(x) + +#include +#include + +#if defined(__GNUC__) && defined(_ASM_INLINES) && \ + (defined(__i386) || defined(__amd64)) +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * macros for conversion between host and (internet) network byte order + */ + +#if defined(_BIG_ENDIAN) && !defined(ntohl) && !defined(__lint) +/* big-endian */ +#if defined(_BIG_ENDIAN) && (defined(__amd64__) || defined(__amd64)) +#error "incompatible ENDIAN / ARCH combination" +#endif +#define ntohl(x) (x) +#define ntohs(x) (x) +#define htonl(x) (x) +#define htons(x) (x) + +#elif !defined(ntohl) /* little-endian */ + +#ifndef _IN_PORT_T +#define _IN_PORT_T +typedef uint16_t in_port_t; +#endif + +#ifndef _IN_ADDR_T +#define _IN_ADDR_T +typedef uint32_t in_addr_t; +#endif + +#if !defined(_XPG4_2) || defined(__EXTENSIONS__) || defined(_XPG5) +extern uint32_t htonl(uint32_t); +extern uint16_t htons(uint16_t); +extern uint32_t ntohl(uint32_t); +extern uint16_t ntohs(uint16_t); +#else +extern in_addr_t htonl(in_addr_t); +extern in_port_t htons(in_port_t); +extern in_addr_t ntohl(in_addr_t); +extern in_port_t ntohs(in_port_t); +#endif /* !defined(_XPG4_2) || defined(__EXTENSIONS__) || defined(_XPG5) */ +#endif + +#if !defined(_XPG4_2) || defined(__EXTENSIONS__) + +/* + * Macros to reverse byte order + */ +#define BSWAP_8(x) ((x) & 0xff) +#define BSWAP_16(x) ((BSWAP_8(x) << 8) | BSWAP_8((x) >> 8)) +#define BSWAP_32(x) ((BSWAP_16(x) << 16) | BSWAP_16((x) >> 16)) +#define BSWAP_64(x) ((BSWAP_32(x) << 32) | BSWAP_32((x) >> 32)) + +#define BMASK_8(x) ((x) & 0xff) +#define BMASK_16(x) ((x) & 0xffff) +#define BMASK_32(x) ((x) & 0xffffffff) +#define BMASK_64(x) (x) + +/* + * Macros to convert from a specific byte order to/from native byte order + */ +#ifdef _BIG_ENDIAN +#define BE_8(x) BMASK_8(x) +#define BE_16(x) BMASK_16(x) +#define BE_32(x) BMASK_32(x) +#define BE_64(x) BMASK_64(x) +#define LE_8(x) BSWAP_8(x) +#define LE_16(x) BSWAP_16(x) +#define LE_32(x) BSWAP_32(x) +#define LE_64(x) BSWAP_64(x) +#else +#define LE_8(x) BMASK_8(x) +#define LE_16(x) BMASK_16(x) +#define LE_32(x) BMASK_32(x) +#define LE_64(x) BMASK_64(x) +#define BE_8(x) BSWAP_8(x) +#define BE_16(x) BSWAP_16(x) +#define BE_32(x) BSWAP_32(x) +#define BE_64(x) BSWAP_64(x) +#endif + +#ifdef _BIG_ENDIAN +static __inline__ uint64_t +htonll(uint64_t n) +{ + return (n); +} + +static __inline__ uint64_t +ntohll(uint64_t n) +{ + return (n); +} +#else +static __inline__ uint64_t +htonll(uint64_t n) +{ + return ((((uint64_t)htonl(n)) << 32) + htonl(n >> 32)); +} + +static __inline__ uint64_t +ntohll(uint64_t n) +{ + return ((((uint64_t)ntohl(n)) << 32) + ntohl(n >> 32)); +} +#endif + +/* + * Macros to read unaligned values from a specific byte order to + * native byte order + */ + +#define BE_IN8(xa) \ + *((uint8_t *)(xa)) + +#define BE_IN16(xa) \ + (((uint16_t)BE_IN8(xa) << 8) | BE_IN8((uint8_t *)(xa)+1)) + +#define BE_IN32(xa) \ + (((uint32_t)BE_IN16(xa) << 16) | BE_IN16((uint8_t *)(xa)+2)) + +#define BE_IN64(xa) \ + (((uint64_t)BE_IN32(xa) << 32) | BE_IN32((uint8_t *)(xa)+4)) + +#define LE_IN8(xa) \ + *((uint8_t *)(xa)) + +#define LE_IN16(xa) \ + (((uint16_t)LE_IN8((uint8_t *)(xa) + 1) << 8) | LE_IN8(xa)) + +#define LE_IN32(xa) \ + (((uint32_t)LE_IN16((uint8_t *)(xa) + 2) << 16) | LE_IN16(xa)) + +#define LE_IN64(xa) \ + (((uint64_t)LE_IN32((uint8_t *)(xa) + 4) << 32) | LE_IN32(xa)) + +/* + * Macros to write unaligned values from native byte order to a specific byte + * order. + */ + +#define BE_OUT8(xa, yv) *((uint8_t *)(xa)) = (uint8_t)(yv); + +#define BE_OUT16(xa, yv) \ + BE_OUT8((uint8_t *)(xa) + 1, yv); \ + BE_OUT8((uint8_t *)(xa), (yv) >> 8); + +#define BE_OUT32(xa, yv) \ + BE_OUT16((uint8_t *)(xa) + 2, yv); \ + BE_OUT16((uint8_t *)(xa), (yv) >> 16); + +#define BE_OUT64(xa, yv) \ + BE_OUT32((uint8_t *)(xa) + 4, yv); \ + BE_OUT32((uint8_t *)(xa), (yv) >> 32); + +#define LE_OUT8(xa, yv) *((uint8_t *)(xa)) = (uint8_t)(yv); + +#define LE_OUT16(xa, yv) \ + LE_OUT8((uint8_t *)(xa), yv); \ + LE_OUT8((uint8_t *)(xa) + 1, (yv) >> 8); + +#define LE_OUT32(xa, yv) \ + LE_OUT16((uint8_t *)(xa), yv); \ + LE_OUT16((uint8_t *)(xa) + 2, (yv) >> 16); + +#define LE_OUT64(xa, yv) \ + LE_OUT32((uint8_t *)(xa), yv); \ + LE_OUT32((uint8_t *)(xa) + 4, (yv) >> 32); + +#endif /* !defined(_XPG4_2) || defined(__EXTENSIONS__) */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_BYTEORDER_H */ diff --git a/lib/libspl/include/os/freebsd/sys/file.h b/lib/libspl/include/os/freebsd/sys/file.h new file mode 100644 index 00000000000..27fd2888f32 --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/file.h @@ -0,0 +1,42 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _LIBSPL_SYS_FILE_H +#define _LIBSPL_SYS_FILE_H + +#include_next + +#define FCREAT O_CREAT +#define FTRUNC O_TRUNC +#define FSYNC O_SYNC +#define FDSYNC O_DSYNC +#define FEXCL O_EXCL + +#define FNODSYNC 0x10000 /* fsync pseudo flag */ +#define FNOFOLLOW 0x20000 /* don't follow symlinks */ +#define FIGNORECASE 0x80000 /* request case-insensitive lookups */ + +#endif diff --git a/lib/libspl/include/os/freebsd/sys/mnttab.h b/lib/libspl/include/os/freebsd/sys/mnttab.h new file mode 100644 index 00000000000..eb6ea2433a6 --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/mnttab.h @@ -0,0 +1,85 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +/* Copyright 2006 Ricardo Correia */ + +#ifndef _SYS_MNTTAB_H +#define _SYS_MNTTAB_H + +#include +#include + +#ifdef MNTTAB +#undef MNTTAB +#endif /* MNTTAB */ + +#include +#include +#define MNTTAB _PATH_DEVZERO +#define MS_NOMNTTAB 0x0 +#define MS_RDONLY 0x1 +#define umount2(p, f) unmount(p, f) +#define MNT_LINE_MAX 4096 + +#define MNT_TOOLONG 1 /* entry exceeds MNT_LINE_MAX */ +#define MNT_TOOMANY 2 /* too many fields in line */ +#define MNT_TOOFEW 3 /* too few fields in line */ + +struct mnttab { + char *mnt_special; + char *mnt_mountp; + char *mnt_fstype; + char *mnt_mntopts; +}; + +/* + * NOTE: fields in extmnttab should match struct mnttab till new fields + * are encountered, this allows hasmntopt to work properly when its arg is + * a pointer to an extmnttab struct cast to a mnttab struct pointer. + */ + +struct extmnttab { + char *mnt_special; + char *mnt_mountp; + char *mnt_fstype; + char *mnt_mntopts; + uint_t mnt_major; + uint_t mnt_minor; +}; + +struct stat64; +struct statfs; + +extern int getmntany(FILE *fp, struct mnttab *mp, struct mnttab *mpref); +extern int _sol_getmntent(FILE *fp, struct mnttab *mp); +extern int getextmntent(const char *path, struct extmnttab *entry, + struct stat64 *statbuf); +extern void statfs2mnttab(struct statfs *sfs, struct mnttab *mp); +char *hasmntopt(struct mnttab *mnt, char *opt); +int getmntent(FILE *fp, struct mnttab *mp); + +#endif diff --git a/lib/libspl/include/os/freebsd/sys/mount.h b/lib/libspl/include/os/freebsd/sys/mount.h new file mode 100644 index 00000000000..b4023910005 --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/mount.h @@ -0,0 +1,108 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + + +#ifndef _LIBSPL_SYS_MOUNT_H +#define _LIBSPL_SYS_MOUNT_H + +#undef _SYS_MOUNT_H_ +#include_next + +#include +#include +#include + +/* + * Some old glibc headers don't define BLKGETSIZE64 + * and we don't want to require the kernel headers + */ +#if !defined(BLKGETSIZE64) +#define BLKGETSIZE64 _IOR(0x12, 114, size_t) +#endif + +/* + * Some old glibc headers don't correctly define MS_DIRSYNC and + * instead use the enum name S_WRITE. When using these older + * headers define MS_DIRSYNC to be S_WRITE. + */ +#if !defined(MS_DIRSYNC) +#define MS_DIRSYNC S_WRITE +#endif + +/* + * Some old glibc headers don't correctly define MS_POSIXACL and + * instead leave it undefined. When using these older headers define + * MS_POSIXACL to the reserved value of (1<<16). + */ +#if !defined(MS_POSIXACL) +#define MS_POSIXACL (1<<16) +#endif + +#define MS_NOSUID MNT_NOSUID +#define MS_NOEXEC MNT_NOEXEC +#define MS_NODEV 0 +#define S_WRITE 0 +#define MS_BIND 0 +#define MS_REMOUNT 0 +#define MS_SYNCHRONOUS MNT_SYNCHRONOUS + +#define MS_USERS (MS_NOEXEC|MS_NOSUID|MS_NODEV) +#define MS_OWNER (MS_NOSUID|MS_NODEV) +#define MS_GROUP (MS_NOSUID|MS_NODEV) +#define MS_COMMENT 0 + +/* + * Older glibc headers did not define all the available + * umount2(2) flags. Both MNT_FORCE and MNT_DETACH are supported in the + * kernel back to 2.4.11 so we define them correctly if they are missing. + */ +#ifdef MNT_FORCE +#define MS_FORCE MNT_FORCE +#else +#define MS_FORCE 0x00000001 +#endif /* MNT_FORCE */ + +#ifdef MNT_DETACH +#define MS_DETACH MNT_DETACH +#else +#define MS_DETACH 0x00000002 +#endif /* MNT_DETACH */ + +/* + * Overlay mount is default in Linux, but for solaris/zfs + * compatibility, MS_OVERLAY is defined to explicitly have the user + * provide a flag (-O) to mount over a non empty directory. + */ +#define MS_OVERLAY 0x00000004 + +/* + * MS_CRYPT indicates that encryption keys should be loaded if they are not + * already available. This is not defined in glibc, but it is never seen by + * the kernel so it will not cause any problems. + */ +#define MS_CRYPT 0x00000008 + +#endif /* _LIBSPL_SYS_MOUNT_H */ diff --git a/lib/libspl/include/os/freebsd/sys/param.h b/lib/libspl/include/os/freebsd/sys/param.h new file mode 100644 index 00000000000..c0ad8d6798c --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/param.h @@ -0,0 +1,70 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _LIBSPL_SYS_PARAM_H +#define _LIBSPL_SYS_PARAM_H + +#include_next +#include + +/* + * File system parameters and macros. + * + * The file system is made out of blocks of at most MAXBSIZE units, + * with smaller units (fragments) only in the last direct block. + * MAXBSIZE primarily determines the size of buffers in the buffer + * pool. It may be made larger without any effect on existing + * file systems; however making it smaller may make some file + * systems unmountable. + * + * Note that the blocked devices are assumed to have DEV_BSIZE + * "sectors" and that fragments must be some multiple of this size. + */ +#define MAXNAMELEN 256 + +#define UID_NOBODY 60001 /* user ID no body */ +#define GID_NOBODY UID_NOBODY +#define UID_NOACCESS 60002 /* user ID no access */ + +#define MAXUID UINT32_MAX /* max user id */ +#define MAXPROJID MAXUID /* max project id */ + +#ifdef PAGESIZE +#undef PAGESIZE +#endif /* PAGESIZE */ + +extern size_t spl_pagesize(void); +#define PAGESIZE (spl_pagesize()) + +extern int execvpe(const char *name, char * const argv[], char * const envp[]); + +struct zfs_handle; +/* + * Attach/detach the given filesystem to/from the given jail. + */ +extern int zfs_jail(struct zfs_handle *zhp, int jailid, int attach); + +#endif diff --git a/lib/libspl/include/os/freebsd/sys/stat.h b/lib/libspl/include/os/freebsd/sys/stat.h new file mode 100644 index 00000000000..82c86262fff --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/stat.h @@ -0,0 +1,71 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. + */ + +#ifndef _LIBSPL_SYS_STAT_H +#define _LIBSPL_SYS_STAT_H + +#include_next + +#include /* for BLKGETSIZE64 */ + +#define stat64 stat + +#define MAXOFFSET_T OFF_MAX + +#ifndef _KERNEL +#include + +static __inline int +fstat64(int fd, struct stat *sb) +{ + int ret; + + ret = fstat(fd, sb); + if (ret == 0) { + if (S_ISCHR(sb->st_mode)) + (void) ioctl(fd, DIOCGMEDIASIZE, &sb->st_size); + } + return (ret); +} +#endif + +/* + * Emulate Solaris' behavior of returning the block device size in fstat64(). + */ +static inline int +fstat64_blk(int fd, struct stat64 *st) +{ + if (fstat64(fd, st) == -1) + return (-1); + + /* In Linux we need to use an ioctl to get the size of a block device */ + if (S_ISBLK(st->st_mode)) { + if (ioctl(fd, BLKGETSIZE64, &st->st_size) != 0) + return (-1); + } + + return (0); +} +#endif /* _LIBSPL_SYS_STAT_H */ diff --git a/lib/libspl/include/os/freebsd/sys/sysmacros.h b/lib/libspl/include/os/freebsd/sys/sysmacros.h new file mode 100644 index 00000000000..d9639d27b60 --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/sysmacros.h @@ -0,0 +1 @@ +/* keep me */ diff --git a/lib/libspl/include/os/freebsd/sys/uio.h b/lib/libspl/include/os/freebsd/sys/uio.h new file mode 100644 index 00000000000..d978b6ad077 --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/uio.h @@ -0,0 +1,98 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +/* + * University Copyright- Copyright (c) 1982, 1986, 1988 + * The Regents of the University of California + * All Rights Reserved + * + * University Acknowledgment- Portions of this document are derived from + * software developed by the University of California, Berkeley, and its + * contributors. + */ + +#ifndef _LIBSPL_SYS_UIO_H +#define _LIBSPL_SYS_UIO_H + +#include_next + +typedef struct iovec iovec_t; +typedef enum uio_seg uio_seg_t; + +typedef struct uio { + struct iovec *uio_iov; /* pointer to array of iovecs */ + int uio_iovcnt; /* number of iovecs */ + offset_t uio_loffset; /* file offset */ + uio_seg_t uio_segflg; /* address space (kernel or user) */ + uint16_t uio_fmode; /* file mode flags */ + uint16_t uio_extflg; /* extended flags */ + offset_t uio_limit; /* u-limit (maximum byte offset) */ + ssize_t uio_resid; /* residual count */ +} uio_t; + +typedef enum xuio_type { + UIOTYPE_ASYNCIO, + UIOTYPE_ZEROCOPY, +} xuio_type_t; + +#define UIOA_IOV_MAX 16 + +typedef struct uioa_page_s { /* locked uio_iov state */ + int uioa_pfncnt; /* count of pfn_t(s) in *uioa_ppp */ + void **uioa_ppp; /* page_t or pfn_t array */ + caddr_t uioa_base; /* address base */ + size_t uioa_len; /* span length */ +} uioa_page_t; + +typedef struct xuio { + uio_t xu_uio; /* embedded UIO structure */ + + /* Extended uio fields */ + enum xuio_type xu_type; /* uio type */ + union { + struct { + uint32_t xu_a_state; /* state of async i/o */ + ssize_t xu_a_mbytes; /* bytes moved */ + uioa_page_t *xu_a_lcur; /* uioa_locked[] pointer */ + void **xu_a_lppp; /* lcur->uioa_pppp[] pointer */ + void *xu_a_hwst[4]; /* opaque hardware state */ + uioa_page_t xu_a_locked[UIOA_IOV_MAX]; + } xu_aio; + + struct { + int xu_zc_rw; /* read or write buffer */ + void *xu_zc_priv; /* fs specific */ + } xu_zc; + } xu_ext; +} xuio_t; + +#define XUIO_XUZC_PRIV(xuio) xuio->xu_ext.xu_zc.xu_zc_priv +#define XUIO_XUZC_RW(xuio) xuio->xu_ext.xu_zc.xu_zc_rw + +#endif /* _SYS_UIO_H */ diff --git a/lib/libspl/include/os/freebsd/sys/vfs.h b/lib/libspl/include/os/freebsd/sys/vfs.h new file mode 100644 index 00000000000..55eb3c23b22 --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/vfs.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef ZFS_SYS_VFS_H_ +#define ZFS_SYS_VFS_H_ + +#include_next + +int fsshare(const char *, const char *, const char *); +int fsunshare(const char *, const char *); + +#endif /* !ZFS_SYS_VFS_H_ */ diff --git a/lib/libspl/include/os/freebsd/sys/zfs_context_os.h b/lib/libspl/include/os/freebsd/sys/zfs_context_os.h new file mode 100644 index 00000000000..25b5a47df92 --- /dev/null +++ b/lib/libspl/include/os/freebsd/sys/zfs_context_os.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef ZFS_CONTEXT_OS_H_ +#define ZFS_CONTEXT_OS_H_ + +#if BYTE_ORDER != BIG_ENDIAN +#undef _BIG_ENDIAN +#endif + +#define ZFS_EXPORTS_PATH "/etc/zfs/exports" + +#endif diff --git a/lib/libspl/os/freebsd/getexecname.c b/lib/libspl/os/freebsd/getexecname.c new file mode 100644 index 00000000000..13e50d32404 --- /dev/null +++ b/lib/libspl/os/freebsd/getexecname.c @@ -0,0 +1,70 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +const char * +getexecname(void) +{ + static char execname[PATH_MAX + 1] = ""; + static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; + char *ptr = NULL; + ssize_t rc; + + (void) pthread_mutex_lock(&mtx); + + if (strlen(execname) == 0) { + int error, name[4]; + size_t len; + + name[0] = CTL_KERN; + name[1] = KERN_PROC; + name[2] = KERN_PROC_PATHNAME; + name[3] = -1; + len = PATH_MAX; + error = sysctl(name, nitems(name), execname, &len, NULL, 0); + if (error != 0) { + rc = -1; + } else { + rc = len; + } + if (rc == -1) { + execname[0] = '\0'; + } else { + execname[rc] = '\0'; + ptr = execname; + } + } else { + ptr = execname; + } + + (void) pthread_mutex_unlock(&mtx); + return (ptr); +} diff --git a/lib/libspl/os/freebsd/gethostid.c b/lib/libspl/os/freebsd/gethostid.c new file mode 100644 index 00000000000..7bd567fe61b --- /dev/null +++ b/lib/libspl/os/freebsd/gethostid.c @@ -0,0 +1,36 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + */ + +#include +#include +#include +#include +#include +#include + +unsigned long +get_system_hostid(void) +{ + return (gethostid()); +} diff --git a/lib/libspl/os/freebsd/getmntany.c b/lib/libspl/os/freebsd/getmntany.c new file mode 100644 index 00000000000..b41e763cee4 --- /dev/null +++ b/lib/libspl/os/freebsd/getmntany.c @@ -0,0 +1,67 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Ricardo Correia. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1988 AT&T */ +/* All Rights Reserved */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define BUFSIZE (MNT_LINE_MAX + 2) + +__thread char buf[BUFSIZE]; + +int +getextmntent(const char *path, struct extmnttab *entry, struct stat64 *statbuf) +{ + struct statfs sfs; + + if (strlen(path) >= MAXPATHLEN) { + (void) fprintf(stderr, "invalid object; pathname too long\n"); + return (-1); + } + + if (stat64(path, statbuf) != 0) { + (void) fprintf(stderr, "cannot open '%s': %s\n", + path, strerror(errno)); + return (-1); + } + + if (statfs(path, &sfs) != 0) { + (void) fprintf(stderr, "%s: %s\n", path, + strerror(errno)); + return (-1); + } + statfs2mnttab(&sfs, (struct mnttab *)entry); + return (0); +} diff --git a/lib/libspl/os/freebsd/mnttab.c b/lib/libspl/os/freebsd/mnttab.c new file mode 100644 index 00000000000..5b9e6429d9e --- /dev/null +++ b/lib/libspl/os/freebsd/mnttab.c @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2006 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * This file implements Solaris compatible getmntany() and hasmntopt() + * functions. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +static char * +mntopt(char **p) +{ + char *cp = *p; + char *retstr; + + while (*cp && isspace(*cp)) + cp++; + + retstr = cp; + while (*cp && *cp != ',') + cp++; + + if (*cp) { + *cp = '\0'; + cp++; + } + + *p = cp; + return (retstr); +} + +char * +hasmntopt(struct mnttab *mnt, char *opt) +{ + char tmpopts[MNT_LINE_MAX]; + char *f, *opts = tmpopts; + + if (mnt->mnt_mntopts == NULL) + return (NULL); + (void) strcpy(opts, mnt->mnt_mntopts); + f = mntopt(&opts); + for (; *f; f = mntopt(&opts)) { + if (strncmp(opt, f, strlen(opt)) == 0) + return (f - tmpopts + mnt->mnt_mntopts); + } + return (NULL); +} + +static void +optadd(char *mntopts, size_t size, const char *opt) +{ + + if (mntopts[0] != '\0') + strlcat(mntopts, ",", size); + strlcat(mntopts, opt, size); +} + +void +statfs2mnttab(struct statfs *sfs, struct mnttab *mp) +{ + static char mntopts[MNTMAXSTR]; + long flags; + + mntopts[0] = '\0'; + + flags = sfs->f_flags; +#define OPTADD(opt) optadd(mntopts, sizeof (mntopts), (opt)) + if (flags & MNT_RDONLY) + OPTADD(MNTOPT_RO); + else + OPTADD(MNTOPT_RW); + if (flags & MNT_NOSUID) + OPTADD(MNTOPT_NOSETUID); + else + OPTADD(MNTOPT_SETUID); + if (flags & MNT_UPDATE) + OPTADD(MNTOPT_REMOUNT); + if (flags & MNT_NOATIME) + OPTADD(MNTOPT_NOATIME); + else + OPTADD(MNTOPT_ATIME); + OPTADD(MNTOPT_NOXATTR); + if (flags & MNT_NOEXEC) + OPTADD(MNTOPT_NOEXEC); + else + OPTADD(MNTOPT_EXEC); +#undef OPTADD + mp->mnt_special = strdup(sfs->f_mntfromname); + mp->mnt_mountp = strdup(sfs->f_mntonname); + mp->mnt_fstype = strdup(sfs->f_fstypename); + mp->mnt_mntopts = strdup(mntopts); +} + +static struct statfs *gsfs = NULL; +static int allfs = 0; + +static int +statfs_init(void) +{ + struct statfs *sfs; + int error; + + if (gsfs != NULL) { + free(gsfs); + gsfs = NULL; + } + allfs = getfsstat(NULL, 0, MNT_WAIT); + if (allfs == -1) + goto fail; + gsfs = malloc(sizeof (gsfs[0]) * allfs * 2); + if (gsfs == NULL) + goto fail; + allfs = getfsstat(gsfs, (long)(sizeof (gsfs[0]) * allfs * 2), + MNT_WAIT); + if (allfs == -1) + goto fail; + sfs = realloc(gsfs, allfs * sizeof (gsfs[0])); + if (sfs != NULL) + gsfs = sfs; + return (0); +fail: + error = errno; + if (gsfs != NULL) + free(gsfs); + gsfs = NULL; + allfs = 0; + return (error); +} + +int +getmntany(FILE *fd __unused, struct mnttab *mgetp, struct mnttab *mrefp) +{ + // struct statfs *sfs; + int i, error; + + error = statfs_init(); + if (error != 0) + return (error); + + for (i = 0; i < allfs; i++) { + if (mrefp->mnt_special != NULL && + strcmp(mrefp->mnt_special, gsfs[i].f_mntfromname) != 0) { + continue; + } + if (mrefp->mnt_mountp != NULL && + strcmp(mrefp->mnt_mountp, gsfs[i].f_mntonname) != 0) { + continue; + } + if (mrefp->mnt_fstype != NULL && + strcmp(mrefp->mnt_fstype, gsfs[i].f_fstypename) != 0) { + continue; + } + statfs2mnttab(&gsfs[i], mgetp); + return (0); + } + return (-1); +} + +int +getmntent(FILE *fp, struct mnttab *mp) +{ + // struct statfs *sfs; + int error, nfs; + + nfs = (int)lseek(fileno(fp), 0, SEEK_CUR); + if (nfs == -1) + return (errno); + /* If nfs is 0, we want to refresh out cache. */ + if (nfs == 0 || gsfs == NULL) { + error = statfs_init(); + if (error != 0) + return (error); + } + if (nfs >= allfs) + return (-1); + statfs2mnttab(&gsfs[nfs], mp); + if (lseek(fileno(fp), 1, SEEK_CUR) == -1) + return (errno); + return (0); +} diff --git a/lib/libuutil/Makefile.am b/lib/libuutil/Makefile.am index c61b66fce32..9eb81f090fc 100644 --- a/lib/libuutil/Makefile.am +++ b/lib/libuutil/Makefile.am @@ -19,6 +19,10 @@ libuutil_la_LIBADD = \ $(top_builddir)/lib/libavl/libavl.la \ $(top_builddir)/lib/libspl/libspl.la +if BUILD_FREEBSD +libuutil_la_LDFLAGS = -pthread -version-info 3:0:0 +else libuutil_la_LDFLAGS = -pthread -version-info 1:1:0 +endif EXTRA_DIST = $(USER_C) diff --git a/lib/libzfs/Makefile.am b/lib/libzfs/Makefile.am index 2747a7e9b3e..0e1e0a53ea7 100644 --- a/lib/libzfs/Makefile.am +++ b/lib/libzfs/Makefile.am @@ -27,6 +27,15 @@ USER_C = \ libzfs_status.c \ libzfs_util.c + +if BUILD_FREEBSD +USER_C += \ + os/freebsd/libzfs_fsshare.c \ + os/freebsd/libzfs_compat.c \ + os/freebsd/libzfs_ioctl_compat.c \ + os/freebsd/libzfs_zmount.c +endif + if BUILD_LINUX USER_C += \ os/linux/libzfs_mount_os.c \ @@ -35,7 +44,6 @@ USER_C += \ os/linux/libzfs_util_os.c endif - KERNEL_C = \ algs/sha2/sha2.c \ cityhash.c \ @@ -70,7 +78,12 @@ libzfs_la_LIBADD += \ $(top_builddir)/lib/libshare/libshare.la endif +if BUILD_FREEBSD +libzfs_la_LIBADD += -lutil -lgeom +libzfs_la_LDFLAGS = -version-info 4:0:0 +else libzfs_la_LDFLAGS = -version-info 2:0:0 +endif libzfs_la_LIBADD += -lm $(LIBSSL) diff --git a/lib/libzfs/libzfs_util.c b/lib/libzfs/libzfs_util.c index c6debd576d5..71ac72ee1ae 100644 --- a/lib/libzfs/libzfs_util.c +++ b/lib/libzfs/libzfs_util.c @@ -881,7 +881,7 @@ libzfs_init(void) return (NULL); } - if ((hdl->libzfs_fd = open(ZFS_DEV, O_RDWR)) < 0) { + if ((hdl->libzfs_fd = open(ZFS_DEV, O_RDWR|O_EXCL)) < 0) { free(hdl); return (NULL); } diff --git a/lib/libzfs/os/freebsd/libzfs_compat.c b/lib/libzfs/os/freebsd/libzfs_compat.c new file mode 100644 index 00000000000..e1c6ef93d5b --- /dev/null +++ b/lib/libzfs/os/freebsd/libzfs_compat.c @@ -0,0 +1,323 @@ +/* + * CDDL HEADER SART + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2013 Martin Matuska . All rights reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int zfs_ioctl_version = ZFS_IOCVER_UNDEF; +// static int zfs_spa_version = -1; + +void +libzfs_set_pipe_max(int infd) +{ + /* FreeBSD automatically resizes */ +} + +static int +execvPe(const char *name, const char *path, char * const *argv, + char * const *envp) +{ + const char **memp; + size_t cnt, lp, ln; + int eacces, save_errno; + char *cur, buf[MAXPATHLEN]; + const char *p, *bp; + struct stat sb; + + eacces = 0; + + /* If it's an absolute or relative path name, it's easy. */ + if (strchr(name, '/')) { + bp = name; + cur = NULL; + goto retry; + } + bp = buf; + + /* If it's an empty path name, fail in the usual POSIX way. */ + if (*name == '\0') { + errno = ENOENT; + return (-1); + } + + cur = alloca(strlen(path) + 1); + if (cur == NULL) { + errno = ENOMEM; + return (-1); + } + strcpy(cur, path); + while ((p = strsep(&cur, ":")) != NULL) { + /* + * It's a SHELL path -- double, leading and trailing colons + * mean the current directory. + */ + if (*p == '\0') { + p = "."; + lp = 1; + } else + lp = strlen(p); + ln = strlen(name); + + /* + * If the path is too long complain. This is a possible + * security issue; given a way to make the path too long + * the user may execute the wrong program. + */ + if (lp + ln + 2 > sizeof (buf)) { + (void) write(STDERR_FILENO, "execvP: ", 8); + (void) write(STDERR_FILENO, p, lp); + (void) write(STDERR_FILENO, ": path too long\n", + 16); + continue; + } + bcopy(p, buf, lp); + buf[lp] = '/'; + bcopy(name, buf + lp + 1, ln); + buf[lp + ln + 1] = '\0'; + +retry: (void) execve(bp, argv, envp); + switch (errno) { + case E2BIG: + goto done; + case ELOOP: + case ENAMETOOLONG: + case ENOENT: + break; + case ENOEXEC: + for (cnt = 0; argv[cnt]; ++cnt) + ; + memp = alloca((cnt + 2) * sizeof (char *)); + if (memp == NULL) { + /* errno = ENOMEM; XXX override ENOEXEC? */ + goto done; + } + memp[0] = "sh"; + memp[1] = bp; + bcopy(argv + 1, memp + 2, cnt * sizeof (char *)); + execve(_PATH_BSHELL, __DECONST(char **, memp), envp); + goto done; + case ENOMEM: + goto done; + case ENOTDIR: + break; + case ETXTBSY: + /* + * We used to retry here, but sh(1) doesn't. + */ + goto done; + default: + /* + * EACCES may be for an inaccessible directory or + * a non-executable file. Call stat() to decide + * which. This also handles ambiguities for EFAULT + * and EIO, and undocumented errors like ESTALE. + * We hope that the race for a stat() is unimportant. + */ + save_errno = errno; + if (stat(bp, &sb) != 0) + break; + if (save_errno == EACCES) { + eacces = 1; + continue; + } + errno = save_errno; + goto done; + } + } + if (eacces) + errno = EACCES; + else + errno = ENOENT; +done: + return (-1); +} + +int +execvpe(const char *name, char * const argv[], char * const envp[]) +{ + const char *path; + + /* Get the path we're searching. */ + if ((path = getenv("PATH")) == NULL) + path = _PATH_DEFPATH; + + return (execvPe(name, path, argv, envp)); +} + +#if 0 +/* + * Get the SPA version + */ +static int +get_zfs_spa_version(void) +{ + size_t ver_size; + int ver = 0; + + ver_size = sizeof (ver); + sysctlbyname("vfs.zfs.version.spa", &ver, &ver_size, NULL, 0); + + return (ver); +} +#endif + +/* + * Get zfs_ioctl_version + */ +int +get_zfs_ioctl_version(void) +{ + size_t ver_size; + int ver = ZFS_IOCVER_NONE; + + ver_size = sizeof (ver); + sysctlbyname("vfs.zfs.version.ioctl", &ver, &ver_size, NULL, 0); + + return (ver); +} + +const char * +libzfs_error_init(int error) +{ + + return (strerror(error)); +} + +int +zfs_ioctl(libzfs_handle_t *hdl, int request, zfs_cmd_t *zc) +{ + return (zfs_ioctl_fd(hdl->libzfs_fd, request, zc)); +} + +/* + * Verify the required ZFS_DEV device is available and optionally attempt + * to load the ZFS modules. Under normal circumstances the modules + * should already have been loaded by some external mechanism. + * + * Environment variables: + * - ZFS_MODULE_LOADING="YES|yes|ON|on" - Attempt to load modules. + * - ZFS_MODULE_TIMEOUT="" - Seconds to wait for ZFS_DEV + */ +int +libzfs_load_module(void) +{ + /* XXX: modname is "zfs" but file is named "openzfs". */ + if (modfind("zfs") < 0) { + /* Not present in kernel, try loading it. */ + if (kldload("openzfs") < 0 && errno != EEXIST) { + return (errno); + } + } + return (0); +} + +int +zpool_relabel_disk(libzfs_handle_t *hdl, const char *path, const char *msg) +{ + return (0); +} + +int +zpool_label_disk(libzfs_handle_t *hdl, zpool_handle_t *zhp, char *name) +{ + return (0); +} + +int +find_shares_object(differ_info_t *di) +{ + return (0); +} + +/* + * Attach/detach the given filesystem to/from the given jail. + */ +int +zfs_jail(zfs_handle_t *zhp, int jailid, int attach) +{ + libzfs_handle_t *hdl = zhp->zfs_hdl; + zfs_cmd_t zc = { { 0 } }; + char errbuf[1024]; + unsigned long cmd; + int ret; + + if (attach) { + (void) snprintf(errbuf, sizeof (errbuf), + dgettext(TEXT_DOMAIN, "cannot jail '%s'"), zhp->zfs_name); + } else { + (void) snprintf(errbuf, sizeof (errbuf), + dgettext(TEXT_DOMAIN, "cannot unjail '%s'"), zhp->zfs_name); + } + + switch (zhp->zfs_type) { + case ZFS_TYPE_VOLUME: + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "volumes can not be jailed")); + return (zfs_error(hdl, EZFS_BADTYPE, errbuf)); + case ZFS_TYPE_SNAPSHOT: + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "snapshots can not be jailed")); + return (zfs_error(hdl, EZFS_BADTYPE, errbuf)); + case ZFS_TYPE_BOOKMARK: + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "bookmarks can not be jailed")); + return (zfs_error(hdl, EZFS_BADTYPE, errbuf)); + case ZFS_TYPE_POOL: + case ZFS_TYPE_FILESYSTEM: + /* OK */ + ; + } + assert(zhp->zfs_type == ZFS_TYPE_FILESYSTEM); + + (void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name)); + zc.zc_objset_type = DMU_OST_ZFS; + zc.zc_zoneid = jailid; + + cmd = attach ? ZFS_IOC_JAIL : ZFS_IOC_UNJAIL; + if ((ret = ioctl(hdl->libzfs_fd, cmd, &zc)) != 0) + zfs_standard_error(hdl, errno, errbuf); + + return (ret); +} + +/* + * Fill given version buffer with zfs kernel version. + * Returns 0 on success, and -1 on error (with errno set) + */ +int +zfs_version_kernel(char *version, int len) +{ + size_t l = len; + + return (sysctlbyname("vfs.zfs.version.module", + version, &l, NULL, 0)); +} diff --git a/lib/libzfs/os/freebsd/libzfs_fsshare.c b/lib/libzfs/os/freebsd/libzfs_fsshare.c new file mode 100644 index 00000000000..0fd75bf2c54 --- /dev/null +++ b/lib/libzfs/os/freebsd/libzfs_fsshare.c @@ -0,0 +1,406 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libzfs_impl.h" + +#define _PATH_MOUNTDPID "/var/run/mountd.pid" +#define FILE_HEADER "# !!! DO NOT EDIT THIS FILE MANUALLY !!!\n\n" +#define OPTSSIZE 1024 +#define MAXLINESIZE (PATH_MAX + OPTSSIZE) + + +void +sa_fini(sa_handle_t handle) +{ +} + +int +sa_parse_legacy_options(sa_group_t group, char *options, char *proto) +{ + return (SA_OK); +} + + +int +zfs_init_libshare(libzfs_handle_t *zhandle, int service) +{ + return (SA_OK); +} + +/* + * Share the given filesystem according to the options in the specified + * protocol specific properties (sharenfs, sharesmb). We rely + * on "libshare" to do the dirty work for us. + */ +int +zfs_share_proto(zfs_handle_t *zhp, zfs_share_proto_t *proto) +{ + char mountpoint[ZFS_MAXPROPLEN]; + char shareopts[ZFS_MAXPROPLEN]; + char sourcestr[ZFS_MAXPROPLEN]; + libzfs_handle_t *hdl = zhp->zfs_hdl; + zfs_share_proto_t *curr_proto; + zprop_source_t sourcetype; + int err, ret; + + if (!zfs_is_mountable(zhp, mountpoint, sizeof (mountpoint), NULL, 0)) + return (0); + + for (curr_proto = proto; *curr_proto != PROTO_END; curr_proto++) { + /* + * Return success if there are no share options. + */ + if (zfs_prop_get(zhp, proto_table[*curr_proto].p_prop, + shareopts, sizeof (shareopts), &sourcetype, sourcestr, + ZFS_MAXPROPLEN, B_FALSE) != 0 || + strcmp(shareopts, "off") == 0) + continue; + + ret = zfs_init_libshare(hdl, SA_INIT_SHARE_API); + if (ret != SA_OK) { + (void) zfs_error_fmt(hdl, EZFS_SHARENFSFAILED, + dgettext(TEXT_DOMAIN, "cannot share '%s': %s"), + zfs_get_name(zhp), sa_errorstr(ret)); + return (-1); + } + + /* + * If the 'zoned' property is set, then zfs_is_mountable() + * will have already bailed out if we are in the global zone. + * But local zones cannot be NFS servers, so we ignore it for + * local zones as well. + */ + if (zfs_prop_get_int(zhp, ZFS_PROP_ZONED)) + continue; + + if (*curr_proto != PROTO_NFS) { + fprintf(stderr, "Unsupported share protocol: %d.\n", + *curr_proto); + continue; + } + + if (strcmp(shareopts, "on") == 0) + err = fsshare(ZFS_EXPORTS_PATH, mountpoint, ""); + else + err = fsshare(ZFS_EXPORTS_PATH, mountpoint, shareopts); + if (err != 0) { + (void) zfs_error_fmt(hdl, + proto_table[*curr_proto].p_share_err, + dgettext(TEXT_DOMAIN, "cannot share '%s'"), + zfs_get_name(zhp)); + return (-1); + } + + } + return (0); +} + +/* + * Unshare a filesystem by mountpoint. + */ +int +unshare_one(libzfs_handle_t *hdl, const char *name, const char *mountpoint, + zfs_share_proto_t proto) +{ + int err; + + if (proto != PROTO_NFS) { + fprintf(stderr, "No SMB support in FreeBSD yet.\n"); + return (EOPNOTSUPP); + } + + err = fsunshare(ZFS_EXPORTS_PATH, mountpoint); + if (err != 0) { + zfs_error_aux(hdl, "%s", strerror(err)); + return (zfs_error_fmt(hdl, EZFS_UNSHARENFSFAILED, + dgettext(TEXT_DOMAIN, + "cannot unshare '%s'"), name)); + } + return (0); +} + +zfs_share_type_t +is_shared_impl(libzfs_handle_t *hdl, const char *mountpoint, + zfs_share_proto_t proto) +{ + char buf[MAXPATHLEN], *tab; + + if (hdl->libzfs_sharetab == NULL) + return (SHARED_NOT_SHARED); + + (void) fseek(hdl->libzfs_sharetab, 0, SEEK_SET); + + while (fgets(buf, sizeof (buf), hdl->libzfs_sharetab) != NULL) { + + /* the mountpoint is the first entry on each line */ + if ((tab = strchr(buf, '\t')) == NULL) + continue; + + *tab = '\0'; + if (strcmp(buf, mountpoint) == 0) { + if (proto == PROTO_NFS) + return (SHARED_NFS); + } + } + + return (SHARED_NOT_SHARED); +} + +static void +restart_mountd(void) +{ + struct pidfh *pfh; + pid_t mountdpid; + + pfh = pidfile_open(_PATH_MOUNTDPID, 0600, &mountdpid); + if (pfh != NULL) { + /* Mountd is not running. */ + pidfile_remove(pfh); + return; + } + if (errno != EEXIST) { + /* Cannot open pidfile for some reason. */ + return; + } + /* We have mountd(8) PID in mountdpid varible. */ + kill(mountdpid, SIGHUP); +} + +/* + * Read one line from a file. Skip comments, empty lines and a line with a + * mountpoint specified in the 'skip' argument. + */ +static char * +zgetline(FILE *fd, const char *skip) +{ + static char line[MAXLINESIZE]; + size_t len, skiplen = 0; + char *s, last; + + if (skip != NULL) + skiplen = strlen(skip); + for (;;) { + s = fgets(line, sizeof (line), fd); + if (s == NULL) + return (NULL); + /* Skip empty lines and comments. */ + if (line[0] == '\n' || line[0] == '#') + continue; + len = strlen(line); + if (line[len - 1] == '\n') + line[len - 1] = '\0'; + last = line[skiplen]; + /* Skip the given mountpoint. */ + if (skip != NULL && strncmp(skip, line, skiplen) == 0 && + (last == '\t' || last == ' ' || last == '\0')) { + continue; + } + break; + } + return (line); +} +/* BEGIN CSTYLED */ +/* + * Function translate options to a format acceptable by exports(5), eg. + * + * -ro -network=192.168.0.0 -mask=255.255.255.0 -maproot=0 freefall.freebsd.org 69.147.83.54 + * + * Accepted input formats: + * + * ro,network=192.168.0.0,mask=255.255.255.0,maproot=0,freefall.freebsd.org + * ro network=192.168.0.0 mask=255.255.255.0 maproot=0 freefall.freebsd.org + * -ro,-network=192.168.0.0,-mask=255.255.255.0,-maproot=0,freefall.freebsd.org + * -ro -network=192.168.0.0 -mask=255.255.255.0 -maproot=0 freefall.freebsd.org + * + * Recognized keywords: + * + * ro, maproot, mapall, mask, network, sec, alldirs, public, webnfs, index, quiet + * + */ +/* END CSTYLED */ + +static const char *known_opts[] = { "ro", "maproot", "mapall", "mask", + "network", "sec", "alldirs", "public", "webnfs", "index", "quiet", + NULL }; +static char * +translate_opts(const char *shareopts) +{ + static char newopts[OPTSSIZE]; + char oldopts[OPTSSIZE]; + char *o, *s = NULL; + unsigned int i; + size_t len; + + strlcpy(oldopts, shareopts, sizeof (oldopts)); + newopts[0] = '\0'; + s = oldopts; + while ((o = strsep(&s, "-, ")) != NULL) { + if (o[0] == '\0') + continue; + for (i = 0; known_opts[i] != NULL; i++) { + len = strlen(known_opts[i]); + if (strncmp(known_opts[i], o, len) == 0 && + (o[len] == '\0' || o[len] == '=')) { + strlcat(newopts, "-", sizeof (newopts)); + break; + } + } + strlcat(newopts, o, sizeof (newopts)); + strlcat(newopts, " ", sizeof (newopts)); + } + return (newopts); +} + +static int +fsshare_main(const char *file, const char *mountpoint, const char *shareopts, + int share) +{ + char tmpfile[PATH_MAX]; + char *line; + FILE *newfd, *oldfd; + int fd, error; + + newfd = oldfd = NULL; + error = 0; + + /* + * Create temporary file in the same directory, so we can atomically + * rename it. + */ + if (strlcpy(tmpfile, file, sizeof (tmpfile)) >= sizeof (tmpfile)) + return (ENAMETOOLONG); + if (strlcat(tmpfile, ".XXXXXXXX", sizeof (tmpfile)) >= sizeof (tmpfile)) + return (ENAMETOOLONG); + fd = mkstemp(tmpfile); + if (fd == -1) + return (errno); + /* + * File name is random, so we don't really need file lock now, but it + * will be needed after rename(2). + */ + error = flock(fd, LOCK_EX); + assert(error == 0 || (error == -1 && errno == EOPNOTSUPP)); + newfd = fdopen(fd, "r+"); + assert(newfd != NULL); + /* Open old exports file. */ + oldfd = fopen(file, "r"); + if (oldfd == NULL) { + if (share) { + if (errno != ENOENT) { + error = errno; + goto out; + } + } else { + /* If there is no exports file, ignore the error. */ + if (errno == ENOENT) + errno = 0; + error = errno; + goto out; + } + } else { + error = flock(fileno(oldfd), LOCK_EX); + assert(error == 0 || (error == -1 && errno == EOPNOTSUPP)); + error = 0; + } + + /* Place big, fat warning at the begining of the file. */ + fprintf(newfd, "%s", FILE_HEADER); + while (oldfd != NULL && (line = zgetline(oldfd, mountpoint)) != NULL) + fprintf(newfd, "%s\n", line); + if (oldfd != NULL && ferror(oldfd) != 0) { + error = ferror(oldfd); + goto out; + } + if (ferror(newfd) != 0) { + error = ferror(newfd); + goto out; + } + if (share) { + fprintf(newfd, "%s\t%s\n", mountpoint, + translate_opts(shareopts)); + } + +out: + if (error != 0) + unlink(tmpfile); + else { + if (rename(tmpfile, file) == -1) { + error = errno; + unlink(tmpfile); + } else { + fflush(newfd); + /* + * Send SIGHUP to mountd, but unlock exports file later. + */ + restart_mountd(); + } + } + if (oldfd != NULL) { + flock(fileno(oldfd), LOCK_UN); + fclose(oldfd); + } + if (newfd != NULL) { + flock(fileno(newfd), LOCK_UN); + fclose(newfd); + } + return (error); +} + +/* + * Add the given mountpoint to the given exports file. + */ +int +fsshare(const char *file, const char *mountpoint, const char *shareopts) +{ + + return (fsshare_main(file, mountpoint, shareopts, 1)); +} + +/* + * Remove the given mountpoint from the given exports file. + */ +int +fsunshare(const char *file, const char *mountpoint) +{ + + return (fsshare_main(file, mountpoint, NULL, 0)); +} diff --git a/lib/libzfs/os/freebsd/libzfs_ioctl_compat.c b/lib/libzfs/os/freebsd/libzfs_ioctl_compat.c new file mode 100644 index 00000000000..18b93fe2796 --- /dev/null +++ b/lib/libzfs/os/freebsd/libzfs_ioctl_compat.c @@ -0,0 +1,432 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2013 Xin Li . All rights reserved. + * Copyright 2013 Martin Matuska . All rights reserved. + * Portions Copyright 2005, 2010, Oracle and/or its affiliates. + * All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "zfs_namecheck.h" +#include + +/* + * FreeBSD zfs_cmd compatibility with older binaries + * appropriately remap/extend the zfs_cmd_t structure + */ +void +zfs_cmd_compat_get(zfs_cmd_t *zc, caddr_t addr, const int cflag) +{ + +} +#if 0 +static int +zfs_ioctl_compat_get_nvlist(uint64_t nvl, size_t size, int iflag, + nvlist_t **nvp) +{ + char *packed; + int error; + nvlist_t *list = NULL; + + /* + * Read in and unpack the user-supplied nvlist. + */ + if (size == 0) + return (EINVAL); + +#ifdef _KERNEL + packed = kmem_alloc(size, KM_SLEEP); + if ((error = ddi_copyin((void *)(uintptr_t)nvl, packed, size, + iflag)) != 0) { + kmem_free(packed, size); + return (error); + } +#else + packed = (void *)(uintptr_t)nvl; +#endif + + error = nvlist_unpack(packed, size, &list, 0); + +#ifdef _KERNEL + kmem_free(packed, size); +#endif + + if (error != 0) + return (error); + + *nvp = list; + return (0); +} + +static int +zfs_ioctl_compat_put_nvlist(zfs_cmd_t *zc, nvlist_t *nvl) +{ + char *packed = NULL; + int error = 0; + size_t size; + + VERIFY(nvlist_size(nvl, &size, NV_ENCODE_NATIVE) == 0); + +#ifdef _KERNEL + packed = kmem_alloc(size, KM_SLEEP); + VERIFY(nvlist_pack(nvl, &packed, &size, NV_ENCODE_NATIVE, + KM_SLEEP) == 0); + + if (ddi_copyout(packed, + (void *)(uintptr_t)zc->zc_nvlist_dst, size, zc->zc_iflags) != 0) + error = EFAULT; + kmem_free(packed, size); +#else + packed = (void *)(uintptr_t)zc->zc_nvlist_dst; + VERIFY(nvlist_pack(nvl, &packed, &size, NV_ENCODE_NATIVE, + 0) == 0); +#endif + + zc->zc_nvlist_dst_size = size; + return (error); +} + +static void +zfs_ioctl_compat_fix_stats_nvlist(nvlist_t *nvl) +{ + nvlist_t **child; + nvlist_t *nvroot = NULL; + vdev_stat_t *vs; + uint_t c, children, nelem; + + if (nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, + &child, &children) == 0) { + for (c = 0; c < children; c++) { + zfs_ioctl_compat_fix_stats_nvlist(child[c]); + } + } + + if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_VDEV_TREE, + &nvroot) == 0) + zfs_ioctl_compat_fix_stats_nvlist(nvroot); + if ((nvlist_lookup_uint64_array(nvl, "stats", + (uint64_t **)&vs, &nelem) == 0)) { + nvlist_add_uint64_array(nvl, + ZPOOL_CONFIG_VDEV_STATS, + (uint64_t *)vs, nelem); + nvlist_remove(nvl, "stats", + DATA_TYPE_UINT64_ARRAY); + } +} + + +static int +zfs_ioctl_compat_fix_stats(zfs_cmd_t *zc, const int nc) +{ + nvlist_t *nv, *nvp = NULL; + nvpair_t *elem; + int error; + + if ((error = zfs_ioctl_compat_get_nvlist(zc->zc_nvlist_dst, + zc->zc_nvlist_dst_size, zc->zc_iflags, &nv)) != 0) + return (error); + + if (nc == 5) { /* ZFS_IOC_POOL_STATS */ + elem = NULL; + while ((elem = nvlist_next_nvpair(nv, elem)) != NULL) { + if (nvpair_value_nvlist(elem, &nvp) == 0) + zfs_ioctl_compat_fix_stats_nvlist(nvp); + } + elem = NULL; + } else + zfs_ioctl_compat_fix_stats_nvlist(nv); + + error = zfs_ioctl_compat_put_nvlist(zc, nv); + + nvlist_free(nv); + + return (error); +} + +static int +zfs_ioctl_compat_pool_get_props(zfs_cmd_t *zc) +{ + nvlist_t *nv, *nva = NULL; + int error; + + if ((error = zfs_ioctl_compat_get_nvlist(zc->zc_nvlist_dst, + zc->zc_nvlist_dst_size, zc->zc_iflags, &nv)) != 0) + return (error); + + if (nvlist_lookup_nvlist(nv, "used", &nva) == 0) { + nvlist_add_nvlist(nv, "allocated", nva); + nvlist_remove(nv, "used", DATA_TYPE_NVLIST); + } + + if (nvlist_lookup_nvlist(nv, "available", &nva) == 0) { + nvlist_add_nvlist(nv, "free", nva); + nvlist_remove(nv, "available", DATA_TYPE_NVLIST); + } + + error = zfs_ioctl_compat_put_nvlist(zc, nv); + + nvlist_free(nv); + + return (error); +} +#endif + +#ifdef _KERNEL +int +zfs_ioctl_compat_pre(zfs_cmd_t *zc, int *vec, const int cflag) +{ + int error = 0; + + /* are we creating a clone? */ + if (*vec == ZFS_IOC_CREATE && zc->zc_value[0] != '\0') + *vec = ZFS_IOC_CLONE; + + if (cflag == ZFS_CMD_COMPAT_V15) { + switch (*vec) { + + case 7: /* ZFS_IOC_POOL_SCRUB (v15) */ + zc->zc_cookie = POOL_SCAN_SCRUB; + break; + } + } + + return (error); +} + +void +zfs_ioctl_compat_post(zfs_cmd_t *zc, int vec, const int cflag) +{ + if (cflag == ZFS_CMD_COMPAT_V15) { + switch (vec) { + case ZFS_IOC_POOL_CONFIGS: + case ZFS_IOC_POOL_STATS: + case ZFS_IOC_POOL_TRYIMPORT: + zfs_ioctl_compat_fix_stats(zc, vec); + break; + case 41: /* ZFS_IOC_POOL_GET_PROPS (v15) */ + zfs_ioctl_compat_pool_get_props(zc); + break; + } + } +} + +nvlist_t * +zfs_ioctl_compat_innvl(zfs_cmd_t *zc, nvlist_t *innvl, const int vec, + const int cflag) +{ + nvlist_t *nvl, *tmpnvl, *hnvl; + nvpair_t *elem; + char *poolname, *snapname; + int err; + + if (cflag == ZFS_CMD_COMPAT_NONE || cflag == ZFS_CMD_COMPAT_LZC || + cflag == ZFS_CMD_COMPAT_ZCMD || cflag == ZFS_CMD_COMPAT_EDBP || + cflag == ZFS_CMD_COMPAT_RESUME || cflag == ZFS_CMD_COMPAT_INLANES) + goto out; + + switch (vec) { + case ZFS_IOC_CREATE: + nvl = fnvlist_alloc(); + fnvlist_add_int32(nvl, "type", zc->zc_objset_type); + if (innvl != NULL) { + fnvlist_add_nvlist(nvl, "props", innvl); + nvlist_free(innvl); + } + return (nvl); + break; + case ZFS_IOC_CLONE: + nvl = fnvlist_alloc(); + fnvlist_add_string(nvl, "origin", zc->zc_value); + if (innvl != NULL) { + fnvlist_add_nvlist(nvl, "props", innvl); + nvlist_free(innvl); + } + return (nvl); + break; + case ZFS_IOC_SNAPSHOT: + if (innvl == NULL) + goto out; + nvl = fnvlist_alloc(); + fnvlist_add_nvlist(nvl, "props", innvl); + tmpnvl = fnvlist_alloc(); + snapname = kmem_asprintf("%s@%s", zc->zc_name, zc->zc_value); + fnvlist_add_boolean(tmpnvl, snapname); + kmem_free(snapname, strlen(snapname + 1)); + /* check if we are doing a recursive snapshot */ + if (zc->zc_cookie) + dmu_get_recursive_snaps_nvl(zc->zc_name, zc->zc_value, + tmpnvl); + fnvlist_add_nvlist(nvl, "snaps", tmpnvl); + fnvlist_free(tmpnvl); + nvlist_free(innvl); + /* strip dataset part from zc->zc_name */ + zc->zc_name[strcspn(zc->zc_name, "/@")] = '\0'; + return (nvl); + break; + case ZFS_IOC_SPACE_SNAPS: + nvl = fnvlist_alloc(); + fnvlist_add_string(nvl, "firstsnap", zc->zc_value); + if (innvl != NULL) + nvlist_free(innvl); + return (nvl); + break; + case ZFS_IOC_DESTROY_SNAPS: + if (innvl == NULL && cflag == ZFS_CMD_COMPAT_DEADMAN) + goto out; + nvl = fnvlist_alloc(); + if (innvl != NULL) { + fnvlist_add_nvlist(nvl, "snaps", innvl); + } else { + /* + * We are probably called by even older binaries, + * allocate and populate nvlist with recursive + * snapshots + */ + if (zfs_component_namecheck(zc->zc_value, NULL, + NULL) == 0) { + tmpnvl = fnvlist_alloc(); + if (dmu_get_recursive_snaps_nvl(zc->zc_name, + zc->zc_value, tmpnvl) == 0) + fnvlist_add_nvlist(nvl, "snaps", + tmpnvl); + nvlist_free(tmpnvl); + } + } + if (innvl != NULL) + nvlist_free(innvl); + /* strip dataset part from zc->zc_name */ + zc->zc_name[strcspn(zc->zc_name, "/@")] = '\0'; + return (nvl); + break; + case ZFS_IOC_HOLD: + nvl = fnvlist_alloc(); + tmpnvl = fnvlist_alloc(); + if (zc->zc_cleanup_fd != -1) + fnvlist_add_int32(nvl, "cleanup_fd", + (int32_t)zc->zc_cleanup_fd); + if (zc->zc_cookie) { + hnvl = fnvlist_alloc(); + if (dmu_get_recursive_snaps_nvl(zc->zc_name, + zc->zc_value, hnvl) == 0) { + elem = NULL; + while ((elem = nvlist_next_nvpair(hnvl, + elem)) != NULL) { + nvlist_add_string(tmpnvl, + nvpair_name(elem), zc->zc_string); + } + } + nvlist_free(hnvl); + } else { + snapname = kmem_asprintf("%s@%s", zc->zc_name, + zc->zc_value); + nvlist_add_string(tmpnvl, snapname, zc->zc_string); + kmem_free(snapname, strlen(snapname + 1)); + } + fnvlist_add_nvlist(nvl, "holds", tmpnvl); + nvlist_free(tmpnvl); + if (innvl != NULL) + nvlist_free(innvl); + /* strip dataset part from zc->zc_name */ + zc->zc_name[strcspn(zc->zc_name, "/@")] = '\0'; + return (nvl); + break; + case ZFS_IOC_RELEASE: + nvl = fnvlist_alloc(); + tmpnvl = fnvlist_alloc(); + if (zc->zc_cookie) { + hnvl = fnvlist_alloc(); + if (dmu_get_recursive_snaps_nvl(zc->zc_name, + zc->zc_value, hnvl) == 0) { + elem = NULL; + while ((elem = nvlist_next_nvpair(hnvl, + elem)) != NULL) { + fnvlist_add_boolean(tmpnvl, + zc->zc_string); + fnvlist_add_nvlist(nvl, + nvpair_name(elem), tmpnvl); + } + } + nvlist_free(hnvl); + } else { + snapname = kmem_asprintf("%s@%s", zc->zc_name, + zc->zc_value); + fnvlist_add_boolean(tmpnvl, zc->zc_string); + fnvlist_add_nvlist(nvl, snapname, tmpnvl); + kmem_free(snapname, strlen(snapname + 1)); + } + nvlist_free(tmpnvl); + if (innvl != NULL) + nvlist_free(innvl); + /* strip dataset part from zc->zc_name */ + zc->zc_name[strcspn(zc->zc_name, "/@")] = '\0'; + return (nvl); + break; + } +out: + return (innvl); +} + +nvlist_t * +zfs_ioctl_compat_outnvl(zfs_cmd_t *zc, nvlist_t *outnvl, const int vec, + const int cflag) +{ + nvlist_t *tmpnvl; + + if (cflag == ZFS_CMD_COMPAT_NONE || cflag == ZFS_CMD_COMPAT_LZC || + cflag == ZFS_CMD_COMPAT_ZCMD || cflag == ZFS_CMD_COMPAT_EDBP || + cflag == ZFS_CMD_COMPAT_RESUME || cflag == ZFS_CMD_COMPAT_INLANES) + return (outnvl); + + switch (vec) { + case ZFS_IOC_SPACE_SNAPS: + (void) nvlist_lookup_uint64(outnvl, "used", &zc->zc_cookie); + (void) nvlist_lookup_uint64(outnvl, "compressed", + &zc->zc_objset_type); + (void) nvlist_lookup_uint64(outnvl, "uncompressed", + &zc->zc_perm_action); + nvlist_free(outnvl); + /* return empty outnvl */ + tmpnvl = fnvlist_alloc(); + return (tmpnvl); + break; + case ZFS_IOC_CREATE: + case ZFS_IOC_CLONE: + case ZFS_IOC_HOLD: + case ZFS_IOC_RELEASE: + nvlist_free(outnvl); + /* return empty outnvl */ + tmpnvl = fnvlist_alloc(); + return (tmpnvl); + break; + } + + return (outnvl); +} +#endif /* KERNEL */ diff --git a/lib/libzfs/os/freebsd/libzfs_zmount.c b/lib/libzfs/os/freebsd/libzfs_zmount.c new file mode 100644 index 00000000000..8ff24f446bd --- /dev/null +++ b/lib/libzfs/os/freebsd/libzfs_zmount.c @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2006 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * This file implements Solaris compatible zmount() function. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void +build_iovec(struct iovec **iov, int *iovlen, const char *name, void *val, + size_t len) +{ + int i; + + if (*iovlen < 0) + return; + i = *iovlen; + *iov = realloc(*iov, sizeof (**iov) * (i + 2)); + if (*iov == NULL) { + *iovlen = -1; + return; + } + (*iov)[i].iov_base = strdup(name); + (*iov)[i].iov_len = strlen(name) + 1; + i++; + (*iov)[i].iov_base = val; + if (len == (size_t)-1) { + if (val != NULL) + len = strlen(val) + 1; + else + len = 0; + } + (*iov)[i].iov_len = (int)len; + *iovlen = ++i; +} + +static int +do_mount_(const char *spec, const char *dir, int mflag, char *fstype, + char *dataptr, int datalen, char *optptr, int optlen) +{ + struct iovec *iov; + char *optstr, *p, *tofree; + int iovlen, rv; + + assert(spec != NULL); + assert(dir != NULL); + assert(fstype != NULL); + assert(strcmp(fstype, MNTTYPE_ZFS) == 0); + assert(dataptr == NULL); + assert(datalen == 0); + assert(optptr != NULL); + assert(optlen > 0); + + tofree = optstr = strdup(optptr); + assert(optstr != NULL); + + iov = NULL; + iovlen = 0; + if (strstr(optstr, MNTOPT_REMOUNT) != NULL) + build_iovec(&iov, &iovlen, "update", NULL, 0); + if (strstr(optstr, MNTOPT_NOXATTR) == NULL && + strstr(optstr, MNTOPT_XATTR) == NULL && + strstr(optstr, MNTOPT_SAXATTR) == NULL && + strstr(optstr, MNTOPT_DIRXATTR) == NULL) + build_iovec(&iov, &iovlen, "xattr", NULL, 0); + if (mflag & MS_RDONLY) + build_iovec(&iov, &iovlen, "ro", NULL, 0); + build_iovec(&iov, &iovlen, "fstype", fstype, (size_t)-1); + build_iovec(&iov, &iovlen, "fspath", __DECONST(char *, dir), + (size_t)-1); + build_iovec(&iov, &iovlen, "from", __DECONST(char *, spec), (size_t)-1); + while ((p = strsep(&optstr, ",/")) != NULL) + build_iovec(&iov, &iovlen, p, NULL, (size_t)-1); + rv = nmount(iov, iovlen, 0); + free(tofree); + if (rv < 0) + return (errno); + return (rv); +} + +int +do_mount(const char *src, const char *mntpt, char *opts, int flags) +{ + + return (do_mount_(src, mntpt, flags, MNTTYPE_ZFS, NULL, 0, opts, + sizeof (mntpt))); +} + +int +do_unmount(const char *mntpt, int flags) +{ + + return (unmount(mntpt, flags)); +} + +int +zfs_mount_delegation_check(void) +{ + return (0); +} + +/* + * Check if we are doing an overlay mount. + * Returns B_TRUE if the mount would overlay, otherwise B_FALSE. + */ +boolean_t +zfs_mount_overlay_check(const char *mountpoint) +{ + /* FreeBSD always allows overlay mounts. */ + return (B_FALSE); +} diff --git a/lib/libzfs_core/Makefile.am b/lib/libzfs_core/Makefile.am index dca81e01ab3..617b1cf3295 100644 --- a/lib/libzfs_core/Makefile.am +++ b/lib/libzfs_core/Makefile.am @@ -12,6 +12,10 @@ libzfs_core_la_LIBADD = \ $(top_builddir)/lib/libuutil/libuutil.la \ $(top_builddir)/lib/libzutil/libzutil.la +if BUILD_FREEBSD +libzfs_core_la_LDFLAGS = -version-info 3:0:0 +libzfs_core_la_LIBADD += -lutil -lgeom +else libzfs_core_la_LDFLAGS = -version-info 1:0:0 - +endif EXTRA_DIST = $(USER_C) diff --git a/lib/libzfs_core/libzfs_core_compat.h b/lib/libzfs_core/libzfs_core_compat.h new file mode 100644 index 00000000000..6527c4b2576 --- /dev/null +++ b/lib/libzfs_core/libzfs_core_compat.h @@ -0,0 +1,47 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2013 by Martin Matuska . All rights reserved. + */ + +#ifndef _LIBZFS_CORE_COMPAT_H +#define _LIBZFS_CORE_COMPAT_H + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int lzc_compat_pre(zfs_cmd_t *, zfs_ioc_t *, nvlist_t **); +void lzc_compat_post(zfs_cmd_t *, const zfs_ioc_t); +int lzc_compat_outnvl(zfs_cmd_t *, const zfs_ioc_t, nvlist_t **); + +#ifdef __cplusplus +} +#endif + +#endif /* _LIBZFS_CORE_COMPAT_H */ diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am index 4225246b968..a9396105bc6 100644 --- a/lib/libzpool/Makefile.am +++ b/lib/libzpool/Makefile.am @@ -199,8 +199,13 @@ libzpool_la_LIBADD = \ $(top_builddir)/lib/libunicode/libunicode.la \ $(top_builddir)/lib/libzutil/libzutil.la +if BUILD_FREEBSD +libzpool_la_LIBADD += $(ZLIB) -ldl -lgeom +libzpool_la_LDFLAGS = -pthread -version-info 4:0:0 +else libzpool_la_LIBADD += $(ZLIB) -ldl libzpool_la_LDFLAGS = -pthread -version-info 2:0:0 +endif EXTRA_DIST = $(USER_C) diff --git a/lib/libzutil/Makefile.am b/lib/libzutil/Makefile.am index e5c6a340d28..092312e0dcd 100644 --- a/lib/libzutil/Makefile.am +++ b/lib/libzutil/Makefile.am @@ -21,13 +21,24 @@ USER_C += \ os/linux/zutil_compat.c endif +if BUILD_FREEBSD +USER_C += \ + os/freebsd/zutil_device_path_os.c \ + os/freebsd/zutil_import_os.c \ + os/freebsd/zutil_compat.c +endif + nodist_libzutil_la_SOURCES = $(USER_C) libzutil_la_LIBADD = \ $(top_builddir)/lib/libavl/libavl.la \ - $(top_builddir)/lib/libefi/libefi.la \ $(top_builddir)/lib/libtpool/libtpool.la +if BUILD_LINUX +libzutil_la_LIBADD += \ + $(top_builddir)/lib/libefi/libefi.la +endif + libzutil_la_LIBADD += -lm $(LIBBLKID) $(LIBUDEV) EXTRA_DIST = $(USER_C) diff --git a/lib/libzutil/os/freebsd/zutil_compat.c b/lib/libzutil/os/freebsd/zutil_compat.c new file mode 100644 index 00000000000..5b0fe85485d --- /dev/null +++ b/lib/libzutil/os/freebsd/zutil_compat.c @@ -0,0 +1,74 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +#include +#include +#include +#include +#include + +static int +zcmd_ioctl_compat(int fd, int request, zfs_cmd_t *zc, const int cflag) +{ + int ret; + void *zc_c; + unsigned long ncmd; + zfs_iocparm_t zp; + + switch (cflag) { + case ZFS_CMD_COMPAT_NONE: + ncmd = _IOWR('Z', request, zfs_iocparm_t); + zp.zfs_cmd = (uint64_t)zc; + zp.zfs_cmd_size = sizeof (zfs_cmd_t); + zp.zfs_ioctl_version = ZFS_IOCVER_ZOF; + return (ioctl(fd, ncmd, &zp)); + default: + abort(); + return (EINVAL); + } + + ret = ioctl(fd, ncmd, zc_c); + zfs_cmd_compat_get(zc, (caddr_t)zc_c, cflag); + free(zc_c); + + return (ret); +} + +/* + * This is FreeBSD version of ioctl, because Solaris' ioctl() updates + * zc_nvlist_dst_size even if an error is returned, on FreeBSD if an + * error is returned zc_nvlist_dst_size won't be updated. + */ +int +zfs_ioctl_fd(int fd, unsigned long request, zfs_cmd_t *zc) +{ + size_t oldsize; + int ret, cflag = ZFS_CMD_COMPAT_NONE; + + oldsize = zc->zc_nvlist_dst_size; + ret = zcmd_ioctl_compat(fd, request, zc, cflag); + + if (ret == 0 && oldsize < zc->zc_nvlist_dst_size) { + ret = -1; + errno = ENOMEM; + } + + return (ret); +} diff --git a/lib/libzutil/os/freebsd/zutil_device_path_os.c b/lib/libzutil/os/freebsd/zutil_device_path_os.c new file mode 100644 index 00000000000..71c93600524 --- /dev/null +++ b/lib/libzutil/os/freebsd/zutil_device_path_os.c @@ -0,0 +1,132 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include + +/* + * We don't strip/append partitions on FreeBSD. + */ + +/* + * Note: The caller must free the returned string. + */ +char * +zfs_strip_partition(char *dev) +{ + return (strdup(dev)); +} + +int +zfs_append_partition(char *path, size_t max_len) +{ + return (strnlen(path, max_len)); +} + +/* + * Strip the path from a device name. + * On FreeBSD we only want to remove "/dev/" from the beginning of + * paths if present. + */ +char * +zfs_strip_path(char *path) +{ + if (strncmp(path, _PATH_DEV, sizeof (_PATH_DEV) - 1) == 0) + return (path + sizeof (_PATH_DEV) - 1); + else + return (path); +} + +char * +zfs_get_underlying_path(const char *dev_name) +{ + + if (dev_name == NULL) + return (NULL); + + return (realpath(dev_name, NULL)); +} + +boolean_t +zfs_dev_is_whole_disk(const char *dev_name) +{ + int fd; + + fd = g_open(dev_name, 0); + if (fd >= 0) { + g_close(fd); + return (B_TRUE); + } + return (B_FALSE); +} + +/* + * Wait up to timeout_ms for udev to set up the device node. The device is + * considered ready when libudev determines it has been initialized, all of + * the device links have been verified to exist, and it has been allowed to + * settle. At this point the device the device can be accessed reliably. + * Depending on the complexity of the udev rules this process could take + * several seconds. + */ +int +zpool_label_disk_wait(const char *path, int timeout_ms) +{ + int settle_ms = 50; + long sleep_ms = 10; + hrtime_t start, settle; + struct stat64 statbuf; + + start = gethrtime(); + settle = 0; + + do { + errno = 0; + if ((stat64(path, &statbuf) == 0) && (errno == 0)) { + if (settle == 0) + settle = gethrtime(); + else if (NSEC2MSEC(gethrtime() - settle) >= settle_ms) + return (0); + } else if (errno != ENOENT) { + return (errno); + } + + usleep(sleep_ms * MILLISEC); + } while (NSEC2MSEC(gethrtime() - start) < timeout_ms); + + return (ENODEV); +} + +/* ARGSUSED */ +boolean_t +is_mpath_whole_disk(const char *path) +{ + return (B_FALSE); +} diff --git a/lib/libzutil/os/freebsd/zutil_import_os.c b/lib/libzutil/os/freebsd/zutil_import_os.c new file mode 100644 index 00000000000..7272a1e0ef0 --- /dev/null +++ b/lib/libzutil/os/freebsd/zutil_import_os.c @@ -0,0 +1,239 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2017 by Delphix. All rights reserved. + * Copyright 2015 RackTop Systems. + * Copyright 2016 Nexenta Systems, Inc. + */ + +/* + * Pool import support functions. + * + * To import a pool, we rely on reading the configuration information from the + * ZFS label of each device. If we successfully read the label, then we + * organize the configuration information in the following hierarchy: + * + * pool guid -> toplevel vdev guid -> label txg + * + * Duplicate entries matching this same tuple will be discarded. Once we have + * examined every device, we pick the best label txg config for each toplevel + * vdev. We then arrange these toplevel vdevs into a complete pool config, and + * update any paths that have changed. Finally, we attempt to import the pool + * using our derived config, and record the results. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#include "zutil_import.h" + +/* + * Update a leaf vdev's persistent device strings + * + * - only applies for a dedicated leaf vdev (aka whole disk) + * - updated during pool create|add|attach|import + * - used for matching device matching during auto-{online,expand,replace} + * - stored in a leaf disk config label (i.e. alongside 'path' NVP) + * - these strings are currently not used in kernel (i.e. for vdev_disk_open) + * + * On FreeBSD we currently just strip devid and phys_path to avoid confusion. + */ +void +update_vdev_config_dev_strs(nvlist_t *nv) +{ + (void) nvlist_remove_all(nv, ZPOOL_CONFIG_DEVID); + (void) nvlist_remove_all(nv, ZPOOL_CONFIG_PHYS_PATH); +} + +/* + * Do not even look at these devices. + */ +static const char * const blacklist_devs[] = { + "nfslock", + "sequencer", + "zfs", +}; +#define BLACKLIST_DIR "/dev/" +#define BLACKLIST_DIR_LEN 5 + +void +zpool_open_func(void *arg) +{ + rdsk_node_t *rn = arg; + struct stat64 statbuf; + nvlist_t *config; + size_t i; + int num_labels; + int fd; + off_t mediasize = 0; + + /* + * Do not even look at blacklisted devices. + */ + if (strncmp(rn->rn_name, BLACKLIST_DIR, BLACKLIST_DIR_LEN) == 0) { + char *name = rn->rn_name + BLACKLIST_DIR_LEN; + for (i = 0; i < nitems(blacklist_devs); ++i) { + const char *badname = blacklist_devs[i]; + size_t len = strlen(badname); + if (strncmp(name, badname, len) == 0) { + return; + } + } + } + + /* + * O_NONBLOCK so we don't hang trying to open things like serial ports. + */ + if ((fd = open(rn->rn_name, O_RDONLY|O_NONBLOCK)) < 0) + return; + + /* + * Ignore failed stats. + */ + if (fstat64(fd, &statbuf) != 0) + goto out; + /* + * We only want regular files, character devs and block devs. + */ + if (S_ISREG(statbuf.st_mode)) { + /* Check if this file is too small to hold a zpool. */ + if (statbuf.st_size < SPA_MINDEVSIZE) { + goto out; + } + } else if (S_ISCHR(statbuf.st_mode) || S_ISBLK(statbuf.st_mode)) { + /* Check if this device is too small to hold a zpool. */ + if (ioctl(fd, DIOCGMEDIASIZE, &mediasize) != 0 || + mediasize < SPA_MINDEVSIZE) { + goto out; + } + } else { + goto out; + } + + if (zpool_read_label(fd, &config, &num_labels) != 0) + goto out; + if (num_labels == 0) { + nvlist_free(config); + goto out; + } + + rn->rn_config = config; + rn->rn_num_labels = num_labels; + + /* TODO: Reuse labelpaths logic from Linux? */ +out: + (void) close(fd); +} + +static const char * +zpool_default_import_path[] = { + "/dev" +}; + +const char * const * +zpool_default_search_paths(size_t *count) +{ + *count = nitems(zpool_default_import_path); + return (zpool_default_import_path); +} + +int +zpool_find_import_blkid(libpc_handle_t *hdl, pthread_mutex_t *lock, + avl_tree_t **slice_cache) +{ + char *end, path[MAXPATHLEN]; + rdsk_node_t *slice; + struct gmesh mesh; + struct gclass *mp; + struct ggeom *gp; + struct gprovider *pp; + avl_index_t where; + size_t pathleft; + int error; + + end = stpcpy(path, "/dev/"); + pathleft = &path[sizeof (path)] - end; + + error = geom_gettree(&mesh); + if (error != 0) + return (error); + + *slice_cache = zutil_alloc(hdl, sizeof (avl_tree_t)); + avl_create(*slice_cache, slice_cache_compare, sizeof (rdsk_node_t), + offsetof(rdsk_node_t, rn_node)); + + LIST_FOREACH(mp, &mesh.lg_class, lg_class) { + LIST_FOREACH(gp, &mp->lg_geom, lg_geom) { + LIST_FOREACH(pp, &gp->lg_provider, lg_provider) { + strlcpy(end, pp->lg_name, pathleft); + slice = zutil_alloc(hdl, sizeof (rdsk_node_t)); + slice->rn_name = zutil_strdup(hdl, path); + slice->rn_vdev_guid = 0; + slice->rn_lock = lock; + slice->rn_avl = *slice_cache; + slice->rn_hdl = hdl; + slice->rn_labelpaths = B_FALSE; + slice->rn_order = IMPORT_ORDER_DEFAULT; + + pthread_mutex_lock(lock); + if (avl_find(*slice_cache, slice, &where)) { + free(slice->rn_name); + free(slice); + } else { + avl_insert(*slice_cache, slice, where); + } + pthread_mutex_unlock(lock); + } + } + } + + geom_deletetree(&mesh); + + return (0); +} + +int +zfs_dev_flush(int fd __unused) +{ + return (0); +} diff --git a/module/.gitignore b/module/.gitignore index 45e5f992223..5f3d7048729 100644 --- a/module/.gitignore +++ b/module/.gitignore @@ -2,6 +2,8 @@ *.ko.unsigned *.ko.out *.ko.out.sig +*.ko.debug +*.ko.full *.dwo .*.cmd .*.d @@ -11,5 +13,13 @@ /.tmp_versions /Module.markers /Module.symvers +/vnode_if* +/bus_if.h +/device_if.h +/opt_global.h + +/export_syms +/machine +/x86 !Makefile.in diff --git a/module/Makefile.bsd b/module/Makefile.bsd new file mode 100644 index 00000000000..6d76796f51e --- /dev/null +++ b/module/Makefile.bsd @@ -0,0 +1,381 @@ +.if !defined(WITH_CTF) +WITH_CTF=1 +.endif + +.include + +SRCDIR= ${.CURDIR} +INCDIR=${.CURDIR:H}/include + +KMOD= openzfs + +.PATH: ${SRCDIR}/avl \ + ${SRCDIR}/lua \ + ${SRCDIR}/nvpair \ + ${SRCDIR}/os/freebsd/spl \ + ${SRCDIR}/os/freebsd/zfs \ + ${SRCDIR}/unicode \ + ${SRCDIR}/zcommon \ + ${SRCDIR}/zfs + + +CFLAGS+= -I${INCDIR} +CFLAGS+= -I${INCDIR}/spl +CFLAGS+= -I${INCDIR}/os/freebsd +CFLAGS+= -I${INCDIR}/os/freebsd/spl +CFLAGS+= -I${INCDIR}/os/freebsd/zfs +CFLAGS+= -include ${INCDIR}/os/freebsd/spl/sys/ccompile.h + +CFLAGS+= -D__KERNEL__ -DFREEBSD_NAMECACHE -DBUILDING_ZFS -D__BSD_VISIBLE=1 +CFLAGS+= -DHAVE_UIO_ZEROCOPY -DWITHOUT_NETDUMP -D__KERNEL -D_SYS_CONDVAR_H_ +CFLAGS+= -D_SYS_VMEM_H_ -D_MACHINE_ENDIAN_H_ -DKDTRACE_HOOKS -DSMP + +.if ${MACHINE_ARCH} == "amd64" +CFLAGS+= -DHAVE_AVX2 -DHAVE_AVX -D__x86_64 -DHAVE_SSE2 -DHAVE_AVX512F +.endif + +.if defined(WITH_DEBUG) && ${WITH_DEBUG} == "true" +CFLAGS+= -DINVARIANTS -DWITNESS -g -O0 -DZFS_DEBUG -DOPENSOLARIS_WITNESS +.else +CFLAGS += -DNDEBUG +.endif + +.if defined(WITH_VFS_DEBUG) && ${WITH_VFS_DEBUG} == "true" +# kernel must also be built with this option for this to work +CFLAGS+= -DDEBUG_VFS_LOCKS +.endif + +.if defined(WITH_GCOV) && ${WITH_GCOV} == "true" +CFLAGS+= -fprofile-arcs -ftest-coverage +.endif + +DEBUG_FLAGS=-g + +.if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "powerpc" || \ + ${MACHINE_ARCH} == "arm" +CFLAGS+= -DBITS_PER_LONG=32 +.else +CFLAGS+= -DBITS_PER_LONG=64 +.endif + +SRCS= vnode_if.h device_if.h bus_if.h + +# avl +SRCS+= avl.c + +#lua +SRCS+= lapi.c \ + lauxlib.c \ + lbaselib.c \ + lcode.c \ + lcompat.c \ + lcorolib.c \ + lctype.c \ + ldebug.c \ + ldo.c \ + lfunc.c \ + lgc.c \ + llex.c \ + lmem.c \ + lobject.c \ + lopcodes.c \ + lparser.c \ + lstate.c \ + lstring.c \ + lstrlib.c \ + ltable.c \ + ltablib.c \ + ltm.c \ + lvm.c \ + lzio.c + +#nvpair +SRCS+= nvpair.c \ + fnvpair.c \ + nvpair_alloc_spl.c \ + nvpair_alloc_fixed.c + +#os/freebsd/spl +SRCS+= acl_common.c \ + btree.c \ + callb.c \ + list.c \ + spl_acl.c \ + spl_cmn_err.c \ + spl_dtrace.c \ + spl_kmem.c \ + spl_kstat.c \ + spl_misc.c \ + spl_policy.c \ + spl_string.c \ + spl_sunddi.c \ + spl_sysevent.c \ + spl_taskq.c \ + spl_uio.c \ + spl_vfs.c \ + spl_vm.c \ + spl_zone.c \ + sha256c.c \ + sha512c.c \ + spl_procfs_list.c \ + spl_zlib.c + + +.if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "powerpc" || \ + ${MACHINE_ARCH} == "arm" +SRCS+= spl_atomic.c +.endif + +#os/freebsd/zfs +SRCS+= abd.c \ + crypto_os.c \ + dmu_os.c \ + hkdf.c \ + kmod_core.c \ + spa_os.c \ + sysctl_os.c \ + vdev_file.c \ + vdev_label_os.c \ + vdev_geom.c \ + zfs_acl.c \ + zfs_ctldir.c \ + zfs_dir.c \ + zfs_ioctl_os.c \ + zfs_log.c \ + zfs_replay.c \ + zfs_vfsops.c \ + zfs_vnops.c \ + zfs_znode.c \ + zio_crypt.c \ + zvol_os.c + +#unicode +SRCS+= uconv.c \ + u8_textprep.c + +#zcommon +SRCS+= zfeature_common.c \ + zfs_comutil.c \ + zfs_deleg.c \ + zfs_fletcher.c \ + zfs_fletcher_avx512.c \ + zfs_fletcher_intel.c \ + zfs_fletcher_sse.c \ + zfs_fletcher_superscalar.c \ + zfs_fletcher_superscalar4.c \ + zfs_namecheck.c \ + zfs_prop.c \ + zpool_prop.c \ + zprop_common.c + +#zfs +SRCS+= aggsum.c \ + arc.c \ + arc_os.c \ + blkptr.c \ + bplist.c \ + bpobj.c \ + cityhash.c \ + dbuf.c \ + dbuf_stats.c \ + bptree.c \ + bqueue.c \ + dataset_kstats.c \ + ddt.c \ + ddt_zap.c \ + dmu.c \ + dmu_diff.c \ + dmu_object.c \ + dmu_objset.c \ + dmu_recv.c \ + dmu_redact.c \ + dmu_send.c \ + dmu_traverse.c \ + dmu_tx.c \ + dmu_zfetch.c \ + dnode.c \ + dnode_sync.c \ + dsl_dataset.c \ + dsl_deadlist.c \ + dsl_deleg.c \ + dsl_bookmark.c \ + dsl_dir.c \ + dsl_crypt.c \ + dsl_destroy.c \ + dsl_pool.c \ + dsl_prop.c \ + dsl_scan.c \ + dsl_synctask.c \ + dsl_userhold.c \ + fm.c \ + gzip.c \ + lzjb.c \ + lz4.c \ + metaslab.c \ + mmp.c \ + multilist.c \ + objlist.c \ + pathname.c \ + range_tree.c \ + refcount.c \ + rrwlock.c \ + sa.c \ + sha256.c \ + skein_zfs.c \ + spa.c \ + spa_boot.c \ + spa_checkpoint.c \ + spa_config.c \ + spa_errlog.c \ + spa_history.c \ + spa_log_spacemap.c \ + spa_misc.c \ + spa_stats.c \ + space_map.c \ + space_reftree.c \ + txg.c \ + uberblock.c \ + unique.c \ + vdev.c \ + vdev_cache.c \ + vdev_indirect.c \ + vdev_indirect_births.c \ + vdev_indirect_mapping.c \ + vdev_initialize.c \ + vdev_label.c \ + vdev_mirror.c \ + vdev_missing.c \ + vdev_queue.c \ + vdev_raidz.c \ + vdev_raidz_math.c \ + vdev_raidz_math_scalar.c \ + vdev_raidz_math_avx2.c \ + vdev_raidz_math_avx512bw.c \ + vdev_raidz_math_avx512f.c \ + vdev_raidz_math_sse2.c \ + vdev_raidz_math_ssse3.c \ + vdev_removal.c \ + vdev_root.c \ + vdev_trim.c \ + zap.c \ + zap_leaf.c \ + zap_micro.c \ + zcp.c \ + zcp_get.c \ + zcp_global.c \ + zcp_iter.c \ + zcp_set.c \ + zcp_synctask.c \ + zfeature.c \ + zfs_byteswap.c \ + zfs_debug.c \ + zfs_file_os.c \ + zfs_fm.c \ + zfs_fuid.c \ + zfs_fuid_os.c \ + zfs_ioctl.c \ + zfs_onexit.c \ + zfs_quota.c \ + zfs_ratelimit.c \ + zfs_rlock.c \ + zfs_sa.c \ + zil.c \ + zio.c \ + zio_checksum.c \ + zio_compress.c \ + zio_inject.c \ + zle.c \ + zrlock.c \ + zthr.c \ + zvol.c + +beforeinstall: +.if ${MK_DEBUG_FILES} != "no" + mtree -eu \ + -f /etc/mtree/BSD.debug.dist \ + -p ${DESTDIR}/usr/lib +.endif + +.include + + +CFLAGS.gcc+= -Wno-pointer-to-int-cast + +CFLAGS.lapi.c= -Wno-cast-qual +CFLAGS.lcompat.c= -Wno-cast-qual -Wno-missing-prototypes +CFLAGS.lobject.c= -Wno-cast-qual +CFLAGS.ltable.c= -Wno-cast-qual +CFLAGS.lvm.c= -Wno-cast-qual +CFLAGS.nvpair.c= -Wno-cast-qual +CFLAGS.acl_common.c= -Wno-strict-prototypes -Wno-missing-prototypes +CFLAGS.callb.c= -Wno-strict-prototypes -Wno-missing-prototypes +CFLAGS.spl_kstat.c= -Wno-missing-prototypes +CFLAGS.spl_string.c= -Wno-cast-qual +CFLAGS.spl_vm.c= -Wno-cast-qual -Wno-missing-prototypes +CFLAGS.spl_zlib.c= -Wno-cast-qual +CFLAGS.abd.c= -Wno-cast-qual +CFLAGS.freebsd_dmu.c= -Wno-missing-prototypes +CFLAGS.freebsd_kmod.c= -Wno-missing-prototypes +CFLAGS.vdev_geom.c= -Wno-missing-prototypes +CFLAGS.zfs_acl.c= -Wno-missing-prototypes +CFLAGS.zfs_ctldir.c= -Wno-missing-prototypes -Wno-strict-prototypes +CFLAGS.zfs_log.c= -Wno-cast-qual +CFLAGS.zfs_vfsops.c= -Wno-missing-prototypes +CFLAGS.zfs_vnops.c= -Wno-missing-prototypes -Wno-strict-prototypes -Wno-pointer-arith +CFLAGS.zfs_znode.c= -Wno-missing-prototypes +CFLAGS.zvol.c= -Wno-missing-prototypes +CFLAGS.u8_textprep.c= -Wno-cast-qual +CFLAGS.zfs_fletcher.c= -Wno-cast-qual -Wno-pointer-arith +CFLAGS.zfs_fletcher_intel.c= -Wno-cast-qual -Wno-pointer-arith +CFLAGS.zfs_fletcher_sse.c= -Wno-cast-qual -Wno-pointer-arith +CFLAGS.zfs_fletcher_avx512.c= -Wno-cast-qual -Wno-pointer-arith +CFLAGS.zprop_common.c= -Wno-cast-qual +CFLAGS.arc.c= -Wno-missing-prototypes +CFLAGS.blkptr.c= -Wno-missing-prototypes +CFLAGS.dbuf.c= -Wno-missing-prototypes +CFLAGS.dbuf_stats.c= -Wno-missing-prototypes +CFLAGS.ddt.c= -Wno-missing-prototypes -Wno-cast-qual +CFLAGS.dmu.c= -Wno-missing-prototypes -Wno-cast-qual +CFLAGS.dmu_object.c= -Wno-missing-prototypes +CFLAGS.dmu_objset.c= -Wno-missing-prototypes +CFLAGS.dmu_traverse.c= -Wno-cast-qual +CFLAGS.dsl_dir.c= -Wno-missing-prototypes -Wno-cast-qual +CFLAGS.dsl_crypt.c= -Wno-missing-prototypes +CFLAGS.dsl_deadlist.c= -Wno-cast-qual +CFLAGS.dsl_pool.c= -Wno-missing-prototypes +CFLAGS.dsl_prop.c= -Wno-cast-qual +CFLAGS.dsl_scan.c= -Wno-missing-prototypes +CFLAGS.fm.c= -Wno-cast-qual +CFLAGS.gzip.c= -Wno-missing-prototypes +CFLAGS.lzjb.c= -Wno-missing-prototypes +CFLAGS.lz4.c= -Wno-missing-prototypes -Wno-cast-qual +CFLAGS.metaslab.c= -Wno-missing-prototypes +CFLAGS.sa.c= -Wno-missing-prototypes +CFLAGS.sha256.c= -Wno-missing-prototypes +CFLAGS.skein_zfs.c= -Wno-missing-prototypes +CFLAGS.spa.c= -Wno-missing-prototypes -Wno-cast-qual +CFLAGS.spa_boot.c= -Wno-missing-prototypes +CFLAGS.spa_misc.c= -Wno-missing-prototypes -Wno-cast-qual +CFLAGS.space_map.c= -Wno-missing-prototypes +CFLAGS.vdev.c= -Wno-missing-prototypes +CFLAGS.vdev_indirect.c= -Wno-missing-prototypes +CFLAGS.vdev_label.c= -Wno-missing-prototypes +CFLAGS.vdev_queue.c= -Wno-missing-prototypes +CFLAGS.vdev_raidz.c= -Wno-cast-qual +CFLAGS.vdev_raidz_math.c= -Wno-cast-qual +CFLAGS.vdev_raidz_math_scalar.c= -Wno-cast-qual -Wno-missing-prototypes +CFLAGS.vdev_raidz_math_avx2.c= -Wno-cast-qual -Wno-duplicate-decl-specifier +CFLAGS.vdev_raidz_math_avx512f.c= -Wno-cast-qual -Wno-duplicate-decl-specifier +CFLAGS.vdev_raidz_math_sse2.c= -Wno-cast-qual -Wno-duplicate-decl-specifier +CFLAGS.zap_leaf.c= -Wno-cast-qual +CFLAGS.zap_micro.c= -Wno-missing-prototypes -Wno-cast-qual +CFLAGS.zcp.c= -Wno-cast-qual +CFLAGS.zcp_get.c= -Wno-missing-prototypes +CFLAGS.zfs_debug.c= -Wno-missing-prototypes +CFLAGS.zfs_fm.c= -Wno-cast-qual +CFLAGS.zfs_ioctl.c= -Wno-missing-prototypes -Wno-cast-qual +CFLAGS.zil.c= -Wno-missing-prototypes -Wno-cast-qual +CFLAGS.zio.c= -Wno-missing-prototypes -Wno-cast-qual +CFLAGS.zio_checksum.c= -Wno-missing-prototypes +CFLAGS.zle.c= -Wno-missing-prototypes +CFLAGS.zrlock.c= -Wno-missing-prototypes -Wno-cast-qual diff --git a/module/Makefile.in b/module/Makefile.in index 24af81aa771..39acdac20b1 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -12,7 +12,7 @@ obj-m += os/linux/zfs/ INSTALL_MOD_DIR ?= extra ZFS_MODULE_CFLAGS += -std=gnu99 -Wno-declaration-after-statement -ZFS_MODULE_CFLAGS += @KERNEL_DEBUG_CFLAGS@ +ZFS_MODULE_CFLAGS += @KERNEL_DEBUG_CFLAGS@ @NO_FORMAT_ZERO_LENGTH@ ZFS_MODULE_CFLAGS += -include @abs_top_builddir@/zfs_config.h ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include/os/linux/kernel ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include/os/linux/spl @@ -40,6 +40,11 @@ modules-Linux: done $(MAKE) -C @LINUX_OBJ@ M=`pwd` @KERNEL_MAKE@ CONFIG_ZFS=m modules +# Only pass down gmake -j flag, if used. +modules-FreeBSD: + flags="$$(echo $$MAKEFLAGS | awk -v RS=' ' /^-j/)"; \ + env MAKEFLAGS="" make $${flags} -f Makefile.bsd + modules-unknown: @true @@ -55,6 +60,10 @@ clean-Linux: find . -name '*.ur-safe' -type f -print | xargs $(RM) +clean-FreeBSD: + flags="$$(echo $$MAKEFLAGS | awk -v RS=' ' /^-j/)"; \ + env MAKEFLAGS="" make $${flags} -f Makefile.bsd clean + clean: clean-@ac_system@ modules_install-Linux: @@ -100,6 +109,11 @@ cscopelist-am: $(am__tagged_files) fi; \ done >> $(top_builddir)/cscope.files +modules_install-FreeBSD: + @# Install the kernel modules + flags="$$(echo $$MAKEFLAGS | awk -v RS=' ' /^-j/)"; \ + env MAKEFLAGS="" make $${flags} -f Makefile.bsd install + modules_install: modules_install-@ac_system@ modules_uninstall-Linux: @@ -109,11 +123,16 @@ modules_uninstall-Linux: $(RM) -R $$kmoddir/$(INSTALL_MOD_DIR)/$$objdir; \ done +modules_uninstall-FreeBSD: + @false + modules_uninstall: modules_uninstall-@ac_system@ distdir: list='$(obj-m)'; for objdir in $$list; do \ - (cd @top_srcdir@/module && find $$objdir \ - -name '*.c' -o -name '*.h' -o -name '*.S' | \ - xargs -r cp --parents -t @abs_top_builddir@/module/$$distdir); \ + (cd @top_srcdir@/module && find $$objdir -name '*.[chS]' | \ + while read path; do \ + mkdir -p @abs_top_builddir@/module/$$distdir/$${path%/*}; \ + cp $$path @abs_top_builddir@/module/$$distdir/$$path; \ + done); \ done diff --git a/module/os/freebsd/spl/acl_common.c b/module/os/freebsd/spl/acl_common.c new file mode 100644 index 00000000000..8eea4695eb0 --- /dev/null +++ b/module/os/freebsd/spl/acl_common.c @@ -0,0 +1,1731 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright 2011 Nexenta Systems, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#if defined(_KERNEL) +#include +#include +#include +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define ASSERT assert +#endif + +#define ACE_POSIX_SUPPORTED_BITS (ACE_READ_DATA | \ + ACE_WRITE_DATA | ACE_APPEND_DATA | ACE_EXECUTE | \ + ACE_READ_ATTRIBUTES | ACE_READ_ACL | ACE_WRITE_ACL) + + +#define ACL_SYNCHRONIZE_SET_DENY 0x0000001 +#define ACL_SYNCHRONIZE_SET_ALLOW 0x0000002 +#define ACL_SYNCHRONIZE_ERR_DENY 0x0000004 +#define ACL_SYNCHRONIZE_ERR_ALLOW 0x0000008 + +#define ACL_WRITE_OWNER_SET_DENY 0x0000010 +#define ACL_WRITE_OWNER_SET_ALLOW 0x0000020 +#define ACL_WRITE_OWNER_ERR_DENY 0x0000040 +#define ACL_WRITE_OWNER_ERR_ALLOW 0x0000080 + +#define ACL_DELETE_SET_DENY 0x0000100 +#define ACL_DELETE_SET_ALLOW 0x0000200 +#define ACL_DELETE_ERR_DENY 0x0000400 +#define ACL_DELETE_ERR_ALLOW 0x0000800 + +#define ACL_WRITE_ATTRS_OWNER_SET_DENY 0x0001000 +#define ACL_WRITE_ATTRS_OWNER_SET_ALLOW 0x0002000 +#define ACL_WRITE_ATTRS_OWNER_ERR_DENY 0x0004000 +#define ACL_WRITE_ATTRS_OWNER_ERR_ALLOW 0x0008000 + +#define ACL_WRITE_ATTRS_WRITER_SET_DENY 0x0010000 +#define ACL_WRITE_ATTRS_WRITER_SET_ALLOW 0x0020000 +#define ACL_WRITE_ATTRS_WRITER_ERR_DENY 0x0040000 +#define ACL_WRITE_ATTRS_WRITER_ERR_ALLOW 0x0080000 + +#define ACL_WRITE_NAMED_WRITER_SET_DENY 0x0100000 +#define ACL_WRITE_NAMED_WRITER_SET_ALLOW 0x0200000 +#define ACL_WRITE_NAMED_WRITER_ERR_DENY 0x0400000 +#define ACL_WRITE_NAMED_WRITER_ERR_ALLOW 0x0800000 + +#define ACL_READ_NAMED_READER_SET_DENY 0x1000000 +#define ACL_READ_NAMED_READER_SET_ALLOW 0x2000000 +#define ACL_READ_NAMED_READER_ERR_DENY 0x4000000 +#define ACL_READ_NAMED_READER_ERR_ALLOW 0x8000000 + + +#define ACE_VALID_MASK_BITS (\ + ACE_READ_DATA | \ + ACE_LIST_DIRECTORY | \ + ACE_WRITE_DATA | \ + ACE_ADD_FILE | \ + ACE_APPEND_DATA | \ + ACE_ADD_SUBDIRECTORY | \ + ACE_READ_NAMED_ATTRS | \ + ACE_WRITE_NAMED_ATTRS | \ + ACE_EXECUTE | \ + ACE_DELETE_CHILD | \ + ACE_READ_ATTRIBUTES | \ + ACE_WRITE_ATTRIBUTES | \ + ACE_DELETE | \ + ACE_READ_ACL | \ + ACE_WRITE_ACL | \ + ACE_WRITE_OWNER | \ + ACE_SYNCHRONIZE) + +#define ACE_MASK_UNDEFINED 0x80000000 + +#define ACE_VALID_FLAG_BITS (ACE_FILE_INHERIT_ACE | \ + ACE_DIRECTORY_INHERIT_ACE | \ + ACE_NO_PROPAGATE_INHERIT_ACE | ACE_INHERIT_ONLY_ACE | \ + ACE_SUCCESSFUL_ACCESS_ACE_FLAG | ACE_FAILED_ACCESS_ACE_FLAG | \ + ACE_IDENTIFIER_GROUP | ACE_OWNER | ACE_GROUP | ACE_EVERYONE) + +/* + * ACL conversion helpers + */ + +typedef enum { + ace_unused, + ace_user_obj, + ace_user, + ace_group, /* includes GROUP and GROUP_OBJ */ + ace_other_obj +} ace_to_aent_state_t; + +typedef struct acevals { + uid_t key; + avl_node_t avl; + uint32_t mask; + uint32_t allowed; + uint32_t denied; + int aent_type; +} acevals_t; + +typedef struct ace_list { + acevals_t user_obj; + avl_tree_t user; + int numusers; + acevals_t group_obj; + avl_tree_t group; + int numgroups; + acevals_t other_obj; + uint32_t acl_mask; + int hasmask; + int dfacl_flag; + ace_to_aent_state_t state; + int seen; /* bitmask of all aclent_t a_type values seen */ +} ace_list_t; + +/* + * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified. + * v = Ptr to array/vector of objs + * n = # objs in the array + * s = size of each obj (must be multiples of a word size) + * f = ptr to function to compare two objs + * returns (-1 = less than, 0 = equal, 1 = greater than + */ +void +ksort(caddr_t v, int n, int s, int (*f)()) +{ + int g, i, j, ii; + unsigned int *p1, *p2; + unsigned int tmp; + + /* No work to do */ + if (v == NULL || n <= 1) + return; + + /* Sanity check on arguments */ + ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0); + ASSERT(s > 0); + for (g = n / 2; g > 0; g /= 2) { + for (i = g; i < n; i++) { + for (j = i - g; j >= 0 && + (*f)(v + j * s, v + (j + g) * s) == 1; + j -= g) { + p1 = (void *)(v + j * s); + p2 = (void *)(v + (j + g) * s); + for (ii = 0; ii < s / 4; ii++) { + tmp = *p1; + *p1++ = *p2; + *p2++ = tmp; + } + } + } + } +} + +/* + * Compare two acls, all fields. Returns: + * -1 (less than) + * 0 (equal) + * +1 (greater than) + */ +int +cmp2acls(void *a, void *b) +{ + aclent_t *x = (aclent_t *)a; + aclent_t *y = (aclent_t *)b; + + /* Compare types */ + if (x->a_type < y->a_type) + return (-1); + if (x->a_type > y->a_type) + return (1); + /* Equal types; compare id's */ + if (x->a_id < y->a_id) + return (-1); + if (x->a_id > y->a_id) + return (1); + /* Equal ids; compare perms */ + if (x->a_perm < y->a_perm) + return (-1); + if (x->a_perm > y->a_perm) + return (1); + /* Totally equal */ + return (0); +} + +static int +cacl_malloc(void **ptr, size_t size) +{ + *ptr = kmem_zalloc(size, KM_SLEEP); + return (0); +} + + +#if !defined(_KERNEL) +acl_t * +acl_alloc(enum acl_type type) +{ + acl_t *aclp; + + if (cacl_malloc((void **)&aclp, sizeof (acl_t)) != 0) + return (NULL); + + aclp->acl_aclp = NULL; + aclp->acl_cnt = 0; + + switch (type) { + case ACE_T: + aclp->acl_type = ACE_T; + aclp->acl_entry_size = sizeof (ace_t); + break; + case ACLENT_T: + aclp->acl_type = ACLENT_T; + aclp->acl_entry_size = sizeof (aclent_t); + break; + default: + acl_free(aclp); + aclp = NULL; + } + return (aclp); +} + +/* + * Free acl_t structure + */ +void +acl_free(acl_t *aclp) +{ + int acl_size; + + if (aclp == NULL) + return; + + if (aclp->acl_aclp) { + acl_size = aclp->acl_cnt * aclp->acl_entry_size; + cacl_free(aclp->acl_aclp, acl_size); + } + + cacl_free(aclp, sizeof (acl_t)); +} + +static uint32_t +access_mask_set(int haswriteperm, int hasreadperm, int isowner, int isallow) +{ + uint32_t access_mask = 0; + int acl_produce; + int synchronize_set = 0, write_owner_set = 0; + int delete_set = 0, write_attrs_set = 0; + int read_named_set = 0, write_named_set = 0; + + acl_produce = (ACL_SYNCHRONIZE_SET_ALLOW | + ACL_WRITE_ATTRS_OWNER_SET_ALLOW | + ACL_WRITE_ATTRS_WRITER_SET_DENY); + + if (isallow) { + synchronize_set = ACL_SYNCHRONIZE_SET_ALLOW; + write_owner_set = ACL_WRITE_OWNER_SET_ALLOW; + delete_set = ACL_DELETE_SET_ALLOW; + if (hasreadperm) + read_named_set = ACL_READ_NAMED_READER_SET_ALLOW; + if (haswriteperm) + write_named_set = ACL_WRITE_NAMED_WRITER_SET_ALLOW; + if (isowner) + write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_ALLOW; + else if (haswriteperm) + write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_ALLOW; + } else { + + synchronize_set = ACL_SYNCHRONIZE_SET_DENY; + write_owner_set = ACL_WRITE_OWNER_SET_DENY; + delete_set = ACL_DELETE_SET_DENY; + if (hasreadperm) + read_named_set = ACL_READ_NAMED_READER_SET_DENY; + if (haswriteperm) + write_named_set = ACL_WRITE_NAMED_WRITER_SET_DENY; + if (isowner) + write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_DENY; + else if (haswriteperm) + write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_DENY; + else + /* + * If the entity is not the owner and does not + * have write permissions ACE_WRITE_ATTRIBUTES will + * always go in the DENY ACE. + */ + access_mask |= ACE_WRITE_ATTRIBUTES; + } + + if (acl_produce & synchronize_set) + access_mask |= ACE_SYNCHRONIZE; + if (acl_produce & write_owner_set) + access_mask |= ACE_WRITE_OWNER; + if (acl_produce & delete_set) + access_mask |= ACE_DELETE; + if (acl_produce & write_attrs_set) + access_mask |= ACE_WRITE_ATTRIBUTES; + if (acl_produce & read_named_set) + access_mask |= ACE_READ_NAMED_ATTRS; + if (acl_produce & write_named_set) + access_mask |= ACE_WRITE_NAMED_ATTRS; + + return (access_mask); +} + +/* + * Given an mode_t, convert it into an access_mask as used + * by nfsace, assuming aclent_t -> nfsace semantics. + */ +static uint32_t +mode_to_ace_access(mode_t mode, boolean_t isdir, int isowner, int isallow) +{ + uint32_t access = 0; + int haswriteperm = 0; + int hasreadperm = 0; + + if (isallow) { + haswriteperm = (mode & S_IWOTH); + hasreadperm = (mode & S_IROTH); + } else { + haswriteperm = !(mode & S_IWOTH); + hasreadperm = !(mode & S_IROTH); + } + + /* + * The following call takes care of correctly setting the following + * mask bits in the access_mask: + * ACE_SYNCHRONIZE, ACE_WRITE_OWNER, ACE_DELETE, + * ACE_WRITE_ATTRIBUTES, ACE_WRITE_NAMED_ATTRS, ACE_READ_NAMED_ATTRS + */ + access = access_mask_set(haswriteperm, hasreadperm, isowner, isallow); + + if (isallow) { + access |= ACE_READ_ACL | ACE_READ_ATTRIBUTES; + if (isowner) + access |= ACE_WRITE_ACL; + } else { + if (! isowner) + access |= ACE_WRITE_ACL; + } + + /* read */ + if (mode & S_IROTH) { + access |= ACE_READ_DATA; + } + /* write */ + if (mode & S_IWOTH) { + access |= ACE_WRITE_DATA | + ACE_APPEND_DATA; + if (isdir) + access |= ACE_DELETE_CHILD; + } + /* exec */ + if (mode & S_IXOTH) { + access |= ACE_EXECUTE; + } + + return (access); +} + +/* + * Given an nfsace (presumably an ALLOW entry), make a + * corresponding DENY entry at the address given. + */ +static void +ace_make_deny(ace_t *allow, ace_t *deny, int isdir, int isowner) +{ + (void) memcpy(deny, allow, sizeof (ace_t)); + + deny->a_who = allow->a_who; + + deny->a_type = ACE_ACCESS_DENIED_ACE_TYPE; + deny->a_access_mask ^= ACE_POSIX_SUPPORTED_BITS; + if (isdir) + deny->a_access_mask ^= ACE_DELETE_CHILD; + + deny->a_access_mask &= ~(ACE_SYNCHRONIZE | ACE_WRITE_OWNER | + ACE_DELETE | ACE_WRITE_ATTRIBUTES | ACE_READ_NAMED_ATTRS | + ACE_WRITE_NAMED_ATTRS); + deny->a_access_mask |= access_mask_set((allow->a_access_mask & + ACE_WRITE_DATA), (allow->a_access_mask & ACE_READ_DATA), isowner, + B_FALSE); +} +/* + * Make an initial pass over an array of aclent_t's. Gather + * information such as an ACL_MASK (if any), number of users, + * number of groups, and whether the array needs to be sorted. + */ +static int +ln_aent_preprocess(aclent_t *aclent, int n, + int *hasmask, mode_t *mask, + int *numuser, int *numgroup, int *needsort) +{ + int error = 0; + int i; + int curtype = 0; + + *hasmask = 0; + *mask = 07; + *needsort = 0; + *numuser = 0; + *numgroup = 0; + + for (i = 0; i < n; i++) { + if (aclent[i].a_type < curtype) + *needsort = 1; + else if (aclent[i].a_type > curtype) + curtype = aclent[i].a_type; + if (aclent[i].a_type & USER) + (*numuser)++; + if (aclent[i].a_type & (GROUP | GROUP_OBJ)) + (*numgroup)++; + if (aclent[i].a_type & CLASS_OBJ) { + if (*hasmask) { + error = EINVAL; + goto out; + } else { + *hasmask = 1; + *mask = aclent[i].a_perm; + } + } + } + + if ((! *hasmask) && (*numuser + *numgroup > 1)) { + error = EINVAL; + goto out; + } + +out: + return (error); +} + +/* + * Convert an array of aclent_t into an array of nfsace entries, + * following POSIX draft -> nfsv4 conversion semantics as outlined in + * the IETF draft. + */ +static int +ln_aent_to_ace(aclent_t *aclent, int n, ace_t **acepp, int *rescount, int isdir) +{ + int error = 0; + mode_t mask; + int numuser, numgroup, needsort; + int resultsize = 0; + int i, groupi = 0, skip; + ace_t *acep, *result = NULL; + int hasmask; + + error = ln_aent_preprocess(aclent, n, &hasmask, &mask, + &numuser, &numgroup, &needsort); + if (error != 0) + goto out; + + /* allow + deny for each aclent */ + resultsize = n * 2; + if (hasmask) { + /* + * stick extra deny on the group_obj and on each + * user|group for the mask (the group_obj was added + * into the count for numgroup) + */ + resultsize += numuser + numgroup; + /* ... and don't count the mask itself */ + resultsize -= 2; + } + + /* sort the source if necessary */ + if (needsort) + ksort((caddr_t)aclent, n, sizeof (aclent_t), cmp2acls); + + if (cacl_malloc((void **)&result, resultsize * sizeof (ace_t)) != 0) + goto out; + + acep = result; + + for (i = 0; i < n; i++) { + /* + * don't process CLASS_OBJ (mask); mask was grabbed in + * ln_aent_preprocess() + */ + if (aclent[i].a_type & CLASS_OBJ) + continue; + + /* If we need an ACL_MASK emulator, prepend it now */ + if ((hasmask) && + (aclent[i].a_type & (USER | GROUP | GROUP_OBJ))) { + acep->a_type = ACE_ACCESS_DENIED_ACE_TYPE; + acep->a_flags = 0; + if (aclent[i].a_type & GROUP_OBJ) { + acep->a_who = (uid_t)-1; + acep->a_flags |= + (ACE_IDENTIFIER_GROUP|ACE_GROUP); + } else if (aclent[i].a_type & USER) { + acep->a_who = aclent[i].a_id; + } else { + acep->a_who = aclent[i].a_id; + acep->a_flags |= ACE_IDENTIFIER_GROUP; + } + if (aclent[i].a_type & ACL_DEFAULT) { + acep->a_flags |= ACE_INHERIT_ONLY_ACE | + ACE_FILE_INHERIT_ACE | + ACE_DIRECTORY_INHERIT_ACE; + } + /* + * Set the access mask for the prepended deny + * ace. To do this, we invert the mask (found + * in ln_aent_preprocess()) then convert it to an + * DENY ace access_mask. + */ + acep->a_access_mask = mode_to_ace_access((mask ^ 07), + isdir, 0, 0); + acep += 1; + } + + /* handle a_perm -> access_mask */ + acep->a_access_mask = mode_to_ace_access(aclent[i].a_perm, + isdir, aclent[i].a_type & USER_OBJ, 1); + + /* emulate a default aclent */ + if (aclent[i].a_type & ACL_DEFAULT) { + acep->a_flags |= ACE_INHERIT_ONLY_ACE | + ACE_FILE_INHERIT_ACE | + ACE_DIRECTORY_INHERIT_ACE; + } + + /* + * handle a_perm and a_id + * + * this must be done last, since it involves the + * corresponding deny aces, which are handled + * differently for each different a_type. + */ + if (aclent[i].a_type & USER_OBJ) { + acep->a_who = (uid_t)-1; + acep->a_flags |= ACE_OWNER; + ace_make_deny(acep, acep + 1, isdir, B_TRUE); + acep += 2; + } else if (aclent[i].a_type & USER) { + acep->a_who = aclent[i].a_id; + ace_make_deny(acep, acep + 1, isdir, B_FALSE); + acep += 2; + } else if (aclent[i].a_type & (GROUP_OBJ | GROUP)) { + if (aclent[i].a_type & GROUP_OBJ) { + acep->a_who = (uid_t)-1; + acep->a_flags |= ACE_GROUP; + } else { + acep->a_who = aclent[i].a_id; + } + acep->a_flags |= ACE_IDENTIFIER_GROUP; + /* + * Set the corresponding deny for the group ace. + * + * The deny aces go after all of the groups, unlike + * everything else, where they immediately follow + * the allow ace. + * + * We calculate "skip", the number of slots to + * skip ahead for the deny ace, here. + * + * The pattern is: + * MD1 A1 MD2 A2 MD3 A3 D1 D2 D3 + * thus, skip is + * (2 * numgroup) - 1 - groupi + * (2 * numgroup) to account for MD + A + * - 1 to account for the fact that we're on the + * access (A), not the mask (MD) + * - groupi to account for the fact that we have + * passed up groupi number of MD's. + */ + skip = (2 * numgroup) - 1 - groupi; + ace_make_deny(acep, acep + skip, isdir, B_FALSE); + /* + * If we just did the last group, skip acep past + * all of the denies; else, just move ahead one. + */ + if (++groupi >= numgroup) + acep += numgroup + 1; + else + acep += 1; + } else if (aclent[i].a_type & OTHER_OBJ) { + acep->a_who = (uid_t)-1; + acep->a_flags |= ACE_EVERYONE; + ace_make_deny(acep, acep + 1, isdir, B_FALSE); + acep += 2; + } else { + error = EINVAL; + goto out; + } + } + + *acepp = result; + *rescount = resultsize; + +out: + if (error != 0) { + if ((result != NULL) && (resultsize > 0)) { + cacl_free(result, resultsize * sizeof (ace_t)); + } + } + + return (error); +} + +static int +convert_aent_to_ace(aclent_t *aclentp, int aclcnt, boolean_t isdir, + ace_t **retacep, int *retacecnt) +{ + ace_t *acep; + ace_t *dfacep; + int acecnt = 0; + int dfacecnt = 0; + int dfaclstart = 0; + int dfaclcnt = 0; + aclent_t *aclp; + int i; + int error; + int acesz, dfacesz; + + ksort((caddr_t)aclentp, aclcnt, sizeof (aclent_t), cmp2acls); + + for (i = 0, aclp = aclentp; i < aclcnt; aclp++, i++) { + if (aclp->a_type & ACL_DEFAULT) + break; + } + + if (i < aclcnt) { + dfaclstart = i; + dfaclcnt = aclcnt - i; + } + + if (dfaclcnt && !isdir) { + return (EINVAL); + } + + error = ln_aent_to_ace(aclentp, i, &acep, &acecnt, isdir); + if (error) + return (error); + + if (dfaclcnt) { + error = ln_aent_to_ace(&aclentp[dfaclstart], dfaclcnt, + &dfacep, &dfacecnt, isdir); + if (error) { + if (acep) { + cacl_free(acep, acecnt * sizeof (ace_t)); + } + return (error); + } + } + + if (dfacecnt != 0) { + acesz = sizeof (ace_t) * acecnt; + dfacesz = sizeof (ace_t) * dfacecnt; + acep = cacl_realloc(acep, acesz, acesz + dfacesz); + if (acep == NULL) + return (ENOMEM); + if (dfaclcnt) { + (void) memcpy(acep + acecnt, dfacep, dfacesz); + } + } + if (dfaclcnt) + cacl_free(dfacep, dfacecnt * sizeof (ace_t)); + + *retacecnt = acecnt + dfacecnt; + *retacep = acep; + return (0); +} + +static int +ace_mask_to_mode(uint32_t mask, o_mode_t *modep, boolean_t isdir) +{ + int error = 0; + o_mode_t mode = 0; + uint32_t bits, wantbits; + + /* read */ + if (mask & ACE_READ_DATA) + mode |= S_IROTH; + + /* write */ + wantbits = (ACE_WRITE_DATA | ACE_APPEND_DATA); + if (isdir) + wantbits |= ACE_DELETE_CHILD; + bits = mask & wantbits; + if (bits != 0) { + if (bits != wantbits) { + error = ENOTSUP; + goto out; + } + mode |= S_IWOTH; + } + + /* exec */ + if (mask & ACE_EXECUTE) { + mode |= S_IXOTH; + } + + *modep = mode; + +out: + return (error); +} + +static void +acevals_init(acevals_t *vals, uid_t key) +{ + bzero(vals, sizeof (*vals)); + vals->allowed = ACE_MASK_UNDEFINED; + vals->denied = ACE_MASK_UNDEFINED; + vals->mask = ACE_MASK_UNDEFINED; + vals->key = key; +} + +static void +ace_list_init(ace_list_t *al, int dfacl_flag) +{ + acevals_init(&al->user_obj, 0); + acevals_init(&al->group_obj, 0); + acevals_init(&al->other_obj, 0); + al->numusers = 0; + al->numgroups = 0; + al->acl_mask = 0; + al->hasmask = 0; + al->state = ace_unused; + al->seen = 0; + al->dfacl_flag = dfacl_flag; +} + +/* + * Find or create an acevals holder for a given id and avl tree. + * + * Note that only one thread will ever touch these avl trees, so + * there is no need for locking. + */ +static acevals_t * +acevals_find(ace_t *ace, avl_tree_t *avl, int *num) +{ + acevals_t key, *rc; + avl_index_t where; + + key.key = ace->a_who; + rc = avl_find(avl, &key, &where); + if (rc != NULL) + return (rc); + + /* this memory is freed by ln_ace_to_aent()->ace_list_free() */ + if (cacl_malloc((void **)&rc, sizeof (acevals_t)) != 0) + return (NULL); + + acevals_init(rc, ace->a_who); + avl_insert(avl, rc, where); + (*num)++; + + return (rc); +} + +static int +access_mask_check(ace_t *acep, int mask_bit, int isowner) +{ + int set_deny, err_deny; + int set_allow, err_allow; + int acl_consume; + int haswriteperm, hasreadperm; + + if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) { + haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 0 : 1; + hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 0 : 1; + } else { + haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 1 : 0; + hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 1 : 0; + } + + acl_consume = (ACL_SYNCHRONIZE_ERR_DENY | + ACL_DELETE_ERR_DENY | + ACL_WRITE_OWNER_ERR_DENY | + ACL_WRITE_OWNER_ERR_ALLOW | + ACL_WRITE_ATTRS_OWNER_SET_ALLOW | + ACL_WRITE_ATTRS_OWNER_ERR_DENY | + ACL_WRITE_ATTRS_WRITER_SET_DENY | + ACL_WRITE_ATTRS_WRITER_ERR_ALLOW | + ACL_WRITE_NAMED_WRITER_ERR_DENY | + ACL_READ_NAMED_READER_ERR_DENY); + + if (mask_bit == ACE_SYNCHRONIZE) { + set_deny = ACL_SYNCHRONIZE_SET_DENY; + err_deny = ACL_SYNCHRONIZE_ERR_DENY; + set_allow = ACL_SYNCHRONIZE_SET_ALLOW; + err_allow = ACL_SYNCHRONIZE_ERR_ALLOW; + } else if (mask_bit == ACE_WRITE_OWNER) { + set_deny = ACL_WRITE_OWNER_SET_DENY; + err_deny = ACL_WRITE_OWNER_ERR_DENY; + set_allow = ACL_WRITE_OWNER_SET_ALLOW; + err_allow = ACL_WRITE_OWNER_ERR_ALLOW; + } else if (mask_bit == ACE_DELETE) { + set_deny = ACL_DELETE_SET_DENY; + err_deny = ACL_DELETE_ERR_DENY; + set_allow = ACL_DELETE_SET_ALLOW; + err_allow = ACL_DELETE_ERR_ALLOW; + } else if (mask_bit == ACE_WRITE_ATTRIBUTES) { + if (isowner) { + set_deny = ACL_WRITE_ATTRS_OWNER_SET_DENY; + err_deny = ACL_WRITE_ATTRS_OWNER_ERR_DENY; + set_allow = ACL_WRITE_ATTRS_OWNER_SET_ALLOW; + err_allow = ACL_WRITE_ATTRS_OWNER_ERR_ALLOW; + } else if (haswriteperm) { + set_deny = ACL_WRITE_ATTRS_WRITER_SET_DENY; + err_deny = ACL_WRITE_ATTRS_WRITER_ERR_DENY; + set_allow = ACL_WRITE_ATTRS_WRITER_SET_ALLOW; + err_allow = ACL_WRITE_ATTRS_WRITER_ERR_ALLOW; + } else { + if ((acep->a_access_mask & mask_bit) && + (acep->a_type & ACE_ACCESS_ALLOWED_ACE_TYPE)) { + return (ENOTSUP); + } + return (0); + } + } else if (mask_bit == ACE_READ_NAMED_ATTRS) { + if (!hasreadperm) + return (0); + + set_deny = ACL_READ_NAMED_READER_SET_DENY; + err_deny = ACL_READ_NAMED_READER_ERR_DENY; + set_allow = ACL_READ_NAMED_READER_SET_ALLOW; + err_allow = ACL_READ_NAMED_READER_ERR_ALLOW; + } else if (mask_bit == ACE_WRITE_NAMED_ATTRS) { + if (!haswriteperm) + return (0); + + set_deny = ACL_WRITE_NAMED_WRITER_SET_DENY; + err_deny = ACL_WRITE_NAMED_WRITER_ERR_DENY; + set_allow = ACL_WRITE_NAMED_WRITER_SET_ALLOW; + err_allow = ACL_WRITE_NAMED_WRITER_ERR_ALLOW; + } else { + return (EINVAL); + } + + if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) { + if (acl_consume & set_deny) { + if (!(acep->a_access_mask & mask_bit)) { + return (ENOTSUP); + } + } else if (acl_consume & err_deny) { + if (acep->a_access_mask & mask_bit) { + return (ENOTSUP); + } + } + } else { + /* ACE_ACCESS_ALLOWED_ACE_TYPE */ + if (acl_consume & set_allow) { + if (!(acep->a_access_mask & mask_bit)) { + return (ENOTSUP); + } + } else if (acl_consume & err_allow) { + if (acep->a_access_mask & mask_bit) { + return (ENOTSUP); + } + } + } + return (0); +} + +static int +ace_to_aent_legal(ace_t *acep) +{ + int error = 0; + int isowner; + + /* only ALLOW or DENY */ + if ((acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE) && + (acep->a_type != ACE_ACCESS_DENIED_ACE_TYPE)) { + error = ENOTSUP; + goto out; + } + + /* check for invalid flags */ + if (acep->a_flags & ~(ACE_VALID_FLAG_BITS)) { + error = EINVAL; + goto out; + } + + /* some flags are illegal */ + if (acep->a_flags & (ACE_SUCCESSFUL_ACCESS_ACE_FLAG | + ACE_FAILED_ACCESS_ACE_FLAG | + ACE_NO_PROPAGATE_INHERIT_ACE)) { + error = ENOTSUP; + goto out; + } + + /* check for invalid masks */ + if (acep->a_access_mask & ~(ACE_VALID_MASK_BITS)) { + error = EINVAL; + goto out; + } + + if ((acep->a_flags & ACE_OWNER)) { + isowner = 1; + } else { + isowner = 0; + } + + error = access_mask_check(acep, ACE_SYNCHRONIZE, isowner); + if (error) + goto out; + + error = access_mask_check(acep, ACE_WRITE_OWNER, isowner); + if (error) + goto out; + + error = access_mask_check(acep, ACE_DELETE, isowner); + if (error) + goto out; + + error = access_mask_check(acep, ACE_WRITE_ATTRIBUTES, isowner); + if (error) + goto out; + + error = access_mask_check(acep, ACE_READ_NAMED_ATTRS, isowner); + if (error) + goto out; + + error = access_mask_check(acep, ACE_WRITE_NAMED_ATTRS, isowner); + if (error) + goto out; + + /* more detailed checking of masks */ + if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) { + if (! (acep->a_access_mask & ACE_READ_ATTRIBUTES)) { + error = ENOTSUP; + goto out; + } + if ((acep->a_access_mask & ACE_WRITE_DATA) && + (! (acep->a_access_mask & ACE_APPEND_DATA))) { + error = ENOTSUP; + goto out; + } + if ((! (acep->a_access_mask & ACE_WRITE_DATA)) && + (acep->a_access_mask & ACE_APPEND_DATA)) { + error = ENOTSUP; + goto out; + } + } + + /* ACL enforcement */ + if ((acep->a_access_mask & ACE_READ_ACL) && + (acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE)) { + error = ENOTSUP; + goto out; + } + if (acep->a_access_mask & ACE_WRITE_ACL) { + if ((acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) && + (isowner)) { + error = ENOTSUP; + goto out; + } + if ((acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) && + (! isowner)) { + error = ENOTSUP; + goto out; + } + } + +out: + return (error); +} + +static int +ace_allow_to_mode(uint32_t mask, o_mode_t *modep, boolean_t isdir) +{ + /* ACE_READ_ACL and ACE_READ_ATTRIBUTES must both be set */ + if ((mask & (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) != + (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) { + return (ENOTSUP); + } + + return (ace_mask_to_mode(mask, modep, isdir)); +} + +static int +acevals_to_aent(acevals_t *vals, aclent_t *dest, ace_list_t *list, + uid_t owner, gid_t group, boolean_t isdir) +{ + int error; + uint32_t flips = ACE_POSIX_SUPPORTED_BITS; + + if (isdir) + flips |= ACE_DELETE_CHILD; + if (vals->allowed != (vals->denied ^ flips)) { + error = ENOTSUP; + goto out; + } + if ((list->hasmask) && (list->acl_mask != vals->mask) && + (vals->aent_type & (USER | GROUP | GROUP_OBJ))) { + error = ENOTSUP; + goto out; + } + error = ace_allow_to_mode(vals->allowed, &dest->a_perm, isdir); + if (error != 0) + goto out; + dest->a_type = vals->aent_type; + if (dest->a_type & (USER | GROUP)) { + dest->a_id = vals->key; + } else if (dest->a_type & USER_OBJ) { + dest->a_id = owner; + } else if (dest->a_type & GROUP_OBJ) { + dest->a_id = group; + } else if (dest->a_type & OTHER_OBJ) { + dest->a_id = 0; + } else { + error = EINVAL; + goto out; + } + +out: + return (error); +} + + +static int +ace_list_to_aent(ace_list_t *list, aclent_t **aclentp, int *aclcnt, + uid_t owner, gid_t group, boolean_t isdir) +{ + int error = 0; + aclent_t *aent, *result = NULL; + acevals_t *vals; + int resultcount; + + if ((list->seen & (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) != + (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) { + error = ENOTSUP; + goto out; + } + if ((! list->hasmask) && (list->numusers + list->numgroups > 0)) { + error = ENOTSUP; + goto out; + } + + resultcount = 3 + list->numusers + list->numgroups; + /* + * This must be the same condition as below, when we add the CLASS_OBJ + * (aka ACL mask) + */ + if ((list->hasmask) || (! list->dfacl_flag)) + resultcount += 1; + + if (cacl_malloc((void **)&result, + resultcount * sizeof (aclent_t)) != 0) { + error = ENOMEM; + goto out; + } + aent = result; + + /* USER_OBJ */ + if (!(list->user_obj.aent_type & USER_OBJ)) { + error = EINVAL; + goto out; + } + + error = acevals_to_aent(&list->user_obj, aent, list, owner, group, + isdir); + + if (error != 0) + goto out; + ++aent; + /* USER */ + vals = NULL; + for (vals = avl_first(&list->user); vals != NULL; + vals = AVL_NEXT(&list->user, vals)) { + if (!(vals->aent_type & USER)) { + error = EINVAL; + goto out; + } + error = acevals_to_aent(vals, aent, list, owner, group, + isdir); + if (error != 0) + goto out; + ++aent; + } + /* GROUP_OBJ */ + if (!(list->group_obj.aent_type & GROUP_OBJ)) { + error = EINVAL; + goto out; + } + error = acevals_to_aent(&list->group_obj, aent, list, owner, group, + isdir); + if (error != 0) + goto out; + ++aent; + /* GROUP */ + vals = NULL; + for (vals = avl_first(&list->group); vals != NULL; + vals = AVL_NEXT(&list->group, vals)) { + if (!(vals->aent_type & GROUP)) { + error = EINVAL; + goto out; + } + error = acevals_to_aent(vals, aent, list, owner, group, + isdir); + if (error != 0) + goto out; + ++aent; + } + /* + * CLASS_OBJ (aka ACL_MASK) + * + * An ACL_MASK is not fabricated if the ACL is a default ACL. + * This is to follow UFS's behavior. + */ + if ((list->hasmask) || (! list->dfacl_flag)) { + if (list->hasmask) { + uint32_t flips = ACE_POSIX_SUPPORTED_BITS; + if (isdir) + flips |= ACE_DELETE_CHILD; + error = ace_mask_to_mode(list->acl_mask ^ flips, + &aent->a_perm, isdir); + if (error != 0) + goto out; + } else { + /* fabricate the ACL_MASK from the group permissions */ + error = ace_mask_to_mode(list->group_obj.allowed, + &aent->a_perm, isdir); + if (error != 0) + goto out; + } + aent->a_id = 0; + aent->a_type = CLASS_OBJ | list->dfacl_flag; + ++aent; + } + /* OTHER_OBJ */ + if (!(list->other_obj.aent_type & OTHER_OBJ)) { + error = EINVAL; + goto out; + } + error = acevals_to_aent(&list->other_obj, aent, list, owner, group, + isdir); + if (error != 0) + goto out; + ++aent; + + *aclentp = result; + *aclcnt = resultcount; + +out: + if (error != 0) { + if (result != NULL) + cacl_free(result, resultcount * sizeof (aclent_t)); + } + + return (error); +} + + +/* + * free all data associated with an ace_list + */ +static void +ace_list_free(ace_list_t *al) +{ + acevals_t *node; + void *cookie; + + if (al == NULL) + return; + + cookie = NULL; + while ((node = avl_destroy_nodes(&al->user, &cookie)) != NULL) + cacl_free(node, sizeof (acevals_t)); + cookie = NULL; + while ((node = avl_destroy_nodes(&al->group, &cookie)) != NULL) + cacl_free(node, sizeof (acevals_t)); + + avl_destroy(&al->user); + avl_destroy(&al->group); + + /* free the container itself */ + cacl_free(al, sizeof (ace_list_t)); +} + +static int +acevals_compare(const void *va, const void *vb) +{ + const acevals_t *a = va, *b = vb; + + if (a->key == b->key) + return (0); + + if (a->key > b->key) + return (1); + + else + return (-1); +} + +/* + * Convert a list of ace_t entries to equivalent regular and default + * aclent_t lists. Return error (ENOTSUP) when conversion is not possible. + */ +static int +ln_ace_to_aent(ace_t *ace, int n, uid_t owner, gid_t group, + aclent_t **aclentp, int *aclcnt, aclent_t **dfaclentp, int *dfaclcnt, + boolean_t isdir) +{ + int error = 0; + ace_t *acep; + uint32_t bits; + int i; + ace_list_t *normacl = NULL, *dfacl = NULL, *acl; + acevals_t *vals; + + *aclentp = NULL; + *aclcnt = 0; + *dfaclentp = NULL; + *dfaclcnt = 0; + + /* we need at least user_obj, group_obj, and other_obj */ + if (n < 6) { + error = ENOTSUP; + goto out; + } + if (ace == NULL) { + error = EINVAL; + goto out; + } + + error = cacl_malloc((void **)&normacl, sizeof (ace_list_t)); + if (error != 0) + goto out; + + avl_create(&normacl->user, acevals_compare, sizeof (acevals_t), + offsetof(acevals_t, avl)); + avl_create(&normacl->group, acevals_compare, sizeof (acevals_t), + offsetof(acevals_t, avl)); + + ace_list_init(normacl, 0); + + error = cacl_malloc((void **)&dfacl, sizeof (ace_list_t)); + if (error != 0) + goto out; + + avl_create(&dfacl->user, acevals_compare, sizeof (acevals_t), + offsetof(acevals_t, avl)); + avl_create(&dfacl->group, acevals_compare, sizeof (acevals_t), + offsetof(acevals_t, avl)); + ace_list_init(dfacl, ACL_DEFAULT); + + /* process every ace_t... */ + for (i = 0; i < n; i++) { + acep = &ace[i]; + + /* rule out certain cases quickly */ + error = ace_to_aent_legal(acep); + if (error != 0) + goto out; + + /* + * Turn off these bits in order to not have to worry about + * them when doing the checks for compliments. + */ + acep->a_access_mask &= ~(ACE_WRITE_OWNER | ACE_DELETE | + ACE_SYNCHRONIZE | ACE_WRITE_ATTRIBUTES | + ACE_READ_NAMED_ATTRS | ACE_WRITE_NAMED_ATTRS); + + /* see if this should be a regular or default acl */ + bits = acep->a_flags & + (ACE_INHERIT_ONLY_ACE | + ACE_FILE_INHERIT_ACE | + ACE_DIRECTORY_INHERIT_ACE); + if (bits != 0) { + /* all or nothing on these inherit bits */ + if (bits != (ACE_INHERIT_ONLY_ACE | + ACE_FILE_INHERIT_ACE | + ACE_DIRECTORY_INHERIT_ACE)) { + error = ENOTSUP; + goto out; + } + acl = dfacl; + } else { + acl = normacl; + } + + if ((acep->a_flags & ACE_OWNER)) { + if (acl->state > ace_user_obj) { + error = ENOTSUP; + goto out; + } + acl->state = ace_user_obj; + acl->seen |= USER_OBJ; + vals = &acl->user_obj; + vals->aent_type = USER_OBJ | acl->dfacl_flag; + } else if ((acep->a_flags & ACE_EVERYONE)) { + acl->state = ace_other_obj; + acl->seen |= OTHER_OBJ; + vals = &acl->other_obj; + vals->aent_type = OTHER_OBJ | acl->dfacl_flag; + } else if (acep->a_flags & ACE_IDENTIFIER_GROUP) { + if (acl->state > ace_group) { + error = ENOTSUP; + goto out; + } + if ((acep->a_flags & ACE_GROUP)) { + acl->seen |= GROUP_OBJ; + vals = &acl->group_obj; + vals->aent_type = GROUP_OBJ | acl->dfacl_flag; + } else { + acl->seen |= GROUP; + vals = acevals_find(acep, &acl->group, + &acl->numgroups); + if (vals == NULL) { + error = ENOMEM; + goto out; + } + vals->aent_type = GROUP | acl->dfacl_flag; + } + acl->state = ace_group; + } else { + if (acl->state > ace_user) { + error = ENOTSUP; + goto out; + } + acl->state = ace_user; + acl->seen |= USER; + vals = acevals_find(acep, &acl->user, + &acl->numusers); + if (vals == NULL) { + error = ENOMEM; + goto out; + } + vals->aent_type = USER | acl->dfacl_flag; + } + + if (!(acl->state > ace_unused)) { + error = EINVAL; + goto out; + } + + if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) { + /* no more than one allowed per aclent_t */ + if (vals->allowed != ACE_MASK_UNDEFINED) { + error = ENOTSUP; + goto out; + } + vals->allowed = acep->a_access_mask; + } else { + /* + * it's a DENY; if there was a previous DENY, it + * must have been an ACL_MASK. + */ + if (vals->denied != ACE_MASK_UNDEFINED) { + /* ACL_MASK is for USER and GROUP only */ + if ((acl->state != ace_user) && + (acl->state != ace_group)) { + error = ENOTSUP; + goto out; + } + + if (! acl->hasmask) { + acl->hasmask = 1; + acl->acl_mask = vals->denied; + /* check for mismatched ACL_MASK emulations */ + } else if (acl->acl_mask != vals->denied) { + error = ENOTSUP; + goto out; + } + vals->mask = vals->denied; + } + vals->denied = acep->a_access_mask; + } + } + + /* done collating; produce the aclent_t lists */ + if (normacl->state != ace_unused) { + error = ace_list_to_aent(normacl, aclentp, aclcnt, + owner, group, isdir); + if (error != 0) { + goto out; + } + } + if (dfacl->state != ace_unused) { + error = ace_list_to_aent(dfacl, dfaclentp, dfaclcnt, + owner, group, isdir); + if (error != 0) { + goto out; + } + } + +out: + if (normacl != NULL) + ace_list_free(normacl); + if (dfacl != NULL) + ace_list_free(dfacl); + + return (error); +} + +static int +convert_ace_to_aent(ace_t *acebufp, int acecnt, boolean_t isdir, + uid_t owner, gid_t group, aclent_t **retaclentp, int *retaclcnt) +{ + int error = 0; + aclent_t *aclentp, *dfaclentp; + int aclcnt, dfaclcnt; + int aclsz, dfaclsz; + + error = ln_ace_to_aent(acebufp, acecnt, owner, group, + &aclentp, &aclcnt, &dfaclentp, &dfaclcnt, isdir); + + if (error) + return (error); + + + if (dfaclcnt != 0) { + /* + * Slap aclentp and dfaclentp into a single array. + */ + aclsz = sizeof (aclent_t) * aclcnt; + dfaclsz = sizeof (aclent_t) * dfaclcnt; + aclentp = cacl_realloc(aclentp, aclsz, aclsz + dfaclsz); + if (aclentp != NULL) { + (void) memcpy(aclentp + aclcnt, dfaclentp, dfaclsz); + } else { + error = ENOMEM; + } + } + + if (aclentp) { + *retaclentp = aclentp; + *retaclcnt = aclcnt + dfaclcnt; + } + + if (dfaclentp) + cacl_free(dfaclentp, dfaclsz); + + return (error); +} + + +int +acl_translate(acl_t *aclp, int target_flavor, boolean_t isdir, uid_t owner, + gid_t group) +{ + int aclcnt; + void *acldata; + int error; + + /* + * See if we need to translate + */ + if ((target_flavor == _ACL_ACE_ENABLED && aclp->acl_type == ACE_T) || + (target_flavor == _ACL_ACLENT_ENABLED && + aclp->acl_type == ACLENT_T)) + return (0); + + if (target_flavor == -1) { + error = EINVAL; + goto out; + } + + if (target_flavor == _ACL_ACE_ENABLED && + aclp->acl_type == ACLENT_T) { + error = convert_aent_to_ace(aclp->acl_aclp, + aclp->acl_cnt, isdir, (ace_t **)&acldata, &aclcnt); + if (error) + goto out; + + } else if (target_flavor == _ACL_ACLENT_ENABLED && + aclp->acl_type == ACE_T) { + error = convert_ace_to_aent(aclp->acl_aclp, aclp->acl_cnt, + isdir, owner, group, (aclent_t **)&acldata, &aclcnt); + if (error) + goto out; + } else { + error = ENOTSUP; + goto out; + } + + /* + * replace old acl with newly translated acl + */ + cacl_free(aclp->acl_aclp, aclp->acl_cnt * aclp->acl_entry_size); + aclp->acl_aclp = acldata; + aclp->acl_cnt = aclcnt; + if (target_flavor == _ACL_ACE_ENABLED) { + aclp->acl_type = ACE_T; + aclp->acl_entry_size = sizeof (ace_t); + } else { + aclp->acl_type = ACLENT_T; + aclp->acl_entry_size = sizeof (aclent_t); + } + return (0); + +out: + +#if !defined(_KERNEL) + errno = error; + return (-1); +#else + return (error); +#endif +} +#endif /* !_KERNEL */ + +#define SET_ACE(acl, index, who, mask, type, flags) { \ + acl[0][index].a_who = (uint32_t)who; \ + acl[0][index].a_type = type; \ + acl[0][index].a_flags = flags; \ + acl[0][index++].a_access_mask = mask; \ +} + +void +acl_trivial_access_masks(mode_t mode, boolean_t isdir, trivial_acl_t *masks) +{ + uint32_t read_mask = ACE_READ_DATA; + uint32_t write_mask = ACE_WRITE_DATA|ACE_APPEND_DATA; + uint32_t execute_mask = ACE_EXECUTE; + + (void) isdir; /* will need this later */ + + masks->deny1 = 0; + if (!(mode & S_IRUSR) && (mode & (S_IRGRP|S_IROTH))) + masks->deny1 |= read_mask; + if (!(mode & S_IWUSR) && (mode & (S_IWGRP|S_IWOTH))) + masks->deny1 |= write_mask; + if (!(mode & S_IXUSR) && (mode & (S_IXGRP|S_IXOTH))) + masks->deny1 |= execute_mask; + + masks->deny2 = 0; + if (!(mode & S_IRGRP) && (mode & S_IROTH)) + masks->deny2 |= read_mask; + if (!(mode & S_IWGRP) && (mode & S_IWOTH)) + masks->deny2 |= write_mask; + if (!(mode & S_IXGRP) && (mode & S_IXOTH)) + masks->deny2 |= execute_mask; + + masks->allow0 = 0; + if ((mode & S_IRUSR) && (!(mode & S_IRGRP) && (mode & S_IROTH))) + masks->allow0 |= read_mask; + if ((mode & S_IWUSR) && (!(mode & S_IWGRP) && (mode & S_IWOTH))) + masks->allow0 |= write_mask; + if ((mode & S_IXUSR) && (!(mode & S_IXGRP) && (mode & S_IXOTH))) + masks->allow0 |= execute_mask; + + masks->owner = ACE_WRITE_ATTRIBUTES|ACE_WRITE_OWNER|ACE_WRITE_ACL| + ACE_WRITE_NAMED_ATTRS|ACE_READ_ACL|ACE_READ_ATTRIBUTES| + ACE_READ_NAMED_ATTRS|ACE_SYNCHRONIZE; + if (mode & S_IRUSR) + masks->owner |= read_mask; + if (mode & S_IWUSR) + masks->owner |= write_mask; + if (mode & S_IXUSR) + masks->owner |= execute_mask; + + masks->group = ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_NAMED_ATTRS| + ACE_SYNCHRONIZE; + if (mode & S_IRGRP) + masks->group |= read_mask; + if (mode & S_IWGRP) + masks->group |= write_mask; + if (mode & S_IXGRP) + masks->group |= execute_mask; + + masks->everyone = ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_NAMED_ATTRS| + ACE_SYNCHRONIZE; + if (mode & S_IROTH) + masks->everyone |= read_mask; + if (mode & S_IWOTH) + masks->everyone |= write_mask; + if (mode & S_IXOTH) + masks->everyone |= execute_mask; +} + +int +acl_trivial_create(mode_t mode, boolean_t isdir, ace_t **acl, int *count) +{ + int index = 0; + int error; + trivial_acl_t masks; + + *count = 3; + acl_trivial_access_masks(mode, isdir, &masks); + + if (masks.allow0) + (*count)++; + if (masks.deny1) + (*count)++; + if (masks.deny2) + (*count)++; + + if ((error = cacl_malloc((void **)acl, *count * sizeof (ace_t))) != 0) + return (error); + + if (masks.allow0) { + SET_ACE(acl, index, -1, masks.allow0, + ACE_ACCESS_ALLOWED_ACE_TYPE, ACE_OWNER); + } + if (masks.deny1) { + SET_ACE(acl, index, -1, masks.deny1, + ACE_ACCESS_DENIED_ACE_TYPE, ACE_OWNER); + } + if (masks.deny2) { + SET_ACE(acl, index, -1, masks.deny2, + ACE_ACCESS_DENIED_ACE_TYPE, ACE_GROUP|ACE_IDENTIFIER_GROUP); + } + + SET_ACE(acl, index, -1, masks.owner, ACE_ACCESS_ALLOWED_ACE_TYPE, + ACE_OWNER); + SET_ACE(acl, index, -1, masks.group, ACE_ACCESS_ALLOWED_ACE_TYPE, + ACE_IDENTIFIER_GROUP|ACE_GROUP); + SET_ACE(acl, index, -1, masks.everyone, ACE_ACCESS_ALLOWED_ACE_TYPE, + ACE_EVERYONE); + + return (0); +} + +/* + * ace_trivial: + * determine whether an ace_t acl is trivial + * + * Trivialness implies that the acl is composed of only + * owner, group, everyone entries. ACL can't + * have read_acl denied, and write_owner/write_acl/write_attributes + * can only be owner@ entry. + */ +int +ace_trivial_common(void *acep, int aclcnt, + uint64_t (*walk)(void *, uint64_t, int aclcnt, + uint16_t *, uint16_t *, uint32_t *)) +{ + uint16_t flags; + uint32_t mask; + uint16_t type; + uint64_t cookie = 0; + + while ((cookie = walk(acep, cookie, aclcnt, &flags, &type, &mask))) { + switch (flags & ACE_TYPE_FLAGS) { + case ACE_OWNER: + case ACE_GROUP|ACE_IDENTIFIER_GROUP: + case ACE_EVERYONE: + break; + default: + return (1); + + } + + if (flags & (ACE_FILE_INHERIT_ACE| + ACE_DIRECTORY_INHERIT_ACE|ACE_NO_PROPAGATE_INHERIT_ACE| + ACE_INHERIT_ONLY_ACE)) + return (1); + + /* + * Special check for some special bits + * + * Don't allow anybody to deny reading basic + * attributes or a files ACL. + */ + if ((mask & (ACE_READ_ACL|ACE_READ_ATTRIBUTES)) && + (type == ACE_ACCESS_DENIED_ACE_TYPE)) + return (1); + + /* + * Delete permissions are never set by default + */ + if (mask & (ACE_DELETE|ACE_DELETE_CHILD)) + return (1); + /* + * only allow owner@ to have + * write_acl/write_owner/write_attributes/write_xattr/ + */ + if (type == ACE_ACCESS_ALLOWED_ACE_TYPE && + (!(flags & ACE_OWNER) && (mask & + (ACE_WRITE_OWNER|ACE_WRITE_ACL| ACE_WRITE_ATTRIBUTES| + ACE_WRITE_NAMED_ATTRS)))) + return (1); + + } + return (0); +} + +uint64_t +ace_walk(void *datap, uint64_t cookie, int aclcnt, uint16_t *flags, + uint16_t *type, uint32_t *mask) +{ + ace_t *acep = datap; + + if (cookie >= aclcnt) + return (0); + + *flags = acep[cookie].a_flags; + *type = acep[cookie].a_type; + *mask = acep[cookie++].a_access_mask; + + return (cookie); +} + +int +ace_trivial(ace_t *acep, int aclcnt) +{ + return (ace_trivial_common(acep, aclcnt, ace_walk)); +} diff --git a/module/os/freebsd/spl/callb.c b/module/os/freebsd/spl/callb.c new file mode 100644 index 00000000000..d4a0e141cfd --- /dev/null +++ b/module/os/freebsd/spl/callb.c @@ -0,0 +1,372 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for delay() */ +#include /* For TASKQ_NAMELEN */ +#include + +#define CB_MAXNAME TASKQ_NAMELEN + +/* + * The callb mechanism provides generic event scheduling/echoing. + * A callb function is registered and called on behalf of the event. + */ +typedef struct callb { + struct callb *c_next; /* next in class or on freelist */ + kthread_id_t c_thread; /* ptr to caller's thread struct */ + char c_flag; /* info about the callb state */ + uchar_t c_class; /* this callb's class */ + kcondvar_t c_done_cv; /* signal callb completion */ + boolean_t (*c_func)(); /* cb function: returns true if ok */ + void *c_arg; /* arg to c_func */ + char c_name[CB_MAXNAME+1]; /* debug:max func name length */ +} callb_t; + +/* + * callb c_flag bitmap definitions + */ +#define CALLB_FREE 0x0 +#define CALLB_TAKEN 0x1 +#define CALLB_EXECUTING 0x2 + +/* + * Basic structure for a callb table. + * All callbs are organized into different class groups described + * by ct_class array. + * The callbs within a class are single-linked and normally run by a + * serial execution. + */ +typedef struct callb_table { + kmutex_t ct_lock; /* protect all callb states */ + callb_t *ct_freelist; /* free callb structures */ + int ct_busy; /* != 0 prevents additions */ + kcondvar_t ct_busy_cv; /* to wait for not busy */ + int ct_ncallb; /* num of callbs allocated */ + callb_t *ct_first_cb[NCBCLASS]; /* ptr to 1st callb in a class */ +} callb_table_t; + +int callb_timeout_sec = CPR_KTHREAD_TIMEOUT_SEC; + +static callb_id_t callb_add_common(boolean_t (*)(void *, int), + void *, int, char *, kthread_id_t); + +static callb_table_t callb_table; /* system level callback table */ +static callb_table_t *ct = &callb_table; +static kmutex_t callb_safe_mutex; +callb_cpr_t callb_cprinfo_safe = { + &callb_safe_mutex, CALLB_CPR_ALWAYS_SAFE, 0, {0, 0} }; + +/* + * Init all callb tables in the system. + */ +void +callb_init(void *dummy __unused) +{ + callb_table.ct_busy = 0; /* mark table open for additions */ + mutex_init(&callb_safe_mutex, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&callb_table.ct_lock, NULL, MUTEX_DEFAULT, NULL); +} + +void +callb_fini(void *dummy __unused) +{ + callb_t *cp; + int i; + + mutex_enter(&ct->ct_lock); + for (i = 0; i < 16; i++) { + while ((cp = ct->ct_freelist) != NULL) { + ct->ct_freelist = cp->c_next; + ct->ct_ncallb--; + kmem_free(cp, sizeof (callb_t)); + } + if (ct->ct_ncallb == 0) + break; + /* Not all callbacks finished, waiting for the rest. */ + mutex_exit(&ct->ct_lock); + tsleep(ct, 0, "callb", hz / 4); + mutex_enter(&ct->ct_lock); + } + if (ct->ct_ncallb > 0) + printf("%s: Leaked %d callbacks!\n", __func__, ct->ct_ncallb); + mutex_exit(&ct->ct_lock); + mutex_destroy(&callb_safe_mutex); + mutex_destroy(&callb_table.ct_lock); +} + +/* + * callout_add() is called to register func() be called later. + */ +static callb_id_t +callb_add_common(boolean_t (*func)(void *arg, int code), + void *arg, int class, char *name, kthread_id_t t) +{ + callb_t *cp; + + ASSERT(class < NCBCLASS); + + mutex_enter(&ct->ct_lock); + while (ct->ct_busy) + cv_wait(&ct->ct_busy_cv, &ct->ct_lock); + if ((cp = ct->ct_freelist) == NULL) { + ct->ct_ncallb++; + cp = (callb_t *)kmem_zalloc(sizeof (callb_t), KM_SLEEP); + } + ct->ct_freelist = cp->c_next; + cp->c_thread = t; + cp->c_func = func; + cp->c_arg = arg; + cp->c_class = (uchar_t)class; + cp->c_flag |= CALLB_TAKEN; +#ifdef DEBUG + if (strlen(name) > CB_MAXNAME) + cmn_err(CE_WARN, "callb_add: name of callback function '%s' " + "too long -- truncated to %d chars", + name, CB_MAXNAME); +#endif + (void) strncpy(cp->c_name, name, CB_MAXNAME); + cp->c_name[CB_MAXNAME] = '\0'; + + /* + * Insert the new callb at the head of its class list. + */ + cp->c_next = ct->ct_first_cb[class]; + ct->ct_first_cb[class] = cp; + + mutex_exit(&ct->ct_lock); + return ((callb_id_t)cp); +} + +/* + * The default function to add an entry to the callback table. Since + * it uses curthread as the thread identifier to store in the table, + * it should be used for the normal case of a thread which is calling + * to add ITSELF to the table. + */ +callb_id_t +callb_add(boolean_t (*func)(void *arg, int code), + void *arg, int class, char *name) +{ + return (callb_add_common(func, arg, class, name, curthread)); +} + +/* + * A special version of callb_add() above for use by threads which + * might be adding an entry to the table on behalf of some other + * thread (for example, one which is constructed but not yet running). + * In this version the thread id is an argument. + */ +callb_id_t +callb_add_thread(boolean_t (*func)(void *arg, int code), + void *arg, int class, char *name, kthread_id_t t) +{ + return (callb_add_common(func, arg, class, name, t)); +} + +/* + * callout_delete() is called to remove an entry identified by id + * that was originally placed there by a call to callout_add(). + * return -1 if fail to delete a callb entry otherwise return 0. + */ +int +callb_delete(callb_id_t id) +{ + callb_t **pp; + callb_t *me = (callb_t *)id; + + mutex_enter(&ct->ct_lock); + + for (;;) { + pp = &ct->ct_first_cb[me->c_class]; + while (*pp != NULL && *pp != me) + pp = &(*pp)->c_next; + +#ifdef DEBUG + if (*pp != me) { + cmn_err(CE_WARN, "callb delete bogus entry 0x%p", + (void *)me); + mutex_exit(&ct->ct_lock); + return (-1); + } +#endif /* DEBUG */ + + /* + * It is not allowed to delete a callb in the middle of + * executing otherwise, the callb_execute() will be confused. + */ + if (!(me->c_flag & CALLB_EXECUTING)) + break; + + cv_wait(&me->c_done_cv, &ct->ct_lock); + } + /* relink the class list */ + *pp = me->c_next; + + /* clean up myself and return the free callb to the head of freelist */ + me->c_flag = CALLB_FREE; + me->c_next = ct->ct_freelist; + ct->ct_freelist = me; + + mutex_exit(&ct->ct_lock); + return (0); +} + +/* + * class: indicates to execute all callbs in the same class; + * code: optional argument for the callb functions. + * return: = 0: success + * != 0: ptr to string supplied when callback was registered + */ +void * +callb_execute_class(int class, int code) +{ + callb_t *cp; + void *ret = NULL; + + ASSERT(class < NCBCLASS); + + mutex_enter(&ct->ct_lock); + + for (cp = ct->ct_first_cb[class]; + cp != NULL && ret == 0; cp = cp->c_next) { + while (cp->c_flag & CALLB_EXECUTING) + cv_wait(&cp->c_done_cv, &ct->ct_lock); + /* + * cont if the callb is deleted while we're sleeping + */ + if (cp->c_flag == CALLB_FREE) + continue; + cp->c_flag |= CALLB_EXECUTING; + +#ifdef CALLB_DEBUG + printf("callb_execute: name=%s func=%p arg=%p\n", + cp->c_name, (void *)cp->c_func, (void *)cp->c_arg); +#endif /* CALLB_DEBUG */ + + mutex_exit(&ct->ct_lock); + /* If callback function fails, pass back client's name */ + if (!(*cp->c_func)(cp->c_arg, code)) + ret = cp->c_name; + mutex_enter(&ct->ct_lock); + + cp->c_flag &= ~CALLB_EXECUTING; + cv_broadcast(&cp->c_done_cv); + } + mutex_exit(&ct->ct_lock); + return (ret); +} + +/* + * callers make sure no recursive entries to this func. + * dp->cc_lockp is registered by callb_add to protect callb_cpr_t structure. + * + * When calling to stop a kernel thread (code == CB_CODE_CPR_CHKPT) we + * use a cv_timedwait() in case the kernel thread is blocked. + * + * Note that this is a generic callback handler for daemon CPR and + * should NOT be changed to accommodate any specific requirement in a daemon. + * Individual daemons that require changes to the handler shall write + * callback routines in their own daemon modules. + */ +boolean_t +callb_generic_cpr(void *arg, int code) +{ + callb_cpr_t *cp = (callb_cpr_t *)arg; + clock_t ret = 0; /* assume success */ + + mutex_enter(cp->cc_lockp); + + switch (code) { + case CB_CODE_CPR_CHKPT: + cp->cc_events |= CALLB_CPR_START; +#ifdef CPR_NOT_THREAD_SAFE + while (!(cp->cc_events & CALLB_CPR_SAFE)) + /* cv_timedwait() returns -1 if it times out. */ + if ((ret = cv_reltimedwait(&cp->cc_callb_cv, + cp->cc_lockp, (callb_timeout_sec * hz), + TR_CLOCK_TICK)) == -1) + break; +#endif + break; + + case CB_CODE_CPR_RESUME: + cp->cc_events &= ~CALLB_CPR_START; + cv_signal(&cp->cc_stop_cv); + break; + } + mutex_exit(cp->cc_lockp); + return (ret != -1); +} + +/* + * The generic callback function associated with kernel threads which + * are always considered safe. + */ +/* ARGSUSED */ +boolean_t +callb_generic_cpr_safe(void *arg, int code) +{ + return (B_TRUE); +} +/* + * Prevent additions to callback table. + */ +void +callb_lock_table(void) +{ + mutex_enter(&ct->ct_lock); + ASSERT(ct->ct_busy == 0); + ct->ct_busy = 1; + mutex_exit(&ct->ct_lock); +} + +/* + * Allow additions to callback table. + */ +void +callb_unlock_table(void) +{ + mutex_enter(&ct->ct_lock); + ASSERT(ct->ct_busy != 0); + ct->ct_busy = 0; + cv_broadcast(&ct->ct_busy_cv); + mutex_exit(&ct->ct_lock); +} + +SYSINIT(sol_callb, SI_SUB_DRIVERS, SI_ORDER_FIRST, callb_init, NULL); +SYSUNINIT(sol_callb, SI_SUB_DRIVERS, SI_ORDER_FIRST, callb_fini, NULL); diff --git a/module/os/freebsd/spl/list.c b/module/os/freebsd/spl/list.c new file mode 100644 index 00000000000..e8db13a5cf6 --- /dev/null +++ b/module/os/freebsd/spl/list.c @@ -0,0 +1,245 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +/* + * Generic doubly-linked list implementation + */ + +#include +#include +#include +#include +#include + +#define list_d2l(a, obj) ((list_node_t *)(((char *)obj) + (a)->list_offset)) +#define list_object(a, node) ((void *)(((char *)node) - (a)->list_offset)) +#define list_empty(a) ((a)->list_head.list_next == &(a)->list_head) + +#define list_insert_after_node(list, node, object) { \ + list_node_t *lnew = list_d2l(list, object); \ + lnew->list_prev = (node); \ + lnew->list_next = (node)->list_next; \ + (node)->list_next->list_prev = lnew; \ + (node)->list_next = lnew; \ +} + +#define list_insert_before_node(list, node, object) { \ + list_node_t *lnew = list_d2l(list, object); \ + lnew->list_next = (node); \ + lnew->list_prev = (node)->list_prev; \ + (node)->list_prev->list_next = lnew; \ + (node)->list_prev = lnew; \ +} + +#define list_remove_node(node) \ + (node)->list_prev->list_next = (node)->list_next; \ + (node)->list_next->list_prev = (node)->list_prev; \ + (node)->list_next = (node)->list_prev = NULL + +void +list_create(list_t *list, size_t size, size_t offset) +{ + ASSERT(list); + ASSERT(size > 0); + ASSERT(size >= offset + sizeof (list_node_t)); + + list->list_size = size; + list->list_offset = offset; + list->list_head.list_next = list->list_head.list_prev = + &list->list_head; +} + +void +list_destroy(list_t *list) +{ + list_node_t *node = &list->list_head; + + ASSERT(list); + ASSERT(list->list_head.list_next == node); + ASSERT(list->list_head.list_prev == node); + + node->list_next = node->list_prev = NULL; +} + +void +list_insert_after(list_t *list, void *object, void *nobject) +{ + if (object == NULL) { + list_insert_head(list, nobject); + } else { + list_node_t *lold = list_d2l(list, object); + list_insert_after_node(list, lold, nobject); + } +} + +void +list_insert_before(list_t *list, void *object, void *nobject) +{ + if (object == NULL) { + list_insert_tail(list, nobject); + } else { + list_node_t *lold = list_d2l(list, object); + list_insert_before_node(list, lold, nobject); + } +} + +void +list_insert_head(list_t *list, void *object) +{ + list_node_t *lold = &list->list_head; + list_insert_after_node(list, lold, object); +} + +void +list_insert_tail(list_t *list, void *object) +{ + list_node_t *lold = &list->list_head; + list_insert_before_node(list, lold, object); +} + +void +list_remove(list_t *list, void *object) +{ + list_node_t *lold = list_d2l(list, object); + ASSERT(!list_empty(list)); + ASSERT(lold->list_next != NULL); + list_remove_node(lold); +} + +void * +list_remove_head(list_t *list) +{ + list_node_t *head = list->list_head.list_next; + if (head == &list->list_head) + return (NULL); + list_remove_node(head); + return (list_object(list, head)); +} + +void * +list_remove_tail(list_t *list) +{ + list_node_t *tail = list->list_head.list_prev; + if (tail == &list->list_head) + return (NULL); + list_remove_node(tail); + return (list_object(list, tail)); +} + +void * +list_head(list_t *list) +{ + if (list_empty(list)) + return (NULL); + return (list_object(list, list->list_head.list_next)); +} + +void * +list_tail(list_t *list) +{ + if (list_empty(list)) + return (NULL); + return (list_object(list, list->list_head.list_prev)); +} + +void * +list_next(list_t *list, void *object) +{ + list_node_t *node = list_d2l(list, object); + + if (node->list_next != &list->list_head) + return (list_object(list, node->list_next)); + + return (NULL); +} + +void * +list_prev(list_t *list, void *object) +{ + list_node_t *node = list_d2l(list, object); + + if (node->list_prev != &list->list_head) + return (list_object(list, node->list_prev)); + + return (NULL); +} + +/* + * Insert src list after dst list. Empty src list thereafter. + */ +void +list_move_tail(list_t *dst, list_t *src) +{ + list_node_t *dstnode = &dst->list_head; + list_node_t *srcnode = &src->list_head; + + ASSERT(dst->list_size == src->list_size); + ASSERT(dst->list_offset == src->list_offset); + + if (list_empty(src)) + return; + + dstnode->list_prev->list_next = srcnode->list_next; + srcnode->list_next->list_prev = dstnode->list_prev; + dstnode->list_prev = srcnode->list_prev; + srcnode->list_prev->list_next = dstnode; + + /* empty src list */ + srcnode->list_next = srcnode->list_prev = srcnode; +} + +void +list_link_replace(list_node_t *lold, list_node_t *lnew) +{ + ASSERT(list_link_active(lold)); + ASSERT(!list_link_active(lnew)); + + lnew->list_next = lold->list_next; + lnew->list_prev = lold->list_prev; + lold->list_prev->list_next = lnew; + lold->list_next->list_prev = lnew; + lold->list_next = lold->list_prev = NULL; +} + +void +list_link_init(list_node_t *link) +{ + link->list_next = NULL; + link->list_prev = NULL; +} + +int +list_link_active(list_node_t *link) +{ + return (link->list_next != NULL); +} + +int +list_is_empty(list_t *list) +{ + return (list_empty(list)); +} diff --git a/module/os/freebsd/spl/sha224.h b/module/os/freebsd/spl/sha224.h new file mode 100644 index 00000000000..0abd4306870 --- /dev/null +++ b/module/os/freebsd/spl/sha224.h @@ -0,0 +1,96 @@ +/* + * Copyright 2005 Colin Percival + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SHA224_H_ +#define _SHA224_H_ + +#ifndef _KERNEL +#include +#endif + +#define SHA224_BLOCK_LENGTH 64 +#define SHA224_DIGEST_LENGTH 28 +#define SHA224_DIGEST_STRING_LENGTH (SHA224_DIGEST_LENGTH * 2 + 1) + +typedef struct SHA224Context { + uint32_t state[8]; + uint64_t count; + uint8_t buf[SHA224_BLOCK_LENGTH]; +} SHA224_CTX; + +__BEGIN_DECLS + +/* Ensure libmd symbols do not clash with libcrypto */ + +#ifndef SHA224_Init +#define SHA224_Init _libmd_SHA224_Init +#endif +#ifndef SHA224_Update +#define SHA224_Update _libmd_SHA224_Update +#endif +#ifndef SHA224_Final +#define SHA224_Final _libmd_SHA224_Final +#endif +#ifndef SHA224_End +#define SHA224_End _libmd_SHA224_End +#endif +#ifndef SHA224_Fd +#define SHA224_Fd _libmd_SHA224_Fd +#endif +#ifndef SHA224_FdChunk +#define SHA224_FdChunk _libmd_SHA224_FdChunk +#endif +#ifndef SHA224_File +#define SHA224_File _libmd_SHA224_File +#endif +#ifndef SHA224_FileChunk +#define SHA224_FileChunk _libmd_SHA224_FileChunk +#endif +#ifndef SHA224_Data +#define SHA224_Data _libmd_SHA224_Data +#endif + +#ifndef SHA224_version +#define SHA224_version _libmd_SHA224_version +#endif + +void SHA224_Init(SHA224_CTX *); +void SHA224_Update(SHA224_CTX *, const void *, size_t); +void SHA224_Final(unsigned char [__min_size(SHA224_DIGEST_LENGTH)], + SHA224_CTX *); +#ifndef _KERNEL +char *SHA224_End(SHA224_CTX *, char *); +char *SHA224_Data(const void *, unsigned int, char *); +char *SHA224_Fd(int, char *); +char *SHA224_FdChunk(int, char *, off_t, off_t); +char *SHA224_File(const char *, char *); +char *SHA224_FileChunk(const char *, char *, off_t, off_t); +#endif +__END_DECLS + +#endif /* !_SHA224_H_ */ diff --git a/module/os/freebsd/spl/sha256.h b/module/os/freebsd/spl/sha256.h new file mode 100644 index 00000000000..193c0c02512 --- /dev/null +++ b/module/os/freebsd/spl/sha256.h @@ -0,0 +1,99 @@ +/* + * Copyright 2005 Colin Percival + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SHA256_H_ +#define _SHA256_H_ + +#ifndef _KERNEL +#include +#endif + +#define SHA256_BLOCK_LENGTH 64 +#define SHA256_DIGEST_LENGTH 32 +#define SHA256_DIGEST_STRING_LENGTH (SHA256_DIGEST_LENGTH * 2 + 1) + +typedef struct SHA256Context { + uint32_t state[8]; + uint64_t count; + uint8_t buf[SHA256_BLOCK_LENGTH]; +} SHA256_CTX; + +__BEGIN_DECLS + +/* Ensure libmd symbols do not clash with libcrypto */ + +#ifndef SHA256_Init +#define SHA256_Init _libmd_SHA256_Init +#endif +#ifndef SHA256_Update +#define SHA256_Update _libmd_SHA256_Update +#endif +#ifndef SHA256_Final +#define SHA256_Final _libmd_SHA256_Final +#endif +#ifndef SHA256_End +#define SHA256_End _libmd_SHA256_End +#endif +#ifndef SHA256_Fd +#define SHA256_Fd _libmd_SHA256_Fd +#endif +#ifndef SHA256_FdChunk +#define SHA256_FdChunk _libmd_SHA256_FdChunk +#endif +#ifndef SHA256_File +#define SHA256_File _libmd_SHA256_File +#endif +#ifndef SHA256_FileChunk +#define SHA256_FileChunk _libmd_SHA256_FileChunk +#endif +#ifndef SHA256_Data +#define SHA256_Data _libmd_SHA256_Data +#endif + +#ifndef SHA256_Transform +#define SHA256_Transform _libmd_SHA256_Transform +#endif +#ifndef SHA256_version +#define SHA256_version _libmd_SHA256_version +#endif + +void SHA256_Init(SHA256_CTX *); +void SHA256_Update(SHA256_CTX *, const void *, size_t); +void SHA256_Final(unsigned char [__min_size(SHA256_DIGEST_LENGTH)], + SHA256_CTX *); +#ifndef _KERNEL +char *SHA256_End(SHA256_CTX *, char *); +char *SHA256_Data(const void *, unsigned int, char *); +char *SHA256_Fd(int, char *); +char *SHA256_FdChunk(int, char *, off_t, off_t); +char *SHA256_File(const char *, char *); +char *SHA256_FileChunk(const char *, char *, off_t, off_t); +#endif +__END_DECLS + +#endif /* !_SHA256_H_ */ diff --git a/module/os/freebsd/spl/sha256c.c b/module/os/freebsd/spl/sha256c.c new file mode 100644 index 00000000000..241cf8c9ae7 --- /dev/null +++ b/module/os/freebsd/spl/sha256c.c @@ -0,0 +1,378 @@ +/* + * Copyright 2005 Colin Percival + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include + +#ifdef _KERNEL +#include +#else +#include +#endif + + +#include +#include +#include "sha224.h" +#include "sha256.h" + +#if BYTE_ORDER == BIG_ENDIAN + +/* Copy a vector of big-endian uint32_t into a vector of bytes */ +#define be32enc_vect(dst, src, len) \ + memcpy((void *)dst, (const void *)src, (size_t)len) + +/* Copy a vector of bytes into a vector of big-endian uint32_t */ +#define be32dec_vect(dst, src, len) \ + memcpy((void *)dst, (const void *)src, (size_t)len) + +#else /* BYTE_ORDER != BIG_ENDIAN */ + +/* + * Encode a length len/4 vector of (uint32_t) into a length len vector of + * (unsigned char) in big-endian form. Assumes len is a multiple of 4. + */ +static void +be32enc_vect(unsigned char *dst, const uint32_t *src, size_t len) +{ + size_t i; + + for (i = 0; i < len / 4; i++) + be32enc(dst + i * 4, src[i]); +} + +/* + * Decode a big-endian length len vector of (unsigned char) into a length + * len/4 vector of (uint32_t). Assumes len is a multiple of 4. + */ +static void +be32dec_vect(uint32_t *dst, const unsigned char *src, size_t len) +{ + size_t i; + + for (i = 0; i < len / 4; i++) + dst[i] = be32dec(src + i * 4); +} + +#endif /* BYTE_ORDER != BIG_ENDIAN */ + +/* SHA256 round constants. */ +static const uint32_t K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +}; + +/* Elementary functions used by SHA256 */ +#define Ch(x, y, z) ((x & (y ^ z)) ^ z) +#define Maj(x, y, z) ((x & (y | z)) | (y & z)) +#define SHR(x, n) (x >> n) +#define ROTR(x, n) ((x >> n) | (x << (32 - n))) +#define S0(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22)) +#define S1(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25)) +#define s0(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHR(x, 3)) +#define s1(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHR(x, 10)) + +/* SHA256 round function */ +#define RND(a, b, c, d, e, f, g, h, k) \ + h += S1(e) + Ch(e, f, g) + k; \ + d += h; \ + h += S0(a) + Maj(a, b, c); + +/* Adjusted round function for rotating state */ +#define RNDr(S, W, i, ii) \ + RND(S[(64 - i) % 8], S[(65 - i) % 8], \ + S[(66 - i) % 8], S[(67 - i) % 8], \ + S[(68 - i) % 8], S[(69 - i) % 8], \ + S[(70 - i) % 8], S[(71 - i) % 8], \ + W[i + ii] + K[i + ii]) + +/* Message schedule computation */ +#define MSCH(W, ii, i) \ + W[i + ii + 16] = s1(W[i + ii + 14]) + W[i + ii + 9] + \ + s0(W[i + ii + 1]) + W[i + ii] + +/* + * SHA256 block compression function. The 256-bit state is transformed via + * the 512-bit input block to produce a new state. + */ +static void +SHA256_Transform(uint32_t *state, const unsigned char block[64]) +{ + uint32_t W[64]; + uint32_t S[8]; + int i; + + /* 1. Prepare the first part of the message schedule W. */ + be32dec_vect(W, block, 64); + + /* 2. Initialize working variables. */ + memcpy(S, state, 32); + + /* 3. Mix. */ + for (i = 0; i < 64; i += 16) { + RNDr(S, W, 0, i); + RNDr(S, W, 1, i); + RNDr(S, W, 2, i); + RNDr(S, W, 3, i); + RNDr(S, W, 4, i); + RNDr(S, W, 5, i); + RNDr(S, W, 6, i); + RNDr(S, W, 7, i); + RNDr(S, W, 8, i); + RNDr(S, W, 9, i); + RNDr(S, W, 10, i); + RNDr(S, W, 11, i); + RNDr(S, W, 12, i); + RNDr(S, W, 13, i); + RNDr(S, W, 14, i); + RNDr(S, W, 15, i); + + if (i == 48) + break; + MSCH(W, 0, i); + MSCH(W, 1, i); + MSCH(W, 2, i); + MSCH(W, 3, i); + MSCH(W, 4, i); + MSCH(W, 5, i); + MSCH(W, 6, i); + MSCH(W, 7, i); + MSCH(W, 8, i); + MSCH(W, 9, i); + MSCH(W, 10, i); + MSCH(W, 11, i); + MSCH(W, 12, i); + MSCH(W, 13, i); + MSCH(W, 14, i); + MSCH(W, 15, i); + } + + /* 4. Mix local working variables into global state */ + for (i = 0; i < 8; i++) + state[i] += S[i]; +} + +static unsigned char PAD[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* Add padding and terminating bit-count. */ +static void +SHA256_Pad(SHA256_CTX * ctx) +{ + size_t r; + + /* Figure out how many bytes we have buffered. */ + r = (ctx->count >> 3) & 0x3f; + + /* Pad to 56 mod 64, transforming if we finish a block en route. */ + if (r < 56) { + /* Pad to 56 mod 64. */ + memcpy(&ctx->buf[r], PAD, 56 - r); + } else { + /* Finish the current block and mix. */ + memcpy(&ctx->buf[r], PAD, 64 - r); + SHA256_Transform(ctx->state, ctx->buf); + + /* The start of the final block is all zeroes. */ + memset(&ctx->buf[0], 0, 56); + } + + /* Add the terminating bit-count. */ + be64enc(&ctx->buf[56], ctx->count); + + /* Mix in the final block. */ + SHA256_Transform(ctx->state, ctx->buf); +} + +/* SHA-256 initialization. Begins a SHA-256 operation. */ +void +SHA256_Init(SHA256_CTX * ctx) +{ + + /* Zero bits processed so far */ + ctx->count = 0; + + /* Magic initialization constants */ + ctx->state[0] = 0x6A09E667; + ctx->state[1] = 0xBB67AE85; + ctx->state[2] = 0x3C6EF372; + ctx->state[3] = 0xA54FF53A; + ctx->state[4] = 0x510E527F; + ctx->state[5] = 0x9B05688C; + ctx->state[6] = 0x1F83D9AB; + ctx->state[7] = 0x5BE0CD19; +} + +/* Add bytes into the hash */ +void +SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len) +{ + uint64_t bitlen; + uint32_t r; + const unsigned char *src = in; + + /* Number of bytes left in the buffer from previous updates */ + r = (ctx->count >> 3) & 0x3f; + + /* Convert the length into a number of bits */ + bitlen = len << 3; + + /* Update number of bits */ + ctx->count += bitlen; + + /* Handle the case where we don't need to perform any transforms */ + if (len < 64 - r) { + memcpy(&ctx->buf[r], src, len); + return; + } + + /* Finish the current block */ + memcpy(&ctx->buf[r], src, 64 - r); + SHA256_Transform(ctx->state, ctx->buf); + src += 64 - r; + len -= 64 - r; + + /* Perform complete blocks */ + while (len >= 64) { + SHA256_Transform(ctx->state, src); + src += 64; + len -= 64; + } + + /* Copy left over data into buffer */ + memcpy(ctx->buf, src, len); +} + +/* + * SHA-256 finalization. Pads the input data, exports the hash value, + * and clears the context state. + */ +void +SHA256_Final(unsigned char digest[static SHA256_DIGEST_LENGTH], SHA256_CTX *ctx) +{ + + /* Add padding */ + SHA256_Pad(ctx); + + /* Write the hash */ + be32enc_vect(digest, ctx->state, SHA256_DIGEST_LENGTH); + + /* Clear the context state */ + explicit_bzero(ctx, sizeof (*ctx)); +} + +/* SHA-224: ******************************************************* */ +/* + * the SHA224 and SHA256 transforms are identical + */ + +/* SHA-224 initialization. Begins a SHA-224 operation. */ +void +SHA224_Init(SHA224_CTX * ctx) +{ + + /* Zero bits processed so far */ + ctx->count = 0; + + /* Magic initialization constants */ + ctx->state[0] = 0xC1059ED8; + ctx->state[1] = 0x367CD507; + ctx->state[2] = 0x3070DD17; + ctx->state[3] = 0xF70E5939; + ctx->state[4] = 0xFFC00B31; + ctx->state[5] = 0x68581511; + ctx->state[6] = 0x64f98FA7; + ctx->state[7] = 0xBEFA4FA4; +} + +/* Add bytes into the SHA-224 hash */ +void +SHA224_Update(SHA224_CTX * ctx, const void *in, size_t len) +{ + + SHA256_Update((SHA256_CTX *)ctx, in, len); +} + +/* + * SHA-224 finalization. Pads the input data, exports the hash value, + * and clears the context state. + */ +void +SHA224_Final(unsigned char digest[static SHA224_DIGEST_LENGTH], SHA224_CTX *ctx) +{ + + /* Add padding */ + SHA256_Pad((SHA256_CTX *)ctx); + + /* Write the hash */ + be32enc_vect(digest, ctx->state, SHA224_DIGEST_LENGTH); + + /* Clear the context state */ + explicit_bzero(ctx, sizeof (*ctx)); +} + +#ifdef WEAK_REFS +/* + * When building libmd, provide weak references. Note: this is not + * activated in the context of compiling these sources for internal + * use in libcrypt. + */ +#undef SHA256_Init +__weak_reference(_libmd_SHA256_Init, SHA256_Init); +#undef SHA256_Update +__weak_reference(_libmd_SHA256_Update, SHA256_Update); +#undef SHA256_Final +__weak_reference(_libmd_SHA256_Final, SHA256_Final); +#undef SHA256_Transform +__weak_reference(_libmd_SHA256_Transform, SHA256_Transform); + +#undef SHA224_Init +__weak_reference(_libmd_SHA224_Init, SHA224_Init); +#undef SHA224_Update +__weak_reference(_libmd_SHA224_Update, SHA224_Update); +#undef SHA224_Final +__weak_reference(_libmd_SHA224_Final, SHA224_Final); +#endif diff --git a/module/os/freebsd/spl/sha384.h b/module/os/freebsd/spl/sha384.h new file mode 100644 index 00000000000..67250cee031 --- /dev/null +++ b/module/os/freebsd/spl/sha384.h @@ -0,0 +1,96 @@ +/* + * Copyright 2005 Colin Percival + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SHA384_H_ +#define _SHA384_H_ + +#ifndef _KERNEL +#include +#endif + +#define SHA384_BLOCK_LENGTH 128 +#define SHA384_DIGEST_LENGTH 48 +#define SHA384_DIGEST_STRING_LENGTH (SHA384_DIGEST_LENGTH * 2 + 1) + +typedef struct SHA384Context { + uint64_t state[8]; + uint64_t count[2]; + uint8_t buf[SHA384_BLOCK_LENGTH]; +} SHA384_CTX; + +__BEGIN_DECLS + +/* Ensure libmd symbols do not clash with libcrypto */ +#ifndef SHA384_Init +#define SHA384_Init _libmd_SHA384_Init +#endif +#ifndef SHA384_Update +#define SHA384_Update _libmd_SHA384_Update +#endif +#ifndef SHA384_Final +#define SHA384_Final _libmd_SHA384_Final +#endif +#ifndef SHA384_End +#define SHA384_End _libmd_SHA384_End +#endif +#ifndef SHA384_Fd +#define SHA384_Fd _libmd_SHA384_Fd +#endif +#ifndef SHA384_FdChunk +#define SHA384_FdChunk _libmd_SHA384_FdChunk +#endif +#ifndef SHA384_File +#define SHA384_File _libmd_SHA384_File +#endif +#ifndef SHA384_FileChunk +#define SHA384_FileChunk _libmd_SHA384_FileChunk +#endif +#ifndef SHA384_Data +#define SHA384_Data _libmd_SHA384_Data +#endif + +#ifndef SHA384_version +#define SHA384_version _libmd_SHA384_version +#endif + +void SHA384_Init(SHA384_CTX *); +void SHA384_Update(SHA384_CTX *, const void *, size_t); +void SHA384_Final(unsigned char [__min_size(SHA384_DIGEST_LENGTH)], + SHA384_CTX *); +#ifndef _KERNEL +char *SHA384_End(SHA384_CTX *, char *); +char *SHA384_Data(const void *, unsigned int, char *); +char *SHA384_Fd(int, char *); +char *SHA384_FdChunk(int, char *, off_t, off_t); +char *SHA384_File(const char *, char *); +char *SHA384_FileChunk(const char *, char *, off_t, off_t); +#endif + +__END_DECLS + +#endif /* !_SHA384_H_ */ diff --git a/module/os/freebsd/spl/sha512.h b/module/os/freebsd/spl/sha512.h new file mode 100644 index 00000000000..b6fb733ca54 --- /dev/null +++ b/module/os/freebsd/spl/sha512.h @@ -0,0 +1,101 @@ +/* + * Copyright 2005 Colin Percival + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SHA512_H_ +#define _SHA512_H_ + +#ifndef _KERNEL +#include +#endif + +#define SHA512_BLOCK_LENGTH 128 +#define SHA512_DIGEST_LENGTH 64 +#define SHA512_DIGEST_STRING_LENGTH (SHA512_DIGEST_LENGTH * 2 + 1) + +typedef struct SHA512Context { + uint64_t state[8]; + uint64_t count[2]; + uint8_t buf[SHA512_BLOCK_LENGTH]; +} SHA512_CTX; + +__BEGIN_DECLS + +/* Ensure libmd symbols do not clash with libcrypto */ +#if 0 +#ifndef SHA512_Init +#define SHA512_Init _libmd_SHA512_Init +#endif +#ifndef SHA512_Update +#define SHA512_Update _libmd_SHA512_Update +#endif +#ifndef SHA512_Final +#define SHA512_Final _libmd_SHA512_Final +#endif +#endif +#ifndef SHA512_End +#define SHA512_End _libmd_SHA512_End +#endif +#ifndef SHA512_Fd +#define SHA512_Fd _libmd_SHA512_Fd +#endif +#ifndef SHA512_FdChunk +#define SHA512_FdChunk _libmd_SHA512_FdChunk +#endif +#ifndef SHA512_File +#define SHA512_File _libmd_SHA512_File +#endif +#ifndef SHA512_FileChunk +#define SHA512_FileChunk _libmd_SHA512_FileChunk +#endif +#ifndef SHA512_Data +#define SHA512_Data _libmd_SHA512_Data +#endif + +#ifndef SHA512_Transform +#define SHA512_Transform _libmd_SHA512_Transform +#endif +#ifndef SHA512_version +#define SHA512_version _libmd_SHA512_version +#endif + +void SHA512_Init(SHA512_CTX *); +void SHA512_Update(SHA512_CTX *, const void *, size_t); +void SHA512_Final(unsigned char [__min_size(SHA512_DIGEST_LENGTH)], + SHA512_CTX *); +#ifndef _KERNEL +char *SHA512_End(SHA512_CTX *, char *); +char *SHA512_Data(const void *, unsigned int, char *); +char *SHA512_Fd(int, char *); +char *SHA512_FdChunk(int, char *, off_t, off_t); +char *SHA512_File(const char *, char *); +char *SHA512_FileChunk(const char *, char *, off_t, off_t); +#endif + +__END_DECLS + +#endif /* !_SHA512_H_ */ diff --git a/module/os/freebsd/spl/sha512c.c b/module/os/freebsd/spl/sha512c.c new file mode 100644 index 00000000000..146f338f0ed --- /dev/null +++ b/module/os/freebsd/spl/sha512c.c @@ -0,0 +1,508 @@ +/* + * Copyright 2005 Colin Percival + * Copyright (c) 2015 Allan Jude + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#ifdef _KERNEL +#include +#else +#include +#endif + +#include "sha512.h" +#include "sha512t.h" +#include "sha384.h" + +#if BYTE_ORDER == BIG_ENDIAN + +/* Copy a vector of big-endian uint64_t into a vector of bytes */ +#define be64enc_vect(dst, src, len) \ + memcpy((void *)dst, (const void *)src, (size_t)len) + +/* Copy a vector of bytes into a vector of big-endian uint64_t */ +#define be64dec_vect(dst, src, len) \ + memcpy((void *)dst, (const void *)src, (size_t)len) + +#else /* BYTE_ORDER != BIG_ENDIAN */ + +/* + * Encode a length len/4 vector of (uint64_t) into a length len vector of + * (unsigned char) in big-endian form. Assumes len is a multiple of 8. + */ +static void +be64enc_vect(unsigned char *dst, const uint64_t *src, size_t len) +{ + size_t i; + + for (i = 0; i < len / 8; i++) + be64enc(dst + i * 8, src[i]); +} + +/* + * Decode a big-endian length len vector of (unsigned char) into a length + * len/4 vector of (uint64_t). Assumes len is a multiple of 8. + */ +static void +be64dec_vect(uint64_t *dst, const unsigned char *src, size_t len) +{ + size_t i; + + for (i = 0; i < len / 8; i++) + dst[i] = be64dec(src + i * 8); +} + +#endif /* BYTE_ORDER != BIG_ENDIAN */ + +/* SHA512 round constants. */ +static const uint64_t K[80] = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL +}; + +/* Elementary functions used by SHA512 */ +#define Ch(x, y, z) ((x & (y ^ z)) ^ z) +#define Maj(x, y, z) ((x & (y | z)) | (y & z)) +#define SHR(x, n) (x >> n) +#define ROTR(x, n) ((x >> n) | (x << (64 - n))) +#define S0(x) (ROTR(x, 28) ^ ROTR(x, 34) ^ ROTR(x, 39)) +#define S1(x) (ROTR(x, 14) ^ ROTR(x, 18) ^ ROTR(x, 41)) +#define s0(x) (ROTR(x, 1) ^ ROTR(x, 8) ^ SHR(x, 7)) +#define s1(x) (ROTR(x, 19) ^ ROTR(x, 61) ^ SHR(x, 6)) + +/* SHA512 round function */ +#define RND(a, b, c, d, e, f, g, h, k) \ + h += S1(e) + Ch(e, f, g) + k; \ + d += h; \ + h += S0(a) + Maj(a, b, c); + +/* Adjusted round function for rotating state */ +#define RNDr(S, W, i, ii) \ + RND(S[(80 - i) % 8], S[(81 - i) % 8], \ + S[(82 - i) % 8], S[(83 - i) % 8], \ + S[(84 - i) % 8], S[(85 - i) % 8], \ + S[(86 - i) % 8], S[(87 - i) % 8], \ + W[i + ii] + K[i + ii]) + +/* Message schedule computation */ +#define MSCH(W, ii, i) \ + W[i + ii + 16] = s1(W[i + ii + 14]) + W[i + ii + 9] + \ + s0(W[i + ii + 1]) + W[i + ii] + +/* + * SHA512 block compression function. The 512-bit state is transformed via + * the 512-bit input block to produce a new state. + */ +static void +SHA512_Transform(uint64_t *state, + const unsigned char block[SHA512_BLOCK_LENGTH]) +{ + uint64_t W[80]; + uint64_t S[8]; + int i; + + /* 1. Prepare the first part of the message schedule W. */ + be64dec_vect(W, block, SHA512_BLOCK_LENGTH); + + /* 2. Initialize working variables. */ + memcpy(S, state, SHA512_DIGEST_LENGTH); + + /* 3. Mix. */ + for (i = 0; i < 80; i += 16) { + RNDr(S, W, 0, i); + RNDr(S, W, 1, i); + RNDr(S, W, 2, i); + RNDr(S, W, 3, i); + RNDr(S, W, 4, i); + RNDr(S, W, 5, i); + RNDr(S, W, 6, i); + RNDr(S, W, 7, i); + RNDr(S, W, 8, i); + RNDr(S, W, 9, i); + RNDr(S, W, 10, i); + RNDr(S, W, 11, i); + RNDr(S, W, 12, i); + RNDr(S, W, 13, i); + RNDr(S, W, 14, i); + RNDr(S, W, 15, i); + + if (i == 64) + break; + MSCH(W, 0, i); + MSCH(W, 1, i); + MSCH(W, 2, i); + MSCH(W, 3, i); + MSCH(W, 4, i); + MSCH(W, 5, i); + MSCH(W, 6, i); + MSCH(W, 7, i); + MSCH(W, 8, i); + MSCH(W, 9, i); + MSCH(W, 10, i); + MSCH(W, 11, i); + MSCH(W, 12, i); + MSCH(W, 13, i); + MSCH(W, 14, i); + MSCH(W, 15, i); + } + + /* 4. Mix local working variables into global state */ + for (i = 0; i < 8; i++) + state[i] += S[i]; +} + +static unsigned char PAD[SHA512_BLOCK_LENGTH] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* Add padding and terminating bit-count. */ +static void +SHA512_Pad(SHA512_CTX * ctx) +{ + size_t r; + + /* Figure out how many bytes we have buffered. */ + r = (ctx->count[1] >> 3) & 0x7f; + + /* Pad to 112 mod 128, transforming if we finish a block en route. */ + if (r < 112) { + /* Pad to 112 mod 128. */ + memcpy(&ctx->buf[r], PAD, 112 - r); + } else { + /* Finish the current block and mix. */ + memcpy(&ctx->buf[r], PAD, 128 - r); + SHA512_Transform(ctx->state, ctx->buf); + + /* The start of the final block is all zeroes. */ + memset(&ctx->buf[0], 0, 112); + } + + /* Add the terminating bit-count. */ + be64enc_vect(&ctx->buf[112], ctx->count, 16); + + /* Mix in the final block. */ + SHA512_Transform(ctx->state, ctx->buf); +} + +/* SHA-512 initialization. Begins a SHA-512 operation. */ +void +SHA512_Init(SHA512_CTX * ctx) +{ + + /* Zero bits processed so far */ + ctx->count[0] = ctx->count[1] = 0; + + /* Magic initialization constants */ + ctx->state[0] = 0x6a09e667f3bcc908ULL; + ctx->state[1] = 0xbb67ae8584caa73bULL; + ctx->state[2] = 0x3c6ef372fe94f82bULL; + ctx->state[3] = 0xa54ff53a5f1d36f1ULL; + ctx->state[4] = 0x510e527fade682d1ULL; + ctx->state[5] = 0x9b05688c2b3e6c1fULL; + ctx->state[6] = 0x1f83d9abfb41bd6bULL; + ctx->state[7] = 0x5be0cd19137e2179ULL; +} + +/* Add bytes into the hash */ +void +SHA512_Update(SHA512_CTX * ctx, const void *in, size_t len) +{ + uint64_t bitlen[2]; + uint64_t r; + const unsigned char *src = in; + + /* Number of bytes left in the buffer from previous updates */ + r = (ctx->count[1] >> 3) & 0x7f; + + /* Convert the length into a number of bits */ + bitlen[1] = ((uint64_t)len) << 3; + bitlen[0] = ((uint64_t)len) >> 61; + + /* Update number of bits */ + if ((ctx->count[1] += bitlen[1]) < bitlen[1]) + ctx->count[0]++; + ctx->count[0] += bitlen[0]; + + /* Handle the case where we don't need to perform any transforms */ + if (len < SHA512_BLOCK_LENGTH - r) { + memcpy(&ctx->buf[r], src, len); + return; + } + + /* Finish the current block */ + memcpy(&ctx->buf[r], src, SHA512_BLOCK_LENGTH - r); + SHA512_Transform(ctx->state, ctx->buf); + src += SHA512_BLOCK_LENGTH - r; + len -= SHA512_BLOCK_LENGTH - r; + + /* Perform complete blocks */ + while (len >= SHA512_BLOCK_LENGTH) { + SHA512_Transform(ctx->state, src); + src += SHA512_BLOCK_LENGTH; + len -= SHA512_BLOCK_LENGTH; + } + + /* Copy left over data into buffer */ + memcpy(ctx->buf, src, len); +} + +/* + * SHA-512 finalization. Pads the input data, exports the hash value, + * and clears the context state. + */ +void +SHA512_Final(unsigned char digest[static SHA512_DIGEST_LENGTH], SHA512_CTX *ctx) +{ + + /* Add padding */ + SHA512_Pad(ctx); + + /* Write the hash */ + be64enc_vect(digest, ctx->state, SHA512_DIGEST_LENGTH); + + /* Clear the context state */ + explicit_bzero(ctx, sizeof (*ctx)); +} + +/* SHA-512t: ******************************************************** */ +/* + * the SHA512t transforms are identical to SHA512 so reuse the existing function + */ +void +SHA512_224_Init(SHA512_CTX * ctx) +{ + + /* Zero bits processed so far */ + ctx->count[0] = ctx->count[1] = 0; + + /* Magic initialization constants */ + ctx->state[0] = 0x8c3d37c819544da2ULL; + ctx->state[1] = 0x73e1996689dcd4d6ULL; + ctx->state[2] = 0x1dfab7ae32ff9c82ULL; + ctx->state[3] = 0x679dd514582f9fcfULL; + ctx->state[4] = 0x0f6d2b697bd44da8ULL; + ctx->state[5] = 0x77e36f7304c48942ULL; + ctx->state[6] = 0x3f9d85a86a1d36c8ULL; + ctx->state[7] = 0x1112e6ad91d692a1ULL; +} + +void +SHA512_224_Update(SHA512_CTX * ctx, const void *in, size_t len) +{ + + SHA512_Update(ctx, in, len); +} + +void +SHA512_224_Final(unsigned char digest[static SHA512_224_DIGEST_LENGTH], + SHA512_CTX *ctx) +{ + + /* Add padding */ + SHA512_Pad(ctx); + + /* Write the hash */ + be64enc_vect(digest, ctx->state, SHA512_224_DIGEST_LENGTH); + + /* Clear the context state */ + explicit_bzero(ctx, sizeof (*ctx)); +} + +void +SHA512_256_Init(SHA512_CTX * ctx) +{ + + /* Zero bits processed so far */ + ctx->count[0] = ctx->count[1] = 0; + + /* Magic initialization constants */ + ctx->state[0] = 0x22312194fc2bf72cULL; + ctx->state[1] = 0x9f555fa3c84c64c2ULL; + ctx->state[2] = 0x2393b86b6f53b151ULL; + ctx->state[3] = 0x963877195940eabdULL; + ctx->state[4] = 0x96283ee2a88effe3ULL; + ctx->state[5] = 0xbe5e1e2553863992ULL; + ctx->state[6] = 0x2b0199fc2c85b8aaULL; + ctx->state[7] = 0x0eb72ddc81c52ca2ULL; +} + +void +SHA512_256_Update(SHA512_CTX * ctx, const void *in, size_t len) +{ + + SHA512_Update(ctx, in, len); +} + +void +SHA512_256_Final(unsigned char digest[static SHA512_256_DIGEST_LENGTH], + SHA512_CTX * ctx) +{ + + /* Add padding */ + SHA512_Pad(ctx); + + /* Write the hash */ + be64enc_vect(digest, ctx->state, SHA512_256_DIGEST_LENGTH); + + /* Clear the context state */ + explicit_bzero(ctx, sizeof (*ctx)); +} + +/* ** SHA-384: ******************************************************** */ +/* + * the SHA384 and SHA512 transforms are identical, so SHA384 is skipped + */ + +/* SHA-384 initialization. Begins a SHA-384 operation. */ +void +SHA384_Init(SHA384_CTX * ctx) +{ + + /* Zero bits processed so far */ + ctx->count[0] = ctx->count[1] = 0; + + /* Magic initialization constants */ + ctx->state[0] = 0xcbbb9d5dc1059ed8ULL; + ctx->state[1] = 0x629a292a367cd507ULL; + ctx->state[2] = 0x9159015a3070dd17ULL; + ctx->state[3] = 0x152fecd8f70e5939ULL; + ctx->state[4] = 0x67332667ffc00b31ULL; + ctx->state[5] = 0x8eb44a8768581511ULL; + ctx->state[6] = 0xdb0c2e0d64f98fa7ULL; + ctx->state[7] = 0x47b5481dbefa4fa4ULL; +} + +/* Add bytes into the SHA-384 hash */ +void +SHA384_Update(SHA384_CTX * ctx, const void *in, size_t len) +{ + + SHA512_Update((SHA512_CTX *)ctx, in, len); +} + +/* + * SHA-384 finalization. Pads the input data, exports the hash value, + * and clears the context state. + */ +void +SHA384_Final(unsigned char digest[static SHA384_DIGEST_LENGTH], SHA384_CTX *ctx) +{ + + /* Add padding */ + SHA512_Pad((SHA512_CTX *)ctx); + + /* Write the hash */ + be64enc_vect(digest, ctx->state, SHA384_DIGEST_LENGTH); + + /* Clear the context state */ + explicit_bzero(ctx, sizeof (*ctx)); +} + +#if 0 +/* + * When building libmd, provide weak references. Note: this is not + * activated in the context of compiling these sources for internal + * use in libcrypt. + */ +#undef SHA512_Init +__weak_reference(_libmd_SHA512_Init, SHA512_Init); +#undef SHA512_Update +__weak_reference(_libmd_SHA512_Update, SHA512_Update); +#undef SHA512_Final +__weak_reference(_libmd_SHA512_Final, SHA512_Final); +#undef SHA512_Transform +__weak_reference(_libmd_SHA512_Transform, SHA512_Transform); + +#undef SHA512_224_Init +__weak_reference(_libmd_SHA512_224_Init, SHA512_224_Init); +#undef SHA512_224_Update +__weak_reference(_libmd_SHA512_224_Update, SHA512_224_Update); +#undef SHA512_224_Final +__weak_reference(_libmd_SHA512_224_Final, SHA512_224_Final); + +#undef SHA512_256_Init +__weak_reference(_libmd_SHA512_256_Init, SHA512_256_Init); +#undef SHA512_256_Update +__weak_reference(_libmd_SHA512_256_Update, SHA512_256_Update); +#undef SHA512_256_Final +__weak_reference(_libmd_SHA512_256_Final, SHA512_256_Final); + +#undef SHA384_Init +__weak_reference(_libmd_SHA384_Init, SHA384_Init); +#undef SHA384_Update +__weak_reference(_libmd_SHA384_Update, SHA384_Update); +#undef SHA384_Final +__weak_reference(_libmd_SHA384_Final, SHA384_Final); +#endif diff --git a/module/os/freebsd/spl/sha512t.h b/module/os/freebsd/spl/sha512t.h new file mode 100644 index 00000000000..703867fc028 --- /dev/null +++ b/module/os/freebsd/spl/sha512t.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2015 Allan Jude + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SHA512T_H_ +#define _SHA512T_H_ + +#include "sha512.h" + +#ifndef _KERNEL +#include +#endif + +#define SHA512_224_DIGEST_LENGTH 28 +#define SHA512_224_DIGEST_STRING_LENGTH (SHA512_224_DIGEST_LENGTH * 2 + 1) +#define SHA512_256_DIGEST_LENGTH 32 +#define SHA512_256_DIGEST_STRING_LENGTH (SHA512_256_DIGEST_LENGTH * 2 + 1) + +__BEGIN_DECLS + +/* Ensure libmd symbols do not clash with libcrypto */ +#ifndef SHA512_224_Init +#define SHA512_224_Init _libmd_SHA512_224_Init +#endif +#ifndef SHA512_224_Update +#define SHA512_224_Update _libmd_SHA512_224_Update +#endif +#ifndef SHA512_224_Final +#define SHA512_224_Final _libmd_SHA512_224_Final +#endif +#ifndef SHA512_224_End +#define SHA512_224_End _libmd_SHA512_224_End +#endif +#ifndef SHA512_224_Fd +#define SHA512_224_Fd _libmd_SHA512_224_Fd +#endif +#ifndef SHA512_224_FdChunk +#define SHA512_224_FdChunk _libmd_SHA512_224_FdChunk +#endif +#ifndef SHA512_224_File +#define SHA512_224_File _libmd_SHA512_224_File +#endif +#ifndef SHA512_224_FileChunk +#define SHA512_224_FileChunk _libmd_SHA512_224_FileChunk +#endif +#ifndef SHA512_224_Data +#define SHA512_224_Data _libmd_SHA512_224_Data +#endif + +#ifndef SHA512_224_Transform +#define SHA512_224_Transform _libmd_SHA512_224_Transform +#endif +#ifndef SHA512_224_version +#define SHA512_224_version _libmd_SHA512_224_version +#endif + +#ifndef SHA512_256_Init +#define SHA512_256_Init _libmd_SHA512_256_Init +#endif +#ifndef SHA512_256_Update +#define SHA512_256_Update _libmd_SHA512_256_Update +#endif +#ifndef SHA512_256_Final +#define SHA512_256_Final _libmd_SHA512_256_Final +#endif +#ifndef SHA512_256_End +#define SHA512_256_End _libmd_SHA512_256_End +#endif +#ifndef SHA512_256_Fd +#define SHA512_256_Fd _libmd_SHA512_256_Fd +#endif +#ifndef SHA512_256_FdChunk +#define SHA512_256_FdChunk _libmd_SHA512_256_FdChunk +#endif +#ifndef SHA512_256_File +#define SHA512_256_File _libmd_SHA512_256_File +#endif +#ifndef SHA512_256_FileChunk +#define SHA512_256_FileChunk _libmd_SHA512_256_FileChunk +#endif +#ifndef SHA512_256_Data +#define SHA512_256_Data _libmd_SHA512_256_Data +#endif + +#ifndef SHA512_256_Transform +#define SHA512_256_Transform _libmd_SHA512_256_Transform +#endif +#ifndef SHA512_256_version +#define SHA512_256_version _libmd_SHA512_256_version +#endif + +void SHA512_224_Init(SHA512_CTX *); +void SHA512_224_Update(SHA512_CTX *, const void *, size_t); +void SHA512_224_Final(unsigned char [__min_size(SHA512_224_DIGEST_LENGTH)], + SHA512_CTX *); +#ifndef _KERNEL +char *SHA512_224_End(SHA512_CTX *, char *); +char *SHA512_224_Data(const void *, unsigned int, char *); +char *SHA512_224_Fd(int, char *); +char *SHA512_224_FdChunk(int, char *, off_t, off_t); +char *SHA512_224_File(const char *, char *); +char *SHA512_224_FileChunk(const char *, char *, off_t, off_t); +#endif +void SHA512_256_Init(SHA512_CTX *); +void SHA512_256_Update(SHA512_CTX *, const void *, size_t); +void SHA512_256_Final(unsigned char [__min_size(SHA512_256_DIGEST_LENGTH)], + SHA512_CTX *); +#ifndef _KERNEL +char *SHA512_256_End(SHA512_CTX *, char *); +char *SHA512_256_Data(const void *, unsigned int, char *); +char *SHA512_256_Fd(int, char *); +char *SHA512_256_FdChunk(int, char *, off_t, off_t); +char *SHA512_256_File(const char *, char *); +char *SHA512_256_FileChunk(const char *, char *, off_t, off_t); +#endif + +__END_DECLS + +#endif /* !_SHA512T_H_ */ diff --git a/module/os/freebsd/spl/spl_acl.c b/module/os/freebsd/spl/spl_acl.c new file mode 100644 index 00000000000..bb4c30728d5 --- /dev/null +++ b/module/os/freebsd/spl/spl_acl.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2008, 2009 Edward Tomasz Napierała + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +struct zfs2bsd { + uint32_t zb_zfs; + int zb_bsd; +}; + +struct zfs2bsd perms[] = {{ACE_READ_DATA, ACL_READ_DATA}, + {ACE_WRITE_DATA, ACL_WRITE_DATA}, + {ACE_EXECUTE, ACL_EXECUTE}, + {ACE_APPEND_DATA, ACL_APPEND_DATA}, + {ACE_DELETE_CHILD, ACL_DELETE_CHILD}, + {ACE_DELETE, ACL_DELETE}, + {ACE_READ_ATTRIBUTES, ACL_READ_ATTRIBUTES}, + {ACE_WRITE_ATTRIBUTES, ACL_WRITE_ATTRIBUTES}, + {ACE_READ_NAMED_ATTRS, ACL_READ_NAMED_ATTRS}, + {ACE_WRITE_NAMED_ATTRS, ACL_WRITE_NAMED_ATTRS}, + {ACE_READ_ACL, ACL_READ_ACL}, + {ACE_WRITE_ACL, ACL_WRITE_ACL}, + {ACE_WRITE_OWNER, ACL_WRITE_OWNER}, + {ACE_SYNCHRONIZE, ACL_SYNCHRONIZE}, + {0, 0}}; + +struct zfs2bsd flags[] = {{ACE_FILE_INHERIT_ACE, + ACL_ENTRY_FILE_INHERIT}, + {ACE_DIRECTORY_INHERIT_ACE, + ACL_ENTRY_DIRECTORY_INHERIT}, + {ACE_NO_PROPAGATE_INHERIT_ACE, + ACL_ENTRY_NO_PROPAGATE_INHERIT}, + {ACE_INHERIT_ONLY_ACE, + ACL_ENTRY_INHERIT_ONLY}, + {ACE_INHERITED_ACE, + ACL_ENTRY_INHERITED}, + {ACE_SUCCESSFUL_ACCESS_ACE_FLAG, + ACL_ENTRY_SUCCESSFUL_ACCESS}, + {ACE_FAILED_ACCESS_ACE_FLAG, + ACL_ENTRY_FAILED_ACCESS}, + {0, 0}}; + +static int +_bsd_from_zfs(uint32_t zfs, const struct zfs2bsd *table) +{ + const struct zfs2bsd *tmp; + int bsd = 0; + + for (tmp = table; tmp->zb_zfs != 0; tmp++) { + if (zfs & tmp->zb_zfs) + bsd |= tmp->zb_bsd; + } + + return (bsd); +} + +static uint32_t +_zfs_from_bsd(int bsd, const struct zfs2bsd *table) +{ + const struct zfs2bsd *tmp; + uint32_t zfs = 0; + + for (tmp = table; tmp->zb_bsd != 0; tmp++) { + if (bsd & tmp->zb_bsd) + zfs |= tmp->zb_zfs; + } + + return (zfs); +} + +int +acl_from_aces(struct acl *aclp, const ace_t *aces, int nentries) +{ + int i; + struct acl_entry *entry; + const ace_t *ace; + + if (nentries < 1) { + printf("acl_from_aces: empty ZFS ACL; returning EINVAL.\n"); + return (EINVAL); + } + + if (nentries > ACL_MAX_ENTRIES) { + /* + * I believe it may happen only when moving a pool + * from SunOS to FreeBSD. + */ + printf("acl_from_aces: ZFS ACL too big to fit " + "into 'struct acl'; returning EINVAL.\n"); + return (EINVAL); + } + + bzero(aclp, sizeof (*aclp)); + aclp->acl_maxcnt = ACL_MAX_ENTRIES; + aclp->acl_cnt = nentries; + + for (i = 0; i < nentries; i++) { + entry = &(aclp->acl_entry[i]); + ace = &(aces[i]); + + if (ace->a_flags & ACE_OWNER) + entry->ae_tag = ACL_USER_OBJ; + else if (ace->a_flags & ACE_GROUP) + entry->ae_tag = ACL_GROUP_OBJ; + else if (ace->a_flags & ACE_EVERYONE) + entry->ae_tag = ACL_EVERYONE; + else if (ace->a_flags & ACE_IDENTIFIER_GROUP) + entry->ae_tag = ACL_GROUP; + else + entry->ae_tag = ACL_USER; + + if (entry->ae_tag == ACL_USER || entry->ae_tag == ACL_GROUP) + entry->ae_id = ace->a_who; + else + entry->ae_id = ACL_UNDEFINED_ID; + + entry->ae_perm = _bsd_from_zfs(ace->a_access_mask, perms); + entry->ae_flags = _bsd_from_zfs(ace->a_flags, flags); + + switch (ace->a_type) { + case ACE_ACCESS_ALLOWED_ACE_TYPE: + entry->ae_entry_type = ACL_ENTRY_TYPE_ALLOW; + break; + case ACE_ACCESS_DENIED_ACE_TYPE: + entry->ae_entry_type = ACL_ENTRY_TYPE_DENY; + break; + case ACE_SYSTEM_AUDIT_ACE_TYPE: + entry->ae_entry_type = ACL_ENTRY_TYPE_AUDIT; + break; + case ACE_SYSTEM_ALARM_ACE_TYPE: + entry->ae_entry_type = ACL_ENTRY_TYPE_ALARM; + break; + default: + panic("acl_from_aces: a_type is 0x%x", ace->a_type); + } + } + + return (0); +} + +void +aces_from_acl(ace_t *aces, int *nentries, const struct acl *aclp) +{ + int i; + const struct acl_entry *entry; + ace_t *ace; + + bzero(aces, sizeof (*aces) * aclp->acl_cnt); + + *nentries = aclp->acl_cnt; + + for (i = 0; i < aclp->acl_cnt; i++) { + entry = &(aclp->acl_entry[i]); + ace = &(aces[i]); + + ace->a_who = entry->ae_id; + + if (entry->ae_tag == ACL_USER_OBJ) + ace->a_flags = ACE_OWNER; + else if (entry->ae_tag == ACL_GROUP_OBJ) + ace->a_flags = (ACE_GROUP | ACE_IDENTIFIER_GROUP); + else if (entry->ae_tag == ACL_GROUP) + ace->a_flags = ACE_IDENTIFIER_GROUP; + else if (entry->ae_tag == ACL_EVERYONE) + ace->a_flags = ACE_EVERYONE; + else /* ACL_USER */ + ace->a_flags = 0; + + ace->a_access_mask = _zfs_from_bsd(entry->ae_perm, perms); + ace->a_flags |= _zfs_from_bsd(entry->ae_flags, flags); + + switch (entry->ae_entry_type) { + case ACL_ENTRY_TYPE_ALLOW: + ace->a_type = ACE_ACCESS_ALLOWED_ACE_TYPE; + break; + case ACL_ENTRY_TYPE_DENY: + ace->a_type = ACE_ACCESS_DENIED_ACE_TYPE; + break; + case ACL_ENTRY_TYPE_ALARM: + ace->a_type = ACE_SYSTEM_ALARM_ACE_TYPE; + break; + case ACL_ENTRY_TYPE_AUDIT: + ace->a_type = ACE_SYSTEM_AUDIT_ACE_TYPE; + break; + default: + panic("aces_from_acl: ae_entry_type is 0x%x", + entry->ae_entry_type); + } + } +} diff --git a/module/os/freebsd/spl/spl_atomic.c b/module/os/freebsd/spl/spl_atomic.c new file mode 100644 index 00000000000..e82fed84740 --- /dev/null +++ b/module/os/freebsd/spl/spl_atomic.c @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include + +#ifdef _KERNEL +#include + +struct mtx atomic_mtx; +MTX_SYSINIT(atomic, &atomic_mtx, "atomic", MTX_DEF); +#else +#include + +#define mtx_lock(lock) pthread_mutex_lock(lock) +#define mtx_unlock(lock) pthread_mutex_unlock(lock) + +static pthread_mutex_t atomic_mtx; + +static __attribute__((constructor)) void +atomic_init(void) +{ + pthread_mutex_init(&atomic_mtx, NULL); +} +#endif + +#if !defined(__LP64__) && !defined(__mips_n32) && \ + !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) +void +atomic_add_64(volatile uint64_t *target, int64_t delta) +{ + + mtx_lock(&atomic_mtx); + *target += delta; + mtx_unlock(&atomic_mtx); +} + +void +atomic_dec_64(volatile uint64_t *target) +{ + + mtx_lock(&atomic_mtx); + *target -= 1; + mtx_unlock(&atomic_mtx); +} +#endif + +uint64_t +atomic_add_64_nv(volatile uint64_t *target, int64_t delta) +{ + uint64_t newval; + + mtx_lock(&atomic_mtx); + newval = (*target += delta); + mtx_unlock(&atomic_mtx); + return (newval); +} + +#if defined(__powerpc__) || defined(__arm__) || defined(__mips__) +void +atomic_or_8(volatile uint8_t *target, uint8_t value) +{ + mtx_lock(&atomic_mtx); + *target |= value; + mtx_unlock(&atomic_mtx); +} +#endif + +uint8_t +atomic_or_8_nv(volatile uint8_t *target, uint8_t value) +{ + uint8_t newval; + + mtx_lock(&atomic_mtx); + newval = (*target |= value); + mtx_unlock(&atomic_mtx); + return (newval); +} + +uint64_t +atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) +{ + uint64_t oldval; + + mtx_lock(&atomic_mtx); + oldval = *target; + if (oldval == cmp) + *target = newval; + mtx_unlock(&atomic_mtx); + return (oldval); +} + +uint32_t +atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) +{ + uint32_t oldval; + + mtx_lock(&atomic_mtx); + oldval = *target; + if (oldval == cmp) + *target = newval; + mtx_unlock(&atomic_mtx); + return (oldval); +} + +void +membar_producer(void) +{ + /* nothing */ +} diff --git a/module/os/freebsd/spl/spl_cmn_err.c b/module/os/freebsd/spl/spl_cmn_err.c new file mode 100644 index 00000000000..23566603f5f --- /dev/null +++ b/module/os/freebsd/spl/spl_cmn_err.c @@ -0,0 +1,74 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2007 John Birrell . All rights reserved. + * Copyright 2012 Martin Matuska . All rights reserved. + */ + +#include + +void +vcmn_err(int ce, const char *fmt, va_list adx) +{ + char buf[256]; + const char *prefix; + + prefix = NULL; /* silence unwitty compilers */ + switch (ce) { + case CE_CONT: + prefix = "Solaris(cont): "; + break; + case CE_NOTE: + prefix = "Solaris: NOTICE: "; + break; + case CE_WARN: + prefix = "Solaris: WARNING: "; + break; + case CE_PANIC: + prefix = "Solaris(panic): "; + break; + case CE_IGNORE: + break; + default: + panic("Solaris: unknown severity level"); + } + if (ce == CE_PANIC) { + vsnprintf(buf, sizeof (buf), fmt, adx); + panic("%s%s", prefix, buf); + } + if (ce != CE_IGNORE) { + printf("%s", prefix); + vprintf(fmt, adx); + printf("\n"); + } +} + +void +cmn_err(int type, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vcmn_err(type, fmt, ap); + va_end(ap); +} diff --git a/module/os/freebsd/spl/spl_dtrace.c b/module/os/freebsd/spl/spl_dtrace.c new file mode 100644 index 00000000000..e7b2ff82309 --- /dev/null +++ b/module/os/freebsd/spl/spl_dtrace.c @@ -0,0 +1,37 @@ +/* + * Copyright 2014 The FreeBSD Project. + * All rights reserved. + * + * This software was developed by Steven Hartland. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +/* CSTYLED */ +SDT_PROBE_DEFINE1(sdt, , , set__error, "int"); diff --git a/module/os/freebsd/spl/spl_kmem.c b/module/os/freebsd/spl/spl_kmem.c new file mode 100644 index 00000000000..af3747c271f --- /dev/null +++ b/module/os/freebsd/spl/spl_kmem.c @@ -0,0 +1,351 @@ +/* + * Copyright (c) 2006-2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include +#include +#include +#include + +#ifdef KMEM_DEBUG +#include +#include +#endif + +#ifdef _KERNEL +MALLOC_DEFINE(M_SOLARIS, "solaris", "Solaris"); +#else +#define malloc(size, type, flags) malloc(size) +#define free(addr, type) free(addr) +#endif + +#ifdef KMEM_DEBUG +struct kmem_item { + struct stack stack; + LIST_ENTRY(kmem_item) next; +}; +static LIST_HEAD(, kmem_item) kmem_items; +static struct mtx kmem_items_mtx; +MTX_SYSINIT(kmem_items_mtx, &kmem_items_mtx, "kmem_items", MTX_DEF); +#endif /* KMEM_DEBUG */ + +#include + +void * +zfs_kmem_alloc(size_t size, int kmflags) +{ + void *p; +#ifdef KMEM_DEBUG + struct kmem_item *i; + + size += sizeof (struct kmem_item); +#endif + p = malloc(MAX(size, 16), M_SOLARIS, kmflags); +#ifndef _KERNEL + if (kmflags & KM_SLEEP) + assert(p != NULL); +#endif +#ifdef KMEM_DEBUG + if (p != NULL) { + i = p; + p = (uint8_t *)p + sizeof (struct kmem_item); + stack_save(&i->stack); + mtx_lock(&kmem_items_mtx); + LIST_INSERT_HEAD(&kmem_items, i, next); + mtx_unlock(&kmem_items_mtx); + } +#endif + return (p); +} + +void +zfs_kmem_free(void *buf, size_t size __unused) +{ +#ifdef KMEM_DEBUG + if (buf == NULL) { + printf("%s: attempt to free NULL\n", __func__); + return; + } + struct kmem_item *i; + + buf = (uint8_t *)buf - sizeof (struct kmem_item); + mtx_lock(&kmem_items_mtx); + LIST_FOREACH(i, &kmem_items, next) { + if (i == buf) + break; + } + ASSERT(i != NULL); + LIST_REMOVE(i, next); + mtx_unlock(&kmem_items_mtx); + memset(buf, 0xDC, MAX(size, 16)); +#endif + free(buf, M_SOLARIS); +} + +static uint64_t kmem_size_val; + +static void +kmem_size_init(void *unused __unused) +{ + + kmem_size_val = (uint64_t)vm_cnt.v_page_count * PAGE_SIZE; + if (kmem_size_val > vm_kmem_size) + kmem_size_val = vm_kmem_size; +} +SYSINIT(kmem_size_init, SI_SUB_KMEM, SI_ORDER_ANY, kmem_size_init, NULL); + +uint64_t +kmem_size(void) +{ + + return (kmem_size_val); +} + +static int +kmem_std_constructor(void *mem, int size __unused, void *private, int flags) +{ + struct kmem_cache *cache = private; + + return (cache->kc_constructor(mem, cache->kc_private, flags)); +} + +static void +kmem_std_destructor(void *mem, int size __unused, void *private) +{ + struct kmem_cache *cache = private; + + cache->kc_destructor(mem, cache->kc_private); +} + +kmem_cache_t * +kmem_cache_create(char *name, size_t bufsize, size_t align, + int (*constructor)(void *, void *, int), void (*destructor)(void *, void *), + void (*reclaim)(void *) __unused, void *private, vmem_t *vmp, int cflags) +{ + kmem_cache_t *cache; + + ASSERT(vmp == NULL); + + cache = kmem_alloc(sizeof (*cache), KM_SLEEP); + strlcpy(cache->kc_name, name, sizeof (cache->kc_name)); + cache->kc_constructor = constructor; + cache->kc_destructor = destructor; + cache->kc_private = private; +#if defined(_KERNEL) && !defined(KMEM_DEBUG) + cache->kc_zone = uma_zcreate(cache->kc_name, bufsize, + constructor != NULL ? kmem_std_constructor : NULL, + destructor != NULL ? kmem_std_destructor : NULL, + NULL, NULL, align > 0 ? align - 1 : 0, cflags); +#else + cache->kc_size = bufsize; +#endif + + return (cache); +} + +void +kmem_cache_destroy(kmem_cache_t *cache) +{ +#if defined(_KERNEL) && !defined(KMEM_DEBUG) + uma_zdestroy(cache->kc_zone); +#endif + kmem_free(cache, sizeof (*cache)); +} + +void * +kmem_cache_alloc(kmem_cache_t *cache, int flags) +{ +#if defined(_KERNEL) && !defined(KMEM_DEBUG) + return (uma_zalloc_arg(cache->kc_zone, cache, flags)); +#else + void *p; + + p = kmem_alloc(cache->kc_size, flags); + if (p != NULL && cache->kc_constructor != NULL) + kmem_std_constructor(p, cache->kc_size, cache, flags); + return (p); +#endif +} + +void +kmem_cache_free(kmem_cache_t *cache, void *buf) +{ +#if defined(_KERNEL) && !defined(KMEM_DEBUG) + uma_zfree_arg(cache->kc_zone, buf, cache); +#else + if (cache->kc_destructor != NULL) + kmem_std_destructor(buf, cache->kc_size, cache); + kmem_free(buf, cache->kc_size); +#endif +} + +/* + * Allow our caller to determine if there are running reaps. + * + * This call is very conservative and may return B_TRUE even when + * reaping activity isn't active. If it returns B_FALSE, then reaping + * activity is definitely inactive. + */ +boolean_t +kmem_cache_reap_active(void) +{ + + return (B_FALSE); +} + +/* + * Reap (almost) everything soon. + * + * Note: this does not wait for the reap-tasks to complete. Caller + * should use kmem_cache_reap_active() (above) and/or moderation to + * avoid scheduling too many reap-tasks. + */ +#ifdef _KERNEL +void +kmem_cache_reap_soon(kmem_cache_t *cache) +{ +#ifndef KMEM_DEBUG +#if __FreeBSD_version >= 1300043 + uma_zone_reclaim(cache->kc_zone, UMA_RECLAIM_DRAIN); +#else + zone_drain(cache->kc_zone); +#endif +#endif +} + +void +kmem_reap(void) +{ +#if __FreeBSD_version >= 1300043 + uma_reclaim(UMA_RECLAIM_TRIM); +#else + uma_reclaim(); +#endif +} +#else +void +kmem_cache_reap_soon(kmem_cache_t *cache __unused) +{ +} + +void +kmem_reap(void) +{ +} +#endif + +int +kmem_debugging(void) +{ + return (0); +} + +void * +calloc(size_t n, size_t s) +{ + return (kmem_zalloc(n * s, KM_NOSLEEP)); +} + +char * +kmem_vasprintf(const char *fmt, va_list adx) +{ + char *msg; + va_list adx2; + + va_copy(adx2, adx); + msg = kmem_alloc(vsnprintf(NULL, 0, fmt, adx) + 1, KM_SLEEP); + (void) vsprintf(msg, fmt, adx2); + va_end(adx2); + + return (msg); +} + +#include +#include +#ifdef KMEM_DEBUG +#error "KMEM_DEBUG not currently supported" +#endif + +uint64_t +spl_kmem_cache_inuse(kmem_cache_t *cache) +{ + return (uma_zone_get_cur(cache->kc_zone)); +} + +uint64_t +spl_kmem_cache_entry_size(kmem_cache_t *cache) +{ + return (cache->kc_zone->uz_size); +} + +/* + * Register a move callback for cache defragmentation. + * XXX: Unimplemented but harmless to stub out for now. + */ +void +spl_kmem_cache_set_move(kmem_cache_t *skc, + kmem_cbrc_t (move)(void *, void *, size_t, void *)) +{ + ASSERT(move != NULL); +} + +#ifdef KMEM_DEBUG +void kmem_show(void *); +void +kmem_show(void *dummy __unused) +{ + struct kmem_item *i; + + mtx_lock(&kmem_items_mtx); + if (LIST_EMPTY(&kmem_items)) + printf("KMEM_DEBUG: No leaked elements.\n"); + else { + printf("KMEM_DEBUG: Leaked elements:\n\n"); + LIST_FOREACH(i, &kmem_items, next) { + printf("address=%p\n", i); + stack_print_ddb(&i->stack); + printf("\n"); + } + } + mtx_unlock(&kmem_items_mtx); +} + +SYSUNINIT(sol_kmem, SI_SUB_CPU, SI_ORDER_FIRST, kmem_show, NULL); +#endif /* KMEM_DEBUG */ diff --git a/module/os/freebsd/spl/spl_kstat.c b/module/os/freebsd/spl/spl_kstat.c new file mode 100644 index 00000000000..fda03a3d788 --- /dev/null +++ b/module/os/freebsd/spl/spl_kstat.c @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include + +static MALLOC_DEFINE(M_KSTAT, "kstat_data", "Kernel statistics"); + +SYSCTL_ROOT_NODE(OID_AUTO, kstat, CTLFLAG_RW, 0, "Kernel statistics"); + +void +__kstat_set_raw_ops(kstat_t *ksp, + int (*headers)(char *buf, size_t size), + int (*data)(char *buf, size_t size, void *data), + void *(*addr)(kstat_t *ksp, loff_t index)) +{ + ksp->ks_raw_ops.headers = headers; + ksp->ks_raw_ops.data = data; + ksp->ks_raw_ops.addr = addr; +} + +static int +kstat_default_update(kstat_t *ksp, int rw) +{ + ASSERT(ksp != NULL); + + if (rw == KSTAT_WRITE) + return (EACCES); + + return (0); +} + +kstat_t * +__kstat_create(const char *module, int instance, const char *name, + const char *class, uchar_t ks_type, uint_t ks_ndata, uchar_t flags) +{ + struct sysctl_oid *root; + kstat_t *ksp; + + KASSERT(instance == 0, ("instance=%d", instance)); + if ((ks_type == KSTAT_TYPE_INTR) || (ks_type == KSTAT_TYPE_IO)) + ASSERT(ks_ndata == 1); + + /* + * Allocate the main structure. We don't need to copy module/class/name + * stuff in here, because it is only used for sysctl node creation + * done in this function. + */ + ksp = malloc(sizeof (*ksp), M_KSTAT, M_WAITOK|M_ZERO); + + ksp->ks_crtime = gethrtime(); + ksp->ks_snaptime = ksp->ks_crtime; + ksp->ks_instance = instance; + strncpy(ksp->ks_name, name, KSTAT_STRLEN); + strncpy(ksp->ks_class, class, KSTAT_STRLEN); + ksp->ks_type = ks_type; + ksp->ks_flags = flags; + ksp->ks_update = kstat_default_update; + + switch (ksp->ks_type) { + case KSTAT_TYPE_RAW: + ksp->ks_ndata = 1; + ksp->ks_data_size = ks_ndata; + break; + case KSTAT_TYPE_NAMED: + ksp->ks_ndata = ks_ndata; + ksp->ks_data_size = ks_ndata * sizeof (kstat_named_t); + break; + case KSTAT_TYPE_INTR: + ksp->ks_ndata = ks_ndata; + ksp->ks_data_size = ks_ndata * sizeof (kstat_intr_t); + break; + case KSTAT_TYPE_IO: + ksp->ks_ndata = ks_ndata; + ksp->ks_data_size = ks_ndata * sizeof (kstat_io_t); + break; + case KSTAT_TYPE_TIMER: + ksp->ks_ndata = ks_ndata; + ksp->ks_data_size = ks_ndata * sizeof (kstat_timer_t); + break; + default: + panic("Undefined kstat type %d\n", ksp->ks_type); + } + + if (ksp->ks_flags & KSTAT_FLAG_VIRTUAL) { + ksp->ks_data = NULL; + } else { + ksp->ks_data = kmem_zalloc(ksp->ks_data_size, KM_SLEEP); + if (ksp->ks_data == NULL) { + kmem_free(ksp, sizeof (*ksp)); + ksp = NULL; + } + } + /* + * Create sysctl tree for those statistics: + * + * kstat.... + */ + sysctl_ctx_init(&ksp->ks_sysctl_ctx); + root = SYSCTL_ADD_NODE(&ksp->ks_sysctl_ctx, + SYSCTL_STATIC_CHILDREN(_kstat), OID_AUTO, module, CTLFLAG_RW, 0, + ""); + if (root == NULL) { + printf("%s: Cannot create kstat.%s tree!\n", __func__, module); + sysctl_ctx_free(&ksp->ks_sysctl_ctx); + free(ksp, M_KSTAT); + return (NULL); + } + root = SYSCTL_ADD_NODE(&ksp->ks_sysctl_ctx, SYSCTL_CHILDREN(root), + OID_AUTO, class, CTLFLAG_RW, 0, ""); + if (root == NULL) { + printf("%s: Cannot create kstat.%s.%s tree!\n", __func__, + module, class); + sysctl_ctx_free(&ksp->ks_sysctl_ctx); + free(ksp, M_KSTAT); + return (NULL); + } + root = SYSCTL_ADD_NODE(&ksp->ks_sysctl_ctx, SYSCTL_CHILDREN(root), + OID_AUTO, name, CTLFLAG_RW, 0, ""); + if (root == NULL) { + printf("%s: Cannot create kstat.%s.%s.%s tree!\n", __func__, + module, class, name); + sysctl_ctx_free(&ksp->ks_sysctl_ctx); + free(ksp, M_KSTAT); + return (NULL); + } + ksp->ks_sysctl_root = root; + + return (ksp); +} + +static int +kstat_sysctl(SYSCTL_HANDLER_ARGS) +{ + kstat_named_t *ksent = arg1; + uint64_t val; + + val = ksent->value.ui64; + return (sysctl_handle_64(oidp, &val, 0, req)); +} + +void +kstat_install(kstat_t *ksp) +{ + kstat_named_t *ksent; + char *namelast; + int typelast; + + ksent = ksp->ks_data; + if (ksp->ks_ndata == UINT32_MAX) { +#ifdef INVARIANTS + printf("can't handle raw ops yet!!!\n"); +#endif + return; + } + if (ksent == NULL) { + printf("%s ksp->ks_data == NULL!!!!\n", __func__); + return; + } + typelast = 0; + namelast = NULL; + for (int i = 0; i < ksp->ks_ndata; i++, ksent++) { + if (ksent->data_type != 0) { + typelast = ksent->data_type; + namelast = ksent->name; + } + switch (typelast) { + case KSTAT_DATA_INT32: + SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx, + SYSCTL_CHILDREN(ksp->ks_sysctl_root), + OID_AUTO, namelast, + CTLTYPE_S32 | CTLFLAG_RD, ksent, + sizeof (*ksent), kstat_sysctl, "I", + namelast); + break; + case KSTAT_DATA_UINT32: + SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx, + SYSCTL_CHILDREN(ksp->ks_sysctl_root), + OID_AUTO, namelast, + CTLTYPE_U32 | CTLFLAG_RD, ksent, + sizeof (*ksent), kstat_sysctl, "IU", + namelast); + break; + case KSTAT_DATA_INT64: + SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx, + SYSCTL_CHILDREN(ksp->ks_sysctl_root), + OID_AUTO, namelast, + CTLTYPE_S64 | CTLFLAG_RD, ksent, + sizeof (*ksent), kstat_sysctl, "Q", + namelast); + break; + case KSTAT_DATA_UINT64: + SYSCTL_ADD_PROC(&ksp->ks_sysctl_ctx, + SYSCTL_CHILDREN(ksp->ks_sysctl_root), + OID_AUTO, namelast, + CTLTYPE_U64 | CTLFLAG_RD, ksent, + sizeof (*ksent), kstat_sysctl, "QU", + namelast); + break; + default: + panic("unsupported type: %d", typelast); + } + + } +} + +void +kstat_delete(kstat_t *ksp) +{ + + sysctl_ctx_free(&ksp->ks_sysctl_ctx); + free(ksp, M_KSTAT); +} + +void +kstat_set_string(char *dst, const char *src) +{ + + bzero(dst, KSTAT_STRLEN); + (void) strncpy(dst, src, KSTAT_STRLEN - 1); +} + +void +kstat_named_init(kstat_named_t *knp, const char *name, uchar_t data_type) +{ + + kstat_set_string(knp->name, name); + knp->data_type = data_type; +} + +void +kstat_waitq_enter(kstat_io_t *kiop) +{ + hrtime_t new, delta; + ulong_t wcnt; + + new = gethrtime(); + delta = new - kiop->wlastupdate; + kiop->wlastupdate = new; + wcnt = kiop->wcnt++; + if (wcnt != 0) { + kiop->wlentime += delta * wcnt; + kiop->wtime += delta; + } +} + +void +kstat_waitq_exit(kstat_io_t *kiop) +{ + hrtime_t new, delta; + ulong_t wcnt; + + new = gethrtime(); + delta = new - kiop->wlastupdate; + kiop->wlastupdate = new; + wcnt = kiop->wcnt--; + ASSERT((int)wcnt > 0); + kiop->wlentime += delta * wcnt; + kiop->wtime += delta; +} + +void +kstat_runq_enter(kstat_io_t *kiop) +{ + hrtime_t new, delta; + ulong_t rcnt; + + new = gethrtime(); + delta = new - kiop->rlastupdate; + kiop->rlastupdate = new; + rcnt = kiop->rcnt++; + if (rcnt != 0) { + kiop->rlentime += delta * rcnt; + kiop->rtime += delta; + } +} + +void +kstat_runq_exit(kstat_io_t *kiop) +{ + hrtime_t new, delta; + ulong_t rcnt; + + new = gethrtime(); + delta = new - kiop->rlastupdate; + kiop->rlastupdate = new; + rcnt = kiop->rcnt--; + ASSERT((int)rcnt > 0); + kiop->rlentime += delta * rcnt; + kiop->rtime += delta; +} diff --git a/module/os/freebsd/spl/spl_misc.c b/module/os/freebsd/spl/spl_misc.c new file mode 100644 index 00000000000..ab4702574b0 --- /dev/null +++ b/module/os/freebsd/spl/spl_misc.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include + +char hw_serial[11] = "0"; + +static struct opensolaris_utsname hw_utsname = { + .machine = MACHINE +}; + +static void +opensolaris_utsname_init(void *arg) +{ + + hw_utsname.sysname = ostype; + hw_utsname.nodename = prison0.pr_hostname; + hw_utsname.release = osrelease; + snprintf(hw_utsname.version, sizeof (hw_utsname.version), + "%d", osreldate); +} + +char * +kmem_strdup(const char *s) +{ + char *buf; + + buf = kmem_alloc(strlen(s) + 1, KM_SLEEP); + strcpy(buf, s); + return (buf); +} + +int +ddi_copyin(const void *from, void *to, size_t len, int flags) +{ + /* Fake ioctl() issued by kernel, 'from' is a kernel address */ + if (flags & FKIOCTL) { + memcpy(to, from, len); + return (0); + } + + return (copyin(from, to, len)); +} + +int +ddi_copyout(const void *from, void *to, size_t len, int flags) +{ + /* Fake ioctl() issued by kernel, 'from' is a kernel address */ + if (flags & FKIOCTL) { + memcpy(to, from, len); + return (0); + } + + return (copyout(from, to, len)); +} + +int +spl_panic(const char *file, const char *func, int line, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vpanic(fmt, ap); + va_end(ap); +} + +utsname_t * +utsname(void) +{ + return (&hw_utsname); +} +SYSINIT(opensolaris_utsname_init, SI_SUB_TUNABLES, SI_ORDER_ANY, + opensolaris_utsname_init, NULL); diff --git a/module/os/freebsd/spl/spl_policy.c b/module/os/freebsd/spl/spl_policy.c new file mode 100644 index 00000000000..53732836ff9 --- /dev/null +++ b/module/os/freebsd/spl/spl_policy.c @@ -0,0 +1,429 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +int +secpolicy_nfs(cred_t *cr) +{ + + return (spl_priv_check_cred(cr, PRIV_NFS_DAEMON)); +} + +int +secpolicy_zfs(cred_t *cr) +{ + + return (spl_priv_check_cred(cr, PRIV_VFS_MOUNT)); +} + +int +secpolicy_sys_config(cred_t *cr, int checkonly __unused) +{ + + return (spl_priv_check_cred(cr, PRIV_ZFS_POOL_CONFIG)); +} + +int +secpolicy_zinject(cred_t *cr) +{ + + return (spl_priv_check_cred(cr, PRIV_ZFS_INJECT)); +} + +int +secpolicy_fs_unmount(cred_t *cr, struct mount *vfsp __unused) +{ + + return (spl_priv_check_cred(cr, PRIV_VFS_UNMOUNT)); +} + +int +secpolicy_fs_owner(struct mount *mp, cred_t *cr) +{ + + if (zfs_super_owner) { + if (cr->cr_uid == mp->mnt_cred->cr_uid && + cr->cr_prison == mp->mnt_cred->cr_prison) { + return (0); + } + } + return (EPERM); +} + +/* + * This check is done in kern_link(), so we could just return 0 here. + */ +extern int hardlink_check_uid; +int +secpolicy_basic_link(vnode_t *vp, cred_t *cr) +{ + + if (!hardlink_check_uid) + return (0); + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + return (spl_priv_check_cred(cr, PRIV_VFS_LINK)); +} + +int +secpolicy_vnode_stky_modify(cred_t *cr) +{ + + return (EPERM); +} + +int +secpolicy_vnode_remove(vnode_t *vp, cred_t *cr) +{ + + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + return (spl_priv_check_cred(cr, PRIV_VFS_ADMIN)); +} + +int +secpolicy_vnode_access(cred_t *cr, vnode_t *vp, uid_t owner, accmode_t accmode) +{ + + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + + if ((accmode & VREAD) && spl_priv_check_cred(cr, PRIV_VFS_READ) != 0) + return (EACCES); + if ((accmode & VWRITE) && + spl_priv_check_cred(cr, PRIV_VFS_WRITE) != 0) { + return (EACCES); + } + if (accmode & VEXEC) { + if (vp->v_type == VDIR) { + if (spl_priv_check_cred(cr, PRIV_VFS_LOOKUP) != 0) + return (EACCES); + } else { + if (spl_priv_check_cred(cr, PRIV_VFS_EXEC) != 0) + return (EACCES); + } + } + return (0); +} + +/* + * Like secpolicy_vnode_access() but we get the actual wanted mode and the + * current mode of the file, not the missing bits. + */ +int +secpolicy_vnode_access2(cred_t *cr, vnode_t *vp, uid_t owner, + accmode_t curmode, accmode_t wantmode) +{ + accmode_t mode; + + mode = ~curmode & wantmode; + + if (mode == 0) + return (0); + + return (secpolicy_vnode_access(cr, vp, owner, mode)); +} + +int +secpolicy_vnode_any_access(cred_t *cr, vnode_t *vp, uid_t owner) +{ + static int privs[] = { + PRIV_VFS_ADMIN, + PRIV_VFS_READ, + PRIV_VFS_WRITE, + PRIV_VFS_EXEC, + PRIV_VFS_LOOKUP + }; + int i; + + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + + /* Same as secpolicy_vnode_setdac */ + if (owner == cr->cr_uid) + return (0); + + for (i = 0; i < sizeof (privs)/sizeof (int); i++) { + int priv; + + switch (priv = privs[i]) { + case PRIV_VFS_EXEC: + if (vp->v_type == VDIR) + continue; + break; + case PRIV_VFS_LOOKUP: + if (vp->v_type != VDIR) + continue; + break; + } + if (spl_priv_check_cred(cr, priv) == 0) + return (0); + } + return (EPERM); +} + +int +secpolicy_vnode_setdac(vnode_t *vp, cred_t *cr, uid_t owner) +{ + + if (owner == cr->cr_uid) + return (0); + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + return (spl_priv_check_cred(cr, PRIV_VFS_ADMIN)); +} + +int +secpolicy_vnode_setattr(cred_t *cr, vnode_t *vp, struct vattr *vap, + const struct vattr *ovap, int flags, + int unlocked_access(void *, int, cred_t *), void *node) +{ + int mask = vap->va_mask; + int error; + + if (mask & AT_SIZE) { + if (vp->v_type == VDIR) + return (EISDIR); + error = unlocked_access(node, VWRITE, cr); + if (error) + return (error); + } + if (mask & AT_MODE) { + /* + * If not the owner of the file then check privilege + * for two things: the privilege to set the mode at all + * and, if we're setting setuid, we also need permissions + * to add the set-uid bit, if we're not the owner. + * In the specific case of creating a set-uid root + * file, we need even more permissions. + */ + error = secpolicy_vnode_setdac(vp, cr, ovap->va_uid); + if (error) + return (error); + error = secpolicy_setid_setsticky_clear(vp, vap, ovap, cr); + if (error) + return (error); + } else { + vap->va_mode = ovap->va_mode; + } + if (mask & (AT_UID | AT_GID)) { + error = secpolicy_vnode_setdac(vp, cr, ovap->va_uid); + if (error) + return (error); + + /* + * To change the owner of a file, or change the group of + * a file to a group of which we are not a member, the + * caller must have privilege. + */ + if (((mask & AT_UID) && vap->va_uid != ovap->va_uid) || + ((mask & AT_GID) && vap->va_gid != ovap->va_gid && + !groupmember(vap->va_gid, cr))) { + if (secpolicy_fs_owner(vp->v_mount, cr) != 0) { + error = spl_priv_check_cred(cr, PRIV_VFS_CHOWN); + if (error) + return (error); + } + } + + if (((mask & AT_UID) && vap->va_uid != ovap->va_uid) || + ((mask & AT_GID) && vap->va_gid != ovap->va_gid)) { + secpolicy_setid_clear(vap, vp, cr); + } + } + if (mask & (AT_ATIME | AT_MTIME)) { + /* + * From utimes(2): + * If times is NULL, ... The caller must be the owner of + * the file, have permission to write the file, or be the + * super-user. + * If times is non-NULL, ... The caller must be the owner of + * the file or be the super-user. + */ + error = secpolicy_vnode_setdac(vp, cr, ovap->va_uid); + if (error && (vap->va_vaflags & VA_UTIMES_NULL)) + error = unlocked_access(node, VWRITE, cr); + if (error) + return (error); + } + return (0); +} + +int +secpolicy_vnode_create_gid(cred_t *cr) +{ + + return (EPERM); +} + +int +secpolicy_vnode_setids_setgids(vnode_t *vp, cred_t *cr, gid_t gid) +{ + + if (groupmember(gid, cr)) + return (0); + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + return (spl_priv_check_cred(cr, PRIV_VFS_SETGID)); +} + +int +secpolicy_vnode_setid_retain(vnode_t *vp, cred_t *cr, + boolean_t issuidroot __unused) +{ + + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + return (spl_priv_check_cred(cr, PRIV_VFS_RETAINSUGID)); +} + +void +secpolicy_setid_clear(struct vattr *vap, vnode_t *vp, cred_t *cr) +{ + + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return; + + if ((vap->va_mode & (S_ISUID | S_ISGID)) != 0) { + if (spl_priv_check_cred(cr, PRIV_VFS_RETAINSUGID)) { + vap->va_mask |= AT_MODE; + vap->va_mode &= ~(S_ISUID|S_ISGID); + } + } +} + +int +secpolicy_setid_setsticky_clear(vnode_t *vp, struct vattr *vap, + const struct vattr *ovap, cred_t *cr) +{ + int error; + + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + + /* + * Privileged processes may set the sticky bit on non-directories, + * as well as set the setgid bit on a file with a group that the process + * is not a member of. Both of these are allowed in jail(8). + */ + if (vp->v_type != VDIR && (vap->va_mode & S_ISTXT)) { + if (spl_priv_check_cred(cr, PRIV_VFS_STICKYFILE)) + return (EFTYPE); + } + /* + * Check for privilege if attempting to set the + * group-id bit. + */ + if ((vap->va_mode & S_ISGID) != 0) { + error = secpolicy_vnode_setids_setgids(vp, cr, ovap->va_gid); + if (error) + return (error); + } + /* + * Deny setting setuid if we are not the file owner. + */ + if ((vap->va_mode & S_ISUID) && ovap->va_uid != cr->cr_uid) { + error = spl_priv_check_cred(cr, PRIV_VFS_ADMIN); + if (error) + return (error); + } + return (0); +} + +int +secpolicy_fs_mount(cred_t *cr, vnode_t *mvp, struct mount *vfsp) +{ + + return (spl_priv_check_cred(cr, PRIV_VFS_MOUNT)); +} + +int +secpolicy_vnode_owner(vnode_t *vp, cred_t *cr, uid_t owner) +{ + + if (owner == cr->cr_uid) + return (0); + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + + /* XXX: vfs_suser()? */ + return (spl_priv_check_cred(cr, PRIV_VFS_MOUNT_OWNER)); +} + +int +secpolicy_vnode_chown(vnode_t *vp, cred_t *cr, uid_t owner) +{ + + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + return (spl_priv_check_cred(cr, PRIV_VFS_CHOWN)); +} + +void +secpolicy_fs_mount_clearopts(cred_t *cr, struct mount *vfsp) +{ + + if (spl_priv_check_cred(cr, PRIV_VFS_MOUNT_NONUSER) != 0) { + MNT_ILOCK(vfsp); + vfsp->vfs_flag |= VFS_NOSETUID | MNT_USER; + vfs_clearmntopt(vfsp, MNTOPT_SETUID); + vfs_setmntopt(vfsp, MNTOPT_NOSETUID, NULL, 0); + MNT_IUNLOCK(vfsp); + } +} + +/* + * Check privileges for setting xvattr attributes + */ +int +secpolicy_xvattr(vnode_t *vp, xvattr_t *xvap, uid_t owner, cred_t *cr, + vtype_t vtype) +{ + + if (secpolicy_fs_owner(vp->v_mount, cr) == 0) + return (0); + return (spl_priv_check_cred(cr, PRIV_VFS_SYSFLAGS)); +} + +int +secpolicy_smb(cred_t *cr) +{ + + return (spl_priv_check_cred(cr, PRIV_NETSMB)); +} diff --git a/module/os/freebsd/spl/spl_procfs_list.c b/module/os/freebsd/spl/spl_procfs_list.c new file mode 100644 index 00000000000..7b4ae9d0e35 --- /dev/null +++ b/module/os/freebsd/spl/spl_procfs_list.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +void +seq_printf(struct seq_file *m, const char *fmt, ...) +{} + +void +procfs_list_install(const char *module, + const char *name, + mode_t mode, + procfs_list_t *procfs_list, + int (*show)(struct seq_file *f, void *p), + int (*show_header)(struct seq_file *f), + int (*clear)(procfs_list_t *procfs_list), + size_t procfs_list_node_off) +{ + mutex_init(&procfs_list->pl_lock, NULL, MUTEX_DEFAULT, NULL); + list_create(&procfs_list->pl_list, + procfs_list_node_off + sizeof (procfs_list_node_t), + procfs_list_node_off + offsetof(procfs_list_node_t, pln_link)); + procfs_list->pl_next_id = 1; + procfs_list->pl_node_offset = procfs_list_node_off; +} + +void +procfs_list_uninstall(procfs_list_t *procfs_list) +{} + +void +procfs_list_destroy(procfs_list_t *procfs_list) +{ + ASSERT(list_is_empty(&procfs_list->pl_list)); + list_destroy(&procfs_list->pl_list); + mutex_destroy(&procfs_list->pl_lock); +} + +#define NODE_ID(procfs_list, obj) \ + (((procfs_list_node_t *)(((char *)obj) + \ + (procfs_list)->pl_node_offset))->pln_id) + +void +procfs_list_add(procfs_list_t *procfs_list, void *p) +{ + ASSERT(MUTEX_HELD(&procfs_list->pl_lock)); + NODE_ID(procfs_list, p) = procfs_list->pl_next_id++; + list_insert_tail(&procfs_list->pl_list, p); +} diff --git a/module/os/freebsd/spl/spl_string.c b/module/os/freebsd/spl/spl_string.c new file mode 100644 index 00000000000..14d816b5c32 --- /dev/null +++ b/module/os/freebsd/spl/spl_string.c @@ -0,0 +1,106 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include + +#define IS_DIGIT(c) ((c) >= '0' && (c) <= '9') + +#define IS_ALPHA(c) \ + (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z')) + +char * +strpbrk(const char *s, const char *b) +{ + const char *p; + + do { + for (p = b; *p != '\0' && *p != *s; ++p) + ; + if (*p != '\0') + return ((char *)s); + } while (*s++); + + return (NULL); +} + +/* + * Convert a string into a valid C identifier by replacing invalid + * characters with '_'. Also makes sure the string is nul-terminated + * and takes up at most n bytes. + */ +void +strident_canon(char *s, size_t n) +{ + char c; + char *end = s + n - 1; + + if ((c = *s) == 0) + return; + + if (!IS_ALPHA(c) && c != '_') + *s = '_'; + + while (s < end && ((c = *(++s)) != 0)) { + if (!IS_ALPHA(c) && !IS_DIGIT(c) && c != '_') + *s = '_'; + } + *s = 0; +} + +/* + * Do not change the length of the returned string; it must be freed + * with strfree(). + */ +char * +kmem_asprintf(const char *fmt, ...) +{ + int size; + va_list adx; + char *buf; + + va_start(adx, fmt); + size = vsnprintf(NULL, 0, fmt, adx) + 1; + va_end(adx); + + buf = kmem_alloc(size, KM_SLEEP); + + va_start(adx, fmt); + (void) vsnprintf(buf, size, fmt, adx); + va_end(adx); + + return (buf); +} + +void +kmem_strfree(char *str) +{ + ASSERT(str != NULL); + kmem_free(str, strlen(str) + 1); +} diff --git a/module/os/freebsd/spl/spl_sunddi.c b/module/os/freebsd/spl/spl_sunddi.c new file mode 100644 index 00000000000..1fa4f56f1f8 --- /dev/null +++ b/module/os/freebsd/spl/spl_sunddi.c @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2010 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include + +int +ddi_strtol(const char *str, char **nptr, int base, long *result) +{ + + *result = strtol(str, nptr, base); + return (0); +} + +int +ddi_strtoul(const char *str, char **nptr, int base, unsigned long *result) +{ + + if (str == hw_serial) { + *result = prison0.pr_hostid; + return (0); + } + + *result = strtoul(str, nptr, base); + return (0); +} + +int +ddi_strtoull(const char *str, char **nptr, int base, unsigned long long *result) +{ + + *result = (unsigned long long)strtouq(str, nptr, base); + return (0); +} + +int +ddi_strtoll(const char *str, char **nptr, int base, long long *result) +{ + + *result = (long long)strtoq(str, nptr, base); + return (0); +} diff --git a/module/os/freebsd/spl/spl_sysevent.c b/module/os/freebsd/spl/spl_sysevent.c new file mode 100644 index 00000000000..d3748276a71 --- /dev/null +++ b/module/os/freebsd/spl/spl_sysevent.c @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2010 Pawel Jakub Dawidek + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int +log_sysevent(nvlist_t *event) +{ + struct sbuf *sb; + const char *type; + char typestr[128]; + nvpair_t *elem = NULL; + + sb = sbuf_new_auto(); + if (sb == NULL) + return (ENOMEM); + type = NULL; + + while ((elem = nvlist_next_nvpair(event, elem)) != NULL) { + switch (nvpair_type(elem)) { + case DATA_TYPE_BOOLEAN: + { + boolean_t value; + + (void) nvpair_value_boolean_value(elem, &value); + sbuf_printf(sb, " %s=%s", nvpair_name(elem), + value ? "true" : "false"); + break; + } + case DATA_TYPE_UINT8: + { + uint8_t value; + + (void) nvpair_value_uint8(elem, &value); + sbuf_printf(sb, " %s=%hhu", nvpair_name(elem), value); + break; + } + case DATA_TYPE_INT32: + { + int32_t value; + + (void) nvpair_value_int32(elem, &value); + sbuf_printf(sb, " %s=%jd", nvpair_name(elem), + (intmax_t)value); + break; + } + case DATA_TYPE_UINT32: + { + uint32_t value; + + (void) nvpair_value_uint32(elem, &value); + sbuf_printf(sb, " %s=%ju", nvpair_name(elem), + (uintmax_t)value); + break; + } + case DATA_TYPE_INT64: + { + int64_t value; + + (void) nvpair_value_int64(elem, &value); + sbuf_printf(sb, " %s=%jd", nvpair_name(elem), + (intmax_t)value); + break; + } + case DATA_TYPE_UINT64: + { + uint64_t value; + + (void) nvpair_value_uint64(elem, &value); + sbuf_printf(sb, " %s=%ju", nvpair_name(elem), + (uintmax_t)value); + break; + } + case DATA_TYPE_STRING: + { + char *value; + + (void) nvpair_value_string(elem, &value); + sbuf_printf(sb, " %s=%s", nvpair_name(elem), value); + if (strcmp(FM_CLASS, nvpair_name(elem)) == 0) + type = value; + break; + } + case DATA_TYPE_UINT8_ARRAY: + { + uint8_t *value; + uint_t ii, nelem; + + (void) nvpair_value_uint8_array(elem, &value, &nelem); + sbuf_printf(sb, " %s=", nvpair_name(elem)); + for (ii = 0; ii < nelem; ii++) + sbuf_printf(sb, "%02hhx", value[ii]); + break; + } + case DATA_TYPE_UINT16_ARRAY: + { + uint16_t *value; + uint_t ii, nelem; + + (void) nvpair_value_uint16_array(elem, &value, &nelem); + sbuf_printf(sb, " %s=", nvpair_name(elem)); + for (ii = 0; ii < nelem; ii++) + sbuf_printf(sb, "%04hx", value[ii]); + break; + } + case DATA_TYPE_UINT32_ARRAY: + { + uint32_t *value; + uint_t ii, nelem; + + (void) nvpair_value_uint32_array(elem, &value, &nelem); + sbuf_printf(sb, " %s=", nvpair_name(elem)); + for (ii = 0; ii < nelem; ii++) + sbuf_printf(sb, "%08jx", (uintmax_t)value[ii]); + break; + } + case DATA_TYPE_INT64_ARRAY: + { + int64_t *value; + uint_t ii, nelem; + + (void) nvpair_value_int64_array(elem, &value, &nelem); + sbuf_printf(sb, " %s=", nvpair_name(elem)); + for (ii = 0; ii < nelem; ii++) + sbuf_printf(sb, "%016lld", + (long long)value[ii]); + break; + } + case DATA_TYPE_UINT64_ARRAY: + { + uint64_t *value; + uint_t ii, nelem; + + (void) nvpair_value_uint64_array(elem, &value, &nelem); + sbuf_printf(sb, " %s=", nvpair_name(elem)); + for (ii = 0; ii < nelem; ii++) + sbuf_printf(sb, "%016jx", (uintmax_t)value[ii]); + break; + } + case DATA_TYPE_STRING_ARRAY: + { + char **strarr; + uint_t ii, nelem; + + (void) nvpair_value_string_array(elem, &strarr, &nelem); + + for (ii = 0; ii < nelem; ii++) { + if (strarr[ii] == NULL) { + sbuf_printf(sb, " "); + continue; + } + + sbuf_printf(sb, " %s", strarr[ii]); + if (strcmp(FM_CLASS, strarr[ii]) == 0) + type = strarr[ii]; + } + break; + } + case DATA_TYPE_NVLIST: + /* XXX - requires recursing in log_sysevent */ + break; + default: + printf("%s: type %d is not implemented\n", __func__, + nvpair_type(elem)); + break; + } + } + + if (sbuf_finish(sb) != 0) { + sbuf_delete(sb); + return (ENOMEM); + } + + if (type == NULL) + type = ""; + if (strncmp(type, "ESC_ZFS_", 8) == 0) { + snprintf(typestr, sizeof (typestr), "misc.fs.zfs.%s", type + 8); + type = typestr; + } + devctl_notify("ZFS", "ZFS", type, sbuf_data(sb)); + sbuf_delete(sb); + + return (0); +} + +static void +sysevent_worker(void *arg __unused) +{ + zfs_zevent_t *ze; + nvlist_t *event; + uint64_t dropped = 0; + uint64_t dst_size; + int error; + + zfs_zevent_init(&ze); + for (;;) { + dst_size = 131072; + dropped = 0; + event = NULL; + error = zfs_zevent_next(ze, &event, + &dst_size, &dropped); + if (error) { + error = zfs_zevent_wait(ze); + if (error == ESHUTDOWN) + break; + } else { + VERIFY(event != NULL); + log_sysevent(event); + nvlist_free(event); + } + } + zfs_zevent_destroy(ze); + kthread_exit(); +} + +void +ddi_sysevent_init(void) +{ + kproc_kthread_add(sysevent_worker, NULL, &zfsproc, NULL, 0, 0, + "zfskern", "sysevent"); +} diff --git a/module/os/freebsd/spl/spl_taskq.c b/module/os/freebsd/spl/spl_taskq.c new file mode 100644 index 00000000000..b6a501f6773 --- /dev/null +++ b/module/os/freebsd/spl/spl_taskq.c @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2009 Pawel Jakub Dawidek + * All rights reserved. + * + * Copyright (c) 2012 Spectra Logic Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static uint_t taskq_tsd; +static uma_zone_t taskq_zone; + +taskq_t *system_taskq = NULL; +taskq_t *system_delay_taskq = NULL; +taskq_t *dynamic_taskq = NULL; + +extern int uma_align_cache; + +#define TQ_MASK uma_align_cache +#define TQ_PTR_MASK ~uma_align_cache + +#define TIMEOUT_TASK 1 +#define NORMAL_TASK 2 + +static int +taskqent_init(void *mem, int size, int flags) +{ + bzero(mem, sizeof (taskq_ent_t)); + return (0); +} + +static int +taskqent_ctor(void *mem, int size, void *arg, int flags) +{ + return (0); +} + +static void +taskqent_dtor(void *mem, int size, void *arg) +{ + taskq_ent_t *ent = mem; + + ent->tqent_gen = (ent->tqent_gen + 1) & TQ_MASK; +} + +static void +system_taskq_init(void *arg) +{ + + tsd_create(&taskq_tsd, NULL); + taskq_zone = uma_zcreate("taskq_zone", sizeof (taskq_ent_t), + taskqent_ctor, taskqent_dtor, taskqent_init, NULL, + UMA_ALIGN_CACHE, UMA_ZONE_NOFREE); + system_taskq = taskq_create("system_taskq", mp_ncpus, minclsyspri, + 0, 0, 0); + system_delay_taskq = taskq_create("system_delay_taskq", mp_ncpus, + minclsyspri, 0, 0, 0); +} +SYSINIT(system_taskq_init, SI_SUB_CONFIGURE, SI_ORDER_ANY, system_taskq_init, + NULL); + +static void +system_taskq_fini(void *arg) +{ + + taskq_destroy(system_taskq); + uma_zdestroy(taskq_zone); + tsd_destroy(&taskq_tsd); +} +SYSUNINIT(system_taskq_fini, SI_SUB_CONFIGURE, SI_ORDER_ANY, system_taskq_fini, + NULL); + +static void +taskq_tsd_set(void *context) +{ + taskq_t *tq = context; + + tsd_set(taskq_tsd, tq); +} + +static taskq_t * +taskq_create_with_init(const char *name, int nthreads, pri_t pri, + int minalloc __unused, int maxalloc __unused, uint_t flags) +{ + taskq_t *tq; + + if ((flags & TASKQ_THREADS_CPU_PCT) != 0) + nthreads = MAX((mp_ncpus * nthreads) / 100, 1); + + tq = kmem_alloc(sizeof (*tq), KM_SLEEP); + tq->tq_queue = taskqueue_create(name, M_WAITOK, + taskqueue_thread_enqueue, &tq->tq_queue); + taskqueue_set_callback(tq->tq_queue, TASKQUEUE_CALLBACK_TYPE_INIT, + taskq_tsd_set, tq); + taskqueue_set_callback(tq->tq_queue, TASKQUEUE_CALLBACK_TYPE_SHUTDOWN, + taskq_tsd_set, NULL); + (void) taskqueue_start_threads(&tq->tq_queue, nthreads, pri, + "%s", name); + + return ((taskq_t *)tq); +} + +taskq_t * +taskq_create(const char *name, int nthreads, pri_t pri, int minalloc __unused, + int maxalloc __unused, uint_t flags) +{ + + return (taskq_create_with_init(name, nthreads, pri, minalloc, maxalloc, + flags)); +} + +taskq_t * +taskq_create_proc(const char *name, int nthreads, pri_t pri, int minalloc, + int maxalloc, proc_t *proc __unused, uint_t flags) +{ + + return (taskq_create_with_init(name, nthreads, pri, minalloc, maxalloc, + flags)); +} + +void +taskq_destroy(taskq_t *tq) +{ + + taskqueue_free(tq->tq_queue); + kmem_free(tq, sizeof (*tq)); +} + +int +taskq_member(taskq_t *tq, kthread_t *thread) +{ + + return (taskqueue_member(tq->tq_queue, thread)); +} + +taskq_t * +taskq_of_curthread(void) +{ + return (tsd_get(taskq_tsd)); +} + +int +taskq_cancel_id(taskq_t *tq, taskqid_t tid) +{ + uint32_t pend; + int rc; + taskq_ent_t *ent = (void*)(tid & TQ_PTR_MASK); + + if (ent == NULL) + return (0); + if ((tid & TQ_MASK) != ent->tqent_gen) + return (0); + if (ent->tqent_type == TIMEOUT_TASK) { + rc = taskqueue_cancel_timeout(tq->tq_queue, + &ent->tqent_timeout_task, &pend); + } else + rc = taskqueue_cancel(tq->tq_queue, &ent->tqent_task, &pend); + if (rc == EBUSY) + taskq_wait_id(tq, tid); + else + uma_zfree(taskq_zone, ent); + return (rc); +} + +static void +taskq_run(void *arg, int pending __unused) +{ + taskq_ent_t *task = arg; + + task->tqent_func(task->tqent_arg); + uma_zfree(taskq_zone, task); +} + +taskqid_t +taskq_dispatch_delay(taskq_t *tq, task_func_t func, void *arg, + uint_t flags, clock_t expire_time) +{ + taskq_ent_t *task; + taskqid_t tid; + clock_t timo; + int mflag; + + timo = expire_time - ddi_get_lbolt(); + if (timo <= 0) + return (taskq_dispatch(tq, func, arg, flags)); + + if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP) + mflag = M_WAITOK; + else + mflag = M_NOWAIT; + + task = uma_zalloc(taskq_zone, mflag); + if (task == NULL) + return (0); + tid = (uintptr_t)task; + MPASS((tid & TQ_MASK) == 0); + task->tqent_func = func; + task->tqent_arg = arg; + task->tqent_type = TIMEOUT_TASK; + tid |= task->tqent_gen; + TIMEOUT_TASK_INIT(tq->tq_queue, &task->tqent_timeout_task, 0, + taskq_run, task); + + taskqueue_enqueue_timeout(tq->tq_queue, &task->tqent_timeout_task, + timo); + return (tid); +} + +taskqid_t +taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags) +{ + taskq_ent_t *task; + int mflag, prio; + taskqid_t tid; + + if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP) + mflag = M_WAITOK; + else + mflag = M_NOWAIT; + /* + * If TQ_FRONT is given, we want higher priority for this task, so it + * can go at the front of the queue. + */ + prio = !!(flags & TQ_FRONT); + + task = uma_zalloc(taskq_zone, mflag); + if (task == NULL) + return (0); + + tid = (uintptr_t)task; + MPASS((tid & TQ_MASK) == 0); + task->tqent_func = func; + task->tqent_arg = arg; + task->tqent_type = NORMAL_TASK; + TASK_INIT(&task->tqent_task, prio, taskq_run, task); + tid |= task->tqent_gen; + taskqueue_enqueue(tq->tq_queue, &task->tqent_task); + return (tid); +} + +static void +taskq_run_ent(void *arg, int pending __unused) +{ + taskq_ent_t *task = arg; + + task->tqent_func(task->tqent_arg); +} + +void +taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint32_t flags, + taskq_ent_t *task) +{ + int prio; + + /* + * If TQ_FRONT is given, we want higher priority for this task, so it + * can go at the front of the queue. + */ + prio = !!(flags & TQ_FRONT); + + task->tqent_func = func; + task->tqent_arg = arg; + + TASK_INIT(&task->tqent_task, prio, taskq_run_ent, task); + taskqueue_enqueue(tq->tq_queue, &task->tqent_task); +} + +void +taskq_wait(taskq_t *tq) +{ + taskqueue_quiesce(tq->tq_queue); +} + +void +taskq_wait_id(taskq_t *tq, taskqid_t tid) +{ + taskq_ent_t *ent = (void*)(tid & TQ_PTR_MASK); + + if ((tid & TQ_MASK) != ent->tqent_gen) + return; + + taskqueue_drain(tq->tq_queue, &ent->tqent_task); +} + +void +taskq_wait_outstanding(taskq_t *tq, taskqid_t id __unused) +{ + taskqueue_drain_all(tq->tq_queue); +} + +int +taskq_empty_ent(taskq_ent_t *t) +{ + return (t->tqent_task.ta_pending == 0); +} diff --git a/module/os/freebsd/spl/spl_uio.c b/module/os/freebsd/spl/spl_uio.c new file mode 100644 index 00000000000..05dbfd06d79 --- /dev/null +++ b/module/os/freebsd/spl/spl_uio.c @@ -0,0 +1,92 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +/* + * University Copyright- Copyright (c) 1982, 1986, 1988 + * The Regents of the University of California + * All Rights Reserved + * + * University Acknowledgment- Portions of this document are derived from + * software developed by the University of California, Berkeley, and its + * contributors. + */ + +/* + * $FreeBSD$ + */ + +#include +#include +#include + +/* + * same as uiomove() but doesn't modify uio structure. + * return in cbytes how many bytes were copied. + */ +int +uiocopy(void *p, size_t n, enum uio_rw rw, struct uio *uio, size_t *cbytes) +{ + struct iovec small_iovec[1]; + struct uio small_uio_clone; + struct uio *uio_clone; + int error; + + ASSERT3U(uio->uio_rw, ==, rw); + if (uio->uio_iovcnt == 1) { + small_uio_clone = *uio; + small_iovec[0] = *uio->uio_iov; + small_uio_clone.uio_iov = small_iovec; + uio_clone = &small_uio_clone; + } else { + uio_clone = cloneuio(uio); + } + + error = vn_io_fault_uiomove(p, n, uio_clone); + *cbytes = uio->uio_resid - uio_clone->uio_resid; + if (uio_clone != &small_uio_clone) + free(uio_clone, M_IOV); + return (error); +} + +/* + * Drop the next n chars out of *uiop. + */ +void +uioskip(uio_t *uio, size_t n) +{ + enum uio_seg segflg; + + /* For the full compatibility with illumos. */ + if (n > uio->uio_resid) + return; + + segflg = uio->uio_segflg; + uio->uio_segflg = UIO_NOCOPY; + uiomove(NULL, n, uio->uio_rw, uio); + uio->uio_segflg = segflg; +} diff --git a/module/os/freebsd/spl/spl_vfs.c b/module/os/freebsd/spl/spl_vfs.c new file mode 100644 index 00000000000..99da8c97653 --- /dev/null +++ b/module/os/freebsd/spl/spl_vfs.c @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2006-2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +MALLOC_DECLARE(M_MOUNT); + +void +vfs_setmntopt(vfs_t *vfsp, const char *name, const char *arg, + int flags __unused) +{ + struct vfsopt *opt; + size_t namesize; + int locked; + + if (!(locked = mtx_owned(MNT_MTX(vfsp)))) + MNT_ILOCK(vfsp); + + if (vfsp->mnt_opt == NULL) { + void *opts; + + MNT_IUNLOCK(vfsp); + opts = malloc(sizeof (*vfsp->mnt_opt), M_MOUNT, M_WAITOK); + MNT_ILOCK(vfsp); + if (vfsp->mnt_opt == NULL) { + vfsp->mnt_opt = opts; + TAILQ_INIT(vfsp->mnt_opt); + } else { + free(opts, M_MOUNT); + } + } + + MNT_IUNLOCK(vfsp); + + opt = malloc(sizeof (*opt), M_MOUNT, M_WAITOK); + namesize = strlen(name) + 1; + opt->name = malloc(namesize, M_MOUNT, M_WAITOK); + strlcpy(opt->name, name, namesize); + opt->pos = -1; + opt->seen = 1; + if (arg == NULL) { + opt->value = NULL; + opt->len = 0; + } else { + opt->len = strlen(arg) + 1; + opt->value = malloc(opt->len, M_MOUNT, M_WAITOK); + bcopy(arg, opt->value, opt->len); + } + + MNT_ILOCK(vfsp); + TAILQ_INSERT_TAIL(vfsp->mnt_opt, opt, link); + if (!locked) + MNT_IUNLOCK(vfsp); +} + +void +vfs_clearmntopt(vfs_t *vfsp, const char *name) +{ + int locked; + + if (!(locked = mtx_owned(MNT_MTX(vfsp)))) + MNT_ILOCK(vfsp); + vfs_deleteopt(vfsp->mnt_opt, name); + if (!locked) + MNT_IUNLOCK(vfsp); +} + +int +vfs_optionisset(const vfs_t *vfsp, const char *opt, char **argp) +{ + struct vfsoptlist *opts = vfsp->mnt_optnew; + int error; + + if (opts == NULL) + return (0); + error = vfs_getopt(opts, opt, (void **)argp, NULL); + return (error != 0 ? 0 : 1); +} + +int +mount_snapshot(kthread_t *td, vnode_t **vpp, const char *fstype, char *fspath, + char *fspec, int fsflags) +{ + struct vfsconf *vfsp; + struct mount *mp; + vnode_t *vp, *mvp; + struct ucred *cr; + int error; + + ASSERT_VOP_ELOCKED(*vpp, "mount_snapshot"); + + vp = *vpp; + *vpp = NULL; + error = 0; + + /* + * Be ultra-paranoid about making sure the type and fspath + * variables will fit in our mp buffers, including the + * terminating NUL. + */ + if (strlen(fstype) >= MFSNAMELEN || strlen(fspath) >= MNAMELEN) + error = ENAMETOOLONG; + if (error == 0 && (vfsp = vfs_byname_kld(fstype, td, &error)) == NULL) + error = ENODEV; + if (error == 0 && vp->v_type != VDIR) + error = ENOTDIR; + /* + * We need vnode lock to protect v_mountedhere and vnode interlock + * to protect v_iflag. + */ + if (error == 0) { + VI_LOCK(vp); + if ((vp->v_iflag & VI_MOUNT) == 0 && vp->v_mountedhere == NULL) + vp->v_iflag |= VI_MOUNT; + else + error = EBUSY; + VI_UNLOCK(vp); + } + if (error != 0) { + vput(vp); + return (error); + } + VOP_UNLOCK1(vp); + + /* + * Allocate and initialize the filesystem. + * We don't want regular user that triggered snapshot mount to be able + * to unmount it, so pass credentials of the parent mount. + */ + mp = vfs_mount_alloc(vp, vfsp, fspath, vp->v_mount->mnt_cred); + + mp->mnt_optnew = NULL; + vfs_setmntopt(mp, "from", fspec, 0); + mp->mnt_optnew = mp->mnt_opt; + mp->mnt_opt = NULL; + + /* + * Set the mount level flags. + */ + mp->mnt_flag = fsflags & MNT_UPDATEMASK; + /* + * Snapshots are always read-only. + */ + mp->mnt_flag |= MNT_RDONLY; + /* + * We don't want snapshots to allow access to vulnerable setuid + * programs, so we turn off setuid when mounting snapshots. + */ + mp->mnt_flag |= MNT_NOSUID; + /* + * We don't want snapshots to be visible in regular + * mount(8) and df(1) output. + */ + mp->mnt_flag |= MNT_IGNORE; + /* + * XXX: This is evil, but we can't mount a snapshot as a regular user. + * XXX: Is is safe when snapshot is mounted from within a jail? + */ + cr = td->td_ucred; + td->td_ucred = kcred; + error = VFS_MOUNT(mp); + td->td_ucred = cr; + + if (error != 0) { + /* + * Clear VI_MOUNT and decrement the use count "atomically", + * under the vnode lock. This is not strictly required, + * but makes it easier to reason about the life-cycle and + * ownership of the covered vnode. + */ + vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); + VI_LOCK(vp); + vp->v_iflag &= ~VI_MOUNT; + VI_UNLOCK(vp); + vput(vp); + vfs_unbusy(mp); + vfs_freeopts(mp->mnt_optnew); + mp->mnt_vnodecovered = NULL; + vfs_mount_destroy(mp); + return (error); + } + + if (mp->mnt_opt != NULL) + vfs_freeopts(mp->mnt_opt); + mp->mnt_opt = mp->mnt_optnew; + (void) VFS_STATFS(mp, &mp->mnt_stat); + + /* + * Prevent external consumers of mount options from reading + * mnt_optnew. + */ + mp->mnt_optnew = NULL; + + vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); +#ifdef FREEBSD_NAMECACHE + cache_purge(vp); +#endif + VI_LOCK(vp); + vp->v_iflag &= ~VI_MOUNT; + VI_UNLOCK(vp); + + vp->v_mountedhere = mp; + /* Put the new filesystem on the mount list. */ + mtx_lock(&mountlist_mtx); + TAILQ_INSERT_TAIL(&mountlist, mp, mnt_list); + mtx_unlock(&mountlist_mtx); + vfs_event_signal(NULL, VQ_MOUNT, 0); + if (VFS_ROOT(mp, LK_EXCLUSIVE, &mvp)) + panic("mount: lost mount"); + VOP_UNLOCK1(vp); +#if __FreeBSD_version >= 1300048 + vfs_op_exit(mp); +#endif + vfs_unbusy(mp); + *vpp = mvp; + return (0); +} + +/* + * Like vn_rele() except if we are going to call VOP_INACTIVE() then do it + * asynchronously using a taskq. This can avoid deadlocks caused by re-entering + * the file system as a result of releasing the vnode. Note, file systems + * already have to handle the race where the vnode is incremented before the + * inactive routine is called and does its locking. + * + * Warning: Excessive use of this routine can lead to performance problems. + * This is because taskqs throttle back allocation if too many are created. + */ +void +vn_rele_async(vnode_t *vp, taskq_t *taskq) +{ + VERIFY(vp->v_count > 0); + if (refcount_release_if_not_last(&vp->v_usecount)) { +#if __FreeBSD_version < 1300045 + vdrop(vp); +#endif + return; + } + VERIFY(taskq_dispatch((taskq_t *)taskq, + (task_func_t *)vrele, vp, TQ_SLEEP) != 0); +} diff --git a/module/os/freebsd/spl/spl_vm.c b/module/os/freebsd/spl/spl_vm.c new file mode 100644 index 00000000000..cd18ebb7adf --- /dev/null +++ b/module/os/freebsd/spl/spl_vm.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2013 EMC Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +const int zfs_vm_pagerret_bad = VM_PAGER_BAD; +const int zfs_vm_pagerret_error = VM_PAGER_ERROR; +const int zfs_vm_pagerret_ok = VM_PAGER_OK; +const int zfs_vm_pagerput_sync = VM_PAGER_PUT_SYNC; +const int zfs_vm_pagerput_inval = VM_PAGER_PUT_INVAL; + +void +zfs_vmobject_assert_wlocked(vm_object_t object) +{ + + /* + * This is not ideal because FILE/LINE used by assertions will not + * be too helpful, but it must be an hard function for + * compatibility reasons. + */ + VM_OBJECT_ASSERT_WLOCKED(object); +} + +void +zfs_vmobject_wlock(vm_object_t object) +{ + + VM_OBJECT_WLOCK(object); +} + +void +zfs_vmobject_wunlock(vm_object_t object) +{ + + VM_OBJECT_WUNLOCK(object); +} diff --git a/module/os/freebsd/spl/spl_zlib.c b/module/os/freebsd/spl/spl_zlib.c new file mode 100644 index 00000000000..7549483d8bc --- /dev/null +++ b/module/os/freebsd/spl/spl_zlib.c @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#if __FreeBSD_version >= 1300041 +#include +#else +#include +#endif +#include + + +/*ARGSUSED*/ +static void * +zcalloc(void *opaque, uint_t items, uint_t size) +{ + + return (malloc((size_t)items*size, M_SOLARIS, M_NOWAIT)); +} + +/*ARGSUSED*/ +static void +zcfree(void *opaque, void *ptr) +{ + + free(ptr, M_SOLARIS); +} + +static int +zlib_deflateInit(z_stream *stream, int level) +{ + + stream->zalloc = zcalloc; + stream->opaque = NULL; + stream->zfree = zcfree; + + return (deflateInit(stream, level)); +} + +static int +zlib_deflate(z_stream *stream, int flush) +{ + return (deflate(stream, flush)); +} + +static int +zlib_deflateEnd(z_stream *stream) +{ + return (deflateEnd(stream)); +} + +static int +zlib_inflateInit(z_stream *stream) +{ + stream->zalloc = zcalloc; + stream->opaque = NULL; + stream->zfree = zcfree; + + return (inflateInit(stream)); +} + +static int +zlib_inflate(z_stream *stream, int finish) +{ +#if __FreeBSD_version >= 1300024 + return (inflate(stream, finish)); +#else + return (_zlib104_inflate(stream, finish)); +#endif +} + + +static int +zlib_inflateEnd(z_stream *stream) +{ + return (inflateInit(stream)); +} + +/* + * A kmem_cache is used for the zlib workspaces to avoid having to vmalloc + * and vfree for every call. Using a kmem_cache also has the advantage + * that improves the odds that the memory used will be local to this cpu. + * To further improve things it might be wise to create a dedicated per-cpu + * workspace for use. This would take some additional care because we then + * must disable preemption around the critical section, and verify that + * zlib_deflate* and zlib_inflate* never internally call schedule(). + */ +static void * +zlib_workspace_alloc(int flags) +{ + // return (kmem_cache_alloc(zlib_workspace_cache, flags)); + return (NULL); +} + +static void +zlib_workspace_free(void *workspace) +{ + // kmem_cache_free(zlib_workspace_cache, workspace); +} + +/* + * Compresses the source buffer into the destination buffer. The level + * parameter has the same meaning as in deflateInit. sourceLen is the byte + * length of the source buffer. Upon entry, destLen is the total size of the + * destination buffer, which must be at least 0.1% larger than sourceLen plus + * 12 bytes. Upon exit, destLen is the actual size of the compressed buffer. + * + * compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + * memory, Z_BUF_ERROR if there was not enough room in the output buffer, + * Z_STREAM_ERROR if the level parameter is invalid. + */ +int +z_compress_level(void *dest, size_t *destLen, const void *source, + size_t sourceLen, int level) +{ + z_stream stream; + int err; + + bzero(&stream, sizeof (stream)); + stream.next_in = (Byte *)source; + stream.avail_in = (uInt)sourceLen; + stream.next_out = dest; + stream.avail_out = (uInt)*destLen; + stream.opaque = NULL; + + if ((size_t)stream.avail_out != *destLen) + return (Z_BUF_ERROR); + + stream.opaque = zlib_workspace_alloc(KM_SLEEP); +#if 0 + if (!stream.opaque) + return (Z_MEM_ERROR); +#endif + err = zlib_deflateInit(&stream, level); + if (err != Z_OK) { + zlib_workspace_free(stream.opaque); + return (err); + } + + err = zlib_deflate(&stream, Z_FINISH); + if (err != Z_STREAM_END) { + zlib_deflateEnd(&stream); + zlib_workspace_free(stream.opaque); + return (err == Z_OK ? Z_BUF_ERROR : err); + } + *destLen = stream.total_out; + + err = zlib_deflateEnd(&stream); + zlib_workspace_free(stream.opaque); + return (err); +} + +/* + * Decompresses the source buffer into the destination buffer. sourceLen is + * the byte length of the source buffer. Upon entry, destLen is the total + * size of the destination buffer, which must be large enough to hold the + * entire uncompressed data. (The size of the uncompressed data must have + * been saved previously by the compressor and transmitted to the decompressor + * by some mechanism outside the scope of this compression library.) + * Upon exit, destLen is the actual size of the compressed buffer. + * This function can be used to decompress a whole file at once if the + * input file is mmap'ed. + * + * uncompress returns Z_OK if success, Z_MEM_ERROR if there was not + * enough memory, Z_BUF_ERROR if there was not enough room in the output + * buffer, or Z_DATA_ERROR if the input data was corrupted. + */ +int +z_uncompress(void *dest, size_t *destLen, const void *source, size_t sourceLen) +{ + z_stream stream; + int err; + + bzero(&stream, sizeof (stream)); + + stream.next_in = (Byte *)source; + stream.avail_in = (uInt)sourceLen; + stream.next_out = dest; + stream.avail_out = (uInt)*destLen; + + if ((size_t)stream.avail_out != *destLen) + return (Z_BUF_ERROR); + + stream.opaque = zlib_workspace_alloc(KM_SLEEP); +#if 0 + if (!stream.opaque) + return (Z_MEM_ERROR); +#endif + err = zlib_inflateInit(&stream); + if (err != Z_OK) { + zlib_workspace_free(stream.opaque); + return (err); + } + + err = zlib_inflate(&stream, Z_FINISH); + if (err != Z_STREAM_END) { + zlib_inflateEnd(&stream); + zlib_workspace_free(stream.opaque); + + if (err == Z_NEED_DICT || + (err == Z_BUF_ERROR && stream.avail_in == 0)) + return (Z_DATA_ERROR); + + return (err); + } + *destLen = stream.total_out; + + err = zlib_inflateEnd(&stream); + zlib_workspace_free(stream.opaque); + + return (err); +} + +#if 0 +int +spl_zlib_init(void) +{ + int size; + + size = MAX(spl_zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL), + zlib_inflate_workspacesize()); + + zlib_workspace_cache = kmem_cache_create( + "spl_zlib_workspace_cache", + size, 0, NULL, NULL, NULL, NULL, NULL, + KMC_VMEM | KMC_NOEMERGENCY); + if (!zlib_workspace_cache) + return (1); + + return (0); +} + +void +spl_zlib_fini(void) +{ + kmem_cache_destroy(zlib_workspace_cache); + zlib_workspace_cache = NULL; +} +#endif diff --git a/module/os/freebsd/spl/spl_zone.c b/module/os/freebsd/spl/spl_zone.c new file mode 100644 index 00000000000..40f21934ef6 --- /dev/null +++ b/module/os/freebsd/spl/spl_zone.c @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2007 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static MALLOC_DEFINE(M_ZONES, "zones_data", "Zones data"); + +/* + * Structure to record list of ZFS datasets exported to a zone. + */ +typedef struct zone_dataset { + LIST_ENTRY(zone_dataset) zd_next; + char zd_dataset[0]; +} zone_dataset_t; + +LIST_HEAD(zone_dataset_head, zone_dataset); + +static int zone_slot; + +int +zone_dataset_attach(struct ucred *cred, const char *dataset, int jailid) +{ + struct zone_dataset_head *head; + zone_dataset_t *zd, *zd2; + struct prison *pr; + int dofree, error; + + if ((error = spl_priv_check_cred(cred, PRIV_ZFS_JAIL)) != 0) + return (error); + + /* Allocate memory before we grab prison's mutex. */ + zd = malloc(sizeof (*zd) + strlen(dataset) + 1, M_ZONES, M_WAITOK); + + sx_slock(&allprison_lock); + pr = prison_find(jailid); /* Locks &pr->pr_mtx. */ + sx_sunlock(&allprison_lock); + if (pr == NULL) { + free(zd, M_ZONES); + return (ENOENT); + } + + head = osd_jail_get(pr, zone_slot); + if (head != NULL) { + dofree = 0; + LIST_FOREACH(zd2, head, zd_next) { + if (strcmp(dataset, zd2->zd_dataset) == 0) { + free(zd, M_ZONES); + error = EEXIST; + goto end; + } + } + } else { + dofree = 1; + prison_hold_locked(pr); + mtx_unlock(&pr->pr_mtx); + head = malloc(sizeof (*head), M_ZONES, M_WAITOK); + LIST_INIT(head); + mtx_lock(&pr->pr_mtx); + error = osd_jail_set(pr, zone_slot, head); + KASSERT(error == 0, ("osd_jail_set() failed (error=%d)", + error)); + } + strcpy(zd->zd_dataset, dataset); + LIST_INSERT_HEAD(head, zd, zd_next); +end: + if (dofree) + prison_free_locked(pr); + else + mtx_unlock(&pr->pr_mtx); + return (error); +} + +int +zone_dataset_detach(struct ucred *cred, const char *dataset, int jailid) +{ + struct zone_dataset_head *head; + zone_dataset_t *zd; + struct prison *pr; + int error; + + if ((error = spl_priv_check_cred(cred, PRIV_ZFS_JAIL)) != 0) + return (error); + + sx_slock(&allprison_lock); + pr = prison_find(jailid); + sx_sunlock(&allprison_lock); + if (pr == NULL) + return (ENOENT); + head = osd_jail_get(pr, zone_slot); + if (head == NULL) { + error = ENOENT; + goto end; + } + LIST_FOREACH(zd, head, zd_next) { + if (strcmp(dataset, zd->zd_dataset) == 0) + break; + } + if (zd == NULL) + error = ENOENT; + else { + LIST_REMOVE(zd, zd_next); + free(zd, M_ZONES); + if (LIST_EMPTY(head)) + osd_jail_del(pr, zone_slot); + error = 0; + } +end: + mtx_unlock(&pr->pr_mtx); + return (error); +} + +/* + * Returns true if the named dataset is visible in the current zone. + * The 'write' parameter is set to 1 if the dataset is also writable. + */ +int +zone_dataset_visible(const char *dataset, int *write) +{ + struct zone_dataset_head *head; + zone_dataset_t *zd; + struct prison *pr; + size_t len; + int ret = 0; + + if (dataset[0] == '\0') + return (0); + if (INGLOBALZONE(curproc)) { + if (write != NULL) + *write = 1; + return (1); + } + pr = curthread->td_ucred->cr_prison; + mtx_lock(&pr->pr_mtx); + head = osd_jail_get(pr, zone_slot); + if (head == NULL) + goto end; + + /* + * Walk the list once, looking for datasets which match exactly, or + * specify a dataset underneath an exported dataset. If found, return + * true and note that it is writable. + */ + LIST_FOREACH(zd, head, zd_next) { + len = strlen(zd->zd_dataset); + if (strlen(dataset) >= len && + bcmp(dataset, zd->zd_dataset, len) == 0 && + (dataset[len] == '\0' || dataset[len] == '/' || + dataset[len] == '@')) { + if (write) + *write = 1; + ret = 1; + goto end; + } + } + + /* + * Walk the list a second time, searching for datasets which are parents + * of exported datasets. These should be visible, but read-only. + * + * Note that we also have to support forms such as 'pool/dataset/', with + * a trailing slash. + */ + LIST_FOREACH(zd, head, zd_next) { + len = strlen(dataset); + if (dataset[len - 1] == '/') + len--; /* Ignore trailing slash */ + if (len < strlen(zd->zd_dataset) && + bcmp(dataset, zd->zd_dataset, len) == 0 && + zd->zd_dataset[len] == '/') { + if (write) + *write = 0; + ret = 1; + goto end; + } + } +end: + mtx_unlock(&pr->pr_mtx); + return (ret); +} + +static void +zone_destroy(void *arg) +{ + struct zone_dataset_head *head; + zone_dataset_t *zd; + + head = arg; + while ((zd = LIST_FIRST(head)) != NULL) { + LIST_REMOVE(zd, zd_next); + free(zd, M_ZONES); + } + free(head, M_ZONES); +} + +uint32_t +zone_get_hostid(void *ptr) +{ + + KASSERT(ptr == NULL, ("only NULL pointer supported in %s", __func__)); + + return ((uint32_t)curthread->td_ucred->cr_prison->pr_hostid); +} + +boolean_t +in_globalzone(struct proc *p) +{ + return (!jailed(FIRST_THREAD_IN_PROC((p))->td_ucred)); +} + +static void +zone_sysinit(void *arg __unused) +{ + + zone_slot = osd_jail_register(zone_destroy, NULL); +} + +static void +zone_sysuninit(void *arg __unused) +{ + + osd_jail_deregister(zone_slot); +} + +SYSINIT(zone_sysinit, SI_SUB_DRIVERS, SI_ORDER_ANY, zone_sysinit, NULL); +SYSUNINIT(zone_sysuninit, SI_SUB_DRIVERS, SI_ORDER_ANY, zone_sysuninit, NULL); diff --git a/module/os/freebsd/zfs/abd.c b/module/os/freebsd/zfs/abd.c new file mode 100644 index 00000000000..888a113a429 --- /dev/null +++ b/module/os/freebsd/zfs/abd.c @@ -0,0 +1,1134 @@ +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2014 by Chunwei Chen. All rights reserved. + * Copyright (c) 2016 by Delphix. All rights reserved. + */ + +/* + * ARC buffer data (ABD). + * + * ABDs are an abstract data structure for the ARC which can use two + * different ways of storing the underlying data: + * + * (a) Linear buffer. In this case, all the data in the ABD is stored in one + * contiguous buffer in memory (from a zio_[data_]buf_* kmem cache). + * + * +-------------------+ + * | ABD (linear) | + * | abd_flags = ... | + * | abd_size = ... | +--------------------------------+ + * | abd_buf ------------->| raw buffer of size abd_size | + * +-------------------+ +--------------------------------+ + * no abd_chunks + * + * (b) Scattered buffer. In this case, the data in the ABD is split into + * equal-sized chunks (from the abd_chunk_cache kmem_cache), with pointers + * to the chunks recorded in an array at the end of the ABD structure. + * + * +-------------------+ + * | ABD (scattered) | + * | abd_flags = ... | + * | abd_size = ... | + * | abd_offset = 0 | +-----------+ + * | abd_chunks[0] ----------------------------->| chunk 0 | + * | abd_chunks[1] ---------------------+ +-----------+ + * | ... | | +-----------+ + * | abd_chunks[N-1] ---------+ +------->| chunk 1 | + * +-------------------+ | +-----------+ + * | ... + * | +-----------+ + * +----------------->| chunk N-1 | + * +-----------+ + * + * Using a large proportion of scattered ABDs decreases ARC fragmentation since + * when we are at the limit of allocatable space, using equal-size chunks will + * allow us to quickly reclaim enough space for a new large allocation (assuming + * it is also scattered). + * + * In addition to directly allocating a linear or scattered ABD, it is also + * possible to create an ABD by requesting the "sub-ABD" starting at an offset + * within an existing ABD. In linear buffers this is simple (set abd_buf of + * the new ABD to the starting point within the original raw buffer), but + * scattered ABDs are a little more complex. The new ABD makes a copy of the + * relevant abd_chunks pointers (but not the underlying data). However, to + * provide arbitrary rather than only chunk-aligned starting offsets, it also + * tracks an abd_offset field which represents the starting point of the data + * within the first chunk in abd_chunks. For both linear and scattered ABDs, + * creating an offset ABD marks the original ABD as the offset's parent, and the + * original ABD's abd_children refcount is incremented. This data allows us to + * ensure the root ABD isn't deleted before its children. + * + * Most consumers should never need to know what type of ABD they're using -- + * the ABD public API ensures that it's possible to transparently switch from + * using a linear ABD to a scattered one when doing so would be beneficial. + * + * If you need to use the data within an ABD directly, if you know it's linear + * (because you allocated it) you can use abd_to_buf() to access the underlying + * raw buffer. Otherwise, you should use one of the abd_borrow_buf* functions + * which will allocate a raw buffer if necessary. Use the abd_return_buf* + * functions to return any raw buffers that are no longer necessary when you're + * done using them. + * + * There are a variety of ABD APIs that implement basic buffer operations: + * compare, copy, read, write, and fill with zeroes. If you need a custom + * function which progressively accesses the whole ABD, use the abd_iterate_* + * functions. + */ + +#include +#include +#include +#include +#include + +typedef struct abd_stats { + kstat_named_t abdstat_struct_size; + kstat_named_t abdstat_scatter_cnt; + kstat_named_t abdstat_scatter_data_size; + kstat_named_t abdstat_scatter_chunk_waste; + kstat_named_t abdstat_linear_cnt; + kstat_named_t abdstat_linear_data_size; +} abd_stats_t; + +static abd_stats_t abd_stats = { + /* Amount of memory occupied by all of the abd_t struct allocations */ + { "struct_size", KSTAT_DATA_UINT64 }, + /* + * The number of scatter ABDs which are currently allocated, excluding + * ABDs which don't own their data (for instance the ones which were + * allocated through abd_get_offset()). + */ + { "scatter_cnt", KSTAT_DATA_UINT64 }, + /* Amount of data stored in all scatter ABDs tracked by scatter_cnt */ + { "scatter_data_size", KSTAT_DATA_UINT64 }, + /* + * The amount of space wasted at the end of the last chunk across all + * scatter ABDs tracked by scatter_cnt. + */ + { "scatter_chunk_waste", KSTAT_DATA_UINT64 }, + /* + * The number of linear ABDs which are currently allocated, excluding + * ABDs which don't own their data (for instance the ones which were + * allocated through abd_get_offset() and abd_get_from_buf()). If an + * ABD takes ownership of its buf then it will become tracked. + */ + { "linear_cnt", KSTAT_DATA_UINT64 }, + /* Amount of data stored in all linear ABDs tracked by linear_cnt */ + { "linear_data_size", KSTAT_DATA_UINT64 }, +}; + +#define ABDSTAT(stat) (abd_stats.stat.value.ui64) +#define ABDSTAT_INCR(stat, val) \ + atomic_add_64(&abd_stats.stat.value.ui64, (val)) +#define ABDSTAT_BUMP(stat) ABDSTAT_INCR(stat, 1) +#define ABDSTAT_BUMPDOWN(stat) ABDSTAT_INCR(stat, -1) + +/* + * It is possible to make all future ABDs be linear by setting this to B_FALSE. + * Otherwise, ABDs are allocated scattered by default unless the caller uses + * abd_alloc_linear(). + */ +boolean_t zfs_abd_scatter_enabled = B_TRUE; + +/* + * The size of the chunks ABD allocates. Because the sizes allocated from the + * kmem_cache can't change, this tunable can only be modified at boot. Changing + * it at runtime would cause ABD iteration to work incorrectly for ABDs which + * were allocated with the old size, so a safeguard has been put in place which + * will cause the machine to panic if you change it and try to access the data + * within a scattered ABD. + */ +size_t zfs_abd_chunk_size = 4096; + +#if defined(_KERNEL) +SYSCTL_DECL(_vfs_zfs); + +SYSCTL_INT(_vfs_zfs, OID_AUTO, abd_scatter_enabled, CTLFLAG_RWTUN, + &zfs_abd_scatter_enabled, 0, "Enable scattered ARC data buffers"); +SYSCTL_ULONG(_vfs_zfs, OID_AUTO, abd_chunk_size, CTLFLAG_RDTUN, + &zfs_abd_chunk_size, 0, "The size of the chunks ABD allocates"); +#endif + +kmem_cache_t *abd_chunk_cache; +static kstat_t *abd_ksp; + +extern inline boolean_t abd_is_linear(abd_t *abd); +extern inline void abd_copy(abd_t *dabd, abd_t *sabd, size_t size); +extern inline void abd_copy_from_buf(abd_t *abd, const void *buf, size_t size); +extern inline void abd_copy_to_buf(void* buf, abd_t *abd, size_t size); +extern inline int abd_cmp_buf(abd_t *abd, const void *buf, size_t size); +extern inline void abd_zero(abd_t *abd, size_t size); + +static void * +abd_alloc_chunk() +{ + void *c = kmem_cache_alloc(abd_chunk_cache, KM_PUSHPAGE); + ASSERT3P(c, !=, NULL); + return (c); +} + +static void +abd_free_chunk(void *c) +{ + kmem_cache_free(abd_chunk_cache, c); +} + +void +abd_init(void) +{ + abd_chunk_cache = kmem_cache_create("abd_chunk", zfs_abd_chunk_size, 0, + NULL, NULL, NULL, NULL, 0, KMC_NOTOUCH | KMC_NODEBUG); + + abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED, + sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); + if (abd_ksp != NULL) { + abd_ksp->ks_data = &abd_stats; + kstat_install(abd_ksp); + } +} + +void +abd_fini(void) +{ + if (abd_ksp != NULL) { + kstat_delete(abd_ksp); + abd_ksp = NULL; + } + + kmem_cache_destroy(abd_chunk_cache); + abd_chunk_cache = NULL; +} + +static inline size_t +abd_chunkcnt_for_bytes(size_t size) +{ + return (P2ROUNDUP(size, zfs_abd_chunk_size) / zfs_abd_chunk_size); +} + +static inline size_t +abd_scatter_chunkcnt(abd_t *abd) +{ + ASSERT(!abd_is_linear(abd)); + return (abd_chunkcnt_for_bytes( + abd->abd_u.abd_scatter.abd_offset + abd->abd_size)); +} + +static inline void +abd_verify(abd_t *abd) +{ + ASSERT3U(abd->abd_size, >, 0); + ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE); + ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR | + ABD_FLAG_OWNER | ABD_FLAG_META)); + IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER)); + IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER); + if (abd_is_linear(abd)) { + ASSERT3P(abd->abd_u.abd_linear.abd_buf, !=, NULL); + } else { + ASSERT3U(abd->abd_u.abd_scatter.abd_offset, <, + zfs_abd_chunk_size); + size_t n = abd_scatter_chunkcnt(abd); + for (int i = 0; i < n; i++) { + ASSERT3P( + abd->abd_u.abd_scatter.abd_chunks[i], !=, NULL); + } + } +} + +static inline abd_t * +abd_alloc_struct(size_t chunkcnt) +{ + size_t size = offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]); + abd_t *abd = kmem_alloc(size, KM_PUSHPAGE); + ASSERT3P(abd, !=, NULL); + ABDSTAT_INCR(abdstat_struct_size, size); + + return (abd); +} + +static inline void +abd_free_struct(abd_t *abd) +{ + size_t chunkcnt = abd_is_linear(abd) ? 0 : abd_scatter_chunkcnt(abd); + int size = offsetof(abd_t, abd_u.abd_scatter.abd_chunks[chunkcnt]); + kmem_free(abd, size); + ABDSTAT_INCR(abdstat_struct_size, -size); +} + +/* + * Allocate an ABD, along with its own underlying data buffers. Use this if you + * don't care whether the ABD is linear or not. + */ +abd_t * +abd_alloc(size_t size, boolean_t is_metadata) +{ + if (!zfs_abd_scatter_enabled || size <= zfs_abd_chunk_size) + return (abd_alloc_linear(size, is_metadata)); + + VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); + + size_t n = abd_chunkcnt_for_bytes(size); + abd_t *abd = abd_alloc_struct(n); + + abd->abd_flags = ABD_FLAG_OWNER; + if (is_metadata) { + abd->abd_flags |= ABD_FLAG_META; + } + abd->abd_size = size; + abd->abd_parent = NULL; + zfs_refcount_create(&abd->abd_children); + + abd->abd_u.abd_scatter.abd_offset = 0; + abd->abd_u.abd_scatter.abd_chunk_size = zfs_abd_chunk_size; + + for (int i = 0; i < n; i++) { + void *c = abd_alloc_chunk(); + ASSERT3P(c, !=, NULL); + abd->abd_u.abd_scatter.abd_chunks[i] = c; + } + + ABDSTAT_BUMP(abdstat_scatter_cnt); + ABDSTAT_INCR(abdstat_scatter_data_size, size); + ABDSTAT_INCR(abdstat_scatter_chunk_waste, + n * zfs_abd_chunk_size - size); + + return (abd); +} + +static void +abd_free_scatter(abd_t *abd) +{ + size_t n = abd_scatter_chunkcnt(abd); + for (int i = 0; i < n; i++) { + abd_free_chunk(abd->abd_u.abd_scatter.abd_chunks[i]); + } + + zfs_refcount_destroy(&abd->abd_children); + ABDSTAT_BUMPDOWN(abdstat_scatter_cnt); + ABDSTAT_INCR(abdstat_scatter_data_size, -(int)abd->abd_size); + ABDSTAT_INCR(abdstat_scatter_chunk_waste, + abd->abd_size - n * zfs_abd_chunk_size); + + abd_free_struct(abd); +} + +/* + * Allocate an ABD that must be linear, along with its own underlying data + * buffer. Only use this when it would be very annoying to write your ABD + * consumer with a scattered ABD. + */ +abd_t * +abd_alloc_linear(size_t size, boolean_t is_metadata) +{ + abd_t *abd = abd_alloc_struct(0); + + VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); + + abd->abd_flags = ABD_FLAG_LINEAR | ABD_FLAG_OWNER; + if (is_metadata) { + abd->abd_flags |= ABD_FLAG_META; + } + abd->abd_size = size; + abd->abd_parent = NULL; + zfs_refcount_create(&abd->abd_children); + + if (is_metadata) { + abd->abd_u.abd_linear.abd_buf = zio_buf_alloc(size); + } else { + abd->abd_u.abd_linear.abd_buf = zio_data_buf_alloc(size); + } + + ABDSTAT_BUMP(abdstat_linear_cnt); + ABDSTAT_INCR(abdstat_linear_data_size, size); + + return (abd); +} + +static void +abd_free_linear(abd_t *abd) +{ + if (abd->abd_flags & ABD_FLAG_META) { + zio_buf_free(abd->abd_u.abd_linear.abd_buf, abd->abd_size); + } else { + zio_data_buf_free(abd->abd_u.abd_linear.abd_buf, abd->abd_size); + } + + zfs_refcount_destroy(&abd->abd_children); + ABDSTAT_BUMPDOWN(abdstat_linear_cnt); + ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size); + + abd_free_struct(abd); +} + +/* + * Free an ABD. Only use this on ABDs allocated with abd_alloc() or + * abd_alloc_linear(). + */ +void +abd_free(abd_t *abd) +{ + if (abd == NULL) + return; + + abd_verify(abd); + ASSERT3P(abd->abd_parent, ==, NULL); + ASSERT(abd->abd_flags & ABD_FLAG_OWNER); + if (abd_is_linear(abd)) + abd_free_linear(abd); + else + abd_free_scatter(abd); +} + +/* + * Allocate an ABD of the same format (same metadata flag, same scatterize + * setting) as another ABD. + */ +abd_t * +abd_alloc_sametype(abd_t *sabd, size_t size) +{ + boolean_t is_metadata = (sabd->abd_flags & ABD_FLAG_META) != 0; + if (abd_is_linear(sabd)) { + return (abd_alloc_linear(size, is_metadata)); + } else { + return (abd_alloc(size, is_metadata)); + } +} + +/* + * If we're going to use this ABD for doing I/O using the block layer, the + * consumer of the ABD data doesn't care if it's scattered or not, and we don't + * plan to store this ABD in memory for a long period of time, we should + * allocate the ABD type that requires the least data copying to do the I/O. + * + * Currently this is linear ABDs, however if ldi_strategy() can ever issue I/Os + * using a scatter/gather list we should switch to that and replace this call + * with vanilla abd_alloc(). + */ +abd_t * +abd_alloc_for_io(size_t size, boolean_t is_metadata) +{ + return (abd_alloc_linear(size, is_metadata)); +} + +/* + * Allocate a new ABD to point to offset off of sabd. It shares the underlying + * buffer data with sabd. Use abd_put() to free. sabd must not be freed while + * any derived ABDs exist. + */ +/* ARGSUSED */ +static inline abd_t * +abd_get_offset_impl(abd_t *sabd, size_t off, size_t size) +{ + abd_t *abd; + + abd_verify(sabd); + ASSERT3U(off, <=, sabd->abd_size); + + if (abd_is_linear(sabd)) { + abd = abd_alloc_struct(0); + + /* + * Even if this buf is filesystem metadata, we only track that + * if we own the underlying data buffer, which is not true in + * this case. Therefore, we don't ever use ABD_FLAG_META here. + */ + abd->abd_flags = ABD_FLAG_LINEAR; + + abd->abd_u.abd_linear.abd_buf = + (char *)sabd->abd_u.abd_linear.abd_buf + off; + } else { + size_t new_offset = sabd->abd_u.abd_scatter.abd_offset + off; + size_t chunkcnt = abd_scatter_chunkcnt(sabd) - + (new_offset / zfs_abd_chunk_size); + + abd = abd_alloc_struct(chunkcnt); + + /* + * Even if this buf is filesystem metadata, we only track that + * if we own the underlying data buffer, which is not true in + * this case. Therefore, we don't ever use ABD_FLAG_META here. + */ + abd->abd_flags = 0; + + abd->abd_u.abd_scatter.abd_offset = + new_offset % zfs_abd_chunk_size; + abd->abd_u.abd_scatter.abd_chunk_size = zfs_abd_chunk_size; + + /* Copy the scatterlist starting at the correct offset */ + (void) memcpy(&abd->abd_u.abd_scatter.abd_chunks, + &sabd->abd_u.abd_scatter.abd_chunks[new_offset / + zfs_abd_chunk_size], + chunkcnt * sizeof (void *)); + } + + if (size == 0) + abd->abd_size = sabd->abd_size - off; + else + abd->abd_size = size; + abd->abd_parent = sabd; + zfs_refcount_create(&abd->abd_children); + (void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd); + + return (abd); +} + +abd_t * +abd_get_offset(abd_t *sabd, size_t off) +{ + + return (abd_get_offset_impl(sabd, off, 0)); +} + +abd_t * +abd_get_offset_size(abd_t *sabd, size_t off, size_t size) +{ + ASSERT3U(off + size, <=, sabd->abd_size); + + return (abd_get_offset_impl(sabd, off, size)); +} + + +/* + * Allocate a linear ABD structure for buf. You must free this with abd_put() + * since the resulting ABD doesn't own its own buffer. + */ +abd_t * +abd_get_from_buf(void *buf, size_t size) +{ + abd_t *abd = abd_alloc_struct(0); + + VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); + + /* + * Even if this buf is filesystem metadata, we only track that if we + * own the underlying data buffer, which is not true in this case. + * Therefore, we don't ever use ABD_FLAG_META here. + */ + abd->abd_flags = ABD_FLAG_LINEAR; + abd->abd_size = size; + abd->abd_parent = NULL; + zfs_refcount_create(&abd->abd_children); + + abd->abd_u.abd_linear.abd_buf = buf; + + return (abd); +} + +/* + * Free an ABD allocated from abd_get_offset() or abd_get_from_buf(). Will not + * free the underlying scatterlist or buffer. + */ +void +abd_put(abd_t *abd) +{ + if (abd == NULL) + return; + abd_verify(abd); + ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER)); + + if (abd->abd_parent != NULL) { + (void) zfs_refcount_remove_many(&abd->abd_parent->abd_children, + abd->abd_size, abd); + } + + zfs_refcount_destroy(&abd->abd_children); + abd_free_struct(abd); +} + +/* + * Get the raw buffer associated with a linear ABD. + */ +void * +abd_to_buf(abd_t *abd) +{ + ASSERT(abd_is_linear(abd)); + abd_verify(abd); + return (abd->abd_u.abd_linear.abd_buf); +} + +/* + * Borrow a raw buffer from an ABD without copying the contents of the ABD + * into the buffer. If the ABD is scattered, this will allocate a raw buffer + * whose contents are undefined. To copy over the existing data in the ABD, use + * abd_borrow_buf_copy() instead. + */ +void * +abd_borrow_buf(abd_t *abd, size_t n) +{ + void *buf; + abd_verify(abd); + ASSERT3U(abd->abd_size, >=, n); + if (abd_is_linear(abd)) { + buf = abd_to_buf(abd); + } else { + buf = zio_buf_alloc(n); + } + (void) zfs_refcount_add_many(&abd->abd_children, n, buf); + + return (buf); +} + +void * +abd_borrow_buf_copy(abd_t *abd, size_t n) +{ + void *buf = abd_borrow_buf(abd, n); + if (!abd_is_linear(abd)) { + abd_copy_to_buf(buf, abd, n); + } + return (buf); +} + +/* + * Return a borrowed raw buffer to an ABD. If the ABD is scattered, this will + * not change the contents of the ABD and will ASSERT that you didn't modify + * the buffer since it was borrowed. If you want any changes you made to buf to + * be copied back to abd, use abd_return_buf_copy() instead. + */ +void +abd_return_buf(abd_t *abd, void *buf, size_t n) +{ + abd_verify(abd); + ASSERT3U(abd->abd_size, >=, n); + if (abd_is_linear(abd)) { + ASSERT3P(buf, ==, abd_to_buf(abd)); + } else { + ASSERT0(abd_cmp_buf(abd, buf, n)); + zio_buf_free(buf, n); + } + (void) zfs_refcount_remove_many(&abd->abd_children, n, buf); +} + +void +abd_return_buf_copy(abd_t *abd, void *buf, size_t n) +{ + if (!abd_is_linear(abd)) { + abd_copy_from_buf(abd, buf, n); + } + abd_return_buf(abd, buf, n); +} + +/* + * Give this ABD ownership of the buffer that it's storing. Can only be used on + * linear ABDs which were allocated via abd_get_from_buf(), or ones allocated + * with abd_alloc_linear() which subsequently released ownership of their buf + * with abd_release_ownership_of_buf(). + */ +void +abd_take_ownership_of_buf(abd_t *abd, boolean_t is_metadata) +{ + ASSERT(abd_is_linear(abd)); + ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER)); + abd_verify(abd); + + abd->abd_flags |= ABD_FLAG_OWNER; + if (is_metadata) { + abd->abd_flags |= ABD_FLAG_META; + } + + ABDSTAT_BUMP(abdstat_linear_cnt); + ABDSTAT_INCR(abdstat_linear_data_size, abd->abd_size); +} + +void +abd_release_ownership_of_buf(abd_t *abd) +{ + ASSERT(abd_is_linear(abd)); + ASSERT(abd->abd_flags & ABD_FLAG_OWNER); + abd_verify(abd); + + abd->abd_flags &= ~ABD_FLAG_OWNER; + /* Disable this flag since we no longer own the data buffer */ + abd->abd_flags &= ~ABD_FLAG_META; + + ABDSTAT_BUMPDOWN(abdstat_linear_cnt); + ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size); +} + +struct abd_iter { + abd_t *iter_abd; /* ABD being iterated through */ + size_t iter_pos; /* position (relative to abd_offset) */ + void *iter_mapaddr; /* addr corresponding to iter_pos */ + size_t iter_mapsize; /* length of data valid at mapaddr */ +}; + +static inline size_t +abd_iter_scatter_chunk_offset(struct abd_iter *aiter) +{ + ASSERT(!abd_is_linear(aiter->iter_abd)); + return ((aiter->iter_abd->abd_u.abd_scatter.abd_offset + + aiter->iter_pos) % zfs_abd_chunk_size); +} + +static inline size_t +abd_iter_scatter_chunk_index(struct abd_iter *aiter) +{ + ASSERT(!abd_is_linear(aiter->iter_abd)); + return ((aiter->iter_abd->abd_u.abd_scatter.abd_offset + + aiter->iter_pos) / zfs_abd_chunk_size); +} + +/* + * Initialize the abd_iter. + */ +static void +abd_iter_init(struct abd_iter *aiter, abd_t *abd) +{ + abd_verify(abd); + aiter->iter_abd = abd; + aiter->iter_pos = 0; + aiter->iter_mapaddr = NULL; + aiter->iter_mapsize = 0; +} + +/* + * Advance the iterator by a certain amount. Cannot be called when a chunk is + * in use. This can be safely called when the aiter has already exhausted, in + * which case this does nothing. + */ +static void +abd_iter_advance(struct abd_iter *aiter, size_t amount) +{ + ASSERT3P(aiter->iter_mapaddr, ==, NULL); + ASSERT0(aiter->iter_mapsize); + + /* There's nothing left to advance to, so do nothing */ + if (aiter->iter_pos == aiter->iter_abd->abd_size) + return; + + aiter->iter_pos += amount; +} + +/* + * Map the current chunk into aiter. This can be safely called when the aiter + * has already exhausted, in which case this does nothing. + */ +static void +abd_iter_map(struct abd_iter *aiter) +{ + void *paddr; + size_t offset = 0; + + ASSERT3P(aiter->iter_mapaddr, ==, NULL); + ASSERT0(aiter->iter_mapsize); + + /* Panic if someone has changed zfs_abd_chunk_size */ + IMPLY(!abd_is_linear(aiter->iter_abd), zfs_abd_chunk_size == + aiter->iter_abd->abd_u.abd_scatter.abd_chunk_size); + + /* There's nothing left to iterate over, so do nothing */ + if (aiter->iter_pos == aiter->iter_abd->abd_size) + return; + + if (abd_is_linear(aiter->iter_abd)) { + offset = aiter->iter_pos; + aiter->iter_mapsize = aiter->iter_abd->abd_size - offset; + paddr = aiter->iter_abd->abd_u.abd_linear.abd_buf; + } else { + size_t index = abd_iter_scatter_chunk_index(aiter); + offset = abd_iter_scatter_chunk_offset(aiter); + aiter->iter_mapsize = MIN(zfs_abd_chunk_size - offset, + aiter->iter_abd->abd_size - aiter->iter_pos); + paddr = aiter->iter_abd->abd_u.abd_scatter.abd_chunks[index]; + } + aiter->iter_mapaddr = (char *)paddr + offset; +} + +/* + * Unmap the current chunk from aiter. This can be safely called when the aiter + * has already exhausted, in which case this does nothing. + */ +static void +abd_iter_unmap(struct abd_iter *aiter) +{ + /* There's nothing left to unmap, so do nothing */ + if (aiter->iter_pos == aiter->iter_abd->abd_size) + return; + + ASSERT3P(aiter->iter_mapaddr, !=, NULL); + ASSERT3U(aiter->iter_mapsize, >, 0); + + aiter->iter_mapaddr = NULL; + aiter->iter_mapsize = 0; +} + +int +abd_iterate_func(abd_t *abd, size_t off, size_t size, + abd_iter_func_t *func, void *private) +{ + int ret = 0; + struct abd_iter aiter; + + abd_verify(abd); + ASSERT3U(off + size, <=, abd->abd_size); + + abd_iter_init(&aiter, abd); + abd_iter_advance(&aiter, off); + + while (size > 0) { + abd_iter_map(&aiter); + + size_t len = MIN(aiter.iter_mapsize, size); + ASSERT3U(len, >, 0); + + ret = func(aiter.iter_mapaddr, len, private); + + abd_iter_unmap(&aiter); + + if (ret != 0) + break; + + size -= len; + abd_iter_advance(&aiter, len); + } + + return (ret); +} + +struct buf_arg { + void *arg_buf; +}; + +static int +abd_copy_to_buf_off_cb(void *buf, size_t size, void *private) +{ + struct buf_arg *ba_ptr = private; + + (void) memcpy(ba_ptr->arg_buf, buf, size); + ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size; + + return (0); +} + +/* + * Copy abd to buf. (off is the offset in abd.) + */ +void +abd_copy_to_buf_off(void *buf, abd_t *abd, size_t off, size_t size) +{ + struct buf_arg ba_ptr = { buf }; + + (void) abd_iterate_func(abd, off, size, abd_copy_to_buf_off_cb, + &ba_ptr); +} + +static int +abd_cmp_buf_off_cb(void *buf, size_t size, void *private) +{ + int ret; + struct buf_arg *ba_ptr = private; + + ret = memcmp(buf, ba_ptr->arg_buf, size); + ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size; + + return (ret); +} + +/* + * Compare the contents of abd to buf. (off is the offset in abd.) + */ +int +abd_cmp_buf_off(abd_t *abd, const void *buf, size_t off, size_t size) +{ + struct buf_arg ba_ptr = { (void *) buf }; + + return (abd_iterate_func(abd, off, size, abd_cmp_buf_off_cb, &ba_ptr)); +} + +static int +abd_copy_from_buf_off_cb(void *buf, size_t size, void *private) +{ + struct buf_arg *ba_ptr = private; + + (void) memcpy(buf, ba_ptr->arg_buf, size); + ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size; + + return (0); +} + +/* + * Copy from buf to abd. (off is the offset in abd.) + */ +void +abd_copy_from_buf_off(abd_t *abd, const void *buf, size_t off, size_t size) +{ + struct buf_arg ba_ptr = { (void *) buf }; + + (void) abd_iterate_func(abd, off, size, abd_copy_from_buf_off_cb, + &ba_ptr); +} + +/*ARGSUSED*/ +static int +abd_zero_off_cb(void *buf, size_t size, void *private) +{ + (void) memset(buf, 0, size); + return (0); +} + +/* + * Zero out the abd from a particular offset to the end. + */ +void +abd_zero_off(abd_t *abd, size_t off, size_t size) +{ + (void) abd_iterate_func(abd, off, size, abd_zero_off_cb, NULL); +} + +/* + * Iterate over two ABDs and call func incrementally on the two ABDs' data in + * equal-sized chunks (passed to func as raw buffers). func could be called many + * times during this iteration. + */ +int +abd_iterate_func2(abd_t *dabd, abd_t *sabd, size_t doff, size_t soff, + size_t size, abd_iter_func2_t *func, void *private) +{ + int ret = 0; + struct abd_iter daiter, saiter; + + abd_verify(dabd); + abd_verify(sabd); + + ASSERT3U(doff + size, <=, dabd->abd_size); + ASSERT3U(soff + size, <=, sabd->abd_size); + + abd_iter_init(&daiter, dabd); + abd_iter_init(&saiter, sabd); + abd_iter_advance(&daiter, doff); + abd_iter_advance(&saiter, soff); + + while (size > 0) { + abd_iter_map(&daiter); + abd_iter_map(&saiter); + + size_t dlen = MIN(daiter.iter_mapsize, size); + size_t slen = MIN(saiter.iter_mapsize, size); + size_t len = MIN(dlen, slen); + ASSERT(dlen > 0 || slen > 0); + + ret = func(daiter.iter_mapaddr, saiter.iter_mapaddr, len, + private); + + abd_iter_unmap(&saiter); + abd_iter_unmap(&daiter); + + if (ret != 0) + break; + + size -= len; + abd_iter_advance(&daiter, len); + abd_iter_advance(&saiter, len); + } + + return (ret); +} + +/*ARGSUSED*/ +static int +abd_copy_off_cb(void *dbuf, void *sbuf, size_t size, void *private) +{ + (void) memcpy(dbuf, sbuf, size); + return (0); +} + +/* + * Copy from sabd to dabd starting from soff and doff. + */ +void +abd_copy_off(abd_t *dabd, abd_t *sabd, size_t doff, size_t soff, size_t size) +{ + (void) abd_iterate_func2(dabd, sabd, doff, soff, size, + abd_copy_off_cb, NULL); +} + +/*ARGSUSED*/ +static int +abd_cmp_cb(void *bufa, void *bufb, size_t size, void *private) +{ + return (memcmp(bufa, bufb, size)); +} + +/* + * Compares the contents of two ABDs. + */ +int +abd_cmp(abd_t *dabd, abd_t *sabd) +{ + ASSERT3U(dabd->abd_size, ==, sabd->abd_size); + return (abd_iterate_func2(dabd, sabd, 0, 0, dabd->abd_size, + abd_cmp_cb, NULL)); +} + +/* + * Iterate over code ABDs and a data ABD and call @func_raidz_gen. + * + * @cabds parity ABDs, must have equal size + * @dabd data ABD. Can be NULL (in this case @dsize = 0) + * @func_raidz_gen should be implemented so that its behaviour + * is the same when taking linear and when taking scatter + */ +void +abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd, + ssize_t csize, ssize_t dsize, const unsigned parity, + void (*func_raidz_gen)(void **, const void *, size_t, size_t)) +{ + int i; + ssize_t len, dlen; + struct abd_iter caiters[3]; + struct abd_iter daiter = {0}; + void *caddrs[3]; + + ASSERT3U(parity, <=, 3); + + for (i = 0; i < parity; i++) + abd_iter_init(&caiters[i], cabds[i]); + + if (dabd) + abd_iter_init(&daiter, dabd); + + ASSERT3S(dsize, >=, 0); + + critical_enter(); + while (csize > 0) { + len = csize; + + if (dabd && dsize > 0) + abd_iter_map(&daiter); + + for (i = 0; i < parity; i++) { + abd_iter_map(&caiters[i]); + caddrs[i] = caiters[i].iter_mapaddr; + } + + switch (parity) { + case 3: + len = MIN(caiters[2].iter_mapsize, len); + case 2: + len = MIN(caiters[1].iter_mapsize, len); + case 1: + len = MIN(caiters[0].iter_mapsize, len); + } + + /* must be progressive */ + ASSERT3S(len, >, 0); + + if (dabd && dsize > 0) { + /* this needs precise iter.length */ + len = MIN(daiter.iter_mapsize, len); + dlen = len; + } else + dlen = 0; + + /* must be progressive */ + ASSERT3S(len, >, 0); + /* + * The iterated function likely will not do well if each + * segment except the last one is not multiple of 512 (raidz). + */ + ASSERT3U(((uint64_t)len & 511ULL), ==, 0); + + func_raidz_gen(caddrs, daiter.iter_mapaddr, len, dlen); + + for (i = parity-1; i >= 0; i--) { + abd_iter_unmap(&caiters[i]); + abd_iter_advance(&caiters[i], len); + } + + if (dabd && dsize > 0) { + abd_iter_unmap(&daiter); + abd_iter_advance(&daiter, dlen); + dsize -= dlen; + } + + csize -= len; + + ASSERT3S(dsize, >=, 0); + ASSERT3S(csize, >=, 0); + } + critical_exit(); +} + +/* + * Iterate over code ABDs and data reconstruction target ABDs and call + * @func_raidz_rec. Function maps at most 6 pages atomically. + * + * @cabds parity ABDs, must have equal size + * @tabds rec target ABDs, at most 3 + * @tsize size of data target columns + * @func_raidz_rec expects syndrome data in target columns. Function + * reconstructs data and overwrites target columns. + */ +void +abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds, + ssize_t tsize, const unsigned parity, + void (*func_raidz_rec)(void **t, const size_t tsize, void **c, + const unsigned *mul), + const unsigned *mul) +{ + int i; + ssize_t len; + struct abd_iter citers[3]; + struct abd_iter xiters[3]; + void *caddrs[3], *xaddrs[3]; + + ASSERT3U(parity, <=, 3); + + for (i = 0; i < parity; i++) { + abd_iter_init(&citers[i], cabds[i]); + abd_iter_init(&xiters[i], tabds[i]); + } + + critical_enter(); + while (tsize > 0) { + + for (i = 0; i < parity; i++) { + abd_iter_map(&citers[i]); + abd_iter_map(&xiters[i]); + caddrs[i] = citers[i].iter_mapaddr; + xaddrs[i] = xiters[i].iter_mapaddr; + } + + len = tsize; + switch (parity) { + case 3: + len = MIN(xiters[2].iter_mapsize, len); + len = MIN(citers[2].iter_mapsize, len); + case 2: + len = MIN(xiters[1].iter_mapsize, len); + len = MIN(citers[1].iter_mapsize, len); + case 1: + len = MIN(xiters[0].iter_mapsize, len); + len = MIN(citers[0].iter_mapsize, len); + } + /* must be progressive */ + ASSERT3S(len, >, 0); + /* + * The iterated function likely will not do well if each + * segment except the last one is not multiple of 512 (raidz). + */ + ASSERT3U(((uint64_t)len & 511ULL), ==, 0); + + func_raidz_rec(xaddrs, len, caddrs, mul); + + for (i = parity-1; i >= 0; i--) { + abd_iter_unmap(&xiters[i]); + abd_iter_unmap(&citers[i]); + abd_iter_advance(&xiters[i], len); + abd_iter_advance(&citers[i], len); + } + + tsize -= len; + ASSERT3S(tsize, >=, 0); + } + critical_exit(); +} diff --git a/module/os/freebsd/zfs/arc_os.c b/module/os/freebsd/zfs/arc_os.c new file mode 100644 index 00000000000..f0c2724471f --- /dev/null +++ b/module/os/freebsd/zfs/arc_os.c @@ -0,0 +1,245 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern struct vfsops zfs_vfsops; + +/* vmem_size typemask */ +#define VMEM_ALLOC 0x01 +#define VMEM_FREE 0x02 +#define VMEM_MAXFREE 0x10 +typedef size_t vmem_size_t; +extern vmem_size_t vmem_size(vmem_t *vm, int typemask); + +uint_t zfs_arc_free_target = 0; + +int64_t last_free_memory; +free_memory_reason_t last_free_reason; + +int64_t +arc_available_memory(void) +{ + int64_t lowest = INT64_MAX; + int64_t n __unused; + free_memory_reason_t r = FMR_UNKNOWN; + +#ifdef _KERNEL + /* + * Cooperate with pagedaemon when it's time for it to scan + * and reclaim some pages. + */ + n = PAGESIZE * ((int64_t)freemem - zfs_arc_free_target); + if (n < lowest) { + lowest = n; + r = FMR_LOTSFREE; + } +#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC) + /* + * If we're on an i386 platform, it's possible that we'll exhaust the + * kernel heap space before we ever run out of available physical + * memory. Most checks of the size of the heap_area compare against + * tune.t_minarmem, which is the minimum available real memory that we + * can have in the system. However, this is generally fixed at 25 pages + * which is so low that it's useless. In this comparison, we seek to + * calculate the total heap-size, and reclaim if more than 3/4ths of the + * heap is allocated. (Or, in the calculation, if less than 1/4th is + * free) + */ + n = uma_avail() - (long)(uma_limit() / 4); + if (n < lowest) { + lowest = n; + r = FMR_HEAP_ARENA; + } +#endif + + /* + * If zio data pages are being allocated out of a separate heap segment, + * then enforce that the size of available vmem for this arena remains + * above about 1/4th (1/(2^arc_zio_arena_free_shift)) free. + * + * Note that reducing the arc_zio_arena_free_shift keeps more virtual + * memory (in the zio_arena) free, which can avoid memory + * fragmentation issues. + */ + if (zio_arena != NULL) { + n = (int64_t)vmem_size(zio_arena, VMEM_FREE) - + (vmem_size(zio_arena, VMEM_ALLOC) >> + arc_zio_arena_free_shift); + if (n < lowest) { + lowest = n; + r = FMR_ZIO_ARENA; + } + } + +#else /* _KERNEL */ + /* Every 100 calls, free a small amount */ + if (spa_get_random(100) == 0) + lowest = -1024; +#endif /* _KERNEL */ + + last_free_memory = lowest; + last_free_reason = r; + DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r); + return (lowest); +} + +/* + * Return a default max arc size based on the amount of physical memory. + */ +uint64_t +arc_default_max(uint64_t min, uint64_t allmem) +{ + uint64_t size; + + if (allmem >= 1 << 30) + size = allmem - (1 << 30); + else + size = min; + return (MAX(allmem * 5 / 8, size)); +} + +/* + * Helper function for arc_prune_async() it is responsible for safely + * handling the execution of a registered arc_prune_func_t. + */ +static void +arc_prune_task(void *arg) +{ + int64_t nr_scan = *(int64_t *)arg; + + arc_reduce_target_size(ptob(nr_scan)); + free(arg, M_TEMP); + vnlru_free(nr_scan, &zfs_vfsops); +} + +/* + * Notify registered consumers they must drop holds on a portion of the ARC + * buffered they reference. This provides a mechanism to ensure the ARC can + * honor the arc_meta_limit and reclaim otherwise pinned ARC buffers. This + * is analogous to dnlc_reduce_cache() but more generic. + * + * This operation is performed asynchronously so it may be safely called + * in the context of the arc_reclaim_thread(). A reference is taken here + * for each registered arc_prune_t and the arc_prune_task() is responsible + * for releasing it once the registered arc_prune_func_t has completed. + */ +void +arc_prune_async(int64_t adjust) +{ + + int64_t *adjustptr; + + if ((adjustptr = malloc(sizeof (int64_t), M_TEMP, M_NOWAIT)) == NULL) + return; + + *adjustptr = adjust; + taskq_dispatch(arc_prune_taskq, arc_prune_task, adjustptr, TQ_SLEEP); + ARCSTAT_BUMP(arcstat_prune); +} + +uint64_t +arc_all_memory(void) +{ + return ((uint64_t)ptob(physmem)); +} + +int +arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg) +{ + return (0); +} + +uint64_t +arc_free_memory(void) +{ + /* XXX */ + return (0); +} + +static eventhandler_tag arc_event_lowmem = NULL; + +static void +arc_lowmem(void *arg __unused, int howto __unused) +{ + int64_t free_memory, to_free; + + arc_no_grow = B_TRUE; + arc_warm = B_TRUE; + arc_growtime = gethrtime() + SEC2NSEC(arc_grow_retry); + free_memory = arc_available_memory(); + to_free = (arc_c >> arc_shrink_shift) - MIN(free_memory, 0); + DTRACE_PROBE2(arc__needfree, int64_t, free_memory, int64_t, to_free); + arc_reduce_target_size(to_free); + + mutex_enter(&arc_adjust_lock); + arc_adjust_needed = B_TRUE; + zthr_wakeup(arc_adjust_zthr); + + /* + * It is unsafe to block here in arbitrary threads, because we can come + * here from ARC itself and may hold ARC locks and thus risk a deadlock + * with ARC reclaim thread. + */ + if (curproc == pageproc) + (void) cv_wait(&arc_adjust_waiters_cv, &arc_adjust_lock); + mutex_exit(&arc_adjust_lock); +} + +void +arc_lowmem_init(void) +{ + arc_event_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, arc_lowmem, NULL, + EVENTHANDLER_PRI_FIRST); + +} + +void +arc_lowmem_fini(void) +{ + if (arc_event_lowmem != NULL) + EVENTHANDLER_DEREGISTER(vm_lowmem, arc_event_lowmem); +} diff --git a/module/os/freebsd/zfs/crypto_os.c b/module/os/freebsd/zfs/crypto_os.c new file mode 100644 index 00000000000..cc86074c264 --- /dev/null +++ b/module/os/freebsd/zfs/crypto_os.c @@ -0,0 +1,613 @@ +/* + * Copyright (c) 2005-2010 Pawel Jakub Dawidek + * Copyright (c) 2018 Sean Eric Fagan + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#ifdef _KERNEL +#include +#include +#include +#include +#include +#else +#include +#endif + +#include +#include +#include + +#include + +#define SHA512_HMAC_BLOCK_SIZE 128 + +static int crypt_sessions = 0; +SYSCTL_DECL(_vfs_zfs); +SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD, + &crypt_sessions, 0, "Number of cryptographic sessions created"); + +void +crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key) +{ + uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE], + k_opad[SHA512_HMAC_BLOCK_SIZE], + key[SHA512_HMAC_BLOCK_SIZE]; + SHA512_CTX lctx; + int i; + size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length); + + /* + * This code is based on the similar code in geom/eli/g_eli_hmac.c + */ + explicit_bzero(key, sizeof (key)); + if (c_key->ck_length == 0) + /* do nothing */; + else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE) + bcopy(c_key->ck_data, key, cl_bytes); + else { + /* + * If key is longer than 128 bytes reset it to + * key = SHA512(key). + */ + SHA512_Init(&lctx); + SHA512_Update(&lctx, c_key->ck_data, cl_bytes); + SHA512_Final(key, &lctx); + } + + /* XOR key with ipad and opad values. */ + for (i = 0; i < sizeof (key); i++) { + k_ipad[i] = key[i] ^ 0x36; + k_opad[i] = key[i] ^ 0x5c; + } + explicit_bzero(key, sizeof (key)); + + /* Start inner SHA512. */ + SHA512_Init(&ctx->innerctx); + SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad)); + explicit_bzero(k_ipad, sizeof (k_ipad)); + /* Start outer SHA512. */ + SHA512_Init(&ctx->outerctx); + SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad)); + explicit_bzero(k_opad, sizeof (k_opad)); +} + +void +crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize) +{ + SHA512_Update(&ctx->innerctx, data, datasize); +} + +void +crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize) +{ + uint8_t digest[SHA512_DIGEST_LENGTH]; + + /* Complete inner hash */ + SHA512_Final(digest, &ctx->innerctx); + + /* Complete outer hash */ + SHA512_Update(&ctx->outerctx, digest, sizeof (digest)); + SHA512_Final(digest, &ctx->outerctx); + + explicit_bzero(ctx, sizeof (*ctx)); + /* mdsize == 0 means "Give me the whole hash!" */ + if (mdsize == 0) + mdsize = SHA512_DIGEST_LENGTH; + bcopy(digest, md, mdsize); + explicit_bzero(digest, sizeof (digest)); +} + +void +crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size, + void *out_data, size_t out_data_size) +{ + struct hmac_ctx ctx; + + crypto_mac_init(&ctx, key); + crypto_mac_update(&ctx, in_data, in_data_size); + crypto_mac_final(&ctx, out_data, out_data_size); +} + +static int +freebsd_zfs_crypt_done(struct cryptop *crp) +{ + freebsd_crypt_session_t *ses; + + ses = crp->crp_opaque; + mtx_lock(&ses->fs_lock); + ses->fs_done = true; + mtx_unlock(&ses->fs_lock); + wakeup(crp); + return (0); +} + +void +freebsd_crypt_freesession(freebsd_crypt_session_t *sess) +{ + mtx_destroy(&sess->fs_lock); + crypto_freesession(sess->fs_sid); + explicit_bzero(sess, sizeof (*sess)); +} + +static int +zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp) +{ + int error; + + crp->crp_opaque = session; + crp->crp_callback = freebsd_zfs_crypt_done; + for (;;) { + error = crypto_dispatch(crp); + if (error) + break; + mtx_lock(&session->fs_lock); + while (session->fs_done == false) + msleep(crp, &session->fs_lock, PRIBIO, + "zfs_crypto", hz/5); + mtx_unlock(&session->fs_lock); + + if (crp->crp_etype != EAGAIN) { + error = crp->crp_etype; + break; + } + crp->crp_etype = 0; + crp->crp_flags &= ~CRYPTO_F_DONE; + session->fs_done = false; +#if __FreeBSD_version < 1300087 + /* + * Session ID changed, so we should record that, + * and try again + */ + session->fs_sid = crp->crp_session; +#endif + } + return (error); +} +static void +freebsd_crypt_uio_debug_log(boolean_t encrypt, + freebsd_crypt_session_t *input_sessionp, + struct zio_crypt_info *c_info, + uio_t *data_uio, + crypto_key_t *key, + uint8_t *ivbuf, + size_t datalen, + size_t auth_len) +{ +#ifdef FCRYPTO_DEBUG + struct cryptodesc *crd; + uint8_t *p = NULL; + size_t total = 0; + + printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %d, %p, %u }, " + "%p, %u, %u)\n", + __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp, + c_info->ci_algname, c_info->ci_crypt_type, + (unsigned int)c_info->ci_keylen, c_info->ci_name, + data_uio, key->ck_format, key->ck_data, + (unsigned int)key->ck_length, + ivbuf, (unsigned int)datalen, (unsigned int)auth_len); + printf("\tkey = { "); + for (int i = 0; i < key->ck_length / 8; i++) { + uint8_t *b = (uint8_t *)key->ck_data; + printf("%02x ", b[i]); + } + printf("}\n"); + for (int i = 0; i < data_uio->uio_iovcnt; i++) { + printf("\tiovec #%d: <%p, %u>\n", i, + data_uio->uio_iov[i].iov_base, + (unsigned int)data_uio->uio_iov[i].iov_len); + total += data_uio->uio_iov[i].iov_len; + } + data_uio->uio_resid = total; +#endif +} +/* + * Create a new cryptographic session. This should + * happen every time the key changes (including when + * it's first loaded). + */ +#if __FreeBSD_version >= 1300087 +int +freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, + struct zio_crypt_info *c_info, crypto_key_t *key) +{ + struct crypto_session_params csp; + int error = 0; + +#ifdef FCRYPTO_DEBUG + printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n", + __FUNCTION__, sessp, + c_info->ci_algname, c_info->ci_crypt_type, + (unsigned int)c_info->ci_keylen, c_info->ci_name, + key->ck_format, key->ck_data, (unsigned int)key->ck_length); + printf("\tkey = { "); + for (int i = 0; i < key->ck_length / 8; i++) { + uint8_t *b = (uint8_t *)key->ck_data; + printf("%02x ", b[i]); + } + printf("}\n"); +#endif + bzero(&csp, sizeof (csp)); + csp.csp_mode = CSP_MODE_AEAD; + csp.csp_cipher_key = key->ck_data; + csp.csp_cipher_klen = key->ck_length / 8; + switch (c_info->ci_crypt_type) { + case ZC_TYPE_GCM: + csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; + csp.csp_ivlen = AES_GCM_IV_LEN; + switch (key->ck_length/8) { + case AES_128_GMAC_KEY_LEN: + case AES_192_GMAC_KEY_LEN: + case AES_256_GMAC_KEY_LEN: + break; + default: + error = EINVAL; + goto bad; + } + break; + case ZC_TYPE_CCM: + csp.csp_cipher_alg = CRYPTO_AES_CCM_16; + csp.csp_ivlen = AES_CCM_IV_LEN; + switch (key->ck_length/8) { + case AES_128_CBC_MAC_KEY_LEN: + case AES_192_CBC_MAC_KEY_LEN: + case AES_256_CBC_MAC_KEY_LEN: + break; + default: + error = EINVAL; + goto bad; + break; + } + break; + default: + error = ENOTSUP; + goto bad; + } + error = crypto_newsession(&sessp->fs_sid, &csp, + CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); + mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", + NULL, MTX_DEF); + crypt_sessions++; +bad: +#ifdef FCRYPTO_DEBUG + if (error) + printf("%s: returning error %d\n", __FUNCTION__, error); +#endif + return (error); +} + +int +freebsd_crypt_uio(boolean_t encrypt, + freebsd_crypt_session_t *input_sessionp, + struct zio_crypt_info *c_info, + uio_t *data_uio, + crypto_key_t *key, + uint8_t *ivbuf, + size_t datalen, + size_t auth_len) +{ + struct cryptop *crp; + freebsd_crypt_session_t *session = NULL; + int error = 0; + size_t total = 0; + + freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, + key, ivbuf, datalen, auth_len); + for (int i = 0; i < data_uio->uio_iovcnt; i++) + total += data_uio->uio_iov[i].iov_len; + data_uio->uio_resid = total; + if (input_sessionp == NULL) { + session = kmem_zalloc(sizeof (*session), KM_SLEEP); + error = freebsd_crypt_newsession(session, c_info, key); + if (error) + goto out; + } else + session = input_sessionp; + + crp = crypto_getreq(session->fs_sid, M_WAITOK); + if (encrypt) { + crp->crp_op = CRYPTO_OP_ENCRYPT | + CRYPTO_OP_COMPUTE_DIGEST; + } else { + crp->crp_op = CRYPTO_OP_DECRYPT | + CRYPTO_OP_VERIFY_DIGEST; + } + crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE; + crp->crp_buf_type = CRYPTO_BUF_UIO; + crp->crp_uio = (void*)data_uio; + crp->crp_ilen = data_uio->uio_resid; + + crp->crp_aad_start = 0; + crp->crp_aad_length = auth_len; + crp->crp_payload_start = auth_len; + crp->crp_payload_length = datalen; + crp->crp_digest_start = auth_len + datalen; + + bcopy(ivbuf, crp->crp_iv, ZIO_DATA_IV_LEN); + error = zfs_crypto_dispatch(session, crp); + crypto_freereq(crp); +out: +#ifdef FCRYPTO_DEBUG + if (error) + printf("%s: returning error %d\n", __FUNCTION__, error); +#endif + if (input_sessionp == NULL) { + freebsd_crypt_freesession(session); + kmem_free(session, sizeof (*session)); + } + return (error); +} + +#else +int +freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, + struct zio_crypt_info *c_info, crypto_key_t *key) +{ + struct cryptoini cria, crie, *crip; + struct enc_xform *xform; + struct auth_hash *xauth; + int error = 0; + crypto_session_t sid; + +#ifdef FCRYPTO_DEBUG + printf("%s(%p, { %s, %d, %d, %s }, { %d, %p, %u })\n", + __FUNCTION__, sessp, + c_info->ci_algname, c_info->ci_crypt_type, + (unsigned int)c_info->ci_keylen, c_info->ci_name, + key->ck_format, key->ck_data, (unsigned int)key->ck_length); + printf("\tkey = { "); + for (int i = 0; i < key->ck_length / 8; i++) { + uint8_t *b = (uint8_t *)key->ck_data; + printf("%02x ", b[i]); + } + printf("}\n"); +#endif + switch (c_info->ci_crypt_type) { + case ZC_TYPE_GCM: + xform = &enc_xform_aes_nist_gcm; + switch (key->ck_length/8) { + case AES_128_GMAC_KEY_LEN: + xauth = &auth_hash_nist_gmac_aes_128; + break; + case AES_192_GMAC_KEY_LEN: + xauth = &auth_hash_nist_gmac_aes_192; + break; + case AES_256_GMAC_KEY_LEN: + xauth = &auth_hash_nist_gmac_aes_256; + break; + default: + error = EINVAL; + goto bad; + } + break; + case ZC_TYPE_CCM: + xform = &enc_xform_ccm; + switch (key->ck_length/8) { + case AES_128_CBC_MAC_KEY_LEN: + xauth = &auth_hash_ccm_cbc_mac_128; + break; + case AES_192_CBC_MAC_KEY_LEN: + xauth = &auth_hash_ccm_cbc_mac_192; + break; + case AES_256_CBC_MAC_KEY_LEN: + xauth = &auth_hash_ccm_cbc_mac_256; + break; + default: + error = EINVAL; + goto bad; + break; + } + break; + default: + error = ENOTSUP; + goto bad; + } +#ifdef FCRYPTO_DEBUG + printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " + "auth %s (key length %d)\n", + __FUNCTION__, __LINE__, + xform->name, (unsigned int)key->ck_length, + (unsigned int)key->ck_length/8, + xauth->name, xauth->keysize); +#endif + + bzero(&crie, sizeof (crie)); + bzero(&cria, sizeof (cria)); + + crie.cri_alg = xform->type; + crie.cri_key = key->ck_data; + crie.cri_klen = key->ck_length; + + cria.cri_alg = xauth->type; + cria.cri_key = key->ck_data; + cria.cri_klen = key->ck_length; + + cria.cri_next = &crie; + crie.cri_next = NULL; + crip = &cria; + // Everything else is bzero'd + + error = crypto_newsession(&sid, crip, + CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); + if (error != 0) { + printf("%s(%d): crypto_newsession failed with %d\n", + __FUNCTION__, __LINE__, error); + goto bad; + } + sessp->fs_sid = sid; + mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", + NULL, MTX_DEF); + crypt_sessions++; +bad: + return (error); +} + +/* + * The meat of encryption/decryption. + * If sessp is NULL, then it will create a + * temporary cryptographic session, and release + * it when done. + */ +int +freebsd_crypt_uio(boolean_t encrypt, + freebsd_crypt_session_t *input_sessionp, + struct zio_crypt_info *c_info, + uio_t *data_uio, + crypto_key_t *key, + uint8_t *ivbuf, + size_t datalen, + size_t auth_len) +{ + struct cryptop *crp; + struct cryptodesc *enc_desc, *auth_desc; + struct enc_xform *xform; + struct auth_hash *xauth; + freebsd_crypt_session_t *session = NULL; + int error; + + freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, + key, ivbuf, datalen, auth_len); + switch (c_info->ci_crypt_type) { + case ZC_TYPE_GCM: + xform = &enc_xform_aes_nist_gcm; + switch (key->ck_length/8) { + case AES_128_GMAC_KEY_LEN: + xauth = &auth_hash_nist_gmac_aes_128; + break; + case AES_192_GMAC_KEY_LEN: + xauth = &auth_hash_nist_gmac_aes_192; + break; + case AES_256_GMAC_KEY_LEN: + xauth = &auth_hash_nist_gmac_aes_256; + break; + default: + error = EINVAL; + goto bad; + } + break; + case ZC_TYPE_CCM: + xform = &enc_xform_ccm; + switch (key->ck_length/8) { + case AES_128_CBC_MAC_KEY_LEN: + xauth = &auth_hash_ccm_cbc_mac_128; + break; + case AES_192_CBC_MAC_KEY_LEN: + xauth = &auth_hash_ccm_cbc_mac_192; + break; + case AES_256_CBC_MAC_KEY_LEN: + xauth = &auth_hash_ccm_cbc_mac_256; + break; + default: + error = EINVAL; + goto bad; + break; + } + break; + default: + error = ENOTSUP; + goto bad; + } + +#ifdef FCRYPTO_DEBUG + printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " + "auth %s (key length %d)\n", + __FUNCTION__, __LINE__, + xform->name, (unsigned int)key->ck_length, + (unsigned int)key->ck_length/8, + xauth->name, xauth->keysize); +#endif + + if (input_sessionp == NULL) { + session = kmem_zalloc(sizeof (*session), KM_SLEEP); + error = freebsd_crypt_newsession(session, c_info, key); + if (error) + goto out; + } else + session = input_sessionp; + + crp = crypto_getreq(2); + if (crp == NULL) { + error = ENOMEM; + goto bad; + } + + auth_desc = crp->crp_desc; + enc_desc = auth_desc->crd_next; + + crp->crp_session = session->fs_sid; + crp->crp_ilen = auth_len + datalen; + crp->crp_buf = (void*)data_uio; + crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC; + + auth_desc->crd_skip = 0; + auth_desc->crd_len = auth_len; + auth_desc->crd_inject = auth_len + datalen; + auth_desc->crd_alg = xauth->type; +#ifdef FCRYPTO_DEBUG + printf("%s: auth: skip = %u, len = %u, inject = %u\n", + __FUNCTION__, auth_desc->crd_skip, auth_desc->crd_len, + auth_desc->crd_inject); +#endif + + enc_desc->crd_skip = auth_len; + enc_desc->crd_len = datalen; + enc_desc->crd_inject = auth_len; + enc_desc->crd_alg = xform->type; + enc_desc->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; + bcopy(ivbuf, enc_desc->crd_iv, ZIO_DATA_IV_LEN); + enc_desc->crd_next = NULL; + +#ifdef FCRYPTO_DEBUG + printf("%s: enc: skip = %u, len = %u, inject = %u\n", + __FUNCTION__, enc_desc->crd_skip, enc_desc->crd_len, + enc_desc->crd_inject); +#endif + + if (encrypt) + enc_desc->crd_flags |= CRD_F_ENCRYPT; + + error = zfs_crypto_dispatch(session, crp); + crypto_freereq(crp); +out: + if (input_sessionp == NULL) { + freebsd_crypt_freesession(session); + kmem_free(session, sizeof (*session)); + } +bad: +#ifdef FCRYPTO_DEBUG + if (error) + printf("%s: returning error %d\n", __FUNCTION__, error); +#endif + return (error); +} +#endif diff --git a/module/os/freebsd/zfs/dmu_os.c b/module/os/freebsd/zfs/dmu_os.c new file mode 100644 index 00000000000..268c843e50e --- /dev/null +++ b/module/os/freebsd/zfs/dmu_os.c @@ -0,0 +1,346 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#ifndef IDX_TO_OFF +#define IDX_TO_OFF(idx) (((vm_ooffset_t)(idx)) << PAGE_SHIFT) +#endif + +#if __FreeBSD_version < 1300051 +#define VM_ALLOC_BUSY_FLAGS VM_ALLOC_NOBUSY +#else +#define VM_ALLOC_BUSY_FLAGS VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY +#endif + + +#if __FreeBSD_version < 1300072 +#define dmu_page_lock(m) vm_page_lock(m) +#define dmu_page_unlock(m) vm_page_unlock(m) +#else +#define dmu_page_lock(m) +#define dmu_page_unlock(m) +#endif + +static int +dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset, + uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp) +{ + dnode_t *dn; + int err; + + err = dnode_hold(os, object, FTAG, &dn); + if (err) + return (err); + + err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag, + numbufsp, dbpp, DMU_READ_PREFETCH); + + dnode_rele(dn, FTAG); + + return (err); +} + +int +dmu_write_pages(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, + vm_page_t *ma, dmu_tx_t *tx) +{ + dmu_buf_t **dbp; + struct sf_buf *sf; + int numbufs, i; + int err; + + if (size == 0) + return (0); + + err = dmu_buf_hold_array(os, object, offset, size, + FALSE, FTAG, &numbufs, &dbp); + if (err) + return (err); + + for (i = 0; i < numbufs; i++) { + int tocpy, copied, thiscpy; + int bufoff; + dmu_buf_t *db = dbp[i]; + caddr_t va; + + ASSERT(size > 0); + ASSERT3U(db->db_size, >=, PAGESIZE); + + bufoff = offset - db->db_offset; + tocpy = (int)MIN(db->db_size - bufoff, size); + + ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); + + if (tocpy == db->db_size) + dmu_buf_will_fill(db, tx); + else + dmu_buf_will_dirty(db, tx); + + for (copied = 0; copied < tocpy; copied += PAGESIZE) { + ASSERT3U(ptoa((*ma)->pindex), ==, + db->db_offset + bufoff); + thiscpy = MIN(PAGESIZE, tocpy - copied); + va = zfs_map_page(*ma, &sf); + bcopy(va, (char *)db->db_data + bufoff, thiscpy); + zfs_unmap_page(sf); + ma += 1; + bufoff += PAGESIZE; + } + + if (tocpy == db->db_size) + dmu_buf_fill_done(db, tx); + + offset += tocpy; + size -= tocpy; + } + dmu_buf_rele_array(dbp, numbufs, FTAG); + return (err); +} + +int +dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count, + int *rbehind, int *rahead, int last_size) +{ + struct sf_buf *sf; + vm_object_t vmobj; + vm_page_t m; + dmu_buf_t **dbp; + dmu_buf_t *db; + caddr_t va; + int numbufs, i; + int bufoff, pgoff, tocpy; + int mi, di; + int err; + + ASSERT3U(ma[0]->pindex + count - 1, ==, ma[count - 1]->pindex); + ASSERT(last_size <= PAGE_SIZE); + + err = dmu_buf_hold_array(os, object, IDX_TO_OFF(ma[0]->pindex), + IDX_TO_OFF(count - 1) + last_size, TRUE, FTAG, &numbufs, &dbp); + if (err != 0) + return (err); + +#ifdef DEBUG + IMPLY(last_size < PAGE_SIZE, *rahead == 0); + if (dbp[0]->db_offset != 0 || numbufs > 1) { + for (i = 0; i < numbufs; i++) { + ASSERT(ISP2(dbp[i]->db_size)); + ASSERT((dbp[i]->db_offset % dbp[i]->db_size) == 0); + ASSERT3U(dbp[i]->db_size, ==, dbp[0]->db_size); + } + } +#endif + + vmobj = ma[0]->object; + zfs_vmobject_wlock(vmobj); + + db = dbp[0]; + for (i = 0; i < *rbehind; i++) { + m = vm_page_grab(vmobj, ma[0]->pindex - 1 - i, + VM_ALLOC_NORMAL | VM_ALLOC_NOWAIT | VM_ALLOC_BUSY_FLAGS); + if (m == NULL) + break; + if (!vm_page_none_valid(m)) { + ASSERT3U(m->valid, ==, VM_PAGE_BITS_ALL); + vm_page_do_sunbusy(m); + break; + } + ASSERT(m->dirty == 0); + ASSERT(!pmap_page_is_mapped(m)); + + ASSERT(db->db_size > PAGE_SIZE); + bufoff = IDX_TO_OFF(m->pindex) % db->db_size; + va = zfs_map_page(m, &sf); + bcopy((char *)db->db_data + bufoff, va, PAGESIZE); + zfs_unmap_page(sf); + vm_page_valid(m); + dmu_page_lock(m); + if ((m->busy_lock & VPB_BIT_WAITERS) != 0) + vm_page_activate(m); + else + vm_page_deactivate(m); + dmu_page_unlock(m); + vm_page_do_sunbusy(m); + } + *rbehind = i; + + bufoff = IDX_TO_OFF(ma[0]->pindex) % db->db_size; + pgoff = 0; + for (mi = 0, di = 0; mi < count && di < numbufs; ) { + if (pgoff == 0) { + m = ma[mi]; + if (m != bogus_page) { + vm_page_assert_xbusied(m); + ASSERT(vm_page_none_valid(m)); + ASSERT(m->dirty == 0); + ASSERT(!pmap_page_is_mapped(m)); + va = zfs_map_page(m, &sf); + } + } + if (bufoff == 0) + db = dbp[di]; + + if (m != bogus_page) { + ASSERT3U(IDX_TO_OFF(m->pindex) + pgoff, ==, + db->db_offset + bufoff); + } + + /* + * We do not need to clamp the copy size by the file + * size as the last block is zero-filled beyond the + * end of file anyway. + */ + tocpy = MIN(db->db_size - bufoff, PAGESIZE - pgoff); + if (m != bogus_page) + bcopy((char *)db->db_data + bufoff, va + pgoff, tocpy); + + pgoff += tocpy; + ASSERT(pgoff <= PAGESIZE); + if (pgoff == PAGESIZE) { + if (m != bogus_page) { + zfs_unmap_page(sf); + vm_page_valid(m); + } + ASSERT(mi < count); + mi++; + pgoff = 0; + } + + bufoff += tocpy; + ASSERT(bufoff <= db->db_size); + if (bufoff == db->db_size) { + ASSERT(di < numbufs); + di++; + bufoff = 0; + } + } + +#ifdef DEBUG + /* + * Three possibilities: + * - last requested page ends at a buffer boundary and , thus, + * all pages and buffers have been iterated; + * - all requested pages are filled, but the last buffer + * has not been exhausted; + * the read-ahead is possible only in this case; + * - all buffers have been read, but the last page has not been + * fully filled; + * this is only possible if the file has only a single buffer + * with a size that is not a multiple of the page size. + */ + if (mi == count) { + ASSERT(di >= numbufs - 1); + IMPLY(*rahead != 0, di == numbufs - 1); + IMPLY(*rahead != 0, bufoff != 0); + ASSERT(pgoff == 0); + } + if (di == numbufs) { + ASSERT(mi >= count - 1); + ASSERT(*rahead == 0); + IMPLY(pgoff == 0, mi == count); + if (pgoff != 0) { + ASSERT(mi == count - 1); + ASSERT((dbp[0]->db_size & PAGE_MASK) != 0); + } + } +#endif + if (pgoff != 0) { + ASSERT(m != bogus_page); + bzero(va + pgoff, PAGESIZE - pgoff); + zfs_unmap_page(sf); + vm_page_valid(m); + } + + for (i = 0; i < *rahead; i++) { + m = vm_page_grab(vmobj, ma[count - 1]->pindex + 1 + i, + VM_ALLOC_NORMAL | VM_ALLOC_NOWAIT | VM_ALLOC_BUSY_FLAGS); + if (m == NULL) + break; + if (!vm_page_none_valid(m)) { + ASSERT3U(m->valid, ==, VM_PAGE_BITS_ALL); + vm_page_do_sunbusy(m); + break; + } + ASSERT(m->dirty == 0); + ASSERT(!pmap_page_is_mapped(m)); + + ASSERT(db->db_size > PAGE_SIZE); + bufoff = IDX_TO_OFF(m->pindex) % db->db_size; + tocpy = MIN(db->db_size - bufoff, PAGESIZE); + va = zfs_map_page(m, &sf); + bcopy((char *)db->db_data + bufoff, va, tocpy); + if (tocpy < PAGESIZE) { + ASSERT(i == *rahead - 1); + ASSERT((db->db_size & PAGE_MASK) != 0); + bzero(va + tocpy, PAGESIZE - tocpy); + } + zfs_unmap_page(sf); + vm_page_valid(m); + dmu_page_lock(m); + if ((m->busy_lock & VPB_BIT_WAITERS) != 0) + vm_page_activate(m); + else + vm_page_deactivate(m); + dmu_page_unlock(m); + vm_page_do_sunbusy(m); + } + *rahead = i; + zfs_vmobject_wunlock(vmobj); + + dmu_buf_rele_array(dbp, numbufs, FTAG); + return (0); +} diff --git a/module/os/freebsd/zfs/hkdf.c b/module/os/freebsd/zfs/hkdf.c new file mode 100644 index 00000000000..8324ff2319b --- /dev/null +++ b/module/os/freebsd/zfs/hkdf.c @@ -0,0 +1,102 @@ +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2017, Datto, Inc. All rights reserved. + */ + +#include +#include +#include +#include + +static int +hkdf_sha512_extract(uint8_t *salt, uint_t salt_len, uint8_t *key_material, + uint_t km_len, uint8_t *out_buf) +{ + crypto_key_t key; + + /* initialize the salt as a crypto key */ + key.ck_format = CRYPTO_KEY_RAW; + key.ck_length = CRYPTO_BYTES2BITS(salt_len); + key.ck_data = salt; + + crypto_mac(&key, key_material, km_len, out_buf, SHA512_DIGEST_LENGTH); + + return (0); +} + +static int +hkdf_sha512_expand(uint8_t *extract_key, uint8_t *info, uint_t info_len, + uint8_t *out_buf, uint_t out_len) +{ + struct hmac_ctx ctx; + crypto_key_t key; + uint_t i, T_len = 0, pos = 0; + uint8_t c; + uint_t N = (out_len + SHA512_DIGEST_LENGTH) / SHA512_DIGEST_LENGTH; + uint8_t T[SHA512_DIGEST_LENGTH]; + + if (N > 255) + return (SET_ERROR(EINVAL)); + + /* initialize the salt as a crypto key */ + key.ck_format = CRYPTO_KEY_RAW; + key.ck_length = CRYPTO_BYTES2BITS(SHA512_DIGEST_LENGTH); + key.ck_data = extract_key; + + for (i = 1; i <= N; i++) { + c = i; + + crypto_mac_init(&ctx, &key); + crypto_mac_update(&ctx, T, T_len); + crypto_mac_update(&ctx, info, info_len); + crypto_mac_update(&ctx, &c, 1); + crypto_mac_final(&ctx, T, SHA512_DIGEST_LENGTH); + bcopy(T, out_buf + pos, + (i != N) ? SHA512_DIGEST_LENGTH : (out_len - pos)); + pos += SHA512_DIGEST_LENGTH; + } + + return (0); +} + +/* + * HKDF is designed to be a relatively fast function for deriving keys from a + * master key + a salt. We use this function to generate new encryption keys + * so as to avoid hitting the cryptographic limits of the underlying + * encryption modes. Note that, for the sake of deriving encryption keys, the + * info parameter is called the "salt" everywhere else in the code. + */ +int +hkdf_sha512(uint8_t *key_material, uint_t km_len, uint8_t *salt, + uint_t salt_len, uint8_t *info, uint_t info_len, uint8_t *output_key, + uint_t out_len) +{ + int ret; + uint8_t extract_key[SHA512_DIGEST_LENGTH]; + + ret = hkdf_sha512_extract(salt, salt_len, key_material, km_len, + extract_key); + if (ret != 0) + return (ret); + + ret = hkdf_sha512_expand(extract_key, info, info_len, output_key, + out_len); + if (ret != 0) + return (ret); + + return (0); +} diff --git a/module/os/freebsd/zfs/kmod_core.c b/module/os/freebsd/zfs/kmod_core.c new file mode 100644 index 00000000000..10807afa3d3 --- /dev/null +++ b/module/os/freebsd/zfs/kmod_core.c @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "zfs_namecheck.h" +#include "zfs_prop.h" +#include "zfs_deleg.h" +#include "zfs_comutil.h" + +SYSCTL_DECL(_vfs_zfs); +SYSCTL_DECL(_vfs_zfs_vdev); + + +static int zfs_version_ioctl = ZFS_IOCVER_ZOF; +SYSCTL_DECL(_vfs_zfs_version); +SYSCTL_INT(_vfs_zfs_version, OID_AUTO, ioctl, CTLFLAG_RD, &zfs_version_ioctl, + 0, "ZFS_IOCTL_VERSION"); + +static struct cdev *zfsdev; + +extern void zfs_init(void); +extern void zfs_fini(void); +extern void zfs_ioctl_init(void); + + +static struct root_hold_token *zfs_root_token; + +extern uint_t rrw_tsd_key; +extern uint_t zfs_allow_log_key; +extern uint_t zfs_geom_probe_vdev_key; + +static int zfs__init(void); +static int zfs__fini(void); +static void zfs_shutdown(void *, int); + +static eventhandler_tag zfs_shutdown_event_tag; +extern zfsdev_state_t *zfsdev_state_list; + +#define ZFS_MIN_KSTACK_PAGES 4 + +static void +zfs_cmd_bsd12_to_zof(zfs_cmd_legacy_t *src, zfs_cmd_t *dst) +{ + memcpy(dst, src, offsetof(zfs_cmd_t, zc_objset_stats)); + *&dst->zc_objset_stats = *&src->zc_objset_stats; + memcpy(&dst->zc_begin_record, &src->zc_begin_record, + offsetof(zfs_cmd_t, zc_sendobj) - + offsetof(zfs_cmd_t, zc_begin_record)); + memcpy(&dst->zc_sendobj, &src->zc_sendobj, + sizeof (zfs_cmd_t) - 8 - offsetof(zfs_cmd_t, zc_sendobj)); + dst->zc_zoneid = src->zc_jailid; +} + +static void +zfs_cmd_zof_to_bsd12(zfs_cmd_t *src, zfs_cmd_legacy_t *dst) +{ + memcpy(dst, src, offsetof(zfs_cmd_t, zc_objset_stats)); + *&dst->zc_objset_stats = *&src->zc_objset_stats; + memcpy(&dst->zc_begin_record, &src->zc_begin_record, + offsetof(zfs_cmd_t, zc_sendobj) - + offsetof(zfs_cmd_t, zc_begin_record)); + memcpy(&dst->zc_sendobj, &src->zc_sendobj, + sizeof (zfs_cmd_t) - 8 - offsetof(zfs_cmd_t, zc_sendobj)); + dst->zc_jailid = src->zc_zoneid; +} + +static int +zfsdev_ioctl(struct cdev *dev, ulong_t zcmd, caddr_t arg, int flag, + struct thread *td) +{ + uint_t len, vecnum; + zfs_iocparm_t *zp; + zfs_cmd_t *zc; + zfs_cmd_legacy_t *zcl; + int rc, error; + void *uaddr; + + len = IOCPARM_LEN(zcmd); + vecnum = zcmd & 0xff; + zp = (void *)arg; + uaddr = (void *)zp->zfs_cmd; + error = 0; + zcl = NULL; + + if (len != sizeof (zfs_iocparm_t)) { + printf("len %d vecnum: %d sizeof (zfs_cmd_t) %lu\n", + len, vecnum, sizeof (zfs_cmd_t)); + return (EINVAL); + } + + zc = kmem_zalloc(sizeof (zfs_cmd_t), KM_SLEEP); + /* + * Remap ioctl code for legacy user binaries + */ + if (zp->zfs_ioctl_version == ZFS_IOCVER_FREEBSD) { + if (vecnum >= sizeof (zfs_ioctl_bsd12_to_zof)/sizeof (long)) { + kmem_free(zc, sizeof (zfs_cmd_t)); + return (ENOTSUP); + } + zcl = kmem_zalloc(sizeof (zfs_cmd_legacy_t), KM_SLEEP); + vecnum = zfs_ioctl_bsd12_to_zof[vecnum]; + if (copyin(uaddr, zcl, sizeof (zfs_cmd_legacy_t))) { + error = SET_ERROR(EFAULT); + goto out; + } + zfs_cmd_bsd12_to_zof(zcl, zc); + } else if (copyin(uaddr, zc, sizeof (zfs_cmd_t))) { + error = SET_ERROR(EFAULT); + goto out; + } + error = zfsdev_ioctl_common(vecnum, zc); + if (zcl) { + zfs_cmd_zof_to_bsd12(zc, zcl); + rc = copyout(zcl, uaddr, sizeof (*zcl)); + } else { + rc = copyout(zc, uaddr, sizeof (*zc)); + } + if (error == 0 && rc != 0) + error = SET_ERROR(EFAULT); +out: + if (zcl) + kmem_free(zcl, sizeof (zfs_cmd_legacy_t)); + kmem_free(zc, sizeof (zfs_cmd_t)); + return (error); +} + +static void +zfsdev_close(void *data) +{ + zfsdev_state_t *zs, *zsp = data; + + mutex_enter(&zfsdev_state_lock); + for (zs = zfsdev_state_list; zs != NULL; zs = zs->zs_next) { + if (zs == zsp) + break; + } + if (zs == NULL || zs->zs_minor <= 0) { + mutex_exit(&zfsdev_state_lock); + return; + } + zs->zs_minor = -1; + zfs_onexit_destroy(zs->zs_onexit); + zfs_zevent_destroy(zs->zs_zevent); + mutex_exit(&zfsdev_state_lock); +} + +static int +zfs_ctldev_init(struct cdev *devp) +{ + boolean_t newzs = B_FALSE; + minor_t minor; + zfsdev_state_t *zs, *zsprev = NULL; + + ASSERT(MUTEX_HELD(&zfsdev_state_lock)); + + minor = zfsdev_minor_alloc(); + if (minor == 0) + return (SET_ERROR(ENXIO)); + + for (zs = zfsdev_state_list; zs != NULL; zs = zs->zs_next) { + if (zs->zs_minor == -1) + break; + zsprev = zs; + } + + if (!zs) { + zs = kmem_zalloc(sizeof (zfsdev_state_t), KM_SLEEP); + newzs = B_TRUE; + } + + devfs_set_cdevpriv(zs, zfsdev_close); + + zfs_onexit_init((zfs_onexit_t **)&zs->zs_onexit); + zfs_zevent_init((zfs_zevent_t **)&zs->zs_zevent); + + if (newzs) { + zs->zs_minor = minor; + wmb(); + zsprev->zs_next = zs; + } else { + wmb(); + zs->zs_minor = minor; + } + return (0); +} + +static int +zfsdev_open(struct cdev *devp, int flag, int mode, struct thread *td) +{ + int error; + + mutex_enter(&zfsdev_state_lock); + error = zfs_ctldev_init(devp); + mutex_exit(&zfsdev_state_lock); + + return (error); +} + +static struct cdevsw zfs_cdevsw = { + .d_version = D_VERSION, + .d_open = zfsdev_open, + .d_ioctl = zfsdev_ioctl, + .d_name = ZFS_DRIVER +}; + +int +zfsdev_attach(void) +{ + zfsdev = make_dev(&zfs_cdevsw, 0x0, UID_ROOT, GID_OPERATOR, 0666, + ZFS_DRIVER); + return (0); +} + +void +zfsdev_detach(void) +{ + if (zfsdev != NULL) + destroy_dev(zfsdev); +} + +int +zfs__init(void) +{ + int error; + +#if KSTACK_PAGES < ZFS_MIN_KSTACK_PAGES + printf("ZFS NOTICE: KSTACK_PAGES is %d which could result in stack " + "overflow panic!\nPlease consider adding " + "'options KSTACK_PAGES=%d' to your kernel config\n", KSTACK_PAGES, + ZFS_MIN_KSTACK_PAGES); +#endif + zfs_root_token = root_mount_hold("ZFS"); + if ((error = zfs_kmod_init()) != 0) { + printf("ZFS: Failed to Load ZFS Filesystem" + ", rc = %d\n", error); + root_mount_rel(zfs_root_token); + return (error); + } + + + tsd_create(&zfs_geom_probe_vdev_key, NULL); + + printf("ZFS storage pool version: features support (" + SPA_VERSION_STRING ")\n"); + root_mount_rel(zfs_root_token); + ddi_sysevent_init(); + return (0); +} + +int +zfs__fini(void) +{ + if (zfs_busy() || zvol_busy() || + zio_injection_enabled) { + return (EBUSY); + } + zfs_kmod_fini(); + tsd_destroy(&zfs_geom_probe_vdev_key); + return (0); +} + +static void +zfs_shutdown(void *arg __unused, int howto __unused) +{ + + /* + * ZFS fini routines can not properly work in a panic-ed system. + */ + if (panicstr == NULL) + zfs__fini(); +} + + +static int +zfs_modevent(module_t mod, int type, void *unused __unused) +{ + int err; + + switch (type) { + case MOD_LOAD: + err = zfs__init(); + if (err == 0) + zfs_shutdown_event_tag = EVENTHANDLER_REGISTER( + shutdown_post_sync, zfs_shutdown, NULL, + SHUTDOWN_PRI_FIRST); + return (err); + case MOD_UNLOAD: + err = zfs__fini(); + if (err == 0 && zfs_shutdown_event_tag != NULL) + EVENTHANDLER_DEREGISTER(shutdown_post_sync, + zfs_shutdown_event_tag); + return (err); + case MOD_SHUTDOWN: + return (0); + default: + break; + } + return (EOPNOTSUPP); +} + +static moduledata_t zfs_mod = { + "zfsctrl", + zfs_modevent, + 0 +}; + +#ifdef _KERNEL +EVENTHANDLER_DEFINE(mountroot, spa_boot_init, NULL, 0); +#endif + +DECLARE_MODULE(zfsctrl, zfs_mod, SI_SUB_CLOCKS, SI_ORDER_ANY); +MODULE_VERSION(zfsctrl, 1); +MODULE_DEPEND(zfsctrl, krpc, 1, 1, 1); +MODULE_DEPEND(zfsctrl, acl_nfs4, 1, 1, 1); +MODULE_DEPEND(zfsctrl, crypto, 1, 1, 1); +MODULE_DEPEND(zfsctrl, cryptodev, 1, 1, 1); diff --git a/module/os/freebsd/zfs/spa_os.c b/module/os/freebsd/zfs/spa_os.c new file mode 100644 index 00000000000..ed124a5faf8 --- /dev/null +++ b/module/os/freebsd/zfs/spa_os.c @@ -0,0 +1,280 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011 by Delphix. All rights reserved. + * Copyright (c) 2013 Martin Matuska . All rights reserved. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zfs_prop.h" +#include "zfs_comutil.h" + +extern int vdev_geom_read_pool_label(const char *name, nvlist_t ***configs, + uint64_t *count); + +static nvlist_t * +spa_generate_rootconf(const char *name) +{ + nvlist_t **configs, **tops; + nvlist_t *config; + nvlist_t *best_cfg, *nvtop, *nvroot; + uint64_t *holes; + uint64_t best_txg; + uint64_t nchildren; + uint64_t pgid; + uint64_t count; + uint64_t i; + uint_t nholes; + + if (vdev_geom_read_pool_label(name, &configs, &count) != 0) + return (NULL); + + ASSERT3U(count, !=, 0); + best_txg = 0; + for (i = 0; i < count; i++) { + uint64_t txg; + + VERIFY(nvlist_lookup_uint64(configs[i], ZPOOL_CONFIG_POOL_TXG, + &txg) == 0); + if (txg > best_txg) { + best_txg = txg; + best_cfg = configs[i]; + } + } + + nchildren = 1; + nvlist_lookup_uint64(best_cfg, ZPOOL_CONFIG_VDEV_CHILDREN, &nchildren); + holes = NULL; + nvlist_lookup_uint64_array(best_cfg, ZPOOL_CONFIG_HOLE_ARRAY, + &holes, &nholes); + + tops = kmem_zalloc(nchildren * sizeof (void *), KM_SLEEP); + for (i = 0; i < nchildren; i++) { + if (i >= count) + break; + if (configs[i] == NULL) + continue; + VERIFY(nvlist_lookup_nvlist(configs[i], ZPOOL_CONFIG_VDEV_TREE, + &nvtop) == 0); + nvlist_dup(nvtop, &tops[i], KM_SLEEP); + } + for (i = 0; holes != NULL && i < nholes; i++) { + if (i >= nchildren) + continue; + if (tops[holes[i]] != NULL) + continue; + nvlist_alloc(&tops[holes[i]], NV_UNIQUE_NAME, KM_SLEEP); + VERIFY(nvlist_add_string(tops[holes[i]], ZPOOL_CONFIG_TYPE, + VDEV_TYPE_HOLE) == 0); + VERIFY(nvlist_add_uint64(tops[holes[i]], ZPOOL_CONFIG_ID, + holes[i]) == 0); + VERIFY(nvlist_add_uint64(tops[holes[i]], ZPOOL_CONFIG_GUID, + 0) == 0); + } + for (i = 0; i < nchildren; i++) { + if (tops[i] != NULL) + continue; + nvlist_alloc(&tops[i], NV_UNIQUE_NAME, KM_SLEEP); + VERIFY(nvlist_add_string(tops[i], ZPOOL_CONFIG_TYPE, + VDEV_TYPE_MISSING) == 0); + VERIFY(nvlist_add_uint64(tops[i], ZPOOL_CONFIG_ID, + i) == 0); + VERIFY(nvlist_add_uint64(tops[i], ZPOOL_CONFIG_GUID, + 0) == 0); + } + + /* + * Create pool config based on the best vdev config. + */ + nvlist_dup(best_cfg, &config, KM_SLEEP); + + /* + * Put this pool's top-level vdevs into a root vdev. + */ + VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, + &pgid) == 0); + VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_add_string(nvroot, ZPOOL_CONFIG_TYPE, + VDEV_TYPE_ROOT) == 0); + VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_ID, 0ULL) == 0); + VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_GUID, pgid) == 0); + VERIFY(nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, + tops, nchildren) == 0); + + /* + * Replace the existing vdev_tree with the new root vdev in + * this pool's configuration (remove the old, add the new). + */ + VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, nvroot) == 0); + + /* + * Drop vdev config elements that should not be present at pool level. + */ + nvlist_remove(config, ZPOOL_CONFIG_GUID, DATA_TYPE_UINT64); + nvlist_remove(config, ZPOOL_CONFIG_TOP_GUID, DATA_TYPE_UINT64); + + for (i = 0; i < count; i++) + nvlist_free(configs[i]); + kmem_free(configs, count * sizeof (void *)); + for (i = 0; i < nchildren; i++) + nvlist_free(tops[i]); + kmem_free(tops, nchildren * sizeof (void *)); + nvlist_free(nvroot); + return (config); +} + +int +spa_import_rootpool(const char *name) +{ + spa_t *spa; + vdev_t *rvd; + nvlist_t *config, *nvtop; + uint64_t txg; + char *pname; + int error; + + /* + * Read the label from the boot device and generate a configuration. + */ + config = spa_generate_rootconf(name); + + mutex_enter(&spa_namespace_lock); + if (config != NULL) { + VERIFY(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME, + &pname) == 0 && strcmp(name, pname) == 0); + VERIFY(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, &txg) + == 0); + + if ((spa = spa_lookup(pname)) != NULL) { + /* + * The pool could already be imported, + * e.g., after reboot -r. + */ + if (spa->spa_state == POOL_STATE_ACTIVE) { + mutex_exit(&spa_namespace_lock); + nvlist_free(config); + return (0); + } + + /* + * Remove the existing root pool from the namespace so + * that we can replace it with the correct config + * we just read in. + */ + spa_remove(spa); + } + spa = spa_add(pname, config, NULL); + + /* + * Set spa_ubsync.ub_version as it can be used in vdev_alloc() + * via spa_version(). + */ + if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, + &spa->spa_ubsync.ub_version) != 0) + spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; + } else if ((spa = spa_lookup(name)) == NULL) { + mutex_exit(&spa_namespace_lock); + nvlist_free(config); + cmn_err(CE_NOTE, "Cannot find the pool label for '%s'", + name); + return (EIO); + } else { + VERIFY(nvlist_dup(spa->spa_config, &config, KM_SLEEP) == 0); + } + spa->spa_is_root = B_TRUE; + spa->spa_import_flags = ZFS_IMPORT_VERBATIM; + + /* + * Build up a vdev tree based on the boot device's label config. + */ + VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, + &nvtop) == 0); + spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); + error = spa_config_parse(spa, &rvd, nvtop, NULL, 0, + VDEV_ALLOC_ROOTPOOL); + spa_config_exit(spa, SCL_ALL, FTAG); + if (error) { + mutex_exit(&spa_namespace_lock); + nvlist_free(config); + cmn_err(CE_NOTE, "Can not parse the config for pool '%s'", + pname); + return (error); + } + + spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); + vdev_free(rvd); + spa_config_exit(spa, SCL_ALL, FTAG); + mutex_exit(&spa_namespace_lock); + + nvlist_free(config); + return (0); +} + +const char * +spa_history_zone(void) +{ + return ("freebsd"); +} diff --git a/module/os/freebsd/zfs/spa_stats.c b/module/os/freebsd/zfs/spa_stats.c new file mode 100644 index 00000000000..45c880ada24 --- /dev/null +++ b/module/os/freebsd/zfs/spa_stats.c @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include + +void +spa_stats_init(spa_t *spa) +{ + +} + +void +spa_stats_destroy(spa_t *spa) +{ + +} + +void +spa_iostats_trim_add(spa_t *spa, trim_type_t type, + uint64_t extents_written, uint64_t bytes_written, + uint64_t extents_skipped, uint64_t bytes_skipped, + uint64_t extents_failed, uint64_t bytes_failed) +{ +} + +void +spa_read_history_add(spa_t *spa, const zbookmark_phys_t *zb, uint32_t aflags) +{ +} + +void +spa_txg_history_add(spa_t *spa, uint64_t txg, hrtime_t birth_time) +{ + +} +/* + * Set txg state completion time and increment current state. + */ +int +spa_txg_history_set(spa_t *spa, uint64_t txg, txg_state_t completed_state, + hrtime_t completed_time) +{ + return (0); +} + +txg_stat_t * +spa_txg_history_init_io(spa_t *spa, uint64_t txg, dsl_pool_t *dp) +{ + return (NULL); +} + +void +spa_txg_history_fini_io(spa_t *spa, txg_stat_t *ts) +{ + +} + +void +spa_tx_assign_add_nsecs(spa_t *spa, uint64_t nsecs) +{ + +} + +void +spa_mmp_history_add(spa_t *spa, uint64_t txg, uint64_t timestamp, + uint64_t mmp_delay, vdev_t *vd, int label, uint64_t mmp_node_id, + int error) +{ + +} + +int +spa_mmp_history_set(spa_t *spa, uint64_t mmp_node_id, int io_error, + hrtime_t duration) +{ + return (0); +} + +int +spa_mmp_history_set_skip(spa_t *spa, uint64_t mmp_node_id) +{ + return (0); +} diff --git a/module/os/freebsd/zfs/sysctl_os.c b/module/os/freebsd/zfs/sysctl_os.c new file mode 100644 index 00000000000..ea9c1b3f1f1 --- /dev/null +++ b/module/os/freebsd/zfs/sysctl_os.c @@ -0,0 +1,699 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include <../zfs_config.h> + +/* BEGIN CSTYLED */ +SYSCTL_DECL(_vfs_zfs); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, zevent, CTLFLAG_RW, 0, "ZFS events"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, zil, CTLFLAG_RW, 0, "ZFS ZIL"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, vdev, CTLFLAG_RW, 0, "ZFS VDEV"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, trim, CTLFLAG_RW, 0, "ZFS TRIM"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, spa, CTLFLAG_RW, 0, "space allocation"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, reconstruct, CTLFLAG_RW, 0, "reconstruct"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, prefetch, CTLFLAG_RW, 0, "ZFS ZFETCH"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, multihost, CTLFLAG_RW, 0, "multihost protection"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, mg, CTLFLAG_RW, 0, "metaslab group"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, metaslab, CTLFLAG_RW, 0, "ZFS metaslab"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, lua, CTLFLAG_RW, 0, "lua"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, l2arc, CTLFLAG_RW, 0, "l2arc"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, dbuf, CTLFLAG_RW, 0, "ZFS disk buf cache"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, dbuf_cache, CTLFLAG_RW, 0, "ZFS disk buf cache"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, deadman, CTLFLAG_RW, 0, "ZFS deadman"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, condense, CTLFLAG_RW, 0, "ZFS condense"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, arc, CTLFLAG_RW, 0, "ZFS Adaptive Replacement Cache"); + +SYSCTL_NODE(_vfs_zfs_vdev, OID_AUTO, mirror, CTLFLAG_RD, 0, + "ZFS VDEV Mirror"); +SYSCTL_NODE(_vfs_zfs_vdev, OID_AUTO, cache, CTLFLAG_RW, 0, "ZFS VDEV Cache"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, livelist, CTLFLAG_RW, 0, "livelist state"); +SYSCTL_NODE(_vfs_zfs_livelist, OID_AUTO, condense, CTLFLAG_RW, 0, "condense knobs"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, recv, CTLFLAG_RW, 0, "receive knobs"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, send, CTLFLAG_RW, 0, "send knobs"); + +SYSCTL_DECL(_vfs_zfs_version); +SYSCTL_CONST_STRING(_vfs_zfs_version, OID_AUTO, module, CTLFLAG_RD, + (ZFS_META_VERSION "-" ZFS_META_RELEASE), "OpenZFS module version"); + +extern arc_state_t ARC_anon; +extern arc_state_t ARC_mru; +extern arc_state_t ARC_mru_ghost; +extern arc_state_t ARC_mfu; +extern arc_state_t ARC_mfu_ghost; +extern arc_state_t ARC_l2c_only; + +/* + * minimum lifespan of a prefetch block in clock ticks + * (initialized in arc_init()) + */ + +/* arc.c */ + +/* legacy compat */ +extern unsigned long l2arc_write_max; /* def max write size */ +extern unsigned long l2arc_write_boost; /* extra warmup write */ +extern unsigned long l2arc_headroom; /* # of dev writes */ +extern unsigned long l2arc_headroom_boost; +extern unsigned long l2arc_feed_secs; /* interval seconds */ +extern unsigned long l2arc_feed_min_ms; /* min interval msecs */ +extern int l2arc_noprefetch; /* don't cache prefetch bufs */ +extern int l2arc_feed_again; /* turbo warmup */ +extern int l2arc_norw; /* no reads during writes */ + +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_write_max, CTLFLAG_RW, + &l2arc_write_max, 0, "max write size (LEGACY)"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_write_boost, CTLFLAG_RW, + &l2arc_write_boost, 0, "extra write during warmup (LEGACY)"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_headroom, CTLFLAG_RW, + &l2arc_headroom, 0, "number of dev writes (LEGACY)"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_feed_secs, CTLFLAG_RW, + &l2arc_feed_secs, 0, "interval seconds (LEGACY)"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2arc_feed_min_ms, CTLFLAG_RW, + &l2arc_feed_min_ms, 0, "min interval milliseconds (LEGACY)"); + +SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_noprefetch, CTLFLAG_RW, + &l2arc_noprefetch, 0, "don't cache prefetch bufs (LEGACY)"); +SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_feed_again, CTLFLAG_RW, + &l2arc_feed_again, 0, "turbo warmup (LEGACY)"); +SYSCTL_INT(_vfs_zfs, OID_AUTO, l2arc_norw, CTLFLAG_RW, + &l2arc_norw, 0, "no reads during writes (LEGACY)"); +#if 0 +extern int zfs_compressed_arc_enabled; +SYSCTL_INT(_vfs_zfs, OID_AUTO, compressed_arc_enabled, CTLFLAG_RW, + &zfs_compressed_arc_enabled, 1, "compressed arc buffers (LEGACY)"); +#endif + +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_size, CTLFLAG_RD, + &ARC_anon.arcs_size.rc_count, 0, "size of anonymous state"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_metadata_esize, CTLFLAG_RD, + &ARC_anon.arcs_esize[ARC_BUFC_METADATA].rc_count, 0, + "size of anonymous state"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, anon_data_esize, CTLFLAG_RD, + &ARC_anon.arcs_esize[ARC_BUFC_DATA].rc_count, 0, + "size of anonymous state"); + +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_size, CTLFLAG_RD, + &ARC_mru.arcs_size.rc_count, 0, "size of mru state"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_metadata_esize, CTLFLAG_RD, + &ARC_mru.arcs_esize[ARC_BUFC_METADATA].rc_count, 0, + "size of metadata in mru state"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_data_esize, CTLFLAG_RD, + &ARC_mru.arcs_esize[ARC_BUFC_DATA].rc_count, 0, + "size of data in mru state"); + +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_size, CTLFLAG_RD, + &ARC_mru_ghost.arcs_size.rc_count, 0, "size of mru ghost state"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_metadata_esize, CTLFLAG_RD, + &ARC_mru_ghost.arcs_esize[ARC_BUFC_METADATA].rc_count, 0, + "size of metadata in mru ghost state"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mru_ghost_data_esize, CTLFLAG_RD, + &ARC_mru_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0, + "size of data in mru ghost state"); + +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_size, CTLFLAG_RD, + &ARC_mfu.arcs_size.rc_count, 0, "size of mfu state"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_metadata_esize, CTLFLAG_RD, + &ARC_mfu.arcs_esize[ARC_BUFC_METADATA].rc_count, 0, + "size of metadata in mfu state"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_data_esize, CTLFLAG_RD, + &ARC_mfu.arcs_esize[ARC_BUFC_DATA].rc_count, 0, + "size of data in mfu state"); + +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_size, CTLFLAG_RD, + &ARC_mfu_ghost.arcs_size.rc_count, 0, "size of mfu ghost state"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_metadata_esize, CTLFLAG_RD, + &ARC_mfu_ghost.arcs_esize[ARC_BUFC_METADATA].rc_count, 0, + "size of metadata in mfu ghost state"); +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_data_esize, CTLFLAG_RD, + &ARC_mfu_ghost.arcs_esize[ARC_BUFC_DATA].rc_count, 0, + "size of data in mfu ghost state"); + +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2c_only_size, CTLFLAG_RD, + &ARC_l2c_only.arcs_size.rc_count, 0, "size of mru state"); + +extern int arc_no_grow_shift; +extern int arc_shrink_shift; + +extern arc_stats_t arc_stats; +#define ARCSTAT(stat) (arc_stats.stat.value.ui64) +#define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ +#define arc_c ARCSTAT(arcstat_c) /* target size of cache */ +#define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ +#define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ +#define arc_no_grow ARCSTAT(arcstat_no_grow) /* do not grow cache size */ +#define arc_tempreserve ARCSTAT(arcstat_tempreserve) +#define arc_loaned_bytes ARCSTAT(arcstat_loaned_bytes) +#define arc_meta_limit ARCSTAT(arcstat_meta_limit) /* max size for metadata */ +#define arc_dnode_limit ARCSTAT(arcstat_dnode_limit) /* max size for dnodes */ +#define arc_meta_min ARCSTAT(arcstat_meta_min) /* min size for metadata */ +#define arc_meta_max ARCSTAT(arcstat_meta_max) /* max size of metadata */ +#define arc_need_free ARCSTAT(arcstat_need_free) /* bytes to be freed */ +#define arc_sys_free ARCSTAT(arcstat_sys_free) /* target system free bytes */ + +static int +sysctl_vfs_zfs_arc_no_grow_shift(SYSCTL_HANDLER_ARGS) +{ + uint32_t val; + int err; + + val = arc_no_grow_shift; + err = sysctl_handle_32(oidp, &val, 0, req); + if (err != 0 || req->newptr == NULL) + return (err); + + if (val >= arc_shrink_shift) + return (EINVAL); + + arc_no_grow_shift = val; + return (0); +} + +SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_no_grow_shift, CTLTYPE_U32 | CTLFLAG_RWTUN, + 0, sizeof (uint32_t), sysctl_vfs_zfs_arc_no_grow_shift, "U", + "log2(fraction of ARC which must be free to allow growing)"); +/* dbuf.c */ + + +/* dmu.c */ + +/* dmu_zfetch.c */ +SYSCTL_NODE(_vfs_zfs, OID_AUTO, zfetch, CTLFLAG_RW, 0, "ZFS ZFETCH"); + +/* max bytes to prefetch per stream (default 8MB) */ +extern uint32_t zfetch_max_distance; +SYSCTL_UINT(_vfs_zfs_zfetch, OID_AUTO, max_distance, CTLFLAG_RWTUN, + &zfetch_max_distance, 0, "Max bytes to prefetch per stream (LEGACY)"); + +/* max bytes to prefetch indirects for per stream (default 64MB) */ +extern uint32_t zfetch_max_idistance; +SYSCTL_UINT(_vfs_zfs_prefetch, OID_AUTO, max_idistance, CTLFLAG_RWTUN, + &zfetch_max_idistance, 0, "Max bytes to prefetch indirects for per stream"); + +/* dsl_pool.c */ + +/* dnode.c */ +extern int zfs_default_bs; +SYSCTL_INT(_vfs_zfs, OID_AUTO, default_bs, CTLFLAG_RWTUN, + &zfs_default_bs, 0, "Default dnode block shift"); + +extern int zfs_default_ibs; +SYSCTL_INT(_vfs_zfs, OID_AUTO, default_ibs, CTLFLAG_RWTUN, + &zfs_default_ibs, 0, "Default dnode indirect block shift"); + + +/* dsl_scan.c */ + +/* metaslab.c */ + +/* + * Enable/disable lba weighting (i.e. outer tracks are given preference). + */ +extern boolean_t metaslab_lba_weighting_enabled; +SYSCTL_INT(_vfs_zfs_metaslab, OID_AUTO, lba_weighting, CTLFLAG_RWTUN, + &metaslab_lba_weighting_enabled, 0, + "Enable LBA weighting (i.e. outer tracks are given preference)"); + + +/* + * In pools where the log space map feature is not enabled we touch + * multiple metaslabs (and their respective space maps) with each + * transaction group. Thus, we benefit from having a small space map + * block size since it allows us to issue more I/O operations scattered + * around the disk. So a sane default for the space map block size + * is 8~16K. + */ +extern int zfs_metaslab_sm_blksz_no_log; +SYSCTL_INT(_vfs_zfs_metaslab, OID_AUTO, sm_blksz_no_log, CTLFLAG_RDTUN, + &zfs_metaslab_sm_blksz_no_log, 0, + "Block size for space map in pools with log space map disabled. " + "Power of 2 and greater than 4096."); + +/* + * When the log space map feature is enabled, we accumulate a lot of + * changes per metaslab that are flushed once in a while so we benefit + * from a bigger block size like 128K for the metaslab space maps. + */ +extern int zfs_metaslab_sm_blksz_with_log; +SYSCTL_INT(_vfs_zfs_metaslab, OID_AUTO, sm_blksz_with_log, CTLFLAG_RDTUN, + &zfs_metaslab_sm_blksz_with_log, 0, + "Block size for space map in pools with log space map enabled. " + "Power of 2 and greater than 4096."); + +/* + * The in-core space map representation is more compact than its on-disk form. + * The zfs_condense_pct determines how much more compact the in-core + * space map representation must be before we compact it on-disk. + * Values should be greater than or equal to 100. + */ +extern int zfs_condense_pct; +SYSCTL_INT(_vfs_zfs, OID_AUTO, condense_pct, CTLFLAG_RWTUN, + &zfs_condense_pct, 0, + "Condense on-disk spacemap when it is more than this many percents" + " of in-memory counterpart"); + +extern int zfs_remove_max_segment; +SYSCTL_INT(_vfs_zfs, OID_AUTO, remove_max_segment, CTLFLAG_RWTUN, + &zfs_remove_max_segment, 0, "Largest contiguous segment ZFS will attempt to" + " allocate when removing a device"); + +extern int zfs_removal_suspend_progress; +SYSCTL_INT(_vfs_zfs, OID_AUTO, removal_suspend_progress, CTLFLAG_RWTUN, + &zfs_removal_suspend_progress, 0, "Ensures certain actions can happen while" + " in the middle of a removal"); + + +/* + * Minimum size which forces the dynamic allocator to change + * it's allocation strategy. Once the space map cannot satisfy + * an allocation of this size then it switches to using more + * aggressive strategy (i.e search by size rather than offset). + */ +extern uint64_t metaslab_df_alloc_threshold; +SYSCTL_QUAD(_vfs_zfs_metaslab, OID_AUTO, df_alloc_threshold, CTLFLAG_RWTUN, + &metaslab_df_alloc_threshold, 0, + "Minimum size which forces the dynamic allocator to change it's allocation strategy"); + +/* + * The minimum free space, in percent, which must be available + * in a space map to continue allocations in a first-fit fashion. + * Once the space map's free space drops below this level we dynamically + * switch to using best-fit allocations. + */ +extern int metaslab_df_free_pct; +SYSCTL_INT(_vfs_zfs_metaslab, OID_AUTO, df_free_pct, CTLFLAG_RWTUN, + &metaslab_df_free_pct, 0, + "The minimum free space, in percent, which must be available in a " + "space map to continue allocations in a first-fit fashion"); + +/* + * Percentage of all cpus that can be used by the metaslab taskq. + */ +extern int metaslab_load_pct; +SYSCTL_INT(_vfs_zfs_metaslab, OID_AUTO, load_pct, CTLFLAG_RWTUN, + &metaslab_load_pct, 0, + "Percentage of cpus that can be used by the metaslab taskq"); + +/* + * Max number of metaslabs per group to preload. + */ +extern int metaslab_preload_limit; +SYSCTL_INT(_vfs_zfs_metaslab, OID_AUTO, preload_limit, CTLFLAG_RWTUN, + &metaslab_preload_limit, 0, + "Max number of metaslabs per group to preload"); + +/* refcount.c */ +extern int reference_tracking_enable; +SYSCTL_INT(_vfs_zfs, OID_AUTO, reference_tracking_enable, CTLFLAG_RDTUN, + &reference_tracking_enable, 0, + "Track reference holders to refcount_t objects, used mostly by ZFS"); + +/* spa.c */ +extern int zfs_ccw_retry_interval; +SYSCTL_INT(_vfs_zfs, OID_AUTO, ccw_retry_interval, CTLFLAG_RWTUN, + &zfs_ccw_retry_interval, 0, + "Configuration cache file write, retry after failure, interval (seconds)"); + +extern uint64_t zfs_max_missing_tvds_cachefile; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, max_missing_tvds_cachefile, CTLFLAG_RWTUN, + &zfs_max_missing_tvds_cachefile, 0, + "allow importing pools with missing top-level vdevs in cache file"); + +extern uint64_t zfs_max_missing_tvds_scan; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, max_missing_tvds_scan, CTLFLAG_RWTUN, + &zfs_max_missing_tvds_scan, 0, + "allow importing pools with missing top-level vdevs during scan"); + +/* spa_misc.c */ +extern int zfs_flags; +static int +sysctl_vfs_zfs_debug_flags(SYSCTL_HANDLER_ARGS) +{ + int err, val; + + val = zfs_flags; + err = sysctl_handle_int(oidp, &val, 0, req); + if (err != 0 || req->newptr == NULL) + return (err); + + /* + * ZFS_DEBUG_MODIFY must be enabled prior to boot so all + * arc buffers in the system have the necessary additional + * checksum data. However, it is safe to disable at any + * time. + */ + if (!(zfs_flags & ZFS_DEBUG_MODIFY)) + val &= ~ZFS_DEBUG_MODIFY; + zfs_flags = val; + + return (0); +} + +SYSCTL_PROC(_vfs_zfs, OID_AUTO, debugflags, + CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RWTUN, NULL, 0, + sysctl_vfs_zfs_debug_flags, "IU", "Debug flags for ZFS testing."); + +int +param_set_deadman_synctime(SYSCTL_HANDLER_ARGS) +{ + unsigned long val; + int err; + + val = zfs_deadman_synctime_ms; + err = sysctl_handle_long(oidp, &val, 0, req); + if (err != 0 || req->newptr == NULL) + return (err); + zfs_deadman_synctime_ms = val; + + spa_set_deadman_synctime(MSEC2NSEC(zfs_deadman_synctime_ms)); + + return (0); +} + +int +param_set_deadman_ziotime(SYSCTL_HANDLER_ARGS) +{ + unsigned long val; + int err; + + val = zfs_deadman_ziotime_ms; + err = sysctl_handle_long(oidp, &val, 0, req); + if (err != 0 || req->newptr == NULL) + return (err); + zfs_deadman_ziotime_ms = val; + + spa_set_deadman_ziotime(MSEC2NSEC(zfs_deadman_synctime_ms)); + + return (0); +} + +int +param_set_deadman_failmode(SYSCTL_HANDLER_ARGS) +{ + char buf[16]; + int rc; + + if (req->newptr == NULL) + strlcpy(buf, zfs_deadman_failmode, sizeof (buf)); + + rc = sysctl_handle_string(oidp, buf, sizeof (buf), req); + if (rc || req->newptr == NULL) + return (rc); + if (strcmp(buf, zfs_deadman_failmode) == 0) + return (0); + if (!strcmp(buf, "wait")) + zfs_deadman_failmode = "wait"; + if (!strcmp(buf, "continue")) + zfs_deadman_failmode = "continue"; + if (!strcmp(buf, "panic")) + zfs_deadman_failmode = "panic"; + + return (-param_set_deadman_failmode_common(buf)); +} + + +/* spacemap.c */ +extern int space_map_ibs; +SYSCTL_INT(_vfs_zfs, OID_AUTO, space_map_ibs, CTLFLAG_RWTUN, + &space_map_ibs, 0, "Space map indirect block shift"); + + +/* vdev.c */ +#ifdef notyet +extern uint64_t zfs_max_auto_ashift; +extern uint64_t zfs_min_auto_ashift; + +static int +sysctl_vfs_zfs_max_auto_ashift(SYSCTL_HANDLER_ARGS) +{ + uint64_t val; + int err; + + val = zfs_max_auto_ashift; + err = sysctl_handle_64(oidp, &val, 0, req); + if (err != 0 || req->newptr == NULL) + return (err); + + if (val > ASHIFT_MAX || val < zfs_min_auto_ashift) + return (EINVAL); + + zfs_max_auto_ashift = val; + + return (0); +} +SYSCTL_PROC(_vfs_zfs, OID_AUTO, max_auto_ashift, + CTLTYPE_U64 | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof (uint64_t), + sysctl_vfs_zfs_max_auto_ashift, "QU", + "Max ashift used when optimising for logical -> physical sectors size on " + "new top-level vdevs."); +static int +sysctl_vfs_zfs_min_auto_ashift(SYSCTL_HANDLER_ARGS) +{ + uint64_t val; + int err; + + val = zfs_min_auto_ashift; + err = sysctl_handle_64(oidp, &val, 0, req); + if (err != 0 || req->newptr == NULL) + return (err); + + if (val < ASHIFT_MIN || val > zfs_max_auto_ashift) + return (EINVAL); + + zfs_min_auto_ashift = val; + + return (0); +} +SYSCTL_PROC(_vfs_zfs, OID_AUTO, min_auto_ashift, + CTLTYPE_U64 | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof (uint64_t), + sysctl_vfs_zfs_min_auto_ashift, "QU", + "Min ashift used when creating new top-level vdevs."); +#endif + +/* + * Since the DTL space map of a vdev is not expected to have a lot of + * entries, we default its block size to 4K. + */ +extern int zfs_vdev_dtl_sm_blksz; +SYSCTL_INT(_vfs_zfs, OID_AUTO, dtl_sm_blksz, CTLFLAG_RDTUN, + &zfs_vdev_dtl_sm_blksz, 0, + "Block size for DTL space map. Power of 2 and greater than 4096."); + +/* + * vdev-wide space maps that have lots of entries written to them at + * the end of each transaction can benefit from a higher I/O bandwidth + * (e.g. vdev_obsolete_sm), thus we default their block size to 128K. + */ +extern int zfs_vdev_standard_sm_blksz; +SYSCTL_INT(_vfs_zfs, OID_AUTO, standard_sm_blksz, CTLFLAG_RDTUN, + &zfs_vdev_standard_sm_blksz, 0, + "Block size for standard space map. Power of 2 and greater than 4096."); + +extern int vdev_validate_skip; +SYSCTL_INT(_vfs_zfs, OID_AUTO, validate_skip, CTLFLAG_RDTUN, + &vdev_validate_skip, 0, + "Enable to bypass vdev_validate()."); + + +/* vdev_cache.c */ + +/* vdev_mirror.c */ +/* + * The load configuration settings below are tuned by default for + * the case where all devices are of the same rotational type. + * + * If there is a mixture of rotating and non-rotating media, setting + * non_rotating_seek_inc to 0 may well provide better results as it + * will direct more reads to the non-rotating vdevs which are more + * likely to have a higher performance. + */ + + +/* vdev_queue.c */ +#define ZFS_VDEV_QUEUE_KNOB_MIN(name) \ +extern uint32_t zfs_vdev_ ## name ## _min_active; \ +SYSCTL_UINT(_vfs_zfs_vdev, OID_AUTO, name ## _min_active, CTLFLAG_RWTUN,\ + &zfs_vdev_ ## name ## _min_active, 0, \ + "Initial number of I/O requests of type " #name \ + " active for each device"); + +#define ZFS_VDEV_QUEUE_KNOB_MAX(name) \ +extern uint32_t zfs_vdev_ ## name ## _max_active; \ +SYSCTL_UINT(_vfs_zfs_vdev, OID_AUTO, name ## _max_active, CTLFLAG_RWTUN, \ + &zfs_vdev_ ## name ## _max_active, 0, \ + "Maximum number of I/O requests of type " #name \ + " active for each device"); + + +#undef ZFS_VDEV_QUEUE_KNOB + +extern uint32_t zfs_vdev_max_active; +SYSCTL_UINT(_vfs_zfs, OID_AUTO, top_maxinflight, CTLFLAG_RWTUN, + &zfs_vdev_max_active, 0, + "The maximum number of I/Os of all types active for each device. (LEGACY)"); + +extern int zfs_vdev_def_queue_depth; +SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, def_queue_depth, CTLFLAG_RWTUN, + &zfs_vdev_def_queue_depth, 0, + "Default queue depth for each allocator"); + +/*extern uint64_t zfs_multihost_history; +SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, multihost_history, CTLFLAG_RWTUN, + &zfs_multihost_history, 0, + "Historical staticists for the last N multihost updates");*/ + +#ifdef notyet +SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, trim_on_init, CTLFLAG_RW, + &vdev_trim_on_init, 0, "Enable/disable full vdev trim on initialisation"); +#endif + + +/* zio.c */ +#if defined(__LP64__) +int zio_use_uma = 1; +#else +int zio_use_uma = 0; +#endif + +SYSCTL_INT(_vfs_zfs_zio, OID_AUTO, use_uma, CTLFLAG_RDTUN, &zio_use_uma, 0, + "Use uma(9) for ZIO allocations"); +SYSCTL_INT(_vfs_zfs_zio, OID_AUTO, exclude_metadata, CTLFLAG_RDTUN, &zio_exclude_metadata, 0, + "Exclude metadata buffers from dumps as well"); + + +int +param_set_arc_long(SYSCTL_HANDLER_ARGS) +{ + int err; + + err = sysctl_handle_long(oidp, arg1, 0, req); + if (err != 0 || req->newptr == NULL) + return (err); + + arc_tuning_update(B_TRUE); + + return (0); +} + +int +param_set_arc_int(SYSCTL_HANDLER_ARGS) +{ + int err; + + err = sysctl_handle_int(oidp, arg1, 0, req); + if (err != 0 || req->newptr == NULL) + return (err); + + arc_tuning_update(B_TRUE); + + return (0); +} + +int +param_set_slop_shift(SYSCTL_HANDLER_ARGS) +{ + int val; + int err; + + val = *(int *)arg1; + + err = sysctl_handle_int(oidp, &val, 0, req); + if (err != 0 || req->newptr == NULL) + return (err); + + if (val < 1 || val > 31) + return (EINVAL); + + *(int *)arg1 = val; + + return (0); +} diff --git a/module/os/freebsd/zfs/vdev_file.c b/module/os/freebsd/zfs/vdev_file.c new file mode 100644 index 00000000000..01851378e71 --- /dev/null +++ b/module/os/freebsd/zfs/vdev_file.c @@ -0,0 +1,326 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2016 by Delphix. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Virtual device vector for files. + */ + +static taskq_t *vdev_file_taskq; + +void +vdev_file_init(void) +{ + vdev_file_taskq = taskq_create("z_vdev_file", MAX(max_ncpus, 16), + minclsyspri, max_ncpus, INT_MAX, 0); +} + +void +vdev_file_fini(void) +{ + taskq_destroy(vdev_file_taskq); +} + +static void +vdev_file_hold(vdev_t *vd) +{ + ASSERT(vd->vdev_path != NULL); +} + +static void +vdev_file_rele(vdev_t *vd) +{ + ASSERT(vd->vdev_path != NULL); +} + +static mode_t +vdev_file_open_mode(spa_mode_t spa_mode) +{ + mode_t mode = 0; + + if ((spa_mode & SPA_MODE_READ) && (spa_mode & SPA_MODE_WRITE)) { + mode = O_RDWR; + } else if (spa_mode & SPA_MODE_READ) { + mode = O_RDONLY; + } else if (spa_mode & SPA_MODE_WRITE) { + mode = O_WRONLY; + } + + return (mode | O_LARGEFILE); +} + +static int +vdev_file_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize, + uint64_t *ashift) +{ + vdev_file_t *vf; + zfs_file_t *fp; + zfs_file_attr_t zfa; + int error; + + /* + * Rotational optimizations only make sense on block devices. + */ + vd->vdev_nonrot = B_TRUE; + + /* + * Allow TRIM on file based vdevs. This may not always be supported, + * since it depends on your kernel version and underlying filesystem + * type but it is always safe to attempt. + */ + vd->vdev_has_trim = B_TRUE; + + /* + * Disable secure TRIM on file based vdevs. There is no way to + * request this behavior from the underlying filesystem. + */ + vd->vdev_has_securetrim = B_FALSE; + + /* + * We must have a pathname, and it must be absolute. + */ + if (vd->vdev_path == NULL || vd->vdev_path[0] != '/') { + vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL; + return (SET_ERROR(EINVAL)); + } + + /* + * Reopen the device if it's not currently open. Otherwise, + * just update the physical size of the device. + */ + if (vd->vdev_tsd != NULL) { + ASSERT(vd->vdev_reopening); + vf = vd->vdev_tsd; + goto skip_open; + } + + vf = vd->vdev_tsd = kmem_zalloc(sizeof (vdev_file_t), KM_SLEEP); + + /* + * We always open the files from the root of the global zone, even if + * we're in a local zone. If the user has gotten to this point, the + * administrator has already decided that the pool should be available + * to local zone users, so the underlying devices should be as well. + */ + ASSERT(vd->vdev_path != NULL && vd->vdev_path[0] == '/'); + + error = zfs_file_open(vd->vdev_path, + vdev_file_open_mode(spa_mode(vd->vdev_spa)), 0, &fp); + if (error) { + vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED; + return (error); + } + + vf->vf_file = fp; + +#ifdef _KERNEL + /* + * Make sure it's a regular file. + */ + if (zfs_file_getattr(fp, &zfa)) { + return (SET_ERROR(ENODEV)); + } + if (!S_ISREG(zfa.zfa_mode)) { + vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED; + return (SET_ERROR(ENODEV)); + } +#endif + +skip_open: + + error = zfs_file_getattr(vf->vf_file, &zfa); + if (error) { + vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED; + return (error); + } + + *max_psize = *psize = zfa.zfa_size; + *ashift = SPA_MINBLOCKSHIFT; + + return (0); +} + +static void +vdev_file_close(vdev_t *vd) +{ + vdev_file_t *vf = vd->vdev_tsd; + + if (vd->vdev_reopening || vf == NULL) + return; + + if (vf->vf_file != NULL) { + zfs_file_close(vf->vf_file); + } + + vd->vdev_delayed_close = B_FALSE; + kmem_free(vf, sizeof (vdev_file_t)); + vd->vdev_tsd = NULL; +} + +/* + * Implements the interrupt side for file vdev types. This routine will be + * called when the I/O completes allowing us to transfer the I/O to the + * interrupt taskqs. For consistency, the code structure mimics disk vdev + * types. + */ +static void +vdev_file_io_intr(zio_t *zio) +{ + zio_delay_interrupt(zio); +} + +static void +vdev_file_io_strategy(void *arg) +{ + zio_t *zio = arg; + vdev_t *vd = zio->io_vd; + vdev_file_t *vf; + void *buf; + ssize_t resid; + loff_t off; + ssize_t size; + int err; + + off = zio->io_offset; + size = zio->io_size; + resid = 0; + + vf = vd->vdev_tsd; + + ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE); + if (zio->io_type == ZIO_TYPE_READ) { + buf = abd_borrow_buf(zio->io_abd, zio->io_size); + err = zfs_file_pread(vf->vf_file, buf, size, off, &resid); + abd_return_buf_copy(zio->io_abd, buf, size); + } else { + buf = abd_borrow_buf_copy(zio->io_abd, zio->io_size); + err = zfs_file_pwrite(vf->vf_file, buf, size, off, &resid); + abd_return_buf(zio->io_abd, buf, size); + } + if (resid != 0 && zio->io_error == 0) + zio->io_error = ENOSPC; + + vdev_file_io_intr(zio); +} + +static void +vdev_file_io_start(zio_t *zio) +{ + vdev_t *vd = zio->io_vd; + vdev_file_t *vf = vd->vdev_tsd; + + if (zio->io_type == ZIO_TYPE_IOCTL) { + /* XXPOLICY */ + if (!vdev_readable(vd)) { + zio->io_error = SET_ERROR(ENXIO); + zio_interrupt(zio); + return; + } + + switch (zio->io_cmd) { + case DKIOCFLUSHWRITECACHE: + zio->io_error = zfs_file_fsync(vf->vf_file, + O_SYNC|O_DSYNC); + break; + default: + zio->io_error = SET_ERROR(ENOTSUP); + } + + zio_execute(zio); + return; + } else if (zio->io_type == ZIO_TYPE_TRIM) { +#ifdef notyet + int mode = 0; + + ASSERT3U(zio->io_size, !=, 0); + + /* XXX FreeBSD has no fallocate routine in file ops */ + zio->io_error = zfs_file_fallocate(vf->vf_file, + mode, zio->io_offset, zio->io_size); +#endif + zio->io_error = SET_ERROR(ENOTSUP); + zio_execute(zio); + return; + } + ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE); + zio->io_target_timestamp = zio_handle_io_delay(zio); + + VERIFY3U(taskq_dispatch(vdev_file_taskq, vdev_file_io_strategy, zio, + TQ_SLEEP), !=, 0); +} + +/* ARGSUSED */ +static void +vdev_file_io_done(zio_t *zio) +{ +} + +vdev_ops_t vdev_file_ops = { + vdev_file_open, + vdev_file_close, + vdev_default_asize, + vdev_file_io_start, + vdev_file_io_done, + NULL, + NULL, + vdev_file_hold, + vdev_file_rele, + NULL, + vdev_default_xlate, + VDEV_TYPE_FILE, /* name of this vdev type */ + B_TRUE /* leaf vdev */ +}; + +/* + * From userland we access disks just like files. + */ +#ifndef _KERNEL + +vdev_ops_t vdev_disk_ops = { + vdev_file_open, + vdev_file_close, + vdev_default_asize, + vdev_file_io_start, + vdev_file_io_done, + NULL, + NULL, + vdev_file_hold, + vdev_file_rele, + NULL, + vdev_default_xlate, + VDEV_TYPE_DISK, /* name of this vdev type */ + B_TRUE /* leaf vdev */ +}; + +#endif diff --git a/module/os/freebsd/zfs/vdev_geom.c b/module/os/freebsd/zfs/vdev_geom.c new file mode 100644 index 00000000000..d87bbbc1815 --- /dev/null +++ b/module/os/freebsd/zfs/vdev_geom.c @@ -0,0 +1,1195 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2006 Pawel Jakub Dawidek + * All rights reserved. + * + * Portions Copyright (c) 2012 Martin Matuska + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Virtual device vector for GEOM. + */ + +static g_attrchanged_t vdev_geom_attrchanged; +struct g_class zfs_vdev_class = { + .name = "ZFS::VDEV", + .version = G_VERSION, + .attrchanged = vdev_geom_attrchanged, +}; + +struct consumer_vdev_elem { + SLIST_ENTRY(consumer_vdev_elem) elems; + vdev_t *vd; +}; + +SLIST_HEAD(consumer_priv_t, consumer_vdev_elem); +/* BEGIN CSTYLED */ +_Static_assert(sizeof (((struct g_consumer *)NULL)->private) + == sizeof (struct consumer_priv_t*), + "consumer_priv_t* can't be stored in g_consumer.private"); + +DECLARE_GEOM_CLASS(zfs_vdev_class, zfs_vdev); + +SYSCTL_DECL(_vfs_zfs_vdev); +/* Don't send BIO_FLUSH. */ +static int vdev_geom_bio_flush_disable; +SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, bio_flush_disable, CTLFLAG_RWTUN, + &vdev_geom_bio_flush_disable, 0, "Disable BIO_FLUSH"); +/* Don't send BIO_DELETE. */ +static int vdev_geom_bio_delete_disable; +SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, bio_delete_disable, CTLFLAG_RWTUN, + &vdev_geom_bio_delete_disable, 0, "Disable BIO_DELETE"); +/* END CSTYLED */ + +/* Declare local functions */ +static void vdev_geom_detach(struct g_consumer *cp, boolean_t open_for_read); + +/* + * Thread local storage used to indicate when a thread is probing geoms + * for their guids. If NULL, this thread is not tasting geoms. If non NULL, + * it is looking for a replacement for the vdev_t* that is its value. + */ +uint_t zfs_geom_probe_vdev_key; + +static void +vdev_geom_set_physpath(vdev_t *vd, struct g_consumer *cp, + boolean_t do_null_update) +{ + boolean_t needs_update = B_FALSE; + char *physpath; + int error, physpath_len; + + physpath_len = MAXPATHLEN; + physpath = g_malloc(physpath_len, M_WAITOK|M_ZERO); + error = g_io_getattr("GEOM::physpath", cp, &physpath_len, physpath); + if (error == 0) { + char *old_physpath; + + /* g_topology lock ensures that vdev has not been closed */ + g_topology_assert(); + old_physpath = vd->vdev_physpath; + vd->vdev_physpath = spa_strdup(physpath); + + if (old_physpath != NULL) { + needs_update = (strcmp(old_physpath, + vd->vdev_physpath) != 0); + spa_strfree(old_physpath); + } else + needs_update = do_null_update; + } + g_free(physpath); + + /* + * If the physical path changed, update the config. + * Only request an update for previously unset physpaths if + * requested by the caller. + */ + if (needs_update) + spa_async_request(vd->vdev_spa, SPA_ASYNC_CONFIG_UPDATE); + +} + +static void +vdev_geom_attrchanged(struct g_consumer *cp, const char *attr) +{ + struct consumer_priv_t *priv; + struct consumer_vdev_elem *elem; + + priv = (struct consumer_priv_t *)&cp->private; + if (SLIST_EMPTY(priv)) + return; + + SLIST_FOREACH(elem, priv, elems) { + vdev_t *vd = elem->vd; + if (strcmp(attr, "GEOM::physpath") == 0) { + vdev_geom_set_physpath(vd, cp, /* null_update */B_TRUE); + return; + } + } +} + +static void +vdev_geom_resize(struct g_consumer *cp) +{ + struct consumer_priv_t *priv; + struct consumer_vdev_elem *elem; + spa_t *spa; + vdev_t *vd; + + priv = (struct consumer_priv_t *)&cp->private; + if (SLIST_EMPTY(priv)) + return; + + SLIST_FOREACH(elem, priv, elems) { + vd = elem->vd; + if (vd->vdev_state != VDEV_STATE_HEALTHY) + continue; + spa = vd->vdev_spa; + if (!spa->spa_autoexpand) + continue; + vdev_online(spa, vd->vdev_guid, ZFS_ONLINE_EXPAND, NULL); + } +} + +static void +vdev_geom_orphan(struct g_consumer *cp) +{ + struct consumer_priv_t *priv; + // cppcheck-suppress uninitvar + struct consumer_vdev_elem *elem; + + g_topology_assert(); + + priv = (struct consumer_priv_t *)&cp->private; + if (SLIST_EMPTY(priv)) + /* Vdev close in progress. Ignore the event. */ + return; + + /* + * Orphan callbacks occur from the GEOM event thread. + * Concurrent with this call, new I/O requests may be + * working their way through GEOM about to find out + * (only once executed by the g_down thread) that we've + * been orphaned from our disk provider. These I/Os + * must be retired before we can detach our consumer. + * This is most easily achieved by acquiring the + * SPA ZIO configuration lock as a writer, but doing + * so with the GEOM topology lock held would cause + * a lock order reversal. Instead, rely on the SPA's + * async removal support to invoke a close on this + * vdev once it is safe to do so. + */ + // cppcheck-suppress All + SLIST_FOREACH(elem, priv, elems) { + // cppcheck-suppress uninitvar + vdev_t *vd = elem->vd; + + vd->vdev_remove_wanted = B_TRUE; + spa_async_request(vd->vdev_spa, SPA_ASYNC_REMOVE); + } +} + +static struct g_consumer * +vdev_geom_attach(struct g_provider *pp, vdev_t *vd, boolean_t sanity) +{ + struct g_geom *gp; + struct g_consumer *cp; + int error; + + g_topology_assert(); + + ZFS_LOG(1, "Attaching to %s.", pp->name); + + if (sanity) { + if (pp->sectorsize > VDEV_PAD_SIZE || !ISP2(pp->sectorsize)) { + ZFS_LOG(1, "Failing attach of %s. " + "Incompatible sectorsize %d\n", + pp->name, pp->sectorsize); + return (NULL); + } else if (pp->mediasize < SPA_MINDEVSIZE) { + ZFS_LOG(1, "Failing attach of %s. " + "Incompatible mediasize %ju\n", + pp->name, pp->mediasize); + return (NULL); + } + } + + /* Do we have geom already? No? Create one. */ + LIST_FOREACH(gp, &zfs_vdev_class.geom, geom) { + if (gp->flags & G_GEOM_WITHER) + continue; + if (strcmp(gp->name, "zfs::vdev") != 0) + continue; + break; + } + if (gp == NULL) { + gp = g_new_geomf(&zfs_vdev_class, "zfs::vdev"); + gp->orphan = vdev_geom_orphan; + gp->attrchanged = vdev_geom_attrchanged; + gp->resize = vdev_geom_resize; + cp = g_new_consumer(gp); + error = g_attach(cp, pp); + if (error != 0) { + ZFS_LOG(1, "%s(%d): g_attach failed: %d\n", __func__, + __LINE__, error); + vdev_geom_detach(cp, B_FALSE); + return (NULL); + } + error = g_access(cp, 1, 0, 1); + if (error != 0) { + ZFS_LOG(1, "%s(%d): g_access failed: %d\n", __func__, + __LINE__, error); + vdev_geom_detach(cp, B_FALSE); + return (NULL); + } + ZFS_LOG(1, "Created geom and consumer for %s.", pp->name); + } else { + /* Check if we are already connected to this provider. */ + LIST_FOREACH(cp, &gp->consumer, consumer) { + if (cp->provider == pp) { + ZFS_LOG(1, "Found consumer for %s.", pp->name); + break; + } + } + if (cp == NULL) { + cp = g_new_consumer(gp); + error = g_attach(cp, pp); + if (error != 0) { + ZFS_LOG(1, "%s(%d): g_attach failed: %d\n", + __func__, __LINE__, error); + vdev_geom_detach(cp, B_FALSE); + return (NULL); + } + error = g_access(cp, 1, 0, 1); + if (error != 0) { + ZFS_LOG(1, "%s(%d): g_access failed: %d\n", + __func__, __LINE__, error); + vdev_geom_detach(cp, B_FALSE); + return (NULL); + } + ZFS_LOG(1, "Created consumer for %s.", pp->name); + } else { + error = g_access(cp, 1, 0, 1); + if (error != 0) { + ZFS_LOG(1, "%s(%d): g_access failed: %d\n", + __func__, __LINE__, error); + return (NULL); + } + ZFS_LOG(1, "Used existing consumer for %s.", pp->name); + } + } + + if (vd != NULL) + vd->vdev_tsd = cp; + + cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE; + return (cp); +} + +static void +vdev_geom_detach(struct g_consumer *cp, boolean_t open_for_read) +{ + struct g_geom *gp; + + g_topology_assert(); + + ZFS_LOG(1, "Detaching from %s.", + cp->provider && cp->provider->name ? cp->provider->name : "NULL"); + + gp = cp->geom; + if (open_for_read) + g_access(cp, -1, 0, -1); + /* Destroy consumer on last close. */ + if (cp->acr == 0 && cp->ace == 0) { + if (cp->acw > 0) + g_access(cp, 0, -cp->acw, 0); + if (cp->provider != NULL) { + ZFS_LOG(1, "Destroying consumer for %s.", + cp->provider->name ? cp->provider->name : "NULL"); + g_detach(cp); + } + g_destroy_consumer(cp); + } + /* Destroy geom if there are no consumers left. */ + if (LIST_EMPTY(&gp->consumer)) { + ZFS_LOG(1, "Destroyed geom %s.", gp->name); + g_wither_geom(gp, ENXIO); + } +} + +static void +vdev_geom_close_locked(vdev_t *vd) +{ + struct g_consumer *cp; + struct consumer_priv_t *priv; + struct consumer_vdev_elem *elem, *elem_temp; + + g_topology_assert(); + + cp = vd->vdev_tsd; + vd->vdev_delayed_close = B_FALSE; + if (cp == NULL) + return; + + ZFS_LOG(1, "Closing access to %s.", cp->provider->name); + KASSERT(cp->private != NULL, ("%s: cp->private is NULL", __func__)); + priv = (struct consumer_priv_t *)&cp->private; + vd->vdev_tsd = NULL; + SLIST_FOREACH_SAFE(elem, priv, elems, elem_temp) { + if (elem->vd == vd) { + SLIST_REMOVE(priv, elem, consumer_vdev_elem, elems); + g_free(elem); + } + } + + vdev_geom_detach(cp, B_TRUE); +} + +/* + * Issue one or more bios to the vdev in parallel + * cmds, datas, offsets, errors, and sizes are arrays of length ncmds. Each IO + * operation is described by parallel entries from each array. There may be + * more bios actually issued than entries in the array + */ +static void +vdev_geom_io(struct g_consumer *cp, int *cmds, void **datas, off_t *offsets, + off_t *sizes, int *errors, int ncmds) +{ + struct bio **bios; + uint8_t *p; + off_t off, maxio, s, end; + int i, n_bios, j; + size_t bios_size; + + maxio = MAXPHYS - (MAXPHYS % cp->provider->sectorsize); + n_bios = 0; + + /* How many bios are required for all commands ? */ + for (i = 0; i < ncmds; i++) + n_bios += (sizes[i] + maxio - 1) / maxio; + + /* Allocate memory for the bios */ + bios_size = n_bios * sizeof (struct bio *); + bios = kmem_zalloc(bios_size, KM_SLEEP); + + /* Prepare and issue all of the bios */ + for (i = j = 0; i < ncmds; i++) { + off = offsets[i]; + p = datas[i]; + s = sizes[i]; + end = off + s; + ASSERT((off % cp->provider->sectorsize) == 0); + ASSERT((s % cp->provider->sectorsize) == 0); + + for (; off < end; off += maxio, p += maxio, s -= maxio, j++) { + bios[j] = g_alloc_bio(); + bios[j]->bio_cmd = cmds[i]; + bios[j]->bio_done = NULL; + bios[j]->bio_offset = off; + bios[j]->bio_length = MIN(s, maxio); + bios[j]->bio_data = (caddr_t)p; + g_io_request(bios[j], cp); + } + } + ASSERT(j == n_bios); + + /* Wait for all of the bios to complete, and clean them up */ + for (i = j = 0; i < ncmds; i++) { + off = offsets[i]; + s = sizes[i]; + end = off + s; + + for (; off < end; off += maxio, s -= maxio, j++) { + errors[i] = biowait(bios[j], "vdev_geom_io") || + errors[i]; + g_destroy_bio(bios[j]); + } + } + kmem_free(bios, bios_size); +} + +/* + * Read the vdev config from a device. Return the number of valid labels that + * were found. The vdev config will be returned in config if and only if at + * least one valid label was found. + */ +static int +vdev_geom_read_config(struct g_consumer *cp, nvlist_t **configp) +{ + struct g_provider *pp; + nvlist_t *config; + vdev_phys_t *vdev_lists[VDEV_LABELS]; + char *buf; + size_t buflen; + uint64_t psize, state, txg; + off_t offsets[VDEV_LABELS]; + off_t size; + off_t sizes[VDEV_LABELS]; + int cmds[VDEV_LABELS]; + int errors[VDEV_LABELS]; + int l, nlabels; + + g_topology_assert_not(); + + pp = cp->provider; + ZFS_LOG(1, "Reading config from %s...", pp->name); + + psize = pp->mediasize; + psize = P2ALIGN(psize, (uint64_t)sizeof (vdev_label_t)); + + size = sizeof (*vdev_lists[0]) + pp->sectorsize - + ((sizeof (*vdev_lists[0]) - 1) % pp->sectorsize) - 1; + + buflen = sizeof (vdev_lists[0]->vp_nvlist); + + /* Create all of the IO requests */ + for (l = 0; l < VDEV_LABELS; l++) { + cmds[l] = BIO_READ; + vdev_lists[l] = kmem_alloc(size, KM_SLEEP); + offsets[l] = vdev_label_offset(psize, l, 0) + VDEV_SKIP_SIZE; + sizes[l] = size; + errors[l] = 0; + ASSERT(offsets[l] % pp->sectorsize == 0); + } + + /* Issue the IO requests */ + vdev_geom_io(cp, cmds, (void**)vdev_lists, offsets, sizes, errors, + VDEV_LABELS); + + /* Parse the labels */ + config = *configp = NULL; + nlabels = 0; + for (l = 0; l < VDEV_LABELS; l++) { + if (errors[l] != 0) + continue; + + buf = vdev_lists[l]->vp_nvlist; + + if (nvlist_unpack(buf, buflen, &config, 0) != 0) + continue; + + if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, + &state) != 0 || state > POOL_STATE_L2CACHE) { + nvlist_free(config); + continue; + } + + if (state != POOL_STATE_SPARE && + state != POOL_STATE_L2CACHE && + (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, + &txg) != 0 || txg == 0)) { + nvlist_free(config); + continue; + } + + if (*configp != NULL) + nvlist_free(*configp); + *configp = config; + nlabels++; + } + + /* Free the label storage */ + for (l = 0; l < VDEV_LABELS; l++) + kmem_free(vdev_lists[l], size); + + return (nlabels); +} + +static void +resize_configs(nvlist_t ***configs, uint64_t *count, uint64_t id) +{ + nvlist_t **new_configs; + uint64_t i; + + if (id < *count) + return; + new_configs = kmem_zalloc((id + 1) * sizeof (nvlist_t *), + KM_SLEEP); + for (i = 0; i < *count; i++) + new_configs[i] = (*configs)[i]; + if (*configs != NULL) + kmem_free(*configs, *count * sizeof (void *)); + *configs = new_configs; + *count = id + 1; +} + +static void +process_vdev_config(nvlist_t ***configs, uint64_t *count, nvlist_t *cfg, + const char *name, uint64_t *known_pool_guid) +{ + nvlist_t *vdev_tree; + uint64_t pool_guid; + uint64_t vdev_guid; + uint64_t id, txg, known_txg; + char *pname; + + if (nvlist_lookup_string(cfg, ZPOOL_CONFIG_POOL_NAME, &pname) != 0 || + strcmp(pname, name) != 0) + goto ignore; + + if (nvlist_lookup_uint64(cfg, ZPOOL_CONFIG_POOL_GUID, &pool_guid) != 0) + goto ignore; + + if (nvlist_lookup_uint64(cfg, ZPOOL_CONFIG_TOP_GUID, &vdev_guid) != 0) + goto ignore; + + if (nvlist_lookup_nvlist(cfg, ZPOOL_CONFIG_VDEV_TREE, &vdev_tree) != 0) + goto ignore; + + if (nvlist_lookup_uint64(vdev_tree, ZPOOL_CONFIG_ID, &id) != 0) + goto ignore; + + VERIFY(nvlist_lookup_uint64(cfg, ZPOOL_CONFIG_POOL_TXG, &txg) == 0); + + if (*known_pool_guid != 0) { + if (pool_guid != *known_pool_guid) + goto ignore; + } else + *known_pool_guid = pool_guid; + + resize_configs(configs, count, id); + + if ((*configs)[id] != NULL) { + VERIFY(nvlist_lookup_uint64((*configs)[id], + ZPOOL_CONFIG_POOL_TXG, &known_txg) == 0); + if (txg <= known_txg) + goto ignore; + nvlist_free((*configs)[id]); + } + + (*configs)[id] = cfg; + return; + +ignore: + nvlist_free(cfg); +} + +int +vdev_geom_read_pool_label(const char *name, + nvlist_t ***configs, uint64_t *count) +{ + struct g_class *mp; + struct g_geom *gp; + struct g_provider *pp; + struct g_consumer *zcp; + nvlist_t *vdev_cfg; + uint64_t pool_guid; + int nlabels; + + DROP_GIANT(); + g_topology_lock(); + + *configs = NULL; + *count = 0; + pool_guid = 0; + LIST_FOREACH(mp, &g_classes, class) { + if (mp == &zfs_vdev_class) + continue; + LIST_FOREACH(gp, &mp->geom, geom) { + if (gp->flags & G_GEOM_WITHER) + continue; + LIST_FOREACH(pp, &gp->provider, provider) { + if (pp->flags & G_PF_WITHER) + continue; + zcp = vdev_geom_attach(pp, NULL, B_TRUE); + if (zcp == NULL) + continue; + g_topology_unlock(); + nlabels = vdev_geom_read_config(zcp, &vdev_cfg); + g_topology_lock(); + vdev_geom_detach(zcp, B_TRUE); + if (nlabels == 0) + continue; + ZFS_LOG(1, "successfully read vdev config"); + + process_vdev_config(configs, count, + vdev_cfg, name, &pool_guid); + } + } + } + g_topology_unlock(); + PICKUP_GIANT(); + + return (*count > 0 ? 0 : ENOENT); +} + +enum match { + NO_MATCH = 0, /* No matching labels found */ + TOPGUID_MATCH = 1, /* Labels match top guid, not vdev guid */ + ZERO_MATCH = 1, /* Should never be returned */ + ONE_MATCH = 2, /* 1 label matching the vdev_guid */ + TWO_MATCH = 3, /* 2 label matching the vdev_guid */ + THREE_MATCH = 4, /* 3 label matching the vdev_guid */ + FULL_MATCH = 5 /* all labels match the vdev_guid */ +}; + +static enum match +vdev_attach_ok(vdev_t *vd, struct g_provider *pp) +{ + nvlist_t *config; + uint64_t pool_guid, top_guid, vdev_guid; + struct g_consumer *cp; + int nlabels; + + cp = vdev_geom_attach(pp, NULL, B_TRUE); + if (cp == NULL) { + ZFS_LOG(1, "Unable to attach tasting instance to %s.", + pp->name); + return (NO_MATCH); + } + g_topology_unlock(); + nlabels = vdev_geom_read_config(cp, &config); + g_topology_lock(); + vdev_geom_detach(cp, B_TRUE); + if (nlabels == 0) { + ZFS_LOG(1, "Unable to read config from %s.", pp->name); + return (NO_MATCH); + } + + pool_guid = 0; + (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid); + top_guid = 0; + (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TOP_GUID, &top_guid); + vdev_guid = 0; + (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_GUID, &vdev_guid); + nvlist_free(config); + + /* + * Check that the label's pool guid matches the desired guid. + * Inactive spares and L2ARCs do not have any pool guid in the label. + */ + if (pool_guid != 0 && pool_guid != spa_guid(vd->vdev_spa)) { + ZFS_LOG(1, "pool guid mismatch for provider %s: %ju != %ju.", + pp->name, + (uintmax_t)spa_guid(vd->vdev_spa), (uintmax_t)pool_guid); + return (NO_MATCH); + } + + /* + * Check that the label's vdev guid matches the desired guid. + * The second condition handles possible race on vdev detach, when + * remaining vdev receives GUID of destroyed top level mirror vdev. + */ + if (vdev_guid == vd->vdev_guid) { + ZFS_LOG(1, "guids match for provider %s.", pp->name); + return (ZERO_MATCH + nlabels); + } else if (top_guid == vd->vdev_guid && vd == vd->vdev_top) { + ZFS_LOG(1, "top vdev guid match for provider %s.", pp->name); + return (TOPGUID_MATCH); + } + ZFS_LOG(1, "vdev guid mismatch for provider %s: %ju != %ju.", + pp->name, (uintmax_t)vd->vdev_guid, (uintmax_t)vdev_guid); + return (NO_MATCH); +} + +static struct g_consumer * +vdev_geom_attach_by_guids(vdev_t *vd) +{ + struct g_class *mp; + struct g_geom *gp; + struct g_provider *pp, *best_pp; + struct g_consumer *cp; + const char *vdpath; + enum match match, best_match; + + g_topology_assert(); + + vdpath = vd->vdev_path + sizeof ("/dev/") - 1; + cp = NULL; + best_pp = NULL; + best_match = NO_MATCH; + LIST_FOREACH(mp, &g_classes, class) { + if (mp == &zfs_vdev_class) + continue; + LIST_FOREACH(gp, &mp->geom, geom) { + if (gp->flags & G_GEOM_WITHER) + continue; + LIST_FOREACH(pp, &gp->provider, provider) { + match = vdev_attach_ok(vd, pp); + if (match > best_match) { + best_match = match; + best_pp = pp; + } else if (match == best_match) { + if (strcmp(pp->name, vdpath) == 0) { + best_pp = pp; + } + } + if (match == FULL_MATCH) + goto out; + } + } + } + +out: + if (best_pp) { + cp = vdev_geom_attach(best_pp, vd, B_TRUE); + if (cp == NULL) { + printf("ZFS WARNING: Unable to attach to %s.\n", + best_pp->name); + } + } + return (cp); +} + +static struct g_consumer * +vdev_geom_open_by_guids(vdev_t *vd) +{ + struct g_consumer *cp; + char *buf; + size_t len; + + g_topology_assert(); + + ZFS_LOG(1, "Searching by guids [%ju:%ju].", + (uintmax_t)spa_guid(vd->vdev_spa), (uintmax_t)vd->vdev_guid); + cp = vdev_geom_attach_by_guids(vd); + if (cp != NULL) { + len = strlen(cp->provider->name) + strlen("/dev/") + 1; + buf = kmem_alloc(len, KM_SLEEP); + + snprintf(buf, len, "/dev/%s", cp->provider->name); + spa_strfree(vd->vdev_path); + vd->vdev_path = buf; + + ZFS_LOG(1, "Attach by guid [%ju:%ju] succeeded, provider %s.", + (uintmax_t)spa_guid(vd->vdev_spa), + (uintmax_t)vd->vdev_guid, cp->provider->name); + } else { + ZFS_LOG(1, "Search by guid [%ju:%ju] failed.", + (uintmax_t)spa_guid(vd->vdev_spa), + (uintmax_t)vd->vdev_guid); + } + + return (cp); +} + +static struct g_consumer * +vdev_geom_open_by_path(vdev_t *vd, int check_guid) +{ + struct g_provider *pp; + struct g_consumer *cp; + + g_topology_assert(); + + cp = NULL; + pp = g_provider_by_name(vd->vdev_path + sizeof ("/dev/") - 1); + if (pp != NULL) { + ZFS_LOG(1, "Found provider by name %s.", vd->vdev_path); + if (!check_guid || vdev_attach_ok(vd, pp) == FULL_MATCH) + cp = vdev_geom_attach(pp, vd, B_FALSE); + } + + return (cp); +} + +static int +vdev_geom_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize, + uint64_t *logical_ashift) +{ + struct g_provider *pp; + struct g_consumer *cp; + int error, has_trim; + uint16_t rate; + + /* + * Set the TLS to indicate downstack that we + * should not access zvols + */ + VERIFY(tsd_set(zfs_geom_probe_vdev_key, vd) == 0); + + /* + * We must have a pathname, and it must be absolute. + */ + if (vd->vdev_path == NULL || strncmp(vd->vdev_path, "/dev/", 5) != 0) { + vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL; + return (EINVAL); + } + + /* + * Reopen the device if it's not currently open. Otherwise, + * just update the physical size of the device. + */ + if ((cp = vd->vdev_tsd) != NULL) { + ASSERT(vd->vdev_reopening); + goto skip_open; + } + + DROP_GIANT(); + g_topology_lock(); + error = 0; + + if (vd->vdev_spa->spa_is_splitting || + ((vd->vdev_prevstate == VDEV_STATE_UNKNOWN && + (vd->vdev_spa->spa_load_state == SPA_LOAD_NONE || + vd->vdev_spa->spa_load_state == SPA_LOAD_CREATE)))) { + /* + * We are dealing with a vdev that hasn't been previously + * opened (since boot), and we are not loading an + * existing pool configuration. This looks like a + * vdev add operation to a new or existing pool. + * Assume the user knows what he/she is doing and find + * GEOM provider by its name, ignoring GUID mismatches. + * + * XXPOLICY: It would be safer to only allow a device + * that is unlabeled or labeled but missing + * GUID information to be opened in this fashion, + * unless we are doing a split, in which case we + * should allow any guid. + */ + cp = vdev_geom_open_by_path(vd, 0); + } else { + /* + * Try using the recorded path for this device, but only + * accept it if its label data contains the expected GUIDs. + */ + cp = vdev_geom_open_by_path(vd, 1); + if (cp == NULL) { + /* + * The device at vd->vdev_path doesn't have the + * expected GUIDs. The disks might have merely + * moved around so try all other GEOM providers + * to find one with the right GUIDs. + */ + cp = vdev_geom_open_by_guids(vd); + } + } + + /* Clear the TLS now that tasting is done */ + VERIFY(tsd_set(zfs_geom_probe_vdev_key, NULL) == 0); + + if (cp == NULL) { + ZFS_LOG(1, "Vdev %s not found.", vd->vdev_path); + error = ENOENT; + } else { + struct consumer_priv_t *priv; + struct consumer_vdev_elem *elem; + int spamode; + + priv = (struct consumer_priv_t *)&cp->private; + if (cp->private == NULL) + SLIST_INIT(priv); + elem = g_malloc(sizeof (*elem), M_WAITOK|M_ZERO); + elem->vd = vd; + SLIST_INSERT_HEAD(priv, elem, elems); + + spamode = spa_mode(vd->vdev_spa); + if (cp->provider->sectorsize > VDEV_PAD_SIZE || + !ISP2(cp->provider->sectorsize)) { + ZFS_LOG(1, "Provider %s has unsupported sectorsize.", + cp->provider->name); + + vdev_geom_close_locked(vd); + error = EINVAL; + cp = NULL; + } else if (cp->acw == 0 && (spamode & FWRITE) != 0) { + int i; + + for (i = 0; i < 5; i++) { + error = g_access(cp, 0, 1, 0); + if (error == 0) + break; + g_topology_unlock(); + tsleep(vd, 0, "vdev", hz / 2); + g_topology_lock(); + } + if (error != 0) { + printf("ZFS WARNING: Unable to open %s for " + "writing (error=%d).\n", + cp->provider->name, error); + vdev_geom_close_locked(vd); + cp = NULL; + } + } + } + + /* Fetch initial physical path information for this device. */ + if (cp != NULL) { + vdev_geom_attrchanged(cp, "GEOM::physpath"); + + /* Set other GEOM characteristics */ + vdev_geom_set_physpath(vd, cp, /* do_null_update */B_FALSE); + } + + g_topology_unlock(); + PICKUP_GIANT(); + if (cp == NULL) { + vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED; + vdev_dbgmsg(vd, "vdev_geom_open: failed to open [error=%d]", + error); + return (error); + } +skip_open: + pp = cp->provider; + + /* + * Determine the actual size of the device. + */ + *max_psize = *psize = pp->mediasize; + + /* + * Determine the device's minimum transfer size and preferred + * transfer size. + */ + *logical_ashift = highbit(MAX(pp->sectorsize, SPA_MINBLOCKSIZE)) - 1; +#ifdef notyet + if (pp->stripesize > (1 << *logical_ashift) && ISP2(pp->stripesize) && + pp->stripesize <= (1 << ASHIFT_MAX) && pp->stripeoffset == 0) + *physical_ashift = highbit(pp->stripesize) - 1; +#endif + /* + * Clear the nowritecache settings, so that on a vdev_reopen() + * we will try again. + */ + vd->vdev_nowritecache = B_FALSE; + + /* Inform the ZIO pipeline that we are non-rotational. */ + error = g_getattr("GEOM::rotation_rate", cp, &rate); + if (error == 0 && rate == DISK_RR_NON_ROTATING) + vd->vdev_nonrot = B_TRUE; + else + vd->vdev_nonrot = B_FALSE; + + /* Set when device reports it supports TRIM. */ + error = g_getattr("GEOM::candelete", cp, &has_trim); + vd->vdev_has_trim = (error == 0 && has_trim); + + /* Set when device reports it supports secure TRIM. */ + /* unavailable on FreeBSD */ + vd->vdev_has_securetrim = B_FALSE; + + return (0); +} + +static void +vdev_geom_close(vdev_t *vd) +{ + struct g_consumer *cp; + + cp = vd->vdev_tsd; + + DROP_GIANT(); + g_topology_lock(); + + if (!vd->vdev_reopening || + (cp != NULL && ((cp->flags & G_CF_ORPHAN) != 0 || + (cp->provider != NULL && cp->provider->error != 0)))) + vdev_geom_close_locked(vd); + + g_topology_unlock(); + PICKUP_GIANT(); +} + +static void +vdev_geom_io_intr(struct bio *bp) +{ + vdev_t *vd; + zio_t *zio; + + zio = bp->bio_caller1; + vd = zio->io_vd; + zio->io_error = bp->bio_error; + if (zio->io_error == 0 && bp->bio_resid != 0) + zio->io_error = SET_ERROR(EIO); + + switch (zio->io_error) { + case ENOTSUP: + /* + * If we get ENOTSUP for BIO_FLUSH or BIO_DELETE we know + * that future attempts will never succeed. In this case + * we set a persistent flag so that we don't bother with + * requests in the future. + */ + switch (bp->bio_cmd) { + case BIO_FLUSH: + vd->vdev_nowritecache = B_TRUE; + break; + case BIO_DELETE: + break; + } + break; + case ENXIO: + if (!vd->vdev_remove_wanted) { + /* + * If provider's error is set we assume it is being + * removed. + */ + if (bp->bio_to->error != 0) { + vd->vdev_remove_wanted = B_TRUE; + spa_async_request(zio->io_spa, + SPA_ASYNC_REMOVE); + } else if (!vd->vdev_delayed_close) { + vd->vdev_delayed_close = B_TRUE; + } + } + break; + } + + /* + * We have to split bio freeing into two parts, because the ABD code + * cannot be called in this context and vdev_op_io_done is not called + * for ZIO_TYPE_IOCTL zio-s. + */ + if (zio->io_type != ZIO_TYPE_READ && zio->io_type != ZIO_TYPE_WRITE) { + g_destroy_bio(bp); + zio->io_bio = NULL; + } + zio_delay_interrupt(zio); +} + +static void +vdev_geom_io_start(zio_t *zio) +{ + vdev_t *vd; + struct g_consumer *cp; + struct bio *bp; + + vd = zio->io_vd; + + switch (zio->io_type) { + case ZIO_TYPE_IOCTL: + /* XXPOLICY */ + if (!vdev_readable(vd)) { + zio->io_error = SET_ERROR(ENXIO); + zio_interrupt(zio); + return; + } else { + switch (zio->io_cmd) { + case DKIOCFLUSHWRITECACHE: + if (zfs_nocacheflush || + vdev_geom_bio_flush_disable) + break; + if (vd->vdev_nowritecache) { + zio->io_error = SET_ERROR(ENOTSUP); + break; + } + goto sendreq; + default: + zio->io_error = SET_ERROR(ENOTSUP); + } + } + + zio_execute(zio); + return; + case ZIO_TYPE_TRIM: + if (!vdev_geom_bio_delete_disable) { + goto sendreq; + } + zio_execute(zio); + return; + default: + ; + /* PASSTHROUGH --- placate compiler */ + } +sendreq: + ASSERT(zio->io_type == ZIO_TYPE_READ || + zio->io_type == ZIO_TYPE_WRITE || + zio->io_type == ZIO_TYPE_TRIM || + zio->io_type == ZIO_TYPE_IOCTL); + + cp = vd->vdev_tsd; + if (cp == NULL) { + zio->io_error = SET_ERROR(ENXIO); + zio_interrupt(zio); + return; + } + bp = g_alloc_bio(); + bp->bio_caller1 = zio; + switch (zio->io_type) { + case ZIO_TYPE_READ: + case ZIO_TYPE_WRITE: + zio->io_target_timestamp = zio_handle_io_delay(zio); + bp->bio_offset = zio->io_offset; + bp->bio_length = zio->io_size; + if (zio->io_type == ZIO_TYPE_READ) { + bp->bio_cmd = BIO_READ; + bp->bio_data = + abd_borrow_buf(zio->io_abd, zio->io_size); + } else { + bp->bio_cmd = BIO_WRITE; + bp->bio_data = + abd_borrow_buf_copy(zio->io_abd, zio->io_size); + } + break; + case ZIO_TYPE_TRIM: + bp->bio_cmd = BIO_DELETE; + bp->bio_data = NULL; + bp->bio_offset = zio->io_offset; + bp->bio_length = zio->io_size; + break; + case ZIO_TYPE_IOCTL: + bp->bio_cmd = BIO_FLUSH; + bp->bio_flags |= BIO_ORDERED; + bp->bio_data = NULL; + bp->bio_offset = cp->provider->mediasize; + bp->bio_length = 0; + break; + default: + panic("invalid zio->io_type: %d\n", zio->io_type); + } + bp->bio_done = vdev_geom_io_intr; + zio->io_bio = bp; + + g_io_request(bp, cp); +} + +static void +vdev_geom_io_done(zio_t *zio) +{ + struct bio *bp = zio->io_bio; + + if (zio->io_type != ZIO_TYPE_READ && zio->io_type != ZIO_TYPE_WRITE) { + ASSERT(bp == NULL); + return; + } + + if (bp == NULL) { + ASSERT3S(zio->io_error, ==, ENXIO); + return; + } + + if (zio->io_type == ZIO_TYPE_READ) + abd_return_buf_copy(zio->io_abd, bp->bio_data, zio->io_size); + else + abd_return_buf(zio->io_abd, bp->bio_data, zio->io_size); + + g_destroy_bio(bp); + zio->io_bio = NULL; +} + +static void +vdev_geom_hold(vdev_t *vd) +{ +} + +static void +vdev_geom_rele(vdev_t *vd) +{ +} + +vdev_ops_t vdev_disk_ops = { + vdev_geom_open, + vdev_geom_close, + vdev_default_asize, + vdev_geom_io_start, + vdev_geom_io_done, + NULL, + NULL, + vdev_geom_hold, + vdev_geom_rele, + NULL, + vdev_default_xlate, + VDEV_TYPE_DISK, /* name of this vdev type */ + B_TRUE /* leaf vdev */ +}; diff --git a/module/os/freebsd/zfs/vdev_label_os.c b/module/os/freebsd/zfs/vdev_label_os.c new file mode 100644 index 00000000000..e734a2af837 --- /dev/null +++ b/module/os/freebsd/zfs/vdev_label_os.c @@ -0,0 +1,74 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int +vdev_label_write_pad2(vdev_t *vd, const char *buf, size_t size) +{ + spa_t *spa = vd->vdev_spa; + zio_t *zio; + abd_t *pad2; + int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL; + int error; + + if (size > VDEV_PAD_SIZE) + return (EINVAL); + + if (!vd->vdev_ops->vdev_op_leaf) + return (ENODEV); + if (vdev_is_dead(vd)) + return (ENXIO); + + ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); + + pad2 = abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE); + abd_zero(pad2, VDEV_PAD_SIZE); + abd_copy_from_buf(pad2, buf, size); + +retry: + zio = zio_root(spa, NULL, NULL, flags); + vdev_label_write(zio, vd, 0, pad2, + offsetof(vdev_label_t, vl_pad2), + VDEV_PAD_SIZE, NULL, NULL, flags); + error = zio_wait(zio); + if (error != 0 && !(flags & ZIO_FLAG_TRYHARD)) { + flags |= ZIO_FLAG_TRYHARD; + goto retry; + } + + abd_free(pad2); + return (error); +} diff --git a/module/os/freebsd/zfs/zfs_acl.c b/module/os/freebsd/zfs/zfs_acl.c new file mode 100644 index 00000000000..c11e1643750 --- /dev/null +++ b/module/os/freebsd/zfs/zfs_acl.c @@ -0,0 +1,2738 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013 by Delphix. All rights reserved. + * Copyright 2017 Nexenta Systems, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define ALLOW ACE_ACCESS_ALLOWED_ACE_TYPE +#define DENY ACE_ACCESS_DENIED_ACE_TYPE +#define MAX_ACE_TYPE ACE_SYSTEM_ALARM_CALLBACK_OBJECT_ACE_TYPE +#define MIN_ACE_TYPE ALLOW + +#define OWNING_GROUP (ACE_GROUP|ACE_IDENTIFIER_GROUP) +#define EVERYONE_ALLOW_MASK (ACE_READ_ACL|ACE_READ_ATTRIBUTES | \ + ACE_READ_NAMED_ATTRS|ACE_SYNCHRONIZE) +#define EVERYONE_DENY_MASK (ACE_WRITE_ACL|ACE_WRITE_OWNER | \ + ACE_WRITE_ATTRIBUTES|ACE_WRITE_NAMED_ATTRS) +#define OWNER_ALLOW_MASK (ACE_WRITE_ACL | ACE_WRITE_OWNER | \ + ACE_WRITE_ATTRIBUTES|ACE_WRITE_NAMED_ATTRS) + +#define ZFS_CHECKED_MASKS (ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_DATA| \ + ACE_READ_NAMED_ATTRS|ACE_WRITE_DATA|ACE_WRITE_ATTRIBUTES| \ + ACE_WRITE_NAMED_ATTRS|ACE_APPEND_DATA|ACE_EXECUTE|ACE_WRITE_OWNER| \ + ACE_WRITE_ACL|ACE_DELETE|ACE_DELETE_CHILD|ACE_SYNCHRONIZE) + +#define WRITE_MASK_DATA (ACE_WRITE_DATA|ACE_APPEND_DATA|ACE_WRITE_NAMED_ATTRS) +#define WRITE_MASK_ATTRS (ACE_WRITE_ACL|ACE_WRITE_OWNER|ACE_WRITE_ATTRIBUTES| \ + ACE_DELETE|ACE_DELETE_CHILD) +#define WRITE_MASK (WRITE_MASK_DATA|WRITE_MASK_ATTRS) + +#define OGE_CLEAR (ACE_READ_DATA|ACE_LIST_DIRECTORY|ACE_WRITE_DATA| \ + ACE_ADD_FILE|ACE_APPEND_DATA|ACE_ADD_SUBDIRECTORY|ACE_EXECUTE) + +#define OKAY_MASK_BITS (ACE_READ_DATA|ACE_LIST_DIRECTORY|ACE_WRITE_DATA| \ + ACE_ADD_FILE|ACE_APPEND_DATA|ACE_ADD_SUBDIRECTORY|ACE_EXECUTE) + +#define ALL_INHERIT (ACE_FILE_INHERIT_ACE|ACE_DIRECTORY_INHERIT_ACE | \ + ACE_NO_PROPAGATE_INHERIT_ACE|ACE_INHERIT_ONLY_ACE|ACE_INHERITED_ACE) + +#define RESTRICTED_CLEAR (ACE_WRITE_ACL|ACE_WRITE_OWNER) + +#define V4_ACL_WIDE_FLAGS (ZFS_ACL_AUTO_INHERIT|ZFS_ACL_DEFAULTED|\ + ZFS_ACL_PROTECTED) + +#define ZFS_ACL_WIDE_FLAGS (V4_ACL_WIDE_FLAGS|ZFS_ACL_TRIVIAL|ZFS_INHERIT_ACE|\ + ZFS_ACL_OBJ_ACE) + +#define ALL_MODE_EXECS (S_IXUSR | S_IXGRP | S_IXOTH) + +static uint16_t +zfs_ace_v0_get_type(void *acep) +{ + return (((zfs_oldace_t *)acep)->z_type); +} + +static uint16_t +zfs_ace_v0_get_flags(void *acep) +{ + return (((zfs_oldace_t *)acep)->z_flags); +} + +static uint32_t +zfs_ace_v0_get_mask(void *acep) +{ + return (((zfs_oldace_t *)acep)->z_access_mask); +} + +static uint64_t +zfs_ace_v0_get_who(void *acep) +{ + return (((zfs_oldace_t *)acep)->z_fuid); +} + +static void +zfs_ace_v0_set_type(void *acep, uint16_t type) +{ + ((zfs_oldace_t *)acep)->z_type = type; +} + +static void +zfs_ace_v0_set_flags(void *acep, uint16_t flags) +{ + ((zfs_oldace_t *)acep)->z_flags = flags; +} + +static void +zfs_ace_v0_set_mask(void *acep, uint32_t mask) +{ + ((zfs_oldace_t *)acep)->z_access_mask = mask; +} + +static void +zfs_ace_v0_set_who(void *acep, uint64_t who) +{ + ((zfs_oldace_t *)acep)->z_fuid = who; +} + +/*ARGSUSED*/ +static size_t +zfs_ace_v0_size(void *acep) +{ + return (sizeof (zfs_oldace_t)); +} + +static size_t +zfs_ace_v0_abstract_size(void) +{ + return (sizeof (zfs_oldace_t)); +} + +static int +zfs_ace_v0_mask_off(void) +{ + return (offsetof(zfs_oldace_t, z_access_mask)); +} + +/*ARGSUSED*/ +static int +zfs_ace_v0_data(void *acep, void **datap) +{ + *datap = NULL; + return (0); +} + +static acl_ops_t zfs_acl_v0_ops = { + zfs_ace_v0_get_mask, + zfs_ace_v0_set_mask, + zfs_ace_v0_get_flags, + zfs_ace_v0_set_flags, + zfs_ace_v0_get_type, + zfs_ace_v0_set_type, + zfs_ace_v0_get_who, + zfs_ace_v0_set_who, + zfs_ace_v0_size, + zfs_ace_v0_abstract_size, + zfs_ace_v0_mask_off, + zfs_ace_v0_data +}; + +static uint16_t +zfs_ace_fuid_get_type(void *acep) +{ + return (((zfs_ace_hdr_t *)acep)->z_type); +} + +static uint16_t +zfs_ace_fuid_get_flags(void *acep) +{ + return (((zfs_ace_hdr_t *)acep)->z_flags); +} + +static uint32_t +zfs_ace_fuid_get_mask(void *acep) +{ + return (((zfs_ace_hdr_t *)acep)->z_access_mask); +} + +static uint64_t +zfs_ace_fuid_get_who(void *args) +{ + uint16_t entry_type; + zfs_ace_t *acep = args; + + entry_type = acep->z_hdr.z_flags & ACE_TYPE_FLAGS; + + if (entry_type == ACE_OWNER || entry_type == OWNING_GROUP || + entry_type == ACE_EVERYONE) + return (-1); + return (((zfs_ace_t *)acep)->z_fuid); +} + +static void +zfs_ace_fuid_set_type(void *acep, uint16_t type) +{ + ((zfs_ace_hdr_t *)acep)->z_type = type; +} + +static void +zfs_ace_fuid_set_flags(void *acep, uint16_t flags) +{ + ((zfs_ace_hdr_t *)acep)->z_flags = flags; +} + +static void +zfs_ace_fuid_set_mask(void *acep, uint32_t mask) +{ + ((zfs_ace_hdr_t *)acep)->z_access_mask = mask; +} + +static void +zfs_ace_fuid_set_who(void *arg, uint64_t who) +{ + zfs_ace_t *acep = arg; + + uint16_t entry_type = acep->z_hdr.z_flags & ACE_TYPE_FLAGS; + + if (entry_type == ACE_OWNER || entry_type == OWNING_GROUP || + entry_type == ACE_EVERYONE) + return; + acep->z_fuid = who; +} + +static size_t +zfs_ace_fuid_size(void *acep) +{ + zfs_ace_hdr_t *zacep = acep; + uint16_t entry_type; + + switch (zacep->z_type) { + case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE: + case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE: + case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE: + case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE: + return (sizeof (zfs_object_ace_t)); + case ALLOW: + case DENY: + entry_type = + (((zfs_ace_hdr_t *)acep)->z_flags & ACE_TYPE_FLAGS); + if (entry_type == ACE_OWNER || + entry_type == OWNING_GROUP || + entry_type == ACE_EVERYONE) + return (sizeof (zfs_ace_hdr_t)); + /*FALLTHROUGH*/ + default: + return (sizeof (zfs_ace_t)); + } +} + +static size_t +zfs_ace_fuid_abstract_size(void) +{ + return (sizeof (zfs_ace_hdr_t)); +} + +static int +zfs_ace_fuid_mask_off(void) +{ + return (offsetof(zfs_ace_hdr_t, z_access_mask)); +} + +static int +zfs_ace_fuid_data(void *acep, void **datap) +{ + zfs_ace_t *zacep = acep; + zfs_object_ace_t *zobjp; + + switch (zacep->z_hdr.z_type) { + case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE: + case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE: + case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE: + case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE: + zobjp = acep; + *datap = (caddr_t)zobjp + sizeof (zfs_ace_t); + return (sizeof (zfs_object_ace_t) - sizeof (zfs_ace_t)); + default: + *datap = NULL; + return (0); + } +} + +static acl_ops_t zfs_acl_fuid_ops = { + zfs_ace_fuid_get_mask, + zfs_ace_fuid_set_mask, + zfs_ace_fuid_get_flags, + zfs_ace_fuid_set_flags, + zfs_ace_fuid_get_type, + zfs_ace_fuid_set_type, + zfs_ace_fuid_get_who, + zfs_ace_fuid_set_who, + zfs_ace_fuid_size, + zfs_ace_fuid_abstract_size, + zfs_ace_fuid_mask_off, + zfs_ace_fuid_data +}; + +/* + * The following three functions are provided for compatibility with + * older ZPL version in order to determine if the file use to have + * an external ACL and what version of ACL previously existed on the + * file. Would really be nice to not need this, sigh. + */ +uint64_t +zfs_external_acl(znode_t *zp) +{ + zfs_acl_phys_t acl_phys; + int error; + + if (zp->z_is_sa) + return (0); + + /* + * Need to deal with a potential + * race where zfs_sa_upgrade could cause + * z_isa_sa to change. + * + * If the lookup fails then the state of z_is_sa should have + * changed. + */ + + if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_ZNODE_ACL(zp->z_zfsvfs), + &acl_phys, sizeof (acl_phys))) == 0) + return (acl_phys.z_acl_extern_obj); + else { + /* + * after upgrade the SA_ZPL_ZNODE_ACL should have been + * removed + */ + VERIFY(zp->z_is_sa && error == ENOENT); + return (0); + } +} + +/* + * Determine size of ACL in bytes + * + * This is more complicated than it should be since we have to deal + * with old external ACLs. + */ +static int +zfs_acl_znode_info(znode_t *zp, int *aclsize, int *aclcount, + zfs_acl_phys_t *aclphys) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + uint64_t acl_count; + int size; + int error; + + ASSERT(MUTEX_HELD(&zp->z_acl_lock)); + if (zp->z_is_sa) { + if ((error = sa_size(zp->z_sa_hdl, SA_ZPL_DACL_ACES(zfsvfs), + &size)) != 0) + return (error); + *aclsize = size; + if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_DACL_COUNT(zfsvfs), + &acl_count, sizeof (acl_count))) != 0) + return (error); + *aclcount = acl_count; + } else { + if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_ZNODE_ACL(zfsvfs), + aclphys, sizeof (*aclphys))) != 0) + return (error); + + if (aclphys->z_acl_version == ZFS_ACL_VERSION_INITIAL) { + *aclsize = ZFS_ACL_SIZE(aclphys->z_acl_size); + *aclcount = aclphys->z_acl_size; + } else { + *aclsize = aclphys->z_acl_size; + *aclcount = aclphys->z_acl_count; + } + } + return (0); +} + +int +zfs_znode_acl_version(znode_t *zp) +{ + zfs_acl_phys_t acl_phys; + + if (zp->z_is_sa) + return (ZFS_ACL_VERSION_FUID); + else { + int error; + + /* + * Need to deal with a potential + * race where zfs_sa_upgrade could cause + * z_isa_sa to change. + * + * If the lookup fails then the state of z_is_sa should have + * changed. + */ + if ((error = sa_lookup(zp->z_sa_hdl, + SA_ZPL_ZNODE_ACL(zp->z_zfsvfs), + &acl_phys, sizeof (acl_phys))) == 0) + return (acl_phys.z_acl_version); + else { + /* + * After upgrade SA_ZPL_ZNODE_ACL should have + * been removed. + */ + VERIFY(zp->z_is_sa && error == ENOENT); + return (ZFS_ACL_VERSION_FUID); + } + } +} + +static int +zfs_acl_version(int version) +{ + if (version < ZPL_VERSION_FUID) + return (ZFS_ACL_VERSION_INITIAL); + else + return (ZFS_ACL_VERSION_FUID); +} + +static int +zfs_acl_version_zp(znode_t *zp) +{ + return (zfs_acl_version(zp->z_zfsvfs->z_version)); +} + +zfs_acl_t * +zfs_acl_alloc(int vers) +{ + zfs_acl_t *aclp; + + aclp = kmem_zalloc(sizeof (zfs_acl_t), KM_SLEEP); + list_create(&aclp->z_acl, sizeof (zfs_acl_node_t), + offsetof(zfs_acl_node_t, z_next)); + aclp->z_version = vers; + if (vers == ZFS_ACL_VERSION_FUID) + aclp->z_ops = &zfs_acl_fuid_ops; + else + aclp->z_ops = &zfs_acl_v0_ops; + return (aclp); +} + +zfs_acl_node_t * +zfs_acl_node_alloc(size_t bytes) +{ + zfs_acl_node_t *aclnode; + + aclnode = kmem_zalloc(sizeof (zfs_acl_node_t), KM_SLEEP); + if (bytes) { + aclnode->z_acldata = kmem_alloc(bytes, KM_SLEEP); + aclnode->z_allocdata = aclnode->z_acldata; + aclnode->z_allocsize = bytes; + aclnode->z_size = bytes; + } + + return (aclnode); +} + +static void +zfs_acl_node_free(zfs_acl_node_t *aclnode) +{ + if (aclnode->z_allocsize) + kmem_free(aclnode->z_allocdata, aclnode->z_allocsize); + kmem_free(aclnode, sizeof (zfs_acl_node_t)); +} + +static void +zfs_acl_release_nodes(zfs_acl_t *aclp) +{ + zfs_acl_node_t *aclnode; + + while ((aclnode = list_head(&aclp->z_acl))) { + list_remove(&aclp->z_acl, aclnode); + zfs_acl_node_free(aclnode); + } + aclp->z_acl_count = 0; + aclp->z_acl_bytes = 0; +} + +void +zfs_acl_free(zfs_acl_t *aclp) +{ + zfs_acl_release_nodes(aclp); + list_destroy(&aclp->z_acl); + kmem_free(aclp, sizeof (zfs_acl_t)); +} + +static boolean_t +zfs_acl_valid_ace_type(uint_t type, uint_t flags) +{ + uint16_t entry_type; + + switch (type) { + case ALLOW: + case DENY: + case ACE_SYSTEM_AUDIT_ACE_TYPE: + case ACE_SYSTEM_ALARM_ACE_TYPE: + entry_type = flags & ACE_TYPE_FLAGS; + return (entry_type == ACE_OWNER || + entry_type == OWNING_GROUP || + entry_type == ACE_EVERYONE || entry_type == 0 || + entry_type == ACE_IDENTIFIER_GROUP); + default: + if (type >= MIN_ACE_TYPE && type <= MAX_ACE_TYPE) + return (B_TRUE); + } + return (B_FALSE); +} + +static boolean_t +zfs_ace_valid(vtype_t obj_type, zfs_acl_t *aclp, uint16_t type, uint16_t iflags) +{ + /* + * first check type of entry + */ + + if (!zfs_acl_valid_ace_type(type, iflags)) + return (B_FALSE); + + switch (type) { + case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE: + case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE: + case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE: + case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE: + if (aclp->z_version < ZFS_ACL_VERSION_FUID) + return (B_FALSE); + aclp->z_hints |= ZFS_ACL_OBJ_ACE; + } + + /* + * next check inheritance level flags + */ + + if (obj_type == VDIR && + (iflags & (ACE_FILE_INHERIT_ACE|ACE_DIRECTORY_INHERIT_ACE))) + aclp->z_hints |= ZFS_INHERIT_ACE; + + if (iflags & (ACE_INHERIT_ONLY_ACE|ACE_NO_PROPAGATE_INHERIT_ACE)) { + if ((iflags & (ACE_FILE_INHERIT_ACE| + ACE_DIRECTORY_INHERIT_ACE)) == 0) { + return (B_FALSE); + } + } + + return (B_TRUE); +} + +static void * +zfs_acl_next_ace(zfs_acl_t *aclp, void *start, uint64_t *who, + uint32_t *access_mask, uint16_t *iflags, uint16_t *type) +{ + zfs_acl_node_t *aclnode; + + ASSERT(aclp); + + if (start == NULL) { + aclnode = list_head(&aclp->z_acl); + if (aclnode == NULL) + return (NULL); + + aclp->z_next_ace = aclnode->z_acldata; + aclp->z_curr_node = aclnode; + aclnode->z_ace_idx = 0; + } + + aclnode = aclp->z_curr_node; + + if (aclnode == NULL) + return (NULL); + + if (aclnode->z_ace_idx >= aclnode->z_ace_count) { + aclnode = list_next(&aclp->z_acl, aclnode); + if (aclnode == NULL) + return (NULL); + else { + aclp->z_curr_node = aclnode; + aclnode->z_ace_idx = 0; + aclp->z_next_ace = aclnode->z_acldata; + } + } + + if (aclnode->z_ace_idx < aclnode->z_ace_count) { + void *acep = aclp->z_next_ace; + size_t ace_size; + + /* + * Make sure we don't overstep our bounds + */ + ace_size = aclp->z_ops->ace_size(acep); + + if (((caddr_t)acep + ace_size) > + ((caddr_t)aclnode->z_acldata + aclnode->z_size)) { + return (NULL); + } + + *iflags = aclp->z_ops->ace_flags_get(acep); + *type = aclp->z_ops->ace_type_get(acep); + *access_mask = aclp->z_ops->ace_mask_get(acep); + *who = aclp->z_ops->ace_who_get(acep); + aclp->z_next_ace = (caddr_t)aclp->z_next_ace + ace_size; + aclnode->z_ace_idx++; + + return ((void *)acep); + } + return (NULL); +} + +/*ARGSUSED*/ +static uint64_t +zfs_ace_walk(void *datap, uint64_t cookie, int aclcnt, + uint16_t *flags, uint16_t *type, uint32_t *mask) +{ + zfs_acl_t *aclp = datap; + zfs_ace_hdr_t *acep = (zfs_ace_hdr_t *)(uintptr_t)cookie; + uint64_t who; + + acep = zfs_acl_next_ace(aclp, acep, &who, mask, + flags, type); + return ((uint64_t)(uintptr_t)acep); +} + +/* + * Copy ACE to internal ZFS format. + * While processing the ACL each ACE will be validated for correctness. + * ACE FUIDs will be created later. + */ +int +zfs_copy_ace_2_fuid(zfsvfs_t *zfsvfs, vtype_t obj_type, zfs_acl_t *aclp, + void *datap, zfs_ace_t *z_acl, uint64_t aclcnt, size_t *size, + zfs_fuid_info_t **fuidp, cred_t *cr) +{ + int i; + uint16_t entry_type; + zfs_ace_t *aceptr = z_acl; + ace_t *acep = datap; + zfs_object_ace_t *zobjacep; + ace_object_t *aceobjp; + + for (i = 0; i != aclcnt; i++) { + aceptr->z_hdr.z_access_mask = acep->a_access_mask; + aceptr->z_hdr.z_flags = acep->a_flags; + aceptr->z_hdr.z_type = acep->a_type; + entry_type = aceptr->z_hdr.z_flags & ACE_TYPE_FLAGS; + if (entry_type != ACE_OWNER && entry_type != OWNING_GROUP && + entry_type != ACE_EVERYONE) { + aceptr->z_fuid = zfs_fuid_create(zfsvfs, acep->a_who, + cr, (entry_type == 0) ? + ZFS_ACE_USER : ZFS_ACE_GROUP, fuidp); + } + + /* + * Make sure ACE is valid + */ + if (zfs_ace_valid(obj_type, aclp, aceptr->z_hdr.z_type, + aceptr->z_hdr.z_flags) != B_TRUE) + return (SET_ERROR(EINVAL)); + + switch (acep->a_type) { + case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE: + case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE: + case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE: + case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE: + zobjacep = (zfs_object_ace_t *)aceptr; + aceobjp = (ace_object_t *)acep; + + bcopy(aceobjp->a_obj_type, zobjacep->z_object_type, + sizeof (aceobjp->a_obj_type)); + bcopy(aceobjp->a_inherit_obj_type, + zobjacep->z_inherit_type, + sizeof (aceobjp->a_inherit_obj_type)); + acep = (ace_t *)((caddr_t)acep + sizeof (ace_object_t)); + break; + default: + acep = (ace_t *)((caddr_t)acep + sizeof (ace_t)); + } + + aceptr = (zfs_ace_t *)((caddr_t)aceptr + + aclp->z_ops->ace_size(aceptr)); + } + + *size = (caddr_t)aceptr - (caddr_t)z_acl; + + return (0); +} + +/* + * Copy ZFS ACEs to fixed size ace_t layout + */ +static void +zfs_copy_fuid_2_ace(zfsvfs_t *zfsvfs, zfs_acl_t *aclp, cred_t *cr, + void *datap, int filter) +{ + uint64_t who; + uint32_t access_mask; + uint16_t iflags, type; + zfs_ace_hdr_t *zacep = NULL; + ace_t *acep = datap; + ace_object_t *objacep; + zfs_object_ace_t *zobjacep; + size_t ace_size; + uint16_t entry_type; + + while ((zacep = zfs_acl_next_ace(aclp, zacep, + &who, &access_mask, &iflags, &type))) { + + switch (type) { + case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE: + case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE: + case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE: + case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE: + if (filter) { + continue; + } + zobjacep = (zfs_object_ace_t *)zacep; + objacep = (ace_object_t *)acep; + bcopy(zobjacep->z_object_type, + objacep->a_obj_type, + sizeof (zobjacep->z_object_type)); + bcopy(zobjacep->z_inherit_type, + objacep->a_inherit_obj_type, + sizeof (zobjacep->z_inherit_type)); + ace_size = sizeof (ace_object_t); + break; + default: + ace_size = sizeof (ace_t); + break; + } + + entry_type = (iflags & ACE_TYPE_FLAGS); + if ((entry_type != ACE_OWNER && + entry_type != OWNING_GROUP && + entry_type != ACE_EVERYONE)) { + acep->a_who = zfs_fuid_map_id(zfsvfs, who, + cr, (entry_type & ACE_IDENTIFIER_GROUP) ? + ZFS_ACE_GROUP : ZFS_ACE_USER); + } else { + acep->a_who = (uid_t)(int64_t)who; + } + acep->a_access_mask = access_mask; + acep->a_flags = iflags; + acep->a_type = type; + acep = (ace_t *)((caddr_t)acep + ace_size); + } +} + +static int +zfs_copy_ace_2_oldace(vtype_t obj_type, zfs_acl_t *aclp, ace_t *acep, + zfs_oldace_t *z_acl, int aclcnt, size_t *size) +{ + int i; + zfs_oldace_t *aceptr = z_acl; + + for (i = 0; i != aclcnt; i++, aceptr++) { + aceptr->z_access_mask = acep[i].a_access_mask; + aceptr->z_type = acep[i].a_type; + aceptr->z_flags = acep[i].a_flags; + aceptr->z_fuid = acep[i].a_who; + /* + * Make sure ACE is valid + */ + if (zfs_ace_valid(obj_type, aclp, aceptr->z_type, + aceptr->z_flags) != B_TRUE) + return (SET_ERROR(EINVAL)); + } + *size = (caddr_t)aceptr - (caddr_t)z_acl; + return (0); +} + +/* + * convert old ACL format to new + */ +void +zfs_acl_xform(znode_t *zp, zfs_acl_t *aclp, cred_t *cr) +{ + zfs_oldace_t *oldaclp; + int i; + uint16_t type, iflags; + uint32_t access_mask; + uint64_t who; + void *cookie = NULL; + zfs_acl_node_t *newaclnode; + + ASSERT(aclp->z_version == ZFS_ACL_VERSION_INITIAL); + /* + * First create the ACE in a contiguous piece of memory + * for zfs_copy_ace_2_fuid(). + * + * We only convert an ACL once, so this won't happen + * everytime. + */ + oldaclp = kmem_alloc(sizeof (zfs_oldace_t) * aclp->z_acl_count, + KM_SLEEP); + i = 0; + while ((cookie = zfs_acl_next_ace(aclp, cookie, &who, + &access_mask, &iflags, &type))) { + oldaclp[i].z_flags = iflags; + oldaclp[i].z_type = type; + oldaclp[i].z_fuid = who; + oldaclp[i++].z_access_mask = access_mask; + } + + newaclnode = zfs_acl_node_alloc(aclp->z_acl_count * + sizeof (zfs_object_ace_t)); + aclp->z_ops = &zfs_acl_fuid_ops; + VERIFY(zfs_copy_ace_2_fuid(zp->z_zfsvfs, ZTOV(zp)->v_type, aclp, + oldaclp, newaclnode->z_acldata, aclp->z_acl_count, + &newaclnode->z_size, NULL, cr) == 0); + newaclnode->z_ace_count = aclp->z_acl_count; + aclp->z_version = ZFS_ACL_VERSION; + kmem_free(oldaclp, aclp->z_acl_count * sizeof (zfs_oldace_t)); + + /* + * Release all previous ACL nodes + */ + + zfs_acl_release_nodes(aclp); + + list_insert_head(&aclp->z_acl, newaclnode); + + aclp->z_acl_bytes = newaclnode->z_size; + aclp->z_acl_count = newaclnode->z_ace_count; + +} + +/* + * Convert unix access mask to v4 access mask + */ +static uint32_t +zfs_unix_to_v4(uint32_t access_mask) +{ + uint32_t new_mask = 0; + + if (access_mask & S_IXOTH) + new_mask |= ACE_EXECUTE; + if (access_mask & S_IWOTH) + new_mask |= ACE_WRITE_DATA; + if (access_mask & S_IROTH) + new_mask |= ACE_READ_DATA; + return (new_mask); +} + +static void +zfs_set_ace(zfs_acl_t *aclp, void *acep, uint32_t access_mask, + uint16_t access_type, uint64_t fuid, uint16_t entry_type) +{ + uint16_t type = entry_type & ACE_TYPE_FLAGS; + + aclp->z_ops->ace_mask_set(acep, access_mask); + aclp->z_ops->ace_type_set(acep, access_type); + aclp->z_ops->ace_flags_set(acep, entry_type); + if ((type != ACE_OWNER && type != OWNING_GROUP && + type != ACE_EVERYONE)) + aclp->z_ops->ace_who_set(acep, fuid); +} + +/* + * Determine mode of file based on ACL. + */ +uint64_t +zfs_mode_compute(uint64_t fmode, zfs_acl_t *aclp, + uint64_t *pflags, uint64_t fuid, uint64_t fgid) +{ + int entry_type; + mode_t mode; + mode_t seen = 0; + zfs_ace_hdr_t *acep = NULL; + uint64_t who; + uint16_t iflags, type; + uint32_t access_mask; + boolean_t an_exec_denied = B_FALSE; + + mode = (fmode & (S_IFMT | S_ISUID | S_ISGID | S_ISVTX)); + + while ((acep = zfs_acl_next_ace(aclp, acep, &who, + &access_mask, &iflags, &type))) { + + if (!zfs_acl_valid_ace_type(type, iflags)) + continue; + + entry_type = (iflags & ACE_TYPE_FLAGS); + + /* + * Skip over any inherit_only ACEs + */ + if (iflags & ACE_INHERIT_ONLY_ACE) + continue; + + if (entry_type == ACE_OWNER || (entry_type == 0 && + who == fuid)) { + if ((access_mask & ACE_READ_DATA) && + (!(seen & S_IRUSR))) { + seen |= S_IRUSR; + if (type == ALLOW) { + mode |= S_IRUSR; + } + } + if ((access_mask & ACE_WRITE_DATA) && + (!(seen & S_IWUSR))) { + seen |= S_IWUSR; + if (type == ALLOW) { + mode |= S_IWUSR; + } + } + if ((access_mask & ACE_EXECUTE) && + (!(seen & S_IXUSR))) { + seen |= S_IXUSR; + if (type == ALLOW) { + mode |= S_IXUSR; + } + } + } else if (entry_type == OWNING_GROUP || + (entry_type == ACE_IDENTIFIER_GROUP && who == fgid)) { + if ((access_mask & ACE_READ_DATA) && + (!(seen & S_IRGRP))) { + seen |= S_IRGRP; + if (type == ALLOW) { + mode |= S_IRGRP; + } + } + if ((access_mask & ACE_WRITE_DATA) && + (!(seen & S_IWGRP))) { + seen |= S_IWGRP; + if (type == ALLOW) { + mode |= S_IWGRP; + } + } + if ((access_mask & ACE_EXECUTE) && + (!(seen & S_IXGRP))) { + seen |= S_IXGRP; + if (type == ALLOW) { + mode |= S_IXGRP; + } + } + } else if (entry_type == ACE_EVERYONE) { + if ((access_mask & ACE_READ_DATA)) { + if (!(seen & S_IRUSR)) { + seen |= S_IRUSR; + if (type == ALLOW) { + mode |= S_IRUSR; + } + } + if (!(seen & S_IRGRP)) { + seen |= S_IRGRP; + if (type == ALLOW) { + mode |= S_IRGRP; + } + } + if (!(seen & S_IROTH)) { + seen |= S_IROTH; + if (type == ALLOW) { + mode |= S_IROTH; + } + } + } + if ((access_mask & ACE_WRITE_DATA)) { + if (!(seen & S_IWUSR)) { + seen |= S_IWUSR; + if (type == ALLOW) { + mode |= S_IWUSR; + } + } + if (!(seen & S_IWGRP)) { + seen |= S_IWGRP; + if (type == ALLOW) { + mode |= S_IWGRP; + } + } + if (!(seen & S_IWOTH)) { + seen |= S_IWOTH; + if (type == ALLOW) { + mode |= S_IWOTH; + } + } + } + if ((access_mask & ACE_EXECUTE)) { + if (!(seen & S_IXUSR)) { + seen |= S_IXUSR; + if (type == ALLOW) { + mode |= S_IXUSR; + } + } + if (!(seen & S_IXGRP)) { + seen |= S_IXGRP; + if (type == ALLOW) { + mode |= S_IXGRP; + } + } + if (!(seen & S_IXOTH)) { + seen |= S_IXOTH; + if (type == ALLOW) { + mode |= S_IXOTH; + } + } + } + } else { + /* + * Only care if this IDENTIFIER_GROUP or + * USER ACE denies execute access to someone, + * mode is not affected + */ + if ((access_mask & ACE_EXECUTE) && type == DENY) + an_exec_denied = B_TRUE; + } + } + + /* + * Failure to allow is effectively a deny, so execute permission + * is denied if it was never mentioned or if we explicitly + * weren't allowed it. + */ + if (!an_exec_denied && + ((seen & ALL_MODE_EXECS) != ALL_MODE_EXECS || + (mode & ALL_MODE_EXECS) != ALL_MODE_EXECS)) + an_exec_denied = B_TRUE; + + if (an_exec_denied) + *pflags &= ~ZFS_NO_EXECS_DENIED; + else + *pflags |= ZFS_NO_EXECS_DENIED; + + return (mode); +} + +/* + * Read an external acl object. If the intent is to modify, always + * create a new acl and leave any cached acl in place. + */ +int +zfs_acl_node_read(znode_t *zp, boolean_t have_lock, zfs_acl_t **aclpp, + boolean_t will_modify) +{ + zfs_acl_t *aclp; + int aclsize; + int acl_count; + zfs_acl_node_t *aclnode; + zfs_acl_phys_t znode_acl; + int version; + int error; + + ASSERT(MUTEX_HELD(&zp->z_acl_lock)); + if (zp->z_zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_LOCKED(ZTOV(zp), __func__); + + if (zp->z_acl_cached && !will_modify) { + *aclpp = zp->z_acl_cached; + return (0); + } + + version = zfs_znode_acl_version(zp); + + if ((error = zfs_acl_znode_info(zp, &aclsize, + &acl_count, &znode_acl)) != 0) { + goto done; + } + + aclp = zfs_acl_alloc(version); + + aclp->z_acl_count = acl_count; + aclp->z_acl_bytes = aclsize; + + aclnode = zfs_acl_node_alloc(aclsize); + aclnode->z_ace_count = aclp->z_acl_count; + aclnode->z_size = aclsize; + + if (!zp->z_is_sa) { + if (znode_acl.z_acl_extern_obj) { + error = dmu_read(zp->z_zfsvfs->z_os, + znode_acl.z_acl_extern_obj, 0, aclnode->z_size, + aclnode->z_acldata, DMU_READ_PREFETCH); + } else { + bcopy(znode_acl.z_ace_data, aclnode->z_acldata, + aclnode->z_size); + } + } else { + error = sa_lookup(zp->z_sa_hdl, SA_ZPL_DACL_ACES(zp->z_zfsvfs), + aclnode->z_acldata, aclnode->z_size); + } + + if (error != 0) { + zfs_acl_free(aclp); + zfs_acl_node_free(aclnode); + /* convert checksum errors into IO errors */ + if (error == ECKSUM) + error = SET_ERROR(EIO); + goto done; + } + + list_insert_head(&aclp->z_acl, aclnode); + + *aclpp = aclp; + if (!will_modify) + zp->z_acl_cached = aclp; +done: + return (error); +} + +/*ARGSUSED*/ +void +zfs_acl_data_locator(void **dataptr, uint32_t *length, uint32_t buflen, + boolean_t start, void *userdata) +{ + zfs_acl_locator_cb_t *cb = (zfs_acl_locator_cb_t *)userdata; + + if (start) { + cb->cb_acl_node = list_head(&cb->cb_aclp->z_acl); + } else { + cb->cb_acl_node = list_next(&cb->cb_aclp->z_acl, + cb->cb_acl_node); + } + *dataptr = cb->cb_acl_node->z_acldata; + *length = cb->cb_acl_node->z_size; +} + +int +zfs_acl_chown_setattr(znode_t *zp) +{ + int error; + zfs_acl_t *aclp; + + if (zp->z_zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_ELOCKED(ZTOV(zp), __func__); + ASSERT(MUTEX_HELD(&zp->z_acl_lock)); + + if ((error = zfs_acl_node_read(zp, B_TRUE, &aclp, B_FALSE)) == 0) + zp->z_mode = zfs_mode_compute(zp->z_mode, aclp, + &zp->z_pflags, zp->z_uid, zp->z_gid); + return (error); +} + +/* + * common code for setting ACLs. + * + * This function is called from zfs_mode_update, zfs_perm_init, and zfs_setacl. + * zfs_setacl passes a non-NULL inherit pointer (ihp) to indicate that it's + * already checked the acl and knows whether to inherit. + */ +int +zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx) +{ + int error; + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + dmu_object_type_t otype; + zfs_acl_locator_cb_t locate = { 0 }; + uint64_t mode; + sa_bulk_attr_t bulk[5]; + uint64_t ctime[2]; + int count = 0; + zfs_acl_phys_t acl_phys; + + mode = zp->z_mode; + + mode = zfs_mode_compute(mode, aclp, &zp->z_pflags, + zp->z_uid, zp->z_gid); + + zp->z_mode = mode; + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL, + &mode, sizeof (mode)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, + &zp->z_pflags, sizeof (zp->z_pflags)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, + &ctime, sizeof (ctime)); + + if (zp->z_acl_cached) { + zfs_acl_free(zp->z_acl_cached); + zp->z_acl_cached = NULL; + } + + /* + * Upgrade needed? + */ + if (!zfsvfs->z_use_fuids) { + otype = DMU_OT_OLDACL; + } else { + if ((aclp->z_version == ZFS_ACL_VERSION_INITIAL) && + (zfsvfs->z_version >= ZPL_VERSION_FUID)) + zfs_acl_xform(zp, aclp, cr); + ASSERT(aclp->z_version >= ZFS_ACL_VERSION_FUID); + otype = DMU_OT_ACL; + } + + /* + * Arrgh, we have to handle old on disk format + * as well as newer (preferred) SA format. + */ + + if (zp->z_is_sa) { /* the easy case, just update the ACL attribute */ + locate.cb_aclp = aclp; + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_DACL_ACES(zfsvfs), + zfs_acl_data_locator, &locate, aclp->z_acl_bytes); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_DACL_COUNT(zfsvfs), + NULL, &aclp->z_acl_count, sizeof (uint64_t)); + } else { /* Painful legacy way */ + zfs_acl_node_t *aclnode; + uint64_t off = 0; + uint64_t aoid; + + if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_ZNODE_ACL(zfsvfs), + &acl_phys, sizeof (acl_phys))) != 0) + return (error); + + aoid = acl_phys.z_acl_extern_obj; + + if (aclp->z_acl_bytes > ZFS_ACE_SPACE) { + /* + * If ACL was previously external and we are now + * converting to new ACL format then release old + * ACL object and create a new one. + */ + if (aoid && + aclp->z_version != acl_phys.z_acl_version) { + error = dmu_object_free(zfsvfs->z_os, aoid, tx); + if (error) + return (error); + aoid = 0; + } + if (aoid == 0) { + aoid = dmu_object_alloc(zfsvfs->z_os, + otype, aclp->z_acl_bytes, + otype == DMU_OT_ACL ? + DMU_OT_SYSACL : DMU_OT_NONE, + otype == DMU_OT_ACL ? + DN_OLD_MAX_BONUSLEN : 0, tx); + } else { + (void) dmu_object_set_blocksize(zfsvfs->z_os, + aoid, aclp->z_acl_bytes, 0, tx); + } + acl_phys.z_acl_extern_obj = aoid; + for (aclnode = list_head(&aclp->z_acl); aclnode; + aclnode = list_next(&aclp->z_acl, aclnode)) { + if (aclnode->z_ace_count == 0) + continue; + dmu_write(zfsvfs->z_os, aoid, off, + aclnode->z_size, aclnode->z_acldata, tx); + off += aclnode->z_size; + } + } else { + void *start = acl_phys.z_ace_data; + /* + * Migrating back embedded? + */ + if (acl_phys.z_acl_extern_obj) { + error = dmu_object_free(zfsvfs->z_os, + acl_phys.z_acl_extern_obj, tx); + if (error) + return (error); + acl_phys.z_acl_extern_obj = 0; + } + + for (aclnode = list_head(&aclp->z_acl); aclnode; + aclnode = list_next(&aclp->z_acl, aclnode)) { + if (aclnode->z_ace_count == 0) + continue; + bcopy(aclnode->z_acldata, start, + aclnode->z_size); + start = (caddr_t)start + aclnode->z_size; + } + } + /* + * If Old version then swap count/bytes to match old + * layout of znode_acl_phys_t. + */ + if (aclp->z_version == ZFS_ACL_VERSION_INITIAL) { + acl_phys.z_acl_size = aclp->z_acl_count; + acl_phys.z_acl_count = aclp->z_acl_bytes; + } else { + acl_phys.z_acl_size = aclp->z_acl_bytes; + acl_phys.z_acl_count = aclp->z_acl_count; + } + acl_phys.z_acl_version = aclp->z_version; + + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ZNODE_ACL(zfsvfs), NULL, + &acl_phys, sizeof (acl_phys)); + } + + /* + * Replace ACL wide bits, but first clear them. + */ + zp->z_pflags &= ~ZFS_ACL_WIDE_FLAGS; + + zp->z_pflags |= aclp->z_hints; + + if (ace_trivial_common(aclp, 0, zfs_ace_walk) == 0) + zp->z_pflags |= ZFS_ACL_TRIVIAL; + + zfs_tstamp_update_setup(zp, STATE_CHANGED, NULL, ctime); + return (sa_bulk_update(zp->z_sa_hdl, bulk, count, tx)); +} + +static void +zfs_acl_chmod(vtype_t vtype, uint64_t mode, boolean_t split, boolean_t trim, + zfs_acl_t *aclp) +{ + void *acep = NULL; + uint64_t who; + int new_count, new_bytes; + int ace_size; + int entry_type; + uint16_t iflags, type; + uint32_t access_mask; + zfs_acl_node_t *newnode; + size_t abstract_size = aclp->z_ops->ace_abstract_size(); + void *zacep; + boolean_t isdir; + trivial_acl_t masks; + + new_count = new_bytes = 0; + + isdir = (vtype == VDIR); + + acl_trivial_access_masks((mode_t)mode, isdir, &masks); + + newnode = zfs_acl_node_alloc((abstract_size * 6) + aclp->z_acl_bytes); + + zacep = newnode->z_acldata; + if (masks.allow0) { + zfs_set_ace(aclp, zacep, masks.allow0, ALLOW, -1, ACE_OWNER); + zacep = (void *)((uintptr_t)zacep + abstract_size); + new_count++; + new_bytes += abstract_size; + } + if (masks.deny1) { + zfs_set_ace(aclp, zacep, masks.deny1, DENY, -1, ACE_OWNER); + zacep = (void *)((uintptr_t)zacep + abstract_size); + new_count++; + new_bytes += abstract_size; + } + if (masks.deny2) { + zfs_set_ace(aclp, zacep, masks.deny2, DENY, -1, OWNING_GROUP); + zacep = (void *)((uintptr_t)zacep + abstract_size); + new_count++; + new_bytes += abstract_size; + } + + while ((acep = zfs_acl_next_ace(aclp, acep, &who, &access_mask, + &iflags, &type))) { + entry_type = (iflags & ACE_TYPE_FLAGS); + /* + * ACEs used to represent the file mode may be divided + * into an equivalent pair of inherit-only and regular + * ACEs, if they are inheritable. + * Skip regular ACEs, which are replaced by the new mode. + */ + if (split && (entry_type == ACE_OWNER || + entry_type == OWNING_GROUP || + entry_type == ACE_EVERYONE)) { + if (!isdir || !(iflags & + (ACE_FILE_INHERIT_ACE|ACE_DIRECTORY_INHERIT_ACE))) + continue; + /* + * We preserve owner@, group@, or @everyone + * permissions, if they are inheritable, by + * copying them to inherit_only ACEs. This + * prevents inheritable permissions from being + * altered along with the file mode. + */ + iflags |= ACE_INHERIT_ONLY_ACE; + } + + /* + * If this ACL has any inheritable ACEs, mark that in + * the hints (which are later masked into the pflags) + * so create knows to do inheritance. + */ + if (isdir && (iflags & + (ACE_FILE_INHERIT_ACE|ACE_DIRECTORY_INHERIT_ACE))) + aclp->z_hints |= ZFS_INHERIT_ACE; + + if ((type != ALLOW && type != DENY) || + (iflags & ACE_INHERIT_ONLY_ACE)) { + switch (type) { + case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE: + case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE: + case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE: + case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE: + aclp->z_hints |= ZFS_ACL_OBJ_ACE; + break; + } + } else { + /* + * Limit permissions granted by ACEs to be no greater + * than permissions of the requested group mode. + * Applies when the "aclmode" property is set to + * "groupmask". + */ + if ((type == ALLOW) && trim) + access_mask &= masks.group; + } + zfs_set_ace(aclp, zacep, access_mask, type, who, iflags); + ace_size = aclp->z_ops->ace_size(acep); + zacep = (void *)((uintptr_t)zacep + ace_size); + new_count++; + new_bytes += ace_size; + } + zfs_set_ace(aclp, zacep, masks.owner, ALLOW, -1, ACE_OWNER); + zacep = (void *)((uintptr_t)zacep + abstract_size); + zfs_set_ace(aclp, zacep, masks.group, ALLOW, -1, OWNING_GROUP); + zacep = (void *)((uintptr_t)zacep + abstract_size); + zfs_set_ace(aclp, zacep, masks.everyone, ALLOW, -1, ACE_EVERYONE); + + new_count += 3; + new_bytes += abstract_size * 3; + zfs_acl_release_nodes(aclp); + aclp->z_acl_count = new_count; + aclp->z_acl_bytes = new_bytes; + newnode->z_ace_count = new_count; + newnode->z_size = new_bytes; + list_insert_tail(&aclp->z_acl, newnode); +} + +int +zfs_acl_chmod_setattr(znode_t *zp, zfs_acl_t **aclp, uint64_t mode) +{ + int error = 0; + + mutex_enter(&zp->z_acl_lock); + if (zp->z_zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_ELOCKED(ZTOV(zp), __func__); + if (zp->z_zfsvfs->z_acl_mode == ZFS_ACL_DISCARD) + *aclp = zfs_acl_alloc(zfs_acl_version_zp(zp)); + else + error = zfs_acl_node_read(zp, B_TRUE, aclp, B_TRUE); + + if (error == 0) { + (*aclp)->z_hints = zp->z_pflags & V4_ACL_WIDE_FLAGS; + zfs_acl_chmod(ZTOV(zp)->v_type, mode, B_TRUE, + (zp->z_zfsvfs->z_acl_mode == ZFS_ACL_GROUPMASK), *aclp); + } + mutex_exit(&zp->z_acl_lock); + + return (error); +} + +/* + * Should ACE be inherited? + */ +static int +zfs_ace_can_use(vtype_t vtype, uint16_t acep_flags) +{ + int iflags = (acep_flags & 0xf); + + if ((vtype == VDIR) && (iflags & ACE_DIRECTORY_INHERIT_ACE)) + return (1); + else if (iflags & ACE_FILE_INHERIT_ACE) + return (!((vtype == VDIR) && + (iflags & ACE_NO_PROPAGATE_INHERIT_ACE))); + return (0); +} + +/* + * inherit inheritable ACEs from parent + */ +static zfs_acl_t * +zfs_acl_inherit(zfsvfs_t *zfsvfs, vtype_t vtype, zfs_acl_t *paclp, + uint64_t mode, boolean_t *need_chmod) +{ + void *pacep = NULL; + void *acep; + zfs_acl_node_t *aclnode; + zfs_acl_t *aclp = NULL; + uint64_t who; + uint32_t access_mask; + uint16_t iflags, newflags, type; + size_t ace_size; + void *data1, *data2; + size_t data1sz, data2sz; + uint_t aclinherit; + boolean_t isdir = (vtype == VDIR); + boolean_t isreg = (vtype == VREG); + + *need_chmod = B_TRUE; + + aclp = zfs_acl_alloc(paclp->z_version); + aclinherit = zfsvfs->z_acl_inherit; + if (aclinherit == ZFS_ACL_DISCARD || vtype == VLNK) + return (aclp); + + while ((pacep = zfs_acl_next_ace(paclp, pacep, &who, + &access_mask, &iflags, &type))) { + + /* + * don't inherit bogus ACEs + */ + if (!zfs_acl_valid_ace_type(type, iflags)) + continue; + + /* + * Check if ACE is inheritable by this vnode + */ + if ((aclinherit == ZFS_ACL_NOALLOW && type == ALLOW) || + !zfs_ace_can_use(vtype, iflags)) + continue; + + /* + * If owner@, group@, or everyone@ inheritable + * then zfs_acl_chmod() isn't needed. + */ + if ((aclinherit == ZFS_ACL_PASSTHROUGH || + aclinherit == ZFS_ACL_PASSTHROUGH_X) && + ((iflags & (ACE_OWNER|ACE_EVERYONE)) || + ((iflags & OWNING_GROUP) == OWNING_GROUP)) && + (isreg || (isdir && (iflags & ACE_DIRECTORY_INHERIT_ACE)))) + *need_chmod = B_FALSE; + + /* + * Strip inherited execute permission from file if + * not in mode + */ + if (aclinherit == ZFS_ACL_PASSTHROUGH_X && type == ALLOW && + !isdir && ((mode & (S_IXUSR|S_IXGRP|S_IXOTH)) == 0)) { + access_mask &= ~ACE_EXECUTE; + } + + /* + * Strip write_acl and write_owner from permissions + * when inheriting an ACE + */ + if (aclinherit == ZFS_ACL_RESTRICTED && type == ALLOW) { + access_mask &= ~RESTRICTED_CLEAR; + } + + ace_size = aclp->z_ops->ace_size(pacep); + aclnode = zfs_acl_node_alloc(ace_size); + list_insert_tail(&aclp->z_acl, aclnode); + acep = aclnode->z_acldata; + + zfs_set_ace(aclp, acep, access_mask, type, + who, iflags|ACE_INHERITED_ACE); + + /* + * Copy special opaque data if any + */ + if ((data1sz = paclp->z_ops->ace_data(pacep, &data1)) != 0) { + VERIFY((data2sz = aclp->z_ops->ace_data(acep, + &data2)) == data1sz); + bcopy(data1, data2, data2sz); + } + + aclp->z_acl_count++; + aclnode->z_ace_count++; + aclp->z_acl_bytes += aclnode->z_size; + newflags = aclp->z_ops->ace_flags_get(acep); + + /* + * If ACE is not to be inherited further, or if the vnode is + * not a directory, remove all inheritance flags + */ + if (!isdir || (iflags & ACE_NO_PROPAGATE_INHERIT_ACE)) { + newflags &= ~ALL_INHERIT; + aclp->z_ops->ace_flags_set(acep, + newflags|ACE_INHERITED_ACE); + continue; + } + + /* + * This directory has an inheritable ACE + */ + aclp->z_hints |= ZFS_INHERIT_ACE; + + /* + * If only FILE_INHERIT is set then turn on + * inherit_only + */ + if ((iflags & (ACE_FILE_INHERIT_ACE | + ACE_DIRECTORY_INHERIT_ACE)) == ACE_FILE_INHERIT_ACE) { + newflags |= ACE_INHERIT_ONLY_ACE; + aclp->z_ops->ace_flags_set(acep, + newflags|ACE_INHERITED_ACE); + } else { + newflags &= ~ACE_INHERIT_ONLY_ACE; + aclp->z_ops->ace_flags_set(acep, + newflags|ACE_INHERITED_ACE); + } + } + + return (aclp); +} + +/* + * Create file system object initial permissions + * including inheritable ACEs. + * Also, create FUIDs for owner and group. + */ +int +zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr, + vsecattr_t *vsecp, zfs_acl_ids_t *acl_ids) +{ + int error; + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + zfs_acl_t *paclp; + gid_t gid; + boolean_t need_chmod = B_TRUE; + boolean_t trim = B_FALSE; + boolean_t inherited = B_FALSE; + + if ((flag & IS_ROOT_NODE) == 0) { + if (zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_ELOCKED(ZTOV(dzp), __func__); + } else + ASSERT(dzp->z_vnode == NULL); + bzero(acl_ids, sizeof (zfs_acl_ids_t)); + acl_ids->z_mode = MAKEIMODE(vap->va_type, vap->va_mode); + + if (vsecp) + if ((error = zfs_vsec_2_aclp(zfsvfs, vap->va_type, vsecp, cr, + &acl_ids->z_fuidp, &acl_ids->z_aclp)) != 0) + return (error); + /* + * Determine uid and gid. + */ + if ((flag & IS_ROOT_NODE) || zfsvfs->z_replay || + ((flag & IS_XATTR) && (vap->va_type == VDIR))) { + acl_ids->z_fuid = zfs_fuid_create(zfsvfs, + (uint64_t)vap->va_uid, cr, + ZFS_OWNER, &acl_ids->z_fuidp); + acl_ids->z_fgid = zfs_fuid_create(zfsvfs, + (uint64_t)vap->va_gid, cr, + ZFS_GROUP, &acl_ids->z_fuidp); + gid = vap->va_gid; + } else { + acl_ids->z_fuid = zfs_fuid_create_cred(zfsvfs, ZFS_OWNER, + cr, &acl_ids->z_fuidp); + acl_ids->z_fgid = 0; + if (vap->va_mask & AT_GID) { + acl_ids->z_fgid = zfs_fuid_create(zfsvfs, + (uint64_t)vap->va_gid, + cr, ZFS_GROUP, &acl_ids->z_fuidp); + gid = vap->va_gid; + if (acl_ids->z_fgid != dzp->z_gid && + !groupmember(vap->va_gid, cr) && + secpolicy_vnode_create_gid(cr) != 0) + acl_ids->z_fgid = 0; + } + if (acl_ids->z_fgid == 0) { + if (dzp->z_mode & S_ISGID) { + char *domain; + uint32_t rid; + + acl_ids->z_fgid = dzp->z_gid; + gid = zfs_fuid_map_id(zfsvfs, acl_ids->z_fgid, + cr, ZFS_GROUP); + + if (zfsvfs->z_use_fuids && + IS_EPHEMERAL(acl_ids->z_fgid)) { + domain = zfs_fuid_idx_domain( + &zfsvfs->z_fuid_idx, + FUID_INDEX(acl_ids->z_fgid)); + rid = FUID_RID(acl_ids->z_fgid); + zfs_fuid_node_add(&acl_ids->z_fuidp, + domain, rid, + FUID_INDEX(acl_ids->z_fgid), + acl_ids->z_fgid, ZFS_GROUP); + } + } else { + acl_ids->z_fgid = zfs_fuid_create_cred(zfsvfs, + ZFS_GROUP, cr, &acl_ids->z_fuidp); +#ifdef __FreeBSD_kernel__ + gid = acl_ids->z_fgid = dzp->z_gid; +#else + gid = crgetgid(cr); +#endif + } + } + } + + /* + * If we're creating a directory, and the parent directory has the + * set-GID bit set, set in on the new directory. + * Otherwise, if the user is neither privileged nor a member of the + * file's new group, clear the file's set-GID bit. + */ + + if (!(flag & IS_ROOT_NODE) && (dzp->z_mode & S_ISGID) && + (vap->va_type == VDIR)) { + acl_ids->z_mode |= S_ISGID; + } else { + if ((acl_ids->z_mode & S_ISGID) && + secpolicy_vnode_setids_setgids(ZTOV(dzp), cr, gid) != 0) + acl_ids->z_mode &= ~S_ISGID; + } + + if (acl_ids->z_aclp == NULL) { + mutex_enter(&dzp->z_acl_lock); + if (!(flag & IS_ROOT_NODE) && + (dzp->z_pflags & ZFS_INHERIT_ACE) && + !(dzp->z_pflags & ZFS_XATTR)) { + VERIFY0(zfs_acl_node_read(dzp, B_TRUE, + &paclp, B_FALSE)); + acl_ids->z_aclp = zfs_acl_inherit(zfsvfs, + vap->va_type, paclp, acl_ids->z_mode, &need_chmod); + inherited = B_TRUE; + } else { + acl_ids->z_aclp = + zfs_acl_alloc(zfs_acl_version_zp(dzp)); + acl_ids->z_aclp->z_hints |= ZFS_ACL_TRIVIAL; + } + mutex_exit(&dzp->z_acl_lock); + + if (need_chmod) { + if (vap->va_type == VDIR) + acl_ids->z_aclp->z_hints |= + ZFS_ACL_AUTO_INHERIT; + + if (zfsvfs->z_acl_mode == ZFS_ACL_GROUPMASK && + zfsvfs->z_acl_inherit != ZFS_ACL_PASSTHROUGH && + zfsvfs->z_acl_inherit != ZFS_ACL_PASSTHROUGH_X) + trim = B_TRUE; + zfs_acl_chmod(vap->va_type, acl_ids->z_mode, B_FALSE, + trim, acl_ids->z_aclp); + } + } + + if (inherited || vsecp) { + acl_ids->z_mode = zfs_mode_compute(acl_ids->z_mode, + acl_ids->z_aclp, &acl_ids->z_aclp->z_hints, + acl_ids->z_fuid, acl_ids->z_fgid); + if (ace_trivial_common(acl_ids->z_aclp, 0, zfs_ace_walk) == 0) + acl_ids->z_aclp->z_hints |= ZFS_ACL_TRIVIAL; + } + + return (0); +} + +/* + * Free ACL and fuid_infop, but not the acl_ids structure + */ +void +zfs_acl_ids_free(zfs_acl_ids_t *acl_ids) +{ + if (acl_ids->z_aclp) + zfs_acl_free(acl_ids->z_aclp); + if (acl_ids->z_fuidp) + zfs_fuid_info_free(acl_ids->z_fuidp); + acl_ids->z_aclp = NULL; + acl_ids->z_fuidp = NULL; +} + +boolean_t +zfs_acl_ids_overquota(zfsvfs_t *zv, zfs_acl_ids_t *acl_ids, uint64_t projid) +{ + return (zfs_id_overquota(zv, DMU_USERUSED_OBJECT, acl_ids->z_fuid) || + zfs_id_overquota(zv, DMU_GROUPUSED_OBJECT, acl_ids->z_fgid) || + (projid != ZFS_DEFAULT_PROJID && projid != ZFS_INVALID_PROJID && + zfs_id_overquota(zv, DMU_PROJECTUSED_OBJECT, projid))); +} + +/* + * Retrieve a file's ACL + */ +int +zfs_getacl(znode_t *zp, vsecattr_t *vsecp, boolean_t skipaclchk, cred_t *cr) +{ + zfs_acl_t *aclp; + ulong_t mask; + int error; + int count = 0; + int largeace = 0; + + mask = vsecp->vsa_mask & (VSA_ACE | VSA_ACECNT | + VSA_ACE_ACLFLAGS | VSA_ACE_ALLTYPES); + + if (mask == 0) + return (SET_ERROR(ENOSYS)); + + if ((error = zfs_zaccess(zp, ACE_READ_ACL, 0, skipaclchk, cr))) + return (error); + + mutex_enter(&zp->z_acl_lock); + + if (zp->z_zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_LOCKED(ZTOV(zp), __func__); + error = zfs_acl_node_read(zp, B_TRUE, &aclp, B_FALSE); + if (error != 0) { + mutex_exit(&zp->z_acl_lock); + return (error); + } + + /* + * Scan ACL to determine number of ACEs + */ + if ((zp->z_pflags & ZFS_ACL_OBJ_ACE) && !(mask & VSA_ACE_ALLTYPES)) { + void *zacep = NULL; + uint64_t who; + uint32_t access_mask; + uint16_t type, iflags; + + while ((zacep = zfs_acl_next_ace(aclp, zacep, + &who, &access_mask, &iflags, &type))) { + switch (type) { + case ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE: + case ACE_ACCESS_DENIED_OBJECT_ACE_TYPE: + case ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE: + case ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE: + largeace++; + continue; + default: + count++; + } + } + vsecp->vsa_aclcnt = count; + } else + count = (int)aclp->z_acl_count; + + if (mask & VSA_ACECNT) { + vsecp->vsa_aclcnt = count; + } + + if (mask & VSA_ACE) { + size_t aclsz; + + aclsz = count * sizeof (ace_t) + + sizeof (ace_object_t) * largeace; + + vsecp->vsa_aclentp = kmem_alloc(aclsz, KM_SLEEP); + vsecp->vsa_aclentsz = aclsz; + + if (aclp->z_version == ZFS_ACL_VERSION_FUID) + zfs_copy_fuid_2_ace(zp->z_zfsvfs, aclp, cr, + vsecp->vsa_aclentp, !(mask & VSA_ACE_ALLTYPES)); + else { + zfs_acl_node_t *aclnode; + void *start = vsecp->vsa_aclentp; + + for (aclnode = list_head(&aclp->z_acl); aclnode; + aclnode = list_next(&aclp->z_acl, aclnode)) { + bcopy(aclnode->z_acldata, start, + aclnode->z_size); + start = (caddr_t)start + aclnode->z_size; + } + ASSERT((caddr_t)start - (caddr_t)vsecp->vsa_aclentp == + aclp->z_acl_bytes); + } + } + if (mask & VSA_ACE_ACLFLAGS) { + vsecp->vsa_aclflags = 0; + if (zp->z_pflags & ZFS_ACL_DEFAULTED) + vsecp->vsa_aclflags |= ACL_DEFAULTED; + if (zp->z_pflags & ZFS_ACL_PROTECTED) + vsecp->vsa_aclflags |= ACL_PROTECTED; + if (zp->z_pflags & ZFS_ACL_AUTO_INHERIT) + vsecp->vsa_aclflags |= ACL_AUTO_INHERIT; + } + + mutex_exit(&zp->z_acl_lock); + + return (0); +} + +int +zfs_vsec_2_aclp(zfsvfs_t *zfsvfs, umode_t obj_type, + vsecattr_t *vsecp, cred_t *cr, zfs_fuid_info_t **fuidp, zfs_acl_t **zaclp) +{ + zfs_acl_t *aclp; + zfs_acl_node_t *aclnode; + int aclcnt = vsecp->vsa_aclcnt; + int error; + + if (vsecp->vsa_aclcnt > MAX_ACL_ENTRIES || vsecp->vsa_aclcnt <= 0) + return (SET_ERROR(EINVAL)); + + aclp = zfs_acl_alloc(zfs_acl_version(zfsvfs->z_version)); + + aclp->z_hints = 0; + aclnode = zfs_acl_node_alloc(aclcnt * sizeof (zfs_object_ace_t)); + if (aclp->z_version == ZFS_ACL_VERSION_INITIAL) { + if ((error = zfs_copy_ace_2_oldace(obj_type, aclp, + (ace_t *)vsecp->vsa_aclentp, aclnode->z_acldata, + aclcnt, &aclnode->z_size)) != 0) { + zfs_acl_free(aclp); + zfs_acl_node_free(aclnode); + return (error); + } + } else { + if ((error = zfs_copy_ace_2_fuid(zfsvfs, obj_type, aclp, + vsecp->vsa_aclentp, aclnode->z_acldata, aclcnt, + &aclnode->z_size, fuidp, cr)) != 0) { + zfs_acl_free(aclp); + zfs_acl_node_free(aclnode); + return (error); + } + } + aclp->z_acl_bytes = aclnode->z_size; + aclnode->z_ace_count = aclcnt; + aclp->z_acl_count = aclcnt; + list_insert_head(&aclp->z_acl, aclnode); + + /* + * If flags are being set then add them to z_hints + */ + if (vsecp->vsa_mask & VSA_ACE_ACLFLAGS) { + if (vsecp->vsa_aclflags & ACL_PROTECTED) + aclp->z_hints |= ZFS_ACL_PROTECTED; + if (vsecp->vsa_aclflags & ACL_DEFAULTED) + aclp->z_hints |= ZFS_ACL_DEFAULTED; + if (vsecp->vsa_aclflags & ACL_AUTO_INHERIT) + aclp->z_hints |= ZFS_ACL_AUTO_INHERIT; + } + + *zaclp = aclp; + + return (0); +} + +/* + * Set a file's ACL + */ +int +zfs_setacl(znode_t *zp, vsecattr_t *vsecp, boolean_t skipaclchk, cred_t *cr) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + zilog_t *zilog = zfsvfs->z_log; + ulong_t mask = vsecp->vsa_mask & (VSA_ACE | VSA_ACECNT); + dmu_tx_t *tx; + int error; + zfs_acl_t *aclp; + zfs_fuid_info_t *fuidp = NULL; + boolean_t fuid_dirtied; + uint64_t acl_obj; + + if (zp->z_zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_ELOCKED(ZTOV(zp), __func__); + if (mask == 0) + return (SET_ERROR(ENOSYS)); + + if (zp->z_pflags & ZFS_IMMUTABLE) + return (SET_ERROR(EPERM)); + + if ((error = zfs_zaccess(zp, ACE_WRITE_ACL, 0, skipaclchk, cr))) + return (error); + + error = zfs_vsec_2_aclp(zfsvfs, ZTOV(zp)->v_type, vsecp, cr, &fuidp, + &aclp); + if (error) + return (error); + + /* + * If ACL wide flags aren't being set then preserve any + * existing flags. + */ + if (!(vsecp->vsa_mask & VSA_ACE_ACLFLAGS)) { + aclp->z_hints |= + (zp->z_pflags & V4_ACL_WIDE_FLAGS); + } +top: + mutex_enter(&zp->z_acl_lock); + + tx = dmu_tx_create(zfsvfs->z_os); + + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE); + + fuid_dirtied = zfsvfs->z_fuid_dirty; + if (fuid_dirtied) + zfs_fuid_txhold(zfsvfs, tx); + + /* + * If old version and ACL won't fit in bonus and we aren't + * upgrading then take out necessary DMU holds + */ + + if ((acl_obj = zfs_external_acl(zp)) != 0) { + if (zfsvfs->z_version >= ZPL_VERSION_FUID && + zfs_znode_acl_version(zp) <= ZFS_ACL_VERSION_INITIAL) { + dmu_tx_hold_free(tx, acl_obj, 0, + DMU_OBJECT_END); + dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, + aclp->z_acl_bytes); + } else { + dmu_tx_hold_write(tx, acl_obj, 0, aclp->z_acl_bytes); + } + } else if (!zp->z_is_sa && aclp->z_acl_bytes > ZFS_ACE_SPACE) { + dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, aclp->z_acl_bytes); + } + + zfs_sa_upgrade_txholds(tx, zp); + error = dmu_tx_assign(tx, TXG_NOWAIT); + if (error) { + mutex_exit(&zp->z_acl_lock); + + if (error == ERESTART) { + dmu_tx_wait(tx); + dmu_tx_abort(tx); + goto top; + } + dmu_tx_abort(tx); + zfs_acl_free(aclp); + return (error); + } + + error = zfs_aclset_common(zp, aclp, cr, tx); + ASSERT(error == 0); + ASSERT(zp->z_acl_cached == NULL); + zp->z_acl_cached = aclp; + + if (fuid_dirtied) + zfs_fuid_sync(zfsvfs, tx); + + zfs_log_acl(zilog, tx, zp, vsecp, fuidp); + + if (fuidp) + zfs_fuid_info_free(fuidp); + dmu_tx_commit(tx); + mutex_exit(&zp->z_acl_lock); + + return (error); +} + +/* + * Check accesses of interest (AoI) against attributes of the dataset + * such as read-only. Returns zero if no AoI conflict with dataset + * attributes, otherwise an appropriate errno is returned. + */ +static int +zfs_zaccess_dataset_check(znode_t *zp, uint32_t v4_mode) +{ + if ((v4_mode & WRITE_MASK) && + (zp->z_zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) && + (!IS_DEVVP(ZTOV(zp)) || + (IS_DEVVP(ZTOV(zp)) && (v4_mode & WRITE_MASK_ATTRS)))) { + return (SET_ERROR(EROFS)); + } + + /* + * Intentionally allow ZFS_READONLY through here. + * See zfs_zaccess_common(). + */ + if ((v4_mode & WRITE_MASK_DATA) && + (zp->z_pflags & ZFS_IMMUTABLE)) { + return (SET_ERROR(EPERM)); + } + + /* + * In FreeBSD we allow to modify directory's content is ZFS_NOUNLINK + * (sunlnk) is set. We just don't allow directory removal, which is + * handled in zfs_zaccess_delete(). + */ + if ((v4_mode & ACE_DELETE) && + (zp->z_pflags & ZFS_NOUNLINK)) { + return (EPERM); + } + + if (((v4_mode & (ACE_READ_DATA|ACE_EXECUTE)) && + (zp->z_pflags & ZFS_AV_QUARANTINED))) { + return (SET_ERROR(EACCES)); + } + + return (0); +} + +/* + * The primary usage of this function is to loop through all of the + * ACEs in the znode, determining what accesses of interest (AoI) to + * the caller are allowed or denied. The AoI are expressed as bits in + * the working_mode parameter. As each ACE is processed, bits covered + * by that ACE are removed from the working_mode. This removal + * facilitates two things. The first is that when the working mode is + * empty (= 0), we know we've looked at all the AoI. The second is + * that the ACE interpretation rules don't allow a later ACE to undo + * something granted or denied by an earlier ACE. Removing the + * discovered access or denial enforces this rule. At the end of + * processing the ACEs, all AoI that were found to be denied are + * placed into the working_mode, giving the caller a mask of denied + * accesses. Returns: + * 0 if all AoI granted + * EACCESS if the denied mask is non-zero + * other error if abnormal failure (e.g., IO error) + * + * A secondary usage of the function is to determine if any of the + * AoI are granted. If an ACE grants any access in + * the working_mode, we immediately short circuit out of the function. + * This mode is chosen by setting anyaccess to B_TRUE. The + * working_mode is not a denied access mask upon exit if the function + * is used in this manner. + */ +static int +zfs_zaccess_aces_check(znode_t *zp, uint32_t *working_mode, + boolean_t anyaccess, cred_t *cr) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + zfs_acl_t *aclp; + int error; + uid_t uid = crgetuid(cr); + uint64_t who; + uint16_t type, iflags; + uint16_t entry_type; + uint32_t access_mask; + uint32_t deny_mask = 0; + zfs_ace_hdr_t *acep = NULL; + boolean_t checkit; + uid_t gowner; + uid_t fowner; + + zfs_fuid_map_ids(zp, cr, &fowner, &gowner); + + mutex_enter(&zp->z_acl_lock); + + if (zp->z_zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_LOCKED(ZTOV(zp), __func__); + error = zfs_acl_node_read(zp, B_TRUE, &aclp, B_FALSE); + if (error != 0) { + mutex_exit(&zp->z_acl_lock); + return (error); + } + + ASSERT(zp->z_acl_cached); + + while ((acep = zfs_acl_next_ace(aclp, acep, &who, &access_mask, + &iflags, &type))) { + uint32_t mask_matched; + + if (!zfs_acl_valid_ace_type(type, iflags)) + continue; + + if (ZTOV(zp)->v_type == VDIR && (iflags & ACE_INHERIT_ONLY_ACE)) + continue; + + /* Skip ACE if it does not affect any AoI */ + mask_matched = (access_mask & *working_mode); + if (!mask_matched) + continue; + + entry_type = (iflags & ACE_TYPE_FLAGS); + + checkit = B_FALSE; + + switch (entry_type) { + case ACE_OWNER: + if (uid == fowner) + checkit = B_TRUE; + break; + case OWNING_GROUP: + who = gowner; + /*FALLTHROUGH*/ + case ACE_IDENTIFIER_GROUP: + checkit = zfs_groupmember(zfsvfs, who, cr); + break; + case ACE_EVERYONE: + checkit = B_TRUE; + break; + + /* USER Entry */ + default: + if (entry_type == 0) { + uid_t newid; + + newid = zfs_fuid_map_id(zfsvfs, who, cr, + ZFS_ACE_USER); + if (newid != UID_NOBODY && + uid == newid) + checkit = B_TRUE; + break; + } else { + mutex_exit(&zp->z_acl_lock); + return (SET_ERROR(EIO)); + } + } + + if (checkit) { + if (type == DENY) { + DTRACE_PROBE3(zfs__ace__denies, + znode_t *, zp, + zfs_ace_hdr_t *, acep, + uint32_t, mask_matched); + deny_mask |= mask_matched; + } else { + DTRACE_PROBE3(zfs__ace__allows, + znode_t *, zp, + zfs_ace_hdr_t *, acep, + uint32_t, mask_matched); + if (anyaccess) { + mutex_exit(&zp->z_acl_lock); + return (0); + } + } + *working_mode &= ~mask_matched; + } + + /* Are we done? */ + if (*working_mode == 0) + break; + } + + mutex_exit(&zp->z_acl_lock); + + /* Put the found 'denies' back on the working mode */ + if (deny_mask) { + *working_mode |= deny_mask; + return (SET_ERROR(EACCES)); + } else if (*working_mode) { + return (-1); + } + + return (0); +} + +/* + * Return true if any access whatsoever granted, we don't actually + * care what access is granted. + */ +boolean_t +zfs_has_access(znode_t *zp, cred_t *cr) +{ + uint32_t have = ACE_ALL_PERMS; + + if (zfs_zaccess_aces_check(zp, &have, B_TRUE, cr) != 0) { + uid_t owner; + + owner = zfs_fuid_map_id(zp->z_zfsvfs, zp->z_uid, cr, ZFS_OWNER); + return (secpolicy_vnode_any_access(cr, ZTOV(zp), owner) == 0); + } + return (B_TRUE); +} + +static int +zfs_zaccess_common(znode_t *zp, uint32_t v4_mode, uint32_t *working_mode, + boolean_t *check_privs, boolean_t skipaclchk, cred_t *cr) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + int err; + + *working_mode = v4_mode; + *check_privs = B_TRUE; + + /* + * Short circuit empty requests + */ + if (v4_mode == 0 || zfsvfs->z_replay) { + *working_mode = 0; + return (0); + } + + if ((err = zfs_zaccess_dataset_check(zp, v4_mode)) != 0) { + *check_privs = B_FALSE; + return (err); + } + + /* + * The caller requested that the ACL check be skipped. This + * would only happen if the caller checked VOP_ACCESS() with a + * 32 bit ACE mask and already had the appropriate permissions. + */ + if (skipaclchk) { + *working_mode = 0; + return (0); + } + + /* + * Note: ZFS_READONLY represents the "DOS R/O" attribute. + * When that flag is set, we should behave as if write access + * were not granted by anything in the ACL. In particular: + * We _must_ allow writes after opening the file r/w, then + * setting the DOS R/O attribute, and writing some more. + * (Similar to how you can write after fchmod(fd, 0444).) + * + * Therefore ZFS_READONLY is ignored in the dataset check + * above, and checked here as if part of the ACL check. + * Also note: DOS R/O is ignored for directories. + */ + if ((v4_mode & WRITE_MASK_DATA) && + (ZTOV(zp)->v_type != VDIR) && + (zp->z_pflags & ZFS_READONLY)) { + return (SET_ERROR(EPERM)); + } + + return (zfs_zaccess_aces_check(zp, working_mode, B_FALSE, cr)); +} + +static int +zfs_zaccess_append(znode_t *zp, uint32_t *working_mode, boolean_t *check_privs, + cred_t *cr) +{ + if (*working_mode != ACE_WRITE_DATA) + return (SET_ERROR(EACCES)); + + return (zfs_zaccess_common(zp, ACE_APPEND_DATA, working_mode, + check_privs, B_FALSE, cr)); +} + +/* + * Check if VEXEC is allowed. + * + * This routine is based on zfs_fastaccesschk_execute which has slowpath + * calling zfs_zaccess. This would be incorrect on FreeBSD (see + * zfs_freebsd_access for the difference). Thus this variant let's the + * caller handle the slowpath (if necessary). + * + * On top of that we perform a lockless check for ZFS_NO_EXECS_DENIED. + * + * Safe access to znode_t is provided by the vnode lock. + */ +int +zfs_fastaccesschk_execute(znode_t *zdp, cred_t *cr) +{ + boolean_t owner = B_FALSE; + boolean_t groupmbr = B_FALSE; + boolean_t is_attr; + uid_t uid = crgetuid(cr); + + if (zdp->z_pflags & ZFS_AV_QUARANTINED) + return (1); + + is_attr = ((zdp->z_pflags & ZFS_XATTR) && + (ZTOV(zdp)->v_type == VDIR)); + if (is_attr) + return (1); + + if (zdp->z_pflags & ZFS_NO_EXECS_DENIED) + return (0); + + mutex_enter(&zdp->z_acl_lock); + if (FUID_INDEX(zdp->z_uid) != 0 || FUID_INDEX(zdp->z_gid) != 0) { + goto out_slow; + } + + if (uid == zdp->z_uid) { + owner = B_TRUE; + if (zdp->z_mode & S_IXUSR) { + goto out; + } else { + goto out_slow; + } + } + if (groupmember(zdp->z_gid, cr)) { + groupmbr = B_TRUE; + if (zdp->z_mode & S_IXGRP) { + goto out; + } else { + goto out_slow; + } + } + if (!owner && !groupmbr) { + if (zdp->z_mode & S_IXOTH) { + goto out; + } + } +out: + mutex_exit(&zdp->z_acl_lock); + return (0); +out_slow: + mutex_exit(&zdp->z_acl_lock); + return (1); +} + + +/* + * Determine whether Access should be granted/denied. + * + * The least priv subsystem is always consulted as a basic privilege + * can define any form of access. + */ +int +zfs_zaccess(znode_t *zp, int mode, int flags, boolean_t skipaclchk, cred_t *cr) +{ + uint32_t working_mode; + int error; + int is_attr; + boolean_t check_privs; + znode_t *xzp = NULL; + znode_t *check_zp = zp; + mode_t needed_bits; + uid_t owner; + + is_attr = ((zp->z_pflags & ZFS_XATTR) && (ZTOV(zp)->v_type == VDIR)); + +#ifdef __FreeBSD_kernel__ + /* + * In FreeBSD, we don't care about permissions of individual ADS. + * Note that not checking them is not just an optimization - without + * this shortcut, EA operations may bogusly fail with EACCES. + */ + if (zp->z_pflags & ZFS_XATTR) + return (0); +#else + /* + * If attribute then validate against base file + */ + if (is_attr) { + uint64_t parent; + + if ((error = sa_lookup(zp->z_sa_hdl, + SA_ZPL_PARENT(zp->z_zfsvfs), &parent, + sizeof (parent))) != 0) + return (error); + + if ((error = zfs_zget(zp->z_zfsvfs, + parent, &xzp)) != 0) { + return (error); + } + + check_zp = xzp; + + /* + * fixup mode to map to xattr perms + */ + + if (mode & (ACE_WRITE_DATA|ACE_APPEND_DATA)) { + mode &= ~(ACE_WRITE_DATA|ACE_APPEND_DATA); + mode |= ACE_WRITE_NAMED_ATTRS; + } + + if (mode & (ACE_READ_DATA|ACE_EXECUTE)) { + mode &= ~(ACE_READ_DATA|ACE_EXECUTE); + mode |= ACE_READ_NAMED_ATTRS; + } + } +#endif + + owner = zfs_fuid_map_id(zp->z_zfsvfs, zp->z_uid, cr, ZFS_OWNER); + /* + * Map the bits required to the standard vnode flags VREAD|VWRITE|VEXEC + * in needed_bits. Map the bits mapped by working_mode (currently + * missing) in missing_bits. + * Call secpolicy_vnode_access2() with (needed_bits & ~checkmode), + * needed_bits. + */ + needed_bits = 0; + + working_mode = mode; + if ((working_mode & (ACE_READ_ACL|ACE_READ_ATTRIBUTES)) && + owner == crgetuid(cr)) + working_mode &= ~(ACE_READ_ACL|ACE_READ_ATTRIBUTES); + + if (working_mode & (ACE_READ_DATA|ACE_READ_NAMED_ATTRS| + ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_SYNCHRONIZE)) + needed_bits |= VREAD; + if (working_mode & (ACE_WRITE_DATA|ACE_WRITE_NAMED_ATTRS| + ACE_APPEND_DATA|ACE_WRITE_ATTRIBUTES|ACE_SYNCHRONIZE)) + needed_bits |= VWRITE; + if (working_mode & ACE_EXECUTE) + needed_bits |= VEXEC; + + if ((error = zfs_zaccess_common(check_zp, mode, &working_mode, + &check_privs, skipaclchk, cr)) == 0) { + if (is_attr) + VN_RELE(ZTOV(xzp)); + return (secpolicy_vnode_access2(cr, ZTOV(zp), owner, + needed_bits, needed_bits)); + } + + if (error && !check_privs) { + if (is_attr) + VN_RELE(ZTOV(xzp)); + return (error); + } + + if (error && (flags & V_APPEND)) { + error = zfs_zaccess_append(zp, &working_mode, &check_privs, cr); + } + + if (error && check_privs) { + mode_t checkmode = 0; + vnode_t *check_vp = ZTOV(check_zp); + + /* + * First check for implicit owner permission on + * read_acl/read_attributes + */ + + error = 0; + ASSERT(working_mode != 0); + + if ((working_mode & (ACE_READ_ACL|ACE_READ_ATTRIBUTES) && + owner == crgetuid(cr))) + working_mode &= ~(ACE_READ_ACL|ACE_READ_ATTRIBUTES); + + if (working_mode & (ACE_READ_DATA|ACE_READ_NAMED_ATTRS| + ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_SYNCHRONIZE)) + checkmode |= VREAD; + if (working_mode & (ACE_WRITE_DATA|ACE_WRITE_NAMED_ATTRS| + ACE_APPEND_DATA|ACE_WRITE_ATTRIBUTES|ACE_SYNCHRONIZE)) + checkmode |= VWRITE; + if (working_mode & ACE_EXECUTE) + checkmode |= VEXEC; + + error = secpolicy_vnode_access2(cr, check_vp, owner, + needed_bits & ~checkmode, needed_bits); + + if (error == 0 && (working_mode & ACE_WRITE_OWNER)) + error = secpolicy_vnode_chown(check_vp, cr, owner); + if (error == 0 && (working_mode & ACE_WRITE_ACL)) + error = secpolicy_vnode_setdac(check_vp, cr, owner); + + if (error == 0 && (working_mode & + (ACE_DELETE|ACE_DELETE_CHILD))) + error = secpolicy_vnode_remove(check_vp, cr); + + if (error == 0 && (working_mode & ACE_SYNCHRONIZE)) { + error = secpolicy_vnode_chown(check_vp, cr, owner); + } + if (error == 0) { + /* + * See if any bits other than those already checked + * for are still present. If so then return EACCES + */ + if (working_mode & ~(ZFS_CHECKED_MASKS)) { + error = SET_ERROR(EACCES); + } + } + } else if (error == 0) { + error = secpolicy_vnode_access2(cr, ZTOV(zp), owner, + needed_bits, needed_bits); + } + + + if (is_attr) + VN_RELE(ZTOV(xzp)); + + return (error); +} + +/* + * Translate traditional unix VREAD/VWRITE/VEXEC mode into + * native ACL format and call zfs_zaccess() + */ +int +zfs_zaccess_rwx(znode_t *zp, mode_t mode, int flags, cred_t *cr) +{ + return (zfs_zaccess(zp, zfs_unix_to_v4(mode >> 6), flags, B_FALSE, cr)); +} + +/* + * Access function for secpolicy_vnode_setattr + */ +int +zfs_zaccess_unix(znode_t *zp, mode_t mode, cred_t *cr) +{ + int v4_mode = zfs_unix_to_v4(mode >> 6); + + return (zfs_zaccess(zp, v4_mode, 0, B_FALSE, cr)); +} + +static int +zfs_delete_final_check(znode_t *zp, znode_t *dzp, + mode_t available_perms, cred_t *cr) +{ + int error; + uid_t downer; + + downer = zfs_fuid_map_id(dzp->z_zfsvfs, dzp->z_uid, cr, ZFS_OWNER); + + error = secpolicy_vnode_access2(cr, ZTOV(dzp), + downer, available_perms, VWRITE|VEXEC); + + if (error == 0) + error = zfs_sticky_remove_access(dzp, zp, cr); + + return (error); +} + +/* + * Determine whether Access should be granted/deny, without + * consulting least priv subsystem. + * + * The following chart is the recommended NFSv4 enforcement for + * ability to delete an object. + * + * ------------------------------------------------------- + * | Parent Dir | Target Object Permissions | + * | permissions | | + * ------------------------------------------------------- + * | | ACL Allows | ACL Denies| Delete | + * | | Delete | Delete | unspecified| + * ------------------------------------------------------- + * | ACL Allows | Permit | Permit | Permit | + * | DELETE_CHILD | | + * ------------------------------------------------------- + * | ACL Denies | Permit | Deny | Deny | + * | DELETE_CHILD | | | | + * ------------------------------------------------------- + * | ACL specifies | | | | + * | only allow | Permit | Permit | Permit | + * | write and | | | | + * | execute | | | | + * ------------------------------------------------------- + * | ACL denies | | | | + * | write and | Permit | Deny | Deny | + * | execute | | | | + * ------------------------------------------------------- + * ^ + * | + * No search privilege, can't even look up file? + * + */ +int +zfs_zaccess_delete(znode_t *dzp, znode_t *zp, cred_t *cr) +{ + uint32_t dzp_working_mode = 0; + uint32_t zp_working_mode = 0; + int dzp_error, zp_error; + mode_t available_perms; + boolean_t dzpcheck_privs = B_TRUE; + boolean_t zpcheck_privs = B_TRUE; + + /* + * We want specific DELETE permissions to + * take precedence over WRITE/EXECUTE. We don't + * want an ACL such as this to mess us up. + * user:joe:write_data:deny,user:joe:delete:allow + * + * However, deny permissions may ultimately be overridden + * by secpolicy_vnode_access(). + * + * We will ask for all of the necessary permissions and then + * look at the working modes from the directory and target object + * to determine what was found. + */ + + if (zp->z_pflags & (ZFS_IMMUTABLE | ZFS_NOUNLINK)) + return (SET_ERROR(EPERM)); + + /* + * First row + * If the directory permissions allow the delete, we are done. + */ + if ((dzp_error = zfs_zaccess_common(dzp, ACE_DELETE_CHILD, + &dzp_working_mode, &dzpcheck_privs, B_FALSE, cr)) == 0) + return (0); + + /* + * If target object has delete permission then we are done + */ + if ((zp_error = zfs_zaccess_common(zp, ACE_DELETE, &zp_working_mode, + &zpcheck_privs, B_FALSE, cr)) == 0) + return (0); + + ASSERT(dzp_error && zp_error); + + if (!dzpcheck_privs) + return (dzp_error); + if (!zpcheck_privs) + return (zp_error); + + /* + * Second row + * + * If directory returns EACCES then delete_child was denied + * due to deny delete_child. In this case send the request through + * secpolicy_vnode_remove(). We don't use zfs_delete_final_check() + * since that *could* allow the delete based on write/execute permission + * and we want delete permissions to override write/execute. + */ + + if (dzp_error == EACCES) { + /* XXXPJD: s/dzp/zp/ ? */ + return (secpolicy_vnode_remove(ZTOV(dzp), cr)); + } + /* + * Third Row + * only need to see if we have write/execute on directory. + */ + + dzp_error = zfs_zaccess_common(dzp, ACE_EXECUTE|ACE_WRITE_DATA, + &dzp_working_mode, &dzpcheck_privs, B_FALSE, cr); + + if (dzp_error != 0 && !dzpcheck_privs) + return (dzp_error); + + /* + * Fourth row + */ + + available_perms = (dzp_working_mode & ACE_WRITE_DATA) ? 0 : VWRITE; + available_perms |= (dzp_working_mode & ACE_EXECUTE) ? 0 : VEXEC; + + return (zfs_delete_final_check(zp, dzp, available_perms, cr)); + +} + +int +zfs_zaccess_rename(znode_t *sdzp, znode_t *szp, znode_t *tdzp, + znode_t *tzp, cred_t *cr) +{ + int add_perm; + int error; + + if (szp->z_pflags & ZFS_AV_QUARANTINED) + return (SET_ERROR(EACCES)); + + add_perm = (ZTOV(szp)->v_type == VDIR) ? + ACE_ADD_SUBDIRECTORY : ACE_ADD_FILE; + + /* + * Rename permissions are combination of delete permission + + * add file/subdir permission. + * + * BSD operating systems also require write permission + * on the directory being moved from one parent directory + * to another. + */ + if (ZTOV(szp)->v_type == VDIR && ZTOV(sdzp) != ZTOV(tdzp)) { + if ((error = zfs_zaccess(szp, ACE_WRITE_DATA, 0, B_FALSE, cr))) + return (error); + } + + /* + * first make sure we do the delete portion. + * + * If that succeeds then check for add_file/add_subdir permissions + */ + + if ((error = zfs_zaccess_delete(sdzp, szp, cr))) + return (error); + + /* + * If we have a tzp, see if we can delete it? + */ + if (tzp && (error = zfs_zaccess_delete(tdzp, tzp, cr))) + return (error); + + /* + * Now check for add permissions + */ + error = zfs_zaccess(tdzp, add_perm, 0, B_FALSE, cr); + + return (error); +} diff --git a/module/os/freebsd/zfs/zfs_ctldir.c b/module/os/freebsd/zfs/zfs_ctldir.c new file mode 100644 index 00000000000..ed6652c3bab --- /dev/null +++ b/module/os/freebsd/zfs/zfs_ctldir.c @@ -0,0 +1,1345 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. + * Copyright 2015, OmniTI Computer Consulting, Inc. All rights reserved. + */ + +/* + * ZFS control directory (a.k.a. ".zfs") + * + * This directory provides a common location for all ZFS meta-objects. + * Currently, this is only the 'snapshot' directory, but this may expand in the + * future. The elements are built using the GFS primitives, as the hierarchy + * does not actually exist on disk. + * + * For 'snapshot', we don't want to have all snapshots always mounted, because + * this would take up a huge amount of space in /etc/mnttab. We have three + * types of objects: + * + * ctldir ------> snapshotdir -------> snapshot + * | + * | + * V + * mounted fs + * + * The 'snapshot' node contains just enough information to lookup '..' and act + * as a mountpoint for the snapshot. Whenever we lookup a specific snapshot, we + * perform an automount of the underlying filesystem and return the + * corresponding vnode. + * + * All mounts are handled automatically by the kernel, but unmounts are + * (currently) handled from user land. The main reason is that there is no + * reliable way to auto-unmount the filesystem when it's "no longer in use". + * When the user unmounts a filesystem, we call zfsctl_unmount(), which + * unmounts any snapshots within the snapshot directory. + * + * The '.zfs', '.zfs/snapshot', and all directories created under + * '.zfs/snapshot' (ie: '.zfs/snapshot/') are all GFS nodes and + * share the same vfs_t as the head filesystem (what '.zfs' lives under). + * + * File systems mounted ontop of the GFS nodes '.zfs/snapshot/' + * (ie: snapshots) are ZFS nodes and have their own unique vfs_t. + * However, vnodes within these mounted on file systems have their v_vfsp + * fields set to the head filesystem to make NFS happy (see + * zfsctl_snapdir_lookup()). We VFS_HOLD the head filesystem's vfs_t + * so that it cannot be freed until all snapshots have been unmounted. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zfs_namecheck.h" + +#include + +/* Common access mode for all virtual directories under the ctldir */ +const uint16_t zfsctl_ctldir_mode = S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | + S_IROTH | S_IXOTH; + +/* + * "Synthetic" filesystem implementation. + */ + +/* + * Assert that A implies B. + */ +#define KASSERT_IMPLY(A, B, msg) KASSERT(!(A) || (B), (msg)); + +static MALLOC_DEFINE(M_SFSNODES, "sfs_nodes", "synthetic-fs nodes"); + +typedef struct sfs_node { + char sn_name[ZFS_MAX_DATASET_NAME_LEN]; + uint64_t sn_parent_id; + uint64_t sn_id; +} sfs_node_t; + +/* + * Check the parent's ID as well as the node's to account for a chance + * that IDs originating from different domains (snapshot IDs, artifical + * IDs, znode IDs) may clash. + */ +static int +sfs_compare_ids(struct vnode *vp, void *arg) +{ + sfs_node_t *n1 = vp->v_data; + sfs_node_t *n2 = arg; + bool equal; + + equal = n1->sn_id == n2->sn_id && + n1->sn_parent_id == n2->sn_parent_id; + + /* Zero means equality. */ + return (!equal); +} + +static int +sfs_vnode_get(const struct mount *mp, int flags, uint64_t parent_id, + uint64_t id, struct vnode **vpp) +{ + sfs_node_t search; + int err; + + search.sn_id = id; + search.sn_parent_id = parent_id; + err = vfs_hash_get(mp, (uint32_t)id, flags, curthread, vpp, + sfs_compare_ids, &search); + return (err); +} + +static int +sfs_vnode_insert(struct vnode *vp, int flags, uint64_t parent_id, + uint64_t id, struct vnode **vpp) +{ + int err; + + KASSERT(vp->v_data != NULL, ("sfs_vnode_insert with NULL v_data")); + err = vfs_hash_insert(vp, (uint32_t)id, flags, curthread, vpp, + sfs_compare_ids, vp->v_data); + return (err); +} + +static void +sfs_vnode_remove(struct vnode *vp) +{ + vfs_hash_remove(vp); +} + +typedef void sfs_vnode_setup_fn(vnode_t *vp, void *arg); + +static int +sfs_vgetx(struct mount *mp, int flags, uint64_t parent_id, uint64_t id, + const char *tag, struct vop_vector *vops, + sfs_vnode_setup_fn setup, void *arg, + struct vnode **vpp) +{ + struct vnode *vp; + int error; + + error = sfs_vnode_get(mp, flags, parent_id, id, vpp); + if (error != 0 || *vpp != NULL) { + KASSERT_IMPLY(error == 0, (*vpp)->v_data != NULL, + "sfs vnode with no data"); + return (error); + } + + /* Allocate a new vnode/inode. */ + error = getnewvnode(tag, mp, vops, &vp); + if (error != 0) { + *vpp = NULL; + return (error); + } + + /* + * Exclusively lock the vnode vnode while it's being constructed. + */ + lockmgr(vp->v_vnlock, LK_EXCLUSIVE, NULL); + error = insmntque(vp, mp); + if (error != 0) { + *vpp = NULL; + return (error); + } + + setup(vp, arg); + + error = sfs_vnode_insert(vp, flags, parent_id, id, vpp); + if (error != 0 || *vpp != NULL) { + KASSERT_IMPLY(error == 0, (*vpp)->v_data != NULL, + "sfs vnode with no data"); + return (error); + } + + *vpp = vp; + return (0); +} + +static void +sfs_print_node(sfs_node_t *node) +{ + printf("\tname = %s\n", node->sn_name); + printf("\tparent_id = %ju\n", (uintmax_t)node->sn_parent_id); + printf("\tid = %ju\n", (uintmax_t)node->sn_id); +} + +static sfs_node_t * +sfs_alloc_node(size_t size, const char *name, uint64_t parent_id, uint64_t id) +{ + struct sfs_node *node; + + KASSERT(strlen(name) < sizeof (node->sn_name), + ("sfs node name is too long")); + KASSERT(size >= sizeof (*node), ("sfs node size is too small")); + node = malloc(size, M_SFSNODES, M_WAITOK | M_ZERO); + strlcpy(node->sn_name, name, sizeof (node->sn_name)); + node->sn_parent_id = parent_id; + node->sn_id = id; + + return (node); +} + +static void +sfs_destroy_node(sfs_node_t *node) +{ + free(node, M_SFSNODES); +} + +static void * +sfs_reclaim_vnode(vnode_t *vp) +{ + void *data; + + sfs_vnode_remove(vp); + data = vp->v_data; + vp->v_data = NULL; + return (data); +} + +static int +sfs_readdir_common(uint64_t parent_id, uint64_t id, struct vop_readdir_args *ap, + uio_t *uio, off_t *offp) +{ + struct dirent entry; + int error; + + /* Reset ncookies for subsequent use of vfs_read_dirent. */ + if (ap->a_ncookies != NULL) + *ap->a_ncookies = 0; + + if (uio->uio_resid < sizeof (entry)) + return (SET_ERROR(EINVAL)); + + if (uio->uio_offset < 0) + return (SET_ERROR(EINVAL)); + if (uio->uio_offset == 0) { + entry.d_fileno = id; + entry.d_type = DT_DIR; + entry.d_name[0] = '.'; + entry.d_name[1] = '\0'; + entry.d_namlen = 1; + entry.d_reclen = sizeof (entry); + error = vfs_read_dirent(ap, &entry, uio->uio_offset); + if (error != 0) + return (SET_ERROR(error)); + } + + if (uio->uio_offset < sizeof (entry)) + return (SET_ERROR(EINVAL)); + if (uio->uio_offset == sizeof (entry)) { + entry.d_fileno = parent_id; + entry.d_type = DT_DIR; + entry.d_name[0] = '.'; + entry.d_name[1] = '.'; + entry.d_name[2] = '\0'; + entry.d_namlen = 2; + entry.d_reclen = sizeof (entry); + error = vfs_read_dirent(ap, &entry, uio->uio_offset); + if (error != 0) + return (SET_ERROR(error)); + } + + if (offp != NULL) + *offp = 2 * sizeof (entry); + return (0); +} + + +/* + * .zfs inode namespace + * + * We need to generate unique inode numbers for all files and directories + * within the .zfs pseudo-filesystem. We use the following scheme: + * + * ENTRY ZFSCTL_INODE + * .zfs 1 + * .zfs/snapshot 2 + * .zfs/snapshot/ objectid(snap) + */ +#define ZFSCTL_INO_SNAP(id) (id) + +static struct vop_vector zfsctl_ops_root; +static struct vop_vector zfsctl_ops_snapdir; +static struct vop_vector zfsctl_ops_snapshot; +static struct vop_vector zfsctl_ops_shares_dir; + +void +zfsctl_init(void) +{ +} + +void +zfsctl_fini(void) +{ +} + +boolean_t +zfsctl_is_node(vnode_t *vp) +{ + return (vn_matchops(vp, zfsctl_ops_root) || + vn_matchops(vp, zfsctl_ops_snapdir) || + vn_matchops(vp, zfsctl_ops_snapshot) || + vn_matchops(vp, zfsctl_ops_shares_dir)); + +} + +typedef struct zfsctl_root { + sfs_node_t node; + sfs_node_t *snapdir; + timestruc_t cmtime; +} zfsctl_root_t; + + +/* + * Create the '.zfs' directory. + */ +void +zfsctl_create(zfsvfs_t *zfsvfs) +{ + zfsctl_root_t *dot_zfs; + sfs_node_t *snapdir; + vnode_t *rvp; + uint64_t crtime[2]; + + ASSERT(zfsvfs->z_ctldir == NULL); + + snapdir = sfs_alloc_node(sizeof (*snapdir), "snapshot", ZFSCTL_INO_ROOT, + ZFSCTL_INO_SNAPDIR); + dot_zfs = (zfsctl_root_t *)sfs_alloc_node(sizeof (*dot_zfs), ".zfs", 0, + ZFSCTL_INO_ROOT); + dot_zfs->snapdir = snapdir; + + VERIFY(VFS_ROOT(zfsvfs->z_vfs, LK_EXCLUSIVE, &rvp) == 0); + VERIFY(0 == sa_lookup(VTOZ(rvp)->z_sa_hdl, SA_ZPL_CRTIME(zfsvfs), + &crtime, sizeof (crtime))); + ZFS_TIME_DECODE(&dot_zfs->cmtime, crtime); + vput(rvp); + + zfsvfs->z_ctldir = dot_zfs; +} + +/* + * Destroy the '.zfs' directory. Only called when the filesystem is unmounted. + * The nodes must not have any associated vnodes by now as they should be + * vflush-ed. + */ +void +zfsctl_destroy(zfsvfs_t *zfsvfs) +{ + sfs_destroy_node(zfsvfs->z_ctldir->snapdir); + sfs_destroy_node((sfs_node_t *)zfsvfs->z_ctldir); + zfsvfs->z_ctldir = NULL; +} + +static int +zfsctl_fs_root_vnode(struct mount *mp, void *arg __unused, int flags, + struct vnode **vpp) +{ + return (VFS_ROOT(mp, flags, vpp)); +} + +static void +zfsctl_common_vnode_setup(vnode_t *vp, void *arg) +{ + ASSERT_VOP_ELOCKED(vp, __func__); + + /* We support shared locking. */ + VN_LOCK_ASHARE(vp); + vp->v_type = VDIR; + vp->v_data = arg; +} + +static int +zfsctl_root_vnode(struct mount *mp, void *arg __unused, int flags, + struct vnode **vpp) +{ + void *node; + int err; + + node = ((zfsvfs_t *)mp->mnt_data)->z_ctldir; + err = sfs_vgetx(mp, flags, 0, ZFSCTL_INO_ROOT, "zfs", &zfsctl_ops_root, + zfsctl_common_vnode_setup, node, vpp); + return (err); +} + +static int +zfsctl_snapdir_vnode(struct mount *mp, void *arg __unused, int flags, + struct vnode **vpp) +{ + void *node; + int err; + + node = ((zfsvfs_t *)mp->mnt_data)->z_ctldir->snapdir; + err = sfs_vgetx(mp, flags, ZFSCTL_INO_ROOT, ZFSCTL_INO_SNAPDIR, "zfs", + &zfsctl_ops_snapdir, zfsctl_common_vnode_setup, node, vpp); + return (err); +} + +/* + * Given a root znode, retrieve the associated .zfs directory. + * Add a hold to the vnode and return it. + */ +int +zfsctl_root(zfsvfs_t *zfsvfs, int flags, vnode_t **vpp) +{ + int error; + + error = zfsctl_root_vnode(zfsvfs->z_vfs, NULL, flags, vpp); + return (error); +} + +/* + * Common open routine. Disallow any write access. + */ +static int +zfsctl_common_open(struct vop_open_args *ap) +{ + int flags = ap->a_mode; + + if (flags & FWRITE) + return (SET_ERROR(EACCES)); + + return (0); +} + +/* + * Common close routine. Nothing to do here. + */ +/* ARGSUSED */ +static int +zfsctl_common_close(struct vop_close_args *ap) +{ + return (0); +} + +/* + * Common access routine. Disallow writes. + */ +static int +zfsctl_common_access(struct vop_access_args *ap) +{ + accmode_t accmode = ap->a_accmode; + + if (accmode & VWRITE) + return (SET_ERROR(EACCES)); + return (0); +} + +/* + * Common getattr function. Fill in basic information. + */ +static void +zfsctl_common_getattr(vnode_t *vp, vattr_t *vap) +{ + timestruc_t now; + sfs_node_t *node; + + node = vp->v_data; + + vap->va_uid = 0; + vap->va_gid = 0; + vap->va_rdev = 0; + /* + * We are a purely virtual object, so we have no + * blocksize or allocated blocks. + */ + vap->va_blksize = 0; + vap->va_nblocks = 0; + vap->va_seq = 0; + vn_fsid(vp, vap); + vap->va_mode = zfsctl_ctldir_mode; + vap->va_type = VDIR; + /* + * We live in the now (for atime). + */ + gethrestime(&now); + vap->va_atime = now; + /* FreeBSD: Reset chflags(2) flags. */ + vap->va_flags = 0; + + vap->va_nodeid = node->sn_id; + + /* At least '.' and '..'. */ + vap->va_nlink = 2; +} + +#ifndef _OPENSOLARIS_SYS_VNODE_H_ +struct vop_fid_args { + struct vnode *a_vp; + struct fid *a_fid; +}; +#endif + +static int +zfsctl_common_fid(struct vop_fid_args *ap) +{ + vnode_t *vp = ap->a_vp; + fid_t *fidp = (void *)ap->a_fid; + sfs_node_t *node = vp->v_data; + uint64_t object = node->sn_id; + zfid_short_t *zfid; + int i; + + zfid = (zfid_short_t *)fidp; + zfid->zf_len = SHORT_FID_LEN; + + for (i = 0; i < sizeof (zfid->zf_object); i++) + zfid->zf_object[i] = (uint8_t)(object >> (8 * i)); + + /* .zfs nodes always have a generation number of 0 */ + for (i = 0; i < sizeof (zfid->zf_gen); i++) + zfid->zf_gen[i] = 0; + + return (0); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_reclaim_args { + struct vnode *a_vp; + struct thread *a_td; +}; +#endif + +static int +zfsctl_common_reclaim(struct vop_reclaim_args *ap) +{ + vnode_t *vp = ap->a_vp; + + (void) sfs_reclaim_vnode(vp); + return (0); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_print_args { + struct vnode *a_vp; +}; +#endif + +static int +zfsctl_common_print(struct vop_print_args *ap) +{ + sfs_print_node(ap->a_vp->v_data); + return (0); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_getattr_args { + struct vnode *a_vp; + struct vattr *a_vap; + struct ucred *a_cred; +}; +#endif + +/* + * Get root directory attributes. + */ +static int +zfsctl_root_getattr(struct vop_getattr_args *ap) +{ + struct vnode *vp = ap->a_vp; + struct vattr *vap = ap->a_vap; + zfsctl_root_t *node = vp->v_data; + + zfsctl_common_getattr(vp, vap); + vap->va_ctime = node->cmtime; + vap->va_mtime = vap->va_ctime; + vap->va_birthtime = vap->va_ctime; + vap->va_nlink += 1; /* snapdir */ + vap->va_size = vap->va_nlink; + return (0); +} + +/* + * When we lookup "." we still can be asked to lock it + * differently, can't we? + */ +int +zfsctl_relock_dot(vnode_t *dvp, int ltype) +{ + vref(dvp); + if (ltype != VOP_ISLOCKED(dvp)) { + if (ltype == LK_EXCLUSIVE) + vn_lock(dvp, LK_UPGRADE | LK_RETRY); + else /* if (ltype == LK_SHARED) */ + vn_lock(dvp, LK_DOWNGRADE | LK_RETRY); + + /* Relock for the "." case may left us with reclaimed vnode. */ + if (VN_IS_DOOMED(dvp)) { + vrele(dvp); + return (SET_ERROR(ENOENT)); + } + } + return (0); +} + +/* + * Special case the handling of "..". + */ +int +zfsctl_root_lookup(struct vop_lookup_args *ap) +{ + struct componentname *cnp = ap->a_cnp; + vnode_t *dvp = ap->a_dvp; + vnode_t **vpp = ap->a_vpp; + int flags = ap->a_cnp->cn_flags; + int lkflags = ap->a_cnp->cn_lkflags; + int nameiop = ap->a_cnp->cn_nameiop; + int err; + + ASSERT(dvp->v_type == VDIR); + + if ((flags & ISLASTCN) != 0 && nameiop != LOOKUP) + return (SET_ERROR(ENOTSUP)); + + if (cnp->cn_namelen == 1 && *cnp->cn_nameptr == '.') { + err = zfsctl_relock_dot(dvp, lkflags & LK_TYPE_MASK); + if (err == 0) + *vpp = dvp; + } else if ((flags & ISDOTDOT) != 0) { + err = vn_vget_ino_gen(dvp, zfsctl_fs_root_vnode, NULL, + lkflags, vpp); + } else if (strncmp(cnp->cn_nameptr, "snapshot", cnp->cn_namelen) == 0) { + err = zfsctl_snapdir_vnode(dvp->v_mount, NULL, lkflags, vpp); + } else { + err = SET_ERROR(ENOENT); + } + if (err != 0) + *vpp = NULL; + return (err); +} + +static int +zfsctl_root_readdir(struct vop_readdir_args *ap) +{ + struct dirent entry; + vnode_t *vp = ap->a_vp; + zfsvfs_t *zfsvfs = vp->v_vfsp->vfs_data; + zfsctl_root_t *node = vp->v_data; + uio_t *uio = ap->a_uio; + int *eofp = ap->a_eofflag; + off_t dots_offset; + int error; + + ASSERT(vp->v_type == VDIR); + + error = sfs_readdir_common(zfsvfs->z_root, ZFSCTL_INO_ROOT, ap, uio, + &dots_offset); + if (error != 0) { + if (error == ENAMETOOLONG) /* ran out of destination space */ + error = 0; + return (error); + } + if (uio->uio_offset != dots_offset) + return (SET_ERROR(EINVAL)); + + CTASSERT(sizeof (node->snapdir->sn_name) <= sizeof (entry.d_name)); + entry.d_fileno = node->snapdir->sn_id; + entry.d_type = DT_DIR; + strcpy(entry.d_name, node->snapdir->sn_name); + entry.d_namlen = strlen(entry.d_name); + entry.d_reclen = sizeof (entry); + error = vfs_read_dirent(ap, &entry, uio->uio_offset); + if (error != 0) { + if (error == ENAMETOOLONG) + error = 0; + return (SET_ERROR(error)); + } + if (eofp != NULL) + *eofp = 1; + return (0); +} + +static int +zfsctl_root_vptocnp(struct vop_vptocnp_args *ap) +{ + static const char dotzfs_name[4] = ".zfs"; + vnode_t *dvp; + int error; + + if (*ap->a_buflen < sizeof (dotzfs_name)) + return (SET_ERROR(ENOMEM)); + + error = vn_vget_ino_gen(ap->a_vp, zfsctl_fs_root_vnode, NULL, + LK_SHARED, &dvp); + if (error != 0) + return (SET_ERROR(error)); + + VOP_UNLOCK1(dvp); + *ap->a_vpp = dvp; + *ap->a_buflen -= sizeof (dotzfs_name); + bcopy(dotzfs_name, ap->a_buf + *ap->a_buflen, sizeof (dotzfs_name)); + return (0); +} + +static int +zfsctl_common_pathconf(struct vop_pathconf_args *ap) +{ + /* + * We care about ACL variables so that user land utilities like ls + * can display them correctly. Since the ctldir's st_dev is set to be + * the same as the parent dataset, we must support all variables that + * it supports. + */ + switch (ap->a_name) { + case _PC_LINK_MAX: + *ap->a_retval = MIN(LONG_MAX, ZFS_LINK_MAX); + return (0); + + case _PC_FILESIZEBITS: + *ap->a_retval = 64; + return (0); + + case _PC_MIN_HOLE_SIZE: + *ap->a_retval = (int)SPA_MINBLOCKSIZE; + return (0); + + case _PC_ACL_EXTENDED: + *ap->a_retval = 0; + return (0); + + case _PC_ACL_NFS4: + *ap->a_retval = 1; + return (0); + + case _PC_ACL_PATH_MAX: + *ap->a_retval = ACL_MAX_ENTRIES; + return (0); + + case _PC_NAME_MAX: + *ap->a_retval = NAME_MAX; + return (0); + + default: + return (vop_stdpathconf(ap)); + } +} + +/* + * Returns a trivial ACL + */ +int +zfsctl_common_getacl(struct vop_getacl_args *ap) +{ + int i; + + if (ap->a_type != ACL_TYPE_NFS4) + return (EINVAL); + + acl_nfs4_sync_acl_from_mode(ap->a_aclp, zfsctl_ctldir_mode, 0); + /* + * acl_nfs4_sync_acl_from_mode assumes that the owner can always modify + * attributes. That is not the case for the ctldir, so we must clear + * those bits. We also must clear ACL_READ_NAMED_ATTRS, because xattrs + * aren't supported by the ctldir. + */ + for (i = 0; i < ap->a_aclp->acl_cnt; i++) { + struct acl_entry *entry; + entry = &(ap->a_aclp->acl_entry[i]); + entry->ae_perm &= ~(ACL_WRITE_ACL | ACL_WRITE_OWNER | + ACL_WRITE_ATTRIBUTES | ACL_WRITE_NAMED_ATTRS | + ACL_READ_NAMED_ATTRS); + } + + return (0); +} + +static struct vop_vector zfsctl_ops_root = { + .vop_default = &default_vnodeops, + .vop_open = zfsctl_common_open, + .vop_close = zfsctl_common_close, + .vop_ioctl = VOP_EINVAL, + .vop_getattr = zfsctl_root_getattr, + .vop_access = zfsctl_common_access, + .vop_readdir = zfsctl_root_readdir, + .vop_lookup = zfsctl_root_lookup, + .vop_inactive = VOP_NULL, + .vop_reclaim = zfsctl_common_reclaim, + .vop_fid = zfsctl_common_fid, + .vop_print = zfsctl_common_print, + .vop_vptocnp = zfsctl_root_vptocnp, + .vop_pathconf = zfsctl_common_pathconf, + .vop_getacl = zfsctl_common_getacl, +}; +VFS_VOP_VECTOR_REGISTER(zfsctl_ops_root); + +static int +zfsctl_snapshot_zname(vnode_t *vp, const char *name, int len, char *zname) +{ + objset_t *os = ((zfsvfs_t *)((vp)->v_vfsp->vfs_data))->z_os; + + dmu_objset_name(os, zname); + if (strlen(zname) + 1 + strlen(name) >= len) + return (SET_ERROR(ENAMETOOLONG)); + (void) strcat(zname, "@"); + (void) strcat(zname, name); + return (0); +} + +static int +zfsctl_snapshot_lookup(vnode_t *vp, const char *name, uint64_t *id) +{ + objset_t *os = ((zfsvfs_t *)((vp)->v_vfsp->vfs_data))->z_os; + int err; + + err = dsl_dataset_snap_lookup(dmu_objset_ds(os), name, id); + return (err); +} + +/* + * Given a vnode get a root vnode of a filesystem mounted on top of + * the vnode, if any. The root vnode is referenced and locked. + * If no filesystem is mounted then the orinal vnode remains referenced + * and locked. If any error happens the orinal vnode is unlocked and + * released. + */ +static int +zfsctl_mounted_here(vnode_t **vpp, int flags) +{ + struct mount *mp; + int err; + + ASSERT_VOP_LOCKED(*vpp, __func__); + ASSERT3S((*vpp)->v_type, ==, VDIR); + + if ((mp = (*vpp)->v_mountedhere) != NULL) { + err = vfs_busy(mp, 0); + KASSERT(err == 0, ("vfs_busy(mp, 0) failed with %d", err)); + KASSERT(vrefcnt(*vpp) > 1, ("unreferenced mountpoint")); + vput(*vpp); + err = VFS_ROOT(mp, flags, vpp); + vfs_unbusy(mp); + return (err); + } + return (EJUSTRETURN); +} + +typedef struct { + const char *snap_name; + uint64_t snap_id; +} snapshot_setup_arg_t; + +static void +zfsctl_snapshot_vnode_setup(vnode_t *vp, void *arg) +{ + snapshot_setup_arg_t *ssa = arg; + sfs_node_t *node; + + ASSERT_VOP_ELOCKED(vp, __func__); + + node = sfs_alloc_node(sizeof (sfs_node_t), + ssa->snap_name, ZFSCTL_INO_SNAPDIR, ssa->snap_id); + zfsctl_common_vnode_setup(vp, node); + + /* We have to support recursive locking. */ + VN_LOCK_AREC(vp); +} + +/* + * Lookup entry point for the 'snapshot' directory. Try to open the + * snapshot if it exist, creating the pseudo filesystem vnode as necessary. + * Perform a mount of the associated dataset on top of the vnode. + * There are four possibilities: + * - the snapshot node and vnode do not exist + * - the snapshot vnode is covered by the mounted snapshot + * - the snapshot vnode is not covered yet, the mount operation is in progress + * - the snapshot vnode is not covered, because the snapshot has been unmounted + * The last two states are transient and should be relatively short-lived. + */ +int +zfsctl_snapdir_lookup(struct vop_lookup_args *ap) +{ + vnode_t *dvp = ap->a_dvp; + vnode_t **vpp = ap->a_vpp; + struct componentname *cnp = ap->a_cnp; + char name[NAME_MAX + 1]; + char fullname[ZFS_MAX_DATASET_NAME_LEN]; + char *mountpoint; + size_t mountpoint_len; + zfsvfs_t *zfsvfs = dvp->v_vfsp->vfs_data; + uint64_t snap_id; + int nameiop = cnp->cn_nameiop; + int lkflags = cnp->cn_lkflags; + int flags = cnp->cn_flags; + int err; + + ASSERT(dvp->v_type == VDIR); + + if ((flags & ISLASTCN) != 0 && nameiop != LOOKUP) + return (SET_ERROR(ENOTSUP)); + + if (cnp->cn_namelen == 1 && *cnp->cn_nameptr == '.') { + err = zfsctl_relock_dot(dvp, lkflags & LK_TYPE_MASK); + if (err == 0) + *vpp = dvp; + return (err); + } + if (flags & ISDOTDOT) { + err = vn_vget_ino_gen(dvp, zfsctl_root_vnode, NULL, lkflags, + vpp); + return (err); + } + + if (cnp->cn_namelen >= sizeof (name)) + return (SET_ERROR(ENAMETOOLONG)); + + strlcpy(name, ap->a_cnp->cn_nameptr, ap->a_cnp->cn_namelen + 1); + err = zfsctl_snapshot_lookup(dvp, name, &snap_id); + if (err != 0) + return (SET_ERROR(ENOENT)); + + for (;;) { + snapshot_setup_arg_t ssa; + + ssa.snap_name = name; + ssa.snap_id = snap_id; + err = sfs_vgetx(dvp->v_mount, LK_SHARED, ZFSCTL_INO_SNAPDIR, + snap_id, "zfs", &zfsctl_ops_snapshot, + zfsctl_snapshot_vnode_setup, &ssa, vpp); + if (err != 0) + return (err); + + /* Check if a new vnode has just been created. */ + if (VOP_ISLOCKED(*vpp) == LK_EXCLUSIVE) + break; + + /* + * Check if a snapshot is already mounted on top of the vnode. + */ + err = zfsctl_mounted_here(vpp, lkflags); + if (err != EJUSTRETURN) + return (err); + + /* + * If the vnode is not covered, then either the mount operation + * is in progress or the snapshot has already been unmounted + * but the vnode hasn't been inactivated and reclaimed yet. + * We can try to re-use the vnode in the latter case. + */ + VI_LOCK(*vpp); + if (((*vpp)->v_iflag & VI_MOUNT) == 0) { + /* + * Upgrade to exclusive lock in order to: + * - avoid race conditions + * - satisfy the contract of mount_snapshot() + */ + err = VOP_LOCK(*vpp, LK_TRYUPGRADE | LK_INTERLOCK); + if (err == 0) + break; + } else { + VI_UNLOCK(*vpp); + } + + /* + * In this state we can loop on uncontested locks and starve + * the thread doing the lengthy, non-trivial mount operation. + * So, yield to prevent that from happening. + */ + vput(*vpp); + kern_yield(PRI_USER); + } + + VERIFY0(zfsctl_snapshot_zname(dvp, name, sizeof (fullname), fullname)); + + mountpoint_len = strlen(dvp->v_vfsp->mnt_stat.f_mntonname) + + strlen("/" ZFS_CTLDIR_NAME "/snapshot/") + strlen(name) + 1; + mountpoint = kmem_alloc(mountpoint_len, KM_SLEEP); + (void) snprintf(mountpoint, mountpoint_len, + "%s/" ZFS_CTLDIR_NAME "/snapshot/%s", + dvp->v_vfsp->mnt_stat.f_mntonname, name); + + err = mount_snapshot(curthread, vpp, "zfs", mountpoint, fullname, 0); + kmem_free(mountpoint, mountpoint_len); + if (err == 0) { + /* + * Fix up the root vnode mounted on .zfs/snapshot/. + * + * This is where we lie about our v_vfsp in order to + * make .zfs/snapshot/ accessible over NFS + * without requiring manual mounts of . + */ + ASSERT(VTOZ(*vpp)->z_zfsvfs != zfsvfs); + VTOZ(*vpp)->z_zfsvfs->z_parent = zfsvfs; + + /* Clear the root flag (set via VFS_ROOT) as well. */ + (*vpp)->v_vflag &= ~VV_ROOT; + } + + if (err != 0) + *vpp = NULL; + return (err); +} + +static int +zfsctl_snapdir_readdir(struct vop_readdir_args *ap) +{ + char snapname[ZFS_MAX_DATASET_NAME_LEN]; + struct dirent entry; + vnode_t *vp = ap->a_vp; + zfsvfs_t *zfsvfs = vp->v_vfsp->vfs_data; + uio_t *uio = ap->a_uio; + int *eofp = ap->a_eofflag; + off_t dots_offset; + int error; + + ASSERT(vp->v_type == VDIR); + + error = sfs_readdir_common(ZFSCTL_INO_ROOT, ZFSCTL_INO_SNAPDIR, ap, uio, + &dots_offset); + if (error != 0) { + if (error == ENAMETOOLONG) /* ran out of destination space */ + error = 0; + return (error); + } + + ZFS_ENTER(zfsvfs); + for (;;) { + uint64_t cookie; + uint64_t id; + + cookie = uio->uio_offset - dots_offset; + + dsl_pool_config_enter(dmu_objset_pool(zfsvfs->z_os), FTAG); + error = dmu_snapshot_list_next(zfsvfs->z_os, sizeof (snapname), + snapname, &id, &cookie, NULL); + dsl_pool_config_exit(dmu_objset_pool(zfsvfs->z_os), FTAG); + if (error != 0) { + if (error == ENOENT) { + if (eofp != NULL) + *eofp = 1; + error = 0; + } + ZFS_EXIT(zfsvfs); + return (error); + } + + entry.d_fileno = id; + entry.d_type = DT_DIR; + strcpy(entry.d_name, snapname); + entry.d_namlen = strlen(entry.d_name); + entry.d_reclen = sizeof (entry); + error = vfs_read_dirent(ap, &entry, uio->uio_offset); + if (error != 0) { + if (error == ENAMETOOLONG) + error = 0; + ZFS_EXIT(zfsvfs); + return (SET_ERROR(error)); + } + uio->uio_offset = cookie + dots_offset; + } + /* NOTREACHED */ +} + +static int +zfsctl_snapdir_getattr(struct vop_getattr_args *ap) +{ + vnode_t *vp = ap->a_vp; + vattr_t *vap = ap->a_vap; + zfsvfs_t *zfsvfs = vp->v_vfsp->vfs_data; + dsl_dataset_t *ds = dmu_objset_ds(zfsvfs->z_os); + uint64_t snap_count; + int err; + + ZFS_ENTER(zfsvfs); + zfsctl_common_getattr(vp, vap); + vap->va_ctime = dmu_objset_snap_cmtime(zfsvfs->z_os); + vap->va_mtime = vap->va_ctime; + vap->va_birthtime = vap->va_ctime; + if (dsl_dataset_phys(ds)->ds_snapnames_zapobj != 0) { + err = zap_count(dmu_objset_pool(ds->ds_objset)->dp_meta_objset, + dsl_dataset_phys(ds)->ds_snapnames_zapobj, &snap_count); + if (err != 0) { + ZFS_EXIT(zfsvfs); + return (err); + } + vap->va_nlink += snap_count; + } + vap->va_size = vap->va_nlink; + + ZFS_EXIT(zfsvfs); + return (0); +} + +static struct vop_vector zfsctl_ops_snapdir = { + .vop_default = &default_vnodeops, + .vop_open = zfsctl_common_open, + .vop_close = zfsctl_common_close, + .vop_getattr = zfsctl_snapdir_getattr, + .vop_access = zfsctl_common_access, + .vop_readdir = zfsctl_snapdir_readdir, + .vop_lookup = zfsctl_snapdir_lookup, + .vop_reclaim = zfsctl_common_reclaim, + .vop_fid = zfsctl_common_fid, + .vop_print = zfsctl_common_print, + .vop_pathconf = zfsctl_common_pathconf, + .vop_getacl = zfsctl_common_getacl, +}; +VFS_VOP_VECTOR_REGISTER(zfsctl_ops_snapdir); + + +static int +zfsctl_snapshot_inactive(struct vop_inactive_args *ap) +{ + vnode_t *vp = ap->a_vp; + + VERIFY(vrecycle(vp) == 1); + return (0); +} + +static int +zfsctl_snapshot_reclaim(struct vop_reclaim_args *ap) +{ + vnode_t *vp = ap->a_vp; + void *data = vp->v_data; + + sfs_reclaim_vnode(vp); + sfs_destroy_node(data); + return (0); +} + +static int +zfsctl_snapshot_vptocnp(struct vop_vptocnp_args *ap) +{ + struct mount *mp; + vnode_t *dvp; + vnode_t *vp; + sfs_node_t *node; + size_t len; + int locked; + int error; + + vp = ap->a_vp; + node = vp->v_data; + len = strlen(node->sn_name); + if (*ap->a_buflen < len) + return (SET_ERROR(ENOMEM)); + + /* + * Prevent unmounting of the snapshot while the vnode lock + * is not held. That is not strictly required, but allows + * us to assert that an uncovered snapshot vnode is never + * "leaked". + */ + mp = vp->v_mountedhere; + if (mp == NULL) + return (SET_ERROR(ENOENT)); + error = vfs_busy(mp, 0); + KASSERT(error == 0, ("vfs_busy(mp, 0) failed with %d", error)); + + /* + * We can vput the vnode as we can now depend on the reference owned + * by the busied mp. But we also need to hold the vnode, because + * the reference may go after vfs_unbusy() which has to be called + * before we can lock the vnode again. + */ + locked = VOP_ISLOCKED(vp); +#if __FreeBSD_version >= 1300045 + enum vgetstate vs = vget_prep(vp); +#else + vhold(vp); +#endif + vput(vp); + + /* Look up .zfs/snapshot, our parent. */ + error = zfsctl_snapdir_vnode(vp->v_mount, NULL, LK_SHARED, &dvp); + if (error == 0) { + VOP_UNLOCK1(dvp); + *ap->a_vpp = dvp; + *ap->a_buflen -= len; + bcopy(node->sn_name, ap->a_buf + *ap->a_buflen, len); + } + vfs_unbusy(mp); +#if __FreeBSD_version >= 1300045 + vget_finish(vp, locked | LK_RETRY, vs); +#else + vget(vp, locked | LK_VNHELD | LK_RETRY, curthread); +#endif + return (error); +} + +/* + * These VP's should never see the light of day. They should always + * be covered. + */ +static struct vop_vector zfsctl_ops_snapshot = { + .vop_default = NULL, /* ensure very restricted access */ + .vop_inactive = zfsctl_snapshot_inactive, +#if __FreeBSD_version >= 1300045 + .vop_need_inactive = vop_stdneed_inactive, +#endif + .vop_reclaim = zfsctl_snapshot_reclaim, + .vop_vptocnp = zfsctl_snapshot_vptocnp, + .vop_lock1 = vop_stdlock, + .vop_unlock = vop_stdunlock, + .vop_islocked = vop_stdislocked, + .vop_advlockpurge = vop_stdadvlockpurge, /* called by vgone */ + .vop_print = zfsctl_common_print, +}; +VFS_VOP_VECTOR_REGISTER(zfsctl_ops_snapshot); + +int +zfsctl_lookup_objset(vfs_t *vfsp, uint64_t objsetid, zfsvfs_t **zfsvfsp) +{ + zfsvfs_t *zfsvfs __unused = vfsp->vfs_data; + vnode_t *vp; + int error; + + ASSERT(zfsvfs->z_ctldir != NULL); + *zfsvfsp = NULL; + error = sfs_vnode_get(vfsp, LK_EXCLUSIVE, + ZFSCTL_INO_SNAPDIR, objsetid, &vp); + if (error == 0 && vp != NULL) { + /* + * XXX Probably need to at least reference, if not busy, the mp. + */ + if (vp->v_mountedhere != NULL) + *zfsvfsp = vp->v_mountedhere->mnt_data; + vput(vp); + } + if (*zfsvfsp == NULL) + return (SET_ERROR(EINVAL)); + return (0); +} + +/* + * Unmount any snapshots for the given filesystem. This is called from + * zfs_umount() - if we have a ctldir, then go through and unmount all the + * snapshots. + */ +int +zfsctl_umount_snapshots(vfs_t *vfsp, int fflags, cred_t *cr) +{ + char snapname[ZFS_MAX_DATASET_NAME_LEN]; + zfsvfs_t *zfsvfs = vfsp->vfs_data; + struct mount *mp; + vnode_t *vp; + uint64_t cookie; + int error; + + ASSERT(zfsvfs->z_ctldir != NULL); + + cookie = 0; + for (;;) { + uint64_t id; + + dsl_pool_config_enter(dmu_objset_pool(zfsvfs->z_os), FTAG); + error = dmu_snapshot_list_next(zfsvfs->z_os, sizeof (snapname), + snapname, &id, &cookie, NULL); + dsl_pool_config_exit(dmu_objset_pool(zfsvfs->z_os), FTAG); + if (error != 0) { + if (error == ENOENT) + error = 0; + break; + } + + for (;;) { + error = sfs_vnode_get(vfsp, LK_EXCLUSIVE, + ZFSCTL_INO_SNAPDIR, id, &vp); + if (error != 0 || vp == NULL) + break; + + mp = vp->v_mountedhere; + + /* + * v_mountedhere being NULL means that the + * (uncovered) vnode is in a transient state + * (mounting or unmounting), so loop until it + * settles down. + */ + if (mp != NULL) + break; + vput(vp); + } + if (error != 0) + break; + if (vp == NULL) + continue; /* no mountpoint, nothing to do */ + + /* + * The mount-point vnode is kept locked to avoid spurious EBUSY + * from a concurrent umount. + * The vnode lock must have recursive locking enabled. + */ + vfs_ref(mp); + error = dounmount(mp, fflags, curthread); + KASSERT_IMPLY(error == 0, vrefcnt(vp) == 1, + ("extra references after unmount")); + vput(vp); + if (error != 0) + break; + } + KASSERT_IMPLY((fflags & MS_FORCE) != 0, error == 0, + ("force unmounting failed")); + return (error); +} + +int +zfsctl_snapshot_unmount(char *snapname, int flags __unused) +{ + vfs_t *vfsp = NULL; + zfsvfs_t *zfsvfs = NULL; + + if (strchr(snapname, '@') == NULL) + return (0); + + int err = getzfsvfs(snapname, &zfsvfs); + if (err != 0) { + ASSERT3P(zfsvfs, ==, NULL); + return (0); + } + vfsp = zfsvfs->z_vfs; + + ASSERT(!dsl_pool_config_held(dmu_objset_pool(zfsvfs->z_os))); + + vfs_ref(vfsp); + vfs_unbusy(vfsp); + return (dounmount(vfsp, MS_FORCE, curthread)); +} diff --git a/module/os/freebsd/zfs/zfs_debug.c b/module/os/freebsd/zfs/zfs_debug.c new file mode 100644 index 00000000000..2f5962b25a8 --- /dev/null +++ b/module/os/freebsd/zfs/zfs_debug.c @@ -0,0 +1,254 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + */ + +#include +#include + +typedef struct zfs_dbgmsg { + list_node_t zdm_node; + time_t zdm_timestamp; + int zdm_size; + char zdm_msg[1]; /* variable length allocation */ +} zfs_dbgmsg_t; + +list_t zfs_dbgmsgs; +int zfs_dbgmsg_size = 0; +kmutex_t zfs_dbgmsgs_lock; +int zfs_dbgmsg_maxsize = 4<<20; /* 4MB */ +kstat_t *zfs_dbgmsg_kstat; + +/* + * Internal ZFS debug messages are enabled by default. + * + * # Print debug messages + * cat /proc/spl/kstat/zfs/dbgmsg + * + * # Disable the kernel debug message log. + * echo 0 > /sys/module/zfs/parameters/zfs_dbgmsg_enable + * + * # Clear the kernel debug message log. + * echo 0 >/proc/spl/kstat/zfs/dbgmsg + */ +int zfs_dbgmsg_enable = 1; + +static int +zfs_dbgmsg_headers(char *buf, size_t size) +{ + (void) snprintf(buf, size, "%-12s %-8s\n", "timestamp", "message"); + + return (0); +} + +static int +zfs_dbgmsg_data(char *buf, size_t size, void *data) +{ + zfs_dbgmsg_t *zdm = (zfs_dbgmsg_t *)data; + + (void) snprintf(buf, size, "%-12llu %-s\n", + (u_longlong_t)zdm->zdm_timestamp, zdm->zdm_msg); + + return (0); +} + +static void * +zfs_dbgmsg_addr(kstat_t *ksp, loff_t n) +{ + zfs_dbgmsg_t *zdm = (zfs_dbgmsg_t *)ksp->ks_private; + + ASSERT(MUTEX_HELD(&zfs_dbgmsgs_lock)); + + if (n == 0) + ksp->ks_private = list_head(&zfs_dbgmsgs); + else if (zdm) + ksp->ks_private = list_next(&zfs_dbgmsgs, zdm); + + return (ksp->ks_private); +} + +static void +zfs_dbgmsg_purge(int max_size) +{ + zfs_dbgmsg_t *zdm; + int size; + + ASSERT(MUTEX_HELD(&zfs_dbgmsgs_lock)); + + while (zfs_dbgmsg_size > max_size) { + zdm = list_remove_head(&zfs_dbgmsgs); + if (zdm == NULL) + return; + + size = zdm->zdm_size; + kmem_free(zdm, size); + zfs_dbgmsg_size -= size; + } +} + +static int +zfs_dbgmsg_update(kstat_t *ksp, int rw) +{ + if (rw == KSTAT_WRITE) + zfs_dbgmsg_purge(0); + + return (0); +} + +void +zfs_dbgmsg_init(void) +{ + list_create(&zfs_dbgmsgs, sizeof (zfs_dbgmsg_t), + offsetof(zfs_dbgmsg_t, zdm_node)); + mutex_init(&zfs_dbgmsgs_lock, NULL, MUTEX_DEFAULT, NULL); + + zfs_dbgmsg_kstat = kstat_create("zfs", 0, "dbgmsg", "misc", + KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); + if (zfs_dbgmsg_kstat) { + zfs_dbgmsg_kstat->ks_lock = &zfs_dbgmsgs_lock; + zfs_dbgmsg_kstat->ks_ndata = UINT32_MAX; + zfs_dbgmsg_kstat->ks_private = NULL; + zfs_dbgmsg_kstat->ks_update = zfs_dbgmsg_update; + kstat_set_raw_ops(zfs_dbgmsg_kstat, zfs_dbgmsg_headers, + zfs_dbgmsg_data, zfs_dbgmsg_addr); + kstat_install(zfs_dbgmsg_kstat); + } +} + +void +zfs_dbgmsg_fini(void) +{ + if (zfs_dbgmsg_kstat) + kstat_delete(zfs_dbgmsg_kstat); + /* + * TODO - decide how to make this permanent + */ +#ifdef _KERNEL + mutex_enter(&zfs_dbgmsgs_lock); + zfs_dbgmsg_purge(0); + mutex_exit(&zfs_dbgmsgs_lock); + mutex_destroy(&zfs_dbgmsgs_lock); +#endif +} + +void +__zfs_dbgmsg(char *buf) +{ + zfs_dbgmsg_t *zdm; + int size; + + DTRACE_PROBE1(zfs__dbgmsg, char *, buf); + + size = sizeof (zfs_dbgmsg_t) + strlen(buf); + zdm = kmem_zalloc(size, KM_SLEEP); + zdm->zdm_size = size; + zdm->zdm_timestamp = gethrestime_sec(); + strcpy(zdm->zdm_msg, buf); + + mutex_enter(&zfs_dbgmsgs_lock); + list_insert_tail(&zfs_dbgmsgs, zdm); + zfs_dbgmsg_size += size; + zfs_dbgmsg_purge(MAX(zfs_dbgmsg_maxsize, 0)); + mutex_exit(&zfs_dbgmsgs_lock); +} + +void +__set_error(const char *file, const char *func, int line, int err) +{ + /* + * To enable this: + * + * $ echo 512 >/sys/module/zfs/parameters/zfs_flags + */ + if (zfs_flags & ZFS_DEBUG_SET_ERROR) + __dprintf(B_FALSE, file, func, line, "error %lu", err); +} + +#ifdef _KERNEL +void +__dprintf(boolean_t dprint, const char *file, const char *func, + int line, const char *fmt, ...) +{ + const char *newfile; + va_list adx; + size_t size; + char *buf; + char *nl; + int i; + + size = 1024; + buf = kmem_alloc(size, KM_SLEEP); + + /* + * Get rid of annoying prefix to filename. + */ + newfile = strrchr(file, '/'); + if (newfile != NULL) { + newfile = newfile + 1; /* Get rid of leading / */ + } else { + newfile = file; + } + + i = snprintf(buf, size, "%s:%d:%s(): ", newfile, line, func); + + if (i < size) { + va_start(adx, fmt); + (void) vsnprintf(buf + i, size - i, fmt, adx); + va_end(adx); + } + + /* + * Get rid of trailing newline. + */ + nl = strrchr(buf, '\n'); + if (nl != NULL) + *nl = '\0'; + + __zfs_dbgmsg(buf); + + kmem_free(buf, size); +} + +#else + +void +zfs_dbgmsg_print(const char *tag) +{ + zfs_dbgmsg_t *zdm; + + (void) printf("ZFS_DBGMSG(%s):\n", tag); + mutex_enter(&zfs_dbgmsgs_lock); + for (zdm = list_head(&zfs_dbgmsgs); zdm; + zdm = list_next(&zfs_dbgmsgs, zdm)) + (void) printf("%s\n", zdm->zdm_msg); + mutex_exit(&zfs_dbgmsgs_lock); +} +#endif /* _KERNEL */ + +#ifdef _KERNEL +module_param(zfs_dbgmsg_enable, int, 0644); +MODULE_PARM_DESC(zfs_dbgmsg_enable, "Enable ZFS debug message log"); + +module_param(zfs_dbgmsg_maxsize, int, 0644); +MODULE_PARM_DESC(zfs_dbgmsg_maxsize, "Maximum ZFS debug log size"); +#endif diff --git a/module/os/freebsd/zfs/zfs_dir.c b/module/os/freebsd/zfs/zfs_dir.c new file mode 100644 index 00000000000..e93b3e2cf2f --- /dev/null +++ b/module/os/freebsd/zfs/zfs_dir.c @@ -0,0 +1,961 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2016 by Delphix. All rights reserved. + * Copyright 2017 Nexenta Systems, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * zfs_match_find() is used by zfs_dirent_lookup() to peform zap lookups + * of names after deciding which is the appropriate lookup interface. + */ +static int +zfs_match_find(zfsvfs_t *zfsvfs, znode_t *dzp, const char *name, + matchtype_t mt, uint64_t *zoid) +{ + int error; + + if (zfsvfs->z_norm) { + + /* + * In the non-mixed case we only expect there would ever + * be one match, but we need to use the normalizing lookup. + */ + error = zap_lookup_norm(zfsvfs->z_os, dzp->z_id, name, 8, 1, + zoid, mt, NULL, 0, NULL); + } else { + error = zap_lookup(zfsvfs->z_os, dzp->z_id, name, 8, 1, zoid); + } + *zoid = ZFS_DIRENT_OBJ(*zoid); + + return (error); +} + +/* + * Look up a directory entry under a locked vnode. + * dvp being locked gives us a guarantee that there are no concurrent + * modification of the directory and, thus, if a node can be found in + * the directory, then it must not be unlinked. + * + * Input arguments: + * dzp - znode for directory + * name - name of entry to lock + * flag - ZNEW: if the entry already exists, fail with EEXIST. + * ZEXISTS: if the entry does not exist, fail with ENOENT. + * ZXATTR: we want dzp's xattr directory + * + * Output arguments: + * zpp - pointer to the znode for the entry (NULL if there isn't one) + * + * Return value: 0 on success or errno on failure. + * + * NOTE: Always checks for, and rejects, '.' and '..'. + */ +int +zfs_dirent_lookup(znode_t *dzp, const char *name, znode_t **zpp, int flag) +{ + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + znode_t *zp; + matchtype_t mt = 0; + uint64_t zoid; + int error = 0; + + if (zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_LOCKED(ZTOV(dzp), __func__); + + *zpp = NULL; + + /* + * Verify that we are not trying to lock '.', '..', or '.zfs' + */ + if (name[0] == '.' && + (((name[1] == '\0') || (name[1] == '.' && name[2] == '\0')) || + (zfs_has_ctldir(dzp) && strcmp(name, ZFS_CTLDIR_NAME) == 0))) + return (SET_ERROR(EEXIST)); + + /* + * Case sensitivity and normalization preferences are set when + * the file system is created. These are stored in the + * zfsvfs->z_case and zfsvfs->z_norm fields. These choices + * affect how we perform zap lookups. + * + * When matching we may need to normalize & change case according to + * FS settings. + * + * Note that a normalized match is necessary for a case insensitive + * filesystem when the lookup request is not exact because normalization + * can fold case independent of normalizing code point sequences. + * + * See the table above zfs_dropname(). + */ + if (zfsvfs->z_norm != 0) { + mt = MT_NORMALIZE; + + /* + * Determine if the match needs to honor the case specified in + * lookup, and if so keep track of that so that during + * normalization we don't fold case. + */ + if (zfsvfs->z_case == ZFS_CASE_MIXED) { + mt |= MT_MATCH_CASE; + } + } + + /* + * Only look in or update the DNLC if we are looking for the + * name on a file system that does not require normalization + * or case folding. We can also look there if we happen to be + * on a non-normalizing, mixed sensitivity file system IF we + * are looking for the exact name. + * + * NB: we do not need to worry about this flag for ZFS_CASE_SENSITIVE + * because in that case MT_EXACT and MT_FIRST should produce exactly + * the same result. + */ + + if (dzp->z_unlinked && !(flag & ZXATTR)) + return (ENOENT); + if (flag & ZXATTR) { + error = sa_lookup(dzp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), &zoid, + sizeof (zoid)); + if (error == 0) + error = (zoid == 0 ? ENOENT : 0); + } else { + error = zfs_match_find(zfsvfs, dzp, name, mt, &zoid); + } + if (error) { + if (error != ENOENT || (flag & ZEXISTS)) { + return (error); + } + } else { + if (flag & ZNEW) { + return (SET_ERROR(EEXIST)); + } + error = zfs_zget(zfsvfs, zoid, &zp); + if (error) + return (error); + ASSERT(!zp->z_unlinked); + *zpp = zp; + } + + return (0); +} + +static int +zfs_dd_lookup(znode_t *dzp, znode_t **zpp) +{ + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + znode_t *zp; + uint64_t parent; + int error; + + if (zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_LOCKED(ZTOV(dzp), __func__); + ASSERT(RRM_READ_HELD(&zfsvfs->z_teardown_lock)); + + if (dzp->z_unlinked) + return (ENOENT); + + if ((error = sa_lookup(dzp->z_sa_hdl, + SA_ZPL_PARENT(zfsvfs), &parent, sizeof (parent))) != 0) + return (error); + + error = zfs_zget(zfsvfs, parent, &zp); + if (error == 0) + *zpp = zp; + return (error); +} + +int +zfs_dirlook(znode_t *dzp, const char *name, znode_t **zpp) +{ + zfsvfs_t *zfsvfs __unused = dzp->z_zfsvfs; + znode_t *zp = NULL; + int error = 0; + +#ifdef ZFS_DEBUG + if (zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_LOCKED(ZTOV(dzp), __func__); + ASSERT(RRM_READ_HELD(&zfsvfs->z_teardown_lock)); +#endif + if (dzp->z_unlinked) + return (SET_ERROR(ENOENT)); + + if (name[0] == 0 || (name[0] == '.' && name[1] == 0)) { + *zpp = dzp; + } else if (name[0] == '.' && name[1] == '.' && name[2] == 0) { + error = zfs_dd_lookup(dzp, &zp); + if (error == 0) + *zpp = zp; + } else { + error = zfs_dirent_lookup(dzp, name, &zp, ZEXISTS); + if (error == 0) { + dzp->z_zn_prefetch = B_TRUE; /* enable prefetching */ + *zpp = zp; + } + } + return (error); +} + +/* + * unlinked Set (formerly known as the "delete queue") Error Handling + * + * When dealing with the unlinked set, we dmu_tx_hold_zap(), but we + * don't specify the name of the entry that we will be manipulating. We + * also fib and say that we won't be adding any new entries to the + * unlinked set, even though we might (this is to lower the minimum file + * size that can be deleted in a full filesystem). So on the small + * chance that the nlink list is using a fat zap (ie. has more than + * 2000 entries), we *may* not pre-read a block that's needed. + * Therefore it is remotely possible for some of the assertions + * regarding the unlinked set below to fail due to i/o error. On a + * nondebug system, this will result in the space being leaked. + */ +void +zfs_unlinked_add(znode_t *zp, dmu_tx_t *tx) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + + ASSERT(zp->z_unlinked); + ASSERT(zp->z_links == 0); + + VERIFY3U(0, ==, + zap_add_int(zfsvfs->z_os, zfsvfs->z_unlinkedobj, zp->z_id, tx)); +} + +/* + * Clean up any znodes that had no links when we either crashed or + * (force) umounted the file system. + */ +void +zfs_unlinked_drain(zfsvfs_t *zfsvfs) +{ + zap_cursor_t zc; + zap_attribute_t zap; + dmu_object_info_t doi; + znode_t *zp; + dmu_tx_t *tx; + int error; + + /* + * Interate over the contents of the unlinked set. + */ + for (zap_cursor_init(&zc, zfsvfs->z_os, zfsvfs->z_unlinkedobj); + zap_cursor_retrieve(&zc, &zap) == 0; + zap_cursor_advance(&zc)) { + + /* + * See what kind of object we have in list + */ + + error = dmu_object_info(zfsvfs->z_os, + zap.za_first_integer, &doi); + if (error != 0) + continue; + + ASSERT((doi.doi_type == DMU_OT_PLAIN_FILE_CONTENTS) || + (doi.doi_type == DMU_OT_DIRECTORY_CONTENTS)); + /* + * We need to re-mark these list entries for deletion, + * so we pull them back into core and set zp->z_unlinked. + */ + error = zfs_zget(zfsvfs, zap.za_first_integer, &zp); + + /* + * We may pick up znodes that are already marked for deletion. + * This could happen during the purge of an extended attribute + * directory. All we need to do is skip over them, since they + * are already in the system marked z_unlinked. + */ + if (error != 0) + continue; + + vn_lock(ZTOV(zp), LK_EXCLUSIVE | LK_RETRY); + + /* + * Due to changes in zfs_rmnode we need to make sure the + * link count is set to zero here. + */ + if (zp->z_links != 0) { + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error != 0) { + dmu_tx_abort(tx); + vput(ZTOV(zp)); + continue; + } + zp->z_links = 0; + VERIFY0(sa_update(zp->z_sa_hdl, SA_ZPL_LINKS(zfsvfs), + &zp->z_links, sizeof (zp->z_links), tx)); + dmu_tx_commit(tx); + } + + zp->z_unlinked = B_TRUE; + vput(ZTOV(zp)); + } + zap_cursor_fini(&zc); +} + +/* + * Delete the entire contents of a directory. Return a count + * of the number of entries that could not be deleted. If we encounter + * an error, return a count of at least one so that the directory stays + * in the unlinked set. + * + * NOTE: this function assumes that the directory is inactive, + * so there is no need to lock its entries before deletion. + * Also, it assumes the directory contents is *only* regular + * files. + */ +static int +zfs_purgedir(znode_t *dzp) +{ + zap_cursor_t zc; + zap_attribute_t zap; + znode_t *xzp; + dmu_tx_t *tx; + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + int skipped = 0; + int error; + + for (zap_cursor_init(&zc, zfsvfs->z_os, dzp->z_id); + (error = zap_cursor_retrieve(&zc, &zap)) == 0; + zap_cursor_advance(&zc)) { + error = zfs_zget(zfsvfs, + ZFS_DIRENT_OBJ(zap.za_first_integer), &xzp); + if (error) { + skipped += 1; + continue; + } + + vn_lock(ZTOV(xzp), LK_EXCLUSIVE | LK_RETRY); + ASSERT((ZTOV(xzp)->v_type == VREG) || + (ZTOV(xzp)->v_type == VLNK)); + + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_zap(tx, dzp->z_id, FALSE, zap.za_name); + dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL); + /* Is this really needed ? */ + zfs_sa_upgrade_txholds(tx, xzp); + dmu_tx_mark_netfree(tx); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + vput(ZTOV(xzp)); + skipped += 1; + continue; + } + + error = zfs_link_destroy(dzp, zap.za_name, xzp, tx, 0, NULL); + if (error) + skipped += 1; + dmu_tx_commit(tx); + + vput(ZTOV(xzp)); + } + zap_cursor_fini(&zc); + if (error != ENOENT) + skipped += 1; + return (skipped); +} + +extern taskq_t *zfsvfs_taskq; + +void +zfs_rmnode(znode_t *zp) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + objset_t *os = zfsvfs->z_os; + dmu_tx_t *tx; + uint64_t acl_obj; + uint64_t xattr_obj; + uint64_t count; + int error; + + ASSERT(zp->z_links == 0); + if (zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_ELOCKED(ZTOV(zp), __func__); + + /* + * If this is an attribute directory, purge its contents. + */ + if (ZTOV(zp) != NULL && ZTOV(zp)->v_type == VDIR && + (zp->z_pflags & ZFS_XATTR)) { + if (zfs_purgedir(zp) != 0) { + /* + * Not enough space to delete some xattrs. + * Leave it in the unlinked set. + */ + zfs_znode_dmu_fini(zp); + zfs_znode_free(zp); + return; + } + } else { + /* + * Free up all the data in the file. We don't do this for + * XATTR directories because we need truncate and remove to be + * in the same tx, like in zfs_znode_delete(). Otherwise, if + * we crash here we'll end up with an inconsistent truncated + * zap object in the delete queue. Note a truncated file is + * harmless since it only contains user data. + */ + error = dmu_free_long_range(os, zp->z_id, 0, DMU_OBJECT_END); + if (error) { + /* + * Not enough space or we were interrupted by unmount. + * Leave the file in the unlinked set. + */ + zfs_znode_dmu_fini(zp); + zfs_znode_free(zp); + return; + } + } + + /* + * If the file has extended attributes, we're going to unlink + * the xattr dir. + */ + error = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), + &xattr_obj, sizeof (xattr_obj)); + if (error) + xattr_obj = 0; + + acl_obj = zfs_external_acl(zp); + + /* + * Set up the final transaction. + */ + tx = dmu_tx_create(os); + dmu_tx_hold_free(tx, zp->z_id, 0, DMU_OBJECT_END); + dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL); + if (xattr_obj) + dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, TRUE, NULL); + if (acl_obj) + dmu_tx_hold_free(tx, acl_obj, 0, DMU_OBJECT_END); + + zfs_sa_upgrade_txholds(tx, zp); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + /* + * Not enough space to delete the file. Leave it in the + * unlinked set, leaking it until the fs is remounted (at + * which point we'll call zfs_unlinked_drain() to process it). + */ + dmu_tx_abort(tx); + zfs_znode_dmu_fini(zp); + zfs_znode_free(zp); + return; + } + + /* + * FreeBSD's implemention of zfs_zget requires a vnode to back it. + * This means that we could end up calling into getnewvnode while + * calling zfs_rmnode as a result of a prior call to getnewvnode + * trying to clear vnodes out of the cache. If this repeats we can + * recurse enough that we overflow our stack. To avoid this, we + * avoid calling zfs_zget on the xattr znode and instead simply add + * it to the unlinked set and schedule a call to zfs_unlinked_drain. + */ + if (xattr_obj) { + /* Add extended attribute directory to the unlinked set. */ + VERIFY3U(0, ==, + zap_add_int(os, zfsvfs->z_unlinkedobj, xattr_obj, tx)); + } + + mutex_enter(&os->os_dsl_dataset->ds_dir->dd_activity_lock); + + /* Remove this znode from the unlinked set */ + VERIFY3U(0, ==, + zap_remove_int(os, zfsvfs->z_unlinkedobj, zp->z_id, tx)); + + if (zap_count(os, zfsvfs->z_unlinkedobj, &count) == 0 && count == 0) { + cv_broadcast(&os->os_dsl_dataset->ds_dir->dd_activity_cv); + } + + mutex_exit(&os->os_dsl_dataset->ds_dir->dd_activity_lock); + + zfs_znode_delete(zp, tx); + + dmu_tx_commit(tx); + + if (xattr_obj) { + /* + * We're using the FreeBSD taskqueue API here instead of + * the Solaris taskq API since the FreeBSD API allows for a + * task to be enqueued multiple times but executed once. + */ + taskqueue_enqueue(zfsvfs_taskq->tq_queue, + &zfsvfs->z_unlinked_drain_task); + } +} + +static uint64_t +zfs_dirent(znode_t *zp, uint64_t mode) +{ + uint64_t de = zp->z_id; + + if (zp->z_zfsvfs->z_version >= ZPL_VERSION_DIRENT_TYPE) + de |= IFTODT(mode) << 60; + return (de); +} + +/* + * Link zp into dzp. Can only fail if zp has been unlinked. + */ +int +zfs_link_create(znode_t *dzp, const char *name, znode_t *zp, dmu_tx_t *tx, + int flag) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + vnode_t *vp = ZTOV(zp); + uint64_t value; + int zp_is_dir = (vp->v_type == VDIR); + sa_bulk_attr_t bulk[5]; + uint64_t mtime[2], ctime[2]; + int count = 0; + int error; + + if (zfsvfs->z_replay == B_FALSE) { + ASSERT_VOP_ELOCKED(ZTOV(dzp), __func__); + ASSERT_VOP_ELOCKED(ZTOV(zp), __func__); + } + if (zp_is_dir) { + if (dzp->z_links >= ZFS_LINK_MAX) + return (SET_ERROR(EMLINK)); + } + if (!(flag & ZRENAMING)) { + if (zp->z_unlinked) { /* no new links to unlinked zp */ + ASSERT(!(flag & (ZNEW | ZEXISTS))); + return (SET_ERROR(ENOENT)); + } + if (zp->z_links >= ZFS_LINK_MAX - zp_is_dir) { + return (SET_ERROR(EMLINK)); + } + zp->z_links++; + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL, + &zp->z_links, sizeof (zp->z_links)); + + } else { + ASSERT(zp->z_unlinked == 0); + } + value = zfs_dirent(zp, zp->z_mode); + error = zap_add(zp->z_zfsvfs->z_os, dzp->z_id, name, + 8, 1, &value, tx); + + /* + * zap_add could fail to add the entry if it exceeds the capacity of the + * leaf-block and zap_leaf_split() failed to help. + * The caller of this routine is responsible for failing the transaction + * which will rollback the SA updates done above. + */ + if (error != 0) { + if (!(flag & ZRENAMING) && !(flag & ZNEW)) + zp->z_links--; + return (error); + } + + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zfsvfs), NULL, + &dzp->z_id, sizeof (dzp->z_id)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, + &zp->z_pflags, sizeof (zp->z_pflags)); + + if (!(flag & ZNEW)) { + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, + ctime, sizeof (ctime)); + zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, + ctime); + } + error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); + ASSERT0(error); + + dzp->z_size++; + dzp->z_links += zp_is_dir; + count = 0; + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL, + &dzp->z_size, sizeof (dzp->z_size)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL, + &dzp->z_links, sizeof (dzp->z_links)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, + mtime, sizeof (mtime)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, + ctime, sizeof (ctime)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, + &dzp->z_pflags, sizeof (dzp->z_pflags)); + zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime); + error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx); + ASSERT0(error); + return (0); +} + +/* + * The match type in the code for this function should conform to: + * + * ------------------------------------------------------------------------ + * fs type | z_norm | lookup type | match type + * ---------|-------------|-------------|---------------------------------- + * CS !norm | 0 | 0 | 0 (exact) + * CS norm | formX | 0 | MT_NORMALIZE + * CI !norm | upper | !ZCIEXACT | MT_NORMALIZE + * CI !norm | upper | ZCIEXACT | MT_NORMALIZE | MT_MATCH_CASE + * CI norm | upper|formX | !ZCIEXACT | MT_NORMALIZE + * CI norm | upper|formX | ZCIEXACT | MT_NORMALIZE | MT_MATCH_CASE + * CM !norm | upper | !ZCILOOK | MT_NORMALIZE | MT_MATCH_CASE + * CM !norm | upper | ZCILOOK | MT_NORMALIZE + * CM norm | upper|formX | !ZCILOOK | MT_NORMALIZE | MT_MATCH_CASE + * CM norm | upper|formX | ZCILOOK | MT_NORMALIZE + * + * Abbreviations: + * CS = Case Sensitive, CI = Case Insensitive, CM = Case Mixed + * upper = case folding set by fs type on creation (U8_TEXTPREP_TOUPPER) + * formX = unicode normalization form set on fs creation + */ +static int +zfs_dropname(znode_t *dzp, const char *name, znode_t *zp, dmu_tx_t *tx, + int flag) +{ + int error; + + if (zp->z_zfsvfs->z_norm) { + matchtype_t mt = MT_NORMALIZE; + + if (zp->z_zfsvfs->z_case == ZFS_CASE_MIXED) { + mt |= MT_MATCH_CASE; + } + + error = zap_remove_norm(zp->z_zfsvfs->z_os, dzp->z_id, + name, mt, tx); + } else { + error = zap_remove(zp->z_zfsvfs->z_os, dzp->z_id, name, tx); + } + + return (error); +} + +/* + * Unlink zp from dzp, and mark zp for deletion if this was the last link. + * Can fail if zp is a mount point (EBUSY) or a non-empty directory (EEXIST). + * If 'unlinkedp' is NULL, we put unlinked znodes on the unlinked list. + * If it's non-NULL, we use it to indicate whether the znode needs deletion, + * and it's the caller's job to do it. + */ +int +zfs_link_destroy(znode_t *dzp, const char *name, znode_t *zp, dmu_tx_t *tx, + int flag, boolean_t *unlinkedp) +{ + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + vnode_t *vp = ZTOV(zp); + int zp_is_dir = (vp->v_type == VDIR); + boolean_t unlinked = B_FALSE; + sa_bulk_attr_t bulk[5]; + uint64_t mtime[2], ctime[2]; + int count = 0; + int error; + + if (zfsvfs->z_replay == B_FALSE) { + ASSERT_VOP_ELOCKED(ZTOV(dzp), __func__); + ASSERT_VOP_ELOCKED(ZTOV(zp), __func__); + } + if (!(flag & ZRENAMING)) { + + if (zp_is_dir && !zfs_dirempty(zp)) + return (SET_ERROR(ENOTEMPTY)); + + /* + * If we get here, we are going to try to remove the object. + * First try removing the name from the directory; if that + * fails, return the error. + */ + error = zfs_dropname(dzp, name, zp, tx, flag); + if (error != 0) { + return (error); + } + + if (zp->z_links <= zp_is_dir) { + zfs_panic_recover("zfs: link count on vnode %p is %u, " + "should be at least %u", zp->z_vnode, + (int)zp->z_links, + zp_is_dir + 1); + zp->z_links = zp_is_dir + 1; + } + if (--zp->z_links == zp_is_dir) { + zp->z_unlinked = B_TRUE; + zp->z_links = 0; + unlinked = B_TRUE; + } else { + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), + NULL, &ctime, sizeof (ctime)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), + NULL, &zp->z_pflags, sizeof (zp->z_pflags)); + zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, + ctime); + } + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), + NULL, &zp->z_links, sizeof (zp->z_links)); + error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); + count = 0; + ASSERT0(error); + } else { + ASSERT(zp->z_unlinked == 0); + error = zfs_dropname(dzp, name, zp, tx, flag); + if (error != 0) + return (error); + } + + dzp->z_size--; /* one dirent removed */ + dzp->z_links -= zp_is_dir; /* ".." link from zp */ + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), + NULL, &dzp->z_links, sizeof (dzp->z_links)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), + NULL, &dzp->z_size, sizeof (dzp->z_size)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), + NULL, ctime, sizeof (ctime)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), + NULL, mtime, sizeof (mtime)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), + NULL, &dzp->z_pflags, sizeof (dzp->z_pflags)); + zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime); + error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx); + ASSERT0(error); + + if (unlinkedp != NULL) + *unlinkedp = unlinked; + else if (unlinked) + zfs_unlinked_add(zp, tx); + + return (0); +} + +/* + * Indicate whether the directory is empty. + */ +boolean_t +zfs_dirempty(znode_t *dzp) +{ + return (dzp->z_size == 2); +} + +int +zfs_make_xattrdir(znode_t *zp, vattr_t *vap, znode_t **xvpp, cred_t *cr) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + znode_t *xzp; + dmu_tx_t *tx; + int error; + zfs_acl_ids_t acl_ids; + boolean_t fuid_dirtied; + uint64_t parent __unused; + + *xvpp = NULL; + + if ((error = zfs_acl_ids_create(zp, IS_XATTR, vap, cr, NULL, + &acl_ids)) != 0) + return (error); + if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, 0)) { + zfs_acl_ids_free(&acl_ids); + return (SET_ERROR(EDQUOT)); + } + + getnewvnode_reserve_(); + + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes + + ZFS_SA_BASE_ATTR_SIZE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE); + dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL); + fuid_dirtied = zfsvfs->z_fuid_dirty; + if (fuid_dirtied) + zfs_fuid_txhold(zfsvfs, tx); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + zfs_acl_ids_free(&acl_ids); + dmu_tx_abort(tx); + getnewvnode_drop_reserve(); + return (error); + } + zfs_mknode(zp, vap, tx, cr, IS_XATTR, &xzp, &acl_ids); + + if (fuid_dirtied) + zfs_fuid_sync(zfsvfs, tx); + +#ifdef DEBUG + error = sa_lookup(xzp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs), + &parent, sizeof (parent)); + ASSERT(error == 0 && parent == zp->z_id); +#endif + + VERIFY(0 == sa_update(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), &xzp->z_id, + sizeof (xzp->z_id), tx)); + + (void) zfs_log_create(zfsvfs->z_log, tx, TX_MKXATTR, zp, + xzp, "", NULL, acl_ids.z_fuidp, vap); + + zfs_acl_ids_free(&acl_ids); + dmu_tx_commit(tx); + + getnewvnode_drop_reserve(); + + *xvpp = xzp; + + return (0); +} + +/* + * Return a znode for the extended attribute directory for zp. + * ** If the directory does not already exist, it is created ** + * + * IN: zp - znode to obtain attribute directory from + * cr - credentials of caller + * flags - flags from the VOP_LOOKUP call + * + * OUT: xzpp - pointer to extended attribute znode + * + * RETURN: 0 on success + * error number on failure + */ +int +zfs_get_xattrdir(znode_t *zp, znode_t **xzpp, cred_t *cr, int flags) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + znode_t *xzp; + vattr_t va; + int error; +top: + error = zfs_dirent_lookup(zp, "", &xzp, ZXATTR); + if (error) + return (error); + + if (xzp != NULL) { + *xzpp = xzp; + return (0); + } + + + if (!(flags & CREATE_XATTR_DIR)) + return (SET_ERROR(ENOATTR)); + + if (zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) { + return (SET_ERROR(EROFS)); + } + + /* + * The ability to 'create' files in an attribute + * directory comes from the write_xattr permission on the base file. + * + * The ability to 'search' an attribute directory requires + * read_xattr permission on the base file. + * + * Once in a directory the ability to read/write attributes + * is controlled by the permissions on the attribute file. + */ + va.va_mask = AT_MODE | AT_UID | AT_GID; + va.va_type = VDIR; + va.va_mode = S_IFDIR | S_ISVTX | 0777; + zfs_fuid_map_ids(zp, cr, &va.va_uid, &va.va_gid); + + error = zfs_make_xattrdir(zp, &va, xzpp, cr); + + if (error == ERESTART) { + /* NB: we already did dmu_tx_wait() if necessary */ + goto top; + } + if (error == 0) + VOP_UNLOCK1(ZTOV(*xzpp)); + + return (error); +} + +/* + * Decide whether it is okay to remove within a sticky directory. + * + * In sticky directories, write access is not sufficient; + * you can remove entries from a directory only if: + * + * you own the directory, + * you own the entry, + * the entry is a plain file and you have write access, + * or you are privileged (checked in secpolicy...). + * + * The function returns 0 if remove access is granted. + */ +int +zfs_sticky_remove_access(znode_t *zdp, znode_t *zp, cred_t *cr) +{ + uid_t uid; + uid_t downer; + uid_t fowner; + zfsvfs_t *zfsvfs = zdp->z_zfsvfs; + + if (zdp->z_zfsvfs->z_replay) + return (0); + + if ((zdp->z_mode & S_ISVTX) == 0) + return (0); + + downer = zfs_fuid_map_id(zfsvfs, zdp->z_uid, cr, ZFS_OWNER); + fowner = zfs_fuid_map_id(zfsvfs, zp->z_uid, cr, ZFS_OWNER); + + if ((uid = crgetuid(cr)) == downer || uid == fowner || + (ZTOV(zp)->v_type == VREG && + zfs_zaccess(zp, ACE_WRITE_DATA, 0, B_FALSE, cr) == 0)) + return (0); + else + return (secpolicy_vnode_remove(ZTOV(zp), cr)); +} diff --git a/module/os/freebsd/zfs/zfs_file_os.c b/module/os/freebsd/zfs/zfs_file_os.c new file mode 100644 index 00000000000..ec7c04717c8 --- /dev/null +++ b/module/os/freebsd/zfs/zfs_file_os.c @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int +zfs_file_open(const char *path, int flags, int mode, zfs_file_t **fpp) +{ + struct thread *td; + int rc, fd; + + td = curthread; + pwd_ensure_dirs(); + /* 12.x doesn't take a const char * */ + rc = kern_openat(td, AT_FDCWD, __DECONST(char *, path), + UIO_SYSSPACE, flags, mode); + if (rc) + return (SET_ERROR(rc)); + fd = td->td_retval[0]; + td->td_retval[0] = 0; + if (fget(curthread, fd, &cap_no_rights, fpp)) + kern_close(td, fd); + return (0); +} + +void +zfs_file_close(zfs_file_t *fp) +{ + fo_close(fp, curthread); +} + +static int +zfs_file_write_impl(zfs_file_t *fp, const void *buf, size_t count, loff_t *offp, + ssize_t *resid) +{ + ssize_t rc; + struct uio auio; + struct thread *td; + struct iovec aiov; + + td = curthread; + aiov.iov_base = (void *)(uintptr_t)buf; + aiov.iov_len = count; + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; + auio.uio_segflg = UIO_SYSSPACE; + auio.uio_resid = count; + auio.uio_rw = UIO_WRITE; + auio.uio_td = td; + auio.uio_offset = *offp; + + if ((fp->f_flag & FWRITE) == 0) + return (SET_ERROR(EBADF)); + + if (fp->f_type == DTYPE_VNODE) + bwillwrite(); + + rc = fo_write(fp, &auio, td->td_ucred, FOF_OFFSET, td); + if (rc) + return (SET_ERROR(rc)); + if (resid) + *resid = auio.uio_resid; + else if (auio.uio_resid) + return (SET_ERROR(EIO)); + *offp += count - auio.uio_resid; + return (rc); +} + +int +zfs_file_write(zfs_file_t *fp, const void *buf, size_t count, ssize_t *resid) +{ + loff_t off = fp->f_offset; + ssize_t rc; + + rc = zfs_file_write_impl(fp, buf, count, &off, resid); + if (rc == 0) + fp->f_offset = off; + + return (SET_ERROR(rc)); +} + +int +zfs_file_pwrite(zfs_file_t *fp, const void *buf, size_t count, loff_t off, + ssize_t *resid) +{ + return (zfs_file_write_impl(fp, buf, count, &off, resid)); +} + +static int +zfs_file_read_impl(zfs_file_t *fp, void *buf, size_t count, loff_t *offp, + ssize_t *resid) +{ + ssize_t rc; + struct uio auio; + struct thread *td; + struct iovec aiov; + + td = curthread; + aiov.iov_base = (void *)(uintptr_t)buf; + aiov.iov_len = count; + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; + auio.uio_segflg = UIO_SYSSPACE; + auio.uio_resid = count; + auio.uio_rw = UIO_READ; + auio.uio_td = td; + auio.uio_offset = *offp; + + if ((fp->f_flag & FREAD) == 0) + return (SET_ERROR(EBADF)); + + rc = fo_read(fp, &auio, td->td_ucred, FOF_OFFSET, td); + if (rc) + return (SET_ERROR(rc)); + *resid = auio.uio_resid; + *offp += count - auio.uio_resid; + return (SET_ERROR(0)); +} + +int +zfs_file_read(zfs_file_t *fp, void *buf, size_t count, ssize_t *resid) +{ + loff_t off = fp->f_offset; + ssize_t rc; + + rc = zfs_file_read_impl(fp, buf, count, &off, resid); + if (rc == 0) + fp->f_offset = off; + return (rc); +} + +int +zfs_file_pread(zfs_file_t *fp, void *buf, size_t count, loff_t off, + ssize_t *resid) +{ + return (zfs_file_read_impl(fp, buf, count, &off, resid)); +} + +int +zfs_file_seek(zfs_file_t *fp, loff_t *offp, int whence) +{ + int rc; + struct thread *td; + + td = curthread; + if ((fp->f_ops->fo_flags & DFLAG_SEEKABLE) == 0) + return (SET_ERROR(ESPIPE)); + rc = fo_seek(fp, *offp, whence, td); + if (rc == 0) + *offp = td->td_uretoff.tdu_off; + return (SET_ERROR(rc)); +} + +int +zfs_file_getattr(zfs_file_t *fp, zfs_file_attr_t *zfattr) +{ + struct thread *td; + struct stat sb; + int rc; + + td = curthread; + + rc = fo_stat(fp, &sb, td->td_ucred, td); + if (rc) + return (SET_ERROR(rc)); + zfattr->zfa_size = sb.st_size; + zfattr->zfa_mode = sb.st_mode; + + return (0); +} + +static __inline int +zfs_vop_fsync(vnode_t *vp) +{ + struct mount *mp; + int error; + + if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0) + goto drop; + vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); + error = VOP_FSYNC(vp, MNT_WAIT, curthread); + VOP_UNLOCK1(vp); + vn_finished_write(mp); +drop: + return (SET_ERROR(error)); +} + +int +zfs_file_fsync(zfs_file_t *fp, int flags) +{ + struct vnode *v; + + if (fp->f_type != DTYPE_VNODE) + return (EINVAL); + + v = fp->f_data; + return (zfs_vop_fsync(v)); +} + +int +zfs_file_get(int fd, zfs_file_t **fpp) +{ + struct file *fp; + + if (fget(curthread, fd, &cap_no_rights, &fp)) + return (SET_ERROR(EBADF)); + + *fpp = fp; + return (0); +} + +void +zfs_file_put(int fd) +{ + struct file *fp; + + /* No CAP_ rights required, as we're only releasing. */ + if (fget(curthread, fd, &cap_no_rights, &fp) == 0) { + fdrop(fp, curthread); + fdrop(fp, curthread); + } +} + +loff_t +zfs_file_off(zfs_file_t *fp) +{ + return (fp->f_offset); +} + +void * +zfs_file_private(zfs_file_t *fp) +{ + file_t *tmpfp; + void *data; + int error; + + tmpfp = curthread->td_fpop; + curthread->td_fpop = fp; + error = devfs_get_cdevpriv(&data); + curthread->td_fpop = tmpfp; + if (error != 0) + return (NULL); + return (data); +} + +int +zfs_file_unlink(const char *fnamep) +{ + enum uio_seg seg = UIO_SYSSPACE; + int rc; + +#if __FreeBSD_version >= 1300018 + rc = kern_funlinkat(curthread, AT_FDCWD, fnamep, FD_NONE, seg, 0, 0); +#else +#ifdef AT_BENEATH + rc = kern_unlinkat(curthread, AT_FDCWD, fnamep, seg, 0, 0); +#else + rc = kern_unlinkat(curthread, AT_FDCWD, __DECONST(char *, fnamep), + seg, 0); +#endif +#endif + return (SET_ERROR(rc)); +} diff --git a/module/os/freebsd/zfs/zfs_fuid_os.c b/module/os/freebsd/zfs/zfs_fuid_os.c new file mode 100644 index 00000000000..ebd09abd65e --- /dev/null +++ b/module/os/freebsd/zfs/zfs_fuid_os.c @@ -0,0 +1,52 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#ifdef _KERNEL +#include +#include +#include +#endif +#include + +uint64_t +zfs_fuid_create_cred(zfsvfs_t *zfsvfs, zfs_fuid_type_t type, + cred_t *cr, zfs_fuid_info_t **fuidp) +{ + uid_t id; + + VERIFY(type == ZFS_OWNER || type == ZFS_GROUP); + + id = (type == ZFS_OWNER) ? crgetuid(cr) : crgetgid(cr); + + if (IS_EPHEMERAL(id)) + return ((type == ZFS_OWNER) ? UID_NOBODY : GID_NOBODY); + + return ((uint64_t)id); +} diff --git a/module/os/freebsd/zfs/zfs_ioctl_os.c b/module/os/freebsd/zfs/zfs_ioctl_os.c new file mode 100644 index 00000000000..4b7e8646709 --- /dev/null +++ b/module/os/freebsd/zfs/zfs_ioctl_os.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2020 iXsystems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int +zfs_vfs_ref(zfsvfs_t **zfvp) +{ + int error = 0; + + if (*zfvp == NULL) + return (SET_ERROR(ESRCH)); + + error = vfs_busy((*zfvp)->z_vfs, 0); + if (error != 0) { + *zfvp = NULL; + error = SET_ERROR(ESRCH); + } + return (error); +} + +int +zfs_vfs_held(zfsvfs_t *zfsvfs) +{ + return (zfsvfs->z_vfs != NULL); +} + +void +zfs_vfs_rele(zfsvfs_t *zfsvfs) +{ + vfs_unbusy(zfsvfs->z_vfs); +} + +static const zfs_ioc_key_t zfs_keys_nextboot[] = { + {"command", DATA_TYPE_STRING, 0}, + { ZPOOL_CONFIG_POOL_GUID, DATA_TYPE_UINT64, 0}, + { ZPOOL_CONFIG_GUID, DATA_TYPE_UINT64, 0} +}; + +static int +zfs_ioc_jail(zfs_cmd_t *zc) +{ + + return (zone_dataset_attach(curthread->td_ucred, zc->zc_name, + (int)zc->zc_zoneid)); +} + +static int +zfs_ioc_unjail(zfs_cmd_t *zc) +{ + + return (zone_dataset_detach(curthread->td_ucred, zc->zc_name, + (int)zc->zc_zoneid)); +} + +static int +zfs_ioc_nextboot(const char *unused, nvlist_t *innvl, nvlist_t *outnvl) +{ + char name[MAXNAMELEN]; + spa_t *spa; + vdev_t *vd; + char *command; + uint64_t pool_guid; + uint64_t vdev_guid; + int error; + + if (nvlist_lookup_uint64(innvl, + ZPOOL_CONFIG_POOL_GUID, &pool_guid) != 0) + return (EINVAL); + if (nvlist_lookup_uint64(innvl, + ZPOOL_CONFIG_GUID, &vdev_guid) != 0) + return (EINVAL); + if (nvlist_lookup_string(innvl, + "command", &command) != 0) + return (EINVAL); + + mutex_enter(&spa_namespace_lock); + spa = spa_by_guid(pool_guid, vdev_guid); + if (spa != NULL) + strcpy(name, spa_name(spa)); + mutex_exit(&spa_namespace_lock); + if (spa == NULL) + return (ENOENT); + + if ((error = spa_open(name, &spa, FTAG)) != 0) + return (error); + spa_vdev_state_enter(spa, SCL_ALL); + vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE); + if (vd == NULL) { + (void) spa_vdev_state_exit(spa, NULL, ENXIO); + spa_close(spa, FTAG); + return (ENODEV); + } + error = vdev_label_write_pad2(vd, command, strlen(command)); + (void) spa_vdev_state_exit(spa, NULL, 0); + txg_wait_synced(spa->spa_dsl_pool, 0); + spa_close(spa, FTAG); + return (error); +} + + +void +zfs_ioctl_init_os(void) +{ + zfs_ioctl_register_dataset_nolog(ZFS_IOC_JAIL, zfs_ioc_jail, + zfs_secpolicy_config, POOL_CHECK_NONE); + zfs_ioctl_register_dataset_nolog(ZFS_IOC_UNJAIL, zfs_ioc_unjail, + zfs_secpolicy_config, POOL_CHECK_NONE); + zfs_ioctl_register("fbsd_nextboot", ZFS_IOC_NEXTBOOT, + zfs_ioc_nextboot, zfs_secpolicy_config, NO_NAME, + POOL_CHECK_NONE, B_FALSE, B_FALSE, zfs_keys_nextboot, 3); + +} diff --git a/module/os/freebsd/zfs/zfs_onexit_os.c b/module/os/freebsd/zfs/zfs_onexit_os.c new file mode 100644 index 00000000000..8b22f2fdc3b --- /dev/null +++ b/module/os/freebsd/zfs/zfs_onexit_os.c @@ -0,0 +1,70 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013 by Delphix. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +static int +zfs_onexit_minor_to_state(minor_t minor, zfs_onexit_t **zo) +{ + *zo = zfsdev_get_state(minor, ZST_ONEXIT); + if (*zo == NULL) + return (SET_ERROR(EBADF)); + + return (0); +} + +int +zfs_onexit_fd_hold(int fd, minor_t *minorp) +{ + file_t *fp, *tmpfp; + zfs_onexit_t *zo; + void *data; + int error; + + if ((error = zfs_file_get(fd, &fp))) + return (error); + + tmpfp = curthread->td_fpop; + curthread->td_fpop = fp; + error = devfs_get_cdevpriv(&data); + if (error == 0) + *minorp = (minor_t)(uintptr_t)data; + curthread->td_fpop = tmpfp; + if (error != 0) + return (SET_ERROR(EBADF)); + return (zfs_onexit_minor_to_state(*minorp, &zo)); +} + +void +zfs_onexit_fd_rele(int fd) +{ + zfs_file_put(fd); +} diff --git a/module/os/freebsd/zfs/zfs_vfsops.c b/module/os/freebsd/zfs/zfs_vfsops.c new file mode 100644 index 00000000000..d6f7fc11e9b --- /dev/null +++ b/module/os/freebsd/zfs/zfs_vfsops.c @@ -0,0 +1,2448 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011 Pawel Jakub Dawidek . + * All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. + * Copyright (c) 2014 Integros [integros.com] + * Copyright 2016 Nexenta Systems, Inc. All rights reserved. + */ + +/* Portions Copyright 2010 Robert Milkowski */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zfs_comutil.h" + +#ifndef MNTK_VMSETSIZE_BUG +#define MNTK_VMSETSIZE_BUG 0 +#endif +#ifndef MNTK_NOMSYNC +#define MNTK_NOMSYNC 8 +#endif + +/* BEGIN CSTYLED */ +struct mtx zfs_debug_mtx; +MTX_SYSINIT(zfs_debug_mtx, &zfs_debug_mtx, "zfs_debug", MTX_DEF); + +SYSCTL_NODE(_vfs, OID_AUTO, zfs, CTLFLAG_RW, 0, "ZFS file system"); + +int zfs_super_owner; +SYSCTL_INT(_vfs_zfs, OID_AUTO, super_owner, CTLFLAG_RW, &zfs_super_owner, 0, + "File system owner can perform privileged operation on his file systems"); + +int zfs_debug_level; +SYSCTL_INT(_vfs_zfs, OID_AUTO, debug, CTLFLAG_RWTUN, &zfs_debug_level, 0, + "Debug level"); + +SYSCTL_NODE(_vfs_zfs, OID_AUTO, version, CTLFLAG_RD, 0, "ZFS versions"); +static int zfs_version_acl = ZFS_ACL_VERSION; +SYSCTL_INT(_vfs_zfs_version, OID_AUTO, acl, CTLFLAG_RD, &zfs_version_acl, 0, + "ZFS_ACL_VERSION"); +static int zfs_version_spa = SPA_VERSION; +SYSCTL_INT(_vfs_zfs_version, OID_AUTO, spa, CTLFLAG_RD, &zfs_version_spa, 0, + "SPA_VERSION"); +static int zfs_version_zpl = ZPL_VERSION; +SYSCTL_INT(_vfs_zfs_version, OID_AUTO, zpl, CTLFLAG_RD, &zfs_version_zpl, 0, + "ZPL_VERSION"); +/* END CSTYLED */ + +static int zfs_quotactl(vfs_t *vfsp, int cmds, uid_t id, void *arg); +static int zfs_mount(vfs_t *vfsp); +static int zfs_umount(vfs_t *vfsp, int fflag); +static int zfs_root(vfs_t *vfsp, int flags, vnode_t **vpp); +static int zfs_statfs(vfs_t *vfsp, struct statfs *statp); +static int zfs_vget(vfs_t *vfsp, ino_t ino, int flags, vnode_t **vpp); +static int zfs_sync(vfs_t *vfsp, int waitfor); +static int zfs_checkexp(vfs_t *vfsp, struct sockaddr *nam, int *extflagsp, + struct ucred **credanonp, int *numsecflavors, int **secflavors); +static int zfs_fhtovp(vfs_t *vfsp, fid_t *fidp, int flags, vnode_t **vpp); +static void zfs_freevfs(vfs_t *vfsp); + +struct vfsops zfs_vfsops = { + .vfs_mount = zfs_mount, + .vfs_unmount = zfs_umount, +#if __FreeBSD_version >= 1300049 + .vfs_root = vfs_cache_root, + .vfs_cachedroot = zfs_root, +#else + .vfs_root = zfs_root, +#endif + .vfs_statfs = zfs_statfs, + .vfs_vget = zfs_vget, + .vfs_sync = zfs_sync, + .vfs_checkexp = zfs_checkexp, + .vfs_fhtovp = zfs_fhtovp, + .vfs_quotactl = zfs_quotactl, +}; + +VFS_SET(zfs_vfsops, zfs, VFCF_JAIL | VFCF_DELEGADMIN); + +/* + * We need to keep a count of active fs's. + * This is necessary to prevent our module + * from being unloaded after a umount -f + */ +static uint32_t zfs_active_fs_count = 0; + +int +zfs_get_temporary_prop(dsl_dataset_t *ds, zfs_prop_t zfs_prop, uint64_t *val, + char *setpoint) +{ + int error; + zfsvfs_t *zfvp; + vfs_t *vfsp; + objset_t *os; + uint64_t tmp = *val; + + error = dmu_objset_from_ds(ds, &os); + if (error != 0) + return (error); + + error = getzfsvfs_impl(os, &zfvp); + if (error != 0) + return (error); + if (zfvp == NULL) + return (ENOENT); + vfsp = zfvp->z_vfs; + switch (zfs_prop) { + case ZFS_PROP_ATIME: + if (vfs_optionisset(vfsp, MNTOPT_NOATIME, NULL)) + tmp = 0; + if (vfs_optionisset(vfsp, MNTOPT_ATIME, NULL)) + tmp = 1; + break; + case ZFS_PROP_DEVICES: + if (vfs_optionisset(vfsp, MNTOPT_NODEVICES, NULL)) + tmp = 0; + if (vfs_optionisset(vfsp, MNTOPT_DEVICES, NULL)) + tmp = 1; + break; + case ZFS_PROP_EXEC: + if (vfs_optionisset(vfsp, MNTOPT_NOEXEC, NULL)) + tmp = 0; + if (vfs_optionisset(vfsp, MNTOPT_EXEC, NULL)) + tmp = 1; + break; + case ZFS_PROP_SETUID: + if (vfs_optionisset(vfsp, MNTOPT_NOSETUID, NULL)) + tmp = 0; + if (vfs_optionisset(vfsp, MNTOPT_SETUID, NULL)) + tmp = 1; + break; + case ZFS_PROP_READONLY: + if (vfs_optionisset(vfsp, MNTOPT_RW, NULL)) + tmp = 0; + if (vfs_optionisset(vfsp, MNTOPT_RO, NULL)) + tmp = 1; + break; + case ZFS_PROP_XATTR: + if (zfvp->z_flags & ZSB_XATTR) + tmp = zfvp->z_xattr; + break; + case ZFS_PROP_NBMAND: + if (vfs_optionisset(vfsp, MNTOPT_NONBMAND, NULL)) + tmp = 0; + if (vfs_optionisset(vfsp, MNTOPT_NBMAND, NULL)) + tmp = 1; + break; + default: + vfs_unbusy(vfsp); + return (ENOENT); + } + + vfs_unbusy(vfsp); + if (tmp != *val) { + (void) strcpy(setpoint, "temporary"); + *val = tmp; + } + return (0); +} + +static int +zfs_getquota(zfsvfs_t *zfsvfs, uid_t id, int isgroup, struct dqblk64 *dqp) +{ + int error = 0; + char buf[32]; + uint64_t usedobj, quotaobj; + uint64_t quota, used = 0; + timespec_t now; + + usedobj = isgroup ? DMU_GROUPUSED_OBJECT : DMU_USERUSED_OBJECT; + quotaobj = isgroup ? zfsvfs->z_groupquota_obj : zfsvfs->z_userquota_obj; + + if (quotaobj == 0 || zfsvfs->z_replay) { + error = ENOENT; + goto done; + } + (void) sprintf(buf, "%llx", (longlong_t)id); + if ((error = zap_lookup(zfsvfs->z_os, quotaobj, + buf, sizeof (quota), 1, "a)) != 0) { + dprintf("%s(%d): quotaobj lookup failed\n", + __FUNCTION__, __LINE__); + goto done; + } + /* + * quota(8) uses bsoftlimit as "quoota", and hardlimit as "limit". + * So we set them to be the same. + */ + dqp->dqb_bsoftlimit = dqp->dqb_bhardlimit = btodb(quota); + error = zap_lookup(zfsvfs->z_os, usedobj, buf, sizeof (used), 1, &used); + if (error && error != ENOENT) { + dprintf("%s(%d): usedobj failed; %d\n", + __FUNCTION__, __LINE__, error); + goto done; + } + dqp->dqb_curblocks = btodb(used); + dqp->dqb_ihardlimit = dqp->dqb_isoftlimit = 0; + vfs_timestamp(&now); + /* + * Setting this to 0 causes FreeBSD quota(8) to print + * the number of days since the epoch, which isn't + * particularly useful. + */ + dqp->dqb_btime = dqp->dqb_itime = now.tv_sec; +done: + return (error); +} + +static int +zfs_quotactl(vfs_t *vfsp, int cmds, uid_t id, void *arg) +{ + zfsvfs_t *zfsvfs = vfsp->vfs_data; + struct thread *td; + int cmd, type, error = 0; + int bitsize; + zfs_userquota_prop_t quota_type; + struct dqblk64 dqblk = { 0 }; + + td = curthread; + cmd = cmds >> SUBCMDSHIFT; + type = cmds & SUBCMDMASK; + + ZFS_ENTER(zfsvfs); + if (id == -1) { + switch (type) { + case USRQUOTA: + id = td->td_ucred->cr_ruid; + break; + case GRPQUOTA: + id = td->td_ucred->cr_rgid; + break; + default: + error = EINVAL; + if (cmd == Q_QUOTAON || cmd == Q_QUOTAOFF) + vfs_unbusy(vfsp); + goto done; + } + } + /* + * Map BSD type to: + * ZFS_PROP_USERUSED, + * ZFS_PROP_USERQUOTA, + * ZFS_PROP_GROUPUSED, + * ZFS_PROP_GROUPQUOTA + */ + switch (cmd) { + case Q_SETQUOTA: + case Q_SETQUOTA32: + if (type == USRQUOTA) + quota_type = ZFS_PROP_USERQUOTA; + else if (type == GRPQUOTA) + quota_type = ZFS_PROP_GROUPQUOTA; + else + error = EINVAL; + break; + case Q_GETQUOTA: + case Q_GETQUOTA32: + if (type == USRQUOTA) + quota_type = ZFS_PROP_USERUSED; + else if (type == GRPQUOTA) + quota_type = ZFS_PROP_GROUPUSED; + else + error = EINVAL; + break; + } + + /* + * Depending on the cmd, we may need to get + * the ruid and domain (see fuidstr_to_sid?), + * the fuid (how?), or other information. + * Create fuid using zfs_fuid_create(zfsvfs, id, + * ZFS_OWNER or ZFS_GROUP, cr, &fuidp)? + * I think I can use just the id? + * + * Look at zfs_id_overquota() to look up a quota. + * zap_lookup(something, quotaobj, fuidstring, + * sizeof (long long), 1, "a) + * + * See zfs_set_userquota() to set a quota. + */ + if ((uint32_t)type >= MAXQUOTAS) { + error = EINVAL; + goto done; + } + + switch (cmd) { + case Q_GETQUOTASIZE: + bitsize = 64; + error = copyout(&bitsize, arg, sizeof (int)); + break; + case Q_QUOTAON: + // As far as I can tell, you can't turn quotas on or off on zfs + error = 0; + vfs_unbusy(vfsp); + break; + case Q_QUOTAOFF: + error = ENOTSUP; + vfs_unbusy(vfsp); + break; + case Q_SETQUOTA: + error = copyin(&dqblk, arg, sizeof (dqblk)); + if (error == 0) + error = zfs_set_userquota(zfsvfs, quota_type, + "", id, dbtob(dqblk.dqb_bhardlimit)); + break; + case Q_GETQUOTA: + error = zfs_getquota(zfsvfs, id, type == GRPQUOTA, &dqblk); + if (error == 0) + error = copyout(&dqblk, arg, sizeof (dqblk)); + break; + default: + error = EINVAL; + break; + } +done: + ZFS_EXIT(zfsvfs); + return (error); +} + + +boolean_t +zfs_is_readonly(zfsvfs_t *zfsvfs) +{ + return (!!(zfsvfs->z_vfs->vfs_flag & VFS_RDONLY)); +} + +/*ARGSUSED*/ +static int +zfs_sync(vfs_t *vfsp, int waitfor) +{ + + /* + * Data integrity is job one. We don't want a compromised kernel + * writing to the storage pool, so we never sync during panic. + */ + if (panicstr) + return (0); + + /* + * Ignore the system syncher. ZFS already commits async data + * at zfs_txg_timeout intervals. + */ + if (waitfor == MNT_LAZY) + return (0); + + if (vfsp != NULL) { + /* + * Sync a specific filesystem. + */ + zfsvfs_t *zfsvfs = vfsp->vfs_data; + dsl_pool_t *dp; + int error; + + error = vfs_stdsync(vfsp, waitfor); + if (error != 0) + return (error); + + ZFS_ENTER(zfsvfs); + dp = dmu_objset_pool(zfsvfs->z_os); + + /* + * If the system is shutting down, then skip any + * filesystems which may exist on a suspended pool. + */ + if (rebooting && spa_suspended(dp->dp_spa)) { + ZFS_EXIT(zfsvfs); + return (0); + } + + if (zfsvfs->z_log != NULL) + zil_commit(zfsvfs->z_log, 0); + + ZFS_EXIT(zfsvfs); + } else { + /* + * Sync all ZFS filesystems. This is what happens when you + * run sync(1M). Unlike other filesystems, ZFS honors the + * request by waiting for all pools to commit all dirty data. + */ + spa_sync_allpools(); + } + + return (0); +} + +static void +atime_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + + if (newval == TRUE) { + zfsvfs->z_atime = TRUE; + zfsvfs->z_vfs->vfs_flag &= ~MNT_NOATIME; + vfs_clearmntopt(zfsvfs->z_vfs, MNTOPT_NOATIME); + vfs_setmntopt(zfsvfs->z_vfs, MNTOPT_ATIME, NULL, 0); + } else { + zfsvfs->z_atime = FALSE; + zfsvfs->z_vfs->vfs_flag |= MNT_NOATIME; + vfs_clearmntopt(zfsvfs->z_vfs, MNTOPT_ATIME); + vfs_setmntopt(zfsvfs->z_vfs, MNTOPT_NOATIME, NULL, 0); + } +} + +static void +xattr_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + + if (newval == ZFS_XATTR_OFF) { + zfsvfs->z_flags &= ~ZSB_XATTR; + } else { + zfsvfs->z_flags |= ZSB_XATTR; + + if (newval == ZFS_XATTR_SA) + zfsvfs->z_xattr_sa = B_TRUE; + else + zfsvfs->z_xattr_sa = B_FALSE; + } +} + +static void +blksz_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + ASSERT3U(newval, <=, spa_maxblocksize(dmu_objset_spa(zfsvfs->z_os))); + ASSERT3U(newval, >=, SPA_MINBLOCKSIZE); + ASSERT(ISP2(newval)); + + zfsvfs->z_max_blksz = newval; + zfsvfs->z_vfs->mnt_stat.f_iosize = newval; +} + +static void +readonly_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + + if (newval) { + /* XXX locking on vfs_flag? */ + zfsvfs->z_vfs->vfs_flag |= VFS_RDONLY; + vfs_clearmntopt(zfsvfs->z_vfs, MNTOPT_RW); + vfs_setmntopt(zfsvfs->z_vfs, MNTOPT_RO, NULL, 0); + } else { + /* XXX locking on vfs_flag? */ + zfsvfs->z_vfs->vfs_flag &= ~VFS_RDONLY; + vfs_clearmntopt(zfsvfs->z_vfs, MNTOPT_RO); + vfs_setmntopt(zfsvfs->z_vfs, MNTOPT_RW, NULL, 0); + } +} + +static void +setuid_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + + if (newval == FALSE) { + zfsvfs->z_vfs->vfs_flag |= VFS_NOSETUID; + vfs_clearmntopt(zfsvfs->z_vfs, MNTOPT_SETUID); + vfs_setmntopt(zfsvfs->z_vfs, MNTOPT_NOSETUID, NULL, 0); + } else { + zfsvfs->z_vfs->vfs_flag &= ~VFS_NOSETUID; + vfs_clearmntopt(zfsvfs->z_vfs, MNTOPT_NOSETUID); + vfs_setmntopt(zfsvfs->z_vfs, MNTOPT_SETUID, NULL, 0); + } +} + +static void +exec_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + + if (newval == FALSE) { + zfsvfs->z_vfs->vfs_flag |= VFS_NOEXEC; + vfs_clearmntopt(zfsvfs->z_vfs, MNTOPT_EXEC); + vfs_setmntopt(zfsvfs->z_vfs, MNTOPT_NOEXEC, NULL, 0); + } else { + zfsvfs->z_vfs->vfs_flag &= ~VFS_NOEXEC; + vfs_clearmntopt(zfsvfs->z_vfs, MNTOPT_NOEXEC); + vfs_setmntopt(zfsvfs->z_vfs, MNTOPT_EXEC, NULL, 0); + } +} + +/* + * The nbmand mount option can be changed at mount time. + * We can't allow it to be toggled on live file systems or incorrect + * behavior may be seen from cifs clients + * + * This property isn't registered via dsl_prop_register(), but this callback + * will be called when a file system is first mounted + */ +static void +nbmand_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + if (newval == FALSE) { + vfs_clearmntopt(zfsvfs->z_vfs, MNTOPT_NBMAND); + vfs_setmntopt(zfsvfs->z_vfs, MNTOPT_NONBMAND, NULL, 0); + } else { + vfs_clearmntopt(zfsvfs->z_vfs, MNTOPT_NONBMAND); + vfs_setmntopt(zfsvfs->z_vfs, MNTOPT_NBMAND, NULL, 0); + } +} + +static void +snapdir_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + + zfsvfs->z_show_ctldir = newval; +} + +static void +vscan_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + + zfsvfs->z_vscan = newval; +} + +static void +acl_mode_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + + zfsvfs->z_acl_mode = newval; +} + +static void +acl_inherit_changed_cb(void *arg, uint64_t newval) +{ + zfsvfs_t *zfsvfs = arg; + + zfsvfs->z_acl_inherit = newval; +} + +static int +zfs_register_callbacks(vfs_t *vfsp) +{ + struct dsl_dataset *ds = NULL; + objset_t *os = NULL; + zfsvfs_t *zfsvfs = NULL; + uint64_t nbmand; + boolean_t readonly = B_FALSE; + boolean_t do_readonly = B_FALSE; + boolean_t setuid = B_FALSE; + boolean_t do_setuid = B_FALSE; + boolean_t exec = B_FALSE; + boolean_t do_exec = B_FALSE; + boolean_t xattr = B_FALSE; + boolean_t atime = B_FALSE; + boolean_t do_atime = B_FALSE; + boolean_t do_xattr = B_FALSE; + int error = 0; + + ASSERT(vfsp); + zfsvfs = vfsp->vfs_data; + ASSERT(zfsvfs); + os = zfsvfs->z_os; + + /* + * This function can be called for a snapshot when we update snapshot's + * mount point, which isn't really supported. + */ + if (dmu_objset_is_snapshot(os)) + return (EOPNOTSUPP); + + /* + * The act of registering our callbacks will destroy any mount + * options we may have. In order to enable temporary overrides + * of mount options, we stash away the current values and + * restore them after we register the callbacks. + */ + if (vfs_optionisset(vfsp, MNTOPT_RO, NULL) || + !spa_writeable(dmu_objset_spa(os))) { + readonly = B_TRUE; + do_readonly = B_TRUE; + } else if (vfs_optionisset(vfsp, MNTOPT_RW, NULL)) { + readonly = B_FALSE; + do_readonly = B_TRUE; + } + if (vfs_optionisset(vfsp, MNTOPT_NOSETUID, NULL)) { + setuid = B_FALSE; + do_setuid = B_TRUE; + } else if (vfs_optionisset(vfsp, MNTOPT_SETUID, NULL)) { + setuid = B_TRUE; + do_setuid = B_TRUE; + } + if (vfs_optionisset(vfsp, MNTOPT_NOEXEC, NULL)) { + exec = B_FALSE; + do_exec = B_TRUE; + } else if (vfs_optionisset(vfsp, MNTOPT_EXEC, NULL)) { + exec = B_TRUE; + do_exec = B_TRUE; + } + if (vfs_optionisset(vfsp, MNTOPT_NOXATTR, NULL)) { + zfsvfs->z_xattr = xattr = ZFS_XATTR_OFF; + do_xattr = B_TRUE; + } else if (vfs_optionisset(vfsp, MNTOPT_XATTR, NULL)) { + zfsvfs->z_xattr = xattr = ZFS_XATTR_DIR; + do_xattr = B_TRUE; + } else if (vfs_optionisset(vfsp, MNTOPT_DIRXATTR, NULL)) { + zfsvfs->z_xattr = xattr = ZFS_XATTR_DIR; + do_xattr = B_TRUE; + } else if (vfs_optionisset(vfsp, MNTOPT_SAXATTR, NULL)) { + zfsvfs->z_xattr = xattr = ZFS_XATTR_SA; + do_xattr = B_TRUE; + } + if (vfs_optionisset(vfsp, MNTOPT_NOATIME, NULL)) { + atime = B_FALSE; + do_atime = B_TRUE; + } else if (vfs_optionisset(vfsp, MNTOPT_ATIME, NULL)) { + atime = B_TRUE; + do_atime = B_TRUE; + } + + /* + * We need to enter pool configuration here, so that we can use + * dsl_prop_get_int_ds() to handle the special nbmand property below. + * dsl_prop_get_integer() can not be used, because it has to acquire + * spa_namespace_lock and we can not do that because we already hold + * z_teardown_lock. The problem is that spa_write_cachefile() is called + * with spa_namespace_lock held and the function calls ZFS vnode + * operations to write the cache file and thus z_teardown_lock is + * acquired after spa_namespace_lock. + */ + ds = dmu_objset_ds(os); + dsl_pool_config_enter(dmu_objset_pool(os), FTAG); + + /* + * nbmand is a special property. It can only be changed at + * mount time. + * + * This is weird, but it is documented to only be changeable + * at mount time. + */ + if (vfs_optionisset(vfsp, MNTOPT_NONBMAND, NULL)) { + nbmand = B_FALSE; + } else if (vfs_optionisset(vfsp, MNTOPT_NBMAND, NULL)) { + nbmand = B_TRUE; + } else if ((error = dsl_prop_get_int_ds(ds, "nbmand", &nbmand) != 0)) { + dsl_pool_config_exit(dmu_objset_pool(os), FTAG); + return (error); + } + + /* + * Register property callbacks. + * + * It would probably be fine to just check for i/o error from + * the first prop_register(), but I guess I like to go + * overboard... + */ + error = dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_ATIME), atime_changed_cb, zfsvfs); + error = error ? error : dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_XATTR), xattr_changed_cb, zfsvfs); + error = error ? error : dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_RECORDSIZE), blksz_changed_cb, zfsvfs); + error = error ? error : dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_READONLY), readonly_changed_cb, zfsvfs); + error = error ? error : dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_SETUID), setuid_changed_cb, zfsvfs); + error = error ? error : dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_EXEC), exec_changed_cb, zfsvfs); + error = error ? error : dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_SNAPDIR), snapdir_changed_cb, zfsvfs); + error = error ? error : dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_ACLMODE), acl_mode_changed_cb, zfsvfs); + error = error ? error : dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_ACLINHERIT), acl_inherit_changed_cb, + zfsvfs); + error = error ? error : dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_VSCAN), vscan_changed_cb, zfsvfs); + dsl_pool_config_exit(dmu_objset_pool(os), FTAG); + if (error) + goto unregister; + + /* + * Invoke our callbacks to restore temporary mount options. + */ + if (do_readonly) + readonly_changed_cb(zfsvfs, readonly); + if (do_setuid) + setuid_changed_cb(zfsvfs, setuid); + if (do_exec) + exec_changed_cb(zfsvfs, exec); + if (do_xattr) + xattr_changed_cb(zfsvfs, xattr); + if (do_atime) + atime_changed_cb(zfsvfs, atime); + + nbmand_changed_cb(zfsvfs, nbmand); + + return (0); + +unregister: + dsl_prop_unregister_all(ds, zfsvfs); + return (error); +} + +/* + * Associate this zfsvfs with the given objset, which must be owned. + * This will cache a bunch of on-disk state from the objset in the + * zfsvfs. + */ +static int +zfsvfs_init(zfsvfs_t *zfsvfs, objset_t *os) +{ + int error; + uint64_t val; + + zfsvfs->z_max_blksz = SPA_OLD_MAXBLOCKSIZE; + zfsvfs->z_show_ctldir = ZFS_SNAPDIR_VISIBLE; + zfsvfs->z_os = os; + + error = zfs_get_zplprop(os, ZFS_PROP_VERSION, &zfsvfs->z_version); + if (error != 0) + return (error); + if (zfsvfs->z_version > + zfs_zpl_version_map(spa_version(dmu_objset_spa(os)))) { + (void) printf("Can't mount a version %lld file system " + "on a version %lld pool\n. Pool must be upgraded to mount " + "this file system.", (u_longlong_t)zfsvfs->z_version, + (u_longlong_t)spa_version(dmu_objset_spa(os))); + return (SET_ERROR(ENOTSUP)); + } + error = zfs_get_zplprop(os, ZFS_PROP_NORMALIZE, &val); + if (error != 0) + return (error); + zfsvfs->z_norm = (int)val; + + error = zfs_get_zplprop(os, ZFS_PROP_UTF8ONLY, &val); + if (error != 0) + return (error); + zfsvfs->z_utf8 = (val != 0); + + error = zfs_get_zplprop(os, ZFS_PROP_CASE, &val); + if (error != 0) + return (error); + zfsvfs->z_case = (uint_t)val; + + /* + * Fold case on file systems that are always or sometimes case + * insensitive. + */ + if (zfsvfs->z_case == ZFS_CASE_INSENSITIVE || + zfsvfs->z_case == ZFS_CASE_MIXED) + zfsvfs->z_norm |= U8_TEXTPREP_TOUPPER; + + zfsvfs->z_use_fuids = USE_FUIDS(zfsvfs->z_version, zfsvfs->z_os); + zfsvfs->z_use_sa = USE_SA(zfsvfs->z_version, zfsvfs->z_os); + + uint64_t sa_obj = 0; + if (zfsvfs->z_use_sa) { + /* should either have both of these objects or none */ + error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_SA_ATTRS, 8, 1, + &sa_obj); + if (error != 0) + return (error); + } + + error = sa_setup(os, sa_obj, zfs_attr_table, ZPL_END, + &zfsvfs->z_attr_table); + if (error != 0) + return (error); + + if (zfsvfs->z_version >= ZPL_VERSION_SA) + sa_register_update_callback(os, zfs_sa_upgrade); + + error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_ROOT_OBJ, 8, 1, + &zfsvfs->z_root); + if (error != 0) + return (error); + ASSERT(zfsvfs->z_root != 0); + + error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_UNLINKED_SET, 8, 1, + &zfsvfs->z_unlinkedobj); + if (error != 0) + return (error); + + error = zap_lookup(os, MASTER_NODE_OBJ, + zfs_userquota_prop_prefixes[ZFS_PROP_USERQUOTA], + 8, 1, &zfsvfs->z_userquota_obj); + if (error == ENOENT) + zfsvfs->z_userquota_obj = 0; + else if (error != 0) + return (error); + + error = zap_lookup(os, MASTER_NODE_OBJ, + zfs_userquota_prop_prefixes[ZFS_PROP_GROUPQUOTA], + 8, 1, &zfsvfs->z_groupquota_obj); + if (error == ENOENT) + zfsvfs->z_groupquota_obj = 0; + else if (error != 0) + return (error); + + error = zap_lookup(os, MASTER_NODE_OBJ, + zfs_userquota_prop_prefixes[ZFS_PROP_PROJECTQUOTA], + 8, 1, &zfsvfs->z_projectquota_obj); + if (error == ENOENT) + zfsvfs->z_projectquota_obj = 0; + else if (error != 0) + return (error); + + error = zap_lookup(os, MASTER_NODE_OBJ, + zfs_userquota_prop_prefixes[ZFS_PROP_USEROBJQUOTA], + 8, 1, &zfsvfs->z_userobjquota_obj); + if (error == ENOENT) + zfsvfs->z_userobjquota_obj = 0; + else if (error != 0) + return (error); + + error = zap_lookup(os, MASTER_NODE_OBJ, + zfs_userquota_prop_prefixes[ZFS_PROP_GROUPOBJQUOTA], + 8, 1, &zfsvfs->z_groupobjquota_obj); + if (error == ENOENT) + zfsvfs->z_groupobjquota_obj = 0; + else if (error != 0) + return (error); + + error = zap_lookup(os, MASTER_NODE_OBJ, + zfs_userquota_prop_prefixes[ZFS_PROP_PROJECTOBJQUOTA], + 8, 1, &zfsvfs->z_projectobjquota_obj); + if (error == ENOENT) + zfsvfs->z_projectobjquota_obj = 0; + else if (error != 0) + return (error); + + error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_FUID_TABLES, 8, 1, + &zfsvfs->z_fuid_obj); + if (error == ENOENT) + zfsvfs->z_fuid_obj = 0; + else if (error != 0) + return (error); + + error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_SHARES_DIR, 8, 1, + &zfsvfs->z_shares_dir); + if (error == ENOENT) + zfsvfs->z_shares_dir = 0; + else if (error != 0) + return (error); + + /* + * Only use the name cache if we are looking for a + * name on a file system that does not require normalization + * or case folding. We can also look there if we happen to be + * on a non-normalizing, mixed sensitivity file system IF we + * are looking for the exact name (which is always the case on + * FreeBSD). + */ + zfsvfs->z_use_namecache = !zfsvfs->z_norm || + ((zfsvfs->z_case == ZFS_CASE_MIXED) && + !(zfsvfs->z_norm & ~U8_TEXTPREP_TOUPPER)); + + return (0); +} + +taskq_t *zfsvfs_taskq; + +static void +zfsvfs_task_unlinked_drain(void *context, int pending __unused) +{ + + zfs_unlinked_drain((zfsvfs_t *)context); +} + +int +zfsvfs_create(const char *osname, boolean_t readonly, zfsvfs_t **zfvp) +{ + objset_t *os; + zfsvfs_t *zfsvfs; + int error; + boolean_t ro = (readonly || (strchr(osname, '@') != NULL)); + + /* + * XXX: Fix struct statfs so this isn't necessary! + * + * The 'osname' is used as the filesystem's special node, which means + * it must fit in statfs.f_mntfromname, or else it can't be + * enumerated, so libzfs_mnttab_find() returns NULL, which causes + * 'zfs unmount' to think it's not mounted when it is. + */ + if (strlen(osname) >= MNAMELEN) + return (SET_ERROR(ENAMETOOLONG)); + + zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP); + + error = dmu_objset_own(osname, DMU_OST_ZFS, ro, B_TRUE, zfsvfs, + &os); + if (error != 0) { + kmem_free(zfsvfs, sizeof (zfsvfs_t)); + return (error); + } + + error = zfsvfs_create_impl(zfvp, zfsvfs, os); + + return (error); +} + + +int +zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os) +{ + int error; + + zfsvfs->z_vfs = NULL; + zfsvfs->z_parent = zfsvfs; + + mutex_init(&zfsvfs->z_znodes_lock, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&zfsvfs->z_lock, NULL, MUTEX_DEFAULT, NULL); + list_create(&zfsvfs->z_all_znodes, sizeof (znode_t), + offsetof(znode_t, z_link_node)); + TASK_INIT(&zfsvfs->z_unlinked_drain_task, 0, + zfsvfs_task_unlinked_drain, zfsvfs); +#ifdef DIAGNOSTIC + rrm_init(&zfsvfs->z_teardown_lock, B_TRUE); +#else + rrm_init(&zfsvfs->z_teardown_lock, B_FALSE); +#endif + rw_init(&zfsvfs->z_teardown_inactive_lock, NULL, RW_DEFAULT, NULL); + rw_init(&zfsvfs->z_fuid_lock, NULL, RW_DEFAULT, NULL); + for (int i = 0; i != ZFS_OBJ_MTX_SZ; i++) + mutex_init(&zfsvfs->z_hold_mtx[i], NULL, MUTEX_DEFAULT, NULL); + + error = zfsvfs_init(zfsvfs, os); + if (error != 0) { + dmu_objset_disown(os, B_TRUE, zfsvfs); + *zfvp = NULL; + kmem_free(zfsvfs, sizeof (zfsvfs_t)); + return (error); + } + + *zfvp = zfsvfs; + return (0); +} + +static int +zfsvfs_setup(zfsvfs_t *zfsvfs, boolean_t mounting) +{ + int error; + + /* + * Check for a bad on-disk format version now since we + * lied about owning the dataset readonly before. + */ + if (!(zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) && + dmu_objset_incompatible_encryption_version(zfsvfs->z_os)) + return (SET_ERROR(EROFS)); + + error = zfs_register_callbacks(zfsvfs->z_vfs); + if (error) + return (error); + + zfsvfs->z_log = zil_open(zfsvfs->z_os, zfs_get_data); + + /* + * If we are not mounting (ie: online recv), then we don't + * have to worry about replaying the log as we blocked all + * operations out since we closed the ZIL. + */ + if (mounting) { + boolean_t readonly; + + /* + * During replay we remove the read only flag to + * allow replays to succeed. + */ + readonly = zfsvfs->z_vfs->vfs_flag & VFS_RDONLY; + if (readonly != 0) { + zfsvfs->z_vfs->vfs_flag &= ~VFS_RDONLY; + } else { + dsl_dir_t *dd; + + zfs_unlinked_drain(zfsvfs); + dd = zfsvfs->z_os->os_dsl_dataset->ds_dir; + dd->dd_activity_cancelled = B_FALSE; + } + + /* + * Parse and replay the intent log. + * + * Because of ziltest, this must be done after + * zfs_unlinked_drain(). (Further note: ziltest + * doesn't use readonly mounts, where + * zfs_unlinked_drain() isn't called.) This is because + * ziltest causes spa_sync() to think it's committed, + * but actually it is not, so the intent log contains + * many txg's worth of changes. + * + * In particular, if object N is in the unlinked set in + * the last txg to actually sync, then it could be + * actually freed in a later txg and then reallocated + * in a yet later txg. This would write a "create + * object N" record to the intent log. Normally, this + * would be fine because the spa_sync() would have + * written out the fact that object N is free, before + * we could write the "create object N" intent log + * record. + * + * But when we are in ziltest mode, we advance the "open + * txg" without actually spa_sync()-ing the changes to + * disk. So we would see that object N is still + * allocated and in the unlinked set, and there is an + * intent log record saying to allocate it. + */ + if (spa_writeable(dmu_objset_spa(zfsvfs->z_os))) { + if (zil_replay_disable) { + zil_destroy(zfsvfs->z_log, B_FALSE); + } else { + boolean_t use_nc = zfsvfs->z_use_namecache; + zfsvfs->z_use_namecache = B_FALSE; + zfsvfs->z_replay = B_TRUE; + zil_replay(zfsvfs->z_os, zfsvfs, + zfs_replay_vector); + zfsvfs->z_replay = B_FALSE; + zfsvfs->z_use_namecache = use_nc; + } + } + + /* restore readonly bit */ + if (readonly != 0) + zfsvfs->z_vfs->vfs_flag |= VFS_RDONLY; + } + + /* + * Set the objset user_ptr to track its zfsvfs. + */ + mutex_enter(&zfsvfs->z_os->os_user_ptr_lock); + dmu_objset_set_user(zfsvfs->z_os, zfsvfs); + mutex_exit(&zfsvfs->z_os->os_user_ptr_lock); + + return (0); +} + +extern krwlock_t zfsvfs_lock; /* in zfs_znode.c */ + +void +zfsvfs_free(zfsvfs_t *zfsvfs) +{ + int i; + + /* + * This is a barrier to prevent the filesystem from going away in + * zfs_znode_move() until we can safely ensure that the filesystem is + * not unmounted. We consider the filesystem valid before the barrier + * and invalid after the barrier. + */ + rw_enter(&zfsvfs_lock, RW_READER); + rw_exit(&zfsvfs_lock); + + zfs_fuid_destroy(zfsvfs); + + mutex_destroy(&zfsvfs->z_znodes_lock); + mutex_destroy(&zfsvfs->z_lock); + ASSERT(zfsvfs->z_nr_znodes == 0); + list_destroy(&zfsvfs->z_all_znodes); + rrm_destroy(&zfsvfs->z_teardown_lock); + rw_destroy(&zfsvfs->z_teardown_inactive_lock); + rw_destroy(&zfsvfs->z_fuid_lock); + for (i = 0; i != ZFS_OBJ_MTX_SZ; i++) + mutex_destroy(&zfsvfs->z_hold_mtx[i]); + kmem_free(zfsvfs, sizeof (zfsvfs_t)); +} + +static void +zfs_set_fuid_feature(zfsvfs_t *zfsvfs) +{ + zfsvfs->z_use_fuids = USE_FUIDS(zfsvfs->z_version, zfsvfs->z_os); + if (zfsvfs->z_vfs) { + if (zfsvfs->z_use_fuids) { + vfs_set_feature(zfsvfs->z_vfs, VFSFT_XVATTR); + vfs_set_feature(zfsvfs->z_vfs, VFSFT_SYSATTR_VIEWS); + vfs_set_feature(zfsvfs->z_vfs, VFSFT_ACEMASKONACCESS); + vfs_set_feature(zfsvfs->z_vfs, VFSFT_ACLONCREATE); + vfs_set_feature(zfsvfs->z_vfs, VFSFT_ACCESS_FILTER); + vfs_set_feature(zfsvfs->z_vfs, VFSFT_REPARSE); + } else { + vfs_clear_feature(zfsvfs->z_vfs, VFSFT_XVATTR); + vfs_clear_feature(zfsvfs->z_vfs, VFSFT_SYSATTR_VIEWS); + vfs_clear_feature(zfsvfs->z_vfs, VFSFT_ACEMASKONACCESS); + vfs_clear_feature(zfsvfs->z_vfs, VFSFT_ACLONCREATE); + vfs_clear_feature(zfsvfs->z_vfs, VFSFT_ACCESS_FILTER); + vfs_clear_feature(zfsvfs->z_vfs, VFSFT_REPARSE); + } + } + zfsvfs->z_use_sa = USE_SA(zfsvfs->z_version, zfsvfs->z_os); +} + +static int +zfs_domount(vfs_t *vfsp, char *osname) +{ + uint64_t recordsize, fsid_guid; + int error = 0; + zfsvfs_t *zfsvfs; + + ASSERT(vfsp); + ASSERT(osname); + + error = zfsvfs_create(osname, vfsp->mnt_flag & MNT_RDONLY, &zfsvfs); + if (error) + return (error); + zfsvfs->z_vfs = vfsp; + + if ((error = dsl_prop_get_integer(osname, + "recordsize", &recordsize, NULL))) + goto out; + zfsvfs->z_vfs->vfs_bsize = SPA_MINBLOCKSIZE; + zfsvfs->z_vfs->mnt_stat.f_iosize = recordsize; + + vfsp->vfs_data = zfsvfs; + vfsp->mnt_flag |= MNT_LOCAL; + vfsp->mnt_kern_flag |= MNTK_LOOKUP_SHARED; + vfsp->mnt_kern_flag |= MNTK_SHARED_WRITES; + vfsp->mnt_kern_flag |= MNTK_EXTENDED_SHARED; + /* + * This can cause a loss of coherence between ARC and page cache + * on ZoF - unclear if the problem is in FreeBSD or ZoF + */ + vfsp->mnt_kern_flag |= MNTK_NO_IOPF; /* vn_io_fault can be used */ + vfsp->mnt_kern_flag |= MNTK_NOMSYNC; + vfsp->mnt_kern_flag |= MNTK_VMSETSIZE_BUG; + + /* + * The fsid is 64 bits, composed of an 8-bit fs type, which + * separates our fsid from any other filesystem types, and a + * 56-bit objset unique ID. The objset unique ID is unique to + * all objsets open on this system, provided by unique_create(). + * The 8-bit fs type must be put in the low bits of fsid[1] + * because that's where other Solaris filesystems put it. + */ + fsid_guid = dmu_objset_fsid_guid(zfsvfs->z_os); + ASSERT((fsid_guid & ~((1ULL<<56)-1)) == 0); + vfsp->vfs_fsid.val[0] = fsid_guid; + vfsp->vfs_fsid.val[1] = ((fsid_guid>>32) << 8) | + (vfsp->mnt_vfc->vfc_typenum & 0xFF); + + /* + * Set features for file system. + */ + zfs_set_fuid_feature(zfsvfs); + if (zfsvfs->z_case == ZFS_CASE_INSENSITIVE) { + vfs_set_feature(vfsp, VFSFT_DIRENTFLAGS); + vfs_set_feature(vfsp, VFSFT_CASEINSENSITIVE); + vfs_set_feature(vfsp, VFSFT_NOCASESENSITIVE); + } else if (zfsvfs->z_case == ZFS_CASE_MIXED) { + vfs_set_feature(vfsp, VFSFT_DIRENTFLAGS); + vfs_set_feature(vfsp, VFSFT_CASEINSENSITIVE); + } + vfs_set_feature(vfsp, VFSFT_ZEROCOPY_SUPPORTED); + + if (dmu_objset_is_snapshot(zfsvfs->z_os)) { + uint64_t pval; + + atime_changed_cb(zfsvfs, B_FALSE); + readonly_changed_cb(zfsvfs, B_TRUE); + if ((error = dsl_prop_get_integer(osname, + "xattr", &pval, NULL))) + goto out; + xattr_changed_cb(zfsvfs, pval); + zfsvfs->z_issnap = B_TRUE; + zfsvfs->z_os->os_sync = ZFS_SYNC_DISABLED; + + mutex_enter(&zfsvfs->z_os->os_user_ptr_lock); + dmu_objset_set_user(zfsvfs->z_os, zfsvfs); + mutex_exit(&zfsvfs->z_os->os_user_ptr_lock); + } else { + if ((error = zfsvfs_setup(zfsvfs, B_TRUE))) + goto out; + } + + vfs_mountedfrom(vfsp, osname); + + if (!zfsvfs->z_issnap) + zfsctl_create(zfsvfs); +out: + if (error) { + dmu_objset_disown(zfsvfs->z_os, B_TRUE, zfsvfs); + zfsvfs_free(zfsvfs); + } else { + atomic_inc_32(&zfs_active_fs_count); + } + + return (error); +} + +void +zfs_unregister_callbacks(zfsvfs_t *zfsvfs) +{ + objset_t *os = zfsvfs->z_os; + + if (!dmu_objset_is_snapshot(os)) + dsl_prop_unregister_all(dmu_objset_ds(os), zfsvfs); +} + +#ifdef SECLABEL +/* + * Convert a decimal digit string to a uint64_t integer. + */ +static int +str_to_uint64(char *str, uint64_t *objnum) +{ + uint64_t num = 0; + + while (*str) { + if (*str < '0' || *str > '9') + return (SET_ERROR(EINVAL)); + + num = num*10 + *str++ - '0'; + } + + *objnum = num; + return (0); +} + +/* + * The boot path passed from the boot loader is in the form of + * "rootpool-name/root-filesystem-object-number'. Convert this + * string to a dataset name: "rootpool-name/root-filesystem-name". + */ +static int +zfs_parse_bootfs(char *bpath, char *outpath) +{ + char *slashp; + uint64_t objnum; + int error; + + if (*bpath == 0 || *bpath == '/') + return (SET_ERROR(EINVAL)); + + (void) strcpy(outpath, bpath); + + slashp = strchr(bpath, '/'); + + /* if no '/', just return the pool name */ + if (slashp == NULL) { + return (0); + } + + /* if not a number, just return the root dataset name */ + if (str_to_uint64(slashp+1, &objnum)) { + return (0); + } + + *slashp = '\0'; + error = dsl_dsobj_to_dsname(bpath, objnum, outpath); + *slashp = '/'; + + return (error); +} + +/* + * Check that the hex label string is appropriate for the dataset being + * mounted into the global_zone proper. + * + * Return an error if the hex label string is not default or + * admin_low/admin_high. For admin_low labels, the corresponding + * dataset must be readonly. + */ +int +zfs_check_global_label(const char *dsname, const char *hexsl) +{ + if (strcasecmp(hexsl, ZFS_MLSLABEL_DEFAULT) == 0) + return (0); + if (strcasecmp(hexsl, ADMIN_HIGH) == 0) + return (0); + if (strcasecmp(hexsl, ADMIN_LOW) == 0) { + /* must be readonly */ + uint64_t rdonly; + + if (dsl_prop_get_integer(dsname, + zfs_prop_to_name(ZFS_PROP_READONLY), &rdonly, NULL)) + return (SET_ERROR(EACCES)); + return (rdonly ? 0 : EACCES); + } + return (SET_ERROR(EACCES)); +} + +/* + * Determine whether the mount is allowed according to MAC check. + * by comparing (where appropriate) label of the dataset against + * the label of the zone being mounted into. If the dataset has + * no label, create one. + * + * Returns 0 if access allowed, error otherwise (e.g. EACCES) + */ +static int +zfs_mount_label_policy(vfs_t *vfsp, char *osname) +{ + int error, retv; + zone_t *mntzone = NULL; + ts_label_t *mnt_tsl; + bslabel_t *mnt_sl; + bslabel_t ds_sl; + char ds_hexsl[MAXNAMELEN]; + + retv = EACCES; /* assume the worst */ + + /* + * Start by getting the dataset label if it exists. + */ + error = dsl_prop_get(osname, zfs_prop_to_name(ZFS_PROP_MLSLABEL), + 1, sizeof (ds_hexsl), &ds_hexsl, NULL); + if (error) + return (SET_ERROR(EACCES)); + + /* + * If labeling is NOT enabled, then disallow the mount of datasets + * which have a non-default label already. No other label checks + * are needed. + */ + if (!is_system_labeled()) { + if (strcasecmp(ds_hexsl, ZFS_MLSLABEL_DEFAULT) == 0) + return (0); + return (SET_ERROR(EACCES)); + } + + /* + * Get the label of the mountpoint. If mounting into the global + * zone (i.e. mountpoint is not within an active zone and the + * zoned property is off), the label must be default or + * admin_low/admin_high only; no other checks are needed. + */ + mntzone = zone_find_by_any_path(vfsp->vfs_mntpt, B_FALSE); + if (mntzone->zone_id == GLOBAL_ZONEID) { + uint64_t zoned; + + zone_rele(mntzone); + + if (dsl_prop_get_integer(osname, + zfs_prop_to_name(ZFS_PROP_ZONED), &zoned, NULL)) + return (SET_ERROR(EACCES)); + if (!zoned) + return (zfs_check_global_label(osname, ds_hexsl)); + else + /* + * This is the case of a zone dataset being mounted + * initially, before the zone has been fully created; + * allow this mount into global zone. + */ + return (0); + } + + mnt_tsl = mntzone->zone_slabel; + ASSERT(mnt_tsl != NULL); + label_hold(mnt_tsl); + mnt_sl = label2bslabel(mnt_tsl); + + if (strcasecmp(ds_hexsl, ZFS_MLSLABEL_DEFAULT) == 0) { + /* + * The dataset doesn't have a real label, so fabricate one. + */ + char *str = NULL; + + if (l_to_str_internal(mnt_sl, &str) == 0 && + dsl_prop_set_string(osname, + zfs_prop_to_name(ZFS_PROP_MLSLABEL), + ZPROP_SRC_LOCAL, str) == 0) + retv = 0; + if (str != NULL) + kmem_free(str, strlen(str) + 1); + } else if (hexstr_to_label(ds_hexsl, &ds_sl) == 0) { + /* + * Now compare labels to complete the MAC check. If the + * labels are equal then allow access. If the mountpoint + * label dominates the dataset label, allow readonly access. + * Otherwise, access is denied. + */ + if (blequal(mnt_sl, &ds_sl)) + retv = 0; + else if (bldominates(mnt_sl, &ds_sl)) { + vfs_setmntopt(vfsp, MNTOPT_RO, NULL, 0); + retv = 0; + } + } + + label_rele(mnt_tsl); + zone_rele(mntzone); + return (retv); +} +#endif /* SECLABEL */ + +static int +getpoolname(const char *osname, char *poolname) +{ + char *p; + + p = strchr(osname, '/'); + if (p == NULL) { + if (strlen(osname) >= MAXNAMELEN) + return (ENAMETOOLONG); + (void) strcpy(poolname, osname); + } else { + if (p - osname >= MAXNAMELEN) + return (ENAMETOOLONG); + (void) strncpy(poolname, osname, p - osname); + poolname[p - osname] = '\0'; + } + return (0); +} + +/*ARGSUSED*/ +static int +zfs_mount(vfs_t *vfsp) +{ + kthread_t *td = curthread; + vnode_t *mvp = vfsp->mnt_vnodecovered; + cred_t *cr = td->td_ucred; + char *osname; + int error = 0; + int canwrite; + + if (vfs_getopt(vfsp->mnt_optnew, "from", (void **)&osname, NULL)) + return (SET_ERROR(EINVAL)); + + /* + * If full-owner-access is enabled and delegated administration is + * turned on, we must set nosuid. + */ + if (zfs_super_owner && + dsl_deleg_access(osname, ZFS_DELEG_PERM_MOUNT, cr) != ECANCELED) { + secpolicy_fs_mount_clearopts(cr, vfsp); + } + + /* + * Check for mount privilege? + * + * If we don't have privilege then see if + * we have local permission to allow it + */ + error = secpolicy_fs_mount(cr, mvp, vfsp); + if (error) { + if (dsl_deleg_access(osname, ZFS_DELEG_PERM_MOUNT, cr) != 0) + goto out; + + if (!(vfsp->vfs_flag & MS_REMOUNT)) { + vattr_t vattr; + + /* + * Make sure user is the owner of the mount point + * or has sufficient privileges. + */ + + vattr.va_mask = AT_UID; + + vn_lock(mvp, LK_SHARED | LK_RETRY); + if (VOP_GETATTR(mvp, &vattr, cr)) { + VOP_UNLOCK1(mvp); + goto out; + } + + if (secpolicy_vnode_owner(mvp, cr, vattr.va_uid) != 0 && + VOP_ACCESS(mvp, VWRITE, cr, td) != 0) { + VOP_UNLOCK1(mvp); + goto out; + } + VOP_UNLOCK1(mvp); + } + + secpolicy_fs_mount_clearopts(cr, vfsp); + } + + /* + * Refuse to mount a filesystem if we are in a local zone and the + * dataset is not visible. + */ + if (!INGLOBALZONE(curproc) && + (!zone_dataset_visible(osname, &canwrite) || !canwrite)) { + error = SET_ERROR(EPERM); + goto out; + } + +#ifdef SECLABEL + error = zfs_mount_label_policy(vfsp, osname); + if (error) + goto out; +#endif + + vfsp->vfs_flag |= MNT_NFS4ACLS; + + /* + * When doing a remount, we simply refresh our temporary properties + * according to those options set in the current VFS options. + */ + if (vfsp->vfs_flag & MS_REMOUNT) { + zfsvfs_t *zfsvfs = vfsp->vfs_data; + + /* + * Refresh mount options with z_teardown_lock blocking I/O while + * the filesystem is in an inconsistent state. + * The lock also serializes this code with filesystem + * manipulations between entry to zfs_suspend_fs() and return + * from zfs_resume_fs(). + */ + rrm_enter(&zfsvfs->z_teardown_lock, RW_WRITER, FTAG); + zfs_unregister_callbacks(zfsvfs); + error = zfs_register_callbacks(vfsp); + rrm_exit(&zfsvfs->z_teardown_lock, FTAG); + goto out; + } + + /* Initial root mount: try hard to import the requested root pool. */ + if ((vfsp->vfs_flag & MNT_ROOTFS) != 0 && + (vfsp->vfs_flag & MNT_UPDATE) == 0) { + char pname[MAXNAMELEN]; + + error = getpoolname(osname, pname); + if (error == 0) + error = spa_import_rootpool(pname); + if (error) + goto out; + } + DROP_GIANT(); + error = zfs_domount(vfsp, osname); + PICKUP_GIANT(); + +out: + return (error); +} + +static int +zfs_statfs(vfs_t *vfsp, struct statfs *statp) +{ + zfsvfs_t *zfsvfs = vfsp->vfs_data; + uint64_t refdbytes, availbytes, usedobjs, availobjs; + + statp->f_version = STATFS_VERSION; + + ZFS_ENTER(zfsvfs); + + dmu_objset_space(zfsvfs->z_os, + &refdbytes, &availbytes, &usedobjs, &availobjs); + + /* + * The underlying storage pool actually uses multiple block sizes. + * We report the fragsize as the smallest block size we support, + * and we report our blocksize as the filesystem's maximum blocksize. + */ + statp->f_bsize = SPA_MINBLOCKSIZE; + statp->f_iosize = zfsvfs->z_vfs->mnt_stat.f_iosize; + + /* + * The following report "total" blocks of various kinds in the + * file system, but reported in terms of f_frsize - the + * "fragment" size. + */ + + statp->f_blocks = (refdbytes + availbytes) >> SPA_MINBLOCKSHIFT; + statp->f_bfree = availbytes / statp->f_bsize; + statp->f_bavail = statp->f_bfree; /* no root reservation */ + + /* + * statvfs() should really be called statufs(), because it assumes + * static metadata. ZFS doesn't preallocate files, so the best + * we can do is report the max that could possibly fit in f_files, + * and that minus the number actually used in f_ffree. + * For f_ffree, report the smaller of the number of object available + * and the number of blocks (each object will take at least a block). + */ + statp->f_ffree = MIN(availobjs, statp->f_bfree); + statp->f_files = statp->f_ffree + usedobjs; + + /* + * We're a zfs filesystem. + */ + strlcpy(statp->f_fstypename, "zfs", + sizeof (statp->f_fstypename)); + + strlcpy(statp->f_mntfromname, vfsp->mnt_stat.f_mntfromname, + sizeof (statp->f_mntfromname)); + strlcpy(statp->f_mntonname, vfsp->mnt_stat.f_mntonname, + sizeof (statp->f_mntonname)); + + statp->f_namemax = MAXNAMELEN - 1; + + ZFS_EXIT(zfsvfs); + return (0); +} + +static int +zfs_root(vfs_t *vfsp, int flags, vnode_t **vpp) +{ + zfsvfs_t *zfsvfs = vfsp->vfs_data; + znode_t *rootzp; + int error; + + ZFS_ENTER(zfsvfs); + + error = zfs_zget(zfsvfs, zfsvfs->z_root, &rootzp); + if (error == 0) + *vpp = ZTOV(rootzp); + + ZFS_EXIT(zfsvfs); + + if (error == 0) { + error = vn_lock(*vpp, flags); + if (error != 0) { + VN_RELE(*vpp); + *vpp = NULL; + } + } + return (error); +} + +/* + * Teardown the zfsvfs::z_os. + * + * Note, if 'unmounting' is FALSE, we return with the 'z_teardown_lock' + * and 'z_teardown_inactive_lock' held. + */ +static int +zfsvfs_teardown(zfsvfs_t *zfsvfs, boolean_t unmounting) +{ + znode_t *zp; + dsl_dir_t *dd; + + /* + * If someone has not already unmounted this file system, + * drain the zrele_taskq to ensure all active references to the + * zfsvfs_t have been handled only then can it be safely destroyed. + */ + if (zfsvfs->z_os) { + /* + * If we're unmounting we have to wait for the list to + * drain completely. + * + * If we're not unmounting there's no guarantee the list + * will drain completely, but zreles run from the taskq + * may add the parents of dir-based xattrs to the taskq + * so we want to wait for these. + * + * We can safely read z_nr_znodes without locking because the + * VFS has already blocked operations which add to the + * z_all_znodes list and thus increment z_nr_znodes. + */ + int round = 0; + while (zfsvfs->z_nr_znodes > 0) { + taskq_wait_outstanding(dsl_pool_zrele_taskq( + dmu_objset_pool(zfsvfs->z_os)), 0); + if (++round > 1 && !unmounting) + break; + } + } + rrm_enter(&zfsvfs->z_teardown_lock, RW_WRITER, FTAG); + + if (!unmounting) { + /* + * We purge the parent filesystem's vfsp as the parent + * filesystem and all of its snapshots have their vnode's + * v_vfsp set to the parent's filesystem's vfsp. Note, + * 'z_parent' is self referential for non-snapshots. + */ +#ifdef FREEBSD_NAMECACHE + cache_purgevfs(zfsvfs->z_parent->z_vfs, true); +#endif + } + + /* + * Close the zil. NB: Can't close the zil while zfs_inactive + * threads are blocked as zil_close can call zfs_inactive. + */ + if (zfsvfs->z_log) { + zil_close(zfsvfs->z_log); + zfsvfs->z_log = NULL; + } + + rw_enter(&zfsvfs->z_teardown_inactive_lock, RW_WRITER); + + /* + * If we are not unmounting (ie: online recv) and someone already + * unmounted this file system while we were doing the switcheroo, + * or a reopen of z_os failed then just bail out now. + */ + if (!unmounting && (zfsvfs->z_unmounted || zfsvfs->z_os == NULL)) { + rw_exit(&zfsvfs->z_teardown_inactive_lock); + rrm_exit(&zfsvfs->z_teardown_lock, FTAG); + return (SET_ERROR(EIO)); + } + + /* + * At this point there are no vops active, and any new vops will + * fail with EIO since we have z_teardown_lock for writer (only + * relavent for forced unmount). + * + * Release all holds on dbufs. + */ + mutex_enter(&zfsvfs->z_znodes_lock); + for (zp = list_head(&zfsvfs->z_all_znodes); zp != NULL; + zp = list_next(&zfsvfs->z_all_znodes, zp)) + if (zp->z_sa_hdl) { + ASSERT(ZTOV(zp)->v_count >= 0); + zfs_znode_dmu_fini(zp); + } + mutex_exit(&zfsvfs->z_znodes_lock); + + /* + * If we are unmounting, set the unmounted flag and let new vops + * unblock. zfs_inactive will have the unmounted behavior, and all + * other vops will fail with EIO. + */ + if (unmounting) { + zfsvfs->z_unmounted = B_TRUE; + rw_exit(&zfsvfs->z_teardown_inactive_lock); + rrm_exit(&zfsvfs->z_teardown_lock, FTAG); + } + + /* + * z_os will be NULL if there was an error in attempting to reopen + * zfsvfs, so just return as the properties had already been + * unregistered and cached data had been evicted before. + */ + if (zfsvfs->z_os == NULL) + return (0); + + /* + * Unregister properties. + */ + zfs_unregister_callbacks(zfsvfs); + + /* + * Evict cached data + */ + if (!zfs_is_readonly(zfsvfs)) + txg_wait_synced(dmu_objset_pool(zfsvfs->z_os), 0); + dmu_objset_evict_dbufs(zfsvfs->z_os); + dd = zfsvfs->z_os->os_dsl_dataset->ds_dir; + dsl_dir_cancel_waiters(dd); + + return (0); +} + +/*ARGSUSED*/ +static int +zfs_umount(vfs_t *vfsp, int fflag) +{ + kthread_t *td = curthread; + zfsvfs_t *zfsvfs = vfsp->vfs_data; + objset_t *os; + cred_t *cr = td->td_ucred; + int ret; + + ret = secpolicy_fs_unmount(cr, vfsp); + if (ret) { + if (dsl_deleg_access((char *)vfsp->vfs_resource, + ZFS_DELEG_PERM_MOUNT, cr)) + return (ret); + } + + /* + * Unmount any snapshots mounted under .zfs before unmounting the + * dataset itself. + */ + if (zfsvfs->z_ctldir != NULL) { + if ((ret = zfsctl_umount_snapshots(vfsp, fflag, cr)) != 0) + return (ret); + } + + if (fflag & MS_FORCE) { + /* + * Mark file system as unmounted before calling + * vflush(FORCECLOSE). This way we ensure no future vnops + * will be called and risk operating on DOOMED vnodes. + */ + rrm_enter(&zfsvfs->z_teardown_lock, RW_WRITER, FTAG); + zfsvfs->z_unmounted = B_TRUE; + rrm_exit(&zfsvfs->z_teardown_lock, FTAG); + } + + /* + * Flush all the files. + */ + ret = vflush(vfsp, 0, (fflag & MS_FORCE) ? FORCECLOSE : 0, td); + if (ret != 0) + return (ret); + while (taskqueue_cancel(zfsvfs_taskq->tq_queue, + &zfsvfs->z_unlinked_drain_task, NULL) != 0) + taskqueue_drain(zfsvfs_taskq->tq_queue, + &zfsvfs->z_unlinked_drain_task); + + VERIFY(zfsvfs_teardown(zfsvfs, B_TRUE) == 0); + os = zfsvfs->z_os; + + /* + * z_os will be NULL if there was an error in + * attempting to reopen zfsvfs. + */ + if (os != NULL) { + /* + * Unset the objset user_ptr. + */ + mutex_enter(&os->os_user_ptr_lock); + dmu_objset_set_user(os, NULL); + mutex_exit(&os->os_user_ptr_lock); + + /* + * Finally release the objset + */ + dmu_objset_disown(os, B_TRUE, zfsvfs); + } + + /* + * We can now safely destroy the '.zfs' directory node. + */ + if (zfsvfs->z_ctldir != NULL) + zfsctl_destroy(zfsvfs); + zfs_freevfs(vfsp); + + return (0); +} + +static int +zfs_vget(vfs_t *vfsp, ino_t ino, int flags, vnode_t **vpp) +{ + zfsvfs_t *zfsvfs = vfsp->vfs_data; + znode_t *zp; + int err; + + /* + * zfs_zget() can't operate on virtual entries like .zfs/ or + * .zfs/snapshot/ directories, that's why we return EOPNOTSUPP. + * This will make NFS to switch to LOOKUP instead of using VGET. + */ + if (ino == ZFSCTL_INO_ROOT || ino == ZFSCTL_INO_SNAPDIR || + (zfsvfs->z_shares_dir != 0 && ino == zfsvfs->z_shares_dir)) + return (EOPNOTSUPP); + + ZFS_ENTER(zfsvfs); + err = zfs_zget(zfsvfs, ino, &zp); + if (err == 0 && zp->z_unlinked) { + vrele(ZTOV(zp)); + err = EINVAL; + } + if (err == 0) + *vpp = ZTOV(zp); + ZFS_EXIT(zfsvfs); + if (err == 0) { + err = vn_lock(*vpp, flags); + if (err != 0) + vrele(*vpp); + } + if (err != 0) + *vpp = NULL; + return (err); +} + +static int +zfs_checkexp(vfs_t *vfsp, struct sockaddr *nam, int *extflagsp, + struct ucred **credanonp, int *numsecflavors, int **secflavors) +{ + zfsvfs_t *zfsvfs = vfsp->vfs_data; + + /* + * If this is regular file system vfsp is the same as + * zfsvfs->z_parent->z_vfs, but if it is snapshot, + * zfsvfs->z_parent->z_vfs represents parent file system + * which we have to use here, because only this file system + * has mnt_export configured. + */ + return (vfs_stdcheckexp(zfsvfs->z_parent->z_vfs, nam, extflagsp, + credanonp, numsecflavors, secflavors)); +} + +CTASSERT(SHORT_FID_LEN <= sizeof (struct fid)); +CTASSERT(LONG_FID_LEN <= sizeof (struct fid)); + +static int +zfs_fhtovp(vfs_t *vfsp, fid_t *fidp, int flags, vnode_t **vpp) +{ + struct componentname cn; + zfsvfs_t *zfsvfs = vfsp->vfs_data; + znode_t *zp; + vnode_t *dvp; + uint64_t object = 0; + uint64_t fid_gen = 0; + uint64_t gen_mask; + uint64_t zp_gen; + int i, err; + + *vpp = NULL; + + ZFS_ENTER(zfsvfs); + + /* + * On FreeBSD we can get snapshot's mount point or its parent file + * system mount point depending if snapshot is already mounted or not. + */ + if (zfsvfs->z_parent == zfsvfs && fidp->fid_len == LONG_FID_LEN) { + zfid_long_t *zlfid = (zfid_long_t *)fidp; + uint64_t objsetid = 0; + uint64_t setgen = 0; + + for (i = 0; i < sizeof (zlfid->zf_setid); i++) + objsetid |= ((uint64_t)zlfid->zf_setid[i]) << (8 * i); + + for (i = 0; i < sizeof (zlfid->zf_setgen); i++) + setgen |= ((uint64_t)zlfid->zf_setgen[i]) << (8 * i); + + ZFS_EXIT(zfsvfs); + + err = zfsctl_lookup_objset(vfsp, objsetid, &zfsvfs); + if (err) + return (SET_ERROR(EINVAL)); + ZFS_ENTER(zfsvfs); + } + + if (fidp->fid_len == SHORT_FID_LEN || fidp->fid_len == LONG_FID_LEN) { + zfid_short_t *zfid = (zfid_short_t *)fidp; + + for (i = 0; i < sizeof (zfid->zf_object); i++) + object |= ((uint64_t)zfid->zf_object[i]) << (8 * i); + + for (i = 0; i < sizeof (zfid->zf_gen); i++) + fid_gen |= ((uint64_t)zfid->zf_gen[i]) << (8 * i); + } else { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + /* + * A zero fid_gen means we are in .zfs or the .zfs/snapshot + * directory tree. If the object == zfsvfs->z_shares_dir, then + * we are in the .zfs/shares directory tree. + */ + if ((fid_gen == 0 && + (object == ZFSCTL_INO_ROOT || object == ZFSCTL_INO_SNAPDIR)) || + (zfsvfs->z_shares_dir != 0 && object == zfsvfs->z_shares_dir)) { + ZFS_EXIT(zfsvfs); + VERIFY0(zfsctl_root(zfsvfs, LK_SHARED, &dvp)); + if (object == ZFSCTL_INO_SNAPDIR) { + cn.cn_nameptr = "snapshot"; + cn.cn_namelen = strlen(cn.cn_nameptr); + cn.cn_nameiop = LOOKUP; + cn.cn_flags = ISLASTCN | LOCKLEAF; + cn.cn_lkflags = flags; + VERIFY0(VOP_LOOKUP(dvp, vpp, &cn)); + vput(dvp); + } else if (object == zfsvfs->z_shares_dir) { + /* + * XXX This branch must not be taken, + * if it is, then the lookup below will + * explode. + */ + cn.cn_nameptr = "shares"; + cn.cn_namelen = strlen(cn.cn_nameptr); + cn.cn_nameiop = LOOKUP; + cn.cn_flags = ISLASTCN; + cn.cn_lkflags = flags; + VERIFY0(VOP_LOOKUP(dvp, vpp, &cn)); + vput(dvp); + } else { + *vpp = dvp; + } + return (err); + } + + gen_mask = -1ULL >> (64 - 8 * i); + + dprintf("getting %llu [%u mask %llx]\n", object, fid_gen, gen_mask); + if ((err = zfs_zget(zfsvfs, object, &zp))) { + ZFS_EXIT(zfsvfs); + return (err); + } + (void) sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs), &zp_gen, + sizeof (uint64_t)); + zp_gen = zp_gen & gen_mask; + if (zp_gen == 0) + zp_gen = 1; + if (zp->z_unlinked || zp_gen != fid_gen) { + dprintf("znode gen (%u) != fid gen (%u)\n", zp_gen, fid_gen); + vrele(ZTOV(zp)); + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + *vpp = ZTOV(zp); + ZFS_EXIT(zfsvfs); + err = vn_lock(*vpp, flags); + if (err == 0) + vnode_create_vobject(*vpp, zp->z_size, curthread); + else + *vpp = NULL; + return (err); +} + +/* + * Block out VOPs and close zfsvfs_t::z_os + * + * Note, if successful, then we return with the 'z_teardown_lock' and + * 'z_teardown_inactive_lock' write held. We leave ownership of the underlying + * dataset and objset intact so that they can be atomically handed off during + * a subsequent rollback or recv operation and the resume thereafter. + */ +int +zfs_suspend_fs(zfsvfs_t *zfsvfs) +{ + int error; + + if ((error = zfsvfs_teardown(zfsvfs, B_FALSE)) != 0) + return (error); + + return (0); +} + +/* + * Rebuild SA and release VOPs. Note that ownership of the underlying dataset + * is an invariant across any of the operations that can be performed while the + * filesystem was suspended. Whether it succeeded or failed, the preconditions + * are the same: the relevant objset and associated dataset are owned by + * zfsvfs, held, and long held on entry. + */ +int +zfs_resume_fs(zfsvfs_t *zfsvfs, dsl_dataset_t *ds) +{ + int err; + znode_t *zp; + + ASSERT(RRM_WRITE_HELD(&zfsvfs->z_teardown_lock)); + ASSERT(RW_WRITE_HELD(&zfsvfs->z_teardown_inactive_lock)); + + /* + * We already own this, so just update the objset_t, as the one we + * had before may have been evicted. + */ + objset_t *os; + VERIFY3P(ds->ds_owner, ==, zfsvfs); + VERIFY(dsl_dataset_long_held(ds)); + dsl_pool_t *dp = spa_get_dsl(dsl_dataset_get_spa(ds)); + dsl_pool_config_enter(dp, FTAG); + VERIFY0(dmu_objset_from_ds(ds, &os)); + dsl_pool_config_exit(dp, FTAG); + + err = zfsvfs_init(zfsvfs, os); + if (err != 0) + goto bail; + + ds->ds_dir->dd_activity_cancelled = B_FALSE; + VERIFY(zfsvfs_setup(zfsvfs, B_FALSE) == 0); + + zfs_set_fuid_feature(zfsvfs); + + /* + * Attempt to re-establish all the active znodes with + * their dbufs. If a zfs_rezget() fails, then we'll let + * any potential callers discover that via ZFS_ENTER_VERIFY_VP + * when they try to use their znode. + */ + mutex_enter(&zfsvfs->z_znodes_lock); + for (zp = list_head(&zfsvfs->z_all_znodes); zp; + zp = list_next(&zfsvfs->z_all_znodes, zp)) { + (void) zfs_rezget(zp); + } + mutex_exit(&zfsvfs->z_znodes_lock); + +bail: + /* release the VOPs */ + rw_exit(&zfsvfs->z_teardown_inactive_lock); + rrm_exit(&zfsvfs->z_teardown_lock, FTAG); + + if (err) { + /* + * Since we couldn't setup the sa framework, try to force + * unmount this file system. + */ + if (vn_vfswlock(zfsvfs->z_vfs->vfs_vnodecovered) == 0) { + vfs_ref(zfsvfs->z_vfs); + (void) dounmount(zfsvfs->z_vfs, MS_FORCE, curthread); + } + } + return (err); +} + +static void +zfs_freevfs(vfs_t *vfsp) +{ + zfsvfs_t *zfsvfs = vfsp->vfs_data; + + zfsvfs_free(zfsvfs); + + atomic_dec_32(&zfs_active_fs_count); +} + +#ifdef __i386__ +static int desiredvnodes_backup; +#endif + +static void +zfs_vnodes_adjust(void) +{ +#ifdef __i386__ + int newdesiredvnodes; + + desiredvnodes_backup = desiredvnodes; + + /* + * We calculate newdesiredvnodes the same way it is done in + * vntblinit(). If it is equal to desiredvnodes, it means that + * it wasn't tuned by the administrator and we can tune it down. + */ + newdesiredvnodes = min(maxproc + vm_cnt.v_page_count / 4, 2 * + vm_kmem_size / (5 * (sizeof (struct vm_object) + + sizeof (struct vnode)))); + if (newdesiredvnodes == desiredvnodes) + desiredvnodes = (3 * newdesiredvnodes) / 4; +#endif +} + +static void +zfs_vnodes_adjust_back(void) +{ + +#ifdef __i386__ + desiredvnodes = desiredvnodes_backup; +#endif +} + +void +zfs_init(void) +{ + + printf("ZFS filesystem version: " ZPL_VERSION_STRING "\n"); + + /* + * Initialize .zfs directory structures + */ + zfsctl_init(); + + /* + * Initialize znode cache, vnode ops, etc... + */ + zfs_znode_init(); + + /* + * Reduce number of vnodes. Originally number of vnodes is calculated + * with UFS inode in mind. We reduce it here, because it's too big for + * ZFS/i386. + */ + zfs_vnodes_adjust(); + + dmu_objset_register_type(DMU_OST_ZFS, zfs_space_delta_cb); + + zfsvfs_taskq = taskq_create("zfsvfs", 1, minclsyspri, 0, 0, 0); +} + +void +zfs_fini(void) +{ + taskq_destroy(zfsvfs_taskq); + zfsctl_fini(); + zfs_znode_fini(); + zfs_vnodes_adjust_back(); +} + +int +zfs_busy(void) +{ + return (zfs_active_fs_count != 0); +} + +/* + * Release VOPs and unmount a suspended filesystem. + */ +int +zfs_end_fs(zfsvfs_t *zfsvfs, dsl_dataset_t *ds) +{ + ASSERT(RRM_WRITE_HELD(&zfsvfs->z_teardown_lock)); + ASSERT(RW_WRITE_HELD(&zfsvfs->z_teardown_inactive_lock)); + + /* + * We already own this, so just hold and rele it to update the + * objset_t, as the one we had before may have been evicted. + */ + objset_t *os; + VERIFY3P(ds->ds_owner, ==, zfsvfs); + VERIFY(dsl_dataset_long_held(ds)); + dsl_pool_t *dp = spa_get_dsl(dsl_dataset_get_spa(ds)); + dsl_pool_config_enter(dp, FTAG); + VERIFY0(dmu_objset_from_ds(ds, &os)); + dsl_pool_config_exit(dp, FTAG); + zfsvfs->z_os = os; + + /* release the VOPs */ + rw_exit(&zfsvfs->z_teardown_inactive_lock); + rrm_exit(&zfsvfs->z_teardown_lock, FTAG); + + /* + * Try to force unmount this file system. + */ + (void) zfs_umount(zfsvfs->z_vfs, 0); + zfsvfs->z_unmounted = B_TRUE; + return (0); +} + +int +zfs_set_version(zfsvfs_t *zfsvfs, uint64_t newvers) +{ + int error; + objset_t *os = zfsvfs->z_os; + dmu_tx_t *tx; + + if (newvers < ZPL_VERSION_INITIAL || newvers > ZPL_VERSION) + return (SET_ERROR(EINVAL)); + + if (newvers < zfsvfs->z_version) + return (SET_ERROR(EINVAL)); + + if (zfs_spa_version_map(newvers) > + spa_version(dmu_objset_spa(zfsvfs->z_os))) + return (SET_ERROR(ENOTSUP)); + + tx = dmu_tx_create(os); + dmu_tx_hold_zap(tx, MASTER_NODE_OBJ, B_FALSE, ZPL_VERSION_STR); + if (newvers >= ZPL_VERSION_SA && !zfsvfs->z_use_sa) { + dmu_tx_hold_zap(tx, MASTER_NODE_OBJ, B_TRUE, + ZFS_SA_ATTRS); + dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL); + } + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + return (error); + } + + error = zap_update(os, MASTER_NODE_OBJ, ZPL_VERSION_STR, + 8, 1, &newvers, tx); + + if (error) { + dmu_tx_commit(tx); + return (error); + } + + if (newvers >= ZPL_VERSION_SA && !zfsvfs->z_use_sa) { + uint64_t sa_obj; + + ASSERT3U(spa_version(dmu_objset_spa(zfsvfs->z_os)), >=, + SPA_VERSION_SA); + sa_obj = zap_create(os, DMU_OT_SA_MASTER_NODE, + DMU_OT_NONE, 0, tx); + + error = zap_add(os, MASTER_NODE_OBJ, + ZFS_SA_ATTRS, 8, 1, &sa_obj, tx); + ASSERT0(error); + + VERIFY(0 == sa_set_sa_object(os, sa_obj)); + sa_register_update_callback(os, zfs_sa_upgrade); + } + + spa_history_log_internal_ds(dmu_objset_ds(os), "upgrade", tx, + "from %lu to %lu", zfsvfs->z_version, newvers); + + dmu_tx_commit(tx); + + zfsvfs->z_version = newvers; + os->os_version = newvers; + + zfs_set_fuid_feature(zfsvfs); + + return (0); +} + +/* + * Read a property stored within the master node. + */ +int +zfs_get_zplprop(objset_t *os, zfs_prop_t prop, uint64_t *value) +{ + uint64_t *cached_copy = NULL; + + /* + * Figure out where in the objset_t the cached copy would live, if it + * is available for the requested property. + */ + if (os != NULL) { + switch (prop) { + case ZFS_PROP_VERSION: + cached_copy = &os->os_version; + break; + case ZFS_PROP_NORMALIZE: + cached_copy = &os->os_normalization; + break; + case ZFS_PROP_UTF8ONLY: + cached_copy = &os->os_utf8only; + break; + case ZFS_PROP_CASE: + cached_copy = &os->os_casesensitivity; + break; + default: + break; + } + } + if (cached_copy != NULL && *cached_copy != OBJSET_PROP_UNINITIALIZED) { + *value = *cached_copy; + return (0); + } + + /* + * If the property wasn't cached, look up the file system's value for + * the property. For the version property, we look up a slightly + * different string. + */ + const char *pname; + int error = ENOENT; + if (prop == ZFS_PROP_VERSION) { + pname = ZPL_VERSION_STR; + } else { + pname = zfs_prop_to_name(prop); + } + + if (os != NULL) { + ASSERT3U(os->os_phys->os_type, ==, DMU_OST_ZFS); + error = zap_lookup(os, MASTER_NODE_OBJ, pname, 8, 1, value); + } + + if (error == ENOENT) { + /* No value set, use the default value */ + switch (prop) { + case ZFS_PROP_VERSION: + *value = ZPL_VERSION; + break; + case ZFS_PROP_NORMALIZE: + case ZFS_PROP_UTF8ONLY: + *value = 0; + break; + case ZFS_PROP_CASE: + *value = ZFS_CASE_SENSITIVE; + break; + default: + return (error); + } + error = 0; + } + + /* + * If one of the methods for getting the property value above worked, + * copy it into the objset_t's cache. + */ + if (error == 0 && cached_copy != NULL) { + *cached_copy = *value; + } + + return (error); +} + +/* + * Return true if the coresponding vfs's unmounted flag is set. + * Otherwise return false. + * If this function returns true we know VFS unmount has been initiated. + */ +boolean_t +zfs_get_vfs_flag_unmounted(objset_t *os) +{ + zfsvfs_t *zfvp; + boolean_t unmounted = B_FALSE; + + ASSERT(dmu_objset_type(os) == DMU_OST_ZFS); + + mutex_enter(&os->os_user_ptr_lock); + zfvp = dmu_objset_get_user(os); + if (zfvp != NULL && zfvp->z_vfs != NULL && + (zfvp->z_vfs->mnt_kern_flag & MNTK_UNMOUNT)) + unmounted = B_TRUE; + mutex_exit(&os->os_user_ptr_lock); + + return (unmounted); +} + +#ifdef _KERNEL +void +zfsvfs_update_fromname(const char *oldname, const char *newname) +{ + char tmpbuf[MAXPATHLEN]; + struct mount *mp; + char *fromname; + size_t oldlen; + + oldlen = strlen(oldname); + + mtx_lock(&mountlist_mtx); + TAILQ_FOREACH(mp, &mountlist, mnt_list) { + fromname = mp->mnt_stat.f_mntfromname; + if (strcmp(fromname, oldname) == 0) { + (void) strlcpy(fromname, newname, + sizeof (mp->mnt_stat.f_mntfromname)); + continue; + } + if (strncmp(fromname, oldname, oldlen) == 0 && + (fromname[oldlen] == '/' || fromname[oldlen] == '@')) { + (void) snprintf(tmpbuf, sizeof (tmpbuf), "%s%s", + newname, fromname + oldlen); + (void) strlcpy(fromname, tmpbuf, + sizeof (mp->mnt_stat.f_mntfromname)); + continue; + } + } + mtx_unlock(&mountlist_mtx); +} +#endif diff --git a/module/os/freebsd/zfs/zfs_vnops.c b/module/os/freebsd/zfs/zfs_vnops.c new file mode 100644 index 00000000000..d7b92035f7c --- /dev/null +++ b/module/os/freebsd/zfs/zfs_vnops.c @@ -0,0 +1,6533 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. + * Copyright (c) 2014 Integros [integros.com] + * Copyright 2017 Nexenta Systems, Inc. + */ + +/* Portions Copyright 2007 Jeremy Teo */ +/* Portions Copyright 2010 Robert Milkowski */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#ifndef VN_OPEN_INVFS +#define VN_OPEN_INVFS 0x0 +#endif + +#if __FreeBSD_version >= 1300047 +#define vm_page_wire_lock(pp) +#define vm_page_wire_unlock(pp) +#else +#define vm_page_wire_lock(pp) vm_page_lock(pp) +#define vm_page_wire_unlock(pp) vm_page_unlock(pp) +#endif + +static int +zfs_u8_validate(const char *u8str, size_t n, char **list, int flag, int *errnum) +{ + + return (u8_validate(__DECONST(char *, u8str), n, list, flag, errnum)); +} +#define u8_validate zfs_u8_validate + +#ifdef DEBUG_VFS_LOCKS +#define VNCHECKREF(vp) \ + VNASSERT((vp)->v_holdcnt > 0 && (vp)->v_usecount > 0, vp, \ + ("%s: wrong ref counts", __func__)); +#else +#define VNCHECKREF(vp) +#endif + +/* + * Programming rules. + * + * Each vnode op performs some logical unit of work. To do this, the ZPL must + * properly lock its in-core state, create a DMU transaction, do the work, + * record this work in the intent log (ZIL), commit the DMU transaction, + * and wait for the intent log to commit if it is a synchronous operation. + * Moreover, the vnode ops must work in both normal and log replay context. + * The ordering of events is important to avoid deadlocks and references + * to freed memory. The example below illustrates the following Big Rules: + * + * (1) A check must be made in each zfs thread for a mounted file system. + * This is done avoiding races using ZFS_ENTER(zfsvfs). + * A ZFS_EXIT(zfsvfs) is needed before all returns. Any znodes + * must be checked with ZFS_VERIFY_ZP(zp). Both of these macros + * can return EIO from the calling function. + * + * (2) VN_RELE() should always be the last thing except for zil_commit() + * (if necessary) and ZFS_EXIT(). This is for 3 reasons: + * First, if it's the last reference, the vnode/znode + * can be freed, so the zp may point to freed memory. Second, the last + * reference will call zfs_zinactive(), which may induce a lot of work -- + * pushing cached pages (which acquires range locks) and syncing out + * cached atime changes. Third, zfs_zinactive() may require a new tx, + * which could deadlock the system if you were already holding one. + * If you must call VN_RELE() within a tx then use VN_RELE_ASYNC(). + * + * (3) All range locks must be grabbed before calling dmu_tx_assign(), + * as they can span dmu_tx_assign() calls. + * + * (4) If ZPL locks are held, pass TXG_NOWAIT as the second argument to + * dmu_tx_assign(). This is critical because we don't want to block + * while holding locks. + * + * If no ZPL locks are held (aside from ZFS_ENTER()), use TXG_WAIT. This + * reduces lock contention and CPU usage when we must wait (note that if + * throughput is constrained by the storage, nearly every transaction + * must wait). + * + * Note, in particular, that if a lock is sometimes acquired before + * the tx assigns, and sometimes after (e.g. z_lock), then failing + * to use a non-blocking assign can deadlock the system. The scenario: + * + * Thread A has grabbed a lock before calling dmu_tx_assign(). + * Thread B is in an already-assigned tx, and blocks for this lock. + * Thread A calls dmu_tx_assign(TXG_WAIT) and blocks in txg_wait_open() + * forever, because the previous txg can't quiesce until B's tx commits. + * + * If dmu_tx_assign() returns ERESTART and zfsvfs->z_assign is TXG_NOWAIT, + * then drop all locks, call dmu_tx_wait(), and try again. On subsequent + * calls to dmu_tx_assign(), pass TXG_NOTHROTTLE in addition to TXG_NOWAIT, + * to indicate that this operation has already called dmu_tx_wait(). + * This will ensure that we don't retry forever, waiting a short bit + * each time. + * + * (5) If the operation succeeded, generate the intent log entry for it + * before dropping locks. This ensures that the ordering of events + * in the intent log matches the order in which they actually occurred. + * During ZIL replay the zfs_log_* functions will update the sequence + * number to indicate the zil transaction has replayed. + * + * (6) At the end of each vnode op, the DMU tx must always commit, + * regardless of whether there were any errors. + * + * (7) After dropping all locks, invoke zil_commit(zilog, foid) + * to ensure that synchronous semantics are provided when necessary. + * + * In general, this is how things should be ordered in each vnode op: + * + * ZFS_ENTER(zfsvfs); // exit if unmounted + * top: + * zfs_dirent_lookup(&dl, ...) // lock directory entry (may VN_HOLD()) + * rw_enter(...); // grab any other locks you need + * tx = dmu_tx_create(...); // get DMU tx + * dmu_tx_hold_*(); // hold each object you might modify + * error = dmu_tx_assign(tx, (waited ? TXG_NOTHROTTLE : 0) | TXG_NOWAIT); + * if (error) { + * rw_exit(...); // drop locks + * zfs_dirent_unlock(dl); // unlock directory entry + * VN_RELE(...); // release held vnodes + * if (error == ERESTART) { + * waited = B_TRUE; + * dmu_tx_wait(tx); + * dmu_tx_abort(tx); + * goto top; + * } + * dmu_tx_abort(tx); // abort DMU tx + * ZFS_EXIT(zfsvfs); // finished in zfs + * return (error); // really out of space + * } + * error = do_real_work(); // do whatever this VOP does + * if (error == 0) + * zfs_log_*(...); // on success, make ZIL entry + * dmu_tx_commit(tx); // commit DMU tx -- error or not + * rw_exit(...); // drop locks + * zfs_dirent_unlock(dl); // unlock directory entry + * VN_RELE(...); // release held vnodes + * zil_commit(zilog, foid); // synchronous when necessary + * ZFS_EXIT(zfsvfs); // finished in zfs + * return (error); // done, report error + */ + +/* ARGSUSED */ +static int +zfs_open(vnode_t **vpp, int flag, cred_t *cr) +{ + znode_t *zp = VTOZ(*vpp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + if ((flag & FWRITE) && (zp->z_pflags & ZFS_APPENDONLY) && + ((flag & FAPPEND) == 0)) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EPERM)); + } + + if (!zfs_has_ctldir(zp) && zp->z_zfsvfs->z_vscan && + ZTOV(zp)->v_type == VREG && + !(zp->z_pflags & ZFS_AV_QUARANTINED) && zp->z_size > 0) { + if (fs_vscan(*vpp, cr, 0) != 0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EACCES)); + } + } + + /* Keep a count of the synchronous opens in the znode */ + if (flag & (FSYNC | FDSYNC)) + atomic_inc_32(&zp->z_sync_cnt); + + ZFS_EXIT(zfsvfs); + return (0); +} + +/* ARGSUSED */ +static int +zfs_close(vnode_t *vp, int flag, int count, offset_t offset, cred_t *cr) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + /* Decrement the synchronous opens in the znode */ + if ((flag & (FSYNC | FDSYNC)) && (count == 1)) + atomic_dec_32(&zp->z_sync_cnt); + + if (!zfs_has_ctldir(zp) && zp->z_zfsvfs->z_vscan && + ZTOV(zp)->v_type == VREG && + !(zp->z_pflags & ZFS_AV_QUARANTINED) && zp->z_size > 0) + VERIFY(fs_vscan(vp, cr, 1) == 0); + + ZFS_EXIT(zfsvfs); + return (0); +} + +/* + * Lseek support for finding holes (cmd == _FIO_SEEK_HOLE) and + * data (cmd == _FIO_SEEK_DATA). "off" is an in/out parameter. + */ +static int +zfs_holey(vnode_t *vp, ulong_t cmd, offset_t *off) +{ + znode_t *zp = VTOZ(vp); + uint64_t noff = (uint64_t)*off; /* new offset */ + uint64_t file_sz; + int error; + boolean_t hole; + + file_sz = zp->z_size; + if (noff >= file_sz) { + return (SET_ERROR(ENXIO)); + } + + if (cmd == _FIO_SEEK_HOLE) + hole = B_TRUE; + else + hole = B_FALSE; + + error = dmu_offset_next(zp->z_zfsvfs->z_os, zp->z_id, hole, &noff); + + if (error == ESRCH) + return (SET_ERROR(ENXIO)); + + /* file was dirty, so fall back to using generic logic */ + if (error == EBUSY) { + if (hole) + *off = file_sz; + + return (0); + } + + /* + * We could find a hole that begins after the logical end-of-file, + * because dmu_offset_next() only works on whole blocks. If the + * EOF falls mid-block, then indicate that the "virtual hole" + * at the end of the file begins at the logical EOF, rather than + * at the end of the last block. + */ + if (noff > file_sz) { + ASSERT(hole); + noff = file_sz; + } + + if (noff < *off) + return (error); + *off = noff; + return (error); +} + +/* ARGSUSED */ +static int +zfs_ioctl(vnode_t *vp, ulong_t com, intptr_t data, int flag, cred_t *cred, + int *rvalp) +{ + offset_t off; + int error; + zfsvfs_t *zfsvfs; + znode_t *zp; + + switch (com) { + case _FIOFFS: + { + return (0); + + /* + * The following two ioctls are used by bfu. Faking out, + * necessary to avoid bfu errors. + */ + } + case _FIOGDIO: + case _FIOSDIO: + { + return (0); + } + + case _FIO_SEEK_DATA: + case _FIO_SEEK_HOLE: + { + off = *(offset_t *)data; + zp = VTOZ(vp); + zfsvfs = zp->z_zfsvfs; + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + /* offset parameter is in/out */ + error = zfs_holey(vp, com, &off); + ZFS_EXIT(zfsvfs); + if (error) + return (error); + *(offset_t *)data = off; + return (0); + } + } + return (SET_ERROR(ENOTTY)); +} + +static vm_page_t +page_busy(vnode_t *vp, int64_t start, int64_t off, int64_t nbytes) +{ + vm_object_t obj; + vm_page_t pp; + int64_t end; + + /* + * At present vm_page_clear_dirty extends the cleared range to DEV_BSIZE + * aligned boundaries, if the range is not aligned. As a result a + * DEV_BSIZE subrange with partially dirty data may get marked as clean. + * It may happen that all DEV_BSIZE subranges are marked clean and thus + * the whole page would be considred clean despite have some dirty data. + * For this reason we should shrink the range to DEV_BSIZE aligned + * boundaries before calling vm_page_clear_dirty. + */ + end = rounddown2(off + nbytes, DEV_BSIZE); + off = roundup2(off, DEV_BSIZE); + nbytes = end - off; + + obj = vp->v_object; + zfs_vmobject_assert_wlocked(obj); +#if __FreeBSD_version < 1300050 + for (;;) { + if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL && + pp->valid) { + if (vm_page_xbusied(pp)) { + /* + * Reference the page before unlocking and + * sleeping so that the page daemon is less + * likely to reclaim it. + */ + vm_page_reference(pp); + vm_page_lock(pp); + zfs_vmobject_wunlock(obj); + vm_page_busy_sleep(pp, "zfsmwb", true); + zfs_vmobject_wlock(obj); + continue; + } + vm_page_sbusy(pp); + } else if (pp != NULL) { + ASSERT(!pp->valid); + pp = NULL; + } + if (pp != NULL) { + ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL); + vm_object_pip_add(obj, 1); + pmap_remove_write(pp); + if (nbytes != 0) + vm_page_clear_dirty(pp, off, nbytes); + } + break; + } +#else + vm_page_grab_valid(&pp, obj, OFF_TO_IDX(start), VM_ALLOC_NOCREAT | + VM_ALLOC_SBUSY | VM_ALLOC_NORMAL | VM_ALLOC_IGN_SBUSY); + if (pp != NULL) { + ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL); + vm_object_pip_add(obj, 1); + pmap_remove_write(pp); + if (nbytes != 0) + vm_page_clear_dirty(pp, off, nbytes); + } +#endif + return (pp); +} + +static void +page_unbusy(vm_page_t pp) +{ + + vm_page_sunbusy(pp); +#if __FreeBSD_version >= 1300041 + vm_object_pip_wakeup(pp->object); +#else + vm_object_pip_subtract(pp->object, 1); +#endif +} + +#if __FreeBSD_version > 1300051 +static vm_page_t +page_hold(vnode_t *vp, int64_t start) +{ + vm_object_t obj; + vm_page_t m; + + obj = vp->v_object; + zfs_vmobject_assert_wlocked(obj); + + vm_page_grab_valid(&m, obj, OFF_TO_IDX(start), VM_ALLOC_NOCREAT | + VM_ALLOC_WIRED | VM_ALLOC_IGN_SBUSY | VM_ALLOC_NOBUSY); + return (m); +} +#else +static vm_page_t +page_hold(vnode_t *vp, int64_t start) +{ + vm_object_t obj; + vm_page_t pp; + + obj = vp->v_object; + zfs_vmobject_assert_wlocked(obj); + + for (;;) { + if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL && + pp->valid) { + if (vm_page_xbusied(pp)) { + /* + * Reference the page before unlocking and + * sleeping so that the page daemon is less + * likely to reclaim it. + */ + vm_page_reference(pp); + vm_page_lock(pp); + zfs_vmobject_wunlock(obj); + vm_page_busy_sleep(pp, "zfsmwb", true); + zfs_vmobject_wlock(obj); + continue; + } + + ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL); + vm_page_wire_lock(pp); + vm_page_hold(pp); + vm_page_wire_unlock(pp); + + } else + pp = NULL; + break; + } + return (pp); +} +#endif + +static void +page_unhold(vm_page_t pp) +{ + + vm_page_wire_lock(pp); +#if __FreeBSD_version >= 1300035 + vm_page_unwire(pp, PQ_ACTIVE); +#else + vm_page_unhold(pp); +#endif + vm_page_wire_unlock(pp); +} + +/* + * When a file is memory mapped, we must keep the IO data synchronized + * between the DMU cache and the memory mapped pages. What this means: + * + * On Write: If we find a memory mapped page, we write to *both* + * the page and the dmu buffer. + */ +static void +update_pages(vnode_t *vp, int64_t start, int len, objset_t *os, uint64_t oid, + int segflg, dmu_tx_t *tx) +{ + vm_object_t obj; + struct sf_buf *sf; + caddr_t va; + int off; + + ASSERT(segflg != UIO_NOCOPY); + ASSERT(vp->v_mount != NULL); + obj = vp->v_object; + ASSERT(obj != NULL); + + off = start & PAGEOFFSET; + zfs_vmobject_wlock(obj); +#if __FreeBSD_version >= 1300041 + vm_object_pip_add(obj, 1); +#endif + for (start &= PAGEMASK; len > 0; start += PAGESIZE) { + vm_page_t pp; + int nbytes = imin(PAGESIZE - off, len); + + if ((pp = page_busy(vp, start, off, nbytes)) != NULL) { + zfs_vmobject_wunlock(obj); + + va = zfs_map_page(pp, &sf); + (void) dmu_read(os, oid, start+off, nbytes, + va+off, DMU_READ_PREFETCH); + zfs_unmap_page(sf); + + zfs_vmobject_wlock(obj); + page_unbusy(pp); + } + len -= nbytes; + off = 0; + } +#if __FreeBSD_version >= 1300041 + vm_object_pip_wakeup(obj); +#else + vm_object_pip_wakeupn(obj, 0); +#endif + zfs_vmobject_wunlock(obj); +} + +/* + * Read with UIO_NOCOPY flag means that sendfile(2) requests + * ZFS to populate a range of page cache pages with data. + * + * NOTE: this function could be optimized to pre-allocate + * all pages in advance, drain exclusive busy on all of them, + * map them into contiguous KVA region and populate them + * in one single dmu_read() call. + */ +static int +mappedread_sf(vnode_t *vp, int nbytes, uio_t *uio) +{ + znode_t *zp = VTOZ(vp); + objset_t *os = zp->z_zfsvfs->z_os; + struct sf_buf *sf; + vm_object_t obj; + vm_page_t pp; + int64_t start; + caddr_t va; + int len = nbytes; + int error = 0; + + ASSERT(uio->uio_segflg == UIO_NOCOPY); + ASSERT(vp->v_mount != NULL); + obj = vp->v_object; + ASSERT(obj != NULL); + ASSERT((uio->uio_loffset & PAGEOFFSET) == 0); + + zfs_vmobject_wlock(obj); + for (start = uio->uio_loffset; len > 0; start += PAGESIZE) { + int bytes = MIN(PAGESIZE, len); + + pp = vm_page_grab(obj, OFF_TO_IDX(start), VM_ALLOC_SBUSY | + VM_ALLOC_NORMAL | VM_ALLOC_IGN_SBUSY); + if (vm_page_none_valid(pp)) { + zfs_vmobject_wunlock(obj); + va = zfs_map_page(pp, &sf); + error = dmu_read(os, zp->z_id, start, bytes, va, + DMU_READ_PREFETCH); + if (bytes != PAGESIZE && error == 0) + bzero(va + bytes, PAGESIZE - bytes); + zfs_unmap_page(sf); + zfs_vmobject_wlock(obj); + vm_page_do_sunbusy(pp); +#if __FreeBSD_version >= 1300047 && __FreeBSD_version < 1300051 +#error "unsupported version window" +#elif __FreeBSD_version >= 1300051 + if (error == 0) { + vm_page_valid(pp); + vm_page_lock(pp); + vm_page_activate(pp); + vm_page_unlock(pp); + } + vm_page_do_sunbusy(pp); + if (error != 0 && !vm_page_wired(pp) == 0 && + pp->valid == 0 && vm_page_tryxbusy(pp)) + vm_page_free(pp); +#else + vm_page_lock(pp); + if (error) { + if (pp->wire_count == 0 && pp->valid == 0 && + !vm_page_busied(pp)) + vm_page_free(pp); + } else { + pp->valid = VM_PAGE_BITS_ALL; + vm_page_activate(pp); + } + vm_page_unlock(pp); +#endif + } else { + ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL); + vm_page_do_sunbusy(pp); + } + if (error) + break; + uio->uio_resid -= bytes; + uio->uio_offset += bytes; + len -= bytes; + } + zfs_vmobject_wunlock(obj); + return (error); +} + +/* + * When a file is memory mapped, we must keep the IO data synchronized + * between the DMU cache and the memory mapped pages. What this means: + * + * On Read: We "read" preferentially from memory mapped pages, + * else we default from the dmu buffer. + * + * NOTE: We will always "break up" the IO into PAGESIZE uiomoves when + * the file is memory mapped. + */ +static int +mappedread(vnode_t *vp, int nbytes, uio_t *uio) +{ + znode_t *zp = VTOZ(vp); + vm_object_t obj; + int64_t start; + int len = nbytes; + int off; + int error = 0; + + ASSERT(vp->v_mount != NULL); + obj = vp->v_object; + ASSERT(obj != NULL); + + start = uio->uio_loffset; + off = start & PAGEOFFSET; + zfs_vmobject_wlock(obj); + for (start &= PAGEMASK; len > 0; start += PAGESIZE) { + vm_page_t pp; + uint64_t bytes = MIN(PAGESIZE - off, len); + + if ((pp = page_hold(vp, start))) { + struct sf_buf *sf; + caddr_t va; + + zfs_vmobject_wunlock(obj); + va = zfs_map_page(pp, &sf); + error = vn_io_fault_uiomove(va + off, bytes, uio); + zfs_unmap_page(sf); + zfs_vmobject_wlock(obj); + page_unhold(pp); + } else { + zfs_vmobject_wunlock(obj); + error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl), + uio, bytes); + zfs_vmobject_wlock(obj); + } + len -= bytes; + off = 0; + if (error) + break; + } + zfs_vmobject_wunlock(obj); + return (error); +} + +offset_t zfs_read_chunk_size = 1024 * 1024; /* Tunable */ + +/* + * Read bytes from specified file into supplied buffer. + * + * IN: vp - vnode of file to be read from. + * uio - structure supplying read location, range info, + * and return buffer. + * ioflag - SYNC flags; used to provide FRSYNC semantics. + * cr - credentials of caller. + * ct - caller context + * + * OUT: uio - updated offset and range, buffer filled. + * + * RETURN: 0 on success, error code on failure. + * + * Side Effects: + * vp - atime updated if byte count > 0 + */ +/* ARGSUSED */ +static int +zfs_read(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + ssize_t n, nbytes; + int error = 0; + zfs_locked_range_t *lr; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + if (zp->z_pflags & ZFS_AV_QUARANTINED) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EACCES)); + } + + /* + * Validate file offset + */ + if (uio->uio_loffset < (offset_t)0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + /* + * Fasttrack empty reads + */ + if (uio->uio_resid == 0) { + ZFS_EXIT(zfsvfs); + return (0); + } + + /* + * If we're in FRSYNC mode, sync out this znode before reading it. + */ + if (zfsvfs->z_log && + (ioflag & FRSYNC || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)) + zil_commit(zfsvfs->z_log, zp->z_id); + + /* + * Lock the range against changes. + */ + lr = zfs_rangelock_enter(&zp->z_rangelock, uio->uio_loffset, + uio->uio_resid, RL_READER); + + /* + * If we are reading past end-of-file we can skip + * to the end; but we might still need to set atime. + */ + if (uio->uio_loffset >= zp->z_size) { + error = 0; + goto out; + } + + ASSERT(uio->uio_loffset < zp->z_size); + n = MIN(uio->uio_resid, zp->z_size - uio->uio_loffset); + + while (n > 0) { + nbytes = MIN(n, zfs_read_chunk_size - + P2PHASE(uio->uio_loffset, zfs_read_chunk_size)); + + if (uio->uio_segflg == UIO_NOCOPY) + error = mappedread_sf(vp, nbytes, uio); + else if (vn_has_cached_data(vp)) { + error = mappedread(vp, nbytes, uio); + } else { + error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl), + uio, nbytes); + } + if (error) { + /* convert checksum errors into IO errors */ + if (error == ECKSUM) + error = SET_ERROR(EIO); + break; + } + + n -= nbytes; + } +out: + zfs_rangelock_exit(lr); + + ZFS_ACCESSTIME_STAMP(zfsvfs, zp); + ZFS_EXIT(zfsvfs); + return (error); +} + +/* + * Write the bytes to a file. + * + * IN: vp - vnode of file to be written to. + * uio - structure supplying write location, range info, + * and data buffer. + * ioflag - FAPPEND, FSYNC, and/or FDSYNC. FAPPEND is + * set if in append mode. + * cr - credentials of caller. + * ct - caller context (NFS/CIFS fem monitor only) + * + * OUT: uio - updated offset and range. + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * vp - ctime|mtime updated if byte count > 0 + */ + +/* ARGSUSED */ +static int +zfs_write(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr) +{ + znode_t *zp = VTOZ(vp); + rlim64_t limit = MAXOFFSET_T; + ssize_t start_resid = uio->uio_resid; + ssize_t tx_bytes; + uint64_t end_size; + dmu_buf_impl_t *db; + dmu_tx_t *tx; + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + zilog_t *zilog; + offset_t woff; + ssize_t n, nbytes; + zfs_locked_range_t *lr; + int max_blksz = zfsvfs->z_max_blksz; + int error = 0; + arc_buf_t *abuf; + iovec_t *aiov = NULL; + xuio_t *xuio = NULL; + int i_iov = 0; + int iovcnt __unused = uio->uio_iovcnt; + iovec_t *iovp = uio->uio_iov; + int write_eof; + int count = 0; + sa_bulk_attr_t bulk[4]; + uint64_t mtime[2], ctime[2]; + uint64_t uid, gid, projid; + + /* + * Fasttrack empty write + */ + n = start_resid; + if (n == 0) + return (0); + + if (limit == RLIM64_INFINITY || limit > MAXOFFSET_T) + limit = MAXOFFSET_T; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL, + &zp->z_size, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, + &zp->z_pflags, 8); + + /* + * Callers might not be able to detect properly that we are read-only, + * so check it explicitly here. + */ + if (zfs_is_readonly(zfsvfs)) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EROFS)); + } + + /* + * If immutable or not appending then return EPERM. + * Intentionally allow ZFS_READONLY through here. + * See zfs_zaccess_common() + */ + if ((zp->z_pflags & ZFS_IMMUTABLE) || + ((zp->z_pflags & ZFS_APPENDONLY) && !(ioflag & FAPPEND) && + (uio->uio_loffset < zp->z_size))) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EPERM)); + } + + zilog = zfsvfs->z_log; + + /* + * Validate file offset + */ + woff = ioflag & FAPPEND ? zp->z_size : uio->uio_loffset; + if (woff < 0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + /* + * If in append mode, set the io offset pointer to eof. + */ + if (ioflag & FAPPEND) { + /* + * Obtain an appending range lock to guarantee file append + * semantics. We reset the write offset once we have the lock. + */ + lr = zfs_rangelock_enter(&zp->z_rangelock, 0, n, RL_APPEND); + woff = lr->lr_offset; + if (lr->lr_length == UINT64_MAX) { + /* + * We overlocked the file because this write will cause + * the file block size to increase. + * Note that zp_size cannot change with this lock held. + */ + woff = zp->z_size; + } + uio->uio_loffset = woff; + } else { + /* + * Note that if the file block size will change as a result of + * this write, then this range lock will lock the entire file + * so that we can re-write the block safely. + */ + lr = zfs_rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER); + } + + if (vn_rlimit_fsize(vp, uio, uio->uio_td)) { + zfs_rangelock_exit(lr); + ZFS_EXIT(zfsvfs); + return (EFBIG); + } + + if (woff >= limit) { + zfs_rangelock_exit(lr); + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EFBIG)); + } + + if ((woff + n) > limit || woff > (limit - n)) + n = limit - woff; + + /* Will this write extend the file length? */ + write_eof = (woff + n > zp->z_size); + + end_size = MAX(zp->z_size, woff + n); + + uid = zp->z_uid; + gid = zp->z_gid; + projid = zp->z_projid; + + /* + * Write the file in reasonable size chunks. Each chunk is written + * in a separate transaction; this keeps the intent log records small + * and allows us to do more fine-grained space accounting. + */ + while (n > 0) { + woff = uio->uio_loffset; + + if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, uid) || + zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, gid) || + (projid != ZFS_DEFAULT_PROJID && + zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT, + projid))) { + error = SET_ERROR(EDQUOT); + break; + } + + abuf = NULL; + if (xuio) { + ASSERT(i_iov < iovcnt); + aiov = &iovp[i_iov]; + abuf = dmu_xuio_arcbuf(xuio, i_iov); + dmu_xuio_clear(xuio, i_iov); + DTRACE_PROBE3(zfs_cp_write, int, i_iov, + iovec_t *, aiov, arc_buf_t *, abuf); + ASSERT((aiov->iov_base == abuf->b_data) || + ((char *)aiov->iov_base - (char *)abuf->b_data + + aiov->iov_len == arc_buf_size(abuf))); + i_iov++; + } else if (n >= max_blksz && + woff >= zp->z_size && + P2PHASE(woff, max_blksz) == 0 && + zp->z_blksz == max_blksz) { + /* + * This write covers a full block. "Borrow" a buffer + * from the dmu so that we can fill it before we enter + * a transaction. This avoids the possibility of + * holding up the transaction if the data copy hangs + * up on a pagefault (e.g., from an NFS server mapping). + */ + size_t cbytes; + + abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl), + max_blksz); + ASSERT(abuf != NULL); + ASSERT(arc_buf_size(abuf) == max_blksz); + if ((error = uiocopy(abuf->b_data, max_blksz, + UIO_WRITE, uio, &cbytes))) { + dmu_return_arcbuf(abuf); + break; + } + ASSERT(cbytes == max_blksz); + } + + /* + * Start a transaction. + */ + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl); + DB_DNODE_ENTER(db); + dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff, + MIN(n, max_blksz)); + DB_DNODE_EXIT(db); + zfs_sa_upgrade_txholds(tx, zp); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + if (abuf != NULL) + dmu_return_arcbuf(abuf); + break; + } + + /* + * If zfs_range_lock() over-locked we grow the blocksize + * and then reduce the lock range. This will only happen + * on the first iteration since zfs_range_reduce() will + * shrink down r_len to the appropriate size. + */ + if (lr->lr_length == UINT64_MAX) { + uint64_t new_blksz; + + if (zp->z_blksz > max_blksz) { + /* + * File's blocksize is already larger than the + * "recordsize" property. Only let it grow to + * the next power of 2. + */ + ASSERT(!ISP2(zp->z_blksz)); + new_blksz = MIN(end_size, + 1 << highbit64(zp->z_blksz)); + } else { + new_blksz = MIN(end_size, max_blksz); + } + zfs_grow_blocksize(zp, new_blksz, tx); + zfs_rangelock_reduce(lr, woff, n); + } + + /* + * XXX - should we really limit each write to z_max_blksz? + * Perhaps we should use SPA_MAXBLOCKSIZE chunks? + */ + nbytes = MIN(n, max_blksz - P2PHASE(woff, max_blksz)); + + if (woff + nbytes > zp->z_size) + vnode_pager_setsize(vp, woff + nbytes); + + if (abuf == NULL) { + tx_bytes = uio->uio_resid; + error = dmu_write_uio_dbuf(sa_get_db(zp->z_sa_hdl), + uio, nbytes, tx); + tx_bytes -= uio->uio_resid; + } else { + tx_bytes = nbytes; + ASSERT(xuio == NULL || tx_bytes == aiov->iov_len); + /* + * If this is not a full block write, but we are + * extending the file past EOF and this data starts + * block-aligned, use assign_arcbuf(). Otherwise, + * write via dmu_write(). + */ + if (tx_bytes < max_blksz && (!write_eof || + aiov->iov_base != abuf->b_data)) { + ASSERT(xuio); + dmu_write(zfsvfs->z_os, zp->z_id, woff, + aiov->iov_len, aiov->iov_base, tx); + dmu_return_arcbuf(abuf); + xuio_stat_wbuf_copied(); + } else { + ASSERT(xuio || tx_bytes == max_blksz); + dmu_assign_arcbuf(sa_get_db(zp->z_sa_hdl), woff, + abuf, tx); + } + ASSERT(tx_bytes <= uio->uio_resid); + uioskip(uio, tx_bytes); + } + if (tx_bytes && vn_has_cached_data(vp)) { + update_pages(vp, woff, tx_bytes, zfsvfs->z_os, + zp->z_id, uio->uio_segflg, tx); + } + + /* + * If we made no progress, we're done. If we made even + * partial progress, update the znode and ZIL accordingly. + */ + if (tx_bytes == 0) { + (void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs), + (void *)&zp->z_size, sizeof (uint64_t), tx); + dmu_tx_commit(tx); + ASSERT(error != 0); + break; + } + + /* + * Clear Set-UID/Set-GID bits on successful write if not + * privileged and at least one of the excute bits is set. + * + * It would be nice to to this after all writes have + * been done, but that would still expose the ISUID/ISGID + * to another app after the partial write is committed. + * + * Note: we don't call zfs_fuid_map_id() here because + * user 0 is not an ephemeral uid. + */ + mutex_enter(&zp->z_acl_lock); + if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) | + (S_IXUSR >> 6))) != 0 && + (zp->z_mode & (S_ISUID | S_ISGID)) != 0 && + secpolicy_vnode_setid_retain(vp, cr, + (zp->z_mode & S_ISUID) != 0 && zp->z_uid == 0) != 0) { + uint64_t newmode; + zp->z_mode &= ~(S_ISUID | S_ISGID); + newmode = zp->z_mode; + (void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs), + (void *)&newmode, sizeof (uint64_t), tx); + } + mutex_exit(&zp->z_acl_lock); + + zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); + + /* + * Update the file size (zp_size) if it has changed; + * account for possible concurrent updates. + */ + while ((end_size = zp->z_size) < uio->uio_loffset) { + (void) atomic_cas_64(&zp->z_size, end_size, + uio->uio_loffset); + ASSERT(error == 0 || error == EFAULT); + } + /* + * If we are replaying and eof is non zero then force + * the file size to the specified eof. Note, there's no + * concurrency during replay. + */ + if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0) + zp->z_size = zfsvfs->z_replay_eof; + + if (error == 0) + error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); + else + (void) sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); + + zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, + ioflag, NULL, NULL); + dmu_tx_commit(tx); + + if (error != 0) + break; + ASSERT(tx_bytes == nbytes); + n -= nbytes; + + } + + zfs_rangelock_exit(lr); + + /* + * If we're in replay mode, or we made no progress, return error. + * Otherwise, it's at least a partial write, so it's successful. + */ + if (zfsvfs->z_replay || uio->uio_resid == start_resid) { + ZFS_EXIT(zfsvfs); + return (error); + } + + /* + * EFAULT means that at least one page of the source buffer was not + * available. VFS will re-try remaining I/O upon this error. + */ + if (error == EFAULT) { + ZFS_EXIT(zfsvfs); + return (error); + } + + if (ioflag & (FSYNC | FDSYNC) || + zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zilog, zp->z_id); + + ZFS_EXIT(zfsvfs); + return (0); +} + +int +zfs_write_simple(znode_t *zp, const void *data, size_t len, + loff_t pos, size_t *presid) +{ + int error = 0; + ssize_t resid; + + error = vn_rdwr(UIO_WRITE, ZTOV(zp), __DECONST(void *, data), len, pos, + UIO_SYSSPACE, IO_SYNC, kcred, NOCRED, &resid, curthread); + + if (error) { + return (SET_ERROR(error)); + } else if (presid == NULL) { + if (resid != 0) { + error = SET_ERROR(EIO); + } + } else { + *presid = resid; + } + return (error); +} + +void +zfs_get_done(zgd_t *zgd, int error) +{ + znode_t *zp = zgd->zgd_private; + objset_t *os = zp->z_zfsvfs->z_os; + + if (zgd->zgd_db) + dmu_buf_rele(zgd->zgd_db, zgd); + + zfs_rangelock_exit(zgd->zgd_lr); + + /* + * Release the vnode asynchronously as we currently have the + * txg stopped from syncing. + */ + VN_RELE_ASYNC(ZTOV(zp), dsl_pool_zrele_taskq(dmu_objset_pool(os))); + + kmem_free(zgd, sizeof (zgd_t)); +} + +#ifdef DEBUG +static int zil_fault_io = 0; +#endif + +/* + * Get data to generate a TX_WRITE intent log record. + */ +int +zfs_get_data(void *arg, lr_write_t *lr, char *buf, struct lwb *lwb, zio_t *zio) +{ + zfsvfs_t *zfsvfs = arg; + objset_t *os = zfsvfs->z_os; + znode_t *zp; + uint64_t object = lr->lr_foid; + uint64_t offset = lr->lr_offset; + uint64_t size = lr->lr_length; + dmu_buf_t *db; + zgd_t *zgd; + int error = 0; + + ASSERT3P(lwb, !=, NULL); + ASSERT3P(zio, !=, NULL); + ASSERT3U(size, !=, 0); + + /* + * Nothing to do if the file has been removed + */ + if (zfs_zget(zfsvfs, object, &zp) != 0) + return (SET_ERROR(ENOENT)); + if (zp->z_unlinked) { + /* + * Release the vnode asynchronously as we currently have the + * txg stopped from syncing. + */ + VN_RELE_ASYNC(ZTOV(zp), + dsl_pool_zrele_taskq(dmu_objset_pool(os))); + return (SET_ERROR(ENOENT)); + } + + zgd = (zgd_t *)kmem_zalloc(sizeof (zgd_t), KM_SLEEP); + zgd->zgd_lwb = lwb; + zgd->zgd_private = zp; + + /* + * Write records come in two flavors: immediate and indirect. + * For small writes it's cheaper to store the data with the + * log record (immediate); for large writes it's cheaper to + * sync the data and get a pointer to it (indirect) so that + * we don't have to write the data twice. + */ + if (buf != NULL) { /* immediate write */ + zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock, offset, + size, RL_READER); + /* test for truncation needs to be done while range locked */ + if (offset >= zp->z_size) { + error = SET_ERROR(ENOENT); + } else { + error = dmu_read(os, object, offset, size, buf, + DMU_READ_NO_PREFETCH); + } + ASSERT(error == 0 || error == ENOENT); + } else { /* indirect write */ + /* + * Have to lock the whole block to ensure when it's + * written out and its checksum is being calculated + * that no one can change the data. We need to re-check + * blocksize after we get the lock in case it's changed! + */ + for (;;) { + uint64_t blkoff; + size = zp->z_blksz; + blkoff = ISP2(size) ? P2PHASE(offset, size) : offset; + offset -= blkoff; + zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock, + offset, size, RL_READER); + if (zp->z_blksz == size) + break; + offset += blkoff; + zfs_rangelock_exit(zgd->zgd_lr); + } + /* test for truncation needs to be done while range locked */ + if (lr->lr_offset >= zp->z_size) + error = SET_ERROR(ENOENT); +#ifdef DEBUG + if (zil_fault_io) { + error = SET_ERROR(EIO); + zil_fault_io = 0; + } +#endif + if (error == 0) + error = dmu_buf_hold(os, object, offset, zgd, &db, + DMU_READ_NO_PREFETCH); + + if (error == 0) { + blkptr_t *bp = &lr->lr_blkptr; + + zgd->zgd_db = db; + zgd->zgd_bp = bp; + + ASSERT(db->db_offset == offset); + ASSERT(db->db_size == size); + + error = dmu_sync(zio, lr->lr_common.lrc_txg, + zfs_get_done, zgd); + ASSERT(error || lr->lr_length <= size); + + /* + * On success, we need to wait for the write I/O + * initiated by dmu_sync() to complete before we can + * release this dbuf. We will finish everything up + * in the zfs_get_done() callback. + */ + if (error == 0) + return (0); + + if (error == EALREADY) { + lr->lr_common.lrc_txtype = TX_WRITE2; + /* + * TX_WRITE2 relies on the data previously + * written by the TX_WRITE that caused + * EALREADY. We zero out the BP because + * it is the old, currently-on-disk BP, + * so there's no need to zio_flush() its + * vdevs (flushing would needlesly hurt + * performance, and doesn't work on + * indirect vdevs). + */ + zgd->zgd_bp = NULL; + BP_ZERO(bp); + error = 0; + } + } + } + + zfs_get_done(zgd, error); + + return (error); +} + +/*ARGSUSED*/ +static int +zfs_access(vnode_t *vp, int mode, int flag, cred_t *cr, + caller_context_t *ct) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + int error; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + if (flag & V_ACE_MASK) + error = zfs_zaccess(zp, mode, flag, B_FALSE, cr); + else + error = zfs_zaccess_rwx(zp, mode, flag, cr); + + ZFS_EXIT(zfsvfs); + return (error); +} + +static int +zfs_dd_callback(struct mount *mp, void *arg, int lkflags, struct vnode **vpp) +{ + int error; + + *vpp = arg; + error = vn_lock(*vpp, lkflags); + if (error != 0) + vrele(*vpp); + return (error); +} + +static int +zfs_lookup_lock(vnode_t *dvp, vnode_t *vp, const char *name, int lkflags) +{ + znode_t *zdp = VTOZ(dvp); + zfsvfs_t *zfsvfs __unused = zdp->z_zfsvfs; + int error; + int ltype; + + if (zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_LOCKED(dvp, __func__); +#ifdef DIAGNOSTIC + if ((zdp->z_pflags & ZFS_XATTR) == 0) + VERIFY(!RRM_LOCK_HELD(&zfsvfs->z_teardown_lock)); +#endif + + if (name[0] == 0 || (name[0] == '.' && name[1] == 0)) { + ASSERT3P(dvp, ==, vp); + vref(dvp); + ltype = lkflags & LK_TYPE_MASK; + if (ltype != VOP_ISLOCKED(dvp)) { + if (ltype == LK_EXCLUSIVE) + vn_lock(dvp, LK_UPGRADE | LK_RETRY); + else /* if (ltype == LK_SHARED) */ + vn_lock(dvp, LK_DOWNGRADE | LK_RETRY); + + /* + * Relock for the "." case could leave us with + * reclaimed vnode. + */ + if (VN_IS_DOOMED(dvp)) { + vrele(dvp); + return (SET_ERROR(ENOENT)); + } + } + return (0); + } else if (name[0] == '.' && name[1] == '.' && name[2] == 0) { + /* + * Note that in this case, dvp is the child vnode, and we + * are looking up the parent vnode - exactly reverse from + * normal operation. Unlocking dvp requires some rather + * tricky unlock/relock dance to prevent mp from being freed; + * use vn_vget_ino_gen() which takes care of all that. + * + * XXX Note that there is a time window when both vnodes are + * unlocked. It is possible, although highly unlikely, that + * during that window the parent-child relationship between + * the vnodes may change, for example, get reversed. + * In that case we would have a wrong lock order for the vnodes. + * All other filesystems seem to ignore this problem, so we + * do the same here. + * A potential solution could be implemented as follows: + * - using LK_NOWAIT when locking the second vnode and retrying + * if necessary + * - checking that the parent-child relationship still holds + * after locking both vnodes and retrying if it doesn't + */ + error = vn_vget_ino_gen(dvp, zfs_dd_callback, vp, lkflags, &vp); + return (error); + } else { + error = vn_lock(vp, lkflags); + if (error != 0) + vrele(vp); + return (error); + } +} + +/* + * Lookup an entry in a directory, or an extended attribute directory. + * If it exists, return a held vnode reference for it. + * + * IN: dvp - vnode of directory to search. + * nm - name of entry to lookup. + * pnp - full pathname to lookup [UNUSED]. + * flags - LOOKUP_XATTR set if looking for an attribute. + * rdir - root directory vnode [UNUSED]. + * cr - credentials of caller. + * ct - caller context + * + * OUT: vpp - vnode of located entry, NULL if not found. + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * NA + */ +/* ARGSUSED */ +static int +zfs_lookup(vnode_t *dvp, char *nm, vnode_t **vpp, struct componentname *cnp, + int nameiop, cred_t *cr, kthread_t *td, int flags, boolean_t cached) +{ + znode_t *zdp = VTOZ(dvp); + znode_t *zp; + zfsvfs_t *zfsvfs = zdp->z_zfsvfs; + int error = 0; + + /* + * Fast path lookup, however we must skip DNLC lookup + * for case folding or normalizing lookups because the + * DNLC code only stores the passed in name. This means + * creating 'a' and removing 'A' on a case insensitive + * file system would work, but DNLC still thinks 'a' + * exists and won't let you create it again on the next + * pass through fast path. + */ + if (!(flags & LOOKUP_XATTR)) { + if (dvp->v_type != VDIR) { + return (SET_ERROR(ENOTDIR)); + } else if (zdp->z_sa_hdl == NULL) { + return (SET_ERROR(EIO)); + } + } + + DTRACE_PROBE2(zfs__fastpath__lookup__miss, vnode_t *, dvp, char *, nm); + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zdp); + + *vpp = NULL; + + if (flags & LOOKUP_XATTR) { + /* + * If the xattr property is off, refuse the lookup request. + */ + if (!(zfsvfs->z_flags & ZSB_XATTR)) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EOPNOTSUPP)); + } + + /* + * We don't allow recursive attributes.. + * Maybe someday we will. + */ + if (zdp->z_pflags & ZFS_XATTR) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + if ((error = zfs_get_xattrdir(VTOZ(dvp), &zp, cr, flags))) { + ZFS_EXIT(zfsvfs); + return (error); + } + *vpp = ZTOV(zp); + + /* + * Do we have permission to get into attribute directory? + */ + error = zfs_zaccess(zp, ACE_EXECUTE, 0, B_FALSE, cr); + if (error) { + vrele(ZTOV(zp)); + } + + ZFS_EXIT(zfsvfs); + return (error); + } + + /* + * Check accessibility of directory if we're not coming in via + * VOP_CACHEDLOOKUP. + */ + if (!cached) { +#ifdef NOEXECCHECK + if ((cnp->cn_flags & NOEXECCHECK) != 0) { + cnp->cn_flags &= ~NOEXECCHECK; + } else +#endif + if ((error = zfs_zaccess(zdp, ACE_EXECUTE, 0, B_FALSE, cr))) { + ZFS_EXIT(zfsvfs); + return (error); + } + } + + if (zfsvfs->z_utf8 && u8_validate(nm, strlen(nm), + NULL, U8_VALIDATE_ENTIRE, &error) < 0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EILSEQ)); + } + + + /* + * First handle the special cases. + */ + if ((cnp->cn_flags & ISDOTDOT) != 0) { + /* + * If we are a snapshot mounted under .zfs, return + * the vp for the snapshot directory. + */ + if (zdp->z_id == zfsvfs->z_root && zfsvfs->z_parent != zfsvfs) { + struct componentname cn; + vnode_t *zfsctl_vp; + int ltype; + + ZFS_EXIT(zfsvfs); + ltype = VOP_ISLOCKED(dvp); + VOP_UNLOCK1(dvp); + error = zfsctl_root(zfsvfs->z_parent, LK_SHARED, + &zfsctl_vp); + if (error == 0) { + cn.cn_nameptr = "snapshot"; + cn.cn_namelen = strlen(cn.cn_nameptr); + cn.cn_nameiop = cnp->cn_nameiop; + cn.cn_flags = cnp->cn_flags & ~ISDOTDOT; + cn.cn_lkflags = cnp->cn_lkflags; + error = VOP_LOOKUP(zfsctl_vp, vpp, &cn); + vput(zfsctl_vp); + } + vn_lock(dvp, ltype | LK_RETRY); + return (error); + } + } + if (zfs_has_ctldir(zdp) && strcmp(nm, ZFS_CTLDIR_NAME) == 0) { + ZFS_EXIT(zfsvfs); + if ((cnp->cn_flags & ISLASTCN) != 0 && nameiop != LOOKUP) + return (SET_ERROR(ENOTSUP)); + error = zfsctl_root(zfsvfs, cnp->cn_lkflags, vpp); + return (error); + } + + /* + * The loop is retry the lookup if the parent-child relationship + * changes during the dot-dot locking complexities. + */ + for (;;) { + uint64_t parent; + + error = zfs_dirlook(zdp, nm, &zp); + if (error == 0) + *vpp = ZTOV(zp); + + ZFS_EXIT(zfsvfs); + if (error != 0) + break; + + error = zfs_lookup_lock(dvp, *vpp, nm, cnp->cn_lkflags); + if (error != 0) { + /* + * If we've got a locking error, then the vnode + * got reclaimed because of a force unmount. + * We never enter doomed vnodes into the name cache. + */ + *vpp = NULL; + return (error); + } + + if ((cnp->cn_flags & ISDOTDOT) == 0) + break; + + ZFS_ENTER(zfsvfs); + if (zdp->z_sa_hdl == NULL) { + error = SET_ERROR(EIO); + } else { + error = sa_lookup(zdp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs), + &parent, sizeof (parent)); + } + if (error != 0) { + ZFS_EXIT(zfsvfs); + vput(ZTOV(zp)); + break; + } + if (zp->z_id == parent) { + ZFS_EXIT(zfsvfs); + break; + } + vput(ZTOV(zp)); + } + + if (error != 0) + *vpp = NULL; + + /* Translate errors and add SAVENAME when needed. */ + if (cnp->cn_flags & ISLASTCN) { + switch (nameiop) { + case CREATE: + case RENAME: + if (error == ENOENT) { + error = EJUSTRETURN; + cnp->cn_flags |= SAVENAME; + break; + } + /* FALLTHROUGH */ + case DELETE: + if (error == 0) + cnp->cn_flags |= SAVENAME; + break; + } + } + + /* Insert name into cache (as non-existent) if appropriate. */ + if (zfsvfs->z_use_namecache && !zfsvfs->z_replay && + error == ENOENT && (cnp->cn_flags & MAKEENTRY) != 0) + cache_enter(dvp, NULL, cnp); + + /* Insert name into cache if appropriate. */ + if (zfsvfs->z_use_namecache && !zfsvfs->z_replay && + error == 0 && (cnp->cn_flags & MAKEENTRY)) { + if (!(cnp->cn_flags & ISLASTCN) || + (nameiop != DELETE && nameiop != RENAME)) { + cache_enter(dvp, *vpp, cnp); + } + } + + return (error); +} + +/* + * Attempt to create a new entry in a directory. If the entry + * already exists, truncate the file if permissible, else return + * an error. Return the vp of the created or trunc'd file. + * + * IN: dvp - vnode of directory to put new file entry in. + * name - name of new file entry. + * vap - attributes of new file. + * excl - flag indicating exclusive or non-exclusive mode. + * mode - mode to open file with. + * cr - credentials of caller. + * flag - large file flag [UNUSED]. + * ct - caller context + * vsecp - ACL to be set + * + * OUT: vpp - vnode of created or trunc'd entry. + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * dvp - ctime|mtime updated if new entry created + * vp - ctime|mtime always, atime if new + */ + +/* ARGSUSED */ +int +zfs_create(znode_t *dzp, char *name, vattr_t *vap, int excl, int mode, + znode_t **zpp, cred_t *cr, int flag, vsecattr_t *vsecp) +{ + znode_t *zp; + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + zilog_t *zilog; + objset_t *os; + dmu_tx_t *tx; + int error; + ksid_t *ksid; + uid_t uid; + gid_t gid = crgetgid(cr); + uint64_t projid = ZFS_DEFAULT_PROJID; + zfs_acl_ids_t acl_ids; + boolean_t fuid_dirtied; + uint64_t txtype; +#ifdef DEBUG_VFS_LOCKS + vnode_t *dvp = ZTOV(dzp); +#endif + + /* + * If we have an ephemeral id, ACL, or XVATTR then + * make sure file system is at proper version + */ + + ksid = crgetsid(cr, KSID_OWNER); + if (ksid) + uid = ksid_getid(ksid); + else + uid = crgetuid(cr); + + if (zfsvfs->z_use_fuids == B_FALSE && + (vsecp || (vap->va_mask & AT_XVATTR) || + IS_EPHEMERAL(uid) || IS_EPHEMERAL(gid))) + return (SET_ERROR(EINVAL)); + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(dzp); + os = zfsvfs->z_os; + zilog = zfsvfs->z_log; + + if (zfsvfs->z_utf8 && u8_validate(name, strlen(name), + NULL, U8_VALIDATE_ENTIRE, &error) < 0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EILSEQ)); + } + + if (vap->va_mask & AT_XVATTR) { + if ((error = secpolicy_xvattr(ZTOV(dzp), (xvattr_t *)vap, + crgetuid(cr), cr, vap->va_type)) != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + } + + *zpp = NULL; + + if ((vap->va_mode & S_ISVTX) && secpolicy_vnode_stky_modify(cr)) + vap->va_mode &= ~S_ISVTX; + + error = zfs_dirent_lookup(dzp, name, &zp, ZNEW); + if (error) { + ZFS_EXIT(zfsvfs); + return (error); + } + ASSERT3P(zp, ==, NULL); + + /* + * Create a new file object and update the directory + * to reference it. + */ + if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, B_FALSE, cr))) { + goto out; + } + + /* + * We only support the creation of regular files in + * extended attribute directories. + */ + + if ((dzp->z_pflags & ZFS_XATTR) && + (vap->va_type != VREG)) { + error = SET_ERROR(EINVAL); + goto out; + } + + if ((error = zfs_acl_ids_create(dzp, 0, vap, + cr, vsecp, &acl_ids)) != 0) + goto out; + + if (S_ISREG(vap->va_mode) || S_ISDIR(vap->va_mode)) + projid = zfs_inherit_projid(dzp); + if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, projid)) { + zfs_acl_ids_free(&acl_ids); + error = SET_ERROR(EDQUOT); + goto out; + } + + getnewvnode_reserve_(); + + tx = dmu_tx_create(os); + + dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes + + ZFS_SA_BASE_ATTR_SIZE); + + fuid_dirtied = zfsvfs->z_fuid_dirty; + if (fuid_dirtied) + zfs_fuid_txhold(zfsvfs, tx); + dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE); + if (!zfsvfs->z_use_sa && + acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) { + dmu_tx_hold_write(tx, DMU_NEW_OBJECT, + 0, acl_ids.z_aclp->z_acl_bytes); + } + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + zfs_acl_ids_free(&acl_ids); + dmu_tx_abort(tx); + getnewvnode_drop_reserve(); + ZFS_EXIT(zfsvfs); + return (error); + } + zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids); + if (fuid_dirtied) + zfs_fuid_sync(zfsvfs, tx); + + (void) zfs_link_create(dzp, name, zp, tx, ZNEW); + txtype = zfs_log_create_txtype(Z_FILE, vsecp, vap); + zfs_log_create(zilog, tx, txtype, dzp, zp, name, + vsecp, acl_ids.z_fuidp, vap); + zfs_acl_ids_free(&acl_ids); + dmu_tx_commit(tx); + + getnewvnode_drop_reserve(); + +out: + VNCHECKREF(dvp); + if (error == 0) { + *zpp = zp; + } + + if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zilog, 0); + + ZFS_EXIT(zfsvfs); + return (error); +} + +/* + * Remove an entry from a directory. + * + * IN: dvp - vnode of directory to remove entry from. + * name - name of entry to remove. + * cr - credentials of caller. + * ct - caller context + * flags - case flags + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * dvp - ctime|mtime + * vp - ctime (if nlink > 0) + */ + +/*ARGSUSED*/ +static int +zfs_remove_(vnode_t *dvp, vnode_t *vp, char *name, cred_t *cr) +{ + znode_t *dzp = VTOZ(dvp); + znode_t *zp; + znode_t *xzp; + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + zilog_t *zilog; + uint64_t xattr_obj; + uint64_t obj = 0; + dmu_tx_t *tx; + boolean_t unlinked; + uint64_t txtype; + int error; + + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(dzp); + zp = VTOZ(vp); + ZFS_VERIFY_ZP(zp); + zilog = zfsvfs->z_log; + + xattr_obj = 0; + xzp = NULL; + + if ((error = zfs_zaccess_delete(dzp, zp, cr))) { + goto out; + } + + /* + * Need to use rmdir for removing directories. + */ + if (vp->v_type == VDIR) { + error = SET_ERROR(EPERM); + goto out; + } + + vnevent_remove(vp, dvp, name, ct); + + obj = zp->z_id; + + /* are there any extended attributes? */ + error = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), + &xattr_obj, sizeof (xattr_obj)); + if (error == 0 && xattr_obj) { + error = zfs_zget(zfsvfs, xattr_obj, &xzp); + ASSERT0(error); + } + + /* + * We may delete the znode now, or we may put it in the unlinked set; + * it depends on whether we're the last link, and on whether there are + * other holds on the vnode. So we dmu_tx_hold() the right things to + * allow for either case. + */ + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + zfs_sa_upgrade_txholds(tx, zp); + zfs_sa_upgrade_txholds(tx, dzp); + + if (xzp) { + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE); + dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE); + } + + /* charge as an update -- would be nice not to charge at all */ + dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL); + + /* + * Mark this transaction as typically resulting in a net free of space + */ + dmu_tx_mark_netfree(tx); + + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + ZFS_EXIT(zfsvfs); + return (error); + } + + /* + * Remove the directory entry. + */ + error = zfs_link_destroy(dzp, name, zp, tx, ZEXISTS, &unlinked); + + if (error) { + dmu_tx_commit(tx); + goto out; + } + + if (unlinked) { + zfs_unlinked_add(zp, tx); + vp->v_vflag |= VV_NOSYNC; + } + /* XXX check changes to linux vnops */ + txtype = TX_REMOVE; + zfs_log_remove(zilog, tx, txtype, dzp, name, obj, unlinked); + + dmu_tx_commit(tx); +out: + + if (xzp) + vrele(ZTOV(xzp)); + + if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zilog, 0); + + + ZFS_EXIT(zfsvfs); + return (error); +} + + +int +zfs_lookup_internal(znode_t *dzp, char *name, vnode_t **vpp, + struct componentname *cnp, int nameiop) +{ + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + int error; + + cnp->cn_nameptr = name; + cnp->cn_namelen = strlen(name); + cnp->cn_nameiop = nameiop; + cnp->cn_flags = ISLASTCN | SAVENAME; + cnp->cn_lkflags = LK_EXCLUSIVE | LK_RETRY; + cnp->cn_cred = kcred; + cnp->cn_thread = curthread; + + if (zfsvfs->z_use_namecache && !zfsvfs->z_replay) { + struct vop_lookup_args a; + + a.a_gen.a_desc = &vop_lookup_desc; + a.a_dvp = ZTOV(dzp); + a.a_vpp = vpp; + a.a_cnp = cnp; + error = vfs_cache_lookup(&a); + } else { + error = zfs_lookup(ZTOV(dzp), name, vpp, cnp, nameiop, kcred, + curthread, 0, B_FALSE); + } +#ifdef ZFS_DEBUG + if (error) { + printf("got error %d on name %s on op %d\n", error, name, + nameiop); + kdb_backtrace(); + } +#endif + return (error); +} + +int +zfs_remove(znode_t *dzp, char *name, cred_t *cr, int flags) +{ + vnode_t *vp; + int error; + struct componentname cn; + + if ((error = zfs_lookup_internal(dzp, name, &vp, &cn, DELETE))) + return (error); + + error = zfs_remove_(ZTOV(dzp), vp, name, cr); + vput(vp); + return (error); +} +/* + * Create a new directory and insert it into dvp using the name + * provided. Return a pointer to the inserted directory. + * + * IN: dvp - vnode of directory to add subdir to. + * dirname - name of new directory. + * vap - attributes of new directory. + * cr - credentials of caller. + * ct - caller context + * flags - case flags + * vsecp - ACL to be set + * + * OUT: vpp - vnode of created directory. + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * dvp - ctime|mtime updated + * vp - ctime|mtime|atime updated + */ +/*ARGSUSED*/ +int +zfs_mkdir(znode_t *dzp, char *dirname, vattr_t *vap, znode_t **zpp, cred_t *cr, + int flags, vsecattr_t *vsecp) +{ + znode_t *zp; + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + zilog_t *zilog; + uint64_t txtype; + dmu_tx_t *tx; + int error; + ksid_t *ksid; + uid_t uid; + gid_t gid = crgetgid(cr); + zfs_acl_ids_t acl_ids; + boolean_t fuid_dirtied; + + ASSERT(vap->va_type == VDIR); + + /* + * If we have an ephemeral id, ACL, or XVATTR then + * make sure file system is at proper version + */ + + ksid = crgetsid(cr, KSID_OWNER); + if (ksid) + uid = ksid_getid(ksid); + else + uid = crgetuid(cr); + if (zfsvfs->z_use_fuids == B_FALSE && + ((vap->va_mask & AT_XVATTR) || + IS_EPHEMERAL(uid) || IS_EPHEMERAL(gid))) + return (SET_ERROR(EINVAL)); + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(dzp); + zilog = zfsvfs->z_log; + + if (dzp->z_pflags & ZFS_XATTR) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + if (zfsvfs->z_utf8 && u8_validate(dirname, + strlen(dirname), NULL, U8_VALIDATE_ENTIRE, &error) < 0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EILSEQ)); + } + + if (vap->va_mask & AT_XVATTR) { + if ((error = secpolicy_xvattr(ZTOV(dzp), (xvattr_t *)vap, + crgetuid(cr), cr, vap->va_type)) != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + } + + if ((error = zfs_acl_ids_create(dzp, 0, vap, cr, + NULL, &acl_ids)) != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + + /* + * First make sure the new directory doesn't exist. + * + * Existence is checked first to make sure we don't return + * EACCES instead of EEXIST which can cause some applications + * to fail. + */ + *zpp = NULL; + + if ((error = zfs_dirent_lookup(dzp, dirname, &zp, ZNEW))) { + zfs_acl_ids_free(&acl_ids); + ZFS_EXIT(zfsvfs); + return (error); + } + ASSERT3P(zp, ==, NULL); + + if ((error = zfs_zaccess(dzp, ACE_ADD_SUBDIRECTORY, 0, B_FALSE, cr))) { + zfs_acl_ids_free(&acl_ids); + ZFS_EXIT(zfsvfs); + return (error); + } + + if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, zfs_inherit_projid(dzp))) { + zfs_acl_ids_free(&acl_ids); + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EDQUOT)); + } + + /* + * Add a new entry to the directory. + */ + getnewvnode_reserve_(); + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_zap(tx, dzp->z_id, TRUE, dirname); + dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL); + fuid_dirtied = zfsvfs->z_fuid_dirty; + if (fuid_dirtied) + zfs_fuid_txhold(zfsvfs, tx); + if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) { + dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, + acl_ids.z_aclp->z_acl_bytes); + } + + dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes + + ZFS_SA_BASE_ATTR_SIZE); + + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + zfs_acl_ids_free(&acl_ids); + dmu_tx_abort(tx); + getnewvnode_drop_reserve(); + ZFS_EXIT(zfsvfs); + return (error); + } + + /* + * Create new node. + */ + zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids); + + if (fuid_dirtied) + zfs_fuid_sync(zfsvfs, tx); + + /* + * Now put new name in parent dir. + */ + (void) zfs_link_create(dzp, dirname, zp, tx, ZNEW); + + *zpp = zp; + + txtype = zfs_log_create_txtype(Z_DIR, NULL, vap); + zfs_log_create(zilog, tx, txtype, dzp, zp, dirname, NULL, + acl_ids.z_fuidp, vap); + + zfs_acl_ids_free(&acl_ids); + + dmu_tx_commit(tx); + + getnewvnode_drop_reserve(); + + if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zilog, 0); + + ZFS_EXIT(zfsvfs); + return (0); +} + +/* + * Remove a directory subdir entry. If the current working + * directory is the same as the subdir to be removed, the + * remove will fail. + * + * IN: dvp - vnode of directory to remove from. + * name - name of directory to be removed. + * cwd - vnode of current working directory. + * cr - credentials of caller. + * ct - caller context + * flags - case flags + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * dvp - ctime|mtime updated + */ +/*ARGSUSED*/ +static int +zfs_rmdir_(vnode_t *dvp, vnode_t *vp, char *name, cred_t *cr) +{ + znode_t *dzp = VTOZ(dvp); + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + zilog_t *zilog; + dmu_tx_t *tx; + int error; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(dzp); + ZFS_VERIFY_ZP(zp); + zilog = zfsvfs->z_log; + + + if ((error = zfs_zaccess_delete(dzp, zp, cr))) { + goto out; + } + + if (vp->v_type != VDIR) { + error = SET_ERROR(ENOTDIR); + goto out; + } + + vnevent_rmdir(vp, dvp, name, ct); + + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL); + zfs_sa_upgrade_txholds(tx, zp); + zfs_sa_upgrade_txholds(tx, dzp); + dmu_tx_mark_netfree(tx); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + ZFS_EXIT(zfsvfs); + return (error); + } + + cache_purge(dvp); + + error = zfs_link_destroy(dzp, name, zp, tx, ZEXISTS, NULL); + + if (error == 0) { + uint64_t txtype = TX_RMDIR; + zfs_log_remove(zilog, tx, txtype, dzp, name, + ZFS_NO_OBJECT, B_FALSE); + } + + dmu_tx_commit(tx); + + cache_purge(vp); +out: + if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zilog, 0); + + ZFS_EXIT(zfsvfs); + return (error); +} + +int +zfs_rmdir(znode_t *dzp, char *name, znode_t *cwd, cred_t *cr, int flags) +{ + struct componentname cn; + vnode_t *vp; + int error; + + if ((error = zfs_lookup_internal(dzp, name, &vp, &cn, DELETE))) + return (error); + + error = zfs_rmdir_(ZTOV(dzp), vp, name, cr); + vput(vp); + return (error); +} + +/* + * Read as many directory entries as will fit into the provided + * buffer from the given directory cursor position (specified in + * the uio structure). + * + * IN: vp - vnode of directory to read. + * uio - structure supplying read location, range info, + * and return buffer. + * cr - credentials of caller. + * ct - caller context + * flags - case flags + * + * OUT: uio - updated offset and range, buffer filled. + * eofp - set to true if end-of-file detected. + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * vp - atime updated + * + * Note that the low 4 bits of the cookie returned by zap is always zero. + * This allows us to use the low range for "special" directory entries: + * We use 0 for '.', and 1 for '..'. If this is the root of the filesystem, + * we use the offset 2 for the '.zfs' directory. + */ +/* ARGSUSED */ +static int +zfs_readdir(vnode_t *vp, uio_t *uio, cred_t *cr, int *eofp, + int *ncookies, ulong_t **cookies) +{ + znode_t *zp = VTOZ(vp); + iovec_t *iovp; + edirent_t *eodp; + dirent64_t *odp; + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + objset_t *os; + caddr_t outbuf; + size_t bufsize; + zap_cursor_t zc; + zap_attribute_t zap; + uint_t bytes_wanted; + uint64_t offset; /* must be unsigned; checks for < 1 */ + uint64_t parent; + int local_eof; + int outcount; + int error; + uint8_t prefetch; + boolean_t check_sysattrs; + uint8_t type; + int ncooks; + ulong_t *cooks = NULL; + int flags = 0; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs), + &parent, sizeof (parent))) != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + + /* + * If we are not given an eof variable, + * use a local one. + */ + if (eofp == NULL) + eofp = &local_eof; + + /* + * Check for valid iov_len. + */ + if (uio->uio_iov->iov_len <= 0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + /* + * Quit if directory has been removed (posix) + */ + if ((*eofp = zp->z_unlinked) != 0) { + ZFS_EXIT(zfsvfs); + return (0); + } + + error = 0; + os = zfsvfs->z_os; + offset = uio->uio_loffset; + prefetch = zp->z_zn_prefetch; + + /* + * Initialize the iterator cursor. + */ + if (offset <= 3) { + /* + * Start iteration from the beginning of the directory. + */ + zap_cursor_init(&zc, os, zp->z_id); + } else { + /* + * The offset is a serialized cursor. + */ + zap_cursor_init_serialized(&zc, os, zp->z_id, offset); + } + + /* + * Get space to change directory entries into fs independent format. + */ + iovp = uio->uio_iov; + bytes_wanted = iovp->iov_len; + if (uio->uio_segflg != UIO_SYSSPACE || uio->uio_iovcnt != 1) { + bufsize = bytes_wanted; + outbuf = kmem_alloc(bufsize, KM_SLEEP); + odp = (struct dirent64 *)outbuf; + } else { + bufsize = bytes_wanted; + outbuf = NULL; + odp = (struct dirent64 *)iovp->iov_base; + } + eodp = (struct edirent *)odp; + + if (ncookies != NULL) { + /* + * Minimum entry size is dirent size and 1 byte for a file name. + */ + ncooks = uio->uio_resid / (sizeof (struct dirent) - + sizeof (((struct dirent *)NULL)->d_name) + 1); + cooks = malloc(ncooks * sizeof (ulong_t), M_TEMP, M_WAITOK); + *cookies = cooks; + *ncookies = ncooks; + } + /* + * If this VFS supports the system attribute view interface; and + * we're looking at an extended attribute directory; and we care + * about normalization conflicts on this vfs; then we must check + * for normalization conflicts with the sysattr name space. + */ +#ifdef TODO + check_sysattrs = vfs_has_feature(vp->v_vfsp, VFSFT_SYSATTR_VIEWS) && + (vp->v_flag & V_XATTRDIR) && zfsvfs->z_norm && + (flags & V_RDDIR_ENTFLAGS); +#else + check_sysattrs = 0; +#endif + + /* + * Transform to file-system independent format + */ + outcount = 0; + while (outcount < bytes_wanted) { + ino64_t objnum; + ushort_t reclen; + off64_t *next = NULL; + + /* + * Special case `.', `..', and `.zfs'. + */ + if (offset == 0) { + (void) strcpy(zap.za_name, "."); + zap.za_normalization_conflict = 0; + objnum = zp->z_id; + type = DT_DIR; + } else if (offset == 1) { + (void) strcpy(zap.za_name, ".."); + zap.za_normalization_conflict = 0; + objnum = parent; + type = DT_DIR; + } else if (offset == 2 && zfs_show_ctldir(zp)) { + (void) strcpy(zap.za_name, ZFS_CTLDIR_NAME); + zap.za_normalization_conflict = 0; + objnum = ZFSCTL_INO_ROOT; + type = DT_DIR; + } else { + /* + * Grab next entry. + */ + if ((error = zap_cursor_retrieve(&zc, &zap))) { + if ((*eofp = (error == ENOENT)) != 0) + break; + else + goto update; + } + + if (zap.za_integer_length != 8 || + zap.za_num_integers != 1) { + cmn_err(CE_WARN, "zap_readdir: bad directory " + "entry, obj = %lld, offset = %lld\n", + (u_longlong_t)zp->z_id, + (u_longlong_t)offset); + error = SET_ERROR(ENXIO); + goto update; + } + + objnum = ZFS_DIRENT_OBJ(zap.za_first_integer); + /* + * MacOS X can extract the object type here such as: + * uint8_t type = ZFS_DIRENT_TYPE(zap.za_first_integer); + */ + type = ZFS_DIRENT_TYPE(zap.za_first_integer); + + if (check_sysattrs && !zap.za_normalization_conflict) { +#ifdef TODO + zap.za_normalization_conflict = + xattr_sysattr_casechk(zap.za_name); +#else + panic("%s:%u: TODO", __func__, __LINE__); +#endif + } + } + + if (flags & V_RDDIR_ACCFILTER) { + /* + * If we have no access at all, don't include + * this entry in the returned information + */ + znode_t *ezp; + if (zfs_zget(zp->z_zfsvfs, objnum, &ezp) != 0) + goto skip_entry; + if (!zfs_has_access(ezp, cr)) { + vrele(ZTOV(ezp)); + goto skip_entry; + } + vrele(ZTOV(ezp)); + } + + if (flags & V_RDDIR_ENTFLAGS) + reclen = EDIRENT_RECLEN(strlen(zap.za_name)); + else + reclen = DIRENT64_RECLEN(strlen(zap.za_name)); + + /* + * Will this entry fit in the buffer? + */ + if (outcount + reclen > bufsize) { + /* + * Did we manage to fit anything in the buffer? + */ + if (!outcount) { + error = SET_ERROR(EINVAL); + goto update; + } + break; + } + if (flags & V_RDDIR_ENTFLAGS) { + /* + * Add extended flag entry: + */ + eodp->ed_ino = objnum; + eodp->ed_reclen = reclen; + /* NOTE: ed_off is the offset for the *next* entry */ + next = &(eodp->ed_off); + eodp->ed_eflags = zap.za_normalization_conflict ? + ED_CASE_CONFLICT : 0; + (void) strncpy(eodp->ed_name, zap.za_name, + EDIRENT_NAMELEN(reclen)); + eodp = (edirent_t *)((intptr_t)eodp + reclen); + } else { + /* + * Add normal entry: + */ + odp->d_ino = objnum; + odp->d_reclen = reclen; + odp->d_namlen = strlen(zap.za_name); + /* NOTE: d_off is the offset for the *next* entry. */ + next = &odp->d_off; + strlcpy(odp->d_name, zap.za_name, odp->d_namlen + 1); + odp->d_type = type; + dirent_terminate(odp); + odp = (dirent64_t *)((intptr_t)odp + reclen); + } + outcount += reclen; + + ASSERT(outcount <= bufsize); + + /* Prefetch znode */ + if (prefetch) + dmu_prefetch(os, objnum, 0, 0, 0, + ZIO_PRIORITY_SYNC_READ); + + skip_entry: + /* + * Move to the next entry, fill in the previous offset. + */ + if (offset > 2 || (offset == 2 && !zfs_show_ctldir(zp))) { + zap_cursor_advance(&zc); + offset = zap_cursor_serialize(&zc); + } else { + offset += 1; + } + + /* Fill the offset right after advancing the cursor. */ + if (next != NULL) + *next = offset; + if (cooks != NULL) { + *cooks++ = offset; + ncooks--; + KASSERT(ncooks >= 0, ("ncookies=%d", ncooks)); + } + } + zp->z_zn_prefetch = B_FALSE; /* a lookup will re-enable pre-fetching */ + + /* Subtract unused cookies */ + if (ncookies != NULL) + *ncookies -= ncooks; + + if (uio->uio_segflg == UIO_SYSSPACE && uio->uio_iovcnt == 1) { + iovp->iov_base += outcount; + iovp->iov_len -= outcount; + uio->uio_resid -= outcount; + } else if ((error = uiomove(outbuf, (long)outcount, UIO_READ, uio))) { + /* + * Reset the pointer. + */ + offset = uio->uio_loffset; + } + +update: + zap_cursor_fini(&zc); + if (uio->uio_segflg != UIO_SYSSPACE || uio->uio_iovcnt != 1) + kmem_free(outbuf, bufsize); + + if (error == ENOENT) + error = 0; + + ZFS_ACCESSTIME_STAMP(zfsvfs, zp); + + uio->uio_loffset = offset; + ZFS_EXIT(zfsvfs); + if (error != 0 && cookies != NULL) { + free(*cookies, M_TEMP); + *cookies = NULL; + *ncookies = 0; + } + return (error); +} + +ulong_t zfs_fsync_sync_cnt = 4; + +static int +zfs_fsync(vnode_t *vp, int syncflag, cred_t *cr, caller_context_t *ct) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + + (void) tsd_set(zfs_fsyncer_key, (void *)zfs_fsync_sync_cnt); + + if (zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) { + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + zil_commit(zfsvfs->z_log, zp->z_id); + ZFS_EXIT(zfsvfs); + } + tsd_set(zfs_fsyncer_key, NULL); + return (0); +} + + +/* + * Get the requested file attributes and place them in the provided + * vattr structure. + * + * IN: vp - vnode of file. + * vap - va_mask identifies requested attributes. + * If AT_XVATTR set, then optional attrs are requested + * flags - ATTR_NOACLCHECK (CIFS server context) + * cr - credentials of caller. + * + * OUT: vap - attribute values. + * + * RETURN: 0 (always succeeds). + */ +/* ARGSUSED */ +static int +zfs_getattr(vnode_t *vp, vattr_t *vap, int flags, cred_t *cr) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + int error = 0; + uint32_t blksize; + u_longlong_t nblocks; + uint64_t mtime[2], ctime[2], crtime[2], rdev; + xvattr_t *xvap = (xvattr_t *)vap; /* vap may be an xvattr_t * */ + xoptattr_t *xoap = NULL; + boolean_t skipaclchk = (flags & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE; + sa_bulk_attr_t bulk[4]; + int count = 0; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + zfs_fuid_map_ids(zp, cr, &vap->va_uid, &vap->va_gid); + + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CRTIME(zfsvfs), NULL, &crtime, 16); + if (vp->v_type == VBLK || vp->v_type == VCHR) + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_RDEV(zfsvfs), NULL, + &rdev, 8); + + if ((error = sa_bulk_lookup(zp->z_sa_hdl, bulk, count)) != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + + /* + * If ACL is trivial don't bother looking for ACE_READ_ATTRIBUTES. + * Also, if we are the owner don't bother, since owner should + * always be allowed to read basic attributes of file. + */ + if (!(zp->z_pflags & ZFS_ACL_TRIVIAL) && + (vap->va_uid != crgetuid(cr))) { + if ((error = zfs_zaccess(zp, ACE_READ_ATTRIBUTES, 0, + skipaclchk, cr))) { + ZFS_EXIT(zfsvfs); + return (error); + } + } + + /* + * Return all attributes. It's cheaper to provide the answer + * than to determine whether we were asked the question. + */ + + vap->va_type = IFTOVT(zp->z_mode); + vap->va_mode = zp->z_mode & ~S_IFMT; + vn_fsid(vp, vap); + vap->va_nodeid = zp->z_id; + vap->va_nlink = zp->z_links; + if ((vp->v_flag & VROOT) && zfs_show_ctldir(zp) && + zp->z_links < ZFS_LINK_MAX) + vap->va_nlink++; + vap->va_size = zp->z_size; + if (vp->v_type == VBLK || vp->v_type == VCHR) + vap->va_rdev = zfs_cmpldev(rdev); + vap->va_seq = zp->z_seq; + vap->va_flags = 0; /* FreeBSD: Reset chflags(2) flags. */ + vap->va_filerev = zp->z_seq; + + /* + * Add in any requested optional attributes and the create time. + * Also set the corresponding bits in the returned attribute bitmap. + */ + if ((xoap = xva_getxoptattr(xvap)) != NULL && zfsvfs->z_use_fuids) { + if (XVA_ISSET_REQ(xvap, XAT_ARCHIVE)) { + xoap->xoa_archive = + ((zp->z_pflags & ZFS_ARCHIVE) != 0); + XVA_SET_RTN(xvap, XAT_ARCHIVE); + } + + if (XVA_ISSET_REQ(xvap, XAT_READONLY)) { + xoap->xoa_readonly = + ((zp->z_pflags & ZFS_READONLY) != 0); + XVA_SET_RTN(xvap, XAT_READONLY); + } + + if (XVA_ISSET_REQ(xvap, XAT_SYSTEM)) { + xoap->xoa_system = + ((zp->z_pflags & ZFS_SYSTEM) != 0); + XVA_SET_RTN(xvap, XAT_SYSTEM); + } + + if (XVA_ISSET_REQ(xvap, XAT_HIDDEN)) { + xoap->xoa_hidden = + ((zp->z_pflags & ZFS_HIDDEN) != 0); + XVA_SET_RTN(xvap, XAT_HIDDEN); + } + + if (XVA_ISSET_REQ(xvap, XAT_NOUNLINK)) { + xoap->xoa_nounlink = + ((zp->z_pflags & ZFS_NOUNLINK) != 0); + XVA_SET_RTN(xvap, XAT_NOUNLINK); + } + + if (XVA_ISSET_REQ(xvap, XAT_IMMUTABLE)) { + xoap->xoa_immutable = + ((zp->z_pflags & ZFS_IMMUTABLE) != 0); + XVA_SET_RTN(xvap, XAT_IMMUTABLE); + } + + if (XVA_ISSET_REQ(xvap, XAT_APPENDONLY)) { + xoap->xoa_appendonly = + ((zp->z_pflags & ZFS_APPENDONLY) != 0); + XVA_SET_RTN(xvap, XAT_APPENDONLY); + } + + if (XVA_ISSET_REQ(xvap, XAT_NODUMP)) { + xoap->xoa_nodump = + ((zp->z_pflags & ZFS_NODUMP) != 0); + XVA_SET_RTN(xvap, XAT_NODUMP); + } + + if (XVA_ISSET_REQ(xvap, XAT_OPAQUE)) { + xoap->xoa_opaque = + ((zp->z_pflags & ZFS_OPAQUE) != 0); + XVA_SET_RTN(xvap, XAT_OPAQUE); + } + + if (XVA_ISSET_REQ(xvap, XAT_AV_QUARANTINED)) { + xoap->xoa_av_quarantined = + ((zp->z_pflags & ZFS_AV_QUARANTINED) != 0); + XVA_SET_RTN(xvap, XAT_AV_QUARANTINED); + } + + if (XVA_ISSET_REQ(xvap, XAT_AV_MODIFIED)) { + xoap->xoa_av_modified = + ((zp->z_pflags & ZFS_AV_MODIFIED) != 0); + XVA_SET_RTN(xvap, XAT_AV_MODIFIED); + } + + if (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP) && + vp->v_type == VREG) { + zfs_sa_get_scanstamp(zp, xvap); + } + + if (XVA_ISSET_REQ(xvap, XAT_REPARSE)) { + xoap->xoa_reparse = ((zp->z_pflags & ZFS_REPARSE) != 0); + XVA_SET_RTN(xvap, XAT_REPARSE); + } + if (XVA_ISSET_REQ(xvap, XAT_GEN)) { + xoap->xoa_generation = zp->z_gen; + XVA_SET_RTN(xvap, XAT_GEN); + } + + if (XVA_ISSET_REQ(xvap, XAT_OFFLINE)) { + xoap->xoa_offline = + ((zp->z_pflags & ZFS_OFFLINE) != 0); + XVA_SET_RTN(xvap, XAT_OFFLINE); + } + + if (XVA_ISSET_REQ(xvap, XAT_SPARSE)) { + xoap->xoa_sparse = + ((zp->z_pflags & ZFS_SPARSE) != 0); + XVA_SET_RTN(xvap, XAT_SPARSE); + } + + if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT)) { + xoap->xoa_projinherit = + ((zp->z_pflags & ZFS_PROJINHERIT) != 0); + XVA_SET_RTN(xvap, XAT_PROJINHERIT); + } + + if (XVA_ISSET_REQ(xvap, XAT_PROJID)) { + xoap->xoa_projid = zp->z_projid; + XVA_SET_RTN(xvap, XAT_PROJID); + } + } + + ZFS_TIME_DECODE(&vap->va_atime, zp->z_atime); + ZFS_TIME_DECODE(&vap->va_mtime, mtime); + ZFS_TIME_DECODE(&vap->va_ctime, ctime); + ZFS_TIME_DECODE(&vap->va_birthtime, crtime); + + + sa_object_size(zp->z_sa_hdl, &blksize, &nblocks); + vap->va_blksize = blksize; + vap->va_bytes = nblocks << 9; /* nblocks * 512 */ + + if (zp->z_blksz == 0) { + /* + * Block size hasn't been set; suggest maximal I/O transfers. + */ + vap->va_blksize = zfsvfs->z_max_blksz; + } + + ZFS_EXIT(zfsvfs); + return (0); +} + +/* + * Set the file attributes to the values contained in the + * vattr structure. + * + * IN: zp - znode of file to be modified. + * vap - new attribute values. + * If AT_XVATTR set, then optional attrs are being set + * flags - ATTR_UTIME set if non-default time values provided. + * - ATTR_NOACLCHECK (CIFS context only). + * cr - credentials of caller. + * ct - caller context + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * vp - ctime updated, mtime updated if size changed. + */ +/* ARGSUSED */ +int +zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr) +{ + vnode_t *vp = ZTOV(zp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + objset_t *os = zfsvfs->z_os; + zilog_t *zilog; + dmu_tx_t *tx; + vattr_t oldva; + xvattr_t tmpxvattr; + uint_t mask = vap->va_mask; + uint_t saved_mask = 0; + uint64_t saved_mode; + int trim_mask = 0; + uint64_t new_mode; + uint64_t new_uid, new_gid; + uint64_t xattr_obj; + uint64_t mtime[2], ctime[2]; + uint64_t projid = ZFS_INVALID_PROJID; + znode_t *attrzp; + int need_policy = FALSE; + int err, err2; + zfs_fuid_info_t *fuidp = NULL; + xvattr_t *xvap = (xvattr_t *)vap; /* vap may be an xvattr_t * */ + xoptattr_t *xoap; + zfs_acl_t *aclp; + boolean_t skipaclchk = (flags & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE; + boolean_t fuid_dirtied = B_FALSE; + sa_bulk_attr_t bulk[7], xattr_bulk[7]; + int count = 0, xattr_count = 0; + + if (mask == 0) + return (0); + + if (mask & AT_NOSET) + return (SET_ERROR(EINVAL)); + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + zilog = zfsvfs->z_log; + + /* + * Make sure that if we have ephemeral uid/gid or xvattr specified + * that file system is at proper version level + */ + + if (zfsvfs->z_use_fuids == B_FALSE && + (((mask & AT_UID) && IS_EPHEMERAL(vap->va_uid)) || + ((mask & AT_GID) && IS_EPHEMERAL(vap->va_gid)) || + (mask & AT_XVATTR))) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + if (mask & AT_SIZE && vp->v_type == VDIR) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EISDIR)); + } + + if (mask & AT_SIZE && vp->v_type != VREG && vp->v_type != VFIFO) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + /* + * If this is an xvattr_t, then get a pointer to the structure of + * optional attributes. If this is NULL, then we have a vattr_t. + */ + xoap = xva_getxoptattr(xvap); + + xva_init(&tmpxvattr); + + /* + * Immutable files can only alter immutable bit and atime + */ + if ((zp->z_pflags & ZFS_IMMUTABLE) && + ((mask & (AT_SIZE|AT_UID|AT_GID|AT_MTIME|AT_MODE)) || + ((mask & AT_XVATTR) && XVA_ISSET_REQ(xvap, XAT_CREATETIME)))) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EPERM)); + } + + /* + * Note: ZFS_READONLY is handled in zfs_zaccess_common. + */ + + /* + * Verify timestamps doesn't overflow 32 bits. + * ZFS can handle large timestamps, but 32bit syscalls can't + * handle times greater than 2039. This check should be removed + * once large timestamps are fully supported. + */ + if (mask & (AT_ATIME | AT_MTIME)) { + if (((mask & AT_ATIME) && TIMESPEC_OVERFLOW(&vap->va_atime)) || + ((mask & AT_MTIME) && TIMESPEC_OVERFLOW(&vap->va_mtime))) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EOVERFLOW)); + } + } + if (xoap != NULL && (mask & AT_XVATTR)) { + if (XVA_ISSET_REQ(xvap, XAT_CREATETIME) && + TIMESPEC_OVERFLOW(&vap->va_birthtime)) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EOVERFLOW)); + } + + if (XVA_ISSET_REQ(xvap, XAT_PROJID)) { + if (!dmu_objset_projectquota_enabled(os) || + (!S_ISREG(zp->z_mode) && !S_ISDIR(zp->z_mode))) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EOPNOTSUPP)); + } + + projid = xoap->xoa_projid; + if (unlikely(projid == ZFS_INVALID_PROJID)) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + if (projid == zp->z_projid && zp->z_pflags & ZFS_PROJID) + projid = ZFS_INVALID_PROJID; + else + need_policy = TRUE; + } + + if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT) && + (xoap->xoa_projinherit != + ((zp->z_pflags & ZFS_PROJINHERIT) != 0)) && + (!dmu_objset_projectquota_enabled(os) || + (!S_ISREG(zp->z_mode) && !S_ISDIR(zp->z_mode)))) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EOPNOTSUPP)); + } + } + + attrzp = NULL; + aclp = NULL; + + if (zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EROFS)); + } + + /* + * First validate permissions + */ + + if (mask & AT_SIZE) { + /* + * XXX - Note, we are not providing any open + * mode flags here (like FNDELAY), so we may + * block if there are locks present... this + * should be addressed in openat(). + */ + /* XXX - would it be OK to generate a log record here? */ + err = zfs_freesp(zp, vap->va_size, 0, 0, FALSE); + if (err) { + ZFS_EXIT(zfsvfs); + return (err); + } + } + + if (mask & (AT_ATIME|AT_MTIME) || + ((mask & AT_XVATTR) && (XVA_ISSET_REQ(xvap, XAT_HIDDEN) || + XVA_ISSET_REQ(xvap, XAT_READONLY) || + XVA_ISSET_REQ(xvap, XAT_ARCHIVE) || + XVA_ISSET_REQ(xvap, XAT_OFFLINE) || + XVA_ISSET_REQ(xvap, XAT_SPARSE) || + XVA_ISSET_REQ(xvap, XAT_CREATETIME) || + XVA_ISSET_REQ(xvap, XAT_SYSTEM)))) { + need_policy = zfs_zaccess(zp, ACE_WRITE_ATTRIBUTES, 0, + skipaclchk, cr); + } + + if (mask & (AT_UID|AT_GID)) { + int idmask = (mask & (AT_UID|AT_GID)); + int take_owner; + int take_group; + + /* + * NOTE: even if a new mode is being set, + * we may clear S_ISUID/S_ISGID bits. + */ + + if (!(mask & AT_MODE)) + vap->va_mode = zp->z_mode; + + /* + * Take ownership or chgrp to group we are a member of + */ + + take_owner = (mask & AT_UID) && (vap->va_uid == crgetuid(cr)); + take_group = (mask & AT_GID) && + zfs_groupmember(zfsvfs, vap->va_gid, cr); + + /* + * If both AT_UID and AT_GID are set then take_owner and + * take_group must both be set in order to allow taking + * ownership. + * + * Otherwise, send the check through secpolicy_vnode_setattr() + * + */ + + if (((idmask == (AT_UID|AT_GID)) && take_owner && take_group) || + ((idmask == AT_UID) && take_owner) || + ((idmask == AT_GID) && take_group)) { + if (zfs_zaccess(zp, ACE_WRITE_OWNER, 0, + skipaclchk, cr) == 0) { + /* + * Remove setuid/setgid for non-privileged users + */ + secpolicy_setid_clear(vap, vp, cr); + trim_mask = (mask & (AT_UID|AT_GID)); + } else { + need_policy = TRUE; + } + } else { + need_policy = TRUE; + } + } + + oldva.va_mode = zp->z_mode; + zfs_fuid_map_ids(zp, cr, &oldva.va_uid, &oldva.va_gid); + if (mask & AT_XVATTR) { + /* + * Update xvattr mask to include only those attributes + * that are actually changing. + * + * the bits will be restored prior to actually setting + * the attributes so the caller thinks they were set. + */ + if (XVA_ISSET_REQ(xvap, XAT_APPENDONLY)) { + if (xoap->xoa_appendonly != + ((zp->z_pflags & ZFS_APPENDONLY) != 0)) { + need_policy = TRUE; + } else { + XVA_CLR_REQ(xvap, XAT_APPENDONLY); + XVA_SET_REQ(&tmpxvattr, XAT_APPENDONLY); + } + } + + if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT)) { + if (xoap->xoa_projinherit != + ((zp->z_pflags & ZFS_PROJINHERIT) != 0)) { + need_policy = TRUE; + } else { + XVA_CLR_REQ(xvap, XAT_PROJINHERIT); + XVA_SET_REQ(&tmpxvattr, XAT_PROJINHERIT); + } + } + + if (XVA_ISSET_REQ(xvap, XAT_NOUNLINK)) { + if (xoap->xoa_nounlink != + ((zp->z_pflags & ZFS_NOUNLINK) != 0)) { + need_policy = TRUE; + } else { + XVA_CLR_REQ(xvap, XAT_NOUNLINK); + XVA_SET_REQ(&tmpxvattr, XAT_NOUNLINK); + } + } + + if (XVA_ISSET_REQ(xvap, XAT_IMMUTABLE)) { + if (xoap->xoa_immutable != + ((zp->z_pflags & ZFS_IMMUTABLE) != 0)) { + need_policy = TRUE; + } else { + XVA_CLR_REQ(xvap, XAT_IMMUTABLE); + XVA_SET_REQ(&tmpxvattr, XAT_IMMUTABLE); + } + } + + if (XVA_ISSET_REQ(xvap, XAT_NODUMP)) { + if (xoap->xoa_nodump != + ((zp->z_pflags & ZFS_NODUMP) != 0)) { + need_policy = TRUE; + } else { + XVA_CLR_REQ(xvap, XAT_NODUMP); + XVA_SET_REQ(&tmpxvattr, XAT_NODUMP); + } + } + + if (XVA_ISSET_REQ(xvap, XAT_AV_MODIFIED)) { + if (xoap->xoa_av_modified != + ((zp->z_pflags & ZFS_AV_MODIFIED) != 0)) { + need_policy = TRUE; + } else { + XVA_CLR_REQ(xvap, XAT_AV_MODIFIED); + XVA_SET_REQ(&tmpxvattr, XAT_AV_MODIFIED); + } + } + + if (XVA_ISSET_REQ(xvap, XAT_AV_QUARANTINED)) { + if ((vp->v_type != VREG && + xoap->xoa_av_quarantined) || + xoap->xoa_av_quarantined != + ((zp->z_pflags & ZFS_AV_QUARANTINED) != 0)) { + need_policy = TRUE; + } else { + XVA_CLR_REQ(xvap, XAT_AV_QUARANTINED); + XVA_SET_REQ(&tmpxvattr, XAT_AV_QUARANTINED); + } + } + + if (XVA_ISSET_REQ(xvap, XAT_REPARSE)) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EPERM)); + } + + if (need_policy == FALSE && + (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP) || + XVA_ISSET_REQ(xvap, XAT_OPAQUE))) { + need_policy = TRUE; + } + } + + if (mask & AT_MODE) { + if (zfs_zaccess(zp, ACE_WRITE_ACL, 0, skipaclchk, cr) == 0) { + err = secpolicy_setid_setsticky_clear(vp, vap, + &oldva, cr); + if (err) { + ZFS_EXIT(zfsvfs); + return (err); + } + trim_mask |= AT_MODE; + } else { + need_policy = TRUE; + } + } + + if (need_policy) { + /* + * If trim_mask is set then take ownership + * has been granted or write_acl is present and user + * has the ability to modify mode. In that case remove + * UID|GID and or MODE from mask so that + * secpolicy_vnode_setattr() doesn't revoke it. + */ + + if (trim_mask) { + saved_mask = vap->va_mask; + vap->va_mask &= ~trim_mask; + if (trim_mask & AT_MODE) { + /* + * Save the mode, as secpolicy_vnode_setattr() + * will overwrite it with ova.va_mode. + */ + saved_mode = vap->va_mode; + } + } + err = secpolicy_vnode_setattr(cr, vp, vap, &oldva, flags, + (int (*)(void *, int, cred_t *))zfs_zaccess_unix, zp); + if (err) { + ZFS_EXIT(zfsvfs); + return (err); + } + + if (trim_mask) { + vap->va_mask |= saved_mask; + if (trim_mask & AT_MODE) { + /* + * Recover the mode after + * secpolicy_vnode_setattr(). + */ + vap->va_mode = saved_mode; + } + } + } + + /* + * secpolicy_vnode_setattr, or take ownership may have + * changed va_mask + */ + mask = vap->va_mask; + + if ((mask & (AT_UID | AT_GID)) || projid != ZFS_INVALID_PROJID) { + err = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), + &xattr_obj, sizeof (xattr_obj)); + + if (err == 0 && xattr_obj) { + err = zfs_zget(zp->z_zfsvfs, xattr_obj, &attrzp); + if (err == 0) { + err = vn_lock(ZTOV(attrzp), LK_EXCLUSIVE); + if (err != 0) + vrele(ZTOV(attrzp)); + } + if (err) + goto out2; + } + if (mask & AT_UID) { + new_uid = zfs_fuid_create(zfsvfs, + (uint64_t)vap->va_uid, cr, ZFS_OWNER, &fuidp); + if (new_uid != zp->z_uid && + zfs_id_overquota(zfsvfs, DMU_USERUSED_OBJECT, + new_uid)) { + if (attrzp) + vput(ZTOV(attrzp)); + err = SET_ERROR(EDQUOT); + goto out2; + } + } + + if (mask & AT_GID) { + new_gid = zfs_fuid_create(zfsvfs, (uint64_t)vap->va_gid, + cr, ZFS_GROUP, &fuidp); + if (new_gid != zp->z_gid && + zfs_id_overquota(zfsvfs, DMU_GROUPUSED_OBJECT, + new_gid)) { + if (attrzp) + vput(ZTOV(attrzp)); + err = SET_ERROR(EDQUOT); + goto out2; + } + } + + if (projid != ZFS_INVALID_PROJID && + zfs_id_overquota(zfsvfs, DMU_PROJECTUSED_OBJECT, projid)) { + if (attrzp) + vput(ZTOV(attrzp)); + err = SET_ERROR(EDQUOT); + goto out2; + } + } + tx = dmu_tx_create(os); + + if (mask & AT_MODE) { + uint64_t pmode = zp->z_mode; + uint64_t acl_obj; + new_mode = (pmode & S_IFMT) | (vap->va_mode & ~S_IFMT); + + if (zp->z_zfsvfs->z_acl_mode == ZFS_ACL_RESTRICTED && + !(zp->z_pflags & ZFS_ACL_TRIVIAL)) { + err = SET_ERROR(EPERM); + goto out; + } + + if ((err = zfs_acl_chmod_setattr(zp, &aclp, new_mode))) + goto out; + + if (!zp->z_is_sa && ((acl_obj = zfs_external_acl(zp)) != 0)) { + /* + * Are we upgrading ACL from old V0 format + * to V1 format? + */ + if (zfsvfs->z_version >= ZPL_VERSION_FUID && + zfs_znode_acl_version(zp) == + ZFS_ACL_VERSION_INITIAL) { + dmu_tx_hold_free(tx, acl_obj, 0, + DMU_OBJECT_END); + dmu_tx_hold_write(tx, DMU_NEW_OBJECT, + 0, aclp->z_acl_bytes); + } else { + dmu_tx_hold_write(tx, acl_obj, 0, + aclp->z_acl_bytes); + } + } else if (!zp->z_is_sa && aclp->z_acl_bytes > ZFS_ACE_SPACE) { + dmu_tx_hold_write(tx, DMU_NEW_OBJECT, + 0, aclp->z_acl_bytes); + } + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE); + } else { + if (((mask & AT_XVATTR) && + XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP)) || + (projid != ZFS_INVALID_PROJID && + !(zp->z_pflags & ZFS_PROJID))) + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE); + else + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + } + + if (attrzp) { + dmu_tx_hold_sa(tx, attrzp->z_sa_hdl, B_FALSE); + } + + fuid_dirtied = zfsvfs->z_fuid_dirty; + if (fuid_dirtied) + zfs_fuid_txhold(zfsvfs, tx); + + zfs_sa_upgrade_txholds(tx, zp); + + err = dmu_tx_assign(tx, TXG_WAIT); + if (err) + goto out; + + count = 0; + /* + * Set each attribute requested. + * We group settings according to the locks they need to acquire. + * + * Note: you cannot set ctime directly, although it will be + * updated as a side-effect of calling this function. + */ + + if (projid != ZFS_INVALID_PROJID && !(zp->z_pflags & ZFS_PROJID)) { + /* + * For the existed object that is upgraded from old system, + * its on-disk layout has no slot for the project ID attribute. + * But quota accounting logic needs to access related slots by + * offset directly. So we need to adjust old objects' layout + * to make the project ID to some unified and fixed offset. + */ + if (attrzp) + err = sa_add_projid(attrzp->z_sa_hdl, tx, projid); + if (err == 0) + err = sa_add_projid(zp->z_sa_hdl, tx, projid); + + if (unlikely(err == EEXIST)) + err = 0; + else if (err != 0) + goto out; + else + projid = ZFS_INVALID_PROJID; + } + + if (mask & (AT_UID|AT_GID|AT_MODE)) + mutex_enter(&zp->z_acl_lock); + + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, + &zp->z_pflags, sizeof (zp->z_pflags)); + + if (attrzp) { + if (mask & (AT_UID|AT_GID|AT_MODE)) + mutex_enter(&attrzp->z_acl_lock); + SA_ADD_BULK_ATTR(xattr_bulk, xattr_count, + SA_ZPL_FLAGS(zfsvfs), NULL, &attrzp->z_pflags, + sizeof (attrzp->z_pflags)); + if (projid != ZFS_INVALID_PROJID) { + attrzp->z_projid = projid; + SA_ADD_BULK_ATTR(xattr_bulk, xattr_count, + SA_ZPL_PROJID(zfsvfs), NULL, &attrzp->z_projid, + sizeof (attrzp->z_projid)); + } + } + + if (mask & (AT_UID|AT_GID)) { + + if (mask & AT_UID) { + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL, + &new_uid, sizeof (new_uid)); + zp->z_uid = new_uid; + if (attrzp) { + SA_ADD_BULK_ATTR(xattr_bulk, xattr_count, + SA_ZPL_UID(zfsvfs), NULL, &new_uid, + sizeof (new_uid)); + attrzp->z_uid = new_uid; + } + } + + if (mask & AT_GID) { + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), + NULL, &new_gid, sizeof (new_gid)); + zp->z_gid = new_gid; + if (attrzp) { + SA_ADD_BULK_ATTR(xattr_bulk, xattr_count, + SA_ZPL_GID(zfsvfs), NULL, &new_gid, + sizeof (new_gid)); + attrzp->z_gid = new_gid; + } + } + if (!(mask & AT_MODE)) { + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), + NULL, &new_mode, sizeof (new_mode)); + new_mode = zp->z_mode; + } + err = zfs_acl_chown_setattr(zp); + ASSERT(err == 0); + if (attrzp) { + err = zfs_acl_chown_setattr(attrzp); + ASSERT(err == 0); + } + } + + if (mask & AT_MODE) { + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL, + &new_mode, sizeof (new_mode)); + zp->z_mode = new_mode; + ASSERT3U((uintptr_t)aclp, !=, 0); + err = zfs_aclset_common(zp, aclp, cr, tx); + ASSERT0(err); + if (zp->z_acl_cached) + zfs_acl_free(zp->z_acl_cached); + zp->z_acl_cached = aclp; + aclp = NULL; + } + + + if (mask & AT_ATIME) { + ZFS_TIME_ENCODE(&vap->va_atime, zp->z_atime); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL, + &zp->z_atime, sizeof (zp->z_atime)); + } + + if (mask & AT_MTIME) { + ZFS_TIME_ENCODE(&vap->va_mtime, mtime); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, + mtime, sizeof (mtime)); + } + + if (projid != ZFS_INVALID_PROJID) { + zp->z_projid = projid; + SA_ADD_BULK_ATTR(bulk, count, + SA_ZPL_PROJID(zfsvfs), NULL, &zp->z_projid, + sizeof (zp->z_projid)); + } + + /* XXX - shouldn't this be done *before* the ATIME/MTIME checks? */ + if (mask & AT_SIZE && !(mask & AT_MTIME)) { + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), + NULL, mtime, sizeof (mtime)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, + &ctime, sizeof (ctime)); + zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); + } else if (mask != 0) { + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, + &ctime, sizeof (ctime)); + zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, ctime); + if (attrzp) { + SA_ADD_BULK_ATTR(xattr_bulk, xattr_count, + SA_ZPL_CTIME(zfsvfs), NULL, + &ctime, sizeof (ctime)); + zfs_tstamp_update_setup(attrzp, STATE_CHANGED, + mtime, ctime); + } + } + + /* + * Do this after setting timestamps to prevent timestamp + * update from toggling bit + */ + + if (xoap && (mask & AT_XVATTR)) { + + if (XVA_ISSET_REQ(xvap, XAT_CREATETIME)) + xoap->xoa_createtime = vap->va_birthtime; + /* + * restore trimmed off masks + * so that return masks can be set for caller. + */ + + if (XVA_ISSET_REQ(&tmpxvattr, XAT_APPENDONLY)) { + XVA_SET_REQ(xvap, XAT_APPENDONLY); + } + if (XVA_ISSET_REQ(&tmpxvattr, XAT_NOUNLINK)) { + XVA_SET_REQ(xvap, XAT_NOUNLINK); + } + if (XVA_ISSET_REQ(&tmpxvattr, XAT_IMMUTABLE)) { + XVA_SET_REQ(xvap, XAT_IMMUTABLE); + } + if (XVA_ISSET_REQ(&tmpxvattr, XAT_NODUMP)) { + XVA_SET_REQ(xvap, XAT_NODUMP); + } + if (XVA_ISSET_REQ(&tmpxvattr, XAT_AV_MODIFIED)) { + XVA_SET_REQ(xvap, XAT_AV_MODIFIED); + } + if (XVA_ISSET_REQ(&tmpxvattr, XAT_AV_QUARANTINED)) { + XVA_SET_REQ(xvap, XAT_AV_QUARANTINED); + } + if (XVA_ISSET_REQ(&tmpxvattr, XAT_PROJINHERIT)) { + XVA_SET_REQ(xvap, XAT_PROJINHERIT); + } + + if (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP)) + ASSERT(vp->v_type == VREG); + + zfs_xvattr_set(zp, xvap, tx); + } + + if (fuid_dirtied) + zfs_fuid_sync(zfsvfs, tx); + + if (mask != 0) + zfs_log_setattr(zilog, tx, TX_SETATTR, zp, vap, mask, fuidp); + + if (mask & (AT_UID|AT_GID|AT_MODE)) + mutex_exit(&zp->z_acl_lock); + + if (attrzp) { + if (mask & (AT_UID|AT_GID|AT_MODE)) + mutex_exit(&attrzp->z_acl_lock); + } +out: + if (err == 0 && attrzp) { + err2 = sa_bulk_update(attrzp->z_sa_hdl, xattr_bulk, + xattr_count, tx); + ASSERT(err2 == 0); + } + + if (attrzp) + vput(ZTOV(attrzp)); + + if (aclp) + zfs_acl_free(aclp); + + if (fuidp) { + zfs_fuid_info_free(fuidp); + fuidp = NULL; + } + + if (err) { + dmu_tx_abort(tx); + } else { + err2 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); + dmu_tx_commit(tx); + } + +out2: + if (os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zilog, 0); + + ZFS_EXIT(zfsvfs); + return (err); +} + +/* + * We acquire all but fdvp locks using non-blocking acquisitions. If we + * fail to acquire any lock in the path we will drop all held locks, + * acquire the new lock in a blocking fashion, and then release it and + * restart the rename. This acquire/release step ensures that we do not + * spin on a lock waiting for release. On error release all vnode locks + * and decrement references the way tmpfs_rename() would do. + */ +static int +zfs_rename_relock(struct vnode *sdvp, struct vnode **svpp, + struct vnode *tdvp, struct vnode **tvpp, + const struct componentname *scnp, const struct componentname *tcnp) +{ + zfsvfs_t *zfsvfs; + struct vnode *nvp, *svp, *tvp; + znode_t *sdzp, *tdzp, *szp, *tzp; + const char *snm = scnp->cn_nameptr; + const char *tnm = tcnp->cn_nameptr; + int error; + + VOP_UNLOCK1(tdvp); + if (*tvpp != NULL && *tvpp != tdvp) + VOP_UNLOCK1(*tvpp); + +relock: + error = vn_lock(sdvp, LK_EXCLUSIVE); + if (error) + goto out; + sdzp = VTOZ(sdvp); + + error = vn_lock(tdvp, LK_EXCLUSIVE | LK_NOWAIT); + if (error != 0) { + VOP_UNLOCK1(sdvp); + if (error != EBUSY) + goto out; + error = vn_lock(tdvp, LK_EXCLUSIVE); + if (error) + goto out; + VOP_UNLOCK1(tdvp); + goto relock; + } + tdzp = VTOZ(tdvp); + + /* + * Before using sdzp and tdzp we must ensure that they are live. + * As a porting legacy from illumos we have two things to worry + * about. One is typical for FreeBSD and it is that the vnode is + * not reclaimed (doomed). The other is that the znode is live. + * The current code can invalidate the znode without acquiring the + * corresponding vnode lock if the object represented by the znode + * and vnode is no longer valid after a rollback or receive operation. + * z_teardown_lock hidden behind ZFS_ENTER and ZFS_EXIT is the lock + * that protects the znodes from the invalidation. + */ + zfsvfs = sdzp->z_zfsvfs; + ASSERT3P(zfsvfs, ==, tdzp->z_zfsvfs); + ZFS_ENTER(zfsvfs); + + /* + * We can not use ZFS_VERIFY_ZP() here because it could directly return + * bypassing the cleanup code in the case of an error. + */ + if (tdzp->z_sa_hdl == NULL || sdzp->z_sa_hdl == NULL) { + ZFS_EXIT(zfsvfs); + VOP_UNLOCK1(sdvp); + VOP_UNLOCK1(tdvp); + error = SET_ERROR(EIO); + goto out; + } + + /* + * Re-resolve svp to be certain it still exists and fetch the + * correct vnode. + */ + error = zfs_dirent_lookup(sdzp, snm, &szp, ZEXISTS); + if (error != 0) { + /* Source entry invalid or not there. */ + ZFS_EXIT(zfsvfs); + VOP_UNLOCK1(sdvp); + VOP_UNLOCK1(tdvp); + if ((scnp->cn_flags & ISDOTDOT) != 0 || + (scnp->cn_namelen == 1 && scnp->cn_nameptr[0] == '.')) + error = SET_ERROR(EINVAL); + goto out; + } + svp = ZTOV(szp); + + /* + * Re-resolve tvp, if it disappeared we just carry on. + */ + error = zfs_dirent_lookup(tdzp, tnm, &tzp, 0); + if (error != 0) { + ZFS_EXIT(zfsvfs); + VOP_UNLOCK1(sdvp); + VOP_UNLOCK1(tdvp); + vrele(svp); + if ((tcnp->cn_flags & ISDOTDOT) != 0) + error = SET_ERROR(EINVAL); + goto out; + } + if (tzp != NULL) + tvp = ZTOV(tzp); + else + tvp = NULL; + + /* + * At present the vnode locks must be acquired before z_teardown_lock, + * although it would be more logical to use the opposite order. + */ + ZFS_EXIT(zfsvfs); + + /* + * Now try acquire locks on svp and tvp. + */ + nvp = svp; + error = vn_lock(nvp, LK_EXCLUSIVE | LK_NOWAIT); + if (error != 0) { + VOP_UNLOCK1(sdvp); + VOP_UNLOCK1(tdvp); + if (tvp != NULL) + vrele(tvp); + if (error != EBUSY) { + vrele(nvp); + goto out; + } + error = vn_lock(nvp, LK_EXCLUSIVE); + if (error != 0) { + vrele(nvp); + goto out; + } + VOP_UNLOCK1(nvp); + /* + * Concurrent rename race. + * XXX ? + */ + if (nvp == tdvp) { + vrele(nvp); + error = SET_ERROR(EINVAL); + goto out; + } + vrele(*svpp); + *svpp = nvp; + goto relock; + } + vrele(*svpp); + *svpp = nvp; + + if (*tvpp != NULL) + vrele(*tvpp); + *tvpp = NULL; + if (tvp != NULL) { + nvp = tvp; + error = vn_lock(nvp, LK_EXCLUSIVE | LK_NOWAIT); + if (error != 0) { + VOP_UNLOCK1(sdvp); + VOP_UNLOCK1(tdvp); + VOP_UNLOCK1(*svpp); + if (error != EBUSY) { + vrele(nvp); + goto out; + } + error = vn_lock(nvp, LK_EXCLUSIVE); + if (error != 0) { + vrele(nvp); + goto out; + } + vput(nvp); + goto relock; + } + *tvpp = nvp; + } + + return (0); + +out: + return (error); +} + +/* + * Note that we must use VRELE_ASYNC in this function as it walks + * up the directory tree and vrele may need to acquire an exclusive + * lock if a last reference to a vnode is dropped. + */ +static int +zfs_rename_check(znode_t *szp, znode_t *sdzp, znode_t *tdzp) +{ + zfsvfs_t *zfsvfs; + znode_t *zp, *zp1; + uint64_t parent; + int error; + + zfsvfs = tdzp->z_zfsvfs; + if (tdzp == szp) + return (SET_ERROR(EINVAL)); + if (tdzp == sdzp) + return (0); + if (tdzp->z_id == zfsvfs->z_root) + return (0); + zp = tdzp; + for (;;) { + ASSERT(!zp->z_unlinked); + if ((error = sa_lookup(zp->z_sa_hdl, + SA_ZPL_PARENT(zfsvfs), &parent, sizeof (parent))) != 0) + break; + + if (parent == szp->z_id) { + error = SET_ERROR(EINVAL); + break; + } + if (parent == zfsvfs->z_root) + break; + if (parent == sdzp->z_id) + break; + + error = zfs_zget(zfsvfs, parent, &zp1); + if (error != 0) + break; + + if (zp != tdzp) + VN_RELE_ASYNC(ZTOV(zp), + dsl_pool_zrele_taskq( + dmu_objset_pool(zfsvfs->z_os))); + zp = zp1; + } + + if (error == ENOTDIR) + panic("checkpath: .. not a directory\n"); + if (zp != tdzp) + VN_RELE_ASYNC(ZTOV(zp), + dsl_pool_zrele_taskq(dmu_objset_pool(zfsvfs->z_os))); + return (error); +} + +/* + * Move an entry from the provided source directory to the target + * directory. Change the entry name as indicated. + * + * IN: sdvp - Source directory containing the "old entry". + * snm - Old entry name. + * tdvp - Target directory to contain the "new entry". + * tnm - New entry name. + * cr - credentials of caller. + * ct - caller context + * flags - case flags + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * sdvp,tdvp - ctime|mtime updated + */ +/*ARGSUSED*/ +static int +zfs_rename_(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp, + vnode_t *tdvp, vnode_t **tvpp, struct componentname *tcnp, + cred_t *cr, int log) +{ + zfsvfs_t *zfsvfs; + znode_t *sdzp, *tdzp, *szp, *tzp; + zilog_t *zilog = NULL; + dmu_tx_t *tx; + char *snm = scnp->cn_nameptr; + char *tnm = tcnp->cn_nameptr; + int error = 0; + + /* Reject renames across filesystems. */ + if ((*svpp)->v_mount != tdvp->v_mount || + ((*tvpp) != NULL && (*svpp)->v_mount != (*tvpp)->v_mount)) { + error = SET_ERROR(EXDEV); + goto out; + } + + if (zfsctl_is_node(tdvp)) { + error = SET_ERROR(EXDEV); + goto out; + } + + /* + * Lock all four vnodes to ensure safety and semantics of renaming. + */ + error = zfs_rename_relock(sdvp, svpp, tdvp, tvpp, scnp, tcnp); + if (error != 0) { + /* no vnodes are locked in the case of error here */ + return (error); + } + + tdzp = VTOZ(tdvp); + sdzp = VTOZ(sdvp); + zfsvfs = tdzp->z_zfsvfs; + zilog = zfsvfs->z_log; + + /* + * After we re-enter ZFS_ENTER() we will have to revalidate all + * znodes involved. + */ + ZFS_ENTER(zfsvfs); + + if (zfsvfs->z_utf8 && u8_validate(tnm, + strlen(tnm), NULL, U8_VALIDATE_ENTIRE, &error) < 0) { + error = SET_ERROR(EILSEQ); + goto unlockout; + } + + /* If source and target are the same file, there is nothing to do. */ + if ((*svpp) == (*tvpp)) { + error = 0; + goto unlockout; + } + + if (((*svpp)->v_type == VDIR && (*svpp)->v_mountedhere != NULL) || + ((*tvpp) != NULL && (*tvpp)->v_type == VDIR && + (*tvpp)->v_mountedhere != NULL)) { + error = SET_ERROR(EXDEV); + goto unlockout; + } + + /* + * We can not use ZFS_VERIFY_ZP() here because it could directly return + * bypassing the cleanup code in the case of an error. + */ + if (tdzp->z_sa_hdl == NULL || sdzp->z_sa_hdl == NULL) { + error = SET_ERROR(EIO); + goto unlockout; + } + + szp = VTOZ(*svpp); + tzp = *tvpp == NULL ? NULL : VTOZ(*tvpp); + if (szp->z_sa_hdl == NULL || (tzp != NULL && tzp->z_sa_hdl == NULL)) { + error = SET_ERROR(EIO); + goto unlockout; + } + + /* + * This is to prevent the creation of links into attribute space + * by renaming a linked file into/outof an attribute directory. + * See the comment in zfs_link() for why this is considered bad. + */ + if ((tdzp->z_pflags & ZFS_XATTR) != (sdzp->z_pflags & ZFS_XATTR)) { + error = SET_ERROR(EINVAL); + goto unlockout; + } + + /* + * If we are using project inheritance, means if the directory has + * ZFS_PROJINHERIT set, then its descendant directories will inherit + * not only the project ID, but also the ZFS_PROJINHERIT flag. Under + * such case, we only allow renames into our tree when the project + * IDs are the same. + */ + if (tdzp->z_pflags & ZFS_PROJINHERIT && + tdzp->z_projid != szp->z_projid) { + error = SET_ERROR(EXDEV); + goto unlockout; + } + + /* + * Must have write access at the source to remove the old entry + * and write access at the target to create the new entry. + * Note that if target and source are the same, this can be + * done in a single check. + */ + if ((error = zfs_zaccess_rename(sdzp, szp, tdzp, tzp, cr))) + goto unlockout; + + if ((*svpp)->v_type == VDIR) { + /* + * Avoid ".", "..", and aliases of "." for obvious reasons. + */ + if ((scnp->cn_namelen == 1 && scnp->cn_nameptr[0] == '.') || + sdzp == szp || + (scnp->cn_flags | tcnp->cn_flags) & ISDOTDOT) { + error = EINVAL; + goto unlockout; + } + + /* + * Check to make sure rename is valid. + * Can't do a move like this: /usr/a/b to /usr/a/b/c/d + */ + if ((error = zfs_rename_check(szp, sdzp, tdzp))) + goto unlockout; + } + + /* + * Does target exist? + */ + if (tzp) { + /* + * Source and target must be the same type. + */ + if ((*svpp)->v_type == VDIR) { + if ((*tvpp)->v_type != VDIR) { + error = SET_ERROR(ENOTDIR); + goto unlockout; + } else { + cache_purge(tdvp); + if (sdvp != tdvp) + cache_purge(sdvp); + } + } else { + if ((*tvpp)->v_type == VDIR) { + error = SET_ERROR(EISDIR); + goto unlockout; + } + } + } + + vnevent_rename_src(*svpp, sdvp, scnp->cn_nameptr, ct); + if (tzp) + vnevent_rename_dest(*tvpp, tdvp, tnm, ct); + + /* + * notify the target directory if it is not the same + * as source directory. + */ + if (tdvp != sdvp) { + vnevent_rename_dest_dir(tdvp, ct); + } + + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_zap(tx, sdzp->z_id, FALSE, snm); + dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, tnm); + if (sdzp != tdzp) { + dmu_tx_hold_sa(tx, tdzp->z_sa_hdl, B_FALSE); + zfs_sa_upgrade_txholds(tx, tdzp); + } + if (tzp) { + dmu_tx_hold_sa(tx, tzp->z_sa_hdl, B_FALSE); + zfs_sa_upgrade_txholds(tx, tzp); + } + + zfs_sa_upgrade_txholds(tx, szp); + dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + goto unlockout; + } + + + if (tzp) /* Attempt to remove the existing target */ + error = zfs_link_destroy(tdzp, tnm, tzp, tx, 0, NULL); + + if (error == 0) { + error = zfs_link_create(tdzp, tnm, szp, tx, ZRENAMING); + if (error == 0) { + szp->z_pflags |= ZFS_AV_MODIFIED; + + error = sa_update(szp->z_sa_hdl, SA_ZPL_FLAGS(zfsvfs), + (void *)&szp->z_pflags, sizeof (uint64_t), tx); + ASSERT0(error); + + error = zfs_link_destroy(sdzp, snm, szp, tx, ZRENAMING, + NULL); + if (error == 0) { + zfs_log_rename(zilog, tx, TX_RENAME, sdzp, + snm, tdzp, tnm, szp); + + /* + * Update path information for the target vnode + */ + vn_renamepath(tdvp, *svpp, tnm, strlen(tnm)); + } else { + /* + * At this point, we have successfully created + * the target name, but have failed to remove + * the source name. Since the create was done + * with the ZRENAMING flag, there are + * complications; for one, the link count is + * wrong. The easiest way to deal with this + * is to remove the newly created target, and + * return the original error. This must + * succeed; fortunately, it is very unlikely to + * fail, since we just created it. + */ + VERIFY3U(zfs_link_destroy(tdzp, tnm, szp, tx, + ZRENAMING, NULL), ==, 0); + } + } + if (error == 0) { + cache_purge(*svpp); + if (*tvpp != NULL) + cache_purge(*tvpp); + cache_purge_negative(tdvp); + } + } + + dmu_tx_commit(tx); + +unlockout: /* all 4 vnodes are locked, ZFS_ENTER called */ + ZFS_EXIT(zfsvfs); + VOP_UNLOCK1(*svpp); + VOP_UNLOCK1(sdvp); + +out: /* original two vnodes are locked */ + if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zilog, 0); + + if (*tvpp != NULL) + VOP_UNLOCK1(*tvpp); + if (tdvp != *tvpp) + VOP_UNLOCK1(tdvp); + return (error); +} + +int +zfs_rename(znode_t *sdzp, char *sname, znode_t *tdzp, char *tname, + cred_t *cr, int flags) +{ + struct componentname scn, tcn; + vnode_t *sdvp, *tdvp; + vnode_t *svp, *tvp; + int error; + svp = tvp = NULL; + + sdvp = ZTOV(sdzp); + tdvp = ZTOV(tdzp); + error = zfs_lookup_internal(sdzp, sname, &svp, &scn, DELETE); + if (sdzp->z_zfsvfs->z_replay == B_FALSE) + VOP_UNLOCK1(sdvp); + if (error != 0) + goto fail; + VOP_UNLOCK1(svp); + + vn_lock(tdvp, LK_EXCLUSIVE | LK_RETRY); + error = zfs_lookup_internal(tdzp, tname, &tvp, &tcn, RENAME); + if (error == EJUSTRETURN) + tvp = NULL; + else if (error != 0) { + VOP_UNLOCK1(tdvp); + goto fail; + } + + error = zfs_rename_(sdvp, &svp, &scn, tdvp, &tvp, &tcn, cr, 0); +fail: + if (svp != NULL) + vrele(svp); + if (tvp != NULL) + vrele(tvp); + + return (error); +} + +/* + * Insert the indicated symbolic reference entry into the directory. + * + * IN: dvp - Directory to contain new symbolic link. + * link - Name for new symlink entry. + * vap - Attributes of new entry. + * cr - credentials of caller. + * ct - caller context + * flags - case flags + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * dvp - ctime|mtime updated + */ +/*ARGSUSED*/ +int +zfs_symlink(znode_t *dzp, const char *name, vattr_t *vap, + const char *link, znode_t **zpp, cred_t *cr, int flags) +{ + znode_t *zp; + dmu_tx_t *tx; + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + zilog_t *zilog; + uint64_t len = strlen(link); + int error; + zfs_acl_ids_t acl_ids; + boolean_t fuid_dirtied; + uint64_t txtype = TX_SYMLINK; + + ASSERT(vap->va_type == VLNK); + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(dzp); + zilog = zfsvfs->z_log; + + if (zfsvfs->z_utf8 && u8_validate(name, strlen(name), + NULL, U8_VALIDATE_ENTIRE, &error) < 0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EILSEQ)); + } + + if (len > MAXPATHLEN) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(ENAMETOOLONG)); + } + + if ((error = zfs_acl_ids_create(dzp, 0, + vap, cr, NULL, &acl_ids)) != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + + /* + * Attempt to lock directory; fail if entry already exists. + */ + error = zfs_dirent_lookup(dzp, name, &zp, ZNEW); + if (error) { + zfs_acl_ids_free(&acl_ids); + ZFS_EXIT(zfsvfs); + return (error); + } + + if ((error = zfs_zaccess(dzp, ACE_ADD_FILE, 0, B_FALSE, cr))) { + zfs_acl_ids_free(&acl_ids); + ZFS_EXIT(zfsvfs); + return (error); + } + + if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, + 0 /* projid */)) { + zfs_acl_ids_free(&acl_ids); + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EDQUOT)); + } + + getnewvnode_reserve_(); + tx = dmu_tx_create(zfsvfs->z_os); + fuid_dirtied = zfsvfs->z_fuid_dirty; + dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, MAX(1, len)); + dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name); + dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes + + ZFS_SA_BASE_ATTR_SIZE + len); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE); + if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) { + dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, + acl_ids.z_aclp->z_acl_bytes); + } + if (fuid_dirtied) + zfs_fuid_txhold(zfsvfs, tx); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + zfs_acl_ids_free(&acl_ids); + dmu_tx_abort(tx); + getnewvnode_drop_reserve(); + ZFS_EXIT(zfsvfs); + return (error); + } + + /* + * Create a new object for the symlink. + * for version 4 ZPL datsets the symlink will be an SA attribute + */ + zfs_mknode(dzp, vap, tx, cr, 0, &zp, &acl_ids); + + if (fuid_dirtied) + zfs_fuid_sync(zfsvfs, tx); + + if (zp->z_is_sa) + error = sa_update(zp->z_sa_hdl, SA_ZPL_SYMLINK(zfsvfs), + __DECONST(void *, link), len, tx); + else + zfs_sa_symlink(zp, __DECONST(char *, link), len, tx); + + zp->z_size = len; + (void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs), + &zp->z_size, sizeof (zp->z_size), tx); + /* + * Insert the new object into the directory. + */ + (void) zfs_link_create(dzp, name, zp, tx, ZNEW); + + zfs_log_symlink(zilog, tx, txtype, dzp, zp, + __DECONST(char *, name), __DECONST(char *, link)); + *zpp = zp; + + zfs_acl_ids_free(&acl_ids); + + dmu_tx_commit(tx); + + getnewvnode_drop_reserve(); + + if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zilog, 0); + + ZFS_EXIT(zfsvfs); + return (error); +} + +/* + * Return, in the buffer contained in the provided uio structure, + * the symbolic path referred to by vp. + * + * IN: vp - vnode of symbolic link. + * uio - structure to contain the link path. + * cr - credentials of caller. + * ct - caller context + * + * OUT: uio - structure containing the link path. + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * vp - atime updated + */ +/* ARGSUSED */ +static int +zfs_readlink(vnode_t *vp, uio_t *uio, cred_t *cr, caller_context_t *ct) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + int error; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + if (zp->z_is_sa) + error = sa_lookup_uio(zp->z_sa_hdl, + SA_ZPL_SYMLINK(zfsvfs), uio); + else + error = zfs_sa_readlink(zp, uio); + + ZFS_ACCESSTIME_STAMP(zfsvfs, zp); + + ZFS_EXIT(zfsvfs); + return (error); +} + +/* + * Insert a new entry into directory tdvp referencing svp. + * + * IN: tdvp - Directory to contain new entry. + * svp - vnode of new entry. + * name - name of new entry. + * cr - credentials of caller. + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * tdvp - ctime|mtime updated + * svp - ctime updated + */ +/* ARGSUSED */ +int +zfs_link(znode_t *tdzp, znode_t *szp, char *name, cred_t *cr, + int flags) +{ + znode_t *tzp; + zfsvfs_t *zfsvfs = tdzp->z_zfsvfs; + zilog_t *zilog; + dmu_tx_t *tx; + int error; + uint64_t parent; + uid_t owner; + + ASSERT(ZTOV(tdzp)->v_type == VDIR); + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(tdzp); + zilog = zfsvfs->z_log; + + /* + * POSIX dictates that we return EPERM here. + * Better choices include ENOTSUP or EISDIR. + */ + if (ZTOV(szp)->v_type == VDIR) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EPERM)); + } + + ZFS_VERIFY_ZP(szp); + + /* + * If we are using project inheritance, means if the directory has + * ZFS_PROJINHERIT set, then its descendant directories will inherit + * not only the project ID, but also the ZFS_PROJINHERIT flag. Under + * such case, we only allow hard link creation in our tree when the + * project IDs are the same. + */ + if (tdzp->z_pflags & ZFS_PROJINHERIT && + tdzp->z_projid != szp->z_projid) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EXDEV)); + } + + if (szp->z_pflags & (ZFS_APPENDONLY | + ZFS_IMMUTABLE | ZFS_READONLY)) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EPERM)); + } + + /* Prevent links to .zfs/shares files */ + + if ((error = sa_lookup(szp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs), + &parent, sizeof (uint64_t))) != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + if (parent == zfsvfs->z_shares_dir) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EPERM)); + } + + if (zfsvfs->z_utf8 && u8_validate(name, + strlen(name), NULL, U8_VALIDATE_ENTIRE, &error) < 0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EILSEQ)); + } + + /* + * We do not support links between attributes and non-attributes + * because of the potential security risk of creating links + * into "normal" file space in order to circumvent restrictions + * imposed in attribute space. + */ + if ((szp->z_pflags & ZFS_XATTR) != (tdzp->z_pflags & ZFS_XATTR)) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + + owner = zfs_fuid_map_id(zfsvfs, szp->z_uid, cr, ZFS_OWNER); + if (owner != crgetuid(cr) && secpolicy_basic_link(ZTOV(szp), cr) != 0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EPERM)); + } + + if ((error = zfs_zaccess(tdzp, ACE_ADD_FILE, 0, B_FALSE, cr))) { + ZFS_EXIT(zfsvfs); + return (error); + } + + /* + * Attempt to lock directory; fail if entry already exists. + */ + error = zfs_dirent_lookup(tdzp, name, &tzp, ZNEW); + if (error) { + ZFS_EXIT(zfsvfs); + return (error); + } + + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE); + dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, name); + zfs_sa_upgrade_txholds(tx, szp); + zfs_sa_upgrade_txholds(tx, tdzp); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + ZFS_EXIT(zfsvfs); + return (error); + } + + error = zfs_link_create(tdzp, name, szp, tx, 0); + + if (error == 0) { + uint64_t txtype = TX_LINK; + zfs_log_link(zilog, tx, txtype, tdzp, szp, name); + } + + dmu_tx_commit(tx); + + if (error == 0) { + vnevent_link(ZTOV(szp), ct); + } + + if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zilog, 0); + + ZFS_EXIT(zfsvfs); + return (error); +} + +/* + * Free or allocate space in a file. Currently, this function only + * supports the `F_FREESP' command. However, this command is somewhat + * misnamed, as its functionality includes the ability to allocate as + * well as free space. + * + * IN: ip - inode of file to free data in. + * cmd - action to take (only F_FREESP supported). + * bfp - section of file to free/alloc. + * flag - current file open mode flags. + * offset - current file offset. + * cr - credentials of caller. + * + * RETURN: 0 on success, error code on failure. + * + * Timestamps: + * ip - ctime|mtime updated + */ +/* ARGSUSED */ +int +zfs_space(znode_t *zp, int cmd, flock64_t *bfp, int flag, + offset_t offset, cred_t *cr) +{ + zfsvfs_t *zfsvfs = ZTOZSB(zp); + uint64_t off, len; + int error; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + if (cmd != F_FREESP) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + /* + * Callers might not be able to detect properly that we are read-only, + * so check it explicitly here. + */ + if (zfs_is_readonly(zfsvfs)) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EROFS)); + } + + if (bfp->l_len < 0) { + ZFS_EXIT(zfsvfs); + return (SET_ERROR(EINVAL)); + } + + /* + * Permissions aren't checked on Solaris because on this OS + * zfs_space() can only be called with an opened file handle. + * On Linux we can get here through truncate_range() which + * operates directly on inodes, so we need to check access rights. + */ + if ((error = zfs_zaccess(zp, ACE_WRITE_DATA, 0, B_FALSE, cr))) { + ZFS_EXIT(zfsvfs); + return (error); + } + + off = bfp->l_start; + len = bfp->l_len; /* 0 means from off to end of file */ + + error = zfs_freesp(zp, off, len, flag, TRUE); + + ZFS_EXIT(zfsvfs); + return (error); +} + +/*ARGSUSED*/ +void +zfs_inactive(vnode_t *vp, cred_t *cr, caller_context_t *ct) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + int error; + + rw_enter(&zfsvfs->z_teardown_inactive_lock, RW_READER); + if (zp->z_sa_hdl == NULL) { + /* + * The fs has been unmounted, or we did a + * suspend/resume and this file no longer exists. + */ + rw_exit(&zfsvfs->z_teardown_inactive_lock); + vrecycle(vp); + return; + } + + if (zp->z_unlinked) { + /* + * Fast path to recycle a vnode of a removed file. + */ + rw_exit(&zfsvfs->z_teardown_inactive_lock); + vrecycle(vp); + return; + } + + if (zp->z_atime_dirty && zp->z_unlinked == 0) { + dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os); + + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + zfs_sa_upgrade_txholds(tx, zp); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + } else { + (void) sa_update(zp->z_sa_hdl, SA_ZPL_ATIME(zfsvfs), + (void *)&zp->z_atime, sizeof (zp->z_atime), tx); + zp->z_atime_dirty = 0; + dmu_tx_commit(tx); + } + } + rw_exit(&zfsvfs->z_teardown_inactive_lock); +} + + +CTASSERT(sizeof (struct zfid_short) <= sizeof (struct fid)); +CTASSERT(sizeof (struct zfid_long) <= sizeof (struct fid)); + +/*ARGSUSED*/ +static int +zfs_fid(vnode_t *vp, fid_t *fidp, caller_context_t *ct) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + uint32_t gen; + uint64_t gen64; + uint64_t object = zp->z_id; + zfid_short_t *zfid; + int size, i, error; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs), + &gen64, sizeof (uint64_t))) != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + + gen = (uint32_t)gen64; + + size = (zfsvfs->z_parent != zfsvfs) ? LONG_FID_LEN : SHORT_FID_LEN; + fidp->fid_len = size; + + zfid = (zfid_short_t *)fidp; + + zfid->zf_len = size; + + for (i = 0; i < sizeof (zfid->zf_object); i++) + zfid->zf_object[i] = (uint8_t)(object >> (8 * i)); + + /* Must have a non-zero generation number to distinguish from .zfs */ + if (gen == 0) + gen = 1; + for (i = 0; i < sizeof (zfid->zf_gen); i++) + zfid->zf_gen[i] = (uint8_t)(gen >> (8 * i)); + + if (size == LONG_FID_LEN) { + uint64_t objsetid = dmu_objset_id(zfsvfs->z_os); + zfid_long_t *zlfid; + + zlfid = (zfid_long_t *)fidp; + + for (i = 0; i < sizeof (zlfid->zf_setid); i++) + zlfid->zf_setid[i] = (uint8_t)(objsetid >> (8 * i)); + + /* XXX - this should be the generation number for the objset */ + for (i = 0; i < sizeof (zlfid->zf_setgen); i++) + zlfid->zf_setgen[i] = 0; + } + + ZFS_EXIT(zfsvfs); + return (0); +} + +static int +zfs_pathconf(vnode_t *vp, int cmd, ulong_t *valp, cred_t *cr, + caller_context_t *ct) +{ + + switch (cmd) { + case _PC_LINK_MAX: + *valp = MIN(LONG_MAX, ZFS_LINK_MAX); + return (0); + + case _PC_FILESIZEBITS: + *valp = 64; + return (0); + case _PC_MIN_HOLE_SIZE: + *valp = (int)SPA_MINBLOCKSIZE; + return (0); + case _PC_ACL_EXTENDED: + *valp = 0; + return (0); + + case _PC_ACL_NFS4: + *valp = 1; + return (0); + + case _PC_ACL_PATH_MAX: + *valp = ACL_MAX_ENTRIES; + return (0); + + default: + return (EOPNOTSUPP); + } +} + +/*ARGSUSED*/ +static int +zfs_getsecattr(vnode_t *vp, vsecattr_t *vsecp, int flag, cred_t *cr, + caller_context_t *ct) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + int error; + boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + error = zfs_getacl(zp, vsecp, skipaclchk, cr); + ZFS_EXIT(zfsvfs); + + return (error); +} + +/*ARGSUSED*/ +int +zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + int error; + boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE; + zilog_t *zilog = zfsvfs->z_log; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + error = zfs_setacl(zp, vsecp, skipaclchk, cr); + + if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zilog, 0); + + ZFS_EXIT(zfsvfs); + return (error); +} + +static int +zfs_getpages(struct vnode *vp, vm_page_t *ma, int count, int *rbehind, + int *rahead) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + objset_t *os = zp->z_zfsvfs->z_os; + zfs_locked_range_t *lr; + vm_object_t object; + off_t start, end, obj_size; + uint_t blksz; + int pgsin_b, pgsin_a; + int error; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + start = IDX_TO_OFF(ma[0]->pindex); + end = IDX_TO_OFF(ma[count - 1]->pindex + 1); + + /* + * Lock a range covering all required and optional pages. + * Note that we need to handle the case of the block size growing. + */ + for (;;) { + blksz = zp->z_blksz; + lr = zfs_rangelock_enter(&zp->z_rangelock, + rounddown(start, blksz), + roundup(end, blksz) - rounddown(start, blksz), RL_READER); + if (blksz == zp->z_blksz) + break; + zfs_rangelock_exit(lr); + } + + object = ma[0]->object; + zfs_vmobject_wlock(object); + obj_size = object->un_pager.vnp.vnp_size; + zfs_vmobject_wunlock(object); + if (IDX_TO_OFF(ma[count - 1]->pindex) >= obj_size) { + zfs_rangelock_exit(lr); + ZFS_EXIT(zfsvfs); + return (zfs_vm_pagerret_bad); + } + + pgsin_b = 0; + if (rbehind != NULL) { + pgsin_b = OFF_TO_IDX(start - rounddown(start, blksz)); + pgsin_b = MIN(*rbehind, pgsin_b); + } + + pgsin_a = 0; + if (rahead != NULL) { + pgsin_a = OFF_TO_IDX(roundup(end, blksz) - end); + if (end + IDX_TO_OFF(pgsin_a) >= obj_size) + pgsin_a = OFF_TO_IDX(round_page(obj_size) - end); + pgsin_a = MIN(*rahead, pgsin_a); + } + + /* + * NB: we need to pass the exact byte size of the data that we expect + * to read after accounting for the file size. This is required because + * ZFS will panic if we request DMU to read beyond the end of the last + * allocated block. + */ + error = dmu_read_pages(os, zp->z_id, ma, count, &pgsin_b, &pgsin_a, + MIN(end, obj_size) - (end - PAGE_SIZE)); + + zfs_rangelock_exit(lr); + ZFS_ACCESSTIME_STAMP(zfsvfs, zp); + ZFS_EXIT(zfsvfs); + + if (error != 0) + return (zfs_vm_pagerret_error); + + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, count + pgsin_b + pgsin_a); + if (rbehind != NULL) + *rbehind = pgsin_b; + if (rahead != NULL) + *rahead = pgsin_a; + return (zfs_vm_pagerret_ok); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_getpages_args { + struct vnode *a_vp; + vm_page_t *a_m; + int a_count; + int *a_rbehind; + int *a_rahead; +}; +#endif + +static int +zfs_freebsd_getpages(struct vop_getpages_args *ap) +{ + + return (zfs_getpages(ap->a_vp, ap->a_m, ap->a_count, ap->a_rbehind, + ap->a_rahead)); +} + +static int +zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags, + int *rtvals) +{ + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + zfs_locked_range_t *lr; + dmu_tx_t *tx; + struct sf_buf *sf; + vm_object_t object; + vm_page_t m; + caddr_t va; + size_t tocopy; + size_t lo_len; + vm_ooffset_t lo_off; + vm_ooffset_t off; + uint_t blksz; + int ncount; + int pcount; + int err; + int i; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + object = vp->v_object; + pcount = btoc(len); + ncount = pcount; + + KASSERT(ma[0]->object == object, ("mismatching object")); + KASSERT(len > 0 && (len & PAGE_MASK) == 0, ("unexpected length")); + + for (i = 0; i < pcount; i++) + rtvals[i] = zfs_vm_pagerret_error; + + off = IDX_TO_OFF(ma[0]->pindex); + blksz = zp->z_blksz; + lo_off = rounddown(off, blksz); + lo_len = roundup(len + (off - lo_off), blksz); + lr = zfs_rangelock_enter(&zp->z_rangelock, lo_off, lo_len, RL_WRITER); + + zfs_vmobject_wlock(object); + if (len + off > object->un_pager.vnp.vnp_size) { + if (object->un_pager.vnp.vnp_size > off) { + int pgoff; + + len = object->un_pager.vnp.vnp_size - off; + ncount = btoc(len); + if ((pgoff = (int)len & PAGE_MASK) != 0) { + /* + * If the object is locked and the following + * conditions hold, then the page's dirty + * field cannot be concurrently changed by a + * pmap operation. + */ + m = ma[ncount - 1]; + vm_page_assert_sbusied(m); + KASSERT(!pmap_page_is_write_mapped(m), + ("zfs_putpages: page %p is not read-only", + m)); + vm_page_clear_dirty(m, pgoff, PAGE_SIZE - + pgoff); + } + } else { + len = 0; + ncount = 0; + } + if (ncount < pcount) { + for (i = ncount; i < pcount; i++) { + rtvals[i] = zfs_vm_pagerret_bad; + } + } + } + zfs_vmobject_wunlock(object); + + if (ncount == 0) + goto out; + + if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, zp->z_uid) || + zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, zp->z_gid) || + (zp->z_projid != ZFS_DEFAULT_PROJID && + zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT, + zp->z_projid))) { + goto out; + } + + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_write(tx, zp->z_id, off, len); + + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + zfs_sa_upgrade_txholds(tx, zp); + err = dmu_tx_assign(tx, TXG_WAIT); + if (err != 0) { + dmu_tx_abort(tx); + goto out; + } + + if (zp->z_blksz < PAGE_SIZE) { + for (i = 0; len > 0; off += tocopy, len -= tocopy, i++) { + tocopy = len > PAGE_SIZE ? PAGE_SIZE : len; + va = zfs_map_page(ma[i], &sf); + dmu_write(zfsvfs->z_os, zp->z_id, off, tocopy, va, tx); + zfs_unmap_page(sf); + } + } else { + err = dmu_write_pages(zfsvfs->z_os, zp->z_id, off, len, ma, tx); + } + + if (err == 0) { + uint64_t mtime[2], ctime[2]; + sa_bulk_attr_t bulk[3]; + int count = 0; + + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, + &mtime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, + &ctime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, + &zp->z_pflags, 8); + zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); + err = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); + ASSERT0(err); + /* + * XXX we should be passing a callback to undirty + * but that would make the locking messier + */ + zfs_log_write(zfsvfs->z_log, tx, TX_WRITE, zp, off, + len, 0, NULL, NULL); + + zfs_vmobject_wlock(object); + for (i = 0; i < ncount; i++) { + rtvals[i] = zfs_vm_pagerret_ok; + vm_page_undirty(ma[i]); + } + zfs_vmobject_wunlock(object); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, ncount); + } + dmu_tx_commit(tx); + +out: + zfs_rangelock_exit(lr); + if ((flags & (zfs_vm_pagerput_sync | zfs_vm_pagerput_inval)) != 0 || + zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) + zil_commit(zfsvfs->z_log, zp->z_id); + ZFS_EXIT(zfsvfs); + return (rtvals[0]); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_putpages_args { + struct vnode *a_vp; + vm_page_t *a_m; + int a_count; + int a_sync; + int *a_rtvals; +}; +#endif + +int +zfs_freebsd_putpages(struct vop_putpages_args *ap) +{ + + return (zfs_putpages(ap->a_vp, ap->a_m, ap->a_count, ap->a_sync, + ap->a_rtvals)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_bmap_args { + struct vnode *a_vp; + daddr_t a_bn; + struct bufobj **a_bop; + daddr_t *a_bnp; + int *a_runp; + int *a_runb; +}; +#endif + +static int +zfs_freebsd_bmap(struct vop_bmap_args *ap) +{ + + if (ap->a_bop != NULL) + *ap->a_bop = &ap->a_vp->v_bufobj; + if (ap->a_bnp != NULL) + *ap->a_bnp = ap->a_bn; + if (ap->a_runp != NULL) + *ap->a_runp = 0; + if (ap->a_runb != NULL) + *ap->a_runb = 0; + + return (0); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_open_args { + struct vnode *a_vp; + int a_mode; + struct ucred *a_cred; + struct thread *a_td; +}; +#endif + +static int +zfs_freebsd_open(struct vop_open_args *ap) +{ + vnode_t *vp = ap->a_vp; + znode_t *zp = VTOZ(vp); + int error; + + error = zfs_open(&vp, ap->a_mode, ap->a_cred); + if (error == 0) + vnode_create_vobject(vp, zp->z_size, ap->a_td); + return (error); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_close_args { + struct vnode *a_vp; + int a_fflag; + struct ucred *a_cred; + struct thread *a_td; +}; +#endif + +static int +zfs_freebsd_close(struct vop_close_args *ap) +{ + + return (zfs_close(ap->a_vp, ap->a_fflag, 1, 0, ap->a_cred)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_ioctl_args { + struct vnode *a_vp; + ulong_t a_command; + caddr_t a_data; + int a_fflag; + struct ucred *cred; + struct thread *td; +}; +#endif + +static int +zfs_freebsd_ioctl(struct vop_ioctl_args *ap) +{ + + return (zfs_ioctl(ap->a_vp, ap->a_command, (intptr_t)ap->a_data, + ap->a_fflag, ap->a_cred, NULL)); +} + +static int +ioflags(int ioflags) +{ + int flags = 0; + + if (ioflags & IO_APPEND) + flags |= FAPPEND; + if (ioflags & IO_NDELAY) + flags |= FNONBLOCK; + if (ioflags & IO_SYNC) + flags |= (FSYNC | FDSYNC | FRSYNC); + + return (flags); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_read_args { + struct vnode *a_vp; + struct uio *a_uio; + int a_ioflag; + struct ucred *a_cred; +}; +#endif + +static int +zfs_freebsd_read(struct vop_read_args *ap) +{ + + return (zfs_read(ap->a_vp, ap->a_uio, ioflags(ap->a_ioflag), + ap->a_cred)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_write_args { + struct vnode *a_vp; + struct uio *a_uio; + int a_ioflag; + struct ucred *a_cred; +}; +#endif + +static int +zfs_freebsd_write(struct vop_write_args *ap) +{ + + return (zfs_write(ap->a_vp, ap->a_uio, ioflags(ap->a_ioflag), + ap->a_cred)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_access_args { + struct vnode *a_vp; + accmode_t a_accmode; + struct ucred *a_cred; + struct thread *a_td; +}; +#endif + +static int +zfs_freebsd_access(struct vop_access_args *ap) +{ + vnode_t *vp = ap->a_vp; + znode_t *zp = VTOZ(vp); + accmode_t accmode; + int error = 0; + + + if (ap->a_accmode == VEXEC) { + if (zfs_fastaccesschk_execute(zp, ap->a_cred) == 0) + return (0); + } + + /* + * ZFS itself only knowns about VREAD, VWRITE, VEXEC and VAPPEND, + */ + accmode = ap->a_accmode & (VREAD|VWRITE|VEXEC|VAPPEND); + if (accmode != 0) + error = zfs_access(ap->a_vp, accmode, 0, ap->a_cred, NULL); + + /* + * VADMIN has to be handled by vaccess(). + */ + if (error == 0) { + accmode = ap->a_accmode & ~(VREAD|VWRITE|VEXEC|VAPPEND); + if (accmode != 0) { + error = vaccess(vp->v_type, zp->z_mode, zp->z_uid, + zp->z_gid, accmode, ap->a_cred, NULL); + } + } + + /* + * For VEXEC, ensure that at least one execute bit is set for + * non-directories. + */ + if (error == 0 && (ap->a_accmode & VEXEC) != 0 && vp->v_type != VDIR && + (zp->z_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) == 0) { + error = EACCES; + } + + return (error); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_lookup_args { + struct vnode *a_dvp; + struct vnode **a_vpp; + struct componentname *a_cnp; +}; +#endif + +static int +zfs_freebsd_lookup(struct vop_lookup_args *ap, boolean_t cached) +{ + struct componentname *cnp = ap->a_cnp; + char nm[NAME_MAX + 1]; + + ASSERT(cnp->cn_namelen < sizeof (nm)); + strlcpy(nm, cnp->cn_nameptr, MIN(cnp->cn_namelen + 1, sizeof (nm))); + + return (zfs_lookup(ap->a_dvp, nm, ap->a_vpp, cnp, cnp->cn_nameiop, + cnp->cn_cred, cnp->cn_thread, 0, cached)); +} + +static int +zfs_freebsd_cachedlookup(struct vop_cachedlookup_args *ap) +{ + + return (zfs_freebsd_lookup((struct vop_lookup_args *)ap, B_TRUE)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_lookup_args { + struct vnode *a_dvp; + struct vnode **a_vpp; + struct componentname *a_cnp; +}; +#endif + +static int +zfs_cache_lookup(struct vop_lookup_args *ap) +{ + zfsvfs_t *zfsvfs; + + zfsvfs = ap->a_dvp->v_mount->mnt_data; + if (zfsvfs->z_use_namecache) + return (vfs_cache_lookup(ap)); + else + return (zfs_freebsd_lookup(ap, B_FALSE)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_create_args { + struct vnode *a_dvp; + struct vnode **a_vpp; + struct componentname *a_cnp; + struct vattr *a_vap; +}; +#endif + +static int +zfs_freebsd_create(struct vop_create_args *ap) +{ + zfsvfs_t *zfsvfs; + struct componentname *cnp = ap->a_cnp; + vattr_t *vap = ap->a_vap; + znode_t *zp = NULL; + int rc, mode; + + ASSERT(cnp->cn_flags & SAVENAME); + + vattr_init_mask(vap); + mode = vap->va_mode & ALLPERMS; + zfsvfs = ap->a_dvp->v_mount->mnt_data; + *ap->a_vpp = NULL; + + rc = zfs_create(VTOZ(ap->a_dvp), cnp->cn_nameptr, vap, !EXCL, mode, + &zp, cnp->cn_cred, 0 /* flag */, NULL /* vsecattr */); + if (rc == 0) + *ap->a_vpp = ZTOV(zp); + if (zfsvfs->z_use_namecache && + rc == 0 && (cnp->cn_flags & MAKEENTRY) != 0) + cache_enter(ap->a_dvp, *ap->a_vpp, cnp); + + return (rc); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_remove_args { + struct vnode *a_dvp; + struct vnode *a_vp; + struct componentname *a_cnp; +}; +#endif + +static int +zfs_freebsd_remove(struct vop_remove_args *ap) +{ + + ASSERT(ap->a_cnp->cn_flags & SAVENAME); + + return (zfs_remove_(ap->a_dvp, ap->a_vp, ap->a_cnp->cn_nameptr, + ap->a_cnp->cn_cred)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_mkdir_args { + struct vnode *a_dvp; + struct vnode **a_vpp; + struct componentname *a_cnp; + struct vattr *a_vap; +}; +#endif + +static int +zfs_freebsd_mkdir(struct vop_mkdir_args *ap) +{ + vattr_t *vap = ap->a_vap; + znode_t *zp = NULL; + int rc; + + ASSERT(ap->a_cnp->cn_flags & SAVENAME); + + vattr_init_mask(vap); + *ap->a_vpp = NULL; + + rc = zfs_mkdir(VTOZ(ap->a_dvp), ap->a_cnp->cn_nameptr, vap, &zp, + ap->a_cnp->cn_cred, 0, NULL); + + if (rc == 0) + *ap->a_vpp = ZTOV(zp); + return (rc); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_rmdir_args { + struct vnode *a_dvp; + struct vnode *a_vp; + struct componentname *a_cnp; +}; +#endif + +static int +zfs_freebsd_rmdir(struct vop_rmdir_args *ap) +{ + struct componentname *cnp = ap->a_cnp; + + ASSERT(cnp->cn_flags & SAVENAME); + + return (zfs_rmdir_(ap->a_dvp, ap->a_vp, cnp->cn_nameptr, cnp->cn_cred)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_readdir_args { + struct vnode *a_vp; + struct uio *a_uio; + struct ucred *a_cred; + int *a_eofflag; + int *a_ncookies; + ulong_t **a_cookies; +}; +#endif + +static int +zfs_freebsd_readdir(struct vop_readdir_args *ap) +{ + + return (zfs_readdir(ap->a_vp, ap->a_uio, ap->a_cred, ap->a_eofflag, + ap->a_ncookies, ap->a_cookies)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_fsync_args { + struct vnode *a_vp; + int a_waitfor; + struct thread *a_td; +}; +#endif + +static int +zfs_freebsd_fsync(struct vop_fsync_args *ap) +{ + + vop_stdfsync(ap); + return (zfs_fsync(ap->a_vp, 0, ap->a_td->td_ucred, NULL)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_getattr_args { + struct vnode *a_vp; + struct vattr *a_vap; + struct ucred *a_cred; +}; +#endif + +static int +zfs_freebsd_getattr(struct vop_getattr_args *ap) +{ + vattr_t *vap = ap->a_vap; + xvattr_t xvap; + ulong_t fflags = 0; + int error; + + xva_init(&xvap); + xvap.xva_vattr = *vap; + xvap.xva_vattr.va_mask |= AT_XVATTR; + + /* Convert chflags into ZFS-type flags. */ + /* XXX: what about SF_SETTABLE?. */ + XVA_SET_REQ(&xvap, XAT_IMMUTABLE); + XVA_SET_REQ(&xvap, XAT_APPENDONLY); + XVA_SET_REQ(&xvap, XAT_NOUNLINK); + XVA_SET_REQ(&xvap, XAT_NODUMP); + XVA_SET_REQ(&xvap, XAT_READONLY); + XVA_SET_REQ(&xvap, XAT_ARCHIVE); + XVA_SET_REQ(&xvap, XAT_SYSTEM); + XVA_SET_REQ(&xvap, XAT_HIDDEN); + XVA_SET_REQ(&xvap, XAT_REPARSE); + XVA_SET_REQ(&xvap, XAT_OFFLINE); + XVA_SET_REQ(&xvap, XAT_SPARSE); + + error = zfs_getattr(ap->a_vp, (vattr_t *)&xvap, 0, ap->a_cred); + if (error != 0) + return (error); + + /* Convert ZFS xattr into chflags. */ +#define FLAG_CHECK(fflag, xflag, xfield) do { \ + if (XVA_ISSET_RTN(&xvap, (xflag)) && (xfield) != 0) \ + fflags |= (fflag); \ +} while (0) + FLAG_CHECK(SF_IMMUTABLE, XAT_IMMUTABLE, + xvap.xva_xoptattrs.xoa_immutable); + FLAG_CHECK(SF_APPEND, XAT_APPENDONLY, + xvap.xva_xoptattrs.xoa_appendonly); + FLAG_CHECK(SF_NOUNLINK, XAT_NOUNLINK, + xvap.xva_xoptattrs.xoa_nounlink); + FLAG_CHECK(UF_ARCHIVE, XAT_ARCHIVE, + xvap.xva_xoptattrs.xoa_archive); + FLAG_CHECK(UF_NODUMP, XAT_NODUMP, + xvap.xva_xoptattrs.xoa_nodump); + FLAG_CHECK(UF_READONLY, XAT_READONLY, + xvap.xva_xoptattrs.xoa_readonly); + FLAG_CHECK(UF_SYSTEM, XAT_SYSTEM, + xvap.xva_xoptattrs.xoa_system); + FLAG_CHECK(UF_HIDDEN, XAT_HIDDEN, + xvap.xva_xoptattrs.xoa_hidden); + FLAG_CHECK(UF_REPARSE, XAT_REPARSE, + xvap.xva_xoptattrs.xoa_reparse); + FLAG_CHECK(UF_OFFLINE, XAT_OFFLINE, + xvap.xva_xoptattrs.xoa_offline); + FLAG_CHECK(UF_SPARSE, XAT_SPARSE, + xvap.xva_xoptattrs.xoa_sparse); + +#undef FLAG_CHECK + *vap = xvap.xva_vattr; + vap->va_flags = fflags; + return (0); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_setattr_args { + struct vnode *a_vp; + struct vattr *a_vap; + struct ucred *a_cred; +}; +#endif + +static int +zfs_freebsd_setattr(struct vop_setattr_args *ap) +{ + vnode_t *vp = ap->a_vp; + vattr_t *vap = ap->a_vap; + cred_t *cred = ap->a_cred; + xvattr_t xvap; + ulong_t fflags; + uint64_t zflags; + + vattr_init_mask(vap); + vap->va_mask &= ~AT_NOSET; + + xva_init(&xvap); + xvap.xva_vattr = *vap; + + zflags = VTOZ(vp)->z_pflags; + + if (vap->va_flags != VNOVAL) { + zfsvfs_t *zfsvfs = VTOZ(vp)->z_zfsvfs; + int error; + + if (zfsvfs->z_use_fuids == B_FALSE) + return (EOPNOTSUPP); + + fflags = vap->va_flags; + /* + * XXX KDM + * We need to figure out whether it makes sense to allow + * UF_REPARSE through, since we don't really have other + * facilities to handle reparse points and zfs_setattr() + * doesn't currently allow setting that attribute anyway. + */ + if ((fflags & ~(SF_IMMUTABLE|SF_APPEND|SF_NOUNLINK|UF_ARCHIVE| + UF_NODUMP|UF_SYSTEM|UF_HIDDEN|UF_READONLY|UF_REPARSE| + UF_OFFLINE|UF_SPARSE)) != 0) + return (EOPNOTSUPP); + /* + * Unprivileged processes are not permitted to unset system + * flags, or modify flags if any system flags are set. + * Privileged non-jail processes may not modify system flags + * if securelevel > 0 and any existing system flags are set. + * Privileged jail processes behave like privileged non-jail + * processes if the PR_ALLOW_CHFLAGS permission bit is set; + * otherwise, they behave like unprivileged processes. + */ + if (secpolicy_fs_owner(vp->v_mount, cred) == 0 || + spl_priv_check_cred(cred, PRIV_VFS_SYSFLAGS) == 0) { + if (zflags & + (ZFS_IMMUTABLE | ZFS_APPENDONLY | ZFS_NOUNLINK)) { + error = securelevel_gt(cred, 0); + if (error != 0) + return (error); + } + } else { + /* + * Callers may only modify the file flags on + * objects they have VADMIN rights for. + */ + if ((error = VOP_ACCESS(vp, VADMIN, cred, + curthread)) != 0) + return (error); + if (zflags & + (ZFS_IMMUTABLE | ZFS_APPENDONLY | + ZFS_NOUNLINK)) { + return (EPERM); + } + if (fflags & + (SF_IMMUTABLE | SF_APPEND | SF_NOUNLINK)) { + return (EPERM); + } + } + +#define FLAG_CHANGE(fflag, zflag, xflag, xfield) do { \ + if (((fflags & (fflag)) && !(zflags & (zflag))) || \ + ((zflags & (zflag)) && !(fflags & (fflag)))) { \ + XVA_SET_REQ(&xvap, (xflag)); \ + (xfield) = ((fflags & (fflag)) != 0); \ + } \ +} while (0) + /* Convert chflags into ZFS-type flags. */ + /* XXX: what about SF_SETTABLE?. */ + FLAG_CHANGE(SF_IMMUTABLE, ZFS_IMMUTABLE, XAT_IMMUTABLE, + xvap.xva_xoptattrs.xoa_immutable); + FLAG_CHANGE(SF_APPEND, ZFS_APPENDONLY, XAT_APPENDONLY, + xvap.xva_xoptattrs.xoa_appendonly); + FLAG_CHANGE(SF_NOUNLINK, ZFS_NOUNLINK, XAT_NOUNLINK, + xvap.xva_xoptattrs.xoa_nounlink); + FLAG_CHANGE(UF_ARCHIVE, ZFS_ARCHIVE, XAT_ARCHIVE, + xvap.xva_xoptattrs.xoa_archive); + FLAG_CHANGE(UF_NODUMP, ZFS_NODUMP, XAT_NODUMP, + xvap.xva_xoptattrs.xoa_nodump); + FLAG_CHANGE(UF_READONLY, ZFS_READONLY, XAT_READONLY, + xvap.xva_xoptattrs.xoa_readonly); + FLAG_CHANGE(UF_SYSTEM, ZFS_SYSTEM, XAT_SYSTEM, + xvap.xva_xoptattrs.xoa_system); + FLAG_CHANGE(UF_HIDDEN, ZFS_HIDDEN, XAT_HIDDEN, + xvap.xva_xoptattrs.xoa_hidden); + FLAG_CHANGE(UF_REPARSE, ZFS_REPARSE, XAT_REPARSE, + xvap.xva_xoptattrs.xoa_reparse); + FLAG_CHANGE(UF_OFFLINE, ZFS_OFFLINE, XAT_OFFLINE, + xvap.xva_xoptattrs.xoa_offline); + FLAG_CHANGE(UF_SPARSE, ZFS_SPARSE, XAT_SPARSE, + xvap.xva_xoptattrs.xoa_sparse); +#undef FLAG_CHANGE + } + if (vap->va_birthtime.tv_sec != VNOVAL) { + xvap.xva_vattr.va_mask |= AT_XVATTR; + XVA_SET_REQ(&xvap, XAT_CREATETIME); + } + return (zfs_setattr(VTOZ(vp), (vattr_t *)&xvap, 0, cred)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_rename_args { + struct vnode *a_fdvp; + struct vnode *a_fvp; + struct componentname *a_fcnp; + struct vnode *a_tdvp; + struct vnode *a_tvp; + struct componentname *a_tcnp; +}; +#endif + +static int +zfs_freebsd_rename(struct vop_rename_args *ap) +{ + vnode_t *fdvp = ap->a_fdvp; + vnode_t *fvp = ap->a_fvp; + vnode_t *tdvp = ap->a_tdvp; + vnode_t *tvp = ap->a_tvp; + int error; + + ASSERT(ap->a_fcnp->cn_flags & (SAVENAME|SAVESTART)); + ASSERT(ap->a_tcnp->cn_flags & (SAVENAME|SAVESTART)); + + error = zfs_rename_(fdvp, &fvp, ap->a_fcnp, tdvp, &tvp, + ap->a_tcnp, ap->a_fcnp->cn_cred, 1); + + vrele(fdvp); + vrele(fvp); + vrele(tdvp); + if (tvp != NULL) + vrele(tvp); + + return (error); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_symlink_args { + struct vnode *a_dvp; + struct vnode **a_vpp; + struct componentname *a_cnp; + struct vattr *a_vap; + char *a_target; +}; +#endif + +static int +zfs_freebsd_symlink(struct vop_symlink_args *ap) +{ + struct componentname *cnp = ap->a_cnp; + vattr_t *vap = ap->a_vap; + znode_t *zp = NULL; + int rc; + + ASSERT(cnp->cn_flags & SAVENAME); + + vap->va_type = VLNK; /* FreeBSD: Syscall only sets va_mode. */ + vattr_init_mask(vap); + *ap->a_vpp = NULL; + + rc = zfs_symlink(VTOZ(ap->a_dvp), cnp->cn_nameptr, vap, + ap->a_target, &zp, cnp->cn_cred, 0 /* flags */); + if (rc == 0) + *ap->a_vpp = ZTOV(zp); + return (rc); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_readlink_args { + struct vnode *a_vp; + struct uio *a_uio; + struct ucred *a_cred; +}; +#endif + +static int +zfs_freebsd_readlink(struct vop_readlink_args *ap) +{ + + return (zfs_readlink(ap->a_vp, ap->a_uio, ap->a_cred, NULL)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_link_args { + struct vnode *a_tdvp; + struct vnode *a_vp; + struct componentname *a_cnp; +}; +#endif + +static int +zfs_freebsd_link(struct vop_link_args *ap) +{ + struct componentname *cnp = ap->a_cnp; + vnode_t *vp = ap->a_vp; + vnode_t *tdvp = ap->a_tdvp; + + if (tdvp->v_mount != vp->v_mount) + return (EXDEV); + + ASSERT(cnp->cn_flags & SAVENAME); + + return (zfs_link(VTOZ(tdvp), VTOZ(vp), + cnp->cn_nameptr, cnp->cn_cred, 0)); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_inactive_args { + struct vnode *a_vp; + struct thread *a_td; +}; +#endif + +static int +zfs_freebsd_inactive(struct vop_inactive_args *ap) +{ + vnode_t *vp = ap->a_vp; + + zfs_inactive(vp, ap->a_td->td_ucred, NULL); + return (0); +} + +#if __FreeBSD_version >= 1300042 +#ifndef _SYS_SYSPROTO_H_ +struct vop_need_inactive_args { + struct vnode *a_vp; + struct thread *a_td; +}; +#endif + +static int +zfs_freebsd_need_inactive(struct vop_need_inactive_args *ap) +{ + vnode_t *vp = ap->a_vp; + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + bool need; + + if (!rw_tryenter(&zfsvfs->z_teardown_inactive_lock, RW_READER)) + return (true); + need = (zp->z_sa_hdl == NULL || zp->z_unlinked || zp->z_atime_dirty); + rw_exit(&zfsvfs->z_teardown_inactive_lock); + + return (need); +} +#endif + +#ifndef _SYS_SYSPROTO_H_ +struct vop_reclaim_args { + struct vnode *a_vp; + struct thread *a_td; +}; +#endif + +static int +zfs_freebsd_reclaim(struct vop_reclaim_args *ap) +{ + vnode_t *vp = ap->a_vp; + znode_t *zp = VTOZ(vp); + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + + ASSERT(zp != NULL); + +#if __FreeBSD_version < 1300042 + /* Destroy the vm object and flush associated pages. */ + vnode_destroy_vobject(vp); +#endif + /* + * z_teardown_inactive_lock protects from a race with + * zfs_znode_dmu_fini in zfsvfs_teardown during + * force unmount. + */ + rw_enter(&zfsvfs->z_teardown_inactive_lock, RW_READER); + if (zp->z_sa_hdl == NULL) + zfs_znode_free(zp); + else + zfs_zinactive(zp); + rw_exit(&zfsvfs->z_teardown_inactive_lock); + + vp->v_data = NULL; + return (0); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_fid_args { + struct vnode *a_vp; + struct fid *a_fid; +}; +#endif + +static int +zfs_freebsd_fid(struct vop_fid_args *ap) +{ + + return (zfs_fid(ap->a_vp, (void *)ap->a_fid, NULL)); +} + + +#ifndef _SYS_SYSPROTO_H_ +struct vop_pathconf_args { + struct vnode *a_vp; + int a_name; + register_t *a_retval; +} *ap; +#endif + +static int +zfs_freebsd_pathconf(struct vop_pathconf_args *ap) +{ + ulong_t val; + int error; + + error = zfs_pathconf(ap->a_vp, ap->a_name, &val, + curthread->td_ucred, NULL); + if (error == 0) { + *ap->a_retval = val; + return (error); + } + if (error != EOPNOTSUPP) + return (error); + + switch (ap->a_name) { + case _PC_NAME_MAX: + *ap->a_retval = NAME_MAX; + return (0); + case _PC_PIPE_BUF: + if (ap->a_vp->v_type == VDIR || ap->a_vp->v_type == VFIFO) { + *ap->a_retval = PIPE_BUF; + return (0); + } + return (EINVAL); + default: + return (vop_stdpathconf(ap)); + } +} + +/* + * FreeBSD's extended attributes namespace defines file name prefix for ZFS' + * extended attribute name: + * + * NAMESPACE PREFIX + * system freebsd:system: + * user (none, can be used to access ZFS fsattr(5) attributes + * created on Solaris) + */ +static int +zfs_create_attrname(int attrnamespace, const char *name, char *attrname, + size_t size) +{ + const char *namespace, *prefix, *suffix; + + /* We don't allow '/' character in attribute name. */ + if (strchr(name, '/') != NULL) + return (EINVAL); + /* We don't allow attribute names that start with "freebsd:" string. */ + if (strncmp(name, "freebsd:", 8) == 0) + return (EINVAL); + + bzero(attrname, size); + + switch (attrnamespace) { + case EXTATTR_NAMESPACE_USER: +#if 0 + prefix = "freebsd:"; + namespace = EXTATTR_NAMESPACE_USER_STRING; + suffix = ":"; +#else + /* + * This is the default namespace by which we can access all + * attributes created on Solaris. + */ + prefix = namespace = suffix = ""; +#endif + break; + case EXTATTR_NAMESPACE_SYSTEM: + prefix = "freebsd:"; + namespace = EXTATTR_NAMESPACE_SYSTEM_STRING; + suffix = ":"; + break; + case EXTATTR_NAMESPACE_EMPTY: + default: + return (EINVAL); + } + if (snprintf(attrname, size, "%s%s%s%s", prefix, namespace, suffix, + name) >= size) { + return (ENAMETOOLONG); + } + return (0); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_getextattr { + IN struct vnode *a_vp; + IN int a_attrnamespace; + IN const char *a_name; + INOUT struct uio *a_uio; + OUT size_t *a_size; + IN struct ucred *a_cred; + IN struct thread *a_td; +}; +#endif + +/* + * Vnode operating to retrieve a named extended attribute. + */ +static int +zfs_getextattr(struct vop_getextattr_args *ap) +{ + zfsvfs_t *zfsvfs = VTOZ(ap->a_vp)->z_zfsvfs; + struct thread *td = ap->a_td; + struct nameidata nd; + char attrname[255]; + struct vattr va; + vnode_t *xvp = NULL, *vp; + int error, flags; + + /* + * If the xattr property is off, refuse the request. + */ + if (!(zfsvfs->z_flags & ZSB_XATTR)) { + return (SET_ERROR(EOPNOTSUPP)); + } + + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, + ap->a_cred, ap->a_td, VREAD); + if (error != 0) + return (error); + + error = zfs_create_attrname(ap->a_attrnamespace, ap->a_name, attrname, + sizeof (attrname)); + if (error != 0) + return (error); + + ZFS_ENTER(zfsvfs); + + error = zfs_lookup(ap->a_vp, NULL, &xvp, NULL, 0, ap->a_cred, td, + LOOKUP_XATTR, B_FALSE); + if (error != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + + flags = FREAD; + NDINIT_ATVP(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, attrname, + xvp, td); + error = vn_open_cred(&nd, &flags, VN_OPEN_INVFS, 0, ap->a_cred, NULL); + vp = nd.ni_vp; + NDFREE(&nd, NDF_ONLY_PNBUF); + if (error != 0) { + ZFS_EXIT(zfsvfs); + if (error == ENOENT) + error = ENOATTR; + return (error); + } + + if (ap->a_size != NULL) { + error = VOP_GETATTR(vp, &va, ap->a_cred); + if (error == 0) + *ap->a_size = (size_t)va.va_size; + } else if (ap->a_uio != NULL) + error = VOP_READ(vp, ap->a_uio, IO_UNIT, ap->a_cred); + + VOP_UNLOCK1(vp); + vn_close(vp, flags, ap->a_cred, td); + ZFS_EXIT(zfsvfs); + return (error); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_deleteextattr { + IN struct vnode *a_vp; + IN int a_attrnamespace; + IN const char *a_name; + IN struct ucred *a_cred; + IN struct thread *a_td; +}; +#endif + +/* + * Vnode operation to remove a named attribute. + */ +int +zfs_deleteextattr(struct vop_deleteextattr_args *ap) +{ + zfsvfs_t *zfsvfs = VTOZ(ap->a_vp)->z_zfsvfs; + struct thread *td = ap->a_td; + struct nameidata nd; + char attrname[255]; + vnode_t *xvp = NULL, *vp; + int error; + + /* + * If the xattr property is off, refuse the request. + */ + if (!(zfsvfs->z_flags & ZSB_XATTR)) { + return (SET_ERROR(EOPNOTSUPP)); + } + + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, + ap->a_cred, ap->a_td, VWRITE); + if (error != 0) + return (error); + + error = zfs_create_attrname(ap->a_attrnamespace, ap->a_name, attrname, + sizeof (attrname)); + if (error != 0) + return (error); + + ZFS_ENTER(zfsvfs); + + error = zfs_lookup(ap->a_vp, NULL, &xvp, NULL, 0, ap->a_cred, td, + LOOKUP_XATTR, B_FALSE); + if (error != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + + NDINIT_ATVP(&nd, DELETE, NOFOLLOW | LOCKPARENT | LOCKLEAF, + UIO_SYSSPACE, attrname, xvp, td); + error = namei(&nd); + vp = nd.ni_vp; + if (error != 0) { + ZFS_EXIT(zfsvfs); + NDFREE(&nd, NDF_ONLY_PNBUF); + if (error == ENOENT) + error = ENOATTR; + return (error); + } + + error = VOP_REMOVE(nd.ni_dvp, vp, &nd.ni_cnd); + NDFREE(&nd, NDF_ONLY_PNBUF); + + vput(nd.ni_dvp); + if (vp == nd.ni_dvp) + vrele(vp); + else + vput(vp); + ZFS_EXIT(zfsvfs); + + return (error); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_setextattr { + IN struct vnode *a_vp; + IN int a_attrnamespace; + IN const char *a_name; + INOUT struct uio *a_uio; + IN struct ucred *a_cred; + IN struct thread *a_td; +}; +#endif + +/* + * Vnode operation to set a named attribute. + */ +static int +zfs_setextattr(struct vop_setextattr_args *ap) +{ + zfsvfs_t *zfsvfs = VTOZ(ap->a_vp)->z_zfsvfs; + struct thread *td = ap->a_td; + struct nameidata nd; + char attrname[255]; + struct vattr va; + vnode_t *xvp = NULL, *vp; + int error, flags; + + /* + * If the xattr property is off, refuse the request. + */ + if (!(zfsvfs->z_flags & ZSB_XATTR)) { + return (SET_ERROR(EOPNOTSUPP)); + } + + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, + ap->a_cred, ap->a_td, VWRITE); + if (error != 0) + return (error); + error = zfs_create_attrname(ap->a_attrnamespace, ap->a_name, attrname, + sizeof (attrname)); + if (error != 0) + return (error); + + ZFS_ENTER(zfsvfs); + + error = zfs_lookup(ap->a_vp, NULL, &xvp, NULL, 0, ap->a_cred, td, + LOOKUP_XATTR | CREATE_XATTR_DIR, B_FALSE); + if (error != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + + flags = FFLAGS(O_WRONLY | O_CREAT); + NDINIT_ATVP(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, attrname, + xvp, td); + error = vn_open_cred(&nd, &flags, 0600, VN_OPEN_INVFS, ap->a_cred, + NULL); + vp = nd.ni_vp; + NDFREE(&nd, NDF_ONLY_PNBUF); + if (error != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + + VATTR_NULL(&va); + va.va_size = 0; + error = VOP_SETATTR(vp, &va, ap->a_cred); + if (error == 0) + VOP_WRITE(vp, ap->a_uio, IO_UNIT, ap->a_cred); + + VOP_UNLOCK1(vp); + vn_close(vp, flags, ap->a_cred, td); + ZFS_EXIT(zfsvfs); + return (error); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_listextattr { + IN struct vnode *a_vp; + IN int a_attrnamespace; + INOUT struct uio *a_uio; + OUT size_t *a_size; + IN struct ucred *a_cred; + IN struct thread *a_td; +}; +#endif + +/* + * Vnode operation to retrieve extended attributes on a vnode. + */ +static int +zfs_listextattr(struct vop_listextattr_args *ap) +{ + zfsvfs_t *zfsvfs = VTOZ(ap->a_vp)->z_zfsvfs; + struct thread *td = ap->a_td; + struct nameidata nd; + char attrprefix[16]; + uint8_t dirbuf[sizeof (struct dirent)]; + struct dirent *dp; + struct iovec aiov; + struct uio auio, *uio = ap->a_uio; + size_t *sizep = ap->a_size; + size_t plen; + vnode_t *xvp = NULL, *vp; + int done, error, eof, pos; + + /* + * If the xattr property is off, refuse the request. + */ + if (!(zfsvfs->z_flags & ZSB_XATTR)) { + return (SET_ERROR(EOPNOTSUPP)); + } + + error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, + ap->a_cred, ap->a_td, VREAD); + if (error != 0) + return (error); + + error = zfs_create_attrname(ap->a_attrnamespace, "", attrprefix, + sizeof (attrprefix)); + if (error != 0) + return (error); + plen = strlen(attrprefix); + + ZFS_ENTER(zfsvfs); + + if (sizep != NULL) + *sizep = 0; + + error = zfs_lookup(ap->a_vp, NULL, &xvp, NULL, 0, ap->a_cred, td, + LOOKUP_XATTR, B_FALSE); + if (error != 0) { + ZFS_EXIT(zfsvfs); + /* + * ENOATTR means that the EA directory does not yet exist, + * i.e. there are no extended attributes there. + */ + if (error == ENOATTR) + error = 0; + return (error); + } + + NDINIT_ATVP(&nd, LOOKUP, NOFOLLOW | LOCKLEAF | LOCKSHARED, + UIO_SYSSPACE, ".", xvp, td); + error = namei(&nd); + vp = nd.ni_vp; + NDFREE(&nd, NDF_ONLY_PNBUF); + if (error != 0) { + ZFS_EXIT(zfsvfs); + return (error); + } + + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; + auio.uio_segflg = UIO_SYSSPACE; + auio.uio_td = td; + auio.uio_rw = UIO_READ; + auio.uio_offset = 0; + + do { + uint8_t nlen; + + aiov.iov_base = (void *)dirbuf; + aiov.iov_len = sizeof (dirbuf); + auio.uio_resid = sizeof (dirbuf); + error = VOP_READDIR(vp, &auio, ap->a_cred, &eof, NULL, NULL); + done = sizeof (dirbuf) - auio.uio_resid; + if (error != 0) + break; + for (pos = 0; pos < done; ) { + dp = (struct dirent *)(dirbuf + pos); + pos += dp->d_reclen; + /* + * XXX: Temporarily we also accept DT_UNKNOWN, as this + * is what we get when attribute was created on Solaris. + */ + if (dp->d_type != DT_REG && dp->d_type != DT_UNKNOWN) + continue; + if (plen == 0 && + strncmp(dp->d_name, "freebsd:", 8) == 0) + continue; + else if (strncmp(dp->d_name, attrprefix, plen) != 0) + continue; + nlen = dp->d_namlen - plen; + if (sizep != NULL) + *sizep += 1 + nlen; + else if (uio != NULL) { + /* + * Format of extattr name entry is one byte for + * length and the rest for name. + */ + error = uiomove(&nlen, 1, uio->uio_rw, uio); + if (error == 0) { + error = uiomove(dp->d_name + plen, nlen, + uio->uio_rw, uio); + } + if (error != 0) + break; + } + } + } while (!eof && error == 0); + + vput(vp); + ZFS_EXIT(zfsvfs); + + return (error); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_getacl_args { + struct vnode *vp; + acl_type_t type; + struct acl *aclp; + struct ucred *cred; + struct thread *td; +}; +#endif + +int +zfs_freebsd_getacl(struct vop_getacl_args *ap) +{ + int error; + vsecattr_t vsecattr; + + if (ap->a_type != ACL_TYPE_NFS4) + return (EINVAL); + + vsecattr.vsa_mask = VSA_ACE | VSA_ACECNT; + if ((error = zfs_getsecattr(ap->a_vp, &vsecattr, 0, ap->a_cred, NULL))) + return (error); + + error = acl_from_aces(ap->a_aclp, vsecattr.vsa_aclentp, + vsecattr.vsa_aclcnt); + if (vsecattr.vsa_aclentp != NULL) + kmem_free(vsecattr.vsa_aclentp, vsecattr.vsa_aclentsz); + + return (error); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_setacl_args { + struct vnode *vp; + acl_type_t type; + struct acl *aclp; + struct ucred *cred; + struct thread *td; +}; +#endif + +int +zfs_freebsd_setacl(struct vop_setacl_args *ap) +{ + int error; + vsecattr_t vsecattr; + int aclbsize; /* size of acl list in bytes */ + aclent_t *aaclp; + + if (ap->a_type != ACL_TYPE_NFS4) + return (EINVAL); + + if (ap->a_aclp == NULL) + return (EINVAL); + + if (ap->a_aclp->acl_cnt < 1 || ap->a_aclp->acl_cnt > MAX_ACL_ENTRIES) + return (EINVAL); + + /* + * With NFSv4 ACLs, chmod(2) may need to add additional entries, + * splitting every entry into two and appending "canonical six" + * entries at the end. Don't allow for setting an ACL that would + * cause chmod(2) to run out of ACL entries. + */ + if (ap->a_aclp->acl_cnt * 2 + 6 > ACL_MAX_ENTRIES) + return (ENOSPC); + + error = acl_nfs4_check(ap->a_aclp, ap->a_vp->v_type == VDIR); + if (error != 0) + return (error); + + vsecattr.vsa_mask = VSA_ACE; + aclbsize = ap->a_aclp->acl_cnt * sizeof (ace_t); + vsecattr.vsa_aclentp = kmem_alloc(aclbsize, KM_SLEEP); + aaclp = vsecattr.vsa_aclentp; + vsecattr.vsa_aclentsz = aclbsize; + + aces_from_acl(vsecattr.vsa_aclentp, &vsecattr.vsa_aclcnt, ap->a_aclp); + error = zfs_setsecattr(VTOZ(ap->a_vp), &vsecattr, 0, ap->a_cred); + kmem_free(aaclp, aclbsize); + + return (error); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vop_aclcheck_args { + struct vnode *vp; + acl_type_t type; + struct acl *aclp; + struct ucred *cred; + struct thread *td; +}; +#endif + +int +zfs_freebsd_aclcheck(struct vop_aclcheck_args *ap) +{ + + return (EOPNOTSUPP); +} + +static int +zfs_vptocnp(struct vop_vptocnp_args *ap) +{ + vnode_t *covered_vp; + vnode_t *vp = ap->a_vp; + zfsvfs_t *zfsvfs = vp->v_vfsp->vfs_data; + znode_t *zp = VTOZ(vp); + int ltype; + int error; + + ZFS_ENTER(zfsvfs); + ZFS_VERIFY_ZP(zp); + + /* + * If we are a snapshot mounted under .zfs, run the operation + * on the covered vnode. + */ + if (zp->z_id != zfsvfs->z_root || zfsvfs->z_parent == zfsvfs) { + char name[MAXNAMLEN + 1]; + znode_t *dzp; + size_t len; + + error = zfs_znode_parent_and_name(zp, &dzp, name); + if (error == 0) { + len = strlen(name); + if (*ap->a_buflen < len) + error = SET_ERROR(ENOMEM); + } + if (error == 0) { + *ap->a_buflen -= len; + bcopy(name, ap->a_buf + *ap->a_buflen, len); + *ap->a_vpp = ZTOV(dzp); + } + ZFS_EXIT(zfsvfs); + return (error); + } + ZFS_EXIT(zfsvfs); + + covered_vp = vp->v_mount->mnt_vnodecovered; +#if __FreeBSD_version >= 1300045 + enum vgetstate vs = vget_prep(covered_vp); +#else + vhold(covered_vp); +#endif + ltype = VOP_ISLOCKED(vp); + VOP_UNLOCK1(vp); +#if __FreeBSD_version >= 1300045 + error = vget_finish(covered_vp, LK_SHARED, vs); +#else + error = vget(covered_vp, LK_SHARED | LK_VNHELD, curthread); +#endif + if (error == 0) { + error = VOP_VPTOCNP(covered_vp, ap->a_vpp, ap->a_cred, + ap->a_buf, ap->a_buflen); + vput(covered_vp); + } + vn_lock(vp, ltype | LK_RETRY); + if (VN_IS_DOOMED(vp)) + error = SET_ERROR(ENOENT); + return (error); +} + +#ifdef DIAGNOSTIC +#ifndef _SYS_SYSPROTO_H_ +struct vop_lock1_args { + struct vnode *a_vp; + int a_flags; + char *file; + int line; +}; +#endif + +static int +zfs_lock(struct vop_lock1_args *ap) +{ + vnode_t *vp; + znode_t *zp; + int err; + +#if __FreeBSD_version >= 1300064 + err = vop_lock(ap); +#else + err = vop_stdlock(ap); +#endif + if (err == 0 && (ap->a_flags & LK_NOWAIT) == 0) { + vp = ap->a_vp; + zp = vp->v_data; + if (vp->v_mount != NULL && !VN_IS_DOOMED(vp) && + zp != NULL && (zp->z_pflags & ZFS_XATTR) == 0) + VERIFY(!RRM_LOCK_HELD(&zp->z_zfsvfs->z_teardown_lock)); + } + return (err); +} +#endif + +struct vop_vector zfs_vnodeops; +struct vop_vector zfs_fifoops; +struct vop_vector zfs_shareops; + +struct vop_vector zfs_vnodeops = { + .vop_default = &default_vnodeops, + .vop_inactive = zfs_freebsd_inactive, +#if __FreeBSD_version >= 1300042 + .vop_need_inactive = zfs_freebsd_need_inactive, +#endif + .vop_reclaim = zfs_freebsd_reclaim, + .vop_access = zfs_freebsd_access, + .vop_allocate = VOP_EINVAL, + .vop_lookup = zfs_cache_lookup, + .vop_cachedlookup = zfs_freebsd_cachedlookup, + .vop_getattr = zfs_freebsd_getattr, + .vop_setattr = zfs_freebsd_setattr, + .vop_create = zfs_freebsd_create, + .vop_mknod = (vop_mknod_t *)zfs_freebsd_create, + .vop_mkdir = zfs_freebsd_mkdir, + .vop_readdir = zfs_freebsd_readdir, + .vop_fsync = zfs_freebsd_fsync, + .vop_open = zfs_freebsd_open, + .vop_close = zfs_freebsd_close, + .vop_rmdir = zfs_freebsd_rmdir, + .vop_ioctl = zfs_freebsd_ioctl, + .vop_link = zfs_freebsd_link, + .vop_symlink = zfs_freebsd_symlink, + .vop_readlink = zfs_freebsd_readlink, + .vop_read = zfs_freebsd_read, + .vop_write = zfs_freebsd_write, + .vop_remove = zfs_freebsd_remove, + .vop_rename = zfs_freebsd_rename, + .vop_pathconf = zfs_freebsd_pathconf, + .vop_bmap = zfs_freebsd_bmap, + .vop_fid = zfs_freebsd_fid, + .vop_getextattr = zfs_getextattr, + .vop_deleteextattr = zfs_deleteextattr, + .vop_setextattr = zfs_setextattr, + .vop_listextattr = zfs_listextattr, + .vop_getacl = zfs_freebsd_getacl, + .vop_setacl = zfs_freebsd_setacl, + .vop_aclcheck = zfs_freebsd_aclcheck, + .vop_getpages = zfs_freebsd_getpages, + .vop_putpages = zfs_freebsd_putpages, + .vop_vptocnp = zfs_vptocnp, +#if __FreeBSD_version >= 1300064 +#ifdef DIAGNOSTIC + .vop_lock1 = zfs_lock, +#else + .vop_lock1 = vop_lock, +#endif + .vop_unlock = vop_unlock, + .vop_islocked = vop_islocked, +#else +#ifdef DIAGNOSTIC + .vop_lock1 = zfs_lock, +#endif +#endif +}; +VFS_VOP_VECTOR_REGISTER(zfs_vnodeops); + +struct vop_vector zfs_fifoops = { + .vop_default = &fifo_specops, + .vop_fsync = zfs_freebsd_fsync, + .vop_access = zfs_freebsd_access, + .vop_getattr = zfs_freebsd_getattr, + .vop_inactive = zfs_freebsd_inactive, + .vop_read = VOP_PANIC, + .vop_reclaim = zfs_freebsd_reclaim, + .vop_setattr = zfs_freebsd_setattr, + .vop_write = VOP_PANIC, + .vop_pathconf = zfs_freebsd_pathconf, + .vop_fid = zfs_freebsd_fid, + .vop_getacl = zfs_freebsd_getacl, + .vop_setacl = zfs_freebsd_setacl, + .vop_aclcheck = zfs_freebsd_aclcheck, +}; +VFS_VOP_VECTOR_REGISTER(zfs_fifoops); + +/* + * special share hidden files vnode operations template + */ +struct vop_vector zfs_shareops = { + .vop_default = &default_vnodeops, + .vop_access = zfs_freebsd_access, + .vop_inactive = zfs_freebsd_inactive, + .vop_reclaim = zfs_freebsd_reclaim, + .vop_fid = zfs_freebsd_fid, + .vop_pathconf = zfs_freebsd_pathconf, +}; +VFS_VOP_VECTOR_REGISTER(zfs_shareops); diff --git a/module/os/freebsd/zfs/zfs_znode.c b/module/os/freebsd/zfs/zfs_znode.c new file mode 100644 index 00000000000..b9d5472b25b --- /dev/null +++ b/module/os/freebsd/zfs/zfs_znode.c @@ -0,0 +1,1987 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2014 Integros [integros.com] + */ + +/* Portions Copyright 2007 Jeremy Teo */ +/* Portions Copyright 2011 Martin Matuska */ + +#ifdef _KERNEL +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif /* _KERNEL */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zfs_prop.h" +#include "zfs_comutil.h" + +/* Used by fstat(1). */ +SYSCTL_INT(_debug_sizeof, OID_AUTO, znode, CTLFLAG_RD, + SYSCTL_NULL_INT_PTR, sizeof (znode_t), "sizeof(znode_t)"); + +/* + * Define ZNODE_STATS to turn on statistic gathering. By default, it is only + * turned on when DEBUG is also defined. + */ +#ifdef DEBUG +#define ZNODE_STATS +#endif /* DEBUG */ + +#ifdef ZNODE_STATS +#define ZNODE_STAT_ADD(stat) ((stat)++) +#else +#define ZNODE_STAT_ADD(stat) /* nothing */ +#endif /* ZNODE_STATS */ + +/* + * Functions needed for userland (ie: libzpool) are not put under + * #ifdef_KERNEL; the rest of the functions have dependencies + * (such as VFS logic) that will not compile easily in userland. + */ +#ifdef _KERNEL +/* + * Needed to close a small window in zfs_znode_move() that allows the zfsvfs to + * be freed before it can be safely accessed. + */ +krwlock_t zfsvfs_lock; + +static kmem_cache_t *znode_cache = NULL; + +extern struct vop_vector zfs_vnodeops; +extern struct vop_vector zfs_fifoops; +extern struct vop_vector zfs_shareops; + + +/* + * This callback is invoked when acquiring a RL_WRITER or RL_APPEND lock on + * z_rangelock. It will modify the offset and length of the lock to reflect + * znode-specific information, and convert RL_APPEND to RL_WRITER. This is + * called with the rangelock_t's rl_lock held, which avoids races. + */ +static void +zfs_rangelock_cb(zfs_locked_range_t *new, void *arg) +{ + znode_t *zp = arg; + + /* + * If in append mode, convert to writer and lock starting at the + * current end of file. + */ + if (new->lr_type == RL_APPEND) { + new->lr_offset = zp->z_size; + new->lr_type = RL_WRITER; + } + + /* + * If we need to grow the block size then lock the whole file range. + */ + uint64_t end_size = MAX(zp->z_size, new->lr_offset + new->lr_length); + if (end_size > zp->z_blksz && (!ISP2(zp->z_blksz) || + zp->z_blksz < ZTOZSB(zp)->z_max_blksz)) { + new->lr_offset = 0; + new->lr_length = UINT64_MAX; + } +} + +static int +zfs_znode_cache_constructor(void *buf, void *arg, int kmflags) +{ + znode_t *zp = buf; + + POINTER_INVALIDATE(&zp->z_zfsvfs); + + list_link_init(&zp->z_link_node); + + mutex_init(&zp->z_acl_lock, NULL, MUTEX_DEFAULT, NULL); + + zfs_rangelock_init(&zp->z_rangelock, zfs_rangelock_cb, zp); + + zp->z_acl_cached = NULL; + zp->z_vnode = NULL; + zp->z_moved = 0; + return (0); +} + +/*ARGSUSED*/ +static void +zfs_znode_cache_destructor(void *buf, void *arg) +{ + znode_t *zp = buf; + + ASSERT(!POINTER_IS_VALID(zp->z_zfsvfs)); + ASSERT3P(zp->z_vnode, ==, NULL); + ASSERT(!list_link_active(&zp->z_link_node)); + mutex_destroy(&zp->z_acl_lock); + zfs_rangelock_fini(&zp->z_rangelock); + + ASSERT(zp->z_acl_cached == NULL); +} + +void +zfs_znode_init(void) +{ + /* + * Initialize zcache + */ + rw_init(&zfsvfs_lock, NULL, RW_DEFAULT, NULL); + ASSERT(znode_cache == NULL); + znode_cache = kmem_cache_create("zfs_znode_cache", + sizeof (znode_t), 0, zfs_znode_cache_constructor, + zfs_znode_cache_destructor, NULL, NULL, NULL, 0); + // kmem_cache_set_move(znode_cache, zfs_znode_move); +} + +void +zfs_znode_fini(void) +{ + /* + * Cleanup zcache + */ + if (znode_cache) + kmem_cache_destroy(znode_cache); + znode_cache = NULL; + rw_destroy(&zfsvfs_lock); +} + + +static int +zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx) +{ + zfs_acl_ids_t acl_ids; + vattr_t vattr; + znode_t *sharezp; + znode_t *zp; + int error; + + vattr.va_mask = AT_MODE|AT_UID|AT_GID; + vattr.va_type = VDIR; + vattr.va_mode = S_IFDIR|0555; + vattr.va_uid = crgetuid(kcred); + vattr.va_gid = crgetgid(kcred); + + sharezp = kmem_cache_alloc(znode_cache, KM_SLEEP); + ASSERT(!POINTER_IS_VALID(sharezp->z_zfsvfs)); + sharezp->z_moved = 0; + sharezp->z_unlinked = 0; + sharezp->z_atime_dirty = 0; + sharezp->z_zfsvfs = zfsvfs; + sharezp->z_is_sa = zfsvfs->z_use_sa; + + VERIFY(0 == zfs_acl_ids_create(sharezp, IS_ROOT_NODE, &vattr, + kcred, NULL, &acl_ids)); + zfs_mknode(sharezp, &vattr, tx, kcred, IS_ROOT_NODE, &zp, &acl_ids); + ASSERT3P(zp, ==, sharezp); + POINTER_INVALIDATE(&sharezp->z_zfsvfs); + error = zap_add(zfsvfs->z_os, MASTER_NODE_OBJ, + ZFS_SHARES_DIR, 8, 1, &sharezp->z_id, tx); + zfsvfs->z_shares_dir = sharezp->z_id; + + zfs_acl_ids_free(&acl_ids); + sa_handle_destroy(sharezp->z_sa_hdl); + kmem_cache_free(znode_cache, sharezp); + + return (error); +} + +/* + * define a couple of values we need available + * for both 64 and 32 bit environments. + */ +#ifndef NBITSMINOR64 +#define NBITSMINOR64 32 +#endif +#ifndef MAXMAJ64 +#define MAXMAJ64 0xffffffffUL +#endif +#ifndef MAXMIN64 +#define MAXMIN64 0xffffffffUL +#endif + +/* + * Create special expldev for ZFS private use. + * Can't use standard expldev since it doesn't do + * what we want. The standard expldev() takes a + * dev32_t in LP64 and expands it to a long dev_t. + * We need an interface that takes a dev32_t in ILP32 + * and expands it to a long dev_t. + */ +static uint64_t +zfs_expldev(dev_t dev) +{ + return (((uint64_t)major(dev) << NBITSMINOR64) | minor(dev)); +} +/* + * Special cmpldev for ZFS private use. + * Can't use standard cmpldev since it takes + * a long dev_t and compresses it to dev32_t in + * LP64. We need to do a compaction of a long dev_t + * to a dev32_t in ILP32. + */ +dev_t +zfs_cmpldev(uint64_t dev) +{ + return (makedev((dev >> NBITSMINOR64), (dev & MAXMIN64))); +} + +static void +zfs_znode_sa_init(zfsvfs_t *zfsvfs, znode_t *zp, + dmu_buf_t *db, dmu_object_type_t obj_type, sa_handle_t *sa_hdl) +{ + ASSERT(!POINTER_IS_VALID(zp->z_zfsvfs) || (zfsvfs == zp->z_zfsvfs)); + ASSERT(MUTEX_HELD(ZFS_OBJ_MUTEX(zfsvfs, zp->z_id))); + + ASSERT(zp->z_sa_hdl == NULL); + ASSERT(zp->z_acl_cached == NULL); + if (sa_hdl == NULL) { + VERIFY(0 == sa_handle_get_from_db(zfsvfs->z_os, db, zp, + SA_HDL_SHARED, &zp->z_sa_hdl)); + } else { + zp->z_sa_hdl = sa_hdl; + sa_set_userp(sa_hdl, zp); + } + + zp->z_is_sa = (obj_type == DMU_OT_SA) ? B_TRUE : B_FALSE; + + /* + * Slap on VROOT if we are the root znode unless we are the root + * node of a snapshot mounted under .zfs. + */ + if (zp->z_id == zfsvfs->z_root && zfsvfs->z_parent == zfsvfs) + ZTOV(zp)->v_flag |= VROOT; + + vn_exists(ZTOV(zp)); +} + +void +zfs_znode_dmu_fini(znode_t *zp) +{ + ASSERT(MUTEX_HELD(ZFS_OBJ_MUTEX(zp->z_zfsvfs, zp->z_id)) || + zp->z_unlinked || + RW_WRITE_HELD(&zp->z_zfsvfs->z_teardown_inactive_lock)); + + sa_handle_destroy(zp->z_sa_hdl); + zp->z_sa_hdl = NULL; +} + +static void +zfs_vnode_forget(vnode_t *vp) +{ + + /* copied from insmntque_stddtr */ + vp->v_data = NULL; + vp->v_op = &dead_vnodeops; + vgone(vp); + vput(vp); +} + +/* + * Construct a new znode/vnode and intialize. + * + * This does not do a call to dmu_set_user() that is + * up to the caller to do, in case you don't want to + * return the znode + */ +static znode_t * +zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz, + dmu_object_type_t obj_type, sa_handle_t *hdl) +{ + znode_t *zp; + vnode_t *vp; + uint64_t mode; + uint64_t parent; +#ifdef notyet + uint64_t mtime[2], ctime[2]; +#endif + uint64_t projid = ZFS_DEFAULT_PROJID; + sa_bulk_attr_t bulk[9]; + int count = 0; + int error; + + zp = kmem_cache_alloc(znode_cache, KM_SLEEP); + +#if __FreeBSD_version >= 1300076 + KASSERT(curthread->td_vp_reserved != NULL, + ("zfs_znode_alloc: getnewvnode without any vnodes reserved")); +#else + KASSERT(curthread->td_vp_reserv > 0, + ("zfs_znode_alloc: getnewvnode without any vnodes reserved")); +#endif + error = getnewvnode("zfs", zfsvfs->z_parent->z_vfs, &zfs_vnodeops, &vp); + if (error != 0) { + kmem_cache_free(znode_cache, zp); + return (NULL); + } + zp->z_vnode = vp; + vp->v_data = zp; + + ASSERT(!POINTER_IS_VALID(zp->z_zfsvfs)); + zp->z_moved = 0; + + /* + * Defer setting z_zfsvfs until the znode is ready to be a candidate for + * the zfs_znode_move() callback. + */ + zp->z_sa_hdl = NULL; + zp->z_unlinked = 0; + zp->z_atime_dirty = 0; + zp->z_mapcnt = 0; + zp->z_id = db->db_object; + zp->z_blksz = blksz; + zp->z_seq = 0x7A4653; + zp->z_sync_cnt = 0; + + vp = ZTOV(zp); + + zfs_znode_sa_init(zfsvfs, zp, db, obj_type, hdl); + + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL, &mode, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GEN(zfsvfs), NULL, &zp->z_gen, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL, + &zp->z_size, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL, + &zp->z_links, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, + &zp->z_pflags, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zfsvfs), NULL, &parent, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL, + &zp->z_atime, 16); +#ifdef notyet + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, + &mtime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, + &ctime, 16); +#endif + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL, + &zp->z_uid, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), NULL, + &zp->z_gid, 8); + + if (sa_bulk_lookup(zp->z_sa_hdl, bulk, count) != 0 || zp->z_gen == 0 || + (dmu_objset_projectquota_enabled(zfsvfs->z_os) && + (zp->z_pflags & ZFS_PROJID) && + sa_lookup(zp->z_sa_hdl, SA_ZPL_PROJID(zfsvfs), &projid, 8) != 0)) { + if (hdl == NULL) + sa_handle_destroy(zp->z_sa_hdl); + zfs_vnode_forget(vp); + zp->z_vnode = NULL; + kmem_cache_free(znode_cache, zp); + return (NULL); + } + + zp->z_projid = projid; + zp->z_mode = mode; + + /* Cache the xattr parent id */ + if (zp->z_pflags & ZFS_XATTR) + zp->z_xattr_parent = parent; + + vp->v_type = IFTOVT((mode_t)mode); + + switch (vp->v_type) { + case VDIR: + zp->z_zn_prefetch = B_TRUE; /* z_prefetch default is enabled */ + break; + case VFIFO: + vp->v_op = &zfs_fifoops; + break; + case VREG: + if (parent == zfsvfs->z_shares_dir) { + ASSERT(zp->z_uid == 0 && zp->z_gid == 0); + vp->v_op = &zfs_shareops; + } + break; + default: + break; + } + + mutex_enter(&zfsvfs->z_znodes_lock); + list_insert_tail(&zfsvfs->z_all_znodes, zp); + zfsvfs->z_nr_znodes++; + membar_producer(); + /* + * Everything else must be valid before assigning z_zfsvfs makes the + * znode eligible for zfs_znode_move(). + */ + zp->z_zfsvfs = zfsvfs; + mutex_exit(&zfsvfs->z_znodes_lock); + + /* + * Acquire vnode lock before making it available to the world. + */ + vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); + VN_LOCK_AREC(vp); + if (vp->v_type != VFIFO) + VN_LOCK_ASHARE(vp); + + return (zp); +} + +static uint64_t empty_xattr; +static uint64_t pad[4]; +static zfs_acl_phys_t acl_phys; +/* + * Create a new DMU object to hold a zfs znode. + * + * IN: dzp - parent directory for new znode + * vap - file attributes for new znode + * tx - dmu transaction id for zap operations + * cr - credentials of caller + * flag - flags: + * IS_ROOT_NODE - new object will be root + * IS_XATTR - new object is an attribute + * bonuslen - length of bonus buffer + * setaclp - File/Dir initial ACL + * fuidp - Tracks fuid allocation. + * + * OUT: zpp - allocated znode + * + */ +void +zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr, + uint_t flag, znode_t **zpp, zfs_acl_ids_t *acl_ids) +{ + uint64_t crtime[2], atime[2], mtime[2], ctime[2]; + uint64_t mode, size, links, parent, pflags; + uint64_t dzp_pflags = 0; + uint64_t rdev = 0; + zfsvfs_t *zfsvfs = dzp->z_zfsvfs; + dmu_buf_t *db; + timestruc_t now; + uint64_t gen, obj; + int err; + int bonuslen; + int dnodesize; + sa_handle_t *sa_hdl; + dmu_object_type_t obj_type; + sa_bulk_attr_t *sa_attrs; + int cnt = 0; + zfs_acl_locator_cb_t locate = { 0 }; + + ASSERT(vap && ((vap->va_mask & AT_MODE) == AT_MODE)); + + if (zfsvfs->z_replay) { + obj = vap->va_nodeid; + now = vap->va_ctime; /* see zfs_replay_create() */ + gen = vap->va_nblocks; /* ditto */ + dnodesize = vap->va_fsid; /* ditto */ + } else { + obj = 0; + vfs_timestamp(&now); + gen = dmu_tx_get_txg(tx); + dnodesize = dmu_objset_dnodesize(zfsvfs->z_os); + } + + if (dnodesize == 0) + dnodesize = DNODE_MIN_SIZE; + + obj_type = zfsvfs->z_use_sa ? DMU_OT_SA : DMU_OT_ZNODE; + bonuslen = (obj_type == DMU_OT_SA) ? + DN_BONUS_SIZE(dnodesize) : ZFS_OLD_ZNODE_PHYS_SIZE; + + /* + * Create a new DMU object. + */ + /* + * There's currently no mechanism for pre-reading the blocks that will + * be needed to allocate a new object, so we accept the small chance + * that there will be an i/o error and we will fail one of the + * assertions below. + */ + if (vap->va_type == VDIR) { + if (zfsvfs->z_replay) { + VERIFY0(zap_create_claim_norm_dnsize(zfsvfs->z_os, obj, + zfsvfs->z_norm, DMU_OT_DIRECTORY_CONTENTS, + obj_type, bonuslen, dnodesize, tx)); + } else { + obj = zap_create_norm_dnsize(zfsvfs->z_os, + zfsvfs->z_norm, DMU_OT_DIRECTORY_CONTENTS, + obj_type, bonuslen, dnodesize, tx); + } + } else { + if (zfsvfs->z_replay) { + VERIFY0(dmu_object_claim_dnsize(zfsvfs->z_os, obj, + DMU_OT_PLAIN_FILE_CONTENTS, 0, + obj_type, bonuslen, dnodesize, tx)); + } else { + obj = dmu_object_alloc_dnsize(zfsvfs->z_os, + DMU_OT_PLAIN_FILE_CONTENTS, 0, + obj_type, bonuslen, dnodesize, tx); + } + } + + ZFS_OBJ_HOLD_ENTER(zfsvfs, obj); + VERIFY(0 == sa_buf_hold(zfsvfs->z_os, obj, NULL, &db)); + + /* + * If this is the root, fix up the half-initialized parent pointer + * to reference the just-allocated physical data area. + */ + if (flag & IS_ROOT_NODE) { + dzp->z_id = obj; + } else { + dzp_pflags = dzp->z_pflags; + } + + /* + * If parent is an xattr, so am I. + */ + if (dzp_pflags & ZFS_XATTR) { + flag |= IS_XATTR; + } + + if (zfsvfs->z_use_fuids) + pflags = ZFS_ARCHIVE | ZFS_AV_MODIFIED; + else + pflags = 0; + + if (vap->va_type == VDIR) { + size = 2; /* contents ("." and "..") */ + links = (flag & (IS_ROOT_NODE | IS_XATTR)) ? 2 : 1; + } else { + size = links = 0; + } + + if (vap->va_type == VBLK || vap->va_type == VCHR) { + rdev = zfs_expldev(vap->va_rdev); + } + + parent = dzp->z_id; + mode = acl_ids->z_mode; + if (flag & IS_XATTR) + pflags |= ZFS_XATTR; + + /* + * No execs denied will be deterimed when zfs_mode_compute() is called. + */ + pflags |= acl_ids->z_aclp->z_hints & + (ZFS_ACL_TRIVIAL|ZFS_INHERIT_ACE|ZFS_ACL_AUTO_INHERIT| + ZFS_ACL_DEFAULTED|ZFS_ACL_PROTECTED); + + ZFS_TIME_ENCODE(&now, crtime); + ZFS_TIME_ENCODE(&now, ctime); + + if (vap->va_mask & AT_ATIME) { + ZFS_TIME_ENCODE(&vap->va_atime, atime); + } else { + ZFS_TIME_ENCODE(&now, atime); + } + + if (vap->va_mask & AT_MTIME) { + ZFS_TIME_ENCODE(&vap->va_mtime, mtime); + } else { + ZFS_TIME_ENCODE(&now, mtime); + } + + /* Now add in all of the "SA" attributes */ + VERIFY(0 == sa_handle_get_from_db(zfsvfs->z_os, db, NULL, SA_HDL_SHARED, + &sa_hdl)); + + /* + * Setup the array of attributes to be replaced/set on the new file + * + * order for DMU_OT_ZNODE is critical since it needs to be constructed + * in the old znode_phys_t format. Don't change this ordering + */ + sa_attrs = kmem_alloc(sizeof (sa_bulk_attr_t) * ZPL_END, KM_SLEEP); + + if (obj_type == DMU_OT_ZNODE) { + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_ATIME(zfsvfs), + NULL, &atime, 16); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_MTIME(zfsvfs), + NULL, &mtime, 16); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_CTIME(zfsvfs), + NULL, &ctime, 16); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_CRTIME(zfsvfs), + NULL, &crtime, 16); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_GEN(zfsvfs), + NULL, &gen, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_MODE(zfsvfs), + NULL, &mode, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_SIZE(zfsvfs), + NULL, &size, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_PARENT(zfsvfs), + NULL, &parent, 8); + } else { + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_MODE(zfsvfs), + NULL, &mode, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_SIZE(zfsvfs), + NULL, &size, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_GEN(zfsvfs), + NULL, &gen, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_UID(zfsvfs), + NULL, &acl_ids->z_fuid, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_GID(zfsvfs), + NULL, &acl_ids->z_fgid, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_PARENT(zfsvfs), + NULL, &parent, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_FLAGS(zfsvfs), + NULL, &pflags, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_ATIME(zfsvfs), + NULL, &atime, 16); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_MTIME(zfsvfs), + NULL, &mtime, 16); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_CTIME(zfsvfs), + NULL, &ctime, 16); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_CRTIME(zfsvfs), + NULL, &crtime, 16); + } + + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_LINKS(zfsvfs), NULL, &links, 8); + + if (obj_type == DMU_OT_ZNODE) { + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_XATTR(zfsvfs), NULL, + &empty_xattr, 8); + } + if (obj_type == DMU_OT_ZNODE || + (vap->va_type == VBLK || vap->va_type == VCHR)) { + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_RDEV(zfsvfs), + NULL, &rdev, 8); + + } + if (obj_type == DMU_OT_ZNODE) { + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_FLAGS(zfsvfs), + NULL, &pflags, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_UID(zfsvfs), NULL, + &acl_ids->z_fuid, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_GID(zfsvfs), NULL, + &acl_ids->z_fgid, 8); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_PAD(zfsvfs), NULL, pad, + sizeof (uint64_t) * 4); + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_ZNODE_ACL(zfsvfs), NULL, + &acl_phys, sizeof (zfs_acl_phys_t)); + } else if (acl_ids->z_aclp->z_version >= ZFS_ACL_VERSION_FUID) { + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_DACL_COUNT(zfsvfs), NULL, + &acl_ids->z_aclp->z_acl_count, 8); + locate.cb_aclp = acl_ids->z_aclp; + SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_DACL_ACES(zfsvfs), + zfs_acl_data_locator, &locate, + acl_ids->z_aclp->z_acl_bytes); + mode = zfs_mode_compute(mode, acl_ids->z_aclp, &pflags, + acl_ids->z_fuid, acl_ids->z_fgid); + } + + VERIFY(sa_replace_all_by_template(sa_hdl, sa_attrs, cnt, tx) == 0); + + if (!(flag & IS_ROOT_NODE)) { + *zpp = zfs_znode_alloc(zfsvfs, db, 0, obj_type, sa_hdl); + ASSERT(*zpp != NULL); + } else { + /* + * If we are creating the root node, the "parent" we + * passed in is the znode for the root. + */ + *zpp = dzp; + + (*zpp)->z_sa_hdl = sa_hdl; + } + + (*zpp)->z_pflags = pflags; + (*zpp)->z_mode = mode; + (*zpp)->z_dnodesize = dnodesize; + + if (vap->va_mask & AT_XVATTR) + zfs_xvattr_set(*zpp, (xvattr_t *)vap, tx); + + if (obj_type == DMU_OT_ZNODE || + acl_ids->z_aclp->z_version < ZFS_ACL_VERSION_FUID) { + VERIFY0(zfs_aclset_common(*zpp, acl_ids->z_aclp, cr, tx)); + } + if (!(flag & IS_ROOT_NODE)) { + vnode_t *vp; + + vp = ZTOV(*zpp); + vp->v_vflag |= VV_FORCEINSMQ; + err = insmntque(vp, zfsvfs->z_vfs); + vp->v_vflag &= ~VV_FORCEINSMQ; + KASSERT(err == 0, ("insmntque() failed: error %d", err)); + } + kmem_free(sa_attrs, sizeof (sa_bulk_attr_t) * ZPL_END); + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj); +} + +/* + * Update in-core attributes. It is assumed the caller will be doing an + * sa_bulk_update to push the changes out. + */ +void +zfs_xvattr_set(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx) +{ + xoptattr_t *xoap; + + xoap = xva_getxoptattr(xvap); + ASSERT(xoap); + + if (XVA_ISSET_REQ(xvap, XAT_CREATETIME)) { + uint64_t times[2]; + ZFS_TIME_ENCODE(&xoap->xoa_createtime, times); + (void) sa_update(zp->z_sa_hdl, SA_ZPL_CRTIME(zp->z_zfsvfs), + ×, sizeof (times), tx); + XVA_SET_RTN(xvap, XAT_CREATETIME); + } + if (XVA_ISSET_REQ(xvap, XAT_READONLY)) { + ZFS_ATTR_SET(zp, ZFS_READONLY, xoap->xoa_readonly, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_READONLY); + } + if (XVA_ISSET_REQ(xvap, XAT_HIDDEN)) { + ZFS_ATTR_SET(zp, ZFS_HIDDEN, xoap->xoa_hidden, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_HIDDEN); + } + if (XVA_ISSET_REQ(xvap, XAT_SYSTEM)) { + ZFS_ATTR_SET(zp, ZFS_SYSTEM, xoap->xoa_system, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_SYSTEM); + } + if (XVA_ISSET_REQ(xvap, XAT_ARCHIVE)) { + ZFS_ATTR_SET(zp, ZFS_ARCHIVE, xoap->xoa_archive, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_ARCHIVE); + } + if (XVA_ISSET_REQ(xvap, XAT_IMMUTABLE)) { + ZFS_ATTR_SET(zp, ZFS_IMMUTABLE, xoap->xoa_immutable, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_IMMUTABLE); + } + if (XVA_ISSET_REQ(xvap, XAT_NOUNLINK)) { + ZFS_ATTR_SET(zp, ZFS_NOUNLINK, xoap->xoa_nounlink, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_NOUNLINK); + } + if (XVA_ISSET_REQ(xvap, XAT_APPENDONLY)) { + ZFS_ATTR_SET(zp, ZFS_APPENDONLY, xoap->xoa_appendonly, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_APPENDONLY); + } + if (XVA_ISSET_REQ(xvap, XAT_NODUMP)) { + ZFS_ATTR_SET(zp, ZFS_NODUMP, xoap->xoa_nodump, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_NODUMP); + } + if (XVA_ISSET_REQ(xvap, XAT_OPAQUE)) { + ZFS_ATTR_SET(zp, ZFS_OPAQUE, xoap->xoa_opaque, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_OPAQUE); + } + if (XVA_ISSET_REQ(xvap, XAT_AV_QUARANTINED)) { + ZFS_ATTR_SET(zp, ZFS_AV_QUARANTINED, + xoap->xoa_av_quarantined, zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_AV_QUARANTINED); + } + if (XVA_ISSET_REQ(xvap, XAT_AV_MODIFIED)) { + ZFS_ATTR_SET(zp, ZFS_AV_MODIFIED, xoap->xoa_av_modified, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_AV_MODIFIED); + } + if (XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP)) { + zfs_sa_set_scanstamp(zp, xvap, tx); + XVA_SET_RTN(xvap, XAT_AV_SCANSTAMP); + } + if (XVA_ISSET_REQ(xvap, XAT_REPARSE)) { + ZFS_ATTR_SET(zp, ZFS_REPARSE, xoap->xoa_reparse, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_REPARSE); + } + if (XVA_ISSET_REQ(xvap, XAT_OFFLINE)) { + ZFS_ATTR_SET(zp, ZFS_OFFLINE, xoap->xoa_offline, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_OFFLINE); + } + if (XVA_ISSET_REQ(xvap, XAT_SPARSE)) { + ZFS_ATTR_SET(zp, ZFS_SPARSE, xoap->xoa_sparse, + zp->z_pflags, tx); + XVA_SET_RTN(xvap, XAT_SPARSE); + } +} + +int +zfs_zget(zfsvfs_t *zfsvfs, uint64_t obj_num, znode_t **zpp) +{ + dmu_object_info_t doi; + dmu_buf_t *db; + znode_t *zp; + vnode_t *vp; + sa_handle_t *hdl; + struct thread *td; + int locked; + int err; + + td = curthread; + getnewvnode_reserve_(); +again: + *zpp = NULL; + ZFS_OBJ_HOLD_ENTER(zfsvfs, obj_num); + + err = sa_buf_hold(zfsvfs->z_os, obj_num, NULL, &db); + if (err) { + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + getnewvnode_drop_reserve(); + return (err); + } + + dmu_object_info_from_db(db, &doi); + if (doi.doi_bonus_type != DMU_OT_SA && + (doi.doi_bonus_type != DMU_OT_ZNODE || + (doi.doi_bonus_type == DMU_OT_ZNODE && + doi.doi_bonus_size < sizeof (znode_phys_t)))) { + sa_buf_rele(db, NULL); + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + getnewvnode_drop_reserve(); + return (SET_ERROR(EINVAL)); + } + + hdl = dmu_buf_get_user(db); + if (hdl != NULL) { + zp = sa_get_userdata(hdl); + + /* + * Since "SA" does immediate eviction we + * should never find a sa handle that doesn't + * know about the znode. + */ + ASSERT3P(zp, !=, NULL); + ASSERT3U(zp->z_id, ==, obj_num); + if (zp->z_unlinked) { + err = SET_ERROR(ENOENT); + } else { + vp = ZTOV(zp); + /* + * Don't let the vnode disappear after + * ZFS_OBJ_HOLD_EXIT. + */ + VN_HOLD(vp); + *zpp = zp; + err = 0; + } + + sa_buf_rele(db, NULL); + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + + if (err) { + getnewvnode_drop_reserve(); + return (err); + } + + locked = VOP_ISLOCKED(vp); + VI_LOCK(vp); + if (VN_IS_DOOMED(vp) && locked != LK_EXCLUSIVE) { + /* + * The vnode is doomed and this thread doesn't + * hold the exclusive lock on it, so the vnode + * must be being reclaimed by another thread. + * Otherwise the doomed vnode is being reclaimed + * by this thread and zfs_zget is called from + * ZIL internals. + */ + VI_UNLOCK(vp); + + /* + * XXX vrele() locks the vnode when the last reference + * is dropped. Although in this case the vnode is + * doomed / dead and so no inactivation is required, + * the vnode lock is still acquired. That could result + * in a LOR with z_teardown_lock if another thread holds + * the vnode's lock and tries to take z_teardown_lock. + * But that is only possible if the other thread peforms + * a ZFS vnode operation on the vnode. That either + * should not happen if the vnode is dead or the thread + * should also have a refrence to the vnode and thus + * our reference is not last. + */ + VN_RELE(vp); + goto again; + } + VI_UNLOCK(vp); + getnewvnode_drop_reserve(); + return (err); + } + + /* + * Not found create new znode/vnode + * but only if file exists. + * + * There is a small window where zfs_vget() could + * find this object while a file create is still in + * progress. This is checked for in zfs_znode_alloc() + * + * if zfs_znode_alloc() fails it will drop the hold on the + * bonus buffer. + */ + zp = zfs_znode_alloc(zfsvfs, db, doi.doi_data_block_size, + doi.doi_bonus_type, NULL); + if (zp == NULL) { + err = SET_ERROR(ENOENT); + } else { + *zpp = zp; + } + if (err == 0) { + vnode_t *vp = ZTOV(zp); + + err = insmntque(vp, zfsvfs->z_vfs); + if (err == 0) { + vp->v_hash = obj_num; + VOP_UNLOCK1(vp); + } else { + zp->z_vnode = NULL; + zfs_znode_dmu_fini(zp); + zfs_znode_free(zp); + *zpp = NULL; + } + } + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + getnewvnode_drop_reserve(); + return (err); +} + +int +zfs_rezget(znode_t *zp) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + dmu_object_info_t doi; + dmu_buf_t *db; + vnode_t *vp; + uint64_t obj_num = zp->z_id; + uint64_t mode, size; + sa_bulk_attr_t bulk[8]; + int err; + int count = 0; + uint64_t gen; + + /* + * Remove cached pages before reloading the znode, so that they are not + * lingering after we run into any error. Ideally, we should vgone() + * the vnode in case of error, but currently we cannot do that + * because of the LOR between the vnode lock and z_teardown_lock. + * So, instead, we have to "doom" the znode in the illumos style. + */ + vp = ZTOV(zp); + vn_pages_remove(vp, 0, 0); + + ZFS_OBJ_HOLD_ENTER(zfsvfs, obj_num); + + mutex_enter(&zp->z_acl_lock); + if (zp->z_acl_cached) { + zfs_acl_free(zp->z_acl_cached); + zp->z_acl_cached = NULL; + } + + mutex_exit(&zp->z_acl_lock); + ASSERT(zp->z_sa_hdl == NULL); + err = sa_buf_hold(zfsvfs->z_os, obj_num, NULL, &db); + if (err) { + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + return (err); + } + + dmu_object_info_from_db(db, &doi); + if (doi.doi_bonus_type != DMU_OT_SA && + (doi.doi_bonus_type != DMU_OT_ZNODE || + (doi.doi_bonus_type == DMU_OT_ZNODE && + doi.doi_bonus_size < sizeof (znode_phys_t)))) { + sa_buf_rele(db, NULL); + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + return (SET_ERROR(EINVAL)); + } + + zfs_znode_sa_init(zfsvfs, zp, db, doi.doi_bonus_type, NULL); + size = zp->z_size; + + /* reload cached values */ + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GEN(zfsvfs), NULL, + &gen, sizeof (gen)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL, + &zp->z_size, sizeof (zp->z_size)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL, + &zp->z_links, sizeof (zp->z_links)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, + &zp->z_pflags, sizeof (zp->z_pflags)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL, + &zp->z_atime, sizeof (zp->z_atime)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL, + &zp->z_uid, sizeof (zp->z_uid)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), NULL, + &zp->z_gid, sizeof (zp->z_gid)); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL, + &mode, sizeof (mode)); + + if (sa_bulk_lookup(zp->z_sa_hdl, bulk, count)) { + zfs_znode_dmu_fini(zp); + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + return (SET_ERROR(EIO)); + } + + zp->z_mode = mode; + + if (gen != zp->z_gen) { + zfs_znode_dmu_fini(zp); + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + return (SET_ERROR(EIO)); + } + + /* + * It is highly improbable but still quite possible that two + * objects in different datasets are created with the same + * object numbers and in transaction groups with the same + * numbers. znodes corresponding to those objects would + * have the same z_id and z_gen, but their other attributes + * may be different. + * zfs recv -F may replace one of such objects with the other. + * As a result file properties recorded in the replaced + * object's vnode may no longer match the received object's + * properties. At present the only cached property is the + * files type recorded in v_type. + * So, handle this case by leaving the old vnode and znode + * disassociated from the actual object. A new vnode and a + * znode will be created if the object is accessed + * (e.g. via a look-up). The old vnode and znode will be + * recycled when the last vnode reference is dropped. + */ + if (vp->v_type != IFTOVT((mode_t)zp->z_mode)) { + zfs_znode_dmu_fini(zp); + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + return (SET_ERROR(EIO)); + } + + /* + * If the file has zero links, then it has been unlinked on the send + * side and it must be in the received unlinked set. + * We call zfs_znode_dmu_fini() now to prevent any accesses to the + * stale data and to prevent automatical removal of the file in + * zfs_zinactive(). The file will be removed either when it is removed + * on the send side and the next incremental stream is received or + * when the unlinked set gets processed. + */ + zp->z_unlinked = (zp->z_links == 0); + if (zp->z_unlinked) { + zfs_znode_dmu_fini(zp); + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + return (0); + } + + zp->z_blksz = doi.doi_data_block_size; + if (zp->z_size != size) + vnode_pager_setsize(vp, zp->z_size); + + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj_num); + + return (0); +} + +void +zfs_znode_delete(znode_t *zp, dmu_tx_t *tx) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + objset_t *os = zfsvfs->z_os; + uint64_t obj = zp->z_id; + uint64_t acl_obj = zfs_external_acl(zp); + + ZFS_OBJ_HOLD_ENTER(zfsvfs, obj); + if (acl_obj) { + VERIFY(!zp->z_is_sa); + VERIFY(0 == dmu_object_free(os, acl_obj, tx)); + } + VERIFY(0 == dmu_object_free(os, obj, tx)); + zfs_znode_dmu_fini(zp); + ZFS_OBJ_HOLD_EXIT(zfsvfs, obj); + zfs_znode_free(zp); +} + +void +zfs_zinactive(znode_t *zp) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + uint64_t z_id = zp->z_id; + + ASSERT(zp->z_sa_hdl); + + /* + * Don't allow a zfs_zget() while were trying to release this znode + */ + ZFS_OBJ_HOLD_ENTER(zfsvfs, z_id); + + /* + * If this was the last reference to a file with no links, remove + * the file from the file system unless the file system is mounted + * read-only. That can happen, for example, if the file system was + * originally read-write, the file was opened, then unlinked and + * the file system was made read-only before the file was finally + * closed. The file will remain in the unlinked set. + */ + if (zp->z_unlinked) { + ASSERT(!zfsvfs->z_issnap); + if ((zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) == 0) { + ZFS_OBJ_HOLD_EXIT(zfsvfs, z_id); + zfs_rmnode(zp); + return; + } + } + + zfs_znode_dmu_fini(zp); + ZFS_OBJ_HOLD_EXIT(zfsvfs, z_id); + zfs_znode_free(zp); +} + +void +zfs_znode_free(znode_t *zp) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + + ASSERT(zp->z_sa_hdl == NULL); + zp->z_vnode = NULL; + mutex_enter(&zfsvfs->z_znodes_lock); + POINTER_INVALIDATE(&zp->z_zfsvfs); + list_remove(&zfsvfs->z_all_znodes, zp); + zfsvfs->z_nr_znodes--; + mutex_exit(&zfsvfs->z_znodes_lock); + + if (zp->z_acl_cached) { + zfs_acl_free(zp->z_acl_cached); + zp->z_acl_cached = NULL; + } + + kmem_cache_free(znode_cache, zp); + +} + +void +zfs_tstamp_update_setup_ext(znode_t *zp, uint_t flag, uint64_t mtime[2], + uint64_t ctime[2], boolean_t have_tx) +{ + timestruc_t now; + + vfs_timestamp(&now); + + if (have_tx) { /* will sa_bulk_update happen really soon? */ + zp->z_atime_dirty = 0; + zp->z_seq++; + } else { + zp->z_atime_dirty = 1; + } + + if (flag & AT_ATIME) { + ZFS_TIME_ENCODE(&now, zp->z_atime); + } + + if (flag & AT_MTIME) { + ZFS_TIME_ENCODE(&now, mtime); + if (zp->z_zfsvfs->z_use_fuids) { + zp->z_pflags |= (ZFS_ARCHIVE | + ZFS_AV_MODIFIED); + } + } + + if (flag & AT_CTIME) { + ZFS_TIME_ENCODE(&now, ctime); + if (zp->z_zfsvfs->z_use_fuids) + zp->z_pflags |= ZFS_ARCHIVE; + } +} + + +void +zfs_tstamp_update_setup(znode_t *zp, uint_t flag, uint64_t mtime[2], + uint64_t ctime[2]) +{ + zfs_tstamp_update_setup_ext(zp, flag, mtime, ctime, B_TRUE); +} +/* + * Grow the block size for a file. + * + * IN: zp - znode of file to free data in. + * size - requested block size + * tx - open transaction. + * + * NOTE: this function assumes that the znode is write locked. + */ +void +zfs_grow_blocksize(znode_t *zp, uint64_t size, dmu_tx_t *tx) +{ + int error; + u_longlong_t dummy; + + if (size <= zp->z_blksz) + return; + /* + * If the file size is already greater than the current blocksize, + * we will not grow. If there is more than one block in a file, + * the blocksize cannot change. + */ + if (zp->z_blksz && zp->z_size > zp->z_blksz) + return; + + error = dmu_object_set_blocksize(zp->z_zfsvfs->z_os, zp->z_id, + size, 0, tx); + + if (error == ENOTSUP) + return; + ASSERT0(error); + + /* What blocksize did we actually get? */ + dmu_object_size_from_db(sa_get_db(zp->z_sa_hdl), &zp->z_blksz, &dummy); +} + +/* + * Increase the file length + * + * IN: zp - znode of file to free data in. + * end - new end-of-file + * + * RETURN: 0 on success, error code on failure + */ +static int +zfs_extend(znode_t *zp, uint64_t end) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + dmu_tx_t *tx; + zfs_locked_range_t *lr; + uint64_t newblksz; + int error; + + /* + * We will change zp_size, lock the whole file. + */ + lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_WRITER); + + /* + * Nothing to do if file already at desired length. + */ + if (end <= zp->z_size) { + zfs_rangelock_exit(lr); + return (0); + } + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + zfs_sa_upgrade_txholds(tx, zp); + if (end > zp->z_blksz && + (!ISP2(zp->z_blksz) || zp->z_blksz < zfsvfs->z_max_blksz)) { + /* + * We are growing the file past the current block size. + */ + if (zp->z_blksz > zp->z_zfsvfs->z_max_blksz) { + /* + * File's blocksize is already larger than the + * "recordsize" property. Only let it grow to + * the next power of 2. + */ + ASSERT(!ISP2(zp->z_blksz)); + newblksz = MIN(end, 1 << highbit64(zp->z_blksz)); + } else { + newblksz = MIN(end, zp->z_zfsvfs->z_max_blksz); + } + dmu_tx_hold_write(tx, zp->z_id, 0, newblksz); + } else { + newblksz = 0; + } + + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + zfs_rangelock_exit(lr); + return (error); + } + + if (newblksz) + zfs_grow_blocksize(zp, newblksz, tx); + + zp->z_size = end; + + VERIFY(0 == sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zp->z_zfsvfs), + &zp->z_size, sizeof (zp->z_size), tx)); + + vnode_pager_setsize(ZTOV(zp), end); + + zfs_rangelock_exit(lr); + + dmu_tx_commit(tx); + + return (0); +} + +/* + * Free space in a file. + * + * IN: zp - znode of file to free data in. + * off - start of section to free. + * len - length of section to free. + * + * RETURN: 0 on success, error code on failure + */ +static int +zfs_free_range(znode_t *zp, uint64_t off, uint64_t len) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + zfs_locked_range_t *lr; + int error; + + /* + * Lock the range being freed. + */ + lr = zfs_rangelock_enter(&zp->z_rangelock, off, len, RL_WRITER); + + /* + * Nothing to do if file already at desired length. + */ + if (off >= zp->z_size) { + zfs_rangelock_exit(lr); + return (0); + } + + if (off + len > zp->z_size) + len = zp->z_size - off; + + error = dmu_free_long_range(zfsvfs->z_os, zp->z_id, off, len); + + if (error == 0) { + /* + * In FreeBSD we cannot free block in the middle of a file, + * but only at the end of a file, so this code path should + * never happen. + */ + vnode_pager_setsize(ZTOV(zp), off); + } + + zfs_rangelock_exit(lr); + + return (error); +} + +/* + * Truncate a file + * + * IN: zp - znode of file to free data in. + * end - new end-of-file. + * + * RETURN: 0 on success, error code on failure + */ +static int +zfs_trunc(znode_t *zp, uint64_t end) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + vnode_t *vp = ZTOV(zp); + dmu_tx_t *tx; + zfs_locked_range_t *lr; + int error; + sa_bulk_attr_t bulk[2]; + int count = 0; + + /* + * We will change zp_size, lock the whole file. + */ + lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_WRITER); + + /* + * Nothing to do if file already at desired length. + */ + if (end >= zp->z_size) { + zfs_rangelock_exit(lr); + return (0); + } + + error = dmu_free_long_range(zfsvfs->z_os, zp->z_id, end, + DMU_OBJECT_END); + if (error) { + zfs_rangelock_exit(lr); + return (error); + } + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + zfs_sa_upgrade_txholds(tx, zp); + dmu_tx_mark_netfree(tx); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + zfs_rangelock_exit(lr); + return (error); + } + + zp->z_size = end; + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), + NULL, &zp->z_size, sizeof (zp->z_size)); + + if (end == 0) { + zp->z_pflags &= ~ZFS_SPARSE; + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), + NULL, &zp->z_pflags, 8); + } + VERIFY(sa_bulk_update(zp->z_sa_hdl, bulk, count, tx) == 0); + + dmu_tx_commit(tx); + + /* + * Clear any mapped pages in the truncated region. This has to + * happen outside of the transaction to avoid the possibility of + * a deadlock with someone trying to push a page that we are + * about to invalidate. + */ + vnode_pager_setsize(vp, end); + + zfs_rangelock_exit(lr); + + return (0); +} + +/* + * Free space in a file + * + * IN: zp - znode of file to free data in. + * off - start of range + * len - end of range (0 => EOF) + * flag - current file open mode flags. + * log - TRUE if this action should be logged + * + * RETURN: 0 on success, error code on failure + */ +int +zfs_freesp(znode_t *zp, uint64_t off, uint64_t len, int flag, boolean_t log) +{ + dmu_tx_t *tx; + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + zilog_t *zilog = zfsvfs->z_log; + uint64_t mode; + uint64_t mtime[2], ctime[2]; + sa_bulk_attr_t bulk[3]; + int count = 0; + int error; + + if ((error = sa_lookup(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs), &mode, + sizeof (mode))) != 0) + return (error); + + if (off > zp->z_size) { + error = zfs_extend(zp, off+len); + if (error == 0 && log) + goto log; + else + return (error); + } + + if (len == 0) { + error = zfs_trunc(zp, off); + } else { + if ((error = zfs_free_range(zp, off, len)) == 0 && + off + len > zp->z_size) + error = zfs_extend(zp, off+len); + } + if (error || !log) + return (error); +log: + tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + zfs_sa_upgrade_txholds(tx, zp); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + return (error); + } + + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, mtime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, ctime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), + NULL, &zp->z_pflags, 8); + zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); + error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); + ASSERT(error == 0); + + zfs_log_truncate(zilog, tx, TX_TRUNCATE, zp, off, len); + + dmu_tx_commit(tx); + return (0); +} + +void +zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) +{ + uint64_t moid, obj, sa_obj, version; + uint64_t sense = ZFS_CASE_SENSITIVE; + uint64_t norm = 0; + nvpair_t *elem; + int error; + int i; + znode_t *rootzp = NULL; + zfsvfs_t *zfsvfs; + vattr_t vattr; + znode_t *zp; + zfs_acl_ids_t acl_ids; + + /* + * First attempt to create master node. + */ + /* + * In an empty objset, there are no blocks to read and thus + * there can be no i/o errors (which we assert below). + */ + moid = MASTER_NODE_OBJ; + error = zap_create_claim(os, moid, DMU_OT_MASTER_NODE, + DMU_OT_NONE, 0, tx); + ASSERT(error == 0); + + /* + * Set starting attributes. + */ + version = zfs_zpl_version_map(spa_version(dmu_objset_spa(os))); + elem = NULL; + while ((elem = nvlist_next_nvpair(zplprops, elem)) != NULL) { + /* For the moment we expect all zpl props to be uint64_ts */ + uint64_t val; + char *name; + + ASSERT(nvpair_type(elem) == DATA_TYPE_UINT64); + VERIFY(nvpair_value_uint64(elem, &val) == 0); + name = nvpair_name(elem); + if (strcmp(name, zfs_prop_to_name(ZFS_PROP_VERSION)) == 0) { + if (val < version) + version = val; + } else { + error = zap_update(os, moid, name, 8, 1, &val, tx); + } + ASSERT(error == 0); + if (strcmp(name, zfs_prop_to_name(ZFS_PROP_NORMALIZE)) == 0) + norm = val; + else if (strcmp(name, zfs_prop_to_name(ZFS_PROP_CASE)) == 0) + sense = val; + } + ASSERT(version != 0); + error = zap_update(os, moid, ZPL_VERSION_STR, 8, 1, &version, tx); + + /* + * Create zap object used for SA attribute registration + */ + + if (version >= ZPL_VERSION_SA) { + sa_obj = zap_create(os, DMU_OT_SA_MASTER_NODE, + DMU_OT_NONE, 0, tx); + error = zap_add(os, moid, ZFS_SA_ATTRS, 8, 1, &sa_obj, tx); + ASSERT(error == 0); + } else { + sa_obj = 0; + } + /* + * Create a delete queue. + */ + obj = zap_create(os, DMU_OT_UNLINKED_SET, DMU_OT_NONE, 0, tx); + + error = zap_add(os, moid, ZFS_UNLINKED_SET, 8, 1, &obj, tx); + ASSERT(error == 0); + + /* + * Create root znode. Create minimal znode/vnode/zfsvfs + * to allow zfs_mknode to work. + */ + VATTR_NULL(&vattr); + vattr.va_mask = AT_MODE|AT_UID|AT_GID; + vattr.va_type = VDIR; + vattr.va_mode = S_IFDIR|0755; + vattr.va_uid = crgetuid(cr); + vattr.va_gid = crgetgid(cr); + + zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP); + + rootzp = kmem_cache_alloc(znode_cache, KM_SLEEP); + ASSERT(!POINTER_IS_VALID(rootzp->z_zfsvfs)); + rootzp->z_moved = 0; + rootzp->z_unlinked = 0; + rootzp->z_atime_dirty = 0; + rootzp->z_is_sa = USE_SA(version, os); + + zfsvfs->z_os = os; + zfsvfs->z_parent = zfsvfs; + zfsvfs->z_version = version; + zfsvfs->z_use_fuids = USE_FUIDS(version, os); + zfsvfs->z_use_sa = USE_SA(version, os); + zfsvfs->z_norm = norm; + + error = sa_setup(os, sa_obj, zfs_attr_table, ZPL_END, + &zfsvfs->z_attr_table); + + ASSERT(error == 0); + + /* + * Fold case on file systems that are always or sometimes case + * insensitive. + */ + if (sense == ZFS_CASE_INSENSITIVE || sense == ZFS_CASE_MIXED) + zfsvfs->z_norm |= U8_TEXTPREP_TOUPPER; + + mutex_init(&zfsvfs->z_znodes_lock, NULL, MUTEX_DEFAULT, NULL); + list_create(&zfsvfs->z_all_znodes, sizeof (znode_t), + offsetof(znode_t, z_link_node)); + + for (i = 0; i != ZFS_OBJ_MTX_SZ; i++) + mutex_init(&zfsvfs->z_hold_mtx[i], NULL, MUTEX_DEFAULT, NULL); + + rootzp->z_zfsvfs = zfsvfs; + VERIFY(0 == zfs_acl_ids_create(rootzp, IS_ROOT_NODE, &vattr, + cr, NULL, &acl_ids)); + zfs_mknode(rootzp, &vattr, tx, cr, IS_ROOT_NODE, &zp, &acl_ids); + ASSERT3P(zp, ==, rootzp); + error = zap_add(os, moid, ZFS_ROOT_OBJ, 8, 1, &rootzp->z_id, tx); + ASSERT(error == 0); + zfs_acl_ids_free(&acl_ids); + POINTER_INVALIDATE(&rootzp->z_zfsvfs); + + sa_handle_destroy(rootzp->z_sa_hdl); + kmem_cache_free(znode_cache, rootzp); + + /* + * Create shares directory + */ + + error = zfs_create_share_dir(zfsvfs, tx); + + ASSERT(error == 0); + + for (i = 0; i != ZFS_OBJ_MTX_SZ; i++) + mutex_destroy(&zfsvfs->z_hold_mtx[i]); + kmem_free(zfsvfs, sizeof (zfsvfs_t)); +} +#endif /* _KERNEL */ + +static int +zfs_sa_setup(objset_t *osp, sa_attr_type_t **sa_table) +{ + uint64_t sa_obj = 0; + int error; + + error = zap_lookup(osp, MASTER_NODE_OBJ, ZFS_SA_ATTRS, 8, 1, &sa_obj); + if (error != 0 && error != ENOENT) + return (error); + + error = sa_setup(osp, sa_obj, zfs_attr_table, ZPL_END, sa_table); + return (error); +} + +static int +zfs_grab_sa_handle(objset_t *osp, uint64_t obj, sa_handle_t **hdlp, + dmu_buf_t **db, void *tag) +{ + dmu_object_info_t doi; + int error; + + if ((error = sa_buf_hold(osp, obj, tag, db)) != 0) + return (error); + + dmu_object_info_from_db(*db, &doi); + if ((doi.doi_bonus_type != DMU_OT_SA && + doi.doi_bonus_type != DMU_OT_ZNODE) || + (doi.doi_bonus_type == DMU_OT_ZNODE && + doi.doi_bonus_size < sizeof (znode_phys_t))) { + sa_buf_rele(*db, tag); + return (SET_ERROR(ENOTSUP)); + } + + error = sa_handle_get(osp, obj, NULL, SA_HDL_PRIVATE, hdlp); + if (error != 0) { + sa_buf_rele(*db, tag); + return (error); + } + + return (0); +} + +void +zfs_release_sa_handle(sa_handle_t *hdl, dmu_buf_t *db, void *tag) +{ + sa_handle_destroy(hdl); + sa_buf_rele(db, tag); +} + +/* + * Given an object number, return its parent object number and whether + * or not the object is an extended attribute directory. + */ +static int +zfs_obj_to_pobj(objset_t *osp, sa_handle_t *hdl, sa_attr_type_t *sa_table, + uint64_t *pobjp, int *is_xattrdir) +{ + uint64_t parent; + uint64_t pflags; + uint64_t mode; + uint64_t parent_mode; + sa_bulk_attr_t bulk[3]; + sa_handle_t *sa_hdl; + dmu_buf_t *sa_db; + int count = 0; + int error; + + SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_PARENT], NULL, + &parent, sizeof (parent)); + SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_FLAGS], NULL, + &pflags, sizeof (pflags)); + SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_MODE], NULL, + &mode, sizeof (mode)); + + if ((error = sa_bulk_lookup(hdl, bulk, count)) != 0) + return (error); + + /* + * When a link is removed its parent pointer is not changed and will + * be invalid. There are two cases where a link is removed but the + * file stays around, when it goes to the delete queue and when there + * are additional links. + */ + error = zfs_grab_sa_handle(osp, parent, &sa_hdl, &sa_db, FTAG); + if (error != 0) + return (error); + + error = sa_lookup(sa_hdl, ZPL_MODE, &parent_mode, sizeof (parent_mode)); + zfs_release_sa_handle(sa_hdl, sa_db, FTAG); + if (error != 0) + return (error); + + *is_xattrdir = ((pflags & ZFS_XATTR) != 0) && S_ISDIR(mode); + + /* + * Extended attributes can be applied to files, directories, etc. + * Otherwise the parent must be a directory. + */ + if (!*is_xattrdir && !S_ISDIR(parent_mode)) + return (SET_ERROR(EINVAL)); + + *pobjp = parent; + + return (0); +} + +/* + * Given an object number, return some zpl level statistics + */ +static int +zfs_obj_to_stats_impl(sa_handle_t *hdl, sa_attr_type_t *sa_table, + zfs_stat_t *sb) +{ + sa_bulk_attr_t bulk[4]; + int count = 0; + + SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_MODE], NULL, + &sb->zs_mode, sizeof (sb->zs_mode)); + SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_GEN], NULL, + &sb->zs_gen, sizeof (sb->zs_gen)); + SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_LINKS], NULL, + &sb->zs_links, sizeof (sb->zs_links)); + SA_ADD_BULK_ATTR(bulk, count, sa_table[ZPL_CTIME], NULL, + &sb->zs_ctime, sizeof (sb->zs_ctime)); + + return (sa_bulk_lookup(hdl, bulk, count)); +} + +static int +zfs_obj_to_path_impl(objset_t *osp, uint64_t obj, sa_handle_t *hdl, + sa_attr_type_t *sa_table, char *buf, int len) +{ + sa_handle_t *sa_hdl; + sa_handle_t *prevhdl = NULL; + dmu_buf_t *prevdb = NULL; + dmu_buf_t *sa_db = NULL; + char *path = buf + len - 1; + int error; + + *path = '\0'; + sa_hdl = hdl; + + uint64_t deleteq_obj; + VERIFY0(zap_lookup(osp, MASTER_NODE_OBJ, + ZFS_UNLINKED_SET, sizeof (uint64_t), 1, &deleteq_obj)); + error = zap_lookup_int(osp, deleteq_obj, obj); + if (error == 0) { + return (ESTALE); + } else if (error != ENOENT) { + return (error); + } + error = 0; + + for (;;) { + uint64_t pobj; + char component[MAXNAMELEN + 2]; + size_t complen; + int is_xattrdir; + + if (prevdb) + zfs_release_sa_handle(prevhdl, prevdb, FTAG); + + if ((error = zfs_obj_to_pobj(osp, sa_hdl, sa_table, &pobj, + &is_xattrdir)) != 0) + break; + + if (pobj == obj) { + if (path[0] != '/') + *--path = '/'; + break; + } + + component[0] = '/'; + if (is_xattrdir) { + (void) sprintf(component + 1, ""); + } else { + error = zap_value_search(osp, pobj, obj, + ZFS_DIRENT_OBJ(-1ULL), component + 1); + if (error != 0) + break; + } + + complen = strlen(component); + path -= complen; + ASSERT(path >= buf); + bcopy(component, path, complen); + obj = pobj; + + if (sa_hdl != hdl) { + prevhdl = sa_hdl; + prevdb = sa_db; + } + error = zfs_grab_sa_handle(osp, obj, &sa_hdl, &sa_db, FTAG); + if (error != 0) { + sa_hdl = prevhdl; + sa_db = prevdb; + break; + } + } + + if (sa_hdl != NULL && sa_hdl != hdl) { + ASSERT(sa_db != NULL); + zfs_release_sa_handle(sa_hdl, sa_db, FTAG); + } + + if (error == 0) + (void) memmove(buf, path, buf + len - path); + + return (error); +} + +int +zfs_obj_to_path(objset_t *osp, uint64_t obj, char *buf, int len) +{ + sa_attr_type_t *sa_table; + sa_handle_t *hdl; + dmu_buf_t *db; + int error; + + error = zfs_sa_setup(osp, &sa_table); + if (error != 0) + return (error); + + error = zfs_grab_sa_handle(osp, obj, &hdl, &db, FTAG); + if (error != 0) + return (error); + + error = zfs_obj_to_path_impl(osp, obj, hdl, sa_table, buf, len); + + zfs_release_sa_handle(hdl, db, FTAG); + return (error); +} + +int +zfs_obj_to_stats(objset_t *osp, uint64_t obj, zfs_stat_t *sb, + char *buf, int len) +{ + char *path = buf + len - 1; + sa_attr_type_t *sa_table; + sa_handle_t *hdl; + dmu_buf_t *db; + int error; + + *path = '\0'; + + error = zfs_sa_setup(osp, &sa_table); + if (error != 0) + return (error); + + error = zfs_grab_sa_handle(osp, obj, &hdl, &db, FTAG); + if (error != 0) + return (error); + + error = zfs_obj_to_stats_impl(hdl, sa_table, sb); + if (error != 0) { + zfs_release_sa_handle(hdl, db, FTAG); + return (error); + } + + error = zfs_obj_to_path_impl(osp, obj, hdl, sa_table, buf, len); + + zfs_release_sa_handle(hdl, db, FTAG); + return (error); +} + +#ifdef _KERNEL +int +zfs_znode_parent_and_name(znode_t *zp, znode_t **dzpp, char *buf) +{ + zfsvfs_t *zfsvfs = zp->z_zfsvfs; + uint64_t parent; + int is_xattrdir; + int err; + + /* Extended attributes should not be visible as regular files. */ + if ((zp->z_pflags & ZFS_XATTR) != 0) + return (SET_ERROR(EINVAL)); + + err = zfs_obj_to_pobj(zfsvfs->z_os, zp->z_sa_hdl, zfsvfs->z_attr_table, + &parent, &is_xattrdir); + if (err != 0) + return (err); + ASSERT0(is_xattrdir); + + /* No name as this is a root object. */ + if (parent == zp->z_id) + return (SET_ERROR(EINVAL)); + + err = zap_value_search(zfsvfs->z_os, parent, zp->z_id, + ZFS_DIRENT_OBJ(-1ULL), buf); + if (err != 0) + return (err); + err = zfs_zget(zfsvfs, parent, dzpp); + return (err); +} +#endif /* _KERNEL */ diff --git a/module/os/freebsd/zfs/zio_crypt.c b/module/os/freebsd/zfs/zio_crypt.c new file mode 100644 index 00000000000..d6496a7eff2 --- /dev/null +++ b/module/os/freebsd/zfs/zio_crypt.c @@ -0,0 +1,1882 @@ +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2017, Datto, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * This file is responsible for handling all of the details of generating + * encryption parameters and performing encryption and authentication. + * + * BLOCK ENCRYPTION PARAMETERS: + * Encryption /Authentication Algorithm Suite (crypt): + * The encryption algorithm, mode, and key length we are going to use. We + * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit + * keys. All authentication is currently done with SHA512-HMAC. + * + * Plaintext: + * The unencrypted data that we want to encrypt. + * + * Initialization Vector (IV): + * An initialization vector for the encryption algorithms. This is used to + * "tweak" the encryption algorithms so that two blocks of the same data are + * encrypted into different ciphertext outputs, thus obfuscating block patterns. + * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is + * never reused with the same encryption key. This value is stored unencrypted + * and must simply be provided to the decryption function. We use a 96 bit IV + * (as recommended by NIST) for all block encryption. For non-dedup blocks we + * derive the IV randomly. The first 64 bits of the IV are stored in the second + * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of + * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits + * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count + * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of + * level 0 blocks is the number of allocated dnodes in that block. The on-disk + * format supports at most 2^15 slots per L0 dnode block, because the maximum + * block size is 16MB (2^24). In either case, for level 0 blocks this number + * will still be smaller than UINT32_MAX so it is safe to store the IV in the + * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count + * for the dnode code. + * + * Master key: + * This is the most important secret data of an encrypted dataset. It is used + * along with the salt to generate that actual encryption keys via HKDF. We + * do not use the master key to directly encrypt any data because there are + * theoretical limits on how much data can actually be safely encrypted with + * any encryption mode. The master key is stored encrypted on disk with the + * user's wrapping key. Its length is determined by the encryption algorithm. + * For details on how this is stored see the block comment in dsl_crypt.c + * + * Salt: + * Used as an input to the HKDF function, along with the master key. We use a + * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt + * can be used for encrypting many blocks, so we cache the current salt and the + * associated derived key in zio_crypt_t so we do not need to derive it again + * needlessly. + * + * Encryption Key: + * A secret binary key, generated from an HKDF function used to encrypt and + * decrypt data. + * + * Message Authentication Code (MAC) + * The MAC is an output of authenticated encryption modes such as AES-GCM and + * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted + * data on disk and return garbage to the application. Effectively, it is a + * checksum that can not be reproduced by an attacker. We store the MAC in the + * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated + * regular checksum of the ciphertext which can be used for scrubbing. + * + * OBJECT AUTHENTICATION: + * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because + * they contain some info that always needs to be readable. To prevent this + * data from being altered, we authenticate this data using SHA512-HMAC. This + * will produce a MAC (similar to the one produced via encryption) which can + * be used to verify the object was not modified. HMACs do not require key + * rotation or IVs, so we can keep up to the full 3 copies of authenticated + * data. + * + * ZIL ENCRYPTION: + * ZIL blocks have their bp written to disk ahead of the associated data, so we + * cannot store the MAC there as we normally do. For these blocks the MAC is + * stored in the embedded checksum within the zil_chain_t header. The salt and + * IV are generated for the block on bp allocation instead of at encryption + * time. In addition, ZIL blocks have some pieces that must be left in plaintext + * for claiming even though all of the sensitive user data still needs to be + * encrypted. The function zio_crypt_init_uios_zil() handles parsing which + * pieces of the block need to be encrypted. All data that is not encrypted is + * authenticated using the AAD mechanisms that the supported encryption modes + * provide for. In order to preserve the semantics of the ZIL for encrypted + * datasets, the ZIL is not protected at the objset level as described below. + * + * DNODE ENCRYPTION: + * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left + * in plaintext for scrubbing and claiming, but the bonus buffers might contain + * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing + * which which pieces of the block need to be encrypted. For more details about + * dnode authentication and encryption, see zio_crypt_init_uios_dnode(). + * + * OBJECT SET AUTHENTICATION: + * Up to this point, everything we have encrypted and authenticated has been + * at level 0 (or -2 for the ZIL). If we did not do any further work the + * on-disk format would be susceptible to attacks that deleted or rearranged + * the order of level 0 blocks. Ideally, the cleanest solution would be to + * maintain a tree of authentication MACs going up the bp tree. However, this + * presents a problem for raw sends. Send files do not send information about + * indirect blocks so there would be no convenient way to transfer the MACs and + * they cannot be recalculated on the receive side without the master key which + * would defeat one of the purposes of raw sends in the first place. Instead, + * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs + * from the level below. We also include some portable fields from blk_prop such + * as the lsize and compression algorithm to prevent the data from being + * misinterpreted. + * + * At the objset level, we maintain 2 separate 256 bit MACs in the + * objset_phys_t. The first one is "portable" and is the logical root of the + * MAC tree maintained in the metadnode's bps. The second, is "local" and is + * used as the root MAC for the user accounting objects, which are also not + * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload + * of the send file. The useraccounting code ensures that the useraccounting + * info is not present upon a receive, so the local MAC can simply be cleared + * out at that time. For more info about objset_phys_t authentication, see + * zio_crypt_do_objset_hmacs(). + * + * CONSIDERATIONS FOR DEDUP: + * In order for dedup to work, blocks that we want to dedup with one another + * need to use the same IV and encryption key, so that they will have the same + * ciphertext. Normally, one should never reuse an IV with the same encryption + * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both + * blocks. In this case, however, since we are using the same plaintext as + * well all that we end up with is a duplicate of the original ciphertext we + * already had. As a result, an attacker with read access to the raw disk will + * be able to tell which blocks are the same but this information is given away + * by dedup anyway. In order to get the same IVs and encryption keys for + * equivalent blocks of data we use an HMAC of the plaintext. We use an HMAC + * here so that a reproducible checksum of the plaintext is never available to + * the attacker. The HMAC key is kept alongside the master key, encrypted on + * disk. The first 64 bits of the HMAC are used in place of the random salt, and + * the next 96 bits are used as the IV. As a result of this mechanism, dedup + * will only work within a clone family since encrypted dedup requires use of + * the same master and HMAC keys. + */ + +/* + * After encrypting many blocks with the same key we may start to run up + * against the theoretical limits of how much data can securely be encrypted + * with a single key using the supported encryption modes. The most obvious + * limitation is that our risk of generating 2 equivalent 96 bit IVs increases + * the more IVs we generate (which both GCM and CCM modes strictly forbid). + * This risk actually grows surprisingly quickly over time according to the + * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have + * generated n IVs with a cryptographically secure RNG, the approximate + * probability p(n) of a collision is given as: + * + * p(n) ~= e^(-n*(n-1)/(2*(2^96))) + * + * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html] + * + * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion + * we must not write more than 398,065,730 blocks with the same encryption key. + * Therefore, we rotate our keys after 400,000,000 blocks have been written by + * generating a new random 64 bit salt for our HKDF encryption key generation + * function. + */ +#define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000 +#define ZFS_CURRENT_MAX_SALT_USES \ + (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT)) +unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT; + +/* + * Set to a nonzero value to cause zio_do_crypt_uio() to fail 1/this many + * calls, to test decryption error handling code paths. + */ +uint64_t zio_decrypt_fail_fraction = 0; + +typedef struct blkptr_auth_buf { + uint64_t bab_prop; /* blk_prop - portable mask */ + uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */ + uint64_t bab_pad; /* reserved for future use */ +} blkptr_auth_buf_t; + +zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = { + {"", ZC_TYPE_NONE, 0, "inherit"}, + {"", ZC_TYPE_NONE, 0, "on"}, + {"", ZC_TYPE_NONE, 0, "off"}, + {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 16, "aes-128-ccm"}, + {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 24, "aes-192-ccm"}, + {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 32, "aes-256-ccm"}, + {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 16, "aes-128-gcm"}, + {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 24, "aes-192-gcm"}, + {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"} +}; + +static void +zio_crypt_key_destroy_early(zio_crypt_key_t *key) +{ + rw_destroy(&key->zk_salt_lock); + + /* free crypto templates */ + bzero(&key->zk_session, sizeof (key->zk_session)); + + /* zero out sensitive data */ + bzero(key, sizeof (zio_crypt_key_t)); +} + +void +zio_crypt_key_destroy(zio_crypt_key_t *key) +{ + + freebsd_crypt_freesession(&key->zk_session); + zio_crypt_key_destroy_early(key); +} + +int +zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key) +{ + int ret; + crypto_mechanism_t mech __unused; + uint_t keydata_len; + zio_crypt_info_t *ci = NULL; + + ASSERT(key != NULL); + ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); + + ci = &zio_crypt_table[crypt]; + if (ci->ci_crypt_type != ZC_TYPE_GCM && + ci->ci_crypt_type != ZC_TYPE_CCM) + return (ENOTSUP); + + keydata_len = zio_crypt_table[crypt].ci_keylen; + bzero(key, sizeof (zio_crypt_key_t)); + rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); + + /* fill keydata buffers and salt with random data */ + ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t)); + if (ret != 0) + goto error; + + ret = random_get_bytes(key->zk_master_keydata, keydata_len); + if (ret != 0) + goto error; + + ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN); + if (ret != 0) + goto error; + + ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); + if (ret != 0) + goto error; + + /* derive the current key from the master key */ + ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, + key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, + keydata_len); + if (ret != 0) + goto error; + + /* initialize keys for the ICP */ + key->zk_current_key.ck_format = CRYPTO_KEY_RAW; + key->zk_current_key.ck_data = key->zk_current_keydata; + key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); + + key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; + key->zk_hmac_key.ck_data = &key->zk_hmac_key; + key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); + + ci = &zio_crypt_table[crypt]; + if (ci->ci_crypt_type != ZC_TYPE_GCM && + ci->ci_crypt_type != ZC_TYPE_CCM) + return (ENOTSUP); + + ret = freebsd_crypt_newsession(&key->zk_session, ci, + &key->zk_current_key); + if (ret) + goto error; + + key->zk_crypt = crypt; + key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION; + key->zk_salt_count = 0; + + return (0); + +error: + zio_crypt_key_destroy_early(key); + return (ret); +} + +static int +zio_crypt_key_change_salt(zio_crypt_key_t *key) +{ + int ret = 0; + uint8_t salt[ZIO_DATA_SALT_LEN]; + crypto_mechanism_t mech __unused; + + uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen; + + /* generate a new salt */ + ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN); + if (ret != 0) + goto error; + + rw_enter(&key->zk_salt_lock, RW_WRITER); + + /* someone beat us to the salt rotation, just unlock and return */ + if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES) + goto out_unlock; + + /* derive the current key from the master key and the new salt */ + ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, + salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); + if (ret != 0) + goto out_unlock; + + /* assign the salt and reset the usage count */ + bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN); + key->zk_salt_count = 0; + + freebsd_crypt_freesession(&key->zk_session); + ret = freebsd_crypt_newsession(&key->zk_session, + &zio_crypt_table[key->zk_crypt], &key->zk_current_key); + if (ret != 0) + goto out_unlock; + + rw_exit(&key->zk_salt_lock); + + return (0); + +out_unlock: + rw_exit(&key->zk_salt_lock); +error: + return (ret); +} + +/* See comment above zfs_key_max_salt_uses definition for details */ +int +zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt) +{ + int ret; + boolean_t salt_change; + + rw_enter(&key->zk_salt_lock, RW_READER); + + bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN); + salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >= + ZFS_CURRENT_MAX_SALT_USES); + + rw_exit(&key->zk_salt_lock); + + if (salt_change) { + ret = zio_crypt_key_change_salt(key); + if (ret != 0) + goto error; + } + + return (0); + +error: + return (ret); +} + +void *failed_decrypt_buf; +int failed_decrypt_size; + +/* + * This function handles all encryption and decryption in zfs. When + * encrypting it expects puio to reference the plaintext and cuio to + * reference the ciphertext. cuio must have enough space for the + * ciphertext + room for a MAC. datalen should be the length of the + * plaintext / ciphertext alone. + */ +/* + * The implemenation for FreeBSD's OpenCrypto. + * + * The big difference between ICP and FOC is that FOC uses a single + * buffer for input and output. This means that (for AES-GCM, the + * only one supported right now) the source must be copied into the + * destination, and the destination must have the AAD, and the tag/MAC, + * already associated with it. (Both implementations can use a uio.) + * + * Since the auth data is part of the iovec array, all we need to know + * is the length: 0 means there's no AAD. + * + */ +static int +zio_do_crypt_uio_opencrypto(boolean_t encrypt, freebsd_crypt_session_t *sess, + uint64_t crypt, crypto_key_t *key, uint8_t *ivbuf, uint_t datalen, + uio_t *uio, uint_t auth_len) +{ + zio_crypt_info_t *ci; + int ret; + + ci = &zio_crypt_table[crypt]; + if (ci->ci_crypt_type != ZC_TYPE_GCM && + ci->ci_crypt_type != ZC_TYPE_CCM) + return (ENOTSUP); + + + ret = freebsd_crypt_uio(encrypt, sess, ci, uio, key, ivbuf, + datalen, auth_len); + if (ret != 0) { +#ifdef FCRYPTO_DEBUG + printf("%s(%d): Returning error %s\n", + __FUNCTION__, __LINE__, encrypt ? "EIO" : "ECKSUM"); +#endif + ret = SET_ERROR(encrypt ? EIO : ECKSUM); + } + + return (ret); +} + +int +zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, + uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out) +{ + int ret; + uint64_t aad[3]; + /* + * With OpenCrypto in FreeBSD, the same buffer is used for + * input and output. Also, the AAD (for AES-GMC at least) + * needs to logically go in front. + */ + uio_t cuio; + iovec_t iovecs[4]; + uint64_t crypt = key->zk_crypt; + uint_t enc_len, keydata_len, aad_len; + + ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); + ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); + + keydata_len = zio_crypt_table[crypt].ci_keylen; + + /* generate iv for wrapping the master and hmac key */ + ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN); + if (ret != 0) + goto error; + + /* + * Since we only support one buffer, we need to copy + * the plain text (source) to the cipher buffer (dest). + * We set iovecs[0] -- the authentication data -- below. + */ + bcopy((void*)key->zk_master_keydata, keydata_out, keydata_len); + bcopy((void*)key->zk_hmac_keydata, hmac_keydata_out, + SHA512_HMAC_KEYLEN); + iovecs[1].iov_base = keydata_out; + iovecs[1].iov_len = keydata_len; + iovecs[2].iov_base = hmac_keydata_out; + iovecs[2].iov_len = SHA512_HMAC_KEYLEN; + iovecs[3].iov_base = mac; + iovecs[3].iov_len = WRAPPING_MAC_LEN; + + /* + * Although we don't support writing to the old format, we do + * support rewrapping the key so that the user can move and + * quarantine datasets on the old format. + */ + if (key->zk_version == 0) { + aad_len = sizeof (uint64_t); + aad[0] = LE_64(key->zk_guid); + } else { + ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); + aad_len = sizeof (uint64_t) * 3; + aad[0] = LE_64(key->zk_guid); + aad[1] = LE_64(crypt); + aad[2] = LE_64(key->zk_version); + } + + iovecs[0].iov_base = aad; + iovecs[0].iov_len = aad_len; + enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN; + + cuio.uio_iov = iovecs; + cuio.uio_iovcnt = 4; + cuio.uio_segflg = UIO_SYSSPACE; + + /* encrypt the keys and store the resulting ciphertext and mac */ + ret = zio_do_crypt_uio_opencrypto(B_TRUE, NULL, crypt, cwkey, + iv, enc_len, &cuio, aad_len); + if (ret != 0) + goto error; + + return (0); + +error: + return (ret); +} + +int +zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, + uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv, + uint8_t *mac, zio_crypt_key_t *key) +{ + int ret; + uint64_t aad[3]; + /* + * With OpenCrypto in FreeBSD, the same buffer is used for + * input and output. Also, the AAD (for AES-GMC at least) + * needs to logically go in front. + */ + uio_t cuio; + iovec_t iovecs[4]; + void *src, *dst; + uint_t enc_len, keydata_len, aad_len; + + ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); + ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); + + keydata_len = zio_crypt_table[crypt].ci_keylen; + rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); + + /* + * Since we only support one buffer, we need to copy + * the encrypted buffer (source) to the plain buffer + * (dest). We set iovecs[0] -- the authentication data -- + * below. + */ + dst = key->zk_master_keydata; + src = keydata; + + bcopy(src, dst, keydata_len); + + dst = key->zk_hmac_keydata; + src = hmac_keydata; + bcopy(src, dst, SHA512_HMAC_KEYLEN); + + iovecs[1].iov_base = key->zk_master_keydata; + iovecs[1].iov_len = keydata_len; + iovecs[2].iov_base = key->zk_hmac_keydata; + iovecs[2].iov_len = SHA512_HMAC_KEYLEN; + iovecs[3].iov_base = mac; + iovecs[3].iov_len = WRAPPING_MAC_LEN; + + if (version == 0) { + aad_len = sizeof (uint64_t); + aad[0] = LE_64(guid); + } else { + ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); + aad_len = sizeof (uint64_t) * 3; + aad[0] = LE_64(guid); + aad[1] = LE_64(crypt); + aad[2] = LE_64(version); + } + + enc_len = keydata_len + SHA512_HMAC_KEYLEN; + iovecs[0].iov_base = aad; + iovecs[0].iov_len = aad_len; + + cuio.uio_iov = iovecs; + cuio.uio_iovcnt = 4; + cuio.uio_segflg = UIO_SYSSPACE; + + /* decrypt the keys and store the result in the output buffers */ + ret = zio_do_crypt_uio_opencrypto(B_FALSE, NULL, crypt, cwkey, + iv, enc_len, &cuio, aad_len); + + if (ret != 0) + goto error; + + /* generate a fresh salt */ + ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); + if (ret != 0) + goto error; + + /* derive the current key from the master key */ + ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, + key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, + keydata_len); + if (ret != 0) + goto error; + + /* initialize keys for ICP */ + key->zk_current_key.ck_format = CRYPTO_KEY_RAW; + key->zk_current_key.ck_data = key->zk_current_keydata; + key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); + + key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; + key->zk_hmac_key.ck_data = key->zk_hmac_keydata; + key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); + + ret = freebsd_crypt_newsession(&key->zk_session, + &zio_crypt_table[crypt], &key->zk_current_key); + if (ret != 0) + goto error; + + key->zk_crypt = crypt; + key->zk_version = version; + key->zk_guid = guid; + key->zk_salt_count = 0; + + return (0); + +error: + zio_crypt_key_destroy_early(key); + return (ret); +} + +int +zio_crypt_generate_iv(uint8_t *ivbuf) +{ + int ret; + + /* randomly generate the IV */ + ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN); + if (ret != 0) + goto error; + + return (0); + +error: + bzero(ivbuf, ZIO_DATA_IV_LEN); + return (ret); +} + +int +zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, + uint8_t *digestbuf, uint_t digestlen) +{ + uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH]; + + ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH); + + crypto_mac(&key->zk_hmac_key, data, datalen, + raw_digestbuf, SHA512_DIGEST_LENGTH); + + bcopy(raw_digestbuf, digestbuf, digestlen); + + return (0); +} + +int +zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data, + uint_t datalen, uint8_t *ivbuf, uint8_t *salt) +{ + int ret; + uint8_t digestbuf[SHA512_DIGEST_LENGTH]; + + ret = zio_crypt_do_hmac(key, data, datalen, + digestbuf, SHA512_DIGEST_LENGTH); + if (ret != 0) + return (ret); + + bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN); + bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN); + + return (0); +} + +/* + * The following functions are used to encode and decode encryption parameters + * into blkptr_t and zil_header_t. The ICP wants to use these parameters as + * byte strings, which normally means that these strings would not need to deal + * with byteswapping at all. However, both blkptr_t and zil_header_t may be + * byteswapped by lower layers and so we must "undo" that byteswap here upon + * decoding and encoding in a non-native byteorder. These functions require + * that the byteorder bit is correct before being called. + */ +void +zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv) +{ + uint64_t val64; + uint32_t val32; + + ASSERT(BP_IS_ENCRYPTED(bp)); + + if (!BP_SHOULD_BYTESWAP(bp)) { + bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t)); + bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t)); + bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); + BP_SET_IV2(bp, val32); + } else { + bcopy(salt, &val64, sizeof (uint64_t)); + bp->blk_dva[2].dva_word[0] = BSWAP_64(val64); + + bcopy(iv, &val64, sizeof (uint64_t)); + bp->blk_dva[2].dva_word[1] = BSWAP_64(val64); + + bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); + BP_SET_IV2(bp, BSWAP_32(val32)); + } +} + +void +zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv) +{ + uint64_t val64; + uint32_t val32; + + ASSERT(BP_IS_PROTECTED(bp)); + + /* for convenience, so callers don't need to check */ + if (BP_IS_AUTHENTICATED(bp)) { + bzero(salt, ZIO_DATA_SALT_LEN); + bzero(iv, ZIO_DATA_IV_LEN); + return; + } + + if (!BP_SHOULD_BYTESWAP(bp)) { + bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t)); + bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t)); + + val32 = (uint32_t)BP_GET_IV2(bp); + bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); + } else { + val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]); + bcopy(&val64, salt, sizeof (uint64_t)); + + val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]); + bcopy(&val64, iv, sizeof (uint64_t)); + + val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp)); + bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); + } +} + +void +zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac) +{ + uint64_t val64; + + ASSERT(BP_USES_CRYPT(bp)); + ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET); + + if (!BP_SHOULD_BYTESWAP(bp)) { + bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t)); + bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3], + sizeof (uint64_t)); + } else { + bcopy(mac, &val64, sizeof (uint64_t)); + bp->blk_cksum.zc_word[2] = BSWAP_64(val64); + + bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t)); + bp->blk_cksum.zc_word[3] = BSWAP_64(val64); + } +} + +void +zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac) +{ + uint64_t val64; + + ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp)); + + /* for convenience, so callers don't need to check */ + if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { + bzero(mac, ZIO_DATA_MAC_LEN); + return; + } + + if (!BP_SHOULD_BYTESWAP(bp)) { + bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t)); + bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t), + sizeof (uint64_t)); + } else { + val64 = BSWAP_64(bp->blk_cksum.zc_word[2]); + bcopy(&val64, mac, sizeof (uint64_t)); + + val64 = BSWAP_64(bp->blk_cksum.zc_word[3]); + bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t)); + } +} + +void +zio_crypt_encode_mac_zil(void *data, uint8_t *mac) +{ + zil_chain_t *zilc = data; + + bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t)); + bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3], + sizeof (uint64_t)); +} + +void +zio_crypt_decode_mac_zil(const void *data, uint8_t *mac) +{ + /* + * The ZIL MAC is embedded in the block it protects, which will + * not have been byteswapped by the time this function has been called. + * As a result, we don't need to worry about byteswapping the MAC. + */ + const zil_chain_t *zilc = data; + + bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t)); + bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t), + sizeof (uint64_t)); +} + +/* + * This routine takes a block of dnodes (src_abd) and copies only the bonus + * buffers to the same offsets in the dst buffer. datalen should be the size + * of both the src_abd and the dst buffer (not just the length of the bonus + * buffers). + */ +void +zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen) +{ + uint_t i, max_dnp = datalen >> DNODE_SHIFT; + uint8_t *src; + dnode_phys_t *dnp, *sdnp, *ddnp; + + src = abd_borrow_buf_copy(src_abd, datalen); + + sdnp = (dnode_phys_t *)src; + ddnp = (dnode_phys_t *)dst; + + for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { + dnp = &sdnp[i]; + if (dnp->dn_type != DMU_OT_NONE && + DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && + dnp->dn_bonuslen != 0) { + bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), + DN_MAX_BONUS_LEN(dnp)); + } + } + + abd_return_buf(src_abd, src, datalen); +} + +/* + * This function decides what fields from blk_prop are included in + * the on-disk various MAC algorithms. + */ +static void +zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) +{ + int avoidlint = SPA_MINBLOCKSIZE; + /* + * Version 0 did not properly zero out all non-portable fields + * as it should have done. We maintain this code so that we can + * do read-only imports of pools on this version. + */ + if (version == 0) { + BP_SET_DEDUP(bp, 0); + BP_SET_CHECKSUM(bp, 0); + BP_SET_PSIZE(bp, avoidlint); + return; + } + + ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); + + /* + * The hole_birth feature might set these fields even if this bp + * is a hole. We zero them out here to guarantee that raw sends + * will function with or without the feature. + */ + if (BP_IS_HOLE(bp)) { + bp->blk_prop = 0ULL; + return; + } + + /* + * At L0 we want to verify these fields to ensure that data blocks + * can not be reinterpreted. For instance, we do not want an attacker + * to trick us into returning raw lz4 compressed data to the user + * by modifying the compression bits. At higher levels, we cannot + * enforce this policy since raw sends do not convey any information + * about indirect blocks, so these values might be different on the + * receive side. Fortunately, this does not open any new attack + * vectors, since any alterations that can be made to a higher level + * bp must still verify the correct order of the layer below it. + */ + if (BP_GET_LEVEL(bp) != 0) { + BP_SET_BYTEORDER(bp, 0); + BP_SET_COMPRESS(bp, 0); + + /* + * psize cannot be set to zero or it will trigger + * asserts, but the value doesn't really matter as + * long as it is constant. + */ + BP_SET_PSIZE(bp, avoidlint); + } + + BP_SET_DEDUP(bp, 0); + BP_SET_CHECKSUM(bp, 0); +} + +static void +zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp, + blkptr_auth_buf_t *bab, uint_t *bab_len) +{ + blkptr_t tmpbp = *bp; + + if (should_bswap) + byteswap_uint64_array(&tmpbp, sizeof (blkptr_t)); + + ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp)); + ASSERT0(BP_IS_EMBEDDED(&tmpbp)); + + zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac); + + /* + * We always MAC blk_prop in LE to ensure portability. This + * must be done after decoding the mac, since the endianness + * will get zero'd out here. + */ + zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version); + bab->bab_prop = LE_64(tmpbp.blk_prop); + bab->bab_pad = 0ULL; + + /* version 0 did not include the padding */ + *bab_len = sizeof (blkptr_auth_buf_t); + if (version == 0) + *bab_len -= sizeof (uint64_t); +} + +static int +zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version, + boolean_t should_bswap, blkptr_t *bp) +{ + uint_t bab_len; + blkptr_auth_buf_t bab; + + zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); + crypto_mac_update(ctx, &bab, bab_len); + + return (0); +} + +static void +zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version, + boolean_t should_bswap, blkptr_t *bp) +{ + uint_t bab_len; + blkptr_auth_buf_t bab; + + zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); + SHA2Update(ctx, &bab, bab_len); +} + +static void +zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version, + boolean_t should_bswap, blkptr_t *bp) +{ + uint_t bab_len; + blkptr_auth_buf_t bab; + + zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); + bcopy(&bab, *aadp, bab_len); + *aadp += bab_len; + *aad_len += bab_len; +} + +static int +zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, + boolean_t should_bswap, dnode_phys_t *dnp) +{ + int ret, i; + dnode_phys_t *adnp; + boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); + uint8_t tmp_dncore[offsetof(dnode_phys_t, dn_blkptr)]; + + /* authenticate the core dnode (masking out non-portable bits) */ + bcopy(dnp, tmp_dncore, sizeof (tmp_dncore)); + adnp = (dnode_phys_t *)tmp_dncore; + if (le_bswap) { + adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec); + adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen); + adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid); + adnp->dn_used = BSWAP_64(adnp->dn_used); + } + adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; + adnp->dn_used = 0; + + crypto_mac_update(ctx, adnp, sizeof (tmp_dncore)); + + for (i = 0; i < dnp->dn_nblkptr; i++) { + ret = zio_crypt_bp_do_hmac_updates(ctx, version, + should_bswap, &dnp->dn_blkptr[i]); + if (ret != 0) + goto error; + } + + if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { + ret = zio_crypt_bp_do_hmac_updates(ctx, version, + should_bswap, DN_SPILL_BLKPTR(dnp)); + if (ret != 0) + goto error; + } + + return (0); + +error: + return (ret); +} + +/* + * objset_phys_t blocks introduce a number of exceptions to the normal + * authentication process. objset_phys_t's contain 2 separate HMACS for + * protecting the integrity of their data. The portable_mac protects the + * metadnode. This MAC can be sent with a raw send and protects against + * reordering of data within the metadnode. The local_mac protects the user + * accounting objects which are not sent from one system to another. + * + * In addition, objset blocks are the only blocks that can be modified and + * written to disk without the key loaded under certain circumstances. During + * zil_claim() we need to be able to update the zil_header_t to complete + * claiming log blocks and during raw receives we need to write out the + * portable_mac from the send file. Both of these actions are possible + * because these fields are not protected by either MAC so neither one will + * need to modify the MACs without the key. However, when the modified blocks + * are written out they will be byteswapped into the host machine's native + * endianness which will modify fields protected by the MAC. As a result, MAC + * calculation for objset blocks works slightly differently from other block + * types. Where other block types MAC the data in whatever endianness is + * written to disk, objset blocks always MAC little endian version of their + * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP() + * and le_bswap indicates whether a byteswap is needed to get this block + * into little endian format. + */ +/* ARGSUSED */ +int +zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, + boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac) +{ + int ret; + struct hmac_ctx hash_ctx; + struct hmac_ctx *ctx = &hash_ctx; + objset_phys_t *osp = data; + uint64_t intval; + boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); + uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH]; + uint8_t raw_local_mac[SHA512_DIGEST_LENGTH]; + + + /* calculate the portable MAC from the portable fields and metadnode */ + crypto_mac_init(ctx, &key->zk_hmac_key); + + /* add in the os_type */ + intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type); + crypto_mac_update(ctx, &intval, sizeof (uint64_t)); + + /* add in the portable os_flags */ + intval = osp->os_flags; + if (should_bswap) + intval = BSWAP_64(intval); + intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK; + /* CONSTCOND */ + if (!ZFS_HOST_BYTEORDER) + intval = BSWAP_64(intval); + + crypto_mac_update(ctx, &intval, sizeof (uint64_t)); + + /* add in fields from the metadnode */ + ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, + should_bswap, &osp->os_meta_dnode); + if (ret) + goto error; + + crypto_mac_final(ctx, raw_portable_mac, SHA512_DIGEST_LENGTH); + + bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); + + /* + * The local MAC protects the user, group and project accounting. + * If these objects are not present, the local MAC is zeroed out. + */ + if ((datalen >= OBJSET_PHYS_SIZE_V3 && + osp->os_userused_dnode.dn_type == DMU_OT_NONE && + osp->os_groupused_dnode.dn_type == DMU_OT_NONE && + osp->os_projectused_dnode.dn_type == DMU_OT_NONE) || + (datalen >= OBJSET_PHYS_SIZE_V2 && + osp->os_userused_dnode.dn_type == DMU_OT_NONE && + osp->os_groupused_dnode.dn_type == DMU_OT_NONE) || + (datalen <= OBJSET_PHYS_SIZE_V1)) { + bzero(local_mac, ZIO_OBJSET_MAC_LEN); + return (0); + } + + /* calculate the local MAC from the userused and groupused dnodes */ + crypto_mac_init(ctx, &key->zk_hmac_key); + + /* add in the non-portable os_flags */ + intval = osp->os_flags; + if (should_bswap) + intval = BSWAP_64(intval); + intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK; + /* CONSTCOND */ + if (!ZFS_HOST_BYTEORDER) + intval = BSWAP_64(intval); + + crypto_mac_update(ctx, &intval, sizeof (uint64_t)); + + /* XXX check dnode type ... */ + /* add in fields from the user accounting dnodes */ + if (osp->os_userused_dnode.dn_type != DMU_OT_NONE) { + ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, + should_bswap, &osp->os_userused_dnode); + if (ret) + goto error; + } + + if (osp->os_groupused_dnode.dn_type != DMU_OT_NONE) { + ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, + should_bswap, &osp->os_groupused_dnode); + if (ret) + goto error; + } + + if (osp->os_projectused_dnode.dn_type != DMU_OT_NONE && + datalen >= OBJSET_PHYS_SIZE_V3) { + ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, + should_bswap, &osp->os_projectused_dnode); + if (ret) + goto error; + } + + crypto_mac_final(ctx, raw_local_mac, SHA512_DIGEST_LENGTH); + + bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN); + + return (0); + +error: + bzero(portable_mac, ZIO_OBJSET_MAC_LEN); + bzero(local_mac, ZIO_OBJSET_MAC_LEN); + return (ret); +} + +static void +zio_crypt_destroy_uio(uio_t *uio) +{ + if (uio->uio_iov) + kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t)); +} + +/* + * This function parses an uncompressed indirect block and returns a checksum + * of all the portable fields from all of the contained bps. The portable + * fields are the MAC and all of the fields from blk_prop except for the dedup, + * checksum, and psize bits. For an explanation of the purpose of this, see + * the comment block on object set authentication. + */ +static int +zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf, + uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum) +{ + blkptr_t *bp; + int i, epb = datalen >> SPA_BLKPTRSHIFT; + SHA2_CTX ctx; + uint8_t digestbuf[SHA512_DIGEST_LENGTH]; + + /* checksum all of the MACs from the layer below */ + SHA2Init(SHA512, &ctx); + for (i = 0, bp = buf; i < epb; i++, bp++) { + zio_crypt_bp_do_indrect_checksum_updates(&ctx, version, + byteswap, bp); + } + SHA2Final(digestbuf, &ctx); + + if (generate) { + bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN); + return (0); + } + + if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0) { +#ifdef FCRYPTO_DEBUG + printf("%s(%d): Setting ECKSUM\n", __FUNCTION__, __LINE__); +#endif + return (SET_ERROR(ECKSUM)); + } + return (0); +} + +int +zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf, + uint_t datalen, boolean_t byteswap, uint8_t *cksum) +{ + int ret; + + /* + * Unfortunately, callers of this function will not always have + * easy access to the on-disk format version. This info is + * normally found in the DSL Crypto Key, but the checksum-of-MACs + * is expected to be verifiable even when the key isn't loaded. + * Here, instead of doing a ZAP lookup for the version for each + * zio, we simply try both existing formats. + */ + ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf, + datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum); + if (ret == ECKSUM) { + ASSERT(!generate); + ret = zio_crypt_do_indirect_mac_checksum_impl(generate, + buf, datalen, 0, byteswap, cksum); + } + + return (ret); +} + +int +zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd, + uint_t datalen, boolean_t byteswap, uint8_t *cksum) +{ + int ret; + void *buf; + + buf = abd_borrow_buf_copy(abd, datalen); + ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen, + byteswap, cksum); + abd_return_buf(abd, buf, datalen); + + return (ret); +} + +/* + * Special case handling routine for encrypting / decrypting ZIL blocks. + * We do not check for the older ZIL chain because the encryption feature + * was not available before the newer ZIL chain was introduced. The goal + * here is to encrypt everything except the blkptr_t of a lr_write_t and + * the zil_chain_t header. Everything that is not encrypted is authenticated. + */ +/* + * The OpenCrypto used in FreeBSD does not use seperate source and + * destination buffers; instead, the same buffer is used. Further, to + * accomodate some of the drivers, the authbuf needs to be logically before + * the data. This means that we need to copy the source to the destination, + * and set up an extra iovec_t at the beginning to handle the authbuf. + * It also means we'll only return one uio_t, which we do via the clumsy + * ifdef in the function declaration. + */ + +/* ARGSUSED */ +static int +zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, + uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, uio_t *puio, + uio_t *out_uio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, + boolean_t *no_crypt) +{ + int ret; + uint64_t txtype, lr_len; + uint_t nr_src, nr_dst, crypt_len; + uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; + iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; + uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp; + zil_chain_t *zilc; + lr_t *lr; + uint8_t *aadbuf = zio_buf_alloc(datalen); + + /* cipherbuf always needs an extra iovec for the MAC */ + if (encrypt) { + src = plainbuf; + dst = cipherbuf; + nr_src = 0; + nr_dst = 1; + } else { + src = cipherbuf; + dst = plainbuf; + nr_src = 1; + nr_dst = 0; + } + + /* + * We need at least two iovecs -- one for the AAD, + * one for the MAC. + */ + bcopy(src, dst, datalen); + nr_dst = 2; + + /* find the start and end record of the log block */ + zilc = (zil_chain_t *)src; + slrp = src + sizeof (zil_chain_t); + aadp = aadbuf; + blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused); + + /* calculate the number of encrypted iovecs we will need */ + for (; slrp < blkend; slrp += lr_len) { + lr = (lr_t *)slrp; + + if (!byteswap) { + txtype = lr->lrc_txtype; + lr_len = lr->lrc_reclen; + } else { + txtype = BSWAP_64(lr->lrc_txtype); + lr_len = BSWAP_64(lr->lrc_reclen); + } + + nr_iovecs++; + if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t)) + nr_iovecs++; + } + + nr_src = 0; + nr_dst += nr_iovecs; + + /* allocate the iovec arrays */ + if (nr_src != 0) { + src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); + if (src_iovecs == NULL) { + ret = SET_ERROR(ENOMEM); + goto error; + } + bzero(src_iovecs, nr_src * sizeof (iovec_t)); + } + + if (nr_dst != 0) { + dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); + if (dst_iovecs == NULL) { + ret = SET_ERROR(ENOMEM); + goto error; + } + bzero(dst_iovecs, nr_dst * sizeof (iovec_t)); + } + + /* + * Copy the plain zil header over and authenticate everything except + * the checksum that will store our MAC. If we are writing the data + * the embedded checksum will not have been calculated yet, so we don't + * authenticate that. + */ + bcopy(src, dst, sizeof (zil_chain_t)); + bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t)); + aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t); + aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t); + + /* loop over records again, filling in iovecs */ + /* The first one will contain the authbuf */ + nr_iovecs = 1; + + slrp = src + sizeof (zil_chain_t); + dlrp = dst + sizeof (zil_chain_t); + + for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) { + lr = (lr_t *)slrp; + + if (!byteswap) { + txtype = lr->lrc_txtype; + lr_len = lr->lrc_reclen; + } else { + txtype = BSWAP_64(lr->lrc_txtype); + lr_len = BSWAP_64(lr->lrc_reclen); + } + + /* copy the common lr_t */ + bcopy(slrp, dlrp, sizeof (lr_t)); + bcopy(slrp, aadp, sizeof (lr_t)); + aadp += sizeof (lr_t); + aad_len += sizeof (lr_t); + + ASSERT3P(dst_iovecs, !=, NULL); + + /* + * If this is a TX_WRITE record we want to encrypt everything + * except the bp if exists. If the bp does exist we want to + * authenticate it. + */ + if (txtype == TX_WRITE) { + crypt_len = sizeof (lr_write_t) - + sizeof (lr_t) - sizeof (blkptr_t); + dst_iovecs[nr_iovecs].iov_base = (char *)dlrp + + sizeof (lr_t); + dst_iovecs[nr_iovecs].iov_len = crypt_len; + + /* copy the bp now since it will not be encrypted */ + bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), + dlrp + sizeof (lr_write_t) - sizeof (blkptr_t), + sizeof (blkptr_t)); + bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), + aadp, sizeof (blkptr_t)); + aadp += sizeof (blkptr_t); + aad_len += sizeof (blkptr_t); + nr_iovecs++; + total_len += crypt_len; + + if (lr_len != sizeof (lr_write_t)) { + crypt_len = lr_len - sizeof (lr_write_t); + dst_iovecs[nr_iovecs].iov_base = (char *) + dlrp + sizeof (lr_write_t); + dst_iovecs[nr_iovecs].iov_len = crypt_len; + nr_iovecs++; + total_len += crypt_len; + } + } else { + crypt_len = lr_len - sizeof (lr_t); + dst_iovecs[nr_iovecs].iov_base = (char *)dlrp + + sizeof (lr_t); + dst_iovecs[nr_iovecs].iov_len = crypt_len; + nr_iovecs++; + total_len += crypt_len; + } + } + + *no_crypt = (nr_iovecs == 0); + *enc_len = total_len; + *authbuf = aadbuf; + *auth_len = aad_len; + dst_iovecs[0].iov_base = aadbuf; + dst_iovecs[0].iov_len = aad_len; + + out_uio->uio_iov = dst_iovecs; + out_uio->uio_iovcnt = nr_dst; + + return (0); + +error: + zio_buf_free(aadbuf, datalen); + if (src_iovecs != NULL) + kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); + if (dst_iovecs != NULL) + kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); + + *enc_len = 0; + *authbuf = NULL; + *auth_len = 0; + *no_crypt = B_FALSE; + puio->uio_iov = NULL; + puio->uio_iovcnt = 0; + out_uio->uio_iov = NULL; + out_uio->uio_iovcnt = 0; + + return (ret); +} + +/* + * Special case handling routine for encrypting / decrypting dnode blocks. + */ +static int +zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, + uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, + uio_t *puio, uio_t *out_uio, uint_t *enc_len, uint8_t **authbuf, + uint_t *auth_len, boolean_t *no_crypt) +{ + int ret; + uint_t nr_src, nr_dst, crypt_len; + uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; + uint_t i, j, max_dnp = datalen >> DNODE_SHIFT; + iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; + uint8_t *src, *dst, *aadp; + dnode_phys_t *dnp, *adnp, *sdnp, *ddnp; + uint8_t *aadbuf = zio_buf_alloc(datalen); + + if (encrypt) { + src = plainbuf; + dst = cipherbuf; + nr_src = 0; + nr_dst = 1; + } else { + src = cipherbuf; + dst = plainbuf; + nr_src = 1; + nr_dst = 0; + } + + bcopy(src, dst, datalen); + nr_dst = 2; + + sdnp = (dnode_phys_t *)src; + ddnp = (dnode_phys_t *)dst; + aadp = aadbuf; + + /* + * Count the number of iovecs we will need to do the encryption by + * counting the number of bonus buffers that need to be encrypted. + */ + for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { + /* + * This block may still be byteswapped. However, all of the + * values we use are either uint8_t's (for which byteswapping + * is a noop) or a * != 0 check, which will work regardless + * of whether or not we byteswap. + */ + if (sdnp[i].dn_type != DMU_OT_NONE && + DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) && + sdnp[i].dn_bonuslen != 0) { + nr_iovecs++; + } + } + + nr_src = 0; + nr_dst += nr_iovecs; + + if (nr_src != 0) { + src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); + if (src_iovecs == NULL) { + ret = SET_ERROR(ENOMEM); + goto error; + } + bzero(src_iovecs, nr_src * sizeof (iovec_t)); + } + + if (nr_dst != 0) { + dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); + if (dst_iovecs == NULL) { + ret = SET_ERROR(ENOMEM); + goto error; + } + bzero(dst_iovecs, nr_dst * sizeof (iovec_t)); + } + + nr_iovecs = 1; + + /* + * Iterate through the dnodes again, this time filling in the uios + * we allocated earlier. We also concatenate any data we want to + * authenticate onto aadbuf. + */ + for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { + dnp = &sdnp[i]; + + /* copy over the core fields and blkptrs (kept as plaintext) */ + bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp); + + if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { + bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]), + sizeof (blkptr_t)); + } + + /* + * Handle authenticated data. We authenticate everything in + * the dnode that can be brought over when we do a raw send. + * This includes all of the core fields as well as the MACs + * stored in the bp checksums and all of the portable bits + * from blk_prop. We include the dnode padding here in case it + * ever gets used in the future. Some dn_flags and dn_used are + * not portable so we mask those out values out of the + * authenticated data. + */ + crypt_len = offsetof(dnode_phys_t, dn_blkptr); + bcopy(dnp, aadp, crypt_len); + adnp = (dnode_phys_t *)aadp; + adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; + adnp->dn_used = 0; + aadp += crypt_len; + aad_len += crypt_len; + + for (j = 0; j < dnp->dn_nblkptr; j++) { + zio_crypt_bp_do_aad_updates(&aadp, &aad_len, + version, byteswap, &dnp->dn_blkptr[j]); + } + + if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { + zio_crypt_bp_do_aad_updates(&aadp, &aad_len, + version, byteswap, DN_SPILL_BLKPTR(dnp)); + } + + /* + * If this bonus buffer needs to be encrypted, we prepare an + * iovec_t. The encryption / decryption functions will fill + * this in for us with the encrypted or decrypted data. + * Otherwise we add the bonus buffer to the authenticated + * data buffer and copy it over to the destination. The + * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that + * we can guarantee alignment with the AES block size + * (128 bits). + */ + crypt_len = DN_MAX_BONUS_LEN(dnp); + if (dnp->dn_type != DMU_OT_NONE && + DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && + dnp->dn_bonuslen != 0) { + ASSERT3U(nr_iovecs, <, nr_dst); + ASSERT3P(dst_iovecs, !=, NULL); + dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]); + dst_iovecs[nr_iovecs].iov_len = crypt_len; + + nr_iovecs++; + total_len += crypt_len; + } else { + bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len); + bcopy(DN_BONUS(dnp), aadp, crypt_len); + aadp += crypt_len; + aad_len += crypt_len; + } + } + + *no_crypt = (nr_iovecs == 0); + *enc_len = total_len; + *authbuf = aadbuf; + *auth_len = aad_len; + + dst_iovecs[0].iov_base = aadbuf; + dst_iovecs[0].iov_len = aad_len; + out_uio->uio_iov = dst_iovecs; + out_uio->uio_iovcnt = nr_dst; + + return (0); + +error: + zio_buf_free(aadbuf, datalen); + if (src_iovecs != NULL) + kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); + if (dst_iovecs != NULL) + kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); + + *enc_len = 0; + *authbuf = NULL; + *auth_len = 0; + *no_crypt = B_FALSE; + out_uio->uio_iov = NULL; + out_uio->uio_iovcnt = 0; + + return (ret); +} + +/* ARGSUSED */ +static int +zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, + uint8_t *cipherbuf, uint_t datalen, uio_t *puio, uio_t *out_uio, + uint_t *enc_len) +{ + int ret; + uint_t nr_plain = 1, nr_cipher = 2; + iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; + void *src, *dst; + + cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t), + KM_SLEEP); + if (!cipher_iovecs) { + ret = SET_ERROR(ENOMEM); + goto error; + } + bzero(cipher_iovecs, nr_cipher * sizeof (iovec_t)); + + if (encrypt) { + src = plainbuf; + dst = cipherbuf; + } else { + src = cipherbuf; + dst = plainbuf; + } + bcopy(src, dst, datalen); + cipher_iovecs[0].iov_base = dst; + cipher_iovecs[0].iov_len = datalen; + + *enc_len = datalen; + out_uio->uio_iov = cipher_iovecs; + out_uio->uio_iovcnt = nr_cipher; + + return (0); + +error: + if (plain_iovecs != NULL) + kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t)); + if (cipher_iovecs != NULL) + kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t)); + + *enc_len = 0; + out_uio->uio_iov = NULL; + out_uio->uio_iovcnt = 0; + + return (ret); +} + +/* + * This function builds up the plaintext (puio) and ciphertext (cuio) uios so + * that they can be used for encryption and decryption by zio_do_crypt_uio(). + * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks + * requiring special handling to parse out pieces that are to be encrypted. The + * authbuf is used by these special cases to store additional authenticated + * data (AAD) for the encryption modes. + */ +static int +zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot, + uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, + uint8_t *mac, uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, + uint_t *auth_len, boolean_t *no_crypt) +{ + int ret; + iovec_t *mac_iov; + + ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE); + + /* route to handler */ + switch (ot) { + case DMU_OT_INTENT_LOG: + ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf, + datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, + no_crypt); + break; + case DMU_OT_DNODE: + ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf, + cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, + auth_len, no_crypt); + break; + default: + ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf, + datalen, puio, cuio, enc_len); + *authbuf = NULL; + *auth_len = 0; + *no_crypt = B_FALSE; + break; + } + + if (ret != 0) + goto error; + + /* populate the uios */ + cuio->uio_segflg = UIO_SYSSPACE; + + mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]); + mac_iov->iov_base = (void *)mac; + mac_iov->iov_len = ZIO_DATA_MAC_LEN; + + return (0); + +error: + return (ret); +} + +void *failed_decrypt_buf; +int faile_decrypt_size; + +/* + * Primary encryption / decryption entrypoint for zio data. + */ +int +zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, + dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv, + uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf, + boolean_t *no_crypt) +{ + int ret; + boolean_t locked = B_FALSE; + uint64_t crypt = key->zk_crypt; + uint_t keydata_len = zio_crypt_table[crypt].ci_keylen; + uint_t enc_len, auth_len; + uio_t puio, cuio; + uint8_t enc_keydata[MASTER_KEY_MAX_LEN]; + crypto_key_t tmp_ckey, *ckey = NULL; + freebsd_crypt_session_t *tmpl = NULL; + uint8_t *authbuf = NULL; + + bzero(&puio, sizeof (uio_t)); + bzero(&cuio, sizeof (uio_t)); + +#ifdef FCRYPTO_DEBUG + printf("%s(%s, %p, %p, %d, %p, %p, %u, %s, %p, %p, %p)\n", + __FUNCTION__, + encrypt ? "encrypt" : "decrypt", + key, salt, ot, iv, mac, datalen, + byteswap ? "byteswap" : "native_endian", plainbuf, + cipherbuf, no_crypt); + + printf("\tkey = {"); + for (int i = 0; i < key->zk_current_key.ck_length/8; i++) + printf("%02x ", ((uint8_t *)key->zk_current_key.ck_data)[i]); + printf("}\n"); +#endif + /* create uios for encryption */ + ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, + cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, + &authbuf, &auth_len, no_crypt); + if (ret != 0) + return (ret); + + /* + * If the needed key is the current one, just use it. Otherwise we + * need to generate a temporary one from the given salt + master key. + * If we are encrypting, we must return a copy of the current salt + * so that it can be stored in the blkptr_t. + */ + rw_enter(&key->zk_salt_lock, RW_READER); + locked = B_TRUE; + + if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) { + ckey = &key->zk_current_key; + tmpl = &key->zk_session; + } else { + rw_exit(&key->zk_salt_lock); + locked = B_FALSE; + + ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, + salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len); + if (ret != 0) + goto error; + tmp_ckey.ck_format = CRYPTO_KEY_RAW; + tmp_ckey.ck_data = enc_keydata; + tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len); + + ckey = &tmp_ckey; + tmpl = NULL; + } + + /* perform the encryption / decryption */ + ret = zio_do_crypt_uio_opencrypto(encrypt, tmpl, key->zk_crypt, + ckey, iv, enc_len, &cuio, auth_len); + if (ret != 0) + goto error; + if (locked) { + rw_exit(&key->zk_salt_lock); + locked = B_FALSE; + } + + if (authbuf != NULL) + zio_buf_free(authbuf, datalen); + if (ckey == &tmp_ckey) + bzero(enc_keydata, keydata_len); + zio_crypt_destroy_uio(&puio); + zio_crypt_destroy_uio(&cuio); + + return (0); + +error: + if (!encrypt) { + if (failed_decrypt_buf != NULL) + kmem_free(failed_decrypt_buf, failed_decrypt_size); + failed_decrypt_buf = kmem_alloc(datalen, KM_SLEEP); + failed_decrypt_size = datalen; + bcopy(cipherbuf, failed_decrypt_buf, datalen); + } + if (locked) + rw_exit(&key->zk_salt_lock); + if (authbuf != NULL) + zio_buf_free(authbuf, datalen); + if (ckey == &tmp_ckey) + bzero(enc_keydata, keydata_len); + zio_crypt_destroy_uio(&puio); + zio_crypt_destroy_uio(&cuio); + return (SET_ERROR(ret)); +} + +/* + * Simple wrapper around zio_do_crypt_data() to work with abd's instead of + * linear buffers. + */ +int +zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, + boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac, + uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt) +{ + int ret; + void *ptmp, *ctmp; + + if (encrypt) { + ptmp = abd_borrow_buf_copy(pabd, datalen); + ctmp = abd_borrow_buf(cabd, datalen); + } else { + ptmp = abd_borrow_buf(pabd, datalen); + ctmp = abd_borrow_buf_copy(cabd, datalen); + } + + ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac, + datalen, ptmp, ctmp, no_crypt); + if (ret != 0) + goto error; + + if (encrypt) { + abd_return_buf(pabd, ptmp, datalen); + abd_return_buf_copy(cabd, ctmp, datalen); + } else { + abd_return_buf_copy(pabd, ptmp, datalen); + abd_return_buf(cabd, ctmp, datalen); + } + + return (0); + +error: + if (encrypt) { + abd_return_buf(pabd, ptmp, datalen); + abd_return_buf_copy(cabd, ctmp, datalen); + } else { + abd_return_buf_copy(pabd, ptmp, datalen); + abd_return_buf(cabd, ctmp, datalen); + } + + return (SET_ERROR(ret)); +} + +#if defined(_KERNEL) && defined(HAVE_SPL) +/* BEGIN CSTYLED */ +module_param(zfs_key_max_salt_uses, ulong, 0644); +MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value " + "can be used for generating encryption keys before it is rotated"); +/* END CSTYLED */ +#endif diff --git a/module/os/freebsd/zfs/zvol_os.c b/module/os/freebsd/zfs/zvol_os.c new file mode 100644 index 00000000000..bef97a9b34a --- /dev/null +++ b/module/os/freebsd/zfs/zvol_os.c @@ -0,0 +1,1476 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * + * Copyright (c) 2006-2010 Pawel Jakub Dawidek + * All rights reserved. + * + * Portions Copyright 2010 Robert Milkowski + * + * Copyright 2011 Nexenta Systems, Inc. All rights reserved. + * Copyright (c) 2012, 2017 by Delphix. All rights reserved. + * Copyright (c) 2013, Joyent, Inc. All rights reserved. + * Copyright (c) 2014 Integros [integros.com] + */ + +/* Portions Copyright 2011 Martin Matuska */ + +/* + * ZFS volume emulation driver. + * + * Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes. + * Volumes are accessed through the symbolic links named: + * + * /dev/zvol// + * + * Volumes are persistent through reboot. No user command needs to be + * run before opening and using a device. + * + * On FreeBSD ZVOLs are simply GEOM providers like any other storage device + * in the system. Except when they're simply character devices (volmode=dev). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "zfs_namecheck.h" + +#define ZVOL_DUMPSIZE "dumpsize" + +#ifdef ZVOL_LOCK_DEBUG +#define ZVOL_RW_READER RW_WRITER +#define ZVOL_RW_READ_HELD RW_WRITE_HELD +#else +#define ZVOL_RW_READER RW_READER +#define ZVOL_RW_READ_HELD RW_READ_HELD +#endif + +enum zvol_geom_state { + ZVOL_GEOM_UNINIT, + ZVOL_GEOM_STOPPED, + ZVOL_GEOM_RUNNING, +}; + +struct zvol_state_os { + int zso_volmode; +#define zso_dev _zso_state._zso_dev +#define zso_geom _zso_state._zso_geom + union { + /* volmode=dev */ + struct zvol_state_dev { + struct cdev *zsd_cdev; + uint64_t zsd_sync_cnt; + } _zso_dev; + + /* volmode=geom */ + struct zvol_state_geom { + struct g_provider *zsg_provider; + struct bio_queue_head zsg_queue; + struct mtx zsg_queue_mtx; + enum zvol_geom_state zsg_state; + } _zso_geom; + } _zso_state; +}; + +struct proc *zfsproc; + +static uint32_t zvol_minors; + +SYSCTL_DECL(_vfs_zfs); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, vol, CTLFLAG_RW, 0, "ZFS VOLUME"); +SYSCTL_INT(_vfs_zfs_vol, OID_AUTO, mode, CTLFLAG_RWTUN, &zvol_volmode, 0, + "Expose as GEOM providers (1), device files (2) or neither"); +static boolean_t zpool_on_zvol = B_FALSE; +SYSCTL_INT(_vfs_zfs_vol, OID_AUTO, recursive, CTLFLAG_RWTUN, &zpool_on_zvol, 0, + "Allow zpools to use zvols as vdevs (DANGEROUS)"); + +/* + * Toggle unmap functionality. + */ +boolean_t zvol_unmap_enabled = B_TRUE; + +SYSCTL_INT(_vfs_zfs_vol, OID_AUTO, unmap_enabled, CTLFLAG_RWTUN, + &zvol_unmap_enabled, 0, "Enable UNMAP functionality"); + +/* + * zvol maximum transfer in one DMU tx. + */ +int zvol_maxphys = DMU_MAX_ACCESS / 2; + +static void zvol_ensure_zilog(zvol_state_t *zv); + +static d_open_t zvol_cdev_open; +static d_close_t zvol_cdev_close; +static d_ioctl_t zvol_cdev_ioctl; +static d_read_t zvol_cdev_read; +static d_write_t zvol_cdev_write; +static d_strategy_t zvol_geom_bio_strategy; + +static struct cdevsw zvol_cdevsw = { + .d_name = "zvol", + .d_version = D_VERSION, + .d_flags = D_DISK | D_TRACKCLOSE, + .d_open = zvol_cdev_open, + .d_close = zvol_cdev_close, + .d_ioctl = zvol_cdev_ioctl, + .d_read = zvol_cdev_read, + .d_write = zvol_cdev_write, + .d_strategy = zvol_geom_bio_strategy, +}; + +extern uint_t zfs_geom_probe_vdev_key; + +struct g_class zfs_zvol_class = { + .name = "ZFS::ZVOL", + .version = G_VERSION, +}; + +DECLARE_GEOM_CLASS(zfs_zvol_class, zfs_zvol); + +static int zvol_geom_open(struct g_provider *pp, int flag, int count); +static int zvol_geom_close(struct g_provider *pp, int flag, int count); +static void zvol_geom_run(zvol_state_t *zv); +static void zvol_geom_destroy(zvol_state_t *zv); +static int zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace); +static void zvol_geom_worker(void *arg); +static void zvol_geom_bio_start(struct bio *bp); +static int zvol_geom_bio_getattr(struct bio *bp); +static void zvol_geom_bio_check_zilog(struct bio *bp); +/* static d_strategy_t zvol_geom_bio_strategy; (declared elsewhere) */ + +/* + * GEOM mode implementation + */ + +/*ARGSUSED*/ +static int +zvol_geom_open(struct g_provider *pp, int flag, int count) +{ + zvol_state_t *zv; + int err = 0; + boolean_t drop_suspend = B_TRUE; + + if (!zpool_on_zvol && tsd_get(zfs_geom_probe_vdev_key) != NULL) { + /* + * if zfs_geom_probe_vdev_key is set, that means that zfs is + * attempting to probe geom providers while looking for a + * replacement for a missing VDEV. In this case, the + * spa_namespace_lock will not be held, but it is still illegal + * to use a zvol as a vdev. Deadlocks can result if another + * thread has spa_namespace_lock + */ + return (SET_ERROR(EOPNOTSUPP)); + } + + rw_enter(&zvol_state_lock, ZVOL_RW_READER); + zv = pp->private; + if (zv == NULL) { + rw_exit(&zvol_state_lock); + return (SET_ERROR(ENXIO)); + } + + mutex_enter(&zv->zv_state_lock); + + ASSERT(zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM); + + /* + * make sure zvol is not suspended during first open + * (hold zv_suspend_lock) and respect proper lock acquisition + * ordering - zv_suspend_lock before zv_state_lock + */ + if (zv->zv_open_count == 0) { + if (!rw_tryenter(&zv->zv_suspend_lock, ZVOL_RW_READER)) { + mutex_exit(&zv->zv_state_lock); + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); + mutex_enter(&zv->zv_state_lock); + /* check to see if zv_suspend_lock is needed */ + if (zv->zv_open_count != 0) { + rw_exit(&zv->zv_suspend_lock); + drop_suspend = B_FALSE; + } + } + } else { + drop_suspend = B_FALSE; + } + rw_exit(&zvol_state_lock); + + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); + + if (zv->zv_open_count == 0) { + ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock)); + err = zvol_first_open(zv, !(flag & FWRITE)); + if (err) + goto out_mutex; + pp->mediasize = zv->zv_volsize; + pp->stripeoffset = 0; + pp->stripesize = zv->zv_volblocksize; + } + + /* + * Check for a bad on-disk format version now since we + * lied about owning the dataset readonly before. + */ + if ((flag & FWRITE) && ((zv->zv_flags & ZVOL_RDONLY) || + dmu_objset_incompatible_encryption_version(zv->zv_objset))) { + err = EROFS; + goto out_open_count; + } + if (zv->zv_flags & ZVOL_EXCL) { + err = EBUSY; + goto out_open_count; + } +#ifdef FEXCL + if (flag & FEXCL) { + if (zv->zv_open_count != 0) { + err = EBUSY; + goto out_open_count; + } + zv->zv_flags |= ZVOL_EXCL; + } +#endif + + zv->zv_open_count += count; + mutex_exit(&zv->zv_state_lock); + if (drop_suspend) + rw_exit(&zv->zv_suspend_lock); + return (0); + +out_open_count: + if (zv->zv_open_count == 0) + zvol_last_close(zv); +out_mutex: + mutex_exit(&zv->zv_state_lock); + if (drop_suspend) + rw_exit(&zv->zv_suspend_lock); + return (SET_ERROR(err)); +} + +/*ARGSUSED*/ +static int +zvol_geom_close(struct g_provider *pp, int flag, int count) +{ + zvol_state_t *zv; + boolean_t drop_suspend = B_TRUE; + + rw_enter(&zvol_state_lock, ZVOL_RW_READER); + zv = pp->private; + if (zv == NULL) { + rw_exit(&zvol_state_lock); + return (SET_ERROR(ENXIO)); + } + + mutex_enter(&zv->zv_state_lock); + if (zv->zv_flags & ZVOL_EXCL) { + ASSERT(zv->zv_open_count == 1); + zv->zv_flags &= ~ZVOL_EXCL; + } + + ASSERT(zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM); + + /* + * If the open count is zero, this is a spurious close. + * That indicates a bug in the kernel / DDI framework. + */ + ASSERT(zv->zv_open_count > 0); + + /* + * make sure zvol is not suspended during last close + * (hold zv_suspend_lock) and respect proper lock acquisition + * ordering - zv_suspend_lock before zv_state_lock + */ + if ((zv->zv_open_count - count) == 0) { + if (!rw_tryenter(&zv->zv_suspend_lock, ZVOL_RW_READER)) { + mutex_exit(&zv->zv_state_lock); + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); + mutex_enter(&zv->zv_state_lock); + /* check to see if zv_suspend_lock is needed */ + if (zv->zv_open_count != 1) { + rw_exit(&zv->zv_suspend_lock); + drop_suspend = B_FALSE; + } + } + } else { + drop_suspend = B_FALSE; + } + rw_exit(&zvol_state_lock); + + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); + + /* + * You may get multiple opens, but only one close. + */ + zv->zv_open_count -= count; + + if (zv->zv_open_count == 0) { + ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock)); + zvol_last_close(zv); + } + + mutex_exit(&zv->zv_state_lock); + + if (drop_suspend) + rw_exit(&zv->zv_suspend_lock); + return (0); +} + +static void +zvol_geom_run(zvol_state_t *zv) +{ + struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; + struct g_provider *pp = zsg->zsg_provider; + + ASSERT(zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM); + + g_error_provider(pp, 0); + + kproc_kthread_add(zvol_geom_worker, zv, &zfsproc, NULL, 0, 0, + "zfskern", "zvol %s", pp->name + sizeof (ZVOL_DRIVER)); +} + +static void +zvol_geom_destroy(zvol_state_t *zv) +{ + struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; + struct g_provider *pp = zsg->zsg_provider; + + ASSERT(zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM); + + g_topology_assert(); + + mutex_enter(&zv->zv_state_lock); + VERIFY(zsg->zsg_state == ZVOL_GEOM_RUNNING); + mutex_exit(&zv->zv_state_lock); + zsg->zsg_provider = NULL; + pp->private = NULL; + g_wither_geom(pp->geom, ENXIO); +} + +static int +zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace) +{ + int count, error, flags; + + g_topology_assert(); + + /* + * To make it easier we expect either open or close, but not both + * at the same time. + */ + KASSERT((acr >= 0 && acw >= 0 && ace >= 0) || + (acr <= 0 && acw <= 0 && ace <= 0), + ("Unsupported access request to %s (acr=%d, acw=%d, ace=%d).", + pp->name, acr, acw, ace)); + + if (pp->private == NULL) { + if (acr <= 0 && acw <= 0 && ace <= 0) + return (0); + return (pp->error); + } + + /* + * We don't pass FEXCL flag to zvol_geom_open()/zvol_geom_close() if + * ace != 0, because GEOM already handles that and handles it a bit + * differently. GEOM allows for multiple read/exclusive consumers and + * ZFS allows only one exclusive consumer, no matter if it is reader or + * writer. I like better the way GEOM works so I'll leave it for GEOM + * to decide what to do. + */ + + count = acr + acw + ace; + if (count == 0) + return (0); + + flags = 0; + if (acr != 0 || ace != 0) + flags |= FREAD; + if (acw != 0) + flags |= FWRITE; + + g_topology_unlock(); + if (count > 0) + error = zvol_geom_open(pp, flags, count); + else + error = zvol_geom_close(pp, flags, -count); + g_topology_lock(); + return (error); +} + +static void +zvol_geom_worker(void *arg) +{ + zvol_state_t *zv = arg; + struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; + struct bio *bp; + + ASSERT(zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM); + + thread_lock(curthread); + sched_prio(curthread, PRIBIO); + thread_unlock(curthread); + + for (;;) { + mtx_lock(&zsg->zsg_queue_mtx); + bp = bioq_takefirst(&zsg->zsg_queue); + if (bp == NULL) { + if (zsg->zsg_state == ZVOL_GEOM_STOPPED) { + zsg->zsg_state = ZVOL_GEOM_RUNNING; + wakeup(&zsg->zsg_state); + mtx_unlock(&zsg->zsg_queue_mtx); + kthread_exit(); + } + msleep(&zsg->zsg_queue, &zsg->zsg_queue_mtx, + PRIBIO | PDROP, "zvol:io", 0); + continue; + } + mtx_unlock(&zsg->zsg_queue_mtx); + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); + zvol_geom_bio_check_zilog(bp); + switch (bp->bio_cmd) { + case BIO_FLUSH: + zil_commit(zv->zv_zilog, ZVOL_OBJ); + g_io_deliver(bp, 0); + break; + case BIO_READ: + case BIO_WRITE: + case BIO_DELETE: + zvol_geom_bio_strategy(bp); + break; + default: + g_io_deliver(bp, EOPNOTSUPP); + break; + } + rw_exit(&zv->zv_suspend_lock); + } +} + +static void +zvol_geom_bio_start(struct bio *bp) +{ + zvol_state_t *zv = bp->bio_to->private; + struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; + boolean_t first; + + if (bp->bio_cmd == BIO_GETATTR) { + if (zvol_geom_bio_getattr(bp)) + g_io_deliver(bp, EOPNOTSUPP); + return; + } + + if (!THREAD_CAN_SLEEP()) { + mtx_lock(&zsg->zsg_queue_mtx); + first = (bioq_first(&zsg->zsg_queue) == NULL); + bioq_insert_tail(&zsg->zsg_queue, bp); + mtx_unlock(&zsg->zsg_queue_mtx); + if (first) + wakeup_one(&zsg->zsg_queue); + return; + } + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); + zvol_geom_bio_check_zilog(bp); + + switch (bp->bio_cmd) { + case BIO_FLUSH: + zil_commit(zv->zv_zilog, ZVOL_OBJ); + g_io_deliver(bp, 0); + break; + case BIO_READ: + case BIO_WRITE: + case BIO_DELETE: + zvol_geom_bio_strategy(bp); + break; + default: + g_io_deliver(bp, EOPNOTSUPP); + break; + } + rw_exit(&zv->zv_suspend_lock); +} + +static int +zvol_geom_bio_getattr(struct bio *bp) +{ + zvol_state_t *zv; + + zv = bp->bio_to->private; + ASSERT(zv != NULL); + + spa_t *spa = dmu_objset_spa(zv->zv_objset); + uint64_t refd, avail, usedobjs, availobjs; + + if (g_handleattr_int(bp, "GEOM::candelete", 1)) + return (0); + if (strcmp(bp->bio_attribute, "blocksavail") == 0) { + dmu_objset_space(zv->zv_objset, &refd, &avail, + &usedobjs, &availobjs); + if (g_handleattr_off_t(bp, "blocksavail", avail / DEV_BSIZE)) + return (0); + } else if (strcmp(bp->bio_attribute, "blocksused") == 0) { + dmu_objset_space(zv->zv_objset, &refd, &avail, + &usedobjs, &availobjs); + if (g_handleattr_off_t(bp, "blocksused", refd / DEV_BSIZE)) + return (0); + } else if (strcmp(bp->bio_attribute, "poolblocksavail") == 0) { + avail = metaslab_class_get_space(spa_normal_class(spa)); + avail -= metaslab_class_get_alloc(spa_normal_class(spa)); + if (g_handleattr_off_t(bp, "poolblocksavail", + avail / DEV_BSIZE)) + return (0); + } else if (strcmp(bp->bio_attribute, "poolblocksused") == 0) { + refd = metaslab_class_get_alloc(spa_normal_class(spa)); + if (g_handleattr_off_t(bp, "poolblocksused", refd / DEV_BSIZE)) + return (0); + } + return (1); +} + +static void +zvol_geom_bio_check_zilog(struct bio *bp) +{ + zvol_state_t *zv; + + zv = bp->bio_to->private; + ASSERT(zv != NULL); + + switch (bp->bio_cmd) { + case BIO_FLUSH: + case BIO_WRITE: + case BIO_DELETE: + zvol_ensure_zilog(zv); + default: + break; + } +} + +static void +zvol_geom_bio_strategy(struct bio *bp) +{ + zvol_state_t *zv; + uint64_t off, volsize; + size_t resid; + char *addr; + objset_t *os; + zfs_locked_range_t *lr; + int error = 0; + boolean_t doread = 0; + boolean_t is_dumpified; + boolean_t sync; + + if (bp->bio_to) + zv = bp->bio_to->private; + else + zv = bp->bio_dev->si_drv2; + + if (zv == NULL) { + error = SET_ERROR(ENXIO); + goto out; + } + + if (bp->bio_cmd != BIO_READ && (zv->zv_flags & ZVOL_RDONLY)) { + error = SET_ERROR(EROFS); + goto out; + } + + switch (bp->bio_cmd) { + case BIO_FLUSH: + goto sync; + case BIO_READ: + doread = 1; + case BIO_WRITE: + case BIO_DELETE: + break; + default: + error = EOPNOTSUPP; + goto out; + } + + off = bp->bio_offset; + volsize = zv->zv_volsize; + + os = zv->zv_objset; + ASSERT(os != NULL); + + addr = bp->bio_data; + resid = bp->bio_length; + + if (resid > 0 && (off < 0 || off >= volsize)) { + error = SET_ERROR(EIO); + goto out; + } + + is_dumpified = B_FALSE; + sync = !doread && !is_dumpified && + zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS; + + /* + * There must be no buffer changes when doing a dmu_sync() because + * we can't change the data whilst calculating the checksum. + */ + lr = zfs_rangelock_enter(&zv->zv_rangelock, off, resid, + doread ? RL_READER : RL_WRITER); + + if (bp->bio_cmd == BIO_DELETE) { + dmu_tx_t *tx = dmu_tx_create(zv->zv_objset); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error != 0) { + dmu_tx_abort(tx); + } else { + zvol_log_truncate(zv, tx, off, resid, sync); + dmu_tx_commit(tx); + error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, + off, resid); + resid = 0; + } + goto unlock; + } + while (resid != 0 && off < volsize) { + size_t size = MIN(resid, zvol_maxphys); + if (doread) { + error = dmu_read(os, ZVOL_OBJ, off, size, addr, + DMU_READ_PREFETCH); + } else { + dmu_tx_t *tx = dmu_tx_create(os); + dmu_tx_hold_write_by_dnode(tx, zv->zv_dn, off, size); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + } else { + dmu_write(os, ZVOL_OBJ, off, size, addr, tx); + zvol_log_write(zv, tx, off, size, sync); + dmu_tx_commit(tx); + } + } + if (error) { + /* convert checksum errors into IO errors */ + if (error == ECKSUM) + error = SET_ERROR(EIO); + break; + } + off += size; + addr += size; + resid -= size; + } +unlock: + zfs_rangelock_exit(lr); + + bp->bio_completed = bp->bio_length - resid; + if (bp->bio_completed < bp->bio_length && off > volsize) + error = EINVAL; + + if (sync) { +sync: + zil_commit(zv->zv_zilog, ZVOL_OBJ); + } +out: + if (bp->bio_to) + g_io_deliver(bp, error); + else + biofinish(bp, NULL, error); +} + +/* + * Character device mode implementation + */ + +static int +zvol_cdev_read(struct cdev *dev, struct uio *uio, int ioflag) +{ + zvol_state_t *zv; + uint64_t volsize; + zfs_locked_range_t *lr; + int error = 0; + + zv = dev->si_drv2; + + volsize = zv->zv_volsize; + /* + * uio_loffset == volsize isn't an error as + * its required for EOF processing. + */ + if (uio->uio_resid > 0 && + (uio->uio_loffset < 0 || uio->uio_loffset > volsize)) + return (SET_ERROR(EIO)); + + lr = zfs_rangelock_enter(&zv->zv_rangelock, uio->uio_loffset, + uio->uio_resid, RL_READER); + while (uio->uio_resid > 0 && uio->uio_loffset < volsize) { + uint64_t bytes = MIN(uio->uio_resid, DMU_MAX_ACCESS >> 1); + + /* don't read past the end */ + if (bytes > volsize - uio->uio_loffset) + bytes = volsize - uio->uio_loffset; + + error = dmu_read_uio_dnode(zv->zv_dn, uio, bytes); + if (error) { + /* convert checksum errors into IO errors */ + if (error == ECKSUM) + error = SET_ERROR(EIO); + break; + } + } + zfs_rangelock_exit(lr); + + return (error); +} + +static int +zvol_cdev_write(struct cdev *dev, struct uio *uio, int ioflag) +{ + zvol_state_t *zv; + uint64_t volsize; + zfs_locked_range_t *lr; + int error = 0; + boolean_t sync; + + zv = dev->si_drv2; + + volsize = zv->zv_volsize; + + if (uio->uio_resid > 0 && + (uio->uio_loffset < 0 || uio->uio_loffset > volsize)) + return (SET_ERROR(EIO)); + + sync = (ioflag & IO_SYNC) || + (zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS); + + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); + zvol_ensure_zilog(zv); + rw_exit(&zv->zv_suspend_lock); + + lr = zfs_rangelock_enter(&zv->zv_rangelock, uio->uio_loffset, + uio->uio_resid, RL_WRITER); + while (uio->uio_resid > 0 && uio->uio_loffset < volsize) { + uint64_t bytes = MIN(uio->uio_resid, DMU_MAX_ACCESS >> 1); + uint64_t off = uio->uio_loffset; + dmu_tx_t *tx = dmu_tx_create(zv->zv_objset); + + if (bytes > volsize - off) /* don't write past the end */ + bytes = volsize - off; + + dmu_tx_hold_write_by_dnode(tx, zv->zv_dn, off, bytes); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error) { + dmu_tx_abort(tx); + break; + } + error = dmu_write_uio_dnode(zv->zv_dn, uio, bytes, tx); + if (error == 0) + zvol_log_write(zv, tx, off, bytes, sync); + dmu_tx_commit(tx); + + if (error) + break; + } + zfs_rangelock_exit(lr); + if (sync) + zil_commit(zv->zv_zilog, ZVOL_OBJ); + return (error); +} + +static int +zvol_cdev_open(struct cdev *dev, int flags, int fmt, struct thread *td) +{ + zvol_state_t *zv; + struct zvol_state_dev *zsd; + int err = 0; + boolean_t drop_suspend = B_TRUE; + + rw_enter(&zvol_state_lock, ZVOL_RW_READER); + zv = dev->si_drv2; + if (zv == NULL) { + rw_exit(&zvol_state_lock); + return (SET_ERROR(ENXIO)); + } + + mutex_enter(&zv->zv_state_lock); + + ASSERT(zv->zv_zso->zso_volmode == ZFS_VOLMODE_DEV); + + /* + * make sure zvol is not suspended during first open + * (hold zv_suspend_lock) and respect proper lock acquisition + * ordering - zv_suspend_lock before zv_state_lock + */ + if (zv->zv_open_count == 0) { + if (!rw_tryenter(&zv->zv_suspend_lock, ZVOL_RW_READER)) { + mutex_exit(&zv->zv_state_lock); + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); + mutex_enter(&zv->zv_state_lock); + /* check to see if zv_suspend_lock is needed */ + if (zv->zv_open_count != 0) { + rw_exit(&zv->zv_suspend_lock); + drop_suspend = B_FALSE; + } + } + } else { + drop_suspend = B_FALSE; + } + rw_exit(&zvol_state_lock); + + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); + + if (zv->zv_open_count == 0) { + ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock)); + err = zvol_first_open(zv, !(flags & FWRITE)); + if (err) + goto out_locked; + } + + if ((flags & FWRITE) && (zv->zv_flags & ZVOL_RDONLY)) { + err = EROFS; + goto out_opened; + } + if (zv->zv_flags & ZVOL_EXCL) { + err = EBUSY; + goto out_opened; + } +#ifdef FEXCL + if (flags & FEXCL) { + if (zv->zv_open_count != 0) { + err = EBUSY; + goto out_opened; + } + zv->zv_flags |= ZVOL_EXCL; + } +#endif + + zv->zv_open_count++; + if (flags & (FSYNC | FDSYNC)) { + zsd = &zv->zv_zso->zso_dev; + zsd->zsd_sync_cnt++; + if (zsd->zsd_sync_cnt == 1) + zil_async_to_sync(zv->zv_zilog, ZVOL_OBJ); + } + + mutex_exit(&zv->zv_state_lock); + if (drop_suspend) + rw_exit(&zv->zv_suspend_lock); + return (0); + +out_opened: + if (zv->zv_open_count == 0) + zvol_last_close(zv); +out_locked: + mutex_exit(&zv->zv_state_lock); + if (drop_suspend) + rw_exit(&zv->zv_suspend_lock); + return (SET_ERROR(err)); +} + +static int +zvol_cdev_close(struct cdev *dev, int flags, int fmt, struct thread *td) +{ + zvol_state_t *zv; + struct zvol_state_dev *zsd; + boolean_t drop_suspend = B_TRUE; + + rw_enter(&zvol_state_lock, ZVOL_RW_READER); + zv = dev->si_drv2; + if (zv == NULL) { + rw_exit(&zvol_state_lock); + return (SET_ERROR(ENXIO)); + } + + mutex_enter(&zv->zv_state_lock); + if (zv->zv_flags & ZVOL_EXCL) { + ASSERT(zv->zv_open_count == 1); + zv->zv_flags &= ~ZVOL_EXCL; + } + + ASSERT(zv->zv_zso->zso_volmode == ZFS_VOLMODE_DEV); + + /* + * If the open count is zero, this is a spurious close. + * That indicates a bug in the kernel / DDI framework. + */ + ASSERT(zv->zv_open_count > 0); + /* + * make sure zvol is not suspended during last close + * (hold zv_suspend_lock) and respect proper lock acquisition + * ordering - zv_suspend_lock before zv_state_lock + */ + if (zv->zv_open_count == 1) { + if (!rw_tryenter(&zv->zv_suspend_lock, ZVOL_RW_READER)) { + mutex_exit(&zv->zv_state_lock); + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); + mutex_enter(&zv->zv_state_lock); + /* check to see if zv_suspend_lock is needed */ + if (zv->zv_open_count != 1) { + rw_exit(&zv->zv_suspend_lock); + drop_suspend = B_FALSE; + } + } + } else { + drop_suspend = B_FALSE; + } + rw_exit(&zvol_state_lock); + + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); + + /* + * You may get multiple opens, but only one close. + */ + zv->zv_open_count--; + if (flags & (FSYNC | FDSYNC)) { + zsd = &zv->zv_zso->zso_dev; + zsd->zsd_sync_cnt--; + } + + if (zv->zv_open_count == 0) { + ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock)); + zvol_last_close(zv); + } + + mutex_exit(&zv->zv_state_lock); + + if (drop_suspend) + rw_exit(&zv->zv_suspend_lock); + return (0); +} + +static int +zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data, + int fflag, struct thread *td) +{ + zvol_state_t *zv; + zfs_locked_range_t *lr; + off_t offset, length; + int i, error; + boolean_t sync; + + zv = dev->si_drv2; + + error = 0; + KASSERT(zv->zv_open_count > 0, + ("Device with zero access count in %s", __func__)); + + i = IOCPARM_LEN(cmd); + switch (cmd) { + case DIOCGSECTORSIZE: + *(uint32_t *)data = DEV_BSIZE; + break; + case DIOCGMEDIASIZE: + *(off_t *)data = zv->zv_volsize; + break; + case DIOCGFLUSH: + if (zv->zv_zilog != NULL) + zil_commit(zv->zv_zilog, ZVOL_OBJ); + break; + case DIOCGDELETE: + if (!zvol_unmap_enabled) + break; + + offset = ((off_t *)data)[0]; + length = ((off_t *)data)[1]; + if ((offset % DEV_BSIZE) != 0 || (length % DEV_BSIZE) != 0 || + offset < 0 || offset >= zv->zv_volsize || + length <= 0) { + printf("%s: offset=%jd length=%jd\n", __func__, offset, + length); + error = EINVAL; + break; + } + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); + zvol_ensure_zilog(zv); + lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, length, + RL_WRITER); + dmu_tx_t *tx = dmu_tx_create(zv->zv_objset); + error = dmu_tx_assign(tx, TXG_WAIT); + if (error != 0) { + sync = FALSE; + dmu_tx_abort(tx); + } else { + sync = (zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS); + zvol_log_truncate(zv, tx, offset, length, sync); + dmu_tx_commit(tx); + error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, + offset, length); + } + zfs_rangelock_exit(lr); + if (sync) + zil_commit(zv->zv_zilog, ZVOL_OBJ); + rw_exit(&zv->zv_suspend_lock); + break; + case DIOCGSTRIPESIZE: + *(off_t *)data = zv->zv_volblocksize; + break; + case DIOCGSTRIPEOFFSET: + *(off_t *)data = 0; + break; + case DIOCGATTR: { + spa_t *spa = dmu_objset_spa(zv->zv_objset); + struct diocgattr_arg *arg = (struct diocgattr_arg *)data; + uint64_t refd, avail, usedobjs, availobjs; + + if (strcmp(arg->name, "GEOM::candelete") == 0) + arg->value.i = 1; + else if (strcmp(arg->name, "blocksavail") == 0) { + dmu_objset_space(zv->zv_objset, &refd, &avail, + &usedobjs, &availobjs); + arg->value.off = avail / DEV_BSIZE; + } else if (strcmp(arg->name, "blocksused") == 0) { + dmu_objset_space(zv->zv_objset, &refd, &avail, + &usedobjs, &availobjs); + arg->value.off = refd / DEV_BSIZE; + } else if (strcmp(arg->name, "poolblocksavail") == 0) { + avail = metaslab_class_get_space(spa_normal_class(spa)); + avail -= metaslab_class_get_alloc( + spa_normal_class(spa)); + arg->value.off = avail / DEV_BSIZE; + } else if (strcmp(arg->name, "poolblocksused") == 0) { + refd = metaslab_class_get_alloc(spa_normal_class(spa)); + arg->value.off = refd / DEV_BSIZE; + } else + error = ENOIOCTL; + break; + } + case FIOSEEKHOLE: + case FIOSEEKDATA: { + off_t *off = (off_t *)data; + uint64_t noff; + boolean_t hole; + + hole = (cmd == FIOSEEKHOLE); + noff = *off; + error = dmu_offset_next(zv->zv_objset, ZVOL_OBJ, hole, &noff); + *off = noff; + break; + } + default: + error = ENOIOCTL; + } + + return (error); +} + +/* + * Misc. helpers + */ + +static void +zvol_ensure_zilog(zvol_state_t *zv) +{ + ASSERT(ZVOL_RW_READ_HELD(&zv->zv_suspend_lock)); + + /* + * Open a ZIL if this is the first time we have written to this + * zvol. We protect zv->zv_zilog with zv_suspend_lock rather + * than zv_state_lock so that we don't need to acquire an + * additional lock in this path. + */ + if (zv->zv_zilog == NULL) { + rw_exit(&zv->zv_suspend_lock); + rw_enter(&zv->zv_suspend_lock, RW_WRITER); + if (zv->zv_zilog == NULL) { + zv->zv_zilog = zil_open(zv->zv_objset, + zvol_get_data); + zv->zv_flags |= ZVOL_WRITTEN_TO; + } + rw_downgrade(&zv->zv_suspend_lock); + } +} + +static boolean_t +zvol_is_zvol_impl(const char *device) +{ + return (device && strncmp(device, ZVOL_DIR, strlen(ZVOL_DIR)) == 0); +} + +static void +zvol_rename_minor(zvol_state_t *zv, const char *newname) +{ + ASSERT(RW_LOCK_HELD(&zvol_state_lock)); + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); + + /* move to new hashtable entry */ + zv->zv_hash = zvol_name_hash(zv->zv_name); + hlist_del(&zv->zv_hlink); + hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash)); + + if (zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM) { + struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; + struct g_provider *pp = zsg->zsg_provider; + struct g_geom *gp; + + g_topology_lock(); + gp = pp->geom; + ASSERT(gp != NULL); + + zsg->zsg_provider = NULL; + g_wither_provider(pp, ENXIO); + + pp = g_new_providerf(gp, "%s/%s", ZVOL_DRIVER, newname); + pp->flags |= G_PF_DIRECT_RECEIVE | G_PF_DIRECT_SEND; + pp->sectorsize = DEV_BSIZE; + pp->mediasize = zv->zv_volsize; + pp->private = zv; + zsg->zsg_provider = pp; + g_error_provider(pp, 0); + g_topology_unlock(); + } else if (zv->zv_zso->zso_volmode == ZFS_VOLMODE_DEV) { + struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev; + struct cdev *dev; + struct make_dev_args args; + + dev = zsd->zsd_cdev; + if (dev != NULL) { + destroy_dev(dev); + dev = zsd->zsd_cdev = NULL; + if (zv->zv_open_count > 0) { + zv->zv_flags &= ~ZVOL_EXCL; + zv->zv_open_count = 0; + /* XXX need suspend lock but lock order */ + zvol_last_close(zv); + } + } + + make_dev_args_init(&args); + args.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK; + args.mda_devsw = &zvol_cdevsw; + args.mda_cr = NULL; + args.mda_uid = UID_ROOT; + args.mda_gid = GID_OPERATOR; + args.mda_mode = 0640; + args.mda_si_drv2 = zv; + if (make_dev_s(&args, &dev, "%s/%s", ZVOL_DRIVER, newname) + == 0) { + dev->si_iosize_max = MAXPHYS; + zsd->zsd_cdev = dev; + } + } + strlcpy(zv->zv_name, newname, sizeof (zv->zv_name)); +} + +/* + * Remove minor node for the specified volume. + */ +static void +zvol_free(zvol_state_t *zv) +{ + ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock)); + ASSERT(!MUTEX_HELD(&zv->zv_state_lock)); + ASSERT(zv->zv_open_count == 0); + + ZFS_LOG(1, "ZVOL %s destroyed.", zv->zv_name); + + rw_destroy(&zv->zv_suspend_lock); + zfs_rangelock_fini(&zv->zv_rangelock); + + if (zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM) { + struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; + + g_topology_lock(); + zvol_geom_destroy(zv); + g_topology_unlock(); + mtx_destroy(&zsg->zsg_queue_mtx); + } else if (zv->zv_zso->zso_volmode == ZFS_VOLMODE_DEV) { + struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev; + struct cdev *dev = zsd->zsd_cdev; + + if (dev != NULL) + destroy_dev(dev); + } + + mutex_destroy(&zv->zv_state_lock); + kmem_free(zv->zv_zso, sizeof (struct zvol_state_os)); + kmem_free(zv, sizeof (zvol_state_t)); + zvol_minors--; +} + +/* + * Create a minor node (plus a whole lot more) for the specified volume. + */ +static int +zvol_create_minor_impl(const char *name) +{ + zvol_state_t *zv; + objset_t *os; + dmu_object_info_t *doi; + uint64_t volsize; + uint64_t volmode, hash; + int error; + + ZFS_LOG(1, "Creating ZVOL %s...", name); + + hash = zvol_name_hash(name); + if ((zv = zvol_find_by_name_hash(name, hash, RW_NONE)) != NULL) { + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); + mutex_exit(&zv->zv_state_lock); + return (SET_ERROR(EEXIST)); + } + + DROP_GIANT(); + /* lie and say we're read-only */ + error = dmu_objset_own(name, DMU_OST_ZVOL, B_TRUE, B_TRUE, FTAG, &os); + doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP); + + if (error) + goto out_doi; + + error = dmu_object_info(os, ZVOL_OBJ, doi); + if (error) + goto out_dmu_objset_disown; + + error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize); + if (error) + goto out_dmu_objset_disown; + + error = dsl_prop_get_integer(name, + zfs_prop_to_name(ZFS_PROP_VOLMODE), &volmode, NULL); + if (error != 0 || volmode == ZFS_VOLMODE_DEFAULT) + volmode = zvol_volmode; + /* + * zvol_alloc equivalent ... + */ + zv = kmem_zalloc(sizeof (*zv), KM_SLEEP); + zv->zv_hash = hash; + mutex_init(&zv->zv_state_lock, NULL, MUTEX_DEFAULT, NULL); + zv->zv_zso = kmem_zalloc(sizeof (struct zvol_state_os), KM_SLEEP); + zv->zv_zso->zso_volmode = volmode; + if (zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM) { + struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; + struct g_provider *pp; + struct g_geom *gp; + + zsg->zsg_state = ZVOL_GEOM_UNINIT; + mtx_init(&zsg->zsg_queue_mtx, "zvol", NULL, MTX_DEF); + + g_topology_lock(); + gp = g_new_geomf(&zfs_zvol_class, "zfs::zvol::%s", name); + gp->start = zvol_geom_bio_start; + gp->access = zvol_geom_access; + pp = g_new_providerf(gp, "%s/%s", ZVOL_DRIVER, name); + /* TODO: NULL check? */ + pp->flags |= G_PF_DIRECT_RECEIVE | G_PF_DIRECT_SEND; + pp->sectorsize = DEV_BSIZE; + pp->mediasize = 0; + pp->private = zv; + + zsg->zsg_provider = pp; + bioq_init(&zsg->zsg_queue); + } else if (zv->zv_zso->zso_volmode == ZFS_VOLMODE_DEV) { + struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev; + struct cdev *dev; + struct make_dev_args args; + + make_dev_args_init(&args); + args.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK; + args.mda_devsw = &zvol_cdevsw; + args.mda_cr = NULL; + args.mda_uid = UID_ROOT; + args.mda_gid = GID_OPERATOR; + args.mda_mode = 0640; + args.mda_si_drv2 = zv; + error = make_dev_s(&args, &dev, "%s/%s", ZVOL_DRIVER, name); + if (error != 0) { + mutex_destroy(&zv->zv_state_lock); + kmem_free(zv->zv_zso, sizeof (struct zvol_state_os)); + kmem_free(zv, sizeof (*zv)); + dmu_objset_disown(os, B_TRUE, FTAG); + goto out_giant; + } + dev->si_iosize_max = MAXPHYS; + zsd->zsd_cdev = dev; + } + (void) strlcpy(zv->zv_name, name, MAXPATHLEN); + rw_init(&zv->zv_suspend_lock, NULL, RW_DEFAULT, NULL); + zfs_rangelock_init(&zv->zv_rangelock, NULL, NULL); + + if (dmu_objset_is_snapshot(os) || !spa_writeable(dmu_objset_spa(os))) + zv->zv_flags |= ZVOL_RDONLY; + + zv->zv_volblocksize = doi->doi_data_block_size; + zv->zv_volsize = volsize; + zv->zv_objset = os; + + if (spa_writeable(dmu_objset_spa(os))) { + if (zil_replay_disable) + zil_destroy(dmu_objset_zil(os), B_FALSE); + else + zil_replay(os, zv, zvol_replay_vector); + } + + /* XXX do prefetch */ + + zv->zv_objset = NULL; +out_dmu_objset_disown: + dmu_objset_disown(os, B_TRUE, FTAG); + + if (zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM) { + if (error == 0) + zvol_geom_run(zv); + g_topology_unlock(); + } +out_doi: + kmem_free(doi, sizeof (dmu_object_info_t)); + if (error == 0) { + rw_enter(&zvol_state_lock, RW_WRITER); + zvol_insert(zv); + zvol_minors++; + rw_exit(&zvol_state_lock); + } + ZFS_LOG(1, "ZVOL %s created.", name); +out_giant: + PICKUP_GIANT(); + return (error); +} + +static void +zvol_clear_private(zvol_state_t *zv) +{ + ASSERT(RW_LOCK_HELD(&zvol_state_lock)); + if (zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM) { + struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; + struct g_provider *pp = zsg->zsg_provider; + + if (pp == NULL) /* XXX when? */ + return; + + mtx_lock(&zsg->zsg_queue_mtx); + zsg->zsg_state = ZVOL_GEOM_STOPPED; + pp->private = NULL; + wakeup_one(&zsg->zsg_queue); + while (zsg->zsg_state != ZVOL_GEOM_RUNNING) + msleep(&zsg->zsg_state, + &zsg->zsg_queue_mtx, + 0, "zvol:w", 0); + mtx_unlock(&zsg->zsg_queue_mtx); + ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock)); + } +} + +static int +zvol_update_volsize(zvol_state_t *zv, uint64_t volsize) +{ + zv->zv_volsize = volsize; + if (zv->zv_zso->zso_volmode == ZFS_VOLMODE_GEOM) { + struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; + struct g_provider *pp = zsg->zsg_provider; + + if (pp == NULL) /* XXX when? */ + return (0); + + g_topology_lock(); + + /* + * Do not invoke resize event when initial size was zero. + * ZVOL initializes the size on first open, this is not + * real resizing. + */ + if (pp->mediasize == 0) + pp->mediasize = zv->zv_volsize; + else + g_resize_provider(pp, zv->zv_volsize); + + g_topology_unlock(); + } + return (0); +} + +static void +zvol_set_disk_ro_impl(zvol_state_t *zv, int flags) +{ + // XXX? set_disk_ro(zv->zv_zso->zvo_disk, flags); +} + +static void +zvol_set_capacity_impl(zvol_state_t *zv, uint64_t capacity) +{ + // XXX? set_capacity(zv->zv_zso->zvo_disk, capacity); +} + +const static zvol_platform_ops_t zvol_freebsd_ops = { + .zv_free = zvol_free, + .zv_rename_minor = zvol_rename_minor, + .zv_create_minor = zvol_create_minor_impl, + .zv_update_volsize = zvol_update_volsize, + .zv_clear_private = zvol_clear_private, + .zv_is_zvol = zvol_is_zvol_impl, + .zv_set_disk_ro = zvol_set_disk_ro_impl, + .zv_set_capacity = zvol_set_capacity_impl, +}; + +/* + * Public interfaces + */ + +int +zvol_busy(void) +{ + return (zvol_minors != 0); +} + +int +zvol_init(void) +{ + zvol_init_impl(); + zvol_register_ops(&zvol_freebsd_ops); + return (0); +} + +void +zvol_fini(void) +{ + zvol_fini_impl(); +} diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh index e1e114128e3..1f142455a9c 100755 --- a/scripts/zfs-tests.sh +++ b/scripts/zfs-tests.sh @@ -221,7 +221,7 @@ create_links() { [ ! -e "$STF_PATH/$i" ] || continue if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then - ln -s "$j/$i" "$STF_PATH/$i" || \ + ln -sf "$j/$i" "$STF_PATH/$i" || \ fail "Couldn't link $i" break fi diff --git a/tests/runfiles/Makefile.am b/tests/runfiles/Makefile.am index cc630a5e9e6..c7cf2a20c1d 100644 --- a/tests/runfiles/Makefile.am +++ b/tests/runfiles/Makefile.am @@ -1,6 +1,7 @@ pkgdatadir = $(datadir)/@PACKAGE@/runfiles dist_pkgdata_DATA = \ common.run \ + freebsd.run \ linux.run \ longevity.run \ perf-regression.run \ diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index ccf03af89dc..b75be778bdc 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -654,6 +654,12 @@ tests = ['online_offline_001_pos', 'online_offline_002_neg', 'online_offline_003_neg'] tags = ['functional', 'online_offline'] +[tests/functional/persist_l2arc] +tests = ['persist_l2arc_001_pos', 'persist_l2arc_002_pos', + 'persist_l2arc_003_neg', 'persist_l2arc_004_pos', 'persist_l2arc_005_pos', + 'persist_l2arc_006_pos', 'persist_l2arc_007_pos', 'persist_l2arc_008_pos'] +tags = ['functional', 'persist_l2arc'] + [tests/functional/pool_checkpoint] tests = ['checkpoint_after_rewind', 'checkpoint_big_rewind', 'checkpoint_capacity', 'checkpoint_conf_change', 'checkpoint_discard', diff --git a/tests/runfiles/freebsd.run b/tests/runfiles/freebsd.run new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 61df6d4208c..897a6a95582 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -164,9 +164,3 @@ tags = ['functional', 'user_namespace'] tests = ['groupspace_001_pos', 'groupspace_002_pos', 'groupspace_003_pos', 'userquota_013_pos', 'userspace_003_pos'] tags = ['functional', 'userquota'] - -[tests/functional/persist_l2arc:Linux] -tests = ['persist_l2arc_001_pos', 'persist_l2arc_002_pos', - 'persist_l2arc_003_neg', 'persist_l2arc_004_pos', 'persist_l2arc_005_pos', - 'persist_l2arc_006_pos', 'persist_l2arc_007_pos', 'persist_l2arc_008_pos'] -tags = ['functional', 'persist_l2arc'] diff --git a/tests/test-runner/bin/zts-report.py b/tests/test-runner/bin/zts-report.py index 7fc84fcecf9..d74aa9d7aef 100755 --- a/tests/test-runner/bin/zts-report.py +++ b/tests/test-runner/bin/zts-report.py @@ -244,9 +244,10 @@ if sys.platform.startswith('freebsd'): maybe.update({ 'cli_root/zfs_copies/zfs_copies_002_pos': ['FAIL', known_reason], - 'cli_root/zpool_import/zpool_import_missing_003_pos': - ['FAIL', known_reason], + 'cli_root/zfs_inherit/zfs_inherit_001_neg': ['FAIL', known_reason], 'delegate/zfs_allow_003_pos': ['FAIL', known_reason], + 'removal/removal_condense_export': ['FAIL', known_reason], + 'removal/removal_with_export': ['FAIL', known_reason], 'resilver/resilver_restart_001': ['FAIL', known_reason], }) diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index a641a0b7a7e..f917c58129a 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -4056,13 +4056,19 @@ function ls_xattr # path function get_arcstat # stat { - if is_linux; then - typeset stat=$1 + typeset stat=$1 + + case $(uname) in + FreeBSD) + sysctl -n kstat.zfs.misc.arcstats.$stat + ;; + Linux) typeset zfs_arcstats="/proc/spl/kstat/zfs/arcstats" [[ -f "$zfs_arcstats" ]] || return 1 grep $stat $zfs_arcstats | awk '{print $3}' - return $? - else - return 1 - fi + ;; + *) + false + ;; + esac } diff --git a/tests/zfs-tests/include/tunables.cfg b/tests/zfs-tests/include/tunables.cfg index 680fcf42cb2..d7bd44b6035 100644 --- a/tests/zfs-tests/include/tunables.cfg +++ b/tests/zfs-tests/include/tunables.cfg @@ -36,8 +36,8 @@ INITIALIZE_CHUNK_SIZE initialize_chunk_size zfs_initialize_chunk_size INITIALIZE_VALUE initialize_value zfs_initialize_value KEEP_LOG_SPACEMAPS_AT_EXPORT keep_log_spacemaps_at_export zfs_keep_log_spacemaps_at_export L2ARC_NOPREFETCH l2arc.noprefetch l2arc_noprefetch -L2ARC_REBUILD_BLOCKS_MIN_L2SIZE UNSUPPORTED l2arc_rebuild_blocks_min_l2size -L2ARC_REBUILD_ENABLED UNSUPPORTED l2arc_rebuild_enabled +L2ARC_REBUILD_BLOCKS_MIN_L2SIZE l2arc.rebuild_blocks_min_l2size l2arc_rebuild_blocks_min_l2size +L2ARC_REBUILD_ENABLED l2arc.rebuild_enabled l2arc_rebuild_enabled L2ARC_WRITE_BOOST l2arc.write_boost l2arc_write_boost L2ARC_WRITE_MAX l2arc.write_max l2arc_write_max LIVELIST_CONDENSE_NEW_ALLOC livelist.condense.new_alloc zfs_livelist_condense_new_alloc diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_import/zpool_import_missing_003_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_import/zpool_import_missing_003_pos.ksh index 815d409aa1a..6fa55250a77 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_import/zpool_import_missing_003_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_import/zpool_import_missing_003_pos.ksh @@ -59,8 +59,8 @@ verify_runnable "global" -# See issue: https://github.com/zfsonlinux/zfs/issues/6839 -if is_linux; then +# See issue: https://github.com/openzfs/zfs/issues/6839 +if ! is_illumos; then log_unsupported "Test case may be slow" fi From 6b1139e82c0a96c267752b437a3b14f6787535ec Mon Sep 17 00:00:00 2001 From: alaviss Date: Tue, 14 Apr 2020 18:47:40 +0000 Subject: [PATCH 016/133] sys/mnttab.h: include sys/stat.h for stat64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Musl libc defined `stat64` as a macro, which causes the build to fail upon compiling os/linux/getmntany.c due to conflicts between the forward declaration and the implementation. This commit fixes that by including in "sys/mnttab.h" directly. Reviewed-by: Brian Behlendorf Signed-off-by: Hiếu Lê Closes #10195 --- lib/libspl/include/os/linux/sys/mnttab.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/libspl/include/os/linux/sys/mnttab.h b/lib/libspl/include/os/linux/sys/mnttab.h index 7b30bf1d547..093218d6853 100644 --- a/lib/libspl/include/os/linux/sys/mnttab.h +++ b/lib/libspl/include/os/linux/sys/mnttab.h @@ -32,6 +32,7 @@ #include #include +#include #include #ifdef MNTTAB @@ -67,7 +68,6 @@ struct extmnttab { uint_t mnt_minor; }; -struct stat64; struct statfs; extern int getmntany(FILE *fp, struct mnttab *mp, struct mnttab *mpref); From 813c8564ee756ad0a51cd5a67750b97292611efe Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Tue, 14 Apr 2020 16:18:23 -0400 Subject: [PATCH 017/133] Fix SC2086 note in zpool.d/smart ./cmd/zpool/zpool.d/smart:78:32: note: Double quote to prevent globbing and word splitting. [SC2086] Reported by latest shellcheck on FreeBSD. Reviewed-by: Brian Behlendorf Signed-off-by: Ryan Moeller Closes #10194 --- cmd/zpool/zpool.d/smart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zpool/zpool.d/smart b/cmd/zpool/zpool.d/smart index bd18e9d0443..874528d66ec 100755 --- a/cmd/zpool/zpool.d/smart +++ b/cmd/zpool/zpool.d/smart @@ -75,7 +75,7 @@ if [ -b "$VDEV_UPATH" ] && [ -x "$smartctl_path" ] || [ -n "$samples" ] ; then if [ -n "$samples" ] ; then # cat a smartctl output text file instead of running smartctl # on a vdev (only used for developer testing). - file=$(get_filename_from_dir $samples) + file=$(get_filename_from_dir "$samples") echo "file=$file" raw_out=$(cat "$samples/$file") else From af99094dee7b47d32678b271ca90563e9b89bb78 Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Wed, 15 Apr 2020 12:21:40 -0400 Subject: [PATCH 018/133] Don't delete freebsd.run in distclean Add a comment so the file is not empty. The comment can be removed when FreeBSD-specific tests are added. Reviewed-by: George Melikov Reviewed-by: Sean Eric Fagan Reviewed-by: Brian Behlendorf Signed-off-by: Ryan Moeller Closes #10206 --- tests/runfiles/freebsd.run | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/runfiles/freebsd.run b/tests/runfiles/freebsd.run index e69de29bb2d..447463699c3 100644 --- a/tests/runfiles/freebsd.run +++ b/tests/runfiles/freebsd.run @@ -0,0 +1 @@ +# don't delete this file From a7929f31373497590f5884efbc7cde29104e94d5 Mon Sep 17 00:00:00 2001 From: Ryan Moeller Date: Wed, 15 Apr 2020 14:14:47 -0400 Subject: [PATCH 019/133] Update FreeBSD tunables Remove some obsolete legacy compat, rename some misnamed, and add some missing tunables for FreeBSD. Reviewed-by: Brian Behlendorf Signed-off-by: Ryan Moeller Closes #10203 --- include/os/linux/kernel/linux/mod_compat.h | 5 +- module/os/freebsd/zfs/arc_os.c | 47 +++++++++++++++--- module/os/freebsd/zfs/sysctl_os.c | 56 ++++++++++------------ module/zfs/ddt.c | 2 +- module/zfs/txg.c | 2 +- tests/zfs-tests/include/tunables.cfg | 2 +- 6 files changed, 71 insertions(+), 43 deletions(-) diff --git a/include/os/linux/kernel/linux/mod_compat.h b/include/os/linux/kernel/linux/mod_compat.h index 1a53c66e522..4b83fe41333 100644 --- a/include/os/linux/kernel/linux/mod_compat.h +++ b/include/os/linux/kernel/linux/mod_compat.h @@ -54,6 +54,7 @@ enum scope_prefix_types { zfs_dbuf, zfs_dbuf_cache, zfs_deadman, + zfs_dedup, zfs_l2arc, zfs_livelist, zfs_livelist_condense, @@ -67,13 +68,13 @@ enum scope_prefix_types { zfs_send, zfs_spa, zfs_trim, + zfs_txg, zfs_vdev, zfs_vdev_cache, zfs_vdev_mirror, zfs_zevent, zfs_zio, - zfs_zil, - spa + zfs_zil }; /* diff --git a/module/os/freebsd/zfs/arc_os.c b/module/os/freebsd/zfs/arc_os.c index f0c2724471f..d7b842f84e5 100644 --- a/module/os/freebsd/zfs/arc_os.c +++ b/module/os/freebsd/zfs/arc_os.c @@ -60,6 +60,46 @@ uint_t zfs_arc_free_target = 0; int64_t last_free_memory; free_memory_reason_t last_free_reason; +static void +arc_free_target_init(void *unused __unused) +{ + zfs_arc_free_target = vm_cnt.v_free_target; +} +SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, + arc_free_target_init, NULL); + +/* + * We don't have a tunable for arc_free_target due to the dependency on + * pagedaemon initialisation. + */ +static int +sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS) +{ + uint_t val; + int err; + + val = zfs_arc_free_target; + err = sysctl_handle_int(oidp, &val, 0, req); + if (err != 0 || req->newptr == NULL) + return (err); + + if (val < minfree) + return (EINVAL); + if (val > vm_cnt.v_page_count) + return (EINVAL); + + zfs_arc_free_target = val; + + return (0); +} +SYSCTL_DECL(_vfs_zfs); +/* BEGIN CSTYLED */ +SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_free_target, + CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof (uint_t), + sysctl_vfs_zfs_arc_free_target, "IU", + "Desired number of free pages below which ARC triggers reclaim"); +/* END CSTYLED */ + int64_t arc_available_memory(void) { @@ -67,7 +107,6 @@ arc_available_memory(void) int64_t n __unused; free_memory_reason_t r = FMR_UNKNOWN; -#ifdef _KERNEL /* * Cooperate with pagedaemon when it's time for it to scan * and reclaim some pages. @@ -115,12 +154,6 @@ arc_available_memory(void) } } -#else /* _KERNEL */ - /* Every 100 calls, free a small amount */ - if (spa_get_random(100) == 0) - lowest = -1024; -#endif /* _KERNEL */ - last_free_memory = lowest; last_free_reason = r; DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r); diff --git a/module/os/freebsd/zfs/sysctl_os.c b/module/os/freebsd/zfs/sysctl_os.c index ea9c1b3f1f1..544a8895696 100644 --- a/module/os/freebsd/zfs/sysctl_os.c +++ b/module/os/freebsd/zfs/sysctl_os.c @@ -94,32 +94,35 @@ __FBSDID("$FreeBSD$"); /* BEGIN CSTYLED */ SYSCTL_DECL(_vfs_zfs); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, zevent, CTLFLAG_RW, 0, "ZFS events"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, zil, CTLFLAG_RW, 0, "ZFS ZIL"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, vdev, CTLFLAG_RW, 0, "ZFS VDEV"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, trim, CTLFLAG_RW, 0, "ZFS TRIM"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, spa, CTLFLAG_RW, 0, "space allocation"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, reconstruct, CTLFLAG_RW, 0, "reconstruct"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, prefetch, CTLFLAG_RW, 0, "ZFS ZFETCH"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, multihost, CTLFLAG_RW, 0, "multihost protection"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, mg, CTLFLAG_RW, 0, "metaslab group"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, metaslab, CTLFLAG_RW, 0, "ZFS metaslab"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, lua, CTLFLAG_RW, 0, "lua"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, l2arc, CTLFLAG_RW, 0, "l2arc"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, arc, CTLFLAG_RW, 0, "ZFS adaptive replacement cache"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, condense, CTLFLAG_RW, 0, "ZFS condense"); SYSCTL_NODE(_vfs_zfs, OID_AUTO, dbuf, CTLFLAG_RW, 0, "ZFS disk buf cache"); SYSCTL_NODE(_vfs_zfs, OID_AUTO, dbuf_cache, CTLFLAG_RW, 0, "ZFS disk buf cache"); SYSCTL_NODE(_vfs_zfs, OID_AUTO, deadman, CTLFLAG_RW, 0, "ZFS deadman"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, condense, CTLFLAG_RW, 0, "ZFS condense"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, arc, CTLFLAG_RW, 0, "ZFS Adaptive Replacement Cache"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, dedup, CTLFLAG_RW, 0, "ZFS dedup"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, l2arc, CTLFLAG_RW, 0, "ZFS l2arc"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, livelist, CTLFLAG_RW, 0, "ZFS livelist"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, lua, CTLFLAG_RW, 0, "ZFS lua"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, metaslab, CTLFLAG_RW, 0, "ZFS metaslab"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, mg, CTLFLAG_RW, 0, "ZFS metaslab group"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, multihost, CTLFLAG_RW, 0, "ZFS multihost protection"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, prefetch, CTLFLAG_RW, 0, "ZFS prefetch"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, reconstruct, CTLFLAG_RW, 0, "ZFS reconstruct"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, recv, CTLFLAG_RW, 0, "ZFS receive"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, send, CTLFLAG_RW, 0, "ZFS send"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, spa, CTLFLAG_RW, 0, "ZFS space allocation"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, trim, CTLFLAG_RW, 0, "ZFS TRIM"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, txg, CTLFLAG_RW, 0, "ZFS transaction group"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, vdev, CTLFLAG_RW, 0, "ZFS VDEV"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, zevent, CTLFLAG_RW, 0, "ZFS event"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, zil, CTLFLAG_RW, 0, "ZFS ZIL"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); -SYSCTL_NODE(_vfs_zfs_vdev, OID_AUTO, mirror, CTLFLAG_RD, 0, - "ZFS VDEV Mirror"); +SYSCTL_NODE(_vfs_zfs_livelist, OID_AUTO, condense, CTLFLAG_RW, 0, + "ZFS livelist condense"); SYSCTL_NODE(_vfs_zfs_vdev, OID_AUTO, cache, CTLFLAG_RW, 0, "ZFS VDEV Cache"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, livelist, CTLFLAG_RW, 0, "livelist state"); -SYSCTL_NODE(_vfs_zfs_livelist, OID_AUTO, condense, CTLFLAG_RW, 0, "condense knobs"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, recv, CTLFLAG_RW, 0, "receive knobs"); -SYSCTL_NODE(_vfs_zfs, OID_AUTO, send, CTLFLAG_RW, 0, "send knobs"); +SYSCTL_NODE(_vfs_zfs_vdev, OID_AUTO, mirror, CTLFLAG_RD, 0, + "ZFS VDEV mirror"); SYSCTL_DECL(_vfs_zfs_version); SYSCTL_CONST_STRING(_vfs_zfs_version, OID_AUTO, module, CTLFLAG_RD, @@ -267,7 +270,7 @@ SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_no_grow_shift, CTLTYPE_U32 | CTLFLAG_RWTUN, /* dmu.c */ /* dmu_zfetch.c */ -SYSCTL_NODE(_vfs_zfs, OID_AUTO, zfetch, CTLFLAG_RW, 0, "ZFS ZFETCH"); +SYSCTL_NODE(_vfs_zfs, OID_AUTO, zfetch, CTLFLAG_RW, 0, "ZFS ZFETCH (LEGACY)"); /* max bytes to prefetch per stream (default 8MB) */ extern uint32_t zfetch_max_distance; @@ -295,15 +298,6 @@ SYSCTL_INT(_vfs_zfs, OID_AUTO, default_ibs, CTLFLAG_RWTUN, /* metaslab.c */ -/* - * Enable/disable lba weighting (i.e. outer tracks are given preference). - */ -extern boolean_t metaslab_lba_weighting_enabled; -SYSCTL_INT(_vfs_zfs_metaslab, OID_AUTO, lba_weighting, CTLFLAG_RWTUN, - &metaslab_lba_weighting_enabled, 0, - "Enable LBA weighting (i.e. outer tracks are given preference)"); - - /* * In pools where the log space map feature is not enabled we touch * multiple metaslabs (and their respective space maps) with each diff --git a/module/zfs/ddt.c b/module/zfs/ddt.c index 5e7b31c7639..5dcb36ce6f8 100644 --- a/module/zfs/ddt.c +++ b/module/zfs/ddt.c @@ -1188,6 +1188,6 @@ ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde) } /* BEGIN CSTYLED */ -ZFS_MODULE_PARAM(zfs, zfs_, dedup_prefetch, INT, ZMOD_RW, +ZFS_MODULE_PARAM(zfs_dedup, zfs_dedup_, prefetch, INT, ZMOD_RW, "Enable prefetching dedup-ed blks"); /* END CSTYLED */ diff --git a/module/zfs/txg.c b/module/zfs/txg.c index 8d1c2a5c8ba..2284234feb9 100644 --- a/module/zfs/txg.c +++ b/module/zfs/txg.c @@ -1054,6 +1054,6 @@ EXPORT_SYMBOL(txg_stalled); EXPORT_SYMBOL(txg_sync_waiting); /* BEGIN CSTYLED */ -ZFS_MODULE_PARAM(zfs, zfs_, txg_timeout, INT, ZMOD_RW, +ZFS_MODULE_PARAM(zfs_txg, zfs_txg_, timeout, INT, ZMOD_RW, "Max seconds worth of delta per txg"); /* END CSTYLED */ diff --git a/tests/zfs-tests/include/tunables.cfg b/tests/zfs-tests/include/tunables.cfg index d7bd44b6035..f85b156586b 100644 --- a/tests/zfs-tests/include/tunables.cfg +++ b/tests/zfs-tests/include/tunables.cfg @@ -71,7 +71,7 @@ TRIM_EXTENT_BYTES_MIN trim.extent_bytes_min zfs_trim_extent_bytes_min TRIM_METASLAB_SKIP trim.metaslab_skip zfs_trim_metaslab_skip TRIM_TXG_BATCH trim.txg_batch zfs_trim_txg_batch TXG_HISTORY UNSUPPORTED zfs_txg_history -TXG_TIMEOUT txg_timeout zfs_txg_timeout +TXG_TIMEOUT txg.timeout zfs_txg_timeout UNLINK_SUSPEND_PROGRESS UNSUPPORTED zfs_unlink_suspend_progress VDEV_MIN_MS_COUNT vdev.min_ms_count zfs_vdev_min_ms_count VDEV_VALIDATE_SKIP vdev.validate_skip vdev_validate_skip From 9249f1272ec08316f4482cba145b42ba935d3b02 Mon Sep 17 00:00:00 2001 From: George Amanakis Date: Fri, 17 Apr 2020 12:27:40 -0400 Subject: [PATCH 020/133] Persistent L2ARC minor fixes Minor fixes on persistent L2ARC improving code readability and fixing a typo in zdb.c when byte-swapping a log block. It also improves the pesist_l2arc_007_pos.ksh test by giving it more time to retrieve log blocks on the cache device. Reviewed-by: Brian Behlendorf Reviewed-by: Adam D. Moss Signed-off-by: George Amanakis Closes #10210 --- cmd/zdb/zdb.c | 3 +-- module/zfs/arc.c | 14 ++++++-------- .../persist_l2arc/persist_l2arc_007_pos.ksh | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index dab0d8b68f8..f5492666f10 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -3606,8 +3606,7 @@ dump_l2arc_log_blocks(int fd, l2arc_dev_hdr_phys_t l2dhdr) } if (this_lb.lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC)) - byteswap_uint64_array(&this_lb, psize); - + byteswap_uint64_array(&this_lb, sizeof (this_lb)); if (this_lb.lb_magic != L2ARC_LOG_BLK_MAGIC) { (void) printf("Invalid log block magic\n\n"); break; diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 74bfbfc7029..5b34d4abd4a 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -9578,6 +9578,7 @@ l2arc_log_blk_read(l2arc_dev_t *dev, int err = 0; zio_cksum_t cksum; abd_t *abd = NULL; + uint64_t psize; ASSERT(this_lbp != NULL && next_lbp != NULL); ASSERT(this_lb != NULL && next_lb != NULL); @@ -9616,8 +9617,8 @@ l2arc_log_blk_read(l2arc_dev_t *dev, } /* Make sure the buffer checks out */ - fletcher_4_native(this_lb, - L2BLK_GET_PSIZE((this_lbp)->lbp_prop), NULL, &cksum); + psize = L2BLK_GET_PSIZE((this_lbp)->lbp_prop); + fletcher_4_native(this_lb, psize, NULL, &cksum); if (!ZIO_CHECKSUM_EQUAL(cksum, this_lbp->lbp_cksum)) { ARCSTAT_BUMP(arcstat_l2_rebuild_abort_cksum_lb_errors); zfs_dbgmsg("L2ARC log block cksum failed, offset: %llu, " @@ -9633,14 +9634,11 @@ l2arc_log_blk_read(l2arc_dev_t *dev, case ZIO_COMPRESS_OFF: break; case ZIO_COMPRESS_LZ4: - abd = abd_alloc_for_io(L2BLK_GET_PSIZE( - (this_lbp)->lbp_prop), B_TRUE); - abd_copy_from_buf_off(abd, this_lb, 0, - L2BLK_GET_PSIZE((this_lbp)->lbp_prop)); + abd = abd_alloc_for_io(psize, B_TRUE); + abd_copy_from_buf_off(abd, this_lb, 0, psize); if ((err = zio_decompress_data( L2BLK_GET_COMPRESS((this_lbp)->lbp_prop), - abd, this_lb, L2BLK_GET_PSIZE((this_lbp)->lbp_prop), - sizeof (*this_lb))) != 0) { + abd, this_lb, psize, sizeof (*this_lb))) != 0) { err = SET_ERROR(EINVAL); goto cleanup; } diff --git a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh index e3c983be847..c79c3927652 100755 --- a/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh +++ b/tests/zfs-tests/tests/functional/persist_l2arc/persist_l2arc_007_pos.ksh @@ -72,7 +72,7 @@ log_must fio $FIO_SCRIPTS/random_reads.fio log_must zpool offline $TESTPOOL $VDEV_CACHE -sleep 5 +sleep 10 typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks) @@ -81,7 +81,7 @@ typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \ log_must zpool online $TESTPOOL $VDEV_CACHE -sleep 5 +sleep 10 typeset l2_rebuild_log_blk_end=$(get_arcstat l2_rebuild_log_blks) From c614fd6e12c0fe2636a82b341e9ab9d150ee3c8a Mon Sep 17 00:00:00 2001 From: Matthew Macy Date: Fri, 17 Apr 2020 09:30:26 -0700 Subject: [PATCH 021/133] Use new FreeBSD API to largely eliminate object locking Propagate changes in HEAD that mostly eliminate object locking. Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Signed-off-by: Matt Macy Closes #10205 --- include/os/freebsd/spl/sys/vm.h | 16 ++++++++ module/os/freebsd/zfs/dmu_os.c | 12 +++--- module/os/freebsd/zfs/zfs_vnops.c | 67 ++++++++++++++++--------------- 3 files changed, 56 insertions(+), 39 deletions(-) diff --git a/include/os/freebsd/spl/sys/vm.h b/include/os/freebsd/spl/sys/vm.h index 07ee6bc191a..7b3830be8a5 100644 --- a/include/os/freebsd/spl/sys/vm.h +++ b/include/os/freebsd/spl/sys/vm.h @@ -41,6 +41,22 @@ void zfs_vmobject_assert_wlocked(vm_object_t object); void zfs_vmobject_wlock(vm_object_t object); void zfs_vmobject_wunlock(vm_object_t object); +#if __FreeBSD_version >= 1300081 +#define zfs_vmobject_assert_wlocked_12(x) +#define zfs_vmobject_wlock_12(x) +#define zfs_vmobject_wunlock_12(x) +#else +#define zfs_vmobject_assert_wlocked_12(x) \ + zfs_vmobject_assert_wlocked((x)) +#define zfs_vmobject_wlock_12(x) \ + zfs_vmobject_wlock(x) +#define zfs_vmobject_wunlock_12(x) \ + zfs_vmobject_wunlock(x) +#define vm_page_grab_unlocked(obj, idx, flags) \ + vm_page_grab((obj), (idx), (flags)) +#define vm_page_grab_valid_unlocked(m, obj, idx, flags) \ + vm_page_grab_valid((m), (obj), (idx), (flags)) +#endif static inline caddr_t zfs_map_page(vm_page_t pp, struct sf_buf **sfp) { diff --git a/module/os/freebsd/zfs/dmu_os.c b/module/os/freebsd/zfs/dmu_os.c index 268c843e50e..0c806df66fd 100644 --- a/module/os/freebsd/zfs/dmu_os.c +++ b/module/os/freebsd/zfs/dmu_os.c @@ -186,11 +186,11 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count, #endif vmobj = ma[0]->object; - zfs_vmobject_wlock(vmobj); + zfs_vmobject_wlock_12(vmobj); db = dbp[0]; for (i = 0; i < *rbehind; i++) { - m = vm_page_grab(vmobj, ma[0]->pindex - 1 - i, + m = vm_page_grab_unlocked(vmobj, ma[0]->pindex - 1 - i, VM_ALLOC_NORMAL | VM_ALLOC_NOWAIT | VM_ALLOC_BUSY_FLAGS); if (m == NULL) break; @@ -200,7 +200,7 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count, break; } ASSERT(m->dirty == 0); - ASSERT(!pmap_page_is_mapped(m)); + ASSERT(!pmap_page_is_write_mapped(m)); ASSERT(db->db_size > PAGE_SIZE); bufoff = IDX_TO_OFF(m->pindex) % db->db_size; @@ -227,7 +227,7 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count, vm_page_assert_xbusied(m); ASSERT(vm_page_none_valid(m)); ASSERT(m->dirty == 0); - ASSERT(!pmap_page_is_mapped(m)); + ASSERT(!pmap_page_is_write_mapped(m)); va = zfs_map_page(m, &sf); } } @@ -306,7 +306,7 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count, } for (i = 0; i < *rahead; i++) { - m = vm_page_grab(vmobj, ma[count - 1]->pindex + 1 + i, + m = vm_page_grab_unlocked(vmobj, ma[count - 1]->pindex + 1 + i, VM_ALLOC_NORMAL | VM_ALLOC_NOWAIT | VM_ALLOC_BUSY_FLAGS); if (m == NULL) break; @@ -339,7 +339,7 @@ dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count, vm_page_do_sunbusy(m); } *rahead = i; - zfs_vmobject_wunlock(vmobj); + zfs_vmobject_wunlock_12(vmobj); dmu_buf_rele_array(dbp, numbufs, FTAG); return (0); diff --git a/module/os/freebsd/zfs/zfs_vnops.c b/module/os/freebsd/zfs/zfs_vnops.c index d7b92035f7c..711e0b1f75a 100644 --- a/module/os/freebsd/zfs/zfs_vnops.c +++ b/module/os/freebsd/zfs/zfs_vnops.c @@ -394,7 +394,7 @@ page_busy(vnode_t *vp, int64_t start, int64_t off, int64_t nbytes) nbytes = end - off; obj = vp->v_object; - zfs_vmobject_assert_wlocked(obj); + zfs_vmobject_assert_wlocked_12(obj); #if __FreeBSD_version < 1300050 for (;;) { if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL && @@ -427,8 +427,9 @@ page_busy(vnode_t *vp, int64_t start, int64_t off, int64_t nbytes) break; } #else - vm_page_grab_valid(&pp, obj, OFF_TO_IDX(start), VM_ALLOC_NOCREAT | - VM_ALLOC_SBUSY | VM_ALLOC_NORMAL | VM_ALLOC_IGN_SBUSY); + vm_page_grab_valid_unlocked(&pp, obj, OFF_TO_IDX(start), + VM_ALLOC_NOCREAT | VM_ALLOC_SBUSY | VM_ALLOC_NORMAL | + VM_ALLOC_IGN_SBUSY); if (pp != NULL) { ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL); vm_object_pip_add(obj, 1); @@ -460,10 +461,9 @@ page_hold(vnode_t *vp, int64_t start) vm_page_t m; obj = vp->v_object; - zfs_vmobject_assert_wlocked(obj); - - vm_page_grab_valid(&m, obj, OFF_TO_IDX(start), VM_ALLOC_NOCREAT | - VM_ALLOC_WIRED | VM_ALLOC_IGN_SBUSY | VM_ALLOC_NOBUSY); + vm_page_grab_valid_unlocked(&m, obj, OFF_TO_IDX(start), + VM_ALLOC_NOCREAT | VM_ALLOC_WIRED | VM_ALLOC_IGN_SBUSY | + VM_ALLOC_NOBUSY); return (m); } #else @@ -541,7 +541,7 @@ update_pages(vnode_t *vp, int64_t start, int len, objset_t *os, uint64_t oid, ASSERT(obj != NULL); off = start & PAGEOFFSET; - zfs_vmobject_wlock(obj); + zfs_vmobject_wlock_12(obj); #if __FreeBSD_version >= 1300041 vm_object_pip_add(obj, 1); #endif @@ -550,14 +550,14 @@ update_pages(vnode_t *vp, int64_t start, int len, objset_t *os, uint64_t oid, int nbytes = imin(PAGESIZE - off, len); if ((pp = page_busy(vp, start, off, nbytes)) != NULL) { - zfs_vmobject_wunlock(obj); + zfs_vmobject_wunlock_12(obj); va = zfs_map_page(pp, &sf); (void) dmu_read(os, oid, start+off, nbytes, va+off, DMU_READ_PREFETCH); zfs_unmap_page(sf); - zfs_vmobject_wlock(obj); + zfs_vmobject_wlock_12(obj); page_unbusy(pp); } len -= nbytes; @@ -568,7 +568,7 @@ update_pages(vnode_t *vp, int64_t start, int len, objset_t *os, uint64_t oid, #else vm_object_pip_wakeupn(obj, 0); #endif - zfs_vmobject_wunlock(obj); + zfs_vmobject_wunlock_12(obj); } /* @@ -599,36 +599,37 @@ mappedread_sf(vnode_t *vp, int nbytes, uio_t *uio) ASSERT(obj != NULL); ASSERT((uio->uio_loffset & PAGEOFFSET) == 0); - zfs_vmobject_wlock(obj); + zfs_vmobject_wlock_12(obj); for (start = uio->uio_loffset; len > 0; start += PAGESIZE) { int bytes = MIN(PAGESIZE, len); - pp = vm_page_grab(obj, OFF_TO_IDX(start), VM_ALLOC_SBUSY | - VM_ALLOC_NORMAL | VM_ALLOC_IGN_SBUSY); + pp = vm_page_grab_unlocked(obj, OFF_TO_IDX(start), + VM_ALLOC_SBUSY | VM_ALLOC_NORMAL | VM_ALLOC_IGN_SBUSY); if (vm_page_none_valid(pp)) { - zfs_vmobject_wunlock(obj); + zfs_vmobject_wunlock_12(obj); va = zfs_map_page(pp, &sf); error = dmu_read(os, zp->z_id, start, bytes, va, DMU_READ_PREFETCH); if (bytes != PAGESIZE && error == 0) bzero(va + bytes, PAGESIZE - bytes); zfs_unmap_page(sf); - zfs_vmobject_wlock(obj); - vm_page_do_sunbusy(pp); -#if __FreeBSD_version >= 1300047 && __FreeBSD_version < 1300051 -#error "unsupported version window" -#elif __FreeBSD_version >= 1300051 + zfs_vmobject_wlock_12(obj); +#if __FreeBSD_version >= 1300081 if (error == 0) { vm_page_valid(pp); - vm_page_lock(pp); vm_page_activate(pp); - vm_page_unlock(pp); + vm_page_do_sunbusy(pp); + } else { + zfs_vmobject_wlock(obj); + if (!vm_page_wired(pp) && pp->valid == 0 && + vm_page_busy_tryupgrade(pp)) + vm_page_free(pp); + else + vm_page_sunbusy(pp); + zfs_vmobject_wunlock(obj); } - vm_page_do_sunbusy(pp); - if (error != 0 && !vm_page_wired(pp) == 0 && - pp->valid == 0 && vm_page_tryxbusy(pp)) - vm_page_free(pp); #else + vm_page_do_sunbusy(pp); vm_page_lock(pp); if (error) { if (pp->wire_count == 0 && pp->valid == 0 && @@ -650,7 +651,7 @@ mappedread_sf(vnode_t *vp, int nbytes, uio_t *uio) uio->uio_offset += bytes; len -= bytes; } - zfs_vmobject_wunlock(obj); + zfs_vmobject_wunlock_12(obj); return (error); } @@ -680,7 +681,7 @@ mappedread(vnode_t *vp, int nbytes, uio_t *uio) start = uio->uio_loffset; off = start & PAGEOFFSET; - zfs_vmobject_wlock(obj); + zfs_vmobject_wlock_12(obj); for (start &= PAGEMASK; len > 0; start += PAGESIZE) { vm_page_t pp; uint64_t bytes = MIN(PAGESIZE - off, len); @@ -689,24 +690,24 @@ mappedread(vnode_t *vp, int nbytes, uio_t *uio) struct sf_buf *sf; caddr_t va; - zfs_vmobject_wunlock(obj); + zfs_vmobject_wunlock_12(obj); va = zfs_map_page(pp, &sf); error = vn_io_fault_uiomove(va + off, bytes, uio); zfs_unmap_page(sf); - zfs_vmobject_wlock(obj); + zfs_vmobject_wlock_12(obj); page_unhold(pp); } else { - zfs_vmobject_wunlock(obj); + zfs_vmobject_wunlock_12(obj); error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl), uio, bytes); - zfs_vmobject_wlock(obj); + zfs_vmobject_wlock_12(obj); } len -= bytes; off = 0; if (error) break; } - zfs_vmobject_wunlock(obj); + zfs_vmobject_wunlock_12(obj); return (error); } From 1f043c8be1bf575bae1088f150cf17b35184ad12 Mon Sep 17 00:00:00 2001 From: Matthew Ahrens Date: Mon, 20 Apr 2020 10:12:48 -0700 Subject: [PATCH 022/133] Fix zfs send progress reporting The progress of a send is supposed to be reported by `zfs send -v`, but it is not. This works by creating a new user thread (with pthread_create()) which does ZFS_IOC_SEND_PROGRESS ioctls to check how much progress has been made. This IOCTL finds the specified send (since there may be multiple concurrent sends in the system). The IOCTL also checks that the specified send was started by the current process. On Linux, different threads of the same process are represented as different `struct task_struct`s (and, confusingly, have different PID's). To check if if two threads are in the same process, we need to check if they have the same `struct task_struct:group_leader`. We used to to this correctly, but it was inadvertently changed by 30af21b02569 (Redacted Send) to simply check if the current `struct task_struct` is the one that started the send. This commit changes the code back to checking if the send was started by a `struct task_struct` with the same `group_leader` as the calling thread. Reviewed-by: Ryan Moeller Reviewed-by: Chris Wedgwood Reviewed-by: Paul Dagnelie Reviewed-by: Brian Behlendorf Signed-off-by: Matthew Ahrens Closes #10215 Closes #10216 --- include/os/freebsd/spl/sys/proc.h | 7 +++++++ include/os/linux/spl/sys/proc.h | 7 +++++++ module/zfs/zfs_ioctl.c | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/include/os/freebsd/spl/sys/proc.h b/include/os/freebsd/spl/sys/proc.h index fca833018ea..07201dd6a09 100644 --- a/include/os/freebsd/spl/sys/proc.h +++ b/include/os/freebsd/spl/sys/proc.h @@ -117,4 +117,11 @@ do_thread_create(caddr_t stk, size_t stksize, void (*proc)(void *), void *arg, int uread(proc_t *, void *, size_t, uintptr_t); int uwrite(proc_t *, void *, size_t, uintptr_t); + +static inline boolean_t +zfs_proc_is_caller(proc_t *p) +{ + return (p == curproc); +} + #endif /* _OPENSOLARIS_SYS_PROC_H_ */ diff --git a/include/os/linux/spl/sys/proc.h b/include/os/linux/spl/sys/proc.h index 28768392077..fefce515eb2 100644 --- a/include/os/linux/spl/sys/proc.h +++ b/include/os/linux/spl/sys/proc.h @@ -26,10 +26,17 @@ #define _SPL_PROC_H #include +#include extern struct proc_dir_entry *proc_spl_kstat; int spl_proc_init(void); void spl_proc_fini(void); +static inline boolean_t +zfs_proc_is_caller(struct task_struct *t) +{ + return (t->group_leader == current->group_leader); +} + #endif /* SPL_PROC_H */ diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c index fb9435341db..5c53eb63779 100644 --- a/module/zfs/zfs_ioctl.c +++ b/module/zfs/zfs_ioctl.c @@ -5468,7 +5468,7 @@ zfs_ioc_send_progress(zfs_cmd_t *zc) for (dsp = list_head(&ds->ds_sendstreams); dsp != NULL; dsp = list_next(&ds->ds_sendstreams, dsp)) { if (dsp->dss_outfd == zc->zc_cookie && - dsp->dss_proc == curproc) + zfs_proc_is_caller(dsp->dss_proc)) break; } From a84c92f93364116b5e7f4685eb6d665526251a51 Mon Sep 17 00:00:00 2001 From: Niklas Haas Date: Tue, 21 Apr 2020 18:29:31 +0200 Subject: [PATCH 023/133] Don't attempt trimming "hole" vdevs On zpools containing hole vdevs (e.g. removed log devices), the `zpool trim` (and presumably `zpool initialize`) commands will attempt calling their respective functions on "hole", which fails, as this is not a real vdev. Avoid this by removing HOLE vdevs in zpool_collect_leaves. Reviewed-by: Brian Behlendorf Reviewed-by: George Melikov Signed-off-by: Niklas Haas Closes #10227 --- cmd/zpool/zpool_main.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index e8e94cd94eb..d62a6eb3c9a 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -445,7 +445,8 @@ zpool_collect_leaves(zpool_handle_t *zhp, nvlist_t *nvroot, nvlist_t *res) char *path = zpool_vdev_name(g_zfs, zhp, nvroot, VDEV_NAME_PATH); - if (strcmp(path, VDEV_TYPE_INDIRECT) != 0) + if (strcmp(path, VDEV_TYPE_INDIRECT) != 0 && + strcmp(path, VDEV_TYPE_HOLE) != 0) fnvlist_add_boolean(res, path); free(path); From 32d805c3e2888cbb71956454ced38b4882c27c00 Mon Sep 17 00:00:00 2001 From: Matthew Ahrens Date: Wed, 22 Apr 2020 10:26:56 -0700 Subject: [PATCH 024/133] Use a struct to organize metaslab-group-allocator fields Each metaslab group (of which there is one per top-level vdev) has several (4, by default) "metaslab group allocators". Each "allocator" has its own metaslab that it prefers to allocate from (the "primary" allocator), and each can perform allocations concurrently with the other allocators. In addition to the primary metaslab, there are several other fields that need to be tracked separately for each allocator. These are currently stored as several arrays in the metaslab_group_t, each array indexed by allocator number. This change organizes all the metaslab-group-allocator-specific fields into a new struct, metaslab_group_allocator_t. The metaslab_group_t now needs only one array indexed by the allocator number - which contains the metaslab_group_allocator_t's. Reviewed-by: Paul Dagnelie Reviewed-by: Brian Behlendorf Signed-off-by: Matthew Ahrens Closes #10213 --- include/sys/metaslab_impl.h | 15 ++++-- module/zfs/metaslab.c | 95 ++++++++++++++++++------------------- module/zfs/spa.c | 9 ++-- 3 files changed, 62 insertions(+), 57 deletions(-) diff --git a/include/sys/metaslab_impl.h b/include/sys/metaslab_impl.h index d140f741d85..4a7475256a2 100644 --- a/include/sys/metaslab_impl.h +++ b/include/sys/metaslab_impl.h @@ -203,6 +203,16 @@ struct metaslab_class { multilist_t *mc_metaslab_txg_list; }; +/* + * Per-allocator data structure. + */ +typedef struct metaslab_group_allocator { + uint64_t mga_cur_max_alloc_queue_depth; + zfs_refcount_t mga_alloc_queue_depth; + metaslab_t *mga_primary; + metaslab_t *mga_secondary; +} metaslab_group_allocator_t; + /* * Metaslab groups encapsulate all the allocatable regions (i.e. metaslabs) * of a top-level vdev. They are linked together to form a circular linked @@ -214,8 +224,6 @@ struct metaslab_class { */ struct metaslab_group { kmutex_t mg_lock; - metaslab_t **mg_primaries; - metaslab_t **mg_secondaries; avl_tree_t mg_metaslab_tree; uint64_t mg_aliquot; boolean_t mg_allocatable; /* can we allocate? */ @@ -263,9 +271,8 @@ struct metaslab_group { * groups are unable to handle their share of allocations. */ uint64_t mg_max_alloc_queue_depth; - uint64_t *mg_cur_max_alloc_queue_depth; - zfs_refcount_t *mg_alloc_queue_depth; int mg_allocators; + metaslab_group_allocator_t *mg_allocator; /* array */ /* * A metalab group that can no longer allocate the minimum block * size will set mg_no_free_space. Once a metaslab group is out diff --git a/module/zfs/metaslab.c b/module/zfs/metaslab.c index 2fc017b5b19..1fc44399f21 100644 --- a/module/zfs/metaslab.c +++ b/module/zfs/metaslab.c @@ -814,10 +814,6 @@ metaslab_group_create(metaslab_class_t *mc, vdev_t *vd, int allocators) mutex_init(&mg->mg_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&mg->mg_ms_disabled_lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&mg->mg_ms_disabled_cv, NULL, CV_DEFAULT, NULL); - mg->mg_primaries = kmem_zalloc(allocators * sizeof (metaslab_t *), - KM_SLEEP); - mg->mg_secondaries = kmem_zalloc(allocators * sizeof (metaslab_t *), - KM_SLEEP); avl_create(&mg->mg_metaslab_tree, metaslab_compare, sizeof (metaslab_t), offsetof(metaslab_t, ms_group_node)); mg->mg_vd = vd; @@ -827,13 +823,11 @@ metaslab_group_create(metaslab_class_t *mc, vdev_t *vd, int allocators) mg->mg_no_free_space = B_TRUE; mg->mg_allocators = allocators; - mg->mg_alloc_queue_depth = kmem_zalloc(allocators * - sizeof (zfs_refcount_t), KM_SLEEP); - mg->mg_cur_max_alloc_queue_depth = kmem_zalloc(allocators * - sizeof (uint64_t), KM_SLEEP); + mg->mg_allocator = kmem_zalloc(allocators * + sizeof (metaslab_group_allocator_t), KM_SLEEP); for (int i = 0; i < allocators; i++) { - zfs_refcount_create_tracked(&mg->mg_alloc_queue_depth[i]); - mg->mg_cur_max_alloc_queue_depth[i] = 0; + metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; + zfs_refcount_create_tracked(&mga->mga_alloc_queue_depth); } mg->mg_taskq = taskq_create("metaslab_group_taskq", metaslab_load_pct, @@ -856,21 +850,16 @@ metaslab_group_destroy(metaslab_group_t *mg) taskq_destroy(mg->mg_taskq); avl_destroy(&mg->mg_metaslab_tree); - kmem_free(mg->mg_primaries, mg->mg_allocators * sizeof (metaslab_t *)); - kmem_free(mg->mg_secondaries, mg->mg_allocators * - sizeof (metaslab_t *)); mutex_destroy(&mg->mg_lock); mutex_destroy(&mg->mg_ms_disabled_lock); cv_destroy(&mg->mg_ms_disabled_cv); for (int i = 0; i < mg->mg_allocators; i++) { - zfs_refcount_destroy(&mg->mg_alloc_queue_depth[i]); - mg->mg_cur_max_alloc_queue_depth[i] = 0; + metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; + zfs_refcount_destroy(&mga->mga_alloc_queue_depth); } - kmem_free(mg->mg_alloc_queue_depth, mg->mg_allocators * - sizeof (zfs_refcount_t)); - kmem_free(mg->mg_cur_max_alloc_queue_depth, mg->mg_allocators * - sizeof (uint64_t)); + kmem_free(mg->mg_allocator, mg->mg_allocators * + sizeof (metaslab_group_allocator_t)); kmem_free(mg, sizeof (metaslab_group_t)); } @@ -951,14 +940,15 @@ metaslab_group_passivate(metaslab_group_t *mg) spa_config_enter(spa, locks & ~(SCL_ZIO - 1), spa, RW_WRITER); metaslab_group_alloc_update(mg); for (int i = 0; i < mg->mg_allocators; i++) { - metaslab_t *msp = mg->mg_primaries[i]; + metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; + metaslab_t *msp = mga->mga_primary; if (msp != NULL) { mutex_enter(&msp->ms_lock); metaslab_passivate(msp, metaslab_weight_from_range_tree(msp)); mutex_exit(&msp->ms_lock); } - msp = mg->mg_secondaries[i]; + msp = mga->mga_secondary; if (msp != NULL) { mutex_enter(&msp->ms_lock); metaslab_passivate(msp, @@ -1218,9 +1208,9 @@ metaslab_group_allocatable(metaslab_group_t *mg, metaslab_group_t *rotor, * regardless of the mg_allocatable or throttle settings. */ if (mg->mg_allocatable) { - metaslab_group_t *mgp; + metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; int64_t qdepth; - uint64_t qmax = mg->mg_cur_max_alloc_queue_depth[allocator]; + uint64_t qmax = mga->mga_cur_max_alloc_queue_depth; if (!mc->mc_alloc_throttle_enabled) return (B_TRUE); @@ -1239,8 +1229,7 @@ metaslab_group_allocatable(metaslab_group_t *mg, metaslab_group_t *rotor, */ qmax = qmax * (4 + d) / 4; - qdepth = zfs_refcount_count( - &mg->mg_alloc_queue_depth[allocator]); + qdepth = zfs_refcount_count(&mga->mga_alloc_queue_depth); /* * If this metaslab group is below its qmax or it's @@ -1258,11 +1247,14 @@ metaslab_group_allocatable(metaslab_group_t *mg, metaslab_group_t *rotor, * racy since we can't hold the locks for all metaslab * groups at the same time when we make this check. */ - for (mgp = mg->mg_next; mgp != rotor; mgp = mgp->mg_next) { - qmax = mgp->mg_cur_max_alloc_queue_depth[allocator]; + for (metaslab_group_t *mgp = mg->mg_next; + mgp != rotor; mgp = mgp->mg_next) { + metaslab_group_allocator_t *mgap = + &mgp->mg_allocator[allocator]; + qmax = mgap->mga_cur_max_alloc_queue_depth; qmax = qmax * (4 + d) / 4; - qdepth = zfs_refcount_count( - &mgp->mg_alloc_queue_depth[allocator]); + qdepth = + zfs_refcount_count(&mgap->mga_alloc_queue_depth); /* * If there is another metaslab group that @@ -3205,6 +3197,7 @@ static int metaslab_activate_allocator(metaslab_group_t *mg, metaslab_t *msp, int allocator, uint64_t activation_weight) { + metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; ASSERT(MUTEX_HELD(&msp->ms_lock)); /* @@ -3219,16 +3212,16 @@ metaslab_activate_allocator(metaslab_group_t *mg, metaslab_t *msp, return (0); } - metaslab_t **arr = (activation_weight == METASLAB_WEIGHT_PRIMARY ? - mg->mg_primaries : mg->mg_secondaries); + metaslab_t **mspp = (activation_weight == METASLAB_WEIGHT_PRIMARY ? + &mga->mga_primary : &mga->mga_secondary); mutex_enter(&mg->mg_lock); - if (arr[allocator] != NULL) { + if (*mspp != NULL) { mutex_exit(&mg->mg_lock); return (EEXIST); } - arr[allocator] = msp; + *mspp = msp; ASSERT3S(msp->ms_allocator, ==, -1); msp->ms_allocator = allocator; msp->ms_primary = (activation_weight == METASLAB_WEIGHT_PRIMARY); @@ -3237,7 +3230,6 @@ metaslab_activate_allocator(metaslab_group_t *mg, metaslab_t *msp, msp->ms_activation_weight = msp->ms_weight; metaslab_group_sort_impl(mg, msp, msp->ms_weight | activation_weight); - mutex_exit(&mg->mg_lock); return (0); @@ -3337,14 +3329,15 @@ metaslab_passivate_allocator(metaslab_group_t *mg, metaslab_t *msp, ASSERT3S(0, <=, msp->ms_allocator); ASSERT3U(msp->ms_allocator, <, mg->mg_allocators); + metaslab_group_allocator_t *mga = &mg->mg_allocator[msp->ms_allocator]; if (msp->ms_primary) { - ASSERT3P(mg->mg_primaries[msp->ms_allocator], ==, msp); + ASSERT3P(mga->mga_primary, ==, msp); ASSERT(msp->ms_weight & METASLAB_WEIGHT_PRIMARY); - mg->mg_primaries[msp->ms_allocator] = NULL; + mga->mga_primary = NULL; } else { - ASSERT3P(mg->mg_secondaries[msp->ms_allocator], ==, msp); + ASSERT3P(mga->mga_secondary, ==, msp); ASSERT(msp->ms_weight & METASLAB_WEIGHT_SECONDARY); - mg->mg_secondaries[msp->ms_allocator] = NULL; + mga->mga_secondary = NULL; } msp->ms_allocator = -1; metaslab_group_sort_impl(mg, msp, weight); @@ -4493,22 +4486,24 @@ metaslab_group_alloc_increment(spa_t *spa, uint64_t vdev, void *tag, int flags, if (!mg->mg_class->mc_alloc_throttle_enabled) return; - (void) zfs_refcount_add(&mg->mg_alloc_queue_depth[allocator], tag); + metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; + (void) zfs_refcount_add(&mga->mga_alloc_queue_depth, tag); } static void metaslab_group_increment_qdepth(metaslab_group_t *mg, int allocator) { + metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; uint64_t max = mg->mg_max_alloc_queue_depth; - uint64_t cur = mg->mg_cur_max_alloc_queue_depth[allocator]; + uint64_t cur = mga->mga_cur_max_alloc_queue_depth; while (cur < max) { - if (atomic_cas_64(&mg->mg_cur_max_alloc_queue_depth[allocator], + if (atomic_cas_64(&mga->mga_cur_max_alloc_queue_depth, cur, cur + 1) == cur) { atomic_inc_64( &mg->mg_class->mc_alloc_max_slots[allocator]); return; } - cur = mg->mg_cur_max_alloc_queue_depth[allocator]; + cur = mga->mga_cur_max_alloc_queue_depth; } } @@ -4524,7 +4519,8 @@ metaslab_group_alloc_decrement(spa_t *spa, uint64_t vdev, void *tag, int flags, if (!mg->mg_class->mc_alloc_throttle_enabled) return; - (void) zfs_refcount_remove(&mg->mg_alloc_queue_depth[allocator], tag); + metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; + (void) zfs_refcount_remove(&mga->mga_alloc_queue_depth, tag); if (io_complete) metaslab_group_increment_qdepth(mg, allocator); } @@ -4540,8 +4536,8 @@ metaslab_group_alloc_verify(spa_t *spa, const blkptr_t *bp, void *tag, for (int d = 0; d < ndvas; d++) { uint64_t vdev = DVA_GET_VDEV(&dva[d]); metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg; - VERIFY(zfs_refcount_not_held( - &mg->mg_alloc_queue_depth[allocator], tag)); + metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; + VERIFY(zfs_refcount_not_held(&mga->mga_alloc_queue_depth, tag)); } #endif } @@ -4716,6 +4712,7 @@ metaslab_group_alloc_normal(metaslab_group_t *mg, zio_alloc_list_t *zal, */ if (mg->mg_ms_ready < mg->mg_allocators * 3) allocator = 0; + metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; ASSERT3U(mg->mg_vd->vdev_ms_count, >=, 2); @@ -4737,8 +4734,8 @@ metaslab_group_alloc_normal(metaslab_group_t *mg, zio_alloc_list_t *zal, mutex_enter(&mg->mg_lock); if (activation_weight == METASLAB_WEIGHT_PRIMARY && - mg->mg_primaries[allocator] != NULL) { - msp = mg->mg_primaries[allocator]; + mga->mga_primary != NULL) { + msp = mga->mga_primary; /* * Even though we don't hold the ms_lock for the @@ -4753,8 +4750,8 @@ metaslab_group_alloc_normal(metaslab_group_t *mg, zio_alloc_list_t *zal, was_active = B_TRUE; ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); } else if (activation_weight == METASLAB_WEIGHT_SECONDARY && - mg->mg_secondaries[allocator] != NULL) { - msp = mg->mg_secondaries[allocator]; + mga->mga_secondary != NULL) { + msp = mga->mga_secondary; /* * See comment above about the similar assertions diff --git a/module/zfs/spa.c b/module/zfs/spa.c index aface90afc6..bd1e091cadc 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -8720,13 +8720,14 @@ spa_sync_adjust_vdev_max_queue_depth(spa_t *spa) * allocations look at mg_max_alloc_queue_depth, and async * allocations all happen from spa_sync(). */ - for (int i = 0; i < spa->spa_alloc_count; i++) + for (int i = 0; i < mg->mg_allocators; i++) { ASSERT0(zfs_refcount_count( - &(mg->mg_alloc_queue_depth[i]))); + &(mg->mg_allocator[i].mga_alloc_queue_depth))); + } mg->mg_max_alloc_queue_depth = max_queue_depth; - for (int i = 0; i < spa->spa_alloc_count; i++) { - mg->mg_cur_max_alloc_queue_depth[i] = + for (int i = 0; i < mg->mg_allocators; i++) { + mg->mg_allocator[i].mga_cur_max_alloc_queue_depth = zfs_vdev_def_queue_depth; } slots_per_allocator += zfs_vdev_def_queue_depth; From 70e5ad31f6425868b8a173bbc2be4ef08a8d949b Mon Sep 17 00:00:00 2001 From: Joao Carlos Mendes Luis Date: Wed, 22 Apr 2020 14:40:34 -0300 Subject: [PATCH 025/133] Fix more leaks detected by ASAN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit fixes a bunch of missing free() calls in a10d50f99 Reviewed-by: Brian Behlendorf Signed-off-by: João Carlos Mendes Luís Closes #10219 --- lib/libzfs/libzfs_dataset.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/libzfs/libzfs_dataset.c b/lib/libzfs/libzfs_dataset.c index 45e7a79fb49..90c6b68e387 100644 --- a/lib/libzfs/libzfs_dataset.c +++ b/lib/libzfs/libzfs_dataset.c @@ -931,10 +931,15 @@ libzfs_mnttab_add(libzfs_handle_t *hdl, const char *special, * Another thread may have already added this entry * via libzfs_mnttab_update. If so we should skip it. */ - if (avl_find(&hdl->libzfs_mnttab_cache, mtn, NULL) != NULL) + if (avl_find(&hdl->libzfs_mnttab_cache, mtn, NULL) != NULL) { + free(mtn->mtn_mt.mnt_special); + free(mtn->mtn_mt.mnt_mountp); + free(mtn->mtn_mt.mnt_fstype); + free(mtn->mtn_mt.mnt_mntopts); free(mtn); - else + } else { avl_add(&hdl->libzfs_mnttab_cache, mtn); + } } pthread_mutex_unlock(&hdl->libzfs_mnttab_cache_lock); } From 196bee4cfd576fb15baa6a64ad6501c594f45497 Mon Sep 17 00:00:00 2001 From: Matthew Ahrens Date: Thu, 23 Apr 2020 10:06:57 -0700 Subject: [PATCH 026/133] Remove deduplicated send/receive code Deduplicated send streams (i.e. `zfs send -D` and `zfs receive` of such streams) are deprecated. Deduplicated send streams can be received by first converting them to non-deduplicated with the `zstream redup` command. This commit removes the code for sending and receiving deduplicated send streams. `zfs send -D` will now print a warning, ignore the `-D` flag, and generate a regular (non-deduplicated) send stream. `zfs receive` of a deduplicated send stream will print an error message and fail. The resulting code simplification (especially in the kernel's support for receiving dedup streams) should help enable future performance enhancements. Several new tests are added which leverage `zstream redup`. Reviewed-by: Paul Dagnelie Reviewed-by: Brian Behlendorf Signed-off-by: Matthew Ahrens Issue #7887 Issue #10117 Issue #10156 Closes #10212 --- cmd/zfs/zfs_main.c | 21 +- include/libzfs.h | 6 +- include/libzfs_impl.h | 3 +- include/sys/dmu.h | 4 +- include/sys/dmu_recv.h | 18 +- include/sys/zfs_ioctl.h | 5 +- include/sys/zfs_onexit.h | 5 +- lib/libzfs/libzfs_sendrecv.c | 523 +----------------- lib/libzfs_core/libzfs_core.c | 44 +- man/man8/zfs-receive.8 | 15 +- man/man8/zfs-send.8 | 18 +- module/zfs/dmu.c | 52 +- module/zfs/dmu_recv.c | 229 +------- module/zfs/zfs_ioctl.c | 36 +- module/zfs/zfs_onexit.c | 79 +-- tests/runfiles/common.run | 14 +- .../tests/functional/rsend/Makefile.am | 9 +- .../tests/functional/rsend/dedup.zsend.bz2 | Bin 0 -> 18561 bytes .../functional/rsend/dedup_encrypted_zvol.bz2 | Bin 0 -> 1482112 bytes .../rsend/dedup_encrypted_zvol.zsend.bz2 | Bin 0 -> 836685 bytes .../tests/functional/rsend/fs.tar.gz | Bin 0 -> 13196 bytes .../tests/functional/rsend/recv_dedup.ksh | 53 ++ .../rsend/recv_dedup_encrypted_zvol.ksh | 60 ++ .../tests/functional/rsend/send-cD.ksh | 89 --- ...ed_zvol.ksh => send-wR_encrypted_zvol.ksh} | 15 +- .../functional/rsend/send_partial_dataset.ksh | 2 +- 26 files changed, 219 insertions(+), 1081 deletions(-) create mode 100644 tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 create mode 100644 tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 create mode 100644 tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 create mode 100644 tests/zfs-tests/tests/functional/rsend/fs.tar.gz create mode 100755 tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh create mode 100755 tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh delete mode 100755 tests/zfs-tests/tests/functional/rsend/send-cD.ksh rename tests/zfs-tests/tests/functional/rsend/{send-wDR_encrypted_zvol.ksh => send-wR_encrypted_zvol.ksh} (84%) diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c index ae71cdc881f..15e350313af 100644 --- a/cmd/zfs/zfs_main.c +++ b/cmd/zfs/zfs_main.c @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2019 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright 2012 Milan Jurik. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. * Copyright (c) 2013 Steven Hartland. All rights reserved. @@ -4266,7 +4266,10 @@ zfs_do_send(int argc, char **argv) flags.progress = B_TRUE; break; case 'D': - flags.dedup = B_TRUE; + (void) fprintf(stderr, + gettext("WARNING: deduplicated send is no " + "longer supported. A regular,\n" + "non-deduplicated stream will be generated.\n\n")); break; case 'n': flags.dryrun = B_TRUE; @@ -4333,16 +4336,6 @@ zfs_do_send(int argc, char **argv) } } - if (flags.dedup) { - (void) fprintf(stderr, - gettext("WARNING: deduplicated send is " - "deprecated, and will be removed in a\n" - "future release. (In the future, the flag will be " - "accepted, but a\n" - "regular, non-deduplicated stream will be " - "generated.)\n\n")); - } - if (flags.parsable && flags.verbosity == 0) flags.verbosity = 1; @@ -4351,7 +4344,7 @@ zfs_do_send(int argc, char **argv) if (resume_token != NULL) { if (fromname != NULL || flags.replicate || flags.props || - flags.backup || flags.dedup || flags.holds || + flags.backup || flags.holds || flags.saved || redactbook != NULL) { (void) fprintf(stderr, gettext("invalid flags combined with -t\n")); @@ -4375,7 +4368,7 @@ zfs_do_send(int argc, char **argv) if (flags.saved) { if (fromname != NULL || flags.replicate || flags.props || - flags.doall || flags.backup || flags.dedup || + flags.doall || flags.backup || flags.holds || flags.largeblock || flags.embed_data || flags.compress || flags.raw || redactbook != NULL) { (void) fprintf(stderr, gettext("incompatible flags " diff --git a/include/libzfs.h b/include/libzfs.h index 7633579d434..c4f08882ed6 100644 --- a/include/libzfs.h +++ b/include/libzfs.h @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright Joyent, Inc. * Copyright (c) 2013 Steven Hartland. All rights reserved. * Copyright (c) 2016, Intel Corporation. @@ -651,8 +651,8 @@ typedef struct sendflags { /* if dataset is a clone, do incremental from its origin */ boolean_t fromorigin; - /* do deduplication */ - boolean_t dedup; + /* field no longer used, maintained for backwards compatibility */ + boolean_t pad; /* send properties (ie, -p) */ boolean_t props; diff --git a/include/libzfs_impl.h b/include/libzfs_impl.h index 45c344c8366..5cec629132a 100644 --- a/include/libzfs_impl.h +++ b/include/libzfs_impl.h @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2017 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright (c) 2018 Datto Inc. */ @@ -71,7 +71,6 @@ struct libzfs_handle { int libzfs_pool_iter; char libzfs_chassis_id[256]; boolean_t libzfs_prop_debug; - boolean_t libzfs_dedup_warning_printed; }; #define ZFSSHARE_MISS 0x01 /* Didn't find entry in cache */ diff --git a/include/sys/dmu.h b/include/sys/dmu.h index 928ee763d48..139f3cbdfea 100644 --- a/include/sys/dmu.h +++ b/include/sys/dmu.h @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. * Copyright 2014 HybridCluster. All rights reserved. @@ -864,8 +864,6 @@ int dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, int dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset, struct arc_buf *buf, dmu_tx_t *tx); #define dmu_assign_arcbuf dmu_assign_arcbuf_by_dbuf -void dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset, - dmu_buf_t *handle, dmu_tx_t *tx); #ifdef HAVE_UIO_ZEROCOPY int dmu_xuio_init(struct xuio *uio, int niov); void dmu_xuio_fini(struct xuio *uio); diff --git a/include/sys/dmu_recv.h b/include/sys/dmu_recv.h index e3b54e3d198..c0a562115a2 100644 --- a/include/sys/dmu_recv.h +++ b/include/sys/dmu_recv.h @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2018 by Delphix. All rights reserved. + * Copyright (c) 2012, 2020 by Delphix. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. */ @@ -53,10 +53,8 @@ typedef struct dmu_recv_cookie { boolean_t drc_raw; boolean_t drc_clone; boolean_t drc_spill; - struct avl_tree *drc_guid_to_ds_map; nvlist_t *drc_keynvl; uint64_t drc_fromsnapobj; - uint64_t drc_newsnapobj; uint64_t drc_ivset_guid; void *drc_owner; cred_t *drc_cred; @@ -80,13 +78,11 @@ typedef struct dmu_recv_cookie { objlist_t *drc_ignore_objlist; } dmu_recv_cookie_t; -int dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin, - boolean_t force, boolean_t resumable, nvlist_t *localprops, - nvlist_t *hidden_args, char *origin, dmu_recv_cookie_t *drc, - zfs_file_t *fp, offset_t *voffp); -int dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd, - uint64_t *action_handlep, offset_t *voffp); -int dmu_recv_end(dmu_recv_cookie_t *drc, void *owner); -boolean_t dmu_objset_is_receiving(objset_t *os); +int dmu_recv_begin(char *, char *, dmu_replay_record_t *, + boolean_t, boolean_t, nvlist_t *, nvlist_t *, char *, + dmu_recv_cookie_t *, zfs_file_t *, offset_t *); +int dmu_recv_stream(dmu_recv_cookie_t *, offset_t *); +int dmu_recv_end(dmu_recv_cookie_t *, void *); +boolean_t dmu_objset_is_receiving(objset_t *); #endif /* _DMU_RECV_H */ diff --git a/include/sys/zfs_ioctl.h b/include/sys/zfs_ioctl.h index da5a3ee60df..d4ffe70bbe3 100644 --- a/include/sys/zfs_ioctl.h +++ b/include/sys/zfs_ioctl.h @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2018 by Delphix. All rights reserved. + * Copyright (c) 2012, 2020 by Delphix. All rights reserved. * Copyright 2016 RackTop Systems. * Copyright (c) 2017, Intel Corporation. */ @@ -111,8 +111,7 @@ typedef enum drr_headertype { /* * Mask of all supported backup features */ -#define DMU_BACKUP_FEATURE_MASK (DMU_BACKUP_FEATURE_DEDUP | \ - DMU_BACKUP_FEATURE_DEDUPPROPS | DMU_BACKUP_FEATURE_SA_SPILL | \ +#define DMU_BACKUP_FEATURE_MASK (DMU_BACKUP_FEATURE_SA_SPILL | \ DMU_BACKUP_FEATURE_EMBED_DATA | DMU_BACKUP_FEATURE_LZ4 | \ DMU_BACKUP_FEATURE_RESUMING | DMU_BACKUP_FEATURE_LARGE_BLOCKS | \ DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_LARGE_DNODE | \ diff --git a/include/sys/zfs_onexit.h b/include/sys/zfs_onexit.h index 4982bd4d0af..0fab23ff849 100644 --- a/include/sys/zfs_onexit.h +++ b/include/sys/zfs_onexit.h @@ -21,6 +21,7 @@ /* * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 by Delphix. All rights reserved. */ #ifndef _SYS_ZFS_ONEXIT_H @@ -54,10 +55,6 @@ extern int zfs_onexit_fd_hold(int fd, minor_t *minorp); extern void zfs_onexit_fd_rele(int fd); extern int zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data, uint64_t *action_handle); -extern int zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, - boolean_t fire); -extern int zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, - void **data); #ifdef __cplusplus } diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 43a39e789c5..c65673dd35b 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. * Copyright (c) 2012 Pawel Jakub Dawidek . * All rights reserved @@ -73,22 +73,14 @@ extern void zfs_setprop_error(libzfs_handle_t *, zfs_prop_t, int, char *); static int zfs_receive_impl(libzfs_handle_t *, const char *, const char *, - recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **, int, - uint64_t *, const char *, nvlist_t *); + recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **, + const char *, nvlist_t *); static int guid_to_name_redact_snaps(libzfs_handle_t *hdl, const char *parent, uint64_t guid, boolean_t bookmark_ok, uint64_t *redact_snap_guids, uint64_t num_redact_snaps, char *name); static int guid_to_name(libzfs_handle_t *, const char *, uint64_t, boolean_t, char *); -static const zio_cksum_t zero_cksum = { { 0 } }; - -typedef struct dedup_arg { - int inputfd; - int outputfd; - libzfs_handle_t *dedup_hdl; -} dedup_arg_t; - typedef struct progress_arg { zfs_handle_t *pa_zhp; int pa_fd; @@ -97,112 +89,6 @@ typedef struct progress_arg { int pa_verbosity; } progress_arg_t; -typedef struct dataref { - uint64_t ref_guid; - uint64_t ref_object; - uint64_t ref_offset; -} dataref_t; - -typedef struct dedup_entry { - struct dedup_entry *dde_next; - zio_cksum_t dde_chksum; - uint64_t dde_prop; - dataref_t dde_ref; -} dedup_entry_t; - -#define MAX_DDT_PHYSMEM_PERCENT 20 -#define SMALLEST_POSSIBLE_MAX_DDT_MB 128 - -typedef struct dedup_table { - dedup_entry_t **dedup_hash_array; - umem_cache_t *ddecache; - uint64_t max_ddt_size; /* max dedup table size in bytes */ - uint64_t cur_ddt_size; /* current dedup table size in bytes */ - uint64_t ddt_count; - int numhashbits; - boolean_t ddt_full; -} dedup_table_t; - -static int -high_order_bit(uint64_t n) -{ - int count; - - for (count = 0; n != 0; count++) - n >>= 1; - return (count); -} - -static size_t -ssread(void *buf, size_t len, FILE *stream) -{ - size_t outlen; - - if ((outlen = fread(buf, len, 1, stream)) == 0) - return (0); - - return (outlen); -} - -static void -ddt_hash_append(libzfs_handle_t *hdl, dedup_table_t *ddt, dedup_entry_t **ddepp, - zio_cksum_t *cs, uint64_t prop, dataref_t *dr) -{ - dedup_entry_t *dde; - - if (ddt->cur_ddt_size >= ddt->max_ddt_size) { - if (ddt->ddt_full == B_FALSE) { - zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "Dedup table full. Deduplication will continue " - "with existing table entries")); - ddt->ddt_full = B_TRUE; - } - return; - } - - if ((dde = umem_cache_alloc(ddt->ddecache, UMEM_DEFAULT)) - != NULL) { - assert(*ddepp == NULL); - dde->dde_next = NULL; - dde->dde_chksum = *cs; - dde->dde_prop = prop; - dde->dde_ref = *dr; - *ddepp = dde; - ddt->cur_ddt_size += sizeof (dedup_entry_t); - ddt->ddt_count++; - } -} - -/* - * Using the specified dedup table, do a lookup for an entry with - * the checksum cs. If found, return the block's reference info - * in *dr. Otherwise, insert a new entry in the dedup table, using - * the reference information specified by *dr. - * - * return value: true - entry was found - * false - entry was not found - */ -static boolean_t -ddt_update(libzfs_handle_t *hdl, dedup_table_t *ddt, zio_cksum_t *cs, - uint64_t prop, dataref_t *dr) -{ - uint32_t hashcode; - dedup_entry_t **ddepp; - - hashcode = BF64_GET(cs->zc_word[0], 0, ddt->numhashbits); - - for (ddepp = &(ddt->dedup_hash_array[hashcode]); *ddepp != NULL; - ddepp = &((*ddepp)->dde_next)) { - if (ZIO_CHECKSUM_EQUAL(((*ddepp)->dde_chksum), *cs) && - (*ddepp)->dde_prop == prop) { - *dr = (*ddepp)->dde_ref; - return (B_TRUE); - } - } - ddt_hash_append(hdl, ddt, ddepp, cs, prop, dr); - return (B_FALSE); -} - static int dump_record(dmu_replay_record_t *drr, void *payload, int payload_len, zio_cksum_t *zc, int outfd) @@ -228,274 +114,6 @@ dump_record(dmu_replay_record_t *drr, void *payload, int payload_len, return (0); } -/* - * This function is started in a separate thread when the dedup option - * has been requested. The main send thread determines the list of - * snapshots to be included in the send stream and makes the ioctl calls - * for each one. But instead of having the ioctl send the output to the - * the output fd specified by the caller of zfs_send()), the - * ioctl is told to direct the output to a pipe, which is read by the - * alternate thread running THIS function. This function does the - * dedup'ing by: - * 1. building a dedup table (the DDT) - * 2. doing checksums on each data block and inserting a record in the DDT - * 3. looking for matching checksums, and - * 4. sending a DRR_WRITE_BYREF record instead of a write record whenever - * a duplicate block is found. - * The output of this function then goes to the output fd requested - * by the caller of zfs_send(). - */ -static void * -cksummer(void *arg) -{ - dedup_arg_t *dda = arg; - char *buf = zfs_alloc(dda->dedup_hdl, SPA_MAXBLOCKSIZE); - dmu_replay_record_t thedrr = { 0 }; - dmu_replay_record_t *drr = &thedrr; - FILE *ofp; - int outfd; - dedup_table_t ddt; - zio_cksum_t stream_cksum; - uint64_t numbuckets; - -#ifdef _ILP32 - ddt.max_ddt_size = SMALLEST_POSSIBLE_MAX_DDT_MB << 20; -#else - uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); - ddt.max_ddt_size = - MAX((physmem * MAX_DDT_PHYSMEM_PERCENT) / 100, - SMALLEST_POSSIBLE_MAX_DDT_MB << 20); -#endif - - numbuckets = ddt.max_ddt_size / (sizeof (dedup_entry_t)); - - /* - * numbuckets must be a power of 2. Increase number to - * a power of 2 if necessary. - */ - if (!ISP2(numbuckets)) - numbuckets = 1ULL << high_order_bit(numbuckets); - - ddt.dedup_hash_array = calloc(numbuckets, sizeof (dedup_entry_t *)); - ddt.ddecache = umem_cache_create("dde", sizeof (dedup_entry_t), 0, - NULL, NULL, NULL, NULL, NULL, 0); - ddt.cur_ddt_size = numbuckets * sizeof (dedup_entry_t *); - ddt.numhashbits = high_order_bit(numbuckets) - 1; - ddt.ddt_full = B_FALSE; - - outfd = dda->outputfd; - ofp = fdopen(dda->inputfd, "r"); - while (ssread(drr, sizeof (*drr), ofp) != 0) { - - /* - * kernel filled in checksum, we are going to write same - * record, but need to regenerate checksum. - */ - if (drr->drr_type != DRR_BEGIN) { - bzero(&drr->drr_u.drr_checksum.drr_checksum, - sizeof (drr->drr_u.drr_checksum.drr_checksum)); - } - - switch (drr->drr_type) { - case DRR_BEGIN: - { - struct drr_begin *drrb = &drr->drr_u.drr_begin; - int fflags; - int sz = 0; - ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); - - ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC); - - /* set the DEDUP feature flag for this stream */ - fflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo); - fflags |= (DMU_BACKUP_FEATURE_DEDUP | - DMU_BACKUP_FEATURE_DEDUPPROPS); - DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags); - - if (drr->drr_payloadlen != 0) { - sz = drr->drr_payloadlen; - - if (sz > SPA_MAXBLOCKSIZE) { - buf = zfs_realloc(dda->dedup_hdl, buf, - SPA_MAXBLOCKSIZE, sz); - } - (void) ssread(buf, sz, ofp); - if (ferror(stdin)) - perror("fread"); - } - if (dump_record(drr, buf, sz, &stream_cksum, - outfd) != 0) - goto out; - break; - } - - case DRR_END: - { - struct drr_end *drre = &drr->drr_u.drr_end; - /* use the recalculated checksum */ - drre->drr_checksum = stream_cksum; - if (dump_record(drr, NULL, 0, &stream_cksum, - outfd) != 0) - goto out; - break; - } - - case DRR_OBJECT: - { - struct drr_object *drro = &drr->drr_u.drr_object; - if (drro->drr_bonuslen > 0) { - (void) ssread(buf, - DRR_OBJECT_PAYLOAD_SIZE(drro), ofp); - } - if (dump_record(drr, buf, DRR_OBJECT_PAYLOAD_SIZE(drro), - &stream_cksum, outfd) != 0) - goto out; - break; - } - - case DRR_SPILL: - { - struct drr_spill *drrs = &drr->drr_u.drr_spill; - (void) ssread(buf, DRR_SPILL_PAYLOAD_SIZE(drrs), ofp); - if (dump_record(drr, buf, DRR_SPILL_PAYLOAD_SIZE(drrs), - &stream_cksum, outfd) != 0) - goto out; - break; - } - - case DRR_FREEOBJECTS: - { - if (dump_record(drr, NULL, 0, &stream_cksum, - outfd) != 0) - goto out; - break; - } - - case DRR_WRITE: - { - struct drr_write *drrw = &drr->drr_u.drr_write; - dataref_t dataref; - uint64_t payload_size; - - payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); - (void) ssread(buf, payload_size, ofp); - - /* - * Use the existing checksum if it's dedup-capable, - * else calculate a SHA256 checksum for it. - */ - - if (ZIO_CHECKSUM_EQUAL(drrw->drr_key.ddk_cksum, - zero_cksum) || - !DRR_IS_DEDUP_CAPABLE(drrw->drr_flags)) { - SHA2_CTX ctx; - zio_cksum_t tmpsha256; - - SHA2Init(SHA256, &ctx); - SHA2Update(&ctx, buf, payload_size); - SHA2Final(&tmpsha256, &ctx); - - drrw->drr_key.ddk_cksum.zc_word[0] = - BE_64(tmpsha256.zc_word[0]); - drrw->drr_key.ddk_cksum.zc_word[1] = - BE_64(tmpsha256.zc_word[1]); - drrw->drr_key.ddk_cksum.zc_word[2] = - BE_64(tmpsha256.zc_word[2]); - drrw->drr_key.ddk_cksum.zc_word[3] = - BE_64(tmpsha256.zc_word[3]); - drrw->drr_checksumtype = ZIO_CHECKSUM_SHA256; - drrw->drr_flags |= DRR_CHECKSUM_DEDUP; - } - - dataref.ref_guid = drrw->drr_toguid; - dataref.ref_object = drrw->drr_object; - dataref.ref_offset = drrw->drr_offset; - - if (ddt_update(dda->dedup_hdl, &ddt, - &drrw->drr_key.ddk_cksum, drrw->drr_key.ddk_prop, - &dataref)) { - dmu_replay_record_t wbr_drr = {0}; - struct drr_write_byref *wbr_drrr = - &wbr_drr.drr_u.drr_write_byref; - - /* block already present in stream */ - wbr_drr.drr_type = DRR_WRITE_BYREF; - - wbr_drrr->drr_object = drrw->drr_object; - wbr_drrr->drr_offset = drrw->drr_offset; - wbr_drrr->drr_length = drrw->drr_logical_size; - wbr_drrr->drr_toguid = drrw->drr_toguid; - wbr_drrr->drr_refguid = dataref.ref_guid; - wbr_drrr->drr_refobject = - dataref.ref_object; - wbr_drrr->drr_refoffset = - dataref.ref_offset; - - wbr_drrr->drr_checksumtype = - drrw->drr_checksumtype; - wbr_drrr->drr_flags = drrw->drr_flags; - wbr_drrr->drr_key.ddk_cksum = - drrw->drr_key.ddk_cksum; - wbr_drrr->drr_key.ddk_prop = - drrw->drr_key.ddk_prop; - - if (dump_record(&wbr_drr, NULL, 0, - &stream_cksum, outfd) != 0) - goto out; - } else { - /* block not previously seen */ - if (dump_record(drr, buf, payload_size, - &stream_cksum, outfd) != 0) - goto out; - } - break; - } - - case DRR_WRITE_EMBEDDED: - { - struct drr_write_embedded *drrwe = - &drr->drr_u.drr_write_embedded; - (void) ssread(buf, - P2ROUNDUP((uint64_t)drrwe->drr_psize, 8), ofp); - if (dump_record(drr, buf, - P2ROUNDUP((uint64_t)drrwe->drr_psize, 8), - &stream_cksum, outfd) != 0) - goto out; - break; - } - - case DRR_FREE: - { - if (dump_record(drr, NULL, 0, &stream_cksum, - outfd) != 0) - goto out; - break; - } - - case DRR_OBJECT_RANGE: - { - if (dump_record(drr, NULL, 0, &stream_cksum, - outfd) != 0) - goto out; - break; - } - - default: - (void) fprintf(stderr, "INVALID record type 0x%x\n", - drr->drr_type); - /* should never happen, so assert */ - assert(B_FALSE); - } - } -out: - umem_cache_destroy(ddt.ddecache); - free(ddt.dedup_hash_array); - free(buf); - (void) fclose(ofp); - - return (NULL); -} - /* * Routines for dealing with the AVL tree of fs-nvlists */ @@ -2478,7 +2096,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap, int spa_version; pthread_t tid = 0; int pipefd[2]; - dedup_arg_t dda = { 0 }; int featureflags = 0; FILE *fout; @@ -2502,33 +2119,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap, if (flags->holds) featureflags |= DMU_BACKUP_FEATURE_HOLDS; - /* - * Start the dedup thread if this is a dedup stream. We do not bother - * doing this if this a raw send of an encrypted dataset with dedup off - * because normal encrypted blocks won't dedup. - */ - if (flags->dedup && !flags->dryrun && !(flags->raw && - zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) != ZIO_CRYPT_OFF && - zfs_prop_get_int(zhp, ZFS_PROP_DEDUP) == ZIO_CHECKSUM_OFF)) { - featureflags |= (DMU_BACKUP_FEATURE_DEDUP | - DMU_BACKUP_FEATURE_DEDUPPROPS); - if ((err = socketpair(AF_UNIX, SOCK_STREAM, 0, pipefd)) != 0) { - zfs_error_aux(zhp->zfs_hdl, strerror(errno)); - return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED, - errbuf)); - } - dda.outputfd = outfd; - dda.inputfd = pipefd[1]; - dda.dedup_hdl = zhp->zfs_hdl; - if ((err = pthread_create(&tid, NULL, cksummer, &dda)) != 0) { - (void) close(pipefd[0]); - (void) close(pipefd[1]); - zfs_error_aux(zhp->zfs_hdl, strerror(errno)); - return (zfs_error(zhp->zfs_hdl, - EZFS_THREADCREATEFAILED, errbuf)); - } - } - if (flags->replicate || flags->doall || flags->props || flags->holds || flags->backup) { char full_tosnap_name[ZFS_MAX_DATASET_NAME_LEN]; @@ -2706,34 +2296,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap, return (err); } -static int -get_dedup_fd(zfs_handle_t *zhp, dedup_arg_t *dda, int fd, pthread_t *tid, - int *outfd) -{ - int pipefd[2]; - char errbuf[1024]; - int err; - (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, - "warning: cannot send '%s'"), zhp->zfs_name); - if ((err = socketpair(AF_UNIX, SOCK_STREAM, 0, pipefd)) != 0) { - zfs_error_aux(zhp->zfs_hdl, strerror(errno)); - return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED, - errbuf)); - } - dda->outputfd = fd; - dda->inputfd = pipefd[1]; - dda->dedup_hdl = zhp->zfs_hdl; - if ((err = pthread_create(tid, NULL, cksummer, dda)) != 0) { - (void) close(pipefd[0]); - (void) close(pipefd[1]); - zfs_error_aux(zhp->zfs_hdl, strerror(err)); - return (zfs_error(zhp->zfs_hdl, EZFS_THREADCREATEFAILED, - errbuf)); - } - *outfd = pipefd[0]; - return (0); -} - zfs_handle_t * name_to_dir_handle(libzfs_handle_t *hdl, const char *snapname) { @@ -2819,9 +2381,8 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags, libzfs_handle_t *hdl = zhp->zfs_hdl; char *name = zhp->zfs_name; int orig_fd = fd; - pthread_t ddtid, ptid; + pthread_t ptid; progress_arg_t pa = { 0 }; - dedup_arg_t dda = { 0 }; char errbuf[1024]; (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, @@ -2914,16 +2475,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags, if (flags->dryrun) return (0); - /* - * If deduplication is requested, spawn a thread that will deduplicate - * the data coming out of the kernel. - */ - if (flags->dedup) { - err = get_dedup_fd(zhp, &dda, fd, &ddtid, &fd); - if (err != 0) - return (err); - } - /* * If progress reporting is requested, spawn a new thread to poll * ZFS_IOC_SEND_PROGRESS at a regular interval. @@ -2939,11 +2490,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags, send_progress_thread, &pa); if (err != 0) { zfs_error_aux(zhp->zfs_hdl, strerror(errno)); - if (flags->dedup) { - (void) pthread_cancel(ddtid); - (void) close(fd); - (void) pthread_join(ddtid, NULL); - } return (zfs_error(zhp->zfs_hdl, EZFS_THREADCREATEFAILED, errbuf)); } @@ -2966,12 +2512,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags, return (zfs_standard_error(hdl, error, errbuf)); } } - if (flags->dedup) { - if (err != 0) - (void) pthread_cancel(ddtid); - (void) close(fd); - (void) pthread_join(ddtid, NULL); - } if (flags->props || flags->holds || flags->backup) { /* Write the final end record. */ @@ -3965,8 +3505,7 @@ recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs, static int zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, recvflags_t *flags, dmu_replay_record_t *drr, zio_cksum_t *zc, - char **top_zfs, int cleanup_fd, uint64_t *action_handlep, - nvlist_t *cmdprops) + char **top_zfs, nvlist_t *cmdprops) { nvlist_t *stream_nv = NULL; avl_tree_t *stream_avl = NULL; @@ -4143,8 +3682,7 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, * recv_skip() and return 0). */ error = zfs_receive_impl(hdl, destname, NULL, flags, fd, - sendfs, stream_nv, stream_avl, top_zfs, cleanup_fd, - action_handlep, sendsnap, cmdprops); + sendfs, stream_nv, stream_avl, top_zfs, sendsnap, cmdprops); if (error == ENODATA) { error = 0; break; @@ -4530,8 +4068,8 @@ static int zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, const char *originsnap, recvflags_t *flags, dmu_replay_record_t *drr, dmu_replay_record_t *drr_noswap, const char *sendfs, nvlist_t *stream_nv, - avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd, - uint64_t *action_handlep, const char *finalsnap, nvlist_t *cmdprops) + avl_tree_t *stream_avl, char **top_zfs, + const char *finalsnap, nvlist_t *cmdprops) { time_t begin_time; int ioctl_err, ioctl_errno, err; @@ -4741,24 +4279,16 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, (void) printf("found clone origin %s\n", origin); } - if (!hdl->libzfs_dedup_warning_printed && - (DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) & + if ((DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) & DMU_BACKUP_FEATURE_DEDUP)) { (void) fprintf(stderr, - gettext("WARNING: This is a deduplicated send stream. " - "The ability to send and\n" - "receive deduplicated send streams is deprecated. " - "In the future, the\n" - "ability to receive a deduplicated send stream with " - "\"zfs receive\" will be\n" - "removed. However, in the future, a utility will be " - "provided to convert a\n" - "deduplicated send stream to a regular " - "(non-deduplicated) stream. This\n" - "future utility will require that the send stream be " - "located in a\n" - "seek-able file, rather than provided by a pipe.\n\n")); - hdl->libzfs_dedup_warning_printed = B_TRUE; + gettext("ERROR: \"zfs receive\" no longer supports " + "deduplicated send streams. Use\n" + "the \"zstream redup\" command to convert this stream " + "to a regular,\n" + "non-deduplicated stream.\n")); + err = zfs_error(hdl, EZFS_NOTSUP, errbuf); + goto out; } boolean_t resuming = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) & @@ -5103,8 +4633,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, err = ioctl_err = lzc_receive_with_cmdprops(destsnap, rcvprops, oxprops, wkeydata, wkeylen, origin, flags->force, flags->resumable, - raw, infd, drr_noswap, cleanup_fd, &read_bytes, &errflags, - action_handlep, &prop_errors); + raw, infd, drr_noswap, -1, &read_bytes, &errflags, + NULL, &prop_errors); ioctl_errno = ioctl_err; prop_errflags = errflags; @@ -5435,8 +4965,8 @@ zfs_receive_checkprops(libzfs_handle_t *hdl, nvlist_t *props, static int zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, const char *originsnap, recvflags_t *flags, int infd, const char *sendfs, - nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd, - uint64_t *action_handlep, const char *finalsnap, nvlist_t *cmdprops) + nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, + const char *finalsnap, nvlist_t *cmdprops) { int err; dmu_replay_record_t drr, drr_noswap; @@ -5546,12 +5076,12 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, } return (zfs_receive_one(hdl, infd, tosnap, originsnap, flags, &drr, &drr_noswap, sendfs, stream_nv, stream_avl, top_zfs, - cleanup_fd, action_handlep, finalsnap, cmdprops)); + finalsnap, cmdprops)); } else { assert(DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == DMU_COMPOUNDSTREAM); return (zfs_receive_package(hdl, infd, tosnap, flags, &drr, - &zcksum, top_zfs, cleanup_fd, action_handlep, cmdprops)); + &zcksum, top_zfs, cmdprops)); } } @@ -5568,8 +5098,6 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props, { char *top_zfs = NULL; int err; - int cleanup_fd; - uint64_t action_handle = 0; struct stat sb; char *originsnap = NULL; @@ -5595,13 +5123,8 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props, return (err); } - cleanup_fd = open(ZFS_DEV, O_RDWR); - VERIFY(cleanup_fd >= 0); - err = zfs_receive_impl(hdl, tosnap, originsnap, flags, infd, NULL, NULL, - stream_avl, &top_zfs, cleanup_fd, &action_handle, NULL, props); - - VERIFY(0 == close(cleanup_fd)); + stream_avl, &top_zfs, NULL, props); if (err == 0 && !flags->nomount && flags->domount && top_zfs) { zfs_handle_t *zhp = NULL; diff --git a/lib/libzfs_core/libzfs_core.c b/lib/libzfs_core/libzfs_core.c index 18143d36408..4e83b624b26 100644 --- a/lib/libzfs_core/libzfs_core.c +++ b/lib/libzfs_core/libzfs_core.c @@ -20,7 +20,7 @@ */ /* - * Copyright (c) 2012, 2018 by Delphix. All rights reserved. + * Copyright (c) 2012, 2020 by Delphix. All rights reserved. * Copyright (c) 2013 Steven Hartland. All rights reserved. * Copyright (c) 2017 Datto Inc. * Copyright 2017 RackTop Systems. @@ -783,9 +783,8 @@ static int recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops, uint8_t *wkeydata, uint_t wkeylen, const char *origin, boolean_t force, boolean_t resumable, boolean_t raw, int input_fd, - const dmu_replay_record_t *begin_record, int cleanup_fd, - uint64_t *read_bytes, uint64_t *errflags, uint64_t *action_handle, - nvlist_t **errors) + const dmu_replay_record_t *begin_record, uint64_t *read_bytes, + uint64_t *errflags, nvlist_t **errors) { dmu_replay_record_t drr; char fsname[MAXPATHLEN]; @@ -868,12 +867,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops, if (resumable) fnvlist_add_boolean(innvl, "resumable"); - if (cleanup_fd >= 0) - fnvlist_add_int32(innvl, "cleanup_fd", cleanup_fd); - - if (action_handle != NULL) - fnvlist_add_uint64(innvl, "action_handle", - *action_handle); error = lzc_ioctl(ZFS_IOC_RECV_NEW, fsname, innvl, &outnvl); @@ -885,10 +878,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops, error = nvlist_lookup_uint64(outnvl, "error_flags", errflags); - if (error == 0 && action_handle != NULL) - error = nvlist_lookup_uint64(outnvl, "action_handle", - action_handle); - if (error == 0 && errors != NULL) { nvlist_t *nvl; error = nvlist_lookup_nvlist(outnvl, "errors", &nvl); @@ -931,12 +920,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops, zc.zc_cleanup_fd = -1; zc.zc_action_handle = 0; - if (cleanup_fd >= 0) - zc.zc_cleanup_fd = cleanup_fd; - - if (action_handle != NULL) - zc.zc_action_handle = *action_handle; - zc.zc_nvlist_dst_size = 128 * 1024; zc.zc_nvlist_dst = (uint64_t)(uintptr_t) malloc(zc.zc_nvlist_dst_size); @@ -951,9 +934,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops, if (errflags != NULL) *errflags = zc.zc_obj; - if (action_handle != NULL) - *action_handle = zc.zc_action_handle; - if (errors != NULL) VERIFY0(nvlist_unpack( (void *)(uintptr_t)zc.zc_nvlist_dst, @@ -986,7 +966,7 @@ lzc_receive(const char *snapname, nvlist_t *props, const char *origin, boolean_t force, boolean_t raw, int fd) { return (recv_impl(snapname, props, NULL, NULL, 0, origin, force, - B_FALSE, raw, fd, NULL, -1, NULL, NULL, NULL, NULL)); + B_FALSE, raw, fd, NULL, NULL, NULL, NULL)); } /* @@ -1000,7 +980,7 @@ lzc_receive_resumable(const char *snapname, nvlist_t *props, const char *origin, boolean_t force, boolean_t raw, int fd) { return (recv_impl(snapname, props, NULL, NULL, 0, origin, force, - B_TRUE, raw, fd, NULL, -1, NULL, NULL, NULL, NULL)); + B_TRUE, raw, fd, NULL, NULL, NULL, NULL)); } /* @@ -1023,7 +1003,7 @@ lzc_receive_with_header(const char *snapname, nvlist_t *props, return (EINVAL); return (recv_impl(snapname, props, NULL, NULL, 0, origin, force, - resumable, raw, fd, begin_record, -1, NULL, NULL, NULL, NULL)); + resumable, raw, fd, begin_record, NULL, NULL, NULL)); } /* @@ -1039,9 +1019,7 @@ lzc_receive_with_header(const char *snapname, nvlist_t *props, * The 'errflags' value will contain zprop_errflags_t flags which are * used to describe any failures. * - * The 'action_handle' is used to pass the handle for this guid/ds mapping. - * It should be set to zero on first call and will contain an updated handle - * on success, it should be passed in subsequent calls. + * The 'action_handle' and 'cleanup_fd' are no longer used, and are ignored. * * The 'errors' nvlist contains an entry for each unapplied received * property. Callers are responsible for freeing this nvlist. @@ -1053,8 +1031,8 @@ int lzc_receive_one(const char *snapname, nvlist_t *props, nvlist_t **errors) { return (recv_impl(snapname, props, NULL, NULL, 0, origin, force, - resumable, raw, input_fd, begin_record, cleanup_fd, read_bytes, - errflags, action_handle, errors)); + resumable, raw, input_fd, begin_record, + read_bytes, errflags, errors)); } /* @@ -1073,8 +1051,8 @@ int lzc_receive_with_cmdprops(const char *snapname, nvlist_t *props, nvlist_t **errors) { return (recv_impl(snapname, props, cmdprops, wkeydata, wkeylen, origin, - force, resumable, raw, input_fd, begin_record, cleanup_fd, - read_bytes, errflags, action_handle, errors)); + force, resumable, raw, input_fd, begin_record, + read_bytes, errflags, errors)); } /* diff --git a/man/man8/zfs-receive.8 b/man/man8/zfs-receive.8 index a60a5c7bbb6..2d799a2da69 100644 --- a/man/man8/zfs-receive.8 +++ b/man/man8/zfs-receive.8 @@ -105,17 +105,11 @@ destroyed by using the .Nm zfs Cm destroy Fl d command. .Pp -Deduplicated send streams can be generated by using the -.Nm zfs Cm send Fl D +The ability to send and receive deduplicated send streams has been removed. +However, a deduplicated send stream created with older software can be converted +to a regular (non-deduplicated) stream by using the +.Nm zstream Cm redup command. -\fBThe ability to send and receive deduplicated send streams is deprecated.\fR -In the future, the ability to receive a deduplicated send stream with -.Nm zfs Cm receive -will be removed. -However, in the future, a utility will be provided to convert a -deduplicated send stream to a regular (non-deduplicated) stream. -This future utility will require that the send stream be located in a -seek-able file, rather than provided by a pipe. .Pp If .Fl o Em property Ns = Ns Ar value @@ -378,3 +372,4 @@ deleting its saved partially received state. .El .Sh SEE ALSO .Xr zfs-send 8 +.Xr zstream 8 diff --git a/man/man8/zfs-send.8 b/man/man8/zfs-send.8 index 1deedc214bb..c1b2134d118 100644 --- a/man/man8/zfs-send.8 +++ b/man/man8/zfs-send.8 @@ -86,21 +86,9 @@ The output can be redirected to a file or to a different system By default, a full stream is generated. .Bl -tag -width "-D" .It Fl D, -dedup -Generate a deduplicated stream. -\fBDeduplicated send is deprecated and will be removed in a future release.\fR -(In the future, the flag will be accepted but a regular, non-deduplicated -stream will be generated.) -Blocks which would have been sent multiple times in the send stream will only be -sent once. -The receiving system must also support this feature to receive a deduplicated -stream. -This flag can be used regardless of the dataset's -.Sy dedup -property, but performance will be much better if the filesystem uses a -dedup-capable checksum -.Po for example, -.Sy sha256 -.Pc . +Deduplicated send is no longer supported. +This flag is accepted for backwards compatibility, but a regular, +non-deduplicated stream will be generated. .It Fl I Ar snapshot Generate a stream package that sends all intermediary snapshots from the first snapshot to the second snapshot. diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index 6eb935720f4..a21ac8d74a4 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright (c) 2016, Nexenta Systems, Inc. All rights reserved. @@ -1560,56 +1560,6 @@ dmu_return_arcbuf(arc_buf_t *buf) arc_buf_destroy(buf, FTAG); } -void -dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset, - dmu_buf_t *handle, dmu_tx_t *tx) -{ - dmu_buf_t *dst_handle; - dmu_buf_impl_t *dstdb; - dmu_buf_impl_t *srcdb = (dmu_buf_impl_t *)handle; - dmu_object_type_t type; - arc_buf_t *abuf; - uint64_t datalen; - boolean_t byteorder; - uint8_t salt[ZIO_DATA_SALT_LEN]; - uint8_t iv[ZIO_DATA_IV_LEN]; - uint8_t mac[ZIO_DATA_MAC_LEN]; - - ASSERT3P(srcdb->db_buf, !=, NULL); - - /* hold the db that we want to write to */ - VERIFY0(dmu_buf_hold(os, object, offset, FTAG, &dst_handle, - DMU_READ_NO_DECRYPT)); - dstdb = (dmu_buf_impl_t *)dst_handle; - datalen = arc_buf_size(srcdb->db_buf); - - DB_DNODE_ENTER(dstdb); - type = DB_DNODE(dstdb)->dn_type; - DB_DNODE_EXIT(dstdb); - - /* allocated an arc buffer that matches the type of srcdb->db_buf */ - if (arc_is_encrypted(srcdb->db_buf)) { - arc_get_raw_params(srcdb->db_buf, &byteorder, salt, iv, mac); - abuf = arc_loan_raw_buf(os->os_spa, dmu_objset_id(os), - byteorder, salt, iv, mac, type, - datalen, arc_buf_lsize(srcdb->db_buf), - arc_get_compression(srcdb->db_buf)); - } else { - /* we won't get a compressed db back from dmu_buf_hold() */ - ASSERT3U(arc_get_compression(srcdb->db_buf), - ==, ZIO_COMPRESS_OFF); - abuf = arc_loan_buf(os->os_spa, - DMU_OT_IS_METADATA(type), datalen); - } - - ASSERT3U(datalen, ==, arc_buf_size(abuf)); - - /* copy the data to the new buffer and assign it to the dstdb */ - bcopy(srcdb->db_buf->b_data, abuf->b_data, datalen); - dbuf_assign_arcbuf(dstdb, abuf, tx); - dmu_buf_rele(dst_handle, FTAG); -} - /* * When possible directly assign passed loaned arc buffer to a dbuf. * If this is not possible copy the contents of passed arc buf via diff --git a/module/zfs/dmu_recv.c b/module/zfs/dmu_recv.c index 9c82cecc6a1..817c85b935b 100644 --- a/module/zfs/dmu_recv.c +++ b/module/zfs/dmu_recv.c @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright (c) 2014, Joyent, Inc. All rights reserved. * Copyright 2014 HybridCluster. All rights reserved. * Copyright (c) 2018, loli10K . All rights reserved. @@ -101,8 +101,6 @@ struct receive_writer_arg { boolean_t done; int err; - /* A map from guid to dataset to help handle dedup'd streams. */ - avl_tree_t *guid_to_ds_map; boolean_t resumable; boolean_t raw; /* DMU_BACKUP_FEATURE_RAW set */ boolean_t spill; /* DRR_FLAG_SPILL_BLOCK set */ @@ -123,13 +121,6 @@ struct receive_writer_arg { boolean_t or_byteorder; }; -typedef struct guid_map_entry { - uint64_t guid; - boolean_t raw; - objset_t *gme_os; - avl_node_t avlnode; -} guid_map_entry_t; - typedef struct dmu_recv_begin_arg { const char *drba_origin; dmu_recv_cookie_t *drba_cookie; @@ -1212,38 +1203,6 @@ dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin, return (err); } -static int -guid_compare(const void *arg1, const void *arg2) -{ - const guid_map_entry_t *gmep1 = (const guid_map_entry_t *)arg1; - const guid_map_entry_t *gmep2 = (const guid_map_entry_t *)arg2; - - return (TREE_CMP(gmep1->guid, gmep2->guid)); -} - -static void -free_guid_map_onexit(void *arg) -{ - avl_tree_t *ca = arg; - void *cookie = NULL; - guid_map_entry_t *gmep; - - while ((gmep = avl_destroy_nodes(ca, &cookie)) != NULL) { - ds_hold_flags_t dsflags = DS_HOLD_FLAG_DECRYPT; - - if (gmep->raw) { - gmep->gme_os->os_raw_receive = B_FALSE; - dsflags &= ~DS_HOLD_FLAG_DECRYPT; - } - - dsl_dataset_disown(gmep->gme_os->os_dsl_dataset, - dsflags, gmep); - kmem_free(gmep, sizeof (guid_map_entry_t)); - } - avl_destroy(ca); - kmem_free(ca, sizeof (avl_tree_t)); -} - static int receive_read(dmu_recv_cookie_t *drc, int len, void *buf) { @@ -1845,81 +1804,6 @@ receive_process_write_record(struct receive_writer_arg *rwa, return (EAGAIN); } -/* - * Handle a DRR_WRITE_BYREF record. This record is used in dedup'ed - * streams to refer to a copy of the data that is already on the - * system because it came in earlier in the stream. This function - * finds the earlier copy of the data, and uses that copy instead of - * data from the stream to fulfill this write. - */ -noinline static int -receive_write_byref(struct receive_writer_arg *rwa, - struct drr_write_byref *drrwbr) -{ - dmu_tx_t *tx; - int err; - guid_map_entry_t gmesrch; - guid_map_entry_t *gmep; - avl_index_t where; - objset_t *ref_os = NULL; - int flags = DMU_READ_PREFETCH; - dmu_buf_t *dbp; - - if (drrwbr->drr_offset + drrwbr->drr_length < drrwbr->drr_offset) - return (SET_ERROR(EINVAL)); - - /* - * If the GUID of the referenced dataset is different from the - * GUID of the target dataset, find the referenced dataset. - */ - if (drrwbr->drr_toguid != drrwbr->drr_refguid) { - gmesrch.guid = drrwbr->drr_refguid; - if ((gmep = avl_find(rwa->guid_to_ds_map, &gmesrch, - &where)) == NULL) { - return (SET_ERROR(EINVAL)); - } - ref_os = gmep->gme_os; - } else { - ref_os = rwa->os; - } - - if (drrwbr->drr_object > rwa->max_object) - rwa->max_object = drrwbr->drr_object; - - if (rwa->raw) - flags |= DMU_READ_NO_DECRYPT; - - /* may return either a regular db or an encrypted one */ - err = dmu_buf_hold(ref_os, drrwbr->drr_refobject, - drrwbr->drr_refoffset, FTAG, &dbp, flags); - if (err != 0) - return (err); - - tx = dmu_tx_create(rwa->os); - - dmu_tx_hold_write(tx, drrwbr->drr_object, - drrwbr->drr_offset, drrwbr->drr_length); - err = dmu_tx_assign(tx, TXG_WAIT); - if (err != 0) { - dmu_tx_abort(tx); - return (err); - } - - if (rwa->raw) { - dmu_copy_from_buf(rwa->os, drrwbr->drr_object, - drrwbr->drr_offset, dbp, tx); - } else { - dmu_write(rwa->os, drrwbr->drr_object, - drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx); - } - dmu_buf_rele(dbp, FTAG); - - /* See comment in restore_write. */ - save_resume_state(rwa, drrwbr->drr_object, drrwbr->drr_offset, tx); - dmu_tx_commit(tx); - return (0); -} - static int receive_write_embedded(struct receive_writer_arg *rwa, struct drr_write_embedded *drrwe, void *data) @@ -2607,13 +2491,6 @@ receive_process_record(struct receive_writer_arg *rwa, } break; } - case DRR_WRITE_BYREF: - { - struct drr_write_byref *drrwbr = - &rrd->header.drr_u.drr_write_byref; - err = receive_write_byref(rwa, drrwbr); - break; - } case DRR_WRITE_EMBEDDED: { struct drr_write_embedded *drrwe = @@ -2754,8 +2631,7 @@ resume_check(dmu_recv_cookie_t *drc, nvlist_t *begin_nvl) * NB: callers *must* call dmu_recv_end() if this succeeds. */ int -dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd, - uint64_t *action_handlep, offset_t *voffp) +dmu_recv_stream(dmu_recv_cookie_t *drc, offset_t *voffp) { int err = 0; struct receive_writer_arg *rwa = kmem_zalloc(sizeof (*rwa), KM_SLEEP); @@ -2779,41 +2655,6 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd, ASSERT0(drc->drc_os->os_encrypted && (drc->drc_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)); - /* if this stream is dedup'ed, set up the avl tree for guid mapping */ - if (drc->drc_featureflags & DMU_BACKUP_FEATURE_DEDUP) { - minor_t minor; - - if (cleanup_fd == -1) { - err = SET_ERROR(EBADF); - goto out; - } - err = zfs_onexit_fd_hold(cleanup_fd, &minor); - if (err != 0) { - cleanup_fd = -1; - goto out; - } - - if (*action_handlep == 0) { - rwa->guid_to_ds_map = - kmem_alloc(sizeof (avl_tree_t), KM_SLEEP); - avl_create(rwa->guid_to_ds_map, guid_compare, - sizeof (guid_map_entry_t), - offsetof(guid_map_entry_t, avlnode)); - err = zfs_onexit_add_cb(minor, - free_guid_map_onexit, rwa->guid_to_ds_map, - action_handlep); - if (err != 0) - goto out; - } else { - err = zfs_onexit_cb_data(minor, *action_handlep, - (void **)&rwa->guid_to_ds_map); - if (err != 0) - goto out; - } - - drc->drc_guid_to_ds_map = rwa->guid_to_ds_map; - } - /* handle DSL encryption key payload */ if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) { nvlist_t *keynvl = NULL; @@ -2980,9 +2821,6 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd, kmem_free(rwa, sizeof (*rwa)); nvlist_free(drc->drc_begin_nvl); - if ((drc->drc_featureflags & DMU_BACKUP_FEATURE_DEDUP) && - (cleanup_fd != -1)) - zfs_onexit_fd_rele(cleanup_fd); if (err != 0) { /* @@ -3083,6 +2921,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) dmu_recv_cookie_t *drc = arg; dsl_pool_t *dp = dmu_tx_pool(tx); boolean_t encrypted = drc->drc_ds->ds_dir->dd_crypto_obj != 0; + uint64_t newsnapobj; spa_history_log_internal_ds(drc->drc_ds, "finish receiving", tx, "snap=%s", drc->drc_tosnap); @@ -3148,7 +2987,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) dsl_dataset_phys(origin_head)->ds_flags &= ~DS_FLAG_INCONSISTENT; - drc->drc_newsnapobj = + newsnapobj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj; dsl_dataset_rele(origin_head, FTAG); @@ -3188,7 +3027,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) (void) zap_remove(dp->dp_meta_objset, ds->ds_object, DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, tx); } - drc->drc_newsnapobj = + newsnapobj = dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj; } @@ -3203,9 +3042,9 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) * value. */ if (drc->drc_raw && drc->drc_ivset_guid != 0) { - dmu_object_zapify(dp->dp_meta_objset, drc->drc_newsnapobj, + dmu_object_zapify(dp->dp_meta_objset, newsnapobj, DMU_OT_DSL_DATASET, tx); - VERIFY0(zap_update(dp->dp_meta_objset, drc->drc_newsnapobj, + VERIFY0(zap_update(dp->dp_meta_objset, newsnapobj, DS_FIELD_IVSET_GUID, sizeof (uint64_t), 1, &drc->drc_ivset_guid, tx)); } @@ -3226,54 +3065,6 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) drc->drc_ds = NULL; } -static int -add_ds_to_guidmap(const char *name, avl_tree_t *guid_map, uint64_t snapobj, - boolean_t raw) -{ - dsl_pool_t *dp; - dsl_dataset_t *snapds; - guid_map_entry_t *gmep; - objset_t *os; - ds_hold_flags_t dsflags = (raw) ? 0 : DS_HOLD_FLAG_DECRYPT; - int err; - - ASSERT(guid_map != NULL); - - err = dsl_pool_hold(name, FTAG, &dp); - if (err != 0) - return (err); - gmep = kmem_alloc(sizeof (*gmep), KM_SLEEP); - err = dsl_dataset_own_obj(dp, snapobj, dsflags, gmep, &snapds); - - if (err == 0) { - err = dmu_objset_from_ds(snapds, &os); - if (err != 0) { - dsl_dataset_disown(snapds, dsflags, FTAG); - dsl_pool_rele(dp, FTAG); - kmem_free(gmep, sizeof (*gmep)); - return (err); - } - /* - * If this is a deduplicated raw send stream, we need - * to make sure that we can still read raw blocks from - * earlier datasets in the stream, so we set the - * os_raw_receive flag now. - */ - if (raw) - os->os_raw_receive = B_TRUE; - - gmep->raw = raw; - gmep->guid = dsl_dataset_phys(snapds)->ds_guid; - gmep->gme_os = os; - avl_add(guid_map, gmep); - } else { - kmem_free(gmep, sizeof (*gmep)); - } - - dsl_pool_rele(dp, FTAG); - return (err); -} - static int dmu_recv_end_modified_blocks = 3; static int @@ -3325,12 +3116,6 @@ dmu_recv_end(dmu_recv_cookie_t *drc, void *owner) drc->drc_tofs, drc->drc_tosnap); zvol_create_minor(snapname); kmem_strfree(snapname); - - if (drc->drc_guid_to_ds_map != NULL) { - (void) add_ds_to_guidmap(drc->drc_tofs, - drc->drc_guid_to_ds_map, - drc->drc_newsnapobj, drc->drc_raw); - } } return (error); } diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c index 5c53eb63779..2104bef714c 100644 --- a/module/zfs/zfs_ioctl.c +++ b/module/zfs/zfs_ioctl.c @@ -27,7 +27,7 @@ * Copyright (c) 2014, 2016 Joyent, Inc. All rights reserved. * Copyright 2016 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2014, Joyent, Inc. All rights reserved. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2020 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013 Steven Hartland. All rights reserved. * Copyright (c) 2014 Integros [integros.com] @@ -4765,9 +4765,9 @@ static boolean_t zfs_ioc_recv_inject_err; static int zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, nvlist_t *recvprops, nvlist_t *localprops, nvlist_t *hidden_args, boolean_t force, - boolean_t resumable, int input_fd, dmu_replay_record_t *begin_record, - int cleanup_fd, uint64_t *read_bytes, uint64_t *errflags, - uint64_t *action_handle, nvlist_t **errors) + boolean_t resumable, int input_fd, + dmu_replay_record_t *begin_record, uint64_t *read_bytes, + uint64_t *errflags, nvlist_t **errors) { dmu_recv_cookie_t drc; int error = 0; @@ -4896,7 +4896,7 @@ zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, nvlist_t *recvprops, nvlist_free(xprops); } - error = dmu_recv_stream(&drc, cleanup_fd, action_handle, &off); + error = dmu_recv_stream(&drc, &off); if (error == 0) { zfsvfs_t *zfsvfs = NULL; @@ -5088,13 +5088,10 @@ zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, nvlist_t *recvprops, * zc_cookie file descriptor to recv from * zc_begin_record the BEGIN record of the stream (not byteswapped) * zc_guid force flag - * zc_cleanup_fd cleanup-on-exit file descriptor - * zc_action_handle handle for this guid/ds mapping (or zero on first call) * * outputs: * zc_cookie number of bytes read * zc_obj zprop_errflags_t - * zc_action_handle handle for this guid/ds mapping * zc_nvlist_dst{_size} error for each unapplied received property */ static int @@ -5137,8 +5134,7 @@ zfs_ioc_recv(zfs_cmd_t *zc) error = zfs_ioc_recv_impl(tofs, tosnap, origin, recvdprops, localprops, NULL, zc->zc_guid, B_FALSE, zc->zc_cookie, &begin_record, - zc->zc_cleanup_fd, &zc->zc_cookie, &zc->zc_obj, - &zc->zc_action_handle, &errors); + &zc->zc_cookie, &zc->zc_obj, &errors); nvlist_free(recvdprops); nvlist_free(localprops); @@ -5171,15 +5167,14 @@ zfs_ioc_recv(zfs_cmd_t *zc) * "input_fd" -> file descriptor to read stream from (int32) * (optional) "force" -> force flag (value ignored) * (optional) "resumable" -> resumable flag (value ignored) - * (optional) "cleanup_fd" -> cleanup-on-exit file descriptor - * (optional) "action_handle" -> handle for this guid/ds mapping + * (optional) "cleanup_fd" -> unused + * (optional) "action_handle" -> unused * (optional) "hidden_args" -> { "wkeydata" -> value } * } * * outnvl: { * "read_bytes" -> number of bytes read * "error_flags" -> zprop_errflags_t - * "action_handle" -> handle for this guid/ds mapping * "errors" -> error for each unapplied received property (nvlist) * } */ @@ -5212,11 +5207,9 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl) char tofs[ZFS_MAX_DATASET_NAME_LEN]; boolean_t force; boolean_t resumable; - uint64_t action_handle = 0; uint64_t read_bytes = 0; uint64_t errflags = 0; int input_fd = -1; - int cleanup_fd = -1; int error; snapname = fnvlist_lookup_string(innvl, "snapname"); @@ -5244,14 +5237,6 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl) force = nvlist_exists(innvl, "force"); resumable = nvlist_exists(innvl, "resumable"); - error = nvlist_lookup_int32(innvl, "cleanup_fd", &cleanup_fd); - if (error && error != ENOENT) - return (error); - - error = nvlist_lookup_uint64(innvl, "action_handle", &action_handle); - if (error && error != ENOENT) - return (error); - /* we still use "props" here for backwards compatibility */ error = nvlist_lookup_nvlist(innvl, "props", &recvprops); if (error && error != ENOENT) @@ -5266,12 +5251,11 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl) return (error); error = zfs_ioc_recv_impl(tofs, tosnap, origin, recvprops, localprops, - hidden_args, force, resumable, input_fd, begin_record, cleanup_fd, - &read_bytes, &errflags, &action_handle, &errors); + hidden_args, force, resumable, input_fd, begin_record, + &read_bytes, &errflags, &errors); fnvlist_add_uint64(outnvl, "read_bytes", read_bytes); fnvlist_add_uint64(outnvl, "error_flags", errflags); - fnvlist_add_uint64(outnvl, "action_handle", action_handle); fnvlist_add_nvlist(outnvl, "errors", errors); nvlist_free(errors); diff --git a/module/zfs/zfs_onexit.c b/module/zfs/zfs_onexit.c index bf86446d4fa..2a1332e715e 100644 --- a/module/zfs/zfs_onexit.c +++ b/module/zfs/zfs_onexit.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2013 by Delphix. All rights reserved. + * Copyright (c) 2013, 2020 by Delphix. All rights reserved. */ #include @@ -171,80 +171,3 @@ zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data, return (0); } - -static zfs_onexit_action_node_t * -zfs_onexit_find_cb(zfs_onexit_t *zo, uint64_t action_handle) -{ - zfs_onexit_action_node_t *match; - zfs_onexit_action_node_t *ap; - list_t *l; - - ASSERT(MUTEX_HELD(&zo->zo_lock)); - - match = (zfs_onexit_action_node_t *)(uintptr_t)action_handle; - l = &zo->zo_actions; - for (ap = list_head(l); ap != NULL; ap = list_next(l, ap)) { - if (match == ap) - break; - } - return (ap); -} - -/* - * Delete the callback, triggering it first if 'fire' is set. - */ -int -zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire) -{ - zfs_onexit_t *zo; - zfs_onexit_action_node_t *ap; - int error; - - error = zfs_onexit_minor_to_state(minor, &zo); - if (error) - return (error); - - mutex_enter(&zo->zo_lock); - ap = zfs_onexit_find_cb(zo, action_handle); - if (ap != NULL) { - list_remove(&zo->zo_actions, ap); - mutex_exit(&zo->zo_lock); - if (fire) - ap->za_func(ap->za_data); - kmem_free(ap, sizeof (zfs_onexit_action_node_t)); - } else { - mutex_exit(&zo->zo_lock); - error = SET_ERROR(ENOENT); - } - - return (error); -} - -/* - * Return the data associated with this callback. This allows consumers - * of the cleanup-on-exit interfaces to stash kernel data across system - * calls, knowing that it will be cleaned up if the calling process exits. - */ -int -zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data) -{ - zfs_onexit_t *zo; - zfs_onexit_action_node_t *ap; - int error; - - *data = NULL; - - error = zfs_onexit_minor_to_state(minor, &zo); - if (error) - return (error); - - mutex_enter(&zo->zo_lock); - ap = zfs_onexit_find_cb(zo, action_handle); - if (ap != NULL) - *data = ap->za_data; - else - error = SET_ERROR(ENOENT); - mutex_exit(&zo->zo_lock); - - return (error); -} diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index b75be778bdc..6f83dcd8ba4 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -766,22 +766,22 @@ tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos'] tags = ['functional', 'rootpool'] [tests/functional/rsend] -tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', - 'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', - 'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', - 'rsend_013_pos', 'rsend_014_pos', 'rsend_016_neg', - 'rsend_019_pos', 'rsend_020_pos', +tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos', + 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos', + 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', 'rsend_009_pos', + 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', 'rsend_013_pos', + 'rsend_014_pos', 'rsend_016_neg', 'rsend_019_pos', 'rsend_020_pos', 'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos', 'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props', 'send-c_incremental', 'send-c_volume', 'send-c_zstreamdump', 'send-c_lz4_disabled', 'send-c_recv_lz4_disabled', - 'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD', + 'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize', 'send-c_recv_dedup', 'send_encrypted_hierarchy', 'send_encrypted_props', 'send_encrypted_truncated_files', 'send_freeobjects', 'send_realloc_files', 'send_realloc_encrypted_files', 'send_spill_block', 'send_holds', - 'send_hole_birth', 'send_mixed_raw', 'send-wDR_encrypted_zvol', + 'send_hole_birth', 'send_mixed_raw', 'send-wR_encrypted_zvol', 'send_partial_dataset'] tags = ['functional', 'rsend'] diff --git a/tests/zfs-tests/tests/functional/rsend/Makefile.am b/tests/zfs-tests/tests/functional/rsend/Makefile.am index 8e16f884766..7728a6481da 100644 --- a/tests/zfs-tests/tests/functional/rsend/Makefile.am +++ b/tests/zfs-tests/tests/functional/rsend/Makefile.am @@ -2,6 +2,8 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/rsend dist_pkgdata_SCRIPTS = \ setup.ksh \ cleanup.ksh \ + recv_dedup.ksh \ + recv_dedup_encrypted_zvol.ksh \ rsend_001_pos.ksh \ rsend_002_pos.ksh \ rsend_003_pos.ksh \ @@ -25,7 +27,6 @@ dist_pkgdata_SCRIPTS = \ send_encrypted_hierarchy.ksh \ send_encrypted_props.ksh \ send_encrypted_truncated_files.ksh \ - send-cD.ksh \ send-c_embedded_blocks.ksh \ send-c_incremental.ksh \ send-c_lz4_disabled.ksh \ @@ -49,8 +50,12 @@ dist_pkgdata_SCRIPTS = \ send_holds.ksh \ send_hole_birth.ksh \ send_mixed_raw.ksh \ - send-wDR_encrypted_zvol.ksh + send-wR_encrypted_zvol.ksh dist_pkgdata_DATA = \ + dedup.zsend.bz2 \ + dedup_encrypted_zvol.bz2 \ + dedup_encrypted_zvol.zsend.bz2 \ + fs.tar.gz \ rsend.cfg \ rsend.kshlib diff --git a/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 b/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..585e14852662476fab6d4f1ccaab5d5c1b18f37c GIT binary patch literal 18561 zcmafZXH=6<^d=xki3J1_u^}i5m_k!gq#INarMFNdf_8eY>j$^c=>;>6OG_ucnDraY56hgfWN}Y42+k<6&TJjFfjQsFj%pi2|6`R z9x-`)w;|Sk8vPa#%=zw>-tW{JaINi`$ght+Xj9Ckx?cEy>$Z~MeteQ=KaKWO>T&1U zry?_ru_ALxKU6+Qsf#h(6xZJTAZ67k)gdl%?cCYNA?_#qXiaI>c(E=It8*v$ce9cs zBQ8s+r>Q6_SiS!sc1`7)wiMPnu$x=`8u(o5+%>6KCXeHRgCB_I-y|%Wzm(nztULbx z8bg{y60MurTJl`{Gf~eji=@XH=dK3FW-h2?PFP27$fISScB|g&kvOffo|Hyg4o(m8 zi1$#THEu}UOG#U|e#-Y$gZ4A?wDyCL7b0^WNhxz4@vl{EZmBENI=|AS?9{()oO>bi z)kbPLsH6T?Qs%6z$~nI%dC>@~5ZR~P>ZzB~XF2USJFFO_x@jE@jL8;F$;Z_h8teI_ ztSqFCDI8Z(2lK=jX0dVdY`oQRmG^$izqj|hR~{A6 zl4!BA3Y^~+-^O3F@@u^G)$09a@#EK282P?+bJM;*z7~6pO{#vKo4KPyP)mTxf+{P; zpe}jeTK0MxTl(9F>s@wFRpPm+vM1StHtuEqme-(l`2^2d-_9)hWG6xviT@B{Ct+hT zpklQl#dj<115v_5ySwLBTK&d@%o&MzyK5F6nuRydmAi!bRx0J}@MPP{)&*V+?BxkX zW-}D=UC5ebxl8;$7xIAdG)HxU_Sk96bk^?`(w6O(cQswc?9$RFeHf#^W5cM<*%Y6M zb&**u)6$i;O3szO+pRf5Loo4i=s!nxZlX?C|lIh>?Q9xJ~r%=uO3 ztx_Rr^W5~$t2M)-*1!pixWu=HmU@`y`ei*Yoy8MB96wt}SK>U3cqG6U5H{b&<`Xi@ z^Ohqv<|NzWNA=WOFFAXPEe2zC{EV|~M8BSYf4@#eLhP%MuCDFZYw)PcFyT6>vCMuh z;X1VT-=SLXvW14->?HEt_1V7s{{8cFxluk@uO8Q|(ps)Ih8pnSm!5)*2*if{P}~nh z1U~%k!gcwbm6SK6r)rNrYbe6!44T#Q+q_Ro)L z&-X8MHP6b*{dKq)&GGvW%dKjOv~X|DPoh8XREld~S)J(ro(*H0f-CtK3A;SdaDD?p zFDbiu`ext!{aVaX&Dlm!K(|vdC4I|9__H69Sx-qJLgDJ^W6I+jvd2dUCxepuC&wF` zu3Eeij?cSPm=hQl_&!YVVUwnL0k8g5o7zr#k&xvmqXJK{s-W@8U9NDU4$^e2upqCP zGA}1HbHC#8@t~9im*t$i5{+a9y$rlY?X@ri$9mQ;0(K0Rr281t|7ZoaIr9*y64x;F zF-=Yu!yw_z&;rdnvF8X~m?$1E9O+K6eS;W9n1Q-7-0GA^FDWo|A^a>UV77;KQkGtp47lQg`IrSmw#(6J%Ad`Wt#TUF5bVq5sH3mDT!pVRfmv39Ja^^I7or>TUOoln>@)UC+h)-G^wDHF) zE<9q`Tsc=7=~QsPr(T57LTiE%yNlOsTqulU$>>wPCnfCnTrQHHe|R{id``vpK{Vun znVvxl(y&RPTdBj6Q8~JeHKfPN@`G^UryQrmhDWon`defJ^v|}B*xdAzCC(%>#7CRF zarw)9s)(Ju7~TEg;dc+wh|!bNEH+e~6*zAbbW?&{8b4Q{Q3x5pI?z%1w)!vh)u4r! z`b%roi+PQ)3mY4V3fh=->)6BW?$(1IW`+v$%L?UensZ^7vsiynwyyc*iwP0^m@YpH zL*HU%b|!mUUFfNrDG=SQ4Jvv1qdHgkfH%b0P<8vcoB>++>!-^(si!T__3$bSkTWQH- zpO2|pxBV7XJ2O=={)stU!ZDODODGtt(-*I#6MFUJYow#PQ?WGbqSGmn2I3fDB(v>R0>SG zlYVcG=YiN<;`2P$Q%(aiXg(ERR@+7Xra_tXb+^!+5gAK?v3o2c!y+NL*bMn%iu*^t z$$RKWbon0*x|gsyYe{d~J)Ua*y>Z%bR`y-V)}_^wtuM#Mz0k7y@>R+`$8wd);~t0g zVOIl9;#Gp5N7$)Z&toKVXx4i9tT)AztR%AK) z?5gvMf!VDVhwtfsi<3v!WM%Hv6AZjsy)uI5Hbb zf4Yvc=<(O$#Mt4xueG`Qew^QTn66pgZtvfnxU4SSgu8Nk{Aj+{#3lClZ#Z9=)kK%F zd=Y={Ne1i}h~KhT`XC0+Qv<7ee3lqu%5v$az;<iM`tLWiz_)J)fO|zu zggpMN_$c66?Em~UxaGg+9V%lN&oqZ{D*JumbYo4lF#g~Bhb;f^HKHu{EHzRe5<_(V zPe^(pdf=7+EQf$!E>niH75CKleIJoGTMq^jm~GBCTx^EFvz*ZJZK^HZw>ukj>7nA{ zq3^L@gATPNrQfgrNl{P^@n4^KRkX_FUg#V)f~y=1=dSf5i^$INu1y4m=2*$voqzi_ zP?_w{X*Jqi!gNw8T6-Ek5hDC`z?^3n45`M5RdDYAPLPN*j|UH#L&S#17etLb&a8<> zftgEP_6PsU{r)#fF{S?#Irx$i$G>l;iD_RI2?s0S|L=320~Vh4ko*5Wd4ZP#(-?bj z^DCQw(DcEV{Jv0Y>z&cUtu5`7iQmd65-3B9((j~yqPtDK-H=?ZJF7Ps6BPK8izVa} z@-8q-hVlQ{a&=|UU?}d1JbRzHZ`)aie|y*2Q<_2l>=|dPmk|No7c>K^#TX%2=vdUX z6c2~04Er-HvZ_JC#C|?~q@Gc9u}A**UDbZC zqd42bxIL9ZJiPP7kF9wgEUe;Fk2Q}Rd_Y_IY$rcozwtQR^Yxzo<##GcfBu-(pYCVA z<^SQWLu%jWT(!WRh}uUYY2s(>o{p0{0xJ?eFse2hZm{cYMk~h-9-8&kjW|}nSl|egJbL@~IMcD44abbc<#o@woEOPwR-OKc znpRk}vmVulp&1uODlvD;ADrp!eyRR8?^QxrO_TkecizR_AGF0L#8M-Fu-}@yl>V7< zo&KOPlsj~r*jFDN zA7CkW`tV0#amG0OPQ|>|pP{dhUuHa~FtUGp<2JRJwQcO4gjP7bnD42Vu%xm4wYX7y zu;oZyPP5hN%&CO?wbU7-GeLqLOU}CM-u(O925uc=o~?uTONGpsf8MRJ^7yq^IcuGCt8QzkQkvnId2p%*QZ(bj(%G|OFzNEK)%0yC$Gvf?XRLHZ z>f(eY{`9jSp#{5_^1Qp3IcNXc=J(%~TMjz-lx1nYQi4;>iE1D*w$2tv9sOAz=iuSs zVY=&nddM%h;!UpK+e!v{a`4xAVZXcm46-?jtC#Vy#|9KfsV7UA?>rQYk3Hpcx+8!z zBg&t|KrT5}@3$hxyy?g@Gk!6UQTryt=_UpVF}Wwi07R%t5Rabv4f$b51%U^wE|iH& znT{2?%U+OrFI{f{C z)m(q`yS-RB;N}HB$n)tK*0l`soWzK`TzkupS2qWa%cZs0f4(h`tXseIQ;3zLrhD(! zI=Q;7=irMz zHssIdMSRm?@^O5+Nbi*AX_=19mfrLQ!oGX$SJVN1JN4j>FB}77=`TYK!i4s@Y|R`i zp@fK4IEn>N94;r+`O*=hwck*Csry^yi88;zku zv^Ow@;V6f4aI~!{HC2xgL7CAx=qSO!0ybuhgNG|)0_HGX*^>r3>u_C9Dt^2QrZsPh zA6sf^t+AxujZsRY6FbdqO;Zca9ZjTB!A>w1Jjod)F-@z;sD@Je%lfO)gGhWw1)^or z2o>(dLUJ@|>8J^HC8W5aBnFZ6CJ&gndw&H1_iGN)U1`#S>n|uV9e_oe+(tkNXj(xT z9MSR~2_d*!IwR?Y?z1qG>(Vk3Pa_i-Xe1PN-h`z`7f&lf zs%3h@NKUkZ>K0)zp#dGAqha#|$haw_+LT^VhI&|mq!atg5G{i9MhF6S6(!M%XlbBl z5vmPLT2gEfFd^cM0VbN-}djtpxX95Yq8An6uqewOE4z$y44IV980SA$}pyU!%%XMAh zoz8H?azOzZf?Y$V;#X0ET>B0UT!IV}LDVuNt zwmG}Cpen1Hkd9kNM()5e9Zj}SqNreVM^ifeN6Kp`jH*ihB1nhKTvuPDb{CMRGSmpg$7bL_|KhxYxtO`x5AV6@NFqQ>8J`sYRG{9)( zsYP_+Ow_(%(51HzZ^T;_RKdW|)B={)5SxgWj!IbE>ll_+BNWRd1}0SKrDjzHOL-k5 zL~;Y8Uc|(R+nNfOn5c~#7&~>8m_q)*wB|6;qLtA^Fnd!*2q?_=VO2CN#nD8~MLz@; zN@%V0h+)~2SVv5(!%-6J{Uw-~IJ=?g{>o8(`^e#Pl)p0yJ6RQ73CnchZbssCU%_yl zu1H8@8H#1isB(R)jWBN5t;=yfAFD*W|KK@H1K z@^Gp1#tw{vr2fm))Q%DZ0~0meeAajcD$c{I8it)Wz(8TuMhG-<9s>(Ee@zXyH?{1q zhC=DInsdZqlCHTc0!qWcxV(kxa3+M<0+4lRl5JLhq3)E4U>~;A5!`_OFBo)_qX`6( z*n;jTLO{{@kUR)v$`DG8q|BIF7J5X(of?5x2cUXj zEM16+4LB;f2n?&x+%v0^+E0dzgRViz#x{cD94|vsX$2(2E|j`}?;!sx5!{>!0i;Vy zBgm~zb6gfktU3oU2rIA_R^@OccD%$GCT>**C!p(1A=t@+Y68g-1H*2^s;^EnNF)n zJaMuzCdC=Vja?NTHB?Llhx{c`;lr>rgFIKI-D_kM?s01#Jt%_FG4GMf;%gVQF%oGmdd5AqZ`{Q>K>qoK<9@3ld*VngJQwY9dtV3`dIO66$O* zF!JGS_LIwJKf+&4^SK+p(9oiEsbSs#8m_##TBJBX&epH zT1UWmz#2htG$TT5L1j#w8`2w63JX|*(uYAVt;4kX$pm8$nARYHrcS^%mJwRP+l3x6 z0U!pJIU{6hxT7H&Pb)D+*V!ObCykKQvW#*9tsFssKq4B+YP=?d)*AE_hDuyE6`V7` z#9-Hv)F~549nKVwAFn_{5_>SPG=iqNy(xX148e`)exn(q3;!)>3eFfx8!jjZeHK*K zuYgXEIV2sAr^5=J3Ao{M(2ZlsEsd2hp?~R2dhHHku{RN<^n;E#ZyduiZz@Qk8OLCU zLH%fr*%1msxw^&FxKQDx>MdagwgB8Piy7C9J*zB{*-wYog{B=LN~54i{7( z2=-UQeDLcqrFC!}N^2}iZxr2MX%|-<@T=VS4uAKN-Qni1$4~9X|C8Jm? z#U#eyy57&_tVCbWJF~}cLQ_6+A3X4oI3M^}UcZh#!(r%wfy4SQhBp7<_zCOt=KN06 zzGLTm@E`sfErnN^zW5l(s?j>>$BZcb-mjLJ_xMV7>tWx@$m>_m6P){aK^D~{=CR8* zIX=2xbKg$fdnX>L{ZU(ake1$~HK3Myr$;U%HWi*}Yes=mI|}}#;U){oij1ACgdM0N zAt^4P0np0kvKE|?_~9aCYQ2e7bqo3%C}+^BV&cp}K~giyk@Px8r8V0>jNDO1!`HmW zMXm26dCxHZ$30}2GphEgMByQkA05mYLC^P)52+fTs(zW+IDg>n%J%Hz`OLN{oe;Ck zIIWK-zM+1gej;EKV8=4Y8Kax53d*N(ii)v!MRQ<=Yp(s$2*W*e}E zk{AW`ZwE_p0F7%3lxd2O9|_ffghb4neFddPLJh%it0eZ`;K+H>(r8X z7U$@~qk*5p&sX}L3}pB>k)lm1S<#+++Zc??g+H>vEy1m5*9Wz3)l3=P8bu z&9~{_-{U$TTHDZZOFZCtvqzhJXnNv7b$a_F7fmwT@36t2M*roH9KKg@o8XI;l*BW& zJxC0^b}WTV6FkYq#_C;OELw4W7`ZkKIa$({J#Z`8FCu#&NLG-i5np9;b$$6~^-+G6 zZ~e)mcfrV@x+}{*F+Wx{6mEWb6;~WuT=JaH=P(g*t+3`l0THJ=cKt!I0w>1waucsH zh0?g?)R6IAcY+_3Qw&gz!8XsN(r>e~Ukz6`vVS`F-e2(HH^P<6jW8}IAt&t{>>l)V z*~`kcq$9z?0mhK$kuA=9b!QftPHXZ@qe(HO-N7--RvUM(XGe*b`i}RJDn`T_bCfb3 zUwL;15m>0DTBagy_4)3zfjB?Ty&+%SP04Y{8{Y8EtC1Fhk7<4#NgnsA1I6Cd$2(rU z{}b)DA&e&|vk&{V9=|NN^T9(q5$P^bXSKw{hf!%dVQpl-wZ#_T@#7~@W2pWp`Fh0C z&8i(F&z&Fxb7}I!{02h4|9>SBMokKS`$6s|uaBs=hk92A^9~s2A3rVsZZt0Y?zN&< zMPk&sQcm*m48|l_j^D|h_=W$d4KA*qzQ5r@A7p&fu*j^j{&{uSHtEeKJVeplv4=DG zO0AF|jL^w)mHV@y1oQ8B z!daiX?t1^aw1z;1#vo0(VBZ|eb$sG7m9zcVtqzZtLb2U_2l=$tr*iz>@yjySJ=(IP zm{a%N`un$exMCXP78@$V3Py}anH2}NL$F&UJG;aq&B!^oHPpwwMV&ru1G91RT<&12 zuXLaI4rj-I_xd2t*U$e*_`&h(;nBzE@0UtU$RiDEzt0ti-eSF=)zr@!a83O2a1itT z5XGm}*Di8Gw$qnpIGZLGrAE|`?xZw?PJPZj+ zPyMBi1g!HWC9nOg6DTVymFOxOxjw!8Nw&1Ue_7to@2tP%UHR_w$4rV949?MYt`W5a zH4NE{1`dEdHDt%YLU0O%e?A+lJ9t{wo3( zgxwO@KO=vl`FcF%oHIY{TGOkYWm_w3N9${BZ!owcEeciP#)%Ii*bHz2G!f3l+0q&Z zrmN+Mgu?C{Y{Y(P_TbkR_d*5VGFKG#e@P9u3b-sgi0oimSN^;;L#ND?;^qUKy|^RC zdVjf;{J^C#lz)1j!f_?s_X0EZ)t7MA4K_v|zi(P1zqVzU5?uT)d5QU6hxs|2;^{Vo z=qHALHQ)a^?Mw3FC#gv_<~lVNaK+?tSnVpGQ%a`*YDNeT`Vu( zBNgyW0t}+~(1oUzDj3DZkk~WGiOdm?Gz~x5%Z+-oisOHuOndlPaw~Uxt@zXsMWegH z>0Ws5HtF_-PdCQbg?`^4d(#dWh`#e73T)q`D|}(StXapt?8+29HB_9Vn}1O}@;JJG zr<3bBEC0xdOytDf$v-U}1~ zoJVS2*$dMA*`M^XZw4#4zOjc~VVzjJeW3rVb8AG*`DoHCtXlrAhI`}9Ym0BvA7^NI z2;cf~?D8}2k@(>L78OGo*cZYVa@O>7ZaiPeUl+sitV;VR!`P%XwJ#6ZpQz_I*}t`7itw8A9$WuF;5*T0zGz9EhT&R5c%p3~gPofA#XS!o5W2FY8^AT4Q3YlPkf74_7?xE1qU?ZiASs|HP1M%$QzmNe6{tb#!L;}*G#;RQ z69Nm#nX=AZ=mt9n1dUF+Z9H)k6`zd*a><@e9+l)7vw|nt8W>Xt2a%K|^msLtok$t4 z5zXBMR4IV621kvkL2_>hHtEJ)natWk1$&HmPf&&ns(jlK6Z2VAJs7qt`!-7HsIGy~ zU&211OYG4sv@n7L6uwRm0%&7B8cKl7B4G8^#9=aVEy|SQC4pj@GNt#$R#Hpx(S(;1 zh9oaSoijkX>s|}0=H9o3R*?7^V_27|Wr}k{M@k>Y#3RTU4M}Zp$P}VaS&9V>x z-unvnWo46G;b=U*Ia?%Ek5|edtJB>4*%zkmm^d$jrZZA4QPa^BPjW@3E*h8+3Vn*k zYwEmN>b!^)svb2Y2d?`XR$pT|gNgZ`ISUh-Hxq36ynzb#9Dz)j8Iw}(!qRNXYMQU= z%wIiKS_#Q%hEw%Fn?UT6wZgbqQ#cQcv}Fm|H`p&N)IV5aE|NKUcbj+oeg(1H*1V|O8%UnFtcmek!MBx5_`F}JcI2=`v zfC9Qj0RtjH=DV?g%0jS1n)Vn+V|FSUA7E8MumYM1WJL`(rEB?srOwB%42{i(K(M_~ zf;VAzK}e5^-wUOu+y%V421=YP)Fs&)rP0HkQ5D8O_o0laiZF&?l6fm@#!_|0da?t1-|OJ53^pO813ofq|hYrB!{aD;p4@ESiBK z6upW}CH5B-6vaWcVgY^4056dc)m#FX9lD{!tR9<<4hO_C4vc2T2-UxDFL>m+Nj}Qj z%W?z2wV(*36ol8{8su+GM0P|jp%8lm5-|uzsXK_)OwsF{k*QQMa7SK}3n7FXf+g4> zi8d(p5<}79GJgUzq}BUmud_Nm+^ANk zPW6W(rie$mHrhPjn2()HjbfZs3)n9L2-bt(TaRctylNH=gn?ZUOuvC?NPY`~5HJTs zjWWc#sgpBmPdHBmkW5XNjH+k?j?!rf%gLfN79pu)rb+0IvNfO<1Ol!cOD6cF;zz=r zps;|stgh^DCIr>o7MX1XECdb+HEsh4P5~6kqA8xZM4Sf_$LnZnJ1o=OF{>HraZog* ziOtSK1Azh}xQId1{-Jvh>UgcGS&ci&D0|ZfO(Ux{)^ORILLkFMfIO31ApPW);WEHa zjS%4`5Ht}8Kt1Q+ghE?mD0PVBrG_0s11+KXpR@xDm@y#;F2S5lCbkHjx*_>2k!DN- zx+GUb3lMaSVFo||A(;aKOViv10Slna8=$l(oj`?%7_Nc>rQ|w}GF*XdLsyVnYRk2# z`t{|=Lbo00T`0A^BnwOxamF;2xCvBCN~depQTox3CZE%Peb7rYrgWgqr3dUfn-G{L z3v@lY|G?C`Yvoes&2qR)VV?~}NvLtxD6_t3YHf!Jp~RHBGl)Xvnc{m<*g2!U90V9_ zDG|&bc9-7dMi@6UCaVES3J`8soE;2H0Rk@2R9Yao$r9l$MBru`C>t~lFr@yXz*VGL z2MLZKpl3`=`l|`(T2({~ZnB^febA-VY1dI5DHF4>-!g-2&I`%E>wRYx3F$>d0l}vB z9#D$)KbwZz8kjnP2?8_QX@Q~QNmfO0R7^b?XsFS!W|ZL0Z?BdVm+^8!BN^ll{gB-o z4WuP>jC-YM4H}5Z3%*_$n5ABnF-)k*OD)BN`WFb0c>D@Va2ra`D5%o3gla<|lcxBh z8&d|?=uLJ__F2OvI-OoEmU^&4H$)4q0znYM!15!F2oci}yV&oaoe}7rtjX#L(6Zf7 z*kLjx)0wgY`!X<{VpR$wS>Eig5u$rqRYGC9U<)BxBxv{7bXt`>{Y{7^?Pk&nqM=aH z8kUUe1Q>1;WugDQQo9!=v88M7&E=wNRmxsy?z@B_bkwBki#wyfBf~v_h)X1L=)FGM zetWO{lDg^YpZK*5)v?oWFCyCj6WvK zLq&^0c^bl4GRooHQD84n%|U>lAn4wTGLF&efl`}*fY=i=Ls-%^w*MM2Sp}^_--3y1 zGE_32DbW&ng%hbao2hi3CojDVyTGq(TQ2$x^QCOh8&YkuOZ`HQF_$s~En?9G(vBdk zL53?~vJg$@8BA6U4Md7>vM@b>p#Lr=60nmcW{x%6C0UdqO%G8LlD+_Tqj^*Guq>!os~-j+hI|NrXzBPIC)0SxvA;RQ#vLO6Up>LnNzVDJ!eG_5cUC zqNK~9)M(LSi0FIJc8TlAP8Z59q#hGJN*xIPEWoDtXc>jpBV^j4OU#Pk+$LC1^E)vn zS=taG{S<8W?jd9zI8nbycCdg2bhs-@aE}+1Z^R~Y#RgS4lhqLi#ufz?1&$0X#i|T6 zajG6v#{kpm>$MuSLAcsOoXp%51j6Bfr2cw1^H)Al!5{kZH`B+ zl^{U9;hMAo{QL(c%`Kr&h+s2Z7n{A3-v$;|cn6w6R!hJ{YrbqO`{zRZ#7-9S?oift zVl4Ftg&yd610aD*X1XBI^}xuXb^zTRuLoem>c3=*MdJI(`D@j%r?i-W?HF-y`HX|7 zQ9Ic=%^IsI{@ZSPkwZ4~q869^jkQEr1j7ySkp^Z4rRvXughyiDP#Os_tpMnlH|b5_kbji<~o5Z^6>C z8f$N1s2w%?nN5zySvA{0P;B!4ZAXRm-Gz;TiX-6+vc}N4EkgPMf9xRvVAm{kaz~cv z9()}4<@>oHpVzlK=C*&Xnn0KK&MaTUHR+ZZm<_~GfZ=0=3dUxSo8<%ELV)1)&_xI; zvA+a##imjylo}4uNG}x#ieSosvq_Bvc%{%b7EBNj>n(tHz*2&t7Mk1qgS}?E;3@*d z!`6r(NTKR)Z5#cQ=V2XY{@&pmDRzgp@X9hXiatZpC|tc!GjE|K3|n^_o-r5; zmbbM6spbI(abPKcI5=XU%U`yL6beAB4C=0?rb^D4R&ZL6qQc-FEio8=8(Fm~tXz z5JC8?3oI8FP*;MJWgH@@t3LL{*7z3#YKEu&BsYD0I&D?QIwBI_=M^Q$TZ-OjStOtf zvB2S~fWr}Vy^LCVG?XA*P^gQKIBNi$H3Ay{G|#{|`hI~3;}hAZ=WlcyYCi+2t9LD5 z$?n*g)2O8mSvyEC6}Z*rzUM#zbeOqgu-Q{O`zt<*_Qs+$hgBg_ZgALrJzs^a1=wRG z1h*>(%&;XwozG)TjM6*+sH@mRuiC2>!`XAdv3?c{!0_WUjU-CDjocTXEPqM=TH_Ql_q=@~Nh@EKKHX1);_#`B^#W+Sd|1m+V=57y+CBPd;87~5M z6=lYxB~YJbuYwF{NI?ROWzp1f0Xn!>v>Eg35N^u69TU?|5G#g6H0cuSB)*)PF%zWh zM8on3>-bLHO)OhN?5k4f)cl5KvSdEKPmF-&>E$D(-K8XggoPLuC}i=Oqug} z@iU;V-2p|tb+7jF_2JLfSXjrv-q_Q=N&GEwHR|Q`kw0=Zo=ftl;HYgVB+~~ErEhbY z`*2xR=z=!Yy>*6$0mb!=t-$QF8di)V+IWE!*$zJmG!9_b}7Y4!x~23P+&vpqAPuIxPS$kTSsh zwkp!K1+Bfss>HyEpb4xtxUPlF++XHPD-Tn7LTq%1i38smxWj1nxRd&EJ|$$xA(u~1 z?bRZbQezod#qodU-Z~m!p$SThfDJ+u>^#{JLCXJdUn7*#L2u#=`4d^BzX)*`<^)a< z2p;^ZsEh#$2yqS*voCSrYz&)g(X=WG97V2}J9Fm}H-K$olOPI=(pf^pGA65tP;YPz zs_6(2)9MB&8A6>^W^;C?g%O~8>9bxfBy$%ejE{H6j0u>DGp0HQMu;A;j$!418&LqY z1-lBPzS5k5S#Qh-LU*B~X_jmYM|KFjx}~E+{cqovDxl&h!Nu!Bzr@`M^Sd#5o`BSu z+vw(M0<#^xh>3oQ9WE%znm1i>gA-N>ka{3mWV*~580Z4fhfdX-gjT65w;F73ZFD>Q z3J~s`i4S(SYL1TXx*m1D6LEKvVbN6RQ4CziOU*;NEt^r=g|wDQNrb5NgUc>|IQsK-Xmz@8Y`nki@=IO zQh@jE4z|n^LjY$h5HRe@IOrpF-bBERNsd`Fn3*anzQ3GM28+a?XAJ(C#_)gJH}nvq znm3FlQkrz@%zfA$0T!VZ6m{avendx^sQnCRvOxmm;(L1RqXTdnQ$up0)NP?@=s!LW zeB*zlvZn1&Wpt-_El@uz8**mb08|ytz!(Y(cYvmIy^hvyE!15Fo~^lMotK^G(2}Mr z0TM6)M}YbQP8Vj;7}k!&j_6LBlw>uBLWE0U;Z8QFg#*Buu{{q=pag2);hRm!rrVO6 z){L2=g{kGdQIu(+$98(DL3G7{IN}LVH8`Mq<#%g zutqYS+(nZAW@ZW}%n=rs)nTL2^dqr~eL*1ixgkVL%skn*!fg@-2a&oM7<(TE0=5}m zfEO|yptMdGq#HM`)4S;4dx?qQ98iIf*r_2+A72QTs*mloH-_2cn{2^pjEPQb-6k7U zg&|Db2@fnex`UgpPyXgu-cUyclDe}x;(Gw_&+yXT)-)sq7qUB~o#I9K1BKm$HI^el z?E*#!2YUw^A8rRbA5HJ5Naq&@M;V4IfWhvf2c?;%?1IBBwkEuw71v_Nfm@C{Dn0x- zVMew$E%&6An1s8+LyX{`n*C~WQlM0Da78sYqcW=ug`2^ktFs{i9l8KX?tt|2=*oVy zXadfJpqPBmJL98PkZKu8knT+PG?-Nt`rC}j1YqLW8%Ley+Tdu#OnCudM^jm=+_@5P zxRxGN6b>hLVp-#yQA*8v^n+TVLZE-*@w8Id6qE`uImEpR{io}n{;>qI~-U7n2Ufx?!kB<~tNBxDivczl!hFgTXbMNrL;`D|F39SelGzYugp{+OE5aB-$X zN6mmaI1YAjG>0lNA?^(#!KM@j*3;Gy-+V(DNMUrmX=>4m>ImBf{Sj%KeKM5KB+Q;11EC&A#9H~NYZ+7|~SBWXPLqop&azcITr zw?E2#CuLqGrDj~!T{)$9ILSM z^I9z0QQOdk_G}a z<^R&AJrf+acF8WQpW_%84x4$jrP5L!pB@?&dR>E=b&r!2e&O_=3se5`foaVeVQwPc zGyDnPlS^Q8%2gG~-%Z6gjHM&Mi61yPEmw&^mtdk{bC49fCvHeV%4`M+EM=3USs1`d zbyj5vYMoC>WjDIN)EQ9)9Z^fMH;y@z{eieoYy#>90^MI?_A7d9r79Yr4nuIL!&(>L zWbO_p?DDrAw#54cK83^gE%j2LHUMN!*?IWx>7#SM%uHPGU(NC75mT!_YH!}DOqNad zlfIRMq<#{(>0RT^AnnQP0Pkz38#Co?UQEcp4&!@Y^wa4T;y(G{)8~1$8`nOMu)Mrw zbWPSLCbc)e?_1Q=O-WTYnfK-7xR-7#)e3i$Tuxyg(=Qc!isFx*Xbhm2>|Z_GTZ8(% z#$cuJf1&v(C1NE|z6pBx)8Yb_JHIHL&t(ZInow(sv2Ny202Q<*k?@wIUqyVffe4bRQ- z&Pg#O{=xec|+(cJFcN9cT5x zk!bsmo6P50+oz+4s6`6JgE>oR2etTe*^29Dp^vh6NP?RhPylJRi-@AV;NI!$Rj!2=>S%DLX#q;7Ba z0Ct!c8?=LXtJvAdPY(XYrm>K`c~%i_PO;by23&;QE)l~Um{FnhI-RQ;## zPMEt>m>b_i_gQNn_(zW;#_CYc-XmA|&nSNKi?^_`frG)sg15O}%UGWLj%`)an39h- zb4SE>gs9tGbsiW0aN^#R!KX4j9dP)jx&)#ki|$j?CO7v{Ti{bdws}y^@TF#!^S2G% zJo!U+Gvlk}Y-&yp%l3MC>fV%dCGh>%%m3ocu>`A(lVdV8i}R+i zuTJA7UH$#Ws|O?VKC6$W`MF0stgdX}oCdDd1vGU3xyCS~rdeS9w=~Ey*}-S<{jpf= zl~*T?F6PEOvYs1TYt@a{julJ$k9o4Om|I}=ef0KK+bhZk# zC{O0^_DGi#RbOm2_Y^rm)+}1=3K>?%)3iUg>7=77(+g`!i&FisM{Ylwx0QkGO|T3y z=GB|Cb@_2F+%^1<%?vV)jfl7vLcJd3{pWc>)N)6Q_Hf0iKcmQZ*M0Kjw<;p^d@i=N zRrr1s9*?=`xnR^tZQNKHB{m1Fcyt`79}`>;S>`_3S&r(K}r#iP@0 zhCT(9AA-JB_ZVIpyhZxHZhI}FbcFFp9W%ucui4bG>t|%PE4mEO*Z357wN;U) z9z>^*{ER++loR~{iR@w5T%XvE8)4Bec-ndD2J?Dyq<;?U2S``m?G7t~`S%L*cief` zuO6Ne|BFT2$IJg&h_{TC)p3*oiga>TXBy;V|{Ev;?s-L`m?*U_AOaZ`3Lo1UQ*`K+ni zo-4)yQV%nwNYWs7!zOo&_VXw333!@+GsEurXDz`8EQ(<%e zj5$N?l=LTN3e)vBbzYT9s%aNf-!~k}KVm-CmUJ9%`p47I{5_kI3^9;eYTW{V$a(Go zs{5=^VO5k=MAW%y!wo-jgJ3IMw*5&Yfzar~%pbBS#3RdeZ*PGK^L`O$NxP%hbl@U- zWusk^@igJ%&fe{5{SQl6j=ncvV%;vC6)U|EZ+DXd6}u4?dU0!!cB>I`GxS5NtICNe zWNC0azHH^cl;`OJSNs_q&MLm4%4!}|CI6nZCAu}}Y^;8M#~Jd8^sd}|Lj`w1+}YPQ z>|JM~Uz3+{>z`vSN51KmWsQeVcbgslWNb^VIm5rx{|+rP{G>44K+E^eB<0T+Zgmy6 zEN13dE?GR6_>R#HIPVu;{Zw()y`NP2$Y;s*`;Xb}_v7oH2iI!Zf3*z!MdcskXZ-Ok1(mVjh6^&7#?tMPTs6&Pg9AxI$;|tf_v4G+iCtWaambo*7)|V z(--fbe`>O<{B7h2?r4zT&My;E%w}fmUl!orvU+bg+J$!1k-<@G?yYe?EVosVla&xzQxn+ET`_ueOBZcstdfZUC z_*4wQB6dA`%! zVRVebG_|c&)+VoyHR`MJQy<-eoFi8i8doBmtk zf5~;x$?Yb`U90={D<1!ew#bm1k`nmY2}Gls#5otwgN@qs;Pr!(Z&KdlLc@QV$;4U+ z+O-CJtWv4<3mO-P~_OGM?c@2l4D^NZ!r%riw`Py zzjfC-`1}J~?eAXLldJXUT6t}5Qfy9^(oeUpI?2>ID*5kqV5SI=Pm35j2RNBkduzc}dclgUlk`SN1s;@PW@hp)#dU&QTWloDQE zp58hfk~{qgQ~1fYu176c#=GR_A(rXc-#TK2#C?X{|263ci?%mz-?nuTN5HA_*+H@I zzf4tl^VdA!N0_p~qwm%6-+tZxy~Xs*!#@uce;!C^IQzMNuk?zLU8TP==AFJ9JN@>* z9l~!9s@t0i$|gN^)w|7aRy}VlubJ#OFlb|69pr%NF^5x#ICB zNxH@>BV=EHkKX-MpzUg4er9~u6X_Gn_}*17`&>|#dH53N{1lZnZgT1jSC16(-BpOK z<@@|Axpnq=)$a@(`H#;VZ74~;H}Ph&NzS)-lUB68eY!3+ktJC?KivH2itMM`qd%93 zFS`}~cbnJVoe*S` zw{yL@cXie>fx15b%@uR+TnY+ywQXCbzP#aosMWeJ%iEu1PkH9j_%ZjMyP!+b`ugdC zzrVhBe=?^gqWa8L*Vb*nPp(b=Y|Or;aQh4K58HeGto{`9T2AF1n|IF<+v2=q-;1`f z9QqkAt82|u#$UF3rEAjLEi)?Eb)RV&exEOVaZ?G`7U=}p?)}lPs^hW_zpXp7y&z+y z&7xz>1t+x5tV+?&`NcW+-0O=!tCJr+x<9W))iTCBYk9)cEV%^rm`(maDrb2}yjE_A zE#3B5^4Y@U`j_$!)SrLw=%?|{KO$dGzpCP&-@Q;^@unp+*9c_GA5B(yCnfp%D(A&d zdmA{kWew$(-#t|Q*WI4?RX*g4=Zb^=I}U~{u(kR*!D!cB_RHFa=dRqGwQa_?7Dqu9 z_kawe8F`z1SQ#d6;+y$C>|Jus$CtO9Prg~ra&hOU-wAfRs;3G0YKlJpu=~%1C%r1B z@$xr*o6kJ=e!^Fl5AQ$Q_8q?Pafjvp-><&U4csg!C}`&5wx?qVf)rs zwz|JvuqW%^va;{V`uS39q9wxf-x!+4R>?nVy?+0f{jBR*VqcyrOkbkUvHGGw{X|h- h@zYqqYA003l)c|`yK literal 0 HcmV?d00001 diff --git a/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 b/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..73a5742fc302098db888e30753fbfb599614cb94 GIT binary patch literal 1482112 zcmeFYXHb+;ur{co1c?HYgOZV)hKwRe96&^oEHE&nA?J)lfgwxIl9fC#QirZpVvmV(jtn6TzZDaZ&u#i zyYChJ_rIcTZkBtj+9CJwL+*W`xOY$O-v3CC|3~(}n|t(M8Nhq?Um^Z)>HkIH|I0-E zzp^=={}s^xBbfWI{u49zpZzDw|4CKwKS}iQM@kx@h^0QG#3Hpj1!kI~Xzr2u#eh<&$ z3bg2pN4^Zt+!rOGnyT0nRu6e#evj)v+aX8yec^jl{~6+;hB8;mhnLij4BXKL33|4p zK}lub{mSaCbY4g%Lv(a?MhY9Kep2ynzog8g7ZIP*vG=7{{qAS?^&~%(+HPB}tYTMQ z?ERg%ZlhevKc8$-TGS_6;ujI}N?P*r&m)VpUd4R>KO=F!;6FV1=h6R_G;Z$c9hs!v z*ZhzciI}C#D|hjnGp0)UaX*Uq(d0@{MTm<)!Mj`C?}#j1=@UU>^_t2pfiHmR_4m~! z(N3I5`8vIAd}fJ}vj&LhR~>X-V`xn-dyqq@n*#R1(I1u z@9@)Ad|9;oK@+HcR8IYjWKb-xb38K}{W8$BC%?|`lc6BLKFwIRBn|XY+qdtu21S6Z z$x`xg#`>Nf*OJf?Zw51J#}C>^IWyT#XTupqGsG57PoACV7IEP7M)DMm^pMb#YURB% z!$6*G5{>pA73Kt6kNM=5CZxYxO|bhr`}jr8UH_x^+Ke2^5psO(}$t9Zd|b%`n$0=oS{*aS~0sI z{)n#;G3k^~{!(NUJ#L|BE__UhQL<${_gCJ;2{JnolgLZ_F}6CZm$R!lDq z%YJ&c=IYl}&w2i4sWw-8ZaVTgaP|22NFuu`J_Q-@vP)b6^7+^Or~=ORdjZdF@WUvd zNw442D}MDmigG@f)_oPnC|}mi>u^ zsx%0%bS=Y~VL3d5Qgl3D2qW}+w0!jzinWn1+SEd2zuYs5OQTM{&xU6kTJ}?rRAaD z?|m@@{t+VhgnL7SqpqhOT@hCt7SZ~_>`Ikv-pocAm&|=B#g-{)w7Hb}ix9LJs%Q?M zkk~bfe-k@Ts55A3e3``;MBwt4T>S3qH@plVQi88TDxvp%l;G~$Wb((KAmy@s19q!2 zdM@u@xb{5GW3aW$4FC$dW_Er%i+)Lq~fh8fd15A0f?whFYVbMS>_J(l@j)VP?9mRhstA8nD8FFGi>fLrFkJ`a7lS6$LwdmAdX1u zx?25E^y2``@pW0RzQ3SjyC3bc>vO9+h_IDsyu<JzWh5)Safo%hpUFICWX*2nv413ZUEWKCX z8eG9b{xu*o$vfVl!W$Zxo@;kHGWN2h>?H9If!R%*D$S^vTA%ij5o^UF<9uyU*oEfH zCohQ~+?`BZ1ffsXkT$U@zq2o#m)jzxNNq&7&8B)XxPiuwOazbpN>tvTElHLA4E$Dv zWssyi9q{3fwL55^*=}T2YqOi_=&O1mHq;Bsia%cYdcL?pw|QGW-?`uJhsSVWcHefg z{w(73CPDZQjK|;>cvG6&EfJhen&k0Tf{bT#{M7JRxL3hpI8`=G4P(>kf3U4EcGOME2o0CKxz$tCy<~;XjJ#Wz80QRkGHirWGBB{Z^ku(>2#`@syB$H zXRpE56-=~FcR`$D{vis!ooDw*9ID(StN%DK_eLXVup*2q@@F>&x~6dIhF$7gvAAY= z`)f3v%a(yf-EWfAINi^!dOUx;PyfWpw=q#BGCIXVf-h-uH0Pga%vs zuu@I6<_4KHz8tx`((|Rifb3B|2#N#=eX_ts<6(ai>p~gsj90Yjoqh;Sm7wqjD6@X` zG3!+gKvMYiNNlD>6?9E{@Q=r^|Ei_7?ru|2%*i6C7oshZRjt+`kq*J>6cMnK5bm+g zfL|wo;BvULaG3`&7pAh#!Qt|xV#F$+Tc}pm*XlmKr_6r%djw8!GOqEt%tHFxw;@Icl+}UMf&v?L z){x*b6Dpb4Mqoic$(a*r`mT^*LJePC-WyF>sQTLEahluYGxIQ-(8Uc+?udYNZm&f@ zB}=rK3o<_wYcgZKwRSCmxp&rZQR#^|jYJ128&p$PEDSem#m#ppg8(;s{2=BC~r`Pm?n&j2PL+mcEv`M;*F!gD3CLG_jJYR1+y=R}%*8qwCoW?v(8` zyaa?e8Cd|?NqOgK**AZxg?R})$}I30h57qJtHQ)ShQC@pC9+hRz3oif(NhEBu?rMi zhhkPM@HC0#-GCJtQj)#cCF4@xH&e^fdX>6apuRwU>lA8?f|}Wlj;+#(IdGu_rX)Oq z(xc%clc;7K{f#dC`3yqobHePKZw^0I zxyqi>wZJjMnz@5UKWS2!UtE^{8h$qU2~}R#;CX91k+fyYRnHvGQVr@JrU{(Q`NP&rdu{ zC#zfZDc05OZe5(Bb8|1BO^q7IxC5{LG}^FjB}nu7LF(GYzO9cZ=#{;gIM}Ew!>Q-S zQiwXvh65s*{*3#(x(a31h&KhDFA*;|kH0^V(62v>W2_a1RPMY4gm#NHaZEESK^-QJ zFKYElyT!{a-x;!h0KKu^t5u>|WPFkpq#76fq_@-y3S)PzUK#1Y=l?~N$n~b3PK8=A zcVtAs`a^9Be)ms0ss>K{H#@c7I3VN5xH~^&sAX;m$H4d6!Et?xrd0fI-WX6w2+}M) z6ux3gOyQu!3qml@QNi!u(Bg0QWPyTZv>)!vUVPa#WcIT#VO+hk4-Zr$V+R1 zw9L7T{VWF}=^DkzCK~ZeXwoRSH&0mk z2?u*z#nwr>KKxqCRBCXpwbqd*%hf{!PI9(-ieCwB@=af4OM5c7z(cy!x+sHY2pRpm~+HcZ!fd! zAnxS!HR(3us}s8HuH#?pq?+)D$3WUb0>bI3Y2cOolnb9X-QAL4h4V!|S?|ip(6nCM zS&qB1kB)k5n@!NiOr>&99SN+Y3S3vy;g!R|@ld0U`G-%1yEX?KWAwB^`*iLEym8P0 zxw*%~z%~R2E+?-Y3tS{!e8CSEAC>YN{FUFM66E+QqRpFZV)-W6^^q1zT=e@&Y{fAT zMi|?8I{IEqBu%Ua7}h{b(L<(`4Tg+&G>IizGmI^=3+^s5MObb3Cs4VB&OcM9`;1Sn z_~4mwDfRKvj$)4{5ooJ3uD;}VPGmO4TL2pCEo_$g z(}TR5BmE7UXJu>xfr>r zwcSj2uDEs>wpfn{?Pm+!8B_3L**Ct?b9EG%ltHh)BgA9*OX!~Jark?oGx>{sPFtoR zNlUdD{@HE}kpx%Sy-kTrvc){(H*Sbi-1riRz@vB<-dqG7W=SD-Cc9b`PdfpTZ}b8V znsaETMaQ!G&Sj8`>nxX@tISjR{wfuhGBX%#t}a1<^VCA4(gjk>-zDHkV7CL5;k+=C zYEM2jq<|Le8ESSMchBUvUWv{eVmbOWBjNi-3YS$xiPK8)_fSE6mMN{qsmuC zi<^MIrxErxQzy(P8hsi?mA<6XLya^nr{rxSeggOMvW7x(CCdki-3h$V6tV~RQxbxUIdVPE_!pJ8S73y@igsR427>8w$2u`3qTAgA1P zT6{s}{JLCoZMFJVHsKyqC5%n=M$yaTdwnMNpZmq-&I@C@2kGg7&LkgFB{!LNvu4P> zwGb%EdYa)Zst|X<=|L7uB5{&=^#)Xe=BKj> z=$#n9KsO9sfXR+MmOyG%(Bp^d8xd=_HebqNSK59(W3el6tWr!B=lU~9taoD>6>>v6 zKX_)s8|R`TotzJ+ng zUn;b58_12$KaGSOgm-tOWEo{D+4_eM3<@iWx@*tySZXZ}ibkHfW+^!fSrJyXUZm}z zl(d}$pKZ3={KQ44gatZ9lsC=Uzgjk)<47pzlxjM&$l#-Q#69^1&}YDDs>gCVsv$&G z%sp0!c>)sglg#LCblDMck!H z(J2>SCzkmFy;Myz>MxhP42fUcIgeC!g;AsOln#Zt49Z$n*)~bWw4Tx3d}5z?`zI^y z#a8Z3g*U$Zc_O)W&Jz&N_9z~IaErQ^e?qy}24 z6>VZ~No)EHk4C!E29Pzr) z&iRY(oI*~QiSj!+72L$9>}^3PMU&X&SGP4oMp^#KGATU!?Ob3 z?20Ja$aFI8YXs2`dp{fN11bFPZfmk{{2_Avl34tq6|oZb!ID^*%Co*)s>g0N4M+X?n;7q78ds}Ew*>ZIxq%Kbf-oTEsN87e3d&B#yCqY-u0FXQGL{Y{Y#Ab z71O?hvv&WUMCn`m>=6ae_{v@yONK!{`&ZE?<|LX;?)a}F+q_?2+x8P4c3|m_k$Q9< zPST4K|E@y%l&7|pyThqs%X%aJJu(k*5iD4szv&oWkj?o)8h)+*Cte;;bkXy8wIddN za6T2;h4DU@EY7ozVLoWxXzX8BqR*ut?A>45-*HLc%x0Kh&{C5%awDUkrlbk?%5e90 zRDBXEPGjC{AQA%RZz)fP87yX=>iRo=D3eY%_><3A9<17v(*Xi3nOPJ)*WOwE%GH03 zBGv{xyc%In8xE4MJRZ!JR6$QWQyps{bFph2Ol=vWJ`vRYQY}!)X;ZIadY67 z_(uHVCd;z)%NI<;80>M|JzZ%uf0@Dvy`nM-GcFbMvWwP)aQ7h~{_#uNhfvu|;QIwV zycf{$aRn}cDv0WB?+BX;<9-_L=lf_uztB;VcoU1UNFoTY z8rYOQM~Nc@F?Bg`^wtIwdt4BKKsi8f0ZA;a4T*G2mMLvVBxLiw)TzRB(8|5db-#0i zn&dYUlrOH#>~Lx6CJ1|lz5;OddH_*NR$7p#7F80nMHHPU_^e27aLy^e=i2kIr$rK7 zdz;|PAOkIqPf16rN24{p9ObdpVKD)tj8S~#M0vs~qHd<3J3D;WO_@^_pM)O(DWCx? zjh*6-#9cc8S+}1o$z`CQyB`9YaK4|$c3gyZPXQl0G0Q)Er4AC;;c#2U z)kRN{xT?a`QZvJtq*sFcygw2FNTV1SmX8dvOL;MPtBta@Kcw#Mn8&?}cW4S!5xAh8 z7gx{S{q9SCyRZsE^Qi=OzWgX^m5G-1#cs{M#7p}-;FwuQsj|9AjDFT1k&+Z@g)(fx zfT|Y4Xee`*Ked$GkY%Q}VBh{`XyfD;@;!Jt|9+P4_<;rYPR3J}x28sX!QR#C-`m4p z4H#}@dRW<2!Cn{t)^X@EKP;@Q$8H8J?-B-;o#DDt%&wq#7r}xNl`yo&79-WDMO@tx z?6oj%m@gtA?z}~kHg7>wkCP|vX|80N2S^>QC06y}cjD%&?p4`jPTcag)JO3Fa_Ky` znvtKhDrujF7ug&SI{IN@?}x|nj>N$M$&%cYcc|ETKc~zuq9>D1#?nv^t=m!Iu*Y-D zGikClb(gFv&DX3{Q6k|}U$eR4d#gqB-u&__!f7%aM(%+e(B&JEvhKc^*e_0kgkzm^ z#gimTm9EA8gEDy9=$Il?v5EGsj2W4mJQTgyqpOsh(layisjs{xNqeoYzg)+k;^YNC zG<0E#521Jh5>C;-@3194JmB5<>rxUf}6{(?~{_rWjK zpVahP<20ebjknn<%#2fA&|MQTbW1wjHP_qDvm&Y z(aDAjQIq_UZbsX`QXVOvAT~%h9URpUd>7``k z`#eRQ48LeRA}dx|qNZf09YZq~zkB8FtR4nWqXC(wa5#X48#>uqw{wXNLHMR+GB%UH z7#+mj;vtjtHhCHuidsSZyojglgX?*Egs$NwFaR}-;!*L1jO6=wq_OIpyanKahoi8$ zS3aMvE^4DRR!w&E{rI(tJIbVX==+{Hjq_urN|&}6KX>o|na-NsPexcPzQ1skj!(kw zTmxtJ18NXGV%&n$W3bg{D%;;-LsqLYb491;jMK5~d zYC8`J#AgZllpsefJ1igVJz|u`tQ;-QrQym`j8rIRGUdYl?SedQUHdylxzab6ScnI|1@jsFnyk}R5M zl?gA=ZjfjRehIlF8Pns{`TV5?u56VL`)Du3JHSZr;EpK!n>@Jttm%?)?-OyOBYww; z2<-PLWmOSAa#l4 zJ2e-xQ%|p&-Dwd_190r~m17O_U+GI|PYx=Q=b=S!-CWL^*(uCK_xymo3H5RdXn9pGo)|*rlD&4N8SJ~~*;~TIeIrMi9?ZW$E&7xqzt+{-+EDMzM z8{4R66myeCFhVi0%e(R|N{?Bprl1CH`TIPZea9xwbN?kb#y6v=LfGb3(_~E9x)Ond zi9*ikiR-nB;#{;zC&j>+9@08?i`O)G^~xx87#ptn8zna;c%1M`wiM#GYW!5>aH~{*Z|+S<}aAXK3#^mbJ@#yax-q?mqzVYY*y6}NhE(t}kbN@>UzLfs=AReEb+Yz8JH z`qp)dbtW~vZoYWEz}!}>0C`>|MO!e+NW>|GAKeYdznC89Td}RoY=VdC|I{c#%94s# zh^V$=M=cJmkQ7tWmj=7hvQmCz%mp$JTF`7;NeFp>ou6uHW!!xMKas;A8N-c0JuKaql;yW<(Gsb*YJf?B_(vLy*u8rWQLm%(VZb8_jdi6YQ;NIfc@U3rauM)bwR)M$1(&$40mu2{UuZ{hS;R#1me zf?A6CbVbPEXa`yU1M!27*A({TzvuJees&p8p1VI>iEV}J`(>VHIBAdUQ_kOl#`?Zn z#~n0eaZxHQQ2pkvAu6JcGJidxw_*KAy1(Ip@cPmfNG^^dt&vu6JLdG9Q|mJ>QBrUu z!{T93+|wg*{aQpm9ztrue70`Tc3~kK045drP8C+ZZS1=*jc6#Br+u+#B4#5cL|EKN zAb0j{borRptQl9>7MkAX? z=c%GJe}2;Iew8h)s+UX4P&1N6^S;^bQJ8P|G;a4a=wR|ormE$kv7J7s3W#LB1kaLB zeFu>}xy%kLK8|7k77VTGl?pREG%7OT#KHVekJ{(p zBe}#B9!}39C8Essh^Y|vHE+H$0EVYVqYLGSb}Ja>A%Vss}(u^vAlA}&jHQrDp3tv}YHHwS|ZV!`|9Y_$^A6SovJF}|UOce$$lR z8>DeM0U3BxDX&MZM4!?sqtias;q_Qg%QzZ>2>1=aB(Ipwj0GZfuj{c{$|k2@6o9xk z32^*thA};eGfz(wZFs;UeY|8ODIY;qSN2Ja6^r|&qaX*i&IWUwWv2}(-DnaSTGCz7 z73HgL9+B~Sd~LMf1x{d&>$KUkG^RqzX@9Nk((gsbCc#14cR%5m6#L^ToSuSl$i_6E zvlZ7GwyHu*owxX&n(u4j_{KD5%(!d3L%U;j$RRaoZj-5Rg@a^rpgA&andP&DS2d(C zZ$@KOeJqp+^ft(`BeQvs!hTJ;~clx0Qah#K#hg?A9*e{(%c3ap_|=p)@K#i&z9*3c6^Mp3n38c z3kfc2r+koNV9DJ%^|ys-lEQAAsGie?^sNu^{wea?rlIZd#~6dy4*Me!gm69J)^Pu| zd(F-fMIV>Rs3wo&HsNQM!LN5(vZz3)dWC!`rCToEnC%dBq3zxEKJyb7ccj1#jnW^l zQu#`XOMDB~3V9F5M?Ya{=HnSjakkZ;Cpy@S`j4Xg-*2qk*!n{wM!tUhGL!%Ny6K|{ zlCDp{qB1d+ItsqXvOg{7#FHqT(lDRTm}P!=f4S{4)GSZY6o7PGZ-`{@W$Ej18RFdr zOkaS%&})I?%-bkL<8Lx{*KJ7yZkjFi1IMYtT(+n7+Vmi6Bkl+0jvF1dec^#|nY-SQ zz>1gp8Dp$`y+Sp+tx}SZ%TZq@K1aHF`7*-48XIqXusOL@Fcl|b!SlL-NqCx1(Fz^(aj}pGHa#<#m}yag~GwnAP3v{%&&<6pUWloXDT*sk4+Zj0;H?*JdSJKE*Jf2cn^b=>JdT2ySjG3;GCv4kjDv-WIWNz3 zQtGR(glU-5(I2uOZuh5O_?>1vdF*F>|AGbGZ!!7<;C3{LP7Rww8EBwU!ye_@_Skr@0<|J%-QA?Bt?sEwzbirMv`>bNDJOk=9u$VYO(}qGmzoz?eoA zH_tr(dshsnYqmnazGj5JQ)hK|0Z%N~G|gpr0e}3qf3$Fx#Ri_wR1&g^lk1m31A8NV zf^*wA-2HI~ zfKuWtOVOdH^5+TaIiB++JZ*c3VWN!cw-1@S!rs~&->Ig9Y?zK;_(Uz@0FiV3bRbKo zxDWH~N+d%oc#Y0v0(*&;?LZ@Rl-o8(em$nua1D>uN$@_bp^RckE9tRhjrk5mQSI0l z^r`P)PGocPH(L!^d9Hztq&n}MF^0NiagwVk_~7VS0v-aaJv>&cAM%PG{L?L$Xw&=9 zWLm+7Vv>>QJBN_^Pg#QSzgcKzHfYGW=uG{&hM2cMw`a4cUhOC&x=|sjc=3tN!)3bD zdlz)Ohv4zz?zY#}D7BVGm+@bt&G^F8*;M2_RX$0Kq5&?MH{H%ox44dXJ63D^eE~Jk z^IzJq&3tm$aF}>i-kkPsGlTEk1IODX4EnzODfY%KGBdT=^8Iqw2IuqsSJw(+lgmrS zRe@F#PUKsq_2}{eHG+180SM+X@p-9LpJXIo)89gXdZhlc_dF)4807dK)#aUdK`841 z5Gq`3u?ey_>`CNCo)5wR+K83DY6lX+7#V`s=lK?cekIRSI_kyIX8Va2j=Dc2=O=OW zdI^Emcll58*|rw;!-KVXSCWP}2D4$AHi*pT#<@LBVo-@Qj{i;^drwj4s=RBi(j4_9 zL4y1Ki2NmPTRnQ#|Dj@MqWkDLB89n?Oz|E{lhQdN6f3g_fbrGgT|9$F_THfPKyBl6rKwn9n!3iBkG9OEEY$M?f69~n;adQLz{ z`C~aRpA#S+l8^sYy{zc{oi;4B#uDF@M?XmevyIvE5-8MJYhXB?tMt8@{|{@G51p{- z<|BIaE*FQzLI?O=KSOq^eYNXEC&32G$tDl`75ChH2qQ-B}o1>ica5Bmq9kg;^1FN2)wj!*tcvE_ba>OBy=^FQNt@L z8V#3R-^^T6~=`W93ZH_%|lZ&+CnHoSbU>E)x-vV^nmWg5sot zcfr&O(Mw;el3%SuuMlc4qoEzWCjK#eS~iDt-5tH{gRwsd0}rNPOc)0J1;(VnO1Y*d zkzRWYpa1ovdWQHrUezj~-(A!D`f^$QI&0E@y_$7Q=38=*&r&7|j7ymSX%YR+R=NUf z_ZQ|xop`hUm~pKNKubmS?`DgLO6s%6N;S#9^NRu!N-Rx~UJWsZ>pbL@W_@e&p! z%A!4KcFm79@~SBrSFl+>Uu;?MB^8;;N^AXua+tD>s2$D(K!OpAW2}-i{brh+0PV9f z=@h4rP?0@5{_^z{F5zJ$yT2&E=L9_Jnf*eT8EX&*c8$OC9TVhO#{;l-^%} zJBJ6M{|aY9_wrZKFWU5)v_ZI$0g1l6qg6DkT^Ze)co_e7KK(p6&a_7?s0amp`v`)X(HOQ9rA~%RirTNoOwj1F|uAV5$wdKabsLESivDd_bY>0e( zL_SmCw?zmOo=;m^)D89ne(h-#qdt#EnF(UC!|ugbTX(rNCbg&~4(9LvQh zHO@7OV_nC2uBBnrkm0^nFv(6*9!L~p021Xy)NP7*Lsx+Jk!}LK7cce47xJj$tX1hx z0Iv_HNyLQw5z8fTxVVm15zvE!K23U1ZEYz<6GY-TE@{(!&QQeZQ@lGbn#M3{Cf#Tu zB=NoN^!ko&MoSNVT({<`<@5b(8OMe3Vv8+~DZEn&f?AGMb9FQ+V*ue_ETD2M9(_4#CkUO&Fz;SF zeQU21kwt4`qL{BFChR(R4SC@hB>;}{D92F zg7`*S6C`YYg(gHUqAw*&7ICfE_OEGzYxITk-uy)QU$HoS`Xnnqa$aQJ=Ji4LKEPsJ zwysO8(93$yGiyUJ(zbRVDDF6w=9iQ;Ta1ep#(WVh6@nFK%Ah};bo1V%EG>U)fnnmN~!}Nw$|oA~FPp&+ zjkS-q)9IVOE9LAens|0dAS&fsr!r4q+Q%_Y&${2 zHIb&cDE{4##!UsqxDh{&t&$dzDO{f9*`FK`G5~suQ}}4>`>C!Rf?r;q{!U|f(KYB< zXTNZX?p81am?*e+L4q3zJEg# z^=d1Z%GbM?{(JHu9J@euDuVnlmgN3zZi%QHU>wQsa`njWt#@D3p zPDra{RE*2uOh>aKpvt$Y?i-U~99qOlhyF~qEMyfG+bc9Un~;;Nzc)_&E>xK7;gIoj z9%lr{NZky;os@nTz@@uh$K{69`fT}(Xt#^wo7>a zy^{uR!;B{+hm_CiDx<&{GdkD}2F$e&MWQo_`*?*qmc-KP&oW)%0u!F?FFzXfBv8eJ zfoiFe>NQ8W%}p4#%q0N!f!V- z`4Q_=b36@KE!fClq;;JLiO51Jh0$#OG{Ion5W z{OFi}x3`uOXi+ltyEg6Hq2|pAA7|mU{Gk5y7;4D{TG%K9n4Bk5lJZR`!3w9F^epYd zyh)xP0^9`^M^3HebP7&uBgNmZP+?yRZk$Pfq=fFG?&&w$o&fi zl}#@)R0TIlpF3D$t@@gq-W_Qe3fTAa*S>@W7=JQ`SUz$L6JuxF$aRplMZArb6KnpH zus&^Yj3RepB&oi39eI+dU~RDgFG9Y}t!T75a81jYcY>^?H1KZ6 zZ2;2s$KvfMJp?(=+b|k`@@^dLg8%?;V=8=ygvK=~Y`C)-qn`x{9*?)s@-g>T6JWD? zBcCeV(Q&f4P3imJFCc>8DEX8wu4kCMY#r+==}IQr_$?^al7-dNWtxj-J-S6hkk;=| zIBn}5(C%2!c{aZ@x%7yL-O4|Xr(9xAK{5M>miyL1sT3w}ooqit##*tz4gF<~MZ{Fd zSvaR1roW+CIBZpAw< zN5ZRfzWC(g=VBJssYkp?Sm!$y1k+~{n{mgp+8K>S)_3o+Q|#R)Dkhr&TSr=_q}K7A zM=g@^_K)?(xPcpcxn-MsWT&aY`0pu+YAMzWWyD{NDtt^J`1LId?JNk zTB|Ys-zf$$%Wm@VZ^IAT#zEgNpeNrM*K&3`M-bNS{%;Im$gI zJsB`!x2d)7dd@_4%%Mbahv8y$`_ovIZFNXp7yNJ@iH1Jk@@#ELpT?+Dv3ei1{At%G z8;GM?^W!qbN@Ds?mf1T6BgYHHE>&I=DU}CfVtzBYXZQhir)f*8HP2Zrgc1^!+sR(4meYB9a5G}0}%jXaqDS4FKir9QE`A5Zlo z3>*MIjP)n-i*USd0|%fqVncWZYH^Y@XCOv;UXO>;u?>_bUTa1E-lcDjBnkWs7N+9e zfV1(bk>S_$>Qv?jXDOms8I%#I*ITFu`ZZ-)e^FK!q*YOtG~+bGaudvte^uBuf8DF* zzHLF;aSV?PJ&uD-xQ!1)w?>r$^HcI|5~`{dU^@_$gRZ4=)tZc3|HxV)l}y!EiirMe z1OwxpZMbr;6t~8zMwlODP;0$w#_2={ByC`odWkeSx_(f(46cTkF7ag`yh0h2HyXBSR{q>`jrGBkt;;)id_s1C21p{4{K zeZEn{|3G5!xFjYDyBh)K{TdFZ!2<6?CB44QFNayX)u_YjqY4fg?wdOZNJMpf`~pg# z`U!#5Wytr{BT$-wd;~v{Om17Rs6Qr8FFAbid6SthFUf#d5WDlN)cV|rE#A-2d@fM7 zoG!;zJGYRc$%dSrr=~S99v34AksjdMqTV~@sjE0xG3R&ceMgY8zj8Su`jV;i$Kfb? zhDPu0a85M6nD%n5`%OQ^ljFftQ+FK3l;Ckp=6lRtL! zB7WsdyhuI8T51sJHpsv9?P;p{jYw*OY^dnxgI`?;de3uNs`1aMx%}*QS8&N2Wad$d0Svk!Is8 zcB+A7_ll%wdG?T0Ieh$;H%|!u^;@5E$U5Fj?#aA?TM4M4x*{JQEw^!aBLol`-h{v7 zW<0Em`zM<9amQrFUoGObCPbRJ{&g|Z_+U7`QA8R98mk{j@z2KBH9^%4T({Y9q^VC~ zF3Ls~-#p<}Z;syAwe2{EpVQu%@@m>QwH&kVEKzTFL-uNZ_L5waKpL?to<#UBOey z-&4`q!T-81t0k2l&L=o5OOs`mq|+838^E*`63X-2<_4{x*@Cv!!|P%v-w)S#m24+C zTb{7p?~zifg|PAgg{>3oMO2w^bT=Gq_#r!;dJMhvZBIR_N795#Pq1oXq*CN(6-+ex zGbqMC3LJG|jd*)TgmtvRe*1(Cfo1a*hY@3S5eFaDhDPNI&P%mDA*j^rREQyZ-OT>IX(Qs;J8Z& zk<_MS+8Hy;3WoU#7f$cl4dHvi~pD& zPkZtoFF(~6o}j~j*rO@t&J_`drqO>tK_@coidgBF!z))#tTpmvYSFf3F zQ24}8MP^&}_nF)@`rt=QQqgvMttMe4_)Dgx-#M)?fplNQ*Ip@bFAtz#szJ)-FsjJ6 zOG{yAJ|y7?fLNMW)))j_E%K#dUSDmEz1A-3YmM0D)_cNzZOJ&TrY>4aH#1kFfPwAf zW^XI~$G(w{;}*M(O%x||D2(&Lqn|HNJD=;EUw{4go)zjJV2L|8OV#HxWkjX7^g|E7tW8IUF^UEuBh*` zaaWnN3ByKz8{wH)RL4Q<0e{ApcxRjs?-Fb0-p-+(^aKj2Hw(}2crqx-OKnB(cF+u3 z((6Tfk4G2KT!#E43nQ{FPO4Sl61>ihMOs7fq}{DbMnyM~dMqeg|2XTy&eyN^7hy&s z2R@nY7dWR_ywsq|?KBBV7F*Uj8SDO}5;>)q!`!x1+oR7ru#+92rUO+PYwZ2=Gg6+z7>oKbg&CDJ2jb3)DUxdMX|A)QiqlKWpC@M}vyqlI^%+}% zYwyAQzUI2b*~uMXZs6iImnPcFxx6p#0qYk7ws#&GiD;*Mk+>I7;mixYY0e0!7KV(~ z8IwJAuZ-Y47Es?>8zV91Vdb9In(!HmOn0tT{<(H&#+``;e2*dPGH15V*)xAMhGqA< z#zOZU37I%1&5}5VGGK~Hk9BQfl|M)3-kyt{c}?-Uu+uhB7*oP_1~9jk!?x0loBn3+ zbeR$Mmhj$5!YPc1z_YjXS{uVfc%~4%J*J$%G^Vor-E(a|Zk7Hv;s(MT%iMYoAl4k$ zT3r}Bz&j-z{W_-*&l&NBGR-WInK$5XpA!>#j_k$0l9th)6VG~vRm!qv*!|qIw`Xo8 z!@V}r{9T(6dTlZFu;;A66{CiJ-3uOj#|WMjcGB`q5nDb^8G*h>bYF|3LL-H-m^pKh zMjrzVXN+Xul9nFCox9_8t?i%?qF7U2QT}YDY1I?6WNDsz>3WR4?Gnb`&Y8=IEapv+ zoHyXa9FqoW?-|J-V(ei81PWu425a7n0N zq~x2sM;7qi1D9@1t(P~E$8&b|^-+S$8l zYv#F?JvY2@9J`TV>`DC?MqtTa^ErKuFvcHu7V${qiGHjssXxY0NgdIECM_MQx93RF z4V$rlFQlzM7FK9q>-cL2>64eYT3A>?S!RV*{kT?GbIc>Qc4R!!k>~8{Tq|>7t<=K0 zpwl&@io#wChdM4ymaqgIr9C5-^$AJNb>!ffxE6X&$`i9Mt`yk2*XZ$I+lY4M;e)vL zzCjpEr*}qe>AZ&!)Co!Xb8qOmH&f8@&O7vMEIh}iA@Vc=-t}I~G=GIzoe>6v@f|C` zd8Sd_Kj$9ajN#~EM996GNZDK}q>+vjha62ztDZP-(2_C(q(WJNWPe6Y$3I7$@eh;S zV#SHbU=2}tOFGS6?vBrSd%*(ki&k?{# z$t%ogFTv0-B2fECgAQnK36{KP*y~uK=OS$Qn!NVLZ=CB{X3Y%hIMW>M6r*Cv4D*;R1XYfZH`wD_F*Ic6 zJlDN9z~>m_O=_;x$+to>%orPPduH9nv|{FBSewr=W$nSfl7RP08`dDst@gPRdhgkr zDQ~Rl_8k-6SDM>McyD=~xySDBos+6?PI>bKUu zR9KTDBus$*nn!%v%2Q5i<|%eKL^D33Hy8? zhMmQ|7o_0{(Nc6R@$H@lwqKl!ntsk9;ku$u?3<%pH7qIXy^|L6Nn>Sm&HU^Vmjd^h zxxH^~>FYUjfY}?vIU#O6;xJO&^WJ+lI*z%z9amJt*fB#qFXf~*cVNYuBaJ4=-ZnXJZ}B|4P5BDM83G!GUl1f`#KLfgBnsOQlGnTXv{5;H^&NU zox41JZ&54eytRMY>fEQIwFkm+QTz6 zr`hqgrgr$qGcIF}XqvExd~(@C_&qPw?HQmF+4}y z=oZI*OB{PmbOZsGJFi6loYS~ruaUF8$7J~$6Kp>x(9akLe#Blmr*usq`MQ_B(27wZ zKWCB1GxC7%$-_WkM{)HZCWQQ&1Cx1fbki~xQdb>u*D{1P**90vGID%(9_wc)ZFHls zGW^CF8=PsZ+2$8_T<@DHVQvl};F@-R+s>FD7!&af@{scD1zR&_aAzs@H>MIwPn!ll zBtg))-OIzY@JG-*yLu{~Gj(i+1IG$v%Eu*Ry?Ukl!PtwG!zv&c!F6UQ-*o&TD_ zaer-n^so$_k}=|pZ(cLIe+NaTk+Q<>T`6IH#bJ+?Mufnhxj{Y+tg1TKDozeVgla`3 zh-c2rop3Dhk(>fdR3E#gJBLNc5RwMhT&Y)Su9^N9LSDg2>mhW87-73dN*%r^_$jeW1BhAdX6oEvIi8**o)mY=0v|b0#L)7gPeUt zG~_o&66OmVreb7utGH&u!yTzHBn14wIHuU-9kcB>@2ulFfnvU38C)+ZSfm z!jCaWZcZ7@um>rX8YYHjSb5WXZ{X!0m%7ML0j6rEl$XBOL|U86!Ex+G|BlI2ea;Np zF%yc-*$cR31yR?v!@Bz4V;N-x#q_x+M)RAQIxa;SwlSwv*h{-wZbWR^9Cox&OWPD? zj9k_|hN4AULwhggJc|4r$(>t7Wo&rrxwfR}TuXd-=55Nb_bAogd-5+X-PE{t z?CIV6&vtL{yE^uU`rg5;D$J3Qw&qq#82dD4rRlJ`_O8Q>(dc%D2(U0CHNHD%frqgY zIDeZtXd%zUvNQ4?%bZ~vK8(rF5NFoR9%&LJghjbOhpum0vm!K&tfaA~0Lru1bkoR@ zxp75#rM1Fl)Y_rwCPtyZJ;u`i9*LbJ4xNd<^W^eb8PIACY0x}Ilw=-xOCrbV)3c^d zP25vKa|}JVoR_lbNi&CIZt1o%r~WvP8MwZpn%xfzp=1nc*)db@bC{!SW(TdIHr9;V zPtg%HhA8+lQ*8I$v$Sc(nTfw9tlydg$s%t-sl0dS&)w5kY7A-2wPI|;*mHz_1`)ZJ zqwdoWyA^FlnD@D|VBnsc5hCUs#5Q7bQW){IWDHU29#fE59pfH!Oj+17=CZ>_%KFcn zeQj+a=rRK=&x`qnVP;ILmsYfS%FC20PNa?!w-(8WiTFK_6sxz!e9s=CATsX++c9GZ z`ChxHFb!N4^b>HE^)-UvRGFdt3iGTWzn?f=6u=9*KjXbz_#Z=>f95Ob&Z|j z7iS1f*&!Z%L@nVmLqu6m$sTYI0I@n3WJ%6*aA7Vf*&DZzSqpTJ@TER^6t*VsCaET-ZJ+z&l_{-Wi9BHm68tNn}bU!%}m7>;}%XB`+jPyIr_Ob z9BGe(?R8|4+d8+3NL`5l*2)7XY)o9Yo(3Rc4#Dqv%$>lz(xBQHV~aYD=#rfGUh|DR z+F@@r`LII1HUc`rm`i?bF7(+Gmm=?pnFBj*_=&S32;SaH_;c^&rWkj&O3RteF)cly zkS6fX+7Ym5%a0&&OIlj+Zw?NV9(IXwr66)45P6xi~&q) z%~*~*Ldf8VsZ@TA0pU1u!ov?^%`N0bCXD$X+tY5Xc(Xev zdgKVZ8$2$t+_EO>Ks9o@>f2+sHUxp)vX)%SO`A4!&rq>3t4s(Nk@9_OG0`%<5wwsD(2+8-#Nyv^4{EK|?J(yo$XR0-eC;ux zypyWK95LlOXWgw3Qh-H`qtz*;Q1HBFnCPGR?K099SKhwAZ5Di^*Mc4CT6? zw?60G^Z0Defygq3O4S|1g>Y=C!nhZ><5(lJdav2@J7YDM-8biW7hEbA{#}4vYv95m)RNWT#^y!ToeI!Leue-8V z&>ZU~ckS7-GUICL&nuuRuLzr#2Wa}<8F42?D1c`T@#;3>B=wr3HFYe}@UpUA;ZITL zYA!*>nGd zENYL*vL05_?VVF&koNa0krxwAb8Q+naQ7PLT1mwth=mYa1ZUz@HU|Hsjg(S2OSR^);gY_8Y09Am#1- zzvkpin4zqGE@as|V{Y}D5ms~VsgbvLR{n^?bvkTWjMWA4nBv_fRin=>zCO~vdnv$Xo1Df?}Y z`Q|!yX2snpJSJwO$TO#A*W7cgz=NBI=-FC%TXrSr-#GKO`UqKsF|HY{ zn@5(-2;sD8#y!@$SH|hzG5%(yHPjv^Qq~Adm3A#jh9ni^Jog%UTHAOd#qFxOmh#n~ zVf}H8)qs2C8R(dI{Pxc|Eq`x4@{|{t+1Rt(E36&Zox-s18C%|AZ%L@MCs^TH)BijM zj@eFosdJ3Xj5x;3Jo4T`-Vq#T<~W=em(uFpW700K38ND;GRPWP3t7+!NkH2z$=3g;3ea=J|zK7PzjtP`+E^WX!*8W9}^B6i!QRFqJ{KZ~z&1S95pR_kZ z;uu>PB5c`|KNjND50T?Kg;9Yzf{J+@d!{lZS+lyLjKZ6nB5=&zyfo*8Xx%MdYnOQY9Z;9}?hr~t>(@=2k zn87ioz}4E()pCcmA?304AE&rt-@zwwO|i*8)ZQ91h-27`6CsV&ue#Pg`5Wtidyg&A zx%XJ^8o5DzZ_%KbVo2d0!!u4>&MD0tr4<*d&)Ngccr0Y=6K7)k4`ZP|roE{?_LkQe zgCj0a&4M#GGVETV_G-^{yq%|>?^{!JW(7pYG{-bSi8C5vCC%)!herCFV=*U0A=VaW z0?}Hl*muQc&@(rZ`dC>tbWPlYIyRug-BT_xZcV{hO?b~twlZ_3>AHH_$-wby+6N-4i`M`6w%M_kkoGgotEmHW8zH1!&jRcg+d z;yD+3?iq9WZBF31v{TITjlqmzFHxj7cFNhw!!lc#_(;QojJ&pN*B`rj# zwRhm*+tDI(rS0yx_KMaTBaVN>3H36jJjGrs31Mxl$ds0T<%AnXG)e@&h5y3=s$ z9?3dS32J6Wv7$JJOmqmrdTfWC#FjQ>Y2D*Obj(1zALcsL9vfD4rs0mTw!nIiNo{jX z;L8*SJY0%X{9vw7u9Y)9bDtr1Dh}X}H3KrqS)&tUjl`@!$J)(avneQMk<^f9++!FS ziE?ak`!>cfBM6nz++C)7yLTW=B<( zJJJ<7VkB9rx#!+=8mkC%1wgsI<7m&?n}uoz#GR3{DE8l3)gcFUTYsF z=G63-V_N!;Q;ata9rZb~zTyvw{}-0@VwnlbVP~nKIQB~5T2ne}Z87m1HX_JQtBP~a z`2Mt}G|b#nej@G2(zM1-!J7LfuC|8=_#FGSaRd$39rqB~+q*nuFD1abGipuSvsG_w z#QMJ0Akm!@1thFJxwe+R)|?q*ZY~j^J#$3&O$&-;jb(5e;g`1#)hDI%qKj6Y{s z!Ox5KVkMB!8%EH_9*Yxn2Yk*SqIzJ;8|y5s0i-{dy3O2+vwp0}`jbX*^qoT)Ek}{z zJ?6~!p4*c$W;OgXv(!x=0Z4t0{lgH4#>-sOz%9<yk`W<8hQU6 zV*=_}gOnhoS=Ao~QgU1i)Nbzmx0B+8!CD)sXRV>VIu8Q)gui zY|1oqZtss{!gpsylauG#b(`aLHEu=EJEnTb&yxo-EqR=$QRO~&)YKno*C?+k^q$66 zQr$C{eNFi9F_sG7TmvLG#KiFx6DaN28H_6?49+}HffOV9U7AS~cMK8w zzcv!znE72LEa;fA=4f0Qa}8&N`IwWFy5WtHoMDfkWW-eHzNUa^9+`k_Wrc#7x4_pJ zt2BA1)zi6WZeZJ6lXZo`-8AO_T#ozFJ;l_LG-K%AP0J~J=5&LYMz})S5kqZ;jFPps zn#Y24XkZ3YdCljpKh z+^GsWZtcZ7wo?0>Q*&akCDpW+*4v!RIy3Cy{yz8QVoY;wbZ-6JotKJ0n1i}+jp4+Y zaum?t3uk&Jkm0`vJi*^#Pd;tA@f^1%LLAd-J*~B$HbY2hnkzzfB(>5ulThTCGm2uZ zz?T=pigBEaZD7sGrWR)CO$ifSZZE*AGdIxmo~wFkOccMp2DI%&B20{!x{^fdoH5yX&2^}jR}RnI5piR$gvl{SEYR3Puw@JhhCPOY zR$rT0F|P%^Kl5UBrdjPZBIa{m3+X6Mn7fz5q)v~k1v)KI!4`%z&K)BiU_}kiKGOWv zoI?Qf*^$g+>d7c0B(c`|Ep81(XuU_DL9_#Axh*(!Uol=v|oxGA)g`J`!?fS&5b% zf^m(R(`w8tm^;TZ_!xPYH;lE#xQA|EjO)QQWhAMbBeLmPLxo`_{nRtkPRdI2_jv^s z$vQCW8v@2%SwR?bE|JSIW|YpIfvJDu%;(=b-)GFYlbPoJ#|)#6KgMMHJeT6y2ysYw zZDF##MnGkoyBHqHr*AI_o}Pk0WzGRTBMotyvXgRQpWDYK@4%3?hm!VB$s#I_A=Q;~ zGSFFT7J9__(YSK}_h0MdD(BSOJ=Pk`oJm$Yrr^OoXP)#Q(*tJ?Dbv3f-uv7OCpWH) zusEiyc1Sz%VNUVwH&-U+9=Z2w#VnmS7P!$KQAlhIVY!*39`YN5Gixu1sy~LrXG)7H zanD7ZF((}13rk8Nt?AaBGjh|8srGGzZPmZ?2xV_1!MHbm!kk+vJ%?HCJ_1DLjwvx_ z?bzEkX13*+5o&Ej8I_(_YVD1Sb0SRTjJWos&)W;>XJ+jt^T4LSxDX^rDKT#h-03pH zbl*w~eKCYd&-Q1S=^bE=0^&loIFo0sy1c@ z_TGt)bdK%wmzH9~&*Qp!P28Ek*ZR#Ei!CfJ#q>NEdPP_hJ3lV0qp@Qod|9c7dZdxn zGZ*Z}jjP*j41CTrhH~kQ8y#ybEts6rGXEVrXn)OZf*f+}-;CSIaAjrKwMOFlS!=f~ z22_wWmVQv#(K0CwVSyE<($w4|qG;{}vN$J3=9%a9flCVu)p;d}ij>#j+#G`d#mc)v zI4>mlyjO&4UBlWh428itw<5=#8%J=3NXs!3R?D1o*&|I=shS5)a2Y}TXr#>RIhU%* z9og@8uBo5AF@hNOvP%!63}$Wp;4{+navix>e61bfu~r=aUfYCjWrV`L!&t{1Q7UZ? z)P}PL(sK(lKWA->g|#j_>~jP?`m*D`<<6^QVa)A>w?ne%4P)0f20e_k5{T$n`x%qG z_AdC!!;^cgO|QFvlDjwZPMG`ZKTKh|7k18MTH6_6?s?e~1{8f?8}~d!%&)%}kX?^E zC4UBlu{3tH&>2&6e~!_TZ&koFVwnG4$##9OxQ#j|#@St?rEScp>j^9XW=!P`G0M(3HSf^m!$fi_m+;T@UjWY$(v``CN7FsDqFx8m4! z8$)VgEQIGDM-Y0?lUZmDp~#%K3i6uUMQ4w_^{_|M$W7~2V6Cj37$)*=O!;zl41LFt z>kG$w%+pkc6MER$Mma&HP+gbUgdd}4Kyr%rt3aL{k?)Afy z<|J3xK^<;ReTtS!Ny`2^IEZ>RAOMXq2`!Sa` zZV1D(a)&^kG}Bb+U#lJWwsM20@USwvy z$21}u@gQBd{Pmq_f^khKoHq7oSxPiE~jsjB%#1Vo2CXqd#T`rMVD;F3X)u z|23mZWr=&icZ9v3701jim{b<&D7(Nw=U9MX?JR7VadPoQhF~Os1xHJ zSI7rG{mN^kZmr1nvmzwcTcPwXET4Mbn2Q8&=1HR%m;QcbIhYa$1oa6M-+S&blo8`> zS+c3|e6{<#y- z=v?Edc#M&pH^cbt$b-RhjwQgjqVnRH>(O$?S(UmbX2{O5|35O=-d!P{cP?ey5F_l& zDeMbabd6I)AqT|LvB&nzh(o(?=gG7e*O+|C39fgIIhhvI^kPk0t2=I0!ML*Udddl< zH*87ONt&BNWvoon9D+Q>9>W)8ZE4;ShMdQndy0CDNt_#}@_5+GU}(;T)Hs&@$DO;z zWR1Ygmd1AW&hzABMYR34vh2iO!gd-vqcDb{`I^=YeGFscGVdU+ou_2cO!LJkZ#u!_k`wC~o zCA2dq_{1E`G%8G`s+RVG_SsX9ANv~p<3UUN@&%~`xL*A~RvL&|5ZZM;5nB;1%WX*w;a=dq`lUNm4@)_&mkjX&$+}n_ImtFDQ9{uA)UMCynPLu*lG>o?le-aYg(bFBj+{knTN9O z+HpQ(ZYAtChYU*!saSi2VXmpq=f2wJ zC*(MxnszeynzQsYF4_0DrvPl5%j`L>&6GW|ir@{)*kkQ|mb*r-$Xqe{E-tmBG8gpk znd8PYE?Jq9gC1L+d1pLqwX{2uCfbYhb71d*tRJ$jz?*@NB5aY^Fk>jpO(BzKPu=}E z)9`!`+qiy=!O1ZfGSXTbzIu*m)3Wkt=AW^NX-ws?wj;_y9$^$?Elv4578Jr?3tMFk zWbrcB%<5){La~XG!7M(kZ0!biIKl=2mJk=@#SQ!8TK-E`oN8W18XP2 zkiV0d+RedoI>kBkGLi`H3*(%04Y|HJmx}IMd)`dR4J&L2$~N-=-`-P;E{(mXm6q1! z3uE$n?Je&cw&MF8s|amOp~EspI?~$9}(LW;XqZIb3e70q(bxp6|~iK4K0DhY}XpNXQ4W zLeA@CFfK97GS{SAnv()4EXji$=5*y4>(OH@!HOI~P#PBfR}w z^AmQ=Nv#^ka_r10KX**U^O{q}*I(iKWNz`!zBi7;-6`pL=luT~8^Ag&<;A%6UPl-M z&iY;XtZ~KNq_rj-Lda`UZSM)dG1JuD8k<5Q4OzE0mqP0q8?Jwcoz0l09zRFs=iMV0 zc`RMBH{+bZ*z?D zniEug&H%waHe}n})1-C9aGE-|2t=84=PoBq`>?}uLSBoMZ3hsGz1DQw3{%7?MHp`#* z)f`vgL)~jDaj(>tF-By53fbr?4p5vtvy9YCL%Vj&keHhH3f7O~mu<}ihA~p!R-aiF zWsXtzKN5J{O{4Z_4*1WKQW8YJ(xYBz-}GO-VY;Pb);qI9~K~L%d>N7Z|#mY z#)x)ZyOnZnfswh!I%{8}oj50n&l+atA2_W!%s;hVeIY4H1e=&n{m25 zlZI?!NEG#DCUkMAT_QrsWm?{4_23SEEN%%eGJ*~3i1o_DW zJap|Gvp)v3(H^`1KSpHe&r3CY&q1z`wp#2NgWYQ`kdHe@UgC^_(|cvT{WL;2<_J;N zU~iF>F@gFrm$b>)nY>|a?TWh66ywgDIC5@XgOi4c>x%=YZ4Q~9I)hx=TXVl?ZKc*e z)>dlXdF^;jnD)Dugw~F$4=eAGr#3_G>>6{wHjdqinwA{I4BNGGE#1GqqOwicV|#Ii z1(mUq8f6^Yoo{Se+dhZz=NmgqHw@gY9*5}UT)P!94|t(Dqqg2lquhEdB<&No7<=7& zBXLYT=r-oAe;+|VXv{&QzvI}~+yjqe1k~3$W^7{0Q-@;BsMMJwHfo;91$ShP^u1OX z=3SXrcMi4c8gg_&ntPORCS|*lwr)k+F|s3v?SN)Yw1=1X3c}g3e|c>HpuaZO^P5A! zG=`b@xQB3HUMmJL?tR)5RuWIiluM;cWgkAv=?0W3i}FWO^kvVckcR( z^XO;pWz#vsWcyq5+;UHFg)+y6S6pN9XAFV4Hiv*h4xv;#&P0tkLfY`pQQvXRDYF(= z4DZ|<-)LsG6?<~BYuBGq|Cnq1i_t~c_8kT zy0S)O>r0b{Xhz|6(f3d~EH?kPMXG zvL~AG-}#O
_ZV@Pk$ouMCc=FM5-S9q_X;yBiJ>fCcdYY%zWIp^lyOq*V7jfj{( z=FZuit2tCpO&Qdo^wZHt_{Bv^J4v78-X^* z&8j_jL}`z6{xhZg);?DfOry@pNXh3Ou?^{7aCdF3y5rQy#5+)Rn%E+?_*wsy#WiAjNUr#0g{2fX$SAyjzm_0l$yaQ_>SgqX-(MI zw}z~5%L{#NMm6g?_SWc6gG_fuN#ZdFo_`A4gL7?Z?h>OC>dK4hDGpuwH}jBmNIA@B zra1b%6E;Ll30osgK!dbr&PYv*BpG>aXm3r9vuE(} zy`NV^@m%4ndd;A=mR6S5%=1Nct&kS^;m$MCeC!@mUvEsA?wp38)84~3 zIRu@9ns)BXj-v^4uQ~0t2J+I6TLODb(c86_uJK+YMl7x=^|DtwQ68(qcrA&H7wMFCUQ+55qxTH!Q(i#p2*2@5MWK=lNd)(YtK7TC5{EBG4>*H%=xP^?g@mD z<4!bUa{Zk{pkXZOjT1vm=}&X^Xw8|hlcTcqSO?%|82iy>4OHwqhsbc8+t+9Ab?6+2 zdgzNQ?l4c4%{?Z}{LTUPYvs7xw`b;N9|Hww2AzYKGRoA<(-<&KIiHx4^35JQP(7@T z!Wx&D?90P~H!TgZF(V*NU7LAn?V!QA7ozkGVbXQ)yp1)2gh3n2@GYki<{u|))9*J&F!&r-ria{?l;UKw3pZF?OD6NWJJ}iJO*G}-dQGL=OME`b_88dtEOrsHO4b$ zp2NziIU>vvi#GR?!$@KKaAutCG#5Pl*m+$$?6t}@r>yW>OI9?*{NXU?=s}NRY;;cr zgfK#Y_M1zxatzh%6b1&+$&;OQ42k-%G6Ad=M7VpD8?|XvFD=q9E(eT&D6mkqxk#X%hG*D<=GxGQv4lbN_%e+r8C!>%-c)OcWv0T zl$J*D$hpO3FD>x12O|7gG1+TQ;fl4EuF@Gfp)kxGnxFGoIp^5Do|h)d+L@Fw?Lexv zmiqNyyHI$ftjxS7R(@V_+-UAW=ehPI>mEa6V2r)3k6pT+cYsb`yH9wm-Gvzk9?IKG zhGZ|z#XaMe=t@~Hd9C=WpTk^##d*UK7PiEhJKrGZ#F9TZVtd(}UnV96`Lr`u<_POW zbxsKDJ7$>i3@JZ##%0hk2T=Oh8-;Ig>CruRDr;NWBzz6Q;538$K@W4TY_5!l5r#_U zj2YTzXXJ%HS2EAdVU1+xZG$yK_F9=^25D{l{4|0n=uI1xXwEd4w09!#O#zc9uSu7XH}vSSBznwq1!#h9rU`_#MvLIUpfyx zkeC;6$yxc;Y%FEqnnEb*Sab6rr(uga=MdDGix+T&RfD$Y{S3M0 zZzb{UA0|wCT`5&~rgiYSR<^|ndHQrE<${f7P!9`iVa=(lKcgsSit`$DjVXsd7Zz=wTRkriRk$&- zF4+vz_--z=fH@6)uAisCVqL>oa;`a#zn9|j$;r@ggu&UH#zxZLxr474G?hL!%F9}D zk!XkU@SLZ(cpF2_aAlzTy4P^+8F6@fE=BpX?jz7;ztR%{b|~N1Df+`%QjiDU+14zRq47 zEORc@`MS0= z^`1lRbm#o?84DF*<>}WE2iV(;!DG8A2x*d&of|a?ChJ9#?a*+tIlJN!Lcwl>e`&!zIMgI>okH)O^s`cd#)Vcux5VT z2s^QQjcmOd2Xw+>%i z%Uyj;>7TdOc3TN?F?od8(iOI3=+C*XJkE@onbIOmT{%iJZP}U`_s;xG(`Y>}q5ie! zq{msiSZwZX$1vgq&(A?9XK(q^H5X3!i0eIJC&AOP23*t#Lqj*@Wr@FMNNbooplog7 znY}hh(q60SVb7VfnN~(riE;jRZYkq7Lr(VF0jFZEA>+K#2FgtnFMfv&jk@EO&YDSe zHANw)H}+U{O4}oP%`xn^qa5rHgQ_RbJ*k#A?C9B(;%848@iS9`*w0g%cc)yMH#2@< zodNVV%w?9DXLj$|iNs({ZO@X{24~-shdm^*v=g@`=-UGYdv00(G53(p-%&+>rHI10 zN2vY`E7W&vt-2Yjm16|x@?^b2Xs#D$(b{b zXBugCWX>@Eo@s@83Ga7~b=@{M1bhhd0wyjT!?)v7b{R`dGE8ycGdHr?%G*SLh7HNG zv&#C&L+Wr1(9|(g6hcmWgn5t2kvmr0+26_RJdga}+p9(>jaaR`au(!_`H*N%ILN=3 zalbXfD#00wMp=V&WCkeQ z7#GyV*fSSDk3gP1_cGm?p)YQZ7{fAl2uoa}1$(cI#W;41Lz!VLaSo-~JcDrch?{Uc zj*-^0mtN@H>j`hA$;&x1M%K;SqHPSlfxIFP>`#MSaSzS*8J0{^TEPu4F4(8BV}|5e zE30X3@vxDX2IR=WqC1Uc`MVZ!?w*@Ya|}`Dv_sNmO3TqArGc|I))?;}X>N3Hu&B64 zjPn?)1aA(Fw>%Pt!HXNUB5$>|zSr7m*n3rMZb9!jhw{)}t0OCy3=Y5XMoRh`@&bih~?h50d zX-;XsG4`xup4(7)59QjMXHwCht4w|EP3WIS_UB#!(R4(7k+jwX^h*omX>aVkvsXM? zpP9pHBr%J#!ph>_(>inI8I`$qTIpHy&S-6b*c8)L^iB(QZI4-l8+SBe+wt za+3Mo)1oBLV52)?2KSh`*k#T|!Ze3U?_Vn>CQgyr8W$GSN;B|vt?A0M!+y}&t8F68 zHHwrM7+#)Phh*iE{upDbSKcc~H;jP5ohCHX9^)P~?Rl&br|$Ypb1rOb2;GuWR%BmW zqHFJnkQI_baTtjPer^4dItNazj>23i%XEziq_j5p z&Y0mJZX~qOF_J{q*fUB!2cU&D;~Z|B>l-nT1;aE#X7e2@CO5@-u#uN0O`SuiZl&3p zI^&k_j9~;4C@IYqvo!OB&z*6nA?;1G9~PWQ3?Tz>g)FKNqQ>Ih z(U@n3p!Ytqw$95-oh7Vo=pSZq(AWoh`&|j3b4|RbH752~jF|w)o>}TI4KThGcgAR2 zYguf~pyUzK9?#x^#&e9}=9NQU^azvQY%KuBIW|(m-P0;$4qeK*BS!Pg!3t^z8Je*t zrtF*ZlWmP{{~vbLMo6hOHm^n4Gq&vZ8>=~N4C$)1hR*qp!`vq+-*?X7rn;9n@!b>0 ze(YtAJ{Ppcj$6igBz?=Y(g6sgDy!U~iwpRdG*s^cF&f;mBLqXD-mpF~+LbO0(WSrzOK4 zhG=L^_*-0ysbQ{obM8IXoo0;cUcsn7%_OT8Mo!S#10Xj~Y}PVY#>Ss>>@Tm)!L%nD z{20?}aE#HTI5xoHo@<_9XBmUBcaYcK3k`0>LCqg$?#|pv3?wZr+BCun&Pqc?b}cN; zkrt$VT*FF#rRlvIa+doZq1qviP=h%Z=HeUC+C0a&#<8LLD& z?||r-N4)$VA+~x9_~tu96i*N1PkIKqgR!&TeBUX!XXNqQH`iWg%t0D;EJ6F2=Hf}+ zql$I!Ip#fg9P!)<=XyZ0n@j7R;#$9t1KBPE+B;_IFJGY|v z-ot)k#OePS_uk-GgTQueto;!eu;+@qs(CIUuo{AJbqUi|dhX=kpSGmQ3L7hhVPorLhNmb6D#kWDXg@H&#N-jS&_jgpu15O$%!`E)ASG zqG;gTtHxoERpP#un%S^zjM4NvgN*eEQ{gQvprM}wY|vO+?JUI& zlbIsE{+=7aZY>bbHiG;{OydnO4*l>o2X1Bw<3wi#q24}MhS%$9VrEQtz`DIW(IjovuS?JEb)-ndPyJCL3G45!ZJtn zcgkZdZw_&Xxb`+&9Scut&5YT%0>aeJ^YLbFtgjh&@=@DkL@ms<&KGum?1{=$8>^6Z z#ASmV5=`sdo567=5z3OL4*rbu6K2kwzPrP6@J#`hZ!Kx7xWc6G2|I3dZ_(Q~c5;3m z%OHQ|Rz)1~xi(K&-WTU;_DJzrI?v(Ty2m`*j5{J`&Ha_Iwh(g};}>AY-rqjs$zlt0(*a2eYS zd=BY`zgO(qp0j;=jD542w{GDa8|-8(g`PNftp8Y9s(CM|*}KE&%;S^E`8J+mIUe=^K5H} zS^u3w|2z|p$O>D5Esb#Alh)GH-TNPC#6{w{*9!TWS&t`773P;0H1S=jHEygytrw;+ zf6aM`zlZW;+yS*92rH$0EKr=W!(!eJ`)hCSAfdM7ywhHDntv_W&^Gg`+u1WTb1WH& zmZFH;+j(O>C(VU5$LeRvX@_kt35B+!*z8@CFL92c!9Ta~`5r4kFVA(f8^`EP8j})X zq%`-G#x%o>X;E_~35pQ{=;4~H6=ThXhaE$1&ks@kWvAuOn)Xs>AGy_VL=o+r*Q)WI zo0n!z6{E0YoWPm`{UyZorW>Xj${m{=XHC7mKbDx=%}dQEtp(c?W>Q=aSu|%QrIE05 z+Q`@;!D!9=p1tQN_#cCXBTbp+yfb!n+<~8b>2r51X{0o>ZtWPGLVJbu_c`{kP7Gtx zU~UYCIQIm0o!dZ6dsJ)BagMY`xK5ixuzE+;u^ZQV^qyO1e0pS)VY6RPS56fMJeV ztGV`k{@?k+u=hOQSv#3+uNmr;2R3~e6Rvfv$jlH@4(u8;a%ar=$ej|#(BB)-BhOLw zoR_rGSwnhrE_La@)BIl!D^fOWR;x>IOli)^vFf$67VKOT=rqJ6_d4e= z*&91TH|{*56emQ<9^+$q&0YO7w1u%|m~la~VA*?d?YS;{uq;~Fb#U8g#*qX6QG)4`+y4K)Y3j3vOPx+CvGOTak z+oNj?J*Y7Qnnq2TWn&MF>a|up>yKNvYxr~Q-(FMhaqL~iJ>#Z$k3n5`Erpst24rJS zJIy5x6!$XbDA?Ic9C4=+looa*%$-YwX3bQo5XK_Z7$a(MWT@>o7qq}Pr(9W@*~oJQ zZKJ+dH2RvGKx{AIfxHHA%osbUbw+INGM5B)8LK2a%(1AD)~45-+qr24)QY+y{=!;g zOBu1mer{QXlSZK1NK;*9M`)xna%4@OW7sSVNyWL6j7Lck?lmOQ%rl2PPK@C)E-oq0 z8CMo(Thrfd?|9`umN?8x^GagQq=h!84trdKQ+W*ix}EoqS=ks8CMUfMxPXD4~mHHYqh30rzP?cJ>&c4}-% z`}1rMgr>0)7WCMW+C7Y=$`Th`f1PtzD8>z}vU6f?%uyRCMjhai)|TW-`!;+onCCZ# zc*M@*CqFHz+dIZQa+uMXB#hC!9VU7%&F^8u4 z%@YDXtlXlv2hQIgyHhPhbdH&}aA?_+@?oyHt2P(Z&fNoBZx0mdv(nPwSh-(xE@5#cyC##0;fp(w0T=M|=2bIFr~a}Ax!yOYl98QW<&4Q13ZXZ|Ti&8)Ci zT2UI)cxjAiq&emU=iB*AV1(KBHy1i%UO71@PNb@vBEaMslguz>jF&dHg6mlWaCQXj z_r7L;+{x*?Jgoq_6{l$5oUsu%Wn7z(Hth3Q%ldY#nAx@_^zaS)7;`Kwvb2*R>&c;e ze$7prxAKbMn|aPSZ%OwTrz+0cTYzWnsH&ERhVPgm;%^THj=3iSN)GFxA_dXpvF3pG z%$t!n&uxsk_bB0-0j+Uv74?w^B;uc|;AG}`+7w2H<(U&>4KovdE_9rb4;8DM2E5$Z zb8Kfvuk?zCpzQVd(JRL91o8?~B55uxu(5X<#9h;{dqjoynWkn` z9*a+9#2u%$#+3D5Loa!6$&eW*OncZve}0X9rZy(z{D=eGz&KMdeB1lsb}a#;lGY$+ z9J^C~?M<7!2H4Y>;o5%=fsvkes&}1X>}sZEnLSqo-P>VoZYCj$ImckeTw6YN1nIgG z0<2<5;gl(k>GYH19Q_{Cy>di(ftXef%pW78KL@eSv6s%qj#~?BE&cVlQ!;%Iu_R+9 zWt+be6zW+qk#sD~^*82{?g&Y-d9G0Sx<_L9m@7^yt_-4;_hR20lUIFiDWaG0<+zp{ z@fo=nC?qIlq^#9BXPm>DTcRg#d80hnn&I8C(s68A^1jx>(VydSbWCyhyZ5Mbo;!mn zCxwa?CYZ*Y!_sf9^uaU63{Bjj@g+_)`9D$~T;3z;b!I_pcmbTXAFw5CM3ZhYZG-x35^o7AVd!HIy>+2 zh&k8P$4RSsY>l~(Kc>R$9=k+f?)>1fr)ozYq2gfgy`-DR#!1f;hvGM8gv#5o`ECu| zwX-7bQC~A6G^Jgxog!4m++n_PFF@RpHW+2w>GO1_iTl5|Y(U--^>6Ogs5K^J?g~MA zD6Zw8K6kv;52NopE|Jc-c@Px<||CN}z;u{~^0 zd9S<|LekvB(|RTthP75W>6ycLDy4m`Fn4I=NmFBI3~k)G#?JT4gRpXh+1E1j)^D9T zJ!X$ch!y7E#vJ*a_z_=^cY&EQZB{ z6-PekTqz2BjSb7ZhBW?Onbl;aK>fH^WLsVLO* zBUNs!yuiMDY6nX%3;KImbf#8S@=)tU2oz;?VBPQ<7}04BIw# zQ05uggl5Hgho9z<-P^;VCT+Q(8)n%5nhQ}Zhd}8&qUPn9i%4OtMY+7!*xX%n3o_49 zsyBwVz@1~SJZzxWoCi=;%^4SR?`^a_M=tjq$xCES46czg3db4?pED20)VBxH+ndV} zI4AMFJ_c&gO-Vdx%|No0*J{U23o|RN0iZr-1aqI0`gslY=d=bgQkWCEdhTJs86sqQ z-`fWwOoXRB*2=;hQ;%`(;pmwcX6p(INpVHR;4%lqOi!WvV2u^+5jKeTnNtF7?}gzr z$5K|`S*U#MMASBC4qTZV?{sV(_a{8#Hq$ik9H|d>>~*FT6NuzktD|S^f%gskhdoz3 z_Me;9Z>4OzwB`)U*|XYyEG_z*Ruo{~dn7Vv441GDC6m4*c6pD9IVdkxqZU%i>7AR< zBM!mXzVq1g+hfl@PX*2r;*S20vwt*gLBWsl-8bfD;9BccJIrD9F~?r@%5g_@?|8?s z1~rYdW}Z|TTRU*2n8+1(^y}KoRBTS|&pj4KbKgN~Vr7(+H-V;`4`4|~=HZAOeKX(ezi~BBb7^A&?2O0l4V$#r$dlY=FxQIPRfK46YKx(h0$&@Ed?u^4z zcdkXFwk8VN4vXYHhtba#XYkq>AtYcN`{#BJ8Pt#_ibEJX5PHwS&Y3p6-$*-PYsYnv zyyxO~9eW{QO}U`EQ$kQ0J3DXh_3k+5+~}NJ3w-aGkv0Zq-XEJ&X(dgR8AqDk8Z)9} zEFI6elLpZlyD&X%2)#KM_+*&7v~2AKikpWx!;RbPeq@!nmv^>C-RsF~?I51G)+F@V z8Pj9W(U3Cs2<0A={Aq_Mg`5ZW^O+O$a3+x!mjn%%E z!e&})LL-l%kw52zc@JwfZ!baDIb%}Y98oSj<|WZRQj|~F5%_a1aninWphk(4cXF+j zn3R(A(Mr=`e+{j&9Y+q|h#M(yO@!2$$71#0^YtT4;hsA}kZBE@9e$13gb-smd)a}4 zd~R9i7l*dzW3jNSZ+ce5?_{y_WdM7-L#{ zCOC|e84$!zZXM|fB zyEc1H{k=MOeEgabGiDA+=e+j3*N)RyJna$6Hj_s7Oq&CCuBE1!=5WDVBk(`XNX|M3 zR7je$8h(!;tFi_%O$n2!H7B{TJ|?2so+IdZ?r8SAGr-x8Qygn-E#fqXwo_U=6MXM2 zroRS$?c0+gXTeNu1ugBJ6Xr(8gwB~kpek=ntdK@b&YtOcc7!m)wgWcH-(ebbB*}!7 zHZFWzQ7LT(@uiejeE1F1Ln;N)x0cs(f1GKZYm9B-w01De--A>=?jeu8L(J~mLwkSC zUBnUhXy!;6U@|TpvoROAQQRAaV2#PoJw}jJ-5G{_E}f%4({9Gvskb@C&E6Ha*2)NL zwSDZZo*F`Y*qICFYDIL4u_7RPovB1ECe6kY7cyU+i>@{+u(DDkn<2%YWN@7 zwO~ha*EVAq+*yHZFKxw^k~X^b+41^$r>{I$cyn%9zI{g7e(PIBZ%W||v#N4$jCss` z_M+hSA3?!r=YiHgW_t7AODt%Nv9~#fw9;Bpa66B+g0u!){@nXUFU0V)mxdzDj{9zS zW?0ppf+BKFaVdI?>9`b=;#1m#U^Q)_wmY_f>>6RdY>a@8G$$I+nQOWxrgV!u)&Zgt z7ZmkfN&g;F9!#D~d1#D$o*XB9;of_8Z*4U7Hxg!1$t%xd58#%YXJr3Nn^}E?k>Nd8 zLdV`igncYcs+1Gl#~W#pc#ie;n8wa#Te~+f#^BnpCUW1MlZKphJuH}-KE_aD3=2IV?=|$X!oJ{K8FXM~rL#1qh;GgU$}ojt#JG0C*4%UE zI>!jR7k9AT*~x2s#SE6dV&>l2TXcHtnUFd6fXEt?IC+n$$i8zLR^9RXW>0O?7vqF) z344QRZYhQpwwQF=TUUQ?EwZ~tDo>r^?`{v(x-!SA_|DV$An!PqKJyqYkbGcJ8JAFo)dMUiler%)O#N(jrIA0m&z& zarZv<4Ax1@EN87Hwi>q5$KJ6Fe@unpvh%p;-J!%Jj6JWghVXEho7H`U<%+%s9^y{3 z7d~M-aLoOwH|I{$4xtKlFMy-9XFTH!>rF8%P2#v#z}(*ZWOZ+K>l(vE=1rl*Bb zIj%gu;u#b7X-~}4Hx?%F$*U1!2FbKx zp5u!lO+oq)b69^FQ!`^F!N499rut4>BW?w8q?$Lf$BR4lVUMV*x@Q9Kju zoPZ%u;M%*kOvGP1v|t`PoG-4xn;NGOc~~K_Xzy)=zo$U)P8(!lPH~z)C(7!avu}G0 zQU16$@JNcG3T0%qjuVnB&zNi6V`eGc8scm~nfakLCdJc~X8dHztG_G-P0O1G@blhl zS8*?-=DOlabx5PID+~$XGB;M;m?3&2ZMg9pM+nQ>;U;OOo#qjj_{*6St#EIQ!n_iE zUk@ReehTj@EmK?V+6#-xE3TuS!uK^ z%su`cR`kh;2dX#%INRJ&=V4^=@(}~NcOH9kDer}xoaTU8Ao<+s z(sIwSl^3!`#T;P(;oG}ff9HPm?*%N*u4k@lOlSa=OE0=amS;;+@9Lk9~%4nwjn6l?8YR(ISJ}t%gIks-$8gnFn zj_J}gX3p^)=}IxIN%;_?Zp_)iHgE16lsgu}LEj6Qa4(ddl496!nKP<9q)nI9px&M9wq$Hozd44`=#6VrF%PZHHwHM*j^h(%&9#uTMyl!BL)mmr1(y_J zhHD$?5MysG-nF*yat+JYAf;URva(>q*n|6NPH3dQ#`?vXqhn>vjrYCBB=lP&KQ|-w zeeX$)mNWQH&QW0}&f%&xSKdX85u15$mG?ZBR??dBr7>e|W$y8@w6@emjnl$5jm@~3@&sDdUBG}qYlNdP;Fy%1=X3-C^;%a5l-nEYFC~Pd$gEGc6+{wFfDkNo> zGEhF#l0&cFnud}D7K#PlH zZVpZSK9c}jj7wK~OknU7hmdw&8&h}A1cSB5lxrAEuXoQ-?=d3yQrw|hd&HgoJ|kA* z-W^s*|_FJ(%l0`B`heQ5DuNGI_ExZn)AzP zY>mJ;(lF=V1L0*Z&B3^bu7&>^OIhdU=8{h2G4H7?<-vDO~#o!Q@i41BY($FkF% zVN@ohL5M$=a$pG2oqvWP;WT5m(F(I!Zf&vlIoI^Z3JVt+8`5lS{hPbfSjpK_Bw!!Y zBQ)hTi?lNOTUoo3C$3?qvFC{WnwjTiW0hl$wATsZP0h0s z$1qJDabb7v)!sB`I!w*0wSNvJ_qd0M`rSL%AjU6nq_< zVLB|4l@iwmc$xdMdPOzdu*bl6+*>s|gr(^;w_MuKYs+ek71gqKs`OvG_%kj@uQWDF zM;}AHXfJ`*ImXICSUc8v?=VTYlR9_DCC4|G)awn4PiyZXq_`(C+zgw9FwEtzGDmXrUvoHS zF5LG%gX;Sov9o5ZIkr0U>|31EdTehotTWfr@=0qEc?I0GmR7X>&jJ2mroH^0vRu~L z>6~FNiJG;y#77&;=zZ@XsWTQXM;?3aA%@}Q9kzP(+8HWtPQCHAmnPtjtH@{0A*!~w z0?3#%P;E~Mg&QWy_l?uOG0&8%7M9}l$-&ZQ%oV&66P8!oQL=nQk;6Oo=FFR;vuV%y znlop5=Uu~uCvGH-oi>v1+*4v;8+$!*B;coI{g$IIG|y@9kf-iSTB`uXjpIFc%!!;a;#6{48%S~t9kV+J(AL_c?S2F( z-8hCMOj{E}dxqGou+#43&O_*7OyH(J)@JZn;Xq;M^_-h07=K%lk9ei+_ZOC3o@|ZL zpS6~j*v#{1c0`c5Jm!+=j)|~t&1C;Kmn8W}+bM6Q(Tg#c(BzCm>@A1cioZ4>z{nds zdXC+qCp@>fCnUpJ8<%&6W&fK~@*s~D$g?NxZJ1l*X6zu|H^x#+8nDIO8y7Q1L`uy+2&+WC29&Yph6gGm;Ny+phtpSt2^NBTVq4c!UVC0As`g0A;)42Cs>=+wAf5z#^mGgAaiBl&{ z@a7(~%VCd*;hG1u+Zx;SaPLgSy?4A-O-t8h?}gL5Cc0-{!8Cj>!I+o_F!kDVQ)i6a z{jujz@fr(SJ5RyUkrpOq55trq%>c-iS4!B7i6e97ed@g@xL4l`taEQ*r@Lo_V4eYC zajyXGF*5o|4x7;_t~sDP;v&qNsqS{IDXutUT+^CM%W{uUs2u~`;2SeAaqO+;HRi5t znn8s;Zjg#PBR17pi{vq{X@j^HVC4#1En`hd<35)tciltoe=caXJYu|Gh@(Iy#{t5e zw|Kyg)01i~kmfoUwEtX-`Xi0WiZ$k{%ZTelJa5UzvRCM43v)>=FCDeD*Al>ti&A8+ z`I|9EAoE)L+cd1%)}4l|>DdEHdQY*nx3gaGT}vY&jr?=2E%6$1NY)&)JZ^0GtsFO2 z{GBo1dnYNv5{JIW9Ah7KE?Kg_66)O!J9{zAt)sFNqH0?UZhp;4qY{P^^;>gycTXAH z5`vm@PwP`?Wi5~u6QKXhOl^j>j5vl$a$hq$YzC-+bEVybJ%Y^Fo7?qkMu6iogVfWD zODrR;yoZohN>|xgCppZC@Ur&2NZP^0Xax0#pLay{-HX>h2C>JvM!d`&X{BUm38c7| zC{!L}@Fa%uw>`G7a-2JNKd&9UJ7%a*87WC_h2{4emI`8+L*96-QIjHojN-nA{=;68aeZzrl01f@-XHny7N%N% zPMgJR&#;=e(wOhc3&(531N>{J?TjvEua%NsKQMw*!nW=-LO7si(F zoT+X-j1l;@H?+iyv14ka6@e1t6no#>Y-3E^u@(a`_08K?ea5)^wWD&~PFdJ4%!RMJ z#%kZ4OX6(DOyjoZQ1n@YeRd9k=et5=?%L}Xbq@{8K49ie}y7t(?@@p+|nY&_`-WzL!a4rb7 zymoMCAMxOLZHc-OVussWxlkwxeC>K9MtEG4Gi46VlQ|c-RLwz#FeKdKG8foD*f~XOL~N>-H{AP~OHFP? z>F~T_9{d@}2`JHo8Sp^hlvzw@Tk}4~g}IOnh?}!E`s>L9CpD~u>6wF= z_1c3MZ0sq5l*6P!h|?@?&LsCd27q~=;R9_VaK7SFk-^f-#b($jzH-(MiA*A18;0E zS&h4A;`>V5cxPvIgR>_Z@?N{)H|9vIo^yU-m?>W^j|GgkM_kF9bD?DK1%Xx&mS)ilt2lI|$)`O6Zo=H@)pbsl_p@e% z-CNsLG^A0du%^t^T8ox_#T4?J7xK-E8606PXrG;@5Z0W@K7J*P<@p1YxC z?`iI}a?r!hd2xKKL9;j$YV+I6qj@fouAVk{)=ewZZ?06aGm`|zO;eR<&MD=!vrh4f z@jG*lMW(*@QotKG5NKK}88z5nN7Gbfqry!RS-o%wfX4_USlGEUu&YukV3 zl-`_S0Mp(Z<91~H_7;}9OOBI3U`8nFG}l^s$ce*k?Zou5hfK{M>E34V^xT&gZa|I0 z2YJke%M-@@^k4B>V~i=_F}EW33OfaIj9~J#5F_^GtI{ZwYaDaEwKV5a%v*TRSdmMk(Zwww~AC z!~J%}fvqt{P~;8cR&~z+#~l}p#7PsTBE_|Yw8nP*oGWTCE$PI$bIj6NgIFn#So*dX zcH_wjVmU5Yk{)LiTZ}OwJ_h-lH5Y<+pX%u1JwMX2iyd`BVXlm?nm4rlTPa;F<7aedO~yVFD(y*2FEVU}(Gyqh)j|m89N|)Cm0%s_Z#b|ZSN?IxuT@+NP7i;CY0H?w)AY=LAi46 z^_mt2TSa4RZL1a+BE-+rGiy&_)jxMM=^t78D$Rk%H)n`_&PgL@&k*{Yms($1k>GRf zU7tPo9ME6!nm&i^gPz)p13TuC;xS?_abG<6D2$D z(d)2Aq+8EX{c!Jjfw)$NT-_tKVJxlo7sqNsO3D8?R@m8_tGgkO`If$B*8B+T(mtl4 zxwdymS&vzpY%h(k9>Yr7No(1BrbM_t_8dWqk%J{Bf&IGj7-Jq8%4?2I?iVMd`kix2 zaZCxeGFL)l-!qzF?g;3)N4{L$qm+LxUEnh2wECSpV|T2W?3C67_F03vC`WO}8iIsy zpLuC+d3WaB+T&1g=R}1S$EwELd-*XBfxo|pfa%|nEPF0(hPqeG%b#;PIK~maIfjyC z8^fz;hCIQON0hIElgPpeF-|zg zO{}!X09f1mTz-dC(lz%4K%OISe1sjIz2_`k-)oz2jL@nPHx$)b%YR}n{Gv3*Ecu#~ zw|EBZ(=|t|_??q)c*dRbHWr3#SfdtrXZ3--^H%2>Tdj015!Sk6q{$lzm3(gjnwaML zLt5(?b1a3VJ?BLB9^1ZP=GEIbH<*9U-RivtO8Ok}o_s9~RBIAH6X? zJO=^3GiLtT9jo$dW>u*dvs&mMqo_G<8MH7BVgDM|p!!)eMR;!Q)0g8$)tPHTJ;c;Tm zRnamgB-U7K5+UudzaMjU(ul*Ud9EbXH=(pKcFgnK3vMS(3Bj?)Aj4bngmG@=lN{$b zc3fk#e$7;|yrweJTa#RNWu>^fc7*U8!^}I*iR+XHR`;GO)O4gl-85JDMVZ;+I`91W z-@7Vw4Eg0fSB&jn8+&Oa5&jWJ&ghsaATWpAzZNH!_SxHFCghNv872h(oXfi_&E1i; zMr85`yNF--q8S!~=A3I=F^?&vyk?NhNxQ>xrtQNPW>!Er2PV=_ z*&=79VaL5!40qYH4{AolxF5rw{LBl%4b7arGy38W;7r`JLSbe_`VrIO=vf=R zW=wVKkpnbx+bg1Lj&;&A$7t=I1ASt~t*5&8ROX#YG%?LB!;S0_9`9E#|-yJ3h-W((6GX`DNofbl1U(*m`q`BiaW>DSVYny$B z$>fuk^vjI9c6vuq=C#rw$(&QR=p0i+v0s=)EV#YFrD1E=GCyyYkpY*um*G<+cBh%W!`U$)vvbE=b=w zmU{2C>^}#}-dZz^b#8&Ekn%3vU70{_!Qpf7DZH~X#PAtg>NX8|lQI)_MjT_rYo_hq zp2l!dU-*i?_R3HkQw(U&0hE){s7nu5e69RPVswmPdvIa zcWOl6yDDP`dAIrG z)2(c7jGZ%w24_yw+9$`s!Weg6_8&V*ekX8?KL_4g+Zh;n$F-fjhXmb9yTgBMjr6=m z%zYX;vU^VLm>Ay54 zuIrCsxOr|J&^|YW&&X3BcWm7Ey_Vkfm`lKHkD-8hXCbvcvP4xJq0Tad0k(f{5!t$}HZ=t#RZa4byDGnX#rT&o##jb-q&Mq)+G^ZRW_J^HjJFj3l(l539X znYcop{t8QeaOZ)|6eC{T-3y9o&ppXAl8RvqlM-UiwWFV-eDvFsv3ak6W^c69I~OM2 z4Ph5-&25lAml*Jv%lJG@beA>u#A@0@i+s#Yw;MP1{T>4)EG*T|xMO(P%VC6lj1lxX zGNj5~8yjcO1^b&541F8h)L}*Wvb&}*Y)O-GY;7UfnIkM|9!qCzY$dcgrkux*BYtpf zIK8)b)ZJR)jW_J^mNW-aX3x98BWECw9H$!P+F5pD<&G0$j8mGkG+|ERfcV`O>_-yx)#@jZrG!=b;LdVIfAy=%#(Oz4H30I7xwo| zxrQ?Z%;P+X)?&^S)VL@3{g`=?X>1+3GRGkD**mUlO`))oC&J{-@r8dajfp#w z7W!J-W+U!&k2vyDcnkAEVP-LtwB};PTp4L}WvTEH)`V~0OPgg3C7_aGSk0P4zH81w zxHrZk*cq$QX(#OJ5GPnvNC(u~S))>DMd{Q%v%vqDQMEbEA=5l&F8kPvY-fyg;l9&I z%L;jAXpa@rwdUyY&(Vx=P29h_1Jun3(OW+zIFXxk8dF^fH8$*!Y$arvvohTDTQj(F z&w-;gLfTXak(X?ac;2)o(&bvoCTy=rzMAK<`rVT)VD3ba5K}DOp9`op&A7=iA~y91 zb2V#@@${F50hPCM6x!QS$}%nK@|`AJeVlu_HSQIHH8U>p2;(?7kMXQDwov+?BVAa> zo~^^#*(@`BuQykj#9H+;F)yle&4w!j%HZ_bo?jDP{NQ4-InI8&Ylh~fqlOMvU=$tE;XwJc_Ipz{b z-5a)RjH&fB!gTGO(@iRFoys={WYAelCnyH!vbwUi*;xaPabt66jX2Rg$3*p+!-H)v z4D=NC2IpSW^>U3gpt|?YWL$adYQ(kd8Z-2B$%Dpa2O!qCr+mvFEAf7>K&db%bbZN7 z=wVG&>9Tg*@Y<=Je1<`tGEy}5*|`LG#H5xmGYrk1x!Y+@0JNDyaz&j3b#sIzl{2Qu z%Sa(TV+F01f6e5Bk#U;2H%!FeBg!>KWyQQ!IPIUi3T*F;;*qCR;~nb;Z-m|NJhwpZ z*<+48F2%*S_O8^4OPX)ysqYZC25UDh7_cv%Kcx?T*JQFbg zO*`#-<;Az3N9^bikq~>%A&@hp7V?ZS4ld0hu(&cZ;mG@8dJoOKu$GR~8Hu5J1d!b} zg1Yn0>#A^1fzG+M)bozfIWca1l(I%N+}@jHaLp0b5SOBPTM;LFq&bbVH*D5l`PjMl z1Y?ie9xzS;@H=;o;8|;?d@Q++zf+d(OM6m$j|rx*M&8PZE2DfxeZai79MGG)D=$PS z+cBn+;LTgbZBG2VGIs2G4I5`@4Md8+hmy}6>)LkiN-Ln>fZ)z?N)` zu_Ivli?fMs%*};3^9<+7Iahx#5u~ywD#_lPTxQJxjXYx>cg^W|E-We4G(wPj%c}|? zjaiPBrvmYbLs(#qpx3dt!t%*`EosMX)z>1WG>y+wE{p&A0w-AEF_$fGH7C58$oK$1fCos z@XX$EC@F^V^0~K!S&GB6W9RL;Fy>^%9i!Z1r|@dV?X2 zGOfkdGFH&upDSJ>jE(q}r{qh^k*;^=)cL+fxcA#z07@Gh&vDFYoSJsz+Mc-(c?VRa z7ecU1T;pwE&u!(Bl1OP8Yb0!p?ZLE1GDe&G3uF%oyfp`8c3*Q;Z-k+Z6!+ZHoO{G% z2Qi(nR^;g0qYQT@?8&*NbjyoN8D>Pm_P7#e@LA&sX(m(M@u7^cF%l=y*GMAS&J)rF0`mH&x2Dp z&ur^97vA%T_4k0ZcI6!G)A2Ai33b%3ATR^1dKj%3in)rgKj4r zgtUV~e%h;>XwHnFv$OC<-YKwU&9KQdW5!|`BXV&C6`l}w0n;#&4#?jzU2(@y*c4W9 zXk1}jFs%XYxdwRm$;-n&gx${+SH9w&@tJGPSh*UOFvXt3A1}_G-8v&?*PmH3WGtx>0Jq_*q z78iWt$x$?Iq)~*PS042m!%uUr)uTIRaPFAv;yL80nZ6^G+}@emF((DYzlT51d<^y7 zGZ%JSnAyoa$B~`6*2dn*^XoP(<(0UCl-rys4ti#N^p)~X=UX|Se(m^$6Jq$t8f%v; z&DDrIgEYn+lWI36v51prEX-e%#%*s=yAk#-;~#^+JWY|uGsBYN-T4xGtQn}eCc^BT zb5nCgtogKNkj|M)qGX0VQ!7@q~*&tqvp=DZ(;Sa#;EvQJ2P^uIr)*-e(V@~?PbL2 z@*l?%|4C7cYpmd*FrxgyNO@I0WtFNKR!-Yp^V)MQOpTtGX$4uE} za1SM~F~;ugoT~z8?KH}q^RQ*z<6wVht>?Zrvgpi%V}8Vaytwud@QNF2Wha@7v{szh zTB9m*FX79d1{}p(16niA#f`dRUg;Y_baX5=);p(+ah$P%Eo~v*KX#hj+>uIW&hgST z^KM*RE2A~<%#)v|Kv&AEUqZ@Gpa=E&(< zV?t?$ftxyqj>y~ND06Jxgt%5>##-BEagSkw5{Ihb*fTq9uX)d#hsNHX`RsnIxuP^T z^!iHrW1UGgYtG@r7PI1735h6SjXat&y{%dIhi)sH18O}>vV4wh#%8(M&ILSdd&>kK4V(lUtyYYW);hv zr}Xn5^Gh*I35L7p9{F0URA=N>6B84@1GM|U~f^NM=suz&D0$ z(Oz4Bc?IZ!IW}}(oMCiwjosZJ2kJ_lV{UwpIl31^_R*eMV}BaYvf6Ih3lwsM42Ut7_+JlU7Gw?#kQJ7W=T8HFG(=_Ny9PqW8slGg@6m`iBid-l zOS)<0nbk7#g7u#%drbrCEd;^jvzKB*8T0vM4+zGw2g>c*n~*ooCA*pzy7ynhgDebf z{;?M7bDDDqFDGcyJqB7y8Jov@ME&+PmcIEI({yi+P0TiToXeY=NjdGEyfijObQxI| zdgf`08nzr@Sxe$#4zSN3m+V6f)4p_MjN!hb&cfWw8*xRk+q7etM%fXSchAJvH=|_6 znHvOcY=z{KrUX!!$yj&h`R6y+I^|1KCwH$^$2sD-P0928H|?;~IOe)yp3zkyPptSg zBEt9|2|y&J&8!wD5X8!Jqh&|9_d0j5^qniSYe$ryI%W`3+B27TrD4E4Hx|d*GdVjg zG2j-L?#7c^X|aglRG5Ob?LJMfaXaVjc!du zw!h=p;Q+NjO21qSOOm=)LfW4z{ABM{$g`Jr@!2c*Yea#Mx#x!0&LbaluLal`(pKl4 zI|L@|>Akc9$npvS%y5m7%DMLl&ImEkdrT;{Ii{S{nR_xVry1$FmgMD6iF+u{S^Ag; zhT4X>d|abtCTvhQOIn7VA=_ZJ}l2MlLW1S1@V@Vry6kCOTlV}E&4h4v|V3YIdo3_>^T#}K^UlLol)f= zma|%AOwmYsrs4KJLwe2IQxRis39&On{&(+Ph&3l*)f@RYam*RBJhL{|S^+(Mgq85L z^1SVyv4C<-p}iVX;?`Nqm2ibQlRpMVT3oY~ZHF1qFlHi6jZ>gyE^W3n0!;Pa3!q>f zv1VnBg{8Sx4#nGB6k#Xfqcvhq%?e?bI}g#Eu;&KM8X;der6h%zM{H`_f$3n4UHg|K zz)4!0MR!kem^Nab-OBURC@<{ty>kxTNMn&`3=PT@Hfm;^5e;%qIEI@BsN@Sv5@g2& z*pkCI=bJ+-MKk-j#t#NT^BY=+h7K68BIo-s^f zZgKFCw%GO_Ic;iB&A6IIILC==>T>Nl|B5N_dj|oQGg89)TML3Yue74RN4#R$*&l!A zfZZPpB6csmvM_c|^<4X~X(X+*5vEAU-aEK+Z8*y^=Cb==fj@l>mB)~x1mW8&)OgOU zz%eH7VM!C&a;9MapG(*;ZVkQ~ChE;w+q_`zY^FJ9UPhd=5O-$GfwhK2*4?SCYAh|O z5QYlFm~#UpFLdX%Ry@caTL&*sp@Y0*AnMO6)qO?Tu{t*H$j+=WZ1)BVeF{5Glq7|jFBxr;@-iXJE3vT?3}lE z5@6eVTyd;LoU@lg!yj>eIs}>4u;Sj<*jwjuhe?({SIqpId88pE5QDWhu3m}rFnC8n z|BDk=cI?fMx@KxePl4yTcRtRT(~5d*RF^ikAm>;iKya^=gN5x0k+8PBQ~>4!r0*$I@#WgPLqC`C0b_bgMLArT5WVPsD^r#|iGt3FS7Yh{&C@swL!c{Xf%sa-XSlZ0Gg8A7?Pzi(v_9>@Ai)$9Df3vmqmm4UjXp z#9Cf6E;?j%uf0R0T-sw@We!A*wpUhZn;Xd~E782?lG5l}Ned`_87Sr9!J1)nx_Zk;8-5%qM zY7B9~n0M~dm`l=Zi~+%M#~9I`_XNb*LsfpyQRcGuOh#WxS7hZK$GO)$#$QY4A!qcX zGv;K|8@tqL4>{N{_EKT-Oo`x|GOpX+%Lw1THkjR8gQ#q5jf)jF^7h+F$t&(HygiaE zT$+RLacv-$u;PgN+Z#G%B^jbI=K{&zt7kk79GJMGU_lr;2Q6nbqdVtv>I$pfr^NwI#n=yPZB%8ITKI+-Cyll)>vK&X~?%A7pB1M_^zSH*Mn%g}s z&UxBB_J;CILmFj=rPeezROg)Q4s-+{@{(5o<<054B#lMYG3Lb+VozbQwHL;1U(?$% zO`)S41`hPvi5hSxaK^QQ40)RS`fO!<@wx*_%36EVD38(PzbEAO97Az?t_=DzCQ@%) z!^%934X7KE@W&17Ok&0vh`q*OXB??Ia1FSuw|1Do5RyP}pG)m+$7%AprcCvSo8&OB zjnWc_KIh9DyJ(Jd`8pE7^4qflVs41x5DcxVw+6U>8!LWlPH?ptqSkF*v+!j~jaHLq3aIFeq#Idky(jEkRUrme+0M&!i~6M`{KEYlo9l--N# zAS;i_fHKb{^|9A%#GeT|FD)so8*+5RUt>Kn%qiov$L`UXi*aE_$ksR~cxPNuO>%El z)v=<2@Ew`cVk`mvb4}gEjcv0TN3O?QLppV@ER~mnJl~ixbTW^?@;25^?w!jWHD`$H zH6vc|n)96|PpP>VhwycXTl{&>h2I?Fc+;Ijv^Gw)qrB%7*hwMSGLJ>kH#byd8e>&- zY+c!~@*3sY@f&InaKSZJSV7r?WMb{H#xoa=#2E>6bu8uW6*64d4AIdqhFSc%WN3_CV%ZCpWbKrww@0MLnqwS#CWYiT#(4eM(EAUYo0u$=nxu!k&E+ZzgZ&rRB%M{fMwv-^F<-GMnW{y^C4y??Fo#xYkI+MEL@ zXT@=ZJVq2yn3MN?M~wLvR}AML3k7(^{nnTE9^PJ~R4NZe$d@wGW7vc7E=-~JyXP9g z*n^{Vk5TNuM%-+RgA{trncflxkWyaz@jT~=){_@T_ntdtWe45R5mx}(jC($Ojam3R zG9-82L&qd97}TCu*5KKTeQ+<}lC)#~F7EM*nO69E*?S>>Pj$*XmtcC_scs^NM4UE9 z3Q(BaU1^QQ+OQ@TMIT`wb7hpXJx1o&PFsk2gpJA?b_URzff{^o?WZ2n;P!{~M4UT# zEDlhEv;v6xS<{?qCMC`_=APmnOGzQctL^DOXxt@cZ_Lw`=Dh`S7HOAaq zNW+Cb#L4PAf)-gz>k)k~nA*9wp3jN9=xWZ9(Yw}`#hhzndhbN?J65RGAIpO@Cw2L> z5*SDfK|f}1Q1}<78q~`19dac-{I!=<>RT}udq%~Ox<^dmAKNQwtkK94MnLLZ(JFa` z$>zFtj^+p>1ZU5soH%oe&)jn=BhGo0JhEi=3yC*>Z288s2OPo5gS<0kDZjO&RKwrf zI&AL&<-Eqo$yo7UW9)3Cx`%GiTBG|qE{LcRvzp7FVYg)t4BoYO)YxB>vMFw9owmo~ z`ks4SW#vfYH`36^3WOlaP7tIoiA?XmZXQm}Sq2?Y#yLW{!i9Y)t*A zy;tn;9bs{D?WFx1=WauqS!Za^e1W}(AkE8LwL3;YwKY@v{@YXCbj^^KG{RK*Por^W z#9V|q6H-V>L+BvQz4IIfJjw=kR?hkCVa*-1g^onv$l1(!CPkw;C%*Zdd7o_0*_}26 zw*Fr!zAi6^GVoku4Irm@tTb0t&))m|dgZ*8Jcb<5n#&Gzr6Kx|#*Wn6lgK^9Y2v>J zl;xfq|2|W8#+utoaff~5GluTLUu&6Yhe5Hs1Aa~%feT_RN%}XZZ2DQdC}D=}&N@dJ zQXH8YXid1zGB!rQ*}L;UZvox6M+C_Rs}ySubd8x;j?M{lQe`f!{1Jm%=iLKXKFlo6 zn#W-EpHmDxFQl*-lhj3CQ$2PMfxR0R6ztkNPjEz;iXB%R>dm|LY-|{-zlTm&OevCM zOpS&ScP{jZ!?0xy;r5nRuuw>gZ)OLrzn$Xfafw?7cn{H`zUBa`GNv}~-Gkz9t}u-i zH`ee9gXAhCk%KuFu;v)cp(KZ3gqp*`%^j22YOK)jHX>m0-OHppules7HYn7~v*>or zVeC9|Fya`g>1@Tp(iCEv)gQUUByXgJ5eG)*pL6VPWo@?|mjq}SlZ9uckew73=Kl!6 z=yxuq(;mX+$4e{maptjw5wbqRouM~(4^7-WqXxv;dktqV4gR$EF#pXN=Cubt%MCM+ zBQNpxlBWp8jkDf$%Y+=|tqQcEd1LrMmVT~~cMB!NLVRx;4=rDGm>>E=EYQ%->G{@4(Tl3j0 zk4)Q?cFOq-R#*qcFnE3zQT^w7!x3U%`Da!wmkV- z)6`{WaITwnb~VP>^Bq%yWDKdKv4Tp+oeQjQ%xF6;MfE<`#>*d3p>$06;kCB9=#7gs zaV><6KSRn}8}pQC4*}8_cTQedY1lp|K#0HN@X(35|2c=?Vb6PVHiwed6J0ldDqtVCbSaXYRgx<8ieW?DIwW+iyswevLQ3c1g5ZfK%A zrcmKb%gJ-Cbnv_OX4l!tl4>L&m>gmxN*K#deQm}0oX1*Mi6OpVhkWRj#z4|uqrG}g zb^bj^_|Bb^cX4f~ueqmEQ&~aYHmz`<6-JW&nPcE_hQXtf_d5Iz)9)hQrwot3r&7)vIRNpV`x9aeh2@a)lFuAtL~&+~ z_c+F6#b0x`buE0nxt2^xA8RBkr4{o(){tCYyALd^M6{acd<+=rKgXPH zpTq1wE_wd3m$rPEQOYBSq5Yg!me0wfAbbpM)SPC%;|t-MC+zv<8KP3jT+`sV<>M1N-rju4`&YdICZmi{}KW4V&ayai@BYbwIWXF~U zNYWb%Q+F+q%o>8Q@tsk!a&Of8le4@@T3fh&XEENtmNNTY(?uanG@G}EPSBb2c4=g= z%e4m>dmEcsZjH5wF(yvr&ZF^itc0KycgpPC16*sZg})RgTI$|$1}TU9){HC1BxGEG zV$F%RJGYqTPE#-~ErrLoQgZ2>6OL+(P@ES=f2XL>={ z0;1np6H6TnKXpc(;=VIt@n2!pY9uZ4AF`T2pjimZU~eyZi#uY5)d$Gc3A4s8%t@w| z#*R=KyWA;9q^~|V=)xUS*es?6x4*Vz^xt!7bOx>NzD5M@2oXOtOniejhaks^X(u!X zh3T-u-en)@#dd8th`f^K&>Dk!ax7)Ow=;^>-E+NY?X34Q;uQaw!`*XF^tLv^p*JR~ zSWolsY0Tw}yrUrRi@`cIjg9agL-^6#$#Z(o(bl_nc<#(=YC7f#$UcUPd}O7q^RU;5 zYpK3Y^;Tnv&L>^nA6Q*7^7|| z2Suej2bRg-OWANN8S68L_{?2lD{gF&o-}r#=32}7XbxSzy7PF@oHGq^F5suNR-RKH z(b_p~RqzzEOvQ;a?tWxhhrivwU*J&F2?lpv8{ioFWVi{Fx_4#-Ee7 zB@eLAHTJM>T#Jfe3F8!M&SCl$GI&*tk*;HIK-xRA81-JGwsEi6th=_Zc9;WrTMJP? zB`;yNI0FvZO}oi4&w#;~Habt8D*(g`qs(KCwAsG)2-S)yOJWW|kOyhAwx&?_oAYfd zFBzFU*4*vgsVHhqq4mFVP)W`c=y43R#6ISh`xql3e2yX0wKq)3&bvw_PhIo6A_~b~ zd$1=Yl(aAo5%|4llwlvcq(8@D-@Z~RR@mc%ZEhXqxq~2PjG5|mFYSZ8rv&NB`+GDj z$gB|pQuRuiQD!A1fsq5OT?(W3YVOS07-MGJp0R~wCBf#uR%ApO6D@PkMBuYW6v^M? zzjZD|oHb^`e_GjzbxvrHwc>8i+H+?mZCr#Bf)e)0i~1=pQ1iZ$V8sYqLuX};{GLKQ zY93phF~msb5+`s(Tth8rZ-sz=Z=~Qj*E;hZdpa+y;Eorg9_^W{KPhJY=$E&2^@~}h zIL&3xy!Ir;AE~x@Mv0KS(cU&@#6hYE9!i!^2janQ7u4$Gcv0LR;lZ6Zz&5h!6l1oi-R;N^^mEPC$gacL>)F!;*i_#re25N^2R55peHO*t5gL_D(}= zB<+pow0DlenfsbGZnftSW-jiT`@1LRL7P4oQsf>(7$fa1u@u+N^POp9Iq%@gw1zNa z$ie7x?v$m!f*$4GLq>F_5uCWw{wl@zn7c<_-W$soYwTd?l(J^Pm>XGpjD*@SHbQ;h z@nAE|`Q94lY`~pk@n;T2!jq=f=uG?5V{F*rGIvzSpM$PoZ>@#7mYU=n%Qj>NO{ct) zxMA4~?<%c$w>3A`;@M%5FV3ZmG-5FM49UYzu`6v3E%3Z{_+J_8nJQ+5<1v%s_T2Lb zct`oW{HWn_MY|~D=$8bjL5QPHsB324tef=KWk?$r0A1knrE5muRm=$ z(J-gdAc1m(VV+P zeNWtm8-}3p9(!y#1u5)4XME0Hp#pgZG4&hQCUl8oXMZNZ%R07}+#m7$V-BU)mWIg0 znX5E>1zFCLlCWJ%QR`uEtjIC9sCCT?h-jtIx-oXr!-~87Vr0nwBsG^E)>282$$~o% zeYrWp=1k2CEMg6?%Dbbuep<|pjWu^reH`Pkd+k-BHx?{RpZUqPrwr^H!Ot=*eD*T3 zM&z8C86~8liIbB|=bY0^bp)b zA1kmSWI*Z`H~2va<0vO?mDCu-Qr~U5h%C!daR7Pi+n*tF$+?MV!$}Y|QZImNH!P z->C&_Zu!MKH^$@|ODSvTb+nciX!IHh5pXOqq`pQtVA!koBMorewbw{w9eW2eZ2j#r zgVy)WYgc#2smz^Y$axHbwr}i=z_HdsTpbe~H>Uvhyt6oL95b7EhcS_}7NYzPfi82U znb$J6j>lWUj&LQF%aa4L(VKa}XUuKwIfn-99eFN%#%$O&ruyz4WB7UQF!mhdlE90( zCTFE_;JpSA_t+^tsCVEb1^vChgzzVtnV+khQk1_REp8 zAg@i8zsL0Q9&7J$&7tGG2F&6}W3+h>%$PVQ*wR~LEqqM5nmPjRW7)~_W{-@fJolE} zTB%AkO%;nYcc@~VgJycK>98^u=+j@(Dk#s%!#QTa{@LTaa%EYYH^zp<+l%pN&4`AR zm!4@J0oq~Zh2JqZhQwM)%WK4elduN#oHtf7PuVlOCvOd~6efo5U&Cc>C&`zxV%*u< z(_C+F4E`KaoGY)9p8$CBp9oxGzn#Nm^t8vY^h!Ka(OWa$+ZzR>%wRYIh z*?V1e%rKQWLy+EzBlt6^S0)+tZH{=Aapdz>o^Z|s%1xAFn>vgR85-Fp-?Z87LF1CHSt zql$2^!0S2Z%zuo*wr?-h^b^-u!;d?@Dh6So9FjQUi(8&LOhw!@r<`^eYXWCwfv~by zX61>)o^t1Lvym6XO4(aYaV%YJ?4y}aZvs6KL9_zUA>KFz_$G3UT&%X7VH zNA#$lBko^Z@ee)h0ZU_MP2Zjttoj-ens{y%*0F{>^P8y|FE4z$V%IjA#4wk&Ynkb!kmNZJ&jqWG&bZ?2;)FD4nX1* z_rmrJnWZ-@A^E!#;1(OZ$&}R)VJZB6|#hrp)V~@+>WDO0jzv6&+*kc}HEn&vC7cyu=#Pi9! z{wnRnzddub(Hr~vWN(F^8x!>9h>;X-L>a^$mLgDybH#QArK3DoAZ*>4Cum9NEKMzx znKod`S@TPF$E1$11UafcXK2V@tJHAj-NLh?y2F^8U3hIUin}+y$=G@ID{XzNG8Y=l zi?bql2Xx1gw{GnV<2Gf_D9gA866wl$1Z~a*yt}uSP8bmdF7IUCG-9MvjYI2sY^nLV z7OHk#;dgoLk@vm^M)ey>urCaSs2bN`V~SJeJ*-8Dk)kZ;%ahlAsXu$KOv^Sh80#5h z^Lfs|p%Ye$;$FdaYlLLk7}t8k+XkSj5*V@gShBj;RH7X;oP#O zitG=&#wtX3)VT+O(c9UoVr_JkwT2|*NHf@VZi((aW{&NcsRk$nmDarnY-FDyA993E z+BavSSWdBvU{3M5H3QOM*vmzI$AH^BM{3$wi+wb2F~pb$KFVKv8)UDQ-m+IN(w{kr zZw}e#u~!03T^smd+`bZ+#2XtcCr`bMlhdmAOBuj^&nUIA=dRJ+(XM0f zRn|Xd)?7}PlZy1pBcmlw;gUPHVEA6kfoIJjgPn4w-q@?2elG!_xwe?Y80#W!Bw*h# zhep=USrsKm5s5yta`hf7*LG*A;=ZS-?%&}aWAEAXwD0rsactnc6s9=-7}GdTDStcd zb>kOj81#=5*kDdw^s(2@&{!K_eGcTVJW|%n91BEhgt@Xmrc}us!{j^$0jjcNtl`|5 zdVH@O@Rb&DXo;yKFJ=+avv%(1-aE*9&hf*&=48#hVyk~;jUt?HcPNBpeH%!Nm zV}doWy|XeRIM5pNe<$aGxU-W`kvH7*nM3A&&+(JJa|p_dJFaof4dk?z=6PAWl_&*e ztsYj`Zd@Y^F-#HYH@EKdozvVorJ3QFH;mq%Q}2B3*??yx4c|9o$j;A$5gJkPv z`dk|_JdSC#Fycb}%R6~#Y~;)^)|k&u`@n6d$?GxJmikGHODRU^=e{`Ke3Y>naH z5%W0w-Ls5#Mx9Y)Ld(HZ{&@OH3#O^i(4{% zrp(qEXMFX`^HOB&#L~Cs=JZG#>mUrPPIKlklD?B*=pT_Ef6XDizNe({4skptS6iJ=|x;p0xs}l1>>UznJ$_7ei@Nf-@0t5jU`{!^kV5E3 z%lj=Uh4J1K!s0<36EZvu9oRVcG|<^&+G2%afj*XwZd@r%X6y-*9oOE^8rvRaPjIG? zwg^aE>BDXgz2mz!;Pr^x2YaMo)-$H`^P1C}It($ev?J#Iio4bz$576ZM#%D;%gZYy zh`^Co@X`p$9(a!}>oDiyXkN(|b?@=BH}?p~m{9^Ir7hVPMySDwNfIP4{A3$|%ukDo zV-3mao@Vq~AFHx$#<`K4W*Xug%Lrj^-KITP_Sy^kDmAR=(!3K?bc~y6BPBQknSA!G zz^FThczd6FyL#+EmAYbd+>dideGV<&8W+0N4p9SpOf{d9x5RWDfjxPzfuyn*g6>^o zEMaA>-n~bp#hD2fDD0e@HdAcb4>RX}O*zwzM4uMt_(R{5{%`E;qcMXH_)bxoe9q*8 zy@w3v%9~qcOu5mt=IX{9Qx-EVQ0O=Zr0vWp-F!xQj~`Yl&z>V}Bju^pvB$1$PUG!* zuKYFb5b`y*L{U#0RWlB}#=YijTNw*ZeFiPoKLaA!*lPxT1yR+NH}JumQPE^ifY_ES zQ|&NE$gQx~Uc*Xz>v83Dy|w3{@EU0neI&{LHwJv!OB+-rZwd821MEuQqdGDKJe3`H zg3*|x6lH~2^D^QV*vqTTE)V6nm_m&B*>mM9EWE|CCbr#+F&lhO#JI4hWKy3|&w9>8)te@AX$jM6 zYfR1kFotfxj!9N>M$E}MXGY4Mp@we>5_zxHtG5Qi&6`PIGv*nuwn=e!eZ{nf5vF+L z$wTfXfGJk2?vw8nJH8oP64gyoMB);#u4 zo0)Yj&HcD%e$b4mL4C(C+8F1GZCryfJtS%AJ+e^3n`2CCvrtyfj71= z+zl}R*xIuWElik+n8xn&jeD~p#+CdO2W(Leli(!`k*%Eu>fFyUV|wHuw?3AR&zrk9 zeb2p`z305@A2E9}gqida<3!fp(`9`RdCI@C#Pm&@0QpFp(s4|MrnJJ$${M*YBdu`U zGWVM3hHUfNyC@@X!H&46ZdF?ujem_bjyU#gf5lwI84}FO$%8R!ZCTs7=CZ*V!x(4p ztj`|TNd1}vM`p>tX9i^b8E3TZ9P8kAr1iZrBjS3CQAuirrPDVTvPM`l7A-Fs?l^Y* z#$6LCW3Hi(HB&NYPHSpsWZA>9(-!vItGYQ3jQ@z^Y;i~Ya2i=wWQSq5FxHgE9dSx* zPaKLIHW1^Di|;0l$l*Qnj@VBrRV|ExsWLX2`d%?{bPc_c9cE0{o@@Sj1$>K@ru6<^ zi?Vi1X|6vP+T$MSHFGVo$us9hw)9z$t52DzI)7jovxk@06Qb)^YPYL}X;m%^7pX%ii-nYc7G{7>DZU zU)i*JE`8iS=gv)-L(p_hu+oyl8pg|eZY9NWo;m_>=A0RII1JtOnn%)E+v`6m#JQ~y zSFrBbvmyUKPsX1^Y zh3>z@w9VX;FE33Q{Tc?c%Z)R*Z|%t4Hl{w+-)jpZ?N!ORcC^(RDFk%K#mYGH`ph4x zmuxL0k~;>L$Xt zW+bby=05hAE0}b}`I9=wzVn?cU2ld>$+7pQ)fjnUUmMcuU*sLza3Mt4oAwKazz?3iobe}t&>ljl6b%{$L6FD=Qu#?otyvwkN{g_$)b zo=+TM17z)Sw>y_m)m$lad5*Qow8oCI&XLHJcZkF3Hqmo`ex zYq2%2tdG71?EFg^VtGw{nKRP-`29z!U5W*Mov_7>9`Lvbx`dMGl+dj^?+Y#5GQ5$Q{d+p`@I8x+hoP(`&M5Ko~V%W@G>)3kin2tARG({N8er|0Y z>AA-oaacokcyC#=7lx*DS__h6tx1|4g5c(lQ^7N24a7I&UV2#TSt~_%@cAC;C}j-Q z-!TW6)X3|qcLfQqn8OBHk5fijYs-_FBBnURxYJ-% zpG(Gh03%u|s)!~^MjEnLDl z7EZ|-JEL$XEQz%yURg{N5GpQ(kRB&8PMGWTCI{)KIW~ag-)W3J4wd%0_X70XgHCYG zjjI;7BJWJQMQjzsk~)u#f3N4ILDbQ zy>Twtw>u-HW5#O-qh}hgAsCAC4 z!nZc;;n_>=a>bp;wWs_-j={1y%pJSd#l`!-COF!TqjWMZu!ONzbYRN?+Gb3U*D#|@%i5Wo8f3LsNJ>)`1IS`*ko;e?zDo~+}U|pbZjloF=LMWTSF#wCsECpRtDT3qp)F4 zjFyxaY{8EUgF4Ur_1J5}Yh<~=n)gIj+AA@3$5fg&23r3au@+;D5v#F7nA6^0`QEoj z1k@Yb@n}zZs-EWb>r6wWb!REGHTDedoY^;gf948h-J|+#q{Y572QtLm^NBYG)Y&&z z68IiNZ*C2=r#=!Y{9SvsWC3e>?787Oh7{fkQK4|pVB8whw-cveutu5`ecG69mOY3QEVWyLk_`orf9%;*i5Pe65_%s%5+#g|DaxJZ) zzM?$Wj+3x?tc8cMwnpV!Y3CptVIg~lk*`0}{7_gyaeOZ|%CdKY_gK>zJ<6#dp-!xL5*%_03XwONtv_`z>S`l(T;?BZd zOPMZDakV^BnrB<{R3Y!_>ojAgavW*+Zzk=wvW85^SX0zAge0Ul)&%OC%Q879;H4KP zw)~z^-)6@E>6wP2e~X)`FK_wfGRCatiK_^5OtJYEhYtIV89^_NwTUouQp^cMjxFrX z?6Y<(=G+4dG%bnDwx?FqNHa2VuLaPT(@J59q3LXo%F!8-DLS*Exqa-rRGRBkoa%zc2D@cx4H&k%1H~Zaf~&IHA9AZ8$*j@u4w)?29o26Gg@d(p@g>sxL}=O z3UqE&I{QpAn#3twYU7%-9uY;#D)I6W3Y9dL)dnNx#B{~1DfAod!T$yZGtst8o`?j z@^UOKnXz}IM%#HYJjd|yI>t=MSlgo|E_txKGFVQVD}^?!dBY@-4Z zv7;o?A1N?#EmXO;QXbfb*5BJd^I&36u*#7K(#jwC`yGZ@-HLOGV&vJG5hLKy-@{vJu5`V&V#Z~k>3$%tdHkPq zDB9l>7=28z^OJKf{u#SWWzBKiKQ@l=+M`)^?RCwwQVwVwQ`sdi%8#jOW6mh>9D+*5NRhffrn$nfLfZ2SF`^)aZQ7RxVEPRkgEQnYz_n%s;R+K4 zC#}3UZ@lHY*PikXJK%LJosSlSGD=G$PA*Pa#TFKhM#>v1C#6ZyJ{R&<8uLqH&y41i z*0A8&saAHyL8dci80;RIOM1qQ))eM?^&iU)e5FN^9*5ZUjw=Zzt|^GTCcx*NTTN-L z?C_L=j>u2p1TZdrx0W|dVZL^!K)*GY?AXae z-6Re1^|?bZej8&AAgpMjHYb*K3KN=cVTPs_DsEWy6MH+16}fww;eq`jRZ?A%>T3@h%$%9kc)bq+J1ZU>a& zJqH@e&hs>SO<>C#SI*f<**+nsaq7J%sN7h&r*4lC&Og>LZJbfpcuxrR8%MtUPm^+C zkENrXMsUtpF>@$Po$o!8M(3UB$$PGF__DIRL|Qw{EN?*l9F{cWU5k+$%ic63LGQlS zyndTm$94v+_B!&U(;1VYb5HfiyRz2g*!z+pChg`Px0-3$69i`NxcM-LUee7=+HcLI zjWw2j?VI!Jb|v_qlLxxdm@6S>j>+IQHa>kEtC(_xAhfbV-tFI0?KqEp%DUD{-QQze ze&oT}x8^pTqkP;=(LiKu(cZOIg5yY2VmFSVrXAv}`JBTMYQ^b|wFipK->Z3Vj~MPc zSIWkp3yXM;0R6wG>crUl)@`ng?>=Tu&|LZLW-r;QwBnTHSQ7(3W+nJDM-=i|D>4~j zl0Qda@Hs|~%iTeUJWo}nHb+oh+iQzxto8bs*Yes~tFwBI6~;L>c4W(&zkbf`vOI?5 zl^I?f|bL60dmIp4<9n;Wh zPT;AVM;6OnA!%hULGdtml13eIdm+X_{2fM^b;zL@chBYWJhyso3&{^}Co%V&0x;)F zn`AWYf#$ikM#^2`BylZ`mNX{x^_dG_F{d5DH#XLgC+tL>!~Ynk8gWfayJrv3@w~E1 z^&PRGZ>=c5G_zRpp0T59j9t$$(-ic`sn9xPVWg7Q6z&W=VR8>t^fs43}A=tmL%CpJGg5O0qHxlB-KdaMLg3@E%>31qd*`^UktVK;j0Kr{Zsd`WR^4#;ydXFupG!hi>9ZShLZxP=f zwhm}c!=`mabf^`=8dBJsl7EHm&XtCmTpdwrX$`p6kS2Q3p6Ts;L|EMy_n=vw%S&zV z{6<-;uV|#9wGme+ONc>)bgi(~78Vv{8>^gjMJ&A#hq~vfAETzjv-|*|D=8 z>Ra3HV$BHDw=(Yfi(A8i2FkYIh>H?yjLEYwgED5A0$A+XBdBbQ;Gws6;Bm?!hkUL~ z;Wj2*XAN1mYQ|CIog=E`9gC`P&ZWD$wuE@vi*_bW(YG1b*6+!4_IPh_f^$i6o-~c% z);e~YR88@~c*a?#lV@n^9s?U5^B&{> zItJ{|TiJPK50URXXJp6D%SCJKw52vibDn<&LDU`gs!N?QnPex~tDUEo^Vz#(M<1(j?!OQdRa?WkPkSB)Q-pRrt zuBD#7mVEk|!-s888M3>F=F4A0b$X<2sj>%v`JM{|87Vqo?x^B4r?BK33o0@#3BI2a z9{*W+Cr&ZhYj0JX7q*<_U4hwcOsU{L(tu}~QIl*9xwRbPUgBLd>U3_X_PgdHZQ4WN zAubU8w4>DK8EbbYr^)EQXVT2y*?@WNwD7wpFlSroMk+>`ma~Q~a!A9IIP7h<8kWTL z9cc+;&e_j9W(L+-p}sfA{QDbI)-uiT*Rb|#UmsJkFAioNHfckGS`kVj$h# zgZ^Qsb&EMdaQj{ha(Zp0@txPeeI4NkH>K>cJ5nfEU*nj2Z@II)Lk!r9Q+;eq%(c1# z*j36C&mx4e(zCaI_Kit-ZLj?8pE=Mc&mH`p2D0K^dk!OHK%_VZDDxbX+HU5x<1{y% z>RX#tIqx0&xN;i8pMf}J?!?I!cHZTWGjcb~wTv2qMg-?-5o6%U*lBc<^PObwDd?5rLP{Fjoj8xp zq>^Ve(TnNCaqfAvyBChp-XSV>FWKKW_te}?X}5PwmCQ1Rl4)I2lqRhe!@P6&#ap2_ zGVYk|yYe<^n_EgQ4%FS62Rhc;Lws_K5UG~O0$p6Yk9W;c+%RJ{#h$rcKdjZbu=XU* z85=h$r*z!DXNdopA&WK*iOwGf$Xl8-i8X~_<+CSv_ZZ8}ZKql9n^&UW7~=>dFTv=eCxhZ_h(lZN>GZ zIx`^p3mIi2MOCAf_lEq=A>4C}*~K5T7R8ubdNWKhp0T66RUaFEIZe5ioYxNFSTlQR zj_9x$vS{>=fp=vMq4zyjX5JfvZEnU5`#I(`*PQ#JGz|@?mm*|#S)*-zB>CQv=YUBJ zfkSQV?ZTes2Jp?n*Ei<8iX8HU{MqSmXKm4nyh0qv87sG7jE(5Db}-foYwImJqIwnx%VjJTa%$Lj4Y)wwkE`hDNS^Zd8QxY1m&J% zaXU{H>^kC1+~1RLDeQo@KX(G)4$(kw57_$<(n9M^tJ-01`QJB|rpVpP4|Rkk*ctQ6 zQ=92vXYGN5G}f|W96?(@u2uOoV;01XqpLZmb-)_(0nR(4NLT)^?bMfsFvf_PIBx~X znXz6rt01r!zO-H!woR%^th|9~{TXW?(wY7HK)|<25{l(W=v4fED3gdCe0!|KYYXr5!_c*H#Wuy#n`7_0vi##qeSGsb8xQH{6awC5VL*JTe;l@*h;=%1Oq zEv*%_w-*xlo-=zkY{`f{Q}W#m^U8C_)ttFT(9qhkPC2E~=eAOH)zXl*q{-UD+IuWfxSS^JNek(tWUaBSy+$rWpJQHVZFsso z7S8b4Io4uq5w^G%)aQw#$a;oxin~G#XIR^(E^XoUm6Lc#Sn-B52l2DDheqsOqx^s7 z;M5!YIeLbO%smIQG!NwIJQ?drSZ5^#yjyKF2l#T`N<${Z1` zekWm)JaX7pE&_A2SkbJTatA;dN&cxxYvd}~iMq#EMT+0J1%V@2H7ls9tGU9$&m z&AEm#R$$znv9E3JY2Utf;(DKZsB7-X))~@7@E$wTYt60jwAQ3)4wEZstcBXPvx-Vc z6Q+F(ETzBm_5*A8P1scsF^Z)y(Zl)qP$`-@?uXpK3Gkg|sH zmVu)@k?7&AI2 zP1K3Dc1CI&vt?kfxsH|>hVt8k<~1#_?YIIu+L{3wbgwn(u?9pGodrXaZ5xJPMM0!Q zlujv8dN3MP6r>dtX(h*C0gNt1N;(C`2nj)?yHUC}29DTh7;JP)tKYug@YMZW*LfZX zjvnCxqbS9j$iKF%h=R7DJAYLRB9^T#R_sY)t%;@|doopweMg|1&58=r6(g}AdN;ICt1ri!D2lv{2+Yla`U(v~+Q8?Q{ZfSOyfz-e^w ze22Qy5rt#f-D##Htub~CpUTzy_|FoC*&Hc3dzt00^ z2k|QzNQ0m_v!mvzcfV(C|7o!CK8`Ptzb7Y%B+Cl-bii4h_cMgi1ongAjW|h{bJLE-d#r;zUKKX5N)*@J>wjm4h1r2Y$z8~f{+$0k3#A9o!Nanamqqh`R zD0+Gd^!bS__J>6NG#erOXA@(2Bx(>#miIwlau+Zl7|TEL*ejml1+R(dTaVt#?@;jO zo{DDC0BmLLC`?tIeG^Vj`_=OOHr&DTyz4ViIXg1>SIq%Z5G}HcPeh)$_3~_L`(cz8*|lr% zJhmGmG#sQh>dUzyx;>W>hE;j3Dk6jBtPF$s4zM6|Jw9a2tC(XVi}k_TTb6ZODMG8- z^0_7#amQiA>y$lGmhjg66uPB@)llF+*cAE)*6_y&l@L2dbp|L9GtQXT4~(6tN?;;MYpV9aqnlsrnJx&Iy?2M zs>8!mQX-Ut=mL1&V>2dIWtY*YS{g)UY?$1#)#Dh17)bv^VPnz9!-ain zp9-&JW>BzYR`;RrQ`Jw>G`)K&Cl8)xmWIjqo`uNL{Bk$4R}SbiD6Iv0cvueqeJJwj zb1gZHJiR&YC+QcRQsCK-q;Wpr+?9B)o!tFEz+7pe=_tAg6V1hN=LK} z+YARjfF)(h(?G@tne)A{TX^-fw8@!*hu&I)c<$rV#~h*3J}ZtVEV2i?Z@U%5dh(K$ zM00FtntGp`1h_c4r8h)gCFUGz1=3`1Ndudv*-K*s7@kKovQRh}CR^Yl+LEka{F@dr z2ltDw6Puh^duD4B1$tLk`J6_eTp=*AN_$N1i$_ zi4x429uk_@T}xGQgSM!RKzbpkfu8I{t8O*+SmcR*raeA z?LWKe=2Z5oU4i=M0&l6ed?j+d!zptbe-SIbE!5dcRby1#$H5){3qz*4ZDeR_$r%k!RZWRI6K1^p=qCy_*vzI- zq5nC~0k~wu9U@z)U-g)WZac&0N^hG!osOr=bT{h#3z*EB;B55_i2N_NmFVKM0C0AX zUrU!s|I{}(!=>Ub)ti<<>N%hQ zMFL`J`#Gs(wX^irybQno_yM?gl#?e6i{nU3(UK-|Z@McRt!$)qQhGy-D?BnZk}&r( z843wv7~m5azw|G9a3Yw#enahea@HtGOBgMLYNAP;%qjgF)T36`!n*8iDM%BrMU~+y z?srim+O<4mvY@mgw$MSHsl?VZow*s{T=BS~mQ>f5aaaIk5=P$j7`;H@NUE^*Vl_6o zx4QP6iX`Y3(8t#RpHc@d8$~s-s1jn&t_NY0R_mN5&t~;h!Ij|~O|1TRZavs(kI823 z(x(f^RSxL`m#sHY?(-eS%tsk|7j`{j`7+*x2XDooju0vq7a_kPz?)l}24XSJ*U**H zqRM!_&hJVR)(xs!pXSX1e8sl#TWZE1$S68$L3AClw)+&EQO)Zy;@8)Fva`XsIpQJp zJ4^03VlaNVU9gDIDnqRb%k=AZ8(|-;k!{=JC8k28MzOL$*-pZYHVh&^qO3(t%2G*y zSIV3#fh-1WhF@rihqp93O`p=M#&EvM*x2-6awQs&N z_PX*r=zk~{MU@Kk(_Dm`?v3HnfU>y=KDPVsb+5R2V3N{rO?K(ipB)SDL>J1ebL>IRu|Yj4Y&~FhrEg#?bCpbl|C(XxtH2)2 z8TXqz3)SCINxhNM3YX%&Ypw!gzCTraW^+%kEwk6XC4VrCLgFsccs+T!Ld-vz!*54#T_4F6zK&>O za~b2+Jo86i5%k0gFXVy_a5Jda(Cx%swe-5qSXzzuWzP$EeO&$!LO(l+NSxemTztkZ zSfdirmJXO|`^C`R+gp*Z(nYFG|rYa2;OqDRZ+yYE6d4+kh?DLEz&*#j8Of}Y4QdB^=^Vp&C$XLVa}iA zn9k~*Y#S4xc~7djE%gaIA=XcdpK~)r^4iOAwe%N8QICV%p&9Fq2eZ>TXJx+HPsdeg zf|qrOLh_JLZR!QaKT!`;;0cM#)mrZA;yTEomaci-Ilbr7y*fYS5w|ex?uG#{<_Zle zp&fkn5wy7cPpgUkhCyq!pv2cjb>PlHMpS!1LikJvTOy2m6Xb+7+iN8U!ZT=6pXj%~ zlrF?9*H7NK;B5^7K?$UsuTN-1l74d*Bq&Sy4J#J}du0P;0*h_KT5GMQ zpNBjEORb+MX*mzRb*FzoS|3y^8YSkKB>I%MA9MCO{mO_Da)6ds?CH?0Jk)V@h*(^> zbIeF$k0}R1|NFsK9K1n)CdR+>GGC26`j<}N)w|d~ZR*silY9tDK*o9$VDiLoW?lBd zj#pT!32ElWS^Yye^Ox)?;4q;z%)1{_S{*t5l1a={MV~uYF5Q_lJ;nf{DcNr%-XGvn zic&W;TWe)8wa>8kaWvTPL)Lh_&Co0N5nmmUn^JYc#~i}9NAp{q-t9a$(RGyYO1{+} zd@(!I9mue~(e!a-GY9n*Vt9GPqxJJw0-_BpvJ}rl4!q3P2EUG3>Pj&!Bn!C1e6%ZtyBO-yw4d}rX*5$sy6C6 zOB72w2+h%0Pn-{AR|)oE`qrFHH9l;)wrQn;P%IlfSFzvo9V)y2qnQ61Mq z(I|;ruaQHj&Z1Gu_Kbgun*k2!=SuANcCN(yP`EW^q!JAdV-Bn8ciTjDpSg&xA6Bz< zoN;rc`@_e+s;1Ntttp8@sinb&#v2oyme;%sFN=?YCsQ6`Q4ithZYW)KuF{ib0O4c<^G z6l&rEiEqqxI(smAeymNSALdp0qX91&ttq&_t;=Er^-musU~1hZ%Y@e&QWA+wDq*(b zss}s;Y5F#uFevtMvnuXLU)MbfEJ3@*t;GtmQgOcDd%2l$NE)*W2=d%+PnA}1TSdFl z04m*E>h9SLWDL`I_o9&;wd#Kp^hAQi&5(+KEH7PKAw|%$^(x>bxinGCvY)L^h8R~DO zoCLq%cqpYda(zi!bZCfAC03+-u2b<|;e1TNf^DzbxB@vE$6>D)%y{>RU5oqgY?3Sj z`*;)yQg`AQ=5Q`6!E^OT_~evS%SzUxYFV?+k@j)-jH0mBU4?ox3UMRcHD;6_g63cY zxefV9`b)m02se@YtC()4nM8-_Uq^7y4#S>oK9*_k%>}N z`p~|S`&DT)b78%Y^Ap{(?+_#ULp2X-{noAQ8%@s$-rAt5gzj%E1 z;flcOk4e=P8`p2X62hIg6z*df&x|C){nD#Xj%;Ty(p|(?e#;y`Iekv7Kp6cjO%fsX z$Kuk!B!daLsrW7PF(nx6bF`Y~-Ug3ODWtIXZ~DjR+1Wulajn=sjpC8v!IzDK+p}vm zq>T-86Cc&DSz+=KPN$s1&-^K37D_(?olQ^@DOVe4vr`|pK~1~Fs$pdTTYDB)o<5M! zIb>hX+{nMwVrVBXdm~ z|FFBGd_E&VQNutmgIX!?amQpVDn@wsT1Ep%?iEkzL$`8_nyfg-et|a$mHPW$J&lmJHe*;uDtV&8;f8$ z{CF}fqZ&wN?rsD6uUn%^#0{TCs8OG8#IT6oi_HsDB1-o2(K#ybn0;y{jr=o0usM(i z@%0Vwc5>D$op$}YJ4g?PtzOm#kX|)uH0+VgI~81@*@=MVhW&+yv%ta<5@*pJA4+yn z5?h}s$7->E6MzFi^RTz=YnB(dxEvvMUt5yr`w1*hAr5E7O*Gy**bt&aKViR-tp0nB zJFzue4-+A+4R4{z*qACB@n3)XN z9?~@4YTweEYgr{yE;Gb(IP`m#MGL53|3d))T$!fw0XvQtR`)&cwi$avnRZSzO%`sX z+!}wLcMrZYHr6z`xWQD*8xVQ9%p>CsK~uOSAML=_hY}uJ(&XBc51QF2FN9zMHNZ*z z$?V~FvC60avaUD=LXs+(agUBaUG%`o^kklV=TqA5jGzeTsD|`g&vN%I6hO^23bOY{ zxKzUY8Mb~gs(!1Xmev{6Ot%XQ?*P)M>v0x%J3xJ#>NRz{XajaMXzD^5SZ$V>l>Q1D z1+CmZ%$mqVoO#4`j>;GL{^PMdewQ%1m`h{!`}q>B%5F(nHL;1XfRu9u33N2r*Gx0$ zm4WG>pX4@5LI&;4Wp*zebBQ|durgzb$M;XnHS+>Un6M6S@owJ1GABcZJQd@@%^WT_ zs1(n%g7(25M$~&(4D$>x4YdWv^09E-e+}*0H-~g<`M~{RSt^?Nek6`n%qSzG_JDEG zMki7p%UrJJ)YfCPDXtLb8x0V>Kd*C<9UP(bO(yjCICjiYUHsKXJ(clUj{B@nEF>CO zyY3-Ep~kUL_l+zB3Papr>_%HQ)_Up(4qJ6pjd_FoKE!P`woW9gjB7TAg;&n z?yr=DrZ3~&EgoJ{h_!jGBKPc{uZ#FHbzUe_@t?&oyG+#f>-P+@EICzr4zU!#{v~jc zAC%vy*GS_$&56N#>vLeMJ)WLqipBb;1Iob2m?+oyu^+r_Kb_tNY>UJ@OgG9tntP6j zd@uT8rw&JL`P|;hGi)&Cn&WCsWAYk@{hGt|sGWvBt0Tkaht8|xX*@Lk5V}%ZULwD{ z)Rm4@d-1gqgggYcwJ3WWe3H>AYwHwSO?iOfHy?g@uP?L5`9^)fV*gwX^fa?B-@=V& z7$p^P=L3xKa~B`mlbwO?ZhvT9UDXqpw;Z_7@(j)thHwXzJ89KJ{AJ#qerg+n(s#Gm ziSl%9=ilbdDZ)++M+*7NuNjbwelUUBCX)C2w^SWo<@hmm`+;4q7+JwegMzw-G?S=9 zqUhGo4)y#aLe|W0Vd0mJ*}ygDTtpr$b|s{a)bLj?Pe^8DYhF6d&_2HTw?9l^TIfRo ztGD+mlL{@sz%7wk!EJ43GvB4cD^a1~1CdX&`91t|m>TZUWG({Ekz{Qv;WyA&6pW6_ z8}Y`@l{bodZ!eTywZFcOt8GvoMm1N-7=a_?Ssz`{hK%?^#PYa(EV?iV7pTNUNX@ZJ zr8;_9?HXLm=FXtl-^6z<2tC9kcS4 zYtdeBQ^fot5{pz!9=weUMSGX6<~F|JJwWdF$wDXcWxG9x7Lx9GMhZ7H9k2)JGuL@n zW~R{pgV0Qb(9Uti3UL$UBEBL4wA-pdRq?@d9O|w+!^=0<6oRe}*f*IVL=3Npqp( z&u*vl?~my9f*_9Zsc)#~W{006w3vx%q*t(MK%DayrX-n~*V3xF#&`F1v9i_&owq6f zfv|enZsB`V*+6U=vsjJy$ER`kRwtZ4Yfv-~i@a@&r%^#(%kx zW62^}sAM@N?ERAMr+3bw-Pd4D3YV)196-J_5QKl7x~*;TcWCCGc(~A>oYwcAoGZ`# z%U&;6X@7Y>t5W)|XD&{*;oG29%7F`Ze@MFc#_u_I3@Ci{hjfIb^PKsd+RC%gs=nLZ=owk?^`~puxTnNif!N^d=7(t!RevzU zD7$OxyKGlSVb*#D@5Rq5ZcoNO;uf{B9ci`mVYIH-2(@!~t10Sj76WoVZS@iuJuJr=xS8 zD_6Y?%)%A4*|*tE!sLx!^#Y4Nbs2HR-iN%bt}@~8(U18RSV=pAjQBF0O}7f#NhG&u zKia*igm#NZ=4k)}=Y~0sWogzss^0it7(D!^<|N&lUELhaur(vk%GhmYqTv#|lvt&e zxpkB9-al@3OELWe>}~5imu*OGL;v%OnC@1@_WoK2E9c9>@$KX~!S zcxVFETAivQeX+3cE72HvmzjI7!}`YNH}9uqVW|G&Z60udtHX^}Y0ZQ4g?WSDA&1Iu2rp^rS7F0ce@1lm2f_IdrG(`V1r!{GJ(WhNSkr{h<#i&H3 zggbJd7;muSs6mW>F60UxE%2FqdJkJU`C)1XHCMsHY1CF#PKMyq01g8HEFUp_c8~q-fQQ2m3>ugT z@z6(N0SfoFcwcfo82N9DCFSX&Zpc?@N!rhVt|2mqUKg-pC5yX1aFy#J(Su?8Ta`i$ zI0yGZD$<~mTeMhuGpqn*km%znAPq%@ko>iU=WnwX zZrX4?b;4w<))Yk;S{9r*LE2NQsa66kLzZ2j&&m}6>f>1#tM6kc64*q)`F^Th@GVD{ z)<#D6>OFi34=>MD2Hn}C<(dE+s-7ktN>9(U4hP3|9;neMuIUMu`CMD-P+)zv$9Lt_ zUE6U-wRYtsn~NEQee_u|4bqFMzO6*hoUYz@XMrzdHlPSx3}9oY=(&|eB)6~`jZcq< zao7FH&%YzD>SLm8JLH}XhBbOCpTx={*KQ|D4jVq`%PUmW-1L$W}I2 zfVW80fkw^BDWBy>D1BY1n#cS~Zb^t2{6?pS{Vnt?_*@OTU!h84)q?>%bQMi4L&}a2 zcu?#Qg>RK}+>^fKp_25CyIxU zak4tu#MU5+lQknfy310fSc$?Q)hNDC?EIN?as{B|LFnA{W@i3(gfK%*COivzKq%&UuhOhlfC+|6M zJx)M`M@6*0S<#LGD0x)IHr2loADVutq)=Y8isyzE49XV-=vhDVk7;Tc%HpDU^Zn|P#XQ~_53=s65 zIPEIMs*0xC+r%h?w(;mX!7lHNl6RhEKXYoSouLFl70Aw&H01FUz4(iafU>6PRGOY0 z^B52Ku5gaBRy2_#hTysVV!rK{-T95f0{d9$x^zL_zqx0fDkRqD8E5Q3h*xj5$R{{^ z^^wpBxTJ+<As=jDmzQjCQ1iOY3a}H69@DlQOGM!qBO&fx-&zC%SCc+ zOoOyXHba2HFv57Ff{@Yt=8^MGTV__o0l5?n2yB`z@Jx__)7%_DBL+JmcJY%;u zvO;p;hT$VsDfpcPl_f93lKPZ_9x&tKsBOjXqE^sP2wiQ6dgXd?n&9XlxBZSr?@DR< zWPFeXAPepoehQd8KglV6jq_TJ)m!`UrznUL>oohCL< z-ua1n?0jE*saKX5?0>^HR)9>oPg!l?Gci1#kj{2))37rgOCupI{p~gp#?2wuF$g+n zs2X54WvD7-TfQgGN#_4l(8K z5R^dyMV}xqJk=QfEZ=Dy3ZkR3b%U{IqHb#wfKFanRVFUl>`7N)A|cqjtm4zzEa!F% zgP}h_Is4P0{Z3HN>Big3ptn@d6%Dh}Z`1VX>mw9h4G3onraS0a+Qx@HGTnF5ej0FJ z+_hT~8c&HF8s`#t;V+@sC>x=5K{rF|OL|0ZN4)xFZg~@WeQ%FVVcLrjVyDF9Rl(v^ z_$McZ&kqznCms30jw`eiENU>VN<9R%z5;M6LpC$}bR6(;ti2bXJ5-l--e=U*RM=8# zYlxWN$hFLKbL1HTh!h^tCJA~JhR1OhTWuIjN`%I)?^XAAyqul+3d2t9$oBCEp>s0SnCu3MX z*!7g8XB><#&MD0aLB~$3S&lH40b3H#uXv!sA{h@D^Vn%Fs_KvR)75+6g8tgvMsw@u z9n!`Wfkl*m)vS@LU~ga{={r2Ka5Hu4FR;5;rfe@lH5BMNuZeYnCoM{Q5oPW*k?H9S zyekN?^~TqXvB(}7(ZcOIRtnND(IVLO4W^sDHMPp-i$4=;l01?1S zd*t1|)ZGjfUGey)9ORRoVKZFiy@)T#jUo@$kul-#;?KgHsq^8O89~9yXQ!3@KY*1M@@7}#{H;m&qWDJO+#(ABJ9_cOCVNvLPLrlSw` zbh=wgoaw**TI2Y?W(X&cDB)jRXpghg*s@~LlgEa1PL5Ir=%h*3al;F;r$kbjEZef4 zq28xpSp6S`t9zGd0KG%`Wn1=DHyq(Y)gyKer+9*IK?aKqQLd0XjH>BC^#h3bf&9&<#q3v@KJ zz3AEwN>35e>owm%d~reBx?sEFh{^)aZlY}C1Iq40C6UMswx^nArL4iQ&qO&9l8qr? zfs$G*&;Yp%p(ryv2BnjBCm4|nAx?2jlq51771XoB&ZYtRcDuj(71)#?DW*MoZPYmu zXaG)XEVlB>NcjL;K|I3Db3M7IU!NeOE#7OPndmfPFeOWR@q}lvJ{bM=gJ0z524!tk za|^((d-&G0qW!Y*tb33%IuM&%?ETAPvlNDj?Nkt*bwR}_1z>*m-j#i_Fh{%kzT)S2 zGca~ND(15k({^`eH*Ac%fJXp2SgM>KkFhc-bgQh6;Z6HFk(-w!%;dgQZ67wu7b+*z zD_+q)wPkEZV>Ow@72y`ok8Q1`J{~I3_GcQH1{^*icKOL+20VTmfh*Y zQ0h~Em9T>oHLTKuLpsCCTaYrgNpS*x5amGKgl%XLAIOTb@zcIs$PgHpm6FInP6}bT z#C|)9GtB;{^B4SNW|;xGDn^+O?N-{*W&I{U2gLn*er$a^2})* zYO3oK>RF~(JFQvrH5~Ux)FL^JpVxAMD(FL*^UwE^wl!UHq+y)|Wi*UGZQ&*;k6TG_fgR(XWy|H!}7|UF`G6W%Q-IvFWiS&dD$iG&Jt9|%Xil+t%i}5~(w7mVj!T0Ku!P^+k^hv|ck7ZaVDsHJ_s7)!0SgAYN_crzV_R{m zF~%>*JF>Z{y|LnCj#=RWr1u}PbQfH19;t*NO+?$nJq+Y))f*uO$bw?~*bkL8-n_GK zW9&o{di%MHJT06!G)-fV+*iAwy#a`NkU8 zS+8__;4Fap-nJ!2D}^zb!xfg1>}~8eT$(3cA++2O@oLKa;y&DHtRXSQ!GP~@NVQNc zO2s>I-Ar)Z15LXt`=+ly{-VZho8ik3ALGfahb^uK;L0}5$&U2DrIi25&kK*xi9Va zaLkE>l)+C?TkCc5;3RPUTNg;`r|IThsyFuWv zzvjI|xR5L6-#2d_^nu+!y5keJfJ8Y2>c{=5608`T*5iS}ol_M_@oQXBVL-{b)(+|s zZQZk#O14Pl>G}0@Q25|R+wl_}I11v+wK?duW?0-vy{254ZWe zx-g;(@u8%5Y7giKa+SjTjoiakYES>YEtmx{*|td6cAVvGa=eSQ{orzKeM}km6K#0d z2GSRsRTjB%nQPhUuv1=^s@a3(25=+9CoUC$9+Su`t`66iQrjBeP4jh&ysNBRN)}-~ z*72VBBx~X@V2u+MlMud8Xw2D|VQClAmL641Yg*>gafE**tV;t@SIrS8Q1RYEHj_;2 z=*>UDy_kJLzHquiimU6h#R>yO?z74+CJSWdajnxN8yB9CE zYD_@}xN7f%MckL=O@KC>=J-JD-L@KEA_t5f`Q^vqt@rWU^FJo@yB92}RbbLlHDvxJ zw5y@05jeh@bxSix=d+CH#Btlb#YHT^(Y`q9M7`MQ1$(pCyKVPzRhph@F4xxrQg>ry zNmgA3)gNWTgws^mE>~=1!qrEzk5oXq_?#s$e+ADj>oR_UyPGa3Zm>!EWNmLo2B9%> zB_Wrl>Fr?DS3d*4?9ZV^qcOuL!8>m4+##o2{j#qNntT1r+Qzc-pG@%;s3^rh{nFaU zi8qQlgz88LN!qR}^#A%D)f)ehy1AC}#_FSZ%{{mCPL%qN1}sdTDoVMx*W_Qpa%vB~ zWTJHuz-%r4NWU3u*!t)_scZIa(l4)K%x56Ah}~SZkdJs?t*opNIY*!qA@SdI+LNYt zaYS!_tkrrwZrX>?3H43*8LK*_{xd6T(cegZpytHeNN+d-7LKmqc;ca%5CeD+m)D#t zs(fzK%#YalQeR7>HJF2F1@%rO)v%(c-f~RXmC?2}o_K7}g?GV@v-;bRoW`W=hG3rnwvrR-W7lKo7cgZI~O3H3mmkvT3^ zY4{wmgX-XzG4vL+5ZcvraOA8<#peX6Mprd*Z$%5by)*p11?c{F{v(gOnw+JkCITw60nMbrWa1Tztc6|tl{PYMx=XXzPAHNcsi8J z`vk={mfB!qra+E+kHjJ`*?ELlMc$Vwj?|dmbI3V1WDhNGRx$NH9XbSwKFdpQR_HLy z5|CB+8uh2P;ToC)yoQLxbk!ifZ*U_9-`xbLI_y?|ZlcydVpK_0(--^AebVBTD=46h zz1u6>59`D#EvrNSYpU;e>o+i|fWTgUZ)a1sL@CvRr1+ z9RxuJ`|d_~L+Jr8DAvi72G#~j8Z^Q_E^FfR3H^kT<}JZhol6eZjlURsP5+9hfgk%J zK+xibO!5PJcJLQ|63+-wl%JR>m>i%rkt!3hxPJr31;Zv)vXLkq8U(%LPFwO_k`|u( zVJ>LIHmi+1?!`#=CjeuWosG!fnSsH+Zh-#mXBPnYh_Bva0uA4cL5#JO(M5>seu z21c%oyROQW^Rg`;aR`F_4gc#?>i$Bad-xUAdUMs(>^3LY6tpkoV|9&N4;ndWe;vPe>z?kE zp=%J-XuGI=Y6!wL_|acdX|hGzV~EaN2#_FgAkXHuOpAN(y**nZ+n5D=TfG>}I55P? z8vhK5i*{%5ORZ`AzW&@6n4=vFzNi?nrYfjjm`XD=!(56rXAT-r-GPq7~>ni;9w^o_rqu1R5^ z&aCb6jXj-e*W~Xq(EJI8GVdq(OxmJYarf6P z2Lr?$gtuBlr@gX~UGg+@y$z98X1qjwd-Um8%L)HwY@QCNG69g0rf0=R&bHNj(;3E@f3S{aRxR}?nf-NAv*4|MUCr|M%jd7Ktrs3-!q=3dOA-gM$a9DJgzZho zEW3MWfYCQhYpA!~{Mg(>(AF8>GAl8m+l!xr3+$yWF!zJnrPI9myDenAJC*Do%t==% ze%*aevG0TN4VE1-*nZC)sy*;q&BnmpyQl5HA`s@Nac^x}LIXK*Hb?0VkZGwGdFZh+ zEfB8?N;&|7>>g!X@;Pa%Ewt8p{&{=I^^T`o*|At>uJ$UPIE+-0Dw`;jX4qOqnl!!0 z0+je0&K;{aRA6rhM`FUygt_yKwrtAGZTn7g~M66-h|- z?u=c18MrO|?@;AQPRcH*%}?JwgYDpp?*Prc3toVx^0Vf1Ma_4{If&|DdWj2-h7R%! zbDp?k`&#y-8)h1LA|5xfsr>Pa#|UZ+jE!AtHZ_~Z3rszX3+a#*RO}>_rpc`cQ((*) z`<$IZ?`amJ>wq=~46Z-j4Q!9AYky%4&bZ%W>T5J;^3MkZH=LLp$*n$|rh4FOPO*!- zw@Lm>9Jps-ahc6rh3RG8$Bp))&0hgDm%Z5qZ#I2K+5^v%-aNvi5S|Drr3l{1D-}Gz zp%nx7VZrUEZPm-szjFhUA6)O;;K;X|wZI7p(ISJPaMC>KgmE-q}=opng|6_JQHOye~lX}F* z*T|5fcNp=cWYAD&bTig(_48^f=l~pr`|VrN8>bk_zY^ug=NJ17tMDZMh!=PLy}?;o zl2fX04zq{DE`705tC8d9#wqrfs8)?2dSr$o>GC9VUqL+5Dpv)U^@P3qr)#VdMUdNb zP5>|Z#%`C~?)$aKAi#>Cul}V#lRWyo^t0=T+_q}*JF5(^FA$c!qjGXN7)STZt@!$% zW?)q(C(anYFxh%q~9iTnN+)l-}&93t@StClB5uh zT#!5X3C$F1lPHqN>bR%nZ_V5uAbz^rNF9G+fa?tCw+`!YYI~)V_Z&`rC8g1u8(}l% z=NcQ^)Bt$UPdO!1X6)%r#1%YtT>~zagqIV&sZY2#MbPUvvwFkyZqmICgixueNy@b- z9@p&UBz?ETJWBjQh6T*cl-w)7vLVj~L)*|c-lOFAlYds4!yu{_9&Ko8WvAEvoALM9bp62?%cpXw5RpF!gl9$pDTN0A=@d@6< z4bFIUS#5uH6N7-;!}rM5Y0hIdn_y6J*?4>ZJI~y+rp$uNd3M~Tzy8Y$8NkyOlbX!7Ur5c_oReYRX~A%5=_=yVD1$;Q$sKH^=g!NqH4H+kNjgg z-F?Nwm$;lh^;olPD(?D~H?pBerZsW=g>n0LinRu3!B6d>Upr}(mVjON``8gj9-@iI zs-zt!UaBV|(1`=_47v3 zo10rR_Aph0r4IFvH}`7te<|#q}Sj+|_RDy@|^3B6F0=6xo5<5wZ^o zzZTtPi|EZYQ3@=#{aFvK$e z;Z@3%{01+_MHBOk8Q7fw8qB~?7YgBFcTW7 zZW52P0(9Sl?KQTt{+d|mW=2HxB6)rt=;qqrbpQJGnfjs|6%{wMKC18%-nAUIxeGA**` z2tya;xE;04A;K8Ir*jZciD_4Q^B*DCU05T29VtVd>YiH~s#56Hq|g}XmoWo{8Hu;Z zcmMYXPhEJFQ4JEWKvvfm-#X6-c;-_!SlFR3Nw%~V^@xA8uB9Yq6j3xYt(e%5fcV^0nncO~4d?@|mlGzxdIgoMNFdBr#7E;)j+eMR-xx zaL(76lemswObxrwD8~OOZzZRma#brTW29qLSyvwCCfyf-$)BpFNpI+iNxMk!AXCym zAP5Dj8j>7a8Cn&`gVjcmV!yPGd`}ZT3f&v4ns!IDq*r-i?61_G~55iaom4pa=SiQAx%YV65I3JEM1-GUVvL&;7QY zb_8}BXx2>eTybvLvWObdoDqW<0(3g9WnicosapB-cdHMjMU@hOV8qO#YPUA@?dAO; z9Gfj7VXpX@MSBRt@DHDnX=}|qXA!@uRG+_)O1biek1q~U%@<<&sh&6FYYXSURO_23u$v#j z;RXd9{Vk)JrM*zYwNzQriXOtlGbc>Y5Hk(_PcC9Npg$}7hpF2U$Jntl)4R4hQq~MD zPTd=Jtt&h7e1UGi+dHl|t?JE5IZ;9p#8}qdgieQ+5jylxIrFf`3N^0J(KAx zn=o@7@`Q31EBWxhvMhb43~xWEo66U;$g_ONc&+9$ThIFR{(Uz?nf&s3d>enY&;C!9L7oX&0T^G+#QG`TaqXjn8vSw_OY8dKzf1tsoIq zL2>niz^jZH>nmEI2*3I!`_$hvgmmF;`VFpp&qH<*2xUroe)9J(+lr5WRr#ejz>7+|Uzky+iDv5Y%RRqdF@_WuYQ`*`k=j~5q!U0iu(H!c9)JC~A1 zOfzh2?Csq%_UcQF`Hyy_RF{>ecE()ekZNWb=NBTrM_7BDd~M*P6&JeBo*Nu#?X<1D z*F@tUYYQ~3J(f6^RNq?5UpcM~fPF8J?=+V(P1(CKajwOJvbR>%UekX!1klnuC-%~t z6P0Z&71)uYEdNa_=YNLr>$I2DT3Bo2VC@aLwc|`%3mJ(%1tqzZH*U+1arR|SWx*OV zi0Mj$qJOT5y}gHC^xvEPC5{z^8bXTi+Ih@>;-1f&(;6>L>E0XXLiE~6eQ+)1#=Z8o zz&mGn^;mP3E>AtG9%pRV8-piz4C$ynQu_B@(aRwZiO`;>ymMNi!g7S5-4I63&s%X% zb`J@PH711H-+>imEbZGhN7D1$t4cjbWcrs!H0zl&_9=wT+dcNg+Z{nOGmIpWm6H1B z3DcT#=g7vC_h`V|5wT#6ZNWUJVxl`*H?kQ3%m++GW2YiQVx~r;$K9^B213t{i-UEKX`i>|v|r5&OENDNthZBA_=x#sWlfpNHIj%|os;!03}K71 z6FS036KZp9NtwS_ChC}*wq*?6%s*n-@}8U5Wh^lGKBEv>imUQz?r_|d*P7Fvn=~r$ z4JghV78GGv^Vw&Fu+ADchRfK~aA?f&&=uCAaUOehf9F!dp3`e1?p3@!$7=S@LxduZ zP4<3TOK~)Z82K0n!pB(=>_3iO;kDLUSzS}$We!b;JkxyO-6_y|tufmaHi8nSzVzH% z7Au(K^ck!Bcds0cyas^yoV%%H&n)XQ(iU)Dqmy^c`TLnuMql1* zcOdT&nVyq${hwK!Z|ym~IMZr(+DnveEsexE*O=?v>G5+-neVYPNYa{F7k_kA$yJa(H@f?RSd~Ef2N_a zG!A*mp3JfSnvRQz6ZOKC*OlfU!Ec!bUPnU=o5GvgfJ%#qhF&M}P| z0#5fD!^vmv-On-?1X<207a&it(vRGTn&WD`^|7>;pjX>l zAa8B3@3f)<{t@kU>M_3YA)c^KLQ44ADb*>#f_MdhfwgzV;FU%`I)vRJl@Hh@NG|s)j9(T z-rXyls6GUxKS%j1pCP_@w>~bUx!8f*^QlHbCW6dR(K6c>5U+E!wWR-)p zcV=~&JM(2se7L%HeAvj#yJ+n&$F)*qX-FZDH;%pdo;GgCOAC51uf&!y#uULzq5CMT zA;`Q!EJGY~Xn9Ti(z3Up#T&bbb1%JuJEr*g3%kH}M;WWONc#Zsn%@` z>bpa}OPWK(Ye$^7F&AXiN~yhchDhNWCsg4X`)V@{4b?Ih9O{evmM-L#tQkjucMqAA zGcE+QohLfWo>^CP4aMxe7X0JdJ2@*)3GX;!UTR3W7&k4lD4q@N9lcLhdQ=@GyiH8xg zs^APc!!KsZ#<51mO`D_LDGv#g8wU>EnOjFU#W|odQt*A+6V+!AS>-z8!bTd2#&n1H z!nm@A-WWrbbEoa$Hl|AR*%QS*ts#gulLTm(DJ6Mi38}OOa!VYUsCk8mkT|yl@tSi% zWo6Bux&zM0-dlbn<&m?sr;6A$M)=Sjx0qU4`}SdMb@9KJG~(ZzP-O3sh&^`Fa$56> zVQwJczQTZPOpB^9=OKx;C&uK81Ir~X6t^%&u>9P)?0(PHvN2O`{9eOBVU01+6Ni-J zoTIa8We~@)(o|BP!76?4k^emx$a_oSYC2Ex^bl8U%FG)hb51et9=9%JPU&(!O*r?F z_aJ;5+q_^*jl{RsL{(T@8(6vCcO=ND~uw2d&P%mVD)p$yz#PxxcX%67QPh!7eOpw-+{8@(&x=JB$#TzgK2c z-C0C>t!bv8QhMqQq2+Q%;M15BJj95>dm)dNg1o2B@(W>`bq;y!v?g5Fnjy?_jOo6# zvr_(v>04_?U5qqFUfqZ*vuEwG*AxdnRo^LmZY?2|yM~}eT~UX4F5$T~c989wgEM@m zUBR~!ph`;%6@LtkmKpZm#a_$XW#+X06@wVa*r~5zM-;g|_5}Q#TODPrfrc^H{(P^E zj+nRfOc+skA*~ea7Po*-$OE42-Z469O`Pa76U6MCiAr*A=+Pa+`sN)`;dstD%^IWr zZ4N258iwZLO$%Fou8pNKQx@V{D{`fom((0~~8F+3qoyfY6i&uRom4!Cv%JoU(#b@Lt-nN!5>(A-TF|KKwJ4T?-A2atQY{AKz$4J4NTb^|d zM9wzydgF@G?K|u^g}S#?^ju4yW^P4{5;ivKT@%tGE)|x&GmcZ3vlK5)6~G(!u5}w@ zqdX*C{WycP;aa=icq{{>d8B#DxTb>ep848$jY-b3HkQs?OWc3OWses^?(|vH0N0KQ zOE!mm;}h0$TOYgSY-J^suyX?I51Vi>u2{ga_YB+!JE>$Y(6%|s9lNu}kWQSTb}bJ*>%2Cy=~%OWKaP3b zwUbcbPMJ+VuTb8(_eA>-Q~W$H)XcD!)P5T2jCD-a)<3eOS&j=7YmXs;KWEDFTuVt~ zjJ4vQ=K4aINgQ-1Rj|6|pzlg!94pVc^%Vv%;hXt;AWxm>vG#~vi1UzttX<&0QyN}e zaYkm2S&F^Z+UZ+kPIV?g$(jeE!QBbDH|*t)xTAbn4$JyEEEU&1Qgp#y+vhS*1->_9 z`u+;T5pd?@`4O_1XI!J+EHAa+6|;u!*~9aC4=taYhaAeAv2Hm;4aG3Bu)&YBW@H28 zDrE`WI)j+Sh_lynjP30*c8Xk>^Qt)Sfta^;M&DjbVQ*&jlQRbr<=Y9PHYIiHA17+* z8N*w6&wTf^2Ta9Z3*9gTyzsV%)^%Jv+B%2Cy}Q?HVcbdLaO~;u4_T75=ITgJqZ>8s zg!VEA;=x#9Ng*#V){{mW{Tmy7ZVkxA7iVHeoEyS=Z~6S4_W;{pvxF}Wjjo+{?CTtf z(Q?Od*t3%G!yCg(BxS9hITld)hz4NB-q|r{&xx3yCZzqH6FPon0Pwm7f@Pa|$!G5o z+Z;D)*BSf&Go}_>+_@!v4IRHRmwMcpqro&LJ&KbzlGPkj6m$fT%|CY@$R4AxWzOB$ zzsA7tp7S3oZ&B^O0`ko{>^XrHL4vSDVAqMGxgq3*z#T%YW?Es*XJpZeG*ZIVN&}`P zj3v1`hWbOAbCG1u!R5DuMr7Dy&t}9i_Y>C)@EeQXb0&?0o`)Lp*(2t6SK`MSvozRxIN?qHT=yGOF zhPr2D-5+a|V2!z`y3(Y_+o?Sv4b+}9Ru;ut6BA($?cY9zgzSprIba)G;ArhE>5@mr z(N6OsaE@%9G`BF+4-?ySCK0%ihf3VdBcyER9lI9CF7MszvNlc`nHr})LE8gcEsll% z6QYPgUK4_KZ>YjOCl*58S@=CfmGGY?5XsG3Mm4X=%#wp(a|p@MDeo1gxt7GiO$&Q% z%-oi=ma0z8Tex=&(ak-#$Yz_{4RkK0<`YIb)EYaGdk#tR9|BrW-}5bLFG2k~^QM2# zsL!|}F4!62j&aWT#I-igTM3EBaZD+Su=gDRa zl)ss_N>U8Tzj`d)l(WADSk61`S-Xt(&Sr_ z{X6Yh_nFq3#ai=1d?%Ukw>HAfotxS)$N7(t$71!JDe5`_#4>LoydBpR{$7FpbFMYy zJLfq3oiVIsP7#i<_DIrLk=rq3#e}?&bjwI_}YuNJ}lIPw{y;Yi0gQ3F6qlMSFV4G`?@}bMckiP#M_9W z%4d#r#J48?%$Z9ZY|YexvO>7;-IIqc3}EZCMwZq~gMKQem6{b-U`vYg=5fyjp)r$A z{2pn7X9kG2oQ#ub-f!n+vA=e*P@i*-Z3UIz6jvT$8WYAL zj%Cj}a?b0{vwJBn?Xk3D1VBHt7Veoz7b-*){E;K{dR+riGsN+azm{m4^7Q9R+gmkn zG{GJNX3|Kj1~Cskrjy6I@Y#DSX$A$VIk&z?-@&jv4wZovre@&|6ZJHW)%G_h0@T}+ z7GsV*ww7Xz8-eUzLsx#UJ^Ho>^weBS7H_AVzY)g##@?HcGp%*dI->OCnnAE&u29h( zWA^zUb3lHDIsCIDe%PHezBr@^wYx{A!5SmOV-9@&7lxA5OF5)w?D_MXrr_HS0|3sP zBV={Vp}{@}2FlCnSZ2(W#kdl*%pB8YXDyw`GWTNen|oDvP87Pc_K?Vr5hEe3iJ~%t z%=j6xfF}=4=(W0I`eqkb%<`TCeSMGB*AR0)#SFXoWv;D(7uGP(P5Xf)Wre%4w#?IB z!{;L9`RN_Dmg9^YVtQ@7q#h^6W6X;gDFx+(HK#QC7#jdV3!9=ZO^M`~CuG)`8)I;- zan8Q?j_42Dd3J;q_&CClFJpwk#TBW`WU%dwUm z+8Wu%DsD8dkR!<99BEQEj}6wjr~Ky+3v^^niOxBeHg=!EC3cPN!8#Jk&RgS&GDdj6 zIHEE&r=ESzdqE?u9KbVXNZyPQMttmL__l{C>CPEHF%NZvIae^-&5NWgr#+by!&>JW zTrm?V?9rPQ*9h0x^Da1r{e%<-5ZVd@fMJaw z%s*4A?qg`jmwa=Er+QN%L z19I+=mA3;_`d-5tJIAP=y#f&BTw!QzEg`cr=SDmdWAdVEgF>;(!3{l}@ zq}{uI3HFt#13ey8?tXZ@;@|1R2 zyPY-0y^6fXsML%zQ*loXHm|&sy_Qn*8DrOS?^M;2GfK#c!z6z0t?;tO*z#HfX&d=H zEKbeXJBPBuNm;0H=Ru(~mc0MXas0B!Y+Tsu(r|Am;xL!y=bh_5Ywq-rI|p9t+^KzI zj;Yr+#}3n&TQFj;E%zNVR#zJV!f6LB@UpiqYFp!v?at2wv zy%y&FSO==#AF&F1t-Y5s0w{iodti0P>9w>2P|DqVUUI}}oVx}j_?c5)W3J_&n#R)V;YH|}x3)6bn*nrkgmhvp<=Qb9!v39+ z3~i0Uoljc(*?w%#um$4W2|t^0oAv5%JbcM znlsJBpAs@^eA{vSer64-u><(|+|vhnjOgONH%Qc0EeQl`J9HypO-KpAj53Sf2mPWzK`-ET^saYUpg{>JUhDM&rd2-D7k2-@e+6x;n zVJuO!7xFAun&YW;MlG2ZMom9q4>5|oCNTHgi%nxq{iCBTmH)qX}r_VTm`AQt}_mfI6+=rN5R2Mj8>bEKdo5J+H;$ zv1hpb7@K{2j^L@dMsVyN>tr=;1@F8jFyEhxu6E~5xi&ZO(qG9HZmi*}Hg_!l9iz^4 zB-xvsqJDiJqfB-Mfx$l)4#C?|i#&(K)4x}Ac$t|6X3k{yw^o+;-UDrNjb-b(lFVA1 zWAAOviN>+QwqOZEGG&AziZn8w>E1)vJ`G8hzIL2$9(jEyW|4=oXJqS~JAH6RIgG#7 zj{csberBzdx3pshTi>%sEJsy>yf>!k8cDV!4$=;*eCeBT$H)fnn4m)RT?e(-hc0fy;3o2x74f7c{E@YbJl=D+Fe;|Z-xoLKIgdO91-S!W^IW%#@1b%o4bC+bd|P;j^A9vLS)Zb=#UJxk)C#1 zbBKfMBQI@|v8EP5n(41^B`BB`78vE7V~jV?T;)6R7-P$827GN%;jjzC?j^rG7tZEOlYK0tZP6abn0^xU>zH}5dZi(>o)co{Uz5mX@7cY+ z7Zm>ru@hw{A}c89UpQ<$sFCvW zZjbA*#VDnL|E<_a+LX9iy?n( zSf4+mg5VpQhj|Ajn-;db|69X!Iu10JvO?tV2_r#yY|-_+Lt<@CIh|#P;g6nYx_lWs zB5Ez|hqu-g%nxBAeFfE}x+j))it|Eh?-|aWwvhQBdn0Kqh1i=y>w&fbH=D5Q9fwx$l%OVN6B?F`jErWkR_+2tTbh26MD z&RUwI>}qYux4WlQ`xtZ3e#M!~nm2ykPb1=EO*M-)Ct79<3zRC)Nu|3NeC&@?_XRL>{wEdS=bvwiYV(PE+6z*9e*{>;mLtAxh(lO5CiLK) z!Zc|bQ!8kM$)KCp_Fo!tJAaG=zkVhphd(!r>q|R}Ab2fl<$EJab&d7D9~08>-2(vD z-QgT~#l7LRgJ@?+qjxdR5*a3;a6Gm|=153!+XZw1+tcWBF6s|auH zS^Y70;^)ePf?_YBj30t3R$gJWc`qcSo^n{sHm%vwoHzXb88K#JCUl=Vhu+dyL)2pq z5v)9B{?c1}x+aG?%(51q^d1?hBaTVJmZMntT8TO=P7I>HrWVQ@v&1&9wf{YG)ahPR zqF`)v-;wsN?A!CSVUEe-H|IE!6X;ahv!gUm3A~kqe&wC1LuJOHnlOjJXc@!3ctwfs zx0WhNp2O5^jH&$)B4Y8_1{hjcYj8WwrL?#QDtyU%9z6`v;j<=wW6hE4BqVv2x%LeE z+bj5H&GDVOCs5EH1F&j`;fxk@s()VdBxDWHp1L!l~?fWsKO2FUS zz9&qX`#XcmWedwfYzI*JxpvCzo@3~DEcLuN1A_KnL5(JiVZa%u^wiHQc{m2By0W&C zR~loRXau#~o0EjzO#3S;PMz$()<)Re>+F3G(9b*OVroeV@_I!bmom1ZUWlRaVT|$n zI98TI-z)iX4I!X7BJjamY4<0uV1pH>!rY#jVPlLy`Zv;M(%5sHdTv$I6z87!7z3_I z8M(|qPC)GzGb-d-N&ggfoaf%#M>y2H>>uI1Z>0H@GKOy9j;n|!kEoorHuUoi z)5s;v&CV1z%-e|@6=KavtC-h%<{MFOH|4C}Hzz*Q2_vFmhAEOWchLCX>)Lw8;m)T) z@;`^X`pQGmC`=Xl9*353TDx0#EC~NQa=>L?gJ*l>g^Rtm98;Q8DQAp{y0K?a;u@Pk zVl4oGbf%!s7nT9pyLQt0UMc841<8fC)@ITeA@^{Nl&3Ogd{*64ASg`{z!dlF%AKnz zX(Wxtv2(J@T-76N7Ap0f9ee;^!d?khSleZ4+9a}3sjr_zh6888S zdH697h`O-?kX;z7X*%R2t~0{o)!8AUE6sG8KEr}&9(kK{&IzrsqE!4H%bRY@(YU>) zp3NAc&vHju{g#J*$qieTdyM?eo(G0`9zjT9&9LaSclt)1!}=`m0lhZn?B3kV(q&Je zkFy8l!iuvKXQwgLmGc0~h*6(zu1Uo+_Y7Lv`G|ck6#25!lF5v#PiSt0n7Z~F>CMSh zY)zezyVgw286&SLE>VQFh9p;>L4{z?-G((rD9&7|SUPX4;5_2;^A1_|bk9}Q69=rXxP9;F#GUt&c?|)fDh%n%yOwmpn(2IfZ{4&!N1W){**qX@ zz?v4eh~(WXID15am^g<_ZB5e}e-7Y*G$YDe-wQftObEdr!Ys?1F{UyPm5eYZENY5- zA#7&N@fo&v!m>b<`b-l_&X`*uDUS88l2WAYNYSe~ZH&wmwhnolgJ5Py#lXJzQ0UrQ z5Hd``yt0RkZj5;Xcmb<8t>K5Ah9c(JW4A4hiJiaZ@a~)4n{J57=%U%!+lTi;>Fr2adX#%7p%V`?RB@UiDUZrWla>tKR3Yt9?3c{tigx3Cw9``!-FTyLA|=- z?!nw~{yuDM-y6raaGB%Ubq(o&F9s-`y|#SNTC)avrl_+rmS)`_d7fb8fyj_|G}c*z z^mA<>@itc8cL-zRI0gB&IL4IfT*0d&@BHowt9B)<2+tpr9^Xi@LT;rU{~EGj-CJ4t zZjBM%G6(2zOfyDkrrfNwHeB)F!}oTEjM6j$>}(#3DkY~8&^CvRSj=mFcHDOx1&wD*;kxX+AJ);2ByjTN#;U|spFe$QCUJqFy)TAKjYAB$~ujER*vhe~3a zF&JbGDDjymPDn`G#doe;zMGOJ{fkk-dZpc)KF9dtiJKC2>{0D8rd;|SDLiiOVc;@` zC`ylGogydHzCDI&)R-IjeMV7%lIOV1&Ns4RBMkojvNxMPfm+_Y_4JG5VN9Mo-=51%wf43 z7tGq)>%(|#_?5YnG|V28BP>lk?luQTa2{!jcE@GJJXYBBSTUqvt!4YX6ZF}bqu*_> zJ)|>+LgpBm&~T*C;58Qp=vu2WeMgm@yJN6)O0nf=4s@F{7JAy+Glys_n8UZ`w&`E9 z!g}rP#<_#+_?w&hW<~9)HufxC%Q>?xPf)}+7qs{uVWwzL>9iPEqE6nif@CaZ%$D~I z$Q+aPc8xHf8&-D1&$}UZ&dK&aW{lSxvF&2a3BWOTsLqSy1vbrrw>Q!zT^Y+5YNv(5 zxu@FpT0?h!B<MB80~+9xj!(>vCR{+;tse$R~gHP=4OUW0-N zTdFz?naHy?;QbvtPA$bH;6E02%ib}NZ{=COw^kI^o?}&UL~+nJR=9jv+bM9y-LJj2 zzS^C0#e1(Y$hxyI^&biAFyxW;zZU{Qo!gK(tXY63PN~2Zrl97}NzZt#HMg(~vD6k9 z$k~c3@M^8G>lgMu{#?@sZsooBvo;`186z%wjDd~5cBWCmE|n{RuM` zH)p|zyyBSqn~}?UN70j(w?g~g!%8s?$;gp%Mqb%lb2NpZfy z*n#~gq{)pC_w-`gGj=R5VU<4C+CrV_?<%ax>$k=@Z5jzdGw$H;v3Gpej{`Mzj9t2) zBQn+>yM!jkq2Qc{23X9?4nIznsU@o{@R)KI_T5PW zYK%F#Kc`qr9C=r3XT8-EmNsZys|0<<(BqV|GV5Ofe{PP@!L*lt$WHmu4{>G+H8su^ z_HOLk!6hO`#q6`jD)FB45NF1bsXq6v)1GVFe?@_*J>opeU-_k%!Xm|+@p)hhGfyfk zFp;yO?ry>)&$JTYo0IUP8%R|L>axF zSNPuEv+sLv(T6buZsQ)Cuq^E$J+2kX7J@R-O9|X0PdLb)=F)FVDF--i!GS-+X5LxJ z%P&X8v_F=vbWBtAY0ikilcr41n`7a5uf4}Jhi3SiL!2*+k)S-nuJH(yGIOpZ^)grL zUs(BoZX_9wn=*jUO8YT>#VMCIMu_5#(=au!+2b5$C|z2)3?bza!|Kev zLl)AS`4rBRw4n2}G@?=F@oAPBQ zafuxUiuX>_^L|a3t~3%}`b@i`VC8nw`d>kdX^lUxYnl8+4C-HPKn5k@!`BT_`ukC-*C)K^`FybL0@}Se+Rs+ zlXKYCp81o1XSKP!rV>MpgZX3bn6EvCN@-oY$8~Hy@H|Gs>YQWMYb`Cmw}%SLT04Sn zB#nr+cap;z@vvyDfx3|QjKfJGL1V?u=#cXY_1weUYRu(>F$1n+h{HoQO_9_V<}_TH z;|O}k5ZbhYdiLG{>SJ!zgfM2>QD5PMdqs7(u#YpMYNvd_I<`Ph3F%&HEp_9$w%kUY zTby7GDKTZNDB80U4pATRgK~$^sF?RS_}Pp8U`;IV6vyn_-6P^}>_y_Sx8U3z;p%H- zT%a0<{(8k_y**}L=+5iOb;X2$XQu3;HupYO8X*onEu6%^7xwm=YQ*J-kaqyt*@M|H1d+JBHuUhydHQwFZR5S;5KdpgO=eDA|25{c zdD;WPF~<$76{bewTLV{cj?}%h!=`p!F~2x1t(&{IWZ|93nPG;E@*h`((B0WtX9=V= zOu>x3@*GzlN$zSyLI0o2H*n6R^|Qj#+RJmdK8HZ0p7t`xo)cDZ&26+hGdyceL-sQd z=!v+8rgd5SwIQr*?7sGZ-rD&*X2*@5yn_s68H-GGuAG}Sf`abdYlLlyX)kTz!?kx* zM9RrlbHRM09Pu*uL`g)0+E@U>a*;GHg`QFz4`X z97C~WjuG9N=Q!_5+dF@(#rL=aCPo^Ig>#3X(>^x}-&_%?bncbU61LP*j?wsb&Y7n* zXBNSqW65o11h2ZcV%y4q+}kT}YYiEJz7}p%-;tdup|RvMax&A~lVT`kk%zlua8TIm z>^e`O#lFTC^jd>JeJ;W1G3Lb99|5;?ZE3kW1IFr!E01mOr2V%-irgLhx;rIBvXXLq zWsO7Fek_FKFxJk=*c%@v`g2}mOK@gYoVgO5=Nj8We#PwP zF$WY{8-w9(O!dAxBJkK-noJoPI6LKiuQVj8Gq%D;8WS0E@92>+Ct67e1D;oFtVrQJ zbG%JkgQarCto%2ICTLva5@;_yq&x>?pwrwjqeo$NM7AZ zE@x+CrL-25M4E{aXD0QTu)|Q$9s5rxYzeY5Q?}(=!%1|ljlZ|^6yn$u{c^?;h&Ch0 zVjj!SV^4UUvo<7uNR#<-jj_DA7ZO(tQ%FCqk=7Y)Kc%`j%c+p@Q|b{k>NU>{THDlHi8AH)7q}mz%~K zTwKc_e9v6zo!1cm3!&;|rr7J5Bhc>31Fv=D8LPc#R!iP{{ct9&-MqGD+{#nQIpuwY zop$DP%^6c9&%vRaW58qB!|Z%y{H&bT!sl7hZ*}egxHDF~R3B?>VGiBj9fCsdjoT+B zkFl5(qM-K}i}`r1;O3VmYE;iN({&|;)wgC?#@W#{W$xvSJ2xin*{gVG&IzcP_VV%` z6SyLe@xU||$j2W89yM;A_P=vL)}BM5Zw+DhHd5~K+Og<8P9=dC)9zA<1NU@KdCEV> z+Ed&EfFWiX=9i{8+8U!DX-=T)5faq!oN-ijZ}pIwmUPcggG?voxR*KCF6&-Vb~(+c zx*q1T>0Fr)cWoqypTq9$ThSjmPr#}h2V_H-v0G`bNrjPyM&ZdT-fgCJuph<-*O_?= zGHjKGoMJ9Q4^xIE03?z(%ln^IUe4p9BVxpjk_S?{cr{_cS*gY8lG_-X7ByZs*;_kS7@6$}xm$WKoDW$4GD4 zd$MAO<X(gmdjuWE%R~kdD zZEV!0I5v*d2zeVX?rE~Vl6YHLgD!o|DCHePQbZZ!fjlSt`dq8EevT!e7zS|F*s+vu z&F$qpwl-c{3*~DEDe1dMWL((0op2@Ce4y3`wKsw@uq`4v;@Yt0*7cW!;cvI270TjQ&23=rI&vy5NZyW(gE=$AE? zn01*O3U)`y;j{C=_Z}%VbWC`PIyZd58hh|5#SM(N1C~_S24rNJ8}cUaEs-!Ma8(|A zXnq9*p+CcbZrBguD1q8m7=xB%F*1}PBEo0)UkJ}_lX%~`aKp1b6fH6a;+fZ zJGTJU+M8H(41Kkj$0Yn75imI}Rn4AeYT(#8csva3-8!Sf&<*PVRvi(rbT8cZwxTlI zSd$fL=Apm7@($JxdE;|U{pi1kbmoc?g+6RZfw6W*!Abcxea7H~9=1N%o>3Ec&J_1L zCh~OBxakVhr#dXe!Mw&4_!|o+Wknq99&-S6S&L?B`|))?1c^LclKaqBf#23wl@ zs%ozx?le=vTpJtjZ$!oTo$~PYnYlo2jbQyY_Z(B1Yl3G*y@;~MF5lk+IAq81p}!}5 z#LmNOXAhyby0^B^$um?pMXjDZXVB){%iDO(NZ2zn!g1UId0|cE(-LR++L|%{G6sVA z8GCv?O@;V91~N_#i_(7%>5aWtwBnv?<7zK-n79@`c3LA!b?nupu?9Zj+c~CdOvT~2 zv&emllVCP3Jg+uq%2r)V4kzZRthDAVT~EomZp{gmxYLlqUR#zkFC40$hY02glj?BI zkcu;7XmJj)-G9eyf|}-tQH+aFW$YcSvrU5_$zF7}-}vS#l^-L?{N++2%Cb!}PKn#RIL zPD$l22Tj)xbfq?u=xW}P7HF*@>yiQr?nw*&YHkqoyvL|X-P_V_Pu0|z=hAJ4)glJFY&%g6oB}I7nI0N{_nUU&x=9T+Df(&-tBX>A$?aMyKd|X)b8Y0Dz zyE4YgXc!Tze}oOH9A>P~o{?c-rjgmPcNld`qrqqnLC~A?TYo(Q?v3IWi$=k|yCA6@zC$#1Y zK}J6B{OF(KAa5;5hdnm9^p5$+c87uIJmQ3M+JlgFhI!Q-`BF z)OU|nu^2Z@Ld|ofXs`LRwU!v+UL)ab#|@)4hE`b(dya5V<=MUFw(?nVm^O}pvYSz< zHs|p9+7q8^?YX}k$Ljdr8w7R7X@M{%uIHYkc5BYPnz`rxW38FKv{t-&ib35g|Mz9B zmXn~+oCErO&8(0y2U7o9i_2&SfY`Jn4(;5VwR|m6n%8-=i>ZgjJ%s_Y~VqySQv@p^=%F6mg2P zlwfQ{oivv^(-|?-X>a+Z6IR@En^O#E4YZInZu_|@M6sa1;t=KB^P_5x@Q}Jj%wQa$ z?_tk<^}m)#`(H7hF2*Uovqm~c3PboVWmv_Rx3*ndqa!jc=#H`GF42ffAA785jJ8H} z-drn`KaLfHHYcWNSbO7ZZP1dx*BoOBJC9(Fbl4GtQ1+j>Zexsut+(d3M$h<(G{-Dj z+k^8p?)=y_$MV5k%ZhRZ<;XFnB>bAY>~@chkh!Mvd|M+nXN*m`IQEL$jf?bW&vnSX z({SgU$%<~yVZ*k^2GCm5DQ*tY{=S2--PluTJ;yS?c4v%dFINQvFKsRUH_uR2MGc7oZuxEr;87XCcMDf+Or%-lWyVH6N?3A}hqT=5p zEIkY{fg01mM$Te4MK@dyJ{9otBdR&8q;=m=g&qW@M(bhD<`(lU;mnHHy23 z9@PGQl$r)u9{n_q1u#s0m6lxbMoSaVJx zj+&=}N}rRwJZ{l~nN}M3owNCQ@8#3H7w|-hi%@NavAL0>((%gy&U#0&{jj21#n@wV zbWMGaUw6b!v_#as)ZDzRCvD19( z%j0Y!guRWt1D@voT`=0^5Y)_T(l9HnQ+;jPG#1!f^5@1Lf+uv&D3E4bXg7(>24KF6q zrMRPJ*4b;}VCS*yI&#eKpBXt|rt#{Qr;zVn=`CYsweUYjn9|uZ!)0durV%$X++RDt zY<8C>(wqZVac_Z*8i&^9nuF{)ZY71dw-nCbYfx-0HLe+k6zNNeMRTq}sk!nlX&SSL zJno6yH^%mO8TpEC<;CxnCbVox+tzW-w3QhHDlsR%^bZqYY%aO!6Qg?giKC!(#$lMa z6PUyqYZoW)ak#nzgzyP5R%%T(o*8B)*`L{Id`5xE8pi(g+({sCjNp+L@__rEiz#LW zy|gk@&dE(vd~Iws{S#I&`kbSrA`B_iJ7?N#-=Qiigz3<+=MK!so82VjDTk2b#fHOS5t8IKr@(2>suq*g8)grIl ziNuO6Hpi~LiV&kUtF>dV+JwXwBKB6RYOkI*|3QBFe)2r`bKlqH_B%sR_=ux*(8tg& z8ld%Z2*q(PxRQQQ>BE%1DN|rDwxSTg4CzOO8U%V z<%C4&30z8U&h{^+x(^u%Igvk}dF&%PogMyA0=F|zSK9rbMGwvwzOaJE$Npgj5Nx4O zO0u=@zRnwGm(-V9QQn%EWp~%}q>KE+)OhzDC8Y`iE-Q{D(*^50r|0-6>@KxZ2q%=s zsc7G`UFdq`ug}#i1Nh}<^J7Ou3M15MIf z>J{2U55M-x+t%uv*fe>$+-qgb_-A<}>(5j1uIXyPlrfRR(7vj8qP9EPY{(ohtS!)- zC-nPtq0CiCuV#)!uGITD?C6O5e3>KE(9gj{DeTgg%hT}M7FX5~Y?pwM(%$J@KgsAh8zfFf`1S?m$Q8_cc^}%&@P>EsCkEgZVA5%1j%UdPu`5oHqN$5(i+s<7KN$e8`xjmbUtlTC z`>(etPQg}dABSS`iqyQS8K9q;8~*O&y75Pv&%g%b?&V0AUY{B!Jv6$q==YSXU+$&# z2lm3p;AoxMM;VUBxr!VLhMGgw8U9`bH6C$^gann+cRLRqxE!cuySUO``Nwet_-PQ5 zKr{WlmR}8>BZ5}tX1E7<7|bPc7+#8^-LyitRo$KO5NvY?cfU&yd!g$SoM!yb?sD7Q z-8B1h<#r{g3kepv>P{!0qriBA$dza`7(j)N@QPL7q@x~!^*#4J_azhM1poD+G9o%h zi5{jhBiHh>6hJg7H$T%h{}My_6^g$RInP$XbHv|DrSO3Im!7?0~f-kev{Bf^R&tlN4XDuS-r(;U6|6Pu(%-GR>>1l@E z^eusHW%O;vfCZZ`ScD{J%ov7U5^l%1@p5WQWE=c?Z)@M?wL0? zmOPYiXq<%Fnx@f*xV&jo_%Jk!bw6}-W-D!F)aW&27-*Z$)Gu zWjlNlB44O^gn!+e1HM+(7fbkDpz=PBKP)g_K0PZ#X`KfUE=m_VA!F1wj|snZp{|Yr zG-`}ys~xLqKd7mBX`!LxWMY$`k23O>!)2hpPr0<{EY}?CAC377@{GOAU2ziI z&&^})|KvF96sI`y&AnHuouS)#mWHg^%kFD6$3Wdw_6mzn|4P2hgw@e7)kA3fx2+5x z`ySICU;h5)HkBDU2hL|3@|r6Xpbr`ttkEv)URIidPw*A3&6#MFK)7n7)-EU0U0YF_ z%hXI6JjTe-FN!ez7>M}Hl89l|@$lLFXG~F>d(`k*S;*+ktyPcfvr*2$bn^Jl9HRrz z%80eL9_)>UVxDKh(P&3B`%t`yF$|Vogo6I_GS8C<6rsN)duH!8M<(eD(>Sd$qTig| zeTUE^$tELFf(6o`w~kJeu9gsz|DYgP~-C`XfB2lYH$ZDzs0?= zAg^g%#n7S(UWR=Fe}IineNOZ7pNw?~aFj@|&)BTnd0>8*m-g7d`A|76r57=TQq=_SZ_T#beHN2-)(JI-XoS+?(#}T*F^#L5ezS7|V9PGv zeWL}G(T(#nmK$ZeNtOQ8DHFB1L88$<9let$m-*-R;m#4XQvOT3_H+Bi@KvBDLwqwr zYo5TN=&{#aqM|+j7o%PVzrP-y*W;_RxC?XNk56QsnKBoP}evYSIb*v!6)mw z=t)P3oYK@)>~(gJ4k9Y%a~-qU5P|G1qMR+UZtLKPCgroe==y(Bg0{!{o<)#e`&+q^ zB2P1_O+V`G+Q*xV7gm`6BulVkRPMiQXkxZ%@og|mj%J(v&pWvdX9MggR}qazi>f8| zOZp;`ipArz^e=xM%@fo-$~Ci8KK#34NqW_+a<2kNL#z&yn!`E63p7ka>Fu`Key9qG z`V60z#*BjA2^5Fy{9BGz70}$gw3nX${oy>G#WW>leEA76|yM$hO6n} zo)K@cqTITSPg;6ZO#EU`G8^B0`kPO6C{Ilj`*RJlj~uQ7B!1p*hs};I>L&VZ;HF!& z3~tOgyd)z&-$9u^Vi~mOMpk-3cPAP}A_(bW~Q$W5WI*Ot$iCt~gh2%RilTaJ-@o`z9<<|sWOF%p& zL8hN}*Zg=%b?YDNS#*{NwE9MY;>jNq#Ui2mUphOsIDZoZGMp3M*Qu1yds?)Xic{~9 z9QFvm*In$^iMQ!0(Fpjcxt3qzbQj6N!{O>H@RNLi#(NRN#ac}-x*#L2xOZxsB%rM8La7FrI zZPY5M<`*lJm=mF>^~zCiT*`|sr$u#Q?u}IdAt2#?RnFFO2zI_mCbjK9-GD{cXwLk0 z3gnFG^)e!fKm-=X-pqy4I#yNeS2r@Q6AL@#q6i(5CC=(Rt(zf2FMhQQ@DW&vK|Zeb zsOo-5pQ*ZCfJ;hMHVf1tj?742D{?Of@ zdlx_!aUxIFoRhOmE{UD>3nkSoM0dXho!W3}K~>7P=aBL!?ECuCMO{n7Lj)nTbVN!AbeWLkvv?V%cV_|X$ zE)WyVEcnu&$mI46BDPbIV7u;Il^qPmQ`!TT_-8NIi!Y~-NSeFwSP zPX|%DUqFTnQPm)FhpgrXKC$}26g>UJprWAT{<959W9}*QG&YHZ{XQjf2!3u?+UwqY zj#8?@0oO3M=YZN28df`t!{pi;+}~=83clhA&9HgegiY};Nr9)?IPRu^{vIr{1o0|D+IFMe&-^VX=ytB!K z18FaJQ=)%qnV6&9%Xe-$`hSh+nnni*rDv;v822%8o`DisW7$3r)VTuQ!lSW$AVKVA zPP3-HzWhpBZ7m)d9vM=;kt)rS;meaY2Q|ETKg?_8>nv8LNdD427d--Tji_L<71t(` z)!Ja)hh_WcW{6VbA@|;{0AbscTn>NiCCbJgM;T+-6ssWmnCtX@6P;LIGx-S46-F&a zEGcUCA5g~8pFi4t$dts_=eweSX0+hxNtJjLwLF=v8}(#EzQ6H25X;@A;Gy4y$}RkK z`1-mtgzgfrgrIk^B#u7WpiWnKIfq56v3g^(A~_MxkO$y@#gg<oj0n(^Pu{ac?A|6;DFEJH=wW1KV2(s5+=0B}Bm;fc|7IqYrWTK#A4fufMm z$o;Dp=DO1dMnFjKKi{gg%r0|QUii=itx8SIJG1`r$EK7^hzYDqBf9Mp0?NH$c1`HB zo28Fg%ANv{1t8F1G~q`vY>O>PQK;5>zR<+CN-DiI1GxI2wa%$XBNnhpn&rr^1(Mpu zD44~6S$@lYqL#~#@fr>^l+XP7-tMvj2hDVTTzVT;)WiQ(T+Dx3@Uu}w4x9{y=eUG^BSxD=C7nO**7bZJ&x^Bz9|n%n zYyDclssSA&AeC0?j8^G z*1`^7>@-G0OkS;=c|+sVvsSBL#G!%00=VC-h;5@eDiikv%FlIR_KT0;JnzlAI;r!z zX%2oW@d1XURIOjjL@BIT@ex6BGskY8)DooVTbBn;T<|byK3lI}$kggSCh>pVNoGi( zhsshG@rYmRXe)KbYz-6V>Qt)M`LqAcXpZi&bCvf{$^h2fM0dp}v0|!y>j}xL8000! zb|NDbNO~_R_kxm!2V{2|LT?!uYx#3*qDyuck*YLqfE3-eI z$YBk2NpH(WbKY=mGamOj;pchQ(KpO8@M?E*o%i)%le- zLvNG2q988bhpE=C?Hg3om%s*HNEb3L=!f3}SGM64gdW(>Gd+Cn%G<+AP8!}Hler&! zP30k@-HV@Ow8qM>g7Aw+@u;e&hx(uE@SoM=CJUb=VUn2yhe*V7fBTaA2Y}jF2==sY z?*rby^o`6f&B$7O9X1stR+g)6Yl%pJgHMJ0E!$-Lf4%hit%!SvNKXzYQO#7jsIzM` z&k*-Y%wPFZR$n@lxYt2_S|iKyC^GymctB5D?9G^Rs>;~MU7?U9sj_;OUUy2i`>NVO zH%bVs=h9L2b8)UEZqUhkBF50(AS*|u^ncSlcY0JuH_`!jXZ1{UDgBZ#z%P)`^6apugmCf3s-%|6n?;W5z$EJVpGc8eUfQ=68CtQ+S;sBU z4qn7|2zFi}6?+sC?(Bn%{~hK8CqKoK^#&wCHX0_(@VUFwpdv2<0Lc2%l+8`nplc58 zvw9NOB_2GY)T|6}fe}Q!KpHXW%35_hmRa1whUr~s3dlZd^S8iF(;{WTu+4rkhS+sKs_>U-Ee--Kof8F}_M-dKwT_Jl`ZJ{35%C+6n%+OC_WB zuX&-p6Sc{NDkJABD}EMX@!y*qn+Buo?tIS&x@MS#25&wwsd&Jj#nq7karsB3E|p&s z8p)hjEaLuAzy{?j!wPnXcFV<|P9N0uSZolDb_1|G2XdJP*c|nrwn>;jjTsMCh{wU) z$2LKRxxLH-dfN{b3+y+!s*oOm>$)TxgTC{T3IMS^+xik-CHsTR=d3@zHQfus_6ONO zAA%IAvBO?Y*9z?|&rU)e;`=5W80l|$MBc%%3!8OY-$=v66W7tpD%8v1X^zPWpoASp zR%>}I!D$3+ckklS`I`_OBZxaJXaNZRt^b-g#i2wk3rUf2{VGk-Ens7|APomGKvIwYQ%+C(eV0hjr zNSgLCwzi?Nn+Ni~q>Ld-Ws5zHKmuTpP^Fs_)HfrDfn*H?Sp6!0*5?7ku zxdo-W`+(_Q{!G)7Wf}yb5wo0!NOd$�iks-Kfs^LbBsvVlJ{bX=7z`c$&aG^D@wMau*@- zZ%XUg*ei}W3f~0p5*eSHvqAMIwg$@c(&yi2ki;%wyGJ`f*oc0P4!g642@z-v92&A?``3&^#v@pN^ zK+t7(beXQo!hi4LmcRCQh4(2%FCQX*xZ03PNp)wxruCe@xg;+Cl`h0zvhu%ja(RLR z@g)yu$?wb2$X&|fxx3g4zYD>SI?$WkT^2ps&eRM8vVzc=IjS{#a??=G8F4()v)s#$ zH%w&}vSuQ00k=EtVwL_f@XBf0ry>E<1bb;_y8A(NzamWvzZqcaM-Rd_|IS>?7}RgY zg(OaIc0lLc63EbQRlKn+2x(lt2&1I9{8=VB?U-)BUSP3KUK>l^S`3vieBknHHSFrW zsgAKN1ar*MAbgz^E(6zFzOII@{zkIR`tD6fG6OmP!CXUaG#8eWwYPOqKWbnDY`=cO zATw3~Sut?}$+>8I{L#o*G_kAn5kTdD&4x`*+%%jW_|gd@@=yly`?oT=9LyB+KR*-m zO6&f}42I|f4Ks8S6cPzaVB40QK~hwFGK@ zYU8)`MAm?CwSh*U^R6Y?4NUB`PpXmnk15$xAiU-b09)6ys`=z0VAGM(KeEB)pb0C^ zYEBkH9Oo;F1@wNwBKamJ*cd$c6RMARZ1H}HWP4usrZBBrof8}YIfwplaZEFKEmI@X z=QD<_^>H@M?qaeR>?)Q$>$qJ~xg%br1IG~W@*`0pj{jzVVGtY-vMULDL%X1n5 zB&g)>Q>6->$y#RG5Uyu54!wQTO{9Frl<>t5-rm_>Yu{%@mMudgbpojUXIO71!V@bN zm04hNkqdaG9Odw>rQq8p=Wu6m0>ef7=ADh29Baaj|8#Dp-cRc3WYmh2y)@2De|7 z2fqD||6$|@%Q|NASi~=t!Mrt_=VcI!81w5_!p@otd1rT8gU!hVuGbnt@HLDK=EIsfx` z(W|^@G6$ADYuU`K0Ii>B=0ljrf@#H|VwsG5%iF}(&rjgv9l$>_yL5y#5krm24t$v5 zY|mc@IBTs+1Np#QtF!&R_9kQ|cb|EU`V%;=yF8&g)>c%U))2}58WzrVWZYYu(Xyr*me{^mo~{1a&}zAJ{F zL;Z(e7Y_lG4i)XyO8ugPhEzH(5`AYfpl@eHCviDk=T}vxe;qA3R^cN}^Gatf+63Zf zn6^N5Nkoox)R);o6b=q4t7r$dolqI^i;p%Tn>KG|xKaELf_*n=pD;w$@k79-Y(W#g zV4P-c*=9sEF(s98I~UdJ+q?g#?!)@DDS8M>ZvTTj!?=eq+$wJ9ewcsv7Z};v;O`>2 zQ6H@Z=ZLe19g5A>qm3sP)?NfUSI4GCefL@#xA%r0EwvX~_`c4152`8)ZS!Ow$x^Qn zG2CxyFm`)spjNF!?$-_gyzJ#MQU~Jh-P9sZX7UJ2eiE8LQYmtJQ08+11}kHYt1Yj% zQ6V0<@t#_i^1oR|_Yf7IGSeEWQHbY%Oo#bg);mKL2dID3_OSYEg?7|XY)*5U#wgO2 zRW_|{dS(+9+(%cEK$FnH{IX@CXt74rnO-mF@xD~0Ttf}H%OP!ZscV#P8T-V`u3-n5 z&%VZIX8u>MI>grjG}CCb^q#L36~PUrHp1b<9+>Oi?S_4@pS5;2&2GwyNJJJelC!E~ zbtwY$#ZrNdzdJwm@cZDJ7joWxAALynJV>HBr1h&%IrCjq0PmP+mM_S)cv)AzM5)K!S1O-Qtx+dTeo0LbJuit!p zX5##`FHFvpdfFVE<06jzn;cg3k-g`Wx552JB_L3TC(cHAV>~S5ZuJ|JXU=x|q1`E4 zVM{qZ{POTNqEgR(a{%rAT_xye2OjlHG3|TBGJ-FiKkoQI=r+#MfW3dQYBm9poW(LF zvFGd};NTHXJP?9zYcQL)=$R0uoj$F-JAXB@`@+RlbC9)m_E%1!vj^_;Vm;$4H*O|`Uloju@%wd;)?t9iA*>rPgOt=^d7 z+XMx}^bA=%@Gw``MM11oVcI$4~&>p3F4S}*vdFAwugWrUEK%>U^!_Q$Le9raC{kK|TT+&4rOLo7fG`PU#t z3*0ZdU5lsX_|Qo-duA$H?7Q_Cx;cfVdhB`6YoDtjn=rWc#@^7f1pTKP&%yuKNl&d$ zim80UR7BF#Wh{z&(&`{@_Zj~^o3BzIu9jn|EWJgihLJvQYY)vCV0&(x$}A1txXBxr z@>ijX(C1Dj?vh}-_6Ze$bFW$Wi9*E!G3SrUk|`6^C-HxcIZ;n+MYPT^AZ}#p>7|*GyPXU?#<#$xUNwGfJ)( z0^}tIzJ<}_m3;HUL+okyD`m;MofZQo0u9Bwa$pyJgQ$I=xc8_`L;3FqHOv8!od{rt zJ)ZFOuB(2`VE!iSBDZe7D$Uz9OozSe&fkW28Y`5$5-VwJ=sH>}AN%I!pdy6zfPg;j; z>9F-TNDdG0V*PX3tySGQb%L+f#U^7_UoO*S2NsG!<(Yew)gmdqTo`Th#@ z?ujc8$Dupy=&wpti4BiP7#M61tMnsnzWp!s*SKMk@4t7DFGELnp^}tkuJ@*YCC`lA z3{j5R!*(hZqhz`)b&He^Px=zmC#y50|9%=Cm7<+p<0#4Kqhq=Vi2sz;Uh-v?Ym)bw z!8ekkR>7Khxdk1_s%9(@9>pH(+R~ktHtIBh1G@szXU@(c0>o-Z~<1OOCl@ zH#Tm`zw+5CN8y_lWat}S*FAi4L+Vgo99xLt#=gpjehp&lH~8T~VH0!RzD1#oU!wg* z19Qs+D(%XxC^-xmZre4FHVE3I#Iz24hg+t}ij>!+avA@AYeiXa<0{ofAO72mqbx#T z&oYl`rr-(-W^gD~n^rCCm^W9Wv!|dxM|rGYAc+n(Nfk?qrLX+x(8f?~4F=8fz%vP; zAlX+Ffv!!oqdt!F+mi~djjFrIZr$gtve{%OPd=SxXnPJ$g7h_)wt)1!&C(P-BeS9( zXMWbQoP1*U8NM1g3k7D_UE?Lm{~-P9#+a z3DL~lANTcz{mA}IUDVm@EDj-OA=9s*G&!AocQU&J%Sl-+=r$Gqp8d#pPs_xgbuRu! z@V$uhOs!z!k81Nz?CyM%@?;+Qh`Qhtlftxy1NAt`Oa!8C+n5?s4@apOz*{;6KZR^Z zteoH~0bmHEoDAd6LDUX$bs4|#Sv`T^ zB+7EdGsek89ZMfOflDRhvR$8eyMffq&b}qKk9@RBgp_v{nph{7Ae`EoyjDg~j?w=P z7evY(;hHrqnVqB;msjp|0{`W>pE*ecRkIt*Xc3%FeMc$3g2-!axY(@iqTg4vlm8Dd zM^0}=++}v9A=D>BNBUyWmB9XuaaO_91vJnF|#!tqaW# zDwlDdXbxbIHu(Lo-P5KT&JVN1VbdXySQeTGkVl|tZ zde%4f8T|-$^a*KvzpA8--?0cl2_9Uew&15WEPiad(3xJ->XoYkB%$V)PweR2>bV6o z`nY!g_1qIUA>EdP2eZ1sd*6O`fEDD76DK$U>K^N%RUORnddTA+Ps^t}n0(0f^Z*D+ z3FYbbt^R$3t_w(lKw?Wz_JUzE$}!Qovjf=Hgz&zgzy08$Kk0!( z6P!*b;t^_*iWE&4Zj&y;;FT$wRLUk?F+x3bObYi^w=yGyU&`)c>K)K=s+uv_K?}Za~ z|D(@*iQ^!v_ZK4kD>R~Z+|++&uR{p2*4r2Zl383W77qEkx+gpWJwnV5d$}gHUrL>= z|C-I}@Ya(2)zTfmh-&rbP29hfuY)%Dj9L(40!TA=1^^Iz~jZ%*Tf?jNV zO65$qQPbmFUf1{aY(C_gNrZOM{rK9n#=+pxI6wOY-fv7LPi9O-_Ph_<)9MPJn&kb^ zhSSItwU&nUuogDEiXDYFl3Se2p*{#EWZ}{>s|T|f-b(MckqmBTYgqxT4!wayCWMu_ ziO98AVoE|?ROK|~Fw~XiMg3|V2SC~D;i{x2UtDKhzfn14 zxK%!bY`+cw1V_&kQ<+)E=GX*8L{0E2WXq>SVn1TD><^Ov;M_c|(@-tICdn%@Yv#*- zY@v!Do%PS`hU!g2PYTPzVy|m8LuwAfF5Uqph=LB+=PYG zp?!iwS~jEaasDML#lz(nO8vmW#i1_6$GZDf*K^2bE~(WH?BhhM@N0a-E8v%hXS1q; z4eyLec8mA1nO}Mp31rvdt!7xAmR%&0=jdl5W=7aZ7qE;su0%hO3_N~bp6O&K$@9LX z{a;CAi&MW~P?Nypn z-9#q-G1vW#)8WxD+kjO+&E&BcJmw6*oHwP9?g~7RIi$xsy5wLgd;}lk0sOXIOS<0n zdpp^R;WB#qKOAF~eO^o~?luMp{_jxb!|TJvv^MXmw`V*By~fDUEjTUpZXNTzw>qY$ zK9w?ajPC@m%|dgY_2y30XkXz?u6xAektwWNpZD{~a-e$d`6GF{U&N^0OEOBYNcKVV zLA6KTqmKtNJx&}s#=~5lH&)UMF05bg=-NLWYfmdV=|fH$xzQ?Rwvf*q&xzdgIfU=G zMPKX^-FNxKWpe?kf>#7olu$zNioD>WZ=%j$8wvVf?4YKRq2bRy@RbEwXu}0Eu7aPM zfmEmM@)<2Jt%KS+uyMHh7%CMjkX!4G92ddto<`;kK(}Lcij_3HKUE()W7xZB11jCW z)U8_0TR7YgpqX7(G=6=Og?b9#zb&brx(4}J8<-Ct3PY3AQNR~Hl%5g6(Ndo>dieS= z=G%ON-fQi!yPHXrV2F!yl_LK-=j7h)(zv-F#IAz7acpZA{oG}*EP}1)uaXva7J%<| zwNOO5&PO9ck~UAI_4B2GQ01rxFe6vBYHA8@xSi zi(WM03@;C)rkrR%v>Ya&Xh6>P8F`r=Ues}{_O4y6tfd*B`a@&hf$7*PC8; zN=r)NbH;Pmm0l9tWdMY*_u}r9UDauKZ5p}f91|f(kPyZa{%iG9RwlInF;1@EYL*6k zEObwHt^;(kokjfUndGp{3hfPC#2Ba~jW4zd7U;!~-}5)1`KF?EHcn#`EE@T?w0=f( zGl|zQFtNC@#)$iLv1`sfg5gUu_~wjA1`bXbA5Zf{^e!pp`^WDLwy=0z&J(#1w74I6 zCDK@8+80Md`RXj6mtDBDH#ROsKLME$D>VG3_F_6%;N8I6#XCQQ`WE^*_+IJ1K7JvG z!<@ANy}V30Od)iB4XHE0$%Q+%!jL-B_-;>gByKUFf@@dm_>Z57DYI%h2~GDd`F4~E z;zV9lN57hPL2WYf;&f8*~FT^f#KYMj^ zTE|G<^G47B@e05 z`p%n_d)zNyY*?Cx1c?U?fzB%j>BxfYoi`fmf$LLClk$$Dre9mmhag#G7Y@{35%O!~ z7;4V~kA!w)Tjus3T-+-gm|vvgd8`&Shf(nUlPHcx(Os^XYaiOkf(&xK{!bz*{Dzp} z+J5ZuL#L>q#JUBA5C6{Qg?HLfyf|Pa+q*Ci06FT5^|YF7E* zN~8?7Y3p^@8x-_I+tJa{+hz;Z`UM>x!J2Cha#6WNV9G()4$3D5nG4 zu}FaEgA(b?Cr~!EtEc_g7LZ`Pi%Ec$_m+HEhf$NR#W1zIhR&!0llTh&Sml*PoS2-@ zi?2M!2DpzuV7^-?SGik%UHj>=t$c147|Jd&jpomCAC#tFnZL0%kndA$M2;qu~< z9Q<*g%m>B0l5{Io)s1|Mi*aJ$%H*N^f+t6#?iLWk;)Lpg3j=d9I?sLic#yHEp{cuH zK$Koau8%k-`$w>|*xTG1SeTZ%QL%%zU7q~R*fHhN9F#6}tQPdbHM{Ve_dxC<06Gih zwqbF-&sC#H@ym3!D+1Q*{W%Ryf1u+q>o`(T*~_}p;7)Bb*krU#_{fwKl%%%G5Kg^g zb~FQ%xSx}UeBz#<{WH9h=k6DQc2Vrwf>I;N|= z*>FVgmgNDEc^UGFI3tTqm_T0U_Uvwo;^Pa^9pJ--&9Kbk#med2F0> zoE;h`N3*|Io7UjX%~Td3_^2_x(16m^GP|?zDE&_H<^9T?=3zh(}R`xHR|FM*04SeRB$6dF^Lt#Ar*mzo_^@`q# z-6;)%-#@NnemKpEG7+RZYEh!-)v{6)1C<}Ob3{Gy7zHFc=&>a#g+bOoc9}cU1&#j% zeY*Lm`=4s#pa1$IGistE_Be@Diu#n@sxRyM=E(?T^R@;f2g`vZ)E{R?sGr`|4)&tzt} zyK|UI9HUv!I=UyINWiH8wMi=5=7Hwo9U4TRO=*jeMxRNSk;6dsSLTf0eX%Hi5bbSm z>(-zPozu8qaaWV7y0k6fc+pWp7TXBrT%rk;=kJ3zH)edcjzQNdLcChrq*txy?o(6YpS#Lo9(hJ>E(S=z?INQZ1q0z$3@`iyAeh=hH*8L8vac*Al$D1(-oKYm0coPo{V8mo}~8bU^co4 zI;#A)5r}o?s*!Nj7z{;%_e|WyP1{?T;{+nShST}&&q@Du?ymFQOwrtn zd44g%din!7h28ICua)==yJjCxR(8%0>mn$4M?o_a`?*XDUbM0ioQOv!^MqmP4HD{d z^_yPyajG-TgTE@9@AenR-E6QmH)!&2gSxG6{9I1URKsew7=yvtG4tN!CQC>_(G1Vp z^8lz`t0G8a?UNCWgv8fkC6=*(eDtZKYfePd*3c1>tp%c)^7pYIpR*qEI?&*LgM9S= z*5^~i0u0L6Siqg*Njl*Q;<=qt{r856Ka{*+lDmxOzlf|oK@hC$BxgZ`A1nH<1KoI5 z+kn>wmmAzyhMUcD`%d2JXp-KUu7UTVA-1G!|?A1cS;eJJEzHD#L-6w&p_vkjp_{KB|(HwHu8TFq-=4C%IL?_ zzWb-Z2LG9l{1x7#&dE|UZs*)-aV~mnU)TWt=N9e{5qhkDl-C$>s9}eYtvVAL0Hyc( z&0aoG=?HmbXh$tDC|i0DAKTpVoZ9rd7Sws0d*^FN(%XZMfpyCvkpq!|;xF{~L%*kJ zTR5>%4GQ-L5hd{k)A?)nJQHq9K@?c4dGrDA-tBRHYMXkV(|d%((Ul4d_i%nV%%ei) zi!^!VwiQ{YC!F`Hj-}{7T~xGiY1fx(y8Fb24}ad$Qbvd!u!h!ywyzf0j%2FtY~?}e zceq+}?X+oH>Vdy71a;fMiX9@8rfX3QK68zchO15ySEKZ z>H_{o@?QrC+WnB+`e)B#I84=GdlC4~xZhMnUqLJ~g}14$nK!{>{00#v(eUJ7lYHU%KZ`D=BO_zzgK!IxF=2o z?zC4E*}~WG*L--+1$x_6pGj@Kgr4vem20a9SRLl54+g_8j~}!2;&g1Ddl|Ox%aYBy z&X03@>o2s(zoQQsVZB5T(2jos1lwT~LU{zw^sN9T3L9$GsuW7j^K7cV`|!QI=wMsI z^$?^2cay$)5esyS?&DI8UOTME--4)R;u=Acx^vgOe6DbG zoyUt~?wSV@qH1+qAGC%HdYFBwHRn`Hl+K2D*MP2azgv%m@gH|h_^K?MmA7vE4A@Ro zlL$H4wUQo~Nul7qXK9Mi>Rl`FhbManSH)t=PoCGtB~J;JLW;eY{hvLb zPnZ~`iPQG2)Ei66thsoYD?qH!^-?8YqiT%)9HNLUK1rgtpp4~16iH4L5tGO56ZO|B zZNNt2sEh%6bxSoYZ7Zso*M>$6CG^C3lUt(Xw5AALm{SSu`eEf6=R>4vQ=pjXR83le zktyYcpJ8VWV-LjDQbgUmGIvqtrNJ5@W*mOVoAddu#(O$4TJh4XTA9qY)9fFf2<+E* zX5hW`T<~|d5x-&iKL;OkP}~lrSQ$OQ{2$v7Ca7~o;p6l51%lHXJBqz-X{f=tg}lCl z-Pgv}v;IpHju_kc@$Yib&!pY8`g7ONI0c{B^_;>(*kR;A`yV`bX{|%f;2EigRx-!U zzwf_)|62pOt!a#uh4X-3f0lZsNdn*zoJ7B8)ynCaC0@C>Dkr5F$4@R0a_p!be%k60 zt$oUKN7@Iz&*8{Eh>j$5G;Maka)F<&?2lU%(P8D3}+L@HE$^4dv9r<#kpd)+uK)#XH3CL0j2Ge z(kw&6OKT#KEr#A&t+2tx@F7U0*lc<~aC`9wm2ASvG0GVDn>FAO5LcY46g9MDiP5qX zl^t*Go;w*<|1D0j9t*lFjg4XOe|DgB(wxb97IQh}rym%bJJ5L%!>{fl=78f|6I`Yw zQ}Cu!CPgY?ALE@`Jj9=2WX}I|%^B7DSIJ)0FjsC8s1|G!P7h=<9{nD1pG0h=A1k8m z=dDCiY)fCPNzOnIOx1R+%Vihpw69ENv+nuQzG=v%_;)_x(jP2lGtVoak_IKf|=%v+YJe z`yKL`3%0vxDRlx;KWL3{G1+?*lzXS2 z*xQEc^SuEaTGhSxa#v-|YKJZkX=89d6omMLrDa>49vdfyHu=4XAAXrajx;&Rbld3n5{o#XH7}$n|^v97pME=;P$X# zCiY+!)g$Q2hGqCO8#up{>N6tqS$VQcil*`R4=yi`B__zY7*jYOWY{Wh1xc2l*-b=hD zPu=Xha+FP&qhochA%_#!g8Gev*JrF1>XPFqT^n;aC(Y5#lE*q^idon%X5rPkC*Z}6 z>;8N#x#*q(PQuzDId#Xqsu}qq z-3V)>Z_Ht}KS#z*+CvH^207>wmU`=qqb@ha8P=VqrpuiZ+-VHGgtzkm^;@GKB@c`z z?s3*X))LWJi+_8MEr`9=it^c0p(k$*`@0v8+ZYr5G%l^v9~Z9W-z)HR4K?R92a5b$ zi6A4g1Wb$!yFrjCXbM=Hg|SOo#88dr6j}@cIwX)l zwl)e}oI@Kk?lFKjPL0|*=a#}*tB)*<6s{I6KEA z>fFO_WbYaF6$ehmSm81`4_)UHqx4svDHbiP@zFi9*3;Z0hIYnbp|F<(c8sfcWDV5V zJI46vnR(16hD7AB_TJYUYH0}E>0P`7q%8$51CMB zPt}1J^77?gb9-s8-SIfW7|xBb#~lltZ_L5enFrwd+*4$7+zF|jt!xuh7Q zqD;MxWaBq8=Jg&SsXA`p)H3Iu%wM4}dG1M_xibu8TLTC_rp4|*!|>mkJJKo-h4(lD zO4y3KgD=d%h;&CliZ*gy?VWp0B9DQgxwhiM z+i^;B?SAOe9C?y5?pdm|S7_zklkh#Q#LcjyK-QVdsA=tq|2l&J)lJi>e+TKkJC|rx z-`Nsr4^*W+lUnUci(YW9U8*v&1a8>rYkH+o-@ig&z~76gGY2XAzL(@;T7wf~4Iq;= z$C&5d`)PZxXs9<<0Kt&*O841|x^!(>pR>lK@R@U8Va^(w zjsF~`Jo28or+lr@#Xpv?`kNaSEpL%5#nr+xM%>HXu@HEz@z*t1*xHPHlYeZr=Dp^u z+sOMFbPOH*9OHWM*vkORUE`#A#GTH(V|;lZF~4~XHH^J?Fv1KmIW`1Hiocea*B#jq zW^XCqG8S&_+0jFH4)~k7WBUKc{M=isPiXJq#+_Cg=LySxWCYEk9QPi`Oq+~qM3CE< zrWQoqbEIm_;n5JcNb;W(H)Ll4_dlmvd>yI!YGyU|G*isz*ef3{$K17==W@*0Q3G%e zL7l!bGRWM^%s(%&yB_xHbxMQaWX3?w9hbny3UQZm#r5Sk!>ZYg%V~RN->T*tlOJ=B zrQ{RRHt(A=Fe?Vg*){gUK$)QyD@7T`JH{SrTM+DmIMZ)MTw9!K6^Nz1H01bLzxN61%ML4RoORM*|N4CXiL-dbuGQ_G*_PBjsszSrR~eMGR{Fu z1FB|k)Py)kgzuYEi*--M?lG41TV4@3cV``}x0jmx8-urT&EdI_gPz~s3yo(5?S?Sc zI_6nR)i4jd$hY^rWtfvqe@;osw>Ns=A3HlEW)1cmx4hRKv$HS_NTWV>w#6R1=Xnf` z>ohmw&YH1pDbCcNHaGlWN^3fQPQ1StXDI1k6BRuKjoUl7isjAYt9E5Awv;9u+>fCn zY|fFaJa?+-+jGox4h^^+X4+_)W7BzUT)~`H?tI@1!auAriIoEAa+vYEJZ#CY5>lMU zoCD!;Pb{gJHw?&(3s!K=MW(xRT-l4`7AGyVuQ>-gZ&?EpC@ih?HiKr~kK3GP&s60) zS7>HRRoJvv9$=i2JULE@)0aa6WDDblH>^nGmy+IH`Nneujo`SK zXwplefGq8uo7NDT@0xgH$*-BK8Ik{ z$n&2guU*c*LuORoDS0Pu?a&oQ7R+9|r+e)khC5c+>5SVKXN}#@unfhrvGzXOp5bsT zM&!?ugQ({_C4t4ampJ}UNh*6S*`qX9O97{UznSwW26kHvq!9VS_5o1 z?R4%JR$|fK1E^pbTUcc5sojwFazsnh@hh%zpc6Kf%pE&obEVPhnFAKl+FLq$?tIIc zGaT35VFx6q&Hb|`j8Dnqh&x4nt+-PnRGRUkcJFw_IpXH{$SW#+g?W*(CcI%7F|2c? zwfZ=BQ03kOhH0+(;WEdj(#>0xf8(@KOi5&ZWMPPu(>l!}jz`2w* zjzf&$C}d1!>=uHq^Vl&=cCTsiJQB3@nY-k9t^Lq5CotC;i%MvX3H&z`#@`M@ry`FX z%9B?f;vZ9?WKAHFnj?U3nVTggE~LL3;}X%CljSULfbJTX1WQ?g32)Ev*t~{P`5QSP zI&HnSopzw{8xt)tuaUJi7jD5#OC}-hb*vJ0XziYhMK|T)l{I%VO&u{dD6UAZJ~F&Z zojZXrhK#$MB7EkVdqX0GoUyy|d{^1i@*$;3{+Nb*@fhI*bA`RW8}{JB35lO_4XNJ} zHmK$c>o9ADZQHeCfX*Lrj38~Gsuc4KbqNDxe?-~4v?jp)iTm7eEsVK67qG@yn~*+@ zb(l+J$Vl3gYxe{{eozoO_EOF=(SFmHulQ3&fT*We1n%9`2 zT0hS@hdJUZ<{I1HVz0sbJ(F0}pBWfFuYLX&cC61EOATyqT(`4^d|Fu{z$$MEf**D& z?=LO6jxgr}@LWS@WzFrP5<+Ie2%5O;pj1j%I{0bK5S>fyRqV= z{Yn8*ccvM*JW`%{nQPu>Z(-jz*Mj_vJ1S(vy^pe!zRjEg^J-2hi@l~E^~thu*!vPC zr=i2SwjkS0(aa<#Rn{_hzT{k46mu+;&AXP$Oxvk?HYarVI`_n8*>UA>?=aoE*Zy_S zl$97#_|q5zWb&OMsb$P?*#~)n6lV_L+iR_M4~c^s*F4hO*+*!^y|gm&+{&E8R(UP` z$rhHH)kzthe+ICUvImM}%xNJut{J@(!h&GmTe)tp^!^fpw9K42TxJe=r4$Cn+e@L6 zI7bx79#Vk#4k_#=kIk+&GK$XLsYWv-6}~eU&i5LV2z#xyqPzlT&fN1sV=U3Xv?tV0 z$zxk)4Z+3DY{~5YCUYJ}rfHjj*G*NLh<1An$DDowk6}T_dJuFKw(fr|w-FlkOvi zaho%vgwopU-Fa>C#k+De+IyNq5Nbvx)4X?Rz~1>Ea_%MaoTFIL*~4sP z&UB!$li1OoNpNIEQPVy$3g?{@qHN~fvy$WHN7<9@c_%gDo0dxC-K)ZI?C8y~M?U16 znM-$%`LDGm*82#d3U7_DzB}{C))_O>FphoeJ>ziu4}-{k&z;Jq{O1%#w$NVrBrQyU zh&smF{@#=1C$8bMIRoO)UbAXsMIA8DU9i03u;7m~I(>`{x48pwa9We4YDZj^wx@Dy zm^-s8%{|(e67tiUGcb8&@SQxTa{XFsw{iq!wKmpL*j!=Der>dqH)aG?TnkZq>=5l1 zhwNThgR;R|OX0wKq`A4jf{1z82jt7n6P{^~*w&br7-`&rR(HnG-@8Y2@n7>ybi^^P zKNqrT$pKGzOdv0<{qH#zesI`{5o?SzzMGS9<=qj?ZLE#bwFB}+iX&HH%(&8)vlMF? zd;KL%MC!b!aj~j~s$FH~i<$6aFVHy!|~^ zjOW|KuXHE*tQR9N(VhwOEu>}Q9;T*q-Fr(V4e`&u=2Gcdt7I)xHz@s|$su>yC{F>fc#F`nk&zdq#5Q&PahG!f2_RV z6;ibCh!Y}x?npj)YJ#2+N{x}Bi#2ra@V-0+UyVhpyol6xVEGdgW^J`>n zVX_djqT$$EEOJc6jTDC1(ip=;c8p+-GX@BM&!K^mlMwtHIXx&XUBJG=!q;30wkX9> z{z7J_ib zh)Zg2tw^&tgAm+WGul3`3C}RIs$JXTCw&gF+OyYqNs3!VZjX@t6GkfOj?p@| zHZ*w}tE405u&gBS&tb2(A_~RdL+v21obK;s$+vbkVwuZ^DXuu) zKR0^qAImsm@7VsZ!$Q+ri2`}7jioS_Sjt>e!Z6H8iMXdwL;I_^yIFtRMh-AkHa zPUX)!wl+ZD>lG@jDVGvc3P>7Tadpi(vO0&*(%WNGaYV(^zT*<(%YjvA%w6<0<1*@s zb2uMswKRmmpRx92^A5veb8O)LI0s06ANZmjhJ5&*=`nNfovg6)uH=u?Vr?(Y)IRqR z&so!4V$Hex84|4f9!m{*jQNSP){;<|dvSA*rNuX+jL6tqNq+@NpSwr6)rkAjX(frb z6=zb!2m)y7OfyJtCV1|g=b(O!(=|Dd?Z-Oj4A2Ti-?>A;?Vsy%DhE-jG&W$yjj3oj?hVqqCSY+5x$1Te zdGow7plHa^#5*pDr8ZaI{Mgeta17bdJ6F)y$(uks4EUt77FO%nqmDN%Xy3kPJkVQ# zqI;|<*PCW0+Zr1ZcdSvxH$n(;*jqp;kGT z6|&IkU-{TE#(?vho8)oqi1w7Hz{OtCN`6k2lO2{eKo{_G5?nT5ncY@AZ`6x6FdHonSNW)k; zqi*hXy1NDlZJk4eAtptp5q9KHoEfM#rFEN;hZNmOGge_w)ugqvTK*j=zk1|ByE2yk z(n*7PGN(Y)wT6P(-fMwm?@_I~M&{6-6K!p8?ftyg3~k$UW@Z()j#&un*E4SzsJJ%t zXPA*mJde=LmggqK&I4(7O-QUa#u#RtJ5PVEP>dCqs#Mtv{~5=GP*0<{YmQl^8K)f9 z-fsijnKKWrb~yM?O1~jYb?0(u+j`p%mKn|%$(kk=bU6tBUvVG zJ^h&W^2yx8+kdUW$G+miY+RG~GRK|3oo7J*2y1gI@66g32VnBaVPt8~P4h7Z2dVTTY;q4Rr#Q!6 z$=ER~FOOBayvA_B+2bu@%%Rk|X1x7VQ0`v~5IH9u>6DiwXWZMKW6r$Jmp6vvT=N<( z4%L^FLlkkGv59AoOz1G<*zt=&A$=@O+A~IO(OF9*d<_AKxss0R-q9$3&WyOR_nuD} z6ZkjGsfRq5_IDVIuX*jbgEP1K+>U|wZ!Mj^HWGBspS$vGZYhPD=5q2&gY7M*@y5I- zg67YIw|dS6s1||_Pn$YvSi7BmPeJWAqHOla+rwfH2!aHpF(^Yp!aJQQ1BQKIzIixjIB_u{TEq;ocFQDvrpzxMKR& z$m{Sfj4Et7i zCjFf}lg59X8*L`8MasOg6#9+BLt+l0#J8g?NZl(?aZDuNJm)Oq+7Vr6%%QThR!HUA zlMOoM4Xli9yql+F?U*Z)GA?c9m^SKrT)Q88524)|;&{Ye88l?gyyiEu>~mX_QELrk ztG?D+;M=nbafdnZv8NE!89PI7ra0D{XMXhxy9Im2!Pz>NnC@Bm-ejy5&JkC1w=($Riupfx z&8gwJGo1Zh6VZ8RE$BFRSnb&vRd9qn`o0IC{Ga1gV}~uTGtzFuTdT!s%n95dmxx^0 z`66$Qq{22=0PIe|A$etOoHgb`+L=q}XoWG1KbCg*T3cgbO-a=<2SUr0MP4f`b z_SPJu=6p=e#gfM|V_aJVbj84!v%)rbn<1Wi?e(6vm&EXogI#M(CB-|VYSfB56E6%k zv9MFjN)2(cWY1)QJal=^x3I7WNcCWl~c6hnjtK2P6huK z(*W+sTfSsx;nBJlTw9n1u-J&hE-{Bdt+z7nRLfcFbL^y(Kl27=PGya@@+AKZA+c}o>HQg}m|>3F z<zWfNjS#@jJ#g#G8B8Z?Eb1op+$vUemI51;Fo}rflWhlaV9t{PUGVIMs=3aePFK zz&_Tb%$?KFU`73=HuG>_3ZVol1>vN$=bYvo!TNa&NaY>2yzSf(5o>v}l-OUp4t>u& zxgECxSKFaaVvl*xzf*eh+PRB*q&c!QW6Jc|x$rLxK(;m3q~%?kAU!N0nUhuw@tQ-j zXM`y6FoRU}%(HcV?RfP+*9t+~L%wwHNvpX>j>BGSYHP+-ro948d7m2!I!s)wwxagY zjgvKWMji36cgWY8Idg0bIHou4X{BM#5ZFEPuIAZWH!3Vmt1^=8+Z`kHcx;*Ck!R9F z-r4^>cL2@Z>3VRE#ooAMuI3xVqGhE#nJ^};LG9fJ#LhdHRB**+!JblhkcZsQ$)qynGAQN-Lkxb zUTs=)IdjFOhB>Ex(+c}YE-XaAw&#H6U&~=>Cdu5Cw#xOKle}xp39T1LZgJgPA%9P? z#65#7d0HcSYmGIsG6QyheT6mGM&4aBTQ&#u>NEy^e^|3D zdqfHCJ=a{`Qu0*`vpj$1e#e+|NPTbZ(mK~lX3y(GC=WEFlqbN$PC=q%d*x%#IYzn&MvE9y@?91n{RlW-`j0**<&h$=fiuOybyUsdC2o z*qJ6!bXrT9X>3rQv!+P!pIZZEjcD1sclz^;3zIVr@xY(Q=t3XiDRB%mmN5em&|f3` zU>MWue@&sA7em7CpF2?{@7>!Iw{l`#i z?ZCP{_X^ybD=Z_X@aLyYfw+SD%ARvWdyF{nI9A@;U&E1hO=Yhqm4gwts_7UZvV8A3 z@3B_wZe1HeaqMh@G3He88iR>1h26872IB0`Tf{4F8HBhtwC^5^cQlUVuoZUNVGWxA z)>z^9ZI8&JIG18?%%e+x3_ZcVMySD^t2lW~&Dys^IPTmd!EvT3(U^nMZJ$GvJkAl5 zyLOs;&tWSnZrS;<25iC@BW-35vA{dWQhD5=bZzc^-8ti0XCHa?X^y;;v;&4-3VR4W zr0v!{hv4*@DL7!vRL;A`SkT+S5NB>xiJmev%v*EUW9BKgI#<^GiJ`GdOD%AOkcJ!Q z642S(`zQ_w#uejM|C*r$V-0canG(3`PkWVg4IP{?w}jDOi(r1w5Q4h~%<>*Mj8&nqGW6RRd1oT+t(Cr(pm7=lLVhMmg`6jn z)mqD0X6M9(lm|-55Bbj!_www^>k@iy5zv!o0PIW|8hk8N>a_E&!yaJ_EDwd=6myJ3 zNdaFY?785YCt6J&n^-=sc(1zF=EB=+8gwlYoiulJ_DV}EG;KMT9M`z!O5sN`#`)Mf z)?&bjacCwi=;k$Ng5X^fm1(W0%9yhn^BrlhIL`^u5r*2ph_mfEtc{&KHrU)4GY54| zxT}n0(7e|$#GIore-G)?w&zY!+bKqLPeh)vC*sWrODr#^Wvj3Z1D|>)0R1*c8c&$} z7Jd!TmA9sR$J&#!U>(!uc#l2F9HZvN-21CCj_~z7_M+b08wPJpUE4B;2ImhkDq%!1 zk+>$hz?lKPI?dvaQ)OXnfXxx<<>w%8lP*&tqqNaIC$ywzs-X z9@C^PZmh^PQrt`RkjbcB(D9s`8j zp7C~atR2*{lAiX+Bad??fU32|GI39f$aF5<$-i^rL`;(jGsUr$H0Ew$9K#bPFEPUv zhK^R+`x<0SdFB>}#@J5twkpgVi8;o={TXv^HxG4?n5JM{8Pm-z4F!=H_vYFSn?f>f z*wC}4u;H3HYa@<%nmZfY07bq$&9xpp8_*!h!X=Y-gqwpiHRGp~9Moq|4g%I)8iEM-o`{~faO$%OAi6BH>Vke@M=g7wZDbu-4b`?eCW@y}DLW{*|$oR_BLons<84L$TZ zrcm#h*{d%O6sy0pa9P>AoF$I=thIA4UQ6M*cyDl*g1Yv+?G2NlA&nfcI<&&F$720m>uWkjH2jdah-F@j7JqFBu&~qM z^iRv2eTPW?u&2z&-E*m8OjWr%S1i$=Lknq!$(FNr%t;*!vu%f!$TiZG;+qRlJuju$ z5R)!ST}ugRgx$Q7Ck#T}>p?Ur_H=G_tCn{1&Kc<-cf}C9yHbAPog%V3& z-<`WDJI;Z~yB3OL3+b{YZBg&C@``Ur!fzSj&3+ESg*|uVV%qbCYL4-=yoN+fN%Jdr zCArHx<}U5c%fNRmT+hB1w$z)sJ8H(stFaO)(i%a;V@J)ewPU8-UdvxHENQs2HvR1UunvsWXMPc_fLo(nB8wqOe<*UD@oZFbYjc+H}?z*SE(b{3P zbdN2#6Ve1?*xPk!rs0q`mcr-Vd1r79@#GeAglOGSR67UB>9ew$VxJ>`Du+3ey$9^= zPkVGK@70{UC$#eoBc32+Nv1wR^vqt%vSY?Hx;XagejO8Ff6T$CF$YZX9{!=H8|TR6 z-$N{Z&z000;x25;0o`s4sm3^WKJpv!M_~qq+cr1Qb4go+I^;>$n4<9C%1Pia?8)U5 zX8>^CcRdy^o$ZX4F{&QYGw=v%40qd!K9a zAcp02|-IZ4VkT!!`|s%5gje>go+r)YTH}W_I3FhxU*DZ z+XHlS4(ZXFGp^@MnBHbA4h;7Y{*|Fko@LQXfek9=FJ@)!cOi^rY?O2zU)+%wHqhENeWat^$OKxDt z8L*f#no3C{4{~nM$(#0e#F!&_CQN10wzuNo-a9ii4uS0-!a(>Q`&4KRjieZNM0Z*d zcYZAC!yG35(}ujs(MLY(s7>4+nyY5b9gmMZ)d~<25INn zqcr!Dc}qKLV~^RUlw5jfLRPD(z*k~#D@LP+EpvG8SWM9@3OPSe;E@Mq=yniNJ({!QafB?V=sG^4st82LwT?kV>@ z0)q9}i^yRty~eg@gzO!IJ1vJyoHhp7+g#hDbVbdxGv<879J_RR22JWVwq8ij1FbSo zeZv%YM*Uu4zAW#N*f~c;)KB4$DFn^I5KHKCcqko(!0mv}|d*Dy9f zMoKgLcf@g_AIHu@iGx^uhI!C1wy;XZU-s# zKc=#93$vy!gkZN50EgYieR9 zJ(W79RQ+9p>}Cx;)|g}dVazbjur|JHACt3XtRavQM$+zF^C&SV39GWgnrYl4)OU{2 zgR)i(=$fH0ehqlMJ4fW$TT|p~5B&KI^9Mc-IKrL7MA!|JL11o`gt6D$_6&n-Fb^p8 zAI7R{+#$&(>}hf3p6w@nZptzK*yV#mUOK6k~rr4c?=07X)H~v5QhrajXQcg zjWvP00%~{ITXrs{wZ1$iL|aP>5o%1u^_hp3&CN5YGYsT|y@oo}iPM8`F2LZrb_`KW zc|3K*K0fDItIc?g=#rcUz}d@-n|SWje6Pi{u_gq;8|!+156SSC7rf$Gi@_$1*@d__ z{&LL`n-_LM_y`*xFOAUgwT50sTT3lvjAVd!kBRU&hHS=PBSB~HEu9{>i1Ju@31DVT z)t_fh)k|xSIV{ECv6Fb^7Ql3P{|MuPpE3(~~B;)LpZ4cQ3@s8^grx-Q&Assgx~^&D69q+TB|@#AfFW z^gA~EcAD$ndyG+vGB&9WiCLg(?Wp}T){xqqF+*f*FqN|g{OOB9=4q~&qm{-EW7zXYZ->G0ocCPX zU$cff=G4iyx5)hoF_k%w>AAIIp3~WrqhSto!4uXHP+id~Xf3h0la_kfien6LXC3?u ze2%#W8pTXgpmb0C*p4$YEY3Z*Gm~n3&&yhJB-ORMg0S`;8(uDqz3{$6T4N3Q&2i^M zhc&iN@f(v&dQ2Uzkw*+%ptb9Hbm7=Q9&yVO{W@{M$XBbMkY_~(XhgB>t36z zcr0X?o|m3V+6nF@M0~)x=NkVWu?u{K^^-T&?(iNnmMIJsjht7AP+KFyGH<1_vnL9} z2+K8ZC4|p3mlRu@gPLrIb^1JJjNzGsgdvQ;;~rPs{R_hrHm7`$HHN@L&RJY%48fTd z;u8PO{rS8XeA7!CnsTnCmN$Yx(%3sxd2ZOkoThf@8IcYyO{}9k77+bSS)Dj->7g*! zZ0kt@n`9*5*pay;oP~mI>%p0Y&A}W zo3$q%cUbeLCJrIOwgxu(nd3q$MM3@?2Li+0`$amf$i}nw$okBw^LlQbthU04VOcQ| zVMUFan)gb6pCd_P?)BEWQj%kydHra`!L2pcVs4##n5Oz*Ty_|G$a`vXZv5Z9hgL(L z8G~`wa%KJPy0%JsUHKz3?G?%t=l(oKEc3IbQr=t3 zzG;o|@H``8lLlaIOH-+2tflrJ7G~~U(b;5;gr}9)B=6lj{}zWfcpuxmdQIK%z1Q;h z&TDQeETQ$cLTvsUQy6J&so=C$q|whoxn_r0*FQs|N)0O!W~LGC7{cShtHOERoIw;KG}1HvF?Mb)ym8gANS zayVqbsXB8!*;skFbj|s(z6N;7U%O;=uJHE}6147&JH;qvrOUGy;=qj&RBJ$Kf4*$Yl8EQOu0HrT`5vF0;vsOz_~664P^VrMM)xSB&eWFH$1I|ohE zwu0dIkDH}42aJd~BWB%>8`)rHk=8LbN`BhQS#?fP)Sm}9aT=>^IIVf~xF!a4Oe=r=m-1|i)jKRx4mnwT5vDb9WwBk9(ZpVyMH8>2df|OH8(@o<)a1Ax7 zxMvFbi#uU{XEnZ^0vy>}Y5QvLiJ>$W@>rcokai7l&^_14^W3R0Xs`TJNol=s&V7Qp zGQjg*W43zE(UcO`=+z8i-(w}2+dap!?O4$9ZYD2FFj^WNM_1ud?>C(HNheVv*|I{3}Wqi7AS;IfB;;aDS?V25;; zm1n}tUh`detSIiiHyqDdL%}%a6u+HkyyRYc4{}dv>=f4$NXs(-$ygiAd+m+GwHB1+ z*;z?q?0}Iwma5@SNg-f|W^u_=A8#xzvNR(;XiSrCcWu1UKLWJf%`0PeXCcC!qT0ucL0mshS>U}krsrHC za(2d5`yK`i{!Z)Hd1REF81m5h9+OaUtf8Gd_DcQUGtzG?t++Zp zc+cs;ITMCm54(bO?V;(rR|e!*J4a{_iN86=x?-C1c5$sN>a&)P$)1D4Ypo%j9kU)( z8j~SltTEd-X6jB^J1Ssq?X|M=#B1Jb{2}eozdZtqY~O1NV8ktx5F&c!SV7z>gz(nB zMnKFPi`;Jn15{$>o%cCs7F=UAg}A?%@qzotBJ%v-Z9XZ(6uF>P$j zd8s`^3Ta$puu zqPf<9{Mh@04>9;&X?BQI4xN?L&v9|^u z$H{AOARhwva?jfN*SRz zjTy+Z_e5PyQHwO?soT8cHsP3)oOg|ZyAbjm-re)}IuESHJBQ3$+q;lujd_-{w>Ev4 zi~k+RV)#qxu5bmtuD(Ko^w>$HD~yoAyrX9LjpI9ZuYl4&=kh?9Bf=)K^xHPky-VAa|4{C93V=r#5-_Fn^>V8n67Fo&?g z+c_C_ECa`L?X1YK_H0g=%i4Di-Pbjyh}|5~`D#q{n7f#3e|}F8zq-R9_796wOuJZa z3_ZRz*8o(H+3|g44S;v71L0*yP{O&wR&U=ic026pjk)5)!yfC5ZjWHYwK5`iUW4Fv z<>-*Q_NvXw%amcP4EVRwxb7OEbTSVSslC>6dR$SyChpmrGp7LR+*!V8ZDEo$$Ff8o zD?>K#Wz`nvhT0r6H7^ZG@Uw>Y(u+IxV9Z6mo5tE(&ZCez>`l|3Q^5aP6GUVqmUV^1 z$r1u$*%=FG(3Ll)!q^AUP*~%gd(V96H5MA_4m%igP4MJ5Bc{&TAs#XgS;;rYJm%S< zu4m8O-L|sydS9akXb;KHmEtOO&*=wa%`J+y_NMpT>xz4=1iCoZ1Z$X^RCK3-_C7L3 z>I@4qZcG{3Gj^K)ocXV5Me zpLt|2q_m(AmcaQR0f}J8waK%Ss{NePwJEL$xIgBC{1}n$Y7L#yyXLy@i^KVAN41YP zS6b9w6Xb2D#gMlK5c?U!dSk88{|xlLxw3e{%`@^Z50Ul~My$pf;{#((<=(PqsQAx& zx_L~|<(Af5UK-JNOw&aqFKMp7781zYDJ*8~LF+JM9{5gCl`02)j+hdXVcp4+B+ePN zmD0jd8}oK{Z6VZ|!b0ZX!|NgL_5X}h{&=j})}4oNYZ<8&WR3-sJ@)`QjgTd`BNa4PvZujG5C8@7nthWDU*vx~C}bjY$hB zC5_fMlfd@egCuTlWRp6kL{p0k|H79B+U`I5mwq|N>Q$8B+;NeSLl2h zS+jpE`JlX~qVYZ@IWPCk*vXL;7T|iS-e4!du+AJz=Dz zdn|GDvo|{Lm=X9VZba3xlf-{pTgGM%S=zG~gjQRt?_!PNwVkH~TiNkfe~fXzm}9`j ziK*mghXI~7=B(jKqxocpIe>RAIf|1;MD`oupLLG-?VTrd;2i6%Yy@$ey;olD4$&t% zkG;Mz$NXCo`cIERF=Ec)o);osTbS7;d?x(szUScZ%`+Y~r(yqyvn_E<+>$ZY5`LY@ zoPR78;*oZ4%3llJamBFhyE9(>pOcJs1cCn+_BQ!i(d1=qIfc3hY}rjKHELwRt~R&y z&l_ujdSpcZp2LA4t-X&O7V_E|TdHx60p2_2-0sTD*k#Tn>Jyi4VO@!HbL9oNw}Obt z8Sxi4WSQ+UH)`x0QK)n8*~_xG9K)K^*MChQ)06_5++K;-IE`uF9ipK7+jGQtr6tml z$F^<=+aG#Gl=wChMBd!%A${!NsXu2@&fJ4KVGlvmJ(hG~nsdQo%;3PDGdSp8I|g>` z&H6n;C~rvQJJ8!2JF<7Av9Z1PX7XAaAb;oN{+#(YaLs`FHTInN7(+^D59HdqRuKCh zgZ_7IWa5}7xaFIf#yad_%{ns}QCWFsYKJ6|nf8G59P?{_=OwN%N9s*nQQUs7v9k?i zt~;WzS>2KLW3GXixzkcuT479oF5!wdhe*>56I6WWA-@u~g5;eO4snE;u)N2z)R^&S zKCF4}zcZLa8ms7kXZfw4@hd$h0h_c2V%d9**r&IbVsc*#&~l`VrnV>gMPK6#Y;9$% zGuPzx*zpf`thn<#XLewXt805qu!*^X=>3d~J|}NU)v`B;=pV@|BW>CElvY6dPh&1S z&7|9twu;RgGjlV~FswczL}nV1;Am!5mbVv-(_FitCTs|G#Asph&SCfSQiFJ-N{m$OD%^-IINEiTO5GiE;BAG;S~ z%#roBcS6sLQD$_`?A0~mAW~oJCpHbq*S2#~!pU=6WQ0w-JhlO|l~+#98%wMq?-8*# zq;%|F(*U0I|3ai)#tHDb2)UJoY9p@TvKFV}?aW(>Z>;f$9>)4+n;R%_g%#u;HpYKjx#xb2xye1!NZ88z z`)IBB>p1f`V$B=2X{B(0cnnCkv7%V)8qrN+54@u|IXcCZ zro4wLcM1DoXz#h=viHVTPjfVCjSQT>cIbGELs4>$l;1zH26N2`Yc0*4^E%Y!`IC&fS=rO0b(maPKcgcGwXzq!=xTo0q-x(8T##Nb<7C`de+Z1i? zh|@T;nD1ZnGiO9S!5B95NgnIBDDR!$H>RLN+xgLBZ1nz~liXRzDJ*4WsJssiqMvq( zbJ?@UBX2duGU8Nt$y4uYg)z06$l_Wfy>-qhy^!+e@SYnxYwhrkzqjJ?%cGil zL{-9;#-7+t5iM^mEYmhN*5RF-E-Gztp}vCr=bouCeQq?SKexuf+C#>2EN$r#o527JacGQ%hK4WDonH>J7&{(}#TAK9i>jMT5J2C|cg5ea?JuxO1Siatx7}s=^RfEIrswTu8ogk@ z*$Zwf=miqF!m-MfdFHw+RmWD~XJX9l!M)$tS*QMJ6h-k7rXyBoO%5*iii^8q>22DZ zod8G@52q}kOo??{4k*$^1QW7gO$6*L5tpUS=~esv6byb9s8d?*TYtp2L$vzAt*QS< zYZv1Z_9-U9!lk)Xxe|gwUDg++lmQ;?h$I4&mU|4 zH7i|q*FsDgx;NGY?2$g6qque=#2o|FzwXVM#({^Ft{9Rc45J7AZ7P6EP3UKnOr68Q zSZt<(deGT?*nOzZw<~Ef*%PL_Wn?> z1)A9ELm}0zcbOxrJLKNruQGhP-bcyPrS&d=Gu*MBrAWSq(^}vYtD?-41~S(*3yO!& zM^5q89aG!d$O(9bQY(Ap$-u&dfDqO=LUQHL7<=cS>wTG%Ct<(F$+^k6?@)DHE>!<5Nvy)wSPK5~ILRH=hrOA6AkYNXC9gn^Q(Ci?%s+P_ z`lH`~|do+%W8tWV?euv4a3-?(>Q^U%lkD4)Q<5<@JK9~uZ z*<&r^82Nq>@D#N2Waj-yvT;?Wu(AC(c`O z@eMGQ(o=h?#*Nx(_w_iEPET!>c&1)6wS}u_?ykoZvJ4G6bI8QNy_6zk*xlde{ZN1H z?2>c?E62sbMEl+T>n9)H&YBG3U4`0jlcv8~S%kqM!kq1y-afa30;nJrsy>!8A9M5M z+km03Th(3~D@FxpaKR@i#VK9Cd61qjk0iR9j88k4V$Ir_@+<|)V`aW0=oOzgEbcwf zi5NTpBZM|2=x)abLBcjVaVr46nZH?Ja~M^?7SM1~*HlRmNI8#{kGw~$+wHvIwNDne zg$-o+w)#SyW*!$`UuXC?cnW^4c|!WwDZ4XDhDNAVG;(i>iFP-bQt6~n;ja>uo<%l2gVL~|PZVdajh8NT`=6MKS&PLo>e%Z3v`LXHI7mki9?>v&+I z>~P~}xipVXHu$t7RewwiJl_q8j(*m)y(5iLfNCip_(%NiOH*Ilngw~w4f$Twa)mBr z;WRyWGoqX5V`k>B_Pwy(At_TUZ`gt2z6hUmMTuv_Z~TK@w0*Y}rUg%R;07+LaT_U^ z$s@k?W45rlr?hF_!Z%M=i?i{8#TIrPSahEhit~Me^nx_Q!`C^>>-Grdys9Euy$mg3 zFcyW+#$=6|?+s-AWQ67R9_V>y(v=;Iq*z zmQ*iVWx3kn>rp*SS4Z?9`#Y@kn9HlsMz#Y5YYGV=ht(4)-c;43!S{_TSbHzJnIXjh z9b)nX2AabaMdXv^lK!`8q-^Rhcyo#=X1g#VN*;d|hpf}p z90eL9Z@l8mJDF`6R1n-t?oj5>O`+fCvfLV9B7W7i)eA1Xx!6a9vB~c?&SHAw;pUvd z{MOfVNc72e{KQ6pbAL@W>5?^bX^<#o*G+Cu&4sST9e2OC659m} zMq!jzapdQ+UK`_?%Y_c%f~(?I3WIa-n*2JVRuT!)!-iBJtL-lwb4nm)#zE@4i&SDG zeuWteImBT#Yu=b0i&y!vVqhl?C%ZNl42Q#d4p+{^tnA!>6vWC2$AN~zy8WGe;JB^L zN7G0DhELal^sj!;9KQtUHe_de)tsW*X`VfucqNI*j^UX^Dtrh%(1>o6-uxd@%IaB= zg;o>zepgu=-}N8k33l%hIeZ=TY?#Y_En4=^?N^in8q|<#4=AswidVT+Z=SLn7>^J;2mS;pKZheZii^Y1o zRzBjL{jBGgTBHN8W~{nCalCz5H4nUv;SJt=cFi)F1|uT@Y4^R!!g_TOeAE2zwyqiZe(Mb|MVcg}t>lc2n?Ik@}1|Tj`E9_$|y!IZ*ALhRtPt;Z&T8 z6wM#M$>q-B>dE`uT#xB8T2=IRrfO8^II`~#>j&;Nd=}%X?H+kCD68u`fD%7a#R_N< zmzk?Ay$f9l5PwhS({xd*=l!U$O_+<*zc|w+523t^^vL-CQMlc=8BxN?c~AX`=H&K= zv1f`!eDY_C-L=sP*v&qc8Dt^hlOX8)LpdI}>y>AVPaPbwGfR4l0O!-E_qMYs_Mh`S z_)7;k$?bWBj}C)7sEw^*ytsN6C|abym5fWC1?XB7cnU`B9%Oa6cMcmgO7WD%%SnIw zt(}?bT-JEn1otBTZCAq9wbL*kDhxC(w*(0`kqQND8>XFWoa##X5>VTNeq(ocQk{$1 zqT4-inYp_}?m`^ouj~=fkS0 z$w1ACm0%~QshVTSS?1%n0oW&bOop<*6qBPbEVX~feK8Es;JEthR#rF3RKTZ)3%m3z z)?W}iH=8ui3>J(RjFG?$bg_U~U?P+&>CiQ`K9Gk~;T=sLLGuIl4Z*2l{REHUxQwdv z=kw-`L@ccDJ}b`>i>p-V-o`(f&N^e>X%OD)Z)i!P&t=V&%03GLy0R=U(My5h>he?StIsPG#n?b^P&LEJ%F`r`BQ6yJr)| zrTTzg;NX{&AvRJAFVnC-NY9raxqK=!Ih~)uI)qJyqLUG3`xLN_=(nFASaxTQF*f(* zR^-%}y3BpGL`2+BRiP}mZ>n<~uyVVSUPh|A1;<{IY-Nu>JT_mzS3M(KtkTzyFU*?0 z4T2I|0KF9lnwexT;@9V;Cw^=#vStLw^OV(+F+mslQ-Y}j>zmT<rIUUUu&0pRX?TWdhjGpZq(7`pbr;47Qisk1XW~K$B$E*#AI&u z7vEWJoi`Dgw>D+KJce^KJ+qMKeqHTrV(;dD#j+Oh!br5-giq16tJ*u}$w0qKm7i5b zudb*`eshZ)qEpWp`jL+B+`@&brPFM&A4`@*Hy^Km;kO=r(ENi>_bpO(b99~sB<$Yp zl&Yv!t7+_b8#v5Vz!FKhkWmV=?xMYDBdf3-9&8LIyY9}#^?Qc%reu1KGg#cX)w#&gT;N!%)ln`}^yP3^; zouOj1;{xo>#yd)WiMo+*@7=zG3o3lDb+GH!wlB zWI(6c^2g{Na6-!wyV~%@yuMDLaOz&&8A~$hpI%dAJE_9{mBy)GE9@a0QkC6PD6)&A z1W1Q7S-^Ip^62(ryADx}KG~4p=gWpfI8)27N+^HtnBh+H(EfU$=>7K^r=QOyWY#EV zXuG{&YHfvZ#RA}uZdhX#eJe|Dn%)f=3>e%Eq_Vk^fj>JJKre zxj=de7&uD2z75WHxE4T)?PR@cW@JJHpdy}aDv)K((sYIdQZ=N?M>cfF4q^g}e16yT zyxcIRb>$bFjW{)exJg38m$E;!o%YeLK=qhQ07ZZ22lFZ&^gIyAsi_N^%*QWW>UuC% zpDwebpIfd|R@C+U@iE@!dk$s)35syQWtYMN$Y>Ej?^jb6FCKEQrwlhiUw>dvPO;|g zFRoT!TQd^-?|QzJ*&ac@s#x>7Ye4>DNA{d_!=gFFZueCO3zztNTymwX9@{?qJ8E5O zRt;N3;-GvdX0F#+*8=y;vDf!UBfWJek~7goJFZuLQ%x6T$|q0#J1?1m^1y@>@5xW7 zZ|)pzXT=BDC#Z(Ya$9a8qup$}%;PcYyK6(-dye0*s}1LDS0-T|9+bTOGCjWt8JG0# zf|q?y`51br@HYW2V^iYw)*}y*xt0lKRT5`O7tD?g)?}^qvf@ok1IHS!1m^sAGa+yM z%9uAL8G2S`^~G)X+OIf7Lu=FQdT_C|fr7YDg|H!;IXuFy%%@mUSk;(vLM~{^L;Kh1(-qjMg(EzOrtv!qfA*ijOIKGN@~JPqeTQ236FX9s zse1Ll+w5?9{(H3PisC9ORBlA6pzpC9pUkQGgs2zxVn9Yb;AkMxR+d^4NoP=H@zgY! ze@}D1#Wvo>0vWDQmU8qHVz$lFek5K7jH)A>+%qu{m*Nj8m6XQVyrFP>xm!|~gIOEF zIhOja-ITN%JW!>$l(g!)n2_D@Tc1B>@UN^Rv_c>4X&bxH!cVZ-f5NS%aG+?9Niid9bRmYq;x z3Lr@Xn@M~3xznAsmZFXEtkV1P{0D^k^wEB8#|xWbv7IdStiLo9A&$ahbGgzF7jOoJ z{_&Z0)du<9Gs-bw9;)toW#JPGk_FEM@|~39TX7Y!>E?6lzgb8>JCVQyal`O!Tmj6r zjJz;=Q|>GiT`4&f@Grw4#fiV=+hfMRGY?r6-1i7GK*QTXHV}7^%goQ3m!a5e}}UFb+zzOt~^bA037DLYRjkVK*1$p*u-;%Y1{o=2nDDx zDQy70`@Q_sqLZ(sK_PXhS@(SRFaQjZ2e%%;HAIk|JHlYmO_)(#& ze5mJtHFM9HBJ+fu_cF>9#;wgW!oGTRc`T-SA1vufl(_EdPtz+8z~f>~<#t=QtB-v4 zqmjvOlu|@|cHggD5DQedY7gz5vuGMJ>O!O6ik?u|dNXz@wl;KJ$8Y=BIPX}3AlB0a zc(BZhm-Q(o&(EPJrr-L%S!2JzMqriIMb`@~=nr0>nTJI;_S$hs6Ir3|^F;9?;>%0Q zPMvlmS>3_P+*m&{X-*)#{;Hu{(ALL&yE z@(u-b`iH7FDfc+=r@kWA@nQyL+c{T11%|3N*bVI*j9FJjDCb%L5wDL-;5&B1qiDqs zIla05z(AqICSZ}-83*&ArLJmP5~lWik=2m>4h0y>_7g-JI+S%7@5|%4nvxW;n}y#D z=#g3xn;zGpn3#6PK1;=%z4`kw@g0=lJC_DQESWgM`P5QBJuh}@Y2}8ke2l#gBBdRw zPAgkuE8uX^eKz@Y+@zhEUWMN&I<~22L(6pV7*-x#_mhmeo$c@f+?`k`^KNvCn!++r zSE^a&1NH4d@{(5(`H>`_g>KGemrb1JWVPDY(@btFf2aHVL>t^WC2N_xGiN)o>)XGG z{_7HVy_aaqGMcg7Ya8i%i7Dz^Lef^-Qh{^o41YLegJZ1MpMYk+{hSHLHM0EAHc+^88%oX_P^jhA$HM{p$dC&e=#aiSPz+wacC9e=-G87n%$m$; z=;!f_D=+x)i($^2xov&7`zoiBk@xl&aGTZ@%;AqP_f+x3Ml1c5{J!tCT;x!9?9P8a zvpeQMN}*p8#W|fUn$aSv9((oYRZH~$ZDyBJxWZl~|0F#ane)%~6j@43Wt{6oi8($` zi2hrhCi6E#z|vitu>P|iHj`73WTR_?q$jP}Sb!&q5L-DioW9*UmrS3G8L{CqHwQdFbMzC6gY zZ(dYoD7}E3m~|^Mo2io?-yIq(_OC?!N$dzcf4U8ZyQmUcHhKIh$qimqs1z`e_z(sh ze+R~weXc)$s6=&VN_PKy)lWK)KO>M5XR3_I$FGOS3MA~}b+G`CK8H3?*}x|-JlTEK z-0oA2Kc$=NU)1_)%OCzPrk%}q-DeTG89M&viXNgJc+eE_O#5TAun$dMh* zWcoYhndVq)9@OXNp5Zr{jXf|Ja1^5)KSu*zaf+MV6^*PoTR2v-(^ZNA80h}hi~cJT zXFgpeO{cnO@c&McS$T;ROZG!~c$e{f#x#?bE_a7t#p$ocdb3+oQ-bZT=w~4OtFejc z5V(RVALX8nWimE&9@40)iO}6;1<_jZZ}-Tg3+kAX#EJ9$3PIVP{WT^@=Zyy+vmc3{ zG7lQVY&|!`xU~b#mZv~*-IK`3-F0KoiC0B+O#{D%zDWr`R0NpTKr%X}6}x0V!t{ls zC+nXDc`MiY{d$?S4z*l}yc_kM(6qDKlSSM+kd8`CLnKDU%-_UezYZfoI=-$I#trE< z>wi@Ici(idOi<_d_ZnNd0}F`@yB_(AX4@6QgTuc*yqAkrhFM!V`r?7*>-$W-K?FQ! zt-J`;*XF6vJru2m3sp$l7{9lun(Sb?-9UAzJC@aAgIpzR%_aw zP!5K{ov^qXX~XfME$#P} zVl{pnY@I%iYVwh-wp9U?npe)fm-E$phMTC)B!9~4tSsoogov(6W3+15j z5B2SdYBfy&k%Bt}?EJ4MtFarhH=kNd^6bw8ts|JrX$vsfXnYY~^#T!+3*>t@JHPtX zZ=`iW{z!74)wc-Nb<>NE>>Fo-C9aa7p#tI$S{fJw%(ox+ID_&4x zJ$~Tla{sqnPsRooq)dd_{lVBiy#8oVBSuDbln59)=uejL62|UIMA(|;KvJC|OAgX) z^GCC|NrKhijwbgc4d_NA9aP_TicMFIMOE)S*9-WQl^nWgp#+q6(eH2V^_eztn}QG5 zmY}@9jArCTx2Od z^ig5P0HLsL4p2>iDgmFuJ%0GHG$L?Ve?}f1dKy2G;;&~JUklH$M^3ni3Re#TNM#6S z_zoK#Ffwg*9G(T{wzQS16W=`PLIU+)y5xPSj~BVSfJ0v*m#}8c0&ZP`1J!^Z1ZsO>$s(l<2Jg@X z^_SZT%l0nVjPfS_8Q-2Ze4OKsyOYOhbN%CPBY!%n1lr1avV%UY0kh z4_G_JJe@z|ex{ATPe8`N)NmIK48Qs^;a8R}^Kk&UkD3N6%)N3gCAYT?j}4oa(UHHx zq^BLs9bVgnzmNZdnH`8}cCjO{H%qS=1^3)(uEM;*QYQixHAez;pX@)H&Nzv#-nB3C z98n}1ax#@;__`@avLU2@qDVezO_QElP`*VWYZzQLuDFDCUX)3}Un=F6 zj&oB352PXY)SePOd_AH#`40h;8iN(V{O=Jha3jyb=GT_x_gmN^pKWF6Z$kMmkjxvd z$1LP!Vk5hADdW^yZ_th}tSmf>s^=(Om8T zUT+Ohm;QlJ$zg%~cKxQzE_Sw5OK4J`ja=~afW7%h`l&7W>>$97%=y5usRHOG(URWwIB<*LzfjOY4)vm9ZgyF&(#?DiTeTDhoijE(D{DuI^`5oxD* z$5h1)+XF0q%x0*ut=`-Eg1Hncz0Nx!3w>sWEO_ivE|rM}lueB2-%8(WM@dJfIWe2g z2l95O0`u=|KdoojSvc~E^Y8bsY_FBI4OrG190Qg;YG_UR-+CXe*tg>Hn^?rV;q2H9 zJL=1H+M0u|Y>KxYDw23<^qj^=gLVJ1=r-Ao_`|2o+Ct-8ba*C`j?Ug&-_1rsy_eN+ zT&$79kW`GkyZE35eZG98%0f#`lb z{Bd6^`z3ZBv^;~plycFGY3SGrW^+4*$4d^&ul^ayR(|5Gc4xeH;^198{41Yz=Cx?A z579CiW3#7pjqHi?;XqQ_@6nda7|M?57{s}jz4(p7q}XXE3MqV3O|F*6{=+fRXxp8U zhe_73ra@o4bh^oDvFdVXY95q9e2;3R`!cE*on%9Ex-0{s26HTfUfS5-)*C+b)u7)5 z5$|4&=qM{(Se+I1i*GII3-5GHL_UvpU`f60<4abGCELyRL{|8bo%tXuw-Zx!Cg7q=1KsSF09&v#XofzuGxT>elJqh*bH=FY zMuwn%_MLSQL9;@BI%nspftXFWeoh+%GlW9VL!Cvp>U#^fsq)q#JeLKGh7XQ%wB8Ho zJvoq1nt5vsZ+xcfvC3+YKhhu84#El+BR9CV7sckR_~sCehWGwv`hRmFsA3tys7DfZ zoz%_{1Yqxn-V2M!k%^<@4j9z(!uCBKLO z+c_L=T_OIkNirl;mcNR)TkjAr>c5sOW;4olEU<~gr6*xnbj&%wf)N_O3OO^ILP-yE z2gRkjPDUm3k6sOgy^?Xlb+qw6#v@n1{_)k;^5cTdG$?j(Xfv@+5}^sf#>tX)F7mkz zuNSt=;R=BA@@7o(K_^oA(#odf0K^!tVcv*xDM#U3d!L_wR0{RBvAGsY&)U{V_H`Nt zw$?Iu;i(N2Z1lQg*}bCt;z@p4>HSVgPJcL?eTlLwY)_~+;5uPxuUA|RSjf?md@iz^ z(db3waF<#+BVG&uHu%mS3~GB20{g_zhKE@>x1A$1FOoa8EjgMOMX2sHCmLcVa~gAn z@8=J_(}6&oT$Wae!<0*tiES5Rz52^$O@9K!*S%OMzcKm8^4yMXpz880En~(&20gQz z&H6EitqNqI^t(&()Db*XVX0h5RLK#p2tRLNq8hixT^BBs4zH(T& zGz}Q$9Yud|ofw2X4-j_xqR0?%x)7G@P?>%9#P;uDU#FM&H_u-z^`z5_;6Qk>(m>0Z zbBCEylk5I+c22$RdF&ynrK(afitl?Lp5iYzthP~>^v9?P@EE+F86q)}@+L<4kc($~ zw92YyhZ@)9?7X`v{XxSb%0w>*h~(V8UPz2POyN{5iB;s<>R^GBDQHvmW72%|jaiEn z5_soj3^`^{6sayYLNTe*IU8(YRo(>qZQ4$&j(+kb0e-OrSqMg%s+tf06J2|3yKAmN z@})h;0Odc-S^HDkYmLQU%p>93PPVt#wE5wMgDvO&)B370=HriUy$3yt(|;D&1c)Q_ z!*Pk}3e{ZYa3|i>o}cjX4T~Ljn{VZ9jP&`Ji|9e?xWot+I$0RCRt)v_X8{`61;Eha zpH2zhqq1=hRDaYFho*N;koi_OVfoJ-!sojoR;}PJZo?$;$)zVGhRtxMbDkR}GlVOz zJyRK+88m(2g8h>naTLs+%mAi7%*uge1?O;{+jWP`qP$-oWb1cApt!%e8{xQFWuktD zIEW$*xXNl~bW&R)!szm$a|OK~IzxHY?z_`*j=s*s9WDJ`$x$=60h=MG<+l7sTbam9 z53JZx&r3Q>KiK)Uar;UWB{ZG*#zE0b;~c?3>BtxII}IwC@0d7^0hxsY}q^ApHAs; zH)3k8w_Mj5sAkZ3W1=PdEYdEjs&nARIOMB`>Stssri`#iZprH+MyMyPIiR+W4G<=# zuRrWLiXzdT)%(K**ei52JzZ8l%)oDNaKV5}r(J`(!6>dhp)ws86o3W>sdk zQmy`EiMY9WOlC;P0==|KU|t5Pe>IQjn>GQ>z+gr{+6>z|X1Q;C?h*0X1uJ+|;u}cW zGeZ4z=&l6_A4p9oe`Nc5E3@D%h<6g(=Jlu7I5G33Q$v0Wsq8SD1e8nX{jman{yfLT zj~A*iJi6X$V*rva+P(cHi>r2my>Y@W?sGNMx%g4Pfj=*zKd+h55KC2na~YxhrEuo8 zF$enSPeVy8I$$RI!bok(d9O02hyt?2{9(sr06z`6XS`@fEPY_LGhjw5&65(8N?@{Pv43=15h3I}N zZ?v+&BeiO5w58^f!iGonxGx+COS>(}4S3dO7ibxCjwr^@`C;|zXOt6tuDgiMR=zuh ztf4bjv6|u{P8!LrMO=&2$FaWMslQT>zRWGeDnSLStC>3(Hi;fQULKVlP-h>WrP|nr zrnR;$f7w809QsUeMnL{0Ol*(I%Wh&0WDUe_kF$IjTA{kb?!V6&fu0upjYrz`lqahvlkCX%9((hXn6rm=Gsk*h zL|1)lr1-#awVCwr%&+VonVA_{i$LD7g|_iKeUcbms~jzJN4>Wo7qyWN3eymOt8@E$?nxvN>A|3#c7IK7tXhGbX54zY|94esG_CDn+7$H|1%j9 zF!a^tt){2@b*lx+ezrCgPD^LCfGXIv>prcPTE2& zYY%A(CxgTO3D4nDGt)<0jKTK>cfkjBYP!4gO5OH_`go>f(Y4&p6&SRENPd}G%I*9i z=Im4WsHf;eO9?Hw(e>9PbNtqP?n)X~>C-{_;&l zwUbq(#EHhGpKsUu2>RkrJFFpsZfi=1=={C`3-}~24R4!**W>OO2`V>&Xl^oOOBa=V zZLcO2(;#lLlhG+}v$qu67CK}S0>kzQ2`=&zV?n(wU}PCnO;xhK1|V zBHo_B9rrvbY2%eStENWK@XbY9M6Dy1p20^5amhmJI+e+t=i9A80KwVt3+?n?y0l#u z8F^`5p9`aF?}~3Omg0Q&2%luNQ+{OGKAS>~eX=2v$U}a+47Y!8Sv`9bAM5Fj2tmjd z6)NN1BH#%ps{nKaF$pU9GU01?Tm9UbCbx&UHs@tC!@a{Cc#NwcQ-UIZYXeKZ3fa}! zV6=6e9_q~U*w=2T$NcY*m0Osf@M2$v+0-LAp**aH{~>PX?(6wnbPJC*RpukK$I471 zLR?*Sy=lVkYTooxCbBWDWpj8RRsD$f1teHdHM=eS%c%L7k^_xgPi>8`ZYYuTRVOQMJ^04J!k3a{L!jvSv6SJGllBBe!ATP%YdAi# zxU=rDOSOS!-?WN-%w4^#dUcSp=eE6w{*ruL_h zMG$7)^dS{tarnggQp6X9_-eTox?!<+5JfI*^R#cRab8Oj+wv?b*vsV{T(75t~Ty zw3L&qRV^P?rIW^pp;yrqwE;t%dD2r6hq>sxiU^(6CyX^qG0MY<MMB;ziqYm*JKh zLM_t09)1?dg#hZ>F%~arLPlbUleSM^E$K5U3BNUjGNr4n+`Kq6z)j2Xy_4Gg*_`@N zf|37?_Z6VxjHNU$NY81vi7xckjKshx;eCgk_v=B6D{jk5v1aMCTBDg#0l0M_8V&Kn zEh=XP#NFSoQ-7?Rvq~OVrrjXZW4A`-&+Kh$AKRREG%i-*{a<(`^j+ zRgehyM_O^y1+Jh0pc=lB5|K2HRL&1jJ6%*b@_1)N@8MQ~ZOiPnRc5AWb>a7Il@Eg* z>l%L$89tqsyE{8UUhr9x=lzLzI!w=qv)QUV9G4u~po-`IpV%%@T$r}9{rBB|7uI#3jmm^F9&jTv5 z|I#LF%q%ue26dG*O7T6|$U2tvLAV{Xp#K=s_&fWK#Qm%h*MM*+M+&&mcCr>F&2lXF z_(r&%%eZs=i_-ceOln@396A2_LA3%1W>m}O^?jaIdB%^_dLA_O${Wc_Vcq?_vQC=C zn7O&%o5pE5*-!(^ED=W?X5qemuO&q~fG|^nuw-w9faAMQ0Aa6o=4n^lVAT!sq3~yr z`x>bjbLLwc8ur%umA9kDJ8wPj33fA{>k{J7e;tKHvzdDz+nUqYyFpvRGa!7!tceMN zq=9MTXMg-Oh@Bg3o9HTT^@EjKcdHnL>D*NLMs72_P5ML2d3GrggweH%VHn+^JkESU zbFC6n@t&a^5v65`}ggQWOlyxtRZ;W0+HAqmT?nxPG$Rc#ldfYU|TT1o%;z&nJJl z9x>)#V81jW!*<9GXj>4CGbm zqYHKJ*Bvb8rA|z&kIdTWFBuLdJ1nU_e|gT@EbY3FssNq&b$uM{a<#GEtG_#X%KoCz zp-eSdvc~J&c>5bsEVM2*H8+FOdjiaI7QIZ!Bl}`t88IY!F?Wa5F)}))=3Spv?%;^5;OIYzsmTsw;JI)3 z?}Lo=rFJdTJzgrN{j9L~`MJ^znBM$mvp3PUH$me>Xoy5bc+D(^cm-fvXBoY3{Ue>0 z8Kl=HQLHwOk+2yB{-{4$kOC{xFI%nrq28<%DBt1M5OK`+^{iyLq)2oXYf}E|o9!-v zee68IFlQk8a8r6n?BZu~ywZTy$Pzz$j z_-`;D)jp_a>dY*RDu~yuvAO8`hnZNUoH{N}4C>`#fsXe{z@~~T4{u_{`dqs{FEerf zS;^{D6L#_Z_M}*{V}zDn(y5M^^q@S>UM6r2A6CewJwMEH%L|O-se3xrG`hGe9mnxs;Qi){xce&lKd_+WeQv?6q<9 z5n@Jk8mF`{(w)!Z+ojf`)3b_upgAu-y>JO24piwbXSN?%hhtkoA7pf*$)nuMD;tGj&SgLDpvPw?|v$%$7D?t zQfw!3(Euc2@zX#n1$Nulq}s}^b5KYbZoeoz%3k9`E^8s)IpoYzhAyGr5nia zv1+dL!G1UHE4ufY5&i`ytYW=e5@3Z*C@u7|9UW4zYFI_Fnz#$PwN$;6w9lyyknJuz z07}qZFt_JAsh&(SeUz;7zYfd}E26u4-2uK;_a{ z0Pn%ufRpPL@5@eRc8lLGkvzW*th_|Do+H%eh}&7A&#^VV9xJ#4s^AeBtHQq3w#ec0 z!5t7#PX|@ELfyfvECE}y*`|+tWSgsGX1|GL!L&mkAxDV>&p?jFk=+^NAE_x@pPD^e%o+q3RSfZ}|bkWjXTk7y-vbyQy~ z#`63rc>bBwaxA;STO8XUPDYjLU#`-cC~wKRoZ8%aMq;q7@b(O7Qz7WVRsZvZC#|U@ zT%HM9Juh`V%7LLqF6@Vo0|bTQB`~Ci7O@EsRiQ+gyC=IjrCvhV>}n3o11ml52oII^ zOP2gUI&UN%fy;ekBtHC5T^yt26Zw1y+RW<8)@p>SG4IQHXDOLz+IdrbnWi7NhR1_) zENxEm8X;Y}efkyB&ddi=9c+Zz!yka*(GY29)>JgeK&&zjC5n0VKNilyK*7c`0N;Oc z!2Ac39L6hlu*Q92G;A8N^V%AQI+pMfRnEYJg1WY^r8c`3VpdHdy1U6$eD{p7?bNb* zhv~Eo89`>%IB=XVN<5izdnop+0JYqOxY~)yoPkxKpx9pyZW9Y!>Ay$CEig)h1Hg#( zcNyunUSoZTQ?X}7^I2DKMwp{a)mUOZerjAvL^qV z)_ZLYiA?0IyqK&yPrDj{q*{K9n?md?{LY;CgI2T0jzW4q&he68(8+V>K)uY-fOy58*ca&Q>4l7w7GiEOE zSxm^X1t1+m7?2S3CTI*X(ot&9I<^GBP+6>`cH3;xdA5RIdK)$z##t{%KoOT|QpEVX z(JI)c*lbuT-FfdR{!c*H=Qbep0WSVf!Y#qL_2-B1?vHuZHeE_xa%8q9Gs{@PAvw<{%&sQt-FEM?21kEPCuz*Aa5SftXFg4e6= zd1VI2_3hJY_{wO;;4eEJr&aj~m1{imQLq|$zGx^9E^92_b>H)*YWlzT8uam2q`n=B z-r}p~_*d3c_L)!8tVTvaw=P7NqtxBqo|v$A0YiP0a*Tb`<`Pi?P~-8cw4{!l%qnT; z$%>yFKOp_3ISvS=voFO*ZgGj|K<$%y8IpH%@yQPr2(9PnNI!h$aw;#c=&3Z>%jrbh-AmxBUHK8W_OwYGvbOjmdPbf45Toj>l(637mqVs5K4e0$;7n$}_(gUIclOX4ih9#?AjEW0)WIpA;?Eo&?bk z<7qTq{KL&M8O8eu{r*Sx!{ARuS>7=4-J2`ZMt$d@>hs3l$B$!f+Q0*=^uyCizdQG$F2}4?7 z)Gi%n7)c`SEcmGICuDLowmA##bhwYigj+7ascj?V*3jwHBW#0Uhh2@ly5di2Pdl#^ z6MU~|CD#0?wEz%Z{hTCH%Ast>8?m9&8$j{HE{^0_x&8j1fQa~ACW&Ohz5lAyNXnpQ zB&{;eqU+)xQt(3pfezouqv(hN9eI(|euZPB-bFl8!4@@BH8G&Xrkv!W+WPA;g!(U@ zyYl5QE)5`E*p&bp0XrqLbZS$Kh~2ltq@s6Ma|B+?QXhuNPp<@hqXa7WQpT8-I8+4{xxvW-n^xE>qJkh=X^_|E{~s zax@dd^ODN-mo*Fm36^pPbzrF2WBh-74BVx zDlm@liA9VJfu_&&P{~n2{3%I|^e6ZXeP&)*W)}eS%qP<^TN<3{Wf}cI%I2}ubE!a3-E34$!s#rBAWj~ZSI$dbI+);7gWNv z5j~(@vsE(*TEzsn;h)C|!Evcs?CF1Qamt(4aig7uK3_P}{icv0e{pF0OijUznwj#T zmCnp+4&C+Az!l@OuJ!Prd>f-okXq^8Cugk~uM;L#FV8rcaunblF@}B3U4fl#;VZkDtP~8A7Y#llAmSnmM!Fx zM}VDl3PmHW{(J=Of(WO%b;|$gPzls{@4s&*f~Vid>L)v@z8$NL2sJkb&e?BTRw6ex zdZ|leh$N;YWW=1+tU-Zu9C>VA4-3BkGS^l(Zae!!P?t$Gi4FGsCL^QCff2e%RuHw& zymF$2CCBSCM%EM={GBZgtURRNTSs|>hYTwq-MR#f%)V>?enCkhI=S3xRIb5?B+yjn0 z8*4q&-e!SVIFz|9|Ll!9ir2bz-&~F%egFvb5{10BO0A+t3IiiMf!y7FUqOs?%R`O# z+@SCG8^{92SsgW?uzGxyARTat zJapz}W{fdFxTK4zh)s|A&Z^7$<&?PL&{!NWmx%2DUc*4TJk;N4voDGg?@T&k75Ja` zFu0Ft!?iO5$>uYZcZ#kb#mghg4xwm zwXOf7=&b*mdfYzz^@E^DDJUJ%A~^=5K|~2D6{SIPBQ|PuDWG)sXaxyrMtA3ekQf^v zwSja<2HNs8=yr>sNv=9jxH(pYZF46 zjO16xRb`0HEXUn_bc$yv?XPJmO`Bj{nj$rUkPR7W<5km%W#!tC6K3WQXzai}H?mc& zyENNbn=QtEPh_B=h3_ZU=pblUeMX)+_SK@R;CswPen05Jd6xlUg)4KxHuuq)&AXD=v zza8R1oQM(ffj%hDJ0ovr_<1EE-ST>GlQIUQTf$=dZJM`pVjuFB!8?XXgRgaTC zDx3Ff5qO@k+$oawGWlx5nP5GQcqkas#Cx@SaVD0_P7k2sf8t&*D zd5!SK5muub_dV@Y#F?ghjy=gn>w9d0v6N2SRZ)l=ndsN3(M?2-NMYZmcCCx`Eh`X} zG1PzaBEHNe=uSL#4_+O{`w_q@Z_s)m5tcnxssMwbSPBd1xR=-XgcJ7*uXQvgMGM6vs8l)f^`B$H`;F z&)EDL2DsB`!Ss$eM&JHzi>X`Q`iKM}+v9bM54bg#S+X2i0u56M35enBqGhHT*0n$P(#<3LErzQ{LOt;Z8q zX)8WSvRGzf@Jc~NZzy`RQ2FT#0v;dZ{Ji2jlF@&eT8G9D+&iNTBup;Y>WB_=W~wz1 z=JR&Gy(ijeaL+sm5U5UW98vXscQR{RGkaR7iP2vu=5jsQu56f-L81ATRjTPy8--D} z;}&-WP?llHFQE(M+7v#$u2gwprz4IqPtSya3mHv5jGmH?3q(Ip#w64OU2iHWfog9n z&2wL@JadCRU+Zq(zcRSNK<^0|^*#ZK#-FNZ%xRcYD@Y4HI0S*(Kg6 zY1H)k%>ncYN*LXiK8am6#05$2%A|1&pH9A*;1hRGn5`#SUd9io2w-+wSwWM>IU*7v z+uYtl@ks~*yFPijJ1}V1 zAW+MAllFv7?&DH^-BTA#?9L3N#x`3{+Vo%I)c&I+g^!&qu_m~Pez#Bg%gCsz(PezO zy!eg9F5yQ86HH)nltOVGv2?Fs%-j<@Hp@2m_k}h`gXhhM2Z|$9+ib;sF3)^^b&$v& zEsDC}>aSx>XCHpX8hN|C>)zmxf62e&9=r=?C%kA7`gX2j2P= zmTMfU&0c&vK%lp!mF2JG9%uT4(&GyQfS$AXy#v)!C}70bNQYy?bgLV|0Db2w_Tn;>;$yXk6Q)#+n}9G zq|S@B5D1PSH~l_eZq!0ZZ$UlPO_k<)B zR=SB_=5QS=t{+1#@0x&ou&$9%$DtTdak_dE@E(jRB#U{3I8ngC0^e^VWk#8gzN0wy zuB}E&HKbdZ>bm+$gq*1OvRdLb#%%ZgS2O=^#@8MFYc`LlIm@cEFXmoRAlp*+B40z% z7kxLn=cLG8SfQOsXn!6*(Qij3IMzkVu`-8`graYLE;z3;!{7KBWSEe4?}UCk!9Gmj z;ChNMrsp80UkZw{Su`LaEC0QzB@)KiIxc)Yzg)s0;4xrKpT zQR%<~y7T)~+$2s>H|SndT+>MCgw@FkUsYHfXm~DIhXx`TWtP=Vygh!0X2M0Bn%GL( z{lS3?3B7gQXC3y|SqU*WbzRJh8wW3vSqetRs-keq{J`aP{5AU7-3uX|{ z<1yY19c3bi56Tak{q8I+9Nibb^>TtmNHG-0Ul{=XHIkqnRY%u)DC=AE-OANnhW5ey zedFZ|o(l5sMPDQPQ=z?rOL6ZS%3L5(Y|-Ksv){-Ni5-;gf!n^|cXv7iMG-9ub#??9 zMDnDGyT8Zni-W;3bu10xeMxEG+=mGXoqp-4l}am^kYjJx^!-2PF|PxXaL34IWkTW1 z91#2cvTbBx2%!rLz6l$AP!^6wXnb@y$p4pnj|*BQQkBLJUgbJE@T6Zd|W>}fBeGWbjw}S3Y=Gw0X?TI6~T8zz> zjTr19&@i%W$B!M~=PR)lE)^blg!jb4>8a;|;0p({bq*-8E3;$7>j%%wI`GEo!^n{G zPW$OWS|(MeTmF5DXM-2nY@UzY9EMrEstheB8e~d#i9HniCv0&jms-;VC9w4JlCH6r zB;}aQS>w5ml9zYFC8lG}7 z)8E+SY!3g6`0U|JYqQX<#X7RAAtWy*Sl!U_VWS|Z6V-An zecF0J1+M2A&EEfW#lnKdZvSR{Yicxs{uN9`_`x9KNyD%~J={mkv9_Z>eFhH*^Bbm& zY0A_zTINsil9X9s8d+S-wqFN7%U^4k#*nQw zXh`r}wwW-n!oL&*{k)z9SG3^9zqO6mACu2IJYK+=dnN*+q)I23N7cuKY4xz_TF`&l zDaqdS`Dbm?w`a}nSq4j3g(;V2Uo126N@B?8MV84ROQf@nzG+N7U(xqm;4^_pn?%Kt ztzq3DLHE6}uRXr>n;bE$j2^GDr`SMiJ;RyTyMI~QZ-_nTY?ovFiBF@o==5&*ov}=b1LQ*@J^~?(Q^NbBm?meN<~EBzl|f zX3`vKY|BuTugc<$wpfLURo@?Flq)^uvJVy&Z~fZLUD?m47xtCicX*Z>c;%fGOj0ls zo996)Lol%4?WBi0RvK55jPg=79hTUF7@T>oN4qy3F!hi`_a41QUS>BvZ$wKtt$pTx; z9e-9J3c7oVkfHqGFpTCPd=gm}3qy57`q4eHOzpg#YW!P|r)q_Dk z4GA%~#{{v;@0n$0NY}aX&}i|JT36-HEHQaEzyc0Km2HsZbL#*ZUvbo$T*O0Sb_YvP zl}F@*@IWlx`sw?dAM+W{zVA>2lANJOjq*kk&&DHwKnAl$07dMWGO;5}3Ve2FZM9(=~Lcp-gB1JzPxe zuyEAcY*G@6#UgY@AB`v!b%TlmIj_3>N?{AFE#Ea0*DpP*y~(vP%)|P*octR{3vwg0 z+jnxwIDk)f)``p(yXQAwp{dw*@S{CbJl-Vl!X3N8a*-eC-S}@R=mlm3MLe!Qej_q%dgNJf>9Q< zXmn|7ncft4ae826QJ3^FYI2xDOa39;^vU_$IQ`rQ#MQ(aJ?%XEdkNL2JB`vA9 z132uzcn=LNrp=9`se_6F7)(iBrL1RqaC2K)y-!kQw=o8yP;XlFPtH;|94C#SFzWIP zx!vfsIjhDb?2aAQ`l@~in>Ct&w4Y|eX{GTV7`*C%D%SIM<)Km2&|Mn6ulA=sHRt+p$z(>vjQIyj=!J4HXQ#nhvL}Q zP5Xey>!lvsgmW;}R8pE&|E6ArrUI%WE0>CUj6lI+FLXaojvuPY6W0c%q$a2PnEh51 zJbm3i@3n?IpcoWhx)3Y+?={b6(ls*NG%F~s6B3IV?atC(eWz;y6RXJSg5UkeG4KrT zsL8o$v7rMVv09O&0xUmR|ypbbS z_To?*t|=b>rnHbLun(VhEGaqF!ejEr*EB88zltqQ%2#{QA`TBuRKT@`o4egn`t0Y{ zJD)NC-F3qLmx5kz3A1kzZaj<(3+MOCHJrKQcm4$KQh+?F&`EX1d1K4C74SYRF)zw= z#LCR|T#Alsvm?tFy0pGd*3$0hfnINMRH}JaKas;`KV-gej7; z@69z1*C78rD`dyW;`t)HI!LT2Rjusqi1}EvuLIyqCWH^Y9HA;K8HVkAy)jGR2!}-q zw)6Cd_nECekWKoU*g4>}#Q~rRdM0?$Ug|Myfw*lew%wjGcV8gbExhZSzSF*`ZO8GMS9_cozdoX^CtSk#JlQge;_*NY;YFE80d4)Q8_>IP#n=`tb#y_gdFt$V$0wth{DyO-( zm`|cy8&~}##S-q{#s%kDh&(%v8yj3?r&=vpA!EWcw+pq%HuuZAJ@4C4Ag27 z1#*n$=8L&~kVqR$XV!gUw8EMbOk}m7v;;=GV+s?v%;1EB`j=@?{`%t6c`eBVoq>E| z3GsSWIl}d_h7|dkF=^mP3tAmYpK4PqGfVd93w47Kfz9DYVdxJ+{SesQbXir?J9u1l zNl_r6NM6`r*xJm{TbMS#>!kgBDc8`;AmwLf=G-3QRj)I4me660vQ1ncdQTOD=P=fO zntS8N!>a2F*=d16Hrr_D!TKT&;*+uHt9Kp_GOMz4jaKg{Xaz8BMfJk$i}6zgZ#n`I zaIR{0hd(T2V>3d!?Cjk#g(;YF<6P-KFT(A(GL@u+=B#G_9d;u}xsu@e#xI;(T<`^- zU76~x$Z1zgUY3XP=om#R34bmlJ_W;~91dXVJ8paKps35CjuY-{YYSdi@bgmOO#d9m zwu=khv)Z1_ZW8OnNwD6lxlV}3CcYjhrZM=)RmV*9CG|XdouW+Dlx# zL5}hEuQ;U$2`ubDfl6w|I>OA^?a@Zdh_(-dsA$MD+L+d1#$UOjBWeamdG{cZ&uG(< zykK`1*fqn~3UpE&3=n^$>vCdk?Lo zW_#dybZb9I_pv>T%LG~XRj^Bv3tI9ZU zDSb4Z(B>5CKo_GX6gpgRA}GJ9{)r9wS4Q&M^iBX{p!>6iHjU_?g|GH~Xq&#Owrp9) zM|k;FIrHf!$4i@(k0;@^ z@JUbG8v9tT(*JN=K_In{+oxQ|Yrj`!Fpi~{4T{AZ>xD|Hq6rm=vYOqF)cmay42U*l zgVcG1_Utmnq@r+2TW6-U6FECI-vPS7=$cXP~>@-3grsU&}lZuZq1 zzkai_z6#rHyl!Xd-bPPwdNNpsw-4*hjQ*!Rm=p2KkIFNTsks$zz%~++*fz$vZrW#N z^dsa@ibuRW?4k=DPvVWp4s_3Vf5Q3uQ~sPP1%$`2v_^y5GId`##ts$YW?L64~ zw(XLOy9iEkI25~>dk;t-kgC{n{m1lfi218A2l_p{%t+1XqPnCVYPG1b_^lv}Q5v7;RBa|Pzl~1;ormnUQkuN0KSq+@GT%=X1d+**gq6{$|Lyy1AR~PoXxlNvLi>|YoqQTW618Rc4s%`9F)HuYjSnYD*G@UymbVdbE#LqB$%A^-y}zK2P5!~)+PKRHkl(U^{-maYM z4O$W#V%G+>bOao6I7h_@3us<;fgog_ zEY}-Ai|9}pDVbeKnTEzsjj1+#Jj>HPmabLbmC_3TZn5)5GV#yvw8&9sQ;SkK3Kk({ z|IIm%=64|K`NG|&s(yG1DphFbG|!Qw<34o0b+~RKo1@rEf*03nV!3KK=FGpDzt8^P z+Ful0zy`*akx|wQ&pqM~(hr&MI9qeH+);NiSUpbFbQ#1s2dbi5t202Y`-QG6?(X-d zICt>?-jSm2lqXJgS%Q!4HJXJL%-JJ01Y>G)ykxLfe{{a@9>IS-Yb`%)ijK;%Q`Uk z(hutjkOD)UfFQrG+N0^hQ87`9uJ zJqW#?7f_2!mY!dnUJ7Y^fm@r~@Cx|2p9S8J{uiAl!r;33VeMcda#4>=Ufg zd#FCF>4@I9-*m(%t|m`KbdTAHOtd>DgF{KJal5$z~A>9MlCWIFn+| z8-*9wt<~*(sc;Pq>+;_XHSW07Pp&r=tSB zC3V%3Rz5~Jf_Vzp*S0oH@39@(!S`jqfS3xN(FE-T)TEFUqh2r43|Ods-979o2K`)Q z%5^}!jO$5jcF~{tUr=0Sqc44HKXBm1%!`?<5clet`lt&G`_yQ#e!^clzb&GkQI-GD z=W`GrW+V4R0aT_?(hbx+iLdzLQ_b}|KP`tG_+$&*Na*0y62tP)lm?YPPwy$?Ga%2% zQkq|**&=!mMvvaPV`Zm%qGGpn(kSiGQ=ijrY0iw|z6L&{Zp{zp9AB6?@z=Uv+6n&n zlMWx^nfN@JNfWi5C%R4sVa zwh{3meQ}qUQ+wfk+Wt01;4>_?E4CwZpt3tpN{A7kuZ4GkXjb{eF?|$ zm@0)N{TefA*0F3KAN_|hGI?D;LNj!CFA2zUpGaHJ#4GmXYj@hCOY?ygh(3C~z^|cK zS$NII>asISC^n`&(?4URkUa4<-ykY?Qa!xkZEUyaI;IidBX(dZMeN=TF% zp$g97=kU|lUYCkQVX}=jc*VZE<`1?`-pY5?2eEOCvY9wt?b8s|fOZ6N>M%3L(CYVQ&_AW^HuW zK@t|rG?1#PJ#Y|!q5P-_p?$ez%jn16vvJi|+U8hFp$?YJ#gM{rQu$X|2g*!MT@McP#cQ0^i^Z?>#^W_`v zlQ?#aeCCu$v85S4mUQalPW>>=MiXL}ut(^ady27k z{t_fy-YE;Yy}M+z6X$b#q|<+Rdy*)|KsT#1$jpZ7Yg}wjMFh`^&0?L zFuX(|EGRklwbQ=wMjZ-t)L%@!quqqiXLU&z z0PAZn?0@nu*4bMS4BJr1eE1I~-lOW(xsSNw7u~1(+{!$?Wl{V_ur}4uL&&lxQrMBYtWzj-4T}sC zW^kTW)42PZGvK*<{N#CKL+~T~vZ8n;lgBpNd_P`V!kSHPZF(ck13DQ0fc$0f&+0 zolH>m_aiQ?^zTV0Nv?aH!vNo#Gr3Gos$^O*IsDkbEFb}Fn1NsdFaM%Ry9fDH)X1fE zyvQly1ltoiQkk0@uz(K*V_zjImI-VY8a2KjSuIsOpFS;^DD=PSiR%L0$FL4~NxE;@ z>_?Z@7ncTc=59I}hz(>27cCcx2K&_P#-z-1R6->GwCr2sNBN(% zA4508)S~WBQ?KLvDGyuP%j)_}?Z<^~{uyXqQ#{Ap*8&bMhUz27p=3(Hp4K_6c}_DF zktQsW=pWr2#kZ1Uk>yG*t++q8u&=MK3DKiYxE~?E$7&QEBpi&9jKMKB%v(+j!D<9jXeuU1=jk zpoxM<(gXBA2|H>Iq}956z8(j35e>eQ3YALJ_ovy7{<=ij?sY)8^^yiUAs3QIg8`_a zlJ1e;1dDob;HRa2!>kv^-D_GlWe7ylm82#}?(Q=*}KA%WSa11D&v*?TVUSC!=`A z0P$DhExvBkI)HHqtwL+vz?)YKI^ynnxLP(y=p6fvFhBk4>beKS$6Jdt!H>|F;Z0?3 z)96aFCKnaSWo&>pr^$lRQw}ziiUk_ZwEDPb7sz~(IpETG=OR|I+N9}VHg=@tTwQ(9 z{396|)xA7=!W>Exsp@J-)JCf>+UpM;s38|nmY=unKT6GIBBKu%gLXn7Z>PzdzjYpk ze=>5@q`Wqm4`o}8yot45!9*4e)fU}jm*_HaL)pX4r?G7=+<<#Wz@zF}eHW-3!(-Meh^$$;0CB)?aY}MhzfUD4fR*Dk8CkF zBGgs#a!)nu=LH8si?6hG55dRng1Z;-P~R${1x)sHX^@3!O?A>|7n?CbXO9Fq`+A6{ zXbxntOi4;pvSvSg;;8Ond@!E%?8h<>|6j!9-z?yeJb;abZ@EYvK29&L0$4%^4ImOVRVH#+i@092^($4{*ob%G-=tLrlT2Y6iN zoj16`q@U`&!H<>tQf-wY!R?mTuhQJ3a9=w5C*8@po{+M!vgY-N-Z{FAf_q`}smvu) zx0S}A(1b|2V>0zv%n?ddXqB=~o8*7S_FZ1s`UfY!rXwumhxFXTf97z4Dil%OLBeP+ zSWpKJKG{?W=lfkSfqpbBGsbCLPGZ0O;fl#0YiQ1X#}kq1x(6-Gpt`Xul|{c|dR5>I z^|@@NYG2`61&Wr49|AS0&c~4%;63UqP%+^&>qWn0XH z6Z!D*<3+%n7v}9xrn56l_RNsL*OYH^RMKy((v&2~iB1*p52z_}-s1pPTd1M9x%f&_ zXt$(F3Bvp?S69ke-)w>B>Es{Fi8?jX?cMCD17&&uQsy<-1Q#O%srowAUgrM{d$bnH z@b;q1?(83m4$*v4K7$Ps^ni8^01ubA=7_Kyrf`3TyUnSZzxUVxe}kXDZI$s8fbg_& zO4Ft6%g>LVn(HEEs$S(PLdtgtl}|lj&tv5ItTl3iR-6)M2N~tW-l6a13X1QY0$zK( zANEX98i`$8_zyLQzY0@Mq$GBvF}_;_O7{6QPMK9AT_9~S$x(Wk~Lv=aATxl;7|FmKzx#ma)|) z^kvWt&jNIyYfmS2kH&Xs!6WmU>yDEh3o7S=s7{L%BP|K603<~3MxwXDQacY4?8$|6 z=7VNb0!L~FvJs6w?g=GoBR1|F$m7t>ga)YyR($@Io)o?KG{>#SUbvY)djt8#yKwON z(#}DsnCY?#sbGvpVdmjTRUgzqqkxRttlu-S{$b8$VTYPZ&x`ndK$s#4fj>1i8pSf2 zEWimJ>0mvdblJ3;ZA(10cCO3gsM)!O=9*2e+!kAi?Pgtr1J**}KcA|@D5VBFyeTpQ zK0?JSQ3%8<4k|KIub7S9hA+-|5C-f%iVBn(vH!ICFa<$1W1xsS;JQS;%n2}r5 zsB=`0JJ%QbC?BKW=E|1c?L}-`!^Cd)$eFw$;wBi~qGAr4y<-4!P=ghZle{Yr@tV6Q zbfauk38ta?`vHY?6>(kB?MeB(cB#NEs7E=@|+b z8B!CrFAq`$yf0OWC|#_ML+OZwBn3S>16w#>H21wL{$$)Eff)7B`K!s4Pi z&*J+`;f(JY?yl9Bb^90QhWZ@5oo-}o-&l?~f{MS%Y?o`DTf*RZ0< z89mCN>F?|sWZpX?LGg9b)##P2qi{^Aa`VEO6@QzEgnED4`o}BIN_G&qIC2}U zfDd>XN=!3 zV}U&$No*%wAgNWHQE4cqVxV3tcnpDql8XX%L2|SqoVgF=_M5_USneDoiK4-7pE?!# z3+Cv;vTWRAR?}Mk!;*`eh2lB?bQ-(N@&HcU7@{wgO^9RYtEaA|-_O~NU9d(48wDMB z(0+7X-H0_g6`Ssk?JNNGn@*T@Fxj|u_NWMLkwqY=U?_Sj<1G0Azy4qF0ip7qFyC{| zw14dLZ$=qJ-1u&N|KMB|Fab0U4&oGz@`pZ0bWQgjQcnm5V;;<@l>UDv|E*g!VU>+J z+9ZEM$$s$YDmRLg@Xi(j+6-uJDGl7QjuCXKIz-n~X`9ESWZBRb6C@*XLm<1*Ukpf; zBIV#O1!?(1+z}E+7t_|12_|w;l)BQt>c4Mica>cn7zE?|t$1#|u8nLFm5?cDDowtO zz2tSMSxs&CCF~WtTG+sR{>8_cd($-)q|W`uS}%;V;|+q6#E{L!k<*+JS&n7u-|Oa8 z$CC!mMQX-2+2mfnvE-0ernD_*&Ad5hrX(t0TsA-#qAtZS_#D z+Ip)S6YV;ik&vyHGcMOF7)&S1C?kH9t;Uamk0SD9>2B`lvH0t+Y&WP8yWI><1XT@Q zMQ~#S|Urb>t9L}QGm&gww-(!U$jSsRBexS z8B>vi2ZeH&k^#1n+rGIpB_aC%t78sdUNh4W;&a{+P84iyx|OcStwWF!Q%I@Tytx>U z)OsHkwgEf`FR)EyLJTfP*!ufV&DX#3iJ!K~gSHMi;SPU*=h6MS)G;plHobrm{{u7o z=Pu%}&Q*$+H}2SfdkkTlY&-Fo)l+I0tNxRLoSy@8yF&OSL}nDe*sSITQ95WwpQ@Ec z&zKcMrQ;Q5qN+eqZ2b*D+K>cf4vfUxF2fkSX~|#bO22oSQD%A(kNfEU@p-&d^VS=ktbd-yU(jObM zC1sMcXx@=(8>tefAkFDNN+ol>5b?)VT;;l_x%3%NoHx{9VLInO10U81qRky%Q!mPr zt$U@9RtPEVJ_bko1tX6~bpY%@G0Rrvk+VG%d@00`j?}|qZO^0h!$wmqx$XSnKTF(z z)1J=?QfqX{qQXf&=|O`GWY9@F?WG&`QLDmy@BmL+(|Z!RdFje8z@dl$n$GsY!GmNE z9>&Xkd9kw4Aqx=;i&T+=eJaXcC$kGqbXAapUs}wP?ztqw$+RWzdMZ3Jy5=zG~na7 z;WODBm1E2qLmyHLAk`D^=*A+-fZ{Qq6<;`u?_@53Xk; zyu5^;M{}H)9xYU2g~_&LHL|K}vG_GaGx?Xa@99W%JrG_!^{)^Zp+et!(p~oRl;@&f z#pv5qVO%e`q%XfYLUnfuXc&4^cDLL2jYn*Ns~hoi?focIeRiOdARXm!i|~u>Jmd=p ziLXDUMvkxVtZs-v1?ujVV$S063BGzU{oO+0>x-96KJJt`4MO)P2%=t&HIM34Z%JI` z)5YA5a9RCmwYk>8G@`N5805ArawK<;aS zI3Hvr*OmUSs+AP^&pIePkaRRC>nCq%t&rn767sRlIN4x~+SM+h0tdLYKTpn8L3dM# z>?XHi2V)z^(D zKSwYXc-YX*K4m^;XA1<9e0Xk|xr_ zT7ms~p^qka;FsjURC_yU@Vj}%nH`DlR65i92bmnTcSzrB2F63I2Bh63tkSbh#*J6j z7i1#8lhbclRmR;pw&E*(G^z|X@ZT+&Yj0@}V!b-#r*r(q z*^;_wh(~#$$Hv+pSYH~;{ACNn{ z*k8VM1ykNA?{3*5<}-D{+=rs%bZvR0J^>;)SiEgb?|t(A_#@{*n#*Rq5qa8+ZvsAj z^puTa9wfOsq4M9ItxG`}^zs|h1r1RG~iEX7@j$7Ya^nYj$=Sd&P(eGy8QgS1ugy+cKF@RKea2rtOUDJeV zDbCZvP1H$lP^A0Pe*K*cCLS^#`w%}NK_=Zu_Ae&#({%g!pBJNAB zuM2@dje_a5&2@tq$=I^sLz%v-mOIXYZ|#888rj9#`Vqf0=N;A|c6p?#w3 zPjQB@#a7|qeYTEmI|)Ea&v1FjXfdNZdJ->BJp4ha8oOc2%PdU1gks26#mg zi9a1k>2VdW!^35cIh7K!3T6TEzNh%6N5Cp4gN&*oN)WWifWq6{)&pBzp9)! zuj7xn{SwqkJ;te1adA11>3zl;e7}t4-H^A{B9uyLnyW8x^a2g?yAks>{u~_9uzhHp zjBxCA;kI3Hwa*GyRKd}^;(C?+{6AU5?^jl~@{b%5YmJQDvs}4HAy~lzxu~4)3rek? zsEhmzeu)dhP8&!sBc}l#JaM}Z6*?>!c5kixo$kbX5M6JXG~|Eh1@{m>3B1G>sGqY6qDKrJVFxM6C6(+lBzaSwW|Dh(sDiY5pB&T*>jfE7ckgnO4t0$ zy~WBVcQNrC9%iDwzQwj|EoBYBi;rk^xe!vzRhK5mx@8duvrP(EZ#8!KvyfXUZ+Jb6 zNLvlQ`R7aW8F(^@T)vSg@-2T3bd}w4BzS;3m6sD(hyi-r4_ejBnWX5F4 zqLEwU|`l;RJgWc!`2dE%_$Lu@H|Etvjy#bjWXYbwhxmNZ6F=PzMa z&}H9x#)I7E{V0&2lzDs16tF?Ach5>&mmhi2CLap>-q}|66Kz^d=O(UK-d38uQfTxR zrbU^b9d&1%fMa$T{@g7`uCw9_=<=q0JE<&eZAJ*f2JE6gfg&51vdKT3;vxs#=f>-W z#ZTvx?4;TPu?Iv?Bx1Fk_?ofGdYe#>B;v7o5L zRW4U^@7;jL?ztcRi$Q_zhGXM0k&`=^u~b{F%S({SsTDVl!V#8&mfFf3n+HE#yExGQv;&ZyE;_ao1l! zD1y8K2s)2FYSP;kjAafAB7Qky#IFG7RsZ4x*n=WdR+<>~7U>qQxGii$J}MltL|u$o z+?aLN>iofS)J-IFxz>`sl4AVh;fc^e58Qqp_hm*pe7FR;4bz4R3l)Z`9BIrstZp+6Gji!;}j zG#&(tTc)c-uAOt|wv+YaBlm|(;p6Hic3NLcyR?WK2x?tfOkV{H#F`Lm-w&sJPLe?e z|K^$Dk=9o@N(m3%v3eyCfciQLl0F{S+qo<4@N0nkmgb@5=ULc+=XtAe=tOph9<=uR zzu2(vBGo=+NWMF()|Ww02dkX+hBsJF>KI8);X6h#2YZ9Q?-jUApITQy_IE4OmKl?oNI;Z16f(S@syY>Bu{pgv?C!abd z5VOVY4p081`fqX#hauWXqVLVYYtVX?rXhcv=Yc|LXIA)x8O4f9!UK{D7;0XK_{5vg zRs~!O-(OTx2Gr=RFotWJ(d{^eB)p2P%QAkKO$FAqH2UB65=Eyikt-t?u6W7qnP!tj ziSIspJ;1~E9x5q@SM`0=U!c3uYrdEn@S_iCyPgC z0lOXz3a{n=;O&{cpZ9k3gxeXLU6GNx)1U1lKhXv|I|0vR*Az6^cL*9i3MJavY;rk+ zixeU9Xs!%-hz|JngT>?brsJN^hBF*0R}gKhhslvAerOM zmxB6b%A|g_G^9?l{?KH%Ka;BcJkLb6ugi0JmPtIDOiU^8O30pNG2#BbNUpJ|xv`r{ zwPc=TM=A0;tX)V<@FynAhtCs(nJpE(Z`o=F;+WIk-$SnhW+|~P=dpX?e4VQVI zB@Tighh;KpF|4A&_6|o)nM}9w`DwWKQ8DfH^X*QI(4~A`fHY*UNG)q^iH(}g2USrC zN>Gd_#9_J}zMA0AB3OnNF$m(?e7#e-MgY8rm-(CWH+ynkH-XAJj1zvg1 z>k80{-BjGY1aB;&Q+AH~tr^9fe^N?C!}{aHWg~yjUHQ{2;{O57Kr+9;z?xysWyj3C zH`8j?A4_~EPME~K$FAQSDN}0%0SkQWsmT;`g3eEYvo);wpdJ_O?hg5FV~pvJzZag~ z9Lvmm#aZmO*04njX(nNfU6{Am_*&TW7&k1vh%k4|QCw5lbkEhOv}U;dN`r-Y?CspL z)*kL3i%ogv-0eO?0CgHk@o)!ij34v=Kk(U2vtM9r^_iL18d;qyiaLyFyS!Hx+#acE za83}hn%5+8h#SXXOeootmauIa6K-@Fe0`T4eCvJ!QPo6_nCk7F^G50uP3Ipwa zO#pta0F|;An8Mlf4mAZa-o3Yg*_^YuE)LD#FvrsE9H~)w4mrlYHk5hKGw3RhIft`W zz|kJjsDExDygAb#ZQ27uVg~5@x@R6^+}m6(C1|Fz_tJV;bFqE|A)T=E(0Cie+k56D z{+{B(;vA{^VQ!iGm6QHuMTGI1(xyZo`K~c{@c+eSyqyDjX4*SvYHYo_JSOV$UCYB| zja}Zqf>hO;d;c)P#LyUVmT)Jug}kPoSDbU2cZ@N>wMJ&;9)qhb=76EQCyM-v$v|h$ zrHYhS=3|}9IDSry;4$|6!I>M`a3oR6vu51znM-tSE-cKnXENVggUx6KeDOMy24fma z(tKw9#1VHM?#p9hFi*X`y4IfDSc3;*4%NF7w*XvDlXz@xY5E>Ua7o{5Ut)yW*0vV< z+MFvCb0yu|5~gO(NbCPRvlQPOOUz*`;l@A4utpDSjx8nZnzWPn*NnjojU~yk*N%Okdyh3vFt-z@2J_2t?S9T-sp5JIK$c#gkgn6!Zk-`8$#CO-&q4Gjy=P-cdGoD>x?;V z(2=$#Qs!7GwsOYV%{m5p>ey3vA#7#dw5HZi38}hcZfLJDr^NLMk%?;O*sV3jKt;=o zseDdRq_UQL!Jd)najm7v6cZHWS>rV*rBR=jP4R_sWYzIw-CIs$r+us$w~AsNaSU}V zkoY+kSV_)v6e|pfix79N)f+qdH79MA6a$dV8S^W2hrpwrCoNBO^b!Q&wiZa%iZiZrXUWC05|&Mwq1t4xeAJs0x>R2Kr+dwr^giaIbeXBCX-$dO zzGkA^TS_xafVo3E_vyCt9Y3w#e zT*TSqHYKbP{y0+(``xodZs%#nKBrv$oAWa@MwOH~_j+YYI|gB-F^#oixb|2vIUx<5 z&XINy^xrdrH3vMAm{xqtPOGRb?1jfLmYDuslSgEQEatQmxMY|UG;_@PlDM;8-x>p7 zV&&|Np2PU<8cVTb=UDX>=WOg;D^Y9=C6~A+a9kdlU?IjC!WFVgLWqmOFU>XOG=uzk z4Ph65Z_&87Mkc|G)9Gg~6^EHt`0yU1#&-=c?-Pb9^4<|eaE~FvwkGh)%6V@kC4s%Q zcRbl1VfZrSG=OTae2q4iO3cjZ%y=*5#=a6F+FsEpB5fGazQSzpUMWCtj+DW-w~Wgf zgF1CBEP)d~5qP`y1CJ&rX|1b+6EyI0Hm&Njpp<4o#|(=iY0I`|mf-#Kkh_g3{mHt}^Tp^0~%r ze2&|{WR0Ell@r!~NWnEAtc~bDLiWu|L1K2Kg}$&SoXXqz`hF};%DVS7+uE}SCGHT_ zA4aar5BZl7x7K_b*?D!vWWX~AO2mw5>?Fi}+O~$4!CH%|V1&)5I)d8V-|JCzj%DJo z^3Gz)voUf7Y38%X2G|KJ4<&DG{}i_j*V?PmD=m1yG#Ad!og1DmY(b6@$6m@?xs_+H z1eO#7M9vs9kYQ}yr?ls2`Ptj%Dhk(O=o+ zXHHazvDeDTS|i6`8T%J$Zn*U_7idqJqZMRtHJUi5hH425gm~t(-L^JHY}>Q`KdmL+ zJ*LXk-m51yCxP-a2Xw@mJNIa&h09DAcLFE!klrnF{R3kP7wxTL$YeEZp>CUI}M$GqoE z_F6mVW^GBk6{kd6oM}ld4%O0?7BtUVF)U;yLDD-i*x{M;`gjH$pr7L^=uWfaIpq{- z&Rw&y!vfc^&;0br2X=Cvc}#6bU9pmq+Vq@D?R!M*iWAl#;fxVkFD=;pA5)mn55ZG- zj{U6|H^fNKE5v6<*}S;s&U1=0taMJ?mAS@L*x1`OcJ4@;ydv7qiz{^`#l7y7gSOck z>%wrZF}b>y?0*hR5pt|?v$aPA_eo>)eg)C#7^2!uoI7+OX0hWi1);%|7eG{7F(_nZ z6@(Mkoc>v>=yW7~h&Y#c%gXyGdTbf9yLVpMJ4fN+yQVhk%WEJkOgXi)XGqGMLx4D} zgpIR8Ue}rPxM7C)uRKSXaV*vnE!~8%uU&?5(mLw-{~N3u}MP3E#bE3h+p? zCvXjU?zM7I)(*QmVr*ocJoifLnQLKfhM~bZmlRwI6P!7ONT;zAJnh?Aqb3Y>uRlj# z-^*L0I}cITKjM1N8x!$(%ovh7MhM-TQ=w%AnX$EJq~IG98D^(3jj>b0atcd6Xzc-u z7iO%@nfqoiMU1>3M>NFRdm?8B)UY-3DBqsZlr^t##y(ck&l@XCD$lv4us5Dpo4Yx8 zZv@ITr#$sd1CwQrHLEgL9KfHMrecl+mbiB+%3X7Pb1wy#97iVRUt26WZ$YB5vH{qi zhGyTGGkkgt*_fFZ{7K(YP&8n==kEs4}Ot$)ma{k5P>jceun~1OG6yhIY%zqz)^yXb#QWwWb>69g7EP%?Z;S zGD^#u(FbcpUEDSHg6fMCnQsiW^E0A|(Akp~WW?C)Ge#WGo7oC?tZeO+lb&x{Ge2>L z2%)?Z0iCrbM8gP!m|$+L=r#vBYuu|qEX19%vUV~;ni0NmM+M(H*6vnV^Qn9eJG|89-;$Yqr7k-U3%NKT(UCiNCap#G! zv6r~^n~4!P%&C_?)x;7+5`ZLi#yKG&+q8QZ^R z#FXH)W=h*iDV%o9ecUrPD%{VJOmePaxjM##+JlP72-{J9PqfmxqMqWMgV1dZB&;6!zNv3gbR+tg-Mk=BoGK1A=$X2-%d!i0#Q6bZYJJoE5?<+*pg_d&dQ)G=_A` zjuEA3PFVOhh9Y&}OGZDYL4!56dP|+@g?UYBk-Vnv_Sgr&NlcLvZSMh%8;1T;4RP%# zP7szg_9lNxdt6{-Rq-{~+TsjrVK;{GvXG~~Qke^ydCYN;m{;If8T)i^&S{am1Hi?f zQ4?}bfrz-*YVV0-crGqcwmA2^*~&9eCohEi6$Yy491FT>r%C5C7n;P$S@ANZVT(0} z&U;ufPiI9CmmlZ0K^k!tHcuJiH~04ApF_eh#YLnZgQE9O15|3xaGskoI`mt~3oor< z&>x2q^BRLdBF5aynwI3+NK0RTg^abi7VvmY;gKaoDEqNDMAFzxvTNlPq&0UM$sU{0 zd@Wg?HMiK(m~(J5B#p1K#+=(4Q&fHp$<4GvYSrDjBzMM%xVPp!-iWat)-wc2c!}GD^q)B*@>LzTEN-UZfH)_r@vQv=-&BzEX>{BIg)P5SbO4Zte~hH zR>;kna}F#|Y`!zb49uO8@OaN0x3(sJ;MjS3V$Qsb5Z8!r&e1C=E~(ExW~k1cOSUHj zN$MD*bpOW@hczRF(ogxs8`g^0&ha*3EdiFB_o!N!^Vm4;?eLJM3gKR(wmHpV)-Z>Z z*qSlCXG}HCI`T^Q+{sCBWjXY;cI;wU(GhO1)w33}jP}cs`)=>~|DG{HaZO>NGi_sJ zeZ)M;wx-tk&2uGinZDvdkM{Kz%L6&^RZ~)J$U*d5lo0x%Pa+ zh{J|wkI}-MV)|a1QHOht0iqt07+G9Hq;q7^%sew3!&sAbFpuHhwny}Li-`p*uZ7T@ z=LYPK)6{Ry5ZFH_2zZ;zRdDY-qB?et+KXeAc28xRvD4c7NF$kl;|$dZBNSnV2=S5j z!um_=tTYC3D00YB<~6g>Xx{T2bxmljyf&z1 z+reRNME$fe@|s4Cc{@HV{gIzmjQAWQCUvec+#BSpJ=a9do`F|xPl@)l*DPM#K3I<Fa z9-B^ZMtS8u_Qd&^yFnw5b>+V&l*yZk?QjQV>6b?I^4WR1eJ{zgwg;@kiQBkvt*L^V z^T5s-8{~0Kc%{BKmff2hICG7h_nhOd@ZP%vF2>pEKKI7o+Ic`E=18a&v)Jled5s`# z3E>fQs`wlsF>=lMi@Ik{^c}ehEl*kDv|?uPSg}Ma=b^hj6VC2V;k0b+Jii_Wyx81B zlx|F|wU~2o@tASFIS;+|J{N@liSbQm?D_1FS4v2WW8OdK;7Jb=s&vm~>zDQrz}dsn zc5Hcyuy^p!49jOX=R~SH2cqAaA}}h# zMzZsn$#iHfVTBuZWK`Wjh;y#Ig|@fQ&9gBo7ozB?6J}^_bAAUgC%U{ zWcwNBdc#P=$0M&Qo=dtR2Fc|*huq&Bks50)n3%EVifG-_ z@+;<@@;7GcTF7g^F-FAl9Cswbnq#merI^gT!^Dy!OS>D(Ax|NxNYnZ=Q(%Y z_g;yvZw}3uy&}BFUi)@*r<|3x_vm?_>)$bN;hVXZknl=NYk6%bi?a3@#*6VgVZ<$m zn8plp-#Z?54`K8j5;D|V<6$M`w9+tka@L<4Jur^_nV$9}>7L7ABJO?c62soW9rMgm zU0a4}CT*@dryS!P0#li*9B$5q$Can_@EJ>lVGl_6Gy+^~pJN zHUPxEH!^HpF^e*$mGd=+qQ)JoXmRcxg1>gI%!~tDJWu@D9+6>hrA*(nW2VpBTgfl) zXrdnDn9~^V_PM8SLkzitF-&!m7;;GNnWJQItf|?tw*26j+jM>fMfkC@boos~3S)19raxyQ z+!?!7GsVHczQd$$S?k1q#6_ty!V1yNyF_CQ!IQERGJjn=CuHU9%f7QX=uHD{BTmhw zJfg;Xni+*TkFoPMr>0)d3L=7RKXB1a|vof&Xx&3W6gXBtsSYb|Jo zX_63URO}9mreo#+*_85(KO=TqU0XeV<*}lY=7RIjkHE3b;WR5pC@Y~;;s(OvMmK>63 z(jFt-X-*xwv7-7(TEnYx?sf5;1j?RvsPG$G=xRkp(mA&X?%Y9Ba*R2jH+Ogz2L$5V zX;pQEM3FY;8qwZ+G%95H^AM)E@QSlXd#uQ*y_W84o`HBaXF-}Uc6v!2({W>tee|&> zI!fC6;AuuF$h-n#^xj+bWzT7*p8b)3CuC*RxYJh3OB*0$ZE=u0rX*8pqGZ(%d&+N)^|Lf*boyRnQDtlu{Ty=Y^PYfFjG>>7%6yr?0D-o^G@Ll<98*_QIQm;AbQ+0JtMEZ-#IhNLtV*xeC{m7 zKPR$HpIcCQu7%b=rgUqXGodQ);r6^nwr?A${~cCT!C7m~G|XMbI}w&Px4wHA+m3Yy zQII&7g6$jIR(8w`>^4KPQC&GSc@O!sy%$XM4VfEhk8!}gw;IDqqj5ba#QGO<7U>=< zK`YG2`<=G`b!IJab%g) zzCvVcnYraKE+Cb&W9Z%-%Pws1fsCB@O#j7T%b7-!;9mRNZ4CX-JaU+MUt<(1ZXvLf z7k=&7^SvSN$&xu{xY-^7?=4KU#k?|(ab9EDU{7VgzT&KS-gAa{wEya<(wv$lu znA-<&291h7qWEc=qv(1jnTQe>Ud`B(G<1#pp*hB6_!#?LZKiFUFgLK-oZB@ggfQT` zCg{lxv!`_@5VSsL?%-O>O=HGssW)O!ZOj8|cF%Q;zSrc?om(|;F6p7O=LXqL@mFyz zNuai}IN%!_ohwdEm^}s*bBb|qBgXK^xbjTLTnXADW@XX1)VjJu4DFk1p>k#T>NAp% zKUdK1PQxN)Pd$scf-dviTaRdkxa<(NW?WBGGiS}k;}SNq&mIGJY%f*Lw?nSo8Ci65 z&H$pB*7DJd`Ry_9p_`W$_{N+0Oe9V%^&fL+`Cqd)APAXPbWI(VAGe}#Nf{+FuHc-u zmx@bYn}l(XC8rX$RN$MV5B_QDGXW8xT6ayWv*eZoFllx zh$*CU44td8)9POu={zS6$?HDm{MS!&FL(tdhP0AK!`j2lca24zIO9xQ8hgB9$GGRP z2aG%~DU`Fe#8pX~5^N2~!ML_I@tZ4OV6MshHTKwM+Y96;uN3{aCcb5z+wg5An5`4W zddOT64SkP|*gSV$(4SFnZw&#PITv)z-BXKXug%?*;+Vpoxv(t_x!AWx_S)N_zk02m z$vHN1eO*h)dyUERHL`r$9~+E#F0G(GW)N<_Qbv+k-;=H;<;K6Vt(Ua{?ehe7zTcS_|7gE?=FVWPLzT64-vn<_-i zz!%r#@mtxdV(q=@J%%hni=mTmOreh%*D_~W(UWISA;mt&UfIh7g>0q#w6dqX#hdHZ zVQk62G)JmHG&X?cA2H}FWGwi!<|N>oF@tEYVEI3WcHEgEtT9b(+_bZt=$^|DGjCz^ z7nhL7%0IQWGUf``+3TES&8?HR0)F7w0t|c413G*y4a&LJa8L*bykZ=y5F;-fvbhK5 z+09uaa&IyBFoq!O-f3$pOd;Tt0;uQO8*yb$-KsEhAl95af@BZK{1wKG@(!E)ajdD3 z5D#?6xfiU;T62YOjgf+ohX!(tdweLYNwPA={Co~W!aGmhqPqsHdE6`GWDW_Zy4NVh z$qROHrPb@Z_b_xyi#%YYp`6jHMCDt%s80jhCg!!%9CJ48#$dpU!{=}gw8<4>(9a$tReDFA?Y&37 zP1%unZcYu~xpNxv9ixLgCZy0Xqd?_c6H$APX^AoxGVGtzRCZ3iowsvt;SF1@Z!b;8 zI^yclADhc}%)s$JQ)=r@+3IO7z1h1H!0I|KJh8M^Xw)46nR2XM&ATT){EwUMK5VG% zKGG8L8QHvYPrTDLg2u~;dxbB}n4`LuOwykL@@VOBLIP~@ASIFba ziHB(Hpwl$A=d@c3AF;t7>!vB(=2z9P1 z_Php$LfHdNXbnw`H@9B-9TVm>EREv0Qg+hZf$}!3DZVkryx$LdEOzYVxwj^g-3dFR zeD5&6pR=yV+*>4aFTvV0qX6uS)0TYZjQAFpu8Llea@Y?ee=X%%)wOb#>78>WIgEwaz48+LT=C9(?UA#% zh7#^iIgxb^Ii|KIbo|XDzI11e)IGC?;MnO}Y)x&dFs5+CS__0=??te__t4LqyPs^1 zA@MRd==O-Av|$Ea_!yQNeH=@fe2t9iwHI(q+2iOu?s=6R@?b?=)0SXQefU2EFtgP!0;`TvFYj!a0MSv}?oxV9oV)NZG5`E42 zuph<nGsLEMESbJx+pg4DW+S-fKCd|03k;b6YSi43$Y!T4AlZeudv%`31 zDY2ZFmf}x)Z*Av|&z?ep?jE5Xa_r^a6Gy(!ox#s%4(R`ylTU1Ip^_XIy5XNIC~JnD zkr@NB%-Ng1U>PfGaPASxJhpNZN6tV#0$^328y9=-#oaUZ;Qt@N**kB=t3CGm+S~C6 zFD`|czNhT!TzN@#&wQyA(pcC^Yc6n(>E^q~j6gGDa(hk-K5wm@g%zh}#aXF;Vr3DR zm{X4b&W+7DCS===)j1Z#l&!?W4Wdgw#*VLOBN=qL7lX?VqF7W-WxvzDFW(-IIkWt%;1e6JAOjgXDe= zO~$`RY}lRyhh*(-=dxFP-J9c>Y=t?EJ2xKkNGX42P9>flw@`5zgC2H{VWl^goWR@r zhAL4srVsrk>DPDbIRm;p`fQV)>d&eReJI$&$l(;T}<*HpS`q zJom=vjq|L2uQk*gm#%IdQ%P;^LGwS7vf7!!COZz5j5VUr`&)BWd}I-u7x%bk3X#kq z&dK37A|_WE3uPwn)r&pm{CVD^aD7g#y`01Bb{wIHctu5-w^JtV412pNZF!luw)S-y z^A0pkD5EyVSp8f<&MQr+nzr`D)ZK#>G6#6ZyY}WnoGCVbPQ>ySQuNUnL)K`mw5gl2 zSo%nLNi77>s=1dK!I(p!c1ET1wu2;J+0*!RB+-GIx6a~C)2e3V3FR}QyjfTig=>ev z`MxKh=MPJbYv-A^GZ*m7&numI&z*^q0|ep;OBQV{#Q7I;hC>PQqxt8X^9ve(?1vq8*XWh?c}nfJm(pyw_xP4y1C}wTH1rebtZ+$9ztkB zPWdGvPKm=j6X1CoYhFK26_yi6lF^A zm=i5(4N1c`Cg$v);iqB**o+&8%;{b;Pa%wL%^s(~|68H2W#>`38b>Jin*%UuOqrq@ z0tUgEGYe=<1^=0YJ#>!|`4soe<(zX!D}~{azwy~IWqpLOr-=jD>GHngl-pd>AvA30 zvNLyBaLtpFF|G~sIyXS$O<6v0j9u5WHg;AHuZT*%9n=4W;Zl zqC#1o8?1W`k++vu_)Q+s14)a|Hw+x@k>a9ATj@nLMd9%_#%Mttxw$p0jikHg-pbhP zM>B;T+C9ep{jqZPu2LRLTskEngcO%PPZ|^EXzW$)5<`ymT%(drBXllI4d^+8X2Kui zLqEpVues7-=pVsHat}b)m4;~ch`Xs{F3t2hw;pOtBUfXM;mSDkutzhUBN0vj40R&~P?W;Fq@J`-a$2*R|&c8P3+MCOdXJ>Jv zwMKeS-P>C@%o*^wrXojPV>5XT!S|bkYHAvzqb%>m-o17#-cR}eHa2?a9;0l3=YZQU zt%;eKhsbpr%b_yIiOf2)O2pbjC~bxb)gMEQ1Sha&C^3p;j(Ou@aDGI?0}?{LH|{J0X7$(~92XGAFBvXiLx z+|wa*z}1*LwLY)Zo*2h2=Lk6katHLFx}#KS%W+k5j6Jiq zH-7lvfyQD^VbqaEBw3t6C1xy5j=7=%y}mZu=3cu7Jx0*WwIf{NkE4b$ZMFB9W-jN= zQ+#XZnSnSb=2FRPJ#xo%mO2M`+M7dHWknV6oPy|Y*{fK22PO2C_pa($6WeV~S+o?R z0{dL6ICy0pf|R%3Y0i=PAdboHxTdtx8XFNlkIk1hraWX1t8Hk-9sDa% z6z~p194btW!LF_8fq*#v$p0`>mH*+ zZH*XbWrRLfRK-ZMIy~&uwG-#mN(>XaE6)IhIJZLHOB;VT4kVR4lcv;9 zF%xaN>CY)=8{VvlU3w}Lk1-Af;Bt~Jj(M>hHj14$uI`Lr`q zDpt!&;cyQ1^1T-5O-fnke5NG7J!aIz+{12fZI$wvrpVkLYYHk4>7)?icyO4LG-u7B z^u1ERyFy@ZWa65K8t+^2m3A#Xh_|+INE+)EJq-2jkutQ(i&?>E4jHdE z7U1hzW94iNg}}U{!gyX|P-Si)?mD-c@eW&hAf!d>79yBv98o>bMa`YE8t-2#Z6su+n>ZIx`W`d1cV_{%w4+pb zT-jlKjctvS2L#=j+mC4t)$6sg6v^3Hnq}v#{IG`{=$hLLdySdXp7UVY9FgmFu06q* zrlR=XQ+9AJz4y>~X#T=O2^m0VFL|=I;x@pWbK`8LwYj*4PTxzL zWphsP$`(=(?hLUVE+!O=zt(1b%HtDe?Q!KY*C1LLqmCv{S-3Q&id>#6OmWO%5ny?MeX?ekH)x zz1NP~-opib?B&He=G54W%O)}>9r>GgSpSb}WOQu3=C#A-#T$upH4e?-9Y?yvPeJv0 zZ5@{zgG3Wa0oUyJoM!dkCc1q35tJH8N zedLg`V#13X5pb>b?K)yW=ZS+}b|f*B8gfAU*t1V-EFIw*HnMEmJ56E@^@6tcaDLhQ zD0{{I$}-jr@EDsLJ|#K4%ZT@G^(e z)>vDOWz58ml~V4@8~ashPo>0@W){#$D+MVnZLu>_EHqLm-rYlsU~llapVQdNU4yzI zPldQPWa_&cJN{}f#L=Ax4F4I!y<`m?{W)i_W*xh5Bd2wqnxio0Oyip}PbAwOQhM## zQDtav9Nst=!u3v5C@$|9jGH##a9?q`cWn`>yF>JA-uu*lB$3t{)AGZanFuE5Scj1( z7Q-I1muC;zh%r;tXWDz0ZS1L*G$+#Nn#tKVj$M%!7PNWTOOtwyWQILf22acp5N#)Q zsg&o+^55C%V$8M3v-9M5nM0>!juHPK7c%gho7r>^48Aev1W=l@AUBOw{W=EFP8?%_ zam^vVGFLW6U6Y?I3<<6shJw!^3=OKVFj5x72_b$WUqwhE* zIJ}xScHE9z-8v2A&bP)=`Iy_kb*@0{zUEL+92swH&AGa^)8_8qqepyabh;fET6A6$ zi+t`e=exFi>&dgjJ3h|AzIdIJQ>pfGR_Zhj9WFJ<}!(aC{JFD+P{xW>BPohvA31VM#45v2Mff!`E_Did z17J;s_?IVER0t9LeyvT_vFAQ>8KZA8?I5?Gr=)fnIb~^YP1}(6G{BpI;&lyZ{=C-u z+e*ttA%(cNG6o9PnL$H-#MQ<)w+{+G%=G)Q$JF_siHss81<@AMR_UFKUM}vToEBF` z|6aq#e+)T-JL9PKk7;~!(}DJnVmH9>cHCrfp1UQg0$Cc_Q=7hF0Db=G9pUhnmg7i zF0JZ5BkJAQvxFtA9pgQgD0C0;#Un=~mM|l@%vpo&E8CYc{og}fIgG6BGy-n?T+2Ch zgkaw=W+Q5UrUBqS5<=9-!RB~n9jYB>2KOEt*>eYt${2B}I%cMGSm|G5@2I|)hG=FR zqmXg#tjDlMg3MgeAvMIw&$*pp&@MakekhB9Yai>+rwmANpMSp8qQ zRCTT`q%`tu#2!1hE(g8;GqYCSoD&@?E&bCsmlob0xdki^*?=-;fz!F>y3m=)Z9b-u z@ixb*`Wy@7c;^9~6LS1|3Y*6>ZXo!y!z^|hQ9*X^#oWI3+RPkL_iV3S;1y<0XkN1< ze=Q-8yLV_v-Xn)5jh)S!#=`nsNd$5ZEH;nu?GpDa=1I{LWbS>0JhpDi2^pSfEH$LQ zW^!bSV_|RY(Wf4U5>Z-n;D4+Q{hh*0+M8RvWv@y3KbH2vn7gKRr!Cr^XVmeGTUmLe zne{dzj(V9f1vD)YxV$&w-WfywYVKjrnxhQon^DMcM)8ESveacyv%M>g0pq;)iqgr` zF?P)%p1;OYOCHPQJxtiV#(iPDx=n$KKYJ!j3yt2IN$Y0T5(bnRu9 zv4RwAoLl5`?eUM9hosLP+aw{3)VYv1h{78o9&UyO)i!gq%vlkMehmf7n?^)Sid#-& zF7d;kb_~`|@k3xu3CzB8cJ57!=qWDQlDMK8;9TQOY9~447-oj)+be8n?@hRy*Urqw zP}m+5KR4`&&9o*aNX|*qI4()(6Jnm(n+yA7W`yrNX5#Hg%TpqTJ@t{M{$!bJ?|3H= z|C#foHzq{0y!U$7osk@7jdl01XAauT>kw`&^}aQxbafs9qi=74mNmwz?w@P-CQtRF zmNqud9x=)=t+n7d*T6{|A@hB0*`%E25Y@?15opY%`?^+oNDf0&b}uQp9~RWhTB)>t zPeq-#ws_tOIb3+mU6(c!uy{=Y&VR*8rM43=&kS2zYD_@;pH?bZ8XUfOg zV^3kt<=&Hb2yTtD$#PFTiL&OV%3fQoDefrGn5Oo|Ov_Cp`=Yl3Aa-Y3!X4X7c}!sI z8RyFPm@%euZP}!~2Q0!K5uGEArJs?;1l`}0voB2fiaSR7+u8ehVT}x=7BklNi=!1I zubHGaC(zMK!-{fc`N}cFg3{Tu<~2k?;g{yf&I=nkKWv!46SSv0798y!;~g|FQMR|n z=Ifptb$*V0#x&*>%o{18bS%}gGltA*4;zGQPCdK3m&n&!bBSyQ1=>DlRN9+cPICoa z$2%6wUV7!>zd8fF^odJwVJ2O*v<8G+3G1bEWUZ*XQ<%ZavA%MyE&ex0=*S$qL~5*A zud$a9;2BFieUA0>yk`v4A7gW4ZVBp_2QJ=Nv&3~RmDRdK`c~Ql_+qVb+P&BG>4{?| za}KfjnRY;72zyR>Zw1~n;;znI%Y1p|kIj1 z-XnrA>}A`SW~$3b8zN>d9juw7Skhc^#Cq?6>^$}|%-<8nbY?}FwuWxg8#z=Uj}@sf zCc0KmbNMSwrSGzK!rIwk=3^$jnYTB#>>3JG4AYu8ZCUFu_OQ~3JHtOO6_hkWa^P6Y ztUqoMyq$J#JOUu}-qRm+j|G`GhM@ik^9?#h1&6tl2+?2RO)sygiakRx!ds)=YiwZm zHdi!NS#eK1PhI*jSEl3NP}qxU1AWhp!MG-(d5Uw`bMFw*I-@%N+5@~brzxF27m&}I z*&=BKu!24Ywnx}$P;Exkzm_*PVv2)(eQm|QAM?Fw4Mr>W28_#L(MeCE((8x^Fc4AE2(vfBe zVcb(_aqRG?pEpwCo(Wfcr<~6bcPML&VbpHy1Gi<*pzO3}DqC91S0=21ti0xc-CjEz zHP5lRy3>Bn%FAtX=8%@P5+utTGpll&KyYjUP%FfNj5#J4dfd5McgM}RoEJ3r*n=c- zZ79YZW~SU73w3@kXq^#u^jru*ZEq~R-yWjOTi+AKC?{~&G6phB9NA@RjL4f1R;J;K zNyTC3o!hnc4&&H?1~{+0(iQ^zQHenfFHawpR1K_?~$;bw*tz|qc>tMEx|cQ zNX=ferE=#v#=q7&`<-#?ab!H)IC25fu%__n%o_)6rP!aewsPm5Gbv*3^!G6*xXm9~ z)_*QowmkQ8)mk~deUHHXy?6Tb8yTQ%L}<=7_k_sXgAsD=CH^%=Yq`TFe=T*}nggE08j=4s2IkS;N!%hXp}jK}ntF-*NOo=|j1!ht?oE4yaPK*uGvXH1 z9D5%xPPLsdL)Ly@dn;z_e8RiazUPjEwP!8St+%2;cbjW4eQoW=w1K_w)_8(hKX$>)rxyPR33cEIEZ#nw4Lzc~2!}DMm5yDA}J8?~kra1R>YThei zVa*V(J;TDvibJPngk<-&BOFs(V<+FP7~CTk~|%aOwx;TmJcC`^&dyp~Ab z+w-eGjO~=a2ax*@u?%wrjDj0tQ2bd_9%=1~j5l^>c%4hWZI7w3m(nU%T?40K#wnq> zhN9od)0{poy{5UwwDg|4UVcOngt%t-+Z+?SVP!4inAF7&bBu8hEaTkk)3N8S^cjXZH7=t*H99Ek9^?c0??KH+s4mzd8 z?>@pp_lt4#DI~0kwV~0lRzA#*JLhP{(A^kTLU@_$u6iTXFt4Dzy<#ru-^&npP1&(K zXa09Z)$2WTy!i>ic4jTftGP4sXAR@Lag3d!k~UUBNbA}o&N1CGW`6V?+a_%e@i35lL_ z7|KjZ!78n>nYHKm`CAJBUff%+E@yGC8za2I+@qI$tx=V}#$3Z030J%U$zi4F* zO}98FwoTeYsBCU9sK0Waafp%iWUOp~wr1Xa9hsbQuQ|`0cerc`V`e-|U5qs&4%p4x zq<6%P_`PG6)S9!UWlj~wv-h;ZNfX^~3`E#8Ml$_eYtMN{;r6~ZR^wiy0(y)b{ykS- z;fpw~9rGn{i_-QHvMXgwa~7`D0Ge$2f8p$DFBCF08qxx%;sWIzxdV z?J)8_wg~lJ$w*_R_0qH##$}m7NOp}Rjg?ox?ieFJeJ(ZB5(k=7n(NPfZ@sUZQ~b&U z4&si%)@zM_bw_nfq4~FyZq1x25@3ul>NA4=?@qBid2HG7v%*Z)j&Z(iCe4+V77)PM zIjeUk9mpIqQxqU=Ad5Vv)Z2@3xnZV>^gSe>HPTGe$kTLZukrc0wg&&teYv?u`d&># zzJ3kp{x!EEQX3onIt+1}v!-y%S-YoU2h7DY#@^q~Yg=NDH2XQX_`%&H198tFg`5YZ zM+k|Wd!>caJcc$=OMBa955?I!CK&dc`!j6J<^7%JDB;OFU_V9ZzC2^hQXUa6V$HqT zH-nz^8&QRNhEUWr!>I6|5r8A_(2^22W^G&1Hf@f1&pQ{A=*)3nZ_G8EyN5DtoC{wm zOdX3q7KnBl`HN^yG^VnH?!{QMD={w7|C#GBdW7ZtIkGI~nv?o!<&2BC#t8l!s}Fjw z39hjx#MqA0u`Tamo;Fub_Q}C;-hI1O3Ex_3Zt-Kp+nx@WdyoV(<2<^c1T_TEUJ zDfEBL6@i~)G=H1PUMS7ogOPI5=NXffV+Dl%G{>Gy8&gmyZluM#mPF9sJ5G8J6uB@{ z?#CEWn`2F3-!pbrL7J;6Xl-D|n%0JT*rR?ZrS*igHxS0#tI&Th5u!2Y9(IevkRt6- zjI~D+OC6IkXRjRko|kCIAECQ5Z^9*qfq?wtPL_nF-JbTUn<~I@=<&shVXbLGwZw1)mf4T}?Q zMIfjcS3GK)1MX#HWZ^dhX4IdPBRB0Op|=(mV@eZVeunw|A10dJ9FtpntW49AV#eWH zlUgXQ?SVfxT41ZX+MA0;!QOYs`GRd1es3zz6ytn7V)*SI# zYVLIBw^vNU+sk5e4861zbL3V@+jl4CM9i@^TKUW)T0PBF=Dhc;%?$DRa!+y9xCa>I zSjlr{u7L2n);3m&TP$;KFyT1oO8;M>zHBDMksIcw){KLCYAp%-yb|i%3`xr_M)>iY z2N2g>8wO>rb(lLdHt9=|`g@F&$(z?)U7I7BC8T+_vE%Abi&GL~U!w#h%>mLk z_I$)%W4mC*#ibKB((#-F?=cTexDj%|N|<}4b&p}}JO@DSOG)2}1!Tmk8s}Gr~UXC4?D<+}zx240FtpijbFH)Cf7_b52m>KBuD8UxPw8 z4LHcSR{Y@@bC_cX9n-KPf_BXld@OHZ_%hd=dCiN8V9j*=Iffu*4Lg@-#~H|R+^V*Q^>p)D+o zj5|k;URdkzd#r7X8lz;&9wX;x#6^W2R=&#I5%6-%ZS}tg-rk&34{@gs)Z&||d@Juc*7 zwoL5{D{>&NsO&aFjMbZy=4FO}m^+ec*VvK%HLbj;yBC7a8w1`rk1dnG!lvWjlh!(q zvAH_eM8=(q!D;Q}ian>CSRPRsEY5MQGslcYOv4N`FV(^s*O+U|D^e^CjG_?dZuQyQ z3w{l4^RwpMeV=0tIj?}&8@Ars9V7gEr(K|$#{lq{6KZy@J>-+Z;{0CA_--v||Czzs zdTxoM8AE=@9@{2F5p-%S zvEsYa4$hq;8EIzJnZKfpd)o7RG4GZ9y2c{#m{B8lr>M@G^L+7)!!2)yM7X$ztmO>z z{$Xq}rxMma`(4u&CuGI&F}EC38B>5e&n?&=^OJ8TH6iBN;*jRP`5T*yZ7jGZ?19TY zM^40^AwYes(9}O-(&(9KF@Hx{%{E4M$(xA~Y-DWUIu{cDj@g5MPu+o=Mi|1+EAC=s zV4J;G9{ZU2xIWEcudxPnLB|-&ogs~2&Y<)?cWmTZLz``dG|MptBL50uXg=h;w-sjy z%$dWiXeYgc5OY{g32~2dZS}36R?u7=`w}+C{LL84mv%%E;4x!-)y&)eYwW4^l7e{I z*y}JijPc?hhHCuY0dqSKi0v|lWcy3wXKgGc!ZXvVb{oUDbO-t0y0hfe-#JHWO+d0W z0^mX08&Pb`CDJ?-4%*&ZW;2CdH0{~wx|dw(9t%!BM6t}UhdSoZV|Y2_h^xB8O6HC; zH7&+9{FSl@(27BeYo&nxGKTuY%cB)-=XI+)=4{R%5!rJ@naUV*T=NfED$*^(c6tx-$%yNq}#A|Lj-WFzP z-d&3CDI(>huAJvGUW%JSbIrks zI_4(WS}}ui%>mTDb0%CL6Nq`P>9`P!8pws1!Y;ylx~=Z<6YWzMMNze9rOPb2JM zEakixMg;gs1>ZN~Tw9!LuP|?Ykh<6I)l1{)CS{1lvl712OM4D2XT7g5cKTTfJ0EFI z1@b<2W@8vJ{C^K+#Ik2XS6^A^EN@}NvF2vtjB|Bjt_g%XbIj};!u%q zU!6m`XC^7ay0R$6&RcGNtO<*=XAb{b^NMmNjM5yJg!38u`7K8^w!Q+m=8u__Ht&hj zy@%*<+RK?DrS-iuVnSXXF$#Q+z>c}rib5O9U}}v?);!kq-$-*wa;(MHyC&-Ynt85t zhPCE7hL-Bt6JT!U>F2s9G)35{_Hzt%iIS%Vc8_bWY!5J3s)*id`Jj{vEoWs2Q-K!mDt(mWof`C}rd+l!wrO2~K zDC-^hs351PhdT$d$%|t`Ax89=H`nUI-{WL2?-}hlVt(156Z&GzEP@dd+<6+K3>rI? zI?WZ|wN{+zT49%K&WNIvC+6#3v)+4WobtOCfZ9&W<#erKoEEa`%p3dNc1{S;G-r5W zo%z>zZ!wG(!{T!flWu>F05-${2xg6`%n>&J?hX?IWNk6;FrtW3+S{ah23Yd52a?^L zY5rtx4e~L&qA5$iN2C2}Q5)Qx>ldgLYgPLh?_41LX z?A{)^s&9?}x*TUnejQn9F9aygl;?{5OltymuYHpdBWQgeW05{Ck&v=xhRxoaKzIf1 zr#EwOX$pf2dTwOcIEHB8*|WnTjFrcqw!Z&cYrA7)efY3aMD?G0n=*}k{gJnZX^A;m zJ!Q$pG_&T`UU5r!j`5f?r^aYrgS{}$F}%E&DAsXW@N~?%&Y5QH;tXS9HEtl!JqAkR8I$&TCDg*X1Mo*) z6GUb&IjtABAXu3pDQD*?>lY(f^507+FpL2ZaBo!1GG~VDODilU4$Y*qvKGbIftP-d zt>+#0TId=%$1RS)tq@0K;9098BQ0sWJl3XkNlVQz50Q_)R~B`fJ5zkG>9#wDlKC6! zF)h#0*th41ve7I{GPpsuJGRbe|hX6(zmvL`dy1yBt>!4Iy18RT6-mI@95_pVmeXX^W9?& zT$(mATxSk5TX4)7hCPRH#8^>2Bdlz|H{z!A8M7>Sr-ko6_Vi0wgLGkyCBvTAUfW&6 zVreGzt2%e?-O59(YA?;N7PeqlSTSU5My2?YR|@UPYb13J5tg;bFn>z>C}&OK%$VaY zU&x8meGeUyJywuf%3BtHt`WQ%0?6svGgv2%Na{QjveMf#y?<@UyD&ns&>8DWVDAl? z8nf&}+{>V9%;D=hq83m}3tc4*QQ$PgP}kU@D05BWkTFTFeXo>(GY`} z)lC7zcyCz#o5l#^PuX2>j4a=q$8`CbL&|-{WcVG1@b61gWOl9X?YYMS!P`43B939# zKKH6mnvwH$Zz+wxcB1`_<1B7$ARO2^$}Eq)D&m$D*C&WZDVAZQh6 zER2^n2dMVg8@px&fxMDdozFw&9rrL-tk#=j416wStuZ#3OIm}2FmF`D5JD{cUr9`4j-T=C-A7UV=zygbAdCiIodum>cE`y(?8}u#$BUkeJs)XIfke~nmbcD zEj6YUN07*kLq|DJotZZh&fiN*@NDb>%WCXZy}GmZ%oro@V`iYc6-Ug#%!|Y)Z+Xi$ zwy4a@0iG`}vC_HIWbxgzykkx^|Bp-9YiFqQy%Pk)TN~73h9&QtSJ3I&<6d~IW#qY| zlFLfbAYxDWjvXdAZAnQ2eJ3IGm&Y1Rp9}nGEs67(LV7`3F{6F2DZetd;M-2I%VCFY z;GD_pX>2IBLWDWq#8K*bhz*h8GFVs1|H!(_8t~H|;(B zIP${5+!<|dCe^YXmQ4AWlg43fg{Zn`aN{0pb72L|*Aw=@{8_6EH0)Hez88wiTPuie zq!64lf+}SHqzO0fQTDOpqWjzHRcbHPq&RbqYa8SAcF#%Dw{v>v4O3P!2Jo^s$JBaS zDd2T(goU|McHl_a3@r?4y|Twp`%MEdD&)1zx3{#(i)$An4h;RYW=wfs*_3bu0~I+h z9MqSRcF5m@hH8dQ`@RFtHCSR#w_G3w%ZW!#_sw{F^x{Gta=e zGFRO5-kXSL@5$E__ORrMGqN;Kyv!Xo;QNS!a4iSvi_M=341sv z&*-V00+8lgfrK-SINUTxiv9{ozf&#R<%_cB08$ z>#aG3fcH4Z?q3M`MtZHh$vd|6*PC-;GcL@CHkWeI-FskTZ9w5Xg4lG;W5hOXA-1x% zitL$zj&^T3#x@rG*B;UQAttG=n+Dinj;ldwq|E=EJD@TR0r5GLa{3xGG;6OkhO-9( z`kPUPeIz`$m2!aa52L+x58;fP0@CZ-DU^2S$dDNKmR;GC5+N-R?Y+LbMkerfWp3}bka{-o_0imgSt0T^Ui6AGX_NDnbJH|nCUEh zW}PyGSmHM~jOkd@Mr{qCxScmf*PWXHa7*yPna|<(zITkuiwp33?LFYW2V~mWffaQw zVX`--r0m)2_fNs3eUJIIxQ4Xv*@L`mPO02EXSBi$b5JjCK$gFjdR*S~@MVQ?&JjkO z@Z4LxV$5-e81`(&$a_3t?5)STw+z~tGbU|s9oieF-h2qFVmi(A)t07+O$ZSwH{^_( zx#Dn8N^=u84M@X2b`H~?iSA_#4f?%DXnvks*>H_Lq?|{3`yTt+FYV-yxsv{4UGWQV zhI>_XoAtYd6B-ks+N=9!x>X~b#kx`Xg(-Gc{qZs5tcgEIHe z;Snk?;r=j&#$pL;hhY z+=-EG4uPk**5vTXv#@al;jlY`zW&VV&}Jvyhp|TjO2+0&>$%w)xsa*FHRxi@qtvB|ZDcJNF4t!^b5wm$-lN8U3LEib(EkrImR zO{w~Bj#;!C2Flf%%XVZ2=$kzzmSrDPrDLa5i!}BG)tD=#c?{_37j_2wlgxD6DQR)8 zz~?_FXu#Pkk|~5B*RVI(+nLMGeUDAgvzJKu8nO9k=lJ291EPD1Gw~|TXqhz9zS)nV z!+nhLs3Z)o5i(@V$cr;{Mfjo_78zcBivzNW@uFm2eG(*|4Uvf6Q6+zT>W39%BP6ZeW+4M&!)PtDJF!QKUWR zWb@2<*L`QC&a)E){@lCgb|fT_F_+x>U6Uku>}1xy#@ycWLwhbSy^vtEK%Y!x28-TdqiR`6_z_T2E<_R@y8sN zs>GNhTOzM~nibP@TnhpHBW-c^wxS%>2`fWr&MBjp=BV=@=};>Mk+y#|(4oMY&IEX0_fDHn#Wbw&&Aj%~=$N~VeC-9pv2qm8Otbnq4CU~& zw=n$(8&Pj14B9+4*wbG_Id6rWgRmy7Qcv3-B5p*kG|i;NYgng4B6(9CY0S>qp@sE9N;%b*kGKYa%L{no|A{N zTU~HO zE#--}JGM5-nPZ4CFTK#eLSSdzJ4|oP9jOr~VriHw4SA+ui4w+k(3!*gE$*q}o!5NM zT??3LZrGMN)?RHG>AZN2#q1GBmgGuVk~GZyli^reH&FQ*;NJhv9=4Xdwe?mQO?-H5~NX~fXUAE!)0+v7SlXBqi97DDEl6A2_n#QG4&Q0QFiKPZK*_Lift zM~##CC=S8tHs^HSivzo8hU|N6h_AgSVCWq4Bw$TMg1aMB^BUu`e9kS}mUq&@9z(Wp z#7Om@2hRK6k;FK!*sU=K2G0or32jHr>pd1E${P7yXGe|Mn#cZsuPu-_R#I3`E7B@V zNt-q!-qu+A=6y}gpg5=Y?~X}GYVSOfGecrc2=o4N=Sj3MG75EAJIOmP3H3S?(B~O5 zF*gSp^Rn}b@J&HbYejj}lJ>gpUBmu!4sEBlHg46;ThC_CtlSs3+S!}SOnL?Qr3};5_8ahsR`5l+eURf&}aO@S$7B(t+8A}3sZWOaLmdxj# z`*kh_$Y z_*)v8$bF>2(m8`z(AeXjY!Be-8iz)F*t7b0?*z>;){Na+18HneQS&+jtkw)!RCUI| z(=?)J=*e?ye(hz0p7N$fi$QpB?m_yUhcsizyS*!??f-~-X?YD5+Bf!A;a;JrI47~k zwYFr#*r8uwuN}F$)6DFip;vH3K*Kmj67b8*@H)q!|DAiWB<}H$KL*U-SR;pUuBq7^ zN0#SYQ4xAfamzolpmojjuriKOiZ`c7>Y8x_DXl!{G-BrdTMLeU?&9mjyl8)1WPr4X$>hCWzNb0u=j4ZIk}BJc|P z9%;_`zL&?&>fEauZ*KvPHrMX>30o{A%^2Q22h!%rJC!hO<*2>Gs7PAtRDZ_2lAH%R z^T=t)V$3z5x+kXD46E#P4H3bUH*!-LV8Z1gYRYSeUz9NaK{WYiejy7ijo87*IY|%cO*>WzZ2~79s{;) zZ-wn0^MGOv3lKG}dB?E!JZlaccQ#HHtTi+8(AYbbE(ch;pVDwgotrLTuVnW>7B2o- zBkXBSm7g`H7}k$l1Sd?WqqoN-Y)h-`dBy#k82`mOkT;nUG?w4fv9V+)3GE6n0MW z*qCO7`Aw^%e8)VKHI}e(%KPAXh0Ve__6)|~;Zt;HA=)-3e96x{GbYaIf;ICBO<3`? zF|J&#H$ph|Tlrys#({w`_CnpyDHkPg5UxG8%zPeem2}Nf;+;Zj@z~)NZDv)Hw?bCK zjnjE|j&$>shU(^+v4n2~_~ z)7HzAP{#^!AacXuJT5@Go7eKt2vNCx&oq#>5~5}rt3EcTdCNBfKFS;cjXe!KpcTh* zLY-sCer>42l{ZZ6&x=}ZuHomm!kB4IlTS9UahyGufXiF6GH9*Dw-@KsL7UqeZOs7i znX{hLnM+AMWXYhHN6vIxdkSMM-21gt6v$n>$YI`M zt*Owx7C!&XIGQrH>U|m8+-xmTp*XilJz_1O<1n&N_Z?|$E{-YJwI@JQj+0d`gt4zYV|LQn3;1~imBt&Q7R-zT zM0{-x`Zo8fRLN5%G3{NNwHNB}-Kp_!P3i5QqZ~0pw%(sHh%O18)WLZHKZLXZoIwLN`+v|jGFZIVaRv^uZ za{%;STfc2a<+nArntF(F!)NC7g*6xQLz-JvH_QWOG^`ws8-`xWTr(J88Hukb#4U|6 zLx|WLnXWG^;lD8=AYh%-6=;o#j1!ltOxnYLb>|S9JV$)b9mCrwP92{)7Xb0fxs!G+ zRp+qsOy*vrgfg$Ou(YNQatm8sGevQVzGet&S_@i#=IFz^P%iVKKjXyx-nrOpM}erh zS2E~H`w$G-s&np9$hb#t$eh!sVaLJIHq#8;9_xZ(W$EQM_mFhhBcW*Rq2IWZ=En(> zl5dT9=|0v(_wXodCC zFw)xmpOZCo?G4Hv_qNf`)3jpdG@zK*oL7!p`91BBsFD(D!V571cu!E;JNK^Ji(85{ zObyb#W?tKz+m~r?rR=h1uIZRt6?{Zdu)KG|{LCw_XD7k0KNfQC8QXp&ZV|?oR=~sB zd*^*D#l*7{RN-6`r)rF`k{U8HnG-~DF1Y!%MhsF;nUFaS z4Y{-Cmhm3TE^$p+zB(h?=vqS`dTp$)vbKg>4`aJ64k@;hSE$9>Ga_*8^tQTk$kN}r zsVjzY%NetL{M&PzVDDkCw9@$Z8!PZ)1Td;Ka~S5D(HS?!1epe1|_asCJ!UlYcKAj6KJ)dYuuUbj4x4Iff9! z4r9DPX-!GWzE))DPBFP*@A;OQ zwuaE&iE}t6`G+*u#N%FpN;gi0ySrzW&)+GRGR|?Iu+v)gnnAaIkJXr%S0+(P3-fL4 z+1xWX%IQeiUMHqetvB{|by>kWe60EQpSN=59hpOFZ|Sj&d@H6Tq5@3>N|=3MKdAly}->q_ykVT|be9RlL-O(SAFPM!0-*2wMKp%ZFu z@xL@Cl*$<6Ix-IR!!qY+a#{PgBh3+^5Qfb2-(wm$Y(=>n2XbLtvz~NrAn-dUh=0r( zraP7p*BbK%I>dFjHUhvs|O6U1Y$`NSB9s#Dn;T60by%@Ma!*AD|je6MY%I3mJ!iF0FV z4y~;jQmX4-nU#5L{f!sq8pYoUZg$7L{Ti0=?awp#XvP775teetUo+r1>~YDvCN6r; z(|K{E)!mYI9_(7vS#vCH;Wam`$4PNXI1CKbl_nlujT63LZ?Uwu21sXLOLu6^L6te8 zp8iP7#dgk_#J1v6^UDKUdG7@ImlkaM9CJv14CRv<1ENO=`5<&8@c$UYmeHC6<9E&J zhqDr{-J#aw_W`8%lJ~*{8BYdc{eDVl9k8u(-E!;hg)?Ztg7gnN}|I8aw)bjycmC z!-9L+``Kg(J2`Bi9b zgvYu@w9VUl{&eOI&9lQ~bsGTyY!6GIV$9LGnbT1EPg}<>FBt4IXX?}4gZwP)2>m|N zl6cF(u3(Mz%(ufFd0oSuVlBm^Gjy7k>T=x?tb&Y0&w za35oQX^+^#5ay6W47+n_&6U_RW1`o{sa8E_$%Qz`lfpj-IHD3(&T(0r2yN%xjg*!I zd)P~CbPfIYGWSE<6a4bRkxU&M@n)#n_&x!Cg1D@1hBX4n!+1oSs zyy%+4K5^z1hC3D<&zl?VZLK-Lk|Tu8&Z}c(g<$`WqXlJ7XxTd!X7!zGK_F!*xV08C z%UIEyAcSq(x3=hhOc|7DgyFN3H>yNgll40%(EUC`c*;z{A#ZQpoVAynS(*`Oe+?;@ zJcC?&TPeSFO}T~}b~eUX5!F6y?A9~0YGBR74s1-+#lNDE?2jAqa4bEAmDfgInlm4B zgh{PCcSP}B^T2|7v4!r5GlO}P>D>xyA z8K9nq*w~H%Vsiw*s~Xp8z{m%zdRsfpX-{pmGk0=wTGO{bhCu9`*GT2tku@=HH2;}d zdO9xM^tD!u+8GNFIfdbwF#?`lm@D>iZgHg-c7o$i1J-uzRqquyeCnIqjAA9hz&5AK zRt+mrbM8$4iaTX%4*C2wLZ;jpi-dgaMb|yEZ2ed>q;(`!(l|Hl_+BF}cWAv=$`y3HNa}MpmOvsy~*eK04rc&t`>v?R2 z)%!ByvS`gIuptal`MH)*`A_kmd2gkPnKoS8pGmtQ&oTBCcO2GRqX{C0LE=8vHe3$d zsw7YGiL)2{Xo?H?XHCtQvnF6+n1jzVh85ht5}r@lTf{a@gq9K(0mHgx9{AXc@@S>O z`?e-l*@){iVQwLx7bj9%&LOC1MtO)bw&LYWO9FI^5xlu0toNB&(S8jeoRji?(auQ* zVJrjEVQi_-vSOOsSm{1=jNsCpCUW^%kyLsv#r8T!NaY;)*fNgIr8+Vc;n}fMVeN5@ zoYE%MpZT$z!t&BwYyLGZU4=L|cfXzPC2Wkw9)}u1+6fOa zq`ktrhYVg%8^1Yi?WP`9e#P4fE_LsL!@IMXQQk3ib8jryGB&2xh+F1)j|s0m#@Ox| z6Bcp|ZN)Z&Fx6VwtvfA!#WdFXXOAnYcr783xRPA;T5GH=hZxGali2Xw+n;j@zj(|U zkUbNw?~Q|Gc8B%Dma^rrV=MY^|lsMc7UUucJ!=Canb_ndRx zY{xC9n|F?SoQtA!Nqch)`1P04#O>I7&LYi#?Krpe#vY> zLh|Yf>sBTX)vCS*n)w?0R6pjn|Id({ICGpu-N_wg4}h?jqbyEdSu!yVl;E+W_`_YX zA35xhijxDX|5;&lbB;;fycf#OTC;p)EYb0|XFTPexfE~gDeW`j(%MLSH+8HPhmk^V z%^!pEaF1Y-J|_Uh*n_$>4Ee~sGqU=~QM7T4RnIskdf6IFnRw1wX1`b*bl=+e+}HWK1by1nX7SrZ{_i}V{Gq@ z)3t7fotw4xXidw*UU_Dq$UM_3{R%;$ex#+bnbH886vo6%46AQ^ugJ!-cS`%sQ?Y)J z+4H?8dcfZEe`wD&&pK8LLfk_;unv&%IfAHC%wx=Uta-ILm-yLVbK+x+ zWdDrQ*l+~Z*F6J%V_T_{BupiqvA251PjOdf4k3oSS8`a|D~LG_(BnDw2xs5ho^Pe} z;2P$N!cX(&ZH(Erok#e`4Y3Pgq$Q%ale}6@S)O6dK$<-!&hCjb+kB7B;4 zdPgPpFt^ZUkL!O(y^vsX&mXFBF!1cGo$+2TuGL6Mf~%- z-YG&iWi0JIBGT&}W65KN1nrcjdiGdLKYq^f_c=0>cbL->WGrNZJlDp{NbBr)Ol|$R zhWhx-DVJyORKhp|g320eG9rwT&N#yeQ5^d|d+k-bHCOy>+T%QS?jVsDrV8|&k)Ci4 zjkvXk7Dn0Oyf==Og|*U(_=w4aZ*Ezlu?AApU9rV~uN0EAhK$cyu^>JyDg7A37C{UP znka2~l01h%)*qWmAkXcQn=;?sf4r=Pbz15r`xQAi%fgDCZu-4|Rt~th2V# z(OT1JbML8*xKnEG4jbYs&n1kqCjP?-p{jhYMZ7x4RK?9($80S9%Qk03{etfE$YW%9 zhcLRjS2EOE>qH_Z%-Ebq==q#T&Q41OVvm7_5Vxw;m`g)#gw&!vB2ez1JAi3zO|Y=` zFy`JQ#Gc)|zT-lm%j~vsycKFp_gB5*^Jf%10%)*_EhHtEh@wz6Y;#%|S zYGobqvgd;4nwxEV&t-|fM%w$y!A@Z;HRUm)+R=^kPI*na$~PmZWL*m!doD!nHAb*l zU9($xZ7{X5gZ?g$HN3MHpymm=Ut?`-zn|kc;M*a2V5gn1Fh+j!AG7#s&%xBYw%}7< zs}*r%LF7BvgwvTph$2qa$h|{)!`h4gJ9ebf-y1}4E`hGNwvJJp`6)S11k*5P{zlw$ zQethH+@2@Y^9uR1IR|L?w-(IW-}8-XZrzlWHt_M>K@%;mt@Rz|M*7&XlX)y*yS<09 z*I2PFJ47&|zZcB%pVP84OmUAshf>#A^Zz`y*xL#F%xs29`aVY5X4;XrWADM85hJkk z8#`Wkj&Y7K;@HYqTcc#Ix%e`J!fwmkb$n0oOdLVcAmkmBzP9S|9DCkr?VZiI2O{KMcbCXeQ=)B&p<{GU z_;s$>p%!+;O58ioJZ@p9x5IMJjmw*F#SyZdqgK*PaVTg7p^CVYfYBa{p=D2HpE8mN zPmSx!Z%m=lzJs*<+Do8#r}Vm%((ck9gO4On-H#bop7vfjWMl_*`7$y_#2p(CY9!s? zG(%EiSmA(tj*O3&S3drlqho9C$gh^hlGR!etduI=o!8t=_@JsP0e@^x0H}~%9T)7D| zj|i`fVZ#=N$oEds$z?5xry8Qn*q_S-dBqj|zH^xAOpAv)Z9M0<<2ujVIk{%cT#U8H z$Y9vhgf&c=nY0F0p?dtV~~#EMx_lw6>JSv}Z

b3+Kodq`QOejx_?((H|)jnOD)q5%aa7kyF^t^S1=! zpSDsBVY%L{U~ZM@)iNm$^BzTNPYK>aP;DC^`8;AUEJ?xybVo~{JvFn}IjwBr*Z$40 zpn2?=_L5axA-uX0WG*^GYIhIcs)?3J+mbM~0=(qjio3WhAgoN%8H?J^fmNwE(c>BRx>vWVyDV)$P-4c}$vK=wbS9w1w-sS#Q z2RyGRFP8c9)z$GBoBF4|pBW93KCf*}CM#J=a6e4U??FW3x=y@uj@AD?Cx$*h479e< zZ^-PE(-gMbUOk${cEpFDzPKu5KHbILVQ)a_wg(FChnT;pzchXpXdaZ;vRS1jQU7Ex z6dXAFiddUVDGvjWJjYt(DYsF@%i-p^#$JTq3jW+mK}V0?B{*-6X__z^eUG@$Y!~J& zsLOlwBofge(goc4kS*>%?G<-6{8R63bQsjYj8m}O&@r;9oO5dx_Qorl{f8JksSc_ zlQep?#AJsNo#joqySiI9b)?R?PtaKNWP`X{$*NT*ks$$mQ%#g9mp6NJt3_+tBc1Q% zoK^1+?#^PzK~WgTX=}R@#s3*{AF^xpiax@-<-bH5shG(os(l$3_nAbl)m`sD8*Mgg znN;K`JQIyO8h!UW{C>k1q{>K(~vvbZF3Z32x^_>qGnKlM|qxefBg`DIF@~u zUW{~m>GHH(!KkYfkqLy`g%Et8{QF+MFJ0y%N0)=2U^7uSZ2D0p81}zFYnH|l*(0UA z!9rrM9VXV_^&fH}x%UVH%QfvNgXb)F300Og zwBBg!gM&!3)t3jA*^w}-qOZ`rpt&_kuNf~QV!8l+EaGdQtkqKfWwpDE7^TukCot8X zITskKEcN|3uorQ2HRBg3E)((%l`8ZGIb@}N(>wLg?0I{oG&bEG&nr=JW$-?n3nQ)^ z2AL@T(c4BoBB%edc}(yg>u4uU=KU8fG1yyc%K<-T|7t?8JpWORG0}OIACxVUbZ`ZI z8)=Xy`!&Kp`0u6g;IOpK$L`}NH)Ov(*hnaHytPWsylgsNL7&ix-)A>=OT5y=rsF ztw>K#%l2FdQ9D+@Y}0)V<^dwPxt9xRIY^Lxc1*#KrLdtI^v9Pa)auD=b`g6GM&9)( z-}H@c7lLRT?o7}uFQ`}&$sySQ?8LyB@l4fKi13qW3zYgrB5H}fbV_&m;=69XO!bd+ zQd)!g4xd2SD7s|^`P?j*=LNZR^Y*c=b8nVUN_3}Xc9{QvyOQHAE>6k6$J@Si=!pkK z&!&It)EqDhyM7!tMMdKLTuKWYl3L9s@$KXu#|i{=I#iI6-96`R#6WULOQY$&!V!t! zy!cfs^dE1ioN?COYnO$@xDz8VnI1D!rVQH+)zr3iXyV`NDXXHJC)fGE{dNpS?;tL@ zM1LRS!oIie54MNZi11hw9gT)s%HEG+o~nWz1mDW6k&D0c`By>Na`qvdOl#wu5N&S?OtZ zdh)f)jty@Ev9!Ybym_oqrm`F+-U(%0Jrus_CVe<^IR>ZOW@y>3M0(Z6kDkZ0;3L5n zA31tuMjcbrFMcXBo2&S0d~d#3URhK!sbxC1`ndMrpa~Z<4X@7^h;DJ+A1^hikX+uV zlq@S*n@}Sbx;QsQ893~Y9TG4&Wz|V|%)-swV|}VHO12g8(I!ZsjH5a3AV%JFfeKIp z9-JjxD*>>=9^+|ti|cm+gG88v#h?-3+ZWy{4_NVsnT2bT5hzAYD)7!zH$tV4Rq0N| z!4XTk+Z2fB?liJV5;5?PQ5v^ULYm>U&|A9R#}zf|&da#xmifwlb&)d0GLi65-uC%+u~;4wJ=g=&#;y+2<80Dh|o@?`+g_tB6cJ=HObFQ=tTYxSR}4CINqS z>#Wsss31PlMJ>e!M3asqw;3fm+deHBL_(({4orj5`XnKvWjs9uj}+ebQOePkkd?|C zJqH0lRX0%G))aP$ppfolxy1Zwdp0O`V+M!Iu zxWWgQ9j03>B z_}<0w$;$Wi&>20-GfIyz!MxzWB?li4kKJE#^f1K@rPkq|x2gNvh^;nO395}=jeV!f zS=;RYY^?OxQvjTwShf9CJw{^aD)*|V-dC})WgCw}L}~hYf2bPUk`XwQvLAH}PUSYrY=SNr}T z1QEoa55|MG+cYy8~x*r$OM6?trU7(MGD>d0Rea9G}N|-Jns!rCTV6ubBAfffYIEBz=c@#lt+r zXnb9(P08q+Wf5C-`Q2@R^zDG{Mk;y5_Db@(Lz3urw6JeJV9<}f$8djMltolL*s=T6 zQK`ngrX+G^M%0Xk<07=K_B?wb`Dm`P5JL%|JO4MmEmi>{SmmrUX=Lo(K$c$}kbJoX zy-uM^ZNWPZ(Q61s-Z6rk#gi}l_BQ?JIbUe}7PGPUy65E}sEDI|CJ7y+Ebl$^jr}o% z;n3YXdQJ1itEmcg>Fe1e-?Jb)4M7)v7_#{d`8JCW;NzM$Fm**Y^9*<2D2l;Fthk4D zEcDVdMXHR^G1U+KsJolOy^8FKyS>v9QRULyDuaICzn8aYlBHDb86Ff#dfQJTx(F>+ z)M48?%NnflQt!aFGn(($MaPl>J`t8Ff%ZXja|rce|579GiD`50dVMh)RB`0Ma=(E8 zhcAVDe1-)w9$>~@NQP-gdW8PPTx?jJkN4ErILeJDwR$gV^zNMwM>0NYyAG6Rfv#wg z_pcN%J`eHOS;5e&@?@~(=CI^x`FZ)9xMZyuH82_2-Js3Er%Ks!9ou_s zMad4&JW5uil|USnr@uIP9{t%XLHpof4@4Civ4(!>Zfi*E7N6Xfv(GB504R0NvZNcI zD>bZzi@su2Jl5(LW03!e?>=5|B6&M+l$a`>7tUal(&j-`67{Yfq4l z)_PJD;QDscW8tM$Jz;`=7HWUrYFgX!wXV#!Q z2F_7gB9m(GgIRP9q@c9c#>PUX=QJanzViV4i*kxLCR3BD%`Z>2BFBVVrTjUbwx1lj z%EPJt<);oM0p@G*hS02^!L1Err;(|Xjv>{Uka>d0trVKu4ys80p;0}PD= z8lV6B7Mj*8aTCbRnzz#ScUkLQr$#fki6t7I zMCN5Fh z(nSJiVlg%sQ1Wk!=i}9PZ+Vc4`#<8gUS5LVQ=rqu`u|uLv`#3J)T!`<{E#|yb&LJQ zaRot?F{En`0gdR<^kw;XWYEHYsT6AcF+yf79#bJwZraU z^%8Jjq?lPpMO#kc{ zI0g@Fo^*pzpBulo_D_n}?=l`)@qDsdmXdb%Ao`M8_aCwe;od(9B8pS#Pi3Xy=AWKP z>rW0wM>|AUzPI5Ar*JD=J@w~aZ7-pI+79ul4h6HV|6Di`hW-uB%P-<4slGZ&#b&N6 z`0UPghGz)xDksUmCQhQx(!}z#j15+0Zu}Rj7G^FCXBaPw&%KDH_~2ww{glYZqQ7c> zu%C>ou^xJThDlb0Onu&M1yryc@`$tY6gLkBn9SkqJiZmlNve+(X|L+9PT<8|7+N&+ zoMe;M-MFsVPgc!8AJsgM^Nhw^q)t6yuYQ_}YiKMuE@WVr$v(AS)%+Y43nOLLOS?iU zf<2e6tDmLOAhCf=Wm*iynDISn8Sw;_vwGipdGnqOOJi!Z6{<8!q{TEj|NDs;jYUeA8>FmVs(PqNrA-HVb018H z8*7B82G=m}KoRC%>^9Qk3vGoq8MC8W)_))pu>*9Gl%Sr2VjRnN{%P@2q?oVIYi)Mf zn;8wGmQ{3k(*n$ezg|x3iY;7=BfP&@YAih7>xU_{rbePV2_|c9!CknZ$`|sHu8<&5 zO7}NADZxdq?qC03q1y1M)yITXgXNtY#?Fca6`KO96f{YxhKX=z#{|vnH31br1k@Wg zyuC(V4mtAU$|o1H6o(cr9dJqC?byHIg&+M0B}yiy?H%}4*qHbJV;C~uM?VFhu4);c zHPCQ2SzS@mR{DVmeZ@qLP~=A0`f~Rb`R}2o^1A_0l@bquWTZB?2JAuHUrn{U523>m zxzBG67lpbPYjPBvnhC%{!=(b2^4*h5ODeZ=w8Sk}`J}2FJTBURs3A>dCYhu#JC+;< z&v}JvGR7SFMAeqG?OpH+tM2H=d1Yq~%;MSTYCWC*bwWs+P^`^3oMwBAs4Ub%IiI)) zO^>_u*7k;oPv4`sRO_`*%~s3DSWz~s<0X}z>D6%>6Hx+W zimJ>dm*Ya!ypUndrwQZZ(~I|zvey%O+#{xQdX%**+ERnzIblg71i2@qZy(J8CwrK5 zKOM^zGK_WPm%IX?1@AqZuQIItx$~l6I%1@J@w;x)mbelXY9BWGjF=qoBxuLcl;4G0 zV-ssDDgQg1spPelGxsq*G=c$S!J0l?ew^0=JW=QMiQuq;~x zOx{;--<+M&{lvOagX^EDW#B1Ym#WxAEvR~zJ6|q@>=*p<&I90mbCaRi@m z38nNq)jJA2{nzYk$T16!p4I+0Rt+YW*DYwtkcB&7B1B+gMuJ#a-=un2jLsziEGInnqr#{ie#*N`# z4B@dsGwk;a8bR!~>BO{(Egh1)WFX;GlosnfH?8>_E;R_jEBixB<1PLlorkU93=Z%Z ze;hh>+K~BTrKq+K(-2j2eBZ&2(7gM{&$wsaoS(I{8YgK;y-4A6mf|~S2PI{@MMV8{ z;PRfl8t%?Lqx|lfVzL2i##R1IjaZZDUg=9hy|IU{uy$PRj&S0+;W3W$;J6^9egO~Q zY}}NSG+Z#8ZY@-ZU5qpdAQl4a>3lpirQ>PDBe|FGvGFCWp+Pi1loa5g6#N|QQu-68H-L2ErJ9u^M8u?q9a#Gp zs*@V#LO6Ase-N&c@-5VMS*i~62M=!cen}VP^{yXgR^4-a)~lzp7LLlOp4$22gB3qv zrHt`*6qtSquBCy%h2HZC`7ekBKESbG0m*>{8zpXxFFRI%{_9GUSlEnRv~l1gFX}(Z zbxGnmdMBR9yZe%;cmJY@GBMMkn73tg(Yk?PeA+3@z}G?V6@KNK(qx!8QIlx#?z=E0 zf9kN!+Un=ta${;eKbozuhvEp|3GMz-A+z-UHF)zCz6NaAgQ(J)-E$*9>n&_GCfDTx zw704b@;X=M(Ifogrp6?I)A6h^>(_H$&g2f^QExip3BjK)GV*QpB~2~TcQ{&-^%zbS zPqg+UHJ2V{aneiVr3nw^2Z}C4jPp-2yWb3@l&rj8rTahCUge1ReoPK za8{gK2|K1tYn6aMwYGCqMbL@fOU9|`A5P7c9-a2Go1OCZWclUwBrD5k8#+ZHZZ9H) zH#B3r{_h-&`8_EHD10-CCa;(-$mAG+)x7NsSGpckOU=gPVSVSxx&(O}PPPtgqGIqf z5L@$<1u@vU`5UeXw;q-l*u?8Pu2ZvXfTkYkfI(}njH zC0wBA*m<7?M;H4GU-?IkRemR5Tr3sgeV430;p#y|S`tq)Ape|4LGq zgz236j*rH@SAJ^P$|Ugh`vA8s-PcqKZouRI(9Yf3bxSU#y|1GX4q#j_Y1+Bf!ZL4z z9WyB?vJV0eW?wGghkJ1R4|NjVITR?NE7NjZ9$&9{bds)gJaHa-CsBRbmvi^>M^1b$ z4EX{&j-ggC*b^` z&|H4hir+ZQ1n0V4097qOk3}1Ck}}onTD;}sPY5tuqUxhi4}rS&i0X-W=J^3jOAsRs z8pdxP$m4%g4P;vtzm^_~<19|3kCwN2+<8M2JgN~YyDJ|u@FEwoQS;gkm_O(|G4uAn zFlfc4=5%GRuM_0MNY~aC(kj4Yl*U`J+h0X9-(R&b>Jx~G)~~1u4c5~onWZTYV7`}Vw|`6&$cS>%PO_V z`413{M+z@I8RjQ?G)`*zctohTH|$XnGk1svggdJdwt-uV z0n46j3U2`sgN6FDJ2DkjWmM!rt?BBl9k2F30bz(oJ1vHgwiWu6jnr$a37vtz?YPhQ z+OEojR!*rJW5NeGg^^l^bT|EHF}dNkMw-*rM^wkp4KLlKw=X;v+&gxFm7OuM&gVQp zVz!vpGx%5eQ49Qret;P^l<(eEI5UFq(lTN4-AVd-|AjrX!Fm65gnhB5?=U)!?D7Wj zRte^%6H4a8mJdaglj{7>jsGqF=5(m3E;K$A-;*z$B=(YJ<{vrdly-HdQ^CKt3u|})~}>m=27~VcF5(orEct@DkUpp$dI@& zKc$hA+)47b(KAVDb4&j3XYHc?vDol9dBS2Nl^ciF%BYst_^Sx` zxh_+^+B0;<&2T!h0-v}E7>uQmz2<@F-{Gj;mZLYz^;^qn=mLhHYG$GqQ_NdGuN$^P zf8$HXS0nt*w#%fu4EZKypY*h8hN)m;oyb^g8?ZsmO78R7uksY0Bo^RWw?z#|E*|ErD@5q?*|h$QX3u}0XhBVMaI~r9I|`PG~Axe4Coy7xxFOwD=zjh zy6A@+1^&A##pEV4ABR??JPits(#X4YcnUC`{`69y=3=UJR-^a+>vNvAihYA|bsA_W zBwcEvnzwO8n*)i3A6X;A{WU`WTWdcdW6=s>Lo2TKGci|ar-JcF8EmJAReiO5WZRus zZMq9S5^{e1lwZ(tb$x>EQZ~-yPzdqJ98PO`#fofdMpxqT$!o30<2o<{!Hnx1?WR@- zqI}r*vK`*BaIJR*7r@@qMuRim(=*u<^`z_cyZ1lA&E{d_UA}KPE*XFHRa5`OPMZnF z_LbdF8G~VaQd9j|vw4~8K#?~0;78JyFg7?56n|W>GXW&xig$0hj*0s% zuL6x0JjYaTsv?Eu<8lcK_Wb3`HGQj+aOuv|XD_JDdcg?TAd3fo55>uSt~((<5IIlY zf?VNg?ARW3*cK{>#|QO^&No9(haje7{de!DoWDr1y71w4Bb94S`>fP`^%9M!OBw$4>WfCTIQVh9uE&K8b+? zRmPo;jcroCd#>|i1&T=sI}?Z$3g4ic@7w_9I2)VLQE^cl`a}VP9*n#?6V90cf`4anVspG zcKGsZJk8L>?o6X@=X6*|X8IY3_^bs4|1n5mt&cdUE80yA9(}_XQF@%$g@+NJ&vy0^ zn^n9Eh9OTH`+(0hRg%Sa0V@tr@0$<5HodaK`XZXx-_`DyGGGQ%AA7x3VI27NM2>Pd zLZQ0$Ra8UO!H=-x_98P*pzxuFQ~8%7h4F*x@fF?YZ!UKkHuo;gM|h37b#h-Er}$LB zDT6;2cCu4H5)!v|^xYyo{G5I)+Z9RGgjy&%*2G#AZ-^V-ZtW3z6cTP^Fotvr7Z*4y z(7dZlh?45;*hddzUG53HI+YyM0OhUfck#1ALn+#ca|8aEvu*J9) zD}GtRp)4m96Bj@AW9tHLDy6mABm;c^oV-e8ald>#val4yH>GcV9H7c4&#_Nf%Y-2Q z#PwgmlKRUGN%Jon!ZFjLfmUnGp6kG$DN<&cJ1tc8Gj9i_X=6W}R{;H>*hs=B6HV1F zUl!hlss0MfO<9Yi{T*enGKpmII6(XIwb)aXja%%odgpXICEwWpW!EsV%C%}KS)FOe zY9uP|Sj%t0sUk`wi`}{%GTp;@bva_;1t~D+KUv~gV_&oBw_SvaG%amdaF@RI|m z`)tZgRV4?@qDlr;oAQGu)|32Wf1tc8@_j>>K+h}MjHXw7VR^nTxJ|SxT7zl*M2>&*KL50d1WQES*5{fz!%Go z$u$k-w;6`ynbbN9ho?kW%9Xcker^NCcH)DHg`Q}>IBGa`Ok%LV6?6;L)@%12;!nkR zAJoZM_0x#FCJod43L&7o52;sS`p=z9tghZE2&W3?ez ztWh-cI}P~lzc5S0kX@CSF6z3fqSNxf^xfUglUDT<&6^Uvu}l3NI2Yu4K#ngfiz}9e zhd%&rOsLsZvdWelp8F5_(fPYgF-)OXLv1D9XIUb`I-$1f-adI%3dgOPt?E;zTuO)O zOOi77VrX<_q4litRj1~?nLU!41zTTL?wOQ2=JkMjlr!oC<9n33GHuKLOnWByyd?q? zEMvSd4Sk81mm7PSV?}<9t&SrTyxq8G2&m>aSNzc!D#lhN3v|uE)TXfxgnkd=*zGpv zFnwfTHp`*@rB6@TITC|hlQt{D<3gB5fQFBaQSsA#U(W1jzpMJaKQNpU$rCF(-vJgb z>K>lT1+E(`{HA*F^mThSxHGL)F3*-!+kcJ*l^?Z^aJ_NLyyeky=P&J0yVAmGT;58*R|Aq?XA*NW$k>u2jc19 z$bC)b2;DakFg&n3nzIhw!25F!V4bjhvlj7MF+4VR6W z1s1k$7hGDViFgoPa7g{jcF(F3O5|Gh$)ZhlV2DQP*~vWfj$%qL^0Z7`ZS=#Ed4SLbnO}eI%6G6dyk)kp_s>BO^jF{xRJQ!Y zud{o##{93MGJi11(!w?|KljApvLQj7AZ23zRymQ)i2F|D=h)^e9N-ZqfsBv8*Pr%F zkePMb_pqEX$L)np8U8ahcOQIu*ZiG{8pUh1BmRw|M3_%RVG>{tnL0_c`pZGR;-gy>!wKE* z&uuPdiq+G@e#yo&)_+w&AW!thms&q}@baj49yT*eJ&(4v>=dJ3bhfE9S3P7eJHR!x z=$bp{*auB3=kT>mI&pm{$L$N%)ysZ&C(n7Yxvw4%+lgA0*`h+_^yb_6Zff%T|eJig`sI>j*K*TPAzwj7o{RBcU2?3IYm zv77iz@3>`2b(BTeFEzTgcu@F*d|&f#udULU|971KWl5V5djA^PgOc4EOsf*vDgQaD zSXxqqd8cXh5IY(fds}Fbey*;UJimo{-b|?W;7hXesiwIWocC&k0piISGa|o@D{)c<#C7i23B;I^iyl7l$v!YHwP{r(;&T2_& zM!8vBO+982Q3y9wv`mj}1P6?R;L$3)!|It{E}QM>f!E6w%FTwQLFN7o2SAroB4y8Rgc ze6rdu`7*$f^XCP}U3afzwAtTb#<5e!92~4pSvIJT$*PxsC5y^XChDHE@;`t0Z-Rp{ zsvFl<5t+CZ(T)rLqkp9OCWu~HBn@dy*47Sg^PQY+Q;)S7&8xwRCT|-O^aiYxbsv4O zFh`8TVX|UDX?}9&b7_^^fB6N7rO1rd z$J5ouN4>8ELull1e!v}f(-e8@fvW7+I%yZf%^{A5w}_X^M$^*N$o}Z0y)|cjs@U+|Naacl4Vke6V*liE-V*1l zC%Q8zTAiNQ>dvMsXK*oDKQQYoHipsw!I!piiv004tuGD>%!l;qRgG`vf!P{v1ykKe zDYFd{ReyTqrp&BkFgfzxP62!E#Z}7mUyivv@#juQW&5QggSo%!%(TA(H!wnwtRbHQ zjx8bDx{B#Pok7+wdKw?tlet4>ceIpZ`rB{lHPOfXusGT1jajlyg`ftiDUmS>L!J#* zQsDeh^`WXkVsZs$ZAEgQk#HEJ6nY9CxRvbA6Eja`mFd2_fQA%b!K{k|JCBcn338R@ zFcM(Ni(^h5DwQbd$mv!l7XFxNx?>W)E&qRXjgyqd#c9s2I&rqmtm5xD>*cVDH#5Tq z(cGa6>iIoyy{@wCZ)fdY^UM;t}$=t4sirFf)2mS%`>P8CdvojxYR(Fq`T9?upS7mXg2=!9Vtt ze(i;d8QS%#W@OYE6wnjYBk1$&r^*q`lx1L5CGfVf%zWRcB_KD-+cB)e%Z2`8mJ`nh zaXOvIps^ryHKSwJy!Nn^g>Oer#w{fCJxbTYYnNRU!RCVAOY*BsqsQUOCBKu@=W+}d z97c1ji{w`V|;jn-+x`i<2h<4G_0E6j6YJ_T)`I<^`iO67%hnt)tk zuZYTrUaBr2x7X)SD<4=-d^YpnKQO$GKM@R|WqsnOdXmV$ok_R!=0vIPlHOCM&at?# z>K+)%L!Oq%I(3Om>y?omU)I^N_FOD)ea+g}f0zo~Z1h+*ZxAwiE{APZ$~28Zjdn=h zA0h12`hi?)C7H}Uk|UE+!k!Qi&qVl&51%2DW$kA!(_A?RFJMwUi~Y?2=ad2`A11X= zG-P@rb~n?|BKE=#yn}Kiq+dCaPLskj5N7LSP8+*lao!7YYIB^cypJ!zXraqxKicv^ zZHwGdX47kzfCK*bB!8=*K`k{M{c2{|tDgYV=ClfDOMDlX(B*4=7R+UCc6(T#miAaE zABxNi;%VKkcyLGl-z2HnmTh0|-r!xW;>$0fJiwRLf#MkFE@em!D-(?BOmbh1^Le~r zbZXrz!8}5(%bQQjZIZeoNZ~#Vl%@86{;J!A>zViKa%IPz}?_mDUdd5&GgFM}s=it&MRD1vb zor`AMqHt-eW*&3wH`xUBJHq;tdVjQ%9e7FTHFL0slU8T}e-9|2ragfyogNJ^Y&g*P zSZ=+&Yt5isHC08*3JeM#3Kqd~oV{n(wJxjhc&&tZSU!Q4+GiN5Y_!bRQsN~+xQgEz zndYn?9fD!6N`SB5c?rwhisPyW945|hV)V2n3me7-jMA~kzf-mDxxM#!eBbab)sZrm zqhgms$WE-dF3EMCIJg-t{SM+6nCX-v74TK)KGY20Ux2+bXG7J@S8FuzjImtWSI{ij z|NA@PDBRjC!(W^f%D466!0tw!jQB@34uY$5=0ndTzIz37r4l}b#ll_e*~+#tMy|F3 zjzy7>{IquRv`A&yC4pPV0a|V4SFjz^vV7Pe(e>L6jvHxiYa_M2yv(#QPK6tl?@nlY z?h_Opw`8`Tql%@lYZ#KTMJ~_mY&-J%A=|14 zN&YT(4(;QYim;yr`RluhNLLtnokLJzNQo;0;ssxg zU!D!uRM*eLVE^Vxu8WqGp4b&*8?;7%t!~aydud(Kh|rb9&SUt4hxbGV^b`YpwDGh= zmxAu=LFE&&$nGTNl-1_L3=LxJugYb$@`rqeejBwHWsP;>?x*HR5Z{@1Igkde$ z?B1;MS+!athO%d1v$;*(krzPCmueLKy44?QgkLI@k+ z0kFp0AeX@x^Pg}c*A~hvIkHmao%P!I=@K)p!kCb7X7}c*E4v4^Jl(*B=XM+t&yD{M z&3$Z{zg%9%oPHJGIdvuoIasJm_>Hjpd%4a@2OPa-tr;yP^h)5r{7(;aVp=18&#Nlh z?tR)dsT(uV-Z;rD?dyYRr_WG=Ge6I6DaOlFns)e-b1yCXaV5KqYhdg{^RBhN?vB1I zUs8=R??(Cf%$+hntAkO4Ay!u`xf^L_lN;(t40N$08nGH{GC``Gea-yBd|8>(zdg;9 z44l&&+^bipP^^3k;r_HW6`gd}XtzNYO$rEHlr%6wxr%8N6>I8lK078Y?|pA`FVnxw zO{O%3Iz^o1>zZY>>n)|mKp=g3ilR|cA0i%l4|t_qV7Md58lvB@CA$_2AbL#Vo#)P( zzO{DCib3~&r&Mf0pvO{-tc9r+)cW27_Ov-j6jM#yj5lcooQ;p4a1x=(uMZ;;@*H?N zk@ftAKt+<{%!{-Vn!@X-d);KE-MCYK7nSj_-CGb`@ktns9Ye?rnNOYD7vB=Z9h+N? zold}H5<=W3n6dr-DT-}!b~zB|i}~zB9G={7DTYxepx4BKA_=vn+)>l z5c(jdKy66$%2z7Di4EyRZDD*?3p7BnC7ie46-Fdj(4rppQP>dEu4^Y8J(WtBo|uv> zOKW8)^~a8dA!~Z+Y#|`_S?8tyv(EB%wLHR*+Sj98fjqZH)&E`SMym@-_MOFxlA~SoNeTAurHdayYklp=B zR?PeJP{;mqWo`6lD--q^N0ezcP*)Y9Z{jeaCvNWj`0&Q)xwYZpSTb&hz;2t`w!r>w z>T{FrljQ?zI!6tAOA?n_*QuJ(=P6t+T0Wkhow zpBxeQ(~fhF^K0?h_tj@=dNbd;$)&$o8h7fVdP_pLln}7i7E$JMc@`_*_p!I<5&*i$X;YU)ZzTI(te<2c>!#DYExiDHJNf4d{w&HjG-KnJ<^FMS{Qn&xIuHl{> z^xJt_Z#B7Zu&F9c-(;>g@OY5TMDWiem(0p3@p1paOA%A>XkTe;{V+&6{qr#*?#%CsSBG$k z9T*x8=CM5fWnAXd3PrJv)cSCTzTQJecYSQKa>@qhsShFxp|J@Y!Pfh+`c;!VAHFX!ahZ8um#P1uHI=h zJ5+EeJ`r^KO~VZ!3xghy#3nc`Y9`$%vajCxrX93|RYAXKtws<$r0AuTXH^ z$-2(_i0uVXLcOx^m^^k4;&!80I{q%>cU~%&g|BtWictcD&@k8dNH&F*a?yM+&x&AB1yC}tNx+l6kP5Gdj zH2Jw zYQ~Q%9WXDjCKT?(39PD_`e_Mgw!)!QjWGJM$+b*uk+7Kg1lP7g%w#ZbcJV6bdOnG` zj?t&7-0ys||3oacR8syjR(`;cOXQY(ySn&IY6}D+%@23C0B+5Ya3zhW86~O5-0KH) zjziUTT>nAZ+VqBI>B=Cu)z?-5X;-b3h5i1s7x&kxhF-oLb=)Ls*ziSW)dKf>CbaJ5 ziB|DZK@YL@p@K!o)@7tB#F=VaEFZhyv8x&B^8z{*E!+xLG-<9QRu78I#n}@Le>s=1F2#dM0yl4=t@|q4`HAPZ+RcVyn#HFgT{n1`dwp|Sm+`L) zr`ii&UgiAJQsyWA#}v|fF@@2HdV~of%=Nu>4nq9?-A3tjh!DcR(EjD`P6RTH(#>*EU~6>fT(5e)3bQumuRCQ-5CMAZM_yoF{cBqFqpYqrEA4rKun2d=&5snDgim?1rr{m*PzNREvx*#u#`8{ZN>u?&qX zc@ZBI>Ib}^Pu@#=<|cHrUrlLcJwHjD3U#io345}#*5Y$zWXAFUY6&J<+%RZS?LV12 zlI+lrI29QPnMtld0ffVvfYh#x$Xx@Kb=9*a$GN;K;|~V6!56_0LbB$T&;$2c-n#+w z^))I`#{XsPCNfZ*X!?L3wZKlQ3$z`sUvGywKLXs2K6?{Hj^QcR?@5WdmH6mf-D$q;uR; zbbM!f^Gf&6tUA4(1y9%fIroTO=t)OkQpIkIu~yniI)j%nk#Ae5rL(F;gWHz)naMUoYnXxPm88Nn{=HwuPW)CJWFC3UVr(d2>Yu857@Nlj(M{c>Tz}V*c|GVgOHKlsU(@bU8_FO z2=bOYJ0A=qtpSpm=K*UPrVF@YeI7MKLq7umMohoFkgOd9Ik2^X^ z(3Tc^vip%Iy1l!1Z@eRwYLHRlBo`LDw`3VTad?&q;&G zTzADWTXUNT+@M{^)MM#^+CjbKd+9?S#F+vQ=Bl=C65WW2FM~(t zPB9d>OC)N2JSp|Ea$cPQrTPlfCcdplOUPfXIi_g-Gy>%4>s>4?X5#HuZ0lZ3PY++Z zIKFQxLMf7Q2+sdY65={9ox)lPzQS1WPt<89>q2rkDqdJS6tG6D%$g+Bd#;E_ckQ_? zPlz==CpfPp*mJdJ#*kB)fm7i-j$L75nBrrH^disP0c-GH6AzzvzFvXJmED8mucx`z zHiT-gxr0J<@ZIRui^hFo`^i3pFqtVX)#s>yp4R{Iy^1zb)fB?Yb|cjq)M^Gd7TT^9p4n;KDWOVsPFu`&aA(o|`C z;(;@JgOb$skW+qGYoBQtUysmn0*zA)%JCXy;!fO8BE(R$nNv?X6pS);ceEc1L>C5{ z<6%X>)p+2Df^JvNjvB)84rvzHDz8?!2RR83i3dC!}*ioKk>iL4w}gLtk>Tr^ZH7u{ z{6~XZuEI|HR8ghB$w}|}hnsBzi>RLy;sx9wuI#J*V)rc6nSY`8*`Z}L#q?l$c3U0# ztGRBJ^whKhwPN9#I#p&Gn|h7I7mK?m{K~d7zahwS@vtU6Cz7;l>Fe&(dH#`{X zDW6wwuTy7SZI>RJbF+R;USB!rK(npg)|wFd)7ufp*4Won`4HVz2`GNP4AIFNwPVz~ zO0ZIcQ9?{Mg`RRW{Hf7AtKKA6a9)BoV+LmtjJGZ$hnP%EL$Sa{#SrI|YZjFEpMk=- zRSnAAy=k=`!=G`9jpd~nKNFlZ4^2)ytStIho$#>fPCyvYND=Lqlv4!o35v#Ut%&6s z${7}R|J;9GuCzG@y&RJZV7|ocg^PmG8I>)P(|c#uddH4Q;nBnwliG}9MD1;IH&qn1 ze0NvMAmiIr{xF;1szXMJm%h~Coa_>&$C0@0?#Rj-XgjV?p!xGnEUE8@PA2rB!$m&6 zyWHKku5vtsdM2j}e^O?tuW&%;RtR)x#|oG<1o~LVK7vWq8Sy%Npcat?8A*B`y0| znCO2eJW4ZJ@cNRh(ps@d2+tMRT(0_rsbhe)f4(*6^kJT)i zvoUAo*7{&uzbkQSCbTdnWpq?~>l(lq&B+Y9Xo&#gvwXLa41rS$q_gMcnYsnJJ@LcC z*W7*BcBZy#S?^S~n|zD2gY1ROlXa0Nr^g*yMbmm5DY!`xKep8?UD7jRL|s#kw7e0N z-LTJk|F7SNFT|9OP?~wckFF`dsud(JXM%VAL{?=ncPrngH%W@*T@kCO1*{yvFvSO$ ze@@howW!P80cWBPb9t=@HQR1Y<98?L)pMcEMNR_m`EK?ly)R#23egiPBPuOSW*$;* z$46MYSGp;iDQNBS!a{E$gIbws|F#t&Hhkvq@J)rRT;W7kgy%cQnj20bFOQ>n+VUWe zK+Ww6Z^xoHx)%rP?fI&#(fNk+Svr)wK>+m|5z6^{%U4j0um83f%4oaiF=Ww z6B~xKa8~0@6|3pzMFw!E%V$3SgOOBirjzhOm$bu_1LfM%2K9di1s~j+*F@Yo2{>W0 zrLQ^5`=&PAc{3c?bkT({_1-eXBj`W{W=NqoN4505%TLWgT$zd^72sd4wgQepL8AfYVMf$x?@ z`qp8{Ub0YIA+3U&7w-J-Q`#qk9oh{%N&4oE8-*D1by3f1Wsy}*M%op-Bui#wy$j?s)iz$@{BNEDpzyL>c6d%| zS2eYW3=FeS>xJzB$!Wc6tfdKJkgL^@-A}FAJ#>M3maMim7W_8>H($2xVzVq4nU4F- zlphik4hk^_VUR(K6*p7XU&k&vgSYb#@u_*`b3gu*U~F4+hpHAeDXa3amBpOgj*QkZ z|LP@!*P1IB`6WZJ+!vJ85{n_5Sh;4aG{MI+tsD)u+YU@)NEnC?7y zm3PYL64TBEk1p_P5fm*EMR3ZuT@)T9bg!!FO~EdEN23fGQ@5~wbwm6?qiPDR1DcnA zdUvRtryQzLM3yw_4r&%TyKEWQ|L5gkrZuPYY$1RPyS_FwnS6ft25dd-&a2!FGnW0O z(zw>JqhKc@lBx5n=JMA3Kya0K;!_&5-8BV!aunW*%_>!mQX?DnEO+YAF)g9?Q_ z5+`iOx4zdi1syd=nvcZo*`;ho{E!{|8|oYPxwE^fpLQ<{BZJkMk|qFJM(W2apzt7e9y%9oaY!%T(tx%QHZ|DkHbPUd?%sJ*>RlrKfb zdXuH=SfjyxV2XmFyl(pB6ao@3@Cd?$`wIrAx9m}W@OSi$&;x#!vrK%n48+Mw#ry=L z#eCRcECFG5mV0SS%&36yuaWIqw%^?XC#(H^s|oP2cl~65X<47{(vaS&Wtn+_$NV&# zKtf7~YH6ps%gzb-v+9{@wo>$BSIPyW91FtMGNA#%)O$TD&Lk|_dc#TM)~k0Cs{}k~ zBYZst8~EhhqG(imnQ=(kPn&p}cbIy*%%^XGP~(pwhxso=eSaUZrs6`IY{7R-OHs*z zp@R>js)Mm^N>HN4+)%p4l6w8$l^512HEQAvZ|BsL>4rM}!qxM~cqXbDTOCZcdR3o+ z(xReAXJ=W9Q>psC7RMasBrXEgg}?l4JW%R$l~C-&pB01UeDr|q?JGo*{s+i|f(7B+?T=+Z6OpmQp z6B4<=AQ@O=O_4C|EG+&4o>gzJVO%icJTCaRbY=L~nC4>%azp2O2VsgK6~`RV`7<$) zr}hUfXvw-4uON=2+Gui5LVLCxT~-)TRG?047nsBY5eh_4Dg1)D%|h>kcD*Eec>vna;zG24F|` z`G)11T4AS`gI<~$Mi$!lU{T^B#fk?E8Nc2g-y&m1P68S3ERUvGmC?mHc>m+{x!h&ySO7+9f?k&Fc7qclctv|b+b#n)t ze%E&>M)y!gDlmV&8}PWq4XAepVHpa{}|Kvycks0}p^E&m`rR-g0lv z_6+CgWNPOnmDAzVFG*&?SuUv^>sZ%9@l2aOh23!ja?ZTQR8>t-g*VWk*0h7AcUwxl_VWk@Z;S(BeHv$ANn_+XKff} zAmCqG@bM?A0s-ysSrrnll%DLYt+hwgD0;wkj$3;=Me8p*ghgFzAvr0!(FP2M9T>~*XyeHP_IjceC)!iUk9~BeVtgbE6^s@y4$|_LP_{K&?EI5 z!la!*Wd*`E+PpcGB~-h#y8rmhIJK(?ILpUHcf431oDHOiB`At;8Ow-sH+AUbQ1>tE zywSgoK^(RIySUw2jm{?Or-MVh$i}=PfYJ|RJ_mODxnYIgUMHpm@OP>!NY!=XZMOva z+hD6WL&7`21TpeB~Fs$OW2%+^_d3#OhUJw{9%7#rTax3d$Dr1VwBv=W_GA3)kQGO-;Kfg;48edO-XFX#^&OY^3LE9Cl>lwMKC{r&Ow$b{zt-C1 z`2AChSgT{6;LBpol`rMc*21V;AcU+}Qq(206L2CJMkA=W4}>k05>Ic|vvYm2Sh#mq zvwJ%aF~YUesp{qw4$W6!=WXMaj&P%X)ZsG{;UtGoZ?Ypj(UZj@-#pM;ZmmiZ*kTTr zoJm6VHMHe787H zd=fcBAbCR_wXz}+oB8i*Lpoly2?=iCjkUD4N|2*l0uUt@Md&wrSp=_VD67Fg4NVIp z38@W;%+f3HkdK(|gzHF|_?udTaHJzaM-4ZpCdr|6@W^@I94n)D6+5id&qN^c>^hwA zo1CepoxiMH&FY2AU7X#v%(YM#s++)^SLQngbM#*`5%($3dO{>rc&2`)HI4cexJhnr<}mocg^cW zd1nH~Q5lV1U*I^z24x3NNJaP)%F~ zl9IFxzlcDEaTc0uZE0^^%lds*3Q(g%fx#zU55>*)gQt$KEc&(z3@W;7H@p9_5 z_ix7b_gP)0hp^knfuvhcQe`M>@nuRw2AMzByw<6JXapEaJ>MfU;6Go-aR7pmM?qEX zk*D8bl9=OZ9Bo3nmR%y$hb&BL3Xi+TJnWZ@`$@;`(h*YokIBkWOcO9IAq6Bq7h8qd}Xl&FQmCJcYL{z~H z3w0AT%Wg7Gw|{yB9Y(R#ilAj$bJ}SBDz`~#SQtV-kTFPXiz>ZN?`1yijr>FzUpo*W z{cC03c(N?>n^3{vdNa1xaoAE*TAVvKQY^L`<9<1yUrNOHka31aiI*{Q0q2FfyJ}<4 zM?i^75Sdvp>fWLVu)P_(r7&dNAz595H&4RPkgzK5VSQ zSJGg2OPBc9szj`2TS=$rXo||0SGZ`%HvFNq22T9+u}eqM2lI2a>qLxH19AfV!_1D! z+X9ow^-F*@Q3i^#{bIa>wEm~5EozRMD3mE;uq3_TX$VeLEVr6waS+K3{+j_5_wun^ za^65F>8tC~!1$`U$!Yw0tMcWjWc4UI0;j)PJ_%@sH1i&pn6Z&N>> zGi9NnZ!qMsewR@;9AifFm;4L<+Dz1Ez0^Nq_;uS$Z2P3^E%bm$m@1=mSY z8-Uxh)pvO{qfbi8Ez~54lx7h<1i>E0kNYpq5YU)*KBbOzK6wuT?e;qHXbyvmYdOo% zWl#Pi`|Lo3vki-_Ot1h**ey={RHbr0>6+)$hjGz^2!ulL?kK*sj8F`$F=ht&khR>- zq+;{Gu8(JMiIR1N2YaOasY}%FP%yFAc*?1*=h!aQ2*o`xnp3LmZJA=nTsItH#n<>< zM(poo?tbYXN5tU&U@}^NofRg_Rm^$vMr=z3>v8h*zU-d{R+*CzQ}TnX|kXv{-R$>GSbbQ65tlZ=5bDZKw z@#QAd{)@dH;ujx2$tq_oU^y?)MkV)#Dc$O_O`8Wt>;B00|4o2n)~uP$;u{PaLKGH- zWBjhki2>77Dx?E^-L&4H)%g4=V-Bx zhT(EUrQoqO37uFmq(-sE_pOCSkzJC<1WneFS%aZxk68Kcm&A&AWKcOcm0JsUR81G` zQ{Xy|#mq>|%ebOP`&k~_qZ7fsMT=Bkemog5p(hL+ojrDWlPNT|RhNmBY;Fv6$ z@i#5$(RBtLybpBD)pwtC%@^qhKJCjCBiR?djN>qCkj`$Mlod-Y`_LdjpH=@j64>*S z`8y=!xXG_GSf5N059FI?Rz8pJR?(~$fbTyf530!1T;0qt3jk0L_R1${QvVCgX~`j8 z8}rJB#Vzib9or!tg-mC4Jf;=?phY_Mg-S%m zWL^~*ucWpqsIvb$4PS@4-X>#{Tdg@77Znb=KO_k{owhYFC&CUMZf$EzNI_V4&Veao z>pr3=_FpL0eK&_>vHnI$#f|1sx&~0eRebxrjg2*nE>+DkO7|FFks50*ys?%I^Zjzf z-rr2l??L&fLYF|10Q+p^Oc?XpN_8+KWjkJ%$NkIIj1ZP*Gr#1SyzF5d855I%;g!+b z=*ZsloGyflSXvu^ZW zF<1ZKvCFh#*}AzcS;vz(?ahkVinV6RYeN0@W;VCd9F8}`7-R+1$-%3(Z?~R)d5Nul zt7)(XNoEO9b&09bIpVFA|E1Zq z2zR$avbsMEnR)5HcY3M1bulhjmiHD^Jn?E%0x_=>Kbe~Zv{+aAcgi~?Y5P=Z>_B$- zjakXQiT9#`yJLO|Tv9}xwWdfs%Sm`XW_~G{&YES%Ucu+%f~KOK`IM+q+@B7@QpaXi zlpxF%^%*p}!6ScGTCZ?UA$HlrayRpfnjh%jXg_tE)yRKgq(UkRuY`Qdc_zugB|&V) zzM#E^@A03QB=&aZYHL0(KAt!=o&!~hObPj6uhW6hMzdb;=5S;fNU?hbvKSP!(> zjJ9@#)>ea+t&Yy#pR`FMMpDV`w z*m=O0Ljl#O43n~Fh=V2qq<`{$2@D+-7G@0kbz;W!Tpn1cl z{hT&#yp^t7w;{UYcbR(B#|a8w0LN*hu)%nEL#iL=*aK=3vn^JbvKDj=P*cVQ7 z2jL}2JHjPX3(M(#d!F)A*+HU25a%ctKFD5rvB4g%s$NEzhY5fHCB z1?07E-cVybDbSQVYtok!k6iAKq)1fW`PSLGsrcHu+GyU3!gFsbs;^RVy&(D7BkM=S zFAs6GWTumz*=SU%c-+oMj+kd*Dp9oZ!`jn>;Q+%FuYylEJe0CBuM*)dMczPphy9*K z(>SG16l~d5M2dpXpGtB$6kRzxmo}>?PBeslC8( z_#5p;=wu4-Ae2G-+s4Gr-Pv&c)uaRD}8tO^j$Zj{s6AROIsRAQ~0kZ2%y+MMuo2QLF?Y}v#!1#`Z zw&#mCUYv>THmvvnyYSf0xwUN_QE6Qj54{8V6_regaI575j$+Y%#s8|A93PE@t&OMB321|=og{yN-vyB?%f_Q4ZuZAGlR z=JL1TU$dTcuGaR-KCd|x)t`H}SS|G{kk|?Kki6dMRa7*!jacLsKJl2A7k19>0~4=2 zvKMN86lwnz>6%p}^6H?)B8k*$%!ltUrZas_VXTicee`!WQaqx4k4XFo+&>%pVBb?)&{E=)zwgE9#pV(nK0l$d)YgbYT1q}L?c9kRq{Lo2q0wiLI*Yi@>?=a!l)mkDPgUt<6oZ$cw=82E<Cz1P<+wmET`g70D*SP6 z!aUA2P;c(2W=upcy%Z<&sL(30;Z-pV+4bN4MbNy=^TVc{M^-kGk>dK6A~A2RN5>7u zUg)8Bh;{K7rAV3I?y2kdWRIr_v1}=S_j(^1`bl(GKxa<#?J_|urH3^;2ie;GyK%?! zW|Q!_z~W~j&w}Is=aYXtV>(&hBjk9Z&Xnu0*o-{w4?p8fsa-)P{-`@OSC#q919|y} z7|WB7k3Zz_oYwse7uIvfy;PuCn^W>Q+FGTKbg_!>DIX3_1=G6cUgDw)s773cpPm`< zSmsCHd7Gs9L?=(v@VwU$t7ld8@ELOR$%2P1{BMylZ-pS>UxBgBq67!vBML)mE#v6Tv^pJT%?k*X?M^N$jb6yZE;AYg>ySFx-?l&MzA@@qgQLq{^T( zgfiLa!5JWE>&5i9xw2yE3HOg+izX( zg)wBg*}Gp3&dc}p#JWCgJQE+Q8p{z7>c}QYtg6^f{qiakl)hCbfw`_Y(Xw|6rzfhK z$CO>p3>By?cNmxz%R(GZ4+}}_4bin(i$b*z88=R5XfCLh0JK`_QZ7~$<@6fMJudx&=fZceH5p zl&s`;^_T|OOsYcq`C_&8%Nyb)AblPC;>_afK&#*bI6~&_Wmf0_`pRoiWb|5u?sdAe zLr&=6$N$BOP;ZHd>s-|Hr-#-BU3#V{2f>3kcTc!_t^Z%CMp=BUoaLYH>P@^V*!JS^ zZ?z2VY+k<~%?soA*?-K}_-+2BV$C)+#%IV^))#C*Wa<=PGyAz5-WF}UooMbD%Wyw= zQwbrH%F$SG5l=Q@r0Ki1IjymxFn;~DW}#jbHd--O5fr@(jx+`Z+-vs8)feEcGc^}7 zuJlWHL+;wCALAWarcTF@`Ynoz{f9{*b(RyM(=O z&6TEZUt^l>D{p@#KUiYAjJ`LjvjVrZooX#}|K70U`_wBz`aQ0hUx5svFLG&PnwFxZ z)}2ghxy6Rr(ID7|WCtBh3_v=0s+ZOmaUQ{<`g;BK+jlHqBQhtb#QJ^NoC)?n9fAGI zKU7&YPe}YDD9trl1{xNTY8h*(a;w|#BEqF$3;w5p4of=O9o=J@q3?}V3`%N)9~F+Z z5+(-S@;mb}<+o<*+z6P8PO7(rVYBtySns?h!RSrWDSo$F$0k)71hxn07V4;V3i$+1++#?h21C|^-+;qp9) zBp3AShTWxfB9%hQEe7Xx^~Nq6%|^ULt;m&qJi zX|bEz=ib@bE_?zNV)`N>xuA_hp+3H>G|%XskiRqw;8El9t2|uwtKJg9Yc~1I1f660FVwW&jfDnDm}~gs;Qz|-uvr97SN+{(7=vF%5aeEV-?*|N zZGPzrV^nzAd59?Zx=gcw8o&uHV1=yyRDzWcjB!6H`fBQYg)bAe2ufmq-Ty4O@_c%} zH5JLsX|j0B%En%Q6t&g#<;gUvR2su3XFnKRYtJ7U9*M(6O5 z&`?NaIt#e(Zkn*Jliq=^ui; ze_6R=0K&_^0s;zM^3EHF!k>@;;huvBHrdIl=XUv_Qc#}O50#@8F9L(AYs!nBv^Bru zc>^cx@pvAY4kIQI6fE8RnQydSJ@ zAndU;Ao>>+YTT^(M%A@1isuW*vRvxRAX+8BvRn~8v=oX4(ds?<4{i^FDQL_E^Ku#n z8s+j?Qb`kpFC=-o+Q4*4;|Y^RRyu2Z%h-tkR`rFxIfz5fJ<$pyJIHwu`}gQVp_N6d zaJ8HML<6SO@TJHu5vfFE(^7OaH0{d~71esxxY6(?k-BWS~>~qbaf~lmH+7wVC;lH*zF6g7D8R}0kKa4OcG>eH2W3x z%s@aefGcTs!6fD4kH8aKT`;>v6JQ#o>h1$;RY(TI_Krv0h`8*qvN*aDFYdnYf_kRN zDmwx`@Tjm=ZgCPi_r>#2tLIyu0BNsKMAlMn)_=RrX0C89k$3dVHERgczoaoHOULr8 zX)&PHw&c3vW?=hf>VbU1A08(7(ic$*PZzO1LZGvBuqXBPT7*lqHe+w%)Iu z&&5T`*mYKLhqj$iA@PYIjA4fV`oE4@3b^^W=F_<#uY_tllLDffp!X|hg6f9z8=kw_ zk@GLSy(CGQ7P8)Ysq|J8vgI`ha_AK<(q_CvpnW=_8TskEL%c?^=E{W$G%3F=%5WA1 z`<;oeSGm|yW5fFLTCE5zw?(98*E9-?Mwsy7HX7!@o7@f zW2EyDr)!7273Fc+l^Tn@@KA(Pa?f_uTALP&q>pd9YKPnRs^&2b0M|y5K(wzm+dpSs zgI(Juz#?0PAT;N=PpjTQ){(<_wL;QK|EyC|z-82+3|c*j!iz@uZ)rv?>s=Rlitp^? z$iB0)_P=DEA%iqM9sL<2U)AJk+Jj<{pj=&E-yllCKbN`ld!`IuCk_}`)Cw4q(k%ir zqC!n$P31=_$WMy)RcVamEZbv3`&*8wHtrbbqXgA!=#t}4vN{c)de3W>L%Kp6)5uxR zA(?njeSdGd?U#%P1=;qlHuqe}W@ZYycCd$Q?wgVo$w-WH+!e-cZ~(ynaosCPThgdj z7RGj6@c*{3OzZ`nXk$D6H8bR==sDi2zPXUjBjkXVbpsdUxZ=$|^~ddf$6?^;C3|2& z&d-K$_(;keo4l>18@To}k2GMd?x;V~K}}w@@~vICrkgLfvh&lkR;&;S3F$|cJ;M1;2=-8F1X;EbZmu6tb zN?-xmcG~-J!F7S4=oFn8S5@!bV(E9gfci?)CQVay#3HHW%_SUA_SBoV?n`xK3_Pj^ zz+3Z~Z1S8OfnjBPUJ;w-FcKNG)^ zxk%fTd}8u0q1cF3D$;32{z0&v0Q8?f%8>jGOo`+&fD7z0A0`Ewlkj9bIZ_;p50=dA z1f(Um3C1QOID7ZTF3W3ay>9w3^o#_76)n}*T}ow;t%d$}oB7S5ZU?5vJ3#|7)={zV z^?@|>r!#E5bk}Nvc8UNeNv}(KKV<%4bVxMB5PQ7fwAghU{nX2-ba4IEJu z{;zpD+tJeQNf5VC@w}Qq#n8F!J;MoHgHI^jv)fEc9A1}F6_%@-9URB#++PoQV>Hm= zq8JUdCA`x2RMQ|ufowJDTEj+9Js=jTzxpjRN=sx!$yUQ1F+X``jl=$dtz2X0wsj z$)mcTk_$A2xaPP73ltDD!5NvhE$aj-LF+ZtL zOjM5cYHxqpTVz}gb-Z)Iu3&j?1hz{g8|1nuy2eHLnxqDR;^vb^RBsZG-a3UoTl!YF z2x?jn!`B(SAFW5QNeoPnx63c8AvJ7L=HbYbL-u=35uxcoPFBu~ZRN7Uv4Cjln&FOU zKfB= z^}> ztd^pjSgwTJ@!(7)mt47V1EY*nOFn)>JCB3nca_`UJU(t_&>`Zd7W|jrT*D=~r%-RM zAT0Mg)!sL)Xa=l_V%9viF%d>wp53|y2mXoT{-(0+d^JZI7C7%^#-o*D4D;}8oOb16 zgG9N1Ziurp`VBD%75d-%r6Dj=k5d;+N(ee*-drL%`qK;v$mT4XI=|&G%>*q*b)9OH z^PH1PTdS;d?uL!NmS2)#s8nmfUo$u<%Ib5Rj2(D_8MAqb-LpDN34xitueM+4)=hD{VnMymfedV!a&@LJ)8@iCCxV;rjZ-M^lgwtn@dY)YEhs}Y^KFiR z*PUmF30k|oWm$1*HM%$8`_lLkxz17NiJpW>lr|rZAYIH61P`AZyN*?h6RPeUw#S_j{N@)L` z)p$uy+U{>cvonKgE}n#;-<;$G^fz@u;gw!?z`L@+!Buxf&&-I$5|K3kUmI^t?y!T& z^QsQES#c53I*O^FY8C>?H7V}Ja5PW~4!%N#Nzf_uT_~1azh8~p!Pk>J;iyv>e0N}L zpk~hup5Z;_8}38x{kEZ5+o#WusZ7-F{d2M1b>^7G0+G$X#TQ>txFXj72a{Y>rttr7 zrpO2onwRBm)7?b&U{BSQF-oN|rb?mB-97-ebh{M4*d(0NXU8lU6?xIba-J!t)9$#Sp#DV%GOwXNUd=WL49{1qQtS=Pv_FO<6DY%ckLZPyU&3L-5ZNOdgvAcZa+o6x{&&yUi_pa3N zQTG>8W=7QRbM&Z4jP<2uzAkq|28BrH>$}u~V|Owhuhd4ehYPFg*k0ZZU$@7kwO}Cw zyEsOJNBa4BC*_&@XY3|+viom}$##Ag6a{0m!{V_{7lItN5A1jyH2l!{B1zT8C;#8k@0<)aI{_0%O{aP7GC=FAHI+{U(WU`e0XA9fSC^6 z$%5K$EeFNBq(dB%_AZUl!6sIT3S59woaf~zNErKAVxO3u9fB}%m~WLUTNAa#188KS zC41;!o9Db%H9Zq*_}PJ2OxQy2r~K2UT8UPT=#Yknp&Uz4&^}+zdGh4?uA*?EUk(k5 zV_HTO?sT&(YPMOdmLKQ&xnLky#Ls~D$y;4wJ4eNf-3juZx4C%Y%(2B; z;2&yoIDJak11qbW#HHikwy8mxVyBx0azU&wmm7yO&c?Snlh)VWfh7SNCZUh+d}r0X z-HzpnO>WO?_o%3pR47f*A@UuWsYEz1gDR8|oBM!^0#i48vOcjty9>W3%Vj*?BWNTa= z>&SG&dq~~Zn*9_qitB(RvCU}5^O(wF9GdD=+@4ySo=p zkk~o+awmg~GA;JPeAX);zgctl43Y;4(y!L~kV^Dw#=8~gd-DiNhlrRDx#QD&a}HjG z3prPt*8C>tv(WMdwX2$ZGDp(5ngZ1^LwkT!KqFq(c-67u8NK==E#%2#K@XJ>q$)Lp zI#-19GktIBJ#1;rnEqzUrhsLsZD-eOy>v+Zq8&UVK>7@x4(!Oi4p~fP{A`iZ6XmC4 zJ}EFhFkG^O)*7lZ=sFC=vQPVhAzx}SrrLGG(q099zL^U&hOi#Xf|)p*pdFKWrqUv{ zNbLPg)uP#Fev6p#h?}8XKzd=c$i@L?h%ppuzQh-LEHtn0?tDIb(y@rjtz5uLR_N3` zSLbGsf_2Q^Bu>NhNj!`PLDh{$go{g6C%Xu$n0o=f_AN~G^OgbeWqP|g^K1V^)`Vnn z#8Ag$qFSK6*?UVA@vq$~Nr4gch4+e4lV&zD%oDn`+Ui`fzr!Uk9CDKHIE2!i>`;s3 zk(LBr;MYHu;L09BnD5z7B5X(|m;76CoU7&SJ#ghkf)k)fm9;+cBBwd(QysSYCjV63 z(bfr3VnWnO$HuxBCln8ugiw-!ZF>S`kXWiZ&1#)h@04nF= z&CitgLZaEC=j!pqlH+2rs%@CA;I`_97~tu#)Yw*y!Ds6cxa1Dj>l;ptxC%|Kc+)w% z#;!6yXjVLfVkWgU%;Mn?nPVj9|0pc?3*P@3zo&PLE>qYp28gGXw5N59nnc~HPie(* z%YhxUE}ZF<3|5pSln*R;2Z$Vk-{p%-M(J1Ns{KuJlvgu*c7&1o)$gyC;PApAzT`ne za~$^ph)@`-tDl%%7&g360SaIkIDzGe=IjzD8@{xvh%0rrO5YG0=Y}T=`$dB3ciOcV z)&<7in}T|t0!%{If%7&FrC7@2f>&$?de*7$((XO&3JQ|@>vK!F*8~X!##qaEv*1K8 zSGtE&AXOeRCUuqipyWZl<1*9MBvOIWEz@>5Z5Jjy1YEndhG;2L1r-`Pv$|I(5|5^g z?fySQXW^HI{=WU+_v~yt3%AF1<~io<*)3;b2eZ3djM;(cS?L6kS=h4Kjh&#IirCI> zogx-07Gj}~|HS8YfA0Ib-fw@ZL~=nNaPCW?iG6WD($F$p!0L!`dUbPsS~KL`yXDId zg(1ls!+FtrTmfi1wzH)c(vlnyp4|3B1VUvVF!ktXq`_3gu$^1f9bNBWiaGv>s3K{s zN8%s-XUdWz0=k|v;5lk!*tp~=sZ)KLs?gA(Zl+E#f-G&`|Dy~0e7cthvI7tiJqTne zX=jtA-d?s#TbeQ^mtx~(rpL4(V4lHGzr>G5LpEB-UPFl%M-7(awuGtBmZ$Ik z^EfF3w)Dvg*IBUNlfxUP$Bne!Ha(m!B|_ZpLP>+mJXSO<1T<Z{H<=2&`yyKF87DvoF2XsZMHPVo-(+e zrS><~bS0|VY3`YKK{`pR9MgVt$NO}4R9XF0LJ?bpg-7}pe3)0q{WEPTRgMmkXla*; z)$-?=)Ss8o^PK2)Q-R+t>4|CC6fUg+cnr}vma<0aZ_@Ojg~eB66($p7&6)`idB z#H9A3C#|5lDc(l+1fzNPm7hi264zD+K4U;D;+G9GO(uC+St8{EL7~+fvNyA6HM$rw z2o2CCXtv>FGf_!{;TUvMyo>4CV0qqsdflK%6NRyL2|tPN)()2T1)mwhp4>FKTjTFq zc!&u9iKuvYlk!*RGRSZs`i331rZVcLliS&(Y<;n0isRR@&z|%zsKy^$#vhq5;s(A9 zQ>~geU4m-L;zEaNHJ*JPQ~_EGB(XG;IElnAS38)=BbIZ5AWwVWz5!HW0powOycNIUR3-klgYBWDu2aV%j*y`Ieao*JNlBkZ;2dl zJ0_r*LoyRRILMI*9_8Zx7Cq5zXTCw}{Hi|BS|e#|?(AtGOGDO6P#` zm~}@BX~^7q#i?pAyyN(9O!VOkES5u{$2Lh-L+toGoLxDF0}J2acipeGFIQ@f^?3%qG9Lo~LefE+i_KwyCwITqfM3 zslvsgKzDA}Ao(ZhXM{38(-gP|a4|i9OY`PZJA5$D84&U$f3(R^MyOGLnYg!F1!6ko zuWrx7qDMgG1dI+>oAGXKzuqp4#Iumw8jyc`EeCU+F|d@OLey>^gmv-LWh=2hd}Yy{ z&~;j;o>l+u`a0tmE*E1qpT0xOoXOZwbqD=Qgz8bG%5`5qyZd0b1^VdAZDmn4F7?#b zUJL>w{5Chyv|pYmhiBIew?tn>$b?bW1^hV}7fqr^k6yD`qn#vz+u{pSJ1Rk(cE+wN zgvi5NIpp_qeme1|y*HMcL_r{<>rxt~R}df=aw@*Bn{l*kvg6a)*f?vv%;`Hl6Cv)v zq(HosFRviE+Bj$${S$?t9}e}9I&5RJxLcLoiuegJK^pr65IKF zd0`Ul&tBkR%XaU$3r4-Wp;G&|?b5~*C7wI|5OldNx=H5}%p-k-Q|I|?ndldg47s+h zTPibzahA0{sh@mzyZRt`@zl9D)Oyz$pIgl3o%>ZPQnW8703w!^(N!vYk$WMj3sM~d1!CM%`^`3w|Srvnc7Qwgb5Kgm31>j1k%;OX#tX2iX z{n?qyqz#+_HcE%F?us0n9{n6fH2Fw=7o;uNX&p~CVb6!o|Fv@G#p8E~; zwmGNf=J_n6Q(g0atLm&?FPxJ#i*_|u%WjI3ej+lX?U1J_No`q;ieXr8mDzZpT`XjD z1I%~|G4YySSky7wz(v?HTF&iTvSc>9V8f7?idSsMNTgdCE5yEW2k+8Ld|%&oy>eE3j5CMAmS;|k3`z#r6$M;sC+S`w6mc1|wAzgv;NPY2zV8qXt z;D!$414?#DlB3JA{hfv@+vcWjHKCM&Ag+6&oaq>ko)JkIiK4ib$Z59O$=aAf4oV@a z#7~ZHB7YLsKNu-)a+^kjcpwKM5t_uCY-TktzkL0^PLI!n`jNUz4~DC><`|2-CBz?H z+b)H!i9A>swcx(ebvJ7J7Oi_{&*etwx_qA7d-miRQbZvj5ApJ46lkK$wZM#3(@NrK z;ifad??i0h&QMk191>*lhH!qE2iTgPw)IHuR@`pJZW-1-Qu&j*BgXE&-LnTZ%oBB@ zR6knt{3;I=9&>ihh$m`je8YW+&3>U?GX3L<%+i!xSDS0c?qOygf&QH9&S;5PJ(eve ze=6>XF>p1yWuzol&RitD(Pa%kuQTfNv4+}$Y{Kgsd@oXzO*!Hl(@dk{17{Dd?Njy; z55n!DSG;19jv8)Z6AiHX|F^OrH2wuY#m$SA%W3uv8gS0Khn-BdJCg2d6iEHY9o|9$t3*I2Sva1o08_G7e0`36#Zb?jP|k|7z5G4WrVe^Wa8lOWV`cZww%158=vJ&sK^9Y^RVS3?j z{Ng;up&T_2c&c%3T z`4djZMZt6Dw%)W9cvLu(1J`rANtOq-BL@?4!k1yZFXsl7U=u@#ft|jIRFy}%BOW;v zGt(~h;*bNph2I=NaScz7Fya08r@*$d-*8*fzmO61!MyEC)^7JPFBB&~5?awH4Si@j zQDibr+;@l$IX~)k`qw}EtjT$t>{j41$1);kv=1Rw1w?uTCG}_FUw#jpnx1nqm(;85 z?KxJee)R9j+?&1h$n;(CdCWL|2M-V(B-uI)koh9dCTUUv>-Zu}H|)x(Yd+0@H|~J>SozghaGyC2#j?Zt@zkdQfE(yNs7+m*%5v znMl3kAoi<2mX9-MX_gux3ff+Nnt{38Gv}<6IP738J#&~qF<4jBu&rK79GEuU$k%FR zHuuThZmM6~(E>kTxR!T!WZ(x)k|A;|qRQiYkwJU>QOlKc+t`I9Ts4^2ex+fww#1d8 z(QeaqMTI%zVnt7m2M7rdZ)>c|kJ;*bbJ!E3pE6EK1_}T`mC)D4`4yo=ui_ukYJH2e z;X6qUR;ZoI6L5-04aJs~Ru8Tf3*<31$?$0Cw3AEpws_{&FLJF!uNR}2zn*ut3wyI7 zTd`A`rIxt*k$PKEiw^_SZ=H>nK1W?{g23Q#PMcOP*>We+~06 z`Azx_xiY5Zeks8d{H$&@Y6XYEa7J#N7P-mSt=0XW{rJqqlSVG0gmv7rGp%&u3fD8h zR$c|Pf)n3~pY8{g2vmV%cyU~4m* zNaWYTou+au^V!~x_4IyqpU@5UHPx}RXp|cxWw;b5XmVd*%4kjjdWG_aqB^(G^6>-2 z?~_3*4@) zTIWVUHUWb9@8yWG;KoqCCE#?L6H)EH!ozw=`gfo_Xxx~d#XZ_guYtnhyP(+_Xp*L& z_{c)tkfJI6I%aHJ;Ne3U!(TNrRn|LeYJ$XXNTJ%}Xk*u<;hP~B$)2b~ix|nO>EyyP z(vA_|zefb#%38fth(E*#`iJMYDqr6snhjP3&m? z5CuyfFW-hXv)^6%+pr*pOYY9G8#Y2OCS z{B6pmA-Za3{AL<(|8AysjxJ-bbF#c#mHi5NycUmsLz&XC5#N1r{CTe-GT@rm9H;c< zTRD@*B&t!inf-;%!dWyKe4}+l*6g<7zH8L$%i6S;>bWo7T)&1fU*b>5Jx>nK-6~Q& z|I^VWc|d|*XDn>~tg?3OMw0?_SP;5T2wHCNULj`@F%-D~$b%5OnV{*O7pxnrK_tbe_<*?Vq<#CSUFn_?f);vMD3g410E>-wLt8OYW`bnP-r;+kEY_1h_JYDXq zLf0Pg%-&aK7Tz2=g;j@!`lcR~;cRTq<&t}lH^VjM*?L*nk0$G>nX`xv_J3P7adIIJ zat{K~vf|7%mqfiD8@9d$q2@0Muj#l!j_G}C<}py;q;?@iN{r3z_6&%PdeV zT6?0Tj_TUm&Gws2RzUn|&n>!JhIN$O2(wON`{N?3-}8*qGY zSI#T&vceVLR+WG{znrl}tKW=V=1jP`Am`$w&HP>Tz$CTq&}AaH8mI${MmsHc_nQ3^ z-~9`i9rmT2U)1DYmYW$q_gPpAmFIl3-XNteT@|mfd<6?H2cZYw`=TOPlqP<`#_ zjGa>mH~9FxC6ECK;LMOV`r(!f2FZs zdV2bML?eBWrabDZ6D@eyn{F7+$hGD! z3ITLS2eMl1Kx z)?_M&ivd8=9ks@~Crp|3SAretA?kD1sER<^_5q<^o*MoDwrd#QO5I`LO+uni)zXGKwvS5Zx}n+7JmoA!FH9h zBdSZ*S}KEeMGl-_FlFzOB)lJGE!0*Asn@9EN6rY3PlNSA?_a=Gn0L!yw#2udto9-~ z>-E)tR8NX9X-4pdn8WI`ePjHE+vCk&{z{PlDP}vN^$9Osw)16H?S2zh<21V9w`sAk zpdLT{XII|er`(qvhv&7D2YL1L(*oxAxTXkF+b)ddkyTM0(m>yf8sVK^T}mRI$hOwq z^6?`cg=gaZq)xWop6|cyGtL_8YvJp{BgaeLY9V>NDAtm1Gz-CDlYk=n#vH|`#1p{t zW7|LDQy|h|cCGHH1)Yzftg<8gXe_Ez*s`h3IJ2Cp`??KtG9@&yi?3kef;zA^v`d$X zA2-FcR7yS-Cn>|=zWbE<%|Hps{-w8mh69G_f|teThk(!jGmFyITef=VBc0?9hMIKg z*F07G517+A++9@rG@9T0)#9A|(+3-*Rs^ozs^D7Zh)a74BSd3E(q7nI^=jFV4_k#j zqGC(;WZFn-4<4z0RV$hnjQU*G63(cIE3QyR4k_#Ahc4qP+VD;K4vX70wNYv6S)!lI%m#w~qn*^@k#lAF*My9U9iJQ8G2JavMxy@Hk2`?P{ z;87|XU4#l*O&FJXt~FT7 z#K{g*X_V^rKsP3uvE;gUtPhyHPJE@Ehqi7z3)=e=m;*PFD8<8=ZQ)+G22&8nu3d%& z<|4%p8Ya=9p}fNJFeQHm6RUf4=l~FS{tn_=8=lb=&$^pku6K94G3qdhEasj6m12Lt zsSSY7DES>pW(SDQ2R7EUd;T_84UHc~L^xu5uzC*U9I{r~757e($jhoI-ZfeDih+qF54TqK--|Qq?AkWxyK_7@sqe{yD^AA+y zkGZn`K#nOYDFJFEoC+hr2t)y$BIKbGHG8U9L z8G3r;-!GYPp;2kRBI3FtL2~sg=+yq+hEQ7TBgl4?(q%tCvq)WlUp%LCf%XWr_ufX5 zoA+Or-!{(I%cXS&Yfe>+E_<-H*X$2E9!)u<$4ZQHOv4rGY9A^WR&;);toMWMf!gU@ z;M~=<)FPllirpEmjkj>GtAwjfoCyuPPVthyD3IrlY}g>JzcYc@wy=>i&oA855;f(XjY` z5&?3fjo;mKsS;`kb)o&mpT7;$*m_k&2J*%mg(o3&jO6dP{4d zlSb%-R}J*kYZu@iR*LG?xKa~(_7kCI9vPr_YNJIT=Qu~r|6O{mx2jX{CB4`y&unnE zgwEmXSjVtSzfG#=q`b&2Dy!ir)}T49^Ut5=L0M1XW|5D@r%ua6@keuQL(0Fe%unH+ z9YT8(&cr~4fV!yuVDRBub86Z)4X|;;aLJRSZ#;nPi0(gN7&;v%Jllp*okhqzZ!bC1 zXRBX`1#4Sb6sEkW5KFoi8~LDLQ~szsoHmbt&}Ng-leZn~kW#FJ2`dn7a0xN_>&gK& z+3Urcb~iMfCxv^ycFqmg5-xnZNUH0!L~6Lge}n0>fyw1T@410;(lOA5)Mk+} zK8OzQ*@0^B@Tq`zo#m-rDQ$9^s2;=A*G==`@yn)lYGo`nmW$-0jkD4Y_a{`-N0x=W z^50-XmNG8p5W{~1!>f=j% z_TP~v-bfsKca&hva?{{?kLbf(=iR1-D=cV$5LTfOe^UZYC&Zd=f}-lfg$X;qV;`TCB;W!ovJS?>#_#&!ENcuuY%tSINO=U?5j~ zdrNBA9MfMIeAbQpX!FT>$4o*?p?Hc$r=`7s@8@{k2wPL0*9xqgW>>$asJ<-`UQxik zwASR-kJi{G*lcPEN6GBE+1^maSm*+?c$6#e*HOEfnViG)_Nd?;QvJ7*`TSo^i<87T z7b6E7%ht68I;>fg`7QEld}jZb!YTfN#&)qtVa(%=|9#ctmz0;@&R&S}&)g6feMRQ6 zznfP(dNW|G0Zw#2=gzTY-YNjYVRSO?)*0y)FqzJqG|RpEGDa@8lI)SNIv|~b*Ua<- z{k7&ahumOOru)r@7(DfrztzR-9VX* zjXR@&OvxZ0PP->ONvPwJK8Rtn^cKBW_ROh&Gh~@KKgW?mH z3&$M8PsX-!O8?l=NSuyK_yxO`H^pj-YlGx5HP?yB?AR4G%W=VMfhy!RcbdtGGnWs$ zacjAk$F<26i<@9qh_&_(?(Pe3&hv#gEkPE4UC0@}8ec4JUZG#Dw{%AeKgxF6Pzs;{ z9T=@^)hFL81P2F|W-1L|9yqYY98M+(#5-Jpfh9ismAWAPRQ2^b^``81qtBoIlbgE7 z$)We+V83>~s6SBqlwh``YmyduxzW;Lrs7Y?2RYg|o$L2%ZJ654I5G|_%+j}}ni$e~;p+oZ-!=VhGn ztj#i035Hj5D7R*viLJkhAL!a`9&jmQ>AM*nyAl!yO@Ns1-TVe|X(e266btcQyR_vs zlRzV56%uWZQ(6W;5#q=74?ij(A(aJvp3G_a;W-UP{UPJ`jrlvC7R`f2U~Ayou|z)k zng(c(o$0OvD2^()=Nb{93LX%Yj=Z)l!CWnT=~cf^%yh|PSg#iahx)GQvS}Ekg_HPv zzx8yUgre>cbh0)g_CsTBcjfn$Cu`b}Z5%t14@;W*mE-wf1aaS!r9@p6@a!Lp~d(bXDoz05dRuo?vb z6Rk@1oky#@P<#v7FU0u_I6U(^L!OE)x2Cz@y)Hi8_o%y`&IAj~M3O_)j|y6T*T9Z9 zBE{0*Lp`pFsD{}k{ts0QKQuDl6IO!g1s?qpK8alo2c@}JVM-bv z7skz4^Qu$j)g>)x;q3~0 z#_6}pJneljp`GzE)SZ!jrw+=@p{?{IIp#DkuHtRDKT?arHm%WVuCT`xwZeYRZEC_s z_3dsFkh&z!D)zMQ7NGl%DGOm^e17JZ@$+M+ZgUqCKl<^KlhO3*Ubc7xJr5Vazo88e z#mpwnxcAqvB7o_e@zR~5-R^6d?Pawfqd)vxs zndD5qqir%OIkl)($1~jmiV+&j-J1xnN^l9U(vPTCHrT%fT;+^c2HxjV^6xyUef!Sk z8qPP2AlgI08<0Krx2JY2(L#xObu+`uD#%kCq8s0*#OF-JBINIiVe9c6)y=dMteJ@F z^N|kKkM%G77qkn_*5ZF&CkxdTujb|V4oW4dB*Ya`Ghz=Xq74?&oHe1Sph1xRu}tsr z;`I0R;(&^!qDuw3miUw4iv$Sx%%9cmAFtwN)bUJ=*k$={6;Ow_R|zDD%iU22Xc)JP zD`@;7f9PZ~ow`Et2E)&e)mv&RX&kxJ<^n zMMa;XCP{L{FXRMMbqjX@+3V%^0Ge-SW@fevWI86dQskrAL+~rSmW2l*v&`u5g+!p; zRWZ3dV@&}HO*m(D-QH6#R$BrmHdTC?9N03qAsCM@N$QYxK-ny$OSoxRrfk35yEWQD zE$MQ}uV|0cZReO6is3*2OXGa2NyTGIiZewEaq6fl-4iCY>g{`4@1Oj zaGzhi8}DV#i0}(yEc0h>X8vZyJ>9`T<+Xcz57@8uNc7$%n7!kWSgnmn_)J%Y?eN6UGoDXRxH5QaH-FEx%=9 zI_%KuA5~xOdrgTKrWSOOCInphlHF{TAcFMZ+04`$kzK$k2^zKJyaaGq7OnuD?!JhA z4akibX`nYmKUvg@%3d8s^$~z>rLU$`5UJMB% zxTHf_P$TwLxWANw*Ie4%2>66`U`x^4d5p(nvf^DjrT$khfQns#Bve|PYF{|O4j>|w zD$T^?p~D#eaxo^Ja@Al%_6ijzqGEM-K((oJ%+IFIG`WwbbN9rDq0o^b@OXI3^G5^6 z%%Q`jw5!zj+tDi654oK- z0Y?)v(1Gyk2w!$zvhT8TKQ~NbLak|a(a}n?=x3pNT*Q0u$+;*VfmBTkI8&>dQs4CszyB+Iz*|im{J*VFwX@oJevY3$q8`$rcn;z>*|4&OwG~;qc@fq9e|myZl>aaJaxwguU)WL*Cctra^Sb!8G2`W zL>jrE>hbLS1t6s)(Y9dL(ey5_fI|30%!F${i=`w6no~ZA2O7c_EJVppfM~|&7xSW<$v@OuF<56z^UjGq zpIu*!ua7GM7N~0v*1Y5yGIlwHW-DE6n#_DkYk3Uct2WQ+`yQR9#hf~=_Pj^9vpkp} zW0Y-?^Vg|~#ri=?vBt!1jWQ3sGdDC-AQ=#h+YqV7SQ?+cmK|zDzdV%}UE5jc#X8nj z-H5+8ykKV5Xo?vXb`OQbVGeEj;ulVVlO8cc*>g*lGonvxk9M8k6lqS0!c9K+M9%V% z5h|GCS-F>;=GO|%%*FI@1BbTYwISx7Dq01+e&2_|Jxk&V$aNL7k}N6O(Vt~LgD$S0 zNtmF9%@XU>73Z{?T4f|dgrPCtCTL@&HK1|-=Id|O)e9tZ=2#BvD;6y-e6q~ITK}y% z{=faTIsEumt+*`kx@c)@wt)MHOJLl?96DWPWyh3}u4KUEWTJSU#j|p9=Ca~#9Q(4K zAOLtx4r4)LDQ_qQ!O?%@kI1)NzyrFw__S|qP&sJZyMu*{1{ z;sk@7w2VeXRr zp5T}k{K;eDjOQQKIF#@+?pzKhg~IJ@*h^>5`(J&@kVKm+J-y{Bc7VfwI-QUGew*2F zXk^3Muxz`ncDKE}PDtv9j=aI~+jILm%E)bx_{4y3-ypM-#B!ABxzzl4GTf*mo@5kf zz>6U-ZWy&?3bh(2*e&_2-1$OdW1D{gFAUAlC^K=f$DIS*eD@f$<_oXxzb{E^2;3=f zVDpI0GB}GK=9Wl`fZISK!Uy#DJ47 zywk}cq;pZob+gbgQ-~p(zveFghVB$<9gQBlZqd=bPK~ZsPOg$Dqzi?xbWdZdX~=SKd;lC@2B zH`T!5l@;q}@LwHr^L8JW20Yxz%_^UclOJUD{1EU1Ns^~nEx)ytsi+^y$)yWBKnWnt zXn(;FI&?=RR`F zg*pSnhHvxseNzx2FKG;d8ezatI&fT%7it%b^$?M15-|6n6zs(SSHDF*V*hZl)*a(i zM3xaumaZbrA_7$s$`7)!`~`@r5kT6o+E7g%Sdmmg_WG3g$gorOK?)6FuWFz|t}hM{ zbnqtVVE=w^e!6!dNuCHiTITC>irD*{J3RIc999Hp1iL6QDL_aA z=EaG-5!khzxf@5GsLy31&XDJ1i!^Qq3X*gR@xLzlo}CK>1UOD`x{!@*3{nQd%c6_E z(@fKoI6ezB{_dOFoMI;x>>=bryYQ9gQ7{X;bOI(e|?{DY&df>C;P zEx#i`S^d^QoC*fV9rwTGs5V~6opKG%hmq(x&l`;0D~5tywi0b42RH(zA|k+7&*^K! zWyR$wDi$_uCmdRM2DaQfNxMQrZsmqO?DHv%#@LNAz0}(jV#T4+a?k*HWKiOCt%PZ7 z*oTjRz-=^%B3FMbRl`!0XU&86K#ui|dW?PU1d@J!aC^&kTtk0xz`FpE1|c`Ku7QkL ztShW*oXTP`kq6~?%F0}26Q32ai3JvwM2k+8r)to(n`N+mn?T!S5B8V6+3vEQeZF*< zm8C+o^SS*UTU$p@7gb`YixP~jM4DJ&2!8@qO*fPP+IaK6IC{>lk=Tye-24h;IO}U- zE-JY3n!_zjN3mLB0L>@)lC_7@0V#+-JhN1ht@5<%k~1>D6mKaew%WwWIOPo%kdW1T z!>y2t_U48#$OE@OSg#SeT7_i#Zf>oNQWpw_E*F7Yd%+gdVFNf3eWA8oH9IEAQzv3? zol0;=`=S9~_g*n_^Xry6ye_b2j&(ComAX6FEYbbnoJ{=5+q^&H!->5B1jhhqNtf?f zH?O?14B(O4P)^RaU^qbn3_ngMzA^LL0lSa5Zv_kj&*$A*U&Z+UJSuiz&EB0%M(!0} zT{Dm7<=6TPpGk``e99<)k`o^>cUC)=(ZS!?32%rX$A?$Nj5b0oylPY8gi%pRYYK2_ zlYgV>Zbq6^pS)(9dJZC4cd7L%0d?~<1!!6_v_RB{(p=-K%j}#y)uIlu{qeD9rA1m& zT1IU0dhM&?$`s*VZ=OOMPWOso1l>o-n3w%B!IO4V7F#9`5li%a9=U7ky$rfcG@dUk zGNSr2J$=^ac(xZOL$%;9k9nUSW}R_vb?-blG`acNuptNDZ8<$J?it#3|gZwHZ!pjG~@bypty#a>YC%EDLdQxwCOq}#`Ms>NNJ{FxwOAm(7< zHk=V&aVb5j5vt;yM^QDg6G$pHP#!6mbebSb9mpl2L;Nq~AQ zAV$0GoHy9d1Vt0cMspua{_5*70lIs6UEAu9le2odZ$(BDgKu!Th>8B(Co|UJ6r*e|kU?~LPt%J%(3A6(hkFz>+E_8;Q*6(qVC6^iV}}dD zrOcu58DhHN$yy38q_(>QC#8V6Q?X!|R|zgBbSEmCQ#0<@bIZnGM(WFqM#AVcoPC!q z%+@6KKR+0XPI&B9$a7qYJBfL`AkIz5W|2evUrh)9if(Q12*0G=InHSDZ9yc}tjJy( z&8&!sfA19I`Z2LABI&vuV@6kD;VpB+XT}2-jdV=agbjaf6i&CZe7ll@Nwak6zqJL} zI|SxUcf@`WtlHUTGJOvYP+K714+nE;&Yn8YF&yF-8V*5j^XK!Fs;tT!&DKkiECTq z*nb>~J$Lw+!kU|McUv#|p0kAELNjw%;UTujw@)uqQdq!i!o7?La+!r+t%`b^ zDTO_h=s*51vsRCbc%UUP6(0Y(SpMpAZ3~PF ze@N`C{UH4oek9sm*E@O@I)CqjATU`~aJxvG$9lR;HlW2#Nc=U)EH(VMR>DNPRPcAA z%{@cyj)o!Vf5+M4AC<3eHpMeKHE)mg%CA}P9R?x{B+Wd$hNAsX%a$UxK?C~B)$WOX zvV)i`>G+_+99<+a^Kggu=QgEH|ItXu#bI5jv7oZ#SWI~N!`YNxjTgw;DAtSh3Z6-; zwD2xmfo#i)plDEGsuCi9v={=Uh=wDoZi+e@9C#N|)wA+#Dr~64%CdG8;}QJmpbt;~ zJf9v+YJHegmIq@SE#I=;Fn2@KzbSaW=M4`)AsL!J>LroLeLqEe5xtLFg{Vt3z5)A3v_PK5!n|=sXV3vi8i0 zBdu6-cO7$S^&Ty-dL`>6`@d}0Nwb))E!Oc49H9WrtCwlH8UfSC>|chpl)NF^Z4q4S z@wH*>Q(1y=8B!!#hKTUOc)OE=NH}$k$ljXT$0F+;QM=0819DF$p3GJ62Ju)1XPnC2 z6jZU`xYmUaV+(75x_Z;?9+nF2T7+8Yql_yv)8Sq?ryq~cyny&o1$R*38o!!3t=Dhv zLDoxi`^7@kQeYmL@iSE&WnXvD>_6P>?&4E}7f^F$=s~z~XX6oAO0UST@%GX0uxKyq zb7n1QHDDIi=Hoqjg|Az{K)Z$+2c<7rv`^TW-Br#0aYIntEo%}U8?*L4yqW@N!eh*A zHVePveg?qvk9yGbH|41_Gx7;6MH8Xfa(8=9>=~aF0g9)3EEt>IGbq-A_EOP%MnnnH z$C{GzF)UbT^T#w{iZZihB37YqpYL^(Kd~ngzw5=^>-1XJK4EMy(zmfVnwo3=&p+<_ z<9+Es?o<=r0RondW^2}%5f)Q;KJ|7b6?8Qd;2Dm733+Rurn4?YPJDs5Dp30?Y1Mw| zp6s%i!}lg{9uCeSnX-;n;R`;?of6vh)2_Of;bY^}7mp1VO82qy8XdNic$tH0{8oh+ zB(?fYvDSFIaP3vogY^*gxamxRLmlso4$@-SbXsPi^;#dYWEYxK^SsI#GYW8rd^D3j zB7P2h>tG5&2OQK&b;A}m&5&AkqZo=#J#1e0khADPMWvuOWaZ3d%m%^Hj}WJIFGT{~ zE=#=zwKsbgHtR7kFv2FxIPRzv-?9?(kk7Pqa=G{W`w{KwCU-FjM&ucrLTo!u$K09V zJTC5!)rCCt{(W7rF)IT0c}+p!|L=r*XPbq>)vuGSu(%MI4+v8_zHL_cO4xB)*gUQi z1O*TKg&Rag`u5(20VdQ!0u`$Af?|i&1tjEvc3(rXJ=5!y?<(YPPGwHfNg(=<`LB8? z8#_$6*J1K`vhb{dX&J;FFTcQBt+1k~i2ZzNME|r@zFwYs88(vMGI7Z2n)-DlrgnXr zT{bB_EQYCZt?7YsPKSUVpV-$%$*R+$q`ZIyfu(XUlcxSoOL;L7>l5R2GU%&Xdp6FK z@CDmY+i{YL)Zx9IxrQQu->0lPEE+boa=v4)K^HM2qj&p#YrXkB@KXcAet`ZRLLDn%3nE0?n#dFwrtmJC$Ck4>9>oX z;<>NvMZ2vn%xIrn7ZvOQ0>e)E{E|rq_8flYzbeIn-$iZNK4*#gB+p{i1>_zmo= zbdop0{+B76g``VK=KgFn+n{9Yw<2v@W?ZD3dtLB6Km_iUY2D#bd1=4-VBd~?=CA|) zF|N*I_i(>~EI=7?uKY%OTITsx_6adk0qyoddr6KW<-1nKb~3u1$Y+})x}##m5}Fj; z)koMfS-?kMU-z7?UpHH|L6FCj6oj_#4IjD5EQSjF-coDD5 zWoPce=(nY8IXjdDAhNk zBVOG1tWc3sKwmrgWUB*7g)g<5OTq55v3C(M;*4vqb5l}$!ECVt6VOYZUy*2c!BHZy zBcmZgkM4qx=rjo_-ftTR1b@HD*3OU@Zo}j4q!VnN(#;nhu#~a;Gj+_HZ>gpC?!0az ztdhMMv}?)?UW-v!o^N8Gul9PKRE%JMWHozPJF zCma|dqni_L5{c2$iHl6~NxZUlQO}%Q*R6Qm)kHU;wCkH3n)FkbMIhi{&4uFZ@WzYz zG~`(GgZzG!Hsh#SdqYj7z=LQ?l;CQwb7qO$@0c;vDQ8$BjmLi5{Ax{c!1*2nqMrtHOUbxQ`WnL-mmuQO6UpW%ikBZA4Vy^N(6&Inv|K(fW z+?tHSI#$&><;c}^FxmWCVAy)AVnV?H!d+cqhqP&~9vr_d{dJ>PIXGM0g0PDR`D%uy zKuLm)9ihZ{y^N*@Fv`m7zzdX*Sl#5*`rFLj3qhbrl8dQTV}M_Znl9-+b9`1yml}TO zRIQoJCe+9b=v4m-2yTt_v?tuMu?>_LXhn*ag)NL(N@AtSy8)5 zQX*h&2ideEw^XmEFl|?q_m%QeYNx9&F)?IKZTBlJg7yIF_)(`hPgBRyfW|hOH2R@3 zraEYpnb-6gTTv}?p2&yZafoR$HHsb9J!vw#yx40+E@UZONQ3!3e$V2EWhPF5q7waq z_E+AM>^RmRmesYUcf1zDWZoS|TNPdq?!7(w^%58=BeT9Ar5)^@=`7@v)miCxdSos% zX1P$}U}H#`hG7XknQK<}k8!NeMa)_*{N8)!zE4I(8a*^h11#c1&+K*8_Y_$Z^e;01 znb5tSV8*w;^7~wlG=?~3r{ML_N96*K(fWNAxGIDOVcl}+M`9%MWOj`3Pqs#KQY3E9 z$DWy1TrH=$Z;Z5z;|h5b6zb4?GHJ|`uHeI6=69y&E+(Y3>)TY*ZQ+&NC>6BBSp=zVWc8--RTnIwl+9fqKB(WEe zCAREMr7fqXwwHqnO$+RoK7YmEyl@nX_77KBsnSkgvKJ(d0yyV1Ie7?e6l=XP`b%>9 zad21we$!gKnxhf}J^M*ll#5@Lmk^mY0aVPk3n}8S4sag$kWoq265A!*Pi6H{)U(|gsxF@RM=k3H1|o+!9v^^gcpp3S302|1Moxzp(t*)n2-WXj--7at z&7P+7%`=w2DhNbcb%ceURW`^u*&HBjXB)@r{MO#-gSn;}ta=wSsFYk&D_cWZjx^;^ z*_L3PVK^+=;!6}(+h=a0jtSy7XG2hbj}i2zgl`Yb>Ng0vx=v)}wz)%_Qo$UYh!Ia< z7HwIFbkU+Xl`~XsEFI&wH5?PyX7M~A_n);3NQUV|S%ZufIdA?~$J+WNE+I-C8pbjC zMyGYmZZ$LFh@YTQHQO7b70Guy1)Y)x7j;1vH^OmI+~`Ls@e)#lHI`QT;N z2+WyDQ@^M!)k?wa;^VW$GRS*3kx?iIXhP`SqUEAypolT*L4%zYPjQ}vP1E>y?MB{d z`_U=-J%pfyCq?ZLd5v#3&R-jU7L5*QC!_m~rwZ|^dM1;xl{IL&fYY=){W}KauD5v2 z!R>%a$nAg2$!Yqnzo$`$pGRCyg#>frDs)ZW#+-lj3Ug0A)KG?Q}B_- zAOAt_p3lQ?HE0J4B5=TYpS7jX0(FmxoL_BT-YYuu+Z#4?P2f@Xu(U_YJ}Xuq?t7d< z0Ukp`Xvn&pR;A0%7qkiNHH2NdVuV|udCcI80Mv$Gv2nS}!{WIPf=${bw7KxzeWsJ=OnNsS|k`Odj@KqYUDwK4Sk zCo9?#5}T*{l`9>HP>DX3LOW>A&2odvh7izuvR+uhCP3_NLh@W8Ps*TKHvi#&ln{-| z!;$V@P1-=<+hSyv?1-@X3(Wg%Ifaec<9#_!Cx`a zQ@H7_P{}uq4idrfsxlwpIL zWb$lp;)ZUM+nywlWfFa-+FwL9X*_}@Vvx%kXj#*3Rq$x`lxa+sB)qsIJgvqfBd19< zD4tCm@#jbMX1Wcwk?oD)?xsnUMT*Zus8fgJ1;Z`WTM)~|z`a&w2-KkW9VB%h`W^4v*MAe9^P_vbW?6x2Mcu24V zk4FwnK8=R3SL~io>;Gi(spfCWR=>Bg%-ECyG`aX~wv0rajUp`IKlR;cZKiK4E*($C z1O;x+R$s1cDELtnVxFoIA`mi-#Ue`)f$OdYZ_95meu<^TGx~uDiWUs z5~EDJLyNy-Nz*l|feQgMm$$ToxgR0_L`qAI&Qw%giO$jZ4_OU3?*itpTEBwzEP}i2 z*@QO3>q33xgt$QOc^cuco6i%eXJVgo9*Wj*Y3y^FueZ@ffQX12x6nUTR7L&ht05lJqt zzXlx)PR|#~YWVI!U}Ugv?!#A;kJ6;aMN{VuxEOlBy5#$H#fQApjs98?&I=bhS1>Z< zZMa3fH2&~I?2BS+#rzVN7vB|K9nk5}D$Q8qNCiMg5by)pS++HLYPi89{$GSyx&EC=AWxGX+6$KM<;*92d*D#MuwAy}JvEbGdO zQ<26@6bxZ)MVrvSSgq(+I?d9V{#<#bV^i}srY0k&{9Pz16>Z-A7vA74;$d#A(p(-u zSF^biv&-Vle!W_C5qB@>o#0&uTA}&;KeV{An<{I51>)@mgB%^V3{7;zJ2*~^E-EVI zei*z)F*ih8;XwoSq>(pQZbSqw9+bn~tB_K6SVo)Nc~)~YN1CDQetmhi*gw2)6ER&; znIFT+DzUG`>ONU<Az8J25!O*E;DxADp8Y3_Syf^v+SS8u_tBhpY+vYZ&q`f@Cc5%bGxT5ty2Kc;M_%k z((@x7)kyc(5MOVgLqVuh;*(cc^V9_IbcAwj%-C+^yPEekh^gc<)+j_cNPG&S` zP1+~XoroJDd0S!>o9jI0`?+a!iPWD-TMK>;ADjv9YIh%A3J=$&>7SM8n{E+FbB+`X zYk!{X+ykv1V`2>aZdh$gjAL17M;5mKbT@G}XsW*m^Dmu3GDMwRU7+33=#y;0aIQGS zSD%;Jn!uEK$$RFAO+Zzl&RjqT^O*MunZJJI%WGbwhu(SgGSM>9=-qU{w?XI#DEojG z7|ntS%|fsO8j|=2tcyxndL4K|za}S*Etq6J-iYy$?D<);vt=_$;GS6{@b4lh6V^xN zx}AtKLO_pVhqb@V9hb;iUlIFL@f4P5ut_L` z85WcLVxk#5uDe>QO)rg5|9pl-aH~#k)siPj)3*DAA+W1BzC41DbbTb!ocNi+&2uyw z3aQ&l=*y(E^o{m9#7a~d z*|^QT{9KK!0;9HLIX$ioM3~ExM-D{3AFFRu9KINEYEbzAT-GK9O_&qVE~$1>jpi5O zQ+Oe-8NNMHU;SS8&VJGt;_8=SjV+^BMTGh{OzDiwC%z+>nRl}eP#q%q=_Y`>Vmv27f%{Whd$#X zs49VQigxX-^v36)L|Nm<{cKmC?pCp~NAhLHe>W_G(x&}|x&p8XAiw{;PG3_oKJ!PN zoCE~ccnTkHR+^#Y%jGc{rVXrQ`=6R@`rB1~iL|zE^b-E$^xiM8MssuJzE~NfKrSBT zN5h_qe%cHBPyAeCb_-)696$3OD^$9Q2>lA0$|Q>$7n?L_g^8-Y=67Y7M`6lBjci6q z9X#$s)(fE7?iTmBUb*JJCf`KN{9`$b0jP*|l#%b|-#+)dt9!ogO)_#vKvrS#?)Cad zm+X&9FJt{-HZ5B3Gq?3q21%d44UD@fyvqwso0@`AdVXwG+(BG}ZTXn&S-T zG+5EGwp?y9FA@$mwY0VWTo4}|S8A+U|LO2EDLjz%whpw5&Nu6lQ-Le>CZVi4o4kC? zFUq`_-Bj!#aFLrj)bPZfVgd1;`?N|}&lIoX;NfJg_6|Z~jJ9hE8$4tuG^JoK3cDF= z?OcopS>AJ(@kUAI%o7E+K?ki*}tiCj|w;zD)Db}n&omFH~s(KJ8HG3BP> zANuObv4;Go3pIuUBbVtP^QJ2Txays=rqYJhlG2OT^il&~gUm}A9m@7vjm|%e1pbcC z3I(pG1SbZ0{&@lqjeCDA0#W(&Lvy#~t?NR?RCl4;FVf0>UG4{_F3++FncdCIrO3pV zS};sdhLy!<>^VY6gqC(@X1#GF_BrdDO?8|*h%eJ3kZ#?&vhvG;^3UdKT)3eX%=cRL zn{sU%2ED@6N2o?KwH0)~F;_jdfb_X-H)FL(Rfmofwl_FE@e^|HrPg?1Z6qDK`_vt~ zQx@twmu7C>N_1IR5fwQR;A~avEskDM2*?7MQ3ub(Wfl9Dw8h>|w?qJ5q{wb^hHy-@ zD&vA*S>E`%qZhLwb=HWZl?^Jf3bdyG)zR@!q`Yw*GXkE@TDTWuRpusU$Z?%^+-4s-smT@}w|>E0e)E65ithnvs35q*Z2KZ4@Mr>Fn&;I(X;#HcEWr^(9;d_r>QUkrn1|OaGDXm9P`Amgp+e%!VNL#$3g>|(x z=5hxX>^Bs0;&$lXCk>meo;zjIA-+`2c&Xheh2Rk3S*Av))j#`rTz7wl#1d!TI75SQ z3RFbr>}hILw9r!LaI@U8d;mDaLy?tHu>K917RBSou>%));}o7pei&sKIjq}O258Pv z-U`D!mxc(zBRM>7i}6$rBqQSbL!A$Z>V%hC*-~_y#cLVo`7(T$QITl?y47B>5fj`t z@~3h4`J7cI?~=*TH?sSWfRs9}@Mfh-?t4;ft*~9~7wgT7ciDA%-oEcr$L06xg)bGT z_FwA@ohjYdF6TSQKUX`kCzLHMiXwJ9E4?L=A!Za_P8HKnmm=7p%#e}IDG6GBR_ zKeta=_!+wN^tVjqj4AiM?We8q9@jc9U`BPluqgtT5K8!XBJ|rq_L_>yMzl>)Vyewv zkqBqH7WOoVd|6$2>@9p8VZZoJm4Y?ajQZDF=uWtGToECD#l>;C#^K0s$q8;bGa)E< zX?flGDrgQ#mSsoolnz>wz!FeRmHomrkCdc@7mi78#7%Vqc2>kmK|il54z zc}gFwG+0n%jo6(o!bD!t=8vXLCsq4}`fO%m<=l?ScHY(W>niN{Pceez*?Q(AQYW%o z&9n^DY?~x@*nQMrS*@nhUql zV?J4U1qz$gD5RB{+v|;Fo?H$pokn2t)AeF;>m_VzuU3%UFn}`Y1)X?LMeyo*2Fy zQJHTp8X5xUUq3~iGT3iC2zi|@@FEoHUDB@G_?2UNb54;tPCXI3nEP*&HI@0#iEn~* zB#qKM$O7QZ6upqzZ^(V=fu(9LpK|k;m`O|N1<59`&*SXxi^J56Lgx14q$+jUvvh}@ zEyL5!gqU+r{nG9i;--q~Ho`}(kIT-Go6Vcja)yjrOnNXV@k;#T)Y@)bDn0MYGj-PS z8`=N)SKY8P{7(?SR+=@hX!17k>5i5@&JD)4q9i@H3l;o3{@BimG%)lc*)}GrJ1R<; z>rk`$`JSnZLe9hTc2E%=@ zp9|0XW;h!DOyZ&&3d&+BPfOk!kFKQDsg|jCOjMuL$rjrE89}XklaqpiwobbLv^RY_ zp17I3-f{oH>4fsrZQQLv1!6_6yVMS&!A))TMJuADtAdK?uK8s~Q$$9O69M5R6(xj9 z5Lj#D08O* z*ZJ`5eDn65M6**~lJ4l-grIV*+*P0G2y}l316OQhjOpgSM97NndVgO9+NaDZ?&pnD z{Y)uc-*>L|w~|KplI6zI}XU;ANH zi?z=ERJao@bYo2BBS{&oB%iA2Z@Hi7GGUk2HiS@iK&bCMp#d2=KR*NR)AL0?$gq@8 zGh9uUUS`T~1uGF^eM96GDqNvAnu*mPE%Ql3nG%VUE~@vQy%QSUJa>yO1tcMgX&~>+ zL&Um4g?TQ${eALY&s%Y+5ZW?6Iepn)@a9%ZAdvG-VE7`xSWP!gJ4y6`(@;gSQlBB& zs9^_(4HR*OW{)Q#3FPkTRcY+mK~et$HVKo$2)gR|dTvYLkE?LqI&$!6T)@7#uy#bp z(%l$zApHtH%6xDP{=aonZS* zN&PyDZQU+7bDw3XJ@MnT*ZrYqX*D`O=Sfvr8uuVjv9EzMH@TM8Q=4rqLt2y&Z$2@2 z!X$fgYiY7@R2K{fSXVP{{W1|;)OF^;E5ZpUi9|$;=QJ9Y&u{5<8-!1g6q>AirxATkNlS!WU7gA18Ovb{V zByP<^@_G1_Q|W55bs~Xa+t}7iaVKT!0v0|t19UPLq?j(g-T98&c;UA=_Uno}vm|Z- zw4S{w`>8|Uff3PP0z`Rnbx2f)Q#w6Hb)1T_<(qprVn+7rg`T+?oRV{6%%~o6f0V1#IYk3D6y22Fllb>w}iIkkVoOCqaTM!fZZ%qT5 z|9>-T-Ic+jK1(WX7c2CYXU@8BlZ;G20zoZ*eTFXB7wARcz>tyPk1oSx-# z<2OUgFF7QGU$=!7OsGrJXG7g0XPdLaL0rpNwmkFs*f17U+j;Rcx}qKFl>mqR5q3;= zJ}i!&0{py0W*nnBVpOY&rUPCjJW${D~~5Erl>9 zbhAg3MK~D{;M+}GTN}}YZ2)(|xU&$PKvU$Kl+yRLWpC=i)mLc*M1sNae|uGs{_av2 zluj33$Vd?U`MT3m_u-@riEs7%wc*F&)48?VI#V)AZuQjJoNnD)5mCb*g3$;3_j~K) zP_?-$V>}=BCv5YMF_5{u*aGU(K9ZozE$@|?Cfjd`7fuQ&hJC}a(hWMBZu>0y+LuX4 z`GNa8O^TZYBxAi+wV7WzTuYbr1J@LWFzH;Ep@>)fw#Du3D-&ym%hbz)Lghp9pi-|( z`*LrqU4f2LZy+@`$xkqBVJMmJ_3skeo8;hl!53q-lTcyf4sL_$RzIj@kxa()=b5P5 zOKg({!pp=$sP=8!zQQhJLzj#0vI>M#WWKQ>>Y_E7whlF+T8G*kUo5I_4HA z9xWjKt`YM!9lYcLAmeExKBF9-o0THHE4qRWGyrm3 z-ny*Vh<7@Dgju2f^x0@>&3ipDP^>DOPW!4Y-g%C)qDt2og$+ns$x)wHRI&>Cx~6;1 z+l;?|qi37GG4bOS9Fr)Nw5E8TBgD+Y=mm_%?(nNMotWeOj6+$^9l(@m;*nSIf9~fX zpVS=0uq0Z2mLxz@@H!0~!~2fHXp>9!EPKGm@-Qp4KGHg=8EX*`eK+H^W!Q=393rqX z5uUW(!gg8i_ja)^GO#gFFtZl}edymNku3AjsGKxgq&!$RLu~6XO8FUGFhB0bksWzR zG_`#_2RfRnmt9p|Zzs+S9A(}?T2#W*KMEE2ko#$H&Fdn|iUXS&`TeMXWtr;w(7aVc zn@D=jv>cl5Pe1kNs4d=rRzebC7TX;YZLvsyr1wW5#h`UkK_1$|+xa|VRMaF<`u8I* zML&AY$^FW`(GR#c&LV>m5tn#G<=C0^+&iJ5p>y$a(Md%a57adAnU_JvmdPq zEQc_o`Vc-ci_bS3iCZ4}2?8EJPa%qnR2|Q)mfmZ_U~vAr!Z$0<=zHN!zfLSi^mBTW%(Fq*VyVX+j0q+F$=~pr-Z@6a? ziEA}2y4l{WY`v1Ag~;^z6`ZDgUE74hy$`Kkm8pTdX30-nNoCPPlWS~0oG%}^0vD)yW1W&IIlm3id^?$RnybohJclX@#dG6s*b(2N#D=nwueT8MkI?CmTNS;8vAo8wu8HCbLusVTt2Ew?@MXDH+7 z<>x&=UWaoeXCQ1(>_MM%w38%liuFssM6ouk>NG7a%>~~o_p0(Kk_j^w$kcP4yYSMK zp7cc0*EZVj6sHly$4P$i2hf5QDTQOLk4eS0@|JPs?s;;(O=*|CyKAs01lzLqZt9o7 zb*ZWp4NcpjHX12>pY2^jh=tI@59dp#5*E$9I&5PWD86qdP=+fnDCM^lq3X?d-C& zN-QCWpd|>h$SeXmJfhh|k7wjXu1ffe`iVn8EqB0BQXGqj+1WTL@SQ7&r!jvTxb{8! zG$W+pY>?O8&(=L}w++72p4)$x0*u)k1PjCt{}?Gdk$uPzk|G>u@k_SuUEFj^P3)hu zvTFi_$Bj&%gFBq^)pzq?d1Ie?~dv0=)6QF+-_ z5?mw4HMh5extp6~?v8cBSvacw=7)cH^y(dNP3DNgC|o(c;)yS8!Cml5$J`rFHEK0f zZ$2126iS9G-hNI^9Mnx`6pcx50SwNH>P>UPX;z^zJ#PPoFY<^#IuV9fTsZOO`|z+V zt;pI5W(P@Q)#Za#O&!Wcl`Xzm6n^M!jUxC=B|JOJt&_n%T?+-@wAil+?2k5wX*bP% zBjI`SH;nnJwz|Xc=LK)lC+m7YFn%X!UQkPV6}=Rq(K6_~VS>cgXte)+@#rN+X=F!k z7i!ASujHlUvhV>fYmZ{a%IKioqX#NAD$1oeIJiA8^By-oGECpQ93EBC`jC7qX8e3F zN%COUhdqR5{t}B-P>oE@i1K>(znSt2fy2;as<|_%kuS+`&DzQL&Xc_>F_uG z)v@(qe1z>->)W7xfkl?~5xrJz{k|XVA8S?=&xET_j(EiHz5p-x)|U1JEop6>G;w5S zcF}Fi`f?gn10O8aNLuzpAm0EjO6|v-%rvEZ(HH&h3?1DP_ucvrUytR({3!?Sr?`l| z{fi#r-9-Zhg-qD(rS=9bwJ%Zu$+nSXA94tnkpAj8mMtMHcb(}TxO_@4UHZ2~r%J#a z)m%#0QuGt<>W{L>$pGV7D@U$O?hWhgADwLMAxpy$OTi+k`9qiH{YDcC)x?%`Yfb+z z#T{khIVo?U3BJ@IY8bP6z0D6TTdTL)?lHs3)ZMKv)OQ*;O_T8vsA|9M62mpsVC4xl zxo~iQmREM>=c7fN@@PJz9Qcu2t%cg8shGQMTJ671j*CKM5i@|>U*;D#42r;n6D>oah6tt zN=;YPjk;q;AO>6&KX6(mLlC|r$XwjJBqg2=y4e3 z8T9Y3-v-8JOxcy!hLcn!4pd|~IKONN=!Yo>giCD1-lbEmm4yiEa$bS?`g^z+Z?>zb zrRb7+*a*(yh}GOOv%cegn}Qs4yQXGSb0@j-9HNB%F=dC+*sL?*583e)u|4DT_+8s# zl%H{GrF={Zmyz)GOY>h0oIA+j20mu9YUol+H!qQwUK+Z_+=7a!n>Ld%=e zfz&&H&Pc?67wZRze#$8^Y*BGO94x{jAikfe?cqm5q~!Y~X#1~0WM?wVGbCEtgJWoeb;%B}JMY&{%5u#x>*VXhZtv_ToI6 zRoUg>Pr1!Do`lQ(#V!gaGJN3BQe*jbM!(PBMDxX^ply=Z1B=`bn^VH+2S*w8`&wd# zhVTPxZL5ahgxS3+Ww6M<%a4YKf4byO#1F?%+5d2amyF8Js?N59AP>9g&{d=a_FV`j z`kqM?OQLlB*{-ZCt=X8{x==EFynLv~n!Jl==PKj55wu>`P(3zclw)?fy5%tWm3U$p~J2>rmG^0Elv`ePXWH%J){As>#0Djz>Vrq6~5 z^^XEkoCgc4O{%fC5{oDVQ9I3(rEt3CJnNy{R`Jp~rKNQ|_YN`1l6kiZ=j@W`d+OpC21Y}ub>-Oy_Cq+i#M zvR#20y8VY-%o)$h52wi6!9oaXd5 z4F@hn-OjW)Xb#}$vCs7ud6=)6$Rf}TUP>F#AKK7N-G**T)J+CyseAxDP6HSr1+|jX z@$oN5xy%JFL;~I{&c-Xou<&tXL$l~b79Rg-RP`>0dSy|cRmHa&Z0??M z(WqI`YX=6-BV@w%@L|P8k>zt?%Im$EO4tkZE9F}b|1IoBRcvbDqMbO2=WC?qQ`9YK zMCo36f>qqePI=%ieK+wh|8AMdh^^9@4}l4hg-pN4-Vaw(*X?@V30+Kk4ma(FQtJF% zot3u<%#dArTEd{6?2m>Zo^1ALPIJDG1oTw^KI3&xDxe|xw}RBro5wsM#2r^q&QZF@ z5)pGNINGry$`ah#>xnkx8sz=CI8DA7eGd^TJW0|9Xs1aroKq zrIW25D1VRptcuf{qA70X_hWKf_K2sImMmM?vUaGO7XiPKxER|dv(M6n-TqfnMr9G(_u0*lYDwYe{-MjZBQBed0vxXf7RFXXu1)(bJ}aIf zG#DOgc~CS8wKUVBTZWMQ-1)JztFLL|vUBVhRk=yEmh3e%8f2&hlwz5OgNkC@RAz z6PAv4*5~5k$bvi#RtY2`#u=Q%D8lV#)bOrdy1ldz;609DtcC+-)_6|7PVgeB- z?ru_49PT<0$MzYzp!eoND`L6b?L^EJSm#D@(kJOS-`t?nl8>ip_&VZsY7m+ZV z+=UIy%Nbo8DeuyS?~%w9J!VLCyW6GBm}VL&3~GXrelt*%N)Q zhW3ymsjwM^h*?odbcm(rtnjHMlbt?9j5j0>`d4E`Yhv!z(NpqUcy0)i4T_-Pi9uxD z<9)}RLVTj+$f0>WB*C);g?I%b7 zC2EJg`PXqCVW!EPM(}PmSMuc$%6%C4ie^z`sXND8spcNj$(I@cyXO_n6?X>g^CmC% zQf_~rIP7Dj6o#7G2vEiu81f!r9V2D zIl_1R9UFjNk!yi-M(*(Wlvoy30*PAi3pYlkud%ppb6bEML}<(V3`%?njtza}_3(&_ zU&Ot9J$O^*#Y@YK^0}CtoM{>3cb?0-&kH$1t!ogXslNQGLs@0$c5BdK zOoPj;q|~YlI9=V-cJ1Vvu2x;$TcBqX9*6oc<*lc4me{X3*MB6bWEV-Qf?OPr_0<-`t0QB zVQ6X^oRM{F>ty!Ph@a{Bp``q9<$y(T$S|y?&RTGSV=bh>>zX?M?mf(A^F56-v|buv z%TSay?G1rmfG0v1H}ZOJrN<16h1JO*G8x(=y~`uXcAc+JBLd|s>}*8qr{Any&|`}`ZPmuw0yo|@|i=<|MnpE`RL z2F+NVDzQ&(PvV)${P63&VU|MgG&&w@^ps}6l1p!^@MqxmwU}=HC+xN;%}jIN5*Q2e zKRe%cjOH)M+UhWCQR?=_oE`b(1g$0)GnR_yKE6cOnVs>M1+BbsxF1lYtv0ROCy8)4 zsQyOVl=JF?YgjN$+>b*z+)V-#lZP~&X|0ySlgyGPgBunfLofp|Mf;~ z_|n_qQR!yMvELEa;YxqZ@*n3^NiH1~itRu&0NJa3)(xy*3D#@E*rThTKWs-yz}(03 zrPAciilUNX01Xh{pMdC5LMN70`DL_RgQrR3Fa6~t;V-YY*UXF;SrnhZc|R!;_M`r& zdrabTHI`S?b|5hG(x&gG92=)MjT3nwVC@D& z*uKjf{_yUL91^$Zl8)yz;m~yN1!K~ZHh4P?#S{wuUZ+nUZeWZPkF?!N^AM>rvEw`& z4-Dr#@mLxC)h*q?>7*yH5yhC4Yw_&mIr(Dbg6M&*?MTF}ecMR$NL2Wo*p9SW$4#5Y zFos+yc-2LXMu1mVYpv_eJPGH%kUBC8><_jbQ`~yKmfxd(iF*6B<3@jdy-rh8DdM4Q z$WZ5aeAL0lWU{I=fK0VI}h+;P|OnLf9Nwj z;m0F-z||&))*q{ zy1!qMH*YzJ9k?^9B^tjfd*|&lrc2Rpsurj(0~idl5qmUX(~#gb%K&bc09GcXV?p5n zk6?%l+Z%_PKDhZOcgl3u>6S_L8Q@dbv0s`ePI@?Ppi4Cl-hNg8w%ZB4-<;Yq^!|gK z^wroZmC~ZD%4Oex|7W1=u4NZ6pQrPzUXGzTN1ahVHV=>VVyQS4VWz+jPw!{C9JP0V z>HhPVLs61+Rbl?13`yxW5(sKG+On75Av!hT1&QKb##)X4=%zc=Z2sFNzAFlN1^t_$ zyjA>VWS94!Osz$Z`-iqpe^=8~>O$sf^9a{BrI8u-wK(|;p|+(6L6yuFC#ZL&-U1=c zVqY!w%$D*uO0?f?RLq>K(|wlOwjH#vGdkZcGrWKWdsd57qY==@gBVEED`xNpkp~5j zt?uWsH(nTa*P$=H=A5tt2QR0uWam(OB#N+Nh{%j2RvU$+O5h$6l}8&p$=_X9M`Z19 z;z76GmDcXM>8HS(a7+>IQt>-{ugkkj7cEg0b*i}I7BtlCNVxyEQB$1N7ERa%r3m0q!+>t<;4um!_>vq0#euD4^Rpbo*5 zJ>2Voi_1%j5%|neV9UXI999LDoojdfaYBIX$R?T}YxxC870dwrr%8W==R@yH_GL}! z5AAcqx=6`mONhewz}eRRqmUw_)#XNNy08pFY`KV_Ngazzp^#u?)w1@*Irqpp(BidPrUSaN=I4vcbtt6n`;_SX)cf-$i6Ho) z)mhobj_zNdr_{LhCVZoIAZAi>q4Y9iV?8o@SW*lt1uSmE_GTpVEuSpl|2i4Kd!CQ0 zwi$+jg-e+Dzgv<+fvST7Qwkw%c(KD5eM*rTLo$Be7FlMSf#1`*OD9*+@DHLnNm0&o z2wj@A z={yN)*;l={wzN#20<|(`Zb=8aOC{Fgo7qetnb~hf;!;;SREER;V4PT}5^7#)lTB+0 zA)x!E>H*HsonRCMY;ltC48AwbHKgr95i~#oh5;bRe2?436NO(sU~8vfgm?fS(;?=Q z|6)Pozb!3Q(qfZbf5vR)I~0l|u(-xPxsfumJp^{Pj!6WAH6_}Mm9VKG_k|jxKVwmw zd=V8)(Nym%OuPZ&DJH#jDCvzTQF}1%!?srG*L7`@;A00!-JOLtg8lbZjd0i|=|0cU zq#kc3Jh&UU!d`)u#jx~2`Jx$ zDK?Y#(cQb30FA`cgh^c3&p0-4GBL2*XX@q+`H@=J!ftfZK`m`G#HDQ&7vsM4!@zTv z+YP$|Lu5(c>k_JweZs7Ba_0%jHM5pCt$ebvYA<8|6%?5V;+<=hzEhaHXwBawo9X}* zo~tQgg00M!666saAR+S-Eb%|tOo)=MGEupsbD4xMObZz&N=%vNcs#|y^`?T<8IOo^+8EiN%;u8^lRYZUhSmjhZ^olo3kZ4&Lwe6FqL8DCFB>GOw#xmK{Q<8 z7iqunk$I5jP&&GEi7Kl!5c~bEDvUUs zK{kP^vW+2d+lq3nIx4-#0MYYF-&)N3tEsKo2~U$K)l1y#-yeP-B*UYj=v!G#_r5b1B*X5s|93U zqK@cSA_~1wA?TB1Ze{DZTs5<;Dy05lS6D>yY~>Yi%&ox}lH*W2J)x(lZc^rvH&o`w zp&6Hi-L7p-e&%`#9WqQ;R@i7~oHs>7V0p~x>9Vt&hCg3laaeb%Nsj+OGVN59dn>BU zsQ?F_M-3i+sMlrsomi{MtXm&E@Hwq=(NoSYfjs|QisZ{iW@|DoExCDq?K{CWPSyx3 z3C~-NPS_j~65dJf&%(YCojyj=813#Oj*ytlnRb)>3Hv99Vsc*?e)&Kpc7A{R|ALvgr0d7(PZD zva)y>*1O;`zQ6eVzjH*X=?Ebce-DHxgQ)zZX9NtIWTb`7R`&%?fHjk93^=+on`0## z9@M(>8Ce!KCcKSUm9G)mPsTKu#sv&kc`mX{k9v$xIR_^E zdUt<%33l9bS9;FgW*gi)XTaQ>&B30?Cp$OO7Pb-$ya6XH|VCO>`5bQdrE7uU&G?=()ErC zC-*}~&A2qRJO$l$q-1yR%6J#`w>fx)?W zIul5a&;6GXcsqa3P~d~ho#nwqC1T~f58^(u^u>{hQH()O?_uE#INo~ zLYvRI58$dD0~R7Biw(w0AFm#$)f|3qaUqX6N3=^%b6mFX%(p@z0Z4%oXVj3nx2+NSxu6dyfjJN62mF8G~T07 z*Q2)SSDSHnIIk-DDjWtl?P)i+RHlX>or^+)YBoXoDS5ie|1&J@hYlvR0$ad#%yGs( z=42bMgXuDxRNRc)+|RQS&EL{tRT^It8$`TmtE@EJHg@q|Cq0*m{&||Syt@!c{yU|r z1ZN72yF^{luMYf+tC*;LM_$!5wEl*9YqPQ~y>WO!(vk5089M8SCinmEe~)<-m7A9S zNJ>jL9FGFh5~D-9L1M&!?HHisBu0lc8;sF8m?AJ52aFEs7&SI>8{GU2*ALh0{jA4R za`B(vf_B0i;x$O|&+l_RF5wy(IIV>EcTII8f+I~)D$U*o#i4kmmEKJ?Eg)Gr&@tZU z|EMfC*|JV87%;Bbc%{yaTJrZ*{+EV6ug<>S5J2i3#E$_C_D8LRR8gINeZc-q65X;{ z_ICO29E&=q%D#~@FrDl`(|b-|;C(=q+2+B)MOb7?w}Sxvpj3g)+eKBBsBpk?BdvrI zZyLT-o9y!crj3{MVSAbu*~@X10|vAgO_*2CzBcNxOO{&+E|LbJJYOiPaf9E)5AIg_FjK{Jzn~7 zR17MzgE@9%@+!{R#I&5AGLA0!lN7B1RypmX>DDf2%AR9d^)|}bFvLtv&`Hk>*LeKVl$hUhs$>$FMTdAuo{fJ3igT zyfsVn#F|)xoV!@G$Zc}+G3T?p&h8x*w47 zi;Vo}&34B|fyCIHhP(xP`1`D3*{Se@M_y~c%`)9je0K%u%#E{~c8i``!WM$90Qwpu znwBh?ktufm?K|r=U70d~DU;t3CdAsS0u{M9afgWl8P6Q*L$ZG%)0FX~(vN22PW{!2 z(=tT{o_}q|U;px?EtkX+s`VZY{L-Iz@accwdjRPbCZ4`s<##iJ3qB;88y^U7aPDuC|E8D15}EUkUY z1m-suH{ay%9@%Wp*9H!1k%^=mZt1-hvPN?nx>e`AMM~7hD-p7_oj2q-gus>gQ+D5L zRMKqy*DNX|5R^ke=Vp>^BGdm{lYN{Uf{_(5a4gVgii@BlW$t1B`Vu~}$=LMNBHHt? zkDwU1+4t!P%3$_xF)4W7Yt?tRJ>?~^y3oKn{a#s~(VSirJ4s&rxc0Y}*E0wZpeI2$ zfq9i6=iY$$IM-;>#fr>xp!Pi@^-5T}+C}iWy!WrS!~WjAKpi03;jHC)%46g5i(#du z)zTG~1mdyiDU0B{chw@YNc1-$xRA(y4Vv!R8oExPI+?LM78xi&RPx%MJ;W_q-xcph zRC+Il8pyaTWDMq+T9mt<_9jC_G`5|Fu%+`jZXG@iyZiU|bHjDrnzq|G|m*e6O6|`k*VT z6Qkv{*;xh`aIs+(6pUMU^$+X`oCUgzjXkJ^UN~W?4h=Ev%fy9(|bjrMd0AMyPdxuJ@3r(}wOGi*AA0HEGAWfEs<+(uh+7 zjw~2VcoxXA^|nmvdV6U>XjVYn2}B zGQTpj9K5twoWqp*^M7G=E!=jo*{k-S7Bal7sdEd;xv_ocBJ}o&PhfB7fsZj>)zLBcJYZ((Vfa;nUz;ezc)&RhG2B}#N%?0&$?tQ> zY9@E+PLxmPO#E?sZsl$saApJkCj!4&N0EU(TkUor3ajUahKaTF)Ms9oP~Iy*NBlC^_6SoCc>Zx?ZddhU8=Vj9c9n@ zd`Uxpu#~7CYv1C}bYHeT!-l?@4X|adqHX<=3UR6S9E%f2Wc`HH=q(qdMIqmL1JQ+XYj$vCZ2R8hU?VvXRS{ z<-|q_==`R5)kNw*AOK*hru%Eq)8(Jb)fV5#fL*~ScdXg8_HlnuJ0CZ5&xv|o0Jopl z#hwkK0T5jHZm|DQSLQNX>#oX>6x6}P6r<7IrxWz9tQ5+At~7R%?XllF_A0$dN6S1_ z)BN-!PW?=!bZoQgTynRUWoDyP{(hLOZrtuwWqn=C8;5_%#}bmF7CDD`@+p@B#GhL( z{HEQ#T1YRKM3QwZr^qfwvu^GidV5Gi=)unRT$!?@9lJ2ZgS9W>&F@az}W=aw;?_B{)=4#rWEFCUDm7ZOx9%CGDd$4%*~=Kq{aABvn3esVLdZ6vsPOCeMW7k-qb4k*ott5**5WV zmnX?O+6n_TTpiZk_{X-p1n?8#BbR9xa+1uJ{$a-QBhA>pZ+|k%hJ^Pf z4A{#=Qn0U_z(YS}d#nUF=m@JrBFwP@;QLJuZ0;MJXFO7tZ)j=iP~QaY&>t??W;_Gj$R{ho_8@kjW-Ud18yw$*paDGG3XHK#+v&Mc~>^C>mbM5Ll3k?VbNe`fuJN6Ih9sY16ct-IUW z&+=$la~@Lvn|3>u&I|YFrwnU8U(=9!oE0sn*u?mGexC}FO$HlR)3oBhk5Zscx9 zZNKLJZ6m77#9Mx?vE)~F=@N6U8;NC_bQ|g$IzjX7c(!Zx3|)6)-rM9jJC6yFXy}sW zJutf>dl!tTK)ns+L7ppy)?Z-?DYbyIq zE#YfYL8r8i0K1u40e?hCU!=dU!K{t#M)#XE;i|Xs|LS`fgK@}cEY0X>f1(Sb;mco zEwqVRsg~_^t2UUO5-CZBPQcT`MsW4C%NTTVexfdT`- z*u&2^#PPE&GAMKBW}DpDO(Gh1EMqWk$!&nYxU?${qGWnrzh|(s!wa*28TE7@{0 zF|7lb9r)~2&@y|e&Y2eY6ksoP9FWvwR;a31jLD7y?F?nFw1^MpT-jTZHL;H?!QRP$ zDlqko#%hWDZ$)6}5eTK#UZAsZQ?M$zm5r9lE>0JWpWw)8_x@PCa!{$8wD=};y4POX zfmu}{t~ea$5x$=`y>w3y{4$9ZouEHg7^{S-StEi&iSjhvrwI)zlD@@BC9B! z^5Nb)v*SO-4~twSEDj>2JSd=9p^a}5$9=I<2KC9aFXoi_cfxq|*MfK|J}-k%NC3## zTQu-cV-kNtfg73hg<5SFVfxNKl7yl?*PDlKd*{|Xa2fF;k6nuSt_HJ|Bas00FUQ4; zVMr|?F84a30aq~r{r=Ing9SRIje6I%;ZFsB>4W(owUt04Y2_zATHf!#G$ z0w^|HTtY?XtIynY-jL`5ph(cIGww&PW8#9fx87CqW`G8iTz z6P8}5axffloo~i^=shdsV24Hnh&$2M6pcYmsI%-%we?eJ@MI%$1yi0O>iHc#U}rro zq#OaTW5H7q58SnA*K}qo*BWf_IzPtpXB81fRSMtA(T!fE{R)}#-jhD(nuCnNak)UE2 zMvvxvMO_7q_BlHtfQ(kFP^R@@<9M2-Pqo}N);kl9^5Et{yo()k)N$KXEw+lgVNj5| z;tJ2JIrFS$uWt^r$coit=9yAseuXn0pYu4&zk=Or>CE=xHy?^U@^^T=hCKH>j%m>SbxX|G z%h1H!f)ZuoD~CeW_f5ZxcOMX+bUfVg`Pp0ye6!Gx?gYJ~~%LGH8C@>`J z9y9PQG;&T;3w6dXncZxj!};saYXM>2N<%&gR7punag9zEfy6!yTrV4;444;wwwzKr zw>+O$VaOlG`4UKdz@Tfi#ae<}bh7GtSEbQf#+GSrle6T&uFbgYF2!m|E{VVTa?PZ~ zgqLYBE_XN_Lw`zjH1CW_Xg5FQLYhq+2us($Cq2m7*%eHf?kd(&=kww> z`~7%%lK+EY6hqhdWp_}EgxJoNsxyQ<$If?8%Srx(Uh%rxIJU6eCDjTrRZ8=zL_sz5H zx~INN<7Lw4E)-cF=rC@2>9m$lM$Da`&38o)bo}blM;lA}cUPuihw|-QR^53%wvrs> zh_9AK=u?7{J5=RURlMw+rhr@VeJ{?oJHYftm*3E>9h%mUubA=jwSb98eVgu@2+IoG z^&*&Qg3P}$T&fs8i}`y)bLu77000;n3T`v9Y2)I!9O3QYS?!c>7vEJ?{1zRss%kY! ztHkKc@)URd4vtK0x8+Smah`1z19*O+JA@VssF3x(7YUP&F1{DSEMe2^kAmE*<&Ehs zBatPXFAL^f;5-w2*V!HEMyn^b?@~Yiq)H9?NnRhtFb%RB3iyx}nnEl0e`go=MaK3uOm{DRdYFOHPzLHqL z&$QgD78A2a$%4RN`7^f=Sgp>;s4n4~Jn9T~b3?RF$9)lSPvc8FsbzQ_WNnE!(AjNA zNad=~Tw0702E(!F^9#KE z46U?T;qP)0F(&oqa#{i7lW&QCzR_oojqoeki|Z&)5w9@~Eqh&H$>tjKJ(1##Y-5@- z^BBC!VBXrXSvB3Rl+p$4pUzk$B;w|m+haERLIpFP@2VH*cSF{oWsHS}X6eElhyOfN4N|&)2{m&1xd!Z|eA zt$`&aC19^Wai0M$y*~Nnzs5qjzWgRx3H{tuKG?Horq@%QIWH9%{dH!t}vO~su@$?2tQxUU)d8Ly1nc%ZJBjZQ0z-w z6h5WcZ85xeSW^^@ANoVCK|$l5wzMC_Lt~f9ihUX)nmfH`%j? zweT$GU@=Rpl`z@S;pJHf&TI*F7eWJfY#vp{8h-8!%k)_tnL_D-_n%A1YlVSlAOMMDKUr+ zH<~W|`9^W1R4dNvP`*BlM45##$JOA`%8w#duI!1Nlt(K%Z+S zMd0sfU6ps~nr?xY72b^j%wc-w z!)GqTc2*%NZ+6$`%UuS+3Be8(=Co;&}7Un{{@YQW9+zekf7Fna}iQ7 zqU^ZL#5HVHhbW%l4XqEImPr)dCA#jZ3Jtg5I0f_nC2^52DxzwNGy3`>@+PmL8c+`4 z!@Z;ItQ*8?+FF0 zv6RzZEiwP?4E|!q%rVz?wN|QG!DKMdjXtj2&rXZ>pJ^5#IlixzKDugGM^wBzmzVSN zzGL`skMj(ytv(XB(gpK@8=Q{YicIGQcby5vrjHl#w?E2^r?* zHM~f;{kBSGH25gNIsDsfkWcC0=&>|B-jC;_|Fp=%&vu2jJ9eR7W{Y4{>VG3`51DZG zfj4LW*GA3$RRI&pC=P0$0iHBG}AA^qSHfBgBZN7mV0!BUY1)S=S!*3(|&zj_he)wU^kBtt9v7zxc;EX1p+c-X=Y`vRQWY-1E%C?~%cX+gZbjLn=)9mF>0nti?q_eGt;{<;Ps0fE5@#ba#c z^T`cEZ3+K=Uv0WRaQc?u4PypAk!8o>jA3=9L0z|@Ir))HIjQZ?qDtw(YkaiL+65%}_%JhCY`5_gka;l|Mc87v z&g5RD@Y-q*HgL!_@Dg<#M0|#F$Ac&Cf3KHhXbf8tn|(vR?umlwu_#_T9T}gR7a+&V z5zEqXGC99{HO^P-#`Zyx#$KzcO5SBEvZ90E7KFJ^+0aJ@ES9jH%E92Xcd|_G9G)13 zmb=oazHkFmeamnOMuGToGPpVEgE7k}Ys{`uwl?Zok{l{xlDPVQL#K^-^8^IldUDYa zjK|Nq6@TGXMa@SCKa zqr=lyvL)NEr}wA!Rec2@hcE5^c;c01qOC!wmodphcV;^OzFNLVUnd{f+0wqC$7jg% zImN*J%5x0P@HKj$)cT zo!aYkSbTFe7(nN2(W!I3)`S>h?0=s#iNk^Eor;o4&bu8c#|XMfEq2EvZT%~63%Zs- z!2p{K!Es{L7c4XR;-53l6W;0ErakXUgj$7^&5>GW{}I}9X}xc%kS2LB_uod}pum2b z?v5-S<9N>d0^biH_snKI$)u3cMNKzNvOH8|SyASjaUdq+R{uaP0qj zBa*~s!wp_dIXRHl^996_BcE+}`1kD64ej(G2%Y1~C;HKJ_QaB=BKZqb^ysD;{T#jgN zJ)0@4(IW>|=?{hSH=gD_9l17nnz>3dvwxL_^1iOt-v<*9R0I6}m}Z8NcN8RUOBS`W z`I!+63(AVhd~I~13;fHQ4E4$jlt$Lp!+pZ02S4607!g)>eu< zKT&X1_?oThHFw{YJv788S*no+Of<_F)~Lu^?s7DemhgF{bfg$>8#29++8cZn&-m}z z+Av5fUM^y%gx&I-l>g!t<@!}mF7|N`ac&FOphqs_W+`C{Eue3 z1PhPsnWletlu9%lC4^zXSxCLplQ)jV7u9CPP-1P$$EZV`kJllse_Szulkhba1=Tvl zkPgi>p#v{A4(%$)}uP^wuJxfV*~zQrdZM)*E~#7k|z?y;1@M8F|EtvJiUV4y=_%fUzm{h zLlWAxRHLZZAV6fDukSx8Tx|N$(ICm^Ny=se{tx04?H07AVm(ZHGfB*IZ~BGMlpn`8 zwFu}L5Q0i%5~kh!BQiK03$=bz61_i|M=c18)HKPMT=cHE7%y%ujPy)o@^S%rnPEq2 zf+NS`jpaK6Kv*joaDU@9MDJGu>%3X$#q{Rw%_JW_H1?%pvP%4~zEu%gOM{Hrye~OI z+6=t6;K-$~FU9(hH?>HE8Sal=%bF^)$CL%1H+G=4JSgA7Yc9``Sdol(nk3PZKHdK% ztt+`WIozy2aSayo2tDr4}Uyg5$jzmKp z23vfj#*2q$d(_%JhqQK)avtd-fipJY|@OzC7TvscR4I>&=3(mE?fRQtCd^z zs5m6#3E?3!GM>p1A7B<`6DIVd1$weYlM~lYa5N1;Nvl$ASYnG&hdlK8{z#d_qUK-+ z+09Y>1h7k#Yv#q4iQk70BN zPR5I|V(`KisG_FzGE5KOEH4&67g>C|{kUo~m&C)Df$q?_CKKeRzf9KmuDtBRMN*sK zyAEcSh(?iwV4HUAaBD~MYa_G!DzXy)zAKB?GZR|v2;jZSUCNsc@#8c0+}EN1?aTPI zTLCrt<7BP2x}4^DbOf}(S@|xT4wDPy>kff8Ol>C&;$)c&_* z*w&tny?0Zc?oOmvy61jS^e+6MmSHxL*IwFcv9xYWdW6%S()c9Ll5d_%J}~x%k_GTS zBr5o@U(7Q$Y6`y;_h)0?lz-Hr&S3=&RhgR^lgeAK8}t!ReAqRk^H=&p?@x%MQkCky z|C#itvqgEDe|neFH1@~7*#eY1#&9B$iJL29#4wT`K$~#et@;Nim?5w~;kx8lFbS+m z3AmbqK{~jeSBCEd3vWO$9HAtRQdk9$`=t4~;T@BVmInl-#8Tgxvgi)eT>N|&k@UFm zJH{`P80`?CdPOl6sSQs+vN0l$8g*l%>LqH1gonE z!>E;g5qf2UNLYESj4s_^(C&kqNlA#~V|ovEkyy!$%aQ{>O6!UJG$f_iF#Eod@W}O7 zl<2_ij+)6%)X#NsdU#d@Q}YI&Ew5Ix*brz$FB_wju{mXNziimYN}$LV@?bTzxna0{ zxhW_gd#L>#NmTWKh0S6hLnt?Lu%gxdbrP%U@#p}8>cEts9XioJ;Jf5jm;H|Qe%ESO z)9mG&^rRLAwTR=hEhMhTgw>DQWpQr4X|mHBTuxBkf9+Z$qUU|CO6lm)^+S0JF^s>p zim`4Vtq}RrgY{yL0Y*g{bo|pTnsRAnclIM?dCgJ=Iw!ni9(`HSdnNAQ3sO%QtEZU3 z0t-S}D71&-&}aRIFighNdpSZ)pEJabSMou5@{V}Ng5X}~LX|!AF>7V1rlnRUqK!?f z?HDsE2E4iCW}=f6;Ir>2CVBBP{wL;Nj$v9$U(agu^28`nT<*_0ZTxWBKJ3Qu4dv1C zx!vc1a_u#B9GQOk?_DzK`9^V7(x0n(y@Uc_GmF!CBONLos6w+J^Y3K*Bh}hFwJ(bs zs`rE2czAWcncLnKv@pqKF(j#zH;`5n8TP4w0@1B1xYAr`;q$u6I$TmiI}Ta5)+hMf zVwR0FK&(-)Mzj(wVEuQdM#WEu2-Fv$Q$A{n{Ql}ES9ewE_YlPGKlU!8rVPmmZu!-Q zG{pfPvq!(09^EmW55XbcvIsXA+n+4kPU><4ylNYo>7z5CnLB%Zz?W`qb&2iIkF|iF zIaQN>dKML;Xx5nX2Kq2&63gEy%ES89`1FK13N8cM13Yt#E zEL>}EN)OXj(l|UgC4XS<+s~4l5e~5WM7i+n3Mb!@U|SJVN7PuK@~VcE@Jam0zQIYR z8;jN~nD{AR`hb@W!@FwuuVbM%Io;g4kK0Gje7fpsrZvB(U`oX-nnSqzB=0q490avw zoeE-{b~8U^9GtJdp-8}AM5(c|pI6g^RdAXi)X}>j9>b9`j9oYE`R^*Ff4oU z!Qb@8$u>US(786(Zz16UN~h2MY<{oX|QMwxm`+rPa&?z<_-R3FWA z`2311`7<;3^nQI;i{b6qisz5$!n#5n22ERp(Ud zw&=Lr>nLKtR=e?pK&p(9?Qp!==hL+_8n=!}?s_$GSB*@B+S-4q;2A=sDWwmQ4kHPd zIvcer+uw!h+?&dQ&gmOxx&C=rMYs1Z=4+i+t>|qcOC?P0bau`tSBI9GR+nVd5|`H5 zsYPt?Wd6HBy~&a1)aL6ZYR^Ryr+x{3$@lI;5#YDk*+@IVK%d2Vo@x-#eL8i|-j?96 z_VgRX7dmzJzKs?Cq@ANWx2jOdV#&7`#-jsn17BrdLdD zX_cB=+QrAcvo7FtQk*!*T<7m-D0VBRQryQN!JI*Oc1S`>m8bgxPqt6md(~kQ8wy^5n1w*RzO?dPIOfvu9TS)Z@nrAQF)SOjnqV zrH`*C6@qsj`S8?7T0%`uHdxtHW6x{~cNG(vJ^>d+0-9XR*zc=v>(DZ;3+*hjrstL7 zq(&Q+e15%8>Fm9m1NE5&4XKusO8)Br}z z-Y}1td!PW-m zy7-gi9CCU_)gnZC__-yjRP1l60@<<=Cxzy2UA8=e&z+aCuzYJ);D-pP&L;Ty`KpGy zgz|c?*4}U5r0Wk0qo{|rbv%siRqYZ%{okY{oIhT=w3D9pU*=Ynimq1bEnoG5^b=t7i|4_!JT;qo=^7%>v0k9 zH9`Y-8wd$whC2=r!8|Cd-&1Hxcc_cnRz=`uFTgq8^v&@CNbl8eyVa^6O`6Sjf&N>f z#U@Yhe{vv5M7b%;`&Wpqx_(>bA8N;t+{c&fo2*5G?1ldof^&{iW(2jTzP>mW=3%oKMZ25j^9vg~n>}G7hssFz3BUy%T zyf~Z%fd@BYiYb}B>b2$z*m+LDhy5LX=Odee{JQ1DKanWpKE|D$EHR*6=H|@wbOiIc z3>1}2`fE9W$&E(nlCOJfS~4DCjF87g7)f;H$B|yZ>rcr!g948H)o{2GVQnW!z52F7 zu3Tduw_*@%i{V7fs9|uVGn5AK$*)j;s_s{kVnn0weSsISCOD0`@K$E8F|Z zS#^M-ag4*)Vh=da!1UY)+Wg6FA~}8BvSnft#hEV=2xwapM32;!hTWrCTI$p6m#bEN z>jv(?kVD$nwYhn4R(vr%2>AKa%+69U0#cQ> z`$yUbAERWFT~mCDLK#Q*1DV36<|0PoKQDHX;}6M@jgEf{UZ)rg3F%Mo^OX9!+`+!* zp9@><%EaE%xJxWPci5!8d-D{)4sWj+ny#6z-U$KI-<3Yaf?gb`88OCsOUpGkDk@Dpm?cU0IAj{$C0P4t%xbT|F^Rc*1>w*28yjOy8Iv6!2ckJ=_AIp7c zfn?wT1{);oOIeJf|3clY6PJtfdU)J|GS*iqz3%D>*fIuQnV5#>9)aMXV#-EI(If0xnTjt+(XF>fK2h{bgNZlDA&(bqapCtZ4Pbls=&Q z?`q>(&e!2#?}pm&4Zc2KrjYRNgnukbL$vM6mM2YeQzDs@sOuaY;s--aA1y{gsU0gY zq5T-J*Q?36hLG_aoVSUK<8jTQjXX*uy>g7b>o)0Tp?lTA*=#99s3E5^A3pi+&73X)}dhwv+C`*SS z@TW+XSuQUqBwfWT9!geyQuX(&hEk_C69AYrG#BhDmTEW9Cou(nbazP%xs~}2!Z4nX zcgArG3!;3u3eUc{QI_?6Qamy&mS^3$EB}Zg=o$Ma@WteHOrOg_SQq$se7j6{&b7AP zKz3?JN4Y-Xp~aHCWc=hdrd(U@4Bs|RayA&iJSR=$Hueem2)%qqo*FxSzGH#n{x?Zh z=&0})-qR*iYI;e6xHKCbKuIP`K7U{6q(pokG1S0C z7^pY%H#x&|vUic|%?Y39MM%76(=JXWuw!XJnVTCgMnM}(7B z_n8wB@ArC%mi{!03q7mqY8l64P)!eB11v2_p0^@1->;*IaP=yJFRV_0mqGsH&wJeM zzI6XMA2ux~mf*r{ThQI34#qPXJ&OZ0m*ru=6bSSZ110_}QhC_um#fg_eKC{yZM>{? z3us)WE>{zUhL6Dxbp}(+w)LwVwi@F&BXAZWJGs9>DNAvDXdFa(Ibps{AXzVngjWNt z?MM*E_3U(j;R$O+>Hc+1F?Xx%m_sF=`=;_H^_wh~B6U2|pYN9R`9zoqmQzqB8xguK z%9`HWDT4OAuj_10)=pdt34QveSE^wLLh(K{x9qPqow*^7Q68xG>%C($5VkH+ir|TB z2P=0%uBxvg*ox6UL? z_3R_s}M4mLZ=ahe>-k!DvrfHp1Q2g7WtLNdt2`O z4DKSvv`IK|s6jE!*+q$jr4Cw|TfUlbU^dKUkN_KY)K6eY1LsR;>#GgM*+$`fRn6an zGOeZt}Mmc>m?j(ZK9;uqAp5yy_laix5s_54u;h9SkA$2ex@c$?-#a& zUc@12j(W4bJIx^uy~@YJ$ACqBQzhMbha#;F4G7ZQi&u1vo1!kr=gTx3CdikB$iRsp zCQW(K;2ZRlwTrrNuC_!DK+WjYV{Y*7K2O3<)L~F(h`BAtx=#j+sK(#v(MGpx_Gv!) z)y z$BgTs4zS+Q@lG$z!Y>V+IPt&B$+G;tI^2JzyjWm1$hx;rriw#{@g}`sIemPlem}WZ zr>~;dexXyXqVDb6f^l?}PR60yMyfcW>g}i?{MOq)s$LKiLMf_9W(l+BK7KQj5e}24 z^XVZBbBX$MQ&+io_lr*%v@8~aaPHl-@=QgJbqzE`h-=un<*N~Tn@=479Dx# zc+UDOi|G~yDX2c)!xUe%kVOLbE}D#r)LjUq6hniQJ3~KQxO4Rqp-B|7JCB_W%wK+7 zFp#PiMq1IfPI0z^GmU@&tS>(YXQ7_?Btty@Kxr%$O|*vUe?Mn6NKv|pb}$Ke;y98g z;h#&hr#;PeB+OYldGV~c=Vq%fhdd#dveG!r+IgmWj(;K~^^A*UC*0jaEDf$C85qy{-0e0VTx zvjGRW3XZwUduQep)Ura>(Btxc#~DjPDZOFL3$YRr7JQS{TB7dc0y1dqr@2i!>*FNd z%sX|@LM9Jxh0Km~Bz{UP-+Jl+O~z>Ug{spQKD0oe9lc{`c_?6D#>&Pa~89A++|O}egp2Yb=~arK6F=@G;=cB(!`q=@a{18yb;2G&ySLAl zsdZGcdNa<{-JuZzbry?%eV~@U;slv*GE85ru2b;QxRWeRNAXXWsm>XlTAO)(!KW?{ z)IJI07A{6n;kPThwVXtaXOz3UMs-#RzwIgrI#Uu%6v}SYA(Py+pY>LZD%fE2WzvzN z2yA8AXF|BURwOV#sGy`^LQ(Hk^-jeTEiM0~u?=46A*=afvx^0Q|98h?9su6!;c)+! zVnEWIKP*P@wITncd+;#Rh~=ss$p)J>8iBApXu{9*;3GP&x@hp`RVPTojCu0HqF95Xd-`Iz?(w!7_r*jVD3Lb#TEskG)tkU}A1ykDqMjP3t0bjW$amIwLlT1a?mM zlM$+%tsx~~c~{CaK*4517yz(RX?h~ z7z^>y6n9(-o&ZZrSg@8e$R?m6k63@YT^4Wz#w1TH%~Vt|dPPH5r7jenST;n8#pm<7 zlrak%5`*jLXY&)(-saD4{H2iG@o>Z!ZlVyM*1CPs#UHR1QA^^VpVZJD1kI_P4arz6 zaFkdc)&$Tw4+nxDR9j;zS2PwOzbf4TU+iIk4CX8K18qV770o_vc`3i|5m{RD9!V2i z6(YN)PM$72Ym1>X^lO8D!BvP911r`X}gI)r5yox;}5_y>S8 zB(_DVcy~>m_?Z`@1hRRS=XaSZ2k9BTXf|~#hY20U)SvXPq3oq!U$Y(PnEqrCX0iV*#!Vmx{D(;vU5 z*{8coW2R&43P!uLQsGlr^_Fn6n8$L(!d06R?+-DtNN5mG8b5QJhl55BOeY+YS;UlW zX?&OV)#`hq_1A$$*gc%$Y=?1lKvIVH>+5RK zbkDA{P$g9Sx9{qAR?oZT{kUv)Z%SPiW}K`^rf&JIQ3odxp3veV&H}vw=NVV&WYKkU zrRi9+nhq{k-%$_B)U-lFrg#U=ec}5~TRTahY(yb$d6K!A6H0693}3C4g~LybentA* z%Hhv#_{S3?gRb%{u~PY$(w{>iTR1NN$Z^Qyy=Z+Ysh(o(Gw&*mu^KAXWzCPP*#rTR zQ0;ck@;p+%AE6P+asPp(O$lLIGi$g*depe9^a2ujFnDpkUW zvZ04TNh!9?CicvJQ$XqI1yefl>YcwADC1(eH?SN8{`7T0lnN}mu0Y~1zCotCPE9s0 zMskXqvD=8g-sWif6vd!)-c?aC!t*7RcvPf_FlXC-y=17hx=k*N|A3E~xpx|z$Hx5h zLOR$YGER0g&fyuvu%tg*&Hb)SQehec*gxsxs1Wl>L+Y95g1qias} z`ANQJITH&fNV(k}1^wsaYH|)`$l6N)P zJ7(gA_xcZWgq-#?>ExxUR@Z_Lg~o7}18OxK8>|8e7w;^gCxNc`0~w2^>cb6RZ?iJ+ z3r(9(TtXjQKn#2rietpNd4b3#3Nt*wD!Ch(8i1eo39k+)MB%_)q!NPuNDBo#RrE$waPl@M{27%c=&#has zNicf_JAPgGifA#D#1$FHJAU3zl}?1avU;xdCW|Ig^_fG0mM%R>1u@DIP`V|vMZ-Jq zf_Qg}CtIXb-%iVYkf(7z%@WwV{cu~hH8f7lXk43)5#4SGnqW0jXJ2^dvI5>G@lzZ3 z0*bzx%z7fUWJK{(@==S}%RXh_leTT?_=i=`3<`n~e=x^!nFfoQJKWE+oNck2gX)ff zqx#G&89(D}`M#g8eZ0mUmecoAwPEIg&WlIboKZ?Xwu~$YrC#zT-Mwlv`5dt=4ReaXHT)8~;9=%%QTZ}iB}*N*n#63=o^o|5)qS)a?8AxnCR;yt9IxB9D7Ei4}Uc{gz*lXZ|lkGsfqX!vQIKpdKt6(J>4b z^EEHXGrTIC5AUyvw;AGlI~`hV)lE<_{t3Tu84qzB#!)*r>!Z1z@K#YV%k4T6kh(pP zio{X%J@iR=`Rz8Z z2cHx7o-giDE@si25#^?BwoIhoL9qUnnZF@-bo)gK%T3wDmO~PXisti^Y zb~8f9a!W2-zr_6I;3X!P8Xi9;`r^P=d92Y}+Ax*gdTRNH9n}Y0x5WL#)-0H^P>B6c z$A%^M&oo2%?-W);<{zFGx}ex~0La`acJX?8oge!uYGH}p;(-TMbuQL9QT!i%y1R@+0aSV8w)3%NyLXwa zUOj<%tm^-tp|fCXGuztmH`D2KDwKm;A*X2Z;usj|*XMa=C34{ZJu6I0}!68_>&8TVYFBO^(hK1r| zCBysj4zD@b;OrNeea~6v#Y+{Pan*do25H*|jxE1gHy_rTgUS7T$*E-<3-B47`-jKZ z`q#^Mm!QH!L`Mhx z{ZP7s6Ax9(SI|smRlQqkwdL(qh7;zu_kruH{ragq8(X5OQ6lJwZp%U95>c)%5Uy8s z<;;|In14}E=1dwX*Ca9*BX6TOE$)KSR6&f(WrS#_@f(20QHr5$$#!h9nC*lo^Ti)n z(@fHlSv)z=F9f^R|4mH;5+?L;OS1A=r}nqnF!bV{1b^|)PPm&ie}lv9*Qc2fYxFi% zX*TkKd?v5n8$7ZYuM~i`yp;3m#4~)YPHrfKylrUnvPm3Rox)bxVSw!@MAEWUuRoh{ z3{eIqT$&VNx*XEQoBXb#j7B$V)N-mN2KI!tJ?wYQK8Z;fq zySeA?+&-T*`zfU%pjvKTGWWrnRm&uw#H&dXj*P#abXhxYZjP8vw_kvcA4cofL|3w= zAg&h$Au9C;?GvcKMY^^-I5^wadrY{P=pRA$RlRmm`2UO!dkKZZcy72+Yu$Yk@#39V zGp27g4^v)UavJZ;=mAb`K+e0qn#^e0u^$3;$6xa&L9B)}45B(45+wbY;JmnYZmn{$ z1S{JI$AOZhmpk3mqvo~3dWpJrjNXYI9vX{keafcG!;FO%(G7!!?DGY=G9vH3k_B!1 z-*kE|l^MK}i+&D7tH}CTCeD8U0jzuMU~c*C;OC7!vwY7F79Wb~s zq{8Oc?0od(@T%k?`GgId47M)kl-Ih1gqCrGk@9YIFZrtdK90?zTshn&7@39>Cyuq9 zgHH7Ie9Jt^X-!wPk>(%6snHm#f~v`-y?nTU)KLVo$J{DIO0-c5f&WBmrLr_hQ2+1t!SRwBy1!#MULyq3I6TKMmXs~^ltC^afmdJWzKY0YlYaW^ zJz(4)d18}iV6}to-M|R$M;@(9PE@-uB1ud<%gd0)xQpZJT#@Mzz(C`z!yQLc0K9M4A?kSfQFoy( z7XyH{^_r!y{1x$OZ5|SnT9GchAVT61G+jYMO|&Kjq^UxrTY_|wozHS!k4%Io~P4s(4$fP~;#7TW(E^O7|` zncXo~6In}S#uVK1L$nkffJ~u9W{%|H;OYjM1_hg0XP&VL-x3egwkoy+ybGO{R&!Px z`ufJZUNCr(E`5gonIOTV^)WuV&J%vLy0pf+Q&~|BoyG7q)Q0(BL683|uP!y&%^jcX zMDe!~TkF2gIb*>iUWKQDl|iO@JN~pm;*;Bg6W2+U1(@l!3(I-gy6Vua_~IDu!!h9c zUVdtp$~WbA_`Q{+GZ>Y~<8+r+I&4d4`Wshf*-L9n>I?cRPrQX{mX0cur5m`z1w6$3 zd>5DLYbHJCTw)jb^ht!x8; z+QG|B+uX7{cJQ}$mh=tB6=H{dfl{gz?Ax(nbe81mj59?z)%puG)!)@gPH?I z-YtDamhJ85Z5_PJyN$(Pp&*rzun89_HUCwr*%~IPH_`PHG#?G`=7}r|k-oao zJ9uY+`*b1%2fHanE%7H$IXS#fdBq>Ri?Kh`4sd!AGI*;uicS*!n|ngp$`W~5sI+ym z=f_1&&s}rH&9L*0XZJH+{lpF5!(866TkREHHxxQn4W2La;Kr$rsv6mPTt_7AMN{P( zgDg0NA`-qGDe=3P9t-=WZJf<&F z3vL~9d%89u6q-Q=mYP74U? zkh$HS+^fDFnc6rp^6YV8H5`^Tjw;<}kV_gb)Z2vpja4Esp<*lBN`PeR zNKpWDe>eKEe0b8y?}$pMTudbO$>-JK+W(12K2kpW9e(tsSH2G6 z@?w5?S`FTGZk8`N6KLb_%QWE`U_-qK^uch)@ZY;`d&5BNniVHJ@u``BKMB=hW2yf) z%S)OBduMgx+1W4W>L*p~_Y@EK4(=3maJN`8D*8vQ3c`ZWlZm+9Y_N|9w=VM}E&Kfz ze-iYj-Ndj;O1Q-f_j^#aR_T$uL>*aF@1t~jsjqIi%;NdOe)UTf4KRM-+v2#Q*-7?A zAD7v}`IZu+=+I27aMLcOU_UZHAj}824o-`uk<~(9>`VwjNX3=4`~FE4t>fSeiy0(; z(Fcn3-=8LjfGYu(Rg$I-$CDITq`+N4p7j!PtVw~|^2F|FzrS~3?;cbO`K49LBRbds*=#;6WI8&*DLFWJMNQw?aM4)wSLX1i^j*yoJPiF!(@bsi#i)>0 z?&WVq;}oSmaW8uADT!%oy^P6Q5vA5LwKT?D%jXn4B0a)$TNpVfXlbLxs=f zz=UxNcl(-cCHIC;fMix8-9MX+_d4lw!5D;%iVfyfhhifSCd{}tU@^Wao^;z98etP) z_<=wb5f?r6U>o+)+37a(n%U**TITuq{csoy_4_bKw?!*yvB+5H}Ap5`Erco~|^%a!tu@F4!t^S}|e#rw$2 zaHpuIG37+1-|>p$?dk767vFo8h{yNyY(5pd7UwwIS96~*kjL9!((d4dGSSS;*kvJG zDlrT1?5(1W<~p|!Psc(_$IepyRP}378lyC=yA;?MZ+;46Kvq-P;#u{COtaYVoL zcypNm)HMHxl??%xE&;REJ}Ow2P3;ajj1u-3JX~r`31`d;G9PU zxV~4z>U&BMutASa1_~g=GdeaFw{*7;^toi*0F)=cIyl%ZHq~5jL)g~RL1V@zBBk3A z9h~6*1?CMdZncxd$0p9`Vtp$tRyM6Yana9H-tO+LowphkNAFSV%Y_n$XvN(>tM`i} zvFaYC#{4a-gJ08b5xqJlYlUG@rz`+y^OsBU*#<4c=P@?1re%eJ!q6>Cx3H-UYB1 z#fCrql+{^3N+2FowiChCj9R zZsy(dxee8oZ!x-Esqu&gR8?oXR^A=ltEA=5Ap2V&?(A%bY@=mEr5lD8s*d(Xk(}E* zj#L1k@9|4tTBey_%ELT9k%VrSiK*W0mr#>jWYpA{F>o6x89MsJWPX;mz$iT_OjCC^ z9)e$-0Exc*7^#cvIA7sn-mZptETK5BX+y4Y)*N{6-<`h2;aVxfCv+~09K?)uc`=sl zNAtl)`z{0Qm!t*{as4=Zvmk5hK<0lahC?(^Et9x&%Zpl^bRzwDxPtvc9ni}QFiF@; z6%b8yw6uhKIZ`Ycl6RUWuV=Q!0+$|k3K04Sl1;D!NYjXeC)LQxc#)V!tz2fZzgTaQ zrsq$w?#-Vqp`O~&q_&TMBooNpXAf)8dscF2}LEpaV$*|2$h@w34+QG zTwjE?*gG2GR8USLNwdEYD$hp>1c~QVcc82a56)y!n-;fs(Bz;4SP$GaBQ{pFcAiVI zuy3cNd5?DA*YUN5Nu(ZR$&?{ObQ2mVEL49FIA!>hua>o_%DbyN(l^{H^~A^|$1%0Z zUxO)w#yQ%Hi-~z&L$lt_vqmH{Y~=qnpebf7pC zb$GtBq)T5c4!&CBdh^1xiYv%^;Uo&uM~MiHFV-H@S~9JeheIXYSeyou0poL#eT6Jp zL9>}TM=V(@!tVV!HR+lH7-*wjhQm24nqK{ z8jTR(2(C|h`_Wh1jzu*XuA%05s#!!URlG}@HgLN}Fj`8vz$zo0`s?80u--?O3QFM7;FqrHBXZ686$t*2oF zJ}ejV63GaxxfD3|RKpZSNuPPc1tyiGQGZLvodWoL z2}6nh(=vUODxiUxKLKc4WJsQ^)yZj<+mjX5?=6yD%K6HR9igJrDf(Pc7QU1Gcu(Iv z!x(ja55-^fioQi{7(LdND)(m=x|K3_!9Bsq8Z zyv%VaoIO)-3pYa`3CCdh@lvk=$6+B&Tot3!l+xFdP37s^;dO^Zc}R7WR(|4vtu*RE z$}yJJox)U6tG3*bURwL)is_>GKn=C)@1S0yXoL_E#H9l|e{_~58#qNf#VTTW5l5?m zR9ROHdOYykwf?1{YBveWeiL=a*C?GOt32VV8AK8Af>U&cy4d;5eAkkOPWQ=ubIj%M zo3*G;QvmLLg;Ih-m8cu7quibyF?-lGRUHlDl?U9}~ zG7}KW#d#3xCmka_Fl&`HY<=C+4h z6ZD%wv3H0kXV*wb|2`c^A(Bo+Fp^Pqwb9>X;&aVur*2B|>6&6?L4bi&;s4zjghA~XJ}SDg-xt}7dl zqH1!XUnWleB5>6(wJ^%a*DR;rTx5d7@yKxtD189x_urO{NGz*kGULX#FG!A z2#%pozw-Mh`MZ8In3IPP+Cor6lnM>X-;!cq+M>HnX#aPc>iG3<%0k-MZ)|g9>G_m@<9=PZz` zNe!Z*V*8u!Y3GV7PR5(2FEvh&En&DL_RaM+c z_sC7(u(;Aclnt6pMDN_N;?w(3+XG>)8bxbI9k=JaS|85lWVUL0GUP5>^;E^oRALQI z7M<XFlFq^Y#80tY|hL^hKr!%n&vutYxuOu4M9&ccQcwSx3*yDN{#tAW)6(+*7|p zhB|o7mB-5K%}nid*fzPjn%F4Zy{b-0Dp7Q)y_+nxvS>LQIeD?gdpA(j(zPU?H2`vY z^17ECT5N*JbHQ~386!nZ=C`Kt>*usgqIa+6Gm={|HC1dD`N-H~e*<(_BK@7y504c+ zEC?(9fnqUi^P1W?P9MFkaBPgS1Xf3%U zwt@55^GYKPJenk;I~%p5Yvwfj?N*vbUVnb@YwtLBHMw)H zLh~K-hFo3Oop9oO1wq)T`>9_?8I1^-nEdYt8;@Z6mcU)0F~3r{PPc|3y6i?Uv}OE2 z6Rq&YlY{f$Q=3PlyWK`P{*#Uc^BU%Qb41b^Fkw%U$v z6IRyBtQ(dyaj8WtRnI!@!_cKw4jbS&Q=?ozIV`zz+F_Ny!l!f?H=6eM zR~L@V(wTNX&}pUTgTu8qT5f_1p?%7C71RngC((S^QnkVt$~mvJz$QQ2Z_;0S!SV;) zyBGQ!j(kS$2%h~8za$z&W5B0{Odh;m7A_n z|KAj@y;gjyb;xCEd%Y6-Q$kv`&P*<6=u_1k1qgviINlNMPn)U2&PD5mHSTm-8O6M| zQgogkPWloh2KQOIGY$b;L_*TAjb2QLV}11KyEHzpLx&rh@IxM$>84vSWQxG!gIByW z;ODlL_1#3K+$xUa24M!k?Xa7nNGdKum?nMYO>0uRg8hea@b%3HQw~ii*2h^SBPHK^TX+rND;(g4fu-@KzmMsp#~O z#nSm3%x#I$&02=NoCV%kABJR-n< z5We8fo4v76}gkMPF9(R`}T3Is9KtTyE^uit!*RRt36WZ0_M!Z_fLL@iayS$ z5~0ZprBbY^_riT8NlC~%=+kXuSM1pU!6gh|X?~bkKb#R1{Ta00x5lGa*X;54`rgpu zc9)G--mq^JvZ5Nqb1Z_gzo{iLltg$d+KH2#>$isGUfY6^us6P*YoCdF7B7_eQ4X9G z{>WsC!1+9<9zz=yNY@Nb*qmL%{pbSn#jQ2_8U@auvD>+2RyHl$F_@8tj`=XV5UWF# z9{;g++pA2McD8w~DH!mg9rlulY&+@R$(A{q2q>hJK3tm!VEcQ0#(V*V12b;aD%R9S zw!0b6_ug8RfAn<&OH)|SbX1-BYs4=}$W?Yr`hd)=Gk;Go>4d_xlVa~yIU9FE7d+B6 zd>}v#ypSZZez`4B*i>Ksi+E3;o_SyV_7u>lN?|n_II>&Y)30wRk4&;u9dJy1^Df?+g|`%IWVN=O48I#UpV7E(E$tHfpN&y7 z2`cXcR&MHbY4779ZKbU*0mOeC-vvv}+7U-1qIU{~rZXHpwL!>%{z{H6#~(vdM@sCC zH=Dhs1>Ay#ZYr(?{t?OL+T*{&_n6ubFFga^UkeN{?E>Zea~g$}PK;Nebh{ceOy=MM zYh&p@f}rn?U}ws~OD>awLoz<%ud$7RsbD6RKxGdOgAWMBCnV=Oxv^>NoD|~4Cq7ine_^qwJ1v1IH<9I94P5o<+{Sg@RHXoP z&36g(7zxfB{K=&r2!RiYhwqX03V8V6hV!{&273_R86#=J_uI&ZI3!&iLyGeY9=WL~ zpDHny)d?S%d$9PVXRg@qG!UtTF^tg#3E7d&)e?Iorf0C{OBs8vFzoa0+10CJPJA><|#sq=f?7S(8BY zV&1e(By93?uy|6*W#CE~)221s#szb6q!Pkm;}|etGbia2`NMmSZH@x|H;;^RTaA`R z9{@Mk2AxX8Z#fZ~zBLkA)qlz-clIFnog#cz-I7YCLk?ngW8*(hJ1h*={%S!U zZ(fV^XQf{-4-IS0nz6zebo~#SPS^P@jN`5uE^N4$ys(RTw^GM3s6a(T zM2Ykr+s#w9K}=~qx~Q23zt78BP^ZY{cq!UgWHMxFVA=jnRTrd z(9KH@`bkg_5!N&SQ{|Jbs6V{Fq*?~9Vqz&8Xz9l_;euSmG^d52Bf{WlidEhd@pL*e zjLGU8_-OCW=C9-tbNop_$P6RrAP@e{jQ(XtS%1 zF^FTaWCGPV+k%bx#^=roL#~SS(utVod?HWTbnFWp-t8GtT)Tr)O?r-xnBs1&jEaSP ze2HEAe+HDRo#Q*|^kH7$B^~K%A&(MYCROVrsTzcvfoDN=bYu2#14#sNCma~hiXG+l zXv~^L-jfs@e)Lq1uFWAj_q6p{Di%%6hKIjj$%kOS`H` z_g`97_n zQ5`~(`<~k!T#CK77G`f_-**4F3Uu9-76}(9t*5$<>@_U@b@-YP^w7niIOLoph{r|J zY;FLQ*xqsE+8+d1xn5;qsA){R3lc&q>0R^STXpIs%kMMd&@Fp|dL^he5ZZVs;$~$w zJT{8vm-k&Pje4Z0g)mGi+LB34?o$YAeI-F~KqJHD;|?%(5pGl1kKAvt*!Q#rd}S~V zb#u@-)>=q^fDYGH^jq!uG_PqEU0Qr_wp14eVnFX9lKST(z?KiQ(LS7r>26iX+k!?Q zUm;sox}@;6@qgWll~1!rrW&YLlqYvo zB_Z$yYI;Y67R5g}1>B*KF%PFZpc{7d8G1SjM7cU^oTuyTqu7~iaq7u_h!3ZfFVQ!3 zT>uo*jiS`(8 z;&7p`wjC}@>n50gfFy(^55i#h6Apo1RBe!1ai2)I0H*>*-uenPV>jDofWGef<( zkXFp5zzq|OOgCPUzdU`_ZfT<|)0qGp3Yv)20nL#6qzbk$L&kEBmg;@td-8!!8vZ^n zqZ(dQv+Vb@*LGt=*Rc=mJi{bj{>nfvb=FTHDkz>D#_7$t?26y-D?sXMJl&7wihL%@1#jK5J5t(BzV@crZdzQ3t*A8`QnzzrwdKbz z=ukG`Vg$%OAEG-ZB9gkZN}o1b07;)=QV(5M7S&6GP_{ep!te5$mK;~`Jcrq;}mPPGQ^jQJvtY%_bVbD`gJ!QyzP zejbICdZo^asucJz>(nY>rF1s%DaBBCP3JY%L{Xxq^7nVWNi zVHSBCe&cF|Psw7_8zGvrGvQU50R?=|dyab5S2u&D+iYz*$;Y;=qvwJLo{_>v>lo{l zz!@QIm-BWngSlv^Xl2H6A$|D1!`yAH2J|x<{6}dm|Cq0t_jWk1%9tmQ0QFCOkCl0T zuaQvb6Xy(MiPhpSM&~dfCD%tNX}N};KIT%8Bqw9N9F#s+V6u_{42yC@#w~;ZPW^G! z&3?UVE5&LFec?@Iv9CZUZ9l!h=Uzt6DthK6zGRVv&l?W8PwG0Td0aFZ(DKSP9&<9E zgailB+D?L9?{C0eso!}}8st&uBd9lP*W0ikE0~+q7q{XlrzLM$;S}G~`)cHE#W2Np zSt+)h4~Cx#^&z_Nb?arwBtu;{t38rwk-jTdM(OAsF_BLjx$X=uK@o?$k8_f8?=3Y9 z2mzma$-ByvHjdgq5~2*zSMxqTH09x<_P0%f5=ci*Y?)wGF6D$SE%~c^Du(II2FnJy zy&EA&`g9O`hCbGx%_DnQ_t*_U3*Ut9O0sH)hpwez=4uf>=41qQ=tghG)q1Hu%*m{I z)xmKFd3lV>ehxXkV};MExY0Qm51w%hmzAh9g_|6MT&E5hcPJ(IRsWFfLc~f?!R$L_ zkJPe2S8Iu+rQl!Dn#~^W35pskS886Z5l1QzgZICBvqq$J{t82A{of4S!(s+5hVmcz zV3Qa6)V;J>PP$b%b?I8jVG{s<=n)B<&Ins(UJ%ebAGlpymAz@r9b))qlifw2tdx-( zonQ-5zMG>d!(wO+@NYZ17h{gqm6KI|9g3%mN{~C!r5Fcpwph2=;z!>$c;1Lx|8tbb zr)U#

+)K<(ZxG;~|v6MoT zdm4$`FmJ*6nulER9~)a^ZaKHLhK|HqJF{fWtgt?}igpe2_;zj(=sAWG<=B(Eac@DI zKL#>SS{bWhW^I-@WBz4L0OqzA{xc5wthqw&Pfs!Gd{J)(T%-m8akZrQWBCm{V_!)|G;jruuuHf+vItbHyZ z<+iq7_gt~aeC|o>v-Yb0n$cA&W_gQ0hD>hGGaxnX#p}ExVE)hXv2%=h_`6rA)!*r7 zEe~v`9Rfzg97A3_&b+g_Cyv~jvpakyVU)FlhHje+A7$o1v%R*k-;P5|VJu~<5D(PQ zJhsm17*XOq?h);{k`i-_(O-Nf`PjC%5Jz4UEIEwbnXy+0%^mSfU~ln>95%Shoyiq2 zrD>U%gVti&^QUspDX+dmnC6eGO>eCcil0Ji>)Fd*V@|!@8@6(38!6vm?7ikTB4o}O zi7`5^L6sd+X3EQvi+DwujxX(KX)C%(i zWp2RFo-+5eTrsj{kj&;mUzr3du&0jMIdaSL}5VuNv zT(K{44}H|Pwov=p*>8Gg4f7P2a_mbZ$SJ0Q^g055-sC5k)xjWV{N8NF=F@%J>pLQJLnj2Jm@6ERna$MBjamaCRjqWlsGTC3N zH7*SB;C=S<_!}ZY`9Zw^ZmI!(er9xO?o4%aUS<#$97HH?D=e zp0iNqT!W1zXL;H`$Bf#KBja-nVequZK=s%ow=xg)fv_g@_Rmp*aiw6e7bhm~&k?tF zPPUXk;LXvxhEk84Ji~%zWLlrk22s z1CT%FddwXnV+ey#IqyZ8zQ=fW-D&-0&spu7hGz60VM=rk#rMBA&Sam7t#}7e$~4Cg z{$C?OYXvm4GPmZ%*cpsF>>2gG2OL2fvvX_)N$fLYnq3L&WPVK4%|DVbc$)c?Xf45) zxi%!#Pw~`X@BGEx8+d#T&4`x9nCIVfIegD0iI?;0#GK2FbMM`mzhhe7kK+w^u3^s< zg?eGl}{kv3-d8>6vkO_k8Gc4paFGy5;* znTx(+U`gJAwJ=YO(mF#d$&Y&sWsND;m4fJN2|MRy??~e@cNETxvx9R7Dbt#khT{s6 z9Cc23&oov%_F7{cU<@nPajvnHzOr)p9x*jy4cVu>rbu_r%R6LE6`eSyD$LA*1bBtb z`?2>#>>3$Rcx)xeFt!lwoN?PbuVI%y62wj3OM!K+(ft{>WMvxL{(U4llo2*A@fkTr za1WHtn}bN+8R=DRuDP_FlbB43>wsx4jj=NpBLC0a(lmm~@^V%)ytwGr)72<78^) zeE&9rpx~WBw`A4EghJh^d?xR`xVJMX^vH4QYOZP3ke7Ju+QThqrDT>m$4K4}d(&vH zf$A_PR_9wYPhsS3);#Ap`QKABDo&B`mbTdQUptm9jg^PGr=(0@5&mhd#Euy^uKFJH zP-muP^*yHo#agNOeC&m#yLMvV8xu|N_85pv{l+N;EBP58r-VouZA%cXRLz_XvnGRfch(2^JQbKjGnFoxm5zvjf|7||1G z%&na~M(WZJQ-dxIW%{$T9QIlBY9bGvmb-S+#Tsj~CyaT97G}PE$m4x|#sS5+wwBbH zn|E>!{Hi;rf@_)krfn@qxVl!PO`1#DXiO2Bxt7Mj+8HcpuMO6+hYZZh(a>wJxud#b zJX@S2GBB=W>$lf7(ppR0er=HcH8#|69sw|6E@{ZMV%qTCx%@cpjlRDI4s#kKYHTNU z?6fCd-kv*WGtX&(F_$#V+pF4V2$AZxhb15bat*f!)IBg#rm_MhX!@5`M%2?xg zb&l=UoOj^ZoFm&StqA!%clg6gq5gD_2)`W@82;aZ>SC=p*|auzPTc#%E{^%*G$(fa z-1FUc&BfO+awfqUlUf)#Dt=8V@VTc9!rq&EZ4LyyG6p!z$P*W2rugkKwj%bO0p&IC zFpfJ0Sm;}WtS@Xq)Uy_Z)3tndp5W1PO1jm^f7hp|QFK?arIOj_AS#y1StX6{1f)3z!LTb>4P$04jWW!j^*+&W9)Jo8O3zR8PT~$9@wA3hhS!r ztQ1z7SsPQ(Z-*%Go<@Y}4@u-WEQzGKwqoPRdoUE8ha*-0|Hr?d6v+x@kL=3ma=A8@ zomDAYwrkvL&y2E(E1PSEtU|cBc3IcD_gaN(TyeRGYh7FPyTAY8yw5rB*X#LwJnZTo z)FDpxwnyPNeAY2-(l2FSb4K3)5qr{`QKi%Pr0>C)>W>QGS!lx4%?HMb2lA%DGcZD) z6aDKRHL;p7UGfL_y41pK>fWGP))sFPsXzhD6T~5O4rTfkR{#XzW)q=O*o5geohw(& ziw}x%uXgjqm)xl@kAc41PH1TbFiq>eH0XKQqMM=*Rx-VN=KfLMMQNZ_$wi7-Rlh zti#raOR`i2FP7H#Vi&lqc>G}ZuhiH@r{CDiNss*(-)0)MYB161DA|?Tc1?B<|h zt@=g9~A@PrTOE}hbFR6kR34&U;2 zCH*+g_ESOO)2=+A(#!lSxBUw{ECW|#tBYN@BO*lQ4zq|~O!W{}zNq~KPCuomn2wDI zFHTHHOToDX-6KB+XE<_j#O%%d=LenxZJj|?o;LsO0KFK4DL_U0nb4klL&a&Z~SYXC&lKu>u}ll^Sz*rB4Y zK6cDZ6Jh#uKK-vtIxG3CIja*DleN&??5}>zDWe_6)+(DSHZE;Ero(73sP>mUUcAfz z_Y~o{to0AuAKkh|J>hfeW~iRz7q?v@$?KpD11H@K27k2AB|_Ub0-j+Wh>6SvFb*fG zIph*KoZyDFm|ov`tKxU~0}`KYobf-ggcC2KdN28QXKeJz zevsOq?N7h#nUgOZ86znP!!HZ`9oL;|B2SP(b+6upXu8MPA1G7*v9-u(R3l}rfP6oj z(DsQre``Q;mE8yMs=W7x>1!MiJy5&5i0gk!-%S1ZldwnXrGzlo+Gr zp!-&T)KdK$&s+|Oj}jgU^QU)BmeCnP^N_E*Kn6aw>cm)cjq}BX5KOT{Hd5{UfNPk7!LlVRakK9H!_9V@O^sZlD@VC*G_v#*^KtLP2-}Xk zlmSw0bKLD`qxxzBYjU|DYRw2dr_0+|!$f&7wUq@`vmB=9qVr`tmeb14-fJU4sZ{EM zkJ(8IJW-}r0a@bgkWtpD{iVp>c{V~&b5B`B${;__7@5Qe6M3&%kAGekhwg_m$Q7p_ z`_9;K8z>rtC4Q$Gd1W*e@Q#Q*7!z_(U+)~@*Rb;y)-7cd#gAeWDV&A}8%-@BorHXA zSB6zm+e`e3ZD^y{V8NhzixN8F;-4()GsUay7cq9U+X{N`z~N!+=d3!elCAXeqeZf> zQEsyOdcq#(yiS-7^g}aS)9(dl0X4}rbYqmR10OI#_C{A18;jAUc?L!XwV7P69cI7O z5F<`6T_hW|L9*Svp$c0}?V@Sg8xi`VG;nt;or*fb1{mZw)>9!XmiLN#bjQ2D76}}D zwSvwciD2+42gwM|(<_?^rijEMCJ&DF^??sb40vRm3qTVY`yE-MzmDt@`dm7@A1zH23}(>UW0- zSCd*Q1;=!w@d{#qLkGl0?Md8XKt6oYQh z&oT?RZtGq)9iNhDX(NRshJV+WL2PubzN$FId{Gfq>d&_{OG@Lg@89TCm2qF@N&K!$ zMu5Tey%`Q%v$%Xuk2=>RAobrdeT`30QZiD88sN#zx+MJZ@0jIQv8T~dMVEYkk8a6` zzf@Kql5JbBkX-MT!py+4ajk`yFD;sB0n=mqXua;rxR-!(E9 zEeV;5(6)Akbm4xBc`0Bl^JND0F=Tk_6cn?Z{$2aONM+zn2V=s(FGo*0SEZtJvVdG;j}87I zd)&V~TkS0YbxmR}b=aS;e(wP^-sQzkKkos+(1u#zpW8eKg5%VVE{Q5K(% zX-SVrkcK*t?;t9k|3FYD-M;EKZpb(X8; zJYc$it=8ptB)xGikl#c5$BuNxf=2Pwg@R~nWsaGk#+(ogGKTxj1^Rppqe7`N#-@YJ zw)<@+%G^I1p2)wXUU@@su}hJz%Ut*V1;Zsc?Sy=92LB}4_F+ACyH>ji(_@^YFgJ7*ALNeHUUDl zq~>~4K(CDlOdiTvtrCese1;IK`p@IVE^t@ZXY{Fa|Gc5;e})V`Nvc7}?tcF_C||z-wNk@ex##@*EVP@Na|X=y*aZqLfy@IghP{FfsfYpb7;NQN|#{%xV(c(AeTw|us+ zjoC8!K+k~F2B6IPhX%WFFsWDJIAv-+qty-?i{N4V`P^FHJ2=+7P|fteDz%`Z(~nkZ z(lCDnNGvtnmSFo1yOEJ0mf-jij+d6`!gNeS2>2Jv0jQ?bPQUl3`7z@y?I>a0`J^$Q z9h=0~^*>JBp){1v@^?Vcwk8wV|DKjsm1^L#0glmgg_!4GJLS*E3l-mm2_McVu5e5i zicbd%i|lgij7-_$T*4|df2fgL2XNeWfn(zZ-_v&tbSH|OwDr=|z=v0cqvGz5ZWlJT8B{HvYiv4Eh$<@R4;k-tgy|;OOAwFZNAMQACjM}~^Quf^Z z^SyEfY^3qyAp^P?d3@z*B)%HBT|L$5;6R2;D@kGePgTX7jOO((Ls3~j8)!PS;~wrZ z=~Yt>$ag^e%61WRm3l45Fh9k-zySvZ?M@^&X&?6Bp72Icm-Mn}-&rC{u989?h$YzB#yM@zcoKnu20x9g zwjB)wkcupElS>^i56R1TwqDRVmxo#C_UrCWR{vA3$T;U6|7>*U2Y0;+$*>CLj;~U2d%>*UuerrZ>>C$=dL$N+T8ycmui04y<8NYon zPii}KPM_bR^M5NwO6V*lj?E5RzST~yX+org(3_kFi22k~XnI411TIibgvH)ZbqbH)$FQKeSNGiLs}+?rFEt7Yl%G?_W?2Z zXDX3^_gV|+;4|GoV>#uVYZnfAD_1S8$jhpfYHbH1vp-3{M7{2Mt_+RvKE43~_JY)9 z9`BJj9FqX_xf@-k2?NFx$|_QktKBO@Aq&xa0Su8DZK&-10%qHXOsSo%NbPk)>0OFf zthV)-=m0~oK6s@1u>kRR(V`6Gu5B>XB)?#mNd6K|pPE&V9C4hxcmSrR-O6O^bWfz? z_mC9m%p*MCu?_hwZRq_;j!kJI`D>sDWZ{Il_x$JL*3EFt+NZ+!=aY+$*F|bM(-nyi zugtB|g~tB=;T!e#xQ_YRJ{5d9H@xr6XkMzwmOIDLCkxq5{jA9L2UnPJB>u5{Qnn0Y zge|)%qQ!ir;1abkr~KNfu6YSSm_9}O96rKMz*NeFUlkIx0w6WGY#q%BroM9#DGRJR zIm`vxmcc`$J~Fb`ax%nK$v8ZJOJ7|U$`lN16=PiD-I7pPQ>~GXt~s=^;_yhOY6@gL zTWt#Z>@|#V2$i^fQgBjb|6$+L+j##^BO%+y#_@H8PHJm&;H_!Kt-IoA-LC8@Y#-0) zRHzQ_vXQ18H>dYSOsA)!o^dm?$m~3*D6z&7d$!ihc730hZqMUUY*W#+;6AMf%3eZZ zSe2MS;{JNzLM$)pA|{+N2@DDw1`xid-&swB-}Qjx#xxwDwm&95fyVhq*9YxTs(xM@I#bi00UA8_3wTTX1m93JT@ZCD+h7XN;3HjVC~v1B6% zG&FA!(hOll4=FSnyuDn!cm%0p~GaKb=BCJB$!Axnh z3TF2W@zuPpb;DWp!28Rb2v(O>#bFg0UCM*Q;Z%1N*Q&uto_jE8Pl0~}Dg?uxT`Wi& zM_x*^A39m%x`YfQ4uC=FyA;l>bc)@vQnu*7H_E|E5ZG76vQNhOk(7PSy>In`uXSXM%e4`}NtaobWo4;a2wCE&9+fN*3d| zJ3dfvF*?OlLZrC`b$y4CD%6+0_MoGpAgtiY)kBOySzeC&AY7WtmHPEw$7CLkUm#L+ zQ1M)rZ6Q1%G6p)3`KvA4`--8@791Pgtk4+Y!AG_PX?`wmPrvPS)5AsFx%ns~&ilvh z3UzMu{_FYis=XgI(RyYx`VdXh*h~RjpJndJzEv)_E}*X|Nor~ zFXN_(@0Bq|{L0DxDFcP-g9+n>&`++;E9%ip~7 zujEr+9CwhPUKTzulpyZUcg^%~E|6KBzTN#)Gzi(`uzLG8MYDlISsh$jc2(UiDM6qt zhZ)Y|JO_XDH_E497>1k$Yn2o?`98}Z?DSs*{;wB4Q4lbucUoBb=5RKe7y@Njyc2&% z8=cV!&}u$bBc`l#>2dCC@G2d5gBG=IonZ;AQsp8`JxJ`%_zPJl-4I!Tt{WYG08(?j2qJsIrdg7uCuQ ztKum>5vam&F{dsMCq4t+?eeyyuHgY#Dr_=+l$0;Q-k;hW`d7)@y z2ycHR@#w$(PP6tybB|pG@KCqh@s6gsxOhkJ*08e|Vvh)1B z+2OAnN1vdAL}xT`DlDWBxELI(wGYsFxx?du34_T_sEVPcIF^ZaMlsp^?APY-n^yr$ z4bgcmQEqu1%zmFaPn(4P{NQ2Bz2s>q64$mNwfD9|cxW4ak{_2TX*9YO`#ksDzBi{B zlufb(zK!TlMFIrpo-c8a%0geHOIHX#XS|3E+A*v{$ho;fU>(2-WcXJMbNbyHEDr8HaYK?3SK(B|CH&WY~E=2BlGr9H@OttLW ze64WD`Xl&t?!B`va|&b>_7mZ$yQ}^8Pf^*jAn|TiqhHfNQpkyT%0i{GgSl7XmYb)i zUGJTf!=U1`1%;q1XDN6Zca+m5ZHAo#V`L`3eELyB$p7}{H8KL7mpATEScsgXY(9g_ zYsG%Kc+UB12lSl=isjT7&HB~L;Ak?3)qRa;blw*&P}RgfeP(NM?hD&U?0O0bv?)6e zFu)D9i+P*yE3$`f*Hq-{6ukS;5$pBMuup+}1;RpeyVr>rxW2itsB38&_|^nIlp-;0 zC~Ky1x6n&#br}mkm3672*bIOYDDz*u$1i_brOA2=JFQdRv1eA~3hx`sN7)COPKkZ0 z&KRHH3NE6g1@DtLCf=djZoj0ppN*icu6v-*X1d~9CauWdJdy<}ZZpF2a<4E>%5Y6$v9-c&h?lT zjZ>WHn%*FF(+3mz2tP;)>$n z4wq$MUQ>@o4;cZMd}GQBR0dUs62E5KrzB2)e+oh4_gNv2N){TC3ri`u^0$we=hZ|- z^>NWx-gUn|D44mgbL;;PKYMmVmCDN%Xe8mx$+K>gAdA9=T9cppLz`20D{Fqu zUw}bcu8JY|)ef`7 z&7CDybA)4-@qi4~H>66BJ>^p)M--mb|rFGCyLY)LjA*TAc z*7b-JTa(91JHKKT*yx=t%T7Eh?eh?W_pfO6i5`?FkTlJ7xTQ%@$Zm3y3^H@Vaz4u) zN@{GNdbZ4Y_Q&tDB7z6;2g@Dm5Rd{p;Vey!(BrIf@>0C$&{p%}4B=20X4ub+>Yf}H zSdzfle1%7??i-Yp%B;$5AhqLA7SZn5K-E_CGj_E*=yD-BGGf#q)@%q{X2jZe(?U412%mw0rZPZn^GN^Sr5 z(XWDGQ$4=o9j5t+BWh)fvnSGSTk)`~F>sd0v@{9UnMAx zxY=eRz1h6Jeb~A+T+Z~5XWl%k;=n|&g1mJP#-~rM#(!ESn(r#5;X9Y!H*cmVSFG_8 zQ$y6Qr=z1YVRa&{cf_!N^V)7p8~3&8BSVzMZ3`LA2;>egE`#mAh8Gew=-mInIglz$7IRe4lM8JN<@IM`v>yK zsL3~>JP@s_?*O8j6Rl&Rx{Km)vk9Y(l{h9ua;>}>e0pX*dLJTXn3XNsd(IhB>l{OS z0&3~DT1cA?M)Q8GC#nr6&*!U&#N@EAsQgf4Z&_FjEfXqr&&(lAXijCkQ}P+%GPT9d&SQWm<*o90Mzx zr)O=cv%78)+|ja18*+e|S7gn^;kh2n*&E4SsRxcYBzrs#Bo%&Qixcd> zwbkT-Z9SGnMvTI(sP1cVklW+EFLgUKpK!8G-ccBLRWnEjGP*1 ziOJ>e4b`mb?j&CSYt8xu(2wi*{-Rv%;NIqF5(i}4@Q;jH>6~S?sgk5m7k|a80}rU- zXSVfm+k`~*`!t1zoYw#ocyZH*Og*2Zr(aHSgf@%!Z;nzL7>^enc|r`sd%Fg2T^S0< zPyiq+KtbWd5x-Z07|o6E>>C3&skv6nsraA^)e-lct3D|7-v61LSRbY%J<@!_cD7K) zrfyW-j;Qopvjhr?O5CgK&?&&q3jFke%H=A9t&g7x{dI!gIOJ{=9JI`MuVQT1&?SA< zUM9OhPisJldl5KLpTaE4EbOqaaWam*rnDxXRYf`5)J!ddIg+RVVh!+I!GynX+Bd=? zZ4>GmMq35I8<0n|$DsA5nWa9bcM=VsbBOp&xcG+uqrwAq8uZm-!^(rW)B1MP+jF{! zu8P7wC?%F$(tPxk1LFc2u5(9|JAYD&VQpFk#P-5)v&ip;1&WgB8N12-F(Ua#TBa}L z;G;4z*wY!KPL5X@n;7Xd$-62@vVE|-KUwe+a>z#*cK|$<*|aLDQ;U}_{Qt~Rp+-BDGLVoolQCVdt_8;{P7 z<`){H&u&eB+KUI;N`1T6mML`ZXOVyI`>|(VaVGb``XbobGxCdq_H#FGAy!!E`y;3IY1(Db1piVk1A{RIU)* z`Td(UWA&Uj^4R-x*G{G%{%V%886!C*v#;=7T3R_d*T53xd|lVfr87mxhE`i#K5by~ z^Q3q0R~Hj~o%8)}LM5__;Vhk|vuA2RnkT2ZW&URt9X#I9i6H=L2TjqsysL97?GAc6 zjVDOENi1IE0~`Cf8-}4o=eh_7t;^3@>pb!wo)Hr6TSeO$)h8e)ouwLt1rk>6&kYi%MExaP5w5Xa6|GsI69<}A^gpIu7rPyghRyFpDV!0fiy&1(x-5-pvXM`UR?i)??Z1m2>R>NjqPq59z1eH28Y%Oj;EIB??#Y$m5zcsscF6Bm zw&z#8g!VvZ!#3rqMjhB_VE~2E1gyNZ!xKtn&!~k|3o?Lm|2G=(NMyS)@bDG?)Ke!X z#Yz<#;lI|KXC8(PsqJ9;NxY&kl(9BO8r0O2S2F%i^jz%1pJIe+tH!1tpwz!8AvyuIBF}S>Xm%uf>|0a}S0VvfLVIqh4u(LtI-|oO6 z;zHRMwU4v~@QUrni&Ucik^$`<)UpJ)BZETnuZsF<-6dW(lBibKmF!8s;Z4Hrhu6bD z-bH?y(MxN%N17c0vcCOAPskRpUEWm=JlC?|4);P^45&<}TNh|7JE0{%wdJ&v;6k$y z=f@>E^5fdi2=1U{0i*);l@DLvM|)^ME-2gOD&MNiO)@sAh0Si@V_6nOIY8)U4Kyh1 zc&>cI0N%m=7Lr$X_K13IPceREarPR|&zt$1p)kdn)~K6W{9l>v)&M<-^%!DGhJlb% zO=>6GX$0I&2C@tPN6`rCp#ADYF_6t1LXlXXSOW{iwWCiO%Tk>!A78gI;|16z7_oMFg^5J3j*8s+lX%2_E1&`9~N6 zqVFf!lAtU1C=9RMvVoli_RAXn{n_O55(`fX_r$7pHn8>w7QZkR)TT$Kv%L>UB!3T( z?3`1hR`Q-pYKx$!jpCwYT=JvEAMV~r=U!?2F{fcBCS^FOMqUJ^y5=+4AdP^$V&%^0Tw(swR6t+}Ut>Kf#?9l2_{4CJV!I+sMi`pO*pCzzeJ4WB4yB4Mcm;{*0Y@3hf2Qy@08q;N`hPaQV zzSY!94}SPjobN6GZc9y@5w_x($3Q>YB~vQ=_MBUHf{c6;nBzJR(0vyH49larzW!^o zJ~yBhxg@2Q_W?}9wD&5MJ+W3>En%WH10C#uP_F!X=P_PP_tW0xTP6Uv%8g~fWRd%5 zMwUZAW=vXxUfpc+l@;!vs`lYqn~O#-*u<6!d+NzE69xxNW+{)(r(+(v&=6zJ*rK3x zvQ^ToBl>c#ksqnZc|DQUl-ITV>DrlSZNt%4KKV@nz07Baa&mOw81s>Tu?N{^A zM3Id$W}U~L`RTt%@2iqmYldNmZDks zXhmxEmBg)16GL8~MAhpK2Fgz~iZ>YI6K5gPj|c~Qtb zY}g4xis@*mtO|V={mz9RE>4=QZzMv_^~KMi10iV?ZVTRkThhL^_FwMAZup}cBzbwe zs|Af-p%jU4>iacEh#-aep{**H0sBQn&d#E~ILf(}gVZi+O&*@QbV+vJZWHsry0sbM z)Ze+z7r!vIn0Om1GTu}~te>6RjC_~OyT}>Dw^RHC1ol*n%hF~ce(004Gw$zb$=-II z?V+~A=1x**@>ff^P8VqZ`*0@jD7EpG9bA`EPSe!e*ggt~y0R-Bj7Vo2wTH!D^1& zVFO*7mw}yN|IP6Sb`LKN*E`oq9P{aTzIy{ylQ4`e&Jl9?t0(@&UeXS(W)|#+W?dB5 zX{&0;Y|3;{>a+AoP>(xJQkM^&Uh3(6P~NkS09Nd9mfe+uF?=YNXWX|`=DEks8U`|W zJfGeY=9ZFw1`um2&z$ze!lQ{YFxVG>!+#%D(}Nd_UZlOl+9x)(C|vNmJ%`kav~+9y zJwm$;-b=<+w5>lq=QEl05H=`K6h&oLVD)t`f?Q!119|K_1Dv5+9+kgpn=0Z0c{H-! zTpCKd`&^89djurDb*&W>Uzu3|&z-;7oguW@wD>Y+-qJfK$n zZC$tHiEi}-zu0VN{ojhknUyEgMjuaH;Oms6Jz=8-Z0buMlChAjq}US}Kaz2lS4b^b zB6YlZF!?ye11wh4oYw`SALq_^?1fcqR%9dg@i&->X(=prBcm?hX>XtscU#qhTz}5B zH7O8vk0~GLCv;oWu65;IZS2QR7JXLE*Ks-OG@pNSY`F!IR1;JPLy)3qIT& z&Y-}}g2c4@5Y;o~$`8XZp+za|eMBni;pT_0c6Q}y9bCxgV{bzB9#$-$E7832ct%+A zqxoN#+|24FuYRr{wtic&Y`b5d>izv#QV%Qn3z;{>B=B26hm7{=UwT5NnCQJ+d7VG~ zmYb*`i*C;YWpoez5swyB7m&w#l^V?_}KzYpWwD)8>Y%%dLPSzrqNe zn2F-2uSga`Y0h;u6UZ6B0d6^7Hu>n!UFuuJyd)|xwNsvaT>A@46J9X5t47vjmXMrw zPn;fuqD-hZ+NrKRK@0tnq3>C(H*4%$q+J#ETCCys-H7-wbqi#$yxt_{e_@kU7eG>o z=GX2G;yL|QR5k#OhbK3(>3S?Ir-#!xm+-P_&(HS^ihJ6uHyxXATM$ZBliRQ@$A1Gp zKFEMFR!uhbMqC!2As_5YPAcX6c*{v{ix5${Y7%q$XHIaI8{3>xk@5dysZ6o8^2G41 zSZ~gqS--6l-ff{qKP!m*5c-kxeP{*rteJ*!%-ruvC3(2CO$V}yE71|6e^AV&VPIq< zUgA*8yweIcUCU*a3LiVFl=hVww{ql)FlvkGnyJEIri2~G%D<@5NKDK%TvIn%8|a9a zjX-l6*^#7i^4ivY^Mt!{HC-8Z&A5K)3lJTIUw=dBte)sub*j{C4;2U|EEr9EcqcWy z&zt=hv!o0Rj5(xOxlmgpZS;lIib`5taW-r9M@Yz4<^ID+k%)E1x=MI{dh-=-K|Xw6lR|%Pt~-} zQ+*Opqp}4AmAV*qTsjHgd@oH6IqwSkG;GQJS}OKEDkr{kRu}PG+6D8WRpY}2-x&~^ z+*Ou${B#&@k?sNQ?~w(6M!`0_>?Nd=lyY88V%#XCK<^7x7G0;geg9!2lqopx!Er2D-8E2o1|9ZIWOTpWY!8MUb&VF%#fX8 zq{{KbmMZ_xH21xFr#Xq}xE7;vlOz2HH0lP!Q3f!`q8&4Mpy=O6r2^iB7fN05Z$e|LkZ zW2SCK>iHT=bOXx{F$VjyQ2N(j6a@Fk_~8UFl$T7I@V>sun(J11{~yq$h-5)m=A10C zT%Ufrte;fjji#6+*)^|jS%pd&Y%b+i-1aWIXi*@!A zr&!q~kP*I<83pn(&&R1h()!UbO&wHBM-5JP?gXrRhgiQLhKsbFvo*GypWEIFpXzS+3O`EwIAu#uk3&2=UDg34r{| zIj4EVa(`M7a=lV&&+oEPKf~q!^kGnF0f<>o6(X2{19#hy**lA4U+ciNuK!>bVvv}l zbiib)2Zw&M6uonF^Sb7~ zjnY%wr-hDHK|2Nqye-``i;jF?L&F+5khVWPtI{JWfkpYs+;NQk>04OXx{@|Z&xYc4 zp{ZI12zhVNH5gGQC;y5wM(=4!!`WH*7C+FYs<_yonoeqFIjd#E>lSxa18)sjO4A6f z_xx;N`4VnKh!Z81zGC}6rgNl0j9G8>~3hO&Sr$_Ug z+{)18`ZOf*9BrZ|F1Z#%;klqJ6QjIuED}LTOIsVNhI^iAag>msD=+gwh*sE$DwS+c zp~($brnAP39L+6bp*sJR!sR>EF7t(<)4|s>{Ai~W9wI9x7M^$eJ=PyiqVa`Pr!#BM z*>VX)I@8>X%#R{Yak4H+t{QeJvg{B>A|D5kv=EI%(S821y?5U><1<<6ulPdMoi)S8 zc&06__*sJ<_P-ubH>Mr=nw3bVOTlPCK5uiqIMOe+bw9$=Y!oReStQJl`*$7amh zi&@Wfn#kh#DP&>B*Tr0Mz0dMT-+Hdrw=~4zrwXSqZ%MDnAgVXeH~F7s85_uKIxG5o z%Kz)JM6h~Gm2a`pJRhcOkb%}TSI8ki#Cd{#VDjHMd6PhwLLY&kZONwGmMfgT6^H0I z+~@C&@1Nli)&TFJG#Vu7Rr@>E?>&7MY3-Q;MXSl*bhX)<34R2f>puRm zHWE7T4H1a%C_V4Tg(8yHo_TqO zMX8rP55V-@h^SRq%#_z97R{ePwd}``C!%;uH%8a7!g7|zAA1Z`myXUpOT6gEjsPrM*qYOO?no_s(gli|;ZCBFO6J(Rd%)wGA7bvHx%^jU zrpe8GKEkd>c0nnw(S?yv=6d7j0BcYcG|$7=`#SBEvOKoaqQ3XNT# zG1XysO!6o&jS&;*w=qs;!`XO8+LtbR+G6)@1yjVjqQy8x5K%9(bbP{yu#sx=Q-8;BIfiFg{M5VVI zXH9SfCD&xLvHIc);3K~4%gaEG1&WMK&&SAMT<*mEkk9eV+4Fi)Cb>Th;~^#dEwvhP z!eMNi(H6QRciMe)7lgU7+TiD64~xcl8ASFerX_bx|Mf3UDiJ`HUm$b*&$%^R0xICB z?#a*2a1FzEj0-99@wry`2S#p2L!a)tNcCZ@zIrF)CPCWUTihlq?;e#m-Z`El7l!l) zd5U~*E#02Rg|Y;As^Ym#Z+jFlUkXat5xa6${O70+++xDdzsFgIS&-fAy$r_LbsUJC zC2G)h+*I?)l}Fw6HNI;+ZwfQ$osCw{Zd12)@apWGF6hk$CR4;?)7FbtX{Qk>zxq_; zSXPXuR)y&AEO7-+>seTFI`r#ys18m)22_I3x+XZgIM`fkpwWoUS2@M)yWeHB@A%HO z@%WVUOw0R1CT#Xg6&RZmrKedA4d{A7TD;3hRQv40g`QhYIHoPb^okES?OPYQSPXK< zGCuKf&~iSf%;2r_E^zj%7UMEe?tSS02mx_5$^26aD41dbRBH2+EHH4!B8jq64@1a{ zSyT9BPzY4>S0T`I20z;EqUrSgF6h*pH)GGGAKKJ~dbY5Xbm<5U`IL!l(&40R!gm+r zK|z_ii$Iv@Zem3j%RMU|3qzUstnaLWH%Sc9CBa5_YSc_n|M-F0E?C|2+l11C#TPVg z{b^tMsnHuZ;hS$YohOlkM6@N&#>L=lTwd@cW}a&Gt+v2aOZ?4g+}GZFbw`!*jjt|K z8hC_Bh1Ci|o_mX5A3GZFWq~Slr>$94{S8V+yE0^6@DcVQjK6+m2BP%Fw#z*1cg7D((hEa2R@K#oJ30S(iBfTev83*tsqI>gmEPi$Re36+uDT0&n)RhsMEuSIzSO2 zOQ$gY=qBt3jZ=q5%l+Mz)L2}fF?h{Ci8Ux~Wq-=XUfjXH0B@v#m>R0dZlBQ>`daa3M06N6l$_+7rVQni@OZ>rhT z$M}n35UDe|i`{+q=BT&5r^o?-0Y0vd;(_S1VmzH>=P4m;JlB>-PP8tQn0->i0 zM)lKe$3vKTX&Hb)03utbf5m!KTsAQ_V!*ktrL1mbEX2gMu0DZtdW4b^sd13BIRw!x zynA!esNi(jjtJ}>Or{`iP+_fIW^?bXLf<==J9RgaUH6B!0)oee_ZqT~-K;kq08Uli zK*wNfqY2}H>-Rq0j*D9#*5C&j?d8g3OOcHCTi=Noz;oDM8FLAwJ%Z})Ame!qm)t@n z#xOO^6)%6&=l^hVZm6lBX>uQ?dnMrsC8Ya)3VJgdDs8;j!be@I&nx8R>{VFVB)?~3 z4*z@q#*c;!7nui!{|6T+V?Q0s&;6mmo9`b}^%}~8eh2?x+wN^mw z>tt!vm;Qyb1NFM)jm8te1ztH(wRZfIG{R`bQh8fNQuvcx>CKnV2*rW@6DP%}RAWX* zJ73a7i{d0Bfoqh-zJEWirI8CVr2ojr*!}{+j^6+9U?iO0bxL9V;A+W=9pX}7p|rw7 z=2Hv~HNXrsxd=w!HMqf2HXUqFwSRLq*!T{FOx|fJu(lk1FK&q%bM9~pu@HyyNBkwM)8%H&gRCB8 zglqQnf{?^5{QPg>Z}16#S>E3T`vSY2dq3FX-c!Vl3#wW#u!@!UP79?E9ncSwx&59S z;UfLqVKyH!Xy|bZ;$!z*Aa(rg({m|SHy;kF_9C)Nk5lrM0^>O*JEET**a|IrF~(*A zz?R17>V$h}Eu0ckROeP(@;joa9oA~YE@k#hhpCDNzmK)2OZ}qi%snYfkw^0y2uLK9|+^Vo@bE&)Wc_tkqeSu7^H%j`_>zeBt@h0G=9un2t3Yz+mio1ub6DvXyOezbXvHC@ zxcU~zNb7@AMJeymQ9g(7k4CYM(pMHBOPspN7We z%=-~9&4|`E)&|B}BadfinY$auwDQ@jfHlUjpfS^w!w(zdbcY;$$Rdwez%4AJy5^+D39D#5OjX@67s6oA$p$&EZJRJwv~u4w z?sungoj%8?(_LdoA`F$8Fw$nlPV@C6Mrqp-cKpE^E7Wl&c+fZ_EI|&zPH)bgn;K#y z_u1R5J&!ToJ+_S0UqMoJZL#4$r?f_!3ng|eP5rP|Qe+>gH)m}P)|VFS%O6?sWKYoB z5`wD8+3THb?yA5#&5MxTSj4QsK=(Ja`?V4NJKCiKq6qa0Hn(;zs&5VaS76#^Ds~{-OL4%$KPR!dOGinWB zq!%{e=+ArbAZFN$9D-EZm^0jbt__edd>%VF2Zw!>%u6VCyiJdP7}%*J4LNg1cup{hhPEFl1=R zJNMS}n**jUh~=F!vmL3M^)2p1SskTN)wmCkA-2wboo`7#nzK z<%#by#&&Mr!z4UJ#qbvA`bL=pkv+}D?>NKe%^2fNEe{m`HnZ--8FSQP4P>%0COqv* zldLCgwZxl3km-)A2QhEOvb5%$@K}TQYmD{6u;*aK->Z~luHn?Yv()wZIO+~D-D+hJvF1{H%{yCcB|*5Dcc|JJ`)PA- z&DcDW>Qvv+G=2_owmmjv=NMB|XO7*^p0e0yT=Cs)kBRRvVm?w?SrB<7sN0ZJ>h?~f zf@sVYpE753NKT3Jc1#J8lD9%y-kXDJt-++V##GjybLL^M@sc<~fW#ZY9Byt+fF^Cp z%n)LX+sK(~I*hTxK8CtZnValr%{ZWz7W(ko^OkHbS%^7Sy4u=N5_*lbhq_nrP|X{h zFYaaew8rSc2-_BJF4e`F7l`HEGo@h0+_b%On)}OJ-fYGJxwrR@Ti&C}A*Tt>Hx^96 z4H1B=6zSi*T%_GfZ?|_&wXYyOw0Xcpw{lmDn#N3==9dgY{}lv^egOTgf(Zla$95hC#=l&nwOT;-nqDP zZ)B7bw$5(Yn=*K9e65-DUSnA!6gLM5jI$TE(44dQdagma6<5N?36m^$tVq6;rhLZ_ zvHEaNxV4!kUe3;Y>wl(Q#5(t|{9IdRZiWH>IKsGKn#sF*<}mEBqZGkiGwyuuWTc%l zp6?iWNGHwY+LPk4MHnG&KF=Z1KL?^o-Juw1&YhvQ z+8}|Ib+i_xZXF2XZMquVyGoe0=$h|kWSpOQ+m?0)r zs2cJ_>`0@=DF$`yF*bPKpJQ-4Ou44F=KyY8>Ah^tA-$URnr>SIbY~3l&#GU)Ywuayl!nsl z9=o6~tnIS47L?eT(;#mTfSwQ!A=VW0I_=qWad{^pU_=n&JuL?J_PIrV{gX^IKCZa%`dTo=05I+i8Y-$N9jP zr{LHOk+pV?^~twa-qu*V7;7%bz8YrENeV+De+(_UzP6m<8>#d)28G$YQgp&u8Lm3- zp`|ioLhMRYiDfJ?$}^_W{|no%H05lCzca*Zt>6=2T=v%#v1E*%=#DAZJ{dwboeONK;rbjrGH`671C8^V4w;nb|hy*nZE$E_2SQ_^<^` z!ncMP*O02?F zZqH?}m$$0T2}@Wg4Po)P7Fy2TlWRAvAiX~qx=|h@ENtwlq&7oTWQZ9?W+o7exYv;B z+Iw(rO;O@H#~9$s@m_MK9j`doVD%h}7H|Y<+Bx^+UE0w`XfMT=9|uZLU8$abXC?lh zR+@cU186n{u-!i5(oN2@xo!z)Tr)1RAydd~#jpVp$mpZSrNCUj#R3mI#V8RoSU zYV=)0A!|=`qqEo2^a_*YHl~F8I)+Nmn4@Gc&aLV;R$A^{!_OtfftNMbvc%sR7k*6P z$P-s;O!aM%?;gi+?pRw3Fs1F+6jroZii3qekI9F<_Q2eao24X-DE78|BMiLo|2haI1?HXvIYBQkof+3*>5V8n@2Mrv(9+Z_gsU>O5# zbA&jCyl3EO+Hr4s62Zse-C$GY~O!!>$u!Ri$QNMcGkt1QP2gOj$<)z7n&c}$S>yT<_Q-YKqlr3srh zhpxdIBR+2~>65z$8p#?fnj@|i=bLv>LD;*aeh$s%ItQ-xnt4@YWH_ll_sUsZqmnso zdGoclr4BmVec@*F-Anf&4V)`Z87FJ_sC(4dq{B4O{*R@qWzDP&wR%v!McJ( zO9>$_YE4AXI-+v_8SD6Xgq(^W;=0!gBVseno!U948f}gv(tM4egP3?85?j7wXD8|K-UY|S2j%fi4-^V-&<3qB+Q+aur}P{%EMJKjBt-QH@3kT zJO4GNs`#0kz%R_*{XFAD(Fij#YHa1izD9y=9ZT;lEg`?K^BCvM(>{NU-JUgatow?i zCwdOa?=}N&OsKx2GZysg9#x|x#>Pqki*-9w|dX4!ZA`% zTi*e-YVT~AKSrwJ&l4UajBK9~l8X1wb0Blcim%V}fx31dA!PBqhz=JxHK=_4dhP5&K-ki^{4w`PZ2qdK>y=2?S2WG955I}<)|7;`9b zhefuScdX+XnHnd})V#Oj5@}vDj42Im&@g81L)sabECvm{xi`$?$vaGJ#HG#`cAQ_| zJCiDe;Quuz=GWO<$YiXgi5oNA@ED`xX;10*mN%?jADbh2#37HnB2aAID@ti({K!1l zdeayK-tbPNAbw^sr8xH@?HjXrH>Pobcn@LTIr6Aj%{z@@uaMEbcLZEt%j|P(-Q<=6 z4#tf;3~KLD{GF1{^qeVCaVBh&oa18FU26&_Zb_v)0$_JqQMP*}9L>CTJnx=U?s2ab zfj=ig$w{O7Vo&A0w#I_$50lb4#8sEPrqJYCLkxK=^|=yqgwh&`{(f$a(Y_Y2?pA*sL+dT)Io=ay=Gq-&(RfTbqfMM;yxwnI&=p7sKFGO|ooHs7t+LIwSMrq5NcHltC>2@cM4X+e}IQ`7?9c!ey z=P;+r#aoNJaA%ylmRByv*n?X;XZgz)VsPxqBim+;AdMX7^6khYem5`7^R(tHRY_Sz zckDgIyds|MPJ6s6j(O1 zf%syF+@GHUB=#O#h-!`S?lH$i{*vNAnZxciPc^j>Q^3(%n;|uANy9ucwsBlzy(PrS z<(M{tVID!_JWY{|x&{zVSQ%=3Y;mAI*4WV=37|NK?8G+KKGuwL8#8SH_A-OyN6MSS zG7asV7*=ji+dCm_q&fGzQ)Fq2%M5A{{Lv1Z;Wq8Ov%97aQ=XgXY$WXKFmf#H+2My^ zW;u$R2Fmf=yZ3bF_0pKb*2o=M4R4Me$DCtC#opoWe8!=ZGovza*-7Pngk8umvtU6T z;k0r_!1FrS2=W=rTyE`E_z_~Jc%HewA!aSWG`1q)4RaT81yQ)PV#@TH%SAVb6v(;b zeCuCZhcm1ptFuG&T%0kUHSC4C95;So9OEn`CDH92GniMONoaKpmHj_=SVb7Kd}dF9 zffk}X^-1gIYi|v*v_@9N$a6ShPARvT=jwdf>*amV4XQN)-qM`wz;aLZ>^S1~;Z36= zJgo8f7P7+Sj)NO5rI6l^P1Zd&c=(;uODt~WtdQ5d<=vZZW+cJPyH+UD8*3LS=b_uR zhfHT&V|HY$z3#fVG{O%9%6m+86U zt8$E}gdB&!@?7caB+g99I)ak!8B>cSFIki2*3 zLRv#3B!sPqn4(s4S{sx<4z1a+f}X<9+3#=d1=^G*%yZfUa%(O4Bb9F_CHXiuXn9_Pb1hC~=bWb)&x<3{eT*TDH6oz#p9`sE z&II?hmmvR~Tfi+2iMYQae92$)Wp~a2xwTT>*BddZV~yPD6&D8NpVO0WkAd(NhB`pa z@ke83A%eTspkJ8F32Q|iwLCX4OWPZ4I?mOuu)>7U3oDl#xh!wT1-ZAU zw8)GTZevc+*}VdCR2<7*B_sjjD9$YAGzOeV9?MWVZ~R&g8)I%wErT+~L`Di5eseFZ z%A4at^BP+#X(oKN*aIp0n1zYFx8} zBBqhMxo1w_O-rnOuN=jm#zfl~h$1ttjksRg{&7L8xZ7x0Dk=IIN z3e!X{2VKFirySyqgA8xX`RA~+gxpMvEOU)0wJ?S{P+kkYcIJ5NxmF0)NpUW82erSq z6NuZ~IWjec3BR#}*lh^$ooUQ{#~L>-M;R0FdB-iPI(Nk03>&6*W~HAtB7(?YOLZ}A zl>DAzThw?tc15#3i{vMxH&{v(>qpn z=!ki0X${fNl0rh!&FP&q?o9Q$MnwMDvmtP;?ANeHe&?QRbYY|^(v%Y%M4wB`Erdys z7&GA4$@|AMEX312w>H5~!H{nRdDS_0to2PR-g^z;?>^@~!VHNDbnnHcyEd@ijv;G# zjE%cD^1NOgBdI^f0pK?mY(&kOCV7u6)jXGW^NH!MWoH4jI;PxqALE)hr0u6Y=8Dss z=_z?F<)9mvio;E_B|C=|x45I0O__^aYs{6%l(shL8!7#5g(a%6g6z**^Bg1Py^x(o z?Ay=DcxK1(=n}^;`bpEKX|H*O821|G9K*6^3^kRyW`xcOD@k!AwEvo85N1hDZ46M1 zJ|?KuniCRb<}K%*W}?E&d%7--aQL6|7ILkrqnUyfPuT-5H*EQ>HuoI=9^->A4DqQK z=J?>;@k4XWK)yF7#Qs=2e{P1kzPpE-QQG4fWzJ>iK4)U-pYxYH4mIPD3>@dWwsh;- zGr)SLq1ZK78tt7!GkOncgE2Qm{fQyjC+|S{lBeL{T>*e0?9kzmH;lp>satujorE3s zv}_N1UuLbC>@;Sm%Nvu&YbH6TA7WN?iP@TCqytZXj1~61);QS>TWUWIwfHl03df4e zJa8ltmYPPs?cdQNI?NP<8v{Jd9YJF~$2x-ow>t4i(>@x434UYZ59C5Qw&CTyS24YkDP^$hWshYtP$ zmj|vv9qH^Xt;C3zw-96+E0S*{k@Y)fc2!6luO&xJ?LTJF&RdgQbAOt_+Bo0tnil zQ|UI3^zj$=*x(+sU}FZ&xUxcw&e#zxYmeCbxMJK&h|$%5OmO=WhpO~h>%eag0I0f> zMCzFXHeigjrM*`q!{ZopS_yu2I=Dw=U=#^Q0n;rI$AIlIV+(Q)mYOuDtg`|6Oa3KMZ}HIM&YOoii9U zkFn+zhJ?f13kPM*!M3-uuI3v%Bq#=i)w}Y(Mi>I->KkhSMpz?#crFQ(J%i5t+hYhZ z&so~-26rpWcp>3m|&$+I*f z`fy*np*jvB*0u+R=9^>bJnr3&vA0~_2&>8>jFh1~Cy4tUiS}rxxtOtokaJ$Mjv?wjU!!Y@8tNq7Iw^^Lp68p?3oZpOiNfBRB>+UtVWfpSgd_ZDXS)}OJPXKmffyMy@f*&87w#7yy?g1*j<@l7tytg5y~ zYGodS{&6o^*1cyC&RA=ZW%oaZ<`eAB%)-PNCmF1~g@!V=sCP;kL79boH32Rx@oh<-apx>yF{O zA&zmZIwAzxn3?8h%!%2zXWCL;i*8LrF(J(T-7u2o(Ouh*WDn5onnn)6oO}K`T#PtYs zfP74~^sy7JdK;tLH7vn|oJaER94nq`4Dsha#}Y<%*0?6H&m3FGe+GTxw$~VxlCEEyi??7-#D_m- zZf~9|1Z&Qeo|C88%G(iGd`y7VI~UmMjYEKUkA$~B2h>!bQ&wh-$)U6dB-crclQS*I z_B}HyaT;UTd(FW7yqDB)-@!R>q&3mBC$!^Uo4Y&@o!mM0(DTa^9VjljqdW&#Q%RA2 zV+<(pn=@GZ+ffsB4osRhXAI?9i;jQH)sPyt98%qJ3V(0KiMTh&_f30Nc`X@+zEULR zoLO#qL;=S=#C4=PSI$|VL5^z-9m|{oX3Pi!?oSGPzj=>=&p*>r)mqac#7b9n#|FA4^r8-uT93?Y&`H< z;c;;9d9OC6_T*d}nSN#H(6>V>{bz0L|dJM^hF{cPw4BPT@Wi9g<#uoNYLwzKz{prtM6VY-ed4{qEG{8M0Olz2t zp?VD=&o)!8ZXA<*Y6Ntn8p5jWO6wY7ZArPB)*$DJb89Etj%qS zCFrM{12D*)V}4SDCe7l3EkE5CJY8Ws#%nY%P*MY#C6r!eW9$6f-56`|YAmelHm3IY-oYL$4hf;Zw^VUj zS=2c$A)mX(%=?Us`92NFjyPlLz?>;QeGG`-7IuhBS_7|d<{;9)Lxe@1$=YgV9p;kv z^5~zdXJO4)_!J}j&kPGrE$kqhFz1qY8?%#s4NQf!XI$b>K^!7ve1a54VBnqEsb#Eb z%Q#j*eB1*TWd)V_J9dcB%iG9lN6C;ec8>a7iKTfhVXKvc@b8W35Hsg7r?p~u=pMOd zU=FkFc@0GC5r=&34spwSZ_%kU2X^31i6JA+skS`Bit(CLtA4Fbkvzw+=Uzb*J;in2 z7cwf>jEOpCrTF ziI-+>T==?|2>aXnxn+&f^cN?P@>s(fWsd-gGSVnkAK`^1>_L}3*0e_46O?OBtc9StESSbe%Lt{`(GF{53CO zk+U|S(oI0^HM>W1?wqy|1%@g}6fc-Cs#rX>1AQuyZbQ z8e4^NYu5xFU7&%HdD&fS)tNmkAUYgMkw%H12t|8(4DeoAkLa8=VY$L-k-Jrk-W#i z%$h^hY;LjkzQ$(D9z#($?CsmLcNR%l!Blkag`1Fq}!v5>tc=*x>k4R+3r^)iNxSl%-fe5_%*k+x`iN*VKc?{&eF!&-Sr zA=_dEA)A*%*g=@;!x6t2MTbXgBt;01_M0MFy zOngV^pT1H)`5&7Of8uItU7_tdjfw6s=BDY4Ln&v5!O=YDrp2CFnP)AqjK3mQ+!{m9 zEU)36KatKom$K|1%jS76XpJ%^7D82AhdAyKawtPzgEuhksjZNr9Pb{Pj%}|g zt~~~{L(3UIX6%9ZJaTB=TC3=F&jqc$rZmu_0sZHQg|Fj?`PRA!tXn{podyRnhGq;ZW3OkT( zr8%oTmqbq5gG*rsEw7Z;eoa}qs~IzCH4S8g9Cn`38k2~5jp^_`la%fsv%75ty^Wm~ z`oI+D?C?(8(|is5{1XPi`P|VIeGF)?5SPU8oEvj_tVHO&#~e)`>-gAh2=keVdhrn=L>+Ls=@xQ(kz-wERYA{SprJN&Z ze9rl+AREikaV4RgG*{+oj&aauO{~n6GUm)n8{K%txPT@sJ&%<3nDfcu(-U zU)`B-YJ_#ivQqBT+(Y1Ku7UTv2O>*dlVx#k#jLuAxYeCI`*UrngB7N(P}l~(c1Zy~ zHBX(by@qD(pIeGK?U~Lq=dMD^^U7fjS;97E{@EOv`zqyVq&!z>NS^z?H4Yq&xE6BZ zOvC$oC3*3-*P!^yL9leLUDvuN7}y(g$|5hA#xw_@%ZOVFdv4{{m)FYs*r^0Ok6E(0 z1MJPsS^Ht8t?iii&g$KRP-`!A?!Pix^jWj`e}!@O8t39(+*9jeZh4Fm6F5VUYtm`Y zJ*co#i1!L>}AeXOE%DvT~BZG7{u}o;x&dCdB(22FTDHn+Rt`x!OLq z?%)pN7-h^c`@IL`V2X1tY-ch3KGFf%v*rp$-wQJ@Zx!FYSDf+~qlP^#4eyo*6yjTp zZDH^Dk2P0#_Z`{ZBkc|DI>YqO8=K}fEHR3;hJr*N8-91J4VN;)GE`bK>nv~S_cO<= zNQny)bnJcanTJNt-5WlCY=Fu*whUcZVbFgs{AL|9-bl0m9b!OKjH7Bk zBnY(>7aGN)^Q_Jd={Xvspz&ztn{>^LZF-#Gix>hvUTBA%fMSa<_QtIN{DY9bjpu`QZ z)jgM(%SoHIAm!zZxMwy|TEi22h7r}U4f)@(ri9B0!8&zkVZ=FdQpul#19OcvtPw(> z-y1=NW3OF@7G^y7SYco*M7@A;Y;l1eW7Nl7BThF1DI>2{x3!jv^BFTZcE=U>I-(lc z8WUl5Mn$QdwrJ{~dog%LmFvDneEHqsXKKYU{x>63?pc%bZ_j}2wdbNx90_A5g^{;2 zB2wfYF-ClB9o@2*MDrhWeQ5>3@|vay`&sLiK5n(goO6Uw&r2_C&P5!I$+uSK+IxInnl2TG> z9T`n(ZpGie0xHs)iSTbvWtO|+Qc2j!GJY&b+_ARu(~UtHeuue_w1*;j$a6&_E`{+p zLLhaIDO+$)O|HEr9QRp+D>A2v`V%JZ=LumXZ4LbJGjkHySW8)XO=adecIxL$k#1lL zqeXp9DdxJTFkTw#v1z8!{1?-f6d(>v@L@0_Q&(#)HtU`(NoJ7zR?%u9W5&osjv z<4!tPnB>mmlPXNf%(cc~L>}vRU=9;)D9$<26=%Xk82glStcAxuQm#_ktLS&fg^sr7-eQUSNhz&3 z!m`)6<6O)2B}_%{u>!i&8qt1m1mJFjk=Ji~-|oW!d;U#>&E3J0Lp+sr8W3;>Uk z;LfXVc8&SWk*5^S3F+u}4+(*}X7tKfyIW#UJ=TyS7WPQ1PG=;+-95wVPafm6A_VH{a&~Ui&l3|)Uz`y_XpIo&lSY=^ooT{j z#TC#uW>9#Zfv7yM6`hnvSmX=AA2Tdr$&;e4?Anu-WX!>&KN4s|PPy`LrLCg9rY_aq zi9>&fwBWQR3P#;4v}$kp+O@WB`<%&&dkk3W824^jSVI(VCRnLAvkdQw)5UY<43oEq zYTk~4KQGQzjF@MbPuPJ3Fb)8!yBD@v*mHGw4oRuAcQ9Ov+uLNvHRHO+5dT^+WibYk z`JP8|$4Oz~K2LO~kfY+o3F%K}&jI1K24>pco5^Bioq?3rfLe{y4kAok;h1-jepqD5 znl`s$(+c5MIxH2IJ`yz2SzF+I&)u6gR+LE`3BYtm$@jg(g#X1n!Ial@Pm8;cWG4+Q zZPl*3#;(cS`BQf6?fSSUwo#d@yDrYbsWbM#&L26>We3&WxMynEO<4wZMNz7-Hqc7g zBXN1`X@W9ygnz}Q;x!kFL7fW-X>R4?8F%>NSbI@QZ%K+iP8CQshM$uQ0p(VHd~{GB;OVXj$;Gx9)w+UYeU?d7$$LN4wc zyNqM)dF~OzM*G;IEo|UtilVq!(%tGmbe_m69ObG82Br%)vJ-tWe3ar({(gvwm(R5$CuD7FkMbr8s6S z{xC*#>qY=M4KyN zE{}Ep*Q~_e^40=@)MV^{6x=aO>FH^K_2A(zb)F z+1smPV4c! z7B>p>SlcjeZkgE;a`JMXyB}c8q_n&CLU4(hv30EA**wSoAg)=8xAwxp9trGn1v%if zCjvrBTZ%HwQTv_8e#F>og*ya^#2NSUQi>y4WbMqu83W4jU%7E>CgIn*gCfmXQRrgr zoWc^;fM(e%&nB%I;gbd&VA`wrJLdVGu?E!Hj1$ms4rP%vLDMki-05Bu?Rq5LtT%Re zP~Yp{bC3AsyVve(-BEO7523IaW~kLlSq3ab71TIa(8|oS_i8S|`LNb*`kxaqZEvBC zv!@=(O1XDzY$VLJ;w;yTqor-`W$81fa6}5xD18q_fjoB%KojFg@lPWUd<+f#va)ne z3Yk78PwCKuf4tE22FRab1IwFwa+X?t&Mro=%(i-EKA=GM& zk?Sx+s(Idl`*)3i=C-$#(^`wVb4_jcmj)#M++!$a=P}>8bIjEo@iIFOmHxa(G<)0u zTrWgDt+{i$<`}bXVu!hnwWjRj+atVawgT+_YgWa-MAdn@GO=`{Ah z(9f$UCd8D@9LBuxAHzl`%^lGgMjTZQyFzAXc=?;cepec6sVeW?oETHg-p)&=Zp8W8 zHn$vQ459pB?yat$*NpL;vmU$kcap|*qhchE$N1v)9QL&!|pFHc$YGk z%1=w%x;ZY{`xR5fSxjN%cMX+`Fd}r}oFlbv>?rm*CaCPp6Z(G4Rj<6aGSMDmU~|my zv$;2RVcwgwJ?t#xvS-5Sn<=t#O)aH5(r(Te`+Q{w1*{r^$WEPWr(2OUyY} zJPhdBo+GSQ2}_t~1(A<22hidV<7YVzl*pCV=Ecv$fiVRb=`c1DRLqf}d`78_wKDuw znUVHm&Lzmc2f*{*Yf5v+=-hA0rWO5RyvWp5e%v!)*W-gBdKL|OI|*VIByOWJY{ zaQQS=ZsDF&1ZqTuk`uGyOr9~ya!dhkIW94;KQ^rXSrZvzBrK$or`+R>EB<<~e6Svt z;Ne=6f@tnx`M)#3@=D9IDX;P56BGE>4f9W64$}y4WZj=Q6YAhxD^7TgfYv$Ulw+E6 z+B%PUv^FxNOkjca9hhJojn10?9#69RP&1*JbHSZ0p1peQGl%@C8;P+l9dbPP27JA;z& z8BsAit~JZPr_{)egT6C027>*bE2v=%y{6hkNg7jdXY9%L8YYtIn@RmV42ZlOBCvIfKGQR(Aisdam0}1G*i;r+S_epO%cGl7Cd&Dq49jImE6A)qRXBE zOe3Ummp4Z=ciQQcY-WJLna0@3pFyyH51sA3N7ii*qkeG@^r1GV>Rp(73TW?bmM{fC z?X>oU%^$mVY2`@Ryt43LUZIRN#@VyCH_F&t%a?Qnk=#9#SZdny^Ed6d&^1>6{ma{3 zEDjCfJ43eC*l`teCvdxw=X_$zD>8L$@f*4jv)}LFSWM$>>koQc`%4>XekNM89GA@1IV`^dS)Vs5jfM(4h z>TD$pgSmpJ4ICU#6s!MMU` z+YZ?ja%@%gIYKPTT|>)grg7sRmRfD!yTW*dB-Io%hV2_Gmv)R9_PkS)VqYVAa!d`+ zG!nk%n+vFH?5Xgz=RDco^VnzRiR!xY7;zgR5+zS{)faOV^=*W9m6Giu08({@&wi#L%}{R zjG`U#T++!CH*arwkUQ3RcV2s;ekRDhymqd}h>J5hr!@YyXU@rp1Ng;C+cYvSo#{I= zOxPPMntQAWoiis!$k=-XK5b2)I%Y}pJLm6a#t_h0e1 zV=evWunisG9w#N6+S4^HWC{H;7oJYs=>tAwP@2D1=Hi;+=5NP+ z)IAo?)!wsNW=sjRGP73A9+9MaO&!>=CNOZ%;nIH0@w=D83iTVgNpVias@}U0H*{g& z8*+S%RGY94RQSDyrpa8h25v2NrW=xa!kF9RbY_W^662imoAZx!Y-z7OMl}AK6Hayw z^~yCucy^kTM0M?`w6(VE{1`(JKaTMDl@{>a+QXSTW|h%9HoErSlVEd>G0M0Dn!%o- zPIyk)>a?dY*jXw2C2XjW6h?&d97EhSENz^<2C`yb6CrTtb-pvg2vb|JJ2Yk~-nlUA z-|3D0rI(^AN8cL)J!fR9&5s8ZuE;Xw%Sx$(UxtbWw{*Jru?0IMqy1{$Q_0V z(jP+~JO&K#Kc@iS+EF86reT~B=0erYb9*9Bjf+1ferC#hwtGhb;TCqLSj#ayaW5I= zIJQDwTxsif#O$paH?HT;iGFX4Oy(RGj$lzS_Dav1>DzP0DgHRaYV4fhRz6M1ury~%Y8+7(Ii!7x9hW}9pBa5-jnU>l$C~P! z>w9_33I05m6z__uD=&=c#updJYECI)bkCjgnZr88Uwa>SZ?*8B2TW<-0WV~wEsZ{x zGU}M2v2iUm%@c-_^d9>Zb&iGUI+3%tS1iPh!+9>xdG|190?vvX#C8u=+8(!3SR3*b;>$2nDU=XP&G#(!JT$m)!hqEXpbe|KZn5N-7|1D zCLR1Y!q`HZc~W@n$=)eB_a;*gB5tgL!9;2Hrto5=vR~ppa zdn0-eQQ4RxYJW*ONLvZQ4n;JkEK*wkMX)%gc*%ZV2!{^Ne9{T=p;`oZg;G zO>j-|ls_kO%U{D>a*v_0pLXQYoO{+eE`7xxMk-g^yJ;!yh4;FWX8RsXXJthA$~ae; z`&&C;JM4|_y%WOvpKIzo3^e#WQrzm9F%T?o+~b<(x_OG@lQ%5bw7PRH%%3Ava!={H zFeYaC8BzZ{mTGfZ2_(o$INoO>NL#TC<@v&zMqdB#2Nbjdw-=;B{{ zMslp!uM-nmL>tpXCodVYGd3(jU6HSR#r)#1;>yXMQ*$<~)vvJQ%=FmXi#!c^o|&h_ z$(t+TY45z#yH_?^8H;;1ukeyJ7hcxPE4Dx5JbvEOFfdKPqC7&X(VVLhX$}O5KLhsa znL_~NT}woM?dj~k2AXNuac6w5g|MhBp6PaiMf9O_Cf34(bTg|2P*4U(bmiDCE7Eg{;-Pwl?YAfrTf{eCm?NGE9r3 z^m8r6x;qS=i$Bx z`{sU|`(^HTxkUBbAF;hZkJmZp`K$>cagL6Inc5Q71*F_K!^hmZR*UmWUA%=l|iNa*>OGtn6NmHQjL_Vh;CZ1{mFP-Eg;RG&{*QM zi5{(k?L>W**L^pv&$8%9V0RSxwf^I0`f{Fl+3V|ns?evrrXOC8rLi=4^Xq$lXIY-t}6_JDwq}J2r$y?XLgyS!c-alboEE{>(k|V+OMuVhvUj6 zW0$YQC4JTce>4d$UUCCyJic4aRX<4`Lb9yBNj1Kx5&1bcUuNbg@~1;qz+)BOkB}Rv zx|Cm^IOh|z3GfkWYY=fgoQV#WUE_hPi*|0mDw+s18mQwi^MU0*yY0Z6SQyd;U#=zZ zog$vmN_RisZl-~~c?D(&fzwYEcDTCFkOT>%brUGY$dbQa#l+}O zu3(&|@ixJirsXqx^rO)!&3)xn+wxqN*W{b-z;GS?Ocs1w_;mHN@RG#yCX1c%DV1P1 zZ@biLa!gm!aEY_%odfPnw|2{>t{^FnHR0Q7ifoqa`B{<=5W!{wh&85kTxJ@@^87ykSm&H>%w zH%VBRIs3Fg^!E(^TsK4A&`^(8oIzi1xK+|F%IDP2qFB}fC5mSoPixMY?KkWluR9w} zC2~)E>NJR^ktc7Sr~?9f2qa5auQGBS$od~#j2vl1TT|bmv%{zIJYx0NiPF)|EC(HX zc#3RuGn>!vM7k)KHTOv%Ip7>sw&`qKIagbvr9TBWiAad{uYdy%nArje>=&ZgMR{F!CMyshj>pC-OAN@m8geXWW z3x8``Z*eJ(8nwx%XY9vu0dTA4rd;*~IG7Y6w7J|;^I3N%7Hpc*n)tA^*qNGNpFTB<=4S%|N2EZ+q01~2I(Qber=Adv8nsb62v$U91Yvc`An;(E{k z$5jS-Y!9(g>|7k*voBkC1o`|GPlwUr;isn(8>q-}>fje#I?Q zEm;r9R@M1|Qu=?pi$Zak;jpV&9x!I}(~s9T^;+n5RhMEz1bzN$x*gb$zH($5I8u)O z%P+YxP)BVaVx2|h@9Z~JFCa3-*EH;<85S1qlq%!N&n z7M7etrPK3J%(wmWGD>}85>hfmVE(TJ=>k$|@v>lNw^TY+i1zHarj@%Lg(}+0jOUHb zC7cH~9afRPEyZ_=*v|*ERGoaIE;L@|bfGxrUB9{1XOU9_W&y{}Osh*si&-#w`;T~c z*XL@;7Z)Rz`k?s8K^2P-W0e1&$lcHxb-;nyT*eb%CqwQ%3s=X)4DLu`=$k2oE zbtI8LPQ!xr02NIwlg7WV=G5b4<0vgBG(;Ua@KGkxp^$hqYvHWJZ zwEJrF>4>K#Kuk8~V?veL>+ZDhEfGuW5bvO>@3!7CewmXrTUWZEG$S5#Lsue-=ZURCTUYK9~Y@wH2=ncy@}1;@DWy2S!dhhFC$qoM21>zYd2 zu6QX|c8SBymZDQg9x6C^pM#`CbL)&f`U>E~zaazqbd--OmGQlYkfG^n0tq_opsCU# zOQ!~FMS#Ef#5hG5+qd`vd_qh~*CVJCr)!TtbnzVC@MYZHY|-U~+Ocq}4fdSmkJfz!-gacg@lMShwg`85zt>QnQ=ubEY=)l5K-&28QLR z7=O;8%)F(`qBL?c2MdSsui@NZ0J+wN_S>@eQ$LS`bnh<@QdI$pU!-AzzYY=r8(stO z=~f^`{>sJox*Pqzj*Q-%?YQtKbbiwi*xJtz|0JXxaaQLV7hv1|pOsBWFOok~9`2H) zUb;uLGQSyI%F)Zbe%q!XnJ>iW2KgQ<>zXBrPQzIB9i>S?K|R|zn(%*@v2fZNz(hKv zoU|h3ga`&jCc%X4cA26p{dttUzx3ELhRi<9(H`MkQvIQ2;^fSxh*E4g>}lzaBc(=v zao+Xn29gh^(9^6Qrq7mvy2}THn)fK5MM?4jCZpU@n?b~k-}w-j{`AUSU{_^F%V|v) zqd0@ViTiii31xE1H8v@xVsd`KIWbRj(~;@Q1VN3Mu^defptk#*`NVmbNBq6RceQDy zGwuzW%Ls?TmfTv{(TSyFe7JY$x&rm$!Odz7&(F~vo4{i-*XiWSIv=Fn>9x%4wjwEOc?d?8P0{SIv3`@{PdXJzcH-j@JjW`TfJ%V zT=hY(mTJw1&q-i{hje;*t)OZz99-)^I2c=KF&jB^cp1PnG?-~tGB-na!(%n0m_6E_ zWEBu#cQ}oj#==WFMjd7mCc}=ump0Id|NLgczbvN zsu1XtKi1MAGjMZSUh&4qr2NUZr`+O{3-Y~hlywk(o?kXB#cQN$MdD;C9IwuBBN1(; zb;$782*RWng?{qQO0PpLaNqd1ykI~CJM@vtx2R0ey1eDA=4a1T`Lb%D)&vh7uf)K; z6*=793nOq|z-}m|1S6tvOlRP;HyfqQ@^z{ZdRK_v>LiCHr?aOY^M)XHUP!GtTR2)B zBK_h*MWF3uj^mwRYI~jRX~o`LHG{C%*f?g{vY*hg?ndOG_z4urvxWl^+KyE!-Tz(8 z16~Leb0n(kcnr_=UwKH+Zne7BS}*i_gS8dMc$iQA%**b2l9B%kE-iuZ*?HhZ4AIg* zN;2ifpu9la6eDedyh|Cp3HEyysj><7a+tXn{53KaLmja9d=nLadSw0hn4hrpSL4ae z--XXgRrcN@>O~5skt@-@Mz@3EZ54o1ho|g<$3YT)pMk6oM*3G&z4!S`Da6uWh0n-A zrH6l6Pft(bsB}rD=Nq^Z8M3850A_lg%>FaNh!a()KEOcaIWzH?224JnmrRkz;@jqw zzWSxkWr=2&Hy%xN8XKxa47}JAD?nZ)q}$2pp_H zziMxvO;IU!ZHwbQ(*eH}Ub63N>d4BTCpYO##ny`tZs+ zyiJ3?vZ|q@U@3p;KONIV2De!K1l-TBmxzxJfRvAGnL1Pb*6uy8b@fxSAQpS@EQE&d z#~=4{o`e`D_ED;ZzneifpV*c$fCm~#&CCU>o|=Jg$mGB9_F-3s)m&`!+b%nm|K^j! zZ5?&T=5Xr)=e#H_R|dsx#g4B=sD4l&!}y5oM9As%Xw^3|9N42~4mPgIPzSVJ!bOE8QT*Js`j;qnA{ zeP69nGEex4CcW@-`Ph>KW<4J4KM!PZ&sRKc_OKHC;O7;X$9ZaUbQ3z{J8&1vrrS7S z!(ZlBadA+`^03{s=nY7!x$7PdzeQ;BmDg!eNgJ!EfyrIYa-qCpS_TwFrk0YL6kK*I zca&C?XVC3J!IJIo6tFus28k8!MpwHT!O9EyRR8mK;n-`OJhwD0dKVt6ol9}3j2I|= zbJ=xk=EFW{Cv8JuzA|W_h0_j$pb|Ls7ygF`UVHj7b|PIVwuQ{Be*cR+nAzMM4H^!y|K`qK=)K1?JeN2@#L=tA%z>R#tFoWA!g@%Djz;ts8lqR`uSQ|+-;AT0 zVXRuVbc^1r?&KQLo_T}gEYLTll-yEAk2%9d2cl#-d(Q3MnI>~J>MzACXYg*5(PR#?u<>CV49P@@SL`us3A}@MWk1q6G+ohk zfM9IiBx)))(RyUPHLaqd;c_^w2Cla=q~mk-s*11EL_h5>d+Gc=p8zV|AX)d<`c+7d zfbi;l2o6tYEn1ab=ba3EfToT6D@mV?e+syXHEvP`KY;;JBf`FldMp{>`(HPX#_Q)o zbd$G{8;b4t>7=cgycA(-xn6Yn2s!#LL`3^g1L8)o$$){WfBkBpP@Qa59>}fLt;oE* z{xYQl<|9)41(8{M!ZDif?EB>tQZ&N9FMKMDAeO8EtXi1|EC{bY?re#tc7bmvzwV@x zQ!r*2y4`VeX#mUukPzC&x-$N?67Pq3)sJzqeKkmI&yp3cOy_|J?b&KJzfR4lt`>Rd zWkTl>_Pr`korc+gkB$_0-Fu2##Ag=e-s6%6CsBMaJ1GBoWuu^y?0_$jg{2x7$$M2T ztl#E(!xMZev;QFtQv-|Hb2N_A!U;>iu%Md`q)WOGiF*?*ta)>(ETVD@@YtH%Egx&y z0O?(YsA?98z&XSJ1`wlP!8f#PIkAC{G}wH$h=LC(neYVj%1%!~ha}A8dTC?n$#wx| zgA$vRyw{xHOy$Brz9;r#<$4yU4IRTICBLbP1g>$o-*Yy0kI&1En6gEEfu{AAG?WZY z(AxC*Uhmv#$&6X$y8{9Ah6>Ou=46oVf1<9TJQ!6mZ0o20EVuo|eS0q`odt+qxmTz} zR0&4sVF>qQDHq?H6I`R8w30eR!kA=9m?P(Qq$cVO#2usZu3!-5UZpDer=oVWpfmZn z0c`464ifz<{JE>16*khI4}ggJo}{Xk2u$#Hm`RF0Iz_r3h0h#{Q-Y*TWfrOPf{!bn zl}lxEOKDhM;f}9T6>Z^}lVD0M>(k>S`{_`BU`L3C8t>7;D(q>Fd8=@oE)o70x81(&zmxe@79DrX+(K4^myeDl9+v*HxKQt7)-G zs{)er7RzXu*g7xZkaT1CH4cnpd^!L?N+qs_!0)BX3OHRdxjzkJJUo`e+}hQLat3KF zon*&-_>*vF>$T8_^y(6`3vN1l1>=YdWGQ#IL2_Wg^|ApuHv7rKa#PWYsAyH}%uGK2 z?{H~kF(>tdg*noq>i&mZoX+z@c+oA-3gdQM&0Fu(XB2bVuU?jTw^dH?BY6V+ygB}z z4#Hg_Sn^JJ4VCMP4JuA_brIc)v4(MUJbKWXk)#-XV9(xk-!IR^4!G{*dZ!!?$WpQIE|8o&*Q`FUk&RyHB{T68<)_qGbByRi zP2_xE?8}#0lKTaL9;V1SK-O3n>Eb#hb#qk~`*4?3_5w)rfdq(Gp69gTl`wM7RST9g znb&_S6+b=;Fe4kF`TwePRTeV8q+hX2-VWCq$haq}_%H3aEo49~yOzZabf)7>L7qy{ z51#$7h;z5u{@`i+c!JMjfM8cdlr_0m&m8S`p4aQ+@$LFK2V~+6t;)B+9o5AK9NY^?WRq&Psq7c&Nm( z+GiU6UWQ5QcOw;}Gf7hrP>GzmoppP`L=z`yWV?ckd(?w!T5{gk&(7ER+uLhnC+hS3 zD~Pg-vtBtD(LxgHN*x9lqDcZ;GSOo;I8y5dEJ0VdC(1D z*IZOH-UDx6juh0QGWLnrPyaE|*l9;Hh=g2QRfIV25Td057~78(M&ToT(cR%U5SUhx z&ZQ(u-3tD6lKCabPbh3vGG?j9FxOas)yh`)a2(F~cOF|%IMu?K3bUx~AZ z8x$JLWfmsqPhK_LOIz{1Mk)D}B52o z3>*qa@5%4kRb{g^PZC)4{nsc~Lo)|^M1Pml?q|vY+3c6qur5F2V4^y8-~irR+0> zboe($j$+dAcgpd%59Ws>Wd`_HWp%Bepy?9sGxnzkRJ_<=WAhrF-d6XOq|ZkLUu`Wc zVH9GcQdG=lt~gP*!lf0mav=Zp5LH&hC7-2NgG$K8;XhWpK6Vl8Q?OPbR1~e zt0o`p^sDeX`}Sr6CVBsvCYvdG*`ZoWB?j!H$DGoh9p<01UKJjry(;cp=4Mt6a5b6w zdcR-JV%6HW1Xqbla=`1HPR8I|ok8^8mGW#4yKS3_gLd1uWMGs5PiS%o(Z4oh{eMsM zFtG~jyG^p>CpuHp4^4{hxJz}l)JgZVgvUoFfR1wJ?J#Xl3?EXI%@>!L1g|@qKHbOa z9K9%L8do7awP^XC+|7mL$7#|RR}7-WfpLc^ts!A^^&084-GlQIxZYp*S-7}72D zsTQ&Tiq{OP6vhO&9nuE7EP4$$uZQ}3*N*H|j$VxOf4mEb9>$1JZ6 zqU@YfH_}yMBNw!QUJa(oKTV>G=0@Vu0bVIsJGE~gmK36?mlKagrHDL@3KNRkyqDkoO}(#_%>6oNG3;)s*^Mf35wkYp`Y!a< zjWw)42ROud!s$QO-!17Qr`Y^s|DVqE+%}_M?Akk&rrzv)@*_D@wZ7WYiG_uacT{vQ z#+oy?Hu5jy#Vc2v8!i4DYCe79pIRR2%p^R>PR5RKTDgVok7qZ*uI8<`Enq&hH8QR- zjwb!7!QZ*})lRLS@~X(gjK*TvVRF9*^z&e!?;snNO?n2(=>q7#e9MP)wJb(v?Wxx0!+C4}#e7bv~EyD@9{5LRi2YoJIojG_D9zE{tAl3xZF zcfrvjtcp48mvkMauHPhMRa`5cCWcl-ri}ltW_{vKm-T=@RdzoIc+0DYe>p^J9Mc^4 z=#0A@uEf~0_nwWQZ!gv#ir4Xb;Gsz?Kp?CWKB{~wGL=FV z<8P{jFJO+@$S>y#!w2nX;QG!}thn8FHv6#To0fg;2=~D|gS9%%AcV#HpRk-g zaqZXt8d4mpAmL9D7G%)FD`?>VIi?K&e(g5*%uROZ1zxuFb)Tiuii>e5wn?-(*jke- zh(W6<7k1r%_p;UP^9G!A`C$+5=uBwbjq%ze{DGC&TKJA_sXB{8Kb&^bdmz6GsAEAH zk$i>HR4&z)emyUQ2CHjd`B+y8?avSfr6!S?L=M+3-*$!REc*O znaRq2T@g6*1JvrFAL;=!`S4Fp#CekEaz5Nt@&r_B^q1R_9I@N;jt|&3X-c&c(q*-r ze#1Eq2U*?fOBQw>s4kTSeKJEK|Hzo2$yJQ^s}Lgh@o1`Q|KgMpC7=lf3jRmBlYdV} zr}koI+v^PzdmivfAepP9^S)~@0rce2UXNfXSm)k5bv*WXT71>(`Yyg9_pLS3CNIM0 z&-Xj5ZObUlumG*$nr1Yn1u2kVj2L+n@9BLz=)RneE5i7y{2F85a&2v zyucbyc!OJa4*~bY-Y~(Y=QaJ{fx7wFqd7W&P@MBysU4YlUXcB;5tnDc^%p%L@a^%z z=zAV%b8XpRdmHYLw;A8@S7rXlJ)B6Gt9e@F{-UX5aI>H;$W` z*WHFz6D%!;mBH+tOsf!mN{tCDV8!l2C2p7aJ2|pC6me#`UpNCh^2g+pMSB}g6_=`i z$yB|Hl9XXo4g7C%1BbG=cAq@Cq=(o1T);SS7Ia#BkpjT^@2-8kH+j^?7N6vOGX<>&wLP>^4xZ>db<7-Ajw-T4ARPw*6;*61CNJ!5e?mR_UevWOZ)!JEcrOlY3>Fve3<@`dk~RN`sgoNJDJ;YHO6Gf z52cSCXcFC7N*VqoB^gp$gLSp%5S$K7Q~1P3<+q`?q9qBhJmV5!U$bra=(j_a?!qv% zpxDj6m4`$E(y9t%;wwxHM)!nMg})cZZCoYTl!aA1{;B*aro-BkHn8$yQguCK1Wo(N zjo2`zKn2&c|9Dfp!i!i4QXNB55fpc*bbP*V{>U$B4kMvx`F9@UWP;^7t!?riP^Vzn zL4VzLmL`$Q^3RM9ze+J)qu^=`rWTg$1Dt!yp{aQ~x+g~qZ?UuW)&FlIdHz+)#ErndxDU{$% z|LaOoAfUu$@3Ct5H@8b1=zpvBWs#1A3x0;OWq|0DJMchZU-6dU)$P|QAckEyLuk%!xyujzQGbHg*6xK})r@;f1wvr6qH{6bLa?W8z919Z9dW|s)*TMG6&5`Oh6L*~f^ zqk|NnZmis>g|QLWaiH|07+zMR9n^3=C@prO;ANX7mwA>WWbOlN9O5W#rPtrfYDN~{ zw~D^(lMuEY;>AyN-a?K!rHJ8glfSm7vS7h$C)=&Aj6{xb-`8=~qv|4r*Yqm9_kE^` z%A!Y3HyxA3*Sz-|&QgEQ9Up`fdw%iaw#c*q7+|+8sv3qqF-_)x?fOqM!)`82Z??Eg z4fM0-_8IECABt7h&htRfc7@1I8S0Z|VAj{e0y>WS+|bWz+AS+s5a*@sXt~S&o$PcS z&YeM8m~g2g29X=LzYwkCDp~PUp|7qrgM{H^ z4p#@a{WMK1zUL~|s#Wl|0^~YNZ zY%@W6+q`dlc0RM@Rt4!t`-T}k9+%OPnX#6YhJD_RZGzl3%r9>SGtHjJ1V3;c5HI%G zf83<;iS@uPi^qxHWk-eimS5^+^Xs&s=GvMAGDlhw zEnW7kEhQ%HLb)fd4tD(;uVb#Zu^qpCNPtu~#b94KN;nsZ&FbcNsRXX5*Fjpb zh^(W7j#n!*!33+!)|Xy?NwUR|?}42%I#PHfN$*V@_ax$UnZ(m!7qXVHoEvQ7nvhKA zOXN6{hw16$)75zHJLdlA5V0~W;tF!5EtZ1<|Grwf6W~>S48-LbaY9?Y_SNcgSb+xx zMS?!kZ!2ZipG@(gnqlz34s5sb$g6Fp2PVP?1)oHo@vS(>?y40rVet5-pHECD`7@(>pgN)10mE+p>NQaZI-a+-P7LB7M!&Coa$@5kL5m#O%iZ`;o z1~)o$z>9h5NK@ZSS@hWtWHdjN65g*}iFlT5WiFzf(ZvD#N|8j#czsV0fo|3$njrF>Baq0d%}lgA^gH51As%ae>yQ@eL)%yrJkf{H z%y3Nm)h<)|`4M5|4gbz~uPjyl5{dN9EB;v;Z=9b^zUE)|;@H{N)AKg;I}r<5HJ?^8QDuC3 zh=u&mo74~{@|{xtEm<97%nFt?T3SXfKXIPQN_01kYEEaI#wlms_A;Rg^44m8j}-kn zWqar-SwWl6EXO{Iyj*cB8J||u)H&s5*^w}{d(VzZdNb&F%8>^%}R!H?=Rgk+7!>S>h^832gw@s%6TR0PKD5SS9yM;A~=_+J5#?tGNZy zMu{tg;QrRpr@6C0}%yvQxP5Vm*sZvoC|kWaHGZ0AoGC^ycjVDz@<{@r>t>f5u0D$hK=MSkXu z{{XexK91OPi~-W5?oF|<3OZX6nRDJQFi=w2E~^h!PQ3o9C=$rE`c!2y%wxVt0!S~6 ztv7dDu`+xVv1->=lVyo7I5v`H>c!RQpo!ej*S-~%Qwk$L_3+RA3$pg#44jmxo@V|V z;2ym_(bhd(mhKVP*@if3J~yQTz8Oq7Ze-02v)R7u^I7s^t3{pkm@{l0d64_|P(E?{yaUSpEpoY>IqlhP0?_i|vB81>&MySTSs7YCZTChQl|us(Em z35mlpPIGy>M-sRj6iW{_?Vt!hhP7(GcT!}8l8j$5vi#2R@;EJRqIa5Jr4pOT{v1>z zv~;twmFlV^(cj;}7yTYbcX?T zt`y`T2&y+_P!#W-c7W!=)BiFlk8zj2Q&;5|0j)1YfV_ ziA||U*F{%-k~f_~kt4r3e^JJrJHVk^zCMx{^*U2SmJPQeP16g2CBHY4!D<2C+Bvw6 z#UVT7Zp5x|licb1Q(D9OUW7_C4pQq!IOdg?IxJTdMa&_%>>%H#aWc_Ti+T;_{-Sr` z8M)RMY2s-4^%t`&kg#0@OP`z3?&@th+ZArAe}HH`=U18np{E#TC*G`vu(z0u}_ZsH`--`D@Ki`>D{j1OXPOd+vj=a2w+^SRV)$;?der?{qxh7?ovy=9MPY_mKYiT9mVd3IGtZ8ac#(c!@Cx+b#F5CIN&=&PJ`~-4ZDs6 zKOMCs4XW93qYeg$q$Ha8iP>adQIWT>ST$=---RU?%wPMW^L8#`Ylh1~-CH4{RdD$q zLWTK0v3>E56Yu#vNz8TcMYlq?>hj!lCnTkMxxwtg7=pwkNH08m=Qw>eA-YSZ`06i? zWg~1(yzz~Qb_L*!Fqz7+?8mrMtO`>NFj{%)!q^qZ7vMb134jH~xFld*S)cKiO9$?% z8kc>{lY>|3hWu`AA!} zvX&(DT0^&mTw^cfoC+?c0Et1)91AKF&b>3>>^D;0WA&^~VW=pL*Z9B`;N<5EK00?P z7sdK2f%{Xo$;z)?Z#1x5+3Uu;7Ws!Rigi<9m#`T2_k}>bFV7~(!!?D}ob}$AYdZ<5eK-ZQAZ#(LaCy=^t3X(|^%RO=`|GAMBw_)ftyf3_CmrjZPKV{=Qks+_!D=c;ga+!E}EL=KwVNtTlX5@5uRc zv%}6!JF;E<3~`Z`SPhYWL~1IDJ3%+RcwKSQ&xKpQ81x+X^>s-E(X{>J!zjN_qs%L5 z3U)X;Rv-FWEnx-NF9xG|e2KD^jX1B(!O?st@0*))BO5{NT%FJoqrwJF^x}Mm1sMJ~ z0YSqWG+Qm_NKK#i<4{_DkOIPsCtn#Py|h_rm)+UfM^4uE*@i!QUDDoOmuqH0LM=iF zYu{8@rH*;PHZn1_D8Pza=YSYKLNsEk3g)zIb*v<@b2uZPyR(|8WDRwW{E?l*Nm^)k zbryYwE0{EBBQM~`gMG4%3~a|Unvn?Np|r0DC*;`thdW-b-c`kb(ViHbrdP@ z>|mGnV~A|Uqy>_{h_?u8nOX3@qUgk>%gmYytPVh^(dbzO}w%i z#e;zb6pGV|d)1C)uPgV=9a}-Yt_GS*s=N`7cho9`o0TR$b@7&^xK`Zxo15b^Xx;SM zaYy)lO$V@xw}0x5M1CySr~6Cj#Oq;uheM!}^dJtge{wG$o!WK4T;tojur$=h+|VI^ zY5iePT+$^cZAXB%T5n;1&Nk2V?QkOm;a=ANP6t-#yd>|XtAY3WO$84lwHb=l+2(ES z9PQswKPssO222$uVlE!>1{Xx6MRWvh;>r!f&3HZE5eyWHilFSb^qE|3RfH zdFUG4HmpM`YR=cR)}2=i724ud@3Rq&}B(_m$Rofmz| zrJN+=WOj|%_FXO(^cy|%I1$O&FIA#HPpqI1{dE$OG=R-V_Dux8l)DoeZ0Nr|MeObj z3dt%sU~YOWCry}kNwM#8ujYQ&=+ z@M}lmrB4_Z?uEX~Fz8;wS9Jihm34AUi1GX5W?*AbJ{M>B;9L?e_-8-TW+d1Lz9g+ko&-o^*CsOcxi-0WHvJM=P}AW5bSMx zipz2ynCjILm$ZT_#&11B878|hKF&}HzH4Z6Z8w-+;X8op{Y;+hFG*^i9(7IIZaS!) ztd`&Wd+aAdVYsk^*2zT+u;$X1v=JuFaoKVv|JR`0-OZ(W1Q&oewn5$XsN{$$=!TPL zvrqvh7w&D+T=U^gTB(n1!-HG?9tP`E;NJ;>xF@A8Xx+;kksDRYcnNk}{{yRRkl zTC5`V%tgv(tiRA~2kIR?i3!CeFLalJKNLFBhcX7@Y4cPAL@W)qD?FU*2#w2a|A;ZT zrktCawons)rcjDH2k~S^zfN8Iey>8X?LRKr5`Ty`xe3U*F~71A3b{L63t-&j22i%% z{8ykOt>yXSH^FhTxpE@%ll?_yfeij<9P1-{T;XmI--ORc8HK7PH~;Wu zU4o`o&U$DbsHi~PG>fJcRqU4RMJ7pkYkiq^_7hl^xIgFGoPp{|$ z-6gDA{97=JL~sfIbaIm9e;|;W{a%imy4U)02q+TSoeqJI&ARIg4LGN_oYk(UU2Bfm znQN_rnAs#bjy*^TeV-Bc#TN}$67{+tS0G}sUMN29ntJn{N=d>f!X_Q^O$@2X;&7I~ zy(lzLozc3!7hq>@94bB4APbdvZ1-{0IYmE4jl6SaZJyz?+wxKUe_Ljp7u&9QrY}ysna(TKLFt zA)`1P^C%w|#!I-juzpm?%dUx9MTDa(J#%Y|5Y8*KuKF)U^WAHc?3H_vk{$hzc%dEH zE;-=T!4!X3P^Bd(Qa9RF1BWZAnw zH5chpmndHo=2w{IE-5)wjy%3PB=Va@5WU~aKt5b10p10BfBBajdL{C?x}sq+Zmpo> zvHT!y4k(9tjFaT(^ZnzG0&L)l&GA%tS0pa8G~v=}sa``@I7{hlOsBv9TUZKrX1bG6 z?}Ex=ut{u&M%Kh7mzBP|%e8_gh{Iqimzo4*7>RiDP1B}9+;|ARWAI@j&#>W`forY;;$bl~X z=m0?EuoFYWZNo{j4U6SClELZ?Oce=B2ycD`EN7DWnqGMBcyED6E2z`mS+@Zoz?2PD1O7JH1|{U_Q}#m@O8#TNeI4o*`hks= z@Cu=y;*b&&qx!CMTCc+cDna2HSY#d|K0XYs5Gp`WH+C9cKJp@qo4q5 z+*rt_)wK}%K14_MP`M<5Xj)D4n#r9|2zzwQ^{hBNWqKZ~+xl~k*3DDLxb#Q#M$mM4 zuw<;ue!Xk@=_{5IKcUA(xf2%luMDmInCAZ7F2cH?HCv&32XEl~8@SV|F?hM82mNA{ zLXAmpAy%w0AL8Pu$9VYj#?tRJa* zM1GZ_sXo&5BmDi#h_1Y9mEe^24B&mdD%o$k_v20B>+NlvQRtHJ?$?Za_FK4rr#rEVnd@$Z!f_O*Rv!1y~NuZiv zx{^%GSMIyG4CUznx2M4g{DazPnTer+G)HR>$^4xb-E+L%M(4aCn?P#s5GCgA%ZvUf z`aOM6mFB@d;=BzWMe)}3epJgno$7IK42W@#RXMZhXEIj~Tz}An&1D`#kMb$)R5z*9 z*v@emy>s0e%m48yurCi4&xz6&fqZpltIQ!a+(BiMy)7em)#1eHY`k z^g7l9E1Pn$FdCPYCZO%X^kJV`<>8K+*}IOuXSCqe>Ye;}c&A;t2Dr<6s*cK!$m!WB z>6s>A^WqpDi#fyJ? zpPi-i@D-Qiicp$=d0zC%5;rtMu0$9O9DS%`rQ!tk>8fgmolM)YhzGcIjmR*~{Z;6` zd67Tu9wcEot`TiQ)LX%br0BQRbnBg36=eq8plk?Pny1*Kb;Y!P&9%5P8xfc0@9){2 z-T|I#a+T`^cwaAUJe7BRMf<^#cyovCiTxN-6X(bnkq0PIh4vNcfKzUFW<~q7X*l)l zrDceRdKD+szPvlbgKB%I)zAa%80dL*40mC0+hWD*gd@QH%0!_|`A<)X)=-k)dj6+o z>KVVe$6v#}r(5!lQu|O@g<=C7!>u@v$XBEJ!F!JjS|^`x=3j_H@!4JK<_1r#I?%HF z!=HFh_sFlpL;Owb#@|yP{pMs5R_*TH=K7z`UOzNBR4VjC>zV6s#!)cDH=T%$4;bpb zgn5zkbMHM-)ign>-`{<_t_s?Fky`KIchVY906rXG%kHlLmMHjfocrT(SptBd(uTji~BTI0P&@|O%L!CI*ITW4iT z-cX2DDHWLJX!@docNnVsUq~l9*}$oyct{BZ{AYKm@bg8P zyDcxd7gSY^6F9Za>J#0CFDOzJ7PeW|St1HCoD5q(og=-z<5DBQ>%Y7E+U9{TYz$&bw1jLyg~yp-EQY%U*&x(a(3aq=4o&aiAY2c-^XmYocUTUxnVMd+`)UdHrvRZw0O@ z)!ZY;1lV1my%?@i>T<#?5_^-xQ>wmZF9&wau5^JV4@cx!5?SptYMnrNzUABC~^2{>Cnq##QRw%%OaSjX_Ki z3CwI`J%j%T&OkB06Q^*Fg!`8^#Lx`m7A~$0<`u`J=U)S+a%^0Ioi-X`*_jf5=O*9_ zt88uMr0&1AKK#rp6nbYM=rV`2T8nGZIWKXzGuCYBNt<^$M5wnhhVs=)8!tI!t?xPK zV%`|3fjEV^wzFnPU=QI5ZX}(ckoGpy+oLo%hUM_F14R2BTh4n-h10zTC{`GI2Vlks zvyw+Z+n6!$V8{IFUu$S!?I5_ZQyx@byA@#Wh4K+oHgt&T25&?W@-Txq`W@kRWW*qX z5+|TSSi?La2EdFvLyF>0iQFjd1^1S>PHmrSy=qV4^}e&5(+XP`cOokMu6ELqkw3o zU8^^<6#baFf@B9Y^fv-V)X$-neQjC&6DQt%3u(-7?G=MH*3{5SbM<-0(a;xnuK7ya zXg36@__@<8=b3qvK1Y4kJaa@}jnge~XVrjakJX$UHaOtlaf5Ge9OJdaLQ0S8T3}3Z z?6fnY>Kl0Yoz`e@%4tv{o8T z$mxbOt{wY72Y`H-yE<(RUB$Wbe)USzbtR3&oU%6Nz|84scBUAnI^$OM-t$3qY!&Jn z2BO8;=}T!%jMutj)Xtf!#&hk7=@e#s+>Lo!B~8?%x@Ww>4UQ58Ajs5Sg|`SL=fz_R($A~iwSo{ zsn)S)IL=zj@?lM>!<3_{)R-gKcIWx;u$Devp82*gjBSUuN36}-;fa6eI78oKI&aRA z_Mbukyq97SblsavZfuaz9FneKT~k7CEUe$Vf)3G|(Y|vI1&1}>x!)(Tb_`|5bkthJcuRA<@|TWKxn&$W}*-ppG?Hw+cRwI^QBi(?*t zssVHIP#l*SNMEIWDL3=Kxj<_>o%-dtjAWX%Sw>O&S8+mLl%u$Uw=9F~KyEkEs z`H8ir#M7UF`e={r=Qm?EPs!7;Z*TeIlSU$Kn**LGPtmfpB0l(tVb6FijNd#5V*E}? zGkp%#i@aty-Jj#SX%6I|xl@eI%gHS{WX;^a2A0*F^SOHN5aGOn@cY^!)^n_#nIA@4 z+};6Ea7{7MJQjTZT2X~$Zl&S3R(jK%`;j81*zGuGPFz@XF=lL0t2o!3A5GF7f=Fa^B4v>yl&V;kmY^;_e?CyE10vjlYM^ZAm$fe=JOjpCgdSNCq~`k8zS^ zM|HC>hm69T`%hyht>zbUs?y3KlxhX-_qHOsz&mG}{9B2jH05ohxB||549TA_Yy--1 zt(n^y#^%G1Av$KH7ZBg(zk{WxN z6T)O|0koZWWL#XaX<*IW=Q>gb+Ks6I>z{k~F|KgKpGUI94EvR4XXS^zvV7_e0p4}b z-G(1G%H&S0M=Mx@so2X@XZr(enbP9VZ@}T6Q*R|&M9y)?D^`ILzHxk0mdbdwZu6xaB+(%)iZ5v z_O}#s!T$Cpz<(+c#xrXq~phkoMjS`E4&*);d-iN1F?LJZ@3&utr?n4#Som8xMU( zwC%AYCR|_h7cB(bjTJVC_1ROaIVWk@8rDMF9fM8Yg4o!B z>3POQurihq(jQAPY7R)uI|KArnQM00HCoc8zt7 zx)(}GP7$edPF1k6MhyEKagTq_g`*Hw8sx~+Z*cA?_Pp0l`B-tMV^7twKL-fcPw9(j z1tE(VmUPG&TQqMaMX;9^4r&Z5k8{lt-Z$qE#h**3d(L&Vy9d1QoLiVNPc_$(rdaA) z`{FcC)#o~MB=1Y{d@5zdq@5S!LkpP#bA_a$xaRoJ98vda?exvN#~RB{BbI;X7F?M7 z9AnHhuQfMFTpb~oe@?`iKc-y%+>uIctO@2nQ@Br>3psZWAiA(uWYdXr2RP2@wK~JZ z*PBsCGHqe4JHkBE&ja8u%%S8FR?2Z%tAj1XxxBeT-uqaKIeuh0<34se(HxQXXm6>u z9#UfJ+H=b#&Un^3cJ4`;i<^Fo+{3l!BF0K{U?UE#>a*6M`d`~JB`(FgxMD8<9V{lH~o|JC$wh{lKt>kmZdzuWp4vgCED5(AukbGi129xwag}3X76%%&no5lbF*T zGfO*1K&zTZv~Za*;C6)7^t9vBoHA}mS=*>RthAvxc8v8|W1o7>oYK2fnC_mFmT|=? zfMqU(>@zn!!5?$na!%~jztgb%*)v#R4moUUr3tsZqjGg0yJ~tyiQc-l4*1DiFCmTu zytfwu#T|>3f6ZKhJVKJr2&1tqrxcL0R@`Nq*+D+Xwfnnf68;~vXKX>`wj`TBPN~J9;a4Z z9!r&T>`?bH=K|+g(=23Xz@)ckw$~n^Qe(_X#Wg1aWfM z$85rtH(F>|2N={&i&Zkkb>TLbcEuZ8q-ckHo0en5W#20wXzmH}C0zBoN5073+aP2O zl(jtIn zHExW_x;;0R-k#gtX$PtI5f>iEpKE14hUJzY213%9JL!MM{ev_1gxDYX_Oj-z!P>d@ zKP=FNzo!1iomn9@3~`&b^W6HLBcx}nbit99#$a2+9e(U7x-cjFcHc3{EDy1m6^B~% z-MiFxrMcU@0@}$R%YSKaZQinmrt^(6CL?W7m=-2>@QF)rB_#!#8HOyz909sz1+>_i zR^-J>3kZDeVd%9JXno&FSZRg`z7zKD&R7F#-3bd*Zw*Ygy7qR*9H~-ggk7>3v)=Zd z`-N>Mag#i=z{r_9UU6-_$vGD4&m5WiEsVXgKITfx3n?6Y?cw=4qT0+EtG9O!(cK@n z-qqb(_$h>ey1rLfc^$(ucFvXZx+83IpX2H?q)qLyHr&9;BQti!k*>AZ{xt{bv>F1? z&l^+iI1UNFzt@`fN_)?HgxJ}(Clu75agHuTwZA-P*6`h7pM4Hxm$YY$YF(pAJ?=H{ zyoNk*n;Q>mRc z#dPkr;{wY~S!igi&7rho3c*RcCn09&)VL=eQW)FrWo^CEwR801&Xb*FPLS<27qrir z!7v%yIARS^U$mqGp8a5;9^qgbi zX6+%1K9T(`2#3)TAr82h=eOWX7V-DfD zA9t|b9WiWY&K2{wH{k9X;SXs|(abiMYT^m;r96!tgFi({9kTe&NZX5L z>|NzF=B!GYV=!)xMVqnKta(^7CT(t6-5)1bP~BT-X5{67U{2Am6bH^iit$N$M#aAx zL(E^uOJa1atb; zIwujTly@ZNOW7)9?&!6&#;)IzIPV#=?K2MLfitEq%?%q%acyCtHP?D~-5J+@E)D)R zhM3hF3psd=QQoze;_n!X)Hkm@ubU<$>EDxfe{J#Bw^lM+8Y7Kt&b62`H0oe9uXcxppG{A0gFYFV);M!uG(Q8%}>E^xCoadfJP* zDlCrymp>+0c}zh@X=Lp5J_c@BO`~6VCz1NQ_Nvfa!#;0JUHl&x1k2x%Mj+13&%75l z;u|AZEr$54pJMD|UJD;5?UloqCjik-Qz>JNq1w2XGV9L4y(X?z!J3y0^h*(sb}gyC zx~D>Up200mGtw_EY3`oq(q~CYn{ZB1jUAUjW#4I>cV|h89K$5fiy>BW&cxX`g7QiV z3mkvPsq;S4c-J1=DJMsWv7IJx?}!mKVvNbfySE_8jQI_7ZxqEmcGN)Fx!Na;9HEpZ zC}>_ga%!wGk~gB(_1U8rG){%Vn$`fj9uojbo9pdh90QbfZXM8)W5hz3`(b*;gwnaP z4%*N27B;1Q@*5_&(jJ?}cPs-ZIPAdqxE3<#iBpPYOf;7`N6Ka2`Qb4qti)YulVUGS z%(bVw)CqyEWJFrL_HE3`zO{02aNG%Vc}%^v9h1Us zU5RCTBq6gB*Cg-V(py*g>L~{qdxHh&1QD0N$FYNunxHGP1iOKV0>}9R75)|OuQIuj2z~Y!ETwmR* zuNd=0I|l8XGzYfNOA||N?7aOnr?~2x$)I;e9j7)|_GI2mQEJS*=o|K=?OV$;X|J7; zIwrQ;j#H$14tS2bGg9zb6K^;LVaPK^*2W5xLU>0gnm^MJ!;2eFb`Qb#KjLiMSwr7@ zjXB9Ra~$Jd>v|=n;P^GCKKG9yfHa1!rW#VV>JEDmVb9UCmL{xGT1h;5h9Hu($ArpB z5nFFAC6_)&#N^yrLm_OO#;|w#?->)kdq=szGnSCv*=zAD?g6ndmkRX_Gt4awv9u6Z zbWlixIcW^#^Auy)*=@_S{t+>F_6Po$!&zl+T~Vz= z*SB&q#+;*tC@!$1HbazXi|KiF?m3~DXM#l96OLpA38pp2GUiM94SodZsyOE!$(m`1 zDh)}7oOWv1N;$P+OpW`rXENRh^S>*^Nt(E#mcqimMFqAyR_(fS1@-*DmY8-~`ABr`^_U!jddl3Bs$dQA-eGL~(~M_`7$e!rQqNXKnoQFb+A3x3>yh+FSQD4*`%A7YOm1v%4`aoyZz;+|){= z%`iuZ*FNXmOIj0#bnH3D9H-b*%Im0NMj-S)7cAY{drmQrX~;e!q{~{{N@0Xi_ZH?3 z+uBj2ey6aymhxD^-0`kqt}vWE*S7SX<1}e)Wb%^tz;B*&CvZhP*EhEm=$?7UYsSs7 z7l-Qm3R6CJrxElsM^M)rYd2xXd8oN(baf9C!#b=Gg|>Fi`VJ#LBg9pdoAx&44?F02 z#>vRP^K^Qq#k!P2CSl)0=wc0t%r@ehLR||gZLQ3>ItLu**=ZDGEcE@j7M_2LGtE72 zWwf0}w{|G?Su3<=4k3mT#`f>p1CW2_X3+_m)_aA-ramL|^%$d=ZH~>gF!xl) zo%?t&PG$NycaYARn}Rt6ZIwQD;@z7ez&-7am%Nv}-X5D_bNZWZ9PvR>C3@jPSAIg}nl#$=9j z7JJ4Kn-u39+1q;n^ozUoZLHkRy>fKaTe-hw%>0qDr(V)o;kjUDW#%-OwBy+;Ok;+P zh?BR<``i;0FD$UNvzFGu3mevO4*<%y$I8^)%e(i<(KPUFq-qR>}FF3O@!-i^@69;S!Jej(qyuljl z#xMp;<&Rls@EiNGVJth>B19x?oG-f*4+sU+N&OFYLH^gF^ zqwagm%=;I&-fqdeNhZvd!oCLBZw&hvW{>=6oof$s3|-8<@~YL_8#R3`8PG99_GH-k zv2|o&ytB3%%pEyyYOjggICrW;o|!#nB;ks=cOq5HOL=?*rQJ5>hWm)~$8~O`=QbCV z&>abNZml7|np4PT*t4;J<5bNbW6Wf&MW{O$fZ&aT&w5Qs{T~(>-`UYneFh=+y~1MZ zNs%jkZGgicmU3Xs8y#iOef70MOu?UX1a0kL|BI6Y9|I3=tz_%9_lW+?gMDDn+3yy@ob#SLL|}$B zs~GcI_L||Fc;!H*r(4C7PNBO!k@EYBNue+LuQ7-dKZrYi}j-7Z<4KpR~2K4g}GNk%O0CDZ7*=#xh7Kd+uKB7ZTzRcmniYs8?tv#{BhoaD|qZ^p1nqn zWgHW}cCUT5kPf-uzNVt+-}&q@SHfRh`93?$0j+cI6{0nUKI>XBvwe)g-?YczU7le8 zewcZ+V@83=J_m-uifOVq4k?B{*An1KyZdCXajG8^bjshm8f)$ei7*V2>N1v0V;r;Y zBrdFkJJ$?d8|$oNY;ndh!$Qqjo2oKo@ufPJa%j%GTXY3Qh7khH*qpJNDy%`6I9H}~ ziqV%j4279IR%-NKTkSoJ#s8gKE_5#$v=%c^-dynnIIi81Fjs{B8mR(kjM#{@GIrGr zgI;1Undq8!c50c)_ArhRk!Of{8zYH7PubBu!`8`OB5RR_$DKGCPmO zsJFHx+YAG7ajB0sE_~P+7cS4w+X8uQWuzP9CgR^4IC}0yqBz%1;9Fz)dd-cMxAUG! zi$fN359R8)HnQ$rqs?Y-n2Ivz7WUq=iYIO*h&N^i`5FV0XUu%}9tT!S3rp^JOtJO9 zhsM(gljtf9os$*!4%nPC2zq2(nK?2L-dwvNEyeh@yaSr!8H;9k1vvSTQWnmiX%b?r z;jJ>JK-b;7Bff zKWD8$sUAZb#ajzac5EEZ5yNiX+l#kgYz@U7bFTUiJ44Gvb)wNFjiH<}vR&mYD0C z^TKt7P1cro(CXg%XCb9Mxi|)-;)G} z(w?&h7!zl0O!$p3cW_4!Nen$samqS|$ncAi`ZtY@=A9ED=wCa|Ywdu~5(e;c+3TWx zC+WO9*WSX5>6Ty^5!+>rK-L;|)a{6S_%P zC|DhvVrxwu&M||U^Oy!CZdj2!GlcPxlY>@Cn~N!WO-=Z^;`a5KX;N>FouV-Yme*gi zKVff?$_2dX6QV%Z+?hsr%|+w2XY%%6OF?(dMV3FpD1O~@q;ajG$C2m!@S9QbXoY?5 zKX=?s$(x>V?lFlx<|y!n%<-CAy><<3hZd5$%Gudlevjb0w)W!YoP+T%4pI8DBgFVz z`?-0JS?U-f;K-i|0%)wPnZDO_(3&%FVr8MZ8CHnG&I8Caj;Z}Nclh?*VXAxNIn+3_ zxc?ec#~u6KH%(ZY9cIY;9!t@4h3Ko2QW(PAlV&!gtj!zJc-~xdl6yuei#Z}v-;cT8 zC+w}un@9G?9{WvmPI&x15+dPUvkE58q2Zk70fjs!G|t}JyDKC#+&-t;-wsntaEzt& zGJ-PP*pv4ujM=stcC!8%K}L5BNSm7XpwQowZ)|UU#+dT@?U_68evf^XlSZcC*r_*S z?5)8(Go0yL3y*UQ#pxABykyK-|2C$=?hDa`a*RO0IoE)C+dIoX&*iJN24u+}q0S~v z@#nGUm|&g>@-2r5ffd6Tbs1a5DURKy9(VTr8o6#h&xIz4(anVh7>|r1{Cd?TV-oanfp6-1kqjtWgyRyi$3!t zO^ZcStI>$ zZ0+;6Cfr?Fd#qyY&C8ZX+Q*$4>T705s=em^#hp`qB#&v)H+NQLn4@}Xt<~u}gOp<& zyEr=yE#I*7mdlG{-gOKlpT0*-{aulea&2Ua7-A~i*<)dOk2U8wx0+x|!F6UXNzNCy zxW-#!*JmUh&=PVK-QG*9YmNcRKW0S8PGc2m4RpCWc82prcy?TC!(k_( zzmTRhYmWK6d}dw7wPHwUpTk&kO%?aP14!r`nbd6Wy{ny~`qEy39yYDDk)C(X@d#5$ zZ>|B6IyMIG+XMb!ZlH)imwx6Au?1oc42O`nK=(=;czmqQpRxzU_F0Lubj;n2vXXAu zjr%5YkNNhs=0yF?`&?wify^^Dh)5pu>S0Bw=rV??MO?{VEskusF(xR~%A*f*MX8gQ z#zfc6JDYRvCCHSwdPj^yd}^e*`?-d`a^3+sf5iCTIA_x78xh!UOti>~ zWv;l#q(xif$75{lrkXb@R~y4wHZLTbyu(EP8fgb@C26ucw$Ae$ON1wm^u#hZN_5+4 zX>Di4ueVd!)tU>hYi-fKy7o?3-D5LzXRWZ41Mue=!SH{MB&a#^dg*4*gpRcV>rtqB}>3>X`XrW(C!zw^u^) zjAMs#XF%P*c9!lOt8Q~kw%UUBw%hCeh!KaA0oTqPLfr&PoHod5lT;zjGS<9ZQuoPsq2nx3GDgiw}PW zkf6Q8Tw_=}vLO$#;S&ekew@R(HU_lyIJdM{9qc?ZebHIk5ET3NGjp?stv*n_{k+=#|za zRT>jAW#nCrGbc>y+A*SSj>W(>ruyQUxhrGrJm)q-jMLqra&fG&GXcRCMk1(z5piMwt6o zB?tWDo$I$bZ6TJuml$xIyJcg~75J6rc>N10l6_9eusW8e;F_t{YRt*=ur@+s9m7j| zBy^-PB#XE6_V`?Tfn`N$kiDmzW?4x64l`ykO_~0imU?BMqc>;8sk$-~2F}{cOgv{~ zgC8ei*c-EIZ^Ws;H1pQ`*`xm)az@(NQxtNIIp)45l)&4IU1EowjTT3K_0nusn~5zf zMpcI}wkkqiQ_yGRRhX5==*?YIig^tI`yaOQ#hKe-ATO+XNN1#wpA&+(SlEFbAq5EWzb6*;-`hqmkJYx2#)85M@pf?svA;iZ z?DC%5^m8rU_dFLc_>cMHA2zaBit8>Xj%~CyxBg?V?Y}t_?ERQwM<*?KxV&>-|4Czy zaITH$wNvbOOKWjD48?=I#z@Ou`HXPRiR?e(jM~l1h&YYCf{?dB(Hub!Zf{N6zvAX` zpR4UBZk)Tf7LJxi%Iu%x`gcva&$|}vcArDyD^FF`J%cv*iwS#ttWBj7_i(}6yXAS0 z3BnWN(ng;%ur{qonlV!>>|3eCH-_2Ux>HK|T6y$tL{Y{N#=z`ZBTQi|3CF#*HshGl zs(LSp#XqAg*^A3=GHk%3IVZ;anETBs=f%RfmJ;3!L!Uq6X4c+=Z($4>kuu}Z)LW}N zY;JI(o+b#{*x5r(`(iI7M5(oMB<{#NH6@SWk2C@b_D;1*kdAnF7dVveCDv%@?ek2 zFLUqtzOv&^>szytVecuEvz7tEwg;N!4~v0+=L&lqxhiqxy|cNe&h{DU)n)Dhn778V z(@5)TC1q&PwHH!T9y7isZYbjzT37y)z`m!}dzsUTIFB*> zxR!1~h>2)mWfi-V2VV7FW7s&Xz?!r++*MkG(J#LQ#xv7Wb+o%PS~B>LwgLs z#5-pwSDBm4YA08sX+YtR<|o23m2L^HON8;la8?=E&H2^?l6IxHp#g&6yj8 zX-q)FJD2LmTFXH!%vp@KH_G4IOW1HlCA2X|0O1|Wp=bx)mYJp;{hDFqbgtZ_wnvia z-3dN*?V-C7<8b`nYYur1VZbzpS@va(m7_l*0FAUVI@=jT3@R-Nld$4qNg0a(QjghY ze=HsAI_LUTUjei`4y>q@Q*6abDQhqdZMHJkfXLe`CSpyYq?*PM#TqO3d!(58oFcsX z*%MB4Br&}*7kqx-^Tc${+5d+I@0p>#KJ84SH6jYZ%5%Lvt+3IuVzy*hgFq%P+=xC$ zR{b8ENNG-$n=v4uu-B}27%38H$7JF$rzmqxE3<8;g!Vt?&cz+OUSTHD=e~vpY+u_I zXvBn!Hlkcw4w?RFW_kagAAP|?4BcfcFj@vlg8H0otsB@j_~HN50S4vHwOO9kosJi zux7^*@15re#hg1kWUeLDw}T)@T;rx|kL|0r*68|~IpJ{c5rR4<66aiL1bgKO&^;$0 z_exRHX=cIsyZ7eV%Zspgj?wCvVaORrHs2i+b9YXWv^}=)&fUuoeyo+onuD_EoU>Ik zk72nUVi51n*;^z|b<-I04rSN_QuW!f=wc5Ym7k-$`U>HCGev~{y{Ft<-%EsVgb32R zH&WzW`&f63INcP+K=#l1^|%)zSlXk~XYMJDG4r0)3^Sr;O!?Q4c7VsoOTl#P<%zXs z{8k+U$#rghiZ_z9)EPt7W2eZ8G}qed+_4aG&(Xl0lOjWyS%qxx9jCK*?sQL^)n|?+ z-H}G1+1isBevG)^7sG1Y8;O80CRL}l_723It1N2o%%8QF+QXdNrf?0+_MT=m;)_FB zFJ=&#HJ03Rj+?2IXF3uRo8NkF%#H+uSolbWA1YH76+ii~&+Vj^XPy!l?e9)4Fi( zWbYN$#>UUyX?DeiYqrl*q88>mwMdm%nh{ap! zHFECkkGC^Y$Bg3?dGDp-xK7?+7dQq zRN8}BA?D%gHK%6t+>=gbj=`ZahqUj^bNX?|QG~GL^v(&Z19Gip%e&?}?VQPSbZiZ? z8rJMa$n(r>2Suhehb~DATd8ejINC7N_I;hJX*M#uDq?gjr+gdYwYz$5HnWkX* zT-kPYFX`a8H-4V?r1*~;ReKNV_Lrl4a1Rr%Z!Dj_p-HzM#cTCmiKc`^vTQLCr9ed9Y`qfKynxerJajxj6Dr&R=ssWv#V}HnN&wimA(Cj=06X@~U;) z8wPMs4Z)D(Y~Y(wX*y2xMx_lf(LXf3&l7!v^a2*aHr?Um;|C-z>L6A31+L7}|2 z0Cr!qYBJ6>_?ib$|4R$xU>l2&V{LWryrX#9o9lUD%`NS@*XVhiYixgq75B4Zy!c%E zh;zXNPBrWxwy90Jk9`o69?v0AQQc~XD*hTO%zR;JBQyRLsO{rf#KK+2BG zu5YX*raU5c`kv!#Y|VI(ut$#b*^|Y4M(FgL=c;gu;g4xe2+%*L7DgF!ohq%Z^1srC z`3m{8E39O=Gs7U&ADcvMu7%_trasMEBb6}C;r2deoYGwTBx6NzvXQ2QaLOC`bFVF; zHdmZ{8EYzLrEHG0SD=1eBi1BExSExAB-`0pfNvz__p_6<@L$v2Y@|elHP=Go3EM|$ zY{{9mr@H?iE3<4(A^bhp&Rn09z&&mOg*XNp^Ijrsp7R}TMA()XCO~Q)TiYkbRJ=52 zC}Eue9d1PV;xP9j=8nNZVyy*?pO%o^+(AorW!3XN^Wb6G;{fkp!+~awJ=wY!|Bg9Ae(ojII9Be&iU|pQ zWHG)P^GrpUD|~onjq#UaX2)Ao!ga(0G&oFEthvVS#TZ*!bq1xRy^~Jl*o#9j478D% z7d&!UaSJ7_Y2P{n^ixcOIA9ywPc&^k(HWw)|63zTc8)ywFf&+H8Vlqy1sUr)_Xzcw zTk3vJZ05UWrq&MW!+r(f#W0p6>|0x^G^}XaIMQs(nk#5vWQm==>;_fcjoSjNxnRc81)}R(EduRR%(xtkuws6$lnW3 zdd=;O5hf6FPhlW@j@X_X_F_WX2?#cgIngl2M)4Vw|1x$e{T_=DX)YY}xtF%l*=tZe zjO^Gkru=BmE7Ef2Ny<5Q?C6Q(yg3XVv7N^z<=!*LW$uBiGPb_>8T+a#&JEl&(oFSD z@s@c`>5wv4^7PKjly^=zqL(+a;+(5@ab=P2o&%`(9n*I#Mj_u22IYd3B1-#T6Iga- z!IPGvc5zP2kveBB!QpQ$KNzg8=b=tYN*65wXuwf^a@wP@T zb6rc+bdQt_T@gh$tvHmCc4W$#lMOM)DX1_F)SVHhMAV;?7ilk9%Dy(V(T<^0Zj5B4G!h0` z8M`BSPhEw!6VlgT!5k%Tc(gI6jLcnWx^!pt&%9E;Xi9_pBW!?Tc5Dc3z`%H)qh*Iu6>Ca7V=SD6WV(1 zz4ef?V#Z9P?0ilgwm$M~%UO%eWydA{vqCQA-MjjCC+(9LW=iypn`>i@0jQQzNM+9> z&|+*YfNbx5n6qcV{hm`0E~nLs8<&dOigTK0?2PRYW>QejleKUy{Nf!0KFk?;XJYOx z-mo{;MO(=UInG(ClBWE_-l>Op%|Mzl7Jg|PIrk$@gyJ%GF5Vv-$}SBhxS6K(Y6>Go zW`x=2JBCdAnhW_nES1l*HG>S9_>h&HgE^Q5s+8VP!#hFQAYerFlzPA8o zo{NHDuKkT1A|Cx*@egDTX`nF|=En%Z*nEa% z`W|=EOBiXDBaAWOmo{?V-E;OWuF*`RuXDX$rXK%5VE|c3c{Jgx@1P}qP`Znh|BLB**VGbH%)BNH)J!w| zJ;pWpv?5sANuzmcuC3#`GKTO8i$rP#6~Z=#9Csd5l_hT3>mEnSMql`gk+TZZ9+7`I zj-iG*!c1%0Yg2S=q1u%qC~se>6JSj+$v%UW!%xwIaqShazNTW^o{2+i=c$A_hve+r zsYN}Hx&60h1pQz0<8cm2ox9e^-CxN@Wo!|!)WRXQ+jEJ73(<` zvR&KT#e2yKqDlMq6H6|XbCC!$?i?ws9u{9%v}{Z8>{Y>Xk!Hn!KS)GhUFUT>w8Y2th@$bLY#r@ zdu?^un8PYkT}v%*=0)qVC!W(D!=z~p5!1eBTvHFRj5JKSl)2)H&JKHabf%!C9yc^V zJ+=h%TOlEO3~`b-<817iW9B!-EbbiFU{sp3kvC1D-#$l7We9`1AuN^Q9LA8^++(_Y zFGR{RM+Dv;^V}%xefAsoXiJ{M9eIWj%Cnf?;VGeohaFZQon04D3@@5Vd zkq|?u-`k;LA`hvCzY|Q@7;Be%r3jIn!;tLElb?0(0GPiv2Ez!U4=G0Qz_%k34YQ|dg*pEnf;{VA>uF|;NRYAC@Z?C@ zfg>#e>XpWpO4@r)cF*X-x)x0Kjp^cJCSlqdrZC$WizakL?87`K)^{BP1a{;-gtb=u z(T|BdbnNw^HMVyB3WI%guMDy}=Jv{sDP?QMQNOSZwa7Tb5@lOsbSg&N&=GQSY1%W6 zY|UA)w5D8eo#Az7?nsXoM|{9McaGNzW0rN#Rnb2}+Yr2Caj(?L z8WPm_&-v9IvJ`UKS+s7B-R>DSM8b~a96e8vxxCj@=v--MJ?(VHm-hri-#L{s4DqEC zr&d~6yYptxSEbK$rwonc&^EeH@B$u4Rc#R4jsiZqYUWX`{!jvY0sH6u7mzUk087W37qXJ$45Ei!<$H?~wYo_AHs^&c@$sp?1ZAm9^qHOU_eT zZwKLn9AgGW3+r5E&A8gRg23;d(dIW!q3^UNj!xKfkbkco*0iy|zS9u&9Fy;UO&#JC z#^CIno0c<8DTgp3_WBQbpkvO_+CTG5YR`P}Ffu~gTzT|m?dg=8lb*7gFj{& z!aH~H#-E{XdWW@)6Vo!!ACnU(CS3WswlwG8t8sAznY%w`Ou&fCj&{z;mA!L(L|oH+ zd+#WLJXe0)4D$tZkIDF&rug%l={shm74*0kit-x6AAdygqLF6e>mAdrD^7XTHU?Zv zox|{S4ke*B=Y;Cm8#sP%E%LUv?BibHktdD4-?_3FY@PcXaLo|E5l8xb%LC$O>==r{IU*y_F3=yBYe$83$Uw=|cU(MVHqXa-cVwnr}Q-wSVeO$GI~hWy-~+x}(m z`R6*Pc=bquzi*9H=aj5ty>;hxlbSXh;vM0wc`PmOw36sc8A-S>?D+S-C-BJ6GgL4x zEsve27}3o`4}5K)+%e}?O9`_qG3`~(5oWl{j2jbgY~YO+Q+(f_W7=ya{Liyvu1<~H zM|DhLnw(e0&JUa9c5hUQnx`^-nt?-mr6u*9L#WgZ(OYuH@!LKp#?THC5`FL0ikpIL z+?s3gI4AX(lPAU0qn0KvSKhNrcr9^*p0}FQof#K!u4Ik5H%`+IxsxglIQNjV&gPnX z!D^1_p*EKcNn6`BeC|cEJ+f@eUE2|RjM>r_xGiBA88^Zd)i5ua4 z$J~KDhFnLV!G&;*;o=q|&fQsy1Z_^ulRHLM%^F*faSdFsoVTX?-Xmj3a~prn8G}3b z7Uo-{jyLTYgPR9L<63j3d1i#OITK3on={F3%^>U?7P#k{6W(A?8QB}=Xw4p3YJcW9 zM$4lnbHx3;mV=&K*=u@pjp2tgW0?Mk;cImd9K5^+x_X)Gon&r>kerr?<(X4lWKBiS zn*&t%irXY4@A-q6R+Py}J0^ZE`RKR<9_HO?^mj%z)1DUY(jUQ|KMaubIM!}MU&%jl zg`K09V*2Ksaik+9jflUe*ufd|5`E-(xUjdzK}-p|WXJXO5ci^GT}jM(%&Eu`X9oNU zTjX}n_3Rr*blX`Im}5>gvKJOO(VjDIV@?I?G`EQ57?B!e(XMDyRaOMl?eKUmJ zu{z@9Z`ctAdkrrrBDDfyb#a^Tua$1~4WgS=*ZbeSumCxqasFeVb;iAfTA zWr*{?^F(eAfX%!223^fd#%l)E_?6a5Y1=C)Zw^$XyMshRNHZljrQxtV_M}Z)(SRaDm|kGba?7qE0#ixp(AMA1LjjLHmqA7PGh?79c`%^4AIWzB7dG4`5bUvq+N zug&M15GR0r3PblV{r$1MsS6x&1eZ{2txF%HU8gVLk%(RCwwjAh- zsT3hi*_)WwaG0mweGPkXZbh}-8&h;{jw^d_45-&UmzI6byNqfN;rg97Ce8^fd1@zo z*f2Vyw=$0C@U=G1#*VYfVn$WXJw}wo98&<+%G;A@ZgBE8BT)a(Rl&BCE@0jp8+2wp z;zHD0<%cx$ zp6Ntk@6?#G_kPRFY0-3MRj;z<+}W8)Hg1jx(U*1>`I;klXzk#fu~xk5TEozEC&j9l zmq_QI3Ey#qsgfN=GRRL$<#a@dzdxr~>fS>oW-K`SHU|V(ih*h|4&j}>7fkcd_=p_i z_}g3QQ8=wIq`yY!MV+ZLKaI7JH?{=u&Qp#gO*NLh^R{dq+of(S;OIP(=tPKl^l9vI zyt^WzZ`m7cc`g;;kTSyNSgR#IPl@(3qvlFl>pLyvefzs6_*fjd#5J$w;<;7|=b2IB zCoWmLnI`(on^8+AZS9LaR>tO^LwG2Tos*RZE(`w(F7C8NIPFkTd$by!o6--dsR5>g@(qZ^SPFk*y)5>g_5>5frjbcYHE(lG=^ zij+tQ14g5CjZsRCQ6pvKXi(q1|DWqR|DN-Fp69-;&AVOqe^q_gV%r47f0B>A+*T02 zKRHLXxsr4HP=DUCf)uvzB1|2+xd2$0S0)PCEHRhj0}C&88~^T?e`01N6va2`w2HjKXLCCEa_E`iB4p8(~uQ(TjsYAN=yb>#wAi;U3$ zW7cEvbGx-X<9}>{QaE;*Kgtr7iw&_u+ZMeYZ_v{9vXEG^KF~?>v>>ykfz+9j-Y#ssmXXk62E*kxz>^YyS(vyI*GV+l&P-u`DZ21Sk%6fUp&B`9n9go|;b~SkV z=ON4QlX>F{f08nVYw-BeGBGIRK(1UqOVh%Z%*E+WOB#5vdYAoZQ^Js`Fqh(|$%5sw zN-E(aQZs22qw!eHt1}LMj0)11c`Jc2FK>b%2?^1B_TT#dNoR+iF7v;UO=UQ*C*QnU z=4S`%1^)&P#$Z{o4gS@`t4$9mCv+9ZostlVU@ZwOr-z9NOdw&QY z8TUt)87Bkvn%3CJocgiEcEwa&v@H3dhoz{7iiCL^qU6phuHDGzoB2%#^2UeV!y~GFzNZ&GCQz*GLN^S;nZV>OTx+hq|F)Mp9PyOCWqO_N@_Z6YYaVSDd+B| z9K0acn$m*;`Jv|@z*frQF=L!@0$U64NywhnIPh%$-(dVO*fjQLX?KO++32r~Z>S56 z@w)29uYLxT7e}cwZ=srq=q@rYd;t|mAay5NfhEItB zvyhlb=1r%+%QD!3XA_Mi0ko>*UCyxRD${@({X`JUkQY1um``kXFT)I_p$_}gkY}DN zq&{zVE`moYS)_rfC+c(r&h2ctx~Ft5HfIALQud1YCC4W=HzdmMBq2R0+xsGL&co4F zSpT#1R`upcNmM{G?YQ!mtobm1HSgx5?1yKn3keN-?&Q)Xs?q)n3+>cuTIs#gK__U0 zal3zwRIU7Z`Nk5~Blv(SzQZB(!lHf5SL;rq6phNM(u~rz4OrU+#`#x`na#|KcnaI^ z?|DGBK9@Y5s-~Zlq@4BEHx;a?;G}|VV7|Xm_^v2$YZYB5Uv5XaTkB4@ijY3`kHmK8 z`jhuI{HYoD7ob!U7C zksltj(V1!bdzfnYi-5sojRINWFrbP*M31Zr=Bbj!QBtd&0A>pi&U71Jtp;i38*^za z0n5tjm6}QOLH;4HBztVv?Wid_>V2=;qv8V7IXt&6g#>JP6!m#wD3>U&q~)0+o&Cwy za%V|YkE~4eSyK!f`S!+~^h?1MhAC;GzNv!fEJevX(^6VuPmr( zMeQP1$%XeSlRAUkt*98R90^Zj3-M>YG~EY6cUI$3nofhb`A3m1=8dvZbYTUIU&A!T zaV?H#J**<)j)IR0^LxLVbMSw4PnN@dqaxkGFqZr@l(uLM79Y-|F;eA%#l zusiR|>S(1WWldn4^cTm9WEqeVY*g^RKe`1)uvv_slS(?t0%t7t7u9b`-N~G8jxrdr z!gja`%UxtbvYscjo4R_rey(qpva;&V55A+<)2{w_!OBuhSTJI&16l1~0QT~9?nAc? zP`=I?&GbPZ#i}vK8s6JA6%c%12cG%uz_q*VXV_BG3|C*lGE%#`WX_Vj^zPj4#9*h8 zgf_;T*+pV)*v8KtfEHKZkfpWvGp8ik^OL?5OFbWFpN#J=sGx|o>Ddk$+c~2*;s%|E?E_Ex5LH0Oq@e`mo}}gZ`sX6!O=X!SCVg{$cO1W6 zo=BO7o@(#5IgwxXD`^ETU=_DvNv>Z=!?ypH*wcIrXm?}+-(J~}0e#0ps@fq2e{8~5 zfl#v^CV6%LvHBor>Mfj1hU*{~e8ahxD(`zx%vSO=SehVSl@`=-Jx=Z7m|B|LIgqfH zL=;goXhAB1lt+_7*rjSqTQZQa`FKZ#@Cw(IHj5HbUQ9r#+$-hbJedKZ{nF_oVyS~r z>v=s$Y}pe0${wTmwc+^dRequatfn)zYli&S9R7{?*pJ@`ov!vAA?83BRr)7Hw)CM?*HaXlvNY)Y5EzRD>n2oS9AEJTfL zTTu@?U{>0{PfaY)SB{6F%?D&P7q)FY+Ei(M-b*vHj_P#hK;v|>QNn@YR0tDLGEIw; zQ22~(cZ0ir_$V+o%`N_t*JB`l5>Zk8F9GqyQ}XU#gjR+QN^ms0GT&-shQ0 zxs(_58oNN0QGq_|&WQsWau4tNkFWH~QAoKjx9@t;GVODf|J(pDng!fp`r6g=)ep^| z{P@nN%r5eMb@Nd^2Sc7RrsI(z0RFDyo_~{EbKgcH^`bAWN_5&?!3EjDIbYiNJ1*m| z>(l~~KXdXpK(*2bh3EI4AgUTdJ?-Jrfit@W-G+`51(_WK4i-a+naVWw@0H1{cC7Fy zg*)AZdf~^T*xfRxWKuvW^lT7d|Jc>#S0%h6ayq^tMvgY=T^)tnmh>5opMi%a^iF?8 z8^|9^%?{#Ep`GqJ4?fJzqw!2OI#$|5YZc)~oKs}e6)NbwY>Wfk$5_xN^k6l?+Ty-; z4S{t4eA14}2p)Q1cHJCU2wT>5lrL+HsxO}*0*q39^j{;IAbkBJel2!fqY^%wchi)z z3p4y6#c*bw%TTEf|W^ft^?Ank zd84DjKUO>~gu?nrI(q15!L=O+kzE$Un*)kH`MZd<_n_Ih=>$T1hZ@-2Vf9OOYaQD- zEPv45rn8*vzm7?hM|DvdMm9%%%Kfgn+@CpqTC}SLmGl#o&+1l_7+QmBolaPfIrhgq z=EPt$XRYNA-RR)ekH{MbvQ59>mOZW&=fYXzkkqg>e-vQ6`uIz;iWxIX<6PPH8KG#F+2t{ot`@;8!Y9>|80yoW48u%pF>3KhMyzYnvO{~IR0 zlBlj(;@Ili$5ruqQP8FqW-+_vi~i?@!7YOPVCTA`m63u?E3KP0e+t<5+ztR~tfuPg z*xtav)c>v&H2AKESM7cGZc;a3WrZHP&-W0(`g+$2u~<#ugp* z3l~o+6pQKWDVnNW@6yX(s+wKu3zb{;Uio!$$0vOTVQ*C>7tgw01)rwE-+(>W@uf+H z87wgafD&z`l!uv0D3k4^OE%a2`H7CLpOFTMKmF*OMw%OV&TNa{h0z6MfwU$z3aFoh zX#Ka4jG<@NRf($NC%KJlG)4di z##9~>Coxjzg;N$D;a+r=oxs7hP1|-+MT3y^#)cd30+}CSdpaAy=vtzUV>TETf@v3$ zuxI@J&lYbfF@Cy=%Kmy*z%J>h=N#2mes70v=Q#}1U91esn&!eQIPZDRs>Kdyq*Al{ z4D8%=WzTqGr)$Fv0eu}TF_!vRPS3s_IC9{DhvhAjCs_HiE9=?O0=5&Tx0u0Hp{=_> zd>%kg8m=?H9k)?T@u>{w6%UyJAOAKrye#7m!VGaR#zSia_7z+-cd=ERg5%OL4_$t~ zwr(g3I{1``6gsx#)b(~QyD4la7r0!q2xSP0>yxudTzU`E41hH ztxZGVYKiw=;N7Nl8BHf#o^-Dx{fD+jW{2gxl7^7Y%9AwXJDx?<{K6H#{(g6aK3yJ7 z6{&7&Z3NzgE>@*P+?73RWSN-)w0k&7iu5WvP7D{^n({339R5C&ut08FJ0+6;`s;rN1GYzjB*)HAl0&M9mhOV++jbx_0zAB z6t~X{ddUK6q}PFZIdT}CJW?xQVbPk{z@IA+4O z*a)&IaFg!agMQKO;<}Mp(ULtdX@>8CmhKeEu4R%0ITNnQ*>AUsWUSq5?T5ItOol9^ zGehZkm12TdL#--M1#Hhu5~I7Wnzv z;=1R7snYvun_qq-EFJu5LOvnoQ#|zFZ3CV3=UUFzum2kC)V1Kxw&(qtf}u5I%0RxG zoOB_dayin=fgV=GCqA@HrIxnE^!}Ka9i3ZsO9pumUvh?-7QcP8LHC%LR{Zy@psYc9 zW2BT`%#HK_i8tGuBkdXATCg>=&RL=#a965=IjCbn2hSrq*mTUwUl)wM*7P-;xMrU9 z#m?b{Ba`x*+T|w(=XsUhh|%P5#hqLiO~c-Jx!tIM`1V*!80`+`D`aL$r;=9jGUE(& z*;_*^^Xx}*&uGUvHtTxL*)Kf|o7TEaQ=H6*zo7fxt3`Y-Nl39HVd3sxfWt#)*7=v= zbaO3WA++g--5Ui8=_JK-9%C9nZbU^=dC}GhjE%ihcDyb_(L@uP0;q?1F(skuH=3g!so`-GolwL97?iYJaH#XE&`iL z)1}b~KHBcp4s~xYeQFasjM%j_-o(o+{XM_ulU=D-rAszpLMk+JNua=-(Y)&mCz`%F zrQXF;OnsXQMPmY&Y2j%~)iZCywEni)7YBLp4gmwH)vkd@PXVpFp<|~Q>8*vQFBl&= z5D@P`GP&-F##!!T5ew3TujbH+X2A#M#ViA=N7%~#KuP~g4m{9|?Z}FMA3%Q3 zv<3)0AM6}Bc~hi2sj~khvrR~|+noK8`(5iCsX+;id>Nqd{_s?u@WgV0k4o@+#107} zbD`5vwnYpqObT2TM6HLI;ON+ zS7Dx=1#%ez!`$!@Fx{<>a%VBPSWpmK34}M6=iLL;t%`yvDJ@(41AFrIPBn*nTchmE z;mOh{WfSDOZbZ(c8peT;4IB#|kjr4bF65n6`L@$l*vsmT z4&hbV=$}*+R-PXdRgseQ z0)BZ7*J?r$HwvHMI&jz9LQ`uFy3Vvmv3dGIW7_X>&kirT*7sOgS;Dh|?>eEIArda!VR!r`Iofi2RtQ#kqi@EfzBWM~FXXbn zktj({72N)1*xDHRAVLuNNF;~cwH4dia*m|xAz zcZ&uZe&_{}i&?U^Uf3J-r!;RN{pQI!dG|yrbK!xeI_BendcT@lTp-Uz6D3@#H|kVa zc>XS~ZithwV-b+EDU7I{2aI=u%_ZYEJGHqCy7i**?4A}D``jgYViI0{FPub_h!g|t zbZTuMCtuIL`DV1(;qq$4u<8e__TzA&&>d(&;v-4Px>Q7L)SH65J95we{BA!->d}GI zUf)@RiS`jq`j9_iv6q3-Vaqb{K2OTVoXjShdn#E-=$Q3LKK^Rf z7Jd{&P?&0jKf7GOQo8Mx{R9SG{TS!}W$*kULrSwcY_WSfMCd6VL=oBZ!9LhUGVNah z0Pi%|S&Ph|^tIuy^&(JxMh0dqa5&!Ofrh*dz4N{`D{$h{Vmd-Vm=>Y@;PKTQKaJCV zC0q?jd5jbJ!FSMv0SRjk=nikW*1W!vh9FOt6Wf%Z7q_Uz3Y_QGyL#)(eUo2%IA&Hq z?|yeON6{~7&=B0B`2O%vMi@C}u(SJ^@bp~^5N(>ro?tJ8h%a(Md(VU0JjIM|5PomH z0b#|S;!oQjyjsMKfdw4mJhv5DUil}NXhjM>#pljQ?WZXI3K|a{L(YU-b2EN1!0x+Z zyyxX9+_#dAY3%eK)wYt04FcE$A~`$Q|pQa&$Y zAC07~?|HbH!Z6#{2N<<=hW^5FF-sheuv06>*27z7qc#8{{Zh}LrFXepjikJ@g~Z(- zR@KB;6NZAvjyD*n0QMUE9&B}#rzWV zZqp4odgz};JxZ*a>gapi@vJR?(jNNh9N;^5V1_jh{3f2D%p5$0yJhx9Zx7onLpb3k z`(el0d!XPomccgDw$CYZobeqiIs*9IHLp@B|9TW}$rL2C zx~3@q8DpuD?2AUn=8@an=$muPR&(xv&N8g#-F`)g-K5KAeZXNH_gy`yDwN1S%;Gen z0#p9OD&A%#eQgng#cN0sj;ru-pVZD*7e zxTpWX{H{v!RP@gwwqKRTipgDj}N1PCysO7%_*+AwB^)V83IzI?M(IFW<_a>!|GHqD`8YC5TaPR6K{&lmS7S--2~A0+>{fC~ zN8Ifnb31Qfo{L!svf3l+^a9x6&0;>4ubL1ptFD4H1g@1SufJ(zPTIB@R&}m@KbT*h z7LpY0YDXWY9w?R^1vUX`b6z_Nqn!?9X9LGsb+;n+KjNENSUnJz2&d|nul3XA*rFYP zB6|L!J=CG>Tk?f@e85`hPZtm({tOd}9`tuY<&&4#{a|5;=?$y<<0i z$j>}@g5E84qDyX8%sBg(5Iq@y*56SYQ0T$p^8}s?PDr1z%_j^){NIk z|16|-RjYYF#=-LCCqPHU#Xzc_djNT0t#KFne&gQ*bvysU%UxG}Ky*_aYh(hk5}Yjc z2Sk(_vn=LitI~IZzP;v(mEA3De*vLmIBQ2){4Q!J@iPRhs`A6kbHF~r5|o)zaZUp^ zPapeAoMfbFQJ&`+TLJYui{%4b4o9DgiVD*F%GwIITLzT>0@l|TEHaj*u z;X#t(Q4dV`&K8NCZ=4E7xk0ZNsp0-A5UK$bX$*7EgVP|Vua#)K{RS!Wj-z>s8_pD>JsnZYs)gm`(b#Qp4rj>^jSdfYDrwfIJ|K~iqf*nWe zd-nt96WN{{OLMY3*n!T+d~)@D?ct~1m?u+B(MdSU6`fp}9*O;re^|(~^oKS4efB>s zOB)Z3vielHZi%-lMSq?#tUUD@sEcWfV*9;dOyx_Twu8(XxvDUR zs{=`(+na^SyBBVK@e#kD+XZw8DXfZd!o0-;;SS*=BFaL9;`jm+jti{Jq$})JX<<*P z4Le>%9x|EEjc2>2<$o9v>ox&L{tl<{aNs)_{(e8sVFI@+94M|+Cp+6RYCtN4N*-8wAM!I9t2&(5i&n&O6gZJhOjgGd@L%a>3s z)?Bu75}O#z`UoZ)gF0(l9r0V>+jslejDqx9h+_OaYX&&Wr@H9@$tr~RwNj3dg=IKU zSno?#w$|@sOX@&Tv@$7_((B>w))bCom{udIiXB8mw{Bi-b5nabq2oWaqXten5@{gG z+bK;HY1iDw;o;)iXhmWSUK@`Jmn>d`Z)tiIhFJj#Wxo|QZ?H-1DghNO_02QLpB7|J zkX6-9O&&q zA$2K_OXCZFo9r{@B*4P``|B}0yqb4~JbuNPQ+ki8UDO(0rhxK55CrS^(n%)o!lvX?+$=%o2hnXKf-Vr~{|uRcFUTx)n;`nKlof=`lM-(`o=W{w+%Q4Z)y`@I=x zhfS`O465m9P;2D~K5BXS=X)Ds}SJtiF07@1^bKLA) zlO7hUHm7VEO4G>E{wZ$>yuYw`ne9|s5TXLjh}Ddicu*~MNup#JJl6kd6X13Uux#u_h&`s8dwyi^!KB;2Q`Avr^4@C6>% zk$b~gjM-gomQ)vy$kur!a@Mi~X`EPl&A}-vO^v_~o5W}t5wW^=;MuF`qOb>Un1~sf z%_$;chb)3_y+Va7coC=gL&!@mnk&jd&wLlD@GRzQ3-fw+$~!F;{f`Go$90DS4JmoC zrhNc;m+y<{e;k9_qxy`was~`=I;w63M#|@Ax}7284BJh|5(D}&b*`!68`9ZVf6Ul_ zxEqG_bq1;SaA?><3>Zp2W<2{Z$*#2_Iv+W{+=djd9L&ze*mkfV$DfV$SOUo$Sp1Bi3MSIQrdp2GEN2JU>_go!_4~ zD1=+k8TN3}zC<g)i>+;R-QChzJy>pCJ^3>0I$(uDM1G=) zy_eiSga_OWspS$&?2}7(?ze$q3d)X$75|&yXuYyeeRsfFe(0UI zqwgSD-;{AXX*>_RCxI4;|85dH@*-Z-JC5^2K2Ukryco@s!Z7)rPy2!k`y#@V^Jizt zv%ulec6ZOp6Hd7Ik}h#cOiVDL1Jnn>4!c1l?|%MHwD8o^FK#gP|7Q>_>H&4W0LIcx z6y!;&h?%3m#~%Kr4Ka1p%x})uu(LMz^LewjIA=N9U-wi9 zHkQUB|0E>YvQa#Bs40y`CsXQ8>VgU^OyY*z`n79xx+xPNRYi!5z1SLQ6x<-cJt zP)eO!VRt(Z1-MgwZ*t)KgV%Vl}2Xl^^M3nZ1Ea(L#+AroS6LY&ohmx7;CS!Vk@`!4q?>K&S)S`6u#i8L z6M@i}o7bg2ma6Go$|5@+x7k?;lMuUZ-2|Q|!Fjz8*YsE_p z0w3WdAP#4rxo^X5EMP~#!9nFQgBH{Q{_Dj}9P(3d@;sQs41VF-nj`VUc*+p8NG}+4 zWlwuOAC_!At#}*svrxudaL*o96i9wTFTxF~PCAk|d0}AAAHgRcFxH5=uMcpObn@Ce z`-u8{T}VsyV*%CR={;&Fr9u>dk^wF(W5nh zZf}s~jB;kFD&Boi`M0e4;VyUlZJy+l9}j}qPTBbLy{lJ`A5i?jo0U-8!*B$ox)laH zpAUk)D^dil<43S}Bi1L68iH)}aA6ydJ$1US7-FejxF&_SMFn*Rwa% zc8l;6Uixx9fO7|!g$Bs6He#%q++oz_)*C*3hK281{%^SIuUyuB4$@s7VTv>!`iO_5 z%WB>y04$9lNqybyQ=bQ{0XIMlPlk-i-y;C{1WFqa+d_`~JFlqaTw`?Fa?I0{v-`!* z#plOI_~Bp*SV49pX@`aUNw3xDV<+0X1dbAQqE#5TleqY(RQAar0=pu+_YB{O?|YM{ z0rM94nBPp^H(g-QBenm(C^e#HfM?BEdeE~?a}N(k=O0Mf^A|_#j%}jrA{SD0Dr-uc zD`UfCvThiivAkH=yuoFI^M3zth1kd|Z*jM3m2BAX3-%%hvG+|KE?w@r-6RFm%6f{=nJo13b0@{r+I)z!>|MPd zDv-zeg^6lH)#yKMelKxzPiejG=Ag12>beV_ZupnJ9)DO4MB{X2l*RUzT5q4=M;)8x zDlVrvr=zaMs=CAVdA6kGy{9;@pLwPPdXSp%2s%lf0Lx~*{k~Lg=%)&X`xS&KumsjBLsD6p0^kd z3mH~e)85GEA9m0>ifxZ$UM~{uRvQppY=eA=O!0t(dD8YF$z5^WL=82%VGL3}e;6kk zC73*f0h6cT*t;Kv(fjwvXh2@;G;l?VzwhSU%sn^rDG5eD`3FSNI2HBsyus52aa{RY zU|0Rh5u~%m;_pQJbYVKPcP-g83+Q@LK6H9Z8}XT$zK2*C#ym}trPj5`@|pcNhNbpF z@^I^29$4zmvVy}`?a>`Z2WO+nEcpT5C9FLkxz3t$5Of&}U$Uoq$5)J+wn?$%}`NLZr^-?E7l2C zcr@;?|F(*Sc_AQ`3>e`hp36i=zBYICI}&}_q!z3FdfD-XBmL{ld}OXV0v|7g@l4Kr z@V`prIUkmmy)>NuDgHaC(QJWVi(kSgw{uWn!;0ET!mzPZsv&k9;J6f~2Ot>~|Jp4l z-jj$Oj#Hk^dPW^`1*t8?0ZWuzhNXGJz53daGj$7;qpU)PC-Gp=*{GqPsl#3|93_19 z*LF8CIJR#~&h*9K7k>E2YTmg*KQ2t7?t@S*qh*CP(og2)fa5ov4OY|4*EV4r-t^{I zvO;H5tVH+VzM6)n^3Q(?>c6}Zm#$D9FpFUw*vYpyNSXnj)vk=Uf2CN9Fr-$kw`mp5cd zU7Jt4`4}D%v&&`C<~rie$x=D=VdV_j zO|b*&V~i<2!umxGabnJhcoom&%J&^X7X#e4{*x!sy&B6Yug&12o;5tt4S`u#d6upY z>$n6d;fGMm-71$ok$IHRhv`CUwn*~Jd=gw~hK;&8OcGJa5@ozrB5r zT{bYu?hMX&cFM)O1rsRS>X`aFjTq(f*8kt>N59`cls4TXu)DhUH6p0y=_f|xqT8-X z!KWw|J(6{g#h$LZK!F8#vCLmAbuj$?p~+nT=&;QHn>kRukbZi&crg0ur-vx?Ro>`Z z@8{O5_42<@XvtCnB&@5rvL5;Z$dv7C2(dtr>_8%T(TncO4=}~jfH`|BObAfgT+z3j z5XLV8b*|3ad^?_e)FwgDYi*0WeU=0M1@l_`(s;i1TK|?ksC?*O;?u)$G|BS5nKg5d z7Y&8Ve(Y*EJY+3l_fLL^>M*7B8s05?Q!}}uFdMkSN}>oPd!8@_%JyLyBfR!G*Q8DP z0xYqWzA>kXD9xc$HlvHka&ccFIOA{N{`bU2=FmCZG&~G5*{U40rit}P zgY4}j)@d;N3_aBj8n2JzAcst|nx6&ihitMsbX^t#q!rMDTdeYd1~$aRfST2Aq^WF# zRl|2)ij|Et`PyIc)Qf@r@y889%17}f#^K4g(!X!Ets>c^ z!P&xPA$H2mfEFc#Z#3pE840BAKoki7*x(?2fbEIn8?>zElf#shofO|I_Hdof`EBcJ zJe9bc+@U;MJ7i6r`9?f^rj5LAL9gU0=v(qTCHcj18;z^ry_$IG&Gnuk4Fx;&X>ope z-G({-j@@%;uY!=~;3$8-9iW}eyEeX9W>8@}Nq0~XGj_zuZ=z$FcpVa_-vrVWI5SK% z3dgJ?WwTy)7wP^L|JI4&Fzv9gY!6rJJ3Y?T(%ve>=Lkpy;L^{^$JyJd{LzGK7G_zK zd`Bt&)Vkx2*_81d%h!=fWcUyQ@3dOB`psx!VOS~2{vc0E*2o~?rFKUst)|3q+(p4bazsyJBz8l58 znL$71=`9_2>B{n%`0tJ8P7@!E-6`s#kk~b6FP9j5iI)3Jvkge(fzkthkl-{M-09Tz zPE4Ob@&;DaRjLvK+wbaKkh_%9DkATha$6}V_wu8Mf($;!fXg$rG4EtOQ~g`IHnn|l zt*}toe|xfl560%&gZ(Qek@d|Vp0PT*ilJ*pfopeuEy-lt*C-aJY1oi(ZJLi_oAS~h zz?>IF>11dZqgzdZcJa}+P7*&RIJ6cv6CI&I$<^LAey0UJUSUDZP@{Y^Y=1mtPG#)r zdqfr6n$X6Mq;vvsx;L^FX1(Q)sqjFX<3!R7I5hX|~E0c>&D%=_`ZN%E|#qY+6^ z8E7%f4T}c3@TRZ~4B?85_2g0?mSzr;&lRwns0okVr1^nnaE=!CV`&v_$KK0;_Kwgo zhW@7W#c)=B$Rj$+bj9+EBP@F#x&zww*R`8IVjFy265EJV+JVMIr z_H)A46jMdM$v2D);NZf+dx}>wLdS%mZrX@Aw;!y)HV?yFF`A_-Z&Y&XN3dq*F`wWlHe}x7gJV)u0!HR=Ts{hNZK0_(kGU z$O2fnF_UFlb$|_w3>IMesI=ANhA8$^5VuDuf4*7)BnCKsWBSCIEi^EsA>+Sym#S!uqO7|A0IOrWZXPFiZ&0p;h>Nzah?UVD8j%|t-*t4Zpd>qG+ zk)OrdH)q+)FZshsaAOQ)%(h+FuKT9O^E=#x@O%Mq;9mQuvP)C@m{n#Ns`+WDF_sm> zg?Jd^xzQp67yC^KW^qc^`^_Eq7*;@hC+YMlqb2IqbZR|6kJqo640->n50=(qUPIN* zz_-D;1{O)!>0cH8``zY_o$~b7aTzn)Gvli|P|vZ$B2cN@cP4e?!Wm(OF|I}uRf zU$xDfb(?f}etSLbt(Kt33l9DY2l9xN&bY4GWn%_!QodgwR6aXD9pUNnvcKuGyW42P ziiF`;z{#zL`!d&AvcrJUr z(^HghO1D1$B7|`Ir6cS25m#lH5DjG(YRY${@Yb{k`9nO4vHHTi>Ym$x`ENKLqBXB5 z>?ag)kg>)0kDHy@S@J_QOVrc`_vmY$z4w2?L2Scl@F&POy;8ZEa5T=ZBwU~~j*Uh0 z4BOzROi+o=)LiN-T%~t`y&R6|Zt8TI=mD_*^6Fz6ZLi350AQhBdW&;yL7SUqjA1TN z+Fhyhd(%s!)8eV|B{cU7Wn;sifMq0z*b>vneRHwM%|Z?IG)2(3ax{3b>ul&qSZSbX z3w7>5v|!wMqqo@GO7_slz;Dij*15h2*rfqr&-8GqFI0XH-;qPx&CEx)oJ8x+v-?Yna~GJcu?opcaRb2z;nm)jmsVcUSRC0$f-ujEYEQ zKltP4_kqV-+M_PYn4gVOwB=g+(M5>Y;4Wm{l4YSIc17pYK=VJnAVMyR=^$vclo zP?TXacA&?={I&}DgM*}AR|UfA`zGVh_=0pYzxTr@$nvDu`ISW8n(*H(feQWFV6I?- z+)h(P?qR{D2jtrxLM96{0sv<`k}U3IL8AJ(iL{Q+V_)l=e@-KcXOz4`3OeJ$r@402 zoHyrV@}dcUhT_#86U56}ww_#5YbZ#-3UDm5vCw66W~tH-XKIXMA96;#AXM^=JPr>#E5^DVwEB1Tc&IyaM9TFC66=KNwJ7F7NrY%afQTsMtru0*z3I zO54Hr*|=$Arm!Ce+ZIf`)gz5;gDP@|j!u;FPdC{~fYr zKhic>%@=Vx8Dk)S*v%pO4eWHeC!6u|bZ!_CiS|4PO?4fz)+>7GC-;G-Ye^2nL?2B5 zM-ohQwy^y~5ORN0wmfHu3h&^P8qv`<1NwzK`Wrzfi2e@zeUad2BSj%s2#k6aR0v;y zyg$8QfM-U~B8FYd$G+EXx&F$ZW^=Bg$&n-8-pV&$glXl+D>80lQrx7Syy^$PXWxV; z?*utke2kIWd~Lo&W}+ZjHl_*+zaAgubT|03!Y$9%zsUGWIGM9tD%Ri2rXBAh@$*a- z^z(uWY$gLes;L`*80i_udzL$&b!vT?wr%+2ad7cNbxo?7di7bR3d^!ec4w)|0|C7-$ zA>~{H0iIa)K>*0iUV)$N2&m7VTWqt?2yVW8)ZOrZGxEa!vjH^FD|(vWC_-Ktu}#-1 z3f6oBXHm|~*I%jb?d<@C?mOVvv)qM_4jNfL?B#@6$P-i!m4G)Rht(US1wqxDx$R#9 zs^x}R{mzEQr3sIPnhxb%zJyp>mIto4?nslb-7JbWITO$gisxtlCAQCtr!bw!MBn#8 zuz&OuU+rhY>qXGw5s;LdqWW|6(J=p=aL=+0rL7CkqSa_Suk_L;BHxp|hHPIw5+pRb z?J`-U+n%PCROKJ=E2_5(rPW`3*pRPpU|Hl=ZCT{d}X6mji*;l>v-TriFO^x6hHY9}1w`Ki&Nk-Ro*q+lo^T6XU&Uzb3NscVJ z*#xa|V_*=?Y7Vkl>bIq1ZshnJM%6dxPr<5;sKC$^N1(*UP&-|O@S~5cj<=1>7!$=( zUVm-Pu<@cK#zsnhLNXoG;aT$%#nG=vRp7zTmPgM;*E@Gx<1#~S^NQUK_WtF`D0}F! zCN&&~ZWhhcLugySF>Q%q*=5SyG^Iaxo|hheNLLX^+DUpa%hqGr4VEDvu9c#p?U9Y9 z`ORKpC=R{fDxFn&YUS~PZ|`+efs7S~%gEbV(mIAI`^B{xSXT*B*1 z%k6y~XlUI8jvg4&6QyvY+b(2xSTjqGHu9ifEp27K_h*<4?_DPN0U775 zZ{sKDFS%MM4eXjo>W%a2v))PXA)n3xJA|j~WVaJdcDWq)6eyl|wSlHWNX#V&tPTf^ zQ(fa~ZwuF_{REo+@-LA%0hwL4gAve;qfVUhPjVxfT zJOtCLl>+Cyi`*cxbPU*x9?)+CDi2cOuv$f?8pkDCq?r1$5}Cm>Kuhi8J021*9nJj4 z2EiMye|O{i;B8+HteR&@mK6o%6ZCSbK>=`kELcN$8(`9!a4>tXB|Lqs=hA|K{6bgKyB4>p>gNlgcs< zT2{NvA_h#e(vbgcb5qFvvZ<;JsV4kUd8NE)UVI)1yER9eeEqRz7xIQ)mcdyZdY1S@ z9QKAcWywj0(J+MjxZqkR^GXj_-e5${2P_J*@b^V@i_&|6&j2AdzeUGj`Gfm71hU6w zWLi3$RP}5Y{uDNnc-QOOMH7&Wr1`b{=Ga#{0(Ozuh<|D?HLgF^nXR-ww96i*A3vyc z`vaaktd!*UF0KP@j*jD-NLZ>q8R9_1_~n4g?Qnvv=#-D=4xwQ-}8Cf1@eWw!9eUgk1PzJR5i~%&EQ# z0e|%4KHt|RU7zR98zETs3={WpM*9ds!6VWt0XJS{7xJy?V>*y;JQrK6=oyitM#1Bv zhu3vXb}MW@j@;o=tZ*Hy?^L5eKDP}Ew0HjVwF?yfJI9Y!mZm>WP7CCiH>|~XU_Q!* z6f!WYdgpbZv%<9Ht(l+qU!Y{;6tMZ(Uvu`iP|z}j@0c555FLQ`E!L;7xI#oc%}iM86sMp zcfYSoR{rx#op>)LitT1pw#Cjke96B$cu}DlIG<|i@#bzkqGe6B$Ch74xR!Pk7(1~% zkGJKHQ^L z-q48}QW#~MDB~BDOqHHfq3|Cz@3jDZS$%z6?w@61gcu>9PAW^)SI6f*C{7@)<(zai zFB$hBUs*-<+M^q@b{B?16B|KhlesdJe&j2kS~fMu$GZbUB%RA1FhZO;cLpU= zWTh#jNytevH;cq8r&8|@H6nvtpEI^S`}0V?&zs1a(!%Bp^cbb_mNO=QzIlqL8&}ok zUMA9{8h5AHX4gr9ye@SFgT3%B>qh+_LucXEa1_APq_?e)|)4UEAL0ea<=etsIS=Let_xO0Wxc z;K8X`jxo1A^>G>ChoyM?_P~nRzshFS9p)Gc5wz#5#jM!Ps@YYhDy1T|_+m02E*oqz z(BA+VBl56t36~c|$irW{%^xQ^RZX?YXY9DlQ~h-mepL)O`WeC#pZkzOcUc}he8iDP zZ-*S%uKmC{G>ZvBR4Ar4_}sS3p_%Z*`&Fo&N?F9d&ys~n6lR$PZ|50sI_a42^H4=B z#lqEv4WD*$&ip*E8M2gPh`374zz2iQD}Ab@PL&Ie7P2iuz)0ca=9fOBctH+$CI8pl z8K>#lw4h+dltuaC>9@MB`YyY&<8>z!e{5*Kxr|s;GzT=0XpeOijN3D~fFcJV3!fm9 zfr2{i+FsOgjJ){QP24Rf+j-!w0E}b3Hi{K$W8oqs^5CONYSz$#X`;`w$4M zYyh3!7B1@KCttxWWroNMo|ke^E`;CPjZgg?VjpA}uJ_=8VK%()5k8O~gcdgZ?7X88 zu;`$&0X)0H`v?J(ygiMc_L!b)UJzj{MKRZV9ND^?K0vlKD`f^w4RRzv=zFf@j-J0F zAHH_Rla_gU8CFIgar%pw4@k>%`YcjiIaPT7#TL_baZrszD~I#H%cc9}miG8D1hv)n zEtqqXAP`vhwlwA)xxcH-WiKL)B3@RG)Sir@^RL@%6Oy0Lu~Hf@;rEiI(b{ z73WR3vEDomMdwK2`BeY*J{#0y;T_kVoRhKr*bd7;$@cHLY2uUIY1i6OWwwr0Hwif> z_{CYwqecpYqn623Cz_ z%g>U|y}~;(Y!4|{vDxtb&E+)F^|kD-N=b3`v24iK^O5)(*&^=Y7M#K__VACAo>D)! z*1x?Nfp^1)eLpw%${@0a*5E>~yW5KAuB-L2p=eG=0fuX2k?-stkPSVv1&20apM z#vaLr{Mz2o0;$4UXsp}(DQKT(tkyrHyB`A(NW5KIGo!t{sLbaTH5GMN^GH1}Jlw4) zB~um0e#}kgc$^?xj)ni)iy;u)JSx^yAXfp-_6MlbxV~Y>B0t#Ian@6TGhI-H_hY6W zrCZbUyamPn`;Co!IxYhP=t{AX&u9++IV~=lvH%aS6y*=%YbF@S5V~i7>q@E#1204& zqehsfSWQUz+~+sY^f2C}0JlRk#X=*81dr#li zTQFpp=YT0K@04TvKes*mFQ^sW>1=d4v}za&&~AVv8q^3R>P2MXRCWdc@(&A(4i%#& zdKbchL5#?EDm}l_luj3Z@ETA%8WHE@Ae=<=frpME8~95u@>@4XB3!fJN%-XlPXnbVX2QVt8!z477k+0EeBAc0wO^#+?06wUFYs=xT>L4OEt^)o=xJ5mn2fXy_?zm{s)><$hx-aMqBS>K$xDoU4{mjgv>BS-?oc$ zWaXi~?3+_QT5H;2@sbHq_S}O zqe$i2=igfC{xPX;_a!VlzZy zMBJXrnQhw7ue1m5{5njdKAO>|;g;;F{>Bw8aL6qtC%VCrlz;da;hgiUjTJdz{K8zx zh`tr)Lv+8r?dMl<^J`j3-iv(4EeOMKG_Q7|E;m=4@?)mLob|a&zJhuc&%i6Ls8w)o zJewm@qM%+Ix0rk06e2-$oPM&riYoi5@a%VNMzq&NHYQDWeRHA7toUD&V6$>nPgI`v z7ceas(n*ZjSpB zHvCGo^5&LORKp(MP?*c;pKf-|Xi>0taq7`?4NLo2J08s;Ig`7IET>uC$gUsmqlV+6 z^Ls$dSJzo@7NF31b&F`&h5?A?vD3lNYKY4zO}vHA8q*P%cA?ju>z~xh1HUk9?;YWr z^Zk&&DYrGqO*QT6y@b#6F1N6s?ruEC8x_1!mauEe?3EjdH9&^lcu!Ph>$oFZ5rb-+ zQSr?y7*iYF{3RD)Na9U<4)2YqY&E;BVQ?u1yfwLwW3@bD4+lZkUk|X|s$C#%*ZNBq z;_8{Nj{CWU!A?6DHUB#@Ks8r5tliI0-LP4kbG-5xz1D|1fYg3i-PZfMZ(D{xF7Cwj;~#)e-(3Uf6=B4N)gY}(>|7*ZOn1)`GdE5c?#%y zcnwa#Ys587eKtbRU07Lk_u`2e?*KZj*O6gl15>YYimu@blkgf*5>fii?*7tKyyK!4 zlHHzH#;np)x75Vkd1y&xh>)CWQLR7Q&+2M+7QKsA+i)Xs3syNxx=hx7(OCdVU7sL}l&cW0cm<_XVSJ-3SQPs>c$o|j{zudT;z#Zi#)-?QEf zo}vxj0QXRr8Mlty2OPH{;=+UmIU>Q*0;p8t8h%&au2FwU*V7a_{_Y4Jv~Z zgf40vDgSkj2kw<4wckaaj%Pi3YzMxNmWg^mFwwO1uAKV7yz!}TWc;lY!|My|m!qlq z=76iZUh0iG%*`+fza(5#@GNIRpDIH{RsDv?=ABDE`^p;>W$fla_GJ$LdMJ+N*hk&! z*wFErZ&a_Qgb_5})Lq@*O zSb}Z0{N*4jzZC<^)K%RWd5oXI=FW<-fmuPh-^{*!J@+fBz5tq7vNwsWiWM@Gb4o>XV3_ z206&=uJ!d9x4oTerD%=U^Rzm_g{a|LhXr^7OPa*`03{)KYKIcMRD?-%Dx{X5%#Evh zOIn1dW`s@Ue^w1Pd7qZTCbDO>^1mm&UIlFdcegC3X-qvy~Fq8T&=|m#C#eZ45z6j@o}HUJvu$`Jq{_s=B=9|2;j;Z(Mxqk#eATH27yRmJI1V z!?qQSX@DHFAXWFa)l$_o^BoT%qOFl!Z*n%wwiqHdPW}s6w?T zW=Dl0Vb&w#l}5!*uxtcFMEen=|_f;zzZa0g%BR_wsdpUcVZA$s@-e`WNJUYmpA0WPF)tJoa==ns4HpmNvH(e55dp>VJ~@Yb60B=jmCq$#oJ) zM3&tFP))jvC*<;p+N|lGM^TU?pBkxzLi_xm^Q|Br7v3hZu?=rqy$E@1{Q6__`tx-% zt)WoWR3vZe!%OP|E5#cF*-WC9cPPzWcl$OrqpW}QD#8yS6^&nUG{uNRz+1D+mttGM7hM-1DE_VnW`D(i)A-0v?{D!0PDnnINOl@LT+y z@(v8ASyVgbc_3LKAw6^(B)NnAvw^?3#VFhQ;5NH~G6W_|WEPFW$Az0C9=ognd*iI~ z$>mZb)>xI(lYSzLtAb=t5wqEyaW2P4u!Qlm`vEdji!`DEc&3pyQ{9nVcQ4oOzbPtj zR)cL~<$NN#@hv)m`V_t$)Wmx3Uj6gId9IJpUB9a*oq0=q@I}F>#@}tC-?r!6EmbP$ z+my8zNclq~H=~=#regik{#d3!27a=v&Z_mSLcT`GHKzekyy^OBJdc2Ve>zI!8_%_4$E_wluIFUPXn ze>-_5#3*ZD5EuDhs0wLqKbbyhXH8pQux2q)Eq`CEAQ$yppKu^rBr&5PD)Ms-X?DK$fteT%5ByPM0Rw2FF0Gx32&6D(2A+B)7 zM@Xs`is2*^Q``H32g9d_cpF-!H^^H&)?*9Ym4xjsww@F^43B>dk)-qnHw3%~(6qwz z*69lCos&j1>24DCaYaiF)u{*1^|`blLSu`vq7DCY8hr%m8VOHC@XttMX?7;+72mbR zw@~qqf{nYezzK7A(onzDzHCa%|G*?$cl`jmkG~Y66PG9=Oo5TPa5hb2?|FEWW95T+ ztj^{Z-B7-(gjHTk^pAeNao0e)t>$c>i(#_zfmg*&e=GGXceH790OP?JbHMUTTXnbK z8&+6pu;fc~bh89LXCpJ(@*fRVHSk7b1ABDv@oa&0Xisun+j{k{*d+!BkFTw28_5*? z`wxfmogd$Oxz8*C=aRj!B`HCtMGK~5$0Y0KR(Um@T;+D>%ENZqTUTN0-W5{4(B~tB zc>g}v%7&=H6$AT4g~XeCN|^#Z`5k)@ar%MO8CJan4jaFoffiXd{o`!=f1K-2du_Yo zlq6gJwm>-h*}1NKC6OJ~^)3`@e~hoQ{R#yvyNG@iA})SFR6ZG;wDootaQYoD^M>wx zbZgPRXrutgZ2S*|s+IJ#{w9Y>CX$u=Afh+YQShdLY%W#Ta&~0v8?E2{L-*c#9DHwF z*Ko};;6dQ)QkCDF;Ab_T^NV_qHQwrW<$S>r=(&3i@Xs*KE?^2p#ts9R+tpENKi!R) zrZYQ*7UBsn0wiwce!sOB$i7tURwdMjPSbzOhth-ZR3rckIybD<%OTMNRc($|UOCD+ zk9K17nB^x3iR!ET+IL%-`zS$n+Go$T@?wq&1t;tCV#(p%U-hC=b|mEiUmabnRDfB? zV9k`L;rM2HJzmqADjptbgp|sp!*i18!x)&RTKg2v$V<%vqAs~=r(@dL{1r~4IkBwF z*TQ)NZu=R`x^fy2d@JnpTR;5wYlKA|E?zy&9iCxijUw8us(+L!IXcZ{*>UwyZEnQ% zF5*rC?!gMVDNcl%h85G2N7RkoQisqTq3mQv($F{rHKGALCD(7Ui_!fg#D5wHGu zH}1_c2`Eq-s$&Lo_7?cA|;NS1qy~nB# z2C)PyGISTPm(Y1GSXldcXr1ulo~v6DB%vGhQ{S~`t?qD4yBht))|YIaSbj#qjkqbI zotH-+7|w_#?a;John(hSvPfKTEYhk;fQtOzIf1-EQfpbCzhOj4B&A=$o9Q}8JLAT5&D zMMBYd8B(Tsjt)AUK}5LDVVz>AA5x9r^#Ukp3&*rdzMXJuK$`$_MV43h8jwzJQ zXm^eSivMTyt3uXMzmsR%H*|yk;Jev%B3iWH@;29cfOt>5ECr%TAJiXK4%?L3c-Xht zXcj7R{k%?;*LNQD$y4WWOFBbk(N-hGc1ch2F~@WZL5N_9;B-!Sutj;CtZ~vhNZlCU zK&L(T*L4hmYX8myQk(0VgV;hV8@I8VzC&3A+n_$9W%*?#>;7gRZ+!Zq?DX^=H?5T< z1IH1EMvFJ;y{Lpl&I)!TPm_1s$u41Pr{&EbXX|EGuQk#;J4508`VS9&mX*}y#&@;)mUA311ya*_Snb`eqK6MDtGvIh@lya;l!rC`@ z`T9|J&YxNc=iw;uiBM!kMq|lfXKq)Nnn!<}a)2ZS--3~eRQ=YF#yiKeVs3&_LnO`3 z$z}xGb8PE*75=TtJXtln{m9#&P77V1`3`I?Ms_w{%dPG+9x@;D_8$Qhw4Mlhn6mF?+!H7Nq5O>zmEeD{&$vd&oz5K0 z5xgoP1JkDq%5~I|Fm_I=*+}i;L{!5Wb82&!CpPqy<(*1~e=8I8|JcL_`tI7#ww>tX z&A0jg6L0;B?d`NAbu`09&l=g8vBdnKS3&rj<5-|LiJq&{-NUF%HjR3I2+=5K%v1(Q zyBEkAw23g;BJ*kn&Rc=b#3EhK@20-&w~F5ibALZ2^gu=6SORS3%8u-$_O#M|F?Sj z!9Nsv3odD$)hw?tz-96G$w<*QhKlpzJ>}Dpm3@+l6!N z$H$jh2n_>P8~OySYzPPb$w`oCV%!svdt2O0qkhKe;n@KnCOAu(RgJvE z<))$MHdG~cJrxAC5)~pNhZYbzH4Zd=HxXh8)|BYl2{msFok_qb zMEC9qS~OMXHdmcJgV?;nI`s*AkUvy%ez_KBPBh@Z3N=Z%WX9>55DCgqMyIjr5>CPb z-~)x>@J%7W@=(8&SB@h5&hW>dTW;4VR?>XsN^P(}ZLtLCr0j5*Jq#iNo~u#Q^htHT zJk1S<`t2rX%ZeCU%)jTB-nm_(BH-3Bg0NE@rU|bT;r8U#i;`gQ8_S<)?-@oINRNzT zF>LH$TdLkLd>p!`o6=g0oKEZt5nZ#6NfGfiwnEPdJpNo( zQy*vmx-Jkp%6_GK3f-|)lngXtD#OTKxPew!-!i_gss_*cTJZl49#;I^0LeJCHH8H+ zR^(v`IK8-W8UNAlQdS3_&Dr?d9y?p(TqQ!MS(^8}_ScGXPGyL5i2HAyRic~UgzcFK z-f)1a{6mSQDE0~eqd{O*vtPX^(BsBv?TM8U3L#`aa=pB@-l}U3P%S#FILWEq3n!7*g7GdNajexL z3tRpw-Sk-~jP8ggWJW>X%XFdk^Lr)9Q#>W#Mokj6N20xE_?bARyQRJ@#(QtcY0g(@mFewgY{5> zfH+vV^?Se%#G*xkV^T20%pM}@iz{wfgG_$KLsRO*R+*XUhufy6gJ@=e`4k` zp3i~|bmExDy!=_l&a1J=Wd_#P+v0?1Zzjh77T7ub$vNHWVQu=;Zy!5U z-50!R4)k`BQ0;Fs+Z?+?zAa$L6MtWR_Lq%vk;CJshkOonYs4}|Uk3gR+2|uTE_D?z zCyGJs3vQdk@n!HH|9m~`y&&B(WxsX16e;@1mZl8W>9`J(Tjb}<%&uRPi=gj89;!)- z9~cI6xrkwvtt%`nN}Wc6p57au67S*KyWw@xG`ULg&I^3h`#B z-jJ8)JkTeP`c|e;dSUt(9Pj-@JzfwL)}Im?FfP8kMaLAN*HDh>92BXNEe$Rtb1N4? z5|}c1yV|Bqw%#im0{+PrR1(=6;odjK`||vACe7WyhE_^kah_OtWL+jYM}@xUF&peg zYf6VQCO67L1m2*Psa2fD52SkcdTEsJ`6548)KDud>%G6>Y|J#SWcFa%(w@WKFuAmL z#;tw8odyc2SysxpI*Cu?_2^|<$Cgp{kXgsp1ZvSJz?ft}Xqy$9`cyHrq z#R4E=*LT0G4ku?$M;rEd?ZqIfzPo3%+1-2eSGP3o#t+ot3(3ca&G*=-e_67`sSQ6{ zY3$0H(;MGVqr2CsB<(ZASJoV;@1W123`9ySDNZPaZ~{yHE3%eV!YO@q%>(yB??oKs zRd@ciykOwWOSQn(N6tPZurS><&_h6m{UJ_k3x zb>SQq@bjiYqPt@6Yd7!)wgd&${f|3MD|U01 zT*Gd|=I1%Lu938}Nq8=g2cCYF^=3P}^WkE!QH~=~iL~T(had9A_NlVKKuxPOHapHB z4w%H}CFU*-mxU$cpqr$o?;1)(!=VKP{R<3B^pP%4*_^w!;hem{gALxHH-1`yJ1sh! z{V|v-8DZ}WUco}AfpCSeit!YVUNu+tb7e)!4KjC$)7GLOD_9)1DqZ5{XhVPtCEWXy zj(Pfa^nE1GtlV+YKrf`^!L(YZ&6@tW4z&Dyz2+p$oMe!wHj2$`ImXW2-WGftQ8 zO(;Q1H1V9Gz;dZDTEsE)9@?&0`}V+4t&RS9d>(@NK2!z~t( z@@_oGpej@mC+%p{Gxx%WfB$}~tbh7>4&s|9SZJ({awgz)oDaN_yVRDZ`_JN$q94Qc z%vNfvVapn7c!k6{7d*_F=pGMqx|^mrb8XJ3z>^;~e$Hp+Nexv6>DQ<#KnLrY2TRu+2KuF@|fJGx2JdPz*IF!G<8sCb|`h|$hS5uK90 z2`KnFm0M#OU>K6!iX-;vGQ|X#yjrbGy`!V6{GZ+Kd3qxyvKdtL3cNAi#<|g8Xxo(- z64Cf0Q~Mw+8@20Amy#y+WpHz`Of!_OJzs`eV0Dy+6<#sl`()xa+1emiIZ5|It(W_l zksvBVPl{}qxIN8z-k@A@J+(g5!$2>b3t^IlLW$&z@~Ip8OZpYwoj%ayy^L|#FP#L; zJl{2U_YzI0yJWrf;VmDPu2>73g0RG`**U(LR%$%U|}=D*I8zb=({6p8jTCH>me) zu5`OKjp>oMXJb6Jz@a_q&>AdR*PZJCHeeK74r06=mp*+bn1Bl(K=WA4 z5E{z_2LH@H=9ClXnBaK}Qn839@%z=Z<)C>>IlJ@n`Mv-~dm};eR)K)?2zJ7kEF9 z`$X=vDotnqvveF1D1T3Gi7sBbjd%6?ABrVNF7lT`n4^b0i*{{q?oqiWT{0U z7VZz_D)`IQrol^AnnQ|=bZd?Ov+OWemT(vH9?CLXcBUx{eN-7+!UU9DM=AoKd%vR9 zMAkfG6ofM23&0OnqCD`-_EJo z@an;D?*!95JLiOFy+$d|_(AOmm}TnMHqC{eV7s=a|DE)4@qY;Gj?I@b{o`R(!-%*k z){}f|cQUi{;3qrNDFqVH*-WiJgl^RETNOq&%85IfU}*moKg%&CN`2MCF!Gkh@dGa?%iGDkobG)gdMMVW+RLD+SiJM~w-M5P(nB#xZ8)K_Okd4sxwCnRtmQPbbpPc;my7m|T&l2*EpUM5{QKxX+>1W@ z#~c2|8-sl9Lbiw3A3XG77Efw)16Tc7$`e}zcb`|)M}N8GTRC|f35%E8 z;DuQheBdwlE0;ftnLu9l+gG5X<;|e|pt5V$R+oW->uVc8HieS`)ULmwZ*OYJ5i$E1 zD=|!o1_Pi7JetX+(JsNetQqpD%Lc#%jbhqoVUx4W z6=b(BuS@%m6W3x)8?rMj-|&1woMC5@9C-Z&W`P1he86SJGX#5^78>UOU9 zm8az2;$`#b-UB-Ob$-LT2vz?NzvoQfi&v706iu~%8j_v{4BtJF$xQw^=h*9;kdxO$ z?a-5}iEyl#>`p%w*j-$~K9&s0XzNAV@Thk@Fx0DTloz;~GyP_C{yYF0TF^X$C2}7p zdq;H0d7gJFuu?;FxJ-l-?SErW|NDU&=aXmPm~JW&8Hz894BTJJP|P^7LsIG6(VPAa zel~ysh-~hI;@f?U*Gk(=)2+w=1COea>K0}O^=$Xfz;f43*)Ol!lbY}RXDE)9(TM9d z21Te8h5nPOADrSXSaM*z87n_+Hp3meZ=cXQ)#MmER6~#|kfEYzYy0=u8<-&5C3ti{{WsUwf zPHaI>TXq@G&#}u7$Q-8f%8arZ_dZO|kPv(Diq|oXEYwR$1v4QV-N?NE zUHu+NxgpNzbjdU^2r?Q-he_7`c^^{tAj6fL~ian z?d{Q1PDfxtL-3|?|ChGDRiT50SViMK9FcKVq zOO%hO%2D8}LlaUPvQ>H1tbd81Tkf!Xlq7TFz<{L^XtcG#htd9RQ5NY7wrES~Qkr8I z`>9NyRVyR19BAad;qln6B)gZDr?v^2wDlCB(>>g>Zrn{7a!W)kVeitvU#TlV&XLPX zefI1=H4?Z3Vz=j|Mir{1!P)Mlf+%+r)s5T?%O{Y{`2hgZzSnxW&Md3Lg6oAR68lxD zxP8t1bC8IKbESMq@*F#+Sb*AjR*pr>D%-IVBC7uVW3Rc{9j0Z(sNYAmBmHGk>05D6 z=Y+IkZ7>~4h#%iF)zzTcaN&6}Xpwzd@M&>wP|ZHQhA@>!UfBYQgKn?ee#3mp}uwaPgXj z+OD7ONZ^MD37eA`>dqxns@*~|Axm#{p3;flPHP-?x`~q(k{!UKveT3Z1(bWJI`cdE9VJ+q= zw4$_1@npi6KK9I?(Y21UYC$lvdWc)+LERZb) zgN6zDwHtJ#ROw{;Bn~e0aBr(UFZ$Q0Wf^-w?`nF=wE0lHrpCkFDMDL@iY7-_vC>-5 zPHn(DmCiByY=a}e%64*1y>9>f7QZ|)D;MMM(3Lk%w1c@WH3TrcwI1k}z9q4w26lTz zSECMugv{QD{25(L8FCvfjgsDYx0S7o3N{HUZh>fP(s6YHsY7>@<(C&}2`vgq-AhW$ z(?Qv3^biK$2B2J&jZGQW&dItGb8@sc@`4QjxmKTDka*R&A(bS1x);I3Eg^tFmHsPh;4G>T;# zoUz=(i6zM{HB)Q-jz{zI8Z(BDySL@-Vf6H)nE@VC3xYz3Va1KRQ7W^b zE==I)vv%bfbCS(xdZ#MV50!A;1Kr~S=T+xwEG8LGR{-%{8e+jfG$wo21LA` z&cf|X@EwC6d30Y61E{|+c*^N3Kb5i@7+i-1!8cId01PENw zH`56v-NoFMe%fngAmhW}RLKKwHzC4n`Hgquv*lg9ZOL%EivD^EafX;{`rFO0qCHIm z8p1*A3^B24`L0dfjv8Y3Nsy{%DyBZ;RIX4pxn0t5`d|W8`c8n+w?_`5Kgr7yJTVeiOM_23@o*73cdCp?tcd4X14w=v<4|Hqc;W0eTkf(xbmZZ5eENm>EOL6iTq zYU8}nn!^y-!v<`$E+}7d+1_*YC}ZZv4|Mo9ipQruXWD-|iG#~m{CbF=xE~hREAGvA z2vo%19K@Q5lSdJjZq8cSD*g=q@$sdwKG$48{ousBW}Zl*(Gl>?f0tH@C;m4+|I6p! zLa;Nh9oWGg*4#6)8u#7Gd~n3Has) z*`}E{4DYwm{N)wp36cVfNEvFl)$of(Jut9BBJv(M;$=+1shsiomJ&toWH{vIi7b+* z{GI^R3#WR=70QvY29~}SE#iA4Cu-puxh|CsdX`|hs@L!HX9Py+^;Eqi5^7l|=saUq zv}7yhS5cWet|+prbOOzgRK3fZIkSm12&5`mnt^jnea-G_=*I{pekb6~5;-hn3;_jo z>$*pb-ZBV}omuI$&HXzTA`T4J>CDuU_=T-CebBAI9SeNASp)rfi>9%4KmJ(QUQ-=5?SkdcG zt@7Q@vjjz@@}7AnX}V!c0*`?6!C|(U?&R#{pnZi<(Y7CL$6^UFZ%9( z<)4Z5(^kKEOuOAYVt*-9OJ5mm+jvw3RoCJA5A@+@07I$^MLO7Yi~pT}Bd@pe0<=yo z)UTW?>6cq^KV(xG-tcufxhG1IYnYJ}@#p`f{Tr@wn>XDbpQADewQjV6iHcQ<15f^k zJ>;P2gp7C-zxOE|u&13mKy&dZHmPm;#;st+9~GQY$%I+WnJ1EU)%RX{o#YLb+Qa5o zRYdN#Cz$q9Y%k*gc*}a;^&F`4z*s(RVV+aahC*yx0#dBbXqG5XkN4Ob2a%Tyj`tmQ zB;r%LirshtPAXc37zL5M{lv8V;Kxt%W#pz;xY1(%iLf6^#r-5hz=~_R>$HqW%Rwv% z7gp&UQ%hL*NlS$VM|(#q+{zkb$6NqR2G+2rvj?<!1u(o<8` z4Z#-M^L|FHD|Ke8D>O{_K!%>3C*k*Y!_NlEG{Nv^uTq;==G1kG6YS*V0EYKxEvfx~ zY2O>ecGf=Clw3(w)aem1ElDs?t;*&Uofnvi$A^TQ}eD9E||v}s10bLD3eTVUh5FO3_4+c_x@p-#I$(@vT~k^EDECj4nNN4z})Wm9jV zBN-^n>lTLP!@PRd1VZP-r>sw;;i>&lPCZ4>Mk zSTE6-c#NFFWmZ8|NV&vn8@O$P9_k##Zk6SJ2VOIStWzuQf64+m+O%;kquem1=PmKM-U$QEo7Uh_T!+zOtWGcZyW z5*km3YxFRRNlQdQyX!jfr{;wFc1XXBj~}x1^x>xtBXs2CaO5$=kX}U%u`9$x$ zLC~3hBDsFqa3Srlk7sf<%M2Dj8vo9l;b9w7l*_ci(0ZuE&gV?CM8fT6PhV~CR0S$d zGb{L#(+%@Opqa=-oGHSw3biETA2&AdZ7Bx*94kG6Kb@n^cH2e$ZHn^P&V!fBK2F$a zq|S2B-+JP@W7VM?bv6N7IT798{DxhgGoZZ`w?~RaG@V?mFCOnXIz6b*)Q%%A>It}w ziv4Pf);Og^wH_vbe_1c)-4nHlDpB2deJ&x_LW#7xUX7~-;8$Pak8!-~*AKCfYiBuB z=8)krqFnUO3J9JGO*l*Y4%gn#w`i3Y;%}2)eqd`pbi)f-cOnOrJm)Rl1)B)f)`&d4 z$VvO+xPNm+DYH;_ki;epy&s918SVy=dBDyIv1zxiUBuCTM0i4DGeqz7ahJB+N40)# zJMnc)C}=0WCP*_|KCfmgu`!n*$VwiH|QT`o7PaZs;IC z!a!H)$*D8c7o2cv#Le?X3|P`@5M%DQ_j;8J7^B0UwyRQAT`a>YH_ISc=`ea9qf}6C zGrKJ%#P}*HXZdIr#nPuVytIsC`}!&;tQT zN^#iEA-h_Dy_avG-n-%&N!FiXDGT(?0~#^fqLFtsOQjZ= zD4dbemp+ue(!eh+*2wB(=A8fjiF;4u0FYt|%vV=+7kAyMmbd$dWZ{Gqo42$@i5v6P zd{Tq7eqfI(eq*H=FW8rojR`my=V9@IM8b~K*%St6yPkxabaczh&&|w+>xqiji~}62 z*USmRGDZAf^B`7NB-(Gmt1jnU+FARS@d8eC9W|W(x1G_=GYZ24sWll|2WZ-w ziMkj!FR1lM68{&A9)cT9%^Y;iUrKotZ(^E69c;57`BE_I ze*mJx>vSY$MbF5ly5fjuAVnUSIZC320^ss%Jz8rz;wV1T!KxAXRH+6Dm8e9lICJ_^L7N}1m z`&Q&`)kL!!#tf}LL^giNno$`C32Fi9fA6cXOfP^g-w9IxjrzFWGCN0ABO?L8uxk9; zBA!pU2sK6|*S5@qfUjM!T6>lnJvlWJd)*hTF+<(u1>i zwko~54bNHw?m8@%afDReD{b14(zVfS4Zfw4T+Nt3PXAJFAIC)JHkv!foq1))U0@MX zWE~HTGW%)k-T|-d0Q9~a;+50#1C6tY8zC=&W>S~PM@p7rM5zG6L&&GgUiR zJ84OHzkO6%wa(8KJ#V;z;dN^Tn0HJ=@qd383oIUcI-jQsJSSzLQX~syV*s^gzpc<@ zXyRD0bXxjWUKh6?J%79+F1uqMBnx>K)2g;IOvvf5b5D}3W1`M$pHois+ixguT@LGF z-}e{tkC(J?{K~#EJxldu)_eB4>;rrt9tcnU!9Da(QBYthr%M7MB+)N_{9u@v zs)y7p^fa1}YM3q_;cQG6dn`TxG>hs?P)~T9#519`oc-5`$`+n{gfX5`Lc?wRs@|V$0);90>{(p+j!msJ}ZNonW zL6i^?1O_NA(hNq!gS3=_(p@9Q=nexZ=@1y(XhD=7-8o`G!^Y?oDG3QdefRzi+vob; z_kEq`afns@!nbcpeP`vR2u@C@YA(|liU(!~q~MaWfF63fn4d0{x}A+wlfzaN@{Z21 z-6pD(m-&DzNj%L@d0zA0WOyM20_&HWeKl;p8WmA}(cfCEWq=@^xQty}A`{~Rw3RnH zz<+^;3%T90AMu82P)O1YR_W#_tm&trx!z)+wtL>alAmid?LPv%eN_!#C*-;6tlF&u zgpFR~6=ySiNC{wWyxJE9*)#n_HB6LNM|57XpNAD2+mjTBWaii;TfM8orwnxobyQhX zOB9o;l`E+VP3Ub_;fxTd#-6hAnz8PVt=Qn(xKGWFk!)A-AaA)!K-|Ktm9UOj&0Bn% zlkIQ(yLG8CivI8;oydZACbzPKZSg zj*lpr-H&`;avW^P9rI}D<05^iL=bInL6g=`cN*dQk#r6*D(c142`9+Ns?FItDa+kF z3SN1Ei&oWbXX0YJNT7@8d-?K|*%T8hd#;JpCjtuv%tPqE`KT7jA@U8_pJOR$_oHm( zVydj`+DYOrg)|rsDEZjT9t4n-y@fz8cs0|~*9lo zxlfY2DM!q8mv^fESqK0<>9=8H_e+&YZ3k8om(?l2m5UkO)P)OHn2!=({~a>Uk%i>F zJx`&RA3L)Dvw9?X$45)|L@JZY!%A58P381w*$hfB(n%z&x-+Xj^R1B$BSgkEWU)KA z#gK-)w%cR=Qn&5sln8fs#r7_@fR^S9ghzb5UTi8v2kg`B>;`KLe6OF=`Ktrp08(z& zw}DfB0x(O0v@*weH?u6vsb7TI32V5aL&`u8=hxt^!EOwiGjhi7Ayx^3;%L@2RI?-6 zj%m%_n=npxEU)S(fEBc9yY5pw{L4L(?uWh@(Cq&X6v8O!}9#LvWAtY zp3EWi^qz>Dx7C?y3?V00#fK~^?zz_gM&8{ONWH!!0M(o*r?D*prAX7Z#bhUrC#5fQ z+KWBN%o@oEXXI#m+<_s+BF;r$ccz{<;NPQlS&x02###`5+3qFA#}(H1FyppbQbp&R zfkg}#_5JI3|3iLA#a2D==`M$mq&S%!0un%<6pRpkz%TlBxvl}g| zH9}H_N}HHojau>Y3FF;=$i|7iVo$JDtWF|#QbsTEfSTZ{cd6|x90NMKx541YyDb!k z)@)YnZd&r2xp6#wfN!b}Ue=X}4iYr%&;_6VtajRVhZXEnlmRmZ_0iu@@GZqIbZAc- z3}3PZF?^bnG`YEBq%Pp(%zp_wN6eB)2EHPjKiFdJekZhKNUv z+1KCiP<#%eJc>tA*ykOk5r3c38=>L;8~ zD-1ze*~(HD2;s#(P@5~Jhm&Wp@V9;&i$zK~gcJRjxPYONgRZh0d#G&NG23Wwud1R# zq%S@@rvYsUze}PyJSH@Y#DxPt;=}b*}h{DJd{OtOp)j$+o#h^^H>$z2; z6rWemYChODP_M$xU;O%cu5d8-c5*b3C2X4@d=%z&nWpkA?T`Zl2&8oT%GUICW|7x7 zke}^KfwsLiXLvLO+^_@8mL@Im)`AZj!_n>-T=|&D1$r^4vr8cvb2+YDa&fB16e@A6vMHN| zl@$LiMA&P!p>=5d0Gk;LrXGZ~Y4xG!6I*2tcM{vc^B7$UQwNb4QZ^6m&&8<(7q3?r zwHbN(L<-kn!?ZEwa~3FEy&x!tW;efl_!#MFC$rc4r5jXuXy^Ys$g8InKAU@iOc(bR zi5ON{Q#zV&j~3RP`J;%Io=YKjLJnS6)C#oxCAQ1>(>7G7_$kdgSaURb3Cq|yQ1}x(c`-z^@U(Z2uVDw?v zRYAYG5B}|^JnHXr`B%GMTLX>tvCweK1Zw-c>GM(8q$cU}@3zSP2>Y5FXvFq~Z$39| z1cL-x=DoH0LJ38rh~$i{mb3{yF~yH-1AP#2kf(`WmB*x5eXx${MNUuC1f@Jp|-GwNc$647am^#r7s@ z&p6x(yx~vz?iERCIiD_ND!U}FcD3H5U1AruaowkV*YuEYkMOOTsYl+ zkn>nNHui*s`X>)Io>@v*4cRZ9&sD!~$fgDs(YQ^zH}|wJ&&t@hbU%?Xwrdo_?HN3X z+?QhcU!w#==U-)0(5S7Dw0}AFFDoc4l%e%efcQHYNZ@Uht!G6<5*-=9|8&UODaK1b zZKjdJc`qL&P+207wzx*K8ma;STD?LQ9C zxXE&YT_7Z+M4DQMfc5l@zzLDc#&;D?jMel@enuSN*5GK?Evw881-xdLa+ApK=OwY; z)uHjVvK&k7p44GAGp$_@tqHcPN!(D}M@}C7Q_HKH;z`!ooH_J%^z8!SMhlx6=(OO1 z%QF?%Y>#=M${PW%Zn>d5y@ZLB)ls+Zd&4uGu6E#Z78<=*% zTpi-og#Z47m2~x%KtjSGG>L5uhE~we-xUfRYlO9yq#X_`p1c_g%wXHH?PWiC??wBk z1X&JRwVKkE5)*2!n~iWwrr`JCPsjxEJ|4B>EmcbPtxYSI{{9FU<9cf*32;B;C)+oe zgXZcwV^f1OjlUJdV4$Eb+ni`^@?1J4qKT}39nHNIi!!R{d>DJWGy->hQWgkfd8h?p zy-az6r}@IU)qrN#^A9Rp?NeF*er4M`!0r9GH_a?|Ja1jceP45kQ7go2ME4z(VP>YA z%EOcW>LpNoVZAvnz7GzM@!3}+z_f4n-z|v>to(2$H&i*MOMDqjt4!$5hE5tdL^&E2 z4ufQW>>^BJ6};I{jm^g{1>EY6241aFC)0WiyXkIt-Q|1Dtn2YiWpB2=;GbvGxIzPe zvYQfu7d+J*MI=k4>vMaJ66S=pD_Uv=+`zirdtsH_M44w#M(@4@JBrsX0%ALk@ECv; zzgJeHlkpC5hrxdgJKZ?>YS!rTyCN06RDYOj3Ve^OqUn-!LZ4bv)OynOd~S0J@kmUZ zkujXfL(Wxi<&_jWjU~6s@m!xS>Gk{ZRpm0F!#2k*%ez?+_E^c#05RlM-4F}bipffd z)}}EHTnml2q6!7So~paKv~SVB^LYQ(FBa2r8(IsO;O(Po*79jTP7@sO=18qvU*Um1 zO3YWBCYfWxl zR~JukPc}IF>d~bB#;0`h#P+kOy)hGO&)C;vFsZPEdC%|mcbup^b?HM6PbW~|y*lu0)5yzC|Ghjc| z8}O~vO+4l-ds|?9M9(J>((}I2-;D?$8heh(6v*DETC$B!b#SH2OOO3}Sa31hPJOLy zo^Tf|f=g_ll{-XQ5JYJ?3tCwAOmK`0L9w)3?9KoAm`m2Y5npVtdvLr9{d3cG!(ptsVJPd-@2y2=Ya9LfMP4^?7&^afWp~~)Mg&)N&~mf5KVQvQl5W4h0WEidh@J~!pd_l!^<3Gs z2R=5in3ZLc-d^-%BgU?$h)jy2v4vSOJh}aF6Dv7EM0st_hS4%FTEJ-Wlicj% ztuggGPY)zBn}QV93h!GvwsLjd{IOPDo_vDi+Zqfd_#o^h+FF->c@Ml}g zb@EE$pLZ0tzbuf{??s4r6iCV%F0kHQn9J+k@ra4JNTZ8DD$mVh@?kf5w*Pzgh{fL2 zl7`1sG@(QV6#&(KFipo)0p`2tlP>Su)(@q1)$eVPJUKl{79LV6?JQBzdU05A*UBBu zp9IiK^92AFfk`1rs%w<}DP%XEMsHf9lj#wS9y;iQ+Eml5imvfA;Woy^*U+L0wdHon z3HaiuA9B^ce(HpnDpW8C+;1`7n1f@?yagiX+o)&pWTX$jc5RbNY{UM1I0wI>(ux!~ zew|Cm$|fVxB{Tsh@K;)6Fawso}fs&Td=A6mdu4j%wPqU(FCgZ}B^a zPCm%FXRvfq<1cvcgXGj#b$ENB6PaKkoz>G&@47yf_jxOpYNbjkmG8Pt2mvJ2 z(@^kgL09u_!+iHU0WL084J>{qh`Z+YY^}`l49TapPHoqO%y6(5cO~NFi{X4brzGUE zLiUVmf-SK{iJAr>6;~Ss=#L0W)x4@Vo8KW&rya)ouF7jFx3FzrOeDDnha0^tY!KkV zSX8JyKJ2NiXDJQJWpah=(Iw`(vP>E|g5>U$S!tE8bEjzi&j$>Z61LlWj^= z!P%2zx?hD4Wp-Nx1PB(qqF|9#tkT&=KPx~1qSbiS#?o;0=xEHR{Ki6-seT{LH{^+YxhfApd5VUWMow+ZOpt8c)$J;Jg(rB@37n6&9{ zTQ2ngviATj+pHB}EBVA%Gz}?Z7IAC?zv*OPliaY?u|IFcAC~-XIgLK>AHtUJl6#(4{-j=XBrn3FKPmUXZ;si)4_P(uL`p0!-XANek>#o*pzi< z9&RN%CcgwIA6QAf8!Nr9+yXQE8*5OH`sx;W`T%tTE(2m4gC)IdAxr@2{0J)1l48}W(P;t_Lm?E)u-u!`5G{j{>_F*3FuNbdzly0)#be*3K?sxg`sE48rn zjmfoqp)2#|ix~!AYN3Je)0~25ST12FC&AYQm3n%tg`QSm%6J>d)n4CfKgvxQq=@MWhsx%!95@6j;BymO^Tz2G9ADiV-guOAcu5L4BnKJ_FxD082 zs(VM;@qKO7RG~c{?gHy>{@`qCP!fA3vgpAk!{&Wb0^^Ri5saA(& zX~(8ZyXb8fdY5PB#h;x`AvU%^9>`!};irddc5Ikv7aUvL{bx_+3jB9`D>o*9yiyL!G>Bj$ z&F?@h>0joTiBWv}YmBHh%sh9uqev$n)f{%61Tb=9cuitgubsrB%z~Uh@3aTsCg{nF zf%hRJl4n$tNT)hg=)0Brn=j6380jMSHP29!e)!Sd*B%@E+sr%%K7e+u@Qv3;@30u&i) z?J1Du+pz`&DC-NIbOf1BvZlX(x&hl(F3(33PtXx)wtkNC_x^XIgZYh)E}1gjphgz8 zU=j19jIzo=k{Wx|Slwv>)6Y#(zn}_M^hBEK%f=$gCXA1p)ZSd!LtvZ+ksG45%?l$O zQ*Du=@kMg|#p5@(HjxfBvNf7h3T6$#m>nf$Y}ztY_U%#2R#X7)!!Jn|=Rw^o?#TUt z>m2obQi0W9S?ZTt(x%nKJJFWDha6pU{@*8aVU`Z5RUg`@Z{zecTOuItwJPYAKz7B3 z@a@fI7OmJ7()xT<)vbjr3aLZbiRzmV$YUjxu_meiqkOr@cSZDE$@c$m3%>P?P>o@xU6~PCETXLW?cT2VyXYM#ZugjhGxXJzkV7rIQi_1qW z^J;KjJ6|Yg9q?QQjND-mpLp5p;8kzIa#7VCBE^#1uH&0ZqOQ?=(}JqZ{C_nOy# z;h-`vBp`#=sICJD)EI}?`Aeos)+tEkZH|S1a2V?CU^acGYv9=>PKoSkk6R1TSx<6z zMvk$X_BgB3Qs< zc(ei&?xAk3z%b8TQa4srsUAmJ0xn#fIk8f*A?L$a9>P!|GCJw2hD|g^#-Q*YlFiBg z^XW=gF*$vKIrbBKHz3i^XK6*D1oSqDkqOSWWfSss77#hkH5-4;R$>vJJRaBNls!*e zPlrNY!gZRMLkW{jVaR4TqB2&`sS$OZ&msEqUh+M@Uae*UinrR@s~0*23$Lq3ORG zo!l_`OwYVFJIfIde~(#5e~t8(@BI|VMv%j3dF2-T&Mf zOSj&MMsF;jrrb-5kcg)5=mM*)FzU`W$t{s1!-P{0!^;y@miQn}#z6__&VXz^J)YoB ztx9|GrLGLJOndjGuR!T|cHg9Pip3eQjBdH{xM3@a*BqkFhs@FL+T{$LbK0}0M4er0 z+ga2&>Yt9v&Wd1cFj4-MxYO=Q`e{x~3HdpBl(TtYBr)-u>Id%O{5aclBqq?!**fME z%g@>8S7%W%MTH8W{9)KSKx9KHS?c)SRbr@);~3?F)HCcwVx77V-{}{5M>#{wZ%?A= zTC0?p9NAe$_ia7dh45~9?8UV!1r^1Qt9E`XKU29Nskckqg8ctq?s>j0m1E5jMn1OV zh1Jx$WE|UhyThbpf7eAqrcJOYF_ncJFJ!Oy9h#a|UoZIc^^*S_ulPlea(UNK?#Wjk zuMe0!(ENffY&JmKW~OOKKf;RS520WDsk;7R92w3YRvq$D0o}8P++~R@k8G9BeB7HD z?*4>sv%S;gVhQ7<6eRNEhbq<6{=A8_aBt<6{OpETZ~#hTD5hSrtgru3qBkV9;#$tE z0zc;kjBxl z#JPxip8qdrF+Nmn@#M*EDguclQ~?$wdzQ=5yA~(S66;SLX$M!NgOir+D)<9a0;f=B z!@nNwhWk{Dj4HB7<1MY?p$25kI~aPJ=YJ0foggSbJe`BJ-}1tbwAWGN@yU9-9(4NK z_q2_N5mpPnko51L%pS}V*J->eDWt>=YLU_ucZS&SYRaLBd8+;|t_ zIQ1VPx8$)1?c1Zy5~>KI#g~A}qN!%TuNS&{L4FK3`qHWI{9yp}`FY2W=ek1lVQOk` zsNWHlg7y28m=2>p%XFw@+in*CJcKl9irUtWU+R+0Q@4_4>(Bb8h^ys-+BR*iCb_zV zcihUQe@?plURc2ZsX_xSZ~amkhvd-#mzF~Np`}89nM`ZRqoE^8u;rt?AYOwYKLM#J zT$KeM{3n|;K5DJL{l*zt37lhH2LNV5Agu`uaI6Xi&zT&?vH6NPXXUpDKK9{~XXM6OOfk8N89k!m)@4eCY~Utpwf(L*{bM8O zIoCS2r3?{H9=y=cF%lr9GtMLaNLQQk7I2t42~#Xbzfl4Lj~=jja&c3}#J8FV>>=Al z?dg5zMUz7Z8ney!EHVF-PW}h;dt$ScS>kLc)YLHdd@TzjFSRs}jy$5VlitG?<}r~$=J3fk)JK&PWuw0m^)eI%eOFqd{Rl8CX+J#j zmn5nMOLr-%YN!V9lydEnDJLvioNNe*!a7--2EevVW=?|@tIe{;dU=UvuBokQTml`} zY$u-Lg5R&|jNQ)SuM{FSWn%OCFa8iQHOjQ73v9<3F3Y=%;q;#gN^!p(;Z~u1d(MGz zuNvi>^A>1I?kgW?3_ZMQd@y?BPt*`1B?nD8SHLYTojeHZ_IoxzQnK&Dr?b&E#Nn8~ z-WBP>iT~tk?>G!Q_oni~*ZF&dE>Gx$iCr!A? z6C?c*hwjY`)dG8ZF;4F8m7k+k{bJt?A}1JgR6Jn5(f(RbKFhLDo!a$xDTzUj zWM*rZQ;#e>=vMy~FqMXQ%i%aP2YL%odoop7HL^oWSSu4cM+WRQ%JjkSM{HoF?eBzc z?fD}P6K)SYu~AC)_I81~t$SJPgtQ>!>Jf~EU%#%lBX;1>hMc_@w0Ike!#31T39RKM zPDtTqYM8XrmJO`15>Q`QgVUW?dg1!yi}AM(L5YdQ^>OFJ_-~*Rm1A7Het@@H4WQ)C z>K*xz0IG^v_GYO~0`M;{)&|b3U22<0;pt1y`HnG;a*{5(H?$Jju5b?8G*pMUCI$2Z zoBIyo{V}{rt9c5J526xy_ntrsQ`*o=-f+eDK`s|NjZHqdB@|C6lzE;p=jJ5)5g*tU zm(b)Sca0#G1|^3!JSsEwp^u)Gquk3@&8Gap2@eYs@6GF`I`iO|igDBDoCRvBoLH_` ziGJ8z@1`om?v2uG=cgGi1kXFBj|R-AML* zYc!xFDkN369o2;0=Mv8M9g{4~2S4~}=l{mNnl4T*`o|Tar&?(y;U41=6pH8K7^!QSJ=E$4oJQ!OkbQj3NZc-Sq6dE*uE z%-ga#N+%>>NhqEz`XLj`^jN>73cbsx4yxjH3`yE`qpDRt+YMCq{&(ugTi-pIMTK(0 zd0wl&(&ojxTI0u~7QW#=qBn+N_wQgnmI#Lm-XM!ZH(t~0Zh7gygh>`dhept_^qsw!}GoA&S#877ji>t0P2~YEWFd39w1e_K7XK`ih z$uKgJj{d4{Kw5vQ@otMLiqu_<@>)d6YdxLj z5^nML;;d9Uw+*~=*FyNvd9N`Wp_8i{CR*kDy&cUFeCOM|QcWGAO<-U!*&#JlobXmu zEPYd@PhuGXD~rOpUo)R~5SH4_c&b~m&1a@tcoMS=Z>}sm0sifcuzqWLhw<+|2EF%D+sL&} zt?K^d-4)LoH~uolz;9sH_8kla~yg0bX8kZzxYPSps)sJsl*yL0WeU+GEDa2n75= zc-{=&RO<(>t4z#W(eWP0wA7oatO0;CB4V27ndKKF>?S7;+DG4{&t8NexCgz;z8|Yn z2yEo3Irtv*B3XlMJ=hXdgEy5AyUsF*T*6h*^1;sLCWSfneockLw@s~0P!f^!KQn7s ztb@KA~T57}oq>;6RO6Jhrz&$g}n8=}NG4jN}Y zrVA2Fydl$-T$4?ExVzj^f-Z?I6dY=I)e#Z68${_Lm3cUcn@@ZC9EAQ~K?}9c5suf! zPwZE-Vl-LzLh;aJzX6p|V(5cK>G0HOEVeb5_S&F40%Ls2GJ+j#+2uWzaA`+OYvfT6 zqYfM!PnAH4ygf z)Hc^SNhsfw&%Eft(0d#w6UaA(`B+ZYSSN|Z;=>p6iK#>h`>|ifCpn!|7yQL8A0z&x z6k8{E|ED@@dcErb-S75Tkts{~s4H#b50n7nl;g(Rpu^YjOw*;`q}EQk-&nA^)L*fUShcx`_mKfs}3qPq85Z^o`CIThE#hg#P&WdFO$zk4c0 ze*@jbqQA`!9O0+>Zw-|G^}#vcnBKcxFFNV(S;jmX*me6TtKg^1@^PJc6s6mNG#*gwfu)bW45dz}MuNh&mlbK03}KiD{l^Eiu=r5X zog-Fslb4=)tb8^9FS)tzy}CDG%4PiVfabRSJQyvHflP6IxssD@qES?J>9|TUeP}^{ zxi@XgET|0OL#KvwuEhRh$@!>3mq{gs?L|D4l_om~H^7<^{}cqVBl9Pp-Hn7$X+aG? z3b;8U4PuhIwnksV+j{~;_jVN?KHW{qJc>-V>6A6j|7pjh1ugYWXo)>r&hyRn-kxq~ zIA>ool)R3rw81t_@&)8Gl}8$`Zq)?~?B7P!OVrTxF@i6&p7UD0NeRV#ihXWl8P4mM zB2=FN62Zb1FFdP~YdY5cTN=ys2^wO?AheX0=c7**$n;a@v)?N;7&-sspBG*JvDd`a zP*HlsKI=2u4H7ZM-za9k_3VotfqU+8%q2lC=y;YS3=S-%U2ZRDUA5w+4ifmj)jXN8 zhdJSOGKd*Z13#Fwe(5VYcnfjw){7l!CgX@a#VKc;a(jOd=;X@=@(z zyu&;YLcyb|&)=oI{bE2jvjcp<9j7$Z^cSh97La6s&Ir!m)j5=i0rNL`Ez>b1sZSF3>mMRhS|W-6 zT$#$Phn3~O!`vb@KY`v_h4CPB*;E2h`}Sk8`CUh^Mz{e0en7LCmhElF14I|2H`gD7=McZ~4YP>9zj zR+ASq@si~Cj2lN1YwSdN6qEfvO?OAh^MFLFvQYh^A|>kiNuz{yMT5XanR2PwM{Gnf zT*xG_Y~?Tem2Vhi1gB)G6KIawQ>-fLRETZTNfFiq59}qQnnq7q!<)IWy6+{T3)O7D z87pr+#wrKnFF8=65b#V1ibBgdUy(W0j%ybj_`X=A?^0bPFo^3; z>s!Jns~b!04P;4_1^EJmReko{k&QWo0~-pa5b=my(3H-zgm!5WDz=q}0=B2m>+&S& zeMt4$OA~j=B6u&awCtRabXg7eDl? ze;;8bK^mXtYfVe{U}__xDFffyhEn`)OaAgP&|g(@jDNxEX=Z;&JI1J$Vm8WTDxse{ zuSD@T$`yuT=i%tjeQD*s_p-Yox;_YZq`)Vzy3Q8odO|SCg~rw{bn%+nSRW|{B}=BE zf}TZil*G6=vkgm~^N}-Rv>p(R9=Y=S1Iy{lUW2bUxCEf5M&VyTWtW#N>eCofJKWc+ zvsBedDiXjb8l0kx&40*uXvckrC;7u+ZL$3}JCoK^F}g~*IIcAk=9{0^^lE+19a3(; zL!^5Kk3RPrV2=G#H?hov|Ie4qtx+uOUAs&yf2TC~wk6Tz#CvEkSukrC1-j1DdhqOd z;(Q2YdYxqU+r?#=Mwr+L>K^aFKU{J>%J||BvLw<8&zUJbq;Z*8cMj1WC~u`kY*}4N z1&aApDS&Pqw9Ph+&^S?%8xFCLPVjON?lWblNg`Ktx~u*~d3)UGQl2_g2KxD}ik9!R zQqJf~uHvhRJ@?g4rkf77yA`sj((k%agdYo``w?0zy^=#u*NIFFO3=+%*8Ye^E30Uy z4{q=7GbJr??&<~}ZeDSs3f0);hePm`8Pe~yhOv4RQQJkua1)TDWpn|Q?pK*A(~3XB zKMS&7)GL`mI3lh8qm)R?ZFbyGNYHpH_^DYnXod7hmsT!G@}FczpAPcGR>bw~nCxs` zs1ZOH41Tq%>^P5p)E}u})H~XNeqsA){mS*$^xwK>OW(v3Yd>0wWqwh&-`$RsSAa}| za50)jcE<$^o83%Ls@4=K@c4hyyyM^R>=ycL%U{Tzb(mXNgop4kerV{Mg>u~0a4mN^ zz5h&Uf)f+wIe0oQBP$WL>|Y*uaHEp**}V&E0G&6jeu1j2>2JMtNq>A{TO-(hLPvgx z46TDTQiWM8uD8hH$(0c)^XufP6Co@`o`dL0LYJwC4tX=@IK%-{mHH82c{LMq_#mz& z zZ_Y`7Vw6lhumuxla`=&7Y+KpARyW$h^plqqd5v?Ko?a$bSOti@TVs6hw)W_RmV1w{ z{(jE$@0Lux9(ad7E@}=O?rKs|RODFlX4x+b0CP}{S4B(a+kZ&FrKk8&qBMoZ-?5Yb zt>;JocG-51X~n*uC^?z&lLh1iii;F-wsqutjK+L3aZpuYovI%Y7#tLyq>rD>~@ zSCIygE+94KUi?&n&xXN1isd?Nf#PR)`DW20;4~N>`bIUTf?UU(w_yh^`E3+!DT2~) z@$G4nh8jx}+dMsZlAiFvMsGLR^1F43robL&ZT{pWHr3&cV>$PaHi3_)qyz+*fpU+N zW&A1y75Zc>H$kb%@W77ACCvNY=cc(ka|@pKUiyLc-5_=%1Mr>XfyGp9)6)gAp_a^y ze6OHwzKJ`@a`k?2RL+?#&C)=y0iHp{k-KsycMj(nHAHW@uL7H&ys-mVL?-#1$e=II zIgdtwqI}EJvrE5o>l{L$5mjs__aY4YabXA2$hx>mX9h(aC@vHMXudPNVDg_RniAFq zQt8R1dVyWayy+ObvRo%XKZZ2%;s)=x17+Rp^NxHXwO_&OgR?Jg1ND_D9hUCpJm~*O zya}#;wZ=r%v$W)2?&w{&cHhy*2M$k(FbrLgkR2w!<)*mp8(idOVD*Vtje+D~h9p_H zfj|xM>KhcX-E;#RUS_GJUcZKi|8I#?{n~v$ro00bbZRLn5x}Rp(mM2ea_NRL)VnoZ z3Qagb6a@nHZ)0+9LNBafF9VZ4AyC@+e@U%RWCrYiNv~D!+QJbK=*XD-9F@Hu{9{z< zpUtKqfpDw!WQQ`5^X9eFv&8ieP{V;F`QRR0UN2FZfwNU5N8*rZ@y#J6CSRK3#~ud- zF;w*30qCajtfk(&K&sVc#r2Ty1QQp})J-z>p4IkFKeI~>TfY?3-(dZsO3>`=U#?Ju zve86)2ld{My!Gx7v3ti%)Hlaz**qQSHTORG-^T&`e?~%`kD3J@4{!;m2y3&()XqM6 z!5O;cg}zY-nv9yd6Mbx-#GNk(*F*6y&zpmn91FsOoKuN?IsT9P^Z+f2IBX2c z@hcgm6bY^B8JB#v&~M9|6kL6a919XTxAk0fqNhE7Sa!NVTe8DbdT)`NPyw0J%Bwf{sWCe>%Sm~8PZ_%B1X9i3dFjYEYSMyXS?>Xi(mdMnEzIFEgoY8rl z7s5%@CrkORx^K4p#>&$K)O;awFm3f9EV4Z{d=t6_#}{$^V%~WDZ4R@?zb$~#eAV*t z-GWgB4w@DZyjpoP%hGlV?~h>D8|$*qv<>1+b@e~p(~*yK zsI>T`VEkiW*GeK><8?xJ4%aFB38xGGb#-$b2!o{H38Y;eB_rX(;z!bWKYSj2gnFjO z%H$ea9a`I!O=hm%^?s`9djvdr@3x#N=d{86smttm&$ZRXP?tT+W zszP-=OB#kHgoU|I%%F}ZXRYNG{5~X|qo@LibW9gcn#7EB|3(v|YsHdgqM1P}2J=La zKdnZBA}Y^iu_7W;RFlL)k$;@Liw~|@`IUKMZeq!F=08?TXlwqs#NNK<#&FczCHNLX z1WqVc)@{@Ue;mV z<@zi!?EbM6$4#b>&-3LY{OR5IRhm8yvys-q%4b^IgDYPVnWV$DjNFYT7uaxQ`Dz}8ID?&fX~niY_@!^4KO9t>7@lthYbCLLRIac;XKJK zS=xd07N?)acZ30#IQ@vOSF?t~)^+9|_&;_}Na8q)OcDHT|ZEI2C ztWeoHS?Bah>n@z+2(7463iiRrM@za?5>B$D=vR>@%n)zNa@B{wNw+h%T@P;z%FR7- zM!;P>_d;WoO@c^vPUY^D7HuH)BwusViCZPxI&*di9;YrD6boyTEB^A`F40B>`sw1{ zdfDGco`X_b{P^bcjA=Xd7m<{j=>_#5>s(>8Ff42olpQqAxf`fhCfW`FVh=3VQDEBE ztJ-bWH_yM#s$xHt181kzlP(h>v4y1)+rz6C7AVCbSPRGuG5M%YkcSqTC^M9|5BO8n ztVm@$itKL}7xbzC`qzSCx(-4j&Jzt=Z}k=@Udm|jH?VF0Z*_Thz;A52k8A5M&PEa( zl6u`!a0yq=gl=qdmSxR6G;f zEByx}^7?*`+)&omnBWrT;vZWW&tv(q4|flaqE$|2&H!m!?cxG8nadhaR@Uz7qW!6Q zvQ#sDw&CCQgP^S!EoJZb5yW|~n7}M%*$0NKa2{yf?K8&AXIq@Tb^%E%3o=_|F|*_W z5Ly^p>&hNv)&DkBz&%Z6TOyPd_|=W}b2&>{#}8KCU+x0w>FL2yTQrKZf`+{b%AmAM z11A&jO**s``aes&?uMVYSDbXl+8BoMt&q$92@pL$&Y4*s^m)9zS^Gb&F>ZB-Td<}3 zEcP28CuRatKUVQY+ z9)f3)`Z1|5->Xdjr?2)bc6?E9Te$SsJ}Z?y&gQOz8*O}ZL$Lh@&gq`Z=b1!h^t=JV zN*f%Pd)QFFpq5AB<(F(RCy~vHGxAy10}4fny1kX&t~ysW&F;(HndLU(9#RruAG0xu z;`g4xptf|v&3nJuAMj;=G7a0fqy9_g5LvFuD`5^}f`xrfSO*L_?<3ZWUE&ybJ6t~u zz{Br{#8%}?XutJk^EI_+W0HQ#SC{rkfc9d^q5`_#bD?rp7G^_{_k>U8Oj>YYo)mpw zaNA+{UN%*vk-P?fP3S({tlCUZPcn2b1o;WHMr7~Dh!}m&l*!f_T+r)@z+y}5$kmK| z6K*&3c)h}jMv%?atw`L_QQDcb5nR|(MywU7usu4RhgpTvy@dfya}DRi!~BzN=Hc(( zh7nF2!So+eFlyow!2Nn6HluL&0Z6#N{sEzT%IEE7`98K5tNw;Mzgf>C>{&5zXRnv(K!J;2Hi@Gl{JOL9(WV1B z{o>(=X@w|3+Z`*$CMey#oDNxOTl4g6evz4@W@?}C+gIE2l{>XkCAW`hgvhx_mq9M@ zkLBF71He_MpHa*L(r@0LbS2NK*U5vHEaqa#9k;`{ir&(UmLBrk(Fd48s@{zPbIxY) znK)jrUt}KC=#EvoFA!JyQ?9>9!RZPOwF%$D)Q}&)C+&>_JtfB z%VvL{=TxuqPh#wX%_;l(K?80Z$J%l>sVOdkZz?H!E1ceFa; z_WQgajWQK*h-Z4}K~-4Ll@rm&ursY}P!%I}p7$o1N#OAicOCN_gv9A7s$7NZXnUB^ zOPKniB$nQY$oYCQ&wHC+TnL$`rQp37&F7$vtyb-SFbQ9+lN5LV*=442YTn-^CGzPA z;5tAq7Mb!>+nmUm(Gr%UabX*mi;_T7JofI0JoU9P{K^Yr@bU^id9bo1d=~t9qFE+W z|5~Tt$~BpPt4%ta2NW$&H*?5%4(_>0p=H21>} zZ{)KG2Dllq{Jt0`NB4r9$zG-&v>l~>kn(I zHQ7||YS~!V(?^m5JmOxp5@sH2OYO{3HSSnYoc=W_5fOF@m1kP| z*W4-|^4UC|i$`g5IdcJGSb0=G+dxe67lL4EU`?^6eTniXA}Ncts;)unAK9QGl@(VV zC603`clU)4rni-Y0yYVlRKcqM@|%w7%@}x%q`*GcT);h|_3r%;sODg+Acw_b5s&}g z>JVtI!M`9=PfW}WA+fgPO6V7oV<_6=>i>bZ6~=~&e(7qy`GR1+J{M`oU)Ds;L-rDA zK@S%8mHJkeE&O>U_GU1W{l`17XqreoT|+AmOMijYSD&pV*Z}eB%^L&Fo!Vf88Q?B6bnkM2eduinEP>?zd8H;_ zOXTvAJTUx_ycLxPna(8#Q7wS}<*f9X<*=R!OJubRzPZfy;&}-+Ogo~nb)C|! zU_C0TlDRf(&ZB>+Jt%Ofu8F;TVZzY=(5+fGr8cL`C2zeaP=lj7m*v4+?zef#{hnMY zGxk9Yi5ONn*v~!dKUO-F6%@9`z}WA|Fw)6VuF|{Q-v>6bNpI&>y0p0);+!gjR6fBj zCnffWFyP>FEU}LH{ zn|hTJfwNShVHI%TW}1NG)WWWD5RNUOP22o{Z8ybZ>oY;2SpfUolgiCoxwfvj0bTG? zRypPKH-+t>=Zn2&Z=gq+9n@cmT;Wvm`na-ml<77eNPuY0yi*DOVr)d-$?Uz$%A4D` z~A9jZml^|lb4?2g27j;flL#qa`ZB#3V z9Cw*jl9S#kWZsUTtzdb>KYi4`E&Y?kZKUe;x?5rH!4C_VyD(ktVsjtuFD|Plh`8Fi z!kV-ZVI53DJ-k+1>W%h@3n^yGj+o7D{T?PBGA>CKw!56D9GTK`O~yc6Et`B)skz)# zj^*++_!basu!5H~-dm!77Zf=5E4}4e+ZY}2Ok$5SO{)J+KMwQQ;qc3BUIx`10ZtiY zScKpxkKas1O+-3lPjk1PZqPa1?LV>w{VIUB#Yj&wcs5ip?t5B&cD z%s?~044Z#%O>Ce%m$u~DNv(G-;M$v~7Gan>S1Cj|w6?O&@k?pJG~{f;5of&8Sv!6w zMH$N$R~YHtQ$=HKWQsQuuJTOroNlct&@|^z@ExNIb>?;doTGg!&5iRnX5i6Ui|1y~ zxt12fn#-OkLvT*9?6L-$e%KMyaqlIdvDU0oUx_Dn&*aP7 zr93aO)w{9YIYP|cTg&x#ML4>%CJ@5TqpvhCk=C1s>SJ61w`;8+nzAQk%#SmgAcrKm z8D@-Ko}+_!FZryIcjW(@OEPk=v6i#f*6$meQg!D@%M;R!)SFw1DQ&E;vNrPk-FudL z4W+@fhUCr*^W!i~g~c#()O?9!*YSX$(`Scuh6?F}BoI$g7BO zYzf)E#@yN*yMtp6wezu7#=#7Mz$UJJj~v!&)D8=HHDsySzXLem9Ajc^@2JGJf}D9= z%Zh77SgNr%me!jKnrKIDg1m?D^ckU|a%bJyI1Rxb^ikg15OmF9 zmO9dm?TN#xW5h|Xvz89`m@}tqFIkH=R+ir!5xi)v<*B-2y40J~ha@jC=Cu=$_?;=v zJ1sq~FxCX?-@EcC#~hU!_WsO|vr&C-sG_vD8vNV&r+J1%kq~D(WKLr#YGqZa6^AxY z&KV3bt(fz=<_zK=You!|+4a0~cEe3;SUE%)!?y<9%g>uFGi}Ap8}f*7pKCL2#EtN{ z11NQxF;sR=T$z|AVE&nVYG-9ArZkeKQO(Q0a1621v-a%pm@8;#unTF=Y=;@Rt%gTH34eCuWSCJr-hATzifxj!6Bz22#+<*)L;k@Ya~3PUe^s zd^3ePr;t-*;7QwnaSTWrTxcZJdd;xXyu;k` z+vB0Zl~(X=TVwHU=Pji>hxW>!3->3_xzU?uvesLp-hG9kj5*f=XI|-ZbtV;pw3kZr zpL+o43yZpbZwbMc2Wa$8i%~ZP?dLq=rf3*5GkGWV{x-+j*V>C@JI6tQcw~giH8%+2 zPFwPRF2(A!gOGX&%N%r#sfHgDlGb^I_eRYbBL#7< z^~Rc4tlt{B6(>zqrarf1#?JF1I_`1)6K2SD+f!|9?JtS?n&4IL*T5Z`Ghjp(7fgQ%|N>+%!B1P#FHGf_oO$PpOw^3{9LocDV3fTd^bNshhL+KyDjRk2-GrgcFAdWZXl? zWe!EuFvqsY$m_{uXT;*Q$2Qu{DTscL5aKu%81aakI(81o+_{H@b$Y9UKuCXS{>KucVJEd{PJw|NeoRM{ZkFn~z;-dFSky2v}ji@k&@bM2L^*l|L z;J=3)+>MhMC4-w`>;buJq>YK07Ou+P69_-&e8(Tbu`P%EUQPAJG4SC;3W zE2SYW`0zZIy7kLT{wvPKkTU`@*AL^gYUkX{y2gh74DlvDtUd3vX7J*Sqbp;qHTV!$ z8qS{qu4YbU*gw{a^;;v^Cok#1l0rcB7zS+FUE?))Mtr(H=Ti1vb6#MM;ffy@;7Xh8 z2xabQ?ztBR(;C}7bFQfL9p}DhSrPemj5WZz#+u`ryZ1h>S*kUYOy-N@7G{h|!I0DR zPu}}OWbFXHmR6ed-{T7-hk5%Hq8MvgNg!Zvz|gi9migYhN-+gZ$UDM>SRNCeWJQ?b zAF}f7m`e$GhLOoTcOr3Jd-8Bhv82CL(mt0I)!a$nZcJgLy>>p!n?sa&68LF5~j4CtQ|P-G5} zl@^A`@Y$hAFGTsVGA6W_b{5>*8x(TxRf)3#C|ga*sZJa0KjIAEA4%RUEGgN%c1T1H z`w3%(gpZsOMDw0uPh-rHn>)w&(cZa#b)_YYyEd#zO;L<%&P}$tcKqsCBWq-C2;#9f z$Yaind2;N9=oZ3wz&$3+Y#7^RYXw1rF*g?M9GgIGuIa}-1BmU}GwF7Y`Ovk8ipQTJ zq+m=b?>!^t;vVy;Ja5gxx?(u@-3z#9jIp(!HXQg(I}d0LO`jIVoX6Y~4KWRUh&=|V zWZBzqeJoYizSgcrO&j)lt$mR_gW&oKG3s|L)sLFD9NrjnlsW_m%@`NJV~HcVD{d+2 zK4yH%9jl6IgzfmSrV!!Zdx2)mv6Ha}+*ebUUD#X7Hq9i0vBvuL4eRN5ZcO<;mu$kE z>#liBT%{PY6wKSH(RlB*=p1Lt`b|5dd`8&r5JFJ(otfq|L^0%$M}+;0^I3AQIEWW^ z?EW89)@LrYk~p$(bDop^ey1JUu(KlKUvc9&4mEp?YwAP-%Yb9o> zjyM-gN?VDaY|ISg6qc-U9eZwMMP!(}SFG-v+GF0kcYMvQj=vY?{#zp&Z)`!l7q_U+ zjbqMwMv2irv!>9PnUNr^t>cuUK-di72rmVYm@+mD`wMH!a^`U2zIMPzi4#nHPNdqi zk}Aj8n@e~MVZ5J{3ewx5sCx_<-m-V1)W~C2VXOh%e@vnHw1&L-9^q?h%{ADyb_hsb z3$iKBWT-W^K$_JDblA%>B&R*A7}xanO=E3!3_0}>lJe7yySZ%4vC}&u z2+G_0A$ZS?@Vv)J-57iEJWj3ivKLzETAL4g&&0csM$r7&!%JpO{Qovr?tGcan>@@7 zur?Oz^w=|jH>^p~F$Pfd-HR%8q-?Ug)~aYsbG33#!R0p+U_{QWFnCYl`*p9S-5F*a^jPblXDqbrxl-(N z4sjr9rv0HZN9f>M+v;JD74tf`Z0Jq<%RYong&9KB{xsFL>(_}+_AbB`qNoM&|2d`G#xIJQXQS|f`p$APf8 z2iW9W5&t%}GWOf~9(}EKz8!`l^jguxU}S0LKBrPZ6VkTrSP|NAMVS4tXSzk(^J#l+ zeY-KH=>J`r;BbsU&9GuHa99I8$e%OKaV`bIHs@kZn=z0fPDuM2m&Ev7bB%v41hPD~ z^iNL#wm&b0+p(AS>I}Qsc+I_}Kf_4d3_->}ZCQwzhAd;uW3o3*J-#`1KIq-k!63z` z>o?Qz$69NmEzR+xJrfdbjFHr1?7@sUbE@DAQ*~?36~{YfrpQ>^M`Mnh(l{1m)LXgH zY{p%rxpMmam>V_4{;cE?zP%IbBI~5}@IGQpt4$T_zYiCTUwmDa9 z;as`7VK0@4IA&(X57VSNuWhzJMNvm z8b_+)7-7mR?ERUT_BPAh1G#Qa@UFim;NsYMhI#E-?LU%A(^;F>GVSE(G&Yi7*m?OR zZGohdCW2nb`FeY%oWm3HD8$*jH8)MAu(Ecf-d!vIKbMyE4&iBNOl7va<1AR4^H+3@ zn8mfG{L2~Z#d?ortD06;;9RrwVu$?bUBRn#Ms?n@*F@M~$vb*4O^>Qclv`lW1-yfatiFh)5hui(&_X*qfu^Y8lyoJ`Y^Wy5{Ug8R2v!rG&E=w$9fc z^Lu~h6ir{V`zr@XOErbG`A@E+_4j6Ofj{$Mp)h$yLb=5&ugy1y16#w z;ZMt(Wd=RYALc0L+jAm)Pm#&L<7!ged6;>x_|33}sOT7(eRU5Bwmq-qc8xj#pqR|w<8VJ>aNxR)Ak%n^Gj?}e6= z!%*GLWAr-?DeE&9)cObl_}&e}uX=3(jXtA1NDceJYb~6&wU>U-W3kf}D8qS;Qf<00m?oHS$NO-#F{a_->X9JiqI99!#R@3ob+hZf;2X9eQQE^7c111kI^#~-Trp61&TQPb78HEi@lA28>Cu$7pxR%_OE^r_ z;Snbs!b*AZajg7>KBJ0DN$D?gO{CbdR;)+d`JaC!tll%1c>UVbPjW4t$T0Wt;U8;Y zB*kT^y5`y^eKt&tVMrE#i!m*Y-=$Wg=Xy+lrm3BIRo2jyB?4hwe z2B6$s;qWJJk@mLNNMN69SZ*$8_Ba*_%3W)#el3NO7&h|x%o8boWbE;|cHVm$yQLv5 zT-`l0Qh41{AYu;zw6SwqO3#z(KM&=E7`DFB+Vfg^ZwZ(%hZ<)Z+wp1aXz3i6`ris$ zgK-Y9zmRea>YpRNX$`UEpB7&K#~p^41JM58Ii+U|?Tj(zpjM6(?fJ?^w}(9xPRXn7`C@sd{PN?03Ja6}N%yvHa@ z+)KiKZ8g20w=CCM(@Sw~*x|V(suQAg$B2t+eeLD^Ij7LwnDELTqxLmT(VM;Y;CC2% zw`yh=kJPu*#F>)~Q34>lBZJoFoR?Oa*1`g+3I|4l|$*q^xD9@k6q-;+a z*cBFJc}x?bYEIeYlNVS@-kJ4o4AG((QcQYFL-Ql-UEvgm1j}C23VURcr96WQ^%@f$ zeJzRAwt@)Z7^~HGj4hcn253}U6AWpu*yfa`pnOQP{3}fT#6+(GJOV3 zq_l=;de12QT5-cG4RHUTJ6~we$;usf?AsfIw?AyW*fs}t*&Z>2CI*GBITvipjsvc3 zh7gI625t8<7EHoex{7V=G7157at<+8c=MD!YC z6=#RQxU%!8PFmv;VyR+Gh&ZPupTF(@fzbrXY4uF zoA-R%nR8b%?xnntM^x)r`^0PFs#N zhLw{V=3?{O^A|iVOw_iPbl)2DzF?*Ri@X^jZ!MTI=Ynb7 zi)wsEHSs&=?&?|b33YFPf|4hYbkC_bdrTeR8Q13R+1Woi@2%%G#xz9@8>L_lY5s9< z#m}~5di)7XJ9jT}<+GDq_FH?&J10rLGvgp{UO|68Cpn%mM+(W_E6sa^iKI9dV&@z? z1bl}9!Za3gTnIy|G)#%ZvnCQzT#IdVjxnUO_SVMUYky|#fx?njCd-?Nl{c>S^1e3C z@!FZgGRI+`J@W?4pQ*QUOzrWv*R1$n>t|{Pg!i!$=+zp72x|rHwKlgr`3Q4!aRxE> zHmS=6XY9T7w_=i6Ut{}Yja2S2 zgEUayyO?rMX||izi04{CkZlae^E-1+?cP~|e2x8-8kWlV7~9ih#I3Uw^5|X|Dc5k$ zJ%&GaLekBnv}CNcy*dY?b=pI6eC$QFviBa`$>Ae;r9ID@14P>#JLWP?y^A&nzTn-v z4r>jazqq$z_lUbyd=H81w}SN8-s^g1rx}1{EmYFGmQv*!i!W@=tcD+QC}^6Cu6wSf zx0QB)=NQZPCGC`~I+BQI$*B%u4(Q4fMnXp0t0H6VUGg=CZb#h73t+5yoVYfe!y9WA zC9h23F@iwS89R4p?&z$!2N2uM8yH}WRhPcg=EIngDj;T=%R3gV-kB3$b5C9Pu!GD= zTA@OD=8@$SQ)vC4tC%q4N%*{1=HQ4Ce0(I`-nG)y|Hk!*zL(NU38Oqaqw93+OyxVq z!dRK9w_@(SvoL3N#F-(gW$mrgG=gBtSbP0qk4fFT$41Q?o0oHMnTMRW2H}ajE_ep` z_$8f^Frq-u+S`O}tfVB5-GDX*$ex3E zW9JpfyaTlF8pBdQ&K-=m6H>*RD>XST+44FTklJ6lz;h(!s=88=+@14Pa*vhko$^}J zUQ=N*Z3Lqc*6z)X+p2O!L4!3D(8QXl)nqNXl(Q2?NKQj?U|?v;xa6Ux`f+w3nbDdw@m!s*|8r7CGwd`4u6KjUut9h)9%PWh{nbL#e= zh4=K--;zHf0Y1%^DZ9%Nn!VX{Ig3kaz0Z&T$qc1jxEKR(xjPbHaHf zakV-pj9VVNsdJ7Hz?%oM<`28keQhzDxyNW;%Oh=Q52@y#1E}bmlPP8{V6q%?zT}_N z%QlQ9f;xw==L>=4E>29mJNJ-hh+&>EjD6uYvc%OJ1B5=MP4G6;%J~|J?S0RsrMI?T z&Yt1vcO`v>HRh;9Pf0>xWZ26-LttH8DdTldW$(8}vP7PnNN24WUWP? z5)vrcUVF_ZPdVu|b4I})yGeh|N%j{9EJ{vuczjRflrZ*G_y`-Xd`B6uIPsmc$9n1> zS(a%oW#Tjk@b%eyiE549?2z_q=UBVmI?XA}u@=xw7@3BD;=;qvgT`WSInlpIuvkb- zp*5|XlaYo_+uoz9d#sH8pLe43pL5`N&8gA4mT*_!BX4rV@z9sXI@wyoJt^;*^g5>; z?Tw3qWbVm>v6p<&9)k>Gr(}XSW@Ox*3$A7?mHiZFLeHO3%x5f#w3El++zL~GVn$i# zyfzR)41u9PC2WwsViZDNJIZyArRq7yios6nJR&CuzLIhVK#w!lKW+{El_&IZ*=u}p z2b`)m=BU=s;~ilRmD#ovxa(Z=S#M@7+A+5v-pjMlKTU+Hvj)oF+$qv}53T&P7p`a+ zo2Wi*y`hnp_`se?+If#fpc&SdL|7|GHw-1|8pmYN7=!UYttlynJp42x`pe5pd3w(6 z&Knbq{M&g2B#vdzJ4Z0c+~d77%;=ZB*D}D4OO}650L!$;Oy`*M(r~0zvb5Lk<{H~u zZDe`)Jf{Lc6Sj!;-y?7@kCmJm)7I7-X@)S3ZS@fbcE*jX)nFWJNifeT;5~D$Ob7xH z{~RL`EsW`v9RpU&-+2{mEkUNfl6cx(DI;bD{mGDKR`gC`5^)YRgR|BO>e>Ugc_&Sw zx;C`Iis9Zc1o4BN#?;psn;mcL^?|jQOha8$J~vFYtU8x$a~{K+bme%+xu&G#&lyW~ zEjZ1wHsa9UaqKpP3I8zSz|Nn^&w8Zw*E*L(!Bwd zLtbppD2F;1_XWrPFyDfM`7@!x^ zlK+ko;%hGn=P)Lm$BJA36C(Qb-=i`ljx@Y6xB5a~IV@wZIqR^eu-2O~mUdWn>4ny)Xnx)iL8l+m7iVY_4s&ohLYB zS|M6-%|)3!lZelodoE)yk>E7~rs+@1(l)Ol<-Uh5RUL~LBqlk(utx}V48xaZB^i&t zXHa+B5%NEdY16!>EW#X#3V*Jti5%vf^_|iAee z+HQww@w0}C=~}t!#S@nP%$pJ9bk4AgJ;tg0%%z+?ejX|t9 z6Pjh71FUzgLAyEf!e-vFzv@VTG|$rsSV< zWq;-YvYbb_)*R7scxT|*v!=lI-O;aUhsCV3H&{d4%U5qLVd^zD8ur|~J~Rdm<{LL= z+z7dae@6xKnx-a6o2#jHWVOFM_lC(_L+D`)MDaKmhQds9eK}2Bi?G%P?HsfEdTucJ zoaXjoTf-J?Y>>>c*9iNL$+K$4H0l#)n8ezvnRyQBrZd+zLt6=KFGjh7w<6@!j4LQ) z2T+$b#$I_HleZ(Tt(LzMbm^S22{{C9&9zh9f1H8JJVg1Fl*O%FPHn zwQnz_k~2aoS&osUW-qXcoRX?)o}(FW$KByL2Jr1$%gAj9NbQ)XK2Z(JOnWZCygK&= zYnWStYGy&BJhu$%$+2TG&p`CLhYI$UzKvMeXK27RS0`jAY_4b0+T^0bhQFA)}CHJj|a&P0YV)_XYRVl6z<`Bk4%?NRZXl&%ZoI`-%1`vCj>tAu^p{$$N;`$rmczXwF z%en(9?jJjQCPe^AZ&~Zc3XHDTaNkHKy#@+xxdW1x@N4R<`Df>-TG}wf+*P z{&miA%`ygRU<^y>WdvQGl+vc!S~KBjt+13gN0`IeTXiCDynrcT3I6i>7P?zeWP7xZY8BZR%&e<&3)axL|iI`VSRo+J8o4JeGY(&XLFV-z3^TQzYnpzA+| ziu{@D+&9kj9C7n=O?=@uM$GHnBl08eK$o~Pe$|}guzSuC$1~@q&5C)g zZBOa`6_>33-SOxt%;Bq(X7cvlDS2Y!DK{+bthK_XWEoSKI%TE)lM+nI9b^7~E_tf7 z64pW-%@D+ zN&GsOzCsQmoNT0_#+Y*8?;IJdV24TUz9Pcw9YX_p<=L2>V^r7}2Q2Lyu?cI*vpvk& zp1wE0=U$;tXlylxIEFOfjd`dTTWNM>h`c$6(BD}b`6Ufpi514gOxv3zJ8Yr*6ecKL z3^DR~hq1FhwnD%;7KT-xTa73!=%ln}2=QMjx;jqm=DR1x!x*E|XY4JZu!qw8&GSM( z4K=k~yZ1@)?7od5-axm-edvp4&zxtogCBXFl4T zD{^2Pb37_e5v!MnG}axv;cuiNi@r9j-H!{udW{LqF_K=+&biVwC$Nt`(=ha&1JrS> zDX=gj5@;KX@nbD5gPtS6UfWY8Yi@PZGzO&Q&f9l#O#Jh*mb_;}R85_^u5k~o+p>2s zcb>T(XzYR1vPLfc4!O~1O&t3-H-bW$87Ou~HPA8^cFEi$<$df?(Y)8Lz#`*Fb%B7IFd5| zn_6AM)0^r2 zH0;Rql7hxboZ|*|Od*dt;`-R0BX44Fy~!Ad4A%{-x_u12q&~yUL&^)$BdwvZmM0ee zimSCh1l-WRrxfCv0e&*g<&Cs9z-~>k3Sne@m^ViDRbHXwY|Xu<9P*rfi1S@^4Vl+9 zXJYG4>8v)-;H0-^%1H?uge9$6r@1Fm;?9$qe~xXQH4`#!8QX0yEf|u$2J+wwV}N+Y z5&xM}qj8S;loV1t^iBINdhUsrvsQw_8zHD7Y|yteqx4Ul8{;wz;EU9*d9EhYN4hbmIafwL~Iv6;A6SX|t@!C}`(4Sd zcIR#8wle&ASpoZEt}Vzj#|qHd8`Urk`S7`hB66|MpBy8P}G?t3m$>FGOY^jEvww!8ML&PAgkk-AD+DwT< z1!rUppE#0eTpdG;Gwx-oGB#HA-BYwQjG*8+mi*FNk(Mz>$?&`LgxcKuPkT>&hd*X0 zhd4CEof{x!myVv)!kXXXARu^ zu~z8DUBkP0Bw_Qv2X_5lK_6fqBfCH1x?);md^(PN)Va1+3VVRFql49H1!*s z!z7I!qW>&qj)Xd`Mu_}(;1m%cf`D+GAD>#-vMztCWVBUmXg{S z<3b|LHLyL`v|Wv}sd?=U;xs18R+vftEsZg%GRFStA0f6hrS!xXk`TvSshDL>nX9?S zV9^`naDPTAk+&9@_8qIib*E6;x!367&Wk4?tqANm=i*Qwo2+~V2&*<$D#qODdtv9% z{WypEdd`t>FsvcGxRVadPQl-4Z_L{{7jEX+`R#E;t?Mz?-26<@PG?3Xo4NyvQkwxR zXAd2t3`xK`Honu#^Ymdwi1WX9e*W0QZg);`(YW^N_ek?jf9LA`8Vl=eX5H_$)8tGW z>uNnjb-2E^9^V<85_>19k+WCQ*;soIdjvr7Ia8!)4iklM$3>esmwtE{n*?uVWcHB< z^vu}X3VLk?zdvGD=a|8Ee59qav~uRv2>T>=j}@PrQsni_3vqW1U9XhV1biB4i);2aRM-ZhUmwXsY%F<~u=XnGAG0Q5uI%@+hDya56SHa!ZJ)ej49QwcoOccN>@|m0 zbPs9jcPBK#lJ=s<$}7Zj?lq`8GaOqSxd?aV2%A3EUP;PJZ*J#-^09W>NSc8|V#NTR zGdEu6-f21^gkhn*(lEszn`&@uZJIUq)XEH@d47f{=ekBp$lVigbgc!AI5(P6oKY!v zZ1vYWhY0FUX%v41&Bc^P*vZ&iSvapPlsP71bc|bfJ|=~lzqk5(p2L)P?VXK~4~@mR zbCA+mi>f)!eUuz05dPR1nt6rb=DMf;eWo$ix5jYdinGaj3?0+D2kOIF*=Thx0m(cD zP{f>@08Psq2qO+j#klvbQHgURe~u}KIhNw)+G~MfuduH;vYPZ914?!TjoiFFWoYDto;QX}N*sd_Z!Hwy zIu|U&4pT5GtW^4)LWIiAkzjYlNw~J>K1tj;dVJ;N=dh<#ZkS;dFKwNnGoy4y3|ZG< z4<*Gphq7%>>y2z=m7}pXI7Z%k4rC<-;XlUI=o=Yub%iWtL+m+B=#9Ka3SU^8^$9`;CQi=$9=?a8#YSJG9_gJpgW?DjU|1nk?vJs{>CmbW9W;EN*( zagNl%J;NCN92q?}@92m=Rs_$OW4vWg;j2D#eoWYdLuig^_%#;P+8#qvZSJ|lH>Te0 zSWAy|1-aKV=j2k$0gW@S^|d{g+}EAyqjrplxtUXX$Xh!TXNTm*G3HAA-W$YX&k)C% zGOqtjYmt17rI0a4z`#8x4D#P|KV^<}o;N2<(H+^8c&+8(yN1HR*^7>64cxev;^upa zDFtt@c%_gB9rC?~N^ajVlQ|?U#W&WT;Z3>SZqB&KK2r++OqmUJh1r;}Ha>J8tL-@r zU5S{M`p8e4252V)q#rV1;MrkEYGrKLySEU}%;Si6?NF347V_*`iA`=s-N3gNaQoje z7I-f-{h3ls=7@79Zw58QpCZt~Piw$(uJpq{CpJ}xTjzL8aI`kFhVfh@Z84^`gF8ap z*V%bSde6zP5>o)u7)cv$X2FCUmdscPaR+fl^u?XWZ1Y+XW_9na&Abw5e;g_Bdkl>2 zJ%`rs7$Ms(4iK?DCals9^*1^BK#!W9HH8xwlGK8G9sZ-KBp~^I!B1bUt7&!@A;y)NBZMgn=xg@;e$MP8p0R`Ho%=x=XK3B#y8eF_TKx; zZ?Dk$I%kCR+gbi+4*kWjwj@PfyT~(4(f+#E+S^{ykU7T9+P#AedY$p-VaCzxJwq1Q zpDAB?@1?-DrdIA+`^GMWg_Ay$^3lx0OE8Sp?-a8<^O{SoG4HX)xMSMuUjcbDEs4Q7 zc7E7eYYjf^4WKYLm~>k^g)*f`$GBoFN0=kVI!s`fG{V60PEn$MuG#0e)4F<0BdIWj zsj0O#TGS6?nSRF2u0Q75-VXy3HSc|{H=>AFU-|H}CrZoMyHYHLd9FCKXxmvU5@@DL z_p#z2a9v?;VDF^lGM8RZ%K=z)C+(b_cl_<&(Zgzn>4Ti2l0sTbS###d+A-6@Y@5sK zGVNrQJk}CP+xv)WWG&$s*1FT&saZO$6yvxDpxa6FI%g~ps5Me7&QH-Dan1mqm?l)o zO`%aVPIbXF(+uIv^A9>_D2uv7?8;8dglEh}!!`FzS=?EzVCS`;zk&?y%P|9HjkU+0 zcI5BfG4YyTfd%uZN4+C3*3 zkCWoY&K-g8Ztq#^uqN_T-K%hKrwHpacGC0=BMD*VNc%V!=F3lGCwB~y@3BI}*c=PH ze9TPcvi2zG8oN4wF5SbqMsQl%69qeO9fvqF?8n_B|26i~PL4ZmdM(u1kYa9nT#E{A z&dv6`*LqA$GgT!Ip~aDQ3d~=D-foX+y^?n1<=FFdCM}trwL)@U3xPItY)P!PlZ18~ z3-D?NRn|Y3I!&AF7D`O1YVJAGnAdL0URioJZye(`SGHT<^9pXwUGlZ}0OHK~Jaw+Y$&w~qdS44R zd(BPo7!!6{&tsN)&$-&Qr~L66%aUbHUD^~DF8`gGEhQ$HoxfKy+S#Gqd~V(1lG7IN zT5B_7%~689R_s_Gb8$G1h1s_gfMM86*KK4Ri$Bw1%N?urEaXwIl%^K%3v173EqUo3 zvuJG`A&Gr%f%>`jYS@@jMrNon~y+^St+*%HCryE@WY`F!rEUTH$+=72@vNoyinuFD%71=Ri;!DKu&? z&96B|u+&Jq5@F1F*g1pT_6^gkVa0{uKW8#X$$J2K8>9amx3uFPTOWMPNrpJ~uwLB5 ziePVrnzUACT*^6;BF8nJI+xhU8>u2|EQsJ42BcHlyGUe2$j30o+~41U&o0fCi#&7a za9BAcWK3z!7xqZj+_~6g#`*BDXR`2G%UL`wP3@eLv|3K<1bj`Q<+3Iy%8lC;W3NTk z8RIVHA3?uu26ctDXCg%l)2((4HP<%RO5Yso`)3Y;pq1C!(A}$nJ;&AP6P561a!kpa(>Hak*qpwS`1cOO6KJoMfiX6))0x90 zBgP4;wYFYP%JYL|?6tO?wmQfk^SXU3Rq&GHGWZzV#%Cfj^^=vuSj9>Ejb_AkkhIsV-JeOQI4yzRoP(C&4q5wu%oQxpk>Rv+`fJ-u zoj)v1y1ob4XkYU|cWpJJvr~ro8)M9W1exYC(sbaQ@nCr@0Y`PFiO7^QC~=PSLw!$K zkGD5i-d=O_ZKkp6Fbz@SxO1e#o+)cS}BFf)Wkvoq>khr3<*xdV?Y)-V4JNJgvoVmz3MFrvzb~^kX zd$DmWnY%vcm|_a??=_8i(Uf-n-=2&2CXdkaJl3fFPocjgr47cB*7VYx%YJ)>;m{Mt zFi6|e;wCM{oVJFz^%yfWaz+9Ew9*j7+-dhePtC}b76kemk@snhU9uFWxa8cJQ5pdw zGR0)rKSRjkj3EhfEQdG2&=o!uaj%i!kv5!X7+bzJWHFVPcKE{BbJQTs>BKNotXmGdW<9N4#X3g_ z`WrzvaLjd;69=BycI!r32@ek(2rx3;H#X&y^|eMe}Qn?`8=#2EK5 zr(VN|gDP_EtbsUZ{%=Msyc)+qbKd(VeC~D59QRP=7&9kwEj_0{N3!$Tc}jTB45zeL z)^Zp_Ep+VNoRX#pOj+BVB`zJgI_Dh3Ui;5uj=}Xh#{|?&NydK7?dLS&4#QlteJqD< z_z+`KU!GwLdk=-?vZfARp9$A-r%?Zy)7W*7eTB4Vbom)e@qPvMkg#Wja2#=pb&pw- zJhsTtUfWAD4ONS|cF=B{NtJy~8M2$E(#OtGBzZ;!<(bkH(H;X@H-zxlIYWj>p97$4 zY(?ofCJ@jVtE*s*$;mZW>Ok95XCy^^kQmm|a-8E9G%by;JVti%STlVwPeF`4QnuJg z(YXY5SkyB08D9xKFjOx=jL z_PWd7YprPKaO9pM8rDzYn`w?AkT?>=8 z<`JeH=Y&)U^ISHKK&3t79`3wGMY}hKgvO6^C2VcH%Q@z#O$#YbAk1mFk)!O&pIO6X zWFg8D7PRjjc}QW5fTy`uux4M|m}TXa-?G!ncTDqed#B0avtlGh+}jO!k4@z|C&Ygq zVe(;2Y@{?->R6xYWPc7+l$s`z@eM0faBg+cJ`z+)O4%56%-G2rSMX-cGxTlc@%%8> z1nS?Tq-QLF|24-#@ZEENIjl9Hwt@!0Gqw&xPsx{l#G$sfXR7NDo5L|CWP~>Nw98F# z5hX-zj}meO(;BG?d`@-EG^cv+jw1pf9tl5a?A7<34Z~V>?D@Yt(%eLhOKUjCfSo#G z?Egw5HD-;ytT{%8WSuGZFhtpnn^P868$0D{XV~sMbBuCMA&O>1wd+6gt7;9Iiodt; zTHTW=DGzYIltTL1%~LIQCitNnmcmd83rahz9pbV>X7SsDJa?@;dBw%zy%(-cUrWYs z%rJv7mXi8i8=8A;dCQdtPF#ywFl=TJs~>kD>C6kSZ3bNCAMy8h4UwBUc4)+(QG7X1 z5x28Ptax78*&@XqteWT2-P{pyJFoegl;^C#J4ev`UK4kGEd1m;JPOCwWZ8sxTwEd0K?M~Q)(X!Et2I5&U8=tqhH;*$J$1z(ia&M~){4WX zE@bS=J|^tjm}>}qq)3afqB!_i6Nn(C&7&Lw9C9A}$z%nc!nHdu_$(kv`7U@3;f} z`4DemSOKN2>$v7v7QW^6}bu1yF z7K7Y&Op~5wE!F8Y19<)$c|I;rZLgQduJKMQ$#<_Iu0F;l{SG;oBMw}O6n9#D30tLY zuBqfY#yaQCQ$Knxu(`Z9aN}941#ZQlggVytP8u8GV=a}*I~F8&PQzk)?U}hSBH~XP zcxE1pygduN~FC78>bT!!>cO1++Z&wA`5cZfRyY zuMzT0O55>2WA7-CkVm@FTMI>OrzPVZ_ZZ3?2~A*xY}Py%FyR|Pd~gpPnlf|FUsx+3 zW`-%;x0lpmpG#gZ4(X;br_|nGJKZLZ&DSN_`uDpLtw29*1c2a#oQzI zYKGauHnWuZnQ=%l%sJ{g*NVWK(eZswk>;A$RP~Ef(s=Bh)Sr@C=bag#ZRC06wt~WP z3Tu6DW&NVH!vOMK*-$laIkCP{a#fibeRj-c?=a)U*4ROaaLf>yFhc;w7{U8J1PqNc z$Dm?cYl?gBP^KJ)Y~-Fvq^Gr1`5&J z+dO**CA>c-meSoDyn4k!*D;fFL`uW>IxYn27ls^fohxZDN6h)NGSXTN8;~WBL8=`S zNJd!mYixy;+#Q$n-c57vb#C#iIY+$YUNa1G>~M~;LrlRKgBvAJHP}B##LQfSmUKk1 z$FVb_$KJc4Y(x#<7>9iG&a>`$Y@x0cL)`RAgZwT{ag>_2_Ql`h@@tM+*fD3$!VPYtf6c7d5>mF_8R@BKZ4IXz7Fzil+50wa3Dh!Y zB>WAj?{Y14{}5t6&Pa1=CQK3QFs2NBnSrrjhViz!HW+q`!x}dSAj3WP)wuo098GSMZ#ke@9Qdvzi?=nWgj5SuWMazTT zXzmf@ot7wcS?lO~hh>m87FgC>^Ry!efyxX`xG-jz_#P{ub4(?~Gg7E)Tg&b!4t>QD z!8$Tbs0 z$K0b6Y0S*a6Jxq!9n<@CFGR((bM*WQi2%&s2`4j8p{x}bM0Qx=;yvtr%$Rrb>z@Jl zVkPXLyaP&AiSal;F67uc^MY%cNq%uh1e6$8X2jlmf?%f|_L=6u`Wl_Zp>AQJr+LpwADT%kMW#?VZ8>TBgq@0g7X?ixYUk|`g+7g<{H<$$WGaFVy#@j zy7nI3UxOTZ@9C&G0i79hGW1+S9%!tD!8g{@{f*%`aBb}IIP$8~n#(zM4Vi}2N#y#VNYF>k&Hpd~{J*U*iAA6r6%ys!ZR%G*@DHd&}Ik^?M;a}Hm*+CtM7F$Xvn$)uR~wTqD91UBlNU&$`f;B z+8l`pHZ67Zy%xyAom+}%W|a26R|NFldq8&wA-q0@+H6}p4Rvg}<+9hb+TDYHVgx0G zKLUL44nx~_W&FcCVqVOhE8B91nEE!tgx=hXJ!#C~#+0{c-d}TSaxAU;J6DQI9SeJJ zXITF*SMc_kBj|A@jnKH0x_q5MUm$JW@fa4ue%;IRGR#$xypxXj%1hie&YZM9cc5vR z`zm*@pr)G^4C5IKx@66;)x2X&$KT@%Y{d<=xu@F4O2PanZB?4Im$2R08Om>r(STr# zC6u#wx^SL(>~aPr;uJS}*PP2_WbC}=xMBv*UO~NbMV+3rlf>T+vwUX{DZR25?#9>= zV|T5+%)avYz&yf^{tek|YVJ+IGItj1STkENg;C(O7nE99V+k?Eh1|7LSZdnqNpy`g z=M{Ix{~qyWZ7;owm=YjM-{U4>M6HmRw|abELx?uS82dc-lEBYt`)Y2%+_l%t{2IaU zcg=C4xp&&h4x0onZ)pCvwrbRggY9`|IK`h`G^;HhW1(;+G8(t!M!)&c-X6FCyYVhp2imM-@|lprv2Tp4HG=j zk6Vu-%=oJmHs<$TWA}CL;FXewu=L*nXJRJ>__HSPYS;#5XAh%IERJ}9bnO+}omSA~ zUaK}eO&F#+(qv2BlV@Nl$p zZX=GhtiG4#=NyT?e64Zvl@>_Gotw;OP9>$gR!Zy-S%Y9rE#f%C{%TG=n4L$^P+jBs zWY4{VHTKeA%Y$qxMX-qUF)>Ek~3zD$dGvT-b#`Z$M%LES;uY)wJaxJT;M z$wBmMhf(a5wu(m`F~57w2R*6VvN}}WzHq>y@yEISVJ*$jrf#2 z@-Wg~s|#nWfxa;JWZlc*g>_HS#2Ka<@d{CvWlmuwEqup6ws7r=;g%<5MC6&Lgy5UY zCU}i0>kQHHj&-xKl1|?p$x(JEMYp$8YE4=bk7GqivOm&d%MHnpZLC$C8{*d9n)#+8 ztnuNNhra091N32!VCO#)rd;1689oFQk}!6V=MRCYWkvO{xu-PQ*wJb%#dWN(BTUhZ z8CgDtV8u2E`s0to+;FU1)Vfv(#|pz?bMKXikrVLY-Mcn=hx~oay8vL#fw+6_(cLgc zaQ&a*2rOoh&N8>!;GI#(dJp`?nTr6#+WEn81mM#i0u*=K1G9Us8K|*#u-b|FacQkp z`kNv)#E#2QVJ*#zJQiI3A2W}BCRNLx2D1B@OIbXH#fUirLg}3IC3lT2_?LG2-kwq3 zat{=eK9<1mp4qi_ENQclhoVE=83}1*%(6L0X8GNb10YT<#u*bn(-<-QIt6gCKC@_S z+Cw-cEs49j_IAO@ag!s2V8b-#9R3ay6nf0S);-u$QjT+q5SKq=T3X$?_B5?jN0a|IZwwe~V!iDUmZ zHdN5vTPJW&9f`UY*w5J$nsbDiyfRV>{0ho`U(?-XPdSM>=cL+9OHX5u7}UHcwDpUF z@qg#=_uk{MJMYEnxfg)*1}b9PJA!F1+4MHk7Sx?HnqlwUhP@XE!rm+LV(cZEd|>e zrX1dy@vLI*eXJ7q#Aupx#Bro~(Un$0Zl4(iKgUtUvja5KpHt9bZ>_PpB*n3!xXakf z$R>=C&KoE2^4VJ&bZ_wVHX<5o4=Hpu%!Th4_LOB@k$iZr(YBk1gyfz3k9uXK)VfpP z!{6CdGzLxbzB9V_S!sZ8OwHrDH(J_W%P(yW#DNgT(CJ+%{&6n}$dlIsM+pI^G;Pt6 zmogwgHq(a9UO^c*XWhcR!X9@E8zm-&O@pzQ5b~Jod1P#v?6J3g;~Wd*Ck^1>G^ciY zn?uKUtlguzW?01=69DL3YF3H?-+nyRKx+f$OlBAoy7M+9j`mppkbBTbWa&dXFHFk-`$fn!`tLMN!AGml}K- zDM4{<5tSTApy^pFL21Tyuf9jnP}%u{H7&*OK4b20oO9kW4V3P@b4KqT`vzpj<%B<% z_WmFF+Z7hx(H?7!Yy+M=uXv*~viSE31N1eH5S1{dPWW1DePC?epA@1t+nwu(Gv*bX zkYn~o2ouaLkL>cd_PF%kksC4&{h=}@q}3hs5p6Efgp_Aa&l!>h zs~k4xu(Lnbp8uM=#y{sy*PHn{dWT`YG!nYsoSVQg?j5Z(CU|rkE4X_GRL+(+@XVP@ zFKzB|n3*?z@>|*5YUJggkT-l%-Lbhj22H>mCs6uN`H;A`Fn3uo41X=9pSTzH@Et)y zf6lbwJ0`I74yo^X#BubPcLe4R;R1hz*@iJzeACOzCu2_~n7Sjt*-c4~HpCf_nbs`w z%$p5sXNdfg=PK%ny9j1%m5RT|YHgc=_ht`0r<3OfR!qSC&x$>tfB&`#bXM{o~# z*SWV6-Com7Y;3`-Ju@P2ntLdEPTAiuXN<_1!?9ze4YQUHl#r8Khh*+L!y(4Am&oZ-u?+N+)HSZmVz9&le9iyXU zL}-Mu)?n+{Np3sOwc|0T0`wSDMsJKcwwz}q=^En>C+=9@vBu_Y3tOdj43K#2XwtLh z)ZUKUqGYX+`8|iI(c1$6L0uW1IBZGYuvW~{TKfuQtR>w%SIT`^Lpyy93GKAUd`H~l zU^OpwtTS^sz??e(+sZq;ZZDbQkp?P4Td~b^q;>Ex1~=eOdk)2EAfUB&z|D;5W_^sApP815)R{XfY^>?_xq^)RolCPb&54yc(>_@rlMyCN zMZ~gKI%3b;HYLYZx}6r5$6GTWV~u6;v%<9eNcrAmjP!*c$Na$xDdu}7y~#boYS`bI zICd=Qik9|L^Vwryf2Q%Au&3s9nhTh7u4TErd zwuHjj!(=U{71};`z|`BL`73Oh={KedT3-Xvcdy;|F*eA|3ZsWQ=3(8tw$$;PVKgK! z70ou6u*_ZaK00n)r!*%bY+pecZ|ok7P@wc_-{OAn%F+Q$PpA`lY(whTf zY0p6C8D`u}oRd#{Z+Z7VwlM0N17I>PZK0haY{S_r&ueYX|C`HvWNgjIJN77i+oLKu zhUBBSh89U$!{aqA9L2Er0_a;oFk=omg|-qb^x8u;XO7wX5<>>aiEAHj#LTxFX24rb z8|QfDsJ=U=qU;Y*Tw_Oki?eoQSf3+>Aw{&qIu`apPP12PZ-t`}*G^ZNi;Q9C0olJd z7Qr1mKyVGs=s32B=w8Ehc1~E~zT(*E$s-|TtWf8;hqm<_iFR?OS;diZ`r=E|QFiUk zz#GO&+ez80buHMWx>oe^NP7%xuYivbhB)2dsorOX<&8MTqUsrOpC`vf|2Jbe`ip6e zH%=Am8|MDs+Do^0?P=FOX1GEa^Vnf+J^K`6h~P;}v2sk9%&^CPa9@i;bS-`T69<~z z-La8(#yQ`<(qi)%W3FQ-?Cdl5>|$Oqr(>oa=rV_9USFF`a3!g-I`$sg%A-|x%(1jR z7j*5KJ9v05{HDAH@bcRmi(!SmtGWhU_}`fpV9srlx|fdd+WCWcE%}7B$A;?)3*~(6 z3AVV_1nmi9cXo_D)0K7-@5qz;Y(xpx6o&3^9HAREtYns)2a?!N2{kdTZInAE^7~55 z>0wSS)xVYs;N9CtdMq)QpYdOROg)ykW8l+STZcaEosK*<6x_~ng?UdU?>Sc*N*;@8 zHKsVyxt3gKo4W=-%{kjQQ}*UwX-{RQY~Z;9rsm4a)q9Mwt}q6K_P&vo9OURe5-exl zTQenW)a9`DjBr`m5@n^dqA(9w_`G6tUyI{fbcCIoGjeLk4J)^4?O=}{;|^I}`G98z zEr4*(HMY1{z}SeRA#!d_(LNJC=Uq8FcxH{4wwL($4ckCwZJmjk1N8OD(b=hf%BCQeIUahYlkUF?@BGY9JL_-_=!?Gh5c!#-KygjXr!ZrPz@7s-IEIb!vc?|&7$F~N zPrbt%!qoO(qboSAte&67W?=?Qql`U;W!1&@iqo9XWBbib?rU$8P{OS z-m^|=O?;v?*D%|k>j-`?Ws^Rp@>5zV?R{(+-8)t;;0a-weg*-wF_)&+PaCar$2sV? z2N3oiqqI0rIoUKvZpO^Zo-k}^m>;I_{hfQZa4tR9vBsYA8hO)U47t_17EbeCOV@U% zm9e%LrheUe1SpE;o*O-a0(HfG;jTlFsPO_aJ*SpUzwshmSTLL8YI zGY=i8HNu+w+1nL(?8VAH_r!G_i&tpPwUv?vIO`sx#v>=upSFg!QO$Emac%(alm=Gl z-qR0%j@{t9mxA0H0n{R98S@;sQqImPQGG0Qh!g^jOBkage=Hp2zvtG+O(XYruGy71 z0`B2iY0YDgd9%2-SmK)78BggsbLSPl8rDF}S?dXSZ9V8ew;br(5f3oz;JUKbNJpBZ z8guML#ELTGT+6y=F7(WM{cg_4+rJl@ z_n$*3dCaY~kcQ&mnscExO&!-Uh6MYF>-BxkNvtrl>io!)Pi>?To4hmF)|<07VQfvc z6?b69*qhQcu1TdZvcS+=;eC89`OUc&TGNZ031m&(gtvzh$=rE_EX{S&Kc=+7Osh0> z4eg|p*N}BfD+GP75i)Ld$vXow$r;l`CH^4iGSV`nNaXwA1LaKYQ#5IjWy!P3%l5H*I-I||K3HlPFzSj?#w>_^BmKH|_ z^NsU8at`s2z9zDCoSC3tj(LMR=Sb5|6Xk8}VcER*qHh=@9d*p0xISY5$ew%hCuS+3 zv-f6E8_6(Z%)QVv#)REk;}dahEw8sTzSx_4o;~bU$S@8V@Ux>R@z(l zcdRL?6qABRotwpN1!3GY*W~O<>!mEu!Mrinq~IU1c6|=z&cC<(dRTiiCdD<5lUDZS z%d6`*Z_TwB_AK5WF$6IWsiibWPI3z)oN8{B#XGi)=wCA1H8-0^QQC*8HO zrj%|EtL7_C>B$#Da^XuEx;BkT+q9-!=3OfXZ)PR_KBtV!TRA{Ah1vcc*J#&DQv+=b z8M(63Q1zTkFeD_g!=9pk$qQ3xV&wd$HJ7gE9kc!=Y@MM#S6a%Lfe3dd45Jj%jO>~# zb9Ch?*~4KXMqr&E)_~&~AjL@Ru`Zr0$vdbZ4c_`Ma0$YMH|tapdiczc&Ed+=B{jBpA=Xh8W44 zIma_^wZ=PA7Qi~!Zr&c-4lu9jo3qwh)ZIy4VFWm_Iw$UF*@FOJ+MCIDEDeDS{qr{$ zf=o#}4|VU^#X1t+TONDTb8HNf5Qaw3-l^C*u2`HtmMT~sBfE1A`T4)Ylzv%}n0Bn4 zsG7G@`5fDdVy2zgxAW%XUO6r}k7ePwa=P-|k)(f24br!_>R{fxKXMOE)4i8Q{T%r6 zjWdrR%>=%;refEOv!Y@I(3|n0Ztq!|m!>Mvn7c=Hq*Z`- z1wE&+RwCBO>whOkalNyGCde5X+A@xDhqNb*UEL|CYi>oTm4=o?&SUv3O#Qqw)};B_ zgTf)loYTElXu=q2tu=>9^EwAkXBoo+de1PenvFM4ZJjX9C6^!;fer(7iEo z*4Eo&!epk6;2dH|K-^16eovi=y?3_T-mB;+j}81ZHxo38b*L9^Knplp6yU zc{lQnD2XqPzSe(D^GLY-?QXJ$c!HS##+T3h*Q1?{-I zGUDwU!?9#69qSf$9ATexRdx-f^Eg(@*G>r=F|Nh)GiHLrm`mwyt;E!}hmiDL^Y?m8 zWtX-W7JS@MEp~5BrW%(1{9eQ4a!j>>K7%Y-O8a|%E`ggq(`3-t2YT$zli+u)HSCn; z>mTbqFAp5ho|hWvo>Mw|O}U1>Cw4{~%U^ECUD~^nc2&r$m}kdfkP%0OMF~5X zd9S6QIY)xY&-mexreM`eVX0-VUA~oB>#+j_OAY%`Z-)TK6E}R^+ldx1jTwxTmVD4! z+2>`C2$c~g8eW>|4RS1`<1;sc-ic$XW(6&zGdDcrAFFdX4Yc`|S1QCAX|I3hLh#*_ z&}Ifzsv5E`!ix*%cBk#x8CD|G%rRyqjoiUI(`?+`W1l^2t^b@EXmKaSyfUL=+*oOz zGX(^mJa*(p+EWQ^FYTNeRu)|w$?Ik*LwT{D+wZGGLbv%Y>DgK#d)&A}aG zrfyx!R&x*h*<8!dA%tzMgrUi?Cs4~uTUKc9G~T&GJW8DD`yfs=o49AXRSo-Ubge0y zF((@2o1^%21vQBoav<^xQwKgwaLd0&tYll$HGb_0rnk3*=8Yk{azugsGiPAz-z%FX zOg!Qmmju$^S(k0+^_(>qaCTd>%V2>ZowmoO~Oqk;kG)}O`5oe^(38T_6FIllZ^Zshh0ofh42=rRX zz;~sf_cX^WJBLPiOR;)4%^lq}H)>A|=@UK7LB%w(4pvCRS$a(fv=X*f#2oW)c+Cmn zKT<60ncF8}r!2TJreM@t3-dKb$>JV{YQdPh34Ttrp|hhL)Y-X(c?BWBy93H0)?c&A8)F^3-aO2dz0XOPMg^Q3S~3z&P&ea^l| ztk22IDmrfnx3o6;&ECWFapvHGJZ5m!nF)7iN72)<_E2csL(OI{xv0Iyi0htnMP%g2 zuDI4l+ud6~VU9GYGk3JqS{oN@r{I-67dTXyS^qFHIOz)G7AMajx;iqHT-mv$ckY?> zytm}nnc=Z*D*HTXs*G{zcyrT9t)g&?XBuJ=hV&$Yp-Y|0h)be z+2S-e(Bt2u)Md|Xt~2)pjpN?&b2GcW81PSqXebrJcXGOyVBf zEo9GajUI!J=!%a!Qd-W@S0dL+2@ zwPzgsAKSZV4?(TJcTQOx3rT(DIkUJ^oMBkI;bZ4*(x0Mu@(;s-U`0iimM0|To6*W+ zFAb``A`0;wGs$#};q$wPPU+7JrZ`W5hBotNM_gmyEKkANJC=s~%-OvstZ|IISLoo~ zEBbV-Siv_ZXz7ltZ+B1hoHDmw;9S!TZzVXSwFZ#ROe3~?gxQvmH`rPnQMh!k;jXvm zGUShY;A$`Egp~&r`U(-fdaO5h3#iH0^qD*uU7ziv;3 zml2jY<&O!@B81h<6Ng;SNK?6f@0|6pBgXfe88|y-Ahx;I(*4d(7JJT0M&HNz_OUn{d<=KRAl z*E(d%(+qhfRm2hU+`&mXjd%u4GX!8~?}@~e))?`aV_0S9Esiti0R0>BnP?4-wiyRL z&`;AJB+O;pJ7XZs8`HNiEjjHRqe9afOBZFN37$UZe%&5xd@2O-*)XQ2WDIN1ajh}v zKL%*m9E*!D<*D*A=QPNT>9Hin(5N(07~;uunL8{AvofdP(a&+hVhIv+?{vJrLlWGW zVMTw0RpGq{s$SgF*KbC=@j5f^>KbEpZR`=_o0FLT3=3R-k1&-Nm##}5$(kXJu#~z& zB1YZ&3}oz)h?hfb*BP<(Bn>s}KZ26_TGLKuhoRg*mQ>zc^Gk3~DNkA6aSgG+xaS7& z%{lCLY?!v2gCs-Pfk$!djmEaO*4E2W%5+9Oh&@wI?b^dOW(~QvJC`azn(-Mv4?v-^ z2WH`%ksNKVEvLJd?9Unl!!@lX!!r^f%FgLBZEgw0Iwk?Rop#h#OM$Ot?3JTEw%Wm2 zi)(N$nXDJ~K>1u*lywf3)j#K?L)}YSDeUO1Ft!GA-EpgTW{iqG1`ghygUn#8$*epE z*2r2Rif(Qdfw`xQWFFxKXHMMJHn*te7(0z|M5)C-k{%uApvFzxSAHe2-WWE7Y@R#7 zZtf`TGZWOojcaRr&XJ*nrnhKR%sp_OmXWt}&pc>c({t#PfWwv{%tT9`?;W+g$?xpMY*n~{oP#tehL zmW1G1D=2TTJ@@5pnU{0)TpFo*e@%g}z7~k!%wu>h4duzY7sN(gdp$LW zS=6v{2+@cJ4BgogwQ5dvuD*A+a7kI4ccqx)xb|3n9a)%V?y2T96qq!Je$QNEt7R>; z?2&egUzk%MWaYh?K6j+h+e36Nk2rvC>=Df~leFgzixq5z*^r)A&f6Gc`(frG(z!?I z%}R-WcxLe4u;Og-%VPv24QD;RzZ%z=?J=WsCqonyvdrG+&gWdrWn>3=-FYZGcF~{m=aLl-Wxz_E}`QzX9&g} ztMNWhwez>v%4!;;lrU|9xENF9;|Qy|cg_Kou(yP9on!QMjVbLq;%>xR@ggfMFpw0N z9@@y!i!zTCu(iT)*H1IdehpZXy~dvGTSL`7FLAiN#z=76^T2LSaqc(Q1WsK$zyRg|YXN;9g4%Wsh*PJm)y=h~ZNrM)B*rH?GV|@rhwa0l+%v zfMA*tJa{jK+_pykTi=1dePn&XQw#?BCO1f3D@5 z8eoQsv1Vr)=CRvb4Gy5lNLOB7|Z-2 zCaLnf_d;%6NfK@ieYr6L^wrIiq-zf#>M;k@>KxmAC+|7uG}ln~UlVUPrCo}$H?r@X zlR9#Tl+?Xf!a|r)L~<@Lueay4+gs~&X>4rKx>H#EP6=6SMJdi6m#pmGvt2CbEaR}} zD#!`T);GqLhcH4`*qrgOe}@dZI`Vj1TUoGnkMWAKS1i}wyN!OXEt56oSjh`ZzJG3M z**1m}Q(4I=eNCy+Fn8wsO6wnXjJ3D8^J-@c^CD@6S+z6rI?9@xzBccv)EVN0$Qwz3 zZe?`EmIfB^T!9aI?C_vJH{wUxnR6=Tov6LDK+N1DMLi6Gs2m3paUhXdBr- ze6P9PpOPqIk9)v$PpFlir{-AOV{m=W!Kaa@kWS89o?*l_{Ih5NZDydQG-i-z%^8tn zu8Gt!<4E$IQ%G{J)rgelKJW-rFK`Atzcs@s;he#nA*3Lk7dF)Pm{D4ECw28Q(z4^+ zi|jAWv8}XI9LpISvTclck`bdgUEPt;e9oQUF#_VBvoc|gu_9r{b(XNFT=!Yq=QgLf z;j_kS+un2der-kanHL;)ngi1-#i96;(&|lGs}v`$)u|p5ApPIN9%j#hr#}+{{$JyM zG6o%%mN#x~Tcd~~%X&-u$Z;zZ@jL)0v86@fIiT3y(IL2_?V|Bch7a3*A?6?PQC918?+hbYCd zx2)Ek3tf6e?Z`ET+{X?(O*0Hk;W+ag?OaQiZEbLzHiibwO*1)X1Qo_IW**O9Nr`z4 zx%a$waAFyA4}8wy#}VUF$k^FqVy}Vby2c{W3IpwEX250U81uNMq+=Tka&*lU|Crm+ zeov6FJTvsk+Y5ejCH;#!wlek}IbLn;0fxKBF38;Tc6v=goST*o?%xxFY;DEhwC3=_ zjC*)=?qq?pmyW=Wd+IAkNR=8!s7%dUR(emc%e`Z^!;0f2dX2QCHD<2zT{)g%Ziv;N z*MMkQt5H3tID#>Tx?voFojL4vqqSCM`5wa&ARP(PI|odkx6(*N8yPQa&Jq1TlBm-T zVFPgu!TdCH@a9efylM_fgSBRQ?1-5LJdTL$7x#wYO1mpG4Pd+-*JyTM*&HjbE&dZ0 z9^%Z~CpQnsvOec(&>1s0I!(NiwWcD~Ni%UhF4*5NGEUYSfutu-ZM?CgVnEEBK6k8z z-aqD)(M;*aeQ!XLKbAoD+rj;CuQ1OvGKljHp{RHzwY799h zVD0grFgEP&T2pdrPr=0#_c-2OE1Pw0O^=jA0Oc6lDRBj9+Ore>Y))~GvR8Ik-H}Bn zNBl$@OBi8n;f_7hr1Zkv^wE0?D zbZ4Xq-?$>MLEN$bGUi(3nLGJ*kJ+e~b4qI(+xH+Gk#lkm(eV)ys^X97jBG`9wx4jr zzt>9DS_}ItMF|DJ8>RuXGp6Q7U#lT()61g^e+>=fzZN#rn8Tzfu2iG379z(S z`*|&GMcXoFSm>OKZ#ivsgfvImK^$vwey1J5KBpA>7^7)qj7Ywl79RPY;nFG%1+t!J zpz|8DjeU&4uDge_cpBlbaZJs~9OgXqT^l!wkq;t6FlV(W4aC=-eLjd~>D!!#OjC%AdGvZ8RI8wEue-s2mWEqA;>iLXyRJC?FAXt)x`QzA-h)bI z(h)|A8kdL|X94i(YT4Pdt#@zy|IE<2Fz1d%U$Kj1Ohtz7R5>iQ>I}l>-MTisQM8yhw)+O!Prnt8vaUTO{CN8<^F;}?CU-PgwZB2(5rjTI| z;r(=t@%R^)5Yd@K>@W?mffC~0Y8pY7W=`qgz9-&56LX^M9YgkGW&yH4hTw6^A+v490jv^|8u%Fzd0{6F_OYf+)Q>Zge+-13 zx^{N&*}KPWuBDT>GHlz<3#)HL8H2fYa!8F^m1=H8;khR~cFAe`euUJEIffM38}qU& zZGnlhhr->M^CV^k9r?Bb6j9k>5PKzw>Xyes!B}fEIIfA^yaqzZn-iTity$_b;^OiQ z@d9XsSm+(+;?G<8w{hj!l{qKMU5XL1c+9<(x+Ypt9TTl$k3{7&<|I(stNUgT#pN9b z@W~lVEq-Mvh_nM5+TQ!zG!NDH7GkpIoNJvo%#r9gw&-=Af!aLn;h>pza7W(JU^WI& zz7z8H#$7>4W{hptlm=4wPLtej?h&IIC%VjudxC_U8^xN z_~MK!ae5Do)EMWUOV0aba*Uw9x7Gx1O$iJsjM2}ulk`^)=^JlOk&dy42us~d!fg*g z>KN8AYMO(QC5Ne-vv$;AjN2|O2eGcV*YrlpBOqmNxyljeyx|W?P+-q6?>JX*%-!=O zBhKy9J~mv&o74C*Y{B@wrclL5`@3|F#lCy>c4Wo-3&&g^jhh zmOMqD1Nvb8X;puXh4wObJXjo2 z>ufIh<*{}WeVBvBbLEBS8>VRGPiZQ2?Ww<(*TQ|DJ7#k)!MnWoIQvgKXJgOdk-xTN z``TlkcTA<>zqk5Gn@c7qWc{Hq!k9dV*uWoWCTWXdscMgu<2lmg;MoJs ze9xVtyO(~`0M2G z7=xgG2Z7VR=Cbmg^H^Z)E#y1KbkvNgzhrF8o|}V6`OmXRXRU?2y#uEE8|zVS%sr^G z^7`4@OF(2U`MR-E>fql4-eTrek(xq~$60H$KQHZ@y!U$5o1yL|ZP?Zq=6=}B8P8=7 z-OoAJa(7J&9X-y$wHC*$NZP@^HRhSmuxBd7+M7o$Pu-6f6JYsYSw(iI@%gk@1k+y2 z=YIs0t(m5NQQk`^ZtfA|uy@kdnscvrkBGlHx9Y}@o6vsFrM5X2ioqRAw|uQA#h)gu z;?D63XO5}lwWdbp$-`Y|Ph`wK@`ml*`pS zwkO1X8WYez=8#cbyR~8sZI{2Zmg$?*U1AKSoiS53&> zbE15*=Ao^!_N461JIgQ)kdzzd{7o49g8^(ilpW{LJ+B+Lg7#X|?sCK#t-qEq z=G^({E)KB3K1X`SnL{OZEiv1(w#a82Sv_fGpnx@GppLxun0;S~iz4lOlDs2)aGaxc zDD0uZILCr?&I{&m&F~;)C9txRRCpYlXK<{rvb2V@*IJolYKAqryB2c%S%HCKZtTb~ zBj|q|IW8jaS>7_n1jgCRNNJ65fwQ+9?%R{?Zq5S zE#5UFK<=H>je2BxxVb0D?wa!{b+6sMmY0O|3nStxt@*|>mP*xJo7!pYY1q40a?YOX zv~rKkgScnJ&fJj-I*fIMw8oD4+dG77E;+g$2JUdlfow5E^p&@E5PQn=gM6nMD&9iG%ZDiG-sOWSz}k5XHSL**JhOm*$FQX?LSM>T9%tmJ%sJ*l z?HbWPW3Dyjv8O&^nt|(X&-thy2Jqk?JN!Hhu=zK4yjj|#3Uw`vjh}X=zA<^sF;dGS`}6Pt=YE`Csf&>=0BtI^TD> z%Ur2sBG2`tzc;eWN;{NxtaZ+@CcyDd6KQ_r6@t8SXyggoj&tPw=RG2_^~hs}VUIM_ zmoix73nQ_2?Zw6$hd%LHlejHzdAOb?Eb|{xr(-2(uo*}2(HrX`X)S1^vvK(g5d5%HTyrxX(T01{^4uGgUW>QKVQ!p{68LmC@ zAbA`k=yFEQ#J?6o&|X_6VeGy2Ta$12|F5DbQUcN?Ez*omQ3R1vP#R>!$c-M2FS>I` zjgS^;2BSebN5>e==!Qv2_}Tj(cz^$Z@B9AwI(BT=vE#n)>pairaqhOc<0zpw_(SlA@bkEF7N!6@|fasr{X3 zU92s>J5sYHIz2a3>oNRdMDoq9e~v>Am^1MXAQ^D=m5&do@#C)?mH8$*7ww@@cZK9> z+giMS3lZ|izXJX`$ZqbnkH0a_AyQB*SITJWDq1IFOZ5WZPiH}<_>a1Ax*VSLixc`7 zk@}{->Shbfdb1ud>@=<%1PFU323k3;I9b*TNs9E>_?$h=tDCTisI6}&$q zwdp>zG|=v(LfR@r1JYS?tM_mMPfNN^T%rT7eJg)2w|L^GZ;4dB9KCq*)i)HG}q7pthvqgQE}p~HRi=D!`wNJ0C}@likpNaoVlR5a*^ ze0a%8I;D|o?#mtH3MBqW;i5RaRI|v7^GTMtKFy$+i<*-FX)i;bm%-v{Nx;^lLrI=t zKeOmw?gE$j>hPO6){Y0(s?nh-K?RS}sdBoNO(`08IF{V@wmSE+ksz(duseSI9 zsKy!ggTgei6des?HZ@VfEM&8ahNXsGjd<+K*#w<_gMPrh^Xx{1Ah*Ct6}QxNMy@^} z^vlnB%QsktNpYVzih)Mek@sr306(z*Zi3#-WX5WR3n^Esba=*@8b>tO(dz=l3>1q} zb8sL$K_hFV4o0wUcEhnnRBsICj3&hbHarl3q)iYCacg5EeN08Tj^_pb#zQ&EB)a@A z3>^$Wp~`i_I)nW=#3u3;12v2@EFB~z&n-xES6iVFAkkZ(xS$6j%S~rY)9uQ$?5cyB z$7kKdZJBf3??;oqluFiQ>MJd}K56db10(N33#ZR*&-?stSW26`9O+l;d+COjjeY!i zn9F3gGB%d3*8PsgcPLVym}E$!Xd`slXTpH6I(e@o2y@hmV1QBIMeQE?EAABd+hL)vxBBay$DEd z%Pg|@8(B$I8uZ79C0!N8g8o_>-1EaMw`>05N9jp6{K_oXhOXoWcWruuHy@f&=0#rk zf9|Q0iSD`2;m5(+Br-4wTpBW(u$G0iz-B6`DuUnjChD&ls{^ilA2%*3ZlRADo4I_+ zA2!wYxqn`;+y+&G_rzQm{*6S&RP!Wn8x73z928Ww)4rdg#0L z@_TGJ*7CT&=ZUeFVrA`Cb#dH{i379szH^j_UYowD!(Y!W+oPJ)48wHB?`*N_h}82R zQ!I_e=CPX1-|LSZM@7Q@NBz4RkyhUrvF`Se*fj=_!kj^(HxPVw*G}8Hr>#q8BZNAY zC6qWZBiL|_jVwkYJJX>Y?dE0ZRCVLpAe z$5V)F6i>uf%Cf6>D=MoNk$4SR#+uu_<7$SznXy7o5`zW37iHc)0}m6cnX~edTl8$~ zx6<6tV#Oxw8%+kKjylrrcc za8V6lBXG6ZYU6A$;6M%*>ZzkZ>fJE8op~LKmF4u3T%&m#Tn@z2GgVCX(0et`sE$ra z{eGP|dVDO~v_vMDWQ7HH#iQZbb<@>Yl{GW?NVG*NTD` zjZ@P4+&y23o4pQ8R%>P@-l+M~fwgl>=<|I(o0JrA3Xl{@!+P%)i2^)@THS`GY?fnJ^Y*~SshXKe66-{Y>8P3;uk57ds9&+m{cvH}x z8U`|S00F~E6R7I7&|gn*vZP~`iQ!`>20#U8?XnOCz291>zrjD(_5yQZWjVo@elZ5Ns zc%QYeBTf)SaUgrXJN?Ar*rULIIsxCNwnya=B9uB^2Jk8l@NcYCP=&QxGsw@XWTFvCxXKg_v>JJK-2@QZNBi=* zX&VgZ%)Hsr)V&3c?u^25hexIz!~XGUMRIFpJsH;kmyxp&Ji&G?`kH}5$b3|$Q}|jv zzf3@xzM%x+Hh9LYk8U2HEds5|)zk!FhN+Rh9UulOEap4UEh9T~FZl-T0MiU#eX$$dvS ztvXJsB*%0}L>&CcyatiTnw9LyV^cZqV+BWrk;lnm6Dt;??v>Bu~z_p;&Kf-U)p z$Tl0gXcaF)6s}#eyEBsS+M!}0SuvFkFBRf=LBy8z;z{B$_$ahgp2oP4%lre?iTw7Vd`-qam20lSi!_sM85Uu;ef@JzWiRwqO{#GG|tDvo1J> z(@4bh7$=6ZJq+Bcp~3jF?lf5QN3ON@uFnjVPWT*&#~`}=4j2NFVC(8G>6DZkrfNqg zuEcW56#Ea1SLXs68^2{}HO5RVS@><9Pqn4={>3Oszba3>Xr&=sG9rkRdFXXorE~ga z!Qnq3XX^@*_6{7(Y_H|=PbLU*YaC2(Bk%~%Oh4zliv}3@aCkb6m}oY+Gby`CajyaBax=YShQNWI|M*rjyKJi7RHiS>^%v&<9+# z5Ndp=S^_#?;1I{fU$QHWJTy0(Q@2+s`y*YSv^!{gBEtQov&NGFJu+14JA%TAsOT)R zFz(osZIoxya<0KA&t;pGx!{>oBnR|F zoi3!`jaN-Q7!_eav8ZV6qif^7hxfyLV0TV3J&uugxT`^t^j-@@^6rg)>bZj!mt-*R3Kmr1f zUuF^1uE)DsvxA--3hO=8!jHoMoy>0(-+r7nN-8B`PdkW>dt zZ1S7*GY(pW(03{(m01nzDuByFm&H9_VKfN(WVZZPB$LFB-xE^a8R01kHu4M2$b~M` zp#@9w^08+RGt(MeCA!r1fG%ZECKAGE7PE#_e684U)#qb>K8zPCq8^=#iB4EAR^yT5 zymun`7V}?a^}q3#*Rl3yeq10hvfW_bO*{SAk<-O3rTj8e*0a~XAHHB2OPboqZL&8{ z(J8QhzcnCkvAfaj{Duw~E>ivp@m$plgDE9)=qtHw$JzL6S|zj~CcK%j#vGcD4r)Bx zy5Q`~qBzpl%rzII- zCWd|!EQx2`va(rVWVh<}=+4T2aK4B|`IC27Q{;{9=M3p{9CQRT;_mt8Uk0)z zm4lH_(az*6Im{d@YQv2Yjw3Icn>K%Up%dBh=-X|iUc?&`mt0x0#7H@wJHAnk$mSHS z!Qs+O>+;p^#y1h3jES#o_RCl0&jn3TN}9gIyz3&xKX2&x%B>E+l#1$wU1~S}b>~+K z%S6x(WL??5Jc=D_(y#GAH=}K07A{he*%!u>1VSSby%{tbE?V#(kl4_Du^Ofh1~(VB zSc)GqGc7$ilZ`JczZ}6~Mtu>MkrqeRFtN?-Zo1$>$;y_6WXUHt2JlfU^3Row`BB?U zrI#{VCaa>7q8DlOG(MFsIWSte&NQrq6zLb3cwXoHQ*REoL;U2+j-#I z=X;Hg|Gj#80)>87#M~HV>*&I9~M)`IecNRZmLww&`RLWCv&`&!+|bFwy8hI69^OZ78TlM{UHUh6q|QSbbMOwq&tTVy^xL}H5*$e z_k#MQj0ufkbqCU&Lb;+3%W?Uu0ffhUqWWVSOD8>`7(+PAmc`1VgQf`i`cQSa(E=GC z#-h@CHRJStJvHrL^w5r&8zI-Qd{;LXHLvF%-Vl+}cLN=4b_HpO7g1W|Ee!b1b7`(( zval*^M%cIZVRe5IR1GtvZn>cFtE84$77fg}ja8rUe#T?|rNu6woB+u>{o3yG`vC=> z;?-d(Jmr?za2E)Evesr(>o(W;$L?^hk4guRmN9JdF7Rl9ZEC!G+8FHNIDR2X*lV;@ zt!u$mUMKfBWYa)4ORiVAO*}ln1YN?#Tl8fIU|D3mOi<%QzOz(Yzi?pcmgFn+X}z9@ z(}uL!&qAPafM)jQ)|CFlM*y59ycY78lwi@bjIBANZ^@OEj})(`C>j4KAj@Xko9%I&99y?#}pi53Rwoh{6XD-@j_rWP~&8~xtX z#Qpwve`6(g#rsl;6q>O(Lq6HjpY4s|>-hI2GP}N>VBrhO>*2{QkS#B^Ltl}S$ z_ZKcl;raRpy}k5rXYJxWf|`R_utv>Aw&2?Z3S_`6Y4w#Gnc)yEG{Q1l*IxWcWIEIH zwIAJy9T?I&ic}GPJJ0die&k(RBh>GUQFX&Nt*prw9-k4qpLlUJIG4whW4f*BBW9oV;&tXn=B$gP>En9nAD|**41p=oQ>FS*X`* z*o&uHuqB)hr>2tW9EFoO#I~G1vnkl5YupeWWkaN#{|?Bujm@;m8g~y-uS1cUa;k;jFP?vU}N1ag%=k_1B>_$486B z)%A`OS;mK)k7~mtbRZYYT@3)P^fGF0{#C|A1IOOaVU64${$*47eY3h+$|x$r4!Rhy zk#Zd-O{9EbvP={?@QP~cB>aYQ@07Bf+IWAWfwzI;vSIsIq}WFs>j#5UeNzr}ny8d^ z9_52M-uM90kad9MD|^GPXW*lPkro>kl(-&`CwU7#<}0m-%fDh-9Ph8Nh0n0kIumBVH<^FQJ?DR^S!tMKr4(+97>! zN4{*kMrO-MH>VWp;KW}kkC3}ID_yn_kgq{(QT}C}udw4WY!*MY@1g_5c=yPsNQNgt z7!Y@s?+mlxv}>yXUwzCDu5_%5IgBxm{=>LAEo{QXXPfi zjuj81^I@7(7U3tTk|MW_dp7$3OKxcFa`Y(a843Ehj_}WvA;Gby_SqB_Kr~<;vE?&7 z2W#q6>CaYUuz&2nkX0Xbybzh2zixsk=(dskZULeKrieo0m5g{9Kh zfYJ041OnN+2)_nyyzH;DY&*{(z5{}@(#2^Zcp&Lpu7aLSsTn~ax$!giQSS_CLi-uR zl6%f9^Dz$bT)nMGsZm2qB}tT2b92}Pif$7B47XlEc*^-LS9?nr^x$E0>wnO|J+^ne zCsJ7cNnjkW-v8{@)ej^3b!xIePWKIPZ90RsTkDrh=X4eAE2hUpH$Q*P07QwBJ^6`b|7(vn#EFH+uC;P=b}#Ws@y4bjk!Dg(mZ-r2$#t}dGJS$b-iuN40Z*aIvw_Sl&cP231om;_r(#8a55x5| zKCHG{@4tX4PJ|5@sONL=EDTW-%Iw9$2BkDf@?>Z~4{8(F_&EwxNTf)e4Ma(vM-J@JQ)_3vNtX%xMEZdnMEp;4>YF|j#&S3C$ug4T z;Rg++d{?3J{u;tEd^tMF#c^jdmHlv?Br+%vb>4#l*iu0R4@1bp+wHg z7s#Am@21J%`VW;_H;JfAL z^^tjPSI+VDn!dFOIESn?d26Gb4f2|f?OcNVHa4?Tm{9q7Eb9{*yTz|E|yjlz`mM*F4w`$dMHXYSp#Z zo=V(i5|3I5W#+((^$r$G^+m~ALCnLqq||JhQ_+ovP*@Hva}@t~T`_tOT~ z3*O}h>21Xs<7PWIyj;^|vXejkhMIHu;-B1QpZiU)0vIGl&kpgEc0DY{kcf^1;*zFi6pfxzE}IfO?ccyt@`?i9wG0gJ@+qWvP@Pr zF<<%89uwG&-)td)5|x{re0OnR9cN zyDnOhJ90G0PGUws_P*TOmZIamNbekoQ)qj3IQKTW+kX&M(vU{gknwn>Vw=NHBjUIi zM*|9X_jN9$ERFLSYRBW7DpQqk@vmNnNh%k?@*UZ5yvKSz+KKF>?O9*UKR6YiY(+>s za&u-MqxQl>h@#InEN^A(wRpuJiq}!-XRjDWi@s;%b=IP;*z^rH=7BXvmEjN=F`+-|^*oe$_Hc{o*XH>G@-e-dfIr6E5=qk>#w z!Wucnj%CYnw^_5E^z&(<>lxYgUN6I!tm4#uNJ-qAZ?pV)F~Wum*b=yHaroc8`U@Mxwsd{Xlxx$kS8BoWX84aoA6n446hE;r4a6u15$u-b;F91570UDD=$Vm z#>Ef{5N2PDoD<$ia%w)#0JjPd`cx1Bd;qXmO+{)|lXr zh<41V9M4zSV=aEuV=@U**eG9Ty!g4y zAd>FJ*%RBYx(kh0!*^88XOT0>-P-D5mR6TxjfA_lCcNEXl=w*_&Ayjm{-G6#$B4+| z+AX+RmsF_0NL_xI&0w?ti!%9Dxk(u}Glw)0vc@Cvu%}%nzzzq2#$T}Bp9f#zdz>~e z`)UbKU0Gd5!KC0*${C0bkwb)U5$Zs?_~F6`@Cm4LKcWWYIy4Hm^LwZkS{?NM{Dbc% zQ1E-C;)C4|Gs{k(V|qA%*6T=n4uJ3CPsz@j@2l~_d_jR>HSGW4AL92;F!{ZHu{~2c z(16~IYXf`!jTBb5noBgJ0$2nY`Ls66}KLAv9g5%hh5f0a21yex(a}}m!nlhDwwa>6K;)?{dyJdJ$T@P-8)p3 zv}ZvFy_>le`KKNvjh3rXBZ2MX_e0f^m_df|E^E>s1I@+HHq!D5?RY-sH=phv5>pvK zipJAYFVN~5cI2`ZXP0u4ps}<`AHfm2%oZLdrw<5e$zlo^MHs``vTlaAfa!+F)&s|w zd}A-J*M%(g30oP&%}}BBWEx7&Zup6qgjVhy5^2!KI_YaqqS4kylh#k1`{rdC^I3Dy z#L9aSxXa_kWU1c%VIt$_yh6?=_}S(KLnSs4*Fatq^llh4)BzfG51}Qz2bT@2nNIJz zvLhE8k$ymlT~?~~)*uLXh<~?TLqqb{I9k@7JQj`5BJOIfHuJ`8AQVASYiOXA?<3%B zmWsq?1mXdhd?!&fpZ%>7;+0^r^<+BjlT{`jNhP;X{oFKG(sxu2+-iRO>MG6ckL9{RaT@!p&+$MyTdUlS#PiyY z#SQ{NC$_~#&~vSLoHyj4aU!Q!#drh=m(hydmf?_4V+N3ZBk`(`Ii;I6WP2gU?c7Ao z4ZU50XF6<96iA2JTF#}~lMzcj@SS^I8jR{<6{v&^=#PINh=2Kv9MrY+{!xl) zo)a8>G)3#0yqbY9x7lC3saN4VSY#ssk(?BW6qDT%1OEDx#fHmpgY2?bVc+`7+H0Wp zLk6XzpxBMO|G`y`yj2k^uN#u5Ec=ExI9ZK&>{f%qx>$w^6v9{VnTxjN==>c&a{9g; zq(bik(TE^w14~)Iu=IqniX*-wuofAxIRc>N?0r&JP5AN}#klVlTV2wuQugtG5cWr`kwqtvUUb zw5-;t9wH#K4W=tMMz}6A^R;+|C257*MDT|`+L7h8>=DXO5D)FhuWI3*>t~t4-=RV-8%>}UH4Syr zSK4$p!jd7A8o{U1PWRhekt$O|lkXtC-%rVy!Yk0&aV>4sO_ul0 z3zkL;x6y)T`ml*(v@WGP;j^|j)%Q5Yy&$lV-|w6*v1{`DL4oj%zFNwNw{z-Wtt-DQ zBy5JQMgV_8H$xG1j{{8A393Qs!f8^-yd4uNa+eW-b_pBXwdJm4+c;DsV=(9sYxEZ% zE3O&EnY(`Cd{H3jK%JOq2r9xj-;>lMqOkiai6%ltTV3B{g96bU%)gsO|4;61noVT& zM=z~M@H$%aH)(TUgi-zfvWONPAl9h}g7Y&i-UJ?SVbSwm%(rZSC1~_>9D+c@FItzz zO2BVO^x||g!uPA#el9F|GjD+$(eGy+d#X@xg?`Gr0vuIKKzgFlA4G7D!^gjywvN5~ zbr+r1KkwiHtBZ?#SJ21uz27A-ZL;$v9JvSP>cbYSSo3Q@%4+ozeRS*54((o${yG*_ zgAK;69e>iR)MtJDo|#?i79(_}(g?8;V!n-U+8rs)p8qX<$p7L(a=!*DW2UJ00 z#~H)|0s93da((}47}-(ESL_V7<5qnI)Ebehjw%^lpA{>a6On{guhaHk`WBys3x|I| zXEdW7j}BTDQ|^Z1i5`0#(KjdPemqA<{sn~Q4oguUzK(%^6@t+3HmI!a(!*msP_;+j zK^dn^T?HPbYB6#%VSdc;H23g#xe7LBcWNaQBKxbURS0h1McEaJcg}yQ`d*zcm;RD0 zwjPw+Vp+|Rjx(n>B(r>{2^UW#+h^fx4ydHhtARSc#dJy9@3)>vZtmJ+68-$jpH#@D ztp8(QtsP@S zW@VC+j^btKEZ(Ir%1W%5(P3xg$W4DI=lSVBk$r3z`?`#&_v6Y}V}uJA>8iA3RQG)F zS-`MVxd<)gw%U%RTe;jHF@oRI={pzV2lp~*d!f4wSjK=NJYt^b9*zz$ih*Q`voF|#)#uAFxm(Y4@c&|>A}x57%dazAh)?v+}x9K zYOgHGg`>rT4ZawRAAQr{g5d>66S0=J0o=(k* z-I@NtQx|YxNX_PF_Z;02>qp;@s!odr`aiw)j8prU=0NOCN_x$blHv6=|2MbePD{;$ zi2kg6PoEy~bZNJ|yBL^FK)Jx0FcRvI!5w?X-1g*PWe$d^x)lZF|4`Y4eLx4kc~L&_ zF9M*)sqA+r>9%P|2Jd#|Yz?O%j2bi69d$Vh4WTSj*cymD16Ps`W~7WwFLnI2BV%iC zr20>p6dPjMM}%ugaU9Js>+_KP^m}<*jfk#D#B`w+4 zmmD5gXx;vklZeudi7s~E=*|@l%j}pxi2?}#=Tc~&SdopJ_>|A(%knte#ima=leI+2 z_bsiH!TQ-zM6hQ7F}*m4ZO@o59S!w9z&V(J|B-loQ=ioub3Q%|Ui+hlkY+lI(cft_ zAy*b5F~@Sh%3U>cLF$d!p@tIt6-uO7rue)o8~De6%m0|khTEQw(Z)W;WuU>$UfWj) z7?;*}dP7Q-Ed6-Al%XaGuhQ`VAB0ARCg#@6ayn%bnDc18-(s4SIWdE+3RxZhRG>Pa z{%WTMCQ+MGbbY2X=U~mI^)66+&F|90aNRRLH@0ev=7ebVAz)oP6v6Rfl|lBK9d}Qn zglNn;bMte*Ev{xJ+PFn+K$J6QU0M!h8t$pcJy(;)Zc(tyW|HomRou@%&8oJ(3WwL^{>NC`scjSIqiKJ)z^x;E;fo zlR%n$Uyv&??ms{Bgku?f$Frw<$G)S6*@(?Uz-)^Ne6`!P0u!jN3PSgdlpoWB7r z`;K6gDEnk&XYB&If6!12bsBO@FlUgp@EPU8h)XQl8Bu|4MwsifCH|XcRjTv1YH5?Z z2>31gSac{yx5c46a-o(o54;F>0Fu;2wA7oumh1gEp+Ovg7eN$dhu3a@ph~fDv~3k5 zm^o#68{j917qzGqZD|HVtcsZrlDB$a)2XhjlU$3LYezIUzt#DRiS{-x_b+|6wY@iN zTxk(%I3p`Lwl_bQ=~YOy!lZxNO#(bJuN3y%%AIQG`L(?akaCS12ov3UkICXt57;b( zGw|dsFJhl~h2QT>oq48~-n=NhU%0#Nf z*vbHX6qnPdV+EC8S_^J5)+oCu7{U6nQHO%)yQ?!84?beptoU;OV@+L+lq1|*LmI>u zTs=E-j;0WNu~}Fzrk^lkN^tBkeA$*<{HDWXsK}>cxU4~iOq#0r%aCF??sIBF;zg@y|xo8Z3N9fJ$nn^Em36!s}uF);0 zQN42%tS(QT30r0oA@d};T|hiO)M9pvW()QBK_A{QNgTN8sA8K21N9EQ&vgOvla?|D zJwq=|_W6@7)CvO_J{rtFkaBd6l)V69mR6Oy%6CrF>i@hoCf0DP6LmgwLf^$B&)siQ*m>D>3y`4N0|JNdL9k!IbC-v61kY~y-(H5yf1xJANJFxYN^vp zdq4%9|4|dl4btm6Imfyr1M(s+N2=W|9vd!E1zB9P0tB2Z8dDl1e;pbkYGiGI6Y@nr zj2XOD?5%mz+X>UgW4h!Wdev(2Nxr-Hr85DKz)SSudaE&A_MQ+FPD;gnpKQp^TQ;Kj9hI$A=UfyBZ?XA+i6jscRxK}2@738bf&s+1@O9kBByMIDR#uNuX=$fs^Js@r zzmd$soXy??Rz`6?^4KexA`>k`^*uLmVNZkP;9&N=5ft?uA)@nlH)<#PMGtsagopi8 zOsSyz`dMgoedao`w}{1t(kZeg`zTh|+biCCTQSH&U6aE1)#0O>dK!gE^+Zx((()j& z@Yl=a`JXK+{F>V?GH%RDlpNzxo2)Yl<7PwM0*6ZR)!a<#LnVtcAw;V+Qq4>5PSc%8 zfu6pPEy$hGezKxUrh9luBBeAv0$4 zi}#2|x=O7timF9~LyHCoHzHDf1LMu>Yp7#%1zvC2@58OwojgAO*Ln{dj*gTxG8 zk1pYKNw$;2ZtVb#KPlDv30M}YPVpG?E#^EzXqIoe!)W1rU8jYg8hxxJ35sX^q%bk4 zp`Ur)-)&T67tVQAuanuk2x#{wQR_`nwfHTfI`xeK)@!d^GKcKTo+OnO_bEEw64O8S z%$TRQAiMw)H-w&5V z#wl-ub9ZFkpnJ0-S6v{nur$$5g9t+jU((3KBy3El2@J`DO2oC(fEF=)eeNiVAkk;1 z5?h3zj89$Mo9G33s+J2T5q&#QEkdg3^D!&VloQ2;syl62M02QuigITF#XsCwu4cr_ z4C~u0?Jv}>2)|jUM*va_P=;z~aQIwc_8a{1qgfCrs8j){XZp%Ge&gfR5-&XKyf%Pc z<392)kl-e9A0}qEE$_gw*f9CC@~LxkFWyN{icOiqUPDN~kHH^LcxVto0?JG~8XrGL zF_o}MmI3jZyOisp=o9aa(Eb{gHaxC&)ed|=m%1y;&yY&ME><9$hJYQlWCxIad|WBl zTY!lLm_mzcgB3vAU`7}(MCb=zVb_2DK03tm!0Q;0C-W7K$Ol`YW9q{11oyw(!8$E2 zP_lO~NeYqTHGkGdiC=oeN{41mf;O_~Da^Ozw-NczwdRNpUXm_e$|s2);=5yfq$RM# zJm?u?bC$F~Hje^3gkWdT{&pWXYawvr|%j4(>k zQb+$C%Sf!KGq@G8@o)5t_YtaU@kxq6jt6o8WfKhf>Yh%4om5aqm25t^R8e_PM%cN^ zaIS$!k=+kP+D%+0BA#=Ut5@dY^$Gn|neYDnA0lNh>#Cr1Hl~bHK%PdNbUxYidq#*O z`D%m6meFE%{-gw?v4Xx&1zFeuzM?wwDxC) zjGfjo;@S17AO6u%JoV+iABEgX^HX=QmjJ)RG!oEQJ?`CRB6#N>+c8_)tI~@$QegV# z5@bsG5JC75U|SYs)f7xa6_=kH=on$SZAt?9aHy?BAzyGi6&g|du zz5l!ZZwLNw2mWsd{%;5V|F8o@!uK@pJ^cN@{M3hO_qgvpF#lgxJkYpDb??6L|FYsf z_dTNj?xp=NEAGMm=UZs}Z!0*?zI?o=!A+%+rlBdU@nVu&CFX&JU7+v-yZ`PzoKy*; zqM``t1`20b*hR{5r=_KfTK)c?kJO|i3K1^8rFg*3PHYnTjGg^Ombe{x#KkqS*aw|~ zw{AQUffN*v4Wo>f6k3ZuD}*ZfTIJXMGH2Qs0+9){B?uZ< zG1a!uzZ&{QbfEK&e#Wh6%DYT=-s1Drcf6;A6j%CL4Lci+6YXqJFSBr&j3r-B4e)xh zR$CI3BcR48b%wSn_^!A??QbX7pW1w<(j+CMv{e@U*#SG`@6nLE@cH}%_^a0Oo9C3? zRFCOp>#X-K*NjA@nhh5b{5#9qO?v65mYlUIPFTqQSTZqZs`+Vj%(AS8C3JW{dvK8z zrSg*Otw~P6oD>!{?b_(<6dzxEvRHw3tO3Hff$IpxrCTCkJ5IymTkA(iH0++h)3LQ+nLz&xR zUY=PLT~8jdpWn^fQ2*yI@ap&ksH}eFv1f5y@H2<6)0mYjw!_U>?7N?T+x%G+jPRDG z0p^_1V;JczAm)*OeVjjjGtHIg0nwSM7reuYon?oUrzOj)LKJBk4HAAH<-YP-_L(>o z($h1Xi5aS>mEwIG$TAUfWdNN~IEUU`>-UuWJQ`4>C5{e6ZY^7SF<;j_!>;lrsqZLtS$zAyedhaY|5;-44oZ2}im|zs z2#^?$4cHIZ9wpprxKK*>y$|b=x+Tc-SfyB_TNfPgKO$=&MmbMzl=+(T>PKw-Nl+GIB|S z9Rn-dTLP6`%vztt>ZPXC?HtTYZ7$H)^E#QYy<@@T?Os(3)-~1(o)!5Nm9=|LKli#k zdKLFTu+;vVZiXv7r3V#dVgYBIy}6#hymNKw_A391tQ2qwDpk-VxQ9#W-)MhwX+{aX zGix$vtQRWjjQw!4ygff#*RPeQa|=^73;@l>8_syggcvS8tTWaUKcGBW6Z=N$9?Tooz|o(#Q{$x?FWrnciRsOs$m-VX_i}$I4GG&Ssir&^e76>jryf2`|A)H z`4$}HS25^5S71t)rs#!goSK!M`+COD zKHISAU7Z?_5pORY^*P8hU*_3!N^X5;p-?$H1>_N!$R9SP78*EtNAcc!aDL7_yUVJh zQ0Pmc9us;^Y7i#fah{oSF8e3E|K_=>-Dc%}lmTwyY*5zj?XJuBI;T;GlKqTNMV)q5 zya3T|1cg^X9Ft!KLUs+`J~5$%dLzD-aws>R&s_X&XtHCx^2D~ge1f_}cr52T{}v-F zPf;icw2jj=l2k8N;Hen=fXf%nQmM{}E%ZKbtqV{YEvWuxL)jb>D7bKpy^Fi~EIu5g zjqQ*6YjObw%oc3OOW-!+?ZSX-FO-#)*rOd#&Uh-&`{_Cq3L7vfgsj^+ml7elFlnasijr6stAd~0J4)-|JNYCarJFQm_;2&DzT;W=lsQL9X4V(RN zpFpQqG<-ypnkP{1PLXTKiDna#b|NyMUUA0Z&>V?1PZ6pV2Y%b&5K*?!E28-(?@LUu zX=`xD{-d!x_V2jMNAC(sHDa@2!;>D>qXaV! zk+N?j?1faGA$jeD)`x$Ri|=2EIo}x^sc``rY_OM{g9Y{}Y4g8M6kSa!@^mi!QsfVs zM$`ApMF3VTzv@|xdS~#OPovh%rm;EXzrolN<4e2zqf*!GW%@t7+O4Us|3cD*S6;SF z$Nu+GIDoyPYc;t zPW6s`C2!6AWos>rXHlFs3)Tb_rgoOk`;8~I=$)u!>AxWQoIMZ2H z=#{106f_s&nt7)vzR|{|IQ11>2e|V}ac|ua()Pzh>kMtw?XFq=sUlLJQJ)M`S~+a` z zGq%>=YLf`x4plR?H#I|oh`o}uh!vwYLA16Q5kai*_J6oP_x-uA^Sq8DX2f88Qo~(J z9x!PIoxd@a_+$VdV* z78_N|9o^NR()iRkySM%&Bvt;mHf%pDsXLCZ*C*o>1A{^VBfB6EL!*7{UhwbSZ_qg% zRY~l*_fXD_TiZplFj3-HtgcFc1EW7tKhEo?8T$q%(RE>tDCos(QTsBMv2H!`zA!pt za!K1}S3SAW8>M>B(AZ^^jMk$iGDk^%pn@r0g_cf+Ep=N@r`e(EF-9!bRF3FItJCEs zQ_g=Fc`qjwc|76jw=Y0^%!Z_LppmI1vu9Huh1D2S~RL<+43 zuF!sK3#|;2E76BUe1gWA3cSs7$n=9&TYql`!_k8=sV8y|&N6~=2|@mg!bS#pKRd@$ z!=~-ZQpYh(Xi(Io!sb9*IB!ZmXAQdEmBhqmEd4=%Y_|Uy8z};x2e!T5x*q zW2!2qaN%S`lMz!>o4=(}DqPAha0fHHi0x7lpXtZ&}VzT5C0r}Chjg+$})2E0e$l$dZ z5C9x)oM`ZA+;A^GwBJ-R)jYkvsV^sy_wCT3KjXx~U4020WUm#9@hqJEL(zt7>wKoa z`ExRh^AE>G6Gzm-VNC5Y!4EahXRXhvL9l^&_ZQ^PF|?hGG;Sk{M|(pAdwncc0$xv0FL`*w!+ZX;M6nOnRPEAznr#O z10Iwf>=JMkHVA^{Q&Sf7_}0HG+ko%*buVs-ob)E%%~VtH6UeOwB)V6Lvufh>+p2wO zn?(qwKewwxUABF*{l^|h&MGE!{!42^>$d<+M*4FF&9xDP9l0S5)6Za&lNI{RjS*L? zDcA>RDxb=?w!YdAQ0k%f60GUCEPzvla4H>T<+TYgtcjA?teKa-1jh{&VuW))ECV}y zLt^vZ20RG;qC0cr_I%7dYkbCqWyYppl{Tr+S5I>{eWtzrjQydi+5CoZ91q^eg^0rW^VyN-k#c` zJ1@$qF&k`I*Jc(7wG;Js1EgYds}l@OhmH$7EX+7cC~p+W_;Qscy13}>;^);P@9LYa zNx+cP0aEu9y>dOBe{^=%l(iqLEB;dd)c~tIr)cdAz2SjNUZ)q{c}maMi6rB6_7I6M zc=eF{9>*HKiMZLx`{*pYZ+;+Px#%~G?5@i(?C-vUZY}izM@}ry)^Bf96TW zZu)vUy~bOS3``sV+SoeCH1$PE$!j=O9oTaKu>h> zI%2U7ByCkoY+{+kbca~jS?Js8llWPvco_mtpVWF-WS+S%Q)cV3@ z(2EcGwoS=&Kf2nb`xWsdlhMxC`J@10nBkqPaKp?ns}=rPt3T@65j@FkHP28sVAea6 zI=ne9>RDvIZ7HxBDA0C{Hs}#Y_N{)buFE_2j=*BwerZ^wg;lGB)cq#lS|=a|awm&| z4+f8Il!)#TMO}dFp#HesWQsLTeW5av>|M=(a)3* ze;-r?0O6LjD&n~#(fKRj6c zCwVoJZ?lV`ehWzzTeN7_3oqobJZSZSt9!eFGM3&T0=LyS?meHosI3{CbZxl}L#8H& z&onCg_#>z1D?>x$U${N0_)u3SeLsqNpc&`j91XaG+r+17vIEygk$2c~>5cb&?LwM8 zQaWw2aHFTW*etCzx5Py@FcUdwBa;Eb^E9Vp1++SCA7LbOoS#_ry zrUgqhe`?~T_J2_FW-uNhv(u@aJ|U(D5>f&`l2nHawW$>egnqq8H$8IBEi;Z^O=DsY z5Azil#~`c+YxgbO;M4=$EO9KVqwkN^uDi@`yOvG{zJm_s*-BSzx13WJ6lgQk@?&Z` z#De;O=Q5`vdGO{t6<%7f3ImsPZoq$&aD=@d+h;+rN)N3U(l%)NW*HCxSNn(zT&sy&RNm$Ga$tbCUt0htcV7Ooc!_8v}{n9_joYd zI!i0LEXXO(D=cd7dS$uV=1&J?}486WVl0Yh6d(0DbS_3-Plnlh0e{L6PKQ?ka|Y*k8&&LbJBc zp_Zi^Bd?bDq9P181#X^|dHfp#O#P26=M!6kwC5FhToqysV!NvI*2t8zoBez+QFY<8 zAbccz_b*HRv|8Vut(Z}i*VDu<%f`Xrb;mV;Yb0!NAVv>3L0^8c0aHiPLMhL}ou)lf zdD#D;%TtbnJeSWzz%}OdMg$QhOg%{F}6AVXgl z(YKETiWwCn#u;LKNU}vyNbpiD?`xMY;}eUGAzX$&p1w^q^*)P3J%J3|_UO^>j`3NLuG z!aG&Am3INd6C%^uXD>(9GuOeeW&G+Jfaz=3QcD!OQhI~1_U;rURKeTLhky^&}qMq_KU+RBsxc1CK64UYB_S_I7UBRxP&rO&#Fb(Eqd5amPdI57*S)bN1oFl zk=_}>3;E;UB}Ba_s!o*QP{b96|FL@{pW>fDMAUX1jFcbxUHcDu@8QcFjCP^6=bR{^ z7q1U~fboB|DP0-PF2bwYEO^m+u{p?I@kZ;5ChZ(*1kg@>*$At3DkA~{JTwjq^t_B(rMsDa6 zQqVqL{kTOD#V1v~=xE_hm<0zUqoc-6CZi1W{Tn0i*sX-QKg#LsOkG!^{tG+KtyGql zx5S1;&Uu;VdMBNhC+06YME`?cfluia)G3VKjgmkhrz09Q86$@|I-ze~7o7a51UnIv z#_QQ-iVeXGev;J5-H}R3WTTfL&<51U1zR~NmT(!wTjhoZVm0S7sthLUPGlN6^3kpZ zpT;FUQak)6LiW5|V&wmr9G68dG-XHKeUW!QZ)``glBnU1;jC&`_QgnACEOS(pTqrj zMQCA?7=>fcfdvDj+qN9I>-vW}Yv|L(tj_sk7gMgRRE7>VM_zL&b%cz)WxCsf4f8|1 z;{7}GbwbCdVU6R-vv5w8JJxtDXO&FWK`ndYrs=N6?7?jRpK=eO-cNy2Fs`FpUoo|@fg|2EV@8vXnYSu@tqy9VH$($>HW|2p zn=SPhS}{7jy2mJ(QH`Zd!Rm6s$yE5|C{D3vR2tZRrjiyqEo$ME4!LYLOe6)^3^etv zNYItv=a8@*c=Z&YJ%`mFknNC&0H8r{VYSJ5BpRrD4QlPwd+VazBxxF%t|3~FD>xpne{ih}^9Z`G@Z0sGz zfbWKw^bFnzq9^d@Y8#02V!$ZhVnc9ztwwfzLShK>JBkwBCg|~cFyq<7diJ)e9_XFF z)OESJt&Lgfybl4Ntmu}5;1sM_O5%utEO1E@V*k(eY^ytIQG2=y8qf^>DdGLWY8nt= zvnfV_78{l3t>XWv_y%%_7}6niM^d1b@lkuf8uozm=gL6Th*OR==6Wj+f1S*)aWkr0 zZri}YV_t~xazcPP9m(PJWQ_TwLeTk18(!fK`P&t*n@wU${&pzyz(g}DklZRp+l7q+ zu`eARKg=b6_+>--KpNN5tBN-HI|fJ`ZqDD^M$;2fozw7uNrBP2!KRqL}%wgi6Zmfb|Uf^tjUy5RYB<4cB(yx!M`l@xp`QtIm z>4Ks$;_Xn4H;A($!h|i|G<`dp#T5Qi43(S_O|=Q+?K{b$7V;28B>ERK!cR}xZ~bn9 z4;FLABrJMPwtmLB9E_G}lGQubUNan@UtP?~GphpfP8nGCco|=AVSi1p-vim&EXH?l zlFv~LuhuWXM1NUBbo`Jz4JX7kx1b3k4|LV!Ghxd4(=vX(SqvG>PXXPS{-s1wP7D>O zA8T%O<8^l^+W-}wwkyUUhUukm8D%5x%g}l6c+IPXzkIjf+Lf1xT|Ami8LlmU@%e~F z{*2ffBi)y1fS8dq5!RSJAJ-QM!b|NW9_n8oMUfkaqfN*4Tq>{Z#7oUP%vz#Z$kxuf zmA(#B70Ml5@DY+RQ34}&05@2{W{9OMKa*>20kToI!C@TJJZ;(P$MMT=8*#m7vgMIf zJOp6?zIL$8%s9$DDfF{i`I2Gd2+A#uO3q(4bk<>+iVr%9nR$R+YrgYQ-^Nk|CTFx7 zammw$hx-6Fbe9XF@5bqMIr1~7R+m$_fX3H@-xoGXCU{?1=*x?E6u=; zc%Aa)OUlDV^8&hGSiov&QHxkl%Tz`2i)XBswaJiC?DuG+BIIcjrHZk-0i8Q4Rjuak z*+VB2NvUsl#;HGxcan~_Q}b=6?3^;?=%r3;o#T^1>J9~k7dT<4_=&#w_f5bN)cP zd$a%8BPGB~Q^Jh7LGsDCXqIqQL4F~Nw3VVkUjk_9(+>rGU7D5d(>>|J|eQMpd-KP z(%kcxodtXHM4^_=a!>{NIPzSDT(-1v)fRg!8EM-G`QLu0$ubio-gkKD-z`$UoXEl*m{3*(tRBQK8V?X$=*m9o{Bg6Z6wkKm)Cn{&q&2ZL9|uaV4{C z#YIf-h~Wx;!F^2E zH&p~SBt5dz6P)>IZif4TynQsF>_sDlTo!{e2e(!bCoN^+wZrs zr_1JsaXQlV;Nn%pUrNXZvePVy#;_ItB{1bGUVGEPhd^VhW zSvwTRvXiL)`AXm_IuwpQbcTKlspsJM#m`cXY)7H5+*W@YI_+eCfJLhHfW0!jT*hM7 zLv^7kLHT4&i;|8cn(*b5kCjh%Qx=#vgGvc9rfc`k$Jmxl-zJp15N9v8x;_u}CQed1 zhV7E5(bI|lcU+oKEfwmKn`A_V%|C(EE)11zhLD|mVj|<|kIM~~&uTzu_bt&2XAca@Faq^q`PoE%-qaeF;D9XH?m6H z%q2^iCQWL@osWo1FpX)%APJ&0Ys;V~C{@^Z!%(ZeOpFW;k$TlPzpv#jBH3NYm++lu zGfsOCqf{Zs^x?9lMK053#DHVQ%XX^^vKDb=CwOU|v!ktE99eDV$8qH^y*inFH|D6W zk4Dbzvs?(yGvPI=l8A?_yA^XQa`h#{k`@D;mh6glB*W-_HuL>E-k$846`+V>yNa|U zmq%Z!(DAvuPma06wAEWbkm3;Sd zx9NO+=P?d>==nv=>VLJUoc+B>s}TGBeV3&-VR9RhBMMLk}p0V?m`4 zn>yBMf3F^_M_c-1@Aqt6*^3O-yt2DL!N{}_*z(%GL;JsldZWox2IT<}2#i;@VFz9= zwpQ&+DkEOwZWau1DKSC1iS^nOCKdQ>fF>-i>kT3UH?(}Tn=Z@qbK;RHAe2EvBEXLz_U zuj-6(D{{uo3z?np51P(D8q{9*J?GD8tRX9LXPijwXfG*4Yd8}5%Tie>8z;Rwn{inU zQh&RVT$sKOS~j<9dri!n>5r+Jyfb*q-!mTtw1g)ESl9b{uB=#jiP2Rm0kJzGOvot+ z5$0CwB+3uLu67w-SY7dz$80DZHrnu)xhdsNxv)T|brKcA`0weQSzcKW47d=IUzmNB zxnuCpuxMKI=1O=>*qaJypg#PK#avFMzMs1Wyo6uYb>t-jk??0?OtQh;l95@zybDSt zOEU3W8L%0g;5)gMAzCG^e$-=iXf<$F+ChLA5Tm{__M*Au4BwBv7*;HzUq!(#K4G=L zi}~Bkdcl7Km!oXWwwBb(F}XAg!RB6bwBxRQ(La}F!i=o&Ectg4O=7V1qdJKf7rwbi z8?wysClb<7yUB=x+En9ci8N3^GK^Dl>+8-#7 zTufix5tr_8=C`#vox2%Gt_%IAxT>bhA|tN+0GPxw;#PhvIHKTM;8Di)Nyx@!Db%Pt z@iLmgA(6mo8o)lqx^j`H*qFmNwiDA0(WtG%QZfsIOmd zL=%D^Zwoq_|9QGO+_B#fl;?D&^u^!GEm}C)xnmROF;UmS(pxf8p&~7qR8~G)ahJ)z z1=FWEtIHda1_io#+S#y^I)5I+w6=)RC?p3Omn-=M>P3Wk6r^ zH9&moopvK$Ani&u?ns{{h<Sa z@^X-r{Y+aGYXrC96MQc@V)6JvB}6d$dCt~QzZPy&GyJ|N?i^xZzw#_%p-=3H-WAd< z;V_rlj`~j37bh_V8 zwotXr8ERl+adR{&^ccX%kT)LC!Y0|&H?X?Pw!D?R`yCIz!1id^8#XfmR$=Bc*F`sv z54Fph_f%CCyHoq?=CAezM*LlrkUAS%IJd%Q2_1IhA-UHTwVsf}xb9Qu&JFApH)l>t zcNfJjg3^&hS>XRuf3LjO*WNb!a~7Q7VoPTpRZYW+BM7#*urjWz^*j^UwOR{%My;P2 zP`b!^al!V8DC`$vSFpuc8Y4d41AEd*!R;rw`}Ffcg}oR_Do%)P8(-N{6lgPByC!DL zy%+`C?>9&a@rn)&ZPw>Qe1og$;0kp~i90p|hpu*Q(<&OOTSo#_L|{T*&ufdE=88Cc z{gA+@|LoBQEi;Q77~Qg6EUt(qTh3j@Qs2F%09uj`<2C~fcG6X!_FO7y0gNT z*ey|s4cQ!?YxoqOJ?^U=vUTsEDCzN5x0@gcYeP~goyFduDHvtWb>4di#~oSj z!5tT~5h->@;>3lkUzR;?lk^p5=kqGR)jNOwRhb$be|7;0NPT8AGp*MUD#x;Gz`iWb zpZ;pF!+7RlIF(P7D1oC;X^N8oaGXZZ%kyWzvL4%a}thsa^Qom z8~fOcey^s)yLuN9*1T+l@2y}oOTHO-B&aT!TZRwhEwUCHnZL=JrZ(8S^hXUfQX^77 z7))8yOZCsyl-exvvyF(YK-AJDT#91~-7L;>2;L5EVc3_yd$m-T&1A6Y$)A>LXIUhm z@YULGeQ*_r?X19sayiW6__*qZTgK0u6tZ)V?S6|LgvgMHvKUDg4LaL zgU*rN%<-Q8Yx?x7GqjEc{MdXk<4{r(SR`*q@VT@q>ibl>v{|;wHtZ53sb+7*< zUwORRYHE(QJ;1{7XntWZRXqX@6znC)w=_p_8 zc+n)1Cwq;T%R(g`r)|wg&GJ9Crv&?$nqj<&75YgRnGdK47*smKU}r8-Wr^Pw(L3O1 zK#fWY-i?^?c=je#EN^Y!P4FoIKPuBb0vV>*Qdmkk(s2Mq0pj^5&4N!56}D2A3Cz5j0m2z zY8tc6tbDDZWvm%q%d8zE*Mp{jeaS+BG)&l)etJ8)p;%M#8SNz_7*c)c$1(_Bv;=@pp^(=$;^=~golMcBT*C$CxM2#eO)O9F&KYuu5s#nfQ3ZH#=W#>}jH_1X%! zuK%H0$_JwRGBJ+r(e(&hvtbPSUp>o=)N9cZnaj+Ep`d4cYx+Xl&R?%MZ+ZzsgWVaM z4bjdEkAybj4x4VLWUl<+mpNOPm*N_3>|%YHuYGC9QS7%OC%*6>QN!Q4>1&i8e+L{w zZYbpM@PYZbz2IUObrpqSbl_sv<{c0LU}9jB=}EORn2Gao$jfu=De>9cHa~FZD*;hqZED|sd~lXp0&X!a+!h2mqME@9Rh6v_gp+?+-76FyaHsTD{Oc; z-Wb@emc&{QH^!|SjqVe-;+wLnrnv$a>9WXH!T{Ya!L=`2kRGzQiMgv zs;N3cAESm1-tncO$r`&zIXYiCs3xY%g@<+UTs(=#cSc9{+X48t`H*?Etw-8e7~2iY zOAiM{KIdOL8T~pt#37*R#{w<<7;&t1l(o6 zMl=S<{YwqTS!r&yZ?{AeH)j$Voe-l;DEN9CiG?scFVpn77hG)Y?+p7y_wLm?aI7as znPeWjOWj8pW=q3DE%Ss&dOC`PQos05zz{OX#GwSRxn*I!9@8s=3ZfyA&-_tq-^@7g zb+i!M^#uu>av?KQTc`6SM;(?YasSRQkF;WhO5)V3B`c^|irN1TGeGyYbWBgN9-qsU z9~44@5BOxC6#J353NEd(Mwhp)_9-^L{veQ<^030Y6S$D$XL_sas{r!_>G>dwSY!60ZM^w)*cEKUV$c*3|^ zOq}u2osZbiC29R>+XU9FOo6I}6W2yTYOe(T>=*KhSu0WTFL#t3WFA^oMY1r?tvn>R z_R9V+J-(1SGR}*A_ym%{!8q#o21MQ30J;SoBTR+dfvp(*MRMzc*?bD zf`>~E99Y;7B_ts;=<5-u8C|Et$GmaL`LH1+R6`GEoJZA>xy7JYV184HUjTD&b%=j) z48Lwe>pXk+IGYY}zw0*pizoB--rlE_`X+__;QQ^O z{fuQV*pf$f4LaQ-RA|Tv>KhWpAx-t2>58fyCMr=6wd=$9WMY<_DMog?vr<_eHfI1s zZ{zB1n(@wA^wM&)lysB}C~CPQtG&=Q_8N-Pke1%kVYCuz13h1HWuxq_IpiHAT%Sbz z6iY(v8AH5O|A!Q&aJr*SXV-8FBelkOY2VjnR4{C0Mciv6hFcennHz0j?_a11EgrUE zpDV{}9!3WWc{0P-H;q@vj56a=HVpmzWWKMfaMLdBvVHp_$b10&U1|H(sVZq~0gRty19ag5ecvP|}cn@&hW-q(h{= zeZZ*_^2CpYt)2dRi^bBYdPivKP2wVi$u3#krdW$D>kBT{OUIZ!8(WUYyazvMoJ}5% zJ`*xpIaxr$a=z8w4ogsj;kHu5tLRc#_xS6gL&A&UzG}l0XZ>9=Z!_aq32_wRFq2u; zeTW_=dl%reoQ8jB%`ebwBFe9*G5+RR(Bz+$ydsj%lXmv(_{+NLFcysOX6A4rtvTSs zKTXP17w}#2YR>6NsY($j%Cgypt#ws6k^&Y^upQ$8-Lti`y0yRp|I*h6c%INqCC-i_ zrA^c&pn**ZFA|{+#ih1WlG_A{HmTfL70u1duMztf-zvmAYz}iOjs>uJ{Y6-vp+=lX zQJSR5^r(5p-+NyymmrH6%e)FH20}bh)oLS3@}?~imh0)rqRlJdZmqMn+26LWG8*vy zskD2L#wPDa(5e-Uq0(PRD|TZ*Y>Js|*hhmWoYlidq(MccVXH0+hO_=sY|$FG4ajo1 zGG5LZQ%P4)+Z6}AtHtT6@*x!^jR0LMV3YZGhWe#|>pQ|BnfH_VL!2>i&YcD|f?rN2EH+!TO!Kl z3?(d=6!bC8ytrZG+awzyIh_*ilVwD!M4HU6N>sBwrl@C*&qb1Fb4U&j*`F_+h=_3| zBjo=f8YqDimn`d0J=aCn27%eD^y#bgbf>)?7Ro{wKDGm*PwV{x7tgSEv88B;G&Ode zcwJ#`E@?*t%~W8cXITq!h|Ze_xp9B&>Z}Y#rdF9uv3=Kb@T~#=*K$1`vyErVc}@2_ z%=p<`WR(*`_k|m^sriPU$rXhwILnp`?t4NF&5w&NIzN?;BT(dP@EjzCE z?ER*oHYM|QyZOi@@iqKLk>RsJ)-tc@*zvj=KQa?j$9-tP zYU#DizX!_hvay;Y8(C#`D4A+GRX!NsluhZBn-p2hxwOh` z-sr#D0**S$DCf2Yjx|-@BZl|Ihu%h63)JN3qWoj-{WMbwSTcxIXCF`9?DF`h4`tpUMrZqz#r!teEz~ z^lal*;99mEaOsaQvB$g0V@A5ygSJM=9_&lK-U}WR1vrXUilD%4NS3thtw#6IU@Wa`s4HRsKjSq?s-=Z z%ZEJM(^N>7@n<_@IW}W6*-xHQoYfzl<-&LP(tkusXGHs7yu2#98TkY%*L=%|tN-xS_fGz& z1N$d0hs-05j$SSPlSWJje;QTX4=N7zBF3(dU-GfszRqqPLe-IjeY?xIWv5~&u{-E% z)w^&!R^9#hOw06qX!R^?5kAm^RLklUX8or($RcbUU~>e_cQf^GOxT((A$eU?>XN*(E~y?zp@kLkS{D08{PWK554WW^Y_P%)2b5ggbLD zs$BDF4aOVIdsMZ+bN^{Z0rP*t@D`yWXpzKpqQmr7H*gFzXU{s*zM*{^n~*$yC5tYL zCQo)<6t5XL`;Tc6Xq_)HiDnsWjOU=8?F;71CFV|Ez|J}jenL`*{!d5PlQ^$ zOO1YllO&9ys?IuqgO6b&Zad=4Ej}!{5ne2-uBt}<_?Iu={b!b++43|nJ(_8?aoU1E z!5d!REd0AdULgFv+#6KMYQnrzKK0X`+!MYem|`uVTqF+sdajv(I$&1Qdb9 z-}tctAXj)%BhWv*GdAQ2k*_+X??ND)p88 zpC6Z*YQhJ3o9KyrLqD#;WHVQs(g;h{gsj~@fr$ca(=L^;R_xtsN(^}C2kYUvG{0YXnB z(Blr8Qu*&^UoUdLm?6jqzDOFyanUEM7m&Pn39c)%{CC<$we}rnV_-D9iL;fZ+Jlv zWkr`WpO*(3D8iams29SArI$lGLuagUD{EEpuC@XeU%D!8{(gC^{ZUqUQ$=)_Jc1|DU4BLfHv1N~%2P0{ z-%b}exRa<3pn*MK?s%3(7bnagw$b!|nsttTh(ke#ONi>1V8rTqa^ma`02m4?={i31 zg)KRpvQMAXY|T8fnoXH+FnLS`+>@DxzhX<2gTi!vCCmJ%t$p~?@H&5C*(Z6IxlUTj@3hX9i&K*v^KLp20t@ausM?w-Vtws;rfpdmzHp08#EZJy;l`I? zKT+YO6bRndq}V6OjE`G9a{TLBItlp?;fE!i31DhmCY~_mX_oa#egDtjr!LDGfRgNn zOmW1ojbtfRZZ7=OD1p50TCEOhzM-aPvre70AvL}}n7fsAo&EPQO>0t_V{`2~SFU@E z?=XdoCH?kvhSNA|SR~MqNno6Le{bzmhz7TFc7IE%*hofT^l#5usvx7r;p2)#^>9RW zLqw@)+xo+JJq@;Z%ZJ=Khm454{;*4$+eNdbX^LQsSbldgZi90R)AHC6sX9{DYwARh z+0dJ}ReO~U;1hRm)QYW#fu#|>PUdjswo^V9p%>9(ml+y54+1&wFzX4Xd|-ppxHOnL zUsU0;&%0aOXQe*`xB(E>2iv#RY*;}P-}*w0j5ThKKhG0PL8T$P=s0G!KguHkFGgMY znM0i_ZKr-4x$2qn%pa|nRu=D<0lDLL&z=lj70Z>qymY|75yMs*BaF#UZ!=f0w0=Jq z21xj>_qo9sLHpK@p6$5^sdXE zYyq&7tW=FpdZ$lM-BjM^H_uCXck;jucFhp}fc;vm$nk!4yk)Fjw4Miq=ln_Mq;b9q zKzs4jyYVW*hnz*T%BHakYa2xQ(Qbc`pgX5G+Rgq`J5xCCYQ_K^UU;D@c9iu^zE3nX zVQ$@*L{~)lYu}VHjkQN&#XpYpu0phv{u(evjnK$;JX1rX`%;1agqO*eeY`%eIa{@r zjm%0|Kw|mnbs}qBpZvGe zHB;wbB4eh${s&CRtdV9D)0%i$(d#_==P#)0jR$ivB%h=K#!<`8qQsms4Xbg& zHU@6_HsZ@Xs-gc~s8K-%R_wZ#Q#Ah1|7F@)`o_^d?E+z0$MGk5bQ+XwNh)Hhe5DV{E}ToGgmLzlEMP3?V&zjPEQYV;K%N z433H=oN@aZ7G`)Y1smt+US>ReTwq3be_^Rh)yRs~gYUd{5)D=YxAwYVAVg(RYPvqft8iO&KpGH;w+ z;k+TST*CMe|6G;7nCR<1sDai?)Y*EMTR#yDe2_q_HvZtjp?%+_#)fC(Dy7DO1W2mZ zxz`xFli2uJ?u0XZx_|lUUV*Kqdj45kJ<6d9@__f5sXKRDRRquSM zh};zXcS8rNiyimS1A^cMj=o`!tXLkRI`n19z#x+z`~tn-yTSo(q4vu`w3-xKi*b#A zrpC*#9pHzLLyNlJ7ESKj7T=L{)=R%psy?`IHw*q?M2RTKWDu1GU|Sm(k6*MZV_s-2 zjBV^`>yUQyT-Zo);u8I;EKL_;2mN6a#Ao((t^ugNqemKw+LGLsO`AGL51QEMV60>A z>x_Ow2gDy}vpGd4qy_zijQ-WtSuVawBH(jlqpf>e*;Yws_N^bQCAPV;_9UxI4Y=qa zyzc5?wDBz}2fMtfiSv|?@x@FSGjZw?nyCAp@I5;vthb zX13-{gzv(BJd(LWF~lMl5_S8Y-_k_#K|QGevRUYmS)eoe@5$cU@zQ3N{J>*szR!44oHWM=yVbx>FdpS?+RFHr$N zT#$~S)cU$*w@iRIT$k6~GH)hhQfw_bi?bU;UX{g-wsjhJxUn_8bM#`#3O!OuS~$2K z{Oi~z?Z&SIpy`#G*N#f2hYPkCuMK7XOaT>5d92KhD{klij6|J>Cte-RxXnzHt8SCQ zn?myaS5BmYkaO1IppN5RkmwJMF-v_X?nfwe%w03<=7(QgvpK$wAzDPp^Hs(s{u{`4 zg#4SZcG+=Du(V`*!|jO^vS2FElf8{*>GtNiwE~h%D%Sj`Oo&o#^6JSEU2#7+f?D`D zW_ni3Cw?es?(V{HK<*3`7Nkn0;u9p#AI+-GWp-VaenJZ;bRPNpds`C{hCxS<*pHC&UBV9LtFRh}CzE5XPb>q20U)sAt|fMTCMZx@ZWs+d@h|6S%(!nf+d<3(n6 z%^39 zu66%f=~$k!|5%3!io4NfeP`KCfFkw!gnNdR-&IJl8yvDt3Xr-H6E1&Tt(@@Ig&8w# zhL}yx-YP)}d!T~w+7pH8ZJ`sdf>=ikFJ4|M-KUCDegr#6=={Sc8;RiK+35tmoblk_1) zS;WuC`>W>Z?nKwI5jPM}dQ^^@5HhA|HLp*Lp9BRIYiGHzv3>MIdlVAD*4!+Sxfh{< zPK5z%3ZE-52w0|LN@t|6-i|d`&LA|C|HL{F{SSElQ|5HM-s1raK=N!0m$|VRTtwOZ zY<1%FB7!95z9_$}W%L}BWtCsbYWYpma}yB8a;kuPs5si@4kOuakNd!waPx zU?LGz)lIovJ&F-J3QlLkCcITbH*^4I?Iqw<9!2erTX7KaXsfg5K7M8y(|>LF;+gtX zDJ+B0rvFzh8pHW7i{+Z)a7IQK$S#9nXH~1MFxmOQ#;V#-x=g)ZrOkY?pD5SH_x_fRzD764*++&a%)qX zIP~pwG%Oshd*(CTdqXH6Cmq*ow$)jsn`!A@(}XHGc9t_L1KmH9M&-2~=}0>@z1ueO zhWWS>jxxTHZrYjG%WSu`w)#{CznBu2dDrh4+LX!>b=go@{c}rRdD|c-thC#&ETjtm z5*Peb_I-{GLIj|blelQsUWRmzmoWZ;hI9rEcMCEgXQA!dU)@q&4O`c-{rJfc6@m%XIy*)nT8R35;CO>X_Hn~cS;?t_C;R~XEnxV{<7Xg zQ5VB8{Yu0NRUTZIw?%8rBhM}iHxICbjc%XVbm}FQOY50)>~02Sei`=Cp;j68(SBTk zfzYeB$<{LQ#Xe`&L>L&%=8cnrY@+j}z|+?lH>Q_;itL^FzmBg!xQJB}af z6(mxq=yB@g=hWU5Hso?OkkXHns!n%{iTs}Amujf;c;xIUhIMR1df0lD-2V#RuiD%r zj#_R=kV1a4O~!OOQcW9sOo-Ct=ic?$aC5+(o}yuy`+n1&jS{tlr4O@=$WFQY#*#aD zPLq^m+fM+8vh#Gb=r`vrW(;I+AS1u(lFEJyD7iq zlofUP+V$|=X}?nLfM;8HF6V4dIBfRw#gg1OshW}0)Y{yJ71-L)vKLgZYyT<}os5vm zHOm#hE8)r%l(}fa0}DO~U&+@@0kV;stA>u8kFw7Y-Z%N1*114&u%}|Y?tl1#@gUIk z4zc~=U)~E{&X^wK+2-C~7N>f1$O?P&lHwK=_s5t_np5C4aW&NM$w{Tp>O+Mf{aqbD z<&&gMEu8NYXe;fhiU4xW)5<^24<){2w0IBJSy#`h{*JXN;jDA$&gyXVgG??>sY@{= z>Hh9e^(aE1PBq~DC#!u^m$YJjuK?tdTGX%RMz2?Q-^V`DP0PwzLwPS$F!E+YDK zD(?Ph4fkZ1&lL)C1HXuBAmIdC>(i?_Gr%r^?IF4 z8yNAU9P@n?0+f`4rN7CAq}L@-eMw52Qos+q{l(+Akq3j#5vASs9Hi zm$cX(&){5+o%!PeSenVLdc*!11H;&|R`Cph!O4{-hRszUo<%l;%iBo0aIk6WetC@% zZ70^W^J24IOM}o`DP6FiQF5M(x2M8}8j-uMAkbi)lX96!o2Tchbj~rpu+px$bn-?L z$whFr+}_mmu}u4Az&TYp8=+cd7NIZZsPe>R^w_>QaYxo>`UHzW>AtC!ck`3a(S!W? zyVUlyC32?aKte|Z9cf+*W!5l=a5^yFK5P|x5WXYWA!xi^Gi(xqZ*M;DY9dc*^-~BI zvwPRM`K@2e^;B>u*xHw+x;=eYaM`ylLvX9k7o=!4H-NLVTbW?<;+j=nmm}<`^E#+X zH!LcC?<(1@nj{w49xm~hO=i%dMPjNF-NQ%FRQX}dRyplPajA0dUdu{h* z(5v?bja7aj?px8N`7*G+KbIB*fAiaUQTAG8B^2&MlT4h;RId)hYl&At*aPMf!Mt;) zb8J`Kr&L-)^q)Bkr3cY*G@%Uv-J^%qI`(aKi5>k~%4hi^m6}d<2h5;Exh}+awkeE+ zH{>C|;gcA6Ot!G`nH1_7IQmjA*;&cdp6Bmri$b#zGIPS7Qp(vOu1~l*c6Vx0&*6Iw zVs(gA)8O@eF#1E{qO!+dOK#Z$mA3wA_E!vlafVnsbo|1Wes zDfePxG+^`oJXigK6(b4n&Q=dCn4(rTG>RmU))n|8wo{TuXYHeY+7W0@N4kZ_bem@8 z8fZDy^(61joK{KFA%E54e7wErK}JjS_P6VK)()c(gQ?sLXD|bEkHOqs#=>>QC2aDR z5F!7={)#CFGX17HC27QrNUHNpRq={E)2>Uo=alQcuE!#vEE?~yGFO<9Wus&74SCbl zq%WhrBPg}WC;HO4wkhCq#tC37w!2xJ&jfZ2ygzw^K8&`Ygz$TKH~eNH&JzL z+S8X^Aw}R7{!yJEK zGi?`T7I;Yw1cjc2Y5Gn4y&yr>z`iY%2J%&ELTVVF2ILdt7dz9n&VIrZ`KfDKcga38 zY&dKI>a%LpL|u`m=B$!`bi3usR7CAG%FV$)hL|(Z%1=+IGQVO|dz$QEWLgOlKcv>P*dl|w>(yN4IpF?^7 zse_AF*sy+~NVJb4zJJlrBV(KQ%Da8>&nwO=pMOrn=l{Rti%ISJ;jDmLZi2g^wm!bK zlsQOV-B!AJh1ZIQsLm{jUx;h-9qJ`ZS3H!y8G1UeR;Ax*@u{hwYQLZkd@?!_Or?Qt2y1ko zwLZYs`9OGKuB{Qd;hvq=P(bX4AGF4e9x9A~iEIDqY>*;OlwzX)6r93P>0|(s$K=cj zd!4VeFe zRcIkS$_b!*T`T>7xB_Qv3QYrzY?OOK^_VS__rW<~-fvbe5?<@_WOxL*r#`9Hz;_BP z0-n-)mWt}BZ>-qw6d})=#NK8aid>ckyCI30a0B0EfWMpL=*JP;P#t%p_AlqT6Yal@ z(oRVvK6^2Jz{erP^mbaMQ~vAiGKqFCkUq|%7!ed1ZFP|~-8SIxG{#I9KRKYR(=!j- zF;X%ptft3jUUuG@SBQ}mhWI>XW3(G-LEoIn4b{>=iKsT$yct`z3B#G1GPCtjJ7+Ry zJRrN_V-hk3ldXkPHnYP$CDHyMyG+QEb?&`di8tnow4~mvXJ*e2xCBBZ2~g&!AN{3^ zo68QiPYb`lv%~+@gSuT7%xt{B#*XClXh_Qb=s@=qnKq{3w(R#zl5SbK}8v1(|h);3i|+r*-z9Q=`RP4s}cfvd|5*8Txi?j{V7k-ipXK z+p=nkx#qmMW45Zh8S2RKX*2V1F&SUpXQCDqa@H}cnE>}$j(Fej27e5liK710UnwVE zK5g5jk;>taHXHv`2qjwDl`H2VBR89`2y-N6g`{D;F9=uQ z*b!xYs*fV++eO z&fcr%g;-zyX<|9fn+L-}>_BBm=c4nHb81E}Vp3I$3X_mS>j*lCTpY1a!_Q?k*XK2C zN#r5S!n!Qp&vQmIIw1HOvWN=5F42rv#!a}X7uPH+EcI##gFGelF@?wBF`5ZvLC#OR zQV*B|e@OcD@0CV#;iAPy{8Ll~#3@sCWiYKbUyoBUy0$0s`t@e(^$Zv43fE?7onDy) z>`&O1G}JHhe{b-AQ?w3v>NKl|ry{u_A#{FPtR>eud)77VJLjBQI<8iLmmkI~d65Ig z0C~B5?FCF6>J4a^3pG-~7o!U15a2#htU@}y3(B~6vagX?wzuN5r?OxczCCrCsjinX z(SKJc_}bU*g(*N~Fap-zEu?Rz4ak;$L7#6%kQAnI0c>6lj@Yn~I8xpdj?6SRU7LXX zn;$(vEHBj<+&+7m&C+{VIngmQUe+(5FA5ghhRsbxr~v3yoD=Vv-dy?z1U%$Mpfu2I zt}%=t(d^8Kjp`f}>G3iK9pGsuqp|0KlCh@CRMz#bX;rH=b7+661zH1nDn+Y`#u`m-O+(CLVh_q9#ww* zqkb25=d0Ai;fTGImJX$tOBzK*t#s`akQgn6ddj zcKITOvZeI~JT8%3Csy7|x~nhCtwN+IuAy23DaTEIb35(i`oVA&=y3qMX4Dpl6jMGJ z8n1hCk%Z9+2^p)~+4N4s9a*fuj|!aZirsV=pfZp!26_n2(cZYD(AheO1|U^S7DeynVj8 z4|E7JYAP;lL7PuBtv;Z{rHwhC6jHi0hoX_Lpr~}f$0vC}d4m*C=Tn4@Zn%8$;kv?j zj86{D5*sA)|Cv+W>YIyG z_?OTT_$*?rgY)SBUBhD#!Yj@PIZz#d_1BsrBv|Psbd}Oe;a_Is*L2J*6qPRQ)Y0pY zUtATwCPr%NVtDX`H!^$qbz0Pt>oZSvV zQ{frsKM50P@yLSLrz%D^Js~mpJ=8Q_#Q39Wl2G0Y9sJdnz^LzenxG#%DJw(4H||g5 zWHhpBP)XdPlzW<0#xvr0u;V5=He|eVM(S{&x9!}ev8L`A{mcdV*S;PP7ld;*t)y+t zTGjGLX={prLgm@%usT=f`vV9_p1SDcRGk`svJ;oF*R+tkTXmB=aRM~}mm59R?(apH zSn`rV_VvrQrgs4csJhxk;|vfoj6;4r*~ed9)!mBDsFb7HJkhZ}eqkoRWkzz@5FTUm zfyQY$rM9eZ|7;Iyg|fR4G4ooz+X!JV60oP`syy0JrQ}$$mP(tQ zyBX^i9T$Zq&v+sY-dIqK2QAnNh2J~ovW8r=f0inI3whLLV((jWmKOm)nUl)&hz?`I zx_b~ixd)PDN6*aEKg1m@PVxlZ7NVOjS6j^}{E`+o!tKZX;!xO{6MDOWDxe#xcUW6# zES5dvtJd<-WzZWx|0Z|$Z94{LR(n*`d|pvd&Lz<(cf#iqf7Bgc@NC ztqO35-nSiF9Q&FIZ<2>6P8ojgzKHP+#DLr+jdk9dMKLPs@&3Q`R=TnMMR8%=?&N~` z-7CvjSpOSYSg09+j*AF4G<%aDS2iYsn*L^RYm}ubfCcW#P}-Xciew3P$`Wy}E}i6z zY)*{OXFqT++#f+4>N(iwT`FL_JSCkAjw57Df`b=&?w3-)*tiKbH$X#2!cfvRVrf1Zz2@h>(!pT_d|^=y|vGe#nk&OtW#s*o);XfWx|yFeS>pofgf5cb2u*4 zk?Xu&-Twq=oYYXgt8aT(Lnp|5Vv3MtIvQP>rjh zIaky7YA-6oPRtzMvo^f{Ww!YDWOE$vd1&ViR8*H1U#k)6d)xK|bThMaVmwkzaT)IK z)b%UKiAc!JTH8MbagXUkW(-!PwQw;uYhJ7JI5EjUAZrNDSk9`6@fv7-Lfp>pWRTXbmx$^c}V&%*D~@bHTLmrT$dfaSwH#}0`bJQs-cdw@6Fb|*lU8*I*-fim)UDNt>)i6xK#qdoY8@oXK&gxG z;aS_c#oWrP8(?{^$Nqkv5Itr^a`w|U(BkfgtvFd}H>oT82BNFssaC0@zbEhGj$1`GXN00DA9~3WUdE?OSJi_hRc!}O zF`bOn2Y^%$b)V-(XC;I@!!x`%vqfT>_#W?k+>!641eun0loK}F0g!PEi0R4ai4S{z zz$`RgFq|CPVSwvZ&j>vY|E&wCjYMew^6bq^MN?S~q5m^xI*!@sE%e zuk;9YwMM8ed2;Q zn}xHm;`AWUQ@HeV_R{$xe;>acFERe(HZ_~SiJka%GTX)6@NnB~Y z9n1K?D;1+@>%P{QbQfRl&n-iE3A%=mQq17iA(*t_d;HY(P(9Lm`};wT${JyZTr1I0 zeGqV?#4Lm>g!2(oeq2`95*5cv4Qy>)^a4{lB1tM|GZuk^fR~-OJ+gG#E^~`}o`56c zbn|4)GkN<7%d>e1E=1wUMYl)4~LA=9~An`M2x?VuD;vY zHj*VckXEl_aY$}%Srz^nHZD6-TT9g3tcNo1T<7_p+)g%snjO+maYXdN{=CHqWo9V^jOxthYRhd~+|P zfo5xbfe_pZoIBI-{dX&TulA9JYU=MH0r4TVg7%T^s>DV&o?2oM`T9=Z2j|7qbS8A8 z6oG`FGw1c56Fu+<=gIr8wGJ2c?Su&RFx4OSfFQWi;_@egp7!1Pgn&E!E!8Jh7Nz9^ zKr(ruAn8ai_3mA~h@qHe%kHFR(eE^tr1g+;X&PSBJZe`d$I~F1mdyXLs?f$Ph`bwi z2>pOxQy)kX4Zx06e3yvM`;{q9#B52`k`z3g05|H|r{J7LJ82s=B;@y|s=3wFY^m5n z3C-SieCN$froZ#j+-?OS+gZF;uU$_prpK9@KCsib1#AND811&OF?}28B+J}LML!KI zeP!+zN{=o?z}Pk`>+L2#(>`S@?TuWm27sv>8Z+NWGL-B~IbCjvR33cZ>C{%Xi(_bT zd6ieEh?s$zU|HwC>wTDWiOLwKCRaCPAW*JUaTMHlFvN2k4BJ}12v2LGSi!+LdIRIa zsFnk%y`)>C+4}3VRVvaVaAb=5J9KZ-u~(X%YKr86bZ#Jg-dFY$#&s-9_T~_xJ&ZI> z>#1ZFSpV`zyK=_H@e+CuZkIQZ|LY*6o2B;|@$i;kwvVl0-&MjsbpLj+_iN(VepiB` z`{)M}CB8T>ot3Bm2=&1E6|>K}y|he}@Xh4-Y_4Q9Z`s^sMeIR|OC7Pd9;NeQy9Df2 z#xt_&6o-er%Py*w^wLX`*sh@+&cMY)jggu7@(gU>T7o@HKmQK(T7Ux>GEV7N8Pv8o zOWbPNYfV0t91rrCjqM2^lNUxj#>h>d)9jNY^x}oR;ju<(3rz&o!156bU#aw*u;+q@ z=s)J8@A%rv=9%p^#u0r9n%Ie0Cp95G#Gx@OVhIXDVGu!}Pw_w_COL}TjSwob%{QxY zdDU?_MN^K>(U0e~wHeM|omE7&rE!b@G1IT=4%ctj(Ay@B5)%Vv_H(9_(WYJV?+ssS zU(h_E8_5%yGjTI|vbp{axjlPfuzm}QXBIcQ?4WxGQ*96wt=4DWTJnJPR08#90IT99 z!l!I8)Drz?(Z1HkO1s8;gTPBcDr?rmmsy%5Ldh&PR7buP_YUTq?FxsbYn#Ue-lZK>zpM>#l~ zDD-eCqiXu3C6#oYbNtYI63u6HhOq+ES!OfY9bv<<8*g|%ibs5Jm2vrc<)G%`q~Qlh z`Z`rQ7FQ7=d6EpVipuN^dm4r0QCHLLDJ64_+bnGpUB30jg;@C9CjDC3m_?4uA{OYG zngDf|sa-=uB8w88yc))CU_L7|t)Jc$#6c_sPLe{pRtoppXWqM96yzfB*9Pmf?HcbKkUCnVkzd`pw+W#xEE(20 z2e}y)M=3ieW|1$Oc$!kq=M=t=gz5?wY^|F2k6Su~1a1Fy8MnUoU5K(01pv2~)iB8( zTGw0hK$>dTzuJWB6V%~A zd23~QmqqfBRm19*4KRgRce*+$TA~MOzF6Via91BFD7?Ne6}Kyk(}VDEM)%n9z=IdU zb^e+>oMSC+PY+8;6}SN)N6m#F$6RMlnf8fq7BdjO#~40Sw4Mo0={Wm*)a>ViWO60c zQ+}3sB?^WGNAsLoVhQ%l&j9s;B*}5Am=Vjchb(e?*~WF3A~>2*L0o@jh7je=t1f?V zGH>K)=^sZ`4J340*@J~M;_ox&1$SBh=@aQ80-oba|m|bib>abw`wbLX< z{>w@Br`q}8s;;TuRfH)gyhBz~qhcYStRouZcXj`?{KdGI>&>*GtdEk5P1-iz-qAZ% z;irK2G>gW1J?@p)gt7JCsUZw0D%JT>=6k}1%!?>t>2ghV)x9cXU5lp}=Kb^3v$9gv5QS|+KjnL}-hG%5m3wIcr8SJHW!?q*UzNpwIfv@2e2K3MRk<;Z6b!G|Yx zNB9}S0{7Dt=&Akqm~*Bm`VMD7F%!?q+6PeEreT&q{A`SBN1F|I%FeBN?1D5+z!nn zZKRX_C85cD`{Vc<$X6Nwj$-CFH-*!Atkr3o^-(W3+3$e|oiK*66S8{4+8k`aroC#( z7D@?)?y|0mg$u(mCUqcXyYpiepS9f4d!cM#Z9S!N{~U?`zUn>?okYq9*_`#3_O^-+ zeF|l0v%Q)vwmJcxpciyZSTt3w$-=R#&};c=pgqr2tvD!&yl%Pa#5$f&y5DiKc)DwH#UjUHYKF>i<>B zm#6%p<2}s^Rer3;caSCLhZ#aS?Wy+-y2Eq!;8bkoq^<4Df^@BRG`@gb05vg4vdd_$ zhjQHgcfSL;G~H11B93?BXl04?!^m#4&@BXZY+p*KAMbX`IPdFnnHtg9+$<$de?O;O z$zL@VmDh^3x5>t@4kTk3VeoSuDEKCGrY%a;6vThA-)O|C&_|NSDLkZJP&iW&hc0Gn z2th6_PXy19W=k1ELhO3zsC8$jhDT}m0cu>n*6v3=&~$3No*5@ePxz3K@R}PE$O3UNoaHWG@qZy>7vEVj?3s zKCxMeY8!sHD7a4)fx$3&@jckk5mFo|B%Y>~=wT*{rumt7(f#nXwP-y^E}G6gvTa=SaW&p*rJ%*U(~aH4g7&zBAR> znu5;mSE^0C;@7<%jXmIOc<*OxJ^dLd|86~0k{S>Xfk++Z5N}>Et|>?|5`b6$*7j)% z{|&!XYz*f@f}Ghkjp7G}R9vCI%n8vyR6Uq+#g5=-Vg&~(TiX@tdE|>pA@&!xrw+pO zfSH>SeOIVQOl7+6(~|)FnX~@c$~nG??7@0Bumo@3(Pdr;HtBB>jlFhzhT@SLS1?`4 z&?Lzm{Bq4G-o3VQ_Q>{P*+v6&8&b4>&?a3e#3P?sRLGSzB!rvn`}I`jQFeC8tsZ0} z6!r)m7lFcW>$0pYnDmIay>?i+-<^iRHT9mbAz~&vD=HSNN(79T-HMD7GVJmhc|!fr zPANqfHEF@=!VF0c-Gs3FnhT>{bc;lrj#8LVxj-kE{dE$C@v2VbFL*_;!M(L_q0pXF zm-M2rjBxVm33J*WBovTIF|j&N;U~N@+R(sjF}awoS`d&CzT%Jcp}Z^Fm+DNXmy7pi z@s~XOgBw85PhnKtq3=}lg@xqLOTM9Q+aa0G`bkJK%nUw0XP^IR&Muv0St|zGxY>|x zulx&hPKkaIeKB{kJTR(d@!wU}Y|&X>;K&ls->H4YLZw1u3Ej>R=ay>y9)6xzmb+aI z%U^EW-rh`rM)aK1W)t{(BN$NL#R`)1%HFe`L+5d^XxbG6{ZWHgAJiC~8gcj2Eu@Y% z2^alv)X-YyMJP2|ef7^b_29~7d1ByXO0Uq1;LIL?9iv?-Fkmyzi>|7s!AK7CaqdZtEp2E>QnLg(XxCd)o7EMIQxJS=u^js#%Pn^)fZ34vXV&`85x2&R5T_Uz4;= z@~AA>f#o%^loFa|k1Wa>mjOQjjm^=##L0~-%n~i9kunnLaNS`S8b@HUtV0fA*JM6=Am9s^v4|pDu_Sdu|a3z_l>D+`gs*aDS#;K z`SEJK(wE`D)GDHBBe`q$wfPqE;j@U4)~W3F5ck>d%@C;JZe5LrvE4ODXuQNt@6v=> zlm36Rmr~E|vN2xyr0<>xW2N4LB}Sy`lM4@@-@Z7W5aJgGkP1g6)LA8W72-_i z`L?<85qkI9bZe}av38y&TykWwN;LXmwV9OgM}Blz*IZ_S}jUXGZ5^qo*f3yT$E&C&4OFY#3ws?Z_{ui8Cqp2n=(#G ztO1El*!=`yAM5Bz5V2u*cdP9;8F>G^d)ZX0u@^>22!?yT-x4)Q4ooVAp8n>G9tm5s zg%~~i?UBlp^3y9KHfnUcVk~k^bF&ZP z*d87QawTu(2c$SzeW+H0&q}&B$r{y?ByjbQsczR+`1g0 z0*+MK3pM_#JDb9V4?W~?0a;IpR16!+ha6DHsAngAU{u#so4K|F3i;@GJh{RQ09e0l@1G5x5v}#1u`AFHHPqxKjRvOxPjGb#DZChYl%Pp z|LR@R*h~b6dZ=+h@sn`WZJis0H8MNUJx`AV6S5i%khxPXuJXDtFw}@uou9kN8hy&~ zPrbY+%}Csn8sDx5*@X%jFBv~y+0`Pi1el*Jk5ape@l<2{!eOWFo=Hn}rJgraVLd5S z&+qrKWz)Y@DGIEGNv`#U;=Pu^Va$qYTE12-MaVp0a7&P079Oh}cK5wWYe9F3aF zUO!XJmeMAHtX3I;JQKa!>IzD`zY}*_i+ohZfU+5bFDqMhj!L<))0Yht{!W*LC}-t8 z=pRCoJAG}vA4_XI z-OT-DE5Oo;^DdVhISSmVcn}qJNftsqRx5A2&BM%8Z!9jY9J&$rakk0(cdT2@xhc5> zm}R~Dv(g8Lyc;c14V)`g|FRe^u4i)fTd>AcMKJ;LyaH;EAzA@x~zkt=RXt7#< z=Kn(}^3k+ek&foLGkcd^>vv+S@Zm~g6^d;=qE4;hmr zv)sI&Ol*E?8N~@0ca!_+T&feAl}C^(#|o*;*^Z`bADE`!e%}e2* zYbN!`tpT$LUtz-=jkh5N;c|&^4z>>}fJGE*^L1v^)`BMI<$9k7*nb{dvR@xJWyW2+ z-=@88a?F`pu-kxJv{7+el$+1oQK7#V_bs3+h<*+Dq*F3jV8u=$L{gmB1bx4*U)cC# z2<6$0%;WriussZWvBEWDIOG5R1UG%@Z=R?={w`eVS!c^`z1?Z&KvVOWrrZM}EwoxW zj-;hAfIt-4%S!P2xciwb;uqoz(f;#dM6S1U5mmUxTxA9}>l+9?vtaZYU$yXw=6i`~ z*mwjna8x+1BD9u*{d9$2wSxeY;BH#p|VdAY+ex1#14vT!qk zyl_o^G8=Zw6P7qZD(x*-z;s9cCb`N23HbiHn&!nh7oTSy7VLS=rBVNW(l4k!A*o@z zv%FV$hpbkY@{}t+YIi7BV|aVZ?1xxb0Grw|k)`PdD`$XLYbTNxfz;G+{U>G+48RE_ zN#9N2QcwX~CU*6kpZ32`Nx=@Xp0VUZgoYPCRQN&sdp&7xHy#m&Ec}qjG!MM=hY49cSYeRAk|0a)vP&~sp36~QDHlR_re+*&pjqw;cygCz8qUiB`Evw^vy7fkb8a;P z3%RFtIn#a&Wi`rq*yJNCkTLteD-M1e10EbM>fE6E1#Xa+))5wymPVY!GY(}J4uOM{ z!UV}sW8vsf*9znRoUK?R8!cxnF89Pp<`|z)qd-;rzUhMzhwcLPv)reRa!yXq2d**z zbRh@nik*C5`l|!YSLGk!3_2%#DfcgETtbxIT<@A1iR=zq5oyx1tNgI=xvWSmOf?4a zxuZxF>YiT0!18Rcz@4kTOg97AThMJ`tB_TN~{o2&sL)^1+`$JQo}J{+xf`4B2GNSZ5S}1n@jaP2Cd( z1arINT5VsiR8f(EPI@;YfGs0JoBmdWR-97~#2zVjOPg#A;3p-Wof&pn`PU*%KM zMb+;{y`BZf>{RZGV7HBkoe4hJl50vaCvf{;N@I0E%B1*M*nVYr2JV9C_4H>rkym7Y zi7Ie+IgqQ`XdO0Td1-F3zfsNr5@9^-mwKZE5{xg$K9jcLRqqKqoE|~OnUeNjM|wJP zIhB40;O#*m%W;hp%f!s{1@Xo+a@8i2rq!7O#WcC6wd}7$Xi^IH-=j)*-_FjW_>0ja zfB|^Pg;Q{2c%>`FU?J}LQFMYtP;)IN(Pcy}rmgCtm11Coh;53QmC#dglD&eFxt6r; zadvjXu|pCN;UjXTGLm?{geD?kt-`=k>|17D|a+ z3O{Vv+^)Bnon=xWceI&l82{YULKSqhYfkn?JA}7ueIxQcJe5~>I^YdVoL-oGpU?n3 zTDG@<-HGt%TIDgz5oOpF4(!voP-#s8n?BSl3EGm6q{57_GOCtbDVY=Z)ossfc_#gZ z{^W=5*UQM&gWH4nyEHT`ik4F&5H3mQ)vrP4ocv;!SZnd-M$o6gWS>rkJ(nN%fh2k= zhx89AjlU;J1Z|Ckpz!y3dNmAQgod&v+*WBofhkheqhe1=QU^2cNqNte6564OT77-* z@I03peJzKr23+sin;OcdGx}#_>GE%c$V(at{_bGw0EdgDK1P8X^FqPoZSh(a6rFfj z#ZgXur&h_Xm|YQwPZ*MO#kM(u^PFxU_Q*B;ZO^N}E;NP_FlsZ3u0RnY^q6x`5^B|5 zaKEH05O?rp*_%q!C*TZnvpl0wL=0!7ej5LQ`?89x%gh-bG8bq+g-$kNIsL=x#9pHK zXO2C)S=l>y6wTZdoYWJIz?mgByjMI?#(BAc5%KuAH?vyTKGUu>LJKZAq!whk0s!)@ z8S*@I@VM&P@Fe?5w%`$3H*()@&AI@&IY@VGc~rAf3AX42flV1 z=4tn?T{VRZR>KzECls^79#@#IG=evMRvVyC3||r7OlVCup3kdvHJ~bCg4=3Aj0ndRh53#xsZ;BA^Rao*{sJkqb^L}9^h-WC5%@Y_6Mvc6S{|ld1 zC>bsxnQ$w0TbCQ1bVzJhT|6`bu~lpa033FXrlv-ADP0-{rW~U*wS(%x(Be0tRenC+U(RDqA{GFijl8<9F^TptKyvu2ox}21i zjSbfPMYW_EA0a=)lld}M)_$QGgVo?e*?{ed9beL<*x0l9u7*XwB=gAxbuI`WG=agy z*bdNyd@X^-&=>tK3&Sz!Vxm@$YwMgkP37$;pu<%*TzL}P&(=+-oQ=u}6cs>^$F@5Q zK<2OT5c*_SkaH=ic9}glLgI#QTcO%x;>xCRwOlj8YuwQoA5&B22~9o+9C1-*nP)g% zvQeb87_PKz5h?$G>AUwDn}ysmrAzxTFwm5^>^0xnrs#vyO2*@s8SRF)^DFm&r~U7% zOqgd%8^@Ptkg(tu1rmJf80=^p_L%k`UO3_?b6Z+i<>i*BaLf!|Qis|7I7_j-KOT58 ze_5;E08<_s(&BDvJxv!}I>^6W&`H+r^NmrrT6uQ>Yard}~XvnrJGOtsrA` zpv}l&t4$i3FLfhPUU22(#IpewdRjm#0D~-R{j9;yAXX6PBbzT~3GP{%B z`B_IfR6prS4OZjd`C6Id%^z^x>v)rF1znrLSl~M{uqfr-muTOsSaesUda+0l*t)c$ zvIAtGS^GLtHbVH-$LoS?;!L#Y1Vt3lJAUSe--VO(_45gZb)%X00ikXUp%?YvnwFBM zzP~eGX07bASz`6-*8(15&r8ACnb2vLCxh0`T|2+$AmY(3_C+gEWFXq-VkArckE#fr ziQcTP013XU&sv2Ih(|Fc)sj@0X)cSqO;O?0^MZ?6SMtJ>GerE*9=GZzu(>`sV%mi5 z9<`@ul5N&O64gY6an0gJ+EC*_Nbv*R?V{>~mYff9lY^qN_Z}+-oP~pkP*@dAkY#32 z3&d7zNU$?Fsvd-{dOY9LKNV_kU z8xsG_Q5bx_G7W^H%RS3ou8Sn`fp+w}`)$Cp%m%kp(BETn#)5WiTG834)f(Eh#k`_@ zQk376yM!9idtAY|StoCn8*#XqHC^YgouT#h;8tri5w3nG* zg9ARf?ZQKKRZILi1wpx3w^kcYR;i5|&&=0rOArriEY_zXqiO8j>Rf*ayt6kP*J}cJ z=*ISUO{MvbgREC>%nzVb`f)3p_O^&*{?eB8Em|MVOZ^nEAYjlPLC4Tfz1b-BbQ5Px z4aPmFIDXQ6l|$S>nHTCUvs3|{JS^X~g!LXeS@C#JpD4oy@9TMPYml`Rdgr;-|5GkQUq<4@SS`s0& z&_Y7*FW=|O%yG;!&olRRou@uu+;(qYJjL8)H(2>Wuhn5=d%M?F_uGBnEYZ47ou-QT zhq8xd3>B`>@wnHGCS-OF{%LR+q!8dA47Z(EsFxo#{7UEvHY$lw*kWxSl#FelE@`_~ zd9%)Fy06tS)h@o`YaXR8rL$G`#wyHFdQjcON>fdH-|Lw+y(lI*f{y2<=xGw0u(HL(kWUmi7LIvMa;=498pph@njek9 zd^`%Tk^bYCTKLrp82G(Jtrmd1Wj(E{8|HQ0lVt80jl>Qk#0}t146uDCTkCi7Og;gu z^{RoE9QYI-j!47#o_}zr>^Ok3ZM@vKboJfs=+XK$0JP0-*?O*B*_aBMPaSuIfCE