From 8e11af89429264dbe2cf93e989b4ef6f35fb1ad3 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 8 Mar 2022 09:16:35 -0800 Subject: [PATCH] Fix ENOSPC when unlinking multiple files from full pool When unlinking multiple files from a pool at 100% capacity, it was possible for ENOSPC to be returned after the first unlink. e.g. rm -f /mnt/fs/test1.0.0 /mnt/fs/test1.1.0 /mnt/fs/test1.2.0 rm: cannot remove '/mnt/fs/test1.1.0': No space left on device rm: cannot remove '/mnt/fs/test1.2.0': No space left on device After waiting for the pending deferred frees from the first unlink to be processed the remaining files can then be unlinked. This is caused by the quota limit in dsl_dir_tempreserve_impl() being temporarily decreased to the allocatable pool capacity less any deferred free space. This is resolved using the existing mechanism of returning ERESTART when over quota as long as we know enough space will shortly be available after processing the pending deferred frees. Reviewed-by: Alexander Motin Reviewed-by: Ryan Moeller Reviewed-by: Tony Hutter Signed-off-by: Brian Behlendorf Closes #13172 --- include/sys/dsl_pool.h | 1 + module/zfs/dsl_dir.c | 20 +++++-- module/zfs/dsl_pool.c | 6 ++ tests/runfiles/common.run | 2 +- .../tests/functional/no_space/Makefile.am | 3 +- .../tests/functional/no_space/enospc_rm.ksh | 60 +++++++++++++++++++ 6 files changed, 84 insertions(+), 8 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh diff --git a/include/sys/dsl_pool.h b/include/sys/dsl_pool.h index 371fe67acd9e..1177929fa3dc 100644 --- a/include/sys/dsl_pool.h +++ b/include/sys/dsl_pool.h @@ -162,6 +162,7 @@ int dsl_pool_sync_context(dsl_pool_t *dp); uint64_t dsl_pool_adjustedsize(dsl_pool_t *dp, zfs_space_check_t slop_policy); uint64_t dsl_pool_unreserved_space(dsl_pool_t *dp, zfs_space_check_t slop_policy); +uint64_t dsl_pool_deferred_space(dsl_pool_t *dp); void dsl_pool_wrlog_count(dsl_pool_t *dp, int64_t size, uint64_t txg); boolean_t dsl_pool_wrlog_over_max(dsl_pool_t *dp); void dsl_pool_dirty_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx); diff --git a/module/zfs/dsl_dir.c b/module/zfs/dsl_dir.c index 43ef1897811c..aca32ff9bbb9 100644 --- a/module/zfs/dsl_dir.c +++ b/module/zfs/dsl_dir.c @@ -1334,13 +1334,21 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree, /* * If they are requesting more space, and our current estimate * is over quota, they get to try again unless the actual - * on-disk is over quota and there are no pending changes (which - * may free up space for us). + * on-disk is over quota and there are no pending changes + * or deferred frees (which may free up space for us). */ if (used_on_disk + est_inflight >= quota) { - if (est_inflight > 0 || used_on_disk < quota || - (retval == ENOSPC && used_on_disk < quota)) - retval = ERESTART; + if (est_inflight > 0 || used_on_disk < quota) { + retval = SET_ERROR(ERESTART); + } else { + ASSERT3U(used_on_disk, >=, quota); + + if (retval == ENOSPC && (used_on_disk - quota) < + dsl_pool_deferred_space(dd->dd_pool)) { + retval = SET_ERROR(ERESTART); + } + } + dprintf_dd(dd, "failing: used=%lluK inflight = %lluK " "quota=%lluK tr=%lluK err=%d\n", (u_longlong_t)used_on_disk>>10, @@ -1348,7 +1356,7 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree, (u_longlong_t)quota>>10, (u_longlong_t)asize>>10, retval); mutex_exit(&dd->dd_lock); DMU_TX_STAT_BUMP(dmu_tx_quota); - return (SET_ERROR(retval)); + return (retval); } /* We need to up our estimated delta before dropping dd_lock */ diff --git a/module/zfs/dsl_pool.c b/module/zfs/dsl_pool.c index 568799f2bacd..3c54da1395e0 100644 --- a/module/zfs/dsl_pool.c +++ b/module/zfs/dsl_pool.c @@ -950,6 +950,12 @@ dsl_pool_unreserved_space(dsl_pool_t *dp, zfs_space_check_t slop_policy) return (quota); } +uint64_t +dsl_pool_deferred_space(dsl_pool_t *dp) +{ + return (metaslab_class_get_deferred(spa_normal_class(dp->dp_spa))); +} + boolean_t dsl_pool_need_dirty_delay(dsl_pool_t *dp) { diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index eb0d451351f0..fdc2172b6a65 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -694,7 +694,7 @@ tags = ['functional', 'nestedfs'] [tests/functional/no_space] tests = ['enospc_001_pos', 'enospc_002_pos', 'enospc_003_pos', - 'enospc_df'] + 'enospc_df', 'enospc_rm'] tags = ['functional', 'no_space'] [tests/functional/nopwrite] diff --git a/tests/zfs-tests/tests/functional/no_space/Makefile.am b/tests/zfs-tests/tests/functional/no_space/Makefile.am index c2e42bc2ada4..31584fb17583 100644 --- a/tests/zfs-tests/tests/functional/no_space/Makefile.am +++ b/tests/zfs-tests/tests/functional/no_space/Makefile.am @@ -5,7 +5,8 @@ dist_pkgdata_SCRIPTS = \ enospc_001_pos.ksh \ enospc_002_pos.ksh \ enospc_003_pos.ksh \ - enospc_df.ksh + enospc_df.ksh \ + enospc_rm.ksh dist_pkgdata_DATA = \ enospc.cfg diff --git a/tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh b/tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh new file mode 100755 index 000000000000..065abc75977e --- /dev/null +++ b/tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh @@ -0,0 +1,60 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2014, 2016 by Delphix. All rights reserved. +# Copyright (c) 2022 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/no_space/enospc.cfg + +# +# DESCRIPTION: +# After filling a filesystem, verify the contents can be removed +# without encountering an ENOSPC error. +# + +verify_runnable "both" + +function cleanup +{ + destroy_pool $TESTPOOL + log_must rm -f $all_vdevs +} + +log_onexit cleanup + +log_assert "Files can be removed from full file system." + +all_vdevs=$(echo $TEST_BASE_DIR/file.{01..12}) + +log_must truncate -s $MINVDEVSIZE $all_vdevs + +log_must zpool create -f $TESTPOOL draid2:8d:2s $all_vdevs +log_must zfs create $TESTPOOL/$TESTFS +log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS +log_must zfs set compression=off $TESTPOOL/$TESTFS + +log_note "Writing files until ENOSPC." +log_mustnot_expect "No space left on device" fio --name=test \ + --fallocate=none --rw=write --bs=1M --size=1G --numjobs=4 \ + --sync=1 --directory=$TESTDIR/ --group_reporting + +log_must rm $TESTDIR/test.* +log_must test -z "$(ls -A $TESTDIR)" + +log_pass "All files removed without error"