Fix ENOSPC for extended quota

When unlinking multiple files from a pool at 100% capacity, it
was possible for ENOSPC to be returned after the first few unlinks.
This issue was fixed previously by PR openzfs#13172 but then this was
again introduced by PR openzfs#13839.

This is resolved using the existing mechanism of returning ERESTART
when over quota as long as we know enough space will shortly be
available after processing the pending deferred frees.

Also, updated the existing testcase which reliably reproduced the
issue without this patch.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Reviewed-by: Dipak Ghosh <dipak.ghosh@hpe.com>
Signed-off-by: Akash B <akash-b@hpe.com>
Closes openzfs#15312

module/zfs/dsl_dir.c

@@ -26,6 +26,7 @@
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
  * Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
+ * Copyright (c) 2023 Hewlett Packard Enterprise Development LP.
  */
 
 #include <sys/dmu.h>
@@ -1358,30 +1359,23 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree,
 		ext_quota = 0;
 
 	if (used_on_disk >= quota) {
+		if (retval == ENOSPC && (used_on_disk - quota) <
+		    dsl_pool_deferred_space(dd->dd_pool)) {
+			retval = SET_ERROR(ERESTART);
+		}
 		/* Quota exceeded */
 		mutex_exit(&dd->dd_lock);
 		DMU_TX_STAT_BUMP(dmu_tx_quota);
 		return (retval);
 	} else if (used_on_disk + est_inflight >= quota + ext_quota) {
-		if (est_inflight > 0 || used_on_disk < quota) {
-			retval = SET_ERROR(ERESTART);
-		} else {
-			ASSERT3U(used_on_disk, >=, quota);
-
-			if (retval == ENOSPC && (used_on_disk - quota) <
-			    dsl_pool_deferred_space(dd->dd_pool)) {
-				retval = SET_ERROR(ERESTART);
-			}
-		}
-
 		dprintf_dd(dd, "failing: used=%lluK inflight = %lluK "
-		    "quota=%lluK tr=%lluK err=%d\n",
+		    "quota=%lluK tr=%lluK\n",
 		    (u_longlong_t)used_on_disk>>10,
 		    (u_longlong_t)est_inflight>>10,
-		    (u_longlong_t)quota>>10, (u_longlong_t)asize>>10, retval);
+		    (u_longlong_t)quota>>10, (u_longlong_t)asize>>10);
 		mutex_exit(&dd->dd_lock);
 		DMU_TX_STAT_BUMP(dmu_tx_quota);
-		return (retval);
+		return (SET_ERROR(ERESTART));
 	}
 
 	/* We need to up our estimated delta before dropping dd_lock */

tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh

@@ -17,6 +17,7 @@
 #
 # Copyright (c) 2014, 2016 by Delphix. All rights reserved.
 # Copyright (c) 2022 by Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023 Hewlett Packard Enterprise Development LP.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -51,11 +52,20 @@ log_must zfs create $TESTPOOL/$TESTFS
 log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
 log_must zfs set compression=off $TESTPOOL/$TESTFS
 
-log_note "Writing files until ENOSPC."
+log_note "Writing Big(1G) files until ENOSPC."
 log_mustnot_expect "No space left on device" fio --name=test \
     --fallocate=none --rw=write --bs=1M --size=1G --numjobs=4 \
     --sync=1 --directory=$TESTDIR/ --group_reporting
 
+log_must rm $TESTDIR/test.*
+log_must test -z "$(ls -A $TESTDIR)"
+sync_pool $TESTPOOL true
+
+log_note "Writing small(10M) files until ENOSPC."
+log_mustnot_expect "No space left on device" fio --name=test \
+    --fallocate=none --rw=write --bs=1M --size=10M --numjobs=200 \
+    --sync=1 --directory=$TESTDIR/ --group_reporting
+
 log_must rm $TESTDIR/test.*
 log_must test -z "$(ls -A $TESTDIR)"