Fix cylc#4133 - better error handling when rmtree fails

cylc/flow/exceptions.py

@@ -16,7 +16,7 @@
 """Exceptions for "expected" errors."""
 
 
-from typing import Iterable
+from typing import Callable, Iterable, NoReturn, Tuple, Type
 
 
 class CylcError(Exception):
@@ -99,6 +99,31 @@ class WorkflowFilesError(CylcError):
     """Exception for errors related to workflow files/directories."""
 
 
+def handle_rmtree_err(
+    function: Callable,
+    path: str,
+    excinfo: Tuple[Type[Exception], Exception, object]
+) -> NoReturn:
+    """Error handler for shutil.rmtree."""
+    exc = excinfo[1]
+    if isinstance(exc, OSError) and exc.errno == 39:
+        # "Directory not empty", likely due to filesystem lag
+        raise FileRemovalError(exc)
+    raise exc
+
+
+class FileRemovalError(CylcError):
+    """Exception for errors during deletion of files/directories, which are
+    probably the filesystem's fault, not Cylc's."""
+
+    def __init__(self, exc: Exception) -> None:
+        CylcError.__init__(
+            self,
+            f"{exc}. This is probably a temporary issue with the filesystem, "
+            "not a problem with Cylc."
+        )
+
+
 class TaskRemoteMgmtError(CylcError):
     """Exceptions initialising workflow run directory of remote job host."""
 

cylc/flow/pathutil.py

@@ -23,7 +23,7 @@
 
 from cylc.flow import LOG
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
-from cylc.flow.exceptions import WorkflowFilesError
+from cylc.flow.exceptions import WorkflowFilesError, handle_rmtree_err
 from cylc.flow.platforms import get_localhost_install_target
 
 
@@ -239,7 +239,7 @@ def remove_dir_and_target(path: Union[Path, str]) -> None:
             target = os.path.realpath(path)
             LOG.debug(
                 f'Removing symlink target directory: ({path} ->) {target}')
-            rmtree(target)
+            rmtree(target, onerror=handle_rmtree_err)
             LOG.debug(f'Removing symlink: {path}')
         else:
             LOG.debug(f'Removing broken symlink: {path}')
@@ -248,7 +248,7 @@ def remove_dir_and_target(path: Union[Path, str]) -> None:
         raise FileNotFoundError(path)
     else:
         LOG.debug(f'Removing directory: {path}')
-        rmtree(path)
+        rmtree(path, onerror=handle_rmtree_err)
 
 
 def remove_dir_or_file(path: Union[Path, str]) -> None:
@@ -268,7 +268,7 @@ def remove_dir_or_file(path: Union[Path, str]) -> None:
         os.remove(path)
     else:
         LOG.debug(f"Removing directory: {path}")
-        rmtree(path)
+        rmtree(path, onerror=handle_rmtree_err)
 
 
 def get_next_rundir_number(run_path):

cylc/flow/workflow_files.py

@@ -39,15 +39,18 @@
     PlatformLookupError,
     ServiceFileError,
     TaskRemoteMgmtError,
-    WorkflowFilesError)
+    WorkflowFilesError,
+    handle_rmtree_err
+)
 from cylc.flow.pathutil import (
     expand_path,
     get_workflow_run_dir,
     make_localhost_symlinks,
     parse_dirs,
     remove_dir_and_target,
     get_next_rundir_number,
-    remove_dir_or_file)
+    remove_dir_or_file
+)
 from cylc.flow.platforms import (
     get_install_target_to_platforms_map,
     get_localhost_install_target,
@@ -839,7 +842,7 @@ def remove_keys_on_server(keys):
     # Remove client public key folder
     client_public_key_dir = keys["client_public_key"].key_path
     if os.path.exists(client_public_key_dir):
-        shutil.rmtree(client_public_key_dir)
+        shutil.rmtree(client_public_key_dir, onerror=handle_rmtree_err)
 
 
 def create_server_keys(keys, workflow_srv_dir):