Merge branch 'test-improvements' into develop

cerulean/sftp_file_system.py

@@ -321,7 +321,7 @@ def _set_permission(
             mode = mode | permission.value
         else:
             mode = mode & ~permission.value
-        self._chmod(lpath, mode)
+        self._chmod(lpath, mode)    # type: ignore[arg-type]
 
     def _chmod(self, path: AbstractPath, mode: int) -> None:
         self.__ensure_sftp()

cerulean/ssh_terminal.py

@@ -73,8 +73,15 @@ def _get_sftp_client(self) -> paramiko.SFTPClient:
             An SFTP client object using this terminal's connection.
 
         """
-        self.__transport = self.__ensure_connection(self.__transport)
-        client = paramiko.SFTPClient.from_transport(self.__transport)
+        tries = 0
+        while tries < 3:
+            try:
+                self.__transport = self.__ensure_connection(self.__transport)
+                client = paramiko.SFTPClient.from_transport(self.__transport)
+                break
+            except paramiko.ssh_exception.SSHException as e:
+                tries += 1
+
         if client is None:
             raise RuntimeError('Could not open a channel for SFTP')
         return client
@@ -176,12 +183,14 @@ def __get_data_from_channel(
                 new_data = receive(channel, 1024 * 1024)
         except socket.timeout:
             return False, data.decode('utf-8')
+        except ConnectionError:
+            return False, data.decode('utf-8')
 
         return True, data.decode('utf-8')
 
     def __get_key_from_file(
             self, filename: str, passphrase: Optional[str]) -> paramiko.pkey.PKey:
-        key = None
+        key: Optional[paramiko.pkey.PKey] = None
         messages = ''
         try:
             key = paramiko.ed25519key.Ed25519Key.from_private_key_file(

cerulean/test/conftest.py

@@ -1,4 +1,7 @@
+from contextlib import contextmanager
+from paramiko.ssh_exception import SSHException
 import pytest
+import socket
 from typing import Any, Dict, Generator, Tuple
 
 from cerulean.credential import PasswordCredential
@@ -20,6 +23,46 @@
 # PyTest does not export FixtureRequest, the type of the request attribute.
 # So they're annotated as Any.
 
+
+# TODO: use pytest-rerunfailures instead of the manual retry loop?
+
+# Number of times to try tests interrupted by network issues (for the flaky server)
+NUM_TRIES = 3
+
+
+@contextmanager
+def abort_on_network_error() -> Generator:
+    """Swallows network errors but not other errors.
+
+    Used to save some code when retrying a test when a network error occurs. That code
+    then looks like
+
+    while retries left:
+        with abort_on_network_error():
+            ...run test...
+            break
+
+    If a network error occurs during testing and an exception is raised, then we jump
+    back here and eat the exception, after which the with-block is exited and we run the
+    while loop again to retry.
+    """
+    try:
+        yield
+
+    except SSHException:
+        pass
+    except ConnectionError:
+        pass
+    except socket.timeout:
+        pass
+    except OSError as e:
+        x = f'{e.errno}|{e.strerror}|{e}|{e.args}'
+        if e.errno == 'Socket is closed':
+            pass
+        assert x == 'nope'
+        raise
+
+
 @pytest.fixture(scope='module')
 def password_credential() -> PasswordCredential:
     return PasswordCredential('cerulean', 'kingfisher')
@@ -59,13 +102,14 @@ def webdav_filesystem_quiet() -> Generator[WebdavFileSystem, None, None]:
 
 @pytest.fixture(scope='module', params=['local', 'sftp', 'webdav'])
 def filesystem(
-        request: Any, ssh_terminal: SshTerminal
+        request: Any, password_credential: PasswordCredential
         ) -> Generator[FileSystemImpl, None, None]:
     if request.param == 'local':
         yield LocalFileSystem()
     elif request.param == 'sftp':
-        with SftpFileSystem(ssh_terminal) as fs:
-            yield fs
+        with SshTerminal('cerulean-test-sftp', 22, password_credential) as term:
+            with SftpFileSystem(term) as fs:
+                yield fs
     elif request.param == 'webdav':
         with WebdavFileSystem('http://cerulean-test-webdav/files') as wfs:
             yield wfs
@@ -78,8 +122,8 @@ def filesystem2(
     if request.param == 'local':
         yield LocalFileSystem()
     elif request.param == 'sftp':
-        # don't use the ssh_terminal fixture, we want a separate connection
-        with SshTerminal('cerulean-test-ssh', 22, password_credential) as term:
+        # Intentionally a separate connection
+        with SshTerminal('cerulean-test-sftp', 22, password_credential) as term:
             with SftpFileSystem(term) as fs:
                 yield fs
     elif request.param == 'webdav':
@@ -166,7 +210,6 @@ def terminal(
     'local_direct',
     'ssh_direct',
     'ssh_torque-6',
-    'ssh_slurm-16-05',
     'ssh_slurm-17-02',
     'ssh_slurm-17-11',
     'ssh_slurm-18-08',
@@ -177,7 +220,8 @@ def terminal(
     'ssh_slurm-23-02',
     'ssh_slurm-23-11',
     'flakyssh_direct',
-    'flakyssh_slurm-17-11'])
+    'flakyssh_slurm-17-11'
+    ])
 def scheduler_and_fs(
         request: Any, ssh_terminal: SshTerminal,
         password_credential: PasswordCredential