Merge branch 'master' into cdt-vtools-2043-report-cmd-line

.github/workflows/python.yml

@@ -11,6 +11,7 @@ jobs:
         with:
           python-version: '3.8'
       - name: Install Tox
-        run: pip install tox==3.28.0
+        run: pip install tox==4.11.0
+
       - name: Run Tox
         run: tox -e style,py38,cover

.gitignore

@@ -97,6 +97,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+.virtualenvs/
 
 # Spyder project settings
 .spyderproject

.readthedocs.yaml

@@ -0,0 +1,20 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.7"
+
+sphinx:
+  configuration: docs/conf.py
+
+python:
+  install:
+    - requirements: docs/requirements.txt
+    - method: setuptools
+      path: .

README.md

@@ -1,3 +1,7 @@
+[![Documentation Status](https://readthedocs.org/projects/ducktape/badge/?version=0.8.x)](https://ducktape.readthedocs.io/en/0.8.x/?badge=0.8.x)
+
+
+
 Distributed System Integration & Performance Testing Library
 ============================================================
 
@@ -17,7 +21,7 @@ Ducktape contains tools for running system integration and performance tests. It
 Documentation
 -------------
 
-For detailed documentation on how to install, run, create new tests please refer to: http://ducktape-docs.readthedocs.io/
+For detailed documentation on how to install, run, create new tests please refer to: http://ducktape.readthedocs.io/
 
 Contribute
 ----------

Vagrantfile

@@ -23,7 +23,7 @@ VAGRANTFILE_API_VERSION = "2"
 enable_dns = false
 num_workers = 3
 ram_megabytes = 300
-base_box = "ubuntu/trusty64"
+base_box = "ubuntu/focal64"
 
 local_config_file = File.join(File.dirname(__FILE__), "Vagrantfile.local")
 if File.exists?(local_config_file) then
@@ -51,7 +51,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     name = "ducktape" + i.to_s
     config.vm.define name do |worker|
       worker.vm.hostname = name
-      worker.vm.network :private_network, ip: "192.168.50." + (150 + i).to_s
+      worker.vm.network :private_network, ip: "192.168.56." + (150 + i).to_s
     end
   }
 

docs/README.md

@@ -1,23 +1,35 @@
 Ducktape documentation quick start guide
 ========================================
 
-This file provides a quick guide on how to compile the Ducktape documentation.
 
+Build the documentation
+-----------------------
 
-Setup the environment
----------------------
+To render the pages run::
+```shell
+tox -e docs
+```
+
+The rendered pages will be in ``docs/_build/html``
 
-To compile the documentation you need Sphinx Python library. To install it and all its dependencies run::
 
-    pip install -r requirements.txt
+Specify documentation format
+----------------------------
 
+Documentation is built using [sphinx-build](https://www.sphinx-doc.org/en/master/man/sphinx-build.html) command.
+You can select which builder to use using SPHINX_BUILDER command:
+```shell
+SPHINX_BUILDER=man tox -e docs
+```
+All available values: https://www.sphinx-doc.org/en/master/man/sphinx-build.html#cmdoption-sphinx-build-M
 
-Build the documentation
------------------------
 
-To render the pages run::
+Pass options to sphinx-build
+----------------------------
+Any argument after `--` will be passed to the 
+[sphinx-build](https://www.sphinx-doc.org/en/master/man/sphinx-build.html) command directly:
+```shell
+tox -e docs -- -E
+```
 
-    make html
-
-The rendered pages will be in ``docs/_build/html``
 

docs/changelog.rst

@@ -0,0 +1,31 @@
+.. _topics-changelog:
+
+====
+Changelog
+====
+
+0.8.18
+======
+Friday, August 18th, 2023
+-------------------------
+-
+
+0.8.18
+======
+- Updated `requests` version to `2.31.0`
+
+0.8.17
+======
+- Removed `tox` from requirements. It was not used, but was breaking our builds due to recent pushes to `virtualenv`.
+
+0.8.x
+=====
+- Support test suites
+- Easier way to rerun failed tests - generate test suite with all the failed tests and also print them in the log so that user can copy them and paste as ducktape command line arguments
+- Python 2 is no longer supported, minimum supported version is 3.6
+- Added `--deflake N` flag - if provided, it will attempt to rerun each failed test  up to N times, and if it eventually passes, it will be marked as Flaky - `#299 <https://github.com/confluentinc/ducktape/pull/299>`_
+- [backport, also in 0.9.1] - use a generic network device based on the devices found on the remote machine rather than a hardcoded one - `#314 <https://github.com/confluentinc/ducktape/pull/314>`_ and `#328 <https://github.com/confluentinc/ducktape/pull/328>`_
+- [backport, also in 0.9.1] - clean up process properly after an exception during test runner execution - `#323 <https://github.com/confluentinc/ducktape/pull/323>`_
+- [backport, also in 0.9.1] - log ssh errors - `#319 <https://github.com/confluentinc/ducktape/pull/319>`_
+- [backport, also in 0.9.1] - update vagrant tests to use ubuntu20 - `#328 <https://github.com/confluentinc/ducktape/pull/328>`_
+- [backport, also in 0.9.1] - added command to print the total number of nodes the tests run will require - `#320 <https://github.com/confluentinc/ducktape/pull/320>`_

docs/debug_tests.rst

@@ -97,3 +97,13 @@ Tools for Managing Logs
 =======================
 
 Analyzing and matching up logs from a distributed service could be time consuming. There are many good tools for working with logs. Examples include http://lnav.org/, http://list.xmodulo.com/multitail.html, and http://glogg.bonnefon.org/.
+
+Validating Ssh Issues
+=======================
+
+Ducktape supports running custom validators when an ssh error occurs, allowing you to run your own validation against a host.
+this is done simply by running ducktape with the `--ssh-checker-function`, followed by the module path to your function, so for instance::
+    
+    ducktape my-test.py --ssh-checker-function my.module.validator.validate_ssh
+
+this function will take in the ssh error raised as its first argument, and the remote account object as its second.

docs/index.rst

@@ -21,6 +21,7 @@ Ducktape contains tools for running system integration and performance tests. It
    debug_tests
    api
    misc
+   changelog
 
 Contribute
 ==========

docs/requirements.txt

@@ -1,6 +1,8 @@
-Sphinx==1.5.3
+Sphinx<1.7
 sphinx-argparse==0.1.17
 sphinx-rtd-theme==0.2.4
 boto3==1.15.9
 pycryptodome==3.9.8
 pywinrm==0.2.2
+jinja2==2.11.2
+MarkupSafe<2.0.0

docs/run_tests.rst

@@ -8,9 +8,9 @@ Running Tests
 =============
 
 ducktape discovers and runs tests in the path(s) provided.
-You can specify a folder with tests, a specific test file or even a specific class or test method, via absolute or
-relative paths. You can optionally specify a specific set of parameters
-for tests with ``@parametrize`` or ``@matrix`` annotations::
+You can specify a folder with tests (all tests in Python modules named with "test\_" prefix or "_test" suffix will be
+run), a specific test file (with any name) or even a specific class or test method, via absolute or relative paths.
+You can optionally specify a specific set of parameters for tests with ``@parametrize`` or ``@matrix`` annotations::
 
     ducktape <relative_path_to_testdirectory>                   # e.g. ducktape dir/tests
     ducktape <relative_path_to_file>                            # e.g. ducktape dir/tests/my_test.py

ducktape/__init__.py

@@ -1 +1 @@
-__version__ = '0.8.8'
+__version__ = '0.8.18'

ducktape/cluster/json.py

@@ -90,7 +90,9 @@ def __init__(self, cluster_json=None, *args, **kwargs):
                     "Cluster json has a node without a ssh_config field: %s\n Cluster json: %s" % (ninfo, cluster_json)
 
                 ssh_config = RemoteAccountSSHConfig(**ninfo.get("ssh_config", {}))
-                remote_account = JsonCluster.make_remote_account(ssh_config, ninfo.get("externally_routable_ip"))
+                remote_account = \
+                    JsonCluster.make_remote_account(ssh_config, ninfo.get("externally_routable_ip"),
+                                                    ssh_exception_checks=kwargs.get("ssh_exception_checks"))
                 if remote_account.externally_routable_ip is None:
                     remote_account.externally_routable_ip = self._externally_routable_ip(remote_account)
                 self._available_accounts.add_node(remote_account)
@@ -100,15 +102,13 @@ def __init__(self, cluster_json=None, *args, **kwargs):
         self._id_supplier = 0
 
     @staticmethod
-    def make_remote_account(ssh_config, externally_routable_ip=None):
+    def make_remote_account(ssh_config, *args, **kwargs):
         """Factory function for creating the correct RemoteAccount implementation."""
 
         if ssh_config.host and WINDOWS in ssh_config.host:
-            return WindowsRemoteAccount(ssh_config=ssh_config,
-                                        externally_routable_ip=externally_routable_ip)
+            return WindowsRemoteAccount(ssh_config, *args, **kwargs)
         else:
-            return LinuxRemoteAccount(ssh_config=ssh_config,
-                                      externally_routable_ip=externally_routable_ip)
+            return LinuxRemoteAccount(ssh_config, *args, **kwargs)
 
     def do_alloc(self, cluster_spec):
         allocated_accounts = self._available_accounts.remove_spec(cluster_spec)

ducktape/cluster/linux_remoteaccount.py

@@ -13,14 +13,13 @@
 # limitations under the License.
 
 from ducktape.cluster.cluster_spec import LINUX
-from ducktape.cluster.remoteaccount import RemoteAccount
+from ducktape.cluster.remoteaccount import RemoteAccount, RemoteAccountError
 
 
 class LinuxRemoteAccount(RemoteAccount):
 
-    def __init__(self, ssh_config, externally_routable_ip=None, logger=None):
-        super(LinuxRemoteAccount, self).__init__(ssh_config, externally_routable_ip=externally_routable_ip,
-                                                 logger=logger)
+    def __init__(self, *args, **kwargs):
+        super(LinuxRemoteAccount, self).__init__(*args, **kwargs)
         self._ssh_client = None
         self._sftp_client = None
         self.os = LINUX
@@ -31,11 +30,54 @@ def local(self):
         This is an imperfect heuristic, but should work for simple local testing."""
         return self.hostname == "localhost" and self.user is None and self.ssh_config is None
 
-    def fetch_externally_routable_ip(self, is_aws):
-        if is_aws:
-            cmd = "/sbin/ifconfig eth0 "
-        else:
-            cmd = "/sbin/ifconfig eth1 "
-        cmd += r"| grep 'inet ' | tail -n 1 | egrep -o '[0-9\.]+' | head -n 1 2>&1"
-        output = "".join(self.ssh_capture(cmd))
-        return output.strip()
+    def get_network_devices(self):
+        """
+        Utility to get all network devices on a linux account
+        """
+        return [
+            device
+            for device in self.sftp_client.listdir('/sys/class/net')
+        ]
+
+    def get_external_accessible_network_devices(self):
+        """
+        gets the subset of devices accessible through an external conenction
+        """
+        return [
+            device
+            for device in self.get_network_devices()
+            if device != 'lo'  # do not include local device
+            and (device.startswith("en") or device.startswith('eth'))  # filter out other devices; "en" means ethernet
+            # eth0 can also sometimes happen, see https://unix.stackexchange.com/q/134483
+        ]
+
+    # deprecated, please use the self.externally_routable_ip that is set in your cluster,
+    # not explicitly deprecating it as it's used by vagrant cluster
+    def fetch_externally_routable_ip(self, is_aws=None):
+        if is_aws is not None:
+            self.logger.warning("fetch_externally_routable_ip: is_aws is a deprecated flag, and does nothing")
+
+        devices = self.get_external_accessible_network_devices()
+
+        self.logger.debug("found devices: {}".format(devices))
+
+        if not devices:
+            raise RemoteAccountError(self, "Couldn't find any network devices")
+
+        fmt_cmd = (
+            "/sbin/ifconfig {device} | "
+            "grep 'inet ' | "
+            "tail -n 1 | "
+            r"egrep -o '[0-9\.]+' | "
+            "head -n 1 2>&1"
+        )
+
+        ips = [
+            "".join(
+                self.ssh_capture(fmt_cmd.format(device=device))
+            ).strip()
+            for device in devices
+        ]
+        self.logger.debug("found ips: {}".format(ips))
+        self.logger.debug("returning the first ip found")
+        return next(iter(ips))

ducktape/cluster/localhost.py

@@ -32,7 +32,9 @@ def __init__(self, *args, **kwargs):
         self._available_nodes = NodeContainer()
         for i in range(num_nodes):
             ssh_config = RemoteAccountSSHConfig("localhost%d" % i, hostname="localhost", port=22)
-            self._available_nodes.add_node(ClusterNode(LinuxRemoteAccount(ssh_config)))
+            self._available_nodes.add_node(ClusterNode(
+                LinuxRemoteAccount(ssh_config,
+                                   ssh_exception_checks=kwargs.get("ssh_exception_checks"))))
         self._in_use_nodes = NodeContainer()
 
     def do_alloc(self, cluster_spec):