Merge pull request #36 from redpanda-data/DEVPROD-2200

loader: add option to not fail if no tests are discovered

ducktape/command_line/main.py

@@ -131,7 +131,8 @@ def main():
 
     # Discover and load tests to be run
     loader = TestLoader(session_context, session_logger, repeat=args_dict["repeat"], injected_args=injected_args,
-                        subset=args_dict["subset"], subsets=args_dict["subsets"])
+                        subset=args_dict["subset"], subsets=args_dict["subsets"],
+                        allow_empty_tests_list=args_dict["allow_empty_tests_list"])
     try:
         tests = loader.load(args_dict["test_path"], excluded_test_symbols=args_dict['exclude'])
     except LoaderException as e:

ducktape/command_line/parse_args.py

@@ -100,6 +100,9 @@ def create_ducktape_parser():
                              "to determine flakyness. When not present, deflake will not be used, "
                              "and a test will be marked as either passed or failed. "
                              "When enabled tests will be marked as flaky if it passes on any of the reruns")
+    parser.add_argument("--allow-empty-tests-list", action="store_true",
+                        default=os.environ.get("DUCKTAPE_ALLOW_EMPTY_TESTS_LIST", "0").lower() in ("1", "true", "yes"),
+                        help="Proceeds without failing when no tests are loaded ")
     return parser
 
 

ducktape/tests/loader.py

@@ -52,7 +52,7 @@ class TestLoader(object):
     """Class used to discover and load tests."""
 
     def __init__(self, session_context, logger, repeat=1, injected_args=None, cluster=None, subset=0, subsets=1,
-                 historical_report=None):
+                 historical_report=None, allow_empty_tests_list=False):
         self.session_context = session_context
         self.cluster = cluster
         assert logger is not None
@@ -74,6 +74,7 @@ def __init__(self, session_context, logger, repeat=1, injected_args=None, cluste
         # A non-None value here means the loader will override the injected_args
         # in any discovered test, whether or not it is parametrized
         self.injected_args = injected_args
+        self.allow_empty_tests_list = allow_empty_tests_list
 
     def load(self, symbols, excluded_test_symbols=None):
         """
@@ -146,7 +147,9 @@ def load(self, symbols, excluded_test_symbols=None):
         # Sort to make sure we get a consistent order for when we create subsets
         all_test_context_list = sorted(all_test_context_list, key=attrgetter("test_id"))
         if not all_test_context_list:
-            raise LoaderException("No tests to run!")
+            if not self.allow_empty_tests_list:
+                raise LoaderException("No tests to run!")
+            self.logger.warn("No tests to run!")
         self.logger.debug("Discovered these tests: " + str(all_test_context_list))
         # Select the subset of tests.
         if self.historical_report:

tests/loader/check_loader.py

@@ -403,6 +403,15 @@ def check_test_loader_raises_on_params_not_found(self):
         with pytest.raises(LoaderException, match='No tests to run'):
             loader.load(included)
 
+    def check_test_loader_allow_empty_tests_list(self):
+        loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), allow_empty_tests_list=True)
+        # parameter syntax is valid, but there is no such parameter defined in the test annotation in the code
+        included = [os.path.join(discover_dir(), 'test_decorated.py::TestMatrix.test_thing@{"x": 1,"y": "missing"}')]
+        try:
+            loader.load(included)
+        except LoaderException:
+            pytest.fail("Should not have raised the LoaderException exception")
+
     @pytest.mark.parametrize("symbol", [
         # no class
         'test_decorated.py::.test_thing'