Enable launch test discovery in pytest

launch_testing/README.md

@@ -179,3 +179,67 @@ add_launch_test(
   ARGS "arg1:=foo"
 )
 ```
+
+## Examples
+
+### `good_proc_launch_test.py`
+
+Usage:
+
+```sh
+launch_test test/launch_testing/examples/good_proc_launch_test.py
+```
+
+This test checks a process called good_proc (source found in the [example_processes folder](example_processes)).
+good_proc is a simple python process that prints "Loop 1, Loop2, etc. every second until it's terminated with ctrl+c.
+The test will launch the process, wait for a few loops to complete by monitoring stdout, then terminate the process
+and run some post-shutdown checks.
+
+The pre-shutdown tests check that "Loop 1, Loop 2, Loop 3, Loop 4"
+are all printed to stdout.  Once this test finishes, the process under test is shut down
+
+After shutdown, we run a similar test that checks more output, and also checks the
+order of the output.  `test_out_of_order` demonstrates that the `assertSequentialStdout`
+context manager is able to detect out of order stdout.
+
+### `terminating_proc_launch_test.py`
+
+Usage:
+
+```sh
+launch_test test/launch_testing/examples/terminating_proc_launch_test.py
+```
+
+This test checks proper functionality of the _terminating\_proc_ example (source found in the [example_processes folder](example_processes)).
+
+### `args_launch_test.py`
+
+Usage to view the arguments:
+
+```sh
+launch_test test/launch_testing/examples/args_launch_test.py --show-args
+```
+
+Usage to run the test:
+
+```sh
+launch_test test/launch_testing/examples/args_launch_test.py dut_arg:=hey
+```
+
+This example shows how to pass arguments into a launch test.  The arguments are made avilable
+in the launch description via a launch.substitutions.LaunchConfiguration.  The arguments are made
+available to the test cases via a self.test_args dictionary
+
+This example will fail if no arguments are passed.
+
+### `context_launch_test.py`
+
+Usage:
+
+```sh
+launch_test test/launch_testing/examples/context_launch_test.py
+```
+
+This example shows how the `generate_test_description` function can return a tuple where the second
+item is a dictionary of objects that will be injected into the individual test cases.  Tests that
+wish to use elements of the test context can add arguments with names matching the keys of the dictionary.

launch_testing/launch_testing/loader.py

@@ -106,11 +106,12 @@ def _format_params(self):
 
 
 def LoadTestsFromPythonModule(module, *, name='launch_tests'):
-
-    if hasattr(module.generate_test_description, '__parametrized__'):
-        normalized_test_description_func = module.generate_test_description
+    if not hasattr(module.generate_test_description, '__parametrized__'):
+        normalized_test_description_func = (
+            lambda: [(module.generate_test_description, {})]
+        )
     else:
-        normalized_test_description_func = [(module.generate_test_description, {})]
+        normalized_test_description_func = module.generate_test_description
 
     # If our test description is parameterized, we'll load a set of tests for each
     # individual launch
@@ -119,7 +120,7 @@ def LoadTestsFromPythonModule(module, *, name='launch_tests'):
                     args,
                     PreShutdownTestLoader().loadTestsFromModule(module),
                     PostShutdownTestLoader().loadTestsFromModule(module))
-            for description, args in normalized_test_description_func]
+            for description, args in normalized_test_description_func()]
 
 
 def PreShutdownTestLoader():

launch_testing/launch_testing/parametrize.py

@@ -35,22 +35,16 @@ def parametrize(argnames, argvalues):
     argnames = [x.strip() for x in argnames.split(',') if x.strip()]
     argvalues = [_normalize_to_tuple(x) for x in argvalues]
 
-    class decorator:
-
-        def __init__(self, func):
-            setattr(self, '__parametrized__', True)
-            self.__calls = []
-
+    def _decorator(func):
+        @functools.wraps(func)
+        def _wrapped():
             for val in argvalues:
                 partial_args = dict(zip(argnames, val))
 
                 partial = functools.partial(func, **partial_args)
                 functools.update_wrapper(partial, func)
-                self.__calls.append(
-                    (partial, partial_args)
-                )
-
-        def __iter__(self):
-            return iter(self.__calls)
+                yield partial, partial_args
+        setattr(_wrapped, '__parametrized__', True)
+        return _wrapped
 
-    return decorator
+    return _decorator

launch_testing/launch_testing/pytest/hooks.py

@@ -0,0 +1,134 @@
+# Copyright 2019 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from ..loader import LoadTestsFromPythonModule
+from ..test_runner import LaunchTestRunner
+
+
+class LaunchTestFailure(Exception):
+
+    def __init__(self, message, results):
+        super().__init__()
+        self.message = message
+        self.results = results
+
+    def __str__(self):
+        return self.message
+
+
+class LaunchTestItem(pytest.Item):
+
+    def __init__(self, name, parent, test_runs, runner_cls=LaunchTestRunner):
+        super().__init__(name, parent)
+        self.test_runs = test_runs
+        self.runner_cls = runner_cls
+
+    def runtest(self):
+        launch_args = sum((
+            args_set for args_set in self.config.getoption('--launch-args')
+        ), [])
+        runner = self.runner_cls(
+            test_runs=self.test_runs,
+            launch_file_arguments=launch_args,
+            debug=self.config.getoption('verbose')
+        )
+        try:
+            runner.validate()
+        except Exception as e:
+            raise LaunchTestFailure(message=str(e), results=[])
+
+        results_per_run = runner.run()
+        if any(not result.wasSuccessful() for result in results_per_run.values()):
+            raise LaunchTestFailure(
+                message='some test cases have failed', results=results_per_run
+            )
+
+    def repr_failure(self, excinfo):
+        if isinstance(excinfo.value, LaunchTestFailure):
+            return excinfo.value.message + ':\n' + '\n'.join({
+                '{} failed at {}.{}'.format(
+                    str(test_run),
+                    type(test_case).__name__,
+                    test_case._testMethodName
+                )
+                for test_run, test_result in excinfo.value.results.items()
+                for test_case, _ in (test_result.errors + test_result.failures)
+                if not test_result.wasSuccessful()
+            })
+
+    def reportinfo(self):
+        return self.fspath, 0, 'launch tests: {}'.format(self.name)
+
+
+class LaunchTestModule(pytest.File):
+
+    def makeitem(self, *args, **kwargs):
+        return LaunchTestItem(*args, **kwargs)
+
+    def collect(self):
+        module = self.fspath.pyimport()
+        yield self.makeitem(
+            name=module.__name__, parent=self,
+            test_runs=LoadTestsFromPythonModule(
+                module, name=module.__name__
+            )
+        )
+
+
+def find_launch_test_entrypoint(path):
+    try:
+        return getattr(path.pyimport(), 'generate_test_description', None)
+    except SyntaxError:
+        return None
+
+
+def pytest_pycollect_makemodule(path, parent):
+    entrypoint = find_launch_test_entrypoint(path)
+    if entrypoint is not None:
+        ihook = parent.session.gethookproxy(path)
+        module = ihook.pytest_launch_collect_makemodule(
+            path=path, parent=parent, entrypoint=entrypoint
+        )
+        if module is not None:
+            return module
+    if path.basename == '__init__.py':
+        return pytest.Package(path, parent)
+    return pytest.Module(path, parent)
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_launch_collect_makemodule(path, parent, entrypoint):
+    marks = getattr(entrypoint, 'pytestmark', [])
+    if marks and any(m.name == 'launch_test' for m in marks):
+        return LaunchTestModule(path, parent)
+
+
+def pytest_addhooks(pluginmanager):
+    import launch_testing.pytest.hookspecs as hookspecs
+    pluginmanager.add_hookspecs(hookspecs)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        '--launch-args', action='append', nargs='*',
+        default=[], help='One or more Launch test arguments'
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        'markers', 'launch_test: mark a generate_test_description function as a launch test entrypoint'
+    )

launch_testing/launch_testing/pytest/hookspecs.py

@@ -0,0 +1,21 @@
+# Copyright 2019 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_launch_collect_makemodule(path, parent, entrypoint):
+    """Make launch test module appropriate for the found test entrypoint."""
+    pass

launch_testing/pytest.ini

@@ -1,3 +1,5 @@
 [pytest]
 # Set testpaths, otherwise pytest finds 'tests' in the examples directory
 testpaths = test
+# Add arguments for launch tests
+addopts = --launch-args dut_arg:=test

launch_testing/setup.py

@@ -14,7 +14,8 @@
         ('share/launch_testing/examples', glob.glob('examples/[!_]**')),
     ],
     entry_points={
-        'console_scripts': ['launch_test=launch_testing.launch_test:main']
+        'console_scripts': ['launch_test=launch_testing.launch_test:main'],
+        'pytest11': ['launch = launch_testing.pytest.hooks'],
     },
     install_requires=['setuptools'],
     zip_safe=True,

launch_testing/examples/args.test.py → launch_testing/test/launch_testing/examples/args_launch_test.py

@@ -25,6 +25,9 @@
 import launch_testing
 import launch_testing.util
 
+import pytest
+
+
 dut_process = launch.actions.ExecuteProcess(
     cmd=[
         sys.executable,
@@ -40,6 +43,7 @@
 )
 
 
+@pytest.mark.launch_test
 def generate_test_description(ready_fn):
 
     return launch.LaunchDescription([

launch_testing/examples/example_test_context.test.py → launch_testing/test/launch_testing/examples/context_launch_test.py

@@ -24,6 +24,8 @@
 import launch_testing
 from launch_testing.asserts import assertSequentialStdout
 
+import pytest
+
 
 def get_test_process_action():
     TEST_PROC_PATH = os.path.join(
@@ -42,6 +44,7 @@ def get_test_process_action():
 # This launch description shows the prefered way to let the tests access launch actions.  By
 # adding them to the test context, it's not necessary to scope them at the module level like in
 # the good_proc.test.py example
+@pytest.mark.launch_test
 def generate_test_description(ready_fn):
     dut_process = get_test_process_action()