Enable launch test discovery in pytest (#312)

* Enable launch test discovery in pytest.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Test examples using pytest.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Enable downstream customization of launch tests execution.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Move launch testing examples README content to the root README.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Mark launch tests explicitly.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Refactor parameterization support to play nice with pytest markers.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Improve pytest output on launch test failure.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Please flake8

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Fix failing launch_testing tests.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Prevent pytest from performing any assertion rewriting outside the launch_testing plugin.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Change explicit setattr for regular assignment.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Better document PYTEST_DONT_REWRITE docstring.

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

* Please flake8

Signed-off-by: Michel Hidalgo <michel@ekumenlabs.com>

launch_testing/README.md

@@ -179,3 +179,67 @@ add_launch_test(
   ARGS "arg1:=foo"
 )
 ```
+
+## Examples
+
+### `good_proc_launch_test.py`
+
+Usage:
+
+```sh
+launch_test test/launch_testing/examples/good_proc_launch_test.py
+```
+
+This test checks a process called good_proc (source found in the [example_processes folder](example_processes)).
+good_proc is a simple python process that prints "Loop 1, Loop2, etc. every second until it's terminated with ctrl+c.
+The test will launch the process, wait for a few loops to complete by monitoring stdout, then terminate the process
+and run some post-shutdown checks.
+
+The pre-shutdown tests check that "Loop 1, Loop 2, Loop 3, Loop 4"
+are all printed to stdout.  Once this test finishes, the process under test is shut down
+
+After shutdown, we run a similar test that checks more output, and also checks the
+order of the output.  `test_out_of_order` demonstrates that the `assertSequentialStdout`
+context manager is able to detect out of order stdout.
+
+### `terminating_proc_launch_test.py`
+
+Usage:
+
+```sh
+launch_test test/launch_testing/examples/terminating_proc_launch_test.py
+```
+
+This test checks proper functionality of the _terminating\_proc_ example (source found in the [example_processes folder](example_processes)).
+
+### `args_launch_test.py`
+
+Usage to view the arguments:
+
+```sh
+launch_test test/launch_testing/examples/args_launch_test.py --show-args
+```
+
+Usage to run the test:
+
+```sh
+launch_test test/launch_testing/examples/args_launch_test.py dut_arg:=hey
+```
+
+This example shows how to pass arguments into a launch test.  The arguments are made avilable
+in the launch description via a launch.substitutions.LaunchConfiguration.  The arguments are made
+available to the test cases via a self.test_args dictionary
+
+This example will fail if no arguments are passed.
+
+### `context_launch_test.py`
+
+Usage:
+
+```sh
+launch_test test/launch_testing/examples/context_launch_test.py
+```
+
+This example shows how the `generate_test_description` function can return a tuple where the second
+item is a dictionary of objects that will be injected into the individual test cases.  Tests that
+wish to use elements of the test context can add arguments with names matching the keys of the dictionary.

launch_testing/launch_testing/asserts/assert_exit_codes.py

@@ -12,6 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+A module providing exit code assertions.
+
+To prevent pytest from rewriting this module assertions, please PYTEST_DONT_REWRITE.
+See https://docs.pytest.org/en/latest/assert.html#disabling-assert-rewriting for
+further reference.
+"""
+
+
 import os
 
 from ..util import resolveProcesses

launch_testing/launch_testing/asserts/assert_output.py

@@ -12,6 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+A module providing process output assertions.
+
+To prevent pytest from rewriting this module assertions, please PYTEST_DONT_REWRITE.
+See https://docs.pytest.org/en/latest/assert.html#disabling-assert-rewriting for
+further reference.
+"""
+
+
 import os
 
 from osrf_pycommon.terminal_color import remove_ansi_escape_senquences

launch_testing/launch_testing/asserts/assert_sequential_output.py

@@ -12,6 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+A module providing process output sequence assertions.
+
+To prevent pytest from rewriting this module assertions, please PYTEST_DONT_REWRITE.
+See https://docs.pytest.org/en/latest/assert.html#disabling-assert-rewriting for
+further reference.
+"""
+
+
 from contextlib import contextmanager
 
 from ..util import resolveProcesses

launch_testing/launch_testing/io_handler.py

@@ -12,6 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+A module providing process IO capturing classes.
+
+To prevent pytest from rewriting this module assertions, please PYTEST_DONT_REWRITE.
+See https://docs.pytest.org/en/latest/assert.html#disabling-assert-rewriting for
+further reference.
+"""
+
+
 import threading
 
 from .asserts.assert_output import assertInStdout

launch_testing/launch_testing/loader.py

@@ -106,11 +106,12 @@ def _format_params(self):
 
 
 def LoadTestsFromPythonModule(module, *, name='launch_tests'):
-
-    if hasattr(module.generate_test_description, '__parametrized__'):
-        normalized_test_description_func = module.generate_test_description
+    if not hasattr(module.generate_test_description, '__parametrized__'):
+        normalized_test_description_func = (
+            lambda: [(module.generate_test_description, {})]
+        )
     else:
-        normalized_test_description_func = [(module.generate_test_description, {})]
+        normalized_test_description_func = module.generate_test_description
 
     # If our test description is parameterized, we'll load a set of tests for each
     # individual launch
@@ -119,7 +120,7 @@ def LoadTestsFromPythonModule(module, *, name='launch_tests'):
                     args,
                     PreShutdownTestLoader().loadTestsFromModule(module),
                     PostShutdownTestLoader().loadTestsFromModule(module))
-            for description, args in normalized_test_description_func]
+            for description, args in normalized_test_description_func()]
 
 
 def PreShutdownTestLoader():

launch_testing/launch_testing/parametrize.py

@@ -35,22 +35,16 @@ def parametrize(argnames, argvalues):
     argnames = [x.strip() for x in argnames.split(',') if x.strip()]
     argvalues = [_normalize_to_tuple(x) for x in argvalues]
 
-    class decorator:
-
-        def __init__(self, func):
-            setattr(self, '__parametrized__', True)
-            self.__calls = []
-
+    def _decorator(func):
+        @functools.wraps(func)
+        def _wrapped():
             for val in argvalues:
                 partial_args = dict(zip(argnames, val))
 
                 partial = functools.partial(func, **partial_args)
                 functools.update_wrapper(partial, func)
-                self.__calls.append(
-                    (partial, partial_args)
-                )
-
-        def __iter__(self):
-            return iter(self.__calls)
+                yield partial, partial_args
+        _wrapped.__parametrized__ = True
+        return _wrapped
 
-    return decorator
+    return _decorator

launch_testing/launch_testing/proc_info_handler.py

@@ -12,6 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+A module providing process info capturing classes.
+
+To prevent pytest from rewriting this module assertions, please PYTEST_DONT_REWRITE.
+See https://docs.pytest.org/en/latest/assert.html#disabling-assert-rewriting for
+further reference.
+"""
+
+
 import threading
 from launch.actions import ExecuteProcess  # noqa
 from launch.events.process import ProcessExited

launch_testing/launch_testing/pytest/hooks.py

@@ -0,0 +1,134 @@
+# Copyright 2019 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from ..loader import LoadTestsFromPythonModule
+from ..test_runner import LaunchTestRunner
+
+
+class LaunchTestFailure(Exception):
+
+    def __init__(self, message, results):
+        super().__init__()
+        self.message = message
+        self.results = results
+
+    def __str__(self):
+        return self.message
+
+
+class LaunchTestItem(pytest.Item):
+
+    def __init__(self, name, parent, test_runs, runner_cls=LaunchTestRunner):
+        super().__init__(name, parent)
+        self.test_runs = test_runs
+        self.runner_cls = runner_cls
+
+    def runtest(self):
+        launch_args = sum((
+            args_set for args_set in self.config.getoption('--launch-args')
+        ), [])
+        runner = self.runner_cls(
+            test_runs=self.test_runs,
+            launch_file_arguments=launch_args,
+            debug=self.config.getoption('verbose')
+        )
+
+        runner.validate()
+        results_per_run = runner.run()
+
+        if any(not result.wasSuccessful() for result in results_per_run.values()):
+            raise LaunchTestFailure(
+                message='some test cases have failed', results=results_per_run
+            )
+
+    def repr_failure(self, excinfo):
+        if isinstance(excinfo.value, LaunchTestFailure):
+            return excinfo.value.message + ':\n' + '\n'.join({
+                '{} failed at {}.{}'.format(
+                    str(test_run),
+                    type(test_case).__name__,
+                    test_case._testMethodName
+                )
+                for test_run, test_result in excinfo.value.results.items()
+                for test_case, _ in (test_result.errors + test_result.failures)
+                if not test_result.wasSuccessful()
+            }) if excinfo.value.results else ''
+        return super().repr_failure(excinfo)
+
+    def reportinfo(self):
+        return self.fspath, 0, 'launch tests: {}'.format(self.name)
+
+
+class LaunchTestModule(pytest.File):
+
+    def makeitem(self, *args, **kwargs):
+        return LaunchTestItem(*args, **kwargs)
+
+    def collect(self):
+        module = self.fspath.pyimport()
+        yield self.makeitem(
+            name=module.__name__, parent=self,
+            test_runs=LoadTestsFromPythonModule(
+                module, name=module.__name__
+            )
+        )
+
+
+def find_launch_test_entrypoint(path):
+    try:
+        return getattr(path.pyimport(), 'generate_test_description', None)
+    except SyntaxError:
+        return None
+
+
+def pytest_pycollect_makemodule(path, parent):
+    entrypoint = find_launch_test_entrypoint(path)
+    if entrypoint is not None:
+        ihook = parent.session.gethookproxy(path)
+        module = ihook.pytest_launch_collect_makemodule(
+            path=path, parent=parent, entrypoint=entrypoint
+        )
+        if module is not None:
+            return module
+    if path.basename == '__init__.py':
+        return pytest.Package(path, parent)
+    return pytest.Module(path, parent)
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_launch_collect_makemodule(path, parent, entrypoint):
+    marks = getattr(entrypoint, 'pytestmark', [])
+    if marks and any(m.name == 'launch_test' for m in marks):
+        return LaunchTestModule(path, parent)
+
+
+def pytest_addhooks(pluginmanager):
+    import launch_testing.pytest.hookspecs as hookspecs
+    pluginmanager.add_hookspecs(hookspecs)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        '--launch-args', action='append', nargs='*',
+        default=[], help='One or more Launch test arguments'
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        'markers',
+        'launch_test: mark a generate_test_description function as a launch test entrypoint'
+    )