Merge pull request #3720 from jobh/master

hypothesis-python/RELEASE.rst

@@ -0,0 +1,5 @@
+RELEASE_TYPE: patch
+
+Add a health check that detects if the same test is executed
+several times by :ref:`different executors<custom-function-execution>`.
+This can lead to difficult-to-debug problems such as :issue:`3446`.

hypothesis-python/src/hypothesis/_settings.py

@@ -520,6 +520,18 @@ def all(cls) -> List["HealthCheck"]:
     This check requires the :ref:`Hypothesis pytest plugin<pytest-plugin>`,
     which is enabled by default when running Hypothesis inside pytest."""
 
+    differing_executors = 10
+    """Checks if :func:`@given <hypothesis.given>` has been applied to a test
+    which is executed by different :ref:`executors<custom-function-execution>`.
+    If your test function is defined as a method on a class, that class will be
+    your executor, and subclasses executing an inherited test is a common way
+    for things to go wrong.
+
+    The correct fix is often to bring the executor instance under the control
+    of hypothesis by explicit parametrization over, or sampling from,
+    subclasses, or to refactor so that :func:`@given <hypothesis.given>` is
+    specified on leaf subclasses."""
+
 
 @unique
 class Verbosity(IntEnum):

hypothesis-python/src/hypothesis/core.py

@@ -1188,6 +1188,8 @@ def run_test_as_given(test):
                 )
             given_kwargs[name] = st.from_type(hints[name])
 
+        prev_self = Unset = object()
+
         @impersonate(test)
         @define_function_signature(test.__name__, test.__doc__, new_signature)
         def wrapped_test(*arguments, **kwargs):
@@ -1249,6 +1251,23 @@ def wrapped_test(*arguments, **kwargs):
                     "to ensure that each example is run in a separate "
                     "database transaction."
                 )
+            if settings.database is not None:
+                nonlocal prev_self
+                # Check selfy really is self (not e.g. a mock) before we health-check
+                cur_self = (
+                    stuff.selfy
+                    if getattr(type(stuff.selfy), test.__name__, None) is wrapped_test
+                    else None
+                )
+                if prev_self is Unset:
+                    prev_self = cur_self
+                elif cur_self is not prev_self:
+                    msg = (
+                        f"The method {test.__qualname__} was called from multiple "
+                        "different executors. This may lead to flaky tests and "
+                        "nonreproducible errors when replaying from database."
+                    )
+                    fail_health_check(settings, msg, HealthCheck.differing_executors)
 
             state = StateForActualGivenExecution(
                 test_runner, stuff, test, settings, random, wrapped_test
@@ -1478,6 +1497,9 @@ def find(
     )
 
     if database_key is None and settings.database is not None:
+        # Note: The database key is not guaranteed to be unique. If not, replaying
+        # of database examples may fail to reproduce due to being replayed on the
+        # wrong condition.
         database_key = function_digest(condition)
 
     if not isinstance(specifier, SearchStrategy):

hypothesis-python/src/hypothesis/internal/reflection.py

@@ -88,13 +88,17 @@ def function_digest(function):
     multiple processes and is prone to changing significantly in response to
     minor changes to the function.
 
-    No guarantee of uniqueness though it usually will be.
+    No guarantee of uniqueness though it usually will be. Digest collisions
+    lead to unfortunate but not fatal problems during database replay.
     """
     hasher = hashlib.sha384()
     try:
         src = inspect.getsource(function)
     except (OSError, TypeError):
         # If we can't actually get the source code, try for the name as a fallback.
+        # NOTE: We might want to change this to always adding function.__qualname__,
+        # to differentiate f.x. two classes having the same function implementation
+        # with class-dependent behaviour.
         try:
             hasher.update(function.__name__.encode())
         except AttributeError:

hypothesis-python/tests/cover/test_health_checks.py

@@ -181,6 +181,20 @@ def test(b):
     assert str(HealthCheck.large_base_example) in str(exc.value)
 
 
+class sample_test_runner:
+    @given(st.none())
+    def test(self, _):
+        pass
+
+
+def test_differing_executors_fails_health_check():
+    sample_test_runner().test()
+    with pytest.raises(FailedHealthCheck) as exc:
+        sample_test_runner().test()
+
+    assert str(HealthCheck.differing_executors) in str(exc.value)
+
+
 def test_it_is_an_error_to_suppress_non_iterables():
     with raises(InvalidArgument):
         settings(suppress_health_check=1)

hypothesis-python/tests/cover/test_setup_teardown.py

@@ -28,6 +28,7 @@ def teardown_example(self, ex):
 
 class SomeGivens:
     @given(integers())
+    @settings(suppress_health_check=[HealthCheck.differing_executors])
     def give_me_an_int(self, x):
         pass
 

hypothesis-python/tests/django/toystore/test_basic_configuration.py

@@ -24,7 +24,9 @@
 
 
 class SomeStuff:
-    @settings(suppress_health_check=[HealthCheck.too_slow])
+    @settings(
+        suppress_health_check=[HealthCheck.too_slow, HealthCheck.differing_executors]
+    )
     @given(integers())
     def test_is_blank_slate(self, unused):
         Company.objects.create(name="MickeyCo")