From b39f957b88b6f44252c0ad2eb289b92fd7937f8e Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Wed, 26 Jul 2017 10:08:54 +0100 Subject: [PATCH 1/4] Add test of issue #920 --- testing/python/fixture.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/testing/python/fixture.py b/testing/python/fixture.py index 8dd71341628..1801c91ffb9 100644 --- a/testing/python/fixture.py +++ b/testing/python/fixture.py @@ -2547,6 +2547,40 @@ def test_foo(fix): '*test_foo*alpha*', '*test_foo*beta*']) + @pytest.mark.issue920 + @pytest.mark.xfail(reason="Fixture reordering not deterministic with hash randomisation") + def test_deterministic_fixture_collection(self, testdir, monkeypatch): + testdir.makepyfile(""" + import pytest + + @pytest.fixture(scope="module", + params=["A", + "B", + "C"]) + def A(request): + return request.param + + @pytest.fixture(scope="module", + params=["DDDDDDDDD", "EEEEEEEEEEEE", "FFFFFFFFFFF", "banansda"]) + def B(request, A): + return request.param + + def test_foo(B): + # Something funky is going on here. + # Despite specified seeds, on what is collected, + # sometimes we get unexpected passes. hashing B seems + # to help? + assert hash(B) or True + """) + monkeypatch.setenv("PYTHONHASHSEED", "1") + out1 = testdir.runpytest_subprocess("-v") + monkeypatch.setenv("PYTHONHASHSEED", "2") + out2 = testdir.runpytest_subprocess("-v") + out1 = [line for line in out1.outlines if line.startswith("test_deterministic_fixture_collection.py::test_foo")] + out2 = [line for line in out2.outlines if line.startswith("test_deterministic_fixture_collection.py::test_foo")] + assert len(out1) == 12 + assert out1 == out2 + class TestRequestScopeAccess(object): pytestmark = pytest.mark.parametrize(("scope", "ok", "error"), [ From a546a612bde02292488b6f9b3185a950d61fbe94 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Tue, 25 Jul 2017 19:54:27 +0100 Subject: [PATCH 2/4] Fix nondeterminism in fixture collection order fixtures.reorder_items is non-deterministic because it reorders based on iteration over an (unordered) set. Change the code to use an OrderedDict instead so that we get deterministic behaviour, fixes #920. --- AUTHORS | 1 + _pytest/fixtures.py | 21 +++++++++++++-------- changelog/920.bugfix | 1 + testing/python/fixture.py | 1 - tox.ini | 1 + 5 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 changelog/920.bugfix diff --git a/AUTHORS b/AUTHORS index cd9678b3f97..b6b6b46faa2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -91,6 +91,7 @@ Kale Kundert Katarzyna Jachim Kevin Cox Kodi B. Arfer +Lawrence Mitchell Lee Kamentsky Lev Maximov Llandy Riveron Del Risco diff --git a/_pytest/fixtures.py b/_pytest/fixtures.py index 5505fb4e19e..d475dfd1bda 100644 --- a/_pytest/fixtures.py +++ b/_pytest/fixtures.py @@ -19,6 +19,11 @@ from _pytest.runner import fail from _pytest.compat import FuncargnamesCompatAttr +if sys.version_info[:2] == (2, 6): + from ordereddict import OrderedDict +else: + from collections import OrderedDict + def pytest_sessionstart(session): import _pytest.python @@ -136,10 +141,10 @@ def get_parametrized_fixture_keys(item, scopenum): except AttributeError: pass else: - # cs.indictes.items() is random order of argnames but - # then again different functions (items) can change order of - # arguments so it doesn't matter much probably - for argname, param_index in cs.indices.items(): + # cs.indices.items() is random order of argnames. Need to + # sort this so that different calls to + # get_parametrized_fixture_keys will be deterministic. + for argname, param_index in sorted(cs.indices.items()): if cs._arg2scopenum[argname] != scopenum: continue if scopenum == 0: # session @@ -161,7 +166,7 @@ def reorder_items(items): for scopenum in range(0, scopenum_function): argkeys_cache[scopenum] = d = {} for item in items: - keys = set(get_parametrized_fixture_keys(item, scopenum)) + keys = OrderedDict.fromkeys(get_parametrized_fixture_keys(item, scopenum)) if keys: d[item] = keys return reorder_items_atscope(items, set(), argkeys_cache, 0) @@ -196,9 +201,9 @@ def slice_items(items, ignore, scoped_argkeys_cache): for i, item in enumerate(it): argkeys = scoped_argkeys_cache.get(item) if argkeys is not None: - argkeys = argkeys.difference(ignore) - if argkeys: # found a slicing key - slicing_argkey = argkeys.pop() + newargkeys = OrderedDict.fromkeys(k for k in argkeys if k not in ignore) + if newargkeys: # found a slicing key + slicing_argkey, _ = newargkeys.popitem() items_before = items[:i] items_same = [item] items_other = [] diff --git a/changelog/920.bugfix b/changelog/920.bugfix new file mode 100644 index 00000000000..efe646a6e0e --- /dev/null +++ b/changelog/920.bugfix @@ -0,0 +1 @@ +Fix non-determinism in order of fixture collection. diff --git a/testing/python/fixture.py b/testing/python/fixture.py index 1801c91ffb9..f8aef802fa4 100644 --- a/testing/python/fixture.py +++ b/testing/python/fixture.py @@ -2548,7 +2548,6 @@ def test_foo(fix): '*test_foo*beta*']) @pytest.mark.issue920 - @pytest.mark.xfail(reason="Fixture reordering not deterministic with hash randomisation") def test_deterministic_fixture_collection(self, testdir, monkeypatch): testdir.makepyfile(""" import pytest diff --git a/tox.ini b/tox.ini index 3dce17b344b..1307cff143a 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,7 @@ deps = hypothesis<3.0 nose mock<1.1 + ordereddict [testenv:py27-subprocess] changedir = . From f8bd693f8348dbf5b0536b5529cc5a4884bf06b7 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Wed, 26 Jul 2017 10:58:38 +0100 Subject: [PATCH 3/4] Add ordereddict to install_requires for py26 --- setup.py | 3 ++- tox.ini | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 751868c048e..55607912bf4 100644 --- a/setup.py +++ b/setup.py @@ -46,11 +46,12 @@ def main(): install_requires = ['py>=1.4.33', 'setuptools'] # pluggy is vendored in _pytest.vendored_packages extras_require = {} if has_environment_marker_support(): - extras_require[':python_version=="2.6"'] = ['argparse'] + extras_require[':python_version=="2.6"'] = ['argparse', 'ordereddict'] extras_require[':sys_platform=="win32"'] = ['colorama'] else: if sys.version_info < (2, 7): install_requires.append('argparse') + install_requires.append('ordereddict') if sys.platform == 'win32': install_requires.append('colorama') diff --git a/tox.ini b/tox.ini index 1307cff143a..3dce17b344b 100644 --- a/tox.ini +++ b/tox.ini @@ -34,7 +34,6 @@ deps = hypothesis<3.0 nose mock<1.1 - ordereddict [testenv:py27-subprocess] changedir = . From f047e078e2d1c8aba19015e151c1e78c5cbc1cff Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Wed, 26 Jul 2017 13:56:17 +0100 Subject: [PATCH 4/4] Mention new (py26) ordereddict dependency in changelog and docs --- changelog/920.bugfix | 2 +- doc/en/getting-started.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog/920.bugfix b/changelog/920.bugfix index efe646a6e0e..d2dd2be1b79 100644 --- a/changelog/920.bugfix +++ b/changelog/920.bugfix @@ -1 +1 @@ -Fix non-determinism in order of fixture collection. +Fix non-determinism in order of fixture collection. Adds new dependency (ordereddict) for Python 2.6. diff --git a/doc/en/getting-started.rst b/doc/en/getting-started.rst index fb863e4e0d4..1571e4f6b40 100644 --- a/doc/en/getting-started.rst +++ b/doc/en/getting-started.rst @@ -9,7 +9,8 @@ Installation and Getting Started **dependencies**: `py `_, `colorama (Windows) `_, -`argparse (py26) `_. +`argparse (py26) `_, +`ordereddict (py26) `_. **documentation as PDF**: `download latest `_