From a75da914159af1a4ac360d607564329695d2f61f Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Sun, 22 Nov 2020 20:09:07 +0400 Subject: [PATCH 1/4] feat: add `skip_coverage` as a marker, deprecate the fixture --- brownie/test/fixtures.py | 9 ++++++--- brownie/test/managers/base.py | 3 +++ brownie/test/managers/runner.py | 14 ++++++++------ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/brownie/test/fixtures.py b/brownie/test/fixtures.py index abfc9a915..63f2317b1 100644 --- a/brownie/test/fixtures.py +++ b/brownie/test/fixtures.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import sys +import warnings import pytest @@ -118,9 +119,11 @@ def no_call_coverage(self): @pytest.fixture(scope="session") def skip_coverage(self): - """Skips a test when coverage evaluation is active.""" - # implemented in pytest_collection_modifyitems - pass + warnings.warn( + "`skip_coverage` as a fixture has been deprecated, use it as a marker instead", + DeprecationWarning, + stacklevel=2, + ) @pytest.fixture def state_machine(self): diff --git a/brownie/test/managers/base.py b/brownie/test/managers/base.py index a757a51b2..d7a7cae00 100644 --- a/brownie/test/managers/base.py +++ b/brownie/test/managers/base.py @@ -98,6 +98,9 @@ def pytest_configure(self, config): config.addinivalue_line( "markers", "require_network: only run test when a specific network is active" ) + config.addinivalue_line( + "markers", "skip_coverage: skips a test when coverage evaluation is active" + ) for key in ("coverage", "always_transact"): CONFIG.argv[key] = config.getoption("--coverage") diff --git a/brownie/test/managers/runner.py b/brownie/test/managers/runner.py index ae695fe97..971594372 100644 --- a/brownie/test/managers/runner.py +++ b/brownie/test/managers/runner.py @@ -152,8 +152,7 @@ def pytest_collection_modifyitems(self, items): items in-place. Determines which modules are isolated, and skips tests based on - the `--update` and `--stateful` flags as well as the `skip_coverage` - fixture. + the `--update` and `--stateful` flags. Arguments --------- @@ -166,10 +165,6 @@ def pytest_collection_modifyitems(self, items): tests = {} for i in items: - # apply skip_coverage - if "skip_coverage" in i.fixturenames and CONFIG.argv["coverage"]: - i.add_marker("skip") - # apply --stateful flag if stateful is not None: if stateful == "true" and "state_machine" not in i.fixturenames: @@ -276,6 +271,13 @@ def pytest_runtest_setup(self, item): raise ValueError("`require_network` marker must include a network name") if brownie.network.show_active() not in marker.args: pytest.skip("Active network does not match `require_network` marker") + return + + if CONFIG.argv["coverage"] and ( + next(item.iter_markers(name="skip_coverage"), None) + or "skip_coverage" in item.fixturenames + ): + pytest.skip("`skip_coverage` marker and coverage is active") def pytest_runtest_logreport(self, report): """ From a5cbc05b8f3b4d48a7e9dfe35eb3b85f09b9678b Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Sun, 22 Nov 2020 22:58:34 +0400 Subject: [PATCH 2/4] feat: handle `no_call_coverage` as marker, deprecate the fixture --- brownie/network/transaction.py | 2 +- brownie/test/fixtures.py | 9 +++++---- brownie/test/managers/runner.py | 21 +++++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/brownie/network/transaction.py b/brownie/network/transaction.py index e73df5e6a..d2862f62d 100644 --- a/brownie/network/transaction.py +++ b/brownie/network/transaction.py @@ -525,7 +525,7 @@ def _get_trace(self) -> None: msg = f"Encountered a {type(e).__name__} while requesting " msg += "debug_traceTransaction. The local RPC client has likely crashed." if CONFIG.argv["coverage"]: - msg += " If the error persists, add the skip_coverage fixture to this test." + msg += " If the error persists, add the `skip_coverage` marker to this test." raise RPCRequestError(msg) from None if "error" in trace: diff --git a/brownie/test/fixtures.py b/brownie/test/fixtures.py index 63f2317b1..773f872b4 100644 --- a/brownie/test/fixtures.py +++ b/brownie/test/fixtures.py @@ -109,10 +109,11 @@ def package_loader(project_id): @pytest.fixture def no_call_coverage(self): - """ - Prevents coverage evaluation on contract calls during this test. Useful for speeding - up tests that contain many repetetive calls. - """ + warnings.warn( + "`no_call_coverage` as a fixture has been deprecated, use it as a marker instead", + DeprecationWarning, + stacklevel=2, + ) CONFIG.argv["always_transact"] = False yield CONFIG.argv["always_transact"] = CONFIG.argv["coverage"] diff --git a/brownie/test/managers/runner.py b/brownie/test/managers/runner.py index 971594372..28318bd35 100644 --- a/brownie/test/managers/runner.py +++ b/brownie/test/managers/runner.py @@ -336,6 +336,27 @@ def pytest_runtest_logreport(self, report): "results": "".join(self.results[path]), } + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_call(self, item): + """ + Called to run the test for test item (the call phase). + + * Handles logic for the `always_transact` marker. + + Arguments + --------- + item : _pytest.nodes.Item + Test item for which setup is performed. + """ + no_call_coverage = next(item.iter_markers(name="no_call_coverage"), None) + if no_call_coverage: + CONFIG.argv["always_transact"] = False + + yield + + if no_call_coverage: + CONFIG.argv["always_transact"] = CONFIG.argv["coverage"] + def pytest_report_teststatus(self, report): """ Return result-category, shortletter and verbose word for status reporting. From 7aca4d7f0a1b6c9daecda31c6898682f04d06874 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Sun, 22 Nov 2020 23:01:56 +0400 Subject: [PATCH 3/4] test: coverage fixtures -> coverage markers --- tests/test/plugin/test_coverage.py | 30 ++++++++++++------------- tests/test/plugin/test_skip_coverage.py | 4 +++- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/test/plugin/test_coverage.py b/tests/test/plugin/test_coverage.py index 097fea8f3..e6f919a49 100644 --- a/tests/test/plugin/test_coverage.py +++ b/tests/test/plugin/test_coverage.py @@ -3,37 +3,37 @@ import json test_source = """ +import pytest + +@pytest.mark.no_call_coverage def test_call_and_transact(BrownieTester, accounts, web3, fn_isolation): c = accounts[0].deploy(BrownieTester, True) c.setNum(12, {'from': accounts[0]}) assert web3.eth.blockNumber == 2 c.getTuple(accounts[0]) - assert web3.eth.blockNumber == 2""" + assert web3.eth.blockNumber == 2 -conf_source = """ -import pytest -@pytest.fixture(autouse=True) -def setup(no_call_coverage): - pass""" +def test_call_and_transact_without_decorator(BrownieTester, accounts, web3, fn_isolation): + c = accounts[0].deploy(BrownieTester, True) + c.setNum(12, {'from': accounts[0]}) + assert web3.eth.blockNumber == 2 + c.getTuple(accounts[0]) + assert web3.eth.blockNumber == 2 + """ def test_always_transact(plugintester, mocker, chain): mocker.spy(chain, "undo") + # without coverage eval, there should be no calls to `chain.undo` result = plugintester.runpytest() - result.assert_outcomes(passed=1) + result.assert_outcomes(passed=2) assert chain.undo.call_count == 0 - # with coverage eval - result = plugintester.runpytest("--coverage") - result.assert_outcomes(passed=1) - assert chain.undo.call_count == 1 - - # with coverage and no_call_coverage fixture - plugintester.makeconftest(conf_source) + # with coverage eval, only one of the tests should call `chain.undo` result = plugintester.runpytest("--coverage") - result.assert_outcomes(passed=1) + result.assert_outcomes(passed=2) assert chain.undo.call_count == 1 diff --git a/tests/test/plugin/test_skip_coverage.py b/tests/test/plugin/test_skip_coverage.py index c54755318..97881673b 100644 --- a/tests/test/plugin/test_skip_coverage.py +++ b/tests/test/plugin/test_skip_coverage.py @@ -4,8 +4,10 @@ test_source = """ import brownie +import pytest -def test_call_and_transact(BrownieTester, accounts, skip_coverage): +@pytest.mark.skip_coverage +def test_call_and_transact(BrownieTester, accounts): c = accounts[0].deploy(BrownieTester, True) c.doNothing({'from': accounts[0]}) From 26f59d257224cbd336f1de85806180002820684f Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Sun, 22 Nov 2020 23:22:39 +0400 Subject: [PATCH 4/4] docs: updates re: coverage markers --- docs/api-test.rst | 13 --- docs/tests-coverage.rst | 6 +- docs/tests-pytest-fixtures.rst | 169 ++++++++++++++++----------------- 3 files changed, 86 insertions(+), 102 deletions(-) diff --git a/docs/api-test.rst b/docs/api-test.rst index eab569b2e..0af28e483 100644 --- a/docs/api-test.rst +++ b/docs/api-test.rst @@ -68,19 +68,6 @@ These fixtures are used to effectively isolate tests. If included on every test Applies the :func:`module_isolation ` fixture, and additionally takes a snapshot prior to running each test which is then reverted to after the test completes. The snapshot is taken immediately after any module-scoped fixtures are applied, and before all function-scoped ones. -Coverage Fixtures -***************** - -These fixtures alter the behaviour of tests when coverage evaluation is active. - -.. py:attribute:: fixtures.no_call_coverage - - Function scope. Coverage evaluation will not be performed on called contact methods during this test. - -.. py:attribute:: fixtures.skip_coverage - - Function scope. If coverage evaluation is active, this test will be skipped. - ``brownie.test.strategies`` =========================== diff --git a/docs/tests-coverage.rst b/docs/tests-coverage.rst index 1a8e97f8f..9b6033231 100644 --- a/docs/tests-coverage.rst +++ b/docs/tests-coverage.rst @@ -82,8 +82,8 @@ During coverage analysis, all contract calls are executed as transactions. This Some things to keep in mind that can help to reduce your test runtime when evaluating coverage: 1. Coverage is analyzed on a per-transaction basis, and the results are cached. If you repeat an identical transaction, Brownie will not analyze it the 2nd time. Keep this in mind when designing and sequencing setup fixtures. - 2. For tests that involve many calls to the same getter method, use :func:`no_call_coverage ` to significantly speed execution. - 3. Omit very complex tests altogether with :func:`skip_coverage `. + 2. For tests that involve many calls to the same getter method, use the :func:`no_call_coverage ` marker to significantly speed execution. + 3. Omit very complex tests altogether with the :func:`skip_coverage ` marker. 4. If possible, always run your tests in parralel with :ref:`xdist`. -You can use the ``--durations`` flag to view a profile of your slowest tests. You may find good candidates for optimization, or the use of the :func:`no_call_coverage ` and :func:`skip_coverage ` fixtures. +You can use the ``--durations`` flag to view a profile of your slowest tests. You may find good candidates for optimization, or the use of the :func:`no_call_coverage ` and :func:`skip_coverage ` fixtures. diff --git a/docs/tests-pytest-fixtures.rst b/docs/tests-pytest-fixtures.rst index 6e6c0fe6a..0e21874d8 100644 --- a/docs/tests-pytest-fixtures.rst +++ b/docs/tests-pytest-fixtures.rst @@ -18,105 +18,105 @@ These fixtures provide quick access to Brownie objects that are frequently used Yields an :func:`Accounts ` container for the active project, used to interact with your local accounts. - .. code-block:: python - :linenos: + .. code-block:: python + :linenos: - def test_account_balance(accounts): - assert accounts[0].balance() == "100 ether" + def test_account_balance(accounts): + assert accounts[0].balance() == "100 ether" .. py:attribute:: a Short form of the ``accounts`` fixture. - .. code-block:: python - :linenos: + .. code-block:: python + :linenos: - def test_account_balance(a): - assert a[0].balance() == "100 ether" + def test_account_balance(a): + assert a[0].balance() == "100 ether" .. py:attribute:: chain Yields an :func:`Chain ` object, used to access block data and interact with the local test chain. - .. code-block:: python - :linenos: + .. code-block:: python + :linenos: - def test_account_balance(accounts, chain): - balance = accounts[1].balance() - accounts[0].transfer(accounts[1], "10 ether") - assert accounts[1].balance() == balance + "10 ether" + def test_account_balance(accounts, chain): + balance = accounts[1].balance() + accounts[0].transfer(accounts[1], "10 ether") + assert accounts[1].balance() == balance + "10 ether" - chain.reset() - assert accounts[1].balance() == balance + chain.reset() + assert accounts[1].balance() == balance .. py:attribute:: Contract Yields the :func:`Contract ` class, used to interact with contracts outside of the active project. - .. code-block:: python - :linenos: + .. code-block:: python + :linenos: - @pytest.fixture(scope="session") - def dai(Contract): - yield Contract.from_explorer("0x6B175474E89094C44Da98b954EedeAC495271d0F") + @pytest.fixture(scope="session") + def dai(Contract): + yield Contract.from_explorer("0x6B175474E89094C44Da98b954EedeAC495271d0F") .. py:attribute:: history Yields a :func:`TxHistory ` container for the active project, used to access transaction data. - .. code-block:: python - :linenos: + .. code-block:: python + :linenos: - def test_account_balance(accounts, history): - accounts[0].transfer(accounts[1], "10 ether") - assert len(history) == 1 + def test_account_balance(accounts, history): + accounts[0].transfer(accounts[1], "10 ether") + assert len(history) == 1 .. py:attribute:: interface Yields the :func:`InterfaceContainer ` object for the active project, which provides access to project interfaces. - .. code-block:: python - :linenos: + .. code-block:: python + :linenos: - @pytest.fixture(scope="session") - def dai(interface): - yield interface.Dai("0x6B175474E89094C44Da98b954EedeAC495271d0F") + @pytest.fixture(scope="session") + def dai(interface): + yield interface.Dai("0x6B175474E89094C44Da98b954EedeAC495271d0F") .. py:attribute:: pm Callable fixture that provides access to :func:`Project ` objects, used for testing against installed packages. - .. code-block:: python - :linenos: + .. code-block:: python + :linenos: - @pytest.fixture(scope="module") - def compound(pm, accounts): - ctoken = pm('defi.snakecharmers.eth/compound@1.1.0').CToken - yield ctoken.deploy({'from': accounts[0]}) + @pytest.fixture(scope="module") + def compound(pm, accounts): + ctoken = pm('defi.snakecharmers.eth/compound@1.1.0').CToken + yield ctoken.deploy({'from': accounts[0]}) .. py:attribute:: state_machine Yields the :func:`state_machine ` method, used for running a :ref:`stateful test `. - .. code-block:: python - :linenos: + .. code-block:: python + :linenos: - def test_stateful(Token, accounts, state_machine): - token = Token.deploy("Test Token", "TST", 18, 1e23, {'from': accounts[0]}) + def test_stateful(Token, accounts, state_machine): + token = Token.deploy("Test Token", "TST", 18, 1e23, {'from': accounts[0]}) - state_machine(StateMachine, accounts, token) + state_machine(StateMachine, accounts, token) .. py:attribute:: web3 Yields a :func:`Web3 ` object. - .. code-block:: python - :linenos: + .. code-block:: python + :linenos: - def test_account_balance(accounts, web3): - height = web3.eth.blockNumber - accounts[0].transfer(accounts[1], "10 ether") - assert web3.eth.blockNumber == height + 1 + def test_account_balance(accounts, web3): + height = web3.eth.blockNumber + accounts[0].transfer(accounts[1], "10 ether") + assert web3.eth.blockNumber == height + 1 Contract Fixtures ================= @@ -125,12 +125,12 @@ Brownie creates dynamically named fixtures to access each :func:`ContractContain For example - if your project contains a contract named ``Token``, there will be a ``Token`` fixture available. -.. code-block:: python - :linenos: + .. code-block:: python + :linenos: - def test_token_deploys(Token, accounts): - token = accounts[0].deploy(Token, "Test Token", "TST", 18, 1e24) - assert token.name() == "Test Token" + def test_token_deploys(Token, accounts): + token = accounts[0].deploy(Token, "Test Token", "TST", 18, 1e24) + assert token.name() == "Test Token" Isolation Fixtures @@ -151,53 +151,50 @@ Coverage Fixtures Coverage fixtures alter the behaviour of tests when coverage evaluation is active. They are useful for tests with many repetitive functions, to avoid the slowdown caused by ``debug_traceTransaction`` queries. -.. py:attribute:: no_call_coverage - Coverage evaluation will not be performed on called contact methods during this test. +.. _pytest-fixtures-reference-markers: - .. code-block:: python - :linenos: +Markers +======= - import pytest +Brownie provides the following :ref:`markers` for use within your tests: - @pytest.fixture(scope="module", autouse=True) - def token(Token, accounts): - t = accounts[0].deploy(Token, "Test Token", "TST", 18, 1000) - t.transfer(accounts[1], 100, {'from': accounts[0]}) - yield t +.. py:attribute:: pytest.mark.require_network(network_name) - def test_normal(token): - # this call is handled as a transaction, coverage is evaluated - assert token.balanceOf(accounts[0]) == 900 + Mark a test so that it only runs if the active network is named ``network_name``. This is useful when you have some tests intended for a local development environment and others for a forked mainnet. - def test_no_call_cov(Token, no_call_coverage): - # this call happens normally, no coverage evaluation - assert token.balanceOf(accounts[1]) == 100 + .. code-block:: python + :linenos: -.. py:attribute:: skip_coverage + @pytest.mark.require_network("mainnet-fork") + def test_almost_in_prod(): + pass - Skips a test if coverage evaluation is active. +.. py:attribute:: pytest.mark.no_call_coverage - .. code-block:: python - :linenos: + Only evaluate coverage for transactions made during this test, not calls. - def test_heavy_lifting(skip_coverage): - pass + This marker is useful for speeding up slow tests that involve many calls to the same view method. -.. _pytest-fixtures-reference-markers: + .. code-block:: python + :linenos: -Markers -======= + def test_normal(token): + # during coverage analysis this call is handled as a transaction + assert token.balanceOf(accounts[0]) == 900 -Brownie provides the following :ref:`markers` for use within your tests: + @pytest.mark.no_call_coverage + def test_no_call_cov(Token): + # this call is handled as a call, the test execution is quicker + assert token.balanceOf(accounts[1]) == 100 -.. py:attribute:: pytest.mark.require_network(network_name) +.. py:attribute:: pytest.mark.skip_coverage - Mark a test so that it only runs if the active network is named ``network_name``. This is useful when you have some tests intended for a local development environment and others for a forked mainnet. + Skips a test if coverage evaluation is active. - .. code-block:: python - :linenos: + .. code-block:: python + :linenos: - @pytest.mark.require_network("mainnet-fork") - def test_almost_in_prod(): - pass + @pytest.mark.skip_coverage + def test_heavy_lifting(): + pass