diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a30595ec5..8568ef7093 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,10 @@ For each new detector, at least one regression tests must be present. - If updating an existing detector, identify the respective json artifacts and then delete them, or run `python ./tests/test_detectors.py --overwrite` instead. - Run `pytest ./tests/test_detectors.py` and check that everything worked. -To see the tests coverage, run `pytest tests/test_detectors.py --cov=slither/detectors --cov-branch --cov-report html` +To see the tests coverage, run `pytest tests/test_detectors.py --cov=slither/detectors --cov-branch --cov-report html`. +To run tests for a specific detector, run `pytest tests/test_detectors.py -k ReentrancyReadBeforeWritten` (the detector's class name is the argument). +To run tests for a specific version, run `pytest tests/test_detectors.py -k 0.7.6`. +The IDs of tests can be inspected using `pytest tests/test_detectors.py --collect-only`. ### Parser tests - Create a test in `tests/ast-parsing` @@ -73,6 +76,10 @@ To see the tests coverage, run `pytest tests/test_detectors.py --cov=slither/d - Run `pytest ./tests/test_ast_parsing.py` and check that everything worked. To see the tests coverage, run `pytest tests/test_ast_parsing.py --cov=slither/solc_parsing --cov-branch --cov-report html` +To run tests for a specific test case, run `pytest tests/test_ast_parsing.py -k user_defined_value_type` (the filename is the argument). +To run tests for a specific version, run `pytest tests/test_ast_parsing.py -k 0.8.12`. +To run tests for a specific compiler json format, run `pytest tests/test_ast_parsing.py -k legacy` (can be legacy or compact). +The IDs of tests can be inspected using ``pytest tests/test_ast_parsing.py --collect-only`. ### Synchronization with crytic-compile By default, `slither` follows either the latest version of crytic-compile in pip, or `crytic-compile@master` (look for dependencies in [`setup.py`](./setup.py). If crytic-compile development comes with breaking changes, the process to update `slither` is: diff --git a/tests/test_ast_parsing.py b/tests/test_ast_parsing.py index e96a129b8a..bcd0fe3f72 100644 --- a/tests/test_ast_parsing.py +++ b/tests/test_ast_parsing.py @@ -433,20 +433,21 @@ def make_version(minor: int, patch_min: int, patch_max: int) -> List[str]: pass -@pytest.mark.parametrize("test_item", ALL_TESTS, ids=lambda x: x.test_file) -def test_parsing(test_item: Test): - flavors = ["compact"] - if not test_item.disable_legacy: - flavors += ["legacy"] - for version, flavor in test_item.versions_with_flavors: - test_file = os.path.join( - TEST_ROOT, "compile", f"{test_item.test_file}-{version}-{flavor}.zip" - ) - expected_file = os.path.join( - TEST_ROOT, "expected", f"{test_item.test_file}-{version}-{flavor}.json" - ) +def pytest_generate_tests(metafunc): + test_cases = [] + for test_item in ALL_TESTS: + for version, flavor in test_item.versions_with_flavors: + test_cases.append((test_item.test_file, version, flavor)) + metafunc.parametrize("test_file, version, flavor", test_cases) - cc = load_from_zip(test_file)[0] + +class TestASTParsing: + # pylint: disable=no-self-use + def test_parsing(self, test_file, version, flavor): + actual = os.path.join(TEST_ROOT, "compile", f"{test_file}-{version}-{flavor}.zip") + expected = os.path.join(TEST_ROOT, "expected", f"{test_file}-{version}-{flavor}.json") + + cc = load_from_zip(actual)[0] sl = Slither( cc, @@ -458,26 +459,25 @@ def test_parsing(test_item: Test): actual = generate_output(sl) try: - with open(expected_file, "r", encoding="utf8") as f: + with open(expected, "r", encoding="utf8") as f: expected = json.load(f) except OSError: pytest.xfail("the file for this test was not generated") raise diff = DeepDiff(expected, actual, ignore_order=True, verbose_level=2, view="tree") - if diff: for change in diff.get("values_changed", []): path_list = re.findall(r"\['(.*?)'\]", change.path()) path = "_".join(path_list) with open( - f"test_artifacts/{test_item.test_file}_{path}_expected.dot", + f"test_artifacts/{test_file}_{path}_expected.dot", "w", encoding="utf8", ) as f: f.write(change.t1) with open( - f"test_artifacts/{test_item.test_file}_{version}_{flavor}_{path}_actual.dot", + f"test_artifacts/{test_file}_{version}_{flavor}_{path}_actual.dot", "w", encoding="utf8", ) as f: diff --git a/tests/test_detectors.py b/tests/test_detectors.py index a5ecaae2aa..a45369fcdc 100644 --- a/tests/test_detectors.py +++ b/tests/test_detectors.py @@ -52,7 +52,7 @@ def set_solc(test_item: Test): # pylint: disable=too-many-lines def id_test(test_item: Test): - return f"{test_item.detector}: {test_item.solc_ver}/{test_item.test_file}" + return f"{test_item.detector.__name__}-{test_item.solc_ver}-{test_item.test_file}" ALL_TEST_OBJECTS = [