From 74e5702ae10f14797a4bf062654c3759ae551b5a Mon Sep 17 00:00:00 2001 From: jhall1-godaddy Date: Thu, 21 Jul 2022 06:53:19 -0700 Subject: [PATCH 1/6] Update tomlkit to 0.11.1 --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8f0338c6..2c8d4af8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -864,11 +864,11 @@ python-versions = ">=3.6" [[package]] name = "tomlkit" -version = "0.7.2" +version = "0.11.1" description = "Style preserving TOML library" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6,<4.0" [[package]] name = "tox" @@ -1006,7 +1006,7 @@ docs = ["recommonmark", "sphinx", "sphinx-autodoc-typehints", "sphinx-click", "s [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "a6b7c9893e5ec231d120b929666208426ce4c3c77acc51c23190d260ff34d5bf" +content-hash = "7e3a8db3f206be1a5543dd411f2ce8218876907489e8ce8d52f73940fd77d9fb" [metadata.files] alabaster = [ @@ -1622,8 +1622,8 @@ tomli = [ {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, ] tomlkit = [ - {file = "tomlkit-0.7.2-py2.py3-none-any.whl", hash = "sha256:173ad840fa5d2aac140528ca1933c29791b79a374a0861a80347f42ec9328117"}, - {file = "tomlkit-0.7.2.tar.gz", hash = "sha256:d7a454f319a7e9bd2e249f239168729327e4dd2d27b17dc68be264ad1ce36754"}, + {file = "tomlkit-0.11.1-py3-none-any.whl", hash = "sha256:1c5bebdf19d5051e2e1de6cf70adfc5948d47221f097fcff7a3ffc91e953eaf5"}, + {file = "tomlkit-0.11.1.tar.gz", hash = "sha256:61901f81ff4017951119cd0d1ed9b7af31c821d6845c8c477587bbdcd5e5854e"}, ] tox = [ {file = "tox-3.24.5-py2.py3-none-any.whl", hash = "sha256:be3362472a33094bce26727f5f771ca0facf6dafa217f65875314e9a6600c95c"}, diff --git a/pyproject.toml b/pyproject.toml index 58faf10d..14770ca8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ sphinx-autodoc-typehints = {version = "^1.12.0", optional = true} sphinx-click = {version = "^3.0.2", optional = true} sphinx-rtd-theme = {version = "^1.0.0", optional = true} sphinxcontrib-spelling = {version = "^7.2.1", optional = true} -tomlkit = "^0.7.2" +tomlkit = "^0.11.1" cached-property = "^1.5.2" [tool.poetry.dev-dependencies] From 8baa19796747b03d2f2a76e27b258adb3f18fa85 Mon Sep 17 00:00:00 2001 From: jhall1-godaddy Date: Thu, 21 Jul 2022 06:53:58 -0700 Subject: [PATCH 2/6] Handle the case where exclude-signatures is a list of strings Add error handling when tartufo config is not found Only open the file once when writing the config Add a couple more tests --- tartufo/commands/update_signatures.py | 64 ++++++++++++++++++++------- tests/test_update_signatures.py | 24 ++++++++++ 2 files changed, 72 insertions(+), 16 deletions(-) diff --git a/tartufo/commands/update_signatures.py b/tartufo/commands/update_signatures.py index dea490b6..230247b5 100644 --- a/tartufo/commands/update_signatures.py +++ b/tartufo/commands/update_signatures.py @@ -10,6 +10,7 @@ Optional, Sequence, Tuple, + Union, ) import click @@ -22,6 +23,18 @@ DeprecationSetT = MutableSet[Sequence[str]] +def unwrap_signature(data: Union[str, MutableMapping[str, str]]) -> str: + """Handles the case where a signature can be a string or a dict + + :param data: The string, or dictionary to pull the signature from + :returns: The unwrapped signature + """ + if isinstance(data, str): + return data + + return data["signature"] + + def scan_local_repo( options: types.GlobalOptions, repo_path: str, @@ -101,13 +114,20 @@ def replace_deprecated_signatures( updated = 0 for old_sig, new_sig in deprecations: - targets = functools.partial(lambda o, s: o == s["signature"], old_sig) + targets = functools.partial(lambda o, s: o == unwrap_signature(s), old_sig) # Iterate all the deprecations and update them everywhere # they are found in the exclude-signatures section of config - for target_signature in filter(targets, config_data["exclude_signatures"]): + for i, target_signature in enumerate(config_data["exclude_signatures"]): + if not targets(target_signature): + continue + updated += 1 click.echo(f"{updated}) {old_sig!r} -> {new_sig!r}") - target_signature["signature"] = new_sig + + if isinstance(target_signature, str): + config_data["exclude_signatures"][i] = new_sig + else: + target_signature["signature"] = new_sig return updated @@ -120,16 +140,18 @@ def write_updated_signatures( :param config_path: The path to the tartufo config file :param config_data: The updated config data """ - with open(str(config_path), "r") as file: - result = tomlkit.loads(file.read()) + with open(str(config_path), "r+") as file: + file_content = file.read() + result = tomlkit.loads(file_content) + file.seek(0) - # Assign the new signatures and write it to the config - result["tool"]["tartufo"]["exclude-signatures"] = config_data[ # type: ignore - "exclude_signatures" - ] + # Assign the new signatures and write it to the config + result["tool"]["tartufo"]["exclude-signatures"] = config_data[ # type: ignore + "exclude_signatures" + ] - with open(str(config_path), "w") as file: file.write(tomlkit.dumps(result)) + file.truncate() def remove_duplicated_entries(config_data: MutableMapping[str, Any]) -> int: @@ -138,17 +160,19 @@ def remove_duplicated_entries(config_data: MutableMapping[str, Any]) -> int: :param config_data: The config data to check for duplicates """ - seen = set() + seen: MutableSet[str] = set() count = 0 - for i, exclude in enumerate(config_data["exclude_signatures"].copy()): - if exclude["signature"] in seen: + for i, exclude in enumerate(config_data["exclude_signatures"]): + signature = unwrap_signature(exclude) + + if signature in seen: # Remove this duplicated signature - del config_data["exclude_signatures"][i] + config_data["exclude_signatures"].pop(i) count += 1 else: # Mark this signature as seen - seen.add(exclude["signature"]) + seen.add(signature) return count @@ -202,7 +226,15 @@ def main( remove_duplicates: bool, ) -> GitRepoScanner: """Update deprecated signatures for a local repository.""" - config_path, config_data = load_config_from_path(pathlib.Path(repo_path)) + try: + config_path, config_data = load_config_from_path(pathlib.Path(repo_path)) + except FileNotFoundError: + util.fail( + util.style_warning("No tartufo config found, exiting..."), + ctx, + code=0, + ) + if not config_data.get("exclude_signatures"): util.fail( util.style_warning("No signatures found in configuration, exiting..."), diff --git a/tests/test_update_signatures.py b/tests/test_update_signatures.py index 5677fd8b..edf0246c 100644 --- a/tests/test_update_signatures.py +++ b/tests/test_update_signatures.py @@ -28,6 +28,17 @@ def test_with_no_signatures_in_config( result.output, "No signatures found in configuration, exiting...\n" ) + @mock.patch("tartufo.commands.update_signatures.load_config_from_path") + def test_with_no_config(self, mock_load_config: mock.MagicMock) -> None: + mock_load_config.side_effect = FileNotFoundError() + + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(cli.main, ["update-signatures", "."]) + + mock_load_config.assert_called_once() + self.assertEqual(result.output, "No tartufo config found, exiting...\n") + @mock.patch("tartufo.commands.update_signatures.GitRepoScanner") @mock.patch("tartufo.commands.update_signatures.load_config_from_path") def test_with_no_deprecated_signatures( @@ -280,6 +291,19 @@ def test_found_output_with_no_signatures( mock_scan_local.assert_called_once() self.assertEqual(result.output, "Found 0 deprecated signatures.\n") + def test_replace_deprecated_with_list_of_strings(self) -> None: + deprecations: Set[Sequence[str]] = set() + deprecations.update((("123", "abc"), ("456", "def"))) + config_data = {"exclude_signatures": ["123", "456"]} + expected_result = {"exclude_signatures": ["abc", "def"]} + + count = update_signatures.replace_deprecated_signatures( + deprecations, config_data + ) + + self.assertEqual(count, 2) + self.assertEqual(config_data, expected_result) + def test_remove_duplicated_entries(self) -> None: initial_data = { "exclude_signatures": [ From 86d755f47b986117f42730c261b5378a963501e4 Mon Sep 17 00:00:00 2001 From: jhall1-godaddy Date: Thu, 21 Jul 2022 06:54:27 -0700 Subject: [PATCH 3/6] Add changelog fragment --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d3f624e..b1c27298 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +Unreleased +---------------------- + +Bug fixes: +* [#372](https://github.com/godaddy/tartufo/pull/372) - Handle the case where exclude-signatures is a list of strings + v3.2.1 - 20 July 2022 ---------------------- From 25007b19d58eadc27597c2d4afec4565a676add4 Mon Sep 17 00:00:00 2001 From: jhall1-godaddy Date: Thu, 21 Jul 2022 17:42:49 -0700 Subject: [PATCH 4/6] Pass str to click.echo in util.process_issues Update test --- tartufo/util.py | 2 +- tests/test_util.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tartufo/util.py b/tartufo/util.py index 014cfc62..9e023580 100644 --- a/tartufo/util.py +++ b/tartufo/util.py @@ -104,7 +104,7 @@ def echo_result( ) else: for issue in scanner.scan(): - click.echo(bytes(issue)) + click.echo(str(issue)) if scanner.issue_count == 0: if not options.quiet: click.echo(f"Time: {now}\nAll clear. No secrets detected.") diff --git a/tests/test_util.py b/tests/test_util.py index 3c290197..c775cb00 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -115,13 +115,13 @@ def test_echo_result_echos_all_when_not_json(self, mock_click, mock_scanner): mock_scanner.exclude_signatures = [] mock_scanner.scan.return_value = (1, 2, 3, 4) util.echo_result(options, mock_scanner, "", "") - # Ensure that the issues are output as a byte stream + mock_click.echo.assert_has_calls( [ - mock.call(bytes(1)), - mock.call(bytes(2)), - mock.call(bytes(3)), - mock.call(bytes(4)), + mock.call(str(1)), + mock.call(str(2)), + mock.call(str(3)), + mock.call(str(4)), ] ) self.assertEqual(mock_click.echo.call_count, 4) From 6a2a071882b7e2c95c3402baa06d3d4ebf3e6d83 Mon Sep 17 00:00:00 2001 From: jhall1-godaddy Date: Thu, 21 Jul 2022 17:43:26 -0700 Subject: [PATCH 5/6] Update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1c27298..3d641c6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ Unreleased ---------------------- Bug fixes: -* [#372](https://github.com/godaddy/tartufo/pull/372) - Handle the case where exclude-signatures is a list of strings +* [#372](https://github.com/godaddy/tartufo/pull/372) + - [#371](https://github.com/godaddy/tartufo/issues/371) Handle the case where exclude-signatures is a list of strings + - [#373](https://github.com/godaddy/tartufo/issues/373) Pass a string to click.echo rather than bytes v3.2.1 - 20 July 2022 ---------------------- From 443669eee60eb0855a63c30eb7727ecdddaad016 Mon Sep 17 00:00:00 2001 From: jhall1-godaddy Date: Wed, 17 Aug 2022 11:20:39 -0700 Subject: [PATCH 6/6] Update tomlkit to 0.11.4 --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2c8d4af8..20908e8d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -864,7 +864,7 @@ python-versions = ">=3.6" [[package]] name = "tomlkit" -version = "0.11.1" +version = "0.11.4" description = "Style preserving TOML library" category = "main" optional = false @@ -1006,7 +1006,7 @@ docs = ["recommonmark", "sphinx", "sphinx-autodoc-typehints", "sphinx-click", "s [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "7e3a8db3f206be1a5543dd411f2ce8218876907489e8ce8d52f73940fd77d9fb" +content-hash = "76d48e62fa80585829933c1bdfd3a8936009d6b80edf6541b23625c67d06b56a" [metadata.files] alabaster = [ @@ -1622,8 +1622,8 @@ tomli = [ {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, ] tomlkit = [ - {file = "tomlkit-0.11.1-py3-none-any.whl", hash = "sha256:1c5bebdf19d5051e2e1de6cf70adfc5948d47221f097fcff7a3ffc91e953eaf5"}, - {file = "tomlkit-0.11.1.tar.gz", hash = "sha256:61901f81ff4017951119cd0d1ed9b7af31c821d6845c8c477587bbdcd5e5854e"}, + {file = "tomlkit-0.11.4-py3-none-any.whl", hash = "sha256:25d4e2e446c453be6360c67ddfb88838cfc42026322770ba13d1fbd403a93a5c"}, + {file = "tomlkit-0.11.4.tar.gz", hash = "sha256:3235a9010fae54323e727c3ac06fb720752fe6635b3426e379daec60fbd44a83"}, ] tox = [ {file = "tox-3.24.5-py2.py3-none-any.whl", hash = "sha256:be3362472a33094bce26727f5f771ca0facf6dafa217f65875314e9a6600c95c"}, diff --git a/pyproject.toml b/pyproject.toml index 14770ca8..13a7b89e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ sphinx-autodoc-typehints = {version = "^1.12.0", optional = true} sphinx-click = {version = "^3.0.2", optional = true} sphinx-rtd-theme = {version = "^1.0.0", optional = true} sphinxcontrib-spelling = {version = "^7.2.1", optional = true} -tomlkit = "^0.11.1" +tomlkit = "^0.11.4" cached-property = "^1.5.2" [tool.poetry.dev-dependencies]