Skip to content

Commit

Permalink
Remove deprecated code (#433)
Browse files Browse the repository at this point in the history 
* Remove deprecated code

* Updating changelog

* Addressing review comments

* Removed support for old signatures which generated with +/- chars in git diff

* Fixing test names and removed comments around backwards_compatibility_prefix
  • Loading branch information
@emayuri-godaddy
emayuri-godaddy authored Jan 13, 2023
1 parent a3617f9 commit 24976f1
Show file tree
Hide file tree
Showing 10 changed files with 85 additions and 307 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ vX.X.X - DD MMM YYYY
--------------------

Features:
* [#433](https://github.com/godaddy/tartufo/pull/433) - Dropped support for deprecated flags rules, b64, hex
and corresponding code around deprecated options. Removed support for old signatures which generated with +/-
chars in git diff.

* [#411](https://github.com/godaddy/tartufo/pull/411) - Drop support for python 3.6.
This version reached end of life several years ago, and end of security support at
the end of 2021. Users with a requirement to run tartufo on this python version
Expand Down
20 changes: 0 additions & 20 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,6 @@ Usage: tartufo [OPTIONS] COMMAND [ARGS]...
commit hook.

Options:
--rules FILENAME [DEPRECATED] Use the rule-patterns config
options instead. Path(s) to regex rules json
list file(s).
--default-regexes / --no-default-regexes
Whether to include the default regex list
when configuring search patterns. Only
Expand Down Expand Up @@ -108,23 +105,6 @@ Options:
likelihood that a given string will be
identified as suspicious. [default: 75;
0<=x<=100]
-b64, --b64-entropy-score TEXT [DEPRECATED] Use `--entropy-sensitivity`.
Modify the base64 entropy score. If a value
greater than the default (4.5 in a range of
0.0-6.0) is specified, tartufo lists higher
entropy base64 strings (longer or more
randomized strings. A lower value lists
lower entropy base64 strings (shorter or
less randomized strings).
-hex, --hex-entropy-score TEXT [DEPRECATED] Use `--entropy-sensitivity`.
Modify the hexadecimal entropy score. If a
value greater than the default (3.0 in a
range of 0.0-4.0) is specified, tartufo
lists higher entropy hexadecimal strings
(longer or more randomized strings). A lower
value lists lower entropy hexadecimal
strings (shorter or less randomized
strings).
-V, --version Show the version and exit.
-h, --help Show this message and exit.

Expand Down
25 changes: 0 additions & 25 deletions tartufo/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,6 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Comma
name="tartufo",
context_settings=dict(help_option_names=["-h", "--help"]),
)
@click.option(
"--rules",
multiple=True,
type=click.File("r"),
help="[DEPRECATED] Use the rule-patterns config options instead. Path(s) to regex "
"rules json list file(s).",
)
@click.option(
"--rule-patterns",
multiple=True,
Expand Down Expand Up @@ -250,24 +243,6 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Comma
Decreasing the scanner's sensitivity increases the likelihood that a given
string will be identified as suspicious.""",
)
@click.option(
"-b64",
"--b64-entropy-score",
help="""[DEPRECATED] Use `--entropy-sensitivity`. Modify the base64 entropy score. If
a value greater than the default (4.5 in a range of 0.0-6.0) is specified,
tartufo lists higher entropy base64 strings (longer or more randomized strings.
A lower value lists lower entropy base64 strings (shorter or less randomized
strings).""",
)
@click.option(
"-hex",
"--hex-entropy-score",
help="""[DEPRECATED] Use `--entropy-sensitivity`. Modify the hexadecimal entropy score.
If a value greater than the default (3.0 in a range of 0.0-4.0) is specified,
tartufo lists higher entropy hexadecimal strings (longer or more randomized
strings). A lower value lists lower entropy hexadecimal strings (shorter or less
randomized strings).""",
)
# The first positional argument here would be a hard-coded version, hence the `None`
@click.version_option(None, "-V", "--version")
@click.pass_context
Expand Down
13 changes: 1 addition & 12 deletions tartufo/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import pathlib
import re
import shutil
import warnings
from typing import (
Any,
Dict,
Expand Down Expand Up @@ -149,7 +148,6 @@ def read_pyproject_toml(

def configure_regexes(
include_default: bool = True,
rules_files: Optional[Iterable[TextIO]] = None,
rule_patterns: Optional[Iterable[Dict[str, str]]] = None,
rules_repo: Optional[str] = None,
rules_repo_files: Optional[Iterable[str]] = None,
Expand Down Expand Up @@ -186,16 +184,7 @@ def configure_regexes(
f"Invalid rule-pattern; both reason and pattern are required fields. Rule: {pattern}"
) from exc

if rules_files:
warnings.warn(
"Storing rules in a separate file is deprecated and will be removed "
"in tartufo 4.x. Please use the 'rule-patterns' config "
" option instead.",
DeprecationWarning,
)
all_files: List[TextIO] = list(rules_files)
else:
all_files = []
all_files = []
try:
cloned_repo = False
repo_path = None
Expand Down
99 changes: 3 additions & 96 deletions tartufo/scanner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
Tuple,
IO,
)
import warnings

from cached_property import cached_property
import click
Expand Down Expand Up @@ -175,16 +174,6 @@ def compute_scaled_entropy_limit(self, maximum_bitrate: float) -> float:
def hex_entropy_limit(self) -> float:
"""Returns low entropy limit for suspicious hexadecimal encodings"""

# For backwards compatibility, allow the caller to manipulate this score
# # directly (but complain about it).
if self.global_options.hex_entropy_score:
warnings.warn(
"--hex-entropy-score is deprecated and will be removed in tartufo 4.x. "
"Please use --entropy-sensitivity instead.",
DeprecationWarning,
)
return self.global_options.hex_entropy_score

# Each hexadecimal digit represents a 4-bit number, so we want to scale
# the base score by this amount to account for the efficiency of the
# string representation we're examining.
Expand All @@ -194,16 +183,6 @@ def hex_entropy_limit(self) -> float:
def b64_entropy_limit(self) -> float:
"""Returns low entropy limit for suspicious base64 encodings"""

# For backwards compatibility, allow the caller to manipulate this score
# # directly (but complain about it).
if self.global_options.b64_entropy_score:
warnings.warn(
"--b64-entropy-score is deprecated and will be removed in tartufo 4.x. "
"Please use --entropy-sensitivity instead.",
DeprecationWarning,
)
return self.global_options.b64_entropy_score

# Each 4-character base64 group represents 3 8-bit bytes, i.e. an effective
# bit rate of 24/4 = 6 bits per character. We want to scale the base score
# by this amount to account for the efficiency of the string representation
Expand Down Expand Up @@ -249,7 +228,6 @@ def included_paths(self) -> List[Pattern]:
if self._included_paths is None:
self.logger.info("Initializing included paths")
patterns: Set[str] = set()
deprecated = False
for pattern in tuple(
self.global_options.include_path_patterns or []
) + tuple(self.config_data.get("include_path_patterns", [])):
Expand All @@ -260,21 +238,10 @@ def included_paths(self) -> List[Pattern]:
raise types.ConfigException(
"Required key path-pattern missing in include-path-patterns"
) from exc
elif isinstance(pattern, str):
deprecated = True
patterns.add(pattern)
else:
raise types.ConfigException(
f"{type(pattern).__name__} pattern is illegal in include-path-patterns"
)
if deprecated:
warnings.warn(
"Old format of --include-path-patterns option and config file setup include-path-patterns "
"= ['inclusion pattern'] has been deprecated and will be removed in tartufo 4.x. "
"Make sure all the inclusions are set up using new pattern i.e. include-path-patterns = "
"[{path-pattern='inclusion pattern',reason='reason for inclusion'}] in the config file",
DeprecationWarning,
)
self._included_paths = config.compile_path_rules(patterns)
return self._included_paths

Expand All @@ -298,7 +265,6 @@ def excluded_paths(self) -> List[Pattern]:
if self._excluded_paths is None:
self.logger.info("Initializing excluded paths")
patterns: Set[str] = set()
deprecated = False
for pattern in tuple(
self.global_options.exclude_path_patterns or []
) + tuple(self.config_data.get("exclude_path_patterns", [])):
Expand All @@ -309,21 +275,10 @@ def excluded_paths(self) -> List[Pattern]:
raise types.ConfigException(
"Required key path-pattern missing in exclude-path-patterns"
) from exc
elif isinstance(pattern, str):
deprecated = True
patterns.add(pattern)
else:
raise types.ConfigException(
f"{type(pattern).__name__} pattern is illegal in exclude-path-patterns"
)
if deprecated:
warnings.warn(
"Old format of --exclude-path-patterns option and config file setup exclude-path-patterns "
"= ['exclusion pattern'] has been deprecated and will be removed in tartufo 4.x. "
"Make sure all the exclusions are set up using new pattern i.e. exclude-path-patterns = "
"[{path-pattern='exclusion pattern',reason='reason for exclusion'}] in the config file",
DeprecationWarning,
)
self._excluded_paths = config.compile_path_rules(patterns)
return self._excluded_paths

Expand All @@ -338,7 +293,6 @@ def rules_regexes(self) -> Set[Rule]:
try:
self._rules_regexes = config.configure_regexes(
include_default=self.global_options.default_regexes,
rules_files=self.global_options.rules,
rule_patterns=self.global_options.rule_patterns,
rules_repo=self.global_options.git_rules_repo,
rules_repo_files=self.global_options.git_rules_files,
Expand Down Expand Up @@ -386,7 +340,6 @@ def excluded_signatures(self) -> Tuple[str, ...]:
"""
if self._excluded_signatures is None:
signatures: Set[str] = set()
deprecated = False
for signature in tuple(
self.global_options.exclude_signatures or []
) + tuple(self.config_data.get("exclude_signatures", [])):
Expand All @@ -397,21 +350,10 @@ def excluded_signatures(self) -> Tuple[str, ...]:
raise types.ConfigException(
"Required key signature missing in exclude-signatures"
) from exc
elif isinstance(signature, str):
deprecated = True
signatures.add(signature)
else:
raise types.ConfigException(
f"{type(signature).__name__} signature is illegal in exclude-signatures"
)
if deprecated:
warnings.warn(
"Configuring exclude-signatures as string has been deprecated and support for this format will "
"be removed in tartufo 4.x. Please update your exclude-signatures configuration to "
"an array of tables. For example: exclude-signatures = [{signature='signature', reason='The "
"reason of excluding the signature'}]",
DeprecationWarning,
)
self._excluded_signatures = tuple(signatures)
return self._excluded_signatures

Expand Down Expand Up @@ -599,73 +541,38 @@ def scan_entropy(
# If the chunk is diff output, the first character of each line is
# generated metadata ("+", "-", etc.) that is not part of actual
# repository content, and it should be ignored.
extra_char: Optional[str]
if chunk.is_diff:
extra_char = line[0]
analyze = line[1:]
else:
extra_char = None
analyze = line
analyze = line[1:] if chunk.is_diff else line
for word in analyze.split():
for string in util.find_strings_by_regex(word, BASE64_REGEX):
yield from self.evaluate_entropy_string(
chunk, analyze, string, self.b64_entropy_limit, extra_char
chunk, analyze, string, self.b64_entropy_limit
)
for string in util.find_strings_by_regex(word, HEX_REGEX):
yield from self.evaluate_entropy_string(
chunk, analyze, string, self.hex_entropy_limit, extra_char
chunk, analyze, string, self.hex_entropy_limit
)
extra_char = None

def evaluate_entropy_string(
self,
chunk: types.Chunk,
line: str,
string: str,
min_entropy_score: float,
backwards_compatibility_prefix: Optional[str],
) -> Generator[Issue, None, None]:
"""Check entropy string using entropy characters and score.
:param chunk: The chunk of data to check
:param line: Source line containing string of interest
:param string: String to check
:param min_entropy_score: Minimum entropy score to flag
:param backwards_compatibility_prefix: Possible prefix character
:return: Generator of issues flagged
If the string in "string" would result in an Issue (i.e. it has high
entropy and is not excluded), and backwards_compatibility_prefix is not
None, re-check for exclusions based on "prefix" + "string". This preserves
the utility of signatures generated by earlier tartufo versions which did
not handle "diff" chunks correctly.
"""

if not self.signature_is_excluded(string, chunk.file_path):
entropy_score = self.calculate_entropy(string)
if entropy_score > min_entropy_score:
if self.entropy_string_is_excluded(string, line, chunk.file_path):
self.logger.debug("line containing entropy was excluded: %s", line)
elif (
backwards_compatibility_prefix is not None
and self.signature_is_excluded(
backwards_compatibility_prefix + string, chunk.file_path
)
):
self.logger.debug(
"line containing entropy was excluded (old signature): %s", line
)
# We should tell the user to update their old signature
new_signature = util.generate_signature(string, chunk.file_path)
old_signature = util.generate_signature(
backwards_compatibility_prefix + string, chunk.file_path
)
warnings.warn(
f"Signature {old_signature} was generated by an old version of tartufo and is deprecated. "
"tartufo 4.x will not recognize this signature. "
f"Please update your configuration to use signature {new_signature} instead.",
DeprecationWarning,
)

else:
yield Issue(types.IssueType.Entropy, string, chunk)
Expand Down
Loading

0 comments on commit 24976f1

Please sign in to comment.