Output tweaks

.coveragerc

@@ -15,4 +15,5 @@ source =
     ./tests
 omit =
     pyt/formatters/json.py
+    pyt/formatters/screen.py
     pyt/formatters/text.py

pyt/__main__.py

@@ -1,5 +1,6 @@
 """The comand line module of PyT."""
 
+import logging
 import os
 import sys
 from collections import defaultdict
@@ -12,10 +13,6 @@
     get_directory_modules,
     get_modules
 )
-from .formatters import (
-    json,
-    text
-)
 from .usage import parse_args
 from .vulnerabilities import (
     find_vulnerabilities,
@@ -30,6 +27,8 @@
     is_function_without_leading_
 )
 
+log = logging.getLogger(__name__)
+
 
 def discover_files(targets, excluded_files, recursive=False):
     included_files = list()
@@ -41,11 +40,13 @@ def discover_files(targets, excluded_files, recursive=False):
                     if file.endswith('.py') and file not in excluded_list:
                         fullpath = os.path.join(root, file)
                         included_files.append(fullpath)
+                        log.debug('Discovered file: %s', fullpath)
                 if not recursive:
                     break
         else:
             if target not in excluded_list:
                 included_files.append(target)
+                log.debug('Discovered file: %s', target)
     return included_files
 
 
@@ -64,6 +65,14 @@ def retrieve_nosec_lines(
 def main(command_line_args=sys.argv[1:]):  # noqa: C901
     args = parse_args(command_line_args)
 
+    logging_level = (
+        logging.ERROR if not args.verbose else
+        logging.WARN if args.verbose == 1 else
+        logging.INFO if args.verbose == 2 else
+        logging.DEBUG
+    )
+    logging.basicConfig(level=logging_level, format='[%(levelname)s] %(name)s: %(message)s')
+
     files = discover_files(
         args.targets,
         args.excluded_paths,
@@ -78,6 +87,7 @@ def main(command_line_args=sys.argv[1:]):  # noqa: C901
 
     cfg_list = list()
     for path in sorted(files):
+        log.info("Processing %s", path)
         if not args.ignore_nosec:
             nosec_lines[path] = retrieve_nosec_lines(path)
 
@@ -130,16 +140,13 @@ def main(command_line_args=sys.argv[1:]):  # noqa: C901
             args.baseline
         )
 
-    if args.json:
-        json.report(vulnerabilities, args.output_file)
-    else:
-        text.report(vulnerabilities, args.output_file)
+    args.formatter.report(vulnerabilities, args.output_file, not args.only_unsanitised)
 
-    has_unsanitized_vulnerabilities = any(
+    has_unsanitised_vulnerabilities = any(
         not isinstance(v, SanitisedVulnerability)
         for v in vulnerabilities
     )
-    if has_unsanitized_vulnerabilities:
+    if has_unsanitised_vulnerabilities:
         sys.exit(1)
 
 

pyt/cfg/stmt_visitor.py

@@ -1,6 +1,8 @@
 import ast
 import itertools
+import logging
 import os.path
+from pkgutil import iter_modules
 
 from .alias_helper import (
     as_alias_handler,
@@ -52,6 +54,9 @@
     remove_breaks
 )
 
+log = logging.getLogger(__name__)
+uninspectable_modules = {module.name for module in iter_modules()}  # Don't warn about failing to import these
+
 
 class StmtVisitor(ast.NodeVisitor):
     def __init__(self, allow_local_directory_imports=True):
@@ -429,9 +434,12 @@ def visit_Assign(self, node):
             else:
                 label = LabelVisitor()
                 label.visit(node)
-                print('Assignment not properly handled.',
-                      'Could result in not finding a vulnerability.',
-                      'Assignment:', label.result)
+                log.warn(
+                    'Assignment not properly handled in %s. Could result in not finding a vulnerability.'
+                    'Assignment: %s',
+                    getattr(self, 'filenames', ['?'])[0],
+                    self.label.result,
+                )
                 return self.append_node(AssignmentNode(
                     label.result,
                     label.result,
@@ -1022,6 +1030,10 @@ def visit_Import(self, node):
                         name.asname,
                         retrieve_import_alias_mapping(node.names)
                     )
+        for alias in node.names:
+            if alias.name not in uninspectable_modules:
+                log.warn("Cannot inspect module %s", alias.name)
+                uninspectable_modules.add(alias.name)  # Don't repeatedly warn about this
         return IgnoredNode()
 
     def visit_ImportFrom(self, node):
@@ -1061,4 +1073,7 @@ def visit_ImportFrom(self, node):
                         retrieve_import_alias_mapping(node.names),
                         from_from=True
                     )
+        if node.module not in uninspectable_modules:
+            log.warn("Cannot inspect module %s", node.module)
+            uninspectable_modules.add(node.module)
         return IgnoredNode()

pyt/core/ast_helper.py

@@ -2,25 +2,25 @@
 Useful when working with the ast module."""
 
 import ast
+import logging
 import os
 import subprocess
 from functools import lru_cache
 
 from .transformer import PytTransformer
 
-
+log = logging.getLogger(__name__)
 BLACK_LISTED_CALL_NAMES = ['self']
 recursive = False
 
 
 def _convert_to_3(path):  # pragma: no cover
     """Convert python 2 file to python 3."""
     try:
-        print('##### Trying to convert file to Python 3. #####')
+        log.warn('##### Trying to convert %s to Python 3. #####', path)
         subprocess.call(['2to3', '-w', path])
     except subprocess.SubprocessError:
-        print('Check if 2to3 is installed. '
-              'https://docs.python.org/2/library/2to3.html')
+        log.exception('Check if 2to3 is installed. https://docs.python.org/2/library/2to3.html')
         exit(1)
 
 

pyt/formatters/json.py

@@ -1,12 +1,14 @@
 """This formatter outputs the issues in JSON."""
-
 import json
 from datetime import datetime
 
+from ..vulnerabilities.vulnerability_helper import SanitisedVulnerability
+
 
 def report(
     vulnerabilities,
-    fileobj
+    fileobj,
+    print_sanitised,
 ):
     """
     Prints issues in JSON format.
@@ -19,7 +21,10 @@ def report(
 
     machine_output = {
         'generated_at': time_string,
-        'vulnerabilities': [vuln.as_dict() for vuln in vulnerabilities]
+        'vulnerabilities': [
+            vuln.as_dict() for vuln in vulnerabilities
+            if print_sanitised or not isinstance(vuln, SanitisedVulnerability)
+        ]
     }
 
     result = json.dumps(

pyt/formatters/screen.py

@@ -0,0 +1,104 @@
+"""This formatter outputs the issues as color-coded text."""
+from ..vulnerabilities.vulnerability_helper import SanitisedVulnerability, UnknownVulnerability
+
+RESET = '\033[0m'
+BOLD = '\033[1m'
+UNDERLINE = '\033[4m'
+DANGER = '\033[31m'
+GOOD = '\033[32m'
+HIGHLIGHT = '\033[45;1m'
+RED_ON_WHITE = '\033[31m\033[107m'
+
+
+def color(string, color_string):
+    return color_string + str(string) + RESET
+
+
+def report(
+    vulnerabilities,
+    fileobj,
+    print_sanitised,
+):
+    """
+    Prints issues in color-coded text format.
+
+    Args:
+        vulnerabilities: list of vulnerabilities to report
+        fileobj: The output file object, which may be sys.stdout
+    """
+    n_vulnerabilities = len(vulnerabilities)
+    unsanitised_vulnerabilities = [v for v in vulnerabilities if not isinstance(v, SanitisedVulnerability)]
+    n_unsanitised = len(unsanitised_vulnerabilities)
+    n_sanitised = n_vulnerabilities - n_unsanitised
+    heading = "{} vulnerabilit{} found{}.\n".format(
+        'No' if n_unsanitised == 0 else n_unsanitised,
+        'y' if n_unsanitised == 1 else 'ies',
+        " (plus {} sanitised)".format(n_sanitised) if n_sanitised else "",
+    )
+    vulnerabilities_to_print = vulnerabilities if print_sanitised else unsanitised_vulnerabilities
+    with fileobj:
+        for i, vulnerability in enumerate(vulnerabilities_to_print, start=1):
+            fileobj.write(vulnerability_to_str(i, vulnerability))
+
+        if n_unsanitised == 0:
+            fileobj.write(color(heading, GOOD))
+        else:
+            fileobj.write(color(heading, DANGER))
+
+
+def vulnerability_to_str(i, vulnerability):
+    lines = []
+    lines.append(color('Vulnerability {}'.format(i), UNDERLINE))
+    lines.append('File: {}'.format(color(vulnerability.source.path, BOLD)))
+    lines.append(
+        'User input at line {}, source "{}":'.format(
+            vulnerability.source.line_number,
+            color(vulnerability.source_trigger_word, HIGHLIGHT),
+        )
+    )
+    lines.append('\t{}'.format(color(vulnerability.source.label, RED_ON_WHITE)))
+    if vulnerability.reassignment_nodes:
+        previous_path = None
+        lines.append('Reassigned in:')
+        for node in vulnerability.reassignment_nodes:
+            if node.path != previous_path:
+                lines.append('\tFile: {}'.format(node.path))
+                previous_path = node.path
+            label = node.label
+            if (
+                isinstance(vulnerability, SanitisedVulnerability) and
+                node.label == vulnerability.sanitiser.label
+            ):
+                label = color(label, GOOD)
+            lines.append(
+                '\t  Line {}:\t{}'.format(
+                    node.line_number,
+                    label,
+                )
+            )
+    if vulnerability.source.path != vulnerability.sink.path:
+        lines.append('File: {}'.format(color(vulnerability.sink.path, BOLD)))
+    lines.append(
+        'Reaches line {}, sink "{}"'.format(
+            vulnerability.sink.line_number,
+            color(vulnerability.sink_trigger_word, HIGHLIGHT),
+        )
+    )
+    lines.append('\t{}'.format(
+        color(vulnerability.sink.label, RED_ON_WHITE)
+    ))
+    if isinstance(vulnerability, SanitisedVulnerability):
+        lines.append(
+            'This vulnerability is {}{} by {}'.format(
+                color('potentially ', BOLD) if not vulnerability.confident else '',
+                color('sanitised', GOOD),
+                color(vulnerability.sanitiser.label, BOLD),
+            )
+        )
+    elif isinstance(vulnerability, UnknownVulnerability):
+        lines.append(
+            'This vulnerability is unknown due to "{}"'.format(
+                color(vulnerability.unknown_assignment.label, BOLD),
+            )
+        )
+    return '\n'.join(lines) + '\n\n'

pyt/formatters/text.py

@@ -1,25 +1,33 @@
 """This formatter outputs the issues as plain text."""
+from ..vulnerabilities.vulnerability_helper import SanitisedVulnerability
 
 
 def report(
     vulnerabilities,
-    fileobj
+    fileobj,
+    print_sanitised,
 ):
     """
     Prints issues in text format.
 
     Args:
         vulnerabilities: list of vulnerabilities to report
         fileobj: The output file object, which may be sys.stdout
+        print_sanitised: Print just unsanitised vulnerabilities or sanitised vulnerabilities as well
     """
-    number_of_vulnerabilities = len(vulnerabilities)
+    n_vulnerabilities = len(vulnerabilities)
+    unsanitised_vulnerabilities = [v for v in vulnerabilities if not isinstance(v, SanitisedVulnerability)]
+    n_unsanitised = len(unsanitised_vulnerabilities)
+    n_sanitised = n_vulnerabilities - n_unsanitised
+    heading = "{} vulnerabilit{} found{}{}\n".format(
+        'No' if n_unsanitised == 0 else n_unsanitised,
+        'y' if n_unsanitised == 1 else 'ies',
+        " (plus {} sanitised)".format(n_sanitised) if n_sanitised else "",
+        ':' if n_vulnerabilities else '.',
+    )
+    vulnerabilities_to_print = vulnerabilities if print_sanitised else unsanitised_vulnerabilities
     with fileobj:
-        if number_of_vulnerabilities == 0:
-            fileobj.write('No vulnerabilities found.\n')
-        elif number_of_vulnerabilities == 1:
-            fileobj.write('%s vulnerability found:\n' % number_of_vulnerabilities)
-        else:
-            fileobj.write('%s vulnerabilities found:\n' % number_of_vulnerabilities)
+        fileobj.write(heading)
 
-        for i, vulnerability in enumerate(vulnerabilities, start=1):
+        for i, vulnerability in enumerate(vulnerabilities_to_print, start=1):
             fileobj.write('Vulnerability {}:\n{}\n\n'.format(i, vulnerability))