Merge pull request #877 from Captain-T2004/CREATE_SCAN_COMPARE

Added Scan Compare feature

nettacker/api/engine.py

@@ -27,8 +27,9 @@
 from nettacker.config import Config
 from nettacker.core.app import Nettacker
 from nettacker.core.die import die_failure
+from nettacker.core.graph import create_compare_report
 from nettacker.core.messages import messages as _
-from nettacker.core.utils.common import now
+from nettacker.core.utils.common import now, generate_compare_filepath
 from nettacker.database.db import (
     create_connection,
     get_logs_by_scan_id,
@@ -212,6 +213,43 @@ def new_scan():
     return jsonify(vars(nettacker_app.arguments)), 200
 
 
+@app.route("/compare/scans", methods=["POST"])
+def compare_scans():
+    """
+    compare two scans through the API
+    Returns:
+        Success if the comparision is successfull and report is saved and error if not.
+    """
+    api_key_is_valid(app, flask_request)
+
+    scan_id_first = get_value(flask_request, "scan_id_first")
+    scan_id_second = get_value(flask_request, "scan_id_second")
+    if not scan_id_first or not scan_id_second:
+        return jsonify(structure(status="error", msg="Invalid Scan IDs")), 400
+
+    compare_report_path_filename = get_value(flask_request, "compare_report_path")
+    if not compare_report_path_filename:
+        compare_report_path_filename = generate_compare_filepath(scan_id_first)
+
+    compare_options = {
+        "scan_compare_id": scan_id_second,
+        "compare_report_path_filename": compare_report_path_filename,
+    }
+
+    try:
+        result = create_compare_report(compare_options, scan_id_first)
+        if result:
+            return jsonify(
+                structure(
+                    status="success",
+                    msg="scan_comparison_completed",
+                )
+            ), 200
+        return jsonify(structure(status="error", msg="Scan ID not found")), 404
+    except (FileNotFoundError, PermissionError, IOError):
+        return jsonify(structure(status="error", msg="Invalid file path")), 400
+
+
 @app.route("/session/check", methods=["GET"])
 def session_check():
     """

nettacker/config.py

@@ -145,6 +145,8 @@ class DefaultSettings(ConfigBase):
     usernames_list = None
     verbose_event = False
     verbose_mode = False
+    scan_compare_id = None
+    compare_report_path_filename = ""
 
 
 class Config:

nettacker/core/app.py

@@ -12,7 +12,7 @@
 from nettacker.config import Config, version_info
 from nettacker.core.arg_parser import ArgParser
 from nettacker.core.die import die_failure
-from nettacker.core.graph import create_report
+from nettacker.core.graph import create_report, create_compare_report
 from nettacker.core.ip import (
     get_ip_range,
     generate_ip_range,
@@ -204,6 +204,8 @@ def run(self):
 
         exit_code = self.start_scan(scan_id)
         create_report(self.arguments, scan_id)
+        if self.arguments.scan_compare_id is not None:
+            create_compare_report(self.arguments, scan_id)
         log.info(_("done"))
 
         return exit_code
@@ -231,6 +233,7 @@ def start_scan(self, scan_id):
                             "target": target,
                             "module_name": module_name,
                             "scan_id": scan_id,
+                            "scan_compare_id": self.arguments.scan_compare_id,
                         }
                     )
 

nettacker/core/arg_parser.py

@@ -390,6 +390,22 @@ def add_arguments(self):
             default=Config.settings.ping_before_scan,
             help=_("ping_before_scan"),
         )
+        method_options.add_argument(
+            "-K",
+            "--scan-compare",
+            action="store",
+            dest="scan_compare_id",
+            default=Config.settings.scan_compare_id,
+            help=_("compare_scans"),
+        )
+        method_options.add_argument(
+            "-J",
+            "--compare-report-path",
+            action="store",
+            dest="compare_report_path_filename",
+            default=Config.settings.compare_report_path_filename,
+            help=_("compare_report_path_filename"),
+        )
 
         # API Options
         api_options = self.add_argument_group(_("API"), _("API_options"))

nettacker/core/graph.py

@@ -2,18 +2,25 @@
 import html
 import importlib
 import json
+import os
 from datetime import datetime
 
 import texttable
 
 from nettacker import logger
-from nettacker.config import version_info
+from nettacker.config import Config, version_info
 from nettacker.core.die import die_failure
 from nettacker.core.messages import messages as _
-from nettacker.core.utils.common import merge_logs_to_list, now
-from nettacker.database.db import get_logs_by_scan_id, submit_report_to_db
+from nettacker.core.utils.common import (
+    merge_logs_to_list,
+    now,
+    sanitize_path,
+    generate_compare_filepath,
+)
+from nettacker.database.db import get_logs_by_scan_id, submit_report_to_db, get_options_by_scan_id
 
 log = logger.get_logger()
+nettacker_path_config = Config.path
 
 
 def build_graph(graph_name, events):
@@ -42,6 +49,27 @@ def build_graph(graph_name, events):
     return start(events)
 
 
+def build_compare_report(compare_results):
+    """
+    build the compare report
+    Args:
+        compare_results: Final result of the comparision(dict)
+    Returns:
+        report in html format
+    """
+    log.info(_("build_compare_report"))
+    try:
+        build_report = getattr(
+            importlib.import_module("nettacker.lib.compare_report.engine"),
+            "build_report",
+        )
+    except ModuleNotFoundError:
+        die_failure(_("graph_module_unavailable").format("compare_report"))
+
+    log.info(_("finish_build_report"))
+    return build_report(compare_results)
+
+
 def build_text_table(events):
     """
     value['date'], value["target"], value['module_name'], value['scan_id'],
@@ -77,6 +105,20 @@ def build_text_table(events):
     )
 
 
+def create_compare_text_table(results):
+    table = texttable.Texttable()
+    table_headers = list(results.keys())
+    table.add_rows([table_headers])
+    table.add_rows(
+        [
+            table_headers,
+            [results[col] for col in table_headers],
+        ]
+    )
+    table.set_cols_width([len(i) for i in table_headers])
+    return table.draw() + "\n\n"
+
+
 def create_report(options, scan_id):
     """
     sort all events, create log file in HTML/TEXT/JSON and remove old logs
@@ -168,3 +210,89 @@ def create_report(options, scan_id):
 
     log.info(_("file_saved").format(report_path_filename))
     return True
+
+
+def create_compare_report(options, scan_id):
+    """
+    if compare_id is given then create the report of comparision b/w scans
+    Args:
+        options: parsing options
+        scan_id: scan unique id
+    Returns:
+        True if success otherwise None
+    """
+    comp_id = options["scan_compare_id"] if isinstance(options, dict) else options.scan_compare_id
+    scan_log_curr = get_logs_by_scan_id(scan_id)
+    scan_logs_comp = get_logs_by_scan_id(comp_id)
+
+    if not scan_log_curr:
+        log.info(_("no_events_for_report"))
+        return None
+    if not scan_logs_comp:
+        log.info(_("no_scan_to_compare"))
+        return None
+
+    scan_opts_curr = get_options_by_scan_id(scan_id)
+    scan_opts_comp = get_options_by_scan_id(comp_id)
+
+    def get_targets_set(item):
+        return tuple(json.loads(item["options"])["targets"])
+
+    curr_target_set = set(get_targets_set(item) for item in scan_opts_curr)
+    comp_target_set = set(get_targets_set(item) for item in scan_opts_comp)
+
+    def get_modules_ports(item):
+        return (item["target"], item["module_name"], item["port"])
+
+    curr_modules_ports = set(get_modules_ports(item) for item in scan_log_curr)
+    comp_modules_ports = set(get_modules_ports(item) for item in scan_logs_comp)
+
+    compare_results = {
+        "curr_scan_details": (scan_id, scan_log_curr[0]["date"]),
+        "comp_scan_details": (comp_id, scan_logs_comp[0]["date"]),
+        "curr_target_set": tuple(curr_target_set),
+        "comp_target_set": tuple(comp_target_set),
+        "curr_scan_result": tuple(curr_modules_ports),
+        "comp_scan_result": tuple(comp_modules_ports),
+        "new_targets_discovered": tuple(curr_modules_ports - comp_modules_ports),
+        "old_targets_not_detected": tuple(comp_modules_ports - curr_modules_ports),
+    }
+    if isinstance(options, dict):
+        compare_report_path_filename = options["compare_report_path_filename"]
+    else:
+        compare_report_path_filename = (
+            options.compare_report_path_filename
+            if len(options.compare_report_path_filename) != 0
+            else generate_compare_filepath(scan_id)
+        )
+
+    base_path = str(nettacker_path_config.results_dir)
+    compare_report_path_filename = sanitize_path(compare_report_path_filename)
+    fullpath = os.path.normpath(os.path.join(base_path, compare_report_path_filename))
+
+    if not fullpath.startswith(base_path):
+        raise PermissionError
+
+    if (len(fullpath) >= 5 and fullpath[-5:] == ".html") or (
+        len(fullpath) >= 4 and fullpath[-4:] == ".htm"
+    ):
+        html_report = build_compare_report(compare_results)
+        with open(fullpath, "w", encoding="utf-8") as compare_report:
+            compare_report.write(html_report + "\n")
+    elif len(fullpath) >= 5 and fullpath[-5:] == ".json":
+        with open(fullpath, "w", encoding="utf-8") as compare_report:
+            compare_report.write(str(json.dumps(compare_results)) + "\n")
+    elif len(fullpath) >= 5 and fullpath[-4:] == ".csv":
+        keys = compare_results.keys()
+        with open(fullpath, "a") as csvfile:
+            writer = csv.DictWriter(csvfile, fieldnames=keys)
+            if csvfile.tell() == 0:
+                writer.writeheader()
+            writer.writerow(compare_results)
+    else:
+        with open(fullpath, "w", encoding="utf-8") as compare_report:
+            compare_report.write(create_compare_text_table(compare_results))
+
+    log.write(create_compare_text_table(compare_results))
+    log.info(_("compare_report_saved").format(fullpath))
+    return True

nettacker/core/utils/common.py

@@ -118,10 +118,12 @@ def generate_new_sub_steps(sub_steps, data_matrix, arrays):
                 exec(
                     "original_sub_steps{key_name} = {matrix_value}".format(
                         key_name=re_address_repeaters_key_name(array_name),
-                        matrix_value='"' + str(array[array_name_position]) + '"'
-                        if isinstance(array[array_name_position], int)
-                        or isinstance(array[array_name_position], str)
-                        else array[array_name_position],
+                        matrix_value=(
+                            '"' + str(array[array_name_position]) + '"'
+                            if isinstance(array[array_name_position], int)
+                            or isinstance(array[array_name_position], str)
+                            else array[array_name_position]
+                        ),
                     )
                 )
             array_name_position += 1
@@ -350,3 +352,28 @@ def sort_dictionary(dictionary):
     if etc_flag:
         sorted_dictionary["..."] = {}
     return sorted_dictionary
+
+
+def sanitize_path(path):
+    """
+    Sanitize the file path to preven unathorized access
+    Args:
+        path: filepath(user input)
+
+    Returns:
+        sanitized_path
+    """
+    return "_".join(
+        [
+            component
+            for component in re.split(r"[/\\]", path)
+            if re.match(r"^[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)?$", component)
+        ]
+    )
+
+
+def generate_compare_filepath(scan_id):
+    return "/report_compare_{date_time}_{scan_id}.json".format(
+        date_time=now(format="%Y_%m_%d_%H_%M_%S"),
+        scan_id=scan_id,
+    )

nettacker/database/db.py

@@ -117,6 +117,8 @@ def remove_old_logs(options):
         HostsLog.target == options["target"],
         HostsLog.module_name == options["module_name"],
         HostsLog.scan_unique_id != options["scan_id"],
+        HostsLog.scan_unique_id != options["scan_compare_id"],
+        # Don't remove old logs if they are to be used for the scan reports
     ).delete(synchronize_session=False)
     return send_submit_query(session)
 
@@ -362,6 +364,21 @@ def get_logs_by_scan_id(scan_id):
     ]
 
 
+def get_options_by_scan_id(scan_id):
+    """
+    select all stored options of the scan by scan id hash
+    Args:
+        scan_id: scan id hash
+    Returns:
+        an array with a dict with stored options or an empty array
+    """
+    session = create_connection()
+    return [
+        {"options": log.options}
+        for log in session.query(Report).filter(Report.scan_unique_id == scan_id).all()
+    ]
+
+
 def logs_to_report_json(target):
     """
     select all reports of a host

nettacker/lib/compare_report/engine.py

@@ -0,0 +1,21 @@
+import json
+
+from nettacker.config import Config
+
+
+def build_report(compare_result):
+    """
+    generate a report based on result of comparision b/w scans
+
+    Args:
+        compare_result: dict with result of the compare
+
+    Returns:
+        Compare report in HTML
+    """
+    data = (
+        open(Config.path.web_static_dir / "report/compare_report.html")
+        .read()
+        .replace("__data_will_locate_here__", json.dumps(compare_result))
+    )
+    return data

nettacker/locale/en.yaml

@@ -117,4 +117,9 @@ username_list: username(s) list, separate with ","
 verbose_mode: verbose mode level (0-5) (default 0)
 wrong_hardware_usage: "You must select one of these profiles for hardware usage. (low, normal, high, maximum)"
 invalid_scan_id: your scan id is not valid!
-
+compare_scans: compare current scan to old scans using the unique scan_id
+compare_report_path_filename: the file-path to store the compare_scan report
+no_scan_to_compare: the scan_id to be compared not found
+compare_report_saved: "compare results saved in {0}"
+build_compare_report: "building compare report"
+finish_build_report: "Finished building compare report"

nettacker/web/static/css/style.css

@@ -2,7 +2,7 @@ body {
     background: url("/img/background.jpeg")
 }
 
-#new_scan, #home, #get_results,#crawler_area {
+#new_scan, #home, #get_results,#crawler_area,#compare_area{
     background: #FEFCFF;
     padding: 20px;
     border-radius: 5px;