report html

artemis/frontend.py

@@ -219,6 +219,58 @@ def export_delete_form(request: Request, id: int, csrf_protect: CsrfProtect = De
     )
 
 
+@router.get("/export/view/{id}", include_in_schema=False)
+def view_export(request: Request, id: int) -> Response:
+    task = db.get_report_generation_task(id)
+
+    if not task:
+        raise HTTPException(status_code=404, detail="Report generation task not found")
+
+    vulnerabilities = []
+
+    num_high_severity = 0
+    num_medium_severity = 0
+    num_low_severity = 0
+
+    if task.status == "done":
+        domain_messages = {}
+        messages_path = os.path.join(task.output_location, "messages")
+        for item in os.listdir(messages_path):
+            with open(os.path.join(messages_path, item)) as f:
+                domain_messages[item.removesuffix(".html")] = f.read()
+
+        with open(os.path.join(task.output_location, "advanced", "output.json")) as f:
+            data = json.load(f)
+
+            for message in data["messages"].values():
+                for report in message["reports"]:
+                    vulnerabilities.append(report)
+                    if report["severity"] == "high":
+                        num_high_severity += 1
+                    elif report["severity"] == "medium":
+                        num_medium_severity += 1
+                    elif report["severity"] == "low":
+                        num_low_severity += 1
+                    else:
+                        assert False
+
+    else:
+        domain_messages = {}
+
+    return templates.TemplateResponse(
+        "view_export.jinja2",
+        {
+            "domain_messages": domain_messages,
+            "request": request,
+            "num_high_severity": num_high_severity,
+            "num_medium_severity": num_medium_severity,
+            "num_low_severity": num_low_severity,
+            "vulnerabilities": vulnerabilities,
+            "task": task,
+        },
+    )
+
+
 @router.post("/export/confirm-delete/{id}", include_in_schema=False)
 @csrf.validate_csrf
 async def post_export_delete(request: Request, id: int, csrf_protect: CsrfProtect = Depends()) -> Response:

artemis/reporting/base/report.py

@@ -61,6 +61,9 @@ class Report:
     # The severity (added during report post-processing)
     severity: Optional[Severity] = None
 
+    # HTML render of the report (added during post-processing)
+    html: Optional[str] = None
+
     def __post_init__(self) -> None:
         # Sanity check - at this moment, only URLs and domains are supported
         assert self.target_is_url() or self.target_is_domain()

artemis/reporting/export/main.py

@@ -6,6 +6,7 @@
 from pathlib import Path
 from typing import Dict, Optional
 
+import bs4
 import termcolor
 import typer
 from jinja2 import BaseLoader, Environment, StrictUndefined, Template
@@ -39,6 +40,22 @@
 HOST_ROOT_PATH = "/host-root/"
 
 
+def unwrap(html: str) -> str:
+    """Uwraps html if it's wrapped in a single tag (e.g. <div>)."""
+    html = html.strip()
+    soup = bs4.BeautifulSoup(html)
+    while len(list(soup.children)) == 1:
+        only_child = list(soup.children)[0]
+
+        if only_child.name:  # type: ignore
+            only_child.unwrap()
+            soup = bs4.BeautifulSoup(soup.renderContents().strip())
+        else:
+            break
+
+    return soup.renderContents().decode("utf-8", "ignore")
+
+
 def _build_message_template_and_print_path(output_dir: Path, silent: bool) -> Template:
     output_message_template_file_name = output_dir / "advanced" / "message_template.jinja2"
 
@@ -94,6 +111,17 @@ def _build_messages_and_print_path(
         with open(output_messages_directory_name / (top_level_target_shortened + ".html"), "w") as f:
             f.write(message_template.render({"data": export_data_dict["messages"][top_level_target]}))
 
+    for message in export_data.messages.values():
+        for report in message.reports:
+            message_data = {
+                "contains_type": {report.report_type},
+                "reports": [report],
+                "custom_template_arguments": message.custom_template_arguments,
+            }
+            message_data["custom_template_arguments"]["skip_html_and_body_tags"] = True  # type: ignore
+            message_data["custom_template_arguments"]["skip_header_and_footer_text"] = True  # type: ignore
+            report.html = unwrap(message_template.render({"data": message_data}))
+
     if not silent:
         print()
         print(termcolor.colored(f"Messages written to: {output_messages_directory_name}", attrs=["bold"]))
@@ -131,8 +159,9 @@ def export(
     if not skip_hooks:
         run_export_hooks(output_dir, export_data, silent)
 
-    _dump_export_data_and_print_path(export_data, output_dir, silent)
     message_template = _build_message_template_and_print_path(output_dir, silent)
+    _build_messages_and_print_path(message_template, export_data, output_dir, silent)
+    _dump_export_data_and_print_path(export_data, output_dir, silent)
 
     print_and_save_stats(export_data, output_dir, silent)
 
@@ -149,8 +178,6 @@ def export(
         for tag in sorted([key for key in export_db_connector.tag_stats.keys() if key]):
             print(f"\t{tag}: {export_db_connector.tag_stats[tag]}")
 
-    _build_messages_and_print_path(message_template, export_data, output_dir, silent)
-
     if not silent:
         for alert in export_data.alerts:
             print(termcolor.colored("ALERT:" + alert, color="red"))

templates/exports.jinja2

@@ -88,6 +88,7 @@
                         <td>
                             {%  if report_generation_task.status == "done" %}
                                 <a href="/export/download-zip/{{ report_generation_task.id }}">Download ZIP</a> |
+                                <a href="/export/view/{{ report_generation_task.id }}">Browse</a> |
                             {%  elif report_generation_task.status == "failed" %}
                                 <a href="#" data-bs-toggle="modal" data-bs-target="#error-{{  report_generation_task.id }}">
                                   Show error

templates/view_export.jinja2

@@ -0,0 +1,112 @@
+{% extends "components/base.jinja2" %}
+{% block main %}
+    <h2>Export task detail</h2>
+    <dl>
+        <dt>Created at:</dt><dd>{{ task.created_at }}</dd>
+        <dt>Tag:</dt><dd>{% if task.tag %}{{ task.tag }}{% else %}None{% endif %}</dd>
+        <dt>Comment:</dt><dd>{% if task.comment %}{{ task.comment }}{% else %}None{% endif %}</dd>
+        <dt>Status:</dt><dd>{{ task.status }}</dd>
+
+        {% if task.status == "done" %}
+            <dt>Number of high-severity vulnerabilities:</dt><dd>{{ num_high_severity }}</dd>
+            <dt>Number of medium-severity vulnerabilities:</dt><dd>{{ num_medium_severity }}</dd>
+            <dt>Number of low-severity vulnerabilities:</dt><dd>{{ num_low_severity }}</dd>
+        {% endif %}
+    </dl>
+
+    {% if task.status == "done" %}
+        <a class="btn btn-primary mt-4 mb-4 href="/export/download-zip/{{ task.id }}">Download ZIP</a>
+
+        <ul class="nav nav-tabs" role="tablist">
+          <li class="nav-item" role="presentation">
+              <a class="nav-link active" id="browse-messages-link" href="#" data-bs-toggle="tab" data-bs-target="#browse-messages" aria-selected="true">Browse messages</a>
+          </li>
+          <li class="nav-item" role="presentation">
+              <a class="nav-link" id="browse-vulnerabilities-link" href="#" data-bs-toggle="tab" data-bs-target="#browse-vulnerabilities" aria-selected="false">Browse vulnerabilities</a>
+          </li>
+          {% if task.alerts %}
+              <li class="nav-item" role="presentation">
+                  <a class="nav-link" id="alerts-link" href="#" data-bs-toggle="tab" data-bs-target="#alerts" aria-selected="false">Alerts ({{ len(task.alerts) }})</a>
+              </li>
+          {% endif %}
+        </ul>
+        <div class="tab-content">
+            <div id="browse-messages" class="tab-pane pt-4 active show" role="tabpanel" aria-labelledby="browse-messages-link">
+                {% if domain_messages %}
+                    {% for domain, message in domain_messages.items() %}
+                        <h3>{{ domain }}</h3>
+                        {{ message|safe }}
+                    {% endfor %}
+                {% else %}
+                    None
+                {% endif %}
+            </div>
+            <div id="browse-vulnerabilities" class="tab-pane pt-4" role="tabpanel" aria-labelledby="browse-vulnerabilities-link">
+                {% if vulnerabilities %}
+                    <table class="table" id="vulnerabilities-table" style="word-break: break-word">
+                        <thead>
+                            <tr>
+                                <th style="width: 10%">Severity</th>
+                                <th style="width: 20%">Type</th>
+                                <th style="width: 20%">Target</th>
+                                <th style="width: 50%">Vulnerability</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for vulnerability in vulnerabilities %}
+                                <tr>
+                                    <td>
+                                        <span class="text-{% if vulnerability.severity == "high" %}danger{% elif vulnerability.severity == "medium" %}warning{% else %}info{% endif %}">
+                                            {% if vulnerability.severity == "high" %}1.
+                                            {% elif vulnerability.severity == "medium" %}2.
+                                            {% else %}3.{% endif %}
+
+                                            {{ vulnerability.severity }}
+                                        </span>
+                                    </td>
+                                    <td>{{ vulnerability.report_type }}</td>
+                                    <td>{{ vulnerability.target }}</td>
+                                    <td>
+                                        {{ vulnerability.html|safe }}
+
+                                        <p>
+                                            Found: {{ vulnerability.timestamp }}
+                                        </p>
+                                    </td>
+                                </tr>
+                            {% endfor %}
+                        </tbody>
+                    </table>
+                {% else %}
+                    None
+                {% endif %}
+            </div>
+            <div id="alerts" class="tab-pane" role="tabpanel" aria-labelledby="alerts-link">
+                {% if task.alerts %}
+                    {% for alert in task.alerts %}
+                        <div class="alert alert-danger mb-4">
+                            {{ alert }}
+                        </div>
+                    {% endfor %}
+                {% else %}
+                    None
+                {% endif %}
+            </div>
+        </div>
+    {% elif task.status == "failed" %}
+        <pre>{{ task.error }}</pre>
+    {% endif %}
+{% endblock %}
+
+{% block scripts %}
+    <script>
+        $(document).ready(function() {
+            const table = $("#vulnerabilities-table").dataTable({
+            });
+
+            document.getElementById('browse-vulnerabilities-link').onclick = function() {
+                table.columns.adjust().draw();
+            }
+        });
+    </script>
+{% endblock %}

test/e2e/test_exporting.py

@@ -1,3 +1,4 @@
+import json
 import re
 import tempfile
 import time
@@ -106,6 +107,28 @@ def test_exporting_gui(self) -> None:
                     ).encode("ascii"),
                 )
 
+            with export.open("advanced/output.json", "r") as f:
+                output_data = json.loads(f.read().decode("ascii"))
+                self.assertEqual(
+                    output_data["messages"]["test-smtp-server.artemis"]["reports"][0]["html"],
+                    "\n".join(
+                        [
+                            "The following domains don't have properly configured e-mail sender verification mechanisms:        <ul>",
+                            "<li>",
+                            "                        test-smtp-server.artemis:",
+                            "",
+                            "                            Valid DMARC record not found. We recommend using all three mechanisms: SPF, DKIM and DMARC to decrease the possibility of successful e-mail message spoofing.",
+                            "                        ",
+                            "                    </li>",
+                            "</ul>",
+                            "<p>",
+                            "            These mechanisms greatly increase the chance that the recipient server will reject a spoofed message.",
+                            "            Even if a domain is not used to send e-mails, SPF and DMARC records are needed to reduce the possibility to spoof e-mails.",
+                            "        </p>",
+                        ]
+                    ),
+                )
+
     def test_exporting_api(self) -> None:
         self.submit_tasks_with_modules_enabled(
             ["test-smtp-server.artemis"], "exporting-api", ["mail_dns_scanner", "classifier"]