Added Humble

artemis/config.py

@@ -229,6 +229,17 @@ class Gau:
                 "Additional command-line options that will be passed to gau (https://github.com/lc/gau).",
             ] = get_config("GAU_ADDITIONAL_OPTIONS", default="", cast=decouple.Csv(str, delimiter=" "))
 
+        class Humble:
+            HUMBLE_HEADERS_TO_REPORT: Annotated[
+                List[str],
+                "The list of headers that are considered more important and will be mentioned in the generated text "
+                "reports (all of the missing headers will be visible in the UI).",
+            ] = get_config(
+                "HUMBLE_HEADERS_TO_REPORT",
+                default=",".join(["Content-Security-Policy", "Strict-Transport-Security", "X-Content-Type-Options"]),
+                cast=decouple.Csv(str, delimiter=","),
+            )
+
         class Nuclei:
             NUCLEI_CHECK_TEMPLATE_LIST: Annotated[
                 bool,

artemis/modules/humble.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+import dataclasses
+import json
+import os
+import subprocess
+from typing import Any, Dict, List
+
+from karton.core import Task
+
+from artemis.binds import Service, TaskStatus, TaskType
+from artemis.module_base import ArtemisBase
+from artemis.task_utils import get_target_url
+
+
+@dataclasses.dataclass
+class Message:
+    category: str
+    problems: List[str]
+
+    @property
+    def message(self) -> str:
+        return f"{self.category}: {', '.join(self.problems)}"
+
+
+def process_json_data(result: Dict[str, Any]) -> List[Message]:
+    messages: Dict[str, Message] = {}
+
+    # Iterate through key-value pairs in the result
+    for key, value in result.items():
+        # Split the key to extract the relevant part
+        key_parts = key.replace("[", "").replace("]", "").split(". ")
+
+        if key in [
+            "[2. Fingerprint HTTP Response Headers]",
+            "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]",
+            "[4. Empty HTTP Response Headers Values]",
+            "[5. Browser Compatibility for Enabled HTTP Security Headers]",
+        ]:
+            continue
+
+        # Check if the key has the expected structure
+        if len(key_parts) >= 2:
+            category = key_parts[1].capitalize()
+
+            # Check if the key is not in the excluded categories and there are relevant values
+            if category.lower() != "info":
+                # If the value is a dictionary, iterate through subkey-value pairs
+                if isinstance(value, dict):
+                    for subkey, subvalue in value.items():
+                        # Add subkeys and subvalues to messages
+                        if subvalue and subvalue not in [
+                            "Nothing to report, all seems OK!",
+                            "No HTTP security headers are enabled.",
+                        ]:
+                            problem = f"{subkey} {subvalue}"
+                            if category not in messages:
+                                messages[category] = Message(category=category, problems=[])
+                            messages[category].problems.append(problem)
+
+                # If the value is a list, iterate through list items
+                elif isinstance(value, list):
+                    for item in value:
+                        # Add list items to messages
+                        if item and item not in [
+                            "Nothing to report, all seems OK!",
+                            "No HTTP security headers are enabled.",
+                        ]:
+                            if category not in messages:
+                                messages[category] = Message(category=category, problems=[])
+                            messages[category].problems.append(item)
+
+    return list(messages.values())
+
+
+class Humble(ArtemisBase):
+    """
+    Runs humble -> A HTTP Headers Analyzer
+    """
+
+    identity = "humble"
+    filters = [
+        {"type": TaskType.SERVICE.value, "service": Service.HTTP.value},
+    ]
+
+    def run(self, current_task: Task) -> None:
+        base_url = get_target_url(current_task)
+
+        data = subprocess.check_output(
+            [
+                "python3",
+                "humble.py",
+                "-u",
+                base_url,
+                "-b",
+                "-o",
+                "json",
+            ],
+            cwd="/humble",
+            stderr=subprocess.DEVNULL,
+        )
+
+        # strip boilerplatetext from the output to get the location and filename of the output file
+        filename = (
+            data.decode("ascii", errors="ignore")
+            .removeprefix("\n Analyzing URL and saving the report, please wait ...\n\n\n Report saved to ")
+            .removesuffix("\n")
+        )
+        data_str = open(filename, "r").read()
+
+        # cleanup file
+        os.unlink(filename)
+
+        # Check if the input string is empty
+        if data_str.strip():
+            result = json.loads(data_str)
+        else:
+            result = []
+
+        # Parse the JSON data
+        messages = process_json_data(result)
+
+        if messages:
+            status = TaskStatus.INTERESTING
+            status_reason = ", ".join([message.message for message in messages])
+        else:
+            status = TaskStatus.OK
+            status_reason = None
+
+        self.db.save_task_result(
+            task=current_task,
+            status=status,
+            status_reason=status_reason,
+            data={
+                "original_result": result,
+                "message_data": [dataclasses.asdict(message) for message in messages],
+                "messages": [message.message for message in messages],
+            },
+        )
+
+
+if __name__ == "__main__":
+    Humble().loop()

artemis/reporting/base/reporter.py

@@ -65,8 +65,16 @@ def get_scoring_rules(cls) -> Dict[ReportType, Callable[[Report], List[int]]]:
         return {report_type: Reporter.default_scoring_rule for report_type in cls.get_report_types()}
 
     @staticmethod
-    def dict_to_tuple(d: Dict[str, str]) -> Tuple[Tuple[str, str], ...]:
-        return tuple(d.items())
+    def dict_to_tuple(d: Dict[str, Any]) -> Tuple[Any, ...]:
+        result = []
+        for key, value in d.items():
+            if isinstance(value, dict):
+                result.append((key, Reporter.dict_to_tuple(value)))
+            elif isinstance(value, list):
+                result.append((key, tuple(value)))
+            else:
+                result.append((key, value))
+        return tuple(result)
 
     @staticmethod
     def default_scoring_rule(report: Report) -> List[int]:

artemis/reporting/modules/humble/reporter.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+from typing import Any, Callable, Dict, List
+
+from artemis.config import Config
+from artemis.reporting.base.language import Language
+from artemis.reporting.base.normal_form import NormalForm, get_url_normal_form
+from artemis.reporting.base.report import Report
+from artemis.reporting.base.report_type import ReportType
+from artemis.reporting.base.reporter import Reporter
+from artemis.reporting.base.templating import ReportEmailTemplateFragment
+from artemis.reporting.utils import get_target_url, get_top_level_target
+
+
+class HumbleReporter(Reporter):
+    MISSING_SECURITY_HEADERS = ReportType("missing_security_headers")
+
+    @staticmethod
+    def create_reports(task_result: Dict[str, Any], language: Language) -> List[Report]:
+        if task_result["headers"]["receiver"] != "humble":
+            return []
+
+        if not isinstance(task_result["result"], dict):
+            return []
+
+        return [
+            Report(
+                top_level_target=get_top_level_target(task_result),
+                target=get_target_url(task_result),
+                report_type=HumbleReporter.MISSING_SECURITY_HEADERS,
+                additional_data={
+                    "message_data": HumbleReporter._filter_message_data(task_result["result"]["message_data"]),
+                },
+                timestamp=task_result["created_at"],
+            )
+        ]
+
+    @staticmethod
+    def get_email_template_fragments() -> List[ReportEmailTemplateFragment]:
+        return [
+            ReportEmailTemplateFragment.from_file(
+                str(Path(__file__).parents[0] / "template_missing_security_headers.jinja2"), priority=2
+            ),
+        ]
+
+    @staticmethod
+    def get_normal_form_rules() -> Dict[ReportType, Callable[[Report], NormalForm]]:
+        """See the docstring in the Reporter class."""
+        return {
+            HumbleReporter.MISSING_SECURITY_HEADERS: lambda report: Reporter.dict_to_tuple(
+                {
+                    "type": report.report_type,
+                    "target": get_url_normal_form(report.target),
+                    "message_data": [Reporter.dict_to_tuple(item) for item in report.additional_data["message_data"]],
+                }
+            )
+        }
+
+    @staticmethod
+    def _filter_message_data(message_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        result: List[Dict[str, Any]] = []
+        for item in message_data:
+            if item["category"] == "Missing http security headers":
+                problems = sorted(set(item["problems"]) & set(Config.Modules.Humble.HUMBLE_HEADERS_TO_REPORT))
+                if problems:
+                    result.append({"category": item["category"], "problems": problems})
+            else:
+                result.append(item)
+        return result

artemis/reporting/modules/humble/template_missing_security_headers.jinja2

@@ -0,0 +1,22 @@
+{% if "missing_security_headers" in data.contains_type %}
+    <li>{% trans %}We identified that the following security headers are not set correctly:{% endtrans %}
+        <ul>
+            {% for report in data.reports %}
+                {% if report.report_type == "missing_security_headers" %}
+                    {% for message in report.additional_data.message_data %}
+                        <li>
+                            {{ report.target }}: {{ _(message.category) }}: {{ ', '.join(message.problems) }}
+                        </li>
+                    {% endfor %}
+                    {{ report_meta(report) }}
+                {% endif %}
+            {% endfor %}
+        </ul>
+        <p>
+            {% trans trimmed %}
+                Please verify the configuration, and, if a security header is missing, add it. Security
+                headers can have a deep impact on protecting your application against attacks.
+            {% endtrans %}
+        </p>
+    </li>
+{% endif %}

artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po

@@ -0,0 +1,10 @@
+#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:2
+msgid "We identified that the following security headers are not set correctly:"
+msgstr ""
+
+#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:16
+msgid ""
+"Please verify the configuration, and, if a security header is missing, "
+"add it. Security headers can have a deep impact on protecting your "
+"application against attacks."
+msgstr ""

artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po

@@ -0,0 +1,7 @@
+# This is a separate file so that pybabel doesn't mark it as obsolete because it doesn't
+# see the original strings in our code (they come from the humble library).
+
+msgid ""
+"Missing http security headers"
+msgstr ""
+"Brakujące nagłówki HTTP zwiększające bezpieczeństwo"

artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po

@@ -0,0 +1,15 @@
+#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:2
+msgid "We identified that the following security headers are not set correctly:"
+msgstr ""
+"Wykryto, że następujące nagłówki HTTP zwiększające bezpieczeństwo nie są "
+"ustawione poprawnie:"
+
+#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:16
+msgid ""
+"Please verify the configuration, and, if a security header is missing, "
+"add it. Security headers can have a deep impact on protecting your "
+"application against attacks."
+msgstr ""
+"Rekomendujemy weryfikację konfiguracji i zmianę lub ustawienie nagłówka, "
+"jeśli nie jest skonfigurowany poprawnie. Poprawna konfiguracja powyższych"
+" nagłówków zwiększy bezpieczeństwo aplikacji."

artemis/reporting/modules/nuclei/translations/nuclei_messages/pl_PL.py

@@ -584,6 +584,7 @@
     "Execution After Redirect happens when after emitting a Location header that redirects the user, some other code is executed. This may lead to data leak or application compromise.": "Wykryto podatność Execution After Redirect, czyli sytuację, gdy serwer, pomimo przekierowania użytkownika na inny adres, kontynuuje wykonanie skryptu. Może to doprowadzić do wycieku wrażliwych danych lub uzyskania przez atakującego nieuprawnionego dostępu do aplikacji.",
     "[no description] http/misconfiguration/django-debug-detect.yaml": "Wykryto system Django w konfiguracji debug. Upublicznienie systemu w takiej konfiguracji może umożliwić atakującemu poznanie informacji na temat działania aplikacji lub jej konfiguracji.",
     "DOMPDF Configuration page was detected, which contains paths, library versions and other potentially sensitive information": "Wykryto stronę konfiguracyjną DOMPDF, która zawiera ścieżki, wersje zainstalowanego oprogramowania i inne potencjalnie wrażliwe informacje.",
+    "This check detects if there are any active content loaded over HTTP instead of HTTPS.": "Wykryto, że zasoby takie jak skrypty są ładowane za pomocą nieszyfrowanego połączenia. Może to umożliwić atakującemu ich podmianę, a w konsekwencji zmianę wyglądu lub zachowania strony.",
     "OwnCloud is susceptible to the Installation page exposure due to misconfiguration.": "wykryto, że panel instalacyjny narzędzia OwnCloud jest publicznie dostępny, co może umożliwić atakującemu nieuprawniony dostęp do systemu.",
     "phpMyAdmin panel was detected.": "wykryto panel logowania narzędzia phpMyAdmin.",
     "WordPress login panel was detected.": "wykryto panel logowania systemu WordPress.",

artemis/reporting/severity.py

@@ -52,4 +52,5 @@ class Severity(str, Enum):
     ReportType("exposed_php_var_dump"): Severity.LOW,
     ReportType("exposed_phpinfo"): Severity.LOW,
     ReportType("nuclei_exposed_panel"): Severity.LOW,
+    ReportType("missing_security_headers"): Severity.LOW,
 }

docker-compose.yaml

@@ -172,6 +172,16 @@ services:
     restart: always
     volumes: ["./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/"]
 
+  karton-humble:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+    command: "python3 -m artemis.modules.humble"
+    depends_on: [karton-logger]
+    env_file: .env
+    restart: always
+    volumes: [ "./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/" ]
+
   karton-identifier:
     build:
       context: .

docker/Dockerfile

@@ -15,6 +15,9 @@ RUN git clone https://github.com/projectdiscovery/nuclei.git -b v2.9.7 /nuclei
 COPY docker/patches/nuclei-rate-limiting.patch /nuclei/
 RUN cd /nuclei && git apply nuclei-rate-limiting.patch  && cd v2/cmd/nuclei && go build && GOBIN=/usr/local/bin/ go install
 
+RUN git clone https://github.com/rfc-st/humble.git --branch master /humble
+RUN pip install -r /humble/requirements.txt
+
 COPY requirements.txt requirements.txt
 RUN pip install --no-cache-dir -r requirements.txt $ADDITIONAL_REQUIREMENTS
 

test/modules/test_humble.py

@@ -0,0 +1,36 @@
+import unittest
+
+from artemis.modules import humble
+
+
+class TestProcessJsonData(unittest.TestCase):
+    def test_process_json_data_with_valid_input(self) -> None:
+        # Setup
+        input_data = {
+            "[0. Info]": {"Date": "1970/01/01 - 12:12:12", "URL": "https://test.tld"},
+            "[1. Missing HTTP Security Headers]": ["Cache-Control", "Clear-Site-Data", "Cross-Origin-Embedder-Policy"],
+            "[2. Fingerprint HTTP Response Headers]": [
+                "Server",
+            ],
+            "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]": ["X-XSS-Protection (Unsafe Value)"],
+            "[4. Empty HTTP Response Headers Values]": ["Content-Security-Policy"],
+            "[5. Browser Compatibility for Enabled HTTP Security Headers]": {
+                "X-XSS-Protection": "https://caniuse.com/?search=X-XSS-Protection"
+            },
+        }
+
+        # Exercise
+        result = humble.process_json_data(input_data)
+
+        # Verify
+        expected_result = [
+            humble.Message(
+                category="Missing http security headers",
+                problems=["Cache-Control", "Clear-Site-Data", "Cross-Origin-Embedder-Policy"],
+            )
+        ]
+        self.assertEqual(result, expected_result)
+
+
+if __name__ == "__main__":
+    unittest.main()