Added Humble

artemis/config.py

@@ -229,6 +229,17 @@ class Gau:
                 "Additional command-line options that will be passed to gau (https://github.com/lc/gau).",
             ] = get_config("GAU_ADDITIONAL_OPTIONS", default="", cast=decouple.Csv(str, delimiter=" "))
 
+        class Humble:
+            HUMBLE_HEADERS_TO_REPORT: Annotated[
+                List[str],
+                "The list of headers that are considered more important and will be mentioned in the generated text "
+                "reports (all of the missing headers will be visible in the UI).",
+            ] = get_config(
+                "HUMBLE_HEADERS_TO_REPORT",
+                default=",".join(["Content-Security-Policy", "Strict-Transport-Security", "X-Content-Type-Options"]),
+                cast=decouple.Csv(str, delimiter=","),
+            )
+
         class Nuclei:
             NUCLEI_CHECK_TEMPLATE_LIST: Annotated[
                 bool,

artemis/modules/humble.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+import dataclasses
+import json
+import subprocess
+from typing import Any, Dict, List
+
+from karton.core import Task
+
+from artemis.binds import Service, TaskStatus, TaskType
+from artemis.module_base import ArtemisBase
+from artemis.task_utils import get_target_url
+
+
+@dataclasses.dataclass
+class Message:
+    category: str
+    problems: List[str]
+
+    @property
+    def message(self) -> str:
+        return f"{self.category}: {', '.join(self.problems)}"
+
+
+def process_json_data(result: Dict[str, Any]) -> List[Message]:
+    messages: Dict[str, Message] = {}
+
+    # Iterate through key-value pairs in the result
+    for key, value in result.items():
+        # Split the key to extract the relevant part
+        key_parts = key.replace("[", "").replace("]", "").split(". ")
+
+        # Check if the key has the expected structure
+        if len(key_parts) >= 2:
+            category = key_parts[1].capitalize()
+
+            # Check if the key is not in the excluded categories and there are relevant values
+            if (
+                category.lower() != "info"
+                and key
+                not in [
+                    "[2. Fingerprint HTTP Response Headers]",
+                    "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]",
+                    "[4. Empty HTTP Response Headers Values]",
+                    "[5. Browser Compatibility for Enabled HTTP Security Headers]",
+                ]
+                and (
+                    isinstance(value, dict)
+                    or (
+                        isinstance(value, list)
+                        and any(
+                            subvalue
+                            for subvalue in value
+                            if subvalue
+                            and subvalue
+                            not in ["Nothing to report, all seems OK!", "No HTTP security headers are enabled."]
+                        )
+                    )
+                )
+            ):
+                # If the value is a dictionary, iterate through subkey-value pairs
+                if isinstance(value, dict):
+                    for subkey, subvalue in value.items():
+                        # Add subkeys and subvalues to messages
+                        if subvalue and subvalue not in [
+                            "Nothing to report, all seems OK!",
+                            "No HTTP security headers are enabled.",
+                        ]:
+                            problem = f"{subkey} {subvalue}"
+                            if category not in messages:
+                                messages[category] = Message(category=category, problems=[])
+                            messages[category].problems.append(problem)
+
+                # If the value is a list, iterate through list items
+                elif isinstance(value, list):
+                    for item in value:
+                        # Add list items to messages
+                        if item and item not in [
+                            "Nothing to report, all seems OK!",
+                            "No HTTP security headers are enabled.",
+                        ]:
+                            if category not in messages:
+                                messages[category] = Message(category=category, problems=[])
+                            messages[category].problems.append(item)
+
+    return list(messages.values())
+
+
+class Humble(ArtemisBase):
+    """
+    Runs humble -> A HTTP Headers Analyzer
+    """
+
+    identity = "humble"
+    filters = [
+        {"type": TaskType.SERVICE.value, "service": Service.HTTP.value},
+    ]
+
+    def run(self, current_task: Task) -> None:
+        base_url = get_target_url(current_task)
+
+        data = subprocess.check_output(
+            [
+                "python3",
+                "humble.py",
+                "-u",
+                base_url,
+                "-b",
+                "-o",
+                "json",
+            ],
+            cwd="/humble",
+            stderr=subprocess.DEVNULL,
+        )
+
+        # strip boilerplatetext from the output to get the location and filename of the output file
+        filename = (
+            data.decode("ascii", errors="ignore")
+            .removeprefix("\n Analyzing URL and saving the report, please wait ...\n\n\n Report saved to ")
+            .removesuffix("\n")
+        )
+        data_str = open(filename, "r").read()
+
+        # cleanup file
+        subprocess.run(["rm", filename])
+
+        # Check if the input string is empty
+        if data_str.strip():
+            result = json.loads(data_str)
+        else:
+            result = []
+
+        # Parse the JSON data
+        messages = process_json_data(result)
+
+        if messages:
+            status = TaskStatus.INTERESTING
+            status_reason = ", ".join([message.message for message in messages])
+        else:
+            status = TaskStatus.OK
+            status_reason = None
+
+        self.db.save_task_result(
+            task=current_task,
+            status=status,
+            status_reason=status_reason,
+            data={
+                "original_result": result,
+                "message_data": [dataclasses.asdict(message) for message in messages],
+                "messages": [message.message for message in messages],
+            },
+        )
+
+
+if __name__ == "__main__":
+    Humble().loop()

artemis/reporting/base/reporter.py

@@ -65,8 +65,16 @@ def get_scoring_rules(cls) -> Dict[ReportType, Callable[[Report], List[int]]]:
         return {report_type: Reporter.default_scoring_rule for report_type in cls.get_report_types()}
 
     @staticmethod
-    def dict_to_tuple(d: Dict[str, str]) -> Tuple[Tuple[str, str], ...]:
-        return tuple(d.items())
+    def dict_to_tuple(d: Dict[str, Any]) -> Tuple[Any, ...]:
+        result = []
+        for key, value in d.items():
+            if isinstance(value, dict):
+                result.append((key, Reporter.dict_to_tuple(value)))
+            elif isinstance(value, list):
+                result.append((key, tuple(value)))
+            else:
+                result.append((key, value))
+        return tuple(result)
 
     @staticmethod
     def default_scoring_rule(report: Report) -> List[int]:

artemis/reporting/modules/humble/reporter.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+from typing import Any, Callable, Dict, List
+
+from artemis.config import Config
+from artemis.reporting.base.language import Language
+from artemis.reporting.base.normal_form import NormalForm, get_url_normal_form
+from artemis.reporting.base.report import Report
+from artemis.reporting.base.report_type import ReportType
+from artemis.reporting.base.reporter import Reporter
+from artemis.reporting.base.templating import ReportEmailTemplateFragment
+from artemis.reporting.utils import get_target_url, get_top_level_target
+
+
+class HumbleReporter(Reporter):
+    MISSING_SECURITY_HEADERS = ReportType("missing_security_headers")
+
+    @staticmethod
+    def create_reports(task_result: Dict[str, Any], language: Language) -> List[Report]:
+        if task_result["headers"]["receiver"] != "humble":
+            return []
+
+        if not isinstance(task_result["result"], dict):
+            return []
+
+        return [
+            Report(
+                top_level_target=get_top_level_target(task_result),
+                target=get_target_url(task_result),
+                report_type=HumbleReporter.MISSING_SECURITY_HEADERS,
+                additional_data={
+                    "message_data": HumbleReporter._filter_message_data(task_result["result"]["message_data"]),
+                },
+                timestamp=task_result["created_at"],
+            )
+        ]
+
+    @staticmethod
+    def get_email_template_fragments() -> List[ReportEmailTemplateFragment]:
+        return [
+            ReportEmailTemplateFragment.from_file(
+                str(Path(__file__).parents[0] / "template_missing_security_headers.jinja2"), priority=2
+            ),
+        ]
+
+    @staticmethod
+    def get_normal_form_rules() -> Dict[ReportType, Callable[[Report], NormalForm]]:
+        """See the docstring in the Reporter class."""
+        return {
+            HumbleReporter.MISSING_SECURITY_HEADERS: lambda report: Reporter.dict_to_tuple(
+                {
+                    "type": report.report_type,
+                    "target": get_url_normal_form(report.target),
+                    "message_data": [Reporter.dict_to_tuple(item) for item in report.additional_data["message_data"]],
+                }
+            )
+        }
+
+    @staticmethod
+    def _filter_message_data(message_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        result: List[Dict[str, Any]] = []
+        for item in message_data:
+            if item["category"] == "Missing http security headers":
+                problems = sorted(set(item["problems"]) & set(Config.Modules.Humble.HUMBLE_HEADERS_TO_REPORT))
+                if problems:
+                    result.append({"category": item["category"], "problems": problems})
+            else:
+                result.append(item)
+        return result

artemis/reporting/modules/humble/template_missing_security_headers.jinja2

@@ -0,0 +1,22 @@
+{% if "missing_security_headers" in data.contains_type %}
+    <li>{% trans %}We identified that the following security headers are not set correctly:{% endtrans %}
+        <ul>
+            {% for report in data.reports %}
+                {% if report.report_type == "missing_security_headers" %}
+                    {% for message in report.additional_data.message_data %}
+                        <li>
+                            {{ report.target }}: {{ _(message.category) }}: {{ ', '.join(message.problems) }}
+                        </li>
+                    {% endfor %}
+                    {{ report_meta(report) }}
+                {% endif %}
+            {% endfor %}
+        </ul>
+        <p>
+            {% trans trimmed %}
+                Please verify the configuration, and, if a security header is missing, change it. Security
+                headers can have a deep impact on protecting your application against attacks.
+            {% endtrans %}
+        </p>
+    </li>
+{% endif %}

artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po

@@ -0,0 +1,10 @@
+#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:2
+msgid "We identified that the following security headers are not set correctly:"
+msgstr ""
+
+#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:16
+msgid ""
+"Please verify the configuration, and, if a security header is missing, "
+"change it. Security headers can have a deep impact on protecting your "
+"application against attacks."
+msgstr ""

artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po

@@ -0,0 +1,7 @@
+# This is a separate file so that pybabel doesn't mark it as obsolete because it doesn't
+# see the original strings in our code (they come from the humble library).
+
+msgid ""
+"Missing http security headers"
+msgstr ""
+"Brakujące nagłówki HTTP zwiększające bezpieczeństwo"

artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po

@@ -0,0 +1,15 @@
+#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:2
+msgid "We identified that the following security headers are not set correctly:"
+msgstr ""
+"Wykryto, że następujące nagłówki HTTP zwiększające bezpieczeństwo nie są "
+"ustawione poprawnie:"
+
+#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:16
+msgid ""
+"Please verify the configuration, and, if a security header is missing, "
+"change it. Security headers can have a deep impact on protecting your "
+"application against attacks."
+msgstr ""
+"Rekomendujemy weryfikację konfiguracji i zmianę lub ustawienie nagłówka, "
+"jeśli nie jest skonfigurowany poprawnie. Poprawna konfiguracja powyższych"
+" nagłówków zwiększy bezpieczeństwo aplikacji."

artemis/reporting/severity.py

@@ -52,4 +52,5 @@ class Severity(str, Enum):
     ReportType("exposed_php_var_dump"): Severity.LOW,
     ReportType("exposed_phpinfo"): Severity.LOW,
     ReportType("nuclei_exposed_panel"): Severity.LOW,
+    ReportType("missing_security_headers"): Severity.LOW,
 }

docker-compose.yaml

@@ -172,6 +172,16 @@ services:
     restart: always
     volumes: ["./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/"]
 
+  karton-humble:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+    command: "python3 -m artemis.modules.humble"
+    depends_on: [karton-logger]
+    env_file: .env
+    restart: always
+    volumes: [ "./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/" ]
+
   karton-identifier:
     build:
       context: .

docker/Dockerfile

@@ -15,6 +15,9 @@ RUN git clone https://github.com/projectdiscovery/nuclei.git -b v2.9.7 /nuclei
 COPY docker/patches/nuclei-rate-limiting.patch /nuclei/
 RUN cd /nuclei && git apply nuclei-rate-limiting.patch  && cd v2/cmd/nuclei && go build && GOBIN=/usr/local/bin/ go install
 
+RUN git clone https://github.com/rfc-st/humble.git --branch master /humble
+RUN pip install -r /humble/requirements.txt
+
 COPY requirements.txt requirements.txt
 RUN pip install --no-cache-dir -r requirements.txt $ADDITIONAL_REQUIREMENTS
 

test/modules/test_humble.py

@@ -0,0 +1,36 @@
+import unittest
+
+from artemis.modules import humble
+
+
+class TestProcessJsonData(unittest.TestCase):
+    def test_process_json_data_with_valid_input(self) -> None:
+        # Setup
+        input_data = {
+            "[0. Info]": {"Date": "1970/01/01 - 12:12:12", "URL": "https://test.tld"},
+            "[1. Missing HTTP Security Headers]": ["Cache-Control", "Clear-Site-Data", "Cross-Origin-Embedder-Policy"],
+            "[2. Fingerprint HTTP Response Headers]": [
+                "Server",
+            ],
+            "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]": ["X-XSS-Protection (Unsafe Value)"],
+            "[4. Empty HTTP Response Headers Values]": ["Content-Security-Policy"],
+            "[5. Browser Compatibility for Enabled HTTP Security Headers]": {
+                "X-XSS-Protection": "https://caniuse.com/?search=X-XSS-Protection"
+            },
+        }
+
+        # Exercise
+        result = humble.process_json_data(input_data)
+
+        # Verify
+        expected_result = [
+            humble.Message(
+                category="Missing http security headers",
+                problems=["Cache-Control", "Clear-Site-Data", "Cross-Origin-Embedder-Policy"],
+            )
+        ]
+        self.assertEqual(result, expected_result)
+
+
+if __name__ == "__main__":
+    unittest.main()