-
Notifications
You must be signed in to change notification settings - Fork 45
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
13 changed files
with
336 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,142 @@ | ||
#!/usr/bin/env python3 | ||
import dataclasses | ||
import json | ||
import os | ||
import subprocess | ||
from typing import Any, Dict, List | ||
|
||
from karton.core import Task | ||
|
||
from artemis.binds import Service, TaskStatus, TaskType | ||
from artemis.module_base import ArtemisBase | ||
from artemis.task_utils import get_target_url | ||
|
||
|
||
@dataclasses.dataclass | ||
class Message: | ||
category: str | ||
problems: List[str] | ||
|
||
@property | ||
def message(self) -> str: | ||
return f"{self.category}: {', '.join(self.problems)}" | ||
|
||
|
||
def process_json_data(result: Dict[str, Any]) -> List[Message]: | ||
messages: Dict[str, Message] = {} | ||
|
||
# Iterate through key-value pairs in the result | ||
for key, value in result.items(): | ||
# Split the key to extract the relevant part | ||
key_parts = key.replace("[", "").replace("]", "").split(". ") | ||
|
||
if key in [ | ||
"[2. Fingerprint HTTP Response Headers]", | ||
"[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]", | ||
"[4. Empty HTTP Response Headers Values]", | ||
"[5. Browser Compatibility for Enabled HTTP Security Headers]", | ||
]: | ||
continue | ||
|
||
# Check if the key has the expected structure | ||
if len(key_parts) >= 2: | ||
category = key_parts[1].capitalize() | ||
|
||
# Check if the key is not in the excluded categories and there are relevant values | ||
if category.lower() != "info": | ||
# If the value is a dictionary, iterate through subkey-value pairs | ||
if isinstance(value, dict): | ||
for subkey, subvalue in value.items(): | ||
# Add subkeys and subvalues to messages | ||
if subvalue and subvalue not in [ | ||
"Nothing to report, all seems OK!", | ||
"No HTTP security headers are enabled.", | ||
]: | ||
problem = f"{subkey} {subvalue}" | ||
if category not in messages: | ||
messages[category] = Message(category=category, problems=[]) | ||
messages[category].problems.append(problem) | ||
|
||
# If the value is a list, iterate through list items | ||
elif isinstance(value, list): | ||
for item in value: | ||
# Add list items to messages | ||
if item and item not in [ | ||
"Nothing to report, all seems OK!", | ||
"No HTTP security headers are enabled.", | ||
]: | ||
if category not in messages: | ||
messages[category] = Message(category=category, problems=[]) | ||
messages[category].problems.append(item) | ||
|
||
return list(messages.values()) | ||
|
||
|
||
class Humble(ArtemisBase): | ||
""" | ||
Runs humble -> A HTTP Headers Analyzer | ||
""" | ||
|
||
identity = "humble" | ||
filters = [ | ||
{"type": TaskType.SERVICE.value, "service": Service.HTTP.value}, | ||
] | ||
|
||
def run(self, current_task: Task) -> None: | ||
base_url = get_target_url(current_task) | ||
|
||
data = subprocess.check_output( | ||
[ | ||
"python3", | ||
"humble.py", | ||
"-u", | ||
base_url, | ||
"-b", | ||
"-o", | ||
"json", | ||
], | ||
cwd="/humble", | ||
stderr=subprocess.DEVNULL, | ||
) | ||
|
||
# strip boilerplatetext from the output to get the location and filename of the output file | ||
filename = ( | ||
data.decode("ascii", errors="ignore") | ||
.removeprefix("\n Analyzing URL and saving the report, please wait ...\n\n\n Report saved to ") | ||
.removesuffix("\n") | ||
) | ||
data_str = open(filename, "r").read() | ||
|
||
# cleanup file | ||
os.unlink(filename) | ||
|
||
# Check if the input string is empty | ||
if data_str.strip(): | ||
result = json.loads(data_str) | ||
else: | ||
result = [] | ||
|
||
# Parse the JSON data | ||
messages = process_json_data(result) | ||
|
||
if messages: | ||
status = TaskStatus.INTERESTING | ||
status_reason = ", ".join([message.message for message in messages]) | ||
else: | ||
status = TaskStatus.OK | ||
status_reason = None | ||
|
||
self.db.save_task_result( | ||
task=current_task, | ||
status=status, | ||
status_reason=status_reason, | ||
data={ | ||
"original_result": result, | ||
"message_data": [dataclasses.asdict(message) for message in messages], | ||
"messages": [message.message for message in messages], | ||
}, | ||
) | ||
|
||
|
||
if __name__ == "__main__": | ||
Humble().loop() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
from pathlib import Path | ||
from typing import Any, Callable, Dict, List | ||
|
||
from artemis.config import Config | ||
from artemis.reporting.base.language import Language | ||
from artemis.reporting.base.normal_form import NormalForm, get_url_normal_form | ||
from artemis.reporting.base.report import Report | ||
from artemis.reporting.base.report_type import ReportType | ||
from artemis.reporting.base.reporter import Reporter | ||
from artemis.reporting.base.templating import ReportEmailTemplateFragment | ||
from artemis.reporting.utils import get_target_url, get_top_level_target | ||
|
||
|
||
class HumbleReporter(Reporter): | ||
MISSING_SECURITY_HEADERS = ReportType("missing_security_headers") | ||
|
||
@staticmethod | ||
def create_reports(task_result: Dict[str, Any], language: Language) -> List[Report]: | ||
if task_result["headers"]["receiver"] != "humble": | ||
return [] | ||
|
||
if not isinstance(task_result["result"], dict): | ||
return [] | ||
|
||
return [ | ||
Report( | ||
top_level_target=get_top_level_target(task_result), | ||
target=get_target_url(task_result), | ||
report_type=HumbleReporter.MISSING_SECURITY_HEADERS, | ||
additional_data={ | ||
"message_data": HumbleReporter._filter_message_data(task_result["result"]["message_data"]), | ||
}, | ||
timestamp=task_result["created_at"], | ||
) | ||
] | ||
|
||
@staticmethod | ||
def get_email_template_fragments() -> List[ReportEmailTemplateFragment]: | ||
return [ | ||
ReportEmailTemplateFragment.from_file( | ||
str(Path(__file__).parents[0] / "template_missing_security_headers.jinja2"), priority=2 | ||
), | ||
] | ||
|
||
@staticmethod | ||
def get_normal_form_rules() -> Dict[ReportType, Callable[[Report], NormalForm]]: | ||
"""See the docstring in the Reporter class.""" | ||
return { | ||
HumbleReporter.MISSING_SECURITY_HEADERS: lambda report: Reporter.dict_to_tuple( | ||
{ | ||
"type": report.report_type, | ||
"target": get_url_normal_form(report.target), | ||
"message_data": [Reporter.dict_to_tuple(item) for item in report.additional_data["message_data"]], | ||
} | ||
) | ||
} | ||
|
||
@staticmethod | ||
def _filter_message_data(message_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]: | ||
result: List[Dict[str, Any]] = [] | ||
for item in message_data: | ||
if item["category"] == "Missing http security headers": | ||
problems = sorted(set(item["problems"]) & set(Config.Modules.Humble.HUMBLE_HEADERS_TO_REPORT)) | ||
if problems: | ||
result.append({"category": item["category"], "problems": problems}) | ||
else: | ||
result.append(item) | ||
return result |
22 changes: 22 additions & 0 deletions
22
artemis/reporting/modules/humble/template_missing_security_headers.jinja2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
{% if "missing_security_headers" in data.contains_type %} | ||
<li>{% trans %}We identified that the following security headers are not set correctly:{% endtrans %} | ||
<ul> | ||
{% for report in data.reports %} | ||
{% if report.report_type == "missing_security_headers" %} | ||
{% for message in report.additional_data.message_data %} | ||
<li> | ||
{{ report.target }}: {{ _(message.category) }}: {{ ', '.join(message.problems) }} | ||
</li> | ||
{% endfor %} | ||
{{ report_meta(report) }} | ||
{% endif %} | ||
{% endfor %} | ||
</ul> | ||
<p> | ||
{% trans trimmed %} | ||
Please verify the configuration, and, if a security header is missing, add it. Security | ||
headers can have a deep impact on protecting your application against attacks. | ||
{% endtrans %} | ||
</p> | ||
</li> | ||
{% endif %} |
10 changes: 10 additions & 0 deletions
10
artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:2 | ||
msgid "We identified that the following security headers are not set correctly:" | ||
msgstr "" | ||
|
||
#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:16 | ||
msgid "" | ||
"Please verify the configuration, and, if a security header is missing, " | ||
"add it. Security headers can have a deep impact on protecting your " | ||
"application against attacks." | ||
msgstr "" |
7 changes: 7 additions & 0 deletions
7
artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
# This is a separate file so that pybabel doesn't mark it as obsolete because it doesn't | ||
# see the original strings in our code (they come from the humble library). | ||
|
||
msgid "" | ||
"Missing http security headers" | ||
msgstr "" | ||
"Brakujące nagłówki HTTP zwiększające bezpieczeństwo" |
15 changes: 15 additions & 0 deletions
15
artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:2 | ||
msgid "We identified that the following security headers are not set correctly:" | ||
msgstr "" | ||
"Wykryto, że następujące nagłówki HTTP zwiększające bezpieczeństwo nie są " | ||
"ustawione poprawnie:" | ||
|
||
#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:16 | ||
msgid "" | ||
"Please verify the configuration, and, if a security header is missing, " | ||
"add it. Security headers can have a deep impact on protecting your " | ||
"application against attacks." | ||
msgstr "" | ||
"Rekomendujemy weryfikację konfiguracji i zmianę lub ustawienie nagłówka, " | ||
"jeśli nie jest skonfigurowany poprawnie. Poprawna konfiguracja powyższych" | ||
" nagłówków zwiększy bezpieczeństwo aplikacji." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
import unittest | ||
|
||
from artemis.modules import humble | ||
|
||
|
||
class TestProcessJsonData(unittest.TestCase): | ||
def test_process_json_data_with_valid_input(self) -> None: | ||
# Setup | ||
input_data = { | ||
"[0. Info]": {"Date": "1970/01/01 - 12:12:12", "URL": "https://test.tld"}, | ||
"[1. Missing HTTP Security Headers]": ["Cache-Control", "Clear-Site-Data", "Cross-Origin-Embedder-Policy"], | ||
"[2. Fingerprint HTTP Response Headers]": [ | ||
"Server", | ||
], | ||
"[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]": ["X-XSS-Protection (Unsafe Value)"], | ||
"[4. Empty HTTP Response Headers Values]": ["Content-Security-Policy"], | ||
"[5. Browser Compatibility for Enabled HTTP Security Headers]": { | ||
"X-XSS-Protection": "https://caniuse.com/?search=X-XSS-Protection" | ||
}, | ||
} | ||
|
||
# Exercise | ||
result = humble.process_json_data(input_data) | ||
|
||
# Verify | ||
expected_result = [ | ||
humble.Message( | ||
category="Missing http security headers", | ||
problems=["Cache-Control", "Clear-Site-Data", "Cross-Origin-Embedder-Policy"], | ||
) | ||
] | ||
self.assertEqual(result, expected_result) | ||
|
||
|
||
if __name__ == "__main__": | ||
unittest.main() |