Creating separate e-mail for subdomain (#405)

artemis/config.py

@@ -15,20 +15,6 @@ class Config:
     # Custom User-Agent string used by Artemis (if not set, the tool defaults will be used: requests, Nuclei etc.)
     CUSTOM_USER_AGENT = decouple.config("CUSTOM_USER_AGENT", default="")
 
-    # When creating e-mail reports, what is the vulnerability maximum age (in days) for it to be reported
-    REPORTING_MAX_VULN_AGE_DAYS = decouple.config("REPORTING_MAX_VULN_AGE_DAYS", default=14, cast=int)
-
-    # If a report has already been seen earlier - how much time needs to pass for a second e-mail to be sent
-    MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_LOW = decouple.config(
-        "MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_LOW", default=6 * 30, cast=int
-    )
-    MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_MEDIUM = decouple.config(
-        "MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_MEDIUM", default=3 * 30, cast=int
-    )
-    MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_HIGH = decouple.config(
-        "MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_HIGH", default=14, cast=int
-    )
-
     # Whether we will scan a public suffix (e.g. .pl) if it appears on the target list. This may cause very large
     # number of domains to be scanned.
     ALLOW_SCANNING_PUBLIC_SUFFIXES = decouple.config("ALLOW_SCANNING_PUBLIC_SUFFIXES", default=False, cast=bool)
@@ -62,14 +48,35 @@ class Config:
     # default request timeout (for all protocols)
     REQUEST_TIMEOUT_SECONDS = decouple.config("REQUEST_TIMEOUT_SECONDS", default=10, cast=int)
 
+    # == Reporting settings
+    # When creating e-mail reports, what is the vulnerability maximum age (in days) for it to be reported
+    REPORTING_MAX_VULN_AGE_DAYS = decouple.config("REPORTING_MAX_VULN_AGE_DAYS", default=14, cast=int)
+
+    # Sometimes even if we scan example.com, we want to report subdomain.example.com to a separate contact, because
+    # it is a separate institution. This variable should contain a comma-separated list of domains of such institutions.
+    REPORTING_SEPARATE_INSTITUTIONS = decouple.config(
+        "REPORTING_SEPARATE_INSTITUTIONS", default="", cast=decouple.Csv(str)
+    )
+
     # Ports that we will treat as "standard http/https ports" when deduplicating vulnerabilities - that is,
     # if we observe identical vulnerability of two standard ports (e.g. on 80 and on 443), we will treat
     # such case as the same vulnerability.
     #
     # This is configurable because e.g. we observed some hostings serving mirrors of content from
     # port 80 on ports 81-84.
-    REPORTER_DEDUPLICATION_COMMON_HTTP_PORTS = decouple.config(
-        "REPORTER_DEDUPLICATION_COMMON_HTTP_PORTS", default="80,443", cast=decouple.Csv(int)
+    REPORTING_DEDUPLICATION_COMMON_HTTP_PORTS = decouple.config(
+        "REPORTING_DEDUPLICATION_COMMON_HTTP_PORTS", default="80,443", cast=decouple.Csv(int)
+    )
+
+    # If a report has already been seen earlier - how much time needs to pass for a second e-mail to be sent
+    MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_LOW = decouple.config(
+        "MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_LOW", default=6 * 30, cast=int
+    )
+    MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_MEDIUM = decouple.config(
+        "MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_MEDIUM", default=3 * 30, cast=int
+    )
+    MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_HIGH = decouple.config(
+        "MIN_DAYS_BETWEEN_REMINDERS__SEVERITY_HIGH", default=14, cast=int
     )
 
     # == Rate limit settings

artemis/reporting/base/normal_form.py

@@ -33,7 +33,7 @@ def get_url_normal_form(url: str) -> str:
 
     hostname = get_domain_normal_form(url_parsed.hostname)
 
-    if url_parsed.scheme in ["http", "https"] and port in Config.REPORTER_DEDUPLICATION_COMMON_HTTP_PORTS:
+    if url_parsed.scheme in ["http", "https"] and port in Config.REPORTING_DEDUPLICATION_COMMON_HTTP_PORTS:
         url_parsed_dict["scheme"] = "http_or_https"
         url_parsed_dict["netloc"] = hostname + ":0"
     else:

artemis/reporting/utils.py

@@ -6,6 +6,8 @@
 from karton.core import Task
 
 from artemis import http_requests, task_utils
+from artemis.config import Config
+from artemis.domains import is_subdomain
 
 
 def get_target_url(task_result: Dict[str, Any]) -> str:
@@ -19,8 +21,22 @@ def get_top_level_target(task_result: Dict[str, Any]) -> str:
     For example, the top level target may be example.com, but the actual vulnerability may be
     found on https://subdomain.example.com:443/.
     """
+
     payload_persistent = task_result["payload_persistent"]
 
+    # Sometimes subdomain.example.com is managed by an institution separate from example.com. In such
+    # cases, we return a separate top level target, so that all reports in subdomain.example.com will
+    # get grouped in a separate e-mail.
+    if "last_domain" in task_result["payload"]:
+        last_domain = task_result["payload"]["last_domain"]
+
+        for item in Config.REPORTING_SEPARATE_INSTITUTIONS:
+            if is_subdomain(last_domain, item):
+                assert isinstance(item, str)
+                assert "original_domain" in payload_persistent
+                assert is_subdomain(item, payload_persistent["original_domain"])
+                return item
+
     if "original_domain" in payload_persistent:
         assert isinstance(payload_persistent["original_domain"], str)
         return payload_persistent["original_domain"]

test/reporting/test_bruter_autoreporter_integration.py

@@ -60,6 +60,9 @@ def _run_task_and_get_message(self, host: str) -> str:
             "headers": {
                 "receiver": "bruter",
             },
+            "payload": {
+                "last_domain": host,
+            },
             "payload_persistent": {
                 "original_domain": host,
             },