Locking is optional (#228)

artemis/config.py

@@ -54,10 +54,13 @@ class Config:
     # E.g. when set to 100, Artemis will strive to send no more than 100 port scanning packets per seconds to any host.
     SCANNING_PACKETS_PER_SECOND_PER_IP = decouple.config("SCANNING_PACKETS_PER_SECOND_PER_IP", default=100, cast=int)
 
+    # Whether Artemis should strive to make at most one module scan a target at a given time
+    LOCK_SCANNED_TARGETS = decouple.config("LOCK_SCANNED_TARGETS", default=False, cast=bool)
+
     # When a resource is locked using artemis.resource_lock.ResourceLock, a retry will be performed in the
     # next LOCK_SLEEP_MIN_SECONDS..LOCK_SLEEP_MAX_SECONDS seconds.
     LOCK_SLEEP_MIN_SECONDS = decouple.config("LOCK_SLEEP_MIN_SECONDS", default=0.1, cast=float)
-    LOCK_SLEEP_MAX_SECONDS = decouple.config("LOCK_SLEEP_MAX_SECONDS", default=1, cast=float)
+    LOCK_SLEEP_MAX_SECONDS = decouple.config("LOCK_SLEEP_MAX_SECONDS", default=0.5, cast=float)
 
     # Amount of times module will try to get a lock on scanned destination (with sleeps inbetween)
     # before rescheduling task for later.

artemis/module_base.py

@@ -27,7 +27,9 @@ class ArtemisBase(Karton):
     Artemis base module. Provides helpers (such as e.g. cache) for all modules.
     """
 
-    TASK_POLL_INTERVAL_SECONDS = 2
+    task_poll_interval_seconds = 2
+
+    lock_target = Config.LOCK_SCANNED_TARGETS
 
     def __init__(self, db: Optional[DB] = None, *args, **kwargs) -> None:  # type: ignore[no-untyped-def]
         super().__init__(*args, **kwargs)
@@ -89,7 +91,7 @@ def loop(self) -> None:
                     self.log.info("Binds changed, shutting down.")
                     break
 
-                time.sleep(self.TASK_POLL_INTERVAL_SECONDS)
+                time.sleep(self.task_poll_interval_seconds)
                 task = self._consume_random_routed_task(self.identity)
                 if task:
                     self.internal_process(task)
@@ -125,31 +127,30 @@ def run(self, current_task: Task) -> None:
     def internal_process(self, current_task: Task) -> None:
         scan_destination = self._get_scan_destination(current_task)
 
-        lock = ResourceLock(Config.REDIS, f"lock-{scan_destination}")
-
-        try:
-            lock.acquire(max_tries=Config.SCAN_DESTINATION_LOCK_MAX_TRIES)
-        except FailedToAcquireLockException:
-            self.log.info(
-                "Rescheduling task %s (orig_uid=%s destination=%s)",
-                current_task.uid,
-                current_task.orig_uid,
-                scan_destination,
-            )
-            self.reschedule_task(current_task)
-            return
-
-        self.log.info(
-            "Succeeded to lock task %s (orig_uid=%s destination=%s)",
-            current_task.uid,
-            current_task.orig_uid,
-            scan_destination,
-        )
-
-        try:
+        if self.lock_target:
+            try:
+                with ResourceLock(
+                    Config.REDIS, f"lock-{scan_destination}", max_tries=Config.SCAN_DESTINATION_LOCK_MAX_TRIES
+                ):
+                    self.log.info(
+                        "Succeeded to lock task %s (orig_uid=%s destination=%s)",
+                        current_task.uid,
+                        current_task.orig_uid,
+                        scan_destination,
+                    )
+
+                    super().internal_process(current_task)
+            except FailedToAcquireLockException:
+                self.log.info(
+                    "Rescheduling task %s (orig_uid=%s destination=%s)",
+                    current_task.uid,
+                    current_task.orig_uid,
+                    scan_destination,
+                )
+                self.reschedule_task(current_task)
+                return
+        else:
             super().internal_process(current_task)
-        finally:
-            lock.release()
 
     def process(self, current_task: Task) -> None:
         try:

artemis/modules/crtsh.py

@@ -23,6 +23,9 @@ class CrtshScanner(ArtemisBase):
     filters = [
         {"type": TaskType.DOMAIN.value},
     ]
+    # We don't ensure we're the only module touching the target, as actually we interact with crt.sh, not
+    # the target. We lock crt.sh instead in the run() method.
+    lock_target = False
 
     def query_sql(self, domain: str) -> set[str]:
         ct_domains: set[str] = set()

artemis/resource_lock.py

@@ -15,14 +15,13 @@ class FailedToAcquireLockException(Exception):
 
 
 class ResourceLock:
-    def __init__(self, redis: Redis, res_name: str):  # type: ignore[type-arg]
+    def __init__(self, redis: Redis, res_name: str, max_tries: Optional[int] = None):  # type: ignore[type-arg]
         self.redis = redis
         self.res_name = res_name
         self.lid = str(uuid4())
+        self.max_tries = max_tries
 
-    def acquire(
-        self, expiry: Optional[int] = Config.DEFAULT_LOCK_EXPIRY_SECONDS, max_tries: Optional[int] = None
-    ) -> None:
+    def acquire(self, expiry: Optional[int] = Config.DEFAULT_LOCK_EXPIRY_SECONDS) -> None:
         """
         Acquires a lock.
 
@@ -34,7 +33,7 @@ def acquire(
             return
 
         attempts = 0
-        while max_tries is None or attempts < max_tries:
+        while self.max_tries is None or attempts < self.max_tries:
             if self.redis.set(self.res_name, self.lid, nx=True, ex=expiry):
                 return
 

docs/features.rst

@@ -30,9 +30,10 @@ You may tune these variables by providing the values in the ``.env`` file.
 Rate limiting
 -------------
 
-Artemis is build in such a way that no particular host is overloaded with requests.
-To enable that behavior, configure the following two variables:
+Artemis is build in such a way that no particular host can be overloaded with requests.
+This is disabled by default. To enable that behavior, configure the following variables:
 
+ - set ``LOCK_SCANNED_TARGETS`` to ``True`` to enable locking,
  - ``SECONDS_PER_REQUEST_FOR_ONE_IP`` - e.g. when set to 2, Artemis will strive to make no more than
    one HTTP/MySQL connect/... request per two seconds for any IP,
  - ``SCANNING_PACKETS_PER_SECOND_PER_IP`` - e.g. when set to 100, Artemis will strive to send no more than

docs/quick-start.rst

@@ -22,7 +22,7 @@ After that you should be able to access the Artemis dashboard at ``localhost:500
 
 .. code-block:: console
 
-   docker compose up \
+   docker compose up --build \
        --scale=karton-nuclei=10 \
        --scale=karton-bruter=10 \
        --scale=karton-port_scanner=10

templates/add.jinja2

@@ -3,7 +3,7 @@
     <h1>Add targets</h1>
     <form action="/add" method="post" class="w-100">
         <div class="form-group">
-            <label class="form-text">Targets (new line separated)</label>
+            <label class="form-text">Targets (separated with newlines)</label>
             <textarea class="form-control" name="targets"></textarea>
         </div>
         <div class="form-group">