From 62fc7f68fd93fd11d8a95e386b94ec075d689870 Mon Sep 17 00:00:00 2001 From: Matie26 Date: Mon, 8 Apr 2024 09:16:39 +0200 Subject: [PATCH 1/7] return http headers with HTTPResponse --- artemis/http_requests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/artemis/http_requests.py b/artemis/http_requests.py index e66a67a16..58417553a 100644 --- a/artemis/http_requests.py +++ b/artemis/http_requests.py @@ -26,6 +26,7 @@ class HTTPResponse: encoding: str is_redirect: bool url: str + headers: dict def json(self) -> Any: return json.loads(self.content) @@ -79,6 +80,7 @@ def _internal_request() -> HTTPResponse: encoding=response.encoding if response.encoding else "utf-8", is_redirect=bool(response.history), url=response.url, + headers=response.headers ) return throttle_request(_internal_request) # type: ignore From 9021217a3eadd7ca2449be51b66d70dc674400db Mon Sep 17 00:00:00 2001 From: Matie26 Date: Mon, 8 Apr 2024 09:18:09 +0200 Subject: [PATCH 2/7] add device enum to binds --- artemis/binds.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/artemis/binds.py b/artemis/binds.py index a637be9d1..6529845f4 100644 --- a/artemis/binds.py +++ b/artemis/binds.py @@ -56,6 +56,12 @@ class WebApplication(str, Enum): @classmethod def _missing_(cls, value: object) -> WebApplication: return WebApplication.UNKNOWN + + +class Device(str, Enum): + UNKNOWN = "unknown" + + FORTIOS = "fortios" class TaskStatus(str, Enum): From cfe1be0979961cb3ea460462118f13df152497b6 Mon Sep 17 00:00:00 2001 From: Matie26 Date: Mon, 8 Apr 2024 09:55:45 +0200 Subject: [PATCH 3/7] device identifier module --- artemis/modules/device_identifier.py | 58 ++++++++++++++++++++++++++++ docker-compose.yaml | 11 ++++++ 2 files changed, 69 insertions(+) create mode 100644 artemis/modules/device_identifier.py diff --git a/artemis/modules/device_identifier.py b/artemis/modules/device_identifier.py new file mode 100644 index 000000000..c1d56a5af --- /dev/null +++ b/artemis/modules/device_identifier.py @@ -0,0 +1,58 @@ +from karton.core import Task + +from artemis import http_requests +from artemis.binds import Service, TaskStatus, TaskType, Device +from artemis.module_base import ArtemisBase +from artemis.task_utils import get_target_host, get_target_url + + +class DeviceIdentifier(ArtemisBase): + """ + Tries to identify the device (FortiOS, ...) and produces a SERVICE task with proper type (e.g. Device.FORTIOS) + """ + + identity = "device_identifier" + + filters = [ + {"type": TaskType.SERVICE.value, "service": Service.HTTP.value}, + ] + + @staticmethod + def _identify(url: str) -> Device: + response = http_requests.get(url, allow_redirects=True) + if ("xxxxxxxx-xxxxx" in response.headers.get("Server", "") and + "/remote/login" in response.url): + return Device.FORTIOS + + return Device.UNKNOWN + + def _process(self, current_task: Task, url: str, host: str, port: int) -> None: + device = self._identify(url) + + new_task = Task( + { + "type": TaskType.SERVICE, + "device": device, + }, + payload={ + "host": host, + "port": int(port), + "ssl": current_task.get_payload("ssl") + }, + ) + self.add_task(current_task, new_task) + + self.db.save_task_result(task=current_task, status=TaskStatus.OK, data=device) + + def run(self, current_task: Task) -> None: + url = get_target_url(current_task) + host = get_target_host(current_task) + port = current_task.get_payload("port") + + self.log.info(f"device identifier scanning {url}") + + self._process(current_task, url, host, port) + + +if __name__ == "__main__": + DeviceIdentifier().loop() diff --git a/docker-compose.yaml b/docker-compose.yaml index 6bedd3cac..8adb5adfc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -416,6 +416,17 @@ services: restart: always volumes: ["./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/"] + karton-device-identifier: + build: + context: . + dockerfile: docker/Dockerfile + command: "python3 -m artemis.modules.device_identifier" + depends_on: [karton-system] + env_file: .env + restart: always + volumes: ["./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/"] + + volumes: data-nuclei-templates: data-nuclei-config: From 2438f827259df452474ccb5ea221ced324a5bb68 Mon Sep 17 00:00:00 2001 From: Matie26 Date: Mon, 8 Apr 2024 11:06:03 +0200 Subject: [PATCH 4/7] lint --- artemis/binds.py | 2 +- artemis/http_requests.py | 4 ++-- artemis/modules/device_identifier.py | 19 +++++++------------ 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/artemis/binds.py b/artemis/binds.py index 6529845f4..f97625d8b 100644 --- a/artemis/binds.py +++ b/artemis/binds.py @@ -56,7 +56,7 @@ class WebApplication(str, Enum): @classmethod def _missing_(cls, value: object) -> WebApplication: return WebApplication.UNKNOWN - + class Device(str, Enum): UNKNOWN = "unknown" diff --git a/artemis/http_requests.py b/artemis/http_requests.py index 58417553a..b8300558b 100644 --- a/artemis/http_requests.py +++ b/artemis/http_requests.py @@ -26,7 +26,7 @@ class HTTPResponse: encoding: str is_redirect: bool url: str - headers: dict + headers: Dict[str, str] def json(self) -> Any: return json.loads(self.content) @@ -80,7 +80,7 @@ def _internal_request() -> HTTPResponse: encoding=response.encoding if response.encoding else "utf-8", is_redirect=bool(response.history), url=response.url, - headers=response.headers + headers=response.headers, ) return throttle_request(_internal_request) # type: ignore diff --git a/artemis/modules/device_identifier.py b/artemis/modules/device_identifier.py index c1d56a5af..4df0d3d62 100644 --- a/artemis/modules/device_identifier.py +++ b/artemis/modules/device_identifier.py @@ -1,7 +1,7 @@ from karton.core import Task from artemis import http_requests -from artemis.binds import Service, TaskStatus, TaskType, Device +from artemis.binds import Device, Service, TaskStatus, TaskType from artemis.module_base import ArtemisBase from artemis.task_utils import get_target_host, get_target_url @@ -20,25 +20,20 @@ class DeviceIdentifier(ArtemisBase): @staticmethod def _identify(url: str) -> Device: response = http_requests.get(url, allow_redirects=True) - if ("xxxxxxxx-xxxxx" in response.headers.get("Server", "") and - "/remote/login" in response.url): - return Device.FORTIOS - + if "xxxxxxxx-xxxxx" in response.headers.get("Server", "") and "/remote/login" in response.url: + return Device.FORTIOS + return Device.UNKNOWN def _process(self, current_task: Task, url: str, host: str, port: int) -> None: device = self._identify(url) - + new_task = Task( { "type": TaskType.SERVICE, "device": device, }, - payload={ - "host": host, - "port": int(port), - "ssl": current_task.get_payload("ssl") - }, + payload={"host": host, "port": int(port), "ssl": current_task.get_payload("ssl")}, ) self.add_task(current_task, new_task) @@ -50,7 +45,7 @@ def run(self, current_task: Task) -> None: port = current_task.get_payload("port") self.log.info(f"device identifier scanning {url}") - + self._process(current_task, url, host, port) From 451b4fdbfee4cc45bd7fd69071d02d3c0655c946 Mon Sep 17 00:00:00 2001 From: Matie26 Date: Mon, 8 Apr 2024 12:07:46 +0200 Subject: [PATCH 5/7] added TaskType device --- artemis/binds.py | 3 +++ artemis/modules/device_identifier.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/artemis/binds.py b/artemis/binds.py index f97625d8b..277f4a852 100644 --- a/artemis/binds.py +++ b/artemis/binds.py @@ -22,6 +22,9 @@ class TaskType(str, Enum): # {URL: just a HTTP URL. Must have content attached to make further operations faster} URL = "url" + # {device: having an ip, port, ssl, and a device type, e.g. Device.FORTIOS} + DEVICE = "device" + class Service(str, Enum): # Each of the services can have the SSL flag enabled - therefore HTTP covers both HTTP and HTTPS. diff --git a/artemis/modules/device_identifier.py b/artemis/modules/device_identifier.py index 4df0d3d62..3b4fbb863 100644 --- a/artemis/modules/device_identifier.py +++ b/artemis/modules/device_identifier.py @@ -30,7 +30,7 @@ def _process(self, current_task: Task, url: str, host: str, port: int) -> None: new_task = Task( { - "type": TaskType.SERVICE, + "type": TaskType.DEVICE, "device": device, }, payload={"host": host, "port": int(port), "ssl": current_task.get_payload("ssl")}, From d3d045d734f3f91b04ab2f742c83ed178104882a Mon Sep 17 00:00:00 2001 From: Matie26 Date: Mon, 8 Apr 2024 12:10:02 +0200 Subject: [PATCH 6/7] naming convention --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 8adb5adfc..2a5b3a3a9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -416,7 +416,7 @@ services: restart: always volumes: ["./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/"] - karton-device-identifier: + karton-device_identifier: build: context: . dockerfile: docker/Dockerfile From 60389928a006878f8ed42c9cfe656b6103069a4d Mon Sep 17 00:00:00 2001 From: Matie26 Date: Mon, 8 Apr 2024 13:16:04 +0200 Subject: [PATCH 7/7] minor changes in comments --- artemis/binds.py | 2 +- artemis/modules/device_identifier.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/artemis/binds.py b/artemis/binds.py index 277f4a852..d16c2075e 100644 --- a/artemis/binds.py +++ b/artemis/binds.py @@ -22,7 +22,7 @@ class TaskType(str, Enum): # {URL: just a HTTP URL. Must have content attached to make further operations faster} URL = "url" - # {device: having an ip, port, ssl, and a device type, e.g. Device.FORTIOS} + # {device: having a host, port, ssl, and a device type, e.g. Device.FORTIOS} DEVICE = "device" diff --git a/artemis/modules/device_identifier.py b/artemis/modules/device_identifier.py index 3b4fbb863..b26e933e9 100644 --- a/artemis/modules/device_identifier.py +++ b/artemis/modules/device_identifier.py @@ -8,7 +8,7 @@ class DeviceIdentifier(ArtemisBase): """ - Tries to identify the device (FortiOS, ...) and produces a SERVICE task with proper type (e.g. Device.FORTIOS) + Tries to identify the device (FortiOS, ...) and produces a DEVICE task with proper type (e.g. Device.FORTIOS) """ identity = "device_identifier"