From f1a61a98910e0fe6eca9600345f7b18175a21d34 Mon Sep 17 00:00:00 2001 From: Felix Ortmann Date: Mon, 22 Feb 2021 17:26:20 +0100 Subject: [PATCH 1/6] Introduce optional dependencies for MISP plugin --- plugins/apps/threatbus_misp/setup.py | 5 ++-- .../threatbus_misp/threatbus_misp/plugin.py | 27 +++++++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/plugins/apps/threatbus_misp/setup.py b/plugins/apps/threatbus_misp/setup.py index 5d4893d2..8de01d4b 100644 --- a/plugins/apps/threatbus_misp/setup.py +++ b/plugins/apps/threatbus_misp/setup.py @@ -27,10 +27,9 @@ entry_points={"threatbus.app": ["misp = threatbus_misp.plugin"]}, install_requires=[ "threatbus>=2020.12.16", - "pymisp >= 2.4.120, <= 2.4.134", - "pyzmq>=18.1.1", - "confluent-kafka>=1.3.0", + "pymisp >= 2.4.120", ], + extras_require={"kafka": ["confluent-kafka>=1.3.0"], "zmq": ["pyzmq>=18.1.1"]}, keywords=[ "MISP", "zeromq", diff --git a/plugins/apps/threatbus_misp/threatbus_misp/plugin.py b/plugins/apps/threatbus_misp/threatbus_misp/plugin.py index d1e138a2..911c7eb5 100644 --- a/plugins/apps/threatbus_misp/threatbus_misp/plugin.py +++ b/plugins/apps/threatbus_misp/threatbus_misp/plugin.py @@ -1,4 +1,15 @@ -from confluent_kafka import Consumer +## The MISP plugin lists two optional dependencies via `extras_requires`, so +## users can either install `threatbus-misp[zmq]` or `threatbus-misp[kafka]`. +## The plugin needs at least one of these to work properly, so we we need to log +## an error if none of these is installed. +dep_kafka = False +dep_zmq = False +try: + from confluent_kafka import Consumer + + dep_kafka = True +except ModuleNotFoundError: + pass from confuse import Subview from datetime import datetime from itertools import product @@ -12,7 +23,13 @@ from threatbus_misp.message_mapping import map_to_internal, map_to_misp, is_whitelisted from typing import Callable, List, Dict import warnings -import zmq + +try: + import zmq + + dep_zmq = True +except ModuleNotFoundError: + pass warnings.simplefilter("ignore") # pymisp produces urllib warnings @@ -162,9 +179,15 @@ def validate_config(config: Subview): config["api"]["ssl"].get(bool) config["api"]["key"].get(str) if config["zmq"].get(dict): + assert ( + dep_zmq + ), "MISP attribute export is configured via ZeroMQ, but the dependency is not installed. Install `threatbus-misp[zmq]` to use this setting." config["zmq"]["host"].get(str) config["zmq"]["port"].get(int) if config["kafka"].get(dict): + assert ( + dep_kafka + ), "MISP attribute export is configured via Apache Kafka, but the dependency is not installed. Install `threatbus-misp[kafka]` to use this setting." config["kafka"]["topics"].get(list) config["kafka"]["poll_interval"].add(1.0) config["kafka"]["poll_interval"].get(float) From 52146539ef8cd1a40fe97f848e0646d82c6acdd1 Mon Sep 17 00:00:00 2001 From: Felix Ortmann Date: Mon, 22 Feb 2021 17:28:48 +0100 Subject: [PATCH 2/6] Limit plugin compatibility to pre-STIX-2 rewrite --- plugins/apps/threatbus_misp/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/apps/threatbus_misp/setup.py b/plugins/apps/threatbus_misp/setup.py index 8de01d4b..f08bf7d3 100644 --- a/plugins/apps/threatbus_misp/setup.py +++ b/plugins/apps/threatbus_misp/setup.py @@ -26,7 +26,7 @@ description="A plugin to enable threatbus communication with MISP.", entry_points={"threatbus.app": ["misp = threatbus_misp.plugin"]}, install_requires=[ - "threatbus>=2020.12.16", + "threatbus >= 2020.12.16, < 2021.2.24", "pymisp >= 2.4.120", ], extras_require={"kafka": ["confluent-kafka>=1.3.0"], "zmq": ["pyzmq>=18.1.1"]}, From 991f7a79163a2ea21a3069ba5f7597c5e11e0e1b Mon Sep 17 00:00:00 2001 From: Felix Ortmann Date: Mon, 22 Feb 2021 17:57:08 +0100 Subject: [PATCH 3/6] Update readme w/ detailed installation instruction --- plugins/apps/threatbus_misp/README.md | 55 +++++++++++++++++++-------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/plugins/apps/threatbus_misp/README.md b/plugins/apps/threatbus_misp/README.md index df3961d7..54ff6d1c 100644 --- a/plugins/apps/threatbus_misp/README.md +++ b/plugins/apps/threatbus_misp/README.md @@ -9,32 +9,55 @@ Threat Bus MISP Plugin -A Threat Bus plugin that enables communication to [MISP](https://www.misp-project.org/). +A Threat Bus plugin that enables communication with [MISP](https://www.misp-project.org/). The plugin goes against the pub/sub architecture of Threat Bus (for now), -because the plugin subscribes a listener to ZeroMQ / Kafka, rather than having -MISP subscribe itself to Threat Bus. That will be addressed with a MISP module -in the near future. +because it actively binds to a single MISP instance to receive attribute +(IoC) updates, and report back sightings via the REST API. Following the strict +pub/sub architecture of Threat Bus, it *should be the other way +around*, with MISP binding to Threat Bus. This will eventually be resolved by a +MISP module. + +For now, the plugin supports two ways to retrieve attribute (IoC) updates from +MISP - either via ZeroMQ or via Kafka. Basically, the plugin makes itself a +subscriber to MISP events. ## Installation +Users can specify *optional dependencies* during installation. The plugin uses +either ZeroMQ or Kafka to get IoC updates from MISP. As we don't want to burden +the user to install unused dependencies, both options are available as follows: + + ```sh -pip install threatbus-misp +pip install threatbus-misp[zmq] +pip install threatbus-misp[kafka] ``` -#### Prerequisites +If neither of these dependencies is installed (i.e., you installed +`threatbus-misp` without the `[...]` suffix for optional deps), the plugin throws +an error and exits immediately. + +### Kafka Prerequisites -*Install Kafka on the Threat Bus host* +When you decide to use Kafka to receive IoC updates from MISP, you first need to +install Kafka on the Threat Bus host. This plugin uses the +[confluent-kafka](https://docs.confluent.io/platform/current/clients/confluent-kafka-python/index.html) +Python package which requires `librdkafka`. See also the +[prerequisites](https://github.com/confluentinc/confluent-kafka-python#prerequisites) +section of the `confluent-kafka` Python client for details about setting it up +for your distribution. -The plugin enables communication either via ZeroMQ or Kafka. When using Kafka, -you have to install `librdkafka` for the host system that is running -`threatbus`. See also the [prerequisites](https://github.com/confluentinc/confluent-kafka-python#prerequisites) -section of the `confluent-kafka` python client. +Once installed, go ahead and install the Kafka version of this plugin: + +``` +pip install threatbus-misp[kafka] +``` ## Configuration -The plugin can either use ZeroMQ or Kafka to retrieve intelligence items from -MISP. It uses the MISP REST api to report back sightings of indicators. +The plugin uses the MISP REST API to report back sightings of IoCs. You need to +specify a MISP API key for it to work. ZeroMQ and Kafka are mutually exclusive, such that Threat Bus does not receive all attribute updates twice. See below for an example configuration. @@ -79,7 +102,7 @@ plugins: ... ``` -### Filter +### IoC Filter The plugin can be configured with a list of filters. Every filter describes a whitelist for MISP attributes (IoCs). The MISP plugin will only forward IoCs to @@ -219,7 +242,7 @@ misp-server: make deploy ``` -*Enable the Kafka plugin in the MISP webview* +*Enable the Kafka plugin in the MISP web-view* - Visit https://localhost:80 - login with your configured credentials @@ -250,7 +273,7 @@ service apache2 restart exit # leave the Docker container shell ``` -*Enable the ZMQ plugin in the MISP webview* +*Enable the ZMQ plugin in the MISP web-view* - Visit https://localhost:80 - login with your configured credentials From 6afbe8eff949fb76be8a1f82a1c1e26386ac0636 Mon Sep 17 00:00:00 2001 From: Felix Ortmann Date: Mon, 22 Feb 2021 17:57:24 +0100 Subject: [PATCH 4/6] Add changelog entry for options deps --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f58cb47..7aac7c01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,15 @@ Every entry has a category for which we use the following visual abbreviations: ## Unreleased +- 🎁 Feature + The MISP plugin now uses [extra dependencies](https://www.python.org/dev/peps/pep-0508/#extras). + Users can now chose the wanted dependencies during installation by running + `pip install threatbus-misp[zmq]` to install the ZeroMQ dependency, or + `pip install threatbus-misp[kafka]` to install the Kafka dependency. The + plugin throws a fatal error if none of these dependencies is installed and + exits immediately. + [#99](https://github.com/tenzir/threatbus/pull/99) + - 🎁 Feature The RabbitMQ backbone plugin, the In-memory backbone plugins, and the Zmq-app plugin now support the From 2019a32f8234d66901b13041edbb49ac95153995 Mon Sep 17 00:00:00 2001 From: Felix Ortmann Date: Tue, 23 Feb 2021 10:16:42 +0100 Subject: [PATCH 5/6] Default to zmq for local installations via make --- plugins/apps/threatbus_misp/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/apps/threatbus_misp/Makefile b/plugins/apps/threatbus_misp/Makefile index 6ae7e548..e93095b8 100644 --- a/plugins/apps/threatbus_misp/Makefile +++ b/plugins/apps/threatbus_misp/Makefile @@ -30,8 +30,8 @@ dist: .PHONY: install install: - pip install . + pip install .[zmq] .PHONY: dev-mode dev-mode: - pip install --editable . + pip install --editable .[zmq] From 9c3743a647d26f407012f4b350b5998127a6e27c Mon Sep 17 00:00:00 2001 From: Felix Ortmann Date: Tue, 23 Feb 2021 11:48:21 +0100 Subject: [PATCH 6/6] Document quoting for installation trouble shooting --- plugins/apps/threatbus_misp/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/apps/threatbus_misp/README.md b/plugins/apps/threatbus_misp/README.md index 54ff6d1c..2ba2f690 100644 --- a/plugins/apps/threatbus_misp/README.md +++ b/plugins/apps/threatbus_misp/README.md @@ -38,6 +38,10 @@ If neither of these dependencies is installed (i.e., you installed `threatbus-misp` without the `[...]` suffix for optional deps), the plugin throws an error and exits immediately. +**Depending on your setup, you might want to use quotes to avoid shell expansion +when using `[...]`**. For example, you can do `pip install ".[zmq]"` for local +development. + ### Kafka Prerequisites When you decide to use Kafka to receive IoC updates from MISP, you first need to