From 7d7ce6313eb787e1dfc91db8eb5f02774225cdd0 Mon Sep 17 00:00:00 2001 From: Felix Ortmann Date: Wed, 17 Mar 2021 14:32:16 +0100 Subject: [PATCH 1/4] Depend on newest Threat Bus version to support STIX-2 --- plugins/apps/threatbus_cif3/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/apps/threatbus_cif3/setup.py b/plugins/apps/threatbus_cif3/setup.py index a04603f9..5bb1b0e7 100644 --- a/plugins/apps/threatbus_cif3/setup.py +++ b/plugins/apps/threatbus_cif3/setup.py @@ -26,7 +26,8 @@ description="A plugin to enable indicators to be submitted to CIFv3 in real-time", entry_points={"threatbus.app": ["cif3 = threatbus_cif3.plugin"]}, install_requires=[ - "threatbus >= 2020.12.16, < 2021.2.24", + "stix2 >= 2.1", + "threatbus >= 2021.3.25", "cifsdk > 3.0.0rc4, < 4.0", ], keywords=[ From de8f4ef52c5ce4dd73ba4c37017aaa8da5d4b3b3 Mon Sep 17 00:00:00 2001 From: Felix Ortmann Date: Wed, 17 Mar 2021 14:34:17 +0100 Subject: [PATCH 2/4] Map STIX-2 Indicators to CIF3 format --- .../threatbus_cif3/message_mapping.py | 70 +++++++++++-------- .../threatbus_cif3/threatbus_cif3/plugin.py | 35 +++++----- 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/plugins/apps/threatbus_cif3/threatbus_cif3/message_mapping.py b/plugins/apps/threatbus_cif3/threatbus_cif3/message_mapping.py index 43c5b44f..bfe00d45 100644 --- a/plugins/apps/threatbus_cif3/threatbus_cif3/message_mapping.py +++ b/plugins/apps/threatbus_cif3/threatbus_cif3/message_mapping.py @@ -1,47 +1,61 @@ -from threatbus.data import Intel, IntelType, Operation -from csirtg_indicator import Indicator +from csirtg_indicator import Indicator as CIFIndicator from csirtg_indicator.exceptions import InvalidIndicator +from stix2 import Indicator +from threatbus.data import ThreatBusSTIX2Constants +from threatbus.stix2_helpers import is_point_equality_ioc, split_object_path_and_value +from typing import List, Union + cif_supported_types = [ - IntelType.IPSRC, - IntelType.IPDST, - IntelType.EMAILSRC, - IntelType.HOSTNAME, - IntelType.DOMAIN, - IntelType.URL, - IntelType.MD5, - IntelType.SHA1, - IntelType.SHA256, - IntelType.AUTHENTIHASH, - IntelType.SSDEEP, - IntelType.IMPHASH, - IntelType.PEHASH, + "ipv4-addr:value", + "ipv6-addr:value", + "domain-name:value", + "email-addr:value", + "url:value", + "file:hashes.MD5", + "file:hashes.'SHA-1'", + "file:hashes.'SHA-256'", + "file:hashes.SSDEEP", ] -def map_to_cif(intel: Intel, logger, confidence, tags, tlp, group): +def map_to_cif( + indicator: Indicator, confidence: int, tags: List[str], tlp: str, group: str, logger +) -> Union[CIFIndicator, None]: """ - Maps an Intel item to a CIFv3 compatible indicator format. - @param intel The item to map + Maps a STIX-2 Indicator to a CIFv3 compatible indicator format. + @param indicator The STIX-2 Indicator to map + @param confidence The confidence to use when building the CIF indicator + @param tags The tags to use when building the CIF indicator + @param tlp The tlp to use when building the CIF indicator + @param group The group to use when building the CIF indicator @return the mapped intel item or None """ + if not indicator or type(indicator) is not Indicator: + logger.debug(f"Expected STIX-2 indicator, discarding {indicator}") + return None if ( - not intel - or intel.operation != Operation.ADD - or intel.data["intel_type"] not in cif_supported_types + ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value + in indicator.object_properties() ): + logger.debug( + f"CIFv3 only supports adding indicators, not deleting / editing. Discardig {indicator}" + ) + return None + if not is_point_equality_ioc(indicator.pattern): + logger.debug(f"CIFv3 only supports point indicators, discardig {indicator}") return None - # parse values - indicator = intel.data["indicator"][0] # indicators are tuples in Threatbus - if not indicator: + object_path, ioc_value = split_object_path_and_value(indicator.pattern) + if object_path not in cif_supported_types: + logger.debug(f"Discardig indicator with unsupported object-path {indicator}") return None # convert lasttime - lasttime = intel.ts.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + lasttime = indicator.created.strftime("%Y-%m-%dT%H:%M:%S.%fZ") - ii = { - "indicator": indicator, + ioc_dict = { + "indicator": ioc_value, "confidence": confidence, "tags": tags, "tlp": tlp, @@ -50,7 +64,7 @@ def map_to_cif(intel: Intel, logger, confidence, tags, tlp, group): } try: - return Indicator(**ii) + return CIFIndicator(**ioc_dict) except InvalidIndicator as e: logger.error(f"Invalid CIF indicator {e}") except Exception as e: diff --git a/plugins/apps/threatbus_cif3/threatbus_cif3/plugin.py b/plugins/apps/threatbus_cif3/threatbus_cif3/plugin.py index 09ae4027..ecd8646d 100644 --- a/plugins/apps/threatbus_cif3/threatbus_cif3/plugin.py +++ b/plugins/apps/threatbus_cif3/threatbus_cif3/plugin.py @@ -15,17 +15,17 @@ class CIFPublisher(threatbus.StoppableWorker): """ - Reports / publishes intel items back to the given CIF endpoint. + Reports / publishes Indicators to the given CIF endpoint. """ - def __init__(self, intel_outq: JoinableQueue, cif: Client, config: Subview): + def __init__(self, indicator_q: JoinableQueue, cif: Client, config: Subview): """ - @param intel_outq Publish all intel from this queue to CIF + @param indicator_q Publish all indicators from this queue to CIF @param cif The CIF client to use @config the plugin config """ super(CIFPublisher, self).__init__() - self.intel_outq = intel_outq + self.indicator_q = indicator_q self.cif = cif self.config = config @@ -43,24 +43,25 @@ def run(self): while self._running(): try: - intel = self.intel_outq.get(block=True, timeout=1) + indicator = self.indicator_q.get(block=True, timeout=1) except Empty: continue - if not intel: - logger.warning("Received unparsable intel item") - self.intel_outq.task_done() + if not indicator: + self.indicator_q.task_done() continue - cif_mapped_intel = map_to_cif(intel, logger, confidence, tags, tlp, group) + cif_mapped_intel = map_to_cif( + indicator, confidence, tags, tlp, group, logger + ) if not cif_mapped_intel: - self.intel_outq.task_done() + self.indicator_q.task_done() continue try: - logger.debug(f"Adding intel to CIF: {cif_mapped_intel}") + logger.debug(f"Adding indicator to CIF {cif_mapped_intel}") self.cif.indicators_create(cif_mapped_intel) except Exception as err: - logger.error(f"CIF submission error: {err}") + logger.error(f"Error adding indicator to CIF {err}") finally: - self.intel_outq.task_done() + self.indicator_q.task_done() def validate_config(config: Subview): @@ -106,11 +107,11 @@ def run( ) return - intel_outq = JoinableQueue() - topic = "threatbus/intel" - subscribe_callback(topic, intel_outq) + indicator_q = JoinableQueue() + topic = "stix2/indicator" + subscribe_callback(topic, indicator_q) - workers.append(CIFPublisher(intel_outq, cif, config)) + workers.append(CIFPublisher(indicator_q, cif, config)) for w in workers: w.start() From 3e191966114ee397b4c2ba7b5cfa6119c58a03d1 Mon Sep 17 00:00:00 2001 From: Felix Ortmann Date: Wed, 17 Mar 2021 14:42:41 +0100 Subject: [PATCH 3/4] Update CIF3 readme --- plugins/apps/threatbus_cif3/README.md | 39 ++++++++++++++++++++------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/plugins/apps/threatbus_cif3/README.md b/plugins/apps/threatbus_cif3/README.md index c51a5b35..3bf138af 100644 --- a/plugins/apps/threatbus_cif3/README.md +++ b/plugins/apps/threatbus_cif3/README.md @@ -1,7 +1,23 @@ Threat Bus CIFv3 Plugin ====================== -A Threat Bus plugin that enables communication to [Collective Intelligence Framework v3](https://github.com/csirtgadgets/bearded-avenger). +

+ +[![PyPI Status][pypi-badge]][pypi-url] +[![Build Status][ci-badge]][ci-url] +[![License][license-badge]][license-url] + +

+ +A Threat Bus plugin to push indicators from Threat Bus to +[Collective Intelligence Framework v3](https://github.com/csirtgadgets/bearded-avenger). + +The plugin uses the [cifsdk (v3.x)](https://pypi.org/project/cifsdk/) Python +client to submit indicators received from Threat Bus into a CIFv3 instance. + +The plugin breaks with the pub/sub architecture of Threat Bus, because CIF does +not subscribe itself to the bus. Instead, the plugin actively contacts a CIF +endpoint. ## Installation @@ -11,7 +27,8 @@ pip install threatbus-cif3 ## Configuration -The plugin uses the cifsdk python client to submit indicators received on the threatbus into a CIF instance. +Configure this plugin by adding a section to Threat Bus' `config.yaml` file, as +follows: ```yaml ... @@ -32,7 +49,7 @@ plugins: ## Development Setup -The following guides describe how to set up local, dockerized instances of MISP. +The following guides describe how to set up local, dockerized instances of CIF. ### Dockerized CIFv3 @@ -52,7 +69,8 @@ docker-compose build ```sh vim docker-compose.yml ``` -Find the section `cif` in the configuration and edit the following as appropriate: +Find the section `cif` in the configuration and edit the following as +appropriate to bind port 5000 to your localhost: ```yaml cif: @@ -67,13 +85,14 @@ cif: ```sh docker-compose up -d -# get an interactive shell +# Get an interactive shell in the container: docker-compose exec cif /bin/bash -# become the cif user +# Become the cif user: su cif -# check to see if access tokens were successfully created +# check to see if access tokens were successfully created. Copy the `admin` +# token to the CIF config section: cif-tokens -# ping the router to ensure connectivity +# Ping the router to ensure connectivity: cif --ping ``` @@ -81,8 +100,8 @@ cif --ping Threat Bus comes with a [3-clause BSD license][license-url]. -[pypi-badge]: https://img.shields.io/pypi/v/threatbus-misp.svg -[pypi-url]: https://pypi.org/project/threatbus-misp +[pypi-badge]: https://img.shields.io/pypi/v/threatbus-cif3.svg +[pypi-url]: https://pypi.org/project/threatbus-cif3 [ci-url]: https://github.com/tenzir/threatbus/actions?query=branch%3Amaster [ci-badge]: https://github.com/tenzir/threatbus/workflows/Python%20Egg/badge.svg?branch=master [license-badge]: https://img.shields.io/badge/license-BSD-blue.svg From c21cd21621a5be8f3003188b223c9b1c64fdffd6 Mon Sep 17 00:00:00 2001 From: Felix Ortmann Date: Wed, 17 Mar 2021 14:43:00 +0100 Subject: [PATCH 4/4] Re-enable CIF3 plugin --- Makefile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index f0384a32..5728b262 100644 --- a/Makefile +++ b/Makefile @@ -23,11 +23,8 @@ unit-tests: $(MAKE) -C plugins/apps/threatbus_zmq_app unit-tests $(MAKE) -C plugins/apps/threatbus_misp unit-tests $(MAKE) -C plugins/apps/threatbus_zeek unit-tests + $(MAKE) -C plugins/apps/threatbus_cif3 unit-tests $(MAKE) -C apps/vast unit-tests - # Threat Bus is currently being migrated to use STIX-2 as internal format. - # For the time being, all un-migrated plugins cannot be not tested against the - # current master - #$(MAKE) -C plugins/apps/threatbus_cif3 unit-tests .PHONY: integration-tests integration-tests: @@ -96,7 +93,4 @@ dev-mode: $(MAKE) -C plugins/apps/threatbus_misp dev-mode $(MAKE) -C plugins/apps/threatbus_zeek dev-mode $(MAKE) -C apps/vast dev-mode - # Threat Bus is currently being migrated to use STIX-2 as internal format. - # For the time being, all un-migrated plugins cannot be run in conjunction - # with current master - # $(MAKE) -C plugins/apps/threatbus_cif3 dev-mode + $(MAKE) -C plugins/apps/threatbus_cif3 dev-mode