Merge pull request #106

Rewrite CIF3 plugin to convert STIX-2 Indicators

Makefile

@@ -23,11 +23,8 @@ unit-tests:
 	$(MAKE) -C plugins/apps/threatbus_zmq_app unit-tests
 	$(MAKE) -C plugins/apps/threatbus_misp unit-tests
 	$(MAKE) -C plugins/apps/threatbus_zeek unit-tests
+	$(MAKE) -C plugins/apps/threatbus_cif3 unit-tests
 	$(MAKE) -C apps/vast unit-tests
-  # Threat Bus is currently being migrated to use STIX-2 as internal format.
-  # For the time being, all un-migrated plugins cannot be not tested against the
-  # current master
-	#$(MAKE) -C plugins/apps/threatbus_cif3 unit-tests
 
 .PHONY: integration-tests
 integration-tests:
@@ -96,7 +93,4 @@ dev-mode:
 	$(MAKE) -C plugins/apps/threatbus_misp dev-mode
 	$(MAKE) -C plugins/apps/threatbus_zeek dev-mode
 	$(MAKE) -C apps/vast dev-mode
-  # Threat Bus is currently being migrated to use STIX-2 as internal format.
-  # For the time being, all un-migrated plugins cannot be run in conjunction
-  # with current master
-	# $(MAKE) -C plugins/apps/threatbus_cif3 dev-mode
+	$(MAKE) -C plugins/apps/threatbus_cif3 dev-mode

plugins/apps/threatbus_cif3/README.md

@@ -1,7 +1,23 @@
 Threat Bus CIFv3 Plugin
 ======================
 
-A Threat Bus plugin that enables communication to [Collective Intelligence Framework v3](https://github.com/csirtgadgets/bearded-avenger).
+<h4 align="center">
+
+[![PyPI Status][pypi-badge]][pypi-url]
+[![Build Status][ci-badge]][ci-url]
+[![License][license-badge]][license-url]
+
+</h4>
+
+A Threat Bus plugin to push indicators from Threat Bus to
+[Collective Intelligence Framework v3](https://github.com/csirtgadgets/bearded-avenger).
+
+The plugin uses the [cifsdk (v3.x)](https://pypi.org/project/cifsdk/) Python
+client to submit indicators received from Threat Bus into a CIFv3 instance.
+
+The plugin breaks with the pub/sub architecture of Threat Bus, because CIF does
+not subscribe itself to the bus. Instead, the plugin actively contacts a CIF
+endpoint.
 
 ## Installation
 
@@ -11,7 +27,8 @@ pip install threatbus-cif3
 
 ## Configuration
 
-The plugin uses the cifsdk python client to submit indicators received on the threatbus into a CIF instance.
+Configure this plugin by adding a section to Threat Bus' `config.yaml` file, as
+follows:
 
 ```yaml
 ...
@@ -32,7 +49,7 @@ plugins:
 
 ## Development Setup
 
-The following guides describe how to set up local, dockerized instances of MISP.
+The following guides describe how to set up local, dockerized instances of CIF.
 
 ### Dockerized CIFv3
 
@@ -52,7 +69,8 @@ docker-compose build
 ```sh
 vim docker-compose.yml
 ```
-Find the section `cif` in the configuration and edit the following as appropriate:
+Find the section `cif` in the configuration and edit the following as
+appropriate to bind port 5000 to your localhost:
 
 ```yaml
 cif:
@@ -67,22 +85,23 @@ cif:
 
 ```sh
 docker-compose up -d
-# get an interactive shell
+# Get an interactive shell in the container:
 docker-compose exec cif /bin/bash
-# become the cif user
+# Become the cif user:
 su cif
-# check to see if access tokens were successfully created
+# check to see if access tokens were successfully created. Copy the `admin`
+# token to the CIF config section:
 cif-tokens
-# ping the router to ensure connectivity
+# Ping the router to ensure connectivity:
 cif --ping
 ```
 
 ## License
 
 Threat Bus comes with a [3-clause BSD license][license-url].
 
-[pypi-badge]: https://img.shields.io/pypi/v/threatbus-misp.svg
-[pypi-url]: https://pypi.org/project/threatbus-misp
+[pypi-badge]: https://img.shields.io/pypi/v/threatbus-cif3.svg
+[pypi-url]: https://pypi.org/project/threatbus-cif3
 [ci-url]: https://github.com/tenzir/threatbus/actions?query=branch%3Amaster
 [ci-badge]: https://github.com/tenzir/threatbus/workflows/Python%20Egg/badge.svg?branch=master
 [license-badge]: https://img.shields.io/badge/license-BSD-blue.svg

plugins/apps/threatbus_cif3/setup.py

@@ -26,7 +26,8 @@
     description="A plugin to enable indicators to be submitted to CIFv3 in real-time",
     entry_points={"threatbus.app": ["cif3 = threatbus_cif3.plugin"]},
     install_requires=[
-        "threatbus >= 2020.12.16, < 2021.2.24",
+        "stix2 >= 2.1",
+        "threatbus >= 2021.3.25",
         "cifsdk > 3.0.0rc4, < 4.0",
     ],
     keywords=[

plugins/apps/threatbus_cif3/threatbus_cif3/message_mapping.py

@@ -1,47 +1,61 @@
-from threatbus.data import Intel, IntelType, Operation
-from csirtg_indicator import Indicator
+from csirtg_indicator import Indicator as CIFIndicator
 from csirtg_indicator.exceptions import InvalidIndicator
+from stix2 import Indicator
+from threatbus.data import ThreatBusSTIX2Constants
+from threatbus.stix2_helpers import is_point_equality_ioc, split_object_path_and_value
+from typing import List, Union
+
 
 cif_supported_types = [
-    IntelType.IPSRC,
-    IntelType.IPDST,
-    IntelType.EMAILSRC,
-    IntelType.HOSTNAME,
-    IntelType.DOMAIN,
-    IntelType.URL,
-    IntelType.MD5,
-    IntelType.SHA1,
-    IntelType.SHA256,
-    IntelType.AUTHENTIHASH,
-    IntelType.SSDEEP,
-    IntelType.IMPHASH,
-    IntelType.PEHASH,
+    "ipv4-addr:value",
+    "ipv6-addr:value",
+    "domain-name:value",
+    "email-addr:value",
+    "url:value",
+    "file:hashes.MD5",
+    "file:hashes.'SHA-1'",
+    "file:hashes.'SHA-256'",
+    "file:hashes.SSDEEP",
 ]
 
 
-def map_to_cif(intel: Intel, logger, confidence, tags, tlp, group):
+def map_to_cif(
+    indicator: Indicator, confidence: int, tags: List[str], tlp: str, group: str, logger
+) -> Union[CIFIndicator, None]:
     """
-    Maps an Intel item to a CIFv3 compatible indicator format.
-    @param intel The item to map
+    Maps a STIX-2 Indicator to a CIFv3 compatible indicator format.
+    @param indicator The STIX-2 Indicator to map
+    @param confidence The confidence to use when building the CIF indicator
+    @param tags The tags to use when building the CIF indicator
+    @param tlp The tlp to use when building the CIF indicator
+    @param group The group to use when building the CIF indicator
     @return the mapped intel item or None
     """
+    if not indicator or type(indicator) is not Indicator:
+        logger.debug(f"Expected STIX-2 indicator, discarding {indicator}")
+        return None
     if (
-        not intel
-        or intel.operation != Operation.ADD
-        or intel.data["intel_type"] not in cif_supported_types
+        ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value
+        in indicator.object_properties()
     ):
+        logger.debug(
+            f"CIFv3 only supports adding indicators, not deleting / editing. Discardig {indicator}"
+        )
+        return None
+    if not is_point_equality_ioc(indicator.pattern):
+        logger.debug(f"CIFv3 only supports point indicators, discardig {indicator}")
         return None
 
-    # parse values
-    indicator = intel.data["indicator"][0]  # indicators are tuples in Threatbus
-    if not indicator:
+    object_path, ioc_value = split_object_path_and_value(indicator.pattern)
+    if object_path not in cif_supported_types:
+        logger.debug(f"Discardig indicator with unsupported object-path {indicator}")
         return None
 
     # convert lasttime
-    lasttime = intel.ts.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    lasttime = indicator.created.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
 
-    ii = {
-        "indicator": indicator,
+    ioc_dict = {
+        "indicator": ioc_value,
         "confidence": confidence,
         "tags": tags,
         "tlp": tlp,
@@ -50,7 +64,7 @@ def map_to_cif(intel: Intel, logger, confidence, tags, tlp, group):
     }
 
     try:
-        return Indicator(**ii)
+        return CIFIndicator(**ioc_dict)
     except InvalidIndicator as e:
         logger.error(f"Invalid CIF indicator {e}")
     except Exception as e:

plugins/apps/threatbus_cif3/threatbus_cif3/plugin.py

@@ -15,17 +15,17 @@
 
 class CIFPublisher(threatbus.StoppableWorker):
     """
-    Reports / publishes intel items back to the given CIF endpoint.
+    Reports / publishes Indicators to the given CIF endpoint.
     """
 
-    def __init__(self, intel_outq: JoinableQueue, cif: Client, config: Subview):
+    def __init__(self, indicator_q: JoinableQueue, cif: Client, config: Subview):
         """
-        @param intel_outq Publish all intel from this queue to CIF
+        @param indicator_q Publish all indicators from this queue to CIF
         @param cif The CIF client to use
         @config the plugin config
         """
         super(CIFPublisher, self).__init__()
-        self.intel_outq = intel_outq
+        self.indicator_q = indicator_q
         self.cif = cif
         self.config = config
 
@@ -43,24 +43,25 @@ def run(self):
 
         while self._running():
             try:
-                intel = self.intel_outq.get(block=True, timeout=1)
+                indicator = self.indicator_q.get(block=True, timeout=1)
             except Empty:
                 continue
-            if not intel:
-                logger.warning("Received unparsable intel item")
-                self.intel_outq.task_done()
+            if not indicator:
+                self.indicator_q.task_done()
                 continue
-            cif_mapped_intel = map_to_cif(intel, logger, confidence, tags, tlp, group)
+            cif_mapped_intel = map_to_cif(
+                indicator, confidence, tags, tlp, group, logger
+            )
             if not cif_mapped_intel:
-                self.intel_outq.task_done()
+                self.indicator_q.task_done()
                 continue
             try:
-                logger.debug(f"Adding intel to CIF: {cif_mapped_intel}")
+                logger.debug(f"Adding indicator to CIF {cif_mapped_intel}")
                 self.cif.indicators_create(cif_mapped_intel)
             except Exception as err:
-                logger.error(f"CIF submission error: {err}")
+                logger.error(f"Error adding indicator to CIF {err}")
             finally:
-                self.intel_outq.task_done()
+                self.indicator_q.task_done()
 
 
 def validate_config(config: Subview):
@@ -106,11 +107,11 @@ def run(
         )
         return
 
-    intel_outq = JoinableQueue()
-    topic = "threatbus/intel"
-    subscribe_callback(topic, intel_outq)
+    indicator_q = JoinableQueue()
+    topic = "stix2/indicator"
+    subscribe_callback(topic, indicator_q)
 
-    workers.append(CIFPublisher(intel_outq, cif, config))
+    workers.append(CIFPublisher(indicator_q, cif, config))
     for w in workers:
         w.start()