Markdown summary table

CHANGELOG.md

@@ -16,7 +16,7 @@
 - None
 
 ## New features
-- None
+- Added support for markdown style formatting of aggregation tables - [#415](https://github.com/jertel/elastalert2/pull/415) - @Neuro-HSOC
 
 ## Other changes
 - Fixed typo in default setting accidentally introduced in [#407](https://github.com/jertel/elastalert2/pull/407)  - [#413](https://github.com/jertel/elastalert2/pull/413) - @perceptron01

docs/source/ruletypes.rst

@@ -387,6 +387,8 @@ This should result in 2 alerts: One containing alice's two events, sent at ``201
 
 For aggregations, there can sometimes be a large number of documents present in the viewing medium (email, Jira ticket, etc..). If you set the ``summary_table_fields`` field, ElastAlert 2 will provide a summary of the specified fields from all the results.
 
+The formatting style of the summary table can be switched between ``ascii`` (default) and ``markdown`` with parameter ``summary_table_type``. ``markdown`` might be the more suitable formatting for alerters supporting it like TheHive.
+
 For example, if you wish to summarize the usernames and event_types that appear in the documents so that you can see the most relevant fields at a quick glance, you can set::
 
     summary_table_fields:
@@ -709,6 +711,21 @@ summary_table_fields
 
 ``summary_table_fields``: Specifying the summmary_table_fields in conjunction with an aggregation will make it so that each aggregated alert will contain a table summarizing the values for the specified fields in all the matches that were aggregated together.
 
+summary_table_type
+^^^^^^^^^^^^^^^^^^^^
+
+``summary_table_type``: Either ``ascii`` or ``markdown``. Select the table type to use for the aggregation summary. Defaults to ``ascii`` for the classical text based table.
+
+summary_prefix
+^^^^^^^^^^^^^^^^^^^^
+
+``summary_prefix``: Specify a prefix string, which will be added in front of the aggregation summary table. This string is currently not subject to any formatting.
+
+summary_suffix
+^^^^^^^^^^^^^^^^^^^^
+
+``summary_suffix``: Specify a suffix string, which will be added after the aggregation summary table. This string is currently not subject to any formatting.
+
 timestamp_type
 ^^^^^^^^^^^^^^
 

elastalert/alerts.py

@@ -237,19 +237,26 @@ def get_aggregation_summary_text__maximum_width(self):
     def get_aggregation_summary_text(self, matches):
         text = ''
         if 'aggregation' in self.rule and 'summary_table_fields' in self.rule:
+            summary_table_type = self.rule.get('summary_table_type', 'ascii')
+
+            #Type independent prefix
             text = self.rule.get('summary_prefix', '')
+            # If a prefix is set, ensure there is a newline between it and the hardcoded
+            # 'Aggregation resulted in...' header below
+            if text != '':
+                text += "\n"
+
             summary_table_fields = self.rule['summary_table_fields']
             if not isinstance(summary_table_fields, list):
                 summary_table_fields = [summary_table_fields]
+
             # Include a count aggregation so that we can see at a glance how many of each aggregation_key were encountered
             summary_table_fields_with_count = summary_table_fields + ['count']
             text += "Aggregation resulted in the following data for summary_table_fields ==> {0}:\n\n".format(
                 summary_table_fields_with_count
             )
-            text_table = Texttable(max_width=self.get_aggregation_summary_text__maximum_width())
-            text_table.header(summary_table_fields_with_count)
-            # Format all fields as 'text' to avoid long numbers being shown as scientific notation
-            text_table.set_cols_dtype(['t' for i in summary_table_fields_with_count])
+
+            # Prepare match_aggregation used in both table types
             match_aggregation = {}
 
             # Maintain an aggregate count for each unique key encountered in the aggregation period
@@ -259,10 +266,34 @@ def get_aggregation_summary_text(self, matches):
                     match_aggregation[key_tuple] = 1
                 else:
                     match_aggregation[key_tuple] = match_aggregation[key_tuple] + 1
-            for keys, count in match_aggregation.items():
-                text_table.add_row([key for key in keys] + [count])
-            text += text_table.draw() + '\n\n'
-            text += self.rule.get('summary_prefix', '')
+
+            # Type dependent table style
+            if summary_table_type == 'ascii':
+                text_table = Texttable(max_width=self.get_aggregation_summary_text__maximum_width())
+                text_table.header(summary_table_fields_with_count)
+                # Format all fields as 'text' to avoid long numbers being shown as scientific notation
+                text_table.set_cols_dtype(['t' for i in summary_table_fields_with_count])
+
+                for keys, count in match_aggregation.items():
+                    text_table.add_row([key for key in keys] + [count])
+                text += text_table.draw() + '\n\n'
+
+            elif summary_table_type == 'markdown':
+                # Adapted from https://github.com/codazoda/tomark/blob/master/tomark/tomark.py
+                # Create table header
+                text += '| ' + ' | '.join(map(str, summary_table_fields_with_count)) + ' |\n'
+                # Create header separator
+                text += '|-----' * len(summary_table_fields_with_count) + '|\n'
+                # Create table row
+                for keys, count in match_aggregation.items():
+                    markdown_row = ""
+                    for key in keys:
+                        markdown_row += '| ' + str(key) + ' '
+                    text += markdown_row + '| ' + str(count) + ' |\n'
+                text += '\n'
+
+            # Type independent suffix
+            text += self.rule.get('summary_suffix', '')
         return str(text)
 
     def create_default_title(self, matches):

tests/alerts_test.py

@@ -261,6 +261,83 @@ def test_alert_get_aggregation_summary_text__maximum_width():
     assert 80 == alert.get_aggregation_summary_text__maximum_width()
 
 
+def test_alert_aggregation_summary_markdown_table():
+    rule = {
+        'name': 'test_rule',
+        'type': mock_rule(),
+        'owner': 'the_owner',
+        'priority': 2,
+        'alert_subject': 'A very long subject',
+        'aggregation': 1,
+        'summary_table_fields': ['field', 'abc'],
+        'summary_table_type': 'markdown'
+    }
+    matches = [
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', },
+    ]
+    alert = Alerter(rule)
+    summary_table = str(alert.get_aggregation_summary_text(matches))
+    assert "| field | abc | count |" in summary_table
+    assert "|-----|-----|-----|" in summary_table
+    assert "| field_value | abc from match | 3 |" in summary_table
+    assert "| field_value | cde from match | 2 |" in summary_table
+
+
+def test_alert_aggregation_summary_default_table():
+    rule = {
+        'name': 'test_rule',
+        'type': mock_rule(),
+        'owner': 'the_owner',
+        'priority': 2,
+        'alert_subject': 'A very long subject',
+        'aggregation': 1,
+        'summary_table_fields': ['field', 'abc'],
+    }
+    matches = [
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', },
+    ]
+    alert = Alerter(rule)
+    summary_table = str(alert.get_aggregation_summary_text(matches))
+    assert "+-------------+----------------+-------+" in summary_table
+    assert "|    field    |      abc       | count |" in summary_table
+    assert "+=============+================+=======+" in summary_table
+    assert "| field_value | abc from match | 3     |" in summary_table
+    assert "| field_value | cde from match | 2     |" in summary_table
+
+
+def test_alert_aggregation_summary_table_suffix_prefix():
+    rule = {
+        'name': 'test_rule',
+        'type': mock_rule(),
+        'owner': 'the_owner',
+        'priority': 2,
+        'alert_subject': 'A very long subject',
+        'aggregation': 1,
+        'summary_table_fields': ['field', 'abc'],
+        'summary_prefix': 'This is the prefix',
+        'summary_suffix': 'This is the suffix',
+    }
+    matches = [
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', },
+        {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', },
+    ]
+    alert = Alerter(rule)
+    summary_table = str(alert.get_aggregation_summary_text(matches))
+    assert "This is the prefix" in summary_table
+    assert "This is the suffix" in summary_table
+
+
 def test_alert_subject_size_limit_with_args(ea):
     rule = {
         'name': 'test_rule',