From 00f13d0e7bdfa1753e1b17a2c88ae9d4c56fad25 Mon Sep 17 00:00:00 2001 From: Neuro-HSOC <74316448+Neuro-HSOC@users.noreply.github.com> Date: Fri, 30 Jul 2021 00:04:48 +0200 Subject: [PATCH 1/9] Added markdown table output code --- elastalert/alerts.py | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 6573dc18..48782868 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -237,19 +237,21 @@ def get_aggregation_summary_text__maximum_width(self): def get_aggregation_summary_text(self, matches): text = '' if 'aggregation' in self.rule and 'summary_table_fields' in self.rule: + summary_table_type = self.rule.get('summary_table_type', 'ascii') + + #Type independent prefix text = self.rule.get('summary_prefix', '') summary_table_fields = self.rule['summary_table_fields'] if not isinstance(summary_table_fields, list): summary_table_fields = [summary_table_fields] + # Include a count aggregation so that we can see at a glance how many of each aggregation_key were encountered summary_table_fields_with_count = summary_table_fields + ['count'] text += "Aggregation resulted in the following data for summary_table_fields ==> {0}:\n\n".format( summary_table_fields_with_count ) - text_table = Texttable(max_width=self.get_aggregation_summary_text__maximum_width()) - text_table.header(summary_table_fields_with_count) - # Format all fields as 'text' to avoid long numbers being shown as scientific notation - text_table.set_cols_dtype(['t' for i in summary_table_fields_with_count]) + + # Prepare match_aggregation used in both table types match_aggregation = {} # Maintain an aggregate count for each unique key encountered in the aggregation period @@ -259,10 +261,34 @@ def get_aggregation_summary_text(self, matches): match_aggregation[key_tuple] = 1 else: match_aggregation[key_tuple] = match_aggregation[key_tuple] + 1 - for keys, count in match_aggregation.items(): - text_table.add_row([key for key in keys] + [count]) - text += text_table.draw() + '\n\n' - text += self.rule.get('summary_prefix', '') + + # Type dependent table style + if summary_table_type == 'ascii': + text_table = Texttable(max_width=self.get_aggregation_summary_text__maximum_width()) + text_table.header(summary_table_fields_with_count) + # Format all fields as 'text' to avoid long numbers being shown as scientific notation + text_table.set_cols_dtype(['t' for i in summary_table_fields_with_count]) + + for keys, count in match_aggregation.items(): + text_table.add_row([key for key in keys] + [count]) + text += text_table.draw() + '\n\n' + + elif summary_table_type == 'markdown': + # Adapted from https://github.com/codazoda/tomark/blob/master/tomark/tomark.py + # Create table header + text += '| ' + ' | '.join(map(str, summary_table_fields_with_count)) + ' |\n' + # Create header separator + text += '|-----' * len(summary_table_fields_with_count) + '|\n' + # Create table row + for keys, count in match_aggregation.items(): + markdown_row = "" + for key in keys: + markdown_row += '| ' + str(key) + ' ' + text += markdown_row + '| ' + str(count) + ' |\n' + text += '\n' + + # Type independent suffix + text += self.rule.get('summary_suffix', '') return str(text) def create_default_title(self, matches): From 0006e53bcf5fa39a04f1db0514600278681aabd5 Mon Sep 17 00:00:00 2001 From: Neuro-HSOC <74316448+Neuro-HSOC@users.noreply.github.com> Date: Mon, 23 Aug 2021 16:11:44 +0200 Subject: [PATCH 2/9] Add markdown summary table documentation Added documentation for the configuration options related to the markdown summary table. These options include: - summary_table_type (Defaults to ascii) - summary_prefix - summary_suffix Also fixed one too many newline chars at the end of the markdown table. --- docs/source/ruletypes.rst | 15 +++++++++++++++ elastalert/alerts.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 6aaf7c0f..523f16d4 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -709,6 +709,21 @@ summary_table_fields ``summary_table_fields``: Specifying the summmary_table_fields in conjunction with an aggregation will make it so that each aggregated alert will contain a table summarizing the values for the specified fields in all the matches that were aggregated together. +summary_table_type +^^^^^^^^^^^^^^^^^^^^ + +``summary_table_type``: Either ``ascii`` or ``markdown``. Select the table type to use for the aggregation summary. Defaults to ``ascii`` for the classical text based table. + +summary_prefix +^^^^^^^^^^^^^^^^^^^^ + +``summary_prefix``: Specify a prefix string, which will be added in front of the aggregation summary table. This string is currently not subject to any formating. + +summary_suffix +^^^^^^^^^^^^^^^^^^^^ + +``summary_suffix``: Specify a suffix string, which will be added after the aggregation summary table. This string is currently not subject to any formating. + timestamp_type ^^^^^^^^^^^^^^ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 48782868..7438ef39 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -285,7 +285,7 @@ def get_aggregation_summary_text(self, matches): for key in keys: markdown_row += '| ' + str(key) + ' ' text += markdown_row + '| ' + str(count) + ' |\n' - text += '\n' + text += '\n\n' # Type independent suffix text += self.rule.get('summary_suffix', '') From d0d78b6e76d6f15913b3509443f913d7fb4c0c52 Mon Sep 17 00:00:00 2001 From: Neuro-HSOC <74316448+Neuro-HSOC@users.noreply.github.com> Date: Mon, 23 Aug 2021 16:37:55 +0200 Subject: [PATCH 3/9] Add summary_table_type to aggregation Added a short description of the possible summary_table_types to the documentation of the aggregation parameter. --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 523f16d4..0b168932 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -387,6 +387,8 @@ This should result in 2 alerts: One containing alice's two events, sent at ``201 For aggregations, there can sometimes be a large number of documents present in the viewing medium (email, Jira ticket, etc..). If you set the ``summary_table_fields`` field, ElastAlert 2 will provide a summary of the specified fields from all the results. +The formating style of the summary table can be switched between ``ascii`` (default) and ``markdown`` with parameter ``summary_table_type``. ``markdown`` might be the more suitable formating for alerters supporting it like TheHive. + For example, if you wish to summarize the usernames and event_types that appear in the documents so that you can see the most relevant fields at a quick glance, you can set:: summary_table_fields: From cebd973b363eb82af89a7218b4fd1c8988fdb7e1 Mon Sep 17 00:00:00 2001 From: Neuro-HSOC <74316448+Neuro-HSOC@users.noreply.github.com> Date: Mon, 23 Aug 2021 16:51:26 +0200 Subject: [PATCH 4/9] Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1b76278..4dbe8372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ - None ## New features -- None +- Added support for markdown style formating of aggregation tables - @Neuro-HSOC ## Other changes - Fixed typo in default setting accidentally introduced in [#407](https://github.com/jertel/elastalert2/pull/407) - [#413](https://github.com/jertel/elastalert2/pull/413) - @perceptron01 From 9d80f4284d917a764e95dd95d0c7f99807cc7ea6 Mon Sep 17 00:00:00 2001 From: Neuro-HSOC <74316448+Neuro-HSOC@users.noreply.github.com> Date: Tue, 24 Aug 2021 09:40:38 +0200 Subject: [PATCH 5/9] Fix formating typo Fixed formating vs formatting typo. --- CHANGELOG.md | 2 +- docs/source/ruletypes.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dbe8372..da48ba5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ - None ## New features -- Added support for markdown style formating of aggregation tables - @Neuro-HSOC +- Added support for markdown style formatting of aggregation tables - @Neuro-HSOC ## Other changes - Fixed typo in default setting accidentally introduced in [#407](https://github.com/jertel/elastalert2/pull/407) - [#413](https://github.com/jertel/elastalert2/pull/413) - @perceptron01 diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 8a1f0ec7..f08b3feb 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -387,7 +387,7 @@ This should result in 2 alerts: One containing alice's two events, sent at ``201 For aggregations, there can sometimes be a large number of documents present in the viewing medium (email, Jira ticket, etc..). If you set the ``summary_table_fields`` field, ElastAlert 2 will provide a summary of the specified fields from all the results. -The formating style of the summary table can be switched between ``ascii`` (default) and ``markdown`` with parameter ``summary_table_type``. ``markdown`` might be the more suitable formating for alerters supporting it like TheHive. +The formatting style of the summary table can be switched between ``ascii`` (default) and ``markdown`` with parameter ``summary_table_type``. ``markdown`` might be the more suitable formatting for alerters supporting it like TheHive. For example, if you wish to summarize the usernames and event_types that appear in the documents so that you can see the most relevant fields at a quick glance, you can set:: @@ -719,12 +719,12 @@ summary_table_type summary_prefix ^^^^^^^^^^^^^^^^^^^^ -``summary_prefix``: Specify a prefix string, which will be added in front of the aggregation summary table. This string is currently not subject to any formating. +``summary_prefix``: Specify a prefix string, which will be added in front of the aggregation summary table. This string is currently not subject to any formatting. summary_suffix ^^^^^^^^^^^^^^^^^^^^ -``summary_suffix``: Specify a suffix string, which will be added after the aggregation summary table. This string is currently not subject to any formating. +``summary_suffix``: Specify a suffix string, which will be added after the aggregation summary table. This string is currently not subject to any formatting. timestamp_type ^^^^^^^^^^^^^^ From cc4a101aead9ec94a6d521a41ac573ff5739f9ed Mon Sep 17 00:00:00 2001 From: Neuro-HSOC <74316448+Neuro-HSOC@users.noreply.github.com> Date: Wed, 25 Aug 2021 10:48:50 +0200 Subject: [PATCH 6/9] Add new aggregation summary table unit test Added unit tests to check the new markdown style aggregation summary table, the old and now default text table and the summary table prefix and suffix values. --- tests/alerts_test.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 2c1e25ab..bca857b1 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -261,6 +261,83 @@ def test_alert_get_aggregation_summary_text__maximum_width(): assert 80 == alert.get_aggregation_summary_text__maximum_width() +def test_alert_aggregation_summary_markdown_table(): + rule = { + 'name': 'test_rule', + 'type': mock_rule(), + 'owner': 'the_owner', + 'priority': 2, + 'alert_subject': 'A very long subject', + 'aggregation': 1, + 'summary_table_fields': ['field','abc'], + 'summary_table_type': 'markdown' + } + matches = [ + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + ] + alert = Alerter(rule) + summary_table = str(alert.get_aggregation_summary_text(matches)) + assert "| field | abc | count |" in summary_table + assert "|-----|-----|-----|" in summary_table + assert "| field_value | abc from match | 3 |" in summary_table + assert "| field_value | cde from match | 2 |" in summary_table + + +def test_alert_aggregation_summary_default_table(): + rule = { + 'name': 'test_rule', + 'type': mock_rule(), + 'owner': 'the_owner', + 'priority': 2, + 'alert_subject': 'A very long subject', + 'aggregation': 1, + 'summary_table_fields': ['field','abc'], + } + matches = [ + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + ] + alert = Alerter(rule) + summary_table = str(alert.get_aggregation_summary_text(matches)) + assert "+-------------+----------------+-------+" in summary_table + assert "| field | abc | count |" in summary_table + assert "+=============+================+=======+" in summary_table + assert "| field_value | abc from match | 3 |" in summary_table + assert "| field_value | cde from match | 2 |" in summary_table + + +def test_alert_aggregation_summary_table_suffix_prefix(): + rule = { + 'name': 'test_rule', + 'type': mock_rule(), + 'owner': 'the_owner', + 'priority': 2, + 'alert_subject': 'A very long subject', + 'aggregation': 1, + 'summary_table_fields': ['field','abc'], + 'summary_prefix': 'This is the prefix', + 'summary_suffix': 'This is the suffix', + } + matches = [ + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + ] + alert = Alerter(rule) + summary_table = str(alert.get_aggregation_summary_text(matches)) + assert "This is the prefix" in summary_table + assert "This is the suffix" in summary_table + + def test_alert_subject_size_limit_with_args(ea): rule = { 'name': 'test_rule', From b8fe009775665ca705d9f91a5ded3aba94279d63 Mon Sep 17 00:00:00 2001 From: Neuro-HSOC <74316448+Neuro-HSOC@users.noreply.github.com> Date: Wed, 25 Aug 2021 11:04:58 +0200 Subject: [PATCH 7/9] Ensure linebreak after summary_prefix Added a newline after a non-empty summary_prefix to ensure there is a linebreak between the set summary_prefix and the hardcoded 'Aggregation resulted in the following data...' table header. --- elastalert/alerts.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 7438ef39..31ad2597 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -241,6 +241,11 @@ def get_aggregation_summary_text(self, matches): #Type independent prefix text = self.rule.get('summary_prefix', '') + # If a prefix is set, ensure there is a newline between it and the hardcoded + # 'Aggregation resulted in...' header below + if text != '': + text += "\n" + summary_table_fields = self.rule['summary_table_fields'] if not isinstance(summary_table_fields, list): summary_table_fields = [summary_table_fields] @@ -285,7 +290,7 @@ def get_aggregation_summary_text(self, matches): for key in keys: markdown_row += '| ' + str(key) + ' ' text += markdown_row + '| ' + str(count) + ' |\n' - text += '\n\n' + text += '\n' # Type independent suffix text += self.rule.get('summary_suffix', '') From 8357735497801663316d5beacb48e344bf8cf0a1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 25 Aug 2021 07:18:33 -0400 Subject: [PATCH 8/9] Update change URL for #415 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da48ba5a..f6ffd3a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ - None ## New features -- Added support for markdown style formatting of aggregation tables - @Neuro-HSOC +- Added support for markdown style formatting of aggregation tables - [#415](https://github.com/jertel/elastalert2/pull/415) - @Neuro-HSOC ## Other changes - Fixed typo in default setting accidentally introduced in [#407](https://github.com/jertel/elastalert2/pull/407) - [#413](https://github.com/jertel/elastalert2/pull/413) - @perceptron01 From 8b3ba221b2efd180bfb99e0463d7dd9264cccc4a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 25 Aug 2021 07:38:55 -0400 Subject: [PATCH 9/9] Correct linting format issues --- tests/alerts_test.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index bca857b1..3ea788d5 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -269,15 +269,15 @@ def test_alert_aggregation_summary_markdown_table(): 'priority': 2, 'alert_subject': 'A very long subject', 'aggregation': 1, - 'summary_table_fields': ['field','abc'], + 'summary_table_fields': ['field', 'abc'], 'summary_table_type': 'markdown' } matches = [ - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, ] alert = Alerter(rule) summary_table = str(alert.get_aggregation_summary_text(matches)) @@ -295,14 +295,14 @@ def test_alert_aggregation_summary_default_table(): 'priority': 2, 'alert_subject': 'A very long subject', 'aggregation': 1, - 'summary_table_fields': ['field','abc'], + 'summary_table_fields': ['field', 'abc'], } matches = [ - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, ] alert = Alerter(rule) summary_table = str(alert.get_aggregation_summary_text(matches)) @@ -321,16 +321,16 @@ def test_alert_aggregation_summary_table_suffix_prefix(): 'priority': 2, 'alert_subject': 'A very long subject', 'aggregation': 1, - 'summary_table_fields': ['field','abc'], + 'summary_table_fields': ['field', 'abc'], 'summary_prefix': 'This is the prefix', 'summary_suffix': 'This is the suffix', } matches = [ - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, - { '@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'abc from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, + {'@timestamp': '2016-01-01', 'field': 'field_value', 'abc': 'cde from match', }, ] alert = Alerter(rule) summary_table = str(alert.get_aggregation_summary_text(matches))