diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b587467fcd6..c76fd55cda0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -401,6 +401,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add geoip AS lookup & improve ECS categorization in aws cloudtrail fileset. {issue}18644[18644] {pull}18958[18958] - Improved performance of PANW sample dashboards. {issue}19031[19031] {pull}19032[19032] - Add support for v1 consumer API in Cloud Foundry input, use it by default. {pull}19125[19125] +- Explicitly set ECS version in all Filebeat modules. {pull}19198[19198] *Heartbeat* diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml index 0afd17317d4..183de629867 100644 --- a/filebeat/module/traefik/access/config/traefik-access.yml +++ b/filebeat/module/traefik/access/config/traefik-access.yml @@ -4,3 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 81dde391c11..c62bad7a213 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -109,6 +109,8 @@ def test_fileset_file(self, module, fileset, test_file): def run_on_file(self, module, fileset, test_file, cfgfile): print("Testing {}/{} on {}".format(module, fileset, test_file)) + self.assert_explicit_ecs_version_set(module, fileset) + try: self.es.indices.delete(index=self.index_name) except: diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 13e81746dcf..12531156aab 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -11,6 +11,7 @@ import yaml import hashlib import re +import glob from datetime import datetime, timedelta from .compose import ComposeMixin @@ -722,3 +723,33 @@ def get_beat_version(self): proc.wait() return self.get_log_lines(logfile="version")[0].split()[2] + + def assert_explicit_ecs_version_set(self, module, fileset): + """ + Assert that the module explicitly sets the ECS version field. + """ + def get_config_paths(modules_path, module, fileset): + pathname = os.path.abspath(modules_path + + "/" + + module + + "/" + + fileset + + "/" + + "config/*.yml") + return glob.glob(pathname) + + def is_ecs_version_set(path): + # parsing the yml file would be better but go templates in + # the file make that difficult + with open(path) as fhandle: + for line in fhandle: + if re.search("ecs\.version", line): + return True + return False + + errors = [] + for cfg_path in get_config_paths(self.modules_path, module, fileset): + if not is_ecs_version_set(cfg_path): + errors.append("{}".format(cfg_path)) + if len(errors) > 0: + raise Exception("{}/{} ecs.version not explicitly set in:\n{}".format(module, fileset, '\n'.join(errors))) diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index 9f24f85e3eb..8c4c42d60cf 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -8,3 +8,8 @@ resource_manager_endpoint: {{ .resource_manager_endpoint }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/activitylogs/config/file.yml b/x-pack/filebeat/module/azure/activitylogs/config/file.yml index 456cc5dce7c..498a7906457 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/file.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/file.yml @@ -6,3 +6,9 @@ paths: exclude_files: [".gz$"] tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index f8b88d18a4a..239e7ef2249 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -7,3 +7,8 @@ storage_account_key: {{ .storage_account_key }} resource_manager_endpoint: {{ .resource_manager_endpoint }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/auditlogs/config/file.yml b/x-pack/filebeat/module/azure/auditlogs/config/file.yml index 456cc5dce7c..937446eb523 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/file.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/file.yml @@ -6,3 +6,8 @@ paths: exclude_files: [".gz$"] tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index f8b88d18a4a..239e7ef2249 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -7,3 +7,8 @@ storage_account_key: {{ .storage_account_key }} resource_manager_endpoint: {{ .resource_manager_endpoint }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/azure/signinlogs/config/file.yml b/x-pack/filebeat/module/azure/signinlogs/config/file.yml index 456cc5dce7c..937446eb523 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/file.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/file.yml @@ -6,3 +6,8 @@ paths: exclude_files: [".gz$"] tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml index 637a2899315..12440f8fffe 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml @@ -20,3 +20,7 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - add_locale: ~ + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml index 9f5de43be3b..689bd725530 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml +++ b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml @@ -20,3 +20,7 @@ processors: lang: javascript id: crowdstrike_falcon file: ${path.home}/module/crowdstrike/falcon/config/pipeline.js +- add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml index a85eb4923df..6af16945317 100644 --- a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml @@ -25,3 +25,7 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - add_locale: ~ + - add_fields: + target: '' + fields: + ecs.version: 1.5.0