[Filebeat] Explicitly set ECS version in Filebeat modules. (elastic#1…

…9198)

* Explicitly set ECS version in Filebeat modules.

- Add test to check if ecs.version is set
- add_fields to azure/activitylogs
- add_fields to azure/auditlogs
- add_fields to azure/signinlogs
- add_fields to checkpoint/firewall
- add_fields to crowdstrike/falcon
- add_fields to fortinet/firewall
- add_fields to traefik/access

Relates elastic#17688

CHANGELOG.next.asciidoc

@@ -401,6 +401,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add geoip AS lookup & improve ECS categorization in aws cloudtrail fileset. {issue}18644[18644] {pull}18958[18958]
 - Improved performance of PANW sample dashboards. {issue}19031[19031] {pull}19032[19032]
 - Add support for v1 consumer API in Cloud Foundry input, use it by default. {pull}19125[19125]
+- Explicitly set ECS version in all Filebeat modules. {pull}19198[19198]
 
 *Heartbeat*
 

filebeat/module/traefik/access/config/traefik-access.yml

@@ -4,3 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/tests/system/test_modules.py

@@ -109,6 +109,8 @@ def test_fileset_file(self, module, fileset, test_file):
     def run_on_file(self, module, fileset, test_file, cfgfile):
         print("Testing {}/{} on {}".format(module, fileset, test_file))
 
+        self.assert_explicit_ecs_version_set(module, fileset)
+
         try:
             self.es.indices.delete(index=self.index_name)
         except:

libbeat/tests/system/beat/beat.py

@@ -11,6 +11,7 @@
 import yaml
 import hashlib
 import re
+import glob
 from datetime import datetime, timedelta
 
 from .compose import ComposeMixin
@@ -722,3 +723,33 @@ def get_beat_version(self):
         proc.wait()
 
         return self.get_log_lines(logfile="version")[0].split()[2]
+
+    def assert_explicit_ecs_version_set(self, module, fileset):
+        """
+        Assert that the module explicitly sets the ECS version field.
+        """
+        def get_config_paths(modules_path, module, fileset):
+            pathname = os.path.abspath(modules_path +
+                                       "/" +
+                                       module +
+                                       "/" +
+                                       fileset +
+                                       "/" +
+                                       "config/*.yml")
+            return glob.glob(pathname)
+
+        def is_ecs_version_set(path):
+            # parsing the yml file would be better but go templates in
+            # the file make that difficult
+            with open(path) as fhandle:
+                for line in fhandle:
+                    if re.search("ecs\.version", line):
+                        return True
+            return False
+
+        errors = []
+        for cfg_path in get_config_paths(self.modules_path, module, fileset):
+            if not is_ecs_version_set(cfg_path):
+                errors.append("{}".format(cfg_path))
+        if len(errors) > 0:
+            raise Exception("{}/{} ecs.version not explicitly set in:\n{}".format(module, fileset, '\n'.join(errors)))

x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml

@@ -8,3 +8,8 @@ resource_manager_endpoint: {{ .resource_manager_endpoint }}
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

x-pack/filebeat/module/azure/activitylogs/config/file.yml

@@ -6,3 +6,9 @@ paths:
 exclude_files: [".gz$"]
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml

@@ -7,3 +7,8 @@ storage_account_key: {{ .storage_account_key }}
 resource_manager_endpoint: {{ .resource_manager_endpoint }}
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

x-pack/filebeat/module/azure/auditlogs/config/file.yml

@@ -6,3 +6,8 @@ paths:
 exclude_files: [".gz$"]
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml

@@ -7,3 +7,8 @@ storage_account_key: {{ .storage_account_key }}
 resource_manager_endpoint: {{ .resource_manager_endpoint }}
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

x-pack/filebeat/module/azure/signinlogs/config/file.yml

@@ -6,3 +6,8 @@ paths:
 exclude_files: [".gz$"]
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml

@@ -20,3 +20,7 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 processors:
   - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml

@@ -20,3 +20,7 @@ processors:
     lang: javascript
     id: crowdstrike_falcon
     file: ${path.home}/module/crowdstrike/falcon/config/pipeline.js
+- add_fields:
+    target: ''
+    fields:
+      ecs.version: 1.5.0

x-pack/filebeat/module/fortinet/firewall/config/firewall.yml

@@ -25,3 +25,7 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 processors:
   - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0