Migrate fileset to ECS (#8879)

The following fields were migrated to ECS: * fileset.name -> event.dataset * fileset.module -> event.module Changes: * Update generated files * Update tests * Update changelog * Update migration file
elastic · Nov 6, 2018 · 6535112 · 6535112
1 parent 74779e4
commit 6535112
Show file tree

Hide file tree

Showing 41 changed files with 464 additions and 476 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -20,6 +20,8 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Use `initial_scan` action for new paths. {pull}7954[7954]
 
 *Filebeat*
+- Rename `fileset.name` to `event.name`.
+- Rename `fileset.module` to `event.module`.
 
 - Remove the deprecated `prospector(s)` option in the configuration use `input(s)` instead. {pull}8909[8909]
 - Rename `offset` to `log.offset`.

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -11,7 +11,17 @@
 #   # Copy to is useful for fields where multiple fields map to the same ECS field
 #   copy_to: true-if-field-should-be-copied-to-target-in-6x
 
- - from: offset
-   to: log.offset
-   alias: true
-   copy_to: false
+- from: offset
+  to: log.offset
+  alias: true
+  copy_to: false
+
+- from: fileset.name
+  to: event.dataset
+  alias: true
+  copy_to: false
+
+- from: fileset.module
+  to: event.module
+  alias: true
+  copy_to: false
diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml
@@ -41,14 +41,6 @@
         the original `@timestamp` (representing the time when the log line was read) in this
         field.
 
-    - name: fileset.module
-      description: >
-        The Filebeat module that generated this event.
-
-    - name: fileset.name
-      description: >
-        The Filebeat fileset that generated this event.
-
     - name: syslog.facility
       type: long
       required: false

diff --git a/filebeat/channel/factory.go b/filebeat/channel/factory.go
@@ -103,10 +103,10 @@ func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *c
 
 	fields := common.MapStr{}
 	setMeta(fields, "module", config.Module)
-	setMeta(fields, "name", config.Fileset)
+	setMeta(fields, "dataset", config.Fileset)
 	if len(fields) > 0 {
 		fields = common.MapStr{
-			"fileset": fields,
+			"event": fields,
 		}
 	}
 	if config.Type != "" {

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -4687,22 +4687,6 @@ The input type from which the event was generated. This field is set to the valu
 In case the ingest pipeline parses the timestamp from the log contents, it stores the original `@timestamp` (representing the time when the log line was read) in this field.
 
 
---
-
-*`fileset.module`*::
-+
---
-The Filebeat module that generated this event.
-
-
---
-
-*`fileset.name`*::
-+
---
-The Filebeat fileset that generated this event.
-
-
 --
 
 *`syslog.facility`*::

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/apache2/access/test/test.log-expected.json b/filebeat/module/apache2/access/test/test.log-expected.json
@@ -8,8 +8,8 @@
         "apache2.access.response_code": "404", 
         "apache2.access.url": "/favicon.ico", 
         "apache2.access.user_name": "-", 
-        "fileset.module": "apache2", 
-        "fileset.name": "access", 
+        "event.dataset": "access", 
+        "event.module": "apache2", 
         "input.type": "log", 
         "log.offset": 0
     }, 
@@ -32,8 +32,8 @@
         "apache2.access.user_agent.os_minor": "12", 
         "apache2.access.user_agent.os_name": "Mac OS X", 
         "apache2.access.user_name": "-", 
-        "fileset.module": "apache2", 
-        "fileset.name": "access", 
+        "event.dataset": "access", 
+        "event.module": "apache2", 
         "input.type": "log", 
         "log.offset": 73
     }, 
@@ -42,8 +42,8 @@
         "apache2.access.remote_ip": "::1", 
         "apache2.access.response_code": "408", 
         "apache2.access.user_name": "-", 
-        "fileset.module": "apache2", 
-        "fileset.name": "access", 
+        "event.dataset": "access", 
+        "event.module": "apache2", 
         "input.type": "log", 
         "log.offset": 238
     }, 
@@ -65,8 +65,8 @@
         "apache2.access.user_agent.os_name": "Windows 7", 
         "apache2.access.user_agent.patch": "a2", 
         "apache2.access.user_name": "-", 
-        "fileset.module": "apache2", 
-        "fileset.name": "access", 
+        "event.dataset": "access", 
+        "event.module": "apache2", 
         "input.type": "log", 
         "log.offset": 285
     }

diff --git a/filebeat/module/apache2/error/test/test.log-expected.json b/filebeat/module/apache2/error/test/test.log-expected.json
@@ -4,8 +4,8 @@
         "apache2.error.client": "192.168.33.1", 
         "apache2.error.level": "error", 
         "apache2.error.message": "File does not exist: /var/www/favicon.ico", 
-        "fileset.module": "apache2", 
-        "fileset.name": "error", 
+        "event.dataset": "error", 
+        "event.module": "apache2", 
         "input.type": "log", 
         "log.offset": 0
     }, 
@@ -15,8 +15,8 @@
         "apache2.error.message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", 
         "apache2.error.module": "core", 
         "apache2.error.pid": "11379", 
-        "fileset.module": "apache2", 
-        "fileset.name": "error", 
+        "event.dataset": "error", 
+        "event.module": "apache2", 
         "input.type": "log", 
         "log.offset": 99
     }, 
@@ -28,8 +28,8 @@
         "apache2.error.module": "core", 
         "apache2.error.pid": "35708", 
         "apache2.error.tid": "4328636416", 
-        "fileset.module": "apache2", 
-        "fileset.name": "error", 
+        "event.dataset": "error", 
+        "event.module": "apache2", 
         "input.type": "log", 
         "log.offset": 229
     }

diff --git a/filebeat/module/auditd/log/test/test.log-expected.json b/filebeat/module/auditd/log/test/test.log-expected.json
@@ -11,8 +11,8 @@
         "auditd.log.ses": "4294967295", 
         "auditd.log.src": "192.168.2.0", 
         "auditd.log.src_prefixlen": "24", 
-        "fileset.module": "auditd", 
-        "fileset.name": "log", 
+        "event.dataset": "log", 
+        "event.module": "auditd", 
         "input.type": "log", 
         "log.offset": 0
     }, 
@@ -44,8 +44,8 @@
         "auditd.log.syscall": "44", 
         "auditd.log.tty": "(none)", 
         "auditd.log.uid": "0", 
-        "fileset.module": "auditd", 
-        "fileset.name": "log", 
+        "event.dataset": "log", 
+        "event.module": "auditd", 
         "input.type": "log", 
         "log.offset": 174
     }

diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json
@@ -6,8 +6,8 @@
         "elasticsearch.audit.origin_address": "147.107.128.77", 
         "elasticsearch.audit.principal": "i030648", 
         "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
+        "event.dataset": "audit", 
+        "event.module": "elasticsearch", 
         "input.type": "log", 
         "log.offset": 0, 
         "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed]        origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", 
@@ -21,8 +21,8 @@
         "elasticsearch.audit.principal": "rado", 
         "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
         "elasticsearch.node.name": "v_VJhjV", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
+        "event.dataset": "audit", 
+        "event.module": "elasticsearch", 
         "input.type": "log", 
         "log.offset": 155, 
         "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", 
@@ -37,8 +37,8 @@
         "elasticsearch.audit.origin_type": "local_node", 
         "elasticsearch.audit.principal": "_xpack_security", 
         "elasticsearch.audit.request": "ClearScrollRequest", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
+        "event.dataset": "audit", 
+        "event.module": "elasticsearch", 
         "input.type": "log", 
         "log.offset": 306, 
         "message": "[2018-06-19T05:00:15,778] [transport] [access_granted]  origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", 
@@ -51,8 +51,8 @@
         "elasticsearch.audit.origin_address": "172.22.0.3", 
         "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
         "elasticsearch.node.name": "v_VJhjV", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
+        "event.dataset": "audit", 
+        "event.module": "elasticsearch", 
         "input.type": "log", 
         "log.offset": 519, 
         "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", 
@@ -65,8 +65,8 @@
         "elasticsearch.audit.origin_address": "147.107.128.77", 
         "elasticsearch.audit.principal": "N078801", 
         "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
+        "event.dataset": "audit", 
+        "event.module": "elasticsearch", 
         "input.type": "log", 
         "log.offset": 654, 
         "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", 
@@ -81,8 +81,8 @@
         "elasticsearch.audit.origin_type": "rest", 
         "elasticsearch.audit.principal": "_anonymous", 
         "elasticsearch.audit.request": "MainRequest", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
+        "event.dataset": "audit", 
+        "event.module": "elasticsearch", 
         "input.type": "log", 
         "log.offset": 802, 
         "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", 
@@ -97,8 +97,8 @@
         "elasticsearch.audit.request_body": "body", 
         "elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", 
         "elasticsearch.node.name": "v_VJhjV", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
+        "event.dataset": "audit", 
+        "event.module": "elasticsearch", 
         "input.type": "log", 
         "log.offset": 986, 
         "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", 

diff --git a/filebeat/module/elasticsearch/gc/test/test.log-expected.json b/filebeat/module/elasticsearch/gc/test/test.log-expected.json
@@ -11,8 +11,8 @@
         "elasticsearch.gc.phase.cpu_time.user_sec": "0.01", 
         "elasticsearch.gc.phase.duration_sec": "0.0021716", 
         "elasticsearch.gc.phase.name": "CMS Initial Mark", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "gc", 
+        "event.dataset": "gc", 
+        "event.module": "elasticsearch", 
         "input.type": "log", 
         "log.offset": 0, 
         "message": "2018-03-03T19:37:06.157+0500: 14597.826: [GC (CMS Initial Mark) [1 CMS-initial-mark: 131804K(174784K)] 142444K(253440K), 0.0021716 secs] [Times: user=0.01 sys=0.00, real=0.00 secs]", 
@@ -23,8 +23,8 @@
         "elasticsearch.gc.jvm_runtime_sec": "1396138.752", 
         "elasticsearch.gc.stopping_threads_time_sec": "0.0000702", 
         "elasticsearch.gc.threads_total_stop_time_sec": "0.0083760", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "gc", 
+        "event.dataset": "gc", 
+        "event.module": "elasticsearch", 
         "input.type": "log", 
         "log.offset": 181, 
         "message": "2018-06-11T01:53:11.382+0000: 1396138.752: Total time for which application threads were stopped: 0.0083760 seconds, Stopping threads took: 0.0000702 seconds", 
@@ -49,8 +49,8 @@
         "elasticsearch.gc.phase.weak_refs_processing_time_sec": "0.0003647", 
         "elasticsearch.gc.young_gen.size_kb": "157248", 
         "elasticsearch.gc.young_gen.used_kb": "113198", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "gc", 
+        "event.dataset": "gc", 
+        "event.module": "elasticsearch", 
         "input.type": "log", 
         "log.offset": 339, 
         "message": "2018-06-30T16:35:26.632+0500: 224.671: [GC (CMS Final Remark) [YG occupancy: 113198 K (157248 K)]224.671: [Rescan (parallel) , 0.0148273 secs]224.686: [weak refs processing, 0.0003647 secs]224.687: [class unloading, 0.0188407 secs]224.705: [scrub symbol table, 0.0100207 secs]224.715: [scrub string table, 0.0005253 secs][1 CMS-remark: 277821K(349568K)] 391020K(506816K), 0.0457689 secs] [Times: user=0.12 sys=0.00, real=0.04 secs]",