add ecs-migration.yml

_meta/ecs-migration.yml

@@ -0,0 +1,115 @@
+# The ECS migration file contains the information about all the fields which are migrated to ECS in 7.0.
+# The goal of the file is to potentially have scripts on top of this information to convert visualisations and templates
+# based on this information in an automated way and to keep track of all changes which were applied.
+#
+# The format of the file is as following:
+#
+# - from: source-field-in-6.x
+#   to: target-filed-in-ECS
+#   # Alias field is useful for fields where there is a 1-1 mapping from old to new
+#   alias: true-if-alias-is-required-in-6x (default is true)
+#   # Copy to is useful for fields where multiple fields map to the same ECS field
+#   copy_to: true-if-field-should-be-copied-to-target-in-6x (default is false)
+
+- from: context.service.agent.name
+  to: agent.name
+
+- from: context.service.agent.version
+  to: agent.version
+
+- from: context.system.architecture
+  to: host.architecture
+
+- from: context.system.ip
+  to: host.ip
+
+- from: context.system.hostname
+  to: host.name
+
+- from: context.system.platform
+  to: host.os.platform
+
+- from: context.request.method
+  to: http.method
+
+- from: context.request.http_version
+  to: http.version
+
+- from: context.tags
+  to: labels
+  alias: false
+  copy_to: true
+
+- from: context.process.pid
+  to: process.pid
+
+- from: context.process.ppid
+  to: process.ppid
+
+- from: context.process.title
+  to: process.title
+
+  # not in ECS
+- from: context.service.environment
+  to: service.environment
+
+  # not in ECS
+- from: context.service.framework.name
+  to: service.framework.name
+
+  # not in ECS
+- from: context.service.framework.version
+  to: service.framework.version
+
+  # not in ECS
+- from: context.service.language.name
+  to: service.language.name
+
+  # not in ECS
+- from: context.service.language.version
+  to: service.language.version
+
+- from: context.service.name
+  to: service.name
+
+  # not in ECS
+- from: context.service.runtime.name
+  to: service.runtime.name
+
+  # not in ECS
+- from: context.service.runtime.version
+  to: service.runtime.version
+
+- from: context.request.url.full
+  to: url.original
+
+- from: context.request.url.hash
+  to: url.fragment
+
+- from: context.request.url.hostname
+  to: url.domain
+
+- from: context.request.url.pathname
+  to: url.path
+
+- from: context.request.url.port
+  to: url.port
+  alias: false
+  copy_to: true
+
+- from: context.request.url.search
+  to: url.query
+
+- from: context.request.url.protocol
+  to: url.scheme
+  alias: false
+  copy_to: true
+
+- from: context.user.email
+  to: user.email
+
+- from: context.user.id
+  to: user.id
+
+- from: context.user.username
+  to: user.name

_meta/fields.common.yml

@@ -446,19 +446,9 @@
       type: group
       dynamic: false
       fields:
-        - name: href
-          type: group
-          fields:
-            - name: original
-              type: alias
-              path: context.request.url.raw
-
-        - name: host
-          type: group
-          fields:
-            - name: name
-              type: alias
-              path: context.request.url.hostname
+        - name: domain
+          type: alias
+          path: context.request.url.hostname
 
         - name: fragment
           type: alias
@@ -469,12 +459,8 @@
           path: context.request.url.full
 
         - name: path
-          type: group
-          fields:
-            - name: original
-              type: alias
-              path: context.request.url.pathname
-              # TODO: multifield original.text
+          type: alias
+          path: context.request.url.pathname
 
         # context.request.url.port keyword -> long
         - name: port
@@ -483,12 +469,9 @@
             The port of the request, e.g. 443.
 
         - name: query
-          type: group
-          fields:
-            - name: query
-              type: alias
-              path: context.request.url.search
-              # TODO: multifield original.text
+          type: alias
+          path: context.request.url.search
+          # TODO: multifield original.text
 
         # context.request.url.protocol minus the ":"
         - name: scheme

docs/fields.asciidoc

@@ -685,15 +685,6 @@ type: alias
 --
 
 
-
-*`context.request.url.raw`*::
-+
---
-type: alias
-
---
-
-
 *`context.request.url.hostname`*::
 +
 --
@@ -715,7 +706,6 @@ type: alias
 
 --
 
-
 *`context.request.url.pathname`*::
 +
 --
@@ -733,7 +723,6 @@ The port of the request, e.g. 443.
 
 --
 
-
 *`context.request.url.search`*::
 +
 --

tests/system/test_ecs_mappings.py

@@ -34,10 +34,9 @@ def test_ecs_migration(self):
         all_fields = set()
         alias_fields = set()
         for f, a in flatmap(yaml.load(self.command_output)["mappings"]["doc"]["properties"]):
+            all_fields.add(f)
             if a.get("type") == "alias":
                 alias_fields.add(a["path"])
-            else:
-                all_fields.add(f)
 
         # fields with special exception, due to mapping type changes, etc
         # no comment means unchanged
@@ -68,6 +67,16 @@ def test_ecs_migration(self):
         should_not_be_aliased = alias_fields - all_fields
         self.assertFalse(should_not_be_aliased, json.dumps(sorted(should_not_be_aliased)))
 
+        # check the migration log too
+        with open(self._beat_path_join("_meta", "ecs-migration.yml")) as f:
+            for m in yaml.load(f):
+                if m.get("alias", True):
+                    self.assertIn(m["from"], alias_fields)
+                elif m.get("copy_to", False):
+                    self.assertIn(m["from"], all_fields)
+                self.assertIn(m["to"], all_fields)
+
+        # check that all fields are accounted for
         not_aliased = all_fields - alias_fields - exception_fields
         self.assertFalse(not_aliased,
                          "\nall fields ({:d}):\n{}\n\naliased ({:d}):\n{}\n\nunaccounted for ({:d}):\n{}".format(