From f4ad774b80183d75270370e58362fc312a178fe6 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Mon, 12 Nov 2018 16:36:35 -0500 Subject: [PATCH] add ecs-migration.yml --- _meta/ecs-migration.yml | 115 ++++++++++++++++++++++++++++++ _meta/fields.common.yml | 33 +++------ docs/fields.asciidoc | 11 --- include/fields.go | 2 +- tests/system/test_ecs_mappings.py | 13 +++- 5 files changed, 135 insertions(+), 39 deletions(-) create mode 100644 _meta/ecs-migration.yml diff --git a/_meta/ecs-migration.yml b/_meta/ecs-migration.yml new file mode 100644 index 00000000000..2ba4eccf029 --- /dev/null +++ b/_meta/ecs-migration.yml @@ -0,0 +1,115 @@ +# The ECS migration file contains the information about all the fields which are migrated to ECS in 7.0. +# The goal of the file is to potentially have scripts on top of this information to convert visualisations and templates +# based on this information in an automated way and to keep track of all changes which were applied. +# +# The format of the file is as following: +# +# - from: source-field-in-6.x +# to: target-filed-in-ECS +# # Alias field is useful for fields where there is a 1-1 mapping from old to new +# alias: true-if-alias-is-required-in-6x (default is true) +# # Copy to is useful for fields where multiple fields map to the same ECS field +# copy_to: true-if-field-should-be-copied-to-target-in-6x (default is false) + +- from: context.service.agent.name + to: agent.name + +- from: context.service.agent.version + to: agent.version + +- from: context.system.architecture + to: host.architecture + +- from: context.system.ip + to: host.ip + +- from: context.system.hostname + to: host.name + +- from: context.system.platform + to: host.os.platform + +- from: context.request.method + to: http.method + +- from: context.request.http_version + to: http.version + +- from: context.tags + to: labels + alias: false + copy_to: true + +- from: context.process.pid + to: process.pid + +- from: context.process.ppid + to: process.ppid + +- from: context.process.title + to: process.title + + # not in ECS +- from: context.service.environment + to: service.environment + + # not in ECS +- from: context.service.framework.name + to: service.framework.name + + # not in ECS +- from: context.service.framework.version + to: service.framework.version + + # not in ECS +- from: context.service.language.name + to: service.language.name + + # not in ECS +- from: context.service.language.version + to: service.language.version + +- from: context.service.name + to: service.name + + # not in ECS +- from: context.service.runtime.name + to: service.runtime.name + + # not in ECS +- from: context.service.runtime.version + to: service.runtime.version + +- from: context.request.url.full + to: url.original + +- from: context.request.url.hash + to: url.fragment + +- from: context.request.url.hostname + to: url.domain + +- from: context.request.url.pathname + to: url.path + +- from: context.request.url.port + to: url.port + alias: false + copy_to: true + +- from: context.request.url.search + to: url.query + +- from: context.request.url.protocol + to: url.scheme + alias: false + copy_to: true + +- from: context.user.email + to: user.email + +- from: context.user.id + to: user.id + +- from: context.user.username + to: user.name diff --git a/_meta/fields.common.yml b/_meta/fields.common.yml index 5c0824c5d76..c0c9315e521 100644 --- a/_meta/fields.common.yml +++ b/_meta/fields.common.yml @@ -446,19 +446,9 @@ type: group dynamic: false fields: - - name: href - type: group - fields: - - name: original - type: alias - path: context.request.url.raw - - - name: host - type: group - fields: - - name: name - type: alias - path: context.request.url.hostname + - name: domain + type: alias + path: context.request.url.hostname - name: fragment type: alias @@ -469,12 +459,8 @@ path: context.request.url.full - name: path - type: group - fields: - - name: original - type: alias - path: context.request.url.pathname - # TODO: multifield original.text + type: alias + path: context.request.url.pathname # context.request.url.port keyword -> long - name: port @@ -483,12 +469,9 @@ The port of the request, e.g. 443. - name: query - type: group - fields: - - name: query - type: alias - path: context.request.url.search - # TODO: multifield original.text + type: alias + path: context.request.url.search + # TODO: multifield original.text # context.request.url.protocol minus the ":" - name: scheme diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc index 22c6433feb3..6c907eea0fa 100644 --- a/docs/fields.asciidoc +++ b/docs/fields.asciidoc @@ -685,15 +685,6 @@ type: alias -- - -*`context.request.url.raw`*:: -+ --- -type: alias - --- - - *`context.request.url.hostname`*:: + -- @@ -715,7 +706,6 @@ type: alias -- - *`context.request.url.pathname`*:: + -- @@ -733,7 +723,6 @@ The port of the request, e.g. 443. -- - *`context.request.url.search`*:: + -- diff --git a/include/fields.go b/include/fields.go index 4b13efb8f50..13a97f6d279 100644 --- a/include/fields.go +++ b/include/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzcXf+T27ax/91/xb7LZM6eOdF3Z5+daqbp87PT1JO49sR2X2fajgKBKxE1CdAAeGel0//9Db6QBEmQkk665Pr0Q2JRxO5nF4vFYrHAzeATbuZAyuIBgGY6xzl8jxwlyeHFuzewYpin6gFAiopKVmom+PwB+OfmXwAz4KTAOeRMaeSMr+1TAL0pcW7o3wiZ+mchGfjWPwR4kaYSlQKdISiU1yiBqZYgCJ50WJVSUFRKyMR835Xfu7qVpZI8GCGJ18j1/jRtsx5RzQpUmhRlh9xaiqqMEPOPQt2GxCrVPKop5aLRtvlQUXE9h4vg0YjCzedDDQ7EymreSgCMQ8GoFAqp4KkCxThF+MjZF8BS0KwnIhVc4xe9k4Bhj/MNELlkWhK5qYlUJAfGV0IWxLwPEtdEpsYAGnhnQEmpK4kpLDf2MVnbx8KyIHm+MV15zdL2jUqhTGo4G04KRuewIrnCnsYHKtdkPVS6WP4TqQ4euweLmK10WGpZ4W5d8wJWOdFQkLK05r+yMsxSXDGOqYUFN0xnoLQ0L1yTvEKVDAXItC4HAoTdExW/ba800ZVaUJFiB2DU/CZFAvhgBrelB4ZebXUGIkhUpeAKIyIYyQ8RwbQPvERIZdhXO4hQ06vx52K9xtQMG2dnEQgsPQbz1ylyzVYM5R6ssSAsPwb37wyhfWQuI1x7D6fFfVdzMzzgJkOJoZtSIJEKmWJ6Zqgzasc+gRtcwlKKm2DQt93HlGkolprYgbSSorA0/zr7o5A3xFAz/4IMSYryzCC4yRjN7EsrJpUG5FpuDBXzqAUpJFszTnKgObMTQYS1HaaQiRs0M1zB1pkGLjQsETiaaYRIlm+Mn1DaiEUUMA2UcPPGSsi182kECpIzykSlxvVvHYb1jZF+CDz2Dj3xXqz0DZHe1QKh2vgcZlBlJF8ZDRDL7wxwnXS7AB7D2/dARbFk3Dr1yAiX+LlCpQ8a5DJm5H0CW+R8YXCWOWqEjzI/8x6WZljgGWRCaSA8hZLorNu7EVwd8chN75epQbgVJXg/JMnNGVS8JFJhCh9/+rG2RK/OM8Bkbb2rmj9+jF+IES2hopg/ffrksUIiafaHz7/HnCjNqPv+lRZlMiZHKYUWVOR3IUxNOyZDAidOipNRaKsqvxNYhu4ZlEIptjTOxdj/jCiFxTL/dZRuzG4wex1Jupr2iNID9OOaL4XUd2IQQuo4rqdPn4yjITq7K20Z2iOa8h07riX3+12gcpT9u0t0s9LnCuWmjg3jkHs2OA49I+pOgBu6PWxmPNX4tChPovOaGVuLa5SKCX6skM6GoJ5mH1OOxK1ABGgzh4fLvC6yAnUmjhLpNZgcyX0htfOqC6rv0wLAihVZBdRYXQhxenl+fhpV8opxpjKMqXkpRI6E7zPf+ybAeMoosWHNTYY6Q9kBBTdENZxBSBO0RfStNkpjsVXbE5je+kWsJ+U7Jpnuqq2TxdRo3TJWY9OEx9aG4kY7dSwOnQgvBGfcDNNIzeL9uABDygeCLHOiV0IWxwVYUz0QXG/1NLKo2oqoXbPYmFZnRHvmqoUUMW+fITvEvl8HyZ0SpVmFNY4MQVbcfvWMgiVfSjSxSqIiz5Hqzvjv58s6mFm/u0Yc11at/bkqUDLagHv9qulNlNeMYv3LmGUdH8oAwqmCkkhjUNNYbKL3eCb+vqsARz7mH917hxhQzWrcMcbkPa5PfF0UlSbLHKHi7HOF0HGOHiAWTGuXuWxn6C4dNxSGa1eAnCwxX2gsjOPAOZz86182dfDvf5/03hQl8kXO+KcF4wtaSdP5C02Wg2Sj+VQyIDoQawYF43VkNYeTq+Q8Oe/zMx8LZQ4nSfKYlOXjT2xJOPnqcUpUthREpo+fXiyv0t9dns+ef3N5Mbu4wOezb+jT57NnV8tvnl4tr+hq+eQPC/L7hzZOnbv/LVy4On9IOMk3v+DihuUpJTKd/5c+cy+e+jRx4pVsk//zry8vG/V8fXl5+ujRoyHqvnDPjHAzkpcZufhVZMwJX1dkjfO8oshxUqK/t/3995NTI07UqmNB8EGG/ZduBDxpylFEyK+ZFLzoJ52O4l0C4iPsax1HeQ9zQVsTN3exkvxz4CtKKdaSFIXRbY0dKoXp6GIs3uUHg+p1/K64mnVGxTUb8bD3UO0e7n3Q9ASUZsEjSYE3Qn76T1FvA/g+KHgSTLMyGWTJ77F6XQ7+Hqh2AKSJLiXhilDdQojsCk9tw/bl6QTOo7usExvdGYag4PWrTuAYDcMGIdhfGN7A+5JwFcYLO8df47FXLzSxYVdP+1ujkSf4JMXn5+ez5ymeu2hkeXFxNUtXv6O/O0/JZbq6uFXEFagtYen2WKsnTBhm3bVMIxFWT4IGfhtXBXZLEe6HyVqbbdd3DpoWwWakSxws0SwfFWjRL2dx68B7KY3HFkaSX8F3L993BAid8oHwey7ZUSM5I2EuoyQ6a6pZmnjconBFRgOyQze7P+WaRkf0TKgjST6Sc9sJp81RJR0KQ8MYFpfsRJWVQ1q36SVHrUl7DmiK7dmqyezRVDawD20UXkOl28ltZc6BnTzYd9hFdX4vIfGNj2TcNdXOJo0f4PU7tniJipJhCjbB95ASfqodZV9LpR51dGUnY9XRVqcIa6IAK1Z8NVaSNl105UCMl13F06QH9m0Zcb3TXVBn/kzLIbkD6EUJ9jOJ+1B0bTu662YIb6m7r2xNDePNlAJbUxT7OO6Qwm48Y0u4/Z3RaBJzB0fkoTdAejMa7JhVug2njhfYqqpICuc30VSN484V1TAa6OkI0UsX/KTihzmc30TvHsadq73mM6r1w4K7aFjXLjAPnBQyiauDOqquVbylEutZvpJ5IslNpOpY7FHNFwN4gCWF4MYDw5Uk61vMAR3iRGWRiHOo2n0J22K24dRNdHZfOn2kzOor+PD21ds5FFVu65TztGGa2IrTwBVFyQqp6/ANZt92d0ZHCs6ie6hbckG71pbVLG0m4SDd9yncVvHROrKD1V7XXhaMV64E4GR+MtCCK4QdqGHvTJyjM1XoedKLqIMTAAd6zn41/C5D01ZYu4YDentH05ZYLJC+xSxvaTWnGx74o1xLJLo9y/U/7lukS14KrgnjZilWFILbdl5vQK4Jy+0ON+NA8twnmwwou+ceO/1l2idmTv1F8C0LLZuP9W+CrXzRleTteR1RonTFWL5QRvAg8WVEAtaqxVeN9Gzm5L+bs1cnHdtJia7hGdtjEtPO4rBOBwfv+ULYObyo1pXScPlMZ3B5fvHsDC4u50+u5ldPkidPLncT2p24usmQB2m8XKx9CY6tclnbE3h6mFlvjyONrWGbI1V2mW1re/z5AYUaSpROf7aQHWUkV+/01GPcHAOE/RfgcaCN+dnDHVTwFVtX0tpcXdvRQYBSigknEGfynWlUGzV1HI1ZkTRlvtCO8ZUwVk6Jsj7J8mkKS8b8iEFwmCd00Jyq/ciluajSdui+NF/rs2wSCtQkJZrER/Mb/6s70UI7TZWRNzgvl6YL+8KiJhmcfXzQ298Kckwksa2Smmx/cCDdMgLCXawuwgTeuRJ79CkVIBINwTNYUzwDISFla6ZJLigSnoxiY1xpwiku2Bbze+1fDLLSthiuIDRjvG/+MQ6Bw97GI6wP2o2Lf2ER2FmjZ32ZFJiyqpjm/saRsCa2H3Pv/VnO9GYRePMGQaVmSJSeXdAtziggBNbZs9aRM+XgMDXiwbsmZ/1L06sNFP/L7MvupuebGCzfC7HO0Y20ce4S1617HGHwk31nm3x+oKeCfrLjx4/0V/X3CHH3m62YDqof3TB3v5kxqzIh9cJ50TYSIpxmQtb8Zs0oH5nBG1hxHzvmC71fdXHNQT7xoyunawgCS5MpdkUsUbQXx9AuLDnnnBoAZjJeVizXzQn0OJRo9LYXkpcNT27Ppo/z6iTD4fankieqdK0mHJ/GaP2a3pvsn9y3CJHXZkINDFXIiOtpbdM832qZkxtiY3YpVPIJJcdhvL/3kqUfkJ4qcKTrNFGjpU/V0vygUbW6+iF8FuHU/t5M8t0ZuyUKoaamB33baKt6O6D3U3Ip0iMYf6CBUqQQT9kYVtWhLibg9E6k8PH1q/hSTJUkUqh8W1YtxSEzkeJxNWgojqhwV9exGyO/IVaQyGYu4VxoW+d/NHYByTjPY7rjgC/teOYptkeYkKJ8Hd32qpZZvRjyDubFuzduPdH3L/bhTJVI2YpRd4rCOOQX796MuIJrhjd+DfRgXAjHYg4nqaB/i9Ywn/4jsZF8vfZuinOgJIzn7f7osPoqXnllRWlKr3Yqu4qXXG0pt9pelvR89WxF6PnsOX1GXFkSIVdXsyfL8/Tqkj6/oM/Of6Xi9t2KrY4u0RFK2TuremApMNorFpwyONsqsdMT4+vFJ9wc09xmMYz7Zh5emaEW3lRDuC+YlFhKVMhtOEG4TwAJaqzYX6tAoBCcaWGa1tqs2e11fc1Os6W/NOjp7uFQu2Z2PRHxiVVeSjbM0/funfCsL8dY/7HirmKTkjz3KysTkrtlHCuI3ECJskQtiRb+WpSJA7GhyRzmp7/3lH7ATe9OFGfSxstWyi54a6a3rTl1maJXqAnL72Pt6dXq/BvyzfPjO8ThMP9V60/3lWvELUakiNSghlaKXyiWOrb5fMuTlmQpKt25pCY3Cwspbng9ggPTnDpgPDgGvsfJng+1v3DnvhXqIAtun2ekLJFj6jeDTLiyJCpsNVLAX6BSY+d+BlfdRN1OFG1zpY8D4LmMYRBpdftDlYado9AuFgY3C/W9LYxnpLcxb5x+lFhGeJpj/Lhq7JT9bip97Q7ZC9k5Y+90a8/2kmqdaVCiQHfyl3GfjkmxPXQ/XNSI9SED5UV3L6AZM/UC3Lj15ga6vQZLjtcY323f0SAUXqNketPulVIhxw6x2OlHLkYrJ3bkyfs3m0moU95TR2juaAS2GzX5pjvBTg/GkkhSLKZA3eoo4gtHGLVkv2DaYIDvjLs6fSmqPLU1VlRwjlSDFvC1Ou2f+21uPCpR6k1NBZgCpVmeN9uSZ3aXTmWW7BIBP1ckr8/KdyQ8g2WlXWVqmROKmcjtvo9E+zUdInjN7TgDxXTl19QDqgaR6XLD0g4ovzAELdZ2/DZZr+biC78gPTEr0vdu7/aN0Ra1gUtEu/4lfyFI4d49g5x9Qnj57qPVQIGFkBuonKR2S5NI7CfDYxvHw9S4j4xiS9/O7R07B/o/u2Y/18t1Bbmg7QUeXqL+fuKDrrnSsnrQNc+uAxs1y59pWQ1YG73Z/YK220f3MYUmecKFLJKSDsN1RUmO6WKVC6Ij0WuJknZrqLYV3rgGxrbEyuK0px9VaY+ybOrtGXuJgLt21LohonTsBH1wpR5TYIQguR2XnhKviqW7K9FwokLaJFIKTIMkfI3dshtrROfG1i/Oz79O+l3kjPCWveQaDzrKG/Y+fTXool7lU901y43uyDfVMYauxzLcdyBUVxG2+0ywlkI7ijG1vbCSiNP3N7S1erjT3UNjsm+7BcjhM1xqkIw7Igm8tht4lOS0MqunFExAmoJwkcnb9wm85fAj49UXe1Oj4IoprdrTpw3NHtMyrwxZmnmbXFarFUplyb19/1dDjCkgoCp7aUwIzrxu96s4oZpd189t0/9lPBU36sy3tzNGf/oTtc9KfEND/OeBwfdve9nP4n3rwOTrcV1vcZzZUdl4/MDR93zmxM5j4DZvYZi7Os8p2xx1oFtc6JQT3eEWreM60mO70qEz7attMCZ26Lw3tk2bzTa9xOwNdraSqbnxpm1eSlyxL3M4+ZtV/z9OdupSxX65S3dj68Csy71mMvSMYZ9lpCNIU6SvVDJkd3x8P6FiqTGl96jhPfsFExeNFyZsN1YQgSworeyZLsahIOY/7p2HP7148ygJdy+UqCTFgpTdHYz3weMOwuYHWLEcFSCXjGaYuoA3qItyORzXp47JYsVyjbLp5xk0zJMQRjQeDH6HsdTr7jul/08uP+qvEetLYmzetVFZ51xyDNTR766pddNsyA86YFnxNLcWgdFa+r3yvz8K6vIEtRoaySXmxE7LfqFm+NVVzja/r7uDobS5lHAcuCfdIVASHt/HGyvHtRt5hvi+G23+ioPuUCjJLa9yGPRCZEvCWArspvZXWEqkNhB7+O3v4Vly9SiBF27qzjf1fVzBvWgliZlChl8W43sjO2Lp7oQYTmfAuJYirWiIb8da79GNmR3/moP9qx3GOFCxNW+Mk1hkdSpNUeGqAUlY+RuBuGOB634Qf/D6FStobFlijtc2u1RDbK6yS4WdRx7ieg6n6TIphdJriepzntjc+ukZnNbWm6Bcmu82pD49A9Q0pnmlSeT0yJQL3tHJVv0IaGRSjqpsi9rM5+1qpVAPnEvQiacqKC9n7k8FbOrEpZW7b6qdP/MRUVZaSbL3NkREX7+Jwl558FuE7jZqav+HkpsP42WlF/VLIaHei6LS4ZtEvWF5zibfNX6NuU2r89jhLxk7rLb7saeo4/xgc7wjrtNf0tFxpMG81T04EExfHzo/dDO67U97FaUc/UKhqAPupantaarF1JiPxym32Uwec93hpUVDD64YX+dtBPbQXpz8/Xcf4HGlUKrHc5aexpzg4YcXbu3IEzj1oZBx1ktCP5k+5Ok/xdI77XvrhrbFxXZJl3a8DlNhD/7neh+JqsqH3mfvMlpHJ7jTqAlA4E8fPrzrXAtuPIJ5OLNTHqbh67HJqiDy02/0p5o6t4QEt4dY+7S5QAvOdX/bE2NCJJxcs7W1gA+sYHy4uzkt1UjGyTxmfL1YEaqFnMPFuf2MCj+MneyRi2HUPNwQnjKDoBfbrZxTT/sUblieA+M0r1K0Jw/Do4jN7mwyQocLXcN0lMyDjFy7+UuBrSJwVWrwClekyrWyoZSsovcVl4QvbPix1fdMzROpFGU5sps+dcEg9D5jGwFdihHXNeiTNgfVpnQ8Sq8pV7bfdWDJg/8LAAD//1QJZ1w=" + return "eJzcXf+T27ax/91/xb7LZM6eOdF3Z5+daqbp87PT1JO49sR2X2fajgKBKxE9EqAB8M5Kp//7G3whCZIgJZ10ifv0Q2JRxO5nF4vFYrHAzeAaN3MgZfEAQDOd4xy+R46S5PDi3RtYMcxT9QAgRUUlKzUTfP4A/HPzL4AZcFLgHHKmNHLG1/YpgN6UODf0b4VM/bOQDHzrHwK8SFOJSoHOEBTKG5TAVEsQBE86rEopKColZGK+78rvXd3KUkkejJDEG+R6f5q2WY+oZgUqTYqyQ24tRVVGiPlHoW5DYpVqHtWUctFo23yoqLiew0XwaETh5vOhBgdiZTVvJQDGoWBUCoVU8FSBYpwifOTsM2ApaNYTkQqu8bPeScCwx/kGiFwyLYnc1EQqkgPjKyELYt4HiWsiU2MADbwzoKTUlcQUlhv7mKztY2FZkDzfmK68YWn7RqVQJjWcDScFo3NYkVxhT+MDlWuyHipdLP+JVAeP3YNFzFY6LLWscLeueQGrnGgoSFla819ZGWYprhjH1MKCW6YzUFqaF25IXqFKhgJkWpcDAcLuiYrftlea6EotqEixAzBqfpMiAXwwg9vSA0OvtjoDESSqUnCFERGM5IeIYNoHXiKkMuyrHUSo6dX4c7FeY2qGjbOzCASWHoP56xS5ZiuGcg/WWBCWH4P7d4bQPjKXEa69h9Pivqu5GR5wm6HE0E0pkEiFTDE9M9QZtWOfwC0uYSnFbTDo2+5jyjQUS03sQFpJUViaf539UchbYqiZf0GGJEV5ZhDcZoxm9qUVk0oDci03hop51IIUkq0ZJznQnNmJIMLaDlPIxC2aGa5g60wDFxqWCBzNNEIkyzfGTyhtxCIKmAZKuHljJeTa+TQCBckZZaJS4/q3DsP6xkg/BB57h554L1b6lkjvaoFQbXwOM6gykq+MBojldwa4TrpdAI/h7Xugolgybp16ZIRL/FSh0gcNchkz8j6BLXK+MDjLHDXCR5mfeQ9LMyzwDDKhNBCeQkl01u3dCK6OeOS298vUINyKErwfkuT2DCpeEqkwhY8//VhbolfnGWCytt5VzR8/xs/EiJZQUcyfPn3yWCGRNPvDp99jTpRm1H3/SosyGZOjlEILKvL7EKamHZMhgRMnxckotFWV3wssQ/cMSqEUWxrnYux/RpTCYpn/Oko3ZjeYvY4kXU17ROkB+nHNl0LqezEIIXUc19OnT8bREJ3dl7YM7RFN+Y4d15L7/T5QOcr+3SW6WelThXJTx4ZxyD0bHIeeEXUvwA3dHjYznmp8WpQn0XnNjK3FDUrFBD9WSGdDUE+zjylH4lYgArSZw8NlXhdZgToTR4n0GkyO5L6Q2nnVBdVf0gLAihVZBdRYXQhxenl+fhpV8opxpjKMqXkpRI6E7zPf+ybAeMoosWHNbYY6Q9kBBbdENZxBSBO0RfStNkpjsVXbE5je+kWsJ+U7Jpnuqq2TxdRo3TJWY9OEx9aG4kY7dSwOnQgvBGfcDNNIzeL9uABDygeCLHOiV0IWxwVYUz0QXG/1NLKo2oqoXbPYmFZnRHvmqoUUMW+fITvEvl8HyZ0SpVmFNY4MQVbcfvWMgiVfSjSxSqIiz5Hqzvjv58s6mFm/u0Yc11at/bkqUDLagHv9qulNlDeMYv3LmGUdH8oAwqmCkkhjUNNYbKL3eCb+vqsARz7mH917hxhQzWrcMcbkPa5PfF0UlSbLHKHi7FOF0HGOHiAWTGuXuWxn6C4dNxSGa1eAnCwxX2gsjOPAOZz86182dfDvf5/03hQl8kXO+PWC8QWtpOn8hSbLQbLRfCoZEB2INYOC8TqymsPJVXKenPf5mY+FMoeTJHlMyvLxNVsSTr56nBKVLQWR6eOnF8ur9HeX57Pn31xezC4u8PnsG/r0+ezZ1fKbp1fLK7paPvnDgvz+oY1T5+5/Cxeuzh8STvLNL7i4ZXlKiUzn/6XP3IunPk2ceCXb5P/868vLRj1fX16ePnr0aIi6L9wzI9yM5GVGLn4VGXPC1xVZ4zyvKHKclOjvbX///eTUiBO16lgQfJBh/6UbAU+achQR8hsmBS/6SaejeJeA+Aj7WsdR3sNc0NbEzX2sJP8c+IpSirUkRWF0W2OHSmE6uhiLd/nBoHodvyuuZp1Rcc1GPOwXqHYP90vQ9ASUZsEjSYG3Ql7/p6i3AfwlKHgSTLMyGWTJv2D1uhz8F6DaAZAmupSEK0J1CyGyKzy1DduXpxM4j+6yTmx0ZxiCgtevOoFjNAwbhGB/YXgL70vCVRgv7Bx/jcdevdDEhl097W+NRp7gkxSfn5/Pnqd47qKR5cXF1Sxd/Y7+7jwll+nq4k4RV6C2hKXbY62eMGGYdd8yjURYPQka+G1cFdgtRfgyTNbabLu+c9C0CDYjXeJgiWb5qECLfjmLWwd+kdJ4bGEk+RV89/J9R4DQKR8Iv+eSHTWSMxLmMkqis6aapYnHLQpXZDQgO3Sz+1OuaXREz4Q6kuQjObedcNocVdKhMDSMYXHJTlRZOaR1l15y1Jq054Cm2J6tmsweTWUD+9BG4TVUup3cVuYc2MmDfYddVOf3EhLf+EjGXVPtbNL4AV6/Y4uXqCgZpmATfA8p4afaUfa1VOpRR1d2MlYdbXWKsCYKsGLFV2MladNFVw7EeNlVPE16YN+WEdc73QV15s+0HJI7gF6UYD+TuA9F17aju26G8I66+8rW1DDeTCmwNUWxj+MOKezGM7aE298ZjSYxd3BEHnoDpDejwY5Zpbtw6niBraqKpHB+E03VOO5dUQ2jgZ6OEL10wU8qfpjD+U307mHcu9prPqNaPyy4i4Z17QLzwEkhFQVhd52aK5lPBEwrSdZ38I0d4kRlkUjMl0ceQNgWeQ2nNKKzA4g2FUPBMIm+J6SuQwuYfdvdtRsphoru723JU+xa91SztKvcA+QfVCZ9BR/evno7h6LKbbFxnjZ9l9iy0W2Kqiv5CsYrt6F8Mj8Z4HZllQPge+d1HJ2pssGTXnwW1JMfOA77tdW76N7W67qGw9XUvrGZJRYLy+4wZ1haTa38A38waIlEtyeD/sd9i3TJS8E1YdwE9kUhuG3n9QbkhrDc7pcyDiTPferCgLI7uLGzRKZ9Yjz0L4JvCdttds+/CbaOQleSt6c/RInSlfb4sgvBgzSKEQlYqxZfg9CzmZP/bk7ynHRsJyW6hmdsj0lMO0uNOrkYvOfLKufwolpXSsPlM53B5fnFszO4uJw/uZpfPUmePLncTWh3fuc2Qx4khXKx9gUdtmZibc9z6WGetj3cMrYiag7o2EWbrRTx1egKNZQonf5sWTTKSObX6anHuDlUBvsv5+JAG/OzRwWo4Cu2rqS1ubpSoIMApRQTTiDO5DvTqDZq6jgasyJpynzZFuMrYaycEmV9kuXTlCmM+RGD4DBP6KA5VfuRS3NRpe3QfWm+1iejJBSoSUo0iY/mN/5Xdz6CdpoqI29w+ipNF/aFRU0yOEn3oLdbEmQsSGJbJTXZ/uBAumUEhHsiXYQJvHMF2+gX6EAkGoJnsKZ4BkJCytZMk1xQJDwZxca40oRTXLAt5vfavxjkOG1pVUFoxnjf/GMcAoe9jUdYbbIbF//CIrCzRs/6MikwZVUxzf2NI2FNbD/m3vuznOnNIvDmDYJKzZAoPbugW5xRQAiss2etI2fKwWFqxIN3Tc76l6ZXGyj+l9nn3U3PNzFYvhdinaMbaePcJa5b9zjC4Cf7zjb5/EBPBb2248eP9Ff19whx95utvw1q6dwwd7+ZMasyIfXCedE2EiKcZkLW/GbNKB+ZwRtYcR875gu9X3VxzUE+8aMrzmoIAkuTKXZFLO2wF8fQLiw555waAGYyXlYs18155jiUaPS2F5KXDU9uTzqP8+qkVuHuZ1wnaj6tJhyfxmj9Foc32T+5bxEir82EGhiqkBHX09qmeb7VMie3V8bsUqjkGiXHYby/95KlH5CeKnCk66RDo6Xraml+0KhaXf0QPotwan9vJvnujN0ShVBT04O+bbRVvR3Q+ym5FOkRjD/QQClSiCc6DKvqUBcTcHonUvj4+lV8KaZKEil7vSurluKQmUjxuBo0FEdUuKvr2I2R314pSGRrkHAutK0aPxq7gGSc5zHdccCXdjzzFNsjTEhRvo5ue/HHrF4MeQfz4t0bt57o+xf7cKZKpGzFqKvJNw75xbs3I67ghuGtXwM9GBfCsZjDSSro36IVsaf/SGwkX6+9m1IPKAnjebvbNqzlidfxWFGaQp6dinjiBTxbine2F7k8Xz1bEXo+e06fEVfkQsjV1ezJ8jy9uqTPL+iz81+pVHq30p2jS3SEwujOqh5YCoz2Ss+mDM62Suz0xPh6cY2bY5rbLIZx38zDKzPUwntPCPfldxJLiQq5DScI9wkgQY0V+0P6BArBmRamaa3Nmt1el6HsNFv6K2ie7h4OtWtm1xMRn1jlpWTDzHrvFgPP+nKM9R8r7ur/KMlzv7IyIblbxrGCyA2UKEvUkmjhL9mYOF4Zmsxhfvp7T+kH3PRu2HAmbbxspeyCt2Z61wpGlyl6hZqw/EusZLxanX9Dvnl+fIc4HOa/ajXjvnKNuMWIFJGKxtBK8TPFUse2Mu94bo8sRaU7V57kZmEhxS2vR3BgmlPHVQeHivc4J/Kh9hfuFLFCHWTB7fOMlCVyTP1mkAlXlkSFrUbKwQtUauwUyeDilKjbiaJtLohxADyXMQwire5+RM+wcxTaxcLgnpq+t4XxjPQ25o3TjxLLCE9zjB9+jJ3Z3k2lr92RbSE7J7adbu1JUVKtMw1KFOjOkTLu0zEptke4h4sasT5koLzo7gU0Y6ZegBu33txnttdgyfEG++cE9zIIhTcomd60e6VUyLEjEXb6kYvRio4defL+PVkS6pT31IGMexqB7UZNvulOsNODsSSSFIspUHc62PbCEUYt2S+YNhjgO+OuTl+KKk9txQ4VnCPVoAV8rU77p0ib+3NKlHpTUwGmQGmW58225JndpVOZJbtEwE8VyeuT1x0Jz2BZaVfnWOaEYiZyu+8j0X5NhwheczvOQDFd+TX1gKpBZLrcsLQDyi8MQYu1Hb9N1qu5RsEvSE/MivS927t9Y7RFbeAS0a5/yV8vUbh3zyBn1wgv3320GiiwEHIDlZPUbmkSif1keGzjeJga95FRbOnbuQti50D/Z9fs53q5riAXtL0OwkvU30980DVXWlYPuubZdWCjZvkzLasBa6M3u1/QdvvoPqbQJE+4kEVS0mG4rijJMV2sckF0JHotUdJu5dG2UhnXwNiWWFmc9iydKu3BiE29PWOPpLtLLK0bIkrHzmMHF7QxBUYIkttx6Snxqli6m/cMJyqkTSKlwDRIwtfYrTuzRnRubP3i/PzrpN9Fzgjv2Euu8aCjvGHv01eDLurVKtVds9zojnxTHWPoeizDfQdCdRVhu88Eaym0oxhT2wsriTh9G0Bb4YY73WQzJvu2O2UcPsOlBsm4I5LAa7uBR0lOK7N6SsEEpCkIF5m8fZ/AWw4/Ml59tvf+Ca6Y0qo9y9jQ7DEt88qQpZm3yWW1WqFUltzb9381xJgCAqqyV5CE4Mzrdr+KE6rZTf3cNv1fxlNxq858eztj9Kc/UfusxDc0xH8eGHz/7pD9LN63Dky+Htf1FseZHZWNxw8cfc9nTuw8Bm7zDoa5q/Ocss1RB7rFhU450R3uZDquIz22Kx06077aBmNih857Y9u02WzTS8zeh2YrmZr7U9rmpcQV+zyHk79Z9f/jZKcuVeyX+3Q3tg7MutwbJkPPGPZZRjqCNCXfSiVDdsfH9xMqlhpTeo8a3rNfMHHReGHCdmMFEciC0sqeEGIcCmL+4955+NOLN4+ScPdCiUpSLEjZ3cF4HzzuIGx+gBXLUQFyyWiGqQt4g7ool8NxfeqYLFYs1yibfp5BwzwJYUTjweB3GEu97r5T+v/kKp3+GrG+csTmXRuVdU65xkAd/SaUWjfNhvygA5YVT3NrERitQN8r//ujoC5PUKuhkVxiTuy07Bdqhl9d5Wzz+7o7GEqbSwnHgXvSHQIl4fF9vLFyXLuRZ4jvu9HmD8x3h0JJ7ngxwKAXIlsSxlJgN7W/wlIitYHYw29/D8+Sq0cJvHBTd76pb3cKbtkqScwUMvy8GN8b2RFLdyfEcDoDxrUUaUVDfDvWeo9uzOz4twHs34AwxoGKrXljnMQiq1NpigpXDUjCyt8IxB0LXPeD+IPXr1hBY8sSc7yx2aUaYnMxmjs0Aw9xPYfTdJmUQum1RPUpT2xu/fQMTmvrTVAuzXcbUp+eAWoa07zSJHLeY8oF7+hkq34ENDIpR1W2RW3m83a1UqgHziXoxFMVlJczd/H8pk5cWrn7ptr5oxERZaWVJHtvQ0T09Zso7JUHv0XobqOm9n8oufkwXlZ6Ub8UEuq9KCodvknUG5bnbPJd49eY27Q6jx2ZkrEjXrsfVIo6zg82xzviOv2VDx1HGsxb3YMDwfT1ofNDN6Pb/rRXUcrRr6eJOuBemtqeplpMjfl4nHKXzeQx1x1egTP04Irxdd5GYA/tNbzff/cBHlcKpXo8Z+lpzAkefnjhzo48gVMfChlnvST02vQhT/8plt5pf7FuaFtcbJd0acfrMBX24H+u95GoqnzoffYuo3V0ghtymgAE/vThw7vOJdPGI5iHMzvlYRq+HpusCiKvf6M//NO5cyK4i8Lap80FWnCu+9ueGBMi4eSGra0FfGAF48PdzWmpRjJO5jHj68WKUC3kHC7O7WdU+GHsZI9cDKPm4YbwlBkEvdhu5Zx62qdwy/IcGKd5laI9eRgeRWx2Z5MROlzoGqajZB5k5MbNXwpsFYGrUoNXuCJVrpUNpWQVvf22JHxhw4+tvmdqnkilKMuR3fSp6+qg9xnbCOhSjLiuQZ+0Oag2peNRek25sv2uA0se/F8AAAD//3KC/1s=" } diff --git a/tests/system/test_ecs_mappings.py b/tests/system/test_ecs_mappings.py index 4e651730a5b..81ba5f39fc5 100644 --- a/tests/system/test_ecs_mappings.py +++ b/tests/system/test_ecs_mappings.py @@ -34,10 +34,9 @@ def test_ecs_migration(self): all_fields = set() alias_fields = set() for f, a in flatmap(yaml.load(self.command_output)["mappings"]["doc"]["properties"]): + all_fields.add(f) if a.get("type") == "alias": alias_fields.add(a["path"]) - else: - all_fields.add(f) # fields with special exception, due to mapping type changes, etc # no comment means unchanged @@ -68,6 +67,16 @@ def test_ecs_migration(self): should_not_be_aliased = alias_fields - all_fields self.assertFalse(should_not_be_aliased, json.dumps(sorted(should_not_be_aliased))) + # check the migration log too + with open(self._beat_path_join("_meta", "ecs-migration.yml")) as f: + for m in yaml.load(f): + if m.get("alias", True): + self.assertIn(m["from"], alias_fields) + elif m.get("copy_to", False): + self.assertIn(m["from"], all_fields) + self.assertIn(m["to"], all_fields) + + # check that all fields are accounted for not_aliased = all_fields - alias_fields - exception_fields self.assertFalse(not_aliased, "\nall fields ({:d}):\n{}\n\naliased ({:d}):\n{}\n\nunaccounted for ({:d}):\n{}".format(