Cherry-pick elastic#20540 to 7.x: Fix Winlogbeat export index-pattern…

… with migration.6_to_7.enabled (elastic#20574)

* Test export commands in all Beats (elastic#20016)

This adds 4 new integration tests to all Beats that test export commands:

> beatname export ilm-policy
> beatname export template
> beatname export index-pattern
> beatname export config

* [Filebeat] Add export tests to x-pack/filebeat (elastic#20156)

* Add export tests to x-pack/filebeat

Add export sub-command tests to x-pack/filebeat and add an assertion for the
size the Kibana index-pattern due to the Kibana API limiting payloads to 1 MiB.

* Assert size of index pattern document is less than 1 MiB

* Fix Winlogbeat export index-pattern with migration.6_to_7.enabled (elastic#20540)

- Test export index-pattern with migration.6_to_7.enabled=true
- Always run these tests, not only when integration testing
- Remove conflicting Winlogbeat alias

* Changelog entry

Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>
(cherry picked from commit 9ec7549)

CHANGELOG.next.asciidoc

@@ -185,6 +185,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Winlogbeat*
 
+- Fix invalid IP addresses in DNS query results from Sysmon data. {issue}18432[18432] {pull}18436[18436]
+- Fix duplicated field error when exporting index-pattern with migration.6_to_7.enabled. {issue}20521[20521] {pull}20540[20540]
 
 *Functionbeat*
 

auditbeat/tests/system/test_base.py

@@ -6,9 +6,10 @@
 from auditbeat import *
 from elasticsearch import Elasticsearch
 from beat.beat import INTEGRATION_TESTS
+from beat import common_tests
 
 
-class Test(BaseTest):
+class Test(BaseTest, common_tests.TestExportsMixin):
     def test_start_stop(self):
         """
         Auditbeat starts and stops without error.

filebeat/tests/system/filebeat.py

@@ -18,7 +18,7 @@ def setUpClass(self):
         if not hasattr(self, "beat_name"):
             self.beat_name = "filebeat"
         if not hasattr(self, "beat_path"):
-            self.beat_path = os.path.abspath(os.path.join(curdir, "../../"))
+            self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
 
         super(BaseTest, self).setUpClass()
 

filebeat/tests/system/test_base.py

@@ -3,9 +3,10 @@
 from filebeat import BaseTest
 from elasticsearch import Elasticsearch
 from beat.beat import INTEGRATION_TESTS
+from beat import common_tests
 
 
-class Test(BaseTest):
+class Test(BaseTest, common_tests.TestExportsMixin):
 
     def test_base(self):
         """

heartbeat/tests/system/test_base.py

@@ -1,13 +1,14 @@
+import nose.tools
 import os
 import unittest
 
-from heartbeat import BaseTest
-from elasticsearch import Elasticsearch
 from beat.beat import INTEGRATION_TESTS
-import nose.tools
+from beat import common_tests
+from elasticsearch import Elasticsearch
+from heartbeat import BaseTest
 
 
-class Test(BaseTest):
+class Test(BaseTest, common_tests.TestExportsMixin):
 
     def test_base(self):
         """

journalbeat/tests/system/test_base.py

@@ -6,9 +6,10 @@
 import time
 import yaml
 from shutil import copyfile
+from beat import common_tests
 
 
-class Test(BaseTest):
+class Test(BaseTest, common_tests.TestExportsMixin):
 
     @unittest.skipUnless(sys.platform.startswith("linux"), "Journald only on Linux")
     def test_start_with_local_journal(self):

libbeat/tests/system/beat/common_tests.py

@@ -0,0 +1,80 @@
+import json
+import unittest
+import yaml
+
+from beat.beat import INTEGRATION_TESTS
+
+
+class TestExportsMixin:
+
+    def run_export_cmd(self, cmd, extra=[]):
+        """
+        Runs the given export command and returns the output as a string.
+        Raises an exception if the command fails.
+        :param cmd: the export command
+        :param extra: Extra arguments (optional)
+        :return: The output as a string.
+        """
+        self.render_config_template()
+
+        args = ["export", cmd]
+        if len(extra) != 0:
+            args += extra
+        exit_code = self.run_beat(extra_args=args, logging_args=[])
+        assert exit_code == 0
+        output = self.get_log()
+        trailer = "\nPASS\n"
+        pos = output.rfind(trailer)
+        if pos == -1:
+            raise Exception("didn't return expected trailer:{} got:{}".format(
+                trailer.__repr__(),
+                output[-100:].__repr__()))
+        return output[:pos]
+
+    def test_export_ilm_policy(self):
+        """
+        Test that the ilm-policy can be exported with `export ilm-policy`
+        """
+        output = self.run_export_cmd("ilm-policy")
+        js = json.loads(output)
+        assert "policy" in js
+
+    def test_export_template(self):
+        """
+        Test that the template can be exported with `export template`
+        """
+        output = self.run_export_cmd("template")
+        js = json.loads(output)
+        assert "index_patterns" in js and "mappings" in js
+
+    def test_export_index_pattern(self):
+        """
+        Test that the index-pattern can be exported with `export index-pattern`
+        """
+        output = self.run_export_cmd("index-pattern")
+        js = json.loads(output)
+        assert "objects" in js
+        size = len(output.encode('utf-8'))
+        assert size < 1024*1024, "Kibana index pattern must be less than 1MiB " \
+                                 "to keep the Beat setup request size below " \
+                                 "Kibana's server.maxPayloadBytes."
+
+    def test_export_index_pattern_migration(self):
+        """
+        Test that the index-pattern can be exported with `export index-pattern` (migration enabled)
+        """
+        output = self.run_export_cmd("index-pattern", extra=['-E', 'migration.6_to_7.enabled=true'])
+        js = json.loads(output)
+        assert "objects" in js
+        size = len(output.encode('utf-8'))
+        assert size < 1024*1024, "Kibana index pattern must be less than 1MiB " \
+                                 "to keep the Beat setup request size below " \
+                                 "Kibana's server.maxPayloadBytes."
+
+    def test_export_config(self):
+        """
+        Test that the config can be exported with `export config`
+        """
+        output = self.run_export_cmd("config")
+        yml = yaml.load(output)
+        assert isinstance(yml, dict)

libbeat/tests/system/test_base.py

@@ -1,4 +1,5 @@
 from base import BaseTest
+from beat import common_tests
 
 import json
 import os
@@ -9,7 +10,7 @@
 import unittest
 
 
-class Test(BaseTest):
+class Test(BaseTest, common_tests.TestExportsMixin):
 
     def test_base(self):
         """

metricbeat/tests/system/test_base.py

@@ -3,12 +3,14 @@
 import unittest
 import os
 import shutil
-from metricbeat import BaseTest
-from elasticsearch import Elasticsearch
+
+from beat import common_tests
 from beat.beat import INTEGRATION_TESTS
+from elasticsearch import Elasticsearch
+from metricbeat import BaseTest
 
 
-class Test(BaseTest):
+class Test(BaseTest, common_tests.TestExportsMixin):
 
     COMPOSE_SERVICES = ['elasticsearch', 'kibana']
 

packetbeat/tests/system/test_base.py

@@ -0,0 +1,11 @@
+import os
+import sys
+from packetbeat import BaseTest
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../libbeat/tests/system')))
+
+from beat import common_tests
+
+
+class Test(BaseTest, common_tests.TestExportsMixin):
+    pass

winlogbeat/_meta/fields.common.yml

@@ -508,11 +508,6 @@
       path:  winlog.user.identifier
       migration: true
 
-    - name: user.domain
-      type: alias
-      path: winlog.user.domain
-      migration: true
-
     - name: user.type
       type: alias
       path: winlog.user.type

winlogbeat/docs/fields.asciidoc

@@ -7202,15 +7202,6 @@ alias to: winlog.user.identifier
 
 --
 
-*`user.domain`*::
-+
---
-type: alias
-
-alias to: winlog.user.domain
-
---
-
 *`user.type`*::
 +
 --

winlogbeat/tests/system/test_config.py

@@ -3,14 +3,15 @@
 import sys
 import unittest
 from winlogbeat import BaseTest
+from beat import common_tests
 
 """
 Contains tests for config parsing.
 """
 
 
 @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows")
-class Test(BaseTest):
+class Test(BaseTest, common_tests.TestExportsMixin):
 
     def test_valid_config(self):
         """

x-pack/filebeat/tests/system/test_filebeat_xpack.py

@@ -0,0 +1,30 @@
+import jinja2
+import os
+import sys
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../filebeat/tests/system')))
+
+from filebeat import BaseTest as FilebeatTest
+from beat import common_tests
+
+
+class FilebeatXPackTest(FilebeatTest, common_tests.TestExportsMixin):
+
+    @classmethod
+    def setUpClass(self):
+        self.beat_name = "filebeat"
+        self.beat_path = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), "../../"))
+
+        super(FilebeatTest, self).setUpClass()
+
+    def setUp(self):
+        super(FilebeatTest, self).setUp()
+
+        # Hack to make jinja2 have the right paths
+        self.template_env = jinja2.Environment(
+            loader=jinja2.FileSystemLoader([
+                os.path.abspath(os.path.join(self.beat_path, "../../filebeat")),
+                os.path.abspath(os.path.join(self.beat_path, "../../libbeat"))
+            ])
+        )

x-pack/functionbeat/tests/system/test_base.py

@@ -3,9 +3,10 @@
 import json
 import os
 import unittest
+from beat import common_tests
 
 
-class Test(BaseTest):
+class Test(BaseTest, common_tests.TestExportsMixin):
     @unittest.skip("temporarily disabled")
     def test_base(self):
         """