Unify Kibana dashboard directory logic across Beats

Currently all dashboards end up under `_meta/kibana` in each Beat. The problem with this directory is that for some Beats it contains the automatically collected and generated files and for others the original. This prevents us from having decoded JSON files for all dashboards across Beats.

An other benefit of having the `kibana.generated` directory is that now `_meta/kibana` can become the place for dashboards across modules in one Beat without having to worry about overwriting / removing these dashboards.

Community Beats must convert their dashboards with the script under #7224.

Further changes:

* Decode Packetbeat and Winlogbeat dashboards
* Move encoding step to libbeat so it is applied automatically to all dashboards in the kibana.generated directory.
* Remove _meta/kibana from .gitignore as it can now be used to contain dashboards in all Beats.
* Adjust Makefiles for the new logic.
* Adjust tests to depend on kibana.generated directory

CHANGELOG-developer.asciidoc

@@ -22,6 +22,7 @@ The list below covers the major changes between 6.3.0 and master only.
 - The beat.Pipeline is now passed to cfgfile.RunnerFactory. Beats using libbeat for module reloading or autodiscovery need to be adapted. {pull}7018[7017]
 - Moving of TLS helper functions and structs from `output/tls` to `tlscommon`. {pull}7054[7054]
 - Port fields.yml collector to Golang {pull}6911[6911]
+- Dashboards under _meta/kibana are expected to be decoded. See https://github.com/elastic/beats/pull/7224 for a conversion script. {pull}7265[7265]
 
 ==== Bugfixes
 

auditbeat/.gitignore

@@ -1,5 +1,4 @@
 build
-_meta/kibana
 _meta/beat.yml
 _meta/beat.reference.yml
 module/*/_meta/config.yml

auditbeat/Makefile

@@ -79,8 +79,6 @@ collect-docs: python-env
 # Collects all module dashboards
 .PHONY: kibana
 kibana:
-	@-rm -rf _meta/kibana
-	@mkdir -p _meta/kibana
-	@-cp -pr module/*/_meta/kibana _meta/
-	@# Convert all dashboards to string
-	@python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana/6/dashboard/*.json"
+	@-rm -rf _meta/kibana.generated
+	@mkdir -p _meta/kibana.generated
+	@-cp -pr module/*/_meta/kibana/* _meta/kibana.generated

auditbeat/tests/system/test_base.py

@@ -63,7 +63,7 @@ def test_dashboards(self):
 
         dirs = [self.temp_dir("auditbeat_test")]
         with PathCleanup(dirs):
-            kibana_dir = os.path.join(self.beat_path, "_meta", "kibana")
+            kibana_dir = os.path.join(self.beat_path, "_meta", "kibana.generated")
             shutil.copytree(kibana_dir, os.path.join(self.working_dir, "kibana"))
 
             es = Elasticsearch([self.get_elasticsearch_url()])

filebeat/.gitignore

@@ -5,7 +5,6 @@
 filebeat
 
 build
-_meta/kibana
 _meta/module.generated
 _meta/beat.yml
 _meta/beat.reference.yml

filebeat/Dockerfile

@@ -4,7 +4,7 @@ MAINTAINER Nicolas Ruflin <ruflin@elastic.co>
 RUN set -x && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
-         netcat python-pip virtualenv && \
+         netcat python-pip rsync virtualenv && \
     apt-get clean
 
 RUN pip install --upgrade setuptools

filebeat/Makefile

@@ -17,11 +17,9 @@ before-build:
 # Collects all module dashboards
 .PHONY: kibana
 kibana:
-	@rm -rf _meta/kibana
-	@mkdir -p _meta/kibana
-	@-cp -r module/*/_meta/kibana _meta/
-	@# Convert all dashboards to string
-	@python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana/6/dashboard/*.json"
+	@rm -rf _meta/kibana.generated
+	@mkdir -p _meta/kibana.generated
+	@-cp -pr module/*/_meta/kibana/* _meta/kibana.generated
 
 # Collects all modules files to be packaged in a temporary folder
 .PHONY: modules

filebeat/tests/system/test_modules.py

@@ -63,7 +63,7 @@ def init(self):
                                             "/../../../../module")
 
         self.kibana_path = os.path.abspath(self.working_dir +
-                                           "/../../../../_meta/kibana")
+                                           "/../../../../_meta/kibana.generated")
 
         self.filebeat = os.path.abspath(self.working_dir +
                                         "/../../../../filebeat.test")

heartbeat/.gitignore

@@ -4,7 +4,5 @@
 
 heartbeat
 
-_meta/kibana
-
 build
 /tests/load/logs

heartbeat/Makefile

@@ -31,8 +31,6 @@ create-monitor:
 # Collects all module dashboards
 .PHONY: kibana
 kibana:
-	@rm -rf _meta/kibana
-	@mkdir -p _meta/kibana
-	@-cp -r monitors/active/*/_meta/kibana _meta/
-	@# Convert all dashboards to string
-	@python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana/6/dashboard/*.json"
+	@rm -rf _meta/kibana.generated
+	@mkdir -p _meta/kibana.generated
+	@-cp -r monitors/active/*/_meta/kibana/* _meta/kibana.generated

libbeat/scripts/Makefile

@@ -342,10 +342,13 @@ endif
 	@mkdir -p docs
 	@${PYTHON_ENV}/bin/python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_TITLE} ${ES_BEATS}
 
+	@mkdir -p $(PWD)/_meta/kibana.generated
 	@# Generate Kibana index pattern and copy dashboard files
 	if [ -d $(PWD)/_meta/kibana ]; then \
-		cp -r $(PWD)/_meta/kibana/ $(PWD)/_meta/kibana.generated; \
+		cp -pr $(PWD)/_meta/kibana/* $(PWD)/_meta/kibana.generated ; \
 	fi
+	@# Convert all dashboards to string
+	@python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana.generated/6/dashboard/*.json"
 	@mkdir -p $(PWD)/_meta/kibana.generated/5/index-pattern
 	@mkdir -p $(PWD)/_meta/kibana.generated/6/index-pattern
 	@go run ${ES_BEATS}/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go -index '${BEAT_INDEX_PREFIX}-*' -beat-name ${BEAT_NAME} -beat-dir $(PWD) -version ${BEAT_VERSION}

metricbeat/.gitignore

@@ -1,5 +1,4 @@
 build
-_meta/kibana
 _meta/beat.yml
 _meta/beat.reference.yml
 

metricbeat/Makefile

@@ -18,12 +18,9 @@ include ${ES_BEATS}/libbeat/scripts/Makefile
 # Collects all module dashboards
 .PHONY: kibana
 kibana:
-	@rm -rf _meta/kibana
-	@mkdir -p _meta/kibana
-	@-cp -r module/*/_meta/kibana _meta/
-	@-cp -r module/*/_meta/kibana _meta/
-	@# Convert all dashboards to string
-	@python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana/6/dashboard/*.json"
+	@rm -rf _meta/kibana.generated
+	@mkdir -p _meta/kibana.generated
+	@-cp -pr module/*/_meta/kibana/* _meta/kibana.generated
 
 # Collects all module docs
 .PHONY: collect-docs

metricbeat/tests/system/test_base.py

@@ -59,7 +59,7 @@ def test_dashboards(self):
         Test that the dashboards can be loaded with `setup --dashboards`
         """
 
-        kibana_dir = os.path.join(self.beat_path, "_meta", "kibana")
+        kibana_dir = os.path.join(self.beat_path, "_meta", "kibana.generated")
         shutil.copytree(kibana_dir, os.path.join(self.working_dir, "kibana"))
 
         es = Elasticsearch([self.get_elasticsearch_url()])

packetbeat/.gitignore

@@ -10,7 +10,6 @@
 _obj
 _test
 _output
-_meta/kibana/*/index-pattern
 
 # Architecture specific extensions/prefixes
 *.[568vq]