From 3ad9745b23b76337efcae1f67ef814b6b4fb5e90 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 21 Aug 2018 15:56:15 -0700 Subject: [PATCH 01/12] Support Kibana Spaces via setup.kibana.space.id setting --- auditbeat/auditbeat.yml | 5 +++++ filebeat/filebeat.yml | 5 +++++ heartbeat/heartbeat.yml | 5 +++++ libbeat/_meta/config.yml | 5 +++++ libbeat/kibana/client.go | 7 ++++++- libbeat/kibana/client_config.go | 2 ++ metricbeat/metricbeat.yml | 5 +++++ packetbeat/packetbeat.yml | 5 +++++ winlogbeat/winlogbeat.yml | 5 +++++ 9 files changed, 43 insertions(+), 1 deletion(-) diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml index 56789249281..a3749aa2af0 100644 --- a/auditbeat/auditbeat.yml +++ b/auditbeat/auditbeat.yml @@ -95,6 +95,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using auditbeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 5ae43216b7d..52888b19ea5 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -122,6 +122,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml index d80f7bc0f5f..044616ad816 100644 --- a/heartbeat/heartbeat.yml +++ b/heartbeat/heartbeat.yml @@ -69,6 +69,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using heartbeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/libbeat/_meta/config.yml b/libbeat/_meta/config.yml index 0277cd84fdd..6f460928bac 100644 --- a/libbeat/_meta/config.yml +++ b/libbeat/_meta/config.yml @@ -39,6 +39,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 69b34700ae6..2efa9b0bfa7 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "net/http" "net/url" + "path" "strings" "github.com/pkg/errors" @@ -88,7 +89,11 @@ func NewKibanaClient(cfg *common.Config) (*Client, error) { // NewClientWithConfig creates and returns a kibana client using the given config func NewClientWithConfig(config *ClientConfig) (*Client, error) { - kibanaURL, err := common.MakeURL(config.Protocol, config.Path, config.Host, 5601) + p := config.Path + if config.SpaceID != "" { + p = path.Join(p, "s", config.SpaceID) + } + kibanaURL, err := common.MakeURL(config.Protocol, p, config.Host, 5601) if err != nil { return nil, fmt.Errorf("invalid Kibana host: %v", err) } diff --git a/libbeat/kibana/client_config.go b/libbeat/kibana/client_config.go index 25af90a810c..c5dc101075c 100644 --- a/libbeat/kibana/client_config.go +++ b/libbeat/kibana/client_config.go @@ -28,6 +28,7 @@ type ClientConfig struct { Protocol string `config:"protocol"` Host string `config:"host"` Path string `config:"path"` + SpaceID string `config:"space.id"` Username string `config:"username"` Password string `config:"password"` TLS *tlscommon.Config `config:"ssl"` @@ -39,6 +40,7 @@ var ( Protocol: "http", Host: "localhost:5601", Path: "", + SpaceID: "", Username: "", Password: "", Timeout: 90 * time.Second, diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 8363541d426..5bb313ded91 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -66,6 +66,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 79babc2232a..5bdaa0f6911 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -149,6 +149,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using packetbeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index d5c53bbe686..d711c7f10db 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -70,6 +70,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using winlogbeat with the Elastic Cloud (https://cloud.elastic.co/). From 287915d57857fc44c2f2a4a861ebbae68ec9f817 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 21 Aug 2018 16:31:52 -0700 Subject: [PATCH 02/12] Adding CHANGELOG entry --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 70fa946cd7e..3edfa840779 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -110,6 +110,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Added the `add_process_metadata` processor to enrich events with process information. {pull}6789[6789] - Report number of open file handles on Windows. {pull}8329[8329] - Support for Kafka 2.0.0 in kafka output {pull}8399[8399] +- Add setting `setup.kibana.space.id` to support Kibana Spaces {pull}7942[7942] *Auditbeat* From 47e9986e97114dab575350f85bdab3b5e7c308f1 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 27 Aug 2018 10:56:49 -0700 Subject: [PATCH 03/12] Add space-id option to export_dashboards script --- dev-tools/cmd/dashboards/export_dashboards.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go index f208b8e6c16..a3f85dc0c18 100644 --- a/dev-tools/cmd/dashboards/export_dashboards.go +++ b/dev-tools/cmd/dashboards/export_dashboards.go @@ -27,6 +27,7 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" "strings" @@ -48,11 +49,14 @@ func makeURL(url, path string, params url.Values) string { return strings.Join([]string{url, path, "?", params.Encode()}, "") } -func Export(client *http.Client, conn string, dashboard string, out string) error { +func Export(client *http.Client, conn string, spaceID string, dashboard string, out string) error { params := url.Values{} params.Add("dashboard", dashboard) + if spaceID != "" { + exportAPI = path.Join("s", spaceID, exportAPI) + } fullURL := makeURL(conn, exportAPI, params) if !quiet { log.Printf("Calling HTTP GET %v\n", fullURL) @@ -138,6 +142,7 @@ var quiet = false func main() { kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL") + spaceID := flag.String("space-id", "", "Space ID") dashboard := flag.String("dashboard", "", "Dashboard ID") fileOutput := flag.String("output", "output.json", "Output file") ymlFile := flag.String("yml", "", "Path to the module.yml file containing the dashboards") @@ -171,7 +176,7 @@ func main() { if err != nil { log.Fatalf("fail to create directory %s: %v", directory, err) } - err = Export(client, *kibanaURL, dashboard["id"], filepath.Join(directory, dashboard["file"])) + err = Export(client, *kibanaURL, *spaceID, dashboard["id"], filepath.Join(directory, dashboard["file"])) if err != nil { log.Fatalf("fail to export the dashboards: %s", err) } @@ -180,7 +185,7 @@ func main() { } if len(*dashboard) > 0 { - err := Export(client, *kibanaURL, *dashboard, *fileOutput) + err := Export(client, *kibanaURL, *spaceID, *dashboard, *fileOutput) if err != nil { log.Fatalf("fail to export the dashboards: %s", err) } From 9ec545fb89fcaf3aa38f8fc2aa5659bbd1cc3ade Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 27 Aug 2018 10:57:21 -0700 Subject: [PATCH 04/12] Add integration tests for importing and exporting dashboards with space ID --- libbeat/tests/system/test_dashboard.py | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index e4b6f69f387..4cef3829726 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -36,6 +36,32 @@ def test_load_dashboard(self): assert self.log_contains("Kibana dashboards successfully loaded") is True + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_load_dashboard_into_space(self): + """ + Test loading dashboards into Kibana space + """ + self.render_config_template() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["setup", + "--dashboards", + "-E", "setup.dashboards.file=" + + os.path.join(self.beat_path, "tests", "files", "testbeat-dashboards.zip"), + "-E", "setup.dashboards.beat=testbeat", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-E", "setup.kibana.space.id=foo-bar", + "-E", "output.elasticsearch.hosts=['" + self.get_host() + "']", + "-E", "output.file.enabled=false"] + ) + + beat.check_wait(exit_code=0) + + assert self.log_contains("Kibana dashboards successfully loaded") is True + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') def test_load_only_index_patterns(self): @@ -88,6 +114,32 @@ def test_export_dashboard(self): os.remove("output.json") + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_export_dashboard_from_space(self): + """ + Test export dashboards from Kibana space and remove unsupported characters + """ + + self.test_load_dashboard_into_space() + + path = os.path.normpath(self.beat_path + "/../dev-tools/cmd/dashboards/export_dashboards.go") + command = path + " -kibana http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + command = "go run " + command + " -dashboard Metricbeat-system-overview -space-id foo-bar" + + p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + content, err = p.communicate() + + assert p.returncode == 0 + + assert os.path.isfile("output.json") is True + + with open('output.json') as f: + content = f.read() + assert "Metricbeat-system-overview" in content + + os.remove("output.json") + def get_host(self): return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200') From 820bcf89c66afff49628bcba6bd6b4f4d3cdb5e0 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 27 Aug 2018 11:03:44 -0700 Subject: [PATCH 05/12] Adding -space-id option for export_dashboards.go to documentation --- docs/devguide/newdashboards.asciidoc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/devguide/newdashboards.asciidoc b/docs/devguide/newdashboards.asciidoc index b874dafbfe3..b532655fc20 100644 --- a/docs/devguide/newdashboards.asciidoc +++ b/docs/devguide/newdashboards.asciidoc @@ -283,6 +283,15 @@ go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system ------------------- +===== Export dashboards from a Kibana Space + +If you are using the Kibana Spaces feature and want to export dashboards from a specific Space, pass the Space ID to the `export_dashboards.go` script: + +[source,shell] +------------------- +go run dev-tools/cmd/dashboards/export_dashboards.go -space-id my-space [other-options] +------------------- + ==== Exporting Kibana 5.x dashboards From 5dc34d215eb15e0bdcccd5f3991409b2f36517c2 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 27 Aug 2018 11:09:14 -0700 Subject: [PATCH 06/12] Adding entry to dev CHANGELOG --- CHANGELOG-developer.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index ff80d070070..1581c93df95 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -53,3 +53,4 @@ The list below covers the major changes between 6.3.0 and master only. `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850] - Set current year in generator templates. {pull}8396[8396] - You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449] +- Add `-space-id` option to `export_dashboards.go` script to support Kibana Spaces {pull}7942[7942] From a4d4d8439b25f6531377c5079fc968d5b6f31879 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 28 Aug 2018 06:30:57 -0700 Subject: [PATCH 07/12] Prefix space path with / --- dev-tools/cmd/dashboards/export_dashboards.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go index a3f85dc0c18..421e006017d 100644 --- a/dev-tools/cmd/dashboards/export_dashboards.go +++ b/dev-tools/cmd/dashboards/export_dashboards.go @@ -55,7 +55,7 @@ func Export(client *http.Client, conn string, spaceID string, dashboard string, params.Add("dashboard", dashboard) if spaceID != "" { - exportAPI = path.Join("s", spaceID, exportAPI) + exportAPI = path.Join("/s", spaceID, exportAPI) } fullURL := makeURL(conn, exportAPI, params) if !quiet { From 30ed549f6889b3e8bf45275d4ff3ae6ad41701eb Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 1 Oct 2018 08:01:05 -0700 Subject: [PATCH 08/12] Create Kibana space before trying to use it --- libbeat/tests/system/test_dashboard.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 4cef3829726..0151f07fefb 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -4,7 +4,7 @@ import subprocess from nose.plugins.attrib import attr import unittest - +import requests INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -43,6 +43,8 @@ def test_load_dashboard_into_space(self): Test loading dashboards into Kibana space """ self.render_config_template() + self.create_kibana_space() + beat = self.start_beat( logging_args=["-e", "-d", "*"], extra_args=["setup", @@ -148,3 +150,13 @@ def get_kibana_host(self): def get_kibana_port(self): return os.getenv('KIBANA_PORT', '5601') + + def create_kibana_space(self): + url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \ + "/api/spaces/space" + data = { + "id": "foo-bar", + "name": "Foo bar space" + } + r = requests.post(url, data) + assert r.status_code == 200 From 1f3e5786c9d2e76f62a077df14b580fc99cc5872 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 1 Oct 2018 10:12:42 -0700 Subject: [PATCH 09/12] Make spaces tests conditional on version --- libbeat/tests/system/test_dashboard.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 0151f07fefb..9fe841e98a6 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -5,6 +5,7 @@ from nose.plugins.attrib import attr import unittest import requests +import semver INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -42,6 +43,11 @@ def test_load_dashboard_into_space(self): """ Test loading dashboards into Kibana space """ + version = self.get_version() + if semver.compare(version, "6.5.0") == -1: + # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available + raise SkipTest + self.render_config_template() self.create_kibana_space() @@ -122,6 +128,10 @@ def test_export_dashboard_from_space(self): """ Test export dashboards from Kibana space and remove unsupported characters """ + version = self.get_version() + if semver.compare(version, "6.5.0") == -1: + # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available + raise SkipTest self.test_load_dashboard_into_space() @@ -160,3 +170,13 @@ def create_kibana_space(self): } r = requests.post(url, data) assert r.status_code == 200 + + def get_version(self): + url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \ + "/api/status" + + r = requests.get(url) + body = r.json() + version = body["version"]["number"] + + return version From c90c541557738b4b8e7bf07fc28af5c4bb9e319d Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 1 Oct 2018 10:48:59 -0700 Subject: [PATCH 10/12] Add semver to requirements.txt --- libbeat/tests/system/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 743fdf14c66..35eba8977d5 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -29,3 +29,4 @@ urllib3==1.22 websocket-client==0.47.0 parameterized==0.6.1 jsondiff==1.1.2 +semver==2.8.1 From 80b1b99d71ef42c26d96f2f2bdd580d198a5a256 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 1 Oct 2018 17:18:48 -0700 Subject: [PATCH 11/12] Add kbn-xsrf header to POST request --- libbeat/tests/system/test_dashboard.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 9fe841e98a6..00d60811163 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -168,7 +168,12 @@ def create_kibana_space(self): "id": "foo-bar", "name": "Foo bar space" } - r = requests.post(url, data) + + headers = { + "kbn-xsrf": "1" + } + + r = requests.post(url, json=data, headers=headers) assert r.status_code == 200 def get_version(self): From 89dbafd40cc3a9da777daebccf6aedf37b9d1fe0 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 1 Oct 2018 17:49:52 -0700 Subject: [PATCH 12/12] Don't recreate space in second space test --- libbeat/tests/system/test_dashboard.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 00d60811163..e76d738a9d5 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -39,7 +39,7 @@ def test_load_dashboard(self): @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') - def test_load_dashboard_into_space(self): + def test_load_dashboard_into_space(self, create_space=True): """ Test loading dashboards into Kibana space """ @@ -49,7 +49,8 @@ def test_load_dashboard_into_space(self): raise SkipTest self.render_config_template() - self.create_kibana_space() + if create_space: + self.create_kibana_space() beat = self.start_beat( logging_args=["-e", "-d", "*"], @@ -133,7 +134,7 @@ def test_export_dashboard_from_space(self): # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available raise SkipTest - self.test_load_dashboard_into_space() + self.test_load_dashboard_into_space(False) path = os.path.normpath(self.beat_path + "/../dev-tools/cmd/dashboards/export_dashboards.go") command = path + " -kibana http://" + self.get_kibana_host() + ":" + self.get_kibana_port()