diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index ff80d070070..1581c93df95 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -53,3 +53,4 @@ The list below covers the major changes between 6.3.0 and master only. `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850] - Set current year in generator templates. {pull}8396[8396] - You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449] +- Add `-space-id` option to `export_dashboards.go` script to support Kibana Spaces {pull}7942[7942] diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 70fa946cd7e..3edfa840779 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -110,6 +110,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Added the `add_process_metadata` processor to enrich events with process information. {pull}6789[6789] - Report number of open file handles on Windows. {pull}8329[8329] - Support for Kafka 2.0.0 in kafka output {pull}8399[8399] +- Add setting `setup.kibana.space.id` to support Kibana Spaces {pull}7942[7942] *Auditbeat* diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml index 56789249281..a3749aa2af0 100644 --- a/auditbeat/auditbeat.yml +++ b/auditbeat/auditbeat.yml @@ -95,6 +95,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using auditbeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go index f208b8e6c16..421e006017d 100644 --- a/dev-tools/cmd/dashboards/export_dashboards.go +++ b/dev-tools/cmd/dashboards/export_dashboards.go @@ -27,6 +27,7 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" "strings" @@ -48,11 +49,14 @@ func makeURL(url, path string, params url.Values) string { return strings.Join([]string{url, path, "?", params.Encode()}, "") } -func Export(client *http.Client, conn string, dashboard string, out string) error { +func Export(client *http.Client, conn string, spaceID string, dashboard string, out string) error { params := url.Values{} params.Add("dashboard", dashboard) + if spaceID != "" { + exportAPI = path.Join("/s", spaceID, exportAPI) + } fullURL := makeURL(conn, exportAPI, params) if !quiet { log.Printf("Calling HTTP GET %v\n", fullURL) @@ -138,6 +142,7 @@ var quiet = false func main() { kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL") + spaceID := flag.String("space-id", "", "Space ID") dashboard := flag.String("dashboard", "", "Dashboard ID") fileOutput := flag.String("output", "output.json", "Output file") ymlFile := flag.String("yml", "", "Path to the module.yml file containing the dashboards") @@ -171,7 +176,7 @@ func main() { if err != nil { log.Fatalf("fail to create directory %s: %v", directory, err) } - err = Export(client, *kibanaURL, dashboard["id"], filepath.Join(directory, dashboard["file"])) + err = Export(client, *kibanaURL, *spaceID, dashboard["id"], filepath.Join(directory, dashboard["file"])) if err != nil { log.Fatalf("fail to export the dashboards: %s", err) } @@ -180,7 +185,7 @@ func main() { } if len(*dashboard) > 0 { - err := Export(client, *kibanaURL, *dashboard, *fileOutput) + err := Export(client, *kibanaURL, *spaceID, *dashboard, *fileOutput) if err != nil { log.Fatalf("fail to export the dashboards: %s", err) } diff --git a/docs/devguide/newdashboards.asciidoc b/docs/devguide/newdashboards.asciidoc index b874dafbfe3..b532655fc20 100644 --- a/docs/devguide/newdashboards.asciidoc +++ b/docs/devguide/newdashboards.asciidoc @@ -283,6 +283,15 @@ go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system ------------------- +===== Export dashboards from a Kibana Space + +If you are using the Kibana Spaces feature and want to export dashboards from a specific Space, pass the Space ID to the `export_dashboards.go` script: + +[source,shell] +------------------- +go run dev-tools/cmd/dashboards/export_dashboards.go -space-id my-space [other-options] +------------------- + ==== Exporting Kibana 5.x dashboards diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 5ae43216b7d..52888b19ea5 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -122,6 +122,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml index d80f7bc0f5f..044616ad816 100644 --- a/heartbeat/heartbeat.yml +++ b/heartbeat/heartbeat.yml @@ -69,6 +69,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using heartbeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/libbeat/_meta/config.yml b/libbeat/_meta/config.yml index 0277cd84fdd..6f460928bac 100644 --- a/libbeat/_meta/config.yml +++ b/libbeat/_meta/config.yml @@ -39,6 +39,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 69b34700ae6..2efa9b0bfa7 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "net/http" "net/url" + "path" "strings" "github.com/pkg/errors" @@ -88,7 +89,11 @@ func NewKibanaClient(cfg *common.Config) (*Client, error) { // NewClientWithConfig creates and returns a kibana client using the given config func NewClientWithConfig(config *ClientConfig) (*Client, error) { - kibanaURL, err := common.MakeURL(config.Protocol, config.Path, config.Host, 5601) + p := config.Path + if config.SpaceID != "" { + p = path.Join(p, "s", config.SpaceID) + } + kibanaURL, err := common.MakeURL(config.Protocol, p, config.Host, 5601) if err != nil { return nil, fmt.Errorf("invalid Kibana host: %v", err) } diff --git a/libbeat/kibana/client_config.go b/libbeat/kibana/client_config.go index 25af90a810c..c5dc101075c 100644 --- a/libbeat/kibana/client_config.go +++ b/libbeat/kibana/client_config.go @@ -28,6 +28,7 @@ type ClientConfig struct { Protocol string `config:"protocol"` Host string `config:"host"` Path string `config:"path"` + SpaceID string `config:"space.id"` Username string `config:"username"` Password string `config:"password"` TLS *tlscommon.Config `config:"ssl"` @@ -39,6 +40,7 @@ var ( Protocol: "http", Host: "localhost:5601", Path: "", + SpaceID: "", Username: "", Password: "", Timeout: 90 * time.Second, diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 743fdf14c66..35eba8977d5 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -29,3 +29,4 @@ urllib3==1.22 websocket-client==0.47.0 parameterized==0.6.1 jsondiff==1.1.2 +semver==2.8.1 diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index e4b6f69f387..e76d738a9d5 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -4,7 +4,8 @@ import subprocess from nose.plugins.attrib import attr import unittest - +import requests +import semver INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -36,6 +37,40 @@ def test_load_dashboard(self): assert self.log_contains("Kibana dashboards successfully loaded") is True + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_load_dashboard_into_space(self, create_space=True): + """ + Test loading dashboards into Kibana space + """ + version = self.get_version() + if semver.compare(version, "6.5.0") == -1: + # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available + raise SkipTest + + self.render_config_template() + if create_space: + self.create_kibana_space() + + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["setup", + "--dashboards", + "-E", "setup.dashboards.file=" + + os.path.join(self.beat_path, "tests", "files", "testbeat-dashboards.zip"), + "-E", "setup.dashboards.beat=testbeat", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-E", "setup.kibana.space.id=foo-bar", + "-E", "output.elasticsearch.hosts=['" + self.get_host() + "']", + "-E", "output.file.enabled=false"] + ) + + beat.check_wait(exit_code=0) + + assert self.log_contains("Kibana dashboards successfully loaded") is True + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') def test_load_only_index_patterns(self): @@ -88,6 +123,36 @@ def test_export_dashboard(self): os.remove("output.json") + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_export_dashboard_from_space(self): + """ + Test export dashboards from Kibana space and remove unsupported characters + """ + version = self.get_version() + if semver.compare(version, "6.5.0") == -1: + # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available + raise SkipTest + + self.test_load_dashboard_into_space(False) + + path = os.path.normpath(self.beat_path + "/../dev-tools/cmd/dashboards/export_dashboards.go") + command = path + " -kibana http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + command = "go run " + command + " -dashboard Metricbeat-system-overview -space-id foo-bar" + + p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + content, err = p.communicate() + + assert p.returncode == 0 + + assert os.path.isfile("output.json") is True + + with open('output.json') as f: + content = f.read() + assert "Metricbeat-system-overview" in content + + os.remove("output.json") + def get_host(self): return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200') @@ -96,3 +161,28 @@ def get_kibana_host(self): def get_kibana_port(self): return os.getenv('KIBANA_PORT', '5601') + + def create_kibana_space(self): + url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \ + "/api/spaces/space" + data = { + "id": "foo-bar", + "name": "Foo bar space" + } + + headers = { + "kbn-xsrf": "1" + } + + r = requests.post(url, json=data, headers=headers) + assert r.status_code == 200 + + def get_version(self): + url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \ + "/api/status" + + r = requests.get(url) + body = r.json() + version = body["version"]["number"] + + return version diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 8363541d426..5bb313ded91 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -66,6 +66,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 79babc2232a..5bdaa0f6911 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -149,6 +149,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using packetbeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index d5c53bbe686..d711c7f10db 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -70,6 +70,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using winlogbeat with the Elastic Cloud (https://cloud.elastic.co/).