From cdc8a17e2dc91c0c28ac7fdeb837ad2248f71430 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Mar 2020 18:01:25 -0700 Subject: [PATCH 01/11] Add TODO --- metricbeat/module/elasticsearch/ccr/ccr.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metricbeat/module/elasticsearch/ccr/ccr.go b/metricbeat/module/elasticsearch/ccr/ccr.go index 20d5cc65779..519010ad8bd 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr.go +++ b/metricbeat/module/elasticsearch/ccr/ccr.go @@ -121,6 +121,8 @@ func (m *MetricSet) checkCCRAvailability(currentElasticsearchVersion *common.Ver return } + // TODO: Check feature availability in ES: GET _xpack, check .features.ccr.available + isAvailable := elastic.IsFeatureAvailable(currentElasticsearchVersion, elasticsearch.CCRStatsAPIAvailableVersion) if !isAvailable { From 79e1b8104aa167732b7749ab3f61c215a5656c3b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Mar 2020 06:51:52 -0700 Subject: [PATCH 02/11] Check if CCR feature is available in ES --- metricbeat/module/elasticsearch/ccr/ccr.go | 7 +++++-- .../module/elasticsearch/ccr/ccr_test.go | 1 + .../module/elasticsearch/elasticsearch.go | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 metricbeat/module/elasticsearch/ccr/ccr_test.go diff --git a/metricbeat/module/elasticsearch/ccr/ccr.go b/metricbeat/module/elasticsearch/ccr/ccr.go index 519010ad8bd..4fce92246f5 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr.go +++ b/metricbeat/module/elasticsearch/ccr/ccr.go @@ -121,9 +121,12 @@ func (m *MetricSet) checkCCRAvailability(currentElasticsearchVersion *common.Ver return } - // TODO: Check feature availability in ES: GET _xpack, check .features.ccr.available + xpack, err := elasticsearch.GetXPack(m.HTTP, m.GetServiceURI()) + if err != nil { + return "", errors.Wrap(err, "error determining xpack features") + } - isAvailable := elastic.IsFeatureAvailable(currentElasticsearchVersion, elasticsearch.CCRStatsAPIAvailableVersion) + isAvailable := xpack.Features.CCR.Available && elastic.IsFeatureAvailable(currentElasticsearchVersion, elasticsearch.CCRStatsAPIAvailableVersion) if !isAvailable { metricsetName := m.FullyQualifiedName() diff --git a/metricbeat/module/elasticsearch/ccr/ccr_test.go b/metricbeat/module/elasticsearch/ccr/ccr_test.go new file mode 100644 index 00000000000..8327612bb77 --- /dev/null +++ b/metricbeat/module/elasticsearch/ccr/ccr_test.go @@ -0,0 +1 @@ +package ccr diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 9698f6ce003..6d431ce314e 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -363,6 +363,27 @@ func GetStackUsage(http *helper.HTTP, resetURI string) (common.MapStr, error) { return stackUsage, err } +type XPack struct { + Features struct { + CCR struct { + Available bool `json:"available"` + } `json:"CCR"` + } `json:"features"` +} + +// GetXPack returns information about xpack features. +func GetXPack(http *helper.HTTP, resetURI string) (XPack, error) { + content, err := fetchPath(http, resetURI, "_xpack", "") + + if err != nil { + return XPack{}, err + } + + var xpack XPack + err = json.Unmarshal(content, &xpack) + return xpack, err +} + // IsMLockAllEnabled returns if the given Elasticsearch node has mlockall enabled func IsMLockAllEnabled(http *helper.HTTP, resetURI, nodeID string) (bool, error) { content, err := fetchPath(http, resetURI, "_nodes/"+nodeID, "filter_path=nodes.*.process.mlockall") From 09650366e2c2b9a0ce63cd10dc1f35994c7a8701 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Mar 2020 06:52:16 -0700 Subject: [PATCH 03/11] Allow tests to disable license caching --- metricbeat/module/elasticsearch/elasticsearch.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 6d431ce314e..95c1641d2f7 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -458,8 +458,13 @@ func MergeClusterSettings(clusterSettings common.MapStr) (common.MapStr, error) return settings, nil } -// Global cache for license information. Assumption is that license information changes infrequently. -var licenseCache = &_licenseCache{} +var ( + // Global cache for license information. Assumption is that license information changes infrequently. + licenseCache = &_licenseCache{} + + // LicenseEnableCaching controls whether license caching is enabled or not. Intended for test use. + LicenseEnableCaching = true +) type _licenseCache struct { sync.RWMutex @@ -481,6 +486,10 @@ func (c *_licenseCache) get() *License { } func (c *_licenseCache) set(license *License, ttl time.Duration) { + if !LicenseEnableCaching { + return + } + c.Lock() defer c.Unlock() From be74b1fa44c75eca8193a6904f860dbe226015f6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Mar 2020 06:52:29 -0700 Subject: [PATCH 04/11] Add unit tests --- .../module/elasticsearch/ccr/ccr_test.go | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/metricbeat/module/elasticsearch/ccr/ccr_test.go b/metricbeat/module/elasticsearch/ccr/ccr_test.go index 8327612bb77..24fc224b29a 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr_test.go +++ b/metricbeat/module/elasticsearch/ccr/ccr_test.go @@ -1 +1,99 @@ package ccr + +import ( + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func startESServer(esVersion, license string, ccrAvailable bool) *httptest.Server { + + nodesLocalHandler := func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"nodes": { "foobar": {}}}`)) + } + clusterStateMasterHandler := func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"master_node": "foobar"}`)) + } + rootHandler := func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + } + w.Write([]byte(`{"version": { "number": "` + esVersion + `" } }`)) + } + licenseHandler := func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{ "license": { "type": "` + license + `" } }`)) + } + xpackHandler := func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{ "features": { "ccr": { "available": ` + strconv.FormatBool(ccrAvailable) + `}}}`)) + } + ccrStatsHandler := func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "this should never have been called", 418) + } + + mux := http.NewServeMux() + mux.Handle("/_nodes/_local/nodes", http.HandlerFunc(nodesLocalHandler)) + mux.Handle("/_cluster/state/master_node", http.HandlerFunc(clusterStateMasterHandler)) + mux.Handle("/", http.HandlerFunc(rootHandler)) + mux.Handle("/_license", http.HandlerFunc(licenseHandler)) // for 7.0 and above + mux.Handle("/_xpack/license", http.HandlerFunc(licenseHandler)) // for before 7.0 + mux.Handle("/_xpack", http.HandlerFunc(xpackHandler)) + mux.Handle("/_ccr/stats", http.HandlerFunc(ccrStatsHandler)) + + return httptest.NewServer(mux) +} + +func TestCCRNotAvailable(t *testing.T) { + tests := map[string]struct { + esVersion string + license string + ccrAvailable bool + }{ + "old_version": { + "6.4.0", + "platinum", + true, + }, + "low_license": { + "7.6.0", + "basic", + true, + }, + "feature_unavailable": { + "7.6.0", + "platinum", + false, + }, + } + + // Disable license caching for these tests + elasticsearch.LicenseEnableCaching = false + defer func() { elasticsearch.LicenseEnableCaching = true }() + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + server := startESServer(test.esVersion, test.license, test.ccrAvailable) + defer server.Close() + + ms := mbtest.NewReportingMetricSetV2Error(t, getConfig(server.URL)) + events, errs := mbtest.ReportingFetchV2Error(ms) + + require.Empty(t, errs) + require.Empty(t, events) + }) + } +} + +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": elasticsearch.ModuleName, + "metricsets": []string{"ccr"}, + "hosts": []string{host}, + } +} From 52bf9d4d60bb0bfb99e58f5ecf676b70be724da6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Mar 2020 07:02:36 -0700 Subject: [PATCH 05/11] Separate message --- metricbeat/module/elasticsearch/ccr/ccr.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/elasticsearch/ccr/ccr.go b/metricbeat/module/elasticsearch/ccr/ccr.go index 4fce92246f5..d9d115a6ed6 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr.go +++ b/metricbeat/module/elasticsearch/ccr/ccr.go @@ -126,7 +126,12 @@ func (m *MetricSet) checkCCRAvailability(currentElasticsearchVersion *common.Ver return "", errors.Wrap(err, "error determining xpack features") } - isAvailable := xpack.Features.CCR.Available && elastic.IsFeatureAvailable(currentElasticsearchVersion, elasticsearch.CCRStatsAPIAvailableVersion) + if !xpack.Features.CCR.Available { + message = "the CCR feature not available on your Elasticsearch cluster." + return + } + + isAvailable := elastic.IsFeatureAvailable(currentElasticsearchVersion, elasticsearch.CCRStatsAPIAvailableVersion) if !isAvailable { metricsetName := m.FullyQualifiedName() From d0570ab685adf97d7a8053ce097bd3b8b91042c6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Mar 2020 07:11:13 -0700 Subject: [PATCH 06/11] Adding license header --- metricbeat/module/elasticsearch/ccr/ccr_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/metricbeat/module/elasticsearch/ccr/ccr_test.go b/metricbeat/module/elasticsearch/ccr/ccr_test.go index 24fc224b29a..2dc09169c92 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr_test.go +++ b/metricbeat/module/elasticsearch/ccr/ccr_test.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package ccr import ( From 13e4b243b66b3d3b22f22c2dd583129120238288 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Mar 2020 07:41:47 -0700 Subject: [PATCH 07/11] Adding CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e1a20eccf9c..5901d744935 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -117,6 +117,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add dashboard for `redisenterprise` module. {pull}16752[16752] - Dynamically choose a method for the system/service metricset to support older linux distros. {pull}16902[16902] - Reduce memory usage in `elasticsearch/index` metricset. {issue}16503[16503] {pull}16538[16538] +- Check if CCR feature is available on Elasticsearch cluster before attempting to call CCR APIs from `elasticsearch/ccr` metricset. {issue}16511[16511] {pull}17073[17073] *Packetbeat* From ef82a63fca4ecd67c6a7fd0ddc4932664a342ee4 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Mar 2020 08:16:21 -0700 Subject: [PATCH 08/11] Check enabled, not available --- metricbeat/module/elasticsearch/ccr/ccr.go | 4 ++-- metricbeat/module/elasticsearch/ccr/ccr_test.go | 12 ++++++------ metricbeat/module/elasticsearch/elasticsearch.go | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/metricbeat/module/elasticsearch/ccr/ccr.go b/metricbeat/module/elasticsearch/ccr/ccr.go index d9d115a6ed6..591f3d12e22 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr.go +++ b/metricbeat/module/elasticsearch/ccr/ccr.go @@ -126,8 +126,8 @@ func (m *MetricSet) checkCCRAvailability(currentElasticsearchVersion *common.Ver return "", errors.Wrap(err, "error determining xpack features") } - if !xpack.Features.CCR.Available { - message = "the CCR feature not available on your Elasticsearch cluster." + if !xpack.Features.CCR.Enabled { + message = "the CCR feature is not enabled on your Elasticsearch cluster." return } diff --git a/metricbeat/module/elasticsearch/ccr/ccr_test.go b/metricbeat/module/elasticsearch/ccr/ccr_test.go index 2dc09169c92..90c1329cc80 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr_test.go +++ b/metricbeat/module/elasticsearch/ccr/ccr_test.go @@ -30,7 +30,7 @@ import ( mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) -func startESServer(esVersion, license string, ccrAvailable bool) *httptest.Server { +func startESServer(esVersion, license string, ccrEnabled bool) *httptest.Server { nodesLocalHandler := func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`{"nodes": { "foobar": {}}}`)) @@ -48,7 +48,7 @@ func startESServer(esVersion, license string, ccrAvailable bool) *httptest.Serve w.Write([]byte(`{ "license": { "type": "` + license + `" } }`)) } xpackHandler := func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{ "features": { "ccr": { "available": ` + strconv.FormatBool(ccrAvailable) + `}}}`)) + w.Write([]byte(`{ "features": { "ccr": { "enabled": ` + strconv.FormatBool(ccrEnabled) + `}}}`)) } ccrStatsHandler := func(w http.ResponseWriter, r *http.Request) { http.Error(w, "this should never have been called", 418) @@ -68,9 +68,9 @@ func startESServer(esVersion, license string, ccrAvailable bool) *httptest.Serve func TestCCRNotAvailable(t *testing.T) { tests := map[string]struct { - esVersion string - license string - ccrAvailable bool + esVersion string + license string + ccrEnabled bool }{ "old_version": { "6.4.0", @@ -95,7 +95,7 @@ func TestCCRNotAvailable(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - server := startESServer(test.esVersion, test.license, test.ccrAvailable) + server := startESServer(test.esVersion, test.license, test.ccrEnabled) defer server.Close() ms := mbtest.NewReportingMetricSetV2Error(t, getConfig(server.URL)) diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 95c1641d2f7..e1a841754fd 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -366,7 +366,7 @@ func GetStackUsage(http *helper.HTTP, resetURI string) (common.MapStr, error) { type XPack struct { Features struct { CCR struct { - Available bool `json:"available"` + Enabled bool `json:"enabled"` } `json:"CCR"` } `json:"features"` } From c16a5e1bebeba54d27d21f05a968ff4ba6e5cfd6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Mar 2020 12:22:19 -0700 Subject: [PATCH 09/11] Renaming var --- metricbeat/module/elasticsearch/ccr/ccr_test.go | 4 ++-- metricbeat/module/elasticsearch/elasticsearch.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metricbeat/module/elasticsearch/ccr/ccr_test.go b/metricbeat/module/elasticsearch/ccr/ccr_test.go index 90c1329cc80..f6d94c739e4 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr_test.go +++ b/metricbeat/module/elasticsearch/ccr/ccr_test.go @@ -90,8 +90,8 @@ func TestCCRNotAvailable(t *testing.T) { } // Disable license caching for these tests - elasticsearch.LicenseEnableCaching = false - defer func() { elasticsearch.LicenseEnableCaching = true }() + elasticsearch.LicenseCacheEnabled = false + defer func() { elasticsearch.LicenseCacheEnabled = true }() for name, test := range tests { t.Run(name, func(t *testing.T) { diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index e1a841754fd..45e8ab17a23 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -462,8 +462,8 @@ var ( // Global cache for license information. Assumption is that license information changes infrequently. licenseCache = &_licenseCache{} - // LicenseEnableCaching controls whether license caching is enabled or not. Intended for test use. - LicenseEnableCaching = true + // LicenseCacheEnabled controls whether license caching is enabled or not. Intended for test use. + LicenseCacheEnabled = true ) type _licenseCache struct { @@ -486,7 +486,7 @@ func (c *_licenseCache) get() *License { } func (c *_licenseCache) set(license *License, ttl time.Duration) { - if !LicenseEnableCaching { + if !LicenseCacheEnabled { return } From 0ae344c74b8751af79bb0f62a9cb7b95b98f298c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Mar 2020 12:36:45 -0700 Subject: [PATCH 10/11] Adding documentation --- metricbeat/module/elasticsearch/ccr/_meta/docs.asciidoc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/elasticsearch/ccr/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/ccr/_meta/docs.asciidoc index e38b8702e5f..83c8be226f2 100644 --- a/metricbeat/module/elasticsearch/ccr/_meta/docs.asciidoc +++ b/metricbeat/module/elasticsearch/ccr/_meta/docs.asciidoc @@ -1,3 +1,8 @@ This is the `ccr` metricset of the Elasticsearch module. It interrogates the -Cross Cluster Replication Stats API endpoint to fetch information about shards -in the Elasticsearch cluster that are participating in cross-cluster replication. +Cross-Cluster Replication Stats API endpoint to fetch metrics about cross-cluster +replication from the Elasticsearch clusters that are participating in cross-cluster +replication. + +If the Elasticserach cluster does not have the cross-cluster replication feature +enabled, this metricset will not collect any metrics. A DEBUG log message about this +will be emitted in the Metricbeat log. From cafb9c6c8eb93fbed4db18e6342ca02dce56f01b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Mar 2020 12:58:41 -0700 Subject: [PATCH 11/11] Doc feedback fixes --- .../module/elasticsearch/ccr/_meta/docs.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metricbeat/module/elasticsearch/ccr/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/ccr/_meta/docs.asciidoc index 83c8be226f2..04642b5f101 100644 --- a/metricbeat/module/elasticsearch/ccr/_meta/docs.asciidoc +++ b/metricbeat/module/elasticsearch/ccr/_meta/docs.asciidoc @@ -1,8 +1,8 @@ -This is the `ccr` metricset of the Elasticsearch module. It interrogates the +This is the `ccr` metricset of the {es} module. It uses the Cross-Cluster Replication Stats API endpoint to fetch metrics about cross-cluster -replication from the Elasticsearch clusters that are participating in cross-cluster +replication from the {es} clusters that are participating in cross-cluster replication. -If the Elasticserach cluster does not have the cross-cluster replication feature -enabled, this metricset will not collect any metrics. A DEBUG log message about this -will be emitted in the Metricbeat log. +If the {es} cluster does not have cross-cluster replication enabled, this metricset +will not collect metrics. A DEBUG log message about this will be emitted in the +Metricbeat log.