Module configuration variants

CHANGELOG.asciidoc

@@ -226,7 +226,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...v7.0.0-beta1[Check the
 - The `elasticsearch/deprecation` fileset now indexes the `component` field under `elasticsearch` instead of `elasticsearch.server`. {pull}10445[10445]
 - Remove field `kafka.log.trace.full` from kafka.log fielset. {pull}10398[10398]
 - Change field `kafka.log.class` for kafka.log fileset from text to keyword. {pull}10398[10398]
-- Address add_kubernetes_metadata processor issue where old source field is 
+- Address add_kubernetes_metadata processor issue where old source field is
   still used for matcher. {issue}10505[10505] {pull}10506[10506]
 - Change type of haproxy.source from text to keyword. {pull}10506[10506]
 - Rename `event.type` to `suricata.eve.event_type` in Suricata module because event.type is reserved for future use by ECS. {pull}10575[10575]

CHANGELOG.next.asciidoc

@@ -58,7 +58,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve detection of file deletion on Windows. {pull}10747[10747]
 - Fix goroutine leak happening when harvesters are dynamically stopped. {pull}11263[11263]
 - Fix `add_docker_metadata` source matching, using `log.file.path` field now. {pull}11577[11577]
-- Add missing Kubernetes metadata fields to Filebeat CoreDNS module, and fix a documentation error. {pull}11591[11591] 
+- Add missing Kubernetes metadata fields to Filebeat CoreDNS module, and fix a documentation error. {pull}11591[11591]
 
 *Heartbeat*
 
@@ -117,6 +117,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add s3_daily_storage metricset. {pull}10940[10940] {issue}10055[10055]
 - Add `coredns` metricbeat module. {pull}10585[10585]
 - The `elasticsearch.index` metricset (with `xpack.enabled: true`) now collects `refresh.external_total_time_in_millis` fields from Elasticsearch. {pull}11616[11616]
+- Allow module configurations to have variants {pull}9118[9118]
 
 *Packetbeat*
 

libbeat/cfgfile/glob_manager.go

@@ -20,6 +20,7 @@ package cfgfile
 import (
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -110,6 +111,7 @@ func (g *GlobManager) ListEnabled() []*CfgFile {
 		}
 	}
 
+	sort.Sort(byCfgFileDisplayNames(enabled))
 	return enabled
 }
 
@@ -122,6 +124,7 @@ func (g *GlobManager) ListDisabled() []*CfgFile {
 		}
 	}
 
+	sort.Sort(byCfgFileDisplayNames(disabled))
 	return disabled
 }
 
@@ -182,3 +185,31 @@ func (g *GlobManager) Disable(name string) error {
 
 	return errors.Errorf("module %s not found", name)
 }
+
+// For sorting config files in the desired order, so variants will
+// show up after the default, e.g. elasticsearch-xpack will show up
+// after elasticsearch.
+type byCfgFileDisplayNames []*CfgFile
+
+func (s byCfgFileDisplayNames) Len() int {
+	return len(s)
+}
+
+func (s byCfgFileDisplayNames) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s byCfgFileDisplayNames) Less(i, j int) bool {
+	namei := s[i].Name
+	namej := s[j].Name
+
+	if strings.HasPrefix(namei, namej) {
+		// namei starts with namej, so namei is longer and we want it to come after namej
+		return false
+	} else if strings.HasPrefix(namej, namei) {
+		// namej starts with namei, so namej is longer and we want it to come after namei
+		return true
+	} else {
+		return namei < namej
+	}
+}

libbeat/cfgfile/glob_manager_test.go

@@ -47,6 +47,8 @@ func TestGlobManager(t *testing.T) {
 	assert.NoError(t, err)
 	err = ioutil.WriteFile(dir+"/config2.yml", content, 0644)
 	assert.NoError(t, err)
+	err = ioutil.WriteFile(dir+"/config2-alt.yml.disabled", content, 0644)
+	assert.NoError(t, err)
 	err = ioutil.WriteFile(dir+"/config3.yml.disabled", content, 0644)
 	assert.NoError(t, err)
 
@@ -59,23 +61,25 @@ func TestGlobManager(t *testing.T) {
 
 	assert.True(t, manager.Exists("config1"))
 	assert.True(t, manager.Exists("config2"))
+	assert.True(t, manager.Exists("config2-alt"))
 	assert.True(t, manager.Exists("config3"))
 	assert.False(t, manager.Exists("config4"))
 
 	assert.True(t, manager.Enabled("config1"))
 	assert.True(t, manager.Enabled("config2"))
+	assert.False(t, manager.Enabled("config2-alt"))
 	assert.False(t, manager.Enabled("config3"))
 
 	assert.Equal(t, len(manager.ListEnabled()), 2)
-	assert.Equal(t, len(manager.ListDisabled()), 1)
+	assert.Equal(t, len(manager.ListDisabled()), 2)
 
 	// Test disable
 	if err = manager.Disable("config2"); err != nil {
 		t.Fatal(err)
 	}
 
 	assert.Equal(t, len(manager.ListEnabled()), 1)
-	assert.Equal(t, len(manager.ListDisabled()), 2)
+	assert.Equal(t, len(manager.ListDisabled()), 3)
 
 	enabled := manager.ListEnabled()
 	assert.Equal(t, enabled[0].Name, "config1")
@@ -87,11 +91,13 @@ func TestGlobManager(t *testing.T) {
 	}
 
 	assert.Equal(t, len(manager.ListEnabled()), 2)
-	assert.Equal(t, len(manager.ListDisabled()), 1)
+	assert.Equal(t, len(manager.ListDisabled()), 2)
 
 	disabled := manager.ListDisabled()
 	assert.Equal(t, disabled[0].Name, "config2")
 	assert.Equal(t, disabled[0].Enabled, false)
+	assert.Equal(t, disabled[1].Name, "config2-alt")
+	assert.Equal(t, disabled[1].Enabled, false)
 
 	// Check correct files layout:
 	files, err := filepath.Glob(dir + "/*")
@@ -101,7 +107,32 @@ func TestGlobManager(t *testing.T) {
 
 	assert.Equal(t, files, []string{
 		filepath.Join(dir, "config1.yml"),
+		filepath.Join(dir, "config2-alt.yml.disabled"),
 		filepath.Join(dir, "config2.yml.disabled"),
 		filepath.Join(dir, "config3.yml"),
 	})
 }
+
+func TestCfgFileSorting(t *testing.T) {
+	cfgFiles := byCfgFileDisplayNames{
+		&CfgFile{
+			"foo",
+			"modules.d/foo.yml",
+			false,
+		},
+		&CfgFile{
+			"foo-variant",
+			"modules.d/foo-variant.yml",
+			false,
+		},
+		&CfgFile{
+			"fox",
+			"modules.d/fox.yml",
+			false,
+		},
+	}
+
+	assert.True(t, cfgFiles.Less(0, 1))
+	assert.False(t, cfgFiles.Less(1, 0))
+	assert.True(t, cfgFiles.Less(0, 2))
+}

metricbeat/docs/metricbeat-options.asciidoc

@@ -113,6 +113,20 @@ metricbeat.modules:
   period: 30s
 ----
 
+[float]
+[[config-combos]]
+== Configuration variants
+
+Every module come with a default configuration file. Some modules also come with
+one or more variant configuration files containing common alternative configurations
+for that module.
+
+When you see the list of enabled and disabled modules, those modules with configuration
+variants will be shown as `<module_name>-<variant_name>`. You can enable or disable
+specific configuration variants of a module by specifying `metricbeat modules enable
+<module_name>-<variant_name>` and `metricbeat modules disable <module_name>-<variant_name>`
+respectively.
+
 [float]
 [[config-combos]]
 == Configuration combinations

metricbeat/module/elasticsearch/_meta/config-xpack.yml

@@ -0,0 +1,16 @@
+- module: elasticsearch
+  metricsets:
+    - ccr
+    - cluster_stats
+    - index
+    - index_recovery
+    - index_summary
+    - ml_job
+    - node_stats
+    - shard
+  period: 10s
+  hosts: ["http://localhost:9200"]
+  #username: "user"
+  #password: "secret"
+  xpack.enabled: true
+

metricbeat/module/kibana/_meta/config-xpack.yml

@@ -0,0 +1,9 @@
+- module: kibana
+  metricsets:
+    - stats
+  period: 10s
+  hosts: ["localhost:5601"]
+  #basepath: ""
+  #username: "user"
+  #password: "secret"
+  xpack.enabled: true

metricbeat/modules.d/elasticsearch-xpack.yml.disabled

@@ -0,0 +1,19 @@
+# Module: elasticsearch
+# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-elasticsearch.html
+
+- module: elasticsearch
+  metricsets:
+    - ccr
+    - cluster_stats
+    - index
+    - index_recovery
+    - index_summary
+    - ml_job
+    - node_stats
+    - shard
+  period: 10s
+  hosts: ["http://localhost:9200"]
+  #username: "user"
+  #password: "secret"
+  xpack.enabled: true
+

metricbeat/modules.d/kibana-xpack.yml.disabled

@@ -0,0 +1,12 @@
+# Module: kibana
+# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-kibana.html
+
+- module: kibana
+  metricsets:
+    - stats
+  period: 10s
+  hosts: ["localhost:5601"]
+  #basepath: ""
+  #username: "user"
+  #password: "secret"
+  xpack.enabled: true