diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 61c87ca1364..5c892231068 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -251,6 +251,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add more info to message logged when a duplicated symlink file is found {pull}10845[10845] - Add Netflow module to enrich flow events with geoip data. {pull}10877[10877] - Set `event.category: network_traffic` for Suricata. {pull}10882[10882] +- Add configuration knob for auto-discover hints to control whether log harvesting is enabled for the pod/container. {issue}10811[10811] {pull}10911[10911] *Heartbeat* diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index de56e777765..e9e1afa7224 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -47,6 +47,18 @@ func TestGenerateHints(t *testing.T) { len: 0, result: common.MapStr{}, }, + { + msg: "Hints with logs.disable should return nothing", + event: bus.Event{ + "hints": common.MapStr{ + "logs": common.MapStr{ + "disable": "true", + }, + }, + }, + len: 0, + result: common.MapStr{}, + }, { msg: "Empty event hints should return default config", event: bus.Event{ diff --git a/filebeat/docs/autodiscover-hints.asciidoc b/filebeat/docs/autodiscover-hints.asciidoc index a49a8abd055..9d80d28c7ad 100644 --- a/filebeat/docs/autodiscover-hints.asciidoc +++ b/filebeat/docs/autodiscover-hints.asciidoc @@ -94,6 +94,25 @@ filebeat.autodiscover: hints.enabled: true ------------------------------------------------------------------------------------- +Autodiscover provides a way to control whether log harvesting is by default disabled for the pods/containers when auto-discovery is in use. To enable it, just set `default.disable` to true: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +filebeat.autodiscover: + providers: + - type: kubernetes + hints.enabled: true + default.disable: true +------------------------------------------------------------------------------------- + +Then, for the pods/containers that log harvesting should be enabled, you can annotate with hint: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +annotations: + co.elastic.logs/disable: false +------------------------------------------------------------------------------------- + You can annotate Kubernetes Pods with useful info to spin up {beatname_uc} inputs or modules: ["source","yaml",subs="attributes"] @@ -137,6 +156,26 @@ filebeat.autodiscover: hints.enabled: true ------------------------------------------------------------------------------------- +Autodiscover provides a way to control whether log harvesting is by default disabled for the +containers when auto-discovery is in use. To enable it, just set `default.disable` to true: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +filebeat.autodiscover: + providers: + - type: docker + hints.enabled: true + default.disable: true +------------------------------------------------------------------------------------- + +Then, for the containers that log harvesting should be enabled, you can label Docker containers with: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +annotations: + co.elastic.logs/disable: false +------------------------------------------------------------------------------------- + You can label Docker containers with useful info to spin up {beatname_uc} inputs, for example: ["source","yaml",subs="attributes"] diff --git a/libbeat/autodiscover/builder/helper.go b/libbeat/autodiscover/builder/helper.go index e8a81cfd869..3a574675b91 100644 --- a/libbeat/autodiscover/builder/helper.go +++ b/libbeat/autodiscover/builder/helper.go @@ -155,7 +155,7 @@ func IsNoOp(hints common.MapStr, key string) bool { } // GenerateHints parses annotations based on a prefix and sets up hints that can be picked up by individual Beats. -func GenerateHints(annotations common.MapStr, container, prefix string) common.MapStr { +func GenerateHints(annotations common.MapStr, container, prefix string, defaultDisable bool) common.MapStr { hints := common.MapStr{} if rawEntries, err := annotations.GetValue(prefix); err == nil { if entries, ok := rawEntries.(common.MapStr); ok { @@ -195,5 +195,10 @@ func GenerateHints(annotations common.MapStr, container, prefix string) common.M } } + // Update hints: if .disabled annotation does not exist, set according to disabledByDefault flag + if _, err := hints.GetValue("logs.disable"); err != nil && defaultDisable { + hints.Put("logs.disable", "true") + } + return hints } diff --git a/libbeat/autodiscover/builder/helper_test.go b/libbeat/autodiscover/builder/helper_test.go index 477888b93f0..c282972925e 100644 --- a/libbeat/autodiscover/builder/helper_test.go +++ b/libbeat/autodiscover/builder/helper_test.go @@ -27,13 +27,15 @@ import ( func TestGenerateHints(t *testing.T) { tests := []struct { - annotations map[string]string - result common.MapStr + annotations map[string]string + defaultDisable bool + result common.MapStr }{ // Empty annotations should return empty hints { - annotations: map[string]string{}, - result: common.MapStr{}, + annotations: map[string]string{}, + defaultDisable: false, + result: common.MapStr{}, }, // Scenarios being tested: @@ -50,6 +52,7 @@ func TestGenerateHints(t *testing.T) { "co.elastic.metrics.foobar1/period": "15s", "not.to.include": "true", }, + defaultDisable: false, result: common.MapStr{ "logs": common.MapStr{ "multiline": common.MapStr{ @@ -62,6 +65,67 @@ func TestGenerateHints(t *testing.T) { }, }, }, + // Scenarios being tested: + // logs.disable must be generated when defaultDisable is set and annotations does not + // have co.elastic.logs/disable set to false. + // logs/multiline.pattern must be a nested common.MapStr under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // period is annotated at both container and pod level. Container level value must be in hints + { + annotations: map[string]string{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "not.to.include": "true", + }, + defaultDisable: true, + result: common.MapStr{ + "logs": common.MapStr{ + "multiline": common.MapStr{ + "pattern": "^test", + }, + "disable": "true", + }, + "metrics": common.MapStr{ + "module": "prometheus", + "period": "15s", + }, + }, + }, + // Scenarios being tested: + // logs.disable must not be generated when defaultDisable is set, but annotations + // have co.elastic.logs/disable set to false. + // logs/multiline.pattern must be a nested common.MapStr under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // period is annotated at both container and pod level. Container level value must be in hints + { + annotations: map[string]string{ + "co.elastic.logs/disable": "false", + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "not.to.include": "true", + }, + defaultDisable: true, + result: common.MapStr{ + "logs": common.MapStr{ + "multiline": common.MapStr{ + "pattern": "^test", + }, + "disable": "false", + }, + "metrics": common.MapStr{ + "module": "prometheus", + "period": "15s", + }, + }, + }, } for _, test := range tests { @@ -69,6 +133,6 @@ func TestGenerateHints(t *testing.T) { for k, v := range test.annotations { annMap.Put(k, v) } - assert.Equal(t, GenerateHints(annMap, "foobar", "co.elastic"), test.result) + assert.Equal(t, GenerateHints(annMap, "foobar", "co.elastic", test.defaultDisable), test.result) } } diff --git a/libbeat/autodiscover/providers/docker/config.go b/libbeat/autodiscover/providers/docker/config.go index 02c5fffe936..bba4f32cdb0 100644 --- a/libbeat/autodiscover/providers/docker/config.go +++ b/libbeat/autodiscover/providers/docker/config.go @@ -25,14 +25,15 @@ import ( // Config for docker autodiscover provider type Config struct { - Host string `config:"host"` - TLS *docker.TLSConfig `config:"ssl"` - Prefix string `config:"prefix"` - HintsEnabled bool `config:"hints.enabled"` - Builders []*common.Config `config:"builders"` - Appenders []*common.Config `config:"appenders"` - Templates template.MapperSettings `config:"templates"` - Dedot bool `config:"labels.dedot"` + Host string `config:"host"` + TLS *docker.TLSConfig `config:"ssl"` + Prefix string `config:"prefix"` + HintsEnabled bool `config:"hints.enabled"` + DefaultDisable bool `config:"default.disable"` + Builders []*common.Config `config:"builders"` + Appenders []*common.Config `config:"appenders"` + Templates template.MapperSettings `config:"templates"` + Dedot bool `config:"labels.dedot"` } func defaultConfig() *Config { diff --git a/libbeat/autodiscover/providers/docker/docker.go b/libbeat/autodiscover/providers/docker/docker.go index dea344c3df7..fb0901e55e5 100644 --- a/libbeat/autodiscover/providers/docker/docker.go +++ b/libbeat/autodiscover/providers/docker/docker.go @@ -267,7 +267,7 @@ func (d *Provider) generateHints(event bus.Event) bus.Event { e["port"] = port } if labels, err := dockerMeta.GetValue("labels"); err == nil { - hints := builder.GenerateHints(labels.(common.MapStr), "", d.config.Prefix) + hints := builder.GenerateHints(labels.(common.MapStr), "", d.config.Prefix, d.config.DefaultDisable) e["hints"] = hints } return e diff --git a/libbeat/autodiscover/providers/kubernetes/config.go b/libbeat/autodiscover/providers/kubernetes/config.go index 5ba5f789f68..269cb60cff3 100644 --- a/libbeat/autodiscover/providers/kubernetes/config.go +++ b/libbeat/autodiscover/providers/kubernetes/config.go @@ -33,11 +33,12 @@ type Config struct { SyncPeriod time.Duration `config:"sync_period"` CleanupTimeout time.Duration `config:"cleanup_timeout"` - Prefix string `config:"prefix"` - HintsEnabled bool `config:"hints.enabled"` - Builders []*common.Config `config:"builders"` - Appenders []*common.Config `config:"appenders"` - Templates template.MapperSettings `config:"templates"` + Prefix string `config:"prefix"` + HintsEnabled bool `config:"hints.enabled"` + DefaultDisable bool `config:"default.disable"` + Builders []*common.Config `config:"builders"` + Appenders []*common.Config `config:"appenders"` + Templates template.MapperSettings `config:"templates"` } func defaultConfig() *Config { diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index c3d9a03dda4..41037238edd 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -275,12 +275,13 @@ func (p *Provider) generateHints(event bus.Event) bus.Event { } cname := builder.GetContainerName(container) - hints := builder.GenerateHints(annotations, cname, p.config.Prefix) + hints := builder.GenerateHints(annotations, cname, p.config.Prefix, p.config.DefaultDisable) + logp.Debug("kubernetes", "Generated hints %+v", hints) if len(hints) != 0 { e["hints"] = hints } - logp.Debug("kubernetes", "Generated builder event %v", event) + logp.Debug("kubernetes", "Generated builder event %+v", e) return e } diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 0d0fd4fbc2a..a119227ca51 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.0.0-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:7.1.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] retries: 300 @@ -16,7 +16,7 @@ services: - "xpack.security.enabled=false" logstash: - image: docker.elastic.co/logstash/logstash:7.0.0-SNAPSHOT + image: docker.elastic.co/logstash/logstash:7.1.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -26,7 +26,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.0.0-SNAPSHOT + image: docker.elastic.co/kibana/kibana:7.1.0-SNAPSHOT healthcheck: test: ["CMD-SHELL", 'python -c ''import urllib, json; response = urllib.urlopen("http://localhost:5601/api/status"); data = json.loads(response.read()); exit(1) if data["status"]["overall"]["state"] != "green" else exit(0);'''] retries: 600