diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 09d610d3182..b0ea043ca90 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -124,6 +124,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix the `translate_sid` processor's handling of unconfigured target fields. {issue}18990[18990] {pull}18991[18991] - Fixed a service restart failure under Windows. {issue}18914[18914] {pull}18916[18916] - The `monitoring.elasticsearch.api_key` value is correctly base64-encoded before being sent to the monitoring Elasticsearch cluster. {issue}18939[18939] {pull}18945[18945] +- Fix kafka topic setting not allowing upper case characters. {pull}18854[18854] {issue}18640[18640] +- Fix redis key setting not allowing upper case characters. {pull}18854[18854] {issue}18640[18640] *Auditbeat* diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index 06a56646807..9aab5487301 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -20,6 +20,7 @@ package idxmgmt import ( "errors" "fmt" + "strings" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/beat/events" @@ -198,6 +199,7 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector, MultiKey: "indices", EnableSingleOnly: true, FailEmpty: mode != ilm.ModeEnabled, + Case: outil.SelectorLowerCase, } indexSel, err := outil.BuildSelectorFromConfig(selCfg, buildSettings) @@ -354,13 +356,13 @@ func getEventCustomIndex(evt *beat.Event, beatInfo beat.Info) string { } if alias, err := events.GetMetaStringValue(*evt, events.FieldMetaAlias); err == nil { - return alias + return strings.ToLower(alias) } if idx, err := events.GetMetaStringValue(*evt, events.FieldMetaIndex); err == nil { ts := evt.Timestamp.UTC() return fmt.Sprintf("%s-%d.%02d.%02d", - idx, ts.Year(), ts.Month(), ts.Day()) + strings.ToLower(idx), ts.Year(), ts.Month(), ts.Day()) } // This is functionally identical to Meta["alias"], returning the overriding @@ -368,7 +370,7 @@ func getEventCustomIndex(evt *beat.Event, beatInfo beat.Info) string { // to send the index for particular inputs to formatted string templates, // which are then expanded by a processor to the "raw_index" field. if idx, err := events.GetMetaStringValue(*evt, events.FieldMetaRawIndex); err == nil { - return idx + return strings.ToLower(idx) } return "" diff --git a/libbeat/idxmgmt/std_test.go b/libbeat/idxmgmt/std_test.go index ea23a53fd84..3e934d78d58 100644 --- a/libbeat/idxmgmt/std_test.go +++ b/libbeat/idxmgmt/std_test.go @@ -139,6 +139,11 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, want: stable("test-9.9.9"), }, + "without ilm must be lowercase": { + ilmCalls: noILM, + cfg: map[string]interface{}{"index": "TeSt-%{[agent.version]}"}, + want: stable("test-9.9.9"), + }, "event alias without ilm": { ilmCalls: noILM, cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, @@ -147,6 +152,14 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { "alias": "test", }, }, + "event alias without ilm must be lowercae": { + ilmCalls: noILM, + cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, + want: stable("test"), + meta: common.MapStr{ + "alias": "Test", + }, + }, "event index without ilm": { ilmCalls: noILM, cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, @@ -155,11 +168,24 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { "index": "test", }, }, + "event index without ilm must be lowercase": { + ilmCalls: noILM, + cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, + want: dateIdx("test"), + meta: common.MapStr{ + "index": "Test", + }, + }, "with ilm": { ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), cfg: map[string]interface{}{"index": "wrong-%{[agent.version]}"}, want: stable("test-9.9.9"), }, + "with ilm must be lowercase": { + ilmCalls: ilmTemplateSettings("Test-9.9.9", "Test-9.9.9"), + cfg: map[string]interface{}{"index": "wrong-%{[agent.version]}"}, + want: stable("test-9.9.9"), + }, "event alias wit ilm": { ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, @@ -168,6 +194,14 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { "alias": "event-alias", }, }, + "event alias wit ilm must be lowercase": { + ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), + cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, + want: stable("event-alias"), + meta: common.MapStr{ + "alias": "Event-alias", + }, + }, "event index with ilm": { ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, @@ -186,6 +220,16 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { }, want: stable("myindex"), }, + "use indices settings must be lowercase": { + ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), + cfg: map[string]interface{}{ + "index": "test-%{[agent.version]}", + "indices": []map[string]interface{}{ + {"index": "MyIndex"}, + }, + }, + want: stable("myindex"), + }, } for name, test := range cases { t.Run(name, func(t *testing.T) { diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 74e07f42b97..bd6f9ee7ad0 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "net/http" + "strings" "time" "go.elastic.co/apm" @@ -356,7 +357,7 @@ func getPipeline(event *beat.Event, pipelineSel *outil.Selector) (string, error) return "", errors.New("pipeline metadata is no string") } - return pipeline, nil + return strings.ToLower(pipeline), nil } if pipelineSel != nil { diff --git a/libbeat/outputs/elasticsearch/client_proxy_test.go b/libbeat/outputs/elasticsearch/client_proxy_test.go index b6751860e0a..1e368d234ea 100644 --- a/libbeat/outputs/elasticsearch/client_proxy_test.go +++ b/libbeat/outputs/elasticsearch/client_proxy_test.go @@ -190,7 +190,7 @@ func doClientPing(t *testing.T) { Headers: map[string]string{headerTestField: headerTestValue}, ProxyDisable: proxyDisable != "", }, - Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), + Index: outil.MakeSelector(outil.ConstSelectorExpr("test", outil.SelectorLowerCase)), } if proxy != "" { proxyURL, err := url.Parse(proxy) diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index db152bf9045..bd28fe5850b 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -228,7 +228,7 @@ func TestClientWithHeaders(t *testing.T) { "X-Test": "testing value", }, }, - Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), + Index: outil.MakeSelector(outil.ConstSelectorExpr("test", outil.SelectorLowerCase)), }, nil) assert.NoError(t, err) diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index 512b74895ea..bf1f9bd378e 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -133,12 +133,7 @@ func buildSelectors( return index, pipeline, err } - pipelineSel, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ - Key: "pipeline", - MultiKey: "pipelines", - EnableSingleOnly: true, - FailEmpty: false, - }) + pipelineSel, err := buildPipelineSelector(cfg) if err != nil { return index, pipeline, err } @@ -149,3 +144,13 @@ func buildSelectors( return index, pipeline, err } + +func buildPipelineSelector(cfg *common.Config) (outil.Selector, error) { + return outil.BuildSelectorFromConfig(cfg, outil.Settings{ + Key: "pipeline", + MultiKey: "pipelines", + EnableSingleOnly: true, + FailEmpty: false, + Case: outil.SelectorLowerCase, + }) +} diff --git a/libbeat/outputs/elasticsearch/elasticsearch_test.go b/libbeat/outputs/elasticsearch/elasticsearch_test.go index 60268b59602..df757d570dd 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch_test.go +++ b/libbeat/outputs/elasticsearch/elasticsearch_test.go @@ -21,6 +21,8 @@ import ( "fmt" "testing" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" ) @@ -73,3 +75,59 @@ func TestGlobalConnectCallbacksManagement(t *testing.T) { t.Fatalf("third callback cannot be retrieved") } } + +func TestPipelineSelection(t *testing.T) { + cases := map[string]struct { + cfg map[string]interface{} + event beat.Event + want string + }{ + "no pipline configured": {}, + "pipeline configured": { + cfg: map[string]interface{}{"pipeline": "test"}, + want: "test", + }, + "pipeline must be lowercase": { + cfg: map[string]interface{}{"pipeline": "Test"}, + want: "test", + }, + "pipeline via event meta": { + event: beat.Event{Meta: common.MapStr{"pipeline": "test"}}, + want: "test", + }, + "pipeline via event meta must be lowercase": { + event: beat.Event{Meta: common.MapStr{"pipeline": "Test"}}, + want: "test", + }, + "pipelines setting": { + cfg: map[string]interface{}{ + "pipelines": []map[string]interface{}{{"pipeline": "test"}}, + }, + want: "test", + }, + "pipelines setting must be lowercase": { + cfg: map[string]interface{}{ + "pipelines": []map[string]interface{}{{"pipeline": "Test"}}, + }, + want: "test", + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + selector, err := buildPipelineSelector(common.MustNewConfigFrom(test.cfg)) + if err != nil { + t.Fatalf("Failed to parse configuration: %v", err) + } + + got, err := getPipeline(&test.event, &selector) + if err != nil { + t.Fatalf("Failed to create pipeline name: %v", err) + } + + if test.want != got { + t.Errorf("Pipeline name missmatch (want: %v, got: %v)", test.want, got) + } + }) + } +} diff --git a/libbeat/outputs/kafka/config_test.go b/libbeat/outputs/kafka/config_test.go index 3775cafd47b..816cb8f03e8 100644 --- a/libbeat/outputs/kafka/config_test.go +++ b/libbeat/outputs/kafka/config_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/internal/testutil" "github.com/elastic/beats/v7/libbeat/logp" @@ -131,3 +132,64 @@ func TestBackoffFunc(t *testing.T) { }) } } + +func TestTopicSelection(t *testing.T) { + cases := map[string]struct { + cfg map[string]interface{} + event beat.Event + want string + }{ + "topic configured": { + cfg: map[string]interface{}{"topic": "test"}, + want: "test", + }, + "topic must keep case": { + cfg: map[string]interface{}{"topic": "Test"}, + want: "Test", + }, + "topics setting": { + cfg: map[string]interface{}{ + "topics": []map[string]interface{}{{"topic": "test"}}, + }, + want: "test", + }, + "topics setting must keep case": { + cfg: map[string]interface{}{ + "topics": []map[string]interface{}{{"topic": "Test"}}, + }, + want: "Test", + }, + "use event field": { + cfg: map[string]interface{}{"topic": "test-%{[field]}"}, + event: beat.Event{ + Fields: common.MapStr{"field": "from-event"}, + }, + want: "test-from-event", + }, + "use event field must keep case": { + cfg: map[string]interface{}{"topic": "Test-%{[field]}"}, + event: beat.Event{ + Fields: common.MapStr{"field": "From-Event"}, + }, + want: "Test-From-Event", + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + selector, err := buildTopicSelector(common.MustNewConfigFrom(test.cfg)) + if err != nil { + t.Fatalf("Failed to parse configuration: %v", err) + } + + got, err := selector.Select(&test.event) + if err != nil { + t.Fatalf("Failed to create topic name: %v", err) + } + + if test.want != got { + t.Errorf("Pipeline name missmatch (want: %v, got: %v)", test.want, got) + } + }) + } +} diff --git a/libbeat/outputs/kafka/kafka.go b/libbeat/outputs/kafka/kafka.go index b8c6c4dcfff..9be3970b1c4 100644 --- a/libbeat/outputs/kafka/kafka.go +++ b/libbeat/outputs/kafka/kafka.go @@ -66,12 +66,7 @@ func makeKafka( return outputs.Fail(err) } - topic, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ - Key: "topic", - MultiKey: "topics", - EnableSingleOnly: true, - FailEmpty: true, - }) + topic, err := buildTopicSelector(cfg) if err != nil { return outputs.Fail(err) } @@ -102,3 +97,13 @@ func makeKafka( } return outputs.Success(config.BulkMaxSize, retry, client) } + +func buildTopicSelector(cfg *common.Config) (outil.Selector, error) { + return outil.BuildSelectorFromConfig(cfg, outil.Settings{ + Key: "topic", + MultiKey: "topics", + EnableSingleOnly: true, + FailEmpty: true, + Case: outil.SelectorKeepCase, + }) +} diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index c0969358dfb..9fdbd11d875 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -94,7 +94,7 @@ func esConnect(t *testing.T, index string) *esConnection { host := getElasticsearchHost() indexFmt := fmtstr.MustCompileEvent(fmt.Sprintf("%s-%%{+yyyy.MM.dd}", index)) - indexFmtExpr, _ := outil.FmtSelectorExpr(indexFmt, "") + indexFmtExpr, _ := outil.FmtSelectorExpr(indexFmt, "", outil.SelectorLowerCase) indexSel := outil.MakeSelector(indexFmtExpr) index, _ = indexSel.Select(&beat.Event{ Timestamp: ts, diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go index 6ff629c88e7..1615a3bdb11 100644 --- a/libbeat/outputs/outil/select.go +++ b/libbeat/outputs/outil/select.go @@ -19,7 +19,6 @@ package outil import ( "fmt" - "strings" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" @@ -33,22 +32,6 @@ type Selector struct { sel SelectorExpr } -// Settings configures how BuildSelectorFromConfig creates a Selector from -// a given configuration object. -type Settings struct { - // single selector key and default option keyword - Key string - - // multi-selector key in config - MultiKey string - - // if enabled a selector `key` in config will be generated, if `key` is present - EnableSingleOnly bool - - // Fail building selector if `key` and `multiKey` are missing - FailEmpty bool -} - // SelectorExpr represents an expression object that can be composed with other // expressions in order to build a Selector. type SelectorExpr interface { @@ -73,6 +56,7 @@ type constSelector struct { type fmtSelector struct { f fmtstr.EventFormatString otherwise string + selCase SelectorCase } type mapSelector struct { @@ -142,7 +126,7 @@ func BuildSelectorFromConfig( } for _, config := range table { - action, err := buildSingle(config, key) + action, err := buildSingle(config, key, settings.Case) if err != nil { return Selector{}, err } @@ -167,7 +151,7 @@ func BuildSelectorFromConfig( return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } - fmtsel, err := FmtSelectorExpr(fmtstr, "") + fmtsel, err := FmtSelectorExpr(fmtstr, "", settings.Case) if err != nil { return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } @@ -196,16 +180,16 @@ func EmptySelectorExpr() SelectorExpr { } // ConstSelectorExpr creates a selector expression that always returns the configured string. -func ConstSelectorExpr(s string) SelectorExpr { +func ConstSelectorExpr(s string, selCase SelectorCase) SelectorExpr { if s == "" { return EmptySelectorExpr() } - return &constSelector{strings.ToLower(s)} + return &constSelector{selCase.apply(s)} } // FmtSelectorExpr creates a selector expression using a format string. If the // event can not be applied the default fallback constant string will be returned. -func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (SelectorExpr, error) { +func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string, selCase SelectorCase) (SelectorExpr, error) { if fmt.IsConst() { str, err := fmt.Run(nil) if err != nil { @@ -214,10 +198,10 @@ func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (SelectorEx if str == "" { str = fallback } - return ConstSelectorExpr(str), nil + return ConstSelectorExpr(str, selCase), nil } - return &fmtSelector{*fmt, strings.ToLower(fallback)}, nil + return &fmtSelector{*fmt, selCase.apply(fallback), selCase}, nil } // ConcatSelectorExpr combines multiple expressions that are run one after the other. @@ -241,6 +225,7 @@ func LookupSelectorExpr( evtfmt *fmtstr.EventFormatString, table map[string]string, fallback string, + selCase SelectorCase, ) (SelectorExpr, error) { if evtfmt.IsConst() { str, err := evtfmt.Run(nil) @@ -248,11 +233,11 @@ func LookupSelectorExpr( return nil, err } - str = table[strings.ToLower(str)] + str = table[selCase.apply(str)] if str == "" { str = fallback } - return ConstSelectorExpr(str), nil + return ConstSelectorExpr(str, selCase), nil } return &mapSelector{ @@ -262,15 +247,15 @@ func LookupSelectorExpr( }, nil } -func lowercaseTable(table map[string]string) map[string]string { +func copyTable(selCase SelectorCase, table map[string]string) map[string]string { tmp := make(map[string]string, len(table)) for k, v := range table { - tmp[strings.ToLower(k)] = strings.ToLower(v) + tmp[selCase.apply(k)] = selCase.apply(v) } return tmp } -func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { +func buildSingle(cfg *common.Config, key string, selCase SelectorCase) (SelectorExpr, error) { // TODO: check for unknown fields // 1. extract required key-word handler @@ -295,7 +280,7 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { if err != nil { return nil, err } - otherwise = strings.ToLower(tmp) + otherwise = selCase.apply(tmp) } // 3. extract optional `mapping` @@ -332,9 +317,9 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { // 5. build selector from available fields var sel SelectorExpr if len(mapping.Table) > 0 { - sel, err = LookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) + sel, err = LookupSelectorExpr(evtfmt, copyTable(selCase, mapping.Table), otherwise, selCase) } else { - sel, err = FmtSelectorExpr(evtfmt, otherwise) + sel, err = FmtSelectorExpr(evtfmt, otherwise, selCase) } if err != nil { return nil, err @@ -388,7 +373,7 @@ func (s *fmtSelector) sel(evt *beat.Event) (string, error) { if n == "" { return s.otherwise, nil } - return strings.ToLower(n), nil + return s.selCase.apply(n), nil } func (s *mapSelector) sel(evt *beat.Event) (string, error) { diff --git a/libbeat/outputs/outil/select_test.go b/libbeat/outputs/outil/select_test.go index e16cb602a96..49ea63bbd4b 100644 --- a/libbeat/outputs/outil/select_test.go +++ b/libbeat/outputs/outil/select_test.go @@ -31,181 +31,239 @@ import ( type node map[string]interface{} func TestSelector(t *testing.T) { + useLowerCase := func(s Settings) Settings { + return s.WithSelectorCase(SelectorLowerCase) + } + tests := map[string]struct { config string event common.MapStr - expected string + want string + settings func(Settings) Settings }{ "constant key": { - `key: value`, - common.MapStr{}, - "value", + config: `key: value`, + event: common.MapStr{}, + want: "value", }, "lowercase constant key": { - `key: VaLuE`, - common.MapStr{}, - "value", + config: `key: VaLuE`, + event: common.MapStr{}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase constant key by default": { + config: `key: VaLuE`, + event: common.MapStr{}, + want: "VaLuE", }, "format string key": { - `key: '%{[key]}'`, - common.MapStr{"key": "value"}, - "value", + config: `key: '%{[key]}'`, + event: common.MapStr{"key": "value"}, + want: "value", }, "lowercase format string key": { - `key: '%{[key]}'`, - common.MapStr{"key": "VaLuE"}, - "value", + config: `key: '%{[key]}'`, + event: common.MapStr{"key": "VaLuE"}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase format string by default": { + config: `key: '%{[key]}'`, + event: common.MapStr{"key": "VaLuE"}, + want: "VaLuE", }, "key with empty keys": { - `{key: value, keys: }`, - common.MapStr{}, - "value", + config: `{key: value, keys: }`, + event: common.MapStr{}, + want: "value", }, "lowercase key with empty keys": { - `{key: vAlUe, keys: }`, - common.MapStr{}, - "value", + config: `{key: vAlUe, keys: }`, + event: common.MapStr{}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase key with empty keys by default": { + config: `{key: vAlUe, keys: }`, + event: common.MapStr{}, + want: "vAlUe", }, "constant in multi key": { - `keys: [key: 'value']`, - common.MapStr{}, - "value", + config: `keys: [key: 'value']`, + event: common.MapStr{}, + want: "value", }, "format string in multi key": { - `keys: [key: '%{[key]}']`, - common.MapStr{"key": "value"}, - "value", + config: `keys: [key: '%{[key]}']`, + event: common.MapStr{"key": "value"}, + want: "value", }, "missing format string key with default in rule": { - `keys: + config: `keys: - key: '%{[key]}' default: value`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "value", }, "lowercase missing format string key with default in rule": { - `keys: + config: `keys: + - key: '%{[key]}' + default: vAlUe`, + event: common.MapStr{}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase missing format string key with default in rule": { + config: `keys: - key: '%{[key]}' default: vAlUe`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "vAlUe", }, "empty format string key with default in rule": { - `keys: + config: `keys: - key: '%{[key]}' default: value`, - common.MapStr{"key": ""}, - "value", + event: common.MapStr{"key": ""}, + want: "value", }, "lowercase empty format string key with default in rule": { - `keys: + config: `keys: + - key: '%{[key]}' + default: vAluE`, + event: common.MapStr{"key": ""}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase empty format string key with default in rule": { + config: `keys: - key: '%{[key]}' default: vAluE`, - common.MapStr{"key": ""}, - "value", + event: common.MapStr{"key": ""}, + want: "vAluE", }, "missing format string key with constant in next rule": { - `keys: + config: `keys: - key: '%{[key]}' - key: value`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "value", }, "missing format string key with constant in top-level rule": { - `{ key: value, keys: [key: '%{[key]}']}`, - common.MapStr{}, - "value", + config: `{ key: value, keys: [key: '%{[key]}']}`, + event: common.MapStr{}, + want: "value", }, "apply mapping": { - `keys: + config: `keys: - key: '%{[key]}' mappings: v: value`, - common.MapStr{"key": "v"}, - "value", + event: common.MapStr{"key": "v"}, + want: "value", }, "lowercase applied mapping": { - `keys: + config: `keys: + - key: '%{[key]}' + mappings: + v: vAlUe`, + event: common.MapStr{"key": "v"}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase applied mapping": { + config: `keys: - key: '%{[key]}' mappings: v: vAlUe`, - common.MapStr{"key": "v"}, - "value", + event: common.MapStr{"key": "v"}, + want: "vAlUe", }, "apply mapping with default on empty key": { - `keys: + config: `keys: - key: '%{[key]}' default: value mappings: v: 'v'`, - common.MapStr{"key": ""}, - "value", + event: common.MapStr{"key": ""}, + want: "value", }, "lowercase apply mapping with default on empty key": { - `keys: + config: `keys: + - key: '%{[key]}' + default: vAluE + mappings: + v: 'v'`, + event: common.MapStr{"key": ""}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase apply mapping with default on empty key": { + config: `keys: - key: '%{[key]}' default: vAluE mappings: v: 'v'`, - common.MapStr{"key": ""}, - "value", + event: common.MapStr{"key": ""}, + want: "vAluE", }, "apply mapping with default on empty lookup": { - `keys: + config: `keys: - key: '%{[key]}' default: value mappings: v: ''`, - common.MapStr{"key": "v"}, - "value", + event: common.MapStr{"key": "v"}, + want: "value", }, "apply mapping without match": { - `keys: + config: `keys: - key: '%{[key]}' mappings: v: '' - key: value`, - common.MapStr{"key": "x"}, - "value", + event: common.MapStr{"key": "x"}, + want: "value", }, "mapping with constant key": { - `keys: + config: `keys: - key: k mappings: k: value`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "value", }, "mapping with missing constant key": { - `keys: + config: `keys: - key: unknown mappings: {k: wrong} - key: value`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "value", }, "mapping with missing constant key, but default": { - `keys: + config: `keys: - key: unknown default: value mappings: {k: wrong}`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "value", }, "matching condition": { - `keys: + config: `keys: - key: value when.equals.test: test`, - common.MapStr{"test": "test"}, - "value", + event: common.MapStr{"test": "test"}, + want: "value", }, "failing condition": { - `keys: + config: `keys: - key: wrong when.equals.test: test - key: value`, - common.MapStr{"test": "x"}, - "value", + event: common.MapStr{"test": "x"}, + want: "value", }, } @@ -217,12 +275,17 @@ func TestSelector(t *testing.T) { t.Fatalf("YAML parse error: %v\n%v", err, yaml) } - sel, err := BuildSelectorFromConfig(cfg, Settings{ + settings := Settings{ Key: "key", MultiKey: "keys", EnableSingleOnly: true, FailEmpty: true, - }) + } + if test.settings != nil { + settings = test.settings(settings) + } + + sel, err := BuildSelectorFromConfig(cfg, settings) if err != nil { t.Fatal(err) } @@ -236,7 +299,7 @@ func TestSelector(t *testing.T) { t.Fatal(err) } - assert.Equal(t, test.expected, actual) + assert.Equal(t, test.want, actual) }) } } diff --git a/libbeat/outputs/outil/settings.go b/libbeat/outputs/outil/settings.go new file mode 100644 index 00000000000..380f75d23e6 --- /dev/null +++ b/libbeat/outputs/outil/settings.go @@ -0,0 +1,96 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package outil + +import "strings" + +// Settings configures how BuildSelectorFromConfig creates a Selector from +// a given configuration object. +type Settings struct { + // single selector key and default option keyword + Key string + + // multi-selector key in config + MultiKey string + + // if enabled a selector `key` in config will be generated, if `key` is present + EnableSingleOnly bool + + // Fail building selector if `key` and `multiKey` are missing + FailEmpty bool + + // Case configures the case-sensitivity of generated strings. + Case SelectorCase +} + +// SelectorCase is used to configure a Selector output string casing. +// Use SelectorLowerCase or SelectorUpperCase to enforce the Selector to +// always generate lower case or upper case strings. +type SelectorCase uint8 + +const ( + // SelectorKeepCase instructs the Selector to not modify the string output. + SelectorKeepCase SelectorCase = iota + + // SelectorLowerCase instructs the Selector to always transform the string output to lower case only. + SelectorLowerCase + + // SelectorUpperCase instructs the Selector to always transform the string output to upper case only. + SelectorUpperCase +) + +// WithKey returns a new Settings struct with updated `Key` setting. +func (s Settings) WithKey(key string) Settings { + s.Key = key + return s +} + +// WithMultiKey returns a new Settings struct with updated `MultiKey` setting. +func (s Settings) WithMultiKey(key string) Settings { + s.MultiKey = key + return s +} + +// WithEnableSingleOnly returns a new Settings struct with updated `EnableSingleOnly` setting. +func (s Settings) WithEnableSingleOnly(b bool) Settings { + s.EnableSingleOnly = b + return s +} + +// WithFailEmpty returns a new Settings struct with updated `FailEmpty` setting. +func (s Settings) WithFailEmpty(b bool) Settings { + s.FailEmpty = b + return s +} + +// WithSelectorCase returns a new Settings struct with updated `Case` setting. +func (s Settings) WithSelectorCase(c SelectorCase) Settings { + s.Case = c + return s +} + +func (selCase SelectorCase) apply(in string) string { + switch selCase { + case SelectorLowerCase: + return strings.ToLower(in) + case SelectorUpperCase: + return strings.ToUpper(in) + default: + return in + } +} diff --git a/libbeat/outputs/redis/redis.go b/libbeat/outputs/redis/redis.go index 7ef312016e3..910b69d9f58 100644 --- a/libbeat/outputs/redis/redis.go +++ b/libbeat/outputs/redis/redis.go @@ -92,12 +92,7 @@ func makeRedis( return outputs.Fail(errors.New("Bad Redis data type")) } - key, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, - }) + key, err := buildKeySelector(cfg) if err != nil { return outputs.Fail(err) } @@ -174,3 +169,13 @@ func makeRedis( return outputs.SuccessNet(config.LoadBalance, config.BulkMaxSize, config.MaxRetries, clients) } + +func buildKeySelector(cfg *common.Config) (outil.Selector, error) { + return outil.BuildSelectorFromConfig(cfg, outil.Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + Case: outil.SelectorKeepCase, + }) +} diff --git a/libbeat/outputs/redis/redis_test.go b/libbeat/outputs/redis/redis_test.go index 5ca91d3fef0..deffb1d8de5 100644 --- a/libbeat/outputs/redis/redis_test.go +++ b/libbeat/outputs/redis/redis_test.go @@ -118,3 +118,64 @@ func TestMakeRedis(t *testing.T) { }) } } + +func TestKeySelection(t *testing.T) { + cases := map[string]struct { + cfg map[string]interface{} + event beat.Event + want string + }{ + "key configured": { + cfg: map[string]interface{}{"key": "test"}, + want: "test", + }, + "key must keep case": { + cfg: map[string]interface{}{"key": "Test"}, + want: "Test", + }, + "key setting": { + cfg: map[string]interface{}{ + "keys": []map[string]interface{}{{"key": "test"}}, + }, + want: "test", + }, + "keys setting must keep case": { + cfg: map[string]interface{}{ + "keys": []map[string]interface{}{{"key": "Test"}}, + }, + want: "Test", + }, + "use event field": { + cfg: map[string]interface{}{"key": "test-%{[field]}"}, + event: beat.Event{ + Fields: common.MapStr{"field": "from-event"}, + }, + want: "test-from-event", + }, + "use event field must keep case": { + cfg: map[string]interface{}{"key": "Test-%{[field]}"}, + event: beat.Event{ + Fields: common.MapStr{"field": "From-Event"}, + }, + want: "Test-From-Event", + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + selector, err := buildKeySelector(common.MustNewConfigFrom(test.cfg)) + if err != nil { + t.Fatalf("Failed to parse configuration: %v", err) + } + + got, err := selector.Select(&test.event) + if err != nil { + t.Fatalf("Failed to create key name: %v", err) + } + + if test.want != got { + t.Errorf("Pipeline name missmatch (want: %v, got: %v)", test.want, got) + } + }) + } +}