diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 4cacdc3a60b..b425914e40f 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -23,3 +23,5 @@ The list below covers the major changes between 7.0.0-alpha2 and master only. ==== Bugfixes ==== Added + +- Allow multiple object type configurations per field. {pull}9772[9772] diff --git a/libbeat/common/field.go b/libbeat/common/field.go index 3ab60355b7b..95aff0aeac4 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -21,6 +21,8 @@ import ( "fmt" "strings" + "github.com/pkg/errors" + "github.com/elastic/go-ucfg/yaml" ) @@ -34,25 +36,27 @@ import ( type Fields []Field type Field struct { - Name string `config:"name"` - Type string `config:"type"` - Description string `config:"description"` - Format string `config:"format"` - ScalingFactor int `config:"scaling_factor"` - Fields Fields `config:"fields"` - MultiFields Fields `config:"multi_fields"` - ObjectType string `config:"object_type"` - ObjectTypeMappingType string `config:"object_type_mapping_type"` - Enabled *bool `config:"enabled"` - Analyzer string `config:"analyzer"` - SearchAnalyzer string `config:"search_analyzer"` - Norms bool `config:"norms"` - Dynamic DynamicType `config:"dynamic"` - Index *bool `config:"index"` - DocValues *bool `config:"doc_values"` - CopyTo string `config:"copy_to"` - IgnoreAbove int `config:"ignore_above"` - AliasPath string `config:"path"` + Name string `config:"name"` + Type string `config:"type"` + Description string `config:"description"` + Format string `config:"format"` + Fields Fields `config:"fields"` + MultiFields Fields `config:"multi_fields"` + Enabled *bool `config:"enabled"` + Analyzer string `config:"analyzer"` + SearchAnalyzer string `config:"search_analyzer"` + Norms bool `config:"norms"` + Dynamic DynamicType `config:"dynamic"` + Index *bool `config:"index"` + DocValues *bool `config:"doc_values"` + CopyTo string `config:"copy_to"` + IgnoreAbove int `config:"ignore_above"` + AliasPath string `config:"path"` + + ObjectType string `config:"object_type"` + ObjectTypeMappingType string `config:"object_type_mapping_type"` + ScalingFactor int `config:"scaling_factor"` + ObjectTypeParams []ObjectTypeCfg `config:"object_type_params"` // Kibana specific Analyzed *bool `config:"analyzed"` @@ -73,6 +77,13 @@ type Field struct { Path string } +// ObjectTypeCfg defines type and configuration of object attributes +type ObjectTypeCfg struct { + ObjectType string `config:"object_type"` + ObjectTypeMappingType string `config:"object_type_mapping_type"` + ScalingFactor int `config:"scaling_factor"` +} + type VersionizedString struct { MinVersion string `config:"min_version"` Value string `config:"value"` @@ -94,6 +105,17 @@ func (d *DynamicType) Unpack(s string) error { return nil } +// Validate ensures objectTypeParams are not mixed with top level objectType configuration +func (f *Field) Validate() error { + if len(f.ObjectTypeParams) == 0 { + return nil + } + if f.ScalingFactor != 0 || f.ObjectTypeMappingType != "" || f.ObjectType != "" { + return errors.New("mixing top level objectType configuration with array of object type configurations is forbidden") + } + return nil +} + func LoadFieldsYaml(path string) (Fields, error) { keys := []Field{} diff --git a/libbeat/common/field_test.go b/libbeat/common/field_test.go index ea1fe2729c0..8f41ace61ef 100644 --- a/libbeat/common/field_test.go +++ b/libbeat/common/field_test.go @@ -20,6 +20,8 @@ package common import ( "testing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" "github.com/elastic/go-ucfg/yaml" @@ -189,3 +191,59 @@ func TestGetKeys(t *testing.T) { assert.Equal(t, test.keys, test.fields.GetKeys()) } } + +func TestFieldValidate(t *testing.T) { + tests := []struct { + cfg MapStr + field Field + err bool + name string + }{ + { + cfg: MapStr{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 10}, + field: Field{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 10}, + err: false, + name: "top level object type config", + }, { + cfg: MapStr{"object_type_params": []MapStr{ + {"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 100}}}, + field: Field{ObjectTypeParams: []ObjectTypeCfg{{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 100}}}, + err: false, + name: "multiple object type configs", + }, { + cfg: MapStr{ + "object_type": "scaled_float", + "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + err: true, + name: "invalid config mixing object_type and object_type_params", + }, { + cfg: MapStr{ + "object_type_mapping_type": "float", + "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + err: true, + name: "invalid config mixing object_type_mapping_type and object_type_params", + }, { + cfg: MapStr{ + "scaling_factor": 100, + "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + err: true, + name: "invalid config mixing scaling_factor and object_type_params", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg, err := NewConfigFrom(test.cfg) + require.NoError(t, err) + var f Field + err = cfg.Unpack(&f) + if test.err { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, test.field, f) + } + }) + } + +} diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index e0fbc9e8025..aa933aa89d2 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -34,6 +34,8 @@ var ( defaultIgnoreAbove = 1024 ) +const scalingFactorKey = "scalingFactor" + // Process recursively processes the given fields and writes the template in the given output func (p *Processor) Process(fields common.Fields, path string, output common.MapStr) error { for _, field := range fields { @@ -119,20 +121,28 @@ func (p *Processor) integer(f *common.Field) common.MapStr { return property } -func (p *Processor) scaledFloat(f *common.Field) common.MapStr { +func (p *Processor) scaledFloat(f *common.Field, params ...common.MapStr) common.MapStr { property := getDefaultProperties(f) property["type"] = "scaled_float" if p.EsVersion.IsMajor(2) { property["type"] = "float" - } else { - scalingFactor := f.ScalingFactor - // Set default scaling factor - if scalingFactor == 0 { - scalingFactor = defaultScalingFactor + return property + } + + // Set scaling factor + scalingFactor := defaultScalingFactor + if f.ScalingFactor != 0 && len(f.ObjectTypeParams) == 0 { + scalingFactor = f.ScalingFactor + } + + if len(params) > 0 { + if s, ok := params[0][scalingFactorKey].(int); ok && s != 0 { + scalingFactor = s } - property["scaling_factor"] = scalingFactor } + + property["scaling_factor"] = scalingFactor return property } @@ -256,33 +266,43 @@ func (p *Processor) alias(f *common.Field) common.MapStr { } func (p *Processor) object(f *common.Field) common.MapStr { - dynProperties := getDefaultProperties(f) - - matchType := func(onlyType string) string { - if f.ObjectTypeMappingType != "" { - return f.ObjectTypeMappingType + matchType := func(onlyType string, mt string) string { + if mt != "" { + return mt } return onlyType } - switch f.ObjectType { - case "scaled_float": - dynProperties = p.scaledFloat(f) - addDynamicTemplate(f, dynProperties, matchType("*")) - case "text": - dynProperties["type"] = "text" + var otParams []common.ObjectTypeCfg + if len(f.ObjectTypeParams) != 0 { + otParams = f.ObjectTypeParams + } else { + otParams = []common.ObjectTypeCfg{common.ObjectTypeCfg{ + ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}} + } - if p.EsVersion.IsMajor(2) { - dynProperties["type"] = "string" - dynProperties["index"] = "analyzed" + for _, otp := range otParams { + dynProperties := getDefaultProperties(f) + + switch otp.ObjectType { + case "scaled_float": + dynProperties = p.scaledFloat(f, common.MapStr{scalingFactorKey: otp.ScalingFactor}) + addDynamicTemplate(f, dynProperties, matchType("*", otp.ObjectTypeMappingType)) + case "text": + dynProperties["type"] = "text" + + if p.EsVersion.IsMajor(2) { + dynProperties["type"] = "string" + dynProperties["index"] = "analyzed" + } + addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType)) + case "keyword": + dynProperties["type"] = otp.ObjectType + addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType)) + case "byte", "double", "float", "long", "short", "boolean": + dynProperties["type"] = otp.ObjectType + addDynamicTemplate(f, dynProperties, matchType(otp.ObjectType, otp.ObjectTypeMappingType)) } - addDynamicTemplate(f, dynProperties, matchType("string")) - case "keyword": - dynProperties["type"] = f.ObjectType - addDynamicTemplate(f, dynProperties, matchType("string")) - case "byte", "double", "float", "long", "short": - dynProperties["type"] = f.ObjectType - addDynamicTemplate(f, dynProperties, matchType(f.ObjectType)) } properties := getDefaultProperties(f) diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index 1064a9e91a2..c56c45dfead 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -56,6 +56,27 @@ func TestProcessor(t *testing.T) { "scaling_factor": 100, }, }, + { + output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 0}), + expected: common.MapStr{ + "type": "scaled_float", + "scaling_factor": 1000, + }, + }, + { + output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{"someKey": 10}), + expected: common.MapStr{ + "type": "scaled_float", + "scaling_factor": 1000, + }, + }, + { + output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 10}), + expected: common.MapStr{ + "type": "scaled_float", + "scaling_factor": 10, + }, + }, { output: pEsVersion2.scaledFloat(&common.Field{Type: "scaled_float"}), expected: common.MapStr{"type": "float"}, @@ -264,22 +285,24 @@ func TestProcessor(t *testing.T) { } } -func TestDynamicTemplate(t *testing.T) { +func TestDynamicTemplates(t *testing.T) { p := &Processor{} tests := []struct { field common.Field - expected common.MapStr + expected []common.MapStr }{ { field: common.Field{ Type: "object", ObjectType: "keyword", Name: "context", }, - expected: common.MapStr{ - "context": common.MapStr{ - "mapping": common.MapStr{"type": "keyword"}, - "match_mapping_type": "string", - "path_match": "context.*", + expected: []common.MapStr{ + common.MapStr{ + "context": common.MapStr{ + "mapping": common.MapStr{"type": "keyword"}, + "match_mapping_type": "string", + "path_match": "context.*", + }, }, }, }, @@ -288,11 +311,13 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "long", ObjectTypeMappingType: "futuretype", Path: "language", Name: "english", }, - expected: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "long"}, - "match_mapping_type": "futuretype", - "path_match": "language.english.*", + expected: []common.MapStr{ + common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "long"}, + "match_mapping_type": "futuretype", + "path_match": "language.english.*", + }, }, }, }, @@ -301,11 +326,13 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "long", ObjectTypeMappingType: "*", Path: "language", Name: "english", }, - expected: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "long"}, - "match_mapping_type": "*", - "path_match": "language.english.*", + expected: []common.MapStr{ + common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "long"}, + "match_mapping_type": "*", + "path_match": "language.english.*", + }, }, }, }, @@ -314,11 +341,13 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "long", Path: "language", Name: "english", }, - expected: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "long"}, - "match_mapping_type": "long", - "path_match": "language.english.*", + expected: []common.MapStr{ + common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "long"}, + "match_mapping_type": "long", + "path_match": "language.english.*", + }, }, }, }, @@ -327,11 +356,13 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "text", Path: "language", Name: "english", }, - expected: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "text"}, - "match_mapping_type": "string", - "path_match": "language.english.*", + expected: []common.MapStr{ + common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "text"}, + "match_mapping_type": "string", + "path_match": "language.english.*", + }, }, }, }, @@ -340,14 +371,16 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "scaled_float", Name: "core.*.pct", }, - expected: common.MapStr{ - "core.*.pct": common.MapStr{ - "mapping": common.MapStr{ - "type": "scaled_float", - "scaling_factor": defaultScalingFactor, + expected: []common.MapStr{ + common.MapStr{ + "core.*.pct": common.MapStr{ + "mapping": common.MapStr{ + "type": "scaled_float", + "scaling_factor": defaultScalingFactor, + }, + "match_mapping_type": "*", + "path_match": "core.*.pct", }, - "match_mapping_type": "*", - "path_match": "core.*.pct", }, }, }, @@ -356,35 +389,72 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "scaled_float", Name: "core.*.pct", ScalingFactor: 100, ObjectTypeMappingType: "float", }, - expected: common.MapStr{ - "core.*.pct": common.MapStr{ - "mapping": common.MapStr{ - "type": "scaled_float", - "scaling_factor": 100, + expected: []common.MapStr{ + common.MapStr{ + "core.*.pct": common.MapStr{ + "mapping": common.MapStr{ + "type": "scaled_float", + "scaling_factor": 100, + }, + "match_mapping_type": "float", + "path_match": "core.*.pct", + }, + }, + }, + }, + { + field: common.Field{ + Type: "object", ObjectTypeParams: []common.ObjectTypeCfg{ + {ObjectType: "float", ObjectTypeMappingType: "float"}, + {ObjectType: "boolean"}, + {ObjectType: "scaled_float", ScalingFactor: 10000}, + }, + Name: "context", + }, + expected: []common.MapStr{ + common.MapStr{ + "context": common.MapStr{ + "mapping": common.MapStr{"type": "float"}, + "match_mapping_type": "float", + "path_match": "context.*", + }, + }, + common.MapStr{ + "context": common.MapStr{ + "mapping": common.MapStr{"type": "boolean"}, + "match_mapping_type": "boolean", + "path_match": "context.*", + }, + }, + common.MapStr{ + "context": common.MapStr{ + "mapping": common.MapStr{"type": "scaled_float", "scaling_factor": 10000}, + "match_mapping_type": "*", + "path_match": "context.*", }, - "match_mapping_type": "float", - "path_match": "core.*.pct", }, }, }, } - for _, numericType := range []string{"byte", "double", "float", "long", "short"} { + for _, numericType := range []string{"byte", "double", "float", "long", "short", "boolean"} { gen := struct { field common.Field - expected common.MapStr + expected []common.MapStr }{ field: common.Field{ Type: "object", ObjectType: numericType, Name: "somefield", ObjectTypeMappingType: "long", }, - expected: common.MapStr{ - "somefield": common.MapStr{ - "mapping": common.MapStr{ - "type": numericType, + expected: []common.MapStr{ + common.MapStr{ + "somefield": common.MapStr{ + "mapping": common.MapStr{ + "type": numericType, + }, + "match_mapping_type": "long", + "path_match": "somefield.*", }, - "match_mapping_type": "long", - "path_match": "somefield.*", }, }, } @@ -394,7 +464,7 @@ func TestDynamicTemplate(t *testing.T) { for _, test := range tests { dynamicTemplates = nil p.object(&test.field) - assert.Equal(t, test.expected, dynamicTemplates[0]) + assert.Equal(t, test.expected, dynamicTemplates) } }