From 712398efd3d49fd5e150c658dcda89ebe5ec89cd Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 6 May 2021 16:11:51 +0800 Subject: [PATCH] libbeat: support explicit dynamic templates (#25422) (#25570) * libbeat: support explicit dynamic templates Add the field `DynamicTemplate bool` to `mapping.Field`, indicating that the field represents an explicitly named dynamic template. Update Elasticsearch template generation to create dynamic templates from these fields when the target Elasticsearch is 7.13.0+, when support for these dynamic templates was added and the requirement of match criteria was dropped. Such dynamic templates are suitable only for use in dynamic_templates bulk request parameters and in ingest pipelines; they will have no path or type matching criteria. * Update changelog * Revert signature change * Disallow dynamic_template with object_type_params * Add a constant for min version * libbeat/mapping: fix field validation (cherry picked from commit 7a49ca15950fed297aac20d3d96fe605db65164b) Co-authored-by: Andrew Wilkins --- CHANGELOG.next.asciidoc | 1 + libbeat/mapping/field.go | 19 +++++++++++- libbeat/mapping/field_test.go | 14 +++++++++ libbeat/template/processor.go | 50 ++++++++++++++++++++---------- libbeat/template/processor_test.go | 25 +++++++++++++-- 5 files changed, 88 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e5937493d01..303b2a715fe 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -397,6 +397,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add kubernetes.pod.ip field in kubernetes metadata. {pull}25037[25037] - Discover changes in Kubernetes namespace metadata as soon as they happen. {pull}25117[25117] - Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25115[25115] +- Add support for defining explicitly named dynamic templates without path/type match criteria {pull}25422[25422] - Add new setting `gc_percent` for tuning the garbage collector limits via configuration file. {pull}25394[25394] - Add `unit` and `metric_type` properties to fields.yml for populating field metadata in Elasticsearch templates {pull}25419[25419] - Add new option `suffix` to `logging.files` to control how log files are rotated. {pull}25464[25464] diff --git a/libbeat/mapping/field.go b/libbeat/mapping/field.go index 02d6aa0bb07..72ac861581d 100644 --- a/libbeat/mapping/field.go +++ b/libbeat/mapping/field.go @@ -56,6 +56,14 @@ type Field struct { MigrationAlias bool `config:"migration"` Dimension *bool `config:"dimension"` + // DynamicTemplate controls whether this field represents an explicitly + // named dynamic template. + // + // Such dynamic templates are only suitable for use in dynamic_template + // parameter in bulk requests or in ingest pipelines, as they will have + // no path or type match criteria. + DynamicTemplate bool `config:"dynamic_template"` + // Unit holds a standard unit for numeric fields: "percent", "byte", or a time unit. // See https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-field-meta.html. Unit string `config:"unit"` @@ -147,7 +155,16 @@ func (f *Field) validateType() error { allowedFormatters = []string{"date_range"} case "boolean", "binary", "ip", "alias", "array": // No formatters, metric types, or units allowed. - case "object", "group", "nested", "flattened": + case "object": + if f.DynamicTemplate && (len(f.ObjectTypeParams) > 0 || f.ObjectType != "") { + // When either ObjectTypeParams or ObjectType are set for an object-type field, + // libbeat/template will create dynamic templates. It does not make sense to + // use these with explicit dynamic templates. + return errors.New("dynamic_template not supported with object_type_params") + } + // No further checks for object yet. + return nil + case "group", "nested", "flattened": // No check for them yet return nil case "": diff --git a/libbeat/mapping/field_test.go b/libbeat/mapping/field_test.go index dd0f67a2a2d..e2ad9b6a140 100644 --- a/libbeat/mapping/field_test.go +++ b/libbeat/mapping/field_test.go @@ -350,6 +350,20 @@ func TestFieldValidate(t *testing.T) { cfg: common.MapStr{"type": "long", "metric_type": "timer"}, err: true, }, + "invalid config mixing dynamic_template with object_type": { + cfg: common.MapStr{"dynamic_template": true, "type": "object", "object_type": "text"}, + err: true, + }, + "invalid config mixing dynamic_template with object_type_params": { + cfg: common.MapStr{ + "type": "object", + "object_type_params": []common.MapStr{{ + "object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 100, + }}, + "dynamic_template": true, + }, + err: true, + }, } for name, test := range tests { diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 907a9b5db64..2cd06f6d94e 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -27,10 +27,11 @@ import ( ) var ( - minVersionAlias = common.MustNewVersion("6.4.0") - minVersionFieldMeta = common.MustNewVersion("7.6.0") - minVersionHistogram = common.MustNewVersion("7.6.0") - minVersionWildcard = common.MustNewVersion("7.9.0") + minVersionAlias = common.MustNewVersion("6.4.0") + minVersionFieldMeta = common.MustNewVersion("7.6.0") + minVersionHistogram = common.MustNewVersion("7.6.0") + minVersionWildcard = common.MustNewVersion("7.9.0") + minVersionExplicitDynamicTemplate = common.MustNewVersion("7.13.0") ) // Processor struct to process fields to template @@ -127,7 +128,15 @@ func (p *Processor) Process(fields mapping.Fields, state *fieldState, output com } if len(indexMapping) > 0 { - output.Put(mapping.GenerateKey(field.Name), indexMapping) + if field.DynamicTemplate { + // Explicit dynamic templates were introduced in + // Elasticsearch 7.13, ignore if unsupported + if !p.EsVersion.LessThan(minVersionExplicitDynamicTemplate) { + p.addDynamicTemplate(field.Name, "", "", indexMapping) + } + } else { + output.Put(mapping.GenerateKey(field.Name), indexMapping) + } } } return nil @@ -382,8 +391,11 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { if len(f.ObjectTypeParams) != 0 { otParams = f.ObjectTypeParams } else { - otParams = []mapping.ObjectTypeCfg{mapping.ObjectTypeCfg{ - ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}} + otParams = []mapping.ObjectTypeCfg{{ + ObjectType: f.ObjectType, + ObjectTypeMappingType: f.ObjectTypeMappingType, + ScalingFactor: f.ScalingFactor, + }} } for _, otp := range otParams { @@ -432,7 +444,7 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { if len(otParams) > 1 { path = fmt.Sprintf("%s_%s", path, matchingType) } - p.addDynamicTemplate(path, pathMatch, dynProperties, matchingType) + p.addDynamicTemplate(path, pathMatch, matchingType, dynProperties) } properties := p.getDefaultProperties(f) @@ -449,14 +461,14 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { } type dynamicTemplateKey struct { - path string + name string pathMatch string matchType string } -func (p *Processor) addDynamicTemplate(path string, pathMatch string, properties common.MapStr, matchType string) { +func (p *Processor) addDynamicTemplate(name, pathMatch, matchType string, properties common.MapStr) { key := dynamicTemplateKey{ - path: path, + name: name, pathMatch: pathMatch, matchType: matchType, } @@ -468,13 +480,17 @@ func (p *Processor) addDynamicTemplate(path string, pathMatch string, properties return } } + dynamicTemplateProperties := common.MapStr{ + "mapping": properties, + } + if matchType != "" { + dynamicTemplateProperties["match_mapping_type"] = matchType + } + if pathMatch != "" { + dynamicTemplateProperties["path_match"] = pathMatch + } dynamicTemplate := common.MapStr{ - // Set the path of the field as name - path: common.MapStr{ - "mapping": properties, - "match_mapping_type": matchType, - "path_match": pathMatch, - }, + name: dynamicTemplateProperties, } p.dynamicTemplatesMap[key] = dynamicTemplate p.dynamicTemplates = append(p.dynamicTemplates, dynamicTemplate) diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index d1c8de5ce0a..f5e6d895fd5 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/mapping" @@ -482,6 +483,20 @@ func TestDynamicTemplates(t *testing.T) { }, }, }, + { + field: mapping.Field{ + Name: "dynamic_histogram", + Type: "histogram", + DynamicTemplate: true, + }, + expected: []common.MapStr{ + { + "dynamic_histogram": common.MapStr{ + "mapping": common.MapStr{"type": "histogram"}, + }, + }, + }, + }, } for _, numericType := range []string{"byte", "double", "float", "long", "short", "boolean"} { @@ -509,9 +524,13 @@ func TestDynamicTemplates(t *testing.T) { } for _, test := range tests { - p := &Processor{} - p.object(&test.field) - p.object(&test.field) // should not be added twice + output := make(common.MapStr) + p := &Processor{EsVersion: *common.MustNewVersion("8.0.0")} + err := p.Process(mapping.Fields{ + test.field, + test.field, // should not be added twice + }, &fieldState{Path: test.field.Path}, output) + require.NoError(t, err) assert.Equal(t, test.expected, p.dynamicTemplates) } }