libbeat: support explicit dynamic templates (elastic#25422) (elastic#…

…25570)

* libbeat: support explicit dynamic templates

Add the field `DynamicTemplate bool` to `mapping.Field`,
indicating that the field represents an explicitly named
dynamic template. Update Elasticsearch template generation
to create dynamic templates from these fields when the
target Elasticsearch is 7.13.0+, when support for these
dynamic templates was added and the requirement of match
criteria was dropped.

Such dynamic templates are suitable only for use in
dynamic_templates bulk request parameters and in ingest
pipelines; they will have no path or type matching criteria.

* Update changelog

* Revert signature change

* Disallow dynamic_template with object_type_params

* Add a constant for min version

* libbeat/mapping: fix field validation

(cherry picked from commit 7a49ca1)

Co-authored-by: Andrew Wilkins <axw@elastic.co>

CHANGELOG.next.asciidoc

@@ -397,6 +397,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add kubernetes.pod.ip field in kubernetes metadata. {pull}25037[25037]
 - Discover changes in Kubernetes namespace metadata as soon as they happen. {pull}25117[25117]
 - Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25115[25115]
+- Add support for defining explicitly named dynamic templates without path/type match criteria {pull}25422[25422]
 - Add new setting `gc_percent` for tuning the garbage collector limits via configuration file. {pull}25394[25394]
 - Add `unit` and `metric_type` properties to fields.yml for populating field metadata in Elasticsearch templates {pull}25419[25419]
 - Add new option `suffix` to `logging.files` to control how log files are rotated. {pull}25464[25464]

libbeat/mapping/field.go

@@ -56,6 +56,14 @@ type Field struct {
 	MigrationAlias bool        `config:"migration"`
 	Dimension      *bool       `config:"dimension"`
 
+	// DynamicTemplate controls whether this field represents an explicitly
+	// named dynamic template.
+	//
+	// Such dynamic templates are only suitable for use in dynamic_template
+	// parameter in bulk requests or in ingest pipelines, as they will have
+	// no path or type match criteria.
+	DynamicTemplate bool `config:"dynamic_template"`
+
 	// Unit holds a standard unit for numeric fields: "percent", "byte", or a time unit.
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-field-meta.html.
 	Unit string `config:"unit"`
@@ -147,7 +155,16 @@ func (f *Field) validateType() error {
 		allowedFormatters = []string{"date_range"}
 	case "boolean", "binary", "ip", "alias", "array":
 		// No formatters, metric types, or units allowed.
-	case "object", "group", "nested", "flattened":
+	case "object":
+		if f.DynamicTemplate && (len(f.ObjectTypeParams) > 0 || f.ObjectType != "") {
+			// When either ObjectTypeParams or ObjectType are set for an object-type field,
+			// libbeat/template will create dynamic templates. It does not make sense to
+			// use these with explicit dynamic templates.
+			return errors.New("dynamic_template not supported with object_type_params")
+		}
+		// No further checks for object yet.
+		return nil
+	case "group", "nested", "flattened":
 		// No check for them yet
 		return nil
 	case "":

libbeat/mapping/field_test.go

@@ -350,6 +350,20 @@ func TestFieldValidate(t *testing.T) {
 			cfg: common.MapStr{"type": "long", "metric_type": "timer"},
 			err: true,
 		},
+		"invalid config mixing dynamic_template with object_type": {
+			cfg: common.MapStr{"dynamic_template": true, "type": "object", "object_type": "text"},
+			err: true,
+		},
+		"invalid config mixing dynamic_template with object_type_params": {
+			cfg: common.MapStr{
+				"type": "object",
+				"object_type_params": []common.MapStr{{
+					"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 100,
+				}},
+				"dynamic_template": true,
+			},
+			err: true,
+		},
 	}
 
 	for name, test := range tests {

libbeat/template/processor.go

@@ -27,10 +27,11 @@ import (
 )
 
 var (
-	minVersionAlias     = common.MustNewVersion("6.4.0")
-	minVersionFieldMeta = common.MustNewVersion("7.6.0")
-	minVersionHistogram = common.MustNewVersion("7.6.0")
-	minVersionWildcard  = common.MustNewVersion("7.9.0")
+	minVersionAlias                   = common.MustNewVersion("6.4.0")
+	minVersionFieldMeta               = common.MustNewVersion("7.6.0")
+	minVersionHistogram               = common.MustNewVersion("7.6.0")
+	minVersionWildcard                = common.MustNewVersion("7.9.0")
+	minVersionExplicitDynamicTemplate = common.MustNewVersion("7.13.0")
 )
 
 // Processor struct to process fields to template
@@ -127,7 +128,15 @@ func (p *Processor) Process(fields mapping.Fields, state *fieldState, output com
 		}
 
 		if len(indexMapping) > 0 {
-			output.Put(mapping.GenerateKey(field.Name), indexMapping)
+			if field.DynamicTemplate {
+				// Explicit dynamic templates were introduced in
+				// Elasticsearch 7.13, ignore if unsupported
+				if !p.EsVersion.LessThan(minVersionExplicitDynamicTemplate) {
+					p.addDynamicTemplate(field.Name, "", "", indexMapping)
+				}
+			} else {
+				output.Put(mapping.GenerateKey(field.Name), indexMapping)
+			}
 		}
 	}
 	return nil
@@ -382,8 +391,11 @@ func (p *Processor) object(f *mapping.Field) common.MapStr {
 	if len(f.ObjectTypeParams) != 0 {
 		otParams = f.ObjectTypeParams
 	} else {
-		otParams = []mapping.ObjectTypeCfg{mapping.ObjectTypeCfg{
-			ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}}
+		otParams = []mapping.ObjectTypeCfg{{
+			ObjectType:            f.ObjectType,
+			ObjectTypeMappingType: f.ObjectTypeMappingType,
+			ScalingFactor:         f.ScalingFactor,
+		}}
 	}
 
 	for _, otp := range otParams {
@@ -432,7 +444,7 @@ func (p *Processor) object(f *mapping.Field) common.MapStr {
 		if len(otParams) > 1 {
 			path = fmt.Sprintf("%s_%s", path, matchingType)
 		}
-		p.addDynamicTemplate(path, pathMatch, dynProperties, matchingType)
+		p.addDynamicTemplate(path, pathMatch, matchingType, dynProperties)
 	}
 
 	properties := p.getDefaultProperties(f)
@@ -449,14 +461,14 @@ func (p *Processor) object(f *mapping.Field) common.MapStr {
 }
 
 type dynamicTemplateKey struct {
-	path      string
+	name      string
 	pathMatch string
 	matchType string
 }
 
-func (p *Processor) addDynamicTemplate(path string, pathMatch string, properties common.MapStr, matchType string) {
+func (p *Processor) addDynamicTemplate(name, pathMatch, matchType string, properties common.MapStr) {
 	key := dynamicTemplateKey{
-		path:      path,
+		name:      name,
 		pathMatch: pathMatch,
 		matchType: matchType,
 	}
@@ -468,13 +480,17 @@ func (p *Processor) addDynamicTemplate(path string, pathMatch string, properties
 			return
 		}
 	}
+	dynamicTemplateProperties := common.MapStr{
+		"mapping": properties,
+	}
+	if matchType != "" {
+		dynamicTemplateProperties["match_mapping_type"] = matchType
+	}
+	if pathMatch != "" {
+		dynamicTemplateProperties["path_match"] = pathMatch
+	}
 	dynamicTemplate := common.MapStr{
-		// Set the path of the field as name
-		path: common.MapStr{
-			"mapping":            properties,
-			"match_mapping_type": matchType,
-			"path_match":         pathMatch,
-		},
+		name: dynamicTemplateProperties,
 	}
 	p.dynamicTemplatesMap[key] = dynamicTemplate
 	p.dynamicTemplates = append(p.dynamicTemplates, dynamicTemplate)

libbeat/template/processor_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/mapping"
@@ -482,6 +483,20 @@ func TestDynamicTemplates(t *testing.T) {
 				},
 			},
 		},
+		{
+			field: mapping.Field{
+				Name:            "dynamic_histogram",
+				Type:            "histogram",
+				DynamicTemplate: true,
+			},
+			expected: []common.MapStr{
+				{
+					"dynamic_histogram": common.MapStr{
+						"mapping": common.MapStr{"type": "histogram"},
+					},
+				},
+			},
+		},
 	}
 
 	for _, numericType := range []string{"byte", "double", "float", "long", "short", "boolean"} {
@@ -509,9 +524,13 @@ func TestDynamicTemplates(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		p := &Processor{}
-		p.object(&test.field)
-		p.object(&test.field) // should not be added twice
+		output := make(common.MapStr)
+		p := &Processor{EsVersion: *common.MustNewVersion("8.0.0")}
+		err := p.Process(mapping.Fields{
+			test.field,
+			test.field, // should not be added twice
+		}, &fieldState{Path: test.field.Path}, output)
+		require.NoError(t, err)
 		assert.Equal(t, test.expected, p.dynamicTemplates)
 	}
 }