From 29b013de0daee18b1e900d4240edc2ceb3df37d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 6 Jul 2018 16:22:41 +0200 Subject: [PATCH 1/6] add hint to fields.yml when present instead of guessing using the types dict Closes #7472 --- filebeat/scripts/generator/fields/main.go | 27 ++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/filebeat/scripts/generator/fields/main.go b/filebeat/scripts/generator/fields/main.go index 7a883160368..8e5eef92cc9 100644 --- a/filebeat/scripts/generator/fields/main.go +++ b/filebeat/scripts/generator/fields/main.go @@ -66,6 +66,7 @@ type pipeline struct { type field struct { Type string Elements []string + Hint string } type fieldYml struct { @@ -95,17 +96,19 @@ func newFieldYml(name, typeName string, noDoc bool) *fieldYml { func newField(lp string) field { lp = lp[1 : len(lp)-1] ee := strings.Split(lp, ":") - if len(ee) != 2 { - return field{ - Type: ee[0], - Elements: nil, - } + if 2 < len(ee) && len(ee) < 3 { + return field{} } - e := strings.Split(ee[1], ".") + hint := "" + if len(ee) == 3 { + hint = ee[2] + fmt.Println(hint) + } return field{ Type: ee[0], - Elements: e, + Elements: strings.Split(ee[1], "."), + Hint: hint, } } @@ -275,13 +278,17 @@ func getFieldByName(f []*fieldYml, name string) *fieldYml { return nil } -func insertLastField(f []*fieldYml, name, typeName string, noDoc bool) []*fieldYml { +func insertLastField(f []*fieldYml, name string, field field, noDoc bool) []*fieldYml { ff := getFieldByName(f, name) if ff != nil { return f } - nf := newFieldYml(name, types[typeName], noDoc) + fieldType := field.Hint + if fieldType == "" { + fieldType = types[field.Type] + } + nf := newFieldYml(name, fieldType, noDoc) return append(f, nf) } @@ -301,7 +308,7 @@ func insertGroup(out []*fieldYml, field field, index, count int, noDoc bool) []* func generateField(out []*fieldYml, field field, index, count int, noDoc bool) []*fieldYml { if index+1 == count { - return insertLastField(out, field.Elements[index], field.Type, noDoc) + return insertLastField(out, field.Elements[index], field, noDoc) } return insertGroup(out, field, index, count, noDoc) } From e97c177541b3f402d9c1628c5bb7fea14cdbb303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 9 Jul 2018 08:25:51 +0200 Subject: [PATCH 2/6] fix condition && remove unncessary print --- filebeat/scripts/generator/fields/main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/filebeat/scripts/generator/fields/main.go b/filebeat/scripts/generator/fields/main.go index 8e5eef92cc9..1694a39a906 100644 --- a/filebeat/scripts/generator/fields/main.go +++ b/filebeat/scripts/generator/fields/main.go @@ -96,14 +96,13 @@ func newFieldYml(name, typeName string, noDoc bool) *fieldYml { func newField(lp string) field { lp = lp[1 : len(lp)-1] ee := strings.Split(lp, ":") - if 2 < len(ee) && len(ee) < 3 { + if 2 != len(ee) && len(ee) != 3 { return field{} } hint := "" if len(ee) == 3 { hint = ee[2] - fmt.Println(hint) } return field{ Type: ee[0], From 19660e3820745155a751af640c9fb0a377c53c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 13 Jul 2018 13:38:40 +0200 Subject: [PATCH 3/6] refactor and add comments --- filebeat/scripts/generator/fields/main.go | 27 ++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/filebeat/scripts/generator/fields/main.go b/filebeat/scripts/generator/fields/main.go index 1694a39a906..70501790c98 100644 --- a/filebeat/scripts/generator/fields/main.go +++ b/filebeat/scripts/generator/fields/main.go @@ -33,6 +33,10 @@ import ( const ( pipelinePath = "%s/module/%s/%s/ingest/pipeline.json" fieldsYmlPath = "%s/module/%s/%s/_meta/fields.yml" + + typeIdx = 0 + elementsIdx = 1 + hintIdx = 2 ) var ( @@ -96,21 +100,34 @@ func newFieldYml(name, typeName string, noDoc bool) *fieldYml { func newField(lp string) field { lp = lp[1 : len(lp)-1] ee := strings.Split(lp, ":") - if 2 != len(ee) && len(ee) != 3 { + if !isValidFormat(ee) { return field{} } hint := "" - if len(ee) == 3 { - hint = ee[2] + if containsHint(ee) { + hint = ee[hintIdx] } + return field{ - Type: ee[0], - Elements: strings.Split(ee[1], "."), + Type: ee[typeIdx], + Elements: strings.Split(ee[elementsIdx], "."), Hint: hint, } } +// isValidFormat checks if the input can be split correctly +// 1. if lenght is 2, the format is {type}:{field.elements} +// 2. if the lenght is 3, the format is {type}:{field.elements}:{hint} +func isValidFormat(ee []string) bool { + return len(ee) == 2 || len(ee) == 3 +} + +// the last element is the type hint +func containsHint(ee []string) bool { + return len(ee) == 3 +} + func readPipeline(beatsPath, module, fileset string) (*pipeline, error) { pp := fmt.Sprintf(pipelinePath, beatsPath, module, fileset) r, err := ioutil.ReadFile(pp) From afb2d5f07babbce5804c064a6fdafd87a87a5102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 13 Jul 2018 13:38:49 +0200 Subject: [PATCH 4/6] add unit test --- filebeat/scripts/generator/fields/main_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/filebeat/scripts/generator/fields/main_test.go b/filebeat/scripts/generator/fields/main_test.go index 04f34bbeebe..a83d025e7fc 100644 --- a/filebeat/scripts/generator/fields/main_test.go +++ b/filebeat/scripts/generator/fields/main_test.go @@ -126,6 +126,15 @@ func TestFieldsGenerator(t *testing.T) { &fieldYml{Name: "message", Description: "Please add description", Example: "Please add example", Type: "text"}, }, }, + FieldsGeneratorTestCase{ + patterns: []string{ + "\\[%{TIMESTAMP:timestamp}\\] %{NUMBER:idx:int}", + }, + fields: []*fieldYml{ + &fieldYml{Name: "timestamp", Description: "Please add description", Example: "Please add example", Type: "text"}, + &fieldYml{Name: "idx", Description: "Please add description", Example: "Please add example", Type: "int"}, + }, + }, } for _, tc := range tests { From 2bd678ea46dcddb27aab907cb81d786d989ab8a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 13 Jul 2018 13:50:57 +0200 Subject: [PATCH 5/6] refactor struct && add error checking --- filebeat/scripts/generator/fields/main.go | 44 ++++++++++++----------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/filebeat/scripts/generator/fields/main.go b/filebeat/scripts/generator/fields/main.go index 70501790c98..e0e5e2a3788 100644 --- a/filebeat/scripts/generator/fields/main.go +++ b/filebeat/scripts/generator/fields/main.go @@ -68,9 +68,9 @@ type pipeline struct { } type field struct { - Type string - Elements []string - Hint string + Syntax string + SemanticElements []string + Type string } type fieldYml struct { @@ -98,21 +98,25 @@ func newFieldYml(name, typeName string, noDoc bool) *fieldYml { } func newField(lp string) field { + if len(lp) <= 2 { + return field{} + } lp = lp[1 : len(lp)-1] + ee := strings.Split(lp, ":") if !isValidFormat(ee) { return field{} } hint := "" - if containsHint(ee) { + if containsType(ee) { hint = ee[hintIdx] } return field{ - Type: ee[typeIdx], - Elements: strings.Split(ee[elementsIdx], "."), - Hint: hint, + Syntax: ee[typeIdx], + SemanticElements: strings.Split(ee[elementsIdx], "."), + Type: hint, } } @@ -124,7 +128,7 @@ func isValidFormat(ee []string) bool { } // the last element is the type hint -func containsHint(ee []string) bool { +func containsType(ee []string) bool { return len(ee) == 3 } @@ -153,7 +157,7 @@ func addNewField(fs []field, f field) []field { return append(fs, f) } -func getElementsFromPatterns(patterns []string) ([]field, error) { +func getSemanticElementsFromPatterns(patterns []string) ([]field, error) { r, err := regexp.Compile("{[\\.\\w\\:]*}") if err != nil { return nil, err @@ -164,7 +168,7 @@ func getElementsFromPatterns(patterns []string) ([]field, error) { pp := r.FindAllString(lp, -1) for _, p := range pp { f := newField(p) - if f.Elements == nil { + if f.SemanticElements == nil { continue } fs = addNewField(fs, f) @@ -233,16 +237,16 @@ type processors struct { } func (p *processors) processFields() ([]field, error) { - f, err := getElementsFromPatterns(p.patterns) + f, err := getSemanticElementsFromPatterns(p.patterns) if err != nil { return nil, err } for i, ff := range f { - fs := strings.Join(ff.Elements, ".") + fs := strings.Join(ff.SemanticElements, ".") for k, mv := range p.rename { if k == fs { - ff.Elements = strings.Split(mv, ".") + ff.SemanticElements = strings.Split(mv, ".") } } for _, rm := range p.remove { @@ -300,16 +304,16 @@ func insertLastField(f []*fieldYml, name string, field field, noDoc bool) []*fie return f } - fieldType := field.Hint + fieldType := field.Type if fieldType == "" { - fieldType = types[field.Type] + fieldType = types[field.Syntax] } nf := newFieldYml(name, fieldType, noDoc) return append(f, nf) } func insertGroup(out []*fieldYml, field field, index, count int, noDoc bool) []*fieldYml { - g := getFieldByName(out, field.Elements[index]) + g := getFieldByName(out, field.SemanticElements[index]) if g != nil { g.Fields = generateField(g.Fields, field, index+1, count, noDoc) return out @@ -317,14 +321,14 @@ func insertGroup(out []*fieldYml, field field, index, count int, noDoc bool) []* var groupFields []*fieldYml groupFields = generateField(groupFields, field, index+1, count, noDoc) - group := newFieldYml(field.Elements[index], "group", noDoc) + group := newFieldYml(field.SemanticElements[index], "group", noDoc) group.Fields = groupFields return append(out, group) } func generateField(out []*fieldYml, field field, index, count int, noDoc bool) []*fieldYml { if index+1 == count { - return insertLastField(out, field.Elements[index], field, noDoc) + return insertLastField(out, field.SemanticElements[index], field, noDoc) } return insertGroup(out, field, index, count, noDoc) } @@ -333,10 +337,10 @@ func generateFields(f []field, noDoc bool) []*fieldYml { var out []*fieldYml for _, ff := range f { index := 1 - if len(ff.Elements) == 1 { + if len(ff.SemanticElements) == 1 { index = 0 } - out = generateField(out, ff, index, len(ff.Elements), noDoc) + out = generateField(out, ff, index, len(ff.SemanticElements), noDoc) } return out } From 07b59df3da9826a44895e922ba4176d9cab9201c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 17 Jul 2018 09:54:05 +0200 Subject: [PATCH 6/6] more renaming --- filebeat/scripts/generator/fields/main.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/filebeat/scripts/generator/fields/main.go b/filebeat/scripts/generator/fields/main.go index e0e5e2a3788..385677163e5 100644 --- a/filebeat/scripts/generator/fields/main.go +++ b/filebeat/scripts/generator/fields/main.go @@ -97,25 +97,25 @@ func newFieldYml(name, typeName string, noDoc bool) *fieldYml { } } -func newField(lp string) field { - if len(lp) <= 2 { +func newField(pattern string) field { + if len(pattern) <= 2 { return field{} } - lp = lp[1 : len(lp)-1] + pattern = pattern[1 : len(pattern)-1] - ee := strings.Split(lp, ":") - if !isValidFormat(ee) { + elements := strings.Split(pattern, ":") + if !isValidFormat(elements) { return field{} } hint := "" - if containsType(ee) { - hint = ee[hintIdx] + if containsType(elements) { + hint = elements[hintIdx] } return field{ - Syntax: ee[typeIdx], - SemanticElements: strings.Split(ee[elementsIdx], "."), + Syntax: elements[typeIdx], + SemanticElements: strings.Split(elements[elementsIdx], "."), Type: hint, } }