From d986337d5df2381862986c3bc2b128d535cecd97 Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Thu, 27 Jun 2024 23:07:42 +0300 Subject: [PATCH 01/11] Add nested fields access --- plugin/action/remove_fields/README.md | 4 +-- plugin/action/remove_fields/remove_fields.go | 20 +++++++------ .../remove_fields/remove_fields_test.go | 28 ++++++++++++++++++- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/plugin/action/remove_fields/README.md b/plugin/action/remove_fields/README.md index 21feef1b9..dd664643e 100755 --- a/plugin/action/remove_fields/README.md +++ b/plugin/action/remove_fields/README.md @@ -2,9 +2,9 @@ It removes the list of the event fields and keeps others. ### Config params -**`fields`** *`[]string`* +**`fields`** *`[]cfg.FieldSelector`* -The list of the fields to remove. +The list of the fields to remove. Nested fields supported.
diff --git a/plugin/action/remove_fields/remove_fields.go b/plugin/action/remove_fields/remove_fields.go index ae31f7edb..973d3114d 100644 --- a/plugin/action/remove_fields/remove_fields.go +++ b/plugin/action/remove_fields/remove_fields.go @@ -1,6 +1,7 @@ package remove_fields import ( + "github.com/ozontech/file.d/cfg" "github.com/ozontech/file.d/fd" "github.com/ozontech/file.d/logger" "github.com/ozontech/file.d/pipeline" @@ -11,8 +12,8 @@ It removes the list of the event fields and keeps others. }*/ type Plugin struct { - config *Config - fieldsBuf []string + config *Config + fieldPaths [][]string } // ! config-params @@ -20,8 +21,8 @@ type Plugin struct { type Config struct { // > @3@4@5@6 // > - // > The list of the fields to remove. - Fields []string `json:"fields"` // * + // > The list of the fields to remove. Nested fields supported. + Fields []cfg.FieldSelector `json:"fields"` // * } func init() { @@ -40,20 +41,23 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams if p.config == nil { logger.Panicf("config is nil for the remove fields plugin") } + + p.fieldPaths = make([][]string, 0, len(p.config.Fields)) + for _, field := range p.config.Fields { + p.fieldPaths = append(p.fieldPaths, cfg.ParseFieldSelector(string(field))) + } } func (p *Plugin) Stop() { } func (p *Plugin) Do(event *pipeline.Event) pipeline.ActionResult { - p.fieldsBuf = p.fieldsBuf[:0] - if !event.Root.IsObject() { return pipeline.ActionPass } - for _, field := range p.config.Fields { - event.Root.Dig(field).Suicide() + for _, fieldPath := range p.fieldPaths { + event.Root.Dig(fieldPath...).Suicide() } return pipeline.ActionPass diff --git a/plugin/action/remove_fields/remove_fields_test.go b/plugin/action/remove_fields/remove_fields_test.go index 9d7c24858..45fb20062 100644 --- a/plugin/action/remove_fields/remove_fields_test.go +++ b/plugin/action/remove_fields/remove_fields_test.go @@ -1,6 +1,7 @@ package remove_fields import ( + "github.com/ozontech/file.d/cfg" "sync" "testing" @@ -10,7 +11,7 @@ import ( ) func TestRemoveFields(t *testing.T) { - config := test.NewConfig(&Config{Fields: []string{"field_1", "field_2"}}, nil) + config := test.NewConfig(&Config{Fields: []cfg.FieldSelector{"field_1", "field_2"}}, nil) p, input, output := test.NewPipelineMock(test.NewActionPluginStaticInfo(factory, config, pipeline.MatchModeAnd, nil, false)) wg := &sync.WaitGroup{} wg.Add(3) @@ -33,3 +34,28 @@ func TestRemoveFields(t *testing.T) { assert.Equal(t, `{"b":"c"}`, outEvents[1], "wrong event") assert.Equal(t, `{"field_3":"value_3","a":"b"}`, outEvents[2], "wrong event") } + +func TestRemoveNestedFields(t *testing.T) { + config := test.NewConfig(&Config{Fields: []cfg.FieldSelector{"a.b"}}, nil) + p, input, output := test.NewPipelineMock(test.NewActionPluginStaticInfo(factory, config, pipeline.MatchModeAnd, nil, false)) + wg := &sync.WaitGroup{} + wg.Add(3) + + outEvents := make([]string, 0, 3) + output.SetOutFn(func(e *pipeline.Event) { + outEvents = append(outEvents, e.Root.EncodeToString()) + wg.Done() + }) + + input.In(0, "test.log", 0, []byte(`{"a":"some"}`)) + input.In(0, "test.log", 0, []byte(`{"a":{"b":"deleted"}}`)) + input.In(0, "test.log", 0, []byte(`{"a":{"b":{"c":["deleted"]},"d":"saved"}}`)) + + wg.Wait() + p.Stop() + + assert.Equal(t, 3, len(outEvents), "wrong out events count") + assert.Equal(t, `{"a":"some"}`, outEvents[0], "wrong event") + assert.Equal(t, `{"a":{}}`, outEvents[1], "wrong event") + assert.Equal(t, `{"a":{"d":"saved"}}`, outEvents[2], "wrong event") +} From c6f591d6b15e75e27bb2c829ae627034e7eebb48 Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Thu, 27 Jun 2024 23:13:56 +0300 Subject: [PATCH 02/11] Fix linter error --- plugin/action/remove_fields/remove_fields_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/action/remove_fields/remove_fields_test.go b/plugin/action/remove_fields/remove_fields_test.go index 45fb20062..dcfa847f0 100644 --- a/plugin/action/remove_fields/remove_fields_test.go +++ b/plugin/action/remove_fields/remove_fields_test.go @@ -1,10 +1,10 @@ package remove_fields import ( - "github.com/ozontech/file.d/cfg" "sync" "testing" + "github.com/ozontech/file.d/cfg" "github.com/ozontech/file.d/pipeline" "github.com/ozontech/file.d/test" "github.com/stretchr/testify/assert" From 6b6c98aadfbfa43b74728eeac005aec846ef393d Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Fri, 28 Jun 2024 16:52:45 +0300 Subject: [PATCH 03/11] Validate field paths --- plugin/action/remove_fields/remove_fields.go | 55 +++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/plugin/action/remove_fields/remove_fields.go b/plugin/action/remove_fields/remove_fields.go index 973d3114d..660706e87 100644 --- a/plugin/action/remove_fields/remove_fields.go +++ b/plugin/action/remove_fields/remove_fields.go @@ -1,6 +1,8 @@ package remove_fields import ( + "strings" + "github.com/ozontech/file.d/cfg" "github.com/ozontech/file.d/fd" "github.com/ozontech/file.d/logger" @@ -42,10 +44,59 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams logger.Panicf("config is nil for the remove fields plugin") } - p.fieldPaths = make([][]string, 0, len(p.config.Fields)) + fieldPaths := make([][]string, 0, len(p.config.Fields)) for _, field := range p.config.Fields { - p.fieldPaths = append(p.fieldPaths, cfg.ParseFieldSelector(string(field))) + fieldPath := cfg.ParseFieldSelector(string(field)) + + // Setting empty path leads to digging immortal root node + if len(fieldPath) == 0 { + logger.Fatalf("can't remove entire object; use discard plugin instead") + } + + fieldPaths = append(fieldPaths, fieldPath) + } + + ok := make([]bool, len(fieldPaths)) + for i := range ok { + ok[i] = true + } + + for i := range fieldPaths { + for j := range fieldPaths { + if i == j { + continue + } + + if includes(fieldPaths[i], fieldPaths[j]) { + logger.Warnf( + "'%s' path includes '%s' path; remove nested path", + strings.Join(fieldPaths[i], "."), + strings.Join(fieldPaths[j], "."), + ) + ok[j] = false + } + } } + + p.fieldPaths = make([][]string, 0, len(fieldPaths)) + for i, fieldPath := range fieldPaths { + if ok[i] { + p.fieldPaths = append(p.fieldPaths, fieldPath) + } + } +} + +func includes(a, b []string) bool { + if !(len(a) <= len(b)) { + return false + } + + result := true + for i := 0; i < len(a); i++ { + result = result && a[i] == b[i] + } + + return result } func (p *Plugin) Stop() { From c89ce2209a5ebd5c285e62c3fb56198b92b8db11 Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Fri, 28 Jun 2024 22:45:34 +0300 Subject: [PATCH 04/11] Enable duplicating field paths --- plugin/action/remove_fields/remove_fields.go | 13 +++++++++-- .../remove_fields/remove_fields_test.go | 23 ++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/plugin/action/remove_fields/remove_fields.go b/plugin/action/remove_fields/remove_fields.go index 660706e87..ad1e68319 100644 --- a/plugin/action/remove_fields/remove_fields.go +++ b/plugin/action/remove_fields/remove_fields.go @@ -1,6 +1,7 @@ package remove_fields import ( + "slices" "strings" "github.com/ozontech/file.d/cfg" @@ -24,7 +25,7 @@ type Config struct { // > @3@4@5@6 // > // > The list of the fields to remove. Nested fields supported. - Fields []cfg.FieldSelector `json:"fields"` // * + Fields []string `json:"fields"` // * } func init() { @@ -46,7 +47,7 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams fieldPaths := make([][]string, 0, len(p.config.Fields)) for _, field := range p.config.Fields { - fieldPath := cfg.ParseFieldSelector(string(field)) + fieldPath := cfg.ParseFieldSelector(field) // Setting empty path leads to digging immortal root node if len(fieldPath) == 0 { @@ -67,6 +68,14 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams continue } + if slices.Equal(fieldPaths[i], fieldPaths[j]) { + logger.Warnf( + "duplicate path '%s' found; remove extra occurrence", + strings.Join(fieldPaths[i], "."), + ) + continue + } + if includes(fieldPaths[i], fieldPaths[j]) { logger.Warnf( "'%s' path includes '%s' path; remove nested path", diff --git a/plugin/action/remove_fields/remove_fields_test.go b/plugin/action/remove_fields/remove_fields_test.go index dcfa847f0..685c68488 100644 --- a/plugin/action/remove_fields/remove_fields_test.go +++ b/plugin/action/remove_fields/remove_fields_test.go @@ -4,14 +4,14 @@ import ( "sync" "testing" - "github.com/ozontech/file.d/cfg" "github.com/ozontech/file.d/pipeline" "github.com/ozontech/file.d/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestRemoveFields(t *testing.T) { - config := test.NewConfig(&Config{Fields: []cfg.FieldSelector{"field_1", "field_2"}}, nil) + config := test.NewConfig(&Config{Fields: []string{"field_1", "field_2"}}, nil) p, input, output := test.NewPipelineMock(test.NewActionPluginStaticInfo(factory, config, pipeline.MatchModeAnd, nil, false)) wg := &sync.WaitGroup{} wg.Add(3) @@ -36,7 +36,7 @@ func TestRemoveFields(t *testing.T) { } func TestRemoveNestedFields(t *testing.T) { - config := test.NewConfig(&Config{Fields: []cfg.FieldSelector{"a.b"}}, nil) + config := test.NewConfig(&Config{Fields: []string{"a.b"}}, nil) p, input, output := test.NewPipelineMock(test.NewActionPluginStaticInfo(factory, config, pipeline.MatchModeAnd, nil, false)) wg := &sync.WaitGroup{} wg.Add(3) @@ -59,3 +59,20 @@ func TestRemoveNestedFields(t *testing.T) { assert.Equal(t, `{"a":{}}`, outEvents[1], "wrong event") assert.Equal(t, `{"a":{"d":"saved"}}`, outEvents[2], "wrong event") } + +func TestDuplicatingFieldSelectors(t *testing.T) { + config := test.NewConfig(&Config{Fields: []string{"a.b", "a.b"}}, nil) + p := Plugin{} + p.Start(config, nil) + + require.NotEmpty(t, p.fieldPaths) + require.Equal(t, []string{"a", "b"}, p.fieldPaths[0]) +} + +func TestNestedFieldSelectors(t *testing.T) { + config := test.NewConfig(&Config{Fields: []string{"a.b", "a.b.c", "a.d", "a"}}, nil) + p := Plugin{} + p.Start(config, nil) + + require.Equal(t, [][]string{{"a"}}, p.fieldPaths) +} From f9ddbbb97017f9fdd212e87be3059e46a151d86c Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Fri, 28 Jun 2024 22:51:13 +0300 Subject: [PATCH 05/11] Regenerate doc --- plugin/action/remove_fields/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/action/remove_fields/README.md b/plugin/action/remove_fields/README.md index dd664643e..a297189f3 100755 --- a/plugin/action/remove_fields/README.md +++ b/plugin/action/remove_fields/README.md @@ -2,7 +2,7 @@ It removes the list of the event fields and keeps others. ### Config params -**`fields`** *`[]cfg.FieldSelector`* +**`fields`** *`[]string`* The list of the fields to remove. Nested fields supported. From 58c82e3cc6ce4a374bc887edc0b16e481b81a138 Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Tue, 2 Jul 2024 17:02:54 +0300 Subject: [PATCH 06/11] Refactor --- plugin/action/remove_fields/remove_fields.go | 39 +++++++++----------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/plugin/action/remove_fields/remove_fields.go b/plugin/action/remove_fields/remove_fields.go index ad1e68319..ec5094b11 100644 --- a/plugin/action/remove_fields/remove_fields.go +++ b/plugin/action/remove_fields/remove_fields.go @@ -46,51 +46,48 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams } fieldPaths := make([][]string, 0, len(p.config.Fields)) - for _, field := range p.config.Fields { - fieldPath := cfg.ParseFieldSelector(field) + for i, field := range p.config.Fields { + if field == "" { + logger.Fatalf("empty field; pos = %d", i) + } - // Setting empty path leads to digging immortal root node + fieldPath := cfg.ParseFieldSelector(field) if len(fieldPath) == 0 { - logger.Fatalf("can't remove entire object; use discard plugin instead") + logger.Fatalf("empty field selector parsed; field pos = %d", i) } fieldPaths = append(fieldPaths, fieldPath) } - ok := make([]bool, len(fieldPaths)) - for i := range ok { - ok[i] = true - } + p.fieldPaths = make([][]string, 0, len(fieldPaths)) - for i := range fieldPaths { - for j := range fieldPaths { + for i, p1 := range fieldPaths { + ok := true + for j, p2 := range fieldPaths { if i == j { continue } - if slices.Equal(fieldPaths[i], fieldPaths[j]) { + if slices.Equal(p1, p2) { logger.Warnf( "duplicate path '%s' found; remove extra occurrence", - strings.Join(fieldPaths[i], "."), + strings.Join(p1, "."), ) continue } - if includes(fieldPaths[i], fieldPaths[j]) { + if includes(p2, p1) { logger.Warnf( "'%s' path includes '%s' path; remove nested path", - strings.Join(fieldPaths[i], "."), - strings.Join(fieldPaths[j], "."), + strings.Join(p2, "."), + strings.Join(p1, "."), ) - ok[j] = false + ok = false } } - } - p.fieldPaths = make([][]string, 0, len(fieldPaths)) - for i, fieldPath := range fieldPaths { - if ok[i] { - p.fieldPaths = append(p.fieldPaths, fieldPath) + if ok { + p.fieldPaths = append(p.fieldPaths, p1) } } } From 5a0e6a2ca3e814fb6c1a1b95958326ce5e0fe3a4 Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Tue, 2 Jul 2024 18:57:57 +0300 Subject: [PATCH 07/11] Refactor --- plugin/action/remove_fields/remove_fields.go | 59 +++++++------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/plugin/action/remove_fields/remove_fields.go b/plugin/action/remove_fields/remove_fields.go index ec5094b11..e960821a6 100644 --- a/plugin/action/remove_fields/remove_fields.go +++ b/plugin/action/remove_fields/remove_fields.go @@ -1,7 +1,7 @@ package remove_fields import ( - "slices" + "sort" "strings" "github.com/ozontech/file.d/cfg" @@ -45,66 +45,45 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams logger.Panicf("config is nil for the remove fields plugin") } - fieldPaths := make([][]string, 0, len(p.config.Fields)) - for i, field := range p.config.Fields { - if field == "" { + fields := p.config.Fields + sort.Slice(fields, func(i, j int) bool { + return len(fields[i]) < len(fields[j]) + }) + + p.fieldPaths = make([][]string, 0, len(fields)) + + for i, f1 := range fields { + if f1 == "" { logger.Fatalf("empty field; pos = %d", i) } - fieldPath := cfg.ParseFieldSelector(field) + fieldPath := cfg.ParseFieldSelector(f1) if len(fieldPath) == 0 { logger.Fatalf("empty field selector parsed; field pos = %d", i) } - fieldPaths = append(fieldPaths, fieldPath) - } - - p.fieldPaths = make([][]string, 0, len(fieldPaths)) - - for i, p1 := range fieldPaths { ok := true - for j, p2 := range fieldPaths { - if i == j { - continue - } + for j := 0; j < i; j++ { + f2 := fields[j] - if slices.Equal(p1, p2) { - logger.Warnf( - "duplicate path '%s' found; remove extra occurrence", - strings.Join(p1, "."), - ) + if f1 == f2 { + logger.Warnf("duplicate path '%s' found; remove extra occurrences", f1) continue } - if includes(p2, p1) { - logger.Warnf( - "'%s' path includes '%s' path; remove nested path", - strings.Join(p2, "."), - strings.Join(p1, "."), - ) + if strings.HasPrefix(f1, f2) { + logger.Warnf("path '%s' included in '%s' path; remove nested path", f1, f2) ok = false + break } } if ok { - p.fieldPaths = append(p.fieldPaths, p1) + p.fieldPaths = append(p.fieldPaths, fieldPath) } } } -func includes(a, b []string) bool { - if !(len(a) <= len(b)) { - return false - } - - result := true - for i := 0; i < len(a); i++ { - result = result && a[i] == b[i] - } - - return result -} - func (p *Plugin) Stop() { } From f0c7caf4467634c98ab465b410d3ba0f4554b10b Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Tue, 2 Jul 2024 19:14:25 +0300 Subject: [PATCH 08/11] Edit doc --- plugin/action/remove_fields/README.md | 8 +++++++- plugin/action/remove_fields/remove_fields.go | 10 ++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/plugin/action/remove_fields/README.md b/plugin/action/remove_fields/README.md index a297189f3..a4ebcd405 100755 --- a/plugin/action/remove_fields/README.md +++ b/plugin/action/remove_fields/README.md @@ -4,7 +4,13 @@ It removes the list of the event fields and keeps others. ### Config params **`fields`** *`[]string`* -The list of the fields to remove. Nested fields supported. +The list of the fields to remove. +Nested fields supported: list subfield names separated with dot. +Example: +``` +fields: ["a.b.c"] +{"a":{"b":{"c": 100}}} -> {"a":{"b":{}}} +```
diff --git a/plugin/action/remove_fields/remove_fields.go b/plugin/action/remove_fields/remove_fields.go index e960821a6..8d7b3c874 100644 --- a/plugin/action/remove_fields/remove_fields.go +++ b/plugin/action/remove_fields/remove_fields.go @@ -24,7 +24,13 @@ type Plugin struct { type Config struct { // > @3@4@5@6 // > - // > The list of the fields to remove. Nested fields supported. + // > The list of the fields to remove. + // > Nested fields supported: list subfield names separated with dot. + // > Example: + // > ``` + // > fields: ["a.b.c"] + // > {"a":{"b":{"c": 100}}} -> {"a":{"b":{}}} + // > ``` Fields []string `json:"fields"` // * } @@ -72,7 +78,7 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams } if strings.HasPrefix(f1, f2) { - logger.Warnf("path '%s' included in '%s' path; remove nested path", f1, f2) + logger.Warnf("path '%s' included in path '%s'; remove nested path", f1, f2) ok = false break } From 318f9ddc5dcd3e3f0190d509447ed470831ec605 Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Wed, 3 Jul 2024 18:37:41 +0300 Subject: [PATCH 09/11] Refactor. Improve doc --- plugin/action/remove_fields/README.md | 38 ++++++++++++- plugin/action/remove_fields/remove_fields.go | 56 ++++++++++++++------ 2 files changed, 77 insertions(+), 17 deletions(-) diff --git a/plugin/action/remove_fields/README.md b/plugin/action/remove_fields/README.md index a4ebcd405..ec6339486 100755 --- a/plugin/action/remove_fields/README.md +++ b/plugin/action/remove_fields/README.md @@ -9,7 +9,43 @@ Nested fields supported: list subfield names separated with dot. Example: ``` fields: ["a.b.c"] -{"a":{"b":{"c": 100}}} -> {"a":{"b":{}}} + +# event before processing +{ + "a": { + "b": { + "c": 100, + "d": "some" + } + } +} + +# event after processing +{ + "a": { + "b": { + "d": "some" # "c" removed + } + } +} +``` + +If field name contains dots use backslash for escaping. +Example: +``` +fields: + - exception\.type + +# event before processing +{ + "message": "Exception occurred", + "exception.type": "SomeType" +} + +# event after processing +{ + "message": "Exception occurred" # "exception.type" removed +} ```
diff --git a/plugin/action/remove_fields/remove_fields.go b/plugin/action/remove_fields/remove_fields.go index 8d7b3c874..d32ed410c 100644 --- a/plugin/action/remove_fields/remove_fields.go +++ b/plugin/action/remove_fields/remove_fields.go @@ -29,7 +29,43 @@ type Config struct { // > Example: // > ``` // > fields: ["a.b.c"] - // > {"a":{"b":{"c": 100}}} -> {"a":{"b":{}}} + // > + // > # event before processing + // > { + // > "a": { + // > "b": { + // > "c": 100, + // > "d": "some" + // > } + // > } + // > } + // > + // > # event after processing + // > { + // > "a": { + // > "b": { + // > "d": "some" # "c" removed + // > } + // > } + // > } + // > ``` + // > + // > If field name contains dots use backslash for escaping. + // > Example: + // > ``` + // > fields: + // > - exception\.type + // > + // > # event before processing + // > { + // > "message": "Exception occurred", + // > "exception.type": "SomeType" + // > } + // > + // > # event after processing + // > { + // > "message": "Exception occurred" # "exception.type" removed + // > } // > ``` Fields []string `json:"fields"` // * } @@ -60,23 +96,11 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams for i, f1 := range fields { if f1 == "" { - logger.Fatalf("empty field; pos = %d", i) - } - - fieldPath := cfg.ParseFieldSelector(f1) - if len(fieldPath) == 0 { - logger.Fatalf("empty field selector parsed; field pos = %d", i) + logger.Fatal("empty field found") } ok := true - for j := 0; j < i; j++ { - f2 := fields[j] - - if f1 == f2 { - logger.Warnf("duplicate path '%s' found; remove extra occurrences", f1) - continue - } - + for _, f2 := range fields[:i] { if strings.HasPrefix(f1, f2) { logger.Warnf("path '%s' included in path '%s'; remove nested path", f1, f2) ok = false @@ -85,7 +109,7 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams } if ok { - p.fieldPaths = append(p.fieldPaths, fieldPath) + p.fieldPaths = append(p.fieldPaths, cfg.ParseFieldSelector(f1)) } } } From 1c2b4860831f1d33411e1598e2f38b5cf07bfb94 Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Tue, 9 Jul 2024 13:17:49 +0300 Subject: [PATCH 10/11] Edit doc. Delete fatal, add warnings --- plugin/README.md | 42 +++++++++ plugin/action/README.md | 42 +++++++++ plugin/action/remove_fields/README.md | 10 +-- plugin/action/remove_fields/remove_fields.go | 91 +++++++++++--------- 4 files changed, 137 insertions(+), 48 deletions(-) diff --git a/plugin/README.md b/plugin/README.md index e31c8496d..e729888ee 100755 --- a/plugin/README.md +++ b/plugin/README.md @@ -616,6 +616,48 @@ It parses string from the event field using re2 expression with named subgroups [More details...](plugin/action/parse_re2/README.md) ## remove_fields It removes the list of the event fields and keeps others. +Nested fields supported: list subfield names separated with dot. +Example: +``` +fields: ["a.b.c"] + +# event before processing +{ + "a": { + "b": { + "c": 100, + "d": "some" + } + } +} + +# event after processing +{ + "a": { + "b": { + "d": "some" # "c" removed + } + } +} +``` + +If field name contains dots use backslash for escaping. +Example: +``` +fields: + - exception\.type + +# event before processing +{ + "message": "Exception occurred", + "exception.type": "SomeType" +} + +# event after processing +{ + "message": "Exception occurred" # "exception.type" removed +} +``` [More details...](plugin/action/remove_fields/README.md) ## rename diff --git a/plugin/action/README.md b/plugin/action/README.md index 11bd41711..8cb543fe8 100755 --- a/plugin/action/README.md +++ b/plugin/action/README.md @@ -447,6 +447,48 @@ It parses string from the event field using re2 expression with named subgroups [More details...](plugin/action/parse_re2/README.md) ## remove_fields It removes the list of the event fields and keeps others. +Nested fields supported: list subfield names separated with dot. +Example: +``` +fields: ["a.b.c"] + +# event before processing +{ + "a": { + "b": { + "c": 100, + "d": "some" + } + } +} + +# event after processing +{ + "a": { + "b": { + "d": "some" # "c" removed + } + } +} +``` + +If field name contains dots use backslash for escaping. +Example: +``` +fields: + - exception\.type + +# event before processing +{ + "message": "Exception occurred", + "exception.type": "SomeType" +} + +# event after processing +{ + "message": "Exception occurred" # "exception.type" removed +} +``` [More details...](plugin/action/remove_fields/README.md) ## rename diff --git a/plugin/action/remove_fields/README.md b/plugin/action/remove_fields/README.md index ec6339486..5ad611e25 100755 --- a/plugin/action/remove_fields/README.md +++ b/plugin/action/remove_fields/README.md @@ -1,10 +1,5 @@ # Remove fields plugin It removes the list of the event fields and keeps others. - -### Config params -**`fields`** *`[]string`* - -The list of the fields to remove. Nested fields supported: list subfield names separated with dot. Example: ``` @@ -48,6 +43,11 @@ fields: } ``` +### Config params +**`fields`** *`[]string`* + +The list of the fields to remove. +
diff --git a/plugin/action/remove_fields/remove_fields.go b/plugin/action/remove_fields/remove_fields.go index d32ed410c..3eea02deb 100644 --- a/plugin/action/remove_fields/remove_fields.go +++ b/plugin/action/remove_fields/remove_fields.go @@ -12,6 +12,48 @@ import ( /*{ introduction It removes the list of the event fields and keeps others. +Nested fields supported: list subfield names separated with dot. +Example: +``` +fields: ["a.b.c"] + +# event before processing +{ + "a": { + "b": { + "c": 100, + "d": "some" + } + } +} + +# event after processing +{ + "a": { + "b": { + "d": "some" # "c" removed + } + } +} +``` + +If field name contains dots use backslash for escaping. +Example: +``` +fields: + - exception\.type + +# event before processing +{ + "message": "Exception occurred", + "exception.type": "SomeType" +} + +# event after processing +{ + "message": "Exception occurred" # "exception.type" removed +} +``` }*/ type Plugin struct { @@ -25,48 +67,6 @@ type Config struct { // > @3@4@5@6 // > // > The list of the fields to remove. - // > Nested fields supported: list subfield names separated with dot. - // > Example: - // > ``` - // > fields: ["a.b.c"] - // > - // > # event before processing - // > { - // > "a": { - // > "b": { - // > "c": 100, - // > "d": "some" - // > } - // > } - // > } - // > - // > # event after processing - // > { - // > "a": { - // > "b": { - // > "d": "some" # "c" removed - // > } - // > } - // > } - // > ``` - // > - // > If field name contains dots use backslash for escaping. - // > Example: - // > ``` - // > fields: - // > - exception\.type - // > - // > # event before processing - // > { - // > "message": "Exception occurred", - // > "exception.type": "SomeType" - // > } - // > - // > # event after processing - // > { - // > "message": "Exception occurred" # "exception.type" removed - // > } - // > ``` Fields []string `json:"fields"` // * } @@ -96,7 +96,8 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams for i, f1 := range fields { if f1 == "" { - logger.Fatal("empty field found") + logger.Warn("empty field found") + continue } ok := true @@ -112,6 +113,10 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams p.fieldPaths = append(p.fieldPaths, cfg.ParseFieldSelector(f1)) } } + + if len(p.fieldPaths) == 0 { + logger.Warn("no fields will be removed") + } } func (p *Plugin) Stop() { From 2e0d3efd9de9c967624d462950e60052fb377ecb Mon Sep 17 00:00:00 2001 From: george pogosyan Date: Thu, 1 Aug 2024 15:37:17 +0300 Subject: [PATCH 11/11] Add comment --- .gitignore | 1 + plugin/action/remove_fields/remove_fields.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 2e8efd29c..3259e0b32 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ tests-offsets testdata .idea/ +.DS_Store diff --git a/plugin/action/remove_fields/remove_fields.go b/plugin/action/remove_fields/remove_fields.go index 3eea02deb..9bf264b57 100644 --- a/plugin/action/remove_fields/remove_fields.go +++ b/plugin/action/remove_fields/remove_fields.go @@ -87,6 +87,12 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams logger.Panicf("config is nil for the remove fields plugin") } + // remove nested fields selection; + // for example: + // config `fields: ["a", "a.b"]` is equal to + // config `fields: ["a"]` + // see tests: TestDuplicatingFieldSelectors, TestNestedFieldSelectors + fields := p.config.Fields sort.Slice(fields, func(i, j int) bool { return len(fields[i]) < len(fields[j])