diff --git a/.gitignore b/.gitignore index 2e8efd29c..3259e0b32 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ tests-offsets testdata .idea/ +.DS_Store diff --git a/plugin/README.md b/plugin/README.md index e31c8496d..e729888ee 100755 --- a/plugin/README.md +++ b/plugin/README.md @@ -616,6 +616,48 @@ It parses string from the event field using re2 expression with named subgroups [More details...](plugin/action/parse_re2/README.md) ## remove_fields It removes the list of the event fields and keeps others. +Nested fields supported: list subfield names separated with dot. +Example: +``` +fields: ["a.b.c"] + +# event before processing +{ + "a": { + "b": { + "c": 100, + "d": "some" + } + } +} + +# event after processing +{ + "a": { + "b": { + "d": "some" # "c" removed + } + } +} +``` + +If field name contains dots use backslash for escaping. +Example: +``` +fields: + - exception\.type + +# event before processing +{ + "message": "Exception occurred", + "exception.type": "SomeType" +} + +# event after processing +{ + "message": "Exception occurred" # "exception.type" removed +} +``` [More details...](plugin/action/remove_fields/README.md) ## rename diff --git a/plugin/action/README.md b/plugin/action/README.md index 11bd41711..8cb543fe8 100755 --- a/plugin/action/README.md +++ b/plugin/action/README.md @@ -447,6 +447,48 @@ It parses string from the event field using re2 expression with named subgroups [More details...](plugin/action/parse_re2/README.md) ## remove_fields It removes the list of the event fields and keeps others. +Nested fields supported: list subfield names separated with dot. +Example: +``` +fields: ["a.b.c"] + +# event before processing +{ + "a": { + "b": { + "c": 100, + "d": "some" + } + } +} + +# event after processing +{ + "a": { + "b": { + "d": "some" # "c" removed + } + } +} +``` + +If field name contains dots use backslash for escaping. +Example: +``` +fields: + - exception\.type + +# event before processing +{ + "message": "Exception occurred", + "exception.type": "SomeType" +} + +# event after processing +{ + "message": "Exception occurred" # "exception.type" removed +} +``` [More details...](plugin/action/remove_fields/README.md) ## rename diff --git a/plugin/action/remove_fields/README.md b/plugin/action/remove_fields/README.md index 21feef1b9..5ad611e25 100755 --- a/plugin/action/remove_fields/README.md +++ b/plugin/action/remove_fields/README.md @@ -1,5 +1,47 @@ # Remove fields plugin It removes the list of the event fields and keeps others. +Nested fields supported: list subfield names separated with dot. +Example: +``` +fields: ["a.b.c"] + +# event before processing +{ + "a": { + "b": { + "c": 100, + "d": "some" + } + } +} + +# event after processing +{ + "a": { + "b": { + "d": "some" # "c" removed + } + } +} +``` + +If field name contains dots use backslash for escaping. +Example: +``` +fields: + - exception\.type + +# event before processing +{ + "message": "Exception occurred", + "exception.type": "SomeType" +} + +# event after processing +{ + "message": "Exception occurred" # "exception.type" removed +} +``` ### Config params **`fields`** *`[]string`* diff --git a/plugin/action/remove_fields/remove_fields.go b/plugin/action/remove_fields/remove_fields.go index ae31f7edb..9bf264b57 100644 --- a/plugin/action/remove_fields/remove_fields.go +++ b/plugin/action/remove_fields/remove_fields.go @@ -1,6 +1,10 @@ package remove_fields import ( + "sort" + "strings" + + "github.com/ozontech/file.d/cfg" "github.com/ozontech/file.d/fd" "github.com/ozontech/file.d/logger" "github.com/ozontech/file.d/pipeline" @@ -8,11 +12,53 @@ import ( /*{ introduction It removes the list of the event fields and keeps others. +Nested fields supported: list subfield names separated with dot. +Example: +``` +fields: ["a.b.c"] + +# event before processing +{ + "a": { + "b": { + "c": 100, + "d": "some" + } + } +} + +# event after processing +{ + "a": { + "b": { + "d": "some" # "c" removed + } + } +} +``` + +If field name contains dots use backslash for escaping. +Example: +``` +fields: + - exception\.type + +# event before processing +{ + "message": "Exception occurred", + "exception.type": "SomeType" +} + +# event after processing +{ + "message": "Exception occurred" # "exception.type" removed +} +``` }*/ type Plugin struct { - config *Config - fieldsBuf []string + config *Config + fieldPaths [][]string } // ! config-params @@ -40,20 +86,55 @@ func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams if p.config == nil { logger.Panicf("config is nil for the remove fields plugin") } + + // remove nested fields selection; + // for example: + // config `fields: ["a", "a.b"]` is equal to + // config `fields: ["a"]` + // see tests: TestDuplicatingFieldSelectors, TestNestedFieldSelectors + + fields := p.config.Fields + sort.Slice(fields, func(i, j int) bool { + return len(fields[i]) < len(fields[j]) + }) + + p.fieldPaths = make([][]string, 0, len(fields)) + + for i, f1 := range fields { + if f1 == "" { + logger.Warn("empty field found") + continue + } + + ok := true + for _, f2 := range fields[:i] { + if strings.HasPrefix(f1, f2) { + logger.Warnf("path '%s' included in path '%s'; remove nested path", f1, f2) + ok = false + break + } + } + + if ok { + p.fieldPaths = append(p.fieldPaths, cfg.ParseFieldSelector(f1)) + } + } + + if len(p.fieldPaths) == 0 { + logger.Warn("no fields will be removed") + } } func (p *Plugin) Stop() { } func (p *Plugin) Do(event *pipeline.Event) pipeline.ActionResult { - p.fieldsBuf = p.fieldsBuf[:0] - if !event.Root.IsObject() { return pipeline.ActionPass } - for _, field := range p.config.Fields { - event.Root.Dig(field).Suicide() + for _, fieldPath := range p.fieldPaths { + event.Root.Dig(fieldPath...).Suicide() } return pipeline.ActionPass diff --git a/plugin/action/remove_fields/remove_fields_test.go b/plugin/action/remove_fields/remove_fields_test.go index 9d7c24858..685c68488 100644 --- a/plugin/action/remove_fields/remove_fields_test.go +++ b/plugin/action/remove_fields/remove_fields_test.go @@ -7,6 +7,7 @@ import ( "github.com/ozontech/file.d/pipeline" "github.com/ozontech/file.d/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestRemoveFields(t *testing.T) { @@ -33,3 +34,45 @@ func TestRemoveFields(t *testing.T) { assert.Equal(t, `{"b":"c"}`, outEvents[1], "wrong event") assert.Equal(t, `{"field_3":"value_3","a":"b"}`, outEvents[2], "wrong event") } + +func TestRemoveNestedFields(t *testing.T) { + config := test.NewConfig(&Config{Fields: []string{"a.b"}}, nil) + p, input, output := test.NewPipelineMock(test.NewActionPluginStaticInfo(factory, config, pipeline.MatchModeAnd, nil, false)) + wg := &sync.WaitGroup{} + wg.Add(3) + + outEvents := make([]string, 0, 3) + output.SetOutFn(func(e *pipeline.Event) { + outEvents = append(outEvents, e.Root.EncodeToString()) + wg.Done() + }) + + input.In(0, "test.log", 0, []byte(`{"a":"some"}`)) + input.In(0, "test.log", 0, []byte(`{"a":{"b":"deleted"}}`)) + input.In(0, "test.log", 0, []byte(`{"a":{"b":{"c":["deleted"]},"d":"saved"}}`)) + + wg.Wait() + p.Stop() + + assert.Equal(t, 3, len(outEvents), "wrong out events count") + assert.Equal(t, `{"a":"some"}`, outEvents[0], "wrong event") + assert.Equal(t, `{"a":{}}`, outEvents[1], "wrong event") + assert.Equal(t, `{"a":{"d":"saved"}}`, outEvents[2], "wrong event") +} + +func TestDuplicatingFieldSelectors(t *testing.T) { + config := test.NewConfig(&Config{Fields: []string{"a.b", "a.b"}}, nil) + p := Plugin{} + p.Start(config, nil) + + require.NotEmpty(t, p.fieldPaths) + require.Equal(t, []string{"a", "b"}, p.fieldPaths[0]) +} + +func TestNestedFieldSelectors(t *testing.T) { + config := test.NewConfig(&Config{Fields: []string{"a.b", "a.b.c", "a.d", "a"}}, nil) + p := Plugin{} + p.Start(config, nil) + + require.Equal(t, [][]string{{"a"}}, p.fieldPaths) +}