diff --git a/.chloggen/ottl-log-body-indexing.yaml b/.chloggen/ottl-log-body-indexing.yaml new file mode 100755 index 000000000000..894a45332770 --- /dev/null +++ b/.chloggen/ottl-log-body-indexing.yaml @@ -0,0 +1,20 @@ +# Use this changelog template to create an entry for release notes. +# If your change doesn't affect end users, such as a test fix or a tooling change, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Allow indexing map and slice log bodies + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [17396, 22068] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/pkg/ottl/contexts/internal/map.go b/pkg/ottl/contexts/internal/map.go index 29fa85c74e9a..63be64719379 100644 --- a/pkg/ottl/contexts/internal/map.go +++ b/pkg/ottl/contexts/internal/map.go @@ -9,7 +9,6 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/internal/ottlcommon" ) func GetMapValue(m pcommon.Map, keys []ottl.Key) (interface{}, error) { @@ -24,29 +23,7 @@ func GetMapValue(m pcommon.Map, keys []ottl.Key) (interface{}, error) { if !ok { return nil, nil } - for i := 1; i < len(keys); i++ { - switch val.Type() { - case pcommon.ValueTypeMap: - if keys[i].String == nil { - return nil, fmt.Errorf("map must be indexed by a string") - } - val, ok = val.Map().Get(*keys[i].String) - if !ok { - return nil, nil - } - case pcommon.ValueTypeSlice: - if keys[i].Int == nil { - return nil, fmt.Errorf("slice must be indexed by an int") - } - if int(*keys[i].Int) >= val.Slice().Len() || int(*keys[i].Int) < 0 { - return nil, fmt.Errorf("index %v out of bounds", *keys[i].Int) - } - val = val.Slice().At(int(*keys[i].Int)) - default: - return nil, fmt.Errorf("type %v does not support string indexing", val.Type()) - } - } - return ottlcommon.GetValue(val), nil + return getIndexableValue(val, keys[1:]) } func SetMapValue(m pcommon.Map, keys []ottl.Key, val interface{}) error { @@ -57,57 +34,10 @@ func SetMapValue(m pcommon.Map, keys []ottl.Key, val interface{}) error { return fmt.Errorf("non-string indexing is not supported") } - var newValue pcommon.Value - switch val.(type) { - case []string, []bool, []int64, []float64, [][]byte, []any: - newValue = pcommon.NewValueSlice() - default: - newValue = pcommon.NewValueEmpty() - } - err := SetValue(newValue, val) - if err != nil { - return err - } - currentValue, ok := m.Get(*keys[0].String) if !ok { currentValue = m.PutEmpty(*keys[0].String) } - for i := 1; i < len(keys); i++ { - switch currentValue.Type() { - case pcommon.ValueTypeMap: - if keys[i].String == nil { - return fmt.Errorf("map must be indexed by a string") - } - potentialValue, ok := currentValue.Map().Get(*keys[i].String) - if !ok { - currentValue = currentValue.Map().PutEmpty(*keys[i].String) - } else { - currentValue = potentialValue - } - case pcommon.ValueTypeSlice: - if keys[i].Int == nil { - return fmt.Errorf("slice must be indexed by an int") - } - if int(*keys[i].Int) >= currentValue.Slice().Len() || int(*keys[i].Int) < 0 { - return fmt.Errorf("index %v out of bounds", *keys[i].Int) - } - currentValue = currentValue.Slice().At(int(*keys[i].Int)) - case pcommon.ValueTypeEmpty: - if keys[i].String != nil { - currentValue = currentValue.SetEmptyMap().PutEmpty(*keys[i].String) - } else if keys[i].Int != nil { - currentValue.SetEmptySlice() - for k := 0; k < int(*keys[i].Int); k++ { - currentValue.Slice().AppendEmpty() - } - currentValue = currentValue.Slice().AppendEmpty() - } - default: - return fmt.Errorf("type %v does not support string indexing", currentValue.Type()) - } - } - newValue.CopyTo(currentValue) - return nil + return setIndexableValue(currentValue, val, keys[1:]) } diff --git a/pkg/ottl/contexts/internal/slice.go b/pkg/ottl/contexts/internal/slice.go new file mode 100644 index 000000000000..95bfc3767298 --- /dev/null +++ b/pkg/ottl/contexts/internal/slice.go @@ -0,0 +1,44 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/contexts/internal" + +import ( + "fmt" + + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +func GetSliceValue(s pcommon.Slice, keys []ottl.Key) (interface{}, error) { + if len(keys) == 0 { + return nil, fmt.Errorf("cannot get slice value without key") + } + if keys[0].Int == nil { + return nil, fmt.Errorf("non-integer indexing is not supported") + } + idx := int(*keys[0].Int) + + if idx < 0 || idx >= s.Len() { + return nil, fmt.Errorf("index %d out of bounds", idx) + } + + return getIndexableValue(s.At(int(*keys[0].Int)), keys[1:]) +} + +func SetSliceValue(s pcommon.Slice, keys []ottl.Key, val interface{}) error { + if len(keys) == 0 { + return fmt.Errorf("cannot set slice value without key") + } + if keys[0].Int == nil { + return fmt.Errorf("non-integer indexing is not supported") + } + idx := int(*keys[0].Int) + + if idx < 0 || idx >= s.Len() { + return fmt.Errorf("index %d out of bounds", idx) + } + + return setIndexableValue(s.At(int(*keys[0].Int)), val, keys[1:]) +} diff --git a/pkg/ottl/contexts/internal/slice_test.go b/pkg/ottl/contexts/internal/slice_test.go new file mode 100644 index 000000000000..b0d6d23b9997 --- /dev/null +++ b/pkg/ottl/contexts/internal/slice_test.go @@ -0,0 +1,141 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package internal + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottltest" +) + +func Test_GetSliceValue_Invalid(t *testing.T) { + tests := []struct { + name string + keys []ottl.Key + err error + }{ + { + name: "no keys", + keys: []ottl.Key{}, + err: fmt.Errorf("cannot get slice value without key"), + }, + { + name: "first key not an integer", + keys: []ottl.Key{ + { + String: ottltest.Strp("key"), + }, + }, + err: fmt.Errorf("non-integer indexing is not supported"), + }, + { + name: "index too large", + keys: []ottl.Key{ + { + Int: ottltest.Intp(1), + }, + }, + err: fmt.Errorf("index 1 out of bounds"), + }, + { + name: "index too small", + keys: []ottl.Key{ + { + Int: ottltest.Intp(-1), + }, + }, + err: fmt.Errorf("index -1 out of bounds"), + }, + { + name: "invalid type", + keys: []ottl.Key{ + { + Int: ottltest.Intp(0), + }, + { + String: ottltest.Strp("string"), + }, + }, + err: fmt.Errorf("type Str does not support string indexing"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := pcommon.NewSlice() + s.AppendEmpty().SetStr("val") + + _, err := GetSliceValue(s, tt.keys) + assert.Equal(t, tt.err, err) + }) + } +} + +func Test_SetSliceValue_Invalid(t *testing.T) { + tests := []struct { + name string + keys []ottl.Key + err error + }{ + { + name: "no keys", + keys: []ottl.Key{}, + err: fmt.Errorf("cannot set slice value without key"), + }, + { + name: "first key not an integer", + keys: []ottl.Key{ + { + String: ottltest.Strp("key"), + }, + }, + err: fmt.Errorf("non-integer indexing is not supported"), + }, + { + name: "index too large", + keys: []ottl.Key{ + { + Int: ottltest.Intp(1), + }, + }, + err: fmt.Errorf("index 1 out of bounds"), + }, + { + name: "index too small", + keys: []ottl.Key{ + { + Int: ottltest.Intp(-1), + }, + }, + err: fmt.Errorf("index -1 out of bounds"), + }, + { + name: "invalid type", + keys: []ottl.Key{ + { + Int: ottltest.Intp(0), + }, + { + String: ottltest.Strp("string"), + }, + }, + err: fmt.Errorf("type Str does not support string indexing"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := pcommon.NewSlice() + s.AppendEmpty().SetStr("val") + + err := SetSliceValue(s, tt.keys, "value") + assert.Equal(t, tt.err, err) + }) + } +} diff --git a/pkg/ottl/contexts/internal/value.go b/pkg/ottl/contexts/internal/value.go index 0803ae3617e6..addfdf5c536c 100644 --- a/pkg/ottl/contexts/internal/value.go +++ b/pkg/ottl/contexts/internal/value.go @@ -4,9 +4,13 @@ package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/contexts/internal" import ( + "errors" + "fmt" + "go.opentelemetry.io/collector/pdata/pcommon" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/internal/ottlcommon" ) func SetValue(value pcommon.Value, val interface{}) error { @@ -53,6 +57,8 @@ func SetValue(value pcommon.Value, val interface{}) error { pval := value.Slice().AppendEmpty() err = SetValue(pval, a) } + case pcommon.Slice: + v.CopyTo(value.SetEmptySlice()) case pcommon.Map: v.CopyTo(value.SetEmptyMap()) case map[string]interface{}: @@ -63,3 +69,84 @@ func SetValue(value pcommon.Value, val interface{}) error { } return err } + +func getIndexableValue(value pcommon.Value, keys []ottl.Key) (any, error) { + val, ok := value, false + for i := 0; i < len(keys); i++ { + switch val.Type() { + case pcommon.ValueTypeMap: + if keys[i].String == nil { + return nil, fmt.Errorf("map must be indexed by a string") + } + val, ok = val.Map().Get(*keys[i].String) + if !ok { + return nil, nil + } + case pcommon.ValueTypeSlice: + if keys[i].Int == nil { + return nil, fmt.Errorf("slice must be indexed by an int") + } + if int(*keys[i].Int) >= val.Slice().Len() || int(*keys[i].Int) < 0 { + return nil, fmt.Errorf("index %v out of bounds", *keys[i].Int) + } + val = val.Slice().At(int(*keys[i].Int)) + default: + return nil, fmt.Errorf("type %v does not support string indexing", val.Type()) + } + } + return ottlcommon.GetValue(val), nil +} + +func setIndexableValue(currentValue pcommon.Value, val any, keys []ottl.Key) error { + var newValue pcommon.Value + switch val.(type) { + case []string, []bool, []int64, []float64, [][]byte, []any: + newValue = pcommon.NewValueSlice() + default: + newValue = pcommon.NewValueEmpty() + } + err := SetValue(newValue, val) + if err != nil { + return err + } + + for i := 0; i < len(keys); i++ { + switch currentValue.Type() { + case pcommon.ValueTypeMap: + if keys[i].String == nil { + return errors.New("map must be indexed by a string") + } + potentialValue, ok := currentValue.Map().Get(*keys[i].String) + if !ok { + currentValue = currentValue.Map().PutEmpty(*keys[i].String) + } else { + currentValue = potentialValue + } + case pcommon.ValueTypeSlice: + if keys[i].Int == nil { + return errors.New("slice must be indexed by an int") + } + if int(*keys[i].Int) >= currentValue.Slice().Len() || int(*keys[i].Int) < 0 { + return fmt.Errorf("index %v out of bounds", *keys[i].Int) + } + currentValue = currentValue.Slice().At(int(*keys[i].Int)) + case pcommon.ValueTypeEmpty: + switch { + case keys[i].String != nil: + currentValue = currentValue.SetEmptyMap().PutEmpty(*keys[i].String) + case keys[i].Int != nil: + currentValue.SetEmptySlice() + for k := 0; k < int(*keys[i].Int); k++ { + currentValue.Slice().AppendEmpty() + } + currentValue = currentValue.Slice().AppendEmpty() + default: + return errors.New("neither a string nor an int index was given, this is an error in the OTTL") + } + default: + return fmt.Errorf("type %v does not support string indexing", currentValue.Type()) + } + } + newValue.CopyTo(currentValue) + return nil +} diff --git a/pkg/ottl/contexts/internal/value_test.go b/pkg/ottl/contexts/internal/value_test.go new file mode 100644 index 000000000000..ca3aa95442c7 --- /dev/null +++ b/pkg/ottl/contexts/internal/value_test.go @@ -0,0 +1,21 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/contexts/internal" + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +func Test_SetIndexableValue_EmptyValueNoIndex(t *testing.T) { + keys := []ottl.Key{ + {}, + } + err := setIndexableValue(pcommon.NewValueEmpty(), nil, keys) + assert.Error(t, err) +} diff --git a/pkg/ottl/contexts/ottllog/README.md b/pkg/ottl/contexts/ottllog/README.md index b432f99a63f0..bd2d2aa1931a 100644 --- a/pkg/ottl/contexts/ottllog/README.md +++ b/pkg/ottl/contexts/ottllog/README.md @@ -34,6 +34,8 @@ The following paths are supported. | severity_number | the severity numbner of the log being processed | int64 | | severity_text | the severity text of the log being processed | string | | body | the body of the log being processed | any | +| body\[""\] | a value in a map body of the log being processed. Supports multiple indexes to access nested fields. | string, bool, int64, float64, pcommon.Map, pcommon.Slice, []byte or nil | +| body\[\] | a value in a slice body of the log being processed. Supports multiple indexes to access nested fields. | string, bool, int64, float64, pcommon.Map, pcommon.Slice, []byte or nil | | dropped_attributes_count | the number of dropped attributes of the log being processed | int64 | | flags | the flags of the log being processed | int64 | diff --git a/pkg/ottl/contexts/ottllog/log.go b/pkg/ottl/contexts/ottllog/log.go index 381f794b4749..ecdc78ecc421 100644 --- a/pkg/ottl/contexts/ottllog/log.go +++ b/pkg/ottl/contexts/ottllog/log.go @@ -153,7 +153,11 @@ func newPathGetSetter(path []ottl.Field) (ottl.GetSetter[TransformContext], erro case "severity_text": return accessSeverityText(), nil case "body": - return accessBody(), nil + keys := path[0].Keys + if keys == nil { + return accessBody(), nil + } + return accessBodyKey(keys), nil case "attributes": mapKey := path[0].Keys if mapKey == nil { @@ -275,6 +279,33 @@ func accessBody() ottl.StandardGetSetter[TransformContext] { } } +func accessBodyKey(keys []ottl.Key) ottl.StandardGetSetter[TransformContext] { + return ottl.StandardGetSetter[TransformContext]{ + Getter: func(ctx context.Context, tCtx TransformContext) (interface{}, error) { + body := tCtx.GetLogRecord().Body() + switch body.Type() { + case pcommon.ValueTypeMap: + return internal.GetMapValue(tCtx.GetLogRecord().Body().Map(), keys) + case pcommon.ValueTypeSlice: + return internal.GetSliceValue(tCtx.GetLogRecord().Body().Slice(), keys) + default: + return nil, fmt.Errorf("log bodies of type %s cannot be indexed", body.Type().String()) + } + }, + Setter: func(ctx context.Context, tCtx TransformContext, val interface{}) error { + body := tCtx.GetLogRecord().Body() + switch body.Type() { + case pcommon.ValueTypeMap: + return internal.SetMapValue(tCtx.GetLogRecord().Body().Map(), keys, val) + case pcommon.ValueTypeSlice: + return internal.SetSliceValue(tCtx.GetLogRecord().Body().Slice(), keys, val) + default: + return fmt.Errorf("log bodies of type %s cannot be indexed", body.Type().String()) + } + }, + } +} + func accessAttributes() ottl.StandardGetSetter[TransformContext] { return ottl.StandardGetSetter[TransformContext]{ Getter: func(ctx context.Context, tCtx TransformContext) (interface{}, error) { diff --git a/pkg/ottl/contexts/ottllog/log_test.go b/pkg/ottl/contexts/ottllog/log_test.go index a9b625ba4d9d..caab7154226a 100644 --- a/pkg/ottl/contexts/ottllog/log_test.go +++ b/pkg/ottl/contexts/ottllog/log_test.go @@ -6,6 +6,7 @@ package ottllog import ( "context" "encoding/hex" + "fmt" "testing" "time" @@ -25,7 +26,7 @@ var ( ) func Test_newPathGetSetter(t *testing.T) { - refLog, refIS, refResource := createTelemetry() + refLog, refIS, refResource := createTelemetry("string") newAttrs := pcommon.NewMap() newAttrs.PutStr("hello", "world") @@ -37,6 +38,12 @@ func Test_newPathGetSetter(t *testing.T) { pMap2 := newPMap.PutEmptyMap("k2") pMap2.PutStr("k1", "string") + newBodyMap := pcommon.NewMap() + newBodyMap.PutStr("new", "value") + + newBodySlice := pcommon.NewSlice() + newBodySlice.AppendEmpty().SetStr("data") + newMap := make(map[string]interface{}) newMap2 := make(map[string]interface{}) newMap2["k1"] = "string" @@ -48,6 +55,7 @@ func Test_newPathGetSetter(t *testing.T) { orig interface{} newVal interface{} modified func(log plog.LogRecord, il pcommon.InstrumentationScope, resource pcommon.Resource, cache pcommon.Map) + bodyType string }{ { name: "time_unix_nano", @@ -114,6 +122,81 @@ func Test_newPathGetSetter(t *testing.T) { log.Body().SetStr("head") }, }, + { + name: "map body", + path: []ottl.Field{ + { + Name: "body", + }, + }, + orig: func() pcommon.Map { + log, _, _ := createTelemetry("map") + return log.Body().Map() + }(), + newVal: newBodyMap, + modified: func(log plog.LogRecord, il pcommon.InstrumentationScope, resource pcommon.Resource, cache pcommon.Map) { + newBodyMap.CopyTo(log.Body().Map()) + }, + bodyType: "map", + }, + { + name: "map body index", + path: []ottl.Field{ + { + Name: "body", + Keys: []ottl.Key{ + { + String: ottltest.Strp("key"), + }, + }, + }, + }, + orig: "val", + newVal: "val2", + modified: func(log plog.LogRecord, il pcommon.InstrumentationScope, resource pcommon.Resource, cache pcommon.Map) { + log.Body().Map().PutStr("key", "val2") + }, + bodyType: "map", + }, + { + name: "slice body", + path: []ottl.Field{ + { + Name: "body", + }, + }, + orig: func() pcommon.Slice { + log, _, _ := createTelemetry("slice") + return log.Body().Slice() + }(), + newVal: newBodySlice, + modified: func(log plog.LogRecord, il pcommon.InstrumentationScope, resource pcommon.Resource, cache pcommon.Map) { + fmt.Println(log.Body().Slice().At(0).AsString()) + newBodySlice.CopyTo(log.Body().Slice()) + fmt.Println(log.Body().Slice().At(0).AsString()) + + }, + bodyType: "slice", + }, + { + name: "slice body index", + path: []ottl.Field{ + { + Name: "body", + Keys: []ottl.Key{ + { + Int: ottltest.Intp(0), + }, + }, + }, + }, + orig: "body", + newVal: "head", + modified: func(log plog.LogRecord, il pcommon.InstrumentationScope, resource pcommon.Resource, cache pcommon.Map) { + log.Body().Slice().At(0).SetStr("head") + }, + bodyType: "slice", + }, { name: "flags", path: []ottl.Field{ @@ -572,7 +655,7 @@ func Test_newPathGetSetter(t *testing.T) { accessor, err := newPathGetSetter(tt.path) assert.NoError(t, err) - log, il, resource := createTelemetry() + log, il, resource := createTelemetry(tt.bodyType) tCtx := NewTransformContext(log, il, resource) got, err := accessor.Get(context.Background(), tCtx) @@ -583,11 +666,11 @@ func Test_newPathGetSetter(t *testing.T) { err = accessor.Set(context.Background(), tCtx, tt.newVal) assert.Nil(t, err) - exSpan, exIl, exRes := createTelemetry() + exLog, exIl, exRes := createTelemetry(tt.bodyType) exCache := pcommon.NewMap() - tt.modified(exSpan, exIl, exRes, exCache) + tt.modified(exLog, exIl, exRes, exCache) - assert.Equal(t, exSpan, log) + assert.Equal(t, exLog, log) assert.Equal(t, exIl, il) assert.Equal(t, exRes, resource) assert.Equal(t, exCache, tCtx.getCache()) @@ -595,13 +678,12 @@ func Test_newPathGetSetter(t *testing.T) { } } -func createTelemetry() (plog.LogRecord, pcommon.InstrumentationScope, pcommon.Resource) { +func createTelemetry(bodyType string) (plog.LogRecord, pcommon.InstrumentationScope, pcommon.Resource) { log := plog.NewLogRecord() log.SetTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(100))) log.SetObservedTimestamp(pcommon.NewTimestampFromTime(time.UnixMilli(500))) log.SetSeverityNumber(plog.SeverityNumberFatal) log.SetSeverityText("blue screen of death") - log.Body().SetStr("body") log.Attributes().PutStr("str", "val") log.Attributes().PutBool("bool", true) log.Attributes().PutInt("int", 10) @@ -637,6 +719,17 @@ func createTelemetry() (plog.LogRecord, pcommon.InstrumentationScope, pcommon.Re s := log.Attributes().PutEmptySlice("slice") s.AppendEmpty().SetEmptyMap().PutStr("map", "pass") + switch bodyType { + case "map": + log.Body().SetEmptyMap().PutStr("key", "val") + case "slice": + log.Body().SetEmptySlice().AppendEmpty().SetStr("body") + case "string": + fallthrough + default: + log.Body().SetStr("body") + } + log.SetDroppedAttributesCount(10) log.SetFlags(plog.LogRecordFlags(4)) @@ -654,6 +747,32 @@ func createTelemetry() (plog.LogRecord, pcommon.InstrumentationScope, pcommon.Re return log, il, resource } +func Test_InvalidBodyIndexing(t *testing.T) { + path := []ottl.Field{ + { + Name: "body", + Keys: []ottl.Key{ + { + String: ottltest.Strp("key"), + }, + }, + }, + } + + accessor, err := newPathGetSetter(path) + assert.NoError(t, err) + + log, il, resource := createTelemetry("string") + + tCtx := NewTransformContext(log, il, resource) + _, err = accessor.Get(context.Background(), tCtx) + assert.Error(t, err) + + tCtx = NewTransformContext(log, il, resource) + err = accessor.Set(context.Background(), tCtx, nil) + assert.Error(t, err) +} + func Test_ParseEnum(t *testing.T) { tests := []struct { name string