diff --git a/.chloggen/ottl-dynamic-keep-keys.yaml b/.chloggen/ottl-dynamic-keep-keys.yaml new file mode 100644 index 0000000000000..3bafdc330d1d0 --- /dev/null +++ b/.chloggen/ottl-dynamic-keep-keys.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Support dynamic keys in `keep_keys` and `keep_matching_keys` functions, allowing the keys to be specified at runtime. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [43555] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/pkg/ottl/e2e/e2e_test.go b/pkg/ottl/e2e/e2e_test.go index a119a13998ae8..9a33e7c5df989 100644 --- a/pkg/ottl/e2e/e2e_test.go +++ b/pkg/ottl/e2e/e2e_test.go @@ -64,6 +64,17 @@ func Test_e2e_editors(t *testing.T) { tCtx.GetLogRecord().Attributes().Remove("conflict") }, }, + { + statement: `keep_matching_keys(attributes, Concat(["^", "http"], ""))`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().Remove("flags") + tCtx.GetLogRecord().Attributes().Remove("total.string") + tCtx.GetLogRecord().Attributes().Remove("foo") + tCtx.GetLogRecord().Attributes().Remove("things") + tCtx.GetLogRecord().Attributes().Remove("conflict.conflict1") + tCtx.GetLogRecord().Attributes().Remove("conflict") + }, + }, { statement: `flatten(attributes)`, want: func(tCtx ottllog.TransformContext) { @@ -1159,6 +1170,14 @@ func Test_e2e_converters(t *testing.T) { tCtx.GetLogRecord().Attributes().PutStr("test", `"`) }, }, + { + statement: `keep_keys(attributes["foo"], [Concat(["ba", "r"], "")])`, + want: func(tCtx ottllog.TransformContext) { + // keep_keys should see two arguments + m := tCtx.GetLogRecord().Attributes().PutEmptyMap("foo") + m.PutStr("bar", "pass") + }, + }, { statement: `keep_keys(attributes["foo"], ["\\", "bar"])`, want: func(tCtx ottllog.TransformContext) { diff --git a/pkg/ottl/functions.go b/pkg/ottl/functions.go index ce3b24339e85b..0e73dc19bea41 100644 --- a/pkg/ottl/functions.go +++ b/pkg/ottl/functions.go @@ -26,6 +26,8 @@ type Enum int64 // EnumSymbol is how OTTL represents an enum's string value. type EnumSymbol string +const InvalidRegexErrMsg = "the regex pattern supplied to %s '%q' is not a valid pattern: %w" + func buildOriginalText(path *path) string { var builder strings.Builder if path.Context != "" { diff --git a/pkg/ottl/ottlfuncs/func_keep_keys.go b/pkg/ottl/ottlfuncs/func_keep_keys.go index baf1d921932b1..513059ac6f851 100644 --- a/pkg/ottl/ottlfuncs/func_keep_keys.go +++ b/pkg/ottl/ottlfuncs/func_keep_keys.go @@ -14,7 +14,7 @@ import ( type KeepKeysArguments[K any] struct { Target ottl.PMapGetSetter[K] - Keys []string + Keys []ottl.StringGetter[K] } func NewKeepKeysFactory[K any]() ottl.Factory[K] { @@ -31,17 +31,22 @@ func createKeepKeysFunction[K any](_ ottl.FunctionContext, oArgs ottl.Arguments) return keepKeys(args.Target, args.Keys), nil } -func keepKeys[K any](target ottl.PMapGetSetter[K], keys []string) ottl.ExprFunc[K] { - keySet := make(map[string]struct{}, len(keys)) - for _, key := range keys { - keySet[key] = struct{}{} - } - +func keepKeys[K any](target ottl.PMapGetSetter[K], keys []ottl.StringGetter[K]) ottl.ExprFunc[K] { return func(ctx context.Context, tCtx K) (any, error) { val, err := target.Get(ctx, tCtx) if err != nil { return nil, err } + + keySet := make(map[string]struct{}, len(keys)) + for _, key := range keys { + k, err := key.Get(ctx, tCtx) + if err != nil { + return nil, err + } + keySet[k] = struct{}{} + } + val.RemoveIf(func(key string, _ pcommon.Value) bool { _, ok := keySet[key] return !ok diff --git a/pkg/ottl/ottlfuncs/func_keep_keys_test.go b/pkg/ottl/ottlfuncs/func_keep_keys_test.go index 0926b7385aef2..9c1fa25fc001d 100644 --- a/pkg/ottl/ottlfuncs/func_keep_keys_test.go +++ b/pkg/ottl/ottlfuncs/func_keep_keys_test.go @@ -71,7 +71,17 @@ func Test_keepKeys(t *testing.T) { }, } - exprFunc := keepKeys(target, tt.keys) + keys := make([]ottl.StringGetter[pcommon.Map], len(tt.keys)) + for i, key := range tt.keys { + k := key + keys[i] = ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(_ context.Context, _ pcommon.Map) (any, error) { + return k, nil + }, + } + } + + exprFunc := keepKeys(target, keys) _, err := exprFunc(nil, scenarioMap) assert.NoError(t, err) @@ -96,7 +106,13 @@ func Test_keepKeys_bad_input(t *testing.T) { }, } - keys := []string{"anything"} + keys := []ottl.StringGetter[any]{ + ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "anything", nil + }, + }, + } exprFunc := keepKeys[any](target, keys) @@ -114,7 +130,13 @@ func Test_keepKeys_get_nil(t *testing.T) { }, } - keys := []string{"anything"} + keys := []ottl.StringGetter[any]{ + ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "anything", nil + }, + }, + } exprFunc := keepKeys[any](target, keys) _, err := exprFunc(nil, nil) diff --git a/pkg/ottl/ottlfuncs/func_keep_matching_keys.go b/pkg/ottl/ottlfuncs/func_keep_matching_keys.go index 258caeb791af9..7a4e452f0cda7 100644 --- a/pkg/ottl/ottlfuncs/func_keep_matching_keys.go +++ b/pkg/ottl/ottlfuncs/func_keep_matching_keys.go @@ -15,7 +15,7 @@ import ( type KeepMatchingKeysArguments[K any] struct { Target ottl.PMapGetSetter[K] - Pattern string + Pattern ottl.StringGetter[K] } func NewKeepMatchingKeysFactory[K any]() ottl.Factory[K] { @@ -32,20 +32,37 @@ func createKeepMatchingKeysFunction[K any](_ ottl.FunctionContext, oArgs ottl.Ar return keepMatchingKeys(args.Target, args.Pattern) } -func keepMatchingKeys[K any](target ottl.PMapGetSetter[K], pattern string) (ottl.ExprFunc[K], error) { - compiledPattern, err := regexp.Compile(pattern) - if err != nil { - return nil, fmt.Errorf("the regex pattern provided to keep_matching_keys is not a valid pattern: %w", err) +func keepMatchingKeys[K any](target ottl.PMapGetSetter[K], pattern ottl.StringGetter[K]) (ottl.ExprFunc[K], error) { + literalPattern, ok := ottl.GetLiteralValue(pattern) + var compiledPattern *regexp.Regexp + var err error + if ok { + compiledPattern, err = regexp.Compile(literalPattern) + if err != nil { + return nil, fmt.Errorf(ottl.InvalidRegexErrMsg, "KeepMatchingKeys", literalPattern, err) + } } return func(ctx context.Context, tCtx K) (any, error) { + cp := compiledPattern + if cp == nil { + patternVal, err := pattern.Get(ctx, tCtx) + if err != nil { + return nil, err + } + cp, err = regexp.Compile(patternVal) + if err != nil { + return nil, fmt.Errorf(ottl.InvalidRegexErrMsg, "KeepMatchingKeys", patternVal, err) + } + } + val, err := target.Get(ctx, tCtx) if err != nil { return nil, err } val.RemoveIf(func(key string, _ pcommon.Value) bool { - return !compiledPattern.MatchString(key) + return !cp.MatchString(key) }) return nil, target.Set(ctx, tCtx, val) }, nil diff --git a/pkg/ottl/ottlfuncs/func_keep_matching_keys_test.go b/pkg/ottl/ottlfuncs/func_keep_matching_keys_test.go index a44594d8ae6e7..f352f3e6acbc6 100644 --- a/pkg/ottl/ottlfuncs/func_keep_matching_keys_test.go +++ b/pkg/ottl/ottlfuncs/func_keep_matching_keys_test.go @@ -58,14 +58,6 @@ func Test_keepMatchingKeys(t *testing.T) { return &m }, }, - { - name: "invalid pattern", - pattern: "*", - want: func() *pcommon.Map { - return nil - }, - wantError: true, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -87,7 +79,12 @@ func Test_keepMatchingKeys(t *testing.T) { }, } - exprFunc, err := keepMatchingKeys(target, tt.pattern) + pattern := &ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(_ context.Context, _ pcommon.Map) (any, error) { + return tt.pattern, nil + }, + } + exprFunc, err := keepMatchingKeys(target, pattern) if tt.wantError { assert.Error(t, err) @@ -115,7 +112,37 @@ func Test_keepMatchingKeys_bad_input(t *testing.T) { }, } - exprFunc, err := keepMatchingKeys[any](target, "anything") + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "anything", nil + }, + } + + exprFunc, err := keepMatchingKeys[any](target, pattern) + assert.NoError(t, err) + + _, err = exprFunc(nil, input) + assert.Error(t, err) +} + +func Test_keepMatchingKeys_invalid_pattern(t *testing.T) { + input := pcommon.NewValueInt(1) + target := &ottl.StandardPMapGetSetter[any]{ + Getter: func(_ context.Context, tCtx any) (pcommon.Map, error) { + if v, ok := tCtx.(pcommon.Map); ok { + return v, nil + } + return pcommon.Map{}, errors.New("expected pcommon.Map") + }, + } + + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "*", nil + }, + } + + exprFunc, err := keepMatchingKeys[any](target, pattern) assert.NoError(t, err) _, err = exprFunc(nil, input) @@ -132,7 +159,13 @@ func Test_keepMatchingKeys_get_nil(t *testing.T) { }, } - exprFunc, err := keepMatchingKeys[any](target, "anything") + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "anything", nil + }, + } + + exprFunc, err := keepMatchingKeys[any](target, pattern) assert.NoError(t, err) _, err = exprFunc(nil, nil) assert.Error(t, err)