diff --git a/.chloggen/ottl-dynamic-replace-pattern.yaml b/.chloggen/ottl-dynamic-replace-pattern.yaml new file mode 100644 index 0000000000000..ed193ead3e068 --- /dev/null +++ b/.chloggen/ottl-dynamic-replace-pattern.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Support taking match patterns from runtime data in the `replace_all_patterns` and `replace_pattern` functions." + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [43555] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] \ No newline at end of file diff --git a/pkg/ottl/e2e/e2e_test.go b/pkg/ottl/e2e/e2e_test.go index a119a13998ae8..1dfadd6132445 100644 --- a/pkg/ottl/e2e/e2e_test.go +++ b/pkg/ottl/e2e/e2e_test.go @@ -259,6 +259,13 @@ func Test_e2e_editors(t *testing.T) { tCtx.GetLogRecord().Attributes().PutStr("http.url", "http:@@localhost@health") }, }, + { + statement: `replace_all_patterns(attributes, "value", Concat(["/","health"],""), "@")`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("http.path", "@") + tCtx.GetLogRecord().Attributes().PutStr("http.url", "http://localhost@") + }, + }, { statement: `replace_match(attributes["http.path"], "*/*", "test")`, want: func(tCtx ottllog.TransformContext) { @@ -271,6 +278,12 @@ func Test_e2e_editors(t *testing.T) { tCtx.GetLogRecord().Attributes().PutStr("http.path", "@health") }, }, + { + statement: `replace_pattern(attributes["http.path"], Concat(["/","health"],""), "@")`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("http.path", "@") + }, + }, { statement: `replace_pattern(attributes["http.path"], "/", "@", SHA256)`, want: func(tCtx ottllog.TransformContext) { diff --git a/pkg/ottl/functions.go b/pkg/ottl/functions.go index ce3b24339e85b..0e73dc19bea41 100644 --- a/pkg/ottl/functions.go +++ b/pkg/ottl/functions.go @@ -26,6 +26,8 @@ type Enum int64 // EnumSymbol is how OTTL represents an enum's string value. type EnumSymbol string +const InvalidRegexErrMsg = "the regex pattern supplied to %s '%q' is not a valid pattern: %w" + func buildOriginalText(path *path) string { var builder strings.Builder if path.Context != "" { diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go index 3c498c3e331e9..cc780fdea53f5 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go @@ -22,7 +22,7 @@ const ( type ReplaceAllPatternsArguments[K any] struct { Target ottl.PMapGetSetter[K] Mode string - RegexPattern string + RegexPattern ottl.StringGetter[K] Replacement ottl.StringGetter[K] Function ottl.Optional[ottl.FunctionGetter[K]] ReplacementFormat ottl.Optional[ottl.StringGetter[K]] @@ -42,16 +42,32 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl. return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function, args.ReplacementFormat) } -func replaceAllPatterns[K any](target ottl.PMapGetSetter[K], mode, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { - compiledPattern, err := regexp.Compile(regexPattern) - if err != nil { - return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err) +func replaceAllPatterns[K any](target ottl.PMapGetSetter[K], mode string, regexPattern, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { + literalPattern, ok := ottl.GetLiteralValue(regexPattern) + var compiledPattern *regexp.Regexp + var err error + if ok { + compiledPattern, err = regexp.Compile(literalPattern) + if err != nil { + return nil, fmt.Errorf(ottl.InvalidRegexErrMsg, "replace_all_patterns", literalPattern, err) + } } if mode != modeValue && mode != modeKey { return nil, fmt.Errorf("invalid mode %v, must be either 'key' or 'value'", mode) } return func(ctx context.Context, tCtx K) (any, error) { + cp := compiledPattern + if cp == nil { + patternVal, err := regexPattern.Get(ctx, tCtx) + if err != nil { + return nil, err + } + cp, err = regexp.Compile(patternVal) + if err != nil { + return nil, fmt.Errorf(ottl.InvalidRegexErrMsg, "replace_all_patterns", patternVal, err) + } + } val, err := target.Get(ctx, tCtx) if err != nil { return nil, err @@ -66,17 +82,17 @@ func replaceAllPatterns[K any](target ottl.PMapGetSetter[K], mode, regexPattern switch mode { case modeValue: for _, value := range val.All() { - if value.Type() != pcommon.ValueTypeStr || !compiledPattern.MatchString(value.Str()) { + if value.Type() != pcommon.ValueTypeStr || !cp.MatchString(value.Str()) { continue } if !fn.IsEmpty() { - updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, value.Str(), replacementVal, replacementFormat) + updatedString, err := applyOptReplaceFunction(ctx, tCtx, cp, fn, value.Str(), replacementVal, replacementFormat) if err != nil { continue } value.SetStr(updatedString) } else { - value.SetStr(compiledPattern.ReplaceAllString(value.Str(), replacementVal)) + value.SetStr(cp.ReplaceAllString(value.Str(), replacementVal)) } } case modeKey: @@ -86,18 +102,18 @@ func replaceAllPatterns[K any](target ottl.PMapGetSetter[K], mode, regexPattern updated := pcommon.NewMap() updated.EnsureCapacity(val.Len()) for key, value := range val.All() { - if !compiledPattern.MatchString(key) { + if !cp.MatchString(key) { value.MoveTo(updated.PutEmpty(key)) continue } if !fn.IsEmpty() { - updatedKey, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, key, replacementVal, replacementFormat) + updatedKey, err := applyOptReplaceFunction(ctx, tCtx, cp, fn, key, replacementVal, replacementFormat) if err != nil { continue } value.MoveTo(updated.PutEmpty(updatedKey)) } else { - value.MoveTo(updated.PutEmpty(compiledPattern.ReplaceAllString(key, replacementVal))) + value.MoveTo(updated.PutEmpty(cp.ReplaceAllString(key, replacementVal))) } } updated.MoveTo(val) diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go index a99b5ea701407..4929545eb79d5 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go @@ -534,7 +534,13 @@ func Test_replaceAllPatterns(t *testing.T) { }, } - exprFunc, err := replaceAllPatterns[pcommon.Map](target, tt.mode, tt.pattern, tt.replacement, tt.function, tt.replacementFormat) + pattern := &ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(_ context.Context, _ pcommon.Map) (any, error) { + return tt.pattern, nil + }, + } + + exprFunc, err := replaceAllPatterns[pcommon.Map](target, tt.mode, pattern, tt.replacement, tt.function, tt.replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, scenarioMap) @@ -567,7 +573,13 @@ func Test_replaceAllPatterns_bad_input(t *testing.T) { function := ottl.Optional[ottl.FunctionGetter[any]]{} replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexpattern", replacement, function, replacementFormat) + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "regexpattern", nil + }, + } + + exprFunc, err := replaceAllPatterns[any](target, modeValue, pattern, replacement, function, replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, input) @@ -592,7 +604,13 @@ func Test_replaceAllPatterns_bad_function_input(t *testing.T) { function := ottl.Optional[ottl.FunctionGetter[any]]{} replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "regexp", nil + }, + } + + exprFunc, err := replaceAllPatterns[any](target, modeValue, pattern, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -634,7 +652,13 @@ func Test_replaceAllPatterns_bad_function_result(t *testing.T) { function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "regexp", nil + }, + } + + exprFunc, err := replaceAllPatterns[any](target, modeValue, pattern, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -666,7 +690,13 @@ func Test_replaceAllPatterns_get_nil(t *testing.T) { function := ottl.Optional[ottl.FunctionGetter[any]]{} replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "regexp", nil + }, + } + + exprFunc, err := replaceAllPatterns[any](target, modeValue, pattern, replacement, function, replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, nil) @@ -687,11 +717,15 @@ func Test_replaceAllPatterns_invalid_pattern(t *testing.T) { function := ottl.Optional[ottl.FunctionGetter[any]]{} replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - invalidRegexPattern := "*" - exprFunc, err := replaceAllPatterns[any](target, modeValue, invalidRegexPattern, replacement, function, replacementFormat) - require.Error(t, err) + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "*", nil + }, + } + exprFunc, err := replaceAllPatterns[any](target, modeValue, pattern, replacement, function, replacementFormat) + require.NoError(t, err) + _, err = exprFunc(nil, nil) assert.ErrorContains(t, err, "error parsing regexp:") - assert.Nil(t, exprFunc) } func Test_replaceAllPatterns_invalid_model(t *testing.T) { @@ -709,7 +743,12 @@ func Test_replaceAllPatterns_invalid_model(t *testing.T) { replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} invalidMode := "invalid" - exprFunc, err := replaceAllPatterns[any](target, invalidMode, "regex", replacement, function, replacementFormat) + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "regex", nil + }, + } + exprFunc, err := replaceAllPatterns[any](target, invalidMode, pattern, replacement, function, replacementFormat) assert.Nil(t, exprFunc) assert.ErrorContains(t, err, "invalid mode") } diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 2784543e22804..cf40bb38b9434 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -15,7 +15,7 @@ import ( type ReplacePatternArguments[K any] struct { Target ottl.GetSetter[K] - RegexPattern string + RegexPattern ottl.StringGetter[K] Replacement ottl.StringGetter[K] Function ottl.Optional[ottl.FunctionGetter[K]] ReplacementFormat ottl.Optional[ottl.StringGetter[K]] @@ -98,12 +98,28 @@ func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern return updatedString, nil } -func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { - compiledPattern, err := regexp.Compile(regexPattern) - if err != nil { - return nil, fmt.Errorf("the regex pattern supplied to replace_pattern is not a valid pattern: %w", err) +func replacePattern[K any](target ottl.GetSetter[K], regexPattern, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { + literalPattern, ok := ottl.GetLiteralValue(regexPattern) + var compiledPattern *regexp.Regexp + var err error + if ok { + compiledPattern, err = regexp.Compile(literalPattern) + if err != nil { + return nil, fmt.Errorf(ottl.InvalidRegexErrMsg, "replace_pattern", literalPattern, err) + } } return func(ctx context.Context, tCtx K) (any, error) { + cp := compiledPattern + if cp == nil { + patternVal, err := regexPattern.Get(ctx, tCtx) + if err != nil { + return nil, err + } + cp, err = regexp.Compile(patternVal) + if err != nil { + return nil, fmt.Errorf(ottl.InvalidRegexErrMsg, "replace_pattern", patternVal, err) + } + } originalVal, err := target.Get(ctx, tCtx) var replacementVal string if err != nil { @@ -117,10 +133,10 @@ func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replac return nil, err } if originalValStr, ok := originalVal.(string); ok { - if compiledPattern.MatchString(originalValStr) { + if cp.MatchString(originalValStr) { if !fn.IsEmpty() { var updatedString string - updatedString, err = applyOptReplaceFunction[K](ctx, tCtx, compiledPattern, fn, originalValStr, replacementVal, replacementFormat) + updatedString, err = applyOptReplaceFunction[K](ctx, tCtx, cp, fn, originalValStr, replacementVal, replacementFormat) if err != nil { return nil, err } @@ -129,7 +145,7 @@ func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replac return nil, err } } else { - updatedStr := compiledPattern.ReplaceAllString(originalValStr, replacementVal) + updatedStr := cp.ReplaceAllString(originalValStr, replacementVal) err = target.Set(ctx, tCtx, updatedStr) if err != nil { return nil, err diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go index 9e50012f4a0ed..0d872c2e24d84 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go @@ -234,7 +234,12 @@ func Test_replacePattern(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.function, tt.replacementFormat) + pattern := &ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(_ context.Context, _ pcommon.Value) (any, error) { + return tt.pattern, nil + }, + } + exprFunc, err := replacePattern(tt.target, pattern, tt.replacement, tt.function, tt.replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) @@ -268,7 +273,13 @@ func Test_replacePattern_bad_input(t *testing.T) { function := ottl.Optional[ottl.FunctionGetter[any]]{} replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "regexp", nil + }, + } + + exprFunc, err := replacePattern[any](target, pattern, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -296,7 +307,13 @@ func Test_replacePattern_bad_function_input(t *testing.T) { function := ottl.Optional[ottl.FunctionGetter[any]]{} replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "regexp", nil + }, + } + + exprFunc, err := replacePattern[any](target, pattern, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -330,7 +347,13 @@ func Test_replacePattern_bad_function_result(t *testing.T) { function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "regexp", nil + }, + } + + exprFunc, err := replacePattern[any](target, pattern, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -357,7 +380,13 @@ func Test_replacePattern_get_nil(t *testing.T) { function := ottl.Optional[ottl.FunctionGetter[any]]{} replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replacePattern[any](target, `nomatch\=[^\s]*(\s?)`, replacement, function, replacementFormat) + pattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return `nomatch\=[^\s]*(\s?)`, nil + }, + } + + exprFunc, err := replacePattern[any](target, pattern, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, nil) @@ -384,9 +413,14 @@ func Test_replacePatterns_invalid_pattern(t *testing.T) { function := ottl.Optional[ottl.FunctionGetter[any]]{} replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - invalidRegexPattern := "*" - _, err := replacePattern[any](target, invalidRegexPattern, replacement, function, replacementFormat) - require.Error(t, err) + invalidRegexPattern := &ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "*", nil + }, + } + exprFunc, err := replacePattern[any](target, invalidRegexPattern, replacement, function, replacementFormat) + require.NoError(t, err) + _, err = exprFunc(nil, nil) assert.ErrorContains(t, err, "error parsing regexp:") } @@ -420,7 +454,13 @@ func Test_replacePattern_bad_format_string(t *testing.T) { replacementFormat := ottl.NewTestingOptional[ottl.StringGetter[pcommon.Value]](passwdPrefix) // This is not a valid format string function := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) - exprFunc, err := replacePattern[pcommon.Value](target, `passwd\=[^\s]*`, replacement, function, replacementFormat) + pattern := &ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(_ context.Context, _ pcommon.Value) (any, error) { + return `passwd\=[^\s]*`, nil + }, + } + + exprFunc, err := replacePattern[pcommon.Value](target, pattern, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) require.Error(t, err)