[pkg/ottl] Support taking match patterns from runtime data in the replace_all_patterns() and replace_pattern() functions

.chloggen/ottl-dynamic-replace-pattern.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog)
+component: pkg/ottl
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Support taking match patterns from runtime data in the `replace_all_patterns` and `replace_pattern` functions."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [43555]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

pkg/ottl/e2e/e2e_test.go

@@ -259,6 +259,13 @@ func Test_e2e_editors(t *testing.T) {
 				tCtx.GetLogRecord().Attributes().PutStr("http.url", "http:@@localhost@health")
 			},
 		},
+		{
+			statement: `replace_all_patterns(attributes, "value", Concat(["/","health"],""), "@")`,
+			want: func(tCtx ottllog.TransformContext) {
+				tCtx.GetLogRecord().Attributes().PutStr("http.path", "@")
+				tCtx.GetLogRecord().Attributes().PutStr("http.url", "http://localhost@")
+			},
+		},
 		{
 			statement: `replace_match(attributes["http.path"], "*/*", "test")`,
 			want: func(tCtx ottllog.TransformContext) {
@@ -271,6 +278,12 @@ func Test_e2e_editors(t *testing.T) {
 				tCtx.GetLogRecord().Attributes().PutStr("http.path", "@health")
 			},
 		},
+		{
+			statement: `replace_pattern(attributes["http.path"], Concat(["/","health"],""), "@")`,
+			want: func(tCtx ottllog.TransformContext) {
+				tCtx.GetLogRecord().Attributes().PutStr("http.path", "@")
+			},
+		},
 		{
 			statement: `replace_pattern(attributes["http.path"], "/", "@", SHA256)`,
 			want: func(tCtx ottllog.TransformContext) {

pkg/ottl/functions.go

@@ -26,6 +26,8 @@ type Enum int64
 // EnumSymbol is how OTTL represents an enum's string value.
 type EnumSymbol string
 
+const InvalidRegexErrMsg = "the regex pattern supplied to %s '%q' is not a valid pattern: %w"
+
 func buildOriginalText(path *path) string {
 	var builder strings.Builder
 	if path.Context != "" {

pkg/ottl/ottlfuncs/func_replace_all_patterns.go

@@ -22,7 +22,7 @@ const (
 type ReplaceAllPatternsArguments[K any] struct {
 	Target            ottl.PMapGetSetter[K]
 	Mode              string
-	RegexPattern      string
+	RegexPattern      ottl.StringGetter[K]
 	Replacement       ottl.StringGetter[K]
 	Function          ottl.Optional[ottl.FunctionGetter[K]]
 	ReplacementFormat ottl.Optional[ottl.StringGetter[K]]
@@ -42,16 +42,32 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl.
 	return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function, args.ReplacementFormat)
 }
 
-func replaceAllPatterns[K any](target ottl.PMapGetSetter[K], mode, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) {
-	compiledPattern, err := regexp.Compile(regexPattern)
-	if err != nil {
-		return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err)
+func replaceAllPatterns[K any](target ottl.PMapGetSetter[K], mode string, regexPattern, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) {
+	literalPattern, ok := ottl.GetLiteralValue(regexPattern)
+	var compiledPattern *regexp.Regexp
+	var err error
+	if ok {
+		compiledPattern, err = regexp.Compile(literalPattern)
+		if err != nil {
+			return nil, fmt.Errorf(ottl.InvalidRegexErrMsg, "replace_all_patterns", literalPattern, err)
+		}
 	}
 	if mode != modeValue && mode != modeKey {
 		return nil, fmt.Errorf("invalid mode %v, must be either 'key' or 'value'", mode)
 	}
 
 	return func(ctx context.Context, tCtx K) (any, error) {
+		cp := compiledPattern
+		if cp == nil {
+			patternVal, err := regexPattern.Get(ctx, tCtx)
+			if err != nil {
+				return nil, err
+			}
+			cp, err = regexp.Compile(patternVal)
+			if err != nil {
+				return nil, fmt.Errorf(ottl.InvalidRegexErrMsg, "replace_all_patterns", patternVal, err)
+			}
+		}
 		val, err := target.Get(ctx, tCtx)
 		if err != nil {
 			return nil, err
@@ -66,17 +82,17 @@ func replaceAllPatterns[K any](target ottl.PMapGetSetter[K], mode, regexPattern
 		switch mode {
 		case modeValue:
 			for _, value := range val.All() {
-				if value.Type() != pcommon.ValueTypeStr || !compiledPattern.MatchString(value.Str()) {
+				if value.Type() != pcommon.ValueTypeStr || !cp.MatchString(value.Str()) {
 					continue
 				}
 				if !fn.IsEmpty() {
-					updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, value.Str(), replacementVal, replacementFormat)
+					updatedString, err := applyOptReplaceFunction(ctx, tCtx, cp, fn, value.Str(), replacementVal, replacementFormat)
 					if err != nil {
 						continue
 					}
 					value.SetStr(updatedString)
 				} else {
-					value.SetStr(compiledPattern.ReplaceAllString(value.Str(), replacementVal))
+					value.SetStr(cp.ReplaceAllString(value.Str(), replacementVal))
 				}
 			}
 		case modeKey:
@@ -86,18 +102,18 @@ func replaceAllPatterns[K any](target ottl.PMapGetSetter[K], mode, regexPattern
 			updated := pcommon.NewMap()
 			updated.EnsureCapacity(val.Len())
 			for key, value := range val.All() {
-				if !compiledPattern.MatchString(key) {
+				if !cp.MatchString(key) {
 					value.MoveTo(updated.PutEmpty(key))
 					continue
 				}
 				if !fn.IsEmpty() {
-					updatedKey, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, key, replacementVal, replacementFormat)
+					updatedKey, err := applyOptReplaceFunction(ctx, tCtx, cp, fn, key, replacementVal, replacementFormat)
 					if err != nil {
 						continue
 					}
 					value.MoveTo(updated.PutEmpty(updatedKey))
 				} else {
-					value.MoveTo(updated.PutEmpty(compiledPattern.ReplaceAllString(key, replacementVal)))
+					value.MoveTo(updated.PutEmpty(cp.ReplaceAllString(key, replacementVal)))
 				}
 			}
 			updated.MoveTo(val)

pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go

@@ -534,7 +534,13 @@ func Test_replaceAllPatterns(t *testing.T) {
 				},
 			}
 
-			exprFunc, err := replaceAllPatterns[pcommon.Map](target, tt.mode, tt.pattern, tt.replacement, tt.function, tt.replacementFormat)
+			pattern := &ottl.StandardStringGetter[pcommon.Map]{
+				Getter: func(_ context.Context, _ pcommon.Map) (any, error) {
+					return tt.pattern, nil
+				},
+			}
+
+			exprFunc, err := replaceAllPatterns[pcommon.Map](target, tt.mode, pattern, tt.replacement, tt.function, tt.replacementFormat)
 			assert.NoError(t, err)
 
 			_, err = exprFunc(nil, scenarioMap)
@@ -567,7 +573,13 @@ func Test_replaceAllPatterns_bad_input(t *testing.T) {
 	function := ottl.Optional[ottl.FunctionGetter[any]]{}
 	replacementFormat := ottl.Optional[ottl.StringGetter[any]]{}
 
-	exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexpattern", replacement, function, replacementFormat)
+	pattern := &ottl.StandardStringGetter[any]{
+		Getter: func(_ context.Context, _ any) (any, error) {
+			return "regexpattern", nil
+		},
+	}
+
+	exprFunc, err := replaceAllPatterns[any](target, modeValue, pattern, replacement, function, replacementFormat)
 	assert.NoError(t, err)
 
 	_, err = exprFunc(nil, input)
@@ -592,7 +604,13 @@ func Test_replaceAllPatterns_bad_function_input(t *testing.T) {
 	function := ottl.Optional[ottl.FunctionGetter[any]]{}
 	replacementFormat := ottl.Optional[ottl.StringGetter[any]]{}
 
-	exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat)
+	pattern := &ottl.StandardStringGetter[any]{
+		Getter: func(_ context.Context, _ any) (any, error) {
+			return "regexp", nil
+		},
+	}
+
+	exprFunc, err := replaceAllPatterns[any](target, modeValue, pattern, replacement, function, replacementFormat)
 	assert.NoError(t, err)
 
 	result, err := exprFunc(nil, input)
@@ -634,7 +652,13 @@ func Test_replaceAllPatterns_bad_function_result(t *testing.T) {
 	function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue)
 	replacementFormat := ottl.Optional[ottl.StringGetter[any]]{}
 
-	exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat)
+	pattern := &ottl.StandardStringGetter[any]{
+		Getter: func(_ context.Context, _ any) (any, error) {
+			return "regexp", nil
+		},
+	}
+
+	exprFunc, err := replaceAllPatterns[any](target, modeValue, pattern, replacement, function, replacementFormat)
 	assert.NoError(t, err)
 
 	result, err := exprFunc(nil, input)
@@ -666,7 +690,13 @@ func Test_replaceAllPatterns_get_nil(t *testing.T) {
 	function := ottl.Optional[ottl.FunctionGetter[any]]{}
 	replacementFormat := ottl.Optional[ottl.StringGetter[any]]{}
 
-	exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat)
+	pattern := &ottl.StandardStringGetter[any]{
+		Getter: func(_ context.Context, _ any) (any, error) {
+			return "regexp", nil
+		},
+	}
+
+	exprFunc, err := replaceAllPatterns[any](target, modeValue, pattern, replacement, function, replacementFormat)
 	assert.NoError(t, err)
 
 	_, err = exprFunc(nil, nil)
@@ -687,11 +717,15 @@ func Test_replaceAllPatterns_invalid_pattern(t *testing.T) {
 	function := ottl.Optional[ottl.FunctionGetter[any]]{}
 	replacementFormat := ottl.Optional[ottl.StringGetter[any]]{}
 
-	invalidRegexPattern := "*"
-	exprFunc, err := replaceAllPatterns[any](target, modeValue, invalidRegexPattern, replacement, function, replacementFormat)
-	require.Error(t, err)
+	pattern := &ottl.StandardStringGetter[any]{
+		Getter: func(_ context.Context, _ any) (any, error) {
+			return "*", nil
+		},
+	}
+	exprFunc, err := replaceAllPatterns[any](target, modeValue, pattern, replacement, function, replacementFormat)
+	require.NoError(t, err)
+	_, err = exprFunc(nil, nil)
 	assert.ErrorContains(t, err, "error parsing regexp:")
-	assert.Nil(t, exprFunc)
 }
 
 func Test_replaceAllPatterns_invalid_model(t *testing.T) {
@@ -709,7 +743,12 @@ func Test_replaceAllPatterns_invalid_model(t *testing.T) {
 	replacementFormat := ottl.Optional[ottl.StringGetter[any]]{}
 
 	invalidMode := "invalid"
-	exprFunc, err := replaceAllPatterns[any](target, invalidMode, "regex", replacement, function, replacementFormat)
+	pattern := &ottl.StandardStringGetter[any]{
+		Getter: func(_ context.Context, _ any) (any, error) {
+			return "regex", nil
+		},
+	}
+	exprFunc, err := replaceAllPatterns[any](target, invalidMode, pattern, replacement, function, replacementFormat)
 	assert.Nil(t, exprFunc)
 	assert.ErrorContains(t, err, "invalid mode")
 }

pkg/ottl/ottlfuncs/func_replace_pattern.go

@@ -15,7 +15,7 @@ import (
 
 type ReplacePatternArguments[K any] struct {
 	Target            ottl.GetSetter[K]
-	RegexPattern      string
+	RegexPattern      ottl.StringGetter[K]
 	Replacement       ottl.StringGetter[K]
 	Function          ottl.Optional[ottl.FunctionGetter[K]]
 	ReplacementFormat ottl.Optional[ottl.StringGetter[K]]
@@ -98,12 +98,28 @@ func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern
 	return updatedString, nil
 }
 
-func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) {
-	compiledPattern, err := regexp.Compile(regexPattern)
-	if err != nil {
-		return nil, fmt.Errorf("the regex pattern supplied to replace_pattern is not a valid pattern: %w", err)
+func replacePattern[K any](target ottl.GetSetter[K], regexPattern, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) {
+	literalPattern, ok := ottl.GetLiteralValue(regexPattern)
+	var compiledPattern *regexp.Regexp
+	var err error
+	if ok {
+		compiledPattern, err = regexp.Compile(literalPattern)
+		if err != nil {
+			return nil, fmt.Errorf(ottl.InvalidRegexErrMsg, "replace_pattern", literalPattern, err)
+		}
 	}
 	return func(ctx context.Context, tCtx K) (any, error) {
+		cp := compiledPattern
+		if cp == nil {
+			patternVal, err := regexPattern.Get(ctx, tCtx)
+			if err != nil {
+				return nil, err
+			}
+			cp, err = regexp.Compile(patternVal)
+			if err != nil {
+				return nil, fmt.Errorf(ottl.InvalidRegexErrMsg, "replace_pattern", patternVal, err)
+			}
+		}
 		originalVal, err := target.Get(ctx, tCtx)
 		var replacementVal string
 		if err != nil {
@@ -117,10 +133,10 @@ func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replac
 			return nil, err
 		}
 		if originalValStr, ok := originalVal.(string); ok {
-			if compiledPattern.MatchString(originalValStr) {
+			if cp.MatchString(originalValStr) {
 				if !fn.IsEmpty() {
 					var updatedString string
-					updatedString, err = applyOptReplaceFunction[K](ctx, tCtx, compiledPattern, fn, originalValStr, replacementVal, replacementFormat)
+					updatedString, err = applyOptReplaceFunction[K](ctx, tCtx, cp, fn, originalValStr, replacementVal, replacementFormat)
 					if err != nil {
 						return nil, err
 					}
@@ -129,7 +145,7 @@ func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replac
 						return nil, err
 					}
 				} else {
-					updatedStr := compiledPattern.ReplaceAllString(originalValStr, replacementVal)
+					updatedStr := cp.ReplaceAllString(originalValStr, replacementVal)
 					err = target.Set(ctx, tCtx, updatedStr)
 					if err != nil {
 						return nil, err