From 799b3c4b96b274d24867f9a975c1aa8beb2ce839 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Fri, 24 Oct 2025 00:21:31 +0100 Subject: [PATCH 1/4] Add Action to Logs, Metrics, Traces, Profiles Action determines the behavior when Conditions match. - "drop" (default): logs matching any condition are dropped. - "keep": logs matching any condition are kept, all others are dropped. Example config ``` filter: error_mode: propagate logs: action: "drop" log_record: - 'attributes["some"] == "values"' ``` --- processor/filterprocessor/common.go | 16 + processor/filterprocessor/config.go | 41 ++ processor/filterprocessor/config_test.go | 602 +++++++++--------- processor/filterprocessor/factory.go | 4 + processor/filterprocessor/factory_test.go | 4 + processor/filterprocessor/logs.go | 2 +- processor/filterprocessor/logs_test.go | 59 +- processor/filterprocessor/metrics.go | 2 + processor/filterprocessor/metrics_test.go | 221 ++++++- processor/filterprocessor/profiles.go | 2 +- processor/filterprocessor/profiles_test.go | 59 +- .../filterprocessor/testdata/config_ottl.yaml | 27 + processor/filterprocessor/traces.go | 2 + processor/filterprocessor/traces_test.go | 64 ++ 14 files changed, 790 insertions(+), 315 deletions(-) create mode 100644 processor/filterprocessor/common.go diff --git a/processor/filterprocessor/common.go b/processor/filterprocessor/common.go new file mode 100644 index 0000000000000..4a5d300681fad --- /dev/null +++ b/processor/filterprocessor/common.go @@ -0,0 +1,16 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package filterprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" + +import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter/expr" + +// applyActionToExpr applies the configured action to expr.BoolExpr. +// - keepAction: inverts the expression +// - dropAction: returns expression unchanged +func applyActionToExpr[K any](boolExpr expr.BoolExpr[K], action Action) expr.BoolExpr[K] { + if action == keepAction { + return expr.Not(boolExpr) + } + return boolExpr +} diff --git a/processor/filterprocessor/config.go b/processor/filterprocessor/config.go index 0aca51de3218e..af9ad206ab196 100644 --- a/processor/filterprocessor/config.go +++ b/processor/filterprocessor/config.go @@ -65,6 +65,11 @@ type MetricFilters struct { // If both Include and Exclude are specified, Include filtering occurs first. Exclude *filterconfig.MetricMatchProperties `mapstructure:"exclude"` + // Action determines the behavior when conditions match. + // "drop" (default): logs matching any condition are dropped. + // "keep": logs matching any condition are kept, all others are dropped. + Action Action `mapstructure:"action"` + // RegexpConfig specifies options for the regexp match type RegexpConfig *regexp.Config `mapstructure:"regexp"` @@ -81,6 +86,11 @@ type MetricFilters struct { // TraceFilters filters by OTTL conditions type TraceFilters struct { + // Action determines the behavior when conditions match. + // "drop" (default): logs matching any condition are dropped. + // "keep": logs matching any condition are kept, all others are dropped. + Action Action `mapstructure:"action"` + // SpanConditions is a list of OTTL conditions for an ottlspan context. // If any condition resolves to true, the span will be dropped. // Supports `and`, `or`, and `()` @@ -103,6 +113,11 @@ type LogFilters struct { // If both Include and Exclude are specified, Include filtering occurs first. Exclude *LogMatchProperties `mapstructure:"exclude"` + // Action determines the behavior when conditions match. + // "drop" (default): logs matching any condition are dropped. + // "keep": logs matching any condition are kept, all others are dropped. + Action Action `mapstructure:"action"` + // LogConditions is a list of OTTL conditions for an ottllog context. // If any condition resolves to true, the log event will be dropped. // Supports `and`, `or`, and `()` @@ -281,12 +296,38 @@ func (lmp LogSeverityNumberMatchProperties) validate() error { type ProfileFilters struct { _ struct{} // prevent unkeyed literals + // Action determines the behavior when conditions match. + // "drop" (default): logs matching any condition are dropped. + // "keep": logs matching any condition are kept, all others are dropped. + Action Action `mapstructure:"action"` + // ProfileConditions is a list of OTTL conditions for an ottlprofile context. // If any condition resolves to true, the profile will be dropped. // Supports `and`, `or`, and `()` ProfileConditions []string `mapstructure:"profile"` } +// Action specifies the action to take on logs that match the conditions. +type Action string + +const ( + // dropAction drops logs that match the conditions and retains all others. + dropAction = Action("drop") + // keepAction retains logs that match the conditions and drops all others. + keepAction = Action("keep") +) + +func (a *Action) UnmarshalText(text []byte) error { + str := Action(strings.ToLower(string(text))) + switch str { + case dropAction, keepAction: + *a = str + return nil + default: + return fmt.Errorf("unknown action \"%s\". Valid options are: \"%s\", \"%s\"", str, dropAction, keepAction) + } +} + var _ component.Config = (*Config)(nil) // Validate checks if the processor configuration is valid diff --git a/processor/filterprocessor/config_test.go b/processor/filterprocessor/config_test.go index 4c45848406844..dd66830d0b246 100644 --- a/processor/filterprocessor/config_test.go +++ b/processor/filterprocessor/config_test.go @@ -4,7 +4,9 @@ package filterprocessor import ( + "fmt" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -64,42 +66,33 @@ func TestLoadingConfigStrict(t *testing.T) { }{ { id: component.MustNewIDWithName("filter", "empty"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Include: &filterconfig.MetricMatchProperties{ - MatchType: filterconfig.MetricStrict, - }, - }, - }, - }, { + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Include = &filterconfig.MetricMatchProperties{ + MatchType: filterconfig.MetricStrict, + } + }), + }, + { id: component.MustNewIDWithName("filter", "include"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Include: testDataMetricProperties, - }, - }, - }, { + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Include = testDataMetricProperties + }), + }, + { id: component.MustNewIDWithName("filter", "exclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Exclude: testDataMetricProperties, - }, - }, - }, { + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Exclude = testDataMetricProperties + }), + }, + { id: component.MustNewIDWithName("filter", "includeexclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Include: testDataMetricProperties, - Exclude: &filterconfig.MetricMatchProperties{ - MatchType: filterconfig.MetricStrict, - MetricNames: []string{"hello_world"}, - }, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Include = testDataMetricProperties + cfg.Metrics.Exclude = &filterconfig.MetricMatchProperties{ + MatchType: filterconfig.MetricStrict, + MetricNames: []string{"hello_world"}, + } + }), }, } @@ -150,39 +143,25 @@ func TestLoadingConfigStrictLogs(t *testing.T) { }{ { id: component.MustNewIDWithName("filter", "empty"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: &LogMatchProperties{ - LogMatchType: strictType, - }, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = &LogMatchProperties{LogMatchType: strictType} + }), }, { id: component.MustNewIDWithName("filter", "include"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + }), }, { id: component.MustNewIDWithName("filter", "exclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Exclude: testDataLogPropertiesExclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), }, { id: component.MustNewIDWithName("filter", "includeexclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - Exclude: testDataLogPropertiesExclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), }, } @@ -223,29 +202,22 @@ func TestLoadingConfigSeverityLogsStrict(t *testing.T) { }{ { id: component.MustNewIDWithName("filter", "include"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - }, - }, - }, { + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + }), + }, + { id: component.MustNewIDWithName("filter", "exclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Exclude: testDataLogPropertiesExclude, - }, - }, - }, { + expected: createConfig(func(cfg *Config) { + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), + }, + { id: component.MustNewIDWithName("filter", "includeexclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - Exclude: testDataLogPropertiesExclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), }, } @@ -286,29 +258,22 @@ func TestLoadingConfigSeverityLogsRegexp(t *testing.T) { }{ { id: component.MustNewIDWithName("filter", "include"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - }, - }, - }, { + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + }), + }, + { id: component.MustNewIDWithName("filter", "exclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Exclude: testDataLogPropertiesExclude, - }, - }, - }, { + expected: createConfig(func(cfg *Config) { + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), + }, + { id: component.MustNewIDWithName("filter", "includeexclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - Exclude: testDataLogPropertiesExclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), }, } @@ -349,29 +314,22 @@ func TestLoadingConfigBodyLogsStrict(t *testing.T) { }{ { id: component.MustNewIDWithName("filter", "include"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - }, - }, - }, { + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + }), + }, + { id: component.MustNewIDWithName("filter", "exclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Exclude: testDataLogPropertiesExclude, - }, - }, - }, { + expected: createConfig(func(cfg *Config) { + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), + }, + { id: component.MustNewIDWithName("filter", "includeexclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - Exclude: testDataLogPropertiesExclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), }, } @@ -412,29 +370,20 @@ func TestLoadingConfigBodyLogsRegexp(t *testing.T) { }{ { id: component.MustNewIDWithName("filter", "include"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + }), }, { id: component.MustNewIDWithName("filter", "exclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Exclude: testDataLogPropertiesExclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), }, { id: component.MustNewIDWithName("filter", "includeexclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - Exclude: testDataLogPropertiesExclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), }, } @@ -478,29 +427,20 @@ func TestLoadingConfigMinSeverityNumberLogs(t *testing.T) { }{ { id: component.MustNewIDWithName("filter", "include"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + }), }, { id: component.MustNewIDWithName("filter", "exclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Exclude: testDataLogPropertiesExclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), }, { id: component.MustNewIDWithName("filter", "includeexclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Logs: LogFilters{ - Include: testDataLogPropertiesInclude, - Exclude: testDataLogPropertiesExclude, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Logs.Include = testDataLogPropertiesInclude + cfg.Logs.Exclude = testDataLogPropertiesExclude + }), }, } @@ -548,49 +488,37 @@ func TestLoadingConfigRegexp(t *testing.T) { }{ { id: component.MustNewIDWithName("filter", "include"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Include: testDataMetricProperties, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Include = testDataMetricProperties + }), }, { id: component.MustNewIDWithName("filter", "exclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Exclude: testDataMetricProperties, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Exclude = testDataMetricProperties + }), }, { id: component.MustNewIDWithName("filter", "unlimitedcache"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Include: &filterconfig.MetricMatchProperties{ - MatchType: filterconfig.MetricRegexp, - RegexpConfig: &fsregexp.Config{ - CacheEnabled: true, - }, - MetricNames: testDataFilters, + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Include = &filterconfig.MetricMatchProperties{ + MatchType: filterconfig.MetricRegexp, + RegexpConfig: &fsregexp.Config{ + CacheEnabled: true, }, - }, - }, + MetricNames: testDataFilters, + } + }), }, { id: component.MustNewIDWithName("filter", "limitedcache"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Exclude: &filterconfig.MetricMatchProperties{ - MatchType: filterconfig.MetricRegexp, - RegexpConfig: &fsregexp.Config{ - CacheEnabled: true, - CacheMaxNumEntries: 10, - }, - MetricNames: testDataFilters, + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Exclude = &filterconfig.MetricMatchProperties{ + MatchType: filterconfig.MetricRegexp, + RegexpConfig: &fsregexp.Config{ + CacheEnabled: true, + CacheMaxNumEntries: 10, }, - }, - }, + MetricNames: testDataFilters, + } + }), }, } @@ -620,28 +548,25 @@ func TestLoadingSpans(t *testing.T) { }{ { id: component.MustNewIDWithName("filter", "spans"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Spans: filterconfig.MatchConfig{ - Include: &filterconfig.MatchProperties{ - Config: filterset.Config{ - MatchType: filterset.Strict, - }, - Services: []string{"test", "test2"}, - Attributes: []filterconfig.Attribute{ - {Key: "should_include", Value: "(true|probably_true)"}, - }, + expected: createConfig(func(cfg *Config) { + cfg.Spans.Include = &filterconfig.MatchProperties{ + Config: filterset.Config{ + MatchType: filterset.Strict, }, - Exclude: &filterconfig.MatchProperties{ - Config: filterset.Config{ - MatchType: filterset.Regexp, - }, - Attributes: []filterconfig.Attribute{ - {Key: "should_exclude", Value: "(probably_false|false)"}, - }, + Services: []string{"test", "test2"}, + Attributes: []filterconfig.Attribute{ + {Key: "should_include", Value: "(true|probably_true)"}, }, - }, - }, + } + cfg.Spans.Exclude = &filterconfig.MatchProperties{ + Config: filterset.Config{ + MatchType: filterset.Regexp, + }, + Attributes: []filterconfig.Attribute{ + {Key: "should_exclude", Value: "(probably_false|false)"}, + }, + } + }), }, } @@ -671,64 +596,52 @@ func TestLoadingConfigExpr(t *testing.T) { }{ { id: component.MustNewIDWithName("filter", "empty"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Include: &filterconfig.MetricMatchProperties{ - MatchType: filterconfig.MetricExpr, - }, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Include = &filterconfig.MetricMatchProperties{ + MatchType: filterconfig.MetricExpr, + } + }), }, { id: component.MustNewIDWithName("filter", "include"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Include: &filterconfig.MetricMatchProperties{ - MatchType: filterconfig.MetricExpr, - Expressions: []string{ - `Label("foo") == "bar"`, - `HasLabel("baz")`, - }, + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Include = &filterconfig.MetricMatchProperties{ + MatchType: filterconfig.MetricExpr, + Expressions: []string{ + `Label("foo") == "bar"`, + `HasLabel("baz")`, }, - }, - }, + } + }), }, { id: component.MustNewIDWithName("filter", "exclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Exclude: &filterconfig.MetricMatchProperties{ - MatchType: filterconfig.MetricExpr, - Expressions: []string{ - `Label("foo") == "bar"`, - `HasLabel("baz")`, - }, + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Exclude = &filterconfig.MetricMatchProperties{ + MatchType: filterconfig.MetricExpr, + Expressions: []string{ + `Label("foo") == "bar"`, + `HasLabel("baz")`, }, - }, - }, + } + }), }, { id: component.MustNewIDWithName("filter", "includeexclude"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Metrics: MetricFilters{ - Include: &filterconfig.MetricMatchProperties{ - MatchType: filterconfig.MetricExpr, - Expressions: []string{ - `HasLabel("foo")`, - }, + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Include = &filterconfig.MetricMatchProperties{ + MatchType: filterconfig.MetricExpr, + Expressions: []string{ + `HasLabel("foo")`, }, - Exclude: &filterconfig.MetricMatchProperties{ - MatchType: filterconfig.MetricExpr, - Expressions: []string{ - `HasLabel("bar")`, - }, + } + cfg.Metrics.Exclude = &filterconfig.MetricMatchProperties{ + MatchType: filterconfig.MetricExpr, + Expressions: []string{ + `HasLabel("bar")`, }, - }, - }, + } + }), }, } for _, tt := range tests { @@ -865,53 +778,43 @@ func TestLoadingConfigOTTL(t *testing.T) { require.NoError(t, err) tests := []struct { - id component.ID - expected *Config - errorMessage string + id component.ID + expected *Config + errorMessage string + unmarshalErrMsg string }{ { id: component.MustNewIDWithName("filter", "ottl"), - expected: &Config{ - ErrorMode: ottl.IgnoreError, - Traces: TraceFilters{ - SpanConditions: []string{ - `attributes["test"] == "pass"`, - }, - SpanEventConditions: []string{ - `attributes["test"] == "pass"`, - }, - }, - Metrics: MetricFilters{ - MetricConditions: []string{ - `name == "pass"`, - }, - DataPointConditions: []string{ - `attributes["test"] == "pass"`, - }, - }, - Logs: LogFilters{ - LogConditions: []string{ - `attributes["test"] == "pass"`, - }, - }, - Profiles: ProfileFilters{ - ProfileConditions: []string{ - `attributes["test"] == "pass"`, - }, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.ErrorMode = ottl.IgnoreError + cfg.Traces.SpanConditions = []string{ + `attributes["test"] == "pass"`, + } + cfg.Traces.SpanEventConditions = []string{ + `attributes["test"] == "pass"`, + } + cfg.Metrics.MetricConditions = []string{ + `name == "pass"`, + } + cfg.Metrics.DataPointConditions = []string{ + `attributes["test"] == "pass"`, + } + cfg.Logs.LogConditions = []string{ + `attributes["test"] == "pass"`, + } + cfg.Profiles.ProfileConditions = []string{ + `attributes["test"] == "pass"`, + } + }), }, { id: component.MustNewIDWithName("filter", "multiline"), - expected: &Config{ - ErrorMode: ottl.PropagateError, - Traces: TraceFilters{ - SpanConditions: []string{ - `attributes["test"] == "pass"`, - `attributes["test"] == "also pass"`, - }, - }, - }, + expected: createConfig(func(cfg *Config) { + cfg.Traces.SpanConditions = []string{ + `attributes["test"] == "pass"`, + `attributes["test"] == "also pass"`, + } + }), }, { id: component.NewIDWithName(metadata.Type, "spans_mix_config"), @@ -940,6 +843,35 @@ func TestLoadingConfigOTTL(t *testing.T) { { id: component.NewIDWithName(metadata.Type, "bad_syntax_log"), }, + { + id: component.NewIDWithName(metadata.Type, "bad_action"), + unmarshalErrMsg: func() string { + signals := []string{"metrics", "logs", "traces", "profiles"} + lines := make([]string, len(signals)) + for i, signal := range signals { + lines[i] = fmt.Sprintf("'%s.action' unknown action \"invalid\". Valid options are: \"drop\", \"keep\"", signal) + } + return strings.Join(lines, "\n") + }(), + }, + { + id: component.MustNewIDWithName("filter", "keep_action"), + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Action = keepAction + cfg.Logs.Action = keepAction + cfg.Traces.Action = keepAction + cfg.Profiles.Action = keepAction + }), + }, + { + id: component.MustNewIDWithName("filter", "drop_action"), + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Action = dropAction + cfg.Logs.Action = dropAction + cfg.Traces.Action = dropAction + cfg.Profiles.Action = dropAction + }), + }, } for _, tt := range tests { @@ -949,6 +881,11 @@ func TestLoadingConfigOTTL(t *testing.T) { sub, err := cm.Sub(tt.id.String()) require.NoError(t, err) + + if tt.unmarshalErrMsg != "" { + require.ErrorContains(t, sub.Unmarshal(cfg), tt.unmarshalErrMsg) + return + } require.NoError(t, sub.Unmarshal(cfg)) if tt.expected == nil { @@ -965,3 +902,64 @@ func TestLoadingConfigOTTL(t *testing.T) { }) } } + +func TestAction_UnmarshalText(t *testing.T) { + testCases := []struct { + name string + input string + expected Action + expectedErr bool + }{ + { + name: "valid drop action lowercase", + input: "drop", + expected: dropAction, + }, + { + name: "valid keep action lowercase", + input: "keep", + expected: keepAction, + }, + { + name: "invalid action", + input: "invalid", + expectedErr: true, + }, + { + name: "empty action", + input: "", + expectedErr: true, + }, + { + name: "unknown action", + input: "delete", + expectedErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var action Action + err := action.UnmarshalText([]byte(tc.input)) + + if tc.expectedErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), "unknown action") + } else { + assert.NoError(t, err) + assert.Equal(t, tc.expected, action) + } + }) + } +} + +func createConfig(modify func(cfg *Config)) *Config { + factory := NewFactory() + cfg := factory.CreateDefaultConfig().(*Config) + + if modify != nil { + modify(cfg) + } + + return cfg +} diff --git a/processor/filterprocessor/factory.go b/processor/filterprocessor/factory.go index bf5dcfec2b3f0..4e27f798755cb 100644 --- a/processor/filterprocessor/factory.go +++ b/processor/filterprocessor/factory.go @@ -149,6 +149,10 @@ func NewFactoryWithOptions(options ...FactoryOption) processor.Factory { func (f *filterProcessorFactory) createDefaultConfig() component.Config { return &Config{ ErrorMode: ottl.PropagateError, + Metrics: MetricFilters{Action: dropAction}, + Logs: LogFilters{Action: dropAction}, + Traces: TraceFilters{Action: dropAction}, + Profiles: ProfileFilters{Action: dropAction}, dataPointFunctions: f.dataPointFunctions, logFunctions: f.logFunctions, metricFunctions: f.metricFunctions, diff --git a/processor/filterprocessor/factory_test.go b/processor/filterprocessor/factory_test.go index ab4471ce970a8..45ddfbdb1c6d6 100644 --- a/processor/filterprocessor/factory_test.go +++ b/processor/filterprocessor/factory_test.go @@ -39,6 +39,10 @@ func TestCreateDefaultConfig(t *testing.T) { cfg := factory.CreateDefaultConfig() assert.EqualExportedValues(t, &Config{ ErrorMode: ottl.PropagateError, + Metrics: MetricFilters{Action: dropAction}, + Logs: LogFilters{Action: dropAction}, + Traces: TraceFilters{Action: dropAction}, + Profiles: ProfileFilters{Action: dropAction}, }, cfg) assertConfigContainsDefaultFunctions(t, *cfg.(*Config)) assert.NoError(t, componenttest.CheckConfigStruct(cfg)) diff --git a/processor/filterprocessor/logs.go b/processor/filterprocessor/logs.go index b29f71ba1eb45..66e2049ca616b 100644 --- a/processor/filterprocessor/logs.go +++ b/processor/filterprocessor/logs.go @@ -43,7 +43,7 @@ func newFilterLogsProcessor(set processor.Settings, cfg *Config) (*filterLogProc if errBoolExpr != nil { return nil, errBoolExpr } - flp.skipExpr = skipExpr + flp.skipExpr = applyActionToExpr(skipExpr, cfg.Logs.Action) return flp, nil } diff --git a/processor/filterprocessor/logs_test.go b/processor/filterprocessor/logs_test.go index 0d9885db65c0e..9ad6d677e7e43 100644 --- a/processor/filterprocessor/logs_test.go +++ b/processor/filterprocessor/logs_test.go @@ -705,13 +705,15 @@ var ( func TestFilterLogProcessorWithOTTL(t *testing.T) { tests := []struct { name string + action Action conditions []string filterEverything bool want func(ld plog.Logs) errorMode ottl.ErrorMode }{ { - name: "drop logs", + name: "drop logs", + action: dropAction, conditions: []string{ `body == "operationA"`, }, @@ -726,7 +728,24 @@ func TestFilterLogProcessorWithOTTL(t *testing.T) { errorMode: ottl.IgnoreError, }, { - name: "drop everything by dropping all logs", + name: "keep logs", + action: keepAction, + conditions: []string{ + `body == "operationA"`, + }, + want: func(ld plog.Logs) { + ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().RemoveIf(func(log plog.LogRecord) bool { + return log.Body().AsString() != "operationA" + }) + ld.ResourceLogs().At(0).ScopeLogs().At(1).LogRecords().RemoveIf(func(log plog.LogRecord) bool { + return log.Body().AsString() != "operationA" + }) + }, + errorMode: ottl.IgnoreError, + }, + { + name: "drop everything by dropping all logs", + action: dropAction, conditions: []string{ `IsMatch(body, "operation.*")`, }, @@ -734,7 +753,17 @@ func TestFilterLogProcessorWithOTTL(t *testing.T) { errorMode: ottl.IgnoreError, }, { - name: "multiple conditions", + name: "keep everything", + action: keepAction, + conditions: []string{ + `IsMatch(body, "operation.*")`, + }, + want: func(_ plog.Logs) {}, + errorMode: ottl.IgnoreError, + }, + { + name: "multiple conditions", + action: dropAction, conditions: []string{ `IsMatch(body, "wrong name")`, `IsMatch(body, "operation.*")`, @@ -743,17 +772,37 @@ func TestFilterLogProcessorWithOTTL(t *testing.T) { errorMode: ottl.IgnoreError, }, { - name: "with error conditions", + name: "multiple conditions keep everything", + action: keepAction, + conditions: []string{ + `IsMatch(body, "wrong name")`, + `IsMatch(body, "operation.*")`, + }, + want: func(_ plog.Logs) {}, + errorMode: ottl.IgnoreError, + }, + { + name: "with error conditions", + action: dropAction, conditions: []string{ `Substring("", 0, 100) == "test"`, }, want: func(_ plog.Logs) {}, errorMode: ottl.IgnoreError, }, + { + name: "keep action with error conditions should not keep anything", + action: keepAction, + conditions: []string{ + `Substring("", 0, 100) == "test"`, + }, + filterEverything: true, + errorMode: ottl.IgnoreError, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cfg := &Config{Logs: LogFilters{LogConditions: tt.conditions}, logFunctions: defaultLogFunctionsMap()} + cfg := &Config{Logs: LogFilters{Action: tt.action, LogConditions: tt.conditions}, logFunctions: defaultLogFunctionsMap()} processor, err := newFilterLogsProcessor(processortest.NewNopSettings(metadata.Type), cfg) assert.NoError(t, err) diff --git a/processor/filterprocessor/metrics.go b/processor/filterprocessor/metrics.go index e6855c101b375..47606a6e3dc6a 100644 --- a/processor/filterprocessor/metrics.go +++ b/processor/filterprocessor/metrics.go @@ -52,6 +52,7 @@ func newFilterMetricProcessor(set processor.Settings, cfg *Config) (*filterMetri if err != nil { return nil, err } + fsp.skipMetricExpr = applyActionToExpr(fsp.skipMetricExpr, cfg.Metrics.Action) } if cfg.Metrics.DataPointConditions != nil { @@ -59,6 +60,7 @@ func newFilterMetricProcessor(set processor.Settings, cfg *Config) (*filterMetri if err != nil { return nil, err } + fsp.skipDataPointExpr = applyActionToExpr(fsp.skipDataPointExpr, cfg.Metrics.Action) } return fsp, nil diff --git a/processor/filterprocessor/metrics_test.go b/processor/filterprocessor/metrics_test.go index e095966f41919..9dde0befd96ba 100644 --- a/processor/filterprocessor/metrics_test.go +++ b/processor/filterprocessor/metrics_test.go @@ -559,6 +559,7 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { { name: "drop metrics", conditions: MetricFilters{ + Action: dropAction, MetricConditions: []string{ `name == "operationA"`, }, @@ -570,9 +571,25 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep metrics", + conditions: MetricFilters{ + Action: keepAction, + MetricConditions: []string{ + `name == "operationA"`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Name() != "operationA" + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop everything by dropping all metrics", conditions: MetricFilters{ + Action: dropAction, MetricConditions: []string{ `IsMatch(name, "operation.*")`, }, @@ -580,9 +597,21 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { filterEverything: true, errorMode: ottl.IgnoreError, }, + { + name: "keep everything", + conditions: MetricFilters{ + Action: keepAction, + MetricConditions: []string{ + `IsMatch(name, "operation.*")`, + }, + }, + want: func(_ pmetric.Metrics) {}, + errorMode: ottl.IgnoreError, + }, { name: "drop sum data point", conditions: MetricFilters{ + Action: dropAction, DataPointConditions: []string{ `metric.type == METRIC_DATA_TYPE_SUM and value_double == 1.0`, }, @@ -594,9 +623,28 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep metrics other than specific sum data point", + conditions: MetricFilters{ + Action: keepAction, + DataPointConditions: []string{ + `metric.type == METRIC_DATA_TYPE_SUM and value_double == 1.0`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Type() != pmetric.MetricTypeSum + }) + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().RemoveIf(func(point pmetric.NumberDataPoint) bool { + return point.DoubleValue() != 1.0 + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop all sum data points", conditions: MetricFilters{ + Action: dropAction, DataPointConditions: []string{ `metric.type == METRIC_DATA_TYPE_SUM`, }, @@ -608,9 +656,25 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep all sum data points", + conditions: MetricFilters{ + Action: keepAction, + DataPointConditions: []string{ + `metric.type == METRIC_DATA_TYPE_SUM`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Type() != pmetric.MetricTypeSum + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop gauge data point", conditions: MetricFilters{ + Action: dropAction, DataPointConditions: []string{ `metric.type == METRIC_DATA_TYPE_GAUGE and value_double == 1.0`, }, @@ -622,9 +686,28 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep gauge data point", + conditions: MetricFilters{ + Action: keepAction, + DataPointConditions: []string{ + `metric.type == METRIC_DATA_TYPE_GAUGE and value_double == 1.0`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(4).Gauge().DataPoints().RemoveIf(func(point pmetric.NumberDataPoint) bool { + return point.DoubleValue() != 1.0 + }) + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Type() != pmetric.MetricTypeGauge + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop all gauge data points", conditions: MetricFilters{ + Action: dropAction, DataPointConditions: []string{ `metric.type == METRIC_DATA_TYPE_GAUGE`, }, @@ -636,9 +719,25 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep all gauge data points", + conditions: MetricFilters{ + Action: keepAction, + DataPointConditions: []string{ + `metric.type == METRIC_DATA_TYPE_GAUGE`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Type() != pmetric.MetricTypeGauge + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop histogram data point", conditions: MetricFilters{ + Action: dropAction, DataPointConditions: []string{ `metric.type == METRIC_DATA_TYPE_HISTOGRAM and count == 1`, }, @@ -650,9 +749,28 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep histogram data point", + conditions: MetricFilters{ + Action: keepAction, + DataPointConditions: []string{ + `metric.type == METRIC_DATA_TYPE_HISTOGRAM and count == 1`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().RemoveIf(func(point pmetric.HistogramDataPoint) bool { + return point.Count() != 1 + }) + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Type() != pmetric.MetricTypeHistogram + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop all histogram data points", conditions: MetricFilters{ + Action: dropAction, DataPointConditions: []string{ `metric.type == METRIC_DATA_TYPE_HISTOGRAM`, }, @@ -664,9 +782,25 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep all histogram data points", + conditions: MetricFilters{ + Action: keepAction, + DataPointConditions: []string{ + `metric.type == METRIC_DATA_TYPE_HISTOGRAM`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Type() != pmetric.MetricTypeHistogram + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop exponential histogram data point", conditions: MetricFilters{ + Action: dropAction, DataPointConditions: []string{ `metric.type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM and count == 1`, }, @@ -678,9 +812,28 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep exponential histogram data point", + conditions: MetricFilters{ + Action: keepAction, + DataPointConditions: []string{ + `metric.type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM and count == 1`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().RemoveIf(func(point pmetric.ExponentialHistogramDataPoint) bool { + return point.Count() != 1 + }) + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Type() != pmetric.MetricTypeExponentialHistogram + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop all exponential histogram data points", conditions: MetricFilters{ + Action: dropAction, DataPointConditions: []string{ `metric.type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM`, }, @@ -692,9 +845,25 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep all exponential histogram data points", + conditions: MetricFilters{ + Action: keepAction, + DataPointConditions: []string{ + `metric.type == METRIC_DATA_TYPE_EXPONENTIAL_HISTOGRAM`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Type() != pmetric.MetricTypeExponentialHistogram + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop summary data point", conditions: MetricFilters{ + Action: dropAction, DataPointConditions: []string{ `metric.type == METRIC_DATA_TYPE_SUMMARY and sum == 43.21`, }, @@ -706,9 +875,28 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep summary data point", + conditions: MetricFilters{ + Action: keepAction, + DataPointConditions: []string{ + `metric.type == METRIC_DATA_TYPE_SUMMARY and sum == 43.21`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(3).Summary().DataPoints().RemoveIf(func(point pmetric.SummaryDataPoint) bool { + return point.Sum() != 43.21 + }) + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Type() != pmetric.MetricTypeSummary + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop all summary data points", conditions: MetricFilters{ + Action: dropAction, DataPointConditions: []string{ `metric.type == METRIC_DATA_TYPE_SUMMARY`, }, @@ -720,9 +908,25 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep all summary data points", + conditions: MetricFilters{ + Action: keepAction, + DataPointConditions: []string{ + `metric.type == METRIC_DATA_TYPE_SUMMARY`, + }, + }, + want: func(md pmetric.Metrics) { + md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().RemoveIf(func(metric pmetric.Metric) bool { + return metric.Type() != pmetric.MetricTypeSummary + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "multiple conditions", conditions: MetricFilters{ + Action: dropAction, MetricConditions: []string{ `resource.attributes["not real"] == "unknown"`, `type != nil`, @@ -731,9 +935,22 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { filterEverything: true, errorMode: ottl.IgnoreError, }, + { + name: "multiple conditions keep everything", + conditions: MetricFilters{ + Action: keepAction, + MetricConditions: []string{ + `resource.attributes["not real"] == "unknown"`, + `type != nil`, + }, + }, + want: func(_ pmetric.Metrics) {}, + errorMode: ottl.IgnoreError, + }, { name: "with error conditions", conditions: MetricFilters{ + Action: dropAction, MetricConditions: []string{ `Substring("", 0, 100) == "test"`, }, @@ -744,6 +961,7 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { { name: "HasAttrOnDatapoint", conditions: MetricFilters{ + Action: dropAction, MetricConditions: []string{ `HasAttrOnDatapoint("attr1", "test1")`, }, @@ -754,6 +972,7 @@ func TestFilterMetricProcessorWithOTTL(t *testing.T) { { name: "HasAttrKeyOnDatapoint", conditions: MetricFilters{ + Action: dropAction, MetricConditions: []string{ `HasAttrKeyOnDatapoint("attr1")`, }, @@ -899,7 +1118,7 @@ func fillMetricFive(m pmetric.Metric) { dataPoint0.Attributes().PutStr("attr2", "test2") dataPoint0.Attributes().PutStr("attr3", "test3") - dataPoint1 := m.SetEmptyGauge().DataPoints().AppendEmpty() + dataPoint1 := m.Gauge().DataPoints().AppendEmpty() dataPoint1.SetStartTimestamp(dataPointStartTimestamp) dataPoint1.SetDoubleValue(2.0) dataPoint1.Attributes().PutStr("attr1", "test1") diff --git a/processor/filterprocessor/profiles.go b/processor/filterprocessor/profiles.go index 80347b9026cf4..a2d3dfecab36d 100644 --- a/processor/filterprocessor/profiles.go +++ b/processor/filterprocessor/profiles.go @@ -41,7 +41,7 @@ func newFilterProfilesProcessor(set processor.Settings, cfg *Config) (*filterPro if errBoolExpr != nil { return nil, errBoolExpr } - fpp.skipExpr = skipExpr + fpp.skipExpr = applyActionToExpr(skipExpr, cfg.Profiles.Action) return fpp, nil } diff --git a/processor/filterprocessor/profiles_test.go b/processor/filterprocessor/profiles_test.go index 58965ca1fb9a2..974e6695cf1a2 100644 --- a/processor/filterprocessor/profiles_test.go +++ b/processor/filterprocessor/profiles_test.go @@ -71,13 +71,15 @@ func requireNotPanicsProfiles(t *testing.T, profiles pprofile.Profiles) { func TestFilterProfileProcessorWithOTTL(t *testing.T) { tests := []struct { name string + action Action conditions []string filterEverything bool want func(pprofile.Profiles) errorMode ottl.ErrorMode }{ { - name: "drop profiles", + name: "drop profiles", + action: dropAction, conditions: []string{ `original_payload_format == "legacy"`, }, @@ -92,7 +94,24 @@ func TestFilterProfileProcessorWithOTTL(t *testing.T) { errorMode: ottl.IgnoreError, }, { - name: "drop everything by dropping all profiles", + name: "keep profiles", + action: keepAction, + conditions: []string{ + `original_payload_format == "legacy"`, + }, + want: func(ld pprofile.Profiles) { + ld.ResourceProfiles().At(0).ScopeProfiles().At(0).Profiles().RemoveIf(func(profile pprofile.Profile) bool { + return profile.OriginalPayloadFormat() != "legacy" + }) + ld.ResourceProfiles().At(0).ScopeProfiles().At(1).Profiles().RemoveIf(func(profile pprofile.Profile) bool { + return profile.OriginalPayloadFormat() != "legacy" + }) + }, + errorMode: ottl.IgnoreError, + }, + { + name: "drop everything by dropping all profiles", + action: dropAction, conditions: []string{ `IsMatch(original_payload_format, ".*legacy")`, }, @@ -100,7 +119,17 @@ func TestFilterProfileProcessorWithOTTL(t *testing.T) { errorMode: ottl.IgnoreError, }, { - name: "multiple conditions", + name: "keep everything", + action: keepAction, + conditions: []string{ + `IsMatch(original_payload_format, ".*legacy")`, + }, + want: func(_ pprofile.Profiles) {}, + errorMode: ottl.IgnoreError, + }, + { + name: "multiple conditions", + action: dropAction, conditions: []string{ `IsMatch(original_payload_format, "wrong name")`, `IsMatch(original_payload_format, ".*legacy")`, @@ -109,17 +138,37 @@ func TestFilterProfileProcessorWithOTTL(t *testing.T) { errorMode: ottl.IgnoreError, }, { - name: "with error conditions", + name: "multiple conditions keep everything", + action: keepAction, + conditions: []string{ + `IsMatch(original_payload_format, "wrong name")`, + `IsMatch(original_payload_format, ".*legacy")`, + }, + want: func(_ pprofile.Profiles) {}, + errorMode: ottl.IgnoreError, + }, + { + name: "with error conditions", + action: dropAction, conditions: []string{ `Substring("", 0, 100) == "test"`, }, want: func(_ pprofile.Profiles) {}, errorMode: ottl.IgnoreError, }, + { + name: "keep action with error conditions should not keep anything", + action: keepAction, + conditions: []string{ + `Substring("", 0, 100) == "test"`, + }, + filterEverything: true, + errorMode: ottl.IgnoreError, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cfg := &Config{Profiles: ProfileFilters{ProfileConditions: tt.conditions}, profileFunctions: defaultProfileFunctionsMap()} + cfg := &Config{Profiles: ProfileFilters{Action: tt.action, ProfileConditions: tt.conditions}, profileFunctions: defaultProfileFunctionsMap()} processor, err := newFilterProfilesProcessor(processortest.NewNopSettings(metadata.Type), cfg) assert.NoError(t, err) diff --git a/processor/filterprocessor/testdata/config_ottl.yaml b/processor/filterprocessor/testdata/config_ottl.yaml index 7166395f50d1c..e6eda4d5663bd 100644 --- a/processor/filterprocessor/testdata/config_ottl.yaml +++ b/processor/filterprocessor/testdata/config_ottl.yaml @@ -72,3 +72,30 @@ filter/bad_syntax_log: logs: log_record: - 'attributes[test] == "pass"' +filter/bad_action: + metrics: + action: "invalid" + logs: + action: "invalid" + traces: + action: "invalid" + profiles: + action: "invalid" +filter/keep_action: + metrics: + action: "keep" + logs: + action: "keep" + traces: + action: "keep" + profiles: + action: "keep" +filter/drop_action: + metrics: + action: "drop" + logs: + action: "drop" + traces: + action: "drop" + profiles: + action: "drop" \ No newline at end of file diff --git a/processor/filterprocessor/traces.go b/processor/filterprocessor/traces.go index 94e754f98b754..2d2f70e42a2b1 100644 --- a/processor/filterprocessor/traces.go +++ b/processor/filterprocessor/traces.go @@ -46,12 +46,14 @@ func newFilterSpansProcessor(set processor.Settings, cfg *Config) (*filterSpanPr if err != nil { return nil, err } + fsp.skipSpanExpr = applyActionToExpr(fsp.skipSpanExpr, cfg.Traces.Action) } if cfg.Traces.SpanEventConditions != nil { fsp.skipSpanEventExpr, err = filterottl.NewBoolExprForSpanEvent(cfg.Traces.SpanEventConditions, cfg.spanEventFunctions, cfg.ErrorMode, set.TelemetrySettings) if err != nil { return nil, err } + fsp.skipSpanEventExpr = applyActionToExpr(fsp.skipSpanEventExpr, cfg.Traces.Action) } return fsp, nil } diff --git a/processor/filterprocessor/traces_test.go b/processor/filterprocessor/traces_test.go index 3456026899908..fc53f7d3215f2 100644 --- a/processor/filterprocessor/traces_test.go +++ b/processor/filterprocessor/traces_test.go @@ -204,6 +204,7 @@ func TestFilterTraceProcessorWithOTTL(t *testing.T) { { name: "drop spans", conditions: TraceFilters{ + Action: dropAction, SpanConditions: []string{ `name == "operationA"`, }, @@ -218,9 +219,28 @@ func TestFilterTraceProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep spans", + conditions: TraceFilters{ + Action: keepAction, + SpanConditions: []string{ + `name == "operationA"`, + }, + }, + want: func(td ptrace.Traces) { + td.ResourceSpans().At(0).ScopeSpans().At(1).Spans().RemoveIf(func(span ptrace.Span) bool { + return span.Name() != "operationA" + }) + td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().RemoveIf(func(span ptrace.Span) bool { + return span.Name() != "operationA" + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "drop everything by dropping all spans", conditions: TraceFilters{ + Action: dropAction, SpanConditions: []string{ `IsMatch(name, "operation.*")`, }, @@ -228,9 +248,21 @@ func TestFilterTraceProcessorWithOTTL(t *testing.T) { filterEverything: true, errorMode: ottl.IgnoreError, }, + { + name: "keep everything", + conditions: TraceFilters{ + Action: keepAction, + SpanConditions: []string{ + `IsMatch(name, "operation.*")`, + }, + }, + want: func(_ ptrace.Traces) {}, + errorMode: ottl.IgnoreError, + }, { name: "drop span events", conditions: TraceFilters{ + Action: dropAction, SpanEventConditions: []string{ `name == "spanEventA"`, }, @@ -245,9 +277,28 @@ func TestFilterTraceProcessorWithOTTL(t *testing.T) { }, errorMode: ottl.IgnoreError, }, + { + name: "keep span events", + conditions: TraceFilters{ + Action: keepAction, + SpanEventConditions: []string{ + `name == "spanEventA"`, + }, + }, + want: func(td ptrace.Traces) { + td.ResourceSpans().At(0).ScopeSpans().At(1).Spans().At(1).Events().RemoveIf(func(event ptrace.SpanEvent) bool { + return event.Name() != "spanEventA" + }) + td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Events().RemoveIf(func(event ptrace.SpanEvent) bool { + return event.Name() != "spanEventA" + }) + }, + errorMode: ottl.IgnoreError, + }, { name: "multiple conditions", conditions: TraceFilters{ + Action: dropAction, SpanConditions: []string{ `name == "operationZ"`, `span_id != nil`, @@ -256,9 +307,22 @@ func TestFilterTraceProcessorWithOTTL(t *testing.T) { filterEverything: true, errorMode: ottl.IgnoreError, }, + { + name: "multiple conditions keep everything", + conditions: TraceFilters{ + Action: keepAction, + SpanConditions: []string{ + `name == "operationZ"`, + `span_id != nil`, + }, + }, + want: func(_ ptrace.Traces) {}, + errorMode: ottl.IgnoreError, + }, { name: "with error conditions", conditions: TraceFilters{ + Action: dropAction, SpanConditions: []string{ `Substring("", 0, 100) == "test"`, }, From 375eebe0452ec00c32d11a0fd6e72927c9ae4d9f Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Fri, 24 Oct 2025 18:35:48 +0100 Subject: [PATCH 2/4] Add more tests & comments --- processor/filterprocessor/config.go | 22 +++---- processor/filterprocessor/config_test.go | 59 ------------------- .../filterprocessor/testdata/config_ottl.yaml | 16 ++--- 3 files changed, 15 insertions(+), 82 deletions(-) diff --git a/processor/filterprocessor/config.go b/processor/filterprocessor/config.go index af9ad206ab196..a9d2d8b263c70 100644 --- a/processor/filterprocessor/config.go +++ b/processor/filterprocessor/config.go @@ -65,9 +65,7 @@ type MetricFilters struct { // If both Include and Exclude are specified, Include filtering occurs first. Exclude *filterconfig.MetricMatchProperties `mapstructure:"exclude"` - // Action determines the behavior when conditions match. - // "drop" (default): logs matching any condition are dropped. - // "keep": logs matching any condition are kept, all others are dropped. + // Action specifies the behavior when conditions match. Action Action `mapstructure:"action"` // RegexpConfig specifies options for the regexp match type @@ -86,9 +84,7 @@ type MetricFilters struct { // TraceFilters filters by OTTL conditions type TraceFilters struct { - // Action determines the behavior when conditions match. - // "drop" (default): logs matching any condition are dropped. - // "keep": logs matching any condition are kept, all others are dropped. + // Action specifies the behavior when conditions match. Action Action `mapstructure:"action"` // SpanConditions is a list of OTTL conditions for an ottlspan context. @@ -113,9 +109,7 @@ type LogFilters struct { // If both Include and Exclude are specified, Include filtering occurs first. Exclude *LogMatchProperties `mapstructure:"exclude"` - // Action determines the behavior when conditions match. - // "drop" (default): logs matching any condition are dropped. - // "keep": logs matching any condition are kept, all others are dropped. + // Action specifies the behavior when conditions match. Action Action `mapstructure:"action"` // LogConditions is a list of OTTL conditions for an ottllog context. @@ -296,9 +290,7 @@ func (lmp LogSeverityNumberMatchProperties) validate() error { type ProfileFilters struct { _ struct{} // prevent unkeyed literals - // Action determines the behavior when conditions match. - // "drop" (default): logs matching any condition are dropped. - // "keep": logs matching any condition are kept, all others are dropped. + // Action specifies the behavior when conditions match. Action Action `mapstructure:"action"` // ProfileConditions is a list of OTTL conditions for an ottlprofile context. @@ -307,13 +299,13 @@ type ProfileFilters struct { ProfileConditions []string `mapstructure:"profile"` } -// Action specifies the action to take on logs that match the conditions. +// Action specifies the behavior when conditions match. type Action string const ( - // dropAction drops logs that match the conditions and retains all others. + // dropAction drops signals that match the conditions and retains all others. dropAction = Action("drop") - // keepAction retains logs that match the conditions and drops all others. + // keepAction retains signals that match the conditions and drops all others. keepAction = Action("keep") ) diff --git a/processor/filterprocessor/config_test.go b/processor/filterprocessor/config_test.go index dd66830d0b246..bb635c26327f9 100644 --- a/processor/filterprocessor/config_test.go +++ b/processor/filterprocessor/config_test.go @@ -863,15 +863,6 @@ func TestLoadingConfigOTTL(t *testing.T) { cfg.Profiles.Action = keepAction }), }, - { - id: component.MustNewIDWithName("filter", "drop_action"), - expected: createConfig(func(cfg *Config) { - cfg.Metrics.Action = dropAction - cfg.Logs.Action = dropAction - cfg.Traces.Action = dropAction - cfg.Profiles.Action = dropAction - }), - }, } for _, tt := range tests { @@ -903,56 +894,6 @@ func TestLoadingConfigOTTL(t *testing.T) { } } -func TestAction_UnmarshalText(t *testing.T) { - testCases := []struct { - name string - input string - expected Action - expectedErr bool - }{ - { - name: "valid drop action lowercase", - input: "drop", - expected: dropAction, - }, - { - name: "valid keep action lowercase", - input: "keep", - expected: keepAction, - }, - { - name: "invalid action", - input: "invalid", - expectedErr: true, - }, - { - name: "empty action", - input: "", - expectedErr: true, - }, - { - name: "unknown action", - input: "delete", - expectedErr: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var action Action - err := action.UnmarshalText([]byte(tc.input)) - - if tc.expectedErr { - assert.Error(t, err) - assert.Contains(t, err.Error(), "unknown action") - } else { - assert.NoError(t, err) - assert.Equal(t, tc.expected, action) - } - }) - } -} - func createConfig(modify func(cfg *Config)) *Config { factory := NewFactory() cfg := factory.CreateDefaultConfig().(*Config) diff --git a/processor/filterprocessor/testdata/config_ottl.yaml b/processor/filterprocessor/testdata/config_ottl.yaml index e6eda4d5663bd..072a2dad93549 100644 --- a/processor/filterprocessor/testdata/config_ottl.yaml +++ b/processor/filterprocessor/testdata/config_ottl.yaml @@ -83,19 +83,19 @@ filter/bad_action: action: "invalid" filter/keep_action: metrics: - action: "keep" + action: keep logs: - action: "keep" + action: keep traces: - action: "keep" + action: keep profiles: - action: "keep" + action: keep filter/drop_action: metrics: - action: "drop" + action: drop logs: - action: "drop" + action: drop traces: - action: "drop" + action: drop profiles: - action: "drop" \ No newline at end of file + action: drop \ No newline at end of file From c7fd36fe332bdd2b7d06c376211e6c66d49d5449 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Fri, 24 Oct 2025 18:38:55 +0100 Subject: [PATCH 3/4] Add changelog & docs --- .chloggen/filterprocessor_action_option.yaml | 27 ++++++++++++++++++++ processor/filterprocessor/README.md | 25 ++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 .chloggen/filterprocessor_action_option.yaml diff --git a/.chloggen/filterprocessor_action_option.yaml b/.chloggen/filterprocessor_action_option.yaml new file mode 100644 index 0000000000000..b0e6eb74bd7a9 --- /dev/null +++ b/.chloggen/filterprocessor_action_option.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: "enhancement" + +# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog) +component: processor/filter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Added an "action" option to Logs, Metrics, Traces and Profiles to specify the behavior when conditions match. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [42321] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/processor/filterprocessor/README.md b/processor/filterprocessor/README.md index d4e75538a7710..700d3dada7d86 100644 --- a/processor/filterprocessor/README.md +++ b/processor/filterprocessor/README.md @@ -56,6 +56,15 @@ The filter processor also allows configuring an optional field, `error_mode`, wh If not specified, `propagate` will be used. +The filter processor allows configuring an optional field, `action`, which determines the behavior when conditions match. + +| action | description | +|--------|----------------------------------------------------------------------------------------------------------| +| drop | Drop signals that match the conditions and retain all others. This is the default behavior. | +| keep | Retain signals that match the conditions and drop all others. This effectively inverts the `drop` logic. | + +If not specified, `drop` will be used. + ### Examples ```yaml @@ -63,6 +72,7 @@ processors: filter/ottl: error_mode: ignore traces: + action: drop span: - 'attributes["container.name"] == "app_container_1"' - 'resource.attributes["host.name"] == "localhost"' @@ -71,6 +81,7 @@ processors: - 'attributes["grpc"] == true' - 'IsMatch(name, ".*grpc.*")' metrics: + action: drop metric: - 'name == "my.metric" and resource.attributes["my_label"] == "abc123"' - 'type == METRIC_DATA_TYPE_HISTOGRAM' @@ -78,10 +89,12 @@ processors: - 'metric.type == METRIC_DATA_TYPE_SUMMARY' - 'resource.attributes["service.name"] == "my_service_name"' logs: + action: drop log_record: - 'IsMatch(body, ".*password.*")' - 'severity_number < SEVERITY_NUMBER_WARN' profiles: + action: drop profile: - 'duration_unix_nano > 3000' ``` @@ -92,6 +105,18 @@ processors: filter: error_mode: ignore traces: + action: drop + span: + - IsMatch(resource.attributes["k8s.pod.name"], "my-pod-name.*") +``` + +#### Keeping data based on a resource attribute +```yaml +processors: + filter: + error_mode: ignore + traces: + action: keep span: - IsMatch(resource.attributes["k8s.pod.name"], "my-pod-name.*") ``` From 65c5db9c34590a68b3ac59e685c4708158bd8a3b Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Fri, 24 Oct 2025 19:35:28 +0100 Subject: [PATCH 4/4] Update tests --- processor/filterprocessor/config.go | 12 ++--- processor/filterprocessor/config_test.go | 61 +++++++++++++++++++++++- 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/processor/filterprocessor/config.go b/processor/filterprocessor/config.go index a9d2d8b263c70..55d186c36def9 100644 --- a/processor/filterprocessor/config.go +++ b/processor/filterprocessor/config.go @@ -65,7 +65,7 @@ type MetricFilters struct { // If both Include and Exclude are specified, Include filtering occurs first. Exclude *filterconfig.MetricMatchProperties `mapstructure:"exclude"` - // Action specifies the behavior when conditions match. + // Action specifies the behavior when conditions match. The default is drop. Action Action `mapstructure:"action"` // RegexpConfig specifies options for the regexp match type @@ -84,7 +84,7 @@ type MetricFilters struct { // TraceFilters filters by OTTL conditions type TraceFilters struct { - // Action specifies the behavior when conditions match. + // Action specifies the behavior when conditions match. The default is drop. Action Action `mapstructure:"action"` // SpanConditions is a list of OTTL conditions for an ottlspan context. @@ -109,7 +109,7 @@ type LogFilters struct { // If both Include and Exclude are specified, Include filtering occurs first. Exclude *LogMatchProperties `mapstructure:"exclude"` - // Action specifies the behavior when conditions match. + // Action specifies the behavior when conditions match. The default is drop. Action Action `mapstructure:"action"` // LogConditions is a list of OTTL conditions for an ottllog context. @@ -290,7 +290,7 @@ func (lmp LogSeverityNumberMatchProperties) validate() error { type ProfileFilters struct { _ struct{} // prevent unkeyed literals - // Action specifies the behavior when conditions match. + // Action specifies the behavior when conditions match. The default is drop. Action Action `mapstructure:"action"` // ProfileConditions is a list of OTTL conditions for an ottlprofile context. @@ -299,7 +299,7 @@ type ProfileFilters struct { ProfileConditions []string `mapstructure:"profile"` } -// Action specifies the behavior when conditions match. +// Action specifies the behavior when conditions match. The default is drop. type Action string const ( @@ -316,7 +316,7 @@ func (a *Action) UnmarshalText(text []byte) error { *a = str return nil default: - return fmt.Errorf("unknown action \"%s\". Valid options are: \"%s\", \"%s\"", str, dropAction, keepAction) + return fmt.Errorf("unknown action \"%s\": must be \"%s\" or \"%s\"", str, dropAction, keepAction) } } diff --git a/processor/filterprocessor/config_test.go b/processor/filterprocessor/config_test.go index bb635c26327f9..88dff55c7f620 100644 --- a/processor/filterprocessor/config_test.go +++ b/processor/filterprocessor/config_test.go @@ -849,7 +849,7 @@ func TestLoadingConfigOTTL(t *testing.T) { signals := []string{"metrics", "logs", "traces", "profiles"} lines := make([]string, len(signals)) for i, signal := range signals { - lines[i] = fmt.Sprintf("'%s.action' unknown action \"invalid\". Valid options are: \"drop\", \"keep\"", signal) + lines[i] = fmt.Sprintf("'%s.action' unknown action \"invalid\": must be \"drop\" or \"keep\"", signal) } return strings.Join(lines, "\n") }(), @@ -863,6 +863,15 @@ func TestLoadingConfigOTTL(t *testing.T) { cfg.Profiles.Action = keepAction }), }, + { + id: component.MustNewIDWithName("filter", "drop_action"), + expected: createConfig(func(cfg *Config) { + cfg.Metrics.Action = dropAction + cfg.Logs.Action = dropAction + cfg.Traces.Action = dropAction + cfg.Profiles.Action = dropAction + }), + }, } for _, tt := range tests { @@ -894,6 +903,56 @@ func TestLoadingConfigOTTL(t *testing.T) { } } +func TestAction_UnmarshalText(t *testing.T) { + testCases := []struct { + name string + input string + expected Action + expectedErr bool + }{ + { + name: "valid drop action lowercase", + input: "drop", + expected: dropAction, + }, + { + name: "valid keep action lowercase", + input: "keep", + expected: keepAction, + }, + { + name: "invalid action", + input: "invalid", + expectedErr: true, + }, + { + name: "empty action", + input: "", + expectedErr: true, + }, + { + name: "unknown action", + input: "delete", + expectedErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var action Action + err := action.UnmarshalText([]byte(tc.input)) + + if tc.expectedErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), "unknown action") + } else { + assert.NoError(t, err) + assert.Equal(t, tc.expected, action) + } + }) + } +} + func createConfig(modify func(cfg *Config)) *Config { factory := NewFactory() cfg := factory.CreateDefaultConfig().(*Config)