diff --git a/pkg/custom_detectors/CUSTOM_DETECTORS.md b/pkg/custom_detectors/CUSTOM_DETECTORS.md index 720195bc4b2c..5bca7ee0449c 100644 --- a/pkg/custom_detectors/CUSTOM_DETECTORS.md +++ b/pkg/custom_detectors/CUSTOM_DETECTORS.md @@ -38,6 +38,7 @@ This guide will walk you through setting up a custom detector in TruffleHog to i - **`verify`**: An optional section to validate detected secrets. If you want to verify or unverify detected secrets, this section needs to be configured. If not configured, all detected secrets will be marked as unverified. Read [verification server examples](#verification-server-examples) **Other allowed parameters:** + - **`primary_regex_name`**: This parameter allows you designate the primary regex pattern when multiple regex patterns are defined in the regex section. If a match is found, the match for the designated primary regex will be used to determine the line number. The value must be one of the names specified in the regex section. - **`exclude_regexes_capture`**: This parameter allows you to define regex patterns to exclude specific parts of a detected secret. If a match is found within the detected secret, the portion matching this regex is excluded from the result. - **`exclude_regexes_match`**: This parameter enables you to define regex patterns to exclude entire matches from being reported as secrets. - **`entropy`**: This parameter is used to assess the randomness of detected strings. High entropy often indicates that a string is a potential secret, such as an API key or password, due to its complexity and unpredictability. It helps in filtering false-positives. While an entropy threshold of `3` can be a starting point, it's essential to adjust this value based on your project's specific requirements and the nature of the data you have. diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.go index 24eb56dafab3..2880d856fea0 100644 --- a/pkg/custom_detectors/custom_detectors.go +++ b/pkg/custom_detectors/custom_detectors.go @@ -186,21 +186,29 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string // TODO: Log we're possibly leaving out results. return ctx.Err() } + + result := detectors.Result{ + DetectorType: detectorspb.DetectorType_CustomRegex, + DetectorName: c.GetName(), + ExtraData: map[string]string{}, + } + var raw string - for _, values := range match { + for key, values := range match { // values[0] contains the entire regex match. secret := values[0] if len(values) > 1 { secret = values[1] } raw += secret + + // if the match is of the primary regex, set it's value as primary secret value in result + if c.PrimaryRegexName == key { + result.SetPrimarySecretValue(secret) + } } - result := detectors.Result{ - DetectorType: detectorspb.DetectorType_CustomRegex, - DetectorName: c.GetName(), - Raw: []byte(raw), - ExtraData: map[string]string{}, - } + + result.Raw = []byte(raw) if !verify { select { diff --git a/pkg/custom_detectors/custom_detectors_test.go b/pkg/custom_detectors/custom_detectors_test.go index 90c9e67e45e9..4861be96b2cf 100644 --- a/pkg/custom_detectors/custom_detectors_test.go +++ b/pkg/custom_detectors/custom_detectors_test.go @@ -208,6 +208,25 @@ func TestDetector(t *testing.T) { assert.Equal(t, results[0].Raw, []byte(`123456`)) } +func TestDetectorPrimarySecret(t *testing.T) { + detector, err := NewWebhookCustomRegex(&custom_detectorspb.CustomRegex{ + Name: "test", + Keywords: []string{"secret"}, + Regex: map[string]string{"id": "id_[A-Z0-9]{10}_yy", "secret": "secret_[A-Z0-9]{10}_yy"}, + PrimaryRegexName: "secret", + }) + assert.NoError(t, err) + results, err := detector.FromData(context.Background(), false, []byte(` + // getData returns id and secret + func getData()(string, string){ + return "id_ALPHA10100_yy", "secret_YI7C90ACY1_yy" + } + `)) + assert.NoError(t, err) + assert.Equal(t, 1, len(results)) + assert.Equal(t, "secret_YI7C90ACY1_yy", results[0].GetPrimarySecretValue()) +} + func BenchmarkProductIndices(b *testing.B) { for i := 0; i < b.N; i++ { _ = productIndices(3, 2, 6) diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index 9b3a4ffed841..1be6d1c1f6e3 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -112,6 +112,14 @@ type Result struct { // analysis to run. The keys of the map are analyzer specific and // should match what is expected in the corresponding analyzer. AnalysisInfo map[string]string + + // primarySecret is used when a detector has multiple secret patterns. + // This secret is designated to determine the line number. + // If set, the line number will correspond to this secret. + primarySecret struct { + Value string + Line int64 + } } // CopyVerificationInfo clones verification info (status and error) from another Result struct. This is used when @@ -134,6 +142,26 @@ func (r *Result) VerificationError() error { return r.verificationError } +// SetPrimarySecretValue set the value passed as primary secret in the result +func (r *Result) SetPrimarySecretValue(value string) { + if value != "" { + r.primarySecret.Value = value + } +} + +// SetPrimarySecretLine set the passed line number as primary secret line number +func (r *Result) SetPrimarySecretLine(line int64) { + // line number is only set if value is set for primary secret + if r.primarySecret.Value != "" { + r.primarySecret.Line = line + } +} + +// GetPrimarySecretValue return primary secret match value +func (r *Result) GetPrimarySecretValue() string { + return r.primarySecret.Value +} + // redactSecrets replaces all instances of the given secrets with [REDACTED] in the error message. func redactSecrets(err error, secrets ...string) error { lastErr := unwrapToLast(err) diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index e33018b67dd1..957f90c4b6bb 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -1254,11 +1254,18 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool { // FragmentLineOffset sets the line number for a provided source chunk with a given detector result. func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) { - before, after, found := bytes.Cut(chunk.Data, result.Raw) + // get the primary secret value from the result if set + secret := result.GetPrimarySecretValue() + if secret == "" { + secret = string(result.Raw) + } + + before, after, found := bytes.Cut(chunk.Data, []byte(secret)) if !found { return 0, false } lineNumber := int64(bytes.Count(before, []byte("\n"))) + result.SetPrimarySecretLine(lineNumber) // If the line contains the ignore tag, we should ignore the result. endLine := bytes.Index(after, []byte("\n")) if endLine == -1 { diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index 229280fd381d..88b9db715605 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -159,6 +159,57 @@ func TestFragmentLineOffset(t *testing.T) { } } +func TestFragmentLineOffsetWithPrimarySecret(t *testing.T) { + primarySecretResult1 := &detectors.Result{ + Raw: []byte("id heresecret here"), // RAW has two secrets merged + } + + primarySecretResult1.SetPrimarySecretValue("secret here") // set `secret here` as primary secret value for line number calculation + + primarySecretResult2 := &detectors.Result{ + Raw: []byte("idsecret"), // RAW has two secrets merged + } + + tests := []struct { + name string + chunk *sources.Chunk + result *detectors.Result + expectedLine int64 + ignore bool + }{ + { + name: "primary secret line number - correct line number", + chunk: &sources.Chunk{ + Data: []byte("line1\nline2\nid here\nsecret here\nline5"), + }, + result: primarySecretResult1, + expectedLine: 3, + ignore: false, + }, + { + name: "no primary secret set - wrong line number", + chunk: &sources.Chunk{ + Data: []byte("line1\nline2\nid\nsecret\nline5"), + }, + result: primarySecretResult2, + expectedLine: 0, + ignore: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lineOffset, isIgnored := FragmentLineOffset(tt.chunk, tt.result) + if lineOffset != tt.expectedLine { + t.Errorf("Expected line offset to be %d, got %d", tt.expectedLine, lineOffset) + } + if isIgnored != tt.ignore { + t.Errorf("Expected isIgnored to be %v, got %v", tt.ignore, isIgnored) + } + }) + } +} + func setupFragmentLineOffsetBench(totalLines, needleLine int) (*sources.Chunk, *detectors.Result) { data := make([]byte, 0, 4096) needle := []byte("needle") diff --git a/pkg/pb/custom_detectorspb/custom_detectors.pb.go b/pkg/pb/custom_detectorspb/custom_detectors.pb.go index bc22e937ef50..245d36033e91 100644 --- a/pkg/pb/custom_detectorspb/custom_detectors.pb.go +++ b/pkg/pb/custom_detectorspb/custom_detectors.pb.go @@ -82,6 +82,7 @@ type CustomRegex struct { ExcludeWords []string `protobuf:"bytes,7,rep,name=exclude_words,json=excludeWords,proto3" json:"exclude_words,omitempty"` Entropy float32 `protobuf:"fixed32,8,opt,name=entropy,proto3" json:"entropy,omitempty"` ExcludeRegexesMatch []string `protobuf:"bytes,9,rep,name=exclude_regexes_match,json=excludeRegexesMatch,proto3" json:"exclude_regexes_match,omitempty"` + PrimaryRegexName string `protobuf:"bytes,10,opt,name=primary_regex_name,json=primaryRegexName,proto3" json:"primary_regex_name,omitempty"` } func (x *CustomRegex) Reset() { @@ -179,6 +180,13 @@ func (x *CustomRegex) GetExcludeRegexesMatch() []string { return nil } +func (x *CustomRegex) GetPrimaryRegexName() string { + if x != nil { + return x.PrimaryRegexName + } + return "" +} + type VerifierConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -262,7 +270,7 @@ var file_custom_detectors_proto_rawDesc = []byte{ 0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, 0x78, 0x52, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x73, 0x22, 0xbe, 0x03, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, + 0x6f, 0x72, 0x73, 0x22, 0xec, 0x03, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f, @@ -286,25 +294,28 @@ var file_custom_detectors_proto_rawDesc = []byte{ 0x6e, 0x74, 0x72, 0x6f, 0x70, 0x79, 0x12, 0x32, 0x0a, 0x15, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x65, 0x73, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, - 0x67, 0x65, 0x78, 0x65, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x1a, 0x38, 0x0a, 0x0a, 0x52, 0x65, - 0x67, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8e, 0x01, 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, - 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x75, - 0x6e, 0x73, 0x61, 0x66, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, - 0x24, 0x0a, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, - 0x61, 0x6e, 0x67, 0x65, 0x73, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, - 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, - 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, - 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x67, 0x65, 0x78, 0x65, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x72, + 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x52, + 0x65, 0x67, 0x65, 0x78, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x38, 0x0a, 0x0a, 0x52, 0x65, 0x67, 0x65, + 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x8e, 0x01, 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, + 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x75, + 0x6e, 0x73, 0x61, 0x66, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x75, 0x6e, 0x73, + 0x61, 0x66, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x24, 0x0a, + 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, + 0x67, 0x65, 0x73, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, + 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, + 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go b/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go index 9a08130e0dcb..080cec35f3d7 100644 --- a/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go +++ b/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go @@ -233,6 +233,8 @@ func (m *CustomRegex) validate(all bool) error { // no validation rules for Entropy + // no validation rules for PrimaryRegexName + if len(errors) > 0 { return CustomRegexMultiError(errors) } diff --git a/proto/custom_detectors.proto b/proto/custom_detectors.proto index 2c01d5c2dba6..67d53b467b6c 100644 --- a/proto/custom_detectors.proto +++ b/proto/custom_detectors.proto @@ -20,6 +20,7 @@ message CustomRegex { repeated string exclude_words = 7; float entropy = 8; repeated string exclude_regexes_match = 9; + string primary_regex_name = 10; }