From f53135d33fae18b0844af7e2ac2e55d4dde09870 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Wed, 26 Mar 2025 12:46:54 +0500 Subject: [PATCH 1/5] initial commit --- pkg/custom_detectors/custom_detectors.go | 22 ++++++++--- pkg/detectors/detectors.go | 16 ++++++++ pkg/engine/engine.go | 49 +++++++++++++++++------- 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.go index 24eb56dafab3..1127fdf36427 100644 --- a/pkg/custom_detectors/custom_detectors.go +++ b/pkg/custom_detectors/custom_detectors.go @@ -186,6 +186,13 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string // TODO: Log we're possibly leaving out results. return ctx.Err() } + + result := detectors.Result{ + DetectorType: detectorspb.DetectorType_CustomRegex, + DetectorName: c.GetName(), + ExtraData: map[string]string{}, + } + var raw string for _, values := range match { // values[0] contains the entire regex match. @@ -194,13 +201,16 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string secret = values[1] } raw += secret + + secretData := detectors.Secret{ + Value: secret, + Kind: "", + } + + result.AppendSecretData(secretData) } - result := detectors.Result{ - DetectorType: detectorspb.DetectorType_CustomRegex, - DetectorName: c.GetName(), - Raw: []byte(raw), - ExtraData: map[string]string{}, - } + + result.Raw = []byte(raw) if !verify { select { diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index 9b3a4ffed841..ed4c18341f0c 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -112,6 +112,22 @@ type Result struct { // analysis to run. The keys of the map are analyzer specific and // should match what is expected in the corresponding analyzer. AnalysisInfo map[string]string + + secretsData []Secret +} + +type Secret struct { + Value string + Kind string + Line int64 +} + +func (r *Result) AppendSecretData(secret Secret) { + r.secretsData = append(r.secretsData, secret) +} + +func (r *Result) GetSecretData() []Secret { + return r.secretsData } // CopyVerificationInfo clones verification info (status and error) from another Result struct. This is used when diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 14b4ef0efb8e..c6ae193c1086 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -1251,20 +1251,41 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool { // FragmentLineOffset sets the line number for a provided source chunk with a given detector result. func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) { - before, after, found := bytes.Cut(chunk.Data, result.Raw) - if !found { - return 0, false - } - lineNumber := int64(bytes.Count(before, []byte("\n"))) - // If the line contains the ignore tag, we should ignore the result. - endLine := bytes.Index(after, []byte("\n")) - if endLine == -1 { - endLine = len(after) - } - if bytes.Contains(after[:endLine], []byte(ignoreTag)) { - return lineNumber, true - } - return lineNumber, false + for _, secret := range result.GetSecretData() { + before, after, found := bytes.Cut(chunk.Data, []byte(secret.Value)) + if !found { + return 0, false + } + lineNumber := int64(bytes.Count(before, []byte("\n"))) + + secret.Line = lineNumber + // If the line contains the ignore tag, we should ignore the result. + endLine := bytes.Index(after, []byte("\n")) + if endLine == -1 { + endLine = len(after) + } + if bytes.Contains(after[:endLine], []byte(ignoreTag)) { + return lineNumber, true + } + + return lineNumber, false + } + + return 1, false + // before, after, found := bytes.Cut(chunk.Data, result.Raw) + // if !found { + // return 0, false + // } + // lineNumber := int64(bytes.Count(before, []byte("\n"))) + // // If the line contains the ignore tag, we should ignore the result. + // endLine := bytes.Index(after, []byte("\n")) + // if endLine == -1 { + // endLine = len(after) + // } + // if bytes.Contains(after[:endLine], []byte(ignoreTag)) { + // return lineNumber, true + // } + // return lineNumber, false } // FragmentFirstLineAndLink extracts the first line number and the link from the chunk metadata. From b4b506d96951fb22f297b1c0ea6442e935218ed5 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Wed, 26 Mar 2025 15:41:54 +0500 Subject: [PATCH 2/5] initial commit --- pkg/custom_detectors/CUSTOM_DETECTORS.md | 1 + pkg/custom_detectors/custom_detectors.go | 10 ++-- pkg/detectors/detectors.go | 42 +++++++++------ pkg/detectors/shopify/shopify.go | 3 ++ pkg/engine/engine.go | 54 +++++++------------ .../custom_detectorspb/custom_detectors.pb.go | 51 +++++++++++------- .../custom_detectors.pb.validate.go | 2 + proto/custom_detectors.proto | 1 + 8 files changed, 89 insertions(+), 75 deletions(-) diff --git a/pkg/custom_detectors/CUSTOM_DETECTORS.md b/pkg/custom_detectors/CUSTOM_DETECTORS.md index 720195bc4b2c..5bca7ee0449c 100644 --- a/pkg/custom_detectors/CUSTOM_DETECTORS.md +++ b/pkg/custom_detectors/CUSTOM_DETECTORS.md @@ -38,6 +38,7 @@ This guide will walk you through setting up a custom detector in TruffleHog to i - **`verify`**: An optional section to validate detected secrets. If you want to verify or unverify detected secrets, this section needs to be configured. If not configured, all detected secrets will be marked as unverified. Read [verification server examples](#verification-server-examples) **Other allowed parameters:** + - **`primary_regex_name`**: This parameter allows you designate the primary regex pattern when multiple regex patterns are defined in the regex section. If a match is found, the match for the designated primary regex will be used to determine the line number. The value must be one of the names specified in the regex section. - **`exclude_regexes_capture`**: This parameter allows you to define regex patterns to exclude specific parts of a detected secret. If a match is found within the detected secret, the portion matching this regex is excluded from the result. - **`exclude_regexes_match`**: This parameter enables you to define regex patterns to exclude entire matches from being reported as secrets. - **`entropy`**: This parameter is used to assess the randomness of detected strings. High entropy often indicates that a string is a potential secret, such as an API key or password, due to its complexity and unpredictability. It helps in filtering false-positives. While an entropy threshold of `3` can be a starting point, it's essential to adjust this value based on your project's specific requirements and the nature of the data you have. diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.go index 1127fdf36427..2880d856fea0 100644 --- a/pkg/custom_detectors/custom_detectors.go +++ b/pkg/custom_detectors/custom_detectors.go @@ -194,7 +194,7 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string } var raw string - for _, values := range match { + for key, values := range match { // values[0] contains the entire regex match. secret := values[0] if len(values) > 1 { @@ -202,12 +202,10 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string } raw += secret - secretData := detectors.Secret{ - Value: secret, - Kind: "", + // if the match is of the primary regex, set it's value as primary secret value in result + if c.PrimaryRegexName == key { + result.SetPrimarySecretValue(secret) } - - result.AppendSecretData(secretData) } result.Raw = []byte(raw) diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index ed4c18341f0c..1be6d1c1f6e3 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -113,21 +113,13 @@ type Result struct { // should match what is expected in the corresponding analyzer. AnalysisInfo map[string]string - secretsData []Secret -} - -type Secret struct { - Value string - Kind string - Line int64 -} - -func (r *Result) AppendSecretData(secret Secret) { - r.secretsData = append(r.secretsData, secret) -} - -func (r *Result) GetSecretData() []Secret { - return r.secretsData + // primarySecret is used when a detector has multiple secret patterns. + // This secret is designated to determine the line number. + // If set, the line number will correspond to this secret. + primarySecret struct { + Value string + Line int64 + } } // CopyVerificationInfo clones verification info (status and error) from another Result struct. This is used when @@ -150,6 +142,26 @@ func (r *Result) VerificationError() error { return r.verificationError } +// SetPrimarySecretValue set the value passed as primary secret in the result +func (r *Result) SetPrimarySecretValue(value string) { + if value != "" { + r.primarySecret.Value = value + } +} + +// SetPrimarySecretLine set the passed line number as primary secret line number +func (r *Result) SetPrimarySecretLine(line int64) { + // line number is only set if value is set for primary secret + if r.primarySecret.Value != "" { + r.primarySecret.Line = line + } +} + +// GetPrimarySecretValue return primary secret match value +func (r *Result) GetPrimarySecretValue() string { + return r.primarySecret.Value +} + // redactSecrets replaces all instances of the given secrets with [REDACTED] in the error message. func redactSecrets(err error, secrets ...string) error { lastErr := unwrapToLast(err) diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go index 08a29d9bbc92..be6c5226b4d3 100644 --- a/pkg/detectors/shopify/shopify.go +++ b/pkg/detectors/shopify/shopify.go @@ -53,6 +53,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result Raw: []byte(key + domainRes), } + // set the key match as primary secret as raw is combination of both key and domain + s1.SetPrimarySecretValue(match) + if verify { req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/admin/oauth/access_scopes.json", nil) if err != nil { diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index c6ae193c1086..3e152a96d511 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -1251,41 +1251,27 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool { // FragmentLineOffset sets the line number for a provided source chunk with a given detector result. func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) { - for _, secret := range result.GetSecretData() { - before, after, found := bytes.Cut(chunk.Data, []byte(secret.Value)) - if !found { - return 0, false - } - lineNumber := int64(bytes.Count(before, []byte("\n"))) - - secret.Line = lineNumber - // If the line contains the ignore tag, we should ignore the result. - endLine := bytes.Index(after, []byte("\n")) - if endLine == -1 { - endLine = len(after) - } - if bytes.Contains(after[:endLine], []byte(ignoreTag)) { - return lineNumber, true - } + // get the primary secret value from the result if set + secret := result.GetPrimarySecretValue() + if secret == "" { + secret = string(result.Raw) + } - return lineNumber, false - } - - return 1, false - // before, after, found := bytes.Cut(chunk.Data, result.Raw) - // if !found { - // return 0, false - // } - // lineNumber := int64(bytes.Count(before, []byte("\n"))) - // // If the line contains the ignore tag, we should ignore the result. - // endLine := bytes.Index(after, []byte("\n")) - // if endLine == -1 { - // endLine = len(after) - // } - // if bytes.Contains(after[:endLine], []byte(ignoreTag)) { - // return lineNumber, true - // } - // return lineNumber, false + before, after, found := bytes.Cut(chunk.Data, []byte(secret)) + if !found { + return 0, false + } + lineNumber := int64(bytes.Count(before, []byte("\n"))) + result.SetPrimarySecretLine(lineNumber) + // If the line contains the ignore tag, we should ignore the result. + endLine := bytes.Index(after, []byte("\n")) + if endLine == -1 { + endLine = len(after) + } + if bytes.Contains(after[:endLine], []byte(ignoreTag)) { + return lineNumber, true + } + return lineNumber, false } // FragmentFirstLineAndLink extracts the first line number and the link from the chunk metadata. diff --git a/pkg/pb/custom_detectorspb/custom_detectors.pb.go b/pkg/pb/custom_detectorspb/custom_detectors.pb.go index bc22e937ef50..245d36033e91 100644 --- a/pkg/pb/custom_detectorspb/custom_detectors.pb.go +++ b/pkg/pb/custom_detectorspb/custom_detectors.pb.go @@ -82,6 +82,7 @@ type CustomRegex struct { ExcludeWords []string `protobuf:"bytes,7,rep,name=exclude_words,json=excludeWords,proto3" json:"exclude_words,omitempty"` Entropy float32 `protobuf:"fixed32,8,opt,name=entropy,proto3" json:"entropy,omitempty"` ExcludeRegexesMatch []string `protobuf:"bytes,9,rep,name=exclude_regexes_match,json=excludeRegexesMatch,proto3" json:"exclude_regexes_match,omitempty"` + PrimaryRegexName string `protobuf:"bytes,10,opt,name=primary_regex_name,json=primaryRegexName,proto3" json:"primary_regex_name,omitempty"` } func (x *CustomRegex) Reset() { @@ -179,6 +180,13 @@ func (x *CustomRegex) GetExcludeRegexesMatch() []string { return nil } +func (x *CustomRegex) GetPrimaryRegexName() string { + if x != nil { + return x.PrimaryRegexName + } + return "" +} + type VerifierConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -262,7 +270,7 @@ var file_custom_detectors_proto_rawDesc = []byte{ 0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, 0x78, 0x52, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x73, 0x22, 0xbe, 0x03, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, + 0x6f, 0x72, 0x73, 0x22, 0xec, 0x03, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f, @@ -286,25 +294,28 @@ var file_custom_detectors_proto_rawDesc = []byte{ 0x6e, 0x74, 0x72, 0x6f, 0x70, 0x79, 0x12, 0x32, 0x0a, 0x15, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x65, 0x73, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, - 0x67, 0x65, 0x78, 0x65, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x1a, 0x38, 0x0a, 0x0a, 0x52, 0x65, - 0x67, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8e, 0x01, 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, - 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x75, - 0x6e, 0x73, 0x61, 0x66, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, - 0x24, 0x0a, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, - 0x61, 0x6e, 0x67, 0x65, 0x73, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, - 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, - 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, - 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x67, 0x65, 0x78, 0x65, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x72, + 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x52, + 0x65, 0x67, 0x65, 0x78, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x38, 0x0a, 0x0a, 0x52, 0x65, 0x67, 0x65, + 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x8e, 0x01, 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, + 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x75, + 0x6e, 0x73, 0x61, 0x66, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x75, 0x6e, 0x73, + 0x61, 0x66, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x24, 0x0a, + 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e, + 0x67, 0x65, 0x73, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, + 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, + 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go b/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go index 9a08130e0dcb..080cec35f3d7 100644 --- a/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go +++ b/pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go @@ -233,6 +233,8 @@ func (m *CustomRegex) validate(all bool) error { // no validation rules for Entropy + // no validation rules for PrimaryRegexName + if len(errors) > 0 { return CustomRegexMultiError(errors) } diff --git a/proto/custom_detectors.proto b/proto/custom_detectors.proto index 2c01d5c2dba6..67d53b467b6c 100644 --- a/proto/custom_detectors.proto +++ b/proto/custom_detectors.proto @@ -20,6 +20,7 @@ message CustomRegex { repeated string exclude_words = 7; float entropy = 8; repeated string exclude_regexes_match = 9; + string primary_regex_name = 10; } From 40837cfeaa84b9521b03636c5f30e77c83b54c14 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Tue, 15 Apr 2025 19:45:21 +0500 Subject: [PATCH 3/5] added test cases --- pkg/custom_detectors/custom_detectors_test.go | 22 ++++++++ pkg/engine/engine_test.go | 51 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/pkg/custom_detectors/custom_detectors_test.go b/pkg/custom_detectors/custom_detectors_test.go index 90c9e67e45e9..a414fea6a8be 100644 --- a/pkg/custom_detectors/custom_detectors_test.go +++ b/pkg/custom_detectors/custom_detectors_test.go @@ -208,6 +208,28 @@ func TestDetector(t *testing.T) { assert.Equal(t, results[0].Raw, []byte(`123456`)) } +func TestDetectorPrimarySecret(t *testing.T) { + detector, err := NewWebhookCustomRegex(&custom_detectorspb.CustomRegex{ + Name: "test", + // "password" is normally flagged as a false positive, but CustomRegex + // should allow the user to decide and report it as a result. + Keywords: []string{"secret"}, + Regex: map[string]string{"id": "id_[A-Z0-9]{10}_yy", "secret": "secret_[A-Z0-9]{10}_yy"}, + PrimaryRegexName: "secret", + }) + assert.NoError(t, err) + results, err := detector.FromData(context.Background(), false, []byte(` + // getData returns id and secret + func getData()(string, string){ + return "id_ALPHA10100_yy", "secret_YI7C90ACY1_yy" + } + `)) + assert.NoError(t, err) + assert.Equal(t, 1, len(results)) + assert.Equal(t, "secret_YI7C90ACY1_yy", results[0].GetPrimarySecretValue()) + assert.Equal(t, results[0].Raw, []byte(`id_ALPHA10100_yysecret_YI7C90ACY1_yy`)) +} + func BenchmarkProductIndices(b *testing.B) { for i := 0; i < b.N; i++ { _ = productIndices(3, 2, 6) diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index 229280fd381d..88b9db715605 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -159,6 +159,57 @@ func TestFragmentLineOffset(t *testing.T) { } } +func TestFragmentLineOffsetWithPrimarySecret(t *testing.T) { + primarySecretResult1 := &detectors.Result{ + Raw: []byte("id heresecret here"), // RAW has two secrets merged + } + + primarySecretResult1.SetPrimarySecretValue("secret here") // set `secret here` as primary secret value for line number calculation + + primarySecretResult2 := &detectors.Result{ + Raw: []byte("idsecret"), // RAW has two secrets merged + } + + tests := []struct { + name string + chunk *sources.Chunk + result *detectors.Result + expectedLine int64 + ignore bool + }{ + { + name: "primary secret line number - correct line number", + chunk: &sources.Chunk{ + Data: []byte("line1\nline2\nid here\nsecret here\nline5"), + }, + result: primarySecretResult1, + expectedLine: 3, + ignore: false, + }, + { + name: "no primary secret set - wrong line number", + chunk: &sources.Chunk{ + Data: []byte("line1\nline2\nid\nsecret\nline5"), + }, + result: primarySecretResult2, + expectedLine: 0, + ignore: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lineOffset, isIgnored := FragmentLineOffset(tt.chunk, tt.result) + if lineOffset != tt.expectedLine { + t.Errorf("Expected line offset to be %d, got %d", tt.expectedLine, lineOffset) + } + if isIgnored != tt.ignore { + t.Errorf("Expected isIgnored to be %v, got %v", tt.ignore, isIgnored) + } + }) + } +} + func setupFragmentLineOffsetBench(totalLines, needleLine int) (*sources.Chunk, *detectors.Result) { data := make([]byte, 0, 4096) needle := []byte("needle") From aaa767c1cef162b8bd3e056c30f48cc1dee0f8e2 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Tue, 15 Apr 2025 19:48:13 +0500 Subject: [PATCH 4/5] fixed test cases --- pkg/custom_detectors/custom_detectors_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/custom_detectors/custom_detectors_test.go b/pkg/custom_detectors/custom_detectors_test.go index a414fea6a8be..4861be96b2cf 100644 --- a/pkg/custom_detectors/custom_detectors_test.go +++ b/pkg/custom_detectors/custom_detectors_test.go @@ -210,9 +210,7 @@ func TestDetector(t *testing.T) { func TestDetectorPrimarySecret(t *testing.T) { detector, err := NewWebhookCustomRegex(&custom_detectorspb.CustomRegex{ - Name: "test", - // "password" is normally flagged as a false positive, but CustomRegex - // should allow the user to decide and report it as a result. + Name: "test", Keywords: []string{"secret"}, Regex: map[string]string{"id": "id_[A-Z0-9]{10}_yy", "secret": "secret_[A-Z0-9]{10}_yy"}, PrimaryRegexName: "secret", @@ -227,7 +225,6 @@ func TestDetectorPrimarySecret(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 1, len(results)) assert.Equal(t, "secret_YI7C90ACY1_yy", results[0].GetPrimarySecretValue()) - assert.Equal(t, results[0].Raw, []byte(`id_ALPHA10100_yysecret_YI7C90ACY1_yy`)) } func BenchmarkProductIndices(b *testing.B) { From c38961e843d9deebc77d2fef0f7b7948a40386fb Mon Sep 17 00:00:00 2001 From: Kashif Khan <70996046+kashifkhan0771@users.noreply.github.com> Date: Wed, 14 May 2025 21:21:09 +0500 Subject: [PATCH 5/5] removed primary secret from shopify --- pkg/detectors/shopify/shopify.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go index be6c5226b4d3..08a29d9bbc92 100644 --- a/pkg/detectors/shopify/shopify.go +++ b/pkg/detectors/shopify/shopify.go @@ -53,9 +53,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result Raw: []byte(key + domainRes), } - // set the key match as primary secret as raw is combination of both key and domain - s1.SetPrimarySecretValue(match) - if verify { req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/admin/oauth/access_scopes.json", nil) if err != nil {