Improve extensibility

Move all regexes to config file. No more special processing.
Store collected data as values not keys.

Author: averagesecurityguy

config.go

@@ -19,6 +19,7 @@ type Regex struct {
 	Regex    string
 	compiled *regexp.Regexp
 	Prefix   string
+	Match    string
 }
 
 type Config struct {

config.json

@@ -6,19 +6,46 @@
     "github_token": "github_api_token",
     "save": true,
 	"buckets": [
-		"awskeys",
-		"emails",
-		"creds",
-		"privkeys",
 		"keywords",
 		"regexes",
 		"pastes"
 	],
     "regexes": [
-        {"regex": "\\$[0-9]\\$[a-zA-Z0-9]\\$[a-zA-Z0-9./=]+", "prefix": "pwhash"},
-        {"regex": "[a-zA-Z0-9]+::[a-zA-Z0-9]{10}:[a-z0-9]{32}:[a-z0-9-]+", "prefix": "pwhash"},
-        {"regex": "[a-zA-Z0-9-_]+:[0-9]+:[a-z0-9]{32}:[a-z0-9]{32}", "prefix": "pwhash"},
-        {"regex": "CVE-[0-9]{4}-[0-9]{4,5}", "prefix": "exploit"}
+        {
+            "regex": "(?m)^([a-zA-Z0-9+_.-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]+):([^ ~/$| ].*$)",
+            "prefix": "creds",
+            "match": "all"
+        },
+        {
+            "regex": "[a-zA-Z0-9+_.-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}",
+            "prefix": "email",
+            "match": "all"
+        },
+        {
+            "regex": "(?s)BEGIN (RSA|DSA|) PRIVATE KEY.*END (RSA|DSA|) PRIVATE KEY",
+            "prefix": "privkey",
+            "match": "all"
+        },
+        {
+            "regex": "\\$[0-9]\\$[a-zA-Z0-9]+\\$[a-zA-Z0-9./=]+",
+            "prefix": "pwhash",
+            "match": "all"
+        },
+        {
+            "regex": "[a-zA-Z0-9]+::[a-zA-Z0-9]{10}:[a-z0-9]{32}:[a-z0-9-]+",
+            "prefix": "pwhash",
+            "match": "all"
+        },
+        {
+            "regex": "[a-zA-Z0-9-_]+:[0-9]+:[a-z0-9]{32}:[a-z0-9]{32}",
+            "prefix": "pwhash",
+            "match": "all"
+        },
+        {
+            "regex": "CVE-[0-9]{4}-[0-9]{4,5}",
+            "prefix": "exploit",
+            "match": "one"
+        }
     ],
     "keywords": [
         {"keyword": "`password`", "prefix": "sqlpass"},

process.go

@@ -2,153 +2,64 @@ package main
 
 import (
 	"fmt"
-	"regexp"
 	"strings"
 )
 
-var reCreds = regexp.MustCompile("(?m)^([a-zA-Z0-9+_.-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]+):([^ ~/$| ].*$)")
-var reEmail = regexp.MustCompile("[a-zA-Z0-9+_.-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]+")
-var rePrivKey = regexp.MustCompile("(?s)BEGIN (RSA|DSA|) PRIVATE KEY.*END (RSA|DSA|) PRIVATE KEY")
-var reAwsKey = regexp.MustCompile("(?is)(AKIA[A-Z0-9]{16})[\"',: =]+([A-Za-z0-9+/]{40})")
-var reBase64 = regexp.MustCompile("^([a-zA-Z0-9+/]+)$")
-
-// Find AWS access keys and secrets
-func processAWSKeys(contents, key string) bool {
-	// Base64 content yields false positives. Skip documents that are only Base64
-	if b64 := reBase64.FindString(contents); b64 != "" {
-		return false
-	}
-
-	awsKeys := reAwsKey.FindAllStringSubmatch(contents, -1)
-
-	// No keys found.
-	if awsKeys == nil {
-		return false
-	}
-
-	for _, awsKey := range awsKeys {
-		conf.ds.Write("awskeys", strings.Join(awsKey[1:], ":"), []byte(key))
-	}
-
-	return true
-}
-
-// Look for email addresses and save them to a file.
-func processEmails(contents, key string) bool {
-	emails := reEmail.FindAllString(contents, -1)
-
-	// No emails found.
-	if emails == nil {
-		return false
-	}
-
-	for _, email := range emails {
-		email = cleanEmail(email)
-
-		if email == "" {
-			continue
-		}
-
-		conf.ds.Write("emails", email, []byte(key))
-	}
-
-	return true
-}
-
-// Look for credentials in the format of email:password and save them to a file.
-func processCredentials(contents, key string) bool {
-	creds := reCreds.FindAllString(contents, -1)
-
-	// No creds found.
-	if creds == nil {
-		return false
-	}
-
-	for _, cred := range creds {
-		if cred == "" {
-			continue
-		}
-
-		conf.ds.Write("creds", cred, []byte(key))
-	}
-
-	return true
-}
-
-// Look for private keys.
-func processPrivKey(contents, key string) bool {
-	privKeys := rePrivKey.FindAllString(contents, -1)
-
-	// No keys found.
-	if privKeys == nil {
-		return false
-	}
-
-	for _, privKey := range privKeys {
-		conf.ds.Write("privkeys", privKey, []byte(key))
-	}
-
-	return true
-}
-
-func savePaste(key, value string) {
+func savePaste(key, content string) {
 	if conf.Save == false {
 		return
 	}
 
-	if len(value) > conf.MaxSize {
+	if len(content) > conf.MaxSize {
 		return
 	}
 
-	conf.ds.Write("pastes", key, []byte(value))
+	conf.ds.Write("pastes", key, []byte(content))
 }
 
-func processContent(key, content string) {
-	conf.ds = getStoreConn()
-	defer conf.ds.Close()
-
-	// Find and save specific data.
-	switch {
-	case processCredentials(content, key):
-		savePaste(key, content)
-	case processEmails(content, key):
-		savePaste(key, content)
-	case processPrivKey(content, key):
-		savePaste(key, content)
-	case processAWSKeys(content, key):
-		savePaste(key, content)
-	default:
-	}
-
-	// Save pastes that match any of our regular expressions. Use these to find
-	// interesting data that will eventually be processed with a more specific
-	// method.
+func processRegexes(key, content string) {
 	save := false
 	for i, _ := range conf.Regexes {
 		r := conf.Regexes[i]
-		rKey := fmt.Sprintf("%s-%s", r.Prefix, key)
-		match := r.compiled.FindString(content)
 
-		if match != "" {
-			save = true
-			conf.ds.Write("regexes", rKey, nil)
+		switch r.Match {
+		case "all":
+			items := r.compiled.FindAllString(content, -1)
+
+			if items != nil {
+				save = true
+			}
+
+			for k := range items {
+				rKey := fmt.Sprintf("%s-%s-%d", r.Prefix, key, k)
+				conf.ds.Write("regexes", rKey, []byte(items[k]))
+			}
+		case "one":
+			match := r.compiled.FindString(content)
+			rKey := fmt.Sprintf("%s-%s", r.Prefix, key)
+
+			if match != "" {
+				save = true
+				conf.ds.Write("regexes", rKey, []byte(match))
+			}
+		default:
 		}
 	}
 
 	if save {
 		savePaste(key, content)
 	}
+}
 
-	// Save pastes that match any of our keywords. Use these to find interesting
-	// data that will eventually be processed with a more specific method.
-	save = false
+func processKeywords(key, content string) {
+	save := false
 	for i, _ := range conf.Keywords {
 		kwd := conf.Keywords[i]
 		kwdKey := fmt.Sprintf("%s-%s", kwd.Prefix, key)
 
 		if strings.Contains(strings.ToLower(content), strings.ToLower(kwd.Keyword)) {
 			save = true
-			conf.ds.Write("keywords", kwdKey, nil)
+			conf.ds.Write("keywords", kwdKey, []byte(key))
 		}
 	}
 
@@ -157,16 +68,11 @@ func processContent(key, content string) {
 	}
 }
 
-// Remove common false positives in email addresses.
-func cleanEmail(email string) string {
-	email = strings.ToLower(email)
-
-	switch {
-	case strings.HasSuffix(email, "2x.png"):
-		return ""
-	case strings.HasSuffix(email, ".so"):
-		return ""
-	default:
-		return email
-	}
+
+func processContent(key, content string) {
+	conf.ds = getStoreConn()
+	defer conf.ds.Close()
+
+	processRegexes(key, content)
+	processKeywords(key, content)
 }