Use buckets instead of prefixes.

Remove buckets from the configuration.
Create a bucket for each data type collected.

Author: averagesecurityguy

config.go

@@ -12,13 +12,13 @@ import (
 
 type Keyword struct {
 	Keyword string
-	Prefix  string
+	Bucket  string
 }
 
 type Regex struct {
 	Regex    string
 	compiled *regexp.Regexp
-	Prefix   string
+	Bucket   string
 	Match    string
 }
 
@@ -27,7 +27,6 @@ type Config struct {
 	db       *bolt.DB
 	Keywords []*Keyword // A list of keywords to search for in the data.
 	Regexes  []*Regex   // A list of regular expressions to test against data.
-	Buckets  []string   `json:"buckets"`       // List of buckets we need to create.
 	DbFile   string     `json:"database_file"` // File to use for the Store database.
 	MaxSize  int        `json:"max_size"`      // Do not save files larger than this many bytes.
 	MaxTime  int        `json:"max_time"`      // Max time, in seconds, to store previously downloaded keys.

config.json

@@ -5,52 +5,47 @@
     "database_file": "data/scrape.db",
     "github_token": "github_api_token",
     "save": true,
-	"buckets": [
-		"keywords",
-		"regexes",
-		"pastes"
-	],
     "regexes": [
         {
             "regex": "^([a-zA-Z0-9+_.-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]+):([^ ~/$|: ]+)",
-            "prefix": "creds",
+            "bucket": "creds",
             "match": "all"
         },
         {
             "regex": "[a-zA-Z0-9+_.-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}",
-            "prefix": "email",
+            "bucket": "email",
             "match": "all"
         },
         {
             "regex": "(?s)BEGIN (RSA|DSA|) PRIVATE KEY.*END (RSA|DSA|) PRIVATE KEY",
-            "prefix": "privkey",
+            "bucket": "privkey",
             "match": "all"
         },
         {
             "regex": "\\$[0-9]\\$[a-zA-Z0-9]+\\$[a-zA-Z0-9./=]+",
-            "prefix": "pwhash",
+            "bucket": "pwhash",
             "match": "all"
         },
         {
             "regex": "[a-zA-Z0-9]+::[a-zA-Z0-9]{10}:[a-z0-9]{32}:[a-z0-9-]+",
-            "prefix": "pwhash",
+            "bucket": "pwhash",
             "match": "all"
         },
         {
             "regex": "[a-zA-Z0-9-_]+:[0-9]+:[a-z0-9]{32}:[a-z0-9]{32}",
-            "prefix": "pwhash",
+            "bucket": "pwhash",
             "match": "all"
         },
         {
             "regex": "CVE-[0-9]{4}-[0-9]{4,5}",
-            "prefix": "exploit",
+            "bucket": "exploit",
             "match": "one"
         }
     ],
     "keywords": [
-        {"keyword": "`password`", "prefix": "sqlpass"},
-        {"keyword": "proof of concept", "prefix": "exploit"},
-        {"keyword": "remote code execution", "prefix": "exploit"},
-        {"keyword": "fullz", "prefix": "carder"}
+        {"keyword": "`password`", "bucket": "sqlpass"},
+        {"keyword": "proof of concept", "bucket": "exploit"},
+        {"keyword": "remote code execution", "bucket": "exploit"},
+        {"keyword": "fullz", "bucket": "carder"}
     ]
 }

process.go

@@ -31,16 +31,15 @@ func processRegexes(key, content string) {
 			}
 
 			for k := range items {
-				rKey := fmt.Sprintf("%s-%s-%d", r.Prefix, key, k)
-				writeDB(conf.db, "regexes", rKey, []byte(items[k]))
+				rKey := fmt.Sprintf("%s-%d", key, k)
+				writeDB(conf.db, r.Bucket, rKey, []byte(items[k]))
 			}
 		case "one":
 			match := r.compiled.FindString(content)
-			rKey := fmt.Sprintf("%s-%s", r.Prefix, key)
 
 			if match != "" {
 				save = true
-				writeDB(conf.db, "regexes", rKey, []byte(match))
+				writeDB(conf.db, r.Bucket, key, []byte(match))
 			}
 		default:
 		}
@@ -55,11 +54,10 @@ func processKeywords(key, content string) {
 	save := false
 	for i, _ := range conf.Keywords {
 		kwd := conf.Keywords[i]
-		kwdKey := fmt.Sprintf("%s-%s", kwd.Prefix, key)
 
 		if strings.Contains(strings.ToLower(content), strings.ToLower(kwd.Keyword)) {
 			save = true
-			writeDB(conf.db, "keywords", kwdKey, []byte(key))
+			writeDB(conf.db, kwd.Bucket, key, nil)
 		}
 	}
 

store.go

@@ -25,8 +25,14 @@ func getDBConn() *bolt.DB {
 func initDB() {
 	db := getDBConn()
 
-	for _, bucket := range conf.Buckets {
-		createBucket(db, bucket)
+	createBucket(db, "pastes")
+
+	for _, kw := range conf.Keywords {
+		createBucket(db, kw.Bucket)
+	}
+
+	for _, re := range conf.Regexes {
+		createBucket(db, re.Bucket)
 	}
 
 	db.Close()