Add support for storing location with content

Author: averagesecurityguy

files.go

@@ -43,7 +43,7 @@ func scrapeFiles(c chan<- *ProcessItem) {
 			continue
 		}
 
-		item := &ProcessItem{Source: "Local", Key: file.Name(), Content: string(content)}
+		item := &ProcessItem{Source: "Local", Location: path, Key: file.Name(), Content: string(content)}
 		c <- item
 
 		os.Remove(path)

gist.go

@@ -72,7 +72,7 @@ func scrapeGists(c chan<- *ProcessItem) {
 		g.Download()
 
 		for _, gist := range g.files {
-			item := &ProcessItem{Source: "Gist", Key: g.Key, Content: gist.Content}
+			item := &ProcessItem{Source: "Gist", Location: gist.Url, Key: g.Key, Content: gist.Content}
 			c <- item
 		}
 	}

github.go

@@ -9,7 +9,7 @@ import (
 type PushEventCommit struct {
 	Key   string `json:"sha"`
 	Url   string
-	Files []string
+	Files []GithubCommitFile
 }
 
 type PushEvent struct {
@@ -26,6 +26,7 @@ type GithubEvent struct {
 
 type GithubCommitFile struct {
 	Url string `json:"raw_url"`
+	Content string
 }
 
 type GithubCommit struct {
@@ -48,9 +49,11 @@ func (p *PushEvent) Download() {
 			continue
 		}
 
-		for _, f := range ghc.Files {
-			file := getGithub(f.Url)
-			p.Commits[i].Files = append(p.Commits[i].Files, string(file))
+		for j := range ghc.Files {
+			f := ghc.Files[j]
+
+			f.Content = string(get(f.Url))
+			p.Commits[i].Files = append(p.Commits[i].Files, f)
 		}
 	}
 
@@ -87,8 +90,10 @@ func scrapeGithubEvents(c chan<- *ProcessItem) {
 
 			pe.Download()
 			for i := range pe.Commits {
-				for _, f := range pe.Commits[i].Files {
-					item := &ProcessItem{Source: "GithubCommit", Key: string(pe.Key), Content: f}
+				for j := range pe.Commits[i].Files {
+					f := pe.Commits[i].Files[j]
+
+					item := &ProcessItem{Source: "GithubCommit", Location: f.Url, Key: string(pe.Key), Content: f.Content}
 					c <- item
 				}
 			}

paste.go

@@ -46,7 +46,7 @@ func scrapePastes(c chan<- *ProcessItem) {
 		p := pastes[i]
 		p.Download()
 
-		item := &ProcessItem{Source: "Pastebin", Key: p.Key, Content: p.Content}
+		item := &ProcessItem{Source: "Pastebin", Location: p.Url, Key: p.Key, Content: p.Content}
 		c <- item
 	}
 }

process.go

@@ -1,14 +1,30 @@
 package main
 
 import (
-	"fmt"
-	//"log"
+	"encoding/json"
+	"log"
 	"strings"
 )
 
+type SaveItem struct {
+	Location string
+	Content string
+}
+
+func (s *SaveItem) Json() []byte {
+	data, err := json.Marshal(s)
+	if err != nil {
+		log.Printf("[-] Could not create JSON for %s", s.Location)
+		return nil
+	}
+
+	return data
+}
+
 type ProcessItem struct {
 	Source  string
 	Key     string
+	Location string
 	Content string
 	Save    bool
 }
@@ -22,11 +38,13 @@ func (p *ProcessItem) Write() {
 		return
 	}
 
-	db.Write("pastes", p.Key, []byte(p.Content))
+	s := SaveItem{Location: p.Location, Content: p.Content}
+
+	db.Write("pastes", p.Key, s.Json())
 }
 
 func (p *ProcessItem) Regexes() {
-	for i, _ := range conf.Regexes {
+	for i := range conf.Regexes {
 		r := conf.Regexes[i]
 
 		switch r.Match {
@@ -35,18 +53,24 @@ func (p *ProcessItem) Regexes() {
 
 			if items != nil {
 				p.Save = true
-			}
+				data := strings.Join(items, "\n")
+				s := SaveItem{Location: p.Location, Content: data}
 
-			for k := range items {
-				rKey := fmt.Sprintf("%s-%d", p.Key, k)
-				db.Write(r.Bucket, rKey, []byte(items[k]))
+				db.Write(r.Bucket, p.Key, s.Json())
 			}
+
+			// for k := range items {
+			// 	rKey := fmt.Sprintf("%s-%d", p.Source, k)
+			// 	db.Write(r.Bucket, rKey, []byte(items[k]))
+			// }
 		case "one":
 			match := r.compiled.FindString(p.Content)
 
 			if match != "" {
 				p.Save = true
-				db.Write(r.Bucket, p.Key, []byte(match))
+				s := SaveItem{Location: p.Location, Content: match}
+
+				db.Write(r.Bucket, p.Key, s.Json())
 			}
 		default:
 		}
@@ -59,7 +83,7 @@ func (p *ProcessItem) Keywords() {
 
 		if strings.Contains(strings.ToLower(p.Content), strings.ToLower(kwd.Keyword)) {
 			p.Save = true
-			db.Write(kwd.Bucket, p.Key, nil)
+			db.Write(kwd.Bucket, p.Location, nil)
 		}
 	}
 }

web.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"html/template"
 	"log"
 	"net/http"
@@ -14,17 +15,21 @@ import (
 type Value struct {
 	Bucket string
 	Key    string
-	Value  string
+	Location string
+	Content  string
 }
 
 func NewValue(bucket, key string) *Value {
-	v := new(Value)
+	var s SaveItem
 
-	v.Bucket = bucket
-	v.Key = key
-	v.Value = db.Read(bucket, key)
+	data := db.Read(bucket, key)
+	err := json.Unmarshal([]byte(data), &s)
+	if err != nil {
+		log.Printf("[-] Could not unmarshal %s", key)
+		return &Value{Bucket: bucket, Key: key}
+	}
 
-	return v
+	return &Value{Bucket: bucket, Key: key, Location: s.Location, Content: s.Content}
 }
 
 type DataSet struct {

web/templates/read.html

@@ -5,6 +5,12 @@ <h2>{{ .Bucket }} - {{ .Key }}</h2>
 	<input type="text" id="search" name="{{ .Bucket }}" placeholder="Search term." />
 	<input type="submit" value="Search" onclick="search();" />
 </section>
-<pre>{{ .Value }}</pre>
+<section>
+	<h3>Location</h3>
+	<p><a href="{{ .Location }}">{{ .Location }}</a></p>
+
+	<h3>Content</h3>
+	<pre>{{ .Content }}</pre>
+</section>
 
 {{ end }}