feat: pass metadata & general improvements

Author: nohehf

.gitignore

@@ -11,4 +11,7 @@ testers
 /goctopus
 
 dist/
-.env
+.env
+
+# root go files (often used for testing / prototyping)
+/*.go

pkg/address/address.go

@@ -1,14 +1,18 @@
 package address
 
+import "strings"
+
 type Addr struct {
-	Address string
-	Source  string
+	Address  string
+	Source   string
+	Metadata map[string]string
 }
 
 func New(address string) *Addr {
 	return &Addr{
-		Address: address,
-		Source:  address,
+		Address:  address,
+		Source:   address,
+		Metadata: map[string]string{},
 	}
 }
 
@@ -19,3 +23,19 @@ func NewSourced(address, source string) *Addr {
 		Source:  source,
 	}
 }
+
+func (a *Addr) AddMetadata(key, value string) {
+	a.Metadata[key] = value
+}
+
+func (a *Addr) Copy() *Addr {
+	metadataCopy := make(map[string]string)
+	for k, v := range a.Metadata {
+		metadataCopy[k] = v
+	}
+	return &Addr{
+		Address:  strings.Clone(a.Address),
+		Source:   strings.Clone(a.Source),
+		Metadata: metadataCopy,
+	}
+}

pkg/config/config.go

@@ -76,7 +76,7 @@ func LoadFromArgs() {
 		log.SetLevel(log.ErrorLevel)
 	}
 
-	if err := ValidateConfig(&config); err != nil {
+	if err := validateConfig(&config, true); err != nil {
 		log.Error(err)
 		flag.PrintDefaults()
 		os.Exit(1)
@@ -85,7 +85,10 @@ func LoadFromArgs() {
 	c = &config
 }
 
-func ValidateConfig(conf *Config) error {
+// Validates the config.
+// `cli` is used to determine if the config is loaded from the CLI or from a file.
+// If cli is false, then the addresses check is skipped.
+func validateConfig(conf *Config, isCli bool) error {
 	if conf.MaxWorkers < 1 {
 		return errors.New("[Invalid config] Max workers must be greater than 0")
 	}
@@ -98,15 +101,15 @@ func ValidateConfig(conf *Config) error {
 		return errors.New("[Invalid config] Introspection has to be enabled to use field suggestion fingerprinting")
 	}
 
-	if conf.InputFile == "" && len(conf.Addresses) == 0 {
+	if isCli && c.InputFile == "" && len(c.Addresses) == 0 {
 		return errors.New("[Invalid config] Please specify an input file or a list of addresses")
 	}
 
 	return nil
 }
 
 func Load(config *Config) {
-	if err := ValidateConfig(config); err != nil {
+	if err := validateConfig(config, false); err != nil {
 		log.Error(err)
 		flag.PrintDefaults()
 		os.Exit(1)

pkg/domain/enumeration.go

@@ -11,7 +11,9 @@ import (
 
 func makeCallback(domain *address.Addr, subDomains chan *address.Addr) func(s *resolve.HostEntry) {
 	return func(s *resolve.HostEntry) {
-		subDomains <- address.NewSourced(s.Host, domain.Source)
+		addr := domain.Copy()
+		addr.Address = s.Host
+		subDomains <- addr
 	}
 }
 

pkg/domain/fingerprint.go

@@ -48,6 +48,7 @@ func FingerprintSubDomain(domain *address.Addr) (*output.FingerprintOutput, erro
 		}
 		output.Domain = domain.Address
 		output.Source = domain.Source
+		output.Metadata = domain.Metadata
 		return output, nil
 	}
 	return nil, errors.New("no graphql endpoint found")

pkg/goctopus/fingerprint.go

@@ -17,7 +17,7 @@ func worker(addresses chan *address.Addr, output chan *output.FingerprintOutput,
 	log.Debugf("Worker %d instantiated", workerId)
 	for address := range addresses {
 		log.Debugf("Worker %d started on: %v", workerId, address)
-		res, err := FingerprintAddress(address)
+		res, err := fingerprintAddress(address)
 		if err == nil {
 			log.Debugf("Worker %d found endpoint: %v", workerId, res)
 			output <- res
@@ -26,7 +26,10 @@ func worker(addresses chan *address.Addr, output chan *output.FingerprintOutput,
 	log.Debugf("Worker %d finished", workerId)
 }
 
-func FingerprintAddress(address *address.Addr) (*output.FingerprintOutput, error) {
+/**
+ * Fingerprint an address, without subdomain enumeration
+ */
+func fingerprintAddress(address *address.Addr) (*output.FingerprintOutput, error) {
 	// If the domain is a url, we don't need to crawl it
 	if utils.IsUrl(address.Address) {
 		return endpoint.FingerprintEndpoint(address)
@@ -65,6 +68,6 @@ func FingerprintAddresses(addresses chan *address.Addr, output chan *output.Fing
 	close(enumeratedAddresses)
 	log.Debugf("Waiting for workers to finish...")
 	workersWg.Wait()
-	close(output)
 	log.Debugf("All workers finished")
+	close(output)
 }

pkg/goctopus/single_address.go

@@ -0,0 +1,20 @@
+package goctopus
+
+import (
+	"github.com/Escape-Technologies/goctopus/pkg/address"
+	out "github.com/Escape-Technologies/goctopus/pkg/output"
+)
+
+// Fingerprints a single address and outputs a slice of FingerprintOutput.
+func FingerprintAddress(addr *address.Addr) []*out.FingerprintOutput {
+	outputSlice := make([]*out.FingerprintOutput, 0)
+	output := make(chan *out.FingerprintOutput, 1)
+	addresses := make(chan *address.Addr, 1)
+	addresses <- addr
+	close(addresses)
+	go FingerprintAddresses(addresses, output)
+	for fingerprintOutput := range output {
+		outputSlice = append(outputSlice, fingerprintOutput)
+	}
+	return outputSlice
+}

pkg/output/output.go

@@ -12,7 +12,6 @@ type FingerprintResult string
 const (
 	ResultOpenGraphql         FingerprintResult = "OPEN_GRAPHQL"
 	ResultAuthentifiedGraphql FingerprintResult = "AUTHENTIFIED_GRAPHQL"
-	// ResultMaybeGraphql FingerprintResult = "MAYBE_GRAPHQL"
 )
 
 type FingerprintOutput struct {
@@ -21,7 +20,8 @@ type FingerprintOutput struct {
 	Url             string            `json:"url"`
 	Introspection   bool              `json:"introspection"`
 	FieldSuggestion bool              `json:"field_suggestion"`
-	Source          string            `json:"source"` // the original address used to fingerprint the endpoint
+	Source          string            `json:"source"`   // the original address used to fingerprint the endpoint
+	Metadata        map[string]string `json:"metadata"` // optional metadata
 }
 
 func (o *FingerprintOutput) MarshalJSON() ([]byte, error) {
@@ -56,5 +56,9 @@ func marshalOutput(o *FingerprintOutput, c *config.Config) ([]byte, error) {
 		outputMap["domain"] = utils.DomainFromUrl(o.Url)
 	}
 
+	if o.Metadata == nil || len(o.Metadata) == 0 {
+		delete(outputMap, "metadata")
+	}
+
 	return json.Marshal(outputMap)
 }