Merge pull request #10 from numiralofe/delete-strategy

added delete strategy changes

Author: eduardev

README.md

@@ -99,6 +99,7 @@ Below are all available command line flags for starting **Gonsul**
 --allow-deletes=
 --poll-interval=
 --input-ext=
+--KeepFileExt=
 ```
 Below is the full description for each individual command line flag
 
@@ -290,18 +291,21 @@ no secrets are written to disk.
 > `default:` **false**  
 > `example:` **`--allow-deletes=true`**
 
-This instructs Gonsul how it should proceed in case some Consul deletes are to be made. If the value for this flag is 
-`true`, Gonsul will proceed with the delete operations, but if instead is `false`, Gonsul will not proceed with the 
-deletes, and depending on the `--strategy` it is running at, it respondes with some different behaviors, such as:
-- **`ONCE`** When running in once mode, Gonsul will terminate with __error code 10__ and output to console all the 
-Consul KV paths that are supposed to be deleted.
-- **`HOOK`** Gonsul will repond to the HTTP request with error 503 and will also return the following headers and values:
-	- `X-Gonsul-Error:10`
-	- `X-Gonsul-Delete-Paths:path1/to/be/deleted,path2/to/be/deleted`
-- **`POLL`** Gonsul will log all the paths to be deleted as ERRORS and carry on, over and over. In this mode you should 
-monitor Gonsul logs to detect any found errors, and react appropriately. The errors will follow the syntax: 
-	- `[ERROR] [28-01-2018 17:38:25 1234] error-10 path1/to/be/deleted`
-	- `[ERROR] [28-01-2018 17:38:25 5678] error-10 path2/to/be/deleted`
+This option sets Gonsul behaviour on how it should proceed in case some Consul deletes are to be made. 
+
+- **`true`** With this mode deletes are allowed and Gonsul will proceed with the delete operations, removed files from the repo and/or files added directly on consul k/v will be deleted.
+- **`skip`** In this mode, Gonsul will skip the deletion check and operations and will just sync/push files to consul k/v path (can lead to inconsistencies at the consul k/v path since removed files from the repo and/or files added directly on consul k/v will stay in the consul path because we are skipping deletions).
+- **`false`** _this is the default mode_ and Gonsul will not proceed with the deletes, and depending on the `--strategy` it is running at, will respond with some different behaviors, such as:
+
+    1. **`ONCE`** When running in once mode, Gonsul will terminate with __error code 10__ and output to console all the Consul KV paths that are supposed to be deleted.
+
+    2. **`HOOK`** Gonsul will repond to the HTTP request with error 503 and will also return the following headers and values:
+        - `X-Gonsul-Error:10`
+        - `X-Gonsul-Delete-Paths:path1/to/be/deleted,path2/to/be/deleted`
+
+    3. **`POLL`** Gonsul will log all the paths to be deleted as ERRORS and carry on, over and over. In this mode you should monitor Gonsul logs to detect any found errors, and react appropriately. The errors will follow the syntax: 
+        - `[ERROR] [28-01-2018 17:38:25 1234] error-10 path1/to/be/deleted`
+        - `[ERROR] [28-01-2018 17:38:25 5678] error-10 path2/to/be/deleted`
 
 
 ### `--poll-interval`
@@ -321,6 +325,12 @@ This is the number of seconds you want Gonsul to wait between checks on the repo
 This is the file extensions that Gonsul should consider as inputs to populate our Consul. Please set each extension 
 without the dot, and separate each extension with a comma.
 
+### `--keep-FileExt`
+> `require:` **no**  
+> `default:` **false**  
+> `example:` **`--keep-FileExt=true`**
+
+Gonsul default behavior is to remove/trim the filextension from the filename when pushing files to consul k/v path, if this option is set to true gonsul will keep the file extension.
 
 ### `--timeout`
 > `require:` **no**  

internal/config/config.go

@@ -34,7 +34,7 @@ type config struct {
 	expandJSON      bool
 	doSecrets       bool
 	secretsMap      map[string]string
-	allowDeletes    bool
+	allowDeletes    string
 	pollInterval    int
 	Working         chan bool
 	validExtensions []string
@@ -62,7 +62,7 @@ type IConfig interface {
 	ShouldExpandJSON() bool
 	DoSecrets() bool
 	GetSecretsMap() map[string]string
-	AllowDeletes() bool
+	AllowDeletes() string
 	GetPollInterval() int
 	WorkingChan() chan bool
 	GetValidExtensions() []string
@@ -111,6 +111,12 @@ func buildConfig(flags ConfigFlags) (*config, error) {
 		return nil, errors.New(fmt.Sprintf("strategy invalid, must be one of: %s, %s, %s, %s", StrategyDry, StrategyOnce, StrategyPoll, StrategyHook))
 	}
 
+	// Make sure delete method is properly given
+	allowDeletes := strings.ToLower(*flags.AllowDeletes)
+	if allowDeletes != "true" && allowDeletes != "false" && allowDeletes != "skip" {
+		return nil, errors.New(fmt.Sprintf("AllowDelete method is invalid, please define one of the following valid options as argument: true, false, skip"))
+	}
+
 	// Shall we use a local copy of the repository instead of cloning ourselves
 	// This should be useful if we use Gonsul on a CI stack (such as Bamboo)
 	// And the repo is checked out already, alleviating Gonsul work
@@ -224,8 +230,8 @@ func (config *config) GetSecretsMap() map[string]string {
 	return config.secretsMap
 }
 
-func (config *config) AllowDeletes() bool {
-	return config.allowDeletes
+func (config *config) AllowDeletes() string {
+	return strings.ToLower(config.allowDeletes)
 }
 
 func (config *config) GetPollInterval() int {

internal/config/flags_parser.go

@@ -21,10 +21,10 @@ type ConfigFlags struct {
 	ConsulBasePath  *string
 	ExpandJSON      *bool
 	SecretsFile     *string
-	AllowDeletes    *bool
+	AllowDeletes    *string
 	PollInterval    *int
 	ValidExtensions *string
-	KeepFileExt 	*bool	
+	KeepFileExt     *bool
 	Timeout         *int
 	Version         *bool
 }
@@ -46,7 +46,7 @@ func parseFlags() ConfigFlags {
 	flags.ConsulBasePath = flag.String("consul-base-path", "", "The base KV path will be prefixed to dir path")
 	flags.ExpandJSON = flag.Bool("expand-json", false, "Expand and parse JSON files as full paths? (Default false)")
 	flags.SecretsFile = flag.String("secrets-file", "", "A key value json file with placeholders->secrets mapping, in order to do on the fly replace")
-	flags.AllowDeletes = flag.Bool("allow-deletes", false, "Show Gonsul issue deletes? (If not, nothing will be done and a report on conflicting deletes will be shown) (Default false)")
+	flags.AllowDeletes = flag.String("allow-deletes", "false", "false, nothing will be done and a report on conflicting deletes will be shown; true: deletes reported conflitcs and proceeds; skip: reportes conflitcs, does not performe any deletes and proceeds syncing remaining files.) (Default false)")
 	flags.PollInterval = flag.Int("poll-interval", 60, "The number of seconds for the repository polling interval")
 	flags.ValidExtensions = flag.String("input-ext", "json,txt,ini", "A comma separated list of file extensions valid as input")
 	flags.KeepFileExt = flag.Bool("keep-FileExt", false, "Do we want to keep file name extensions ? (If not set to true defaults by ommiting the file name extension.) (Default false)")

internal/importer/helper.go

@@ -59,7 +59,7 @@ func (i *importer) createOperationMatrix(liveData map[string]string, localData m
 	// Now check for deletes
 	// Check for deletes
 	for liveKey := range liveData {
-		if _, ok := localData[liveKey]; !ok {
+		if _, ok := localData[liveKey]; !ok && i.config.AllowDeletes() != "skip" {
 			// Not found in local - DELETE
 			operations.AddDelete(entities.Entry{KVPath: liveKey, Value: ""})
 		}

internal/importer/importer.go

@@ -1,12 +1,12 @@
 package importer
 
 import (
-	"internal/config"
-	"internal/entities"
-	"internal/util"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"internal/config"
+	"internal/entities"
+	"internal/util"
 	"net/http"
 )
 
@@ -57,7 +57,7 @@ func (i *importer) Start(localData map[string]string) {
 
 func (i *importer) processOperations(matrix entities.OperationMatrix) {
 	// Did we got any deletes and are we allowed to delete them?
-	if !i.config.AllowDeletes() && matrix.HasDeletes() {
+	if i.config.AllowDeletes() == "false" && matrix.HasDeletes() {
 		// We're not supposed to trigger Consul deletes, output report and exit with error
 		i.logger.PrintError("We're stopping as there are deletes and Gonsul is running without delete permission")
 		i.logger.PrintError("Below is all the Consul KV paths that would be deleted")
@@ -83,14 +83,13 @@ func (i *importer) processOperations(matrix entities.OperationMatrix) {
 		verb := op.GetVerb()
 		path := op.GetPath()
 
-
 		currentPayload, err := json.Marshal(transactions)
 		if err != nil {
 			util.ExitError(errors.New("Marshal: "+err.Error()), util.ErrorFailedJsonEncode, i.logger)
 		}
-		currentPayloadSize := len(currentPayload);
+		currentPayloadSize := len(currentPayload)
 
-		var TxnKV entities.ConsulTxnKV;
+		var TxnKV entities.ConsulTxnKV
 
 		if op.GetType() == entities.OperationDelete {
 			TxnKV = entities.ConsulTxnKV{Verb: &verb, Key: &path}
@@ -105,7 +104,7 @@ func (i *importer) processOperations(matrix entities.OperationMatrix) {
 			util.ExitError(errors.New("Marshal: "+err.Error()), util.ErrorFailedJsonEncode, i.logger)
 		}
 
-		if currentPayloadSize + len(nextOpPayload) > maximumPayloadSize {
+		if currentPayloadSize+len(nextOpPayload) > maximumPayloadSize {
 			i.processConsulTransaction(transactions)
 			transactions = []entities.ConsulTxn{}
 		}