Merge pull request #38 from riotkit-org/issue-32-refactor-gpg-support

Refactor GPG support

Author: blackandred

DEVELOPMENT.md

@@ -0,0 +1,29 @@
+Running make & restore locally to test encryption & sending & receiving
+-----------------------------------------------------------------------
+
+#### Server part
+
+1. In backup-repository repository you need to run `make k3d skaffold-deploy`, then you will have a working Backup Repository instance in local Kubernetes.
+
+2. Setup a tunnel for client connections
+
+```bash
+kubectl port-forward svc/server-backup-repository-server -n backups 8080:8080
+```
+
+#### Client part
+
+```bash
+# export basic settings. Execute once in a console
+export BM_COLLECTION_ID=iwa-ait
+export BM_PASSPHRASE=riotkit
+export BM_AUTH_TOKEN=$(curl -s -X POST -d '{"username":"admin","password":"admin"}' -H 'Content-Type: application/json' 'http://127.0.0.1:8080/api/stable/auth/login' | jq '.data.token' -r)
+export BM_URL=http://127.0.0.1:8080
+```
+
+#### Perform testing
+
+```bash
+./.build/backup-maker make --cmd "tar -zcvf - ./" --key ./resources/test/gpg-key.asc
+./.build/backup-maker restore --cmd "cat - > /tmp/restore.tar.gz" --passphrase riotkit --private-key ./resources/test/gpg-key.asc
+```

README.md

@@ -40,7 +40,6 @@ export BM_PASSPHRASE="riotkit"; \
 backup-maker make --url https://example.org \
     -c "tar -zcvf - ./" \
     --key build/test/backup.key \
-    --recipient test@riotkit.org \
     --log-level info
 ```
 
@@ -54,7 +53,6 @@ backup-maker restore --url $$(cat .build/test/domain.txt) \
     -c "cat - > /tmp/test" \
     --private-key .build/test/backup.key \
     --passphrase riotkit \
-    --recipient test@riotkit.org \
     --log-level debug
 ```
 
@@ -65,25 +63,23 @@ please take a look at `Backup Controller` documentation.
 
 **Note: GPG steps are optional**
 
-1. `gpg` keyring is created in a temporary directory, keys are imported
+1. `gpg` keys are loaded
 2. Command specified in `--cmd` or in `-c` is executed
 3. Result of the command, it's stdout is transferred to the `gpg` process
 4. From `gpg` process the encoded data is buffered directly to the server
 5. Feedback is returned
-6. Temporary `gpg` keyring is deleted
 
 ## Restore - How it works?
 
 It is very similar as in backup operation.
 
-1. `gpg` keyring is created in a temporary directory, keys are imported
+1. `gpg` keys are loaded
 2. Command specified in `--cmd` or in `-c` is executed
 3. `gpg` process is started
 4. Backup download is starting
 5. Backup is transmitted on the fly from server to `gpg` -> our shell command
 6. Our shell `--cmd` / `-c` command is taking stdin and performing a restore action
 7. Feedback is returned
-8. Temporary `gpg` keyring is deleted
 
 ## Automated procedures
 
@@ -96,6 +92,7 @@ together with a tool that generates Backup & Restore procedures. Those procedure
 
 - Skip `--private-key` and `--passphrase` to disable GPG
 - Use `debug` log level to see GPG output and more verbose output at all
+- Increase encryption/decryption performance by disabling armoring
 
 
 ## Proposed usage

client/download_test.go

@@ -90,14 +90,11 @@ func TestDownload_SuccessWithValidGPG(t *testing.T) {
 
 	ctx := createExampleContext()
 	ctx.ActionType = "download"
-	ctx.Gpg = context.GPGOperationContext{
+	ctx.Crypto = context.EncryptionOperationContext{
 		PrivateKeyPath: "../resources/test/gpg-key.asc",
 		Passphrase:     "riotkit",
-		Recipient:      "test@riotkit.org",
+		EncType:        "gpg-armored",
 	}
-	initErr := context.InitializeGPGContext(&ctx)
-	assert.Nil(t, initErr)
-	defer ctx.Gpg.CleanUp()
 
 	_ = DownloadBackupIntoProcessStdin(ctx, "cat - > ../.build/TestDownload_SuccessWithValidGPG", client)
 

client/upload.go

@@ -115,7 +115,7 @@ func UploadFromCommandOutput(app actionCtx.Action, client HTTPClient) error {
 		return pipeErr
 	}
 
-	log.Print("Starting cmd.Run()")
+	log.Print("Starting cmd.Encrypt()")
 	execErr := cmd.Start()
 	if execErr != nil {
 		log.Println("Cannot start backup process ", execErr)
@@ -127,6 +127,7 @@ func UploadFromCommandOutput(app actionCtx.Action, client HTTPClient) error {
 	log.Printf("Starting Upload() for PID=%v", cmd.Process.Pid)
 	status, out, uploadErr := Upload(ctx, client, app.Url, app.CollectionId, app.AuthToken, ReadCloserWithCancellationWhenProcessFails{stdout, cmd, cancel}, app.Timeout)
 	if uploadErr != nil {
+		cancel()
 		log.Errorf("Status: %v, Out: %v, Err: %v", status, out, uploadErr)
 		return uploadErr
 	} else {

client/upload_test.go

@@ -25,7 +25,7 @@ func TestUploadFromCommandOutput_PassesOnValidResponse(t *testing.T) {
 		ActionType:       "make",
 		VersionToRestore: "",
 		DownloadPath:     "",
-		Gpg:              context.GPGOperationContext{},
+		Crypto:           context.EncryptionOperationContext{},
 	}, http)
 	assert.Nil(t, err)
 }
@@ -47,7 +47,7 @@ func TestUploadFromCommandOutput_FailOnInvalidResponse(t *testing.T) {
 		ActionType:       "make",
 		VersionToRestore: "",
 		DownloadPath:     "",
-		Gpg:              context.GPGOperationContext{},
+		Crypto:           context.EncryptionOperationContext{},
 	}, http)
 	assert.NotNil(t, err)
 }

cmd/backupmaker/main.go

@@ -21,7 +21,7 @@ func addAddGenericFlags(cmd *cobra.Command, ctx *context.Action) {
 	cmd.Flags().StringVarP(&ctx.AuthToken, "auth-token", "t", os.Getenv("BM_AUTH_TOKEN"), "Access token that allows to upload at least one file successfully, [environment variable: BM_AUTH_TOKEN]")
 	cmd.Flags().Int64VarP(&ctx.Timeout, "timeout", "", 60*20, "Connection and read timeout in summary [environment variable: BM_TIMEOUT]")
 	cmd.Flags().StringVarP(&ctx.LogLevelStr, "log-level", "", "info", "Verbosity level: panic|fatal|error|warn|info|debug|trace")
-	cmd.Flags().BoolVarP(&ctx.Gpg.ShouldShowOutput, "verbose", "", false, "Increase verbosity")
+	cmd.Flags().StringVarP(&ctx.Crypto.EncType, "encryption-type", "", getEnvOrDefault("BM_ENCRYPTION_TYPE", "gpg-armored"), "Encryption type (options: gpg-armored, gpg-binary) [environment variable: BM_ENCRYPTION_TYPE]")
 }
 
 func createMakeCommand() *cobra.Command {
@@ -33,23 +33,19 @@ func createMakeCommand() *cobra.Command {
 		Short:        "Upload a backup to remote",
 		SilenceUsage: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			defer ctx.Gpg.CleanUp()
 			if err := context.InitializeLogLevel(&ctx); err != nil {
 				return err
 			}
-			if err := context.InitializeGPGContext(&ctx); err != nil {
-				return err
-			}
 			if err := client.UploadFromCommandOutput(ctx, client.CreateHttpClient()); err != nil {
 				return err
 			}
 			return nil
 		},
 	}
-	makeCmd.Flags().StringVarP(&ctx.Gpg.PublicKeyPath, "key", "k", os.Getenv("BM_PUBLIC_KEY_PATH"), "GPG public or private key (required if using GPG) [environment variable: BM_PUBLIC_KEY_PATH]")
-	makeCmd.Flags().StringVarP(&ctx.Gpg.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]")
+	makeCmd.Flags().StringVarP(&ctx.Crypto.PublicKeyPath, "key", "k", os.Getenv("BM_PUBLIC_KEY_PATH"), "GPG public or private key (required if using GPG) [environment variable: BM_PUBLIC_KEY_PATH]")
 	makeCmd.Flags().StringVarP(&ctx.Command, "cmd", "c", os.Getenv("BM_CMD"), "Command to execute, which output will be captured and sent to server [environment variable: BM_CMD]")
-	makeCmd.Flags().StringVarP(&ctx.Gpg.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]")
+	makeCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]")
+
 	addAddGenericFlags(&makeCmd, &ctx)
 
 	return &makeCmd
@@ -64,25 +60,21 @@ func createRestoreCommand() *cobra.Command {
 		SilenceUsage: true,
 		Short:        "Restore a backup from remote to local target",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			defer ctx.Gpg.CleanUp()
 			if err := context.InitializeLogLevel(&ctx); err != nil {
 				return err
 			}
-			if err := context.InitializeGPGContext(&ctx); err != nil {
-				return err
-			}
 			if err := client.DownloadBackupIntoProcessStdin(ctx, ctx.Command, client.CreateHttpClient()); err != nil {
 				return err
 			}
 			return nil
 		},
 	}
 
-	restoreCmd.Flags().StringVarP(&ctx.Gpg.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]")
+	restoreCmd.Flags().StringVarP(&ctx.Crypto.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]")
 	restoreCmd.Flags().StringVarP(&ctx.Command, "cmd", "c", os.Getenv("BM_CMD"), "Command which should take downloaded file as stdin stream e.g. some tar, unzip, psql [environment variable: BM_CMD]")
-	restoreCmd.Flags().StringVarP(&ctx.Gpg.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]")
+	restoreCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]")
 	restoreCmd.Flags().StringVarP(&ctx.VersionToRestore, "version", "s", getEnvOrDefault("BM_VERSION", "latest"), "Version number [environment variable: BM_VERSION]")
-	restoreCmd.Flags().StringVarP(&ctx.Gpg.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]")
+
 	addAddGenericFlags(&restoreCmd, &ctx)
 	return &restoreCmd
 }
@@ -96,25 +88,21 @@ func createDownloadCommand() *cobra.Command {
 		SilenceUsage: true,
 		Short:        "Download a remote backup and print into a local file",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			defer ctx.Gpg.CleanUp()
 			if err := context.InitializeLogLevel(&ctx); err != nil {
 				return err
 			}
-			if err := context.InitializeGPGContext(&ctx); err != nil {
-				return err
-			}
 			if err := client.DownloadIntoFile(ctx, ctx.DownloadPath, client.CreateHttpClient()); err != nil {
 				return err
 			}
 			return nil
 		},
 	}
 
-	downloadCmd.Flags().StringVarP(&ctx.Gpg.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]")
+	downloadCmd.Flags().StringVarP(&ctx.Crypto.PrivateKeyPath, "private-key", "p", os.Getenv("BM_PRIVATE_KEY_PATH"), "GPG private key. [environment variable: BM_PRIVATE_KEY_PATH]")
 	downloadCmd.Flags().StringVarP(&ctx.Command, "save-path", "", os.Getenv("BM_SAVE_PATH"), "Place where to save file instead of executing a restore command [environment variable: BM_SAVE_PATH]")
-	downloadCmd.Flags().StringVarP(&ctx.Gpg.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]")
+	downloadCmd.Flags().StringVarP(&ctx.Crypto.Passphrase, "passphrase", "", os.Getenv("BM_PASSPHRASE"), "Secret passphrase for GPG [environment variable: BM_PASSPHRASE]")
 	downloadCmd.Flags().StringVarP(&ctx.VersionToRestore, "version", "s", getEnvOrDefault("BM_VERSION", "latest"), "Version number [environment variable: BM_VERSION]")
-	downloadCmd.Flags().StringVarP(&ctx.Gpg.Recipient, "recipient", "r", os.Getenv("BM_RECIPIENT"), "GPG recipient e-mail (required if using GPG). By default this e-mail SHOULD BE same as e-mail used when restoring/downloading backup [environment variable: BM_RECIPIENT]")
+
 	addAddGenericFlags(&downloadCmd, &ctx)
 	return &downloadCmd
 }

cmd/encryption/main.go

@@ -0,0 +1,90 @@
+package encryption
+
+import (
+	"github.com/pkg/errors"
+	"github.com/riotkit-org/br-backup-maker/crypto"
+	"github.com/spf13/cobra"
+)
+
+func NewEncryptionCommand() *cobra.Command {
+	app := &App{}
+
+	command := &cobra.Command{
+		Use:          "encrypt",
+		SilenceUsage: true,
+		Short:        "Encrypts a stdin and outputs as stdout",
+		RunE: func(command *cobra.Command, args []string) error {
+			return app.Encrypt()
+		},
+	}
+
+	command.Flags().StringVarP(&app.keyPath, "key-path", "k", "", "Path to the key file")
+	command.Flags().StringVarP(&app.encType, "type", "t", "gpg-armored", "Encryption type (options: gpg-armored, gpg-binary)")
+	command.Flags().StringVarP(&app.passphrase, "passphrase", "p", "", "(Optional) passphrase to decrypt the key")
+
+	return command
+}
+
+func NewDecryptionCommand() *cobra.Command {
+	app := &App{}
+
+	command := &cobra.Command{
+		Use:          "decrypt",
+		SilenceUsage: true,
+		Short:        "Decrypts a stdin and outputs as stdout",
+		RunE: func(command *cobra.Command, args []string) error {
+			return app.Decrypt()
+		},
+	}
+
+	command.Flags().StringVarP(&app.keyPath, "key-path", "k", "", "Path to the key file")
+	command.Flags().StringVarP(&app.encType, "type", "t", "gpg-armored", "Encryption type (options: gpg-armored, gpg-binary)")
+	command.Flags().StringVarP(&app.passphrase, "passphrase", "p", "", "(Optional) passphrase to decrypt the key")
+
+	return command
+}
+
+func NewCryptoCommand() *cobra.Command {
+	command := &cobra.Command{
+		Use:          "crypto",
+		SilenceUsage: true,
+		Short:        "Decrypts a stdin and outputs as stdout",
+		RunE: func(command *cobra.Command, args []string) error {
+			return command.Help()
+		},
+	}
+	command.AddCommand(NewEncryptionCommand())
+	command.AddCommand(NewDecryptionCommand())
+	return command
+}
+
+type App struct {
+	keyPath    string
+	encType    string
+	passphrase string
+}
+
+func (encrypt *App) createAlgo() (crypto.Service, error) {
+	if encrypt.encType == "gpg-armored" {
+		return crypto.GPGEncryption{Armored: true}, nil
+	} else if encrypt.encType == "gpg-binary" {
+		return crypto.GPGEncryption{Armored: false}, nil
+	}
+	return nil, errors.New("unsupported encryption type")
+}
+
+func (encrypt *App) Encrypt() error {
+	algo, err := encrypt.createAlgo()
+	if err != nil {
+		return err
+	}
+	return algo.Encrypt(encrypt.keyPath, encrypt.passphrase)
+}
+
+func (encrypt *App) Decrypt() error {
+	algo, err := encrypt.createAlgo()
+	if err != nil {
+		return err
+	}
+	return algo.Decrypt(encrypt.keyPath, encrypt.passphrase)
+}

cmd/root.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"github.com/riotkit-org/br-backup-maker/cmd/backupmaker"
 	"github.com/riotkit-org/br-backup-maker/cmd/bmg"
+	"github.com/riotkit-org/br-backup-maker/cmd/encryption"
 	"github.com/spf13/cobra"
 )
 
@@ -18,5 +19,7 @@ func GetRootCommand() *cobra.Command {
 		cmd.AddCommand(subCmd)
 	}
 	cmd.AddCommand(bmg.CreateCommand())
+	cmd.AddCommand(encryption.NewCryptoCommand())
+
 	return cmd
 }

context/action.go

@@ -13,7 +13,7 @@ type Action struct {
 	VersionToRestore string
 	DownloadPath     string
 
-	Gpg         GPGOperationContext
+	Crypto      EncryptionOperationContext
 	LogLevel    uint32
 	LogLevelStr string // todo: convert to LogLevel
 	//logLevel, _ := log.ParseLevel(*logLevelStr)
@@ -30,21 +30,21 @@ func (that Action) CreateWrappedCommand(custom string) string {
 		cmd = custom
 	}
 
-	if !that.Gpg.Enabled(that.ActionType) {
+	if !that.Crypto.Enabled(that.ActionType) {
 		return cmd
 	}
 
 	if that.ActionType == "make" {
-		return cmd + " | " + that.Gpg.GetEncryptionCommand()
+		return cmd + " | " + that.Crypto.GetEncryptionCommand()
 	}
 
-	return that.Gpg.GetDecryptionCommand() + " | " + cmd
+	return that.Crypto.GetDecryptionCommand() + " | " + cmd
 }
 
 // GetPrintableCommand returns same command as in CreateWrappedCommand(), but with erased credentials
 // so the command could be logged or printed into the console
 func (that Action) GetPrintableCommand(custom string) string {
-	return strings.ReplaceAll(that.CreateWrappedCommand(custom), that.Gpg.Passphrase, "***")
+	return strings.ReplaceAll(that.CreateWrappedCommand(custom), that.Crypto.Passphrase, "***")
 }
 
 func (that Action) ShouldShowCommandsOutput() bool {