Merge pull request #111 from jkaninda/multi-backup

Add Multi database backup

Author: jkaninda

.github/workflows/build.yml

@@ -25,7 +25,7 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           push: true
-          file: "./docker/Dockerfile"
+          file: "./Dockerfile"
           platforms: linux/amd64,linux/arm64,linux/arm/v7
           build-args: |
             appVersion=develop-${{ github.sha }}

.github/workflows/release.yml

@@ -39,7 +39,7 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           push: true
-          file: "./docker/Dockerfile"
+          file: "./Dockerfile"
           platforms: linux/amd64,linux/arm64,linux/arm/v7
           build-args: |
             appVersion=${{ env.TAG_NAME }}

docker/Dockerfile renamed to Dockerfile

@@ -0,0 +1,6 @@
+---
+title: Update deprecated configurations
+layout: default
+parent: How Tos
+nav_order: 11
+---

docs/how-tos/deprecated-configs.md

@@ -0,0 +1,40 @@
+---
+title: Run multiple backup schedules in the same container
+layout: default
+parent: How Tos
+nav_order: 11
+---
+
+Multiple backup schedules with different configuration can be configured by mounting a configuration file into `/config/config.yaml`  `/config/config.yml` or by defining an environment variable `BACKUP_CONFIG_FILE=/backup/config.yaml`.
+
+## Configuration file
+
+```yaml
+#cronExpression: "@every 20m" //Optional for scheduled backups
+cronExpression: "" 
+databases:
+  - host: mysql1
+    port: 3306
+    name: database1
+    user: database1
+    password: password
+    path: /s3-path/database1 #For SSH or FTP you need to define the full path (/home/toto/backup/)
+  - host: mysql2
+    port: 3306
+    name: lldap
+    user: lldap
+    password: password
+    path: /s3-path/lldap #For SSH or FTP you need to define the full path (/home/toto/backup/)
+  - host: mysql3
+    port: 3306
+    name: keycloak
+    user: keycloak
+    password: password
+    path: /s3-path/keycloak #For SSH or FTP you need to define the full path (/home/toto/backup/)
+  - host: mysql4
+    port: 3306
+    name: joplin
+    user: joplin
+    password: password
+    path: /s3-path/joplin #For SSH or FTP you need to define the full path (/home/toto/backup/)
+```

docs/how-tos/mutli-backup.md

@@ -13,6 +13,7 @@ require (
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/spf13/cobra v1.8.0
 	golang.org/x/crypto v0.18.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (

go.mod

@@ -101,4 +101,5 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

go.sum

@@ -20,18 +20,23 @@ import (
 
 func StartBackup(cmd *cobra.Command) {
 	intro()
-	dbConf = initDbConfig(cmd)
 	//Initialize backup configs
 	config := initBackupConfig(cmd)
-
-	if config.cronExpression == "" {
-		BackupTask(dbConf, config)
-	} else {
-		if utils.IsValidCronExpression(config.cronExpression) {
-			scheduledMode(dbConf, config)
+	//Load backup configuration file
+	configFile, err := loadConfigFile()
+	if err != nil {
+		dbConf = initDbConfig(cmd)
+		if config.cronExpression == "" {
+			BackupTask(dbConf, config)
 		} else {
-			utils.Fatal("Cron expression is not valid: %s", config.cronExpression)
+			if utils.IsValidCronExpression(config.cronExpression) {
+				scheduledMode(dbConf, config)
+			} else {
+				utils.Fatal("Cron expression is not valid: %s", config.cronExpression)
+			}
 		}
+	} else {
+		startMultiBackup(config, configFile)
 	}
 
 }
@@ -85,6 +90,64 @@ func BackupTask(db *dbConfig, config *BackupConfig) {
 		localBackup(db, config)
 	}
 }
+func multiBackupTask(databases []Database, bkConfig *BackupConfig) {
+	for _, db := range databases {
+		//Check if path is defined in config file
+		if db.Path != "" {
+			bkConfig.remotePath = db.Path
+		}
+		BackupTask(getDatabase(db), bkConfig)
+	}
+}
+func startMultiBackup(bkConfig *BackupConfig, configFile string) {
+	utils.Info("Starting multiple backup jobs...")
+	var conf = &Config{}
+	conf, err := readConf(configFile)
+	if err != nil {
+		utils.Fatal("Error reading config file: %s", err)
+	}
+	//Check if cronExpression is defined in config file
+	if conf.CronExpression != "" {
+		bkConfig.cronExpression = conf.CronExpression
+	}
+	// Check if cronExpression is defined
+	if bkConfig.cronExpression == "" {
+		multiBackupTask(conf.Databases, bkConfig)
+	} else {
+		// Check if cronExpression is valid
+		if utils.IsValidCronExpression(bkConfig.cronExpression) {
+			utils.Info("Running MultiBackup in Scheduled mode")
+			utils.Info("Backup cron expression:  %s", bkConfig.cronExpression)
+			utils.Info("Storage type %s ", bkConfig.storage)
+
+			//Test backup
+			utils.Info("Testing backup configurations...")
+			multiBackupTask(conf.Databases, bkConfig)
+			utils.Info("Testing backup configurations...done")
+			utils.Info("Creating multi backup job...")
+			// Create a new cron instance
+			c := cron.New()
+
+			_, err := c.AddFunc(bkConfig.cronExpression, func() {
+				// Create a channel
+				multiBackupTask(conf.Databases, bkConfig)
+			})
+			if err != nil {
+				return
+			}
+			// Start the cron scheduler
+			c.Start()
+			utils.Info("Creating multi backup job...done")
+			utils.Info("Backup job started")
+			defer c.Stop()
+			select {}
+
+		} else {
+			utils.Fatal("Cron expression is not valid: %s", bkConfig.cronExpression)
+		}
+	}
+
+}
 
 // BackupDatabase backup database
 func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool) {
@@ -93,7 +156,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool
 
 	utils.Info("Starting database backup...")
 
-	err := os.Setenv("PGPASSWORD", db.dbPassword)
+	err := os.Setenv("MYSQL_PWD", db.dbPassword)
 	if err != nil {
 		return
 	}
@@ -116,7 +179,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool
 		}
 
 		// save output
-		file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
+		file, err := os.Create(filepath.Join(tmpPath, backupFileName))
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -137,7 +200,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool
 		}
 		gzipCmd := exec.Command("gzip")
 		gzipCmd.Stdin = stdout
-		gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
+		gzipCmd.Stdout, err = os.Create(filepath.Join(tmpPath, backupFileName))
 		gzipCmd.Start()
 		if err != nil {
 			log.Fatal(err)
@@ -172,6 +235,7 @@ func localBackup(db *dbConfig, config *BackupConfig) {
 	}
 	//Delete temp
 	deleteTemp()
+	utils.Info("Backup completed successfully")
 }
 
 func s3Backup(db *dbConfig, config *BackupConfig) {
@@ -212,6 +276,8 @@ func s3Backup(db *dbConfig, config *BackupConfig) {
 	utils.NotifySuccess(finalFileName)
 	//Delete temp
 	deleteTemp()
+	utils.Info("Backup completed successfully")
+
 }
 func sshBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to Remote server")
@@ -247,6 +313,8 @@ func sshBackup(db *dbConfig, config *BackupConfig) {
 	utils.NotifySuccess(finalFileName)
 	//Delete temp
 	deleteTemp()
+	utils.Info("Backup completed successfully")
+
 }
 func ftpBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to the remote FTP server")
@@ -282,6 +350,8 @@ func ftpBackup(db *dbConfig, config *BackupConfig) {
 	utils.NotifySuccess(finalFileName)
 	//Delete temp
 	deleteTemp()
+	utils.Info("Backup completed successfully")
+
 }
 
 func encryptBackup(config *BackupConfig) {

pkg/backup.go

@@ -14,7 +14,17 @@ import (
 	"strconv"
 )
 
+type Database struct {
+	Host     string `yaml:"host"`
+	Port     string `yaml:"port"`
+	Name     string `yaml:"name"`
+	User     string `yaml:"user"`
+	Password string `yaml:"password"`
+	Path     string `yaml:"path"`
+}
 type Config struct {
+	Databases      []Database `yaml:"databases"`
+	CronExpression string     `yaml:"cronExpression"`
 }
 
 type dbConfig struct {
@@ -92,6 +102,16 @@ func initDbConfig(cmd *cobra.Command) *dbConfig {
 	return &dConf
 }
 
+func getDatabase(database Database) *dbConfig {
+	return &dbConfig{
+		dbHost:     database.Host,
+		dbPort:     database.Port,
+		dbName:     database.Name,
+		dbUserName: database.User,
+		dbPassword: database.Password,
+	}
+}
+
 // loadSSHConfig loads the SSH configuration from environment variables
 func loadSSHConfig() (*SSHConfig, error) {
 	utils.GetEnvVariable("SSH_HOST", "SSH_HOST_NAME")
@@ -245,3 +265,10 @@ func initTargetDbConfig() *targetDbConfig {
 	}
 	return &tdbConfig
 }
+func loadConfigFile() (string, error) {
+	backupConfigFile, err := checkConfigFile(os.Getenv("BACKUP_CONFIG_FILE"))
+	if err == nil {
+		return backupConfigFile, nil
+	}
+	return "", fmt.Errorf("backup config file not found")
+}

pkg/config.go

@@ -10,6 +10,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/jkaninda/mysql-bkup/utils"
+	"gopkg.in/yaml.v3"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -171,3 +172,42 @@ func checkPrKeyFile(prKey string) (string, error) {
 	// Return an error if neither file exists
 	return "", fmt.Errorf("no public key file found")
 }
+func readConf(configFile string) (*Config, error) {
+	//configFile := filepath.Join("./", filename)
+	if utils.FileExists(configFile) {
+		buf, err := os.ReadFile(configFile)
+		if err != nil {
+			return nil, err
+		}
+
+		c := &Config{}
+		err = yaml.Unmarshal(buf, c)
+		if err != nil {
+			return nil, fmt.Errorf("in file %q: %w", configFile, err)
+		}
+
+		return c, err
+	}
+	return nil, fmt.Errorf("config file %q not found", configFile)
+}
+func checkConfigFile(filePath string) (string, error) {
+	// Define possible config file names
+	configFiles := []string{filepath.Join(workingDir, "config.yaml"), filepath.Join(workingDir, "config.yml"), filePath}
+
+	// Loop through config file names and check if they exist
+	for _, configFile := range configFiles {
+		if _, err := os.Stat(configFile); err == nil {
+			// File exists
+			return configFile, nil
+		} else if os.IsNotExist(err) {
+			// File does not exist, continue to the next one
+			continue
+		} else {
+			// An unexpected error occurred
+			return "", err
+		}
+	}
+
+	// Return an error if neither file exists
+	return "", fmt.Errorf("no config file found")
+}

pkg/helper.go

@@ -7,7 +7,6 @@
 package pkg
 
 import (
-	"fmt"
 	"github.com/jkaninda/mysql-bkup/utils"
 	"github.com/spf13/cobra"
 	"os"
@@ -94,7 +93,7 @@ func RestoreDatabase(db *dbConfig, conf *RestoreConfig) {
 
 	}
 
-	if utils.FileExists(fmt.Sprintf("%s/%s", tmpPath, conf.file)) {
+	if utils.FileExists(filepath.Join(tmpPath, conf.file)) {
 		err := os.Setenv("MYSQL_PWD", db.dbPassword)
 		if err != nil {
 			return

pkg/restore.go

@@ -11,6 +11,7 @@ const tmpPath = "/tmp/backup"
 const algorithm = "aes256"
 const gpgHome = "/config/gnupg"
 const gpgExtension = "gpg"
+const workingDir = "/config"
 
 var (
 	storage            = "local"