Feat: support GCS bucket (#1)

* wip: write output state to gcs bucket

Signed-off-by: vietanhduong <anh.duong@kyber.network>

* supported gcs

Signed-off-by: vietanhduong <anh.duong@kyber.network>

* update README.md

Signed-off-by: vietanhduong <anh.duong@kyber.network>

---------

Signed-off-by: vietanhduong <anh.duong@kyber.network>

Author: Viet-Anh Duong

.gitignore

@@ -26,3 +26,5 @@ go.work
 .idea/
 .backup-state/
 *.state.json
+
+.vscode

README.md

@@ -3,3 +3,78 @@
 A tool support you pause/unpause your GCP resources which save your cost. This tool is designed for running on CI.
 
 Basically, Pause GCP rely-on **Gcloud CLI** that means, you must install the `gcloud` command first.
+
+Currently, we support `gke`, `vm` and `cloud sql` resources.
+
+## Usage
+
+### Pause a GKE cluster
+```console
+$ pause-gcp gke pause --help
+
+Pause a GKE cluster.
+This command require '--location' and '--project' flags.
+
+Usage:
+  pause-gcp gke pause [CLUSTER_NAME] [flags]
+
+Examples:
+# write output from stdout
+$ pause-gcp gke pause dev-cluster -l asia-southeast1 -p develop-project > output_state.json
+
+# write output to gcs bucket
+$ pause-gcp gke pause dev-cluster -l asia-southeast -p develop-project --output-dir=gs://bucket-name/gke-states
+
+# write output to a directory, pause-gcp will try to create the output dir if it not exists
+$ pause-gcp gke pause dev-cluster -p project --output-dir=output_states
+
+# pause cluster with some except pools
+$ pause-gcp gke pause dev-cluster -p project --except-pools=critical-pool
+
+
+Flags:
+      --except-pools strings                                      except node pools
+  -h, --help                                                      help for pause
+  -l, --location string                                           the cluster location (default "asia-southeast1")
+      --output-dir gke_<project>_<location>_<cluster_name>.json   the output directory to write the cluster state. If no path is specified, this will skip the write-to-file process. The output state file has named by format gke_<project>_<location>_<cluster_name>.json
+  -p, --project string                                            the project where contain the cluster
+```
+
+This command will print the privous state of the input cluster after it is paused. This state is used to recover the cluster in the unpause command.
+
+If the `--output-dir` is a GCS bucket (start with `gs://`), this tool will push the state file to the destination directly.
+
+### Unpause a GKE cluster
+
+```console
+$ pause-gcp gke unpause --help
+
+Unpause a GKE cluster.
+This command requires a GKE state file which is created when you pause the cluster.
+
+Usage:
+  pause-gcp gke unpause [STATE_FILE] [flags]
+
+Examples:
+
+# STATE_FILE from local
+$ pause-gcp gke unpause ./gke-states/gke_develop_asia-southeast1_dev-cluster.state.json
+
+# STATE_FILE from a gcs bucket
+$ pause-gcp gke unpause gs://bucket/path/json_file.json --rm
+
+
+Flags:
+  -h, --help   help for unpause
+      --rm     Remove the cluster state after complete
+```
+The input file must be the previos state of a cluster. The input file can be a GCS url (with `gs://` prefix).
+
+## FAQ
+
+### Why do I need this tool?
+This will save your money.
+
+### Can I use `gcloud` CLI instead?
+Yes, you can. This tool builds on top of `gcloud` CLI. If you only need to turn off a VM or a cloud SQL instance,
+you can use cloud CLI instead of this tool. But if you need to turn off a `GKE` cluster, I recommend you use `pause-gcp b ecause turning off a cluster is more complicated than a VM and cloud SQL.

cmd/gke/pause/cmd.go

@@ -16,6 +16,18 @@ func NewCommand() *cobra.Command {
 		Short: "Pause a GKE cluster",
 		Long: `Pause a GKE cluster.
 This command require '--location' and '--project' flags.`,
+		Example: `# write output from stdout
+$ pause-gcp gke pause dev-cluster -l asia-southeast1 -p develop-project > output_state.json
+
+# write output to gcs bucket
+$ pause-gcp gke pause dev-cluster -l asia-southeast -p develop-project --output-dir=gs://bucket-name/gke-states
+
+# write output to a directory, pause-gcp will try to create the output dir if it not exists
+$ pause-gcp gke pause dev-cluster -p project --output-dir=output_states
+
+# pause cluster with some except pools
+$ pause-gcp gke pause dev-cluster -p project --except-pools=critical-pool
+`,
 		Args: func(cmd *cobra.Command, args []string) error {
 			if len(args) != 1 || len(args[0]) == 0 {
 				return errors.Errorf("CLUSTER_NAME is required")

cmd/gke/pause/run.go

@@ -5,11 +5,12 @@ import (
 	"github.com/pkg/errors"
 	apis "github.com/vietanhduong/pause-gcp/apis/v1"
 	"github.com/vietanhduong/pause-gcp/pkg/gcloud/gke"
+	"github.com/vietanhduong/pause-gcp/pkg/utils/exec"
 	"github.com/vietanhduong/pause-gcp/pkg/utils/sets"
 	"google.golang.org/protobuf/encoding/protojson"
 	"log"
 	"os"
-	"path"
+	"strings"
 )
 
 type runConfig struct {
@@ -46,12 +47,18 @@ func run(cfg runConfig) error {
 	log.Printf("Recommend: please keep this information. You can use it to restore (unpause) your cluster.")
 
 	if cfg.outputDir != "" {
-		_ = os.MkdirAll(cfg.outputDir, 0755)
-		dst := path.Join(cfg.outputDir, fmt.Sprintf("gke_%s_%s_%s.state.json", cfg.project, cfg.location, cfg.clusterName))
-		if err = os.WriteFile(dst, b, 0644); err != nil {
-			return err
+		dst := fmt.Sprintf("%s/gke_%s_%s_%s.state.json", strings.TrimSuffix(cfg.outputDir, "/"), cfg.project, cfg.location, cfg.clusterName)
+		if strings.HasPrefix(strings.ToLower(cfg.outputDir), "gs://") {
+			_, err = exec.Run(exec.Command("bash", "-c", fmt.Sprintf(`echo '%s' | gsutil cp -L /dev/null - %s`, string(b), dst)))
+			if err != nil {
+				return err
+			}
+		} else {
+			_ = os.MkdirAll(cfg.outputDir, 0755)
+			if err = os.WriteFile(dst, b, 0644); err != nil {
+				return err
+			}
 		}
-
 		log.Printf("INFO: Cluster's state has been written to %q", dst)
 	}
 	log.Printf("INFO: cluster %q has been paused!", cluster.GetName())

cmd/gke/unpause/cmd.go

@@ -1,11 +1,15 @@
 package unpause
 
 import (
+	"log"
+	"os"
+	"strings"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	apis "github.com/vietanhduong/pause-gcp/apis/v1"
+	"github.com/vietanhduong/pause-gcp/pkg/utils/exec"
 	"github.com/vietanhduong/pause-gcp/pkg/utils/protoutil"
-	"os"
 )
 
 func NewCommand() *cobra.Command {
@@ -19,17 +23,36 @@ func NewCommand() *cobra.Command {
 		Short: "Unpause a GKE cluster",
 		Long: `Unpause a GKE cluster.
 This command requires a GKE state file which is created when you pause the cluster.`,
+		Example: `
+# STATE_FILE from local
+$ pause-gcp gke unpause ./gke-states/gke_develop_asia-southeast1_dev-cluster.state.json
+
+# STATE_FILE from a gcs bucket
+$ pause-gcp gke unpause gs://bucket/path/json_file.json --rm
+`,
 		Args: func(cmd *cobra.Command, args []string) error {
 			if len(args) != 1 {
 				return errors.Errorf("STATE_FILE is missing")
 			}
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			b, err := os.ReadFile(args[0])
-			if err != nil {
-				return err
+			var b []byte
+			var err error
+			// if the input path starts with 'gs://', that means the state is stored at GCS
+			// and we can use gsutil cat to retrieve the file content
+			if strings.HasPrefix(strings.ToLower(args[0]), "gs://") {
+				var raw string
+				if raw, err = exec.Run(exec.Command("gsutil", "cat", args[0])); err != nil {
+					return err
+				}
+				b = []byte(raw)
+			} else {
+				if b, err = os.ReadFile(args[0]); err != nil {
+					return err
+				}
 			}
+			log.Printf("INFO: retrieve cluster state completed!\n")
 			var cluster apis.Cluster
 			if err = protoutil.Unmarshal(b, &cluster); err != nil {
 				return err
@@ -38,8 +61,15 @@ This command requires a GKE state file which is created when you pause the clust
 			if err = run(runCfg); err != nil {
 				return err
 			}
+			// remote the input state file if the --rm flag exists
 			if rm {
-				return os.Remove(args[0])
+				// remove the state file by gsutil if the input path start with gs://
+				if strings.HasPrefix(strings.ToLower(args[0]), "gs://") {
+					_, err = exec.Run(exec.Command("gsutil", "rm", args[0]))
+					return err
+				} else {
+					return os.Remove(args[0])
+				}
 			}
 			return nil
 		},

cmd/main.go

@@ -1,14 +1,15 @@
 package main
 
 import (
+	"log"
+	"os"
+
 	"github.com/spf13/cobra"
 	"github.com/vietanhduong/pause-gcp/cmd/gke"
 	"github.com/vietanhduong/pause-gcp/cmd/sql"
 	"github.com/vietanhduong/pause-gcp/cmd/version"
 	"github.com/vietanhduong/pause-gcp/cmd/vm"
 	"github.com/vietanhduong/pause-gcp/pkg/utils/exec"
-	"log"
-	"os"
 )
 
 func newRootCmd() *cobra.Command {
@@ -30,6 +31,7 @@ func main() {
 		os.Exit(1)
 	}
 }
+
 func testGcloud() error {
 	_, err := exec.Run(exec.Command("which", "gcloud"))
 	return err