Feat: Add velero to EC (#530)

laverya · web-flow · commit fbd4f865f52f · 2024-04-24T21:42:53.000+09:00
* update embedded-cluster-kinds

* add default velero, include in metadata

* install velero when it is enabled in both the cluster spec and license

* add velero to autoupdate cron

* deploy velero in an integration test

* enable snapshots in config

* handle empty license file path

* typo and rearranging functions

* do not use whether velero is enabled

* update embedded-cluster-operator to handle velero

* print charts

* ???

* disable backups/snapshots

* enable node agent

* set k0s pod volume path

* actually wait for velero

* test logging

* specify latest velero image outside of chart release

* reduce CPU and memory requirements for nginx pod

* reduce memcached resources

* actually upgrade application

* cleanup and version test

* wait for restic daemonset too

* add velero plugin for aws

* ?

* nope

* ???

* test

* remove?

* put the repo removal in the right action

* fix daemonset name

* set uploader type to restic
diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml
@@ -7,6 +7,9 @@ inputs:
   airgap-license-id:
     description: 'airgap-enabled license id to use for e2e tests'
     required: true
+  snapshot-license-id:
+    description: 'snapshot-enabled license id to use for e2e tests'
+    required: true
   license-id:
     description: 'license id to use for e2e tests'
     required: true
@@ -69,6 +72,7 @@ runs:
       export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
       export LICENSE_ID=${{ inputs.license-id }}
       export AIRGAP_LICENSE_ID=${{ inputs.airgap-license-id }}
+      export SNAPSHOT_LICENSE_ID=${{ inputs.snapshot-license-id }}
       echo "${{ inputs.license }}" | base64 --decode > e2e/license.yaml
       export TESTIM_ACCESS_TOKEN=${{ inputs.testim-access-token }}
       export TESTIM_BRANCH=${{ inputs.testim-branch }}
diff --git a/.github/workflows/dependencies.yaml b/.github/workflows/dependencies.yaml
@@ -47,6 +47,18 @@ jobs:
       run: |
         export VERSION=`curl https://api.github.com/repos/distribution/distribution/tags | jq -r '.[].name'  | tr -d v | grep -v alpha | grep -v beta | head -n 1`
         sed -i "/^REGISTRY_IMAGE_VERSION/c\REGISTRY_IMAGE_VERSION = $VERSION" Makefile
+    - name: Velero Chart
+      run: |
+        export VERSION=`curl https://api.github.com/repos/vmware-tanzu/helm-charts/tags | jq -r '.[].name' | grep velero | tr -d 'velero-' | head -n 1`
+        sed -i "/^VELERO_CHART_VERSION/c\VELERO_CHART_VERSION = $VERSION" Makefile
+    - name: Velero Image
+      run: |
+        export VERSION=`curl https://api.github.com/repos/vmware-tanzu/velero/tags | jq -r '.[].name' | grep -v 'rc' | head -n 1`
+        sed -i "/^VELERO_IMAGE_VERSION/c\VELERO_IMAGE_VERSION = $VERSION" Makefile
+    - name: Velero AWS Plugin Image
+      run: |
+        export VERSION=`curl https://api.github.com/repos/vmware-tanzu/velero-plugin-for-aws/tags | jq -r '.[].name' | grep -v 'rc' | head -n 1`
+        sed -i "/^VELERO_AWS_PLUGIN_IMAGE_VERSION/c\VELERO_AWS_PLUGIN_IMAGE_VERSION = $VERSION" Makefile
     - name: Kubectl
       run: |
         export VERSION=`curl -L -s https://dl.k8s.io/release/stable.txt`
diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
@@ -184,6 +184,7 @@ jobs:
           - TestMaterialize
           - TestLocalArtifactMirror
           - TestSingleNodeAirgapUpgradeUbuntuJammy
+          - TestInstallSnapshotFromReplicatedApp
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -192,10 +193,12 @@ jobs:
       with:
         name: embedded-release
         path: output/bin
+
     - uses: ./.github/actions/e2e
       with:
         test-name: '${{ matrix.tests }}'
         airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
+        snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
         license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
         license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}
         testim-access-token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
@@ -137,6 +137,7 @@ jobs:
           - TestMaterialize
           - TestLocalArtifactMirror
           - TestSingleNodeAirgapUpgradeUbuntuJammy
+          - TestInstallSnapshotFromReplicatedApp
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -149,6 +150,7 @@ jobs:
         with:
           test-name: '${{ matrix.tests }}'
           airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
+          snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
           license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
           license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}
           testim-access-token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
diff --git a/Makefile b/Makefile
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.28.1
+EMBEDDED_OPERATOR_CHART_VERSION = 0.28.2
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_URL = https://openebs.github.io/charts
@@ -20,6 +20,11 @@ REGISTRY_CHART_URL = https://helm.twun.io
 REGISTRY_CHART_NAME = twuni/docker-registry
 REGISTRY_CHART_VERSION = 2.2.3
 REGISTRY_IMAGE_VERSION = 2.8.3
+VELERO_CHART_URL = https://vmware-tanzu.github.io/helm-charts
+VELERO_CHART_NAME = vmware-tanzu/velero
+VELERO_CHART_VERSION = 6.0.0
+VELERO_IMAGE_VERSION = v1.13.2
+VELERO_AWS_PLUGIN_IMAGE_VERSION = v1.9.2
 KUBECTL_VERSION = v1.30.0
 K0S_VERSION = v1.29.4+k0s.0
 PREVIOUS_K0S_VERSION ?= v1.28.9+k0s.0
@@ -48,7 +53,12 @@ LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/registry.ChartURL=$(REGISTRY_CHART_URL) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/registry.ChartName=$(REGISTRY_CHART_NAME) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/registry.Version=$(REGISTRY_CHART_VERSION) \
-	-X github.com/replicatedhq/embedded-cluster/pkg/addons/registry.ImageVersion=$(REGISTRY_IMAGE_VERSION)
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/registry.ImageVersion=$(REGISTRY_IMAGE_VERSION) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/velero.ChartURL=$(VELERO_CHART_URL) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/velero.ChartName=$(VELERO_CHART_NAME) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/velero.Version=$(VELERO_CHART_VERSION) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/velero.VeleroTag=$(VELERO_IMAGE_VERSION) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/velero.AwsPluginTag=$(VELERO_AWS_PLUGIN_IMAGE_VERSION)
 
 .DEFAULT_GOAL := default
 default: embedded-cluster-linux-amd64
diff --git a/cmd/embedded-cluster/version.go b/cmd/embedded-cluster/version.go
@@ -144,6 +144,13 @@ func gatherVersionMetadata() (*types.ReleaseMetadata, error) {
 		return nil, fmt.Errorf("unable to get airgap images: %w", err)
 	}
 
+	// Additional builtin addons
+	builtinCharts, err := applier.GetBuiltinCharts()
+	if err != nil {
+		return nil, fmt.Errorf("unable to get builtin charts: %w", err)
+	}
+	meta.BuiltinConfigs = builtinCharts
+
 	// Render k0s config to get the images contained within
 	k0sConfig := config.RenderK0sConfig()
 	meta.K0sImages = airgap.GetImageURIs(k0sConfig.Spec, true)
diff --git a/e2e/install_test.go b/e2e/install_test.go
@@ -616,6 +616,53 @@ func TestSingleNodeAirgapUpgradeUbuntuJammy(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
+func TestInstallSnapshotFromReplicatedApp(t *testing.T) {
+	t.Parallel()
+	tc := cluster.NewTestCluster(&cluster.Input{
+		T:     t,
+		Nodes: 1,
+		Image: "ubuntu/jammy",
+	})
+	defer tc.Destroy()
+	t.Logf("%s: downloading embedded-cluster on node 0", time.Now().Format(time.RFC3339))
+	line := []string{"vandoor-prepare.sh", os.Getenv("SHORT_SHA"), os.Getenv("SNAPSHOT_LICENSE_ID"), "false"}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to download embedded-cluster on node 0 %s: %v", tc.Nodes[0], err)
+	}
+
+	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
+	line = []string{"single-node-install.sh", "cli"}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
+	}
+
+	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
+	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v", err)
+	}
+
+	t.Logf("%s: ensuring velero is installed", time.Now().Format(time.RFC3339))
+	line = []string{"check-velero-state.sh", os.Getenv("SHORT_SHA")}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to check velero state: %v", err)
+	}
+
+	t.Logf("%s: running kots upstream upgrade", time.Now().Format(time.RFC3339))
+	line = []string{"kots-upstream-upgrade.sh", os.Getenv("SHORT_SHA")}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to run kots upstream upgrade: %v", err)
+	}
+
+	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to check postupgrade state: %v", err)
+	}
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
+
 func downloadAirgapBundle(t *testing.T, versionLabel string, destPath string) string {
 	// download airgap bundle
 	airgapURL := fmt.Sprintf("https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci-airgap/%s?airgap=true", versionLabel)
diff --git a/e2e/kots-release-install/deployment.yaml b/e2e/kots-release-install/deployment.yaml
@@ -22,8 +22,8 @@ spec:
           image: nginx:1.24-alpine
           resources:
             limits:
-              memory: '256Mi'
-              cpu: '500m'
+              memory: '64Mi'
+              cpu: '50m'
           env:
             - name: APP_SEQUENCE
               value: "{{repl Cursor }}"
diff --git a/e2e/kots-release-upgrade/cluster-config.yaml b/e2e/kots-release-upgrade/cluster-config.yaml
@@ -67,6 +67,14 @@ spec:
         - chartname: oci://registry-1.docker.io/bitnamicharts/memcached
           name: memcached
           namespace: memcached
-          values: "abc: xyz"
+          values: |
+            volumePermissions:
+             resources:
+               requests:
+                 cpu: 25m
+                 memory: 128Mi
+               limits:
+                 cpu: 25m
+                 memory: 256Mi
           order: 4
           version: 6.6.4
diff --git a/e2e/kots-release-upgrade/deployment.yaml b/e2e/kots-release-upgrade/deployment.yaml
@@ -22,8 +22,8 @@ spec:
           image: nginx:1.25-alpine
           resources:
             limits:
-              memory: '256Mi'
-              cpu: '500m'
+              memory: '64Mi'
+              cpu: '50m'
           env:
             - name: APP_SEQUENCE
               value: "{{repl Cursor }}"
diff --git a/e2e/scripts/check-velero-state.sh b/e2e/scripts/check-velero-state.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euox pipefail
+
+wait_for_velero_pods() {
+    ready=$(kubectl get pods -n velero -o jsonpath='{.items[*].metadata.name} {.items[*].status.phase}' | grep "velero" | grep -c Running || true)
+    counter=0
+    while [ "$ready" -lt "1" ]; do
+        if [ "$counter" -gt 36 ]; then
+            echo "velero pods did not appear"
+            kubectl get pods -n velero -o jsonpath='{.items[*].metadata.name} {.items[*].status.phase}'
+            kubectl get pods -n velero
+            return 1
+        fi
+        sleep 5
+        counter=$((counter+1))
+        echo "Waiting for velero pods"
+        ready=$(kubectl get pods -n velero -o jsonpath='{.items[*].metadata.name} {.items[*].status.phase}' | grep "velero" | grep -c Running || true)
+        kubectl get pods -n velero 2>&1 || true
+        echo "ready: $ready"
+    done
+}
+
+main() {
+    sleep 50
+
+    kubectl get pods -A
+    kubectl get installations -o yaml
+    kubectl get charts -A
+
+    if ! wait_for_velero_pods; then
+        echo "Failed waiting for velero"
+        exit 1
+    fi
+}
+
+export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
+export KUBECONFIG=/var/lib/k0s/pki/admin.conf
+export PATH=$PATH:/var/lib/embedded-cluster/bin
+main "$@"
diff --git a/e2e/version_test.go b/e2e/version_test.go
@@ -131,6 +131,26 @@ func TestVersion(t *testing.T) {
 		}
 	}
 
+	expectedVeleroCharts := []string{"velero"}
+	if len(parsed.BuiltinConfigs["velero"].Charts) != len(expectedVeleroCharts) {
+		t.Log(output)
+		t.Fatalf("found %d velero charts in metadata, expected %d", len(parsed.BuiltinConfigs["velero"].Charts), len(expectedVeleroCharts))
+	}
+
+	for _, expectedName := range expectedVeleroCharts {
+		foundName := false
+		for _, foundChart := range parsed.BuiltinConfigs["velero"].Charts {
+			if foundChart.Name == expectedName {
+				foundName = true
+				break
+			}
+		}
+		if !foundName {
+			t.Errorf("failed to find velero chart %s in 'metadata' output", expectedName)
+			failed = true
+		}
+	}
+
 	if failed {
 		t.Log(output)
 		t.FailNow()
diff --git a/pkg/addons/applier.go b/pkg/addons/applier.go
@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1"
 	"github.com/replicatedhq/embedded-cluster-kinds/types"
 	"github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
@@ -19,7 +20,9 @@ import (
 	"github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons/openebs"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons/registry"
+	"github.com/replicatedhq/embedded-cluster/pkg/addons/velero"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
 	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/spinner"
 )
@@ -104,6 +107,26 @@ func (a *Applier) GetAirgapCharts() ([]v1beta1.Chart, []v1beta1.Repository, erro
 	return regChart, regRepo, nil
 }
 
+func (a *Applier) GetBuiltinCharts() (map[string]k0sconfig.HelmExtensions, error) {
+	builtinCharts := map[string]k0sconfig.HelmExtensions{}
+
+	vel, err := velero.New(true)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create velero addon: %w", err)
+	}
+	velChart, velRepo, err := vel.GenerateHelmConfig(true)
+	if err != nil {
+		return nil, fmt.Errorf("unable to generate helm config for velero: %w", err)
+	}
+
+	builtinCharts["velero"] = k0sconfig.HelmExtensions{
+		Repositories: velRepo,
+		Charts:       velChart,
+	}
+
+	return builtinCharts, nil
+}
+
 func (a *Applier) GetAdditionalImages() ([]string, error) {
 	additionalImages := []string{}
 	addons, err := a.load()
@@ -180,6 +203,16 @@ func (a *Applier) load() ([]AddOn, error) {
 	}
 	addons = append(addons, embedoperator)
 
+	snapshotsEnabled, err := helpers.SnapshotsEnabled(a.licenseFile)
+	if err != nil {
+		return nil, fmt.Errorf("unable to check if snapshots are enabled: %w", err)
+	}
+	vel, err := velero.New(snapshotsEnabled)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create velero addon: %w", err)
+	}
+	addons = append(addons, vel)
+
 	aconsole, err := adminconsole.New(defaults.KotsadmNamespace, a.prompt, a.config, a.licenseFile, a.airgapBundle)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create admin console addon: %w", err)
diff --git a/pkg/addons/velero/velero.go b/pkg/addons/velero/velero.go
diff --git a/pkg/helpers/snapshots.go b/pkg/helpers/snapshots.go
diff --git a/pkg/kubeutils/kubeutils.go b/pkg/kubeutils/kubeutils.go
