Add support for HA restores (#672)

* Add support for HA restores

Author: sgalsaleh

.github/actions/e2e/action.yml

@@ -4,6 +4,9 @@ inputs:
   test-name:
     description: 'individual test to run'
     required: true
+  is-large-runner:
+    description: 'Whether the test is running on a large runner'
+    required: true
   airgap-license-id:
     description: 'airgap-enabled license id to use for e2e tests'
     required: true
@@ -61,6 +64,7 @@ runs:
            external_ids:ovn-encap-type=geneve                      \
            external_ids:ovn-encap-ip=127.0.0.1
   - name: Free up runner disk space
+    if: ${{ inputs.is-large-runner == 'false' }}
     shell: bash
     run: |
       df -h

.github/workflows/pull-request.yaml

@@ -194,11 +194,15 @@ jobs:
           - TestMultiNodeHAInstallation
           - TestMultiNodeAirgapHAInstallation
           - TestMultiNodeAirgapUpgradeSameK0s
+          - TestMultiNodeHADisasterRecovery
+          - TestMultiNodeAirgapHADisasterRecovery
         include:
           - test: TestMultiNodeAirgapUpgrade
             runner: embedded-cluster
           - test: TestMultiNodeAirgapHAInstallation
             runner: embedded-cluster
+          - test: TestMultiNodeAirgapHADisasterRecovery
+            runner: embedded-cluster
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -211,6 +215,7 @@ jobs:
     - uses: ./.github/actions/e2e
       with:
         test-name: '${{ matrix.test }}'
+        is-large-runner: ${{ matrix.runner == 'embedded-cluster' }}
         airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
         snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
         snapshot-license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}

.github/workflows/release-dev.yaml

@@ -147,11 +147,15 @@ jobs:
           - TestMultiNodeHAInstallation
           - TestMultiNodeAirgapHAInstallation
           - TestMultiNodeAirgapUpgradeSameK0s
+          - TestMultiNodeHADisasterRecovery
+          - TestMultiNodeAirgapHADisasterRecovery
         include:
           - test: TestMultiNodeAirgapUpgrade
             runner: embedded-cluster
           - test: TestMultiNodeAirgapHAInstallation
             runner: embedded-cluster
+          - test: TestMultiNodeAirgapHADisasterRecovery
+            runner: embedded-cluster
     steps:
       - name: Checkout
         uses: actions/checkout@v4

Makefile

@@ -4,12 +4,12 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_URL = oci://registry.replicated.com/library
 ADMIN_CONSOLE_CHART_NAME = admin-console
-ADMIN_CONSOLE_CHART_VERSION = 1.109.9-build.1
+ADMIN_CONSOLE_CHART_VERSION = 1.109.12
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.34.6
+EMBEDDED_OPERATOR_CHART_VERSION = 0.34.9
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_URL = https://openebs.github.io/openebs
@@ -18,7 +18,7 @@ OPENEBS_CHART_VERSION = 4.0.1
 OPENEBS_UTILS_VERSION = 4.0.0
 SEAWEEDFS_CHART_URL = https://seaweedfs.github.io/seaweedfs/helm
 SEAWEEDFS_CHART_NAME = seaweedfs/seaweedfs
-SEAWEEDFS_CHART_VERSION = 3.67.0
+SEAWEEDFS_CHART_VERSION = 3.68.0
 REGISTRY_CHART_URL = https://helm.twun.io
 REGISTRY_CHART_NAME = twuni/docker-registry
 REGISTRY_CHART_VERSION = 2.2.3

cmd/embedded-cluster/assets/resource-modifiers.yaml

@@ -0,0 +1,54 @@
+version: v1
+resourceModifierRules:
+# convert kotsadm components (rqlite) to non-HA mode
+# as kotsadm will always be restored to a single node
+# because it is used during the restore process to add nodes
+- conditions:
+    groupResource: statefulsets.apps
+    resourceNameRegex: "^kotsadm-rqlite$"
+    namespaces:
+    - kotsadm
+  patches:
+  - operation: replace
+    path: "/spec/replicas"
+    value: 1
+  - operation: replace
+    path: "/spec/template/spec/containers/0/args/2"
+    value: "-bootstrap-expect=1"
+# decouple kotsadm components PVCs from nodes
+# this allows the PVCs to be created on the correct nodes
+# when restoring HA kotsadm to a single node and then converting it to HA again
+- conditions:
+    groupResource: persistentvolumeclaims
+    resourceNameRegex: "kotsadm-rqlite"
+    namespaces:
+    - kotsadm
+  mergePatches:
+  - patchData: |
+      {
+        "metadata": {
+          "annotations": {
+            "volume.kubernetes.io/selected-node": null
+          }
+        }
+      }
+# preserve the registry service IP from the original cluster
+- conditions:
+    groupResource: services
+    resourceNameRegex: "^registry$"
+    namespaces:
+    - registry
+  patches:
+  - operation: replace
+    path: "/spec/clusterIP"
+    value: "__REGISTRY_SERVICE_IP__"
+# preserve the seaweedfs s3 service IP from the original cluster
+- conditions:
+    groupResource: services
+    resourceNameRegex: "^ec-seaweedfs-s3$"
+    namespaces:
+    - seaweedfs
+  patches:
+  - operation: replace
+    path: "/spec/clusterIP"
+    value: "__SEAWEEDFS_S3_SERVICE_IP__"

cmd/embedded-cluster/join.go

@@ -18,8 +18,9 @@ import (
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	k8syaml "sigs.k8s.io/yaml"
 
@@ -212,9 +213,8 @@ var joinCommand = &cli.Command{
 		}
 
 		logrus.Debugf("creating systemd unit files")
-		// both controller and worker nodes will have 'worker' in the join command, but only controllers will have 'enable-worker'
-		// https://github.com/replicatedhq/kots/blob/6a0602f4054d5d5f2d97e649b3303a059f0064d9/pkg/embeddedcluster/node_join.go#L183
-		if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "enable-worker")); err != nil {
+		// both controller and worker nodes will have 'worker' in the join command
+		if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller")); err != nil {
 			err := fmt.Errorf("unable to create systemd unit files: %w", err)
 			metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
 			return err
@@ -461,17 +461,16 @@ func canEnableHA(ctx context.Context, kcli client.Client) (bool, error) {
 	if installation.Spec.HighAvailability {
 		return false, nil
 	}
-	var nodes corev1.NodeList
-	labelSelector := labels.Set(map[string]string{
-		"node-role.kubernetes.io/control-plane": "true",
-	}).AsSelector()
-	if err := kcli.List(ctx, &nodes, &client.ListOptions{LabelSelector: labelSelector}); err != nil {
-		return false, fmt.Errorf("unable to list nodes: %w", err)
+	if err := kcli.Get(ctx, types.NamespacedName{Name: ecRestoreStateCMName, Namespace: "embedded-cluster"}, &corev1.ConfigMap{}); err == nil {
+		return false, nil // cannot enable HA during a restore
+	} else if !errors.IsNotFound(err) {
+		return false, fmt.Errorf("unable to get restore state configmap: %w", err)
 	}
-	if len(nodes.Items) < 3 {
-		return false, nil
+	ncps, err := kubeutils.NumOfControlPlaneNodes(ctx, kcli)
+	if err != nil {
+		return false, fmt.Errorf("unable to check control plane nodes: %w", err)
 	}
-	return true, nil
+	return ncps >= 3, nil
 }
 
 // enableHA enables high availability in the installation object

cmd/embedded-cluster/join_test.go

@@ -190,6 +190,24 @@ func Test_canEnableHA(t *testing.T) {
 			},
 			want: false,
 		},
+		{
+			name: "high availability is not enabled and there is three or more controller nodes but a restore is in progress",
+			args: args{
+				kcli: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+					&embeddedclusterv1beta1.Installation{
+						ObjectMeta: metav1.ObjectMeta{Name: "test-installation"},
+						Spec:       embeddedclusterv1beta1.InstallationSpec{HighAvailability: false},
+					},
+					&corev1.ConfigMap{
+						ObjectMeta: metav1.ObjectMeta{Name: ecRestoreStateCMName, Namespace: "embedded-cluster"},
+					},
+					&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: controllerLabels}},
+					&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: controllerLabels}},
+					&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node3", Labels: controllerLabels}},
+				).Build(),
+			},
+			want: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {