fix: add tolerations and nodeAffinity overrides for registry addon (#1183)

**What problem does this PR solve?**:
Adds antiAffinity to ensure that registry pods are scheduled on the
control plane nodes.
Adds tolerations and nodeAffinity to the registry pods to ensure that
they are scheduled on the control plane nodes.

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->
- E2E tests
- Manual tests

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

Author: supershal

charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml

@@ -25,3 +25,26 @@ statefulSet:
         cpu: 100m
         memory: 75Mi
 tlsSecretName: {{ .TLSSecretName }}
+tolerations:
+  - key: "node-role.kubernetes.io/control-plane"
+    operator: Exists
+    effect: "NoSchedule"
+
+podLabels:
+  cncf-distribution-registry: "true" # ensure the labels match with pod AntiAffinity.
+
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+  podAntiAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        podAffinityTerm:
+          labelSelector:
+            matchLabels:
+              cncf-distribution-registry: "true"
+          topologyKey: kubernetes.io/hostname

test/e2e/quick_start_test.go

@@ -315,6 +315,15 @@ var _ = Describe("Quick start", func() {
 														ClusterProxy:    proxy,
 													},
 												)
+
+												EnsureAntiAffnityForRegistryAddon(
+													ctx,
+													EnsureAntiAffnityForRegistryAddonInput{
+														Registry:        addonsConfig.Registry,
+														WorkloadCluster: workloadCluster,
+														ClusterProxy:    proxy,
+													},
+												)
 											},
 										}
 									})

test/e2e/registry.go

@@ -101,3 +101,57 @@ func EnsureClusterCAForRegistryAddon(
 	const caCrtKey = "ca.crt"
 	Expect(rootCASecret.Data[caCrtKey]).To(Equal(rootCASecret.Data[caCrtKey]))
 }
+
+type EnsureAntiAffnityForRegistryAddonInput struct {
+	Registry        *v1alpha1.RegistryAddon
+	WorkloadCluster *clusterv1.Cluster
+	ClusterProxy    framework.ClusterProxy
+}
+
+func EnsureAntiAffnityForRegistryAddon(
+	ctx context.Context,
+	input EnsureAntiAffnityForRegistryAddonInput,
+) {
+	if input.Registry == nil {
+		return
+	}
+	Log("Ensuring anti-affinity for registry addon in workload cluster")
+	workloadClusterClient := input.ClusterProxy.GetWorkloadCluster(
+		ctx, input.WorkloadCluster.Namespace, input.WorkloadCluster.Name,
+	).GetClient()
+
+	sts := &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "cncf-distribution-registry-docker-registry",
+			Namespace: "registry-system",
+		},
+	}
+	err := workloadClusterClient.Get(ctx, ctrlclient.ObjectKeyFromObject(sts), sts)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(sts.Spec.Template.Spec.Affinity).ToNot(BeNil())
+	Expect(sts.Spec.Template.Spec.Affinity.PodAntiAffinity).ToNot(BeNil())
+	podAntiAffinity := sts.Spec.Template.Spec.Affinity.PodAntiAffinity
+	Expect(
+		podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution,
+	).ToNot(BeEmpty())
+	Expect(
+		podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight,
+	).To(Equal(int32(100)))
+	podAffinityTerm := podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm
+	Expect(podAffinityTerm).ToNot(BeNil())
+	Expect(podAffinityTerm.TopologyKey).To(Equal("kubernetes.io/hostname"))
+	Expect(podAffinityTerm.LabelSelector).ToNot(BeNil())
+	affinityLabels := podAffinityTerm.LabelSelector.MatchLabels
+	Expect(
+		affinityLabels["cncf-distribution-registry"],
+	).To(Equal("true")) // Ensure the label matches the pod AntiAffinity.
+
+	// test node affinity
+	nodeAffinity := sts.Spec.Template.Spec.Affinity.NodeAffinity
+	Expect(nodeAffinity).ToNot(BeNil())
+	Expect(nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).ToNot(BeNil())
+	nodeSelectorTerm := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0]
+	Expect(nodeSelectorTerm).ToNot(BeNil())
+	Expect(nodeSelectorTerm.MatchExpressions).ToNot(BeEmpty())
+	Expect(nodeSelectorTerm.MatchExpressions[0].Key).To(Equal("node-role.kubernetes.io/control-plane"))
+}

test/e2e/self_hosted_test.go

@@ -177,6 +177,15 @@ var _ = Describe("Self-hosted", Serial, func() {
 															ClusterProxy:    proxy,
 														},
 													)
+
+													EnsureAntiAffnityForRegistryAddon(
+														ctx,
+														EnsureAntiAffnityForRegistryAddonInput{
+															Registry:        addonsConfig.Registry,
+															WorkloadCluster: workloadCluster,
+															ClusterProxy:    proxy,
+														},
+													)
 												},
 											}
 										},

test/e2e/statefulset_helpers.go

@@ -21,7 +21,7 @@ type WaitForStatefulSetAvailableInput struct {
 	StatefulSet *appsv1.StatefulSet
 }
 
-// WaitForStatefulSetsAvailable waits until the Deployment has observedGeneration equal to generation and
+// WaitForStatefulSetsAvailable waits until the Statefulset has observedGeneration equal to generation and
 // status.Available = True, that signals that all the desired replicas are in place.
 func WaitForStatefulSetsAvailable(
 	ctx context.Context, input WaitForStatefulSetAvailableInput, intervals ...interface{},