K8SPS-414: add imagePullSecrets to the cr.yaml (#921)

* K8SPS-414: add `imagePullSecrets` to the `cr.yaml`

https://perconadev.atlassian.net/browse/K8SPS-414

* K8SPS-415: add `runtimeClassName` to the `cr.yaml`

https://perconadev.atlassian.net/browse/K8SPS-415

* K8SPS-416: add tolerations to the `cr.yaml`

https://perconadev.atlassian.net/browse/K8SPS-416

* fix manifests

* small fixes

* update mysql, router, xtrabackup tests

* delete haproxy and orchestrator unit-tests before merge

* update unit-test

* fix lint

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

Author: pooknull

api/v1alpha1/perconaservermysql_types.go

@@ -217,7 +217,6 @@ type PMMSpec struct {
 	Resources                corev1.ResourceRequirements `json:"resources,omitempty"`
 	ContainerSecurityContext *corev1.SecurityContext     `json:"containerSecurityContext,omitempty"`
 	ImagePullPolicy          corev1.PullPolicy           `json:"imagePullPolicy,omitempty"`
-	RuntimeClassName         *string                     `json:"runtimeClassName,omitempty"`
 }
 
 type BackupSpec struct {

api/v1alpha1/zz_generated.deepcopy.go

@@ -6450,8 +6450,6 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
-                  runtimeClassName:
-                    type: string
                   serverHost:
                     type: string
                 required:

config/crd/bases/ps.percona.com_perconaservermysqls.yaml

@@ -8373,8 +8373,6 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
-                  runtimeClassName:
-                    type: string
                   serverHost:
                     type: string
                 required:

deploy/bundle.yaml

@@ -41,6 +41,15 @@ spec:
     autoRecovery: true
     image: perconalab/percona-server-mysql-operator:main-psmysql
     imagePullPolicy: Always
+#    runtimeClassName: image-rc
+#    tolerations:
+#    - key: "node.alpha.kubernetes.io/unreachable"
+#      operator: "Exists"
+#      effect: "NoExecute"
+#      tolerationSeconds: 6000
+#    imagePullSecrets:
+#    - "secret-1"
+#    - "secret-2"
 #    initImage: perconalab/percona-server-mysql-operator:main
     size: 3
 
@@ -156,6 +165,15 @@ spec:
 
       image: perconalab/percona-server-mysql-operator:main-haproxy
       imagePullPolicy: Always
+#      runtimeClassName: image-rc
+#      tolerations:
+#      - key: "node.alpha.kubernetes.io/unreachable"
+#        operator: "Exists"
+#        effect: "NoExecute"
+#        tolerationSeconds: 6000
+#      imagePullSecrets:
+#      - "secret-1"
+#      - "secret-2"
 
       resources:
         requests:
@@ -265,6 +283,15 @@ spec:
       enabled: false
       image: perconalab/percona-server-mysql-operator:main-router
       imagePullPolicy: Always
+#      runtimeClassName: image-rc
+#      tolerations:
+#      - key: "node.alpha.kubernetes.io/unreachable"
+#        operator: "Exists"
+#        effect: "NoExecute"
+#        tolerationSeconds: 6000
+#      imagePullSecrets:
+#      - "secret-1"
+#      - "secret-2"
 #      initImage: perconalab/percona-server-mysql-operator:main
 
       size: 3
@@ -320,6 +347,15 @@ spec:
 
     image: perconalab/percona-server-mysql-operator:main-orchestrator
     imagePullPolicy: Always
+#    runtimeClassName: image-rc
+#    tolerations:
+#    - key: "node.alpha.kubernetes.io/unreachable"
+#      operator: "Exists"
+#      effect: "NoExecute"
+#      tolerationSeconds: 6000
+#    imagePullSecrets:
+#    - "secret-1"
+#    - "secret-2"
 #    serviceAccountName: percona-server-mysql-operator-orchestrator
 #    initImage: perconalab/percona-server-mysql-operator:main
 
@@ -394,6 +430,9 @@ spec:
 #        storageName: s3
 #    backoffLimit: 6
     imagePullPolicy: Always
+#    imagePullSecrets:
+#    - "secret-1"
+#    - "secret-2"
 #    initImage: perconalab/percona-server-mysql-operator:main
 #    containerSecurityContext:
 #      privileged: true
@@ -447,6 +486,7 @@ spec:
 #        podSecurityContext:
 #          fsGroup: 1001
 #          supplementalGroups: [1001, 1002, 1003]
+#        runtimeClassName: image-rc
         s3:
           bucket: S3-BACKUP-BUCKET-NAME-HERE
           credentialsSecret: cluster1-s3-credentials

deploy/cr.yaml

@@ -8373,8 +8373,6 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
-                  runtimeClassName:
-                    type: string
                   serverHost:
                     type: string
                 required:

deploy/crd.yaml

@@ -8373,8 +8373,6 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
-                  runtimeClassName:
-                    type: string
                   serverHost:
                     type: string
                 required:

deploy/cw-bundle.yaml

@@ -159,8 +159,9 @@ func StatefulSet(cr *apiv1alpha1.PerconaServerMySQL, initImage, configHash, tlsH
 					Annotations: annotations,
 				},
 				Spec: corev1.PodSpec{
-					NodeSelector: cr.Spec.Proxy.HAProxy.NodeSelector,
-					Tolerations:  cr.Spec.Proxy.HAProxy.Tolerations,
+					NodeSelector:     cr.Spec.Proxy.HAProxy.NodeSelector,
+					Tolerations:      cr.Spec.Proxy.HAProxy.Tolerations,
+					RuntimeClassName: cr.Spec.Proxy.HAProxy.RuntimeClassName,
 					InitContainers: []corev1.Container{
 						k8s.InitContainer(
 							AppName,

pkg/haproxy/haproxy.go

@@ -4,55 +4,133 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 
-	apiv1alpha1 "github.com/percona/percona-server-mysql-operator/api/v1alpha1"
-	"github.com/percona/percona-server-mysql-operator/pkg/naming"
+	"github.com/percona/percona-server-mysql-operator/pkg/platform"
+	"github.com/percona/percona-server-mysql-operator/pkg/version"
 )
 
-func TestStatefulSet(t *testing.T) {
-	configHash := "123abc"
-	tlsHash := "123abc"
-	initImage := "percona/init:latest"
+func TestStatefulset(t *testing.T) {
+	const ns = "haproxy-ns"
+	const initImage = "init-image"
+	const configHash = "config-hash"
+	const tlsHash = "tls-hash"
 
-	cr := &apiv1alpha1.PerconaServerMySQL{
+	cr := readDefaultCluster(t, "cluster", ns)
+	if err := cr.CheckNSetDefaults(t.Context(), &platform.ServerVersion{
+		Platform: platform.PlatformKubernetes,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "cluster1",
-			Namespace: "test-ns",
-		},
-		Spec: apiv1alpha1.PerconaServerMySQLSpec{
-			Proxy: apiv1alpha1.ProxySpec{HAProxy: &apiv1alpha1.HAProxySpec{
-				Enabled: true,
-				PodSpec: apiv1alpha1.PodSpec{
-					Size:                          3,
-					TerminationGracePeriodSeconds: pointerInt64(30),
-				},
-			}},
+			Name:      "some-secret",
+			Namespace: ns,
 		},
+		StringData: map[string]string{},
 	}
 
-	sts := StatefulSet(cr, initImage, configHash, tlsHash, nil)
+	t.Run("object meta", func(t *testing.T) {
+		cluster := cr.DeepCopy()
 
-	assert.NotNil(t, sts)
-	assert.Equal(t, "cluster1-haproxy", sts.Name)
-	assert.Equal(t, "test-ns", sts.Namespace)
-	assert.Equal(t, int32(3), *sts.Spec.Replicas)
+		sts := StatefulSet(cluster, initImage, configHash, tlsHash, secret)
 
-	actualTLSAnn, ok := sts.Spec.Template.Annotations[string(naming.AnnotationTLSHash)]
-	assert.True(t, ok)
-	assert.Equal(t, tlsHash, actualTLSAnn)
+		assert.NotNil(t, sts)
+		assert.Equal(t, "cluster-haproxy", sts.Name)
+		assert.Equal(t, "haproxy-ns", sts.Namespace)
+		labels := map[string]string{
+			"app.kubernetes.io/name":       "haproxy",
+			"app.kubernetes.io/part-of":    "percona-server",
+			"app.kubernetes.io/version":    "v" + version.Version(),
+			"app.kubernetes.io/instance":   "cluster",
+			"app.kubernetes.io/managed-by": "percona-server-mysql-operator",
+			"app.kubernetes.io/component":  "proxy",
+		}
+		assert.Equal(t, labels, sts.Labels)
+	})
 
-	actualConfigAnn, ok := sts.Spec.Template.Annotations[string(naming.AnnotationConfigHash)]
-	assert.True(t, ok)
-	assert.Equal(t, configHash, actualConfigAnn)
+	t.Run("defaults", func(t *testing.T) {
+		cluster := cr.DeepCopy()
 
-	initContainers := sts.Spec.Template.Spec.InitContainers
-	assert.Len(t, initContainers, 1)
-	assert.Equal(t, initImage, initContainers[0].Image)
+		sts := StatefulSet(cluster, initImage, configHash, tlsHash, secret)
 
-	assert.Equal(t, pointerInt64(30), sts.Spec.Template.Spec.TerminationGracePeriodSeconds)
-}
+		assert.Equal(t, int32(3), *sts.Spec.Replicas)
+		initContainers := sts.Spec.Template.Spec.InitContainers
+		assert.Len(t, initContainers, 1)
+		assert.Equal(t, initImage, initContainers[0].Image)
+
+		assert.Equal(t, map[string]string{
+			"percona.com/last-applied-tls":   tlsHash,
+			"percona.com/configuration-hash": configHash,
+		}, sts.Spec.Template.Annotations)
+	})
+
+	t.Run("termination grace period seconds", func(t *testing.T) {
+		cluster := cr.DeepCopy()
+
+		cluster.Spec.Proxy.HAProxy.TerminationGracePeriodSeconds = nil
+		sts := StatefulSet(cluster, initImage, configHash, tlsHash, secret)
+		assert.Equal(t, int64(600), *sts.Spec.Template.Spec.TerminationGracePeriodSeconds)
+
+		cluster.Spec.Proxy.HAProxy.TerminationGracePeriodSeconds = ptr.To(int64(30))
+
+		sts = StatefulSet(cluster, initImage, configHash, tlsHash, secret)
+		assert.Equal(t, int64(30), *sts.Spec.Template.Spec.TerminationGracePeriodSeconds)
+	})
+
+	t.Run("image pull secrets", func(t *testing.T) {
+		cluster := cr.DeepCopy()
+
+		sts := StatefulSet(cluster, initImage, configHash, tlsHash, secret)
+		assert.Equal(t, []corev1.LocalObjectReference(nil), sts.Spec.Template.Spec.ImagePullSecrets)
+
+		imagePullSecrets := []corev1.LocalObjectReference{
+			{
+				Name: "secret-1",
+			},
+			{
+				Name: "secret-2",
+			},
+		}
+		cluster.Spec.Proxy.HAProxy.ImagePullSecrets = imagePullSecrets
+
+		sts = StatefulSet(cluster, initImage, configHash, tlsHash, secret)
+		assert.Equal(t, imagePullSecrets, sts.Spec.Template.Spec.ImagePullSecrets)
+	})
+
+	t.Run("runtime class name", func(t *testing.T) {
+		cluster := cr.DeepCopy()
+		sts := StatefulSet(cluster, initImage, configHash, tlsHash, secret)
+		var e *string
+		assert.Equal(t, e, sts.Spec.Template.Spec.RuntimeClassName)
+
+		const runtimeClassName = "runtimeClassName"
+		cluster.Spec.Proxy.HAProxy.RuntimeClassName = ptr.To(runtimeClassName)
+
+		sts = StatefulSet(cluster, initImage, configHash, tlsHash, secret)
+		assert.Equal(t, runtimeClassName, *sts.Spec.Template.Spec.RuntimeClassName)
+	})
+
+	t.Run("tolerations", func(t *testing.T) {
+		cluster := cr.DeepCopy()
+		sts := StatefulSet(cluster, initImage, configHash, tlsHash, secret)
+		assert.Equal(t, []corev1.Toleration(nil), sts.Spec.Template.Spec.Tolerations)
+
+		tolerations := []corev1.Toleration{
+			{
+				Key:               "node.alpha.kubernetes.io/unreachable",
+				Operator:          "Exists",
+				Value:             "value",
+				Effect:            "NoExecute",
+				TolerationSeconds: ptr.To(int64(1001)),
+			},
+		}
+		cluster.Spec.Proxy.HAProxy.Tolerations = tolerations
 
-func pointerInt64(i int64) *int64 {
-	return &i
+		sts = StatefulSet(cluster, initImage, configHash, tlsHash, secret)
+		assert.Equal(t, tolerations, sts.Spec.Template.Spec.Tolerations)
+	})
 }

pkg/haproxy/haproxy_test.go

@@ -0,0 +1,36 @@
+package haproxy
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"sigs.k8s.io/yaml"
+
+	apiv1alpha1 "github.com/percona/percona-server-mysql-operator/api/v1alpha1"
+)
+
+func readDefaultCluster(t *testing.T, name, namespace string) *apiv1alpha1.PerconaServerMySQL {
+	t.Helper()
+
+	cr := &apiv1alpha1.PerconaServerMySQL{}
+	readDefaultFile(t, "cr.yaml", cr)
+
+	cr.Name = name
+	cr.Namespace = namespace
+	cr.Spec.InitImage = "init-image"
+	return cr
+}
+
+func readDefaultFile[T any](t *testing.T, filename string, obj *T) {
+	t.Helper()
+
+	data, err := os.ReadFile(filepath.Join("..", "..", "deploy", filename))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := yaml.Unmarshal(data, obj); err != nil {
+		t.Fatal(err)
+	}
+}