K8SPSMDB-733: Automate volume scaling (#1487)

* K8SPSMDB-733: Automate volume scaling

* add test

* address review comments

* fix tests

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

Author: egegunes

e2e-tests/pvc-resize/compare/statefulset_some-name-rs0.yml

@@ -0,0 +1,217 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  annotations: {}
+  generation: 1
+  labels:
+    app.kubernetes.io/component: mongod
+    app.kubernetes.io/instance: some-name
+    app.kubernetes.io/managed-by: percona-server-mongodb-operator
+    app.kubernetes.io/name: percona-server-mongodb
+    app.kubernetes.io/part-of: percona-server-mongodb
+    app.kubernetes.io/replset: rs0
+  name: some-name-rs0
+  ownerReferences:
+    - controller: true
+      kind: PerconaServerMongoDB
+      name: some-name
+spec:
+  podManagementPolicy: OrderedReady
+  replicas: 3
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: mongod
+      app.kubernetes.io/instance: some-name
+      app.kubernetes.io/managed-by: percona-server-mongodb-operator
+      app.kubernetes.io/name: percona-server-mongodb
+      app.kubernetes.io/part-of: percona-server-mongodb
+      app.kubernetes.io/replset: rs0
+  serviceName: some-name-rs0
+  template:
+    metadata:
+      annotations: {}
+      labels:
+        app.kubernetes.io/component: mongod
+        app.kubernetes.io/instance: some-name
+        app.kubernetes.io/managed-by: percona-server-mongodb-operator
+        app.kubernetes.io/name: percona-server-mongodb
+        app.kubernetes.io/part-of: percona-server-mongodb
+        app.kubernetes.io/replset: rs0
+    spec:
+      containers:
+        - args:
+            - --bind_ip_all
+            - --auth
+            - --dbpath=/data/db
+            - --port=27017
+            - --replSet=rs0
+            - --storageEngine=wiredTiger
+            - --relaxPermChecks
+            - --sslAllowInvalidCertificates
+            - --clusterAuthMode=x509
+            - --enableEncryption
+            - --encryptionKeyFile=/etc/mongodb-encryption/encryption-key
+            - --wiredTigerCacheSizeGB=0.25
+            - --wiredTigerIndexPrefixCompression=true
+            - --config=/etc/mongodb-config/mongod.conf
+            - --quiet
+          command:
+            - /opt/percona/ps-entry.sh
+          env:
+            - name: SERVICE_NAME
+              value: some-name
+            - name: MONGODB_PORT
+              value: "27017"
+            - name: MONGODB_REPLSET
+              value: rs0
+          envFrom:
+            - secretRef:
+                name: internal-some-name-users
+                optional: false
+          imagePullPolicy: Always
+          livenessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - liveness
+                - --ssl
+                - --sslInsecure
+                - --sslCAFile
+                - /etc/mongodb-ssl/ca.crt
+                - --sslPEMKeyFile
+                - /tmp/tls.pem
+                - --startupDelaySeconds
+                - "7200"
+            failureThreshold: 4
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 10
+          name: mongod
+          ports:
+            - containerPort: 27017
+              name: mongodb
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command:
+                - /opt/percona/mongodb-healthcheck
+                - k8s
+                - readiness
+                - --component
+                - mongod
+            failureThreshold: 8
+            initialDelaySeconds: 10
+            periodSeconds: 3
+            successThreshold: 1
+            timeoutSeconds: 2
+          resources:
+            limits:
+              cpu: 500m
+              memory: 500M
+            requests:
+              cpu: 100m
+              memory: 100M
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1001
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /data/db
+              name: mongod-data
+            - mountPath: /etc/mongodb-secrets
+              name: some-name-mongodb-keyfile
+              readOnly: true
+            - mountPath: /etc/mongodb-ssl
+              name: ssl
+              readOnly: true
+            - mountPath: /etc/mongodb-ssl-internal
+              name: ssl-internal
+              readOnly: true
+            - mountPath: /etc/mongodb-config
+              name: config
+            - mountPath: /opt/percona
+              name: bin
+            - mountPath: /etc/mongodb-encryption
+              name: some-name-mongodb-encryption-key
+              readOnly: true
+            - mountPath: /etc/users-secret
+              name: users-secret-file
+          workingDir: /data/db
+      dnsPolicy: ClusterFirst
+      initContainers:
+        - command:
+            - /init-entrypoint.sh
+          imagePullPolicy: Always
+          name: mongo-init
+          resources:
+            limits:
+              cpu: 500m
+              memory: 500M
+            requests:
+              cpu: 100m
+              memory: 100M
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /data/db
+              name: mongod-data
+            - mountPath: /opt/percona
+              name: bin
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext:
+        fsGroup: 1001
+      serviceAccount: default
+      serviceAccountName: default
+      terminationGracePeriodSeconds: 60
+      volumes:
+        - name: some-name-mongodb-keyfile
+          secret:
+            defaultMode: 288
+            optional: false
+            secretName: some-name-mongodb-keyfile
+        - emptyDir: {}
+          name: bin
+        - configMap:
+            defaultMode: 420
+            name: some-name-rs0-mongod
+            optional: true
+          name: config
+        - name: some-name-mongodb-encryption-key
+          secret:
+            defaultMode: 288
+            optional: false
+            secretName: some-name-mongodb-encryption-key
+        - name: ssl
+          secret:
+            defaultMode: 288
+            optional: false
+            secretName: some-name-ssl
+        - name: ssl-internal
+          secret:
+            defaultMode: 288
+            optional: true
+            secretName: some-name-ssl-internal
+        - name: users-secret-file
+          secret:
+            defaultMode: 420
+            secretName: internal-some-name-users
+  updateStrategy:
+    rollingUpdate:
+      partition: 0
+    type: RollingUpdate
+  volumeClaimTemplates:
+    - metadata:
+        name: mongod-data
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi
+      status:
+        phase: Pending

e2e-tests/pvc-resize/run

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -o errexit
+
+test_dir=$(realpath $(dirname $0))
+. ${test_dir}/../functions
+
+set_debug
+
+if [[ $EKS == 1 ]]; then
+    echo "Skip the test. We don't run it for EKS."
+    exit 0
+fi
+
+create_infra ${namespace}
+
+desc 'create secrets and psmdb client'
+kubectl_bin apply \
+    -f $conf_dir/secrets.yml \
+    -f $conf_dir/client.yml
+
+desc 'create PSMDB cluster'
+cluster="some-name"
+spinup_psmdb "${cluster}-rs0" "$conf_dir/$cluster-rs0.yml"
+
+kubectl_bin patch psmdb ${cluster} --type=json -p='[{"op": "replace", "path": "/spec/replsets/0/volumeSpec/persistentVolumeClaim/resources/requests/storage", "value":"5Gi"}]'
+wait_cluster_consistency "$cluster" 3 2
+
+for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=mongod -o name); do
+    retry=0
+    until [[ $(kubectl_bin get ${pvc} -o jsonpath={.status.capacity.storage}) == "5Gi" ]]; do
+        if [[ $retry -ge 60 ]]; then
+            echo "PVC ${pvc} was not resized, max retries exceeded"
+            exit 1
+        fi
+
+        echo "Waiting for PVC ${pvc} to be resized"
+        sleep 5
+
+        retry=$((retry + 1))
+    done
+    echo "PVC ${pvc} was resized"
+done
+
+destroy "${namespace}"
+desc "test passed"

e2e-tests/run-pr.csv

@@ -27,6 +27,7 @@ operator-self-healing-chaos
 pitr
 pitr-sharded
 pitr-physical
+pvc-resize
 recover-no-primary
 rs-shard-migration
 scaling

e2e-tests/run-release.csv

@@ -28,6 +28,7 @@ operator-self-healing-chaos
 pitr
 pitr-sharded
 pitr-physical
+pvc-resize
 recover-no-primary
 rs-shard-migration
 scaling

pkg/apis/psmdb/v1/psmdb_types.go

@@ -1136,4 +1136,12 @@ func (cr *PerconaServerMongoDB) GetOrderedFinalizers() []string {
 	return orderedFinalizers
 }
 
-const AnnotationResyncPBM = "percona.com/resync-pbm"
+const (
+	AnnotationResyncPBM           = "percona.com/resync-pbm"
+	AnnotationPVCResizeInProgress = "percona.com/pvc-resize-in-progress"
+)
+
+func (cr *PerconaServerMongoDB) PVCResizeInProgress() bool {
+	_, ok := cr.Annotations[AnnotationPVCResizeInProgress]
+	return ok
+}

pkg/controller/perconaservermongodb/statefulset.go

@@ -5,11 +5,9 @@ import (
 
 	api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1"
 	"github.com/percona/percona-server-mongodb-operator/pkg/psmdb"
-	"github.com/percona/percona-server-mongodb-operator/pkg/util"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -49,6 +47,10 @@ func (r *ReconcilePerconaServerMongoDB) reconcileStatefulSet(ctx context.Context
 		return sfs, nil
 	}
 
+	if err := r.reconcilePVCs(ctx, cr, sfs, ls, volumeSpec.PersistentVolumeClaim); err != nil {
+		return nil, errors.Wrapf(err, "reconcile PVCs for %s", sfs.Name)
+	}
+
 	err = r.createOrUpdate(ctx, sfs)
 	if err != nil {
 		return nil, errors.Wrapf(err, "update StatefulSet %s", sfs.Name)
@@ -59,10 +61,6 @@ func (r *ReconcilePerconaServerMongoDB) reconcileStatefulSet(ctx context.Context
 		return nil, errors.Wrapf(err, "PodDisruptionBudget for %s", sfs.Name)
 	}
 
-	if err := r.reconcilePVCs(ctx, sfs, ls, volumeSpec.PersistentVolumeClaim); err != nil {
-		return nil, errors.Wrapf(err, "reconcile PVCs for %s", sfs.Name)
-	}
-
 	if err := r.smartUpdate(ctx, cr, sfs, rs); err != nil {
 		return nil, errors.Wrap(err, "failed to run smartUpdate")
 	}
@@ -126,39 +124,3 @@ func (r *ReconcilePerconaServerMongoDB) getStatefulsetFromReplset(ctx context.Co
 
 	return sfs, nil
 }
-
-func (r *ReconcilePerconaServerMongoDB) reconcilePVCs(ctx context.Context, sts *appsv1.StatefulSet, ls map[string]string, pvcSpec api.PVCSpec) error {
-	pvcList := &corev1.PersistentVolumeClaimList{}
-	err := r.client.List(ctx, pvcList, &client.ListOptions{
-		Namespace:     sts.Namespace,
-		LabelSelector: labels.SelectorFromSet(ls),
-	})
-	if err != nil {
-		return errors.Wrap(err, "list PVCs")
-	}
-
-	for _, pvc := range pvcList.Items {
-		orig := pvc.DeepCopy()
-
-		for k, v := range pvcSpec.Labels {
-			pvc.Labels[k] = v
-		}
-		for k, v := range sts.Labels {
-			pvc.Labels[k] = v
-		}
-		for k, v := range pvcSpec.Annotations {
-			pvc.Annotations[k] = v
-		}
-
-		if util.MapEqual(orig.Labels, pvc.Labels) && util.MapEqual(orig.Annotations, pvc.Annotations) {
-			continue
-		}
-		patch := client.MergeFrom(orig)
-
-		if err := r.client.Patch(ctx, &pvc, patch); err != nil {
-			logf.FromContext(ctx).Error(err, "patch PVC", "PVC", pvc.Name)
-		}
-	}
-
-	return nil
-}