From 380320b23a93c725fde2dd50769e7e37576d694a Mon Sep 17 00:00:00 2001 From: Natalia Marukovich Date: Mon, 2 Dec 2024 17:32:08 +0100 Subject: [PATCH 1/4] K8SPG-645 turn on sidecar conteiners for pgbouncer --- cmd/postgres-operator/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/postgres-operator/main.go b/cmd/postgres-operator/main.go index 0a7c48b6ae..25cc1d150e 100644 --- a/cmd/postgres-operator/main.go +++ b/cmd/postgres-operator/main.go @@ -90,6 +90,7 @@ func main() { features := feature.NewGate() err = features.SetFromMap(map[string]bool{ string(feature.InstanceSidecars): true, // needed for PMM + string(feature.PGBouncerSidecars): true, string(feature.TablespaceVolumes): true, }) assertNoError(err) From 4a4bdb2d45ad929c9072b4b025d45e2a4cc1d553 Mon Sep 17 00:00:00 2001 From: Natalia Marukovich Date: Tue, 3 Dec 2024 15:37:54 +0100 Subject: [PATCH 2/4] Update cmd/postgres-operator/main.go Co-authored-by: Andrii Dema --- cmd/postgres-operator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/postgres-operator/main.go b/cmd/postgres-operator/main.go index 25cc1d150e..b4c7e51335 100644 --- a/cmd/postgres-operator/main.go +++ b/cmd/postgres-operator/main.go @@ -90,7 +90,7 @@ func main() { features := feature.NewGate() err = features.SetFromMap(map[string]bool{ string(feature.InstanceSidecars): true, // needed for PMM - string(feature.PGBouncerSidecars): true, + string(feature.PGBouncerSidecars): true, // K8SPG-645 string(feature.TablespaceVolumes): true, }) assertNoError(err) From da3f76aef1db8e2ba3b6d3a942165e2ff880bd6a Mon Sep 17 00:00:00 2001 From: Natalia Marukovich Date: Mon, 9 Dec 2024 14:15:53 +0100 Subject: [PATCH 3/4] add sidecar test --- e2e-tests/tests/sidecars/00-assert.yaml | 24 +++ .../tests/sidecars/00-deploy-operator.yaml | 14 ++ e2e-tests/tests/sidecars/01-assert.yaml | 197 ++++++++++++++++++ .../tests/sidecars/01-create-cluster.yaml | 18 ++ .../99-remove-cluster-gracefully.yaml | 21 ++ 5 files changed, 274 insertions(+) create mode 100644 e2e-tests/tests/sidecars/00-assert.yaml create mode 100644 e2e-tests/tests/sidecars/00-deploy-operator.yaml create mode 100644 e2e-tests/tests/sidecars/01-assert.yaml create mode 100644 e2e-tests/tests/sidecars/01-create-cluster.yaml create mode 100644 e2e-tests/tests/sidecars/99-remove-cluster-gracefully.yaml diff --git a/e2e-tests/tests/sidecars/00-assert.yaml b/e2e-tests/tests/sidecars/00-assert.yaml new file mode 100644 index 0000000000..ae5a062d84 --- /dev/null +++ b/e2e-tests/tests/sidecars/00-assert.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: perconapgclusters.pgv2.percona.com +spec: + group: pgv2.percona.com + names: + kind: PerconaPGCluster + listKind: PerconaPGClusterList + plural: perconapgclusters + singular: perconapgcluster + scope: Namespaced +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-deploy-status +timeout: 120 +commands: + - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/sidecars/00-deploy-operator.yaml b/e2e-tests/tests/sidecars/00-deploy-operator.yaml new file mode 100644 index 0000000000..544ecbeb95 --- /dev/null +++ b/e2e-tests/tests/sidecars/00-deploy-operator.yaml @@ -0,0 +1,14 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + deploy_operator + deploy_client + diff --git a/e2e-tests/tests/sidecars/01-assert.yaml b/e2e-tests/tests/sidecars/01-assert.yaml new file mode 100644 index 0000000000..b1812e3386 --- /dev/null +++ b/e2e-tests/tests/sidecars/01-assert.yaml @@ -0,0 +1,197 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 180 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: sidecars-repo-host + labels: + postgres-operator.crunchydata.com/cluster: sidecars + postgres-operator.crunchydata.com/data: pgbackrest + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-dedicated: '' + ownerReferences: + - apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: sidecars + controller: true + blockOwnerDeletion: true +spec: + template: + spec: + containers: + - command: + - sleep + - 30d + image: busybox + imagePullPolicy: Always + name: sidecar1 + resources: { } + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /tmp + name: tmp + - command: + - sleep + - 30d + image: busybox + imagePullPolicy: Always + name: sidecar2 + resources: { } + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /tmp + name: tmp +status: + observedGeneration: 1 + replicas: 1 + readyReplicas: 1 + currentReplicas: 1 + updatedReplicas: 1 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: sidecars + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + ownerReferences: + - apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: sidecars + controller: true + blockOwnerDeletion: true +status: + observedGeneration: 1 + replicas: 1 + readyReplicas: 1 + currentReplicas: 1 + updatedReplicas: 1 + collisionCount: 0 +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: sidecars-pgbouncer + labels: + postgres-operator.crunchydata.com/cluster: sidecars + postgres-operator.crunchydata.com/role: pgbouncer + annotations: + deployment.kubernetes.io/revision: '1' + ownerReferences: + - apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: sidecars + controller: true + blockOwnerDeletion: true +status: + observedGeneration: 1 + replicas: 3 + updatedReplicas: 3 + readyReplicas: 3 +--- +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: sidecars + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster + name: sidecars + controller: true + blockOwnerDeletion: true + finalizers: + - postgres-operator.crunchydata.com/finalizer +status: + instances: + - name: instance1 + readyReplicas: 3 + replicas: 3 + updatedReplicas: 3 + observedGeneration: 1 + pgbackrest: + repoHost: + apiVersion: apps/v1 + kind: StatefulSet + ready: true + repos: + - bound: true + name: repo1 + replicaCreateBackupComplete: true + stanzaCreated: true + proxy: + pgBouncer: + readyReplicas: 3 + replicas: 3 +--- +- apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster + metadata: + labels: + e2e: sidecars + name: sidecars + spec: + instances: + - affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/data: postgres + topologyKey: kubernetes.io/hostname + weight: 1 + dataVolumeClaimSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + name: instance1 + replicas: 3 + sidecars: + - command: + - sleep + - 30d + image: busybox + name: sidecar1 + - command: + - sleep + - 30d + image: busybox + name: sidecar2 + proxy: + pgBouncer: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/role: pgbouncer + topologyKey: kubernetes.io/hostname + weight: 1 + replicas: 3 + sidecars: + - command: + - sleep + - 30d + image: busybox + name: sidecar1 + status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/sidecars/01-create-cluster.yaml b/e2e-tests/tests/sidecars/01-create-cluster.yaml new file mode 100644 index 0000000000..9c0ef9187f --- /dev/null +++ b/e2e-tests/tests/sidecars/01-create-cluster.yaml @@ -0,0 +1,18 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 20 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr \ + | yq eval '.spec.instances[0].sidecars = [{"name": "sidecar1", "image": "busybox", "command": ["sleep", "30d"]}]' - \ + | yq eval '.spec.instances[0].sidecars += [{"name": "sidecar2", "image": "busybox", "command": ["sleep", "30d"]}]' - \ + | yq eval '.spec.proxy.pgBouncer.sidecars = [{"name": "sidecar1", "image": "busybox", "command": ["sleep", "30d"]}]' - \ + | kubectl -n "${NAMESPACE}" apply -f - + timeout: 30 + + diff --git a/e2e-tests/tests/sidecars/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/sidecars/99-remove-cluster-gracefully.yaml new file mode 100644 index 0000000000..59f953f9dc --- /dev/null +++ b/e2e-tests/tests/sidecars/99-remove-cluster-gracefully.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster + metadata: + name: self-healing + - apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: self-healing +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + destroy_operator + timeout: 60 + From c61214f1217e529bf340937c11fa0ae324777d59 Mon Sep 17 00:00:00 2001 From: Natalia Marukovich Date: Wed, 11 Dec 2024 12:53:07 +0100 Subject: [PATCH 4/4] update test --- e2e-tests/run-pr.csv | 1 + e2e-tests/run-release.csv | 1 + e2e-tests/tests/sidecars/01-assert.yaml | 174 +++++++++++------------- 3 files changed, 83 insertions(+), 93 deletions(-) diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index 1a005a4e37..5980607e79 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -10,6 +10,7 @@ pitr scaling scheduled-backup self-healing +sidecars start-from-backup tablespaces telemetry-transfer diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index 639f97fc77..412a9c1daa 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -12,6 +12,7 @@ pitr scaling scheduled-backup self-healing +sidecars start-from-backup tablespaces telemetry-transfer diff --git a/e2e-tests/tests/sidecars/01-assert.yaml b/e2e-tests/tests/sidecars/01-assert.yaml index b1812e3386..731258761f 100644 --- a/e2e-tests/tests/sidecars/01-assert.yaml +++ b/e2e-tests/tests/sidecars/01-assert.yaml @@ -17,34 +17,6 @@ metadata: name: sidecars controller: true blockOwnerDeletion: true -spec: - template: - spec: - containers: - - command: - - sleep - - 30d - image: busybox - imagePullPolicy: Always - name: sidecar1 - resources: { } - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /tmp - name: tmp - - command: - - sleep - - 30d - image: busybox - imagePullPolicy: Always - name: sidecar2 - resources: { } - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /tmp - name: tmp status: observedGeneration: 1 replicas: 1 @@ -95,6 +67,22 @@ status: updatedReplicas: 3 readyReplicas: 3 --- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: sidecars + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 +--- apiVersion: postgres-operator.crunchydata.com/v1beta1 kind: PostgresCluster metadata: @@ -129,69 +117,69 @@ status: readyReplicas: 3 replicas: 3 --- -- apiVersion: pgv2.percona.com/v2 - kind: PerconaPGCluster - metadata: - labels: - e2e: sidecars - name: sidecars - spec: +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + labels: + e2e: sidecars + name: sidecars +spec: + instances: + - affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/data: postgres + topologyKey: kubernetes.io/hostname + weight: 1 + dataVolumeClaimSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + name: instance1 + replicas: 3 + sidecars: + - command: + - sleep + - 30d + image: busybox + name: sidecar1 + - command: + - sleep + - 30d + image: busybox + name: sidecar2 + proxy: + pgBouncer: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/role: pgbouncer + topologyKey: kubernetes.io/hostname + weight: 1 + replicas: 3 + sidecars: + - command: + - sleep + - 30d + image: busybox + name: sidecar1 +status: + pgbouncer: + ready: 3 + size: 3 + postgres: instances: - - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - postgres-operator.crunchydata.com/data: postgres - topologyKey: kubernetes.io/hostname - weight: 1 - dataVolumeClaimSpec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - name: instance1 - replicas: 3 - sidecars: - - command: - - sleep - - 30d - image: busybox - name: sidecar1 - - command: - - sleep - - 30d - image: busybox - name: sidecar2 - proxy: - pgBouncer: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - postgres-operator.crunchydata.com/role: pgbouncer - topologyKey: kubernetes.io/hostname - weight: 1 - replicas: 3 - sidecars: - - command: - - sleep - - 30d - image: busybox - name: sidecar1 - status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready