Skip to content

Commit 8321bc0

Browse files
horsgithub-actions[bot]
hors
and
github-actions[bot]
authored
K8SPSMDB-1039 fix deletion from server inventory on pod termination (#1434)
* K8SPSMDB-1039 fix deletion from server inventory on pod termination * use 'pmm-admin unregister' command to the instance from server inventory on pod termination * add test case * Update e2e-tests/monitoring-2-0/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/monitoring-2-0/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/monitoring-2-0/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/monitoring-2-0/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/monitoring-2-0/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/monitoring-2-0/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
1 parent 3231c37 commit 8321bc0

8 files changed

+61
-6
lines changed

‎e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg-oc.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ spec:
241241
command:
242242
- bash
243243
- -c
244-
- pmm-admin inventory remove node --force $(pmm-admin status --json | python -c "import sys, json; print(json.load(sys.stdin)['pmm_agent_status']['node_id'])")
244+
- pmm-admin unregister --force
245245
livenessProbe:
246246
failureThreshold: 3
247247
httpGet:

‎e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ spec:
242242
command:
243243
- bash
244244
- -c
245-
- pmm-admin inventory remove node --force $(pmm-admin status --json | python -c "import sys, json; print(json.load(sys.stdin)['pmm_agent_status']['node_id'])")
245+
- pmm-admin unregister --force
246246
livenessProbe:
247247
failureThreshold: 3
248248
httpGet:

‎e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos-oc.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ spec:
243243
command:
244244
- bash
245245
- -c
246-
- pmm-admin inventory remove node --force $(pmm-admin status --json | python -c "import sys, json; print(json.load(sys.stdin)['pmm_agent_status']['node_id'])")
246+
- pmm-admin unregister --force
247247
livenessProbe:
248248
failureThreshold: 3
249249
httpGet:

‎e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ spec:
244244
command:
245245
- bash
246246
- -c
247-
- pmm-admin inventory remove node --force $(pmm-admin status --json | python -c "import sys, json; print(json.load(sys.stdin)['pmm_agent_status']['node_id'])")
247+
- pmm-admin unregister --force
248248
livenessProbe:
249249
failureThreshold: 3
250250
httpGet:

‎e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-oc.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ spec:
229229
command:
230230
- bash
231231
- -c
232-
- pmm-admin inventory remove node --force $(pmm-admin status --json | python -c "import sys, json; print(json.load(sys.stdin)['pmm_agent_status']['node_id'])")
232+
- pmm-admin unregister --force
233233
livenessProbe:
234234
failureThreshold: 3
235235
httpGet:

‎e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ spec:
230230
command:
231231
- bash
232232
- -c
233-
- pmm-admin inventory remove node --force $(pmm-admin status --json | python -c "import sys, json; print(json.load(sys.stdin)['pmm_agent_status']['node_id'])")
233+
- pmm-admin unregister --force
234234
livenessProbe:
235235
failureThreshold: 3
236236
httpGet:

‎e2e-tests/monitoring-2-0/run

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,25 @@ test_dir=$(realpath $(dirname $0))
66
. ${test_dir}/../functions
77
set_debug
88

9+
get_node_id_from_pmm() {
10+
local -a nodeList=()
11+
for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name'); do
12+
nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
13+
done
14+
15+
echo "${nodeList[@]}"
16+
}
17+
18+
does_node_id_exists() {
19+
local -a nodeList=("$@")
20+
local -a nodeList_from_pmm=()
21+
for node_id in "${nodeList[@]}"; do
22+
nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
23+
done
24+
25+
echo "${nodeList_from_pmm[@]}"
26+
}
27+
928
deploy_cert_manager
1029
create_infra $namespace
1130

@@ -93,6 +112,28 @@ desc 'check QAN data'
93112
get_qan_values mongodb "dev-mongod" admin:admin
94113
get_qan_values mongodb "dev-mongos" admin:admin
95114

115+
nodeList=($(get_node_id_from_pmm))
116+
nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}"))
117+
for node_id in "${nodeList_from_pmm[@]}"; do
118+
if [ -z "$node_id" ]; then
119+
echo "Can't get $node_id node_id from PMM server"
120+
exit 1
121+
fi
122+
done
123+
124+
kubectl_bin patch psmdb ${cluster} --type json -p='[{"op":"add","path":"/spec/pause","value":true}]'
125+
wait_for_delete "pod/${cluster}-mongos-0"
126+
wait_for_delete "pod/${cluster}-rs0-0"
127+
wait_for_delete "pod/${cluster}-cfg-0"
128+
129+
does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}"))
130+
for instance in "${does_node_id_exists_in_pmm[@]}"; do
131+
if [ -n "$instance" ]; then
132+
echo "The $instance pod was not deleted from server inventory"
133+
exit 1
134+
fi
135+
done
136+
96137
if [[ -n ${OPENSHIFT} ]]; then
97138
oc adm policy remove-scc-from-user privileged -z pmm-server
98139
if [ -n "$OPERATOR_NS" ]; then

‎pkg/psmdb/pmm.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,5 +386,19 @@ func AddPMMContainer(ctx context.Context, cr *api.PerconaServerMongoDB, secret *
386386
pmmC.Env = append(pmmC.Env, sidecarEnvs...)
387387
}
388388

389+
if cr.CompareVersion("1.16.0") >= 0 {
390+
pmmC.Lifecycle = &corev1.Lifecycle{
391+
PreStop: &corev1.LifecycleHandler{
392+
Exec: &corev1.ExecAction{
393+
Command: []string{
394+
"bash",
395+
"-c",
396+
"pmm-admin unregister --force",
397+
},
398+
},
399+
},
400+
}
401+
}
402+
389403
return &pmmC
390404
}

0 commit comments

Comments
 (0)