Skip to content

Commit 165b97e

Browse files
egeguneshorsjvpasinattoeleo007gkech
authored
K8SPSMDB-1308: Improve physical restore logs (#1915)
* K8SPSMDB-1308: Improve physical restore logs * K8SPSMDB-1265: Update versions for 1.20.0 release (#1903) * K8SPSMDB-1265: Update versions for 1.20.0 release * Update test dependencies versions * Update cr images * Update Mongo 8 image to 8.0.8-3 * Fix compare files with correct PBM_MONGODB_URI (#1905) * K8SPSMDB-1265: update oc compare for rs-shard-migration (#1909) * K8SPSMDB-1265: clean up basmdb-backup and use kubectl wait for wait_restore (#1911) Use kubectl wait instead of regular loop in `wait_restore()` Add retry for `demand-backup-sharded` test backup presence in minio storage Delete backups during test cleanup before removing finalizers from objects. * Revert to use Mongo 7 as default * Print operator version (#1921) * K8SPSMDB-1265: Print operator version details * fix one-pod on azure * fix balancer on aks * K8SPSMDB-1268 pmm3 support (#1916) * K8SPSMDB-1268 pmm3 support * fix imports * assert the env var lengths * add server host * cover more inits in test * add custom params * improve test * remove spammy logs * e2e tests * fix mounts for pmm3 container * update secret with new token comment * wrapup e2e test * drop unused env vars * ensure that pmm3 test is fully functional * bonus: improve the custom name e2e verification * add small assertion to ensure that disabled pmm and nil secret return no container * make custom cluster name configurable in cr for the e2e test * fix linter * cr: package rename to config * add some more test cases * K8SPSMDB-1216 update to db.hello (#1929) * K8SPSMDB-1216 update to db.hello * update rs-shard-migration test --------- Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com> --------- Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com> Co-authored-by: Julio Pasinatto <julio.pasinatto@percona.com> Co-authored-by: Eleonora Zinchenko <eleonora.zinchenko@percona.com> Co-authored-by: George Kechagias <geo.kechagias@gmail.com> Co-authored-by: Natalia Marukovich <nmarukovich@gmail.com>
1 parent 0da913c commit 165b97e

File tree

2 files changed

+63
-7
lines changed

2 files changed

+63
-7
lines changed

build/physical-restore-ps-entry.sh

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,36 @@
11
#!/bin/bash
22

3-
set -Eeuo pipefail
3+
set -e
44
set -o xtrace
55

6-
log=/tmp/pbm-agent.log
6+
PBM_AGENT_LOG=/tmp/pbm-agent.log
7+
MONGOD_LOG=/tmp/mongod.log
8+
PHYSICAL_RESTORE_DIR=/data/db/pbm-restore-logs
9+
10+
function handle_sigterm() {
11+
echo "Received SIGTERM, cleaning up..."
12+
13+
mkdir ${PHYSICAL_RESTORE_DIR}
14+
mv pbm.restore.log.* ${PBM_AGENT_LOG} ${MONGOD_LOG} ${PHYSICAL_RESTORE_DIR}/
15+
16+
echo "Restore finished, you can find logs in ${PHYSICAL_RESTORE_DIR}"
17+
exit 0
18+
}
19+
20+
trap 'handle_sigterm' 15
721

822
touch /opt/percona/restore-in-progress
923

10-
/opt/percona/pbm-agent 1>&2 2>${log} &
11-
/opt/percona/ps-entry.sh "$@" 1>&2 2>/tmp/mongod.log
24+
/opt/percona/pbm-agent >${PBM_AGENT_LOG} 2>&1 &
25+
pbm_pid=$!
26+
27+
/opt/percona/ps-entry.sh "$@" >${MONGOD_LOG} 2>&1 &
28+
mongod_pid=$!
1229

13-
echo "Physical restore in progress"
14-
tail -n +1 -f ${log}
15-
sleep infinity
30+
set +o xtrace
31+
echo "Physical restore in progress... pbm-agent logs: ${PBM_AGENT_LOG} mongod logs: ${MONGOD_LOG}"
32+
echo "Script PID: $$, pbm-agent PID: $pbm_pid, mongod PID: $mongod_pid"
33+
while true; do
34+
echo "Still in progress at $(date)"
35+
sleep 10
36+
done

e2e-tests/functions

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,36 @@ simple_data_check() {
343343
fi
344344
}
345345

346+
get_mongod_pods() {
347+
local cluster=$1
348+
349+
kubectl_bin get pod \
350+
--no-headers \
351+
-l app.kubernetes.io/instance=${cluster} \
352+
-l app.kubernetes.io/component=mongod
353+
354+
}
355+
356+
collect_physical_restore_logs() {
357+
local cluster=$1
358+
local restore=$2
359+
360+
for pod in $(get_mongod_pods ${cluster}); do
361+
desc "pbm-agent logs from ${pod}"
362+
kubectl_bin exec -it ${pod} -- cat /tmp/pbm-agent.log || true
363+
done
364+
}
365+
366+
is_physical_backup() {
367+
local backup=$1
368+
369+
if [[ $(kubectl_bin get psmdb-backup ${backup} -o jsonpath={.status.type}) == "physical" ]]; then
370+
return 0
371+
fi
372+
373+
return 1
374+
}
375+
346376
wait_restore() {
347377
local backup_name=$1
348378
local cluster_name=$2
@@ -384,6 +414,11 @@ wait_restore() {
384414
| grep -v 'Getting tasks for pod' \
385415
| grep -v 'Getting pods from source' \
386416
| tail -100
417+
418+
if is_physical_backup ${backup_name}; then
419+
collect_physical_restore_logs
420+
fi
421+
387422
kubectl_bin get psmdb-restore restore-${backup_name} -o yaml
388423
kubectl_bin describe psmdb-restore restore-${backup_name}
389424
echo "Restore object restore-${backup_name} is in ${current_state} state."

0 commit comments

Comments
 (0)