feat: run containerdebug in the background

Author: razvan

CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 
 - The lifetime of auto generated TLS certificates is now configurable with the role and roleGroup
   config property `requestedSecretLifetime`. This helps reducing frequent Pod restarts ([#619]).
+- Run a `containerdebug` process in the background of each HDFS container to collect debugging information ([#629]).
 
 ### Fixed
 
@@ -17,6 +18,7 @@ All notable changes to this project will be documented in this file.
 
 [#616]: https://github.com/stackabletech/hdfs-operator/pull/616
 [#619]: https://github.com/stackabletech/hdfs-operator/pull/619
+[#629]: https://github.com/stackabletech/hdfs-operator/pull/629
 
 ## [24.11.0] - 2024-11-18
 

rust/operator-binary/src/container.rs

@@ -612,6 +612,7 @@ impl ContainerConfig {
 {COMMON_BASH_TRAP_FUNCTIONS}
 {remove_vector_shutdown_file_command}
 prepare_signal_handlers
+CONTAINERDEBUG_LOG_DIRECTORY={STACKABLE_LOG_DIR}/containerdebug containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &
 if [[ -d {LISTENER_VOLUME_DIR} ]]; then
     export POD_ADDRESS=$(cat {LISTENER_VOLUME_DIR}/default-address/address)
     for i in {LISTENER_VOLUME_DIR}/default-address/ports/*; do

tests/templates/kuttl/smoke/30-assert.yaml.j2

@@ -143,3 +143,12 @@ status:
   expectedPods: {{ test_scenario['values']['number-of-datanodes'] }}
   currentHealthy: {{ test_scenario['values']['number-of-datanodes'] }}
   disruptionsAllowed: 1
+---
+# This test checks if the containerdebug-state.json file is present and valid
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+commands:
+  - script: kubectl exec -n $NAMESPACE --container namenode hdfs-namenode-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status
+  - script: kubectl exec -n $NAMESPACE --container datanode hdfs-datanode-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status
+  - script: kubectl exec -n $NAMESPACE --container journalnode hdfs-journalnode-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status