diff --git a/.github/workflows/e2e-py-check.yml b/.github/workflows/e2e-py-check.yml
new file mode 100644
index 000000000..2627f3298
--- /dev/null
+++ b/.github/workflows/e2e-py-check.yml
@@ -0,0 +1,31 @@
+name: e2e-tests Python Quality Check
+
+on:
+ pull_request:
+ paths:
+ - 'e2e-tests/**/*.py'
+
+jobs:
+ quality-check:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v5
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version-file: "pyproject.toml"
+
+ - name: Install dependencies
+ run: uv sync --locked
+
+ - name: Run ruff check
+ run: uv run ruff check e2e-tests/
+
+ - name: Run mypy
+ run: uv run mypy e2e-tests/
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 2c8d6a1d7..69c56fdfa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -189,3 +189,7 @@ bin/
projects/
installers/olm/operator_*.yaml
installers/olm/bundles
+
+# Test Reports
+e2e-tests/reports/
+e2e-tests/**/__pycache__/
\ No newline at end of file
diff --git a/Jenkinsfile b/Jenkinsfile
index 846fda3e5..0763c7cfa 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -13,7 +13,20 @@ void createCluster(String CLUSTER_SUFFIX) {
gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE
gcloud config set project $GCP_PROJECT
gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true
- gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.30 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 --enable-ip-alias --workload-pool=cloud-dev-112233.svc.id.goog && \
+ gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} \
+ --cluster-version=1.32 \
+ --machine-type=n1-standard-4 \
+ --preemptible --disk-size 30 \
+ --num-nodes=\$NODES_NUM \
+ --network=jenkins-vpc \
+ --subnetwork=jenkins-${CLUSTER_SUFFIX} \
+ --no-enable-autoupgrade \
+ --cluster-ipv4-cidr=/21 \
+ --labels delete-cluster-after-hours=6 \
+ --enable-ip-alias \
+ --monitoring=NONE \
+ --logging=NONE \
+ --workload-pool=cloud-dev-112233.svc.id.goog && \
kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$?
if [ \${ret_val} -eq 0 ]; then break; fi
ret_num=\$((ret_num + 1))
@@ -130,50 +143,44 @@ void markPassedTests() {
}
}
-void printKubernetesStatus(String LOCATION, String CLUSTER_SUFFIX) {
- sh """
- export KUBECONFIG=/tmp/$CLUSTER_NAME-$CLUSTER_SUFFIX
- echo "========== KUBERNETES STATUS $LOCATION TEST =========="
- gcloud container clusters list|grep -E "NAME|$CLUSTER_NAME-$CLUSTER_SUFFIX "
- echo
- kubectl get nodes
- echo
- kubectl top nodes
- echo
- kubectl get pods --all-namespaces
- echo
- kubectl top pod --all-namespaces
- echo
- kubectl get events --field-selector type!=Normal --all-namespaces --sort-by=".lastTimestamp"
- echo "======================================================"
- """
+String formatTime(def time) {
+ if (!time || time == "N/A") return "N/A"
+
+ try {
+ println("Input time: ${time} (type: ${time.class})")
+ def totalSeconds = time as Double
+ println("Converted to double: ${totalSeconds}")
+
+ def hours = (totalSeconds / 3600) as Integer
+ def minutes = ((totalSeconds % 3600) / 60) as Integer
+ def seconds = (totalSeconds % 60) as Integer
+
+ return String.format("%02d:%02d:%02d", hours, minutes, seconds)
+
+ } catch (Exception e) {
+ println("Error converting time: ${e.message}")
+ return time.toString()
+ }
}
-TestsReport = '| Test name | Status |\r\n| ------------- | ------------- |'
-TestsReportXML = '\n'
+TestsReport = '| Test Name | Result | Time |\r\n| ----------- | -------- | ------ |'
void makeReport() {
- def wholeTestAmount=tests.size()
+ def wholeTestAmount = tests.size()
def startedTestAmount = 0
- for (int i=0; i<'+ testResult +'/>\n'
+ TestsReport = TestsReport + "\r\n| " + testName + " | [" + testResult + "](" + testUrl + ") | " + testTime + " |"
}
- TestsReport = TestsReport + "\r\n| We run $startedTestAmount out of $wholeTestAmount|"
- TestsReportXML = TestsReportXML + '\n'
-
- sh """
- echo "${TestsReportXML}" > TestsReport.xml
- """
+ TestsReport = TestsReport + "\r\n| We run $startedTestAmount out of $wholeTestAmount | | |"
}
void clusterRunner(String cluster) {
@@ -193,6 +200,8 @@ void clusterRunner(String cluster) {
if (clusterCreated >= 1) {
shutdownCluster(cluster)
+ // Re-check for passed tests after execution
+ markPassedTests()
}
}
@@ -215,7 +224,11 @@ void runTest(Integer TEST_ID) {
export DEBUG_TESTS=1
fi
export KUBECONFIG=/tmp/$CLUSTER_NAME-$clusterSuffix
- time ./e2e-tests/$testName/run
+
+ source \$HOME/.local/bin/env
+ uv run pytest e2e-tests/test_pytest_wrapper.py -v -s --test-regex "^${testName}\$" \
+ --html=e2e-tests/reports/$CLUSTER_NAME-$testName-report.html \
+ --junitxml=e2e-tests/reports/$CLUSTER_NAME-$testName-report.xml
"""
}
pushArtifactFile("${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}-$testName")
@@ -223,7 +236,6 @@ void runTest(Integer TEST_ID) {
return true
}
catch (exc) {
- printKubernetesStatus("AFTER","$clusterSuffix")
echo "Test $testName has failed!"
if (retryCount >= 1 || currentBuild.nextBuild != null) {
currentBuild.result = 'FAILURE'
@@ -234,7 +246,7 @@ void runTest(Integer TEST_ID) {
}
finally {
def timeStop = new Date().getTime()
- def durationSec = (timeStop - timeStart) / 1000
+ def durationSec = (timeStop - timeStart) / 1000.0
tests[TEST_ID]["time"] = durationSec
pushLogFile("$testName")
echo "The $testName test was finished!"
@@ -264,6 +276,11 @@ EOF
sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin
curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense
+
+ curl -LsSf https://astral.sh/uv/install.sh | sh
+ source \$HOME/.local/bin/env
+ uv python install 3.13
+ uv sync --locked
"""
}
@@ -499,7 +516,7 @@ pipeline {
}
}
options {
- timeout(time: 3, unit: 'HOURS')
+ timeout(time: 4, unit: 'HOURS')
}
parallel {
stage('cluster1') {
@@ -578,9 +595,28 @@ pipeline {
}
}
makeReport()
- step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0])
- archiveArtifacts '*.xml'
-
+ if (fileExists('e2e-tests/reports')){
+ sh """
+ source \$HOME/.local/bin/env
+ uv run pytest_html_merger -i e2e-tests/reports -o final_report.html
+ uv run junitparser merge --glob 'e2e-tests/reports/*.xml' final_report.xml
+ echo 'DEBUG: Files in reports folder'
+ ls e2e-tests/reports/*.html
+ ls e2e-tests/reports/*.xml
+ """
+ step([$class: 'JUnitResultArchiver', testResults: 'final_report.xml', healthScaleFactor: 1.0])
+ archiveArtifacts 'final_report.xml, final_report.html'
+ // Currently Html Publisher plugin is not available
+ // publishHTML (target : [allowMissing: true,
+ // alwaysLinkToLastBuild: true,
+ // keepAll: false,
+ // reportDir: '.',
+ // reportFiles: 'final_report.html',
+ // reportName: 'PSMDB Test Report',
+ // reportTitles: 'Test Report'])
+ } else {
+ echo "No report files found in e2e-tests/reports, skipping report generation"
+ }
unstash 'IMAGE'
def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim()
TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n"
diff --git a/e2e-tests/conftest.py b/e2e-tests/conftest.py
new file mode 100644
index 000000000..6113da958
--- /dev/null
+++ b/e2e-tests/conftest.py
@@ -0,0 +1,110 @@
+import re
+import pytest
+
+from pathlib import Path
+from typing import Tuple, List
+
+from tools.report_generator import generate_report
+from tools.k8s_resources_collector import collect_k8s_resources, get_namespace
+
+
+def pytest_addoption(parser):
+ """Add custom command line option for test suite file"""
+ parser.addoption(
+ "--test-suite",
+ action="store",
+ default=None,
+ help="Name of the test suite file (will look for run-{name}.csv)",
+ )
+ parser.addoption(
+ "--test-regex",
+ action="store",
+ default=None,
+ help="Run tests matching the given regex pattern",
+ )
+ parser.addoption(
+ "--collect-k8s-resources",
+ action="store_true",
+ default=False,
+ help="Enable collection of K8s resources on test failure",
+ )
+
+
+def get_bash_tests(test_suite: str = "") -> List[Tuple[str, Path]]:
+ """Get bash test scripts from file or all directories"""
+ current_dir = Path(__file__).parent
+ bash_tests: List[Tuple[str, Path]] = []
+
+ if test_suite:
+ file_path = current_dir / f"run-{test_suite}.csv"
+ if not file_path.exists():
+ raise FileNotFoundError(f"Test suite file not found: {file_path}")
+
+ with open(file_path, "r", encoding="utf-8") as f:
+ test_names = [line.strip() for line in f if line.strip()]
+ else:
+ test_names = [d.name for d in current_dir.iterdir() if d.is_dir()]
+
+ for test_name in test_names:
+ test_dir = current_dir / test_name
+ run_script = test_dir / "run"
+ if run_script.exists():
+ bash_tests.append((test_name, run_script))
+
+ return bash_tests
+
+
+def pytest_generate_tests(metafunc):
+ """Generate tests dynamically with regex filtering"""
+ if "test_name" in metafunc.fixturenames and "script_path" in metafunc.fixturenames:
+ test_suite = metafunc.config.getoption("--test-suite")
+ test_regex = metafunc.config.getoption("--test-regex")
+
+ bash_tests = get_bash_tests(test_suite)
+ if test_regex:
+ try:
+ pattern = re.compile(test_regex)
+ filtered_tests = [
+ (name, path) for name, path in bash_tests if pattern.search(name)
+ ]
+ bash_tests = filtered_tests
+
+ print(f"\nFiltered to {len(bash_tests)} test(s) matching regex '{test_regex}':")
+ for name, _ in bash_tests:
+ print(f" - {name}")
+
+ except re.error as e:
+ pytest.exit(f"Invalid regex pattern '{test_regex}': {e}")
+
+ metafunc.parametrize(
+ "test_name,script_path", bash_tests, ids=[name for name, _ in bash_tests]
+ )
+
+
+def pytest_html_report_title(report):
+ report.title = "PSMDB E2E Test Report"
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+ outcome = yield
+ report = outcome.get_result()
+
+ if report.when == "call" and report.failed:
+ try:
+ namespace = get_namespace(str(report.longrepr))
+ html_report = generate_report(namespace)
+
+ if not hasattr(report, "extras"):
+ report.extras = []
+ report.extras.extend(html_report)
+
+ collect_resources = item.config.getoption("--collect-k8s-resources")
+ if collect_resources:
+ collect_k8s_resources(
+ namespace=namespace,
+ custom_resources=["psmdb", "psmdb-backup", "psmdb-restore"],
+ output_dir=f"e2e-tests/reports/{namespace}",
+ )
+ except Exception as e:
+ print(f"Error adding K8s info: {e}")
diff --git a/e2e-tests/functions b/e2e-tests/functions
index e7c26143a..f588c16f3 100755
--- a/e2e-tests/functions
+++ b/e2e-tests/functions
@@ -131,7 +131,7 @@ create_namespace() {
destroy_chaos_mesh
desc 'cleaned up all old namespaces'
kubectl_bin get ns \
- | egrep -v "^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME" \
+ | egrep -v "^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME" \
| awk '{print$1}' \
| xargs kubectl delete ns &
fi
diff --git a/e2e-tests/test_pytest_wrapper.py b/e2e-tests/test_pytest_wrapper.py
new file mode 100644
index 000000000..c21bba6a7
--- /dev/null
+++ b/e2e-tests/test_pytest_wrapper.py
@@ -0,0 +1,36 @@
+import os
+import subprocess
+import pytest
+from pathlib import Path
+
+
+def test_e2e(test_name: str, script_path: Path) -> None:
+ """Run bash script with live output and capture for error reporting"""
+ original_cwd = os.getcwd()
+ script_dir = script_path.parent
+
+ try:
+ os.chdir(script_dir)
+ process = subprocess.Popen(
+ ["bash", "run"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+ )
+
+ output = []
+ if process.stdout is not None:
+ for line in iter(process.stdout.readline, ""):
+ print(line, end="")
+ output.append(line)
+
+ process.wait()
+
+ if process.returncode != 0:
+ error_msg = f"""
+Test {test_name} failed with exit code {process.returncode}
+
+OUTPUT:
+{"".join(output)}
+"""
+ pytest.fail(error_msg)
+
+ finally:
+ os.chdir(original_cwd)
diff --git a/e2e-tests/tools/__init__.py b/e2e-tests/tools/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/e2e-tests/tools/k8s_resources_collector.py b/e2e-tests/tools/k8s_resources_collector.py
new file mode 100644
index 000000000..ff539aab7
--- /dev/null
+++ b/e2e-tests/tools/k8s_resources_collector.py
@@ -0,0 +1,330 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import sys
+import tarfile
+import threading
+import subprocess
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, Future, as_completed
+from typing import List, Optional
+
+
+class K8sCollector:
+ def __init__(self, namespace: str, custom_resources: Optional[List[str]] = None):
+ self.namespace = namespace
+ self.custom_resources = custom_resources or []
+ self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ self.output_dir = f"{namespace}_{self.timestamp}"
+ self.error_log_file = ""
+ self.lock = threading.Lock()
+
+ def run_kubectl(
+ self, args: List[str], capture_output: bool = True, text: bool = True
+ ) -> Optional[subprocess.CompletedProcess]:
+ """Run kubectl command and return result"""
+ try:
+ result = subprocess.run(
+ ["kubectl"] + args, capture_output=capture_output, text=text, check=False
+ )
+ return result
+ except Exception as e:
+ print(f"Error running kubectl command: {e}")
+ return None
+
+ def setup_directories(self) -> None:
+ """Create output directory structure"""
+ print(f"Creating output directory: {self.output_dir}")
+
+ directories = [
+ self.output_dir,
+ f"{self.output_dir}/logs",
+ f"{self.output_dir}/describe",
+ f"{self.output_dir}/events",
+ f"{self.output_dir}/get",
+ ]
+
+ for directory in directories:
+ os.makedirs(directory, exist_ok=True)
+
+ self.error_log_file = f"{self.output_dir}/error_summary.log"
+ with open(self.error_log_file, "w") as f:
+ f.write(
+ f"Error Log Summary for Namespace: {self.namespace} (Extracted on {self.timestamp})\n"
+ )
+ f.write("=" * 73 + "\n\n")
+
+ def get_resource_names(self, resource_type: str) -> List[str]:
+ """Get list of resource names for a given type"""
+ result = self.run_kubectl(["get", resource_type, "-n", self.namespace, "-o", "name"])
+ if result and result.returncode == 0 and result.stdout.strip():
+ return [line.split("/")[-1] for line in result.stdout.strip().split("\n")]
+ return []
+
+ def process_resource_type(self, resource_type: str, singular: str, plural: str) -> List[str]:
+ """Process a specific resource type"""
+ print(f"Extracting {resource_type} information...")
+
+ get_dir = f"{self.output_dir}/get/{plural}"
+ os.makedirs(get_dir, exist_ok=True)
+
+ result = self.run_kubectl(["get", plural, "-n", self.namespace, "-o", "wide"])
+ with open(f"{get_dir}/{plural}.txt", "w") as f:
+ if result and result.returncode == 0:
+ f.write(result.stdout)
+ else:
+ f.write(f"No {resource_type}s found\n")
+
+ resources = self.get_resource_names(plural)
+ with ThreadPoolExecutor(max_workers=5) as executor:
+ futures = []
+ for resource in resources:
+ print(f"Processing {resource_type}: {resource}")
+ futures.append(executor.submit(self.describe_resource, singular, resource))
+ futures.append(
+ executor.submit(self.get_resource_yaml, singular, resource, get_dir)
+ )
+
+ # Wait for all tasks to complete
+ for future in as_completed(futures):
+ try:
+ future.result()
+ except Exception as e:
+ print(f"Error processing resource: {e}")
+
+ return resources
+
+ def describe_resource(self, resource_type: str, resource_name: str) -> None:
+ """Describe a specific resource"""
+ result = self.run_kubectl(["describe", resource_type, resource_name, "-n", self.namespace])
+ if result:
+ with open(f"{self.output_dir}/describe/{resource_type}_{resource_name}.txt", "w") as f:
+ f.write(result.stdout)
+
+ def get_resource_yaml(self, resource_type: str, resource_name: str, output_dir: str) -> None:
+ """Get resource YAML"""
+ result = self.run_kubectl(
+ ["get", resource_type, resource_name, "-n", self.namespace, "-o", "yaml"]
+ )
+ if result:
+ with open(f"{output_dir}/{resource_name}.yaml", "w") as f:
+ f.write(result.stdout)
+
+ def process_custom_resource(self, resource: str) -> None:
+ """Process custom resource"""
+ print(f"Extracting custom resource: {resource}...")
+
+ api_result = self.run_kubectl(["api-resources"])
+ if not api_result or resource not in api_result.stdout:
+ print(f"Warning: Custom resource '{resource}' not found in the cluster. Skipping.")
+ return
+
+ get_dir = f"{self.output_dir}/get/{resource}"
+ os.makedirs(get_dir, exist_ok=True)
+
+ check_result = self.run_kubectl(["get", resource, "-n", self.namespace])
+ if not check_result or check_result.returncode != 0:
+ print(f"No resources of type '{resource}' found in namespace {self.namespace}")
+ return
+
+ result = self.run_kubectl(["get", resource, "-n", self.namespace, "-o", "wide"])
+ with open(f"{get_dir}/{resource}.txt", "w") as f:
+ f.write(result.stdout if result else "")
+
+ resources = self.get_resource_names(resource)
+
+ with ThreadPoolExecutor(max_workers=5) as executor:
+ futures = []
+
+ for resource_name in resources:
+ print(f"Processing custom resource {resource}: {resource_name}")
+
+ futures.append(executor.submit(self.describe_resource, resource, resource_name))
+ futures.append(
+ executor.submit(self.get_resource_yaml, resource, resource_name, get_dir)
+ )
+
+ for future in as_completed(futures):
+ try:
+ future.result()
+ except Exception as e:
+ print(f"Error processing custom resource: {e}")
+
+ def extract_pod_logs(self, pod_name: str, container_name: str) -> None:
+ """Extract logs for a specific container"""
+ print(f" Extracting logs for container: {container_name}")
+
+ log_dir = f"{self.output_dir}/logs/{pod_name}"
+ os.makedirs(log_dir, exist_ok=True)
+
+ result = self.run_kubectl(["logs", pod_name, "-c", container_name, "-n", self.namespace])
+ log_file = f"{log_dir}/{container_name}.log"
+
+ with open(log_file, "w") as f:
+ if result:
+ f.write(result.stdout)
+
+ self.extract_error_logs(pod_name, container_name, log_file)
+
+ def extract_error_logs(self, pod_name: str, container_name: str, log_file: str) -> None:
+ """Extract error logs from container log file"""
+ try:
+ with open(log_file, "r") as f:
+ content = f.read()
+
+ if content and "error" in content.lower():
+ error_lines = [line for line in content.split("\n") if "error" in line.lower()]
+
+ if error_lines:
+ with self.lock:
+ with open(self.error_log_file, "a") as f:
+ f.write(
+ f"=== Error logs from pod: {pod_name}, container: {container_name} ===\n\n"
+ )
+ for line in error_lines:
+ f.write(f"{line}\n")
+ f.write("\n" + "-" * 48 + "\n\n")
+
+ except Exception as e:
+ print(f"Error extracting error logs: {e}")
+
+ def get_container_names(self, pod_name: str) -> List[str]:
+ """Get container names for a pod"""
+ result = self.run_kubectl(
+ [
+ "get",
+ "pod",
+ pod_name,
+ "-n",
+ self.namespace,
+ "-o",
+ "jsonpath={.spec.containers[*].name}",
+ ]
+ )
+
+ if result and result.returncode == 0 and result.stdout.strip():
+ return result.stdout.strip().split()
+ return []
+
+ def process_pods(self) -> None:
+ """Process all pods in the namespace"""
+ print("Extracting Pod information...")
+
+ pods_dir = f"{self.output_dir}/get/pods"
+ os.makedirs(pods_dir, exist_ok=True)
+
+ result = self.run_kubectl(["get", "pods", "-n", self.namespace, "-o", "wide"])
+ with open(f"{pods_dir}/pods.txt", "w") as f:
+ if result and result.returncode == 0:
+ f.write(result.stdout)
+ else:
+ f.write("No Pods found\n")
+
+ pods = self.get_resource_names("pods")
+
+ with ThreadPoolExecutor(max_workers=3) as executor:
+ futures = []
+
+ for pod in pods:
+ print(f"Processing pod: {pod}")
+
+ futures.append(executor.submit(self.describe_resource, "pod", pod))
+ futures.append(executor.submit(self.get_resource_yaml, "pod", pod, pods_dir))
+
+ containers = self.get_container_names(pod)
+ for container in containers:
+ futures.append(executor.submit(self.extract_pod_logs, pod, container))
+
+ # Wait for all tasks
+ for future in as_completed(futures):
+ try:
+ future.result()
+ except Exception as e:
+ print(f"Error processing pod: {e}")
+
+ def extract_namespace_events(self) -> None:
+ """Extract namespace events"""
+ print(f"Extracting events for namespace: {self.namespace}")
+
+ events_dir = f"{self.output_dir}/events"
+
+ print("Getting events...")
+ result = self.run_kubectl(["get", "events", "-n", self.namespace, "-o", "wide"])
+ with open(f"{events_dir}/events.txt", "w") as f:
+ f.write(result.stdout if result else "")
+
+ print("Getting events in JSON format...")
+ result = self.run_kubectl(["get", "events", "-n", self.namespace, "-o", "json"])
+ with open(f"{events_dir}/events.json", "w") as f:
+ f.write(result.stdout if result else "")
+
+ def collect_all(self) -> None:
+ """Main collection method"""
+ print(f"=== Starting extraction for namespace: {self.namespace} ===")
+
+ self.setup_directories()
+ self.process_pods()
+ with ThreadPoolExecutor(max_workers=6) as executor:
+ futures: List[Future] = [
+ executor.submit(
+ self.process_resource_type, "StatefulSet", "statefulset", "statefulsets"
+ ),
+ executor.submit(
+ self.process_resource_type, "Deployment", "deployment", "deployments"
+ ),
+ executor.submit(self.process_resource_type, "Secret", "secret", "secrets"),
+ executor.submit(self.process_resource_type, "Job", "job", "jobs"),
+ executor.submit(
+ self.process_resource_type, "ConfigMap", "configmap", "configmaps"
+ ),
+ executor.submit(self.process_resource_type, "Service", "service", "services"),
+ executor.submit(self.extract_namespace_events),
+ ]
+
+ for future in as_completed(futures):
+ try:
+ future.result()
+ except Exception as e:
+ print(f"Error in parallel processing: {e}")
+
+ if self.custom_resources:
+ print("Processing custom resources...")
+ for resource in self.custom_resources:
+ resource = resource.strip()
+ if resource:
+ self.process_custom_resource(resource)
+
+ print("=== Extraction completed successfully ===")
+ print(f"Output is available in: {self.output_dir}")
+ print(f"Error log summary: {self.error_log_file}")
+
+
+def collect_k8s_resources(
+ namespace: str, custom_resources: Optional[List[str]] = None, output_dir: Optional[str] = None
+) -> None:
+ """
+ Collect Kubernetes resources for a given namespace.
+ """
+ collector = K8sCollector(namespace, custom_resources)
+
+ if output_dir:
+ collector.output_dir = f"{output_dir}_{collector.timestamp}"
+
+ try:
+ collector.collect_all()
+ with tarfile.open(f"{collector.output_dir}.tar.gz", "w:gz") as tar:
+ tar.add(collector.output_dir, arcname=os.path.basename(collector.output_dir))
+ except Exception as e:
+ print(f"error collecting from namespace: {namespace} - {str(e)}")
+
+
+def get_namespace(test_log: str) -> str:
+ """Extract namespace from test log"""
+ match = re.search(r"create namespace (\S*?\d+)\b", test_log)
+ if match:
+ namespace = match.group(1)
+ print(f"Extracted namespace: {namespace}", file=sys.stderr)
+ return namespace
+
+ raise ValueError("Namespace not found in logs")
diff --git a/e2e-tests/tools/report_generator.py b/e2e-tests/tools/report_generator.py
new file mode 100644
index 000000000..531ecd349
--- /dev/null
+++ b/e2e-tests/tools/report_generator.py
@@ -0,0 +1,122 @@
+import subprocess
+
+from pytest_html import extras
+
+
+def run_kubectl_commands(commands: list) -> str:
+ """Execute kubectl commands and return formatted output"""
+ output = []
+
+ for title, cmd in commands:
+ output.append(f"\n=== {title} ===")
+ try:
+ result = subprocess.run(cmd.split(), capture_output=True, text=True, timeout=15)
+ if result.returncode == 0:
+ output.append(result.stdout.strip())
+ else:
+ output.append(f"Error (exit code {result.returncode}): {result.stderr.strip()}")
+ except subprocess.TimeoutExpired:
+ output.append("Command timed out after 15 seconds")
+ except Exception as e:
+ output.append(f"Failed to execute: {str(e)}")
+
+ return "\n".join(output)
+
+
+def capture_k8s_resources(namespace: str) -> str:
+ """Capture Kubernetes resources information"""
+ commands = [
+ ("Nodes", "kubectl get nodes"),
+ (f"All from namespace {namespace}", f"kubectl get all -n {namespace}"),
+ ("Secrets", f"kubectl get secrets -n {namespace}"),
+ (
+ "PSMDB Cluster",
+ f"kubectl get psmdb -n {namespace} -o custom-columns=NAME:.metadata.name,STATE:.status.state",
+ ),
+ ("PSMDB Backup", f"kubectl get psmdb-backup -n {namespace}"),
+ ("PSMDB Restore", f"kubectl get psmdb-restore -n {namespace}"),
+ ]
+ return run_kubectl_commands(commands)
+
+
+def capture_k8s_logs(namespace: str) -> str:
+ """Capture Kubernetes logs"""
+ commands = [
+ (
+ "PSMDB Operator Logs",
+ "kubectl logs -l app.kubernetes.io/name=percona-server-mongodb-operator --tail=50 -n psmdb-operator",
+ ),
+ ]
+ return run_kubectl_commands(commands)
+
+
+def capture_k8s_events(namespace: str) -> str:
+ """Capture Kubernetes events"""
+ commands = [
+ ("PSMDB Operator Events", f"kubectl get events -n {namespace}"),
+ ]
+ return run_kubectl_commands(commands)
+
+
+def highlight_log_levels(logs: str) -> str:
+ """Add basic color highlighting for common log levels"""
+ logs = logs.replace("ERROR", 'ERROR')
+ logs = logs.replace("WARN", 'WARN')
+ logs = logs.replace("INFO", 'INFO')
+ logs = logs.replace("DEBUG", 'DEBUG')
+ logs = logs.replace("Normal", 'Normal')
+ logs = logs.replace("Warning", 'Warning')
+ return logs
+
+
+def generate_report(namespace: str) -> list:
+ def create_collapsible_section(title: str, icon: str, content: str) -> str:
+ return f"""
+
+
+
+ {icon} {title}
+
+
+
+