From a60244219b26106e8ac79c6e9eef82e49b30240e Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Fri, 6 Jun 2025 15:26:48 -0300 Subject: [PATCH 01/11] K8SPSMDB-1192: Introduce e2e-test pytest wrapper --- .github/workflows/e2e-py-check.yml | 32 ++++ .gitignore | 3 + e2e-tests/test_pytest_wrapper.py | 55 +++++++ pyproject.toml | 16 ++ uv.lock | 255 +++++++++++++++++++++++++++++ 5 files changed, 361 insertions(+) create mode 100644 .github/workflows/e2e-py-check.yml create mode 100644 e2e-tests/test_pytest_wrapper.py create mode 100644 pyproject.toml create mode 100644 uv.lock diff --git a/.github/workflows/e2e-py-check.yml b/.github/workflows/e2e-py-check.yml new file mode 100644 index 000000000..3998b1a40 --- /dev/null +++ b/.github/workflows/e2e-py-check.yml @@ -0,0 +1,32 @@ +name: E2E Python Quality Check + +on: + push: + paths: + - 'e2e-tests/**/*.py' + pull_request: + paths: + - 'e2e-tests/**/*.py' + +jobs: + quality-check: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Set up Python + run: uv python install 3.13 + + - name: Install dependencies + run: uv sync + + - name: Run ruff check + run: uv run ruff check e2e-tests/ + + - name: Run mypy + run: uv run mypy e2e-tests/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2c8d6a1d7..089f67b09 100644 --- a/.gitignore +++ b/.gitignore @@ -189,3 +189,6 @@ bin/ projects/ installers/olm/operator_*.yaml installers/olm/bundles + +# Test Reports +e2e-tests/reports/ \ No newline at end of file diff --git a/e2e-tests/test_pytest_wrapper.py b/e2e-tests/test_pytest_wrapper.py new file mode 100644 index 000000000..1338cbcfa --- /dev/null +++ b/e2e-tests/test_pytest_wrapper.py @@ -0,0 +1,55 @@ +import os +import subprocess +from pathlib import Path +from typing import List, Tuple + +import pytest + + +def get_bash_tests() -> List[Tuple[str, Path]]: + """Find all bash test scripts in the same directory as this test file""" + current_dir = Path(__file__).parent + bash_tests: List[Tuple[str, Path]] = [] + + for test_dir in current_dir.iterdir(): + if test_dir.is_dir(): + run_script = test_dir / "run" + if run_script.exists(): + bash_tests.append((test_dir.name, run_script)) + + return bash_tests + + +bash_tests = get_bash_tests() + + +@pytest.mark.parametrize( + "test_name,script_path", bash_tests, ids=[name for name, _ in bash_tests] +) +def test_e2e(test_name: str, script_path: Path) -> None: + """Run bash script and check exit code""" + + original_cwd: str = os.getcwd() + script_dir: Path = script_path.parent + + try: + os.chdir(script_dir) + result: subprocess.CompletedProcess[str] = subprocess.run( + ["bash", "run"], capture_output=True, text=True + ) + + if result.returncode != 0: + print(f"\nSTDOUT:\n{result.stdout}") + print(f"\nSTDERR:\n{result.stderr}") + + k8s_result: subprocess.CompletedProcess[str] = subprocess.run( + ["kubectl", "get", "nodes"], capture_output=True, text=True + ) + print(f"\nK8s LOGS:\n{k8s_result.stdout}") + + assert result.returncode == 0, ( + f"Test {test_name} failed with exit code {result.returncode}" + ) + + finally: + os.chdir(original_cwd) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..a7574ff29 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "psmdb-pytest" +version = "0.1.0" +description = "Tests for PSMDB Operator" +requires-python = ">=3.13" +dependencies = [ + "mypy>=1.16.0", + "pytest>=8.4.0", + "pytest-html>=4.1.1", + "pytest-json-report>=1.5.0", + "pyyaml>=6.0.2", + "ruff>=0.11.12", +] + +[tool.pytest.ini_options] +addopts = "--html=e2e-tests/reports/report.html --self-contained-html --json-report --json-report-file=e2e-tests/reports/report.json --junitxml=e2e-tests/reports/report.xml" \ No newline at end of file diff --git a/uv.lock b/uv.lock new file mode 100644 index 000000000..d63a90f1c --- /dev/null +++ b/uv.lock @@ -0,0 +1,255 @@ +version = 1 +revision = 1 +requires-python = ">=3.13" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050 }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899 }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/0e/67eb10a7ecc77a0c2bbe2b0235765b98d164d81600746914bebada795e97/MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", size = 14274 }, + { url = "https://files.pythonhosted.org/packages/2b/6d/9409f3684d3335375d04e5f05744dfe7e9f120062c9857df4ab490a1031a/MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", size = 12352 }, + { url = "https://files.pythonhosted.org/packages/d2/f5/6eadfcd3885ea85fe2a7c128315cc1bb7241e1987443d78c8fe712d03091/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", size = 24122 }, + { url = "https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", size = 23085 }, + { url = "https://files.pythonhosted.org/packages/c2/cf/c9d56af24d56ea04daae7ac0940232d31d5a8354f2b457c6d856b2057d69/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", size = 22978 }, + { url = "https://files.pythonhosted.org/packages/2a/9f/8619835cd6a711d6272d62abb78c033bda638fdc54c4e7f4272cf1c0962b/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", size = 24208 }, + { url = "https://files.pythonhosted.org/packages/f9/bf/176950a1792b2cd2102b8ffeb5133e1ed984547b75db47c25a67d3359f77/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", size = 23357 }, + { url = "https://files.pythonhosted.org/packages/ce/4f/9a02c1d335caabe5c4efb90e1b6e8ee944aa245c1aaaab8e8a618987d816/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", size = 23344 }, + { url = "https://files.pythonhosted.org/packages/ee/55/c271b57db36f748f0e04a759ace9f8f759ccf22b4960c270c78a394f58be/MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", size = 15101 }, + { url = "https://files.pythonhosted.org/packages/29/88/07df22d2dd4df40aba9f3e402e6dc1b8ee86297dddbad4872bd5e7b0094f/MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", size = 15603 }, + { url = "https://files.pythonhosted.org/packages/62/6a/8b89d24db2d32d433dffcd6a8779159da109842434f1dd2f6e71f32f738c/MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", size = 14510 }, + { url = "https://files.pythonhosted.org/packages/7a/06/a10f955f70a2e5a9bf78d11a161029d278eeacbd35ef806c3fd17b13060d/MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", size = 12486 }, + { url = "https://files.pythonhosted.org/packages/34/cf/65d4a571869a1a9078198ca28f39fba5fbb910f952f9dbc5220afff9f5e6/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", size = 25480 }, + { url = "https://files.pythonhosted.org/packages/0c/e3/90e9651924c430b885468b56b3d597cabf6d72be4b24a0acd1fa0e12af67/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", size = 23914 }, + { url = "https://files.pythonhosted.org/packages/66/8c/6c7cf61f95d63bb866db39085150df1f2a5bd3335298f14a66b48e92659c/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", size = 23796 }, + { url = "https://files.pythonhosted.org/packages/bb/35/cbe9238ec3f47ac9a7c8b3df7a808e7cb50fe149dc7039f5f454b3fba218/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", size = 25473 }, + { url = "https://files.pythonhosted.org/packages/e6/32/7621a4382488aa283cc05e8984a9c219abad3bca087be9ec77e89939ded9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", size = 24114 }, + { url = "https://files.pythonhosted.org/packages/0d/80/0985960e4b89922cb5a0bac0ed39c5b96cbc1a536a99f30e8c220a996ed9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", size = 24098 }, + { url = "https://files.pythonhosted.org/packages/82/78/fedb03c7d5380df2427038ec8d973587e90561b2d90cd472ce9254cf348b/MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", size = 15208 }, + { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739 }, +] + +[[package]] +name = "mypy" +version = "1.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/38/13c2f1abae94d5ea0354e146b95a1be9b2137a0d506728e0da037c4276f6/mypy-1.16.0.tar.gz", hash = "sha256:84b94283f817e2aa6350a14b4a8fb2a35a53c286f97c9d30f53b63620e7af8ab", size = 3323139 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/9c/ca03bdbefbaa03b264b9318a98950a9c683e06472226b55472f96ebbc53d/mypy-1.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a9e056237c89f1587a3be1a3a70a06a698d25e2479b9a2f57325ddaaffc3567b", size = 11059753 }, + { url = "https://files.pythonhosted.org/packages/36/92/79a969b8302cfe316027c88f7dc6fee70129490a370b3f6eb11d777749d0/mypy-1.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0b07e107affb9ee6ce1f342c07f51552d126c32cd62955f59a7db94a51ad12c0", size = 10073338 }, + { url = "https://files.pythonhosted.org/packages/14/9b/a943f09319167da0552d5cd722104096a9c99270719b1afeea60d11610aa/mypy-1.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6fb60cbd85dc65d4d63d37cb5c86f4e3a301ec605f606ae3a9173e5cf34997b", size = 11827764 }, + { url = "https://files.pythonhosted.org/packages/ec/64/ff75e71c65a0cb6ee737287c7913ea155845a556c64144c65b811afdb9c7/mypy-1.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7e32297a437cc915599e0578fa6bc68ae6a8dc059c9e009c628e1c47f91495d", size = 12701356 }, + { url = "https://files.pythonhosted.org/packages/0a/ad/0e93c18987a1182c350f7a5fab70550852f9fabe30ecb63bfbe51b602074/mypy-1.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:afe420c9380ccec31e744e8baff0d406c846683681025db3531b32db56962d52", size = 12900745 }, + { url = "https://files.pythonhosted.org/packages/28/5d/036c278d7a013e97e33f08c047fe5583ab4f1fc47c9a49f985f1cdd2a2d7/mypy-1.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:55f9076c6ce55dd3f8cd0c6fff26a008ca8e5131b89d5ba6d86bd3f47e736eeb", size = 9572200 }, + { url = "https://files.pythonhosted.org/packages/99/a3/6ed10530dec8e0fdc890d81361260c9ef1f5e5c217ad8c9b21ecb2b8366b/mypy-1.16.0-py3-none-any.whl", hash = "sha256:29e1499864a3888bca5c1542f2d7232c6e586295183320caa95758fc84034031", size = 2265773 }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963 }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469 }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, +] + +[[package]] +name = "psmdb-pytest" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "mypy" }, + { name = "pytest" }, + { name = "pytest-html" }, + { name = "pytest-json-report" }, + { name = "pyyaml" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "mypy", specifier = ">=1.16.0" }, + { name = "pytest", specifier = ">=8.4.0" }, + { name = "pytest-html", specifier = ">=4.1.1" }, + { name = "pytest-json-report", specifier = ">=1.5.0" }, + { name = "pyyaml", specifier = ">=6.0.2" }, + { name = "ruff", specifier = ">=0.11.12" }, +] + +[[package]] +name = "pygments" +version = "2.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 }, +] + +[[package]] +name = "pytest" +version = "8.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/aa/405082ce2749be5398045152251ac69c0f3578c7077efc53431303af97ce/pytest-8.4.0.tar.gz", hash = "sha256:14d920b48472ea0dbf68e45b96cd1ffda4705f33307dcc86c676c1b5104838a6", size = 1515232 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/de/afa024cbe022b1b318a3d224125aa24939e99b4ff6f22e0ba639a2eaee47/pytest-8.4.0-py3-none-any.whl", hash = "sha256:f40f825768ad76c0977cbacdf1fd37c6f7a468e460ea6a0636078f8972d4517e", size = 363797 }, +] + +[[package]] +name = "pytest-html" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "pytest" }, + { name = "pytest-metadata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/ab/4862dcb5a8a514bd87747e06b8d55483c0c9e987e1b66972336946e49b49/pytest_html-4.1.1.tar.gz", hash = "sha256:70a01e8ae5800f4a074b56a4cb1025c8f4f9b038bba5fe31e3c98eb996686f07", size = 150773 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c7/c160021cbecd956cc1a6f79e5fe155f7868b2e5b848f1320dad0b3e3122f/pytest_html-4.1.1-py3-none-any.whl", hash = "sha256:c8152cea03bd4e9bee6d525573b67bbc6622967b72b9628dda0ea3e2a0b5dd71", size = 23491 }, +] + +[[package]] +name = "pytest-json-report" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "pytest-metadata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/d3/765dae9712fcd68d820338908c1337e077d5fdadccd5cacf95b9b0bea278/pytest-json-report-1.5.0.tar.gz", hash = "sha256:2dde3c647851a19b5f3700729e8310a6e66efb2077d674f27ddea3d34dc615de", size = 21241 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/35/d07400c715bf8a88aa0c1ee9c9eb6050ca7fe5b39981f0eea773feeb0681/pytest_json_report-1.5.0-py3-none-any.whl", hash = "sha256:9897b68c910b12a2e48dd849f9a284b2c79a732a8a9cb398452ddd23d3c8c325", size = 13222 }, +] + +[[package]] +name = "pytest-metadata" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/85/8c969f8bec4e559f8f2b958a15229a35495f5b4ce499f6b865eac54b878d/pytest_metadata-3.1.1.tar.gz", hash = "sha256:d2a29b0355fbc03f168aa96d41ff88b1a3b44a3b02acbe491801c98a048017c8", size = 9952 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/43/7e7b2ec865caa92f67b8f0e9231a798d102724ca4c0e1f414316be1c1ef2/pytest_metadata-3.1.1-py3-none-any.whl", hash = "sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b", size = 11428 }, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309 }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679 }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428 }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361 }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523 }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660 }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597 }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527 }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446 }, +] + +[[package]] +name = "ruff" +version = "0.11.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/da/9c6f995903b4d9474b39da91d2d626659af3ff1eeb43e9ae7c119349dba6/ruff-0.11.13.tar.gz", hash = "sha256:26fa247dc68d1d4e72c179e08889a25ac0c7ba4d78aecfc835d49cbfd60bf514", size = 4282054 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/ce/a11d381192966e0b4290842cc8d4fac7dc9214ddf627c11c1afff87da29b/ruff-0.11.13-py3-none-linux_armv6l.whl", hash = "sha256:4bdfbf1240533f40042ec00c9e09a3aade6f8c10b6414cf11b519488d2635d46", size = 10292516 }, + { url = "https://files.pythonhosted.org/packages/78/db/87c3b59b0d4e753e40b6a3b4a2642dfd1dcaefbff121ddc64d6c8b47ba00/ruff-0.11.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:aef9c9ed1b5ca28bb15c7eac83b8670cf3b20b478195bd49c8d756ba0a36cf48", size = 11106083 }, + { url = "https://files.pythonhosted.org/packages/77/79/d8cec175856ff810a19825d09ce700265f905c643c69f45d2b737e4a470a/ruff-0.11.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:53b15a9dfdce029c842e9a5aebc3855e9ab7771395979ff85b7c1dedb53ddc2b", size = 10436024 }, + { url = "https://files.pythonhosted.org/packages/8b/5b/f6d94f2980fa1ee854b41568368a2e1252681b9238ab2895e133d303538f/ruff-0.11.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab153241400789138d13f362c43f7edecc0edfffce2afa6a68434000ecd8f69a", size = 10646324 }, + { url = "https://files.pythonhosted.org/packages/6c/9c/b4c2acf24ea4426016d511dfdc787f4ce1ceb835f3c5fbdbcb32b1c63bda/ruff-0.11.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c51f93029d54a910d3d24f7dd0bb909e31b6cd989a5e4ac513f4eb41629f0dc", size = 10174416 }, + { url = "https://files.pythonhosted.org/packages/f3/10/e2e62f77c65ede8cd032c2ca39c41f48feabedb6e282bfd6073d81bb671d/ruff-0.11.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1808b3ed53e1a777c2ef733aca9051dc9bf7c99b26ece15cb59a0320fbdbd629", size = 11724197 }, + { url = "https://files.pythonhosted.org/packages/bb/f0/466fe8469b85c561e081d798c45f8a1d21e0b4a5ef795a1d7f1a9a9ec182/ruff-0.11.13-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:d28ce58b5ecf0f43c1b71edffabe6ed7f245d5336b17805803312ec9bc665933", size = 12511615 }, + { url = "https://files.pythonhosted.org/packages/17/0e/cefe778b46dbd0cbcb03a839946c8f80a06f7968eb298aa4d1a4293f3448/ruff-0.11.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55e4bc3a77842da33c16d55b32c6cac1ec5fb0fbec9c8c513bdce76c4f922165", size = 12117080 }, + { url = "https://files.pythonhosted.org/packages/5d/2c/caaeda564cbe103bed145ea557cb86795b18651b0f6b3ff6a10e84e5a33f/ruff-0.11.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:633bf2c6f35678c56ec73189ba6fa19ff1c5e4807a78bf60ef487b9dd272cc71", size = 11326315 }, + { url = "https://files.pythonhosted.org/packages/75/f0/782e7d681d660eda8c536962920c41309e6dd4ebcea9a2714ed5127d44bd/ruff-0.11.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ffbc82d70424b275b089166310448051afdc6e914fdab90e08df66c43bb5ca9", size = 11555640 }, + { url = "https://files.pythonhosted.org/packages/5d/d4/3d580c616316c7f07fb3c99dbecfe01fbaea7b6fd9a82b801e72e5de742a/ruff-0.11.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4a9ddd3ec62a9a89578c85842b836e4ac832d4a2e0bfaad3b02243f930ceafcc", size = 10507364 }, + { url = "https://files.pythonhosted.org/packages/5a/dc/195e6f17d7b3ea6b12dc4f3e9de575db7983db187c378d44606e5d503319/ruff-0.11.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d237a496e0778d719efb05058c64d28b757c77824e04ffe8796c7436e26712b7", size = 10141462 }, + { url = "https://files.pythonhosted.org/packages/f4/8e/39a094af6967faa57ecdeacb91bedfb232474ff8c3d20f16a5514e6b3534/ruff-0.11.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26816a218ca6ef02142343fd24c70f7cd8c5aa6c203bca284407adf675984432", size = 11121028 }, + { url = "https://files.pythonhosted.org/packages/5a/c0/b0b508193b0e8a1654ec683ebab18d309861f8bd64e3a2f9648b80d392cb/ruff-0.11.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:51c3f95abd9331dc5b87c47ac7f376db5616041173826dfd556cfe3d4977f492", size = 11602992 }, + { url = "https://files.pythonhosted.org/packages/7c/91/263e33ab93ab09ca06ce4f8f8547a858cc198072f873ebc9be7466790bae/ruff-0.11.13-py3-none-win32.whl", hash = "sha256:96c27935418e4e8e77a26bb05962817f28b8ef3843a6c6cc49d8783b5507f250", size = 10474944 }, + { url = "https://files.pythonhosted.org/packages/46/f4/7c27734ac2073aae8efb0119cae6931b6fb48017adf048fdf85c19337afc/ruff-0.11.13-py3-none-win_amd64.whl", hash = "sha256:29c3189895a8a6a657b7af4e97d330c8a3afd2c9c8f46c81e2fc5a31866517e3", size = 11548669 }, + { url = "https://files.pythonhosted.org/packages/ec/bf/b273dd11673fed8a6bd46032c0ea2a04b2ac9bfa9c628756a5856ba113b0/ruff-0.11.13-py3-none-win_arm64.whl", hash = "sha256:b4385285e9179d608ff1d2fb9922062663c658605819a6876d8beef0c30b7f3b", size = 10683928 }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/bc/51647cd02527e87d05cb083ccc402f93e441606ff1f01739a62c8ad09ba5/typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4", size = 107423 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/e0/552843e0d356fbb5256d21449fa957fa4eff3bbc135a74a691ee70c7c5da/typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af", size = 43839 }, +] From 38ea3c0b00d83ea3fe6cd5a3fa70958931ec4d66 Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Thu, 12 Jun 2025 08:06:07 -0300 Subject: [PATCH 02/11] Improve html report and introduce k8s resource collection --- .github/workflows/e2e-py-check.yml | 13 +- e2e-tests/conftest.py | 49 +++ e2e-tests/test_pytest_wrapper.py | 69 +++-- e2e-tests/tools/__init__.py | 0 e2e-tests/tools/k8s_resources_collector.py | 334 +++++++++++++++++++++ e2e-tests/tools/report_generator.py | 122 ++++++++ pyproject.toml | 10 +- 7 files changed, 560 insertions(+), 37 deletions(-) create mode 100644 e2e-tests/conftest.py create mode 100644 e2e-tests/tools/__init__.py create mode 100644 e2e-tests/tools/k8s_resources_collector.py create mode 100644 e2e-tests/tools/report_generator.py diff --git a/.github/workflows/e2e-py-check.yml b/.github/workflows/e2e-py-check.yml index 3998b1a40..2627f3298 100644 --- a/.github/workflows/e2e-py-check.yml +++ b/.github/workflows/e2e-py-check.yml @@ -1,9 +1,6 @@ -name: E2E Python Quality Check +name: e2e-tests Python Quality Check on: - push: - paths: - - 'e2e-tests/**/*.py' pull_request: paths: - 'e2e-tests/**/*.py' @@ -17,13 +14,15 @@ jobs: uses: actions/checkout@v4 - name: Install uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@v5 - name: Set up Python - run: uv python install 3.13 + uses: actions/setup-python@v5 + with: + python-version-file: "pyproject.toml" - name: Install dependencies - run: uv sync + run: uv sync --locked - name: Run ruff check run: uv run ruff check e2e-tests/ diff --git a/e2e-tests/conftest.py b/e2e-tests/conftest.py new file mode 100644 index 000000000..3cf9b5efa --- /dev/null +++ b/e2e-tests/conftest.py @@ -0,0 +1,49 @@ +import pytest + +from tools.report_generator import generate_report +from tools.k8s_resources_collector import collect_k8s_resources, get_namespace + + +def pytest_addoption(parser): + """Add custom command line option for test suite file""" + parser.addoption( + "--test-suite", + action="store", + default=None, + help="Name of the test suite file (will look for run-{name}.csv)", + ) + parser.addoption( + "--collect-k8s-resources", + action="store_true", + default=False, + help="Enable collection of K8s resources on test failure", + ) + + +def pytest_html_report_title(report): + report.title = "PSMDB E2E Test Report" + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + + if report.when == "call" and report.failed: + try: + namespace = get_namespace(str(report.longrepr)) + html_report = generate_report(namespace) + + if not hasattr(report, "extras"): + report.extras = [] + report.extras.extend(html_report) + + collect_resources = item.config.getoption("--collect-k8s-resources") + if collect_resources: + collect_k8s_resources( + namespace=namespace, + custom_resources=["psmdb", "psmdb-backup", "psmdb-restore"], + output_dir=f"e2e-tests/reports/{namespace}", + ) + except Exception as e: + print(f"Error adding K8s info: {e}") diff --git a/e2e-tests/test_pytest_wrapper.py b/e2e-tests/test_pytest_wrapper.py index 1338cbcfa..2bbe610b6 100644 --- a/e2e-tests/test_pytest_wrapper.py +++ b/e2e-tests/test_pytest_wrapper.py @@ -1,55 +1,66 @@ import os import subprocess +import pytest from pathlib import Path from typing import List, Tuple -import pytest - -def get_bash_tests() -> List[Tuple[str, Path]]: - """Find all bash test scripts in the same directory as this test file""" +def get_bash_tests(test_suite: str = "") -> List[Tuple[str, Path]]: + """Get bash test scripts from file or all directories""" current_dir = Path(__file__).parent bash_tests: List[Tuple[str, Path]] = [] - - for test_dir in current_dir.iterdir(): - if test_dir.is_dir(): - run_script = test_dir / "run" - if run_script.exists(): - bash_tests.append((test_dir.name, run_script)) + + if test_suite: + file_path = current_dir / f"run-{test_suite}.csv" + if not file_path.exists(): + raise FileNotFoundError(f"Test suite file not found: {file_path}") + + with open(file_path, "r", encoding="utf-8") as f: + test_names = [line.strip() for line in f if line.strip()] + else: + test_names = [d.name for d in current_dir.iterdir() if d.is_dir()] + + for test_name in test_names: + test_dir = current_dir / test_name + run_script = test_dir / "run" + if run_script.exists(): + bash_tests.append((test_name, run_script)) return bash_tests -bash_tests = get_bash_tests() +def pytest_generate_tests(metafunc): + """Generate tests dynamically""" + if "test_name" in metafunc.fixturenames and "script_path" in metafunc.fixturenames: + test_suite = metafunc.config.getoption("--test-suite") + bash_tests = get_bash_tests(test_suite) + metafunc.parametrize( + "test_name,script_path", bash_tests, ids=[name for name, _ in bash_tests] + ) -@pytest.mark.parametrize( - "test_name,script_path", bash_tests, ids=[name for name, _ in bash_tests] -) def test_e2e(test_name: str, script_path: Path) -> None: """Run bash script and check exit code""" - - original_cwd: str = os.getcwd() - script_dir: Path = script_path.parent + original_cwd = os.getcwd() + script_dir = script_path.parent try: os.chdir(script_dir) - result: subprocess.CompletedProcess[str] = subprocess.run( - ["bash", "run"], capture_output=True, text=True - ) + result = subprocess.run(["bash", "run"], capture_output=True, text=True) if result.returncode != 0: - print(f"\nSTDOUT:\n{result.stdout}") - print(f"\nSTDERR:\n{result.stderr}") + error_msg = f""" +Test {test_name} failed with exit code {result.returncode} - k8s_result: subprocess.CompletedProcess[str] = subprocess.run( - ["kubectl", "get", "nodes"], capture_output=True, text=True - ) - print(f"\nK8s LOGS:\n{k8s_result.stdout}") +STDOUT: +{result.stdout} - assert result.returncode == 0, ( - f"Test {test_name} failed with exit code {result.returncode}" - ) +STDERR: +{result.stderr} +""" + pytest.fail(error_msg) + + assert result.returncode == 0 finally: os.chdir(original_cwd) diff --git a/e2e-tests/tools/__init__.py b/e2e-tests/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/e2e-tests/tools/k8s_resources_collector.py b/e2e-tests/tools/k8s_resources_collector.py new file mode 100644 index 000000000..40f87d2bc --- /dev/null +++ b/e2e-tests/tools/k8s_resources_collector.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 + +import os +import re +import sys +import threading +import subprocess +from datetime import datetime +from concurrent.futures import ThreadPoolExecutor, Future, as_completed +from typing import List, Optional, Dict, Any + + +class K8sCollector: + def __init__(self, namespace: str, custom_resources: Optional[List[str]] = None): + self.namespace = namespace + self.custom_resources = custom_resources or [] + self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + self.output_dir = f"{namespace}_{self.timestamp}" + self.error_log_file = "" + self.lock = threading.Lock() + + def run_kubectl( + self, args: List[str], capture_output: bool = True, text: bool = True + ) -> Optional[subprocess.CompletedProcess]: + """Run kubectl command and return result""" + try: + result = subprocess.run( + ["kubectl"] + args, capture_output=capture_output, text=text, check=False + ) + return result + except Exception as e: + print(f"Error running kubectl command: {e}") + return None + + def setup_directories(self) -> None: + """Create output directory structure""" + print(f"Creating output directory: {self.output_dir}") + + directories = [ + self.output_dir, + f"{self.output_dir}/logs", + f"{self.output_dir}/describe", + f"{self.output_dir}/events", + f"{self.output_dir}/get", + ] + + for directory in directories: + os.makedirs(directory, exist_ok=True) + + self.error_log_file = f"{self.output_dir}/error_summary.log" + with open(self.error_log_file, "w") as f: + f.write( + f"Error Log Summary for Namespace: {self.namespace} (Extracted on {self.timestamp})\n" + ) + f.write("=" * 73 + "\n\n") + + def get_resource_names(self, resource_type: str) -> List[str]: + """Get list of resource names for a given type""" + result = self.run_kubectl(["get", resource_type, "-n", self.namespace, "-o", "name"]) + if result and result.returncode == 0 and result.stdout.strip(): + return [line.split("/")[-1] for line in result.stdout.strip().split("\n")] + return [] + + def process_resource_type(self, resource_type: str, singular: str, plural: str) -> List[str]: + """Process a specific resource type""" + print(f"Extracting {resource_type} information...") + + get_dir = f"{self.output_dir}/get/{plural}" + os.makedirs(get_dir, exist_ok=True) + + result = self.run_kubectl(["get", plural, "-n", self.namespace, "-o", "wide"]) + with open(f"{get_dir}/{plural}.txt", "w") as f: + if result and result.returncode == 0: + f.write(result.stdout) + else: + f.write(f"No {resource_type}s found\n") + + resources = self.get_resource_names(plural) + with ThreadPoolExecutor(max_workers=5) as executor: + futures = [] + for resource in resources: + print(f"Processing {resource_type}: {resource}") + futures.append(executor.submit(self.describe_resource, singular, resource)) + futures.append( + executor.submit(self.get_resource_yaml, singular, resource, get_dir) + ) + + # Wait for all tasks to complete + for future in as_completed(futures): + try: + future.result() + except Exception as e: + print(f"Error processing resource: {e}") + + return resources + + def describe_resource(self, resource_type: str, resource_name: str) -> None: + """Describe a specific resource""" + result = self.run_kubectl(["describe", resource_type, resource_name, "-n", self.namespace]) + if result: + with open(f"{self.output_dir}/describe/{resource_type}_{resource_name}.txt", "w") as f: + f.write(result.stdout) + + def get_resource_yaml(self, resource_type: str, resource_name: str, output_dir: str) -> None: + """Get resource YAML""" + result = self.run_kubectl( + ["get", resource_type, resource_name, "-n", self.namespace, "-o", "yaml"] + ) + if result: + with open(f"{output_dir}/{resource_name}.yaml", "w") as f: + f.write(result.stdout) + + def process_custom_resource(self, resource: str) -> None: + """Process custom resource""" + print(f"Extracting custom resource: {resource}...") + + api_result = self.run_kubectl(["api-resources"]) + if not api_result or resource not in api_result.stdout: + print(f"Warning: Custom resource '{resource}' not found in the cluster. Skipping.") + return + + get_dir = f"{self.output_dir}/get/{resource}" + os.makedirs(get_dir, exist_ok=True) + + check_result = self.run_kubectl(["get", resource, "-n", self.namespace]) + if not check_result or check_result.returncode != 0: + print(f"No resources of type '{resource}' found in namespace {self.namespace}") + return + + result = self.run_kubectl(["get", resource, "-n", self.namespace, "-o", "wide"]) + with open(f"{get_dir}/{resource}.txt", "w") as f: + f.write(result.stdout if result else "") + + resources = self.get_resource_names(resource) + + with ThreadPoolExecutor(max_workers=5) as executor: + futures = [] + + for resource_name in resources: + print(f"Processing custom resource {resource}: {resource_name}") + + futures.append(executor.submit(self.describe_resource, resource, resource_name)) + futures.append( + executor.submit(self.get_resource_yaml, resource, resource_name, get_dir) + ) + + for future in as_completed(futures): + try: + future.result() + except Exception as e: + print(f"Error processing custom resource: {e}") + + def extract_pod_logs(self, pod_name: str, container_name: str) -> None: + """Extract logs for a specific container""" + print(f" Extracting logs for container: {container_name}") + + log_dir = f"{self.output_dir}/logs/{pod_name}" + os.makedirs(log_dir, exist_ok=True) + + result = self.run_kubectl(["logs", pod_name, "-c", container_name, "-n", self.namespace]) + log_file = f"{log_dir}/{container_name}.log" + + with open(log_file, "w") as f: + if result: + f.write(result.stdout) + + self.extract_error_logs(pod_name, container_name, log_file) + + def extract_error_logs(self, pod_name: str, container_name: str, log_file: str) -> None: + """Extract error logs from container log file""" + try: + with open(log_file, "r") as f: + content = f.read() + + if content and "error" in content.lower(): + error_lines = [line for line in content.split("\n") if "error" in line.lower()] + + if error_lines: + with self.lock: + with open(self.error_log_file, "a") as f: + f.write( + f"=== Error logs from pod: {pod_name}, container: {container_name} ===\n\n" + ) + for line in error_lines: + f.write(f"{line}\n") + f.write("\n" + "-" * 48 + "\n\n") + + except Exception as e: + print(f"Error extracting error logs: {e}") + + def get_container_names(self, pod_name: str) -> List[str]: + """Get container names for a pod""" + result = self.run_kubectl( + [ + "get", + "pod", + pod_name, + "-n", + self.namespace, + "-o", + "jsonpath={.spec.containers[*].name}", + ] + ) + + if result and result.returncode == 0 and result.stdout.strip(): + return result.stdout.strip().split() + return [] + + def process_pods(self) -> None: + """Process all pods in the namespace""" + print("Extracting Pod information...") + + pods_dir = f"{self.output_dir}/get/pods" + os.makedirs(pods_dir, exist_ok=True) + + result = self.run_kubectl(["get", "pods", "-n", self.namespace, "-o", "wide"]) + with open(f"{pods_dir}/pods.txt", "w") as f: + if result and result.returncode == 0: + f.write(result.stdout) + else: + f.write("No Pods found\n") + + pods = self.get_resource_names("pods") + + with ThreadPoolExecutor(max_workers=3) as executor: + futures = [] + + for pod in pods: + print(f"Processing pod: {pod}") + + futures.append(executor.submit(self.describe_resource, "pod", pod)) + futures.append(executor.submit(self.get_resource_yaml, "pod", pod, pods_dir)) + + containers = self.get_container_names(pod) + for container in containers: + futures.append(executor.submit(self.extract_pod_logs, pod, container)) + + # Wait for all tasks + for future in as_completed(futures): + try: + future.result() + except Exception as e: + print(f"Error processing pod: {e}") + + def extract_namespace_events(self) -> None: + """Extract namespace events""" + print(f"Extracting events for namespace: {self.namespace}") + + events_dir = f"{self.output_dir}/events" + + print("Getting events...") + result = self.run_kubectl(["get", "events", "-n", self.namespace, "-o", "wide"]) + with open(f"{events_dir}/events.txt", "w") as f: + f.write(result.stdout if result else "") + + print("Getting events in JSON format...") + result = self.run_kubectl(["get", "events", "-n", self.namespace, "-o", "json"]) + with open(f"{events_dir}/events.json", "w") as f: + f.write(result.stdout if result else "") + + def collect_all(self) -> None: + """Main collection method""" + print(f"=== Starting extraction for namespace: {self.namespace} ===") + + self.setup_directories() + self.process_pods() + with ThreadPoolExecutor(max_workers=6) as executor: + futures: List[Future] = [ + executor.submit( + self.process_resource_type, "StatefulSet", "statefulset", "statefulsets" + ), + executor.submit( + self.process_resource_type, "Deployment", "deployment", "deployments" + ), + executor.submit(self.process_resource_type, "Secret", "secret", "secrets"), + executor.submit(self.process_resource_type, "Job", "job", "jobs"), + executor.submit( + self.process_resource_type, "ConfigMap", "configmap", "configmaps" + ), + executor.submit(self.process_resource_type, "Service", "service", "services"), + executor.submit(self.extract_namespace_events), + ] + + for future in as_completed(futures): + try: + future.result() + except Exception as e: + print(f"Error in parallel processing: {e}") + + if self.custom_resources: + print("Processing custom resources...") + for resource in self.custom_resources: + resource = resource.strip() + if resource: + self.process_custom_resource(resource) + + print("=== Extraction completed successfully ===") + print(f"Output is available in: {self.output_dir}") + print(f"Error log summary: {self.error_log_file}") + + +def collect_k8s_resources( + namespace: str, custom_resources: Optional[List[str]] = None, output_dir: Optional[str] = None +) -> Dict[str, Any]: + """ + Collect Kubernetes resources for a given namespace. + """ + collector = K8sCollector(namespace, custom_resources) + + if output_dir: + collector.output_dir = f"{output_dir}_{collector.timestamp}" + + try: + collector.collect_all() + return { + "success": True, + "output_dir": collector.output_dir, + "error_log": collector.error_log_file, + "namespace": namespace, + "timestamp": collector.timestamp, + } + except Exception as e: + return {"success": False, "error": str(e), "namespace": namespace} + + +def get_namespace(test_log: str) -> str: + """Extract namespace from test log""" + match = re.search(r"create namespace (\S*?\d+)\b", test_log) + if match: + namespace = match.group(1) + print(f"Extracted namespace: {namespace}", file=sys.stderr) + return namespace + + raise ValueError("Namespace not found in logs") diff --git a/e2e-tests/tools/report_generator.py b/e2e-tests/tools/report_generator.py new file mode 100644 index 000000000..ffc436518 --- /dev/null +++ b/e2e-tests/tools/report_generator.py @@ -0,0 +1,122 @@ +import subprocess + +from pytest_html import extras + + +def run_kubectl_commands(commands: list) -> str: + """Execute kubectl commands and return formatted output""" + output = [] + + for title, cmd in commands: + output.append(f"\n=== {title} ===") + try: + result = subprocess.run(cmd.split(), capture_output=True, text=True, timeout=15) + if result.returncode == 0: + output.append(result.stdout.strip()) + else: + output.append(f"Error (exit code {result.returncode}): {result.stderr.strip()}") + except subprocess.TimeoutExpired: + output.append("Command timed out after 15 seconds") + except Exception as e: + output.append(f"Failed to execute: {str(e)}") + + return "\n".join(output) + + +def capture_k8s_resources(namespace: str) -> str: + """Capture Kubernetes resources information""" + commands = [ + ("Nodes", "kubectl get nodes"), + (f"All from namespace {namespace}", f"kubectl get all -n {namespace}"), + ("Secrets", f"kubectl get secrets -n {namespace}"), + ( + "PSMDB Cluster", + f"kubectl get psmdb -n {namespace} -o custom-columns=NAME:.metadata.name,STATE:.status.state", + ), + ("PSMDB Backup", f"kubectl get psmdb-backup -n {namespace}"), + ("PSMDB Restore", f"kubectl get psmdb-restore -n {namespace}"), + ] + return run_kubectl_commands(commands) + + +def capture_k8s_logs(namespace: str) -> str: + """Capture Kubernetes logs""" + commands = [ + ( + "PSMDB Operator Logs", + f"kubectl logs -l app.kubernetes.io/name=percona-server-mongodb-operator --tail=50 -n {namespace}", + ), + ] + return run_kubectl_commands(commands) + + +def capture_k8s_events(namespace: str) -> str: + """Capture Kubernetes events""" + commands = [ + ("PSMDB Operator Events", f"kubectl get events -n {namespace}"), + ] + return run_kubectl_commands(commands) + + +def highlight_log_levels(logs: str) -> str: + """Add basic color highlighting for common log levels""" + logs = logs.replace("ERROR", 'ERROR') + logs = logs.replace("WARN", 'WARN') + logs = logs.replace("INFO", 'INFO') + logs = logs.replace("DEBUG", 'DEBUG') + logs = logs.replace("Normal", 'Normal') + logs = logs.replace("Warning", 'Warning') + return logs + + +def generate_report(namespace: str) -> list: + def create_collapsible_section(title: str, icon: str, content: str) -> str: + return f""" +
+
+ + {icon} {title} + +
+ 
{content}
+
+
+
+ """ + + final_report = [] + + pod_logs = capture_k8s_logs(namespace) + highlighted_logs = highlight_log_levels(pod_logs) + final_report.append( + extras.html(create_collapsible_section("Operator Pod Logs", "📋", highlighted_logs)) + ) + + k8s_info = capture_k8s_resources(namespace) + final_report.append( + extras.html(create_collapsible_section("Kubernetes Resources", "🔍", k8s_info)) + ) + + k8s_events = capture_k8s_events(namespace) + highlighted_events = highlight_log_levels(k8s_events) + final_report.append( + extras.html(create_collapsible_section("Kubernetes Events", "📣", highlighted_events)) + ) + + return final_report diff --git a/pyproject.toml b/pyproject.toml index a7574ff29..781eb0e30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,4 +13,12 @@ dependencies = [ ] [tool.pytest.ini_options] -addopts = "--html=e2e-tests/reports/report.html --self-contained-html --json-report --json-report-file=e2e-tests/reports/report.json --junitxml=e2e-tests/reports/report.xml" \ No newline at end of file +addopts = "--html=e2e-tests/reports/report.html --self-contained-html --json-report --json-report-file=e2e-tests/reports/report.json --junitxml=e2e-tests/reports/report.xml" +render_collapsed = "all" + +[[tool.mypy.overrides]] +module = ["pytest_html.*"] +follow_untyped_imports = true + +[tool.ruff] +line-length = 99 \ No newline at end of file From 7204a3ccb3e3898223ccb6ee488be08622f111e2 Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Thu, 12 Jun 2025 14:41:57 -0300 Subject: [PATCH 03/11] Make jenkins PR job execute pytest --- .gitignore | 3 ++- Jenkinsfile | 58 +++++++++++++++++++++++++++++++++++--------------- pyproject.toml | 6 ++++-- uv.lock | 48 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 089f67b09..69c56fdfa 100644 --- a/.gitignore +++ b/.gitignore @@ -191,4 +191,5 @@ installers/olm/operator_*.yaml installers/olm/bundles # Test Reports -e2e-tests/reports/ \ No newline at end of file +e2e-tests/reports/ +e2e-tests/**/__pycache__/ \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 846fda3e5..fbe5bf4bb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -149,31 +149,41 @@ void printKubernetesStatus(String LOCATION, String CLUSTER_SUFFIX) { """ } -TestsReport = '| Test name | Status |\r\n| ------------- | ------------- |' -TestsReportXML = '\n' +String formatTime(def time) { + if (!time || time == "N/A") return "N/A" + + try { + def seconds = time as Double + if (seconds < 60) { + return "${seconds.round(2)}s" + } else { + def minutes = (seconds / 60) as Integer + def remainingSeconds = (seconds % 60).round(2) + return "${minutes}m ${remainingSeconds}s" + } + } catch (Exception e) { + return time.toString() + } +} + +TestsReport = '| Test Name | Result | Time |\r\n| ----------- | -------- | ------ |' void makeReport() { - def wholeTestAmount=tests.size() + def wholeTestAmount = tests.size() def startedTestAmount = 0 - for (int i=0; i<'+ testResult +'/>\n' + TestsReport = TestsReport + "\r\n| " + testName + " | [" + testResult + "](" + testUrl + ") | " + testTime + " |" } - TestsReport = TestsReport + "\r\n| We run $startedTestAmount out of $wholeTestAmount|" - TestsReportXML = TestsReportXML + '\n' - - sh """ - echo "${TestsReportXML}" > TestsReport.xml - """ + TestsReport = TestsReport + "\r\n| We run $startedTestAmount out of $wholeTestAmount | | |" } void clusterRunner(String cluster) { @@ -215,7 +225,10 @@ void runTest(Integer TEST_ID) { export DEBUG_TESTS=1 fi export KUBECONFIG=/tmp/$CLUSTER_NAME-$clusterSuffix - time ./e2e-tests/$testName/run + + uv run pytest -v -s -k "$testName" \ + --html=e2e-tests/reports/$CLUSTER_NAME-$testName-report.html \ + --junitxml=e2e-tests/reports/$CLUSTER_NAME-$testName-report.xml """ } pushArtifactFile("${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}-$testName") @@ -264,6 +277,10 @@ EOF sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense + + curl -LsSf https://astral.sh/uv/install.sh | sh + uv python install 3.13 + uv sync --locked """ } @@ -578,9 +595,16 @@ pipeline { } } makeReport() - step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0]) - archiveArtifacts '*.xml' - + if (fileExists('e2e-tests/reports')){ + sh """ + pytest_html_merger -i e2e-tests/reports -o final_report.html + junitparser merge --glob 'e2e-tests/reports/*.xml' final_report.xml + """ + step([$class: 'JUnitResultArchiver', testResults: 'final_report.xml', healthScaleFactor: 1.0]) + archiveArtifacts 'final_report.xml, final_report.html' + }else { + echo "No report files found in e2e-tests/reports, skipping report generation" + } unstash 'IMAGE' def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim() TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n" diff --git a/pyproject.toml b/pyproject.toml index 781eb0e30..f92619e09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,16 +4,18 @@ version = "0.1.0" description = "Tests for PSMDB Operator" requires-python = ">=3.13" dependencies = [ + "junitparser>=3.2.0", "mypy>=1.16.0", "pytest>=8.4.0", "pytest-html>=4.1.1", + "pytest-html-merger>=0.1.0", "pytest-json-report>=1.5.0", "pyyaml>=6.0.2", "ruff>=0.11.12", ] [tool.pytest.ini_options] -addopts = "--html=e2e-tests/reports/report.html --self-contained-html --json-report --json-report-file=e2e-tests/reports/report.json --junitxml=e2e-tests/reports/report.xml" +addopts = "--self-contained-html --json-report --json-report-file=e2e-tests/reports/report.json" render_collapsed = "all" [[tool.mypy.overrides]] @@ -21,4 +23,4 @@ module = ["pytest_html.*"] follow_untyped_imports = true [tool.ruff] -line-length = 99 \ No newline at end of file +line-length = 99 diff --git a/uv.lock b/uv.lock index d63a90f1c..9bcecfd4d 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,19 @@ version = 1 revision = 1 requires-python = ">=3.13" +[[package]] +name = "beautifulsoup4" +version = "4.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/e4/0c4c39e18fd76d6a628d4dd8da40543d136ce2d1752bd6eeeab0791f4d6b/beautifulsoup4-4.13.4.tar.gz", hash = "sha256:dbb3c4e1ceae6aefebdaf2423247260cd062430a410e38c66f2baa50a8437195", size = 621067 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/cd/30110dc0ffcf3b131156077b90e9f60ed75711223f306da4db08eff8403b/beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b", size = 187285 }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -32,6 +45,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899 }, ] +[[package]] +name = "junitparser" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/88/6a268028a297751ed73be8e291f12aa727caf22adbc218e8dfbafcc974af/junitparser-3.2.0.tar.gz", hash = "sha256:b05e89c27e7b74b3c563a078d6e055d95cf397444f8f689b0ca616ebda0b3c65", size = 20073 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/f9/321d566c9f2af81fdb4bb3d5900214116b47be9e26b82219da8b818d9da9/junitparser-3.2.0-py2.py3-none-any.whl", hash = "sha256:e14fdc0a999edfc15889b637390e8ef6ca09a49532416d3bd562857d42d4b96d", size = 13394 }, +] + [[package]] name = "markupsafe" version = "3.0.2" @@ -121,9 +143,11 @@ name = "psmdb-pytest" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "junitparser" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-html" }, + { name = "pytest-html-merger" }, { name = "pytest-json-report" }, { name = "pyyaml" }, { name = "ruff" }, @@ -131,9 +155,11 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "junitparser", specifier = ">=3.2.0" }, { name = "mypy", specifier = ">=1.16.0" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-html", specifier = ">=4.1.1" }, + { name = "pytest-html-merger", specifier = ">=0.1.0" }, { name = "pytest-json-report", specifier = ">=1.5.0" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "ruff", specifier = ">=0.11.12" }, @@ -178,6 +204,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c7/c160021cbecd956cc1a6f79e5fe155f7868b2e5b848f1320dad0b3e3122f/pytest_html-4.1.1-py3-none-any.whl", hash = "sha256:c8152cea03bd4e9bee6d525573b67bbc6622967b72b9628dda0ea3e2a0b5dd71", size = 23491 }, ] +[[package]] +name = "pytest-html-merger" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/8d/8f1bc3282c636bf29c88579a136ae6add64ddf5f239d8bec52d7434e163c/pytest_html_merger-0.1.0.tar.gz", hash = "sha256:497b1e9c99c12eb06eee5fdf9abad42c10fec78524d740737def9d85b8f995e4", size = 17814 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/33/6ebce34cf14de51d2a422e5f77493468e19afe042e35c3f78bebfa5275d6/pytest_html_merger-0.1.0-py3-none-any.whl", hash = "sha256:c1bf0574245dd67481b21630d68168fdf26a779f55b939f37e7aff4c438ea61b", size = 17200 }, +] + [[package]] name = "pytest-json-report" version = "1.5.0" @@ -245,6 +284,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bf/b273dd11673fed8a6bd46032c0ea2a04b2ac9bfa9c628756a5856ba113b0/ruff-0.11.13-py3-none-win_arm64.whl", hash = "sha256:b4385285e9179d608ff1d2fb9922062663c658605819a6876d8beef0c30b7f3b", size = 10683928 }, ] +[[package]] +name = "soupsieve" +version = "2.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/f4/4a80cd6ef364b2e8b65b15816a843c0980f7a5a2b4dc701fc574952aa19f/soupsieve-2.7.tar.gz", hash = "sha256:ad282f9b6926286d2ead4750552c8a6142bc4c783fd66b0293547c8fe6ae126a", size = 103418 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/9c/0e6afc12c269578be5c0c1c9f4b49a8d32770a080260c333ac04cc1c832d/soupsieve-2.7-py3-none-any.whl", hash = "sha256:6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4", size = 36677 }, +] + [[package]] name = "typing-extensions" version = "4.14.0" From 5cd73184754637c38b5470e5661a8a70738facef Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Thu, 12 Jun 2025 15:00:32 -0300 Subject: [PATCH 04/11] Fix jenkins file --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fbe5bf4bb..80ad3928b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -226,6 +226,7 @@ void runTest(Integer TEST_ID) { fi export KUBECONFIG=/tmp/$CLUSTER_NAME-$clusterSuffix + source \$HOME/.local/bin/env uv run pytest -v -s -k "$testName" \ --html=e2e-tests/reports/$CLUSTER_NAME-$testName-report.html \ --junitxml=e2e-tests/reports/$CLUSTER_NAME-$testName-report.xml @@ -279,6 +280,7 @@ EOF curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense curl -LsSf https://astral.sh/uv/install.sh | sh + source \$HOME/.local/bin/env uv python install 3.13 uv sync --locked """ @@ -602,7 +604,7 @@ pipeline { """ step([$class: 'JUnitResultArchiver', testResults: 'final_report.xml', healthScaleFactor: 1.0]) archiveArtifacts 'final_report.xml, final_report.html' - }else { + } else { echo "No report files found in e2e-tests/reports, skipping report generation" } unstash 'IMAGE' From ed4973065dfeeb1d41bc1c85652ca57fb4af58c1 Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Thu, 12 Jun 2025 18:59:25 -0300 Subject: [PATCH 05/11] Add regex support and live logging --- Jenkinsfile | 2 +- e2e-tests/conftest.py | 61 +++++++++++++++++++++++++++++++ e2e-tests/test_pytest_wrapper.py | 62 +++++++++----------------------- 3 files changed, 78 insertions(+), 47 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 80ad3928b..adb13e732 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -227,7 +227,7 @@ void runTest(Integer TEST_ID) { export KUBECONFIG=/tmp/$CLUSTER_NAME-$clusterSuffix source \$HOME/.local/bin/env - uv run pytest -v -s -k "$testName" \ + uv run pytest e2e-tests/test_pytest_wrapper.py -v -s --test-regex "^${testName}\$" \ --html=e2e-tests/reports/$CLUSTER_NAME-$testName-report.html \ --junitxml=e2e-tests/reports/$CLUSTER_NAME-$testName-report.xml """ diff --git a/e2e-tests/conftest.py b/e2e-tests/conftest.py index 3cf9b5efa..6113da958 100644 --- a/e2e-tests/conftest.py +++ b/e2e-tests/conftest.py @@ -1,5 +1,9 @@ +import re import pytest +from pathlib import Path +from typing import Tuple, List + from tools.report_generator import generate_report from tools.k8s_resources_collector import collect_k8s_resources, get_namespace @@ -12,6 +16,12 @@ def pytest_addoption(parser): default=None, help="Name of the test suite file (will look for run-{name}.csv)", ) + parser.addoption( + "--test-regex", + action="store", + default=None, + help="Run tests matching the given regex pattern", + ) parser.addoption( "--collect-k8s-resources", action="store_true", @@ -20,6 +30,57 @@ def pytest_addoption(parser): ) +def get_bash_tests(test_suite: str = "") -> List[Tuple[str, Path]]: + """Get bash test scripts from file or all directories""" + current_dir = Path(__file__).parent + bash_tests: List[Tuple[str, Path]] = [] + + if test_suite: + file_path = current_dir / f"run-{test_suite}.csv" + if not file_path.exists(): + raise FileNotFoundError(f"Test suite file not found: {file_path}") + + with open(file_path, "r", encoding="utf-8") as f: + test_names = [line.strip() for line in f if line.strip()] + else: + test_names = [d.name for d in current_dir.iterdir() if d.is_dir()] + + for test_name in test_names: + test_dir = current_dir / test_name + run_script = test_dir / "run" + if run_script.exists(): + bash_tests.append((test_name, run_script)) + + return bash_tests + + +def pytest_generate_tests(metafunc): + """Generate tests dynamically with regex filtering""" + if "test_name" in metafunc.fixturenames and "script_path" in metafunc.fixturenames: + test_suite = metafunc.config.getoption("--test-suite") + test_regex = metafunc.config.getoption("--test-regex") + + bash_tests = get_bash_tests(test_suite) + if test_regex: + try: + pattern = re.compile(test_regex) + filtered_tests = [ + (name, path) for name, path in bash_tests if pattern.search(name) + ] + bash_tests = filtered_tests + + print(f"\nFiltered to {len(bash_tests)} test(s) matching regex '{test_regex}':") + for name, _ in bash_tests: + print(f" - {name}") + + except re.error as e: + pytest.exit(f"Invalid regex pattern '{test_regex}': {e}") + + metafunc.parametrize( + "test_name,script_path", bash_tests, ids=[name for name, _ in bash_tests] + ) + + def pytest_html_report_title(report): report.title = "PSMDB E2E Test Report" diff --git a/e2e-tests/test_pytest_wrapper.py b/e2e-tests/test_pytest_wrapper.py index 2bbe610b6..c21bba6a7 100644 --- a/e2e-tests/test_pytest_wrapper.py +++ b/e2e-tests/test_pytest_wrapper.py @@ -2,65 +2,35 @@ import subprocess import pytest from pathlib import Path -from typing import List, Tuple - - -def get_bash_tests(test_suite: str = "") -> List[Tuple[str, Path]]: - """Get bash test scripts from file or all directories""" - current_dir = Path(__file__).parent - bash_tests: List[Tuple[str, Path]] = [] - - if test_suite: - file_path = current_dir / f"run-{test_suite}.csv" - if not file_path.exists(): - raise FileNotFoundError(f"Test suite file not found: {file_path}") - - with open(file_path, "r", encoding="utf-8") as f: - test_names = [line.strip() for line in f if line.strip()] - else: - test_names = [d.name for d in current_dir.iterdir() if d.is_dir()] - - for test_name in test_names: - test_dir = current_dir / test_name - run_script = test_dir / "run" - if run_script.exists(): - bash_tests.append((test_name, run_script)) - - return bash_tests - - -def pytest_generate_tests(metafunc): - """Generate tests dynamically""" - if "test_name" in metafunc.fixturenames and "script_path" in metafunc.fixturenames: - test_suite = metafunc.config.getoption("--test-suite") - bash_tests = get_bash_tests(test_suite) - metafunc.parametrize( - "test_name,script_path", bash_tests, ids=[name for name, _ in bash_tests] - ) def test_e2e(test_name: str, script_path: Path) -> None: - """Run bash script and check exit code""" + """Run bash script with live output and capture for error reporting""" original_cwd = os.getcwd() script_dir = script_path.parent try: os.chdir(script_dir) - result = subprocess.run(["bash", "run"], capture_output=True, text=True) + process = subprocess.Popen( + ["bash", "run"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) - if result.returncode != 0: - error_msg = f""" -Test {test_name} failed with exit code {result.returncode} + output = [] + if process.stdout is not None: + for line in iter(process.stdout.readline, ""): + print(line, end="") + output.append(line) -STDOUT: -{result.stdout} + process.wait() -STDERR: -{result.stderr} + if process.returncode != 0: + error_msg = f""" +Test {test_name} failed with exit code {process.returncode} + +OUTPUT: +{"".join(output)} """ pytest.fail(error_msg) - assert result.returncode == 0 - finally: os.chdir(original_cwd) From 7c54c81a56bbeb1918812572742ff0fde16bd745 Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Fri, 13 Jun 2025 08:53:37 -0300 Subject: [PATCH 06/11] Fix report merge --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index adb13e732..b2802a574 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -599,8 +599,9 @@ pipeline { makeReport() if (fileExists('e2e-tests/reports')){ sh """ - pytest_html_merger -i e2e-tests/reports -o final_report.html - junitparser merge --glob 'e2e-tests/reports/*.xml' final_report.xml + source \$HOME/.local/bin/env + uv run pytest_html_merger -i e2e-tests/reports -o final_report.html + uv run junitparser merge --glob 'e2e-tests/reports/*.xml' final_report.xml """ step([$class: 'JUnitResultArchiver', testResults: 'final_report.xml', healthScaleFactor: 1.0]) archiveArtifacts 'final_report.xml, final_report.html' From d38445d8a8c9942de6d120ed7bd26102393a25a9 Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Mon, 16 Jun 2025 16:22:57 -0300 Subject: [PATCH 07/11] Adjust jenkinsfile --- Jenkinsfile | 22 +++++++++++++++++++--- e2e-tests/functions | 2 +- pyproject.toml | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b2802a574..15cca4051 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,20 @@ void createCluster(String CLUSTER_SUFFIX) { gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE gcloud config set project $GCP_PROJECT gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true - gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.30 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 --enable-ip-alias --workload-pool=cloud-dev-112233.svc.id.goog && \ + gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} \ + --cluster-version=1.32 \ + --machine-type=n1-standard-4 \ + --preemptible --disk-size 30 \ + --num-nodes=\$NODES_NUM \ + --network=jenkins-vpc \ + --subnetwork=jenkins-${CLUSTER_SUFFIX} \ + --no-enable-autoupgrade \ + --cluster-ipv4-cidr=/21 \ + --labels delete-cluster-after-hours=6 \ + --enable-ip-alias \ + --monitoring=NONE \ + --logging=NONE \ + --workload-pool=cloud-dev-112233.svc.id.goog && \ kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$? if [ \${ret_val} -eq 0 ]; then break; fi ret_num=\$((ret_num + 1)) @@ -153,7 +166,9 @@ String formatTime(def time) { if (!time || time == "N/A") return "N/A" try { + println("Input time: ${time} (type: ${time.class})") def seconds = time as Double + println("Converted to double: ${seconds}") if (seconds < 60) { return "${seconds.round(2)}s" } else { @@ -162,6 +177,7 @@ String formatTime(def time) { return "${minutes}m ${remainingSeconds}s" } } catch (Exception e) { + println("Error converting time: ${e.message}") return time.toString() } } @@ -248,7 +264,7 @@ void runTest(Integer TEST_ID) { } finally { def timeStop = new Date().getTime() - def durationSec = (timeStop - timeStart) / 1000 + def durationSec = (timeStop - timeStart) / 1000.0 tests[TEST_ID]["time"] = durationSec pushLogFile("$testName") echo "The $testName test was finished!" @@ -518,7 +534,7 @@ pipeline { } } options { - timeout(time: 3, unit: 'HOURS') + timeout(time: 4, unit: 'HOURS') } parallel { stage('cluster1') { diff --git a/e2e-tests/functions b/e2e-tests/functions index e7c26143a..f588c16f3 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -131,7 +131,7 @@ create_namespace() { destroy_chaos_mesh desc 'cleaned up all old namespaces' kubectl_bin get ns \ - | egrep -v "^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME" \ + | egrep -v "^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME" \ | awk '{print$1}' \ | xargs kubectl delete ns & fi diff --git a/pyproject.toml b/pyproject.toml index f92619e09..d37cb7e80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ ] [tool.pytest.ini_options] -addopts = "--self-contained-html --json-report --json-report-file=e2e-tests/reports/report.json" +addopts = "--html=e2e-tests/reports/report.html --self-contained-html --json-report --json-report-file=e2e-tests/reports/report.json --junitxml=e2e-tests/reports/report.xml" render_collapsed = "all" [[tool.mypy.overrides]] From ae3b240a4579a461e54491bd08976a9a4c2af2a5 Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Mon, 16 Jun 2025 20:51:19 -0300 Subject: [PATCH 08/11] Fix GH Test Report --- Jenkinsfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 15cca4051..6b2ddbb43 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -169,11 +169,12 @@ String formatTime(def time) { println("Input time: ${time} (type: ${time.class})") def seconds = time as Double println("Converted to double: ${seconds}") + if (seconds < 60) { - return "${seconds.round(2)}s" + return "${Math.round(seconds * 100) / 100}s" } else { def minutes = (seconds / 60) as Integer - def remainingSeconds = (seconds % 60).round(2) + def remainingSeconds = Math.round((seconds % 60) * 100) / 100 return "${minutes}m ${remainingSeconds}s" } } catch (Exception e) { @@ -219,6 +220,8 @@ void clusterRunner(String cluster) { if (clusterCreated >= 1) { shutdownCluster(cluster) + // Re-check for passed tests after execution + markPassedTests() } } From 0ddadd8533f8486c74d98452dd17fc47474800fd Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Tue, 17 Jun 2025 10:51:53 -0300 Subject: [PATCH 09/11] Publish html test report --- Jenkinsfile | 44 ++++++++++------------------- e2e-tests/tools/report_generator.py | 2 +- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6b2ddbb43..800227c03 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -143,40 +143,20 @@ void markPassedTests() { } } -void printKubernetesStatus(String LOCATION, String CLUSTER_SUFFIX) { - sh """ - export KUBECONFIG=/tmp/$CLUSTER_NAME-$CLUSTER_SUFFIX - echo "========== KUBERNETES STATUS $LOCATION TEST ==========" - gcloud container clusters list|grep -E "NAME|$CLUSTER_NAME-$CLUSTER_SUFFIX " - echo - kubectl get nodes - echo - kubectl top nodes - echo - kubectl get pods --all-namespaces - echo - kubectl top pod --all-namespaces - echo - kubectl get events --field-selector type!=Normal --all-namespaces --sort-by=".lastTimestamp" - echo "======================================================" - """ -} - String formatTime(def time) { if (!time || time == "N/A") return "N/A" try { println("Input time: ${time} (type: ${time.class})") - def seconds = time as Double - println("Converted to double: ${seconds}") + def totalSeconds = time as Double + println("Converted to double: ${totalSeconds}") + + def hours = (totalSeconds / 3600) as Integer + def minutes = ((totalSeconds % 3600) / 60) as Integer + def seconds = (totalSeconds % 60) as Integer + + return String.format("%02d:%02d:%02d", hours, minutes, seconds) - if (seconds < 60) { - return "${Math.round(seconds * 100) / 100}s" - } else { - def minutes = (seconds / 60) as Integer - def remainingSeconds = Math.round((seconds % 60) * 100) / 100 - return "${minutes}m ${remainingSeconds}s" - } } catch (Exception e) { println("Error converting time: ${e.message}") return time.toString() @@ -256,7 +236,6 @@ void runTest(Integer TEST_ID) { return true } catch (exc) { - printKubernetesStatus("AFTER","$clusterSuffix") echo "Test $testName has failed!" if (retryCount >= 1 || currentBuild.nextBuild != null) { currentBuild.result = 'FAILURE' @@ -624,6 +603,13 @@ pipeline { """ step([$class: 'JUnitResultArchiver', testResults: 'final_report.xml', healthScaleFactor: 1.0]) archiveArtifacts 'final_report.xml, final_report.html' + publishHTML (target : [allowMissing: true, + alwaysLinkToLastBuild: true, + keepAll: false, + reportDir: '.', + reportFiles: 'final_report.html', + reportName: 'PSMDB Test Report', + reportTitles: 'Test Report']) } else { echo "No report files found in e2e-tests/reports, skipping report generation" } diff --git a/e2e-tests/tools/report_generator.py b/e2e-tests/tools/report_generator.py index ffc436518..27dcb6f80 100644 --- a/e2e-tests/tools/report_generator.py +++ b/e2e-tests/tools/report_generator.py @@ -44,7 +44,7 @@ def capture_k8s_logs(namespace: str) -> str: commands = [ ( "PSMDB Operator Logs", - f"kubectl logs -l app.kubernetes.io/name=percona-server-mongodb-operator --tail=50 -n {namespace}", + f"kubectl logs -l app.kubernetes.io/name=percona-server-mongodb-operator --tail=50 -n psmdb-operator", ), ] return run_kubectl_commands(commands) From 49d69793b326b656956b8d869871346ea18fd8e8 Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Tue, 17 Jun 2025 16:05:23 -0300 Subject: [PATCH 10/11] compress k8s resources folder --- Jenkinsfile | 18 +++++++++++------- e2e-tests/tools/k8s_resources_collector.py | 10 +++------- e2e-tests/tools/report_generator.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 800227c03..0763c7cfa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -600,16 +600,20 @@ pipeline { source \$HOME/.local/bin/env uv run pytest_html_merger -i e2e-tests/reports -o final_report.html uv run junitparser merge --glob 'e2e-tests/reports/*.xml' final_report.xml + echo 'DEBUG: Files in reports folder' + ls e2e-tests/reports/*.html + ls e2e-tests/reports/*.xml """ step([$class: 'JUnitResultArchiver', testResults: 'final_report.xml', healthScaleFactor: 1.0]) archiveArtifacts 'final_report.xml, final_report.html' - publishHTML (target : [allowMissing: true, - alwaysLinkToLastBuild: true, - keepAll: false, - reportDir: '.', - reportFiles: 'final_report.html', - reportName: 'PSMDB Test Report', - reportTitles: 'Test Report']) + // Currently Html Publisher plugin is not available + // publishHTML (target : [allowMissing: true, + // alwaysLinkToLastBuild: true, + // keepAll: false, + // reportDir: '.', + // reportFiles: 'final_report.html', + // reportName: 'PSMDB Test Report', + // reportTitles: 'Test Report']) } else { echo "No report files found in e2e-tests/reports, skipping report generation" } diff --git a/e2e-tests/tools/k8s_resources_collector.py b/e2e-tests/tools/k8s_resources_collector.py index 40f87d2bc..0a5714942 100644 --- a/e2e-tests/tools/k8s_resources_collector.py +++ b/e2e-tests/tools/k8s_resources_collector.py @@ -3,6 +3,7 @@ import os import re import sys +import tarfile import threading import subprocess from datetime import datetime @@ -312,13 +313,8 @@ def collect_k8s_resources( try: collector.collect_all() - return { - "success": True, - "output_dir": collector.output_dir, - "error_log": collector.error_log_file, - "namespace": namespace, - "timestamp": collector.timestamp, - } + with tarfile.open(f"{collector.output_dir}.tar.gz", "w:gz") as tar: + tar.add(collector.output_dir, arcname=os.path.basename(collector.output_dir)) except Exception as e: return {"success": False, "error": str(e), "namespace": namespace} diff --git a/e2e-tests/tools/report_generator.py b/e2e-tests/tools/report_generator.py index 27dcb6f80..531ecd349 100644 --- a/e2e-tests/tools/report_generator.py +++ b/e2e-tests/tools/report_generator.py @@ -44,7 +44,7 @@ def capture_k8s_logs(namespace: str) -> str: commands = [ ( "PSMDB Operator Logs", - f"kubectl logs -l app.kubernetes.io/name=percona-server-mongodb-operator --tail=50 -n psmdb-operator", + "kubectl logs -l app.kubernetes.io/name=percona-server-mongodb-operator --tail=50 -n psmdb-operator", ), ] return run_kubectl_commands(commands) From b0fe7cc8869b13713717dba16d054771d1097f0b Mon Sep 17 00:00:00 2001 From: Julio Pasinatto Date: Tue, 17 Jun 2025 20:18:08 -0300 Subject: [PATCH 11/11] Fix mypy issues --- e2e-tests/tools/k8s_resources_collector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e-tests/tools/k8s_resources_collector.py b/e2e-tests/tools/k8s_resources_collector.py index 0a5714942..ff539aab7 100644 --- a/e2e-tests/tools/k8s_resources_collector.py +++ b/e2e-tests/tools/k8s_resources_collector.py @@ -8,7 +8,7 @@ import subprocess from datetime import datetime from concurrent.futures import ThreadPoolExecutor, Future, as_completed -from typing import List, Optional, Dict, Any +from typing import List, Optional class K8sCollector: @@ -302,7 +302,7 @@ def collect_all(self) -> None: def collect_k8s_resources( namespace: str, custom_resources: Optional[List[str]] = None, output_dir: Optional[str] = None -) -> Dict[str, Any]: +) -> None: """ Collect Kubernetes resources for a given namespace. """ @@ -316,7 +316,7 @@ def collect_k8s_resources( with tarfile.open(f"{collector.output_dir}.tar.gz", "w:gz") as tar: tar.add(collector.output_dir, arcname=os.path.basename(collector.output_dir)) except Exception as e: - return {"success": False, "error": str(e), "namespace": namespace} + print(f"error collecting from namespace: {namespace} - {str(e)}") def get_namespace(test_log: str) -> str: