From af35910b3009a7a3967fc8ebf637d5901b43822f Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 25 Jun 2024 00:37:53 +0200 Subject: [PATCH] Re-enable codecov in the project We now combine the coverage from all unit tests that are run with different containers, so we have the overall coverage of the project. --- .../reframe/docker-entrypoint.sh | 3 ++- .github/workflows/main.yml | 27 +++++++++++-------- .github/workflows/test-flux.yaml | 5 +++- .github/workflows/test-schedulers.yaml | 7 ++++- .../dockerfiles/reframe-lmod.dockerfile | 3 ++- .../dockerfiles/reframe-lmod77.dockerfile | 3 ++- .../dockerfiles/reframe-python.dockerfile | 3 ++- .../dockerfiles/reframe-tmod32.dockerfile | 5 ++-- .../dockerfiles/reframe-tmod4.dockerfile | 3 ++- reframe/__init__.py | 4 +-- 10 files changed, 41 insertions(+), 22 deletions(-) diff --git a/.github/pseudo-cluster/reframe/docker-entrypoint.sh b/.github/pseudo-cluster/reframe/docker-entrypoint.sh index ccfe1c406d..71eb08fc03 100755 --- a/.github/pseudo-cluster/reframe/docker-entrypoint.sh +++ b/.github/pseudo-cluster/reframe/docker-entrypoint.sh @@ -8,10 +8,11 @@ sudo service munge start cp -r /usr/local/share/reframe . cd reframe ./bootstrap.sh +pip install pytest-cov echo "Running unittests with backend scheduler: ${BACKEND}" tempdir=$(mktemp -d -p /scratch) -TMPDIR=$tempdir ./test_reframe.py -v \ +TMPDIR=$tempdir ./test_reframe.py --cov=reframe --cov-report=xml \ --rfm-user-config=ci-scripts/configs/ci-cluster.py \ --rfm-user-system=pseudo-cluster:compute-${BACKEND:-squeue} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b215e29b12..3831e4257a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,15 +18,11 @@ jobs: run: | ./bootstrap.sh - name: Generic Unittests - if: matrix.python-version != '3.8' run: | - ./test_reframe.py - - name: Generic Unittests and Coverage Report - if: matrix.python-version == '3.8' - run: | - python -m pip install coverage - coverage run --source=reframe ./test_reframe.py - coverage report -m + pip install pytest-cov + ./test_reframe.py --cov=reframe --cov-report=xml + - name: Upload coverage reports + uses: codecov/codecov-action@v4.2.0 unittest-py-eol: runs-on: ubuntu-latest @@ -40,7 +36,10 @@ jobs: docker build --build-arg PYTHON_VERSION=${{ matrix.python-version }} -f ci-scripts/dockerfiles/reframe-python.dockerfile -t reframe-python${{ matrix.python-version }}:latest . - name: Run Unittests run: | - docker run reframe-python${{ matrix.python-version }}:latest + docker run --name reframe-python${{ matrix.python-version }} reframe-python${{ matrix.python-version }}:latest + docker cp reframe-python${{ matrix.python-version }}:/home/rfmuser/reframe/coverage.xml . + - name: Upload coverage reports + uses: codecov/codecov-action@v4.2.0 unittest-macos: runs-on: macos-latest @@ -58,7 +57,10 @@ jobs: ./bootstrap.sh - name: Generic Unittests run: | - ./test_reframe.py + pip install pytest-cov + ./test_reframe.py --cov=reframe --cov-report=xml + - name: Upload coverage reports + uses: codecov/codecov-action@v4.2.0 modulestest: runs-on: ubuntu-latest @@ -78,7 +80,10 @@ jobs: docker build -f ci-scripts/dockerfiles/reframe-${{ matrix.modules-version }}.dockerfile -t reframe-${{ matrix.modules-version }}:latest . - name: Run Unittests run: | - docker run reframe-${{ matrix.modules-version }}:latest + docker run --name=reframe-${{ matrix.modules-version }} reframe-${{ matrix.modules-version }}:latest + docker cp reframe-${{ matrix.modules-version }}:/home/rfmuser/reframe/coverage.xml . + - name: Upload coverage reports + uses: codecov/codecov-action@v4.2.0 eb-spack-howto: runs-on: ubuntu-latest diff --git a/.github/workflows/test-flux.yaml b/.github/workflows/test-flux.yaml index 8a5f3ff299..5d4f65421b 100644 --- a/.github/workflows/test-flux.yaml +++ b/.github/workflows/test-flux.yaml @@ -30,6 +30,7 @@ jobs: run: | apt-get update && apt-get install -y python3-pip ./bootstrap.sh + pip install pytest-cov export PATH=$PWD/bin:$PATH which reframe @@ -40,4 +41,6 @@ jobs: which reframe flux start reframe -c examples/howto/flux -C examples/howto/flux/settings.py -l flux start reframe -c examples/howto/flux -C examples/howto/flux/settings.py -r - flux start python3 ./test_reframe.py --rfm-user-config=examples/howto/flux/settings.py -vvvv + flux start python3 ./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=examples/howto/flux/settings.py + - name: Upload coverage reports + uses: codecov/codecov-action@v4.2.0 diff --git a/.github/workflows/test-schedulers.yaml b/.github/workflows/test-schedulers.yaml index 747787a4ca..329a3866f4 100644 --- a/.github/workflows/test-schedulers.yaml +++ b/.github/workflows/test-schedulers.yaml @@ -19,6 +19,11 @@ jobs: - name: Build Images run: | docker compose -f .github/pseudo-cluster/docker-compose.yml build - - name: Run Unittests with ${{ matrix.scheduler }} scheduler + - name: Run unit tests with ${{ matrix.scheduler }} scheduler run: | BACKEND=${{ matrix.scheduler }} docker compose -f .github/pseudo-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from unittest-sched + _exitcode=$? + docker cp unittest-sched:/home/admin/reframe/coverage.xml . + exit $_exitcode + - name: Upload coverage reports + uses: codecov/codecov-action@v4.2.0 diff --git a/ci-scripts/dockerfiles/reframe-lmod.dockerfile b/ci-scripts/dockerfiles/reframe-lmod.dockerfile index 3ab4804a6e..f95dae4b67 100644 --- a/ci-scripts/dockerfiles/reframe-lmod.dockerfile +++ b/ci-scripts/dockerfiles/reframe-lmod.dockerfile @@ -20,5 +20,6 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh +RUN pip install pytest-cov -CMD ["/bin/bash", "-c", "./test_reframe.py --rfm-user-config=ci-scripts/configs/lmod.py -v"] +CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/lmod.py"] diff --git a/ci-scripts/dockerfiles/reframe-lmod77.dockerfile b/ci-scripts/dockerfiles/reframe-lmod77.dockerfile index ba8e2c50ed..5ee2500394 100644 --- a/ci-scripts/dockerfiles/reframe-lmod77.dockerfile +++ b/ci-scripts/dockerfiles/reframe-lmod77.dockerfile @@ -20,5 +20,6 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh +RUN pip install pytest-cov -CMD ["/bin/bash", "-c", "./test_reframe.py --rfm-user-config=ci-scripts/configs/lmod.py -v"] +CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/lmod.py"] diff --git a/ci-scripts/dockerfiles/reframe-python.dockerfile b/ci-scripts/dockerfiles/reframe-python.dockerfile index 4330c785c2..d80d09f542 100644 --- a/ci-scripts/dockerfiles/reframe-python.dockerfile +++ b/ci-scripts/dockerfiles/reframe-python.dockerfile @@ -18,5 +18,6 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh +docs +RUN pip install pytest-cov -CMD ["/bin/bash", "-c", "./test_reframe.py -v"] +CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml"] diff --git a/ci-scripts/dockerfiles/reframe-tmod32.dockerfile b/ci-scripts/dockerfiles/reframe-tmod32.dockerfile index ccc1eb357e..df9418589f 100644 --- a/ci-scripts/dockerfiles/reframe-tmod32.dockerfile +++ b/ci-scripts/dockerfiles/reframe-tmod32.dockerfile @@ -5,10 +5,11 @@ FROM ghcr.io/reframe-hpc/tmod:3.2.10 # ReFrame requirements -RUN yum -y install gcc make git python3 +RUN yum -y install gcc make git python3 python3-pip # ReFrame user RUN useradd -ms /bin/bash rfmuser +RUN pip3 install pytest-cov USER rfmuser @@ -19,4 +20,4 @@ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh -CMD ["/bin/bash", "-c", "./test_reframe.py --rfm-user-config=ci-scripts/configs/tmod32.py -v"] +CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/tmod32.py"] diff --git a/ci-scripts/dockerfiles/reframe-tmod4.dockerfile b/ci-scripts/dockerfiles/reframe-tmod4.dockerfile index cf1b5105a9..360a448991 100644 --- a/ci-scripts/dockerfiles/reframe-tmod4.dockerfile +++ b/ci-scripts/dockerfiles/reframe-tmod4.dockerfile @@ -21,5 +21,6 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/ WORKDIR /home/rfmuser/reframe RUN ./bootstrap.sh +RUN pip install pytest-cov -CMD ["/bin/bash", "-c", "./test_reframe.py --rfm-user-config=ci-scripts/configs/tmod4.py -v"] +CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/tmod4.py"] diff --git a/reframe/__init__.py b/reframe/__init__.py index 7a71dbf689..13677e0266 100644 --- a/reframe/__init__.py +++ b/reframe/__init__.py @@ -22,5 +22,5 @@ # Import important names for user tests -from reframe.core.pipeline import * # noqa: F401, F403 -from reframe.core.decorators import * # noqa: F401, F403 +from reframe.core.pipeline import * # noqa: F401, F403, E402 +from reframe.core.decorators import * # noqa: F401, F403, E402