[ci] Re-enable codecov in the project

.github/pseudo-cluster/reframe/docker-entrypoint.sh

@@ -8,10 +8,11 @@ sudo service munge start
 cp -r /usr/local/share/reframe .
 cd reframe
 ./bootstrap.sh
+pip install pytest-cov
 
 echo "Running unittests with backend scheduler: ${BACKEND}"
 
 tempdir=$(mktemp -d -p /scratch)
-TMPDIR=$tempdir ./test_reframe.py -v \
+TMPDIR=$tempdir ./test_reframe.py --cov=reframe --cov-report=xml \
     --rfm-user-config=ci-scripts/configs/ci-cluster.py \
     --rfm-user-system=pseudo-cluster:compute-${BACKEND:-squeue}

.github/workflows/main.yml

@@ -18,15 +18,11 @@ jobs:
       run: |
         ./bootstrap.sh
     - name: Generic Unittests
-      if: matrix.python-version != '3.8'
       run: |
-        ./test_reframe.py
-    - name: Generic Unittests and Coverage Report
-      if:  matrix.python-version == '3.8'
-      run: |
-        python -m pip install coverage
-        coverage run --source=reframe ./test_reframe.py
-        coverage report -m
+        pip install pytest-cov
+        ./test_reframe.py --cov=reframe --cov-report=xml
+    - name: Upload coverage reports
+      uses: codecov/codecov-action@v4.2.0
 
   unittest-py-eol:
     runs-on: ubuntu-latest
@@ -40,7 +36,10 @@ jobs:
         docker build --build-arg PYTHON_VERSION=${{ matrix.python-version }} -f ci-scripts/dockerfiles/reframe-python.dockerfile -t reframe-python${{ matrix.python-version }}:latest .
     - name: Run Unittests
       run: |
-        docker run reframe-python${{ matrix.python-version }}:latest
+        docker run --name reframe-python${{ matrix.python-version }} reframe-python${{ matrix.python-version }}:latest
+        docker cp reframe-python${{ matrix.python-version }}:/home/rfmuser/reframe/coverage.xml .
+    - name: Upload coverage reports
+      uses: codecov/codecov-action@v4.2.0
 
   unittest-macos:
     runs-on: macos-latest
@@ -58,7 +57,10 @@ jobs:
         ./bootstrap.sh
     - name: Generic Unittests
       run: |
-        ./test_reframe.py
+        pip install pytest-cov
+        ./test_reframe.py --cov=reframe --cov-report=xml
+    - name: Upload coverage reports
+      uses: codecov/codecov-action@v4.2.0
 
   modulestest:
     runs-on: ubuntu-latest
@@ -78,7 +80,10 @@ jobs:
         docker build -f ci-scripts/dockerfiles/reframe-${{ matrix.modules-version }}.dockerfile -t reframe-${{ matrix.modules-version }}:latest .
     - name: Run Unittests
       run: |
-        docker run reframe-${{ matrix.modules-version }}:latest
+        docker run --name=reframe-${{ matrix.modules-version }} reframe-${{ matrix.modules-version }}:latest
+        docker cp reframe-${{ matrix.modules-version }}:/home/rfmuser/reframe/coverage.xml .
+    - name: Upload coverage reports
+      uses: codecov/codecov-action@v4.2.0
 
   eb-spack-howto:
     runs-on: ubuntu-latest

.github/workflows/test-flux.yaml

@@ -30,6 +30,7 @@ jobs:
         run: |
           apt-get update && apt-get install -y python3-pip
           ./bootstrap.sh
+          pip install pytest-cov
           export PATH=$PWD/bin:$PATH
           which reframe
 
@@ -40,4 +41,6 @@ jobs:
           which reframe
           flux start reframe -c examples/howto/flux -C examples/howto/flux/settings.py -l
           flux start reframe -c examples/howto/flux -C examples/howto/flux/settings.py -r
-          flux start python3 ./test_reframe.py --rfm-user-config=examples/howto/flux/settings.py -vvvv
+          flux start python3 ./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=examples/howto/flux/settings.py
+      - name: Upload coverage reports
+        uses: codecov/codecov-action@v4.2.0

.github/workflows/test-schedulers.yaml

@@ -19,6 +19,11 @@ jobs:
     - name: Build Images
       run: |
         docker compose -f .github/pseudo-cluster/docker-compose.yml build
-    - name: Run Unittests with ${{ matrix.scheduler }} scheduler
+    - name: Run unit tests with ${{ matrix.scheduler }} scheduler
       run: |
         BACKEND=${{ matrix.scheduler }} docker compose -f .github/pseudo-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from unittest-sched
+        _exitcode=$?
+        docker cp unittest-sched:/home/admin/reframe/coverage.xml .
+        exit $_exitcode
+    - name: Upload coverage reports
+      uses: codecov/codecov-action@v4.2.0

ci-scripts/dockerfiles/reframe-lmod.dockerfile

@@ -20,5 +20,6 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/
 WORKDIR /home/rfmuser/reframe
 
 RUN ./bootstrap.sh
+RUN pip install pytest-cov
 
-CMD ["/bin/bash", "-c", "./test_reframe.py --rfm-user-config=ci-scripts/configs/lmod.py -v"]
+CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/lmod.py"]

ci-scripts/dockerfiles/reframe-lmod77.dockerfile

@@ -20,5 +20,6 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/
 WORKDIR /home/rfmuser/reframe
 
 RUN ./bootstrap.sh
+RUN pip install pytest-cov
 
-CMD ["/bin/bash", "-c", "./test_reframe.py --rfm-user-config=ci-scripts/configs/lmod.py -v"]
+CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/lmod.py"]

ci-scripts/dockerfiles/reframe-python.dockerfile

@@ -18,5 +18,6 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/
 WORKDIR /home/rfmuser/reframe
 
 RUN ./bootstrap.sh +docs
+RUN pip install pytest-cov
 
-CMD ["/bin/bash", "-c", "./test_reframe.py -v"]
+CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml"]

ci-scripts/dockerfiles/reframe-tmod32.dockerfile

@@ -5,10 +5,11 @@
 FROM ghcr.io/reframe-hpc/tmod:3.2.10
 
 # ReFrame requirements
-RUN yum -y install gcc make git python3
+RUN yum -y install gcc make git python3 python3-pip
 
 # ReFrame user
 RUN useradd -ms /bin/bash rfmuser
+RUN pip3 install pytest-cov
 
 USER rfmuser
 
@@ -19,4 +20,4 @@ WORKDIR /home/rfmuser/reframe
 
 RUN ./bootstrap.sh
 
-CMD ["/bin/bash", "-c", "./test_reframe.py --rfm-user-config=ci-scripts/configs/tmod32.py -v"]
+CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/tmod32.py"]

ci-scripts/dockerfiles/reframe-tmod4.dockerfile

@@ -21,5 +21,6 @@ COPY --chown=rfmuser . /home/rfmuser/reframe/
 WORKDIR /home/rfmuser/reframe
 
 RUN ./bootstrap.sh
+RUN pip install pytest-cov
 
-CMD ["/bin/bash", "-c", "./test_reframe.py --rfm-user-config=ci-scripts/configs/tmod4.py -v"]
+CMD ["/bin/bash", "-c", "./test_reframe.py --cov=reframe --cov-report=xml --rfm-user-config=ci-scripts/configs/tmod4.py"]

reframe/__init__.py

@@ -22,5 +22,5 @@
 
 
 # Import important names for user tests
-from reframe.core.pipeline import *     # noqa: F401, F403
-from reframe.core.decorators import *   # noqa: F401, F403
+from reframe.core.pipeline import *     # noqa: F401, F403, E402
+from reframe.core.decorators import *   # noqa: F401, F403, E402