[Bugfix] Codecov does not cover multiprocessed tests #879 (#893)

Co-authored-by: Vincent Moens <vincentmoens@gmail.com>

Author: kadeng

.circleci/unittest/helpers/coverage_run_parallel.py

@@ -0,0 +1,64 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+When coverage run is launched with --concurrency=multiprocessing, it
+needs a config file for further arguments.
+
+This script is a drop-in wrapper to conveniently pass command line arguments
+nevertheless. It writes temporary coverage config files on the fly and
+invokes coverage with proper arguments
+"""
+
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import List
+
+
+def write_config(config_path: Path, argv: List[str]) -> None:
+    """
+    Write a coverage.py config that is equivalent to the command line arguments passed here.
+    Args:
+        config_path: Path to write config to
+        argv: Arguments passed to this script, which need to be converted to config file entries
+    """
+    assert not config_path.exists(), "Temporary coverage config exists already"
+    cmdline = " ".join(shlex.quote(arg) for arg in argv[1:])
+    with open(str(config_path), "wt", encoding="utf-8") as fh:
+        fh.write(
+            f"""# .coveragerc to control coverage.py
+[run]
+parallel=True
+concurrency=
+    multiprocessing
+    thread
+command_line={cmdline}
+"""
+        )
+
+
+def main(argv: List[str]) -> int:
+    if len(argv) < 1:
+        print("Usage: 'python coverage_run_parallel.py <command> [command arguments]'")
+        sys.exit(1)
+    # The temporary config is written into a temp dir that will be deleted
+    # including all contents on context exit.
+    # Note: Do not use tempfile.NamedTemporaryFile as this file cannot
+    # be read by other processes on Windows
+    with tempfile.TemporaryDirectory(prefix=".torchrl_coverage_config_tmp_") as tempdir:
+        config_path = Path(tempdir) / ".coveragerc"
+        os.environ["COVERAGE_RCFILE"] = str(
+            config_path
+        )  # This gets passed down to subprocesses
+        write_config(config_path, argv)
+        return subprocess.run(["coverage", "run"]).returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

.circleci/unittest/linux/scripts/run_test.sh

@@ -24,7 +24,8 @@ lib_dir="${env_dir}/lib"
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$lib_dir
 export MKL_THREADING_LAYER=GNU
 
-coverage run -m pytest test/smoke_test.py -v --durations 20
-coverage run -m pytest test/smoke_test_deps.py -v --durations 20 -k 'test_gym or test_dm_control_pixels or test_dm_control or test_tb'
-coverage run -m pytest --instafail -v --durations 20
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test.py -v --durations 20
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test_deps.py -v --durations 20 -k 'test_gym or test_dm_control_pixels or test_dm_control or test_tb'
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest --instafail -v --durations 20
+coverage combine
 coverage xml -i

.circleci/unittest/linux_examples/scripts/run_test.sh

@@ -24,9 +24,9 @@ lib_dir="${env_dir}/lib"
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$lib_dir
 export MKL_THREADING_LAYER=GNU
 
-coverage run -m pytest test/smoke_test.py -v --durations 20
-coverage run -m pytest test/smoke_test_deps.py -v --durations 20
-coverage run examples/ddpg/ddpg.py \
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test.py -v --durations 20
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test_deps.py -v --durations 20
+python .circleci/unittest/helpers/coverage_run_parallel.py examples/ddpg/ddpg.py \
   total_frames=48 \
   init_random_frames=10 \
   batch_size=10 \
@@ -38,7 +38,7 @@ coverage run examples/ddpg/ddpg.py \
   record_video=True \
   record_frames=4 \
   buffer_size=120
-coverage run examples/a2c/a2c.py \
+python .circleci/unittest/helpers/coverage_run_parallel.py examples/a2c/a2c.py \
   total_frames=48 \
   batch_size=10 \
   frames_per_batch=16 \
@@ -49,7 +49,7 @@ coverage run examples/a2c/a2c.py \
   record_video=True \
   record_frames=4 \
   logger=csv
-coverage run examples/dqn/dqn.py \
+python .circleci/unittest/helpers/coverage_run_parallel.py examples/dqn/dqn.py \
   total_frames=48 \
   init_random_frames=10 \
   batch_size=10 \
@@ -61,7 +61,7 @@ coverage run examples/dqn/dqn.py \
   record_video=True \
   record_frames=4 \
   buffer_size=120
-coverage run examples/redq/redq.py \
+python .circleci/unittest/helpers/coverage_run_parallel.py examples/redq/redq.py \
   total_frames=48 \
   init_random_frames=10 \
   batch_size=10 \
@@ -73,7 +73,7 @@ coverage run examples/redq/redq.py \
   record_video=True \
   record_frames=4 \
   buffer_size=120
-coverage run examples/sac/sac.py \
+python .circleci/unittest/helpers/coverage_run_parallel.py examples/sac/sac.py \
   total_frames=48 \
   init_random_frames=10 \
   batch_size=10 \
@@ -85,7 +85,7 @@ coverage run examples/sac/sac.py \
   record_video=True \
   record_frames=4 \
   buffer_size=120
-coverage run examples/ppo/ppo.py \
+python .circleci/unittest/helpers/coverage_run_parallel.py examples/ppo/ppo.py \
   total_frames=48 \
   batch_size=10 \
   frames_per_batch=16 \
@@ -96,7 +96,7 @@ coverage run examples/ppo/ppo.py \
   record_video=True \
   record_frames=4 \
   lr_scheduler=
-coverage run examples/dreamer/dreamer.py \
+python .circleci/unittest/helpers/coverage_run_parallel.py examples/dreamer/dreamer.py \
   total_frames=48 \
   init_random_frames=10 \
   batch_size=10 \
@@ -109,4 +109,5 @@ coverage run examples/dreamer/dreamer.py \
   record_frames=4 \
   buffer_size=120 \
   rssm_hidden_dim=17
+coverage combine
 coverage xml -i

.circleci/unittest/linux_libs/scripts_brax/run_test.sh

@@ -28,5 +28,6 @@ export MAGNUM_LOG=verbose MAGNUM_GPU_VALIDATION=ON
 # this workflow only tests the libs
 python -c "import brax"
 
-coverage run -m pytest test/test_libs.py --instafail -v --durations 20 --capture no -k TestBrax
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 20 --capture no -k TestBrax
+coverage combine
 coverage xml -i

.circleci/unittest/linux_libs/scripts_envpool/run_test.sh

@@ -27,5 +27,6 @@ export MKL_THREADING_LAYER=GNU
 # this workflow only tests the libs
 python -c "import envpool"
 
-coverage run -m pytest test/test_libs.py --instafail -v --durations 20 --capture no -k TestEnvPool
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 20 --capture no -k TestEnvPool
+coverage combine
 coverage xml -i

.circleci/unittest/linux_libs/scripts_gym/run_test.sh

@@ -21,8 +21,8 @@ lib_dir="${env_dir}/lib"
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$lib_dir
 export MKL_THREADING_LAYER=GNU
 
-coverage run -m pytest test/smoke_test.py -v --durations 20
-coverage run -m pytest test/smoke_test_deps.py -v --durations 20 -k 'test_gym or test_dm_control_pixels or test_dm_control'
-MUJOCO_GL=egl python test/test_libs.py -k test_collect
-MUJOCO_GL=egl coverage run -m pytest --instafail -v --durations 20 -k 'test_libs'
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test.py -v --durations 20
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test_deps.py -v --durations 20 -k 'test_gym or test_dm_control_pixels or test_dm_control'
+MUJOCO_GL=egl python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest --instafail -v --durations 20 -k 'test_libs'
+coverage combine
 coverage xml -i

.circleci/unittest/linux_libs/scripts_habitat/run_test.sh

@@ -41,5 +41,6 @@ env = HabitatEnv('HabitatRenderPick-v0')
 env.reset()
 """
 
-coverage run -m pytest test/test_libs.py --instafail -v --durations 20 --capture no -k TestHabitat
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 20 --capture no -k TestHabitat
+coverage combine
 coverage xml -i

.circleci/unittest/linux_libs/scripts_jumanji/run_test.sh

@@ -28,5 +28,6 @@ export MAGNUM_LOG=verbose MAGNUM_GPU_VALIDATION=ON
 # this workflow only tests the libs
 python -c "import jumanji"
 
-coverage run -m pytest test/test_libs.py --instafail -v --durations 20 --capture no -k TestJumanji
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 20 --capture no -k TestJumanji
+coverage combine
 coverage xml -i

.circleci/unittest/linux_libs/scripts_vmas/run_test.sh

@@ -25,5 +25,6 @@ export MAGNUM_LOG=verbose MAGNUM_GPU_VALIDATION=ON
 # this workflow only tests the libs
 python -c "import vmas"
 
-coverage run -m pytest test/test_libs.py --instafail -v --durations 20 --capture no -k TestVmas
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/test_libs.py --instafail -v --durations 20 --capture no -k TestVmas
+coverage combine
 coverage xml -i

.circleci/unittest/linux_olddeps/scripts_gym_0_13/run_test.sh

@@ -20,10 +20,11 @@ lib_dir="${env_dir}/lib"
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$lib_dir
 export MKL_THREADING_LAYER=GNU
 
-coverage run -m pytest test/smoke_test.py -v --durations 20
-coverage run -m pytest test/smoke_test_deps.py -v --durations 20 -k 'test_gym or test_dm_control_pixels or test_dm_control'
-#MUJOCO_GL=egl coverage run -m xvfb-run -a pytest --instafail -v --durations 20
-MUJOCO_GL=egl coverage run -m pytest --instafail -v --durations 20
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test.py -v --durations 20
+python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test_deps.py -v --durations 20 -k 'test_gym or test_dm_control_pixels or test_dm_control'
+#MUJOCO_GL=egl python .circleci/unittest/helpers/coverage_run_parallel.py -m xvfb-run -a pytest --instafail -v --durations 20
+MUJOCO_GL=egl python .circleci/unittest/helpers/coverage_run_parallel.py -m pytest --instafail -v --durations 20
 #pytest --instafail -v --durations 20
 #python test/test_libs.py
+coverage combine
 coverage xml -i