testing-related fixes

.travis.yml

@@ -11,7 +11,7 @@ env:
   - PIP_DIR="$HOME/virtualenv"
   - GATK_PATH="$CACHE_DIR"
   - PYTHONIOENCODING=UTF8
-  - PYTEST_ADDOPTS="-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov kmer_utils --cov reports --cov taxon_filter --cov tools --cov util --cov file_utils"
+  - PYTEST_ADDOPTS="-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov kmer_utils --cov reports --cov taxon_filter --cov tools --cov util --cov file_utils --parts-tot=2"
 
   - DOCKER_REGISTRY="quay.io"
   - DOCKER_REPO_PROD="quay.io/broadinstitute/viral-ngs"
@@ -159,34 +159,50 @@ jobs:
       #stage: test
       sudo: required
       env:
-        - TRAVIS_JOB=test_py27
+        - TRAVIS_JOB=test_py27_part1
+        - PYTEST_EXTRA_OPTS="--part-num 1"
       before_install: travis/before_install.sh
       install:
         - source travis/install-conda.sh
         - travis/install-gatk.sh
         - travis/install-tools.sh
         - source travis/activate-conda.sh
       script:
-        - travis/tests-unit.sh
-        - travis/tests-long.sh
+        - travis/tests-pytest.sh
+      before_cache:
+        - conda clean --all --yes
+    - language: python
+      python: 2.7
+      #stage: test
+      sudo: required
+      env:
+        - TRAVIS_JOB=test_py27_part2
+        - PYTEST_EXTRA_OPTS="--part-num 2"
+      before_install: travis/before_install.sh
+      install:
+        - source travis/install-conda.sh
+        - travis/install-gatk.sh
+        - travis/install-tools.sh
+        - source travis/activate-conda.sh
+      script:
+        - travis/tests-pytest.sh
       before_cache:
         - conda clean --all --yes
-
     - language: python
       python: 3.6
       #stage: test
       sudo: required
       env:
-        - TRAVIS_JOB=test_py36
+        - TRAVIS_JOB=test_py36_part1
+        - PYTEST_EXTRA_OPTS="--part-num 1"
       before_install: travis/before_install.sh
       install:
         - source travis/install-conda.sh
         - travis/install-gatk.sh
         - travis/install-tools.sh
         - source travis/activate-conda.sh
       script:
-        - travis/tests-long.sh
-        - travis/tests-unit.sh
+        - travis/tests-pytest.sh
       after_success:
         - coveralls
       before_cache:
@@ -197,20 +213,20 @@ jobs:
       #stage: test
       sudo: required
       env:
-        - TRAVIS_JOB=test_snakemake
+        - TRAVIS_JOB=test_py36_part2
+        - PYTEST_EXTRA_OPTS="--part-num 2"
       before_install: travis/before_install.sh
       install:
         - source travis/install-conda.sh
         - travis/install-gatk.sh
         - travis/install-tools.sh
         - source travis/activate-conda.sh
       script:
-        - travis/tests-snakemake.sh
+        - travis/tests-pytest.sh
       after_success:
         - coveralls
       before_cache:
         - conda clean --all --yes
-
     #- language: python
     #  stage: test
     #  env:

conftest.py

@@ -41,11 +41,39 @@ def pytest_addoption(parser):
         action="store_true",
         default=False,
         help="run slow tests."
-    ),
+    )
+    group.addoption('--parts-tot', type=int, help="total number of parts")
+    group.addoption('--part-num', type=int, help="number of this part")
+
 
 def pytest_configure(config):
     reporter = FixtureReporter(config)
     config.pluginmanager.register(reporter, 'fixturereporter')
+    config.post_collection_msgs = []
+
+def pytest_collection_modifyitems(session, config, items):
+    if not config.getoption('--runslow', default=False):
+        non_slow_items = [item for item in items if not item.get_closest_marker(name='slow')]
+        config.post_collection_msgs.append('skipped collection of {} slow tests'.format(len(items) - len(non_slow_items)))
+        items[:] = non_slow_items
+
+    part_num = config.getoption('--part-num', default=None)
+    tot_parts = config.getoption('--parts-tot', default=None)
+    if part_num and tot_parts:
+        util.misc.chk(1 <= part_num <= tot_parts)
+        items.sort(key=operator.attrgetter('nodeid'))
+        all_nodeids = [item.nodeid for item in items]
+        part_size = max(1, int(float(len(all_nodeids)) / float(tot_parts)))
+        part_beg = (part_num - 1) * part_size
+        part_end = part_beg+part_size+1
+        items[:] = items[part_beg:part_end]
+        config.post_collection_msgs.append('part {} of {}: collected items {}-{} of {}'.format(part_num, tot_parts,
+                                                                                               part_beg+1, part_end,
+                                                                                               len(all_nodeids)))
+
+
+def pytest_report_collectionfinish(config, startdir, items):
+    return getattr(config, 'post_collection_msgs')
 
 #
 # Fixtures for creating a temp dir at session/module/class/function scope.

requirements-conda-tests.txt

@@ -1,10 +1,10 @@
-coveralls=1.3.0
+coveralls=1.5.1
 flake8>=3.2.1
 mock=2.0.0
 pycodestyle=2.2.0
-pytest=3.7.1
+pytest=4.1.1
 pytest-cov==2.5.1
 pytest-mock=1.10.0
-pytest-xdist=1.22.5
-coverage=4.5.1
+pytest-xdist=1.26.0
+coverage=4.5.2
 six=1.10.0

test/__init__.py

@@ -28,13 +28,6 @@
 else:
     _CPUS = available_cpu_count()
 
-def make_slow_test_marker():
-    """Create a marker for marking slow tests."""
-    return pytest.mark.skipif(
-        not pytest.config.getoption("--runslow", default=False),
-        reason="need --runslow option to run"
-    )
-
 def assert_equal_contents(testCase, filename1, filename2):
     'Assert contents of two files are equal for a unittest.TestCase'
     testCase.assertTrue(filecmp.cmp(filename1, filename2, shallow=False))
@@ -124,4 +117,4 @@ def inputs(self, *fnames):
 def assert_none_executable():
     testDir = os.path.dirname(__file__)
     assert all(not os.access(os.path.join(testDir, filename), os.X_OK) for filename in os.listdir(testDir)
-               if filename.endswith('.py'))
+               if filename.endswith('.py'))

test/pipelines/snakemake/test_diamond.py

@@ -1,17 +1,20 @@
 #!/usr/bin/env python
 
-import os
+import os, os.path
 import sys
 
 import pytest
 
 import tools
 from test.pipelines.snakemake import snake
-from test.integration.test_diamond import * # for pytest fixtures
+from test.integration.test_diamond import fastq_to_sam, sam_to_fastq, diamond, krona, db_type, input_bam, \
+    input_fastqs, taxonomy_db, diamond_db, krona_db, test_diamond
 
 @pytest.mark.skipif(tools.is_osx(), reason="not currently tested under OSX")
 @pytest.mark.skipif(sys.version_info < (3, 5), reason="Python version is too old for snakemake.")
 def test_pipes(tmpdir_function, diamond_db, taxonomy_db, krona_db, input_bam):
+    join = os.path.join
+
     runner = snake.SnakemakeRunner(workdir=tmpdir_function)
     override_config = {
         'diamond_db': diamond_db,
@@ -32,4 +35,4 @@ def test_pipes(tmpdir_function, diamond_db, taxonomy_db, krona_db, input_bam):
     )
     runner.run([krona_out])
     assert os.path.getsize(os.path.join(runner.workdir, diamond_out)) > 0
-    assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0
+    assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0

test/pipelines/snakemake/test_kraken.py

@@ -1,15 +1,17 @@
 #!/usr/bin/env python
 
-import os
+import os, os.path
+
 import sys
 
 import pytest
 
 from test.pipelines.snakemake import snake
-from test.integration.test_kraken import * # for pytest fixtures
+from test.integration.test_kraken import input_bam, kraken, krona, db_type, kraken_db, krona_db
 
 @pytest.mark.skipif(sys.version_info < (3, 5), reason="Python version is too old for snakemake.")
 def test_pipes(tmpdir_function, kraken_db, krona_db, input_bam):
+    join = os.path.join
     runner = snake.SnakemakeRunner(workdir=tmpdir_function)
     override_config = {
         'kraken_db': kraken_db,
@@ -33,4 +35,4 @@ def test_pipes(tmpdir_function, kraken_db, krona_db, input_bam):
     # runner.run(['all_metagenomics'])
     runner.run([kraken_out, krona_out])
     assert os.path.getsize(os.path.join(runner.workdir, kraken_out)) > 0
-    assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0
+    assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0

test/pipelines/snakemake/test_metagenomics_align.py

@@ -1,15 +1,16 @@
 #!/usr/bin/env python
 
-import os
+import os, os.path
 import sys
 
 import pytest
 
 from test.pipelines.snakemake import snake
-from test.integration.test_metagenomics_align import * # for pytest fixtures
+from test.integration.test_metagenomics_align import fastq_to_sam, taxonomy_db, input_bam, bwa, db_type, bwa_db
 
 @pytest.mark.skipif(sys.version_info < (3, 5), reason="Python version is too old for snakemake.")
 def test_pipes(tmpdir_function, bwa_db, taxonomy_db, input_bam):
+    join = os.path.join
     runner = snake.SnakemakeRunner(workdir=tmpdir_function)
     override_config = {
         'align_rna_db': bwa_db,
@@ -32,4 +33,4 @@ def test_pipes(tmpdir_function, bwa_db, taxonomy_db, input_bam):
 
     runner.run([report_out])
     assert os.path.getsize(os.path.join(runner.workdir, report_out)) > 0
-    assert os.path.getsize(os.path.join(runner.workdir, bam_out)) > 0
+    assert os.path.getsize(os.path.join(runner.workdir, bam_out)) > 0

test/unit/test_kmer_utils.py

@@ -24,12 +24,8 @@
 import tools.kmc
 import tools.samtools
 
-from test import make_slow_test_marker
-
 _log = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
-slow_test = make_slow_test_marker()  # pylint: disable=invalid-name
-
 class TestCommandHelp(unittest.TestCase):
 
     def test_help_parser_for_each_command(self):
@@ -246,6 +242,13 @@ def _stringify(arg):
     """Return a string based on `arg`, suitable for use as a pytest test id"""
     return util.file.string_to_file_name(str(arg))
 
+def _get_build_db_args(t_dir, seq_files, kmer_db_opts):
+    key = (seq_files, kmer_db_opts)
+    k_db = os.path.join(t_dir, 'bld_kmer_db_{}'.format(hash(key)))
+    seq_files = list(map(_inp, seq_files.split()))
+    return util.cmd.parse_cmd(module=kmer_utils, cmd='build_kmer_db',
+                              args=seq_files + [k_db] + kmer_db_opts.split() + ['--memLimitGb', 4])
+
 def _do_build_kmer_db(t_dir, val_cache, seq_files, kmer_db_opts):
     """Build a database of kmers from given sequence file(s) using given options.
 
@@ -302,9 +305,10 @@ def dict_module():
 def kmer_db_fixture(request, tmpdir_module, dict_module):
     yield _do_build_kmer_db(tmpdir_module, dict_module, *request.param)
 
-@pytest.fixture(scope='module', params=KMER_DBS_EMPTY+KMER_DBS_SMALL, ids=_stringify)
-def kmer_db_fixture2(request, tmpdir_module, dict_module):
-    yield _do_build_kmer_db(tmpdir_module, dict_module, *request.param)
+@pytest.fixture(scope='module', params=[p for p in itertools.product(KMER_DBS_EMPTY+KMER_DBS_SMALL, repeat=2) if _get_build_db_args('/tmp', *p[0]).kmer_size == _get_build_db_args('/tmp', *p[1]).kmer_size], ids=_stringify)
+def kmer_db_pair_fixture(request, tmpdir_module, dict_module):
+    yield (_do_build_kmer_db(tmpdir_module, dict_module, *(request.param[0])),
+           _do_build_kmer_db(tmpdir_module, dict_module, *(request.param[1])))
 
 @pytest.mark.parametrize("kmer_db_fixture", KMER_DBS_EMPTY+KMER_DBS_SMALL+KMER_DBS_MEDIUM,
                          ids=_stringify, indirect=["kmer_db_fixture"])
@@ -345,7 +349,7 @@ def _test_build_kmer_db(kmer_db_fixture):
               in itertools.product(SEQ_FILES, KMER_SIZES, STRAND_OPTS,
                                    KMER_OCCS_OPTS, NTHREADS)]
 
-@slow_test
+@pytest.mark.slow
 @pytest.mark.parametrize("kmer_db_fixture", COMBO_OPTS, ids=_stringify, indirect=["kmer_db_fixture"])
 def test_build_kmer_db_combo(kmer_db_fixture):
     _test_build_kmer_db(kmer_db_fixture)
@@ -397,7 +401,7 @@ def test_filter_with_empty_db(kmer_db_fixture, reads_file, filter_opts, tmpdir_f
     _test_filter_reads(**locals())
 
 @pytest.mark.parametrize("kmer_db_fixture", [('ebola.fasta.gz', '-k 7')], ids=_stringify, indirect=["kmer_db_fixture"])
-@pytest.mark.parametrize("reads_file", [pytest.param('G5012.3.testreads.bam', marks=slow_test),
+@pytest.mark.parametrize("reads_file", [pytest.param('G5012.3.testreads.bam', marks=pytest.mark.slow),
                                         'G5012.3.subset.bam'])
 @pytest.mark.parametrize("filter_opts", ['--dbMinOccs 7  --readMinOccs 93',
                                          '--dbMinOccs 4 --readMinOccsFrac .6',
@@ -418,18 +422,16 @@ def test_kmer_set_counts(kmer_db_fixture, tmpdir_function, set_to_val):
 
 
 @pytest.mark.parametrize("op", ('intersect', 'union', 'kmers_subtract', 'counters_subtract'))
-def test_kmers_binary_op(kmer_db_fixture, kmer_db_fixture2, op, tmpdir_function):
-    if kmer_db_fixture.kmer_db_args.kmer_size != kmer_db_fixture2.kmer_db_args.kmer_size:
-        pytest.skip('(always skip) binary ops not defined on kmers of different size')
+def test_kmers_binary_op(kmer_db_pair_fixture, op, tmpdir_function):
     db_result = os.path.join(tmpdir_function, 'op_result')
-    _log.debug('fixture1args=%s', kmer_db_fixture.kmer_db_args)
-    _log.debug('fixture2args=%s', kmer_db_fixture2.kmer_db_args)
+    _log.debug('fixture1args=%s', kmer_db_pair_fixture[0].kmer_db_args)
+    _log.debug('fixture2args=%s', kmer_db_pair_fixture[1].kmer_db_args)
     _log.debug('op=%s', op)
     args = util.cmd.run_cmd(module=kmer_utils, cmd='kmers_binary_op',
-                            args=[op, kmer_db_fixture.kmer_db, kmer_db_fixture2.kmer_db, db_result]).args_parsed
+                            args=[op, kmer_db_pair_fixture[0].kmer_db, kmer_db_pair_fixture[1].kmer_db, db_result]).args_parsed
 
     kmc_counts = tools.kmc.KmcTool().get_kmer_counts(db_result)
-    kmcpy_counts = kmcpy.binary_op(op, kmer_db_fixture.kmc_kmer_counts, kmer_db_fixture2.kmc_kmer_counts,
+    kmcpy_counts = kmcpy.binary_op(op, kmer_db_pair_fixture[0].kmc_kmer_counts, kmer_db_pair_fixture[1].kmc_kmer_counts,
                                    result_counter_cap=args.result_counter_cap)
 
     assert kmc_counts == kmcpy_counts

test/unit/test_tools_samtools.py

@@ -78,6 +78,7 @@ def test_bam2fa(self):
             for fa in (fa1, fa2):
                 assert len(list(Bio.SeqIO.parse(fa, 'fasta')))==1
 
-        assert not os.path.isfile(fa1) and not os.path.isfile(fa2)
+        assert util.file.keep_tmp() or (not os.path.isfile(fa1) and not os.path.isfile(fa2))
+
 
 

test/unit/test_util_file.py

@@ -40,14 +40,15 @@ def testTempFiles():
 
                 tmp_fns.append(fn)
 
-        assert os.path.isfile(my_tmp_fn) and not os.path.isfile(my_tmp_fns[0]) and not os.path.isfile(my_tmp_fns[1])
+        assert os.path.isfile(my_tmp_fn)
+        assert util.file.keep_tmp() or (not os.path.isfile(my_tmp_fns[0]) and not os.path.isfile(my_tmp_fns[1]))
 
         largeString = 'A' * (2*1024*1024)
         util.file.dump_file(fname=my_tmp_fn, value=largeString)
         with pytest.raises(RuntimeError):
             util.file.slurp_file(my_tmp_fn, maxSizeMb=1)
 
-    assert not os.path.isfile(my_tmp_fn)
+    assert util.file.keep_tmp() or not os.path.isfile(my_tmp_fn)
 
 def test_check_paths(tmpdir):
     '''Test the util.file.check_paths()'''

travis/tests-long.sh → travis/tests-pytest.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
-pytest --cov-append test/integration
+set -eu -o pipefail
+
+pytest --cov-append $PYTEST_EXTRA_OPTS test
 
 rc=$?; if [[ $rc != 0 ]]; then sleep 10; exit $rc; fi
 # sleep to allow logs to be printed without truncation in the event of error