From b1e2a59904d667118156bc363135fca36cb47077 Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Thu, 17 Jan 2019 20:50:10 -0500 Subject: [PATCH 01/13] splitting CI into parts --- .travis.yml | 113 ++++++++++++++++-- conftest.py | 17 ++- test/pipelines/snakemake/test_diamond.py | 4 +- test/pipelines/snakemake/test_kraken.py | 4 +- .../snakemake/test_metagenomics_align.py | 4 +- travis/tests-pytest.sh | 8 ++ 6 files changed, 134 insertions(+), 16 deletions(-) create mode 100755 travis/tests-pytest.sh diff --git a/.travis.yml b/.travis.yml index b9729e388..e15351528 100644 --- a/.travis.yml +++ b/.travis.yml @@ -157,7 +157,8 @@ jobs: #stage: test sudo: required env: - - TRAVIS_JOB=test_py27 + - TRAVIS_JOB=test_py27_part1 + - PYTEST_EXTRA_OPTS="--tests-part 1 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -165,8 +166,43 @@ jobs: - travis/install-tools.sh - source travis/activate-conda.sh script: - - travis/tests-unit.sh - - travis/tests-long.sh + - travis/tests-pytest.sh + before_cache: + - conda clean --all --yes + + - language: python + python: 2.7 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py27_part2 + - PYTEST_EXTRA_OPTS="--tests-part 2 4" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + before_cache: + - conda clean --all --yes + + - language: python + python: 2.7 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py27_part3 + - PYTEST_EXTRA_OPTS="--tests-part 3 4" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh before_cache: - conda clean --all --yes @@ -175,7 +211,8 @@ jobs: #stage: test sudo: required env: - - TRAVIS_JOB=test_py36 + - TRAVIS_JOB=test_py36_part1 + - PYTEST_EXTRA_OPTS="--tests-part 1 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -183,8 +220,7 @@ jobs: - travis/install-tools.sh - source travis/activate-conda.sh script: - - travis/tests-long.sh - - travis/tests-unit.sh + - travis/tests-pytest.sh after_success: - coveralls before_cache: @@ -195,7 +231,8 @@ jobs: #stage: test sudo: required env: - - TRAVIS_JOB=test_snakemake + - TRAVIS_JOB=test_py36_part2 + - PYTEST_EXTRA_OPTS="--tests-part 2 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -203,7 +240,67 @@ jobs: - travis/install-tools.sh - source travis/activate-conda.sh script: - - travis/tests-snakemake.sh + - travis/tests-pytest.sh + after_success: + - coveralls + before_cache: + - conda clean --all --yes + + - language: python + python: 3.6 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py36_part3 + - PYTEST_EXTRA_OPTS="--tests-part 3 4" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + after_success: + - coveralls + before_cache: + - conda clean --all --yes + + - language: python + python: 3.6 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py36_part4 + - PYTEST_EXTRA_OPTS="--tests-part 4 4" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + after_success: + - coveralls + before_cache: + - conda clean --all --yes + + - language: python + python: 3.6 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py36 + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-long.sh + - travis/tests-unit.sh after_success: - coveralls before_cache: diff --git a/conftest.py b/conftest.py index 2aa0e040b..b5a712e1f 100644 --- a/conftest.py +++ b/conftest.py @@ -34,12 +34,25 @@ def pytest_addoption(parser): action="store_true", default=False, help="run slow tests." - ), + ) + group.addoption('--tests-part', type=int, nargs=2, help="this is part k of N") + def pytest_configure(config): reporter = FixtureReporter(config) config.pluginmanager.register(reporter, 'fixturereporter') +def pytest_collection_modifyitems(session, config, items): + part_num, tot_parts = config.getoption('--tests-part', default=(0,0)) + if tot_parts: + util.misc.chk(1 <= part_num <= tot_parts) + items.sort(key=operator.attrgetter('nodeid')) + all_nodeids = [item.nodeid for item in items] + part_size = max(1, int(float(len(all_nodeids)) / float(tot_parts))) + part_beg = (part_num - 1) * part_size + part_end = part_beg+part_size+1 + items[:] = items[part_beg:part_end] + # # Fixtures for creating a temp dir at session/module/class/function scope. # Unlike pytest's tmpdir fixture, they use tempfile.mkdtemp to create a @@ -128,4 +141,4 @@ def pytest_terminal_summary(self, terminalreporter, exitstatus): widths = [max(map(len, col)) for col in zip(*rows)] for row in rows: writer.write(" ".join((val.ljust(width) for val, width in zip(row, widths)))) - writer.line() \ No newline at end of file + writer.line() diff --git a/test/pipelines/snakemake/test_diamond.py b/test/pipelines/snakemake/test_diamond.py index 709abcc70..ba02f1381 100755 --- a/test/pipelines/snakemake/test_diamond.py +++ b/test/pipelines/snakemake/test_diamond.py @@ -6,7 +6,7 @@ import pytest import tools -from test.pipelines.snakemake import snake +snake = pytest.importorskip('test.pipelines.snake').snake from test.integration.test_diamond import * # for pytest fixtures @pytest.mark.skipif(tools.is_osx(), reason="not currently tested under OSX") @@ -32,4 +32,4 @@ def test_pipes(tmpdir_function, diamond_db, taxonomy_db, krona_db, input_bam): ) runner.run([krona_out]) assert os.path.getsize(os.path.join(runner.workdir, diamond_out)) > 0 - assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 \ No newline at end of file + assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 diff --git a/test/pipelines/snakemake/test_kraken.py b/test/pipelines/snakemake/test_kraken.py index ac2c446cd..69c2b754d 100755 --- a/test/pipelines/snakemake/test_kraken.py +++ b/test/pipelines/snakemake/test_kraken.py @@ -5,7 +5,7 @@ import pytest -from test.pipelines.snakemake import snake +snake = pytest.importorskip('test.pipelines.snake').snake from test.integration.test_kraken import * # for pytest fixtures @pytest.mark.skipif(sys.version_info < (3, 5), reason="Python version is too old for snakemake.") @@ -33,4 +33,4 @@ def test_pipes(tmpdir_function, kraken_db, krona_db, input_bam): # runner.run(['all_metagenomics']) runner.run([kraken_out, krona_out]) assert os.path.getsize(os.path.join(runner.workdir, kraken_out)) > 0 - assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 \ No newline at end of file + assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 diff --git a/test/pipelines/snakemake/test_metagenomics_align.py b/test/pipelines/snakemake/test_metagenomics_align.py index fc8fd0602..c1031762f 100755 --- a/test/pipelines/snakemake/test_metagenomics_align.py +++ b/test/pipelines/snakemake/test_metagenomics_align.py @@ -5,7 +5,7 @@ import pytest -from test.pipelines.snakemake import snake +snake = pytest.importorskip('test.pipelines.snake').snake from test.integration.test_metagenomics_align import * # for pytest fixtures @pytest.mark.skipif(sys.version_info < (3, 5), reason="Python version is too old for snakemake.") @@ -32,4 +32,4 @@ def test_pipes(tmpdir_function, bwa_db, taxonomy_db, input_bam): runner.run([report_out]) assert os.path.getsize(os.path.join(runner.workdir, report_out)) > 0 - assert os.path.getsize(os.path.join(runner.workdir, bam_out)) > 0 \ No newline at end of file + assert os.path.getsize(os.path.join(runner.workdir, bam_out)) > 0 diff --git a/travis/tests-pytest.sh b/travis/tests-pytest.sh new file mode 100755 index 000000000..fa5ec5f1a --- /dev/null +++ b/travis/tests-pytest.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eu -o pipefail + +pytest --cov-append $PYTEST_EXTRA_OPTS + +rc=$?; if [[ $rc != 0 ]]; then sleep 10; exit $rc; fi +# sleep to allow logs to be printed without truncation in the event of error From df37b037ec724af6eaee46f53a0d1028cb62cd01 Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Thu, 17 Jan 2019 20:53:23 -0500 Subject: [PATCH 02/13] fixed travis.yaml --- .travis.yml | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index e15351528..2bf5a7dcd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -158,7 +158,7 @@ jobs: sudo: required env: - TRAVIS_JOB=test_py27_part1 - - PYTEST_EXTRA_OPTS="--tests-part 1 4" + - PYTEST_EXTRA_OPTS="--tests-part 1 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -176,7 +176,7 @@ jobs: sudo: required env: - TRAVIS_JOB=test_py27_part2 - - PYTEST_EXTRA_OPTS="--tests-part 2 4" + - PYTEST_EXTRA_OPTS="--tests-part 2 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -194,7 +194,7 @@ jobs: sudo: required env: - TRAVIS_JOB=test_py27_part3 - - PYTEST_EXTRA_OPTS="--tests-part 3 4" + - PYTEST_EXTRA_OPTS="--tests-part 3 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -207,12 +207,12 @@ jobs: - conda clean --all --yes - language: python - python: 3.6 + python: 2.7 #stage: test sudo: required env: - - TRAVIS_JOB=test_py36_part1 - - PYTEST_EXTRA_OPTS="--tests-part 1 4" + - TRAVIS_JOB=test_py27_part3 + - PYTEST_EXTRA_OPTS="--tests-part 4 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -221,8 +221,6 @@ jobs: - source travis/activate-conda.sh script: - travis/tests-pytest.sh - after_success: - - coveralls before_cache: - conda clean --all --yes @@ -231,8 +229,8 @@ jobs: #stage: test sudo: required env: - - TRAVIS_JOB=test_py36_part2 - - PYTEST_EXTRA_OPTS="--tests-part 2 4" + - TRAVIS_JOB=test_py36_part1 + - PYTEST_EXTRA_OPTS="--tests-part 1 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -251,8 +249,8 @@ jobs: #stage: test sudo: required env: - - TRAVIS_JOB=test_py36_part3 - - PYTEST_EXTRA_OPTS="--tests-part 3 4" + - TRAVIS_JOB=test_py36_part2 + - PYTEST_EXTRA_OPTS="--tests-part 2 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -271,8 +269,8 @@ jobs: #stage: test sudo: required env: - - TRAVIS_JOB=test_py36_part4 - - PYTEST_EXTRA_OPTS="--tests-part 4 4" + - TRAVIS_JOB=test_py36_part3 + - PYTEST_EXTRA_OPTS="--tests-part 3 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -291,7 +289,8 @@ jobs: #stage: test sudo: required env: - - TRAVIS_JOB=test_py36 + - TRAVIS_JOB=test_py36_part4 + - PYTEST_EXTRA_OPTS="--tests-part 4 4" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -299,8 +298,7 @@ jobs: - travis/install-tools.sh - source travis/activate-conda.sh script: - - travis/tests-long.sh - - travis/tests-unit.sh + - travis/tests-pytest.sh after_success: - coveralls before_cache: From 4118ec7c2c9fa6816a6144bfc52211def2cfa6a9 Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Thu, 17 Jan 2019 21:33:55 -0500 Subject: [PATCH 03/13] now split into 8 parts --- .travis.yml | 173 ++++++++++++++++++++++++++++++++++++++++++++++++---- conftest.py | 8 ++- 2 files changed, 167 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2bf5a7dcd..17ea875f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: - PIP_DIR="$HOME/virtualenv" - GATK_PATH="$CACHE_DIR" - PYTHONIOENCODING=UTF8 - - PYTEST_ADDOPTS="-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov kmer_utils --cov reports --cov taxon_filter --cov tools --cov util --cov file_utils" + - PYTEST_ADDOPTS="-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov kmer_utils --cov reports --cov taxon_filter --cov tools --cov util --cov file_utils --parts-tot=8" - DOCKER_REGISTRY="quay.io" - DOCKER_REPO_PROD="quay.io/broadinstitute/viral-ngs" @@ -158,7 +158,7 @@ jobs: sudo: required env: - TRAVIS_JOB=test_py27_part1 - - PYTEST_EXTRA_OPTS="--tests-part 1 4" + - PYTEST_EXTRA_OPTS="--part-num 1" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -169,14 +169,13 @@ jobs: - travis/tests-pytest.sh before_cache: - conda clean --all --yes - - language: python python: 2.7 #stage: test sudo: required env: - TRAVIS_JOB=test_py27_part2 - - PYTEST_EXTRA_OPTS="--tests-part 2 4" + - PYTEST_EXTRA_OPTS="--part-num 2" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -194,7 +193,7 @@ jobs: sudo: required env: - TRAVIS_JOB=test_py27_part3 - - PYTEST_EXTRA_OPTS="--tests-part 3 4" + - PYTEST_EXTRA_OPTS="--part-num 3" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -211,8 +210,80 @@ jobs: #stage: test sudo: required env: - - TRAVIS_JOB=test_py27_part3 - - PYTEST_EXTRA_OPTS="--tests-part 4 4" + - TRAVIS_JOB=test_py27_part4 + - PYTEST_EXTRA_OPTS="--part-num 4" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + before_cache: + - conda clean --all --yes + + - language: python + python: 2.7 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py27_part5 + - PYTEST_EXTRA_OPTS="--part-num 5" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + before_cache: + - conda clean --all --yes + + - language: python + python: 2.7 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py27_part6 + - PYTEST_EXTRA_OPTS="--part-num 6" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + before_cache: + - conda clean --all --yes + + - language: python + python: 2.7 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py27_part7 + - PYTEST_EXTRA_OPTS="--part-num 7" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + before_cache: + - conda clean --all --yes + + - language: python + python: 2.7 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py27_part8 + - PYTEST_EXTRA_OPTS="--part-num 8" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -230,7 +301,7 @@ jobs: sudo: required env: - TRAVIS_JOB=test_py36_part1 - - PYTEST_EXTRA_OPTS="--tests-part 1 4" + - PYTEST_EXTRA_OPTS="--part-num 1" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -250,7 +321,7 @@ jobs: sudo: required env: - TRAVIS_JOB=test_py36_part2 - - PYTEST_EXTRA_OPTS="--tests-part 2 4" + - PYTEST_EXTRA_OPTS="--part-num 2" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -270,7 +341,7 @@ jobs: sudo: required env: - TRAVIS_JOB=test_py36_part3 - - PYTEST_EXTRA_OPTS="--tests-part 3 4" + - PYTEST_EXTRA_OPTS="--part-num 3" before_install: travis/before_install.sh install: - source travis/install-conda.sh @@ -290,7 +361,87 @@ jobs: sudo: required env: - TRAVIS_JOB=test_py36_part4 - - PYTEST_EXTRA_OPTS="--tests-part 4 4" + - PYTEST_EXTRA_OPTS="--part-num 4" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + after_success: + - coveralls + before_cache: + - conda clean --all --yes + + - language: python + python: 3.6 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py36_part5 + - PYTEST_EXTRA_OPTS="--part-num 5" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + after_success: + - coveralls + before_cache: + - conda clean --all --yes + + - language: python + python: 3.6 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py36_part6 + - PYTEST_EXTRA_OPTS="--part-num 6" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + after_success: + - coveralls + before_cache: + - conda clean --all --yes + + - language: python + python: 3.6 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py36_part7 + - PYTEST_EXTRA_OPTS="--part-num 7" + before_install: travis/before_install.sh + install: + - source travis/install-conda.sh + - travis/install-gatk.sh + - travis/install-tools.sh + - source travis/activate-conda.sh + script: + - travis/tests-pytest.sh + after_success: + - coveralls + before_cache: + - conda clean --all --yes + + - language: python + python: 3.6 + #stage: test + sudo: required + env: + - TRAVIS_JOB=test_py36_part8 + - PYTEST_EXTRA_OPTS="--part-num 8" before_install: travis/before_install.sh install: - source travis/install-conda.sh diff --git a/conftest.py b/conftest.py index b5a712e1f..6678c961c 100644 --- a/conftest.py +++ b/conftest.py @@ -35,7 +35,8 @@ def pytest_addoption(parser): default=False, help="run slow tests." ) - group.addoption('--tests-part', type=int, nargs=2, help="this is part k of N") + group.addoption('--parts-tot', type=int, help="total number of parts") + group.addoption('--part-num', type=int, help="number of this part") def pytest_configure(config): @@ -43,8 +44,9 @@ def pytest_configure(config): config.pluginmanager.register(reporter, 'fixturereporter') def pytest_collection_modifyitems(session, config, items): - part_num, tot_parts = config.getoption('--tests-part', default=(0,0)) - if tot_parts: + part_num = config.getoption('--part-num', default=None) + tot_parts = config.getoption('--parts-tot', default=None) + if part_num and tot_parts: util.misc.chk(1 <= part_num <= tot_parts) items.sort(key=operator.attrgetter('nodeid')) all_nodeids = [item.nodeid for item in items] From 60cb0957b84fd5a52d88affcbce0f5067b2fbd17 Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Sat, 19 Jan 2019 20:00:20 -0500 Subject: [PATCH 04/13] testing - kmer_utils - drop extraneous skipped-test messages --- test/unit/test_kmer_utils.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/test/unit/test_kmer_utils.py b/test/unit/test_kmer_utils.py index 0c0676903..c362e0199 100644 --- a/test/unit/test_kmer_utils.py +++ b/test/unit/test_kmer_utils.py @@ -246,6 +246,13 @@ def _stringify(arg): """Return a string based on `arg`, suitable for use as a pytest test id""" return util.file.string_to_file_name(str(arg)) +def _get_build_db_args(t_dir, seq_files, kmer_db_opts): + key = (seq_files, kmer_db_opts) + k_db = os.path.join(t_dir, 'bld_kmer_db_{}'.format(hash(key))) + seq_files = list(map(_inp, seq_files.split())) + return util.cmd.parse_cmd(module=kmer_utils, cmd='build_kmer_db', + args=seq_files + [k_db] + kmer_db_opts.split() + ['--memLimitGb', 4]) + def _do_build_kmer_db(t_dir, val_cache, seq_files, kmer_db_opts): """Build a database of kmers from given sequence file(s) using given options. @@ -302,9 +309,10 @@ def dict_module(): def kmer_db_fixture(request, tmpdir_module, dict_module): yield _do_build_kmer_db(tmpdir_module, dict_module, *request.param) -@pytest.fixture(scope='module', params=KMER_DBS_EMPTY+KMER_DBS_SMALL, ids=_stringify) -def kmer_db_fixture2(request, tmpdir_module, dict_module): - yield _do_build_kmer_db(tmpdir_module, dict_module, *request.param) +@pytest.fixture(scope='module', params=[p for p in itertools.product(KMER_DBS_EMPTY+KMER_DBS_SMALL, repeat=2) if _get_build_db_args('/tmp', *p[0]).kmer_size == _get_build_db_args('/tmp', *p[1]).kmer_size], ids=_stringify) +def kmer_db_pair_fixture(request, tmpdir_module, dict_module): + yield (_do_build_kmer_db(tmpdir_module, dict_module, *(request.param[0])), + _do_build_kmer_db(tmpdir_module, dict_module, *(request.param[1]))) @pytest.mark.parametrize("kmer_db_fixture", KMER_DBS_EMPTY+KMER_DBS_SMALL+KMER_DBS_MEDIUM, ids=_stringify, indirect=["kmer_db_fixture"]) @@ -418,18 +426,16 @@ def test_kmer_set_counts(kmer_db_fixture, tmpdir_function, set_to_val): @pytest.mark.parametrize("op", ('intersect', 'union', 'kmers_subtract', 'counters_subtract')) -def test_kmers_binary_op(kmer_db_fixture, kmer_db_fixture2, op, tmpdir_function): - if kmer_db_fixture.kmer_db_args.kmer_size != kmer_db_fixture2.kmer_db_args.kmer_size: - pytest.skip('(always skip) binary ops not defined on kmers of different size') +def test_kmers_binary_op(kmer_db_pair_fixture, op, tmpdir_function): db_result = os.path.join(tmpdir_function, 'op_result') - _log.debug('fixture1args=%s', kmer_db_fixture.kmer_db_args) - _log.debug('fixture2args=%s', kmer_db_fixture2.kmer_db_args) + _log.debug('fixture1args=%s', kmer_db_pair_fixture[0].kmer_db_args) + _log.debug('fixture2args=%s', kmer_db_pair_fixture[1].kmer_db_args) _log.debug('op=%s', op) args = util.cmd.run_cmd(module=kmer_utils, cmd='kmers_binary_op', - args=[op, kmer_db_fixture.kmer_db, kmer_db_fixture2.kmer_db, db_result]).args_parsed + args=[op, kmer_db_pair_fixture[0].kmer_db, kmer_db_pair_fixture[1].kmer_db, db_result]).args_parsed kmc_counts = tools.kmc.KmcTool().get_kmer_counts(db_result) - kmcpy_counts = kmcpy.binary_op(op, kmer_db_fixture.kmc_kmer_counts, kmer_db_fixture2.kmc_kmer_counts, + kmcpy_counts = kmcpy.binary_op(op, kmer_db_pair_fixture[0].kmc_kmer_counts, kmer_db_pair_fixture[1].kmc_kmer_counts, result_counter_cap=args.result_counter_cap) assert kmc_counts == kmcpy_counts From a1742da8e02392a8edaa2f65287c966118c8dcd6 Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Sat, 19 Jan 2019 21:55:31 -0500 Subject: [PATCH 05/13] added splitting of ci tests --- .travis.yml | 232 +------------------------------------------ travis/tests-long.sh | 6 -- travis/tests-unit.sh | 6 -- 3 files changed, 1 insertion(+), 243 deletions(-) delete mode 100755 travis/tests-long.sh delete mode 100755 travis/tests-unit.sh diff --git a/.travis.yml b/.travis.yml index 17ea875f6..f30737645 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: - PIP_DIR="$HOME/virtualenv" - GATK_PATH="$CACHE_DIR" - PYTHONIOENCODING=UTF8 - - PYTEST_ADDOPTS="-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov kmer_utils --cov reports --cov taxon_filter --cov tools --cov util --cov file_utils --parts-tot=8" + - PYTEST_ADDOPTS="-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov assembly --cov interhost --cov intrahost --cov metagenomics --cov ncbi --cov read_utils --cov kmer_utils --cov reports --cov taxon_filter --cov tools --cov util --cov file_utils --parts-tot=2" - DOCKER_REGISTRY="quay.io" - DOCKER_REPO_PROD="quay.io/broadinstitute/viral-ngs" @@ -186,115 +186,6 @@ jobs: - travis/tests-pytest.sh before_cache: - conda clean --all --yes - - - language: python - python: 2.7 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py27_part3 - - PYTEST_EXTRA_OPTS="--part-num 3" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - before_cache: - - conda clean --all --yes - - - language: python - python: 2.7 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py27_part4 - - PYTEST_EXTRA_OPTS="--part-num 4" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - before_cache: - - conda clean --all --yes - - - language: python - python: 2.7 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py27_part5 - - PYTEST_EXTRA_OPTS="--part-num 5" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - before_cache: - - conda clean --all --yes - - - language: python - python: 2.7 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py27_part6 - - PYTEST_EXTRA_OPTS="--part-num 6" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - before_cache: - - conda clean --all --yes - - - language: python - python: 2.7 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py27_part7 - - PYTEST_EXTRA_OPTS="--part-num 7" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - before_cache: - - conda clean --all --yes - - - language: python - python: 2.7 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py27_part8 - - PYTEST_EXTRA_OPTS="--part-num 8" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - before_cache: - - conda clean --all --yes - - language: python python: 3.6 #stage: test @@ -334,127 +225,6 @@ jobs: - coveralls before_cache: - conda clean --all --yes - - - language: python - python: 3.6 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py36_part3 - - PYTEST_EXTRA_OPTS="--part-num 3" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - after_success: - - coveralls - before_cache: - - conda clean --all --yes - - - language: python - python: 3.6 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py36_part4 - - PYTEST_EXTRA_OPTS="--part-num 4" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - after_success: - - coveralls - before_cache: - - conda clean --all --yes - - - language: python - python: 3.6 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py36_part5 - - PYTEST_EXTRA_OPTS="--part-num 5" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - after_success: - - coveralls - before_cache: - - conda clean --all --yes - - - language: python - python: 3.6 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py36_part6 - - PYTEST_EXTRA_OPTS="--part-num 6" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - after_success: - - coveralls - before_cache: - - conda clean --all --yes - - - language: python - python: 3.6 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py36_part7 - - PYTEST_EXTRA_OPTS="--part-num 7" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - after_success: - - coveralls - before_cache: - - conda clean --all --yes - - - language: python - python: 3.6 - #stage: test - sudo: required - env: - - TRAVIS_JOB=test_py36_part8 - - PYTEST_EXTRA_OPTS="--part-num 8" - before_install: travis/before_install.sh - install: - - source travis/install-conda.sh - - travis/install-gatk.sh - - travis/install-tools.sh - - source travis/activate-conda.sh - script: - - travis/tests-pytest.sh - after_success: - - coveralls - before_cache: - - conda clean --all --yes - #- language: python # stage: test # env: diff --git a/travis/tests-long.sh b/travis/tests-long.sh deleted file mode 100755 index d185badad..000000000 --- a/travis/tests-long.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -pytest --cov-append test/integration - -rc=$?; if [[ $rc != 0 ]]; then sleep 10; exit $rc; fi -# sleep to allow logs to be printed without truncation in the event of error diff --git a/travis/tests-unit.sh b/travis/tests-unit.sh deleted file mode 100755 index 6310ac49f..000000000 --- a/travis/tests-unit.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -pytest --cov-append test/unit - -rc=$?; if [[ $rc != 0 ]]; then sleep 10; exit $rc; fi -# sleep to allow logs to be printed without truncation in the event of error From 02acdec2065dd074ff865a775a1e0407356e8a3d Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Sat, 19 Jan 2019 22:32:13 -0500 Subject: [PATCH 06/13] drop slow tests, rather than skip --- conftest.py | 3 +++ test/__init__.py | 9 +-------- test/unit/test_kmer_utils.py | 8 ++------ 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/conftest.py b/conftest.py index 6678c961c..e3478fd26 100644 --- a/conftest.py +++ b/conftest.py @@ -44,6 +44,9 @@ def pytest_configure(config): config.pluginmanager.register(reporter, 'fixturereporter') def pytest_collection_modifyitems(session, config, items): + if not config.getoption('--runslow', default=False): + items[:] = [item for item in items if not item.get_closest_marker(name='slow')] + part_num = config.getoption('--part-num', default=None) tot_parts = config.getoption('--parts-tot', default=None) if part_num and tot_parts: diff --git a/test/__init__.py b/test/__init__.py index 9908c2907..1d72a1758 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -28,13 +28,6 @@ else: _CPUS = available_cpu_count() -def make_slow_test_marker(): - """Create a marker for marking slow tests.""" - return pytest.mark.skipif( - not pytest.config.getoption("--runslow", default=False), - reason="need --runslow option to run" - ) - def assert_equal_contents(testCase, filename1, filename2): 'Assert contents of two files are equal for a unittest.TestCase' testCase.assertTrue(filecmp.cmp(filename1, filename2, shallow=False)) @@ -124,4 +117,4 @@ def inputs(self, *fnames): def assert_none_executable(): testDir = os.path.dirname(__file__) assert all(not os.access(os.path.join(testDir, filename), os.X_OK) for filename in os.listdir(testDir) - if filename.endswith('.py')) \ No newline at end of file + if filename.endswith('.py')) diff --git a/test/unit/test_kmer_utils.py b/test/unit/test_kmer_utils.py index c362e0199..ae910bea7 100644 --- a/test/unit/test_kmer_utils.py +++ b/test/unit/test_kmer_utils.py @@ -24,12 +24,8 @@ import tools.kmc import tools.samtools -from test import make_slow_test_marker - _log = logging.getLogger(__name__) # pylint: disable=invalid-name -slow_test = make_slow_test_marker() # pylint: disable=invalid-name - class TestCommandHelp(unittest.TestCase): def test_help_parser_for_each_command(self): @@ -353,7 +349,7 @@ def _test_build_kmer_db(kmer_db_fixture): in itertools.product(SEQ_FILES, KMER_SIZES, STRAND_OPTS, KMER_OCCS_OPTS, NTHREADS)] -@slow_test +@pytest.mark.slow @pytest.mark.parametrize("kmer_db_fixture", COMBO_OPTS, ids=_stringify, indirect=["kmer_db_fixture"]) def test_build_kmer_db_combo(kmer_db_fixture): _test_build_kmer_db(kmer_db_fixture) @@ -405,7 +401,7 @@ def test_filter_with_empty_db(kmer_db_fixture, reads_file, filter_opts, tmpdir_f _test_filter_reads(**locals()) @pytest.mark.parametrize("kmer_db_fixture", [('ebola.fasta.gz', '-k 7')], ids=_stringify, indirect=["kmer_db_fixture"]) -@pytest.mark.parametrize("reads_file", [pytest.param('G5012.3.testreads.bam', marks=slow_test), +@pytest.mark.parametrize("reads_file", [pytest.param('G5012.3.testreads.bam', marks=pytest.mark.slow), 'G5012.3.subset.bam']) @pytest.mark.parametrize("filter_opts", ['--dbMinOccs 7 --readMinOccs 93', '--dbMinOccs 4 --readMinOccsFrac .6', From b7e801edf796e03e30c92481ed1d380a1bb2dd0f Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Sat, 19 Jan 2019 22:45:05 -0500 Subject: [PATCH 07/13] added printing of collection status --- conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index e3478fd26..0deef5a06 100644 --- a/conftest.py +++ b/conftest.py @@ -42,10 +42,13 @@ def pytest_addoption(parser): def pytest_configure(config): reporter = FixtureReporter(config) config.pluginmanager.register(reporter, 'fixturereporter') + config.post_collection_msgs = [] def pytest_collection_modifyitems(session, config, items): if not config.getoption('--runslow', default=False): - items[:] = [item for item in items if not item.get_closest_marker(name='slow')] + non_slow_items = [item for item in items if not item.get_closest_marker(name='slow')] + config.post_collection_msgs.append('skipped collection of {} slow tests'.format(len(items) - len(non_slow_items))) + items[:] = non_slow_items part_num = config.getoption('--part-num', default=None) tot_parts = config.getoption('--parts-tot', default=None) @@ -58,6 +61,9 @@ def pytest_collection_modifyitems(session, config, items): part_end = part_beg+part_size+1 items[:] = items[part_beg:part_end] +def pytest_report_collectionfinish(config, startdir, items): + return getattr(config, 'post_collection_msgs') + # # Fixtures for creating a temp dir at session/module/class/function scope. # Unlike pytest's tmpdir fixture, they use tempfile.mkdtemp to create a From 200c7ac23e5ab0cc33328f5eaff46ffd39a266f6 Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Sat, 19 Jan 2019 22:50:29 -0500 Subject: [PATCH 08/13] added msgs about collection of parts --- conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conftest.py b/conftest.py index 0deef5a06..27b708c9f 100644 --- a/conftest.py +++ b/conftest.py @@ -60,6 +60,10 @@ def pytest_collection_modifyitems(session, config, items): part_beg = (part_num - 1) * part_size part_end = part_beg+part_size+1 items[:] = items[part_beg:part_end] + config.post_collection_msgs.append('part {} of {}: collected items {}-{} of {}'.format(part_num, tot_parts, + part_beg+1, part_end, + len(all_nodeids))) + def pytest_report_collectionfinish(config, startdir, items): return getattr(config, 'post_collection_msgs') From 4e202071a4f5a7e9feb437bb6cd0ce2891ec7837 Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Sat, 19 Jan 2019 23:08:58 -0500 Subject: [PATCH 09/13] updated requirements-conda-tests --- requirements-conda-tests.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-conda-tests.txt b/requirements-conda-tests.txt index efa503e26..7ca1d2e80 100644 --- a/requirements-conda-tests.txt +++ b/requirements-conda-tests.txt @@ -1,10 +1,10 @@ -coveralls=1.3.0 +coveralls=1.5.1 flake8>=3.2.1 mock=2.0.0 pycodestyle=2.2.0 -pytest=3.7.1 +pytest=4.1.1 pytest-cov==2.5.1 pytest-mock=1.10.0 -pytest-xdist=1.22.5 -coverage=4.5.1 +pytest-xdist=1.26.0 +coverage=4.5.2 six=1.10.0 From 2d47ba64a892ff051e4691b6da92c7517b149a8b Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Sat, 19 Jan 2019 23:25:23 -0500 Subject: [PATCH 10/13] fix handling when keep_tmp(); on travis, look for tests under test/ --- test/unit/test_tools_samtools.py | 3 ++- test/unit/test_util_file.py | 5 +++-- travis/tests-pytest.sh | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/unit/test_tools_samtools.py b/test/unit/test_tools_samtools.py index 1540cd786..eb2b45e94 100644 --- a/test/unit/test_tools_samtools.py +++ b/test/unit/test_tools_samtools.py @@ -78,6 +78,7 @@ def test_bam2fa(self): for fa in (fa1, fa2): assert len(list(Bio.SeqIO.parse(fa, 'fasta')))==1 - assert not os.path.isfile(fa1) and not os.path.isfile(fa2) + assert util.file.keep_tmp() or (not os.path.isfile(fa1) and not os.path.isfile(fa2)) + diff --git a/test/unit/test_util_file.py b/test/unit/test_util_file.py index 35211d4c5..ef599319f 100644 --- a/test/unit/test_util_file.py +++ b/test/unit/test_util_file.py @@ -40,14 +40,15 @@ def testTempFiles(): tmp_fns.append(fn) - assert os.path.isfile(my_tmp_fn) and not os.path.isfile(my_tmp_fns[0]) and not os.path.isfile(my_tmp_fns[1]) + assert os.path.isfile(my_tmp_fn) + assert util.file.keep_tmp() or (not os.path.isfile(my_tmp_fns[0]) and not os.path.isfile(my_tmp_fns[1])) largeString = 'A' * (2*1024*1024) util.file.dump_file(fname=my_tmp_fn, value=largeString) with pytest.raises(RuntimeError): util.file.slurp_file(my_tmp_fn, maxSizeMb=1) - assert not os.path.isfile(my_tmp_fn) + assert util.file.keep_tmp() or not os.path.isfile(my_tmp_fn) def test_check_paths(tmpdir): '''Test the util.file.check_paths()''' diff --git a/travis/tests-pytest.sh b/travis/tests-pytest.sh index fa5ec5f1a..d217c0920 100755 --- a/travis/tests-pytest.sh +++ b/travis/tests-pytest.sh @@ -2,7 +2,7 @@ set -eu -o pipefail -pytest --cov-append $PYTEST_EXTRA_OPTS +pytest --cov-append $PYTEST_EXTRA_OPTS test rc=$?; if [[ $rc != 0 ]]; then sleep 10; exit $rc; fi # sleep to allow logs to be printed without truncation in the event of error From 128b0d5a8073c60ae0ce67486dddd6751db4dc30 Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Wed, 30 Jan 2019 04:43:03 -0500 Subject: [PATCH 11/13] restored handling of pipelines/snakemake --- test/pipelines/snakemake/test_diamond.py | 4 ++-- test/pipelines/snakemake/test_kraken.py | 4 ++-- test/pipelines/snakemake/test_metagenomics_align.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/pipelines/snakemake/test_diamond.py b/test/pipelines/snakemake/test_diamond.py index ba02f1381..709abcc70 100755 --- a/test/pipelines/snakemake/test_diamond.py +++ b/test/pipelines/snakemake/test_diamond.py @@ -6,7 +6,7 @@ import pytest import tools -snake = pytest.importorskip('test.pipelines.snake').snake +from test.pipelines.snakemake import snake from test.integration.test_diamond import * # for pytest fixtures @pytest.mark.skipif(tools.is_osx(), reason="not currently tested under OSX") @@ -32,4 +32,4 @@ def test_pipes(tmpdir_function, diamond_db, taxonomy_db, krona_db, input_bam): ) runner.run([krona_out]) assert os.path.getsize(os.path.join(runner.workdir, diamond_out)) > 0 - assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 + assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 \ No newline at end of file diff --git a/test/pipelines/snakemake/test_kraken.py b/test/pipelines/snakemake/test_kraken.py index 69c2b754d..ac2c446cd 100755 --- a/test/pipelines/snakemake/test_kraken.py +++ b/test/pipelines/snakemake/test_kraken.py @@ -5,7 +5,7 @@ import pytest -snake = pytest.importorskip('test.pipelines.snake').snake +from test.pipelines.snakemake import snake from test.integration.test_kraken import * # for pytest fixtures @pytest.mark.skipif(sys.version_info < (3, 5), reason="Python version is too old for snakemake.") @@ -33,4 +33,4 @@ def test_pipes(tmpdir_function, kraken_db, krona_db, input_bam): # runner.run(['all_metagenomics']) runner.run([kraken_out, krona_out]) assert os.path.getsize(os.path.join(runner.workdir, kraken_out)) > 0 - assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 + assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 \ No newline at end of file diff --git a/test/pipelines/snakemake/test_metagenomics_align.py b/test/pipelines/snakemake/test_metagenomics_align.py index c1031762f..fc8fd0602 100755 --- a/test/pipelines/snakemake/test_metagenomics_align.py +++ b/test/pipelines/snakemake/test_metagenomics_align.py @@ -5,7 +5,7 @@ import pytest -snake = pytest.importorskip('test.pipelines.snake').snake +from test.pipelines.snakemake import snake from test.integration.test_metagenomics_align import * # for pytest fixtures @pytest.mark.skipif(sys.version_info < (3, 5), reason="Python version is too old for snakemake.") @@ -32,4 +32,4 @@ def test_pipes(tmpdir_function, bwa_db, taxonomy_db, input_bam): runner.run([report_out]) assert os.path.getsize(os.path.join(runner.workdir, report_out)) > 0 - assert os.path.getsize(os.path.join(runner.workdir, bam_out)) > 0 + assert os.path.getsize(os.path.join(runner.workdir, bam_out)) > 0 \ No newline at end of file From d2ad5d42e30dd564b221fc6a6ffa8524a1eb10dc Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Wed, 30 Jan 2019 04:50:49 -0500 Subject: [PATCH 12/13] added __init__.py files --- test/pipelines/__init__.py | 0 test/pipelines/snakemake/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/pipelines/__init__.py create mode 100644 test/pipelines/snakemake/__init__.py diff --git a/test/pipelines/__init__.py b/test/pipelines/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/pipelines/snakemake/__init__.py b/test/pipelines/snakemake/__init__.py new file mode 100644 index 000000000..e69de29bb From f8ce593335ed8b1dda227efb779f054afb9e8bb3 Mon Sep 17 00:00:00 2001 From: Ilya Shlyakhter Date: Wed, 30 Jan 2019 05:12:18 -0500 Subject: [PATCH 13/13] removed import * --- test/pipelines/snakemake/test_diamond.py | 9 ++++++--- test/pipelines/snakemake/test_kraken.py | 8 +++++--- test/pipelines/snakemake/test_metagenomics_align.py | 7 ++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/test/pipelines/snakemake/test_diamond.py b/test/pipelines/snakemake/test_diamond.py index 709abcc70..0c155eb68 100755 --- a/test/pipelines/snakemake/test_diamond.py +++ b/test/pipelines/snakemake/test_diamond.py @@ -1,17 +1,20 @@ #!/usr/bin/env python -import os +import os, os.path import sys import pytest import tools from test.pipelines.snakemake import snake -from test.integration.test_diamond import * # for pytest fixtures +from test.integration.test_diamond import fastq_to_sam, sam_to_fastq, diamond, krona, db_type, input_bam, \ + input_fastqs, taxonomy_db, diamond_db, krona_db, test_diamond @pytest.mark.skipif(tools.is_osx(), reason="not currently tested under OSX") @pytest.mark.skipif(sys.version_info < (3, 5), reason="Python version is too old for snakemake.") def test_pipes(tmpdir_function, diamond_db, taxonomy_db, krona_db, input_bam): + join = os.path.join + runner = snake.SnakemakeRunner(workdir=tmpdir_function) override_config = { 'diamond_db': diamond_db, @@ -32,4 +35,4 @@ def test_pipes(tmpdir_function, diamond_db, taxonomy_db, krona_db, input_bam): ) runner.run([krona_out]) assert os.path.getsize(os.path.join(runner.workdir, diamond_out)) > 0 - assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 \ No newline at end of file + assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 diff --git a/test/pipelines/snakemake/test_kraken.py b/test/pipelines/snakemake/test_kraken.py index ac2c446cd..d5b1585f9 100755 --- a/test/pipelines/snakemake/test_kraken.py +++ b/test/pipelines/snakemake/test_kraken.py @@ -1,15 +1,17 @@ #!/usr/bin/env python -import os +import os, os.path + import sys import pytest from test.pipelines.snakemake import snake -from test.integration.test_kraken import * # for pytest fixtures +from test.integration.test_kraken import input_bam, kraken, krona, db_type, kraken_db, krona_db @pytest.mark.skipif(sys.version_info < (3, 5), reason="Python version is too old for snakemake.") def test_pipes(tmpdir_function, kraken_db, krona_db, input_bam): + join = os.path.join runner = snake.SnakemakeRunner(workdir=tmpdir_function) override_config = { 'kraken_db': kraken_db, @@ -33,4 +35,4 @@ def test_pipes(tmpdir_function, kraken_db, krona_db, input_bam): # runner.run(['all_metagenomics']) runner.run([kraken_out, krona_out]) assert os.path.getsize(os.path.join(runner.workdir, kraken_out)) > 0 - assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 \ No newline at end of file + assert os.path.getsize(os.path.join(runner.workdir, krona_out)) > 0 diff --git a/test/pipelines/snakemake/test_metagenomics_align.py b/test/pipelines/snakemake/test_metagenomics_align.py index fc8fd0602..7ccdd2ae3 100755 --- a/test/pipelines/snakemake/test_metagenomics_align.py +++ b/test/pipelines/snakemake/test_metagenomics_align.py @@ -1,15 +1,16 @@ #!/usr/bin/env python -import os +import os, os.path import sys import pytest from test.pipelines.snakemake import snake -from test.integration.test_metagenomics_align import * # for pytest fixtures +from test.integration.test_metagenomics_align import fastq_to_sam, taxonomy_db, input_bam, bwa, db_type, bwa_db @pytest.mark.skipif(sys.version_info < (3, 5), reason="Python version is too old for snakemake.") def test_pipes(tmpdir_function, bwa_db, taxonomy_db, input_bam): + join = os.path.join runner = snake.SnakemakeRunner(workdir=tmpdir_function) override_config = { 'align_rna_db': bwa_db, @@ -32,4 +33,4 @@ def test_pipes(tmpdir_function, bwa_db, taxonomy_db, input_bam): runner.run([report_out]) assert os.path.getsize(os.path.join(runner.workdir, report_out)) > 0 - assert os.path.getsize(os.path.join(runner.workdir, bam_out)) > 0 \ No newline at end of file + assert os.path.getsize(os.path.join(runner.workdir, bam_out)) > 0