bump python version 3.8 -> 3.10 (#98)

* python bumped to 3.12 for Docker image, docs build, and GitHub Actions env

* change lz4 pin to allow newer versions

* let's try not restricting pytest to <=8

* try more flexible pinning for a few more dependencies

* verbose mamba install

* remove defaults conda channel

remove defaults conda channel as it is now noted to be incompatible with conda-forge; see: https://mamba.readthedocs.io/en/latest/user_guide/troubleshooting.html#mixing-the-defaults-and-conda-forge-channels

* set CONDA_ENVS_PATH env var (dir containing conda envs subdirs)

https://conda.io/projects/conda/en/latest/user-guide/configuration/settings.html#envs-dirs-specify-environment-directories

* set number of threads for conda to fetch repodata

set number of threads for conda to fetch repodata , via repodata_threads; see: https://conda.io/projects/conda/en/latest/user-guide/configuration/settings.html#configuring-number-of-threads

* bump pigz

* print dependency trees for packages specified (individually)

* set python to 3.10

set python to 3.10, as this is the most recent version in the bioconda build matrix; see: https://github.com/bioconda/bioconda-utils/blob/master/bioconda_utils/bioconda_utils-conda_build_config.yaml#L40

* add $CONDA_CHANNEL_STRING to mamba repodata depends call

* unpin lz4-c

* avoid syntax warning using "is" with a string literal

* add DEBUG variable-based toggle of verbose output during conda install script

* quote python version

* actions/setup-python@v4 -> v5

* more flexible pin of matplotlib for building docs

* viral-baseimage to 0.2.2

Author: tomkinsc

.github/workflows/build.yml

@@ -139,7 +139,7 @@ jobs:
     needs: build_docker
     runs-on: ubuntu-20.04
     env:
-      GITHUB_ACTIONS_PYTHON_VERSION: 3.8
+      GITHUB_ACTIONS_PYTHON_VERSION: "3.10"
       PYTEST_ADDOPTS: "-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov read_utils --cov reports --cov tools --cov util --cov file_utils"
     steps:
       - name: checkout repository
@@ -172,7 +172,7 @@ jobs:
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
       - name: install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "${{ env.GITHUB_ACTIONS_PYTHON_VERSION }}"
       - name: Set up Docker Buildx

.readthedocs.yml

@@ -19,6 +19,6 @@ formats: all
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.8
+  version: 3.10
   install:
     - requirements: docs/requirements.txt

Dockerfile

@@ -16,7 +16,8 @@ ENV \
 	INSTALL_PATH="/opt/viral-ngs" \
 	VIRAL_NGS_PATH="/opt/viral-ngs/source" \
 	MINICONDA_PATH="/opt/miniconda" \
-	CONDA_DEFAULT_ENV=viral-ngs-env
+	CONDA_DEFAULT_ENV=viral-ngs-env \
+	CONDA_ENVS_PATH="$MINICONDA_PATH/envs"
 ENV \
 	PATH="$VIRAL_NGS_PATH:$MINICONDA_PATH/envs/$CONDA_DEFAULT_ENV/bin:$MINICONDA_PATH/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
 	CONDA_PREFIX=$MINICONDA_PATH/envs/$CONDA_DEFAULT_ENV \
@@ -27,7 +28,7 @@ ENV \
 # Set it up so that this slow & heavy build layer is cached
 # unless the requirements* files or the install scripts actually change
 WORKDIR $INSTALL_PATH
-RUN conda create -n $CONDA_DEFAULT_ENV python=3.7
+RUN conda create -n $CONDA_DEFAULT_ENV python=3.10
 RUN echo "source activate $CONDA_DEFAULT_ENV" > ~/.bashrc
 RUN hash -r
 COPY docker $VIRAL_NGS_PATH/docker/

docker/install-conda-dependencies.sh

@@ -9,6 +9,7 @@
 
 set -e -o pipefail
 
+#DEBUG=1 # set DEBUG=1 for more verbose output
 CONDA_INSTALL_TIMEOUT="90m"
 
 echo "PATH:              ${PATH}"
@@ -17,7 +18,10 @@ echo "CONDA_PREFIX:      ${CONDA_PREFIX}"
 echo "VIRAL_NGS_PATH:    ${VIRAL_NGS_PATH}"
 echo "MINICONDA_PATH:    ${MINICONDA_PATH}"
 echo "CONDA_DEFAULT_ENV: ${CONDA_DEFAULT_ENV}"
-CONDA_CHANNEL_STRING="--override-channels -c broad-viral -c conda-forge -c bioconda -c defaults"
+CONDA_CHANNEL_STRING="--override-channels -c broad-viral -c conda-forge -c bioconda"
+
+# ToDo: if confirmed working, move to conda config section of viral-baseimage
+conda config --set repodata_threads $(nproc)
 
 # solving the dependency graph for a conda environment can take a while.
 # so long, in fact, that the conda process can run for >10 minutes without
@@ -43,7 +47,7 @@ function start_keepalive {
 
     >&2 echo "Running..."
     # Start a process that runs as a keep-alive
-    # to avoid travis quitting if there is no output
+    # to avoid having the CI running quit if there is no output
     (while true; do
         sleep 120
         >&2 echo "Still running..."
@@ -70,13 +74,18 @@ sync
 REQUIREMENTS=""
 for condafile in $*; do
 	REQUIREMENTS="$REQUIREMENTS --file $condafile"
+
+    # print dependency tree for all packages in file
+    [[ $DEBUG = 1 ]] && grep -vE '^#' "${condafile}" | xargs -I {} mamba repoquery depends $CONDA_CHANNEL_STRING --quiet --pretty --recursive --tree "{}";
 done
 
 # run conda install with keepalive subshell process running in background
 # to keep travis build going. Enforce a hard timeout via timeout GNU coreutil
 start_keepalive
-#timeout $CONDA_INSTALL_TIMEOUT conda install -y -q $CONDA_CHANNEL_STRING -p "${CONDA_PREFIX}" $REQUIREMENTS
-mamba install -y -q $CONDA_CHANNEL_STRING -p "${CONDA_PREFIX}" $REQUIREMENTS
+if [[ $DEBUG = 1 ]]; then 
+    MAMBA_DEBUG_LEVEL="-vvv"
+fi
+mamba install -y $MAMBA_DEBUG_LEVEL -q $CONDA_CHANNEL_STRING -p "${CONDA_PREFIX}" $REQUIREMENTS
 stop_keepalive
 
 # clean up

docs/requirements.txt

@@ -2,7 +2,7 @@ jinja2==3.1.2 # https://github.com/readthedocs/readthedocs.org/issues/9037#issue
 Sphinx==5.3.0 #override sphinx pinning done by RTD: https://docs.readthedocs.io/en/stable/build-default-versions.html#external-dependencies
 sphinx-argparse
 sphinx-rtd-theme==1.1.1
-matplotlib==2.2.4
+matplotlib>=2.2.4
 PyYAML==6.0
 mock==5.0.1
 recommonmark

requirements-conda-tests.txt

@@ -1,7 +1,7 @@
 coveralls>=1.3.0
 lxml>=4.3.3
 mock>=2.0.0
-pytest>=7.4.3,<=8
+pytest>=7.4.3
 pytest-cov>=2.6.1
 pytest-mock>=1.10.0
 pytest-xdist>=1.26.1

requirements-conda.txt

@@ -1,31 +1,31 @@
 bbmap>=38.56
 bcftools>=1.10
 bedtools>=2.29.2
-bwa=0.7.17
+bwa>=0.7.17
 csvkit>=1.0.4
 cd-hit=4.6.8
 cd-hit-auxtools=4.6.8
 fastqc>=0.11.7
 gatk=3.8
 jq>=1.6
-lbzip2=2.5
-lz4-c=1.8.3
+lbzip2>=2.5
+lz4-c>=1.8.3
 minimap2>=2.17
 mvicuna=1.0
 novoalign=3.09.04
 parallel>=20190922
 picard=2.25.6
-pigz=2.4
-prinseq=0.20.4
+pigz>=2.4
+prinseq>=0.20.4
 samtools>=1.16.1
-trimmomatic=0.38
+trimmomatic>=0.38
 unzip>=6.0
 zstd>=1.3.8
 # Python packages below
 arrow>=0.12.1
 biopython>=1.72
 firecloud>=0.16.35
-lz4=2.2.1
+lz4>=2.2.1
 jinja2>=2.11.3
 matplotlib>=2.2.4
 pysam>=0.20.0

util/file.py

@@ -208,7 +208,7 @@ def extract_tarball(tarfile, out_dir=None, threads=None, compression='auto', pip
     else:
         util.file.mkdir_p(out_dir)
     assert compression in ('gz', 'bz2', 'lz4', 'zip', 'zst', 'none', 'auto')
-    if compression is 'auto':
+    if compression == 'auto':
         assert tarfile != '-' or pipe_hint, "cannot autodetect on stdin input unless pipe_hint provided"
         # auto-detect compression type based on file name
         if tarfile=='-':