From 767c1afdc0f43f086a3213eccc96edaa5346838e Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Thu, 3 Apr 2025 09:52:40 +0100 Subject: [PATCH 1/7] Remove conda env file and conda recipe files --- .github/workflows/httomo_version_tag.yml | 53 ---------------- .scripts/conda_upload.sh | 19 ------ .scripts/decrypt_secret.sh | 8 --- .scripts/my_secret.json.gpg | 1 - conda/environment-cpu.yml | 20 ------ conda/environment.yml | 28 --------- conda/recipe/build.sh | 23 ------- conda/recipe/conda_build_config.yaml | 7 --- conda/recipe/meta.yaml | 79 ------------------------ 9 files changed, 238 deletions(-) delete mode 100644 .github/workflows/httomo_version_tag.yml delete mode 100755 .scripts/conda_upload.sh delete mode 100755 .scripts/decrypt_secret.sh delete mode 100644 .scripts/my_secret.json.gpg delete mode 100644 conda/environment-cpu.yml delete mode 100644 conda/environment.yml delete mode 100755 conda/recipe/build.sh delete mode 100755 conda/recipe/conda_build_config.yaml delete mode 100644 conda/recipe/meta.yaml diff --git a/.github/workflows/httomo_version_tag.yml b/.github/workflows/httomo_version_tag.yml deleted file mode 100644 index 79b9d130e..000000000 --- a/.github/workflows/httomo_version_tag.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: version tag pushed - -# Run the workflow whenever a tag beginning with `v` is pushed to any branch -on: - push: - tags: - - v* - -jobs: - conda-build-upload: - runs-on: ubuntu-latest - - defaults: - run: - shell: bash -l {0} - - steps: - - name: Checkout repository code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.ref_name }} - - # setup Python 3.11 - - name: Setup Python 3.11 - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - # setup conda - - name: Setup conda - uses: mamba-org/setup-micromamba@v2 - with: - post-cleanup: 'all' - init-shell: bash - - # install dependencies - - name: Install dependencies - run: | - micromamba install -c conda-forge conda-build anaconda-client - micromamba list - - - name: Decrypt a secret - run: ./.scripts/decrypt_secret.sh - env: - LARGE_SECRET_PASSPHRASE: ${{ secrets.LARGE_SECRET_PASSPHRASE }} - - - name: Build and upload the package to httomo conda cloud - env: - LABEL: main - run: | - chmod +x ./.scripts/conda_upload.sh - ./.scripts/conda_upload.sh diff --git a/.scripts/conda_upload.sh b/.scripts/conda_upload.sh deleted file mode 100755 index f467a5f35..000000000 --- a/.scripts/conda_upload.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -PKG_NAME=httomo -USER=httomo-team -OS=linux-64 -CONDA_TOKEN=$(cat $HOME/.secrets/my_secret.json) - -mkdir ~/conda-bld -conda config --set anaconda_upload no -export CONDA_BLD_PATH=~/conda-bld - -$CONDA/bin/conda build . -c conda-forge -c https://conda.anaconda.org/httomo/ --no-test - -# upload packages to conda -find $CONDA_BLD_PATH/$OS -name *.tar.bz2 | while read file -do - echo $file - $CONDA/bin/anaconda -v --show-traceback --token $CONDA_TOKEN upload --label $LABEL $file --force -done diff --git a/.scripts/decrypt_secret.sh b/.scripts/decrypt_secret.sh deleted file mode 100755 index db5192c1e..000000000 --- a/.scripts/decrypt_secret.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# Decrypt the file -mkdir $HOME/.secrets -# --batch to prevent interactive command -# --yes to assume "yes" for questions -gpg --quiet --batch --yes --decrypt --passphrase="$LARGE_SECRET_PASSPHRASE" \ ---output $HOME/.secrets/my_secret.json ./.scripts/my_secret.json.gpg diff --git a/.scripts/my_secret.json.gpg b/.scripts/my_secret.json.gpg deleted file mode 100644 index 0101187df..000000000 --- a/.scripts/my_secret.json.gpg +++ /dev/null @@ -1 +0,0 @@ -  -:"k]凞$nʷH1\?۝Q5|hcmGߐLP($ɦ;L5e&XGl3XJ :y~{b \ No newline at end of file diff --git a/conda/environment-cpu.yml b/conda/environment-cpu.yml deleted file mode 100644 index e9a026e0d..000000000 --- a/conda/environment-cpu.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: httomo -channels: - - conda-forge - - httomo -dependencies: - - conda-forge::tomopy==1.15 - - httomo::httomolib==2.1 - - conda-forge::click>=8.1.3 - - conda-forge::mpi4py>=3.1 - - conda-forge::h5py=*=*mpi_openmpi* - - conda-forge::pyyaml - - conda-forge::numpy<=1.25 - - conda-forge::python - - conda-forge::numexpr>=2.8.4 - - conda-forge::ipython - - conda-forge::loguru - - conda-forge::graypy - - conda-forge::plumbum - - conda-forge::tqdm - - conda-forge::typing_extensions diff --git a/conda/environment.yml b/conda/environment.yml deleted file mode 100644 index 52c5f7ddf..000000000 --- a/conda/environment.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: httomo -channels: - - conda-forge - - httomo -dependencies: - - astra-toolbox - - click - - numpy - - cupy=12.3.0 - - graypy - - h5py=*=*mpi_openmpi* - - hdf5plugin - - loguru - - mpi4py - - pyyaml - - nvtx - - python - - plumbum - - tqdm - - typing_extensions - - ipython - - pytest - - pytest-xdist - - pytest-cov - - pytest-mock - - tomopy=1.15 - - ccpi-regulariser - diff --git a/conda/recipe/build.sh b/conda/recipe/build.sh deleted file mode 100755 index 76f9d82ac..000000000 --- a/conda/recipe/build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -python -m pip install . - -# Define var for path to httomo installation dir -# -# `SP_DIR` is the path to the `site-packages/` dir in the conda env, see -# https://docs.conda.io/projects/conda-build/en/latest/user-guide/environment-variables.html#environment-variables-set-during-the-build-process -HTTOMO_TARGET="$SP_DIR/$PKG_NAME" - -# Create dirs for YAML templates -mkdir $HTTOMO_TARGET/yaml_templates/ -mkdir $HTTOMO_TARGET/yaml_templates/httomo -mkdir $HTTOMO_TARGET/yaml_templates/httomolib -mkdir $HTTOMO_TARGET/yaml_templates/httomolibgpu -mkdir $HTTOMO_TARGET/yaml_templates/tomopy - -# Generate YAML templates -python scripts/yaml_templates_generator.py -i httomo/methods_database/packages/httomo_modules.yaml -o $HTTOMO_TARGET/yaml_templates/httomo/ -python scripts/yaml_templates_generator.py -i httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o $HTTOMO_TARGET/yaml_templates/httomolib/ -python scripts/yaml_templates_generator.py -i httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o $HTTOMO_TARGET/yaml_templates/httomolibgpu/ -python scripts/yaml_templates_generator.py -i httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o $HTTOMO_TARGET/yaml_templates/tomopy/ -python scripts/yaml_unsupported_tomopy_remove.py -t $HTTOMO_TARGET/yaml_templates/tomopy/ -l httomo/methods_database/packages/external/tomopy/tomopy.yaml diff --git a/conda/recipe/conda_build_config.yaml b/conda/recipe/conda_build_config.yaml deleted file mode 100755 index 37c414dd4..000000000 --- a/conda/recipe/conda_build_config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -python: - - 3.10 -numpy: - - 1.25 -openmpi_build: - - regular - - external diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml deleted file mode 100644 index 1f9184db0..000000000 --- a/conda/recipe/meta.yaml +++ /dev/null @@ -1,79 +0,0 @@ -{% set pyproject = load_file_data("../../pyproject.toml", from_recipe_dir=True) %} -{% set proj = pyproject.get("project") %} -{% set name = proj.get("name") %} - -package: - name: {{ name|lower }} - version: {{ GIT_DESCRIBE_TAG|trim("v") }} - -source: - path: ../../ - -build: - preserve_egg_dir: False - string: py{{ py }}_openmpi_{{ openmpi_build }} - -test: - files: - - samples/* - - yaml_templates/* - requires: - - pip - - pytest - imports: - - httomo - - httomo.data - source_files: - - tests/* - commands: - - pip check - - pytest -requirements: - build: - - h5py=*=*mpi_openmpi* - - httomolib==2.1.2 - - httomolibgpu==2.1.2 - - tomopy==1.15 - - loguru - - hdf5plugin # added because of the yaml templates dependency script - - mpi4py - - numpy<=1.25 - - pip - - python - - pyyaml - - setuptools - - setuptools-git-versioning - - typing_extensions - run: - - astra-toolbox - - click>=8.1.3 - - cupy=12.3.0 - - cudatoolkit - - graypy - - h5py=*=*mpi_openmpi* - - hdf5plugin - - httomolib==2.1.2 - - httomolibgpu==2.1.2 - - tomopy==1.15 - - ipython - - loguru - - mpi4py - - pyyaml - - numpy<=1.25 - - nvtx - {% if openmpi_build == 'regular' %} - - openmpi - {% else %} - - openmpi * *external* - {% endif %} - - python - - plumbum - - tomobar - - tqdm - - typing_extensions - -about: - home: https://github.com/DiamondLightSource/httomo - license: BSD 3-clause - license_file: LICENSE - summary: 'High Throughput Tomography framework' From e3c22c98d678f1928482bb3e577dd2daed1eba53 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Thu, 3 Apr 2025 10:36:26 +0100 Subject: [PATCH 2/7] Update relevant CI to get minimal dependencies from conda channels --- .github/workflows/run_tests_framework_iris.yml | 11 ++++++----- .github/workflows/run_tests_pipelines_iris.yml | 11 ++++++----- pyproject.toml | 8 +++++--- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/run_tests_framework_iris.yml b/.github/workflows/run_tests_framework_iris.yml index 0501d6e65..44ab5cd21 100644 --- a/.github/workflows/run_tests_framework_iris.yml +++ b/.github/workflows/run_tests_framework_iris.yml @@ -27,18 +27,19 @@ jobs: - name: Create conda environment uses: mamba-org/setup-micromamba@v1 with: - environment-file: conda/environment.yml environment-name: httomo + create-args: >- + cupy==12.3.0 + h5py=*=*mpi_openmpi* + mpi4py + tomopy==1.15 post-cleanup: 'all' init-shell: bash - name: Install httomo libraries and httomo run: | micromamba activate httomo - pip install --upgrade --force-reinstall pillow - pip install httomolibgpu tomobar - pip install --no-deps httomo-backends - pip install . + pip install .[dev-gpu] micromamba list - name: Generate full yaml pipelines using pipeline directives diff --git a/.github/workflows/run_tests_pipelines_iris.yml b/.github/workflows/run_tests_pipelines_iris.yml index d72e0ca21..871ff2571 100644 --- a/.github/workflows/run_tests_pipelines_iris.yml +++ b/.github/workflows/run_tests_pipelines_iris.yml @@ -27,18 +27,19 @@ jobs: - name: Create conda environment uses: mamba-org/setup-micromamba@v1 with: - environment-file: conda/environment.yml environment-name: httomo + create-args: >- + cupy==12.3.0 + h5py=*=*mpi_openmpi* + mpi4py + tomopy==1.15 post-cleanup: 'all' init-shell: bash - name: Install httomo libraries, httomo-backends and httomo run: | micromamba activate httomo - pip install --upgrade --force-reinstall pillow - pip install httomolibgpu tomobar - pip install --no-deps httomo-backends - pip install . + pip install .[dev-gpu] micromamba list - name: Generate full yaml pipelines using pipeline directives diff --git a/pyproject.toml b/pyproject.toml index 8006c5d71..0570c1f4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,9 +39,11 @@ dynamic = ["version"] dependencies = [ "click", "numpy", - "httomolib", + "hdf5plugin", + "httomo-backends", "loguru", "typing-extensions", + "pyyaml", "tqdm", "graypy" ] @@ -83,12 +85,12 @@ dev-gpu = [ "toml", "httomolib[dev]", "httomolibgpu[dev]", - "tomopy", "plumbum", "typing_extensions", "mypy", "types-PyYAML", - "pre-commit" + "pre-commit", + "ccpi-regularisation-cupy", ] [tool.mypy] From 70e88bccfbe11c47e1c58f0dfd5562deb9512c99 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Thu, 3 Apr 2025 12:08:16 +0100 Subject: [PATCH 3/7] Remove unnecessary typing extensions dependency --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0570c1f4a..557ceb6ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,6 @@ dependencies = [ "hdf5plugin", "httomo-backends", "loguru", - "typing-extensions", "pyyaml", "tqdm", "graypy" @@ -69,7 +68,6 @@ dev-cpu = [ "httomolib[dev]", "tomopy", "plumbum", - "typing_extensions", "mypy", "types-PyYAML", "pre-commit" @@ -86,7 +84,6 @@ dev-gpu = [ "httomolib[dev]", "httomolibgpu[dev]", "plumbum", - "typing_extensions", "mypy", "types-PyYAML", "pre-commit", From ae60b1862de2d00cecfc0cc0fa72f4ea717df2dd Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Thu, 3 Apr 2025 12:11:30 +0100 Subject: [PATCH 4/7] Remove `tomopy` dependency from `pyproject.toml` `tomopy` isn't distributed on PyPI, so putting it in any list of optional dependencies in the `pyproject.toml` file and attempting to install those optional dependencies will fail. --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 557ceb6ed..ed1c99245 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ httomo = "httomo.cli:main" repository = "https://github.com/DiamondLightSource/HTTomo" [project.optional-dependencies] -cpu = ["tomopy" ] +cpu = ["httomolib" ] gpu = ["httomolibgpu"] dev-cpu = [ "pytest", @@ -66,7 +66,6 @@ dev-cpu = [ "pydocstyle", "toml", "httomolib[dev]", - "tomopy", "plumbum", "mypy", "types-PyYAML", From 60a680b9fe308f3c6401985cc278c0c0e078e56a Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Thu, 3 Apr 2025 12:13:32 +0100 Subject: [PATCH 5/7] Remove unnecessary `toml` dependency --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ed1c99245..7bc81a3d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,6 @@ dev-cpu = [ "flake8", "pyproject-flake8", "pydocstyle", - "toml", "httomolib[dev]", "plumbum", "mypy", @@ -79,7 +78,6 @@ dev-gpu = [ "flake8", "pyproject-flake8", "pydocstyle", - "toml", "httomolib[dev]", "httomolibgpu[dev]", "plumbum", From 818853e8567f2bb29d65f14bf116806853ca8384 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Mon, 14 Apr 2025 16:34:26 +0100 Subject: [PATCH 6/7] Add missing dependencies to `pyproject.toml` --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 7bc81a3d3..0b7dadcee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,12 @@ requires-python = ">=3.10" dynamic = ["version"] dependencies = [ "click", + "cupy", "numpy", "hdf5plugin", "httomo-backends", "loguru", + "mpi4py", "pyyaml", "tqdm", "graypy" From aef55c64eb09887e616635ba42276e21ce9b30e4 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Mon, 14 Apr 2025 16:36:06 +0100 Subject: [PATCH 7/7] Update installation documentation --- docs/source/howto/installation.rst | 53 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/docs/source/howto/installation.rst b/docs/source/howto/installation.rst index d9fe63b91..06ed39560 100644 --- a/docs/source/howto/installation.rst +++ b/docs/source/howto/installation.rst @@ -1,46 +1,47 @@ Installation Guide ****************** +HTTomo is available on PyPI, so it can be installed into either a virtual environment or a +conda environment. -Install HTTomo as a PyPi package -========================================================= -.. code-block:: console +However, there are certain constraints under which a virtual environment can be used, due to +the dependence on an MPI implementation, the hdf5 library, CUDA libraries, and whether the user +requires using :code:`tomopy` methods in pipelines. - $ pip install httomo # this will install the CPU-only version - $ pip install httomo httomolibgpu # this will install the GPU backend +Virtual environment +=================== -Install HTTomo as a pre-built conda Python package -================================================== +A virtual environment can be used if the following conditions are met: -This installation is preferable as it should take care all of dependencies including :ref:`backends_list` by getting them from the dedicated anaconda channel. +- an MPI implementation is installed on the system (ie, OpenMPI) +- the hdf5 library is installed on the system +- CUDA libraries or CUDA toolkit are installed on the system +- methods from :code:`tomopy` are not required to be used in pipelines .. code-block:: console - $ conda env create --name httomo # create a fresh conda environment - $ conda install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge - -Install as a Python module -=========================== + $ python -m venv httomo + $ source httomo/bin/activate + $ MPICC=$(type -p mpicc) pip install mpi4py==3.1.6 + $ pip install cython numpy pkgconfig setuptools # build dependencies of h5py + $ CC=$(type -p mpicc) HDF5_MPI="ON" HDF5_DIR=/path/to/parallel-hdf5 pip install --no-build-isolation --no-binary=h5py h5py + $ pip install cupy-cuda12x # install cupy-cuda11x if CUDA library/CUDA toolkit version is 11.x + $ pip install aiofiles astra-toolbox ccpi-regularisation-cupy click graypy hdf5plugin loguru nvtx pillow pyyaml scikit-image scipy tomobar tqdm + $ pip install --no-deps httomo httomolib httomolibgpu httomo-backends -If installation above for some reason is not working for you, then the best way to install HTTomo is to create conda environment first and then -`pip install` HTTomo into it. You will need to `git clone` HTTomo repository to your disk first. Use `environment.yml` file to install -the GPU-supported HTTomo. For CPU-only version, please use `environment-cpu.yml` instead. +Conda environment +================= .. code-block:: console - $ git clone git@github.com:DiamondLightSource/HTTomo.git # clone the repo - $ conda env create --name httomo --file conda/environment.yml # install dependencies for GPU version - $ conda activate httomo # activate environment - $ pip install . # Install the module + $ conda env create --name httomo + $ conda activate httomo + $ conda install -c conda-forge cupy==12.3.0 openmpi==4.1.6 h5py[build=*openmpi*] + $ conda install -c conda-forge tomopy==1.15 # optional + $ pip install httomo httomolib httomolibgpu Setup HTTomo development environment: ====================================================== .. code-block:: console $ pip install -e .[dev] # development mode - -Build HTTomo as a conda Python package -====================================================== -.. code-block:: console - - $ conda build conda/recipe/ -c conda-forge -c https://conda.anaconda.org/httomo/ --no-test