From 86af851f4415ed383852e50a1b4ec9b25f8be442 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Thu, 22 Aug 2024 17:13:56 +0100 Subject: [PATCH 01/12] bumps the library versions to 2.1 --- conda/environment-cpu.yml | 2 +- conda/environment.yml | 2 +- conda/recipe/meta.yaml | 10 +++++----- docs/source/backends/templates.rst | 2 +- docs/source/doc-conda-requirements.yml | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/conda/environment-cpu.yml b/conda/environment-cpu.yml index f7aac6987..903f67fdd 100644 --- a/conda/environment-cpu.yml +++ b/conda/environment-cpu.yml @@ -5,7 +5,7 @@ channels: - httomo dependencies: - conda-forge::tomopy==1.15 - - httomo::httomolib==2.0 + - httomo::httomolib==2.1 - conda-forge::click>=8.1.3 - conda-forge::mpi4py>=3.1 - conda-forge::h5py=*=*mpi_openmpi* diff --git a/conda/environment.yml b/conda/environment.yml index 44beb53b4..4e4867d33 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -24,7 +24,7 @@ dependencies: - conda-forge::typing_extensions - anaconda::ipython - conda-forge::tomopy==1.15 - - httomo::httomolib==2.0 + - httomo::httomolib==2.1 - httomo::httomolibgpu==2.1 - httomo::tomobar - rapidsai::cucim diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index 068ce3713..49f33d5b6 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -31,8 +31,9 @@ test: requirements: build: - h5py=*=*mpi_openmpi* - - httomolib==2.0 - - httomolibgpu==2.0 + - httomolib==2.1 + - httomolibgpu==2.1 + - tomopy==1.15 - loguru - mpi4py - numpy<=1.25 @@ -41,7 +42,6 @@ requirements: - pyyaml - setuptools - setuptools-git-versioning - - tomopy==1.15 - typing_extensions run: - astra-toolbox @@ -52,8 +52,8 @@ requirements: - graypy - h5py=*=*mpi_openmpi* - hdf5plugin - - httomolib==2.0 - - httomolibgpu==2.0 + - httomolib==2.1 + - httomolibgpu==2.1 - ipython - loguru - mpi4py diff --git a/docs/source/backends/templates.rst b/docs/source/backends/templates.rst index 428c2d531..6d5e79bfe 100644 --- a/docs/source/backends/templates.rst +++ b/docs/source/backends/templates.rst @@ -36,7 +36,7 @@ v.2.1 HTTomolib Modules ------------------ -v.2.0 +v.2.1 ''''' .. toctree:: diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml index cf3ded103..1ca76f4cd 100644 --- a/docs/source/doc-conda-requirements.yml +++ b/docs/source/doc-conda-requirements.yml @@ -9,7 +9,7 @@ dependencies: - numpy - tomopy==1.15 - httomolibgpu==2.1 - - httomolib==2.0 + - httomolib==2.1 - sphinx - sphinx-book-theme - nbsphinx From a7739a4cf58071eaebe98ecfc4e45160bad8761e Mon Sep 17 00:00:00 2001 From: dkazanc Date: Fri, 23 Aug 2024 11:30:47 +0100 Subject: [PATCH 02/12] ci_changes1 --- .../{doc_conda.yml => doc_github_ci.yml} | 0 .github/workflows/docs_iris.yml | 61 +++++++++++++++++++ .github/workflows/run_tests_iris.yml | 1 + conda/environment.yml | 2 +- 4 files changed, 63 insertions(+), 1 deletion(-) rename .github/workflows/{doc_conda.yml => doc_github_ci.yml} (100%) create mode 100644 .github/workflows/docs_iris.yml diff --git a/.github/workflows/doc_conda.yml b/.github/workflows/doc_github_ci.yml similarity index 100% rename from .github/workflows/doc_conda.yml rename to .github/workflows/doc_github_ci.yml diff --git a/.github/workflows/docs_iris.yml b/.github/workflows/docs_iris.yml new file mode 100644 index 000000000..5c271f44e --- /dev/null +++ b/.github/workflows/docs_iris.yml @@ -0,0 +1,61 @@ +name: HTTomo docs iris + +on: + workflow_dispatch: + pull_request: + branches: + - main + push: + branches: + - main + + +jobs: + iris-gpu: + runs-on: iris-gpu + container: + image: nvidia/cuda:11.6.2-devel-ubi8 + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + + defaults: + run: + shell: bash -l {0} + + steps: + - name: Checkout repository code + uses: actions/checkout@v4 + + - name: Setup Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: httomo-docs-iris + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: false + activate-environment: httomo-docs + environment-file: ./docs/source/doc-conda-requirements.yml + + - name: Build api docs + run: sphinx-apidoc -feT -t=./docs/source/_templates -o ./docs/source/api ./httomo + + - name: Generate yaml templates + run: | + python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o ./docs/build/yaml_templates/tomopy + python ./scripts/yaml_unsupported_tomopy_remove.py -t ./docs/build/yaml_templates/tomopy -l ./httomo/methods_database/packages/external/tomopy/tomopy.yaml + python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o ./docs/build/yaml_templates/httomolib + python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o ./docs/build/yaml_templates/httomolibgpu + + - name: Generate yml docs + run: python ./docs/source/yaml_doc_generator.py + + - name: Build html + run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/ + + - name: Publish docs + if: github.ref_type == 'tag' || github.ref_name == 'main' + run: ghp-import -n -p -f ./docs/build + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/run_tests_iris.yml b/.github/workflows/run_tests_iris.yml index 6a44a71ec..f88f05359 100644 --- a/.github/workflows/run_tests_iris.yml +++ b/.github/workflows/run_tests_iris.yml @@ -36,6 +36,7 @@ jobs: run: | micromamba activate httomo pip install .[dev] + pip install --upgrade --force-reinstall pillow micromamba list - name: Generate yaml templates diff --git a/conda/environment.yml b/conda/environment.yml index 4e4867d33..9abf849d6 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -8,7 +8,7 @@ channels: dependencies: - astra-toolbox::astra-toolbox - conda-forge::click>=8.1.3 - - conda-forge::cupy<=12.3.0 + - conda-forge::cupy==12.3.0 - conda-forge::cudatoolkit - conda-forge::graypy - conda-forge::h5py=*=*mpi_openmpi* From 47ec66a0a4c3f932d30aeb5b7db6eff6e35be69a Mon Sep 17 00:00:00 2001 From: dkazanc Date: Fri, 23 Aug 2024 12:06:05 +0100 Subject: [PATCH 03/12] ci fixes2 --- .github/workflows/docs_iris.yml | 23 +++++++++++++++-------- docs/source/doc-conda-requirements.yml | 3 +++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs_iris.yml b/.github/workflows/docs_iris.yml index 5c271f44e..904b1280a 100644 --- a/.github/workflows/docs_iris.yml +++ b/.github/workflows/docs_iris.yml @@ -8,11 +8,10 @@ on: push: branches: - main - - + jobs: iris-gpu: - runs-on: iris-gpu + runs-on: iris-gpu-docs container: image: nvidia/cuda:11.6.2-devel-ubi8 env: @@ -38,16 +37,24 @@ jobs: activate-environment: httomo-docs environment-file: ./docs/source/doc-conda-requirements.yml + - name: Install httomo-dev + run: | + micromamba activate httomo + pip install .[dev] + pip install --upgrade --force-reinstall pillow + micromamba list + - name: Build api docs run: sphinx-apidoc -feT -t=./docs/source/_templates -o ./docs/source/api ./httomo - name: Generate yaml templates run: | - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o ./docs/build/yaml_templates/tomopy - python ./scripts/yaml_unsupported_tomopy_remove.py -t ./docs/build/yaml_templates/tomopy -l ./httomo/methods_database/packages/external/tomopy/tomopy.yaml - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o ./docs/build/yaml_templates/httomolib - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o ./docs/build/yaml_templates/httomolibgpu - + python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/httomo_modules.yaml -o ./httomo/yaml_templates/httomo + python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o ./httomo/yaml_templates/tomopy + python ./scripts/yaml_unsupported_tomopy_remove.py -t ./httomo/yaml_templates/tomopy -l ./httomo/methods_database/packages/external/tomopy/tomopy.yaml + python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o ./httomo/yaml_templates/httomolib + python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o ./httomo/yaml_templates/httomolibgpu + - name: Generate yml docs run: python ./docs/source/yaml_doc_generator.py diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml index 1ca76f4cd..8b3c7b136 100644 --- a/docs/source/doc-conda-requirements.yml +++ b/docs/source/doc-conda-requirements.yml @@ -10,6 +10,9 @@ dependencies: - tomopy==1.15 - httomolibgpu==2.1 - httomolib==2.1 + - conda-forge::cupy==12.3.0 + - conda-forge::cudatoolkit + - rapidsai::cucim - sphinx - sphinx-book-theme - nbsphinx From 5a535acaddf0cabee6912295d7f5d65fc6fb791f Mon Sep 17 00:00:00 2001 From: dkazanc Date: Fri, 23 Aug 2024 14:43:45 +0100 Subject: [PATCH 04/12] removes anaconda and astra-toolbox channels --- conda/environment-cpu.yml | 5 ++--- conda/environment.yml | 16 +++++++--------- conda/recipe/meta.yaml | 4 ++-- docs/source/doc-conda-requirements.yml | 6 +++--- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/conda/environment-cpu.yml b/conda/environment-cpu.yml index 903f67fdd..e9a026e0d 100644 --- a/conda/environment-cpu.yml +++ b/conda/environment-cpu.yml @@ -1,7 +1,6 @@ name: httomo channels: - conda-forge - - anaconda - httomo dependencies: - conda-forge::tomopy==1.15 @@ -13,8 +12,8 @@ dependencies: - conda-forge::numpy<=1.25 - conda-forge::python - conda-forge::numexpr>=2.8.4 - - anaconda::ipython - - anaconda::loguru + - conda-forge::ipython + - conda-forge::loguru - conda-forge::graypy - conda-forge::plumbum - conda-forge::tqdm diff --git a/conda/environment.yml b/conda/environment.yml index 9abf849d6..71ac8e110 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -1,15 +1,13 @@ name: httomo channels: - - astra-toolbox - conda-forge - httomo - - anaconda - rapidsai dependencies: - - astra-toolbox::astra-toolbox + - conda-forge::astra-toolbox - conda-forge::click>=8.1.3 - - conda-forge::cupy==12.3.0 - - conda-forge::cudatoolkit + - conda-forge::cupy=12.3.0 + - conda-forge::cudatoolkit>=11.2 - conda-forge::graypy - conda-forge::h5py=*=*mpi_openmpi* - conda-forge::hdf5plugin @@ -22,9 +20,9 @@ dependencies: - conda-forge::plumbum - conda-forge::tqdm - conda-forge::typing_extensions - - anaconda::ipython - - conda-forge::tomopy==1.15 - - httomo::httomolib==2.1 - - httomo::httomolibgpu==2.1 + - conda-forge::ipython + - conda-forge::tomopy=1.15 + - httomo::httomolib=2.1 + - httomo::httomolibgpu=2.1 - httomo::tomobar - rapidsai::cucim diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index 49f33d5b6..a3bb699a6 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -47,13 +47,14 @@ requirements: - astra-toolbox - click>=8.1.3 - cucim - - cupy<=12.3.0 + - cupy=12.3.0 - cudatoolkit - graypy - h5py=*=*mpi_openmpi* - hdf5plugin - httomolib==2.1 - httomolibgpu==2.1 + - tomopy==1.15 - ipython - loguru - mpi4py @@ -68,7 +69,6 @@ requirements: - python - plumbum - tomobar - - tomopy==1.15 - tqdm - typing_extensions diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml index 8b3c7b136..4ff0974d7 100644 --- a/docs/source/doc-conda-requirements.yml +++ b/docs/source/doc-conda-requirements.yml @@ -10,9 +10,9 @@ dependencies: - tomopy==1.15 - httomolibgpu==2.1 - httomolib==2.1 - - conda-forge::cupy==12.3.0 - - conda-forge::cudatoolkit - - rapidsai::cucim + - cupy==12.3.0 + - cudatoolkit + - cucim - sphinx - sphinx-book-theme - nbsphinx From c70bcbc3e95e9397562e98199e6f82dabaa9720c Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 27 Aug 2024 16:00:08 +0100 Subject: [PATCH 05/12] removes cucim as a dependency --- .github/workflows/main_conda_package_test.yml | 2 +- .scripts/conda_upload.sh | 2 +- conda/environment.yml | 4 +--- conda/recipe/meta.yaml | 1 - docs/source/doc-conda-requirements.yml | 3 --- docs/source/howto/installation.rst | 4 ++-- 6 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main_conda_package_test.yml b/.github/workflows/main_conda_package_test.yml index 88a07af9f..84dd2f120 100644 --- a/.github/workflows/main_conda_package_test.yml +++ b/.github/workflows/main_conda_package_test.yml @@ -34,7 +34,7 @@ jobs: - name: Install httomo run: | micromamba activate httomo - micromamba install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge -c astra-toolbox -c rapidsai -y + micromamba install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge -y micromamba list - name: Generate yaml templates diff --git a/.scripts/conda_upload.sh b/.scripts/conda_upload.sh index 9a6188fec..f467a5f35 100755 --- a/.scripts/conda_upload.sh +++ b/.scripts/conda_upload.sh @@ -9,7 +9,7 @@ mkdir ~/conda-bld conda config --set anaconda_upload no export CONDA_BLD_PATH=~/conda-bld -$CONDA/bin/conda build . -c conda-forge -c https://conda.anaconda.org/httomo/ -c rapidsai --no-test +$CONDA/bin/conda build . -c conda-forge -c https://conda.anaconda.org/httomo/ --no-test # upload packages to conda find $CONDA_BLD_PATH/$OS -name *.tar.bz2 | while read file diff --git a/conda/environment.yml b/conda/environment.yml index 71ac8e110..a4ab686dd 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -2,7 +2,6 @@ name: httomo channels: - conda-forge - httomo - - rapidsai dependencies: - conda-forge::astra-toolbox - conda-forge::click>=8.1.3 @@ -24,5 +23,4 @@ dependencies: - conda-forge::tomopy=1.15 - httomo::httomolib=2.1 - httomo::httomolibgpu=2.1 - - httomo::tomobar - - rapidsai::cucim + - httomo::tomobar \ No newline at end of file diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index a3bb699a6..a7aed1ab7 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -46,7 +46,6 @@ requirements: run: - astra-toolbox - click>=8.1.3 - - cucim - cupy=12.3.0 - cudatoolkit - graypy diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml index 4ff0974d7..ecd5648a9 100644 --- a/docs/source/doc-conda-requirements.yml +++ b/docs/source/doc-conda-requirements.yml @@ -3,7 +3,6 @@ channels: - conda-forge - defaults - httomo - - rapidsai dependencies: - python>=3.10 - numpy @@ -11,8 +10,6 @@ dependencies: - httomolibgpu==2.1 - httomolib==2.1 - cupy==12.3.0 - - cudatoolkit - - cucim - sphinx - sphinx-book-theme - nbsphinx diff --git a/docs/source/howto/installation.rst b/docs/source/howto/installation.rst index 6aaf83c1a..046ad4f85 100644 --- a/docs/source/howto/installation.rst +++ b/docs/source/howto/installation.rst @@ -9,7 +9,7 @@ This installation is preferable as it should take care all of dependencies inclu .. code-block:: console $ conda env create --name httomo # create a fresh conda environment - $ conda install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge -c astra-toolbox -c rapidsai + $ conda install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge Install as a Python module =========================== @@ -35,4 +35,4 @@ Build HTTomo as a conda Python package ====================================================== .. code-block:: console - $ conda build conda/recipe/ -c conda-forge -c https://conda.anaconda.org/httomo/ -c rapidsai --no-test + $ conda build conda/recipe/ -c conda-forge -c https://conda.anaconda.org/httomo/ --no-test From 6a4abe68f6d2cb47df685ca6511a930e68635728 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 27 Aug 2024 16:12:14 +0100 Subject: [PATCH 06/12] ci fixes --- .github/workflows/docs_iris.yml | 68 ------------------- .../{doc_github_ci.yml => httomo_docs.yml} | 0 ...onda_upload.yml => httomo_version_tag.yml} | 2 +- 3 files changed, 1 insertion(+), 69 deletions(-) delete mode 100644 .github/workflows/docs_iris.yml rename .github/workflows/{doc_github_ci.yml => httomo_docs.yml} (100%) rename .github/workflows/{conda_upload.yml => httomo_version_tag.yml} (98%) diff --git a/.github/workflows/docs_iris.yml b/.github/workflows/docs_iris.yml deleted file mode 100644 index 904b1280a..000000000 --- a/.github/workflows/docs_iris.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: HTTomo docs iris - -on: - workflow_dispatch: - pull_request: - branches: - - main - push: - branches: - - main - -jobs: - iris-gpu: - runs-on: iris-gpu-docs - container: - image: nvidia/cuda:11.6.2-devel-ubi8 - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} - - defaults: - run: - shell: bash -l {0} - - steps: - - name: Checkout repository code - uses: actions/checkout@v4 - - - name: Setup Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: httomo-docs-iris - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: false - activate-environment: httomo-docs - environment-file: ./docs/source/doc-conda-requirements.yml - - - name: Install httomo-dev - run: | - micromamba activate httomo - pip install .[dev] - pip install --upgrade --force-reinstall pillow - micromamba list - - - name: Build api docs - run: sphinx-apidoc -feT -t=./docs/source/_templates -o ./docs/source/api ./httomo - - - name: Generate yaml templates - run: | - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/httomo_modules.yaml -o ./httomo/yaml_templates/httomo - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o ./httomo/yaml_templates/tomopy - python ./scripts/yaml_unsupported_tomopy_remove.py -t ./httomo/yaml_templates/tomopy -l ./httomo/methods_database/packages/external/tomopy/tomopy.yaml - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o ./httomo/yaml_templates/httomolib - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o ./httomo/yaml_templates/httomolibgpu - - - name: Generate yml docs - run: python ./docs/source/yaml_doc_generator.py - - - name: Build html - run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/ - - - name: Publish docs - if: github.ref_type == 'tag' || github.ref_name == 'main' - run: ghp-import -n -p -f ./docs/build - env: - GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/doc_github_ci.yml b/.github/workflows/httomo_docs.yml similarity index 100% rename from .github/workflows/doc_github_ci.yml rename to .github/workflows/httomo_docs.yml diff --git a/.github/workflows/conda_upload.yml b/.github/workflows/httomo_version_tag.yml similarity index 98% rename from .github/workflows/conda_upload.yml rename to .github/workflows/httomo_version_tag.yml index c646e454f..f80c606bb 100644 --- a/.github/workflows/conda_upload.yml +++ b/.github/workflows/httomo_version_tag.yml @@ -1,4 +1,4 @@ -name: HTTomo version build +name: version tag pushed # Run the workflow whenever a tag beginning with `v` is pushed to any branch on: From 9a3b4c86fd996a190db62ce9052b8d16ad3bbeca Mon Sep 17 00:00:00 2001 From: dkazanc Date: Wed, 28 Aug 2024 11:30:17 +0100 Subject: [PATCH 07/12] fixing tests --- httomo/method_wrappers/dezinging.py | 3 ++- .../httomolibgpu/supporting_funcs/prep/phase.py | 2 +- .../DLS/01_diad_pipeline_gpu.yaml | 1 - .../pipeline_360deg_gpu2.yaml | 1 - .../pipeline_template_examples/pipeline_gpu1.yaml | 1 - tests/test_backends/test_httomolibgpu.py | 6 +++--- tests/test_pipeline.py | 12 ++++++------ 7 files changed, 12 insertions(+), 14 deletions(-) diff --git a/httomo/method_wrappers/dezinging.py b/httomo/method_wrappers/dezinging.py index 3ab122dac..0d3e6b562 100644 --- a/httomo/method_wrappers/dezinging.py +++ b/httomo/method_wrappers/dezinging.py @@ -52,7 +52,8 @@ def execute(self, block: T) -> T: # plug in the correct value for axis instead of auto. In case if axis is absent we take # it equal to 0. This is to avoid failure, but (missing) template parameters should be # tested in the yaml checker potentially - self._config_params["axis"] = args.get("axis", 0) + if self.package_name == "tomopy": + self._config_params["axis"] = args.get("axis", 0) with catch_gputime() as t: block.data = self.method(block.data, **self._config_params) diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py index 9a30f76c0..ecb97ee9a 100644 --- a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py +++ b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py @@ -101,7 +101,7 @@ def _calc_memory_bytes_paganin_filter_tomopy( + out_slice_size + 2 * complex_slice + 0.5 * fftplan_slice - + 2 * res_slice + + res_slice ) subtract_bytes = int(filter_size + grid_size) diff --git a/tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml b/tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml index 2be755b7e..7eb04d26c 100644 --- a/tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml +++ b/tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml @@ -30,7 +30,6 @@ parameters: dif: 0.1 kernel_size: 3 - axis: auto - method: normalize module_path: httomolibgpu.prep.normalize parameters: diff --git a/tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml b/tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml index d5e77c0ae..123b52a44 100644 --- a/tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml +++ b/tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml @@ -26,7 +26,6 @@ parameters: dif: 0.1 kernel_size: 3 - axis: auto - method: normalize module_path: httomolibgpu.prep.normalize parameters: diff --git a/tests/samples/pipeline_template_examples/pipeline_gpu1.yaml b/tests/samples/pipeline_template_examples/pipeline_gpu1.yaml index a622075f1..b71643dfd 100644 --- a/tests/samples/pipeline_template_examples/pipeline_gpu1.yaml +++ b/tests/samples/pipeline_template_examples/pipeline_gpu1.yaml @@ -26,7 +26,6 @@ parameters: dif: 0.1 kernel_size: 3 - axis: auto - method: normalize module_path: httomolibgpu.prep.normalize parameters: diff --git a/tests/test_backends/test_httomolibgpu.py b/tests/test_backends/test_httomolibgpu.py index edfff6d11..8225e1eb5 100644 --- a/tests/test_backends/test_httomolibgpu.py +++ b/tests/test_backends/test_httomolibgpu.py @@ -154,9 +154,9 @@ def test_remove_outlier_memoryhook(flats, ensure_clean_memory, dtype, slices): @pytest.mark.cupy -@pytest.mark.parametrize("slices", [64, 128]) -@pytest.mark.parametrize("dim_x", [81, 260, 320]) -@pytest.mark.parametrize("dim_y", [340, 135, 96]) +@pytest.mark.parametrize("slices", [2, 4, 8]) +@pytest.mark.parametrize("dim_x", [1024, 2048]) +@pytest.mark.parametrize("dim_y", [1024, 2048]) def test_paganin_filter_tomopy_memoryhook(slices, dim_x, dim_y, ensure_clean_memory): data = cp.random.random_sample((slices, dim_x, dim_y), dtype=np.float32) hook = MaxMemoryHook() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 311e69feb..8e84fcc41 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -666,9 +666,9 @@ def test_run_diad_pipeline_gpu( assert "Path to data: /entry/imaging/data" in verbose_log_contents assert "Preview: (100:3101, 8:15, 0:26)" in verbose_log_contents assert "Data shape is (3001, 7, 26) of type uint16" in verbose_log_contents - assert "Global min -0.0326580" in verbose_log_contents - assert "Global max 0.037757" in verbose_log_contents - assert "Global mean 0.000327" in verbose_log_contents + assert "Global min -0.00915" in verbose_log_contents + assert "Global max 0.01134" in verbose_log_contents + assert "Global mean 0.00013" in verbose_log_contents @pytest.mark.cupy @@ -698,9 +698,9 @@ def test_run_pipeline_360deg_gpu2( assert "Loading data: tests/test_data/360scan/360scan.hdf" in verbose_log_contents assert "Path to data: entry1/tomo_entry/data/data" in verbose_log_contents assert "Data shape is (3601, 3, 2560) of type uint16" in verbose_log_contents - assert "Global min -0.002859" in verbose_log_contents - assert "Global max 0.00561" in verbose_log_contents - assert "Global mean 0.000889" in verbose_log_contents + assert "Global min -0.00275" in verbose_log_contents + assert "Global max 0.005299" in verbose_log_contents + assert "Global mean 0.0008" in verbose_log_contents @pytest.mark.cupy From 87c20e18b5e590316f2641cb2dbc41e97542e312 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Wed, 28 Aug 2024 12:39:04 +0100 Subject: [PATCH 08/12] adding mockers to docs --- docs/source/conf.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 10c8d4afe..abc8c9335 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -52,6 +52,24 @@ for mod_name in MOCK_MODULES: sys.modules[mod_name] = mock.Mock() +class CustomMock(mock.Mock): + def __repr__(self): + return "" + + +sys.modules["cupy"] = CustomMock() +sys.modules["numpy"] = CustomMock() +sys.modules["cupyx.scipy.interpolate"] = CustomMock() +sys.modules["cupyx.scipy.ndimage"] = CustomMock() +sys.modules["cupyx.scipy.fft"] = CustomMock() +sys.modules["cupyx.scipy.fftpack"] = CustomMock() +sys.modules["scipy.fftpack"] = CustomMock() +sys.modules["tomobar.methodsDIR_CuPy"] = CustomMock() +sys.modules["tomobar.methodsIR_CuPy "] = CustomMock() +sys.modules["skimage.registration"] = CustomMock() +sys.modules["httomolibgpu.cuda_kernels"] = CustomMock() + + # ------------------------------------------------------------------------------ project = "HTTomo" From a1a826fcca626b1c045bcc4f7f972749c9b65585 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Mon, 9 Sep 2024 14:34:15 +0100 Subject: [PATCH 09/12] docs fixes and CI for docs build --- .github/workflows/httomo_docs.yml | 10 - docs/source/backends/list.rst | 19 +- docs/source/backends/templates.rst | 41 +---- docs/source/developers/how_to_contribute.rst | 2 +- docs/source/doc-conda-requirements.yml | 8 +- docs/source/explanation/templates.rst | 2 +- .../process_lists/side_outputs/side_out.rst | 2 +- docs/source/index.rst | 3 +- docs/source/utilities/yaml_generator.rst | 33 ---- docs/source/yaml_doc_generator.py | 171 ------------------ docs/sphinx-build.sh | 10 - 11 files changed, 19 insertions(+), 282 deletions(-) delete mode 100644 docs/source/utilities/yaml_generator.rst delete mode 100644 docs/source/yaml_doc_generator.py diff --git a/.github/workflows/httomo_docs.yml b/.github/workflows/httomo_docs.yml index 275d074fb..3620bfb06 100644 --- a/.github/workflows/httomo_docs.yml +++ b/.github/workflows/httomo_docs.yml @@ -34,16 +34,6 @@ jobs: - name: Build api docs run: sphinx-apidoc -feT -t=./docs/source/_templates -o ./docs/source/api ./httomo - - name: Generate yaml templates - run: | - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o ./docs/build/yaml_templates/tomopy - python ./scripts/yaml_unsupported_tomopy_remove.py -t ./docs/build/yaml_templates/tomopy -l ./httomo/methods_database/packages/external/tomopy/tomopy.yaml - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o ./docs/build/yaml_templates/httomolib - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o ./docs/build/yaml_templates/httomolibgpu - - - name: Generate yml docs - run: python ./docs/source/yaml_doc_generator.py - - name: Build html run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/ diff --git a/docs/source/backends/list.rst b/docs/source/backends/list.rst index 21e81a640..81908b728 100644 --- a/docs/source/backends/list.rst +++ b/docs/source/backends/list.rst @@ -1,20 +1,19 @@ .. _backends_list: -Supported backends -============================== +Supported libraries +=================== HTTomo currently supports several software packages that are used as -backends to perform data processing and reconstruction. The list of +`backends` to perform data processing and reconstruction. The list of the external packages will be growing in future to meet users' needs. If the package has a modular structure with an easy access to every -method, for example as `TomoPy `_ -software or `scikit `_ library, then the +method, for example as in the `TomoPy `_ +software or in the `scikit `_ library, then the integration process is straightforward. -The required YAML template can be generated by using :ref:`utilities_yamlgenerator`, -or manually, and used with HTTomo. More complicated in structure packages would need -an additional wrapping, see for instance the reconstruction methods in -the `HTTomolibgpu `_ library. + +A YAML template can be generated by using `YAML generator `_, +or manually. More complicated in structure packages would need an additional wrapping, see :ref:`info_wrappers`. Please see the provided list of :ref:`reference_templates`. @@ -42,5 +41,3 @@ by Data Analysis Group to work together with the HTTomo software. HTTomolib library (CPU) -------------------------- `HTTomolib `_ library is similar to HTTomolibgpu, but contains mostly CPU modules. - -.. note:: The HTTomo release usually supports a specific version of the backend package. Please check that the YAML templates are linked to the version advertised in the release. diff --git a/docs/source/backends/templates.rst b/docs/source/backends/templates.rst index 6d5e79bfe..7db3f32e6 100644 --- a/docs/source/backends/templates.rst +++ b/docs/source/backends/templates.rst @@ -4,42 +4,13 @@ Methods YAML Templates ====================== -This section contains YAML templates from :ref:`backends_list`. These are ready-to-use templates can be either copy-pasted or -downloaded in order to build your pipeline. +This section refers to YAML templates from :ref:`backends_list` which are distributed separately through `httomo-backends`. +You will be redirected to the `httomo-backends` page where those templates can be either copy-pasted or +downloaded in order to help you to construct your pipeline. -.. note:: When you click on a module you can find a link to the API of that module. This is where the description of the method and its arguments can be found. +`TomoPy Modules `_ +`HTTomolibgpu Modules `_ -TomoPy Modules ---------------- -v.1.15 -'''''' +`HTTomolib Modules `_ -.. toctree:: - :glob: - - ../api/tomopy* - -.. _reference_templates_httomolibgpu: - -HTTomolibgpu Modules --------------------- -v.2.1 -''''' - -.. toctree:: - :glob: - - ../api/httomolibgpu* - -.. _reference_templates_httomolib: - -HTTomolib Modules ------------------- -v.2.1 -''''' - -.. toctree:: - :glob: - - ../api/httomolib.* diff --git a/docs/source/developers/how_to_contribute.rst b/docs/source/developers/how_to_contribute.rst index 0db79dfad..77603d8c5 100644 --- a/docs/source/developers/how_to_contribute.rst +++ b/docs/source/developers/how_to_contribute.rst @@ -29,7 +29,7 @@ the issues section on `HTTomo's Github page `_. diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml index ecd5648a9..8812b9cfd 100644 --- a/docs/source/doc-conda-requirements.yml +++ b/docs/source/doc-conda-requirements.yml @@ -1,15 +1,9 @@ name: httomo-docs channels: - - conda-forge - - defaults - - httomo + - conda-forge dependencies: - python>=3.10 - numpy - - tomopy==1.15 - - httomolibgpu==2.1 - - httomolib==2.1 - - cupy==12.3.0 - sphinx - sphinx-book-theme - nbsphinx diff --git a/docs/source/explanation/templates.rst b/docs/source/explanation/templates.rst index 1fbd5aeae..2db04ef2c 100644 --- a/docs/source/explanation/templates.rst +++ b/docs/source/explanation/templates.rst @@ -26,6 +26,6 @@ by importing the method as: The set of parameter values for that method is given in the *parameters* field. -.. note:: Please note that in the original TomoPy's method, there is also the :code:`arr` parameter. This parameter is not exposed in the template because HTTomo will deal with all I/O aspects behind the scenes by using special wrappers. More on that in :ref:`detailed_about`. :ref:`utilities_yamlgenerator` will automatically generate the ready-to-be-used templates. +.. note:: Please note that in the original TomoPy's method, there is also the :code:`arr` parameter. This parameter is not exposed in the template because HTTomo will deal with all I/O aspects behind the scenes by using special wrappers. More on that in :ref:`detailed_about`. `YAML generator `_ will automatically generate the ready-to-be-used templates. Please see the list of all supported :ref:`reference_templates`. diff --git a/docs/source/howto/process_lists/side_outputs/side_out.rst b/docs/source/howto/process_lists/side_outputs/side_out.rst index ab96f2adf..f20f7d4e0 100644 --- a/docs/source/howto/process_lists/side_outputs/side_out.rst +++ b/docs/source/howto/process_lists/side_outputs/side_out.rst @@ -66,7 +66,7 @@ information of :ref:`centering` is stored in the reference There could be various configurations when this reference is required from other methods as well. We present more verbose :ref:`side_output_example` below. .. note:: Side outputs and references to them are generated automatically with the - :ref:`utilities_yamlgenerator`. Usually there is no need to modify them when + `YAML generator `_. Usually there is no need to modify them when editing a process list. .. _side_output_example: diff --git a/docs/source/index.rst b/docs/source/index.rst index 316299d51..9f87d795a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -36,7 +36,7 @@ .. _backends_content: .. toctree:: - :caption: Templates for backends + :caption: Data processing :maxdepth: 2 backends/list @@ -58,7 +58,6 @@ :maxdepth: 2 utilities/yaml_checker - utilities/yaml_generator .. _developers_content: diff --git a/docs/source/utilities/yaml_generator.rst b/docs/source/utilities/yaml_generator.rst deleted file mode 100644 index 63345b2af..000000000 --- a/docs/source/utilities/yaml_generator.rst +++ /dev/null @@ -1,33 +0,0 @@ -.. _utilities_yamlgenerator: - -Templates generator -******************* -:ref:`backends_content` can be generated automatically using the YAML generator tool `provided `_ in the Github repo. - -The script does the following: - -* Generates a list of YAML files for all accessible on import methods in a chosen software package, e.g., TomoPy. -* Modifies and/or removes some extracted parameters in YAML templates to make the templates compatible with HTTomo. - -How does it work: - -* The user would need to provide a YAML file with the listed *modules* you would like to inspect and extract the methods from. For instance, for the TomoPy package this would be: - -.. code-block:: yaml - - - tomopy.misc.corr - - tomopy.misc.morph - -* The generator can be applied using the following command: - -.. code-block:: console - - $ python -m yaml_templates_generator -i /path/to/modules.yaml -o /path/to/outputfolder/ - -Please note that the package from which the modules are extracted, must be installed into your conda environment. - -**For TomoPy templates only.** After templates have been generated for TomoPy, we need to remove the ones that are not currently supported by HTTomo. We do that by looking into the library file that exists in HTTomo for TomoPy. - -.. code-block:: console - - $ python -m remove_unsupported_templates -t /path/to/templates/ -l /path/to/library/file diff --git a/docs/source/yaml_doc_generator.py b/docs/source/yaml_doc_generator.py deleted file mode 100644 index b998e11d6..000000000 --- a/docs/source/yaml_doc_generator.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------- -# Copyright 2022 Diamond Light Source Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- - -import os - - -def add_function_summary(doc_dir, root, files): - """Append title and function summary to documentation file. - - Parameters - ---------- - doc_dir : Path - Documentation path - root : Path - Path to the yaml template directory. - files : List - List of yaml function templates for each module. - """ - rst_name = root.split("/")[-1] - doc_rst_file = f"{doc_dir}/api/{rst_name}.rst" - with open(doc_rst_file, "a") as edit_doc: - if os.stat(doc_rst_file).st_size == 0: - rst_name = root.split("/")[-1] - add_title(edit_doc, rst_name) - add_backend_link(edit_doc, rst_name) - else: - edit_doc.write(f"\n\n .. rubric:: **Functions**") - edit_doc.write(f"\n\n .. autosummary::\n") - for fi in files: - yml_title = fi.split(".yaml")[0] - edit_doc.write(f"\n {yml_title}") - - -def create_yaml_dropdown(doc_dir, root, files): - """Create dropdown panels to allow yaml functions to be downloaded. - - Parameters - ---------- - doc_dir : Path - Documentation path - root : Path - Path to the yaml template directory. - files : List - List of functions for each module. - """ - rst_name = root.split("/")[-1] - doc_rst_file = f"{doc_dir}/api/{rst_name}.rst" - template_dir = root.split("source")[-1] - download_all_button(template_dir, doc_rst_file) - - for fi in files: - t_name = f"{template_dir}/{fi}" - f_name = fi.split(".yaml")[0] - url = ( - f"https://tomopy.readthedocs.io/en/stable/api/{rst_name}" - f".html#{rst_name}.{f_name}" - ) - with open(doc_rst_file, "a") as edit_doc: - edit_doc.write(f"\n\n.. dropdown:: {fi}") - edit_doc.write(f"\n\n :download:`Download <{t_name}>`\n\n") - if "tomopy" in t_name: - edit_doc.write( - f" |link_icon| `Link to {f_name}" - f" function description <{url}>`_" - ) - edit_doc.write(f"\n\n .. literalinclude:: {t_name}") - - -def download_all_button(template_dir, doc_rst_file): - """Download all yaml function text as one file. - - Parameters - ---------- - template_dir : str - Directory to template files. - doc_rst_file : File - rst file to write to. - """ - download_all_path = f"{template_dir}/download_all.yaml" - # Only include download all when more than one yaml file exists - if len(files) > 1: - with open(doc_rst_file, "a") as edit_doc: - download_str = "\n\n:download:`Download all yaml templates<" - edit_doc.write(f"{download_str}{download_all_path}>`") - - -def add_title(edit_doc, rst_name): - """Add a title to rst file. - Parameters - ---------- - edit_doc : File - Document to write to. - rst_name : str - name of rst file. - """ - edit_doc.write(f".. |link_icon| unicode:: U+1F517\n\n") - title = f":mod:`{rst_name}`" - edit_doc.write(f"{title}\n") - underline = len(title) * "=" - edit_doc.write(f"{underline}\n") - - -def add_backend_link(edit_doc, rst_name): - """Links to backends documentation. - - Parameters - ---------- - edit_doc : File - Document to write to. - rst_name : str - name of rst file. - """ - if "tomopy" in rst_name: - # If it is a tomopy module, insert a link. - url = "https://tomopy.readthedocs.io/en/stable/api/" - elif "httomolibgpu" in rst_name: - url = "https://diamondlightsource.github.io/httomolibgpu/api/" - elif "httomolib" in rst_name: - url = "https://diamondlightsource.github.io/httomolib/api/" - else: - raise ValueError("The name of the backend package is not recognised") - edit_doc.write(f"\n{url}{rst_name}.html\n\n") - - -def save_all_yaml_functions(tmp_dir, yaml_files): - """Create a file including all yaml function definitions. - - Parameters - ---------- - tmp_dir : Path - Path to the yaml template directory. - yaml_files : list - List of individual yaml template files. - """ - file_path = f"{tmp_dir}/download_all.yaml" - # Only merge files when more than one yaml file exists - if len(yaml_files) > 1: - with open(file_path, "w") as outfile: - for f in yaml_files: - with open(f"{tmp_dir}/{f}") as infile: - outfile.write(infile.read()) - - -if __name__ == "__main__": - """Create documentation for modules from httomo, tomopy, httomolib and httomolibgpu. - Append the yaml information to the documentation pages. - """ - doc_source_dir = os.path.dirname(os.path.abspath(__file__)) - path_to_templates = doc_source_dir + "/../build/yaml_templates/" - for root, dirs, files in os.walk(path_to_templates, topdown=True): - dirs[:] = [d for d in dirs] - files[:] = [fi for fi in files if ".yaml" in fi] - if files: - add_function_summary(doc_source_dir, root, files) - save_all_yaml_functions(root, files) - create_yaml_dropdown(doc_source_dir, root, files) diff --git a/docs/sphinx-build.sh b/docs/sphinx-build.sh index cd5a1f84f..36a4c8695 100644 --- a/docs/sphinx-build.sh +++ b/docs/sphinx-build.sh @@ -27,22 +27,12 @@ rm -rf $DIR/build/ # sphinx-apidoc generates source files that use sphinx.ext.autodoc to document all found modules sphinx-apidoc -feT -t=$DIR/source/_templates -o $DIR/source/api $DIR/../httomo -# build yaml templates here: -python $DIR/../scripts/yaml_templates_generator.py -i $DIR/../httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o $DIR/build/yaml_templates/tomopy -python $DIR/../scripts/yaml_unsupported_tomopy_remove.py -t $DIR/build/yaml_templates/tomopy -l $DIR/../httomo/methods_database/packages/external/tomopy/tomopy.yaml -python $DIR/../scripts/yaml_templates_generator.py -i $DIR/../httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o $DIR/build/yaml_templates/httomolibgpu -python $DIR/../scripts/yaml_templates_generator.py -i $DIR/../httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o $DIR/build/yaml_templates/httomolib - -# Append yaml link to rst files -python -m source.yaml_doc_generator - # sphinx-build [options] [filenames] # -a Write all output files. The default is to only write output files for new and changed source files. (This may not apply to all builders.) # -E Don’t use a saved environment (the structure caching all cross-references), but rebuild it completely. The default is to only read and parse source files that are new or have changed since the last run. # -b buildername, build pages of a certain file type sphinx-build -a -E -b html $DIR/source/ $DIR/build/ - echo "***********************************************************************" echo " End of script" echo "***********************************************************************" From 8dfcff4162b7ef556cd033dd74274dccfa4b9045 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Mon, 9 Sep 2024 14:48:17 +0100 Subject: [PATCH 10/12] bring #432 fix instead --- docs/source/conf.py | 1 + httomo/method_wrappers/dezinging.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index abc8c9335..b8903bc44 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -52,6 +52,7 @@ for mod_name in MOCK_MODULES: sys.modules[mod_name] = mock.Mock() + class CustomMock(mock.Mock): def __repr__(self): return "" diff --git a/httomo/method_wrappers/dezinging.py b/httomo/method_wrappers/dezinging.py index 0d3e6b562..5972f82a8 100644 --- a/httomo/method_wrappers/dezinging.py +++ b/httomo/method_wrappers/dezinging.py @@ -48,11 +48,13 @@ def execute(self, block: T) -> T: # check if data needs to be transfered host <-> device block = self._transfer_data(block) - args = self._build_kwargs(self._transform_params(self._config_params), block) - # plug in the correct value for axis instead of auto. In case if axis is absent we take - # it equal to 0. This is to avoid failure, but (missing) template parameters should be - # tested in the yaml checker potentially - if self.package_name == "tomopy": + if "axis" in self.parameters: + args = self._build_kwargs( + self._transform_params(self._config_params), block + ) + # plug in the correct value for axis instead of auto. In case if axis is absent we + # take it equal to 0. This is to avoid failure, but (missing) template parameters + # should be tested in the yaml checker potentially self._config_params["axis"] = args.get("axis", 0) with catch_gputime() as t: From 12949fd43a93d79f86f5ff1ba7d989309b64b0d3 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Mon, 9 Sep 2024 15:28:51 +0100 Subject: [PATCH 11/12] reverting memory estimator for paganin to initial state and add odd size for the test --- .../external/httomolibgpu/supporting_funcs/prep/phase.py | 2 +- tests/test_backends/test_httomolibgpu.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py index ecb97ee9a..9a30f76c0 100644 --- a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py +++ b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py @@ -101,7 +101,7 @@ def _calc_memory_bytes_paganin_filter_tomopy( + out_slice_size + 2 * complex_slice + 0.5 * fftplan_slice - + res_slice + + 2 * res_slice ) subtract_bytes = int(filter_size + grid_size) diff --git a/tests/test_backends/test_httomolibgpu.py b/tests/test_backends/test_httomolibgpu.py index 8225e1eb5..e0c8b140a 100644 --- a/tests/test_backends/test_httomolibgpu.py +++ b/tests/test_backends/test_httomolibgpu.py @@ -155,8 +155,8 @@ def test_remove_outlier_memoryhook(flats, ensure_clean_memory, dtype, slices): @pytest.mark.cupy @pytest.mark.parametrize("slices", [2, 4, 8]) -@pytest.mark.parametrize("dim_x", [1024, 2048]) -@pytest.mark.parametrize("dim_y", [1024, 2048]) +@pytest.mark.parametrize("dim_x", [81, 1024, 2048]) +@pytest.mark.parametrize("dim_y", [345, 1024, 2048]) def test_paganin_filter_tomopy_memoryhook(slices, dim_x, dim_y, ensure_clean_memory): data = cp.random.random_sample((slices, dim_x, dim_y), dtype=np.float32) hook = MaxMemoryHook() From 291739addf1e0eff4aaad54ecab902674cd60f38 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Mon, 9 Sep 2024 15:31:40 +0100 Subject: [PATCH 12/12] bring back old paganin test --- tests/test_backends/test_httomolibgpu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_backends/test_httomolibgpu.py b/tests/test_backends/test_httomolibgpu.py index e0c8b140a..edfff6d11 100644 --- a/tests/test_backends/test_httomolibgpu.py +++ b/tests/test_backends/test_httomolibgpu.py @@ -154,9 +154,9 @@ def test_remove_outlier_memoryhook(flats, ensure_clean_memory, dtype, slices): @pytest.mark.cupy -@pytest.mark.parametrize("slices", [2, 4, 8]) -@pytest.mark.parametrize("dim_x", [81, 1024, 2048]) -@pytest.mark.parametrize("dim_y", [345, 1024, 2048]) +@pytest.mark.parametrize("slices", [64, 128]) +@pytest.mark.parametrize("dim_x", [81, 260, 320]) +@pytest.mark.parametrize("dim_y", [340, 135, 96]) def test_paganin_filter_tomopy_memoryhook(slices, dim_x, dim_y, ensure_clean_memory): data = cp.random.random_sample((slices, dim_x, dim_y), dtype=np.float32) hook = MaxMemoryHook()