New testing structure with auto-pipelines generation

.github/workflows/httomo_docs.yml

@@ -35,9 +35,9 @@ jobs:
           run: |
             pip install --no-deps httomo-backends
 
-        - name: Generate full yaml pipelines using directives
+        - name: Generate full yaml pipelines using pipeline directives
           run: |
-            python ./docs/source/scripts/yaml_pipelines_generator.py -i ./docs/source/pipelines_full/gpu_pipeline1_directive.yaml -o ./docs/source/pipelines_full/gpu_pipeline1.yaml
+            python ./docs/source/scripts/execute_pipelines_build.py -o ./docs/source/pipelines_full/
 
         - name: Build docs
           run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/

.github/workflows/run_tests_iris.yml → .github/workflows/run_tests_framework_iris.yml

@@ -1,4 +1,4 @@
-name: HTTomo tests
+name: HTTomo framework tests
 
 on:
   pull_request:
@@ -38,9 +38,14 @@ jobs:
           pip install --upgrade --force-reinstall pillow
           pip install httomolibgpu tomobar
           pip install --no-deps httomo-backends
-          pip install .         
+          pip install .
           micromamba list
 
-      - name: Run HTTomo tests
+      - name: Generate full yaml pipelines using pipeline directives
+        run: |
+          pip install "ruamel.yaml>0.18.0"
+          python ./docs/source/scripts/execute_pipelines_build.py -o ./docs/source/pipelines_full/
+
+      - name: Run HTTomo framework tests
         run: |
           pytest tests/

.github/workflows/run_tests_pipelines_iris.yml

@@ -0,0 +1,51 @@
+name: HTTomo pipelines tests
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  iris-gpu:
+    runs-on: iris-gpu
+    container:
+      image: nvidia/cuda:12.6.3-devel-ubi8
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - name: Create conda environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: conda/environment.yml
+          environment-name: httomo
+          post-cleanup: 'all'
+          init-shell: bash
+
+      - name: Install httomo libraries, httomo-backends and httomo
+        run: |
+          micromamba activate httomo
+          pip install --upgrade --force-reinstall pillow
+          pip install httomolibgpu tomobar
+          pip install --no-deps httomo-backends
+          pip install .   
+          micromamba list
+
+      - name: Generate full yaml pipelines using pipeline directives
+        run: |
+          pip install "ruamel.yaml>0.18.0"
+          python ./docs/source/scripts/execute_pipelines_build.py -o ./docs/source/pipelines_full/
+
+      - name: Run HTTomo pipelines tests (small data)
+        run: |
+          pytest tests/test_pipeline_small.py --small_data

conda/environment-cpu.yml

@@ -1,20 +1,24 @@
 name: httomo
 channels:
   - conda-forge
-  - httomo
 dependencies:
-  - conda-forge::tomopy==1.15
-  - httomo::httomolib==2.1
-  - conda-forge::click>=8.1.3
-  - conda-forge::mpi4py>=3.1
-  - conda-forge::h5py=*=*mpi_openmpi*
-  - conda-forge::pyyaml
-  - conda-forge::numpy<=1.25
-  - conda-forge::python
-  - conda-forge::numexpr>=2.8.4
-  - conda-forge::ipython
-  - conda-forge::loguru
-  - conda-forge::graypy
-  - conda-forge::plumbum
-  - conda-forge::tqdm
-  - conda-forge::typing_extensions
+  - scipy
+  - pillow
+  - click
+  - numpy
+  - graypy
+  - h5py=*=*mpi_openmpi*
+  - hdf5plugin
+  - loguru
+  - mpi4py
+  - pyyaml
+  - nvtx
+  - python
+  - plumbum
+  - tqdm
+  - typing_extensions
+  - ipython
+  - pytest
+  - pytest-xdist
+  - pytest-cov
+  - pytest-mock

docs/source/pipelines/yaml.rst

@@ -3,70 +3,66 @@
 Full YAML pipelines
 ==============================
 
-This is a collection of ready to be used pipeline templates aka process lists for HTTomo.
+This is a collection of ready to be used full pipelines or process lists for HTTomo.
 See more on :ref:`explanation_process_list` and how to :ref:`howto_process_list`.
 
-.. _tutorials_pl_templates_cpu:
+HTTomo mainly targets GPU computations, therefore the use of :ref:`tutorials_pl_templates_gpu` is 
+preferable. However, when the GPU device is not available or a GPU method is not implemented, the use of 
+:ref:`tutorials_pl_templates_cpu` is possible. 
 
-CPU Pipeline templates
-----------------------------
+.. note:: The combination of both GPU and CPU methods is possible. If one expects to achieve the faster performance, please use the GPU methods provided, where possible.
 
-CPU-pipelines mostly use TomoPy methods that are executed on the CPU and expected to be slower.
+.. _tutorials_pl_templates_gpu:
 
-.. dropdown:: Basic TomoPy's (CPU-only) pipeline for the classical 180-degrees scan
+GPU Pipeline templates
+-----------------------
 
-    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_cpu1.yaml
-        :language: yaml
+The GPU-pipelines consist of methods from httomolibgpu (GPU) and httomolib (CPU) backend :ref:`backends_list`. Those libraries are supported directly by the HTTomo development team.
 
-.. dropdown:: TomoPy's pipeline where :ref:`previewing` is demonstrated
+.. dropdown:: GPU pipeline with auto-centering and the FBP reconstruction method.
 
-    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_cpu2.yaml
+    .. literalinclude:: ../pipelines_full/gpu_pipelineFBP.yaml
         :language: yaml
 
-.. dropdown:: This pipeline shows how "calculate_stats" module extracts global statistics in order to rescale data for saving 8-bit images
+.. dropdown:: GPU pipeline as above and Total Variation denoising on the result of the FBP reconstruction.
 
-    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_cpu3.yaml
+    .. literalinclude:: ../pipelines_full/gpu_pipelineFBP_denoising.yaml
         :language: yaml
 
-.. _tutorials_pl_templates_gpu:
+.. _tutorials_pl_templates_cpu:
 
-GPU Pipeline templates
-----------------------------
+CPU Pipeline templates
+-----------------------
 
-It is recommended to use GPU-based pipelines and methods from the httomolib and httomolibgpu libraries. Those libraries are supported directly by HTTomo development team. 
+The CPU-pipelines mostly use TomoPy methods. They are executed solely on the CPU and therefore expected to be slower than the GPU pipelines.
 
-.. dropdown:: Basic GPU pipeline which uses functions from the httomolibgpu library.
+.. dropdown:: CPU pipeline using auto-centering and the gridrec reconstruction method on the CPU (TomoPy).
 
-    .. literalinclude:: ../pipelines_full/gpu_pipeline1.yaml
+    .. literalinclude:: ../pipelines_full/cpu_pipeline_gridrec.yaml
         :language: yaml
 
 
 .. _tutorials_pl_templates_dls:
 
 DLS-specific templates
-----------------------------
+----------------------
 
-Those pipelines will use the methods from the httomolib and httomolibgpu libraries. 
-
-.. dropdown:: An example of a typical DIAD (k11) beamline piepeline.
-
-    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml
-        :language: yaml
+Those pipelines are specific to Diamond Light Source processing strategies and can vary between different tomographic beamlines. 
 
-.. dropdown:: Pipeline for 360-degrees data with automatic CoR finding and stitching to 180-degrees data.
+.. dropdown:: An example of DIAD-k11 beamline pipeline with auto-centering and FBP reconstruction on the GPU.
 
-    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml
+    .. literalinclude:: ../pipelines_full/gpu_diad_FBP.yaml
         :language: yaml
 
-.. dropdown:: Pipeline for 360-degrees data with automatic CoR finding and stitching to 180-degrees data. Iterative reconstruction
+.. dropdown:: Pipeline for 360-degrees data with automatic CoR/overlap finding and stitching to 180-degrees data.
 
-    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_360deg_iterative_gpu3.yaml
+    .. literalinclude:: ../pipelines_full/gpu_360_paganin_FBP.yaml
         :language: yaml
 
 .. _tutorials_pl_templates_sweeps:
 
 Parameter Sweeps templates
-----------------------------
+--------------------------
 
 These templates demonstrate how to perform a sweep across multiple values of a
 single parameter (see :ref:`parameter_sweeping` for more details).

docs/source/scripts/execute_pipelines_build.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------
+# Copyright 2022 Diamond Light Source Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+# Created By  : Tomography Team <scientificsoftware@diamond.ac.uk>
+# Created Date: 30/January/2025
+# version ='0.1'
+# ---------------------------------------------------------------------------
+"""Executing full-pipeline generation for HTTomo using YAML templates from httomo-backends
+and yaml_pipelines_generator script available also in httomo-backends.
+"""
+
+import argparse
+import os
+import glob
+import httomo_backends
+from httomo_backends.scripts.yaml_pipelines_generator import yaml_pipelines_generator
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description="Script that generates YAML pipelines for HTTomo "
+        "using YAML templates from httomo-backends."
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        default="./",
+        help="Full path to the output pipelines folder.",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    path_to_httomobackends = os.path.dirname(httomo_backends.__file__)
+    args = get_args()
+    path_to_httomo_pipelines = args.output
+    pipelines_folder = path_to_httomobackends + "/pipelines_full/"
+    # loop over all pipeline directive files and running the generator
+    for filepath in glob.iglob(pipelines_folder + "*.yaml"):
+        basename = os.path.basename(filepath)
+        outputfile_name = os.path.normpath(basename.replace(r"_directive", r""))
+        yaml_pipelines_generator(
+            filepath, path_to_httomobackends, path_to_httomo_pipelines + outputfile_name
+        )
+        message_str = f"{outputfile_name} has been generated."
+        print(message_str)