Update pipelines to match changes to image saver and rescaler methods

httomo/transform_layer.py

@@ -73,15 +73,28 @@ def insert_data_reducer(self, pipeline: Pipeline) -> Pipeline:
         return Pipeline(loader, methods)
 
     def insert_save_images_after_sweep(self, pipeline: Pipeline) -> Pipeline:
-        """For sweep methods we add image saving method after.
-        In addition we also add saving the results of the reconstruction,
-        if the module is present"""
+        """For sweep methods we add image saving method after, and also a rescaling method to
+        rescale the data passed to the image saver. In addition we also add saving the results
+        of the reconstruction, if the module is present"""
         loader = pipeline.loader
         methods = []
         sweep_before = False
         for m in pipeline:
             methods.append(m)
             if m.sweep or "recon" in m.module_path and sweep_before:
+                methods.append(
+                    make_method_wrapper(
+                        self._repo,
+                        "httomolibgpu.misc.rescale",
+                        "rescale_to_int",
+                        comm=self._comm,
+                        save_result=False,
+                        task_id=f"rescale_sweep_{m.task_id}",
+                        perc_range_min=0.0,
+                        perc_range_max=100.0,
+                        bits=8,
+                    )
+                )
                 methods.append(
                     make_method_wrapper(
                         self._repo,
@@ -92,8 +105,6 @@ def insert_save_images_after_sweep(self, pipeline: Pipeline) -> Pipeline:
                         task_id=f"saveimage_sweep_{m.task_id}",
                         subfolder_name="images_sweep_" + str(m.method_name),
                         axis=1,
-                        perc_range_min=5,
-                        perc_range_max=95,
                     ),
                 )
                 sweep_before = True

tests/samples/pipeline_template_examples/DLS/01_diad_pipeline_gpu.yaml

@@ -56,14 +56,17 @@
   id: statistics
   side_outputs:
     glob_stats: glob_stats
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+  parameters:
+    perc_range_min: 0.0
+    perc_range_max: 100.0
+    bits: 8
+    glob_stats: ${{statistics.side_outputs.glob_stats}}
 - method: save_to_images
   module_path: httomolib.misc.images
   parameters:
     subfolder_name: images
     axis: auto
     file_format: tif
-    bits: 8
-    perc_range_min: 0.0
-    perc_range_max: 95.0
     jpeg_quality: 95
-    glob_stats: ${{statistics.side_outputs.glob_stats}}

tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml

@@ -63,14 +63,17 @@
   id: statistics
   side_outputs:
     glob_stats: glob_stats
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+  parameters:
+    perc_range_min: 0.0
+    perc_range_max: 100.0
+    bits: 8
+    glob_stats: ${{statistics.side_outputs.glob_stats}}
 - method: save_to_images
   module_path: httomolib.misc.images
   parameters:
     subfolder_name: images
     axis: auto
     file_format: tif
-    bits: 8
-    perc_range_min: 0.0
-    perc_range_max: 95.0
     jpeg_quality: 95
-    glob_stats: ${{statistics.side_outputs.glob_stats}}

tests/samples/pipeline_template_examples/pipeline_cpu1.yaml

@@ -34,13 +34,16 @@
     sinogram_order: false
     algorithm: 'gridrec'
     init_recon: null
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+  parameters:
+    perc_range_min: 0.0
+    perc_range_max: 100.0
+    bits: 8
 - method: save_to_images
   module_path: httomolib.misc.images
   parameters:
     subfolder_name: images
     axis: auto
     file_format: tif
-    bits: 8
-    perc_range_min: 0.0
-    perc_range_max: 100.0
     jpeg_quality: 95

tests/samples/pipeline_template_examples/pipeline_cpu2.yaml

@@ -58,13 +58,16 @@
   parameters:
     size: 3
     axis: auto
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+  parameters:
+    perc_range_min: 0.0
+    perc_range_max: 100.0
+    bits: 8
 - method: save_to_images
   module_path: httomolib.misc.images
   parameters:
     subfolder_name: images
     axis: auto
     file_format: tif
-    bits: 8
-    perc_range_min: 0.0
-    perc_range_max: 100.0
     jpeg_quality: 95

tests/samples/pipeline_template_examples/pipeline_cpu3.yaml

@@ -47,15 +47,17 @@
   id: statistics
   side_outputs:
     glob_stats: glob_stats
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+  parameters:
+    perc_range_min: 0.0
+    perc_range_max: 100.0
+    bits: 8
+    glob_stats: ${{statistics.side_outputs.glob_stats}}
 - method: save_to_images
   module_path: httomolib.misc.images
   parameters:
     subfolder_name: images
     axis: auto
     file_format: tif
-    bits: 8
-    perc_range_min: 0.0
-    perc_range_max: 100.0
     jpeg_quality: 95
-    glob_stats: ${{statistics.side_outputs.glob_stats}}
-

tests/samples/pipeline_template_examples/pipeline_cpu4.yaml

@@ -31,13 +31,16 @@
     sinogram_order: false
     algorithm: 'gridrec'
     init_recon: null
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+  parameters:
+    perc_range_min: 0.0
+    perc_range_max: 100.0
+    bits: 8
 - method: save_to_images
   module_path: httomolib.misc.images
   parameters:
     subfolder_name: images
     axis: auto
     file_format: tif
-    bits: 8
-    perc_range_min: 0.0
-    perc_range_max: 100.0
     jpeg_quality: 95

tests/samples/pipeline_template_examples/pipeline_gpu1.yaml

@@ -46,13 +46,16 @@
     recon_size: null
     recon_mask_radius: null
   save_result: true
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+  parameters:
+    perc_range_min: 0.0
+    perc_range_max: 100.0
+    bits: 8
 - method: save_to_images
   module_path: httomolib.misc.images
   parameters:
     subfolder_name: images
     axis: auto
     file_format: tif
-    bits: 8
-    perc_range_min: 0.0
-    perc_range_max: 100.0
     jpeg_quality: 95

tests/samples/pipeline_template_examples/testing/testing_pipeline.yaml

@@ -34,13 +34,16 @@
     algorithm: gridrec
     init_recon: null
   save_result: true
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+  parameters:
+    perc_range_min: 0.0
+    perc_range_max: 100.0
+    bits: 8
 - method: save_to_images
   module_path: httomolib.misc.images
   parameters:
     subfolder_name: images
     axis: 1
     file_format: tif
-    bits: 8
-    perc_range_min: 0.0
-    perc_range_max: 100.0
     jpeg_quality: 95

tests/samples/python_templates/pipeline_cpu1.py

@@ -73,18 +73,25 @@ def methods_to_list() -> PipelineConfig:
     }
     full_pipeline_list.append(method4)
     method5 = {
+        "method": "rescale_to_int",
+        "module_path": "httomolibgpu.misc.rescale",
+        "parameters": {
+            "perc_range_min": 0.0,
+            "perc_range_max": 100.0,
+            "bits": 8,
+        },
+    }
+    full_pipeline_list.append(method5)
+    method6 = {
         "method": "save_to_images",
         "module_path": "httomolib.misc.images",
         "parameters": {
             "subfolder_name": "images",
             "axis": 1,
             "file_format": "tif",
-            "bits": 8,
-            "perc_range_min": 0.0,
-            "perc_range_max": 100.0,
             "jpeg_quality": 95,
         },
     }
-    full_pipeline_list.append(method5)
+    full_pipeline_list.append(method6)
 
     return full_pipeline_list

tests/samples/python_templates/pipeline_cpu2.py

@@ -105,18 +105,25 @@ def methods_to_list() -> PipelineConfig:
     }
     full_pipeline_list.append(method7)
     method8 = {
+        "method": "rescale_to_int",
+        "module_path": "httomolibgpu.misc.rescale",
+        "parameters": {
+            "perc_range_min": 0.0,
+            "perc_range_max": 100.0,
+            "bits": 8,
+        },
+    }
+    full_pipeline_list.append(method8)
+    method9 = {
         "method": "save_to_images",
         "module_path": "httomolib.misc.images",
         "parameters": {
             "subfolder_name": "images",
             "axis": 1,
             "file_format": "tif",
-            "bits": 8,
-            "perc_range_min": 0.0,
-            "perc_range_max": 100.0,
             "jpeg_quality": 95,
         },
     }
-    full_pipeline_list.append(method8)
+    full_pipeline_list.append(method9)
 
     return full_pipeline_list

tests/samples/python_templates/pipeline_cpu3.py

@@ -91,19 +91,26 @@ def methods_to_list() -> PipelineConfig:
     }
     full_pipeline_list.append(method6)
     method7 = {
+        "method": "rescale_to_int",
+        "module_path": "httomolibgpu.misc.rescale",
+        "parameters": {
+            "perc_range_min": 0.0,
+            "perc_range_max": 100.0,
+            "bits": 8,
+            "glob_stats": "${{statistics.side_outputs.glob_stats}}",
+        },
+    }
+    full_pipeline_list.append(method7)
+    method8 = {
         "method": "save_to_images",
         "module_path": "httomolib.misc.images",
         "parameters": {
             "subfolder_name": "images",
             "axis": 1,
             "file_format": "tif",
-            "bits": 8,
-            "perc_range_min": 0.0,
-            "perc_range_max": 100.0,
             "jpeg_quality": 95,
-            "glob_stats": "${{statistics.side_outputs.glob_stats}}",
         },
     }
-    full_pipeline_list.append(method7)
+    full_pipeline_list.append(method8)
 
     return full_pipeline_list

tests/samples/python_templates/pipeline_gpu1.py

@@ -88,18 +88,25 @@ def methods_to_list() -> PipelineConfig:
     }
     full_pipeline_list.append(method5)
     method6 = {
+        "method": "rescale_to_int",
+        "module_path": "httomolibgpu.misc.rescale",
+        "parameters": {
+            "perc_range_min": 0.0,
+            "perc_range_max": 100.0,
+            "bits": 8,
+        },
+    }
+    full_pipeline_list.append(method6)
+    method7 = {
         "method": "save_to_images",
         "module_path": "httomolib.misc.images",
         "parameters": {
             "subfolder_name": "images",
             "axis": 1,
             "file_format": "tif",
-            "bits": 8,
-            "perc_range_min": 0.0,
-            "perc_range_max": 100.0,
             "jpeg_quality": 95,
         },
     }
-    full_pipeline_list.append(method6)
+    full_pipeline_list.append(method7)
 
     return full_pipeline_list

tests/test_pipeline.py

@@ -454,14 +454,14 @@ def test_tomo_standard_testing_pipeline_output_with_save_all(
     subprocess.check_output(cmd)
 
     files = get_files("output_dir/")
-    assert len(files) == 10
+    assert len(files) == 11
 
     _check_yaml(files, "temp.yaml")
     _check_tif(files, 3, (160, 160))
 
     #: check the generated h5 files
     h5_files = list(filter(lambda x: ".h5" in x, files))
-    assert len(h5_files) == 4
+    assert len(h5_files) == 5
 
     for file_to_open in h5_files:
         if "tomopy-recon-tomo-gridrec.h5" in file_to_open:
@@ -488,7 +488,7 @@ def test_i12_testing_pipeline_output(
     subprocess.check_output(cmd)
 
     files = get_files("output_dir/")
-    assert len(files) == 17
+    assert len(files) == 18
 
     _check_yaml(files, "temp.yaml")
 
@@ -499,7 +499,7 @@ def test_i12_testing_pipeline_output(
     assert len(tif_files) == 10
 
     h5_files = list(filter(lambda x: ".h5" in x, files))
-    assert len(h5_files) == 4
+    assert len(h5_files) == 5
 
     gridrec_recon = list(filter(lambda x: "recon-gridrec.h5" in x, h5_files))[0]
     minus_log_tomo = list(filter(lambda x: "minus_log.h5" in x, h5_files))[0]
@@ -600,14 +600,14 @@ def test_diad_testing_pipeline_output(
     subprocess.check_output(cmd)
 
     files = get_files("output_dir/")
-    assert len(files) == 9
+    assert len(files) == 10
 
     _check_yaml(files, "temp.yaml")
     _check_tif(files, 2, (26, 26))
 
     #: check the generated h5 files
     h5_files = list(filter(lambda x: ".h5" in x, files))
-    assert len(h5_files) == 4
+    assert len(h5_files) == 5
 
     for file_to_open in h5_files:
         if "tomopy-normalize-tomo.h5" in file_to_open: