From 05295367e7014a52cae174308f8904177901688f Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Fri, 6 Dec 2024 16:29:42 +0000 Subject: [PATCH 01/14] Remove backends tests --- tests/test_backends/__init__.py | 0 tests/test_backends/test_httomolibgpu.py | 670 ----------------------- tests/test_backends/test_tomopy.py | 56 -- 3 files changed, 726 deletions(-) delete mode 100644 tests/test_backends/__init__.py delete mode 100644 tests/test_backends/test_httomolibgpu.py delete mode 100644 tests/test_backends/test_tomopy.py diff --git a/tests/test_backends/__init__.py b/tests/test_backends/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_backends/test_httomolibgpu.py b/tests/test_backends/test_httomolibgpu.py deleted file mode 100644 index 1e202c973..000000000 --- a/tests/test_backends/test_httomolibgpu.py +++ /dev/null @@ -1,670 +0,0 @@ -import dataclasses -import inspect -from typing import Literal -import pytest -from mpi4py import MPI -import numpy as np -from numpy import uint16, float32 -from unittest import mock - -from pytest_mock import MockerFixture -from numpy.testing import assert_allclose, assert_equal -import os - -from pytest_mock import MockerFixture - -cupy = pytest.importorskip("cupy") -httomolibgpu = pytest.importorskip("httomolibgpu") -import cupy as cp - -from httomo.methods_database.query import get_method_info -from httomo.runner.output_ref import OutputRef - - -from httomolibgpu.misc.morph import data_resampler, sino_360_to_180 -from httomolibgpu.prep.normalize import normalize -from httomolibgpu.prep.phase import paganin_filter_tomopy, paganin_filter_savu -from httomolibgpu.prep.alignment import distortion_correction_proj_discorpy -from httomolibgpu.prep.stripe import ( - remove_stripe_based_sorting, - remove_stripe_ti, - remove_all_stripe, - raven_filter, -) -from httomolibgpu.misc.corr import remove_outlier -from httomolibgpu.recon.algorithm import FBP, SIRT, CGLS -from httomolibgpu.misc.rescale import rescale_to_int - -from httomo.methods_database.packages.external.httomolibgpu.supporting_funcs.misc.morph import * -from httomo.methods_database.packages.external.httomolibgpu.supporting_funcs.prep.phase import * -from httomo.methods_database.packages.external.httomolibgpu.supporting_funcs.prep.stripe import * -from httomo.methods_database.packages.external.httomolibgpu.supporting_funcs.recon.algorithm import * -from httomo.methods_database.packages.external.httomolibgpu.supporting_funcs.misc.rescale import * -from httomo.methods_database.packages.external.httomolibgpu.supporting_funcs.prep.normalize import * - -from tests.testing_utils import make_test_method - - -module_mem_path = "httomo.methods_database.packages.external." - - -class MaxMemoryHook(cp.cuda.MemoryHook): - def __init__(self, initial=0): - self.max_mem = initial - self.current = initial - - def malloc_postprocess( - self, device_id: int, size: int, mem_size: int, mem_ptr: int, pmem_id: int - ): - self.current += mem_size - self.max_mem = max(self.max_mem, self.current) - - def free_postprocess( - self, device_id: int, mem_size: int, mem_ptr: int, pmem_id: int - ): - self.current -= mem_size - - def alloc_preprocess(self, **kwargs): - pass - - def alloc_postprocess(self, device_id: int, mem_size: int, mem_ptr: int): - pass - - def free_preprocess(self, **kwargs): - pass - - def malloc_preprocess(self, **kwargs): - pass - - -@pytest.mark.parametrize("dtype", ["uint16", "float32"]) -@pytest.mark.parametrize("slices", [50, 121]) -@pytest.mark.cupy -def test_normalize_memoryhook(flats, darks, ensure_clean_memory, dtype, slices): - hook = MaxMemoryHook() - data = cp.random.random_sample( - (slices, flats.shape[1], flats.shape[2]), dtype=np.float32 - ) - if dtype == "uint16": - darks = (darks * 1233).astype(np.uint16) - flats = flats.astype(np.uint16) - data = data.astype(np.uint16) - with hook: - normalize(cp.copy(data), flats, darks, minus_log=True).get() - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_normalize( - data.shape[1:], dtype=data.dtype - ) - - estimated_memory_mb = round(slices * estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 20 - - -@pytest.mark.parametrize("dtype", ["uint16"]) -@pytest.mark.parametrize("slices", [151, 321]) -@pytest.mark.cupy -def test_remove_outlier_memoryhook(flats, ensure_clean_memory, dtype, slices): - hook = MaxMemoryHook() - data = cp.random.random_sample( - (slices, flats.shape[1], flats.shape[2]), dtype=np.float32 - ) - if dtype == "uint16": - data = data.astype(np.uint16) - with hook: - remove_outlier(cp.copy(data)) - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - - # now we estimate how much of the total memory required for this data - library_info = get_method_info( - "httomolibgpu.misc.corr", "remove_outlier", "memory_gpu" - ) - estimated_memory_bytes = ( - library_info["multiplier"] * np.prod(cp.shape(data)) * uint16().nbytes - ) - - estimated_memory_mb = round(estimated_memory_bytes / (1024**2), 2) - - max_mem_mb = round(max_mem / (1024**2), 2) - # now compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 20 - - -@pytest.mark.cupy -@pytest.mark.parametrize("slices", [64, 128]) -@pytest.mark.parametrize("dim_x", [81, 260, 320]) -@pytest.mark.parametrize("dim_y", [340, 135, 96]) -def test_paganin_filter_tomopy_memoryhook(slices, dim_x, dim_y, ensure_clean_memory): - data = cp.random.random_sample((slices, dim_x, dim_y), dtype=np.float32) - hook = MaxMemoryHook() - with hook: - data_filtered = paganin_filter_tomopy(cp.copy(data)).get() - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_paganin_filter_tomopy( - (dim_x, dim_y), dtype=np.float32() - ) - estimated_memory_mb = round(slices * estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 20 - - -@pytest.mark.cupy -@pytest.mark.parametrize("slices", [64, 128]) -@pytest.mark.parametrize("dim_x", [81, 260, 320]) -@pytest.mark.parametrize("dim_y", [340, 135, 96]) -def test_paganin_filter_savu_memoryhook(slices, dim_x, dim_y, ensure_clean_memory): - data = cp.random.random_sample((slices, dim_x, dim_y), dtype=np.float32) - kwargs = {} - kwargs["ratio"] = 250.0 - kwargs["energy"] = 53.0 - kwargs["distance"] = 1.0 - kwargs["resolution"] = 1.28 - kwargs["pad_x"] = 20 - kwargs["pad_y"] = 20 - kwargs["pad_method"] = "edge" - kwargs["increment"] = 0.0 - hook = MaxMemoryHook() - with hook: - data_filtered = paganin_filter_savu(data, **kwargs).get() - - # The amount of bytes used by the method's processing according to the memory hook, plus - # the amount of bytes needed to hold the method's input in GPU memory - max_mem = hook.max_mem + data.nbytes - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_paganin_filter_savu( - (dim_x, dim_y), np.float32(), **kwargs - ) - estimated_memory_mb = round(slices * estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we compare both memory estimations - # - # make sure estimator function is within range (80% min, 100% max) - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 20 - - -@pytest.mark.cupy -@pytest.mark.parametrize("slices", [128, 190, 256]) -def test_distortion_correction_memoryhook( - slices, distortion_correction_path, ensure_clean_memory -): - data_size_dim = 320 - data = cp.random.random_sample( - (slices, data_size_dim, data_size_dim), dtype=np.float32 - ) - - distortion_coeffs_path = os.path.join( - distortion_correction_path, "distortion-coeffs.txt" - ) - shift_xy = [0, 0] - step_xy = [1, 1] - - hook = MaxMemoryHook() - with hook: - data_corrected = distortion_correction_proj_discorpy( - cp.copy(data), distortion_coeffs_path, shift_xy, step_xy - ).get() - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - max_mem_mb = round(max_mem / (1024**2), 2) # now in mbs - - # now we estimate how much of the total memory required for this data - library_info = get_method_info( - "httomolibgpu.prep.alignment", - "distortion_correction_proj_discorpy", - "memory_gpu", - ) - estimated_memory_bytes = ( - library_info["multiplier"] * np.prod(cp.shape(data)) * float32().nbytes - ) - estimated_memory_mb = round(estimated_memory_bytes / (1024**2), 2) - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 20 - - -@pytest.mark.cupy -@pytest.mark.parametrize("slices", [128, 256, 320]) -def test_remove_stripe_based_sorting_memoryhook( - slices, distortion_correction_path, ensure_clean_memory -): - data_size_dim = 300 - data = cp.random.random_sample( - (data_size_dim, slices, data_size_dim), dtype=np.float32 - ) - - hook = MaxMemoryHook() - with hook: - data_filtered = remove_stripe_based_sorting(cp.copy(data)).get() - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - max_mem_mb = round(max_mem / (1024**2), 2) # now in mbs - - # now we estimate how much of the total memory required for this data - library_info = get_method_info( - "httomolibgpu.prep.stripe", "remove_stripe_based_sorting", "memory_gpu" - ) - estimated_memory_bytes = ( - library_info["multiplier"] * np.prod(cp.shape(data)) * float32().nbytes - ) - estimated_memory_mb = round(estimated_memory_bytes / (1024**2), 2) - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 20 - - -@pytest.mark.cupy -@pytest.mark.parametrize("slices", [64, 129]) -def test_remove_stripe_ti_memoryhook(slices, ensure_clean_memory): - dim_x = 156 - dim_y = 216 - data = cp.random.random_sample((slices, dim_x, dim_y), dtype=np.float32) - hook = MaxMemoryHook() - with hook: - data_filtered = remove_stripe_ti(cp.copy(data)).get() - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_remove_stripe_ti( - (dim_x, dim_y), dtype=np.float32() - ) - estimated_memory_mb = round(slices * estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 20 - - -@pytest.mark.cupy -@pytest.mark.parametrize("projections", [180, 360, 720, 1080, 1440]) -def test_raven_filter_memoryhook(projections, ensure_clean_memory): - vert_det = 10 - horiz_det = 2560 - data = cp.random.random_sample((projections, vert_det, horiz_det), dtype=np.float32) - kwargs = {} - kwargs["pad_x"] = 20 - kwargs["pad_y"] = 20 - hook = MaxMemoryHook() - with hook: - data_filtered = raven_filter(cp.copy(data), **kwargs).get() - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_raven_filter( - (projections, horiz_det), dtype=np.float32(), **kwargs - ) - estimated_memory_mb = round(vert_det * estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 30 - - -@pytest.mark.cupy -@pytest.mark.parametrize("angles", [900, 1800]) -@pytest.mark.parametrize("dim_x_slices", [1, 3, 5]) -@pytest.mark.parametrize("dim_y", [1280, 2560]) -def test_remove_all_stripe_memoryhook(angles, dim_x_slices, dim_y, ensure_clean_memory): - data = cp.random.random_sample((angles, dim_x_slices, dim_y), dtype=np.float32) - hook = MaxMemoryHook() - with hook: - data_filtered = remove_all_stripe(cp.copy(data)).get() - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - max_mem_mb = round(max_mem / (1024**2), 2) # now in mbs - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_remove_all_stripe( - (angles, dim_y), dtype=np.float32() - ) - estimated_memory_mb = round(dim_x_slices * estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - assert estimated_memory_mb >= max_mem_mb - # this function is too complex to estimate the memory needed exactly, - # but it works in slice-by-slice fashion. - # We overestimate and ensure that we're always above the memoryhook limit. - - -@pytest.mark.cupy -@pytest.mark.parametrize("interpolation", ["nearest", "linear"]) -@pytest.mark.parametrize("slices", [1, 3, 10]) -@pytest.mark.parametrize("newshape", [[256, 256], [500, 500], [1000, 1000]]) -def test_data_sampler_memoryhook(slices, newshape, interpolation, ensure_clean_memory): - recon_size = 2560 - data = cp.random.random_sample((recon_size, slices, recon_size), dtype=cp.float32) - kwargs = {} - kwargs["newshape"] = newshape - kwargs["interpolation"] = interpolation - kwargs["axis"] = 1 - - hook = MaxMemoryHook() - with hook: - scaled_data = data_resampler(cp.copy(data), **kwargs) - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_data_resampler( - (recon_size, recon_size), dtype=np.float32(), **kwargs - ) - # as this is slice-by-slice implementation we should be adding slices number - estimated_memory_mb = slices * round(estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - assert estimated_memory_mb >= max_mem_mb - # for this function it is difficult to estimate the memory requirements as it works - # slice by slice. Also the memory usage inside interpn/RegularGridInterpolator is - # unknown. We should generally overestitmate the memory here. - - -@pytest.mark.cupy -@pytest.mark.parametrize("projections", [1801, 3601]) -@pytest.mark.parametrize("slices", [3, 5, 7, 11]) -@pytest.mark.parametrize("recon_size_it", [1200, 2560]) -def test_recon_FBP_memoryhook( - slices, recon_size_it, projections, ensure_clean_memory, mocker: MockerFixture -): - data = cp.random.random_sample((projections, slices, 2560), dtype=np.float32) - kwargs = {} - kwargs["angles"] = np.linspace( - 0.0 * np.pi / 180.0, 180.0 * np.pi / 180.0, data.shape[0] - ) - kwargs["center"] = 500 - kwargs["recon_size"] = recon_size_it - kwargs["recon_mask_radius"] = 0.8 - - hook = MaxMemoryHook() - p1 = mocker.patch( - "tomobar.astra_wrappers.astra_base.astra.data3d.delete", - side_effect=lambda id: hook.free_postprocess(0, data.nbytes, 0, 0), - ) - - with hook: - recon_data = FBP(cp.copy(data), **kwargs) - - p1.assert_called_once() - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_FBP( - (projections, 2560), dtype=np.float32(), **kwargs - ) - estimated_memory_mb = round(slices * estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert ( - percents_relative_maxmem <= 100 - ) # overestimation happens here because of the ASTRA's part - - -@pytest.mark.cupy -@pytest.mark.parametrize("slices", [3, 5]) -@pytest.mark.parametrize("recon_size_it", [1200, 2000]) -def test_recon_SIRT_memoryhook(slices, recon_size_it, ensure_clean_memory): - data = cp.random.random_sample((1801, slices, 2560), dtype=np.float32) - kwargs = {} - kwargs["recon_size"] = recon_size_it - - hook = MaxMemoryHook() - with hook: - recon_data = SIRT( - cp.copy(data), - np.linspace(0.0 * np.pi / 180.0, 180.0 * np.pi / 180.0, data.shape[0]), - 1200, - recon_size=recon_size_it, - iterations=2, - nonnegativity=True, - ) - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_SIRT( - (1801, 2560), dtype=np.float32(), **kwargs - ) - estimated_memory_mb = round(slices * estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 25 - - -@pytest.mark.cupy -@pytest.mark.parametrize("slices", [3, 5]) -@pytest.mark.parametrize("recon_size_it", [1200, 2000]) -def test_recon_CGLS_memoryhook(slices, recon_size_it, ensure_clean_memory): - data = cp.random.random_sample((1801, slices, 2560), dtype=np.float32) - kwargs = {} - kwargs["recon_size"] = recon_size_it - - hook = MaxMemoryHook() - with hook: - recon_data = CGLS( - cp.copy(data), - np.linspace(0.0 * np.pi / 180.0, 180.0 * np.pi / 180.0, data.shape[0]), - 1200, - recon_size=recon_size_it, - iterations=2, - nonnegativity=True, - ) - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_CGLS( - (1801, 2560), dtype=np.float32(), **kwargs - ) - estimated_memory_mb = round(slices * estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 20 - - -@pytest.mark.cupy -@pytest.mark.parametrize("bits", [8, 16, 32]) -@pytest.mark.parametrize("slices", [3, 5, 8]) -@pytest.mark.parametrize("glob_stats", [False, True]) -def test_rescale_to_int_memoryhook( - data, ensure_clean_memory, slices: int, bits: Literal[8, 16, 32], glob_stats: bool -): - data = cp.random.random_sample((1801, slices, 600), dtype=np.float32) - kwargs: dict = {} - kwargs["bits"] = bits - if glob_stats: - kwargs["glob_stats"] = (0.0, 10.0, 120.0, data.size) - hook = MaxMemoryHook() - with hook: - rescale_to_int(cp.copy(data), **kwargs).get() - - # make sure estimator function is within range (80% min, 100% max) - max_mem = ( - hook.max_mem - ) # the amount of memory in bytes needed for the method according to memoryhook - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we estimate how much of the total memory required for this data - (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_rescale_to_int( - (data.shape[0], data.shape[2]), dtype=np.float32(), **kwargs - ) - estimated_memory_mb = round(slices * estimated_memory_bytes / (1024**2), 2) - max_mem -= subtract_bytes - max_mem_mb = round(max_mem / (1024**2), 2) - - # now we compare both memory estimations - difference_mb = abs(estimated_memory_mb - max_mem_mb) - percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100) - # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb - # the resulting percent value should not deviate from max_mem on more than 20% - assert estimated_memory_mb >= max_mem_mb - assert percents_relative_maxmem <= 35 - - -@pytest.mark.cupy -@pytest.mark.parametrize("slices", [3, 8, 30, 50]) -@pytest.mark.parametrize("det_x", [600, 2160]) -def test_sino_360_to_180_memoryhook( - ensure_clean_memory, - mocker: MockerFixture, - det_x: int, - slices: int, -): - # Use a different overlap value for stitching based on the width of the 360 sinogram - overlap = 350 if det_x == 600 else 1950 - shape = (1801, slices, det_x) - data = cp.random.random_sample(shape, dtype=np.float32) - - # Run method to see actual memory usage - hook = MaxMemoryHook() - with hook: - sino_360_to_180(cp.copy(data), overlap) - - # Call memory estimator to estimate memory usage - output_ref = OutputRef( - method=make_test_method(mocker), - mapped_output_name="overlap", - ) - with mock.patch( - "httomo.runner.output_ref.OutputRef.value", - new_callable=mock.PropertyMock, - ) as mock_value_property: - mock_value_property.return_value = overlap - (estimated_bytes, subtract_bytes) = _calc_memory_bytes_sino_360_to_180( - non_slice_dims_shape=(shape[0], shape[2]), - dtype=np.float32(), - overlap=output_ref, - ) - estimated_bytes *= slices - - max_mem = hook.max_mem - subtract_bytes - - # For the difference between the actual memory usage and estimated memory usage, calculate - # that as a percentage of the actual memory used - difference = abs(estimated_bytes - max_mem) - percentage_difference = round((difference / max_mem) * 100) - - assert estimated_bytes >= max_mem - assert percentage_difference <= 35 diff --git a/tests/test_backends/test_tomopy.py b/tests/test_backends/test_tomopy.py deleted file mode 100644 index cab71a8e8..000000000 --- a/tests/test_backends/test_tomopy.py +++ /dev/null @@ -1,56 +0,0 @@ -from httomo.methods_database.packages.external.tomopy.supporting_funcs.misc.corr import ( - _calc_padding_median_filter3d, - _calc_padding_remove_outlier3d, -) -from httomo.methods_database.packages.external.tomopy.supporting_funcs.prep.stripe import ( - _calc_padding_stripes_detect3d, - _calc_padding_stripes_mask3d, -) - - -def test_calc_padding_median_filter3d() -> None: - kwargs: dict = {"size": 5, "ncore": 10} - assert _calc_padding_median_filter3d(**kwargs) == (2, 2) - - -def test_calc_padding_median_filter3d_defaults() -> None: - # it defaults to size=3 - kwargs: dict = {} - assert _calc_padding_median_filter3d(**kwargs) == (1, 1) - - -def test_calc_padding_remove_outlier3d() -> None: - kwargs: dict = {"size": 5, "ncore": 10} - assert _calc_padding_remove_outlier3d(**kwargs) == (2, 2) - - -def test_calc_padding_remove_outlier3d_defaults() -> None: - # it defaults to size=3 - kwargs: dict = {} - assert _calc_padding_remove_outlier3d(**kwargs) == (1, 1) - - -# TODO: unclear if the following function's padding behaviour is -# doing the right thing... - - -def test_calc_padding_stripes_detect3d() -> None: - kwargs: dict = {"size": 15, "radius": 2, "ncore": 10} - assert _calc_padding_stripes_detect3d(**kwargs) == (2, 2) - - -def test_calc_padding_stripes_detect3d_defaults() -> None: - # it defaults to radius=3 - kwargs: dict = {} - assert _calc_padding_stripes_detect3d(**kwargs) == (3, 3) - - -def test_calc_padding_stripes_mask3d() -> None: - kwargs: dict = {"threshold": 0.7, "min_stripe_depth": 5} - assert _calc_padding_stripes_mask3d(**kwargs) == (5, 5) - - -def test_calc_padding_stripes_mask3d_defaults() -> None: - # it defaults to min_stripe_depth=10 - kwargs: dict = {} - assert _calc_padding_stripes_mask3d(**kwargs) == (10, 10) From 4eef3f310baa151b23a8e7e24ad62648c5b18278 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Fri, 6 Dec 2024 16:32:46 +0000 Subject: [PATCH 02/14] Remove memory estimators and padding calculators --- .../httomolibgpu/supporting_funcs/__init__.py | 0 .../supporting_funcs/misc/__init__.py | 0 .../supporting_funcs/misc/corr.py | 17 -- .../supporting_funcs/misc/morph.py | 120 ----------- .../supporting_funcs/misc/rescale.py | 25 --- .../supporting_funcs/prep/__init__.py | 0 .../supporting_funcs/prep/normalize.py | 24 --- .../supporting_funcs/prep/phase.py | 169 ---------------- .../supporting_funcs/prep/stripe.py | 130 ------------ .../supporting_funcs/recon/__init__.py | 0 .../supporting_funcs/recon/algorithm.py | 186 ------------------ .../tomopy/supporting_funcs/__init__.py | 0 .../tomopy/supporting_funcs/misc/__init__.py | 0 .../tomopy/supporting_funcs/misc/corr.py | 36 ---- .../tomopy/supporting_funcs/prep/__init__.py | 0 .../tomopy/supporting_funcs/prep/stripe.py | 40 ---- .../tomopy/supporting_funcs/recon/__init__.py | 0 .../supporting_funcs/recon/algorithm.py | 36 ---- 18 files changed, 783 deletions(-) delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/__init__.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/__init__.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/corr.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/morph.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/rescale.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/__init__.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/normalize.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/stripe.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/recon/__init__.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/recon/algorithm.py delete mode 100644 httomo/methods_database/packages/external/tomopy/supporting_funcs/__init__.py delete mode 100644 httomo/methods_database/packages/external/tomopy/supporting_funcs/misc/__init__.py delete mode 100644 httomo/methods_database/packages/external/tomopy/supporting_funcs/misc/corr.py delete mode 100644 httomo/methods_database/packages/external/tomopy/supporting_funcs/prep/__init__.py delete mode 100644 httomo/methods_database/packages/external/tomopy/supporting_funcs/prep/stripe.py delete mode 100644 httomo/methods_database/packages/external/tomopy/supporting_funcs/recon/__init__.py delete mode 100644 httomo/methods_database/packages/external/tomopy/supporting_funcs/recon/algorithm.py diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/__init__.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/__init__.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/corr.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/corr.py deleted file mode 100644 index 904e0522f..000000000 --- a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/corr.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Tuple - - -__all__ = [ - "_calc_padding_remove_outlier", - "_calc_padding_median_filter", -] - - -def _calc_padding_remove_outlier(**kwargs) -> Tuple[int, int]: - kernel_size = kwargs["kernel_size"] - return (kernel_size // 2, kernel_size // 2) - - -def _calc_padding_median_filter(**kwargs) -> Tuple[int, int]: - kernel_size = kwargs["kernel_size"] - return (kernel_size // 2, kernel_size // 2) diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/morph.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/morph.py deleted file mode 100644 index 687ca6029..000000000 --- a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/morph.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------- -# Copyright 2022 Diamond Light Source Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- -# Created By : Tomography Team at DLS -# Created Date: 21 September 2023 -# --------------------------------------------------------------------------- -"""Modules for memory estimation for morph functions""" - -import math -from typing import Tuple -import numpy as np - -from httomo.runner.output_ref import OutputRef - -__all__ = [ - "_calc_memory_bytes_data_resampler", - "_calc_output_dim_data_resampler", - "_calc_memory_bytes_sino_360_to_180", - "_calc_output_dim_sino_360_to_180", -] - - -def _calc_output_dim_data_resampler(non_slice_dims_shape, **kwargs): - return kwargs["newshape"] - - -def _calc_memory_bytes_data_resampler( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - newshape = kwargs["newshape"] - interpolation = kwargs["interpolation"] - - input_size = np.prod(non_slice_dims_shape) * dtype.itemsize - xi = 2 * np.prod(newshape) * dtype.itemsize - output_size = np.prod(newshape) * dtype.itemsize - - # interpolation happens in 2d so we should allocate for it, the exact value is unknown - if interpolation == "nearest": - interpolator = 3 * (input_size + output_size) - if interpolation == "linear": - interpolator = 4 * (input_size + output_size) - - tot_memory_bytes = input_size + output_size + interpolator - return (tot_memory_bytes, xi) - - -def _calc_output_dim_sino_360_to_180( - non_slice_dims_shape: Tuple[int, int], - **kwargs, -) -> Tuple[int, int]: - assert "overlap" in kwargs, "Expected overlap in method parameters" - overlap_side_output = kwargs["overlap"] - assert isinstance( - overlap_side_output, OutputRef - ), "Expected overlap to be in an OutputRef" - overlap: float = overlap_side_output.value - - original_sino_width = non_slice_dims_shape[1] - stitched_sino_width = original_sino_width * 2 - math.ceil(overlap) - return non_slice_dims_shape[0] // 2, stitched_sino_width - - -def _calc_memory_bytes_sino_360_to_180( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - assert "overlap" in kwargs, "Expected overlap in method parameters" - overlap_side_output = kwargs["overlap"] - assert isinstance( - overlap_side_output, OutputRef - ), "Expected overlap to be in an OutputRef" - overlap: float = overlap_side_output.value - - original_sino_width = non_slice_dims_shape[1] - stitched_sino_width = original_sino_width * 2 - math.ceil(overlap) - n = non_slice_dims_shape[0] // 2 - stitched_non_slice_dims = (n, stitched_sino_width) - - input_slice_size = int(np.prod(non_slice_dims_shape)) * dtype.itemsize - output_slice_size = int(np.prod(stitched_non_slice_dims)) * dtype.itemsize - - summand_shape: Tuple[int, int] = (n, int(overlap)) - # Multiplication between a subset of the original data (`float32`) and the 1D weights array - # (`float32`) causes a new array to be created that has dtype `float32` (for example, - # multiplications like `weights * data[:n, :, -overlap]` - summand_size = int(np.prod(summand_shape)) * np.float32().itemsize - - total_memory_bytes = ( - input_slice_size - + output_slice_size - # In both the `if` branch and the `else` branch checking the `rotation` variable value, - # in total, there are 4 copies of subsets of the `data` array that are made (note that - # the expressions below are referencing the `if` branch and are slightly different in - # the `else` branch): - # 1. fancy indexing: `data[n : 2 * n, :, overlap:][:, :, ::-1]` - # 2. multiplication: `weights * data[:n, :, :overlap]` - # 3. multiplication: `weights * data[n : 2 * n, :, :overlap]` - # 4. fancy indexing (performed on the result of 3): - # `(weights * data[n : 2 * n, :, :overlap])[:, :, ::-1]` - + 4 * summand_size - ) - - return total_memory_bytes, 0 diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/rescale.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/rescale.py deleted file mode 100644 index 2e0ef590e..000000000 --- a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/misc/rescale.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Tuple -import numpy as np - -__all__ = [ - "_calc_memory_bytes_rescale_to_int", -] - - -def _calc_memory_bytes_rescale_to_int( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - bits: int = kwargs["bits"] - if bits == 8: - itemsize = 1 - elif bits == 16: - itemsize = 2 - else: - itemsize = 4 - safety = 128 - return ( - int(np.prod(non_slice_dims_shape)) * (dtype.itemsize + itemsize) + safety, - 0, - ) diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/__init__.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/normalize.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/normalize.py deleted file mode 100644 index 6b32d8c9b..000000000 --- a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/normalize.py +++ /dev/null @@ -1,24 +0,0 @@ -from typing import Tuple -import numpy as np - - -__all__ = [ - "_calc_memory_bytes_normalize", -] - - -def _calc_memory_bytes_normalize( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - # this function changes the data type - in_slice_mem = np.prod(non_slice_dims_shape) * dtype.itemsize - out_slice_mem = np.prod(non_slice_dims_shape) * np.float32().itemsize - - # fixed cost for keeping mean of flats and darks - mean_mem = int(np.prod(non_slice_dims_shape) * np.float32().itemsize * 2) - - tot_memory_bytes = int(in_slice_mem + out_slice_mem) - - return (tot_memory_bytes, mean_mem) diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py deleted file mode 100644 index aa837f55b..000000000 --- a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/phase.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------- -# Copyright 2022 Diamond Light Source Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- -# Created By : Tomography Team at DLS -# Created Date: 21 September 2023 -# --------------------------------------------------------------------------- -"""Modules for memory estimation for phase retrieval and phase-contrast enhancement""" - -import math -from typing import Tuple -import numpy as np - -from httomo.cufft import CufftType, cufft_estimate_2d - -__all__ = [ - "_calc_memory_bytes_paganin_filter_savu", - "_calc_memory_bytes_paganin_filter_tomopy", -] - - -def _calc_memory_bytes_paganin_filter_savu( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - pad_x = kwargs["pad_x"] - pad_y = kwargs["pad_y"] - - # Input (unpadded) - unpadded_in_slice_size = np.prod(non_slice_dims_shape) * dtype.itemsize - - # Padded input - padded_non_slice_dims_shape = ( - non_slice_dims_shape[0] + 2 * pad_y, - non_slice_dims_shape[1] + 2 * pad_x, - ) - padded_in_slice_size = ( - padded_non_slice_dims_shape[0] * padded_non_slice_dims_shape[1] * dtype.itemsize - ) - - # Padded input cast to `complex64` - complex_slice = padded_in_slice_size / dtype.itemsize * np.complex64().nbytes - - # Plan size for 2D FFT - fftplan_slice_size = cufft_estimate_2d( - nx=padded_non_slice_dims_shape[1], - ny=padded_non_slice_dims_shape[0], - fft_type=CufftType.CUFFT_C2C, - ) - - # Shape of 2D filter is the same as the padded `complex64` slice shape, so the size will be - # the same - filter_size = complex_slice - - # Size of cropped/unpadded + cast to float32 result of 2D IFFT - cropped_float32_res_slice = np.prod(non_slice_dims_shape) * np.float32().nbytes - - # If the FFT plan size is negligible for some reason, this changes where the peak GPU - # memory usage occurs. Hence, the if/else branching below for calculating the total bytes. - NEGLIGIBLE_FFT_PLAN_SIZE = 16 - if fftplan_slice_size < NEGLIGIBLE_FFT_PLAN_SIZE: - tot_memory_bytes = int( - unpadded_in_slice_size + padded_in_slice_size + complex_slice - ) - else: - tot_memory_bytes = int( - unpadded_in_slice_size - + padded_in_slice_size - + complex_slice - # The padded float32 array is deallocated when a copy is made when casting to complex64 - # and the variable `padded_tomo` is reassigned to the complex64 version - - padded_in_slice_size - + fftplan_slice_size - + cropped_float32_res_slice - ) - - return (tot_memory_bytes, filter_size) - - -def _calc_memory_bytes_paganin_filter_tomopy( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - from httomolibgpu.prep.phase import _shift_bit_length - - # Input (unpadded) - unpadded_in_slice_size = np.prod(non_slice_dims_shape) * dtype.itemsize - - # estimate padding size here based on non_slice dimensions - pad_tup = [] - for dim_len in non_slice_dims_shape: - diff = _shift_bit_length(dim_len + 1) - dim_len - if dim_len % 2 == 0: - pad_width = diff // 2 - pad_width = (pad_width, pad_width) - else: - # need an uneven padding for odd-number lengths - left_pad = diff // 2 - right_pad = diff - left_pad - pad_width = (left_pad, right_pad) - pad_tup.append(pad_width) - - # Padded input - padded_in_slice_size = ( - (non_slice_dims_shape[0] + pad_tup[0][0] + pad_tup[0][1]) - * (non_slice_dims_shape[1] + pad_tup[1][0] + pad_tup[1][1]) - * dtype.itemsize - ) - - # Padded input cast to `complex64` - complex_slice = padded_in_slice_size / dtype.itemsize * np.complex64().nbytes - - # Plan size for 2D FFT - ny = non_slice_dims_shape[0] + pad_tup[0][0] + pad_tup[0][1] - nx = non_slice_dims_shape[1] + pad_tup[1][0] + pad_tup[1][1] - fftplan_slice_size = cufft_estimate_2d( - nx=nx, - ny=ny, - fft_type=CufftType.CUFFT_C2C, - ) - - # Size of "reciprocal grid" generated, based on padded projections shape - grid_size = np.prod((ny, nx)) * np.float32().nbytes - filter_size = grid_size - - # Size of cropped/unpadded + cast to float32 result of 2D IFFT - cropped_float32_res_slice = np.prod(non_slice_dims_shape) * np.float32().nbytes - - # Size of negative log of cropped float32 result of 2D IFFT - negative_log_slice = cropped_float32_res_slice - - # If the FFT plan size is negligible for some reason, this changes where the peak GPU - # memory usage occurs. Hence, the if/else branching below for calculating the total bytes. - NEGLIGIBLE_FFT_PLAN_SIZE = 16 - if fftplan_slice_size < NEGLIGIBLE_FFT_PLAN_SIZE: - tot_memory_bytes = int( - unpadded_in_slice_size + padded_in_slice_size + complex_slice - ) - else: - tot_memory_bytes = int( - unpadded_in_slice_size - + padded_in_slice_size - + complex_slice - # The padded float32 array is deallocated when a copy is made when casting to complex64 - # and the variable `padded_tomo` is reassigned to the complex64 version - - padded_in_slice_size - + fftplan_slice_size - + cropped_float32_res_slice - + negative_log_slice - ) - - subtract_bytes = int(filter_size + grid_size) - - return (tot_memory_bytes, subtract_bytes) diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/stripe.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/stripe.py deleted file mode 100644 index d4af718ba..000000000 --- a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/prep/stripe.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------- -# Copyright 2022 Diamond Light Source Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- -# Created By : Tomography Team at DLS -# Created Date: 21 September 2023 -# --------------------------------------------------------------------------- -"""Modules for memory estimation for stripe removal methods""" - -import math -from typing import Tuple -import numpy as np - -from httomo.cufft import CufftType, cufft_estimate_1d - - -__all__ = [ - "_calc_memory_bytes_remove_stripe_ti", - "_calc_memory_bytes_remove_all_stripe", - "_calc_memory_bytes_raven_filter", -] - - -def _calc_memory_bytes_remove_stripe_ti( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - # This is admittedly a rough estimation, but it should be about right - gamma_mem = non_slice_dims_shape[1] * np.float64().itemsize - - in_slice_mem = np.prod(non_slice_dims_shape) * dtype.itemsize - slice_mean_mem = non_slice_dims_shape[1] * dtype.itemsize * 2 - slice_fft_plan_mem = slice_mean_mem * 3.5 - extra_temp_mem = slice_mean_mem * 8 - - tot_memory_bytes = int( - in_slice_mem + slice_mean_mem + slice_fft_plan_mem + extra_temp_mem - ) - return (tot_memory_bytes, gamma_mem) - - -def _calc_memory_bytes_remove_all_stripe( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - # Extremely memory hungry function but it works slice-by-slice so - # we need to compensate for that. - - input_size = np.prod(non_slice_dims_shape) * dtype.itemsize - output_size = np.prod(non_slice_dims_shape) * dtype.itemsize - - methods_memory_allocations = int(30 * input_size) - - tot_memory_bytes = int(input_size + output_size) - - return (tot_memory_bytes, methods_memory_allocations) - - -def _calc_memory_bytes_raven_filter( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - - pad_x = kwargs["pad_x"] - pad_y = kwargs["pad_y"] - - # Unpadded input - input_size = np.prod(non_slice_dims_shape) * dtype.itemsize - - # Padded input - padded_non_slice_dims_shape = ( - non_slice_dims_shape[0] + 2 * pad_y, - non_slice_dims_shape[1] + 2 * pad_x, - ) - in_slice_size_pad = ( - (padded_non_slice_dims_shape[0]) - * (padded_non_slice_dims_shape[1]) - * dtype.itemsize - ) - - # Conversion of padded input data to `complex64` (implicitly done by `fft2()` function) - complex_slice_fft_data = in_slice_size_pad / dtype.itemsize * np.complex64().nbytes - - # 2D FFT becomes two 1D FFTs (possibly due to applying 2D FFT to non-adjacent axes 0 and - # 2), so a plan for a 1D FFT is needed rather than a plan for a 2D FFT - fft_1d_plan = cufft_estimate_1d( - nx=padded_non_slice_dims_shape[0], - fft_type=CufftType.CUFFT_C2C, - batch=non_slice_dims_shape[1], - ) - - # Copy from applying fftshift to FFT result - complex_slice_fft_data_shifted = complex_slice_fft_data - - # Two copies of `complex64` data come from 2D IFFT becoming a loop over two 1D IFFTs, and - # applying 1D IFFT to non-adjacent axes 0 and 2 causes data to not be C contiguous (thus, - # needing to be copied to get a version of the data which is C contiguous) - # - # NOTE: The same copies are generated by the 2D FFT becoming two 1D FFTs, but the order of - # allocations and deallocations of those copies are such that they don't contribute to peak - # GPU memory usage. Thus, they aren't accounted for in the estimated memory, unlike the - # copies generated by the 2D IFFT becoming two 1D IFFTs - ifft_complex64_copies = 2 * complex_slice_fft_data - - tot_memory_bytes = int( - input_size - + in_slice_size_pad - + complex_slice_fft_data - + 2 * fft_1d_plan - + complex_slice_fft_data_shifted - + ifft_complex64_copies - ) - - return (tot_memory_bytes, 0) diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/recon/__init__.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/recon/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/recon/algorithm.py b/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/recon/algorithm.py deleted file mode 100644 index ec7d8db78..000000000 --- a/httomo/methods_database/packages/external/httomolibgpu/supporting_funcs/recon/algorithm.py +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------- -# Copyright 2022 Diamond Light Source Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- -# Created By : Tomography Team at DLS -# Created Date: 21 September 2023 -# --------------------------------------------------------------------------- -"""Modules for memory estimation for reconstruction algorithms""" - -import math -from typing import Tuple -import numpy as np -from httomo.cufft import CufftType, cufft_estimate_1d - -__all__ = [ - "_calc_memory_bytes_FBP", - "_calc_memory_bytes_SIRT", - "_calc_memory_bytes_CGLS", - "_calc_output_dim_FBP", - "_calc_output_dim_SIRT", - "_calc_output_dim_CGLS", -] - - -def __calc_output_dim_recon(non_slice_dims_shape, **kwargs): - """Function to calculate output dimensions for all reconstructors. - The change of the dimension depends either on the user-provided "recon_size" - parameter or taken as the size of the horizontal detector (default). - - """ - DetectorsLengthH = non_slice_dims_shape[1] - recon_size = kwargs["recon_size"] - if recon_size is None: - recon_size = DetectorsLengthH - output_dims = (recon_size, recon_size) - return output_dims - - -def _calc_output_dim_FBP(non_slice_dims_shape, **kwargs): - return __calc_output_dim_recon(non_slice_dims_shape, **kwargs) - - -def _calc_output_dim_SIRT(non_slice_dims_shape, **kwargs): - return __calc_output_dim_recon(non_slice_dims_shape, **kwargs) - - -def _calc_output_dim_CGLS(non_slice_dims_shape, **kwargs): - return __calc_output_dim_recon(non_slice_dims_shape, **kwargs) - - -def _calc_memory_bytes_FBP( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - det_height = non_slice_dims_shape[0] - det_width = non_slice_dims_shape[1] - SLICES = 200 # dummy multiplier+divisor to pass large batch size threshold - - # 1. input - input_slice_size = np.prod(non_slice_dims_shape) * dtype.itemsize - - ########## FFT / filter / IFFT (filtersync_cupy) - - # 2. RFFT plan (R2C transform) - fftplan_slice_size = ( - cufft_estimate_1d( - nx=det_width, - fft_type=CufftType.CUFFT_R2C, - batch=det_height * SLICES, - ) - / SLICES - ) - - # 3. RFFT output size (proj_f in code) - proj_f_slice = det_height * (det_width // 2 + 1) * np.complex64().itemsize - - # 4. Filter size (independent of number of slices) - filter_size = (det_width // 2 + 1) * np.float32().itemsize - - # 5. IRFFT plan size - ifftplan_slice_size = ( - cufft_estimate_1d( - nx=det_width, - fft_type=CufftType.CUFFT_C2R, - batch=det_height * SLICES, - ) - / SLICES - ) - - # 6. output of filtersync call - filtersync_output_slice_size = input_slice_size - - # since the FFT plans, proj_f, and input data is dropped after the filtersync call, we track it here - # separate - filtersync_size = ( - input_slice_size + fftplan_slice_size + proj_f_slice + ifftplan_slice_size - ) - - # 6. we swap the axes before passing data to Astra in ToMoBAR - # https://github.com/dkazanc/ToMoBAR/blob/54137829b6326406e09f6ef9c95eb35c213838a7/tomobar/methodsDIR_CuPy.py#L135 - pre_astra_input_swapaxis_slice = ( - np.prod(non_slice_dims_shape) * np.float32().itemsize - ) - - # 7. astra backprojection will generate an output array - # https://github.com/dkazanc/ToMoBAR/blob/54137829b6326406e09f6ef9c95eb35c213838a7/tomobar/astra_wrappers/astra_base.py#L524 - output_dims = _calc_output_dim_FBP(non_slice_dims_shape, **kwargs) - recon_output_size = np.prod(output_dims) * np.float32().itemsize - - # 7. astra backprojection makes a copy of the input - astra_input_slice_size = np.prod(non_slice_dims_shape) * np.float32().itemsize - - ## now we calculate back projection memory (2 copies of the input + reconstruction output) - projection_mem_size = ( - pre_astra_input_swapaxis_slice + astra_input_slice_size + recon_output_size - ) - - # 9. apply_circular_mask memory (fixed amount, not per slice) - circular_mask_size = np.prod(output_dims) * np.int64().itemsize - - fixed_amount = max(filter_size, circular_mask_size) - - # 9. this swapaxis makes another copy of the output data - # https://github.com/DiamondLightSource/httomolibgpu/blob/72d98ec7ac44e06ee0318043934fb3f68667d203/httomolibgpu/recon/algorithm.py#L118 - # BUT: this swapaxis happens after the cudaArray inputs and the input swapaxis arrays are dropped, - # so it does not add to the memory overall - - if projection_mem_size > filtersync_size: - tot_memory_bytes = int(filtersync_output_slice_size + projection_mem_size) - else: - # here we do not add recon_output_size as we assume that at least one fft plan will be released before the - # the backprojection step which is SMALLER than the current estimation. - tot_memory_bytes = int(filtersync_output_slice_size + filtersync_size) - - # this account for the memory used for filtration AND backprojection. - return (tot_memory_bytes, fixed_amount) - - -def _calc_memory_bytes_SIRT( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - DetectorsLengthH = non_slice_dims_shape[1] - # calculate the output shape - output_dims = _calc_output_dim_SIRT(non_slice_dims_shape, **kwargs) - - in_data_size = np.prod(non_slice_dims_shape) * dtype.itemsize - out_data_size = np.prod(output_dims) * dtype.itemsize - - astra_projection = 2.5 * (in_data_size + out_data_size) - - tot_memory_bytes = 2 * in_data_size + 2 * out_data_size + astra_projection - return (tot_memory_bytes, 0) - - -def _calc_memory_bytes_CGLS( - non_slice_dims_shape: Tuple[int, int], - dtype: np.dtype, - **kwargs, -) -> Tuple[int, int]: - DetectorsLengthH = non_slice_dims_shape[1] - # calculate the output shape - output_dims = _calc_output_dim_CGLS(non_slice_dims_shape, **kwargs) - - in_data_size = np.prod(non_slice_dims_shape) * dtype.itemsize - out_data_size = np.prod(output_dims) * dtype.itemsize - - astra_projection = 2.5 * (in_data_size + out_data_size) - - tot_memory_bytes = 2 * in_data_size + 2 * out_data_size + astra_projection - return (tot_memory_bytes, 0) diff --git a/httomo/methods_database/packages/external/tomopy/supporting_funcs/__init__.py b/httomo/methods_database/packages/external/tomopy/supporting_funcs/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/tomopy/supporting_funcs/misc/__init__.py b/httomo/methods_database/packages/external/tomopy/supporting_funcs/misc/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/tomopy/supporting_funcs/misc/corr.py b/httomo/methods_database/packages/external/tomopy/supporting_funcs/misc/corr.py deleted file mode 100644 index 4d5dda20e..000000000 --- a/httomo/methods_database/packages/external/tomopy/supporting_funcs/misc/corr.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------- -# Copyright 2022 Diamond Light Source Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- -# Created By : Tomography Team at DLS -# Created Date: 18/July/2024 -# --------------------------------------------------------------------------- -""" Modules for memory estimation, padding, data dims estimation """ - -from typing import Tuple - -__all__ = [ - "_calc_padding_median_filter3d", - "_calc_padding_remove_outlier3d", -] - - -def _calc_padding_median_filter3d(size: int = 3, **kwargs) -> Tuple[int, int]: - return size // 2, size // 2 - - -def _calc_padding_remove_outlier3d(size: int = 3, **kwargs) -> Tuple[int, int]: - return size // 2, size // 2 diff --git a/httomo/methods_database/packages/external/tomopy/supporting_funcs/prep/__init__.py b/httomo/methods_database/packages/external/tomopy/supporting_funcs/prep/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/tomopy/supporting_funcs/prep/stripe.py b/httomo/methods_database/packages/external/tomopy/supporting_funcs/prep/stripe.py deleted file mode 100644 index 9433a4201..000000000 --- a/httomo/methods_database/packages/external/tomopy/supporting_funcs/prep/stripe.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------- -# Copyright 2022 Diamond Light Source Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- -# Created By : Tomography Team at DLS -# Created Date: 18/July/2024 -# --------------------------------------------------------------------------- -""" Modules for memory estimation, padding, data dims estimation """ - -from typing import Tuple - -__all__ = [ - "_calc_padding_stripes_detect3d", - "_calc_padding_stripes_mask3d", -] - - -def _calc_padding_stripes_detect3d(radius=3, **kwargs) -> Tuple[int, int]: - # TODO: confirm if this correponds to the padding here: - # https://github.com/tomopy/tomopy/blob/0c6d18da8a5b8fddde1a3e0d8864f5b3fefff8ae/source/tomopy/prep/stripe.py#L988C5-L988C19 - return radius, radius - - -def _calc_padding_stripes_mask3d(min_stripe_depth=10, **kwargs) -> Tuple[int, int]: - # TODO: confirm if this correponds to the padding here: - # https://github.com/tomopy/tomopy/blob/master/source/tomopy/prep/stripe.py#L1062 - return min_stripe_depth, min_stripe_depth diff --git a/httomo/methods_database/packages/external/tomopy/supporting_funcs/recon/__init__.py b/httomo/methods_database/packages/external/tomopy/supporting_funcs/recon/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/tomopy/supporting_funcs/recon/algorithm.py b/httomo/methods_database/packages/external/tomopy/supporting_funcs/recon/algorithm.py deleted file mode 100644 index 6908d1cbf..000000000 --- a/httomo/methods_database/packages/external/tomopy/supporting_funcs/recon/algorithm.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# --------------------------------------------------------------------------- -# Copyright 2022 Diamond Light Source Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- -# Created By : Tomography Team at DLS -# Created Date: 21 September 2023 -# --------------------------------------------------------------------------- -"""Supporting modules for reconstruction algorithms""" - -from typing import Tuple -import numpy as np - - -__all__ = [ - "_calc_output_dim_recon", -] - - -def _calc_output_dim_recon(non_slice_dims_shape, **kwargs): - """Function to calculate output dimensions for all reconstructors.""" - DetectorsLengthH = non_slice_dims_shape[1] - output_dims = (DetectorsLengthH, DetectorsLengthH) - return output_dims From 13f6bd08694119dabf0f790d4f858cb658663f9a Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Fri, 6 Dec 2024 16:39:58 +0000 Subject: [PATCH 03/14] Remove backends methods database files --- .../packages/external/__init__.py | 0 .../packages/external/httomolib/__init__.py | 0 .../external/httomolib/httomolib.yaml | 34 -- .../external/httomolib/httomolib_modules.yaml | 4 - .../external/httomolibgpu/__init__.py | 0 .../external/httomolibgpu/httomolibgpu.yaml | 184 ------- .../httomolibgpu/httomolibgpu_modules.yaml | 10 - .../packages/external/tomopy/__init__.py | 0 .../packages/external/tomopy/tomopy.yaml | 454 ------------------ .../external/tomopy/tomopy_modules.yaml | 10 - 10 files changed, 696 deletions(-) delete mode 100644 httomo/methods_database/packages/external/__init__.py delete mode 100644 httomo/methods_database/packages/external/httomolib/__init__.py delete mode 100644 httomo/methods_database/packages/external/httomolib/httomolib.yaml delete mode 100644 httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/__init__.py delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/httomolibgpu.yaml delete mode 100644 httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml delete mode 100644 httomo/methods_database/packages/external/tomopy/__init__.py delete mode 100644 httomo/methods_database/packages/external/tomopy/tomopy.yaml delete mode 100644 httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml diff --git a/httomo/methods_database/packages/external/__init__.py b/httomo/methods_database/packages/external/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/httomolib/__init__.py b/httomo/methods_database/packages/external/httomolib/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/httomolib/httomolib.yaml b/httomo/methods_database/packages/external/httomolib/httomolib.yaml deleted file mode 100644 index a0e8cf639..000000000 --- a/httomo/methods_database/packages/external/httomolib/httomolib.yaml +++ /dev/null @@ -1,34 +0,0 @@ -misc: - morph: - data_reducer: - pattern: all - output_dims_change: True - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - images: - save_to_images: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - segm: - binary_thresholding: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False -prep: - phase: - paganin_filter: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False diff --git a/httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml b/httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml deleted file mode 100644 index 6b69f7d0f..000000000 --- a/httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- httomolib.misc.morph -- httomolib.misc.images -- httomolib.misc.segm -- httomolib.prep.phase diff --git a/httomo/methods_database/packages/external/httomolibgpu/__init__.py b/httomo/methods_database/packages/external/httomolibgpu/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/httomolibgpu/httomolibgpu.yaml b/httomo/methods_database/packages/external/httomolibgpu/httomolibgpu.yaml deleted file mode 100644 index 5b919becd..000000000 --- a/httomo/methods_database/packages/external/httomolibgpu/httomolibgpu.yaml +++ /dev/null @@ -1,184 +0,0 @@ -misc: - corr: - median_filter: - pattern: all - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: True - memory_gpu: - multiplier: 2.1 - method: direct - remove_outlier: - pattern: all - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: True - memory_gpu: - multiplier: 2.1 - method: direct - morph: - sino_360_to_180: - pattern: sinogram - output_dims_change: True - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: None - method: module - data_resampler: - pattern: all - output_dims_change: True - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: None - method: module - rescale: - rescale_to_int: - pattern: all - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: None - method: module -prep: - normalize: - normalize: - pattern: projection - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: None - method: module - phase: - paganin_filter_tomopy: - pattern: projection - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: None - method: module - paganin_filter_savu: - pattern: projection - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: None - method: module - alignment: - distortion_correction_proj_discorpy: - pattern: projection - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: 1.2 - method: direct - stripe: - remove_stripe_based_sorting: - pattern: sinogram - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: 1.17 - method: direct - remove_stripe_ti: - pattern: sinogram - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: None - method: module - remove_all_stripe: - pattern: sinogram - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: None - method: module - raven_filter: - pattern: sinogram - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - - datasets: [tomo] - - multipliers: [None] - - methods: [module] -recon: - algorithm: - FBP: - pattern: sinogram - output_dims_change: True - implementation: gpu_cupy - save_result_default: True - padding: False - memory_gpu: - multiplier: None - method: module - SIRT: - pattern: sinogram - output_dims_change: True - implementation: gpu_cupy - save_result_default: True - padding: False - memory_gpu: - multiplier: None - method: module - CGLS: - pattern: sinogram - output_dims_change: True - implementation: gpu_cupy - save_result_default: True - padding: False - memory_gpu: - multiplier: None - method: module - rotation: - find_center_vo: - pattern: projection - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: 0 - method: direct - find_center_360: - pattern: projection - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: 0 - method: direct - find_center_pc: - pattern: projection - output_dims_change: False - implementation: gpu_cupy - save_result_default: False - padding: False - memory_gpu: - multiplier: 0 - method: direct diff --git a/httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml b/httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml deleted file mode 100644 index 2722392e7..000000000 --- a/httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- httomolibgpu.misc.corr -- httomolibgpu.misc.morph -- httomolibgpu.misc.rescale -- httomolibgpu.prep.alignment -- httomolibgpu.prep.normalize -- httomolibgpu.prep.phase -- httomolibgpu.prep.stripe -- httomolibgpu.recon.algorithm -- httomolibgpu.recon.rotation - diff --git a/httomo/methods_database/packages/external/tomopy/__init__.py b/httomo/methods_database/packages/external/tomopy/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/external/tomopy/tomopy.yaml b/httomo/methods_database/packages/external/tomopy/tomopy.yaml deleted file mode 100644 index 20ac7b2cd..000000000 --- a/httomo/methods_database/packages/external/tomopy/tomopy.yaml +++ /dev/null @@ -1,454 +0,0 @@ -sim: - project: - add_drift: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - add_focal_spot_blur: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - add_gaussian: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - add_poisson: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - add_rings: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - add_salt_pepper: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - add_zingers: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - project: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - project2: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - project3: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False -misc: - corr: - adjust_range: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - circ_mask: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - gaussian_filter: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - median_filter: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - median_filter3d: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: True - median_filter_nonfinite: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_nan: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_neg: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_outlier: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_outlier1d: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_outlier3d: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: True - remove_ring: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - sobel_filter: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - morph: - downsample: - pattern: all - output_dims_change: True - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - pad: - pattern: all - output_dims_change: True - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - sino_360_to_180: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - trim_sinogram: - pattern: sinogram - output_dims_change: True - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - upsample: - pattern: all - output_dims_change: True - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False -prep: - alignment: - add_jitter: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - add_noise: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - align_joint: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - align_seq: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - blur_edges: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - distortion_correction_proj: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - distortion_correction_sino: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - scale: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - shift_images: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - normalize: - minus_log: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - normalize: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - normalize_roi: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - normalize_bg: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - normalize_nf: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - phase: - retrieve_phase: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - stripe: - remove_all_stripe: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_stripe_fw: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_stripe_ti: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_stripe_sf: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_stripe_based_sorting: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_stripe_based_filtering: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_stripe_based_fitting: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_large_stripe: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_dead_stripe: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - remove_stripe_based_interpolation: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - stripes_detect3d: - pattern: sinogram - implementation: cpu - output_dims_change: False - memory_gpu: None - save_result_default: False - padding: True - stripes_mask3d: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: True -recon: - algorithm: - recon: - pattern: sinogram - output_dims_change: True - implementation: cpu - memory_gpu: None - save_result_default: True - padding: False - rotation: - find_center: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - find_center_pc: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: True - padding: False - find_center_vo: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - wrappers: - astra: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - ufo_fbp: - pattern: sinogram - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - ufo_dfi: - pattern: sinogram - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - lprec: - pattern: sinogram - output_dims_change: True - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False diff --git a/httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml b/httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml deleted file mode 100644 index 23dcb77c6..000000000 --- a/httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- tomopy.misc.corr -- tomopy.misc.morph -- tomopy.prep.alignment -- tomopy.prep.normalize -- tomopy.prep.phase -- tomopy.prep.stripe -- tomopy.recon.algorithm -- tomopy.recon.rotation -- tomopy.sim.project - From 043ff0aa8d10aeab958f64125627b3588757eb81 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Fri, 6 Dec 2024 17:33:30 +0000 Subject: [PATCH 04/14] Use pattern enum from `httomo-backends` --- httomo/utils.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/httomo/utils.py b/httomo/utils.py index fd20bf552..3972e31d3 100644 --- a/httomo/utils.py +++ b/httomo/utils.py @@ -7,6 +7,8 @@ from mpi4py import MPI import numpy as np +from httomo_backends.methods_database.query import Pattern + gpu_enabled = False try: import cupy as xp @@ -179,16 +181,6 @@ def _parse_preview( return preview_str -class Pattern(Enum): - """Enum for the different slicing-orientations/"patterns" that tomographic - data can have. - """ - - projection = 0 - sinogram = 1 - all = 2 - - def _get_slicing_dim(pattern: Pattern) -> Literal[1, 2]: """Assuming 3D projection data with the axes ordered as - axis 0 = rotation angle From 504f735a786a05f23f37c9e8587b39dd8a0465f3 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Tue, 10 Dec 2024 10:45:17 +0000 Subject: [PATCH 05/14] Import `Pattern` explicitly from `httomo-backends` everywhere --- httomo/loaders/standard_tomo_loader.py | 4 +- httomo/method_wrappers/reconstruction.py | 2 +- httomo/method_wrappers/rotation.py | 4 +- httomo/methods_database/query.py | 4 +- httomo/runner/loader.py | 3 +- httomo/runner/method_wrapper.py | 4 +- httomo/runner/methods_repository_interface.py | 2 +- httomo/runner/pipeline.py | 3 +- httomo/runner/section.py | 4 +- tests/method_wrappers/test_generic.py | 4 +- tests/method_wrappers/test_reconstruction.py | 2 +- tests/method_wrappers/test_rotation.py | 3 +- tests/runner/test_dataset_store_backing.py | 4 +- tests/runner/test_pipeline.py | 3 +- tests/runner/test_section.py | 3 +- tests/sweep_runner/test_param_sweep_runner.py | 3 +- tests/test_method_query.py | 195 ------------------ tests/testing_utils.py | 3 +- 18 files changed, 38 insertions(+), 212 deletions(-) delete mode 100644 tests/test_method_query.py diff --git a/httomo/loaders/standard_tomo_loader.py b/httomo/loaders/standard_tomo_loader.py index 58af0f834..763d41db5 100644 --- a/httomo/loaders/standard_tomo_loader.py +++ b/httomo/loaders/standard_tomo_loader.py @@ -21,7 +21,9 @@ from httomo.runner.dataset_store_interfaces import DataSetSource from httomo.runner.loader import LoaderInterface from httomo.types import generic_array -from httomo.utils import Pattern, log_once, make_3d_shape_from_shape +from httomo.utils import log_once, make_3d_shape_from_shape + +from httomo_backends.methods_database.query import Pattern class StandardTomoLoader(DataSetSource): diff --git a/httomo/method_wrappers/reconstruction.py b/httomo/method_wrappers/reconstruction.py index 42679c31c..c74c6009b 100644 --- a/httomo/method_wrappers/reconstruction.py +++ b/httomo/method_wrappers/reconstruction.py @@ -5,7 +5,7 @@ from typing import Any, Dict, Optional -from httomo.utils import Pattern +from httomo_backends.methods_database.query import Pattern class ReconstructionWrapper(GenericMethodWrapper): diff --git a/httomo/method_wrappers/rotation.py b/httomo/method_wrappers/rotation.py index 92b080105..07a722189 100644 --- a/httomo/method_wrappers/rotation.py +++ b/httomo/method_wrappers/rotation.py @@ -2,7 +2,9 @@ from httomo.method_wrappers.generic import GenericMethodWrapper from httomo.runner.method_wrapper import MethodParameterDictType from httomo.runner.methods_repository_interface import MethodRepository -from httomo.utils import Pattern, catchtime, log_once, xp +from httomo.utils import catchtime, log_once, xp + +from httomo_backends.methods_database.query import Pattern import numpy as np diff --git a/httomo/methods_database/query.py b/httomo/methods_database/query.py index 282f2e013..c0778cb3c 100644 --- a/httomo/methods_database/query.py +++ b/httomo/methods_database/query.py @@ -7,9 +7,11 @@ import yaml from httomo.runner.methods_repository_interface import GpuMemoryRequirement, MethodQuery -from httomo.utils import Pattern, log_exception +from httomo.utils import log_exception from httomo.runner.methods_repository_interface import MethodRepository +from httomo_backends.methods_database.query import Pattern + YAML_DIR = Path(__file__).parent / "packages/" diff --git a/httomo/runner/loader.py b/httomo/runner/loader.py index ab72a7716..53313edbd 100644 --- a/httomo/runner/loader.py +++ b/httomo/runner/loader.py @@ -1,9 +1,10 @@ from typing import Protocol, Tuple from httomo.runner.dataset_store_interfaces import DataSetSource -from httomo.utils import Pattern from httomo.preview import PreviewConfig +from httomo_backends.methods_database.query import Pattern + class LoaderInterface(Protocol): """Interface to a loader object""" diff --git a/httomo/runner/method_wrapper.py b/httomo/runner/method_wrapper.py index 4d937e4c0..7bfb4adfe 100644 --- a/httomo/runner/method_wrapper.py +++ b/httomo/runner/method_wrapper.py @@ -1,7 +1,9 @@ from dataclasses import dataclass from httomo.block_interfaces import T from httomo.runner.methods_repository_interface import GpuMemoryRequirement -from httomo.utils import Pattern, xp +from httomo.utils import xp + +from httomo_backends.methods_database.query import Pattern import numpy as np from mpi4py import MPI diff --git a/httomo/runner/methods_repository_interface.py b/httomo/runner/methods_repository_interface.py index 4162feee4..9b951d50a 100644 --- a/httomo/runner/methods_repository_interface.py +++ b/httomo/runner/methods_repository_interface.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from typing import List, Literal, Optional, Protocol, Tuple, Union -from httomo.utils import Pattern +from httomo_backends.methods_database.query import Pattern @dataclass(frozen=True) diff --git a/httomo/runner/pipeline.py b/httomo/runner/pipeline.py index 59c08caa1..12e8151e8 100644 --- a/httomo/runner/pipeline.py +++ b/httomo/runner/pipeline.py @@ -1,7 +1,8 @@ from httomo.runner.loader import LoaderInterface -from httomo.utils import Pattern from httomo.runner.method_wrapper import MethodWrapper +from httomo_backends.methods_database.query import Pattern + from typing import Iterator, List diff --git a/httomo/runner/section.py b/httomo/runner/section.py index 87086a62a..b323479dc 100644 --- a/httomo/runner/section.py +++ b/httomo/runner/section.py @@ -8,9 +8,11 @@ from httomo.runner.output_ref import OutputRef from httomo.runner.pipeline import Pipeline -from httomo.utils import Pattern, log_once +from httomo.utils import log_once from httomo.runner.method_wrapper import MethodWrapper +from httomo_backends.methods_database.query import Pattern + class Section: """ diff --git a/tests/method_wrappers/test_generic.py b/tests/method_wrappers/test_generic.py index 04bf13a85..2361adf26 100644 --- a/tests/method_wrappers/test_generic.py +++ b/tests/method_wrappers/test_generic.py @@ -6,9 +6,11 @@ from httomo.runner.dataset import DataSetBlock from httomo.runner.methods_repository_interface import GpuMemoryRequirement from httomo.runner.output_ref import OutputRef -from httomo.utils import Pattern, gpu_enabled, xp +from httomo.utils import gpu_enabled, xp from ..testing_utils import make_mock_preview_config, make_mock_repo, make_test_method +from httomo_backends.methods_database.query import Pattern + import pytest from mpi4py import MPI from pytest_mock import MockerFixture diff --git a/tests/method_wrappers/test_reconstruction.py b/tests/method_wrappers/test_reconstruction.py index b08f78013..89675f9ce 100644 --- a/tests/method_wrappers/test_reconstruction.py +++ b/tests/method_wrappers/test_reconstruction.py @@ -3,9 +3,9 @@ from httomo.method_wrappers.reconstruction import ReconstructionWrapper from httomo.runner.auxiliary_data import AuxiliaryData from httomo.runner.dataset import DataSetBlock -from httomo.utils import Pattern from ..testing_utils import make_mock_preview_config, make_mock_repo +from httomo_backends.methods_database.query import Pattern import numpy as np from mpi4py import MPI diff --git a/tests/method_wrappers/test_rotation.py b/tests/method_wrappers/test_rotation.py index 5f919424a..450b8fae8 100644 --- a/tests/method_wrappers/test_rotation.py +++ b/tests/method_wrappers/test_rotation.py @@ -5,9 +5,10 @@ from httomo.method_wrappers.rotation import RotationWrapper from httomo.runner.auxiliary_data import AuxiliaryData from httomo.runner.dataset import DataSetBlock -from httomo.utils import Pattern, gpu_enabled, make_3d_shape_from_shape, xp +from httomo.utils import gpu_enabled, make_3d_shape_from_shape, xp from ..testing_utils import make_mock_preview_config, make_mock_repo +from httomo_backends.methods_database.query import Pattern import pytest from mpi4py import MPI diff --git a/tests/runner/test_dataset_store_backing.py b/tests/runner/test_dataset_store_backing.py index 7b61cef7b..810122cb1 100644 --- a/tests/runner/test_dataset_store_backing.py +++ b/tests/runner/test_dataset_store_backing.py @@ -14,9 +14,11 @@ from httomo.runner.methods_repository_interface import GpuMemoryRequirement from httomo.runner.pipeline import Pipeline from httomo.runner.section import sectionize -from httomo.utils import Pattern, make_3d_shape_from_shape +from httomo.utils import make_3d_shape_from_shape from tests.testing_utils import make_test_loader, make_test_method +from httomo_backends.methods_database.query import Pattern + @pytest.mark.parametrize( "nprocs, rank, section_slicing_dim, section_padding", diff --git a/tests/runner/test_pipeline.py b/tests/runner/test_pipeline.py index 75ef9e331..737ec8908 100644 --- a/tests/runner/test_pipeline.py +++ b/tests/runner/test_pipeline.py @@ -1,9 +1,10 @@ from pytest_mock import MockerFixture from httomo.runner.pipeline import Pipeline -from httomo.utils import Pattern from ..testing_utils import make_test_loader, make_test_method +from httomo_backends.methods_database.query import Pattern + def test_pipeline_get_loader_properties(mocker: MockerFixture): loader = make_test_loader(mocker, pattern=Pattern.projection) diff --git a/tests/runner/test_section.py b/tests/runner/test_section.py index f453204fa..d1b46ade4 100644 --- a/tests/runner/test_section.py +++ b/tests/runner/test_section.py @@ -4,9 +4,10 @@ from httomo.runner.output_ref import OutputRef from httomo.runner.pipeline import Pipeline from httomo.runner.section import determine_section_padding, sectionize, Section -from httomo.utils import Pattern from ..testing_utils import make_test_loader, make_test_method +from httomo_backends.methods_database.query import Pattern + # For reference, we break into new platform sections if and only if: # - the pattern was changed (reslice) # - an output is referenced from an earlier method (so it needs to have diff --git a/tests/sweep_runner/test_param_sweep_runner.py b/tests/sweep_runner/test_param_sweep_runner.py index 928fbbff3..4466c62de 100644 --- a/tests/sweep_runner/test_param_sweep_runner.py +++ b/tests/sweep_runner/test_param_sweep_runner.py @@ -24,9 +24,10 @@ make_test_method, ) from httomo.runner.loader import LoaderInterface -from httomo.utils import Pattern from httomo.runner.dataset_store_interfaces import DataSetSource +from httomo_backends.methods_database.query import Pattern + def test_raises_error_if_no_sweep_detected(mocker: MockerFixture): loader = make_test_loader(mocker) diff --git a/tests/test_method_query.py b/tests/test_method_query.py deleted file mode 100644 index ce800a2f4..000000000 --- a/tests/test_method_query.py +++ /dev/null @@ -1,195 +0,0 @@ -from pathlib import Path -from pytest_mock import MockerFixture -import yaml -from httomo.methods_database.query import ( - YAML_DIR, - MethodsDatabaseQuery, - get_method_info, -) -import pytest -import numpy as np -from httomo.utils import Pattern - - -def test_get_from_tomopy(): - pat = get_method_info("tomopy.misc.corr", "median_filter", "pattern") - assert pat == "all" - - -def test_get_invalid_package(): - with pytest.raises(FileNotFoundError, match="doesn't exist"): - get_method_info("unavailable.misc.corr", "median_filter", "pattern") - - -def test_get_invalid_module(): - with pytest.raises(KeyError, match="key doesntexist is not present"): - get_method_info("tomopy.doesntexist.corr", "median_filter", "pattern") - - -def test_get_invalid_method(): - with pytest.raises(KeyError, match="key doesntexist is not present"): - get_method_info("tomopy.misc.corr", "doesntexist", "pattern") - - -def test_get_invalid_attr(): - with pytest.raises(KeyError, match="attribute doesntexist is not present"): - get_method_info("tomopy.misc.corr", "median_filter", "doesntexist") - - -def test_httomolibgpu_pattern(): - pat = get_method_info("httomolibgpu.prep.normalize", "normalize", "pattern") - assert pat == "projection" - - -def test_httomolibgpu_implementation(): - implementation = get_method_info( - "httomolibgpu.prep.normalize", "normalize", "implementation" - ) - assert implementation == "gpu_cupy" - - -def test_httomolibgpu_output_dims_change(): - output_dims_change = get_method_info( - "httomolibgpu.prep.normalize", "normalize", "output_dims_change" - ) - assert output_dims_change == False - - -def test_httomolibgpu_default_save_result(): - save_result = get_method_info( - "httomolibgpu.prep.normalize", "normalize", "save_result_default" - ) - - assert save_result is False - - -def test_httomolibgpu_default_save_result_recon(): - save_result = get_method_info( - "httomolibgpu.recon.algorithm", "FBP", "save_result_default" - ) - - assert save_result is True - - -def test_httomolibgpu_memory_gpu(): - memory_gpu = get_method_info( - "httomolibgpu.prep.normalize", "normalize", "memory_gpu" - ) - assert len(memory_gpu) == 2 - - -def test_httomolibgpu_padding_false(): - padding = get_method_info("httomolibgpu.prep.normalize", "normalize", "padding") - - assert padding is False - - -def test_httomolibgpu_padding_true(): - padding = get_method_info("tomopy.misc.corr", "median_filter3d", "padding") - - assert padding is True - - -# this is just a quick check - until we have schema validation on the DB files -def test_all_methods_have_padding_parameter(): - # we don't care about the httomo one - easy to check - for m in ["tomopy", "httomolib", "httomolibgpu"]: - yaml_path = Path(YAML_DIR, f"external/{m}/{m}.yaml") - with open(yaml_path, "r") as f: - info = yaml.safe_load(f) - # methods are on 3rd level - for package_name, module in info.items(): - for f_name, file in module.items(): - for method_name, method in file.items(): - assert ( - "padding" in method - ), f"{m}.{package_name}.{f_name}.{method_name}" - assert type(method["padding"]) == bool - - -def test_database_query_object(): - query = MethodsDatabaseQuery("httomolibgpu.prep.normalize", "normalize") - assert query.get_pattern() == Pattern.projection - assert query.get_output_dims_change() is False - assert query.get_implementation() == "gpu_cupy" - assert query.swap_dims_on_output() is False - assert query.save_result_default() is False - assert query.padding() is False - mempars = query.get_memory_gpu_params() - assert mempars.method == "module" - assert mempars.multiplier == "None" - - -def test_database_query_object_recon_swap_output(): - query = MethodsDatabaseQuery("tomopy.recon.algorithm", "recon") - assert query.swap_dims_on_output() is True - - -def test_database_query_calculate_memory(mocker: MockerFixture): - class FakeModule: - def _calc_memory_bytes_testmethod(non_slice_dims_shape, dtype, testparam): - assert non_slice_dims_shape == ( - 42, - 3, - ) - assert dtype == np.float32 - assert testparam == 42.0 - return 10, 20 - - importmock = mocker.patch( - "httomo.methods_database.query.import_module", return_value=FakeModule - ) - query = MethodsDatabaseQuery("sample.module.path", "testmethod") - - mem = query.calculate_memory_bytes((42, 3), np.float32, testparam=42.0) - - importmock.assert_called_with( - "httomo.methods_database.packages.external.sample.supporting_funcs.module.path" - ) - assert mem == (10, 20) - - -def test_database_query_calculate_output_dims(mocker: MockerFixture): - class FakeModule: - def _calc_output_dim_testmethod(non_slice_dims_shape, testparam): - assert non_slice_dims_shape == ( - 42, - 3, - ) - assert testparam == 42.0 - return 10, 20 - - importmock = mocker.patch( - "httomo.methods_database.query.import_module", return_value=FakeModule - ) - query = MethodsDatabaseQuery("sample.module.path", "testmethod") - - dims = query.calculate_output_dims((42, 3), testparam=42.0) - - importmock.assert_called_with( - "httomo.methods_database.packages.external.sample.supporting_funcs.module.path" - ) - assert dims == (10, 20) - - -def test_database_query_calculate_padding(mocker: MockerFixture): - SIZE_PARAMETER = 5 - PADDING_RETURNED = (5, 5) - - class FakeModule: - def _calc_padding_testmethod(size): - assert size == SIZE_PARAMETER - return PADDING_RETURNED - - importmock = mocker.patch( - "httomo.methods_database.query.import_module", return_value=FakeModule - ) - query = MethodsDatabaseQuery("sample.module.path", "testmethod") - - pads = query.calculate_padding(size=SIZE_PARAMETER) - - importmock.assert_called_once_with( - "httomo.methods_database.packages.external.sample.supporting_funcs.module.path" - ) - - assert pads == PADDING_RETURNED diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 79e2e5a4b..e72e5e57e 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -8,7 +8,8 @@ GpuMemoryRequirement, MethodRepository, ) -from httomo.utils import Pattern + +from httomo_backends.methods_database.query import Pattern from pytest_mock import MockerFixture From f4d256895975025eee1d0d26a5de4524ecd9e793 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Tue, 10 Dec 2024 10:51:30 +0000 Subject: [PATCH 06/14] Import `GpuMemoryRequirement` from `httomo-backends` --- httomo/runner/methods_repository_interface.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/httomo/runner/methods_repository_interface.py b/httomo/runner/methods_repository_interface.py index 9b951d50a..88be412b8 100644 --- a/httomo/runner/methods_repository_interface.py +++ b/httomo/runner/methods_repository_interface.py @@ -2,13 +2,7 @@ from dataclasses import dataclass from typing import List, Literal, Optional, Protocol, Tuple, Union -from httomo_backends.methods_database.query import Pattern - - -@dataclass(frozen=True) -class GpuMemoryRequirement: - multiplier: Optional[float] = 1.0 - method: Literal["direct", "module"] = "direct" +from httomo_backends.methods_database.query import GpuMemoryRequirement, Pattern class MethodQuery(Protocol): From 4f9eaf11d13976a368efa37ec763d68413d735a5 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Tue, 10 Dec 2024 12:10:33 +0000 Subject: [PATCH 07/14] Import `GpuMemoryRequirement` explicitly from `httomo-backends` everywhere --- httomo/methods_database/query.py | 4 ++-- httomo/runner/method_wrapper.py | 3 +-- tests/method_wrappers/test_generic.py | 3 +-- tests/runner/test_dataset_store_backing.py | 3 +-- tests/runner/test_task_runner.py | 3 ++- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/httomo/methods_database/query.py b/httomo/methods_database/query.py index c0778cb3c..67a8f5e34 100644 --- a/httomo/methods_database/query.py +++ b/httomo/methods_database/query.py @@ -5,12 +5,12 @@ import numpy as np import yaml -from httomo.runner.methods_repository_interface import GpuMemoryRequirement, MethodQuery +from httomo.runner.methods_repository_interface import MethodQuery from httomo.utils import log_exception from httomo.runner.methods_repository_interface import MethodRepository -from httomo_backends.methods_database.query import Pattern +from httomo_backends.methods_database.query import GpuMemoryRequirement, Pattern YAML_DIR = Path(__file__).parent / "packages/" diff --git a/httomo/runner/method_wrapper.py b/httomo/runner/method_wrapper.py index 7bfb4adfe..77581960c 100644 --- a/httomo/runner/method_wrapper.py +++ b/httomo/runner/method_wrapper.py @@ -1,9 +1,8 @@ from dataclasses import dataclass from httomo.block_interfaces import T -from httomo.runner.methods_repository_interface import GpuMemoryRequirement from httomo.utils import xp -from httomo_backends.methods_database.query import Pattern +from httomo_backends.methods_database.query import GpuMemoryRequirement, Pattern import numpy as np from mpi4py import MPI diff --git a/tests/method_wrappers/test_generic.py b/tests/method_wrappers/test_generic.py index 2361adf26..e62fe0425 100644 --- a/tests/method_wrappers/test_generic.py +++ b/tests/method_wrappers/test_generic.py @@ -4,12 +4,11 @@ from httomo.method_wrappers import make_method_wrapper from httomo.method_wrappers.generic import GenericMethodWrapper from httomo.runner.dataset import DataSetBlock -from httomo.runner.methods_repository_interface import GpuMemoryRequirement from httomo.runner.output_ref import OutputRef from httomo.utils import gpu_enabled, xp from ..testing_utils import make_mock_preview_config, make_mock_repo, make_test_method -from httomo_backends.methods_database.query import Pattern +from httomo_backends.methods_database.query import GpuMemoryRequirement, Pattern import pytest from mpi4py import MPI diff --git a/tests/runner/test_dataset_store_backing.py b/tests/runner/test_dataset_store_backing.py index 810122cb1..4dc9b77b6 100644 --- a/tests/runner/test_dataset_store_backing.py +++ b/tests/runner/test_dataset_store_backing.py @@ -11,13 +11,12 @@ calculate_section_chunk_bytes, determine_store_backing, ) -from httomo.runner.methods_repository_interface import GpuMemoryRequirement from httomo.runner.pipeline import Pipeline from httomo.runner.section import sectionize from httomo.utils import make_3d_shape_from_shape from tests.testing_utils import make_test_loader, make_test_method -from httomo_backends.methods_database.query import Pattern +from httomo_backends.methods_database.query import GpuMemoryRequirement, Pattern @pytest.mark.parametrize( diff --git a/tests/runner/test_task_runner.py b/tests/runner/test_task_runner.py index f17475a06..68d6120ec 100644 --- a/tests/runner/test_task_runner.py +++ b/tests/runner/test_task_runner.py @@ -19,7 +19,6 @@ from httomo.runner.auxiliary_data import AuxiliaryData from httomo.runner.dataset import DataSetBlock from httomo.runner.dataset_store_backing import DataSetStoreBacking -from httomo.runner.methods_repository_interface import GpuMemoryRequirement from httomo.runner.monitoring_interface import MonitoringInterface from httomo.runner.output_ref import OutputRef from httomo.runner.pipeline import Pipeline @@ -33,6 +32,8 @@ from httomo.runner.method_wrapper import GpuTimeInfo, MethodWrapper from ..testing_utils import make_mock_preview_config, make_test_loader, make_test_method +from httomo_backends.methods_database.query import GpuMemoryRequirement + def test_check_params_for_sweep_raises_exception( mocker: MockerFixture, tmp_path: PathLike From b77062459467fd4402551178f64cb7d5c56fa017 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 11 Dec 2024 12:16:39 +0000 Subject: [PATCH 08/14] Remove `query.py` and use `httomo-backends` instead --- httomo/methods_database/query.py | 159 ------------------------------- httomo/transform_layer.py | 3 +- httomo/ui_layer.py | 2 +- tests/runner/test_task_runner.py | 7 +- 4 files changed, 8 insertions(+), 163 deletions(-) delete mode 100644 httomo/methods_database/query.py diff --git a/httomo/methods_database/query.py b/httomo/methods_database/query.py deleted file mode 100644 index 67a8f5e34..000000000 --- a/httomo/methods_database/query.py +++ /dev/null @@ -1,159 +0,0 @@ -from importlib import import_module -from types import ModuleType -from typing import Callable, List, Literal, Optional, Tuple -from pathlib import Path -import numpy as np - -import yaml -from httomo.runner.methods_repository_interface import MethodQuery - -from httomo.utils import log_exception -from httomo.runner.methods_repository_interface import MethodRepository - -from httomo_backends.methods_database.query import GpuMemoryRequirement, Pattern - -YAML_DIR = Path(__file__).parent / "packages/" - - -def get_method_info(module_path: str, method_name: str, attr: str): - """Get the information about the given method associated with `attr` that - is stored in the relevant YAML file in `httomo/methods_database/packages/` - - Parameters - ---------- - module_path : str - The full module path of the method, including the top-level package - name. Ie, `httomolib.misc.images.save_to_images`. - - method_name : str - The name of the method function. - - attr : str - The name of the piece of information about the method being requested - (for example, "pattern"). - - Returns - ------- - The requested piece of information about the method. - """ - method_path = f"{module_path}.{method_name}" - split_method_path = method_path.split(".") - package_name = split_method_path[0] - - # open the library file for the package - ext_package_path = "" - if package_name != "httomo": - ext_package_path = f"external/{package_name}/" - yaml_info_path = Path(YAML_DIR, str(ext_package_path), f"{package_name}.yaml") - if not yaml_info_path.exists(): - err_str = f"The YAML file {yaml_info_path} doesn't exist." - log_exception(err_str) - raise FileNotFoundError(err_str) - - with open(yaml_info_path, "r") as f: - info = yaml.safe_load(f) - for key in split_method_path[1:]: - try: - info = info[key] - except KeyError: - raise KeyError(f"The key {key} is not present ({method_path})") - - try: - return info[attr] - except KeyError: - raise KeyError(f"The attribute {attr} is not present on {method_path}") - - -# Implementation of methods database query class -class MethodsDatabaseQuery(MethodQuery): - def __init__(self, module_path: str, method_name: str): - self.module_path = module_path - self.method_name = method_name - - def get_pattern(self) -> Pattern: - p = get_method_info(self.module_path, self.method_name, "pattern") - assert p in ["projection", "sinogram", "all"], ( - f"The pattern {p} that is listed for the method " - f"{self.module_path}.{self.method_name} is invalid." - ) - if p == "projection": - return Pattern.projection - if p == "sinogram": - return Pattern.sinogram - return Pattern.all - - def get_output_dims_change(self) -> bool: - p = get_method_info(self.module_path, self.method_name, "output_dims_change") - return bool(p) - - def get_implementation(self) -> Literal["cpu", "gpu", "gpu_cupy"]: - p = get_method_info(self.module_path, self.method_name, "implementation") - assert p in [ - "gpu", - "gpu_cupy", - "cpu", - ], f"The implementation arch {p} listed for method {self.module_path}.{self.method_name} is invalid" - return p - - def save_result_default(self) -> bool: - return get_method_info( - self.module_path, self.method_name, "save_result_default" - ) - - def padding(self) -> bool: - return get_method_info(self.module_path, self.method_name, "padding") - - def get_memory_gpu_params( - self, - ) -> Optional[GpuMemoryRequirement]: - p = get_method_info(self.module_path, self.method_name, "memory_gpu") - if p is None or p == "None": - return None - if type(p) == list: - # convert to dict first - d: dict = dict() - for item in p: - d |= item - else: - d = p - return GpuMemoryRequirement(multiplier=d["multiplier"], method=d["method"]) - - def calculate_memory_bytes( - self, non_slice_dims_shape: Tuple[int, int], dtype: np.dtype, **kwargs - ) -> Tuple[int, int]: - smodule = self._import_supporting_funcs_module() - module_mem: Callable = getattr( - smodule, "_calc_memory_bytes_" + self.method_name - ) - memory_bytes: Tuple[int, int] = module_mem( - non_slice_dims_shape, dtype, **kwargs - ) - return memory_bytes - - def calculate_output_dims( - self, non_slice_dims_shape: Tuple[int, int], **kwargs - ) -> Tuple[int, int]: - smodule = self._import_supporting_funcs_module() - module_mem: Callable = getattr(smodule, "_calc_output_dim_" + self.method_name) - return module_mem(non_slice_dims_shape, **kwargs) - - def calculate_padding(self, **kwargs) -> Tuple[int, int]: - smodule = self._import_supporting_funcs_module() - module_pad: Callable = getattr(smodule, "_calc_padding_" + self.method_name) - return module_pad(**kwargs) - - def _import_supporting_funcs_module(self) -> ModuleType: - - module_mem_path = "httomo.methods_database.packages.external." - path = self.module_path.split(".") - path.insert(1, "supporting_funcs") - module_mem_path += ".".join(path) - return import_module(module_mem_path) - - def swap_dims_on_output(self) -> bool: - return self.module_path.startswith("tomopy.recon") - - -class MethodDatabaseRepository(MethodRepository): - def query(self, module_path: str, method_name: str) -> MethodQuery: - return MethodsDatabaseQuery(module_path, method_name) diff --git a/httomo/transform_layer.py b/httomo/transform_layer.py index 36fe44ad7..1d59fb197 100644 --- a/httomo/transform_layer.py +++ b/httomo/transform_layer.py @@ -6,11 +6,12 @@ from httomo.method_wrappers.generic import GenericMethodWrapper from httomo.method_wrappers.images import ImagesWrapper from httomo.method_wrappers.save_intermediate import SaveIntermediateFilesWrapper -from httomo.methods_database.query import MethodDatabaseRepository from httomo.runner.pipeline import Pipeline from mpi4py import MPI import httomo +from httomo_backends.methods_database.query import MethodDatabaseRepository + class TransformLayer: def __init__( diff --git a/httomo/ui_layer.py b/httomo/ui_layer.py index b0a421023..e2e2af33c 100644 --- a/httomo/ui_layer.py +++ b/httomo/ui_layer.py @@ -9,7 +9,6 @@ from mpi4py.MPI import Comm from httomo.darks_flats import DarksFlatsFileConfig -from httomo.methods_database.query import MethodDatabaseRepository from httomo.preview import PreviewConfig from httomo.runner.method_wrapper import MethodWrapper from httomo.runner.pipeline import Pipeline @@ -21,6 +20,7 @@ from httomo.sweep_runner.param_sweep_yaml_loader import get_param_sweep_yaml_loader from httomo.transform_loader_params import parse_angles, parse_preview +from httomo_backends.methods_database.query import MethodDatabaseRepository MethodConfig: TypeAlias = Dict[str, Any] PipelineConfig: TypeAlias = List[MethodConfig] diff --git a/tests/runner/test_task_runner.py b/tests/runner/test_task_runner.py index 68d6120ec..3d642cb74 100644 --- a/tests/runner/test_task_runner.py +++ b/tests/runner/test_task_runner.py @@ -14,7 +14,6 @@ from httomo.loaders import make_loader from httomo.loaders.types import RawAngles from httomo.method_wrappers import make_method_wrapper -from httomo.methods_database.query import MethodDatabaseRepository, MethodsDatabaseQuery from httomo.preview import PreviewConfig, PreviewDimConfig from httomo.runner.auxiliary_data import AuxiliaryData from httomo.runner.dataset import DataSetBlock @@ -32,7 +31,11 @@ from httomo.runner.method_wrapper import GpuTimeInfo, MethodWrapper from ..testing_utils import make_mock_preview_config, make_test_loader, make_test_method -from httomo_backends.methods_database.query import GpuMemoryRequirement +from httomo_backends.methods_database.query import ( + GpuMemoryRequirement, + MethodsDatabaseQuery, + MethodDatabaseRepository, +) def test_check_params_for_sweep_raises_exception( From edd0953b91e1584628860cf61d993e8f3eb9de1c Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 11 Dec 2024 12:23:34 +0000 Subject: [PATCH 09/14] Use autospecced query in task runner padding test This removes the need to patch the `import_module()` call (which is now being done outside of `httomo`, in `httomo-backends`). --- tests/runner/test_task_runner.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/runner/test_task_runner.py b/tests/runner/test_task_runner.py index 3d642cb74..e7d254582 100644 --- a/tests/runner/test_task_runner.py +++ b/tests/runner/test_task_runner.py @@ -18,6 +18,7 @@ from httomo.runner.auxiliary_data import AuxiliaryData from httomo.runner.dataset import DataSetBlock from httomo.runner.dataset_store_backing import DataSetStoreBacking +from httomo.runner.methods_repository_interface import MethodQuery from httomo.runner.monitoring_interface import MonitoringInterface from httomo.runner.output_ref import OutputRef from httomo.runner.pipeline import Pipeline @@ -33,7 +34,6 @@ from httomo_backends.methods_database.query import ( GpuMemoryRequirement, - MethodsDatabaseQuery, MethodDatabaseRepository, ) @@ -573,11 +573,6 @@ def method_using_padding_slices(data: np.ndarray, kernel_size: int): out[i, j, k] = neighbourhood.sum() return out - # Define padding calculator for dummy 3D method - class FakeSupportingFunctionsModule: - def _calc_padding_method_using_padding_slices(**kwargs) -> Tuple[int, int]: - return (kwargs["kernel_size"] // 2, kwargs["kernel_size"] // 2) - # Create method wrapper KERNEL_SIZE = 3 MODULE_PATH = "module_path" @@ -586,11 +581,12 @@ def _calc_padding_method_using_padding_slices(**kwargs) -> Tuple[int, int]: "httomo.method_wrappers.generic.import_module", return_value=FakeMethodsModule ) mock_repo = mocker.MagicMock() - method_query = MethodsDatabaseQuery(MODULE_PATH, METHOD_NAME) + method_query = mocker.create_autospec(MethodQuery) mocker.patch.object(target=method_query, attribute="padding", return_value=True) - mocker.patch( - "httomo.methods_database.query.import_module", - return_value=FakeSupportingFunctionsModule, + mocker.patch.object( + target=method_query, + attribute="calculate_padding", + return_value=(KERNEL_SIZE // 2, KERNEL_SIZE // 2), ) mocker.patch.object( target=method_query, attribute="get_pattern", return_value=Pattern.projection @@ -607,6 +603,9 @@ def _calc_padding_method_using_padding_slices(**kwargs) -> Tuple[int, int]: mocker.patch.object( target=method_query, attribute="save_result_default", return_value=False ) + mocker.patch.object( + target=method_query, attribute="swap_dims_on_output", return_value=False + ) mocker.patch.object(target=mock_repo, attribute="query", return_value=method_query) wrapper = make_method_wrapper( method_repository=mock_repo, From 7b7fea5f7e1868dfc6722dd777d012369c4f10d6 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 11 Dec 2024 12:39:50 +0000 Subject: [PATCH 10/14] Remove unnecessary `httomo-backends` method repo object in test --- tests/runner/test_task_runner.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/runner/test_task_runner.py b/tests/runner/test_task_runner.py index e7d254582..c1a5b2887 100644 --- a/tests/runner/test_task_runner.py +++ b/tests/runner/test_task_runner.py @@ -32,10 +32,7 @@ from httomo.runner.method_wrapper import GpuTimeInfo, MethodWrapper from ..testing_utils import make_mock_preview_config, make_test_loader, make_test_method -from httomo_backends.methods_database.query import ( - GpuMemoryRequirement, - MethodDatabaseRepository, -) +from httomo_backends.methods_database.query import GpuMemoryRequirement def test_check_params_for_sweep_raises_exception( @@ -525,8 +522,9 @@ def test_execute_section_with_padding_produces_correct_result( detector_y=PreviewDimConfig(start=0, stop=DATA_SHAPE[1]), detector_x=PreviewDimConfig(start=0, stop=DATA_SHAPE[2]), ) + mock_repo = mocker.MagicMock() loader_wrapper = make_loader( - repo=MethodDatabaseRepository(), + repo=mock_repo, module_path="httomo.data.hdf.loaders", method_name="standard_tomo", in_file=IN_FILE_PATH, @@ -580,7 +578,6 @@ def method_using_padding_slices(data: np.ndarray, kernel_size: int): mocker.patch( "httomo.method_wrappers.generic.import_module", return_value=FakeMethodsModule ) - mock_repo = mocker.MagicMock() method_query = mocker.create_autospec(MethodQuery) mocker.patch.object(target=method_query, attribute="padding", return_value=True) mocker.patch.object( From ec2a4875e90fb02c643fe90b2e15e8a396eb43f6 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 11 Dec 2024 13:15:00 +0000 Subject: [PATCH 11/14] Unwrap `OutputRef` values before passing params to supporting functions Fixes #359 --- httomo/method_wrappers/generic.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/httomo/method_wrappers/generic.py b/httomo/method_wrappers/generic.py index 241cf385b..a41180ff4 100644 --- a/httomo/method_wrappers/generic.py +++ b/httomo/method_wrappers/generic.py @@ -394,7 +394,7 @@ def calculate_output_dims( """Calculate the dimensions of the output for this method""" if self.output_dims_change: return self._query.calculate_output_dims( - non_slice_dims_shape, **self.config_params + non_slice_dims_shape, **self._unwrap_output_ref_values() ) return non_slice_dims_shape @@ -402,7 +402,7 @@ def calculate_output_dims( def calculate_padding(self) -> Tuple[int, int]: """Calculate the padding required by the method""" if self.padding: - return self._query.calculate_padding(**self.config_params) + return self._query.calculate_padding(**self._unwrap_output_ref_values()) return (0, 0) def calculate_max_slices( @@ -449,7 +449,7 @@ def calculate_max_slices( memory_bytes_method, subtract_bytes, ) = self._query.calculate_memory_bytes( - non_slice_dims_shape, data_dtype, **self.config_params + non_slice_dims_shape, data_dtype, **self._unwrap_output_ref_values() ) if memory_bytes_method == 0: @@ -458,3 +458,22 @@ def calculate_max_slices( return ( available_memory - subtract_bytes ) // memory_bytes_method, available_memory + + def _unwrap_output_ref_values(self) -> Dict[str, Any]: + """ + Iterate through params in `self.config_params` and, for any value of type `OutputRef`, + extract the value inside the `OutputRef`. + + Returns + ------- + Dict[str, Any] + A dict containing all parameters in `self.config_params`, but with any `OutputRef` + values replaced with the value inside the `OutputRef`. + """ + params = dict() + for name, value in self.config_params.items(): + if isinstance(value, OutputRef): + params[name] = value.value + continue + params[name] = value + return params From d4959b83c582d7944a156a882faacb80cbb48dad Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 11 Dec 2024 16:11:55 +0000 Subject: [PATCH 12/14] Remove unused `cufft.py` module --- httomo/cufft.py | 168 ------------------------------------------------ 1 file changed, 168 deletions(-) delete mode 100644 httomo/cufft.py diff --git a/httomo/cufft.py b/httomo/cufft.py deleted file mode 100644 index 0acc77109..000000000 --- a/httomo/cufft.py +++ /dev/null @@ -1,168 +0,0 @@ -import ctypes -import sys -from enum import Enum - - -if "linux" not in sys.platform: - raise RuntimeError("Linux is currently the only supported platform") - -_linux_version_list = [ - "11.3.0.4", - 11.0, - 10.1, - 10.0, - 9.2, - 9.1, - 9.0, - 8.0, - 7.5, - 7.0, - 6.5, - 6.0, - 5.5, - 5.0, - 4.0, -] -_libcufft_libname_list = ["libcufft.so"] + [ - "libcufft.so.%s" % v for v in _linux_version_list -] - -# Load library -_libcufft = None -for _libcufft_libname in _libcufft_libname_list: - try: - _libcufft = ctypes.cdll.LoadLibrary(_libcufft_libname) - except OSError: - pass - else: - break - -# Print understandable error message when library cannot be found: -if _libcufft is None: - raise OSError("cufft library not found") - - -# General CUFFT error -class CufftError(Exception): - """CUFFT error""" - - pass - - -# Errors that can be returned by plan estimation functions -class CufftAllocFailed(CufftError): - """CUFFT failed to allocate GPU memory.""" - - pass - - -class CufftInvalidValue(CufftError): - """The user specified a bad memory pointer.""" - - pass - - -class CufftInternalError(CufftError): - """Internal driver error.""" - - pass - - -class CufftSetupFailed(CufftError): - """The CUFFT library failed to initialize.""" - - pass - - -class CufftInvalidSize(CufftError): - """The user specified an unsupported FFT size.""" - - pass - - -cufftExceptions = { - 0x2: CufftAllocFailed, - 0x4: CufftInvalidValue, - 0x5: CufftInternalError, - 0x7: CufftSetupFailed, - 0x8: CufftInvalidSize, -} - - -class _types: - """Some alias types.""" - - plan = ctypes.c_int - stream = ctypes.c_void_p - worksize = ctypes.c_size_t - - -# Data transformation types -class CufftType(Enum): - CUFFT_R2C = 0x2A - CUFFT_C2R = 0x2C - CUFFT_C2C = 0x29 - CUFFT_D2Z = 0x6A - CUFFT_Z2D = 0x6C - CUFFT_Z2Z = 0x69 - - -def cufftCheckStatus(status: int): - """Raise an exception if the specified CUBLAS status is an error.""" - if status != 0: - try: - e = cufftExceptions[status] - except KeyError: - raise CufftError - else: - raise e - - -_libcufft.cufftEstimate1d.restype = int -_libcufft.cufftEstimate1d.argtypes = [ - ctypes.c_int, - ctypes.c_int, - ctypes.c_int, - ctypes.c_void_p, -] - - -def cufft_estimate_1d(nx: int, fft_type: CufftType, batch: int = 1): - """ - Return estimated work area for 1D FFT. - - References - ---------- - `cufftEstimate1d `_ - """ - worksize = _types.worksize() - assert _libcufft is not None - status = _libcufft.cufftEstimate1d( - nx, fft_type.value, batch, ctypes.byref(worksize) - ) - cufftCheckStatus(status) - return worksize.value - - -_libcufft.cufftEstimate2d.restype = int -_libcufft.cufftEstimate2d.argtypes = [ - ctypes.c_int, - ctypes.c_int, - ctypes.c_int, - ctypes.c_void_p, -] - - -def cufft_estimate_2d(nx: int, ny: int, fft_type: CufftType): - """ - Return estimated work area for 2D FFT. - - References - ---------- - `cufftEstimate2d `_ - """ - worksize = _types.worksize() - assert _libcufft is not None - status = _libcufft.cufftEstimate2d(nx, ny, fft_type.value, ctypes.byref(worksize)) - cufftCheckStatus(status) - return worksize.value From adde0f7dd680023d5a3a38deb208915063f9ddbd Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Fri, 10 Jan 2025 14:02:20 +0000 Subject: [PATCH 13/14] Don't install deps of `httomo-backends` in IRIS CI --- .github/workflows/run_tests_iris.yml | 3 ++- pyproject.toml | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests_iris.yml b/.github/workflows/run_tests_iris.yml index b37616a6d..c107a8513 100644 --- a/.github/workflows/run_tests_iris.yml +++ b/.github/workflows/run_tests_iris.yml @@ -36,7 +36,8 @@ jobs: run: | micromamba activate httomo pip install --upgrade --force-reinstall pillow - pip install httomolibgpu tomobar httomo-backends + pip install httomolibgpu tomobar + pip install --no-deps httomo-backends pip install . micromamba list diff --git a/pyproject.toml b/pyproject.toml index 2aa312f47..fa9fa3da8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ dependencies = [ "click", "numpy", "httomolib", - "httomo-backends", "loguru", "typing-extensions", "tqdm", From 877e6f9d71127844d434a5e6be2b2532af84777f Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Tue, 14 Jan 2025 12:12:29 +0000 Subject: [PATCH 14/14] Remove remaining methods database info --- .../process_lists/process_list_configure.rst | 22 +++++++---------- httomo/methods_database/__init__.py | 0 httomo/methods_database/packages/__init__.py | 0 httomo/methods_database/packages/httomo.yaml | 24 ------------------- .../packages/httomo_modules.yaml | 2 -- httomo/methods_database/packages/version.yaml | 3 --- pyproject.toml | 9 ------- 7 files changed, 8 insertions(+), 52 deletions(-) delete mode 100644 httomo/methods_database/__init__.py delete mode 100644 httomo/methods_database/packages/__init__.py delete mode 100644 httomo/methods_database/packages/httomo.yaml delete mode 100644 httomo/methods_database/packages/httomo_modules.yaml delete mode 100644 httomo/methods_database/packages/version.yaml diff --git a/docs/source/howto/process_lists/process_list_configure.rst b/docs/source/howto/process_lists/process_list_configure.rst index 62ce4284f..99e62b7cb 100644 --- a/docs/source/howto/process_lists/process_list_configure.rst +++ b/docs/source/howto/process_lists/process_list_configure.rst @@ -47,20 +47,14 @@ The pattern of any supported method can be found in :ref:`pl_library`. Library files ------------- -Here is the list of library files for backends where patterns and other fixed arguments for methods are specified. When HTTomo operates -with a certain method it always refers to its library file in order get the specific requirements for that method. - -.. dropdown:: TomoPy's library file - - .. literalinclude:: ../../../../httomo/methods_database/packages/external/tomopy/tomopy.yaml - -.. dropdown:: Httomolibgpu's library file - - .. literalinclude:: ../../../../httomo/methods_database/packages/external/httomolibgpu/httomolibgpu.yaml - -.. dropdown:: Httomolib's library file - - .. literalinclude:: ../../../../httomo/methods_database/packages/external/httomolib/httomolib.yaml +In order for HTTomo to execute a method it requires certain information about the method, such +as its pattern, and (if it's a GPU method) the amount of GPU memory required per-slice. HTTomo +uses a package called :code:`httomo-backends` to get this information. + +See the `httomo-backends +`_ +documentation for more details on how it provides the required information through "library +files" and "supporting functions". .. _pl_grouping: diff --git a/httomo/methods_database/__init__.py b/httomo/methods_database/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/__init__.py b/httomo/methods_database/packages/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/httomo/methods_database/packages/httomo.yaml b/httomo/methods_database/packages/httomo.yaml deleted file mode 100644 index 2841a5abb..000000000 --- a/httomo/methods_database/packages/httomo.yaml +++ /dev/null @@ -1,24 +0,0 @@ -data: - hdf: - loaders: - standard_tomo: - pattern: projection - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False -methods: - calculate_stats: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False - save_intermediate_data: - pattern: all - output_dims_change: False - implementation: cpu - memory_gpu: None - save_result_default: False - padding: False \ No newline at end of file diff --git a/httomo/methods_database/packages/httomo_modules.yaml b/httomo/methods_database/packages/httomo_modules.yaml deleted file mode 100644 index a3375c2c3..000000000 --- a/httomo/methods_database/packages/httomo_modules.yaml +++ /dev/null @@ -1,2 +0,0 @@ -- httomo.data.hdf.loaders -- httomo.methods \ No newline at end of file diff --git a/httomo/methods_database/packages/version.yaml b/httomo/methods_database/packages/version.yaml deleted file mode 100644 index 8bb123e7b..000000000 --- a/httomo/methods_database/packages/version.yaml +++ /dev/null @@ -1,3 +0,0 @@ -# This file defines the current httomo version -httomo: - - 2.0 diff --git a/pyproject.toml b/pyproject.toml index fa9fa3da8..8006c5d71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,15 +9,6 @@ include-package-data = true where = ["."] include = ["httomo*"] -[tool.setuptools.package-data] -httomo = [ - "methods_database/packages/*.yaml", - "methods_database/packages/external/*.yaml", - "methods_database/packages/external/httomolibgpu/*.yaml", - "methods_database/packages/external/httomolib/*.yaml", - "methods_database/packages/external/tomopy/*.yaml", -] - [tool.setuptools-git-versioning] enabled = true template = "{tag}"