Added a synthetic test and control from the command line and saving of those test results

Author: etpeterson

conftest.py

@@ -0,0 +1,103 @@
+import pytest
+import pathlib
+import json
+import csv
+import datetime
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--SNR",
+        default=[100],
+        nargs="+",
+        type=int,
+        help="Evaluation test SNR",
+    )
+    parser.addoption(
+        "--algorithmFile",
+        default="tests/IVIMmodels/unit_tests/algorithms.json",
+        type=str,
+        help="Algorithm file name",
+    )
+    parser.addoption(
+        "--dataFile",
+        default="tests/IVIMmodels/unit_tests/generic.json",
+        type=str,
+        help="Default data file name",
+    )
+    parser.addoption(
+        "--saveFileName",
+        default="",
+        type=str,
+        help="Saved results file name",
+    )
+    parser.addoption(
+        "--rtol",
+        default=1,
+        type=float,
+        help="Relative tolerance",
+    )
+    parser.addoption(
+        "--atol",
+        default=1,
+        type=float,
+        help="Absolute tolerance",
+    )
+
+
+@pytest.fixture(scope="session")
+def save_file(request):
+    filename = request.config.getoption("--saveFileName")
+    if filename:
+        current_folder = pathlib.Path.cwd()
+        filename = current_folder / filename
+        # print(filename)
+        # filename.unlink(missing_ok=True)
+        filename = filename.as_posix()
+        with open(filename, "a") as csv_file:
+            writer = csv.writer(csv_file, delimiter='#')
+            writer.writerow(["", datetime.datetime.now()])
+    return filename
+
+@pytest.fixture(scope="session")
+def rtol(request):
+    return request.config.getoption("--rtol")
+
+
+@pytest.fixture(scope="session")
+def atol(request):
+    return request.config.getoption("--atol")
+
+
+def pytest_generate_tests(metafunc):
+    if "SNR" in metafunc.fixturenames:
+        metafunc.parametrize("SNR", metafunc.config.getoption("SNR"))
+    if "ivim_algorithm" in metafunc.fixturenames:
+        algorithms = algorithm_list(metafunc.config.getoption("algorithmFile"))
+        metafunc.parametrize("ivim_algorithm", algorithms)
+    if "algorithm_file" in metafunc.fixturenames:
+        metafunc.parametrize("algorithm_file", metafunc.config.getoption("algorithmFile"))
+    if "ivim_data" in metafunc.fixturenames:
+        data = data_list(metafunc.config.getoption("dataFile"))
+        metafunc.parametrize("ivim_data", data)
+    if "data_file" in metafunc.fixturenames:
+        metafunc.parametrize("data_file", metafunc.config.getoption("dataFile"))
+
+
+def algorithm_list(filename):
+    current_folder = pathlib.Path.cwd()
+    algorithm_path = current_folder / filename
+    with algorithm_path.open() as f:
+        algorithm_information = json.load(f)
+    return algorithm_information["algorithms"]
+
+def data_list(filename):
+    current_folder = pathlib.Path.cwd()
+    data_path = current_folder / filename
+    with data_path.open() as f:
+        all_data = json.load(f)
+
+    bvals = all_data.pop('config')
+    bvals = bvals['bvalues']
+    for name, data in all_data.items():
+        yield name, bvals, data

tests/IVIMmodels/__init__.py

@@ -10,6 +10,57 @@
 
 #run using python -m pytest from the root folder
 
+
+# @pytest.fixture
+# def algorithm_fixture()
+# def test_fixtures()
+
+# use a fixture to generate data
+# either read a config file for the test or perhaps hard code a few fixtures and usefixtures in the config?
+# use a fixture to save data
+
+# def algorithm_list():
+#     # Find the algorithms from algorithms.json
+#     file = pathlib.Path(__file__)
+#     algorithm_path = file.with_name('algorithms.json')
+#     with algorithm_path.open() as f:
+#         algorithm_information = json.load(f)
+#     return algorithm_information["algorithms"]
+
+# @pytest.fixture(params=algorithm_list())
+# def algorithm_fixture(request):
+#     # assert request.param == "algorithms"
+#     yield request.param
+
+
+
+# @pytest.fixture(params=SNR)
+# def noise_fixture(request):
+#     return request.config.getoption("--noise")
+
+# @pytest.fixture
+# def noise_fixture(request):
+#     yield request.param
+
+# @pytest.mark.parametrize("S", [SNR])
+# @pytest.mark.parametrize("D, Dp, f, bvals", [[0.0015, 0.1, 0.11000000000000007,[0, 5, 10, 50, 100, 200, 300, 500, 1000]]])
+# def test_generated(ivim_algorithm, ivim_data, SNR):
+#     S0 = 1
+#     gd = GenerateData()
+#     name, bvals, data = ivim_data
+#     D = data["D"]
+#     f = data["f"]
+#     Dp = data["Dp"]
+#     if "data" not in data:
+#         signal = gd.ivim_signal(D, Dp, f, S0, bvals, SNR)
+#     else:
+#         signal = data["data"]
+#     fit = OsipiBase(algorithm=ivim_algorithm)
+#     [f_fit, Dp_fit, D_fit] = fit.ivim_fit(signal, bvals)
+#     npt.assert_allclose([f, D, Dp], [f_fit, D_fit, Dp_fit])
+
+
+
 # test_linear_data = [
 #     pytest.param(0, np.linspace(0, 1000, 11), id='0'),
 #     pytest.param(0.01, np.linspace(0, 1000, 11), id='0.1'),
@@ -57,6 +108,32 @@
     #if not np.allclose(f, 0):
         #npt.assert_allclose(Dp, Dp_fit, rtol=1e-2, atol=1e-3)
 
+
+# convert the algorithm list and signal list to fixtures that read from the files into params (scope="session")
+# from that helpers can again parse the files?
+
+def signal_helper(signal):
+    signal = np.asarray(signal)
+    signal = np.abs(signal)
+    signal /= signal[0]
+    ratio = 1 / signal[0]
+    return signal, ratio
+
+def tolerances_helper(tolerances, ratio, noise):
+    if "dynamic_rtol" in tolerances:
+        dyn_rtol = tolerances["dynamic_rtol"]
+        scale = dyn_rtol["offset"] + dyn_rtol["ratio"]*ratio + dyn_rtol["noise"]*noise + dyn_rtol["noiseCrossRatio"]*ratio*noise
+        tolerances["rtol"] = {"f": scale*dyn_rtol["f"], "D": scale*dyn_rtol["D"], "Dp": scale*dyn_rtol["Dp"]}
+    else:
+        tolerances["rtol"] = tolerances.get("rtol", {"f": 5, "D": 5, "Dp": 25})
+    if "dynamic_atol" in tolerances:
+        dyn_atol = tolerances["dynamic_atol"]
+        scale = dyn_atol["offset"] + dyn_atol["ratio"]*ratio + dyn_atol["noise"]*noise + dyn_atol["noiseCrossRatio"]*ratio*noise
+        tolerances["atol"] = {"f": scale*dyn_atol["f"], "D": scale*dyn_atol["D"], "Dp": scale*dyn_atol["Dp"]}
+    else:
+        tolerances["atol"] = tolerances.get("atol", {"f": 1e-2, "D": 1e-2, "Dp": 1e-1})
+    return tolerances
+
 def data_ivim_fit_saved():
     # Find the algorithms from algorithms.json
     file = pathlib.Path(__file__)
@@ -88,40 +165,8 @@ def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances,
         mark = pytest.mark.xfail(reason="xfail", strict=xfail["strict"])
         request.node.add_marker(mark)
     fit = OsipiBase(algorithm=algorithm, **kwargs)
-    signal = np.asarray(data['data'])
-    signal = np.abs(signal)
-    signal /= signal[0]
-    ratio = 1 / signal[0]
-    # has_negatives = np.any(signal<0)
-    if "dynamic_rtol" in tolerances:
-        dyn_rtol = tolerances["dynamic_rtol"]
-        scale = dyn_rtol["offset"] + dyn_rtol["ratio"]*ratio + dyn_rtol["noise"]*data["noise"] + dyn_rtol["noiseCrossRatio"]*ratio*data["noise"]
-        tolerances["rtol"] = {"f": scale*dyn_rtol["f"], "D": scale*dyn_rtol["D"], "Dp": scale*dyn_rtol["Dp"]}
-    else:
-        tolerances["rtol"] = tolerances.get("rtol", {"f": 5, "D": 5, "Dp": 25})
-    if "dynamic_atol" in tolerances:
-        dyn_atol = tolerances["dynamic_atol"]
-        scale = dyn_atol["offset"] + dyn_atol["ratio"]*ratio + dyn_atol["noise"]*data["noise"] + dyn_atol["noiseCrossRatio"]*ratio*data["noise"]
-        tolerances["atol"] = {"f": scale*dyn_atol["f"], "D": scale*dyn_atol["D"], "Dp": scale*dyn_atol["Dp"]}
-    else:
-        tolerances["atol"] = tolerances.get("atol", {"f": 1e-2, "D": 1e-2, "Dp": 1e-1})
-    # if tolerances:
-    #     # signal = np.abs(signal)
-    #     # ratio = 1 / signal[0]
-    #     # signal /= signal[0]
-    #     # tolerance = 1e-2 + 25 * data['noise'] * ratio  # totally empirical
-    #     # tolerance = 1
-    #     tolerances = {"rtol": {"f": 5, "D": 5, "Dp": 25},
-    #         "atol": {"f": 1e-2, "D": 1e-2, "Dp": 1e-1}}
-    #if has_negatives:  # this fitting doesn't do well with negatives
-    #    tolerance += 1
-    #if data['f'] == 1.0:
-        #linear_fit = fit.ivim_fit(np.log(signal), bvals, linear_fit_option=True)
-        #npt.assert_allclose([data['f'], data['Dp']], linear_fit, atol=tolerance)
-    #else:
-        #[f_fit, Dp_fit, D_fit] = fit.ivim_fit(signal, bvals)
-        #npt.assert_allclose([data['f'], data['D']], [f_fit, D_fit], atol=tolerance)
-        #npt.assert_allclose(data['Dp'], Dp_fit, atol=1e-1)  # go easy on the perfusion as it's a linear fake
+    signal, ratio = signal_helper(data["data"])
+    tolerances = tolerances_helper(tolerances, ratio, data["noise"])
     [f_fit, Dp_fit, D_fit] = fit.ivim_fit(signal, bvals)
     npt.assert_allclose(data['f'], f_fit, rtol=tolerances["rtol"]["f"], atol=tolerances["atol"]["f"])
     npt.assert_allclose(data['D'], D_fit, rtol=tolerances["rtol"]["D"], atol=tolerances["atol"]["D"])

tests/IVIMmodels/unit_tests/__init__.py

@@ -0,0 +1,37 @@
+import numpy as np
+import numpy.testing as npt
+import pytest
+import json
+import pathlib
+import os
+import csv
+
+from src.wrappers.OsipiBase import OsipiBase
+from utilities.data_simulation.GenerateData import GenerateData
+
+#run using pytest <path_to_this_file> --saveFileName test_output.txt --SNR 50 100 200
+#e.g. pytest tests/IVIMmodels/unit_tests/test_ivim_synthetic.py --saveFileName test_output.txt --SNR 50 100 200
+def test_generated(ivim_algorithm, ivim_data, SNR, rtol, atol, save_file):
+    # assert save_file == "test"
+    S0 = 1
+    gd = GenerateData()
+    name, bvals, data = ivim_data
+    D = data["D"]
+    f = data["f"]
+    Dp = data["Dp"]
+    # if "data" not in data:
+    signal = gd.ivim_signal(D, Dp, f, S0, bvals, SNR)
+    # else:
+    #     signal = data["data"]
+    fit = OsipiBase(algorithm=ivim_algorithm)
+    [f_fit, Dp_fit, D_fit] = fit.ivim_fit(signal, bvals)
+    if save_file:
+        save_results(save_file, ivim_algorithm, name, SNR, [f, D, Dp], [f_fit, Dp_fit, D_fit])
+    npt.assert_allclose([f, D, Dp], [f_fit, D_fit, Dp_fit], rtol, atol)
+
+
+def save_results(filename, algorithm, name, SNR, truth, fit):
+    with open(filename, "a") as csv_file:
+        writer = csv.writer(csv_file, delimiter=',')
+        data = [algorithm, name, SNR, truth, fit]
+        writer.writerow(data)